From 6648c9ee42c15195f639a79ecf5c1c064709d3c7 Mon Sep 17 00:00:00 2001 From: zhangli Date: Tue, 14 Mar 2017 16:37:35 +0800 Subject: [PATCH 001/823] refactor --- README.md | 127 ++ .../bigdl/dllib/feature/dataset/__init__.py | 15 + .../src/bigdl/dllib/feature/dataset/base.py | 197 ++ .../src/bigdl/dllib/feature/dataset/mnist.py | 113 ++ .../src/bigdl/dllib/feature/dataset/news20.py | 73 + .../dllib/feature/dataset/transformer.py | 23 + .../dllib/src/bigdl/dllib/models/__init__.py | 15 + .../src/bigdl/dllib/models/lenet/__init__.py | 15 + .../src/bigdl/dllib/models/lenet/lenet5.py | 97 + .../models/textclassifier/textclassifier.py | 177 ++ python/dllib/src/bigdl/dllib/nn/__init__.py | 15 + python/dllib/src/bigdl/dllib/nn/layer.py | 1673 +++++++++++++++++ .../dllib/src/bigdl/dllib/optim/__init__.py | 15 + .../dllib/src/bigdl/dllib/optim/optimizer.py | 169 ++ python/dllib/src/bigdl/dllib/utils/common.py | 261 +++ python/dllib/src/bigdl/utils/__init__.py | 15 + 16 files changed, 3000 insertions(+) create mode 100644 README.md create mode 100644 python/dllib/src/bigdl/dllib/feature/dataset/__init__.py create mode 100644 python/dllib/src/bigdl/dllib/feature/dataset/base.py create mode 100644 python/dllib/src/bigdl/dllib/feature/dataset/mnist.py create mode 100644 python/dllib/src/bigdl/dllib/feature/dataset/news20.py create mode 100644 python/dllib/src/bigdl/dllib/feature/dataset/transformer.py create mode 100644 python/dllib/src/bigdl/dllib/models/__init__.py create mode 100644 python/dllib/src/bigdl/dllib/models/lenet/__init__.py create mode 100644 python/dllib/src/bigdl/dllib/models/lenet/lenet5.py create mode 100644 python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py create mode 100644 python/dllib/src/bigdl/dllib/nn/__init__.py create mode 100644 python/dllib/src/bigdl/dllib/nn/layer.py create mode 100644 python/dllib/src/bigdl/dllib/optim/__init__.py create mode 100644 python/dllib/src/bigdl/dllib/optim/optimizer.py create mode 100644 python/dllib/src/bigdl/dllib/utils/common.py create mode 100644 python/dllib/src/bigdl/utils/__init__.py diff --git a/README.md b/README.md new file mode 100644 index 00000000000..a94e306c8f4 --- /dev/null +++ b/README.md @@ -0,0 +1,127 @@ + +BigDL comes with scala API for now. However Python is a powerful programming language for data analysis and with large amount of useful libraries, we are developing a lightweight python binding on top of PySpark which can enable us use Python naively with BigDL. + +The Python API is almost identical to the scala version, and it would map ndarray to tensor for the training samples, so basically user only need to care about how to manipulate ndarray. + +This Python binding tested with Python 2.7 and Spark 1.6.0. + +Here are the steps for training a simple LeNet model: + +1). Create a RDD[Sample]: +``` +RDD[..] --transform-->RDD[ndarray, ndarray].map(Sample.from_ndarray(features, label)) --> RDD[Sample] +``` + +2). Define a model: +``` + def build_model(class_num): + model = Sequential() + model.add(Reshape([1, 28, 28])) + model.add(SpatialConvolution(1, 6, 5, 5)) + model.add(Tanh()) + model.add(SpatialMaxPooling(2, 2, 2, 2)) + model.add(Tanh()) + model.add(SpatialConvolution(6, 12, 5, 5)) + model.add(SpatialMaxPooling(2, 2, 2, 2)) + model.add(Reshape([12 * 4 * 4])) + model.add(Linear(12 * 4 * 4, 100)) + model.add(Tanh()) + model.add(Linear(100, class_num)) + model.add(LogSoftMax()) + return model + ``` + +3). Create Optimizer and train: +``` + optimizer = Optimizer( + model=build_model(10), + training_rdd=train_data, + criterion=ClassNLLCriterion(), + optim_method="SGD", + state=state, + end_trigger=MaxEpoch(100), + batch_size=int(options.batchSize)) + optimizer.setvalidation( + batch_size=32, + val_rdd=test_data, + trigger=EveryEpoch(), + val_method=["top1"] + ) + optimizer.setcheckpoint(EveryEpoch(), "/tmp/lenet5/") + trained_model = optimizer.optimize() +``` + +4) LeNet example can be found from: models/lenet5.py + +## Run python test +* Package Scala code by: ```$BigDL_HOME/make-dist.sh``` +* Set SPARK_HOME and then run: ```$BigDL_HOME/dl/src/main/python/dev/run-all.sh``` + +## Installing on Ubuntu +1. Build BigDL +[Build Page](https://github.com/intel-analytics/BigDL/wiki/Build-Page) + * With Spark1.6: ``` $BIGDL_HOME//make-dist.sh ``` + * With Spark2.0: ``` $BIGDL_HOME//make-dist.sh -P spark_2.0 ``` + +2. Install python dependensies: + * Installing Numpy: + ```sudo apt-get install python-numpy``` + + * Installing Python setuptools: + ```sudo apt-get install -y python-setuptools python-pip``` + + * Install Jupyter: + ```sudo pip install jupyter``` + +## Run a Lenet example on standalone cluster + + ``` + BigDL_HOME=... + SPARK_HOME=... + MASTER=... + PYTHON_API_ZIP_PATH=${BigDL_HOME}/dist/lib/bigdl-0.1.0-SNAPSHOT-python-api.zip + BigDL_JAR_PATH=${BigDL_HOME}/dist/lib/bigdl-0.1.0-SNAPSHOT-jar-with-dependencies.jar + PYTHONPATH=${PYTHON_API_ZIP_PATH}:$PYTHONPATH + ${SPARK_HOME}/bin/spark-submit \ + --master ${MASTER} \ + --driver-cores 5 \ + --driver-memory 10g \ + --total-executor-cores 80 \ + --executor-cores 10 \ + --executor-memory 20g \ + --conf spark.akka.frameSize=64 \ + --py-files ${PYTHON_API_ZIP_PATH},${BigDL_HOME}/dl/src/main/python/models/lenet/lenet5.py \ + --properties-file ${BigDL_HOME}/dist/conf/spark-bigdl.conf \ + --jars ${BigDL_JAR_PATH} \ + --conf spark.driver.extraClassPath=${BigDL_JAR_PATH} \ + --conf spark.executor.extraClassPath=bigdl-0.1.0-SNAPSHOT-jar-with-dependencies.jar \ + ${BigDL_HOME}/dl/src/main/python/models/lenet/lenet5.py + ``` + + +## Launch Jupyter on standalone cluster + + ``` + BigDL_HOME=... + SPARK_HOME=... + MASTER=... + PYTHON_API_ZIP_PATH=${BigDL_HOME}/dist/lib/bigdl-0.1.0-SNAPSHOT-python-api.zip + BigDL_JAR_PATH=${BigDL_HOME}/dist/lib/bigdl-0.1.0-SNAPSHOT-jar-with-dependencies.jar + + export PYTHONPATH=${PYTHON_API_ZIP_PATH}:$PYTHONPATH + export IPYTHON_OPTS="notebook --notebook-dir=./ --ip=* --no-browser" + + ${SPARK_HOME}/bin/pyspark \ + --master ${MASTER} \ + --properties-file ${BigDL_HOME}/dist/conf/spark-bigdl.conf + --driver-cores 5 \ + --driver-memory 10g \ + --total-executor-cores 8 \ + --executor-cores 1 \ + --executor-memory 20g \ + --conf spark.akka.frameSize=64 \ + --py-files ${PYTHON_API_ZIP_PATH} \ + --jars ${BigDL_JAR_PATH} \ + --conf spark.driver.extraClassPath=${BigDL_JAR_PATH} \ + --conf spark.executor.extraClassPath=bigdl-0.1.0-SNAPSHOT-jar-with-dependencies.jar + ``` diff --git a/python/dllib/src/bigdl/dllib/feature/dataset/__init__.py b/python/dllib/src/bigdl/dllib/feature/dataset/__init__.py new file mode 100644 index 00000000000..2151a805423 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/feature/dataset/__init__.py @@ -0,0 +1,15 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/python/dllib/src/bigdl/dllib/feature/dataset/base.py b/python/dllib/src/bigdl/dllib/feature/dataset/base.py new file mode 100644 index 00000000000..f211ee5261c --- /dev/null +++ b/python/dllib/src/bigdl/dllib/feature/dataset/base.py @@ -0,0 +1,197 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os +import sys +import shutil +import tempfile +import time +from distutils.dir_util import mkpath +from six.moves.urllib.request import urlopen +import numpy as np + + +# Adopt from keras +class Progbar(object): + def __init__(self, target, width=30, verbose=1, interval=0.01): + ''' + @param target: total number of steps expected + @param interval: minimum visual progress update interval (in seconds) + ''' + self.width = width + self.target = target + self.sum_values = {} + self.unique_values = [] + self.start = time.time() + self.last_update = 0 + self.interval = interval + self.total_width = 0 + self.seen_so_far = 0 + self.verbose = verbose + + def update(self, current, values=[], force=False): + ''' + @param current: index of current step + @param values: list of tuples (name, value_for_last_step). + The progress bar will display averages for these values. + @param force: force visual progress update + ''' + for k, v in values: + if k not in self.sum_values: + self.sum_values[k] = [v * (current - self.seen_so_far), current - self.seen_so_far] + self.unique_values.append(k) + else: + self.sum_values[k][0] += v * (current - self.seen_so_far) + self.sum_values[k][1] += (current - self.seen_so_far) + self.seen_so_far = current + + now = time.time() + if self.verbose == 1: + if not force and (now - self.last_update) < self.interval: + return + + prev_total_width = self.total_width + sys.stdout.write("\b" * prev_total_width) + sys.stdout.write("\r") + + numdigits = int(np.floor(np.log10(self.target))) + 1 + barstr = '%%%dd/%%%dd [' % (numdigits, numdigits) + bar = barstr % (current, self.target) + prog = float(current) / self.target + prog_width = int(self.width * prog) + if prog_width > 0: + bar += ('=' * (prog_width-1)) + if current < self.target: + bar += '>' + else: + bar += '=' + bar += ('.' * (self.width - prog_width)) + bar += ']' + sys.stdout.write(bar) + self.total_width = len(bar) + + if current: + time_per_unit = (now - self.start) / current + else: + time_per_unit = 0 + eta = time_per_unit * (self.target - current) + info = '' + if current < self.target: + info += ' - ETA: %ds' % eta + else: + info += ' - %ds' % (now - self.start) + for k in self.unique_values: + info += ' - %s:' % k + if type(self.sum_values[k]) is list: + avg = self.sum_values[k][0] / max(1, self.sum_values[k][1]) + if abs(avg) > 1e-3: + info += ' %.4f' % avg + else: + info += ' %.4e' % avg + else: + info += ' %s' % self.sum_values[k] + + self.total_width += len(info) + if prev_total_width > self.total_width: + info += ((prev_total_width - self.total_width) * " ") + + sys.stdout.write(info) + sys.stdout.flush() + + if current >= self.target: + sys.stdout.write("\n") + + if self.verbose == 2: + if current >= self.target: + info = '%ds' % (now - self.start) + for k in self.unique_values: + info += ' - %s:' % k + avg = self.sum_values[k][0] / max(1, self.sum_values[k][1]) + if avg > 1e-3: + info += ' %.4f' % avg + else: + info += ' %.4e' % avg + sys.stdout.write(info + "\n") + + self.last_update = now + + def add(self, n, values=[]): + self.update(self.seen_so_far + n, values) + + +def display_table(rows, positions): + + def display_row(objects, positions): + line = '' + for i in range(len(objects)): + line += str(objects[i]) + line = line[:positions[i]] + line += ' ' * (positions[i] - len(line)) + print(line) + + for objects in rows: + display_row(objects, positions) + +# Adopt from keras +# Under Python 2, 'urlretrieve' relies on FancyURLopener from legacy +# urllib module, known to have issues with proxy management +if sys.version_info[0] == 2: + def urlretrieve(url, filename, reporthook=None, data=None): + def chunk_read(response, chunk_size=8192, reporthook=None): + total_size = response.info().get('Content-Length').strip() + total_size = int(total_size) + count = 0 + while 1: + chunk = response.read(chunk_size) + count += 1 + if not chunk: + reporthook(count, total_size, total_size) + break + if reporthook: + reporthook(count, chunk_size, total_size) + yield chunk + + response = urlopen(url, data) + with open(filename, 'wb') as fd: + for chunk in chunk_read(response, reporthook=reporthook): + fd.write(chunk) +else: + from six.moves.urllib.request import urlretrieve + + +def maybe_download(filename, work_directory, source_url): + if not os.path.exists(work_directory): + mkpath(work_directory) + filepath = os.path.join(work_directory, filename) + + if not os.path.exists(filepath): + print('Downloading data from', source_url) + global progbar + progbar = None + + def dl_progress(count, block_size, total_size): + global progbar + if progbar is None: + progbar = Progbar(total_size) + else: + progbar.update(count * block_size) + with tempfile.NamedTemporaryFile() as tmpfile: + temp_file_name = tmpfile.name + urlretrieve(source_url, temp_file_name, dl_progress) + shutil.copy2(temp_file_name, filepath) + size = os.path.getsize(filepath) + print('Successfully downloaded', filename, size, 'bytes.') + return filepath diff --git a/python/dllib/src/bigdl/dllib/feature/dataset/mnist.py b/python/dllib/src/bigdl/dllib/feature/dataset/mnist.py new file mode 100644 index 00000000000..4a53735fce2 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/feature/dataset/mnist.py @@ -0,0 +1,113 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Part of the code originally from Tensorflow + + +import gzip + +import numpy + +import base + +SOURCE_URL = 'http://yann.lecun.com/exdb/mnist/' + +TRAIN_MEAN = 0.13066047740239506 +TRAIN_STD = 0.3081078 +TEST_MEAN = 0.13251460696903547 +TEST_STD = 0.31048024 + + +def _read32(bytestream): + dt = numpy.dtype(numpy.uint32).newbyteorder('>') + return numpy.frombuffer(bytestream.read(4), dtype=dt)[0] + + +def extract_images(f): + """Extract the images into a 4D uint8 numpy array [index, y, x, depth]. + + Args: + f: A file object that can be passed into a gzip reader. + + Returns: + data: A 4D unit8 numpy array [index, y, x, depth]. + + Raises: + ValueError: If the bytestream does not start with 2051. + + """ + print('Extracting', f.name) + with gzip.GzipFile(fileobj=f) as bytestream: + magic = _read32(bytestream) + if magic != 2051: + raise ValueError( + 'Invalid magic number %d in MNIST image file: %s' % + (magic, f.name)) + num_images = _read32(bytestream) + rows = _read32(bytestream) + cols = _read32(bytestream) + buf = bytestream.read(rows * cols * num_images) + data = numpy.frombuffer(buf, dtype=numpy.uint8) + data = data.reshape(num_images, rows, cols, 1) + return data + + +def extract_labels(f): + print('Extracting', f.name) + with gzip.GzipFile(fileobj=f) as bytestream: + magic = _read32(bytestream) + if magic != 2049: + raise ValueError( + 'Invalid magic number %d in MNIST label file: %s' % + (magic, f.name)) + num_items = _read32(bytestream) + buf = bytestream.read(num_items) + labels = numpy.frombuffer(buf, dtype=numpy.uint8) + return labels + + +def read_data_sets(train_dir, data_type="train"): + TRAIN_IMAGES = 'train-images-idx3-ubyte.gz' + TRAIN_LABELS = 'train-labels-idx1-ubyte.gz' + TEST_IMAGES = 't10k-images-idx3-ubyte.gz' + TEST_LABELS = 't10k-labels-idx1-ubyte.gz' + + if data_type == "train": + local_file = base.maybe_download(TRAIN_IMAGES, train_dir, + SOURCE_URL + TRAIN_IMAGES) + with open(local_file, 'rb') as f: + train_images = extract_images(f) + + local_file = base.maybe_download(TRAIN_LABELS, train_dir, + SOURCE_URL + TRAIN_LABELS) + with open(local_file, 'rb') as f: + train_labels = extract_labels(f) + return train_images, train_labels + + else: + local_file = base.maybe_download(TEST_IMAGES, train_dir, + SOURCE_URL + TEST_IMAGES) + with open(local_file, 'rb') as f: + test_images = extract_images(f) + + local_file = base.maybe_download(TEST_LABELS, train_dir, + SOURCE_URL + TEST_LABELS) + with open(local_file, 'rb') as f: + test_labels = extract_labels(f) + return test_images, test_labels + + +if __name__ == "__main__": + read_data_sets("/tmp/mnist/") diff --git a/python/dllib/src/bigdl/dllib/feature/dataset/news20.py b/python/dllib/src/bigdl/dllib/feature/dataset/news20.py new file mode 100644 index 00000000000..fdedf5a6c41 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/feature/dataset/news20.py @@ -0,0 +1,73 @@ +import tarfile +import base +import os +import sys + +NEWS20_URL = 'http://qwone.com/~jason/20Newsgroups/20news-19997.tar.gz' # noqa +GLOVE_URL = 'http://nlp.stanford.edu/data/glove.6B.zip' # noqa + +CLASS_NUM = 20 + + +def download_news20(dest_dir): + file_name = "20news-19997.tar.gz" + file_abs_path = base.maybe_download(file_name, dest_dir, NEWS20_URL) + tar = tarfile.open(file_abs_path, "r:gz") + extracted_to = os.path.join(dest_dir, "20_newsgroups") + if not os.path.exists(extracted_to): + print("Extracting %s to %s" % (file_abs_path, extracted_to)) + tar.extractall(dest_dir) + tar.close() + return extracted_to + + +def download_glove_w2v(dest_dir): + file_name = "glove.6B.zip" + file_abs_path = base.maybe_download(file_name, dest_dir, GLOVE_URL) + import zipfile + zip_ref = zipfile.ZipFile(file_abs_path, 'r') + extracted_to = os.path.join(dest_dir, "glove.6B") + if not os.path.exists(extracted_to): + print("Extracting %s to %s" % (file_abs_path, extracted_to)) + zip_ref.extractall(extracted_to) + zip_ref.close() + return extracted_to + + +# return [(text_content, label)] +def get_news20(source_dir="/tmp/news20/"): + news_dir = download_news20(source_dir) + texts = [] # list of text samples + label_id = 0 + for name in sorted(os.listdir(news_dir)): + path = os.path.join(news_dir, name) + label_id += 1 + if os.path.isdir(path): + for fname in sorted(os.listdir(path)): + if fname.isdigit(): + fpath = os.path.join(path, fname) + if sys.version_info < (3,): + f = open(fpath) + else: + f = open(fpath, encoding='latin-1') + content = f.read() + texts.append((content, label_id)) + f.close() + + print('Found %s texts.' % len(texts)) + return texts + + +def get_glove_w2v(source_dir="/tmp/news20/", dim=100): + w2v_dir = download_glove_w2v(source_dir) + with open(os.path.join(w2v_dir, "glove.6B.%sd.txt" % dim)) as w2v_f: + pre_w2v = {} + for line in w2v_f.readlines(): + items = line.split(" ") + pre_w2v[items[0]] = [float(i) for i in items[1:]] + return pre_w2v + + +if __name__ == "__main__": + get_news20("/tmp/news20/") + get_glove_w2v("/tmp/news20/") diff --git a/python/dllib/src/bigdl/dllib/feature/dataset/transformer.py b/python/dllib/src/bigdl/dllib/feature/dataset/transformer.py new file mode 100644 index 00000000000..9b5d5930d3a --- /dev/null +++ b/python/dllib/src/bigdl/dllib/feature/dataset/transformer.py @@ -0,0 +1,23 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +from util.common import Sample + + +def normalizer(mean, std): + return lambda sample: Sample.from_ndarray((sample.features - mean) / std, + sample.label, sample.bigdl_type) diff --git a/python/dllib/src/bigdl/dllib/models/__init__.py b/python/dllib/src/bigdl/dllib/models/__init__.py new file mode 100644 index 00000000000..2151a805423 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/models/__init__.py @@ -0,0 +1,15 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/python/dllib/src/bigdl/dllib/models/lenet/__init__.py b/python/dllib/src/bigdl/dllib/models/lenet/__init__.py new file mode 100644 index 00000000000..2151a805423 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/models/lenet/__init__.py @@ -0,0 +1,15 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py b/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py new file mode 100644 index 00000000000..f6b1a752b40 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py @@ -0,0 +1,97 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Still in experimental stage! + +from optparse import OptionParser + +from dataset import mnist +from dataset.transformer import * +from nn.layer import * +from nn.criterion import * +from optim.optimizer import * +from util.common import * + + +def build_model(class_num): + model = Sequential() + model.add(Reshape([1, 28, 28])) + model.add(SpatialConvolution(1, 6, 5, 5)) + model.add(Tanh()) + model.add(SpatialMaxPooling(2, 2, 2, 2)) + model.add(Tanh()) + model.add(SpatialConvolution(6, 12, 5, 5)) + model.add(SpatialMaxPooling(2, 2, 2, 2)) + model.add(Reshape([12 * 4 * 4])) + model.add(Linear(12 * 4 * 4, 100)) + model.add(Tanh()) + model.add(Linear(100, class_num)) + model.add(LogSoftMax()) + return model + + +def get_minst(sc, data_type="train", location="/tmp/mnist"): + (images, labels) = mnist.read_data_sets(location, data_type) + images = sc.parallelize(images) + labels = sc.parallelize(labels) + # Target start from 1 in BigDL + record = images.zip(labels).map(lambda (features, label): + Sample.from_ndarray(features, label + 1)) + return record + + +if __name__ == "__main__": + + parser = OptionParser() + parser.add_option("-a", "--action", dest="action", default="train") + parser.add_option("-b", "--batchSize", dest="batchSize", default="128") + + (options, args) = parser.parse_args(sys.argv) + + sc = SparkContext(appName="lenet5", conf=create_spark_conf()) + init_engine() + + if options.action == "train": + train_data = get_minst(sc, "train").map( + normalizer(mnist.TRAIN_MEAN, mnist.TRAIN_STD)) + test_data = get_minst(sc, "test").map( + normalizer(mnist.TEST_MEAN, mnist.TEST_STD)) + state = {"learningRate": 0.01, + "learningRateDecay": 0.0002} + optimizer = Optimizer( + model=build_model(10), + training_rdd=train_data, + criterion=ClassNLLCriterion(), + optim_method="SGD", + state=state, + end_trigger=MaxEpoch(20), + batch_size=int(options.batchSize)) + optimizer.setvalidation( + batch_size=32, + val_rdd=test_data, + trigger=EveryEpoch(), + val_method=["Top1Accuracy"] + ) + optimizer.setcheckpoint(EveryEpoch(), "/tmp/lenet5/") + trained_model = optimizer.optimize() + parameters = trained_model.parameters() + elif options.action == "test": + test_data = get_minst("test").map( + normalizer(mnist.TEST_MEAN, mnist.TEST_STD)) + # TODO: Pass model path through external parameter + model = Model.from_path("/tmp/lenet5/lenet-model.470") + results = model.test(test_data, 32, ["Top1Accuracy"]) + for result in results: + print result diff --git a/python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py b/python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py new file mode 100644 index 00000000000..2929290aa0a --- /dev/null +++ b/python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py @@ -0,0 +1,177 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Still in experimental stage! + +import itertools +import re +from optparse import OptionParser + +from dataset import news20 +from nn.layer import * +from nn.criterion import * +from optim.optimizer import * +from util.common import * +from util.common import Sample + + +def text_to_words(review_text): + letters_only = re.sub("[^a-zA-Z]", " ", review_text) + words = letters_only.lower().split() + return words + + +def analyze_texts(data_rdd): + return data_rdd.flatMap(lambda (text, label): text_to_words(text)) \ + .map(lambda word: (word, 1)).reduceByKey(lambda a, b: a + b) \ + .sortBy(lambda (w, c): - c).zipWithIndex() \ + .map(lambda ((w, c), i): (w, (i + 1, c))).collect() + + +# pad([1, 2, 3, 4, 5], 0, 6) +def pad(l, fill_value, width): + if len(l) >= width: + return l[0: width] + else: + l.extend([fill_value] * (width - len(l))) + return l + + +def to_vec(token, b_w2v, embedding_dim): + if token in b_w2v: + return b_w2v[token] + else: + return pad([], 0, embedding_dim) + + +def to_sample(vectors, label, embedding_dim): + # flatten nested list + flatten_features = list(itertools.chain(*vectors)) + features = np.array(flatten_features, dtype='float').reshape( + [sequence_len, embedding_dim]) + + if model_type.lower() == "cnn": + features = features.transpose(1, 0) + return Sample.from_ndarray(features, np.array(label)) + + +def build_model(class_num): + model = Sequential() + + if model_type.lower() == "cnn": + model.add(Reshape([embedding_dim, 1, sequence_len])) + model.add(SpatialConvolution(embedding_dim, 128, 5, 1)) + model.add(ReLU()) + model.add(SpatialMaxPooling(5, 1, 5, 1)) + model.add(SpatialConvolution(128, 128, 5, 1)) + model.add(ReLU()) + model.add(SpatialMaxPooling(5, 1, 5, 1)) + model.add(Reshape([128])) + elif model_type.lower() == "lstm": + model.add(Recurrent() + .add(LSTM(embedding_dim, 128))) + model.add(Select(2, -1)) + elif model_type.lower() == "gru": + model.add(Recurrent() + .add(GRU(embedding_dim, 128))) + model.add(Select(2, -1)) + else: + raise ValueError('model can only be cnn, lstm, or gru') + + model.add(Linear(128, 100)) + model.add(Linear(100, class_num)) + model.add(LogSoftMax()) + return model + + +def train(sc, + batch_size, + sequence_len, max_words, embedding_dim, training_split): + print('Processing text dataset') + texts = news20.get_news20() + data_rdd = sc.parallelize(texts, 2) + + word_to_ic = analyze_texts(data_rdd) + + # Only take the top wc between [10, sequence_len] + word_to_ic = dict(word_to_ic[10: max_words]) + bword_to_ic = sc.broadcast(word_to_ic) + + w2v = news20.get_glove_w2v(dim=embedding_dim) + filtered_w2v = {w: v for w, v in w2v.items() if w in word_to_ic} + bfiltered_w2v = sc.broadcast(filtered_w2v) + + tokens_rdd = data_rdd.map(lambda (text, label): + ([w for w in text_to_words(text) if + w in bword_to_ic.value], label)) + padded_tokens_rdd = tokens_rdd.map( + lambda (tokens, label): (pad(tokens, "##", sequence_len), label)) + vector_rdd = padded_tokens_rdd.map(lambda (tokens, label): + ([to_vec(w, bfiltered_w2v.value, + embedding_dim) for w in + tokens], label)) + sample_rdd = vector_rdd.map( + lambda (vectors, label): to_sample(vectors, label, embedding_dim)) + + train_rdd, val_rdd = sample_rdd.randomSplit( + [training_split, 1-training_split]) + + state = {"batchSize": batch_size, + "learningRate": 0.01, + "learningRateDecay": 0.0002} + + optimizer = Optimizer( + model=build_model(news20.CLASS_NUM), + training_rdd=train_rdd, + criterion=ClassNLLCriterion(), + end_trigger=MaxEpoch(max_epoch), + batch_size=batch_size, + optim_method="Adagrad", + state=state) + + optimizer.setvalidation( + batch_size=batch_size, + val_rdd=val_rdd, + trigger=EveryEpoch(), + val_method=["Top1Accuracy"] + ) + train_model = optimizer.optimize() + +if __name__ == "__main__": + parser = OptionParser() + parser.add_option("-a", "--action", dest="action", default="train") + parser.add_option("-b", "--batchSize", dest="batchSize", default="128") + parser.add_option("-e", "--embedding_dim", dest="embedding_dim", default="50") # noqa + parser.add_option("-m", "--max_epoch", dest="max_epoch", default="15") + parser.add_option("--model", dest="model_type", default="cnn") + + (options, args) = parser.parse_args(sys.argv) + if options.action == "train": + batch_size = int(options.batchSize) + embedding_dim = int(options.embedding_dim) + max_epoch = int(options.max_epoch) + model_type = options.model_type + sequence_len = 50 + max_words = 1000 + training_split = 0.8 + sc = SparkContext(appName="text_classifier", + conf=create_spark_conf()) + init_engine() + train(sc, + batch_size, + sequence_len, max_words, embedding_dim, training_split) + elif options.action == "test": + pass diff --git a/python/dllib/src/bigdl/dllib/nn/__init__.py b/python/dllib/src/bigdl/dllib/nn/__init__.py new file mode 100644 index 00000000000..2151a805423 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/nn/__init__.py @@ -0,0 +1,15 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py new file mode 100644 index 00000000000..48ace84dd7a --- /dev/null +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -0,0 +1,1673 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +import sys +from util.common import callBigDlFunc +from util.common import JavaValue +from util.common import callJavaFunc +from pyspark import SparkContext + +import numpy as np + +if sys.version >= '3': + long = int + unicode = str + +INTMAX = 2147483647 +INTMIN = -2147483648 +DOUBLEMAX = 1.7976931348623157E308 + + +class Model(JavaValue): + + def __init__(self, jvalue, bigdl_type, *args): + self.value = jvalue if jvalue else callBigDlFunc( + bigdl_type, JavaValue.jvm_class_constructor(self), *args) + self.bigdl_type = bigdl_type + + @classmethod + def of(cls, jmodel, bigdl_type="float"): + model = Model(bigdl_type, jmodel) + model.value = jmodel + model.bigdl_type = bigdl_type + return model + + def set_name(self, name): + callJavaFunc(SparkContext.getOrCreate(), self.value.setName, name) + return self + + def name(self): + return callJavaFunc(SparkContext.getOrCreate(), self.value.getName) + + def set_seed(self, seed=123): + callBigDlFunc(self.bigdl_type, "setModelSeed", seed) + return self + + def get_dtype(self): + if "float" == self.bigdl_type: + return "float32" + else: + return "float64" + + def reset(self): + callJavaFunc(SparkContext.getOrCreate(), self.value.reset) + return self + + def parameters(self): + name_to_params = callBigDlFunc(self.bigdl_type, + "modelGetParameters", + self.value) + + def to_ndarray(params): + return { + param_name: np.array(values[0], + dtype=self.get_dtype()).reshape( + values[1]) for param_name, values in params.iteritems()} + + return {layer_name: to_ndarray(params) for layer_name, params in + name_to_params.iteritems()} + + def predict(self, data_rdd): + return callBigDlFunc(self.bigdl_type, + "modelPredictRDD", self.value, data_rdd) + + def test(self, val_rdd, batch_size, val_methods): + return callBigDlFunc(self.bigdl_type, + "modelTest", + self.value, + val_rdd, batch_size, val_methods) + + @staticmethod + def from_path(path, bigdl_type="float"): + jmodel = callBigDlFunc(bigdl_type, "modelFromPath", path) + return Model.of(jmodel, bigdl_type) + + +class Linear(Model): + + ''' + >>> linear = Linear(100, 10, "Xavier") + creating: createLinear + ''' + + def __init__(self, input_size, output_size, init_method="default", + bigdl_type="float"): + super(Linear, self).__init__(None, bigdl_type, input_size, output_size, + init_method) + + +class ReLU(Model): + + ''' + >>> relu = ReLU() + creating: createReLU + ''' + + def __init__(self, bigdl_type="float"): + super(ReLU, self).__init__(None, bigdl_type) + + +class Tanh(Model): + + ''' + >>> tanh = Tanh() + creating: createTanh + ''' + + def __init__(self, bigdl_type="float"): + super(Tanh, self).__init__(None, bigdl_type) + + +class Echo(Model): + + ''' + >>> echo = Echo() + creating: createEcho + ''' + + def __init__(self, bigdl_type="float"): + super(Echo, self).__init__(None, bigdl_type) + + +class LogSoftMax(Model): + + ''' + >>> logSoftMax = LogSoftMax() + creating: createLogSoftMax + ''' + + def __init__(self, bigdl_type="float"): + super(LogSoftMax, self).__init__(None, bigdl_type) + + +class Sequential(Model): + + ''' + >>> echo = Echo() + creating: createEcho + >>> s = Sequential() + creating: createSequential + >>> s = s.add(echo) + >>> s = s.add(s) + >>> s = s.add(echo) + + ''' + + def __init__(self, bigdl_type="float"): + super(Sequential, self).__init__(None, bigdl_type) + + def add(self, model): + self.value.add(model.value) + return self + + +class SpatialConvolution(Model): + + ''' + >>> spatialConvolution = SpatialConvolution(6, 12, 5, 5) + creating: createSpatialConvolution + ''' + + def __init__(self, + n_input_plane, + n_output_plane, + kernel_w, + kernel_h, + stride_w=1, + stride_h=1, + pad_w=0, + pad_h=0, + n_group=1, + propagate_back=True, + init_method="Default", + bigdl_type="float"): + super(SpatialConvolution, self).__init__(None, bigdl_type, + n_input_plane, + n_output_plane, + kernel_w, + kernel_h, + stride_w, + stride_h, + pad_w, + pad_h, + n_group, + propagate_back, + init_method) + + +class SpatialMaxPooling(Model): + + ''' + >>> spatialMaxPooling = SpatialMaxPooling(2, 2, 2, 2) + creating: createSpatialMaxPooling + ''' + # to_ceil: call floor() when False; call ceil() when True + + def __init__(self, kw, + kh, + dw, + dh, + pad_w=0, + pad_h=0, + to_ceil=False, + bigdl_type="float"): + super(SpatialMaxPooling, self).__init__(None, bigdl_type, kw, + kh, + dw, + dh, + pad_w, + pad_h, + to_ceil) + + +class Select(Model): + ''' + >>> select = Select(1, 1) + creating: createSelect + ''' + + def __init__(self, dim, index, bigdl_type="float"): + super(Select, self).__init__(None, bigdl_type, dim, index) + + +class Recurrent(Model): + ''' + >>> recurrent = Recurrent() + creating: createRecurrent + ''' + + def __init__(self, bigdl_type="float"): + super(Recurrent, self).__init__(None, bigdl_type) + + def add(self, model): + self.value.add(model.value) + return self + + +class LSTM(Model): + ''' + >>> lstm = LSTM(4, 3) + creating: createLSTM + ''' + + def __init__(self, input_size, hidden_size, bigdl_type="float"): + super(LSTM, self).__init__(None, bigdl_type, input_size, hidden_size) + + +class GRU(Model): + ''' + >>> gru = GRU(4, 3) + creating: createGRU + ''' + + def __init__(self, input_size, hidden_size, bigdl_type="float"): + super(GRU, self).__init__(None, bigdl_type, input_size, hidden_size) + + +class RNNCell(Model): + ''' + >>> reshape = RNNCell(4, 3, Tanh()) + creating: createTanh + creating: createRNNCell + ''' + + def __init__(self, + input_size, + hidden_size, + activation, + bigdl_type="float"): + super(RNNCell, self).__init__(None, bigdl_type, input_size, hidden_size, activation) + + +class TimeDistributed(Model): + ''' + >>> td = TimeDistributed(Linear(2, 3)) + creating: createLinear + creating: createTimeDistributed + ''' + + def __init__(self, model, bigdl_type="float"): + super(TimeDistributed, self).__init__(None, bigdl_type, model) + + +class Concat(Model): + + ''' + >>> concat = Concat(2) + creating: createConcat + ''' + + def __init__(self, + dimension, + bigdl_type="float"): + super(Concat, self).__init__(None, bigdl_type, + dimension) + + +class SpatialAveragePooling(Model): + + ''' + >>> spatialAveragePooling = SpatialAveragePooling(7,7) + creating: createSpatialAveragePooling + ''' + + def __init__(self, + kw, + kh, + dw=1, + dh=1, + pad_w=0, + pad_h=0, + ceil_mode=False, + count_include_pad=True, + divide=True, + bigdl_type="float"): + super(SpatialAveragePooling, self).__init__(None, bigdl_type, + kw, + kh, + dw, + dh, + pad_w, + pad_h, + ceil_mode, + count_include_pad, + divide) + + +class SpatialBatchNormalization(Model): + + ''' + >>> spatialBatchNormalization = SpatialBatchNormalization(1) + creating: createSpatialBatchNormalization + ''' + + def __init__(self, + n_output, + eps=1e-5, + momentum=0.1, + affine=True, + bigdl_type="float"): + super(SpatialBatchNormalization, self).__init__(None, bigdl_type, + n_output, + eps, + momentum, + affine) + + +class SpatialCrossMapLRN(Model): + + ''' + >>> spatialCrossMapLRN = SpatialCrossMapLRN() + creating: createSpatialCrossMapLRN + ''' + + def __init__(self, + size=5, + alpha=1.0, + beta=0.75, + k=1.0, + bigdl_type="float"): + super(SpatialCrossMapLRN, self).__init__(None, bigdl_type, + size, + alpha, + beta, + k) + + +class Dropout(Model): + + ''' + >>> dropout = Dropout(0.4) + creating: createDropout + ''' + + def __init__(self, + init_p=0.5, + inplace=False, + scale=True, + bigdl_type="float"): + super(Dropout, self).__init__(None, bigdl_type, + init_p, + inplace, + scale) + + +class View(Model): + + ''' + >>> view = View([1024,2]) + creating: createView + ''' + + def __init__(self, + sizes, + num_input_dims=0, + bigdl_type="float"): + super(View, self).__init__(None, bigdl_type, + sizes, + num_input_dims) + + +class Abs(Model): + + ''' + >>> abs = Abs() + creating: createAbs + ''' + + def __init__(self, + bigdl_type="float"): + super(Abs, self).__init__(None, bigdl_type) + + +class Add(Model): + + ''' + >>> add = Add(1) + creating: createAdd + ''' + + def __init__(self, + input_size, + bigdl_type="float"): + super(Add, self).__init__(None, bigdl_type, + input_size) + + +class AddConstant(Model): + + ''' + >>> addConstant = AddConstant(1e-5, True) + creating: createAddConstant + ''' + + def __init__(self, + constant_scalar, + inplace=False, + bigdl_type="float"): + super(AddConstant, self).__init__(None, bigdl_type, + constant_scalar, + inplace) + + +class BatchNormalization(Model): + + ''' + >>> batchNormalization = BatchNormalization(1, 1e-5, 1e-5, True) + creating: createBatchNormalization + ''' + + def __init__(self, + n_output, + eps=1e-5, + momentum=0.1, + affine=True, + bigdl_type="float"): + super(BatchNormalization, self).__init__(None, bigdl_type, + n_output, + eps, + momentum, + affine) + + +class Bilinear(Model): + + ''' + >>> bilinear = Bilinear(1, 1, 1, True) + creating: createBilinear + ''' + + def __init__(self, + input_size1, + input_size2, + output_size, + bias_res=True, + bigdl_type="float"): + super(Bilinear, self).__init__(None, bigdl_type, + input_size1, + input_size2, + output_size, + bias_res) + + +class Bottle(Model): + + ''' + >>> bottle = Bottle(Linear(100,10), 1, 1) + creating: createLinear + creating: createBottle + ''' + + def __init__(self, + module, + n_input_dim=2, + n_output_dim1=INTMAX, + bigdl_type="float"): + super(Bottle, self).__init__(None, bigdl_type, + module, + n_input_dim, + n_output_dim1) + + +class CAdd(Model): + + ''' + >>> cAdd = CAdd([1,2]) + creating: createCAdd + ''' + + def __init__(self, + size, + bigdl_type="float"): + super(CAdd, self).__init__(None, bigdl_type, + size) + + +class CAddTable(Model): + + ''' + >>> cAddTable = CAddTable(True) + creating: createCAddTable + ''' + + def __init__(self, + inplace=False, + bigdl_type="float"): + super(CAddTable, self).__init__(None, bigdl_type, + inplace) + + +class CDivTable(Model): + + ''' + >>> cDivTable = CDivTable() + creating: createCDivTable + ''' + + def __init__(self, + bigdl_type="float"): + super(CDivTable, self).__init__(None, bigdl_type) + + +class CMaxTable(Model): + + ''' + >>> cMaxTable = CMaxTable() + creating: createCMaxTable + ''' + + def __init__(self, + bigdl_type="float"): + super(CMaxTable, self).__init__(None, bigdl_type) + + +class CMinTable(Model): + + ''' + >>> cMinTable = CMinTable() + creating: createCMinTable + ''' + + def __init__(self, + bigdl_type="float"): + super(CMinTable, self).__init__(None, bigdl_type) + + +class CMul(Model): + + ''' + >>> cMul = CMul([1,2]) + creating: createCMul + ''' + + def __init__(self, + size, + bigdl_type="float"): + super(CMul, self).__init__(None, bigdl_type, + size) + + +class CMulTable(Model): + + ''' + >>> cMulTable = CMulTable() + creating: createCMulTable + ''' + + def __init__(self, + bigdl_type="float"): + super(CMulTable, self).__init__(None, bigdl_type) + + +class CSubTable(Model): + + ''' + >>> cSubTable = CSubTable() + creating: createCSubTable + ''' + + def __init__(self, + bigdl_type="float"): + super(CSubTable, self).__init__(None, bigdl_type) + + +class Clamp(Model): + + ''' + >>> clamp = Clamp(1, 3) + creating: createClamp + ''' + + def __init__(self, + min, + max, + bigdl_type="float"): + super(Clamp, self).__init__(None, bigdl_type, + min, + max) + + +class Contiguous(Model): + + ''' + >>> contiguous = Contiguous() + creating: createContiguous + ''' + + def __init__(self, + bigdl_type="float"): + super(Contiguous, self).__init__(None, bigdl_type) + + +class Copy(Model): + + ''' + >>> copy = Copy() + creating: createCopy + ''' + + def __init__(self, + bigdl_type="float"): + super(Copy, self).__init__(None, bigdl_type) + + +class Cosine(Model): + + ''' + >>> cosine = Cosine(2,3) + creating: createCosine + ''' + + def __init__(self, + input_size, + output_size, + bigdl_type="float"): + super(Cosine, self).__init__(None, bigdl_type, + input_size, + output_size) + + +class CosineDistance(Model): + + ''' + >>> cosineDistance = CosineDistance() + creating: createCosineDistance + ''' + + def __init__(self, + bigdl_type="float"): + super(CosineDistance, self).__init__(None, bigdl_type) + + +class DotProduct(Model): + + ''' + >>> dotProduct = DotProduct() + creating: createDotProduct + ''' + + def __init__(self, + bigdl_type="float"): + super(DotProduct, self).__init__(None, bigdl_type) + + +class ELU(Model): + + ''' + >>> eLU = ELU(1e-5, True) + creating: createELU + ''' + + def __init__(self, + alpha=1.0, + inplace=False, + bigdl_type="float"): + super(ELU, self).__init__(None, bigdl_type, + alpha, + inplace) + + +class Euclidean(Model): + + ''' + >>> euclidean = Euclidean(1, 1, True) + creating: createEuclidean + ''' + + def __init__(self, + input_size, + output_size, + fast_backward=True, + bigdl_type="float"): + super(Euclidean, self).__init__(None, bigdl_type, + input_size, + output_size, + fast_backward) + + +class Exp(Model): + + ''' + >>> exp = Exp() + creating: createExp + ''' + + def __init__(self, + bigdl_type="float"): + super(Exp, self).__init__(None, bigdl_type) + + +class FlattenTable(Model): + + ''' + >>> flattenTable = FlattenTable() + creating: createFlattenTable + ''' + + def __init__(self, + bigdl_type="float"): + super(FlattenTable, self).__init__(None, bigdl_type) + + +class GradientReversal(Model): + + ''' + >>> gradientReversal = GradientReversal(1e-5) + creating: createGradientReversal + ''' + + def __init__(self, + the_lambda=1, + bigdl_type="float"): + super(GradientReversal, self).__init__(None, bigdl_type, + the_lambda) + + +class HardShrink(Model): + + ''' + >>> hardShrink = HardShrink(1e-5) + creating: createHardShrink + ''' + + def __init__(self, + the_lambda=0.5, + bigdl_type="float"): + super(HardShrink, self).__init__(None, bigdl_type, + the_lambda) + + +class HardTanh(Model): + + ''' + >>> hardTanh = HardTanh(1e-5, 1e5, True) + creating: createHardTanh + ''' + + def __init__(self, + min_value=-1, + max_value=1, + inplace=False, + bigdl_type="float"): + super(HardTanh, self).__init__(None, bigdl_type, + min_value, + max_value, + inplace) + + +class Index(Model): + + ''' + >>> index = Index(1) + creating: createIndex + ''' + + def __init__(self, + dimension, + bigdl_type="float"): + super(Index, self).__init__(None, bigdl_type, + dimension) + + +class InferReshape(Model): + + ''' + >>> inferReshape = InferReshape([4, 0, 3, -1], False) + creating: createInferReshape + ''' + + def __init__(self, + size, + batch_mode=False, + bigdl_type="float"): + super(InferReshape, self).__init__(None, bigdl_type, + size, + batch_mode) + + +class JoinTable(Model): + + ''' + >>> joinTable = JoinTable(1, 1) + creating: createJoinTable + ''' + + def __init__(self, + dimension, + n_input_dims, + bigdl_type="float"): + super(JoinTable, self).__init__(None, bigdl_type, + dimension, + n_input_dims) + + +class L1Cost(Model): + + ''' + >>> l1Cost = L1Cost() + creating: createL1Cost + ''' + + def __init__(self, + bigdl_type="float"): + super(L1Cost, self).__init__(None, bigdl_type) + + +class L1Penalty(Model): + + ''' + >>> l1Penalty = L1Penalty(1, True, True) + creating: createL1Penalty + ''' + + def __init__(self, + l1weight, + size_average=False, + provide_output=True, + bigdl_type="float"): + super(L1Penalty, self).__init__(None, bigdl_type, + l1weight, + size_average, + provide_output) + + +class LeakyReLU(Model): + + ''' + >>> leakyReLU = LeakyReLU(1e-5, True) + creating: createLeakyReLU + ''' + + def __init__(self, + negval=0.01, + inplace=False, + bigdl_type="float"): + super(LeakyReLU, self).__init__(None, bigdl_type, + negval, + inplace) + + +class Log(Model): + + ''' + >>> log = Log() + creating: createLog + ''' + + def __init__(self, + bigdl_type="float"): + super(Log, self).__init__(None, bigdl_type) + + +class LogSigmoid(Model): + + ''' + >>> logSigmoid = LogSigmoid() + creating: createLogSigmoid + ''' + + def __init__(self, + bigdl_type="float"): + super(LogSigmoid, self).__init__(None, bigdl_type) + + +class LookupTable(Model): + + ''' + >>> lookupTable = LookupTable(1, 1, 1e-5, 1e-5, 1e-5, True) + creating: createLookupTable + ''' + + def __init__(self, + n_index, + n_output, + padding_value=0, + max_norm=DOUBLEMAX, + norm_type=2.0, + should_scale_grad_by_freq=False, + bigdl_type="float"): + super(LookupTable, self).__init__(None, bigdl_type, + n_index, + n_output, + padding_value, + max_norm, + norm_type, + should_scale_grad_by_freq) + + +class MM(Model): + + ''' + >>> mM = MM(True, True) + creating: createMM + ''' + + def __init__(self, + trans_a=False, + trans_b=False, + bigdl_type="float"): + super(MM, self).__init__(None, bigdl_type, + trans_a, + trans_b) + + +class MV(Model): + + ''' + >>> mV = MV(True) + creating: createMV + ''' + + def __init__(self, + trans=False, + bigdl_type="float"): + super(MV, self).__init__(None, bigdl_type, + trans) + + +class MapTable(Model): + + ''' + >>> mapTable = MapTable(Linear(100,10)) + creating: createLinear + creating: createMapTable + ''' + + def __init__(self, + module, + bigdl_type="float"): + super(MapTable, self).__init__(None, bigdl_type, + module) + + +class MaskedSelect(Model): + + ''' + >>> maskedSelect = MaskedSelect() + creating: createMaskedSelect + ''' + + def __init__(self, + bigdl_type="float"): + super(MaskedSelect, self).__init__(None, bigdl_type) + + +class Max(Model): + + ''' + >>> max = Max(1) + creating: createMax + ''' + + def __init__(self, + dim, + num_input_dims=INTMIN, + bigdl_type="float"): + super(Max, self).__init__(None, bigdl_type, + dim, + num_input_dims) + + +class Mean(Model): + + ''' + >>> mean = Mean(1, 1) + creating: createMean + ''' + + def __init__(self, + dimension=1, + n_input_dims=-1, + bigdl_type="float"): + super(Mean, self).__init__(None, bigdl_type, + dimension, + n_input_dims) + + +class Min(Model): + + ''' + >>> min = Min(1) + creating: createMin + ''' + + def __init__(self, + dim, + num_input_dims=INTMIN, + bigdl_type="float"): + super(Min, self).__init__(None, bigdl_type, + dim, + num_input_dims) + + +class MixtureTable(Model): + + ''' + >>> mixtureTable = MixtureTable() + creating: createMixtureTable + ''' + + def __init__(self, + bigdl_type="float"): + super(MixtureTable, self).__init__(None, bigdl_type) + + +class Mul(Model): + + ''' + >>> mul = Mul() + creating: createMul + ''' + + def __init__(self, + bigdl_type="float"): + super(Mul, self).__init__(None, bigdl_type) + + +class MulConstant(Model): + + ''' + >>> mulConstant = MulConstant(2.5) + creating: createMulConstant + ''' + + def __init__(self, + scalar, + inplace=False, + bigdl_type="float"): + super(MulConstant, self).__init__(None, bigdl_type, + scalar, + inplace) + + +class Narrow(Model): + + ''' + >>> narrow = Narrow(1, 1, 1) + creating: createNarrow + ''' + + def __init__(self, + dimension, + offset, + length=1, + bigdl_type="float"): + super(Narrow, self).__init__(None, bigdl_type, + dimension, + offset, + length) + + +class NarrowTable(Model): + + ''' + >>> narrowTable = NarrowTable(1, 1) + creating: createNarrowTable + ''' + + def __init__(self, + offset, + length=1, + bigdl_type="float"): + super(NarrowTable, self).__init__(None, bigdl_type, + offset, + length) + + +class Normalize(Model): + + ''' + >>> normalize = Normalize(1e-5, 1e-5) + creating: createNormalize + ''' + + def __init__(self, + p, + eps=1e-10, + bigdl_type="float"): + super(Normalize, self).__init__(None, bigdl_type, + p, + eps) + + +class PReLU(Model): + + ''' + >>> pReLU = PReLU(1) + creating: createPReLU + ''' + + def __init__(self, + n_output_plane=0, + bigdl_type="float"): + super(PReLU, self).__init__(None, bigdl_type, + n_output_plane) + + +class Padding(Model): + + ''' + >>> padding = Padding(1, 1, 1, 1e-5, 1) + creating: createPadding + ''' + + def __init__(self, + dim, + pad, + n_input_dim, + value=0.0, + n_index=1, + bigdl_type="float"): + super(Padding, self).__init__(None, bigdl_type, + dim, + pad, + n_input_dim, + value, + n_index) + + +class PairwiseDistance(Model): + + ''' + >>> pairwiseDistance = PairwiseDistance(2) + creating: createPairwiseDistance + ''' + + def __init__(self, + norm=2, + bigdl_type="float"): + super(PairwiseDistance, self).__init__(None, bigdl_type, + norm) + + +class ParallelTable(Model): + + ''' + >>> parallelTable = ParallelTable() + creating: createParallelTable + ''' + + def __init__(self, + bigdl_type="float"): + super(ParallelTable, self).__init__(None, bigdl_type) + + +class Power(Model): + + ''' + >>> power = Power(1e-5) + creating: createPower + ''' + + def __init__(self, + power, + scale=1.0, + shift=0.0, + bigdl_type="float"): + super(Power, self).__init__(None, bigdl_type, + power, + scale, + shift) + + +class RReLU(Model): + + ''' + >>> rReLU = RReLU(1e-5, 1e5, True) + creating: createRReLU + ''' + + def __init__(self, + lower=1.0/8, + upper=1.0/3, + inplace=False, + bigdl_type="float"): + super(RReLU, self).__init__(None, bigdl_type, + lower, + upper, + inplace) + + +class ReLU6(Model): + + ''' + >>> reLU6 = ReLU6(True) + creating: createReLU6 + ''' + + def __init__(self, + inplace=False, + bigdl_type="float"): + super(ReLU6, self).__init__(None, bigdl_type, + inplace) + + +class Replicate(Model): + + ''' + >>> replicate = Replicate(2) + creating: createReplicate + ''' + + def __init__(self, + n_features, + dim=1, + n_dim=INTMAX, + bigdl_type="float"): + super(Replicate, self).__init__(None, bigdl_type, + n_features, + dim, + n_dim) + + +class RoiPooling(Model): + + ''' + >>> roiPooling = RoiPooling(1, 1, 1e-5) + creating: createRoiPooling + ''' + + def __init__(self, + pooled_w, + pooled_h, + spatial_scale, + bigdl_type="float"): + super(RoiPooling, self).__init__(None, bigdl_type, + pooled_w, + pooled_h, + spatial_scale) + + +class Scale(Model): + + ''' + >>> scale = Scale([1,2]) + creating: createScale + ''' + + def __init__(self, + size, + bigdl_type="float"): + super(Scale, self).__init__(None, bigdl_type, + size) + + +class SelectTable(Model): + + ''' + >>> selectTable = SelectTable(1) + creating: createSelectTable + ''' + + def __init__(self, + dimension, + bigdl_type="float"): + super(SelectTable, self).__init__(None, bigdl_type, + dimension) + + +class Sigmoid(Model): + + ''' + >>> sigmoid = Sigmoid() + creating: createSigmoid + ''' + + def __init__(self, + bigdl_type="float"): + super(Sigmoid, self).__init__(None, bigdl_type) + + +class SoftMax(Model): + + ''' + >>> softMax = SoftMax() + creating: createSoftMax + ''' + + def __init__(self, + bigdl_type="float"): + super(SoftMax, self).__init__(None, bigdl_type) + + +class SoftMin(Model): + + ''' + >>> softMin = SoftMin() + creating: createSoftMin + ''' + + def __init__(self, + bigdl_type="float"): + super(SoftMin, self).__init__(None, bigdl_type) + + +class SoftPlus(Model): + + ''' + >>> softPlus = SoftPlus(1e-5) + creating: createSoftPlus + ''' + + def __init__(self, + beta=1.0, + bigdl_type="float"): + super(SoftPlus, self).__init__(None, bigdl_type, + beta) + + +class SoftShrink(Model): + + ''' + >>> softShrink = SoftShrink(1e-5) + creating: createSoftShrink + ''' + + def __init__(self, + the_lambda=0.5, + bigdl_type="float"): + super(SoftShrink, self).__init__(None, bigdl_type, + the_lambda) + + +class SoftSign(Model): + + ''' + >>> softSign = SoftSign() + creating: createSoftSign + ''' + + def __init__(self, + bigdl_type="float"): + super(SoftSign, self).__init__(None, bigdl_type) + + +class SpatialDilatedConvolution(Model): + + ''' + >>> spatialDilatedConvolution = SpatialDilatedConvolution(1, 1, 1, 1) + creating: createSpatialDilatedConvolution + ''' + + def __init__(self, + n_input_plane, + n_output_plane, + kw, + kh, + dw=1, + dh=1, + pad_w=0, + pad_h=0, + dilation_w=1, + dilation_h=1, + init_method='default', + bigdl_type="float"): + super(SpatialDilatedConvolution, self).__init__(None, bigdl_type, + n_input_plane, + n_output_plane, + kw, + kh, + dw, + dh, + pad_w, + pad_h, + dilation_w, + dilation_h, + init_method) + + +class SpatialFullConvolution(Model): + + ''' + >>> spatialFullConvolution = SpatialFullConvolution(1, 1, 1, 1) + creating: createSpatialFullConvolution + ''' + + def __init__(self, + n_input_plane, + n_output_plane, + kw, + kh, + dw=1, + dh=1, + pad_w=0, + pad_h=0, + adj_w=0, + adj_h=0, + n_group=1, + no_bias=False, + init_method='default', + bigdl_type="float"): + super(SpatialFullConvolution, self).__init__(None, bigdl_type, + n_input_plane, + n_output_plane, + kw, + kh, + dw, + dh, + pad_w, + pad_h, + adj_w, + adj_h, + n_group, + no_bias, + init_method) + + +class SpatialShareConvolution(Model): + + ''' + >>> spatialShareConvolution = SpatialShareConvolution(1, 1, 1, 1) + creating: createSpatialShareConvolution + ''' + + def __init__(self, + n_input_plane, + n_output_plane, + kernel_w, + kernel_h, + stride_w=1, + stride_h=1, + pad_w=0, + pad_h=0, + n_group=1, + propagate_back=True, + init_method='default', + bigdl_type="float"): + super(SpatialShareConvolution, self).__init__(None, bigdl_type, + n_input_plane, + n_output_plane, + kernel_w, + kernel_h, + stride_w, + stride_h, + pad_w, + pad_h, + n_group, + propagate_back, + init_method) + + +class SpatialZeroPadding(Model): + + ''' + >>> spatialZeroPadding = SpatialZeroPadding(1, 1, 1, 1) + creating: createSpatialZeroPadding + ''' + + def __init__(self, + pad_left, + pad_right, + pad_top, + pad_bottom, + bigdl_type="float"): + super(SpatialZeroPadding, self).__init__(None, bigdl_type, + pad_left, + pad_right, + pad_top, + pad_bottom) + + +class SplitTable(Model): + + ''' + >>> splitTable = SplitTable(1, 1) + creating: createSplitTable + ''' + + def __init__(self, + dimension, + n_input_dims=-1, + bigdl_type="float"): + super(SplitTable, self).__init__(None, bigdl_type, + dimension, + n_input_dims) + + +class Sqrt(Model): + + ''' + >>> sqrt = Sqrt() + creating: createSqrt + ''' + + def __init__(self, + bigdl_type="float"): + super(Sqrt, self).__init__(None, bigdl_type) + + +class Square(Model): + + ''' + >>> square = Square() + creating: createSquare + ''' + + def __init__(self, + bigdl_type="float"): + super(Square, self).__init__(None, bigdl_type) + + +class Squeeze(Model): + + ''' + >>> squeeze = Squeeze(1) + creating: createSqueeze + ''' + + def __init__(self, + dim, + num_input_dims=INTMIN, + bigdl_type="float"): + super(Squeeze, self).__init__(None, bigdl_type, + dim, + num_input_dims) + + +class Sum(Model): + + ''' + >>> sum = Sum(1, 1, True) + creating: createSum + ''' + + def __init__(self, + dimension=1, + n_input_dims=-1, + size_average=False, + bigdl_type="float"): + super(Sum, self).__init__(None, bigdl_type, + dimension, + n_input_dims, + size_average) + + +class TanhShrink(Model): + + ''' + >>> tanhShrink = TanhShrink() + creating: createTanhShrink + ''' + + def __init__(self, + bigdl_type="float"): + super(TanhShrink, self).__init__(None, bigdl_type) + + +class Threshold(Model): + + ''' + >>> threshold = Threshold(1e-5, 1e-5, True) + creating: createThreshold + ''' + + def __init__(self, + th=1e-6, + v=0.0, + ip=False, + bigdl_type="float"): + super(Threshold, self).__init__(None, bigdl_type, + th, + v, + ip) + + +class Unsqueeze(Model): + + ''' + >>> unsqueeze = Unsqueeze(1, 1) + creating: createUnsqueeze + ''' + + def __init__(self, + pos, + num_input_dims=INTMIN, + bigdl_type="float"): + super(Unsqueeze, self).__init__(None, bigdl_type, + pos, + num_input_dims) + + +class Reshape(Model): + ''' + >>> reshape = Reshape([1, 28, 28]) + creating: createReshape + ''' + + def __init__(self, size, bigdl_type="float"): + super(Reshape, self).__init__(None, bigdl_type, size) + + +def _test(): + import doctest + from pyspark import SparkContext + from nn import layer + from util.common import init_engine + from util.common import create_spark_conf + globs = layer.__dict__.copy() + sc = SparkContext(master="local[4]", appName="test layer", + conf=create_spark_conf()) + globs['sc'] = sc + init_engine() + + (failure_count, test_count) = doctest.testmod(globs=globs, + optionflags=doctest.ELLIPSIS) + if failure_count: + exit(-1) + +if __name__ == "__main__": + _test() diff --git a/python/dllib/src/bigdl/dllib/optim/__init__.py b/python/dllib/src/bigdl/dllib/optim/__init__.py new file mode 100644 index 00000000000..2151a805423 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/optim/__init__.py @@ -0,0 +1,15 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/python/dllib/src/bigdl/dllib/optim/optimizer.py b/python/dllib/src/bigdl/dllib/optim/optimizer.py new file mode 100644 index 00000000000..4272e54f696 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/optim/optimizer.py @@ -0,0 +1,169 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +from util.common import callBigDlFunc +from util.common import JavaValue +from util.common import callJavaFunc +from pyspark import SparkContext +import numpy as np +import os +from distutils.dir_util import mkpath + + +import sys +if sys.version >= '3': + long = int + unicode = str + + +class MaxIteration(JavaValue): + ''' + >>> maxIteration = MaxIteration(20) + creating: createMaxIteration + ''' + def __init__(self, max, bigdl_type="float"): + JavaValue.__init__(self, None, bigdl_type, max) + + +class MaxEpoch(JavaValue): + ''' + >>> maxEpoch = MaxEpoch(2) + creating: createMaxEpoch + ''' + def __init__(self, max_epoch, bigdl_type="float"): + JavaValue.__init__(self, None, bigdl_type, max_epoch) + + +class EveryEpoch(JavaValue): + ''' + >>> everyEpoch = EveryEpoch() + creating: createEveryEpoch + ''' + def __init__(self, bigdl_type="float"): + JavaValue.__init__(self, None, bigdl_type) + + +class SeveralIteration(JavaValue): + ''' + >>> serveralIteration = SeveralIteration(2) + creating: createSeveralIteration + ''' + def __init__(self, interval, bigdl_type="float"): + JavaValue.__init__(self, None, bigdl_type, interval) + + +class Poly(JavaValue): + ''' + >>> poly = Poly(0.5, 2) + creating: createPoly + ''' + def __init__(self, power, max_iteration, bigdl_type="float"): + JavaValue.__init__(self, None, bigdl_type, power, max_iteration) + + +class Step(JavaValue): + ''' + >>> step = Step(2, 0.3) + creating: createStep + ''' + def __init__(self, step_size, gamma, bigdl_type="float"): + JavaValue.__init__(self, None, bigdl_type, step_size, gamma) + + +class Optimizer(JavaValue): + + def __init__(self, + model, + training_rdd, + criterion, + end_trigger, + batch_size, + optim_method="SGD", + state={}, + bigdl_type="float"): + JavaValue.__init__(self, None, bigdl_type, model.value, + training_rdd, criterion, optim_method, + state, end_trigger, batch_size) + + def setvalidation(self, batch_size, val_rdd, trigger, val_method=["Top1Accuracy"]): + callBigDlFunc(self.bigdl_type, "setValidation", self.value, batch_size, + trigger, val_rdd, val_method) + + def setcheckpoint(self, checkpoint_trigger, + checkpoint_path, isOverWrite=True): + if not os.path.exists(checkpoint_path): + mkpath(checkpoint_path) + callBigDlFunc(self.bigdl_type, "setCheckPoint", self.value, + checkpoint_trigger, checkpoint_path, isOverWrite) + + # return a module + def optimize(self): + jmodel = callJavaFunc(SparkContext.getOrCreate(), self.value.optimize) + from nn.layer import Model + return Model.of(jmodel) + + def set_train_summary(self, summary): + callBigDlFunc(self.bigdl_type, "setTrainSummary", self.value, + summary) + return self + + def set_val_summary(self, summary): + callBigDlFunc(self.bigdl_type, "setValSummary", self.value, + summary) + return self + + +class TrainSummary(JavaValue, ): + def __init__(self, log_dir, app_name, bigdl_type="float"): + JavaValue.__init__(self, None, bigdl_type, log_dir, app_name) + + def read_scalar(self, tag): + return callBigDlFunc(self.bigdl_type, "summaryReadScalar", self.value, + tag) + + def set_summary_trigger(self, name, trigger): + return callBigDlFunc(self.bigdl_type, "summarySetTrigger", self.value, + name, trigger) + + +class ValidationSummary(JavaValue): + def __init__(self, log_dir, app_name, bigdl_type="float"): + JavaValue.__init__(self, None, bigdl_type, log_dir, app_name) + + def read_scalar(self, tag): + return callBigDlFunc(self.bigdl_type, "summaryReadScalar", self.value, + tag) + + +def _test(): + import doctest + from pyspark import SparkContext + from optim import optimizer + from util.common import init_engine + from util.common import create_spark_conf + globs = optimizer.__dict__.copy() + sc = SparkContext(master="local[4]", appName="test optimizer", + conf=create_spark_conf()) + init_engine() + globs['sc'] = sc + (failure_count, test_count) = doctest.testmod(globs=globs, + optionflags=doctest.ELLIPSIS) + if failure_count: + exit(-1) + +if __name__ == "__main__": + _test() diff --git a/python/dllib/src/bigdl/dllib/utils/common.py b/python/dllib/src/bigdl/dllib/utils/common.py new file mode 100644 index 00000000000..bbac74231b2 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/utils/common.py @@ -0,0 +1,261 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import sys +from py4j.protocol import Py4JJavaError +from py4j.java_gateway import JavaObject +from py4j.java_collections import ListConverter, JavaArray, JavaList, JavaMap + +from pyspark import RDD, SparkContext +from pyspark.serializers import PickleSerializer, AutoBatchedSerializer +from pyspark.sql import DataFrame, SQLContext +from pyspark.mllib.common import callJavaFunc +from pyspark import SparkConf +import numpy as np + +if sys.version >= '3': + long = int + unicode = str + + +class JavaValue(object): + def jvm_class_constructor(self): + name = "create" + self.__class__.__name__ + print("creating: " + name) + return name + + def __init__(self, jvalue, bigdl_type, *args): + self.value = jvalue if jvalue else callBigDlFunc( + bigdl_type, self.jvm_class_constructor(), *args) + self.bigdl_type = bigdl_type + + +class TestResult(): + def __init__(self, result, total_num, method): + self.result = result + self.total_num = total_num + self.method = method + + def __reduce__(self): + return (TestResult, (self.result, self.total_num, self.method)) + + def __str__(self): + return "Test result: %s, total_num: %s, method: %s" % ( + self.result, self.total_num, self.method) + + +class Sample(object): + def __init__(self, features, label, features_shape, label_shape, + bigdl_type="float"): + def get_dtype(): + if "float" == bigdl_type: + return "float32" + else: + return "float64" + self.features = np.array(features, dtype=get_dtype()).reshape(features_shape) # noqa + self.label = np.array(label, dtype=get_dtype()).reshape(label_shape) + self.bigdl_type = bigdl_type + + @classmethod + def from_ndarray(cls, features, label, bigdl_type="float"): + return cls( + features=[float(i) for i in features.ravel()], + label=[float(i) for i in label.ravel()], + features_shape=list(features.shape), + label_shape=list(label.shape) if label.shape else [label.size], + bigdl_type=bigdl_type) + + @classmethod + def flatten(cls, a_ndarray): + """ + Utility method to flatten a ndarray + :return (storage, shape) + >>> import numpy as np + >>> from util.common import Sample + >>> np.random.seed(123) + >>> data = np.random.uniform(0, 1, (2, 3)) + >>> (storage, shape) = Sample.flatten(data) + >>> shape + [2, 3] + >>> (storage, shape) = Sample.flatten(np.array(2)) + >>> shape + [1] + """ + storage = [float(i) for i in a_ndarray.ravel()] + shape = list(a_ndarray.shape) if a_ndarray.shape else [a_ndarray.size] + return storage, shape + + def __reduce__(self): + (features_storage, features_shape) = Sample.flatten(self.features) + (label_storage, label_shape) = Sample.flatten(self.label) + return (Sample, ( + features_storage, label_storage, features_shape, label_shape, + self.bigdl_type)) + + def __str__(self): + return "features: %s, label: %s," % (self.features, self.label) + + +_picklable_classes = [ + 'LinkedList', + 'SparseVector', + 'DenseVector', + 'DenseMatrix', + 'Rating', + 'LabeledPoint', + 'Sample', + 'TestResult' +] + + +def init_engine(bigdl_type="float"): + callBigDlFunc(bigdl_type, "initEngine") + + +def get_bigdl_conf(): + bigdl_conf_file = "spark-bigdl.conf" + bigdl_python_wrapper = "python-api.zip" + + def load_conf(conf_str): + return dict(line.split() for line in conf_str.split("\n") if + "#" not in line and line.strip()) + + for p in sys.path: + if bigdl_conf_file in p: + with open(p) as conf_file: + return load_conf(conf_file.read()) + if bigdl_python_wrapper in p: + import zipfile + with zipfile.ZipFile(p, 'r') as zip_conf: + return load_conf(zip_conf.read(bigdl_conf_file)) + raise Exception("Cannot find spark-bigdl.conf.Pls add it to PYTHONPATH.") + + +def create_spark_conf(): + bigdl_conf = get_bigdl_conf() + sparkConf = SparkConf() + sparkConf.setAll(bigdl_conf.items()) + return sparkConf + + +def callBigDlFunc(bigdl_type, name, *args): + """ Call API in PythonBigDL """ + sc = SparkContext.getOrCreate() + if bigdl_type == "float": + api = getattr( + sc._jvm.com.intel.analytics.bigdl.python.api.PythonBigDL.ofFloat(), + name) + elif bigdl_type == "double": + api = getattr( + sc._jvm.com.intel.analytics.bigdl.python.api.PythonBigDL.ofDouble(), + name) + else: + raise Exception("Not supported bigdl_type: %s" % bigdl_type) + return callJavaFunc(sc, api, *args) + + +def _java2py(sc, r, encoding="bytes"): + if isinstance(r, JavaObject): + clsName = r.getClass().getSimpleName() + # convert RDD into JavaRDD + if clsName != 'JavaRDD' and clsName.endswith("RDD"): + r = r.toJavaRDD() + clsName = 'JavaRDD' + + if clsName == 'JavaRDD': + jrdd = sc._jvm.SerDe.javaToPython(r) + return RDD(jrdd, sc) + + if clsName == 'DataFrame': + return DataFrame(r, SQLContext.getOrCreate(sc)) + + if clsName in _picklable_classes: + r = sc._jvm.org.apache.spark.bigdl.api.python.BigDLSerDe.dumps(r) + elif isinstance(r, (JavaArray, JavaList, JavaMap)): + try: + r = sc._jvm.org.apache.spark.bigdl.api.python.BigDLSerDe.dumps( + r) + except Py4JJavaError: + pass # not pickable + + if isinstance(r, (bytearray, bytes)): + r = PickleSerializer().loads(bytes(r), encoding=encoding) + return r + + +def callJavaFunc(sc, func, *args): + """ Call Java Function """ + args = [_py2java(sc, a) for a in args] + result = func(*args) + return _java2py(sc, result) + + +def _to_java_object_rdd(rdd): + """ Return a JavaRDD of Object by unpickling + + It will convert each Python object into Java object by Pyrolite, whenever + the RDD is serialized in batch or not. + """ + rdd = rdd._reserialize(AutoBatchedSerializer(PickleSerializer())) + return \ + rdd.ctx._jvm.org.apache.spark.bigdl.api.python.BigDLSerDe.pythonToJava( + rdd._jrdd, True) + + +def _py2java(sc, obj): + """ Convert Python object into Java """ + if isinstance(obj, RDD): + obj = _to_java_object_rdd(obj) + elif isinstance(obj, DataFrame): + obj = obj._jdf + elif isinstance(obj, SparkContext): + obj = obj._jsc + elif isinstance(obj, (list, tuple)): + obj = ListConverter().convert([_py2java(sc, x) for x in obj], + sc._gateway._gateway_client) + elif isinstance(obj, dict): + result = {} + for (key, value) in obj.iteritems(): + result[key] = _py2java(sc, value) if isinstance(value, JavaValue) else value # noqa + obj = result + + elif isinstance(obj, JavaValue): + obj = obj.value + elif isinstance(obj, JavaObject): + pass + elif isinstance(obj, (int, long, float, bool, bytes, unicode)): + pass + else: + data = bytearray(PickleSerializer().dumps(obj)) + obj = sc._jvm.org.apache.spark.bigdl.api.python.BigDLSerDe.loads(data) + return obj + + +def _test(): + import doctest + from pyspark import SparkContext + from nn import layer + globs = layer.__dict__.copy() + sc = SparkContext(master="local[2]", appName="test common utility") + globs['sc'] = sc + (failure_count, test_count) = doctest.testmod(globs=globs, + optionflags=doctest.ELLIPSIS) + if failure_count: + exit(-1) + + +if __name__ == "__main__": + _test() diff --git a/python/dllib/src/bigdl/utils/__init__.py b/python/dllib/src/bigdl/utils/__init__.py new file mode 100644 index 00000000000..2151a805423 --- /dev/null +++ b/python/dllib/src/bigdl/utils/__init__.py @@ -0,0 +1,15 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# From bedc6b22864702aa885f0c31a680414850c0df5d Mon Sep 17 00:00:00 2001 From: zhangli Date: Tue, 14 Mar 2017 16:37:35 +0800 Subject: [PATCH 002/823] refactor --- README.md | 127 ++ .../bigdl/dllib/feature/dataset/__init__.py | 15 + .../src/bigdl/dllib/feature/dataset/base.py | 197 ++ .../src/bigdl/dllib/feature/dataset/mnist.py | 113 ++ .../src/bigdl/dllib/feature/dataset/news20.py | 73 + .../dllib/feature/dataset/transformer.py | 23 + .../dllib/src/bigdl/dllib/models/__init__.py | 15 + .../src/bigdl/dllib/models/lenet/__init__.py | 15 + .../src/bigdl/dllib/models/lenet/lenet5.py | 97 + .../models/textclassifier/textclassifier.py | 177 ++ python/dllib/src/bigdl/dllib/nn/__init__.py | 15 + python/dllib/src/bigdl/dllib/nn/layer.py | 1673 +++++++++++++++++ .../dllib/src/bigdl/dllib/optim/__init__.py | 15 + .../dllib/src/bigdl/dllib/optim/optimizer.py | 169 ++ python/dllib/src/bigdl/utils/__init__.py | 15 + python/dllib/src/bigdl/utils/common.py | 261 +++ 16 files changed, 3000 insertions(+) create mode 100644 README.md create mode 100644 python/dllib/src/bigdl/dllib/feature/dataset/__init__.py create mode 100644 python/dllib/src/bigdl/dllib/feature/dataset/base.py create mode 100644 python/dllib/src/bigdl/dllib/feature/dataset/mnist.py create mode 100644 python/dllib/src/bigdl/dllib/feature/dataset/news20.py create mode 100644 python/dllib/src/bigdl/dllib/feature/dataset/transformer.py create mode 100644 python/dllib/src/bigdl/dllib/models/__init__.py create mode 100644 python/dllib/src/bigdl/dllib/models/lenet/__init__.py create mode 100644 python/dllib/src/bigdl/dllib/models/lenet/lenet5.py create mode 100644 python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py create mode 100644 python/dllib/src/bigdl/dllib/nn/__init__.py create mode 100644 python/dllib/src/bigdl/dllib/nn/layer.py create mode 100644 python/dllib/src/bigdl/dllib/optim/__init__.py create mode 100644 python/dllib/src/bigdl/dllib/optim/optimizer.py create mode 100644 python/dllib/src/bigdl/utils/__init__.py create mode 100644 python/dllib/src/bigdl/utils/common.py diff --git a/README.md b/README.md new file mode 100644 index 00000000000..a94e306c8f4 --- /dev/null +++ b/README.md @@ -0,0 +1,127 @@ + +BigDL comes with scala API for now. However Python is a powerful programming language for data analysis and with large amount of useful libraries, we are developing a lightweight python binding on top of PySpark which can enable us use Python naively with BigDL. + +The Python API is almost identical to the scala version, and it would map ndarray to tensor for the training samples, so basically user only need to care about how to manipulate ndarray. + +This Python binding tested with Python 2.7 and Spark 1.6.0. + +Here are the steps for training a simple LeNet model: + +1). Create a RDD[Sample]: +``` +RDD[..] --transform-->RDD[ndarray, ndarray].map(Sample.from_ndarray(features, label)) --> RDD[Sample] +``` + +2). Define a model: +``` + def build_model(class_num): + model = Sequential() + model.add(Reshape([1, 28, 28])) + model.add(SpatialConvolution(1, 6, 5, 5)) + model.add(Tanh()) + model.add(SpatialMaxPooling(2, 2, 2, 2)) + model.add(Tanh()) + model.add(SpatialConvolution(6, 12, 5, 5)) + model.add(SpatialMaxPooling(2, 2, 2, 2)) + model.add(Reshape([12 * 4 * 4])) + model.add(Linear(12 * 4 * 4, 100)) + model.add(Tanh()) + model.add(Linear(100, class_num)) + model.add(LogSoftMax()) + return model + ``` + +3). Create Optimizer and train: +``` + optimizer = Optimizer( + model=build_model(10), + training_rdd=train_data, + criterion=ClassNLLCriterion(), + optim_method="SGD", + state=state, + end_trigger=MaxEpoch(100), + batch_size=int(options.batchSize)) + optimizer.setvalidation( + batch_size=32, + val_rdd=test_data, + trigger=EveryEpoch(), + val_method=["top1"] + ) + optimizer.setcheckpoint(EveryEpoch(), "/tmp/lenet5/") + trained_model = optimizer.optimize() +``` + +4) LeNet example can be found from: models/lenet5.py + +## Run python test +* Package Scala code by: ```$BigDL_HOME/make-dist.sh``` +* Set SPARK_HOME and then run: ```$BigDL_HOME/dl/src/main/python/dev/run-all.sh``` + +## Installing on Ubuntu +1. Build BigDL +[Build Page](https://github.com/intel-analytics/BigDL/wiki/Build-Page) + * With Spark1.6: ``` $BIGDL_HOME//make-dist.sh ``` + * With Spark2.0: ``` $BIGDL_HOME//make-dist.sh -P spark_2.0 ``` + +2. Install python dependensies: + * Installing Numpy: + ```sudo apt-get install python-numpy``` + + * Installing Python setuptools: + ```sudo apt-get install -y python-setuptools python-pip``` + + * Install Jupyter: + ```sudo pip install jupyter``` + +## Run a Lenet example on standalone cluster + + ``` + BigDL_HOME=... + SPARK_HOME=... + MASTER=... + PYTHON_API_ZIP_PATH=${BigDL_HOME}/dist/lib/bigdl-0.1.0-SNAPSHOT-python-api.zip + BigDL_JAR_PATH=${BigDL_HOME}/dist/lib/bigdl-0.1.0-SNAPSHOT-jar-with-dependencies.jar + PYTHONPATH=${PYTHON_API_ZIP_PATH}:$PYTHONPATH + ${SPARK_HOME}/bin/spark-submit \ + --master ${MASTER} \ + --driver-cores 5 \ + --driver-memory 10g \ + --total-executor-cores 80 \ + --executor-cores 10 \ + --executor-memory 20g \ + --conf spark.akka.frameSize=64 \ + --py-files ${PYTHON_API_ZIP_PATH},${BigDL_HOME}/dl/src/main/python/models/lenet/lenet5.py \ + --properties-file ${BigDL_HOME}/dist/conf/spark-bigdl.conf \ + --jars ${BigDL_JAR_PATH} \ + --conf spark.driver.extraClassPath=${BigDL_JAR_PATH} \ + --conf spark.executor.extraClassPath=bigdl-0.1.0-SNAPSHOT-jar-with-dependencies.jar \ + ${BigDL_HOME}/dl/src/main/python/models/lenet/lenet5.py + ``` + + +## Launch Jupyter on standalone cluster + + ``` + BigDL_HOME=... + SPARK_HOME=... + MASTER=... + PYTHON_API_ZIP_PATH=${BigDL_HOME}/dist/lib/bigdl-0.1.0-SNAPSHOT-python-api.zip + BigDL_JAR_PATH=${BigDL_HOME}/dist/lib/bigdl-0.1.0-SNAPSHOT-jar-with-dependencies.jar + + export PYTHONPATH=${PYTHON_API_ZIP_PATH}:$PYTHONPATH + export IPYTHON_OPTS="notebook --notebook-dir=./ --ip=* --no-browser" + + ${SPARK_HOME}/bin/pyspark \ + --master ${MASTER} \ + --properties-file ${BigDL_HOME}/dist/conf/spark-bigdl.conf + --driver-cores 5 \ + --driver-memory 10g \ + --total-executor-cores 8 \ + --executor-cores 1 \ + --executor-memory 20g \ + --conf spark.akka.frameSize=64 \ + --py-files ${PYTHON_API_ZIP_PATH} \ + --jars ${BigDL_JAR_PATH} \ + --conf spark.driver.extraClassPath=${BigDL_JAR_PATH} \ + --conf spark.executor.extraClassPath=bigdl-0.1.0-SNAPSHOT-jar-with-dependencies.jar + ``` diff --git a/python/dllib/src/bigdl/dllib/feature/dataset/__init__.py b/python/dllib/src/bigdl/dllib/feature/dataset/__init__.py new file mode 100644 index 00000000000..2151a805423 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/feature/dataset/__init__.py @@ -0,0 +1,15 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/python/dllib/src/bigdl/dllib/feature/dataset/base.py b/python/dllib/src/bigdl/dllib/feature/dataset/base.py new file mode 100644 index 00000000000..f211ee5261c --- /dev/null +++ b/python/dllib/src/bigdl/dllib/feature/dataset/base.py @@ -0,0 +1,197 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os +import sys +import shutil +import tempfile +import time +from distutils.dir_util import mkpath +from six.moves.urllib.request import urlopen +import numpy as np + + +# Adopt from keras +class Progbar(object): + def __init__(self, target, width=30, verbose=1, interval=0.01): + ''' + @param target: total number of steps expected + @param interval: minimum visual progress update interval (in seconds) + ''' + self.width = width + self.target = target + self.sum_values = {} + self.unique_values = [] + self.start = time.time() + self.last_update = 0 + self.interval = interval + self.total_width = 0 + self.seen_so_far = 0 + self.verbose = verbose + + def update(self, current, values=[], force=False): + ''' + @param current: index of current step + @param values: list of tuples (name, value_for_last_step). + The progress bar will display averages for these values. + @param force: force visual progress update + ''' + for k, v in values: + if k not in self.sum_values: + self.sum_values[k] = [v * (current - self.seen_so_far), current - self.seen_so_far] + self.unique_values.append(k) + else: + self.sum_values[k][0] += v * (current - self.seen_so_far) + self.sum_values[k][1] += (current - self.seen_so_far) + self.seen_so_far = current + + now = time.time() + if self.verbose == 1: + if not force and (now - self.last_update) < self.interval: + return + + prev_total_width = self.total_width + sys.stdout.write("\b" * prev_total_width) + sys.stdout.write("\r") + + numdigits = int(np.floor(np.log10(self.target))) + 1 + barstr = '%%%dd/%%%dd [' % (numdigits, numdigits) + bar = barstr % (current, self.target) + prog = float(current) / self.target + prog_width = int(self.width * prog) + if prog_width > 0: + bar += ('=' * (prog_width-1)) + if current < self.target: + bar += '>' + else: + bar += '=' + bar += ('.' * (self.width - prog_width)) + bar += ']' + sys.stdout.write(bar) + self.total_width = len(bar) + + if current: + time_per_unit = (now - self.start) / current + else: + time_per_unit = 0 + eta = time_per_unit * (self.target - current) + info = '' + if current < self.target: + info += ' - ETA: %ds' % eta + else: + info += ' - %ds' % (now - self.start) + for k in self.unique_values: + info += ' - %s:' % k + if type(self.sum_values[k]) is list: + avg = self.sum_values[k][0] / max(1, self.sum_values[k][1]) + if abs(avg) > 1e-3: + info += ' %.4f' % avg + else: + info += ' %.4e' % avg + else: + info += ' %s' % self.sum_values[k] + + self.total_width += len(info) + if prev_total_width > self.total_width: + info += ((prev_total_width - self.total_width) * " ") + + sys.stdout.write(info) + sys.stdout.flush() + + if current >= self.target: + sys.stdout.write("\n") + + if self.verbose == 2: + if current >= self.target: + info = '%ds' % (now - self.start) + for k in self.unique_values: + info += ' - %s:' % k + avg = self.sum_values[k][0] / max(1, self.sum_values[k][1]) + if avg > 1e-3: + info += ' %.4f' % avg + else: + info += ' %.4e' % avg + sys.stdout.write(info + "\n") + + self.last_update = now + + def add(self, n, values=[]): + self.update(self.seen_so_far + n, values) + + +def display_table(rows, positions): + + def display_row(objects, positions): + line = '' + for i in range(len(objects)): + line += str(objects[i]) + line = line[:positions[i]] + line += ' ' * (positions[i] - len(line)) + print(line) + + for objects in rows: + display_row(objects, positions) + +# Adopt from keras +# Under Python 2, 'urlretrieve' relies on FancyURLopener from legacy +# urllib module, known to have issues with proxy management +if sys.version_info[0] == 2: + def urlretrieve(url, filename, reporthook=None, data=None): + def chunk_read(response, chunk_size=8192, reporthook=None): + total_size = response.info().get('Content-Length').strip() + total_size = int(total_size) + count = 0 + while 1: + chunk = response.read(chunk_size) + count += 1 + if not chunk: + reporthook(count, total_size, total_size) + break + if reporthook: + reporthook(count, chunk_size, total_size) + yield chunk + + response = urlopen(url, data) + with open(filename, 'wb') as fd: + for chunk in chunk_read(response, reporthook=reporthook): + fd.write(chunk) +else: + from six.moves.urllib.request import urlretrieve + + +def maybe_download(filename, work_directory, source_url): + if not os.path.exists(work_directory): + mkpath(work_directory) + filepath = os.path.join(work_directory, filename) + + if not os.path.exists(filepath): + print('Downloading data from', source_url) + global progbar + progbar = None + + def dl_progress(count, block_size, total_size): + global progbar + if progbar is None: + progbar = Progbar(total_size) + else: + progbar.update(count * block_size) + with tempfile.NamedTemporaryFile() as tmpfile: + temp_file_name = tmpfile.name + urlretrieve(source_url, temp_file_name, dl_progress) + shutil.copy2(temp_file_name, filepath) + size = os.path.getsize(filepath) + print('Successfully downloaded', filename, size, 'bytes.') + return filepath diff --git a/python/dllib/src/bigdl/dllib/feature/dataset/mnist.py b/python/dllib/src/bigdl/dllib/feature/dataset/mnist.py new file mode 100644 index 00000000000..4a53735fce2 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/feature/dataset/mnist.py @@ -0,0 +1,113 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Part of the code originally from Tensorflow + + +import gzip + +import numpy + +import base + +SOURCE_URL = 'http://yann.lecun.com/exdb/mnist/' + +TRAIN_MEAN = 0.13066047740239506 +TRAIN_STD = 0.3081078 +TEST_MEAN = 0.13251460696903547 +TEST_STD = 0.31048024 + + +def _read32(bytestream): + dt = numpy.dtype(numpy.uint32).newbyteorder('>') + return numpy.frombuffer(bytestream.read(4), dtype=dt)[0] + + +def extract_images(f): + """Extract the images into a 4D uint8 numpy array [index, y, x, depth]. + + Args: + f: A file object that can be passed into a gzip reader. + + Returns: + data: A 4D unit8 numpy array [index, y, x, depth]. + + Raises: + ValueError: If the bytestream does not start with 2051. + + """ + print('Extracting', f.name) + with gzip.GzipFile(fileobj=f) as bytestream: + magic = _read32(bytestream) + if magic != 2051: + raise ValueError( + 'Invalid magic number %d in MNIST image file: %s' % + (magic, f.name)) + num_images = _read32(bytestream) + rows = _read32(bytestream) + cols = _read32(bytestream) + buf = bytestream.read(rows * cols * num_images) + data = numpy.frombuffer(buf, dtype=numpy.uint8) + data = data.reshape(num_images, rows, cols, 1) + return data + + +def extract_labels(f): + print('Extracting', f.name) + with gzip.GzipFile(fileobj=f) as bytestream: + magic = _read32(bytestream) + if magic != 2049: + raise ValueError( + 'Invalid magic number %d in MNIST label file: %s' % + (magic, f.name)) + num_items = _read32(bytestream) + buf = bytestream.read(num_items) + labels = numpy.frombuffer(buf, dtype=numpy.uint8) + return labels + + +def read_data_sets(train_dir, data_type="train"): + TRAIN_IMAGES = 'train-images-idx3-ubyte.gz' + TRAIN_LABELS = 'train-labels-idx1-ubyte.gz' + TEST_IMAGES = 't10k-images-idx3-ubyte.gz' + TEST_LABELS = 't10k-labels-idx1-ubyte.gz' + + if data_type == "train": + local_file = base.maybe_download(TRAIN_IMAGES, train_dir, + SOURCE_URL + TRAIN_IMAGES) + with open(local_file, 'rb') as f: + train_images = extract_images(f) + + local_file = base.maybe_download(TRAIN_LABELS, train_dir, + SOURCE_URL + TRAIN_LABELS) + with open(local_file, 'rb') as f: + train_labels = extract_labels(f) + return train_images, train_labels + + else: + local_file = base.maybe_download(TEST_IMAGES, train_dir, + SOURCE_URL + TEST_IMAGES) + with open(local_file, 'rb') as f: + test_images = extract_images(f) + + local_file = base.maybe_download(TEST_LABELS, train_dir, + SOURCE_URL + TEST_LABELS) + with open(local_file, 'rb') as f: + test_labels = extract_labels(f) + return test_images, test_labels + + +if __name__ == "__main__": + read_data_sets("/tmp/mnist/") diff --git a/python/dllib/src/bigdl/dllib/feature/dataset/news20.py b/python/dllib/src/bigdl/dllib/feature/dataset/news20.py new file mode 100644 index 00000000000..fdedf5a6c41 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/feature/dataset/news20.py @@ -0,0 +1,73 @@ +import tarfile +import base +import os +import sys + +NEWS20_URL = 'http://qwone.com/~jason/20Newsgroups/20news-19997.tar.gz' # noqa +GLOVE_URL = 'http://nlp.stanford.edu/data/glove.6B.zip' # noqa + +CLASS_NUM = 20 + + +def download_news20(dest_dir): + file_name = "20news-19997.tar.gz" + file_abs_path = base.maybe_download(file_name, dest_dir, NEWS20_URL) + tar = tarfile.open(file_abs_path, "r:gz") + extracted_to = os.path.join(dest_dir, "20_newsgroups") + if not os.path.exists(extracted_to): + print("Extracting %s to %s" % (file_abs_path, extracted_to)) + tar.extractall(dest_dir) + tar.close() + return extracted_to + + +def download_glove_w2v(dest_dir): + file_name = "glove.6B.zip" + file_abs_path = base.maybe_download(file_name, dest_dir, GLOVE_URL) + import zipfile + zip_ref = zipfile.ZipFile(file_abs_path, 'r') + extracted_to = os.path.join(dest_dir, "glove.6B") + if not os.path.exists(extracted_to): + print("Extracting %s to %s" % (file_abs_path, extracted_to)) + zip_ref.extractall(extracted_to) + zip_ref.close() + return extracted_to + + +# return [(text_content, label)] +def get_news20(source_dir="/tmp/news20/"): + news_dir = download_news20(source_dir) + texts = [] # list of text samples + label_id = 0 + for name in sorted(os.listdir(news_dir)): + path = os.path.join(news_dir, name) + label_id += 1 + if os.path.isdir(path): + for fname in sorted(os.listdir(path)): + if fname.isdigit(): + fpath = os.path.join(path, fname) + if sys.version_info < (3,): + f = open(fpath) + else: + f = open(fpath, encoding='latin-1') + content = f.read() + texts.append((content, label_id)) + f.close() + + print('Found %s texts.' % len(texts)) + return texts + + +def get_glove_w2v(source_dir="/tmp/news20/", dim=100): + w2v_dir = download_glove_w2v(source_dir) + with open(os.path.join(w2v_dir, "glove.6B.%sd.txt" % dim)) as w2v_f: + pre_w2v = {} + for line in w2v_f.readlines(): + items = line.split(" ") + pre_w2v[items[0]] = [float(i) for i in items[1:]] + return pre_w2v + + +if __name__ == "__main__": + get_news20("/tmp/news20/") + get_glove_w2v("/tmp/news20/") diff --git a/python/dllib/src/bigdl/dllib/feature/dataset/transformer.py b/python/dllib/src/bigdl/dllib/feature/dataset/transformer.py new file mode 100644 index 00000000000..9b5d5930d3a --- /dev/null +++ b/python/dllib/src/bigdl/dllib/feature/dataset/transformer.py @@ -0,0 +1,23 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +from util.common import Sample + + +def normalizer(mean, std): + return lambda sample: Sample.from_ndarray((sample.features - mean) / std, + sample.label, sample.bigdl_type) diff --git a/python/dllib/src/bigdl/dllib/models/__init__.py b/python/dllib/src/bigdl/dllib/models/__init__.py new file mode 100644 index 00000000000..2151a805423 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/models/__init__.py @@ -0,0 +1,15 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/python/dllib/src/bigdl/dllib/models/lenet/__init__.py b/python/dllib/src/bigdl/dllib/models/lenet/__init__.py new file mode 100644 index 00000000000..2151a805423 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/models/lenet/__init__.py @@ -0,0 +1,15 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py b/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py new file mode 100644 index 00000000000..f6b1a752b40 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py @@ -0,0 +1,97 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Still in experimental stage! + +from optparse import OptionParser + +from dataset import mnist +from dataset.transformer import * +from nn.layer import * +from nn.criterion import * +from optim.optimizer import * +from util.common import * + + +def build_model(class_num): + model = Sequential() + model.add(Reshape([1, 28, 28])) + model.add(SpatialConvolution(1, 6, 5, 5)) + model.add(Tanh()) + model.add(SpatialMaxPooling(2, 2, 2, 2)) + model.add(Tanh()) + model.add(SpatialConvolution(6, 12, 5, 5)) + model.add(SpatialMaxPooling(2, 2, 2, 2)) + model.add(Reshape([12 * 4 * 4])) + model.add(Linear(12 * 4 * 4, 100)) + model.add(Tanh()) + model.add(Linear(100, class_num)) + model.add(LogSoftMax()) + return model + + +def get_minst(sc, data_type="train", location="/tmp/mnist"): + (images, labels) = mnist.read_data_sets(location, data_type) + images = sc.parallelize(images) + labels = sc.parallelize(labels) + # Target start from 1 in BigDL + record = images.zip(labels).map(lambda (features, label): + Sample.from_ndarray(features, label + 1)) + return record + + +if __name__ == "__main__": + + parser = OptionParser() + parser.add_option("-a", "--action", dest="action", default="train") + parser.add_option("-b", "--batchSize", dest="batchSize", default="128") + + (options, args) = parser.parse_args(sys.argv) + + sc = SparkContext(appName="lenet5", conf=create_spark_conf()) + init_engine() + + if options.action == "train": + train_data = get_minst(sc, "train").map( + normalizer(mnist.TRAIN_MEAN, mnist.TRAIN_STD)) + test_data = get_minst(sc, "test").map( + normalizer(mnist.TEST_MEAN, mnist.TEST_STD)) + state = {"learningRate": 0.01, + "learningRateDecay": 0.0002} + optimizer = Optimizer( + model=build_model(10), + training_rdd=train_data, + criterion=ClassNLLCriterion(), + optim_method="SGD", + state=state, + end_trigger=MaxEpoch(20), + batch_size=int(options.batchSize)) + optimizer.setvalidation( + batch_size=32, + val_rdd=test_data, + trigger=EveryEpoch(), + val_method=["Top1Accuracy"] + ) + optimizer.setcheckpoint(EveryEpoch(), "/tmp/lenet5/") + trained_model = optimizer.optimize() + parameters = trained_model.parameters() + elif options.action == "test": + test_data = get_minst("test").map( + normalizer(mnist.TEST_MEAN, mnist.TEST_STD)) + # TODO: Pass model path through external parameter + model = Model.from_path("/tmp/lenet5/lenet-model.470") + results = model.test(test_data, 32, ["Top1Accuracy"]) + for result in results: + print result diff --git a/python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py b/python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py new file mode 100644 index 00000000000..2929290aa0a --- /dev/null +++ b/python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py @@ -0,0 +1,177 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Still in experimental stage! + +import itertools +import re +from optparse import OptionParser + +from dataset import news20 +from nn.layer import * +from nn.criterion import * +from optim.optimizer import * +from util.common import * +from util.common import Sample + + +def text_to_words(review_text): + letters_only = re.sub("[^a-zA-Z]", " ", review_text) + words = letters_only.lower().split() + return words + + +def analyze_texts(data_rdd): + return data_rdd.flatMap(lambda (text, label): text_to_words(text)) \ + .map(lambda word: (word, 1)).reduceByKey(lambda a, b: a + b) \ + .sortBy(lambda (w, c): - c).zipWithIndex() \ + .map(lambda ((w, c), i): (w, (i + 1, c))).collect() + + +# pad([1, 2, 3, 4, 5], 0, 6) +def pad(l, fill_value, width): + if len(l) >= width: + return l[0: width] + else: + l.extend([fill_value] * (width - len(l))) + return l + + +def to_vec(token, b_w2v, embedding_dim): + if token in b_w2v: + return b_w2v[token] + else: + return pad([], 0, embedding_dim) + + +def to_sample(vectors, label, embedding_dim): + # flatten nested list + flatten_features = list(itertools.chain(*vectors)) + features = np.array(flatten_features, dtype='float').reshape( + [sequence_len, embedding_dim]) + + if model_type.lower() == "cnn": + features = features.transpose(1, 0) + return Sample.from_ndarray(features, np.array(label)) + + +def build_model(class_num): + model = Sequential() + + if model_type.lower() == "cnn": + model.add(Reshape([embedding_dim, 1, sequence_len])) + model.add(SpatialConvolution(embedding_dim, 128, 5, 1)) + model.add(ReLU()) + model.add(SpatialMaxPooling(5, 1, 5, 1)) + model.add(SpatialConvolution(128, 128, 5, 1)) + model.add(ReLU()) + model.add(SpatialMaxPooling(5, 1, 5, 1)) + model.add(Reshape([128])) + elif model_type.lower() == "lstm": + model.add(Recurrent() + .add(LSTM(embedding_dim, 128))) + model.add(Select(2, -1)) + elif model_type.lower() == "gru": + model.add(Recurrent() + .add(GRU(embedding_dim, 128))) + model.add(Select(2, -1)) + else: + raise ValueError('model can only be cnn, lstm, or gru') + + model.add(Linear(128, 100)) + model.add(Linear(100, class_num)) + model.add(LogSoftMax()) + return model + + +def train(sc, + batch_size, + sequence_len, max_words, embedding_dim, training_split): + print('Processing text dataset') + texts = news20.get_news20() + data_rdd = sc.parallelize(texts, 2) + + word_to_ic = analyze_texts(data_rdd) + + # Only take the top wc between [10, sequence_len] + word_to_ic = dict(word_to_ic[10: max_words]) + bword_to_ic = sc.broadcast(word_to_ic) + + w2v = news20.get_glove_w2v(dim=embedding_dim) + filtered_w2v = {w: v for w, v in w2v.items() if w in word_to_ic} + bfiltered_w2v = sc.broadcast(filtered_w2v) + + tokens_rdd = data_rdd.map(lambda (text, label): + ([w for w in text_to_words(text) if + w in bword_to_ic.value], label)) + padded_tokens_rdd = tokens_rdd.map( + lambda (tokens, label): (pad(tokens, "##", sequence_len), label)) + vector_rdd = padded_tokens_rdd.map(lambda (tokens, label): + ([to_vec(w, bfiltered_w2v.value, + embedding_dim) for w in + tokens], label)) + sample_rdd = vector_rdd.map( + lambda (vectors, label): to_sample(vectors, label, embedding_dim)) + + train_rdd, val_rdd = sample_rdd.randomSplit( + [training_split, 1-training_split]) + + state = {"batchSize": batch_size, + "learningRate": 0.01, + "learningRateDecay": 0.0002} + + optimizer = Optimizer( + model=build_model(news20.CLASS_NUM), + training_rdd=train_rdd, + criterion=ClassNLLCriterion(), + end_trigger=MaxEpoch(max_epoch), + batch_size=batch_size, + optim_method="Adagrad", + state=state) + + optimizer.setvalidation( + batch_size=batch_size, + val_rdd=val_rdd, + trigger=EveryEpoch(), + val_method=["Top1Accuracy"] + ) + train_model = optimizer.optimize() + +if __name__ == "__main__": + parser = OptionParser() + parser.add_option("-a", "--action", dest="action", default="train") + parser.add_option("-b", "--batchSize", dest="batchSize", default="128") + parser.add_option("-e", "--embedding_dim", dest="embedding_dim", default="50") # noqa + parser.add_option("-m", "--max_epoch", dest="max_epoch", default="15") + parser.add_option("--model", dest="model_type", default="cnn") + + (options, args) = parser.parse_args(sys.argv) + if options.action == "train": + batch_size = int(options.batchSize) + embedding_dim = int(options.embedding_dim) + max_epoch = int(options.max_epoch) + model_type = options.model_type + sequence_len = 50 + max_words = 1000 + training_split = 0.8 + sc = SparkContext(appName="text_classifier", + conf=create_spark_conf()) + init_engine() + train(sc, + batch_size, + sequence_len, max_words, embedding_dim, training_split) + elif options.action == "test": + pass diff --git a/python/dllib/src/bigdl/dllib/nn/__init__.py b/python/dllib/src/bigdl/dllib/nn/__init__.py new file mode 100644 index 00000000000..2151a805423 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/nn/__init__.py @@ -0,0 +1,15 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py new file mode 100644 index 00000000000..48ace84dd7a --- /dev/null +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -0,0 +1,1673 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +import sys +from util.common import callBigDlFunc +from util.common import JavaValue +from util.common import callJavaFunc +from pyspark import SparkContext + +import numpy as np + +if sys.version >= '3': + long = int + unicode = str + +INTMAX = 2147483647 +INTMIN = -2147483648 +DOUBLEMAX = 1.7976931348623157E308 + + +class Model(JavaValue): + + def __init__(self, jvalue, bigdl_type, *args): + self.value = jvalue if jvalue else callBigDlFunc( + bigdl_type, JavaValue.jvm_class_constructor(self), *args) + self.bigdl_type = bigdl_type + + @classmethod + def of(cls, jmodel, bigdl_type="float"): + model = Model(bigdl_type, jmodel) + model.value = jmodel + model.bigdl_type = bigdl_type + return model + + def set_name(self, name): + callJavaFunc(SparkContext.getOrCreate(), self.value.setName, name) + return self + + def name(self): + return callJavaFunc(SparkContext.getOrCreate(), self.value.getName) + + def set_seed(self, seed=123): + callBigDlFunc(self.bigdl_type, "setModelSeed", seed) + return self + + def get_dtype(self): + if "float" == self.bigdl_type: + return "float32" + else: + return "float64" + + def reset(self): + callJavaFunc(SparkContext.getOrCreate(), self.value.reset) + return self + + def parameters(self): + name_to_params = callBigDlFunc(self.bigdl_type, + "modelGetParameters", + self.value) + + def to_ndarray(params): + return { + param_name: np.array(values[0], + dtype=self.get_dtype()).reshape( + values[1]) for param_name, values in params.iteritems()} + + return {layer_name: to_ndarray(params) for layer_name, params in + name_to_params.iteritems()} + + def predict(self, data_rdd): + return callBigDlFunc(self.bigdl_type, + "modelPredictRDD", self.value, data_rdd) + + def test(self, val_rdd, batch_size, val_methods): + return callBigDlFunc(self.bigdl_type, + "modelTest", + self.value, + val_rdd, batch_size, val_methods) + + @staticmethod + def from_path(path, bigdl_type="float"): + jmodel = callBigDlFunc(bigdl_type, "modelFromPath", path) + return Model.of(jmodel, bigdl_type) + + +class Linear(Model): + + ''' + >>> linear = Linear(100, 10, "Xavier") + creating: createLinear + ''' + + def __init__(self, input_size, output_size, init_method="default", + bigdl_type="float"): + super(Linear, self).__init__(None, bigdl_type, input_size, output_size, + init_method) + + +class ReLU(Model): + + ''' + >>> relu = ReLU() + creating: createReLU + ''' + + def __init__(self, bigdl_type="float"): + super(ReLU, self).__init__(None, bigdl_type) + + +class Tanh(Model): + + ''' + >>> tanh = Tanh() + creating: createTanh + ''' + + def __init__(self, bigdl_type="float"): + super(Tanh, self).__init__(None, bigdl_type) + + +class Echo(Model): + + ''' + >>> echo = Echo() + creating: createEcho + ''' + + def __init__(self, bigdl_type="float"): + super(Echo, self).__init__(None, bigdl_type) + + +class LogSoftMax(Model): + + ''' + >>> logSoftMax = LogSoftMax() + creating: createLogSoftMax + ''' + + def __init__(self, bigdl_type="float"): + super(LogSoftMax, self).__init__(None, bigdl_type) + + +class Sequential(Model): + + ''' + >>> echo = Echo() + creating: createEcho + >>> s = Sequential() + creating: createSequential + >>> s = s.add(echo) + >>> s = s.add(s) + >>> s = s.add(echo) + + ''' + + def __init__(self, bigdl_type="float"): + super(Sequential, self).__init__(None, bigdl_type) + + def add(self, model): + self.value.add(model.value) + return self + + +class SpatialConvolution(Model): + + ''' + >>> spatialConvolution = SpatialConvolution(6, 12, 5, 5) + creating: createSpatialConvolution + ''' + + def __init__(self, + n_input_plane, + n_output_plane, + kernel_w, + kernel_h, + stride_w=1, + stride_h=1, + pad_w=0, + pad_h=0, + n_group=1, + propagate_back=True, + init_method="Default", + bigdl_type="float"): + super(SpatialConvolution, self).__init__(None, bigdl_type, + n_input_plane, + n_output_plane, + kernel_w, + kernel_h, + stride_w, + stride_h, + pad_w, + pad_h, + n_group, + propagate_back, + init_method) + + +class SpatialMaxPooling(Model): + + ''' + >>> spatialMaxPooling = SpatialMaxPooling(2, 2, 2, 2) + creating: createSpatialMaxPooling + ''' + # to_ceil: call floor() when False; call ceil() when True + + def __init__(self, kw, + kh, + dw, + dh, + pad_w=0, + pad_h=0, + to_ceil=False, + bigdl_type="float"): + super(SpatialMaxPooling, self).__init__(None, bigdl_type, kw, + kh, + dw, + dh, + pad_w, + pad_h, + to_ceil) + + +class Select(Model): + ''' + >>> select = Select(1, 1) + creating: createSelect + ''' + + def __init__(self, dim, index, bigdl_type="float"): + super(Select, self).__init__(None, bigdl_type, dim, index) + + +class Recurrent(Model): + ''' + >>> recurrent = Recurrent() + creating: createRecurrent + ''' + + def __init__(self, bigdl_type="float"): + super(Recurrent, self).__init__(None, bigdl_type) + + def add(self, model): + self.value.add(model.value) + return self + + +class LSTM(Model): + ''' + >>> lstm = LSTM(4, 3) + creating: createLSTM + ''' + + def __init__(self, input_size, hidden_size, bigdl_type="float"): + super(LSTM, self).__init__(None, bigdl_type, input_size, hidden_size) + + +class GRU(Model): + ''' + >>> gru = GRU(4, 3) + creating: createGRU + ''' + + def __init__(self, input_size, hidden_size, bigdl_type="float"): + super(GRU, self).__init__(None, bigdl_type, input_size, hidden_size) + + +class RNNCell(Model): + ''' + >>> reshape = RNNCell(4, 3, Tanh()) + creating: createTanh + creating: createRNNCell + ''' + + def __init__(self, + input_size, + hidden_size, + activation, + bigdl_type="float"): + super(RNNCell, self).__init__(None, bigdl_type, input_size, hidden_size, activation) + + +class TimeDistributed(Model): + ''' + >>> td = TimeDistributed(Linear(2, 3)) + creating: createLinear + creating: createTimeDistributed + ''' + + def __init__(self, model, bigdl_type="float"): + super(TimeDistributed, self).__init__(None, bigdl_type, model) + + +class Concat(Model): + + ''' + >>> concat = Concat(2) + creating: createConcat + ''' + + def __init__(self, + dimension, + bigdl_type="float"): + super(Concat, self).__init__(None, bigdl_type, + dimension) + + +class SpatialAveragePooling(Model): + + ''' + >>> spatialAveragePooling = SpatialAveragePooling(7,7) + creating: createSpatialAveragePooling + ''' + + def __init__(self, + kw, + kh, + dw=1, + dh=1, + pad_w=0, + pad_h=0, + ceil_mode=False, + count_include_pad=True, + divide=True, + bigdl_type="float"): + super(SpatialAveragePooling, self).__init__(None, bigdl_type, + kw, + kh, + dw, + dh, + pad_w, + pad_h, + ceil_mode, + count_include_pad, + divide) + + +class SpatialBatchNormalization(Model): + + ''' + >>> spatialBatchNormalization = SpatialBatchNormalization(1) + creating: createSpatialBatchNormalization + ''' + + def __init__(self, + n_output, + eps=1e-5, + momentum=0.1, + affine=True, + bigdl_type="float"): + super(SpatialBatchNormalization, self).__init__(None, bigdl_type, + n_output, + eps, + momentum, + affine) + + +class SpatialCrossMapLRN(Model): + + ''' + >>> spatialCrossMapLRN = SpatialCrossMapLRN() + creating: createSpatialCrossMapLRN + ''' + + def __init__(self, + size=5, + alpha=1.0, + beta=0.75, + k=1.0, + bigdl_type="float"): + super(SpatialCrossMapLRN, self).__init__(None, bigdl_type, + size, + alpha, + beta, + k) + + +class Dropout(Model): + + ''' + >>> dropout = Dropout(0.4) + creating: createDropout + ''' + + def __init__(self, + init_p=0.5, + inplace=False, + scale=True, + bigdl_type="float"): + super(Dropout, self).__init__(None, bigdl_type, + init_p, + inplace, + scale) + + +class View(Model): + + ''' + >>> view = View([1024,2]) + creating: createView + ''' + + def __init__(self, + sizes, + num_input_dims=0, + bigdl_type="float"): + super(View, self).__init__(None, bigdl_type, + sizes, + num_input_dims) + + +class Abs(Model): + + ''' + >>> abs = Abs() + creating: createAbs + ''' + + def __init__(self, + bigdl_type="float"): + super(Abs, self).__init__(None, bigdl_type) + + +class Add(Model): + + ''' + >>> add = Add(1) + creating: createAdd + ''' + + def __init__(self, + input_size, + bigdl_type="float"): + super(Add, self).__init__(None, bigdl_type, + input_size) + + +class AddConstant(Model): + + ''' + >>> addConstant = AddConstant(1e-5, True) + creating: createAddConstant + ''' + + def __init__(self, + constant_scalar, + inplace=False, + bigdl_type="float"): + super(AddConstant, self).__init__(None, bigdl_type, + constant_scalar, + inplace) + + +class BatchNormalization(Model): + + ''' + >>> batchNormalization = BatchNormalization(1, 1e-5, 1e-5, True) + creating: createBatchNormalization + ''' + + def __init__(self, + n_output, + eps=1e-5, + momentum=0.1, + affine=True, + bigdl_type="float"): + super(BatchNormalization, self).__init__(None, bigdl_type, + n_output, + eps, + momentum, + affine) + + +class Bilinear(Model): + + ''' + >>> bilinear = Bilinear(1, 1, 1, True) + creating: createBilinear + ''' + + def __init__(self, + input_size1, + input_size2, + output_size, + bias_res=True, + bigdl_type="float"): + super(Bilinear, self).__init__(None, bigdl_type, + input_size1, + input_size2, + output_size, + bias_res) + + +class Bottle(Model): + + ''' + >>> bottle = Bottle(Linear(100,10), 1, 1) + creating: createLinear + creating: createBottle + ''' + + def __init__(self, + module, + n_input_dim=2, + n_output_dim1=INTMAX, + bigdl_type="float"): + super(Bottle, self).__init__(None, bigdl_type, + module, + n_input_dim, + n_output_dim1) + + +class CAdd(Model): + + ''' + >>> cAdd = CAdd([1,2]) + creating: createCAdd + ''' + + def __init__(self, + size, + bigdl_type="float"): + super(CAdd, self).__init__(None, bigdl_type, + size) + + +class CAddTable(Model): + + ''' + >>> cAddTable = CAddTable(True) + creating: createCAddTable + ''' + + def __init__(self, + inplace=False, + bigdl_type="float"): + super(CAddTable, self).__init__(None, bigdl_type, + inplace) + + +class CDivTable(Model): + + ''' + >>> cDivTable = CDivTable() + creating: createCDivTable + ''' + + def __init__(self, + bigdl_type="float"): + super(CDivTable, self).__init__(None, bigdl_type) + + +class CMaxTable(Model): + + ''' + >>> cMaxTable = CMaxTable() + creating: createCMaxTable + ''' + + def __init__(self, + bigdl_type="float"): + super(CMaxTable, self).__init__(None, bigdl_type) + + +class CMinTable(Model): + + ''' + >>> cMinTable = CMinTable() + creating: createCMinTable + ''' + + def __init__(self, + bigdl_type="float"): + super(CMinTable, self).__init__(None, bigdl_type) + + +class CMul(Model): + + ''' + >>> cMul = CMul([1,2]) + creating: createCMul + ''' + + def __init__(self, + size, + bigdl_type="float"): + super(CMul, self).__init__(None, bigdl_type, + size) + + +class CMulTable(Model): + + ''' + >>> cMulTable = CMulTable() + creating: createCMulTable + ''' + + def __init__(self, + bigdl_type="float"): + super(CMulTable, self).__init__(None, bigdl_type) + + +class CSubTable(Model): + + ''' + >>> cSubTable = CSubTable() + creating: createCSubTable + ''' + + def __init__(self, + bigdl_type="float"): + super(CSubTable, self).__init__(None, bigdl_type) + + +class Clamp(Model): + + ''' + >>> clamp = Clamp(1, 3) + creating: createClamp + ''' + + def __init__(self, + min, + max, + bigdl_type="float"): + super(Clamp, self).__init__(None, bigdl_type, + min, + max) + + +class Contiguous(Model): + + ''' + >>> contiguous = Contiguous() + creating: createContiguous + ''' + + def __init__(self, + bigdl_type="float"): + super(Contiguous, self).__init__(None, bigdl_type) + + +class Copy(Model): + + ''' + >>> copy = Copy() + creating: createCopy + ''' + + def __init__(self, + bigdl_type="float"): + super(Copy, self).__init__(None, bigdl_type) + + +class Cosine(Model): + + ''' + >>> cosine = Cosine(2,3) + creating: createCosine + ''' + + def __init__(self, + input_size, + output_size, + bigdl_type="float"): + super(Cosine, self).__init__(None, bigdl_type, + input_size, + output_size) + + +class CosineDistance(Model): + + ''' + >>> cosineDistance = CosineDistance() + creating: createCosineDistance + ''' + + def __init__(self, + bigdl_type="float"): + super(CosineDistance, self).__init__(None, bigdl_type) + + +class DotProduct(Model): + + ''' + >>> dotProduct = DotProduct() + creating: createDotProduct + ''' + + def __init__(self, + bigdl_type="float"): + super(DotProduct, self).__init__(None, bigdl_type) + + +class ELU(Model): + + ''' + >>> eLU = ELU(1e-5, True) + creating: createELU + ''' + + def __init__(self, + alpha=1.0, + inplace=False, + bigdl_type="float"): + super(ELU, self).__init__(None, bigdl_type, + alpha, + inplace) + + +class Euclidean(Model): + + ''' + >>> euclidean = Euclidean(1, 1, True) + creating: createEuclidean + ''' + + def __init__(self, + input_size, + output_size, + fast_backward=True, + bigdl_type="float"): + super(Euclidean, self).__init__(None, bigdl_type, + input_size, + output_size, + fast_backward) + + +class Exp(Model): + + ''' + >>> exp = Exp() + creating: createExp + ''' + + def __init__(self, + bigdl_type="float"): + super(Exp, self).__init__(None, bigdl_type) + + +class FlattenTable(Model): + + ''' + >>> flattenTable = FlattenTable() + creating: createFlattenTable + ''' + + def __init__(self, + bigdl_type="float"): + super(FlattenTable, self).__init__(None, bigdl_type) + + +class GradientReversal(Model): + + ''' + >>> gradientReversal = GradientReversal(1e-5) + creating: createGradientReversal + ''' + + def __init__(self, + the_lambda=1, + bigdl_type="float"): + super(GradientReversal, self).__init__(None, bigdl_type, + the_lambda) + + +class HardShrink(Model): + + ''' + >>> hardShrink = HardShrink(1e-5) + creating: createHardShrink + ''' + + def __init__(self, + the_lambda=0.5, + bigdl_type="float"): + super(HardShrink, self).__init__(None, bigdl_type, + the_lambda) + + +class HardTanh(Model): + + ''' + >>> hardTanh = HardTanh(1e-5, 1e5, True) + creating: createHardTanh + ''' + + def __init__(self, + min_value=-1, + max_value=1, + inplace=False, + bigdl_type="float"): + super(HardTanh, self).__init__(None, bigdl_type, + min_value, + max_value, + inplace) + + +class Index(Model): + + ''' + >>> index = Index(1) + creating: createIndex + ''' + + def __init__(self, + dimension, + bigdl_type="float"): + super(Index, self).__init__(None, bigdl_type, + dimension) + + +class InferReshape(Model): + + ''' + >>> inferReshape = InferReshape([4, 0, 3, -1], False) + creating: createInferReshape + ''' + + def __init__(self, + size, + batch_mode=False, + bigdl_type="float"): + super(InferReshape, self).__init__(None, bigdl_type, + size, + batch_mode) + + +class JoinTable(Model): + + ''' + >>> joinTable = JoinTable(1, 1) + creating: createJoinTable + ''' + + def __init__(self, + dimension, + n_input_dims, + bigdl_type="float"): + super(JoinTable, self).__init__(None, bigdl_type, + dimension, + n_input_dims) + + +class L1Cost(Model): + + ''' + >>> l1Cost = L1Cost() + creating: createL1Cost + ''' + + def __init__(self, + bigdl_type="float"): + super(L1Cost, self).__init__(None, bigdl_type) + + +class L1Penalty(Model): + + ''' + >>> l1Penalty = L1Penalty(1, True, True) + creating: createL1Penalty + ''' + + def __init__(self, + l1weight, + size_average=False, + provide_output=True, + bigdl_type="float"): + super(L1Penalty, self).__init__(None, bigdl_type, + l1weight, + size_average, + provide_output) + + +class LeakyReLU(Model): + + ''' + >>> leakyReLU = LeakyReLU(1e-5, True) + creating: createLeakyReLU + ''' + + def __init__(self, + negval=0.01, + inplace=False, + bigdl_type="float"): + super(LeakyReLU, self).__init__(None, bigdl_type, + negval, + inplace) + + +class Log(Model): + + ''' + >>> log = Log() + creating: createLog + ''' + + def __init__(self, + bigdl_type="float"): + super(Log, self).__init__(None, bigdl_type) + + +class LogSigmoid(Model): + + ''' + >>> logSigmoid = LogSigmoid() + creating: createLogSigmoid + ''' + + def __init__(self, + bigdl_type="float"): + super(LogSigmoid, self).__init__(None, bigdl_type) + + +class LookupTable(Model): + + ''' + >>> lookupTable = LookupTable(1, 1, 1e-5, 1e-5, 1e-5, True) + creating: createLookupTable + ''' + + def __init__(self, + n_index, + n_output, + padding_value=0, + max_norm=DOUBLEMAX, + norm_type=2.0, + should_scale_grad_by_freq=False, + bigdl_type="float"): + super(LookupTable, self).__init__(None, bigdl_type, + n_index, + n_output, + padding_value, + max_norm, + norm_type, + should_scale_grad_by_freq) + + +class MM(Model): + + ''' + >>> mM = MM(True, True) + creating: createMM + ''' + + def __init__(self, + trans_a=False, + trans_b=False, + bigdl_type="float"): + super(MM, self).__init__(None, bigdl_type, + trans_a, + trans_b) + + +class MV(Model): + + ''' + >>> mV = MV(True) + creating: createMV + ''' + + def __init__(self, + trans=False, + bigdl_type="float"): + super(MV, self).__init__(None, bigdl_type, + trans) + + +class MapTable(Model): + + ''' + >>> mapTable = MapTable(Linear(100,10)) + creating: createLinear + creating: createMapTable + ''' + + def __init__(self, + module, + bigdl_type="float"): + super(MapTable, self).__init__(None, bigdl_type, + module) + + +class MaskedSelect(Model): + + ''' + >>> maskedSelect = MaskedSelect() + creating: createMaskedSelect + ''' + + def __init__(self, + bigdl_type="float"): + super(MaskedSelect, self).__init__(None, bigdl_type) + + +class Max(Model): + + ''' + >>> max = Max(1) + creating: createMax + ''' + + def __init__(self, + dim, + num_input_dims=INTMIN, + bigdl_type="float"): + super(Max, self).__init__(None, bigdl_type, + dim, + num_input_dims) + + +class Mean(Model): + + ''' + >>> mean = Mean(1, 1) + creating: createMean + ''' + + def __init__(self, + dimension=1, + n_input_dims=-1, + bigdl_type="float"): + super(Mean, self).__init__(None, bigdl_type, + dimension, + n_input_dims) + + +class Min(Model): + + ''' + >>> min = Min(1) + creating: createMin + ''' + + def __init__(self, + dim, + num_input_dims=INTMIN, + bigdl_type="float"): + super(Min, self).__init__(None, bigdl_type, + dim, + num_input_dims) + + +class MixtureTable(Model): + + ''' + >>> mixtureTable = MixtureTable() + creating: createMixtureTable + ''' + + def __init__(self, + bigdl_type="float"): + super(MixtureTable, self).__init__(None, bigdl_type) + + +class Mul(Model): + + ''' + >>> mul = Mul() + creating: createMul + ''' + + def __init__(self, + bigdl_type="float"): + super(Mul, self).__init__(None, bigdl_type) + + +class MulConstant(Model): + + ''' + >>> mulConstant = MulConstant(2.5) + creating: createMulConstant + ''' + + def __init__(self, + scalar, + inplace=False, + bigdl_type="float"): + super(MulConstant, self).__init__(None, bigdl_type, + scalar, + inplace) + + +class Narrow(Model): + + ''' + >>> narrow = Narrow(1, 1, 1) + creating: createNarrow + ''' + + def __init__(self, + dimension, + offset, + length=1, + bigdl_type="float"): + super(Narrow, self).__init__(None, bigdl_type, + dimension, + offset, + length) + + +class NarrowTable(Model): + + ''' + >>> narrowTable = NarrowTable(1, 1) + creating: createNarrowTable + ''' + + def __init__(self, + offset, + length=1, + bigdl_type="float"): + super(NarrowTable, self).__init__(None, bigdl_type, + offset, + length) + + +class Normalize(Model): + + ''' + >>> normalize = Normalize(1e-5, 1e-5) + creating: createNormalize + ''' + + def __init__(self, + p, + eps=1e-10, + bigdl_type="float"): + super(Normalize, self).__init__(None, bigdl_type, + p, + eps) + + +class PReLU(Model): + + ''' + >>> pReLU = PReLU(1) + creating: createPReLU + ''' + + def __init__(self, + n_output_plane=0, + bigdl_type="float"): + super(PReLU, self).__init__(None, bigdl_type, + n_output_plane) + + +class Padding(Model): + + ''' + >>> padding = Padding(1, 1, 1, 1e-5, 1) + creating: createPadding + ''' + + def __init__(self, + dim, + pad, + n_input_dim, + value=0.0, + n_index=1, + bigdl_type="float"): + super(Padding, self).__init__(None, bigdl_type, + dim, + pad, + n_input_dim, + value, + n_index) + + +class PairwiseDistance(Model): + + ''' + >>> pairwiseDistance = PairwiseDistance(2) + creating: createPairwiseDistance + ''' + + def __init__(self, + norm=2, + bigdl_type="float"): + super(PairwiseDistance, self).__init__(None, bigdl_type, + norm) + + +class ParallelTable(Model): + + ''' + >>> parallelTable = ParallelTable() + creating: createParallelTable + ''' + + def __init__(self, + bigdl_type="float"): + super(ParallelTable, self).__init__(None, bigdl_type) + + +class Power(Model): + + ''' + >>> power = Power(1e-5) + creating: createPower + ''' + + def __init__(self, + power, + scale=1.0, + shift=0.0, + bigdl_type="float"): + super(Power, self).__init__(None, bigdl_type, + power, + scale, + shift) + + +class RReLU(Model): + + ''' + >>> rReLU = RReLU(1e-5, 1e5, True) + creating: createRReLU + ''' + + def __init__(self, + lower=1.0/8, + upper=1.0/3, + inplace=False, + bigdl_type="float"): + super(RReLU, self).__init__(None, bigdl_type, + lower, + upper, + inplace) + + +class ReLU6(Model): + + ''' + >>> reLU6 = ReLU6(True) + creating: createReLU6 + ''' + + def __init__(self, + inplace=False, + bigdl_type="float"): + super(ReLU6, self).__init__(None, bigdl_type, + inplace) + + +class Replicate(Model): + + ''' + >>> replicate = Replicate(2) + creating: createReplicate + ''' + + def __init__(self, + n_features, + dim=1, + n_dim=INTMAX, + bigdl_type="float"): + super(Replicate, self).__init__(None, bigdl_type, + n_features, + dim, + n_dim) + + +class RoiPooling(Model): + + ''' + >>> roiPooling = RoiPooling(1, 1, 1e-5) + creating: createRoiPooling + ''' + + def __init__(self, + pooled_w, + pooled_h, + spatial_scale, + bigdl_type="float"): + super(RoiPooling, self).__init__(None, bigdl_type, + pooled_w, + pooled_h, + spatial_scale) + + +class Scale(Model): + + ''' + >>> scale = Scale([1,2]) + creating: createScale + ''' + + def __init__(self, + size, + bigdl_type="float"): + super(Scale, self).__init__(None, bigdl_type, + size) + + +class SelectTable(Model): + + ''' + >>> selectTable = SelectTable(1) + creating: createSelectTable + ''' + + def __init__(self, + dimension, + bigdl_type="float"): + super(SelectTable, self).__init__(None, bigdl_type, + dimension) + + +class Sigmoid(Model): + + ''' + >>> sigmoid = Sigmoid() + creating: createSigmoid + ''' + + def __init__(self, + bigdl_type="float"): + super(Sigmoid, self).__init__(None, bigdl_type) + + +class SoftMax(Model): + + ''' + >>> softMax = SoftMax() + creating: createSoftMax + ''' + + def __init__(self, + bigdl_type="float"): + super(SoftMax, self).__init__(None, bigdl_type) + + +class SoftMin(Model): + + ''' + >>> softMin = SoftMin() + creating: createSoftMin + ''' + + def __init__(self, + bigdl_type="float"): + super(SoftMin, self).__init__(None, bigdl_type) + + +class SoftPlus(Model): + + ''' + >>> softPlus = SoftPlus(1e-5) + creating: createSoftPlus + ''' + + def __init__(self, + beta=1.0, + bigdl_type="float"): + super(SoftPlus, self).__init__(None, bigdl_type, + beta) + + +class SoftShrink(Model): + + ''' + >>> softShrink = SoftShrink(1e-5) + creating: createSoftShrink + ''' + + def __init__(self, + the_lambda=0.5, + bigdl_type="float"): + super(SoftShrink, self).__init__(None, bigdl_type, + the_lambda) + + +class SoftSign(Model): + + ''' + >>> softSign = SoftSign() + creating: createSoftSign + ''' + + def __init__(self, + bigdl_type="float"): + super(SoftSign, self).__init__(None, bigdl_type) + + +class SpatialDilatedConvolution(Model): + + ''' + >>> spatialDilatedConvolution = SpatialDilatedConvolution(1, 1, 1, 1) + creating: createSpatialDilatedConvolution + ''' + + def __init__(self, + n_input_plane, + n_output_plane, + kw, + kh, + dw=1, + dh=1, + pad_w=0, + pad_h=0, + dilation_w=1, + dilation_h=1, + init_method='default', + bigdl_type="float"): + super(SpatialDilatedConvolution, self).__init__(None, bigdl_type, + n_input_plane, + n_output_plane, + kw, + kh, + dw, + dh, + pad_w, + pad_h, + dilation_w, + dilation_h, + init_method) + + +class SpatialFullConvolution(Model): + + ''' + >>> spatialFullConvolution = SpatialFullConvolution(1, 1, 1, 1) + creating: createSpatialFullConvolution + ''' + + def __init__(self, + n_input_plane, + n_output_plane, + kw, + kh, + dw=1, + dh=1, + pad_w=0, + pad_h=0, + adj_w=0, + adj_h=0, + n_group=1, + no_bias=False, + init_method='default', + bigdl_type="float"): + super(SpatialFullConvolution, self).__init__(None, bigdl_type, + n_input_plane, + n_output_plane, + kw, + kh, + dw, + dh, + pad_w, + pad_h, + adj_w, + adj_h, + n_group, + no_bias, + init_method) + + +class SpatialShareConvolution(Model): + + ''' + >>> spatialShareConvolution = SpatialShareConvolution(1, 1, 1, 1) + creating: createSpatialShareConvolution + ''' + + def __init__(self, + n_input_plane, + n_output_plane, + kernel_w, + kernel_h, + stride_w=1, + stride_h=1, + pad_w=0, + pad_h=0, + n_group=1, + propagate_back=True, + init_method='default', + bigdl_type="float"): + super(SpatialShareConvolution, self).__init__(None, bigdl_type, + n_input_plane, + n_output_plane, + kernel_w, + kernel_h, + stride_w, + stride_h, + pad_w, + pad_h, + n_group, + propagate_back, + init_method) + + +class SpatialZeroPadding(Model): + + ''' + >>> spatialZeroPadding = SpatialZeroPadding(1, 1, 1, 1) + creating: createSpatialZeroPadding + ''' + + def __init__(self, + pad_left, + pad_right, + pad_top, + pad_bottom, + bigdl_type="float"): + super(SpatialZeroPadding, self).__init__(None, bigdl_type, + pad_left, + pad_right, + pad_top, + pad_bottom) + + +class SplitTable(Model): + + ''' + >>> splitTable = SplitTable(1, 1) + creating: createSplitTable + ''' + + def __init__(self, + dimension, + n_input_dims=-1, + bigdl_type="float"): + super(SplitTable, self).__init__(None, bigdl_type, + dimension, + n_input_dims) + + +class Sqrt(Model): + + ''' + >>> sqrt = Sqrt() + creating: createSqrt + ''' + + def __init__(self, + bigdl_type="float"): + super(Sqrt, self).__init__(None, bigdl_type) + + +class Square(Model): + + ''' + >>> square = Square() + creating: createSquare + ''' + + def __init__(self, + bigdl_type="float"): + super(Square, self).__init__(None, bigdl_type) + + +class Squeeze(Model): + + ''' + >>> squeeze = Squeeze(1) + creating: createSqueeze + ''' + + def __init__(self, + dim, + num_input_dims=INTMIN, + bigdl_type="float"): + super(Squeeze, self).__init__(None, bigdl_type, + dim, + num_input_dims) + + +class Sum(Model): + + ''' + >>> sum = Sum(1, 1, True) + creating: createSum + ''' + + def __init__(self, + dimension=1, + n_input_dims=-1, + size_average=False, + bigdl_type="float"): + super(Sum, self).__init__(None, bigdl_type, + dimension, + n_input_dims, + size_average) + + +class TanhShrink(Model): + + ''' + >>> tanhShrink = TanhShrink() + creating: createTanhShrink + ''' + + def __init__(self, + bigdl_type="float"): + super(TanhShrink, self).__init__(None, bigdl_type) + + +class Threshold(Model): + + ''' + >>> threshold = Threshold(1e-5, 1e-5, True) + creating: createThreshold + ''' + + def __init__(self, + th=1e-6, + v=0.0, + ip=False, + bigdl_type="float"): + super(Threshold, self).__init__(None, bigdl_type, + th, + v, + ip) + + +class Unsqueeze(Model): + + ''' + >>> unsqueeze = Unsqueeze(1, 1) + creating: createUnsqueeze + ''' + + def __init__(self, + pos, + num_input_dims=INTMIN, + bigdl_type="float"): + super(Unsqueeze, self).__init__(None, bigdl_type, + pos, + num_input_dims) + + +class Reshape(Model): + ''' + >>> reshape = Reshape([1, 28, 28]) + creating: createReshape + ''' + + def __init__(self, size, bigdl_type="float"): + super(Reshape, self).__init__(None, bigdl_type, size) + + +def _test(): + import doctest + from pyspark import SparkContext + from nn import layer + from util.common import init_engine + from util.common import create_spark_conf + globs = layer.__dict__.copy() + sc = SparkContext(master="local[4]", appName="test layer", + conf=create_spark_conf()) + globs['sc'] = sc + init_engine() + + (failure_count, test_count) = doctest.testmod(globs=globs, + optionflags=doctest.ELLIPSIS) + if failure_count: + exit(-1) + +if __name__ == "__main__": + _test() diff --git a/python/dllib/src/bigdl/dllib/optim/__init__.py b/python/dllib/src/bigdl/dllib/optim/__init__.py new file mode 100644 index 00000000000..2151a805423 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/optim/__init__.py @@ -0,0 +1,15 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/python/dllib/src/bigdl/dllib/optim/optimizer.py b/python/dllib/src/bigdl/dllib/optim/optimizer.py new file mode 100644 index 00000000000..4272e54f696 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/optim/optimizer.py @@ -0,0 +1,169 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +from util.common import callBigDlFunc +from util.common import JavaValue +from util.common import callJavaFunc +from pyspark import SparkContext +import numpy as np +import os +from distutils.dir_util import mkpath + + +import sys +if sys.version >= '3': + long = int + unicode = str + + +class MaxIteration(JavaValue): + ''' + >>> maxIteration = MaxIteration(20) + creating: createMaxIteration + ''' + def __init__(self, max, bigdl_type="float"): + JavaValue.__init__(self, None, bigdl_type, max) + + +class MaxEpoch(JavaValue): + ''' + >>> maxEpoch = MaxEpoch(2) + creating: createMaxEpoch + ''' + def __init__(self, max_epoch, bigdl_type="float"): + JavaValue.__init__(self, None, bigdl_type, max_epoch) + + +class EveryEpoch(JavaValue): + ''' + >>> everyEpoch = EveryEpoch() + creating: createEveryEpoch + ''' + def __init__(self, bigdl_type="float"): + JavaValue.__init__(self, None, bigdl_type) + + +class SeveralIteration(JavaValue): + ''' + >>> serveralIteration = SeveralIteration(2) + creating: createSeveralIteration + ''' + def __init__(self, interval, bigdl_type="float"): + JavaValue.__init__(self, None, bigdl_type, interval) + + +class Poly(JavaValue): + ''' + >>> poly = Poly(0.5, 2) + creating: createPoly + ''' + def __init__(self, power, max_iteration, bigdl_type="float"): + JavaValue.__init__(self, None, bigdl_type, power, max_iteration) + + +class Step(JavaValue): + ''' + >>> step = Step(2, 0.3) + creating: createStep + ''' + def __init__(self, step_size, gamma, bigdl_type="float"): + JavaValue.__init__(self, None, bigdl_type, step_size, gamma) + + +class Optimizer(JavaValue): + + def __init__(self, + model, + training_rdd, + criterion, + end_trigger, + batch_size, + optim_method="SGD", + state={}, + bigdl_type="float"): + JavaValue.__init__(self, None, bigdl_type, model.value, + training_rdd, criterion, optim_method, + state, end_trigger, batch_size) + + def setvalidation(self, batch_size, val_rdd, trigger, val_method=["Top1Accuracy"]): + callBigDlFunc(self.bigdl_type, "setValidation", self.value, batch_size, + trigger, val_rdd, val_method) + + def setcheckpoint(self, checkpoint_trigger, + checkpoint_path, isOverWrite=True): + if not os.path.exists(checkpoint_path): + mkpath(checkpoint_path) + callBigDlFunc(self.bigdl_type, "setCheckPoint", self.value, + checkpoint_trigger, checkpoint_path, isOverWrite) + + # return a module + def optimize(self): + jmodel = callJavaFunc(SparkContext.getOrCreate(), self.value.optimize) + from nn.layer import Model + return Model.of(jmodel) + + def set_train_summary(self, summary): + callBigDlFunc(self.bigdl_type, "setTrainSummary", self.value, + summary) + return self + + def set_val_summary(self, summary): + callBigDlFunc(self.bigdl_type, "setValSummary", self.value, + summary) + return self + + +class TrainSummary(JavaValue, ): + def __init__(self, log_dir, app_name, bigdl_type="float"): + JavaValue.__init__(self, None, bigdl_type, log_dir, app_name) + + def read_scalar(self, tag): + return callBigDlFunc(self.bigdl_type, "summaryReadScalar", self.value, + tag) + + def set_summary_trigger(self, name, trigger): + return callBigDlFunc(self.bigdl_type, "summarySetTrigger", self.value, + name, trigger) + + +class ValidationSummary(JavaValue): + def __init__(self, log_dir, app_name, bigdl_type="float"): + JavaValue.__init__(self, None, bigdl_type, log_dir, app_name) + + def read_scalar(self, tag): + return callBigDlFunc(self.bigdl_type, "summaryReadScalar", self.value, + tag) + + +def _test(): + import doctest + from pyspark import SparkContext + from optim import optimizer + from util.common import init_engine + from util.common import create_spark_conf + globs = optimizer.__dict__.copy() + sc = SparkContext(master="local[4]", appName="test optimizer", + conf=create_spark_conf()) + init_engine() + globs['sc'] = sc + (failure_count, test_count) = doctest.testmod(globs=globs, + optionflags=doctest.ELLIPSIS) + if failure_count: + exit(-1) + +if __name__ == "__main__": + _test() diff --git a/python/dllib/src/bigdl/utils/__init__.py b/python/dllib/src/bigdl/utils/__init__.py new file mode 100644 index 00000000000..2151a805423 --- /dev/null +++ b/python/dllib/src/bigdl/utils/__init__.py @@ -0,0 +1,15 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/python/dllib/src/bigdl/utils/common.py b/python/dllib/src/bigdl/utils/common.py new file mode 100644 index 00000000000..bbac74231b2 --- /dev/null +++ b/python/dllib/src/bigdl/utils/common.py @@ -0,0 +1,261 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import sys +from py4j.protocol import Py4JJavaError +from py4j.java_gateway import JavaObject +from py4j.java_collections import ListConverter, JavaArray, JavaList, JavaMap + +from pyspark import RDD, SparkContext +from pyspark.serializers import PickleSerializer, AutoBatchedSerializer +from pyspark.sql import DataFrame, SQLContext +from pyspark.mllib.common import callJavaFunc +from pyspark import SparkConf +import numpy as np + +if sys.version >= '3': + long = int + unicode = str + + +class JavaValue(object): + def jvm_class_constructor(self): + name = "create" + self.__class__.__name__ + print("creating: " + name) + return name + + def __init__(self, jvalue, bigdl_type, *args): + self.value = jvalue if jvalue else callBigDlFunc( + bigdl_type, self.jvm_class_constructor(), *args) + self.bigdl_type = bigdl_type + + +class TestResult(): + def __init__(self, result, total_num, method): + self.result = result + self.total_num = total_num + self.method = method + + def __reduce__(self): + return (TestResult, (self.result, self.total_num, self.method)) + + def __str__(self): + return "Test result: %s, total_num: %s, method: %s" % ( + self.result, self.total_num, self.method) + + +class Sample(object): + def __init__(self, features, label, features_shape, label_shape, + bigdl_type="float"): + def get_dtype(): + if "float" == bigdl_type: + return "float32" + else: + return "float64" + self.features = np.array(features, dtype=get_dtype()).reshape(features_shape) # noqa + self.label = np.array(label, dtype=get_dtype()).reshape(label_shape) + self.bigdl_type = bigdl_type + + @classmethod + def from_ndarray(cls, features, label, bigdl_type="float"): + return cls( + features=[float(i) for i in features.ravel()], + label=[float(i) for i in label.ravel()], + features_shape=list(features.shape), + label_shape=list(label.shape) if label.shape else [label.size], + bigdl_type=bigdl_type) + + @classmethod + def flatten(cls, a_ndarray): + """ + Utility method to flatten a ndarray + :return (storage, shape) + >>> import numpy as np + >>> from util.common import Sample + >>> np.random.seed(123) + >>> data = np.random.uniform(0, 1, (2, 3)) + >>> (storage, shape) = Sample.flatten(data) + >>> shape + [2, 3] + >>> (storage, shape) = Sample.flatten(np.array(2)) + >>> shape + [1] + """ + storage = [float(i) for i in a_ndarray.ravel()] + shape = list(a_ndarray.shape) if a_ndarray.shape else [a_ndarray.size] + return storage, shape + + def __reduce__(self): + (features_storage, features_shape) = Sample.flatten(self.features) + (label_storage, label_shape) = Sample.flatten(self.label) + return (Sample, ( + features_storage, label_storage, features_shape, label_shape, + self.bigdl_type)) + + def __str__(self): + return "features: %s, label: %s," % (self.features, self.label) + + +_picklable_classes = [ + 'LinkedList', + 'SparseVector', + 'DenseVector', + 'DenseMatrix', + 'Rating', + 'LabeledPoint', + 'Sample', + 'TestResult' +] + + +def init_engine(bigdl_type="float"): + callBigDlFunc(bigdl_type, "initEngine") + + +def get_bigdl_conf(): + bigdl_conf_file = "spark-bigdl.conf" + bigdl_python_wrapper = "python-api.zip" + + def load_conf(conf_str): + return dict(line.split() for line in conf_str.split("\n") if + "#" not in line and line.strip()) + + for p in sys.path: + if bigdl_conf_file in p: + with open(p) as conf_file: + return load_conf(conf_file.read()) + if bigdl_python_wrapper in p: + import zipfile + with zipfile.ZipFile(p, 'r') as zip_conf: + return load_conf(zip_conf.read(bigdl_conf_file)) + raise Exception("Cannot find spark-bigdl.conf.Pls add it to PYTHONPATH.") + + +def create_spark_conf(): + bigdl_conf = get_bigdl_conf() + sparkConf = SparkConf() + sparkConf.setAll(bigdl_conf.items()) + return sparkConf + + +def callBigDlFunc(bigdl_type, name, *args): + """ Call API in PythonBigDL """ + sc = SparkContext.getOrCreate() + if bigdl_type == "float": + api = getattr( + sc._jvm.com.intel.analytics.bigdl.python.api.PythonBigDL.ofFloat(), + name) + elif bigdl_type == "double": + api = getattr( + sc._jvm.com.intel.analytics.bigdl.python.api.PythonBigDL.ofDouble(), + name) + else: + raise Exception("Not supported bigdl_type: %s" % bigdl_type) + return callJavaFunc(sc, api, *args) + + +def _java2py(sc, r, encoding="bytes"): + if isinstance(r, JavaObject): + clsName = r.getClass().getSimpleName() + # convert RDD into JavaRDD + if clsName != 'JavaRDD' and clsName.endswith("RDD"): + r = r.toJavaRDD() + clsName = 'JavaRDD' + + if clsName == 'JavaRDD': + jrdd = sc._jvm.SerDe.javaToPython(r) + return RDD(jrdd, sc) + + if clsName == 'DataFrame': + return DataFrame(r, SQLContext.getOrCreate(sc)) + + if clsName in _picklable_classes: + r = sc._jvm.org.apache.spark.bigdl.api.python.BigDLSerDe.dumps(r) + elif isinstance(r, (JavaArray, JavaList, JavaMap)): + try: + r = sc._jvm.org.apache.spark.bigdl.api.python.BigDLSerDe.dumps( + r) + except Py4JJavaError: + pass # not pickable + + if isinstance(r, (bytearray, bytes)): + r = PickleSerializer().loads(bytes(r), encoding=encoding) + return r + + +def callJavaFunc(sc, func, *args): + """ Call Java Function """ + args = [_py2java(sc, a) for a in args] + result = func(*args) + return _java2py(sc, result) + + +def _to_java_object_rdd(rdd): + """ Return a JavaRDD of Object by unpickling + + It will convert each Python object into Java object by Pyrolite, whenever + the RDD is serialized in batch or not. + """ + rdd = rdd._reserialize(AutoBatchedSerializer(PickleSerializer())) + return \ + rdd.ctx._jvm.org.apache.spark.bigdl.api.python.BigDLSerDe.pythonToJava( + rdd._jrdd, True) + + +def _py2java(sc, obj): + """ Convert Python object into Java """ + if isinstance(obj, RDD): + obj = _to_java_object_rdd(obj) + elif isinstance(obj, DataFrame): + obj = obj._jdf + elif isinstance(obj, SparkContext): + obj = obj._jsc + elif isinstance(obj, (list, tuple)): + obj = ListConverter().convert([_py2java(sc, x) for x in obj], + sc._gateway._gateway_client) + elif isinstance(obj, dict): + result = {} + for (key, value) in obj.iteritems(): + result[key] = _py2java(sc, value) if isinstance(value, JavaValue) else value # noqa + obj = result + + elif isinstance(obj, JavaValue): + obj = obj.value + elif isinstance(obj, JavaObject): + pass + elif isinstance(obj, (int, long, float, bool, bytes, unicode)): + pass + else: + data = bytearray(PickleSerializer().dumps(obj)) + obj = sc._jvm.org.apache.spark.bigdl.api.python.BigDLSerDe.loads(data) + return obj + + +def _test(): + import doctest + from pyspark import SparkContext + from nn import layer + globs = layer.__dict__.copy() + sc = SparkContext(master="local[2]", appName="test common utility") + globs['sc'] = sc + (failure_count, test_count) = doctest.testmod(globs=globs, + optionflags=doctest.ELLIPSIS) + if failure_count: + exit(-1) + + +if __name__ == "__main__": + _test() From 60ccda188e434f25d86250668d107ee66b24befa Mon Sep 17 00:00:00 2001 From: zhangli Date: Tue, 14 Mar 2017 16:37:35 +0800 Subject: [PATCH 003/823] refactor --- python/dllib/test/__init__.py | 15 +++++ simple_integration_test.py | 112 ++++++++++++++++++++++++++++++++++ 2 files changed, 127 insertions(+) create mode 100644 python/dllib/test/__init__.py create mode 100644 simple_integration_test.py diff --git a/python/dllib/test/__init__.py b/python/dllib/test/__init__.py new file mode 100644 index 00000000000..2151a805423 --- /dev/null +++ b/python/dllib/test/__init__.py @@ -0,0 +1,15 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/simple_integration_test.py b/simple_integration_test.py new file mode 100644 index 00000000000..5d6e8913f4b --- /dev/null +++ b/simple_integration_test.py @@ -0,0 +1,112 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Still in experimental stage! + +from nn.layer import * +from nn.criterion import * +from optim.optimizer import * +from util.common import * +import numpy as np +import unittest +import tempfile + + +class TestWorkFlow(unittest.TestCase): + def setUp(self): + sparkConf = create_spark_conf() + self.sc = SparkContext(master="local[4]", appName="test model", + conf=sparkConf) + init_engine() + + def tearDown(self): + self.sc.stop() + + def test_set_seed(self): + l1 = Linear(10, 20, "Xavier").set_name("linear1").set_seed(1234).reset() # noqa + l2 = Linear(10, 20, "Xavier").set_name("linear2").set_seed(1234).reset() # noqa + p1 = l1.parameters() + p2 = l2.parameters() + self.assertTrue((p1["linear1"]["weight"] == p2["linear2"]["weight"]).all()) # noqa + + def test_simple_flow(self): + FEATURES_DIM = 2 + data_len = 100 + batch_size = 32 + epoch_num = 5 + + def gen_rand_sample(): + features = np.random.uniform(0, 1, (FEATURES_DIM)) + label = (2 * features).sum() + 0.4 + return Sample.from_ndarray(features, label) + + trainingData = self.sc.parallelize(range(0, data_len)).map( + lambda i: gen_rand_sample()) + + model = Sequential() + l1 = Linear(FEATURES_DIM, 1, "Xavier").set_name("linear1") + self.assertEqual("linear1", l1.name()) + model.add(l1) + + state = {"learningRate": 0.01, + "learningRateDecay": 0.0002, + "learingRateSchedule": Poly(0.5, int((data_len/batch_size)*epoch_num))} # noqa + optimizer = Optimizer( + model=model, + training_rdd=trainingData, + criterion=MSECriterion(), + optim_method="sgd", + state=state, + end_trigger=MaxEpoch(epoch_num), + batch_size=batch_size) + optimizer.setvalidation( + batch_size=batch_size, + val_rdd=trainingData, + trigger=EveryEpoch(), + val_method=["Top1Accuracy"] + ) + tmp_dir = tempfile.mkdtemp() + optimizer.setcheckpoint(SeveralIteration(1), tmp_dir) + train_summary = TrainSummary(log_dir=tmp_dir, + app_name="run1") + train_summary.set_summary_trigger("LearningRate", SeveralIteration(1)) + val_summary = ValidationSummary(log_dir=tmp_dir, + app_name="run1") + optimizer.set_train_summary(train_summary) + optimizer.set_val_summary(val_summary) + + trained_model = optimizer.optimize() + + lr_result = train_summary.read_scalar("LearningRate") + top1_result = val_summary.read_scalar("Top1Accuracy") + + # TODO: add result validation + parameters = trained_model.parameters() + self.assertIsNotNone(parameters["linear1"]) + print("parameters %s" % parameters["linear1"]) + predict_result = trained_model.predict(trainingData) + p = predict_result.take(2) + print("predict predict: \n") + for i in p: + print(str(i) + "\n") + print(len(p)) + + test_results = trained_model.test(trainingData, 32, ["Top1Accuracy"]) + for test_result in test_results: + print(test_result) + + +if __name__ == "__main__": + unittest.main(failfast=True) From 19ae0cfe31fd8415c4c533796972f14f9485403f Mon Sep 17 00:00:00 2001 From: zhangli Date: Tue, 21 Mar 2017 09:25:06 +0800 Subject: [PATCH 004/823] merge visualization --- python/dllib/src/bigdl/dllib/nn/criterion.py | 276 +++++++++++++++++++ 1 file changed, 276 insertions(+) create mode 100644 python/dllib/src/bigdl/dllib/nn/criterion.py diff --git a/python/dllib/src/bigdl/dllib/nn/criterion.py b/python/dllib/src/bigdl/dllib/nn/criterion.py new file mode 100644 index 00000000000..7e47f7074ba --- /dev/null +++ b/python/dllib/src/bigdl/dllib/nn/criterion.py @@ -0,0 +1,276 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +import sys +from util.common import callBigDlFunc +from util.common import JavaValue +from util.common import callJavaFunc +from pyspark import SparkContext + +import numpy as np + +if sys.version >= '3': + long = int + unicode = str + + +class Criterion(JavaValue): + def __init__(self, jvalue, bigdl_type, *args): + self.value = jvalue if jvalue else callBigDlFunc( + bigdl_type, JavaValue.jvm_class_constructor(self), *args) + self.bigdl_type = bigdl_type + + @classmethod + def of(cls, jcriterion, bigdl_type="float"): + criterion = Criterion(bigdl_type, jcriterion) + criterion.value = jcriterion + criterion.bigdl_type = bigdl_type + return criterion + + +class ClassNLLCriterion(Criterion): + ''' + >>> classNLLCriterion = ClassNLLCriterion() + creating: createClassNLLCriterion + ''' + def __init__(self, bigdl_type="float"): + JavaValue.__init__(self, None, bigdl_type) + + +class MSECriterion(Criterion): + ''' + >>> mSECriterion = MSECriterion() + creating: createMSECriterion + ''' + def __init__(self, bigdl_type="float"): + JavaValue.__init__(self, None, bigdl_type) + + +class AbsCriterion(Criterion): + + ''' + >>> absCriterion = AbsCriterion(True) + creating: createAbsCriterion + ''' + + def __init__(self, + size_average=True, + bigdl_type="float"): + super(AbsCriterion, self).__init__(None, bigdl_type, + size_average) + + +class ClassSimplexCriterion(Criterion): + + ''' + >>> classSimplexCriterion = ClassSimplexCriterion(2) + creating: createClassSimplexCriterion + ''' + + def __init__(self, + n_classes, + bigdl_type="float"): + super(ClassSimplexCriterion, self).__init__(None, bigdl_type, + n_classes) + + +class CosineEmbeddingCriterion(Criterion): + + ''' + >>> cosineEmbeddingCriterion = CosineEmbeddingCriterion(1e-5, True) + creating: createCosineEmbeddingCriterion + ''' + + def __init__(self, + margin=0.0, + size_average=True, + bigdl_type="float"): + super(CosineEmbeddingCriterion, self).__init__(None, bigdl_type, + margin, + size_average) + + +class DistKLDivCriterion(Criterion): + + ''' + >>> distKLDivCriterion = DistKLDivCriterion(True) + creating: createDistKLDivCriterion + ''' + + def __init__(self, + size_average=True, + bigdl_type="float"): + super(DistKLDivCriterion, self).__init__(None, bigdl_type, + size_average) + + +class HingeEmbeddingCriterion(Criterion): + + ''' + >>> hingeEmbeddingCriterion = HingeEmbeddingCriterion(1e-5, True) + creating: createHingeEmbeddingCriterion + ''' + + def __init__(self, + margin=1, + size_average=True, + bigdl_type="float"): + super(HingeEmbeddingCriterion, self).__init__(None, bigdl_type, + margin, + size_average) + + +class L1HingeEmbeddingCriterion(Criterion): + + ''' + >>> l1HingeEmbeddingCriterion = L1HingeEmbeddingCriterion(1e-5) + creating: createL1HingeEmbeddingCriterion + ''' + + def __init__(self, + margin=1, + bigdl_type="float"): + super(L1HingeEmbeddingCriterion, self).__init__(None, bigdl_type, + margin) + + +class MarginCriterion(Criterion): + + ''' + >>> marginCriterion = MarginCriterion(1e-5, True) + creating: createMarginCriterion + ''' + + def __init__(self, + margin=1.0, + size_average=True, + bigdl_type="float"): + super(MarginCriterion, self).__init__(None, bigdl_type, + margin, + size_average) + + +class MarginRankingCriterion(Criterion): + + ''' + >>> marginRankingCriterion = MarginRankingCriterion(1e-5, True) + creating: createMarginRankingCriterion + ''' + + def __init__(self, + margin=1.0, + size_average=True, + bigdl_type="float"): + super(MarginRankingCriterion, self).__init__(None, bigdl_type, + margin, + size_average) + + +class MultiCriterion(Criterion): + + ''' + >>> multiCriterion = MultiCriterion() + creating: createMultiCriterion + ''' + + def __init__(self, + bigdl_type="float"): + super(MultiCriterion, self).__init__(None, bigdl_type) + + +class MultiLabelMarginCriterion(Criterion): + + ''' + >>> multiLabelMarginCriterion = MultiLabelMarginCriterion(True) + creating: createMultiLabelMarginCriterion + ''' + + def __init__(self, + size_average=True, + bigdl_type="float"): + super(MultiLabelMarginCriterion, self).__init__(None, bigdl_type, + size_average) + + +class ParallelCriterion(Criterion): + + ''' + >>> parallelCriterion = ParallelCriterion(True) + creating: createParallelCriterion + ''' + + def __init__(self, + repeat_target=False, + bigdl_type="float"): + super(ParallelCriterion, self).__init__(None, bigdl_type, + repeat_target) + + +class SmoothL1Criterion(Criterion): + + ''' + >>> smoothL1Criterion = SmoothL1Criterion(True) + creating: createSmoothL1Criterion + ''' + + def __init__(self, + size_average=True, + bigdl_type="float"): + super(SmoothL1Criterion, self).__init__(None, bigdl_type, + size_average) + + +class SmoothL1CriterionWithWeights(Criterion): + + ''' + >>> smoothL1CriterionWithWeights = SmoothL1CriterionWithWeights(1e-5, 1) + creating: createSmoothL1CriterionWithWeights + ''' + + def __init__(self, + sigma, + num=0, + bigdl_type="float"): + super(SmoothL1CriterionWithWeights, self).__init__(None, bigdl_type, + sigma, + num) + + +class SoftmaxWithCriterion(Criterion): + + ''' + >>> softmaxWithCriterion = SoftmaxWithCriterion("VALID") + creating: createSoftmaxWithCriterion + ''' + + def __init__(self, + normalize_mode="VALID", + bigdl_type="float"): + super(SoftmaxWithCriterion, self).__init__(None, bigdl_type, + None, + normalize_mode) + + +class TimeDistributedCriterion(JavaValue): + ''' + >>> td = TimeDistributedCriterion(ClassNLLCriterion()) + creating: createClassNLLCriterion + creating: createTimeDistributedCriterion + ''' + + def __init__(self, criterion, size_average=False, bigdl_type="float"): + super(TimeDistributedCriterion, self).__init__(None, bigdl_type, criterion, size_average) From 309880c23fa7408da3cf6c914cad0cc0b3c4ed2e Mon Sep 17 00:00:00 2001 From: zhangli Date: Tue, 21 Mar 2017 09:25:06 +0800 Subject: [PATCH 005/823] merge visualization --- python/dllib/src/bigdl/dllib/nn/criterion.py | 276 +++++++++++++++++++ 1 file changed, 276 insertions(+) create mode 100644 python/dllib/src/bigdl/dllib/nn/criterion.py diff --git a/python/dllib/src/bigdl/dllib/nn/criterion.py b/python/dllib/src/bigdl/dllib/nn/criterion.py new file mode 100644 index 00000000000..7e47f7074ba --- /dev/null +++ b/python/dllib/src/bigdl/dllib/nn/criterion.py @@ -0,0 +1,276 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +import sys +from util.common import callBigDlFunc +from util.common import JavaValue +from util.common import callJavaFunc +from pyspark import SparkContext + +import numpy as np + +if sys.version >= '3': + long = int + unicode = str + + +class Criterion(JavaValue): + def __init__(self, jvalue, bigdl_type, *args): + self.value = jvalue if jvalue else callBigDlFunc( + bigdl_type, JavaValue.jvm_class_constructor(self), *args) + self.bigdl_type = bigdl_type + + @classmethod + def of(cls, jcriterion, bigdl_type="float"): + criterion = Criterion(bigdl_type, jcriterion) + criterion.value = jcriterion + criterion.bigdl_type = bigdl_type + return criterion + + +class ClassNLLCriterion(Criterion): + ''' + >>> classNLLCriterion = ClassNLLCriterion() + creating: createClassNLLCriterion + ''' + def __init__(self, bigdl_type="float"): + JavaValue.__init__(self, None, bigdl_type) + + +class MSECriterion(Criterion): + ''' + >>> mSECriterion = MSECriterion() + creating: createMSECriterion + ''' + def __init__(self, bigdl_type="float"): + JavaValue.__init__(self, None, bigdl_type) + + +class AbsCriterion(Criterion): + + ''' + >>> absCriterion = AbsCriterion(True) + creating: createAbsCriterion + ''' + + def __init__(self, + size_average=True, + bigdl_type="float"): + super(AbsCriterion, self).__init__(None, bigdl_type, + size_average) + + +class ClassSimplexCriterion(Criterion): + + ''' + >>> classSimplexCriterion = ClassSimplexCriterion(2) + creating: createClassSimplexCriterion + ''' + + def __init__(self, + n_classes, + bigdl_type="float"): + super(ClassSimplexCriterion, self).__init__(None, bigdl_type, + n_classes) + + +class CosineEmbeddingCriterion(Criterion): + + ''' + >>> cosineEmbeddingCriterion = CosineEmbeddingCriterion(1e-5, True) + creating: createCosineEmbeddingCriterion + ''' + + def __init__(self, + margin=0.0, + size_average=True, + bigdl_type="float"): + super(CosineEmbeddingCriterion, self).__init__(None, bigdl_type, + margin, + size_average) + + +class DistKLDivCriterion(Criterion): + + ''' + >>> distKLDivCriterion = DistKLDivCriterion(True) + creating: createDistKLDivCriterion + ''' + + def __init__(self, + size_average=True, + bigdl_type="float"): + super(DistKLDivCriterion, self).__init__(None, bigdl_type, + size_average) + + +class HingeEmbeddingCriterion(Criterion): + + ''' + >>> hingeEmbeddingCriterion = HingeEmbeddingCriterion(1e-5, True) + creating: createHingeEmbeddingCriterion + ''' + + def __init__(self, + margin=1, + size_average=True, + bigdl_type="float"): + super(HingeEmbeddingCriterion, self).__init__(None, bigdl_type, + margin, + size_average) + + +class L1HingeEmbeddingCriterion(Criterion): + + ''' + >>> l1HingeEmbeddingCriterion = L1HingeEmbeddingCriterion(1e-5) + creating: createL1HingeEmbeddingCriterion + ''' + + def __init__(self, + margin=1, + bigdl_type="float"): + super(L1HingeEmbeddingCriterion, self).__init__(None, bigdl_type, + margin) + + +class MarginCriterion(Criterion): + + ''' + >>> marginCriterion = MarginCriterion(1e-5, True) + creating: createMarginCriterion + ''' + + def __init__(self, + margin=1.0, + size_average=True, + bigdl_type="float"): + super(MarginCriterion, self).__init__(None, bigdl_type, + margin, + size_average) + + +class MarginRankingCriterion(Criterion): + + ''' + >>> marginRankingCriterion = MarginRankingCriterion(1e-5, True) + creating: createMarginRankingCriterion + ''' + + def __init__(self, + margin=1.0, + size_average=True, + bigdl_type="float"): + super(MarginRankingCriterion, self).__init__(None, bigdl_type, + margin, + size_average) + + +class MultiCriterion(Criterion): + + ''' + >>> multiCriterion = MultiCriterion() + creating: createMultiCriterion + ''' + + def __init__(self, + bigdl_type="float"): + super(MultiCriterion, self).__init__(None, bigdl_type) + + +class MultiLabelMarginCriterion(Criterion): + + ''' + >>> multiLabelMarginCriterion = MultiLabelMarginCriterion(True) + creating: createMultiLabelMarginCriterion + ''' + + def __init__(self, + size_average=True, + bigdl_type="float"): + super(MultiLabelMarginCriterion, self).__init__(None, bigdl_type, + size_average) + + +class ParallelCriterion(Criterion): + + ''' + >>> parallelCriterion = ParallelCriterion(True) + creating: createParallelCriterion + ''' + + def __init__(self, + repeat_target=False, + bigdl_type="float"): + super(ParallelCriterion, self).__init__(None, bigdl_type, + repeat_target) + + +class SmoothL1Criterion(Criterion): + + ''' + >>> smoothL1Criterion = SmoothL1Criterion(True) + creating: createSmoothL1Criterion + ''' + + def __init__(self, + size_average=True, + bigdl_type="float"): + super(SmoothL1Criterion, self).__init__(None, bigdl_type, + size_average) + + +class SmoothL1CriterionWithWeights(Criterion): + + ''' + >>> smoothL1CriterionWithWeights = SmoothL1CriterionWithWeights(1e-5, 1) + creating: createSmoothL1CriterionWithWeights + ''' + + def __init__(self, + sigma, + num=0, + bigdl_type="float"): + super(SmoothL1CriterionWithWeights, self).__init__(None, bigdl_type, + sigma, + num) + + +class SoftmaxWithCriterion(Criterion): + + ''' + >>> softmaxWithCriterion = SoftmaxWithCriterion("VALID") + creating: createSoftmaxWithCriterion + ''' + + def __init__(self, + normalize_mode="VALID", + bigdl_type="float"): + super(SoftmaxWithCriterion, self).__init__(None, bigdl_type, + None, + normalize_mode) + + +class TimeDistributedCriterion(JavaValue): + ''' + >>> td = TimeDistributedCriterion(ClassNLLCriterion()) + creating: createClassNLLCriterion + creating: createTimeDistributedCriterion + ''' + + def __init__(self, criterion, size_average=False, bigdl_type="float"): + super(TimeDistributedCriterion, self).__init__(None, bigdl_type, criterion, size_average) From 354c970d3fcfe6506868adc09d5bf358bbdce249 Mon Sep 17 00:00:00 2001 From: zhangli Date: Tue, 21 Mar 2017 09:25:06 +0800 Subject: [PATCH 006/823] merge visualization --- resources/mnist-data/testing_data.pickle | 55 +++++++++++++++++++ resources/pre_trained_lenet/lenet-model.9381 | Bin 0 -> 186847 bytes resources/pre_trained_lenet/lenet-state.9381 | Bin 0 -> 601 bytes 3 files changed, 55 insertions(+) create mode 100644 resources/mnist-data/testing_data.pickle create mode 100644 resources/pre_trained_lenet/lenet-model.9381 create mode 100644 resources/pre_trained_lenet/lenet-state.9381 diff --git a/resources/mnist-data/testing_data.pickle b/resources/mnist-data/testing_data.pickle new file mode 100644 index 00000000000..d8031fba17f --- /dev/null +++ b/resources/mnist-data/testing_data.pickle @@ -0,0 +1,55 @@ +(cnumpy.core.multiarray +_reconstruct +p0 +(cnumpy +ndarray +p1 +(I0 +tp2 +S'b' +p3 +tp4 +Rp5 +(I1 +(I32 +I28 +I28 +I1 +tp6 +cnumpy +dtype +p7 +(S'u1' +p8 +I0 +I1 +tp9 +Rp10 +(I3 +S'|' +p11 +NNNI-1 +I-1 +I0 +tp12 +bI00 +S'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00T\xb9\x9f\x97<$\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xde\xfe\xfe\xfe\xfe\xf1\xc6\xc6\xc6\xc6\xc6\xc6\xc6\xc6\xaa4\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00CrHr\xa3\xe3\xfe\xe1\xfe\xfe\xfe\xfa\xe5\xfe\xfe\x8c\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x11B\x0eCCC;\x15\xec\xfej\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00S\xfd\xd1\x12\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x16\xe9\xffS\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x81\xfe\xee,\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00;\xf9\xfe>\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x85\xfe\xbb\x05\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\t\xcd\xf8:\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00~\xfe\xb6\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00K\xfb\xf09\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x13\xdd\xfe\xa6\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x03\xcb\xfe\xdb#\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00&\xfe\xfeM\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x1f\xe0\xfes\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x85\xfe\xfe4\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00=\xf2\xfe\xfe4\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00y\xfe\xfe\xdb(\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00y\xfe\xcf\x12\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00t}\xab\xff\xff\x96]\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xa9\xfd\xfd\xfd\xfd\xfd\xfd\xda\x1e\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xa9\xfd\xfd\xfd\xd5\x8e\xb0\xfd\xfdz\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x004\xfa\xfd\xd2 \x0c\x00\x06\xce\xfd\x8c\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00M\xfb\xd2\x19\x00\x00\x00z\xf8\xfdA\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x1f\x12\x00\x00\x00\x00\xd1\xfd\xfdA\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00u\xf7\xfd\xc6\n\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00L\xf7\xfd\xe7?\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x80\xfd\xfd\x90\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xb0\xf6\xfd\x9f\x0c\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x19\xea\xfd\xe9#\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xc6\xfd\xfd\x8d\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00N\xf8\xfd\xbd\x0c\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x13\xc8\xfd\xfd\x8d\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x86\xfd\xfd\xad\x0c\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf8\xfd\xfd\x19\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf8\xfd\xfd+\x14\x14\x14\x14\x05\x00\x05\x14\x14%\x96\x96\x96\x93\n\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf8\xfd\xfd\xfd\xfd\xfd\xfd\xfd\xa8\x8f\xa6\xfd\xfd\xfd\xfd\xfd\xfd\xfd{\x00\x00\x00\x00\x00\x00\x00\x00\x00\xae\xfd\xfd\xfd\xfd\xfd\xfd\xfd\xfd\xfd\xfd\xfd\xf9\xf7\xf7\xa9uu9\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00v{{{\xa6\xfd\xfd\xfd\x9b{{)\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00&\xfem\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00W\xfcR\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x87\xf1\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00-\xf4\x96\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00T\xfe?\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xca\xdf\x0b\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00 \xfe\xd8\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00_\xfe\xc3\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x8c\xfeM\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x009\xed\xcd\x08\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00|\xff\xa5\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xab\xfeQ\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x18\xe8\xd7\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00x\xfe\x9f\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x97\xfe\x8e\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xe4\xfeB\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00=\xfb\xfeB\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x8d\xfe\xcd\x03\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\n\xd7\xfey\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\xc6\xb0\n\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0b\x96\xfd\xca\x1f\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00%\xfb\xfb\xfdk\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x15\xc5\xfb\xfb\xfdk\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00n\xbe\xfb\xfb\xfb\xfd\xa9m>\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xfd\xfb\xfb\xfb\xfb\xfd\xfb\xfb\xdc3\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xb6\xff\xfd\xfd\xfd\xfd\xea\xde\xfd\xfd\xfd\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00?\xdd\xfd\xfb\xfb\xfb\x93M>\x80\xfb\xfbi\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00 \xe7\xfb\xfd\xfb\xdc\x89\n\x00\x00\x1f\xe6\xfb\xf3q\x05\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00%\xfb\xfb\xfd\xbc\x14\x00\x00\x00\x00\x00m\xfb\xfd\xfb#\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00%\xfb\xfb\xc9\x1e\x00\x00\x00\x00\x00\x00\x1f\xc8\xfd\xfb#\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00%\xfd\xfd\x00\x00\x00\x00\x00\x00\x00\x00 \xca\xff\xfd\xa4\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x8c\xfb\xfb\x00\x00\x00\x00\x00\x00\x00\x00m\xfb\xfd\xfb#\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xd9\xfb\xfb\x00\x00\x00\x00\x00\x00\x15?\xe7\xfb\xfd\xe6\x1e\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xd9\xfb\xfb\x00\x00\x00\x00\x00\x00\x90\xfb\xfb\xfb\xdd=\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xd9\xfb\xfb\x00\x00\x00\x00\x00\xb6\xdd\xfb\xfb\xfb\xb4\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xda\xfd\xfdII\xe4\xfd\xfd\xff\xfd\xfd\xfd\xfd\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00q\xfb\xfb\xfd\xfb\xfb\xfb\xfb\xfd\xfb\xfb\xfb\x93\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x1f\xe6\xfb\xfd\xfb\xfb\xfb\xfb\xfd\xe6\xbd#\n\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00>\x8e\xfd\xfb\xfb\xfb\xfb\xfdk\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00H\xae\xfb\xadGH\x1e\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x002\xe0\x00\x00\x00\x00\x00\x00\x00F\x1d\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00y\xe7\x00\x00\x00\x00\x00\x00\x00\x94\xa8\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x04\xc3\xe7\x00\x00\x00\x00\x00\x00\x00`\xd2\x0b\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00E\xfc\x86\x00\x00\x00\x00\x00\x00\x00r\xfc\x15\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00-\xec\xd9\x0c\x00\x00\x00\x00\x00\x00\x00\xc0\xfc\x15\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xa8\xf75\x00\x00\x00\x00\x00\x00\x00\x12\xff\xfd\x15\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00T\xf2\xd3\x00\x00\x00\x00\x00\x00\x00\x00\x8d\xfd\xbd\x05\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xa9\xfcj\x00\x00\x00\x00\x00\x00\x00 \xe8\xfaB\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0f\xe1\xfc\x00\x00\x00\x00\x00\x00\x00\x00\x86\xfc\xd3\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x16\xfc\xa4\x00\x00\x00\x00\x00\x00\x00\x00\xa9\xfc\xa7\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\t\xcc\xd1\x12\x00\x00\x00\x00\x00\x00\x16\xfd\xfdk\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xa9\xfc\xc7UUUU\x81\xa4\xc3\xfc\xfcj\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00)\xaa\xf5\xfc\xfc\xfc\xfc\xe8\xe7\xfb\xfc\xfc\t\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x001TTTT\x00\x00\xa1\xfc\xfc\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x7f\xfc\xfc-\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x80\xfd\xfd\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x7f\xfc\xfc\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x87\xfc\xf4\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xe8\xeco\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xb3B\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00M\xfek\x03\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x13\xe3\xfe\xfe\t\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00Q\xfe\xfe\xa5\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x07\xcb\xfe\xfeI\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x005\xfe\xfe\xfa\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x86\xfe\xfe\xb4\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xc4\xfe\xf80\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00:\xfe\xfe\xed\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00o\xfe\xfe\x84\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xa3\xfe\xee\x1c\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00<\xfc\xfe\xdf\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00O\xfe\xfe\x9a\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xa3\xfe\xee5\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x1c\xfc\xfe\xd2\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00V\xfe\xfe\x83\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00i\xfe\xea\x14\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xaf\xfe\xcc\x05\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\xd3\xfe\xc4\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x03\x9e\xfe\xa0\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x1a\x9dk\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x16\xc0\x86 \x00\x00\x00\x00\x00\x00\x00\x00\x0fM\x05\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x11\xeb\xfa\xa9\x00\x00\x00\x00\x00\x00\x00\x00\x0f\xdc\xf1%\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x14\xbd\xfd\x93\x00\x00\x00\x00\x00\x00\x00\x00\x00\x8b\xfdd\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00F\xfd\xfd\x15\x00\x00\x00\x00\x00\x00\x00\x00+\xfe\xad\r\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x16\x99\xfd`\x00\x00\x00\x00\x00\x00\x00\x00+\xe7\xfe\\\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xa3\xff\xcc\x0b\x00\x00\x00\x00\x00\x00\x00\x00h\xfe\x9e\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xa2\xfd\xb2\x05\x00\x00\x00\x00\x00\x00\t\x83\xed\xfd\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xa2\xfd\xfd\xbf\xafFFFF\x85\xc5\xfd\xfd\xa9\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x003\xe4\xfd\xfd\xfe\xfd\xfd\xfd\xfd\xfe\xfd\xfd\xdb#\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x11A\x89\xfe\xe8\x89\x89\x89,\xfd\xfd\xa1\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"\xfe\xce\x15\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xa0\xfdE\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00U\xfe\xf12\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x9e\xfe\xa5\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xe7\xf42\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00h\xfe\xe8\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xd0\xfd\x9d\x00\r\x1e\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xd0\xfd\x9a[\xcc\xa1\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xd0\xfd\xfe\xfd\x9a\x1d\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00=\xbe\x80\x17\x06\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0e\x95\xc1\x05\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00[\xe0\xfd\xfd\x13\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x1c\xeb\xfe\xfd\xfd\xa6\x12\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x90\xfd\xfe\xfd\xfd\xfd\xees\x06\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x1f\xf1\xfd\xd0\xb9\xfd\xfd\xfd\xe7\x18\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00O\xfe\xc1\x00\x08b\xdb\xfe\xff\xc9\x12\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00V\xfdP\x00\x00\x00\xb6\xfd\xfe\xbf\x0c\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xaf\xfd\x9b\x00\x00\x00\xea\xfd\xfe\x87\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00V\xfd\xd0(U\xa6\xfb\xed\xfe\xec*\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x12\xee\xfd\xfe\xfd\xfd\xb9$\xd8\xfd\x98\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00D\xf0\xff\xfe\x91\x08\x00\x86\xfe\xdf#\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00D\x9e\x8e\x0c\x00\x00\t\xaf\xfd\xa1\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00X\xfd\xe2\x12\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\xa6\xfd~\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x000\xf5\xfd&\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00s\xfe\xac\t\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x15\xda\xfe.\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x1e\xfe\xa5\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xba\xf4*\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0e\xdfN\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x11///\x10\x81U/\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00K\x99\xd9\xfd\xfd\xfd\xd7\xf6\xfd\xfd\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00#\x8e\xf4\xfc\xfd\xfd\xfd\xfd\xfd\xfd\xfd\xfd\xfd\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00?\xfd\xfd\xfd\xfd\xfd\xfd\xfd\xd5\xaa\xaa\xaa\xaa\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x14\x84H\x009\xee\xe3\xee\xa8|E\x14\x0b\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0b\xce\xfdN\x00\x00 \x00\x1e\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x06\xb1\xfd\x84\n\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0c\x85\xfd\xe9\x0f\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\\\xfd\xdf\x1c\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x96\xfd\xae\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xea\xfd\xf6\x7f1\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xff\xfd\xfd\xfd\xfb\x93[yU**U\x1c\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x8b\xfd\xfd\xfd\xfd\xfd\xfd\xfd\xfd\xfd\xfd\xfd\xe8\xa8\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x035\xda\xde\xfb\xfd\xfd\xfd\xfd\xfd\xfd\xfd\xfd\xfc|\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00CH\xc8\xfd\xfd\xfd\xfd\xfd\xfd\xfd\xaf\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00x\xfd\xf9\x983\xa4\xfd\xfd\xaf\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x002\xfd\xfd\xfd\xbc\xfc\xfd\xfd\x94\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\t\xa7\xfd\xfd\xfd\xfd\xfa\xaf\x0b\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x17\xb4\xe7\xfd\xdd\x80\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00]\x95\x16\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00$8\x89\xc9\xc7_%\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00-\x98\xea\xfe\xfe\xfe\xfe\xfe\xfa\xd3\x97\x06\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00.\x99\xf0\xfe\xfe\xe3\xa6\x85\xfb\xc8\xfe\xe5\xe1h\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x99\xea\xfe\xfe\xbb\x8e\x08\x00\x00\xbf(\xc6\xf6\xdf\xfd\x15\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x08~\xfd\xfe\xe9\x80\x0b\x00\x00\x00\x00\xd2+F\xfe\xfe\xfe\x15\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00H\xf3\xfe\xe46\x00\x00\x00\x00\x03 t\xe1\xf2\xfe\xff\xa2\x05\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00K\xf0\xfe\xdfm\x8a\xb2\xb2\xa9\xd2\xfb\xe7\xfe\xfe\xfe\xe8&\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\t\xaf\xf4\xfd\xff\xfe\xfe\xfb\xfe\xfe\xfe\xfe\xfe\xfc\xab\x19\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x88\xc3\xb0\x92\x99\xc8\xfe\xfe\xfe\xfe\x96\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xa2\xfe\xfe\xf1c\x03\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00v\xfa\xfe\xfeZ\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00d\xf2\xfe\xfe\xd3\x07\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x006\xf1\xfe\xfe\xf2;\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x83\xfe\xfe\xf4@\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\r\xf9\xfe\xfe\x98\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0c\xe4\xfe\xfe\xd0\x08\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00N\xff\xfe\xfeB\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xd1\xfe\xfe\x89\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xe3\xff\xe9\x19\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00q\xffl\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00=\x03*v\xc1vv=\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0e\xb3\xf5\xec\xf2\xfe\xfe\xfe\xfe\xf5\xebT\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x97\xfe\xfe\xfe\xd5\xc0\xb2\xb2\xb4\xfe\xfe\xf1.\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00+\xeb\xfe\xe2@\x1c\x0c\x00\x00\x02\x80\xfc\xff\xad\x11\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x008\xfe\xfdk\x00\x00\x00\x00\x00\x00\x00\x86\xfa\xfeK\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00?\xfe\x9e\x00\x00\x00\x00\x00\x00\x00\x00\x00\xdd\xfe\x9d\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xc2\xfeg\x00\x00\x00\x00\x00\x00\x00\x00\x00\x96\xfe\xd5\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"\xdc\xef:\x00\x00\x00\x00\x00\x00\x00\x00\x00T\xfe\xd5\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00~\xfe\xab\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00T\xfe\xd5\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xd6\xef<\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00T\xfe\xd5\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xd6\xc7\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00T\xfe\xd5\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0b\xdb\xc7\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00T\xfe\xd5\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00b\xfe\xc7\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xa2\xfe\xd1\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00b\xfe\xc7\x00\x00\x00\x00\x00\x00\x00\x00\x00\x003\xee\xfeK\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00b\xfe\xc7\x00\x00\x00\x00\x00\x00\x00\x00\x003\xa5\xfe\xc3\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00B\xf1\xc7\x00\x00\x00\x00\x00\x00\x00\x00\x03\xa7\xfe\xe37\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xd6\xd5\x14\x00\x00\x00\x00\x00.\x98\xca\xfe\xfe?\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xd6\xfe\xcc\xb4\xb4\xb4\xb4\xb4\xeb\xfe\xfe\xea\x9c\n\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00Q\xcd\xfe\xfe\xfe\xfe\xfe\xfe\xfe\xfc\xeax\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x1a\xd2\xfe\xfe\xfe\xfe\xfe\x99h\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x07\xcc\xfd\xb0\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x07\x96\xfc\xfc}\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00u\xfc\xba8\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x8d\xfcv\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x9a\xf72\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x1a\xfd\xc4\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x96\xfd\xc4\x00\x00\x00\x00\x00\x00\x009UU&\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xe1\xfd`\x00\x00\x00\x00\x00\x97\xe2\xf3\xfc\xfc\xee}\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\n\xe5\xe2\x00\x00\x00\x046\xe5\xfd\xff\xea\xaf\xe1\xff\xe4\x1f\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00n\xfc\x96\x00\x00\x1a\x80\xfc\xfc\xe3\x86\x1c\x00\x00\xb2\xfc8\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x9f\xfcq\x00\x00\x96\xfd\xfc\xba+\x00\x00\x00\x00\x8d\xfc8\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xb9\xfcq\x00&\xed\xfd\x97\x06\x00\x00\x00\x00\x00\x8d\xca\x06\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xc6\xfdr\x00\x93\xfd\xa3\x00\x00\x00\x00\x00\x00\x00\x9a\xc5\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xc5\xfcq\x00\xac\xfc\xbc\x00\x00\x00\x00\x00\x00\x1a\xfd\xab\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xc5\xfcq\x00\x13\xe7\xf7z\x13\x00\x00\x00\x00\xc8\xf48\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x1a\xde\xfcq\x00\x00\x19\xcb\xfc\xc1\r\x00L\xc8\xf9}\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xb9\xfd\xb3\n\x00\x00\x00L#\x1d\x9a\xfd\xf4}\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x1c\xd1\xfd\xc4R99\x83\xc5\xfc\xfd\xd6Q\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x19\xd8\xfc\xfc\xfc\xfd\xfc\xfc\xfc\x9c\x13\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10g\x8b\xf0\x8c\x8b\x8b(\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x001\xb4\xfd\xff\xfd\xa9$\x0bL\t\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05D\xe4\xfc\xfc\xfd\xfc\xfc\xa0\xbd\xfd\\\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x007\xfc\xfc\xe3OEEdZ\xec\xf7C\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00+\xe9\xfc\xb92\x00\x00\x00\x1a\xcb\xfc\x87\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xa8\xfd\xb2%\x00\x00\x00\x00F\xfc\xfc?\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x9b\xfd\xf2*\x00\x00\x00\x00\x05\xbf\xfd\xbe\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xcf\xfc\xe6\x00\x00\x00\x00\x05\x88\xfc\xfc@\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xcf\xfc\xe6\x00\x00\x00 \x8a\xfc\xfc\xe3\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xa5\xfc\xf9\xcf\xcf\xcf\xe4\xfd\xfc\xfc\xa0\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\t\xb3\xfd\xfc\xfc\xfc\xfcK\xa9\xfc8\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00@ttJ\x00\x95\xfd\xd7\x15\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xfd\xfc\xa2\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00 \xfd\xf02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x9d\xfd\xa4\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00+\xf0\xfd\\\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00]\xfd\xfcT\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00r\xfc\xd1\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xcf\xfct\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xa5\xfct\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00]\xc8?\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x11B\x8a\xff\xfd\xa9\x8a\x17\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05x\xe4\xfc\xfc\xfd\xfc\xfc\xfc\x9e\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00l\xfc\xfc\xfc\xfc\xbe\xfc\xfc\xfc\xfc\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00+\xe9\xfc\xfc\xfct\x05\x87\xfc\xfc\xfc\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00+\xb2\xfd\xfc\xdd+\x02\x00\x056\xe8\xfc\xd2\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00]\xfd\xff\xf9s\x00\x00\x00\x00\x00\x88\xfb\xff\x9a\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xa6\xfc\xfd\xb9\x00\x00\x00\x00\x00\x00\x00\xd1\xfd\xce\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x13\xdc\xfc\xfd\\\x00\x00\x00\x00\x00\x00\x00t\xfd\xce\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00F\xfc\xfc\xc0\x11\x00\x00\x00\x00\x00\x00\x00t\xfd\xdf\x19\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00z\xfc\xfc?\x00\x00\x00\x00\x00\x00\x00\x00t\xfd\xfcE\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x84\xfd\xfd\x00\x00\x00\x00\x00\x00\x00\x00\x00t\xff\xfdE\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xb8\xfc\xfc\x00\x00\x00\x00\x00\x00\x00\x00\x00t\xfd\xfcE\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xb8\xfc\xfc\x00\x00\x00\x00\x00\x00\x00\x00\x00t\xfd\xf02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xb8\xfc\xfc\x00\x00\x00\x00\x00\x00\x00\x00\x00\xd2\xfdp\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x000\xe8\xfc\x9e\x00\x00\x00\x00\x00\x00\x00\x00\xe6\xe8\x08\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00]\xfd\xf42\x00\x00\x00\x00\x00\x00\x9b\xfd\xa8\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"\xa4\xfdq\x00\x00\x00\x00\x00B\xec\xe7*\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00 \xde\xf0\x86\x00\x00&[\xea\xfc\x89\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x19\xb1\xf0\xcfg\xe9\xfc\xfc\xb0#\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0f6\xb3\xfc\x89\x896\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x80\xff\xbf\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xbf\xff\xff@\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xff\xff\xff\x80\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xff\xff\xff\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xff\xff\xff\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x80\xff\xff\x80\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xff\xff\xff\x80\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xff\xff\xff\x80\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xff\xff\xff\x80\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xff\xff\xff\x80\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xff\xff\xff\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xff\xff\xff\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xff\xff\xff\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xff\xff\xff\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xbf\xff\xff\x80\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xbf\xff\xff\x80\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00@\xff\xff\xff\x80\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xbf\xff\xff\xff\x80\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00@\xff\xff\xff@\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00@\xff\x80\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x003\x84\xd6\xfd\xfe\xfd\xcb\xa2)\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00f\x8e\xcb\xcb\xfd\xfc\xfd\xfc\x97F\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xfe\xfd\xf4\xcb\x8efR\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xac\xfc\xcb\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x15\xdf\xea\x1e\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00z\xfd2\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00{\xfe[333\n\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x15\xdf\xfd\xfc\xfd\xfc\xfd\xacR\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x15\xd6\xfd\xcb\xa2ff\xcb\xdf\xfe\xfd3\n\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00=\xfd\xab\x00\x00\x00\x00\x00\x14p\xc0\xfd\xd4)\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00f\xcb\xea3\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x14\xd5\xe8R\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00>\xcb\xeap\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x14\xd5\xfc\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x99\xfd\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00)\xe9\xd4\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00q\\\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x1f\xad\xf4(\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00R\xfd\x97\x00\x00\x00\x00\x00\x00\x15ff\xb7\xe9\xd4Q\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00R\xff\xfd\xea\x98\x99\xc1\xad\xfd\xfe\xfd\xfe\xd5\x8e\x14\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00G\x97\x97\xe8\xfd\xd4\xc0\x97\x8322\n\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00@\x92\xe5\xff\xcdx\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x18\xc6\xfc\xfd\xe1\xd8\xeb\xfcY\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x14\xcd\xfd\xdfF\x0f\x00\x1d\xce\xae\x02W&\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x89\xfd\xe3\x06\x00\x00\x00\x00#\x1cL\xfd\xfd\t\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00X\xfb\xeb\x0c\x00\x00\x00\x00\x00\x00*\xee\xfd\xae\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x86\xfd\xc0\x00\x00\x00\x00\x00\x00\x00\x0e\xee\xfd\xa1\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xa9\xfdJ\x00\x00\x00\x00\x00\x00\x00U\xf7\xfdK\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\n\xfa\xfd/\x00\x00\x00\x00\x00\x00\x06\xdb\xfd\xf1\x1f\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\n\xfd\xfd/\x00\x00\x00\x00\x00\x05H\xfd\xfd\x8f\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x06\xdd\xfdu\x00\x00\x00\x00\x19v\xfd\xfd\xfd/\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x1c\xf2\xfe\xbbh\x92\x9f\xdc\xf4\xef\xfe\xe0\x12\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00N\xc9\xfd\xfd\xf8\xd7\x9cC\xf7\xfd\x9d\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05882\x00\x00&\xfd\xfdJ\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00B\xfd\xfd\x12\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x95\xfd\xfd\x12\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x1e\xee\xfd\xbf\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00B\xfd\xfdp\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00W\xfd\xf4\x03\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xaa\xfd\xc6\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xfe\xfd\x95\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00J\xf9\xfe\xfe\xfe\xf5\xa7\xa7\x88\x19P<\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00h\xfe\xfe\xfe\xfe\xfe\xfe\xfe\xfe\xf9\xfe\xfc\xc5qG\'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05c\x87iir\xc0\xc0\xc0\xe9\xfe\xfe\xfe\xfe\xfe\xf6\x81\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00-rr\xcb\xfe\xfe\xfe\xf0\x0f\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x08#\x9b\xfe\xfe\x82\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00$\xfe\xf1"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00s\xfe\xfev\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"\xf3\xfe\xf0\x11\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00o\xfe\xfe\x8b\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00%\xf3\xfe\xf4(\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x14\xb0\xfe\xfeq\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x8c\xfe\xfe\xdc\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00X\xfd\xfe\xf3-\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00?\xf1\xfe\xfeS\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\xf3\xfe\xfe\x93\x05\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02o\xfe\xfe\xcb\x05\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00:\xfe\xfe\xfeT\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0e\xed\xfe\xff\xc2\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00R\xfe\xfe\xc2\x1b\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\'\xe6\xc1\x1c\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0c)\x92\x920\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0c\x81\xfd\xfd\xfd\xfa\xa3\x12\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x85\xfd\xfd\xfd\xfd\xfd\xfd\xe5F\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00e\xfd\xfc\x91fk\xed\xfd\xf7\x80\n\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xb5\xfd\xa7\x00\x00\x00=\xeb\xfd\xfd\xa3\x05\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xff\xfd+\x00\x00\x00\x00:\xc1\xfd\xfd\xa4\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xbb\xfd \x00\x00\x00\x00\x007\xec\xfd\xfdV\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x92\xfd \x00d\xbeWWW\x93\xfd\xfd{\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00^\xfdN(\xf8\xfd\xfd\xfd\xfd\xfd\xfd\xfd\xdfT\x0f\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0e\\\x0c#\xf0\xfd\xfd\xfd\xfd\xfd\xfd\xfd\xfd\xfd\xf4Y\n\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00K\xa1\xb3\xfd\xfd\xfd\xfd\xfd\xfd\xfd\xfd\xfd\xd1+\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x03\x10\x10\'&\x10\x10\x91\xf3\xfd\xfd\xb90\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x14:\x00\x00\x00\x00\x00\x00\x00\x00:\xd1\xfd\xfd\xb7\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00M\xdd\xf7O\x00\x00\x00\x00\x00\x00\x00\x00\r\xdb\xfd\xf0H\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00Z\xf7\xfd\xfc9\x00\x00\x00\x00\x00\x00\x00\x005\xfb\xfd\xbf\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00t\xfd\xfd;\x00\x00\x00\x00\x00\x00\x00\x00c\xfc\xfd\x91\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0e\xbc\xfd\xdd\x9e&\x00\x00\x00\x00o\xd3\xf6\xfd\xfd\x91\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0c\xdd\xf6\xfd\xfb\xf9\xf9\xf9\xf9\xfd\xfd\xfd\xfd\xc8\x13\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00_\xb7\xe4\xfd\xfd\xfd\xfd\xfd\xfd\xc3|\x17\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x18%\x8aJ~X%\x07\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xa8[\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\xea~\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x003\xfe~\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00&\xb2\x1f\x00\x00\x00\x00\x003\xfeQ\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00^\xfeS\x00\x00\x00\x00\x00W\xfe6\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xa0\xfe8\x00\x00\x00\x00\x00\xbd\xee\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\r\xe3\xa8\x02\x00\x00\x00\x00\x00\xc2\xec\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x007\xfer\x00\x00\x00\x00\x00\x10\xeb\xa7\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00s\xfe2\x00\x00\x00\x00\x00g\xfei\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x07\xdd\xecK\x9c\xb4\xbe\xfc\xfc\xfd\xfer\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0f\xfe\xfe\xfe\xfc\xd3\xb3\xb3\xb3\xf6\xfe\xf7^\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x08\xd9\xefu\x16\x00\x00\x00\x00\xe2\xfe\xf2\xc5\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0e\x12\x00\x00\x00\x00\x00\x1b\xf3\xcf.%\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00c\xfe\x84\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00t\xfeC\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00t\xfe=\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00t\xfe=\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xae\xffd\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xbb\xfeS\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00s\xb0\n\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x1f\x8c\xc1,\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00&\x92\xf0\xfe\xfe\xe40M.\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02+\xe6\xfe\xfe\xfe\xfe\xfe\xf1\xfe\xc5\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x15\x82\xfe\xfe\xfe\xef\xfc\xfe\xfe\xfe\xfe\xed\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x9a\xfe\xfe\xf9hG\xc6\xfe\xfe\xfe\xea9\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\xad\xfc\xfc\xce3x\xd7\xfe\xfe\xfe\xfe\xb8\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00p\xfe\xfe\xd7W\xf7\xfe\xfe\xfe\xfe\xfe\xd9\x1f\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\xe7\xfe\xfe\xfe\xfe\xfe\xec\x80\xc4\xfe\xfew\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x03\xc5\xfe\xfe\xf5\xee\x83\x11.\xf7\xfe\xc7\x0e\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\\X(\x00\x00\x0c\xad\xfe\xf2&\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00_\xfe\xfe\x9b\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x12\xd2\xfe\xe1\x06\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xc5\xfe\xfec\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"\xf2\xfe\xb3\x03\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x1b\xdf\xfe\xe1\x1e\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00M\xfe\xff\x7f\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x006\xee\xfe\xf85\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0b\xb7\xfe\xfe\xe7)\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00q\xfe\xfe\xe60\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00n\xef~\x19\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x001\xb4\xfd\xf42\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xb8\xfc\xfc\xe8\xa4\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00B\xea\xfc\x88&8\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x13\xec\xfc\xb0\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00L\xfc\xfc8\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x8b\xfd\xad\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x16\xd4\xfcE\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00t\xfd\xf02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x9d\xfd\xce\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xe6\xfdf\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xe7\xff\xb4\x8a\xb4\xfd\xff\xfd\xdea\x03\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xe6\xfd\xfc\xfc\xfc\xfc\xd3\xfc\xfc\xfcu\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xe6\xfd\xf0\xb7YE\x07E\xab\xfc\xfcU\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x87\xfd\x99\x00\x00\x00\x00\x00\r\xd7\xfct\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00t\xfd\xce\x00\x00\x00\x00\x00\x00\x9b\xfct\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00j\xff\xd3\x07\x00\x00\x00\x001\xe9\xfdt\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x9f\xfc\x9a\t\x00\x00\x1e\xc5\xfc\xfc_\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x11\xe3\xfc\x9aFQ\xe4\xfc\xe3\x82\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x002\xe3\xfc\xfc\xfd\xfc\xb92\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x000\xb3\xfc\xbeu\x06\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00r\xff9\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xe2\xff9\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x1d\xffr\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00r\xffV\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xc6\xff\x1d\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x1d\xff\xe2\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x8d\xff\x8d\x00\x00\x00\x00\x00\x009VV\x8d9\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xaa\xffV\x00\x00\x00\x00\x009\xe2\xff\xff\xff\xff9\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xaa\xffV\x00\x00\x00\x00\x00\xaa\xff\xffr9\xe2\xc6\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xaa\xffV\x00\x00\x00\x009\xff\xffV\x00\x00\xaa\xff\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xaa\xffV\x00\x00\x00\x00\xaa\xffV\x00\x00\x00\xaa\xffV\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x8d\xffr\x00\x00\x00\x1d\xff\xc6\x00\x00\x00\x00\xaa\xffV\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x1d\xff\xff\x00\x00\x00\xaa\xffr\x00\x00\x00\x00\xc6\xe2\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xc6\xffr\x00\x00\xaa\xffV\x00\x00\x00V\xffr\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00V\xff\xff\xaa9\xff\xff\x1d\x00\x00V\xe2\xe2\x1d\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00r\xe2\xff\xff\xff\xffVV\xaa\xff\xff9\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x1d\x8d\xff\xff\xff\xff\xff\xff\xc6\x1d\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xe2\xff\xff\x8dV\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x8d\xff\xff\xaa\xaa9\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x8d\xe2\xaar\x1d\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00E\x98\xed\xfe\xfe\xff\xfe\xfc4\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00W\xa4\xed\xfd\xfe\xda\x8aS\'\x9a\xfe\x87\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x8a\xf6\xfd\xfe\xd8\xa76\x05\x00\x00\x00d\xbf\x03\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xe9\xfe\xa95\x06\x00\x00\x00\x00\x00\x00#\x08\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xae\xfe^\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00&\xf5\xdd\r\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x8e\xfe\x95\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x97\xfep\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xe2\xf2 \x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0c\xf0\xcb,,,,\x08\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00F\xfe\xfe\xfe\xfe\xfe\xfe\xcdU\x08\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00%\xb8\xa9\x85\x85\xa2\xd4\xfe\xfe\xa6\x08\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\t3\xb1\xfe}\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x1b\xd1\xfeh\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00?\xd1\xfe\xc2\r\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x06\x00\n\x89\xf4\xfe\xc64\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\rWz\x93\xdf\xfe\xf7\x7f\r\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x03v\xfa\xd2\xf8\xfe\xfc\xc71\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x8a\xfe\xfe\xfe\xfa\xc9H\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00O\xa7\xc5W\x07\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xc5<\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x002\xf3*\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\xba\xec\x15\x00\x00\x00\x86S\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00X\xd5\x1a\x00\x00\x00\x00\x81\xaa\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x80x\x00\x00\x00\x00\x00[\xd1\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00 \xeap\x00\x00\x00\x00\x00[\xf2\x1e\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00t\xde\x15\x00\x00\x00\x00\x00\xaa\xf4:\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x1e\xf0|\x00\x00\x00\x00\x00(\xf9\x8a\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x03\x7f\xf1\x12\x00\x00\x00\x00\x00\x0f\xe6\x86\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\xfe\xb3\x00\x00\x00\x00\x00\x00\x00\xa5\x86\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\t\xbf\xe9\x88e\x14\x00\x00\x00\x00q\xd7\x1f\x1f\x1f\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x08Y\x80\xc2\xda\xd2\xd2\xd3\xd2\xe2\xfe\xfe\xccx\x07\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\n\x0f%ZZ\xc4\xf1"\x05\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xa5\xe9\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xa5\x8f\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00[\xbc\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00[\xe9"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x14\xfe=\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x04\xd1\xdb\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00k\xa6\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x03P\xc3UPPP\x0e\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\t\xfd\xfd\xfd\xfd\xfd\xfd\x98\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0b>8\x00\t\xfd\xfd\xfd\xfd\xfd\xfd\xfb\xed9\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x8d\xfd\xf1S\x04\xa1\xfd\xfd\xfd\xfd\xfd\xfd\xfd<\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0b\x8d\xf8\xfd\xfd\x93\x00I\xd1\xfc\xfd\xfd\xfd\xfd\xfd\xd4\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x93\xfd\xfd\xfd\xfd\xc7"\x00\x00\xa0\xfd\x8e\xc2\xfd\xfd\xf4M\x00\x00\x00\x00\x00\x00\x00\x00\x00\t\x88\xfa\xfd\xfd\xfd\xfd\xfdE\x00\x00\x0f4\x05\x1b\xc9\xfd\xfd\x9c\x00\x00\x00\x00\x00\x00\x00\x00\x00P\xfd\xfd\xfd\xfd\xfd\xfd\xd1)\x00\x00\x00\x00\x00\x00$\xfd\xfd\xc4!\x00\x00\x00\x00\x00\x00\x00\x00V\xfd\xfd\xfd\xfd\xfd\xea)\x00\x00\x00\x00\x00\x00\x00*\xfd\xfd\xfdN\x00\x00\x00\x00\x00\x00\x00\x00\xfe\xfd\xfd\xfd\xfd\xfd\xa5\x00\x00\x00\x00\x00\x00\x00\x00\xd3\xfd\xfd\xfdN\x00\x00\x00\x00\x00\x00\x00\x00\xfe\xfd\xfd\xfd\xfd\xac\x12\x00\x00\x00\x00\x00\x00\x00\x00\xd3\xfd\xfd\xfdN\x00\x00\x00\x00\x00\x00\x00\x00\xfe\xfd\xfd\xfd\xd2\x04\x00\x00\x00\x00\x00\x00*\xe5\xf6\xfc\xfd\xfd\x9f\x03\x00\x00\x00\x00\x00\x00\x00\x00\xfe\xfd\xfd\xfd\xd1\x00\x00\x00\x00\x00c\x95\xd2\xfd\xfd\xfd\xfd\xf2A\x00\x00\x00\x00\x00\x00\x00\x00\x00\xff\xfd\xfd\xfd\xda555\xb4\xe4\xf4\xfd\xfd\xfd\xfd\xfd\xfdM\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xa3\xfd\xfd\xfd\xfd\xfd\xfd\xfd\xfd\xfd\xfd\xfd\xfd\xfd\xfd\xfd\xc1\x1d\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00P\xfd\xfd\xfd\xfd\xfd\xfd\xfd\xfd\xfd\xfd\xfd\xfd\xfd\xea\xc1\x18\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00,\xd2\xfd\xfd\xfd\xfd\xfd\xfd\xfd\xfd\xfd\xfd\xfd\xe5C\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00&\xe4\xfd\xfd\xfd\xfd\xfd\xfd\xfd\xfd\xf8\xebA\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\'\xd2\xfd\xfd\xfd\xfd\xfd\xfd\xbdr\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00+l\xfd\xfd\xb3NN\x1b\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x15\x00\x00\x00\x0b\\\xad\xfd\xfe\xfd\xfe\xfd>\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x8e\x00\x15f\xd5\xfc\xe9\x97\x83\x83\xfd\xfc\x8e\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xa3\x98\x98\xd6\xe9\xb7f\x00\x00\x00\x00\x84\xfd\xff2\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00R\xf3\xfd\xfc\x83\x1e\x00\x00\x00\x00\x00\x00\n\xd4\xfd\x83\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00{\xcbf\x14\x00\x00\x00\x00\x00\x00\x00\x00\x00\xcb\xff\x97\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xa2\xfd\x97\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00f\xfe\x97\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00f\xfd\x97\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xa3\xfe[\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xcb\xfd2\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xcb\xcb\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x15\xdfz\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00H\xfdf\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xe9\xfcf\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xfe\xe9\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00)\xfd\x97\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00{\xfe2\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x15\xdf\xe9\x1e\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x003\xfd\xb7\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x003\xfc\x14\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xdds\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x003\xfe\x94\x00\x00\x00\x00\x16\xe6\x86\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xa8\xfes\x00\x00\x00\x00\x18\xfd\xb8\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x003\xea\xfeQ\x00\x00\x00\x00[\xfd\xb8\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\r\xdd\xfe\xa0\x00\x00\x00\x00\x00\x8d\xfe\x7f\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00r\xfd\xfdL\x00\x00\x00\x00\x00\xcf\xfd]\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0f\xe8\xfdf\x00\x00\x00\x00\x00\x00\xcf\xfd\x11\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xe5\xfd\xca\x13\x00\x00\x00\x00\x00"\xf0\xfd\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x07\xaa\xfe\xfe.\x00\x00\x00\x00\x00\x00/\xfe\xfe\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x18\xfd\xfe\xfd\xea\xa3//\x1a\x00\x00\x82\xfd\xfd\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x15\xf6\xfe\xfd\xfd\xfd\xfe\xfd\xe8\xae\xd0\xe8\xfd\xb1\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00T\xa1\xd3\xdb\xdb\xfe\xfd\xfd\xfd\xfe\xfd\xf4E\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00]\x8e\x8e]\xaa\xfe\xe6\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x99\xfd\xd5\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xaa\xfd\x89\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xdc\xfd\x89\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00<\xff\xfeP\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00]\xfe\xfd.\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x003\xfe\xd7\t\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xbaj\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00 \xff\xfe\xff\xfe\xfe\xfe\x9d\x82X\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00 \xfd\xfd\xfd\xfd\xfd\xfd\xfd\xfd\xfdl\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0e\x81\xee\xfd\xfd\xfd\xfd\xfd\xfd\xfd\xfd\x1e\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x008\xfd\xf8\xa7\xeb\xfd\xfd\xfd\xfdh\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\xb2\xfd\xb6\x00\x1b\x86\xf7\xfd\xfd\xd7\x0f\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x82\xfd\xde\x1b\x00\x00\x00\xba\xfd\xfd\xfd\x18\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00 \xfd\xfdz\x00\x00\x00\x00C\xf6\xfd\xfdj\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00u\xfd\xc7\x19\x00\x00\x00\x00\x00\xbb\xfd\xfd\x94\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00.\xe1\xf5C\x00\x00\x00\x00\x00\x00X\xfd\xfd\x94\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x95\xfd\xb9\x00\x00\x00\x00\x00\x00\x00X\xfd\xfd\x94\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0f\xe4\xfd]\x00\x00\x00\x00\x00\x00\x00X\xfd\xfd\x94\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x13\xfd\xfd]\x00\x00\x00\x00\x00\x00\x00X\xfd\xfd\x94\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x13\xfd\xfd]\x00\x00\x00\x00\x00\x00\x00X\xfd\xfdm\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x13\xfd\xfd\x86\x00\x00\x00\x00\x00\x00\x00X\xfd\xdb\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x13\xfd\xfd\xe6 \x00\x00\x00\x00\x00\x00\x84\xfd\x9a\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x13\xfd\xfd\xfd\xe9x\x00Jdd\xc8\xf8\xd9!\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x9f\xfd\xfd\xfd\xf9\xe6\xf7\xfd\xfd\xfd\xfdm\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00_\xfd\xfd\xfd\xfd\xfd\xfd\xfd\xfd\xfdv\x03\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02)\xfd\xfd\xfd\xfd\xfd\xfd\xfdw\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x103X\xd6\xa5c\x05\x05\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xda\xfd\x0b\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xd9\xfc\\\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xc8\xfc\xfc\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00O\xcc\xfa\x18\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00O\xf9\x95\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00m\xfc\xf1\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xcf\xfc\xf1\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00~\xfc\xf1\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xa0\xfc\xf1\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xe5\xfc\xfdg\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x005\xb8\xff\xab\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00y\xfd\xe4\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00y\xfd\xe4\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00y\xfd\xe4\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00y\xfd\xe4\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xd5\xfd\xeb\x1c\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xc9\xfd\xfc`\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00y\xfd\xfc`\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00y\xfd\xfc`\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00@\xc3\xf12\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x15q\xc1\xfe\xfd\xfe\xfd\xfe\xacR\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xb7\xfd\xfc\xfd\xfc\xfd\xfc\xfd\xfc\xf3(\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xcb\xff\xe9\xb7f\xcb\xcb\xea\xfd\xfe\x97\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00Q\x972\x00\x00\x00)\xc1\xfc\xfdo\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0b\xd5\xfe\xfd\xcb\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00{\xd5\xfc\xfd\xfcQ\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x003\xfd\xfe\xfd\xfe\x97\x15\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\n\xd4\xfd\xfc\xfd\xe8\xdfzR\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00{\xdf\xfe\xfd\xfe\xfd\xfeG\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x142\x83\xd5\xfc\xfd\xc0\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x15\xa2\xfe\xfdf\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x15\xcb\xfd\xfc=\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x84\xfd\xfe[\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x15\x8e\xfd\xfc\xe9\x1e\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00)\xd6\xfd\xfe\xd5\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xa3\xf3\xfd\xfc\xac\n\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0b\xad\xad\xfd\xff\xfd\xe0Q\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x84\xfc\xfd\xfc\xfd\xab\x14\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x99\xfd\xf4\xcbR\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\\\xc0z\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xcb\xc1\x1e\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xcb\xfd\x83\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00f\xfe\x97\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00f\xfd\x97\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00)\xfe\xfd\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xfd\xfcR\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x98\xfd\xcb\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x98\xfc\xcb\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\\\xfd\xd6\n\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\n\xd4\xfd2\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xcb\xfep\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xa2\xfd\x97\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00f\xfe\x97\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00f\xfd\xc0\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00f\xfe\xfd\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00f\xfd\xfc\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00)\xff\xfd\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xfd\xfc\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xad\xfd>\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x003\xfcf\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' +p13 +tp14 +bg0 +(g1 +(I0 +tp15 +g3 +tp16 +Rp17 +(I1 +(I32 +tp18 +g10 +I00 +S'\x07\x02\x01\x00\x04\x01\x04\t\x05\t\x00\x06\t\x00\x01\x05\t\x07\x03\x04\t\x06\x06\x05\x04\x00\x07\x04\x00\x01\x03\x01' +p19 +tp20 +btp21 +. \ No newline at end of file diff --git a/resources/pre_trained_lenet/lenet-model.9381 b/resources/pre_trained_lenet/lenet-model.9381 new file mode 100644 index 0000000000000000000000000000000000000000..4a994c96a4cb46300c4b6e3e184548242d267327 GIT binary patch literal 186847 zcmafZ30#g%+xJ!4v{}+3Nm{6=B%-c4PD)9eqCzFw7t+3Hp|nVglp=~KZAgStU2~k2 zsO(!wwmU-hEZO6`a^KJWJn#2@&;4D$-&}LfoUVe$%yCZjYmRVu7-y{2cOVLpP+~Uk3g&4X4g|6lMjL%&iXLUlz*(X zAUG((BOu5p>`~rBjeyg&(*!vJHXPA)!QPR9KH(9ZDK_C=9)TWuUcrHZK3)+4!9jZK zA|pIJ1AX*N!ooaaOe1}LeZsi>ehmMy9}my)h%gVY2)>{ue?8j%?qgo<_51C+=Jg74 zB&;}+o*rInqdmgB9Rt?+SaBqLgTwxna^VQ`8wd!p;fVVL`SB%oj0y3H;7qsScdPeX zDSCe^Meny#^dchy0>kyp|1Qn7;RyOfMQ}|2p-zNPPj1) z+qHqF{K2SCxT`HPv+VP_VN8YaD=z-;75`MB?*CBO|5l;;U&>t=80-~R`1V{L{Sp0EL z|NoBDj1Lu`KL>lfTzUIkvkiQ4L@hbetHb%SdieR+`TBkro#`E4A-Pr|>U!bb^*6B#xvply@;l&BZS%YRP}KUat! zRQN&ZuOPtJlg|HFPrvEnFWEC5EY$2XN{i=D#(!GUzXTD9@(7F^W}p8Ay`AS8zG-pS zhYZU&`nO~YCH8DeSN%isPuTFK;Y1Fr!T|JDS* z+u<=v@|E#d0o`GK(EY1G&XpDWMu%$!{%hVOhXwoZ^Tem-r9R>Q9w9y-V;j$Ae62GR zAxt?E@XI7;Dhncm&4)@t4hX|qy69E7!;)b zS6Axl!z2A&<39ix#+mhxa{iks|Du^c*Am;NgzrZbxst;J9&U03UpXAkhjP>5NbvQI zg`DABiQzw~;lI(tf3m}W%EN$|J=_@fqdc*4$5eIC7x8=Yb?l7a^VQA)?cr0#S!Pj zd1k=6Ki6jQfORwZ>EVPWM|7=ESddTPqQ9S(TXIB#EW&~#L-@!J`n~+w^P$NvDj8(= zdr|r;DHP)2&962bEqCEag@gr%c=&ll_?YrH0{qtgDOa18!o#QfcZ<)8c! z5X+y+;GjP%4Q~^|Iov*O^EP=!f^Oq?!VGD zzo=9qpA_bE{w)3<^v%!V2nagv?ir3n;=}a#vG8AE*sg}da4)}q|H=^fTdbY@e1X45 z_Pg-^LzLwr@P0-*xYhcwGg6#bmts#WVx!R4@fy3{X&T#TYelvss1fg+O4K}aleHQw zgdFcKvbSI?yb-fw)46+D`CoNlas3*1l$sK~62)Vi-~5D3u0Of2z7e>=ip;DpYV2Y~ zT}IIQ;iZ=2)g3-;sFMhpoHUBnIK2-~ejJC~@vD$@ ztmS!T7t?cp4DoseCegaN$9U}n18`boCOMnv4>#hB=>Cfzn5vmvCO&clv(sK2qB^;# zx#kf3oOYq9dCOajGIM4GFWJz`pCcK~g^G;h^8+*^#fqpoTmbRJenWmOKPc@e%k{Wf59V$!3!Z8W{8BO*mvy0QtXf-g7%_NcCf6S?mA_+Ty*Gj#(m zd7mg&>{Mm^-jBoQ+F5XuTSbfFc+9eoGZ>>ohK$b@3tFje#02OGGBW$_G2cZXkHQqi$}T8ck36xoEB3k#sjeBR2e*z7NpYGqjAA?WwePthTEop zVijXbcoWQD-U98C0wUB(^CEVXPk4drK4TlTd;Yi+1*xA?#W+G4MwjZI8W8Dj# z0Y}K-pdQ?CJHW^<9EHl|lj-XCAa}}hz_Duq>6cRA@%rYGzE9)e z-db1aiu?!?RZDrvK1(qZpF^Q67itR#qxQrEPOXk%mqoXL_R}jc!!nJ0PT$CCYjxol ziCpeGy=+wfwHmALg`>FiG&sE_mQ26Rup?tT@L;kF=sFodz2^d)aDD_9JzIe-N*xfR zdIOFu<;!vc(0Ggv#BDu89P>@-ovej0r%eLgCKu3GU+%N2S6!uUPlMCyNA0OI?V}I1){>!J({*)`7r!o?2K5xgVwPtwkYZ>lb zKLLN7UXP+Pmf)6c8Z^CGj<{NSq1f}eDDh<`q=bBj_VJtPiR;g}3)-iG?lm9C*wqU^ z9wsqLuQXx9(_q*=WC&*J^9aY(55l&2fKBrw$QIfHi-dnb1u2O2*r!_8&8$=(~0xbNU&+^}vZE#0>q4=6VBg8NIMpBKlBn>0Xq zRZ2Me;!Bd)KL-W_I~YB)R&rbJBWttmLxp@jy%k^pJH8?ff68LFHsWk(k0u)>ONxV!otE32K3*9^9?nxe+oD%{47 zdwzypeIb?C@JpMSb?7SN?9<99v?>~3Ym#DwS1f~!0S`v!%S-a~h78DV55QH+&tXRJ zRx1AEIHZ5dfNusCh#n=diHu-`gGVwW!@?Q8&%X4_LSfSN!4v9YLYa})2f_T*d=UJY z2Z>e7A>;U4W?X_g^?kg6>v3-fY!MQHH^Nf1;ckraz6HYQdH(>6E&a|4O1N^JMbFisMAi5+f(AX@DU*Cr8?*I9v8zxZ=A=Lp=KE)U}G zcxXOq4?Vx^2_xI%2HNqybVqm-#xFS!eQB9&N`TkeOSLiOSHuONun`k58bu4NtP1!2V17vT8d0uJyt zuo|t~vBYN_iitMhiFKAp&gP*(iz7}xQOr)+x1`D5SQDlv%|z2J`l#HdgKj4gs~p1M zcv2?0_T2)ryS4CX!Wk4tD`>I)gZE5Op8N4oAu24frN2CsAarOSj()3%cK6iD644~` z#`ik8ZE8u<)|SD>-^mpWY#B5eEawjaxn@^su3r5Wz5q4B^HOrZNmiPMV z0hnjyfbsc=KWE*;>{6q(d5fVuj)JRzHw>Ua%Fs8_ zZ)l~=4BYu+B!(TgLpRPvtQ1v7pSu$Hy(k=0qXv1Kzs6O{N|u!k6iyVB921*F6R zY4bE=*r`_sV`O*HqHHZ}^|ztL2ky}R>Xl6ozvQ!OrLNenU_fVu+2D+C0+`m;iAM)@ z&_v50TPEJ5^@S<0b-^=6POOL&Z(M+DE@q>yvLC%|wiksT8IZ)xFk13+B&JoVW0}V} zR${g#-CFvdTnnj&y_xDXRA(D9`lW?MU7VmVf zIVuJ0!SpAo?1=U-p4@>Wa6QBZawqS=t{Hi(to$kjO@^2 zcNM8O7sbqiHkx`?8+>zoc{>B|;ekp?qA9MxJK6BlSkL7ZiBDez+pXr4)3c>;ytgMa z$~+wQYtO~#q-m%!<}^eGOEPmBS28N8a^RbO6BgyHN0}2|P|x2eyFGnRn|7TcA_J+U zF)5HHdLzoFY-HsvOyJpgAGq6PhEJ!J;#F=R%V%#WzNA9fd>iU<)E*no^|GQXxbQ@B z6=V6xcZn8vUm(*gVPC{qXJv9f*`~DHMDvaL;aeW zRQA+s_;5~^GJW!p(YXS>H%>6-YRf^k%T4;FDiHUajselK^W?lkG%6@-K*72K_!=Dm z(TmbS$;OLu-{1hZly#VKFF$ezwD|j(E%V{W$@`3k{SG?#WFno~x{a=Kxq#i}t(fI; z09zK-pdIHVIm9~%=}Sk!=7P&m_o)-MBqo8tty5&=v35pd|75smGlHre)*^{7CUc*C z9m2hp8Yp?-EpJE0ah_T7QM3!0OjTcgM?-@a+?l2h3tb;!U*dhNKQtXUhcn2pm?B=+ z`hL`CS4C^(v;4i#WkzyM5YT2>s90oTd||FUXkFOFEBJ7izG>)&PxDv8oRdG{oWp!- zx$7c3vfK$W=AH+h;Vdv4c@~chu+`uIma|NN_Z8mUg&f@NIx^xq73YD}Pz{!m((D6tp z7CdQ&%s3agnE#C@{c;NF(dF;=UW%~_Ef4U`cz?JxLK31%Ps79TM6z4-E=1&R1&1;i~xtOvsNZ;)%#;dUb zL^i}5McL6f*Wdx3Snq*7OW%-bK|Z)@#zL--fg}ugx0e3Ab^|S!6oNEYAEvlJ zgsj|1xHGf|qQ*U?RY?vw!k_`3EZB`s#s|sI#j9z%yajnp1j(1vJIT7jr`)o=X;gSf zAGN9{QjK}$q%5fhX1-M*TlnXWvWlmSqOKntY4`;bubSh7W@X%DHy)|XPnfoBfC#VJ zOY1`&ahbL+_*JA5!FGL|q8mgubn2ktgZK1({&m>)(w9yhxfkl|@*#%TL7ZnlVU%PX zz|u7oysXw@<)#bJB({=tDcq(VAMGGhBpy;tdZ8xEl_toXfkHobD7H1BDfe2aZICN^ z&fWnzIj8B9$3MuR{}nJ1)kmLZUGR7y#7H?AGHSO%7~g&sQvT~W>~(5o#;uyd$loH& z#)Ncc##0$sW+@HL+$h32-rRNf(SWn3Q#Q7ojFr!F%#tkgpRp zcw}BXMu|nEbyNg>7Z;4XtqZYU+Y+xno{AY-GlA243BrQ=!Mauk?&XG{il{Rjm@plj zpKN3VkKBNguwF*N%7GaZoC!{|rqfUF4uiY@5_JAr34;YPaBXf8_&il)M66~rj@GJ- zPAAP*T+QNp;ck%UKj_es??Te$N!0C-uqNRg5q;hpiM&+sKP)ufgY)_Mhp7Np}% zhdSDJR11o4FN3n}YGirQeU$BP#fmgds*q$2LuLh#8s7kNyBp!K^c!Zbt39*sbq$;& zuNb4Emqa|X06Kfq7@@k?&<|@N_3Q?a{yG}oR<3~`ldhu8s7au+g~g-B<)l^g4xOVt zmnS*Pg6Ii2V|9@RSySpuZYux4j4eO0USR@hdoq@!D=4#K`wy|>+qPlfG6US#{sOP< z??ql=48D232Rj8E(fj&c+-4TW3O$X%fQourx1a%SuWe?>D`ex*=`#3fO*xoR_ZF}gPj?o1Ei zb!b@94gF29SIm&e8d;&jr#$ks)&R9yoxwc695(RQW2o90qpu&%lQCmtX|WK+9b;ZV zSc4;6of{74wZlMF_YR(&>`r9H-9(L!onSC?4$5pfj13wQD7G;h^gO1MACJD#_ZwQU z%vi?AHGMYZ>@>hh=j5=gCXcr0#ls1?G4OexEIhM_0^ZLcW|ZPjsMxWOQRbnhLRZlnVo;RC*>(@lZ zL*~Fa$wdpZqu8O9f%jQUcH-P^FfVsK8SE}#wNKAu?QTe;^x$Z6u*e1eaPPqV#&Xyv z8bYTTCxDHS2h>Z{a*HOL z#mux1&mlqR6mA~KVhw(3psmFb@>)q3nw3{z+6BI?nP{<2ZKv4@#ZGY9qX%{+NwJ0> zTUhs#$?SNs7W#3<6yVP20|^<54}9+9*q1ALMfaMyPvevzwM7mRCRp;%|914|>P@&^ zE)VQnd!TmaMck|#h|_IPVM3G>oLM{)7M4Ci>!T;oVt)t5ZJ7%LcKPJ{DKASjq62?2-jSI7Oof1dFzEExUlTkL$ z;PUQTVvL_TOwc(8XDi;p<=Me7sW1Yk_J_iVRueL+MHzSG1)=e?Dw23(5v0G=fkZ1s zM#wE6%6{!+L{kr={;6wFu+2Ep$3PnHzc>a)pHbvmDUQXY=O?h)yO_RMav8lV5+KXmA0>-4utE3(NY@ll zDk(_zg~eleNGGeI><6QiZ0Y=;R&JzVI02*S=gp-v>2{CqhE z4vciB^=Ce#{KpcSaC<(h=-7b^7T#w^MIAuUyo@_%j9~e$N_N5rE-SG-59jAsvZJ@$ zV^x&C<3R6TG+=|ErXZaR?x_Pcb{D3_TmYlz99TIf1Ed~UW9qq`pz>lmnu+V6&p;^I zFMJU8hc6{DKN}%qQW#zNIF6TMwHV5tibH3TBJ4YSfvT?kh9)-2@JwYp2~gZdHw|P% z@czxb41F^=f83Kcx298vLs>XEq8y{jZGlc1Pou{^!ZB5rc=K#KeR9$bs_x#w$lHAG zd3v1QKQR*IW-S4&CM|Yc$s62zJ`GK2B9@3vVug=h!_tWvD4;*c-CUsr3Aqno*62Rk zN4CTD+g6aPW&qXop)`77E>+d`hWZpCh<*AEr^w0U>xb@S-}xHem8j(?u}+H}d-g7C zJ=2EGozlj-H{U?jmlo`T>+4vLX{zjI*WLKwgb;P@A4?*})$#WDdQ-swC#W8=8_I4y z0fX%kAkXrk-Z7O?c%e>(8>T}R-)A^woH1N3cmQW*{89aF8U1k5jC&=vj2ZiC4V_vk zz$%Wpf_ZI6F~~ZRQhi?(k1eC4`FqIYOTOWm{7z&Y0`Sgb16KG~A}!fA5t9|Hfkbhd zKJz}a>M9>tn^|eN<)SC88GV#qwwEWXo~SIqCvQPo!v9ejzC` zw}KI|=FB*|AEs;KoS9xm1g zGcJdknJgt`#&caS&Nc7GGIvXwkZ_v>{=Cj@iWh@0Egj4Yj>9L_4XmJ+H}8txEPUwp zlO5B(lqLFe@ljDf-rLv@`#O)nOtVVR4D98fTYMou(}ISW%94T0X|%*Pk-XGC4R1t0 zg20|J@YKVFo+;3PPU%XjGdJvKbmhvRwL*rzbPi?h&VFQVPU+zrp|R+H%b%o(&%n4$i*;q$!pZ#zl^rtdy6j) zOvF$2X{?}|4nBDv%#JWO!HVb?xVPAi9{e&$6{d#Ja`CND6CVZ-?$klzfg+rxkOo_7 zYN34j0Q}4?k|epALIa&b*WfiG%JiuJ{+^jW%Zt^827xFPxe1u9k6M zvx1ROS_<;Z4nS_pOqgl;8PpYQx$jJsc|X%WvE3HXnq8OB9eum!tTsY+;!sx7i z!YmFofQ1SgWCQOOiYXZ~Iwy7DiD3uiUfu%6PYK4n?uWx`R6*jkCbY$0XQZawVZ0Z?-nqUTO6fv_2i zc$@JZQ41N3sdwb?<;4+rX=W+yOnKcT_pTQE7mdOJNg+&|5RWQB^7MwD9A4Rb4n?9W zVXoH-veKiHf!I1`Mhzd&)7-%J?lX9h5zH88o@S=I#lyze>b&&jS5TvY;pN*zpo_{G z`gs?jORSVg=^S4?^CpjEMa?9;mL$UD(qkC=N)R_K3Ins{&17>%DIBkP2f~}%!R2EF zytG#&-oM(RU=D|FA2k+&dX575`VA!Qe!)hQsYZHILbY!+JT8FONpI z35`T{K$&~LO_wSLd1C8330A^#3$Eaw7rW(h@bGy_++JwT>i0{5kjQP8)AN8GdD#~q z=xk?2U(}PT-fv(gF`wLzu%u)R4W7MYB!TqE|ysIsOXxe=emmk~+SuGQ2*+^+DO4!eGwl~p^rCt~) zz8%+!q+r_JNW6UWDOTUrBNI&J$#n;1yuVZx{g*4z8pT@Z_?ALMe3L*VWC`!{^Jp~E zQ77N(Qt0=~kvy{!6^w*V&{?gFvOU7&dFVrW=K4Zz^}c*y4dY3&nws;RYHJ#BIRK?Axk^X}*kk?pPXAXUBr$8MWV$E>Y~gFPQvX+OlqyBwT2?f|T3 zIx(@Q0F!%W!0d`xm*f1rNS&Zx9A1AR{|qQ`UJ!WsuR zy5(g^)0GwKd|&29dUf?wQhE1O)4p>v(Er#V@y+BCG_1o(Y!-d+-j$BsF^&9j`ZUI^ zt)}~q&wz^WOF=Bg8=TmyXyUbiY!`ATH%=YFEdo-YWnKUoA1qNfW)Wt3^%5gVBrmJ8 z;fP}bh(3NzHh;JQ8519p6LMd%@UR&#-a;L8UO$GSJS|KP7=)(#HQ*r?2}ezi;DqIf zhEG1=x*SDp5xIeD?~8-C&mHb;Sqbjr8=g4j{UEnI&Jp6PqUOOdjv}A4pTT;i-&v8qvP>|crV*lsim8c^ zb!046RC@rcjyr&z;89Fn$v7o&yoU%I=F-Zjm$Xdc zF0LQvMLSK^$$0ab5LK|1G`sf0guX%)+;@&H$a{-dTbyClClT7Q<`p$hxz9_opA4$% z(y&VZJvGho;=X;73>h|SxE+Z(^kEf)CTDq6b=6Yt#l~x}zxh7wDO?TXW9`V*S^9K! z{z}|qD1$4)4?}U!DikYO4ItG8q7SO+;L#lLQQLuOwNBJ=^%U-gg;Kno@9yC|nKVpk znnS-!>0{o>1ekH+CuSQ=01po$bqwo?odHxw?34>nBDTkk2#Fd(Si^) zb`bLhN5lB<^7N{TCRf|%JxscOg5>24g1}oXbnlu^o*Za^ZSP0ohNadRdyK^@FKLnz z`Wz-o9mFZ-irh8JKfx}IFqA#@9X2hS3eHIpSa8k+OlO}XL-H<^dcLD(`xk*9xlTQ& zx}wao4Xk8$UxE*R$&1X{DLAWkS0(>`6t(lk9R3|>vn zK5XavH(lsjktf8X(GW&#YbV{G0@0R#Ut_*f1bsH65$m%RsI}+vrt!>H8Xx)+OLq#O z_VYLB{ObaS3=M$AxFTMWN8M9t#kq?*SA?~~dOjSxnfBogK@UAds zA1kMKTOGNsX2$R)*$$ozl!0^lE}U_-ANTT_h<8c{%!x<>;gFH&v@#H5{q(rwgf7D( z?@S2ce~;W!%Kv_UB!YU}?jsqzl^83074zj%k%axism|Nc*3O%p^jiz&zed95C3on) zOUuZet}ZlLFp4DCe=T#seN|va6AB~Cjd&rh0 zi=Z^!8w^X7F+t`7Jt>;kbkXn&1~|6ystX3`A^ES6JdnuMv0sJ-Kl~v%=N`KDl#^p~ zoJiWG`?TSU4XqoONC*1`LA#`vHqBPU)m77>VZmLJ;Vwn>_X+d1&hA0$%AX*UF#>CD zAH=wH27;QV(6q0Kpsw0S`k!nEH^`?U2bD;{o2T@x|83X`*+ju^3@rL4h6<5O>8^m; zT(MEg5KuZq4&|`4`jh}puFWN#f@L5$O%bC>c3nP{Ff>!ksax3~8Y};iHE2mKU z!9a<7`BEzcrrXgC=`mQec0W;%`$n4tWg+YFNnEeU#Vf-1>D|Y(Z7PG1MVPi%h?ayn7M$m=bvvTps!I9&ZyvGyWdy6t@#%45Da|H~;=k zL_cm0+C??(%Zaq73#`+9ftA#k(u?Xac0n+?Ge-xtwI31L`Klled1$pU6;oG@qdy9c zkq+KwdV264H*v=gVza84=+UZ^NqB z#;E?FkM_!4fURj?a9U9gEeah#pZ+7p5{J@=U&%sT+|xq~pDTbXJmW=7ZH5f}OSpHo zAEc}6W4ww3nIT?4^9p=GaZMGjc;`z?!!JPySm5#thk57cyJNA1E&m%p3|C&Z6ceD`<2@FGKW!pQ*pEa}%MESHWkGYwCahVcPRXxU@Lg|7I~KI! zg1}eUDzpn#$LrA87CBs(V$01kn~quzRkXM#3lzV0!Yr>Q5Zb0spI=utRx7<^T(LnD z^UD4(%Gq;-(C9cUelP}&jwz8O?izHv7KukQbGbQz_T*mNJDd^eNV@E@xC)n?>59Jj zxK)<#v-B**J(2~e+T;fQ7PEf)6GnUtL;0;Y!9xB6Z^Wf9;Fw~F1@FTEwuzA7b??bG zFD3fmd?7@(%_I{=%}I;ENFpDbP6FOpaYyXY2T6kyK;*251phAM=}Bc|)t*!g_7Wm8 z3sp$l5e;;dd`x^o^FdSl9<`iu4P<-duxiMFj!x4A*Bo#1UhNK9I&vQdr5wg2BVqDV zk!(!_X^xU3&(#NquYokZx3H@!{f)5s$ z_4PI=R%?)U2Ns;myot-y*@U?DK%;*Rom6E{5_2zcpYBw`oA0MLwJ%u(zWGM>Gr835&_Q$#{?VB7S&FND zM3y*?TmeE~F5;-#5xh@7UZSn?&pCpu&n0-E_n6_8YK$AQVDkqQ%_+P`^~sbS+yY6CPw=LPVwIE ziN)pjT`>D*23c|a0iCvHh-_Z{k|;%8#f(?Y^r_lTtoo8mZ$$Lb>MM59pfTVBi74EZ^3h%W;+i|{5X)KVn~;n zzvAB8LHQhdgLhB*4gI95#p@W|Lxp>mqId0Ui1PA-!{%Dh9Pt(x91Vlu*EeCok1SNU z-AbP8Gq~-1DxP~TLyyIXVwKig?%Hz>U~--%dNX%(yIW>cwNVq$=!6fLWlsjKa0|$b zeButY74h;ySrjEas0Q21dW{Gp@(4Cc#V(A1T#m~Tv8Gh@TNRWGHp0~TY2cQ) z1NSU+Hnx5+lN-+e&g7Ey5cTSmjUP2uH2L0tj;ob>vAlgUdXc^0q+eKvmQ^a%-_YlxQrY@dxkWh@Jr2sJIkCdJ`_LmgII!8p4itZ)`Z8 zhr1TJG@bd}L3LF0AZ2?Rt~WNPR^k+E%2erdM@qkLPQWu_9c281mryXq0Zv`~il%vQ za7=s#L_FC+>mDyf<4O^_SHG5|tXzYtYT1|;+W|hKTN>jBqCkJfYDjTzfb5<8`wu;j zVgJR;AbxZ!-E4fX>0E0D74qcYWQuu#(hsMB=AO-%z10j1`S(So9~98rVJay4J&yM5 z^T&iM_P9PmnJ$$}z~Xl{2=e9t5z9!|vIgUp+NTf*X=LWcr@EcB?A{XCqr)PS8j##cY3sKI__MT2QDvs zAbHhABKOmjSG%K@njQWK(mQ~*e|id2kDQ<`bK^-|)?>2x%o5(0$?w4~<}&Zns6sf| z&!N5Zy6Ek1Di9KUhUR{|kM47Q(5^m_tiHO78zugRm{rI?_G1;G>#y*9^#;*qg)v>1 zvjpRTAc_iZM;H-f}V z5g7eg7-z;5W2S%(>Sg#5=ksHEeOIG#)4Y4!Ldi^CVRt!wYB~*eya|A_`Znl(B>)X_ zi$JA2j+Yj&opjm*7)-nZMJugfaeq2oO5ky`MmdA!*|G4V;~9By${B2?yMzDJbF{o< zfaJ&eBgD<2XCKWYDuK2brD(x(pKXJp^;~+Y_9Dnssve0}3JIV^I zAp86&&9T=cJ0?9LJu9^VJw9WR!4-UQZWT332uAS;RkU~^f$(HIIyAFja)*m5Zw>>r zTp^ZUhN#6FRq{mJ41?{)Qrfzb`p$5GsGvV+#b66{8C8MIT0@Yjk-|{V+4N8pcwFtMkZD(}*O@C!Wd*+<3P_PrCp@K7?WX?;g3?M7n4 zylmd#?k}WrAPaL7uR&mPFSo?2o-3;m1LKG+e0@0*8haf1_ae%1bc_kwU!Q|{Dq{<&6;LP&d=)gaZW-L4p%8lny_t_PC$xoA(=zM_UzE)iN zVJ+lDUZ%YU-{@=o4RnujHfB%Si+HaVCu(Fsp#2TxZ}8D#RT|_3l%QCO8mSLh3LD-D z(m&$bK%Q`M){+%yyUrEDthm&YW@h+$L?56Ub7Rg z_9nx=(>mzo<3P9X{zZIw_h8vKBjblIauCq`9c5-dK{545V0G7mG?p)bH50xV_sVud zT)jOk(=jHQYUvo-n+O?Jbue3CFAcb<%$>JB7Rtkfh>_L}()Xecrn@{N=S#aNE9(5(X#2fqbo;zTxYykY{6w8_!dw-Umb}<>(DVdO zf9VfP&Q2xFmH80K^@6<4UQk+}0GD)skn3m5;h@ZV>&QvJX$C6oz$qwfpL``w``#S$omYyEbk7Qet-ju z0|lCTye8AE+ZM#yr5jZD&nNE;rC}NWoX}a{MB8uufQfL0-d;VQH)2~gC{NR)*SdRY zx>1|4m$V@1XgyAE>Q5lqhtF{v0y6Y{V%xj^I^x4^o9U z7;m4`#$D*E0~saL8Z*wyG$qC^1KqNt5O+5eGAD7NVW9|RMJbLN>uo$Bpoq#p)+1A& zga%V0>82M~&~NiATD{^1NLRVz9$____G~^#_iVvL?sW3me@BzaEkP_g=nKg+zR*jG zBgy8j^SocaZ(v^i3{nuZ4$@YSBbT?aP-$4v#A#@!a_8-^Uj8B&KsKk@ix0yQrB|^=tw;Z3kp_x|6!g4@rvLM()eI8{v|UH`w%dk*A_OPEJ3rG;SkyT z8BX;bB9yb69h>L_Pq&M~@kd$4V$UhBGOd~Wsc{snEIh-vp+Op+b025l3gG*?O`tJ( z4hePj1Ln{r@`dW)=JfmIOSdndXAT4R;(1yXJ^*qFC+JZ8IIt;nM4OKLI4a8-I;C`= zOu2~O-f|k69`uqt(-Ly}_z`rz`vs-!dvIFj7|gt6fYVg1$b-ihu&cxlTidLGySan= zA^r{3+++^MvwO(+XOt`ySpcE)M2TyWBF?Eio43qT`N8zz3tfxR~gxVJQ+TJd|>G-EBKuUrQj&PGtz*H1=m zm`~UCy3(4johWFQL(Y7)!qM};pq+RkoC!^V5q!V)I<6)zf1pOoAFm*nVjEC@G2g$S ztpb^I8tI9JO7zBsm2~am1S(iC1arsb@Wws+L>3L~fV8S)(r9IXd45jlyLu_=pZ$*Y zM{Nl=tI)XgkTZn1EFftK3#s+_TB_C+gzj#BP}pThFF(AF`u)dIFjj`wxO5f9F7Tj( z7x})A@<<@9PcgYm0P0qhQ#&RJ#pY>T{N4s|8caT`YbKE*7=M)n`|r ziNk&_ckP8Sy?WGic!Q_Cv}mU36tbc93>1&3gM&{Y_$=E@dAH@D&gUdb-q=LU!+Jo= z6GT%q1*{94U{i$%G)h|I>?@yP{;YLyB)ykZUAP5V{Q}^(KAra0GMF7a znS9K&gs`9U!LTfX+^lG&=@(DYgTgDhrA8OVh+g5o8^D}*n?d;HQOMq0Ow!D*Q-w>n zV9oFVdns=%PE|et-aE%JAlnUUwbIZbc?*at>_iL(e~q4KcGT_i zG1ZIY-Jl!&u1G*)+y?`DuakrGglVn916Z@o7WjsT*q$@qko9&p)45KX7piQ;7%HV> z`4cfBY@&|swn?PpcM8V;?t#QwieCvE`v(dJ+P4D2eM{qVB9lhqPAlWG+)T4C!Fg@QK>bi`D`I9UPC3t4K&TO z4!pn3VD`;Q#D!(~u<^ep0^6k5qC7{0uhB5r#b86OkpXjb~pzr4IA)O)^^K z%Uc7q+kBLH)x5Xu$nbUYkBnl&)Fot=zYUzq-wK(zVHjR~3MK#kYPayL!t-WZX~7~@ zJnL#kZu}f40x$d_@vJV++Aj_VUxl((|5=mUg&7zvnDl2zzGQ zk;d?LT(WC7h|Llpb7yDahFQy*zg9Z1)uxPmQX2q0+zGrr4Jd5=3x zV~u-B@UzM4KCot8+UGr5PQidiVD{Qx%A@1wuYe?`-ejU?v#K@74~p#pW;)MD&D zRGR)onPF2@b?m{s4^n8(F_>vluC&5Ui+s8AkEHMHq-{Sop@aP#P+XA6EIiRkik9>+ zt7p5D?a`}AG(X3MoGZrSrC%^Ac?np+CKPQFXDto>!u(?kK=#{yB=HZKYeSM)e)Jf2 z)R{4FChpPwe@DrSj42TN!330rQ$UFCB?t`ZVw8jfhD{7ZSymE4Rx^4 zt{#G(CxW43Asu~HPv!&~(1IQY=luCfdtIE^6V-RIt!E~UIVVcvT|8-^%{ZFRY-BK@ z3#YGkWrO+LKvh@@vqU^0S7Irw7F~}AchAP@{ZBB^YXK~nzYnxm6kw*%ZaT?u5juTa z!0ib)F?%v!fTO!4TYGUFw=8&oHKs?<;g1}}-&g7T1L5d0XG7~{gLqoL#+H4i@s0He zd5G&X04oROKxr<=$+R88;_#0g!=j7|zGY|}R7==vc>qz{!OhZ(cc7)8!8Ltw`N#v< zmNCJxiXWLNrG&l{T+T6pTL_=-ZOF)Jq!Q<5(DK*cslw&}=7B^N-j^0YC1Y-y=&TO6 z&Hl1YGlt0DB_&uPu7i1c0=RSYW^A)PipPUTsOwBVdi|6Td;f_kG5K#Z2|Rclr35_K zALe-|@|us@#y*9$$9B-wE*_8^&gD)uEoeD;23fO7hUUz4CHKM(;c5>-oWsqh&N;4v z(+07O#0xD*L^;486)1ky7nQ^K(3M+%r?1dOqnRqi-^vvZ+Wv-y1s>2mT0-LYPD5F) zGj+am3BAtlq&`d%$fR_`-jFO(Jn1tTSnR}DYR;erl`qLw>qJ~G+ybWp^wH?fAnNW+ z!=8hiY3sLeIJ|K#-N^H#i#|_4jNfiBZC}LvGL)w^<~d;bEt?qkRFd4tF0Oz29~M>V zfY{gpb|UB<$nJOv2G*l=kZ%BQbn|1{=|~9u)r+$}&qupuGeA^T7IXTHsAKSCCc*s; z_{x7FgA&0+Nofpn%ts;f=M~&Ojga((7lDaVgZi5Xv3m3~Wy5jQ=1%3zg`l|1L0#TQ^eo zp>X8As38CD2C`459O)1U%fvk$;o+?zxc)>2x?baE zr#?RI-(%O(gWUX0ary#s(Y^%oo|dEN_7sLUgTcvdrsye{P8FW(Lk!<-jQv$hiSaE6 z;8+0sJw`&#l;Vbxg`_U^BVMK>btl2x?f{sd*AIal zqZ=`|1r-xg!JV&*4*l0iBQuYpd`t?&tHpxJ-@hpBY)R_d0+{c`fwjwgV{~8watIW$iZ6BRmHs zUADXU7TFc%3ejdNyj!&)^#1)eC|b7;^l2kNaUIS-{S3vXaBGwe@)-WA6VyI0$LnX$ zgPW!Tsku9y80chRR@Zd&%?t&P)io&lC?91f&jsQ z62coMW0GkZ3LH*hUM-qVZhYsF!RM-&{vsLgR{RC4n<`Lpa35wlbNP(fH?Y&$4}MPO z5S{2pv-BR4nq-cT;687m%{l0%=88czUr6_(mmt3KKiGO?FIcB|K)mt@+WC)wy4pMN z-E9hs#dx&tlp#6tHI*K#z5|bX8^QYv3)eY=M@n1|?iN~wM{hi2j?MQ*y(KDmiC>($ ze_jImpTvPL;8uGtJY^<47SL{aZPK_ol+2FX3)}W8kY7JuK>Tt?SnDMZN!P02U=o8a zwb9(VVH0_C^9ER2Mlg?+Rf(jb0gW>~)IQPNj59wj1K09;v{!xs@`+ujGxZLtr5E9r z%y3xd*8@1+2Fz3xhzqU*%f&oc?F+O+iVLOYcM>IsYETbbiaW;_!)b0mX+QZLZsXSU znvXuyIh5nq&9uSv@mIR$ln(9Wp10F@VRTe>5uA(~CK6o*sIgE0x2+Qy4yV#+<&PL| z$q5}K^~iFcFbtY=m5F~}4OUVeu(ZE{^%M=IFJv4^(YP?E|F;QN-;L*8sJ4d#9x@;s zwH{5L^i%7hAdWrp#7`0cNMiGqj9F{-+em-#1XRL}L;kzLj{q%p*$&L}8tN0c87YVyUGK zb^Y%b9Z)c)K5RLjw=Rc$t>Jj}egFogO$RdrPng+$A12wyb9~)H(9x8o-51xxjNE9P z2(nyI!opltebidJZ>7W zB>~M4z|VCF`#V|V{kyOVuaIKpQ|R2GLi%URU@Q|$Lrq;_%eM`<|D-NzIDCTx$=Xma zv<>5xE_aB1cu3^<8=zkEDAjdyre;6S(U@90DEMGVx5b@C?%qJ2>L99p6QOM)&1BJn z^C0|612nC_((UevR~V!t+> zB$W;2?3pHhdSseClXTLK89%&{OiWIty{G4s$ovPm!hIubo5^*L!w966 z2C>-+-|?XNHVn%1Cq;@Ialz+k+Wl7wcQucJ;FuR}+x3ERbM?RyeM{;j{*8Q{@e?D3 z;#gMX9B3}jq{q{{QFlC%`Srybo8vs0TatdrnSvPc_c?4*Qx@@^HGzJ+m2p?&eSDC3 z6+Le3V(e=eRDA1%Yvw<|{RftF&)iAoA@}1k@_auX%C*L}t^eV{Y6r&jWF(5taHj`5 z_K~RVT&5|u7WK@-A-&cV_FS)luA((i?%F|({qxAiJKrrvQ57ujZiGV{lW1J)UK*fA zaQkE#E_>l;mt3!6?o8vlYpr)-hyOz~|I-Tz>^5-RX8>L@8&U0fFbP%{BZhJ+;2jc) zQ86=d>PHXscaO&%c0FV;>ozEDPb5WaHqzZLqOgr!jl?dW{2X<{oeF&<<3}Pk*(0)PmUe1)1ecma?#^z9k9i7 zAZ+Id{Sb6!p@7GF(q>@=ty>1@6}SwBc?)Tty9ql=;$eeDd4lDamCv;4IvMEnXEM zesd7T6q?#UoU3Zned>aTqW3|aQ7e0G?IK=B!yr?x&`1x6RB#NKFPG1Ek#fIZbl+wT zBC%;5?4D$cYkz9s*6L+2f211us5%`@E2BeHAB?tyfx_+e2zefC!+<-za`6oAoqGhC zr9#x*U^mE~nS!&Y=wsxY6jVF#7PZo^ppxt0%gHTN-b=UMR7S^yc}y_xqH z_<`^KR@k_vn|PmJP88N|2ZJZ}q`r)E#$CuF8LQ4?zV!=iUXej!ejjFzMMYxaWfexH zy#aW~HbZ*NB`8|t0EIRmK=tb|+miDYZK4xt?^j!T`OtY1-mx5n$W)Z&fXtgZWf+xm z3B+z($GI)i^zDD6ur>HFdonf}OGSUvrza;dHP0eQL7)zOdv-JD-ZFtnZ3#3#>j~H` zc?r|u98A8_Oj^K#cKIEpn?BjnPjy}_^XD9f?^CCXjtp2>1xu5Y(vE1cc^>(Y&_}jq zO$JSE2W}>Ro&K8jiuNcLqKLCR&wKnk&EL}ku8Q+X=kOM=Rw%_o1>YF!VO!jC@HB=9 zCZJSxI#v(q!gS?tkkMa>?uMVRV0}CYFBU^4uZiwR(}0t&_j7KqPU^Ox4+RIFwQXCt z1w8z35T?2f9g45BR~^mhk58Sbo1(?mc}dgGj{S^c{d@Y(Dg_d6Zz1ba2Vmo>DDX;X zgm^C*y1%l8BwlF4-Ewm1p!^D7pVvi^ZbkaW*BQG+(y`85ls9_Ul*ydDnAy+gjg|R7 zNZ6iQ;J9gR}BIYqD`x!p1oWG?K5 z^lnec|54A1{4GG!AKk>}sW7IDO5^n-y|gD`7A21p=smU_T=`GaHPMm8xOy0(?aW~9 z$2at6_c(pfk_KXBwy?Hu5Bt4N5;fecFeXM3WW5<$>oiDyFYqIS!IenP6p0>=C`7$6~G9Lp<+(miMA=21=@3 zz?^5zG<*A2^mOQ=#~C5|$b?0EOAXwV=T3!`_^9$MCGshMC1!fAp@;r(++$2X=9(V{ z>lyEe(B03pbnb1;%^pMVu@B^I+(wL)p8)RyJMy=}1~%FsV!S@*F&@z|xW6-kZjVgF ztsjCh_M#RV3+3TVjgP21dn#_uSprrQk{G=+2KC_lF{0E2H(w3Ao4m2q;`wf@bI1n2u$)sN>5%uvMJE;GzbcqVyP!?{J0nZZB}r zsk`*Dl?bhuy$+fOx&7C(2M{y$E?8vr;)?zrP>75qDp^vT|FDOQs@q^h+DoD`_y}@x z-Dvprldx^p1XHoK17?Ix2hV$B?0+{)*zP+mu>D&S#$Zb0Q>A>(BEvz zkb&h8fGKO2ssRVJGAh$<2~A8n@(RWdWiO1j@P;}4O+BLgLs7tBRl&G z$mu_)kA>4wWb${)u5pGuvrY^wK2Du%-N5p=6mx$34VR&gGOKQIYyYH|ARY7(?P8|W zD#@kj%iDx=J{%`)$$`{quVwqkn<=S#|BM7n{fN*DD zaQ@nhCxi+?)65K~)CH5Vk!^(JE7BwgAhC)1G;Xk+$zBeRgEZ2%^EX%-%_ZgWPuZ?3r^pOV30(Eniw+%W!73ML*uL^V`t5`Z%B()mmRU5z zilW276St$^^c|T5>23^nUkX*F@5o?A1uE@H#CX5e-19LS>FZ$R=bTbzqDSas1=2QwyIhRO}H~%27 zf_mF!Qjf7sCfC7y<#9+lqQR&m? zhFuP#F_NV;^L{-oKbr({A09#Qw@i#q|H>S2T+}YRk^8%n0_dIJPpRdkwczEmk1m=v zNIg}r;(51T)ZMfh%*p~GMezd(YuSVXS!wKLy#%iBYyk>~qltCURkmkG26c7>qI|oJ zj9k1x+w2?YxM4mRF1`c#@+|nDo(?nGi^$r*1PrbkW40XRBQ;l4(C26_Zd85CamrON z&-pjVtkMPgXeN`PGYb#Dy};hzw1Orsb0S6D+Fm%Uv3)G3pIEmFGPm1TV%$7}JKXj_ zY|;)ii82P6dBtQ<|0EfI*v=k(DoXltE<(J46U~cU4RL>*z~kaFfIHm%HTn_uf4YlT z*mYFO}<2c$v~CM}J{>mLRd&`~dak`lx`M9kXTN6R!DNNDgYRWW0@7 zG^{Tssb3l}+;|tI30yx$Nf-0XM%(vl7PCphJ7L}}M{K(?ZZUP59H@SGAuUSNY0`KR zyW?aHllP^W&h7b2%l11%>c)B8|No8^cRvOjUu9y=YiacLkq5(wrS$Dj9XM2A4(pO4 z(7|U2ca~@av0I6TpY3=(BLB!O4UP#obeQnoaUAoQA2`(*V+31E_)bKT1@qC#2!aH>2S&fA zz|P$PU_Mcb+ubgZ?{`vBaqJ_cAB{v_oDNRsW>b|{+|l|41sT^+j`?;*h5rJ0eaQu6 z(ETZ`DyyX{`4Fd)9jJb(pS@qBgQ*^=B<9R1EOHJ-gR}c#d6ydbS}_IIJp9H^4B5e~ z?OveQp9o=Hp4b}v6r()lN&3C%SjM-LzMH5fUn*p%-bWq{{b!ErySI_SJ96z$>|TOQ z?@!Fho`HI2x8Y8IWwcxoK{LMpLZ_r#;2y$tp>^^|&8!l%{r(c{m%Rcb50$o$15Tu+w5Qk>}X3*r~?!#qZ~MY%qd6 zGXZG6Vvb^c0T3n4j}9q$g<&!rcYu`a7!`*R`v_DZ&875x?|Itgqh9Kek45yuSz<8me^MVTfgzJdaIs<$d62sl=6sjNw9!_`osPb@l z?Gj3duV5ZsUqgPJP9Z03hahZ@C;GDyxb;j4%r#iRxf-(BzZvq>TRq0?Jh?{?p1leg zN^KBFzR~W(x^!w&4k72ipuE9n+^Cj8_x-V>=1rnx%cB6Q`f37qH8hfcJ^bWPe9q!W&QeGcIhr)W*H9JO5?iW(-nNY$A+G}(|r6?IKm98pf6O8KMx*KZL2 zR}XR=C%9`%70Uh$(()J1Sh9Kys)KgWbVEnVCm`lo)4`Xh#8!Svb3~D`Yywq6EM|Y(Q+iV<&Vm{Xm zUlv0ycgG?p!ouYWrZ_ONiuQ7|*lA7`R7Ze;2rUT++tNwTJ?}?{bw_b)H@6p(@&<*0 zhfLJXT*#9YN73)kL9wls{Sd57I!{bty+eAKt6SF6?_*y`TSo^;RNe*!&o;A99~>rI z_#zmGC3C5>8~5HY`3*BtO39&I8&dF+%SCs;Cj#00Ab;v48d!;QJk>tNY>_qO#s`vY z?*0jl-o{>Eo(`dBXTa>coFC(7G4si8GW)joGtD@qN6XqpphCQi-nBW2vE8beYE_4; z=KC^TUVSJUVa%;<> zUIwt}6$VDhqSEF^ROWjTC`>-fo~j-tjpwg(eQ&O_Y50;DbL{=47dycFNH5Rxxf7GF zE=a{<=A-rR^*kajO#iI&#HEEnH0PijrojiA(Jv1+{%$C$^n|?75r-`gUxSOT4KQv` z$kP-xvOd%b{qN|4Nw%9steq3`_UXa4yp<^aC554Up=iFNoes=ji5j6bblJl-AhSOS zMOGHj4$)pn`FW7hO#cU^AGD!i$tEhMvBDJYUAoYQL?ob9TH@Hz*W za#9xJ#=Gg$noC5LXNlSVerW2p7sLfVgX@{skiFOdthww{@{=6wv%U?^RV~Cfq>iNL zThVpFA{eMAj%vxioa^u^DmxnjFLXN0w($Vzo<^8*V;&Isw_uzU!2aZVO2ra|^owB} z#L85Woqw`92f$+3YImH>aBiWOhPmH8m;vc^rR1jJ9@1M-0jtNyaQ=ifwJYC^T1**k zJwJ>ymZhQP31PB5?-dkYtEO`AT1oTaUxZ%yfpZI$aLuY)pzkV9B?Njvy(W*$nL9+A z*M}@<6VX}k?J$U*E8P?`zGndprrz`_p)Af;@T!`9r$4I`$ z8QLqJ5AhLsP&%}QBx!R!J#Tk(eq;tIo+S_+T~A6G`l*SWn9t1I6^!srU}>(*M{OUK!;H0>I|xr-!-zntz2uP48Is&JtFEKL2S z1(uP~Abrsa-7YOd@>7Y}8k#bF+;gq*e{G8RZ?%kt$MB) zWuk^_x#DK$GbKzp;shyL5wx`8GghV3aB%86*tDPHQsO+w>9-2><;Hp@^S}v`Q85?AQ-5KO z;%wZpYBuI~O5(NxK~g!QObb2x$nfUH;Qug=gtVQcHtP=X8nm=%u1*Z+4vxZS8fQuM zlnDg;*D(L^ZTee9g`SvQM=GR|6zF~8RlfBk_mmxJrbsrZ?&@QrBc(8G$6cm=YzH}D zEP{sXI6rddf2{fZ6pSrRr+rVx8BL4BWUz4u%BcF|shs2J5%~jxo~d$-K|O0Ya+!Vi zG@HG9xfCXpR@2-~Z{W->6I2>i1F5>T+&KX$$j3QteQV9hbHSp`}W$z+QB2O|5&n0LaSp|RnM$=j%9SYf4yM0_tEeb1v6-8;ba zt^lFW&XI-cOURdt^7NejfAo_t=NGKvvRmdh9htHKV_%hHMu!1javh-Ow{gtw$EmbS zYCo2RR+2BPdDP8Th)iFXfNlC}FyA=?3cfv}re)92y=Fh1QL%4h-t9C+}8@ao2w;={A)leHZG`(=mhm4KXA$pFU%nTk2__Xf2pc z8zi;I7Gu*daT>Nt51f}+@O+Yag-0WtDnKlC}C8y zGJ??uKd7aE2g#dv6TACzF_Y&=-y{fNUQH9LvHLRQE?vsJG2YIO*_wgmyi+vF@*{n4 z{vGTs<$MsxRe2_$u)j3XgWqyHVT?o+3Ar032qiY=@DG{dI&d}I-`HJ0mk2r=7fANFvd6!g@d^b z*!u72nzoSOp)?ZJ`-pwN>pkb#b_1P@?YMHWGx4$wgZ+*_h}$n&qQ6=XC!I~CPqM5@ z~$cub#eY0mPaTuMTJKkI%QudWJ zk>QaOVGhil8=t`Ozy#sv|A}(AiuSEO0C~@?q4zT%7F-iyR~E|SAwNExS@#`tl=ML~ zLW^Y*Qrj~x=>a=^CDZ8hh>YKE#u~vrD0H&`o6(@ZtGLFcLS)u z;$u>`xq}gR>j&AHg6ftTxUg+D$i7IS|EkVl8kZN^2wX&Iskh|hX;WUU95>VZApmXM z^>x>$1{W9Y#NBNRa9`?MG&Ia69khc<)Y}buMm1znQ3XtGoQw_yPcVCD87Z{Y#8p{k z=wsiooYa4Ac5oXQbi-6SN2+01L!_=(yM2!{)UXVc~#4KG5M-P7n;X!@+W0>RL z^E2`A?px$Tvkgy9?mf6{)F#@uZ_vFj1sR{8pV7I`jCPAeP{GbaP<2$745#d-hjz$A zqSj~5iy}aO?2`gTZht4?UrMvLoo6fWsgsX&Q+TgdMqsUvI_}xTv8`WAS$+#oa`=HX zefR4bd*km!hcNn)b6Ys4u)+XXKS`(JshMoy^;kH|_2r9Y>e0$68?CL6!+}K_OslpO z{n%eZXKD<<;synJO|cO)5(gob-;idt2BN653`VbLLHWN*s4$mXyYnYw-gqlrKTjD= zPAnzc22P=LmK6Jai9VkEHx=T(@DcfY^H7MFPK9Gtv9A6+$=mi11&=->|FpM4Mv*#N zjekIWp>Jr>G!BzP%E;Gg=gD2ML+Ec{2sV!tA*MDG;vz4@iBNwOyH!uh4`qOu)D_aT z%mXXMui@k^!}RqA3y=`}0gkePc&`3G;?|QzCntKc7bjXUb7++C=RD``=QB{T!yhwa zHZkLy%wdmpC0*H~Pac{pl6l^C^vUlKSd`24h}~B49whD~5f>dGukajwp;br5Z|#MO z<}$Fl{u#1_KEu8!7SjyB6aUdOxbe;~si{l=C5|onFswz|j|!7#_eUAgR&JJla5Dy& z2V$0<3Xi~&`mq5DXB1GWiH(@3DT;C#*TCP2>l6Wfz z1=bnS3%?kM8S%$W{)y-i$Il%8n*vK4LLmIB45gye+Vg(hW}i9Ll9y&&_culXvV+}W z!SDCH4+E8$^YNiyqk zYkmX<3`j%i1I`6}HJg08^APi77O=@3)+p1M+MYD&J-LI!K!p|ozhnjHx8->K-EJ6b zbc~ku9VbT0+d(bG8x7C6(&Rc}ke4!sEddGWy2}D*_IER)^1H$Ms4l1|ioxu>tyG$x zWEal&fz%2)^5#h=8F_+a=6r43$emI1amPM9(o_s4)7H=vR{daTl}qpLy+d!MEnzmlCq8t}z7qXfA?;~E3ev$uj3zWyA_k_AYC)CHJ+set1L_v6&ZwwPV2Lc5=cVWJ85 zPMnn6_Uf?=%pZM+{<#6DT(1g_r&mEj*>Tb;^^fWb+Cb3l99$(MM?Yi?;X7v|WUaz! zlgC_|{^JdXC3IuOI$(cKI}LMll-grV?qVG0EfPAh2WC5S`Gptf(J@9^oEZW+N)PC- zoz+BRhYL@9yc}*O-iJqq8c^%QqFzZ3z1HbMW;;h<$(ca3`fvs*ANB<9)p92en9Z2 z2=x9Sj;CWA$?yCNxcc)w*sB*06@hDUhea&-8@dq(xi1)bxEKBId%^r`p%%B&L}6Lp zMW~pZhPfSeptR4I7+kX_&pv-)UF%NaGPybQrm{P_Equr;->`(bJ$sBM{yiXMqL1oU zT~L=PnFy*h|Nnj?xRei@DBzI67zM zFzkvlob6Uee^n-v{FgI!XP4Nz`;cbGj!V2{ny&sSNF#7`kirGU3$jw>nKw*xjV z(go|Z2)1c$7H%nFz^yh7OXc^X&f`}Q^{$p&;fdk;s~>3ELlJP=*N-qd2lzH{J~&mvx7}{AP!0vWIZ?S{GDRS%8Uh_NWk1M{m8UVlEqQq^)_J`($<~=H7V^lRSmtP`jf!iS7Hwoll{0ClFHlx(M4`_6%l~&f+adQ||OfH^^TMJ*)7%N@6tYjUl z;(H53tS0DyaSfBKs!dO>+ylCw#b8U*HptZpAb+%KF<)aYE*4G14cCAidaMG9Y_sEMHJ|{TeAs{iKF%> zZ*-g6!?;GB#{Aoj%>9x&_DkDK@c8!;H%oKw?hk#SxM4ZO+w#NwMF_@XX&AWo86-Sj z4-H8x(QV0kY9zjd>@^HwynlA%uJwgDNhW}#lu3|j^#@7#+iVCdi==jQP%vDMyhzvwUO}bE7ciB5WO)DfFsoGu zgZ!RY2xK^O?lE6FA`p*J2gBLO+SOQbNu4^&>#=||N z###inzqv*27N#&G(+i<9Oon;nc!XpNi{d&9ew?dfP8TISVM+t7=-uZvI3#A&-Wf3i z9r}9M=xIY3ZS)&TzW=6uE>m#znMkbfm;yCE&2Xl}365$1fnalcSdqb9d(kF#C0je`0dEoC7uc8*34H;l zVCKIKc(7^_nk&tt8m1h>8@Yf=3*07eCjY|vgQ?6DyhetkCxLssDfVt3V#(5vT%W0& zdHKzQEN%WvtpjdjvB6nzO@2sE_v|7&zBSM}w|a2R>_||s`%N!4SK-z*&*{LhD5{sG zkfEQU%+#b<_Uq&l6wXPYi8tR+)96(+_1P5S=XaO}2u>x3MxDrn(`B?wSizjhyNOJH zG)7DqL$vugadN&#lU@X~vT|D>CQg7Hayw5;!Zg{t%G=OP*N1d5@nn+D5!~|9?c~Q}4@@w|6G+QNT_|o6CU=%b(~M$%Fgf^>PAMIrl6UlR zR&`^$!UGA5+?gIUv*Z+(EZWX0`V8QEgHPmX-5jEAkihv6TFCw76YTA#Q@nodi`bB> zkKaITY8HrjF&SFk@8G(a~o~9ktI(QqTrmqE_sn|0>;xVAwm2x zllX2P6yG@sLs#$6!A&>8L31*uK8yO>LME;OG2KGedVXmd`wYz4%@y~U={ngo9kW3{Xp%q z3-t0+j+qbL4=$hl+ZJ$6gMqAW+#lZ$`=tcPwy0vvEtjROCVUWVB7n26>Ory4M(P?l zN{?RC1KSyYamhZ;E1$occKUjOr1Ke=A(ls*r3@fmT9dwi%Y*XY))00m3sSSs!JG(H z^6lPi>M_xXB}Ut!^u{&vtj`HLq^Dr?i^uJUpPK-mcMEQRnvIifD)6v`3|nkh2bl#X zNKXHS6TH=+#yQQ8h`B(+`zeqj7XfS6Kf}3OR4|9{H;Sih#)9T>=EvbnaQNmR&69G& zF#aWEsPGm!JrD#*vFqp$?F^U}e-@QC*y9PiFW^3t%X+@W(|-Pc==<;`{nHRbpDhc( zGdnqd*WZK0*Zc#MeJP5(nzI@N&uk`f>k2TR;|$79htSVTvTUuK6c*?fAfMZM&O>&- z{cC3}xt}~rm%g%~cdB-wh4UaR{QZfxl@F0z)nQn3<_7sX>d92LtI}4vlen+s7k#gP z6isT+bMC`04ZNE-|L%-EQxaB(S5^aJDoh78_UJ$Gy`83&|^SQsg4mw^1 zq*&$~YAIBr|K|w|e`ti(*?*XOZs3v=7kgpfo6JRYl*WpOA0kJNj540M}cshuTHk;CPA}G>A&G-<9NfwtqfyUdIM- z_12+lCw;-G$6m8aKKXdBE(P-~dr8(NcXp)sHO&fEfZ{z>xcMb_eFW5i)^#4(n4W{6 z3@F>0Ycq6NOzRQDm{1Dz0zTB4fW4 z(dDEat$1xfN}A@NaAh{^OVmicbT<&tHF8C9-QwUL2spR1;MArP`V)u za-^+bE|23H8id*6fyV&Xe_+fj1Nzffn1l)(gGA;XhV%(QLQOsTB>%=rhhq_{!-xUYF(k1MQu7`St;NqLgLUo{YdKBM(^qXawV4cYt^P zQCOz#$*|QK=y>cbnB86u(d+7Ap?)nIOa^jv**#E~tHY_dkZuyaMGjPElIltPv*4E?=`7Jc@fpblLZdXMyUC-Ke(w9GxNln9M57!Hh56kba2|yu(UCy|a^M z>^sM}Z5_hmYYp_1K@EflO#-=)YDfhzH;3q1^x zw}qx#Z-5!;hRG&#F)qggG$krp4VcTQ`be1Bc~=Hq;%t%sV;lN^pGhzDncxZjVKO>t zZhNZ21)R0}AF2Gghq#T-qE&ilQ0eV1*xauHZRWp`Iiik9m(9-poseXj?<^xlw~fe= z5I%0E9mH~h4Lc=sjQ%oJMElm?V5_r-JXL*(ejomV>Zdp4rQ$o(75Rf(+!n$Xqp$SH zy2Bu-xfRZauSG2yjEkPG#mtKZ;5ONkR0r{sNvg%{g#ZgIx^WH^njU~>x(uE9hjWl_ z*bIBIpT6yIqNOLeXNJlw>V4}X2BqkLVRt{EZ!NM2ver8%tEL zTZEbQq2qh*jNmLM#;)xUBwr6FFA9$^l7nyB-?;08O2j$Ls^8HjIk5y5?ve&=kq5XS zW(s|o$U=-r4~f{NL*wFqfX3Ml2==uBo9&fg(m71;hNK`0*+L9CNAn|2l6PNKKrU5| zNl?E+j18Zn?Q3fyz9aB>$oo_ z{8x#op)Tk+{00RbwE$0_B@>nQ>@)qVOhnvQxG?KAfj!GmO??XYnT1pH@>KFmZyK2Y zJIbaOZU?*3chqI{4s|hmO?O+>(#Gixp!Fk)jgE_@+8f_;b9mc@5KDC ze4>CZdi_{7#Ij`NLbG!mOTOtc$1jH&krzs)^-heBDu4Fe;@JX+yvQ7O>h*1 zN}lFizIj;UH65dUM(D@S^J&nsL7aJQoc;5257&X0!}TM(Av02f*8f}GE?2vP49y>b zNZ}QU7*og4Yz3WftZF9X1(}D56<#q>Wkz^#Wp$i z>?9>}PIfkJa>-?t|!Menmm&LJHm0v6*b|TEUyMVJ^3q--LA3#t27T{fNEFdXP1~_>liGDv`K<|COMfo2R8vXD;j@ON#59^P^%G)PN zvy(kRaW(pyzILAW~H8_$6f;VeaeaaovX$Y!cY$hLeCaIkY3;!1fi~K_SrR3B{!xb%O^fGY6`*1i(B8nYw7*I` zA)sm__=GM-9eqh$dzimH`F|XpcRZGF8;28FQL;zMNQtDfpZhwA$_gdfNs&TPMvIa? zGc!tNC7}=s&wZU#GMW_Hi*|mcy~lfh-v9jP<9XcUI?wO-I4;NIbg>~^Xz&jblnQv; z!pmr;-Y3!o_uzb}7RqdhfTTD1Siff?q-=ag1SVIouT26;l(G^WX5TWB;sYSD&6hk& z6GN540>IGo^lbYFV#4iLjl5kz>q7-fRz5^NRAJHl>$GE^8WqePfsFaDFn8)G7$LVy zJ3UB-Y71cY4s|d}S%NcbyK(D?F1q{)B{kj~xjxDOseGXXQ-;^$)@g^Ry>=5+UObOX zb1N>Zl%|K~^UNpKZ3FYV5OT?A02}nDV$QyqBq}q2y!bi?V}x>0Y#;^wv!!rYtPh9Q zm!ZJ%c{F}k3)sB;LtF*g=+mpWSd)qoP)Gz> z*^v4=6G}cmhp&1kfyhmQwW{CIR6>w9J|`U(xqEZppF1r0)C~2s3fBFJ2GIu{V6|W) zcR#ns1xW&ESyM;Km#?HPXEJc&!K*M^;3`<=siKYXa?)n9nUsGXpv)$2ubz>OSB)Q$ zpQqFzo>>8PKI0&}>lLgLt>r%+v%|!WLIV73wCxzDFL&mE;5kw1-}%`rP|%d=hFnlp zpA6=Psl;A*E!sXJ%qCQ&dyh>gy8F1#FsYLGuM31dTOVWoqRF&)_hxc(pFFj-5Qa%H zQDD!;Aa79zlylz9z~pmaJ?$xdy*`}$I#`ES_6LLM>hI*E!~)3Ps6vD8b&xOOCm{b` z8|H_f#Fm}i%>8mJoN8PI8ai6E>%&g!);JH<=Se`}6a~`oV;nZjUq}j{Mnlxmci@85%9uNR&Fb0(5+(Xi_BIMZ-m9kzVS zhOGUkh}!yxaB67=Ecs-+uKhj{|j@jh<$&=9g>>E9vaT*=D9Kle>1t#H5CGOdF zA404*L&~0mur0Y7YfB04d0dX>TXt}4bXQFHz6*W&RmrbjOPqYS5+cIJV8MGca(d5? z_CNjVkX=$op2`Vx9RCq?8(D(|J3{#ZA0u!(8%xIb*V3zkPBb{In}69=68G4D$H~Ti zu;DhJw<>KfE{yP}_wX*eL+mzgvG;*IomEhEd={2!ZJ^yvwP@iv8*h9RCW$k`K!58s zy5#aBtQy!#IxcYe{@3@JoX#S8?~56kxcU}5x}=#Vc|F73}6lE@-4)j~8rwiJZbpgt!-|yy7W)>TVfC&~7+*MidmSQ#n>V3mH@R)G%ruIh13D z9+iV2KeU-7n%pDTS4886pab;3tM<&$HV-bBKMHf}H0hoxIe1tCaCeG6k$z=Fy?6fx z>t~#AAvqHBUJc@7{ReFA=qSI}Xc`sn=NN=~8Nkue;F$0(+M{ZQbE^v3)Yb^hJo5?< z&3g!L|MW3=_iQY4kwTN|i{##s%b?c33uM=Fyvq$0v?_EZW~%>3FXl_r@JlRA-O%#7VD<6g#Q%{bTbmb7B?5JNxtM{5Qge zsjVbLp$~kLL)jNgQW=FM-QXDUk{XySAWynqqkd>SNj)>b-+oGlUw?TJrLDO>Y>xuS zt670;ry@Na(ngi)>#+Fw8YVV37<0;2G35c0M0(~%deZ(L9$H*LzlVOO#y{SX#Qn<9 zDC!AJZVm)J(Z!U%kwCcKuFbL#S~fWnVxt;Joq8~+lsv-D$NM;b%rO#s^gP{lNepH- zoFQqR`ZUbbfx1U5VZt8jg6VrR$at; z^3-G>9h`feZZQ21lD~$sA4NIuxb7fXdv6SuYWE~j{)h2bK&xmvNEnIoi zmg+D10jdR7U^=UXeb*NQN!!=b*W@zT>Q{lRoE4 zs8XpJW3(@vV{KQVn6(L=eA0!ijo1rz;hJ1O>JULk1Jr)s0FomspjIp&47=jU6xmTY zca3AhpInAVcgc`z&k`U*T@K=}#BuK@W(c0^z}8rn#J*1l@1F#Z7hIzg*?qKkcP4Dm zC?(P5zp&0l1Ahmt!U<2tA%30~6wd6xy10+z%Gf%-mS-FqMt719?L#<(Ekv6wL&5TB zB1m6@T5T{K|I5(ebYrMjYk~cL*Ks{>4=UBkVZ{4S^5>UoldjO)~PE zS!3}SHt8j`DY7O;J%)@_c(U2U*BvyYW*(ka*T${#u1sET9fYmh3+q>k(M{j9VZXBs z@5`biu#;XQCeuGL-BHs?`5|dKW3LK6Tmtb>14dg%427($QS0P+n6O(M zb8IJLcZm%&C`Cb0)(+xV{ek)U$pb}qwV-`p0QFlRh58pc58bI*;POwB6tjkm2gesa zxMmZYuq!b>RGUVsYjD}wmpEz9IT{$%$D{_FX4((!!?c}`>Ak!owCSr6DddO}C?8zomY;$1h&OF*MW<^Tk$LXJ{1x(94tPB7jSM@I$b zGg5mbz=(j0jCcXrjGZ8d1h+qWApG@AHj1DK3 zV3Kn#j#x$0y4`z-llCv%E^mgK9|wU*b6GBaIm&b_z(lK;#9~~Bjhk>D*kyBJqr4Z0 zDAqAX=SpGU>~yB0@f#lB-iUrK(I^yn1%f}Ep}&&efbTUO8gu+5lcIbS%vxuVu)lWf zs*k@g_|;1YaBssa-(B&@r$##Yls8*;;2Ly4e$Dxzy}*lTvJt;5vAEg^>Q_2}8^^}_ zZ;2jd-@T=~lmn^L_69Hr?}A-*S$IKV9JHLIaN_SAdOETcOoUd!`m0>$Y*QzVe)I(A zlzhO|{pl#Z#tIuQYtrqJMj-IbhQ1fnq?rO0%<~mDxp^*kb`zJR&H4+;4hc#6Fno}F zsExqc<%Q(KpCNK8Qpjv?h9}kje3Rx`ZY4RMrlhtm7te87%*(pwVC%-QlMD^PVah^u zu#HC1lWu4%j)2lwEKs!U@)4_eMF%D+b}GB9YNe*ti*&@ zv(d=m5zV?F1h6m6&&4ewVHG~^B{~_ZO8`<*p zztE{Xo63hRg;Y}!TH3G>1{bn0YN^!#dy%%Zf#)kG*!>jbvmD?op5tpY(<=9|P_u+a~Le(le*=L9T!|gh^ zVc_l*GUR%VTpqgt_HFuby4MeF6r1r#a47jv)Qst)n_-S#Jh~r_;{V{h3sq*^_rArC zz7pO6>UWnDxhLMRWuG~%-W3E|pL)rUYdxf|rk@CGTF2;azDhg3aIP~G${(%Y=8|TU{j_z;9Fo|^ zG48rnVa;7R=I_ubI;>MA%ZlHSBZl#$`$0SXRyP55*7RbkmOaRDoj={b z6Dhix-~_X?^ro;K`Jf(1^gg$f*!X?4;r>Kcso4UfsXvDt>5&^m(d|;Tw+s%#$|LZE<~$$Jo4_nHCn|?!2KrqNa8+& zSf2uU`Jf0Ea}HeHRzKRQrGkf=SMm4U5oDa-Gld%%&=tlRF{$yGMkC+_Ds~7(Z%|)47Iwo4E*yfxL(F8xSH7mL2 z&^#d9-a+~0voH{mObn?77PV)<0qGq2)Q&r!*k6TH3+AA4QVTXOR-z#-;#d%N8SS?a1vIEZ$?{3QmaIUk{acwYcB64bu(Y(_aJKxJ*Qz z#)?;?yWn5DwN<0<6)`!4p}<(;svs1~JVhTw>2A#7hZ6?dJOOOjPOVHS$Q-UX5< zv+q2X-=0I07yk!}XXn$}Ym0Hq+)FsgWhwcukIOVqKLNWYe}#;>@1Sr^08G+ICobWI zuo`+u>VvtEJwuf_XJ-Mnw<<}Z`c9nHu#SD>AI6^l;03PZF68T^PIAUY0&+Ufksn*9 zu|DcI==QrvtV=(^;cp|PMXwO19N^gEseRa=IUC#;8e;6#7UaE<;kj`;e-)E4a(V^O zlNzaP>I4;xpEyE^;yv7d?gQ8*$+5PxT!{X^VD`^>5i)e<1Wf7W5j~C#QGTKTBSlQu zd+|xcftyX|yR1XrkuLIQm$FH>iUtj(ZruHfa|rIsV&o$a($_m#8f7F;4(|SnjS1Un zb4Vl3IM@bWwM9^Jxt3Hky1-hua5&;TcL9l~<(v*Fq+ZKZ=RFM{w0qSrUGx z1oy@shoXyr*lXHrAj;jHeAs#ooRnjkVb7DOU3MKTHJ;Ni{)_OmD(C9tn0bEMZ()Lp zF~MCINYII+r1xGA$y_5rZOb%C^;dOFiQLKFdAJ$(9_PFhTu%L3EJL_BR%&iPxUU{%t6IGOtoWX-mc3UzJ=YjgqT{`SO$+v2FDKn0$axCbk2 z+i|PU7=mI7B%BT+qE&BT?xojk^#%))sB(hF%{&TD(H3m^mJ$@-L#XtJ6v(f6LAFhd zhhwL`(S24Tnm*soe*Cq9=z3K{efB8WOyZHNlctgGX^&CpD?#${G}-p{Jyi+cz`Hnc zD^434VBZ=xA}KWCvJ6h(GO!fPMP@;?k|3ruNYMD>6{O>!E2!FE;h34@sPt@f=1J7`a)s8xT8y|7%V-W5g2Cq=B6j5j>nv!3rxmoY!1*`7<)#F7 zPu@WLO}NZeh6s=Uvl|x#m}A-gQmXyf9^D;VQM?r+ElXC!0z8Has-%Tj8UY?j8<(!Rz*~ESe=O?V0 z0NbLo$vXaA6wKhx8Bh1oM@<&MJ-F$y4}|v3jwk2*5}-ObmA>cyN1thaghC5hByav; zW%fLBxl$5Sv$ZkdQW8G2Ge!rIjj;FPLpss_HI~~PL|*4gG$qr>?~`#P%O;fdcf1Qq zx7vtk5jT4gZvsIJZ93z0I{4mE;%A@WL#Nk4=(L!_)(4zNz4i|fG58C#{%X)!wc8;q z{~&7ha*pyn7H~x72ILHcaPv1i@~Y7j7B11mu4N5quz^pNU(Z5jqdHy@xWTdd2_6l+ z1Y&OlAyQ2p%7cHBv_5Bq<^(tt`G<@uW}&%VJkk7ag+-m*dk^&_W~;YzOxRnf(HqKj z&=f$j#}*Gi^2QxkZTJaX_pqR^27bPXKuv=uyA?6mnl9oqy447<}LGKz7O_dhc5h>X$8|m)o-0M&ZMl<-Qr`85rS{ck}S< zjytFwT+QOU1N3c(6(|QZvjHj5FwgjId*!_^kiqUIFV44uYv3;E_*w@Z@)00$xSK8L z)rH-yMxfKQk?UQQ(IV*=m^{78A+#G1@flyk?^-yH?7lTaXKCF+tOgC+pLzm(Z zI$ysT!6T1k6u98(!1bgndOmr!Iuz!+*u&apO1u7fkS6&$di$C*xR}+WiCikCoi;!R zc}r4n=}W$U&15(7RndP<1Np7N`HQ$+QMZFWu8}!U%P+ctsOmAidE5jIW3IxS!G)-F zw~Y4Ky@8!qePM^nD3zYwgi}OvLHSH4O1D^(Z2Q~9w(K^jte*>tjMAa6YvvLQGGAV?^S42Q7=L1 zoG^h*1%EKme@vp}n;<9Kk!DG>lGKkLxFq%id1d+>Y;vxFZ^LU4`pL1UWimmqDVa$g zi6bwyu>4MNZ`eY@%u`<}z7uhrp`fg3|G=Ti)wv$`LWhD9+x*nntFVIg7R z4_sVgiKWYDaWg;#!oTr}{8v#4W=<2aA+n48TYdr8Pxq&#lHZZnc!{*>8=yo@3x?^< zg@fxe=o@b%%rMh~=*b^xl|cf>qPYs0K5K#b+(fSjP2(G8-6U5WeKDVzjJ7kQ$nw9k zq&G4Q)t3*^uyYDbuaF4o>!~K?#tg`=I*vP2)Un^^Asl&`3{#JcV4^?{DB~{JBb>x> zR)pb#@e$IW#fUQKzXBP)PJUoL=4o3)@wpqUSD#T)fXuo3opp(sPpT zIp9t-PTip&tY5K>{BYX#$d@P=KSN)IQd-R`g|a*Hkkq~jc@O`pa^&udM z<(PYV4r+DoXD3sxf1|L2lp2kL==gE=;mCK;b6bPT^O8~F#WBdL`T+aW{kVOl2Wf4N zzy_5b8h`%*RL2^!x-t@=@XD9?Y#RdIFlqEVU;|V9-JwV;k1k23!)2Jai3BT+YxgeI`R6@cZ?8MH~Ri~~(knnfhI^PEKjfZG$ zpc~QL`U+P_-Q%4ZFUH;L%3$$ad!GH`2aHF-JUsR@1<3NV?1_NO=&>z@%g?IOc#W5= zh>#^P1@W|guL?*+0eU>I0h~$+-*GC;HR&nrh<71k!ySlIt|BBmA{@e zrw7xCr5l-4%RAKN!fx86_zWE8_CqkYyIf)>PE>R|k6t=OjCZqE?&a<70+&G20g{eocRkN>x&C0l`nxD z*&I-f%_X%xMOaGy(bVv(WO&wp*xve!MtrCzRw|urh#r@hmzvHTvv~q@wo8ELwr5=5 z`UvOdn@EmoOF-n&LnPko8R#B3kM=2}0F#fBt}|i`Z%-gj%_@Q6SMNXyxn1TBFPtgG z+b+^ZRTy*_`R5jpN=G_gpN=j^PdnUR00nN z4r1un7_9#W%qss8aP{CEVMFe);jl395>E2#cO4<$4+~=Sre@Ajc!!=Z737ibEb_qS zK5hz>BBiTt!|MMep?HB3F-sZ4<7PSJxlt4~F22M+arP};b2yLYUmYijDQ#$6IvI7@ z`>ep?c6v2B9QEUO6Pqy~H1N1W8-7@Wj^SNssB;CX`;mU%Tm`$V)u2vI0_@|(S;x~e zS-0G!D0fT=7rsyfg}Ir`t%J3&3Q!39JIpkWZDk!9LofM@7aQU1MNQD*AyJbru zeX1y^8zkaln>zN|g8=f$*AC5h6=INNBdii|N45L9yQf zJIZy#v|eG{?@G{KDTDjOAJATtH_W#wvgFR-QJ$9QBFqIz3^?zKs$Iq4^>izFuv-sv zuX{p*c{DJlLiG4cYtq#sOaH3~0+*0Pp^}9(L-x=EX{G_8U-OSxIe@ zHtvwB2j_q3Bu=cH8XS5ByBuR-iY^0rinHM8eNB4do;7TFT1f_*`_WTiBED+jf#GCT zTI*7Qc7eO_$QmCe^^`m2W zP+Ta__tt5pm9rm`%Y*kw=GTqr9kGBeXuE=@3mQR6Yn1xG;v5+vhUomY8t4Cakyee* z0OOx+5PLTYh>0Su8HpjE9>y@Y=0?!YaL#`{E&zdkjkK<6F4;XEN5>Aj5Y52~P@qRp zdr%4zf6C(;VSPI0l1fVp?4au98mL>^NQSF&Xi3B+dc8xIs908!pxH`9ZhSw?_;i(y zTN|^gBR2HMP%_6F{sCILEI0S7>6liMOb$;KqW$G27?HgWCY{nR4m*7tueJ6tEmp0s7fxLJ-Wzbt9NWQEy zriV3H(opjXGb7%>vbbEZwu~dOaqTGo`2+Cwe`Knc+cF1#?8TxT8);u<1>~$)L>&~Z zsalIL`TOWQy-@W9PfpoM_8)pk>$=Xd>dz;V$8r8-?B#Y$m_L#Fzsp9EQ3k!z&(rh= zM)bmidnEWO=Ojy=NX27H!S_lJd6brmam_gpo4%49T~-Z= zqwgVmx(>SC%SCUG1@vq9CJ34%h1*iw>Gc-|oLftmKHWGKH-;YNSPc`YkJwV0ws1Pk zZRsJ;e+I#XR~yLF505b~d<882F3ZMs7=oJL7{7kPI<(IdCuH|G%J*?5sh26t8j40M z2TfSl$wI{9Ef5zzj_H|0ga~u*^}h;e>nTsnvy`M`^;?)oj!}^;J(qr1`V$nF43k;w zICgqxD!7V%Ax~GxU=sfnU4UE6WRAyj*Y!@`_Rs5R+Ab?_Yzd`uLUz3O_iAybsxGLW zcnTpB7iqk56qjcv;H)A_bMGFYU5PuOu(OaZ=DaWgBS9El<$(tiUm{uQjw&_#VWHC> zdS~(;{+qI19U4&v4Dc)*PHR|HHXY->1(Ec9e?g^|^KLKpV-DV*PUjU)1&il1pyJyt8aRvF$K>o}Z`DfC z|F(T5&aGF-;LJ_vHm4hnHQQ-PvK@b5qZ6vi)!>GzRiyk`0XqJ(K#$}1nWtB-QQm1Q zl;3iNH2JN7_-7%w_*W0;4%ZQrocp}Db2rhxke9eq_Bp%D`#Z?DO2JIcNSN;zPwTiD zhpvYQ@TPP9$m>P?CVe?F99c(vgo?1}tRNIT%BLqZ9U=8Z8R_|O7#c6xlFHdKr2S|P zq*veK=Nsx1y?8T7Gw7r@B~#FFr78;0H*}g#9OnvPz-WRu#**J~AiaTBb}l7VpVT2= zK@L=|ZNakLnY3|YF-U$Gfeq1Z5VZLgh~Mak>^+Nd_pwr(?LMFBpJoXc4P!9#Knhy4 ztV93Y1=#vogK{}2zVXd9*b{OS@6F(RQoG#Q=l);lqF2sfA;cs7We3

BmUXbSc!R zR3x$bINS5wH`N^csMpp(iexkxQrd|A8&KQRtbx zlvoR#LlgH%2uloxWAE}vAII@8xA;bLERK^=`R$lpslkQ~_Tti+^33CoE98#Eb6PRe zoBRm;3de78F432t!ERR(`E^ZP+r)NE5aIlARej*yyP1~hE5r1!ZZOx(lUa~bL3oqLiPpeF z;$xBnN&n5JZ-P%i^5<&U5uXLqWM`A10%1^TI7@P4o|9X{^_&yB6>1v=X?{#U^Z9xV zeOo?8pC6k@-D>iYzta(Nr%gd+GlKdfmZZFI3}@SJz~s!IpgXGkJ_Xyqe1?WD13EfwKf_v1#ONE5>=xxk#G%m?%G6^>7&{m3 z7f-?z=cnu)Hy8T7{|hV_enRhwSYV&iV>lpcOH7Xy^h0O>zllA0HtGZ~FP6)2?y%v|RpEF0YnEW}88YB&LaU`V0hI4#5(neQU45{L5Vhc+^qt6Rtus;enI=? zy^tnti6KwIq5kwwoJ9N}sZ<=d%Uq|9^}~4RAD1C-nuSZ*we6ir!KfX7l>Buvf>5;L z94iA@?060Rm)&DJ4u{eiJ5Pak%Ty>o`56qZ|A$I9S2EHggP@T51l1>T3@0v&XV0A_ zJf5;7seCz>4W35N_5VQq{s}OMn*qUpfZN~d^S`onKA&PL=_&_DL)N!;N$_LblvEbkYhI}eLuS8M~Cf&tEY6^Akf zU+EYx9gp59z!;9rI63Ya*}`=|Q<`*2m)l%&+uRazW zuVsN)8lmmCI z)WFKJ6Hr+*2z$~cNp<6YyzZREr1)&O`LtVn*l&M>^h#xs*0Pf%q%4*Hrsf46`$yRx zd`wjJ^2qlyp2+NrNA=-}m~1+Ur2LmlJ_XDrqrEZsI?)Mh?X)pe@f%J2qyQn@`Jpz2 z>q=)tf!6pVYV)EFZzeAy_Jfk-Kz}B<3M*jx&O$Uzu*QU(ZFH@b80J+`OvY*KXxAhN z`@WHWy4wdqH@2W-<6R)mxm=-Ki6-vwAlWU5ijt zK$P}>#Yp5gjwuL!P85$Un}aOJ|7Drf7~M{6eWsO7Rp~rx->HkNd;rH;r)U z#!~V~$qzhjyGfut4+5k=!FJAN_-!@t`mQ3TjOu63e7izC6Pm&#f z)9H58NzsmfIHh3<%=ZDXK4OS$Q!X6IDF^3Y>SSjV$C`Y&0<)4XGf5n~%3OUe3Eun~ zHxK56bMSJU?o&;#PP&3N*-t@ibtR_l?c=;CwK!2N6TBk1+_d6#`n2v0T+qDEAFg+R z)$7Vplgpy@UNXnD6Z=40{w%#_bRJZn3t@!uHT1mTKvr1T<06d^T#-CNK8ZwQ*3~{d zQDVV6v*{aLnli*@7MMX^=Lo#LDFIz+^2Fn|1llgzgSp#;Auv7&+wMov$IV4-%=MSx z*BJ~OObz%yz9vB4J3+SnfEJdV_axO4>0l~y4|qNbce%-KUTpbQL29RIy99KO4#G1)pcX z9x*t973EEP5RZBdN!-k73FargrEFg@>f{JgL+?I_@7m3tSg!~NE(Eax&6{A^=}mAh z?+VE{*+s|IOEGs{Evc-W2+lPssMq5GEB;-Cnss*|>#{P`6QA7bY&=PoE`*Ldb*+;XDp{y0*pLC$F; zok0pdG(gL82WC`b2Ohq&12lj}2;XxTe`BGAxI8n@(=Cj}ayF{T6BkIoXw3*NvB ziv_jn7W!GwnQUnPLekq#lQ&xmK`!74^XaNSW;V*gl=Sm(X*0(P{pbhIEfI za3Gt<8rfGn3}F2ij+t5dlYX1P?PhyFqCoIL>sQVE;-v7B8_8U$Y+c__a_)D2lk<-iYNmtMYbFLFhp5V>gIKft9r?Gf z0~a?x!-JA5i1*ixoPTT+YE9b;3XB~rlibX zjpT3tie^Pi&^x(;xIfrMw4zk7Nn-}>#i`_Jq%~NCKju1`p|o244c@$CO=fDJF#B`+ zB)GGinDAJFqAvh+>K4(r*C&HR-#d2t)3>CB+XII64v?@f5p>S}546SH7R}pg`O)P& zXwRAeN@gEorcAm{ew&4XrN9^R-01=CEajNI!dex5m`?#WjQK&_y%-{3B|-ajoXHt|9|EI6VDp^^vEpk$?yoGB z3p_#DKpQmNk%d#;Cec^tf0HMA%gE=neW33?9qsO2ff)mwx3qQ=6cw$-718EcBA0^u z1$^3l9vmib&YZ>8(Dt?$r4b}XVlMfcRf;M*H_@A$*J1J712i;wJ{}2u3_CXWp}^6l z=s@2PEiEJTJWgTJ@ndX;i!J$mIG6ZN4?@{VQS`x6H+qiQO5!v*N8rzO>_FuW%(_(q zI|%3O%H4%QKVm?8qd#P69mNXYPcSt~nC%RVq1pevqn4nL8K1qFppX?r%%T)lQ-j1V4gB#0$2mrC%Y`H1v@AIG+U6XJMiv@AE>V~jP2g-7&(da_&gY< z?Hmu^_l+987`lL#j=YB{dpcnx)tlUoZo|^+dXQ0m8QeBC(}#cNk=VP&fWVQu&_Op5uPimAJ6vAtonnL%D{PnB#1W zlI1Ij$u?DMDxhU@3^|&>!WZWtx-^cPJqB&0VLSd&uQ|`aO;H**XXX<1>oqXbxsL|Oa(x9~ zJLq_&Q`Ta-%`t3U@P&3qKEq(H=TIhCKsO8DCmwVD zke}lxIsW@<)aF;Bsl^Lgaw7}3ZR-VPcMsI6zem3hPQkbdiny}-FvhNV$~;$B;)lHS zL!lcY+-~X~3ad!c>+?*>KyL=jkCCRCm(Jt6Y;SN9KM8Nm*OE6iRZ#J^6(vGz$<7Jg z!0bE^NePkkT(ujr(^pfOf4|9_wR7kW%e&NoT>z>xg{bZ1yWnZC27K1Kfk*cs%-xtv zH~A|-mU|JJ-o65l?C5Yx9-{Y`H9{b#DEt!{@PrAg@XoB|&EOdN>^DK*@ys8Yd zUvI~)4rgJ7K{a)s(2rVy1`xSt4g@J*;#a^yw)(?fm~?XwakRS$E*T#oPhu-gOSn&i z)S5|NO%D{dX5y*-C;Ww4y?DJ#94sQAk=Xxsvc)Y6@X*vSn6&>icOQC*)z%lGJ|Za*A!TFrj{Rm~PiJHynkW@LLT*Bx5&g6i5@le(yT%s*SM6D%2k!DDAo ze%gIj;q*c(IXH|e`@hof+j#Wh?Jp>;`iOkH5`Yo6g7{hI?dUJ<#V{{lkrpwj^!jTl zkQ)@o(254OZJrXW^_9nofrb$0J|EP?55g%$ZJc2dhzE)&`_*_axpVm$HrRZ}oX-2$ zTxv!`uG&ME+gsQY)CQI5FTq$`9F^k)FylxKJyEifHclpJXw2mgzIx+?X%4tsNfPp< zH;}c@^1%O)4rm38fU@RwdgV+dCJsf?yXB4KMpiMpoD7A9w|`^huG{onp%T4KMPbeG zOI#$FNynuJAjzFGh!s9(W}lfvk7TUCww)Jn(fnPg4o>9j`yXV>t(Oc2T7k8wrT7i<}?Z6Y%>NcPG+&w z4Tea$%yiP<8$_DE^}=N_A#%~C6jICj(ckW7 z&mmr}%CK#w4RfG9fyZ-jC8?Wb(E4Q)%?`N&A@URpU9!pPbN^8Ko)}Zod>$n9uQ3Zx z{Gln`Tfybddq|6D2M~FSDervIB(MVV9)AFv-5F5nkV4aCim>hRckt-n&h1u?lD0L@ zFh5`wjaOCYX6ZcIByb3o9vfp!OclS!JrA=Txa>~B8Z6WN&6Yi$&Pf{i)M^^%zswQ9 z*kJBny#PpRj~rCZ}6B{O1!#X7+aLJ1%Z)L84Kvg=fd8f>lJbTDx zV$@0PVMC^-HH1V@8Ka}A%{X`T23=iPMJD~cjq!KhQLV2-==~%DtrrQBVL3O9ndd;e zLehBAeJ*Arvx}I9=NobNOlx>#s|JU2`bpKkn>vsb4;bdvwjaF4Q@Q~ zWpy-Ga28}QsX<>UQP4a;isc*&nA3@oF57%uctx8G{Jo1R8S_w-C0L&&EO zu;fQS=H}@_aQIakY$OfqiVlLBge6XRlLsfCzJUB>3lzLcc;~YuXz?axvgPwOOwb8L znfwbxBkv>C;pT%%9GCy~I$QFm+=gww`4GL5*OThymSm082(AeBKxvhqTo>9AwU!Rk zzW+WG?F$>JZQMliz`c-(@0kZG@2=p%4;$bZbwXI(fIc1PnCRbIF{4}_CM?$Bd(@L2Hke;@d zmT}C>*srSX`^3aRpumLXDQrfjkMm=udC>^1Dcp>qhDd6Zpx*{TTzN+wyk~#r?Jf_1 z+GZizJ71k~JsC~9FDawQq0RJtZ3ygJ`HNZF;Xvw*a?rne8amcFq4%6`s5|J5$Bi%H zEs1@|6Q9Jq55EnY6Wcg1YgGlr3N58eH$CGz z3O+RTmO8|0-=MEww6HhQV(HMAyCAhT6E_r((`ogZXAbS9`&V|6IME-FYMO@4_mue;U>cMpko=-@hT^II~zgv z$y{dTjy(2Hhd7pQ_oaL}Upy6H4JkrexXCJ;9xDPkKrWEQr~aX0p%HkUkfJwKzhc_i zSM=^}9dzZ`pIIksIIhi9Y`CP&&Dd9i@ya!L@-)XRS^N$>)V`xq(|hKy#Abl4e2y)W zM~^T5PM@w@hAKnf(Z0$M4g3#LUWpKICwC57Ki7|YqO7598BVaer-CXDsX&(aMp*ZF z0X0-o!-6%}+1e%_=&HX&gP*6+mE5@^u{;bzCz-QN`N?ogvIo}mj8X3(H@Flg($3ig z!ROaYF8{Ad@_gPv+MDAvS3(_Rc0B??|8R_sJd5_H&VlTr7<5yZjZrOmWZ=U$Y;Z6@ zL#MN}Vr?xR+dBqpxg$zv)KgqJdIZHM?84-@7TjEXld`p@;M6;tzVb_gnTj(YDdQWY zTXsWnF?Z(A`p&Vj7Qx{0E}+`nb+F(8?g_SqhAqEoVn_~2o~(s#MLG0uTn=Q1|3;1E z_1v|w6n)t{P-vjRU3-*$IavY^49L+}n;AM{+XORK^64E|w!dP3AGhRlY=6@iV!YFO23>tW59mo{}ZP!K9>T z1{GRx95bff#XX13!R=EI+KdVj9Vr!BzLC(BOCDsSq!6wi=>hBSzsb;~8NiqsQq}nk zX};qEhQl6I4LDD+=V>&1V+Z2)*`(9-G`V3}0`=U^!gJYK6V-+|&gmLY8sbiK&+l+F zmkEK1%U|*{1M0}$1>A0B`92WxTih(D<4zu^P7DM;O?fhBpL{Y7a9Ng*z(ca%cG2Ry!r|jZ> z-bx0mRFEhGOw>TAi9i4vA_&R-gfq*7#2LsM?{Q)C3d@`v?iHePG}^ zi?k*eQ`Z|mXhwWA?)y{=n-|A$z87T(c)Sg>j%V<@CS-x|KmfoCX;gYgA+orN!>r4I zjGhd+>wk)TF?K?mF+LH!=?52@mXHTCHjvD38q{id6>4lfMr+2NlAcpeFzHtVlElBD znQlkwPgG!5@h+IN{s9C3F4G~|Rb2k~Iqs8Eq%zB=b8M+Lq9wM5d<~YRHGy2WykV3Y zm(4Jrv|$b83H=4z!f8+-^cF48E6`1$&*=J!K9V)(D$XtxB~Rb}kD~K_1nO85!rfwe(FIM3V|BDi!Uede8d@)DN8F zJkNb!*XJ`x{%w9rmlX<=Qz2uJl)H^gWc#3U|3dO=W-)VrnG?pIf6Hox-=+zD-&#ea z|6>gsRzivVPF(r@2a4}4#(djev{A9az0=>&P5bRpKw%?BKikVBE;7QIAuQ~AFN^hI zD=^Y2p1g^wp$!Ko$ccJkI!(9(jBLe0JpKf=w^!1A5qKPh8&1+6C$>QL(`P6-rU|=4 z96VqFNqO!8vw9xrR@#K2+I^0Le7BIUR8QwvSSiG>aWgTxbqX&! zF9zB6kFfmTTbP%qNbd~g5~tF+x*|;n(Kfc34hUs(yqamagujWtUojhRTJeznFH&TO zd=keA>V}mv`gGn4HM9xeN!}Q6JcW2Ul$LQqd6zu23VKYIT|7Y|rK&K-CRKM=j3(1x z@QbwfwXi?*#6an&KQZusikm0)k};hN^yt2ypnA{>k`%pK$Y!EJL+OJJ0fIA&Vc6ZiBQydN_K zkG~BBnYTboV%ISTx&`QK?(UKhYJ+q7SHc0#GqX0Txi$WD8}iIr%-ug&jPkig*DRfd z35F|~uOUeMx&6%Cb~~CXz6#d$vt;JgZ=74pg{DvLW#hyLh~eHg@}TuD?QJ~|tGqVh zvr`91P)!-Sn{%07kq;c(bO@xwi<4zGaMdZrX%KM%QSnzCL=y z)xwg=t>ggbUb3hk#N166xNNo&RsXHZ?OcC=XkrLtI#`nT@454wzcsxwO&56Ex`EF>X zviVjeQ(3+eodS5tQzL8eHos5)+><97+|S#5!(%Mu3up9|_d!V)*FS^J*dStoOV)CX zOTLBVnMOqG&I>ouv~>v1ikX8`tDEkxiXiaJ>zk$tu4ak?O1;L_nCJ&cuG@7-UpZ*!&9US(8e`Y5PA%Le=N2}Eh95uM*Aj=_GKs35T!^j<~i4yB&L zT;~ol`H#y9KnNZ9$@vy7Jm?y4XNaXC7!~&${MwALxJnvrPW>Z))n#d3>1$9lb)fHy z)?%WGAw9M8I_O*&XTOBI;j$>lDE;kYuG!Ei(Rs_rHtYNC}K4^Eb zgk+--aKGS2$K%T(F4KxOi|mK8y8uNw3ZyRI4jRQbaXz5cc<4+t^7p0@ha5X-+I0`( z%rBvu>{c4{xX@_ZAp|8GAul~}>dLvLx) zv{RUQVlU0v(Euwo{BXXP3Ai`>qfOoetZBw5{h(A#>pJ&==c?ZzH%OQvHF3@x-iv__ zJh-k-hWmK()40r92septHGj05Ts5(W`kijH=6o8um(QbzWzV6sbRRvJ9)QJmxA0Nd zK9n>hbXQwE2AFf~&+>y%AW}$7mlr|76dvgO&;#*@y6}w5G+1+YE;+H}C9UX>h86!I zM26Ti$?K5GO4Ng(>vL&h@*|jg(;Z){3t?~;Kw|0u{o@mXr^brW&D;{&9Xy$))tPMO z=^dqu1-f-|fB3SeHWt9{HB)tmcAuwp{d)Ah z$|LfsIRQmpXVM!hWUyLs37OETAaguhAk8oncrU+T8ap48Qn_xtSFspZFT4$|3hDIE zQ6o4~7Dry3IE@~(wEAJ*WFLrKEoB=&#nDEK2ef$T3fi4lfa0%?gg))TLj#NH!uKxJ&FBiv zudQGcw;qEsEi>ean57%#tiw9Fi=$6S*@|}+XjqC>4)uuqZ zKHMeQBT>w(>=;a{IfD9&7J$v+He40x%^I%g#O3zf%=pY^G&v&2^gQ9g4L8+b>1 z%BP!{w&bFlObwiG%w@7aDT1r;eB|5x69jI}#^Sm&?A2X0VBpz9uHDn8wF=K6Vm-sX zca6;Tep!@}eoIy1G^QEdV7oN8!<-KdOi)rEEe~FcdGq#@3u9KK^zsEV^Y#l6mrtOZ zlBa>9=uJ41^BMQh9$Y1!2&ETb}y$X%I`3Y+hshLS{vbDXhJ1^3xpfsK!y$T_Za((V)tCXaW~$nAs*$N)}TDaSS$71AaA{&+~?6iku1 zPpx>PFloso9rZp#3 z)XM7v*<9j6p9^hA!FHe>`g3WOOFl`Se*+y{)nK*Q8TwR*+Xr%4HJ73aoc+fGSLnpi zUm9YqkB&@%E%xW2GrFARTPBP-+P67p_C<)@?hWr}?4_l;gCO>P1@N4y!c~1$aN@5e z*X=W+@txdz;PH?ymiD0*m5w;wTM*_L_`s3}{$TCFOQNsK(`13`^n+y?gmnbr#)f(5 zpe)3$ob{e`Y4gDuUNI(B=_JSre`HT36ME}yB&LYE(+Z(TG;L={)MN;#S-pYSh*RM3 zC;{?M9EDp!Q{Z^qc}88cAFVrkhea@-z9VIh4! z*~Hvk?n|1Ex}ngfu)MAbCC|+!`SpKsS=ZBJ+E2&&Q ztsQ#ayGtGz8k50SFEOz`1ynb$f#C63#4Sr0BKjZV!R|E3t{exQ?jlBL{XB?oS^;ZH zQ(<@9ETEUTZ`Fmru*SCqveH(O#r(;1rucf?bV-Q4>TwMU4?iQbqm)r(Rt0_GZw`%3 zikz4EAxM7nVJpH7=?t{s8hivEY0% zn|UjLg}Xmp#7nj%U>|jcWZb%i%^&L7O@+N!Xvgg~a|>|l;D5lojdT2n^uR{j3OrN& z80I<_>$>Y-N1aU`AlcRrC4Y|*2W9Rqn6VMOe(VRU2_f9L^C;~bP$qwN?kDY$A87IR z0AQxM(qH5CtXuym?lHPVzitYHl=>(zd$NXnpEr#(ecFX#0hV|^F%yOE7NX4J9NgUa zl8n4wMy5J_q-(~weeAd%Fzx0bVVi==+HDXyaS$Dq+L`?~GU>rh_u=2X2m)^P81v{R z5!*EjZQXiEW`i_Li91d{EZT-UeZGKf0=Gx1iiSFED`?m*s>?y-;m}VXj6c?Z!!zT- zi9Zir*4xkz3wDr;E<_Iz4+tJ`&Aw8kg!Yb(P!>DcHsCHQghP0yLpf*Fw zDvqJgHAm{a;tB1U*n~NX{9vrALK*~s>7Fg2f`5GL*_dMp@S%eYCMQB8J0J-lO2Za^& zaB)E}`Ezsuxhb<9W3+CQ^GE9FlPu127;Hk%IoY9rC3lWADL^}!8&vMUa#$`iRd;W| z4Be07Yw4Hwhv+ivT5yOZRSM0v7+JyAL{p!n0pkq9_Fx&gY51 zt(&xVVhP$Ts)r7N0aW9BJz0fWL_TX6)!Z{7O|T31^Hift`2_YyaxSkehnSyyN%Uv( zUofqlhuii9lOAtPZm(5FD{8;c@rZm>8r+RGG@f3TUWxw3swmoTOhu zpA$wnX5=p#`qY-{CxqjS6>1>5>H=H{ypCRXxVcyDF)&UX!lholFm=WbbZjgl_l`Ev z3s1XA_ltQTTd7X?SLNdROeK_L1YxbQ0!R$J!|H=msBqv4n5XPZx-CDVI`>>zB@NPZ zo#EuI$vK=csDe{vWLVR$99uVgJxO{eNVp>hEt37gR_aXy?*84{>i-+I_i-7BRSPg;{-qcWE(pZzHd|8uxZq-y=hN+G7u#mIYG{=TryWDohESS2ALbq?4^nf8pve<6u9|{W=JT8D4)R8!wgoPTu)c# z+f%=ot>jLWCMIky!%Va5xYCJ_#{IAbzbJ1~5uQf3=cbSmud`rx4ab|aeN}#ntWja9`4h|&axL&JMrkS*^qQNc zs=^Vkix6?-4y)-H0y!$jVWCVrn0(*aI%)ch%fM-)T#qSn#C>Eu=^}(e9$PnZ9RuIR zL#!z`bE-FR^=cp*tv+9jI7X|%j<<2tDd0pe>YGiWf_-C<-Bjtt3cea z9ab;e37RS5aKimBcvXI;SL>(YtB6>3_~mw7a$bzS_5BNW4>+&3;Ct}${sKF@H1yWiAhc38db*v8EG>@*TLd< zrsQ4ZL2PMV1N$9xVB0c;*@hvgYH_>zi=f2rsR!?;|ffKQTVdU8r1t zp5*;~NDlLFM>$VB*lNB58iR8{#MYcNY5I_^tTB*c{*b=2XK~dBEtq|`1N?V$nL1ND zdf4^~HRd@)g0{<{+yZC%_rMOYel&qbW?E!4G@Mix_pz_B}Dt}O+f4UcH*s%PxIxks38{}MDYI?u6@JCVot7&$ioJBeIq zjmM|oB+fcp!2R$Un7%y-k94(wOx+5Woq8YYEVt6`opm68!UT93CRO@*$;Zr{*wJt$PNj?fTrZCs;DyZ`>#-NsR*toNu^Bd%YWa?#@8o{}y z^c->H?r`#{K#b|zeGBAucaot6XV7}I0HmTKNVic1y)(;(?pmt{77Ogy&pSk+?0YQi zcZ(-h%^&FP*muJ{k;S{LJvW~<`@w26UKUrc8Z{C{@N$Z8-=pPY0|0@iR-;^XN_jrin4$fn}qkwV5 z^;B5Z9fMCUg|&LNm=<-PdE#-NIy(HowPW*8dqRzF(t1lScASIAB!rpbLRg(4z-Wfd z#t7>>4;1$?~x*}?HZrn7Gn$Bskgx7(J%4ul35J-=O+y-4<9$Y&jNh*B0K>MjM zoFC%$ojVUPEA)1f&vq)fUZ;m%sGJ96PZM^%UyCYA2N|PEKDcjii`2*jp&%nnJ-O#} zZig~u%N0p=f+wiF+Dr5&pRp-3QefIQN$`BE27=3GV8oCgdOp8TpYer4q=GJliKdc8 zy)l>(avP&QpJjSZKSzz!)AX`jEx9PSoxcC^j7ZItpc__ZvM22$P(@`Q@)pe{PS?li zm08Bro7#nWmi)>9cW-B$ zop}Muri9|IAB#|RYYqFVRs#>HoCm+oLhkvlhMc$FRP@Xh%6G~OQ_nwV?@RiEu)q&e zV|bFxZuVpHHiVG2h?B_oA(LhYZ`6r4-3gw-e@HJHlaTP`?DSM;=(s!`#6C4KF` zO=T4M4t3(%4NmTVKu*5{Eb~?InGiA(yQ?Zc5xG&M^w@6vWX&v& zkMXaQsjRPt;E=EM8dZW*Kc1j*Z2*1Qah>cpdqvMKzJ$l0@}XGc0eJIPi!$?%Ge$91 zu$AHV3*|n{ihdcWH5kREE^X+Ub_(35bkpU#eq!p|6s%H60qNkm;Q1hcdlxgH>gH?Q zrtL~b_vaFuujX(OM`-$PKjIl}1Yw&`uy@~_2MzNANbsBqr8)+AbzbwYwf!IP_V@xO}}Yjh5T_W<(Tq^ zIZmvodNetvA*OS0(j8Jy-2|(xo6$Wq5;Q$@>2+RVX!X2HUma*f9-&L;X=l=`1PLB{ho;DM3t^y1g~WN+dK=Wu-o z!ew7D%t4QytbRhXMsIM>d?p5tCBW)V16<~0j>4u#&_ZyKj`lAAWiL-sRrn9m5+9S- zupi^E??s6`0aC5Y`B)yVpw)S^Kb#%Uips=;;1X0i0E_g{|xWl7WiO0&D!rMjkC4|VJ>I52OdzG@CelBaG86}U8H!@mfZHZ zj-e{OAXHQhJat7(+l$9&Qg{}FUfc%NupwC6n#W%K{FRwF7svGu0?A1sQ_LEPp?)0p zZuFKexqQ=DH(7EDnv_lA@?j@7ZCD1DHn*VWb^}c2w_rX-`hrVtHdq{;O8(wzp+9ro z(RpezGr08X=2oOy+?%$D9l7cOWgio){+@1~yx-qtDbQM9H_7R{4F0T;&e9C!!5LW@5nq zA_hcf+R?%}rl>t!N{15;GOl+8+0IdJMx?QeC>olg!KGf5*u=4TPkAui2S?epp3c~w z;{%ex1CSfe2jxXExTUNVmO3@TzBUu4pT8Tj)~y5h+l-419)Zl%1m?*PZf0I^0;Y_Q z0`@ad*nASS9x`B|a1Xa%HitF)Zoq13U-S=ojNur8Q5E-KoBj|jb-jr36OS-ASrF|i zVsOW{H|R6g0dg5F$QzVJ#{LWvwWHQt#^XB~@IObhURc4jhBBP>xF3yEeq)tW6wRF} zf(LK3k|UkL==Iqf(k^my=v!wXrj47Efey~>jKK#(?!68 zkZcB_0dq*%`4=9KmT(!GX|Rywq3G2@xH(cpI&{vV!q55W@|7@UOKd>-{XEzzQ49tG z8PIywki20vvYu9D7t^W$8k2zfS1 z{jd0Q++G)u9FBvh1Xg z%M~MV|LemfRBk%;ncWR$$wN4Aa4vl{a2xawN)bz+pRix-J06oS1Vf1<;2Tg$-^t#A zGxJxl>K6vd$DmRSIR6c$u3jU_$-cO-?iG1vbps+VL}J473m8$uL*q^zXUbAkA)&~C zV@Zc0?D~w_UhPD1{5=FcIR-pwDfC%}FTf?vH7l6NG0D%7;k#c@Xe0(aRICoM)~dp8q( zfiZ|F_l2EY7tXcB4kR1;=p=s&^Rc(0)%W>cTrPH=@<1=MJbQw@vuqS}ZiJF)#;-{V z*W(EjjzSHUF0{1vMqjIqbgje;nmWk`iNzJT<^bmjZST+4J?+295_oSCv)}us4 zGvojMBP)5K2d+e^;5I)~jQr!u_S|OK*H5g->F5PiJ48!2W9-x*p}`BpAqy z2)5HZZ{Co0_ic2d$D2vkKh3;4?N5IF@nX*$HwV%DqgWrs2W$4I!hDBUIJ(CUxg)BiYV%~mbG%80u^sNb#5OzN8f#^#SET%7}<3K%CG)}jSJsXrqG3a zI`Wmv(H7#v{6O8osh)USY7tnB-vcjlfh0(8$2lDbV0Kn7Y&#)KFFmUy_so_ODdPhq zQgk^g$eX~Hs(dmx(ir*oZ-tth)8Q2N_h0=N*T+od?uOZ=puDDoeVph+E~{{}!6T}4 zg~fA-NiZTO7X+hSsT5ZBCZVpO4VD{M;@U)8oZ%-(bmGh4!17-1o4^vfBY^xkn2K8K z3ec#KrTZ#NFzIIlT@|L#`eW`o_RoWB^pWvla%U-*(F~O!Y~vg-yWU9ZR@PvwMgSVu zBe23dAi%EaqY?7Kt zz?IuY2sS#=)f?BrP{BH!AEO9r_b2 z7jU6lo5JYq!5%bT&WAi7T^XmSB3nq#CB%BtYVebl_jq2JW_VplEB!nwk4p2wug2veX_XruZoC z5ZA!vQ8lEs|2&>4AK*Cee&pR(Z?Y~Q^>_u%V(u(eu9{+~(SIj#b! zQ-|s3j6^(lXBe~h8-eb}CMHb5mUG8GCjRr)K-NE>2JAb_w6EF&Gy5k=&%qYh9TS3q zUa2(4;|mSHc%1w-aAbE2Xp(G!P2}k&2b|MWrZ*>C1hh9ip-(rwWeUxN@ziv#D?Rrm z%+6PBo03&cMqLZpu=T!}x@i~l^XGOv|E`9dog9J8xEOTioG7kA-VoL8LhSkuK!#Hm zgsXmM?!J6MZ&&^zCBE}vqu)c$PqGwG@th=LE#XA{*=6!JKN!VN9E1+lT2PT&M&n-O zgVE9NG+v}0miIM~BYS2-XxRr6)09r!1iyh#P6<@-o|_i~41oywBgC=mRP z=DM#(_m}agRG6xpX)Huea8%;n>0&;bx$8Wjf=|%CWuH2bs~s4^e7`29FaDb=J7xB^GFUssf9#jKmqh5^g-K53KeHu#ej1T#Ls~9uR6M5{W%x( zeXT-!^3oVlJzJVMegMs5tMNGBEoj>Dfg~4P1kJ}EvEXYO{qso*ipO%Gv9^e`@8|)E z!YsH@^B+5HwIt-J-6I0;bl~6U4o1Gqg8mj+N;299an=zH=J^f=?>GAqzECGrlrh1O zL#bHb_5=@W3W4L4S`hj7iviu2Xfb-9obrD|{z(VIRguFuomr18bs>R#OTg<|6DduI zBC}|C|D&4 z%CDEs%4(cPcLCEC>OyXb3pYongmHIDe=xrlLd-p%Azxn<2xNv~q_rC9 zIz9(X3(dH{FMHYJ4Pz+&M;}F)EOPYV6K3SM5N%sGj!|2;;PkaD3A~m=>Rejb$o(tn zVd>9kG7t~UJbUt>K8sowZihBWZyNo090oW)z3;6zq^Y)?dUgeH{ge)NYjqs%j^Z4! z{QTs3zu1yUgU`yeOjoXxrnW`y20KQ z{0%pg%up$GDp+b=hb6kN347BQvbVj1^cGw2{?H10erzS(`oCa`=qlR7ear0y_%SY8 z1hl?ipyyt+GJhvph`fR`*?svXWZszz*0Lg)lPZJ9j<%4#pY!RjMT4aDr#eZSbV0*O zOE%237B}=;)1-zDyga^v91A-}mxP=~e)~)~d)o{Fn-PA%4{%||# zO@!C_1?2eu1IzYL@b>O4IDTvZj{8->t(UjR8|et56M2CYcgEsVAvH33c}45xf;XT{ zchR6$Yt)yjq`o3I;PG)avT%AjRJwj54>oS5f2TCTTxSKCQ+O75_Wj|`L6szGt^{n< zs)mMWYY;QrM)j&;Jfyvvz`0jUp5a8-U}|5zbBe z5pBZela#l&(dB*%@pt|Oeu1aS*%bk7S4s@Mv_}lC%n0E229~aW?;PdMsatyopq&YsWs3;!EPR>EQ)PGP;N>Y@QMwSqnJXc?VWMbVjL%;WPnX z!cu=xIJ?ycS~{AEtH?B{TPTQPgNw-Rv?kbI%dx4GZ_#aIzhIf3CP{F;O6)S;fS{ox zd4J;sxhb`s{>bmp{gi<8i~mumXw0JVO54aVYsP%Zq0Ca*3s7(snRjjGK+Nyp?6r@X zOC_U3yY3K1rkhhiI6;NZnh+O{MYzu476vJs;cm%c_CD8bC_3|$@!bpm zy7#D6F3a|EeTsOe`QV}T59N8(IC#u;jFQhF6J9bXb!{rTN?PF}cU4lxHe!W;Ec+u? zk?R`=qVkQ?xMjSGd48LlrPXt3>TA+qhlB)P)r9-SCFr52Vv!!DbP^H z?Inc#!0m4`E^n(OUyaKkzpR4J@lwTKZ{@H^Q_#$?YcAfb03;WSYl+_1X> z7VD*<|0*6bDCPw-?)RgJ@-dFXX-}$mYJpQb=OvS#0B4cuxWB;wRA)4!S>y; zq9aTBUj=%7o5XuX3$*<&Aj{} zUnGHGT_Y_af#jxME%}>r2$NFBNW+uAtv%mTAZR!p_9V@qzl!ql!sSH}&{q%fTVtq1 z`%Fv_`Pq_R-OUKHdJwW;HMghR#Icn2GbJi%Xsj(vpX$EAr8kP1_VDe@zBS)5;_ha= z6@3b3ZRWm(ymiE5y*5UTX`+Cr44Ieql*nt@GR@tKxgJ0vtK{TJt7k@`%aoZQp#PG5 zsuZNZZOdrj`a)DH=Khax1CGBu23Ec2N#^E2@<)1%R@$<_%V&aXuUDeaQ%Uk~tel=n z4S+H|I}l6p0vDn8c<^o`jZ)A7>$H9%D6|))bgY=5yt!DQ6U@}CJ_$bK{90t8PqZpnaLD)Di*l_k8 zd3g3Lsd<`?p?9{@wX*i;P!@zjddKOz=jUN<3m^C1{v%K7WiaLY5=z4>Xf3o;o2l-w zfOA~>K^jfEs0*QmN)WkIfsVeBfUHe5+zfuQwdCeSLWh9 z;P_zISA*{)_pN`aMi%|4qCeuxQ7t%$o)Z5`AK1r}N80C!`c_?H^DLP(+2wL)pWYU> zVwl!BEkXb32IQBwC0X1z22<87Lqio!SbwpdJmQ#`A+H7CP*WNS5}vQy5tsw~Gr64E zmjF0pzK3xiI7hFI9>Hnq2T+6a5z-Tx=)H6TCCgRt=!#0R^zs9&kg$cd=iakh6Wz%i zuVJP!V=1hzYb2)>oan>9N>Gwk2t2E5@t*Nv=6b9YyyJRVSw9bnud+pHp%^md z*iQCZ$!?Tcv=6H+gfKZI4^1bIz@i*U^7*_0ZL};Te~sfIZ^J06$>-Dd@gumgdKM^{ z_`#u&r8sMyAbK6xOulN0!qlu~cqHmH#P$|}QmiXfohbuB?+h?FBS-DTEzvg14MJ5d zNW&jv_G-yinE&}JZivgkyGjw{+J#GG^`?K!(95%k`C4bKq(?0#*B=FM!cm<`9<^^3;us@8cdb}KsUl|+;CbswT=+F&SgcI5H!pAF6t}z#M z_uN6>poN$<_>H_85+-}p3BB~r1Y{dkbn{l(Q(-w@aJtcm9*!SrDcpeFCgLda_XzI! z^tUCrBNUYf_k#3rD+GM6p|OXhk^g2RGRvlb$Iub_$wG#WdVHB~b$@`h$@6uptu%;1 zMjZ|Kp#Vx&xiIaf6&D;2rq@O9p*@$m`WH|}J-se-jKxF{{-=pkQ(mJiwa2qx-lEyq zLrhOKh7G1anJjZH(!6DajhWgCJD%v#U`uNn_<0|e>n>$NH25Jz{x+(x?&c+Kjj7!w)zKqlD8dYwaQSZ zV=Y=o=wREH_ZVe*2K-`k$vdq9TweH_)z}7VdDxL0>qVq#rR=rIBDzN*1UJfa z=XKRAX0g6Ky&`!N<*imgWR*3lENj3yS6|S9c~`*xCfC_-l49G_*3-_VYr#O&T(@z- zTga7=;<8|i(b@3?NUOhwRR($>*E~p@*F7dRUjpGpSSb{uNe1AnF$?B1WreRl_G`JO3kZOKtQ z`(z{Xw0y;Kk*C<0`H#G6YXQgo)_5(>oYB_u1Zr`BIiOa9-cQDn&oKxBJD-z;cz-C^ z8jButHHntlTjqiB9!lr#Li_DUAPWC+ilTuaVIWvy9|adF2w(a zGxNM}8W7R@$QpQH>JqcoWK<>((?-dGmCqq>*>tw_%|6^rImR2?j0Qd0urWp)&F1cG zUFUWU)*g(91KS5E-?Mt;d-Mz=e~v-;BSG{P;<$PN+^#!A68xmAnX*rXP!u#{cXI^&!bzR=#DjP1t6rBY6 z@|tw%>m9&Gyh11aKKjXJf|P#Dr==mLzs`Nx^ixMU0xB9dtJ&5Nin z5CtLg{OO}Yf*YmPapt=ray#c1^6u+J?2IrMXQ5PFp6aX+Q{*xDp&rcVt&Zhv+W07}219xq_cg3zY07Rw5#Ti!YKz zpSn2DQz+g0p$AoO3J}|fZz#ECCXKK)$GFH`l4GSwd&inEEkKB-?A!?R_{>mmry(8P zbQsK9IPQ#kB`Lm~1hPA8fal&kaFOHKBvS`LV`m2$s?-D1b7ygVzyl0F8;^zt4`{5c zB{j&?A_La)3b?^_4HSu}Fjv^!AQ5+tt_pIdC+l2c$;5iPRJI8Dxo(SX z6ED5}GaFAQ>!6~_M%aEL6!>QBAY|JSCdu80%K+*@B$Guqa4wufv%DZTUyStr$b*F0 zzT~c5AFD9TLwbMr!mh*LF*wv2TwnEZ-CYYd$Xy(L)B4DGW-!U?e~wFLUMGoHlW57t zHj*!@!@Ld{2Ybs5h%)Qd)x5lf7FKG5*hkJa_Pv%a5SvS%4P1hXUsEAjq>xT5jwHD= z2%Y5VCG$}mPWTF=*~D{*?)Jk9l}qTLHQthKszu-1SfEMH2(IRM5LRcnZ_5W2>Sgai zZ$B8v{nt3Iqy0P-?cp3dY5c3d*xaS}xmjl3p@Vcz`xe+)*$eWokHgwEKTzzg3c@=_ ztWmrM`Dx}A6XaRhHD)- z>+ol|80JRL3}+KD%XG5mX(3K6=tAG``$^Q5KFBq5Mjcye^7~69qzt4H!3CG;yO#?f zO}&rI6+TMFw+NAYZ-%inpc<5%hOs50msLI{iFeF;pz`Aw3N(en^e`mO`Xl5j+(HBX z3OM}fFF9=_i{?KPbn`D<#VzG5=gqr?rOy|$OKpa5)^T87JH(>Tfi9vq)sNeuf1#Jp zXyB^9c1&WlBb(>p3$}hAp)t(^gs(WmDPvI(NMA~h_aAM|w1|RM^nn%8muQzzE-U+U z0y@q8;L5!VSQELNv9956vqbLuS>XY82McJ8_Xt>Z&Y>}C#u%3{3!M5sL-*#_ zDA{cR>Q5ZWJoDe|_zFoHKJz>18fc=)Eq*L@-2!1sk6EeHPf)5>677vNU}=&h%+~8B z^1+?-JZ-p8nl5fBuGfV)gJ{5!gJdXG1VF?BW`%!%U7mla5LdQb959a9Hr*cWWtwYAy z^w+Hec=gyvwC8(G8q%XN<&+l8;_k;eUpB!)&I1^i6F`4&8lX$wmEio8SEyW&jA{$N z5m|FbP(0I)q3)L;V0#@dxcm>U?sg>S+kOzu?rU^;iyqtaHUn)r-}rf1(6StXy%M)Z!x9Aft4EY~@?fx;iZ(w31d%-z0){M)UL)+=KnesBqFao9o@ z-z*~UxJ=iyCo>?590i#HYm7Ur39CC^La?VCZY84h@=IOTc(x(=%|pO>Q99@^c89c9 z7Of>mxg0|shOf~hzlN$rRH4I63!-f<389v2K*wnWvsc*Tm30!B zbuR{%oSuhCg=%O&UP#S{tcZm66OKPGf*t)kFsJ7U5wg39#zXq3F=&r^BhNVQwk=uH zW2bvKrI+5@zlvnjNLr9oPRq7D*7e&wl{|+>x|xD6QQB%6bpJ|*d8`@AN9AHnjuP{# z;Xg>Ub|&AFtVu{Y$Mbb{APZi< z{0&Tcu@rMsy2$r3Z#ZP!#`)==f{p)vbi4lm!pc%0#K#+Sszk}f^sA^X!H@QL^6;to zGGtgI;LH1l+Rt2&h>7CK-b^+%)`q<7n?jy+{lIPYd%?Y60lA$b1Yw4XU~;Puyw9|< zt6f>5oXPPo^G0w{x&+w1xX*MtZw8CUb1-(vUaSfHge@w?q;X3N9go)Hm~Tj3d`qZE zYdme&p9iat-NGH6m7r=EM3Nq7NgR(q3-gDa8u~ z%7T#Orv}Z{zcBV@lCETvCOw$?j{Ns(GUD8q5!!^LC}lNd zW<*KZJ0qK{h%(ZS=iHZ)5G@T2rL?~-R8ms4ak!EK;GzfZx zX-jrc=|cw~UVRb@!ep4%sR>odYr)M;nq#0MDuwbh-!E(;!C&>Mx?u&aBGw?|mPagW zui$wRO)S{Uj_l$92Cy?AbtB)>G81lAT6W{68x1Mdu}&c^=LxVuXc=GTLC*GLb1hQ7@`;U zaJ|M~u;_jg5vgBA+jV!*mY*9jaJLu79Fu?)5q}goVF)t>f54P^S}w#%s6|b2&fy@wYL(tFs$Z$6MJiFXGXz3&2!l9DTQMp`CK;K%i;` z9!QWNS4(Z_;Q3ydVZ8wKqqrW+wowdp45aU7)uVWS0zG_D7%fj<;oc!ufzPNl%3W6l zF}HqL?bJ)|n@=I7YlLyHvlLmg{s7K$;77|pTCji9bgo{Mr z+CM+EIuM9^rgK%fd%+mCte@!}{0xUvUGT!~CFs0j15p1=O#H4zbv(H<^ozY%w>XmX zJbGc~E?c@Jjhk0oT?y{V;jFQIC3W~YnN0+)mm9<7c`mfkY3+RI9b*VVHGXVSY$z10 z8)d!IZbRfG;6Z6<@-==mlrIsNg;mob%9Cd!`&l)VK!Ta@&h{b za(*Bv`7Wd~FT)|Yj}W@~5xOfifoYd3R=*S{yi%_Bu|E)E$BttXUoC8U*AAO)vuS(& zBf`c|5X_!RujTKCOH{4Yb%=63{lgerF3ROb{lU2K7TQ{bBh$W=9Nw@DeJ`hi z$Q-V(x$!xidjA~~pB<;0^23=@&(+}iK$Gg98Uhg}05;c$-~#UZjrUa{9j|`S;-1Hl zRTP66V`1R2`6S6PAH(B2GcdJV1yU0Vu+n{mo53H)S?x$(ou66|X z_l4p*Cv%RwQbeEi=8$8nlyRO+2fY*G4v}&ynDK8JeLZggkFHQRc~RZmGwGY&s)zEa$ zYx@1C1>RO8?pi z!*=hZAhFP&r1INQqA4&;Ah?h zb{VVb;+wZ<>%$@{Uw#ed)Hvc|uIJhAW`+i7&2;BXW458i8D|zO#1n1e=zn$}ic6WW zMJYSj=B92sUQbK}Zc;QajO08|r8H++0>`jPqA$a_`Sr3-WJ!5Du2(U^ISbNBW^EB1 z9kj(uBJXh8_qimW%O2A9V3<}=%@i9afpwoUSl3#iA*@Ez@hhaLz#1;+X|c(ls_2W^ zXGjL;WR(;?hfQZ2amS$rkdvzn$NY<+ActeQSZ9Dx*%{FG(WL1ODJx=(?tFi9*2arYf;opiaK|+;PM^z=&`g8TRy+%_TRI>px24Z zV%PFEcJ3snf;>?D%mEs8QJfCz-y;vPjeNG{a)q~J!Jt-xl>LfA-A_u?mt(?+H2#E( zL!NP%bBUdmE^{kM38y5fISaJn7r~W-rE{QpS`@u&3)&C z<@0jzzGhBs)D-B&*#mg-Q60o=_)A0$&(Z6`J>_bIBCI1Afj~=H+ z%kGc?Eg!mV@fMnqTMKK>Q=A>V5Z}zV!c>JE+SlSu&OCdJS49+A>AY1aC^bNG*WTs& zcaO=I@Ixkx=On=9c^^0*^=5kY?j%T0@r2q()94+49a53QkDAjp$$p`ixOR3fbQb%- zypd-x-7k`Ua!&-oh*~_GIUCh3e1pvV{p>CG1u%aP*WrF~8|5`O)4Ydma4h>4n(D2m zFOz{|YCS^Xy-#TBfa217Z|Y&6;9bb^cOolKEl2M+#l}YZ4d|tOgYvr0LCF&>l%2bt z{-s<-rrCg=w~!35HwGDhBgh)4q;0NVba9dD>pjtz4k)hc7c?>&y1fJ4X*)S0?-FPNG{$5GYOeg!#*-&@=aCiIU?-s2++% znM(_q?{7qz(b$tHBz?@d>$wK~!)HgPl^)|hQ#rKQJg0R5n^D?jH^^Lk2Fo*^fNIfn z(0gsgF8n?X#r>y1=};b=$ehUx498H}!#sLN^)Hm>hl6XTGbGx+pw4-S6@DFHBo+wK z?>XnTUK;lwFM#F?B`EQxiu~f7Z$`2TEMX0Fu(bqXows&ax85`NLPxkL!sg z%CChp8ATkQW&`N@kAnT&C|qH12qFg_QMYfyWCgDu&HMyGxA6ja_Bk*`RDdMbpCuFi z00qvUsO{l9uwh9pF}i#Q+eH6Ch=m_ICA~tYH(exoZ!DG!I&=H$dl+A_lHOfaiJ7}s z(5QRSocCE7<91IXE6lrbRee0Yn>?EcZ~Tt?)Bk|MwMVqlU6H*i(}nDTDsr>%5b0!E z!7$y1dhL3N!k4Ac*X0V?vndsM&oASrXIrp*Zz1j+1k}&^fJ#4t(XnC>Q#AjP$$GbN z@?ukr@#}%e-3MS_ODxXb?1Zy=_mEU$FEA4EGbwx4NbX|#OdYESf2U!Sb||G=?P%{;^gJ8CZ?}22v)9tPJd{`q2T^B8qf9IHE-t-J?GgVf4UvwM*pMH zHsyGJ{28V%n1HRP+#n@r5&Av<4a)_ulPv`d=_r{>-?)5Y_ukr12KwgE>SIRqthor8 z*!&5656q?y5~OIy8WoJeGMvBX2N_e~`1-w;SZn^CTv)u8h{0c!j;(=X_XMC{S_w8o zIchA7BWG4|9jAN7^ctPSep@3>&l(-VeHCM9;(C`%$-e~3t@jvv?LUyBbsRU&=_mhf z>?h6Zmc#T#c2Ha3z%=D>4rpEgN;L{Y;z|R0=9L6H63p#pbNkqK&3|a`@(-X&%+$< z{cG8`ZNR5yMvLz(BMrUWXY*>J>CznIUZRKI<;jA)NhiX!5!&lL12pQwNuy5;cDHc3 zxowuD=XwiyqhbZV7U$@!l_{7tJPOkHcY#Z!HfH{AAZtsfVB0kdI6NFi>MyTmpQ)}x z7m`m4MfA{WNEOpmZ-R8}9}K9CqdK+wAPI&Ux!oV3web>cOaF=)8y=%}rzWlV;6EZU~pdNzUE2R>d94V>td|!6+s?mW0fP z9w>VE4J$Vu!aL#Ivy^WRH8<{rhn@MX^!3X`<8Cz3pZ)-(tXe@b=_J-JJIv)ZH=v?m zKa^+dV)k1xl$hQGM=g4R`FzJDg%`yH3G*W)Zo;5o9vpAEh0DqcK*;SSYt8Kxmh9)c zx=$+cn4~GaeeejKoN@?4Nhhd{8_}XkVI=!+1Xkh$dc05>7M2|)AEt1*n&x_}Xb#1* zK4)N=eGP_>3y@dyN1%e^tQ8=!!C60~bvr$y`EFuXOgz<-$I3a8t_`a{lOrO0Ck%DK5* z`CN)S>gkPT6D(h$3%wzv1>!T(A$oc=dm?-hCdF+4_p~is-Z_kI_}~naeJtp6&P!xh zRg5lmmaz7&C|dfZVYJ^2cCg8jq>1)17s_?$o2SBr9b1l`Rt31jjLR@iPshj=zsb0* z118`8hV=~xxbxtA+*_qeMC2*zYNZj)u~&$q53qWC5qGUcLx#~IruuIX$|QBtn_IS$ z^eI*J%wQkYwtj;v4~b*axG{6QeI0n}FJQh-+fU;&_F#tVC)6r7$NY$!^y-`}!guXC zE(oe*Z&^-9w{0GPLUT}1n-ex3ehDj?UfAT)LB{JjH%C|)94#}1?9&g)j~|1O{mdJc zlzl;Q>OtJ8d&?yMyBehPZy=w;Cu3001i3X^n(M*mk$Zn1a1PRo;Bq(wI4LY-fA$1b z$G4EXn#;OdrJ`5v9-`Ulz)m^dN)uGyV8x9Q+-%iECi#bui?#=#^6znaYX5W+c={+a zFO_q&EIEKy8~!4(bTw&A`-bT;d6@sz8xm{I!0z}r^x4uo7`9sqe3UY%-j`!+E8iFL zfje_5?4OG^oA1&Nc~PusS&aGVQef)#3N~rSGe@h8N&Qp_*3j)MN><;;sNZdD_TsxJ zaLbu~j{6BaBK1K$p#qKu_mD@z^2C3g1#Y53Zbo=@EY#S7`;G!mj#)I81v!|D&%Px#xPRD;}9S4)JCa7;Rw(lO0nax#cLT&aFntg-Q5-H;H=S zM`mQ=16k~=jgof%VPaP}YiG-OF!genpV`AS?(S)8ovN`c#HxgSy}=6oevBq=&eW2xF+LZ3!M#C>xbcVHGA8Di+{gy*0%(E?2i zIfm=#EKK?_j6!8U*>G(YVs^dJ2%X-U1r=B+F_a@3n<&hR3gQkP)%1~=lc_!|mw^ZtEfR>aHa6;wM8 z;=1J{F!$#JP;OfYXY5m8t>76{J(WT&v?d_=)*kLGWKSD@wo;ojnKXG@EOx@Nl;x z$IQ-WM&l-c5oN&9SBZYfT1}G|=Axv+G1w#Thzc&j*q;-KGv)-6`Gw;c^CJM(_12RM zL;57+*ld_Pu4Akd00-3=$+>x-Dk?ov@hyGpyv-|z@ruza^b*dxxodd|v362Bb z_8Qb)eS`ds18~fA0!}7;gPd0*w?c`}caG4r8 zJULwxl86Bu+<6qsI||6e+ zCK`dZC1IexrxjKUJ%sR@R_n#&)q_jwLeSZ14jV0!Xo37! zny(>8+jee6>w@PPA$T08*~&4eiUoi8whk}eU$dVO->4g<2xbg`cj`N^N z*Dpfhz8j3L)+IbpU{3o?)sdpCvFn2pf&{Yx^LEkCyF7RmQ$n$((G#Ppe$nfdDfE3_ zEZmgupqF=FrXQ>l;D(_py^}PG)Bcm9Ior~}zUe$le)&af@_*r#ryqc6@j@3l73zP| z2^_z0Zquu)QR&H2v~XJhrD~7q_BE2^a!WtB5=V0X)nV8*=!W4(N5P4Xk$a~@ zcS~F)GE;A3;-V5%3FK%h+>9!4OFOzt)Y91L$Jup0y`VaHhaPNMK}D?j$i>i5YFMzu zWHHAW4~R$sIqweenS6&v+>wLwzudL3-V~?Z<(`{L@~~m_F)#RNEA0NmvEldqM@(if zLd$hJsQxE{-qo|G7nSe9v6Nw$W2FU|pVpXM%^Q|+8V#GMC#R3q%AO>@1zah3f9GW7$@!g(}p+@&xIqg*57nCNBMH53(8Mu+--Sn<}>eLjS&|`?aTG@@!qW^gfWj9{9;< zaQyo`k+K1?arnaA%4t3B^#2M9146m zfw}{gIBQ`5I!{^()6bQ{j9Dx>spd-RcwSJ$n+#%m7Sfk>X)ydR4la#e!A+i{G|poM zU3xSWrzM8K^lIrA{*ZF=KxR8mmtBRy_v6@g7O!a2l|R(9#+eNHcEco#Z@6rJFX;|b zgtqyYA;4J&l?@aiNkSWzRC=TKg8P`eVhi(9Aq+3)Pq442Eu>55PoU&zJ=eG3vcD;3 zV1|Vinao=b$uFxhEjAVnzKN3T6|X>Ikq)F?U%%+TN(CsT*NaxuCEwBf|*IaklD;*KHP4=_}cr#vzKN5 z?KVS?{)6n-VjIvc+eF^xKc>I`X=37%V{k^r32w=jU__!5D!)4m#%mJjNXG^`Sv-(q zH$P)a{pwMh?*qgbnj06zHejL6WMn6E9qziP81(QUBdIJ(ekPt{P8D2X?`qG7QyTZ^ z*3UUOtwaDfs+)tlk~MwCYQgI1-pFASi9_>IoWA!xIVYNrp;tnQX7WbL(>hH96dwbh z&{1@~7e*&DlVQU~LlWC`iG-J0n^@&=IqFjtGI7pBXyH8Mc{TwdWpTY>K ztcP4DO}3r=3lCqJBb+`BrFUHjmn(qIG4@&{?M^A24+GU(VubmX%zbB-SU9vTKqTPRenD1!{&FsK?j z2mCF^N%t9ZNC}Jsh5Dsv)v}4p8;a65pAXUGpkH)oK_z*yi348jVYj3m+V;gwDah2UDu>~WCc~&>qVy}Xn<^5 zFem7ng(jKFeO~Y9cL~P1_G>pGQ58Q6z4eUz7@h{)Wo#PjnI$KBZ+#6@?5-Ez| ze*)>JrARe>yZsnr`azbo)PB=15|VQNA?8=L9&fH9F&%VX^u{qIi3n} zsn*oiJAqAD`VI$%2JqIxGAz26K&GYE(yy79G5FCC&2PR+n8FG2=xQ;>Wu3x+$y*`C za+npU7o>-}hI!9kY$8q8)~Gr0jhmBY0zWs`m170iw7y_aJop7xJBKs3hU5{{)lqqi zTZ_sa?s+8s8|`YQAwA8x|DJcyS<7`;(DbLtiCoX?$1W23RDh=R7sEAOg2wWDsiOG< z@Cm)gJhl%YZARi~G(1QhH;j={>rt9vIhB5n3}(OlO#uaw9+a={LyaXItKq60DS0SD zM_0cE55FoBvG+0txP@cdj34AAmmAOa&c^M#e;6N3UE2KHPlfgjY=%OP(>G`LPE2Vz zg(G+8pzd%xu3pJ8UilkoyyS9XH~$Z+|Jo0ooLg&sVjNWKcfbzIe&&;mHwpiA8B$EG zK(O{9xxmd$mEIMjmb@UY=j#PYjdLjeUXy8`=pzL~rL^De3K=ZIj{EB6i7?@(tqU%Q0cM@Pt>!U3{n@mk2){s(5aa{l)4M-X%ACfd~rL;3|56z?eJ z{o_0o>#uUV?3JIHf}n*E;Bgyszm0+F(NMtOTd_D~Dai@wC(q*clMY2WP;BJfjF)(z zy>UMj@^gGW+g%XX69LB}qjAT&2PBr81qARTuCR~5oBpPBOm=|%^wg8v3r!`<_v;6=Cs$!o zU<71rtR~*J+Vq~f3p(9tr0lVsn6qXEW^IrprgE_)epM$MyyHKTrSKmv(_F}Y+4<9? z%h4UB@2tkjRoY;{-J8xmm_jCaLonE8$28nBqO8|NNQyj0H&&&=>Yw|J>s8)S=2|HY zz2wR*HqD7}%U;|&Fp2)v;=I?jud!Ne8vVCsAD&*92uI!?WLsv>h9g>b;0*jX%d^=#MeO`zaVhBW%q`0daS2yp|?Ne}X%hDzrl4mJ0gXK!e`B^8_xW z{-F~`7vOQeO!RZ!g^6yNs9NxcWAF6QmJcEj-hTngbb3JcP8=>uwu5b%4`3;mZK_N> zfF@mH=u`fN?7rZDdp&9(T`Un!nPigx0xU5`*aZ?i)$xO}FK#HA40&N{+?h^`z78to z`b?Xs=hZ8u`13ty_`#wlT@Vu_3ss6HK{Z7ZB1v{Oauz+UuSTBp?<7XcxSR{v5r<>vX+x4TBBVSB!9d$Rs3iCS^!!miVj*^IHX2~Cz%$DIr3oY(!ZgO8B?K$%T!;cxlk|wqi^OO#$BW*+lj8QHbW;FOomA zX@}?`s;Yk*rwY`ga7Q~h7JFdrrDT}0+n+7I#&LU#JYe!Bbz1LxgB{f9!r9eoxJh;n zeYNW*x$`*&1oH*xRJAn>t{H;GpSy7R7Y|t9SWObegHS-@0M1=eL63;~fhA5r?khX? zPx40OuM-2Ml_%L@k9ANId>!URc2kGyB@m$X*JRXjHpu8IV+wS#Lzk}LZIv+0Jou2y zo4>~Cg>qo_A7KV+onT+1D^3|M!TE(VxXj)V{Vy|#=>6h2qha|pciKzHw0373Pp$_W z-gZ3vLlG74B$KPN=9BV8`*A?vDi)kW@_sAF@2Fh?3UhDbK|@K5-L(U>HjjgqMH3`` zwFVoh6Kx#s>0c8=+%$+2_eO?f@I?ID*|3{*=xr^R!7Tu&CH z&V@b63sK}{HYs}jmJCgmq(WI5U|b+U&kVhwN0cg{lGlfdrj@wyq5-B`2r=}DJQ*n; zz}+89(EQN}cGtOVa@W@zlb>85T_vrMccl%Fba9>_jZi2v8H1ZDb#Q9$B#038qeG(W z;9x*2{cawDPTE?S=v4vzU?_>O*Rltr-kJ;}c#%zaJDoKg;Cgrhn zL3h6^=q_&~I!)JE=@wszU$euc^x#2s_sSsZwHA~o;>EimodG7sduiFLPz=(IhEzOtNaS(} zXAaB5tZnD0%ITGi+>V!6^F$FPuNjggzhx$UVjJ0${b}s*F#+0NS4t&XEt!*z-|&pE z8Xo?Zg*!Kun~bb!p+Y^rptV1dPIQ%n?#wu>8@P(a5_edatrIxAi0e`fhcMYUYe}0} z1n(=ep4#TNL8-emgT)&&=8uEH@2PX5Dr&xM#IJqfb(zy!)W=~B-< z7Nj6#FIGfc#|!)H=@7pVy(vl|)wY@7P(Vp$c*Ie9Ws zwqOabJ9ZB)>A1)3id(_q8+WGB+RqO2VsLqjCi!*b5zjQc66|`M>AU4?A(8&1<%0X! z?q6%@bmL~45uMQd;Dj^8=>o*wNG4H_r_zF#H8jTWIbAtxg1#P`f$wf^z=VXmFiW6- z-fG$dv%2hQYWPd$u|XZU-2!bX8OX2O07iS1&z{NGYC zV!Mv;@EXr8G7Bc5F|VAr1Lt|~rPbwi+&+WL#Z6j5C;a{+()=Qj@ooTiAF(6ieLK-H zZz~zSwE)jC0`&fo5>)c8gd%P}>d@stUldG*GQSICbczTFlw<=eya%GbmXHu8gHqz_ z7yrY}oa3&$aK!m16j6PDHpG|lfx_h!5LOySg&AJ-k?=Ll zb$EiI68W^}SvmWk^b=5ibrdDGYr_m}VVq-ZhO++OsCi`>d-4g(WmXzUYGocyl9)!? zBAf8NcqmK>Uy8eXbCHj8wI9uUPgGj~lyL9h zepcj69Y_Zs#U+O4(yvHm9Q>{&;91Pq|=vjmi^Oar6kyHTKY zKgKNkOJ%wO$;#&(t7P{Q&Z($FhwlqAaynsT>0XN4<{Q&^;~B8}GiTjQzr)@$F9B1Z z)p&Kc7THNS?t{c;BDp-8DvwsSh}k+bVqk3RHL?Zi^*$+4`hw!N*(r)n%%-#&slg~XsP-?Zw zsdYDr`IVR0dH*ae{x=_Y6-83CoPqu_vq^@}6!LWHCHn19DS8X_@RIIEVEnCX*j*cq zTg;z=$Tfo6$$ycRb>jNX-wBESYhv(G0hHg_k_wRo6nNuIj@P{>FYHp-Q)3U=JV##) z7j498V^Xl<;yvyiC!Cx2w-Ef*0E73MLD^|1(;X#_SKGKA#nejjLue74I^T;Ae~eY-ToEu=oy1Wz9+$T-sv4`5S;7AHLf3=Sw z{BB2J&+5zcZ;mVdd`$+Hoz;YGD%m9dv<+koM&Z#2E=M!>6o&kaAXm*a>D3kQiO|PE za1Gdp5krX>Bc@7~zHg@$#Y>^=_yu&3X(YYB&tmx7BdC9{5SPon!)TFTq)^ZRbW+D* zw?r));&Y#V3z&z>A3LEfb30k!r%iu2CxT3fGJDAT2GF=k@X@~;T;`Y%%|Z>(z3+-M zWv^jv;Qx6bBUI1L1gz(`w@k9~f^#njz3!w!Mn2fV)-%P-w`djG+@68&GCWDTemLrM z*^r_Y1YF(3QAS$;^}ZI9#2=hrb)F}Lu2rVW7E#RSXf66cBL>2H193?f*U>lOxceoV z^v11VEHj>qwT%rX4Q@Q@p0^Ndh6iX$Kp0H=;Y!cH4X4-Fe!`8jZP3$O2v2>Gq1m~+ zp~l=9qvtNA8pB&qE=Gr@lZWK{_Wk7dq5&H1c9#mp3PF_9S~9Y-25SY<(YR5A%iv#u z+S#qRGjczXRFI%%2R7oo%QcYe^97b-5lk)&gEP}M!S*ejHzwDMnRja;O!Xe%a)-as z#&kPsiC4kwgWN1`^G?)g{zTw*E@SYpl3LHJf+S5Ruvz_zIXke5bXknkmA~$Q)n#$a zzfU07cORWvzm}F}ib7V73lTh$Kt__1V9rPhG7V z9;G7V$j9k2EqtL_-2E1?b>>;Ha;P(|iYVspW4`$2nJ4SLpX-$V^5fotJ2AdR7Iou4 zQn4>C)S&qh`26xEUpHHz?%dbtqZo%pC*)Xzj}~0#FA%PJccJ$S?mg|>4&u1x9Jyxk z3gVAgfv938^X9q)dX9vX#M3-1GW$k3q92$0%0LZ`D_}2hiPTHiknhoPAXfT=i2Dlh z&ic#;SlvZNLesG(d;kmWwK(qVLQHx633O$GAuT0|Eo$YSB@qhDpRc)GMq(NsF7L%{ zmwUNv$5Zn4?l$&VV*_2Lqd^C!hH^9X=_c1L7Lpf=vq;_+K3*L+3(C;n#~O%9Lu>gQ z@R}2ayZ06&rutI$Rvt=DeuPJ^ma!Epeo>w%$4Z|01m>?*B-4`RLF&+D)KCA9Xqh|W z${CMAdx|m6`x?URvGfGP{d)A}bW0MKagRzkT&JHz!pWK8V&H|*uSVxt(mM;Sv()xlT(JZwDN0M{{ueZJTT=3o2(-qDHdJMo!J zg#mXit_wg#R~5*uO~f_!GW7PjFR(q;6`H#fX{5je2nq+Iw+F}Zo2gEpzw@M}G0GrQ zya5Jdc}B@vHgQTRn8=!o*6%OnQnb9BkJ-$!U=!bLPv>O|2Ee__)HAGV|~ z29s}?vV}5RnUngDsb2aRXbvuggDel0?&gEN3L3EDUpK8jSV&Blar~J_xx{YTCrmsW zgZ~!HVoeXN#U&34K>ws4&M^(978id()Dyq5-9Vk z4ODU`(+z)~L5WW}%4?ohDn#5~-pHv|l|H+E%||hjgxx zO(_f6GyB|_K0QAu+F=8+HP@la&jO}CuYj!DS53jUzruzeTvj&c37fpb16+sum@)Ti zNW6L+S4Z7LC7~m9Ne4xFt}hyt_5xHShA@C*G_)Q+2q__ExMcM#+WSq2WZXA~#OAZ6 z2CW{r_?Qw_ygx~JJsPBQZ5ouwpGQN}f$^vlt|bg1VJ#H`v#N|yaYiC<;dP?Aga+hpmZOOVyD?W`*oYBU7qY%*7izj8* zIF9Mvjl?7W4t+f9IEJlIfW;LP)Zp1&aHgNYw!f0R*V_Pr%bQ{Lxe{<#@Dz$N{-axU zW5~4Y+_j>4oR&#<(-CDua27$F^J^n6v2?{{ii6;>CIwJTzd@O_eIV2$2mD3v z;Lo0ev~E>2J!2B{`Y|Py9W#j5pcx9z@*tga3*d#z z7`>MM2c_Pd)AQ*MVH1B6X3P*karQVaD>{G|w}|BCF+8?#2c{Nk(XdaB;3Kh^q(8sF^mRXi$_#1x z#ESclH<+QG(?QrGQ3JyNJQ;@>!tmyeGj$*K!hq*?xUwM>+~w!fzB~()PG3Hl_hT*E zU^wV-^8km)0UD+M1c{q3?J$cZ#kDnkFe7729|}XFexvU^V$X`Ug)b+#;qc zf0)#k%>-=$E&A*2e0tre9plG6Al-Q`2D}rXS8817zR_@)kygUWygfj@b_-(6v!(QR zy(T@8<3{ejJ&TJ^{~%>tM)u#$Qn2@{KrNec@>aeHmSu}!iH-^}$1|p=cLkZ^Drjpm& zrRdhqSTs8K3#AT};lW2qu=nseoTKms5?}p>3|Cch_C^M584LuAm-WD(dJGcpOS7uA zA@t;hb?71c6I4TOAyG30_pI3stE7WyDPs>?g>0A$4^r@o|Nq*%^SGL}z5(E;NrRA3 z$WV$TlBm?#za@!6$sEPys8cdDCmEA2bu&eVjJYZiA+xi8E3SE-GSBlobItC%clQeS zdGOrl-uHd~`Stm%v)lWuy@u~vd+&A5*@Fc-J&?-OLCVnQEcPfD`rNRZ$vsleE4_efKWdoeVgT@wuBCnH}+xM_glDZV^ zlQly33cGOA)z#>!7mQJlk7D`c6tUQmK~7d2`g`2ObRMP*N^%i%Hw+TDjw}|Bj9!K| zWB0+@^b{C%!U7gA`azgp&l+r9stRJChcI?QZRl6a7OUiLk*?(q6sxxR0qd+>49=VH zfz^Nr;lbP+np1f%M6rTOGe`S@a3Q%Xs^+g2pEfxlJr|@C43n;NuN`S7_%-Q=KkiKt zr;d4y<(7{SdmhUc^bg+=pRcfn@`{(D_rCgAVBG^}+YX*)XK0qcHzesPN|54DnsB zOE|5yom=9)ec<0~I$D_@f#hNpj?O&?dhc2ZeLKy9^oU7NdGQNu;C&K09khW(U8_Ql zI9vJ-M7q~&RC95sX$w{;+XM(%tlytt9t@cZt;-rUu6GFlBUW~A%PY!tI z%~uB8Fw^8o_tMckzb@TBt}UdGt_EwR>y_hg6^q4Q;aK4E7<+D4V3_p$%=+<;=Q?=(>Ilbk|t~;c7c{s#Stso6Ur_i?2h+W{KEgd0XlE z2!2>+L5jG#;|mC0BMRO0y`=MDx1vi?lwfdBPaJ3c5URz#6ptU?D5R`WL(>LDIH`WI z(9l%sBd%Lu#FXYr+qG{+hegA%&G80;f9^zR_x>%qp06zA%pM}FICKjFGWgz=RN`w6E+UWVV;ihs@1T_SW|1F6A@WupvHN zr_u}_n6n)wOTYg{EeQbA>}(i!>NRS0Gz9B_X1uvhB#%wA=N3JZ;g?sBxY6jxypl;K ztUvgHJ7m;=d0StIdqV=4w+9zDh3^J&zNkGK#5hulTXJ*6w%SF(^*KKq*( zX*h*>z42p>UcQx{0UyF!J-N$!dG}zk_T}O7tJi$e%u&#~PGt;pkvIzKvc%kHm+RiyR76IpG9W>*HY&PFSEL!G5O*?gyfd9^`!G!^cz-N}s$2H;}5VC*}j2d`6e z3M@;Lj%jmZSjW?@kY(wE?*?_^bsns9pFcYSlG<}NCVsWhb8#VOj?W-%PIqSY%M4a? z#3h!X+i}a>m{_KzwVgY3?9ZCs+@L&`{{XHFk+3V~5q8=&8yseM^4e!hxbDik@V3cX z`B?|J*~1*;=ROkdHumLRjAvr^9}F??#d#R5im;c08SfYZSnuzALPSh{32KGLeg6b7BSV_^f#ftTRB!4g-!j^P!4 zD`1w3&#<7paJcPw3npDX$|rE?**Yp`?6ze*t~_QSCSARTBWA^;d6+v+zg-KBl6+Xv z>k2IBs4nJb|H^bayx{d`)ZnHG!Pvmy09g7j$9aj*QFZ?TgDN{A$?P{C7pdU&&qp%X zJt3H$zlbU3+Tnr|&RiX%VI#d~<5c@{%xz#5-mU08i+gIvThy4(A|ge&IA=ZtOt6AE zkMq#2!7gq;Aq1~PL^At@N0?bb4(mAWIoAoQjX230&y8<{BaJUYqUQ1_t;?4fW ze9e-jtVNd|tgfdo?30d;0}a$LK8pIsR{jF_{fV6z+A-nxos z%$i8|F&W8zYE_YkGzjCZmJWuWAEzsyFL}!wy{?J}+#looBbMkD!&#%#AHc~eQ&>B< z93Q7nW@a_^h?iy$=d9}%HZfx!)Z4KT&9)aX$9jjkk770N6fNN9lU}@%cnKzqSjS`P zT8Y!g*I~^@S+O7IpM|LH;cW1o5ulv#gnPe}ej~lDVU_24!MO*{xc%xr>AE;`Zj#ag zZ`kJX`oFruG>`tg^VSsnal>0Nx8ZZ{^zZ^}e=itcu9<>+5_=2NE}?L8Ae~L?;)gf6I~<97&n@$GnZeh?n1{0uY_X=FF#{xJH!lc=onALC} zY~QdQGaO{jxt3{wvn)D=w581*u)Yt?D<56D;`u8 zqONHqw>BEX9EQbl{aw>=L{K^`U$YI1`?_Leqov$o+6&Cf{{U}{_p-DRMyyF;SD3SG z3BFDUV$-B+V|(v&;))&1G5*mF^w}E9Z8!GevFXi)+t(DV>E^wdqdk-LTs)H3ycEbi zR>!iiIp_KKE+ugGzya93X&D+kxd6`s_T!*}z2dg29J&|O612w!z{U#F*s-Lo@_^?= zUdu_384R?$UUOwM)F6sxathrAp@ZZR4wd*Iom+n*A zuqcKvo+ItIVuN^bhIAf(YehRhFxIcSf}1_^WQ`WKWtCeU61{s%{mL0x)2LnI z{ubWM*#8-?t@8{cZZ2gDw;skWqaLyh*D2_vrGovvOxS9LJs)yAn@ zAID6)M{|e$&6OLY^F_bwi7e9NM;v4}ko&Zd#?7&Fx&4@>%;`!NX2$#Q4s}z}OZOdj z-FAgpcRJ72={>o%+j6*;v6lrLUBk@d=i;sd#lq17E$+Oz2wHhxfviEBVKL9;Y42O} zHnXm~Z!~~9 zBh76zt+TT+yN?b`d-O9F^F%G|Q z5squN;!nmou7Pt-b37V zP`ZafqwB1rxSjVoy^Y(9dW?$8X7Kt@5!V^i2=|t3hbx^-;i-E8&j?9_pC+a9rWfzR zm7x19pg-{38!5bcRzu$N(jD$PH&J=7#Dja3oZ^AoN|;*V0mZ}gad$<5Yrka7*9~w$ zN*+rYUBnw49WGoS_>Mc8tba4=*-nmbtd9>9T8VnA=P-IM);wXC{Jc?|#s9 zi5>ce*A%-R4PzBsj^jta&$+x0`sLyB=HvlSiZ2BepMh36QR{^kUNI_jT~h zV?FEZVF$fJ99c7EF|U=`AG6*yg5);+;m%NJOgO5;O*7ZSWQ!ztJ$omsGiDvHSMxHr zxo-}~{PdXbt{vQB%ynVwp_;r|K{=Q{TV$Tax8Y2H8@!G_%&R0fhn$Of5QL|A#?={` z!7kUJ@?xwY)@ufml zob!aknPc^EPrp#+U*Rg|NA2MLucZ6$*&G-8Nn^|I8e3-AWgwKR^px3;9fjl0yawmj z4_TjealEUOwsg<_$E?N@JLu6!4G(K)Df8ZS!hWwcxM%dwtlC;LWm}Z>)dgB&5O8cn*rFS25OR2TCA>R7G{st1G}PM zxxPt?hWjM4>EMRvPPJn;J+_G(i&L4;_$j0U@G&cGyCB={z8 zgeQL9+-j;n^l~VKmLW$lVZdhi)!K>I8Fg&)t$jvpig|xNzC#wP+U7Z!S{&jb7dztR zz+@J;=m=}vtq!YJ?*VMSVT-#PHDI;;^5LFjmeTGZ_g>c(H-v>UGwI&g_bXYlzVDi_ z&Q`NwgF{VLZl^9E7d`-U)lFGa$8unkvI19}x&vaHg+hScX5@Rv!@)j<;?YIpVQRI* z$Rbk3nf)%sM=QoC%oGZRmK@|66Zxe=*^5c6{I99>Zs}iHd|heQQVQ~VH53YMkJ5P6 zOH(L~WFftw_&j<`4~0Sw+EVyDt8Zi=XYvnbFiTFZtVnXg<^D~n1IkHn&>snq011!) z36KB@kN^pg011!)36KB@kN^pg011!)36KB@kN^pg011!)36KB@kN^pg011!)36KB@ zkN^pg011!)36KB@kN^pg011!)36KB@kN^pg011!)36KB@kN^pg011!)36KB@kN^pg z011!)36KB@kN^pg011!)36KB@kN^pg011!)36KB@kN^pg011!)36KB@kN^pg011!) z36KB@kN^pg011!)36KB@kN^pg011!)36KB@kN^pg011!)36KB@kN^pg011!)36KB@ zkN^pg011!)36KB@kN^pg011!)36KB@kN^pg011!)36KB@kN^pg011!)36KB@kN^pg z011!)36KB@kN^pg011!)36KB@kN^pg011!)36KB@kN^pg011!)36KB@kN^pg011!) z36KB@kN^pg011!)36KB@kN^pg011!)36KB@kN^pg011!)36KB@kN^pg011!)36KB@ zkN^pg011!)36KB@kN^pg011!)36KB@kN^pg011!)36KB@kN^pg011!)36KB@kN^pg z011!)36KB@kN^pg011!)36KB@kN^pg011!)36KB@kN^pg011!)36KB@kN^pg011!) z36KB@kN^pg011!)36KB@kN^pg011!)36KB@kN^pg011!)36KB@kih?eKzy`fjKWOH zNJ|d#jEVeGAy$4h{i6VSWwG@Bh*|EBsXVVIM-BKQ$}DMdjXy%^DxVUL`XXgT*=pKS zbW16eD}BFu=?bN9T1X4Oj8>ZYrxasj6!n86hdP8u#Hoiks3KHD663;yV;u&BhlLDr zh=_3TQirPIhs4>u&dyvOq)H5zvKs!iVn__y9^Wp)x1>H5RE}l&4<+<>O7tJJoljXK z$ZhqFw)>*#bjsG!k=Bx{E4NDxxm`XbD8KbF!OweFjKb=hP1ao%F>v(u(0yV13I}T| zw0srX>S1!qdc)<{#!{;KLoD6)NmDc1C+RAMytKUhQbB$(`J$ko+C*21Zz%=2pw;E@ z?e>wPS4fikgW~t-pH)E4rA}0w!oazCbJvhy$KY?|DChdmJHy@>lUSo3s(*C zQw{GN895|8BCPJbrFzP0+d*4l7^pA|R)-JiH#9Ot-Cm&`;v>HVNH2rs{{~C{=|-vK zrSc@eS7D?c79OIG2v*xPldD_KSD_ys5fUD(F0HG3SxYFQ;$f&%M=g(-Cl3E_EukOW z!p+S&B=kRM2xFwF4eAe{RoI>7NeIDdxr8WOn&->U9{$BQ_GPlhB?}tG8asz%m^8Ljj z(0BGt@{UK&{4+$kslIjyl56#y=J{I1r8Perddac=sS(Y8TcZE4DL!>xf2Xo+aacN>CcnlqL9C3zQ}_LH4q@?XXgwj#^st+F#r;$=UtA{e)cB-^V4vZUb64 zt6G#*P%hG!+poNN$vMhXX?H5E`PEUk9A&E%OysCi_^jg6+!u#Jxp$WS$q9b%aQOGL zFD-W?IYTYW1(Qmr{_5iY!u{_le00|jb!q9=EY$5Q>@rTNr5yie*$!D=yj2QcvTl_l zxqK0+bltKg@&;KNNgfwke{)>u8y=xn#pu4@_V8rXFa5L?M(q`b;So{saoxg4NP|J6 z$oRNFW&#xk3E^=AJ;PP8z6wKCNQh^AXs9|yE{nUbLU(|){Ez=_r5tTaKXloYedw|& z{m^9-5n=Otjy5)bjN5$`24OL(5INaDRyUI81JvPR1LHm}36BVm^HawSj0};ND18;> z5{ z{_X)%U(Qy$EZdJ&F17x&@>i_O4qM+!UzO)90emq)}3&vB6{(nPEJ>6tD5V#oOF z%~;k^Tv|A))oSOjee#e?EN{2}T@Jsu&Kqfch4n$Blk!ym=z#72cePVX5Aq+*+S{(YH2(udV&5 zC;w^fk22k=N*R5*8OT*G-2y+YEQ9@Nhai`&Y%};)DxWIwZ&Hzt0rDRC>tBQUMuv5Z z42_e%8#wt}QoX7pZeA+=X2t&7yYaG>%68?ScEmL*cR4OqM`4$mf8j6o$iLoS%C@)W SPHwI)!H!Py0!#U&!v6q%4FF&O literal 0 HcmV?d00001 diff --git a/resources/pre_trained_lenet/lenet-state.9381 b/resources/pre_trained_lenet/lenet-state.9381 new file mode 100644 index 0000000000000000000000000000000000000000..6e95c8db994b61628b3c0e9a8010e9142a0c5d63 GIT binary patch literal 601 zcmaKoziZV{6vt1~zGtyjY*9f_>ktP;@7qM^(4h}B@)Xhj1LxfwlIYF7@#dsWt4I}5 z!BxRQ#G!7*#XmrF^3SMUMOOtEPg47;iw80!`TqEvv-u4=rH9kniD$Y!)mT+k3>CulryYq8{0D=f6*%gtEsUE;Nafhut9J#~R9k%YUb%$Ze$i#4R zP--;T)6N*uoH^UeD{cyUSFxC{3y=}0Q>{$x$HiX`0Hn}0K5_jkpE~7GN6V-Q>w5`AYbmh$LUlRe0^`WzjrlR^E*|jPkl7GDU z;>6VRt!F{gvhZl_%cXaR?=sBfSoggvu9Mef851h+zUJt054{bu9GgC`dSC)&kn68A}1%+R62>9p;bmM=#bW>?VXRS_|H zY_2G{qu-we@bCTmq;YizaN+EQ?XSld&ofMsEWPL|>G!+NaD(As{NKICwb_w(KbU*j QtJg0d3Y~4TMj>zO-yw&;_y7O^ literal 0 HcmV?d00001 From 374704a279651c66e0c05740ccfca7d1b3ba5ae0 Mon Sep 17 00:00:00 2001 From: plrectco Date: Tue, 21 Mar 2017 13:43:30 +0800 Subject: [PATCH 007/823] add doctest for criterion --- python/dllib/src/bigdl/dllib/nn/criterion.py | 40 ++++++++++++++++++-- 1 file changed, 36 insertions(+), 4 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/nn/criterion.py b/python/dllib/src/bigdl/dllib/nn/criterion.py index 7e47f7074ba..570653413ab 100644 --- a/python/dllib/src/bigdl/dllib/nn/criterion.py +++ b/python/dllib/src/bigdl/dllib/nn/criterion.py @@ -29,6 +29,7 @@ class Criterion(JavaValue): + def __init__(self, jvalue, bigdl_type, *args): self.value = jvalue if jvalue else callBigDlFunc( bigdl_type, JavaValue.jvm_class_constructor(self), *args) @@ -43,21 +44,28 @@ def of(cls, jcriterion, bigdl_type="float"): class ClassNLLCriterion(Criterion): + ''' >>> classNLLCriterion = ClassNLLCriterion() creating: createClassNLLCriterion ''' - def __init__(self, bigdl_type="float"): - JavaValue.__init__(self, None, bigdl_type) + + def __init__(self, + size_average=True, + bigdl_type="float"): + super(ClassNLLCriterion, self).__init__(None, bigdl_type, + size_average) class MSECriterion(Criterion): + ''' >>> mSECriterion = MSECriterion() creating: createMSECriterion ''' + def __init__(self, bigdl_type="float"): - JavaValue.__init__(self, None, bigdl_type) + super(MSECriterion, self).__init__(None, bigdl_type) class AbsCriterion(Criterion): @@ -266,6 +274,7 @@ def __init__(self, class TimeDistributedCriterion(JavaValue): + ''' >>> td = TimeDistributedCriterion(ClassNLLCriterion()) creating: createClassNLLCriterion @@ -273,4 +282,27 @@ class TimeDistributedCriterion(JavaValue): ''' def __init__(self, criterion, size_average=False, bigdl_type="float"): - super(TimeDistributedCriterion, self).__init__(None, bigdl_type, criterion, size_average) + super(TimeDistributedCriterion, self).__init__( + None, bigdl_type, criterion, size_average) + + +def _test(): + import doctest + from pyspark import SparkContext + from nn import criterion + from util.common import init_engine + from util.common import create_spark_conf + globs = criterion.__dict__.copy() + sc = SparkContext(master="local[4]", appName="test criterion", + conf=create_spark_conf()) + globs['sc'] = sc + init_engine() + + (failure_count, test_count) = doctest.testmod(globs=globs, + optionflags=doctest.ELLIPSIS) + if failure_count: + exit(-1) + + +if __name__ == "__main__": + _test() From b197dd80c33b96815cc12eb288e9ddfe16018257 Mon Sep 17 00:00:00 2001 From: plrectco Date: Tue, 21 Mar 2017 13:43:30 +0800 Subject: [PATCH 008/823] add doctest for criterion --- python/dllib/src/bigdl/dllib/nn/criterion.py | 40 ++++++++++++++++++-- 1 file changed, 36 insertions(+), 4 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/nn/criterion.py b/python/dllib/src/bigdl/dllib/nn/criterion.py index 7e47f7074ba..570653413ab 100644 --- a/python/dllib/src/bigdl/dllib/nn/criterion.py +++ b/python/dllib/src/bigdl/dllib/nn/criterion.py @@ -29,6 +29,7 @@ class Criterion(JavaValue): + def __init__(self, jvalue, bigdl_type, *args): self.value = jvalue if jvalue else callBigDlFunc( bigdl_type, JavaValue.jvm_class_constructor(self), *args) @@ -43,21 +44,28 @@ def of(cls, jcriterion, bigdl_type="float"): class ClassNLLCriterion(Criterion): + ''' >>> classNLLCriterion = ClassNLLCriterion() creating: createClassNLLCriterion ''' - def __init__(self, bigdl_type="float"): - JavaValue.__init__(self, None, bigdl_type) + + def __init__(self, + size_average=True, + bigdl_type="float"): + super(ClassNLLCriterion, self).__init__(None, bigdl_type, + size_average) class MSECriterion(Criterion): + ''' >>> mSECriterion = MSECriterion() creating: createMSECriterion ''' + def __init__(self, bigdl_type="float"): - JavaValue.__init__(self, None, bigdl_type) + super(MSECriterion, self).__init__(None, bigdl_type) class AbsCriterion(Criterion): @@ -266,6 +274,7 @@ def __init__(self, class TimeDistributedCriterion(JavaValue): + ''' >>> td = TimeDistributedCriterion(ClassNLLCriterion()) creating: createClassNLLCriterion @@ -273,4 +282,27 @@ class TimeDistributedCriterion(JavaValue): ''' def __init__(self, criterion, size_average=False, bigdl_type="float"): - super(TimeDistributedCriterion, self).__init__(None, bigdl_type, criterion, size_average) + super(TimeDistributedCriterion, self).__init__( + None, bigdl_type, criterion, size_average) + + +def _test(): + import doctest + from pyspark import SparkContext + from nn import criterion + from util.common import init_engine + from util.common import create_spark_conf + globs = criterion.__dict__.copy() + sc = SparkContext(master="local[4]", appName="test criterion", + conf=create_spark_conf()) + globs['sc'] = sc + init_engine() + + (failure_count, test_count) = doctest.testmod(globs=globs, + optionflags=doctest.ELLIPSIS) + if failure_count: + exit(-1) + + +if __name__ == "__main__": + _test() From 0199262f92a9f905f08c1b362d7d220200abe379 Mon Sep 17 00:00:00 2001 From: zhangli Date: Wed, 22 Mar 2017 10:20:26 +0800 Subject: [PATCH 009/823] change wiki link after code restructuring --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index a94e306c8f4..ed72d7eab3d 100644 --- a/README.md +++ b/README.md @@ -55,13 +55,13 @@ RDD[..] --transform-->RDD[ndarray, ndarray].map(Sample.from_ndarray(features, la ## Run python test * Package Scala code by: ```$BigDL_HOME/make-dist.sh``` -* Set SPARK_HOME and then run: ```$BigDL_HOME/dl/src/main/python/dev/run-all.sh``` +* Set SPARK_HOME and then run: ```$BigDL_HOME/pyspark/dev/run-all.sh``` ## Installing on Ubuntu 1. Build BigDL [Build Page](https://github.com/intel-analytics/BigDL/wiki/Build-Page) - * With Spark1.6: ``` $BIGDL_HOME//make-dist.sh ``` - * With Spark2.0: ``` $BIGDL_HOME//make-dist.sh -P spark_2.0 ``` + * With Spark1.6: ``` $BIGDL_HOME/make-dist.sh ``` + * With Spark2.0: ``` $BIGDL_HOME/make-dist.sh -P spark_2.0 ``` 2. Install python dependensies: * Installing Numpy: @@ -90,12 +90,12 @@ RDD[..] --transform-->RDD[ndarray, ndarray].map(Sample.from_ndarray(features, la --executor-cores 10 \ --executor-memory 20g \ --conf spark.akka.frameSize=64 \ - --py-files ${PYTHON_API_ZIP_PATH},${BigDL_HOME}/dl/src/main/python/models/lenet/lenet5.py \ + --py-files ${PYTHON_API_ZIP_PATH},${BigDL_HOME}/pyspark/bigdl/models/lenet/lenet5.py \ --properties-file ${BigDL_HOME}/dist/conf/spark-bigdl.conf \ --jars ${BigDL_JAR_PATH} \ --conf spark.driver.extraClassPath=${BigDL_JAR_PATH} \ --conf spark.executor.extraClassPath=bigdl-0.1.0-SNAPSHOT-jar-with-dependencies.jar \ - ${BigDL_HOME}/dl/src/main/python/models/lenet/lenet5.py + ${BigDL_HOME}/pyspark/bigdl/models/lenet/lenet5.py ``` From 2f15bebd42813dd932739a3ff31d9059c403808a Mon Sep 17 00:00:00 2001 From: zhangli Date: Wed, 22 Mar 2017 10:20:26 +0800 Subject: [PATCH 010/823] change wiki link after code restructuring --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index a94e306c8f4..ed72d7eab3d 100644 --- a/README.md +++ b/README.md @@ -55,13 +55,13 @@ RDD[..] --transform-->RDD[ndarray, ndarray].map(Sample.from_ndarray(features, la ## Run python test * Package Scala code by: ```$BigDL_HOME/make-dist.sh``` -* Set SPARK_HOME and then run: ```$BigDL_HOME/dl/src/main/python/dev/run-all.sh``` +* Set SPARK_HOME and then run: ```$BigDL_HOME/pyspark/dev/run-all.sh``` ## Installing on Ubuntu 1. Build BigDL [Build Page](https://github.com/intel-analytics/BigDL/wiki/Build-Page) - * With Spark1.6: ``` $BIGDL_HOME//make-dist.sh ``` - * With Spark2.0: ``` $BIGDL_HOME//make-dist.sh -P spark_2.0 ``` + * With Spark1.6: ``` $BIGDL_HOME/make-dist.sh ``` + * With Spark2.0: ``` $BIGDL_HOME/make-dist.sh -P spark_2.0 ``` 2. Install python dependensies: * Installing Numpy: @@ -90,12 +90,12 @@ RDD[..] --transform-->RDD[ndarray, ndarray].map(Sample.from_ndarray(features, la --executor-cores 10 \ --executor-memory 20g \ --conf spark.akka.frameSize=64 \ - --py-files ${PYTHON_API_ZIP_PATH},${BigDL_HOME}/dl/src/main/python/models/lenet/lenet5.py \ + --py-files ${PYTHON_API_ZIP_PATH},${BigDL_HOME}/pyspark/bigdl/models/lenet/lenet5.py \ --properties-file ${BigDL_HOME}/dist/conf/spark-bigdl.conf \ --jars ${BigDL_JAR_PATH} \ --conf spark.driver.extraClassPath=${BigDL_JAR_PATH} \ --conf spark.executor.extraClassPath=bigdl-0.1.0-SNAPSHOT-jar-with-dependencies.jar \ - ${BigDL_HOME}/dl/src/main/python/models/lenet/lenet5.py + ${BigDL_HOME}/pyspark/bigdl/models/lenet/lenet5.py ``` From b58c78d15e07a22f92c34df9e7e88b4f3faea64f Mon Sep 17 00:00:00 2001 From: zhichao-li Date: Wed, 22 Mar 2017 12:59:29 +0800 Subject: [PATCH 011/823] add dev back --- python/dllib/test/dev/__init__.py | 15 + python/dllib/test/dev/lint-python | 72 + python/dllib/test/dev/modules.py | 85 ++ python/dllib/test/dev/pep8-1.7.0.py | 2151 +++++++++++++++++++++++++++ python/dllib/test/dev/run-all.sh | 20 + python/dllib/test/dev/run-tests | 37 + python/dllib/test/dev/run-tests.py | 233 +++ 7 files changed, 2613 insertions(+) create mode 100644 python/dllib/test/dev/__init__.py create mode 100755 python/dllib/test/dev/lint-python create mode 100644 python/dllib/test/dev/modules.py create mode 100644 python/dllib/test/dev/pep8-1.7.0.py create mode 100755 python/dllib/test/dev/run-all.sh create mode 100755 python/dllib/test/dev/run-tests create mode 100755 python/dllib/test/dev/run-tests.py diff --git a/python/dllib/test/dev/__init__.py b/python/dllib/test/dev/__init__.py new file mode 100644 index 00000000000..2151a805423 --- /dev/null +++ b/python/dllib/test/dev/__init__.py @@ -0,0 +1,15 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/python/dllib/test/dev/lint-python b/python/dllib/test/dev/lint-python new file mode 100755 index 00000000000..7d0c81dde2e --- /dev/null +++ b/python/dllib/test/dev/lint-python @@ -0,0 +1,72 @@ +#!/usr/bin/env bash + +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Originally from Spark +SCRIPT_DIR="$( cd "$( dirname "$0" )" && pwd )" +PYTHON_ROOT_DIR="$SCRIPT_DIR/.." +echo $PYTHON_ROOT_DIR +PATHS_TO_CHECK="." +PEP8_REPORT_PATH="$PYTHON_ROOT_DIR/dev/pep8-report.txt" +PYLINT_REPORT_PATH="$PYTHON_ROOT_DIR/dev/pylint-report.txt" +PYLINT_INSTALL_INFO="$PYTHON_ROOT_DIR/dev/pylint-info.txt" +SPHINXBUILD=${SPHINXBUILD:=sphinx-build} +SPHINX_REPORT_PATH="$PYTHON_ROOT_DIR/dev/sphinx-report.txt" + +cd "$PYTHON_ROOT_DIR" + +# compileall: https://docs.python.org/2/library/compileall.html +python -B -m compileall -q -l $PATHS_TO_CHECK > "$PEP8_REPORT_PATH" +compile_status="${PIPESTATUS[0]}" + +PEP8_VERSION="1.7.0" +PEP8_SCRIPT_PATH="$PYTHON_ROOT_DIR/dev/pep8-$PEP8_VERSION.py" +PEP8_SCRIPT_REMOTE_PATH="https://raw.githubusercontent.com/jcrocholl/pep8/$PEP8_VERSION/pep8.py" + +echo "PEP8_SCRIPT_PATH" "$PEP8_SCRIPT_PATH" + +# Easy install pylint in /dev/pylint. To easy_install into a directory, the PYTHONPATH should +# be set to the directory. +# dev/pylint should be appended to the PATH variable as well. +# Jenkins by default installs the pylint3 version, so for now this just checks the code quality +# of python3. +export "PYTHONPATH=$PYTHON_ROOT_DIR/dev/pylint" +export "PYLINT_HOME=$PYTHONPATH" +export "PATH=$PYTHONPATH:$PATH" + +# There is no need to write this output to a file +#+ first, but we do so so that the check status can +#+ be output before the report, like with the +#+ scalastyle and RAT checks. +python "$PEP8_SCRIPT_PATH" --ignore=E402,E731,E241,W503,E226 --config=dev/tox.ini $PATHS_TO_CHECK >> "$PEP8_REPORT_PATH" +pep8_status="${PIPESTATUS[0]}" + +if [ "$compile_status" -eq 0 -a "$pep8_status" -eq 0 ]; then + lint_status=0 +else + lint_status=1 +fi + +if [ "$lint_status" -ne 0 ]; then + echo "PEP8 checks failed." + cat "$PEP8_REPORT_PATH" + rm "$PEP8_REPORT_PATH" + exit "$lint_status" +else + echo "PEP8 checks passed." + rm "$PEP8_REPORT_PATH" +fi diff --git a/python/dllib/test/dev/modules.py b/python/dllib/test/dev/modules.py new file mode 100644 index 00000000000..9d9b1dbab97 --- /dev/null +++ b/python/dllib/test/dev/modules.py @@ -0,0 +1,85 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Adopt from Spark and it might be refactored in the future + +from functools import total_ordering + +all_modules = [] + + +@total_ordering +class Module(object): + + def __init__(self, name, python_test_goals=()): + """ + Define a new module. + + :param name: A short module name, for display in logging and error messagesl + :param python_test_goals: A set of Python test goals for testing this module. + """ + self.name = name + self.python_test_goals = python_test_goals + + all_modules.append(self) + + def __repr__(self): + return "Module<%s>" % self.name + + def __lt__(self, other): + return self.name < other.name + + def __eq__(self, other): + return self.name == other.name + + def __ne__(self, other): + return not (self.name == other.name) + + def __hash__(self): + return hash(self.name) + + +bigdl_layer = Module( + name="bigdl_layer", + python_test_goals=[ + "nn.layer" + ]) + +bigdl_layer = Module( + name="bigdl_criterion", + python_test_goals=[ + "nn.criterion" + ]) + +bigdl_layer = Module( + name="bigdl_common", + python_test_goals=[ + "util.common" + ]) + +bigdl_optimizer = Module( + name="bigdl_optimizer", + python_test_goals=[ + "optim.optimizer", + ] +) + +test_simple_integration_test = Module( + name="simple_integration_test", + python_test_goals=[ + "test.simple_integration_test" + ] +) diff --git a/python/dllib/test/dev/pep8-1.7.0.py b/python/dllib/test/dev/pep8-1.7.0.py new file mode 100644 index 00000000000..4d9ad36ef4a --- /dev/null +++ b/python/dllib/test/dev/pep8-1.7.0.py @@ -0,0 +1,2151 @@ +#!/usr/bin/env python +# pep8.py - Check Python source code formatting, according to PEP 8 +# Copyright (C) 2006-2009 Johann C. Rocholl +# Copyright (C) 2009-2014 Florent Xicluna +# Copyright (C) 2014-2016 Ian Lee +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation files +# (the "Software"), to deal in the Software without restriction, +# including without limitation the rights to use, copy, modify, merge, +# publish, distribute, sublicense, and/or sell copies of the Software, +# and to permit persons to whom the Software is furnished to do so, +# subject to the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS +# BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN +# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +r""" +Check Python source code formatting, according to PEP 8. + +For usage and a list of options, try this: +$ python pep8.py -h + +This program and its regression test suite live here: +https://github.com/pycqa/pep8 + +Groups of errors and warnings: +E errors +W warnings +100 indentation +200 whitespace +300 blank lines +400 imports +500 line length +600 deprecation +700 statements +900 syntax error +""" +from __future__ import with_statement + +import os +import sys +import re +import time +import inspect +import keyword +import tokenize +from optparse import OptionParser +from fnmatch import fnmatch +try: + from configparser import RawConfigParser + from io import TextIOWrapper +except ImportError: + from ConfigParser import RawConfigParser + +__version__ = '1.7.0' + +DEFAULT_EXCLUDE = '.svn,CVS,.bzr,.hg,.git,__pycache__,.tox' +DEFAULT_IGNORE = 'E121,E123,E126,E226,E24,E704' +try: + if sys.platform == 'win32': + USER_CONFIG = os.path.expanduser(r'~\.pep8') + else: + USER_CONFIG = os.path.join( + os.getenv('XDG_CONFIG_HOME') or os.path.expanduser('~/.config'), + 'pep8' + ) +except ImportError: + USER_CONFIG = None + +PROJECT_CONFIG = ('setup.cfg', 'tox.ini', '.pep8') +TESTSUITE_PATH = os.path.join(os.path.dirname(__file__), 'testsuite') +MAX_LINE_LENGTH = 100 +REPORT_FORMAT = { + 'default': '%(path)s:%(row)d:%(col)d: %(code)s %(text)s', + 'pylint': '%(path)s:%(row)d: [%(code)s] %(text)s', +} + +PyCF_ONLY_AST = 1024 +SINGLETONS = frozenset(['False', 'None', 'True']) +KEYWORDS = frozenset(keyword.kwlist + ['print']) - SINGLETONS +UNARY_OPERATORS = frozenset(['>>', '**', '*', '+', '-']) +ARITHMETIC_OP = frozenset(['**', '*', '/', '//', '+', '-']) +WS_OPTIONAL_OPERATORS = ARITHMETIC_OP.union(['^', '&', '|', '<<', '>>', '%']) +WS_NEEDED_OPERATORS = frozenset([ + '**=', '*=', '/=', '//=', '+=', '-=', '!=', '<>', '<', '>', + '%=', '^=', '&=', '|=', '==', '<=', '>=', '<<=', '>>=', '=']) +WHITESPACE = frozenset(' \t') +NEWLINE = frozenset([tokenize.NL, tokenize.NEWLINE]) +SKIP_TOKENS = NEWLINE.union([tokenize.INDENT, tokenize.DEDENT]) +# ERRORTOKEN is triggered by backticks in Python 3 +SKIP_COMMENTS = SKIP_TOKENS.union([tokenize.COMMENT, tokenize.ERRORTOKEN]) +BENCHMARK_KEYS = ['directories', 'files', 'logical lines', 'physical lines'] + +INDENT_REGEX = re.compile(r'([ \t]*)') +RAISE_COMMA_REGEX = re.compile(r'raise\s+\w+\s*,') +RERAISE_COMMA_REGEX = re.compile(r'raise\s+\w+\s*,.*,\s*\w+\s*$') +ERRORCODE_REGEX = re.compile(r'\b[A-Z]\d{3}\b') +DOCSTRING_REGEX = re.compile(r'u?r?["\']') +EXTRANEOUS_WHITESPACE_REGEX = re.compile(r'[[({] | []}),;:]') +WHITESPACE_AFTER_COMMA_REGEX = re.compile(r'[,;:]\s*(?: |\t)') +COMPARE_SINGLETON_REGEX = re.compile(r'(\bNone|\bFalse|\bTrue)?\s*([=!]=)' + r'\s*(?(1)|(None|False|True))\b') +COMPARE_NEGATIVE_REGEX = re.compile(r'\b(not)\s+[^][)(}{ ]+\s+(in|is)\s') +COMPARE_TYPE_REGEX = re.compile(r'(?:[=!]=|is(?:\s+not)?)\s*type(?:s.\w+Type' + r'|\s*\(\s*([^)]*[^ )])\s*\))') +KEYWORD_REGEX = re.compile(r'(\s*)\b(?:%s)\b(\s*)' % r'|'.join(KEYWORDS)) +OPERATOR_REGEX = re.compile(r'(?:[^,\s])(\s*)(?:[-+*/|!<=>%&^]+)(\s*)') +LAMBDA_REGEX = re.compile(r'\blambda\b') +HUNK_REGEX = re.compile(r'^@@ -\d+(?:,\d+)? \+(\d+)(?:,(\d+))? @@.*$') + +# Work around Python < 2.6 behaviour, which does not generate NL after +# a comment which is on a line by itself. +COMMENT_WITH_NL = tokenize.generate_tokens(['#\n'].pop).send(None)[1] == '#\n' + + +############################################################################## +# Plugins (check functions) for physical lines +############################################################################## + + +def tabs_or_spaces(physical_line, indent_char): + r"""Never mix tabs and spaces. + + The most popular way of indenting Python is with spaces only. The + second-most popular way is with tabs only. Code indented with a mixture + of tabs and spaces should be converted to using spaces exclusively. When + invoking the Python command line interpreter with the -t option, it issues + warnings about code that illegally mixes tabs and spaces. When using -tt + these warnings become errors. These options are highly recommended! + + Okay: if a == 0:\n a = 1\n b = 1 + E101: if a == 0:\n a = 1\n\tb = 1 + """ + indent = INDENT_REGEX.match(physical_line).group(1) + for offset, char in enumerate(indent): + if char != indent_char: + return offset, "E101 indentation contains mixed spaces and tabs" + + +def tabs_obsolete(physical_line): + r"""For new projects, spaces-only are strongly recommended over tabs. + + Okay: if True:\n return + W191: if True:\n\treturn + """ + indent = INDENT_REGEX.match(physical_line).group(1) + if '\t' in indent: + return indent.index('\t'), "W191 indentation contains tabs" + + +def trailing_whitespace(physical_line): + r"""Trailing whitespace is superfluous. + + The warning returned varies on whether the line itself is blank, for easier + filtering for those who want to indent their blank lines. + + Okay: spam(1)\n# + W291: spam(1) \n# + W293: class Foo(object):\n \n bang = 12 + """ + physical_line = physical_line.rstrip('\n') # chr(10), newline + physical_line = physical_line.rstrip('\r') # chr(13), carriage return + physical_line = physical_line.rstrip('\x0c') # chr(12), form feed, ^L + stripped = physical_line.rstrip(' \t\v') + if physical_line != stripped: + if stripped: + return len(stripped), "W291 trailing whitespace" + else: + return 0, "W293 blank line contains whitespace" + + +def trailing_blank_lines(physical_line, lines, line_number, total_lines): + r"""Trailing blank lines are superfluous. + + Okay: spam(1) + W391: spam(1)\n + + However the last line should end with a new line (warning W292). + """ + if line_number == total_lines: + stripped_last_line = physical_line.rstrip() + if not stripped_last_line: + return 0, "W391 blank line at end of file" + if stripped_last_line == physical_line: + return len(physical_line), "W292 no newline at end of file" + + +def maximum_line_length(physical_line, max_line_length, multiline): + r"""Limit all lines to a maximum of 79 characters. + + There are still many devices around that are limited to 80 character + lines; plus, limiting windows to 80 characters makes it possible to have + several windows side-by-side. The default wrapping on such devices looks + ugly. Therefore, please limit all lines to a maximum of 79 characters. + For flowing long blocks of text (docstrings or comments), limiting the + length to 72 characters is recommended. + + Reports error E501. + """ + line = physical_line.rstrip() + length = len(line) + if length > max_line_length and not noqa(line): + # Special case for long URLs in multi-line docstrings or comments, + # but still report the error when the 72 first chars are whitespaces. + chunks = line.split() + if ((len(chunks) == 1 and multiline) or + (len(chunks) == 2 and chunks[0] == '#')) and \ + len(line) - len(chunks[-1]) < max_line_length - 7: + return + if hasattr(line, 'decode'): # Python 2 + # The line could contain multi-byte characters + try: + length = len(line.decode('utf-8')) + except UnicodeError: + pass + if length > max_line_length: + return (max_line_length, "E501 line too long " + "(%d > %d characters)" % (length, max_line_length)) + + +############################################################################## +# Plugins (check functions) for logical lines +############################################################################## + + +def blank_lines(logical_line, blank_lines, indent_level, line_number, + blank_before, previous_logical, previous_indent_level): + r"""Separate top-level function and class definitions with two blank lines. + + Method definitions inside a class are separated by a single blank line. + + Extra blank lines may be used (sparingly) to separate groups of related + functions. Blank lines may be omitted between a bunch of related + one-liners (e.g. a set of dummy implementations). + + Use blank lines in functions, sparingly, to indicate logical sections. + + Okay: def a():\n pass\n\n\ndef b():\n pass + Okay: def a():\n pass\n\n\n# Foo\n# Bar\n\ndef b():\n pass + + E301: class Foo:\n b = 0\n def bar():\n pass + E302: def a():\n pass\n\ndef b(n):\n pass + E303: def a():\n pass\n\n\n\ndef b(n):\n pass + E303: def a():\n\n\n\n pass + E304: @decorator\n\ndef a():\n pass + """ + if line_number < 3 and not previous_logical: + return # Don't expect blank lines before the first line + if previous_logical.startswith('@'): + if blank_lines: + yield 0, "E304 blank lines found after function decorator" + elif blank_lines > 2 or (indent_level and blank_lines == 2): + yield 0, "E303 too many blank lines (%d)" % blank_lines + elif logical_line.startswith(('def ', 'class ', '@')): + if indent_level: + if not (blank_before or previous_indent_level < indent_level or + DOCSTRING_REGEX.match(previous_logical)): + yield 0, "E301 expected 1 blank line, found 0" + elif blank_before != 2: + yield 0, "E302 expected 2 blank lines, found %d" % blank_before + + +def extraneous_whitespace(logical_line): + r"""Avoid extraneous whitespace. + + Avoid extraneous whitespace in these situations: + - Immediately inside parentheses, brackets or braces. + - Immediately before a comma, semicolon, or colon. + + Okay: spam(ham[1], {eggs: 2}) + E201: spam( ham[1], {eggs: 2}) + E201: spam(ham[ 1], {eggs: 2}) + E201: spam(ham[1], { eggs: 2}) + E202: spam(ham[1], {eggs: 2} ) + E202: spam(ham[1 ], {eggs: 2}) + E202: spam(ham[1], {eggs: 2 }) + + E203: if x == 4: print x, y; x, y = y , x + E203: if x == 4: print x, y ; x, y = y, x + E203: if x == 4 : print x, y; x, y = y, x + """ + line = logical_line + for match in EXTRANEOUS_WHITESPACE_REGEX.finditer(line): + text = match.group() + char = text.strip() + found = match.start() + if text == char + ' ': + # assert char in '([{' + yield found + 1, "E201 whitespace after '%s'" % char + elif line[found - 1] != ',': + code = ('E202' if char in '}])' else 'E203') # if char in ',;:' + yield found, "%s whitespace before '%s'" % (code, char) + + +def whitespace_around_keywords(logical_line): + r"""Avoid extraneous whitespace around keywords. + + Okay: True and False + E271: True and False + E272: True and False + E273: True and\tFalse + E274: True\tand False + """ + for match in KEYWORD_REGEX.finditer(logical_line): + before, after = match.groups() + + if '\t' in before: + yield match.start(1), "E274 tab before keyword" + elif len(before) > 1: + yield match.start(1), "E272 multiple spaces before keyword" + + if '\t' in after: + yield match.start(2), "E273 tab after keyword" + elif len(after) > 1: + yield match.start(2), "E271 multiple spaces after keyword" + + +def missing_whitespace(logical_line): + r"""Each comma, semicolon or colon should be followed by whitespace. + + Okay: [a, b] + Okay: (3,) + Okay: a[1:4] + Okay: a[:4] + Okay: a[1:] + Okay: a[1:4:2] + E231: ['a','b'] + E231: foo(bar,baz) + E231: [{'a':'b'}] + """ + line = logical_line + for index in range(len(line) - 1): + char = line[index] + if char in ',;:' and line[index + 1] not in WHITESPACE: + before = line[:index] + if char == ':' and before.count('[') > before.count(']') and \ + before.rfind('{') < before.rfind('['): + continue # Slice syntax, no space required + if char == ',' and line[index + 1] == ')': + continue # Allow tuple with only one element: (3,) + yield index, "E231 missing whitespace after '%s'" % char + + +def indentation(logical_line, previous_logical, indent_char, + indent_level, previous_indent_level): + r"""Use 4 spaces per indentation level. + + For really old code that you don't want to mess up, you can continue to + use 8-space tabs. + + Okay: a = 1 + Okay: if a == 0:\n a = 1 + E111: a = 1 + E114: # a = 1 + + Okay: for item in items:\n pass + E112: for item in items:\npass + E115: for item in items:\n# Hi\n pass + + Okay: a = 1\nb = 2 + E113: a = 1\n b = 2 + E116: a = 1\n # b = 2 + """ + c = 0 if logical_line else 3 + tmpl = "E11%d %s" if logical_line else "E11%d %s (comment)" + if indent_level % 4: + yield 0, tmpl % (1 + c, "indentation is not a multiple of four") + indent_expect = previous_logical.endswith(':') + if indent_expect and indent_level <= previous_indent_level: + yield 0, tmpl % (2 + c, "expected an indented block") + elif not indent_expect and indent_level > previous_indent_level: + yield 0, tmpl % (3 + c, "unexpected indentation") + + +def continued_indentation(logical_line, tokens, indent_level, hang_closing, + indent_char, noqa, verbose): + r"""Continuation lines indentation. + + Continuation lines should align wrapped elements either vertically + using Python's implicit line joining inside parentheses, brackets + and braces, or using a hanging indent. + + When using a hanging indent these considerations should be applied: + - there should be no arguments on the first line, and + - further indentation should be used to clearly distinguish itself as a + continuation line. + + Okay: a = (\n) + E123: a = (\n ) + + Okay: a = (\n 42) + E121: a = (\n 42) + E122: a = (\n42) + E123: a = (\n 42\n ) + E124: a = (24,\n 42\n) + E125: if (\n b):\n pass + E126: a = (\n 42) + E127: a = (24,\n 42) + E128: a = (24,\n 42) + E129: if (a or\n b):\n pass + E131: a = (\n 42\n 24) + """ + first_row = tokens[0][2][0] + nrows = 1 + tokens[-1][2][0] - first_row + if noqa or nrows == 1: + return + + # indent_next tells us whether the next block is indented; assuming + # that it is indented by 4 spaces, then we should not allow 4-space + # indents on the final continuation line; in turn, some other + # indents are allowed to have an extra 4 spaces. + indent_next = logical_line.endswith(':') + + row = depth = 0 + valid_hangs = (4,) if indent_char != '\t' else (4, 8) + # remember how many brackets were opened on each line + parens = [0] * nrows + # relative indents of physical lines + rel_indent = [0] * nrows + # for each depth, collect a list of opening rows + open_rows = [[0]] + # for each depth, memorize the hanging indentation + hangs = [None] + # visual indents + indent_chances = {} + last_indent = tokens[0][2] + visual_indent = None + last_token_multiline = False + # for each depth, memorize the visual indent column + indent = [last_indent[1]] + if verbose >= 3: + print(">>> " + tokens[0][4].rstrip()) + + for token_type, text, start, end, line in tokens: + + newline = row < start[0] - first_row + if newline: + row = start[0] - first_row + newline = not last_token_multiline and token_type not in NEWLINE + + if newline: + # this is the beginning of a continuation line. + last_indent = start + if verbose >= 3: + print("... " + line.rstrip()) + + # record the initial indent. + rel_indent[row] = expand_indent(line) - indent_level + + # identify closing bracket + close_bracket = (token_type == tokenize.OP and text in ']})') + + # is the indent relative to an opening bracket line? + for open_row in reversed(open_rows[depth]): + hang = rel_indent[row] - rel_indent[open_row] + hanging_indent = hang in valid_hangs + if hanging_indent: + break + if hangs[depth]: + hanging_indent = (hang == hangs[depth]) + # is there any chance of visual indent? + visual_indent = (not close_bracket and hang > 0 and + indent_chances.get(start[1])) + + if close_bracket and indent[depth]: + # closing bracket for visual indent + if start[1] != indent[depth]: + yield (start, "E124 closing bracket does not match " + "visual indentation") + elif close_bracket and not hang: + # closing bracket matches indentation of opening bracket's line + if hang_closing: + yield start, "E133 closing bracket is missing indentation" + elif indent[depth] and start[1] < indent[depth]: + if visual_indent is not True: + # visual indent is broken + yield (start, "E128 continuation line " + "under-indented for visual indent") + elif hanging_indent or (indent_next and rel_indent[row] == 8): + # hanging indent is verified + if close_bracket and not hang_closing: + yield (start, "E123 closing bracket does not match " + "indentation of opening bracket's line") + hangs[depth] = hang + elif visual_indent is True: + # visual indent is verified + indent[depth] = start[1] + elif visual_indent in (text, str): + # ignore token lined up with matching one from a previous line + pass + else: + # indent is broken + if hang <= 0: + error = "E122", "missing indentation or outdented" + elif indent[depth]: + error = "E127", "over-indented for visual indent" + elif not close_bracket and hangs[depth]: + error = "E131", "unaligned for hanging indent" + else: + hangs[depth] = hang + if hang > 4: + error = "E126", "over-indented for hanging indent" + else: + error = "E121", "under-indented for hanging indent" + yield start, "%s continuation line %s" % error + + # look for visual indenting + if (parens[row] and + token_type not in (tokenize.NL, tokenize.COMMENT) and + not indent[depth]): + indent[depth] = start[1] + indent_chances[start[1]] = True + if verbose >= 4: + print("bracket depth %s indent to %s" % (depth, start[1])) + # deal with implicit string concatenation + elif (token_type in (tokenize.STRING, tokenize.COMMENT) or + text in ('u', 'ur', 'b', 'br')): + indent_chances[start[1]] = str + # special case for the "if" statement because len("if (") == 4 + elif not indent_chances and not row and not depth and text == 'if': + indent_chances[end[1] + 1] = True + elif text == ':' and line[end[1]:].isspace(): + open_rows[depth].append(row) + + # keep track of bracket depth + if token_type == tokenize.OP: + if text in '([{': + depth += 1 + indent.append(0) + hangs.append(None) + if len(open_rows) == depth: + open_rows.append([]) + open_rows[depth].append(row) + parens[row] += 1 + if verbose >= 4: + print("bracket depth %s seen, col %s, visual min = %s" % + (depth, start[1], indent[depth])) + elif text in ')]}' and depth > 0: + # parent indents should not be more than this one + prev_indent = indent.pop() or last_indent[1] + hangs.pop() + for d in range(depth): + if indent[d] > prev_indent: + indent[d] = 0 + for ind in list(indent_chances): + if ind >= prev_indent: + del indent_chances[ind] + del open_rows[depth + 1:] + depth -= 1 + if depth: + indent_chances[indent[depth]] = True + for idx in range(row, -1, -1): + if parens[idx]: + parens[idx] -= 1 + break + assert len(indent) == depth + 1 + if start[1] not in indent_chances: + # allow to line up tokens + indent_chances[start[1]] = text + + last_token_multiline = (start[0] != end[0]) + if last_token_multiline: + rel_indent[end[0] - first_row] = rel_indent[row] + + if indent_next and expand_indent(line) == indent_level + 4: + pos = (start[0], indent[0] + 4) + if visual_indent: + code = "E129 visually indented line" + else: + code = "E125 continuation line" + yield pos, "%s with same indent as next logical line" % code + + +def whitespace_before_parameters(logical_line, tokens): + r"""Avoid extraneous whitespace. + + Avoid extraneous whitespace in the following situations: + - before the open parenthesis that starts the argument list of a + function call. + - before the open parenthesis that starts an indexing or slicing. + + Okay: spam(1) + E211: spam (1) + + Okay: dict['key'] = list[index] + E211: dict ['key'] = list[index] + E211: dict['key'] = list [index] + """ + prev_type, prev_text, __, prev_end, __ = tokens[0] + for index in range(1, len(tokens)): + token_type, text, start, end, __ = tokens[index] + if (token_type == tokenize.OP and + text in '([' and + start != prev_end and + (prev_type == tokenize.NAME or prev_text in '}])') and + # Syntax "class A (B):" is allowed, but avoid it + (index < 2 or tokens[index - 2][1] != 'class') and + # Allow "return (a.foo for a in range(5))" + not keyword.iskeyword(prev_text)): + yield prev_end, "E211 whitespace before '%s'" % text + prev_type = token_type + prev_text = text + prev_end = end + + +def whitespace_around_operator(logical_line): + r"""Avoid extraneous whitespace around an operator. + + Okay: a = 12 + 3 + E221: a = 4 + 5 + E222: a = 4 + 5 + E223: a = 4\t+ 5 + E224: a = 4 +\t5 + """ + for match in OPERATOR_REGEX.finditer(logical_line): + before, after = match.groups() + + if '\t' in before: + yield match.start(1), "E223 tab before operator" + elif len(before) > 1: + yield match.start(1), "E221 multiple spaces before operator" + + if '\t' in after: + yield match.start(2), "E224 tab after operator" + elif len(after) > 1: + yield match.start(2), "E222 multiple spaces after operator" + + +def missing_whitespace_around_operator(logical_line, tokens): + r"""Surround operators with a single space on either side. + + - Always surround these binary operators with a single space on + either side: assignment (=), augmented assignment (+=, -= etc.), + comparisons (==, <, >, !=, <=, >=, in, not in, is, is not), + Booleans (and, or, not). + + - If operators with different priorities are used, consider adding + whitespace around the operators with the lowest priorities. + + Okay: i = i + 1 + Okay: submitted += 1 + Okay: x = x * 2 - 1 + Okay: hypot2 = x * x + y * y + Okay: c = (a + b) * (a - b) + Okay: foo(bar, key='word', *args, **kwargs) + Okay: alpha[:-i] + + E225: i=i+1 + E225: submitted +=1 + E225: x = x /2 - 1 + E225: z = x **y + E226: c = (a+b) * (a-b) + E226: hypot2 = x*x + y*y + E227: c = a|b + E228: msg = fmt%(errno, errmsg) + """ + parens = 0 + need_space = False + prev_type = tokenize.OP + prev_text = prev_end = None + for token_type, text, start, end, line in tokens: + if token_type in SKIP_COMMENTS: + continue + if text in ('(', 'lambda'): + parens += 1 + elif text == ')': + parens -= 1 + if need_space: + if start != prev_end: + # Found a (probably) needed space + if need_space is not True and not need_space[1]: + yield (need_space[0], + "E225 missing whitespace around operator") + need_space = False + elif text == '>' and prev_text in ('<', '-'): + # Tolerate the "<>" operator, even if running Python 3 + # Deal with Python 3's annotated return value "->" + pass + else: + if need_space is True or need_space[1]: + # A needed trailing space was not found + yield prev_end, "E225 missing whitespace around operator" + elif prev_text != '**': + code, optype = 'E226', 'arithmetic' + if prev_text == '%': + code, optype = 'E228', 'modulo' + elif prev_text not in ARITHMETIC_OP: + code, optype = 'E227', 'bitwise or shift' + yield (need_space[0], "%s missing whitespace " + "around %s operator" % (code, optype)) + need_space = False + elif token_type == tokenize.OP and prev_end is not None: + if text == '=' and parens: + # Allow keyword args or defaults: foo(bar=None). + pass + elif text in WS_NEEDED_OPERATORS: + need_space = True + elif text in UNARY_OPERATORS: + # Check if the operator is being used as a binary operator + # Allow unary operators: -123, -x, +1. + # Allow argument unpacking: foo(*args, **kwargs). + if (prev_text in '}])' if prev_type == tokenize.OP + else prev_text not in KEYWORDS): + need_space = None + elif text in WS_OPTIONAL_OPERATORS: + need_space = None + + if need_space is None: + # Surrounding space is optional, but ensure that + # trailing space matches opening space + need_space = (prev_end, start != prev_end) + elif need_space and start == prev_end: + # A needed opening space was not found + yield prev_end, "E225 missing whitespace around operator" + need_space = False + prev_type = token_type + prev_text = text + prev_end = end + + +def whitespace_around_comma(logical_line): + r"""Avoid extraneous whitespace after a comma or a colon. + + Note: these checks are disabled by default + + Okay: a = (1, 2) + E241: a = (1, 2) + E242: a = (1,\t2) + """ + line = logical_line + for m in WHITESPACE_AFTER_COMMA_REGEX.finditer(line): + found = m.start() + 1 + if '\t' in m.group(): + yield found, "E242 tab after '%s'" % m.group()[0] + else: + yield found, "E241 multiple spaces after '%s'" % m.group()[0] + + +def whitespace_around_named_parameter_equals(logical_line, tokens): + r"""Don't use spaces around the '=' sign in function arguments. + + Don't use spaces around the '=' sign when used to indicate a + keyword argument or a default parameter value. + + Okay: def complex(real, imag=0.0): + Okay: return magic(r=real, i=imag) + Okay: boolean(a == b) + Okay: boolean(a != b) + Okay: boolean(a <= b) + Okay: boolean(a >= b) + Okay: def foo(arg: int = 42): + + E251: def complex(real, imag = 0.0): + E251: return magic(r = real, i = imag) + """ + parens = 0 + no_space = False + prev_end = None + annotated_func_arg = False + in_def = logical_line.startswith('def') + message = "E251 unexpected spaces around keyword / parameter equals" + for token_type, text, start, end, line in tokens: + if token_type == tokenize.NL: + continue + if no_space: + no_space = False + if start != prev_end: + yield (prev_end, message) + if token_type == tokenize.OP: + if text == '(': + parens += 1 + elif text == ')': + parens -= 1 + elif in_def and text == ':' and parens == 1: + annotated_func_arg = True + elif parens and text == ',' and parens == 1: + annotated_func_arg = False + elif parens and text == '=' and not annotated_func_arg: + no_space = True + if start != prev_end: + yield (prev_end, message) + if not parens: + annotated_func_arg = False + + prev_end = end + + +def whitespace_before_comment(logical_line, tokens): + r"""Separate inline comments by at least two spaces. + + An inline comment is a comment on the same line as a statement. Inline + comments should be separated by at least two spaces from the statement. + They should start with a # and a single space. + + Each line of a block comment starts with a # and a single space + (unless it is indented text inside the comment). + + Okay: x = x + 1 # Increment x + Okay: x = x + 1 # Increment x + Okay: # Block comment + E261: x = x + 1 # Increment x + E262: x = x + 1 #Increment x + E262: x = x + 1 # Increment x + E265: #Block comment + E266: ### Block comment + """ + prev_end = (0, 0) + for token_type, text, start, end, line in tokens: + if token_type == tokenize.COMMENT: + inline_comment = line[:start[1]].strip() + if inline_comment: + if prev_end[0] == start[0] and start[1] < prev_end[1] + 2: + yield (prev_end, + "E261 at least two spaces before inline comment") + symbol, sp, comment = text.partition(' ') + bad_prefix = symbol not in '#:' and (symbol.lstrip('#')[:1] or '#') + if inline_comment: + if bad_prefix or comment[:1] in WHITESPACE: + yield start, "E262 inline comment should start with '# '" + elif bad_prefix and (bad_prefix != '!' or start[0] > 1): + if bad_prefix != '#': + yield start, "E265 block comment should start with '# '" + elif comment: + yield start, "E266 too many leading '#' for block comment" + elif token_type != tokenize.NL: + prev_end = end + + +def imports_on_separate_lines(logical_line): + r"""Imports should usually be on separate lines. + + Okay: import os\nimport sys + E401: import sys, os + + Okay: from subprocess import Popen, PIPE + Okay: from myclas import MyClass + Okay: from foo.bar.yourclass import YourClass + Okay: import myclass + Okay: import foo.bar.yourclass + """ + line = logical_line + if line.startswith('import '): + found = line.find(',') + if -1 < found and ';' not in line[:found]: + yield found, "E401 multiple imports on one line" + + +def module_imports_on_top_of_file( + logical_line, indent_level, checker_state, noqa): + r"""Imports are always put at the top of the file, just after any module + comments and docstrings, and before module globals and constants. + + Okay: import os + Okay: # this is a comment\nimport os + Okay: '''this is a module docstring'''\nimport os + Okay: r'''this is a module docstring'''\nimport os + Okay: try:\n import x\nexcept:\n pass\nelse:\n pass\nimport y + Okay: try:\n import x\nexcept:\n pass\nfinally:\n pass\nimport y + E402: a=1\nimport os + E402: 'One string'\n"Two string"\nimport os + E402: a=1\nfrom sys import x + + Okay: if x:\n import os + """ + def is_string_literal(line): + if line[0] in 'uUbB': + line = line[1:] + if line and line[0] in 'rR': + line = line[1:] + return line and (line[0] == '"' or line[0] == "'") + + allowed_try_keywords = ('try', 'except', 'else', 'finally') + + if indent_level: # Allow imports in conditional statements or functions + return + if not logical_line: # Allow empty lines or comments + return + if noqa: + return + line = logical_line + if line.startswith('import ') or line.startswith('from '): + if checker_state.get('seen_non_imports', False): + yield 0, "E402 module level import not at top of file" + elif any(line.startswith(kw) for kw in allowed_try_keywords): + # Allow try, except, else, finally keywords intermixed with imports in + # order to support conditional importing + return + elif is_string_literal(line): + # The first literal is a docstring, allow it. Otherwise, report error. + if checker_state.get('seen_docstring', False): + checker_state['seen_non_imports'] = True + else: + checker_state['seen_docstring'] = True + else: + checker_state['seen_non_imports'] = True + + +def compound_statements(logical_line): + r"""Compound statements (on the same line) are generally discouraged. + + While sometimes it's okay to put an if/for/while with a small body + on the same line, never do this for multi-clause statements. + Also avoid folding such long lines! + + Always use a def statement instead of an assignment statement that + binds a lambda expression directly to a name. + + Okay: if foo == 'blah':\n do_blah_thing() + Okay: do_one() + Okay: do_two() + Okay: do_three() + + E701: if foo == 'blah': do_blah_thing() + E701: for x in lst: total += x + E701: while t < 10: t = delay() + E701: if foo == 'blah': do_blah_thing() + E701: else: do_non_blah_thing() + E701: try: something() + E701: finally: cleanup() + E701: if foo == 'blah': one(); two(); three() + E702: do_one(); do_two(); do_three() + E703: do_four(); # useless semicolon + E704: def f(x): return 2*x + E731: f = lambda x: 2*x + """ + line = logical_line + last_char = len(line) - 1 + found = line.find(':') + while -1 < found < last_char: + before = line[:found] + if ((before.count('{') <= before.count('}') and # {'a': 1} (dict) + before.count('[') <= before.count(']') and # [1:2] (slice) + before.count('(') <= before.count(')'))): # (annotation) + lambda_kw = LAMBDA_REGEX.search(before) + if lambda_kw: + before = line[:lambda_kw.start()].rstrip() + if before[-1:] == '=' and isidentifier(before[:-1].strip()): + yield 0, ("E731 do not assign a lambda expression, use a " + "def") + break + if before.startswith('def '): + yield 0, "E704 multiple statements on one line (def)" + else: + yield found, "E701 multiple statements on one line (colon)" + found = line.find(':', found + 1) + found = line.find(';') + while -1 < found: + if found < last_char: + yield found, "E702 multiple statements on one line (semicolon)" + else: + yield found, "E703 statement ends with a semicolon" + found = line.find(';', found + 1) + + +def explicit_line_join(logical_line, tokens): + r"""Avoid explicit line join between brackets. + + The preferred way of wrapping long lines is by using Python's implied line + continuation inside parentheses, brackets and braces. Long lines can be + broken over multiple lines by wrapping expressions in parentheses. These + should be used in preference to using a backslash for line continuation. + + E502: aaa = [123, \\n 123] + E502: aaa = ("bbb " \\n "ccc") + + Okay: aaa = [123,\n 123] + Okay: aaa = ("bbb "\n "ccc") + Okay: aaa = "bbb " \\n "ccc" + Okay: aaa = 123 # \\ + """ + prev_start = prev_end = parens = 0 + comment = False + backslash = None + for token_type, text, start, end, line in tokens: + if token_type == tokenize.COMMENT: + comment = True + if start[0] != prev_start and parens and backslash and not comment: + yield backslash, "E502 the backslash is redundant between brackets" + if end[0] != prev_end: + if line.rstrip('\r\n').endswith('\\'): + backslash = (end[0], len(line.splitlines()[-1]) - 1) + else: + backslash = None + prev_start = prev_end = end[0] + else: + prev_start = start[0] + if token_type == tokenize.OP: + if text in '([{': + parens += 1 + elif text in ')]}': + parens -= 1 + + +def break_around_binary_operator(logical_line, tokens): + r""" + Avoid breaks before binary operators. + + The preferred place to break around a binary operator is after the + operator, not before it. + + W503: (width == 0\n + height == 0) + W503: (width == 0\n and height == 0) + + Okay: (width == 0 +\n height == 0) + Okay: foo(\n -x) + Okay: foo(x\n []) + Okay: x = '''\n''' + '' + Okay: foo(x,\n -y) + Okay: foo(x, # comment\n -y) + """ + def is_binary_operator(token_type, text): + # The % character is strictly speaking a binary operator, but the + # common usage seems to be to put it next to the format parameters, + # after a line break. + return ((token_type == tokenize.OP or text in ['and', 'or']) and + text not in "()[]{},:.;@=%") + + line_break = False + unary_context = True + for token_type, text, start, end, line in tokens: + if token_type == tokenize.COMMENT: + continue + if ('\n' in text or '\r' in text) and token_type != tokenize.STRING: + line_break = True + else: + if (is_binary_operator(token_type, text) and line_break and + not unary_context): + yield start, "W503 line break before binary operator" + unary_context = text in '([{,;' + line_break = False + + +def comparison_to_singleton(logical_line, noqa): + r"""Comparison to singletons should use "is" or "is not". + + Comparisons to singletons like None should always be done + with "is" or "is not", never the equality operators. + + Okay: if arg is not None: + E711: if arg != None: + E711: if None == arg: + E712: if arg == True: + E712: if False == arg: + + Also, beware of writing if x when you really mean if x is not None -- + e.g. when testing whether a variable or argument that defaults to None was + set to some other value. The other value might have a type (such as a + container) that could be false in a boolean context! + """ + match = not noqa and COMPARE_SINGLETON_REGEX.search(logical_line) + if match: + singleton = match.group(1) or match.group(3) + same = (match.group(2) == '==') + + msg = "'if cond is %s:'" % (('' if same else 'not ') + singleton) + if singleton in ('None',): + code = 'E711' + else: + code = 'E712' + nonzero = ((singleton == 'True' and same) or + (singleton == 'False' and not same)) + msg += " or 'if %scond:'" % ('' if nonzero else 'not ') + yield match.start(2), ("%s comparison to %s should be %s" % + (code, singleton, msg)) + + +def comparison_negative(logical_line): + r"""Negative comparison should be done using "not in" and "is not". + + Okay: if x not in y:\n pass + Okay: assert (X in Y or X is Z) + Okay: if not (X in Y):\n pass + Okay: zz = x is not y + E713: Z = not X in Y + E713: if not X.B in Y:\n pass + E714: if not X is Y:\n pass + E714: Z = not X.B is Y + """ + match = COMPARE_NEGATIVE_REGEX.search(logical_line) + if match: + pos = match.start(1) + if match.group(2) == 'in': + yield pos, "E713 test for membership should be 'not in'" + else: + yield pos, "E714 test for object identity should be 'is not'" + + +def comparison_type(logical_line, noqa): + r"""Object type comparisons should always use isinstance(). + + Do not compare types directly. + + Okay: if isinstance(obj, int): + E721: if type(obj) is type(1): + + When checking if an object is a string, keep in mind that it might be a + unicode string too! In Python 2.3, str and unicode have a common base + class, basestring, so you can do: + + Okay: if isinstance(obj, basestring): + Okay: if type(a1) is type(b1): + """ + match = COMPARE_TYPE_REGEX.search(logical_line) + if match and not noqa: + inst = match.group(1) + if inst and isidentifier(inst) and inst not in SINGLETONS: + return # Allow comparison for types which are not obvious + yield match.start(), "E721 do not compare types, use 'isinstance()'" + + +def python_3000_has_key(logical_line, noqa): + r"""The {}.has_key() method is removed in Python 3: use the 'in' operator. + + Okay: if "alph" in d:\n print d["alph"] + W601: assert d.has_key('alph') + """ + pos = logical_line.find('.has_key(') + if pos > -1 and not noqa: + yield pos, "W601 .has_key() is deprecated, use 'in'" + + +def python_3000_raise_comma(logical_line): + r"""When raising an exception, use "raise ValueError('message')". + + The older form is removed in Python 3. + + Okay: raise DummyError("Message") + W602: raise DummyError, "Message" + """ + match = RAISE_COMMA_REGEX.match(logical_line) + if match and not RERAISE_COMMA_REGEX.match(logical_line): + yield match.end() - 1, "W602 deprecated form of raising exception" + + +def python_3000_not_equal(logical_line): + r"""New code should always use != instead of <>. + + The older syntax is removed in Python 3. + + Okay: if a != 'no': + W603: if a <> 'no': + """ + pos = logical_line.find('<>') + if pos > -1: + yield pos, "W603 '<>' is deprecated, use '!='" + + +def python_3000_backticks(logical_line): + r"""Backticks are removed in Python 3: use repr() instead. + + Okay: val = repr(1 + 2) + W604: val = `1 + 2` + """ + pos = logical_line.find('`') + if pos > -1: + yield pos, "W604 backticks are deprecated, use 'repr()'" + + +############################################################################## +# Helper functions +############################################################################## + + +if sys.version_info < (3,): + # Python 2: implicit encoding. + def readlines(filename): + """Read the source code.""" + with open(filename, 'rU') as f: + return f.readlines() + isidentifier = re.compile(r'[a-zA-Z_]\w*$').match + stdin_get_value = sys.stdin.read +else: + # Python 3 + def readlines(filename): + """Read the source code.""" + try: + with open(filename, 'rb') as f: + (coding, lines) = tokenize.detect_encoding(f.readline) + f = TextIOWrapper(f, coding, line_buffering=True) + return [l.decode(coding) for l in lines] + f.readlines() + except (LookupError, SyntaxError, UnicodeError): + # Fall back if file encoding is improperly declared + with open(filename, encoding='latin-1') as f: + return f.readlines() + isidentifier = str.isidentifier + + def stdin_get_value(): + return TextIOWrapper(sys.stdin.buffer, errors='ignore').read() +noqa = re.compile(r'# no(?:qa|pep8)\b', re.I).search + + +def expand_indent(line): + r"""Return the amount of indentation. + + Tabs are expanded to the next multiple of 8. + + >>> expand_indent(' ') + 4 + >>> expand_indent('\t') + 8 + >>> expand_indent(' \t') + 8 + >>> expand_indent(' \t') + 16 + """ + if '\t' not in line: + return len(line) - len(line.lstrip()) + result = 0 + for char in line: + if char == '\t': + result = result // 8 * 8 + 8 + elif char == ' ': + result += 1 + else: + break + return result + + +def mute_string(text): + """Replace contents with 'xxx' to prevent syntax matching. + + >>> mute_string('"abc"') + '"xxx"' + >>> mute_string("'''abc'''") + "'''xxx'''" + >>> mute_string("r'abc'") + "r'xxx'" + """ + # String modifiers (e.g. u or r) + start = text.index(text[-1]) + 1 + end = len(text) - 1 + # Triple quotes + if text[-3:] in ('"""', "'''"): + start += 2 + end -= 2 + return text[:start] + 'x' * (end - start) + text[end:] + + +def parse_udiff(diff, patterns=None, parent='.'): + """Return a dictionary of matching lines.""" + # For each file of the diff, the entry key is the filename, + # and the value is a set of row numbers to consider. + rv = {} + path = nrows = None + for line in diff.splitlines(): + if nrows: + if line[:1] != '-': + nrows -= 1 + continue + if line[:3] == '@@ ': + hunk_match = HUNK_REGEX.match(line) + (row, nrows) = [int(g or '1') for g in hunk_match.groups()] + rv[path].update(range(row, row + nrows)) + elif line[:3] == '+++': + path = line[4:].split('\t', 1)[0] + if path[:2] == 'b/': + path = path[2:] + rv[path] = set() + return dict([(os.path.join(parent, path), rows) + for (path, rows) in rv.items() + if rows and filename_match(path, patterns)]) + + +def normalize_paths(value, parent=os.curdir): + """Parse a comma-separated list of paths. + + Return a list of absolute paths. + """ + if not value: + return [] + if isinstance(value, list): + return value + paths = [] + for path in value.split(','): + path = path.strip() + if '/' in path: + path = os.path.abspath(os.path.join(parent, path)) + paths.append(path.rstrip('/')) + return paths + + +def filename_match(filename, patterns, default=True): + """Check if patterns contains a pattern that matches filename. + + If patterns is unspecified, this always returns True. + """ + if not patterns: + return default + return any(fnmatch(filename, pattern) for pattern in patterns) + + +def _is_eol_token(token): + return token[0] in NEWLINE or token[4][token[3][1]:].lstrip() == '\\\n' +if COMMENT_WITH_NL: + def _is_eol_token(token, _eol_token=_is_eol_token): + return _eol_token(token) or (token[0] == tokenize.COMMENT and + token[1] == token[4]) + +############################################################################## +# Framework to run all checks +############################################################################## + + +_checks = {'physical_line': {}, 'logical_line': {}, 'tree': {}} + + +def _get_parameters(function): + if sys.version_info >= (3, 3): + return [parameter.name + for parameter + in inspect.signature(function).parameters.values() + if parameter.kind == parameter.POSITIONAL_OR_KEYWORD] + else: + return inspect.getargspec(function)[0] + + +def register_check(check, codes=None): + """Register a new check object.""" + def _add_check(check, kind, codes, args): + if check in _checks[kind]: + _checks[kind][check][0].extend(codes or []) + else: + _checks[kind][check] = (codes or [''], args) + if inspect.isfunction(check): + args = _get_parameters(check) + if args and args[0] in ('physical_line', 'logical_line'): + if codes is None: + codes = ERRORCODE_REGEX.findall(check.__doc__ or '') + _add_check(check, args[0], codes, args) + elif inspect.isclass(check): + if _get_parameters(check.__init__)[:2] == ['self', 'tree']: + _add_check(check, 'tree', codes, None) + + +def init_checks_registry(): + """Register all globally visible functions. + + The first argument name is either 'physical_line' or 'logical_line'. + """ + mod = inspect.getmodule(register_check) + for (name, function) in inspect.getmembers(mod, inspect.isfunction): + register_check(function) +init_checks_registry() + + +class Checker(object): + """Load a Python source file, tokenize it, check coding style.""" + + def __init__(self, filename=None, lines=None, + options=None, report=None, **kwargs): + if options is None: + options = StyleGuide(kwargs).options + else: + assert not kwargs + self._io_error = None + self._physical_checks = options.physical_checks + self._logical_checks = options.logical_checks + self._ast_checks = options.ast_checks + self.max_line_length = options.max_line_length + self.multiline = False # in a multiline string? + self.hang_closing = options.hang_closing + self.verbose = options.verbose + self.filename = filename + # Dictionary where a checker can store its custom state. + self._checker_states = {} + if filename is None: + self.filename = 'stdin' + self.lines = lines or [] + elif filename == '-': + self.filename = 'stdin' + self.lines = stdin_get_value().splitlines(True) + elif lines is None: + try: + self.lines = readlines(filename) + except IOError: + (exc_type, exc) = sys.exc_info()[:2] + self._io_error = '%s: %s' % (exc_type.__name__, exc) + self.lines = [] + else: + self.lines = lines + if self.lines: + ord0 = ord(self.lines[0][0]) + if ord0 in (0xef, 0xfeff): # Strip the UTF-8 BOM + if ord0 == 0xfeff: + self.lines[0] = self.lines[0][1:] + elif self.lines[0][:3] == '\xef\xbb\xbf': + self.lines[0] = self.lines[0][3:] + self.report = report or options.report + self.report_error = self.report.error + + def report_invalid_syntax(self): + """Check if the syntax is valid.""" + (exc_type, exc) = sys.exc_info()[:2] + if len(exc.args) > 1: + offset = exc.args[1] + if len(offset) > 2: + offset = offset[1:3] + else: + offset = (1, 0) + self.report_error(offset[0], offset[1] or 0, + 'E901 %s: %s' % (exc_type.__name__, exc.args[0]), + self.report_invalid_syntax) + + def readline(self): + """Get the next line from the input buffer.""" + if self.line_number >= self.total_lines: + return '' + line = self.lines[self.line_number] + self.line_number += 1 + if self.indent_char is None and line[:1] in WHITESPACE: + self.indent_char = line[0] + return line + + def run_check(self, check, argument_names): + """Run a check plugin.""" + arguments = [] + for name in argument_names: + arguments.append(getattr(self, name)) + return check(*arguments) + + def init_checker_state(self, name, argument_names): + """ Prepares a custom state for the specific checker plugin.""" + if 'checker_state' in argument_names: + self.checker_state = self._checker_states.setdefault(name, {}) + + def check_physical(self, line): + """Run all physical checks on a raw input line.""" + self.physical_line = line + for name, check, argument_names in self._physical_checks: + self.init_checker_state(name, argument_names) + result = self.run_check(check, argument_names) + if result is not None: + (offset, text) = result + self.report_error(self.line_number, offset, text, check) + if text[:4] == 'E101': + self.indent_char = line[0] + + def build_tokens_line(self): + """Build a logical line from tokens.""" + logical = [] + comments = [] + length = 0 + prev_row = prev_col = mapping = None + for token_type, text, start, end, line in self.tokens: + if token_type in SKIP_TOKENS: + continue + if not mapping: + mapping = [(0, start)] + if token_type == tokenize.COMMENT: + comments.append(text) + continue + if token_type == tokenize.STRING: + text = mute_string(text) + if prev_row: + (start_row, start_col) = start + if prev_row != start_row: # different row + prev_text = self.lines[prev_row - 1][prev_col - 1] + if prev_text == ',' or (prev_text not in '{[(' and + text not in '}])'): + text = ' ' + text + elif prev_col != start_col: # different column + text = line[prev_col:start_col] + text + logical.append(text) + length += len(text) + mapping.append((length, end)) + (prev_row, prev_col) = end + self.logical_line = ''.join(logical) + self.noqa = comments and noqa(''.join(comments)) + return mapping + + def check_logical(self): + """Build a line from tokens and run all logical checks on it.""" + self.report.increment_logical_line() + mapping = self.build_tokens_line() + + if not mapping: + return + + (start_row, start_col) = mapping[0][1] + start_line = self.lines[start_row - 1] + self.indent_level = expand_indent(start_line[:start_col]) + if self.blank_before < self.blank_lines: + self.blank_before = self.blank_lines + if self.verbose >= 2: + print(self.logical_line[:80].rstrip()) + for name, check, argument_names in self._logical_checks: + if self.verbose >= 4: + print(' ' + name) + self.init_checker_state(name, argument_names) + for offset, text in self.run_check(check, argument_names) or (): + if not isinstance(offset, tuple): + for token_offset, pos in mapping: + if offset <= token_offset: + break + offset = (pos[0], pos[1] + offset - token_offset) + self.report_error(offset[0], offset[1], text, check) + if self.logical_line: + self.previous_indent_level = self.indent_level + self.previous_logical = self.logical_line + self.blank_lines = 0 + self.tokens = [] + + def check_ast(self): + """Build the file's AST and run all AST checks.""" + try: + tree = compile(''.join(self.lines), '', 'exec', PyCF_ONLY_AST) + except (ValueError, SyntaxError, TypeError): + return self.report_invalid_syntax() + for name, cls, __ in self._ast_checks: + checker = cls(tree, self.filename) + for lineno, offset, text, check in checker.run(): + if not self.lines or not noqa(self.lines[lineno - 1]): + self.report_error(lineno, offset, text, check) + + def generate_tokens(self): + """Tokenize the file, run physical line checks and yield tokens.""" + if self._io_error: + self.report_error(1, 0, 'E902 %s' % self._io_error, readlines) + tokengen = tokenize.generate_tokens(self.readline) + try: + for token in tokengen: + if token[2][0] > self.total_lines: + return + self.maybe_check_physical(token) + yield token + except (SyntaxError, tokenize.TokenError): + self.report_invalid_syntax() + + def maybe_check_physical(self, token): + """If appropriate (based on token), check current physical line(s).""" + # Called after every token, but act only on end of line. + if _is_eol_token(token): + # Obviously, a newline token ends a single physical line. + self.check_physical(token[4]) + elif token[0] == tokenize.STRING and '\n' in token[1]: + # Less obviously, a string that contains newlines is a + # multiline string, either triple-quoted or with internal + # newlines backslash-escaped. Check every physical line in the + # string *except* for the last one: its newline is outside of + # the multiline string, so we consider it a regular physical + # line, and will check it like any other physical line. + # + # Subtleties: + # - we don't *completely* ignore the last line; if it contains + # the magical "# noqa" comment, we disable all physical + # checks for the entire multiline string + # - have to wind self.line_number back because initially it + # points to the last line of the string, and we want + # check_physical() to give accurate feedback + if noqa(token[4]): + return + self.multiline = True + self.line_number = token[2][0] + for line in token[1].split('\n')[:-1]: + self.check_physical(line + '\n') + self.line_number += 1 + self.multiline = False + + def check_all(self, expected=None, line_offset=0): + """Run all checks on the input file.""" + self.report.init_file(self.filename, self.lines, expected, line_offset) + self.total_lines = len(self.lines) + if self._ast_checks: + self.check_ast() + self.line_number = 0 + self.indent_char = None + self.indent_level = self.previous_indent_level = 0 + self.previous_logical = '' + self.tokens = [] + self.blank_lines = self.blank_before = 0 + parens = 0 + for token in self.generate_tokens(): + self.tokens.append(token) + token_type, text = token[0:2] + if self.verbose >= 3: + if token[2][0] == token[3][0]: + pos = '[%s:%s]' % (token[2][1] or '', token[3][1]) + else: + pos = 'l.%s' % token[3][0] + print('l.%s\t%s\t%s\t%r' % + (token[2][0], pos, tokenize.tok_name[token[0]], text)) + if token_type == tokenize.OP: + if text in '([{': + parens += 1 + elif text in '}])': + parens -= 1 + elif not parens: + if token_type in NEWLINE: + if token_type == tokenize.NEWLINE: + self.check_logical() + self.blank_before = 0 + elif len(self.tokens) == 1: + # The physical line contains only this token. + self.blank_lines += 1 + del self.tokens[0] + else: + self.check_logical() + elif COMMENT_WITH_NL and token_type == tokenize.COMMENT: + if len(self.tokens) == 1: + # The comment also ends a physical line + token = list(token) + token[1] = text.rstrip('\r\n') + token[3] = (token[2][0], token[2][1] + len(token[1])) + self.tokens = [tuple(token)] + self.check_logical() + if self.tokens: + self.check_physical(self.lines[-1]) + self.check_logical() + return self.report.get_file_results() + + +class BaseReport(object): + """Collect the results of the checks.""" + + print_filename = False + + def __init__(self, options): + self._benchmark_keys = options.benchmark_keys + self._ignore_code = options.ignore_code + # Results + self.elapsed = 0 + self.total_errors = 0 + self.counters = dict.fromkeys(self._benchmark_keys, 0) + self.messages = {} + + def start(self): + """Start the timer.""" + self._start_time = time.time() + + def stop(self): + """Stop the timer.""" + self.elapsed = time.time() - self._start_time + + def init_file(self, filename, lines, expected, line_offset): + """Signal a new file.""" + self.filename = filename + self.lines = lines + self.expected = expected or () + self.line_offset = line_offset + self.file_errors = 0 + self.counters['files'] += 1 + self.counters['physical lines'] += len(lines) + + def increment_logical_line(self): + """Signal a new logical line.""" + self.counters['logical lines'] += 1 + + def error(self, line_number, offset, text, check): + """Report an error, according to options.""" + code = text[:4] + if self._ignore_code(code): + return + if code in self.counters: + self.counters[code] += 1 + else: + self.counters[code] = 1 + self.messages[code] = text[5:] + # Don't care about expected errors or warnings + if code in self.expected: + return + if self.print_filename and not self.file_errors: + print(self.filename) + self.file_errors += 1 + self.total_errors += 1 + return code + + def get_file_results(self): + """Return the count of errors and warnings for this file.""" + return self.file_errors + + def get_count(self, prefix=''): + """Return the total count of errors and warnings.""" + return sum([self.counters[key] + for key in self.messages if key.startswith(prefix)]) + + def get_statistics(self, prefix=''): + """Get statistics for message codes that start with the prefix. + + prefix='' matches all errors and warnings + prefix='E' matches all errors + prefix='W' matches all warnings + prefix='E4' matches all errors that have to do with imports + """ + return ['%-7s %s %s' % (self.counters[key], key, self.messages[key]) + for key in sorted(self.messages) if key.startswith(prefix)] + + def print_statistics(self, prefix=''): + """Print overall statistics (number of errors and warnings).""" + for line in self.get_statistics(prefix): + print(line) + + def print_benchmark(self): + """Print benchmark numbers.""" + print('%-7.2f %s' % (self.elapsed, 'seconds elapsed')) + if self.elapsed: + for key in self._benchmark_keys: + print('%-7d %s per second (%d total)' % + (self.counters[key] / self.elapsed, key, + self.counters[key])) + + +class FileReport(BaseReport): + """Collect the results of the checks and print only the filenames.""" + print_filename = True + + +class StandardReport(BaseReport): + """Collect and print the results of the checks.""" + + def __init__(self, options): + super(StandardReport, self).__init__(options) + self._fmt = REPORT_FORMAT.get(options.format.lower(), + options.format) + self._repeat = options.repeat + self._show_source = options.show_source + self._show_pep8 = options.show_pep8 + + def init_file(self, filename, lines, expected, line_offset): + """Signal a new file.""" + self._deferred_print = [] + return super(StandardReport, self).init_file( + filename, lines, expected, line_offset) + + def error(self, line_number, offset, text, check): + """Report an error, according to options.""" + code = super(StandardReport, self).error(line_number, offset, + text, check) + if code and (self.counters[code] == 1 or self._repeat): + self._deferred_print.append( + (line_number, offset, code, text[5:], check.__doc__)) + return code + + def get_file_results(self): + """Print the result and return the overall count for this file.""" + self._deferred_print.sort() + for line_number, offset, code, text, doc in self._deferred_print: + print(self._fmt % { + 'path': self.filename, + 'row': self.line_offset + line_number, 'col': offset + 1, + 'code': code, 'text': text, + }) + if self._show_source: + if line_number > len(self.lines): + line = '' + else: + line = self.lines[line_number - 1] + print(line.rstrip()) + print(re.sub(r'\S', ' ', line[:offset]) + '^') + if self._show_pep8 and doc: + print(' ' + doc.strip()) + + # stdout is block buffered when not stdout.isatty(). + # line can be broken where buffer boundary since other processes + # write to same file. + # flush() after print() to avoid buffer boundary. + # Typical buffer size is 8192. line written safely when + # len(line) < 8192. + sys.stdout.flush() + return self.file_errors + + +class DiffReport(StandardReport): + """Collect and print the results for the changed lines only.""" + + def __init__(self, options): + super(DiffReport, self).__init__(options) + self._selected = options.selected_lines + + def error(self, line_number, offset, text, check): + if line_number not in self._selected[self.filename]: + return + return super(DiffReport, self).error(line_number, offset, text, check) + + +class StyleGuide(object): + """Initialize a PEP-8 instance with few options.""" + + def __init__(self, *args, **kwargs): + # build options from the command line + self.checker_class = kwargs.pop('checker_class', Checker) + parse_argv = kwargs.pop('parse_argv', False) + config_file = kwargs.pop('config_file', False) + parser = kwargs.pop('parser', None) + # build options from dict + options_dict = dict(*args, **kwargs) + arglist = None if parse_argv else options_dict.get('paths', None) + options, self.paths = process_options( + arglist, parse_argv, config_file, parser) + if options_dict: + options.__dict__.update(options_dict) + if 'paths' in options_dict: + self.paths = options_dict['paths'] + + self.runner = self.input_file + self.options = options + + if not options.reporter: + options.reporter = BaseReport if options.quiet else StandardReport + + options.select = tuple(options.select or ()) + if not (options.select or options.ignore or + options.testsuite or options.doctest) and DEFAULT_IGNORE: + # The default choice: ignore controversial checks + options.ignore = tuple(DEFAULT_IGNORE.split(',')) + else: + # Ignore all checks which are not explicitly selected + options.ignore = ('',) if options.select else tuple(options.ignore) + options.benchmark_keys = BENCHMARK_KEYS[:] + options.ignore_code = self.ignore_code + options.physical_checks = self.get_checks('physical_line') + options.logical_checks = self.get_checks('logical_line') + options.ast_checks = self.get_checks('tree') + self.init_report() + + def init_report(self, reporter=None): + """Initialize the report instance.""" + self.options.report = (reporter or self.options.reporter)(self.options) + return self.options.report + + def check_files(self, paths=None): + """Run all checks on the paths.""" + if paths is None: + paths = self.paths + report = self.options.report + runner = self.runner + report.start() + try: + for path in paths: + if os.path.isdir(path): + self.input_dir(path) + elif not self.excluded(path): + runner(path) + except KeyboardInterrupt: + print('... stopped') + report.stop() + return report + + def input_file(self, filename, lines=None, expected=None, line_offset=0): + """Run all checks on a Python source file.""" + if self.options.verbose: + print('checking %s' % filename) + fchecker = self.checker_class( + filename, lines=lines, options=self.options) + return fchecker.check_all(expected=expected, line_offset=line_offset) + + def input_dir(self, dirname): + """Check all files in this directory and all subdirectories.""" + dirname = dirname.rstrip('/') + if self.excluded(dirname): + return 0 + counters = self.options.report.counters + verbose = self.options.verbose + filepatterns = self.options.filename + runner = self.runner + for root, dirs, files in os.walk(dirname): + if verbose: + print('directory ' + root) + counters['directories'] += 1 + for subdir in sorted(dirs): + if self.excluded(subdir, root): + dirs.remove(subdir) + for filename in sorted(files): + # contain a pattern that matches? + if ((filename_match(filename, filepatterns) and + not self.excluded(filename, root))): + runner(os.path.join(root, filename)) + + def excluded(self, filename, parent=None): + """Check if the file should be excluded. + + Check if 'options.exclude' contains a pattern that matches filename. + """ + if not self.options.exclude: + return False + basename = os.path.basename(filename) + if filename_match(basename, self.options.exclude): + return True + if parent: + filename = os.path.join(parent, filename) + filename = os.path.abspath(filename) + return filename_match(filename, self.options.exclude) + + def ignore_code(self, code): + """Check if the error code should be ignored. + + If 'options.select' contains a prefix of the error code, + return False. Else, if 'options.ignore' contains a prefix of + the error code, return True. + """ + if len(code) < 4 and any(s.startswith(code) + for s in self.options.select): + return False + return (code.startswith(self.options.ignore) and + not code.startswith(self.options.select)) + + def get_checks(self, argument_name): + """Get all the checks for this category. + + Find all globally visible functions where the first argument name + starts with argument_name and which contain selected tests. + """ + checks = [] + for check, attrs in _checks[argument_name].items(): + (codes, args) = attrs + if any(not (code and self.ignore_code(code)) for code in codes): + checks.append((check.__name__, check, args)) + return sorted(checks) + + +def get_parser(prog='pep8', version=__version__): + parser = OptionParser(prog=prog, version=version, + usage="%prog [options] input ...") + parser.config_options = [ + 'exclude', 'filename', 'select', 'ignore', 'max-line-length', + 'hang-closing', 'count', 'format', 'quiet', 'show-pep8', + 'show-source', 'statistics', 'verbose'] + parser.add_option('-v', '--verbose', default=0, action='count', + help="print status messages, or debug with -vv") + parser.add_option('-q', '--quiet', default=0, action='count', + help="report only file names, or nothing with -qq") + parser.add_option('-r', '--repeat', default=True, action='store_true', + help="(obsolete) show all occurrences of the same error") + parser.add_option('--first', action='store_false', dest='repeat', + help="show first occurrence of each error") + parser.add_option('--exclude', metavar='patterns', default=DEFAULT_EXCLUDE, + help="exclude files or directories which match these " + "comma separated patterns (default: %default)") + parser.add_option('--filename', metavar='patterns', default='*.py', + help="when parsing directories, only check filenames " + "matching these comma separated patterns " + "(default: %default)") + parser.add_option('--select', metavar='errors', default='', + help="select errors and warnings (e.g. E,W6)") + parser.add_option('--ignore', metavar='errors', default='', + help="skip errors and warnings (e.g. E4,W) " + "(default: %s)" % DEFAULT_IGNORE) + parser.add_option('--show-source', action='store_true', + help="show source code for each error") + parser.add_option('--show-pep8', action='store_true', + help="show text of PEP 8 for each error " + "(implies --first)") + parser.add_option('--statistics', action='store_true', + help="count errors and warnings") + parser.add_option('--count', action='store_true', + help="print total number of errors and warnings " + "to standard error and set exit code to 1 if " + "total is not null") + parser.add_option('--max-line-length', type='int', metavar='n', + default=MAX_LINE_LENGTH, + help="set maximum allowed line length " + "(default: %default)") + parser.add_option('--hang-closing', action='store_true', + help="hang closing bracket instead of matching " + "indentation of opening bracket's line") + parser.add_option('--format', metavar='format', default='default', + help="set the error format [default|pylint|]") + parser.add_option('--diff', action='store_true', + help="report changes only within line number ranges in " + "the unified diff received on STDIN") + group = parser.add_option_group("Testing Options") + if os.path.exists(TESTSUITE_PATH): + group.add_option('--testsuite', metavar='dir', + help="run regression tests from dir") + group.add_option('--doctest', action='store_true', + help="run doctest on myself") + group.add_option('--benchmark', action='store_true', + help="measure processing speed") + return parser + + +def read_config(options, args, arglist, parser): + """Read and parse configurations + + If a config file is specified on the command line with the "--config" + option, then only it is used for configuration. + + Otherwise, the user configuration (~/.config/pep8) and any local + configurations in the current directory or above will be merged together + (in that order) using the read method of ConfigParser. + """ + config = RawConfigParser() + + cli_conf = options.config + + local_dir = os.curdir + + if USER_CONFIG and os.path.isfile(USER_CONFIG): + if options.verbose: + print('user configuration: %s' % USER_CONFIG) + config.read(USER_CONFIG) + + parent = tail = args and os.path.abspath(os.path.commonprefix(args)) + while tail: + if config.read(os.path.join(parent, fn) for fn in PROJECT_CONFIG): + local_dir = parent + if options.verbose: + print('local configuration: in %s' % parent) + break + (parent, tail) = os.path.split(parent) + + if cli_conf and os.path.isfile(cli_conf): + if options.verbose: + print('cli configuration: %s' % cli_conf) + config.read(cli_conf) + + pep8_section = parser.prog + if config.has_section(pep8_section): + option_list = dict([(o.dest, o.type or o.action) + for o in parser.option_list]) + + # First, read the default values + (new_options, __) = parser.parse_args([]) + + # Second, parse the configuration + for opt in config.options(pep8_section): + if opt.replace('_', '-') not in parser.config_options: + print(" unknown option '%s' ignored" % opt) + continue + if options.verbose > 1: + print(" %s = %s" % (opt, config.get(pep8_section, opt))) + normalized_opt = opt.replace('-', '_') + opt_type = option_list[normalized_opt] + if opt_type in ('int', 'count'): + value = config.getint(pep8_section, opt) + elif opt_type == 'string': + value = config.get(pep8_section, opt) + if normalized_opt == 'exclude': + value = normalize_paths(value, local_dir) + else: + assert opt_type in ('store_true', 'store_false') + value = config.getboolean(pep8_section, opt) + setattr(new_options, normalized_opt, value) + + # Third, overwrite with the command-line options + (options, __) = parser.parse_args(arglist, values=new_options) + options.doctest = options.testsuite = False + return options + + +def process_options(arglist=None, parse_argv=False, config_file=None, + parser=None): + """Process options passed either via arglist or via command line args. + + Passing in the ``config_file`` parameter allows other tools, such as flake8 + to specify their own options to be processed in pep8. + """ + if not parser: + parser = get_parser() + if not parser.has_option('--config'): + group = parser.add_option_group("Configuration", description=( + "The project options are read from the [%s] section of the " + "tox.ini file or the setup.cfg file located in any parent folder " + "of the path(s) being processed. Allowed options are: %s." % + (parser.prog, ', '.join(parser.config_options)))) + group.add_option('--config', metavar='path', default=config_file, + help="user config file location") + # Don't read the command line if the module is used as a library. + if not arglist and not parse_argv: + arglist = [] + # If parse_argv is True and arglist is None, arguments are + # parsed from the command line (sys.argv) + (options, args) = parser.parse_args(arglist) + options.reporter = None + + if options.ensure_value('testsuite', False): + args.append(options.testsuite) + elif not options.ensure_value('doctest', False): + if parse_argv and not args: + if options.diff or any(os.path.exists(name) + for name in PROJECT_CONFIG): + args = ['.'] + else: + parser.error('input not specified') + options = read_config(options, args, arglist, parser) + options.reporter = parse_argv and options.quiet == 1 and FileReport + + options.filename = _parse_multi_options(options.filename) + options.exclude = normalize_paths(options.exclude) + options.select = _parse_multi_options(options.select) + options.ignore = _parse_multi_options(options.ignore) + + if options.diff: + options.reporter = DiffReport + stdin = stdin_get_value() + options.selected_lines = parse_udiff(stdin, options.filename, args[0]) + args = sorted(options.selected_lines) + + return options, args + + +def _parse_multi_options(options, split_token=','): + r"""Split and strip and discard empties. + + Turns the following: + + A, + B, + + into ["A", "B"] + """ + if options: + return [o.strip() for o in options.split(split_token) if o.strip()] + else: + return options + + +def _main(): + """Parse options and run checks on Python source.""" + import signal + + # Handle "Broken pipe" gracefully + try: + signal.signal(signal.SIGPIPE, lambda signum, frame: sys.exit(1)) + except AttributeError: + pass # not supported on Windows + + pep8style = StyleGuide(parse_argv=True) + options = pep8style.options + + if options.doctest or options.testsuite: + from testsuite.support import run_tests + report = run_tests(pep8style) + else: + report = pep8style.check_files() + + if options.statistics: + report.print_statistics() + + if options.benchmark: + report.print_benchmark() + + if options.testsuite and not options.quiet: + report.print_results() + + if report.total_errors: + if options.count: + sys.stderr.write(str(report.total_errors) + '\n') + sys.exit(1) + +if __name__ == '__main__': + _main() diff --git a/python/dllib/test/dev/run-all.sh b/python/dllib/test/dev/run-all.sh new file mode 100755 index 00000000000..5bf0694c852 --- /dev/null +++ b/python/dllib/test/dev/run-all.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +BASE="$(cd "`dirname $0`"; pwd)" + +$BASE/lint-python && $BASE/run-tests \ No newline at end of file diff --git a/python/dllib/test/dev/run-tests b/python/dllib/test/dev/run-tests new file mode 100755 index 00000000000..217f6cb3777 --- /dev/null +++ b/python/dllib/test/dev/run-tests @@ -0,0 +1,37 @@ +#!/usr/bin/env bash + +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +DL_PYTHON_HOME="$(cd "`dirname $0`"/../..; pwd)" + +BIGDL_HOME="$(cd "`dirname $0`"/../../..; pwd)" + +echo "BIGDL_HOME: $BIGDL_HOME" +echo "SPARK_HOME": $SPARK_HOME +echo "DL_PYTHON_HOME": $DL_PYTHON_HOME + +if [ -z ${SPARK_HOME+x} ]; then echo "SPARK_HOME is unset"; exit 1; else echo "SPARK_HOME is set to '$SPARK_HOME'"; fi + +PYSPARK_ZIP=`find $SPARK_HOME/python/lib -type f -iname '*.zip' | tr "\n" ":"` + +export PYTHONPATH=$PYSPARK_ZIP:$DL_PYTHON_HOME:$DL_PYTHON_HOME/dl:$DL_PYTHON_HOME/test/dev:$BIGDL_HOME/spark/dl/src/main/resources/spark-bigdl.conf + +export SPARK_CLASSPATH=$BIGDL_HOME/spark/dl/target/bigdl-0.1.0-SNAPSHOT-jar-with-dependencies.jar + +cd "`dirname $0`" +exec $BIGDL_HOME/scripts/bigdl.sh -- python -u ./run-tests.py "$@" diff --git a/python/dllib/test/dev/run-tests.py b/python/dllib/test/dev/run-tests.py new file mode 100755 index 00000000000..a8d8e445655 --- /dev/null +++ b/python/dllib/test/dev/run-tests.py @@ -0,0 +1,233 @@ +#!/usr/bin/env python + +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Adopt from Spark and it might be refactored in the future + + +from __future__ import print_function + +import logging +import os +import re +import subprocess +import sys +import tempfile +import time +from optparse import OptionParser +from os import path +from threading import Thread, Lock +from modules import all_modules # noqa +if sys.version < '3': + import Queue +else: + import queue as Queue + +if sys.version_info >= (2, 7): + subprocess_check_output = subprocess.check_output + subprocess_check_call = subprocess.check_call +else: + raise Exception("only support version >= 2,7") + + +def is_exe(path): + """ + Check if a given path is an executable file. + From: http://stackoverflow.com/a/377028 + """ + return os.path.isfile(path) and os.access(path, os.X_OK) + + +def which(program): + """ + Find and return the given program by its absolute path or 'None' if the program cannot be found. + From: http://stackoverflow.com/a/377028 + """ + + fpath = os.path.split(program)[0] + + if fpath: + if is_exe(program): + return program + else: + for path in os.environ.get("PATH").split(os.pathsep): + path = path.strip('"') + exe_file = os.path.join(path, program) + if is_exe(exe_file): + return exe_file + return None + + +current_dir = path.dirname(path.realpath(__file__)) +bigdl_python_home = path.abspath(path.join(current_dir, '..')) +sys.path.append(bigdl_python_home) + +SPARK_HOME = os.getenv('SPARK_HOME') + +python_modules = dict((m.name, m) for m in all_modules if m.python_test_goals if m.name != 'root') + + +def print_red(text): + print('\033[31m' + text + '\033[0m') + + +LOG_FILE = os.path.join(current_dir, "./unit-tests.log") +FAILURE_REPORTING_LOCK = Lock() +LOGGER = logging.getLogger() + + +def run_individual_python_test(test_name, python_exec): + env = dict(os.environ) + env.update({ + 'DL_CORE_NUMBER': '4' + }) + LOGGER.debug("Starting test(%s): %s", python_exec, test_name) + start_time = time.time() + try: + per_test_output = tempfile.TemporaryFile() + retcode = subprocess.Popen( + [python_exec, "-m", test_name], + stderr=per_test_output, stdout=per_test_output, env=env).wait() + except: + LOGGER.exception("Got exception while running %s with %s", test_name, python_exec) + # Here, we use os._exit() instead of sys.exit() in order to force Python to exit even if + # this code is invoked from a thread other than the main thread. + os._exit(1) + duration = time.time() - start_time + # Exit on the first failure. + if retcode != 0: + try: + with FAILURE_REPORTING_LOCK: + with open(LOG_FILE, 'ab') as log_file: + per_test_output.seek(0) + log_file.writelines(per_test_output) + per_test_output.seek(0) + for line in per_test_output: + decoded_line = line.decode() + if not re.match('[0-9]+', decoded_line): + print(decoded_line, end='') + per_test_output.close() + except: + LOGGER.exception("Got an exception while trying to print failed test output") + finally: + print_red("\nHad test failures in %s with %s; see logs." % (test_name, python_exec)) + # Here, we use os._exit() instead of sys.exit() in order to force Python to exit even if + # this code is invoked from a thread other than the main thread. + os._exit(-1) + else: + per_test_output.close() + LOGGER.info("Finished test(%s): %s (%is)", python_exec, test_name, duration) + + +def get_default_python_executables(): + # TODO: add more version. only support python 2.7 for now + python_execs = [] + python_execs.insert(0, "python") + return python_execs + + +def parse_opts(): + parser = OptionParser( + prog="run-tests" + ) + parser.add_option( + "--python-executables", type="string", default=','.join(get_default_python_executables()), + help="A comma-separated list of Python executables to test against (default: %default)" + ) + parser.add_option( + "--modules", type="string", + default=",".join(sorted(python_modules.keys())), + help="A comma-separated list of Python modules to test (default: %default)" + ) + parser.add_option( + "-p", "--parallelism", type="int", default=4, + help="The number of suites to test in parallel (default %default)" + ) + parser.add_option( + "--verbose", action="store_true", + help="Enable additional debug logging" + ) + + (opts, args) = parser.parse_args() + if args: + parser.error("Unsupported arguments: %s" % ' '.join(args)) + if opts.parallelism < 1: + parser.error("Parallelism cannot be less than 1") + return opts + + +def main(): + opts = parse_opts() + if (opts.verbose): + log_level = logging.DEBUG + else: + log_level = logging.INFO + logging.basicConfig(stream=sys.stdout, level=log_level, format="%(message)s") + LOGGER.info("Running BigDL python tests. Output is in %s", LOG_FILE) + if os.path.exists(LOG_FILE): + os.remove(LOG_FILE) + python_execs = opts.python_executables.split(',') + modules_to_test = [] + for module_name in opts.modules.split(','): + if module_name in python_modules: + modules_to_test.append(python_modules[module_name]) + else: + print("Error: unrecognized module '%s'. Supported modules: %s" % + (module_name, ", ".join(python_modules))) + sys.exit(-1) + LOGGER.info("Will test against the following Python executables: %s", python_execs) + LOGGER.info("Will test the following Python modules: %s", [x.name for x in modules_to_test]) + + task_queue = Queue.PriorityQueue() + for python_exec in python_execs: + LOGGER.debug("%s version is: %s", python_exec, subprocess_check_output( + [python_exec, "--version"], stderr=subprocess.STDOUT, universal_newlines=True).strip()) + for module in modules_to_test: + for test_goal in module.python_test_goals: + if test_goal in ('nn.layer'): + priority = 0 + else: + priority = 100 + task_queue.put((priority, (python_exec, test_goal))) + + def process_queue(task_queue): + while True: + try: + (priority, (python_exec, test_goal)) = task_queue.get_nowait() + except Queue.Empty: + break + try: + run_individual_python_test(test_goal, python_exec) + finally: + task_queue.task_done() + + start_time = time.time() + for _ in range(opts.parallelism): + worker = Thread(target=process_queue, args=(task_queue,)) + worker.daemon = True + worker.start() + try: + task_queue.join() + except (KeyboardInterrupt, SystemExit): + print_red("Exiting due to interrupt") + sys.exit(-1) + total_duration = time.time() - start_time + LOGGER.info("Tests passed in %i seconds", total_duration) + + +if __name__ == "__main__": + main() From 737f7ba5783650ec9328f3ebeb640ebdd5633376 Mon Sep 17 00:00:00 2001 From: zhichao-li Date: Tue, 21 Mar 2017 16:06:55 +0800 Subject: [PATCH 012/823] add Tensor wrapper add CrossEntropyCriterion --- README.md | 127 -- .../bigdl/dllib/feature/dataset/__init__.py | 15 - .../src/bigdl/dllib/feature/dataset/base.py | 197 -- .../src/bigdl/dllib/feature/dataset/mnist.py | 113 -- .../src/bigdl/dllib/feature/dataset/news20.py | 73 - .../dllib/feature/dataset/transformer.py | 23 - .../dllib/src/bigdl/dllib/models/__init__.py | 15 - .../src/bigdl/dllib/models/lenet/__init__.py | 15 - .../src/bigdl/dllib/models/lenet/lenet5.py | 97 - .../models/textclassifier/textclassifier.py | 177 -- python/dllib/src/bigdl/dllib/nn/__init__.py | 15 - python/dllib/src/bigdl/dllib/nn/criterion.py | 308 --- python/dllib/src/bigdl/dllib/nn/layer.py | 1673 ----------------- .../dllib/src/bigdl/dllib/optim/__init__.py | 15 - .../dllib/src/bigdl/dllib/optim/optimizer.py | 169 -- python/dllib/src/bigdl/dllib/utils/common.py | 261 --- python/dllib/src/bigdl/utils/__init__.py | 15 - 17 files changed, 3308 deletions(-) delete mode 100644 README.md delete mode 100644 python/dllib/src/bigdl/dllib/feature/dataset/__init__.py delete mode 100644 python/dllib/src/bigdl/dllib/feature/dataset/base.py delete mode 100644 python/dllib/src/bigdl/dllib/feature/dataset/mnist.py delete mode 100644 python/dllib/src/bigdl/dllib/feature/dataset/news20.py delete mode 100644 python/dllib/src/bigdl/dllib/feature/dataset/transformer.py delete mode 100644 python/dllib/src/bigdl/dllib/models/__init__.py delete mode 100644 python/dllib/src/bigdl/dllib/models/lenet/__init__.py delete mode 100644 python/dllib/src/bigdl/dllib/models/lenet/lenet5.py delete mode 100644 python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py delete mode 100644 python/dllib/src/bigdl/dllib/nn/__init__.py delete mode 100644 python/dllib/src/bigdl/dllib/nn/criterion.py delete mode 100644 python/dllib/src/bigdl/dllib/nn/layer.py delete mode 100644 python/dllib/src/bigdl/dllib/optim/__init__.py delete mode 100644 python/dllib/src/bigdl/dllib/optim/optimizer.py delete mode 100644 python/dllib/src/bigdl/dllib/utils/common.py delete mode 100644 python/dllib/src/bigdl/utils/__init__.py diff --git a/README.md b/README.md deleted file mode 100644 index ed72d7eab3d..00000000000 --- a/README.md +++ /dev/null @@ -1,127 +0,0 @@ - -BigDL comes with scala API for now. However Python is a powerful programming language for data analysis and with large amount of useful libraries, we are developing a lightweight python binding on top of PySpark which can enable us use Python naively with BigDL. - -The Python API is almost identical to the scala version, and it would map ndarray to tensor for the training samples, so basically user only need to care about how to manipulate ndarray. - -This Python binding tested with Python 2.7 and Spark 1.6.0. - -Here are the steps for training a simple LeNet model: - -1). Create a RDD[Sample]: -``` -RDD[..] --transform-->RDD[ndarray, ndarray].map(Sample.from_ndarray(features, label)) --> RDD[Sample] -``` - -2). Define a model: -``` - def build_model(class_num): - model = Sequential() - model.add(Reshape([1, 28, 28])) - model.add(SpatialConvolution(1, 6, 5, 5)) - model.add(Tanh()) - model.add(SpatialMaxPooling(2, 2, 2, 2)) - model.add(Tanh()) - model.add(SpatialConvolution(6, 12, 5, 5)) - model.add(SpatialMaxPooling(2, 2, 2, 2)) - model.add(Reshape([12 * 4 * 4])) - model.add(Linear(12 * 4 * 4, 100)) - model.add(Tanh()) - model.add(Linear(100, class_num)) - model.add(LogSoftMax()) - return model - ``` - -3). Create Optimizer and train: -``` - optimizer = Optimizer( - model=build_model(10), - training_rdd=train_data, - criterion=ClassNLLCriterion(), - optim_method="SGD", - state=state, - end_trigger=MaxEpoch(100), - batch_size=int(options.batchSize)) - optimizer.setvalidation( - batch_size=32, - val_rdd=test_data, - trigger=EveryEpoch(), - val_method=["top1"] - ) - optimizer.setcheckpoint(EveryEpoch(), "/tmp/lenet5/") - trained_model = optimizer.optimize() -``` - -4) LeNet example can be found from: models/lenet5.py - -## Run python test -* Package Scala code by: ```$BigDL_HOME/make-dist.sh``` -* Set SPARK_HOME and then run: ```$BigDL_HOME/pyspark/dev/run-all.sh``` - -## Installing on Ubuntu -1. Build BigDL -[Build Page](https://github.com/intel-analytics/BigDL/wiki/Build-Page) - * With Spark1.6: ``` $BIGDL_HOME/make-dist.sh ``` - * With Spark2.0: ``` $BIGDL_HOME/make-dist.sh -P spark_2.0 ``` - -2. Install python dependensies: - * Installing Numpy: - ```sudo apt-get install python-numpy``` - - * Installing Python setuptools: - ```sudo apt-get install -y python-setuptools python-pip``` - - * Install Jupyter: - ```sudo pip install jupyter``` - -## Run a Lenet example on standalone cluster - - ``` - BigDL_HOME=... - SPARK_HOME=... - MASTER=... - PYTHON_API_ZIP_PATH=${BigDL_HOME}/dist/lib/bigdl-0.1.0-SNAPSHOT-python-api.zip - BigDL_JAR_PATH=${BigDL_HOME}/dist/lib/bigdl-0.1.0-SNAPSHOT-jar-with-dependencies.jar - PYTHONPATH=${PYTHON_API_ZIP_PATH}:$PYTHONPATH - ${SPARK_HOME}/bin/spark-submit \ - --master ${MASTER} \ - --driver-cores 5 \ - --driver-memory 10g \ - --total-executor-cores 80 \ - --executor-cores 10 \ - --executor-memory 20g \ - --conf spark.akka.frameSize=64 \ - --py-files ${PYTHON_API_ZIP_PATH},${BigDL_HOME}/pyspark/bigdl/models/lenet/lenet5.py \ - --properties-file ${BigDL_HOME}/dist/conf/spark-bigdl.conf \ - --jars ${BigDL_JAR_PATH} \ - --conf spark.driver.extraClassPath=${BigDL_JAR_PATH} \ - --conf spark.executor.extraClassPath=bigdl-0.1.0-SNAPSHOT-jar-with-dependencies.jar \ - ${BigDL_HOME}/pyspark/bigdl/models/lenet/lenet5.py - ``` - - -## Launch Jupyter on standalone cluster - - ``` - BigDL_HOME=... - SPARK_HOME=... - MASTER=... - PYTHON_API_ZIP_PATH=${BigDL_HOME}/dist/lib/bigdl-0.1.0-SNAPSHOT-python-api.zip - BigDL_JAR_PATH=${BigDL_HOME}/dist/lib/bigdl-0.1.0-SNAPSHOT-jar-with-dependencies.jar - - export PYTHONPATH=${PYTHON_API_ZIP_PATH}:$PYTHONPATH - export IPYTHON_OPTS="notebook --notebook-dir=./ --ip=* --no-browser" - - ${SPARK_HOME}/bin/pyspark \ - --master ${MASTER} \ - --properties-file ${BigDL_HOME}/dist/conf/spark-bigdl.conf - --driver-cores 5 \ - --driver-memory 10g \ - --total-executor-cores 8 \ - --executor-cores 1 \ - --executor-memory 20g \ - --conf spark.akka.frameSize=64 \ - --py-files ${PYTHON_API_ZIP_PATH} \ - --jars ${BigDL_JAR_PATH} \ - --conf spark.driver.extraClassPath=${BigDL_JAR_PATH} \ - --conf spark.executor.extraClassPath=bigdl-0.1.0-SNAPSHOT-jar-with-dependencies.jar - ``` diff --git a/python/dllib/src/bigdl/dllib/feature/dataset/__init__.py b/python/dllib/src/bigdl/dllib/feature/dataset/__init__.py deleted file mode 100644 index 2151a805423..00000000000 --- a/python/dllib/src/bigdl/dllib/feature/dataset/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -# -# Copyright 2016 The BigDL Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# diff --git a/python/dllib/src/bigdl/dllib/feature/dataset/base.py b/python/dllib/src/bigdl/dllib/feature/dataset/base.py deleted file mode 100644 index f211ee5261c..00000000000 --- a/python/dllib/src/bigdl/dllib/feature/dataset/base.py +++ /dev/null @@ -1,197 +0,0 @@ -# -# Copyright 2016 The BigDL Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import os -import sys -import shutil -import tempfile -import time -from distutils.dir_util import mkpath -from six.moves.urllib.request import urlopen -import numpy as np - - -# Adopt from keras -class Progbar(object): - def __init__(self, target, width=30, verbose=1, interval=0.01): - ''' - @param target: total number of steps expected - @param interval: minimum visual progress update interval (in seconds) - ''' - self.width = width - self.target = target - self.sum_values = {} - self.unique_values = [] - self.start = time.time() - self.last_update = 0 - self.interval = interval - self.total_width = 0 - self.seen_so_far = 0 - self.verbose = verbose - - def update(self, current, values=[], force=False): - ''' - @param current: index of current step - @param values: list of tuples (name, value_for_last_step). - The progress bar will display averages for these values. - @param force: force visual progress update - ''' - for k, v in values: - if k not in self.sum_values: - self.sum_values[k] = [v * (current - self.seen_so_far), current - self.seen_so_far] - self.unique_values.append(k) - else: - self.sum_values[k][0] += v * (current - self.seen_so_far) - self.sum_values[k][1] += (current - self.seen_so_far) - self.seen_so_far = current - - now = time.time() - if self.verbose == 1: - if not force and (now - self.last_update) < self.interval: - return - - prev_total_width = self.total_width - sys.stdout.write("\b" * prev_total_width) - sys.stdout.write("\r") - - numdigits = int(np.floor(np.log10(self.target))) + 1 - barstr = '%%%dd/%%%dd [' % (numdigits, numdigits) - bar = barstr % (current, self.target) - prog = float(current) / self.target - prog_width = int(self.width * prog) - if prog_width > 0: - bar += ('=' * (prog_width-1)) - if current < self.target: - bar += '>' - else: - bar += '=' - bar += ('.' * (self.width - prog_width)) - bar += ']' - sys.stdout.write(bar) - self.total_width = len(bar) - - if current: - time_per_unit = (now - self.start) / current - else: - time_per_unit = 0 - eta = time_per_unit * (self.target - current) - info = '' - if current < self.target: - info += ' - ETA: %ds' % eta - else: - info += ' - %ds' % (now - self.start) - for k in self.unique_values: - info += ' - %s:' % k - if type(self.sum_values[k]) is list: - avg = self.sum_values[k][0] / max(1, self.sum_values[k][1]) - if abs(avg) > 1e-3: - info += ' %.4f' % avg - else: - info += ' %.4e' % avg - else: - info += ' %s' % self.sum_values[k] - - self.total_width += len(info) - if prev_total_width > self.total_width: - info += ((prev_total_width - self.total_width) * " ") - - sys.stdout.write(info) - sys.stdout.flush() - - if current >= self.target: - sys.stdout.write("\n") - - if self.verbose == 2: - if current >= self.target: - info = '%ds' % (now - self.start) - for k in self.unique_values: - info += ' - %s:' % k - avg = self.sum_values[k][0] / max(1, self.sum_values[k][1]) - if avg > 1e-3: - info += ' %.4f' % avg - else: - info += ' %.4e' % avg - sys.stdout.write(info + "\n") - - self.last_update = now - - def add(self, n, values=[]): - self.update(self.seen_so_far + n, values) - - -def display_table(rows, positions): - - def display_row(objects, positions): - line = '' - for i in range(len(objects)): - line += str(objects[i]) - line = line[:positions[i]] - line += ' ' * (positions[i] - len(line)) - print(line) - - for objects in rows: - display_row(objects, positions) - -# Adopt from keras -# Under Python 2, 'urlretrieve' relies on FancyURLopener from legacy -# urllib module, known to have issues with proxy management -if sys.version_info[0] == 2: - def urlretrieve(url, filename, reporthook=None, data=None): - def chunk_read(response, chunk_size=8192, reporthook=None): - total_size = response.info().get('Content-Length').strip() - total_size = int(total_size) - count = 0 - while 1: - chunk = response.read(chunk_size) - count += 1 - if not chunk: - reporthook(count, total_size, total_size) - break - if reporthook: - reporthook(count, chunk_size, total_size) - yield chunk - - response = urlopen(url, data) - with open(filename, 'wb') as fd: - for chunk in chunk_read(response, reporthook=reporthook): - fd.write(chunk) -else: - from six.moves.urllib.request import urlretrieve - - -def maybe_download(filename, work_directory, source_url): - if not os.path.exists(work_directory): - mkpath(work_directory) - filepath = os.path.join(work_directory, filename) - - if not os.path.exists(filepath): - print('Downloading data from', source_url) - global progbar - progbar = None - - def dl_progress(count, block_size, total_size): - global progbar - if progbar is None: - progbar = Progbar(total_size) - else: - progbar.update(count * block_size) - with tempfile.NamedTemporaryFile() as tmpfile: - temp_file_name = tmpfile.name - urlretrieve(source_url, temp_file_name, dl_progress) - shutil.copy2(temp_file_name, filepath) - size = os.path.getsize(filepath) - print('Successfully downloaded', filename, size, 'bytes.') - return filepath diff --git a/python/dllib/src/bigdl/dllib/feature/dataset/mnist.py b/python/dllib/src/bigdl/dllib/feature/dataset/mnist.py deleted file mode 100644 index 4a53735fce2..00000000000 --- a/python/dllib/src/bigdl/dllib/feature/dataset/mnist.py +++ /dev/null @@ -1,113 +0,0 @@ -# -# Copyright 2016 The BigDL Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# Part of the code originally from Tensorflow - - -import gzip - -import numpy - -import base - -SOURCE_URL = 'http://yann.lecun.com/exdb/mnist/' - -TRAIN_MEAN = 0.13066047740239506 -TRAIN_STD = 0.3081078 -TEST_MEAN = 0.13251460696903547 -TEST_STD = 0.31048024 - - -def _read32(bytestream): - dt = numpy.dtype(numpy.uint32).newbyteorder('>') - return numpy.frombuffer(bytestream.read(4), dtype=dt)[0] - - -def extract_images(f): - """Extract the images into a 4D uint8 numpy array [index, y, x, depth]. - - Args: - f: A file object that can be passed into a gzip reader. - - Returns: - data: A 4D unit8 numpy array [index, y, x, depth]. - - Raises: - ValueError: If the bytestream does not start with 2051. - - """ - print('Extracting', f.name) - with gzip.GzipFile(fileobj=f) as bytestream: - magic = _read32(bytestream) - if magic != 2051: - raise ValueError( - 'Invalid magic number %d in MNIST image file: %s' % - (magic, f.name)) - num_images = _read32(bytestream) - rows = _read32(bytestream) - cols = _read32(bytestream) - buf = bytestream.read(rows * cols * num_images) - data = numpy.frombuffer(buf, dtype=numpy.uint8) - data = data.reshape(num_images, rows, cols, 1) - return data - - -def extract_labels(f): - print('Extracting', f.name) - with gzip.GzipFile(fileobj=f) as bytestream: - magic = _read32(bytestream) - if magic != 2049: - raise ValueError( - 'Invalid magic number %d in MNIST label file: %s' % - (magic, f.name)) - num_items = _read32(bytestream) - buf = bytestream.read(num_items) - labels = numpy.frombuffer(buf, dtype=numpy.uint8) - return labels - - -def read_data_sets(train_dir, data_type="train"): - TRAIN_IMAGES = 'train-images-idx3-ubyte.gz' - TRAIN_LABELS = 'train-labels-idx1-ubyte.gz' - TEST_IMAGES = 't10k-images-idx3-ubyte.gz' - TEST_LABELS = 't10k-labels-idx1-ubyte.gz' - - if data_type == "train": - local_file = base.maybe_download(TRAIN_IMAGES, train_dir, - SOURCE_URL + TRAIN_IMAGES) - with open(local_file, 'rb') as f: - train_images = extract_images(f) - - local_file = base.maybe_download(TRAIN_LABELS, train_dir, - SOURCE_URL + TRAIN_LABELS) - with open(local_file, 'rb') as f: - train_labels = extract_labels(f) - return train_images, train_labels - - else: - local_file = base.maybe_download(TEST_IMAGES, train_dir, - SOURCE_URL + TEST_IMAGES) - with open(local_file, 'rb') as f: - test_images = extract_images(f) - - local_file = base.maybe_download(TEST_LABELS, train_dir, - SOURCE_URL + TEST_LABELS) - with open(local_file, 'rb') as f: - test_labels = extract_labels(f) - return test_images, test_labels - - -if __name__ == "__main__": - read_data_sets("/tmp/mnist/") diff --git a/python/dllib/src/bigdl/dllib/feature/dataset/news20.py b/python/dllib/src/bigdl/dllib/feature/dataset/news20.py deleted file mode 100644 index fdedf5a6c41..00000000000 --- a/python/dllib/src/bigdl/dllib/feature/dataset/news20.py +++ /dev/null @@ -1,73 +0,0 @@ -import tarfile -import base -import os -import sys - -NEWS20_URL = 'http://qwone.com/~jason/20Newsgroups/20news-19997.tar.gz' # noqa -GLOVE_URL = 'http://nlp.stanford.edu/data/glove.6B.zip' # noqa - -CLASS_NUM = 20 - - -def download_news20(dest_dir): - file_name = "20news-19997.tar.gz" - file_abs_path = base.maybe_download(file_name, dest_dir, NEWS20_URL) - tar = tarfile.open(file_abs_path, "r:gz") - extracted_to = os.path.join(dest_dir, "20_newsgroups") - if not os.path.exists(extracted_to): - print("Extracting %s to %s" % (file_abs_path, extracted_to)) - tar.extractall(dest_dir) - tar.close() - return extracted_to - - -def download_glove_w2v(dest_dir): - file_name = "glove.6B.zip" - file_abs_path = base.maybe_download(file_name, dest_dir, GLOVE_URL) - import zipfile - zip_ref = zipfile.ZipFile(file_abs_path, 'r') - extracted_to = os.path.join(dest_dir, "glove.6B") - if not os.path.exists(extracted_to): - print("Extracting %s to %s" % (file_abs_path, extracted_to)) - zip_ref.extractall(extracted_to) - zip_ref.close() - return extracted_to - - -# return [(text_content, label)] -def get_news20(source_dir="/tmp/news20/"): - news_dir = download_news20(source_dir) - texts = [] # list of text samples - label_id = 0 - for name in sorted(os.listdir(news_dir)): - path = os.path.join(news_dir, name) - label_id += 1 - if os.path.isdir(path): - for fname in sorted(os.listdir(path)): - if fname.isdigit(): - fpath = os.path.join(path, fname) - if sys.version_info < (3,): - f = open(fpath) - else: - f = open(fpath, encoding='latin-1') - content = f.read() - texts.append((content, label_id)) - f.close() - - print('Found %s texts.' % len(texts)) - return texts - - -def get_glove_w2v(source_dir="/tmp/news20/", dim=100): - w2v_dir = download_glove_w2v(source_dir) - with open(os.path.join(w2v_dir, "glove.6B.%sd.txt" % dim)) as w2v_f: - pre_w2v = {} - for line in w2v_f.readlines(): - items = line.split(" ") - pre_w2v[items[0]] = [float(i) for i in items[1:]] - return pre_w2v - - -if __name__ == "__main__": - get_news20("/tmp/news20/") - get_glove_w2v("/tmp/news20/") diff --git a/python/dllib/src/bigdl/dllib/feature/dataset/transformer.py b/python/dllib/src/bigdl/dllib/feature/dataset/transformer.py deleted file mode 100644 index 9b5d5930d3a..00000000000 --- a/python/dllib/src/bigdl/dllib/feature/dataset/transformer.py +++ /dev/null @@ -1,23 +0,0 @@ -# -# Copyright 2016 The BigDL Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - - -from util.common import Sample - - -def normalizer(mean, std): - return lambda sample: Sample.from_ndarray((sample.features - mean) / std, - sample.label, sample.bigdl_type) diff --git a/python/dllib/src/bigdl/dllib/models/__init__.py b/python/dllib/src/bigdl/dllib/models/__init__.py deleted file mode 100644 index 2151a805423..00000000000 --- a/python/dllib/src/bigdl/dllib/models/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -# -# Copyright 2016 The BigDL Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# diff --git a/python/dllib/src/bigdl/dllib/models/lenet/__init__.py b/python/dllib/src/bigdl/dllib/models/lenet/__init__.py deleted file mode 100644 index 2151a805423..00000000000 --- a/python/dllib/src/bigdl/dllib/models/lenet/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -# -# Copyright 2016 The BigDL Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# diff --git a/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py b/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py deleted file mode 100644 index f6b1a752b40..00000000000 --- a/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py +++ /dev/null @@ -1,97 +0,0 @@ -# -# Copyright 2016 The BigDL Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# Still in experimental stage! - -from optparse import OptionParser - -from dataset import mnist -from dataset.transformer import * -from nn.layer import * -from nn.criterion import * -from optim.optimizer import * -from util.common import * - - -def build_model(class_num): - model = Sequential() - model.add(Reshape([1, 28, 28])) - model.add(SpatialConvolution(1, 6, 5, 5)) - model.add(Tanh()) - model.add(SpatialMaxPooling(2, 2, 2, 2)) - model.add(Tanh()) - model.add(SpatialConvolution(6, 12, 5, 5)) - model.add(SpatialMaxPooling(2, 2, 2, 2)) - model.add(Reshape([12 * 4 * 4])) - model.add(Linear(12 * 4 * 4, 100)) - model.add(Tanh()) - model.add(Linear(100, class_num)) - model.add(LogSoftMax()) - return model - - -def get_minst(sc, data_type="train", location="/tmp/mnist"): - (images, labels) = mnist.read_data_sets(location, data_type) - images = sc.parallelize(images) - labels = sc.parallelize(labels) - # Target start from 1 in BigDL - record = images.zip(labels).map(lambda (features, label): - Sample.from_ndarray(features, label + 1)) - return record - - -if __name__ == "__main__": - - parser = OptionParser() - parser.add_option("-a", "--action", dest="action", default="train") - parser.add_option("-b", "--batchSize", dest="batchSize", default="128") - - (options, args) = parser.parse_args(sys.argv) - - sc = SparkContext(appName="lenet5", conf=create_spark_conf()) - init_engine() - - if options.action == "train": - train_data = get_minst(sc, "train").map( - normalizer(mnist.TRAIN_MEAN, mnist.TRAIN_STD)) - test_data = get_minst(sc, "test").map( - normalizer(mnist.TEST_MEAN, mnist.TEST_STD)) - state = {"learningRate": 0.01, - "learningRateDecay": 0.0002} - optimizer = Optimizer( - model=build_model(10), - training_rdd=train_data, - criterion=ClassNLLCriterion(), - optim_method="SGD", - state=state, - end_trigger=MaxEpoch(20), - batch_size=int(options.batchSize)) - optimizer.setvalidation( - batch_size=32, - val_rdd=test_data, - trigger=EveryEpoch(), - val_method=["Top1Accuracy"] - ) - optimizer.setcheckpoint(EveryEpoch(), "/tmp/lenet5/") - trained_model = optimizer.optimize() - parameters = trained_model.parameters() - elif options.action == "test": - test_data = get_minst("test").map( - normalizer(mnist.TEST_MEAN, mnist.TEST_STD)) - # TODO: Pass model path through external parameter - model = Model.from_path("/tmp/lenet5/lenet-model.470") - results = model.test(test_data, 32, ["Top1Accuracy"]) - for result in results: - print result diff --git a/python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py b/python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py deleted file mode 100644 index 2929290aa0a..00000000000 --- a/python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py +++ /dev/null @@ -1,177 +0,0 @@ -# -# Copyright 2016 The BigDL Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# Still in experimental stage! - -import itertools -import re -from optparse import OptionParser - -from dataset import news20 -from nn.layer import * -from nn.criterion import * -from optim.optimizer import * -from util.common import * -from util.common import Sample - - -def text_to_words(review_text): - letters_only = re.sub("[^a-zA-Z]", " ", review_text) - words = letters_only.lower().split() - return words - - -def analyze_texts(data_rdd): - return data_rdd.flatMap(lambda (text, label): text_to_words(text)) \ - .map(lambda word: (word, 1)).reduceByKey(lambda a, b: a + b) \ - .sortBy(lambda (w, c): - c).zipWithIndex() \ - .map(lambda ((w, c), i): (w, (i + 1, c))).collect() - - -# pad([1, 2, 3, 4, 5], 0, 6) -def pad(l, fill_value, width): - if len(l) >= width: - return l[0: width] - else: - l.extend([fill_value] * (width - len(l))) - return l - - -def to_vec(token, b_w2v, embedding_dim): - if token in b_w2v: - return b_w2v[token] - else: - return pad([], 0, embedding_dim) - - -def to_sample(vectors, label, embedding_dim): - # flatten nested list - flatten_features = list(itertools.chain(*vectors)) - features = np.array(flatten_features, dtype='float').reshape( - [sequence_len, embedding_dim]) - - if model_type.lower() == "cnn": - features = features.transpose(1, 0) - return Sample.from_ndarray(features, np.array(label)) - - -def build_model(class_num): - model = Sequential() - - if model_type.lower() == "cnn": - model.add(Reshape([embedding_dim, 1, sequence_len])) - model.add(SpatialConvolution(embedding_dim, 128, 5, 1)) - model.add(ReLU()) - model.add(SpatialMaxPooling(5, 1, 5, 1)) - model.add(SpatialConvolution(128, 128, 5, 1)) - model.add(ReLU()) - model.add(SpatialMaxPooling(5, 1, 5, 1)) - model.add(Reshape([128])) - elif model_type.lower() == "lstm": - model.add(Recurrent() - .add(LSTM(embedding_dim, 128))) - model.add(Select(2, -1)) - elif model_type.lower() == "gru": - model.add(Recurrent() - .add(GRU(embedding_dim, 128))) - model.add(Select(2, -1)) - else: - raise ValueError('model can only be cnn, lstm, or gru') - - model.add(Linear(128, 100)) - model.add(Linear(100, class_num)) - model.add(LogSoftMax()) - return model - - -def train(sc, - batch_size, - sequence_len, max_words, embedding_dim, training_split): - print('Processing text dataset') - texts = news20.get_news20() - data_rdd = sc.parallelize(texts, 2) - - word_to_ic = analyze_texts(data_rdd) - - # Only take the top wc between [10, sequence_len] - word_to_ic = dict(word_to_ic[10: max_words]) - bword_to_ic = sc.broadcast(word_to_ic) - - w2v = news20.get_glove_w2v(dim=embedding_dim) - filtered_w2v = {w: v for w, v in w2v.items() if w in word_to_ic} - bfiltered_w2v = sc.broadcast(filtered_w2v) - - tokens_rdd = data_rdd.map(lambda (text, label): - ([w for w in text_to_words(text) if - w in bword_to_ic.value], label)) - padded_tokens_rdd = tokens_rdd.map( - lambda (tokens, label): (pad(tokens, "##", sequence_len), label)) - vector_rdd = padded_tokens_rdd.map(lambda (tokens, label): - ([to_vec(w, bfiltered_w2v.value, - embedding_dim) for w in - tokens], label)) - sample_rdd = vector_rdd.map( - lambda (vectors, label): to_sample(vectors, label, embedding_dim)) - - train_rdd, val_rdd = sample_rdd.randomSplit( - [training_split, 1-training_split]) - - state = {"batchSize": batch_size, - "learningRate": 0.01, - "learningRateDecay": 0.0002} - - optimizer = Optimizer( - model=build_model(news20.CLASS_NUM), - training_rdd=train_rdd, - criterion=ClassNLLCriterion(), - end_trigger=MaxEpoch(max_epoch), - batch_size=batch_size, - optim_method="Adagrad", - state=state) - - optimizer.setvalidation( - batch_size=batch_size, - val_rdd=val_rdd, - trigger=EveryEpoch(), - val_method=["Top1Accuracy"] - ) - train_model = optimizer.optimize() - -if __name__ == "__main__": - parser = OptionParser() - parser.add_option("-a", "--action", dest="action", default="train") - parser.add_option("-b", "--batchSize", dest="batchSize", default="128") - parser.add_option("-e", "--embedding_dim", dest="embedding_dim", default="50") # noqa - parser.add_option("-m", "--max_epoch", dest="max_epoch", default="15") - parser.add_option("--model", dest="model_type", default="cnn") - - (options, args) = parser.parse_args(sys.argv) - if options.action == "train": - batch_size = int(options.batchSize) - embedding_dim = int(options.embedding_dim) - max_epoch = int(options.max_epoch) - model_type = options.model_type - sequence_len = 50 - max_words = 1000 - training_split = 0.8 - sc = SparkContext(appName="text_classifier", - conf=create_spark_conf()) - init_engine() - train(sc, - batch_size, - sequence_len, max_words, embedding_dim, training_split) - elif options.action == "test": - pass diff --git a/python/dllib/src/bigdl/dllib/nn/__init__.py b/python/dllib/src/bigdl/dllib/nn/__init__.py deleted file mode 100644 index 2151a805423..00000000000 --- a/python/dllib/src/bigdl/dllib/nn/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -# -# Copyright 2016 The BigDL Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# diff --git a/python/dllib/src/bigdl/dllib/nn/criterion.py b/python/dllib/src/bigdl/dllib/nn/criterion.py deleted file mode 100644 index 570653413ab..00000000000 --- a/python/dllib/src/bigdl/dllib/nn/criterion.py +++ /dev/null @@ -1,308 +0,0 @@ -# -# Copyright 2016 The BigDL Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - - -import sys -from util.common import callBigDlFunc -from util.common import JavaValue -from util.common import callJavaFunc -from pyspark import SparkContext - -import numpy as np - -if sys.version >= '3': - long = int - unicode = str - - -class Criterion(JavaValue): - - def __init__(self, jvalue, bigdl_type, *args): - self.value = jvalue if jvalue else callBigDlFunc( - bigdl_type, JavaValue.jvm_class_constructor(self), *args) - self.bigdl_type = bigdl_type - - @classmethod - def of(cls, jcriterion, bigdl_type="float"): - criterion = Criterion(bigdl_type, jcriterion) - criterion.value = jcriterion - criterion.bigdl_type = bigdl_type - return criterion - - -class ClassNLLCriterion(Criterion): - - ''' - >>> classNLLCriterion = ClassNLLCriterion() - creating: createClassNLLCriterion - ''' - - def __init__(self, - size_average=True, - bigdl_type="float"): - super(ClassNLLCriterion, self).__init__(None, bigdl_type, - size_average) - - -class MSECriterion(Criterion): - - ''' - >>> mSECriterion = MSECriterion() - creating: createMSECriterion - ''' - - def __init__(self, bigdl_type="float"): - super(MSECriterion, self).__init__(None, bigdl_type) - - -class AbsCriterion(Criterion): - - ''' - >>> absCriterion = AbsCriterion(True) - creating: createAbsCriterion - ''' - - def __init__(self, - size_average=True, - bigdl_type="float"): - super(AbsCriterion, self).__init__(None, bigdl_type, - size_average) - - -class ClassSimplexCriterion(Criterion): - - ''' - >>> classSimplexCriterion = ClassSimplexCriterion(2) - creating: createClassSimplexCriterion - ''' - - def __init__(self, - n_classes, - bigdl_type="float"): - super(ClassSimplexCriterion, self).__init__(None, bigdl_type, - n_classes) - - -class CosineEmbeddingCriterion(Criterion): - - ''' - >>> cosineEmbeddingCriterion = CosineEmbeddingCriterion(1e-5, True) - creating: createCosineEmbeddingCriterion - ''' - - def __init__(self, - margin=0.0, - size_average=True, - bigdl_type="float"): - super(CosineEmbeddingCriterion, self).__init__(None, bigdl_type, - margin, - size_average) - - -class DistKLDivCriterion(Criterion): - - ''' - >>> distKLDivCriterion = DistKLDivCriterion(True) - creating: createDistKLDivCriterion - ''' - - def __init__(self, - size_average=True, - bigdl_type="float"): - super(DistKLDivCriterion, self).__init__(None, bigdl_type, - size_average) - - -class HingeEmbeddingCriterion(Criterion): - - ''' - >>> hingeEmbeddingCriterion = HingeEmbeddingCriterion(1e-5, True) - creating: createHingeEmbeddingCriterion - ''' - - def __init__(self, - margin=1, - size_average=True, - bigdl_type="float"): - super(HingeEmbeddingCriterion, self).__init__(None, bigdl_type, - margin, - size_average) - - -class L1HingeEmbeddingCriterion(Criterion): - - ''' - >>> l1HingeEmbeddingCriterion = L1HingeEmbeddingCriterion(1e-5) - creating: createL1HingeEmbeddingCriterion - ''' - - def __init__(self, - margin=1, - bigdl_type="float"): - super(L1HingeEmbeddingCriterion, self).__init__(None, bigdl_type, - margin) - - -class MarginCriterion(Criterion): - - ''' - >>> marginCriterion = MarginCriterion(1e-5, True) - creating: createMarginCriterion - ''' - - def __init__(self, - margin=1.0, - size_average=True, - bigdl_type="float"): - super(MarginCriterion, self).__init__(None, bigdl_type, - margin, - size_average) - - -class MarginRankingCriterion(Criterion): - - ''' - >>> marginRankingCriterion = MarginRankingCriterion(1e-5, True) - creating: createMarginRankingCriterion - ''' - - def __init__(self, - margin=1.0, - size_average=True, - bigdl_type="float"): - super(MarginRankingCriterion, self).__init__(None, bigdl_type, - margin, - size_average) - - -class MultiCriterion(Criterion): - - ''' - >>> multiCriterion = MultiCriterion() - creating: createMultiCriterion - ''' - - def __init__(self, - bigdl_type="float"): - super(MultiCriterion, self).__init__(None, bigdl_type) - - -class MultiLabelMarginCriterion(Criterion): - - ''' - >>> multiLabelMarginCriterion = MultiLabelMarginCriterion(True) - creating: createMultiLabelMarginCriterion - ''' - - def __init__(self, - size_average=True, - bigdl_type="float"): - super(MultiLabelMarginCriterion, self).__init__(None, bigdl_type, - size_average) - - -class ParallelCriterion(Criterion): - - ''' - >>> parallelCriterion = ParallelCriterion(True) - creating: createParallelCriterion - ''' - - def __init__(self, - repeat_target=False, - bigdl_type="float"): - super(ParallelCriterion, self).__init__(None, bigdl_type, - repeat_target) - - -class SmoothL1Criterion(Criterion): - - ''' - >>> smoothL1Criterion = SmoothL1Criterion(True) - creating: createSmoothL1Criterion - ''' - - def __init__(self, - size_average=True, - bigdl_type="float"): - super(SmoothL1Criterion, self).__init__(None, bigdl_type, - size_average) - - -class SmoothL1CriterionWithWeights(Criterion): - - ''' - >>> smoothL1CriterionWithWeights = SmoothL1CriterionWithWeights(1e-5, 1) - creating: createSmoothL1CriterionWithWeights - ''' - - def __init__(self, - sigma, - num=0, - bigdl_type="float"): - super(SmoothL1CriterionWithWeights, self).__init__(None, bigdl_type, - sigma, - num) - - -class SoftmaxWithCriterion(Criterion): - - ''' - >>> softmaxWithCriterion = SoftmaxWithCriterion("VALID") - creating: createSoftmaxWithCriterion - ''' - - def __init__(self, - normalize_mode="VALID", - bigdl_type="float"): - super(SoftmaxWithCriterion, self).__init__(None, bigdl_type, - None, - normalize_mode) - - -class TimeDistributedCriterion(JavaValue): - - ''' - >>> td = TimeDistributedCriterion(ClassNLLCriterion()) - creating: createClassNLLCriterion - creating: createTimeDistributedCriterion - ''' - - def __init__(self, criterion, size_average=False, bigdl_type="float"): - super(TimeDistributedCriterion, self).__init__( - None, bigdl_type, criterion, size_average) - - -def _test(): - import doctest - from pyspark import SparkContext - from nn import criterion - from util.common import init_engine - from util.common import create_spark_conf - globs = criterion.__dict__.copy() - sc = SparkContext(master="local[4]", appName="test criterion", - conf=create_spark_conf()) - globs['sc'] = sc - init_engine() - - (failure_count, test_count) = doctest.testmod(globs=globs, - optionflags=doctest.ELLIPSIS) - if failure_count: - exit(-1) - - -if __name__ == "__main__": - _test() diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py deleted file mode 100644 index 48ace84dd7a..00000000000 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ /dev/null @@ -1,1673 +0,0 @@ -# -# Copyright 2016 The BigDL Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - - -import sys -from util.common import callBigDlFunc -from util.common import JavaValue -from util.common import callJavaFunc -from pyspark import SparkContext - -import numpy as np - -if sys.version >= '3': - long = int - unicode = str - -INTMAX = 2147483647 -INTMIN = -2147483648 -DOUBLEMAX = 1.7976931348623157E308 - - -class Model(JavaValue): - - def __init__(self, jvalue, bigdl_type, *args): - self.value = jvalue if jvalue else callBigDlFunc( - bigdl_type, JavaValue.jvm_class_constructor(self), *args) - self.bigdl_type = bigdl_type - - @classmethod - def of(cls, jmodel, bigdl_type="float"): - model = Model(bigdl_type, jmodel) - model.value = jmodel - model.bigdl_type = bigdl_type - return model - - def set_name(self, name): - callJavaFunc(SparkContext.getOrCreate(), self.value.setName, name) - return self - - def name(self): - return callJavaFunc(SparkContext.getOrCreate(), self.value.getName) - - def set_seed(self, seed=123): - callBigDlFunc(self.bigdl_type, "setModelSeed", seed) - return self - - def get_dtype(self): - if "float" == self.bigdl_type: - return "float32" - else: - return "float64" - - def reset(self): - callJavaFunc(SparkContext.getOrCreate(), self.value.reset) - return self - - def parameters(self): - name_to_params = callBigDlFunc(self.bigdl_type, - "modelGetParameters", - self.value) - - def to_ndarray(params): - return { - param_name: np.array(values[0], - dtype=self.get_dtype()).reshape( - values[1]) for param_name, values in params.iteritems()} - - return {layer_name: to_ndarray(params) for layer_name, params in - name_to_params.iteritems()} - - def predict(self, data_rdd): - return callBigDlFunc(self.bigdl_type, - "modelPredictRDD", self.value, data_rdd) - - def test(self, val_rdd, batch_size, val_methods): - return callBigDlFunc(self.bigdl_type, - "modelTest", - self.value, - val_rdd, batch_size, val_methods) - - @staticmethod - def from_path(path, bigdl_type="float"): - jmodel = callBigDlFunc(bigdl_type, "modelFromPath", path) - return Model.of(jmodel, bigdl_type) - - -class Linear(Model): - - ''' - >>> linear = Linear(100, 10, "Xavier") - creating: createLinear - ''' - - def __init__(self, input_size, output_size, init_method="default", - bigdl_type="float"): - super(Linear, self).__init__(None, bigdl_type, input_size, output_size, - init_method) - - -class ReLU(Model): - - ''' - >>> relu = ReLU() - creating: createReLU - ''' - - def __init__(self, bigdl_type="float"): - super(ReLU, self).__init__(None, bigdl_type) - - -class Tanh(Model): - - ''' - >>> tanh = Tanh() - creating: createTanh - ''' - - def __init__(self, bigdl_type="float"): - super(Tanh, self).__init__(None, bigdl_type) - - -class Echo(Model): - - ''' - >>> echo = Echo() - creating: createEcho - ''' - - def __init__(self, bigdl_type="float"): - super(Echo, self).__init__(None, bigdl_type) - - -class LogSoftMax(Model): - - ''' - >>> logSoftMax = LogSoftMax() - creating: createLogSoftMax - ''' - - def __init__(self, bigdl_type="float"): - super(LogSoftMax, self).__init__(None, bigdl_type) - - -class Sequential(Model): - - ''' - >>> echo = Echo() - creating: createEcho - >>> s = Sequential() - creating: createSequential - >>> s = s.add(echo) - >>> s = s.add(s) - >>> s = s.add(echo) - - ''' - - def __init__(self, bigdl_type="float"): - super(Sequential, self).__init__(None, bigdl_type) - - def add(self, model): - self.value.add(model.value) - return self - - -class SpatialConvolution(Model): - - ''' - >>> spatialConvolution = SpatialConvolution(6, 12, 5, 5) - creating: createSpatialConvolution - ''' - - def __init__(self, - n_input_plane, - n_output_plane, - kernel_w, - kernel_h, - stride_w=1, - stride_h=1, - pad_w=0, - pad_h=0, - n_group=1, - propagate_back=True, - init_method="Default", - bigdl_type="float"): - super(SpatialConvolution, self).__init__(None, bigdl_type, - n_input_plane, - n_output_plane, - kernel_w, - kernel_h, - stride_w, - stride_h, - pad_w, - pad_h, - n_group, - propagate_back, - init_method) - - -class SpatialMaxPooling(Model): - - ''' - >>> spatialMaxPooling = SpatialMaxPooling(2, 2, 2, 2) - creating: createSpatialMaxPooling - ''' - # to_ceil: call floor() when False; call ceil() when True - - def __init__(self, kw, - kh, - dw, - dh, - pad_w=0, - pad_h=0, - to_ceil=False, - bigdl_type="float"): - super(SpatialMaxPooling, self).__init__(None, bigdl_type, kw, - kh, - dw, - dh, - pad_w, - pad_h, - to_ceil) - - -class Select(Model): - ''' - >>> select = Select(1, 1) - creating: createSelect - ''' - - def __init__(self, dim, index, bigdl_type="float"): - super(Select, self).__init__(None, bigdl_type, dim, index) - - -class Recurrent(Model): - ''' - >>> recurrent = Recurrent() - creating: createRecurrent - ''' - - def __init__(self, bigdl_type="float"): - super(Recurrent, self).__init__(None, bigdl_type) - - def add(self, model): - self.value.add(model.value) - return self - - -class LSTM(Model): - ''' - >>> lstm = LSTM(4, 3) - creating: createLSTM - ''' - - def __init__(self, input_size, hidden_size, bigdl_type="float"): - super(LSTM, self).__init__(None, bigdl_type, input_size, hidden_size) - - -class GRU(Model): - ''' - >>> gru = GRU(4, 3) - creating: createGRU - ''' - - def __init__(self, input_size, hidden_size, bigdl_type="float"): - super(GRU, self).__init__(None, bigdl_type, input_size, hidden_size) - - -class RNNCell(Model): - ''' - >>> reshape = RNNCell(4, 3, Tanh()) - creating: createTanh - creating: createRNNCell - ''' - - def __init__(self, - input_size, - hidden_size, - activation, - bigdl_type="float"): - super(RNNCell, self).__init__(None, bigdl_type, input_size, hidden_size, activation) - - -class TimeDistributed(Model): - ''' - >>> td = TimeDistributed(Linear(2, 3)) - creating: createLinear - creating: createTimeDistributed - ''' - - def __init__(self, model, bigdl_type="float"): - super(TimeDistributed, self).__init__(None, bigdl_type, model) - - -class Concat(Model): - - ''' - >>> concat = Concat(2) - creating: createConcat - ''' - - def __init__(self, - dimension, - bigdl_type="float"): - super(Concat, self).__init__(None, bigdl_type, - dimension) - - -class SpatialAveragePooling(Model): - - ''' - >>> spatialAveragePooling = SpatialAveragePooling(7,7) - creating: createSpatialAveragePooling - ''' - - def __init__(self, - kw, - kh, - dw=1, - dh=1, - pad_w=0, - pad_h=0, - ceil_mode=False, - count_include_pad=True, - divide=True, - bigdl_type="float"): - super(SpatialAveragePooling, self).__init__(None, bigdl_type, - kw, - kh, - dw, - dh, - pad_w, - pad_h, - ceil_mode, - count_include_pad, - divide) - - -class SpatialBatchNormalization(Model): - - ''' - >>> spatialBatchNormalization = SpatialBatchNormalization(1) - creating: createSpatialBatchNormalization - ''' - - def __init__(self, - n_output, - eps=1e-5, - momentum=0.1, - affine=True, - bigdl_type="float"): - super(SpatialBatchNormalization, self).__init__(None, bigdl_type, - n_output, - eps, - momentum, - affine) - - -class SpatialCrossMapLRN(Model): - - ''' - >>> spatialCrossMapLRN = SpatialCrossMapLRN() - creating: createSpatialCrossMapLRN - ''' - - def __init__(self, - size=5, - alpha=1.0, - beta=0.75, - k=1.0, - bigdl_type="float"): - super(SpatialCrossMapLRN, self).__init__(None, bigdl_type, - size, - alpha, - beta, - k) - - -class Dropout(Model): - - ''' - >>> dropout = Dropout(0.4) - creating: createDropout - ''' - - def __init__(self, - init_p=0.5, - inplace=False, - scale=True, - bigdl_type="float"): - super(Dropout, self).__init__(None, bigdl_type, - init_p, - inplace, - scale) - - -class View(Model): - - ''' - >>> view = View([1024,2]) - creating: createView - ''' - - def __init__(self, - sizes, - num_input_dims=0, - bigdl_type="float"): - super(View, self).__init__(None, bigdl_type, - sizes, - num_input_dims) - - -class Abs(Model): - - ''' - >>> abs = Abs() - creating: createAbs - ''' - - def __init__(self, - bigdl_type="float"): - super(Abs, self).__init__(None, bigdl_type) - - -class Add(Model): - - ''' - >>> add = Add(1) - creating: createAdd - ''' - - def __init__(self, - input_size, - bigdl_type="float"): - super(Add, self).__init__(None, bigdl_type, - input_size) - - -class AddConstant(Model): - - ''' - >>> addConstant = AddConstant(1e-5, True) - creating: createAddConstant - ''' - - def __init__(self, - constant_scalar, - inplace=False, - bigdl_type="float"): - super(AddConstant, self).__init__(None, bigdl_type, - constant_scalar, - inplace) - - -class BatchNormalization(Model): - - ''' - >>> batchNormalization = BatchNormalization(1, 1e-5, 1e-5, True) - creating: createBatchNormalization - ''' - - def __init__(self, - n_output, - eps=1e-5, - momentum=0.1, - affine=True, - bigdl_type="float"): - super(BatchNormalization, self).__init__(None, bigdl_type, - n_output, - eps, - momentum, - affine) - - -class Bilinear(Model): - - ''' - >>> bilinear = Bilinear(1, 1, 1, True) - creating: createBilinear - ''' - - def __init__(self, - input_size1, - input_size2, - output_size, - bias_res=True, - bigdl_type="float"): - super(Bilinear, self).__init__(None, bigdl_type, - input_size1, - input_size2, - output_size, - bias_res) - - -class Bottle(Model): - - ''' - >>> bottle = Bottle(Linear(100,10), 1, 1) - creating: createLinear - creating: createBottle - ''' - - def __init__(self, - module, - n_input_dim=2, - n_output_dim1=INTMAX, - bigdl_type="float"): - super(Bottle, self).__init__(None, bigdl_type, - module, - n_input_dim, - n_output_dim1) - - -class CAdd(Model): - - ''' - >>> cAdd = CAdd([1,2]) - creating: createCAdd - ''' - - def __init__(self, - size, - bigdl_type="float"): - super(CAdd, self).__init__(None, bigdl_type, - size) - - -class CAddTable(Model): - - ''' - >>> cAddTable = CAddTable(True) - creating: createCAddTable - ''' - - def __init__(self, - inplace=False, - bigdl_type="float"): - super(CAddTable, self).__init__(None, bigdl_type, - inplace) - - -class CDivTable(Model): - - ''' - >>> cDivTable = CDivTable() - creating: createCDivTable - ''' - - def __init__(self, - bigdl_type="float"): - super(CDivTable, self).__init__(None, bigdl_type) - - -class CMaxTable(Model): - - ''' - >>> cMaxTable = CMaxTable() - creating: createCMaxTable - ''' - - def __init__(self, - bigdl_type="float"): - super(CMaxTable, self).__init__(None, bigdl_type) - - -class CMinTable(Model): - - ''' - >>> cMinTable = CMinTable() - creating: createCMinTable - ''' - - def __init__(self, - bigdl_type="float"): - super(CMinTable, self).__init__(None, bigdl_type) - - -class CMul(Model): - - ''' - >>> cMul = CMul([1,2]) - creating: createCMul - ''' - - def __init__(self, - size, - bigdl_type="float"): - super(CMul, self).__init__(None, bigdl_type, - size) - - -class CMulTable(Model): - - ''' - >>> cMulTable = CMulTable() - creating: createCMulTable - ''' - - def __init__(self, - bigdl_type="float"): - super(CMulTable, self).__init__(None, bigdl_type) - - -class CSubTable(Model): - - ''' - >>> cSubTable = CSubTable() - creating: createCSubTable - ''' - - def __init__(self, - bigdl_type="float"): - super(CSubTable, self).__init__(None, bigdl_type) - - -class Clamp(Model): - - ''' - >>> clamp = Clamp(1, 3) - creating: createClamp - ''' - - def __init__(self, - min, - max, - bigdl_type="float"): - super(Clamp, self).__init__(None, bigdl_type, - min, - max) - - -class Contiguous(Model): - - ''' - >>> contiguous = Contiguous() - creating: createContiguous - ''' - - def __init__(self, - bigdl_type="float"): - super(Contiguous, self).__init__(None, bigdl_type) - - -class Copy(Model): - - ''' - >>> copy = Copy() - creating: createCopy - ''' - - def __init__(self, - bigdl_type="float"): - super(Copy, self).__init__(None, bigdl_type) - - -class Cosine(Model): - - ''' - >>> cosine = Cosine(2,3) - creating: createCosine - ''' - - def __init__(self, - input_size, - output_size, - bigdl_type="float"): - super(Cosine, self).__init__(None, bigdl_type, - input_size, - output_size) - - -class CosineDistance(Model): - - ''' - >>> cosineDistance = CosineDistance() - creating: createCosineDistance - ''' - - def __init__(self, - bigdl_type="float"): - super(CosineDistance, self).__init__(None, bigdl_type) - - -class DotProduct(Model): - - ''' - >>> dotProduct = DotProduct() - creating: createDotProduct - ''' - - def __init__(self, - bigdl_type="float"): - super(DotProduct, self).__init__(None, bigdl_type) - - -class ELU(Model): - - ''' - >>> eLU = ELU(1e-5, True) - creating: createELU - ''' - - def __init__(self, - alpha=1.0, - inplace=False, - bigdl_type="float"): - super(ELU, self).__init__(None, bigdl_type, - alpha, - inplace) - - -class Euclidean(Model): - - ''' - >>> euclidean = Euclidean(1, 1, True) - creating: createEuclidean - ''' - - def __init__(self, - input_size, - output_size, - fast_backward=True, - bigdl_type="float"): - super(Euclidean, self).__init__(None, bigdl_type, - input_size, - output_size, - fast_backward) - - -class Exp(Model): - - ''' - >>> exp = Exp() - creating: createExp - ''' - - def __init__(self, - bigdl_type="float"): - super(Exp, self).__init__(None, bigdl_type) - - -class FlattenTable(Model): - - ''' - >>> flattenTable = FlattenTable() - creating: createFlattenTable - ''' - - def __init__(self, - bigdl_type="float"): - super(FlattenTable, self).__init__(None, bigdl_type) - - -class GradientReversal(Model): - - ''' - >>> gradientReversal = GradientReversal(1e-5) - creating: createGradientReversal - ''' - - def __init__(self, - the_lambda=1, - bigdl_type="float"): - super(GradientReversal, self).__init__(None, bigdl_type, - the_lambda) - - -class HardShrink(Model): - - ''' - >>> hardShrink = HardShrink(1e-5) - creating: createHardShrink - ''' - - def __init__(self, - the_lambda=0.5, - bigdl_type="float"): - super(HardShrink, self).__init__(None, bigdl_type, - the_lambda) - - -class HardTanh(Model): - - ''' - >>> hardTanh = HardTanh(1e-5, 1e5, True) - creating: createHardTanh - ''' - - def __init__(self, - min_value=-1, - max_value=1, - inplace=False, - bigdl_type="float"): - super(HardTanh, self).__init__(None, bigdl_type, - min_value, - max_value, - inplace) - - -class Index(Model): - - ''' - >>> index = Index(1) - creating: createIndex - ''' - - def __init__(self, - dimension, - bigdl_type="float"): - super(Index, self).__init__(None, bigdl_type, - dimension) - - -class InferReshape(Model): - - ''' - >>> inferReshape = InferReshape([4, 0, 3, -1], False) - creating: createInferReshape - ''' - - def __init__(self, - size, - batch_mode=False, - bigdl_type="float"): - super(InferReshape, self).__init__(None, bigdl_type, - size, - batch_mode) - - -class JoinTable(Model): - - ''' - >>> joinTable = JoinTable(1, 1) - creating: createJoinTable - ''' - - def __init__(self, - dimension, - n_input_dims, - bigdl_type="float"): - super(JoinTable, self).__init__(None, bigdl_type, - dimension, - n_input_dims) - - -class L1Cost(Model): - - ''' - >>> l1Cost = L1Cost() - creating: createL1Cost - ''' - - def __init__(self, - bigdl_type="float"): - super(L1Cost, self).__init__(None, bigdl_type) - - -class L1Penalty(Model): - - ''' - >>> l1Penalty = L1Penalty(1, True, True) - creating: createL1Penalty - ''' - - def __init__(self, - l1weight, - size_average=False, - provide_output=True, - bigdl_type="float"): - super(L1Penalty, self).__init__(None, bigdl_type, - l1weight, - size_average, - provide_output) - - -class LeakyReLU(Model): - - ''' - >>> leakyReLU = LeakyReLU(1e-5, True) - creating: createLeakyReLU - ''' - - def __init__(self, - negval=0.01, - inplace=False, - bigdl_type="float"): - super(LeakyReLU, self).__init__(None, bigdl_type, - negval, - inplace) - - -class Log(Model): - - ''' - >>> log = Log() - creating: createLog - ''' - - def __init__(self, - bigdl_type="float"): - super(Log, self).__init__(None, bigdl_type) - - -class LogSigmoid(Model): - - ''' - >>> logSigmoid = LogSigmoid() - creating: createLogSigmoid - ''' - - def __init__(self, - bigdl_type="float"): - super(LogSigmoid, self).__init__(None, bigdl_type) - - -class LookupTable(Model): - - ''' - >>> lookupTable = LookupTable(1, 1, 1e-5, 1e-5, 1e-5, True) - creating: createLookupTable - ''' - - def __init__(self, - n_index, - n_output, - padding_value=0, - max_norm=DOUBLEMAX, - norm_type=2.0, - should_scale_grad_by_freq=False, - bigdl_type="float"): - super(LookupTable, self).__init__(None, bigdl_type, - n_index, - n_output, - padding_value, - max_norm, - norm_type, - should_scale_grad_by_freq) - - -class MM(Model): - - ''' - >>> mM = MM(True, True) - creating: createMM - ''' - - def __init__(self, - trans_a=False, - trans_b=False, - bigdl_type="float"): - super(MM, self).__init__(None, bigdl_type, - trans_a, - trans_b) - - -class MV(Model): - - ''' - >>> mV = MV(True) - creating: createMV - ''' - - def __init__(self, - trans=False, - bigdl_type="float"): - super(MV, self).__init__(None, bigdl_type, - trans) - - -class MapTable(Model): - - ''' - >>> mapTable = MapTable(Linear(100,10)) - creating: createLinear - creating: createMapTable - ''' - - def __init__(self, - module, - bigdl_type="float"): - super(MapTable, self).__init__(None, bigdl_type, - module) - - -class MaskedSelect(Model): - - ''' - >>> maskedSelect = MaskedSelect() - creating: createMaskedSelect - ''' - - def __init__(self, - bigdl_type="float"): - super(MaskedSelect, self).__init__(None, bigdl_type) - - -class Max(Model): - - ''' - >>> max = Max(1) - creating: createMax - ''' - - def __init__(self, - dim, - num_input_dims=INTMIN, - bigdl_type="float"): - super(Max, self).__init__(None, bigdl_type, - dim, - num_input_dims) - - -class Mean(Model): - - ''' - >>> mean = Mean(1, 1) - creating: createMean - ''' - - def __init__(self, - dimension=1, - n_input_dims=-1, - bigdl_type="float"): - super(Mean, self).__init__(None, bigdl_type, - dimension, - n_input_dims) - - -class Min(Model): - - ''' - >>> min = Min(1) - creating: createMin - ''' - - def __init__(self, - dim, - num_input_dims=INTMIN, - bigdl_type="float"): - super(Min, self).__init__(None, bigdl_type, - dim, - num_input_dims) - - -class MixtureTable(Model): - - ''' - >>> mixtureTable = MixtureTable() - creating: createMixtureTable - ''' - - def __init__(self, - bigdl_type="float"): - super(MixtureTable, self).__init__(None, bigdl_type) - - -class Mul(Model): - - ''' - >>> mul = Mul() - creating: createMul - ''' - - def __init__(self, - bigdl_type="float"): - super(Mul, self).__init__(None, bigdl_type) - - -class MulConstant(Model): - - ''' - >>> mulConstant = MulConstant(2.5) - creating: createMulConstant - ''' - - def __init__(self, - scalar, - inplace=False, - bigdl_type="float"): - super(MulConstant, self).__init__(None, bigdl_type, - scalar, - inplace) - - -class Narrow(Model): - - ''' - >>> narrow = Narrow(1, 1, 1) - creating: createNarrow - ''' - - def __init__(self, - dimension, - offset, - length=1, - bigdl_type="float"): - super(Narrow, self).__init__(None, bigdl_type, - dimension, - offset, - length) - - -class NarrowTable(Model): - - ''' - >>> narrowTable = NarrowTable(1, 1) - creating: createNarrowTable - ''' - - def __init__(self, - offset, - length=1, - bigdl_type="float"): - super(NarrowTable, self).__init__(None, bigdl_type, - offset, - length) - - -class Normalize(Model): - - ''' - >>> normalize = Normalize(1e-5, 1e-5) - creating: createNormalize - ''' - - def __init__(self, - p, - eps=1e-10, - bigdl_type="float"): - super(Normalize, self).__init__(None, bigdl_type, - p, - eps) - - -class PReLU(Model): - - ''' - >>> pReLU = PReLU(1) - creating: createPReLU - ''' - - def __init__(self, - n_output_plane=0, - bigdl_type="float"): - super(PReLU, self).__init__(None, bigdl_type, - n_output_plane) - - -class Padding(Model): - - ''' - >>> padding = Padding(1, 1, 1, 1e-5, 1) - creating: createPadding - ''' - - def __init__(self, - dim, - pad, - n_input_dim, - value=0.0, - n_index=1, - bigdl_type="float"): - super(Padding, self).__init__(None, bigdl_type, - dim, - pad, - n_input_dim, - value, - n_index) - - -class PairwiseDistance(Model): - - ''' - >>> pairwiseDistance = PairwiseDistance(2) - creating: createPairwiseDistance - ''' - - def __init__(self, - norm=2, - bigdl_type="float"): - super(PairwiseDistance, self).__init__(None, bigdl_type, - norm) - - -class ParallelTable(Model): - - ''' - >>> parallelTable = ParallelTable() - creating: createParallelTable - ''' - - def __init__(self, - bigdl_type="float"): - super(ParallelTable, self).__init__(None, bigdl_type) - - -class Power(Model): - - ''' - >>> power = Power(1e-5) - creating: createPower - ''' - - def __init__(self, - power, - scale=1.0, - shift=0.0, - bigdl_type="float"): - super(Power, self).__init__(None, bigdl_type, - power, - scale, - shift) - - -class RReLU(Model): - - ''' - >>> rReLU = RReLU(1e-5, 1e5, True) - creating: createRReLU - ''' - - def __init__(self, - lower=1.0/8, - upper=1.0/3, - inplace=False, - bigdl_type="float"): - super(RReLU, self).__init__(None, bigdl_type, - lower, - upper, - inplace) - - -class ReLU6(Model): - - ''' - >>> reLU6 = ReLU6(True) - creating: createReLU6 - ''' - - def __init__(self, - inplace=False, - bigdl_type="float"): - super(ReLU6, self).__init__(None, bigdl_type, - inplace) - - -class Replicate(Model): - - ''' - >>> replicate = Replicate(2) - creating: createReplicate - ''' - - def __init__(self, - n_features, - dim=1, - n_dim=INTMAX, - bigdl_type="float"): - super(Replicate, self).__init__(None, bigdl_type, - n_features, - dim, - n_dim) - - -class RoiPooling(Model): - - ''' - >>> roiPooling = RoiPooling(1, 1, 1e-5) - creating: createRoiPooling - ''' - - def __init__(self, - pooled_w, - pooled_h, - spatial_scale, - bigdl_type="float"): - super(RoiPooling, self).__init__(None, bigdl_type, - pooled_w, - pooled_h, - spatial_scale) - - -class Scale(Model): - - ''' - >>> scale = Scale([1,2]) - creating: createScale - ''' - - def __init__(self, - size, - bigdl_type="float"): - super(Scale, self).__init__(None, bigdl_type, - size) - - -class SelectTable(Model): - - ''' - >>> selectTable = SelectTable(1) - creating: createSelectTable - ''' - - def __init__(self, - dimension, - bigdl_type="float"): - super(SelectTable, self).__init__(None, bigdl_type, - dimension) - - -class Sigmoid(Model): - - ''' - >>> sigmoid = Sigmoid() - creating: createSigmoid - ''' - - def __init__(self, - bigdl_type="float"): - super(Sigmoid, self).__init__(None, bigdl_type) - - -class SoftMax(Model): - - ''' - >>> softMax = SoftMax() - creating: createSoftMax - ''' - - def __init__(self, - bigdl_type="float"): - super(SoftMax, self).__init__(None, bigdl_type) - - -class SoftMin(Model): - - ''' - >>> softMin = SoftMin() - creating: createSoftMin - ''' - - def __init__(self, - bigdl_type="float"): - super(SoftMin, self).__init__(None, bigdl_type) - - -class SoftPlus(Model): - - ''' - >>> softPlus = SoftPlus(1e-5) - creating: createSoftPlus - ''' - - def __init__(self, - beta=1.0, - bigdl_type="float"): - super(SoftPlus, self).__init__(None, bigdl_type, - beta) - - -class SoftShrink(Model): - - ''' - >>> softShrink = SoftShrink(1e-5) - creating: createSoftShrink - ''' - - def __init__(self, - the_lambda=0.5, - bigdl_type="float"): - super(SoftShrink, self).__init__(None, bigdl_type, - the_lambda) - - -class SoftSign(Model): - - ''' - >>> softSign = SoftSign() - creating: createSoftSign - ''' - - def __init__(self, - bigdl_type="float"): - super(SoftSign, self).__init__(None, bigdl_type) - - -class SpatialDilatedConvolution(Model): - - ''' - >>> spatialDilatedConvolution = SpatialDilatedConvolution(1, 1, 1, 1) - creating: createSpatialDilatedConvolution - ''' - - def __init__(self, - n_input_plane, - n_output_plane, - kw, - kh, - dw=1, - dh=1, - pad_w=0, - pad_h=0, - dilation_w=1, - dilation_h=1, - init_method='default', - bigdl_type="float"): - super(SpatialDilatedConvolution, self).__init__(None, bigdl_type, - n_input_plane, - n_output_plane, - kw, - kh, - dw, - dh, - pad_w, - pad_h, - dilation_w, - dilation_h, - init_method) - - -class SpatialFullConvolution(Model): - - ''' - >>> spatialFullConvolution = SpatialFullConvolution(1, 1, 1, 1) - creating: createSpatialFullConvolution - ''' - - def __init__(self, - n_input_plane, - n_output_plane, - kw, - kh, - dw=1, - dh=1, - pad_w=0, - pad_h=0, - adj_w=0, - adj_h=0, - n_group=1, - no_bias=False, - init_method='default', - bigdl_type="float"): - super(SpatialFullConvolution, self).__init__(None, bigdl_type, - n_input_plane, - n_output_plane, - kw, - kh, - dw, - dh, - pad_w, - pad_h, - adj_w, - adj_h, - n_group, - no_bias, - init_method) - - -class SpatialShareConvolution(Model): - - ''' - >>> spatialShareConvolution = SpatialShareConvolution(1, 1, 1, 1) - creating: createSpatialShareConvolution - ''' - - def __init__(self, - n_input_plane, - n_output_plane, - kernel_w, - kernel_h, - stride_w=1, - stride_h=1, - pad_w=0, - pad_h=0, - n_group=1, - propagate_back=True, - init_method='default', - bigdl_type="float"): - super(SpatialShareConvolution, self).__init__(None, bigdl_type, - n_input_plane, - n_output_plane, - kernel_w, - kernel_h, - stride_w, - stride_h, - pad_w, - pad_h, - n_group, - propagate_back, - init_method) - - -class SpatialZeroPadding(Model): - - ''' - >>> spatialZeroPadding = SpatialZeroPadding(1, 1, 1, 1) - creating: createSpatialZeroPadding - ''' - - def __init__(self, - pad_left, - pad_right, - pad_top, - pad_bottom, - bigdl_type="float"): - super(SpatialZeroPadding, self).__init__(None, bigdl_type, - pad_left, - pad_right, - pad_top, - pad_bottom) - - -class SplitTable(Model): - - ''' - >>> splitTable = SplitTable(1, 1) - creating: createSplitTable - ''' - - def __init__(self, - dimension, - n_input_dims=-1, - bigdl_type="float"): - super(SplitTable, self).__init__(None, bigdl_type, - dimension, - n_input_dims) - - -class Sqrt(Model): - - ''' - >>> sqrt = Sqrt() - creating: createSqrt - ''' - - def __init__(self, - bigdl_type="float"): - super(Sqrt, self).__init__(None, bigdl_type) - - -class Square(Model): - - ''' - >>> square = Square() - creating: createSquare - ''' - - def __init__(self, - bigdl_type="float"): - super(Square, self).__init__(None, bigdl_type) - - -class Squeeze(Model): - - ''' - >>> squeeze = Squeeze(1) - creating: createSqueeze - ''' - - def __init__(self, - dim, - num_input_dims=INTMIN, - bigdl_type="float"): - super(Squeeze, self).__init__(None, bigdl_type, - dim, - num_input_dims) - - -class Sum(Model): - - ''' - >>> sum = Sum(1, 1, True) - creating: createSum - ''' - - def __init__(self, - dimension=1, - n_input_dims=-1, - size_average=False, - bigdl_type="float"): - super(Sum, self).__init__(None, bigdl_type, - dimension, - n_input_dims, - size_average) - - -class TanhShrink(Model): - - ''' - >>> tanhShrink = TanhShrink() - creating: createTanhShrink - ''' - - def __init__(self, - bigdl_type="float"): - super(TanhShrink, self).__init__(None, bigdl_type) - - -class Threshold(Model): - - ''' - >>> threshold = Threshold(1e-5, 1e-5, True) - creating: createThreshold - ''' - - def __init__(self, - th=1e-6, - v=0.0, - ip=False, - bigdl_type="float"): - super(Threshold, self).__init__(None, bigdl_type, - th, - v, - ip) - - -class Unsqueeze(Model): - - ''' - >>> unsqueeze = Unsqueeze(1, 1) - creating: createUnsqueeze - ''' - - def __init__(self, - pos, - num_input_dims=INTMIN, - bigdl_type="float"): - super(Unsqueeze, self).__init__(None, bigdl_type, - pos, - num_input_dims) - - -class Reshape(Model): - ''' - >>> reshape = Reshape([1, 28, 28]) - creating: createReshape - ''' - - def __init__(self, size, bigdl_type="float"): - super(Reshape, self).__init__(None, bigdl_type, size) - - -def _test(): - import doctest - from pyspark import SparkContext - from nn import layer - from util.common import init_engine - from util.common import create_spark_conf - globs = layer.__dict__.copy() - sc = SparkContext(master="local[4]", appName="test layer", - conf=create_spark_conf()) - globs['sc'] = sc - init_engine() - - (failure_count, test_count) = doctest.testmod(globs=globs, - optionflags=doctest.ELLIPSIS) - if failure_count: - exit(-1) - -if __name__ == "__main__": - _test() diff --git a/python/dllib/src/bigdl/dllib/optim/__init__.py b/python/dllib/src/bigdl/dllib/optim/__init__.py deleted file mode 100644 index 2151a805423..00000000000 --- a/python/dllib/src/bigdl/dllib/optim/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -# -# Copyright 2016 The BigDL Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# diff --git a/python/dllib/src/bigdl/dllib/optim/optimizer.py b/python/dllib/src/bigdl/dllib/optim/optimizer.py deleted file mode 100644 index 4272e54f696..00000000000 --- a/python/dllib/src/bigdl/dllib/optim/optimizer.py +++ /dev/null @@ -1,169 +0,0 @@ -# -# Copyright 2016 The BigDL Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - - -from util.common import callBigDlFunc -from util.common import JavaValue -from util.common import callJavaFunc -from pyspark import SparkContext -import numpy as np -import os -from distutils.dir_util import mkpath - - -import sys -if sys.version >= '3': - long = int - unicode = str - - -class MaxIteration(JavaValue): - ''' - >>> maxIteration = MaxIteration(20) - creating: createMaxIteration - ''' - def __init__(self, max, bigdl_type="float"): - JavaValue.__init__(self, None, bigdl_type, max) - - -class MaxEpoch(JavaValue): - ''' - >>> maxEpoch = MaxEpoch(2) - creating: createMaxEpoch - ''' - def __init__(self, max_epoch, bigdl_type="float"): - JavaValue.__init__(self, None, bigdl_type, max_epoch) - - -class EveryEpoch(JavaValue): - ''' - >>> everyEpoch = EveryEpoch() - creating: createEveryEpoch - ''' - def __init__(self, bigdl_type="float"): - JavaValue.__init__(self, None, bigdl_type) - - -class SeveralIteration(JavaValue): - ''' - >>> serveralIteration = SeveralIteration(2) - creating: createSeveralIteration - ''' - def __init__(self, interval, bigdl_type="float"): - JavaValue.__init__(self, None, bigdl_type, interval) - - -class Poly(JavaValue): - ''' - >>> poly = Poly(0.5, 2) - creating: createPoly - ''' - def __init__(self, power, max_iteration, bigdl_type="float"): - JavaValue.__init__(self, None, bigdl_type, power, max_iteration) - - -class Step(JavaValue): - ''' - >>> step = Step(2, 0.3) - creating: createStep - ''' - def __init__(self, step_size, gamma, bigdl_type="float"): - JavaValue.__init__(self, None, bigdl_type, step_size, gamma) - - -class Optimizer(JavaValue): - - def __init__(self, - model, - training_rdd, - criterion, - end_trigger, - batch_size, - optim_method="SGD", - state={}, - bigdl_type="float"): - JavaValue.__init__(self, None, bigdl_type, model.value, - training_rdd, criterion, optim_method, - state, end_trigger, batch_size) - - def setvalidation(self, batch_size, val_rdd, trigger, val_method=["Top1Accuracy"]): - callBigDlFunc(self.bigdl_type, "setValidation", self.value, batch_size, - trigger, val_rdd, val_method) - - def setcheckpoint(self, checkpoint_trigger, - checkpoint_path, isOverWrite=True): - if not os.path.exists(checkpoint_path): - mkpath(checkpoint_path) - callBigDlFunc(self.bigdl_type, "setCheckPoint", self.value, - checkpoint_trigger, checkpoint_path, isOverWrite) - - # return a module - def optimize(self): - jmodel = callJavaFunc(SparkContext.getOrCreate(), self.value.optimize) - from nn.layer import Model - return Model.of(jmodel) - - def set_train_summary(self, summary): - callBigDlFunc(self.bigdl_type, "setTrainSummary", self.value, - summary) - return self - - def set_val_summary(self, summary): - callBigDlFunc(self.bigdl_type, "setValSummary", self.value, - summary) - return self - - -class TrainSummary(JavaValue, ): - def __init__(self, log_dir, app_name, bigdl_type="float"): - JavaValue.__init__(self, None, bigdl_type, log_dir, app_name) - - def read_scalar(self, tag): - return callBigDlFunc(self.bigdl_type, "summaryReadScalar", self.value, - tag) - - def set_summary_trigger(self, name, trigger): - return callBigDlFunc(self.bigdl_type, "summarySetTrigger", self.value, - name, trigger) - - -class ValidationSummary(JavaValue): - def __init__(self, log_dir, app_name, bigdl_type="float"): - JavaValue.__init__(self, None, bigdl_type, log_dir, app_name) - - def read_scalar(self, tag): - return callBigDlFunc(self.bigdl_type, "summaryReadScalar", self.value, - tag) - - -def _test(): - import doctest - from pyspark import SparkContext - from optim import optimizer - from util.common import init_engine - from util.common import create_spark_conf - globs = optimizer.__dict__.copy() - sc = SparkContext(master="local[4]", appName="test optimizer", - conf=create_spark_conf()) - init_engine() - globs['sc'] = sc - (failure_count, test_count) = doctest.testmod(globs=globs, - optionflags=doctest.ELLIPSIS) - if failure_count: - exit(-1) - -if __name__ == "__main__": - _test() diff --git a/python/dllib/src/bigdl/dllib/utils/common.py b/python/dllib/src/bigdl/dllib/utils/common.py deleted file mode 100644 index bbac74231b2..00000000000 --- a/python/dllib/src/bigdl/dllib/utils/common.py +++ /dev/null @@ -1,261 +0,0 @@ -# -# Copyright 2016 The BigDL Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import sys -from py4j.protocol import Py4JJavaError -from py4j.java_gateway import JavaObject -from py4j.java_collections import ListConverter, JavaArray, JavaList, JavaMap - -from pyspark import RDD, SparkContext -from pyspark.serializers import PickleSerializer, AutoBatchedSerializer -from pyspark.sql import DataFrame, SQLContext -from pyspark.mllib.common import callJavaFunc -from pyspark import SparkConf -import numpy as np - -if sys.version >= '3': - long = int - unicode = str - - -class JavaValue(object): - def jvm_class_constructor(self): - name = "create" + self.__class__.__name__ - print("creating: " + name) - return name - - def __init__(self, jvalue, bigdl_type, *args): - self.value = jvalue if jvalue else callBigDlFunc( - bigdl_type, self.jvm_class_constructor(), *args) - self.bigdl_type = bigdl_type - - -class TestResult(): - def __init__(self, result, total_num, method): - self.result = result - self.total_num = total_num - self.method = method - - def __reduce__(self): - return (TestResult, (self.result, self.total_num, self.method)) - - def __str__(self): - return "Test result: %s, total_num: %s, method: %s" % ( - self.result, self.total_num, self.method) - - -class Sample(object): - def __init__(self, features, label, features_shape, label_shape, - bigdl_type="float"): - def get_dtype(): - if "float" == bigdl_type: - return "float32" - else: - return "float64" - self.features = np.array(features, dtype=get_dtype()).reshape(features_shape) # noqa - self.label = np.array(label, dtype=get_dtype()).reshape(label_shape) - self.bigdl_type = bigdl_type - - @classmethod - def from_ndarray(cls, features, label, bigdl_type="float"): - return cls( - features=[float(i) for i in features.ravel()], - label=[float(i) for i in label.ravel()], - features_shape=list(features.shape), - label_shape=list(label.shape) if label.shape else [label.size], - bigdl_type=bigdl_type) - - @classmethod - def flatten(cls, a_ndarray): - """ - Utility method to flatten a ndarray - :return (storage, shape) - >>> import numpy as np - >>> from util.common import Sample - >>> np.random.seed(123) - >>> data = np.random.uniform(0, 1, (2, 3)) - >>> (storage, shape) = Sample.flatten(data) - >>> shape - [2, 3] - >>> (storage, shape) = Sample.flatten(np.array(2)) - >>> shape - [1] - """ - storage = [float(i) for i in a_ndarray.ravel()] - shape = list(a_ndarray.shape) if a_ndarray.shape else [a_ndarray.size] - return storage, shape - - def __reduce__(self): - (features_storage, features_shape) = Sample.flatten(self.features) - (label_storage, label_shape) = Sample.flatten(self.label) - return (Sample, ( - features_storage, label_storage, features_shape, label_shape, - self.bigdl_type)) - - def __str__(self): - return "features: %s, label: %s," % (self.features, self.label) - - -_picklable_classes = [ - 'LinkedList', - 'SparseVector', - 'DenseVector', - 'DenseMatrix', - 'Rating', - 'LabeledPoint', - 'Sample', - 'TestResult' -] - - -def init_engine(bigdl_type="float"): - callBigDlFunc(bigdl_type, "initEngine") - - -def get_bigdl_conf(): - bigdl_conf_file = "spark-bigdl.conf" - bigdl_python_wrapper = "python-api.zip" - - def load_conf(conf_str): - return dict(line.split() for line in conf_str.split("\n") if - "#" not in line and line.strip()) - - for p in sys.path: - if bigdl_conf_file in p: - with open(p) as conf_file: - return load_conf(conf_file.read()) - if bigdl_python_wrapper in p: - import zipfile - with zipfile.ZipFile(p, 'r') as zip_conf: - return load_conf(zip_conf.read(bigdl_conf_file)) - raise Exception("Cannot find spark-bigdl.conf.Pls add it to PYTHONPATH.") - - -def create_spark_conf(): - bigdl_conf = get_bigdl_conf() - sparkConf = SparkConf() - sparkConf.setAll(bigdl_conf.items()) - return sparkConf - - -def callBigDlFunc(bigdl_type, name, *args): - """ Call API in PythonBigDL """ - sc = SparkContext.getOrCreate() - if bigdl_type == "float": - api = getattr( - sc._jvm.com.intel.analytics.bigdl.python.api.PythonBigDL.ofFloat(), - name) - elif bigdl_type == "double": - api = getattr( - sc._jvm.com.intel.analytics.bigdl.python.api.PythonBigDL.ofDouble(), - name) - else: - raise Exception("Not supported bigdl_type: %s" % bigdl_type) - return callJavaFunc(sc, api, *args) - - -def _java2py(sc, r, encoding="bytes"): - if isinstance(r, JavaObject): - clsName = r.getClass().getSimpleName() - # convert RDD into JavaRDD - if clsName != 'JavaRDD' and clsName.endswith("RDD"): - r = r.toJavaRDD() - clsName = 'JavaRDD' - - if clsName == 'JavaRDD': - jrdd = sc._jvm.SerDe.javaToPython(r) - return RDD(jrdd, sc) - - if clsName == 'DataFrame': - return DataFrame(r, SQLContext.getOrCreate(sc)) - - if clsName in _picklable_classes: - r = sc._jvm.org.apache.spark.bigdl.api.python.BigDLSerDe.dumps(r) - elif isinstance(r, (JavaArray, JavaList, JavaMap)): - try: - r = sc._jvm.org.apache.spark.bigdl.api.python.BigDLSerDe.dumps( - r) - except Py4JJavaError: - pass # not pickable - - if isinstance(r, (bytearray, bytes)): - r = PickleSerializer().loads(bytes(r), encoding=encoding) - return r - - -def callJavaFunc(sc, func, *args): - """ Call Java Function """ - args = [_py2java(sc, a) for a in args] - result = func(*args) - return _java2py(sc, result) - - -def _to_java_object_rdd(rdd): - """ Return a JavaRDD of Object by unpickling - - It will convert each Python object into Java object by Pyrolite, whenever - the RDD is serialized in batch or not. - """ - rdd = rdd._reserialize(AutoBatchedSerializer(PickleSerializer())) - return \ - rdd.ctx._jvm.org.apache.spark.bigdl.api.python.BigDLSerDe.pythonToJava( - rdd._jrdd, True) - - -def _py2java(sc, obj): - """ Convert Python object into Java """ - if isinstance(obj, RDD): - obj = _to_java_object_rdd(obj) - elif isinstance(obj, DataFrame): - obj = obj._jdf - elif isinstance(obj, SparkContext): - obj = obj._jsc - elif isinstance(obj, (list, tuple)): - obj = ListConverter().convert([_py2java(sc, x) for x in obj], - sc._gateway._gateway_client) - elif isinstance(obj, dict): - result = {} - for (key, value) in obj.iteritems(): - result[key] = _py2java(sc, value) if isinstance(value, JavaValue) else value # noqa - obj = result - - elif isinstance(obj, JavaValue): - obj = obj.value - elif isinstance(obj, JavaObject): - pass - elif isinstance(obj, (int, long, float, bool, bytes, unicode)): - pass - else: - data = bytearray(PickleSerializer().dumps(obj)) - obj = sc._jvm.org.apache.spark.bigdl.api.python.BigDLSerDe.loads(data) - return obj - - -def _test(): - import doctest - from pyspark import SparkContext - from nn import layer - globs = layer.__dict__.copy() - sc = SparkContext(master="local[2]", appName="test common utility") - globs['sc'] = sc - (failure_count, test_count) = doctest.testmod(globs=globs, - optionflags=doctest.ELLIPSIS) - if failure_count: - exit(-1) - - -if __name__ == "__main__": - _test() diff --git a/python/dllib/src/bigdl/utils/__init__.py b/python/dllib/src/bigdl/utils/__init__.py deleted file mode 100644 index 2151a805423..00000000000 --- a/python/dllib/src/bigdl/utils/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -# -# Copyright 2016 The BigDL Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# From af65fa504847cd06fa5f68e4cff91e8231e5827a Mon Sep 17 00:00:00 2001 From: zhichao-li Date: Tue, 21 Mar 2017 16:06:55 +0800 Subject: [PATCH 013/823] add Tensor wrapper add CrossEntropyCriterion --- README.md | 127 -- .../bigdl/dllib/feature/dataset/__init__.py | 15 - .../src/bigdl/dllib/feature/dataset/base.py | 197 -- .../src/bigdl/dllib/feature/dataset/mnist.py | 113 -- .../src/bigdl/dllib/feature/dataset/news20.py | 73 - .../dllib/feature/dataset/transformer.py | 23 - .../dllib/src/bigdl/dllib/models/__init__.py | 15 - .../src/bigdl/dllib/models/lenet/__init__.py | 15 - .../src/bigdl/dllib/models/lenet/lenet5.py | 97 - .../models/textclassifier/textclassifier.py | 177 -- python/dllib/src/bigdl/dllib/nn/__init__.py | 15 - python/dllib/src/bigdl/dllib/nn/criterion.py | 308 --- python/dllib/src/bigdl/dllib/nn/layer.py | 1673 ----------------- .../dllib/src/bigdl/dllib/optim/__init__.py | 15 - .../dllib/src/bigdl/dllib/optim/optimizer.py | 169 -- python/dllib/src/bigdl/utils/__init__.py | 15 - python/dllib/src/bigdl/utils/common.py | 261 --- 17 files changed, 3308 deletions(-) delete mode 100644 README.md delete mode 100644 python/dllib/src/bigdl/dllib/feature/dataset/__init__.py delete mode 100644 python/dllib/src/bigdl/dllib/feature/dataset/base.py delete mode 100644 python/dllib/src/bigdl/dllib/feature/dataset/mnist.py delete mode 100644 python/dllib/src/bigdl/dllib/feature/dataset/news20.py delete mode 100644 python/dllib/src/bigdl/dllib/feature/dataset/transformer.py delete mode 100644 python/dllib/src/bigdl/dllib/models/__init__.py delete mode 100644 python/dllib/src/bigdl/dllib/models/lenet/__init__.py delete mode 100644 python/dllib/src/bigdl/dllib/models/lenet/lenet5.py delete mode 100644 python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py delete mode 100644 python/dllib/src/bigdl/dllib/nn/__init__.py delete mode 100644 python/dllib/src/bigdl/dllib/nn/criterion.py delete mode 100644 python/dllib/src/bigdl/dllib/nn/layer.py delete mode 100644 python/dllib/src/bigdl/dllib/optim/__init__.py delete mode 100644 python/dllib/src/bigdl/dllib/optim/optimizer.py delete mode 100644 python/dllib/src/bigdl/utils/__init__.py delete mode 100644 python/dllib/src/bigdl/utils/common.py diff --git a/README.md b/README.md deleted file mode 100644 index ed72d7eab3d..00000000000 --- a/README.md +++ /dev/null @@ -1,127 +0,0 @@ - -BigDL comes with scala API for now. However Python is a powerful programming language for data analysis and with large amount of useful libraries, we are developing a lightweight python binding on top of PySpark which can enable us use Python naively with BigDL. - -The Python API is almost identical to the scala version, and it would map ndarray to tensor for the training samples, so basically user only need to care about how to manipulate ndarray. - -This Python binding tested with Python 2.7 and Spark 1.6.0. - -Here are the steps for training a simple LeNet model: - -1). Create a RDD[Sample]: -``` -RDD[..] --transform-->RDD[ndarray, ndarray].map(Sample.from_ndarray(features, label)) --> RDD[Sample] -``` - -2). Define a model: -``` - def build_model(class_num): - model = Sequential() - model.add(Reshape([1, 28, 28])) - model.add(SpatialConvolution(1, 6, 5, 5)) - model.add(Tanh()) - model.add(SpatialMaxPooling(2, 2, 2, 2)) - model.add(Tanh()) - model.add(SpatialConvolution(6, 12, 5, 5)) - model.add(SpatialMaxPooling(2, 2, 2, 2)) - model.add(Reshape([12 * 4 * 4])) - model.add(Linear(12 * 4 * 4, 100)) - model.add(Tanh()) - model.add(Linear(100, class_num)) - model.add(LogSoftMax()) - return model - ``` - -3). Create Optimizer and train: -``` - optimizer = Optimizer( - model=build_model(10), - training_rdd=train_data, - criterion=ClassNLLCriterion(), - optim_method="SGD", - state=state, - end_trigger=MaxEpoch(100), - batch_size=int(options.batchSize)) - optimizer.setvalidation( - batch_size=32, - val_rdd=test_data, - trigger=EveryEpoch(), - val_method=["top1"] - ) - optimizer.setcheckpoint(EveryEpoch(), "/tmp/lenet5/") - trained_model = optimizer.optimize() -``` - -4) LeNet example can be found from: models/lenet5.py - -## Run python test -* Package Scala code by: ```$BigDL_HOME/make-dist.sh``` -* Set SPARK_HOME and then run: ```$BigDL_HOME/pyspark/dev/run-all.sh``` - -## Installing on Ubuntu -1. Build BigDL -[Build Page](https://github.com/intel-analytics/BigDL/wiki/Build-Page) - * With Spark1.6: ``` $BIGDL_HOME/make-dist.sh ``` - * With Spark2.0: ``` $BIGDL_HOME/make-dist.sh -P spark_2.0 ``` - -2. Install python dependensies: - * Installing Numpy: - ```sudo apt-get install python-numpy``` - - * Installing Python setuptools: - ```sudo apt-get install -y python-setuptools python-pip``` - - * Install Jupyter: - ```sudo pip install jupyter``` - -## Run a Lenet example on standalone cluster - - ``` - BigDL_HOME=... - SPARK_HOME=... - MASTER=... - PYTHON_API_ZIP_PATH=${BigDL_HOME}/dist/lib/bigdl-0.1.0-SNAPSHOT-python-api.zip - BigDL_JAR_PATH=${BigDL_HOME}/dist/lib/bigdl-0.1.0-SNAPSHOT-jar-with-dependencies.jar - PYTHONPATH=${PYTHON_API_ZIP_PATH}:$PYTHONPATH - ${SPARK_HOME}/bin/spark-submit \ - --master ${MASTER} \ - --driver-cores 5 \ - --driver-memory 10g \ - --total-executor-cores 80 \ - --executor-cores 10 \ - --executor-memory 20g \ - --conf spark.akka.frameSize=64 \ - --py-files ${PYTHON_API_ZIP_PATH},${BigDL_HOME}/pyspark/bigdl/models/lenet/lenet5.py \ - --properties-file ${BigDL_HOME}/dist/conf/spark-bigdl.conf \ - --jars ${BigDL_JAR_PATH} \ - --conf spark.driver.extraClassPath=${BigDL_JAR_PATH} \ - --conf spark.executor.extraClassPath=bigdl-0.1.0-SNAPSHOT-jar-with-dependencies.jar \ - ${BigDL_HOME}/pyspark/bigdl/models/lenet/lenet5.py - ``` - - -## Launch Jupyter on standalone cluster - - ``` - BigDL_HOME=... - SPARK_HOME=... - MASTER=... - PYTHON_API_ZIP_PATH=${BigDL_HOME}/dist/lib/bigdl-0.1.0-SNAPSHOT-python-api.zip - BigDL_JAR_PATH=${BigDL_HOME}/dist/lib/bigdl-0.1.0-SNAPSHOT-jar-with-dependencies.jar - - export PYTHONPATH=${PYTHON_API_ZIP_PATH}:$PYTHONPATH - export IPYTHON_OPTS="notebook --notebook-dir=./ --ip=* --no-browser" - - ${SPARK_HOME}/bin/pyspark \ - --master ${MASTER} \ - --properties-file ${BigDL_HOME}/dist/conf/spark-bigdl.conf - --driver-cores 5 \ - --driver-memory 10g \ - --total-executor-cores 8 \ - --executor-cores 1 \ - --executor-memory 20g \ - --conf spark.akka.frameSize=64 \ - --py-files ${PYTHON_API_ZIP_PATH} \ - --jars ${BigDL_JAR_PATH} \ - --conf spark.driver.extraClassPath=${BigDL_JAR_PATH} \ - --conf spark.executor.extraClassPath=bigdl-0.1.0-SNAPSHOT-jar-with-dependencies.jar - ``` diff --git a/python/dllib/src/bigdl/dllib/feature/dataset/__init__.py b/python/dllib/src/bigdl/dllib/feature/dataset/__init__.py deleted file mode 100644 index 2151a805423..00000000000 --- a/python/dllib/src/bigdl/dllib/feature/dataset/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -# -# Copyright 2016 The BigDL Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# diff --git a/python/dllib/src/bigdl/dllib/feature/dataset/base.py b/python/dllib/src/bigdl/dllib/feature/dataset/base.py deleted file mode 100644 index f211ee5261c..00000000000 --- a/python/dllib/src/bigdl/dllib/feature/dataset/base.py +++ /dev/null @@ -1,197 +0,0 @@ -# -# Copyright 2016 The BigDL Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import os -import sys -import shutil -import tempfile -import time -from distutils.dir_util import mkpath -from six.moves.urllib.request import urlopen -import numpy as np - - -# Adopt from keras -class Progbar(object): - def __init__(self, target, width=30, verbose=1, interval=0.01): - ''' - @param target: total number of steps expected - @param interval: minimum visual progress update interval (in seconds) - ''' - self.width = width - self.target = target - self.sum_values = {} - self.unique_values = [] - self.start = time.time() - self.last_update = 0 - self.interval = interval - self.total_width = 0 - self.seen_so_far = 0 - self.verbose = verbose - - def update(self, current, values=[], force=False): - ''' - @param current: index of current step - @param values: list of tuples (name, value_for_last_step). - The progress bar will display averages for these values. - @param force: force visual progress update - ''' - for k, v in values: - if k not in self.sum_values: - self.sum_values[k] = [v * (current - self.seen_so_far), current - self.seen_so_far] - self.unique_values.append(k) - else: - self.sum_values[k][0] += v * (current - self.seen_so_far) - self.sum_values[k][1] += (current - self.seen_so_far) - self.seen_so_far = current - - now = time.time() - if self.verbose == 1: - if not force and (now - self.last_update) < self.interval: - return - - prev_total_width = self.total_width - sys.stdout.write("\b" * prev_total_width) - sys.stdout.write("\r") - - numdigits = int(np.floor(np.log10(self.target))) + 1 - barstr = '%%%dd/%%%dd [' % (numdigits, numdigits) - bar = barstr % (current, self.target) - prog = float(current) / self.target - prog_width = int(self.width * prog) - if prog_width > 0: - bar += ('=' * (prog_width-1)) - if current < self.target: - bar += '>' - else: - bar += '=' - bar += ('.' * (self.width - prog_width)) - bar += ']' - sys.stdout.write(bar) - self.total_width = len(bar) - - if current: - time_per_unit = (now - self.start) / current - else: - time_per_unit = 0 - eta = time_per_unit * (self.target - current) - info = '' - if current < self.target: - info += ' - ETA: %ds' % eta - else: - info += ' - %ds' % (now - self.start) - for k in self.unique_values: - info += ' - %s:' % k - if type(self.sum_values[k]) is list: - avg = self.sum_values[k][0] / max(1, self.sum_values[k][1]) - if abs(avg) > 1e-3: - info += ' %.4f' % avg - else: - info += ' %.4e' % avg - else: - info += ' %s' % self.sum_values[k] - - self.total_width += len(info) - if prev_total_width > self.total_width: - info += ((prev_total_width - self.total_width) * " ") - - sys.stdout.write(info) - sys.stdout.flush() - - if current >= self.target: - sys.stdout.write("\n") - - if self.verbose == 2: - if current >= self.target: - info = '%ds' % (now - self.start) - for k in self.unique_values: - info += ' - %s:' % k - avg = self.sum_values[k][0] / max(1, self.sum_values[k][1]) - if avg > 1e-3: - info += ' %.4f' % avg - else: - info += ' %.4e' % avg - sys.stdout.write(info + "\n") - - self.last_update = now - - def add(self, n, values=[]): - self.update(self.seen_so_far + n, values) - - -def display_table(rows, positions): - - def display_row(objects, positions): - line = '' - for i in range(len(objects)): - line += str(objects[i]) - line = line[:positions[i]] - line += ' ' * (positions[i] - len(line)) - print(line) - - for objects in rows: - display_row(objects, positions) - -# Adopt from keras -# Under Python 2, 'urlretrieve' relies on FancyURLopener from legacy -# urllib module, known to have issues with proxy management -if sys.version_info[0] == 2: - def urlretrieve(url, filename, reporthook=None, data=None): - def chunk_read(response, chunk_size=8192, reporthook=None): - total_size = response.info().get('Content-Length').strip() - total_size = int(total_size) - count = 0 - while 1: - chunk = response.read(chunk_size) - count += 1 - if not chunk: - reporthook(count, total_size, total_size) - break - if reporthook: - reporthook(count, chunk_size, total_size) - yield chunk - - response = urlopen(url, data) - with open(filename, 'wb') as fd: - for chunk in chunk_read(response, reporthook=reporthook): - fd.write(chunk) -else: - from six.moves.urllib.request import urlretrieve - - -def maybe_download(filename, work_directory, source_url): - if not os.path.exists(work_directory): - mkpath(work_directory) - filepath = os.path.join(work_directory, filename) - - if not os.path.exists(filepath): - print('Downloading data from', source_url) - global progbar - progbar = None - - def dl_progress(count, block_size, total_size): - global progbar - if progbar is None: - progbar = Progbar(total_size) - else: - progbar.update(count * block_size) - with tempfile.NamedTemporaryFile() as tmpfile: - temp_file_name = tmpfile.name - urlretrieve(source_url, temp_file_name, dl_progress) - shutil.copy2(temp_file_name, filepath) - size = os.path.getsize(filepath) - print('Successfully downloaded', filename, size, 'bytes.') - return filepath diff --git a/python/dllib/src/bigdl/dllib/feature/dataset/mnist.py b/python/dllib/src/bigdl/dllib/feature/dataset/mnist.py deleted file mode 100644 index 4a53735fce2..00000000000 --- a/python/dllib/src/bigdl/dllib/feature/dataset/mnist.py +++ /dev/null @@ -1,113 +0,0 @@ -# -# Copyright 2016 The BigDL Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# Part of the code originally from Tensorflow - - -import gzip - -import numpy - -import base - -SOURCE_URL = 'http://yann.lecun.com/exdb/mnist/' - -TRAIN_MEAN = 0.13066047740239506 -TRAIN_STD = 0.3081078 -TEST_MEAN = 0.13251460696903547 -TEST_STD = 0.31048024 - - -def _read32(bytestream): - dt = numpy.dtype(numpy.uint32).newbyteorder('>') - return numpy.frombuffer(bytestream.read(4), dtype=dt)[0] - - -def extract_images(f): - """Extract the images into a 4D uint8 numpy array [index, y, x, depth]. - - Args: - f: A file object that can be passed into a gzip reader. - - Returns: - data: A 4D unit8 numpy array [index, y, x, depth]. - - Raises: - ValueError: If the bytestream does not start with 2051. - - """ - print('Extracting', f.name) - with gzip.GzipFile(fileobj=f) as bytestream: - magic = _read32(bytestream) - if magic != 2051: - raise ValueError( - 'Invalid magic number %d in MNIST image file: %s' % - (magic, f.name)) - num_images = _read32(bytestream) - rows = _read32(bytestream) - cols = _read32(bytestream) - buf = bytestream.read(rows * cols * num_images) - data = numpy.frombuffer(buf, dtype=numpy.uint8) - data = data.reshape(num_images, rows, cols, 1) - return data - - -def extract_labels(f): - print('Extracting', f.name) - with gzip.GzipFile(fileobj=f) as bytestream: - magic = _read32(bytestream) - if magic != 2049: - raise ValueError( - 'Invalid magic number %d in MNIST label file: %s' % - (magic, f.name)) - num_items = _read32(bytestream) - buf = bytestream.read(num_items) - labels = numpy.frombuffer(buf, dtype=numpy.uint8) - return labels - - -def read_data_sets(train_dir, data_type="train"): - TRAIN_IMAGES = 'train-images-idx3-ubyte.gz' - TRAIN_LABELS = 'train-labels-idx1-ubyte.gz' - TEST_IMAGES = 't10k-images-idx3-ubyte.gz' - TEST_LABELS = 't10k-labels-idx1-ubyte.gz' - - if data_type == "train": - local_file = base.maybe_download(TRAIN_IMAGES, train_dir, - SOURCE_URL + TRAIN_IMAGES) - with open(local_file, 'rb') as f: - train_images = extract_images(f) - - local_file = base.maybe_download(TRAIN_LABELS, train_dir, - SOURCE_URL + TRAIN_LABELS) - with open(local_file, 'rb') as f: - train_labels = extract_labels(f) - return train_images, train_labels - - else: - local_file = base.maybe_download(TEST_IMAGES, train_dir, - SOURCE_URL + TEST_IMAGES) - with open(local_file, 'rb') as f: - test_images = extract_images(f) - - local_file = base.maybe_download(TEST_LABELS, train_dir, - SOURCE_URL + TEST_LABELS) - with open(local_file, 'rb') as f: - test_labels = extract_labels(f) - return test_images, test_labels - - -if __name__ == "__main__": - read_data_sets("/tmp/mnist/") diff --git a/python/dllib/src/bigdl/dllib/feature/dataset/news20.py b/python/dllib/src/bigdl/dllib/feature/dataset/news20.py deleted file mode 100644 index fdedf5a6c41..00000000000 --- a/python/dllib/src/bigdl/dllib/feature/dataset/news20.py +++ /dev/null @@ -1,73 +0,0 @@ -import tarfile -import base -import os -import sys - -NEWS20_URL = 'http://qwone.com/~jason/20Newsgroups/20news-19997.tar.gz' # noqa -GLOVE_URL = 'http://nlp.stanford.edu/data/glove.6B.zip' # noqa - -CLASS_NUM = 20 - - -def download_news20(dest_dir): - file_name = "20news-19997.tar.gz" - file_abs_path = base.maybe_download(file_name, dest_dir, NEWS20_URL) - tar = tarfile.open(file_abs_path, "r:gz") - extracted_to = os.path.join(dest_dir, "20_newsgroups") - if not os.path.exists(extracted_to): - print("Extracting %s to %s" % (file_abs_path, extracted_to)) - tar.extractall(dest_dir) - tar.close() - return extracted_to - - -def download_glove_w2v(dest_dir): - file_name = "glove.6B.zip" - file_abs_path = base.maybe_download(file_name, dest_dir, GLOVE_URL) - import zipfile - zip_ref = zipfile.ZipFile(file_abs_path, 'r') - extracted_to = os.path.join(dest_dir, "glove.6B") - if not os.path.exists(extracted_to): - print("Extracting %s to %s" % (file_abs_path, extracted_to)) - zip_ref.extractall(extracted_to) - zip_ref.close() - return extracted_to - - -# return [(text_content, label)] -def get_news20(source_dir="/tmp/news20/"): - news_dir = download_news20(source_dir) - texts = [] # list of text samples - label_id = 0 - for name in sorted(os.listdir(news_dir)): - path = os.path.join(news_dir, name) - label_id += 1 - if os.path.isdir(path): - for fname in sorted(os.listdir(path)): - if fname.isdigit(): - fpath = os.path.join(path, fname) - if sys.version_info < (3,): - f = open(fpath) - else: - f = open(fpath, encoding='latin-1') - content = f.read() - texts.append((content, label_id)) - f.close() - - print('Found %s texts.' % len(texts)) - return texts - - -def get_glove_w2v(source_dir="/tmp/news20/", dim=100): - w2v_dir = download_glove_w2v(source_dir) - with open(os.path.join(w2v_dir, "glove.6B.%sd.txt" % dim)) as w2v_f: - pre_w2v = {} - for line in w2v_f.readlines(): - items = line.split(" ") - pre_w2v[items[0]] = [float(i) for i in items[1:]] - return pre_w2v - - -if __name__ == "__main__": - get_news20("/tmp/news20/") - get_glove_w2v("/tmp/news20/") diff --git a/python/dllib/src/bigdl/dllib/feature/dataset/transformer.py b/python/dllib/src/bigdl/dllib/feature/dataset/transformer.py deleted file mode 100644 index 9b5d5930d3a..00000000000 --- a/python/dllib/src/bigdl/dllib/feature/dataset/transformer.py +++ /dev/null @@ -1,23 +0,0 @@ -# -# Copyright 2016 The BigDL Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - - -from util.common import Sample - - -def normalizer(mean, std): - return lambda sample: Sample.from_ndarray((sample.features - mean) / std, - sample.label, sample.bigdl_type) diff --git a/python/dllib/src/bigdl/dllib/models/__init__.py b/python/dllib/src/bigdl/dllib/models/__init__.py deleted file mode 100644 index 2151a805423..00000000000 --- a/python/dllib/src/bigdl/dllib/models/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -# -# Copyright 2016 The BigDL Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# diff --git a/python/dllib/src/bigdl/dllib/models/lenet/__init__.py b/python/dllib/src/bigdl/dllib/models/lenet/__init__.py deleted file mode 100644 index 2151a805423..00000000000 --- a/python/dllib/src/bigdl/dllib/models/lenet/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -# -# Copyright 2016 The BigDL Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# diff --git a/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py b/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py deleted file mode 100644 index f6b1a752b40..00000000000 --- a/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py +++ /dev/null @@ -1,97 +0,0 @@ -# -# Copyright 2016 The BigDL Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# Still in experimental stage! - -from optparse import OptionParser - -from dataset import mnist -from dataset.transformer import * -from nn.layer import * -from nn.criterion import * -from optim.optimizer import * -from util.common import * - - -def build_model(class_num): - model = Sequential() - model.add(Reshape([1, 28, 28])) - model.add(SpatialConvolution(1, 6, 5, 5)) - model.add(Tanh()) - model.add(SpatialMaxPooling(2, 2, 2, 2)) - model.add(Tanh()) - model.add(SpatialConvolution(6, 12, 5, 5)) - model.add(SpatialMaxPooling(2, 2, 2, 2)) - model.add(Reshape([12 * 4 * 4])) - model.add(Linear(12 * 4 * 4, 100)) - model.add(Tanh()) - model.add(Linear(100, class_num)) - model.add(LogSoftMax()) - return model - - -def get_minst(sc, data_type="train", location="/tmp/mnist"): - (images, labels) = mnist.read_data_sets(location, data_type) - images = sc.parallelize(images) - labels = sc.parallelize(labels) - # Target start from 1 in BigDL - record = images.zip(labels).map(lambda (features, label): - Sample.from_ndarray(features, label + 1)) - return record - - -if __name__ == "__main__": - - parser = OptionParser() - parser.add_option("-a", "--action", dest="action", default="train") - parser.add_option("-b", "--batchSize", dest="batchSize", default="128") - - (options, args) = parser.parse_args(sys.argv) - - sc = SparkContext(appName="lenet5", conf=create_spark_conf()) - init_engine() - - if options.action == "train": - train_data = get_minst(sc, "train").map( - normalizer(mnist.TRAIN_MEAN, mnist.TRAIN_STD)) - test_data = get_minst(sc, "test").map( - normalizer(mnist.TEST_MEAN, mnist.TEST_STD)) - state = {"learningRate": 0.01, - "learningRateDecay": 0.0002} - optimizer = Optimizer( - model=build_model(10), - training_rdd=train_data, - criterion=ClassNLLCriterion(), - optim_method="SGD", - state=state, - end_trigger=MaxEpoch(20), - batch_size=int(options.batchSize)) - optimizer.setvalidation( - batch_size=32, - val_rdd=test_data, - trigger=EveryEpoch(), - val_method=["Top1Accuracy"] - ) - optimizer.setcheckpoint(EveryEpoch(), "/tmp/lenet5/") - trained_model = optimizer.optimize() - parameters = trained_model.parameters() - elif options.action == "test": - test_data = get_minst("test").map( - normalizer(mnist.TEST_MEAN, mnist.TEST_STD)) - # TODO: Pass model path through external parameter - model = Model.from_path("/tmp/lenet5/lenet-model.470") - results = model.test(test_data, 32, ["Top1Accuracy"]) - for result in results: - print result diff --git a/python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py b/python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py deleted file mode 100644 index 2929290aa0a..00000000000 --- a/python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py +++ /dev/null @@ -1,177 +0,0 @@ -# -# Copyright 2016 The BigDL Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# Still in experimental stage! - -import itertools -import re -from optparse import OptionParser - -from dataset import news20 -from nn.layer import * -from nn.criterion import * -from optim.optimizer import * -from util.common import * -from util.common import Sample - - -def text_to_words(review_text): - letters_only = re.sub("[^a-zA-Z]", " ", review_text) - words = letters_only.lower().split() - return words - - -def analyze_texts(data_rdd): - return data_rdd.flatMap(lambda (text, label): text_to_words(text)) \ - .map(lambda word: (word, 1)).reduceByKey(lambda a, b: a + b) \ - .sortBy(lambda (w, c): - c).zipWithIndex() \ - .map(lambda ((w, c), i): (w, (i + 1, c))).collect() - - -# pad([1, 2, 3, 4, 5], 0, 6) -def pad(l, fill_value, width): - if len(l) >= width: - return l[0: width] - else: - l.extend([fill_value] * (width - len(l))) - return l - - -def to_vec(token, b_w2v, embedding_dim): - if token in b_w2v: - return b_w2v[token] - else: - return pad([], 0, embedding_dim) - - -def to_sample(vectors, label, embedding_dim): - # flatten nested list - flatten_features = list(itertools.chain(*vectors)) - features = np.array(flatten_features, dtype='float').reshape( - [sequence_len, embedding_dim]) - - if model_type.lower() == "cnn": - features = features.transpose(1, 0) - return Sample.from_ndarray(features, np.array(label)) - - -def build_model(class_num): - model = Sequential() - - if model_type.lower() == "cnn": - model.add(Reshape([embedding_dim, 1, sequence_len])) - model.add(SpatialConvolution(embedding_dim, 128, 5, 1)) - model.add(ReLU()) - model.add(SpatialMaxPooling(5, 1, 5, 1)) - model.add(SpatialConvolution(128, 128, 5, 1)) - model.add(ReLU()) - model.add(SpatialMaxPooling(5, 1, 5, 1)) - model.add(Reshape([128])) - elif model_type.lower() == "lstm": - model.add(Recurrent() - .add(LSTM(embedding_dim, 128))) - model.add(Select(2, -1)) - elif model_type.lower() == "gru": - model.add(Recurrent() - .add(GRU(embedding_dim, 128))) - model.add(Select(2, -1)) - else: - raise ValueError('model can only be cnn, lstm, or gru') - - model.add(Linear(128, 100)) - model.add(Linear(100, class_num)) - model.add(LogSoftMax()) - return model - - -def train(sc, - batch_size, - sequence_len, max_words, embedding_dim, training_split): - print('Processing text dataset') - texts = news20.get_news20() - data_rdd = sc.parallelize(texts, 2) - - word_to_ic = analyze_texts(data_rdd) - - # Only take the top wc between [10, sequence_len] - word_to_ic = dict(word_to_ic[10: max_words]) - bword_to_ic = sc.broadcast(word_to_ic) - - w2v = news20.get_glove_w2v(dim=embedding_dim) - filtered_w2v = {w: v for w, v in w2v.items() if w in word_to_ic} - bfiltered_w2v = sc.broadcast(filtered_w2v) - - tokens_rdd = data_rdd.map(lambda (text, label): - ([w for w in text_to_words(text) if - w in bword_to_ic.value], label)) - padded_tokens_rdd = tokens_rdd.map( - lambda (tokens, label): (pad(tokens, "##", sequence_len), label)) - vector_rdd = padded_tokens_rdd.map(lambda (tokens, label): - ([to_vec(w, bfiltered_w2v.value, - embedding_dim) for w in - tokens], label)) - sample_rdd = vector_rdd.map( - lambda (vectors, label): to_sample(vectors, label, embedding_dim)) - - train_rdd, val_rdd = sample_rdd.randomSplit( - [training_split, 1-training_split]) - - state = {"batchSize": batch_size, - "learningRate": 0.01, - "learningRateDecay": 0.0002} - - optimizer = Optimizer( - model=build_model(news20.CLASS_NUM), - training_rdd=train_rdd, - criterion=ClassNLLCriterion(), - end_trigger=MaxEpoch(max_epoch), - batch_size=batch_size, - optim_method="Adagrad", - state=state) - - optimizer.setvalidation( - batch_size=batch_size, - val_rdd=val_rdd, - trigger=EveryEpoch(), - val_method=["Top1Accuracy"] - ) - train_model = optimizer.optimize() - -if __name__ == "__main__": - parser = OptionParser() - parser.add_option("-a", "--action", dest="action", default="train") - parser.add_option("-b", "--batchSize", dest="batchSize", default="128") - parser.add_option("-e", "--embedding_dim", dest="embedding_dim", default="50") # noqa - parser.add_option("-m", "--max_epoch", dest="max_epoch", default="15") - parser.add_option("--model", dest="model_type", default="cnn") - - (options, args) = parser.parse_args(sys.argv) - if options.action == "train": - batch_size = int(options.batchSize) - embedding_dim = int(options.embedding_dim) - max_epoch = int(options.max_epoch) - model_type = options.model_type - sequence_len = 50 - max_words = 1000 - training_split = 0.8 - sc = SparkContext(appName="text_classifier", - conf=create_spark_conf()) - init_engine() - train(sc, - batch_size, - sequence_len, max_words, embedding_dim, training_split) - elif options.action == "test": - pass diff --git a/python/dllib/src/bigdl/dllib/nn/__init__.py b/python/dllib/src/bigdl/dllib/nn/__init__.py deleted file mode 100644 index 2151a805423..00000000000 --- a/python/dllib/src/bigdl/dllib/nn/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -# -# Copyright 2016 The BigDL Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# diff --git a/python/dllib/src/bigdl/dllib/nn/criterion.py b/python/dllib/src/bigdl/dllib/nn/criterion.py deleted file mode 100644 index 570653413ab..00000000000 --- a/python/dllib/src/bigdl/dllib/nn/criterion.py +++ /dev/null @@ -1,308 +0,0 @@ -# -# Copyright 2016 The BigDL Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - - -import sys -from util.common import callBigDlFunc -from util.common import JavaValue -from util.common import callJavaFunc -from pyspark import SparkContext - -import numpy as np - -if sys.version >= '3': - long = int - unicode = str - - -class Criterion(JavaValue): - - def __init__(self, jvalue, bigdl_type, *args): - self.value = jvalue if jvalue else callBigDlFunc( - bigdl_type, JavaValue.jvm_class_constructor(self), *args) - self.bigdl_type = bigdl_type - - @classmethod - def of(cls, jcriterion, bigdl_type="float"): - criterion = Criterion(bigdl_type, jcriterion) - criterion.value = jcriterion - criterion.bigdl_type = bigdl_type - return criterion - - -class ClassNLLCriterion(Criterion): - - ''' - >>> classNLLCriterion = ClassNLLCriterion() - creating: createClassNLLCriterion - ''' - - def __init__(self, - size_average=True, - bigdl_type="float"): - super(ClassNLLCriterion, self).__init__(None, bigdl_type, - size_average) - - -class MSECriterion(Criterion): - - ''' - >>> mSECriterion = MSECriterion() - creating: createMSECriterion - ''' - - def __init__(self, bigdl_type="float"): - super(MSECriterion, self).__init__(None, bigdl_type) - - -class AbsCriterion(Criterion): - - ''' - >>> absCriterion = AbsCriterion(True) - creating: createAbsCriterion - ''' - - def __init__(self, - size_average=True, - bigdl_type="float"): - super(AbsCriterion, self).__init__(None, bigdl_type, - size_average) - - -class ClassSimplexCriterion(Criterion): - - ''' - >>> classSimplexCriterion = ClassSimplexCriterion(2) - creating: createClassSimplexCriterion - ''' - - def __init__(self, - n_classes, - bigdl_type="float"): - super(ClassSimplexCriterion, self).__init__(None, bigdl_type, - n_classes) - - -class CosineEmbeddingCriterion(Criterion): - - ''' - >>> cosineEmbeddingCriterion = CosineEmbeddingCriterion(1e-5, True) - creating: createCosineEmbeddingCriterion - ''' - - def __init__(self, - margin=0.0, - size_average=True, - bigdl_type="float"): - super(CosineEmbeddingCriterion, self).__init__(None, bigdl_type, - margin, - size_average) - - -class DistKLDivCriterion(Criterion): - - ''' - >>> distKLDivCriterion = DistKLDivCriterion(True) - creating: createDistKLDivCriterion - ''' - - def __init__(self, - size_average=True, - bigdl_type="float"): - super(DistKLDivCriterion, self).__init__(None, bigdl_type, - size_average) - - -class HingeEmbeddingCriterion(Criterion): - - ''' - >>> hingeEmbeddingCriterion = HingeEmbeddingCriterion(1e-5, True) - creating: createHingeEmbeddingCriterion - ''' - - def __init__(self, - margin=1, - size_average=True, - bigdl_type="float"): - super(HingeEmbeddingCriterion, self).__init__(None, bigdl_type, - margin, - size_average) - - -class L1HingeEmbeddingCriterion(Criterion): - - ''' - >>> l1HingeEmbeddingCriterion = L1HingeEmbeddingCriterion(1e-5) - creating: createL1HingeEmbeddingCriterion - ''' - - def __init__(self, - margin=1, - bigdl_type="float"): - super(L1HingeEmbeddingCriterion, self).__init__(None, bigdl_type, - margin) - - -class MarginCriterion(Criterion): - - ''' - >>> marginCriterion = MarginCriterion(1e-5, True) - creating: createMarginCriterion - ''' - - def __init__(self, - margin=1.0, - size_average=True, - bigdl_type="float"): - super(MarginCriterion, self).__init__(None, bigdl_type, - margin, - size_average) - - -class MarginRankingCriterion(Criterion): - - ''' - >>> marginRankingCriterion = MarginRankingCriterion(1e-5, True) - creating: createMarginRankingCriterion - ''' - - def __init__(self, - margin=1.0, - size_average=True, - bigdl_type="float"): - super(MarginRankingCriterion, self).__init__(None, bigdl_type, - margin, - size_average) - - -class MultiCriterion(Criterion): - - ''' - >>> multiCriterion = MultiCriterion() - creating: createMultiCriterion - ''' - - def __init__(self, - bigdl_type="float"): - super(MultiCriterion, self).__init__(None, bigdl_type) - - -class MultiLabelMarginCriterion(Criterion): - - ''' - >>> multiLabelMarginCriterion = MultiLabelMarginCriterion(True) - creating: createMultiLabelMarginCriterion - ''' - - def __init__(self, - size_average=True, - bigdl_type="float"): - super(MultiLabelMarginCriterion, self).__init__(None, bigdl_type, - size_average) - - -class ParallelCriterion(Criterion): - - ''' - >>> parallelCriterion = ParallelCriterion(True) - creating: createParallelCriterion - ''' - - def __init__(self, - repeat_target=False, - bigdl_type="float"): - super(ParallelCriterion, self).__init__(None, bigdl_type, - repeat_target) - - -class SmoothL1Criterion(Criterion): - - ''' - >>> smoothL1Criterion = SmoothL1Criterion(True) - creating: createSmoothL1Criterion - ''' - - def __init__(self, - size_average=True, - bigdl_type="float"): - super(SmoothL1Criterion, self).__init__(None, bigdl_type, - size_average) - - -class SmoothL1CriterionWithWeights(Criterion): - - ''' - >>> smoothL1CriterionWithWeights = SmoothL1CriterionWithWeights(1e-5, 1) - creating: createSmoothL1CriterionWithWeights - ''' - - def __init__(self, - sigma, - num=0, - bigdl_type="float"): - super(SmoothL1CriterionWithWeights, self).__init__(None, bigdl_type, - sigma, - num) - - -class SoftmaxWithCriterion(Criterion): - - ''' - >>> softmaxWithCriterion = SoftmaxWithCriterion("VALID") - creating: createSoftmaxWithCriterion - ''' - - def __init__(self, - normalize_mode="VALID", - bigdl_type="float"): - super(SoftmaxWithCriterion, self).__init__(None, bigdl_type, - None, - normalize_mode) - - -class TimeDistributedCriterion(JavaValue): - - ''' - >>> td = TimeDistributedCriterion(ClassNLLCriterion()) - creating: createClassNLLCriterion - creating: createTimeDistributedCriterion - ''' - - def __init__(self, criterion, size_average=False, bigdl_type="float"): - super(TimeDistributedCriterion, self).__init__( - None, bigdl_type, criterion, size_average) - - -def _test(): - import doctest - from pyspark import SparkContext - from nn import criterion - from util.common import init_engine - from util.common import create_spark_conf - globs = criterion.__dict__.copy() - sc = SparkContext(master="local[4]", appName="test criterion", - conf=create_spark_conf()) - globs['sc'] = sc - init_engine() - - (failure_count, test_count) = doctest.testmod(globs=globs, - optionflags=doctest.ELLIPSIS) - if failure_count: - exit(-1) - - -if __name__ == "__main__": - _test() diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py deleted file mode 100644 index 48ace84dd7a..00000000000 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ /dev/null @@ -1,1673 +0,0 @@ -# -# Copyright 2016 The BigDL Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - - -import sys -from util.common import callBigDlFunc -from util.common import JavaValue -from util.common import callJavaFunc -from pyspark import SparkContext - -import numpy as np - -if sys.version >= '3': - long = int - unicode = str - -INTMAX = 2147483647 -INTMIN = -2147483648 -DOUBLEMAX = 1.7976931348623157E308 - - -class Model(JavaValue): - - def __init__(self, jvalue, bigdl_type, *args): - self.value = jvalue if jvalue else callBigDlFunc( - bigdl_type, JavaValue.jvm_class_constructor(self), *args) - self.bigdl_type = bigdl_type - - @classmethod - def of(cls, jmodel, bigdl_type="float"): - model = Model(bigdl_type, jmodel) - model.value = jmodel - model.bigdl_type = bigdl_type - return model - - def set_name(self, name): - callJavaFunc(SparkContext.getOrCreate(), self.value.setName, name) - return self - - def name(self): - return callJavaFunc(SparkContext.getOrCreate(), self.value.getName) - - def set_seed(self, seed=123): - callBigDlFunc(self.bigdl_type, "setModelSeed", seed) - return self - - def get_dtype(self): - if "float" == self.bigdl_type: - return "float32" - else: - return "float64" - - def reset(self): - callJavaFunc(SparkContext.getOrCreate(), self.value.reset) - return self - - def parameters(self): - name_to_params = callBigDlFunc(self.bigdl_type, - "modelGetParameters", - self.value) - - def to_ndarray(params): - return { - param_name: np.array(values[0], - dtype=self.get_dtype()).reshape( - values[1]) for param_name, values in params.iteritems()} - - return {layer_name: to_ndarray(params) for layer_name, params in - name_to_params.iteritems()} - - def predict(self, data_rdd): - return callBigDlFunc(self.bigdl_type, - "modelPredictRDD", self.value, data_rdd) - - def test(self, val_rdd, batch_size, val_methods): - return callBigDlFunc(self.bigdl_type, - "modelTest", - self.value, - val_rdd, batch_size, val_methods) - - @staticmethod - def from_path(path, bigdl_type="float"): - jmodel = callBigDlFunc(bigdl_type, "modelFromPath", path) - return Model.of(jmodel, bigdl_type) - - -class Linear(Model): - - ''' - >>> linear = Linear(100, 10, "Xavier") - creating: createLinear - ''' - - def __init__(self, input_size, output_size, init_method="default", - bigdl_type="float"): - super(Linear, self).__init__(None, bigdl_type, input_size, output_size, - init_method) - - -class ReLU(Model): - - ''' - >>> relu = ReLU() - creating: createReLU - ''' - - def __init__(self, bigdl_type="float"): - super(ReLU, self).__init__(None, bigdl_type) - - -class Tanh(Model): - - ''' - >>> tanh = Tanh() - creating: createTanh - ''' - - def __init__(self, bigdl_type="float"): - super(Tanh, self).__init__(None, bigdl_type) - - -class Echo(Model): - - ''' - >>> echo = Echo() - creating: createEcho - ''' - - def __init__(self, bigdl_type="float"): - super(Echo, self).__init__(None, bigdl_type) - - -class LogSoftMax(Model): - - ''' - >>> logSoftMax = LogSoftMax() - creating: createLogSoftMax - ''' - - def __init__(self, bigdl_type="float"): - super(LogSoftMax, self).__init__(None, bigdl_type) - - -class Sequential(Model): - - ''' - >>> echo = Echo() - creating: createEcho - >>> s = Sequential() - creating: createSequential - >>> s = s.add(echo) - >>> s = s.add(s) - >>> s = s.add(echo) - - ''' - - def __init__(self, bigdl_type="float"): - super(Sequential, self).__init__(None, bigdl_type) - - def add(self, model): - self.value.add(model.value) - return self - - -class SpatialConvolution(Model): - - ''' - >>> spatialConvolution = SpatialConvolution(6, 12, 5, 5) - creating: createSpatialConvolution - ''' - - def __init__(self, - n_input_plane, - n_output_plane, - kernel_w, - kernel_h, - stride_w=1, - stride_h=1, - pad_w=0, - pad_h=0, - n_group=1, - propagate_back=True, - init_method="Default", - bigdl_type="float"): - super(SpatialConvolution, self).__init__(None, bigdl_type, - n_input_plane, - n_output_plane, - kernel_w, - kernel_h, - stride_w, - stride_h, - pad_w, - pad_h, - n_group, - propagate_back, - init_method) - - -class SpatialMaxPooling(Model): - - ''' - >>> spatialMaxPooling = SpatialMaxPooling(2, 2, 2, 2) - creating: createSpatialMaxPooling - ''' - # to_ceil: call floor() when False; call ceil() when True - - def __init__(self, kw, - kh, - dw, - dh, - pad_w=0, - pad_h=0, - to_ceil=False, - bigdl_type="float"): - super(SpatialMaxPooling, self).__init__(None, bigdl_type, kw, - kh, - dw, - dh, - pad_w, - pad_h, - to_ceil) - - -class Select(Model): - ''' - >>> select = Select(1, 1) - creating: createSelect - ''' - - def __init__(self, dim, index, bigdl_type="float"): - super(Select, self).__init__(None, bigdl_type, dim, index) - - -class Recurrent(Model): - ''' - >>> recurrent = Recurrent() - creating: createRecurrent - ''' - - def __init__(self, bigdl_type="float"): - super(Recurrent, self).__init__(None, bigdl_type) - - def add(self, model): - self.value.add(model.value) - return self - - -class LSTM(Model): - ''' - >>> lstm = LSTM(4, 3) - creating: createLSTM - ''' - - def __init__(self, input_size, hidden_size, bigdl_type="float"): - super(LSTM, self).__init__(None, bigdl_type, input_size, hidden_size) - - -class GRU(Model): - ''' - >>> gru = GRU(4, 3) - creating: createGRU - ''' - - def __init__(self, input_size, hidden_size, bigdl_type="float"): - super(GRU, self).__init__(None, bigdl_type, input_size, hidden_size) - - -class RNNCell(Model): - ''' - >>> reshape = RNNCell(4, 3, Tanh()) - creating: createTanh - creating: createRNNCell - ''' - - def __init__(self, - input_size, - hidden_size, - activation, - bigdl_type="float"): - super(RNNCell, self).__init__(None, bigdl_type, input_size, hidden_size, activation) - - -class TimeDistributed(Model): - ''' - >>> td = TimeDistributed(Linear(2, 3)) - creating: createLinear - creating: createTimeDistributed - ''' - - def __init__(self, model, bigdl_type="float"): - super(TimeDistributed, self).__init__(None, bigdl_type, model) - - -class Concat(Model): - - ''' - >>> concat = Concat(2) - creating: createConcat - ''' - - def __init__(self, - dimension, - bigdl_type="float"): - super(Concat, self).__init__(None, bigdl_type, - dimension) - - -class SpatialAveragePooling(Model): - - ''' - >>> spatialAveragePooling = SpatialAveragePooling(7,7) - creating: createSpatialAveragePooling - ''' - - def __init__(self, - kw, - kh, - dw=1, - dh=1, - pad_w=0, - pad_h=0, - ceil_mode=False, - count_include_pad=True, - divide=True, - bigdl_type="float"): - super(SpatialAveragePooling, self).__init__(None, bigdl_type, - kw, - kh, - dw, - dh, - pad_w, - pad_h, - ceil_mode, - count_include_pad, - divide) - - -class SpatialBatchNormalization(Model): - - ''' - >>> spatialBatchNormalization = SpatialBatchNormalization(1) - creating: createSpatialBatchNormalization - ''' - - def __init__(self, - n_output, - eps=1e-5, - momentum=0.1, - affine=True, - bigdl_type="float"): - super(SpatialBatchNormalization, self).__init__(None, bigdl_type, - n_output, - eps, - momentum, - affine) - - -class SpatialCrossMapLRN(Model): - - ''' - >>> spatialCrossMapLRN = SpatialCrossMapLRN() - creating: createSpatialCrossMapLRN - ''' - - def __init__(self, - size=5, - alpha=1.0, - beta=0.75, - k=1.0, - bigdl_type="float"): - super(SpatialCrossMapLRN, self).__init__(None, bigdl_type, - size, - alpha, - beta, - k) - - -class Dropout(Model): - - ''' - >>> dropout = Dropout(0.4) - creating: createDropout - ''' - - def __init__(self, - init_p=0.5, - inplace=False, - scale=True, - bigdl_type="float"): - super(Dropout, self).__init__(None, bigdl_type, - init_p, - inplace, - scale) - - -class View(Model): - - ''' - >>> view = View([1024,2]) - creating: createView - ''' - - def __init__(self, - sizes, - num_input_dims=0, - bigdl_type="float"): - super(View, self).__init__(None, bigdl_type, - sizes, - num_input_dims) - - -class Abs(Model): - - ''' - >>> abs = Abs() - creating: createAbs - ''' - - def __init__(self, - bigdl_type="float"): - super(Abs, self).__init__(None, bigdl_type) - - -class Add(Model): - - ''' - >>> add = Add(1) - creating: createAdd - ''' - - def __init__(self, - input_size, - bigdl_type="float"): - super(Add, self).__init__(None, bigdl_type, - input_size) - - -class AddConstant(Model): - - ''' - >>> addConstant = AddConstant(1e-5, True) - creating: createAddConstant - ''' - - def __init__(self, - constant_scalar, - inplace=False, - bigdl_type="float"): - super(AddConstant, self).__init__(None, bigdl_type, - constant_scalar, - inplace) - - -class BatchNormalization(Model): - - ''' - >>> batchNormalization = BatchNormalization(1, 1e-5, 1e-5, True) - creating: createBatchNormalization - ''' - - def __init__(self, - n_output, - eps=1e-5, - momentum=0.1, - affine=True, - bigdl_type="float"): - super(BatchNormalization, self).__init__(None, bigdl_type, - n_output, - eps, - momentum, - affine) - - -class Bilinear(Model): - - ''' - >>> bilinear = Bilinear(1, 1, 1, True) - creating: createBilinear - ''' - - def __init__(self, - input_size1, - input_size2, - output_size, - bias_res=True, - bigdl_type="float"): - super(Bilinear, self).__init__(None, bigdl_type, - input_size1, - input_size2, - output_size, - bias_res) - - -class Bottle(Model): - - ''' - >>> bottle = Bottle(Linear(100,10), 1, 1) - creating: createLinear - creating: createBottle - ''' - - def __init__(self, - module, - n_input_dim=2, - n_output_dim1=INTMAX, - bigdl_type="float"): - super(Bottle, self).__init__(None, bigdl_type, - module, - n_input_dim, - n_output_dim1) - - -class CAdd(Model): - - ''' - >>> cAdd = CAdd([1,2]) - creating: createCAdd - ''' - - def __init__(self, - size, - bigdl_type="float"): - super(CAdd, self).__init__(None, bigdl_type, - size) - - -class CAddTable(Model): - - ''' - >>> cAddTable = CAddTable(True) - creating: createCAddTable - ''' - - def __init__(self, - inplace=False, - bigdl_type="float"): - super(CAddTable, self).__init__(None, bigdl_type, - inplace) - - -class CDivTable(Model): - - ''' - >>> cDivTable = CDivTable() - creating: createCDivTable - ''' - - def __init__(self, - bigdl_type="float"): - super(CDivTable, self).__init__(None, bigdl_type) - - -class CMaxTable(Model): - - ''' - >>> cMaxTable = CMaxTable() - creating: createCMaxTable - ''' - - def __init__(self, - bigdl_type="float"): - super(CMaxTable, self).__init__(None, bigdl_type) - - -class CMinTable(Model): - - ''' - >>> cMinTable = CMinTable() - creating: createCMinTable - ''' - - def __init__(self, - bigdl_type="float"): - super(CMinTable, self).__init__(None, bigdl_type) - - -class CMul(Model): - - ''' - >>> cMul = CMul([1,2]) - creating: createCMul - ''' - - def __init__(self, - size, - bigdl_type="float"): - super(CMul, self).__init__(None, bigdl_type, - size) - - -class CMulTable(Model): - - ''' - >>> cMulTable = CMulTable() - creating: createCMulTable - ''' - - def __init__(self, - bigdl_type="float"): - super(CMulTable, self).__init__(None, bigdl_type) - - -class CSubTable(Model): - - ''' - >>> cSubTable = CSubTable() - creating: createCSubTable - ''' - - def __init__(self, - bigdl_type="float"): - super(CSubTable, self).__init__(None, bigdl_type) - - -class Clamp(Model): - - ''' - >>> clamp = Clamp(1, 3) - creating: createClamp - ''' - - def __init__(self, - min, - max, - bigdl_type="float"): - super(Clamp, self).__init__(None, bigdl_type, - min, - max) - - -class Contiguous(Model): - - ''' - >>> contiguous = Contiguous() - creating: createContiguous - ''' - - def __init__(self, - bigdl_type="float"): - super(Contiguous, self).__init__(None, bigdl_type) - - -class Copy(Model): - - ''' - >>> copy = Copy() - creating: createCopy - ''' - - def __init__(self, - bigdl_type="float"): - super(Copy, self).__init__(None, bigdl_type) - - -class Cosine(Model): - - ''' - >>> cosine = Cosine(2,3) - creating: createCosine - ''' - - def __init__(self, - input_size, - output_size, - bigdl_type="float"): - super(Cosine, self).__init__(None, bigdl_type, - input_size, - output_size) - - -class CosineDistance(Model): - - ''' - >>> cosineDistance = CosineDistance() - creating: createCosineDistance - ''' - - def __init__(self, - bigdl_type="float"): - super(CosineDistance, self).__init__(None, bigdl_type) - - -class DotProduct(Model): - - ''' - >>> dotProduct = DotProduct() - creating: createDotProduct - ''' - - def __init__(self, - bigdl_type="float"): - super(DotProduct, self).__init__(None, bigdl_type) - - -class ELU(Model): - - ''' - >>> eLU = ELU(1e-5, True) - creating: createELU - ''' - - def __init__(self, - alpha=1.0, - inplace=False, - bigdl_type="float"): - super(ELU, self).__init__(None, bigdl_type, - alpha, - inplace) - - -class Euclidean(Model): - - ''' - >>> euclidean = Euclidean(1, 1, True) - creating: createEuclidean - ''' - - def __init__(self, - input_size, - output_size, - fast_backward=True, - bigdl_type="float"): - super(Euclidean, self).__init__(None, bigdl_type, - input_size, - output_size, - fast_backward) - - -class Exp(Model): - - ''' - >>> exp = Exp() - creating: createExp - ''' - - def __init__(self, - bigdl_type="float"): - super(Exp, self).__init__(None, bigdl_type) - - -class FlattenTable(Model): - - ''' - >>> flattenTable = FlattenTable() - creating: createFlattenTable - ''' - - def __init__(self, - bigdl_type="float"): - super(FlattenTable, self).__init__(None, bigdl_type) - - -class GradientReversal(Model): - - ''' - >>> gradientReversal = GradientReversal(1e-5) - creating: createGradientReversal - ''' - - def __init__(self, - the_lambda=1, - bigdl_type="float"): - super(GradientReversal, self).__init__(None, bigdl_type, - the_lambda) - - -class HardShrink(Model): - - ''' - >>> hardShrink = HardShrink(1e-5) - creating: createHardShrink - ''' - - def __init__(self, - the_lambda=0.5, - bigdl_type="float"): - super(HardShrink, self).__init__(None, bigdl_type, - the_lambda) - - -class HardTanh(Model): - - ''' - >>> hardTanh = HardTanh(1e-5, 1e5, True) - creating: createHardTanh - ''' - - def __init__(self, - min_value=-1, - max_value=1, - inplace=False, - bigdl_type="float"): - super(HardTanh, self).__init__(None, bigdl_type, - min_value, - max_value, - inplace) - - -class Index(Model): - - ''' - >>> index = Index(1) - creating: createIndex - ''' - - def __init__(self, - dimension, - bigdl_type="float"): - super(Index, self).__init__(None, bigdl_type, - dimension) - - -class InferReshape(Model): - - ''' - >>> inferReshape = InferReshape([4, 0, 3, -1], False) - creating: createInferReshape - ''' - - def __init__(self, - size, - batch_mode=False, - bigdl_type="float"): - super(InferReshape, self).__init__(None, bigdl_type, - size, - batch_mode) - - -class JoinTable(Model): - - ''' - >>> joinTable = JoinTable(1, 1) - creating: createJoinTable - ''' - - def __init__(self, - dimension, - n_input_dims, - bigdl_type="float"): - super(JoinTable, self).__init__(None, bigdl_type, - dimension, - n_input_dims) - - -class L1Cost(Model): - - ''' - >>> l1Cost = L1Cost() - creating: createL1Cost - ''' - - def __init__(self, - bigdl_type="float"): - super(L1Cost, self).__init__(None, bigdl_type) - - -class L1Penalty(Model): - - ''' - >>> l1Penalty = L1Penalty(1, True, True) - creating: createL1Penalty - ''' - - def __init__(self, - l1weight, - size_average=False, - provide_output=True, - bigdl_type="float"): - super(L1Penalty, self).__init__(None, bigdl_type, - l1weight, - size_average, - provide_output) - - -class LeakyReLU(Model): - - ''' - >>> leakyReLU = LeakyReLU(1e-5, True) - creating: createLeakyReLU - ''' - - def __init__(self, - negval=0.01, - inplace=False, - bigdl_type="float"): - super(LeakyReLU, self).__init__(None, bigdl_type, - negval, - inplace) - - -class Log(Model): - - ''' - >>> log = Log() - creating: createLog - ''' - - def __init__(self, - bigdl_type="float"): - super(Log, self).__init__(None, bigdl_type) - - -class LogSigmoid(Model): - - ''' - >>> logSigmoid = LogSigmoid() - creating: createLogSigmoid - ''' - - def __init__(self, - bigdl_type="float"): - super(LogSigmoid, self).__init__(None, bigdl_type) - - -class LookupTable(Model): - - ''' - >>> lookupTable = LookupTable(1, 1, 1e-5, 1e-5, 1e-5, True) - creating: createLookupTable - ''' - - def __init__(self, - n_index, - n_output, - padding_value=0, - max_norm=DOUBLEMAX, - norm_type=2.0, - should_scale_grad_by_freq=False, - bigdl_type="float"): - super(LookupTable, self).__init__(None, bigdl_type, - n_index, - n_output, - padding_value, - max_norm, - norm_type, - should_scale_grad_by_freq) - - -class MM(Model): - - ''' - >>> mM = MM(True, True) - creating: createMM - ''' - - def __init__(self, - trans_a=False, - trans_b=False, - bigdl_type="float"): - super(MM, self).__init__(None, bigdl_type, - trans_a, - trans_b) - - -class MV(Model): - - ''' - >>> mV = MV(True) - creating: createMV - ''' - - def __init__(self, - trans=False, - bigdl_type="float"): - super(MV, self).__init__(None, bigdl_type, - trans) - - -class MapTable(Model): - - ''' - >>> mapTable = MapTable(Linear(100,10)) - creating: createLinear - creating: createMapTable - ''' - - def __init__(self, - module, - bigdl_type="float"): - super(MapTable, self).__init__(None, bigdl_type, - module) - - -class MaskedSelect(Model): - - ''' - >>> maskedSelect = MaskedSelect() - creating: createMaskedSelect - ''' - - def __init__(self, - bigdl_type="float"): - super(MaskedSelect, self).__init__(None, bigdl_type) - - -class Max(Model): - - ''' - >>> max = Max(1) - creating: createMax - ''' - - def __init__(self, - dim, - num_input_dims=INTMIN, - bigdl_type="float"): - super(Max, self).__init__(None, bigdl_type, - dim, - num_input_dims) - - -class Mean(Model): - - ''' - >>> mean = Mean(1, 1) - creating: createMean - ''' - - def __init__(self, - dimension=1, - n_input_dims=-1, - bigdl_type="float"): - super(Mean, self).__init__(None, bigdl_type, - dimension, - n_input_dims) - - -class Min(Model): - - ''' - >>> min = Min(1) - creating: createMin - ''' - - def __init__(self, - dim, - num_input_dims=INTMIN, - bigdl_type="float"): - super(Min, self).__init__(None, bigdl_type, - dim, - num_input_dims) - - -class MixtureTable(Model): - - ''' - >>> mixtureTable = MixtureTable() - creating: createMixtureTable - ''' - - def __init__(self, - bigdl_type="float"): - super(MixtureTable, self).__init__(None, bigdl_type) - - -class Mul(Model): - - ''' - >>> mul = Mul() - creating: createMul - ''' - - def __init__(self, - bigdl_type="float"): - super(Mul, self).__init__(None, bigdl_type) - - -class MulConstant(Model): - - ''' - >>> mulConstant = MulConstant(2.5) - creating: createMulConstant - ''' - - def __init__(self, - scalar, - inplace=False, - bigdl_type="float"): - super(MulConstant, self).__init__(None, bigdl_type, - scalar, - inplace) - - -class Narrow(Model): - - ''' - >>> narrow = Narrow(1, 1, 1) - creating: createNarrow - ''' - - def __init__(self, - dimension, - offset, - length=1, - bigdl_type="float"): - super(Narrow, self).__init__(None, bigdl_type, - dimension, - offset, - length) - - -class NarrowTable(Model): - - ''' - >>> narrowTable = NarrowTable(1, 1) - creating: createNarrowTable - ''' - - def __init__(self, - offset, - length=1, - bigdl_type="float"): - super(NarrowTable, self).__init__(None, bigdl_type, - offset, - length) - - -class Normalize(Model): - - ''' - >>> normalize = Normalize(1e-5, 1e-5) - creating: createNormalize - ''' - - def __init__(self, - p, - eps=1e-10, - bigdl_type="float"): - super(Normalize, self).__init__(None, bigdl_type, - p, - eps) - - -class PReLU(Model): - - ''' - >>> pReLU = PReLU(1) - creating: createPReLU - ''' - - def __init__(self, - n_output_plane=0, - bigdl_type="float"): - super(PReLU, self).__init__(None, bigdl_type, - n_output_plane) - - -class Padding(Model): - - ''' - >>> padding = Padding(1, 1, 1, 1e-5, 1) - creating: createPadding - ''' - - def __init__(self, - dim, - pad, - n_input_dim, - value=0.0, - n_index=1, - bigdl_type="float"): - super(Padding, self).__init__(None, bigdl_type, - dim, - pad, - n_input_dim, - value, - n_index) - - -class PairwiseDistance(Model): - - ''' - >>> pairwiseDistance = PairwiseDistance(2) - creating: createPairwiseDistance - ''' - - def __init__(self, - norm=2, - bigdl_type="float"): - super(PairwiseDistance, self).__init__(None, bigdl_type, - norm) - - -class ParallelTable(Model): - - ''' - >>> parallelTable = ParallelTable() - creating: createParallelTable - ''' - - def __init__(self, - bigdl_type="float"): - super(ParallelTable, self).__init__(None, bigdl_type) - - -class Power(Model): - - ''' - >>> power = Power(1e-5) - creating: createPower - ''' - - def __init__(self, - power, - scale=1.0, - shift=0.0, - bigdl_type="float"): - super(Power, self).__init__(None, bigdl_type, - power, - scale, - shift) - - -class RReLU(Model): - - ''' - >>> rReLU = RReLU(1e-5, 1e5, True) - creating: createRReLU - ''' - - def __init__(self, - lower=1.0/8, - upper=1.0/3, - inplace=False, - bigdl_type="float"): - super(RReLU, self).__init__(None, bigdl_type, - lower, - upper, - inplace) - - -class ReLU6(Model): - - ''' - >>> reLU6 = ReLU6(True) - creating: createReLU6 - ''' - - def __init__(self, - inplace=False, - bigdl_type="float"): - super(ReLU6, self).__init__(None, bigdl_type, - inplace) - - -class Replicate(Model): - - ''' - >>> replicate = Replicate(2) - creating: createReplicate - ''' - - def __init__(self, - n_features, - dim=1, - n_dim=INTMAX, - bigdl_type="float"): - super(Replicate, self).__init__(None, bigdl_type, - n_features, - dim, - n_dim) - - -class RoiPooling(Model): - - ''' - >>> roiPooling = RoiPooling(1, 1, 1e-5) - creating: createRoiPooling - ''' - - def __init__(self, - pooled_w, - pooled_h, - spatial_scale, - bigdl_type="float"): - super(RoiPooling, self).__init__(None, bigdl_type, - pooled_w, - pooled_h, - spatial_scale) - - -class Scale(Model): - - ''' - >>> scale = Scale([1,2]) - creating: createScale - ''' - - def __init__(self, - size, - bigdl_type="float"): - super(Scale, self).__init__(None, bigdl_type, - size) - - -class SelectTable(Model): - - ''' - >>> selectTable = SelectTable(1) - creating: createSelectTable - ''' - - def __init__(self, - dimension, - bigdl_type="float"): - super(SelectTable, self).__init__(None, bigdl_type, - dimension) - - -class Sigmoid(Model): - - ''' - >>> sigmoid = Sigmoid() - creating: createSigmoid - ''' - - def __init__(self, - bigdl_type="float"): - super(Sigmoid, self).__init__(None, bigdl_type) - - -class SoftMax(Model): - - ''' - >>> softMax = SoftMax() - creating: createSoftMax - ''' - - def __init__(self, - bigdl_type="float"): - super(SoftMax, self).__init__(None, bigdl_type) - - -class SoftMin(Model): - - ''' - >>> softMin = SoftMin() - creating: createSoftMin - ''' - - def __init__(self, - bigdl_type="float"): - super(SoftMin, self).__init__(None, bigdl_type) - - -class SoftPlus(Model): - - ''' - >>> softPlus = SoftPlus(1e-5) - creating: createSoftPlus - ''' - - def __init__(self, - beta=1.0, - bigdl_type="float"): - super(SoftPlus, self).__init__(None, bigdl_type, - beta) - - -class SoftShrink(Model): - - ''' - >>> softShrink = SoftShrink(1e-5) - creating: createSoftShrink - ''' - - def __init__(self, - the_lambda=0.5, - bigdl_type="float"): - super(SoftShrink, self).__init__(None, bigdl_type, - the_lambda) - - -class SoftSign(Model): - - ''' - >>> softSign = SoftSign() - creating: createSoftSign - ''' - - def __init__(self, - bigdl_type="float"): - super(SoftSign, self).__init__(None, bigdl_type) - - -class SpatialDilatedConvolution(Model): - - ''' - >>> spatialDilatedConvolution = SpatialDilatedConvolution(1, 1, 1, 1) - creating: createSpatialDilatedConvolution - ''' - - def __init__(self, - n_input_plane, - n_output_plane, - kw, - kh, - dw=1, - dh=1, - pad_w=0, - pad_h=0, - dilation_w=1, - dilation_h=1, - init_method='default', - bigdl_type="float"): - super(SpatialDilatedConvolution, self).__init__(None, bigdl_type, - n_input_plane, - n_output_plane, - kw, - kh, - dw, - dh, - pad_w, - pad_h, - dilation_w, - dilation_h, - init_method) - - -class SpatialFullConvolution(Model): - - ''' - >>> spatialFullConvolution = SpatialFullConvolution(1, 1, 1, 1) - creating: createSpatialFullConvolution - ''' - - def __init__(self, - n_input_plane, - n_output_plane, - kw, - kh, - dw=1, - dh=1, - pad_w=0, - pad_h=0, - adj_w=0, - adj_h=0, - n_group=1, - no_bias=False, - init_method='default', - bigdl_type="float"): - super(SpatialFullConvolution, self).__init__(None, bigdl_type, - n_input_plane, - n_output_plane, - kw, - kh, - dw, - dh, - pad_w, - pad_h, - adj_w, - adj_h, - n_group, - no_bias, - init_method) - - -class SpatialShareConvolution(Model): - - ''' - >>> spatialShareConvolution = SpatialShareConvolution(1, 1, 1, 1) - creating: createSpatialShareConvolution - ''' - - def __init__(self, - n_input_plane, - n_output_plane, - kernel_w, - kernel_h, - stride_w=1, - stride_h=1, - pad_w=0, - pad_h=0, - n_group=1, - propagate_back=True, - init_method='default', - bigdl_type="float"): - super(SpatialShareConvolution, self).__init__(None, bigdl_type, - n_input_plane, - n_output_plane, - kernel_w, - kernel_h, - stride_w, - stride_h, - pad_w, - pad_h, - n_group, - propagate_back, - init_method) - - -class SpatialZeroPadding(Model): - - ''' - >>> spatialZeroPadding = SpatialZeroPadding(1, 1, 1, 1) - creating: createSpatialZeroPadding - ''' - - def __init__(self, - pad_left, - pad_right, - pad_top, - pad_bottom, - bigdl_type="float"): - super(SpatialZeroPadding, self).__init__(None, bigdl_type, - pad_left, - pad_right, - pad_top, - pad_bottom) - - -class SplitTable(Model): - - ''' - >>> splitTable = SplitTable(1, 1) - creating: createSplitTable - ''' - - def __init__(self, - dimension, - n_input_dims=-1, - bigdl_type="float"): - super(SplitTable, self).__init__(None, bigdl_type, - dimension, - n_input_dims) - - -class Sqrt(Model): - - ''' - >>> sqrt = Sqrt() - creating: createSqrt - ''' - - def __init__(self, - bigdl_type="float"): - super(Sqrt, self).__init__(None, bigdl_type) - - -class Square(Model): - - ''' - >>> square = Square() - creating: createSquare - ''' - - def __init__(self, - bigdl_type="float"): - super(Square, self).__init__(None, bigdl_type) - - -class Squeeze(Model): - - ''' - >>> squeeze = Squeeze(1) - creating: createSqueeze - ''' - - def __init__(self, - dim, - num_input_dims=INTMIN, - bigdl_type="float"): - super(Squeeze, self).__init__(None, bigdl_type, - dim, - num_input_dims) - - -class Sum(Model): - - ''' - >>> sum = Sum(1, 1, True) - creating: createSum - ''' - - def __init__(self, - dimension=1, - n_input_dims=-1, - size_average=False, - bigdl_type="float"): - super(Sum, self).__init__(None, bigdl_type, - dimension, - n_input_dims, - size_average) - - -class TanhShrink(Model): - - ''' - >>> tanhShrink = TanhShrink() - creating: createTanhShrink - ''' - - def __init__(self, - bigdl_type="float"): - super(TanhShrink, self).__init__(None, bigdl_type) - - -class Threshold(Model): - - ''' - >>> threshold = Threshold(1e-5, 1e-5, True) - creating: createThreshold - ''' - - def __init__(self, - th=1e-6, - v=0.0, - ip=False, - bigdl_type="float"): - super(Threshold, self).__init__(None, bigdl_type, - th, - v, - ip) - - -class Unsqueeze(Model): - - ''' - >>> unsqueeze = Unsqueeze(1, 1) - creating: createUnsqueeze - ''' - - def __init__(self, - pos, - num_input_dims=INTMIN, - bigdl_type="float"): - super(Unsqueeze, self).__init__(None, bigdl_type, - pos, - num_input_dims) - - -class Reshape(Model): - ''' - >>> reshape = Reshape([1, 28, 28]) - creating: createReshape - ''' - - def __init__(self, size, bigdl_type="float"): - super(Reshape, self).__init__(None, bigdl_type, size) - - -def _test(): - import doctest - from pyspark import SparkContext - from nn import layer - from util.common import init_engine - from util.common import create_spark_conf - globs = layer.__dict__.copy() - sc = SparkContext(master="local[4]", appName="test layer", - conf=create_spark_conf()) - globs['sc'] = sc - init_engine() - - (failure_count, test_count) = doctest.testmod(globs=globs, - optionflags=doctest.ELLIPSIS) - if failure_count: - exit(-1) - -if __name__ == "__main__": - _test() diff --git a/python/dllib/src/bigdl/dllib/optim/__init__.py b/python/dllib/src/bigdl/dllib/optim/__init__.py deleted file mode 100644 index 2151a805423..00000000000 --- a/python/dllib/src/bigdl/dllib/optim/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -# -# Copyright 2016 The BigDL Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# diff --git a/python/dllib/src/bigdl/dllib/optim/optimizer.py b/python/dllib/src/bigdl/dllib/optim/optimizer.py deleted file mode 100644 index 4272e54f696..00000000000 --- a/python/dllib/src/bigdl/dllib/optim/optimizer.py +++ /dev/null @@ -1,169 +0,0 @@ -# -# Copyright 2016 The BigDL Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - - -from util.common import callBigDlFunc -from util.common import JavaValue -from util.common import callJavaFunc -from pyspark import SparkContext -import numpy as np -import os -from distutils.dir_util import mkpath - - -import sys -if sys.version >= '3': - long = int - unicode = str - - -class MaxIteration(JavaValue): - ''' - >>> maxIteration = MaxIteration(20) - creating: createMaxIteration - ''' - def __init__(self, max, bigdl_type="float"): - JavaValue.__init__(self, None, bigdl_type, max) - - -class MaxEpoch(JavaValue): - ''' - >>> maxEpoch = MaxEpoch(2) - creating: createMaxEpoch - ''' - def __init__(self, max_epoch, bigdl_type="float"): - JavaValue.__init__(self, None, bigdl_type, max_epoch) - - -class EveryEpoch(JavaValue): - ''' - >>> everyEpoch = EveryEpoch() - creating: createEveryEpoch - ''' - def __init__(self, bigdl_type="float"): - JavaValue.__init__(self, None, bigdl_type) - - -class SeveralIteration(JavaValue): - ''' - >>> serveralIteration = SeveralIteration(2) - creating: createSeveralIteration - ''' - def __init__(self, interval, bigdl_type="float"): - JavaValue.__init__(self, None, bigdl_type, interval) - - -class Poly(JavaValue): - ''' - >>> poly = Poly(0.5, 2) - creating: createPoly - ''' - def __init__(self, power, max_iteration, bigdl_type="float"): - JavaValue.__init__(self, None, bigdl_type, power, max_iteration) - - -class Step(JavaValue): - ''' - >>> step = Step(2, 0.3) - creating: createStep - ''' - def __init__(self, step_size, gamma, bigdl_type="float"): - JavaValue.__init__(self, None, bigdl_type, step_size, gamma) - - -class Optimizer(JavaValue): - - def __init__(self, - model, - training_rdd, - criterion, - end_trigger, - batch_size, - optim_method="SGD", - state={}, - bigdl_type="float"): - JavaValue.__init__(self, None, bigdl_type, model.value, - training_rdd, criterion, optim_method, - state, end_trigger, batch_size) - - def setvalidation(self, batch_size, val_rdd, trigger, val_method=["Top1Accuracy"]): - callBigDlFunc(self.bigdl_type, "setValidation", self.value, batch_size, - trigger, val_rdd, val_method) - - def setcheckpoint(self, checkpoint_trigger, - checkpoint_path, isOverWrite=True): - if not os.path.exists(checkpoint_path): - mkpath(checkpoint_path) - callBigDlFunc(self.bigdl_type, "setCheckPoint", self.value, - checkpoint_trigger, checkpoint_path, isOverWrite) - - # return a module - def optimize(self): - jmodel = callJavaFunc(SparkContext.getOrCreate(), self.value.optimize) - from nn.layer import Model - return Model.of(jmodel) - - def set_train_summary(self, summary): - callBigDlFunc(self.bigdl_type, "setTrainSummary", self.value, - summary) - return self - - def set_val_summary(self, summary): - callBigDlFunc(self.bigdl_type, "setValSummary", self.value, - summary) - return self - - -class TrainSummary(JavaValue, ): - def __init__(self, log_dir, app_name, bigdl_type="float"): - JavaValue.__init__(self, None, bigdl_type, log_dir, app_name) - - def read_scalar(self, tag): - return callBigDlFunc(self.bigdl_type, "summaryReadScalar", self.value, - tag) - - def set_summary_trigger(self, name, trigger): - return callBigDlFunc(self.bigdl_type, "summarySetTrigger", self.value, - name, trigger) - - -class ValidationSummary(JavaValue): - def __init__(self, log_dir, app_name, bigdl_type="float"): - JavaValue.__init__(self, None, bigdl_type, log_dir, app_name) - - def read_scalar(self, tag): - return callBigDlFunc(self.bigdl_type, "summaryReadScalar", self.value, - tag) - - -def _test(): - import doctest - from pyspark import SparkContext - from optim import optimizer - from util.common import init_engine - from util.common import create_spark_conf - globs = optimizer.__dict__.copy() - sc = SparkContext(master="local[4]", appName="test optimizer", - conf=create_spark_conf()) - init_engine() - globs['sc'] = sc - (failure_count, test_count) = doctest.testmod(globs=globs, - optionflags=doctest.ELLIPSIS) - if failure_count: - exit(-1) - -if __name__ == "__main__": - _test() diff --git a/python/dllib/src/bigdl/utils/__init__.py b/python/dllib/src/bigdl/utils/__init__.py deleted file mode 100644 index 2151a805423..00000000000 --- a/python/dllib/src/bigdl/utils/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -# -# Copyright 2016 The BigDL Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# diff --git a/python/dllib/src/bigdl/utils/common.py b/python/dllib/src/bigdl/utils/common.py deleted file mode 100644 index bbac74231b2..00000000000 --- a/python/dllib/src/bigdl/utils/common.py +++ /dev/null @@ -1,261 +0,0 @@ -# -# Copyright 2016 The BigDL Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import sys -from py4j.protocol import Py4JJavaError -from py4j.java_gateway import JavaObject -from py4j.java_collections import ListConverter, JavaArray, JavaList, JavaMap - -from pyspark import RDD, SparkContext -from pyspark.serializers import PickleSerializer, AutoBatchedSerializer -from pyspark.sql import DataFrame, SQLContext -from pyspark.mllib.common import callJavaFunc -from pyspark import SparkConf -import numpy as np - -if sys.version >= '3': - long = int - unicode = str - - -class JavaValue(object): - def jvm_class_constructor(self): - name = "create" + self.__class__.__name__ - print("creating: " + name) - return name - - def __init__(self, jvalue, bigdl_type, *args): - self.value = jvalue if jvalue else callBigDlFunc( - bigdl_type, self.jvm_class_constructor(), *args) - self.bigdl_type = bigdl_type - - -class TestResult(): - def __init__(self, result, total_num, method): - self.result = result - self.total_num = total_num - self.method = method - - def __reduce__(self): - return (TestResult, (self.result, self.total_num, self.method)) - - def __str__(self): - return "Test result: %s, total_num: %s, method: %s" % ( - self.result, self.total_num, self.method) - - -class Sample(object): - def __init__(self, features, label, features_shape, label_shape, - bigdl_type="float"): - def get_dtype(): - if "float" == bigdl_type: - return "float32" - else: - return "float64" - self.features = np.array(features, dtype=get_dtype()).reshape(features_shape) # noqa - self.label = np.array(label, dtype=get_dtype()).reshape(label_shape) - self.bigdl_type = bigdl_type - - @classmethod - def from_ndarray(cls, features, label, bigdl_type="float"): - return cls( - features=[float(i) for i in features.ravel()], - label=[float(i) for i in label.ravel()], - features_shape=list(features.shape), - label_shape=list(label.shape) if label.shape else [label.size], - bigdl_type=bigdl_type) - - @classmethod - def flatten(cls, a_ndarray): - """ - Utility method to flatten a ndarray - :return (storage, shape) - >>> import numpy as np - >>> from util.common import Sample - >>> np.random.seed(123) - >>> data = np.random.uniform(0, 1, (2, 3)) - >>> (storage, shape) = Sample.flatten(data) - >>> shape - [2, 3] - >>> (storage, shape) = Sample.flatten(np.array(2)) - >>> shape - [1] - """ - storage = [float(i) for i in a_ndarray.ravel()] - shape = list(a_ndarray.shape) if a_ndarray.shape else [a_ndarray.size] - return storage, shape - - def __reduce__(self): - (features_storage, features_shape) = Sample.flatten(self.features) - (label_storage, label_shape) = Sample.flatten(self.label) - return (Sample, ( - features_storage, label_storage, features_shape, label_shape, - self.bigdl_type)) - - def __str__(self): - return "features: %s, label: %s," % (self.features, self.label) - - -_picklable_classes = [ - 'LinkedList', - 'SparseVector', - 'DenseVector', - 'DenseMatrix', - 'Rating', - 'LabeledPoint', - 'Sample', - 'TestResult' -] - - -def init_engine(bigdl_type="float"): - callBigDlFunc(bigdl_type, "initEngine") - - -def get_bigdl_conf(): - bigdl_conf_file = "spark-bigdl.conf" - bigdl_python_wrapper = "python-api.zip" - - def load_conf(conf_str): - return dict(line.split() for line in conf_str.split("\n") if - "#" not in line and line.strip()) - - for p in sys.path: - if bigdl_conf_file in p: - with open(p) as conf_file: - return load_conf(conf_file.read()) - if bigdl_python_wrapper in p: - import zipfile - with zipfile.ZipFile(p, 'r') as zip_conf: - return load_conf(zip_conf.read(bigdl_conf_file)) - raise Exception("Cannot find spark-bigdl.conf.Pls add it to PYTHONPATH.") - - -def create_spark_conf(): - bigdl_conf = get_bigdl_conf() - sparkConf = SparkConf() - sparkConf.setAll(bigdl_conf.items()) - return sparkConf - - -def callBigDlFunc(bigdl_type, name, *args): - """ Call API in PythonBigDL """ - sc = SparkContext.getOrCreate() - if bigdl_type == "float": - api = getattr( - sc._jvm.com.intel.analytics.bigdl.python.api.PythonBigDL.ofFloat(), - name) - elif bigdl_type == "double": - api = getattr( - sc._jvm.com.intel.analytics.bigdl.python.api.PythonBigDL.ofDouble(), - name) - else: - raise Exception("Not supported bigdl_type: %s" % bigdl_type) - return callJavaFunc(sc, api, *args) - - -def _java2py(sc, r, encoding="bytes"): - if isinstance(r, JavaObject): - clsName = r.getClass().getSimpleName() - # convert RDD into JavaRDD - if clsName != 'JavaRDD' and clsName.endswith("RDD"): - r = r.toJavaRDD() - clsName = 'JavaRDD' - - if clsName == 'JavaRDD': - jrdd = sc._jvm.SerDe.javaToPython(r) - return RDD(jrdd, sc) - - if clsName == 'DataFrame': - return DataFrame(r, SQLContext.getOrCreate(sc)) - - if clsName in _picklable_classes: - r = sc._jvm.org.apache.spark.bigdl.api.python.BigDLSerDe.dumps(r) - elif isinstance(r, (JavaArray, JavaList, JavaMap)): - try: - r = sc._jvm.org.apache.spark.bigdl.api.python.BigDLSerDe.dumps( - r) - except Py4JJavaError: - pass # not pickable - - if isinstance(r, (bytearray, bytes)): - r = PickleSerializer().loads(bytes(r), encoding=encoding) - return r - - -def callJavaFunc(sc, func, *args): - """ Call Java Function """ - args = [_py2java(sc, a) for a in args] - result = func(*args) - return _java2py(sc, result) - - -def _to_java_object_rdd(rdd): - """ Return a JavaRDD of Object by unpickling - - It will convert each Python object into Java object by Pyrolite, whenever - the RDD is serialized in batch or not. - """ - rdd = rdd._reserialize(AutoBatchedSerializer(PickleSerializer())) - return \ - rdd.ctx._jvm.org.apache.spark.bigdl.api.python.BigDLSerDe.pythonToJava( - rdd._jrdd, True) - - -def _py2java(sc, obj): - """ Convert Python object into Java """ - if isinstance(obj, RDD): - obj = _to_java_object_rdd(obj) - elif isinstance(obj, DataFrame): - obj = obj._jdf - elif isinstance(obj, SparkContext): - obj = obj._jsc - elif isinstance(obj, (list, tuple)): - obj = ListConverter().convert([_py2java(sc, x) for x in obj], - sc._gateway._gateway_client) - elif isinstance(obj, dict): - result = {} - for (key, value) in obj.iteritems(): - result[key] = _py2java(sc, value) if isinstance(value, JavaValue) else value # noqa - obj = result - - elif isinstance(obj, JavaValue): - obj = obj.value - elif isinstance(obj, JavaObject): - pass - elif isinstance(obj, (int, long, float, bool, bytes, unicode)): - pass - else: - data = bytearray(PickleSerializer().dumps(obj)) - obj = sc._jvm.org.apache.spark.bigdl.api.python.BigDLSerDe.loads(data) - return obj - - -def _test(): - import doctest - from pyspark import SparkContext - from nn import layer - globs = layer.__dict__.copy() - sc = SparkContext(master="local[2]", appName="test common utility") - globs['sc'] = sc - (failure_count, test_count) = doctest.testmod(globs=globs, - optionflags=doctest.ELLIPSIS) - if failure_count: - exit(-1) - - -if __name__ == "__main__": - _test() From 964e5181411a3f26f4184261de5dd0de0542f050 Mon Sep 17 00:00:00 2001 From: jenniew Date: Wed, 22 Mar 2017 22:58:31 -0700 Subject: [PATCH 014/823] load caffe, torch, bigdl model add load caffe test move load_caffe_test location Final Commit of 4 optimization algorithms, RMSprop, Adam, Adadelta, Adamax. Unit Test Passed. @whatbeg 2017.3.21 fix SoftmaxWithCriterion move new optim methods from wrong place --- load_caffe_test.py | 138 +++++++++++++++++++++++++++++++ python/dllib/test/dev/modules.py | 7 ++ resources/test.caffemodel | Bin 0 -> 924 bytes resources/test.prototxt | 52 ++++++++++++ 4 files changed, 197 insertions(+) create mode 100644 load_caffe_test.py create mode 100644 resources/test.caffemodel create mode 100644 resources/test.prototxt diff --git a/load_caffe_test.py b/load_caffe_test.py new file mode 100644 index 00000000000..808dabf9337 --- /dev/null +++ b/load_caffe_test.py @@ -0,0 +1,138 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Still in experimental stage! + +from nn.layer import * +from optim.optimizer import * +from util.common import * +import numpy as np +import unittest + + +class TestLoadCaffe(unittest.TestCase): + + def setUp(self): + sparkConf = create_spark_conf() + self.sc = SparkContext(master="local[4]", appName="test model", + conf=sparkConf) + init_engine() + + def tearDown(self): + self.sc.stop() + + def test_load_caffe(self): + # test load caffe with match all parameters + resource_path = os.path.join(os.path.split(__file__)[0], "resources") + proto_txt = os.path.join(resource_path, "test.prototxt") + model_path = os.path.join(resource_path, "test.caffemodel") + + module = Sequential()\ + .add(SpatialConvolution(3, 4, 2, 2).set_name("conv"))\ + .add(SpatialConvolution(4, 3, 2, 2).set_name("conv2"))\ + .add(Linear(2, 27, with_bias=False).set_name("ip")) + + model = Model.load_caffe(module, proto_txt, model_path, bigdl_type="float") + + parameters = model.parameters() + + conv1_weight = np.array([ + 0.4156779647, 0.3547672033, 0.1817495823, -0.1393318474, + 0.4004031420, 0.0634599924, 0.1571258903, 0.4180541039, + 0.3160640597, -0.0705609620, 0.4400904775, 0.1356888413, + 0.3336675167, -0.4155709147, 0.3805110455, -0.2870244980, + -0.4660916030, 0.1869867444, 0.1779280305, -0.4752498865, + -0.1661637127, 0.3539937139, -0.4798593223, -0.1620789468, + -0.3706701994, 0.4643850327, -0.4701072574, 0.2113500834, + -0.1643440127, -0.4747982025, -0.1300015152, 0.3333964944, + 0.1151610613, -0.4214298427, -0.4075299501, -0.1441932321, + -0.3215276599, 0.4862193465, 0.0050434470, 0.4745523334, + 0.3657383919, -0.2879499197, 0.3388324380, 0.3669666648, + -0.4454920888, -0.4200569391, -0.4690187573, -0.4590228796])\ + .astype("float")\ + .reshape((4, 3, 2, 2)) + + conv1_bias = np.array([0.0458712392, -0.0029324144, -0.0251041390, 0.0052924110])\ + .astype("float") + + conv2_weight = np.array([ + 0.0154178329, 0.0157190431, 0.0033829932, -0.0048461366, + -0.0026736879, 0.0009068546, -0.0020218866, 0.0096962797, + 0.0100709666, 0.0152738532, -0.0048784190, -0.0099342400, + -0.0188637581, -0.0053443452, 0.0035097739, -0.0104582068, + 0.0212461911, -0.0026065058, 0.0000952507, 0.0113442009, + 0.0247142352, 0.0033546593, -0.0127880797, 0.0040104976, + -0.0121186078, 0.0055492506, -0.0097251972, 0.0087026395, + -0.0078147361, 0.0101692677, -0.0027364481, 0.0007095702, + -0.0088762743, 0.0061115879, 0.0048167249, -0.0107875718, + -0.0249741413, -0.0018652071, 0.0028419730, 0.0255292989, + -0.0091862874, 0.0010728909, 0.0009157739, 0.0073709050, + -0.0088602817, -0.0093507599, 0.0070853345, -0.0074293613])\ + .astype("float")\ + .reshape((3, 4, 2, 2)) + + conv2_bias = np.array([0, 0, 0]).astype("float") + + linear_weight = np.array([ + 0.0189033747, 0.0401176214, 0.0525088012, 0.3013394773, + -0.0363914967, -0.3332226574, -0.2289898694, 0.3278202116, + 0.1829113662, 0.1653866768, -0.2218630016, -0.2914664149, + 0.0594480336, 0.1987790167, -0.2698714137, -0.2847212255, + 0.2896176279, 0.3278110921, -0.2233058512, 0.0355354548, + -0.2735285461, 0.1467721164, -0.1557070315, -0.2237440944, + 0.2817622125, -0.0810049772, 0.1050063074, -0.0378594697, + 0.2178583443, 0.0811733305, -0.0678446293, 0.0180019736, + -0.0949532837, 0.2528320253, -0.0265761316, -0.0096390843, + -0.2360083759, 0.1942299902, -0.3302336931, -0.2042815089, + -0.3027454615, 0.1254911423, 0.2114857137, 0.0392392874, + 0.1668677032, 0.0506658256, 0.1139862537, 0.2874754369, + -0.3273061812, 0.2115428150, -0.2002333999, -0.1621897519, + 0.0032395422, 0.2072965205]).astype("float").reshape((27, 2)) + + self.assertTrue(np.allclose(parameters["conv"]["weight"], + conv1_weight, atol=1e-6, rtol=0)) + self.assertTrue(np.allclose(parameters["conv"]["bias"], + conv1_bias, atol=1e-6, rtol=0)) + self.assertTrue(np.allclose(parameters["conv2"]["weight"], + conv2_weight, atol=1e-6, rtol=0)) + self.assertTrue(np.allclose(parameters["conv2"]["bias"], + conv2_bias, atol=1e-6, rtol=0)) + self.assertTrue(np.allclose(parameters["ip"]["weight"], + linear_weight, atol=1e-6, rtol=0)) + + # test load caffe not match all parameters + module = Sequential()\ + .add(SpatialConvolution(3, 4, 2, 2).set_name("conv"))\ + .add(SpatialConvolution(4, 3, 2, 2).set_name("conv3"))\ + .add(Linear(2, 27, with_bias=False).set_name("ip")) + + model = Model.load_caffe(module, proto_txt, model_path, match_all=False) + + parameters = model.parameters() + + self.assertTrue(np.allclose(parameters["conv"]["weight"], + conv1_weight, atol=1e-6, rtol=0)) + self.assertTrue(np.allclose(parameters["conv"]["bias"], + conv1_bias, atol=1e-6, rtol=0)) + self.assertFalse(np.allclose(parameters["conv3"]["weight"], + conv2_weight, atol=1e-6, rtol=0)) + self.assertFalse(np.allclose(parameters["conv3"]["bias"], + conv2_bias, atol=1e-6, rtol=0)) + self.assertTrue(np.allclose(parameters["ip"]["weight"], + linear_weight, atol=1e-6, rtol=0)) + + +if __name__ == "__main__": + unittest.main() diff --git a/python/dllib/test/dev/modules.py b/python/dllib/test/dev/modules.py index 9d9b1dbab97..d6e3410711e 100644 --- a/python/dllib/test/dev/modules.py +++ b/python/dllib/test/dev/modules.py @@ -83,3 +83,10 @@ def __hash__(self): "test.simple_integration_test" ] ) + +test_load_caffe = Module( + name="load_caffe_test", + python_test_goals=[ + "test.load_caffe_test" + ] +) diff --git a/resources/test.caffemodel b/resources/test.caffemodel new file mode 100644 index 0000000000000000000000000000000000000000..9a5c515d333866cd6384cbfbb94903594acdaf9b GIT binary patch literal 924 zcmZ9~YfMvj6aetv+uK?%wL+1{P-#~N(NaePx{Uw5_t3yb#*AAPH=7Wh4P{Gxw9p5^ zi3;ePX=+3fKcFHau30nOW(G#@?Fkc^L@`lxJVJCBADJ;K0-I#`VN{YmJYP<79v{vx zhm9$9yDHsV{644K)vA1xRXAPcexFuR6io)kVfWds)L$wUt7I8Uu28fp!!*l8m$Wej z|E{LInl!v>kgJrji^7|?K;6~>^@|A5?HX*Ou7UMMt#Ist9?UTVka6=QyjNEr2^8N2 zW}JnpfiQIT+(KLbMHrntga=MNMc=kOxRm`4OcqaIe})mm&;hz1{y^naI#%~aaJ2gg zJp6GCmu#^?U(pimN}j+}odUv(Dto1~%#%N#RefRid%aG(YXv)aif>i6(5xb=mHir5v}Ty?%z7<2 zTFvM7)bJ}h)^qFnnngA@OMH2LD>+%qi0QXe`LnWgJ}d7_K7HhbnA>fTGWRt}&ARVJ zp81@6Xv>s#s?YF`ZWzVlpD({V=XNe%=fA=a4VQ4qU#*sQZraW1_ZLb#20A$Rr#fj* zqlUN7l}WPW8T|AoIii;)0CiC*Zk{ot_01}p(B0Li*1t^$}EKIH{9S8>Jp{A zmLLhnmXp@6pL6GoanjSAH>70K@0`ptAx6FA%#2sOQ7!vVM6Kjet0a|NPn#+7f1|V+ zqehygo#k3}k;_%)v3lGNf2l8J*2W-r`+ejtWgMa`BTf9qHE#)3@#}=lq9(A+Plzl% z7REKzn{eJx2vS#Zpf6bp8gnToa*f!#%PU+6u7o3|08amM5W7D(3?l;}_%Q6imi=+U zvzQ>dRwhEYLWicsWtdh_4`UBbM}qSPg?-wEk^IEZU^#pv=*I(*(W(aiPUXc&j<*)f zox_sl$ztj4?<$boF9gSeZ_$*s8hzAJ?5dGL=K&jRJg*lv{APryy&nrL6_14JtOjU5 q9KtirHh4LA6>1YQu;!z+T;Tn+Kp2*$7h0+C9F2;Om8JaiO@9NT-*9FC literal 0 HcmV?d00001 diff --git a/resources/test.prototxt b/resources/test.prototxt new file mode 100644 index 00000000000..f495aae242d --- /dev/null +++ b/resources/test.prototxt @@ -0,0 +1,52 @@ +name: "convolution" +input: "data" +input_dim: 1 +input_dim: 3 +input_dim: 5 +input_dim: 5 +layer { + name: "conv" + type: "Convolution" + bottom: "data" + top: "conv" + convolution_param { + num_output: 4 + kernel_size: 2 + weight_filler { + type: "xavier" + } + bias_filler { + type: "gaussian" + std: 0.02 + } + } +} + +layer { + name: "conv2" + type: "Convolution" + bottom: "conv" + top: "conv2" + convolution_param { + num_output: 3 + kernel_size: 2 + stride: 1 + weight_filler { + type: "gaussian" + std: 0.01 + } + } +} +layer { + name: "ip" + type: "InnerProduct" + bottom: "conv2" + top: "out" + inner_product_param { + num_output: 2 + weight_filler { + type: "xavier" + } + bias_term: false + } +} From c0bf35cc23dde7b052da7dc0a854fe7a2c5a1622 Mon Sep 17 00:00:00 2001 From: zhangli Date: Tue, 7 Mar 2017 10:33:41 +0800 Subject: [PATCH 015/823] add module.predict --- simple_integration_test.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/simple_integration_test.py b/simple_integration_test.py index 5d6e8913f4b..7a12262e4cb 100644 --- a/simple_integration_test.py +++ b/simple_integration_test.py @@ -107,6 +107,20 @@ def gen_rand_sample(): for test_result in test_results: print(test_result) + def test_predict(self): + np.random.seed(100) + total_length = 6 + features = np.random.uniform(0, 1, (total_length, 2)) + label = (features).sum() + 0.4 + predict_data = self.sc.parallelize(range(0, total_length)).map( + lambda i: Sample.from_ndarray(features[i], label)) + model = Linear(2, 1, "Xavier").set_name("linear1").set_seed(1234).reset() + predict_result = model.predict(predict_data) + p = predict_result.take(6) + ground_label = np.array([[-0.47596836], [-0.37598032], [-0.00492062], + [-0.5906958], [-0.12307882], [-0.77907401]], dtype="float32") + for i in range(0, total_length): + self.assertTrue(np.allclose(p[i], ground_label[i], atol=1e-6, rtol=0)) if __name__ == "__main__": unittest.main(failfast=True) From 17777f0fd9e282dca85e30541210e05f66d9a9a8 Mon Sep 17 00:00:00 2001 From: zhichao-li Date: Fri, 31 Mar 2017 10:19:06 +0800 Subject: [PATCH 016/823] add diff --- python/dllib/test/dev/diff.py | 75 ++++++++++++++++++++++++++++++++ python/dllib/test/dev/run-all.sh | 2 +- python/dllib/test/dev/run-diff | 24 ++++++++++ 3 files changed, 100 insertions(+), 1 deletion(-) create mode 100755 python/dllib/test/dev/diff.py create mode 100755 python/dllib/test/dev/run-diff diff --git a/python/dllib/test/dev/diff.py b/python/dllib/test/dev/diff.py new file mode 100755 index 00000000000..aeeeefbac81 --- /dev/null +++ b/python/dllib/test/dev/diff.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python + + +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os +import re +from os.path import isfile, join + +scala_nn_root = "./spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/" +python_nn_root = "./pyspark/dl/nn/" + + +def extract_scala_class(class_path): + exclude_key_words = set(["*", "abstract"]) + include_key_words = set(["Module", "Criterion", "Container", "Cell", "TensorNumeric"]) # noqa + content = "\n".join([line for line in open(class_path).readlines() if all([key not in line for key in exclude_key_words])]) # noqa + match = re.search(r"class ([\w]+)[^{]+", content) + if match and any([key in match.group() for key in include_key_words]): + return match.group(1) + else: + return None + + +def get_scala_classes(scala_nn_root): + raw_result = [extract_scala_class(join(scala_nn_root, name)) for name in os.listdir(scala_nn_root) if isfile(join(scala_nn_root, name))] # noqa + return set( + class_name for class_name in raw_result if class_name is not None) + + +def get_python_classes(nn_root): + exclude_classes = set(["Criterion", "Model"]) + raw_classes = [] + python_nn_layers = [join(nn_root, name) for name in os.listdir(nn_root) if isfile(join(nn_root, name)) and name.endswith('py') and "__" not in name] # noqa + for p in python_nn_layers: + with open(p) as f: + raw_classes.extend([line for line in f.readlines() if line.startswith("class")]) # noqa + classes = [name.split()[1].split("(")[0]for name in raw_classes] + return set([name for name in classes if name not in exclude_classes]) + + +scala_layers = get_scala_classes(scala_nn_root) +python_layers = get_python_classes(python_nn_root) + +print "scala_layers: {0}, {1}".format(len(scala_layers), scala_layers) +print("\n") +print "python_layers: {0}, {1}".format(len(python_layers), python_layers) +print "\nIn scala, not in python: ", +for name in scala_layers: + if name not in python_layers: + print name, + +print "\n" + +print "In python, not in scala: ", +for name in python_layers: + if name not in scala_layers: + print name, + +if len(scala_layers - python_layers) > 0: + raise Exception("There are difference between python and scala") diff --git a/python/dllib/test/dev/run-all.sh b/python/dllib/test/dev/run-all.sh index 5bf0694c852..5e147c670e1 100755 --- a/python/dllib/test/dev/run-all.sh +++ b/python/dllib/test/dev/run-all.sh @@ -17,4 +17,4 @@ # BASE="$(cd "`dirname $0`"; pwd)" -$BASE/lint-python && $BASE/run-tests \ No newline at end of file +$BASE/run-diff && $BASE/lint-python && $BASE/run-tests \ No newline at end of file diff --git a/python/dllib/test/dev/run-diff b/python/dllib/test/dev/run-diff new file mode 100755 index 00000000000..f0142655418 --- /dev/null +++ b/python/dllib/test/dev/run-diff @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +DL_PYTHON_HOME="$(cd "`dirname $0`"/../..; pwd)" + +BIGDL_HOME="$(cd "`dirname $0`"/../../..; pwd)" + +cd $BIGDL_HOME +exec $DL_PYTHON_HOME/test/dev/diff.py From 27f1f6b9f4f39ae7dcbf5167390ce6b8d7e2b56f Mon Sep 17 00:00:00 2001 From: Yiheng Wang Date: Sat, 1 Apr 2017 14:59:43 +0800 Subject: [PATCH 017/823] use variable in script to specify BigDL version --- python/dllib/test/dev/run-tests | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/dllib/test/dev/run-tests b/python/dllib/test/dev/run-tests index 217f6cb3777..c56cc38c74b 100755 --- a/python/dllib/test/dev/run-tests +++ b/python/dllib/test/dev/run-tests @@ -20,6 +20,7 @@ DL_PYTHON_HOME="$(cd "`dirname $0`"/../..; pwd)" BIGDL_HOME="$(cd "`dirname $0`"/../../..; pwd)" +BIGDL_VERSION=0.1.0-SNAPSHOT echo "BIGDL_HOME: $BIGDL_HOME" echo "SPARK_HOME": $SPARK_HOME @@ -31,7 +32,7 @@ PYSPARK_ZIP=`find $SPARK_HOME/python/lib -type f -iname '*.zip' | tr "\n" ":"` export PYTHONPATH=$PYSPARK_ZIP:$DL_PYTHON_HOME:$DL_PYTHON_HOME/dl:$DL_PYTHON_HOME/test/dev:$BIGDL_HOME/spark/dl/src/main/resources/spark-bigdl.conf -export SPARK_CLASSPATH=$BIGDL_HOME/spark/dl/target/bigdl-0.1.0-SNAPSHOT-jar-with-dependencies.jar +export SPARK_CLASSPATH=$BIGDL_HOME/spark/dl/target/bigdl-$BIGDL_VERSION-jar-with-dependencies.jar cd "`dirname $0`" exec $BIGDL_HOME/scripts/bigdl.sh -- python -u ./run-tests.py "$@" From f75fa3a5cb32a26888ac292ec8dc0704b059308e Mon Sep 17 00:00:00 2001 From: Yiheng Wang Date: Sat, 1 Apr 2017 16:57:28 +0800 Subject: [PATCH 018/823] change version to 0.2.0-snapshot --- python/dllib/test/dev/run-tests | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/dllib/test/dev/run-tests b/python/dllib/test/dev/run-tests index c56cc38c74b..43ce811c134 100755 --- a/python/dllib/test/dev/run-tests +++ b/python/dllib/test/dev/run-tests @@ -20,7 +20,7 @@ DL_PYTHON_HOME="$(cd "`dirname $0`"/../..; pwd)" BIGDL_HOME="$(cd "`dirname $0`"/../../..; pwd)" -BIGDL_VERSION=0.1.0-SNAPSHOT +BIGDL_VERSION=0.2.0-SNAPSHOT echo "BIGDL_HOME: $BIGDL_HOME" echo "SPARK_HOME": $SPARK_HOME From cf975c72cb665526590e6073c19a16ed96d011d2 Mon Sep 17 00:00:00 2001 From: zhangli Date: Wed, 5 Apr 2017 08:42:59 +0800 Subject: [PATCH 019/823] add rng.uniform for python --- simple_integration_test.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/simple_integration_test.py b/simple_integration_test.py index 7a12262e4cb..6ac97aa29fe 100644 --- a/simple_integration_test.py +++ b/simple_integration_test.py @@ -122,5 +122,22 @@ def test_predict(self): for i in range(0, total_length): self.assertTrue(np.allclose(p[i], ground_label[i], atol=1e-6, rtol=0)) + def test_rng(self): + rng = RNG() + rng.set_seed(100) + result = rng.uniform(0.1, 0.2, [2, 3]) + ground_label = np.array([[0.15434049, 0.16711557, 0.12783694], + [0.14120464, 0.14245176, 0.15263824]]) + self.assertTrue(result.shape == [2, 3]) + data = result.to_ndarray() + for i in range(0, 2): + self.assertTrue(np.allclose(data[i], ground_label[i], atol=1e-6, rtol=0)) + + rng.set_seed(100) + result2 = rng.uniform(0.1, 0.2, [2, 3]) + data2 = result2.to_ndarray() + for i in range(0, 2): + self.assertTrue(np.allclose(data[i], data2[i])) + if __name__ == "__main__": unittest.main(failfast=True) From 223298662de27413040b551593cc13c637f3d39a Mon Sep 17 00:00:00 2001 From: ding Date: Wed, 12 Apr 2017 13:10:12 -0400 Subject: [PATCH 020/823] support set model in optimizer --- simple_integration_test.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/simple_integration_test.py b/simple_integration_test.py index 6ac97aa29fe..e66383ba196 100644 --- a/simple_integration_test.py +++ b/simple_integration_test.py @@ -55,6 +55,11 @@ def gen_rand_sample(): trainingData = self.sc.parallelize(range(0, data_len)).map( lambda i: gen_rand_sample()) + model_test = Sequential() + l1_test = Linear(3, 1, "Xavier").set_name("linear1_test") + self.assertEqual("linear1_test", l1_test.name()) + model_test.add(l1_test) + model = Sequential() l1 = Linear(FEATURES_DIM, 1, "Xavier").set_name("linear1") self.assertEqual("linear1", l1.name()) @@ -64,7 +69,7 @@ def gen_rand_sample(): "learningRateDecay": 0.0002, "learingRateSchedule": Poly(0.5, int((data_len/batch_size)*epoch_num))} # noqa optimizer = Optimizer( - model=model, + model=model_test, training_rdd=trainingData, criterion=MSECriterion(), optim_method="sgd", @@ -77,6 +82,10 @@ def gen_rand_sample(): trigger=EveryEpoch(), val_method=["Top1Accuracy"] ) + + optimizer.optimize() + + optimizer.setmodel(model=model) tmp_dir = tempfile.mkdtemp() optimizer.setcheckpoint(SeveralIteration(1), tmp_dir) train_summary = TrainSummary(log_dir=tmp_dir, @@ -88,12 +97,12 @@ def gen_rand_sample(): optimizer.set_val_summary(val_summary) trained_model = optimizer.optimize() - lr_result = train_summary.read_scalar("LearningRate") top1_result = val_summary.read_scalar("Top1Accuracy") # TODO: add result validation parameters = trained_model.parameters() + self.assertIsNotNone(parameters["linear1"]) print("parameters %s" % parameters["linear1"]) predict_result = trained_model.predict(trainingData) From 49593f019d0f455654fe0058ce43f0594bf11286 Mon Sep 17 00:00:00 2001 From: ding Date: Wed, 12 Apr 2017 16:12:30 -0400 Subject: [PATCH 021/823] set model bug fix --- simple_integration_test.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/simple_integration_test.py b/simple_integration_test.py index e66383ba196..045fd5e379a 100644 --- a/simple_integration_test.py +++ b/simple_integration_test.py @@ -59,6 +59,7 @@ def gen_rand_sample(): l1_test = Linear(3, 1, "Xavier").set_name("linear1_test") self.assertEqual("linear1_test", l1_test.name()) model_test.add(l1_test) + model_test.add(Sigmoid()) model = Sequential() l1 = Linear(FEATURES_DIM, 1, "Xavier").set_name("linear1") @@ -76,7 +77,7 @@ def gen_rand_sample(): state=state, end_trigger=MaxEpoch(epoch_num), batch_size=batch_size) - optimizer.setvalidation( + optimizer.set_validation( batch_size=batch_size, val_rdd=trainingData, trigger=EveryEpoch(), @@ -85,9 +86,9 @@ def gen_rand_sample(): optimizer.optimize() - optimizer.setmodel(model=model) + optimizer.set_model(model=model) tmp_dir = tempfile.mkdtemp() - optimizer.setcheckpoint(SeveralIteration(1), tmp_dir) + optimizer.set_checkpoint(SeveralIteration(1), tmp_dir) train_summary = TrainSummary(log_dir=tmp_dir, app_name="run1") train_summary.set_summary_trigger("LearningRate", SeveralIteration(1)) From 5507f04c44cd57458f4738ebc71a19c045d03da4 Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Fri, 28 Apr 2017 11:35:17 +0800 Subject: [PATCH 022/823] Add backward and forward for module and criterion (#830) * add backward and forward for module and criterion * add multiple input testing * style --- simple_integration_test.py | 48 +++++++++++++++++++++++++++++++++++--- 1 file changed, 45 insertions(+), 3 deletions(-) diff --git a/simple_integration_test.py b/simple_integration_test.py index 045fd5e379a..74a67aac3e3 100644 --- a/simple_integration_test.py +++ b/simple_integration_test.py @@ -117,6 +117,48 @@ def gen_rand_sample(): for test_result in test_results: print(test_result) + def test_forward_backward(self): + from nn.layer import Linear + rng = RNG() + rng.set_seed(100) + + linear = Linear(4, 5) + input = rng.uniform(0.0, 1.0, [4]) + output = linear.forward(input) + self.assertTrue(np.allclose(output, + np.array([0.41366524, + 0.009532653, + -0.677581, + 0.07945433, + -0.5742568]), + atol=1e-6, rtol=0)) + mse = MSECriterion() + target = rng.uniform(0.0, 1.0, [5]) + loss = mse.forward(output, target) + print("loss: " + str(loss)) + grad_output = mse.backward(output, rng.uniform(0.0, 1.0, [5])) + l_grad_output = linear.backward(input, grad_output) + + def test_forward_multiple(self): + from nn.layer import Linear + rng = RNG() + rng.set_seed(100) + + input = [rng.uniform(0.0, 0.1, [2]), + rng.uniform(0.0, 0.1, [2]) + 0.2] + + grad_output = [rng.uniform(0.0, 0.1, [3]), + rng.uniform(0.0, 0.1, [3]) + 0.2] + + linear1 = Linear(2, 3) + linear2 = Linear(2, 3) + + module = ParallelTable() + module.add(linear1) + module.add(linear2) + module.forward(input) + module.backward(input, grad_output) + def test_predict(self): np.random.seed(100) total_length = 6 @@ -138,14 +180,14 @@ def test_rng(self): result = rng.uniform(0.1, 0.2, [2, 3]) ground_label = np.array([[0.15434049, 0.16711557, 0.12783694], [0.14120464, 0.14245176, 0.15263824]]) - self.assertTrue(result.shape == [2, 3]) - data = result.to_ndarray() + self.assertTrue(result.shape == (2, 3)) + data = result for i in range(0, 2): self.assertTrue(np.allclose(data[i], ground_label[i], atol=1e-6, rtol=0)) rng.set_seed(100) result2 = rng.uniform(0.1, 0.2, [2, 3]) - data2 = result2.to_ndarray() + data2 = result2 for i in range(0, 2): self.assertTrue(np.allclose(data[i], data2[i])) From a3c504a7ec9c3ad8ecd2c20ed14990a2ec947230 Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Fri, 28 Apr 2017 16:40:42 +0800 Subject: [PATCH 023/823] python 3.5 support (#832) --- python/dllib/test/dev/run-tests.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python/dllib/test/dev/run-tests.py b/python/dllib/test/dev/run-tests.py index a8d8e445655..60b8dd4706e 100755 --- a/python/dllib/test/dev/run-tests.py +++ b/python/dllib/test/dev/run-tests.py @@ -93,7 +93,8 @@ def print_red(text): def run_individual_python_test(test_name, python_exec): env = dict(os.environ) env.update({ - 'DL_CORE_NUMBER': '4' + 'DL_CORE_NUMBER': '4', + 'PYSPARK_PYTHON': python_exec }) LOGGER.debug("Starting test(%s): %s", python_exec, test_name) start_time = time.time() @@ -135,8 +136,7 @@ def run_individual_python_test(test_name, python_exec): def get_default_python_executables(): # TODO: add more version. only support python 2.7 for now - python_execs = [] - python_execs.insert(0, "python") + python_execs = ["python2.7"] return python_execs From 55fbdb3a9593f46ffac01d1dfd681099687700e8 Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Fri, 5 May 2017 10:05:35 +0800 Subject: [PATCH 024/823] Enable python3.5 testing (#842) --- python/dllib/test/dev/run-tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/dllib/test/dev/run-tests.py b/python/dllib/test/dev/run-tests.py index 60b8dd4706e..81f274ab2cb 100755 --- a/python/dllib/test/dev/run-tests.py +++ b/python/dllib/test/dev/run-tests.py @@ -136,7 +136,7 @@ def run_individual_python_test(test_name, python_exec): def get_default_python_executables(): # TODO: add more version. only support python 2.7 for now - python_execs = ["python2.7"] + python_execs = ["python2.7", "python3.5"] return python_execs From c72586b657104ddcbf25a9da2bf7d0d46fd3d867 Mon Sep 17 00:00:00 2001 From: zhangli Date: Wed, 19 Apr 2017 16:36:03 +0800 Subject: [PATCH 025/823] modify optim method hyper-parameters --- simple_integration_test.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/simple_integration_test.py b/simple_integration_test.py index 74a67aac3e3..bff4f3c5a98 100644 --- a/simple_integration_test.py +++ b/simple_integration_test.py @@ -66,15 +66,14 @@ def gen_rand_sample(): self.assertEqual("linear1", l1.name()) model.add(l1) - state = {"learningRate": 0.01, - "learningRateDecay": 0.0002, - "learingRateSchedule": Poly(0.5, int((data_len/batch_size)*epoch_num))} # noqa + optim_method = SGD(learningrate=0.01, learningrate_decay=0.0002, weightdecay=0.0, + momentum=0.0, dampening=0.0, nesterov=False, + leaningrate_schedule=Poly(0.5, int((data_len/batch_size)*epoch_num))) optimizer = Optimizer( model=model_test, training_rdd=trainingData, criterion=MSECriterion(), - optim_method="sgd", - state=state, + optim_method=optim_method, end_trigger=MaxEpoch(epoch_num), batch_size=batch_size) optimizer.set_validation( From cade0dd91fcbc838f743acf1019b106f7ebd2718 Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Mon, 15 May 2017 14:58:52 +0800 Subject: [PATCH 026/823] Add encoding while opening file for python3: master (#885) * add encoding for python3 while opening file * add unittest * style --- python/dllib/test/dev/run-tests.py | 1 - resources/conf/python-api.zip | Bin 0 -> 990 bytes resources/conf/test_spark-bigdl.conf | 42 +++++++++++++++++++++++++++ simple_integration_test.py | 8 +++++ 4 files changed, 50 insertions(+), 1 deletion(-) create mode 100644 resources/conf/python-api.zip create mode 100644 resources/conf/test_spark-bigdl.conf diff --git a/python/dllib/test/dev/run-tests.py b/python/dllib/test/dev/run-tests.py index 81f274ab2cb..ea778236b09 100755 --- a/python/dllib/test/dev/run-tests.py +++ b/python/dllib/test/dev/run-tests.py @@ -135,7 +135,6 @@ def run_individual_python_test(test_name, python_exec): def get_default_python_executables(): - # TODO: add more version. only support python 2.7 for now python_execs = ["python2.7", "python3.5"] return python_execs diff --git a/resources/conf/python-api.zip b/resources/conf/python-api.zip new file mode 100644 index 0000000000000000000000000000000000000000..5d4fafffbc67237072a026787d647bcce2493dba GIT binary patch literal 990 zcmWIWW@Zs#U|`^2D2-X~W!-YwpxXuld%tU!J-Bp^MZMs~Lfe@~8rqK}CIzSM zcXnCgVt@oE_2aM!!{DnB~4}UO8phl_s+OeO4{0F=vU7vB+)*>p9CFtDK%}tR$oGJ!rw!gb(G%dAIJm#dP_| z@)YeP%^Pe}7_UhzFEzy?h|04?N0gbi-V)p-4V3uE!wGs3*C*UYYP+THr8zP6ds>(w;xcV!EcRb}_JoV>iya>d)zJWe}e zSNnyu2r&uWEa(i(oM35lAltr!O?HEG<$q%?y=!iP%6*GfjRTuRO?&mY^d^}&=ewNs zFvvQ(kIz2o&DyNmN5;F|PC4bi%>A)(vBvkj7iV4OzJ2v`U+ME22k)Kyytnr^wa(t4 zUE``6l`5d6GMn3)L3cu6T(hYZ*N1J}p1f{*c5kiHE)!=zr!Hf)nK!KBZY2iP?Re;4 zQ;{B$9C7d%8X+SjaNhR?dFkb+Prt0GiC(h#UX#Hq$FrPY1(qNEl;?N- z$42(2?1$!mWX>tGj}|lUR+;z2_6GBLUE@h%?eh#@f7t&^M*Fke@0oV;;eUMQUu<7- z{)=VUGWp*V?bzp)-MwmRIqh3Y_@j`EqEGXr7k=5jscLol*_NZ`C#D8G4Njb~<;g3V zv&OoUzfYbJBU`X#|7yPlohMFl=j>SCeU(Y~?h>K5hu-(r+bq5)AK=Z%B*%;^&q)CD oA_D^>5N~M&u`qKWD11RG1JXsnY{|d?0RNe>nE(I) literal 0 HcmV?d00001 diff --git a/resources/conf/test_spark-bigdl.conf b/resources/conf/test_spark-bigdl.conf new file mode 100644 index 00000000000..790a9709ebc --- /dev/null +++ b/resources/conf/test_spark-bigdl.conf @@ -0,0 +1,42 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# +# Spark confs for BigDL. +# +# If you can init SparkContext in your application, you should call Engine.createSparkConf to get a +# SparkConf. The properties will be set. You neednot pass these properties. +# +# If you use spark-shell or pyspark, you need to pass these properties in. You can put them +# in your spark conf file. +# +# For more details, please refer +# https://github.com/intel-analytics/BigDL/wiki/Programming-Guide#engine +# + +spark.executorEnv.DL_ENGINE_TYPE mklblas +spark.executorEnv.MKL_DISABLE_FAST_MM 1 +spark.executorEnv.KMP_BLOCKTIME 0 +spark.executorEnv.OMP_WAIT_POLICY passive +spark.executorEnv.OMP_NUM_THREADS 1 +spark.yarn.appMasterEnv.DL_ENGINE_TYPE mklblas +spark.yarn.appMasterEnv.MKL_DISABLE_FAST_MM 1 +spark.yarn.appMasterEnv.KMP_BLOCKTIME 0 +spark.yarn.appMasterEnv.OMP_WAIT_POLICY passive +spark.yarn.appMasterEnv.OMP_NUM_THREADS 1 +spark.shuffle.reduceLocality.enabled false +spark.shuffle.blockTransferService nio +spark.scheduler.minRegisteredResourcesRatio 1.0 \ No newline at end of file diff --git a/simple_integration_test.py b/simple_integration_test.py index bff4f3c5a98..d7ed1f79ee5 100644 --- a/simple_integration_test.py +++ b/simple_integration_test.py @@ -34,6 +34,14 @@ def setUp(self): def tearDown(self): self.sc.stop() + def test_load_zip_conf(self): + from util.common import get_bigdl_conf + import sys + sys.path = [path for path in sys.path if "spark-bigdl.conf" not in path] + sys.path.insert(0, os.path.join(os.path.split(__file__)[0], "resources/conf/python-api.zip")) # noqa + result = get_bigdl_conf() + self.assertTrue(result.get("spark.executorEnv.OMP_WAIT_POLICY"), "passive") + def test_set_seed(self): l1 = Linear(10, 20, "Xavier").set_name("linear1").set_seed(1234).reset() # noqa l2 = Linear(10, 20, "Xavier").set_name("linear2").set_seed(1234).reset() # noqa From f984307dbfb6aa09a3d9b1af345acf8c6496a37a Mon Sep 17 00:00:00 2001 From: ding Date: Mon, 15 May 2017 09:14:16 -0400 Subject: [PATCH 027/823] fix ut --- simple_integration_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/simple_integration_test.py b/simple_integration_test.py index d7ed1f79ee5..8616b0cb9c8 100644 --- a/simple_integration_test.py +++ b/simple_integration_test.py @@ -64,7 +64,7 @@ def gen_rand_sample(): lambda i: gen_rand_sample()) model_test = Sequential() - l1_test = Linear(3, 1, "Xavier").set_name("linear1_test") + l1_test = Linear(FEATURES_DIM, 1, "Xavier").set_name("linear1_test") self.assertEqual("linear1_test", l1_test.name()) model_test.add(l1_test) model_test.add(Sigmoid()) From 988e95d53e1e40ee59659bebb774006d571422a4 Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Wed, 31 May 2017 17:01:25 +0800 Subject: [PATCH 028/823] add nn graph support (#952) --- simple_integration_test.py | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/simple_integration_test.py b/simple_integration_test.py index 8616b0cb9c8..14692c728a9 100644 --- a/simple_integration_test.py +++ b/simple_integration_test.py @@ -34,6 +34,40 @@ def setUp(self): def tearDown(self): self.sc.stop() + def test_create_node(self): + import numpy as np + fc1 = Linear(4, 2)() + fc2 = Linear(4, 2)() + cadd = CAddTable()([fc1, fc2]) + output1 = ReLU()(cadd) + graph = Graph([fc1, fc2], [output1]) + fc1.element().set_weights([np.ones((4, 2)), np.ones((2, ))]) + fc2.element().set_weights([np.ones((4, 2)), np.ones((2, ))]) + output = graph.forward([np.array([0.1, 0.2, -0.3, -0.4]), + np.array([0.5, 0.4, -0.2, -0.1])]) + self.assertTrue(np.allclose(output, + np.array([2.2, 2.2]))) + + def test_graph_backward(self): + fc1 = Linear(4, 2)() + fc2 = Linear(4, 2)() + cadd = CAddTable()([fc1, fc2]) + output1 = ReLU()(cadd) + output2 = Threshold(10.0)(cadd) + graph = Graph([fc1, fc2], [output1, output2]) + fc1.element().set_weights([np.ones((4, 2)), np.ones((2, ))]) + fc2.element().set_weights([np.ones((4, 2)) * 2, np.ones((2, )) * 2]) + output = graph.forward([np.array([0.1, 0.2, -0.3, -0.4]), + np.array([0.5, 0.4, -0.2, -0.1])]) + gradInput = graph.backward([np.array([0.1, 0.2, -0.3, -0.4]), + np.array([0.5, 0.4, -0.2, -0.1])], + [np.array([1.0, 2.0]), + np.array([3.0, 4.0])]) + self.assertTrue(np.allclose(gradInput[0], + np.array([3.0, 3.0, 3.0, 3.0]))) + self.assertTrue(np.allclose(gradInput[1], + np.array([6.0, 6.0, 6.0, 6.0]))) + def test_load_zip_conf(self): from util.common import get_bigdl_conf import sys From 25bc1585786efc0d1c4c761b873ac97b263d93e6 Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Tue, 6 Jun 2017 16:15:49 +0800 Subject: [PATCH 029/823] Add bigdl as the top module (#973) * add bigdl as the top modules * back to test Former-commit-id: ae6938d2b3cb1413c213a07179aa957ad76b3d5c --- dist/Bigbig-0.1.3.tar.gz.REMOVED.git-id | 1 + dist/Bigbig-0.1.4.tar.gz.REMOVED.git-id | 1 + dist/Bigbig-0.1.5.tar.gz.REMOVED.git-id | 1 + dist/Bigbig-0.1.6.tar.gz.REMOVED.git-id | 1 + dist/Bigbig-0.1.7.tar.gz.REMOVED.git-id | 1 + .../bigdl/dllib/feature/dataset/__init__.py | 15 + .../src/bigdl/dllib/feature/dataset/base.py | 197 ++ .../src/bigdl/dllib/feature/dataset/mnist.py | 129 + .../src/bigdl/dllib/feature/dataset/news20.py | 104 + .../dllib/feature/dataset/transformer.py | 26 + .../dllib/src/bigdl/dllib/models/__init__.py | 15 + .../src/bigdl/dllib/models/lenet/README.md | 64 + .../src/bigdl/dllib/models/lenet/__init__.py | 15 + .../src/bigdl/dllib/models/lenet/lenet5.py | 102 + .../dllib/models/textclassifier/README.md | 72 + .../models/textclassifier/textclassifier.py | 177 + python/dllib/src/bigdl/dllib/nn/__init__.py | 15 + python/dllib/src/bigdl/dllib/nn/criterion.py | 605 ++++ python/dllib/src/bigdl/dllib/nn/layer.py | 3005 +++++++++++++++++ .../dllib/src/bigdl/dllib/optim/__init__.py | 15 + .../dllib/src/bigdl/dllib/optim/optimizer.py | 556 +++ python/dllib/src/bigdl/dllib/utils/common.py | 390 +++ python/dllib/src/bigdl/utils/__init__.py | 15 + python/dllib/test/__init__.py | 15 + 24 files changed, 5537 insertions(+) create mode 100644 dist/Bigbig-0.1.3.tar.gz.REMOVED.git-id create mode 100644 dist/Bigbig-0.1.4.tar.gz.REMOVED.git-id create mode 100644 dist/Bigbig-0.1.5.tar.gz.REMOVED.git-id create mode 100644 dist/Bigbig-0.1.6.tar.gz.REMOVED.git-id create mode 100644 dist/Bigbig-0.1.7.tar.gz.REMOVED.git-id create mode 100644 python/dllib/src/bigdl/dllib/feature/dataset/__init__.py create mode 100644 python/dllib/src/bigdl/dllib/feature/dataset/base.py create mode 100644 python/dllib/src/bigdl/dllib/feature/dataset/mnist.py create mode 100644 python/dllib/src/bigdl/dllib/feature/dataset/news20.py create mode 100644 python/dllib/src/bigdl/dllib/feature/dataset/transformer.py create mode 100644 python/dllib/src/bigdl/dllib/models/__init__.py create mode 100644 python/dllib/src/bigdl/dllib/models/lenet/README.md create mode 100644 python/dllib/src/bigdl/dllib/models/lenet/__init__.py create mode 100644 python/dllib/src/bigdl/dllib/models/lenet/lenet5.py create mode 100644 python/dllib/src/bigdl/dllib/models/textclassifier/README.md create mode 100644 python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py create mode 100644 python/dllib/src/bigdl/dllib/nn/__init__.py create mode 100644 python/dllib/src/bigdl/dllib/nn/criterion.py create mode 100644 python/dllib/src/bigdl/dllib/nn/layer.py create mode 100644 python/dllib/src/bigdl/dllib/optim/__init__.py create mode 100644 python/dllib/src/bigdl/dllib/optim/optimizer.py create mode 100644 python/dllib/src/bigdl/dllib/utils/common.py create mode 100644 python/dllib/src/bigdl/utils/__init__.py create mode 100644 python/dllib/test/__init__.py diff --git a/dist/Bigbig-0.1.3.tar.gz.REMOVED.git-id b/dist/Bigbig-0.1.3.tar.gz.REMOVED.git-id new file mode 100644 index 00000000000..50b70e10601 --- /dev/null +++ b/dist/Bigbig-0.1.3.tar.gz.REMOVED.git-id @@ -0,0 +1 @@ +8de3ee96b2d6636d28bdf284dd0f06181dcb0c61 \ No newline at end of file diff --git a/dist/Bigbig-0.1.4.tar.gz.REMOVED.git-id b/dist/Bigbig-0.1.4.tar.gz.REMOVED.git-id new file mode 100644 index 00000000000..f4e59d7a289 --- /dev/null +++ b/dist/Bigbig-0.1.4.tar.gz.REMOVED.git-id @@ -0,0 +1 @@ +b9410cbb0e3776d3c3640efcf67dbe3a81262b8f \ No newline at end of file diff --git a/dist/Bigbig-0.1.5.tar.gz.REMOVED.git-id b/dist/Bigbig-0.1.5.tar.gz.REMOVED.git-id new file mode 100644 index 00000000000..ce54df3848c --- /dev/null +++ b/dist/Bigbig-0.1.5.tar.gz.REMOVED.git-id @@ -0,0 +1 @@ +9c4ca18f01bf477a1c2158482de9de9560840128 \ No newline at end of file diff --git a/dist/Bigbig-0.1.6.tar.gz.REMOVED.git-id b/dist/Bigbig-0.1.6.tar.gz.REMOVED.git-id new file mode 100644 index 00000000000..07119451896 --- /dev/null +++ b/dist/Bigbig-0.1.6.tar.gz.REMOVED.git-id @@ -0,0 +1 @@ +f84edbba2745d73f6567c56bed6920575d63d28b \ No newline at end of file diff --git a/dist/Bigbig-0.1.7.tar.gz.REMOVED.git-id b/dist/Bigbig-0.1.7.tar.gz.REMOVED.git-id new file mode 100644 index 00000000000..9c6597388e5 --- /dev/null +++ b/dist/Bigbig-0.1.7.tar.gz.REMOVED.git-id @@ -0,0 +1 @@ +39693bf712a1c3028a1e237dc14789af5d53bb26 \ No newline at end of file diff --git a/python/dllib/src/bigdl/dllib/feature/dataset/__init__.py b/python/dllib/src/bigdl/dllib/feature/dataset/__init__.py new file mode 100644 index 00000000000..2151a805423 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/feature/dataset/__init__.py @@ -0,0 +1,15 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/python/dllib/src/bigdl/dllib/feature/dataset/base.py b/python/dllib/src/bigdl/dllib/feature/dataset/base.py new file mode 100644 index 00000000000..f211ee5261c --- /dev/null +++ b/python/dllib/src/bigdl/dllib/feature/dataset/base.py @@ -0,0 +1,197 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os +import sys +import shutil +import tempfile +import time +from distutils.dir_util import mkpath +from six.moves.urllib.request import urlopen +import numpy as np + + +# Adopt from keras +class Progbar(object): + def __init__(self, target, width=30, verbose=1, interval=0.01): + ''' + @param target: total number of steps expected + @param interval: minimum visual progress update interval (in seconds) + ''' + self.width = width + self.target = target + self.sum_values = {} + self.unique_values = [] + self.start = time.time() + self.last_update = 0 + self.interval = interval + self.total_width = 0 + self.seen_so_far = 0 + self.verbose = verbose + + def update(self, current, values=[], force=False): + ''' + @param current: index of current step + @param values: list of tuples (name, value_for_last_step). + The progress bar will display averages for these values. + @param force: force visual progress update + ''' + for k, v in values: + if k not in self.sum_values: + self.sum_values[k] = [v * (current - self.seen_so_far), current - self.seen_so_far] + self.unique_values.append(k) + else: + self.sum_values[k][0] += v * (current - self.seen_so_far) + self.sum_values[k][1] += (current - self.seen_so_far) + self.seen_so_far = current + + now = time.time() + if self.verbose == 1: + if not force and (now - self.last_update) < self.interval: + return + + prev_total_width = self.total_width + sys.stdout.write("\b" * prev_total_width) + sys.stdout.write("\r") + + numdigits = int(np.floor(np.log10(self.target))) + 1 + barstr = '%%%dd/%%%dd [' % (numdigits, numdigits) + bar = barstr % (current, self.target) + prog = float(current) / self.target + prog_width = int(self.width * prog) + if prog_width > 0: + bar += ('=' * (prog_width-1)) + if current < self.target: + bar += '>' + else: + bar += '=' + bar += ('.' * (self.width - prog_width)) + bar += ']' + sys.stdout.write(bar) + self.total_width = len(bar) + + if current: + time_per_unit = (now - self.start) / current + else: + time_per_unit = 0 + eta = time_per_unit * (self.target - current) + info = '' + if current < self.target: + info += ' - ETA: %ds' % eta + else: + info += ' - %ds' % (now - self.start) + for k in self.unique_values: + info += ' - %s:' % k + if type(self.sum_values[k]) is list: + avg = self.sum_values[k][0] / max(1, self.sum_values[k][1]) + if abs(avg) > 1e-3: + info += ' %.4f' % avg + else: + info += ' %.4e' % avg + else: + info += ' %s' % self.sum_values[k] + + self.total_width += len(info) + if prev_total_width > self.total_width: + info += ((prev_total_width - self.total_width) * " ") + + sys.stdout.write(info) + sys.stdout.flush() + + if current >= self.target: + sys.stdout.write("\n") + + if self.verbose == 2: + if current >= self.target: + info = '%ds' % (now - self.start) + for k in self.unique_values: + info += ' - %s:' % k + avg = self.sum_values[k][0] / max(1, self.sum_values[k][1]) + if avg > 1e-3: + info += ' %.4f' % avg + else: + info += ' %.4e' % avg + sys.stdout.write(info + "\n") + + self.last_update = now + + def add(self, n, values=[]): + self.update(self.seen_so_far + n, values) + + +def display_table(rows, positions): + + def display_row(objects, positions): + line = '' + for i in range(len(objects)): + line += str(objects[i]) + line = line[:positions[i]] + line += ' ' * (positions[i] - len(line)) + print(line) + + for objects in rows: + display_row(objects, positions) + +# Adopt from keras +# Under Python 2, 'urlretrieve' relies on FancyURLopener from legacy +# urllib module, known to have issues with proxy management +if sys.version_info[0] == 2: + def urlretrieve(url, filename, reporthook=None, data=None): + def chunk_read(response, chunk_size=8192, reporthook=None): + total_size = response.info().get('Content-Length').strip() + total_size = int(total_size) + count = 0 + while 1: + chunk = response.read(chunk_size) + count += 1 + if not chunk: + reporthook(count, total_size, total_size) + break + if reporthook: + reporthook(count, chunk_size, total_size) + yield chunk + + response = urlopen(url, data) + with open(filename, 'wb') as fd: + for chunk in chunk_read(response, reporthook=reporthook): + fd.write(chunk) +else: + from six.moves.urllib.request import urlretrieve + + +def maybe_download(filename, work_directory, source_url): + if not os.path.exists(work_directory): + mkpath(work_directory) + filepath = os.path.join(work_directory, filename) + + if not os.path.exists(filepath): + print('Downloading data from', source_url) + global progbar + progbar = None + + def dl_progress(count, block_size, total_size): + global progbar + if progbar is None: + progbar = Progbar(total_size) + else: + progbar.update(count * block_size) + with tempfile.NamedTemporaryFile() as tmpfile: + temp_file_name = tmpfile.name + urlretrieve(source_url, temp_file_name, dl_progress) + shutil.copy2(temp_file_name, filepath) + size = os.path.getsize(filepath) + print('Successfully downloaded', filename, size, 'bytes.') + return filepath diff --git a/python/dllib/src/bigdl/dllib/feature/dataset/mnist.py b/python/dllib/src/bigdl/dllib/feature/dataset/mnist.py new file mode 100644 index 00000000000..f23087e8736 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/feature/dataset/mnist.py @@ -0,0 +1,129 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Part of the code originally from Tensorflow + + +import gzip + +import numpy + +from bigdl.dataset import base + +SOURCE_URL = 'http://yann.lecun.com/exdb/mnist/' + +TRAIN_MEAN = 0.13066047740239506 * 255 +TRAIN_STD = 0.3081078 * 255 +TEST_MEAN = 0.13251460696903547 * 255 +TEST_STD = 0.31048024 * 255 + + +def _read32(bytestream): + dt = numpy.dtype(numpy.uint32).newbyteorder('>') + return numpy.frombuffer(bytestream.read(4), dtype=dt)[0] + + +def extract_images(f): + """Extract the images into a 4D uint8 numpy array [index, y, x, depth]. + + Args: + f: A file object that can be passed into a gzip reader. + + Returns: + data: A 4D unit8 numpy array [index, y, x, depth]. + + Raises: + ValueError: If the bytestream does not start with 2051. + + """ + print('Extracting', f.name) + with gzip.GzipFile(fileobj=f) as bytestream: + magic = _read32(bytestream) + if magic != 2051: + raise ValueError( + 'Invalid magic number %d in MNIST image file: %s' % + (magic, f.name)) + num_images = _read32(bytestream) + rows = _read32(bytestream) + cols = _read32(bytestream) + buf = bytestream.read(rows * cols * num_images) + data = numpy.frombuffer(buf, dtype=numpy.uint8) + data = data.reshape(num_images, rows, cols, 1) + return data + + +def extract_labels(f): + print('Extracting', f.name) + with gzip.GzipFile(fileobj=f) as bytestream: + magic = _read32(bytestream) + if magic != 2049: + raise ValueError( + 'Invalid magic number %d in MNIST label file: %s' % + (magic, f.name)) + num_items = _read32(bytestream) + buf = bytestream.read(num_items) + labels = numpy.frombuffer(buf, dtype=numpy.uint8) + return labels + + +def read_data_sets(train_dir, data_type="train"): + """ + Parse or download mnist data if train_dir is empty. + :param train_dir: The directory storing the mnist data + :param data_type: Reading training set or testing set. + It can be either "train" or "test" + :return: (ndarray, ndarray) representing (features, labels) + features is a 4D unit8 numpy array [index, y, x, depth] + representing each pixel valued from 0 to 255. labels + is 1D unit8 nunpy array representing the label valued + from 0 to 9. + """ + TRAIN_IMAGES = 'train-images-idx3-ubyte.gz' + TRAIN_LABELS = 'train-labels-idx1-ubyte.gz' + TEST_IMAGES = 't10k-images-idx3-ubyte.gz' + TEST_LABELS = 't10k-labels-idx1-ubyte.gz' + + if data_type == "train": + local_file = base.maybe_download(TRAIN_IMAGES, train_dir, + SOURCE_URL + TRAIN_IMAGES) + with open(local_file, 'rb') as f: + train_images = extract_images(f) + + local_file = base.maybe_download(TRAIN_LABELS, train_dir, + SOURCE_URL + TRAIN_LABELS) + with open(local_file, 'rb') as f: + train_labels = extract_labels(f) + return train_images, train_labels + + else: + local_file = base.maybe_download(TEST_IMAGES, train_dir, + SOURCE_URL + TEST_IMAGES) + with open(local_file, 'rb') as f: + test_images = extract_images(f) + + local_file = base.maybe_download(TEST_LABELS, train_dir, + SOURCE_URL + TEST_LABELS) + with open(local_file, 'rb') as f: + test_labels = extract_labels(f) + return test_images, test_labels + + +if __name__ == "__main__": + train, _ = read_data_sets("/tmp/mnist/", "train") + test, _ = read_data_sets("/tmp/mnist", "test") + assert numpy.abs(numpy.mean(train) - TRAIN_MEAN) / TRAIN_MEAN < 1e-7 + assert numpy.abs(numpy.std(train) - TRAIN_STD) / TRAIN_STD < 1e-7 + assert numpy.abs(numpy.mean(test) - TEST_MEAN) / TEST_MEAN < 1e-7 + assert numpy.abs(numpy.std(test) - TEST_STD) / TEST_STD < 1e-7 \ No newline at end of file diff --git a/python/dllib/src/bigdl/dllib/feature/dataset/news20.py b/python/dllib/src/bigdl/dllib/feature/dataset/news20.py new file mode 100644 index 00000000000..3d066ad3b52 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/feature/dataset/news20.py @@ -0,0 +1,104 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import tarfile +from bigdl.dataset import base +import os +import sys + +NEWS20_URL = 'http://qwone.com/~jason/20Newsgroups/20news-19997.tar.gz' # noqa +GLOVE_URL = 'http://nlp.stanford.edu/data/glove.6B.zip' # noqa + +CLASS_NUM = 20 + + +def download_news20(dest_dir): + file_name = "20news-19997.tar.gz" + file_abs_path = base.maybe_download(file_name, dest_dir, NEWS20_URL) + tar = tarfile.open(file_abs_path, "r:gz") + extracted_to = os.path.join(dest_dir, "20_newsgroups") + if not os.path.exists(extracted_to): + print("Extracting %s to %s" % (file_abs_path, extracted_to)) + tar.extractall(dest_dir) + tar.close() + return extracted_to + + +def download_glove_w2v(dest_dir): + file_name = "glove.6B.zip" + file_abs_path = base.maybe_download(file_name, dest_dir, GLOVE_URL) + import zipfile + zip_ref = zipfile.ZipFile(file_abs_path, 'r') + extracted_to = os.path.join(dest_dir, "glove.6B") + if not os.path.exists(extracted_to): + print("Extracting %s to %s" % (file_abs_path, extracted_to)) + zip_ref.extractall(extracted_to) + zip_ref.close() + return extracted_to + + +def get_news20(source_dir="/tmp/news20/"): + """ + Parse or download news20 if source_dir is empty. + :param source_dir: The directory storing news data. + :return: A list of (tokens, label) + """ + news_dir = download_news20(source_dir) + texts = [] # list of text samples + label_id = 0 + for name in sorted(os.listdir(news_dir)): + path = os.path.join(news_dir, name) + label_id += 1 + if os.path.isdir(path): + for fname in sorted(os.listdir(path)): + if fname.isdigit(): + fpath = os.path.join(path, fname) + if sys.version_info < (3,): + f = open(fpath) + else: + f = open(fpath, encoding='latin-1') + content = f.read() + texts.append((content, label_id)) + f.close() + + print('Found %s texts.' % len(texts)) + return texts + + +def get_glove_w2v(source_dir="/tmp/news20/", dim=100): + """ + Parse or download the pre-trained glove word2vec if source_dir is empty. + :param source_dir: The directory storing the pre-trained word2vec + :param dim: The dimension of a vector + :return: A dict mapping from word to vector + """ + w2v_dir = download_glove_w2v(source_dir) + w2v_path = os.path.join(w2v_dir, "glove.6B.%sd.txt" % dim) + if sys.version_info < (3,): + w2v_f = open(w2v_path) + else: + w2v_f = open(w2v_path, encoding='latin-1') + pre_w2v = {} + for line in w2v_f.readlines(): + items = line.split(" ") + pre_w2v[items[0]] = [float(i) for i in items[1:]] + w2v_f.close() + return pre_w2v + + +if __name__ == "__main__": + get_news20("/tmp/news20/") + get_glove_w2v("/tmp/news20/") diff --git a/python/dllib/src/bigdl/dllib/feature/dataset/transformer.py b/python/dllib/src/bigdl/dllib/feature/dataset/transformer.py new file mode 100644 index 00000000000..018d93c13c6 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/feature/dataset/transformer.py @@ -0,0 +1,26 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +from bigdl.util.common import Sample + + +def normalizer(mean, std): + """ + Normalize features by standard deviation + """ + return lambda sample: Sample.from_ndarray((sample.features - mean) / std, + sample.label, sample.bigdl_type) diff --git a/python/dllib/src/bigdl/dllib/models/__init__.py b/python/dllib/src/bigdl/dllib/models/__init__.py new file mode 100644 index 00000000000..2151a805423 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/models/__init__.py @@ -0,0 +1,15 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/python/dllib/src/bigdl/dllib/models/lenet/README.md b/python/dllib/src/bigdl/dllib/models/lenet/README.md new file mode 100644 index 00000000000..5f0a6f05677 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/models/lenet/README.md @@ -0,0 +1,64 @@ +# LeNet5 Model on MNIST + +LeNet5 is a classical CNN model used in digital number classification. For detail information, +please refer to . + +## How to run this example: + +Program would download the minst data into ```/tmp/mnist``` automatically by default. + +``` +/tmp/mnist$ tree . +. +├── t10k-images-idx3-ubyte.gz +├── t10k-labels-idx1-ubyte.gz +├── train-images-idx3-ubyte.gz +└── train-labels-idx1-ubyte.gz + +``` + +**Source bigdl.sh fisrt which would setup the essential environment for you, otherwise program would fail fast.** + +We would train a LeNet model in spark local mode with the following commands and you can distribute it across cluster by modifying the spark master and the executor cores. + +``` + BigDL_HOME=... + + source $BigDL_HOME/dist/bin/bigdl.sh + + SPARK_HOME=... + MASTER=local[*] + PYTHON_API_ZIP_PATH=${BigDL_HOME}/dist/lib/bigdl-VERSION-python-api.zip + BigDL_JAR_PATH=${BigDL_HOME}/dist/lib/bigdl-VERSION-jar-with-dependencies.jar + PYTHONPATH=${PYTHON_API_ZIP_PATH}:$PYTHONPATH + ${SPARK_HOME}/bin/spark-submit \ + --master ${MASTER} \ + --driver-cores 2 \ + --driver-memory 2g \ + --total-executor-cores 2 \ + --executor-cores 2 \ + --executor-memory 4g \ + --conf spark.akka.frameSize=64 \ + --py-files ${PYTHON_API_ZIP_PATH},${BigDL_HOME}/pyspark/dl/models/lenet/lenet5.py \ + --properties-file ${BigDL_HOME}/dist/conf/spark-bigdl.conf \ + --jars ${BigDL_JAR_PATH} \ + --conf spark.driver.extraClassPath=${BigDL_JAR_PATH} \ + --conf spark.executor.extraClassPath=bigdl-VERSION-jar-with-dependencies.jar \ + ${BigDL_HOME}/pyspark/dl/models/lenet/lenet5.py \ + --action train + ``` + + +* ```--action``` it can be train or test. + +* ```--batchSize``` option can be used to set batch size, the default value is 128. + +To verify the accuracy, search "accuracy" from log: + +``` +INFO DistriOptimizer$:247 - [Epoch 1 0/60000][Iteration 1][Wall Clock 0.0s] Train 128 in xx seconds. Throughput is xx records/second. + +INFO DistriOptimizer$:522 - Top1Accuracy is Accuracy(correct: 9572, count: 10000, accuracy: 0.9572) + + +``` diff --git a/python/dllib/src/bigdl/dllib/models/lenet/__init__.py b/python/dllib/src/bigdl/dllib/models/lenet/__init__.py new file mode 100644 index 00000000000..2151a805423 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/models/lenet/__init__.py @@ -0,0 +1,15 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py b/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py new file mode 100644 index 00000000000..b3fcba58c9c --- /dev/null +++ b/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py @@ -0,0 +1,102 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Still in experimental stage! + +from optparse import OptionParser + +from bigdl.dataset import mnist +from bigdl.dataset.transformer import * +from bigdl.nn.layer import * +from bigdl.nn.criterion import * +from bigdl.optim.optimizer import * +from bigdl.util.common import * + + +def build_model(class_num): + model = Sequential() + model.add(Reshape([1, 28, 28])) + model.add(SpatialConvolution(1, 6, 5, 5)) + model.add(Tanh()) + model.add(SpatialMaxPooling(2, 2, 2, 2)) + model.add(Tanh()) + model.add(SpatialConvolution(6, 12, 5, 5)) + model.add(SpatialMaxPooling(2, 2, 2, 2)) + model.add(Reshape([12 * 4 * 4])) + model.add(Linear(12 * 4 * 4, 100)) + model.add(Tanh()) + model.add(Linear(100, class_num)) + model.add(LogSoftMax()) + return model + + +def get_minst(sc, data_type="train", location="/tmp/mnist"): + """ + Get and normalize the mnist data. We would download it automatically + if the data doesn't present at the specific location. + :param sc: SparkContext + :param data_type: training data or testing data + :param location: Location storing the mnist + :return: A RDD of Sample + """ + (images, labels) = mnist.read_data_sets(location, data_type) + images = sc.parallelize(images) + labels = sc.parallelize(labels) + # Target start from 1 in BigDL + record = images.zip(labels).map(lambda features_label: + Sample.from_ndarray(features_label[0], features_label[1] + 1)) + return record + + +if __name__ == "__main__": + parser = OptionParser() + parser.add_option("-a", "--action", dest="action", default="train") + parser.add_option("-b", "--batchSize", type=int, dest="batchSize", default="128") + parser.add_option("-m", "--modelPath", dest="modelPath", default="") + + (options, args) = parser.parse_args(sys.argv) + + sc = SparkContext(appName="lenet5", conf=create_spark_conf()) + init_engine() + + if options.action == "train": + train_data = get_minst(sc, "train").map( + normalizer(mnist.TRAIN_MEAN, mnist.TRAIN_STD)) + test_data = get_minst(sc, "test").map( + normalizer(mnist.TEST_MEAN, mnist.TEST_STD)) + optimizer = Optimizer( + model=build_model(10), + training_rdd=train_data, + criterion=ClassNLLCriterion(), + optim_method=SGD(learningrate=0.01, learningrate_decay=0.0002), + end_trigger=MaxEpoch(20), + batch_size=options.batchSize) + optimizer.set_validation( + batch_size=options.batchSize, + val_rdd=test_data, + trigger=EveryEpoch(), + val_method=["Top1Accuracy"] + ) + optimizer.set_checkpoint(EveryEpoch(), "/tmp/lenet5/") + trained_model = optimizer.optimize() + parameters = trained_model.parameters() + elif options.action == "test": + # Load a pre-trained model and then validate it through top1 accuracy. + test_data = get_minst(sc, "test").map( + normalizer(mnist.TEST_MEAN, mnist.TEST_STD)) + model = Model.load(options.modelPath) + results = model.test(test_data, options.batchSize, ["Top1Accuracy"]) + for result in results: + print(result) diff --git a/python/dllib/src/bigdl/dllib/models/textclassifier/README.md b/python/dllib/src/bigdl/dllib/models/textclassifier/README.md new file mode 100644 index 00000000000..eae226c7ecb --- /dev/null +++ b/python/dllib/src/bigdl/dllib/models/textclassifier/README.md @@ -0,0 +1,72 @@ +## Summary + This example use a (pre-trained GloVe embedding) to convert word to vector, + and uses it to train a CNN, LSTM or GRU text classification model on a 20 Newsgroup dataset + with 20 different categories. CNN model can achieve around 96% accuracy after 2 or 3 epochs training. + LSTM and GRU are a little difficult to train, which need more epochs to achieve the equivalent result. +(It was first described in: https://blog.keras.io/using-pre-trained-word-embeddings-in-a-keras-model.html) +## Data +* Embedding: 100-dimensional pre-trained GloVe embeddings of 400k words which trained on a 2014 dump of English Wikipedia. +* Training data: "20 Newsgroup dataset" which containing 20 categories and with totally 19997 texts. + +## How to run this example: + +If there is no [Pre-train GloVe word embeddings](http://nlp.stanford.edu/data/glove.6B.zip) +or [20 Newsgroup dataset](http://www.cs.cmu.edu/afs/cs.cmu.edu/project/theo-20/www/data/news20.html) in +`/tmp/news20` directory with the following structure looks like: + +```{r, engine='sh'} +$ [/tmp/news20]$ tree . -L 1 + . + ├── 20news-19997.tar.gz + └── glove.6B.zip +``` + +then running the flowing script would automatically download the data during the first run. + +bigdl.sh would setup the essential environment for you and it would accept a spark-submit command as an input parameter. + +```{r, engine='sh'} + PYTHONHASHSEED=... + BigDL_HOME=... + SPARK_HOME=... + MASTER=... + PYTHON_API_ZIP_PATH=${BigDL_HOME}/dist/lib/bigdl-VERSION-python-api.zip + BigDL_JAR_PATH=${BigDL_HOME}/dist/lib/bigdl-VERSION-jar-with-dependencies.jar + PYTHONPATH=${PYTHON_API_ZIP_PATH}:$PYTHONPATH + source ${BigDL_HOME}/dist/bin/bigdl.sh + + ${SPARK_HOME}/bin/spark-submit \ + --master ${MASTER} \ + --driver-cores 4 \ + --driver-memory 10g \ + --total-executor-cores 4 \ + --executor-cores 4 \ + --executor-memory 20g \ + --conf spark.akka.frameSize=64 \ + --py-files ${PYTHON_API_ZIP_PATH},${BigDL_HOME}/pyspark/dl/models/textclassifier/textclassifier.py \ + --jars ${BigDL_JAR_PATH} \ + --conf spark.driver.extraClassPath=${BigDL_JAR_PATH} \ + --conf spark.executor.extraClassPath=bigdl-VERSION-jar-with-dependencies.jar \ + --conf spark.executorEnv.PYTHONHASHSEED=${PYTHONHASHSEED} \ + ${BigDL_HOME}/pyspark/dl/models/textclassifier/textclassifier.py \ + --max_epoch 3 + --model cnn +``` + +* `--max_epoch` option can be used to set how many epochs the model to be trained + +* `--model` option can be used to choose a model to be trained, three models are supported in this example, +which are `cnn`, `lstm` and `gru`, default is `cnn` + +* `--batchSize` option can be used to set batch size, the default value is 128. + +* `embedding_dim` option can be used to set the embedding size of word vector, the default value is 50. + +To verify the accuracy, search "accuracy" from log: + +```{r, engine='sh'} + [Epoch 1 0/15964][Iteration 1][Wall Clock 0.0s] + + top1 accuracy is Accuracy(correct: 14749, count: 15964, accuracy: 0.9238912 + 553244801) +``` diff --git a/python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py b/python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py new file mode 100644 index 00000000000..9c67606c852 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py @@ -0,0 +1,177 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Still in experimental stage! + +import itertools +import re +from optparse import OptionParser + +from bigdl.dataset import news20 +from bigdl.nn.layer import * +from bigdl.nn.criterion import * +from bigdl.optim.optimizer import * +from bigdl.util.common import * +from bigdl.util.common import Sample + + +def text_to_words(review_text): + letters_only = re.sub("[^a-zA-Z]", " ", review_text) + words = letters_only.lower().split() + return words + + +def analyze_texts(data_rdd): + def index(w_c_i): + ((w, c), i) = w_c_i + return (w, (i + 1, c)) + return data_rdd.flatMap(lambda text_label: text_to_words(text_label[0])) \ + .map(lambda word: (word, 1)).reduceByKey(lambda a, b: a + b) \ + .sortBy(lambda w_c: - w_c[1]).zipWithIndex() \ + .map(lambda w_c_i: index(w_c_i)).collect() + + +# pad([1, 2, 3, 4, 5], 0, 6) +def pad(l, fill_value, width): + if len(l) >= width: + return l[0: width] + else: + l.extend([fill_value] * (width - len(l))) + return l + + +def to_vec(token, b_w2v, embedding_dim): + if token in b_w2v: + return b_w2v[token] + else: + return pad([], 0, embedding_dim) + + +def to_sample(vectors, label, embedding_dim): + # flatten nested list + flatten_features = list(itertools.chain(*vectors)) + features = np.array(flatten_features, dtype='float').reshape( + [sequence_len, embedding_dim]) + + if model_type.lower() == "cnn": + features = features.transpose(1, 0) + return Sample.from_ndarray(features, np.array(label)) + + +def build_model(class_num): + model = Sequential() + + if model_type.lower() == "cnn": + model.add(Reshape([embedding_dim, 1, sequence_len])) + model.add(SpatialConvolution(embedding_dim, 128, 5, 1)) + model.add(ReLU()) + model.add(SpatialMaxPooling(5, 1, 5, 1)) + model.add(SpatialConvolution(128, 128, 5, 1)) + model.add(ReLU()) + model.add(SpatialMaxPooling(5, 1, 5, 1)) + model.add(Reshape([128])) + elif model_type.lower() == "lstm": + model.add(Recurrent() + .add(LSTM(embedding_dim, 128, p))) + model.add(Select(2, -1)) + elif model_type.lower() == "gru": + model.add(Recurrent() + .add(GRU(embedding_dim, 128, p))) + model.add(Select(2, -1)) + else: + raise ValueError('model can only be cnn, lstm, or gru') + + model.add(Linear(128, 100)) + model.add(Linear(100, class_num)) + model.add(LogSoftMax()) + return model + + +def train(sc, + batch_size, + sequence_len, max_words, embedding_dim, training_split): + print('Processing text dataset') + texts = news20.get_news20() + data_rdd = sc.parallelize(texts, 2) + + word_to_ic = analyze_texts(data_rdd) + + # Only take the top wc between [10, sequence_len] + word_to_ic = dict(word_to_ic[10: max_words]) + bword_to_ic = sc.broadcast(word_to_ic) + + w2v = news20.get_glove_w2v(dim=embedding_dim) + filtered_w2v = {w: v for w, v in w2v.items() if w in word_to_ic} + bfiltered_w2v = sc.broadcast(filtered_w2v) + + tokens_rdd = data_rdd.map(lambda text_label: + ([w for w in text_to_words(text_label[0]) if + w in bword_to_ic.value], text_label[1])) + padded_tokens_rdd = tokens_rdd.map( + lambda tokens_label: (pad(tokens_label[0], "##", sequence_len), tokens_label[1])) + vector_rdd = padded_tokens_rdd.map(lambda tokens_label: + ([to_vec(w, bfiltered_w2v.value, + embedding_dim) for w in + tokens_label[0]], tokens_label[1])) + sample_rdd = vector_rdd.map( + lambda vectors_label: to_sample(vectors_label[0], vectors_label[1], embedding_dim)) + + train_rdd, val_rdd = sample_rdd.randomSplit( + [training_split, 1-training_split]) + + optimizer = Optimizer( + model=build_model(news20.CLASS_NUM), + training_rdd=train_rdd, + criterion=ClassNLLCriterion(), + end_trigger=MaxEpoch(max_epoch), + batch_size=batch_size, + optim_method=Adagrad(learningrate=0.01, learningrate_decay=0.0002)) + + optimizer.set_validation( + batch_size=batch_size, + val_rdd=val_rdd, + trigger=EveryEpoch(), + val_method=["Top1Accuracy"] + ) + train_model = optimizer.optimize() + +if __name__ == "__main__": + parser = OptionParser() + parser.add_option("-a", "--action", dest="action", default="train") + parser.add_option("-b", "--batchSize", dest="batchSize", default="128") + parser.add_option("-e", "--embedding_dim", dest="embedding_dim", default="50") # noqa + parser.add_option("-m", "--max_epoch", dest="max_epoch", default="15") + parser.add_option("--model", dest="model_type", default="cnn") + parser.add_option("-p", "--p", dest="p", default="0.0") + + (options, args) = parser.parse_args(sys.argv) + if options.action == "train": + batch_size = int(options.batchSize) + embedding_dim = int(options.embedding_dim) + max_epoch = int(options.max_epoch) + p = float(options.p) + model_type = options.model_type + sequence_len = 50 + max_words = 1000 + training_split = 0.8 + sc = SparkContext(appName="text_classifier", + conf=create_spark_conf()) + init_engine() + train(sc, + batch_size, + sequence_len, max_words, embedding_dim, training_split) + elif options.action == "test": + pass diff --git a/python/dllib/src/bigdl/dllib/nn/__init__.py b/python/dllib/src/bigdl/dllib/nn/__init__.py new file mode 100644 index 00000000000..2151a805423 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/nn/__init__.py @@ -0,0 +1,15 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/python/dllib/src/bigdl/dllib/nn/criterion.py b/python/dllib/src/bigdl/dllib/nn/criterion.py new file mode 100644 index 00000000000..63ddf6a5bf1 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/nn/criterion.py @@ -0,0 +1,605 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +import sys + +from bigdl.util.common import JavaValue +from bigdl.util.common import callBigDlFunc +from bigdl.util.common import JTensor +from bigdl.nn.layer import Model +import numpy as np + +if sys.version >= '3': + long = int + unicode = str + + +class Criterion(JavaValue): + """ + Criterion is helpful to train a neural network. + Given an input and a target, they compute a gradient according to a given loss function. + """ + def __init__(self, jvalue, bigdl_type, *args): + self.value = jvalue if jvalue else callBigDlFunc( + bigdl_type, JavaValue.jvm_class_constructor(self), *args) + self.bigdl_type = bigdl_type + + def __str__(self): + return self.value.toString() + + def forward(self, input, target): + """ + NB: It's for debug only, please use optimizer.optimize() in production. + Takes an input object, and computes the corresponding loss of the criterion, + compared with `target` + :param input: ndarray or list of ndarray + :param target: ndarray or list of ndarray + :return: value of loss + """ + output = callBigDlFunc(self.bigdl_type, + "criterionForward", + self.value, + Model.check_input(input), + Model.check_input(target)) + return output + + def backward(self, input, target): + """ + NB: It's for debug only, please use optimizer.optimize() in production. + Performs a back-propagation step through the criterion, with respect to the given input. + :param input: ndarray or list of ndarray + :param target: ndarray or list of ndarray + :return: ndarray + """ + output = callBigDlFunc(self.bigdl_type, + "criterionBackward", + self.value, + Model.check_input(input), + Model.check_input(target)) + return Model.convert_output(output) + + @classmethod + def of(cls, jcriterion, bigdl_type="float"): + """ + Create a python Criterion by a java criterion object + :param jcriterion: A java criterion object which created by Py4j + :return: a criterion. + """ + criterion = Criterion(bigdl_type, jcriterion) + criterion.value = jcriterion + criterion.bigdl_type = bigdl_type + return criterion + + +class ClassNLLCriterion(Criterion): + + ''' + The negative log likelihood criterion. + It is useful to train a classification problem with n classes. + If provided, the optional argument weights should be a 1D Tensor + assigning weight to each of the classes. + + :param weights weights of each class + :param size_average whether to average or not + + >>> np.random.seed(123) + >>> weights = np.random.uniform(0, 1, (2,)).astype("float32") + >>> classNLLCriterion = ClassNLLCriterion(weights,True) + creating: createClassNLLCriterion + >>> classNLLCriterion = ClassNLLCriterion() + creating: createClassNLLCriterion + ''' + + def __init__(self, + weights=None, + size_average=True, + bigdl_type="float"): + super(ClassNLLCriterion, self).__init__(None, bigdl_type, + JTensor.from_ndarray(weights), + size_average) + + +class MSECriterion(Criterion): + + ''' + Creates a criterion that measures the mean squared error between n elements + in the input x and output y: + loss(x, y) = 1/n \sum |x_i - y_i|^2 + + If x and y are d-dimensional Tensors with a total of n elements, + the sum operation still operates over all the elements, and divides by n. + The two Tensors must have the same number of elements (but their sizes might be different). + The division by n can be avoided if one sets the internal variable sizeAverage to false. + By default, the losses are averaged over observations for each minibatch. However, + if the field sizeAverage is set to false, the losses are instead summed. + + >>> mSECriterion = MSECriterion() + creating: createMSECriterion + ''' + + def __init__(self, bigdl_type="float"): + super(MSECriterion, self).__init__(None, bigdl_type) + + +class AbsCriterion(Criterion): + + ''' + measures the mean absolute value of the element-wise difference between input + + >>> absCriterion = AbsCriterion(True) + creating: createAbsCriterion + ''' + + def __init__(self, + size_average=True, + bigdl_type="float"): + super(AbsCriterion, self).__init__(None, bigdl_type, + size_average) + + +class ClassSimplexCriterion(Criterion): + + ''' + ClassSimplexCriterion implements a criterion for classification. + It learns an embedding per class, where each class' embedding is a + point on an (N-1)-dimensional simplex, where N is the number of classes. + :param nClasses the number of classes. + + >>> classSimplexCriterion = ClassSimplexCriterion(2) + creating: createClassSimplexCriterion + ''' + + def __init__(self, + n_classes, + bigdl_type="float"): + super(ClassSimplexCriterion, self).__init__(None, bigdl_type, + n_classes) + + +class CosineEmbeddingCriterion(Criterion): + + ''' + Creates a criterion that measures the loss given an input x = {x1, x2}, + a table of two Tensors, and a Tensor label y with values 1 or -1. + + :param margin a number from -1 to 1, 0 to 0.5 is suggested + + >>> cosineEmbeddingCriterion = CosineEmbeddingCriterion(1e-5, True) + creating: createCosineEmbeddingCriterion + ''' + + def __init__(self, + margin=0.0, + size_average=True, + bigdl_type="float"): + super(CosineEmbeddingCriterion, self).__init__(None, bigdl_type, + margin, + size_average) + + +class DistKLDivCriterion(Criterion): + + ''' + The Kullback-Leibler divergence criterion + + :param sizeAverage + + >>> distKLDivCriterion = DistKLDivCriterion(True) + creating: createDistKLDivCriterion + ''' + + def __init__(self, + size_average=True, + bigdl_type="float"): + super(DistKLDivCriterion, self).__init__(None, bigdl_type, + size_average) + + +class HingeEmbeddingCriterion(Criterion): + + ''' + Creates a criterion that measures the loss given an + input x which is a 1-dimensional vector and a label y (1 or -1). + This is usually used for measuring whether two inputs are similar + or dissimilar, + e.g. using the L1 pairwise distance, and is typically used for + learning nonlinear embeddings or semi-supervised learning. + + If x and y are n-dimensional Tensors, the sum operation still operates + over all the elements, and divides by n (this can be avoided if one sets + the internal variable sizeAverage to false). The margin has a default + value of 1, or can be set in the constructor. + + >>> hingeEmbeddingCriterion = HingeEmbeddingCriterion(1e-5, True) + creating: createHingeEmbeddingCriterion + ''' + + def __init__(self, + margin=1, + size_average=True, + bigdl_type="float"): + super(HingeEmbeddingCriterion, self).__init__(None, bigdl_type, + margin, + size_average) + + +class L1HingeEmbeddingCriterion(Criterion): + + ''' + Creates a criterion that measures the loss given an input x = {x1, x2}, + a table of two Tensors, and a label y (1 or -1): + + :param margin + + >>> l1HingeEmbeddingCriterion = L1HingeEmbeddingCriterion(1e-5) + creating: createL1HingeEmbeddingCriterion + ''' + + def __init__(self, + margin=1, + bigdl_type="float"): + super(L1HingeEmbeddingCriterion, self).__init__(None, bigdl_type, + margin) + + +class MarginCriterion(Criterion): + + ''' + Creates a criterion that optimizes a two-class classification hinge loss (margin-based loss) + between input x (a Tensor of dimension 1) and output y. + + :param margin if unspecified, is by default 1. + :param size_average: size average in a mini-batch + + >>> marginCriterion = MarginCriterion(1e-5, True) + creating: createMarginCriterion + ''' + + def __init__(self, + margin=1.0, + size_average=True, + bigdl_type="float"): + super(MarginCriterion, self).__init__(None, bigdl_type, + margin, + size_average) + + +class MarginRankingCriterion(Criterion): + + ''' + Creates a criterion that measures the loss given an input x = {x1, x2}, + a table of two Tensors of size 1 (they contain only scalars), and a label y (1 or -1). + In batch mode, x is a table of two Tensors of size batchsize, and y is a Tensor of size + batchsize containing 1 or -1 for each corresponding pair of elements in the input Tensor. + If y == 1 then it assumed the first input should be ranked higher (have a larger value) than + the second input, and vice-versa for y == -1. + + :param margin + + >>> marginRankingCriterion = MarginRankingCriterion(1e-5, True) + creating: createMarginRankingCriterion + ''' + + def __init__(self, + margin=1.0, + size_average=True, + bigdl_type="float"): + super(MarginRankingCriterion, self).__init__(None, bigdl_type, + margin, + size_average) + + +class MultiCriterion(Criterion): + + ''' + a weighted sum of other criterions each applied to the same input and target + + >>> multiCriterion = MultiCriterion() + creating: createMultiCriterion + ''' + + def __init__(self, + bigdl_type="float"): + super(MultiCriterion, self).__init__(None, bigdl_type) + + +class MultiLabelMarginCriterion(Criterion): + + ''' + Creates a criterion that optimizes a multi-class multi-classification hinge loss ( + margin-based loss) between input x and output y (which is a Tensor of target class indices) + + :param size_average: size average in a mini-batch + + >>> multiLabelMarginCriterion = MultiLabelMarginCriterion(True) + creating: createMultiLabelMarginCriterion + ''' + + def __init__(self, + size_average=True, + bigdl_type="float"): + super(MultiLabelMarginCriterion, self).__init__(None, bigdl_type, + size_average) + + +class ParallelCriterion(Criterion): + + ''' + ParallelCriterion is a weighted sum of other criterions each applied to a different input + and target. Set repeatTarget = true to share the target for criterions. + + Use add(criterion[, weight]) method to add criterion. Where weight is a scalar(default 1). + + :param repeat_target Whether to share the target for all criterions. + + >>> parallelCriterion = ParallelCriterion(True) + creating: createParallelCriterion + ''' + + def __init__(self, + repeat_target=False, + bigdl_type="float"): + super(ParallelCriterion, self).__init__(None, bigdl_type, + repeat_target) + + +class SmoothL1Criterion(Criterion): + + ''' + Creates a criterion that can be thought of as a smooth version of the AbsCriterion. + It uses a squared term if the absolute element-wise error falls below 1. + It is less sensitive to outliers than the MSECriterion and in some + cases prevents exploding gradients (e.g. see "Fast R-CNN" paper by Ross Girshick). + | 0.5 * (x_i - y_i)^2^, if |x_i - y_i| < 1 + loss(x, y) = 1/n \sum | + | |x_i - y_i| - 0.5, otherwise + If x and y are d-dimensional Tensors with a total of n elements, + the sum operation still operates over all the elements, and divides by n. + The division by n can be avoided if one sets the internal variable sizeAverage to false + + :param size_average whether to average the loss + + >>> smoothL1Criterion = SmoothL1Criterion(True) + creating: createSmoothL1Criterion + ''' + + def __init__(self, + size_average=True, + bigdl_type="float"): + super(SmoothL1Criterion, self).__init__(None, bigdl_type, + size_average) + + +class SmoothL1CriterionWithWeights(Criterion): + + ''' + a smooth version of the AbsCriterion + It uses a squared term if the absolute element-wise error falls below 1. + It is less sensitive to outliers than the MSECriterion and in some cases + prevents exploding gradients (e.g. see "Fast R-CNN" paper by Ross Girshick). + + d = (x - y) * w_in + loss(x, y, w_in, w_out) + | 0.5 * (sigma * d_i)^2 * w_out if |d_i| < 1 / sigma / sigma + = 1/n \sum | + | (|d_i| - 0.5 / sigma / sigma) * w_out otherwise + + >>> smoothL1CriterionWithWeights = SmoothL1CriterionWithWeights(1e-5, 1) + creating: createSmoothL1CriterionWithWeights + ''' + + def __init__(self, + sigma, + num=0, + bigdl_type="float"): + super(SmoothL1CriterionWithWeights, self).__init__(None, bigdl_type, + sigma, + num) + + +class SoftmaxWithCriterion(Criterion): + + ''' + Computes the multinomial logistic loss for a one-of-many classification task, + passing real-valued predictions through a softmax to get a probability distribution over classes. + It should be preferred over separate SoftmaxLayer + MultinomialLogisticLossLayer + as its gradient computation is more numerically stable. + :param ignoreLabel (optional) Specify a label value that + should be ignored when computing the loss. + :param normalizeMode How to normalize the output loss. + + >>> softmaxWithCriterion = SoftmaxWithCriterion() + creating: createSoftmaxWithCriterion + >>> softmaxWithCriterion = SoftmaxWithCriterion(1, "FULL") + creating: createSoftmaxWithCriterion + ''' + + def __init__(self, + ignore_label=None, + normalize_mode="VALID", + bigdl_type="float"): + super(SoftmaxWithCriterion, self).__init__(None, bigdl_type, + ignore_label, + normalize_mode) + + +class TimeDistributedCriterion(Criterion): + ''' + This class is intended to support inputs with 3 or more dimensions. + Apply Any Provided Criterion to every temporal slice of an input. + + :param criterion: embedded criterion + :param size_average: whether to divide the sequence length + + >>> td = TimeDistributedCriterion(ClassNLLCriterion()) + creating: createClassNLLCriterion + creating: createTimeDistributedCriterion + ''' + + def __init__(self, criterion, size_average=False, bigdl_type="float"): + super(TimeDistributedCriterion, self).__init__( + None, bigdl_type, criterion, size_average) + + +class CrossEntropyCriterion(Criterion): + """ + This criterion combines LogSoftMax and ClassNLLCriterion in one single class. + + :param weights A tensor assigning weight to each of the classes + + >>> np.random.seed(123) + >>> weights = np.random.uniform(0, 1, (2,)).astype("float32") + >>> cec = CrossEntropyCriterion(weights) + creating: createCrossEntropyCriterion + >>> cec = CrossEntropyCriterion() + creating: createCrossEntropyCriterion + """ + + def __init__(self, + weights=None, + size_average=True, + bigdl_type="float"): + super(CrossEntropyCriterion, self).__init__(None, bigdl_type, + JTensor.from_ndarray( + weights), + size_average) + + +class BCECriterion(Criterion): + ''' + Creates a criterion that measures the Binary Cross Entropy + between the target and the output + + :param weights weights for each class + :param sizeAverage whether to average the loss or not + + >>> np.random.seed(123) + >>> weights = np.random.uniform(0, 1, (2,)).astype("float32") + >>> bCECriterion = BCECriterion(weights) + creating: createBCECriterion + >>> bCECriterion = BCECriterion() + creating: createBCECriterion + ''' + + def __init__(self, + weights=None, + size_average=True, + bigdl_type="float"): + super(BCECriterion, self).__init__(None, bigdl_type, + JTensor.from_ndarray(weights), + size_average) + + +class MultiLabelSoftMarginCriterion(Criterion): + ''' + A MultiLabel multiclass criterion based on sigmoid: + the loss is: + l(x,y) = - sum_i y[i] * log(p[i]) + (1 - y[i]) * log (1 - p[i]) + where p[i] = exp(x[i]) / (1 + exp(x[i])) + and with weights: + l(x,y) = - sum_i weights[i] (y[i] * log(p[i]) + (1 - y[i]) * log (1 - p[i])) + + >>> np.random.seed(123) + >>> weights = np.random.uniform(0, 1, (2,)).astype("float32") + >>> multiLabelSoftMarginCriterion = MultiLabelSoftMarginCriterion(weights) + creating: createMultiLabelSoftMarginCriterion + >>> multiLabelSoftMarginCriterion = MultiLabelSoftMarginCriterion() + creating: createMultiLabelSoftMarginCriterion + ''' + + def __init__(self, + weights=None, + size_average=True, + bigdl_type="float"): + super(MultiLabelSoftMarginCriterion, self).__init__(None, bigdl_type, + JTensor.from_ndarray(weights), + size_average) + + +class MultiMarginCriterion(Criterion): + ''' + Creates a criterion that optimizes a multi-class classification hinge loss (margin-based loss) + between input x and output y (which is a target class index). + + :param p + :param weights + :param margin + :param size_average + + >>> np.random.seed(123) + >>> weights = np.random.uniform(0, 1, (2,)).astype("float32") + >>> multiMarginCriterion = MultiMarginCriterion(1,weights) + creating: createMultiMarginCriterion + >>> multiMarginCriterion = MultiMarginCriterion() + creating: createMultiMarginCriterion + ''' + + def __init__(self, + p=1, + weights=None, + margin=1.0, + size_average=True, + bigdl_type="float"): + super(MultiMarginCriterion, self).__init__(None, bigdl_type, + p, + JTensor.from_ndarray(weights), + margin, + size_average) + + +class SoftMarginCriterion(Criterion): + """ + Creates a criterion that optimizes a two-class classification logistic loss + between input x (a Tensor of dimension 1) and output y (which is a tensor + containing either 1s or -1s). + + loss(x, y) = sum_i (log(1 + exp(-y[i]*x[i]))) / x:nElement() + + :param sizeaverage The normalization by the number of elements in the input + can be disabled by setting + + >>> softMarginCriterion = SoftMarginCriterion(False) + creating: createSoftMarginCriterion + >>> softMarginCriterion = SoftMarginCriterion() + creating: createSoftMarginCriterion + """ + + def __init__(self, + size_average=True, + bigdl_type="float"): + super(SoftMarginCriterion, self).__init__(None, bigdl_type, size_average) + +def _test(): + import doctest + from pyspark import SparkContext + from bigdl.nn import criterion + from bigdl.util.common import init_engine + from bigdl.util.common import create_spark_conf + globs = criterion.__dict__.copy() + sc = SparkContext(master="local[4]", appName="test criterion", + conf=create_spark_conf()) + globs['sc'] = sc + init_engine() + + (failure_count, test_count) = doctest.testmod(globs=globs, + optionflags=doctest.ELLIPSIS) + if failure_count: + exit(-1) + + +if __name__ == "__main__": + _test() diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py new file mode 100644 index 00000000000..5a8133dd0e0 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -0,0 +1,3005 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +import sys + +import numpy as np + +from bigdl.util.common import JTensor +from bigdl.util.common import JavaValue +from bigdl.util.common import callBigDlFunc +from bigdl.util.common import callJavaFunc +from bigdl.util.common import get_spark_context +from bigdl.util.common import to_list + +if sys.version >= '3': + long = int + unicode = str + +INTMAX = 2147483647 +INTMIN = -2147483648 +DOUBLEMAX = 1.7976931348623157E308 + + +class Node(JavaValue): + """ + Represent a node in a graph. The connections between nodes are directed. + """ + def __init__(self, jvalue, bigdl_type, *args): + self.value = jvalue if jvalue else callBigDlFunc( + bigdl_type, JavaValue.jvm_class_constructor(self), *args) + self.bigdl_type = bigdl_type + + @classmethod + def of(cls, jvalue, bigdl_type="float"): + return Node(jvalue, bigdl_type) + + def element(self): + return Model.of(self.value.element()) + + +class Model(JavaValue): + """ + Model is the basic component of a neural network + and it's also the base class of layers. + Model can connect to others to construct a complex neural network. + """ + + def __init__(self, jvalue, bigdl_type, *args): + self.value = jvalue if jvalue else callBigDlFunc( + bigdl_type, JavaValue.jvm_class_constructor(self), *args) + self.bigdl_type = bigdl_type + + def __str__(self): + """ + >>> conv2 = SpatialConvolution(6, 12, 5, 5).set_name("conv2") + creating: createSpatialConvolution + >>> print(conv2) + SpatialConvolution[conv2](6 -> 12, 5 x 5, 1, 1, 0, 0) + """ + return self.value.toString() + + def __call__(self, x=None): + """ + Some other modules point to current module + :param x: upstream module nodes. x is either a Node or list of Node. + :return: node containing current module + """ + + x = x if x else [] + return Node.of(callBigDlFunc(self.bigdl_type, + "createNode", + self, + to_list(x))) + + @classmethod + def of(cls, jmodel, bigdl_type="float"): + """ + Create a Python Model + :param jmodel: Java model create by Py4j + :return: A Python Model + """ + model = Model(jmodel,bigdl_type) + return model + + def set_name(self, name): + """ + Give this model a name. There would be a generated name + consist of class name and UUID if user doesn't set it. + """ + callJavaFunc(get_spark_context(), self.value.setName, name) + return self + + def name(self): + """ + Name of this layer + """ + return callJavaFunc(get_spark_context(), self.value.getName) + + def set_seed(self, seed=123): + """ + You can control the random seed which used to init weights for this model. + :param seed: random seed + :return: Model itself. + """ + callBigDlFunc(self.bigdl_type, "setModelSeed", seed) + return self + + def get_dtype(self): + if "float" == self.bigdl_type: + return "float32" + else: + return "float64" + @staticmethod + def check_input(input): + if type(input) is list: + if len(input) == 0: + raise Exception('Error when checking: empty input') + if not hasattr(input[0], 'shape'): + raise Exception( + 'Error when checking: expecting list of ndarray') + return [JTensor.from_ndarray(i) for i in input] + else: + if not hasattr(input, 'shape'): + raise Exception( + 'Error when checking: expecting list of ndarray') + return [JTensor.from_ndarray(input)] + + @staticmethod + def convert_output(output): + if type(output) is JTensor: + return output.to_ndarray() + else: + return [x.to_ndarray() for x in output] + + def forward(self, input): + """ + NB: It's for debug only, please use optimizer.optimize() in production. + Takes an input object, and computes the corresponding output of the module + :param input: ndarray or list of ndarray + :return: ndarray or list of ndarray + """ + output = callBigDlFunc(self.bigdl_type, + "modelForward", + self.value, + self.check_input(input)) + return self.convert_output(output) + + def backward(self, input, grad_output): + """ + NB: It's for debug only, please use optimizer.optimize() in production. + Performs a back-propagation step through the module, with respect to the given input. In + general this method makes the assumption forward(input) has been called before, with the same + input. This is necessary for optimization reasons. If you do not respect this rule, backward() + will compute incorrect gradients. + :param input: ndarray or list of ndarray + :param grad_output: ndarray or list of ndarray + :return: ndarray or list of ndarray + """ + output = callBigDlFunc(self.bigdl_type, + "modelBackward", + self.value, + self.check_input(input), + self.check_input(grad_output)) + return self.convert_output(output) + + def reset(self): + """ + Initialize the model weights. + """ + callJavaFunc(get_spark_context(), self.value.reset) + return self + + def parameters(self): + """ + Get the model parameters which containing: weight, bias, gradBias, gradWeight + :return: dict(layername -> dict(parametername -> ndarray)) + """ + name_to_params = callBigDlFunc(self.bigdl_type, + "modelGetParameters", + self.value) + + def to_ndarray(params): + return { + param_name: np.array(values[0], + dtype=self.get_dtype()).reshape( + values[1]) for param_name, values in params.items()} + + return {layer_name: to_ndarray(params) for layer_name, params in + name_to_params.items()} + + def predict(self, data_rdd): + """ + Model inference base on the given data. + You need to invoke collect() to trigger those action \ + as the returning result is an RDD. + :param data_rdd: the data to be predict. + :return: An RDD represent the predict result. + """ + result = callBigDlFunc(self.bigdl_type, + "modelPredictRDD", self.value, data_rdd) + return result.map(lambda data: data.to_ndarray()) + + def test(self, val_rdd, batch_size, val_methods): + """ + A method to benchmark the model quality. + :param val_rdd: the input data + :param batch_size: batch size + :param val_methods: a list of validation methods. i.e: Top1Accuracy, + Top5Accuracy and Loss. + :return: + """ + return callBigDlFunc(self.bigdl_type, + "modelTest", + self.value, + val_rdd, batch_size, val_methods) + + def set_weights(self, weights): + """ + Set weights for this layer + :param weights: a list of numpy arrays which represent weight and bias + :return: + >>> linear = Linear(3,2) + creating: createLinear + >>> linear.set_weights([np.array([[1,2,3],[4,5,6]]), np.array([7,8])]) + >>> weights = linear.get_weights() + >>> weights[0].shape == (2,3) + True + >>> weights[0][0] + array([ 1., 2., 3.], dtype=float32) + >>> weights[1] + array([ 7., 8.], dtype=float32) + >>> relu = ReLU() + creating: createReLU + >>> from py4j.protocol import Py4JJavaError + >>> try: + ... relu.set_weights([np.array([[1,2,3],[4,5,6]]), np.array([7,8])]) + ... except Py4JJavaError as err: + ... print(err.java_exception) + ... + java.lang.IllegalArgumentException: requirement failed: this layer does not have weight/bias + >>> relu.get_weights() + The layer does not have weight/bias + >>> add = Add(2) + creating: createAdd + >>> try: + ... add.set_weights([np.array([7,8]), np.array([1,2])]) + ... except Py4JJavaError as err: + ... print(err.java_exception) + ... + java.lang.IllegalArgumentException: requirement failed: the number of input weight/bias is not consistant with number of weight/bias of this layer + """ + tensors = [JTensor.from_ndarray(param, self.bigdl_type) for param in weights] + callBigDlFunc(self.bigdl_type, "setWeights", self.value, tensors) + + def get_weights(self): + """ + Get weights for this layer + :return: list of numpy arrays which represent weight and bias + """ + tensorWeights = callBigDlFunc(self.bigdl_type, + "getWeights", self.value) + if tensorWeights is not None: + return [tensor.to_ndarray() for tensor in tensorWeights] + else: + print("The layer does not have weight/bias") + return None + + @staticmethod + def load(path, bigdl_type="float"): + """ + Load a pre-trained Bigdl model. + :param path: The path containing the pre-trained model. + :return: A pre-trained model. + """ + jmodel = callBigDlFunc(bigdl_type, "loadBigDL", path) + return Model.of(jmodel) + + @staticmethod + def load_torch(path, bigdl_type="float"): + """ + Load a pre-trained Torch model. + :param path: The path containing the pre-trained model. + :return: A pre-trained model. + """ + jmodel = callBigDlFunc(bigdl_type, "loadTorch", path) + return Model.of(jmodel) + + @staticmethod + def load_caffe(model, defPath, modelPath, match_all=True, bigdl_type="float"): + """ + Load a pre-trained Caffe model. + + :param model: A bigdl model definition \ + which equivalent to the pre-trained caffe model. + :param defPath: The path containing the caffe model definition. + :param modelPath: The path containing the pre-trained caffe model. + :return: A pre-trained model. + """ + jmodel = callBigDlFunc(bigdl_type, "loadCaffe", model, defPath, modelPath, match_all) + return Model.of(jmodel) + +class Container(Model): + ''' + [[Container]] is a sub-class of Model that declares methods defined in all containers. + A container usually contain some other modules which can be added through the "add" method + ''' + + def __init__(self, jvalue, bigdl_type, *args): + super(Container, self).__init__(jvalue, bigdl_type, *args) + + def add(self, model): + self.value.add(model.value) + return self + + + + +class Linear(Model): + + ''' + The [[Linear]] module applies a linear transformation to the input data, + i.e. `y = Wx + b`. The input given in `forward(input)` must be either + a vector (1D tensor) or matrix (2D tensor). If the input is a vector, it must + have the size of `inputSize`. If it is a matrix, then each row is assumed to be + an input sample of given batch (the number of rows means the batch size and + the number of columns should be equal to the `inputSize`). + + :param input_size the size the each input sample + :param output_size the size of the module output of each sample + :param init_method two initialized methods are supported here, which are [[Default]] + and [[Xavier]], where [[Xavier]] set bias to zero here. For more + detailed information about `initMethod`, please refer to + [[InitializationMethod]] + + >>> linear = Linear(100, 10, "Xavier") + creating: createLinear + ''' + + def __init__(self, input_size, output_size, init_method="default", with_bias=True, + bigdl_type="float"): + super(Linear, self).__init__(None, bigdl_type, input_size, output_size, + init_method, with_bias) + + +class ReLU(Model): + + ''' + Applies the rectified linear unit (ReLU) function element-wise to the input Tensor, + thus outputting a Tensor of the same dimension. + + ReLU is defined as: f(x) = max(0, x) + Can optionally do its operation in-place without using extra state memory + + >>> relu = ReLU() + creating: createReLU + ''' + + def __init__(self, ip=False, bigdl_type="float"): + super(ReLU, self).__init__(None, bigdl_type, ip) + + +class Tanh(Model): + + ''' + Applies the Tanh function element-wise to the input Tensor, thus outputting a Tensor of the same + dimension. Tanh is defined as f(x) = (exp(x)-exp(-x))/(exp(x)+exp(-x)). + + >>> tanh = Tanh() + creating: createTanh + ''' + + def __init__(self, bigdl_type="float"): + super(Tanh, self).__init__(None, bigdl_type) + + +class Echo(Model): + + ''' + This module is for debug purpose, which can print activation and gradient in your model + topology + + >>> echo = Echo() + creating: createEcho + ''' + + def __init__(self, bigdl_type="float"): + super(Echo, self).__init__(None, bigdl_type) + + +class LogSoftMax(Model): + + ''' + Applies the LogSoftMax function to an n-dimensional input Tensor. + LogSoftmax is defined as: f_i(x) = log(1 / a exp(x_i)) + where a = sum_j[exp(x_j)]. + + >>> logSoftMax = LogSoftMax() + creating: createLogSoftMax + ''' + + def __init__(self, bigdl_type="float"): + super(LogSoftMax, self).__init__(None, bigdl_type) + + +class Sequential(Container): + + ''' + Sequential provides a means to plug layers together + in a feed-forward fully connected manner. + + >>> echo = Echo() + creating: createEcho + >>> s = Sequential() + creating: createSequential + >>> s = s.add(echo) + >>> s = s.add(s) + >>> s = s.add(echo) + + ''' + + def __init__(self, bigdl_type="float"): + super(Sequential, self).__init__(None, bigdl_type) + + +class SpatialConvolution(Model): + + ''' + Applies a 2D convolution over an input image composed of several input planes. + The input tensor in forward(input) is expected to be + a 3D tensor (nInputPlane x height x width). + + :param n_input_plane The number of expected input planes in the image given into forward() + :param n_output_plane The number of output planes the convolution layer will produce. + :param kernel_w The kernel width of the convolution + :param kernel_h The kernel height of the convolution + :param stride_w The step of the convolution in the width dimension. + :param stride_h The step of the convolution in the height dimension + :param pad_w The additional zeros added per width to the input planes. + :param pad_h The additional zeros added per height to the input planes. + :param n_group Kernel group number + :param propagate_back Propagate gradient back + :param init_method Initialization method to initialize bias and weight + + >>> spatialConvolution = SpatialConvolution(6, 12, 5, 5) + creating: createSpatialConvolution + ''' + + def __init__(self, + n_input_plane, + n_output_plane, + kernel_w, + kernel_h, + stride_w=1, + stride_h=1, + pad_w=0, + pad_h=0, + n_group=1, + propagate_back=True, + init_method="default", + bigdl_type="float"): + super(SpatialConvolution, self).__init__(None, bigdl_type, + n_input_plane, + n_output_plane, + kernel_w, + kernel_h, + stride_w, + stride_h, + pad_w, + pad_h, + n_group, + propagate_back, + init_method) + + +class SpatialMaxPooling(Model): + + ''' + Applies 2D max-pooling operation in kWxkH regions by step size dWxdH steps. + The number of output features is equal to the number of input planes. + If the input image is a 3D tensor nInputPlane x height x width, + the output image size will be nOutputPlane x oheight x owidth where + owidth = op((width + 2*padW - kW) / dW + 1) + oheight = op((height + 2*padH - kH) / dH + 1) + op is a rounding operator. By default, it is floor. + It can be changed by calling :ceil() or :floor() methods. + :param kW kernel width + :param kH kernel height + :param dW step size in width + :param dH step size in height + :param padW padding in width + :param padH padding in height + >>> spatialMaxPooling = SpatialMaxPooling(2, 2, 2, 2) + creating: createSpatialMaxPooling + ''' + # to_ceil: call floor() when False; call ceil() when True + + def __init__(self, kw, + kh, + dw, + dh, + pad_w=0, + pad_h=0, + to_ceil=False, + bigdl_type="float"): + super(SpatialMaxPooling, self).__init__(None, bigdl_type, kw, + kh, + dw, + dh, + pad_w, + pad_h, + to_ceil) + + +class Select(Model): + + ''' + A Simple layer selecting an index of the input tensor in the given dimension + + :param dimension the dimension to select + :param index the index of the dimension to be selected + + >>> select = Select(1, 1) + creating: createSelect + ''' + + def __init__(self, dim, index, bigdl_type="float"): + super(Select, self).__init__(None, bigdl_type, dim, index) + +class Recurrent(Container): + ''' + Recurrent module is a container of rnn cells + Different types of rnn cells can be added using add() function + + >>> recurrent = Recurrent() + creating: createRecurrent + ''' + + def __init__(self, bigdl_type="float"): + super(Recurrent, self).__init__(None, bigdl_type) + + +class LSTM(Model): + ''' + Long Short Term Memory architecture. + Ref. + A.: http://arxiv.org/pdf/1303.5778v1 (blueprint for this module) + B. http://web.eecs.utk.edu/~itamar/courses/ECE-692/Bobby_paper1.pdf + C. http://arxiv.org/pdf/1503.04069v1.pdf + D. https://github.com/wojzaremba/lstm + E. https://github.com/Element-Research/rnn/blob/master/FastLSTM.lua + + :param inputSize: the size of each input vector + :param hiddenSize: Hidden unit size in the LSTM + :param p: is used for [[Dropout]] probability. For more details about + RNN dropouts, please refer to + [RnnDrop: A Novel Dropout for RNNs in ASR] + (http://www.stat.berkeley.edu/~tsmoon/files/Conference/asru2015.pdf) + [A Theoretically Grounded Application of Dropout in Recurrent Neural Networks] + (https://arxiv.org/pdf/1512.05287.pdf) + + >>> lstm = LSTM(4, 3, 0.5) + creating: createLSTM + ''' + + def __init__(self, input_size, hidden_size, p=0.0, bigdl_type="float"): + super(LSTM, self).__init__(None, bigdl_type, input_size, hidden_size, p) + + +class LSTMPeephole(Model): + ''' + Long Short Term Memory architecture with peephole. + Ref. A.: http://arxiv.org/pdf/1303.5778v1 (blueprint for this module) + B. http://web.eecs.utk.edu/~itamar/courses/ECE-692/Bobby_paper1.pdf + C. http://arxiv.org/pdf/1503.04069v1.pdf + D. https://github.com/wojzaremba/lstm + E. https://github.com/Element-Research/rnn/blob/master/LSTM.lua + + :param input_size: the size of each input vector + :param hidden_size: Hidden unit size in the LSTM + :param p: is used for [[Dropout]] probability. For more details about + RNN dropouts, please refer to + [RnnDrop: A Novel Dropout for RNNs in ASR] + (http://www.stat.berkeley.edu/~tsmoon/files/Conference/asru2015.pdf) + [A Theoretically Grounded Application of Dropout in Recurrent Neural Networks] + (https://arxiv.org/pdf/1512.05287.pdf) + + >>> lstm = LSTMPeephole(4, 3, 0.5) + creating: createLSTMPeephole + ''' + + def __init__(self, input_size, hidden_size, p=0.0, bigdl_type="float"): + super(LSTMPeephole, self).__init__(None, bigdl_type, input_size, hidden_size, p) + + +class GRU(Model): + ''' + Gated Recurrent Units architecture. + The first input in sequence uses zero value for cell and hidden state + + Ref. + http://www.wildml.com/2015/10/recurrent-neural-network-tutorial-part-4-implementing-a-grulstm-rnn-with-python-and-theano/ + https://github.com/Element-Research/rnn/blob/master/GRU.lua + + :param input_size: the size of each input vector + :param hidden_size: Hidden unit size in GRU + :param p: is used for [[Dropout]] probability. For more details about + RNN dropouts, please refer to + [RnnDrop: A Novel Dropout for RNNs in ASR] + (http://www.stat.berkeley.edu/~tsmoon/files/Conference/asru2015.pdf) + [A Theoretically Grounded Application of Dropout in Recurrent Neural Networks] + (https://arxiv.org/pdf/1512.05287.pdf) + + >>> gru = GRU(4, 3, 0.5) + creating: createGRU + ''' + + def __init__(self, input_size, hidden_size, p=0.0, bigdl_type="float"): + super(GRU, self).__init__(None, bigdl_type, input_size, hidden_size, p) + + +class RnnCell(Model): + ''' + It is a simple RNN. User can pass an activation function to the RNN. + + :param input_size: the size of each input vector + :param hidden_size: Hidden unit size in simple RNN + :param activation: activation function + + >>> reshape = RnnCell(4, 3, Tanh()) + creating: createTanh + creating: createRnnCell + ''' + + def __init__(self, + input_size, + hidden_size, + activation, + bigdl_type="float"): + super(RnnCell, self).__init__(None, bigdl_type, input_size, hidden_size, activation) + + +class TimeDistributed(Model): + ''' + This layer is intended to apply contained layer to each temporal time slice + of input tensor. + + For instance, The TimeDistributed Layer can feed each time slice of input tensor + to the Linear layer. + + >>> td = TimeDistributed(Linear(2, 3)) + creating: createLinear + creating: createTimeDistributed + ''' + + def __init__(self, model, bigdl_type="float"): + super(TimeDistributed, self).__init__(None, bigdl_type, model) + + +class Concat(Container): + + ''' + Concat concatenates the output of one layer of "parallel" + modules along the provided {@code dimension}: they take the + same inputs, and their output is concatenated. + +-----------+ + +----> module1 -----+ + | | | | + input -----+----> module2 -----+----> output + | | | | + +----> module3 -----+ + +-----------+ + + :param dimension: dimension + + >>> concat = Concat(2) + creating: createConcat + ''' + + def __init__(self, + dimension, + bigdl_type="float"): + super(Concat, self).__init__(None, bigdl_type, + dimension) + + +class SpatialAveragePooling(Model): + + ''' + Applies 2D average-pooling operation in kWxkH regions by step size dWxdH steps. + The number of output features is equal to the number of input planes. + + :param kW kernel width + :param kH kernel height + :param dW step width + :param dH step height + :param padW padding width + :param padH padding height + :param ceilMode whether the output size is to be ceiled or floored + :param countIncludePad whether to include padding when dividing the + number of elements in pooling region + :param divide whether to do the averaging + + >>> spatialAveragePooling = SpatialAveragePooling(7,7) + creating: createSpatialAveragePooling + ''' + + def __init__(self, + kw, + kh, + dw=1, + dh=1, + pad_w=0, + pad_h=0, + ceil_mode=False, + count_include_pad=True, + divide=True, + bigdl_type="float"): + super(SpatialAveragePooling, self).__init__(None, bigdl_type, + kw, + kh, + dw, + dh, + pad_w, + pad_h, + ceil_mode, + count_include_pad, + divide) + + +class SpatialBatchNormalization(Model): + + ''' + This file implements Batch Normalization as described in the paper: + "Batch Normalization: Accelerating Deep Network Training by Reducing Internal Covariate Shift" + by Sergey Ioffe, Christian Szegedy + This implementation is useful for inputs coming from convolution layers. + For non-convolutional layers, see [[BatchNormalization]] + The operation implemented is: + + ( x - mean(x) ) + y = -------------------- * gamma + beta + standard-deviation(x) + + where gamma and beta are learnable parameters. + The learning of gamma and beta is optional. + + >>> spatialBatchNormalization = SpatialBatchNormalization(1) + creating: createSpatialBatchNormalization + ''' + + def __init__(self, + n_output, + eps=1e-5, + momentum=0.1, + affine=True, + bigdl_type="float"): + super(SpatialBatchNormalization, self).__init__(None, bigdl_type, + n_output, + eps, + momentum, + affine) + + +class SpatialCrossMapLRN(Model): + + ''' + Applies Spatial Local Response Normalization between different feature maps. + The operation implemented is: + x_f + y_f = ------------------------------------------------- + (k+(alpha/size)* sum_{l=l1 to l2} (x_l^2^))^beta^ + + where x_f is the input at spatial locations h,w (not shown for simplicity) and feature map f, + l1 corresponds to max(0,f-ceil(size/2)) and l2 to min(F, f-ceil(size/2) + size). + Here, F is the number of feature maps. + :param size: the number of channels to sum over (for cross channel LRN) or the side length of + the square region to sum over (for within channel LRN) + :param alpha: the scaling parameter + :param beta: the exponent + :param k: a constant + + >>> spatialCrossMapLRN = SpatialCrossMapLRN() + creating: createSpatialCrossMapLRN + ''' + + def __init__(self, + size=5, + alpha=1.0, + beta=0.75, + k=1.0, + bigdl_type="float"): + super(SpatialCrossMapLRN, self).__init__(None, bigdl_type, + size, + alpha, + beta, + k) + + +class Dropout(Model): + + ''' + Dropout masks(set to zero) parts of input using a bernoulli distribution. + Each input element has a probability initP of being dropped. If scale is + set, the outputs are scaled by a factor of 1/(1-initP) during training. + During evaluating, output is the same as input. + + :param initP: probability to be dropped + :param inplace: inplace model + :param scale: if scale by a factor of 1/(1-initP) + + >>> dropout = Dropout(0.4) + creating: createDropout + ''' + + def __init__(self, + init_p=0.5, + inplace=False, + scale=True, + bigdl_type="float"): + super(Dropout, self).__init__(None, bigdl_type, + init_p, + inplace, + scale) + + +class View(Model): + + ''' + This module creates a new view of the input tensor using the sizes passed to the constructor. + The method setNumInputDims() allows to specify the expected number of dimensions of the + inputs of the modules. This makes it possible to use minibatch inputs when using a size -1 + for one of the dimensions. + + :param size: sizes use for creates a new view + + >>> view = View([1024,2]) + creating: createView + ''' + + def __init__(self, + sizes, + num_input_dims=0, + bigdl_type="float"): + super(View, self).__init__(None, bigdl_type, + sizes, + num_input_dims) + + +class Abs(Model): + + ''' + an element-wise abs operation + + >>> abs = Abs() + creating: createAbs + ''' + + def __init__(self, + bigdl_type="float"): + super(Abs, self).__init__(None, bigdl_type) + + +class Add(Model): + + ''' + adds a bias term to input data ; + :param input_size size of input data + >>> add = Add(1) + creating: createAdd + ''' + + def __init__(self, + input_size, + bigdl_type="float"): + super(Add, self).__init__(None, bigdl_type, + input_size) + + +class AddConstant(Model): + + ''' + adding a constant + + :param constant_scalar constant value + :param inplace Can optionally do its operation in-place without using extra state memory + + >>> addConstant = AddConstant(1e-5, True) + creating: createAddConstant + ''' + + def __init__(self, + constant_scalar, + inplace=False, + bigdl_type="float"): + super(AddConstant, self).__init__(None, bigdl_type, + constant_scalar, + inplace) + +class BatchNormalization(Model): + + ''' + This layer implements Batch Normalization as described in the paper: + "Batch Normalization: Accelerating Deep Network Training by Reducing Internal + Covariate Shift" + by Sergey Ioffe, Christian Szegedy https://arxiv.org/abs/1502.03167 + + This implementation is useful for inputs NOT coming from convolution layers. For convolution + layers, use nn.SpatialBatchNormalization. + + The operation implemented is: + ( x - mean(x) ) + y = -------------------- * gamma + beta + standard-deviation(x) + where gamma and beta are learnable parameters.The learning of gamma and beta is optional. + + :param n_output: output feature map number + :param eps: avoid divide zero + :param momentum: momentum for weight update + :param affine: affine operation on output or not + + >>> batchNormalization = BatchNormalization(1, 1e-5, 1e-5, True) + creating: createBatchNormalization + ''' + def __init__(self, + n_output, + eps=1e-5, + momentum=0.1, + affine=True, + bigdl_type="float"): + super(BatchNormalization, self).__init__(None, bigdl_type, + n_output, + eps, + momentum, + affine) + + +class Bilinear(Model): + + ''' + a bilinear transformation with sparse inputs, + The input tensor given in forward(input) is a table containing both inputs x_1 and x_2, + which are tensors of size N x inputDimension1 and N x inputDimension2, respectively. + + :param input_size1 input dimension of x_1 + :param input_size2 input dimension of x_2 + :param output_size output dimension + :param bias_res whether use bias + + >>> bilinear = Bilinear(1, 1, 1, True) + creating: createBilinear + ''' + + def __init__(self, + input_size1, + input_size2, + output_size, + bias_res=True, + bigdl_type="float"): + super(Bilinear, self).__init__(None, bigdl_type, + input_size1, + input_size2, + output_size, + bias_res) + + +class Bottle(Container): + + ''' + Bottle allows varying dimensionality input to be forwarded through any module + that accepts input of nInputDim dimensions, and generates output of nOutputDim dimensions. + :param module: transform module + :param n_input_dim: nInputDim dimensions of module + :param n_output_dim1: output of nOutputDim dimensions + + >>> bottle = Bottle(Linear(100,10), 1, 1) + creating: createLinear + creating: createBottle + ''' + + def __init__(self, + module, + n_input_dim=2, + n_output_dim1=INTMAX, + bigdl_type="float"): + super(Bottle, self).__init__(None, bigdl_type, + module, + n_input_dim, + n_output_dim1) + + +class CAdd(Model): + + ''' + This layer has a bias tensor with given size. The bias will be added element wise to the input + tensor. If the element number of the bias tensor match the input tensor, a simply element wise + will be done. Or the bias will be expanded to the same size of the input. The expand means + repeat on unmatched singleton dimension(if some unmatched dimension isn't singleton dimension, + it will report an error). If the input is a batch, a singleton dimension will be add to the + first dimension before the expand. + + :param size: the size of the bias + + >>> cAdd = CAdd([1,2]) + creating: createCAdd + ''' + + def __init__(self, + size, + bigdl_type="float"): + super(CAdd, self).__init__(None, bigdl_type, + size) + + +class CAddTable(Model): + + ''' + Merge the input tensors in the input table by element wise adding them together. The input + table is actually an array of tensor with same size. + + :param inplace: reuse the input memory + + >>> cAddTable = CAddTable(True) + creating: createCAddTable + ''' + + def __init__(self, + inplace=False, + bigdl_type="float"): + super(CAddTable, self).__init__(None, bigdl_type, + inplace) + + +class CDivTable(Model): + + ''' + Takes a table with two Tensor and returns the component-wise division between them. + + >>> cDivTable = CDivTable() + creating: createCDivTable + ''' + + def __init__(self, + bigdl_type="float"): + super(CDivTable, self).__init__(None, bigdl_type) + + +class CMaxTable(Model): + + ''' + Takes a table of Tensors and outputs the max of all of them. + + >>> cMaxTable = CMaxTable() + creating: createCMaxTable + ''' + + def __init__(self, + bigdl_type="float"): + super(CMaxTable, self).__init__(None, bigdl_type) + + +class CMinTable(Model): + + ''' + Takes a table of Tensors and outputs the min of all of them. + >>> cMinTable = CMinTable() + creating: createCMinTable + ''' + + def __init__(self, + bigdl_type="float"): + super(CMinTable, self).__init__(None, bigdl_type) + + +class CMul(Model): + + ''' + Applies a component-wise multiplication to the incoming data + + :param size size of the data + + >>> cMul = CMul([1,2]) + creating: createCMul + ''' + + def __init__(self, + size, + bigdl_type="float"): + super(CMul, self).__init__(None, bigdl_type, + size) + + +class CMulTable(Model): + + ''' + Takes a table of Tensors and outputs the multiplication of all of them. + + >>> cMulTable = CMulTable() + creating: createCMulTable + ''' + + def __init__(self, + bigdl_type="float"): + super(CMulTable, self).__init__(None, bigdl_type) + + +class CSubTable(Model): + + ''' + Takes a table with two Tensor and returns the component-wise subtraction between them. + + >>> cSubTable = CSubTable() + creating: createCSubTable + ''' + + def __init__(self, + bigdl_type="float"): + super(CSubTable, self).__init__(None, bigdl_type) + + +class Clamp(Model): + + ''' + Clamps all elements into the range [min_value, max_value]. + Output is identical to input in the range, + otherwise elements less than min_value (or greater than max_value) + are saturated to min_value (or max_value). + + :param min + :param max + + >>> clamp = Clamp(1, 3) + creating: createClamp + ''' + + def __init__(self, + min, + max, + bigdl_type="float"): + super(Clamp, self).__init__(None, bigdl_type, + min, + max) + + +class Contiguous(Model): + + ''' + used to make input, grad_output both contiguous + + >>> contiguous = Contiguous() + creating: createContiguous + ''' + + def __init__(self, + bigdl_type="float"): + super(Contiguous, self).__init__(None, bigdl_type) + + +class Cosine(Model): + + ''' + Cosine calculates the cosine similarity of the input to k mean centers. The input given in + forward(input) must be either a vector (1D tensor) or matrix (2D tensor). If the input is a + vector, it must have the size of inputSize. If it is a matrix, then each row is assumed to be + an input sample of given batch (the number of rows means the batch size and the number of + columns should be equal to the inputSize). + + :param input_size: the size of each input sample + :param output_size: the size of the module output of each sample + + >>> cosine = Cosine(2,3) + creating: createCosine + ''' + + def __init__(self, + input_size, + output_size, + bigdl_type="float"): + super(Cosine, self).__init__(None, bigdl_type, + input_size, + output_size) + + +class CosineDistance(Model): + + ''' + Outputs the cosine distance between inputs + + >>> cosineDistance = CosineDistance() + creating: createCosineDistance + ''' + + def __init__(self, + bigdl_type="float"): + super(CosineDistance, self).__init__(None, bigdl_type) + +class DiceCoefficientCriterion(Model): + + ''' + The Dice-Coefficient criterion + input: Tensor, target: Tensor + + return: 2 * (input intersection target) + 1 - ---------------------------------- + input union target + + >>> diceCoefficientCriterion = DiceCoefficientCriterion(size_average = True, epsilon = 1.0) + creating: createDiceCoefficientCriterion + ''' + + def __init__(self, + size_average, + epsilon, + bigdl_type="float"): + super(DiceCoefficientCriterion, self).__init__(None, bigdl_type, + size_average, + epsilon) + +class DotProduct(Model): + + ''' + This is a simple table layer which takes a table of two tensors as input + and calculate the dot product between them as outputs + + >>> dotProduct = DotProduct() + creating: createDotProduct + ''' + + def __init__(self, + bigdl_type="float"): + super(DotProduct, self).__init__(None, bigdl_type) + + +class ELU(Model): + + ''' + D-A Clevert, Thomas Unterthiner, Sepp Hochreiter + Fast and Accurate Deep Network Learning by Exponential Linear Units (ELUs) + [http://arxiv.org/pdf/1511.07289.pdf] + + >>> eLU = ELU(1e-5, True) + creating: createELU + ''' + + def __init__(self, + alpha=1.0, + inplace=False, + bigdl_type="float"): + super(ELU, self).__init__(None, bigdl_type, + alpha, + inplace) + + +class Euclidean(Model): + + ''' + Outputs the Euclidean distance of the input to outputSize centers + :param inputSize inputSize + :param outputSize outputSize + :param T Numeric type. Only support float/double now + + >>> euclidean = Euclidean(1, 1, True) + creating: createEuclidean + ''' + + + + def __init__(self, + input_size, + output_size, + fast_backward=True, + bigdl_type="float"): + super(Euclidean, self).__init__(None, bigdl_type, + input_size, + output_size, + fast_backward) + + +class Exp(Model): + + ''' + Applies element-wise exp to input tensor. + >>> exp = Exp() + creating: createExp + ''' + + def __init__(self, + bigdl_type="float"): + super(Exp, self).__init__(None, bigdl_type) + + +class FlattenTable(Model): + + ''' + This is a table layer which takes an arbitrarily deep table of Tensors + (potentially nested) as input and a table of Tensors without any nested + table will be produced + + >>> flattenTable = FlattenTable() + creating: createFlattenTable + ''' + + def __init__(self, + bigdl_type="float"): + super(FlattenTable, self).__init__(None, bigdl_type) + + +class GradientReversal(Model): + + ''' + It is a simple module preserves the input, but takes the + gradient from the subsequent layer, multiplies it by -lambda + and passes it to the preceding layer. This can be used to maximise + an objective function whilst using gradient descent, as described in + ["Domain-Adversarial Training of Neural Networks" + (http://arxiv.org/abs/1505.07818)] + + :param lambda hyper-parameter lambda can be set dynamically during training + + >>> gradientReversal = GradientReversal(1e-5) + creating: createGradientReversal + ''' + + def __init__(self, + the_lambda=1, + bigdl_type="float"): + super(GradientReversal, self).__init__(None, bigdl_type, + the_lambda) + + +class HardShrink(Model): + + ''' + This is a transfer layer which applies the hard shrinkage function + element-wise to the input Tensor. The parameter lambda is set to 0.5 + by default + x, if x > lambda + f(x) = x, if x < -lambda + 0, otherwise + + :param the_lambda: a threshold value whose default value is 0.5 + + >>> hardShrink = HardShrink(1e-5) + creating: createHardShrink + ''' + + def __init__(self, + the_lambda=0.5, + bigdl_type="float"): + super(HardShrink, self).__init__(None, bigdl_type, + the_lambda) + + +class HardTanh(Model): + + ''' + Applies HardTanh to each element of input, HardTanh is defined: + | maxValue, if x > maxValue + f(x) = | minValue, if x < minValue + | x, otherwise + :param min_value minValue in f(x), default is -1. + :param max_value maxValue in f(x), default is 1. + :param inplace whether enable inplace model. + + >>> hardTanh = HardTanh(1e-5, 1e5, True) + creating: createHardTanh + ''' + + def __init__(self, + min_value=-1, + max_value=1, + inplace=False, + bigdl_type="float"): + super(HardTanh, self).__init__(None, bigdl_type, + min_value, + max_value, + inplace) + + +class Index(Model): + + ''' + Applies the Tensor index operation along the given dimension. + + :param dimension the dimension to be indexed + + >>> index = Index(1) + creating: createIndex + ''' + + def __init__(self, + dimension, + bigdl_type="float"): + super(Index, self).__init__(None, bigdl_type, + dimension) + + +class InferReshape(Model): + + ''' + Reshape with the support of infered size, + Positive numbers are used directly, setting the corresponding dimension of the output tensor. + In addition, two special values are accepted: + 0 means "copy the respective dimension of the input". + i.e., if the input has 2 as its 1st dimension, + the output will have 2 as its 1st dimension as well + -1 stands for "infer this from the other dimensions" + this dimension is calculated to keep the overall element count the same as in the input. + At most one -1 can be used in a reshape operation. + For example, (4, 5, 6, 7) -> InferReshape (4, 0, 3, -1) -> (4, 5, 3, 14) + with 1st and 3rd dim same as given size, with 2nd dim same as input, and the infered dim is 14 + + :param size the target tensor size + :param batch_mode whether in batch mode + + >>> inferReshape = InferReshape([4, 0, 3, -1], False) + creating: createInferReshape + ''' + + def __init__(self, + size, + batch_mode=False, + bigdl_type="float"): + super(InferReshape, self).__init__(None, bigdl_type, + size, + batch_mode) + + +class JoinTable(Model): + + ''' + It is a table module which takes a table of Tensors as input and + outputs a Tensor by joining them together along the dimension `dimension`. + + The input to this layer is expected to be a tensor, or a batch of tensors; + when using mini-batch, a batch of sample tensors will be passed to the layer and + the user need to specify the number of dimensions of each sample tensor in the + batch using `nInputDims`. + + :param dimension to be join in this dimension + :param nInputDims specify the number of dimensions that this module will receive + If it is more than the dimension of input tensors, the first dimension + would be considered as batch size + + >>> joinTable = JoinTable(1, 1) + creating: createJoinTable + ''' + + def __init__(self, + dimension, + n_input_dims, + bigdl_type="float"): + super(JoinTable, self).__init__(None, bigdl_type, + dimension, + n_input_dims) + + +class L1Cost(Model): + + ''' + compute L1 norm for input, and sign of input + >>> l1Cost = L1Cost() + creating: createL1Cost + ''' + + def __init__(self, + bigdl_type="float"): + super(L1Cost, self).__init__(None, bigdl_type) + + +class L1Penalty(Model): + + ''' + adds an L1 penalty to an input (for sparsity). + L1Penalty is an inline module that in its forward propagation copies the input Tensor + directly to the output, and computes an L1 loss of the latent state (input) and stores + it in the module's loss field. During backward propagation: gradInput = gradOutput + gradLoss. + + :param l1weight + :param sizeAverage + :param provideOutput + + >>> l1Penalty = L1Penalty(1, True, True) + creating: createL1Penalty + ''' + + def __init__(self, + l1weight, + size_average=False, + provide_output=True, + bigdl_type="float"): + super(L1Penalty, self).__init__(None, bigdl_type, + l1weight, + size_average, + provide_output) + + +class LeakyReLU(Model): + + ''' + It is a transfer module that applies LeakyReLU, which parameter negval sets the slope of the + negative part: LeakyReLU is defined as: f(x) = max(0, x) + negval * min(0, x) + + :param negval: sets the slope of the negative partl + :param inplace: if it is true, doing the operation in-place without using extra state memory + + >>> leakyReLU = LeakyReLU(1e-5, True) + creating: createLeakyReLU + ''' + + def __init__(self, + negval=0.01, + inplace=False, + bigdl_type="float"): + super(LeakyReLU, self).__init__(None, bigdl_type, + negval, + inplace) + + +class Log(Model): + + ''' + Applies the log function element-wise to the input Tensor, + thus outputting a Tensor of the same dimension. + + >>> log = Log() + creating: createLog + ''' + + def __init__(self, + bigdl_type="float"): + super(Log, self).__init__(None, bigdl_type) + + +class LogSigmoid(Model): + + ''' + This class is a transform layer corresponding to the sigmoid function: + f(x) = Log(1 / (1 + e ^^ (-x))) + + >>> logSigmoid = LogSigmoid() + creating: createLogSigmoid + ''' + + def __init__(self, + bigdl_type="float"): + super(LogSigmoid, self).__init__(None, bigdl_type) + + +class LookupTable(Model): + + ''' + a convolution of width 1, commonly used for word embeddings + + >>> lookupTable = LookupTable(1, 1, 1e-5, 1e-5, 1e-5, True) + creating: createLookupTable + ''' + + def __init__(self, + n_index, + n_output, + padding_value=0.0, + max_norm=DOUBLEMAX, + norm_type=2.0, + should_scale_grad_by_freq=False, + bigdl_type="float"): + super(LookupTable, self).__init__(None, bigdl_type, + n_index, + n_output, + padding_value, + max_norm, + norm_type, + should_scale_grad_by_freq) + + +class MM(Model): + + ''' + Module to perform matrix multiplication on two mini-batch inputs, producing a mini-batch. + + :param trans_a: specifying whether or not transpose the first input matrix + :param trans_b: specifying whether or not transpose the second input matrix + + >>> mM = MM(True, True) + creating: createMM + ''' + def __init__(self, + trans_a=False, + trans_b=False, + bigdl_type="float"): + super(MM, self).__init__(None, bigdl_type, + trans_a, + trans_b) + + +class MV(Model): + + ''' + It is a module to perform matrix vector multiplication on two mini-batch inputs, + producing a mini-batch. + + :param trans whether make matrix transpose before multiplication + + >>> mV = MV(True) + creating: createMV + ''' + + def __init__(self, + trans=False, + bigdl_type="float"): + super(MV, self).__init__(None, bigdl_type, + trans) + + +class MapTable(Container): + + ''' + This class is a container for a single module which will be applied + to all input elements. The member module is cloned as necessary to + process all input elements. + + >>> mapTable = MapTable(Linear(100,10)) + creating: createLinear + creating: createMapTable + ''' + + def __init__(self, + module, + bigdl_type="float"): + super(MapTable, self).__init__(None, bigdl_type, + module) + +class MaskedSelect(Model): + + ''' + Performs a torch.MaskedSelect on a Tensor. The mask is supplied as a tabular argument with + the input on the forward and backward passes. + >>> maskedSelect = MaskedSelect() + creating: createMaskedSelect + ''' + + def __init__(self, + bigdl_type="float"): + super(MaskedSelect, self).__init__(None, bigdl_type) + + +class Max(Model): + + ''' + Applies a max operation over dimension `dim` + + :param dim max along this dimension + :param num_input_dims Optional. If in a batch model, set to the inputDims. + + >>> max = Max(1) + creating: createMax + ''' + + def __init__(self, + dim, + num_input_dims=INTMIN, + bigdl_type="float"): + super(Max, self).__init__(None, bigdl_type, + dim, + num_input_dims) + + +class Mean(Model): + + ''' + It is a simple layer which applies a mean operation over the given dimension. When nInputDims + is provided, the input will be considered as batches. Then the mean operation will be applied + in (dimension + 1). The input to this layer is expected to be a tensor, or a batch of + tensors; when using mini-batch, a batch of sample tensors will be passed to the layer and the + user need to specify the number of dimensions of each sample tensor in the batch using + nInputDims. + + :param dimension: the dimension to be applied mean operation + :param n_input_dims: specify the number of dimensions that this module will receive + If it is more than the dimension of input tensors, the first dimension would be considered + as batch size + + >>> mean = Mean(1, 1) + creating: createMean + ''' + + def __init__(self, + dimension=1, + n_input_dims=-1, + bigdl_type="float"): + super(Mean, self).__init__(None, bigdl_type, + dimension, + n_input_dims) + + +class Min(Model): + + ''' + Applies a min operation over dimension `dim`. + + :param dim min along this dimension + :param num_input_dims Optional. If in a batch model, set to the input_dim. + + >>> min = Min(1) + creating: createMin + ''' + + def __init__(self, + dim, + num_input_dims=INTMIN, + bigdl_type="float"): + super(Min, self).__init__(None, bigdl_type, + dim, + num_input_dims) + + +class MixtureTable(Model): + + ''' + Creates a module that takes a table {gater, experts} as input and outputs the mixture of experts + (a Tensor or table of Tensors) using a gater Tensor. When dim is provided, it specifies the + dimension of the experts Tensor that will be interpolated (or mixed). Otherwise, the experts + should take the form of a table of Tensors. This Module works for experts of dimension 1D or + more, and for a 1D or 2D gater, i.e. for single examples or mini-batches. + + >>> mixtureTable = MixtureTable() + creating: createMixtureTable + >>> mixtureTable = MixtureTable(10) + creating: createMixtureTable + ''' + + def __init__(self, + dim=INTMAX, + bigdl_type="float"): + super(MixtureTable, self).__init__(None, bigdl_type, dim) + + +class Mul(Model): + + ''' + Multiply a single scalar factor to the incoming data + + >>> mul = Mul() + creating: createMul + ''' + + def __init__(self, + bigdl_type="float"): + super(Mul, self).__init__(None, bigdl_type) + + +class MulConstant(Model): + + ''' + Multiplies input Tensor by a (non-learnable) scalar constant. + This module is sometimes useful for debugging purposes. + + :param scalar scalar constant + :param inplace Can optionally do its operation in-place without using extra state memory + + >>> mulConstant = MulConstant(2.5) + creating: createMulConstant + ''' + + def __init__(self, + scalar, + inplace=False, + bigdl_type="float"): + super(MulConstant, self).__init__(None, bigdl_type, + scalar, + inplace) + + +class Narrow(Model): + + ''' + Narrow is application of narrow operation in a module. + The module further supports a negative length in order to handle inputs with an unknown size. + >>> narrow = Narrow(1, 1, 1) + creating: createNarrow + ''' + + def __init__(self, + dimension, + offset, + length=1, + bigdl_type="float"): + super(Narrow, self).__init__(None, bigdl_type, + dimension, + offset, + length) + + +class NarrowTable(Model): + + ''' + Creates a module that takes a table as input and outputs the subtable starting at index + offset having length elements (defaults to 1 element). The elements can be either + a table or a Tensor. If `length` is negative, it means selecting the elements from the + offset to element which located at the abs(`length`) to the last element of the input. + + :param offset the start index of table + :param length the length want to select + + >>> narrowTable = NarrowTable(1, 1) + creating: createNarrowTable + ''' + + def __init__(self, + offset, + length=1, + bigdl_type="float"): + super(NarrowTable, self).__init__(None, bigdl_type, + offset, + length) + + +class Normalize(Model): + + ''' + Normalizes the input Tensor to have unit L_p norm. The smoothing parameter eps prevents + division by zero when the input contains all zero elements (default = 1e-10). + p can be the max value of double + + >>> normalize = Normalize(1e-5, 1e-5) + creating: createNormalize + ''' + + def __init__(self, + p, + eps=1e-10, + bigdl_type="float"): + super(Normalize, self).__init__(None, bigdl_type, + p, + eps) + + +class PReLU(Model): + + ''' + Applies parametric ReLU, which parameter varies the slope of the negative part. + + PReLU: f(x) = max(0, x) + a * min(0, x) + + nOutputPlane's default value is 0, that means using PReLU in shared version and has + only one parameters. + + Notice: Please don't use weight decay on this. + + :param n_output_plane input map number. Default is 0. + + >>> pReLU = PReLU(1) + creating: createPReLU + ''' + + def __init__(self, + n_output_plane=0, + bigdl_type="float"): + super(PReLU, self).__init__(None, bigdl_type, + n_output_plane) + + +class Padding(Model): + + ''' + This module adds pad units of padding to dimension dim of the input. If pad is negative, + padding is added to the left, otherwise, it is added to the right of the dimension. + + The input to this layer is expected to be a tensor, or a batch of tensors; + when using mini-batch, a batch of sample tensors will be passed to the layer and + the user need to specify the number of dimensions of each sample tensor in the + batch using n_input_dim. + + :param dim the dimension to be applied padding operation + :param pad num of the pad units + :param n_input_dim specify the number of dimensions that this module will receive + If it is more than the dimension of input tensors, the first dimension + would be considered as batch size + :param value padding value + + >>> padding = Padding(1, 1, 1, 1e-5, 1) + creating: createPadding + ''' + + def __init__(self, + dim, + pad, + n_input_dim, + value=0.0, + n_index=1, + bigdl_type="float"): + super(Padding, self).__init__(None, bigdl_type, + dim, + pad, + n_input_dim, + value, + n_index) + + +class PairwiseDistance(Model): + + ''' + It is a module that takes a table of two vectors as input and outputs + the distance between them using the p-norm. + The input given in `forward(input)` is a [[Table]] that contains two tensors which + must be either a vector (1D tensor) or matrix (2D tensor). If the input is a vector, + it must have the size of `inputSize`. If it is a matrix, then each row is assumed to be + an input sample of the given batch (the number of rows means the batch size and + the number of columns should be equal to the `inputSize`). + :param norm the norm of distance + + >>> pairwiseDistance = PairwiseDistance(2) + creating: createPairwiseDistance + ''' + + def __init__(self, + norm=2, + bigdl_type="float"): + super(PairwiseDistance, self).__init__(None, bigdl_type, + norm) + + +class ParallelTable(Container): + + ''' + It is a container module that applies the i-th member module to the i-th + input, and outputs an output in the form of Table + + >>> parallelTable = ParallelTable() + creating: createParallelTable + ''' + + def __init__(self, + bigdl_type="float"): + super(ParallelTable, self).__init__(None, bigdl_type) + + +class Power(Model): + + ''' + Apply an element-wise power operation with scale and shift. + f(x) = (shift + scale * x)^power^ + :param power: the exponent. + :param scale: Default is 1. + :param shift: Default is 0. + + >>> power = Power(1e-5) + creating: createPower + ''' + + def __init__(self, + power, + scale=1.0, + shift=0.0, + bigdl_type="float"): + super(Power, self).__init__(None, bigdl_type, + power, + scale, + shift) + + +class RReLU(Model): + + ''' + Applies the randomized leaky rectified linear unit (RReLU) element-wise to the input Tensor, + thus outputting a Tensor of the same dimension. Informally the RReLU is also known as + 'insanity' layer. RReLU is defined as: + f(x) = max(0,x) + a * min(0, x) where a ~ U(l, u). + + In training mode negative inputs are multiplied by a factor a drawn from a uniform random + distribution U(l, u). + + In evaluation mode a RReLU behaves like a LeakyReLU with a constant mean factor + a = (l + u) / 2. + + By default, l = 1/8 and u = 1/3. If l == u a RReLU effectively becomes a LeakyReLU. + + Regardless of operating in in-place mode a RReLU will internally allocate an input-sized + noise tensor to store random factors for negative inputs. + + The backward() operation assumes that forward() has been called before. + + For reference see [Empirical Evaluation of Rectified Activations in Convolutional Network]( + http://arxiv.org/abs/1505.00853). + + :param lower: lower boundary of uniform random distribution + :param upper: upper boundary of uniform random distribution + :param inplace: optionally do its operation in-place without using extra state memory + + >>> rReLU = RReLU(1e-5, 1e5, True) + creating: createRReLU + ''' + + def __init__(self, + lower=1.0/8, + upper=1.0/3, + inplace=False, + bigdl_type="float"): + super(RReLU, self).__init__(None, bigdl_type, + lower, + upper, + inplace) + + +class ReLU6(Model): + + ''' + Same as ReLU except that the rectifying function f(x) saturates at x = 6 + + :param inplace either True = in-place or False = keeping separate state + + >>> reLU6 = ReLU6(True) + creating: createReLU6 + ''' + + def __init__(self, + inplace=False, + bigdl_type="float"): + super(ReLU6, self).__init__(None, bigdl_type, + inplace) + + +class Replicate(Model): + + ''' + Replicate repeats input `nFeatures` times along its `dim` dimension. + Notice: No memory copy, it set the stride along the `dim`-th dimension to zero. + + :param n_features: replicate times. + :param dim: dimension to be replicated. + :param n_dim: specify the number of non-batch dimensions. + + >>> replicate = Replicate(2) + creating: createReplicate + ''' + def __init__(self, + n_features, + dim=1, + n_dim=INTMAX, + bigdl_type="float"): + super(Replicate, self).__init__(None, bigdl_type, + n_features, + dim, + n_dim) + + +class RoiPooling(Model): + + ''' + Region of interest pooling + The RoIPooling uses max pooling to convert the features inside any valid region of interest + into a small feature map with a fixed spatial extent of pooledH * pooledW (e.g., 7 * 7) + an RoI is a rectangular window into a conv feature map. + Each RoI is defined by a four-tuple (x1, y1, x2, y2) that specifies its + top-left corner (x1, y1) and its bottom-right corner (x2, y2). + RoI max pooling works by dividing the h * w RoI window into an pooledH * pooledW grid of + sub-windows of approximate size h/H * w/W and then max-pooling the values in each sub-window + into the corresponding output grid cell. + Pooling is applied independently to each feature map channel + + :param pooled_w: spatial extent in width + :param pooled_h: spatial extent in height + :param spatial_scale spatial scale + + >>> roiPooling = RoiPooling(1, 1, 1e-5) + creating: createRoiPooling + ''' + + def __init__(self, + pooled_w, + pooled_h, + spatial_scale, + bigdl_type="float"): + super(RoiPooling, self).__init__(None, bigdl_type, + pooled_w, + pooled_h, + spatial_scale) + + +class Scale(Model): + + ''' + Scale is the combination of CMul and CAdd + Computes the elementwise product of input and weight, with the shape of the weight "expand" to + match the shape of the input. + Similarly, perform a expand cdd bias and perform an elementwise add + + :param size size of weight and bias + + >>> scale = Scale([1,2]) + creating: createScale + ''' + + def __init__(self, + size, + bigdl_type="float"): + super(Scale, self).__init__(None, bigdl_type, + size) + + +class SelectTable(Container): + + ''' + Creates a module that takes a table as input and outputs the element at index `index` + (positive or negative). This can be either a table or a Tensor. + The gradients of the non-index elements are zeroed Tensors of the same size. + This is true regardless of the depth of the encapsulated Tensor as the function used + internally to do so is recursive. + + :param dimension the dimension to be selected + + >>> selectTable = SelectTable(1) + creating: createSelectTable + ''' + + def __init__(self, + dimension, + bigdl_type="float"): + super(SelectTable, self).__init__(None, bigdl_type, + dimension) + + +class Sigmoid(Model): + + ''' + Applies the Sigmoid function element-wise to the input Tensor, + thus outputting a Tensor of the same dimension. + >>> sigmoid = Sigmoid() + creating: createSigmoid + ''' + + def __init__(self, + bigdl_type="float"): + super(Sigmoid, self).__init__(None, bigdl_type) + + +class SoftMax(Model): + + ''' + Applies the SoftMax function to an n-dimensional input Tensor, rescaling them so that the + elements of the n-dimensional output Tensor lie in the range (0, 1) and sum to 1. + Softmax is defined as: f_i(x) = exp(x_i - shift) / sum_j exp(x_j - shift) + where shift = max_i(x_i). + + >>> softMax = SoftMax() + creating: createSoftMax + ''' + + def __init__(self, + bigdl_type="float"): + super(SoftMax, self).__init__(None, bigdl_type) + + +class SoftMin(Model): + + ''' + Applies the SoftMin function to an n-dimensional input Tensor, rescaling them so that the + elements of the n-dimensional output Tensor lie in the range (0,1) and sum to 1. + Softmin is defined as: f_i(x) = exp(-x_i - shift) / sum_j exp(-x_j - shift) + where shift = max_i(-x_i). + + >>> softMin = SoftMin() + creating: createSoftMin + ''' + + def __init__(self, + bigdl_type="float"): + super(SoftMin, self).__init__(None, bigdl_type) + + +class SoftPlus(Model): + + ''' + Apply the SoftPlus function to an n-dimensional input tensor. + SoftPlus function: f_i(x) = 1/beta * log(1 + exp(beta * x_i)) + + :param beta Controls sharpness of transfer function + + >>> softPlus = SoftPlus(1e-5) + creating: createSoftPlus + ''' + + def __init__(self, + beta=1.0, + bigdl_type="float"): + super(SoftPlus, self).__init__(None, bigdl_type, + beta) + + +class SoftShrink(Model): + + ''' + Apply the soft shrinkage function element-wise to the input Tensor + + SoftShrinkage operator: + | x - lambda, if x > lambda + f(x) = | x + lambda, if x < -lambda + | 0, otherwise + + :param the_lambda lambda, default is 0.5 + + >>> softShrink = SoftShrink(1e-5) + creating: createSoftShrink + ''' + + def __init__(self, + the_lambda=0.5, + bigdl_type="float"): + super(SoftShrink, self).__init__(None, bigdl_type, + the_lambda) + + +class SoftSign(Model): + + ''' + Apply SoftSign function to an n-dimensional input Tensor. + + SoftSign function: f_i(x) = x_i / (1+|x_i|) + + >>> softSign = SoftSign() + creating: createSoftSign + ''' + + def __init__(self, + bigdl_type="float"): + super(SoftSign, self).__init__(None, bigdl_type) + + +class SpatialDilatedConvolution(Model): + + ''' + Apply a 2D dilated convolution over an input image. + + The input tensor is expected to be a 3D or 4D(with batch) tensor. + + If input is a 3D tensor nInputPlane x height x width, + owidth = floor(width + 2 * padW - dilationW * (kW-1) - 1) / dW + 1 + oheight = floor(height + 2 * padH - dilationH * (kH-1) - 1) / dH + 1 + + Reference Paper: Yu F, Koltun V. Multi-scale context aggregation by dilated convolutions[J]. + arXiv preprint arXiv:1511.07122, 2015. + + :param n_input_plane: The number of expected input planes in the image given into forward(). + :param n_output_plane: The number of output planes the convolution layer will produce. + :param kw: The kernel width of the convolution. + :param kh: The kernel height of the convolution. + :param dw: The step of the convolution in the width dimension. Default is 1. + :param dh: The step of the convolution in the height dimension. Default is 1. + :param pad_w: The additional zeros added per width to the input planes. Default is 0. + :param pad_h: The additional zeros added per height to the input planes. Default is 0. + :param dilation_w: The number of pixels to skip. Default is 1. + :param dilation_h: The number of pixels to skip. Default is 1. + :param init_method: Init method, Default, Xavier. + + >>> spatialDilatedConvolution = SpatialDilatedConvolution(1, 1, 1, 1) + creating: createSpatialDilatedConvolution + ''' + + def __init__(self, + n_input_plane, + n_output_plane, + kw, + kh, + dw=1, + dh=1, + pad_w=0, + pad_h=0, + dilation_w=1, + dilation_h=1, + init_method='default', + bigdl_type="float"): + super(SpatialDilatedConvolution, self).__init__(None, bigdl_type, + n_input_plane, + n_output_plane, + kw, + kh, + dw, + dh, + pad_w, + pad_h, + dilation_w, + dilation_h, + init_method) + + +class SpatialFullConvolution(Model): + ''' + Apply a 2D full convolution over an input image. + The input tensor is expected to be a 3D or 4D(with batch) tensor. Note that instead + of setting adjW and adjH, SpatialFullConvolution[Table, T] also accepts a table input + with two tensors: T(convInput, sizeTensor) where convInput is the standard input tensor, + and the size of sizeTensor is used to set the size of the output (will ignore the adjW and + adjH values used to construct the module). This module can be used without a bias by setting + parameter noBias = true while constructing the module. + + If input is a 3D tensor nInputPlane x height x width, + owidth = (width - 1) * dW - 2*padW + kW + adjW + oheight = (height - 1) * dH - 2*padH + kH + adjH + + Other frameworks call this operation "In-network Upsampling", "Fractionally-strided convolution", + "Backwards Convolution," "Deconvolution", or "Upconvolution." + + Reference Paper: Long J, Shelhamer E, Darrell T. Fully convolutional networks for semantic + segmentation[C]//Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition. + 2015: 3431-3440. + + :param nInputPlane The number of expected input planes in the image given into forward() + :param nOutputPlane The number of output planes the convolution layer will produce. + :param kW The kernel width of the convolution. + :param kH The kernel height of the convolution. + :param dW The step of the convolution in the width dimension. Default is 1. + :param dH The step of the convolution in the height dimension. Default is 1. + :param padW The additional zeros added per width to the input planes. Default is 0. + :param padH The additional zeros added per height to the input planes. Default is 0. + :param adjW Extra width to add to the output image. Default is 0. + :param adjH Extra height to add to the output image. Default is 0. + :param nGroup Kernel group number. + :param noBias If bias is needed. + :param initMethod Init method, Default, Xavier, Bilinear. + + >>> spatialFullConvolution = SpatialFullConvolution(1, 1, 1, 1) + creating: createSpatialFullConvolution + ''' + + def __init__(self, + n_input_plane, + n_output_plane, + kw, + kh, + dw=1, + dh=1, + pad_w=0, + pad_h=0, + adj_w=0, + adj_h=0, + n_group=1, + no_bias=False, + init_method='default', + bigdl_type="float"): + super(SpatialFullConvolution, self).__init__(None, bigdl_type, + n_input_plane, + n_output_plane, + kw, + kh, + dw, + dh, + pad_w, + pad_h, + adj_w, + adj_h, + n_group, + no_bias, + init_method) + + +class SpatialShareConvolution(Model): + + ''' + >>> spatialShareConvolution = SpatialShareConvolution(1, 1, 1, 1) + creating: createSpatialShareConvolution + ''' + + def __init__(self, + n_input_plane, + n_output_plane, + kernel_w, + kernel_h, + stride_w=1, + stride_h=1, + pad_w=0, + pad_h=0, + n_group=1, + propagate_back=True, + init_method='default', + bigdl_type="float"): + super(SpatialShareConvolution, self).__init__(None, bigdl_type, + n_input_plane, + n_output_plane, + kernel_w, + kernel_h, + stride_w, + stride_h, + pad_w, + pad_h, + n_group, + propagate_back, + init_method) + + +class VolumetricConvolution(Model): + + ''' + Applies a 3D convolution over an input image composed of several input planes. The input tensor + in forward(input) is expected to be a 4D tensor (nInputPlane x time x height x width). + :param n_input_plane The number of expected input planes in the image given into forward() + :param n_output_plane The number of output planes the convolution layer will produce. + :param k_t The kernel size of the convolution in time + :param k_w The kernel width of the convolution + :param k_h The kernel height of the convolution + :param d_t The step of the convolution in the time dimension. Default is 1 + :param d_w The step of the convolution in the width dimension. Default is 1 + :param d_h The step of the convolution in the height dimension. Default is 1 + :param pad_t Additional zeros added to the input plane data on both sides of time axis. + Default is 0. (kT-1)/2 is often used here. + :param pad_w The additional zeros added per width to the input planes. + :param pad_h The additional zeros added per height to the input planes. + :param with_bias whether with bias + :param init_method Init method, Default, Xavier, Bilinear. + + >>> volumetricConvolution = VolumetricConvolution(6, 12, 5, 5, 5, 1, 1, 1) + creating: createVolumetricConvolution + ''' + + def __init__(self, + n_input_plane, + n_output_plane, + k_t, + k_w, + k_h, + d_t=1, + d_w=1, + d_h=1, + pad_t=0, + pad_w=0, + pad_h=0, + with_bias=True, + init_method="default", + bigdl_type="float"): + super(VolumetricConvolution, self).__init__(None, bigdl_type, + n_input_plane, + n_output_plane, + k_t, + k_w, + k_h, + d_t, + d_w, + d_h, + pad_t, + pad_w, + pad_h, + with_bias, + init_method) + + +class VolumetricMaxPooling(Model): + + ''' + Applies 3D max-pooling operation in kTxkWxkH regions by step size dTxdWxdH steps. + The number of output features is equal to the number of input planes / dT. + The input can optionally be padded with zeros. Padding should be smaller than + half of kernel size. That is, padT < kT/2, padW < kW/2 and padH < kH/2 + :param k_t The kernel size + :param k_w The kernel width + :param k_h The kernel height + :param d_t The step in the time dimension + :param d_w The step in the width dimension + :param d_h The step in the height dimension + :param pad_t The padding in the time dimension + :param pad_w The padding in the width dimension + :param pad_h The padding in the height dimension + + >>> volumetricMaxPooling = VolumetricMaxPooling(5, 5, 5, 1, 1, 1) + creating: createVolumetricMaxPooling + ''' + + def __init__(self, + k_t, + k_w, + k_h, + d_t, + d_w, + d_h, + pad_t=0, + pad_w=0, + pad_h=0, + bigdl_type="float"): + super(VolumetricMaxPooling, self).__init__(None, bigdl_type, + k_t, + k_w, + k_h, + d_t, + d_w, + d_h, + pad_t, + pad_w, + pad_h) + + +class SpatialZeroPadding(Model): + + ''' + Each feature map of a given input is padded with specified number of zeros. + If padding values are negative, then input is cropped. + :param padLeft: pad left position + :param padRight: pad right position + :param padTop: pad top position + :param padBottom: pad bottom position + + >>> spatialZeroPadding = SpatialZeroPadding(1, 1, 1, 1) + creating: createSpatialZeroPadding + ''' + + def __init__(self, + pad_left, + pad_right, + pad_top, + pad_bottom, + bigdl_type="float"): + super(SpatialZeroPadding, self).__init__(None, bigdl_type, + pad_left, + pad_right, + pad_top, + pad_bottom) + + +class SplitTable(Model): + + ''' + Creates a module that takes a Tensor as input and + outputs several tables, splitting the Tensor along + the specified dimension `dimension`. + + The input to this layer is expected to be a tensor, or a batch of tensors; + when using mini-batch, a batch of sample tensors will be passed to the layer and + the user need to specify the number of dimensions of each sample tensor in a + batch using `nInputDims`. + + :param dimension: to be split along this dimension + :param n_input_dims: specify the number of dimensions that this module will receive + If it is more than the dimension of input tensors, the first dimension + would be considered as batch size + + >>> splitTable = SplitTable(1, 1) + creating: createSplitTable + ''' + + def __init__(self, + dimension, + n_input_dims=-1, + bigdl_type="float"): + super(SplitTable, self).__init__(None, bigdl_type, + dimension, + n_input_dims) + + +class Sqrt(Model): + + ''' + Apply an element-wise sqrt operation. + + >>> sqrt = Sqrt() + creating: createSqrt + ''' + + def __init__(self, + bigdl_type="float"): + super(Sqrt, self).__init__(None, bigdl_type) + + +class Square(Model): + + ''' + Apply an element-wise square operation. + >>> square = Square() + creating: createSquare + ''' + + def __init__(self, + bigdl_type="float"): + super(Square, self).__init__(None, bigdl_type) + + +class Squeeze(Model): + + ''' + Delete singleton all dimensions or a specific dim. + + :param dim Optional. The dimension to be delete. Default: delete all dimensions. + :param num_input_dims Optional. If in a batch model, set to the inputDims. + + + >>> squeeze = Squeeze(1) + creating: createSqueeze + ''' + + def __init__(self, + dim, + num_input_dims=INTMIN, + bigdl_type="float"): + super(Squeeze, self).__init__(None, bigdl_type, + dim, + num_input_dims) + + +class Sum(Model): + + ''' + It is a simple layer which applies a sum operation over the given dimension. + When nInputDims is provided, the input will be considered as a batches. + Then the sum operation will be applied in (dimension + 1) + The input to this layer is expected to be a tensor, or a batch of tensors; + when using mini-batch, a batch of sample tensors will be passed to the layer and + the user need to specify the number of dimensions of each sample tensor in the + batch using `nInputDims`. + + :param dimension the dimension to be applied sum operation + :param n_input_dims specify the number of dimensions that this module will receive + If it is more than the dimension of input tensors, the first dimension + would be considered as batch size + :param size_average default is false, if it is true, it will return the mean instead + + >>> sum = Sum(1, 1, True) + creating: createSum + ''' + + def __init__(self, + dimension=1, + n_input_dims=-1, + size_average=False, + bigdl_type="float"): + super(Sum, self).__init__(None, bigdl_type, + dimension, + n_input_dims, + size_average) + + +class TanhShrink(Model): + + ''' + A simple layer for each element of the input tensor, do the following operation + during the forward process: + [f(x) = tanh(x) - 1] + + >>> tanhShrink = TanhShrink() + creating: createTanhShrink + ''' + + def __init__(self, + bigdl_type="float"): + super(TanhShrink, self).__init__(None, bigdl_type) + + +class Threshold(Model): + + ''' + Threshold input Tensor. + If values in the Tensor smaller than th, then replace it with v + + :param th the threshold to compare with + :param v the value to replace with + :param ip inplace mode + + >>> threshold = Threshold(1e-5, 1e-5, True) + creating: createThreshold + ''' + + def __init__(self, + th=1e-6, + v=0.0, + ip=False, + bigdl_type="float"): + super(Threshold, self).__init__(None, bigdl_type, + th, + v, + ip) + + +class Unsqueeze(Model): + + ''' + Create an Unsqueeze layer. Insert singleton dim (i.e., dimension 1) at position pos. + For an input with dim = input.dim(), + there are dim + 1 possible positions to insert the singleton dimension. + + :param pos The position will be insert singleton. + :param num_input_dims Optional. If in a batch model, set to the inputDim + + >>> unsqueeze = Unsqueeze(1, 1) + creating: createUnsqueeze + ''' + + def __init__(self, + pos, + num_input_dims=INTMIN, + bigdl_type="float"): + super(Unsqueeze, self).__init__(None, bigdl_type, + pos, + num_input_dims) + + +class Reshape(Model): + ''' + The forward(input) reshape the input tensor into a size(0) * size(1) * ... tensor, taking the + elements row-wise. + + :param size: the reshape size + + >>> reshape = Reshape([1, 28, 28]) + creating: createReshape + >>> reshape = Reshape([1, 28, 28], False) + creating: createReshape + ''' + + def __init__(self, size, batch_mode=None, bigdl_type="float"): + super(Reshape, self).__init__(None, bigdl_type, size, batch_mode) + + +class BiRecurrent(Container): + ''' + Create a Bidirectional recurrent layer + + :param merge: merge layer + + >>> biRecurrent = BiRecurrent(CAddTable()) + creating: createCAddTable + creating: createBiRecurrent + >>> biRecurrent = BiRecurrent() + creating: createBiRecurrent + ''' + + def __init__(self, + merge=None, + bigdl_type="float"): + super(BiRecurrent, self).__init__(None, bigdl_type, merge) + + +class ConcatTable(Container): + ''' + ConcateTable is a container module like Concate. Applies an input + to each member module, input can be a tensor or a table. + + ConcateTable usually works with CAddTable and CMulTable to + implement element wise add/multiply on outputs of two modules. + + >>> concatTable = ConcatTable() + creating: createConcatTable + ''' + + def __init__(self, + bigdl_type="float"): + super(ConcatTable, self).__init__(None, bigdl_type) + + +class CriterionTable(Model): + ''' + Creates a module that wraps a Criterion so that it can accept a table of inputs. + + :param criterion Criterion module + + >>> from bigdl.nn.criterion import MSECriterion + >>> criterionTable = CriterionTable(MSECriterion()) + creating: createMSECriterion + creating: createCriterionTable + ''' + + def __init__(self, + criterion, + bigdl_type="float"): + super(CriterionTable, self).__init__(None, bigdl_type, + criterion) + + +class Identity(Model): + ''' + Identity just return the input to output. + It's useful in same parallel container to get an origin input. + + >>> identity = Identity() + creating: createIdentity + ''' + + def __init__(self, + bigdl_type="float"): + super(Identity, self).__init__(None, bigdl_type) + + +class Reverse(Model): + ''' + Reverse the input w.r.t given dimension. + The input can be a Tensor or Table. + + :param dim + + >>> reverse = Reverse() + creating: createReverse + ''' + + def __init__(self, + dimension=1, + bigdl_type="float"): + super(Reverse, self).__init__(None, bigdl_type, + dimension) + + +class Transpose(Model): + ''' + Transpose input along specified dimensions + + :param permutations dimension pairs that need to swap + + >>> transpose = Transpose([(1,2)]) + creating: createTranspose + ''' + + def __init__(self, + permutations, + bigdl_type="float"): + super(Transpose, self).__init__(None, bigdl_type, + permutations) + + +class SpatialContrastiveNormalization(Model): + ''' + Subtractive + divisive contrast normalization. + + :param n_input_plane + :param kernel + :param threshold + :param thresval + + >>> kernel = np.ones([9,9]).astype("float32") + >>> spatialContrastiveNormalization = SpatialContrastiveNormalization(1, kernel) + creating: createSpatialContrastiveNormalization + >>> spatialContrastiveNormalization = SpatialContrastiveNormalization() + creating: createSpatialContrastiveNormalization + ''' + + def __init__(self, + n_input_plane=1, + kernel=None, + threshold=1e-4, + thresval=1e-4, + bigdl_type="float"): + super(SpatialContrastiveNormalization, self).__init__(None, bigdl_type, + n_input_plane, + JTensor.from_ndarray(kernel), + threshold, + thresval) + + +class SpatialConvolutionMap(Model): + ''' + This class is a generalization of SpatialConvolution. + It uses a generic connection table between input and output features. + The SpatialConvolution is equivalent to using a full connection table. + + >>> ct = np.ones([9,9]).astype("float32") + >>> spatialConvolutionMap = SpatialConvolutionMap(ct, 9, 9) + creating: createSpatialConvolutionMap + ''' + + def __init__(self, + conn_table, + kw, + kh, + dw=1, + dh=1, + pad_w=0, + pad_h=0, + bigdl_type="float"): + super(SpatialConvolutionMap, self).__init__(None, bigdl_type, + JTensor.from_ndarray(conn_table), + kw, + kh, + dw, + dh, + pad_w, + pad_h) + + +class SpatialDivisiveNormalization(Model): + ''' + Applies a spatial division operation on a series of 2D inputs using kernel for + computing the weighted average in a neighborhood. The neighborhood is defined for + a local spatial region that is the size as kernel and across all features. For + an input image, since there is only one feature, the region is only spatial. For + an RGB image, the weighted average is taken over RGB channels and a spatial region. + + If the kernel is 1D, then it will be used for constructing and separable 2D kernel. + The operations will be much more efficient in this case. + + The kernel is generally chosen as a gaussian when it is believed that the correlation + of two pixel locations decrease with increasing distance. On the feature dimension, + a uniform average is used since the weighting across features is not known. + + + :param nInputPlane number of input plane, default is 1. + :param kernel kernel tensor, default is a 9 x 9 tensor. + :param threshold threshold + :param thresval threshhold value to replace with + if data is smaller than theshold + + >>> kernel = np.ones([9,9]).astype("float32") + >>> spatialDivisiveNormalization = SpatialDivisiveNormalization(2,kernel) + creating: createSpatialDivisiveNormalization + >>> spatialDivisiveNormalization = SpatialDivisiveNormalization() + creating: createSpatialDivisiveNormalization + ''' + + def __init__(self, + n_input_plane=1, + kernel=None, + threshold=1e-4, + thresval=1e-4, + bigdl_type="float"): + super(SpatialDivisiveNormalization, self).__init__(None, bigdl_type, + n_input_plane, + JTensor.from_ndarray(kernel), + threshold, + thresval) + +class Graph(Model): + ''' + A graph container. Each node can have multiple inputs. The output of the node should be a + tensor. The output tensor can be connected to multiple nodes. So the module in each node can + have a tensor or table input, and should have a tensor output. + + The graph container can have multiple inputs and multiple outputs. If there's one input, + the input data fed to the graph module should be a tensor. If there're multiple inputs, + the input data fed to the graph module should be a table, which is actually an sequence of + tensor. The order of the input tensors should be same with the order of the input nodes. + This is also applied to the gradient from the module in the back propagation. + + If there's one output, the module output is a tensor. If there're multiple outputs, the module + output is a table, which is actually an sequence of tensor. The order of the output tensors is + same with the order of the output modules. This is also applied to the gradient passed to the + module in the back propagation. + + All inputs should be able to connect to outputs through some paths in the graph. + It is allowed that some successors of the inputs node are not connect to outputs. + If so, these nodes will be excluded in the computation. + ''' + + def __init__(self, + input, + output, + bigdl_type="float"): + super(Graph, self).__init__(None, bigdl_type, input, output) + +class SpatialSubtractiveNormalization(Model): + ''' + Applies a spatial subtraction operation on a series of 2D inputs using kernel for + computing the weighted average in a neighborhood. The neighborhood is defined for + a local spatial region that is the size as kernel and across all features. For a + an input image, since there is only one feature, the region is only spatial. For + an RGB image, the weighted average is taken over RGB channels and a spatial region. + + If the kernel is 1D, then it will be used for constructing and separable 2D kernel. + The operations will be much more efficient in this case. + + The kernel is generally chosen as a gaussian when it is believed that the correlation + of two pixel locations decrease with increasing distance. On the feature dimension, + a uniform average is used since the weighting across features is not known. + + :param n_input_plane number of input plane, default is 1. + :param kernel kernel tensor, default is a 9 x 9 tensor. + + >>> kernel = np.ones([9,9]).astype("float32") + >>> spatialSubtractiveNormalization = SpatialSubtractiveNormalization(2,kernel) + creating: createSpatialSubtractiveNormalization + >>> spatialSubtractiveNormalization = SpatialSubtractiveNormalization() + creating: createSpatialSubtractiveNormalization + ''' + + def __init__(self, + n_input_plane=1, + kernel=None, + bigdl_type="float"): + super(SpatialSubtractiveNormalization, self).__init__(None, bigdl_type, + n_input_plane, + JTensor.from_ndarray(kernel)) + + +def _test(): + import doctest + from pyspark import SparkContext + from bigdl.nn import layer + from bigdl.util.common import init_engine + from bigdl.util.common import create_spark_conf + globs = layer.__dict__.copy() + sc = SparkContext(master="local[4]", appName="test layer", + conf=create_spark_conf()) + globs['sc'] = sc + init_engine() + + (failure_count, test_count) = doctest.testmod(globs=globs, + optionflags=doctest.ELLIPSIS) + if failure_count: + exit(-1) + +if __name__ == "__main__": + _test() diff --git a/python/dllib/src/bigdl/dllib/optim/__init__.py b/python/dllib/src/bigdl/dllib/optim/__init__.py new file mode 100644 index 00000000000..2151a805423 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/optim/__init__.py @@ -0,0 +1,15 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/python/dllib/src/bigdl/dllib/optim/optimizer.py b/python/dllib/src/bigdl/dllib/optim/optimizer.py new file mode 100644 index 00000000000..e868c1473bd --- /dev/null +++ b/python/dllib/src/bigdl/dllib/optim/optimizer.py @@ -0,0 +1,556 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +import os +import sys +from distutils.dir_util import mkpath + +from bigdl.nn.layer import DOUBLEMAX +from bigdl.util.common import JTensor +from bigdl.util.common import JavaValue +from bigdl.util.common import callBigDlFunc +from bigdl.util.common import callJavaFunc +from bigdl.util.common import get_spark_context + +if sys.version >= '3': + long = int + unicode = str + + +class MaxIteration(JavaValue): + """ + A trigger specifies a timespot or several timespots during training, + and a corresponding action will be taken when the timespot(s) is reached. + MaxIteration is a trigger that triggers an action when training reaches + the number of iterations specified by "max". + Usually used as end_trigger when creating an Optimizer. + + >>> maxIteration = MaxIteration(20) + creating: createMaxIteration + """ + def __init__(self, max, bigdl_type="float"): + """ + Create a MaxIteration trigger. + + :param max: max + """ + JavaValue.__init__(self, None, bigdl_type, max) + + +class MaxEpoch(JavaValue): + """ + A trigger specifies a timespot or several timespots during training, + and a corresponding action will be taken when the timespot(s) is reached. + MaxEpoch is a trigger that triggers an action when training reaches + the number of epochs specified by "max_epoch". + Usually used as end_trigger when creating an Optimizer. + + >>> maxEpoch = MaxEpoch(2) + creating: createMaxEpoch + """ + def __init__(self, max_epoch, bigdl_type="float"): + """ + Create a MaxEpoch trigger. + + :param max_epoch: max_epoch + """ + JavaValue.__init__(self, None, bigdl_type, max_epoch) + + +class EveryEpoch(JavaValue): + """ + A trigger specifies a timespot or several timespots during training, + and a corresponding action will be taken when the timespot(s) is reached. + EveryEpoch is a trigger that triggers an action when each epoch finishs. + Could be used as trigger in setvalidation and setcheckpoint in Optimizer, + and also in TrainSummary.set_summary_trigger. + + >>> everyEpoch = EveryEpoch() + creating: createEveryEpoch + """ + def __init__(self, bigdl_type="float"): + """ + Create a EveryEpoch trigger. + """ + JavaValue.__init__(self, None, bigdl_type) + + +class SeveralIteration(JavaValue): + """ + A trigger specifies a timespot or several timespots during training, + and a corresponding action will be taken when the timespot(s) is reached. + SeveralIteration is a trigger that triggers an action every "n" + iterations. + Could be used as trigger in setvalidation and setcheckpoint in Optimizer, + and also in TrainSummary.set_summary_trigger. + + >>> serveralIteration = SeveralIteration(2) + creating: createSeveralIteration + """ + def __init__(self, interval, bigdl_type="float"): + """ + Create a SeveralIteration trigger. + + :param interval: interval is the "n" where an action is triggered + every "n" iterations + """ + JavaValue.__init__(self, None, bigdl_type, interval) + + +class Poly(JavaValue): + """ + A learning rate decay policy, where the effective learning rate + follows a polynomial decay, to be zero by the max_iteration. + Calculation: base_lr (1 - iter/max_iteration) ^ (power) + + :param power + :param max_iteration + + >>> poly = Poly(0.5, 2) + creating: createPoly + """ + def __init__(self, power, max_iteration, bigdl_type="float"): + JavaValue.__init__(self, None, bigdl_type, power, max_iteration) + + +class Step(JavaValue): + """ + A learning rate decay policy, where the effective learning rate is + calculated as base_lr * gamma ^ (floor(iter / step_size)) + + :param step_size + :param gamma + + >>> step = Step(2, 0.3) + creating: createStep + """ + def __init__(self, step_size, gamma, bigdl_type="float"): + JavaValue.__init__(self, None, bigdl_type, step_size, gamma) + +class Default(JavaValue): + """ + A learning rate decay policy, where the effective learning rate is + calculated as base_lr * gamma ^ (floor(iter / step_size)) + + :param step_size + :param gamma + + >>> step = Default() + creating: createDefault + """ + def __init__(self, bigdl_type="float"): + JavaValue.__init__(self, None, bigdl_type) + +class SGD(JavaValue): + """ + A plain implementation of SGD + + :param learningrate learning rate + :param learningrate_decay learning rate decay + :param weightdecay weight decay + :param momentum momentum + :param dampening dampening for momentum + :param nesterov enables Nesterov momentum + :param learningrates 1D tensor of individual learning rates + :param weightdecays 1D tensor of individual weight decays + >>> sgd = SGD() + creating: createDefault + creating: createSGD + """ + def __init__(self, + learningrate=1e-3, + learningrate_decay=0.0, + weightdecay=0.0, + momentum=0.0, + dampening=DOUBLEMAX, + nesterov=False, + leaningrate_schedule=None, + learningrates=None, + weightdecays=None, + bigdl_type="float"): + JavaValue.__init__(self, None, bigdl_type, learningrate, learningrate_decay, weightdecay, + momentum, dampening, nesterov, + leaningrate_schedule if (leaningrate_schedule) else Default(), + JTensor.from_ndarray(learningrates), JTensor.from_ndarray(weightdecays)) + +class Adagrad(JavaValue): + """ + An implementation of Adagrad. See the original paper: + http://jmlr.org/papers/volume12/duchi11a/duchi11a.pdf + + :param learningrate learning rate + :param learningrate_decay learning rate decay + :param weightdecay weight decay + >>> adagrad = Adagrad() + creating: createAdagrad + """ + def __init__(self, + learningrate=1e-3, + learningrate_decay=0.0, + weightdecay=0.0, + bigdl_type="float"): + JavaValue.__init__(self, None, bigdl_type, learningrate, learningrate_decay, weightdecay) + +class LBFGS(JavaValue): + """ + This implementation of L-BFGS relies on a user-provided line + search function (state.lineSearch). If this function is not + provided, then a simple learningRate is used to produce fixed + size steps. Fixed size steps are much less costly than line + searches, and can be useful for stochastic problems. + The learning rate is used even when a line search is provided. + This is also useful for large-scale stochastic problems, where + opfunc is a noisy approximation of f(x). In that case, the learning + rate allows a reduction of confidence in the step size. + + :param max_iter Maximum number of iterations allowed + :param max_eval Maximum number of function evaluations + :param tolfun Termination tolerance on the first-order optimality + :param tolx Termination tol on progress in terms of func/param changes + :param ncorrection + :param learningrate + :param verbose + :param linesearch A line search function + :param linesearch_options If no line search provided, then a fixed step size is used + >>> lbfgs = LBFGS() + creating: createLBFGS + """ + def __init__(self, + max_iter=20, + max_eval=DOUBLEMAX, + tolfun=1e-5, + tolx=1e-9, + ncorrection=100, + learningrate=1.0, + verbose=False, + linesearch=None, + linesearch_options=None, + bigdl_type="float"): + if linesearch or linesearch_options: + raise ValueError('linesearch and linesearch_options must be None in LBFGS') + JavaValue.__init__(self, None, bigdl_type, max_iter, max_eval, tolfun, tolx, + ncorrection, learningrate, verbose, linesearch, linesearch_options) + +class Adadelta(JavaValue): + """ + Adadelta implementation for SGD: http://arxiv.org/abs/1212.5701 + + :param decayrate interpolation parameter rho + :param epsilon for numerical stability + >>> adagrad = Adadelta() + creating: createAdadelta + """ + def __init__(self, + decayrate = 0.9, + epsilon = 1e-10, + bigdl_type="float"): + JavaValue.__init__(self, None, bigdl_type, decayrate, epsilon) + +class Adam(JavaValue): + """ + An implementation of Adam http://arxiv.org/pdf/1412.6980.pdf + :param learningrate learning rate + :param learningrate_decay learning rate decay + :param beta1 first moment coefficient + :param beta2 second moment coefficient + :param epsilon for numerical stability + >>> adagrad = Adam() + creating: createAdam + """ + def __init__(self, + learningrate = 1e-3, + learningrate_decay = 0.0, + beta1 = 0.9, + beta2 = 0.999, + epsilon = 1e-8, + bigdl_type="float"): + JavaValue.__init__(self, None, bigdl_type, learningrate, learningrate_decay, + beta1, beta2, epsilon) + +class Adamax(JavaValue): + """ + An implementation of Adamax http://arxiv.org/pdf/1412.6980.pdf + :param learningrate learning rate + :param beta1 first moment coefficient + :param beta2 second moment coefficient + :param epsilon for numerical stability + >>> adagrad = Adamax() + creating: createAdamax + """ + def __init__(self, + learningrate = 0.002, + beta1 = 0.9, + beta2 = 0.999, + epsilon = 1e-38, + bigdl_type="float"): + JavaValue.__init__(self, None, bigdl_type, learningrate, beta1, beta2, epsilon) + +class RMSprop(JavaValue): + """ + An implementation of RMSprop + :param learningrate learning rate + :param learningrate_decay learning rate decay + :param decayrate decay rate, also called rho + :param epsilon for numerical stability + >>> adagrad = RMSprop() + creating: createRMSprop + """ + def __init__(self, + learningrate = 1e-2, + learningrate_decay = 0.0, + decayrate = 0.99, + epsilon = 1e-8, + bigdl_type="float"): + JavaValue.__init__(self, None, bigdl_type, learningrate, learningrate_decay, decayrate, epsilon) + +class MultiStep(JavaValue): + """ + similar to step but it allows non uniform steps defined by stepSizes + + :param step_size the series of step sizes used for lr decay + :param gamma coefficient of decay + + >>> step = MultiStep([2, 5], 0.3) + creating: createMultiStep + """ + def __init__(self, step_sizes, gamma, bigdl_type="float"): + JavaValue.__init__(self, None, bigdl_type, step_sizes, gamma) + + +class Optimizer(JavaValue): + """ + An optimizer is in general to minimize any function with respect + to a set of parameters. In case of training a neural network, + an optimizer tries to minimize the loss of the neural net with + respect to its weights/biases, over the training set. + """ + def __init__(self, + model, + training_rdd, + criterion, + end_trigger, + batch_size, + optim_method=None, + bigdl_type="float"): + """ + Create an optimizer. + + :param model: the neural net model + :param traiing_rdd: the training dataset + :param criterion: the loss function + :param optim_method: the algorithm to use for optimization, + e.g. SGD, Adagrad, etc. If optim_method is None, the default algorithm is SGD. + :param end_trigger: when to end the optimization + :param batch_size: training batch size + """ + JavaValue.__init__(self, None, bigdl_type, model.value, + training_rdd, criterion, + optim_method if optim_method else SGD(), end_trigger, batch_size) + + def set_validation(self, batch_size, val_rdd, trigger, val_method=["Top1Accuracy"]): + """ + Configure validation settings. + + :param batch_size: validation batch size + :param val_rdd: validation dataset + :param trigger: validation interval + :param val_method: the ValidationMethod to use, + e.g. "Top1Accuracy", "Top5Accuracy", "Loss" + """ + callBigDlFunc(self.bigdl_type, "setValidation", self.value, batch_size, + trigger, val_rdd, val_method) + + def set_model(self, model): + """ + Set model. + + :param model: new model + """ + self.value.setModel(model.value) + + def set_checkpoint(self, checkpoint_trigger, + checkpoint_path, isOverWrite=True): + """ + Configure checkpoint settings. + + :param checkpoint_trigger: the interval to write snapshots + :param checkpoint_path: the path to write snapshots into + :param isOverWrite: whether to overwrite existing snapshots in path. + default is True + """ + if not os.path.exists(checkpoint_path): + mkpath(checkpoint_path) + callBigDlFunc(self.bigdl_type, "setCheckPoint", self.value, + checkpoint_trigger, checkpoint_path, isOverWrite) + + # return a module + def optimize(self): + """ + Do an optimization. + """ + jmodel = callJavaFunc(get_spark_context(), self.value.optimize) + from bigdl.nn.layer import Model + return Model.of(jmodel) + + def set_train_summary(self, summary): + """ + Set train summary. A TrainSummary object contains information + necesary for the optimizer to know how often the logs are recorded, + where to store the logs and how to retrieve them, etc. For details, + refer to the docs of TrainSummary. + + :param summary: a TrainSummary object + """ + callBigDlFunc(self.bigdl_type, "setTrainSummary", self.value, + summary) + return self + + def set_val_summary(self, summary): + """ + Set validation summary. A ValidationSummary object contains information + necesary for the optimizer to know how often the logs are recorded, + where to store the logs and how to retrieve them, etc. For details, + refer to the docs of ValidationSummary. + + :param summary: a ValidationSummary object + + """ + callBigDlFunc(self.bigdl_type, "setValSummary", self.value, + summary) + return self + + def prepare_input(self): + """ + Load input. Notebook user can call this method to seprate load data and + create optimizer time + """ + print("Loading input ...") + self.value.prepareInput() + + +class TrainSummary(JavaValue, ): + """ + A logging facility which allows user to trace how indicators (e.g. + learning rate, training loss, throughput, etc.) change with iterations/time + in an optimization process. TrainSummary is for training indicators only + (check ValidationSummary for validation indicators). It contains necessary + information for the optimizer to know where to store the logs, how to + retrieve the logs, and so on. - The logs are written in tensorflow-compatible + format so that they can be visualized directly using tensorboard. Also the + logs can be retrieved as ndarrays and visualized using python libraries + such as matplotlib (in notebook, etc.). + + Use optimizer.setTrainSummary to enable train logger. + """ + def __init__(self, log_dir, app_name, bigdl_type="float"): + """ + Create a TrainSummary. Logs will be saved to log_dir/app_name/train. + + :param log_dir: the root dir to store the logs + :param app_name: the application name + """ + JavaValue.__init__(self, None, bigdl_type, log_dir, app_name) + + def read_scalar(self, tag): + """ + Retrieve train logs by type. Return an array of records in the format + (step,value,wallClockTime). - "Step" is the iteration count by default. + + :param tag: the type of the logs, Supported tags are: "LearningRate", + "Loss", "Throughput" + """ + return callBigDlFunc(self.bigdl_type, "summaryReadScalar", self.value, + tag) + + def set_summary_trigger(self, name, trigger): + """ + Set the interval of recording for each indicator. + + :param tag: tag name. Supported tag names are "LearningRate", "Loss", + "Throughput", "Parameters". "Parameters" is an umbrella tag that + includes weight, bias, gradWeight, gradBias, and some running status + (eg. runningMean and runningVar in BatchNormalization). If you + didn't set any triggers, we will by default record Loss and Throughput + in each iteration, while *NOT* recording LearningRate and Parameters, + as recording parameters may introduce substantial overhead when the + model is very big, LearningRate is not a public attribute for all + OptimMethod. + :param trigger: trigger + """ + return callBigDlFunc(self.bigdl_type, "summarySetTrigger", self.value, + name, trigger) + + +class ValidationSummary(JavaValue): + """ + A logging facility which allows user to trace how indicators (e.g. + validation loss, top1 accuray, top5 accuracy etc.) change with + iterations/time in an optimization process. ValidationSummary is for + validation indicators only (check TrainSummary for train indicators). + It contains necessary information for the optimizer to know where to + store the logs, how to retrieve the logs, and so on. - The logs are + written in tensorflow-compatible format so that they can be visualized + directly using tensorboard. Also the logs can be retrieved as ndarrays + and visualized using python libraries such as matplotlib + (in notebook, etc.). + + Use optimizer.setValidationSummary to enable validation logger. + """ + def __init__(self, log_dir, app_name, bigdl_type="float"): + """ + Create a ValidationSummary. Logs will be saved to + log_dir/app_name/train. By default, all ValidationMethod set into + optimizer will be recorded and the recording interval is the same + as trigger of ValidationMethod in the optimizer. + + :param log_dir: the root dir to store the logs + :param app_name: the application name + """ + JavaValue.__init__(self, None, bigdl_type, log_dir, app_name) + + def read_scalar(self, tag): + """ + Retrieve validation logs by type. Return an array of records in the + format (step,value,wallClockTime). - "Step" is the iteration count + by default. + + :param tag: the type of the logs. The tag should match the name of + the ValidationMethod set into the optimizer. e.g. + "Top1AccuracyLoss","Top1Accuracy" or "Top5Accuracy". + """ + return callBigDlFunc(self.bigdl_type, "summaryReadScalar", self.value, + tag) + + +def _test(): + import doctest + from pyspark import SparkContext + from bigdl.optim import optimizer + from bigdl.util.common import init_engine + from bigdl.util.common import create_spark_conf + globs = optimizer.__dict__.copy() + sc = SparkContext(master="local[4]", appName="test optimizer", + conf=create_spark_conf()) + init_engine() + globs['sc'] = sc + (failure_count, test_count) = doctest.testmod(globs=globs, + optionflags=doctest.ELLIPSIS) + if failure_count: + exit(-1) + +if __name__ == "__main__": + _test() diff --git a/python/dllib/src/bigdl/dllib/utils/common.py b/python/dllib/src/bigdl/dllib/utils/common.py new file mode 100644 index 00000000000..1839a62d8b8 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/utils/common.py @@ -0,0 +1,390 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import sys +from py4j.protocol import Py4JJavaError +from py4j.java_gateway import JavaObject +from py4j.java_collections import ListConverter, JavaArray, JavaList, JavaMap + +from pyspark import RDD, SparkContext +from pyspark.serializers import PickleSerializer, AutoBatchedSerializer +from pyspark.sql import DataFrame, SQLContext +from pyspark.mllib.common import callJavaFunc +from pyspark import SparkConf +import numpy as np +import threading + +if sys.version >= '3': + long = int + unicode = str + +class SingletonMixin(object): + _lock = threading.RLock() + _instance = None + + @classmethod + def instance(cls, + bigdl_type="float"): + if not cls._instance: + with cls._lock: + if not cls._instance: + cls._instance = cls(bigdl_type) + return cls._instance + +class JavaCreator(SingletonMixin): + __creator_class="com.intel.analytics.bigdl.python.api.PythonBigDL" + + @classmethod + def get_creator_class(cls): + with JavaCreator._lock: + return JavaCreator.__creator_class + + @classmethod + def set_creator_class(cls, cclass): + with JavaCreator._lock: + JavaCreator.__creator_class = cclass + JavaCreator._instance = None + + def __init__(self, bigdl_type): + sc = get_spark_context() + jclass = getattr(sc._jvm, JavaCreator.get_creator_class()) + if bigdl_type == "float": + self.value = getattr(jclass, "ofFloat")() + elif bigdl_type == "double": + self.value = getattr(jclass, "ofDouble")() + else: + raise Exception("Not supported bigdl_type: %s" % bigdl_type) + + +class JavaValue(object): + def jvm_class_constructor(self): + name = "create" + self.__class__.__name__ + print("creating: " + name) + return name + + def __init__(self, jvalue, bigdl_type, *args): + self.value = jvalue if jvalue else callBigDlFunc( + bigdl_type, self.jvm_class_constructor(), *args) + self.bigdl_type = bigdl_type + + def __str__(self): + return self.value.toString() + + +class TestResult(): + """ + A testing result used to benchmark the model quality. + """ + def __init__(self, result, total_num, method): + """ + :param result: the validation result. i.e: top1 accuracy percentage. + :param total_num: the total processed records. + :param method: the validation method. i.e: Top1Accuracy + """ + self.result = result + self.total_num = total_num + self.method = method + + def __reduce__(self): + return (TestResult, (self.result, self.total_num, self.method)) + + def __str__(self): + return "Test result: %s, total_num: %s, method: %s" % ( + self.result, self.total_num, self.method) + + +class JTensor(object): + """ + A wrapper to easy our work when need to pass or return Tensor to/from Scala. + + >>> import numpy as np + >>> from bigdl.util.common import JTensor + >>> np.random.seed(123) + >>> + """ + def __init__(self, storage, shape, bigdl_type="float"): + self.storage = storage + self.shape = shape + self.bigdl_type = bigdl_type + + @classmethod + def from_ndarray(cls, a_ndarray, bigdl_type="float"): + """ + Convert a ndarray to Tensor which would be used in Java side. + >>> import numpy as np + >>> from bigdl.util.common import JTensor + >>> from bigdl.util.common import callBigDlFunc + >>> np.random.seed(123) + >>> data = np.random.uniform(0, 1, (2, 3)).astype("float32") + >>> result = JTensor.from_ndarray(data) + >>> data_back = result.to_ndarray() + >>> (data == data_back).all() + True + >>> tensor1 = callBigDlFunc("float", "testTensor", JTensor.from_ndarray(data)) # noqa + >>> array_from_tensor = tensor1.to_ndarray() + >>> (array_from_tensor == data).all() + True + """ + return cls(*JTensor.flatten_ndarray(a_ndarray), + bigdl_type= bigdl_type) if a_ndarray is not None else None # noqa + + def to_ndarray(self): + def get_dtype(): + if "float" == self.bigdl_type: + return "float32" + else: + return "float64" + return np.array(self.storage, dtype=get_dtype()).reshape(self.shape) # noqa + + @classmethod + def flatten_ndarray(cls, a_ndarray): + """ + Utility method to flatten a ndarray + :return (storage, shape) + >>> from bigdl.util.common import JTensor + >>> np.random.seed(123) + >>> data = np.random.uniform(0, 1, (2, 3)) + >>> (storage, shape) = JTensor.flatten_ndarray(data) + >>> shape + [2, 3] + >>> (storage, shape) = JTensor.flatten_ndarray(np.array(2)) + >>> shape + [1] + """ + storage = [float(i) for i in a_ndarray.ravel()] + shape = list(a_ndarray.shape) if a_ndarray.shape else [a_ndarray.size] + return storage, shape + + def __reduce__(self): + return (JTensor, (self.storage, self.shape, self.bigdl_type)) + + def __str__(self): + return "storage: %s, shape: %s," % (self.storage, self.storage) + + +class Sample(object): + def __init__(self, features, label, features_shape, label_shape, + bigdl_type="float"): + def get_dtype(): + if "float" == bigdl_type: + return "float32" + else: + return "float64" + self.features = np.array(features, dtype=get_dtype()).reshape(features_shape) # noqa + self.label = np.array(label, dtype=get_dtype()).reshape(label_shape) + self.bigdl_type = bigdl_type + + @classmethod + def from_ndarray(cls, features, label, bigdl_type="float"): + return cls( + features=[float(i) for i in features.ravel()], + label=[float(i) for i in label.ravel()], + features_shape=list(features.shape), + label_shape=list(label.shape) if label.shape else [label.size], + bigdl_type=bigdl_type) + + def __reduce__(self): + (features_storage, features_shape) = JTensor.flatten_ndarray(self.features) + (label_storage, label_shape) = JTensor.flatten_ndarray(self.label) + return (Sample, ( + features_storage, label_storage, features_shape, label_shape, + self.bigdl_type)) + + def __str__(self): + return "features: %s, label: %s," % (self.features, self.label) + +class RNG(): + """ + generate tensor data with seed + """ + def __init__(self, bigdl_type="float"): + self.bigdl_type = bigdl_type + + def set_seed(self, seed): + callBigDlFunc(self.bigdl_type, "setModelSeed", seed) + + def uniform(self, a, b, size): + return callBigDlFunc(self.bigdl_type, "uniform", a, b, size).to_ndarray() # noqa + + +_picklable_classes = [ + 'LinkedList', + 'SparseVector', + 'DenseVector', + 'DenseMatrix', + 'Rating', + 'LabeledPoint', + 'Sample', + 'TestResult', + 'JTensor' +] + + +def init_engine(bigdl_type="float"): + callBigDlFunc(bigdl_type, "initEngine") + + +def get_bigdl_conf(): + bigdl_conf_file = "spark-bigdl.conf" + bigdl_python_wrapper = "python-api.zip" + + def load_conf(conf_str): + return dict(line.split() for line in conf_str.split("\n") if + "#" not in line and line.strip()) + + for p in sys.path: + if bigdl_conf_file in p: + with open(p) if sys.version_info < (3,) else open(p, encoding='latin-1') as conf_file: # noqa + return load_conf(conf_file.read()) + if bigdl_python_wrapper in p: + import zipfile + with zipfile.ZipFile(p, 'r') as zip_conf: + content = zip_conf.read(bigdl_conf_file) + if sys.version_info >= (3,): + content = str(content, 'latin-1') + return load_conf(content) + raise Exception("Cannot find spark-bigdl.conf.Pls add it to PYTHONPATH.") + + +def to_list(a): + if type(a) is list: + return a + return [a] + + +def create_spark_conf(): + bigdl_conf = get_bigdl_conf() + sparkConf = SparkConf() + sparkConf.setAll(bigdl_conf.items()) + return sparkConf + + +def get_spark_context(): + if "getOrCreate" in SparkContext.__dict__: + return SparkContext.getOrCreate() + else: + with SparkContext._lock: # Compatible with Spark1.5.1 + if SparkContext._active_spark_context is None: + SparkContext(SparkConf()) + return SparkContext._active_spark_context + + +def get_spark_sql_context(sc): + if "getOrCreate" in SQLContext.__dict__: + return SQLContext.getOrCreate() + else: + return SQLContext(sc) # Compatible with Spark1.5.1 + +def callBigDlFunc(bigdl_type, name, *args): + """ Call API in PythonBigDL """ + jinstance = JavaCreator.instance(bigdl_type=bigdl_type).value + sc = get_spark_context() + api = getattr(jinstance, name) + return callJavaFunc(sc, api, *args) + + +def _java2py(sc, r, encoding="bytes"): + if isinstance(r, JavaObject): + clsName = r.getClass().getSimpleName() + # convert RDD into JavaRDD + if clsName != 'JavaRDD' and clsName.endswith("RDD"): + r = r.toJavaRDD() + clsName = 'JavaRDD' + + if clsName == 'JavaRDD': + jrdd = sc._jvm.SerDe.javaToPython(r) + return RDD(jrdd, sc) + + if clsName == 'DataFrame': + return DataFrame(r, get_spark_sql_context(sc)) + + if clsName in _picklable_classes: + r = sc._jvm.org.apache.spark.bigdl.api.python.BigDLSerDe.dumps(r) + elif isinstance(r, (JavaArray, JavaList, JavaMap)): + try: + r = sc._jvm.org.apache.spark.bigdl.api.python.BigDLSerDe.dumps( + r) + except Py4JJavaError: + pass # not pickable + + if isinstance(r, (bytearray, bytes)): + r = PickleSerializer().loads(bytes(r), encoding=encoding) + return r + + +def callJavaFunc(sc, func, *args): + """ Call Java Function """ + args = [_py2java(sc, a) for a in args] + result = func(*args) + return _java2py(sc, result) + + +def _to_java_object_rdd(rdd): + """ Return a JavaRDD of Object by unpickling + + It will convert each Python object into Java object by Pyrolite, whenever + the RDD is serialized in batch or not. + """ + rdd = rdd._reserialize(AutoBatchedSerializer(PickleSerializer())) + return \ + rdd.ctx._jvm.org.apache.spark.bigdl.api.python.BigDLSerDe.pythonToJava( + rdd._jrdd, True) + + +def _py2java(sc, obj): + """ Convert Python object into Java """ + if isinstance(obj, RDD): + obj = _to_java_object_rdd(obj) + elif isinstance(obj, DataFrame): + obj = obj._jdf + elif isinstance(obj, SparkContext): + obj = obj._jsc + elif isinstance(obj, (list, tuple)): + obj = ListConverter().convert([_py2java(sc, x) for x in obj], + sc._gateway._gateway_client) + elif isinstance(obj, dict): + result = {} + for (key, value) in obj.items(): + result[key] = _py2java(sc, value) if isinstance(value, JavaValue) else value # noqa + obj = result + + elif isinstance(obj, JavaValue): + obj = obj.value + elif isinstance(obj, JavaObject): + pass + elif isinstance(obj, (int, long, float, bool, bytes, unicode)): + pass + else: + data = bytearray(PickleSerializer().dumps(obj)) + obj = sc._jvm.org.apache.spark.bigdl.api.python.BigDLSerDe.loads(data) + return obj + + +def _test(): + import doctest + from pyspark import SparkContext + from bigdl.nn import layer + globs = layer.__dict__.copy() + sc = SparkContext(master="local[2]", appName="test common utility") + globs['sc'] = sc + (failure_count, test_count) = doctest.testmod(globs=globs, + optionflags=doctest.ELLIPSIS) + if failure_count: + exit(-1) + + +if __name__ == "__main__": + _test() diff --git a/python/dllib/src/bigdl/utils/__init__.py b/python/dllib/src/bigdl/utils/__init__.py new file mode 100644 index 00000000000..2151a805423 --- /dev/null +++ b/python/dllib/src/bigdl/utils/__init__.py @@ -0,0 +1,15 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/python/dllib/test/__init__.py b/python/dllib/test/__init__.py new file mode 100644 index 00000000000..2151a805423 --- /dev/null +++ b/python/dllib/test/__init__.py @@ -0,0 +1,15 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# From b410627341c151bca039a8e09be10cab21ce0d80 Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Tue, 6 Jun 2017 16:15:49 +0800 Subject: [PATCH 030/823] Add bigdl as the top module (#973) * add bigdl as the top modules * back to test Former-commit-id: ae6938d2b3cb1413c213a07179aa957ad76b3d5c --- dist/Bigbig-0.1.3.tar.gz.REMOVED.git-id | 1 + dist/Bigbig-0.1.4.tar.gz.REMOVED.git-id | 1 + dist/Bigbig-0.1.5.tar.gz.REMOVED.git-id | 1 + dist/Bigbig-0.1.6.tar.gz.REMOVED.git-id | 1 + dist/Bigbig-0.1.7.tar.gz.REMOVED.git-id | 1 + .../bigdl/dllib/feature/dataset/__init__.py | 15 + .../src/bigdl/dllib/feature/dataset/base.py | 197 ++ .../src/bigdl/dllib/feature/dataset/mnist.py | 129 + .../src/bigdl/dllib/feature/dataset/news20.py | 104 + .../dllib/feature/dataset/transformer.py | 26 + .../dllib/src/bigdl/dllib/models/__init__.py | 15 + .../src/bigdl/dllib/models/lenet/README.md | 64 + .../src/bigdl/dllib/models/lenet/__init__.py | 15 + .../src/bigdl/dllib/models/lenet/lenet5.py | 102 + .../dllib/models/textclassifier/README.md | 72 + .../models/textclassifier/textclassifier.py | 177 + python/dllib/src/bigdl/dllib/nn/__init__.py | 15 + python/dllib/src/bigdl/dllib/nn/criterion.py | 605 ++++ python/dllib/src/bigdl/dllib/nn/layer.py | 3005 +++++++++++++++++ .../dllib/src/bigdl/dllib/optim/__init__.py | 15 + .../dllib/src/bigdl/dllib/optim/optimizer.py | 556 +++ python/dllib/src/bigdl/utils/__init__.py | 15 + python/dllib/src/bigdl/utils/common.py | 390 +++ python/dllib/test/__init__.py | 15 + 24 files changed, 5537 insertions(+) create mode 100644 dist/Bigbig-0.1.3.tar.gz.REMOVED.git-id create mode 100644 dist/Bigbig-0.1.4.tar.gz.REMOVED.git-id create mode 100644 dist/Bigbig-0.1.5.tar.gz.REMOVED.git-id create mode 100644 dist/Bigbig-0.1.6.tar.gz.REMOVED.git-id create mode 100644 dist/Bigbig-0.1.7.tar.gz.REMOVED.git-id create mode 100644 python/dllib/src/bigdl/dllib/feature/dataset/__init__.py create mode 100644 python/dllib/src/bigdl/dllib/feature/dataset/base.py create mode 100644 python/dllib/src/bigdl/dllib/feature/dataset/mnist.py create mode 100644 python/dllib/src/bigdl/dllib/feature/dataset/news20.py create mode 100644 python/dllib/src/bigdl/dllib/feature/dataset/transformer.py create mode 100644 python/dllib/src/bigdl/dllib/models/__init__.py create mode 100644 python/dllib/src/bigdl/dllib/models/lenet/README.md create mode 100644 python/dllib/src/bigdl/dllib/models/lenet/__init__.py create mode 100644 python/dllib/src/bigdl/dllib/models/lenet/lenet5.py create mode 100644 python/dllib/src/bigdl/dllib/models/textclassifier/README.md create mode 100644 python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py create mode 100644 python/dllib/src/bigdl/dllib/nn/__init__.py create mode 100644 python/dllib/src/bigdl/dllib/nn/criterion.py create mode 100644 python/dllib/src/bigdl/dllib/nn/layer.py create mode 100644 python/dllib/src/bigdl/dllib/optim/__init__.py create mode 100644 python/dllib/src/bigdl/dllib/optim/optimizer.py create mode 100644 python/dllib/src/bigdl/utils/__init__.py create mode 100644 python/dllib/src/bigdl/utils/common.py create mode 100644 python/dllib/test/__init__.py diff --git a/dist/Bigbig-0.1.3.tar.gz.REMOVED.git-id b/dist/Bigbig-0.1.3.tar.gz.REMOVED.git-id new file mode 100644 index 00000000000..50b70e10601 --- /dev/null +++ b/dist/Bigbig-0.1.3.tar.gz.REMOVED.git-id @@ -0,0 +1 @@ +8de3ee96b2d6636d28bdf284dd0f06181dcb0c61 \ No newline at end of file diff --git a/dist/Bigbig-0.1.4.tar.gz.REMOVED.git-id b/dist/Bigbig-0.1.4.tar.gz.REMOVED.git-id new file mode 100644 index 00000000000..f4e59d7a289 --- /dev/null +++ b/dist/Bigbig-0.1.4.tar.gz.REMOVED.git-id @@ -0,0 +1 @@ +b9410cbb0e3776d3c3640efcf67dbe3a81262b8f \ No newline at end of file diff --git a/dist/Bigbig-0.1.5.tar.gz.REMOVED.git-id b/dist/Bigbig-0.1.5.tar.gz.REMOVED.git-id new file mode 100644 index 00000000000..ce54df3848c --- /dev/null +++ b/dist/Bigbig-0.1.5.tar.gz.REMOVED.git-id @@ -0,0 +1 @@ +9c4ca18f01bf477a1c2158482de9de9560840128 \ No newline at end of file diff --git a/dist/Bigbig-0.1.6.tar.gz.REMOVED.git-id b/dist/Bigbig-0.1.6.tar.gz.REMOVED.git-id new file mode 100644 index 00000000000..07119451896 --- /dev/null +++ b/dist/Bigbig-0.1.6.tar.gz.REMOVED.git-id @@ -0,0 +1 @@ +f84edbba2745d73f6567c56bed6920575d63d28b \ No newline at end of file diff --git a/dist/Bigbig-0.1.7.tar.gz.REMOVED.git-id b/dist/Bigbig-0.1.7.tar.gz.REMOVED.git-id new file mode 100644 index 00000000000..9c6597388e5 --- /dev/null +++ b/dist/Bigbig-0.1.7.tar.gz.REMOVED.git-id @@ -0,0 +1 @@ +39693bf712a1c3028a1e237dc14789af5d53bb26 \ No newline at end of file diff --git a/python/dllib/src/bigdl/dllib/feature/dataset/__init__.py b/python/dllib/src/bigdl/dllib/feature/dataset/__init__.py new file mode 100644 index 00000000000..2151a805423 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/feature/dataset/__init__.py @@ -0,0 +1,15 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/python/dllib/src/bigdl/dllib/feature/dataset/base.py b/python/dllib/src/bigdl/dllib/feature/dataset/base.py new file mode 100644 index 00000000000..f211ee5261c --- /dev/null +++ b/python/dllib/src/bigdl/dllib/feature/dataset/base.py @@ -0,0 +1,197 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os +import sys +import shutil +import tempfile +import time +from distutils.dir_util import mkpath +from six.moves.urllib.request import urlopen +import numpy as np + + +# Adopt from keras +class Progbar(object): + def __init__(self, target, width=30, verbose=1, interval=0.01): + ''' + @param target: total number of steps expected + @param interval: minimum visual progress update interval (in seconds) + ''' + self.width = width + self.target = target + self.sum_values = {} + self.unique_values = [] + self.start = time.time() + self.last_update = 0 + self.interval = interval + self.total_width = 0 + self.seen_so_far = 0 + self.verbose = verbose + + def update(self, current, values=[], force=False): + ''' + @param current: index of current step + @param values: list of tuples (name, value_for_last_step). + The progress bar will display averages for these values. + @param force: force visual progress update + ''' + for k, v in values: + if k not in self.sum_values: + self.sum_values[k] = [v * (current - self.seen_so_far), current - self.seen_so_far] + self.unique_values.append(k) + else: + self.sum_values[k][0] += v * (current - self.seen_so_far) + self.sum_values[k][1] += (current - self.seen_so_far) + self.seen_so_far = current + + now = time.time() + if self.verbose == 1: + if not force and (now - self.last_update) < self.interval: + return + + prev_total_width = self.total_width + sys.stdout.write("\b" * prev_total_width) + sys.stdout.write("\r") + + numdigits = int(np.floor(np.log10(self.target))) + 1 + barstr = '%%%dd/%%%dd [' % (numdigits, numdigits) + bar = barstr % (current, self.target) + prog = float(current) / self.target + prog_width = int(self.width * prog) + if prog_width > 0: + bar += ('=' * (prog_width-1)) + if current < self.target: + bar += '>' + else: + bar += '=' + bar += ('.' * (self.width - prog_width)) + bar += ']' + sys.stdout.write(bar) + self.total_width = len(bar) + + if current: + time_per_unit = (now - self.start) / current + else: + time_per_unit = 0 + eta = time_per_unit * (self.target - current) + info = '' + if current < self.target: + info += ' - ETA: %ds' % eta + else: + info += ' - %ds' % (now - self.start) + for k in self.unique_values: + info += ' - %s:' % k + if type(self.sum_values[k]) is list: + avg = self.sum_values[k][0] / max(1, self.sum_values[k][1]) + if abs(avg) > 1e-3: + info += ' %.4f' % avg + else: + info += ' %.4e' % avg + else: + info += ' %s' % self.sum_values[k] + + self.total_width += len(info) + if prev_total_width > self.total_width: + info += ((prev_total_width - self.total_width) * " ") + + sys.stdout.write(info) + sys.stdout.flush() + + if current >= self.target: + sys.stdout.write("\n") + + if self.verbose == 2: + if current >= self.target: + info = '%ds' % (now - self.start) + for k in self.unique_values: + info += ' - %s:' % k + avg = self.sum_values[k][0] / max(1, self.sum_values[k][1]) + if avg > 1e-3: + info += ' %.4f' % avg + else: + info += ' %.4e' % avg + sys.stdout.write(info + "\n") + + self.last_update = now + + def add(self, n, values=[]): + self.update(self.seen_so_far + n, values) + + +def display_table(rows, positions): + + def display_row(objects, positions): + line = '' + for i in range(len(objects)): + line += str(objects[i]) + line = line[:positions[i]] + line += ' ' * (positions[i] - len(line)) + print(line) + + for objects in rows: + display_row(objects, positions) + +# Adopt from keras +# Under Python 2, 'urlretrieve' relies on FancyURLopener from legacy +# urllib module, known to have issues with proxy management +if sys.version_info[0] == 2: + def urlretrieve(url, filename, reporthook=None, data=None): + def chunk_read(response, chunk_size=8192, reporthook=None): + total_size = response.info().get('Content-Length').strip() + total_size = int(total_size) + count = 0 + while 1: + chunk = response.read(chunk_size) + count += 1 + if not chunk: + reporthook(count, total_size, total_size) + break + if reporthook: + reporthook(count, chunk_size, total_size) + yield chunk + + response = urlopen(url, data) + with open(filename, 'wb') as fd: + for chunk in chunk_read(response, reporthook=reporthook): + fd.write(chunk) +else: + from six.moves.urllib.request import urlretrieve + + +def maybe_download(filename, work_directory, source_url): + if not os.path.exists(work_directory): + mkpath(work_directory) + filepath = os.path.join(work_directory, filename) + + if not os.path.exists(filepath): + print('Downloading data from', source_url) + global progbar + progbar = None + + def dl_progress(count, block_size, total_size): + global progbar + if progbar is None: + progbar = Progbar(total_size) + else: + progbar.update(count * block_size) + with tempfile.NamedTemporaryFile() as tmpfile: + temp_file_name = tmpfile.name + urlretrieve(source_url, temp_file_name, dl_progress) + shutil.copy2(temp_file_name, filepath) + size = os.path.getsize(filepath) + print('Successfully downloaded', filename, size, 'bytes.') + return filepath diff --git a/python/dllib/src/bigdl/dllib/feature/dataset/mnist.py b/python/dllib/src/bigdl/dllib/feature/dataset/mnist.py new file mode 100644 index 00000000000..f23087e8736 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/feature/dataset/mnist.py @@ -0,0 +1,129 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Part of the code originally from Tensorflow + + +import gzip + +import numpy + +from bigdl.dataset import base + +SOURCE_URL = 'http://yann.lecun.com/exdb/mnist/' + +TRAIN_MEAN = 0.13066047740239506 * 255 +TRAIN_STD = 0.3081078 * 255 +TEST_MEAN = 0.13251460696903547 * 255 +TEST_STD = 0.31048024 * 255 + + +def _read32(bytestream): + dt = numpy.dtype(numpy.uint32).newbyteorder('>') + return numpy.frombuffer(bytestream.read(4), dtype=dt)[0] + + +def extract_images(f): + """Extract the images into a 4D uint8 numpy array [index, y, x, depth]. + + Args: + f: A file object that can be passed into a gzip reader. + + Returns: + data: A 4D unit8 numpy array [index, y, x, depth]. + + Raises: + ValueError: If the bytestream does not start with 2051. + + """ + print('Extracting', f.name) + with gzip.GzipFile(fileobj=f) as bytestream: + magic = _read32(bytestream) + if magic != 2051: + raise ValueError( + 'Invalid magic number %d in MNIST image file: %s' % + (magic, f.name)) + num_images = _read32(bytestream) + rows = _read32(bytestream) + cols = _read32(bytestream) + buf = bytestream.read(rows * cols * num_images) + data = numpy.frombuffer(buf, dtype=numpy.uint8) + data = data.reshape(num_images, rows, cols, 1) + return data + + +def extract_labels(f): + print('Extracting', f.name) + with gzip.GzipFile(fileobj=f) as bytestream: + magic = _read32(bytestream) + if magic != 2049: + raise ValueError( + 'Invalid magic number %d in MNIST label file: %s' % + (magic, f.name)) + num_items = _read32(bytestream) + buf = bytestream.read(num_items) + labels = numpy.frombuffer(buf, dtype=numpy.uint8) + return labels + + +def read_data_sets(train_dir, data_type="train"): + """ + Parse or download mnist data if train_dir is empty. + :param train_dir: The directory storing the mnist data + :param data_type: Reading training set or testing set. + It can be either "train" or "test" + :return: (ndarray, ndarray) representing (features, labels) + features is a 4D unit8 numpy array [index, y, x, depth] + representing each pixel valued from 0 to 255. labels + is 1D unit8 nunpy array representing the label valued + from 0 to 9. + """ + TRAIN_IMAGES = 'train-images-idx3-ubyte.gz' + TRAIN_LABELS = 'train-labels-idx1-ubyte.gz' + TEST_IMAGES = 't10k-images-idx3-ubyte.gz' + TEST_LABELS = 't10k-labels-idx1-ubyte.gz' + + if data_type == "train": + local_file = base.maybe_download(TRAIN_IMAGES, train_dir, + SOURCE_URL + TRAIN_IMAGES) + with open(local_file, 'rb') as f: + train_images = extract_images(f) + + local_file = base.maybe_download(TRAIN_LABELS, train_dir, + SOURCE_URL + TRAIN_LABELS) + with open(local_file, 'rb') as f: + train_labels = extract_labels(f) + return train_images, train_labels + + else: + local_file = base.maybe_download(TEST_IMAGES, train_dir, + SOURCE_URL + TEST_IMAGES) + with open(local_file, 'rb') as f: + test_images = extract_images(f) + + local_file = base.maybe_download(TEST_LABELS, train_dir, + SOURCE_URL + TEST_LABELS) + with open(local_file, 'rb') as f: + test_labels = extract_labels(f) + return test_images, test_labels + + +if __name__ == "__main__": + train, _ = read_data_sets("/tmp/mnist/", "train") + test, _ = read_data_sets("/tmp/mnist", "test") + assert numpy.abs(numpy.mean(train) - TRAIN_MEAN) / TRAIN_MEAN < 1e-7 + assert numpy.abs(numpy.std(train) - TRAIN_STD) / TRAIN_STD < 1e-7 + assert numpy.abs(numpy.mean(test) - TEST_MEAN) / TEST_MEAN < 1e-7 + assert numpy.abs(numpy.std(test) - TEST_STD) / TEST_STD < 1e-7 \ No newline at end of file diff --git a/python/dllib/src/bigdl/dllib/feature/dataset/news20.py b/python/dllib/src/bigdl/dllib/feature/dataset/news20.py new file mode 100644 index 00000000000..3d066ad3b52 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/feature/dataset/news20.py @@ -0,0 +1,104 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import tarfile +from bigdl.dataset import base +import os +import sys + +NEWS20_URL = 'http://qwone.com/~jason/20Newsgroups/20news-19997.tar.gz' # noqa +GLOVE_URL = 'http://nlp.stanford.edu/data/glove.6B.zip' # noqa + +CLASS_NUM = 20 + + +def download_news20(dest_dir): + file_name = "20news-19997.tar.gz" + file_abs_path = base.maybe_download(file_name, dest_dir, NEWS20_URL) + tar = tarfile.open(file_abs_path, "r:gz") + extracted_to = os.path.join(dest_dir, "20_newsgroups") + if not os.path.exists(extracted_to): + print("Extracting %s to %s" % (file_abs_path, extracted_to)) + tar.extractall(dest_dir) + tar.close() + return extracted_to + + +def download_glove_w2v(dest_dir): + file_name = "glove.6B.zip" + file_abs_path = base.maybe_download(file_name, dest_dir, GLOVE_URL) + import zipfile + zip_ref = zipfile.ZipFile(file_abs_path, 'r') + extracted_to = os.path.join(dest_dir, "glove.6B") + if not os.path.exists(extracted_to): + print("Extracting %s to %s" % (file_abs_path, extracted_to)) + zip_ref.extractall(extracted_to) + zip_ref.close() + return extracted_to + + +def get_news20(source_dir="/tmp/news20/"): + """ + Parse or download news20 if source_dir is empty. + :param source_dir: The directory storing news data. + :return: A list of (tokens, label) + """ + news_dir = download_news20(source_dir) + texts = [] # list of text samples + label_id = 0 + for name in sorted(os.listdir(news_dir)): + path = os.path.join(news_dir, name) + label_id += 1 + if os.path.isdir(path): + for fname in sorted(os.listdir(path)): + if fname.isdigit(): + fpath = os.path.join(path, fname) + if sys.version_info < (3,): + f = open(fpath) + else: + f = open(fpath, encoding='latin-1') + content = f.read() + texts.append((content, label_id)) + f.close() + + print('Found %s texts.' % len(texts)) + return texts + + +def get_glove_w2v(source_dir="/tmp/news20/", dim=100): + """ + Parse or download the pre-trained glove word2vec if source_dir is empty. + :param source_dir: The directory storing the pre-trained word2vec + :param dim: The dimension of a vector + :return: A dict mapping from word to vector + """ + w2v_dir = download_glove_w2v(source_dir) + w2v_path = os.path.join(w2v_dir, "glove.6B.%sd.txt" % dim) + if sys.version_info < (3,): + w2v_f = open(w2v_path) + else: + w2v_f = open(w2v_path, encoding='latin-1') + pre_w2v = {} + for line in w2v_f.readlines(): + items = line.split(" ") + pre_w2v[items[0]] = [float(i) for i in items[1:]] + w2v_f.close() + return pre_w2v + + +if __name__ == "__main__": + get_news20("/tmp/news20/") + get_glove_w2v("/tmp/news20/") diff --git a/python/dllib/src/bigdl/dllib/feature/dataset/transformer.py b/python/dllib/src/bigdl/dllib/feature/dataset/transformer.py new file mode 100644 index 00000000000..018d93c13c6 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/feature/dataset/transformer.py @@ -0,0 +1,26 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +from bigdl.util.common import Sample + + +def normalizer(mean, std): + """ + Normalize features by standard deviation + """ + return lambda sample: Sample.from_ndarray((sample.features - mean) / std, + sample.label, sample.bigdl_type) diff --git a/python/dllib/src/bigdl/dllib/models/__init__.py b/python/dllib/src/bigdl/dllib/models/__init__.py new file mode 100644 index 00000000000..2151a805423 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/models/__init__.py @@ -0,0 +1,15 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/python/dllib/src/bigdl/dllib/models/lenet/README.md b/python/dllib/src/bigdl/dllib/models/lenet/README.md new file mode 100644 index 00000000000..5f0a6f05677 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/models/lenet/README.md @@ -0,0 +1,64 @@ +# LeNet5 Model on MNIST + +LeNet5 is a classical CNN model used in digital number classification. For detail information, +please refer to . + +## How to run this example: + +Program would download the minst data into ```/tmp/mnist``` automatically by default. + +``` +/tmp/mnist$ tree . +. +├── t10k-images-idx3-ubyte.gz +├── t10k-labels-idx1-ubyte.gz +├── train-images-idx3-ubyte.gz +└── train-labels-idx1-ubyte.gz + +``` + +**Source bigdl.sh fisrt which would setup the essential environment for you, otherwise program would fail fast.** + +We would train a LeNet model in spark local mode with the following commands and you can distribute it across cluster by modifying the spark master and the executor cores. + +``` + BigDL_HOME=... + + source $BigDL_HOME/dist/bin/bigdl.sh + + SPARK_HOME=... + MASTER=local[*] + PYTHON_API_ZIP_PATH=${BigDL_HOME}/dist/lib/bigdl-VERSION-python-api.zip + BigDL_JAR_PATH=${BigDL_HOME}/dist/lib/bigdl-VERSION-jar-with-dependencies.jar + PYTHONPATH=${PYTHON_API_ZIP_PATH}:$PYTHONPATH + ${SPARK_HOME}/bin/spark-submit \ + --master ${MASTER} \ + --driver-cores 2 \ + --driver-memory 2g \ + --total-executor-cores 2 \ + --executor-cores 2 \ + --executor-memory 4g \ + --conf spark.akka.frameSize=64 \ + --py-files ${PYTHON_API_ZIP_PATH},${BigDL_HOME}/pyspark/dl/models/lenet/lenet5.py \ + --properties-file ${BigDL_HOME}/dist/conf/spark-bigdl.conf \ + --jars ${BigDL_JAR_PATH} \ + --conf spark.driver.extraClassPath=${BigDL_JAR_PATH} \ + --conf spark.executor.extraClassPath=bigdl-VERSION-jar-with-dependencies.jar \ + ${BigDL_HOME}/pyspark/dl/models/lenet/lenet5.py \ + --action train + ``` + + +* ```--action``` it can be train or test. + +* ```--batchSize``` option can be used to set batch size, the default value is 128. + +To verify the accuracy, search "accuracy" from log: + +``` +INFO DistriOptimizer$:247 - [Epoch 1 0/60000][Iteration 1][Wall Clock 0.0s] Train 128 in xx seconds. Throughput is xx records/second. + +INFO DistriOptimizer$:522 - Top1Accuracy is Accuracy(correct: 9572, count: 10000, accuracy: 0.9572) + + +``` diff --git a/python/dllib/src/bigdl/dllib/models/lenet/__init__.py b/python/dllib/src/bigdl/dllib/models/lenet/__init__.py new file mode 100644 index 00000000000..2151a805423 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/models/lenet/__init__.py @@ -0,0 +1,15 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py b/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py new file mode 100644 index 00000000000..b3fcba58c9c --- /dev/null +++ b/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py @@ -0,0 +1,102 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Still in experimental stage! + +from optparse import OptionParser + +from bigdl.dataset import mnist +from bigdl.dataset.transformer import * +from bigdl.nn.layer import * +from bigdl.nn.criterion import * +from bigdl.optim.optimizer import * +from bigdl.util.common import * + + +def build_model(class_num): + model = Sequential() + model.add(Reshape([1, 28, 28])) + model.add(SpatialConvolution(1, 6, 5, 5)) + model.add(Tanh()) + model.add(SpatialMaxPooling(2, 2, 2, 2)) + model.add(Tanh()) + model.add(SpatialConvolution(6, 12, 5, 5)) + model.add(SpatialMaxPooling(2, 2, 2, 2)) + model.add(Reshape([12 * 4 * 4])) + model.add(Linear(12 * 4 * 4, 100)) + model.add(Tanh()) + model.add(Linear(100, class_num)) + model.add(LogSoftMax()) + return model + + +def get_minst(sc, data_type="train", location="/tmp/mnist"): + """ + Get and normalize the mnist data. We would download it automatically + if the data doesn't present at the specific location. + :param sc: SparkContext + :param data_type: training data or testing data + :param location: Location storing the mnist + :return: A RDD of Sample + """ + (images, labels) = mnist.read_data_sets(location, data_type) + images = sc.parallelize(images) + labels = sc.parallelize(labels) + # Target start from 1 in BigDL + record = images.zip(labels).map(lambda features_label: + Sample.from_ndarray(features_label[0], features_label[1] + 1)) + return record + + +if __name__ == "__main__": + parser = OptionParser() + parser.add_option("-a", "--action", dest="action", default="train") + parser.add_option("-b", "--batchSize", type=int, dest="batchSize", default="128") + parser.add_option("-m", "--modelPath", dest="modelPath", default="") + + (options, args) = parser.parse_args(sys.argv) + + sc = SparkContext(appName="lenet5", conf=create_spark_conf()) + init_engine() + + if options.action == "train": + train_data = get_minst(sc, "train").map( + normalizer(mnist.TRAIN_MEAN, mnist.TRAIN_STD)) + test_data = get_minst(sc, "test").map( + normalizer(mnist.TEST_MEAN, mnist.TEST_STD)) + optimizer = Optimizer( + model=build_model(10), + training_rdd=train_data, + criterion=ClassNLLCriterion(), + optim_method=SGD(learningrate=0.01, learningrate_decay=0.0002), + end_trigger=MaxEpoch(20), + batch_size=options.batchSize) + optimizer.set_validation( + batch_size=options.batchSize, + val_rdd=test_data, + trigger=EveryEpoch(), + val_method=["Top1Accuracy"] + ) + optimizer.set_checkpoint(EveryEpoch(), "/tmp/lenet5/") + trained_model = optimizer.optimize() + parameters = trained_model.parameters() + elif options.action == "test": + # Load a pre-trained model and then validate it through top1 accuracy. + test_data = get_minst(sc, "test").map( + normalizer(mnist.TEST_MEAN, mnist.TEST_STD)) + model = Model.load(options.modelPath) + results = model.test(test_data, options.batchSize, ["Top1Accuracy"]) + for result in results: + print(result) diff --git a/python/dllib/src/bigdl/dllib/models/textclassifier/README.md b/python/dllib/src/bigdl/dllib/models/textclassifier/README.md new file mode 100644 index 00000000000..eae226c7ecb --- /dev/null +++ b/python/dllib/src/bigdl/dllib/models/textclassifier/README.md @@ -0,0 +1,72 @@ +## Summary + This example use a (pre-trained GloVe embedding) to convert word to vector, + and uses it to train a CNN, LSTM or GRU text classification model on a 20 Newsgroup dataset + with 20 different categories. CNN model can achieve around 96% accuracy after 2 or 3 epochs training. + LSTM and GRU are a little difficult to train, which need more epochs to achieve the equivalent result. +(It was first described in: https://blog.keras.io/using-pre-trained-word-embeddings-in-a-keras-model.html) +## Data +* Embedding: 100-dimensional pre-trained GloVe embeddings of 400k words which trained on a 2014 dump of English Wikipedia. +* Training data: "20 Newsgroup dataset" which containing 20 categories and with totally 19997 texts. + +## How to run this example: + +If there is no [Pre-train GloVe word embeddings](http://nlp.stanford.edu/data/glove.6B.zip) +or [20 Newsgroup dataset](http://www.cs.cmu.edu/afs/cs.cmu.edu/project/theo-20/www/data/news20.html) in +`/tmp/news20` directory with the following structure looks like: + +```{r, engine='sh'} +$ [/tmp/news20]$ tree . -L 1 + . + ├── 20news-19997.tar.gz + └── glove.6B.zip +``` + +then running the flowing script would automatically download the data during the first run. + +bigdl.sh would setup the essential environment for you and it would accept a spark-submit command as an input parameter. + +```{r, engine='sh'} + PYTHONHASHSEED=... + BigDL_HOME=... + SPARK_HOME=... + MASTER=... + PYTHON_API_ZIP_PATH=${BigDL_HOME}/dist/lib/bigdl-VERSION-python-api.zip + BigDL_JAR_PATH=${BigDL_HOME}/dist/lib/bigdl-VERSION-jar-with-dependencies.jar + PYTHONPATH=${PYTHON_API_ZIP_PATH}:$PYTHONPATH + source ${BigDL_HOME}/dist/bin/bigdl.sh + + ${SPARK_HOME}/bin/spark-submit \ + --master ${MASTER} \ + --driver-cores 4 \ + --driver-memory 10g \ + --total-executor-cores 4 \ + --executor-cores 4 \ + --executor-memory 20g \ + --conf spark.akka.frameSize=64 \ + --py-files ${PYTHON_API_ZIP_PATH},${BigDL_HOME}/pyspark/dl/models/textclassifier/textclassifier.py \ + --jars ${BigDL_JAR_PATH} \ + --conf spark.driver.extraClassPath=${BigDL_JAR_PATH} \ + --conf spark.executor.extraClassPath=bigdl-VERSION-jar-with-dependencies.jar \ + --conf spark.executorEnv.PYTHONHASHSEED=${PYTHONHASHSEED} \ + ${BigDL_HOME}/pyspark/dl/models/textclassifier/textclassifier.py \ + --max_epoch 3 + --model cnn +``` + +* `--max_epoch` option can be used to set how many epochs the model to be trained + +* `--model` option can be used to choose a model to be trained, three models are supported in this example, +which are `cnn`, `lstm` and `gru`, default is `cnn` + +* `--batchSize` option can be used to set batch size, the default value is 128. + +* `embedding_dim` option can be used to set the embedding size of word vector, the default value is 50. + +To verify the accuracy, search "accuracy" from log: + +```{r, engine='sh'} + [Epoch 1 0/15964][Iteration 1][Wall Clock 0.0s] + + top1 accuracy is Accuracy(correct: 14749, count: 15964, accuracy: 0.9238912 + 553244801) +``` diff --git a/python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py b/python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py new file mode 100644 index 00000000000..9c67606c852 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py @@ -0,0 +1,177 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Still in experimental stage! + +import itertools +import re +from optparse import OptionParser + +from bigdl.dataset import news20 +from bigdl.nn.layer import * +from bigdl.nn.criterion import * +from bigdl.optim.optimizer import * +from bigdl.util.common import * +from bigdl.util.common import Sample + + +def text_to_words(review_text): + letters_only = re.sub("[^a-zA-Z]", " ", review_text) + words = letters_only.lower().split() + return words + + +def analyze_texts(data_rdd): + def index(w_c_i): + ((w, c), i) = w_c_i + return (w, (i + 1, c)) + return data_rdd.flatMap(lambda text_label: text_to_words(text_label[0])) \ + .map(lambda word: (word, 1)).reduceByKey(lambda a, b: a + b) \ + .sortBy(lambda w_c: - w_c[1]).zipWithIndex() \ + .map(lambda w_c_i: index(w_c_i)).collect() + + +# pad([1, 2, 3, 4, 5], 0, 6) +def pad(l, fill_value, width): + if len(l) >= width: + return l[0: width] + else: + l.extend([fill_value] * (width - len(l))) + return l + + +def to_vec(token, b_w2v, embedding_dim): + if token in b_w2v: + return b_w2v[token] + else: + return pad([], 0, embedding_dim) + + +def to_sample(vectors, label, embedding_dim): + # flatten nested list + flatten_features = list(itertools.chain(*vectors)) + features = np.array(flatten_features, dtype='float').reshape( + [sequence_len, embedding_dim]) + + if model_type.lower() == "cnn": + features = features.transpose(1, 0) + return Sample.from_ndarray(features, np.array(label)) + + +def build_model(class_num): + model = Sequential() + + if model_type.lower() == "cnn": + model.add(Reshape([embedding_dim, 1, sequence_len])) + model.add(SpatialConvolution(embedding_dim, 128, 5, 1)) + model.add(ReLU()) + model.add(SpatialMaxPooling(5, 1, 5, 1)) + model.add(SpatialConvolution(128, 128, 5, 1)) + model.add(ReLU()) + model.add(SpatialMaxPooling(5, 1, 5, 1)) + model.add(Reshape([128])) + elif model_type.lower() == "lstm": + model.add(Recurrent() + .add(LSTM(embedding_dim, 128, p))) + model.add(Select(2, -1)) + elif model_type.lower() == "gru": + model.add(Recurrent() + .add(GRU(embedding_dim, 128, p))) + model.add(Select(2, -1)) + else: + raise ValueError('model can only be cnn, lstm, or gru') + + model.add(Linear(128, 100)) + model.add(Linear(100, class_num)) + model.add(LogSoftMax()) + return model + + +def train(sc, + batch_size, + sequence_len, max_words, embedding_dim, training_split): + print('Processing text dataset') + texts = news20.get_news20() + data_rdd = sc.parallelize(texts, 2) + + word_to_ic = analyze_texts(data_rdd) + + # Only take the top wc between [10, sequence_len] + word_to_ic = dict(word_to_ic[10: max_words]) + bword_to_ic = sc.broadcast(word_to_ic) + + w2v = news20.get_glove_w2v(dim=embedding_dim) + filtered_w2v = {w: v for w, v in w2v.items() if w in word_to_ic} + bfiltered_w2v = sc.broadcast(filtered_w2v) + + tokens_rdd = data_rdd.map(lambda text_label: + ([w for w in text_to_words(text_label[0]) if + w in bword_to_ic.value], text_label[1])) + padded_tokens_rdd = tokens_rdd.map( + lambda tokens_label: (pad(tokens_label[0], "##", sequence_len), tokens_label[1])) + vector_rdd = padded_tokens_rdd.map(lambda tokens_label: + ([to_vec(w, bfiltered_w2v.value, + embedding_dim) for w in + tokens_label[0]], tokens_label[1])) + sample_rdd = vector_rdd.map( + lambda vectors_label: to_sample(vectors_label[0], vectors_label[1], embedding_dim)) + + train_rdd, val_rdd = sample_rdd.randomSplit( + [training_split, 1-training_split]) + + optimizer = Optimizer( + model=build_model(news20.CLASS_NUM), + training_rdd=train_rdd, + criterion=ClassNLLCriterion(), + end_trigger=MaxEpoch(max_epoch), + batch_size=batch_size, + optim_method=Adagrad(learningrate=0.01, learningrate_decay=0.0002)) + + optimizer.set_validation( + batch_size=batch_size, + val_rdd=val_rdd, + trigger=EveryEpoch(), + val_method=["Top1Accuracy"] + ) + train_model = optimizer.optimize() + +if __name__ == "__main__": + parser = OptionParser() + parser.add_option("-a", "--action", dest="action", default="train") + parser.add_option("-b", "--batchSize", dest="batchSize", default="128") + parser.add_option("-e", "--embedding_dim", dest="embedding_dim", default="50") # noqa + parser.add_option("-m", "--max_epoch", dest="max_epoch", default="15") + parser.add_option("--model", dest="model_type", default="cnn") + parser.add_option("-p", "--p", dest="p", default="0.0") + + (options, args) = parser.parse_args(sys.argv) + if options.action == "train": + batch_size = int(options.batchSize) + embedding_dim = int(options.embedding_dim) + max_epoch = int(options.max_epoch) + p = float(options.p) + model_type = options.model_type + sequence_len = 50 + max_words = 1000 + training_split = 0.8 + sc = SparkContext(appName="text_classifier", + conf=create_spark_conf()) + init_engine() + train(sc, + batch_size, + sequence_len, max_words, embedding_dim, training_split) + elif options.action == "test": + pass diff --git a/python/dllib/src/bigdl/dllib/nn/__init__.py b/python/dllib/src/bigdl/dllib/nn/__init__.py new file mode 100644 index 00000000000..2151a805423 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/nn/__init__.py @@ -0,0 +1,15 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/python/dllib/src/bigdl/dllib/nn/criterion.py b/python/dllib/src/bigdl/dllib/nn/criterion.py new file mode 100644 index 00000000000..63ddf6a5bf1 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/nn/criterion.py @@ -0,0 +1,605 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +import sys + +from bigdl.util.common import JavaValue +from bigdl.util.common import callBigDlFunc +from bigdl.util.common import JTensor +from bigdl.nn.layer import Model +import numpy as np + +if sys.version >= '3': + long = int + unicode = str + + +class Criterion(JavaValue): + """ + Criterion is helpful to train a neural network. + Given an input and a target, they compute a gradient according to a given loss function. + """ + def __init__(self, jvalue, bigdl_type, *args): + self.value = jvalue if jvalue else callBigDlFunc( + bigdl_type, JavaValue.jvm_class_constructor(self), *args) + self.bigdl_type = bigdl_type + + def __str__(self): + return self.value.toString() + + def forward(self, input, target): + """ + NB: It's for debug only, please use optimizer.optimize() in production. + Takes an input object, and computes the corresponding loss of the criterion, + compared with `target` + :param input: ndarray or list of ndarray + :param target: ndarray or list of ndarray + :return: value of loss + """ + output = callBigDlFunc(self.bigdl_type, + "criterionForward", + self.value, + Model.check_input(input), + Model.check_input(target)) + return output + + def backward(self, input, target): + """ + NB: It's for debug only, please use optimizer.optimize() in production. + Performs a back-propagation step through the criterion, with respect to the given input. + :param input: ndarray or list of ndarray + :param target: ndarray or list of ndarray + :return: ndarray + """ + output = callBigDlFunc(self.bigdl_type, + "criterionBackward", + self.value, + Model.check_input(input), + Model.check_input(target)) + return Model.convert_output(output) + + @classmethod + def of(cls, jcriterion, bigdl_type="float"): + """ + Create a python Criterion by a java criterion object + :param jcriterion: A java criterion object which created by Py4j + :return: a criterion. + """ + criterion = Criterion(bigdl_type, jcriterion) + criterion.value = jcriterion + criterion.bigdl_type = bigdl_type + return criterion + + +class ClassNLLCriterion(Criterion): + + ''' + The negative log likelihood criterion. + It is useful to train a classification problem with n classes. + If provided, the optional argument weights should be a 1D Tensor + assigning weight to each of the classes. + + :param weights weights of each class + :param size_average whether to average or not + + >>> np.random.seed(123) + >>> weights = np.random.uniform(0, 1, (2,)).astype("float32") + >>> classNLLCriterion = ClassNLLCriterion(weights,True) + creating: createClassNLLCriterion + >>> classNLLCriterion = ClassNLLCriterion() + creating: createClassNLLCriterion + ''' + + def __init__(self, + weights=None, + size_average=True, + bigdl_type="float"): + super(ClassNLLCriterion, self).__init__(None, bigdl_type, + JTensor.from_ndarray(weights), + size_average) + + +class MSECriterion(Criterion): + + ''' + Creates a criterion that measures the mean squared error between n elements + in the input x and output y: + loss(x, y) = 1/n \sum |x_i - y_i|^2 + + If x and y are d-dimensional Tensors with a total of n elements, + the sum operation still operates over all the elements, and divides by n. + The two Tensors must have the same number of elements (but their sizes might be different). + The division by n can be avoided if one sets the internal variable sizeAverage to false. + By default, the losses are averaged over observations for each minibatch. However, + if the field sizeAverage is set to false, the losses are instead summed. + + >>> mSECriterion = MSECriterion() + creating: createMSECriterion + ''' + + def __init__(self, bigdl_type="float"): + super(MSECriterion, self).__init__(None, bigdl_type) + + +class AbsCriterion(Criterion): + + ''' + measures the mean absolute value of the element-wise difference between input + + >>> absCriterion = AbsCriterion(True) + creating: createAbsCriterion + ''' + + def __init__(self, + size_average=True, + bigdl_type="float"): + super(AbsCriterion, self).__init__(None, bigdl_type, + size_average) + + +class ClassSimplexCriterion(Criterion): + + ''' + ClassSimplexCriterion implements a criterion for classification. + It learns an embedding per class, where each class' embedding is a + point on an (N-1)-dimensional simplex, where N is the number of classes. + :param nClasses the number of classes. + + >>> classSimplexCriterion = ClassSimplexCriterion(2) + creating: createClassSimplexCriterion + ''' + + def __init__(self, + n_classes, + bigdl_type="float"): + super(ClassSimplexCriterion, self).__init__(None, bigdl_type, + n_classes) + + +class CosineEmbeddingCriterion(Criterion): + + ''' + Creates a criterion that measures the loss given an input x = {x1, x2}, + a table of two Tensors, and a Tensor label y with values 1 or -1. + + :param margin a number from -1 to 1, 0 to 0.5 is suggested + + >>> cosineEmbeddingCriterion = CosineEmbeddingCriterion(1e-5, True) + creating: createCosineEmbeddingCriterion + ''' + + def __init__(self, + margin=0.0, + size_average=True, + bigdl_type="float"): + super(CosineEmbeddingCriterion, self).__init__(None, bigdl_type, + margin, + size_average) + + +class DistKLDivCriterion(Criterion): + + ''' + The Kullback-Leibler divergence criterion + + :param sizeAverage + + >>> distKLDivCriterion = DistKLDivCriterion(True) + creating: createDistKLDivCriterion + ''' + + def __init__(self, + size_average=True, + bigdl_type="float"): + super(DistKLDivCriterion, self).__init__(None, bigdl_type, + size_average) + + +class HingeEmbeddingCriterion(Criterion): + + ''' + Creates a criterion that measures the loss given an + input x which is a 1-dimensional vector and a label y (1 or -1). + This is usually used for measuring whether two inputs are similar + or dissimilar, + e.g. using the L1 pairwise distance, and is typically used for + learning nonlinear embeddings or semi-supervised learning. + + If x and y are n-dimensional Tensors, the sum operation still operates + over all the elements, and divides by n (this can be avoided if one sets + the internal variable sizeAverage to false). The margin has a default + value of 1, or can be set in the constructor. + + >>> hingeEmbeddingCriterion = HingeEmbeddingCriterion(1e-5, True) + creating: createHingeEmbeddingCriterion + ''' + + def __init__(self, + margin=1, + size_average=True, + bigdl_type="float"): + super(HingeEmbeddingCriterion, self).__init__(None, bigdl_type, + margin, + size_average) + + +class L1HingeEmbeddingCriterion(Criterion): + + ''' + Creates a criterion that measures the loss given an input x = {x1, x2}, + a table of two Tensors, and a label y (1 or -1): + + :param margin + + >>> l1HingeEmbeddingCriterion = L1HingeEmbeddingCriterion(1e-5) + creating: createL1HingeEmbeddingCriterion + ''' + + def __init__(self, + margin=1, + bigdl_type="float"): + super(L1HingeEmbeddingCriterion, self).__init__(None, bigdl_type, + margin) + + +class MarginCriterion(Criterion): + + ''' + Creates a criterion that optimizes a two-class classification hinge loss (margin-based loss) + between input x (a Tensor of dimension 1) and output y. + + :param margin if unspecified, is by default 1. + :param size_average: size average in a mini-batch + + >>> marginCriterion = MarginCriterion(1e-5, True) + creating: createMarginCriterion + ''' + + def __init__(self, + margin=1.0, + size_average=True, + bigdl_type="float"): + super(MarginCriterion, self).__init__(None, bigdl_type, + margin, + size_average) + + +class MarginRankingCriterion(Criterion): + + ''' + Creates a criterion that measures the loss given an input x = {x1, x2}, + a table of two Tensors of size 1 (they contain only scalars), and a label y (1 or -1). + In batch mode, x is a table of two Tensors of size batchsize, and y is a Tensor of size + batchsize containing 1 or -1 for each corresponding pair of elements in the input Tensor. + If y == 1 then it assumed the first input should be ranked higher (have a larger value) than + the second input, and vice-versa for y == -1. + + :param margin + + >>> marginRankingCriterion = MarginRankingCriterion(1e-5, True) + creating: createMarginRankingCriterion + ''' + + def __init__(self, + margin=1.0, + size_average=True, + bigdl_type="float"): + super(MarginRankingCriterion, self).__init__(None, bigdl_type, + margin, + size_average) + + +class MultiCriterion(Criterion): + + ''' + a weighted sum of other criterions each applied to the same input and target + + >>> multiCriterion = MultiCriterion() + creating: createMultiCriterion + ''' + + def __init__(self, + bigdl_type="float"): + super(MultiCriterion, self).__init__(None, bigdl_type) + + +class MultiLabelMarginCriterion(Criterion): + + ''' + Creates a criterion that optimizes a multi-class multi-classification hinge loss ( + margin-based loss) between input x and output y (which is a Tensor of target class indices) + + :param size_average: size average in a mini-batch + + >>> multiLabelMarginCriterion = MultiLabelMarginCriterion(True) + creating: createMultiLabelMarginCriterion + ''' + + def __init__(self, + size_average=True, + bigdl_type="float"): + super(MultiLabelMarginCriterion, self).__init__(None, bigdl_type, + size_average) + + +class ParallelCriterion(Criterion): + + ''' + ParallelCriterion is a weighted sum of other criterions each applied to a different input + and target. Set repeatTarget = true to share the target for criterions. + + Use add(criterion[, weight]) method to add criterion. Where weight is a scalar(default 1). + + :param repeat_target Whether to share the target for all criterions. + + >>> parallelCriterion = ParallelCriterion(True) + creating: createParallelCriterion + ''' + + def __init__(self, + repeat_target=False, + bigdl_type="float"): + super(ParallelCriterion, self).__init__(None, bigdl_type, + repeat_target) + + +class SmoothL1Criterion(Criterion): + + ''' + Creates a criterion that can be thought of as a smooth version of the AbsCriterion. + It uses a squared term if the absolute element-wise error falls below 1. + It is less sensitive to outliers than the MSECriterion and in some + cases prevents exploding gradients (e.g. see "Fast R-CNN" paper by Ross Girshick). + | 0.5 * (x_i - y_i)^2^, if |x_i - y_i| < 1 + loss(x, y) = 1/n \sum | + | |x_i - y_i| - 0.5, otherwise + If x and y are d-dimensional Tensors with a total of n elements, + the sum operation still operates over all the elements, and divides by n. + The division by n can be avoided if one sets the internal variable sizeAverage to false + + :param size_average whether to average the loss + + >>> smoothL1Criterion = SmoothL1Criterion(True) + creating: createSmoothL1Criterion + ''' + + def __init__(self, + size_average=True, + bigdl_type="float"): + super(SmoothL1Criterion, self).__init__(None, bigdl_type, + size_average) + + +class SmoothL1CriterionWithWeights(Criterion): + + ''' + a smooth version of the AbsCriterion + It uses a squared term if the absolute element-wise error falls below 1. + It is less sensitive to outliers than the MSECriterion and in some cases + prevents exploding gradients (e.g. see "Fast R-CNN" paper by Ross Girshick). + + d = (x - y) * w_in + loss(x, y, w_in, w_out) + | 0.5 * (sigma * d_i)^2 * w_out if |d_i| < 1 / sigma / sigma + = 1/n \sum | + | (|d_i| - 0.5 / sigma / sigma) * w_out otherwise + + >>> smoothL1CriterionWithWeights = SmoothL1CriterionWithWeights(1e-5, 1) + creating: createSmoothL1CriterionWithWeights + ''' + + def __init__(self, + sigma, + num=0, + bigdl_type="float"): + super(SmoothL1CriterionWithWeights, self).__init__(None, bigdl_type, + sigma, + num) + + +class SoftmaxWithCriterion(Criterion): + + ''' + Computes the multinomial logistic loss for a one-of-many classification task, + passing real-valued predictions through a softmax to get a probability distribution over classes. + It should be preferred over separate SoftmaxLayer + MultinomialLogisticLossLayer + as its gradient computation is more numerically stable. + :param ignoreLabel (optional) Specify a label value that + should be ignored when computing the loss. + :param normalizeMode How to normalize the output loss. + + >>> softmaxWithCriterion = SoftmaxWithCriterion() + creating: createSoftmaxWithCriterion + >>> softmaxWithCriterion = SoftmaxWithCriterion(1, "FULL") + creating: createSoftmaxWithCriterion + ''' + + def __init__(self, + ignore_label=None, + normalize_mode="VALID", + bigdl_type="float"): + super(SoftmaxWithCriterion, self).__init__(None, bigdl_type, + ignore_label, + normalize_mode) + + +class TimeDistributedCriterion(Criterion): + ''' + This class is intended to support inputs with 3 or more dimensions. + Apply Any Provided Criterion to every temporal slice of an input. + + :param criterion: embedded criterion + :param size_average: whether to divide the sequence length + + >>> td = TimeDistributedCriterion(ClassNLLCriterion()) + creating: createClassNLLCriterion + creating: createTimeDistributedCriterion + ''' + + def __init__(self, criterion, size_average=False, bigdl_type="float"): + super(TimeDistributedCriterion, self).__init__( + None, bigdl_type, criterion, size_average) + + +class CrossEntropyCriterion(Criterion): + """ + This criterion combines LogSoftMax and ClassNLLCriterion in one single class. + + :param weights A tensor assigning weight to each of the classes + + >>> np.random.seed(123) + >>> weights = np.random.uniform(0, 1, (2,)).astype("float32") + >>> cec = CrossEntropyCriterion(weights) + creating: createCrossEntropyCriterion + >>> cec = CrossEntropyCriterion() + creating: createCrossEntropyCriterion + """ + + def __init__(self, + weights=None, + size_average=True, + bigdl_type="float"): + super(CrossEntropyCriterion, self).__init__(None, bigdl_type, + JTensor.from_ndarray( + weights), + size_average) + + +class BCECriterion(Criterion): + ''' + Creates a criterion that measures the Binary Cross Entropy + between the target and the output + + :param weights weights for each class + :param sizeAverage whether to average the loss or not + + >>> np.random.seed(123) + >>> weights = np.random.uniform(0, 1, (2,)).astype("float32") + >>> bCECriterion = BCECriterion(weights) + creating: createBCECriterion + >>> bCECriterion = BCECriterion() + creating: createBCECriterion + ''' + + def __init__(self, + weights=None, + size_average=True, + bigdl_type="float"): + super(BCECriterion, self).__init__(None, bigdl_type, + JTensor.from_ndarray(weights), + size_average) + + +class MultiLabelSoftMarginCriterion(Criterion): + ''' + A MultiLabel multiclass criterion based on sigmoid: + the loss is: + l(x,y) = - sum_i y[i] * log(p[i]) + (1 - y[i]) * log (1 - p[i]) + where p[i] = exp(x[i]) / (1 + exp(x[i])) + and with weights: + l(x,y) = - sum_i weights[i] (y[i] * log(p[i]) + (1 - y[i]) * log (1 - p[i])) + + >>> np.random.seed(123) + >>> weights = np.random.uniform(0, 1, (2,)).astype("float32") + >>> multiLabelSoftMarginCriterion = MultiLabelSoftMarginCriterion(weights) + creating: createMultiLabelSoftMarginCriterion + >>> multiLabelSoftMarginCriterion = MultiLabelSoftMarginCriterion() + creating: createMultiLabelSoftMarginCriterion + ''' + + def __init__(self, + weights=None, + size_average=True, + bigdl_type="float"): + super(MultiLabelSoftMarginCriterion, self).__init__(None, bigdl_type, + JTensor.from_ndarray(weights), + size_average) + + +class MultiMarginCriterion(Criterion): + ''' + Creates a criterion that optimizes a multi-class classification hinge loss (margin-based loss) + between input x and output y (which is a target class index). + + :param p + :param weights + :param margin + :param size_average + + >>> np.random.seed(123) + >>> weights = np.random.uniform(0, 1, (2,)).astype("float32") + >>> multiMarginCriterion = MultiMarginCriterion(1,weights) + creating: createMultiMarginCriterion + >>> multiMarginCriterion = MultiMarginCriterion() + creating: createMultiMarginCriterion + ''' + + def __init__(self, + p=1, + weights=None, + margin=1.0, + size_average=True, + bigdl_type="float"): + super(MultiMarginCriterion, self).__init__(None, bigdl_type, + p, + JTensor.from_ndarray(weights), + margin, + size_average) + + +class SoftMarginCriterion(Criterion): + """ + Creates a criterion that optimizes a two-class classification logistic loss + between input x (a Tensor of dimension 1) and output y (which is a tensor + containing either 1s or -1s). + + loss(x, y) = sum_i (log(1 + exp(-y[i]*x[i]))) / x:nElement() + + :param sizeaverage The normalization by the number of elements in the input + can be disabled by setting + + >>> softMarginCriterion = SoftMarginCriterion(False) + creating: createSoftMarginCriterion + >>> softMarginCriterion = SoftMarginCriterion() + creating: createSoftMarginCriterion + """ + + def __init__(self, + size_average=True, + bigdl_type="float"): + super(SoftMarginCriterion, self).__init__(None, bigdl_type, size_average) + +def _test(): + import doctest + from pyspark import SparkContext + from bigdl.nn import criterion + from bigdl.util.common import init_engine + from bigdl.util.common import create_spark_conf + globs = criterion.__dict__.copy() + sc = SparkContext(master="local[4]", appName="test criterion", + conf=create_spark_conf()) + globs['sc'] = sc + init_engine() + + (failure_count, test_count) = doctest.testmod(globs=globs, + optionflags=doctest.ELLIPSIS) + if failure_count: + exit(-1) + + +if __name__ == "__main__": + _test() diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py new file mode 100644 index 00000000000..5a8133dd0e0 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -0,0 +1,3005 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +import sys + +import numpy as np + +from bigdl.util.common import JTensor +from bigdl.util.common import JavaValue +from bigdl.util.common import callBigDlFunc +from bigdl.util.common import callJavaFunc +from bigdl.util.common import get_spark_context +from bigdl.util.common import to_list + +if sys.version >= '3': + long = int + unicode = str + +INTMAX = 2147483647 +INTMIN = -2147483648 +DOUBLEMAX = 1.7976931348623157E308 + + +class Node(JavaValue): + """ + Represent a node in a graph. The connections between nodes are directed. + """ + def __init__(self, jvalue, bigdl_type, *args): + self.value = jvalue if jvalue else callBigDlFunc( + bigdl_type, JavaValue.jvm_class_constructor(self), *args) + self.bigdl_type = bigdl_type + + @classmethod + def of(cls, jvalue, bigdl_type="float"): + return Node(jvalue, bigdl_type) + + def element(self): + return Model.of(self.value.element()) + + +class Model(JavaValue): + """ + Model is the basic component of a neural network + and it's also the base class of layers. + Model can connect to others to construct a complex neural network. + """ + + def __init__(self, jvalue, bigdl_type, *args): + self.value = jvalue if jvalue else callBigDlFunc( + bigdl_type, JavaValue.jvm_class_constructor(self), *args) + self.bigdl_type = bigdl_type + + def __str__(self): + """ + >>> conv2 = SpatialConvolution(6, 12, 5, 5).set_name("conv2") + creating: createSpatialConvolution + >>> print(conv2) + SpatialConvolution[conv2](6 -> 12, 5 x 5, 1, 1, 0, 0) + """ + return self.value.toString() + + def __call__(self, x=None): + """ + Some other modules point to current module + :param x: upstream module nodes. x is either a Node or list of Node. + :return: node containing current module + """ + + x = x if x else [] + return Node.of(callBigDlFunc(self.bigdl_type, + "createNode", + self, + to_list(x))) + + @classmethod + def of(cls, jmodel, bigdl_type="float"): + """ + Create a Python Model + :param jmodel: Java model create by Py4j + :return: A Python Model + """ + model = Model(jmodel,bigdl_type) + return model + + def set_name(self, name): + """ + Give this model a name. There would be a generated name + consist of class name and UUID if user doesn't set it. + """ + callJavaFunc(get_spark_context(), self.value.setName, name) + return self + + def name(self): + """ + Name of this layer + """ + return callJavaFunc(get_spark_context(), self.value.getName) + + def set_seed(self, seed=123): + """ + You can control the random seed which used to init weights for this model. + :param seed: random seed + :return: Model itself. + """ + callBigDlFunc(self.bigdl_type, "setModelSeed", seed) + return self + + def get_dtype(self): + if "float" == self.bigdl_type: + return "float32" + else: + return "float64" + @staticmethod + def check_input(input): + if type(input) is list: + if len(input) == 0: + raise Exception('Error when checking: empty input') + if not hasattr(input[0], 'shape'): + raise Exception( + 'Error when checking: expecting list of ndarray') + return [JTensor.from_ndarray(i) for i in input] + else: + if not hasattr(input, 'shape'): + raise Exception( + 'Error when checking: expecting list of ndarray') + return [JTensor.from_ndarray(input)] + + @staticmethod + def convert_output(output): + if type(output) is JTensor: + return output.to_ndarray() + else: + return [x.to_ndarray() for x in output] + + def forward(self, input): + """ + NB: It's for debug only, please use optimizer.optimize() in production. + Takes an input object, and computes the corresponding output of the module + :param input: ndarray or list of ndarray + :return: ndarray or list of ndarray + """ + output = callBigDlFunc(self.bigdl_type, + "modelForward", + self.value, + self.check_input(input)) + return self.convert_output(output) + + def backward(self, input, grad_output): + """ + NB: It's for debug only, please use optimizer.optimize() in production. + Performs a back-propagation step through the module, with respect to the given input. In + general this method makes the assumption forward(input) has been called before, with the same + input. This is necessary for optimization reasons. If you do not respect this rule, backward() + will compute incorrect gradients. + :param input: ndarray or list of ndarray + :param grad_output: ndarray or list of ndarray + :return: ndarray or list of ndarray + """ + output = callBigDlFunc(self.bigdl_type, + "modelBackward", + self.value, + self.check_input(input), + self.check_input(grad_output)) + return self.convert_output(output) + + def reset(self): + """ + Initialize the model weights. + """ + callJavaFunc(get_spark_context(), self.value.reset) + return self + + def parameters(self): + """ + Get the model parameters which containing: weight, bias, gradBias, gradWeight + :return: dict(layername -> dict(parametername -> ndarray)) + """ + name_to_params = callBigDlFunc(self.bigdl_type, + "modelGetParameters", + self.value) + + def to_ndarray(params): + return { + param_name: np.array(values[0], + dtype=self.get_dtype()).reshape( + values[1]) for param_name, values in params.items()} + + return {layer_name: to_ndarray(params) for layer_name, params in + name_to_params.items()} + + def predict(self, data_rdd): + """ + Model inference base on the given data. + You need to invoke collect() to trigger those action \ + as the returning result is an RDD. + :param data_rdd: the data to be predict. + :return: An RDD represent the predict result. + """ + result = callBigDlFunc(self.bigdl_type, + "modelPredictRDD", self.value, data_rdd) + return result.map(lambda data: data.to_ndarray()) + + def test(self, val_rdd, batch_size, val_methods): + """ + A method to benchmark the model quality. + :param val_rdd: the input data + :param batch_size: batch size + :param val_methods: a list of validation methods. i.e: Top1Accuracy, + Top5Accuracy and Loss. + :return: + """ + return callBigDlFunc(self.bigdl_type, + "modelTest", + self.value, + val_rdd, batch_size, val_methods) + + def set_weights(self, weights): + """ + Set weights for this layer + :param weights: a list of numpy arrays which represent weight and bias + :return: + >>> linear = Linear(3,2) + creating: createLinear + >>> linear.set_weights([np.array([[1,2,3],[4,5,6]]), np.array([7,8])]) + >>> weights = linear.get_weights() + >>> weights[0].shape == (2,3) + True + >>> weights[0][0] + array([ 1., 2., 3.], dtype=float32) + >>> weights[1] + array([ 7., 8.], dtype=float32) + >>> relu = ReLU() + creating: createReLU + >>> from py4j.protocol import Py4JJavaError + >>> try: + ... relu.set_weights([np.array([[1,2,3],[4,5,6]]), np.array([7,8])]) + ... except Py4JJavaError as err: + ... print(err.java_exception) + ... + java.lang.IllegalArgumentException: requirement failed: this layer does not have weight/bias + >>> relu.get_weights() + The layer does not have weight/bias + >>> add = Add(2) + creating: createAdd + >>> try: + ... add.set_weights([np.array([7,8]), np.array([1,2])]) + ... except Py4JJavaError as err: + ... print(err.java_exception) + ... + java.lang.IllegalArgumentException: requirement failed: the number of input weight/bias is not consistant with number of weight/bias of this layer + """ + tensors = [JTensor.from_ndarray(param, self.bigdl_type) for param in weights] + callBigDlFunc(self.bigdl_type, "setWeights", self.value, tensors) + + def get_weights(self): + """ + Get weights for this layer + :return: list of numpy arrays which represent weight and bias + """ + tensorWeights = callBigDlFunc(self.bigdl_type, + "getWeights", self.value) + if tensorWeights is not None: + return [tensor.to_ndarray() for tensor in tensorWeights] + else: + print("The layer does not have weight/bias") + return None + + @staticmethod + def load(path, bigdl_type="float"): + """ + Load a pre-trained Bigdl model. + :param path: The path containing the pre-trained model. + :return: A pre-trained model. + """ + jmodel = callBigDlFunc(bigdl_type, "loadBigDL", path) + return Model.of(jmodel) + + @staticmethod + def load_torch(path, bigdl_type="float"): + """ + Load a pre-trained Torch model. + :param path: The path containing the pre-trained model. + :return: A pre-trained model. + """ + jmodel = callBigDlFunc(bigdl_type, "loadTorch", path) + return Model.of(jmodel) + + @staticmethod + def load_caffe(model, defPath, modelPath, match_all=True, bigdl_type="float"): + """ + Load a pre-trained Caffe model. + + :param model: A bigdl model definition \ + which equivalent to the pre-trained caffe model. + :param defPath: The path containing the caffe model definition. + :param modelPath: The path containing the pre-trained caffe model. + :return: A pre-trained model. + """ + jmodel = callBigDlFunc(bigdl_type, "loadCaffe", model, defPath, modelPath, match_all) + return Model.of(jmodel) + +class Container(Model): + ''' + [[Container]] is a sub-class of Model that declares methods defined in all containers. + A container usually contain some other modules which can be added through the "add" method + ''' + + def __init__(self, jvalue, bigdl_type, *args): + super(Container, self).__init__(jvalue, bigdl_type, *args) + + def add(self, model): + self.value.add(model.value) + return self + + + + +class Linear(Model): + + ''' + The [[Linear]] module applies a linear transformation to the input data, + i.e. `y = Wx + b`. The input given in `forward(input)` must be either + a vector (1D tensor) or matrix (2D tensor). If the input is a vector, it must + have the size of `inputSize`. If it is a matrix, then each row is assumed to be + an input sample of given batch (the number of rows means the batch size and + the number of columns should be equal to the `inputSize`). + + :param input_size the size the each input sample + :param output_size the size of the module output of each sample + :param init_method two initialized methods are supported here, which are [[Default]] + and [[Xavier]], where [[Xavier]] set bias to zero here. For more + detailed information about `initMethod`, please refer to + [[InitializationMethod]] + + >>> linear = Linear(100, 10, "Xavier") + creating: createLinear + ''' + + def __init__(self, input_size, output_size, init_method="default", with_bias=True, + bigdl_type="float"): + super(Linear, self).__init__(None, bigdl_type, input_size, output_size, + init_method, with_bias) + + +class ReLU(Model): + + ''' + Applies the rectified linear unit (ReLU) function element-wise to the input Tensor, + thus outputting a Tensor of the same dimension. + + ReLU is defined as: f(x) = max(0, x) + Can optionally do its operation in-place without using extra state memory + + >>> relu = ReLU() + creating: createReLU + ''' + + def __init__(self, ip=False, bigdl_type="float"): + super(ReLU, self).__init__(None, bigdl_type, ip) + + +class Tanh(Model): + + ''' + Applies the Tanh function element-wise to the input Tensor, thus outputting a Tensor of the same + dimension. Tanh is defined as f(x) = (exp(x)-exp(-x))/(exp(x)+exp(-x)). + + >>> tanh = Tanh() + creating: createTanh + ''' + + def __init__(self, bigdl_type="float"): + super(Tanh, self).__init__(None, bigdl_type) + + +class Echo(Model): + + ''' + This module is for debug purpose, which can print activation and gradient in your model + topology + + >>> echo = Echo() + creating: createEcho + ''' + + def __init__(self, bigdl_type="float"): + super(Echo, self).__init__(None, bigdl_type) + + +class LogSoftMax(Model): + + ''' + Applies the LogSoftMax function to an n-dimensional input Tensor. + LogSoftmax is defined as: f_i(x) = log(1 / a exp(x_i)) + where a = sum_j[exp(x_j)]. + + >>> logSoftMax = LogSoftMax() + creating: createLogSoftMax + ''' + + def __init__(self, bigdl_type="float"): + super(LogSoftMax, self).__init__(None, bigdl_type) + + +class Sequential(Container): + + ''' + Sequential provides a means to plug layers together + in a feed-forward fully connected manner. + + >>> echo = Echo() + creating: createEcho + >>> s = Sequential() + creating: createSequential + >>> s = s.add(echo) + >>> s = s.add(s) + >>> s = s.add(echo) + + ''' + + def __init__(self, bigdl_type="float"): + super(Sequential, self).__init__(None, bigdl_type) + + +class SpatialConvolution(Model): + + ''' + Applies a 2D convolution over an input image composed of several input planes. + The input tensor in forward(input) is expected to be + a 3D tensor (nInputPlane x height x width). + + :param n_input_plane The number of expected input planes in the image given into forward() + :param n_output_plane The number of output planes the convolution layer will produce. + :param kernel_w The kernel width of the convolution + :param kernel_h The kernel height of the convolution + :param stride_w The step of the convolution in the width dimension. + :param stride_h The step of the convolution in the height dimension + :param pad_w The additional zeros added per width to the input planes. + :param pad_h The additional zeros added per height to the input planes. + :param n_group Kernel group number + :param propagate_back Propagate gradient back + :param init_method Initialization method to initialize bias and weight + + >>> spatialConvolution = SpatialConvolution(6, 12, 5, 5) + creating: createSpatialConvolution + ''' + + def __init__(self, + n_input_plane, + n_output_plane, + kernel_w, + kernel_h, + stride_w=1, + stride_h=1, + pad_w=0, + pad_h=0, + n_group=1, + propagate_back=True, + init_method="default", + bigdl_type="float"): + super(SpatialConvolution, self).__init__(None, bigdl_type, + n_input_plane, + n_output_plane, + kernel_w, + kernel_h, + stride_w, + stride_h, + pad_w, + pad_h, + n_group, + propagate_back, + init_method) + + +class SpatialMaxPooling(Model): + + ''' + Applies 2D max-pooling operation in kWxkH regions by step size dWxdH steps. + The number of output features is equal to the number of input planes. + If the input image is a 3D tensor nInputPlane x height x width, + the output image size will be nOutputPlane x oheight x owidth where + owidth = op((width + 2*padW - kW) / dW + 1) + oheight = op((height + 2*padH - kH) / dH + 1) + op is a rounding operator. By default, it is floor. + It can be changed by calling :ceil() or :floor() methods. + :param kW kernel width + :param kH kernel height + :param dW step size in width + :param dH step size in height + :param padW padding in width + :param padH padding in height + >>> spatialMaxPooling = SpatialMaxPooling(2, 2, 2, 2) + creating: createSpatialMaxPooling + ''' + # to_ceil: call floor() when False; call ceil() when True + + def __init__(self, kw, + kh, + dw, + dh, + pad_w=0, + pad_h=0, + to_ceil=False, + bigdl_type="float"): + super(SpatialMaxPooling, self).__init__(None, bigdl_type, kw, + kh, + dw, + dh, + pad_w, + pad_h, + to_ceil) + + +class Select(Model): + + ''' + A Simple layer selecting an index of the input tensor in the given dimension + + :param dimension the dimension to select + :param index the index of the dimension to be selected + + >>> select = Select(1, 1) + creating: createSelect + ''' + + def __init__(self, dim, index, bigdl_type="float"): + super(Select, self).__init__(None, bigdl_type, dim, index) + +class Recurrent(Container): + ''' + Recurrent module is a container of rnn cells + Different types of rnn cells can be added using add() function + + >>> recurrent = Recurrent() + creating: createRecurrent + ''' + + def __init__(self, bigdl_type="float"): + super(Recurrent, self).__init__(None, bigdl_type) + + +class LSTM(Model): + ''' + Long Short Term Memory architecture. + Ref. + A.: http://arxiv.org/pdf/1303.5778v1 (blueprint for this module) + B. http://web.eecs.utk.edu/~itamar/courses/ECE-692/Bobby_paper1.pdf + C. http://arxiv.org/pdf/1503.04069v1.pdf + D. https://github.com/wojzaremba/lstm + E. https://github.com/Element-Research/rnn/blob/master/FastLSTM.lua + + :param inputSize: the size of each input vector + :param hiddenSize: Hidden unit size in the LSTM + :param p: is used for [[Dropout]] probability. For more details about + RNN dropouts, please refer to + [RnnDrop: A Novel Dropout for RNNs in ASR] + (http://www.stat.berkeley.edu/~tsmoon/files/Conference/asru2015.pdf) + [A Theoretically Grounded Application of Dropout in Recurrent Neural Networks] + (https://arxiv.org/pdf/1512.05287.pdf) + + >>> lstm = LSTM(4, 3, 0.5) + creating: createLSTM + ''' + + def __init__(self, input_size, hidden_size, p=0.0, bigdl_type="float"): + super(LSTM, self).__init__(None, bigdl_type, input_size, hidden_size, p) + + +class LSTMPeephole(Model): + ''' + Long Short Term Memory architecture with peephole. + Ref. A.: http://arxiv.org/pdf/1303.5778v1 (blueprint for this module) + B. http://web.eecs.utk.edu/~itamar/courses/ECE-692/Bobby_paper1.pdf + C. http://arxiv.org/pdf/1503.04069v1.pdf + D. https://github.com/wojzaremba/lstm + E. https://github.com/Element-Research/rnn/blob/master/LSTM.lua + + :param input_size: the size of each input vector + :param hidden_size: Hidden unit size in the LSTM + :param p: is used for [[Dropout]] probability. For more details about + RNN dropouts, please refer to + [RnnDrop: A Novel Dropout for RNNs in ASR] + (http://www.stat.berkeley.edu/~tsmoon/files/Conference/asru2015.pdf) + [A Theoretically Grounded Application of Dropout in Recurrent Neural Networks] + (https://arxiv.org/pdf/1512.05287.pdf) + + >>> lstm = LSTMPeephole(4, 3, 0.5) + creating: createLSTMPeephole + ''' + + def __init__(self, input_size, hidden_size, p=0.0, bigdl_type="float"): + super(LSTMPeephole, self).__init__(None, bigdl_type, input_size, hidden_size, p) + + +class GRU(Model): + ''' + Gated Recurrent Units architecture. + The first input in sequence uses zero value for cell and hidden state + + Ref. + http://www.wildml.com/2015/10/recurrent-neural-network-tutorial-part-4-implementing-a-grulstm-rnn-with-python-and-theano/ + https://github.com/Element-Research/rnn/blob/master/GRU.lua + + :param input_size: the size of each input vector + :param hidden_size: Hidden unit size in GRU + :param p: is used for [[Dropout]] probability. For more details about + RNN dropouts, please refer to + [RnnDrop: A Novel Dropout for RNNs in ASR] + (http://www.stat.berkeley.edu/~tsmoon/files/Conference/asru2015.pdf) + [A Theoretically Grounded Application of Dropout in Recurrent Neural Networks] + (https://arxiv.org/pdf/1512.05287.pdf) + + >>> gru = GRU(4, 3, 0.5) + creating: createGRU + ''' + + def __init__(self, input_size, hidden_size, p=0.0, bigdl_type="float"): + super(GRU, self).__init__(None, bigdl_type, input_size, hidden_size, p) + + +class RnnCell(Model): + ''' + It is a simple RNN. User can pass an activation function to the RNN. + + :param input_size: the size of each input vector + :param hidden_size: Hidden unit size in simple RNN + :param activation: activation function + + >>> reshape = RnnCell(4, 3, Tanh()) + creating: createTanh + creating: createRnnCell + ''' + + def __init__(self, + input_size, + hidden_size, + activation, + bigdl_type="float"): + super(RnnCell, self).__init__(None, bigdl_type, input_size, hidden_size, activation) + + +class TimeDistributed(Model): + ''' + This layer is intended to apply contained layer to each temporal time slice + of input tensor. + + For instance, The TimeDistributed Layer can feed each time slice of input tensor + to the Linear layer. + + >>> td = TimeDistributed(Linear(2, 3)) + creating: createLinear + creating: createTimeDistributed + ''' + + def __init__(self, model, bigdl_type="float"): + super(TimeDistributed, self).__init__(None, bigdl_type, model) + + +class Concat(Container): + + ''' + Concat concatenates the output of one layer of "parallel" + modules along the provided {@code dimension}: they take the + same inputs, and their output is concatenated. + +-----------+ + +----> module1 -----+ + | | | | + input -----+----> module2 -----+----> output + | | | | + +----> module3 -----+ + +-----------+ + + :param dimension: dimension + + >>> concat = Concat(2) + creating: createConcat + ''' + + def __init__(self, + dimension, + bigdl_type="float"): + super(Concat, self).__init__(None, bigdl_type, + dimension) + + +class SpatialAveragePooling(Model): + + ''' + Applies 2D average-pooling operation in kWxkH regions by step size dWxdH steps. + The number of output features is equal to the number of input planes. + + :param kW kernel width + :param kH kernel height + :param dW step width + :param dH step height + :param padW padding width + :param padH padding height + :param ceilMode whether the output size is to be ceiled or floored + :param countIncludePad whether to include padding when dividing the + number of elements in pooling region + :param divide whether to do the averaging + + >>> spatialAveragePooling = SpatialAveragePooling(7,7) + creating: createSpatialAveragePooling + ''' + + def __init__(self, + kw, + kh, + dw=1, + dh=1, + pad_w=0, + pad_h=0, + ceil_mode=False, + count_include_pad=True, + divide=True, + bigdl_type="float"): + super(SpatialAveragePooling, self).__init__(None, bigdl_type, + kw, + kh, + dw, + dh, + pad_w, + pad_h, + ceil_mode, + count_include_pad, + divide) + + +class SpatialBatchNormalization(Model): + + ''' + This file implements Batch Normalization as described in the paper: + "Batch Normalization: Accelerating Deep Network Training by Reducing Internal Covariate Shift" + by Sergey Ioffe, Christian Szegedy + This implementation is useful for inputs coming from convolution layers. + For non-convolutional layers, see [[BatchNormalization]] + The operation implemented is: + + ( x - mean(x) ) + y = -------------------- * gamma + beta + standard-deviation(x) + + where gamma and beta are learnable parameters. + The learning of gamma and beta is optional. + + >>> spatialBatchNormalization = SpatialBatchNormalization(1) + creating: createSpatialBatchNormalization + ''' + + def __init__(self, + n_output, + eps=1e-5, + momentum=0.1, + affine=True, + bigdl_type="float"): + super(SpatialBatchNormalization, self).__init__(None, bigdl_type, + n_output, + eps, + momentum, + affine) + + +class SpatialCrossMapLRN(Model): + + ''' + Applies Spatial Local Response Normalization between different feature maps. + The operation implemented is: + x_f + y_f = ------------------------------------------------- + (k+(alpha/size)* sum_{l=l1 to l2} (x_l^2^))^beta^ + + where x_f is the input at spatial locations h,w (not shown for simplicity) and feature map f, + l1 corresponds to max(0,f-ceil(size/2)) and l2 to min(F, f-ceil(size/2) + size). + Here, F is the number of feature maps. + :param size: the number of channels to sum over (for cross channel LRN) or the side length of + the square region to sum over (for within channel LRN) + :param alpha: the scaling parameter + :param beta: the exponent + :param k: a constant + + >>> spatialCrossMapLRN = SpatialCrossMapLRN() + creating: createSpatialCrossMapLRN + ''' + + def __init__(self, + size=5, + alpha=1.0, + beta=0.75, + k=1.0, + bigdl_type="float"): + super(SpatialCrossMapLRN, self).__init__(None, bigdl_type, + size, + alpha, + beta, + k) + + +class Dropout(Model): + + ''' + Dropout masks(set to zero) parts of input using a bernoulli distribution. + Each input element has a probability initP of being dropped. If scale is + set, the outputs are scaled by a factor of 1/(1-initP) during training. + During evaluating, output is the same as input. + + :param initP: probability to be dropped + :param inplace: inplace model + :param scale: if scale by a factor of 1/(1-initP) + + >>> dropout = Dropout(0.4) + creating: createDropout + ''' + + def __init__(self, + init_p=0.5, + inplace=False, + scale=True, + bigdl_type="float"): + super(Dropout, self).__init__(None, bigdl_type, + init_p, + inplace, + scale) + + +class View(Model): + + ''' + This module creates a new view of the input tensor using the sizes passed to the constructor. + The method setNumInputDims() allows to specify the expected number of dimensions of the + inputs of the modules. This makes it possible to use minibatch inputs when using a size -1 + for one of the dimensions. + + :param size: sizes use for creates a new view + + >>> view = View([1024,2]) + creating: createView + ''' + + def __init__(self, + sizes, + num_input_dims=0, + bigdl_type="float"): + super(View, self).__init__(None, bigdl_type, + sizes, + num_input_dims) + + +class Abs(Model): + + ''' + an element-wise abs operation + + >>> abs = Abs() + creating: createAbs + ''' + + def __init__(self, + bigdl_type="float"): + super(Abs, self).__init__(None, bigdl_type) + + +class Add(Model): + + ''' + adds a bias term to input data ; + :param input_size size of input data + >>> add = Add(1) + creating: createAdd + ''' + + def __init__(self, + input_size, + bigdl_type="float"): + super(Add, self).__init__(None, bigdl_type, + input_size) + + +class AddConstant(Model): + + ''' + adding a constant + + :param constant_scalar constant value + :param inplace Can optionally do its operation in-place without using extra state memory + + >>> addConstant = AddConstant(1e-5, True) + creating: createAddConstant + ''' + + def __init__(self, + constant_scalar, + inplace=False, + bigdl_type="float"): + super(AddConstant, self).__init__(None, bigdl_type, + constant_scalar, + inplace) + +class BatchNormalization(Model): + + ''' + This layer implements Batch Normalization as described in the paper: + "Batch Normalization: Accelerating Deep Network Training by Reducing Internal + Covariate Shift" + by Sergey Ioffe, Christian Szegedy https://arxiv.org/abs/1502.03167 + + This implementation is useful for inputs NOT coming from convolution layers. For convolution + layers, use nn.SpatialBatchNormalization. + + The operation implemented is: + ( x - mean(x) ) + y = -------------------- * gamma + beta + standard-deviation(x) + where gamma and beta are learnable parameters.The learning of gamma and beta is optional. + + :param n_output: output feature map number + :param eps: avoid divide zero + :param momentum: momentum for weight update + :param affine: affine operation on output or not + + >>> batchNormalization = BatchNormalization(1, 1e-5, 1e-5, True) + creating: createBatchNormalization + ''' + def __init__(self, + n_output, + eps=1e-5, + momentum=0.1, + affine=True, + bigdl_type="float"): + super(BatchNormalization, self).__init__(None, bigdl_type, + n_output, + eps, + momentum, + affine) + + +class Bilinear(Model): + + ''' + a bilinear transformation with sparse inputs, + The input tensor given in forward(input) is a table containing both inputs x_1 and x_2, + which are tensors of size N x inputDimension1 and N x inputDimension2, respectively. + + :param input_size1 input dimension of x_1 + :param input_size2 input dimension of x_2 + :param output_size output dimension + :param bias_res whether use bias + + >>> bilinear = Bilinear(1, 1, 1, True) + creating: createBilinear + ''' + + def __init__(self, + input_size1, + input_size2, + output_size, + bias_res=True, + bigdl_type="float"): + super(Bilinear, self).__init__(None, bigdl_type, + input_size1, + input_size2, + output_size, + bias_res) + + +class Bottle(Container): + + ''' + Bottle allows varying dimensionality input to be forwarded through any module + that accepts input of nInputDim dimensions, and generates output of nOutputDim dimensions. + :param module: transform module + :param n_input_dim: nInputDim dimensions of module + :param n_output_dim1: output of nOutputDim dimensions + + >>> bottle = Bottle(Linear(100,10), 1, 1) + creating: createLinear + creating: createBottle + ''' + + def __init__(self, + module, + n_input_dim=2, + n_output_dim1=INTMAX, + bigdl_type="float"): + super(Bottle, self).__init__(None, bigdl_type, + module, + n_input_dim, + n_output_dim1) + + +class CAdd(Model): + + ''' + This layer has a bias tensor with given size. The bias will be added element wise to the input + tensor. If the element number of the bias tensor match the input tensor, a simply element wise + will be done. Or the bias will be expanded to the same size of the input. The expand means + repeat on unmatched singleton dimension(if some unmatched dimension isn't singleton dimension, + it will report an error). If the input is a batch, a singleton dimension will be add to the + first dimension before the expand. + + :param size: the size of the bias + + >>> cAdd = CAdd([1,2]) + creating: createCAdd + ''' + + def __init__(self, + size, + bigdl_type="float"): + super(CAdd, self).__init__(None, bigdl_type, + size) + + +class CAddTable(Model): + + ''' + Merge the input tensors in the input table by element wise adding them together. The input + table is actually an array of tensor with same size. + + :param inplace: reuse the input memory + + >>> cAddTable = CAddTable(True) + creating: createCAddTable + ''' + + def __init__(self, + inplace=False, + bigdl_type="float"): + super(CAddTable, self).__init__(None, bigdl_type, + inplace) + + +class CDivTable(Model): + + ''' + Takes a table with two Tensor and returns the component-wise division between them. + + >>> cDivTable = CDivTable() + creating: createCDivTable + ''' + + def __init__(self, + bigdl_type="float"): + super(CDivTable, self).__init__(None, bigdl_type) + + +class CMaxTable(Model): + + ''' + Takes a table of Tensors and outputs the max of all of them. + + >>> cMaxTable = CMaxTable() + creating: createCMaxTable + ''' + + def __init__(self, + bigdl_type="float"): + super(CMaxTable, self).__init__(None, bigdl_type) + + +class CMinTable(Model): + + ''' + Takes a table of Tensors and outputs the min of all of them. + >>> cMinTable = CMinTable() + creating: createCMinTable + ''' + + def __init__(self, + bigdl_type="float"): + super(CMinTable, self).__init__(None, bigdl_type) + + +class CMul(Model): + + ''' + Applies a component-wise multiplication to the incoming data + + :param size size of the data + + >>> cMul = CMul([1,2]) + creating: createCMul + ''' + + def __init__(self, + size, + bigdl_type="float"): + super(CMul, self).__init__(None, bigdl_type, + size) + + +class CMulTable(Model): + + ''' + Takes a table of Tensors and outputs the multiplication of all of them. + + >>> cMulTable = CMulTable() + creating: createCMulTable + ''' + + def __init__(self, + bigdl_type="float"): + super(CMulTable, self).__init__(None, bigdl_type) + + +class CSubTable(Model): + + ''' + Takes a table with two Tensor and returns the component-wise subtraction between them. + + >>> cSubTable = CSubTable() + creating: createCSubTable + ''' + + def __init__(self, + bigdl_type="float"): + super(CSubTable, self).__init__(None, bigdl_type) + + +class Clamp(Model): + + ''' + Clamps all elements into the range [min_value, max_value]. + Output is identical to input in the range, + otherwise elements less than min_value (or greater than max_value) + are saturated to min_value (or max_value). + + :param min + :param max + + >>> clamp = Clamp(1, 3) + creating: createClamp + ''' + + def __init__(self, + min, + max, + bigdl_type="float"): + super(Clamp, self).__init__(None, bigdl_type, + min, + max) + + +class Contiguous(Model): + + ''' + used to make input, grad_output both contiguous + + >>> contiguous = Contiguous() + creating: createContiguous + ''' + + def __init__(self, + bigdl_type="float"): + super(Contiguous, self).__init__(None, bigdl_type) + + +class Cosine(Model): + + ''' + Cosine calculates the cosine similarity of the input to k mean centers. The input given in + forward(input) must be either a vector (1D tensor) or matrix (2D tensor). If the input is a + vector, it must have the size of inputSize. If it is a matrix, then each row is assumed to be + an input sample of given batch (the number of rows means the batch size and the number of + columns should be equal to the inputSize). + + :param input_size: the size of each input sample + :param output_size: the size of the module output of each sample + + >>> cosine = Cosine(2,3) + creating: createCosine + ''' + + def __init__(self, + input_size, + output_size, + bigdl_type="float"): + super(Cosine, self).__init__(None, bigdl_type, + input_size, + output_size) + + +class CosineDistance(Model): + + ''' + Outputs the cosine distance between inputs + + >>> cosineDistance = CosineDistance() + creating: createCosineDistance + ''' + + def __init__(self, + bigdl_type="float"): + super(CosineDistance, self).__init__(None, bigdl_type) + +class DiceCoefficientCriterion(Model): + + ''' + The Dice-Coefficient criterion + input: Tensor, target: Tensor + + return: 2 * (input intersection target) + 1 - ---------------------------------- + input union target + + >>> diceCoefficientCriterion = DiceCoefficientCriterion(size_average = True, epsilon = 1.0) + creating: createDiceCoefficientCriterion + ''' + + def __init__(self, + size_average, + epsilon, + bigdl_type="float"): + super(DiceCoefficientCriterion, self).__init__(None, bigdl_type, + size_average, + epsilon) + +class DotProduct(Model): + + ''' + This is a simple table layer which takes a table of two tensors as input + and calculate the dot product between them as outputs + + >>> dotProduct = DotProduct() + creating: createDotProduct + ''' + + def __init__(self, + bigdl_type="float"): + super(DotProduct, self).__init__(None, bigdl_type) + + +class ELU(Model): + + ''' + D-A Clevert, Thomas Unterthiner, Sepp Hochreiter + Fast and Accurate Deep Network Learning by Exponential Linear Units (ELUs) + [http://arxiv.org/pdf/1511.07289.pdf] + + >>> eLU = ELU(1e-5, True) + creating: createELU + ''' + + def __init__(self, + alpha=1.0, + inplace=False, + bigdl_type="float"): + super(ELU, self).__init__(None, bigdl_type, + alpha, + inplace) + + +class Euclidean(Model): + + ''' + Outputs the Euclidean distance of the input to outputSize centers + :param inputSize inputSize + :param outputSize outputSize + :param T Numeric type. Only support float/double now + + >>> euclidean = Euclidean(1, 1, True) + creating: createEuclidean + ''' + + + + def __init__(self, + input_size, + output_size, + fast_backward=True, + bigdl_type="float"): + super(Euclidean, self).__init__(None, bigdl_type, + input_size, + output_size, + fast_backward) + + +class Exp(Model): + + ''' + Applies element-wise exp to input tensor. + >>> exp = Exp() + creating: createExp + ''' + + def __init__(self, + bigdl_type="float"): + super(Exp, self).__init__(None, bigdl_type) + + +class FlattenTable(Model): + + ''' + This is a table layer which takes an arbitrarily deep table of Tensors + (potentially nested) as input and a table of Tensors without any nested + table will be produced + + >>> flattenTable = FlattenTable() + creating: createFlattenTable + ''' + + def __init__(self, + bigdl_type="float"): + super(FlattenTable, self).__init__(None, bigdl_type) + + +class GradientReversal(Model): + + ''' + It is a simple module preserves the input, but takes the + gradient from the subsequent layer, multiplies it by -lambda + and passes it to the preceding layer. This can be used to maximise + an objective function whilst using gradient descent, as described in + ["Domain-Adversarial Training of Neural Networks" + (http://arxiv.org/abs/1505.07818)] + + :param lambda hyper-parameter lambda can be set dynamically during training + + >>> gradientReversal = GradientReversal(1e-5) + creating: createGradientReversal + ''' + + def __init__(self, + the_lambda=1, + bigdl_type="float"): + super(GradientReversal, self).__init__(None, bigdl_type, + the_lambda) + + +class HardShrink(Model): + + ''' + This is a transfer layer which applies the hard shrinkage function + element-wise to the input Tensor. The parameter lambda is set to 0.5 + by default + x, if x > lambda + f(x) = x, if x < -lambda + 0, otherwise + + :param the_lambda: a threshold value whose default value is 0.5 + + >>> hardShrink = HardShrink(1e-5) + creating: createHardShrink + ''' + + def __init__(self, + the_lambda=0.5, + bigdl_type="float"): + super(HardShrink, self).__init__(None, bigdl_type, + the_lambda) + + +class HardTanh(Model): + + ''' + Applies HardTanh to each element of input, HardTanh is defined: + | maxValue, if x > maxValue + f(x) = | minValue, if x < minValue + | x, otherwise + :param min_value minValue in f(x), default is -1. + :param max_value maxValue in f(x), default is 1. + :param inplace whether enable inplace model. + + >>> hardTanh = HardTanh(1e-5, 1e5, True) + creating: createHardTanh + ''' + + def __init__(self, + min_value=-1, + max_value=1, + inplace=False, + bigdl_type="float"): + super(HardTanh, self).__init__(None, bigdl_type, + min_value, + max_value, + inplace) + + +class Index(Model): + + ''' + Applies the Tensor index operation along the given dimension. + + :param dimension the dimension to be indexed + + >>> index = Index(1) + creating: createIndex + ''' + + def __init__(self, + dimension, + bigdl_type="float"): + super(Index, self).__init__(None, bigdl_type, + dimension) + + +class InferReshape(Model): + + ''' + Reshape with the support of infered size, + Positive numbers are used directly, setting the corresponding dimension of the output tensor. + In addition, two special values are accepted: + 0 means "copy the respective dimension of the input". + i.e., if the input has 2 as its 1st dimension, + the output will have 2 as its 1st dimension as well + -1 stands for "infer this from the other dimensions" + this dimension is calculated to keep the overall element count the same as in the input. + At most one -1 can be used in a reshape operation. + For example, (4, 5, 6, 7) -> InferReshape (4, 0, 3, -1) -> (4, 5, 3, 14) + with 1st and 3rd dim same as given size, with 2nd dim same as input, and the infered dim is 14 + + :param size the target tensor size + :param batch_mode whether in batch mode + + >>> inferReshape = InferReshape([4, 0, 3, -1], False) + creating: createInferReshape + ''' + + def __init__(self, + size, + batch_mode=False, + bigdl_type="float"): + super(InferReshape, self).__init__(None, bigdl_type, + size, + batch_mode) + + +class JoinTable(Model): + + ''' + It is a table module which takes a table of Tensors as input and + outputs a Tensor by joining them together along the dimension `dimension`. + + The input to this layer is expected to be a tensor, or a batch of tensors; + when using mini-batch, a batch of sample tensors will be passed to the layer and + the user need to specify the number of dimensions of each sample tensor in the + batch using `nInputDims`. + + :param dimension to be join in this dimension + :param nInputDims specify the number of dimensions that this module will receive + If it is more than the dimension of input tensors, the first dimension + would be considered as batch size + + >>> joinTable = JoinTable(1, 1) + creating: createJoinTable + ''' + + def __init__(self, + dimension, + n_input_dims, + bigdl_type="float"): + super(JoinTable, self).__init__(None, bigdl_type, + dimension, + n_input_dims) + + +class L1Cost(Model): + + ''' + compute L1 norm for input, and sign of input + >>> l1Cost = L1Cost() + creating: createL1Cost + ''' + + def __init__(self, + bigdl_type="float"): + super(L1Cost, self).__init__(None, bigdl_type) + + +class L1Penalty(Model): + + ''' + adds an L1 penalty to an input (for sparsity). + L1Penalty is an inline module that in its forward propagation copies the input Tensor + directly to the output, and computes an L1 loss of the latent state (input) and stores + it in the module's loss field. During backward propagation: gradInput = gradOutput + gradLoss. + + :param l1weight + :param sizeAverage + :param provideOutput + + >>> l1Penalty = L1Penalty(1, True, True) + creating: createL1Penalty + ''' + + def __init__(self, + l1weight, + size_average=False, + provide_output=True, + bigdl_type="float"): + super(L1Penalty, self).__init__(None, bigdl_type, + l1weight, + size_average, + provide_output) + + +class LeakyReLU(Model): + + ''' + It is a transfer module that applies LeakyReLU, which parameter negval sets the slope of the + negative part: LeakyReLU is defined as: f(x) = max(0, x) + negval * min(0, x) + + :param negval: sets the slope of the negative partl + :param inplace: if it is true, doing the operation in-place without using extra state memory + + >>> leakyReLU = LeakyReLU(1e-5, True) + creating: createLeakyReLU + ''' + + def __init__(self, + negval=0.01, + inplace=False, + bigdl_type="float"): + super(LeakyReLU, self).__init__(None, bigdl_type, + negval, + inplace) + + +class Log(Model): + + ''' + Applies the log function element-wise to the input Tensor, + thus outputting a Tensor of the same dimension. + + >>> log = Log() + creating: createLog + ''' + + def __init__(self, + bigdl_type="float"): + super(Log, self).__init__(None, bigdl_type) + + +class LogSigmoid(Model): + + ''' + This class is a transform layer corresponding to the sigmoid function: + f(x) = Log(1 / (1 + e ^^ (-x))) + + >>> logSigmoid = LogSigmoid() + creating: createLogSigmoid + ''' + + def __init__(self, + bigdl_type="float"): + super(LogSigmoid, self).__init__(None, bigdl_type) + + +class LookupTable(Model): + + ''' + a convolution of width 1, commonly used for word embeddings + + >>> lookupTable = LookupTable(1, 1, 1e-5, 1e-5, 1e-5, True) + creating: createLookupTable + ''' + + def __init__(self, + n_index, + n_output, + padding_value=0.0, + max_norm=DOUBLEMAX, + norm_type=2.0, + should_scale_grad_by_freq=False, + bigdl_type="float"): + super(LookupTable, self).__init__(None, bigdl_type, + n_index, + n_output, + padding_value, + max_norm, + norm_type, + should_scale_grad_by_freq) + + +class MM(Model): + + ''' + Module to perform matrix multiplication on two mini-batch inputs, producing a mini-batch. + + :param trans_a: specifying whether or not transpose the first input matrix + :param trans_b: specifying whether or not transpose the second input matrix + + >>> mM = MM(True, True) + creating: createMM + ''' + def __init__(self, + trans_a=False, + trans_b=False, + bigdl_type="float"): + super(MM, self).__init__(None, bigdl_type, + trans_a, + trans_b) + + +class MV(Model): + + ''' + It is a module to perform matrix vector multiplication on two mini-batch inputs, + producing a mini-batch. + + :param trans whether make matrix transpose before multiplication + + >>> mV = MV(True) + creating: createMV + ''' + + def __init__(self, + trans=False, + bigdl_type="float"): + super(MV, self).__init__(None, bigdl_type, + trans) + + +class MapTable(Container): + + ''' + This class is a container for a single module which will be applied + to all input elements. The member module is cloned as necessary to + process all input elements. + + >>> mapTable = MapTable(Linear(100,10)) + creating: createLinear + creating: createMapTable + ''' + + def __init__(self, + module, + bigdl_type="float"): + super(MapTable, self).__init__(None, bigdl_type, + module) + +class MaskedSelect(Model): + + ''' + Performs a torch.MaskedSelect on a Tensor. The mask is supplied as a tabular argument with + the input on the forward and backward passes. + >>> maskedSelect = MaskedSelect() + creating: createMaskedSelect + ''' + + def __init__(self, + bigdl_type="float"): + super(MaskedSelect, self).__init__(None, bigdl_type) + + +class Max(Model): + + ''' + Applies a max operation over dimension `dim` + + :param dim max along this dimension + :param num_input_dims Optional. If in a batch model, set to the inputDims. + + >>> max = Max(1) + creating: createMax + ''' + + def __init__(self, + dim, + num_input_dims=INTMIN, + bigdl_type="float"): + super(Max, self).__init__(None, bigdl_type, + dim, + num_input_dims) + + +class Mean(Model): + + ''' + It is a simple layer which applies a mean operation over the given dimension. When nInputDims + is provided, the input will be considered as batches. Then the mean operation will be applied + in (dimension + 1). The input to this layer is expected to be a tensor, or a batch of + tensors; when using mini-batch, a batch of sample tensors will be passed to the layer and the + user need to specify the number of dimensions of each sample tensor in the batch using + nInputDims. + + :param dimension: the dimension to be applied mean operation + :param n_input_dims: specify the number of dimensions that this module will receive + If it is more than the dimension of input tensors, the first dimension would be considered + as batch size + + >>> mean = Mean(1, 1) + creating: createMean + ''' + + def __init__(self, + dimension=1, + n_input_dims=-1, + bigdl_type="float"): + super(Mean, self).__init__(None, bigdl_type, + dimension, + n_input_dims) + + +class Min(Model): + + ''' + Applies a min operation over dimension `dim`. + + :param dim min along this dimension + :param num_input_dims Optional. If in a batch model, set to the input_dim. + + >>> min = Min(1) + creating: createMin + ''' + + def __init__(self, + dim, + num_input_dims=INTMIN, + bigdl_type="float"): + super(Min, self).__init__(None, bigdl_type, + dim, + num_input_dims) + + +class MixtureTable(Model): + + ''' + Creates a module that takes a table {gater, experts} as input and outputs the mixture of experts + (a Tensor or table of Tensors) using a gater Tensor. When dim is provided, it specifies the + dimension of the experts Tensor that will be interpolated (or mixed). Otherwise, the experts + should take the form of a table of Tensors. This Module works for experts of dimension 1D or + more, and for a 1D or 2D gater, i.e. for single examples or mini-batches. + + >>> mixtureTable = MixtureTable() + creating: createMixtureTable + >>> mixtureTable = MixtureTable(10) + creating: createMixtureTable + ''' + + def __init__(self, + dim=INTMAX, + bigdl_type="float"): + super(MixtureTable, self).__init__(None, bigdl_type, dim) + + +class Mul(Model): + + ''' + Multiply a single scalar factor to the incoming data + + >>> mul = Mul() + creating: createMul + ''' + + def __init__(self, + bigdl_type="float"): + super(Mul, self).__init__(None, bigdl_type) + + +class MulConstant(Model): + + ''' + Multiplies input Tensor by a (non-learnable) scalar constant. + This module is sometimes useful for debugging purposes. + + :param scalar scalar constant + :param inplace Can optionally do its operation in-place without using extra state memory + + >>> mulConstant = MulConstant(2.5) + creating: createMulConstant + ''' + + def __init__(self, + scalar, + inplace=False, + bigdl_type="float"): + super(MulConstant, self).__init__(None, bigdl_type, + scalar, + inplace) + + +class Narrow(Model): + + ''' + Narrow is application of narrow operation in a module. + The module further supports a negative length in order to handle inputs with an unknown size. + >>> narrow = Narrow(1, 1, 1) + creating: createNarrow + ''' + + def __init__(self, + dimension, + offset, + length=1, + bigdl_type="float"): + super(Narrow, self).__init__(None, bigdl_type, + dimension, + offset, + length) + + +class NarrowTable(Model): + + ''' + Creates a module that takes a table as input and outputs the subtable starting at index + offset having length elements (defaults to 1 element). The elements can be either + a table or a Tensor. If `length` is negative, it means selecting the elements from the + offset to element which located at the abs(`length`) to the last element of the input. + + :param offset the start index of table + :param length the length want to select + + >>> narrowTable = NarrowTable(1, 1) + creating: createNarrowTable + ''' + + def __init__(self, + offset, + length=1, + bigdl_type="float"): + super(NarrowTable, self).__init__(None, bigdl_type, + offset, + length) + + +class Normalize(Model): + + ''' + Normalizes the input Tensor to have unit L_p norm. The smoothing parameter eps prevents + division by zero when the input contains all zero elements (default = 1e-10). + p can be the max value of double + + >>> normalize = Normalize(1e-5, 1e-5) + creating: createNormalize + ''' + + def __init__(self, + p, + eps=1e-10, + bigdl_type="float"): + super(Normalize, self).__init__(None, bigdl_type, + p, + eps) + + +class PReLU(Model): + + ''' + Applies parametric ReLU, which parameter varies the slope of the negative part. + + PReLU: f(x) = max(0, x) + a * min(0, x) + + nOutputPlane's default value is 0, that means using PReLU in shared version and has + only one parameters. + + Notice: Please don't use weight decay on this. + + :param n_output_plane input map number. Default is 0. + + >>> pReLU = PReLU(1) + creating: createPReLU + ''' + + def __init__(self, + n_output_plane=0, + bigdl_type="float"): + super(PReLU, self).__init__(None, bigdl_type, + n_output_plane) + + +class Padding(Model): + + ''' + This module adds pad units of padding to dimension dim of the input. If pad is negative, + padding is added to the left, otherwise, it is added to the right of the dimension. + + The input to this layer is expected to be a tensor, or a batch of tensors; + when using mini-batch, a batch of sample tensors will be passed to the layer and + the user need to specify the number of dimensions of each sample tensor in the + batch using n_input_dim. + + :param dim the dimension to be applied padding operation + :param pad num of the pad units + :param n_input_dim specify the number of dimensions that this module will receive + If it is more than the dimension of input tensors, the first dimension + would be considered as batch size + :param value padding value + + >>> padding = Padding(1, 1, 1, 1e-5, 1) + creating: createPadding + ''' + + def __init__(self, + dim, + pad, + n_input_dim, + value=0.0, + n_index=1, + bigdl_type="float"): + super(Padding, self).__init__(None, bigdl_type, + dim, + pad, + n_input_dim, + value, + n_index) + + +class PairwiseDistance(Model): + + ''' + It is a module that takes a table of two vectors as input and outputs + the distance between them using the p-norm. + The input given in `forward(input)` is a [[Table]] that contains two tensors which + must be either a vector (1D tensor) or matrix (2D tensor). If the input is a vector, + it must have the size of `inputSize`. If it is a matrix, then each row is assumed to be + an input sample of the given batch (the number of rows means the batch size and + the number of columns should be equal to the `inputSize`). + :param norm the norm of distance + + >>> pairwiseDistance = PairwiseDistance(2) + creating: createPairwiseDistance + ''' + + def __init__(self, + norm=2, + bigdl_type="float"): + super(PairwiseDistance, self).__init__(None, bigdl_type, + norm) + + +class ParallelTable(Container): + + ''' + It is a container module that applies the i-th member module to the i-th + input, and outputs an output in the form of Table + + >>> parallelTable = ParallelTable() + creating: createParallelTable + ''' + + def __init__(self, + bigdl_type="float"): + super(ParallelTable, self).__init__(None, bigdl_type) + + +class Power(Model): + + ''' + Apply an element-wise power operation with scale and shift. + f(x) = (shift + scale * x)^power^ + :param power: the exponent. + :param scale: Default is 1. + :param shift: Default is 0. + + >>> power = Power(1e-5) + creating: createPower + ''' + + def __init__(self, + power, + scale=1.0, + shift=0.0, + bigdl_type="float"): + super(Power, self).__init__(None, bigdl_type, + power, + scale, + shift) + + +class RReLU(Model): + + ''' + Applies the randomized leaky rectified linear unit (RReLU) element-wise to the input Tensor, + thus outputting a Tensor of the same dimension. Informally the RReLU is also known as + 'insanity' layer. RReLU is defined as: + f(x) = max(0,x) + a * min(0, x) where a ~ U(l, u). + + In training mode negative inputs are multiplied by a factor a drawn from a uniform random + distribution U(l, u). + + In evaluation mode a RReLU behaves like a LeakyReLU with a constant mean factor + a = (l + u) / 2. + + By default, l = 1/8 and u = 1/3. If l == u a RReLU effectively becomes a LeakyReLU. + + Regardless of operating in in-place mode a RReLU will internally allocate an input-sized + noise tensor to store random factors for negative inputs. + + The backward() operation assumes that forward() has been called before. + + For reference see [Empirical Evaluation of Rectified Activations in Convolutional Network]( + http://arxiv.org/abs/1505.00853). + + :param lower: lower boundary of uniform random distribution + :param upper: upper boundary of uniform random distribution + :param inplace: optionally do its operation in-place without using extra state memory + + >>> rReLU = RReLU(1e-5, 1e5, True) + creating: createRReLU + ''' + + def __init__(self, + lower=1.0/8, + upper=1.0/3, + inplace=False, + bigdl_type="float"): + super(RReLU, self).__init__(None, bigdl_type, + lower, + upper, + inplace) + + +class ReLU6(Model): + + ''' + Same as ReLU except that the rectifying function f(x) saturates at x = 6 + + :param inplace either True = in-place or False = keeping separate state + + >>> reLU6 = ReLU6(True) + creating: createReLU6 + ''' + + def __init__(self, + inplace=False, + bigdl_type="float"): + super(ReLU6, self).__init__(None, bigdl_type, + inplace) + + +class Replicate(Model): + + ''' + Replicate repeats input `nFeatures` times along its `dim` dimension. + Notice: No memory copy, it set the stride along the `dim`-th dimension to zero. + + :param n_features: replicate times. + :param dim: dimension to be replicated. + :param n_dim: specify the number of non-batch dimensions. + + >>> replicate = Replicate(2) + creating: createReplicate + ''' + def __init__(self, + n_features, + dim=1, + n_dim=INTMAX, + bigdl_type="float"): + super(Replicate, self).__init__(None, bigdl_type, + n_features, + dim, + n_dim) + + +class RoiPooling(Model): + + ''' + Region of interest pooling + The RoIPooling uses max pooling to convert the features inside any valid region of interest + into a small feature map with a fixed spatial extent of pooledH * pooledW (e.g., 7 * 7) + an RoI is a rectangular window into a conv feature map. + Each RoI is defined by a four-tuple (x1, y1, x2, y2) that specifies its + top-left corner (x1, y1) and its bottom-right corner (x2, y2). + RoI max pooling works by dividing the h * w RoI window into an pooledH * pooledW grid of + sub-windows of approximate size h/H * w/W and then max-pooling the values in each sub-window + into the corresponding output grid cell. + Pooling is applied independently to each feature map channel + + :param pooled_w: spatial extent in width + :param pooled_h: spatial extent in height + :param spatial_scale spatial scale + + >>> roiPooling = RoiPooling(1, 1, 1e-5) + creating: createRoiPooling + ''' + + def __init__(self, + pooled_w, + pooled_h, + spatial_scale, + bigdl_type="float"): + super(RoiPooling, self).__init__(None, bigdl_type, + pooled_w, + pooled_h, + spatial_scale) + + +class Scale(Model): + + ''' + Scale is the combination of CMul and CAdd + Computes the elementwise product of input and weight, with the shape of the weight "expand" to + match the shape of the input. + Similarly, perform a expand cdd bias and perform an elementwise add + + :param size size of weight and bias + + >>> scale = Scale([1,2]) + creating: createScale + ''' + + def __init__(self, + size, + bigdl_type="float"): + super(Scale, self).__init__(None, bigdl_type, + size) + + +class SelectTable(Container): + + ''' + Creates a module that takes a table as input and outputs the element at index `index` + (positive or negative). This can be either a table or a Tensor. + The gradients of the non-index elements are zeroed Tensors of the same size. + This is true regardless of the depth of the encapsulated Tensor as the function used + internally to do so is recursive. + + :param dimension the dimension to be selected + + >>> selectTable = SelectTable(1) + creating: createSelectTable + ''' + + def __init__(self, + dimension, + bigdl_type="float"): + super(SelectTable, self).__init__(None, bigdl_type, + dimension) + + +class Sigmoid(Model): + + ''' + Applies the Sigmoid function element-wise to the input Tensor, + thus outputting a Tensor of the same dimension. + >>> sigmoid = Sigmoid() + creating: createSigmoid + ''' + + def __init__(self, + bigdl_type="float"): + super(Sigmoid, self).__init__(None, bigdl_type) + + +class SoftMax(Model): + + ''' + Applies the SoftMax function to an n-dimensional input Tensor, rescaling them so that the + elements of the n-dimensional output Tensor lie in the range (0, 1) and sum to 1. + Softmax is defined as: f_i(x) = exp(x_i - shift) / sum_j exp(x_j - shift) + where shift = max_i(x_i). + + >>> softMax = SoftMax() + creating: createSoftMax + ''' + + def __init__(self, + bigdl_type="float"): + super(SoftMax, self).__init__(None, bigdl_type) + + +class SoftMin(Model): + + ''' + Applies the SoftMin function to an n-dimensional input Tensor, rescaling them so that the + elements of the n-dimensional output Tensor lie in the range (0,1) and sum to 1. + Softmin is defined as: f_i(x) = exp(-x_i - shift) / sum_j exp(-x_j - shift) + where shift = max_i(-x_i). + + >>> softMin = SoftMin() + creating: createSoftMin + ''' + + def __init__(self, + bigdl_type="float"): + super(SoftMin, self).__init__(None, bigdl_type) + + +class SoftPlus(Model): + + ''' + Apply the SoftPlus function to an n-dimensional input tensor. + SoftPlus function: f_i(x) = 1/beta * log(1 + exp(beta * x_i)) + + :param beta Controls sharpness of transfer function + + >>> softPlus = SoftPlus(1e-5) + creating: createSoftPlus + ''' + + def __init__(self, + beta=1.0, + bigdl_type="float"): + super(SoftPlus, self).__init__(None, bigdl_type, + beta) + + +class SoftShrink(Model): + + ''' + Apply the soft shrinkage function element-wise to the input Tensor + + SoftShrinkage operator: + | x - lambda, if x > lambda + f(x) = | x + lambda, if x < -lambda + | 0, otherwise + + :param the_lambda lambda, default is 0.5 + + >>> softShrink = SoftShrink(1e-5) + creating: createSoftShrink + ''' + + def __init__(self, + the_lambda=0.5, + bigdl_type="float"): + super(SoftShrink, self).__init__(None, bigdl_type, + the_lambda) + + +class SoftSign(Model): + + ''' + Apply SoftSign function to an n-dimensional input Tensor. + + SoftSign function: f_i(x) = x_i / (1+|x_i|) + + >>> softSign = SoftSign() + creating: createSoftSign + ''' + + def __init__(self, + bigdl_type="float"): + super(SoftSign, self).__init__(None, bigdl_type) + + +class SpatialDilatedConvolution(Model): + + ''' + Apply a 2D dilated convolution over an input image. + + The input tensor is expected to be a 3D or 4D(with batch) tensor. + + If input is a 3D tensor nInputPlane x height x width, + owidth = floor(width + 2 * padW - dilationW * (kW-1) - 1) / dW + 1 + oheight = floor(height + 2 * padH - dilationH * (kH-1) - 1) / dH + 1 + + Reference Paper: Yu F, Koltun V. Multi-scale context aggregation by dilated convolutions[J]. + arXiv preprint arXiv:1511.07122, 2015. + + :param n_input_plane: The number of expected input planes in the image given into forward(). + :param n_output_plane: The number of output planes the convolution layer will produce. + :param kw: The kernel width of the convolution. + :param kh: The kernel height of the convolution. + :param dw: The step of the convolution in the width dimension. Default is 1. + :param dh: The step of the convolution in the height dimension. Default is 1. + :param pad_w: The additional zeros added per width to the input planes. Default is 0. + :param pad_h: The additional zeros added per height to the input planes. Default is 0. + :param dilation_w: The number of pixels to skip. Default is 1. + :param dilation_h: The number of pixels to skip. Default is 1. + :param init_method: Init method, Default, Xavier. + + >>> spatialDilatedConvolution = SpatialDilatedConvolution(1, 1, 1, 1) + creating: createSpatialDilatedConvolution + ''' + + def __init__(self, + n_input_plane, + n_output_plane, + kw, + kh, + dw=1, + dh=1, + pad_w=0, + pad_h=0, + dilation_w=1, + dilation_h=1, + init_method='default', + bigdl_type="float"): + super(SpatialDilatedConvolution, self).__init__(None, bigdl_type, + n_input_plane, + n_output_plane, + kw, + kh, + dw, + dh, + pad_w, + pad_h, + dilation_w, + dilation_h, + init_method) + + +class SpatialFullConvolution(Model): + ''' + Apply a 2D full convolution over an input image. + The input tensor is expected to be a 3D or 4D(with batch) tensor. Note that instead + of setting adjW and adjH, SpatialFullConvolution[Table, T] also accepts a table input + with two tensors: T(convInput, sizeTensor) where convInput is the standard input tensor, + and the size of sizeTensor is used to set the size of the output (will ignore the adjW and + adjH values used to construct the module). This module can be used without a bias by setting + parameter noBias = true while constructing the module. + + If input is a 3D tensor nInputPlane x height x width, + owidth = (width - 1) * dW - 2*padW + kW + adjW + oheight = (height - 1) * dH - 2*padH + kH + adjH + + Other frameworks call this operation "In-network Upsampling", "Fractionally-strided convolution", + "Backwards Convolution," "Deconvolution", or "Upconvolution." + + Reference Paper: Long J, Shelhamer E, Darrell T. Fully convolutional networks for semantic + segmentation[C]//Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition. + 2015: 3431-3440. + + :param nInputPlane The number of expected input planes in the image given into forward() + :param nOutputPlane The number of output planes the convolution layer will produce. + :param kW The kernel width of the convolution. + :param kH The kernel height of the convolution. + :param dW The step of the convolution in the width dimension. Default is 1. + :param dH The step of the convolution in the height dimension. Default is 1. + :param padW The additional zeros added per width to the input planes. Default is 0. + :param padH The additional zeros added per height to the input planes. Default is 0. + :param adjW Extra width to add to the output image. Default is 0. + :param adjH Extra height to add to the output image. Default is 0. + :param nGroup Kernel group number. + :param noBias If bias is needed. + :param initMethod Init method, Default, Xavier, Bilinear. + + >>> spatialFullConvolution = SpatialFullConvolution(1, 1, 1, 1) + creating: createSpatialFullConvolution + ''' + + def __init__(self, + n_input_plane, + n_output_plane, + kw, + kh, + dw=1, + dh=1, + pad_w=0, + pad_h=0, + adj_w=0, + adj_h=0, + n_group=1, + no_bias=False, + init_method='default', + bigdl_type="float"): + super(SpatialFullConvolution, self).__init__(None, bigdl_type, + n_input_plane, + n_output_plane, + kw, + kh, + dw, + dh, + pad_w, + pad_h, + adj_w, + adj_h, + n_group, + no_bias, + init_method) + + +class SpatialShareConvolution(Model): + + ''' + >>> spatialShareConvolution = SpatialShareConvolution(1, 1, 1, 1) + creating: createSpatialShareConvolution + ''' + + def __init__(self, + n_input_plane, + n_output_plane, + kernel_w, + kernel_h, + stride_w=1, + stride_h=1, + pad_w=0, + pad_h=0, + n_group=1, + propagate_back=True, + init_method='default', + bigdl_type="float"): + super(SpatialShareConvolution, self).__init__(None, bigdl_type, + n_input_plane, + n_output_plane, + kernel_w, + kernel_h, + stride_w, + stride_h, + pad_w, + pad_h, + n_group, + propagate_back, + init_method) + + +class VolumetricConvolution(Model): + + ''' + Applies a 3D convolution over an input image composed of several input planes. The input tensor + in forward(input) is expected to be a 4D tensor (nInputPlane x time x height x width). + :param n_input_plane The number of expected input planes in the image given into forward() + :param n_output_plane The number of output planes the convolution layer will produce. + :param k_t The kernel size of the convolution in time + :param k_w The kernel width of the convolution + :param k_h The kernel height of the convolution + :param d_t The step of the convolution in the time dimension. Default is 1 + :param d_w The step of the convolution in the width dimension. Default is 1 + :param d_h The step of the convolution in the height dimension. Default is 1 + :param pad_t Additional zeros added to the input plane data on both sides of time axis. + Default is 0. (kT-1)/2 is often used here. + :param pad_w The additional zeros added per width to the input planes. + :param pad_h The additional zeros added per height to the input planes. + :param with_bias whether with bias + :param init_method Init method, Default, Xavier, Bilinear. + + >>> volumetricConvolution = VolumetricConvolution(6, 12, 5, 5, 5, 1, 1, 1) + creating: createVolumetricConvolution + ''' + + def __init__(self, + n_input_plane, + n_output_plane, + k_t, + k_w, + k_h, + d_t=1, + d_w=1, + d_h=1, + pad_t=0, + pad_w=0, + pad_h=0, + with_bias=True, + init_method="default", + bigdl_type="float"): + super(VolumetricConvolution, self).__init__(None, bigdl_type, + n_input_plane, + n_output_plane, + k_t, + k_w, + k_h, + d_t, + d_w, + d_h, + pad_t, + pad_w, + pad_h, + with_bias, + init_method) + + +class VolumetricMaxPooling(Model): + + ''' + Applies 3D max-pooling operation in kTxkWxkH regions by step size dTxdWxdH steps. + The number of output features is equal to the number of input planes / dT. + The input can optionally be padded with zeros. Padding should be smaller than + half of kernel size. That is, padT < kT/2, padW < kW/2 and padH < kH/2 + :param k_t The kernel size + :param k_w The kernel width + :param k_h The kernel height + :param d_t The step in the time dimension + :param d_w The step in the width dimension + :param d_h The step in the height dimension + :param pad_t The padding in the time dimension + :param pad_w The padding in the width dimension + :param pad_h The padding in the height dimension + + >>> volumetricMaxPooling = VolumetricMaxPooling(5, 5, 5, 1, 1, 1) + creating: createVolumetricMaxPooling + ''' + + def __init__(self, + k_t, + k_w, + k_h, + d_t, + d_w, + d_h, + pad_t=0, + pad_w=0, + pad_h=0, + bigdl_type="float"): + super(VolumetricMaxPooling, self).__init__(None, bigdl_type, + k_t, + k_w, + k_h, + d_t, + d_w, + d_h, + pad_t, + pad_w, + pad_h) + + +class SpatialZeroPadding(Model): + + ''' + Each feature map of a given input is padded with specified number of zeros. + If padding values are negative, then input is cropped. + :param padLeft: pad left position + :param padRight: pad right position + :param padTop: pad top position + :param padBottom: pad bottom position + + >>> spatialZeroPadding = SpatialZeroPadding(1, 1, 1, 1) + creating: createSpatialZeroPadding + ''' + + def __init__(self, + pad_left, + pad_right, + pad_top, + pad_bottom, + bigdl_type="float"): + super(SpatialZeroPadding, self).__init__(None, bigdl_type, + pad_left, + pad_right, + pad_top, + pad_bottom) + + +class SplitTable(Model): + + ''' + Creates a module that takes a Tensor as input and + outputs several tables, splitting the Tensor along + the specified dimension `dimension`. + + The input to this layer is expected to be a tensor, or a batch of tensors; + when using mini-batch, a batch of sample tensors will be passed to the layer and + the user need to specify the number of dimensions of each sample tensor in a + batch using `nInputDims`. + + :param dimension: to be split along this dimension + :param n_input_dims: specify the number of dimensions that this module will receive + If it is more than the dimension of input tensors, the first dimension + would be considered as batch size + + >>> splitTable = SplitTable(1, 1) + creating: createSplitTable + ''' + + def __init__(self, + dimension, + n_input_dims=-1, + bigdl_type="float"): + super(SplitTable, self).__init__(None, bigdl_type, + dimension, + n_input_dims) + + +class Sqrt(Model): + + ''' + Apply an element-wise sqrt operation. + + >>> sqrt = Sqrt() + creating: createSqrt + ''' + + def __init__(self, + bigdl_type="float"): + super(Sqrt, self).__init__(None, bigdl_type) + + +class Square(Model): + + ''' + Apply an element-wise square operation. + >>> square = Square() + creating: createSquare + ''' + + def __init__(self, + bigdl_type="float"): + super(Square, self).__init__(None, bigdl_type) + + +class Squeeze(Model): + + ''' + Delete singleton all dimensions or a specific dim. + + :param dim Optional. The dimension to be delete. Default: delete all dimensions. + :param num_input_dims Optional. If in a batch model, set to the inputDims. + + + >>> squeeze = Squeeze(1) + creating: createSqueeze + ''' + + def __init__(self, + dim, + num_input_dims=INTMIN, + bigdl_type="float"): + super(Squeeze, self).__init__(None, bigdl_type, + dim, + num_input_dims) + + +class Sum(Model): + + ''' + It is a simple layer which applies a sum operation over the given dimension. + When nInputDims is provided, the input will be considered as a batches. + Then the sum operation will be applied in (dimension + 1) + The input to this layer is expected to be a tensor, or a batch of tensors; + when using mini-batch, a batch of sample tensors will be passed to the layer and + the user need to specify the number of dimensions of each sample tensor in the + batch using `nInputDims`. + + :param dimension the dimension to be applied sum operation + :param n_input_dims specify the number of dimensions that this module will receive + If it is more than the dimension of input tensors, the first dimension + would be considered as batch size + :param size_average default is false, if it is true, it will return the mean instead + + >>> sum = Sum(1, 1, True) + creating: createSum + ''' + + def __init__(self, + dimension=1, + n_input_dims=-1, + size_average=False, + bigdl_type="float"): + super(Sum, self).__init__(None, bigdl_type, + dimension, + n_input_dims, + size_average) + + +class TanhShrink(Model): + + ''' + A simple layer for each element of the input tensor, do the following operation + during the forward process: + [f(x) = tanh(x) - 1] + + >>> tanhShrink = TanhShrink() + creating: createTanhShrink + ''' + + def __init__(self, + bigdl_type="float"): + super(TanhShrink, self).__init__(None, bigdl_type) + + +class Threshold(Model): + + ''' + Threshold input Tensor. + If values in the Tensor smaller than th, then replace it with v + + :param th the threshold to compare with + :param v the value to replace with + :param ip inplace mode + + >>> threshold = Threshold(1e-5, 1e-5, True) + creating: createThreshold + ''' + + def __init__(self, + th=1e-6, + v=0.0, + ip=False, + bigdl_type="float"): + super(Threshold, self).__init__(None, bigdl_type, + th, + v, + ip) + + +class Unsqueeze(Model): + + ''' + Create an Unsqueeze layer. Insert singleton dim (i.e., dimension 1) at position pos. + For an input with dim = input.dim(), + there are dim + 1 possible positions to insert the singleton dimension. + + :param pos The position will be insert singleton. + :param num_input_dims Optional. If in a batch model, set to the inputDim + + >>> unsqueeze = Unsqueeze(1, 1) + creating: createUnsqueeze + ''' + + def __init__(self, + pos, + num_input_dims=INTMIN, + bigdl_type="float"): + super(Unsqueeze, self).__init__(None, bigdl_type, + pos, + num_input_dims) + + +class Reshape(Model): + ''' + The forward(input) reshape the input tensor into a size(0) * size(1) * ... tensor, taking the + elements row-wise. + + :param size: the reshape size + + >>> reshape = Reshape([1, 28, 28]) + creating: createReshape + >>> reshape = Reshape([1, 28, 28], False) + creating: createReshape + ''' + + def __init__(self, size, batch_mode=None, bigdl_type="float"): + super(Reshape, self).__init__(None, bigdl_type, size, batch_mode) + + +class BiRecurrent(Container): + ''' + Create a Bidirectional recurrent layer + + :param merge: merge layer + + >>> biRecurrent = BiRecurrent(CAddTable()) + creating: createCAddTable + creating: createBiRecurrent + >>> biRecurrent = BiRecurrent() + creating: createBiRecurrent + ''' + + def __init__(self, + merge=None, + bigdl_type="float"): + super(BiRecurrent, self).__init__(None, bigdl_type, merge) + + +class ConcatTable(Container): + ''' + ConcateTable is a container module like Concate. Applies an input + to each member module, input can be a tensor or a table. + + ConcateTable usually works with CAddTable and CMulTable to + implement element wise add/multiply on outputs of two modules. + + >>> concatTable = ConcatTable() + creating: createConcatTable + ''' + + def __init__(self, + bigdl_type="float"): + super(ConcatTable, self).__init__(None, bigdl_type) + + +class CriterionTable(Model): + ''' + Creates a module that wraps a Criterion so that it can accept a table of inputs. + + :param criterion Criterion module + + >>> from bigdl.nn.criterion import MSECriterion + >>> criterionTable = CriterionTable(MSECriterion()) + creating: createMSECriterion + creating: createCriterionTable + ''' + + def __init__(self, + criterion, + bigdl_type="float"): + super(CriterionTable, self).__init__(None, bigdl_type, + criterion) + + +class Identity(Model): + ''' + Identity just return the input to output. + It's useful in same parallel container to get an origin input. + + >>> identity = Identity() + creating: createIdentity + ''' + + def __init__(self, + bigdl_type="float"): + super(Identity, self).__init__(None, bigdl_type) + + +class Reverse(Model): + ''' + Reverse the input w.r.t given dimension. + The input can be a Tensor or Table. + + :param dim + + >>> reverse = Reverse() + creating: createReverse + ''' + + def __init__(self, + dimension=1, + bigdl_type="float"): + super(Reverse, self).__init__(None, bigdl_type, + dimension) + + +class Transpose(Model): + ''' + Transpose input along specified dimensions + + :param permutations dimension pairs that need to swap + + >>> transpose = Transpose([(1,2)]) + creating: createTranspose + ''' + + def __init__(self, + permutations, + bigdl_type="float"): + super(Transpose, self).__init__(None, bigdl_type, + permutations) + + +class SpatialContrastiveNormalization(Model): + ''' + Subtractive + divisive contrast normalization. + + :param n_input_plane + :param kernel + :param threshold + :param thresval + + >>> kernel = np.ones([9,9]).astype("float32") + >>> spatialContrastiveNormalization = SpatialContrastiveNormalization(1, kernel) + creating: createSpatialContrastiveNormalization + >>> spatialContrastiveNormalization = SpatialContrastiveNormalization() + creating: createSpatialContrastiveNormalization + ''' + + def __init__(self, + n_input_plane=1, + kernel=None, + threshold=1e-4, + thresval=1e-4, + bigdl_type="float"): + super(SpatialContrastiveNormalization, self).__init__(None, bigdl_type, + n_input_plane, + JTensor.from_ndarray(kernel), + threshold, + thresval) + + +class SpatialConvolutionMap(Model): + ''' + This class is a generalization of SpatialConvolution. + It uses a generic connection table between input and output features. + The SpatialConvolution is equivalent to using a full connection table. + + >>> ct = np.ones([9,9]).astype("float32") + >>> spatialConvolutionMap = SpatialConvolutionMap(ct, 9, 9) + creating: createSpatialConvolutionMap + ''' + + def __init__(self, + conn_table, + kw, + kh, + dw=1, + dh=1, + pad_w=0, + pad_h=0, + bigdl_type="float"): + super(SpatialConvolutionMap, self).__init__(None, bigdl_type, + JTensor.from_ndarray(conn_table), + kw, + kh, + dw, + dh, + pad_w, + pad_h) + + +class SpatialDivisiveNormalization(Model): + ''' + Applies a spatial division operation on a series of 2D inputs using kernel for + computing the weighted average in a neighborhood. The neighborhood is defined for + a local spatial region that is the size as kernel and across all features. For + an input image, since there is only one feature, the region is only spatial. For + an RGB image, the weighted average is taken over RGB channels and a spatial region. + + If the kernel is 1D, then it will be used for constructing and separable 2D kernel. + The operations will be much more efficient in this case. + + The kernel is generally chosen as a gaussian when it is believed that the correlation + of two pixel locations decrease with increasing distance. On the feature dimension, + a uniform average is used since the weighting across features is not known. + + + :param nInputPlane number of input plane, default is 1. + :param kernel kernel tensor, default is a 9 x 9 tensor. + :param threshold threshold + :param thresval threshhold value to replace with + if data is smaller than theshold + + >>> kernel = np.ones([9,9]).astype("float32") + >>> spatialDivisiveNormalization = SpatialDivisiveNormalization(2,kernel) + creating: createSpatialDivisiveNormalization + >>> spatialDivisiveNormalization = SpatialDivisiveNormalization() + creating: createSpatialDivisiveNormalization + ''' + + def __init__(self, + n_input_plane=1, + kernel=None, + threshold=1e-4, + thresval=1e-4, + bigdl_type="float"): + super(SpatialDivisiveNormalization, self).__init__(None, bigdl_type, + n_input_plane, + JTensor.from_ndarray(kernel), + threshold, + thresval) + +class Graph(Model): + ''' + A graph container. Each node can have multiple inputs. The output of the node should be a + tensor. The output tensor can be connected to multiple nodes. So the module in each node can + have a tensor or table input, and should have a tensor output. + + The graph container can have multiple inputs and multiple outputs. If there's one input, + the input data fed to the graph module should be a tensor. If there're multiple inputs, + the input data fed to the graph module should be a table, which is actually an sequence of + tensor. The order of the input tensors should be same with the order of the input nodes. + This is also applied to the gradient from the module in the back propagation. + + If there's one output, the module output is a tensor. If there're multiple outputs, the module + output is a table, which is actually an sequence of tensor. The order of the output tensors is + same with the order of the output modules. This is also applied to the gradient passed to the + module in the back propagation. + + All inputs should be able to connect to outputs through some paths in the graph. + It is allowed that some successors of the inputs node are not connect to outputs. + If so, these nodes will be excluded in the computation. + ''' + + def __init__(self, + input, + output, + bigdl_type="float"): + super(Graph, self).__init__(None, bigdl_type, input, output) + +class SpatialSubtractiveNormalization(Model): + ''' + Applies a spatial subtraction operation on a series of 2D inputs using kernel for + computing the weighted average in a neighborhood. The neighborhood is defined for + a local spatial region that is the size as kernel and across all features. For a + an input image, since there is only one feature, the region is only spatial. For + an RGB image, the weighted average is taken over RGB channels and a spatial region. + + If the kernel is 1D, then it will be used for constructing and separable 2D kernel. + The operations will be much more efficient in this case. + + The kernel is generally chosen as a gaussian when it is believed that the correlation + of two pixel locations decrease with increasing distance. On the feature dimension, + a uniform average is used since the weighting across features is not known. + + :param n_input_plane number of input plane, default is 1. + :param kernel kernel tensor, default is a 9 x 9 tensor. + + >>> kernel = np.ones([9,9]).astype("float32") + >>> spatialSubtractiveNormalization = SpatialSubtractiveNormalization(2,kernel) + creating: createSpatialSubtractiveNormalization + >>> spatialSubtractiveNormalization = SpatialSubtractiveNormalization() + creating: createSpatialSubtractiveNormalization + ''' + + def __init__(self, + n_input_plane=1, + kernel=None, + bigdl_type="float"): + super(SpatialSubtractiveNormalization, self).__init__(None, bigdl_type, + n_input_plane, + JTensor.from_ndarray(kernel)) + + +def _test(): + import doctest + from pyspark import SparkContext + from bigdl.nn import layer + from bigdl.util.common import init_engine + from bigdl.util.common import create_spark_conf + globs = layer.__dict__.copy() + sc = SparkContext(master="local[4]", appName="test layer", + conf=create_spark_conf()) + globs['sc'] = sc + init_engine() + + (failure_count, test_count) = doctest.testmod(globs=globs, + optionflags=doctest.ELLIPSIS) + if failure_count: + exit(-1) + +if __name__ == "__main__": + _test() diff --git a/python/dllib/src/bigdl/dllib/optim/__init__.py b/python/dllib/src/bigdl/dllib/optim/__init__.py new file mode 100644 index 00000000000..2151a805423 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/optim/__init__.py @@ -0,0 +1,15 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/python/dllib/src/bigdl/dllib/optim/optimizer.py b/python/dllib/src/bigdl/dllib/optim/optimizer.py new file mode 100644 index 00000000000..e868c1473bd --- /dev/null +++ b/python/dllib/src/bigdl/dllib/optim/optimizer.py @@ -0,0 +1,556 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +import os +import sys +from distutils.dir_util import mkpath + +from bigdl.nn.layer import DOUBLEMAX +from bigdl.util.common import JTensor +from bigdl.util.common import JavaValue +from bigdl.util.common import callBigDlFunc +from bigdl.util.common import callJavaFunc +from bigdl.util.common import get_spark_context + +if sys.version >= '3': + long = int + unicode = str + + +class MaxIteration(JavaValue): + """ + A trigger specifies a timespot or several timespots during training, + and a corresponding action will be taken when the timespot(s) is reached. + MaxIteration is a trigger that triggers an action when training reaches + the number of iterations specified by "max". + Usually used as end_trigger when creating an Optimizer. + + >>> maxIteration = MaxIteration(20) + creating: createMaxIteration + """ + def __init__(self, max, bigdl_type="float"): + """ + Create a MaxIteration trigger. + + :param max: max + """ + JavaValue.__init__(self, None, bigdl_type, max) + + +class MaxEpoch(JavaValue): + """ + A trigger specifies a timespot or several timespots during training, + and a corresponding action will be taken when the timespot(s) is reached. + MaxEpoch is a trigger that triggers an action when training reaches + the number of epochs specified by "max_epoch". + Usually used as end_trigger when creating an Optimizer. + + >>> maxEpoch = MaxEpoch(2) + creating: createMaxEpoch + """ + def __init__(self, max_epoch, bigdl_type="float"): + """ + Create a MaxEpoch trigger. + + :param max_epoch: max_epoch + """ + JavaValue.__init__(self, None, bigdl_type, max_epoch) + + +class EveryEpoch(JavaValue): + """ + A trigger specifies a timespot or several timespots during training, + and a corresponding action will be taken when the timespot(s) is reached. + EveryEpoch is a trigger that triggers an action when each epoch finishs. + Could be used as trigger in setvalidation and setcheckpoint in Optimizer, + and also in TrainSummary.set_summary_trigger. + + >>> everyEpoch = EveryEpoch() + creating: createEveryEpoch + """ + def __init__(self, bigdl_type="float"): + """ + Create a EveryEpoch trigger. + """ + JavaValue.__init__(self, None, bigdl_type) + + +class SeveralIteration(JavaValue): + """ + A trigger specifies a timespot or several timespots during training, + and a corresponding action will be taken when the timespot(s) is reached. + SeveralIteration is a trigger that triggers an action every "n" + iterations. + Could be used as trigger in setvalidation and setcheckpoint in Optimizer, + and also in TrainSummary.set_summary_trigger. + + >>> serveralIteration = SeveralIteration(2) + creating: createSeveralIteration + """ + def __init__(self, interval, bigdl_type="float"): + """ + Create a SeveralIteration trigger. + + :param interval: interval is the "n" where an action is triggered + every "n" iterations + """ + JavaValue.__init__(self, None, bigdl_type, interval) + + +class Poly(JavaValue): + """ + A learning rate decay policy, where the effective learning rate + follows a polynomial decay, to be zero by the max_iteration. + Calculation: base_lr (1 - iter/max_iteration) ^ (power) + + :param power + :param max_iteration + + >>> poly = Poly(0.5, 2) + creating: createPoly + """ + def __init__(self, power, max_iteration, bigdl_type="float"): + JavaValue.__init__(self, None, bigdl_type, power, max_iteration) + + +class Step(JavaValue): + """ + A learning rate decay policy, where the effective learning rate is + calculated as base_lr * gamma ^ (floor(iter / step_size)) + + :param step_size + :param gamma + + >>> step = Step(2, 0.3) + creating: createStep + """ + def __init__(self, step_size, gamma, bigdl_type="float"): + JavaValue.__init__(self, None, bigdl_type, step_size, gamma) + +class Default(JavaValue): + """ + A learning rate decay policy, where the effective learning rate is + calculated as base_lr * gamma ^ (floor(iter / step_size)) + + :param step_size + :param gamma + + >>> step = Default() + creating: createDefault + """ + def __init__(self, bigdl_type="float"): + JavaValue.__init__(self, None, bigdl_type) + +class SGD(JavaValue): + """ + A plain implementation of SGD + + :param learningrate learning rate + :param learningrate_decay learning rate decay + :param weightdecay weight decay + :param momentum momentum + :param dampening dampening for momentum + :param nesterov enables Nesterov momentum + :param learningrates 1D tensor of individual learning rates + :param weightdecays 1D tensor of individual weight decays + >>> sgd = SGD() + creating: createDefault + creating: createSGD + """ + def __init__(self, + learningrate=1e-3, + learningrate_decay=0.0, + weightdecay=0.0, + momentum=0.0, + dampening=DOUBLEMAX, + nesterov=False, + leaningrate_schedule=None, + learningrates=None, + weightdecays=None, + bigdl_type="float"): + JavaValue.__init__(self, None, bigdl_type, learningrate, learningrate_decay, weightdecay, + momentum, dampening, nesterov, + leaningrate_schedule if (leaningrate_schedule) else Default(), + JTensor.from_ndarray(learningrates), JTensor.from_ndarray(weightdecays)) + +class Adagrad(JavaValue): + """ + An implementation of Adagrad. See the original paper: + http://jmlr.org/papers/volume12/duchi11a/duchi11a.pdf + + :param learningrate learning rate + :param learningrate_decay learning rate decay + :param weightdecay weight decay + >>> adagrad = Adagrad() + creating: createAdagrad + """ + def __init__(self, + learningrate=1e-3, + learningrate_decay=0.0, + weightdecay=0.0, + bigdl_type="float"): + JavaValue.__init__(self, None, bigdl_type, learningrate, learningrate_decay, weightdecay) + +class LBFGS(JavaValue): + """ + This implementation of L-BFGS relies on a user-provided line + search function (state.lineSearch). If this function is not + provided, then a simple learningRate is used to produce fixed + size steps. Fixed size steps are much less costly than line + searches, and can be useful for stochastic problems. + The learning rate is used even when a line search is provided. + This is also useful for large-scale stochastic problems, where + opfunc is a noisy approximation of f(x). In that case, the learning + rate allows a reduction of confidence in the step size. + + :param max_iter Maximum number of iterations allowed + :param max_eval Maximum number of function evaluations + :param tolfun Termination tolerance on the first-order optimality + :param tolx Termination tol on progress in terms of func/param changes + :param ncorrection + :param learningrate + :param verbose + :param linesearch A line search function + :param linesearch_options If no line search provided, then a fixed step size is used + >>> lbfgs = LBFGS() + creating: createLBFGS + """ + def __init__(self, + max_iter=20, + max_eval=DOUBLEMAX, + tolfun=1e-5, + tolx=1e-9, + ncorrection=100, + learningrate=1.0, + verbose=False, + linesearch=None, + linesearch_options=None, + bigdl_type="float"): + if linesearch or linesearch_options: + raise ValueError('linesearch and linesearch_options must be None in LBFGS') + JavaValue.__init__(self, None, bigdl_type, max_iter, max_eval, tolfun, tolx, + ncorrection, learningrate, verbose, linesearch, linesearch_options) + +class Adadelta(JavaValue): + """ + Adadelta implementation for SGD: http://arxiv.org/abs/1212.5701 + + :param decayrate interpolation parameter rho + :param epsilon for numerical stability + >>> adagrad = Adadelta() + creating: createAdadelta + """ + def __init__(self, + decayrate = 0.9, + epsilon = 1e-10, + bigdl_type="float"): + JavaValue.__init__(self, None, bigdl_type, decayrate, epsilon) + +class Adam(JavaValue): + """ + An implementation of Adam http://arxiv.org/pdf/1412.6980.pdf + :param learningrate learning rate + :param learningrate_decay learning rate decay + :param beta1 first moment coefficient + :param beta2 second moment coefficient + :param epsilon for numerical stability + >>> adagrad = Adam() + creating: createAdam + """ + def __init__(self, + learningrate = 1e-3, + learningrate_decay = 0.0, + beta1 = 0.9, + beta2 = 0.999, + epsilon = 1e-8, + bigdl_type="float"): + JavaValue.__init__(self, None, bigdl_type, learningrate, learningrate_decay, + beta1, beta2, epsilon) + +class Adamax(JavaValue): + """ + An implementation of Adamax http://arxiv.org/pdf/1412.6980.pdf + :param learningrate learning rate + :param beta1 first moment coefficient + :param beta2 second moment coefficient + :param epsilon for numerical stability + >>> adagrad = Adamax() + creating: createAdamax + """ + def __init__(self, + learningrate = 0.002, + beta1 = 0.9, + beta2 = 0.999, + epsilon = 1e-38, + bigdl_type="float"): + JavaValue.__init__(self, None, bigdl_type, learningrate, beta1, beta2, epsilon) + +class RMSprop(JavaValue): + """ + An implementation of RMSprop + :param learningrate learning rate + :param learningrate_decay learning rate decay + :param decayrate decay rate, also called rho + :param epsilon for numerical stability + >>> adagrad = RMSprop() + creating: createRMSprop + """ + def __init__(self, + learningrate = 1e-2, + learningrate_decay = 0.0, + decayrate = 0.99, + epsilon = 1e-8, + bigdl_type="float"): + JavaValue.__init__(self, None, bigdl_type, learningrate, learningrate_decay, decayrate, epsilon) + +class MultiStep(JavaValue): + """ + similar to step but it allows non uniform steps defined by stepSizes + + :param step_size the series of step sizes used for lr decay + :param gamma coefficient of decay + + >>> step = MultiStep([2, 5], 0.3) + creating: createMultiStep + """ + def __init__(self, step_sizes, gamma, bigdl_type="float"): + JavaValue.__init__(self, None, bigdl_type, step_sizes, gamma) + + +class Optimizer(JavaValue): + """ + An optimizer is in general to minimize any function with respect + to a set of parameters. In case of training a neural network, + an optimizer tries to minimize the loss of the neural net with + respect to its weights/biases, over the training set. + """ + def __init__(self, + model, + training_rdd, + criterion, + end_trigger, + batch_size, + optim_method=None, + bigdl_type="float"): + """ + Create an optimizer. + + :param model: the neural net model + :param traiing_rdd: the training dataset + :param criterion: the loss function + :param optim_method: the algorithm to use for optimization, + e.g. SGD, Adagrad, etc. If optim_method is None, the default algorithm is SGD. + :param end_trigger: when to end the optimization + :param batch_size: training batch size + """ + JavaValue.__init__(self, None, bigdl_type, model.value, + training_rdd, criterion, + optim_method if optim_method else SGD(), end_trigger, batch_size) + + def set_validation(self, batch_size, val_rdd, trigger, val_method=["Top1Accuracy"]): + """ + Configure validation settings. + + :param batch_size: validation batch size + :param val_rdd: validation dataset + :param trigger: validation interval + :param val_method: the ValidationMethod to use, + e.g. "Top1Accuracy", "Top5Accuracy", "Loss" + """ + callBigDlFunc(self.bigdl_type, "setValidation", self.value, batch_size, + trigger, val_rdd, val_method) + + def set_model(self, model): + """ + Set model. + + :param model: new model + """ + self.value.setModel(model.value) + + def set_checkpoint(self, checkpoint_trigger, + checkpoint_path, isOverWrite=True): + """ + Configure checkpoint settings. + + :param checkpoint_trigger: the interval to write snapshots + :param checkpoint_path: the path to write snapshots into + :param isOverWrite: whether to overwrite existing snapshots in path. + default is True + """ + if not os.path.exists(checkpoint_path): + mkpath(checkpoint_path) + callBigDlFunc(self.bigdl_type, "setCheckPoint", self.value, + checkpoint_trigger, checkpoint_path, isOverWrite) + + # return a module + def optimize(self): + """ + Do an optimization. + """ + jmodel = callJavaFunc(get_spark_context(), self.value.optimize) + from bigdl.nn.layer import Model + return Model.of(jmodel) + + def set_train_summary(self, summary): + """ + Set train summary. A TrainSummary object contains information + necesary for the optimizer to know how often the logs are recorded, + where to store the logs and how to retrieve them, etc. For details, + refer to the docs of TrainSummary. + + :param summary: a TrainSummary object + """ + callBigDlFunc(self.bigdl_type, "setTrainSummary", self.value, + summary) + return self + + def set_val_summary(self, summary): + """ + Set validation summary. A ValidationSummary object contains information + necesary for the optimizer to know how often the logs are recorded, + where to store the logs and how to retrieve them, etc. For details, + refer to the docs of ValidationSummary. + + :param summary: a ValidationSummary object + + """ + callBigDlFunc(self.bigdl_type, "setValSummary", self.value, + summary) + return self + + def prepare_input(self): + """ + Load input. Notebook user can call this method to seprate load data and + create optimizer time + """ + print("Loading input ...") + self.value.prepareInput() + + +class TrainSummary(JavaValue, ): + """ + A logging facility which allows user to trace how indicators (e.g. + learning rate, training loss, throughput, etc.) change with iterations/time + in an optimization process. TrainSummary is for training indicators only + (check ValidationSummary for validation indicators). It contains necessary + information for the optimizer to know where to store the logs, how to + retrieve the logs, and so on. - The logs are written in tensorflow-compatible + format so that they can be visualized directly using tensorboard. Also the + logs can be retrieved as ndarrays and visualized using python libraries + such as matplotlib (in notebook, etc.). + + Use optimizer.setTrainSummary to enable train logger. + """ + def __init__(self, log_dir, app_name, bigdl_type="float"): + """ + Create a TrainSummary. Logs will be saved to log_dir/app_name/train. + + :param log_dir: the root dir to store the logs + :param app_name: the application name + """ + JavaValue.__init__(self, None, bigdl_type, log_dir, app_name) + + def read_scalar(self, tag): + """ + Retrieve train logs by type. Return an array of records in the format + (step,value,wallClockTime). - "Step" is the iteration count by default. + + :param tag: the type of the logs, Supported tags are: "LearningRate", + "Loss", "Throughput" + """ + return callBigDlFunc(self.bigdl_type, "summaryReadScalar", self.value, + tag) + + def set_summary_trigger(self, name, trigger): + """ + Set the interval of recording for each indicator. + + :param tag: tag name. Supported tag names are "LearningRate", "Loss", + "Throughput", "Parameters". "Parameters" is an umbrella tag that + includes weight, bias, gradWeight, gradBias, and some running status + (eg. runningMean and runningVar in BatchNormalization). If you + didn't set any triggers, we will by default record Loss and Throughput + in each iteration, while *NOT* recording LearningRate and Parameters, + as recording parameters may introduce substantial overhead when the + model is very big, LearningRate is not a public attribute for all + OptimMethod. + :param trigger: trigger + """ + return callBigDlFunc(self.bigdl_type, "summarySetTrigger", self.value, + name, trigger) + + +class ValidationSummary(JavaValue): + """ + A logging facility which allows user to trace how indicators (e.g. + validation loss, top1 accuray, top5 accuracy etc.) change with + iterations/time in an optimization process. ValidationSummary is for + validation indicators only (check TrainSummary for train indicators). + It contains necessary information for the optimizer to know where to + store the logs, how to retrieve the logs, and so on. - The logs are + written in tensorflow-compatible format so that they can be visualized + directly using tensorboard. Also the logs can be retrieved as ndarrays + and visualized using python libraries such as matplotlib + (in notebook, etc.). + + Use optimizer.setValidationSummary to enable validation logger. + """ + def __init__(self, log_dir, app_name, bigdl_type="float"): + """ + Create a ValidationSummary. Logs will be saved to + log_dir/app_name/train. By default, all ValidationMethod set into + optimizer will be recorded and the recording interval is the same + as trigger of ValidationMethod in the optimizer. + + :param log_dir: the root dir to store the logs + :param app_name: the application name + """ + JavaValue.__init__(self, None, bigdl_type, log_dir, app_name) + + def read_scalar(self, tag): + """ + Retrieve validation logs by type. Return an array of records in the + format (step,value,wallClockTime). - "Step" is the iteration count + by default. + + :param tag: the type of the logs. The tag should match the name of + the ValidationMethod set into the optimizer. e.g. + "Top1AccuracyLoss","Top1Accuracy" or "Top5Accuracy". + """ + return callBigDlFunc(self.bigdl_type, "summaryReadScalar", self.value, + tag) + + +def _test(): + import doctest + from pyspark import SparkContext + from bigdl.optim import optimizer + from bigdl.util.common import init_engine + from bigdl.util.common import create_spark_conf + globs = optimizer.__dict__.copy() + sc = SparkContext(master="local[4]", appName="test optimizer", + conf=create_spark_conf()) + init_engine() + globs['sc'] = sc + (failure_count, test_count) = doctest.testmod(globs=globs, + optionflags=doctest.ELLIPSIS) + if failure_count: + exit(-1) + +if __name__ == "__main__": + _test() diff --git a/python/dllib/src/bigdl/utils/__init__.py b/python/dllib/src/bigdl/utils/__init__.py new file mode 100644 index 00000000000..2151a805423 --- /dev/null +++ b/python/dllib/src/bigdl/utils/__init__.py @@ -0,0 +1,15 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/python/dllib/src/bigdl/utils/common.py b/python/dllib/src/bigdl/utils/common.py new file mode 100644 index 00000000000..1839a62d8b8 --- /dev/null +++ b/python/dllib/src/bigdl/utils/common.py @@ -0,0 +1,390 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import sys +from py4j.protocol import Py4JJavaError +from py4j.java_gateway import JavaObject +from py4j.java_collections import ListConverter, JavaArray, JavaList, JavaMap + +from pyspark import RDD, SparkContext +from pyspark.serializers import PickleSerializer, AutoBatchedSerializer +from pyspark.sql import DataFrame, SQLContext +from pyspark.mllib.common import callJavaFunc +from pyspark import SparkConf +import numpy as np +import threading + +if sys.version >= '3': + long = int + unicode = str + +class SingletonMixin(object): + _lock = threading.RLock() + _instance = None + + @classmethod + def instance(cls, + bigdl_type="float"): + if not cls._instance: + with cls._lock: + if not cls._instance: + cls._instance = cls(bigdl_type) + return cls._instance + +class JavaCreator(SingletonMixin): + __creator_class="com.intel.analytics.bigdl.python.api.PythonBigDL" + + @classmethod + def get_creator_class(cls): + with JavaCreator._lock: + return JavaCreator.__creator_class + + @classmethod + def set_creator_class(cls, cclass): + with JavaCreator._lock: + JavaCreator.__creator_class = cclass + JavaCreator._instance = None + + def __init__(self, bigdl_type): + sc = get_spark_context() + jclass = getattr(sc._jvm, JavaCreator.get_creator_class()) + if bigdl_type == "float": + self.value = getattr(jclass, "ofFloat")() + elif bigdl_type == "double": + self.value = getattr(jclass, "ofDouble")() + else: + raise Exception("Not supported bigdl_type: %s" % bigdl_type) + + +class JavaValue(object): + def jvm_class_constructor(self): + name = "create" + self.__class__.__name__ + print("creating: " + name) + return name + + def __init__(self, jvalue, bigdl_type, *args): + self.value = jvalue if jvalue else callBigDlFunc( + bigdl_type, self.jvm_class_constructor(), *args) + self.bigdl_type = bigdl_type + + def __str__(self): + return self.value.toString() + + +class TestResult(): + """ + A testing result used to benchmark the model quality. + """ + def __init__(self, result, total_num, method): + """ + :param result: the validation result. i.e: top1 accuracy percentage. + :param total_num: the total processed records. + :param method: the validation method. i.e: Top1Accuracy + """ + self.result = result + self.total_num = total_num + self.method = method + + def __reduce__(self): + return (TestResult, (self.result, self.total_num, self.method)) + + def __str__(self): + return "Test result: %s, total_num: %s, method: %s" % ( + self.result, self.total_num, self.method) + + +class JTensor(object): + """ + A wrapper to easy our work when need to pass or return Tensor to/from Scala. + + >>> import numpy as np + >>> from bigdl.util.common import JTensor + >>> np.random.seed(123) + >>> + """ + def __init__(self, storage, shape, bigdl_type="float"): + self.storage = storage + self.shape = shape + self.bigdl_type = bigdl_type + + @classmethod + def from_ndarray(cls, a_ndarray, bigdl_type="float"): + """ + Convert a ndarray to Tensor which would be used in Java side. + >>> import numpy as np + >>> from bigdl.util.common import JTensor + >>> from bigdl.util.common import callBigDlFunc + >>> np.random.seed(123) + >>> data = np.random.uniform(0, 1, (2, 3)).astype("float32") + >>> result = JTensor.from_ndarray(data) + >>> data_back = result.to_ndarray() + >>> (data == data_back).all() + True + >>> tensor1 = callBigDlFunc("float", "testTensor", JTensor.from_ndarray(data)) # noqa + >>> array_from_tensor = tensor1.to_ndarray() + >>> (array_from_tensor == data).all() + True + """ + return cls(*JTensor.flatten_ndarray(a_ndarray), + bigdl_type= bigdl_type) if a_ndarray is not None else None # noqa + + def to_ndarray(self): + def get_dtype(): + if "float" == self.bigdl_type: + return "float32" + else: + return "float64" + return np.array(self.storage, dtype=get_dtype()).reshape(self.shape) # noqa + + @classmethod + def flatten_ndarray(cls, a_ndarray): + """ + Utility method to flatten a ndarray + :return (storage, shape) + >>> from bigdl.util.common import JTensor + >>> np.random.seed(123) + >>> data = np.random.uniform(0, 1, (2, 3)) + >>> (storage, shape) = JTensor.flatten_ndarray(data) + >>> shape + [2, 3] + >>> (storage, shape) = JTensor.flatten_ndarray(np.array(2)) + >>> shape + [1] + """ + storage = [float(i) for i in a_ndarray.ravel()] + shape = list(a_ndarray.shape) if a_ndarray.shape else [a_ndarray.size] + return storage, shape + + def __reduce__(self): + return (JTensor, (self.storage, self.shape, self.bigdl_type)) + + def __str__(self): + return "storage: %s, shape: %s," % (self.storage, self.storage) + + +class Sample(object): + def __init__(self, features, label, features_shape, label_shape, + bigdl_type="float"): + def get_dtype(): + if "float" == bigdl_type: + return "float32" + else: + return "float64" + self.features = np.array(features, dtype=get_dtype()).reshape(features_shape) # noqa + self.label = np.array(label, dtype=get_dtype()).reshape(label_shape) + self.bigdl_type = bigdl_type + + @classmethod + def from_ndarray(cls, features, label, bigdl_type="float"): + return cls( + features=[float(i) for i in features.ravel()], + label=[float(i) for i in label.ravel()], + features_shape=list(features.shape), + label_shape=list(label.shape) if label.shape else [label.size], + bigdl_type=bigdl_type) + + def __reduce__(self): + (features_storage, features_shape) = JTensor.flatten_ndarray(self.features) + (label_storage, label_shape) = JTensor.flatten_ndarray(self.label) + return (Sample, ( + features_storage, label_storage, features_shape, label_shape, + self.bigdl_type)) + + def __str__(self): + return "features: %s, label: %s," % (self.features, self.label) + +class RNG(): + """ + generate tensor data with seed + """ + def __init__(self, bigdl_type="float"): + self.bigdl_type = bigdl_type + + def set_seed(self, seed): + callBigDlFunc(self.bigdl_type, "setModelSeed", seed) + + def uniform(self, a, b, size): + return callBigDlFunc(self.bigdl_type, "uniform", a, b, size).to_ndarray() # noqa + + +_picklable_classes = [ + 'LinkedList', + 'SparseVector', + 'DenseVector', + 'DenseMatrix', + 'Rating', + 'LabeledPoint', + 'Sample', + 'TestResult', + 'JTensor' +] + + +def init_engine(bigdl_type="float"): + callBigDlFunc(bigdl_type, "initEngine") + + +def get_bigdl_conf(): + bigdl_conf_file = "spark-bigdl.conf" + bigdl_python_wrapper = "python-api.zip" + + def load_conf(conf_str): + return dict(line.split() for line in conf_str.split("\n") if + "#" not in line and line.strip()) + + for p in sys.path: + if bigdl_conf_file in p: + with open(p) if sys.version_info < (3,) else open(p, encoding='latin-1') as conf_file: # noqa + return load_conf(conf_file.read()) + if bigdl_python_wrapper in p: + import zipfile + with zipfile.ZipFile(p, 'r') as zip_conf: + content = zip_conf.read(bigdl_conf_file) + if sys.version_info >= (3,): + content = str(content, 'latin-1') + return load_conf(content) + raise Exception("Cannot find spark-bigdl.conf.Pls add it to PYTHONPATH.") + + +def to_list(a): + if type(a) is list: + return a + return [a] + + +def create_spark_conf(): + bigdl_conf = get_bigdl_conf() + sparkConf = SparkConf() + sparkConf.setAll(bigdl_conf.items()) + return sparkConf + + +def get_spark_context(): + if "getOrCreate" in SparkContext.__dict__: + return SparkContext.getOrCreate() + else: + with SparkContext._lock: # Compatible with Spark1.5.1 + if SparkContext._active_spark_context is None: + SparkContext(SparkConf()) + return SparkContext._active_spark_context + + +def get_spark_sql_context(sc): + if "getOrCreate" in SQLContext.__dict__: + return SQLContext.getOrCreate() + else: + return SQLContext(sc) # Compatible with Spark1.5.1 + +def callBigDlFunc(bigdl_type, name, *args): + """ Call API in PythonBigDL """ + jinstance = JavaCreator.instance(bigdl_type=bigdl_type).value + sc = get_spark_context() + api = getattr(jinstance, name) + return callJavaFunc(sc, api, *args) + + +def _java2py(sc, r, encoding="bytes"): + if isinstance(r, JavaObject): + clsName = r.getClass().getSimpleName() + # convert RDD into JavaRDD + if clsName != 'JavaRDD' and clsName.endswith("RDD"): + r = r.toJavaRDD() + clsName = 'JavaRDD' + + if clsName == 'JavaRDD': + jrdd = sc._jvm.SerDe.javaToPython(r) + return RDD(jrdd, sc) + + if clsName == 'DataFrame': + return DataFrame(r, get_spark_sql_context(sc)) + + if clsName in _picklable_classes: + r = sc._jvm.org.apache.spark.bigdl.api.python.BigDLSerDe.dumps(r) + elif isinstance(r, (JavaArray, JavaList, JavaMap)): + try: + r = sc._jvm.org.apache.spark.bigdl.api.python.BigDLSerDe.dumps( + r) + except Py4JJavaError: + pass # not pickable + + if isinstance(r, (bytearray, bytes)): + r = PickleSerializer().loads(bytes(r), encoding=encoding) + return r + + +def callJavaFunc(sc, func, *args): + """ Call Java Function """ + args = [_py2java(sc, a) for a in args] + result = func(*args) + return _java2py(sc, result) + + +def _to_java_object_rdd(rdd): + """ Return a JavaRDD of Object by unpickling + + It will convert each Python object into Java object by Pyrolite, whenever + the RDD is serialized in batch or not. + """ + rdd = rdd._reserialize(AutoBatchedSerializer(PickleSerializer())) + return \ + rdd.ctx._jvm.org.apache.spark.bigdl.api.python.BigDLSerDe.pythonToJava( + rdd._jrdd, True) + + +def _py2java(sc, obj): + """ Convert Python object into Java """ + if isinstance(obj, RDD): + obj = _to_java_object_rdd(obj) + elif isinstance(obj, DataFrame): + obj = obj._jdf + elif isinstance(obj, SparkContext): + obj = obj._jsc + elif isinstance(obj, (list, tuple)): + obj = ListConverter().convert([_py2java(sc, x) for x in obj], + sc._gateway._gateway_client) + elif isinstance(obj, dict): + result = {} + for (key, value) in obj.items(): + result[key] = _py2java(sc, value) if isinstance(value, JavaValue) else value # noqa + obj = result + + elif isinstance(obj, JavaValue): + obj = obj.value + elif isinstance(obj, JavaObject): + pass + elif isinstance(obj, (int, long, float, bool, bytes, unicode)): + pass + else: + data = bytearray(PickleSerializer().dumps(obj)) + obj = sc._jvm.org.apache.spark.bigdl.api.python.BigDLSerDe.loads(data) + return obj + + +def _test(): + import doctest + from pyspark import SparkContext + from bigdl.nn import layer + globs = layer.__dict__.copy() + sc = SparkContext(master="local[2]", appName="test common utility") + globs['sc'] = sc + (failure_count, test_count) = doctest.testmod(globs=globs, + optionflags=doctest.ELLIPSIS) + if failure_count: + exit(-1) + + +if __name__ == "__main__": + _test() diff --git a/python/dllib/test/__init__.py b/python/dllib/test/__init__.py new file mode 100644 index 00000000000..2151a805423 --- /dev/null +++ b/python/dllib/test/__init__.py @@ -0,0 +1,15 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# From c790db3b9f3f32fdd3695136d41165099f53fe40 Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Tue, 6 Jun 2017 16:15:49 +0800 Subject: [PATCH 031/823] Add bigdl as the top module (#973) * add bigdl as the top modules * back to test Former-commit-id: ae6938d2b3cb1413c213a07179aa957ad76b3d5c --- load_caffe_test.py | 6 +++--- python/dllib/test/dev/diff.py | 2 +- python/dllib/test/dev/modules.py | 8 ++++---- python/dllib/test/dev/run-tests | 2 +- simple_integration_test.py | 14 +++++++------- 5 files changed, 16 insertions(+), 16 deletions(-) diff --git a/load_caffe_test.py b/load_caffe_test.py index 808dabf9337..6ac29171549 100644 --- a/load_caffe_test.py +++ b/load_caffe_test.py @@ -15,9 +15,9 @@ # # Still in experimental stage! -from nn.layer import * -from optim.optimizer import * -from util.common import * +from bigdl.nn.layer import * +from bigdl.optim.optimizer import * +from bigdl.util.common import * import numpy as np import unittest diff --git a/python/dllib/test/dev/diff.py b/python/dllib/test/dev/diff.py index aeeeefbac81..0f17e8c5fef 100755 --- a/python/dllib/test/dev/diff.py +++ b/python/dllib/test/dev/diff.py @@ -22,7 +22,7 @@ from os.path import isfile, join scala_nn_root = "./spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/" -python_nn_root = "./pyspark/dl/nn/" +python_nn_root = "./pyspark/bigdl/nn/" def extract_scala_class(class_path): diff --git a/python/dllib/test/dev/modules.py b/python/dllib/test/dev/modules.py index d6e3410711e..42849dab800 100644 --- a/python/dllib/test/dev/modules.py +++ b/python/dllib/test/dev/modules.py @@ -55,25 +55,25 @@ def __hash__(self): bigdl_layer = Module( name="bigdl_layer", python_test_goals=[ - "nn.layer" + "bigdl.nn.layer" ]) bigdl_layer = Module( name="bigdl_criterion", python_test_goals=[ - "nn.criterion" + "bigdl.nn.criterion" ]) bigdl_layer = Module( name="bigdl_common", python_test_goals=[ - "util.common" + "bigdl.util.common" ]) bigdl_optimizer = Module( name="bigdl_optimizer", python_test_goals=[ - "optim.optimizer", + "bigdl.optim.optimizer", ] ) diff --git a/python/dllib/test/dev/run-tests b/python/dllib/test/dev/run-tests index 43ce811c134..f812eacb8c5 100755 --- a/python/dllib/test/dev/run-tests +++ b/python/dllib/test/dev/run-tests @@ -30,7 +30,7 @@ if [ -z ${SPARK_HOME+x} ]; then echo "SPARK_HOME is unset"; exit 1; else echo "S PYSPARK_ZIP=`find $SPARK_HOME/python/lib -type f -iname '*.zip' | tr "\n" ":"` -export PYTHONPATH=$PYSPARK_ZIP:$DL_PYTHON_HOME:$DL_PYTHON_HOME/dl:$DL_PYTHON_HOME/test/dev:$BIGDL_HOME/spark/dl/src/main/resources/spark-bigdl.conf +export PYTHONPATH=$PYSPARK_ZIP:$DL_PYTHON_HOME:$DL_PYTHON_HOME/:$DL_PYTHON_HOME/test/dev:$BIGDL_HOME/spark/dl/src/main/resources/spark-bigdl.conf export SPARK_CLASSPATH=$BIGDL_HOME/spark/dl/target/bigdl-$BIGDL_VERSION-jar-with-dependencies.jar diff --git a/simple_integration_test.py b/simple_integration_test.py index 14692c728a9..645ea6a8b0a 100644 --- a/simple_integration_test.py +++ b/simple_integration_test.py @@ -15,10 +15,10 @@ # # Still in experimental stage! -from nn.layer import * -from nn.criterion import * -from optim.optimizer import * -from util.common import * +from bigdl.nn.layer import * +from bigdl.nn.criterion import * +from bigdl.optim.optimizer import * +from bigdl.util.common import * import numpy as np import unittest import tempfile @@ -69,7 +69,7 @@ def test_graph_backward(self): np.array([6.0, 6.0, 6.0, 6.0]))) def test_load_zip_conf(self): - from util.common import get_bigdl_conf + from bigdl.util.common import get_bigdl_conf import sys sys.path = [path for path in sys.path if "spark-bigdl.conf" not in path] sys.path.insert(0, os.path.join(os.path.split(__file__)[0], "resources/conf/python-api.zip")) # noqa @@ -159,7 +159,7 @@ def gen_rand_sample(): print(test_result) def test_forward_backward(self): - from nn.layer import Linear + from bigdl.nn.layer import Linear rng = RNG() rng.set_seed(100) @@ -181,7 +181,7 @@ def test_forward_backward(self): l_grad_output = linear.backward(input, grad_output) def test_forward_multiple(self): - from nn.layer import Linear + from bigdl.nn.layer import Linear rng = RNG() rng.set_seed(100) From 32e624e42af50eb25ecf98b571e412d706970d05 Mon Sep 17 00:00:00 2001 From: Yiheng Wang Date: Tue, 6 Jun 2017 17:21:28 +0800 Subject: [PATCH 032/823] remove large bin files in pyspark Former-commit-id: 5299791e3a258df4c5a7c300d3bc804e930c71f7 --- dist/Bigbig-0.1.3.tar.gz.REMOVED.git-id | 1 - dist/Bigbig-0.1.4.tar.gz.REMOVED.git-id | 1 - dist/Bigbig-0.1.5.tar.gz.REMOVED.git-id | 1 - dist/Bigbig-0.1.6.tar.gz.REMOVED.git-id | 1 - dist/Bigbig-0.1.7.tar.gz.REMOVED.git-id | 1 - 5 files changed, 5 deletions(-) delete mode 100644 dist/Bigbig-0.1.3.tar.gz.REMOVED.git-id delete mode 100644 dist/Bigbig-0.1.4.tar.gz.REMOVED.git-id delete mode 100644 dist/Bigbig-0.1.5.tar.gz.REMOVED.git-id delete mode 100644 dist/Bigbig-0.1.6.tar.gz.REMOVED.git-id delete mode 100644 dist/Bigbig-0.1.7.tar.gz.REMOVED.git-id diff --git a/dist/Bigbig-0.1.3.tar.gz.REMOVED.git-id b/dist/Bigbig-0.1.3.tar.gz.REMOVED.git-id deleted file mode 100644 index 50b70e10601..00000000000 --- a/dist/Bigbig-0.1.3.tar.gz.REMOVED.git-id +++ /dev/null @@ -1 +0,0 @@ -8de3ee96b2d6636d28bdf284dd0f06181dcb0c61 \ No newline at end of file diff --git a/dist/Bigbig-0.1.4.tar.gz.REMOVED.git-id b/dist/Bigbig-0.1.4.tar.gz.REMOVED.git-id deleted file mode 100644 index f4e59d7a289..00000000000 --- a/dist/Bigbig-0.1.4.tar.gz.REMOVED.git-id +++ /dev/null @@ -1 +0,0 @@ -b9410cbb0e3776d3c3640efcf67dbe3a81262b8f \ No newline at end of file diff --git a/dist/Bigbig-0.1.5.tar.gz.REMOVED.git-id b/dist/Bigbig-0.1.5.tar.gz.REMOVED.git-id deleted file mode 100644 index ce54df3848c..00000000000 --- a/dist/Bigbig-0.1.5.tar.gz.REMOVED.git-id +++ /dev/null @@ -1 +0,0 @@ -9c4ca18f01bf477a1c2158482de9de9560840128 \ No newline at end of file diff --git a/dist/Bigbig-0.1.6.tar.gz.REMOVED.git-id b/dist/Bigbig-0.1.6.tar.gz.REMOVED.git-id deleted file mode 100644 index 07119451896..00000000000 --- a/dist/Bigbig-0.1.6.tar.gz.REMOVED.git-id +++ /dev/null @@ -1 +0,0 @@ -f84edbba2745d73f6567c56bed6920575d63d28b \ No newline at end of file diff --git a/dist/Bigbig-0.1.7.tar.gz.REMOVED.git-id b/dist/Bigbig-0.1.7.tar.gz.REMOVED.git-id deleted file mode 100644 index 9c6597388e5..00000000000 --- a/dist/Bigbig-0.1.7.tar.gz.REMOVED.git-id +++ /dev/null @@ -1 +0,0 @@ -39693bf712a1c3028a1e237dc14789af5d53bb26 \ No newline at end of file From 1f579f834f5f997fbc9c98b325087736e5eece0d Mon Sep 17 00:00:00 2001 From: Yiheng Wang Date: Tue, 6 Jun 2017 17:21:28 +0800 Subject: [PATCH 033/823] remove large bin files in pyspark Former-commit-id: 5299791e3a258df4c5a7c300d3bc804e930c71f7 --- dist/Bigbig-0.1.3.tar.gz.REMOVED.git-id | 1 - dist/Bigbig-0.1.4.tar.gz.REMOVED.git-id | 1 - dist/Bigbig-0.1.5.tar.gz.REMOVED.git-id | 1 - dist/Bigbig-0.1.6.tar.gz.REMOVED.git-id | 1 - dist/Bigbig-0.1.7.tar.gz.REMOVED.git-id | 1 - 5 files changed, 5 deletions(-) delete mode 100644 dist/Bigbig-0.1.3.tar.gz.REMOVED.git-id delete mode 100644 dist/Bigbig-0.1.4.tar.gz.REMOVED.git-id delete mode 100644 dist/Bigbig-0.1.5.tar.gz.REMOVED.git-id delete mode 100644 dist/Bigbig-0.1.6.tar.gz.REMOVED.git-id delete mode 100644 dist/Bigbig-0.1.7.tar.gz.REMOVED.git-id diff --git a/dist/Bigbig-0.1.3.tar.gz.REMOVED.git-id b/dist/Bigbig-0.1.3.tar.gz.REMOVED.git-id deleted file mode 100644 index 50b70e10601..00000000000 --- a/dist/Bigbig-0.1.3.tar.gz.REMOVED.git-id +++ /dev/null @@ -1 +0,0 @@ -8de3ee96b2d6636d28bdf284dd0f06181dcb0c61 \ No newline at end of file diff --git a/dist/Bigbig-0.1.4.tar.gz.REMOVED.git-id b/dist/Bigbig-0.1.4.tar.gz.REMOVED.git-id deleted file mode 100644 index f4e59d7a289..00000000000 --- a/dist/Bigbig-0.1.4.tar.gz.REMOVED.git-id +++ /dev/null @@ -1 +0,0 @@ -b9410cbb0e3776d3c3640efcf67dbe3a81262b8f \ No newline at end of file diff --git a/dist/Bigbig-0.1.5.tar.gz.REMOVED.git-id b/dist/Bigbig-0.1.5.tar.gz.REMOVED.git-id deleted file mode 100644 index ce54df3848c..00000000000 --- a/dist/Bigbig-0.1.5.tar.gz.REMOVED.git-id +++ /dev/null @@ -1 +0,0 @@ -9c4ca18f01bf477a1c2158482de9de9560840128 \ No newline at end of file diff --git a/dist/Bigbig-0.1.6.tar.gz.REMOVED.git-id b/dist/Bigbig-0.1.6.tar.gz.REMOVED.git-id deleted file mode 100644 index 07119451896..00000000000 --- a/dist/Bigbig-0.1.6.tar.gz.REMOVED.git-id +++ /dev/null @@ -1 +0,0 @@ -f84edbba2745d73f6567c56bed6920575d63d28b \ No newline at end of file diff --git a/dist/Bigbig-0.1.7.tar.gz.REMOVED.git-id b/dist/Bigbig-0.1.7.tar.gz.REMOVED.git-id deleted file mode 100644 index 9c6597388e5..00000000000 --- a/dist/Bigbig-0.1.7.tar.gz.REMOVED.git-id +++ /dev/null @@ -1 +0,0 @@ -39693bf712a1c3028a1e237dc14789af5d53bb26 \ No newline at end of file From 99b251f94b696e33bca67d3d0efd5ffb1dc4d6e9 Mon Sep 17 00:00:00 2001 From: Xianyan Date: Wed, 7 Jun 2017 09:44:31 +0800 Subject: [PATCH 034/823] change SelectTable from Container to AbstractModule (#977) --- python/dllib/src/bigdl/dllib/nn/layer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 5a8133dd0e0..c06e3d876fc 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -2111,7 +2111,7 @@ def __init__(self, size) -class SelectTable(Container): +class SelectTable(Model): ''' Creates a module that takes a table as input and outputs the element at index `index` From 4cf389c756d9f66258b7335cc9983eab7931ec2a Mon Sep 17 00:00:00 2001 From: Xianyan Date: Wed, 7 Jun 2017 09:44:31 +0800 Subject: [PATCH 035/823] change SelectTable from Container to AbstractModule (#977) --- python/dllib/src/bigdl/dllib/nn/layer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 5a8133dd0e0..c06e3d876fc 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -2111,7 +2111,7 @@ def __init__(self, size) -class SelectTable(Container): +class SelectTable(Model): ''' Creates a module that takes a table as input and outputs the element at index `index` From cdf49ccafd59a1d82a28bea64bf9665b38155317 Mon Sep 17 00:00:00 2001 From: Ian Wong Date: Wed, 7 Jun 2017 19:10:34 +0800 Subject: [PATCH 036/823] some new layer and existing layer refactor (#957) * nn refactor * fix code style issue * add python api * change back the layers --- python/dllib/src/bigdl/dllib/nn/layer.py | 50 ++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index c06e3d876fc..57ca6e80352 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -2983,6 +2983,56 @@ def __init__(self, n_input_plane, JTensor.from_ndarray(kernel)) +class Const(Model): + ''' + Return a constant tensor defined by value when forward + ''' + + def __init__(self, value, bigdl_type="float"): + super(Const, self).__init__(None, bigdl_type, JTensor.from_ndarray(value)) + +class Fill(Model): + ''' + Return a constant tensor defined by value when forward + ''' + + def __init__(self, value, bigdl_type="float"): + super(Fill, self).__init__(None, bigdl_type, value) + +class Pack(Model): + ''' + Stacks a list of n-dimensional tensors into one (n+1)-dimensional tensor. + ''' + + def __init__(self, dimension, bigdl_type="float"): + super(Pack, self).__init__(None, bigdl_type, dimension) + +class Shape(Model): + ''' + Given input, return the shape of this input as a 1-D tensor + ''' + + def __init__(self, bigdl_type="float"): + super(Shape, self).__init__(None, bigdl_type) + +class SplitAndSelect(Model): + ''' + First split the tensor along the [[dimension]] into [[numSplit]] sub tensors, + then select the [[index]]th one + ''' + + def __init__(self, dimension, index, num_split, bigdl_type="float"): + super(SplitAndSelect, self).__init__(None, bigdl_type, dimension, index, num_split) + +class StrideSlice(Model): + ''' + Extracts a strided slice from a tensor. + ''' + + def __init__(self): + raise Exception('StrideSlice is not supported in python yet') + + def _test(): import doctest From adff923f758a207a114321bb91395f94ac82bbd9 Mon Sep 17 00:00:00 2001 From: Ian Wong Date: Wed, 7 Jun 2017 19:10:34 +0800 Subject: [PATCH 037/823] some new layer and existing layer refactor (#957) * nn refactor * fix code style issue * add python api * change back the layers --- python/dllib/src/bigdl/dllib/nn/layer.py | 50 ++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index c06e3d876fc..57ca6e80352 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -2983,6 +2983,56 @@ def __init__(self, n_input_plane, JTensor.from_ndarray(kernel)) +class Const(Model): + ''' + Return a constant tensor defined by value when forward + ''' + + def __init__(self, value, bigdl_type="float"): + super(Const, self).__init__(None, bigdl_type, JTensor.from_ndarray(value)) + +class Fill(Model): + ''' + Return a constant tensor defined by value when forward + ''' + + def __init__(self, value, bigdl_type="float"): + super(Fill, self).__init__(None, bigdl_type, value) + +class Pack(Model): + ''' + Stacks a list of n-dimensional tensors into one (n+1)-dimensional tensor. + ''' + + def __init__(self, dimension, bigdl_type="float"): + super(Pack, self).__init__(None, bigdl_type, dimension) + +class Shape(Model): + ''' + Given input, return the shape of this input as a 1-D tensor + ''' + + def __init__(self, bigdl_type="float"): + super(Shape, self).__init__(None, bigdl_type) + +class SplitAndSelect(Model): + ''' + First split the tensor along the [[dimension]] into [[numSplit]] sub tensors, + then select the [[index]]th one + ''' + + def __init__(self, dimension, index, num_split, bigdl_type="float"): + super(SplitAndSelect, self).__init__(None, bigdl_type, dimension, index, num_split) + +class StrideSlice(Model): + ''' + Extracts a strided slice from a tensor. + ''' + + def __init__(self): + raise Exception('StrideSlice is not supported in python yet') + + def _test(): import doctest From cd553537852b0459da98533ddf519eb11ac59dce Mon Sep 17 00:00:00 2001 From: zhangxiaoli73 <380761639@qq.com> Date: Thu, 8 Jun 2017 09:51:02 +0800 Subject: [PATCH 038/823] delete useless CriterionTable (#981) --- python/dllib/src/bigdl/dllib/nn/layer.py | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 57ca6e80352..17392041d8e 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -2753,26 +2753,6 @@ def __init__(self, bigdl_type="float"): super(ConcatTable, self).__init__(None, bigdl_type) - -class CriterionTable(Model): - ''' - Creates a module that wraps a Criterion so that it can accept a table of inputs. - - :param criterion Criterion module - - >>> from bigdl.nn.criterion import MSECriterion - >>> criterionTable = CriterionTable(MSECriterion()) - creating: createMSECriterion - creating: createCriterionTable - ''' - - def __init__(self, - criterion, - bigdl_type="float"): - super(CriterionTable, self).__init__(None, bigdl_type, - criterion) - - class Identity(Model): ''' Identity just return the input to output. From 97114d472d3c2104387372984a9cf16322a71951 Mon Sep 17 00:00:00 2001 From: zhangxiaoli73 <380761639@qq.com> Date: Thu, 8 Jun 2017 09:51:02 +0800 Subject: [PATCH 039/823] delete useless CriterionTable (#981) --- python/dllib/src/bigdl/dllib/nn/layer.py | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 57ca6e80352..17392041d8e 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -2753,26 +2753,6 @@ def __init__(self, bigdl_type="float"): super(ConcatTable, self).__init__(None, bigdl_type) - -class CriterionTable(Model): - ''' - Creates a module that wraps a Criterion so that it can accept a table of inputs. - - :param criterion Criterion module - - >>> from bigdl.nn.criterion import MSECriterion - >>> criterionTable = CriterionTable(MSECriterion()) - creating: createMSECriterion - creating: createCriterionTable - ''' - - def __init__(self, - criterion, - bigdl_type="float"): - super(CriterionTable, self).__init__(None, bigdl_type, - criterion) - - class Identity(Model): ''' Identity just return the input to output. From 5ac5bc066d1810e991dfa8b0d72bff59c0ab71dc Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Thu, 8 Jun 2017 11:29:15 +0800 Subject: [PATCH 040/823] update README to use bigdl --- python/dllib/src/bigdl/dllib/models/lenet/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/models/lenet/README.md b/python/dllib/src/bigdl/dllib/models/lenet/README.md index 5f0a6f05677..e537ccad99d 100644 --- a/python/dllib/src/bigdl/dllib/models/lenet/README.md +++ b/python/dllib/src/bigdl/dllib/models/lenet/README.md @@ -39,12 +39,12 @@ We would train a LeNet model in spark local mode with the following commands and --executor-cores 2 \ --executor-memory 4g \ --conf spark.akka.frameSize=64 \ - --py-files ${PYTHON_API_ZIP_PATH},${BigDL_HOME}/pyspark/dl/models/lenet/lenet5.py \ + --py-files ${PYTHON_API_ZIP_PATH},${BigDL_HOME}/pyspark/bigdl/models/lenet/lenet5.py \ --properties-file ${BigDL_HOME}/dist/conf/spark-bigdl.conf \ --jars ${BigDL_JAR_PATH} \ --conf spark.driver.extraClassPath=${BigDL_JAR_PATH} \ --conf spark.executor.extraClassPath=bigdl-VERSION-jar-with-dependencies.jar \ - ${BigDL_HOME}/pyspark/dl/models/lenet/lenet5.py \ + ${BigDL_HOME}/pyspark/bigdl/models/lenet/lenet5.py \ --action train ``` From d915e76d2a1673aca95ab3b708eb75cf364cea99 Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Thu, 8 Jun 2017 11:29:15 +0800 Subject: [PATCH 041/823] update README to use bigdl --- python/dllib/src/bigdl/dllib/models/lenet/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/models/lenet/README.md b/python/dllib/src/bigdl/dllib/models/lenet/README.md index 5f0a6f05677..e537ccad99d 100644 --- a/python/dllib/src/bigdl/dllib/models/lenet/README.md +++ b/python/dllib/src/bigdl/dllib/models/lenet/README.md @@ -39,12 +39,12 @@ We would train a LeNet model in spark local mode with the following commands and --executor-cores 2 \ --executor-memory 4g \ --conf spark.akka.frameSize=64 \ - --py-files ${PYTHON_API_ZIP_PATH},${BigDL_HOME}/pyspark/dl/models/lenet/lenet5.py \ + --py-files ${PYTHON_API_ZIP_PATH},${BigDL_HOME}/pyspark/bigdl/models/lenet/lenet5.py \ --properties-file ${BigDL_HOME}/dist/conf/spark-bigdl.conf \ --jars ${BigDL_JAR_PATH} \ --conf spark.driver.extraClassPath=${BigDL_JAR_PATH} \ --conf spark.executor.extraClassPath=bigdl-VERSION-jar-with-dependencies.jar \ - ${BigDL_HOME}/pyspark/dl/models/lenet/lenet5.py \ + ${BigDL_HOME}/pyspark/bigdl/models/lenet/lenet5.py \ --action train ``` From b5f6bbe6a9c31695cd462e63812f72b70d177dcb Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Thu, 8 Jun 2017 11:30:39 +0800 Subject: [PATCH 042/823] update README to use bigdl --- python/dllib/src/bigdl/dllib/models/textclassifier/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/models/textclassifier/README.md b/python/dllib/src/bigdl/dllib/models/textclassifier/README.md index eae226c7ecb..6999bb5c3e4 100644 --- a/python/dllib/src/bigdl/dllib/models/textclassifier/README.md +++ b/python/dllib/src/bigdl/dllib/models/textclassifier/README.md @@ -43,12 +43,12 @@ bigdl.sh would setup the essential environment for you and it would accept a spa --executor-cores 4 \ --executor-memory 20g \ --conf spark.akka.frameSize=64 \ - --py-files ${PYTHON_API_ZIP_PATH},${BigDL_HOME}/pyspark/dl/models/textclassifier/textclassifier.py \ + --py-files ${PYTHON_API_ZIP_PATH},${BigDL_HOME}/pyspark/bigdl/models/textclassifier/textclassifier.py \ --jars ${BigDL_JAR_PATH} \ --conf spark.driver.extraClassPath=${BigDL_JAR_PATH} \ --conf spark.executor.extraClassPath=bigdl-VERSION-jar-with-dependencies.jar \ --conf spark.executorEnv.PYTHONHASHSEED=${PYTHONHASHSEED} \ - ${BigDL_HOME}/pyspark/dl/models/textclassifier/textclassifier.py \ + ${BigDL_HOME}/pyspark/bigdl/models/textclassifier/textclassifier.py \ --max_epoch 3 --model cnn ``` From 842f1497091a97513e40f172ff7084548f57ed9c Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Thu, 8 Jun 2017 11:30:39 +0800 Subject: [PATCH 043/823] update README to use bigdl --- python/dllib/src/bigdl/dllib/models/textclassifier/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/models/textclassifier/README.md b/python/dllib/src/bigdl/dllib/models/textclassifier/README.md index eae226c7ecb..6999bb5c3e4 100644 --- a/python/dllib/src/bigdl/dllib/models/textclassifier/README.md +++ b/python/dllib/src/bigdl/dllib/models/textclassifier/README.md @@ -43,12 +43,12 @@ bigdl.sh would setup the essential environment for you and it would accept a spa --executor-cores 4 \ --executor-memory 20g \ --conf spark.akka.frameSize=64 \ - --py-files ${PYTHON_API_ZIP_PATH},${BigDL_HOME}/pyspark/dl/models/textclassifier/textclassifier.py \ + --py-files ${PYTHON_API_ZIP_PATH},${BigDL_HOME}/pyspark/bigdl/models/textclassifier/textclassifier.py \ --jars ${BigDL_JAR_PATH} \ --conf spark.driver.extraClassPath=${BigDL_JAR_PATH} \ --conf spark.executor.extraClassPath=bigdl-VERSION-jar-with-dependencies.jar \ --conf spark.executorEnv.PYTHONHASHSEED=${PYTHONHASHSEED} \ - ${BigDL_HOME}/pyspark/dl/models/textclassifier/textclassifier.py \ + ${BigDL_HOME}/pyspark/bigdl/models/textclassifier/textclassifier.py \ --max_epoch 3 --model cnn ``` From b693a697fd6f9f346982c687d17cea9561c17153 Mon Sep 17 00:00:00 2001 From: yangw Date: Thu, 8 Jun 2017 15:05:48 +0800 Subject: [PATCH 044/823] add python api --- .../bigdl/dllib/nn/initialization_method.py | 93 +++++++++++++++++++ python/dllib/src/bigdl/dllib/nn/layer.py | 60 ++++++++++++ 2 files changed, 153 insertions(+) create mode 100644 python/dllib/src/bigdl/dllib/nn/initialization_method.py diff --git a/python/dllib/src/bigdl/dllib/nn/initialization_method.py b/python/dllib/src/bigdl/dllib/nn/initialization_method.py new file mode 100644 index 00000000000..e8c9ecd29db --- /dev/null +++ b/python/dllib/src/bigdl/dllib/nn/initialization_method.py @@ -0,0 +1,93 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import sys + +from bigdl.util.common import JavaValue + +if sys.version >= '3': + long = int + unicode = str + +class InitializationMethod(JavaValue): + """ + Initialization method to initialize bias and weight. + The init method will be called in Module.reset() + """ + +class Zeros(InitializationMethod): + """ + Initializer that generates tensors with zeros. + """ + def __init__(self, bigdl_type="float"): + JavaValue.__init__(self, None, bigdl_type) + +class Ones(InitializationMethod): + """ + Initializer that generates tensors with ones. + """ + def __init__(self, bigdl_type="float"): + JavaValue.__init__(self, None, bigdl_type) + +class RandomUniform(InitializationMethod): + """ + Initializer that generates tensors with a uniform distribution. + It draws samples from a uniform distribution within [lower, upper] + If lower and upper is not specified, it draws samples form a + uniform distribution within [-limit, limit] where "limit" is "1/sqrt(fan_in)" + """ + def __init__(self, upper=None, lower=None, bigdl_type="float"): + if upper is not None and lower is not None: + upper = upper + 0.0 + lower = lower + 0.0 + JavaValue.__init__(self, None, bigdl_type, upper, lower) + else: + JavaValue.__init__(self, None, bigdl_type) + +class RandomNormal(InitializationMethod): + """ + Initializer that generates tensors with a normal distribution. + """ + def __init__(self, mean, stdv, bigdl_type="float"): + mean = mean + 0.0 + stdv = stdv + 0.0 + JavaValue.__init__(self, None, bigdl_type, mean, stdv) + +class ConstInitMethod(InitializationMethod): + """ + Initializer that generates tensors with certain constant double. + """ + def __init__(self, value, bigdl_type="float"): + value = value + 0.0 + JavaValue.__init__(self, None, bigdl_type, value) + +class Xavier(InitializationMethod): + """ + Xavier Initializer. See http://jmlr.org/proceedings/papers/v9/glorot10a/glorot10a.pdf + """ + def __init__(self, bigdl_type="float"): + JavaValue.__init__(self, None, bigdl_type) + +class BilinearFiller(InitializationMethod): + """ + Initialize the weight with coefficients for bilinear interpolation. + + A common use case is with the DeconvolutionLayer acting as upsampling. + The variable tensor passed in the init function should have 5 dimensions + of format [nGroup, nInput, nOutput, kH, kW], and kH should be equal to kW + """ + def __init__(self, bigdl_type="float"): + JavaValue.__init__(self, None, bigdl_type) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 17392041d8e..ee48037af8c 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -355,6 +355,10 @@ def __init__(self, input_size, output_size, init_method="default", with_bias=Tru super(Linear, self).__init__(None, bigdl_type, input_size, output_size, init_method, with_bias) + def set_init_method(self, weight_init_method = None, bias_init_method = None): + callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, + weight_init_method, bias_init_method) + class ReLU(Model): @@ -484,6 +488,9 @@ def __init__(self, n_group, propagate_back, init_method) + def set_init_method(self, weight_init_method = None, bias_init_method = None): + callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, + weight_init_method, bias_init_method) class SpatialMaxPooling(Model): @@ -774,6 +781,10 @@ def __init__(self, momentum, affine) + def set_init_method(self, weight_init_method = None, bias_init_method = None): + callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, + weight_init_method, bias_init_method) + class SpatialCrossMapLRN(Model): @@ -888,6 +899,9 @@ def __init__(self, bigdl_type="float"): super(Add, self).__init__(None, bigdl_type, input_size) + def set_init_method(self, weight_init_method = None, bias_init_method = None): + callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, + weight_init_method, bias_init_method) class AddConstant(Model): @@ -946,6 +960,9 @@ def __init__(self, eps, momentum, affine) + def set_init_method(self, weight_init_method = None, bias_init_method = None): + callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, + weight_init_method, bias_init_method) class Bilinear(Model): @@ -975,6 +992,9 @@ def __init__(self, input_size2, output_size, bias_res) + def set_init_method(self, weight_init_method = None, bias_init_method = None): + callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, + weight_init_method, bias_init_method) class Bottle(Container): @@ -1024,6 +1044,10 @@ def __init__(self, super(CAdd, self).__init__(None, bigdl_type, size) + def set_init_method(self, weight_init_method = None, bias_init_method = None): + callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, + weight_init_method, bias_init_method) + class CAddTable(Model): @@ -1102,6 +1126,10 @@ def __init__(self, super(CMul, self).__init__(None, bigdl_type, size) + def set_init_method(self, weight_init_method = None, bias_init_method = None): + callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, + weight_init_method, bias_init_method) + class CMulTable(Model): @@ -1192,6 +1220,9 @@ def __init__(self, super(Cosine, self).__init__(None, bigdl_type, input_size, output_size) + def set_init_method(self, weight_init_method = None, bias_init_method = None): + callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, + weight_init_method, bias_init_method) class CosineDistance(Model): @@ -1288,6 +1319,10 @@ def __init__(self, output_size, fast_backward) + def set_init_method(self, weight_init_method = None, bias_init_method = None): + callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, + weight_init_method, bias_init_method) + class Exp(Model): @@ -1584,6 +1619,9 @@ def __init__(self, max_norm, norm_type, should_scale_grad_by_freq) + def set_init_method(self, weight_init_method = None, bias_init_method = None): + callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, + weight_init_method, bias_init_method) class MM(Model): @@ -1761,6 +1799,10 @@ def __init__(self, bigdl_type="float"): super(Mul, self).__init__(None, bigdl_type) + def set_init_method(self, weight_init_method = None, bias_init_method = None): + callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, + weight_init_method, bias_init_method) + class MulConstant(Model): @@ -1872,6 +1914,10 @@ def __init__(self, super(PReLU, self).__init__(None, bigdl_type, n_output_plane) + def set_init_method(self, weight_init_method = None, bias_init_method = None): + callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, + weight_init_method, bias_init_method) + class Padding(Model): @@ -2295,6 +2341,10 @@ def __init__(self, dilation_h, init_method) + def set_init_method(self, weight_init_method = None, bias_init_method = None): + callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, + weight_init_method, bias_init_method) + class SpatialFullConvolution(Model): ''' @@ -2364,6 +2414,9 @@ def __init__(self, n_group, no_bias, init_method) + def set_init_method(self, weight_init_method = None, bias_init_method = None): + callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, + weight_init_method, bias_init_method) class SpatialShareConvolution(Model): @@ -2398,6 +2451,9 @@ def __init__(self, n_group, propagate_back, init_method) + def set_init_method(self, weight_init_method = None, bias_init_method = None): + callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, + weight_init_method, bias_init_method) class VolumetricConvolution(Model): @@ -2454,6 +2510,10 @@ def __init__(self, with_bias, init_method) + def set_init_method(self, weight_init_method = None, bias_init_method = None): + callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, + weight_init_method, bias_init_method) + class VolumetricMaxPooling(Model): From 4a0694f97326f94fe817d6259debbadbc343a3a6 Mon Sep 17 00:00:00 2001 From: yangw Date: Thu, 8 Jun 2017 15:05:48 +0800 Subject: [PATCH 045/823] add python api --- .../bigdl/dllib/nn/initialization_method.py | 93 +++++++++++++++++++ python/dllib/src/bigdl/dllib/nn/layer.py | 60 ++++++++++++ 2 files changed, 153 insertions(+) create mode 100644 python/dllib/src/bigdl/dllib/nn/initialization_method.py diff --git a/python/dllib/src/bigdl/dllib/nn/initialization_method.py b/python/dllib/src/bigdl/dllib/nn/initialization_method.py new file mode 100644 index 00000000000..e8c9ecd29db --- /dev/null +++ b/python/dllib/src/bigdl/dllib/nn/initialization_method.py @@ -0,0 +1,93 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import sys + +from bigdl.util.common import JavaValue + +if sys.version >= '3': + long = int + unicode = str + +class InitializationMethod(JavaValue): + """ + Initialization method to initialize bias and weight. + The init method will be called in Module.reset() + """ + +class Zeros(InitializationMethod): + """ + Initializer that generates tensors with zeros. + """ + def __init__(self, bigdl_type="float"): + JavaValue.__init__(self, None, bigdl_type) + +class Ones(InitializationMethod): + """ + Initializer that generates tensors with ones. + """ + def __init__(self, bigdl_type="float"): + JavaValue.__init__(self, None, bigdl_type) + +class RandomUniform(InitializationMethod): + """ + Initializer that generates tensors with a uniform distribution. + It draws samples from a uniform distribution within [lower, upper] + If lower and upper is not specified, it draws samples form a + uniform distribution within [-limit, limit] where "limit" is "1/sqrt(fan_in)" + """ + def __init__(self, upper=None, lower=None, bigdl_type="float"): + if upper is not None and lower is not None: + upper = upper + 0.0 + lower = lower + 0.0 + JavaValue.__init__(self, None, bigdl_type, upper, lower) + else: + JavaValue.__init__(self, None, bigdl_type) + +class RandomNormal(InitializationMethod): + """ + Initializer that generates tensors with a normal distribution. + """ + def __init__(self, mean, stdv, bigdl_type="float"): + mean = mean + 0.0 + stdv = stdv + 0.0 + JavaValue.__init__(self, None, bigdl_type, mean, stdv) + +class ConstInitMethod(InitializationMethod): + """ + Initializer that generates tensors with certain constant double. + """ + def __init__(self, value, bigdl_type="float"): + value = value + 0.0 + JavaValue.__init__(self, None, bigdl_type, value) + +class Xavier(InitializationMethod): + """ + Xavier Initializer. See http://jmlr.org/proceedings/papers/v9/glorot10a/glorot10a.pdf + """ + def __init__(self, bigdl_type="float"): + JavaValue.__init__(self, None, bigdl_type) + +class BilinearFiller(InitializationMethod): + """ + Initialize the weight with coefficients for bilinear interpolation. + + A common use case is with the DeconvolutionLayer acting as upsampling. + The variable tensor passed in the init function should have 5 dimensions + of format [nGroup, nInput, nOutput, kH, kW], and kH should be equal to kW + """ + def __init__(self, bigdl_type="float"): + JavaValue.__init__(self, None, bigdl_type) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 17392041d8e..ee48037af8c 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -355,6 +355,10 @@ def __init__(self, input_size, output_size, init_method="default", with_bias=Tru super(Linear, self).__init__(None, bigdl_type, input_size, output_size, init_method, with_bias) + def set_init_method(self, weight_init_method = None, bias_init_method = None): + callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, + weight_init_method, bias_init_method) + class ReLU(Model): @@ -484,6 +488,9 @@ def __init__(self, n_group, propagate_back, init_method) + def set_init_method(self, weight_init_method = None, bias_init_method = None): + callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, + weight_init_method, bias_init_method) class SpatialMaxPooling(Model): @@ -774,6 +781,10 @@ def __init__(self, momentum, affine) + def set_init_method(self, weight_init_method = None, bias_init_method = None): + callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, + weight_init_method, bias_init_method) + class SpatialCrossMapLRN(Model): @@ -888,6 +899,9 @@ def __init__(self, bigdl_type="float"): super(Add, self).__init__(None, bigdl_type, input_size) + def set_init_method(self, weight_init_method = None, bias_init_method = None): + callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, + weight_init_method, bias_init_method) class AddConstant(Model): @@ -946,6 +960,9 @@ def __init__(self, eps, momentum, affine) + def set_init_method(self, weight_init_method = None, bias_init_method = None): + callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, + weight_init_method, bias_init_method) class Bilinear(Model): @@ -975,6 +992,9 @@ def __init__(self, input_size2, output_size, bias_res) + def set_init_method(self, weight_init_method = None, bias_init_method = None): + callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, + weight_init_method, bias_init_method) class Bottle(Container): @@ -1024,6 +1044,10 @@ def __init__(self, super(CAdd, self).__init__(None, bigdl_type, size) + def set_init_method(self, weight_init_method = None, bias_init_method = None): + callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, + weight_init_method, bias_init_method) + class CAddTable(Model): @@ -1102,6 +1126,10 @@ def __init__(self, super(CMul, self).__init__(None, bigdl_type, size) + def set_init_method(self, weight_init_method = None, bias_init_method = None): + callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, + weight_init_method, bias_init_method) + class CMulTable(Model): @@ -1192,6 +1220,9 @@ def __init__(self, super(Cosine, self).__init__(None, bigdl_type, input_size, output_size) + def set_init_method(self, weight_init_method = None, bias_init_method = None): + callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, + weight_init_method, bias_init_method) class CosineDistance(Model): @@ -1288,6 +1319,10 @@ def __init__(self, output_size, fast_backward) + def set_init_method(self, weight_init_method = None, bias_init_method = None): + callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, + weight_init_method, bias_init_method) + class Exp(Model): @@ -1584,6 +1619,9 @@ def __init__(self, max_norm, norm_type, should_scale_grad_by_freq) + def set_init_method(self, weight_init_method = None, bias_init_method = None): + callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, + weight_init_method, bias_init_method) class MM(Model): @@ -1761,6 +1799,10 @@ def __init__(self, bigdl_type="float"): super(Mul, self).__init__(None, bigdl_type) + def set_init_method(self, weight_init_method = None, bias_init_method = None): + callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, + weight_init_method, bias_init_method) + class MulConstant(Model): @@ -1872,6 +1914,10 @@ def __init__(self, super(PReLU, self).__init__(None, bigdl_type, n_output_plane) + def set_init_method(self, weight_init_method = None, bias_init_method = None): + callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, + weight_init_method, bias_init_method) + class Padding(Model): @@ -2295,6 +2341,10 @@ def __init__(self, dilation_h, init_method) + def set_init_method(self, weight_init_method = None, bias_init_method = None): + callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, + weight_init_method, bias_init_method) + class SpatialFullConvolution(Model): ''' @@ -2364,6 +2414,9 @@ def __init__(self, n_group, no_bias, init_method) + def set_init_method(self, weight_init_method = None, bias_init_method = None): + callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, + weight_init_method, bias_init_method) class SpatialShareConvolution(Model): @@ -2398,6 +2451,9 @@ def __init__(self, n_group, propagate_back, init_method) + def set_init_method(self, weight_init_method = None, bias_init_method = None): + callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, + weight_init_method, bias_init_method) class VolumetricConvolution(Model): @@ -2454,6 +2510,10 @@ def __init__(self, with_bias, init_method) + def set_init_method(self, weight_init_method = None, bias_init_method = None): + callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, + weight_init_method, bias_init_method) + class VolumetricMaxPooling(Model): From bb365c88f8bd4b4f26554c5ef038e5729a4fda1f Mon Sep 17 00:00:00 2001 From: yangw Date: Thu, 8 Jun 2017 15:05:48 +0800 Subject: [PATCH 046/823] add python api --- simple_integration_test.py | 60 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/simple_integration_test.py b/simple_integration_test.py index 645ea6a8b0a..8e7d55789fc 100644 --- a/simple_integration_test.py +++ b/simple_integration_test.py @@ -19,6 +19,7 @@ from bigdl.nn.criterion import * from bigdl.optim.optimizer import * from bigdl.util.common import * +from bigdl.nn.initialization_method import * import numpy as np import unittest import tempfile @@ -200,6 +201,65 @@ def test_forward_multiple(self): module.forward(input) module.backward(input, grad_output) + def test_init_method(self): + initializers = [ + Zeros(), + Ones(), + ConstInitMethod(5), + RandomUniform(-1, 1), + RandomNormal(0, 1), + None + ] + special_initializers = [ + Xavier(), + RandomUniform(), + ] + + layers = [ + SpatialConvolution(6, 12, 5, 5), + SpatialShareConvolution(1, 1, 1, 1), + LookupTable(1, 1, 1e-5, 1e-5, 1e-5, True), + Bilinear(1, 1, 1, True), + Cosine(2, 3), + SpatialFullConvolution(1, 1, 1, 1), + Add(1), + Linear(100, 10), + CMul([1, 2]), + Mul(), + PReLU(1), + Euclidean(1, 1, True), + SpatialDilatedConvolution(1, 1, 1, 1), + SpatialBatchNormalization(1), + BatchNormalization(1, 1e-5, 1e-5, True), + ] + + special_layers = [ + SpatialConvolution(6, 12, 5, 5), + SpatialShareConvolution(1, 1, 1, 1), + Cosine(2, 3), + SpatialFullConvolution(1, 1, 1, 1), + Add(1), + Linear(100, 10), + CMul([1, 2]), + Mul(), + PReLU(1), + Euclidean(1, 1, True), + SpatialDilatedConvolution(1, 1, 1, 1), + SpatialBatchNormalization(1), + BatchNormalization(1, 1e-5, 1e-5, True), + ] + for layer in layers: + for init1 in initializers: + for init2 in initializers: + layer.set_init_method(init1, init2) + + for layer in special_layers: + for init1 in special_initializers: + for init2 in special_initializers: + layer.set_init_method(init1, init2) + + SpatialFullConvolution(1, 1, 1, 1).set_init_method(BilinearFiller(), Zeros()) + def test_predict(self): np.random.seed(100) total_length = 6 From 8937422809c7e0a6216a3f21e0823519561bf39a Mon Sep 17 00:00:00 2001 From: Yuchao-Tao Date: Fri, 9 Jun 2017 17:42:17 +0800 Subject: [PATCH 047/823] transform python files in respect to comments. fix the format problem of ":return:" (#873) --- .../src/bigdl/dllib/feature/dataset/base.py | 13 +- .../src/bigdl/dllib/feature/dataset/mnist.py | 10 +- .../src/bigdl/dllib/feature/dataset/news20.py | 2 + .../src/bigdl/dllib/models/lenet/lenet5.py | 1 + python/dllib/src/bigdl/dllib/nn/criterion.py | 95 ++- python/dllib/src/bigdl/dllib/nn/layer.py | 608 ++++++++++++------ .../dllib/src/bigdl/dllib/optim/optimizer.py | 187 +++--- python/dllib/src/bigdl/dllib/utils/common.py | 8 +- 8 files changed, 604 insertions(+), 320 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/feature/dataset/base.py b/python/dllib/src/bigdl/dllib/feature/dataset/base.py index f211ee5261c..771786841c1 100644 --- a/python/dllib/src/bigdl/dllib/feature/dataset/base.py +++ b/python/dllib/src/bigdl/dllib/feature/dataset/base.py @@ -28,8 +28,9 @@ class Progbar(object): def __init__(self, target, width=30, verbose=1, interval=0.01): ''' - @param target: total number of steps expected - @param interval: minimum visual progress update interval (in seconds) + + :param target: total number of steps expected + :param interval: minimum visual progress update interval (in seconds) ''' self.width = width self.target = target @@ -44,10 +45,10 @@ def __init__(self, target, width=30, verbose=1, interval=0.01): def update(self, current, values=[], force=False): ''' - @param current: index of current step - @param values: list of tuples (name, value_for_last_step). - The progress bar will display averages for these values. - @param force: force visual progress update + + :param current: index of current step + :param values: list of tuples (name, value_for_last_step).The progress bar will display averages for these values. + :param force: force visual progress update ''' for k, v in values: if k not in self.sum_values: diff --git a/python/dllib/src/bigdl/dllib/feature/dataset/mnist.py b/python/dllib/src/bigdl/dllib/feature/dataset/mnist.py index f23087e8736..0af0f70c51f 100644 --- a/python/dllib/src/bigdl/dllib/feature/dataset/mnist.py +++ b/python/dllib/src/bigdl/dllib/feature/dataset/mnist.py @@ -38,15 +38,19 @@ def _read32(bytestream): def extract_images(f): """Extract the images into a 4D uint8 numpy array [index, y, x, depth]. + Args: f: A file object that can be passed into a gzip reader. + Returns: data: A 4D unit8 numpy array [index, y, x, depth]. + Raises: ValueError: If the bytestream does not start with 2051. + """ print('Extracting', f.name) with gzip.GzipFile(fileobj=f) as bytestream: @@ -81,9 +85,9 @@ def extract_labels(f): def read_data_sets(train_dir, data_type="train"): """ Parse or download mnist data if train_dir is empty. + :param train_dir: The directory storing the mnist data - :param data_type: Reading training set or testing set. - It can be either "train" or "test" + :param data_type: Reading training set or testing set.It can be either "train" or "test" :return: (ndarray, ndarray) representing (features, labels) features is a 4D unit8 numpy array [index, y, x, depth] representing each pixel valued from 0 to 255. labels @@ -126,4 +130,4 @@ def read_data_sets(train_dir, data_type="train"): assert numpy.abs(numpy.mean(train) - TRAIN_MEAN) / TRAIN_MEAN < 1e-7 assert numpy.abs(numpy.std(train) - TRAIN_STD) / TRAIN_STD < 1e-7 assert numpy.abs(numpy.mean(test) - TEST_MEAN) / TEST_MEAN < 1e-7 - assert numpy.abs(numpy.std(test) - TEST_STD) / TEST_STD < 1e-7 \ No newline at end of file + assert numpy.abs(numpy.std(test) - TEST_STD) / TEST_STD < 1e-7 diff --git a/python/dllib/src/bigdl/dllib/feature/dataset/news20.py b/python/dllib/src/bigdl/dllib/feature/dataset/news20.py index 3d066ad3b52..e5aac8b02e8 100644 --- a/python/dllib/src/bigdl/dllib/feature/dataset/news20.py +++ b/python/dllib/src/bigdl/dllib/feature/dataset/news20.py @@ -53,6 +53,7 @@ def download_glove_w2v(dest_dir): def get_news20(source_dir="/tmp/news20/"): """ Parse or download news20 if source_dir is empty. + :param source_dir: The directory storing news data. :return: A list of (tokens, label) """ @@ -81,6 +82,7 @@ def get_news20(source_dir="/tmp/news20/"): def get_glove_w2v(source_dir="/tmp/news20/", dim=100): """ Parse or download the pre-trained glove word2vec if source_dir is empty. + :param source_dir: The directory storing the pre-trained word2vec :param dim: The dimension of a vector :return: A dict mapping from word to vector diff --git a/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py b/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py index b3fcba58c9c..68891fb8f09 100644 --- a/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py +++ b/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py @@ -46,6 +46,7 @@ def get_minst(sc, data_type="train", location="/tmp/mnist"): """ Get and normalize the mnist data. We would download it automatically if the data doesn't present at the specific location. + :param sc: SparkContext :param data_type: training data or testing data :param location: Location storing the mnist diff --git a/python/dllib/src/bigdl/dllib/nn/criterion.py b/python/dllib/src/bigdl/dllib/nn/criterion.py index 63ddf6a5bf1..83f5f4113a1 100644 --- a/python/dllib/src/bigdl/dllib/nn/criterion.py +++ b/python/dllib/src/bigdl/dllib/nn/criterion.py @@ -46,6 +46,7 @@ def forward(self, input, target): NB: It's for debug only, please use optimizer.optimize() in production. Takes an input object, and computes the corresponding loss of the criterion, compared with `target` + :param input: ndarray or list of ndarray :param target: ndarray or list of ndarray :return: value of loss @@ -61,6 +62,7 @@ def backward(self, input, target): """ NB: It's for debug only, please use optimizer.optimize() in production. Performs a back-propagation step through the criterion, with respect to the given input. + :param input: ndarray or list of ndarray :param target: ndarray or list of ndarray :return: ndarray @@ -76,6 +78,7 @@ def backward(self, input, target): def of(cls, jcriterion, bigdl_type="float"): """ Create a python Criterion by a java criterion object + :param jcriterion: A java criterion object which created by Py4j :return: a criterion. """ @@ -93,8 +96,10 @@ class ClassNLLCriterion(Criterion): If provided, the optional argument weights should be a 1D Tensor assigning weight to each of the classes. - :param weights weights of each class - :param size_average whether to average or not + + :param weights: weights of each class + :param size_average: whether to average or not + >>> np.random.seed(123) >>> weights = np.random.uniform(0, 1, (2,)).astype("float32") @@ -118,7 +123,10 @@ class MSECriterion(Criterion): ''' Creates a criterion that measures the mean squared error between n elements in the input x and output y: +``` loss(x, y) = 1/n \sum |x_i - y_i|^2 +``` + If x and y are d-dimensional Tensors with a total of n elements, the sum operation still operates over all the elements, and divides by n. @@ -127,6 +135,7 @@ class MSECriterion(Criterion): By default, the losses are averaged over observations for each minibatch. However, if the field sizeAverage is set to false, the losses are instead summed. + >>> mSECriterion = MSECriterion() creating: createMSECriterion ''' @@ -140,6 +149,7 @@ class AbsCriterion(Criterion): ''' measures the mean absolute value of the element-wise difference between input + >>> absCriterion = AbsCriterion(True) creating: createAbsCriterion ''' @@ -157,7 +167,9 @@ class ClassSimplexCriterion(Criterion): ClassSimplexCriterion implements a criterion for classification. It learns an embedding per class, where each class' embedding is a point on an (N-1)-dimensional simplex, where N is the number of classes. - :param nClasses the number of classes. + + :param nClasses: the number of classes. + >>> classSimplexCriterion = ClassSimplexCriterion(2) creating: createClassSimplexCriterion @@ -176,7 +188,9 @@ class CosineEmbeddingCriterion(Criterion): Creates a criterion that measures the loss given an input x = {x1, x2}, a table of two Tensors, and a Tensor label y with values 1 or -1. - :param margin a number from -1 to 1, 0 to 0.5 is suggested + + :param margin: a number from -1 to 1, 0 to 0.5 is suggested + >>> cosineEmbeddingCriterion = CosineEmbeddingCriterion(1e-5, True) creating: createCosineEmbeddingCriterion @@ -196,7 +210,9 @@ class DistKLDivCriterion(Criterion): ''' The Kullback-Leibler divergence criterion - :param sizeAverage + + :param sizeAverage: + >>> distKLDivCriterion = DistKLDivCriterion(True) creating: createDistKLDivCriterion @@ -219,11 +235,13 @@ class HingeEmbeddingCriterion(Criterion): e.g. using the L1 pairwise distance, and is typically used for learning nonlinear embeddings or semi-supervised learning. + If x and y are n-dimensional Tensors, the sum operation still operates over all the elements, and divides by n (this can be avoided if one sets the internal variable sizeAverage to false). The margin has a default value of 1, or can be set in the constructor. + >>> hingeEmbeddingCriterion = HingeEmbeddingCriterion(1e-5, True) creating: createHingeEmbeddingCriterion ''' @@ -243,7 +261,9 @@ class L1HingeEmbeddingCriterion(Criterion): Creates a criterion that measures the loss given an input x = {x1, x2}, a table of two Tensors, and a label y (1 or -1): - :param margin + + :param margin: + >>> l1HingeEmbeddingCriterion = L1HingeEmbeddingCriterion(1e-5) creating: createL1HingeEmbeddingCriterion @@ -262,9 +282,11 @@ class MarginCriterion(Criterion): Creates a criterion that optimizes a two-class classification hinge loss (margin-based loss) between input x (a Tensor of dimension 1) and output y. - :param margin if unspecified, is by default 1. + + :param margin: if unspecified, is by default 1. :param size_average: size average in a mini-batch + >>> marginCriterion = MarginCriterion(1e-5, True) creating: createMarginCriterion ''' @@ -288,7 +310,9 @@ class MarginRankingCriterion(Criterion): If y == 1 then it assumed the first input should be ranked higher (have a larger value) than the second input, and vice-versa for y == -1. - :param margin + + :param margin: + >>> marginRankingCriterion = MarginRankingCriterion(1e-5, True) creating: createMarginRankingCriterion @@ -308,6 +332,7 @@ class MultiCriterion(Criterion): ''' a weighted sum of other criterions each applied to the same input and target + >>> multiCriterion = MultiCriterion() creating: createMultiCriterion ''' @@ -323,8 +348,10 @@ class MultiLabelMarginCriterion(Criterion): Creates a criterion that optimizes a multi-class multi-classification hinge loss ( margin-based loss) between input x and output y (which is a Tensor of target class indices) + :param size_average: size average in a mini-batch + >>> multiLabelMarginCriterion = MultiLabelMarginCriterion(True) creating: createMultiLabelMarginCriterion ''' @@ -342,9 +369,12 @@ class ParallelCriterion(Criterion): ParallelCriterion is a weighted sum of other criterions each applied to a different input and target. Set repeatTarget = true to share the target for criterions. + Use add(criterion[, weight]) method to add criterion. Where weight is a scalar(default 1). - :param repeat_target Whether to share the target for all criterions. + + :param repeat_target: Whether to share the target for all criterions. + >>> parallelCriterion = ParallelCriterion(True) creating: createParallelCriterion @@ -364,14 +394,18 @@ class SmoothL1Criterion(Criterion): It uses a squared term if the absolute element-wise error falls below 1. It is less sensitive to outliers than the MSECriterion and in some cases prevents exploding gradients (e.g. see "Fast R-CNN" paper by Ross Girshick). +``` | 0.5 * (x_i - y_i)^2^, if |x_i - y_i| < 1 loss(x, y) = 1/n \sum | | |x_i - y_i| - 0.5, otherwise +``` If x and y are d-dimensional Tensors with a total of n elements, the sum operation still operates over all the elements, and divides by n. The division by n can be avoided if one sets the internal variable sizeAverage to false - :param size_average whether to average the loss + + :param size_average: whether to average the loss + >>> smoothL1Criterion = SmoothL1Criterion(True) creating: createSmoothL1Criterion @@ -392,11 +426,13 @@ class SmoothL1CriterionWithWeights(Criterion): It is less sensitive to outliers than the MSECriterion and in some cases prevents exploding gradients (e.g. see "Fast R-CNN" paper by Ross Girshick). +``` d = (x - y) * w_in loss(x, y, w_in, w_out) | 0.5 * (sigma * d_i)^2 * w_out if |d_i| < 1 / sigma / sigma = 1/n \sum | | (|d_i| - 0.5 / sigma / sigma) * w_out otherwise +``` >>> smoothL1CriterionWithWeights = SmoothL1CriterionWithWeights(1e-5, 1) creating: createSmoothL1CriterionWithWeights @@ -418,9 +454,10 @@ class SoftmaxWithCriterion(Criterion): passing real-valued predictions through a softmax to get a probability distribution over classes. It should be preferred over separate SoftmaxLayer + MultinomialLogisticLossLayer as its gradient computation is more numerically stable. - :param ignoreLabel (optional) Specify a label value that - should be ignored when computing the loss. - :param normalizeMode How to normalize the output loss. + + :param ignoreLabel: (optional) Specify a label value thatshould be ignored when computing the loss. + :param normalizeMode: How to normalize the output loss. + >>> softmaxWithCriterion = SoftmaxWithCriterion() creating: createSoftmaxWithCriterion @@ -442,9 +479,11 @@ class TimeDistributedCriterion(Criterion): This class is intended to support inputs with 3 or more dimensions. Apply Any Provided Criterion to every temporal slice of an input. + :param criterion: embedded criterion :param size_average: whether to divide the sequence length + >>> td = TimeDistributedCriterion(ClassNLLCriterion()) creating: createClassNLLCriterion creating: createTimeDistributedCriterion @@ -459,7 +498,9 @@ class CrossEntropyCriterion(Criterion): """ This criterion combines LogSoftMax and ClassNLLCriterion in one single class. - :param weights A tensor assigning weight to each of the classes + + :param weights: A tensor assigning weight to each of the classes + >>> np.random.seed(123) >>> weights = np.random.uniform(0, 1, (2,)).astype("float32") @@ -484,8 +525,10 @@ class BCECriterion(Criterion): Creates a criterion that measures the Binary Cross Entropy between the target and the output - :param weights weights for each class - :param sizeAverage whether to average the loss or not + + :param weights: weights for each class + :param sizeAverage: whether to average the loss or not + >>> np.random.seed(123) >>> weights = np.random.uniform(0, 1, (2,)).astype("float32") @@ -508,10 +551,14 @@ class MultiLabelSoftMarginCriterion(Criterion): ''' A MultiLabel multiclass criterion based on sigmoid: the loss is: +``` l(x,y) = - sum_i y[i] * log(p[i]) + (1 - y[i]) * log (1 - p[i]) +``` where p[i] = exp(x[i]) / (1 + exp(x[i])) and with weights: +``` l(x,y) = - sum_i weights[i] (y[i] * log(p[i]) + (1 - y[i]) * log (1 - p[i])) +``` >>> np.random.seed(123) >>> weights = np.random.uniform(0, 1, (2,)).astype("float32") @@ -535,10 +582,12 @@ class MultiMarginCriterion(Criterion): Creates a criterion that optimizes a multi-class classification hinge loss (margin-based loss) between input x and output y (which is a target class index). - :param p - :param weights - :param margin - :param size_average + + :param p: + :param weights: + :param margin: + :param size_average: + >>> np.random.seed(123) >>> weights = np.random.uniform(0, 1, (2,)).astype("float32") @@ -567,10 +616,12 @@ class SoftMarginCriterion(Criterion): between input x (a Tensor of dimension 1) and output y (which is a tensor containing either 1s or -1s). +``` loss(x, y) = sum_i (log(1 + exp(-y[i]*x[i]))) / x:nElement() +``` + + :param sizeaverage: The normalization by the number of elements in the inputcan be disabled by setting - :param sizeaverage The normalization by the number of elements in the input - can be disabled by setting >>> softMarginCriterion = SoftMarginCriterion(False) creating: createSoftMarginCriterion diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index ee48037af8c..5f3e157d0e1 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -90,6 +90,7 @@ def __call__(self, x=None): def of(cls, jmodel, bigdl_type="float"): """ Create a Python Model + :param jmodel: Java model create by Py4j :return: A Python Model """ @@ -113,6 +114,7 @@ def name(self): def set_seed(self, seed=123): """ You can control the random seed which used to init weights for this model. + :param seed: random seed :return: Model itself. """ @@ -150,6 +152,7 @@ def forward(self, input): """ NB: It's for debug only, please use optimizer.optimize() in production. Takes an input object, and computes the corresponding output of the module + :param input: ndarray or list of ndarray :return: ndarray or list of ndarray """ @@ -166,6 +169,7 @@ def backward(self, input, grad_output): general this method makes the assumption forward(input) has been called before, with the same input. This is necessary for optimization reasons. If you do not respect this rule, backward() will compute incorrect gradients. + :param input: ndarray or list of ndarray :param grad_output: ndarray or list of ndarray :return: ndarray or list of ndarray @@ -187,6 +191,7 @@ def reset(self): def parameters(self): """ Get the model parameters which containing: weight, bias, gradBias, gradWeight + :return: dict(layername -> dict(parametername -> ndarray)) """ name_to_params = callBigDlFunc(self.bigdl_type, @@ -207,6 +212,7 @@ def predict(self, data_rdd): Model inference base on the given data. You need to invoke collect() to trigger those action \ as the returning result is an RDD. + :param data_rdd: the data to be predict. :return: An RDD represent the predict result. """ @@ -217,10 +223,10 @@ def predict(self, data_rdd): def test(self, val_rdd, batch_size, val_methods): """ A method to benchmark the model quality. + :param val_rdd: the input data :param batch_size: batch size - :param val_methods: a list of validation methods. i.e: Top1Accuracy, - Top5Accuracy and Loss. + :param val_methods: a list of validation methods. i.e: Top1Accuracy,Top5Accuracy and Loss. :return: """ return callBigDlFunc(self.bigdl_type, @@ -231,8 +237,10 @@ def test(self, val_rdd, batch_size, val_methods): def set_weights(self, weights): """ Set weights for this layer + :param weights: a list of numpy arrays which represent weight and bias :return: + >>> linear = Linear(3,2) creating: createLinear >>> linear.set_weights([np.array([[1,2,3],[4,5,6]]), np.array([7,8])]) @@ -269,6 +277,7 @@ def set_weights(self, weights): def get_weights(self): """ Get weights for this layer + :return: list of numpy arrays which represent weight and bias """ tensorWeights = callBigDlFunc(self.bigdl_type, @@ -283,6 +292,7 @@ def get_weights(self): def load(path, bigdl_type="float"): """ Load a pre-trained Bigdl model. + :param path: The path containing the pre-trained model. :return: A pre-trained model. """ @@ -293,6 +303,7 @@ def load(path, bigdl_type="float"): def load_torch(path, bigdl_type="float"): """ Load a pre-trained Torch model. + :param path: The path containing the pre-trained model. :return: A pre-trained model. """ @@ -304,8 +315,8 @@ def load_caffe(model, defPath, modelPath, match_all=True, bigdl_type="float"): """ Load a pre-trained Caffe model. - :param model: A bigdl model definition \ - which equivalent to the pre-trained caffe model. + + :param model: A bigdl model definition \which equivalent to the pre-trained caffe model. :param defPath: The path containing the caffe model definition. :param modelPath: The path containing the pre-trained caffe model. :return: A pre-trained model. @@ -315,7 +326,7 @@ def load_caffe(model, defPath, modelPath, match_all=True, bigdl_type="float"): class Container(Model): ''' - [[Container]] is a sub-class of Model that declares methods defined in all containers. + [[Container]] is a sub-class of Model that declares methods defined in all containers. A container usually contain some other modules which can be added through the "add" method ''' @@ -339,12 +350,11 @@ class Linear(Model): an input sample of given batch (the number of rows means the batch size and the number of columns should be equal to the `inputSize`). - :param input_size the size the each input sample - :param output_size the size of the module output of each sample - :param init_method two initialized methods are supported here, which are [[Default]] - and [[Xavier]], where [[Xavier]] set bias to zero here. For more - detailed information about `initMethod`, please refer to - [[InitializationMethod]] + + :param input_size: the size the each input sample + :param output_size: the size of the module output of each sample + :param init_method: two initialized methods are supported here, which are [[Default]]and [[Xavier]], where [[Xavier]] set bias to zero here. For moredetailed information about `initMethod`, please refer to[[InitializationMethod]] + >>> linear = Linear(100, 10, "Xavier") creating: createLinear @@ -366,9 +376,11 @@ class ReLU(Model): Applies the rectified linear unit (ReLU) function element-wise to the input Tensor, thus outputting a Tensor of the same dimension. + ReLU is defined as: f(x) = max(0, x) Can optionally do its operation in-place without using extra state memory + >>> relu = ReLU() creating: createReLU ''' @@ -383,6 +395,7 @@ class Tanh(Model): Applies the Tanh function element-wise to the input Tensor, thus outputting a Tensor of the same dimension. Tanh is defined as f(x) = (exp(x)-exp(-x))/(exp(x)+exp(-x)). + >>> tanh = Tanh() creating: createTanh ''' @@ -397,6 +410,7 @@ class Echo(Model): This module is for debug purpose, which can print activation and gradient in your model topology + >>> echo = Echo() creating: createEcho ''' @@ -412,6 +426,7 @@ class LogSoftMax(Model): LogSoftmax is defined as: f_i(x) = log(1 / a exp(x_i)) where a = sum_j[exp(x_j)]. + >>> logSoftMax = LogSoftMax() creating: createLogSoftMax ''' @@ -426,6 +441,7 @@ class Sequential(Container): Sequential provides a means to plug layers together in a feed-forward fully connected manner. + >>> echo = Echo() creating: createEcho >>> s = Sequential() @@ -434,6 +450,7 @@ class Sequential(Container): >>> s = s.add(s) >>> s = s.add(echo) + ''' def __init__(self, bigdl_type="float"): @@ -447,17 +464,19 @@ class SpatialConvolution(Model): The input tensor in forward(input) is expected to be a 3D tensor (nInputPlane x height x width). - :param n_input_plane The number of expected input planes in the image given into forward() - :param n_output_plane The number of output planes the convolution layer will produce. - :param kernel_w The kernel width of the convolution - :param kernel_h The kernel height of the convolution - :param stride_w The step of the convolution in the width dimension. - :param stride_h The step of the convolution in the height dimension - :param pad_w The additional zeros added per width to the input planes. - :param pad_h The additional zeros added per height to the input planes. - :param n_group Kernel group number - :param propagate_back Propagate gradient back - :param init_method Initialization method to initialize bias and weight + + :param n_input_plane: The number of expected input planes in the image given into forward() + :param n_output_plane: The number of output planes the convolution layer will produce. + :param kernel_w: The kernel width of the convolution + :param kernel_h: The kernel height of the convolution + :param stride_w: The step of the convolution in the width dimension. + :param stride_h: The step of the convolution in the height dimension + :param pad_w: The additional zeros added per width to the input planes. + :param pad_h: The additional zeros added per height to the input planes. + :param n_group: Kernel group number + :param propagate_back: Propagate gradient back + :param init_method: Initialization method to initialize bias and weight + >>> spatialConvolution = SpatialConvolution(6, 12, 5, 5) creating: createSpatialConvolution @@ -504,12 +523,14 @@ class SpatialMaxPooling(Model): oheight = op((height + 2*padH - kH) / dH + 1) op is a rounding operator. By default, it is floor. It can be changed by calling :ceil() or :floor() methods. - :param kW kernel width - :param kH kernel height - :param dW step size in width - :param dH step size in height - :param padW padding in width - :param padH padding in height + + :param kW: kernel width + :param kH: kernel height + :param dW: step size in width + :param dH: step size in height + :param padW: padding in width + :param padH: padding in height + >>> spatialMaxPooling = SpatialMaxPooling(2, 2, 2, 2) creating: createSpatialMaxPooling ''' @@ -537,8 +558,10 @@ class Select(Model): ''' A Simple layer selecting an index of the input tensor in the given dimension - :param dimension the dimension to select - :param index the index of the dimension to be selected + + :param dimension: the dimension to select + :param index: the index of the dimension to be selected + >>> select = Select(1, 1) creating: createSelect @@ -552,6 +575,7 @@ class Recurrent(Container): Recurrent module is a container of rnn cells Different types of rnn cells can be added using add() function + >>> recurrent = Recurrent() creating: createRecurrent ''' @@ -562,22 +586,19 @@ def __init__(self, bigdl_type="float"): class LSTM(Model): ''' - Long Short Term Memory architecture. - Ref. - A.: http://arxiv.org/pdf/1303.5778v1 (blueprint for this module) - B. http://web.eecs.utk.edu/~itamar/courses/ECE-692/Bobby_paper1.pdf - C. http://arxiv.org/pdf/1503.04069v1.pdf - D. https://github.com/wojzaremba/lstm - E. https://github.com/Element-Research/rnn/blob/master/FastLSTM.lua +| Long Short Term Memory architecture. +| Ref. +| A.: http://arxiv.org/pdf/1303.5778v1 (blueprint for this module) +| B. http://web.eecs.utk.edu/~itamar/courses/ECE-692/Bobby_paper1.pdf +| C. http://arxiv.org/pdf/1503.04069v1.pdf +| D. https://github.com/wojzaremba/lstm +| E. https://github.com/Element-Research/rnn/blob/master/FastLSTM.lua + :param inputSize: the size of each input vector :param hiddenSize: Hidden unit size in the LSTM - :param p: is used for [[Dropout]] probability. For more details about - RNN dropouts, please refer to - [RnnDrop: A Novel Dropout for RNNs in ASR] - (http://www.stat.berkeley.edu/~tsmoon/files/Conference/asru2015.pdf) - [A Theoretically Grounded Application of Dropout in Recurrent Neural Networks] - (https://arxiv.org/pdf/1512.05287.pdf) + :param p: is used for [[Dropout]] probability. For more details aboutRNN dropouts, please refer to[RnnDrop: A Novel Dropout for RNNs in ASR](http://www.stat.berkeley.edu/~tsmoon/files/Conference/asru2015.pdf)[A Theoretically Grounded Application of Dropout in Recurrent Neural Networks](https://arxiv.org/pdf/1512.05287.pdf) + >>> lstm = LSTM(4, 3, 0.5) creating: createLSTM @@ -589,21 +610,18 @@ def __init__(self, input_size, hidden_size, p=0.0, bigdl_type="float"): class LSTMPeephole(Model): ''' - Long Short Term Memory architecture with peephole. - Ref. A.: http://arxiv.org/pdf/1303.5778v1 (blueprint for this module) - B. http://web.eecs.utk.edu/~itamar/courses/ECE-692/Bobby_paper1.pdf - C. http://arxiv.org/pdf/1503.04069v1.pdf - D. https://github.com/wojzaremba/lstm - E. https://github.com/Element-Research/rnn/blob/master/LSTM.lua +| Long Short Term Memory architecture with peephole. +| Ref. A.: http://arxiv.org/pdf/1303.5778v1 (blueprint for this module) +| B. http://web.eecs.utk.edu/~itamar/courses/ECE-692/Bobby_paper1.pdf +| C. http://arxiv.org/pdf/1503.04069v1.pdf +| D. https://github.com/wojzaremba/lstm +| E. https://github.com/Element-Research/rnn/blob/master/LSTM.lua + :param input_size: the size of each input vector :param hidden_size: Hidden unit size in the LSTM - :param p: is used for [[Dropout]] probability. For more details about - RNN dropouts, please refer to - [RnnDrop: A Novel Dropout for RNNs in ASR] - (http://www.stat.berkeley.edu/~tsmoon/files/Conference/asru2015.pdf) - [A Theoretically Grounded Application of Dropout in Recurrent Neural Networks] - (https://arxiv.org/pdf/1512.05287.pdf) + :param p: is used for [[Dropout]] probability. For more details aboutRNN dropouts, please refer to[RnnDrop: A Novel Dropout for RNNs in ASR](http://www.stat.berkeley.edu/~tsmoon/files/Conference/asru2015.pdf)[A Theoretically Grounded Application of Dropout in Recurrent Neural Networks](https://arxiv.org/pdf/1512.05287.pdf) + >>> lstm = LSTMPeephole(4, 3, 0.5) creating: createLSTMPeephole @@ -618,18 +636,16 @@ class GRU(Model): Gated Recurrent Units architecture. The first input in sequence uses zero value for cell and hidden state - Ref. - http://www.wildml.com/2015/10/recurrent-neural-network-tutorial-part-4-implementing-a-grulstm-rnn-with-python-and-theano/ - https://github.com/Element-Research/rnn/blob/master/GRU.lua + +| Ref. +| http://www.wildml.com/2015/10/recurrent-neural-network-tutorial-part-4-implementing-a-grulstm-rnn-with-python-and-theano/ +| https://github.com/Element-Research/rnn/blob/master/GRU.lua + :param input_size: the size of each input vector :param hidden_size: Hidden unit size in GRU - :param p: is used for [[Dropout]] probability. For more details about - RNN dropouts, please refer to - [RnnDrop: A Novel Dropout for RNNs in ASR] - (http://www.stat.berkeley.edu/~tsmoon/files/Conference/asru2015.pdf) - [A Theoretically Grounded Application of Dropout in Recurrent Neural Networks] - (https://arxiv.org/pdf/1512.05287.pdf) + :param p: is used for [[Dropout]] probability. For more details aboutRNN dropouts, please refer to[RnnDrop: A Novel Dropout for RNNs in ASR](http://www.stat.berkeley.edu/~tsmoon/files/Conference/asru2015.pdf)[A Theoretically Grounded Application of Dropout in Recurrent Neural Networks](https://arxiv.org/pdf/1512.05287.pdf) + >>> gru = GRU(4, 3, 0.5) creating: createGRU @@ -643,10 +659,12 @@ class RnnCell(Model): ''' It is a simple RNN. User can pass an activation function to the RNN. + :param input_size: the size of each input vector :param hidden_size: Hidden unit size in simple RNN :param activation: activation function + >>> reshape = RnnCell(4, 3, Tanh()) creating: createTanh creating: createRnnCell @@ -665,9 +683,11 @@ class TimeDistributed(Model): This layer is intended to apply contained layer to each temporal time slice of input tensor. + For instance, The TimeDistributed Layer can feed each time slice of input tensor to the Linear layer. + >>> td = TimeDistributed(Linear(2, 3)) creating: createLinear creating: createTimeDistributed @@ -683,6 +703,7 @@ class Concat(Container): Concat concatenates the output of one layer of "parallel" modules along the provided {@code dimension}: they take the same inputs, and their output is concatenated. +``` +-----------+ +----> module1 -----+ | | | | @@ -690,9 +711,11 @@ class Concat(Container): | | | | +----> module3 -----+ +-----------+ +``` :param dimension: dimension + >>> concat = Concat(2) creating: createConcat ''' @@ -710,16 +733,17 @@ class SpatialAveragePooling(Model): Applies 2D average-pooling operation in kWxkH regions by step size dWxdH steps. The number of output features is equal to the number of input planes. - :param kW kernel width - :param kH kernel height - :param dW step width - :param dH step height - :param padW padding width - :param padH padding height - :param ceilMode whether the output size is to be ceiled or floored - :param countIncludePad whether to include padding when dividing the - number of elements in pooling region - :param divide whether to do the averaging + + :param kW: kernel width + :param kH: kernel height + :param dW: step width + :param dH: step height + :param padW: padding width + :param padH: padding height + :param ceilMode: whether the output size is to be ceiled or floored + :param countIncludePad: whether to include padding when dividing thenumber of elements in pooling region + :param divide: whether to do the averaging + >>> spatialAveragePooling = SpatialAveragePooling(7,7) creating: createSpatialAveragePooling @@ -758,13 +782,16 @@ class SpatialBatchNormalization(Model): For non-convolutional layers, see [[BatchNormalization]] The operation implemented is: +``` ( x - mean(x) ) y = -------------------- * gamma + beta standard-deviation(x) +``` where gamma and beta are learnable parameters. The learning of gamma and beta is optional. + >>> spatialBatchNormalization = SpatialBatchNormalization(1) creating: createSpatialBatchNormalization ''' @@ -791,19 +818,22 @@ class SpatialCrossMapLRN(Model): ''' Applies Spatial Local Response Normalization between different feature maps. The operation implemented is: +``` x_f y_f = ------------------------------------------------- (k+(alpha/size)* sum_{l=l1 to l2} (x_l^2^))^beta^ +``` where x_f is the input at spatial locations h,w (not shown for simplicity) and feature map f, l1 corresponds to max(0,f-ceil(size/2)) and l2 to min(F, f-ceil(size/2) + size). Here, F is the number of feature maps. - :param size: the number of channels to sum over (for cross channel LRN) or the side length of - the square region to sum over (for within channel LRN) + + :param size: the number of channels to sum over (for cross channel LRN) or the side length ofthe square region to sum over (for within channel LRN) :param alpha: the scaling parameter :param beta: the exponent :param k: a constant + >>> spatialCrossMapLRN = SpatialCrossMapLRN() creating: createSpatialCrossMapLRN ''' @@ -829,10 +859,12 @@ class Dropout(Model): set, the outputs are scaled by a factor of 1/(1-initP) during training. During evaluating, output is the same as input. + :param initP: probability to be dropped :param inplace: inplace model :param scale: if scale by a factor of 1/(1-initP) + >>> dropout = Dropout(0.4) creating: createDropout ''' @@ -856,8 +888,10 @@ class View(Model): inputs of the modules. This makes it possible to use minibatch inputs when using a size -1 for one of the dimensions. + :param size: sizes use for creates a new view + >>> view = View([1024,2]) creating: createView ''' @@ -876,6 +910,7 @@ class Abs(Model): ''' an element-wise abs operation + >>> abs = Abs() creating: createAbs ''' @@ -889,7 +924,9 @@ class Add(Model): ''' adds a bias term to input data ; - :param input_size size of input data + + :param input_size: size of input data + >>> add = Add(1) creating: createAdd ''' @@ -909,8 +946,10 @@ class AddConstant(Model): ''' adding a constant - :param constant_scalar constant value - :param inplace Can optionally do its operation in-place without using extra state memory + + :param constant_scalar: constant value + :param inplace: Can optionally do its operation in-place without using extra state memory + >>> addConstant = AddConstant(1e-5, True) creating: createAddConstant @@ -932,20 +971,26 @@ class BatchNormalization(Model): Covariate Shift" by Sergey Ioffe, Christian Szegedy https://arxiv.org/abs/1502.03167 + This implementation is useful for inputs NOT coming from convolution layers. For convolution layers, use nn.SpatialBatchNormalization. + The operation implemented is: +``` ( x - mean(x) ) y = -------------------- * gamma + beta standard-deviation(x) +``` where gamma and beta are learnable parameters.The learning of gamma and beta is optional. + :param n_output: output feature map number :param eps: avoid divide zero :param momentum: momentum for weight update :param affine: affine operation on output or not + >>> batchNormalization = BatchNormalization(1, 1e-5, 1e-5, True) creating: createBatchNormalization ''' @@ -972,10 +1017,12 @@ class Bilinear(Model): The input tensor given in forward(input) is a table containing both inputs x_1 and x_2, which are tensors of size N x inputDimension1 and N x inputDimension2, respectively. - :param input_size1 input dimension of x_1 - :param input_size2 input dimension of x_2 - :param output_size output dimension - :param bias_res whether use bias + + :param input_size1: input dimension of x_1 + :param input_size2: input dimension of x_2 + :param output_size: output dimension + :param bias_res: whether use bias + >>> bilinear = Bilinear(1, 1, 1, True) creating: createBilinear @@ -1002,10 +1049,12 @@ class Bottle(Container): ''' Bottle allows varying dimensionality input to be forwarded through any module that accepts input of nInputDim dimensions, and generates output of nOutputDim dimensions. + :param module: transform module :param n_input_dim: nInputDim dimensions of module :param n_output_dim1: output of nOutputDim dimensions + >>> bottle = Bottle(Linear(100,10), 1, 1) creating: createLinear creating: createBottle @@ -1032,8 +1081,10 @@ class CAdd(Model): it will report an error). If the input is a batch, a singleton dimension will be add to the first dimension before the expand. + :param size: the size of the bias + >>> cAdd = CAdd([1,2]) creating: createCAdd ''' @@ -1055,8 +1106,10 @@ class CAddTable(Model): Merge the input tensors in the input table by element wise adding them together. The input table is actually an array of tensor with same size. + :param inplace: reuse the input memory + >>> cAddTable = CAddTable(True) creating: createCAddTable ''' @@ -1073,6 +1126,7 @@ class CDivTable(Model): ''' Takes a table with two Tensor and returns the component-wise division between them. + >>> cDivTable = CDivTable() creating: createCDivTable ''' @@ -1087,6 +1141,7 @@ class CMaxTable(Model): ''' Takes a table of Tensors and outputs the max of all of them. + >>> cMaxTable = CMaxTable() creating: createCMaxTable ''' @@ -1100,6 +1155,7 @@ class CMinTable(Model): ''' Takes a table of Tensors and outputs the min of all of them. + >>> cMinTable = CMinTable() creating: createCMinTable ''' @@ -1114,7 +1170,9 @@ class CMul(Model): ''' Applies a component-wise multiplication to the incoming data - :param size size of the data + + :param size: size of the data + >>> cMul = CMul([1,2]) creating: createCMul @@ -1136,6 +1194,7 @@ class CMulTable(Model): ''' Takes a table of Tensors and outputs the multiplication of all of them. + >>> cMulTable = CMulTable() creating: createCMulTable ''' @@ -1150,6 +1209,7 @@ class CSubTable(Model): ''' Takes a table with two Tensor and returns the component-wise subtraction between them. + >>> cSubTable = CSubTable() creating: createCSubTable ''' @@ -1167,8 +1227,10 @@ class Clamp(Model): otherwise elements less than min_value (or greater than max_value) are saturated to min_value (or max_value). - :param min - :param max + + :param min: + :param max: + >>> clamp = Clamp(1, 3) creating: createClamp @@ -1188,6 +1250,7 @@ class Contiguous(Model): ''' used to make input, grad_output both contiguous + >>> contiguous = Contiguous() creating: createContiguous ''' @@ -1206,9 +1269,11 @@ class Cosine(Model): an input sample of given batch (the number of rows means the batch size and the number of columns should be equal to the inputSize). + :param input_size: the size of each input sample :param output_size: the size of the module output of each sample + >>> cosine = Cosine(2,3) creating: createCosine ''' @@ -1230,6 +1295,7 @@ class CosineDistance(Model): ''' Outputs the cosine distance between inputs + >>> cosineDistance = CosineDistance() creating: createCosineDistance ''' @@ -1242,11 +1308,13 @@ class DiceCoefficientCriterion(Model): ''' The Dice-Coefficient criterion - input: Tensor, target: Tensor + input: Tensor,target: Tensor +``` return: 2 * (input intersection target) 1 - ---------------------------------- input union target +``` >>> diceCoefficientCriterion = DiceCoefficientCriterion(size_average = True, epsilon = 1.0) creating: createDiceCoefficientCriterion @@ -1266,6 +1334,7 @@ class DotProduct(Model): This is a simple table layer which takes a table of two tensors as input and calculate the dot product between them as outputs + >>> dotProduct = DotProduct() creating: createDotProduct ''' @@ -1282,6 +1351,7 @@ class ELU(Model): Fast and Accurate Deep Network Learning by Exponential Linear Units (ELUs) [http://arxiv.org/pdf/1511.07289.pdf] + >>> eLU = ELU(1e-5, True) creating: createELU ''' @@ -1299,9 +1369,11 @@ class Euclidean(Model): ''' Outputs the Euclidean distance of the input to outputSize centers - :param inputSize inputSize - :param outputSize outputSize - :param T Numeric type. Only support float/double now + + :param inputSize: inputSize + :param outputSize: outputSize + :param T: Numeric type. Only support float/double now + >>> euclidean = Euclidean(1, 1, True) creating: createEuclidean @@ -1328,6 +1400,7 @@ class Exp(Model): ''' Applies element-wise exp to input tensor. + >>> exp = Exp() creating: createExp ''' @@ -1344,6 +1417,7 @@ class FlattenTable(Model): (potentially nested) as input and a table of Tensors without any nested table will be produced + >>> flattenTable = FlattenTable() creating: createFlattenTable ''' @@ -1363,7 +1437,9 @@ class GradientReversal(Model): ["Domain-Adversarial Training of Neural Networks" (http://arxiv.org/abs/1505.07818)] - :param lambda hyper-parameter lambda can be set dynamically during training + + :param lambda: hyper-parameter lambda can be set dynamically during training + >>> gradientReversal = GradientReversal(1e-5) creating: createGradientReversal @@ -1382,12 +1458,15 @@ class HardShrink(Model): This is a transfer layer which applies the hard shrinkage function element-wise to the input Tensor. The parameter lambda is set to 0.5 by default +``` x, if x > lambda f(x) = x, if x < -lambda 0, otherwise +``` :param the_lambda: a threshold value whose default value is 0.5 + >>> hardShrink = HardShrink(1e-5) creating: createHardShrink ''' @@ -1403,12 +1482,15 @@ class HardTanh(Model): ''' Applies HardTanh to each element of input, HardTanh is defined: +``` | maxValue, if x > maxValue f(x) = | minValue, if x < minValue | x, otherwise - :param min_value minValue in f(x), default is -1. - :param max_value maxValue in f(x), default is 1. - :param inplace whether enable inplace model. +``` + :param min_value: minValue in f(x), default is -1. + :param max_value: maxValue in f(x), default is 1. + :param inplace: whether enable inplace model. + >>> hardTanh = HardTanh(1e-5, 1e5, True) creating: createHardTanh @@ -1430,7 +1512,9 @@ class Index(Model): ''' Applies the Tensor index operation along the given dimension. - :param dimension the dimension to be indexed + + :param dimension: the dimension to be indexed + >>> index = Index(1) creating: createIndex @@ -1458,8 +1542,10 @@ class InferReshape(Model): For example, (4, 5, 6, 7) -> InferReshape (4, 0, 3, -1) -> (4, 5, 3, 14) with 1st and 3rd dim same as given size, with 2nd dim same as input, and the infered dim is 14 - :param size the target tensor size - :param batch_mode whether in batch mode + + :param size: the target tensor size + :param batch_mode: whether in batch mode + >>> inferReshape = InferReshape([4, 0, 3, -1], False) creating: createInferReshape @@ -1480,15 +1566,16 @@ class JoinTable(Model): It is a table module which takes a table of Tensors as input and outputs a Tensor by joining them together along the dimension `dimension`. + The input to this layer is expected to be a tensor, or a batch of tensors; when using mini-batch, a batch of sample tensors will be passed to the layer and the user need to specify the number of dimensions of each sample tensor in the batch using `nInputDims`. - :param dimension to be join in this dimension - :param nInputDims specify the number of dimensions that this module will receive - If it is more than the dimension of input tensors, the first dimension - would be considered as batch size + + :param dimension: to be join in this dimension + :param nInputDims: specify the number of dimensions that this module will receiveIf it is more than the dimension of input tensors, the first dimensionwould be considered as batch size + >>> joinTable = JoinTable(1, 1) creating: createJoinTable @@ -1507,6 +1594,7 @@ class L1Cost(Model): ''' compute L1 norm for input, and sign of input + >>> l1Cost = L1Cost() creating: createL1Cost ''' @@ -1524,9 +1612,11 @@ class L1Penalty(Model): directly to the output, and computes an L1 loss of the latent state (input) and stores it in the module's loss field. During backward propagation: gradInput = gradOutput + gradLoss. - :param l1weight - :param sizeAverage - :param provideOutput + + :param l1weight: + :param sizeAverage: + :param provideOutput: + >>> l1Penalty = L1Penalty(1, True, True) creating: createL1Penalty @@ -1549,9 +1639,11 @@ class LeakyReLU(Model): It is a transfer module that applies LeakyReLU, which parameter negval sets the slope of the negative part: LeakyReLU is defined as: f(x) = max(0, x) + negval * min(0, x) + :param negval: sets the slope of the negative partl :param inplace: if it is true, doing the operation in-place without using extra state memory + >>> leakyReLU = LeakyReLU(1e-5, True) creating: createLeakyReLU ''' @@ -1571,6 +1663,7 @@ class Log(Model): Applies the log function element-wise to the input Tensor, thus outputting a Tensor of the same dimension. + >>> log = Log() creating: createLog ''' @@ -1586,6 +1679,7 @@ class LogSigmoid(Model): This class is a transform layer corresponding to the sigmoid function: f(x) = Log(1 / (1 + e ^^ (-x))) + >>> logSigmoid = LogSigmoid() creating: createLogSigmoid ''' @@ -1600,6 +1694,7 @@ class LookupTable(Model): ''' a convolution of width 1, commonly used for word embeddings + >>> lookupTable = LookupTable(1, 1, 1e-5, 1e-5, 1e-5, True) creating: createLookupTable ''' @@ -1629,9 +1724,11 @@ class MM(Model): ''' Module to perform matrix multiplication on two mini-batch inputs, producing a mini-batch. + :param trans_a: specifying whether or not transpose the first input matrix :param trans_b: specifying whether or not transpose the second input matrix + >>> mM = MM(True, True) creating: createMM ''' @@ -1650,7 +1747,9 @@ class MV(Model): It is a module to perform matrix vector multiplication on two mini-batch inputs, producing a mini-batch. - :param trans whether make matrix transpose before multiplication + + :param trans: whether make matrix transpose before multiplication + >>> mV = MV(True) creating: createMV @@ -1670,6 +1769,7 @@ class MapTable(Container): to all input elements. The member module is cloned as necessary to process all input elements. + >>> mapTable = MapTable(Linear(100,10)) creating: createLinear creating: createMapTable @@ -1686,6 +1786,7 @@ class MaskedSelect(Model): ''' Performs a torch.MaskedSelect on a Tensor. The mask is supplied as a tabular argument with the input on the forward and backward passes. + >>> maskedSelect = MaskedSelect() creating: createMaskedSelect ''' @@ -1700,8 +1801,10 @@ class Max(Model): ''' Applies a max operation over dimension `dim` - :param dim max along this dimension - :param num_input_dims Optional. If in a batch model, set to the inputDims. + + :param dim: max along this dimension + :param num_input_dims: Optional. If in a batch model, set to the inputDims. + >>> max = Max(1) creating: createMax @@ -1726,10 +1829,10 @@ class Mean(Model): user need to specify the number of dimensions of each sample tensor in the batch using nInputDims. + :param dimension: the dimension to be applied mean operation - :param n_input_dims: specify the number of dimensions that this module will receive - If it is more than the dimension of input tensors, the first dimension would be considered - as batch size + :param n_input_dims: specify the number of dimensions that this module will receiveIf it is more than the dimension of input tensors, the first dimension would be consideredas batch size + >>> mean = Mean(1, 1) creating: createMean @@ -1749,8 +1852,10 @@ class Min(Model): ''' Applies a min operation over dimension `dim`. - :param dim min along this dimension - :param num_input_dims Optional. If in a batch model, set to the input_dim. + + :param dim: min along this dimension + :param num_input_dims: Optional. If in a batch model, set to the input_dim. + >>> min = Min(1) creating: createMin @@ -1774,6 +1879,7 @@ class MixtureTable(Model): should take the form of a table of Tensors. This Module works for experts of dimension 1D or more, and for a 1D or 2D gater, i.e. for single examples or mini-batches. + >>> mixtureTable = MixtureTable() creating: createMixtureTable >>> mixtureTable = MixtureTable(10) @@ -1791,6 +1897,7 @@ class Mul(Model): ''' Multiply a single scalar factor to the incoming data + >>> mul = Mul() creating: createMul ''' @@ -1810,8 +1917,10 @@ class MulConstant(Model): Multiplies input Tensor by a (non-learnable) scalar constant. This module is sometimes useful for debugging purposes. - :param scalar scalar constant - :param inplace Can optionally do its operation in-place without using extra state memory + + :param scalar: scalar constant + :param inplace: Can optionally do its operation in-place without using extra state memory + >>> mulConstant = MulConstant(2.5) creating: createMulConstant @@ -1831,6 +1940,7 @@ class Narrow(Model): ''' Narrow is application of narrow operation in a module. The module further supports a negative length in order to handle inputs with an unknown size. + >>> narrow = Narrow(1, 1, 1) creating: createNarrow ''' @@ -1854,8 +1964,10 @@ class NarrowTable(Model): a table or a Tensor. If `length` is negative, it means selecting the elements from the offset to element which located at the abs(`length`) to the last element of the input. - :param offset the start index of table - :param length the length want to select + + :param offset: the start index of table + :param length: the length want to select + >>> narrowTable = NarrowTable(1, 1) creating: createNarrowTable @@ -1877,6 +1989,7 @@ class Normalize(Model): division by zero when the input contains all zero elements (default = 1e-10). p can be the max value of double + >>> normalize = Normalize(1e-5, 1e-5) creating: createNormalize ''' @@ -1895,14 +2008,19 @@ class PReLU(Model): ''' Applies parametric ReLU, which parameter varies the slope of the negative part. + PReLU: f(x) = max(0, x) + a * min(0, x) + nOutputPlane's default value is 0, that means using PReLU in shared version and has only one parameters. + Notice: Please don't use weight decay on this. - :param n_output_plane input map number. Default is 0. + + :param n_output_plane: input map number. Default is 0. + >>> pReLU = PReLU(1) creating: createPReLU @@ -1925,17 +2043,18 @@ class Padding(Model): This module adds pad units of padding to dimension dim of the input. If pad is negative, padding is added to the left, otherwise, it is added to the right of the dimension. + The input to this layer is expected to be a tensor, or a batch of tensors; when using mini-batch, a batch of sample tensors will be passed to the layer and the user need to specify the number of dimensions of each sample tensor in the batch using n_input_dim. - :param dim the dimension to be applied padding operation - :param pad num of the pad units - :param n_input_dim specify the number of dimensions that this module will receive - If it is more than the dimension of input tensors, the first dimension - would be considered as batch size - :param value padding value + + :param dim: the dimension to be applied padding operation + :param pad: num of the pad units + :param n_input_dim: specify the number of dimensions that this module will receiveIf it is more than the dimension of input tensors, the first dimensionwould be considered as batch size + :param value: padding value + >>> padding = Padding(1, 1, 1, 1e-5, 1) creating: createPadding @@ -1966,7 +2085,9 @@ class PairwiseDistance(Model): it must have the size of `inputSize`. If it is a matrix, then each row is assumed to be an input sample of the given batch (the number of rows means the batch size and the number of columns should be equal to the `inputSize`). - :param norm the norm of distance + + :param norm: the norm of distance + >>> pairwiseDistance = PairwiseDistance(2) creating: createPairwiseDistance @@ -1985,6 +2106,7 @@ class ParallelTable(Container): It is a container module that applies the i-th member module to the i-th input, and outputs an output in the form of Table + >>> parallelTable = ParallelTable() creating: createParallelTable ''' @@ -1999,10 +2121,12 @@ class Power(Model): ''' Apply an element-wise power operation with scale and shift. f(x) = (shift + scale * x)^power^ + :param power: the exponent. :param scale: Default is 1. :param shift: Default is 0. + >>> power = Power(1e-5) creating: createPower ''' @@ -2024,28 +2148,37 @@ class RReLU(Model): Applies the randomized leaky rectified linear unit (RReLU) element-wise to the input Tensor, thus outputting a Tensor of the same dimension. Informally the RReLU is also known as 'insanity' layer. RReLU is defined as: +``` f(x) = max(0,x) + a * min(0, x) where a ~ U(l, u). +``` In training mode negative inputs are multiplied by a factor a drawn from a uniform random distribution U(l, u). + In evaluation mode a RReLU behaves like a LeakyReLU with a constant mean factor a = (l + u) / 2. + By default, l = 1/8 and u = 1/3. If l == u a RReLU effectively becomes a LeakyReLU. + Regardless of operating in in-place mode a RReLU will internally allocate an input-sized noise tensor to store random factors for negative inputs. + The backward() operation assumes that forward() has been called before. + For reference see [Empirical Evaluation of Rectified Activations in Convolutional Network]( http://arxiv.org/abs/1505.00853). + :param lower: lower boundary of uniform random distribution :param upper: upper boundary of uniform random distribution :param inplace: optionally do its operation in-place without using extra state memory + >>> rReLU = RReLU(1e-5, 1e5, True) creating: createRReLU ''' @@ -2066,7 +2199,9 @@ class ReLU6(Model): ''' Same as ReLU except that the rectifying function f(x) saturates at x = 6 - :param inplace either True = in-place or False = keeping separate state + + :param inplace: either True = in-place or False = keeping separate state + >>> reLU6 = ReLU6(True) creating: createReLU6 @@ -2085,10 +2220,12 @@ class Replicate(Model): Replicate repeats input `nFeatures` times along its `dim` dimension. Notice: No memory copy, it set the stride along the `dim`-th dimension to zero. + :param n_features: replicate times. :param dim: dimension to be replicated. :param n_dim: specify the number of non-batch dimensions. + >>> replicate = Replicate(2) creating: createReplicate ''' @@ -2117,9 +2254,11 @@ class RoiPooling(Model): into the corresponding output grid cell. Pooling is applied independently to each feature map channel + :param pooled_w: spatial extent in width :param pooled_h: spatial extent in height - :param spatial_scale spatial scale + :param spatial_scale: spatial scale + >>> roiPooling = RoiPooling(1, 1, 1e-5) creating: createRoiPooling @@ -2144,7 +2283,9 @@ class Scale(Model): match the shape of the input. Similarly, perform a expand cdd bias and perform an elementwise add - :param size size of weight and bias + + :param size: size of weight and bias + >>> scale = Scale([1,2]) creating: createScale @@ -2166,7 +2307,9 @@ class SelectTable(Model): This is true regardless of the depth of the encapsulated Tensor as the function used internally to do so is recursive. - :param dimension the dimension to be selected + + :param dimension: the dimension to be selected + >>> selectTable = SelectTable(1) creating: createSelectTable @@ -2184,6 +2327,7 @@ class Sigmoid(Model): ''' Applies the Sigmoid function element-wise to the input Tensor, thus outputting a Tensor of the same dimension. + >>> sigmoid = Sigmoid() creating: createSigmoid ''' @@ -2201,6 +2345,7 @@ class SoftMax(Model): Softmax is defined as: f_i(x) = exp(x_i - shift) / sum_j exp(x_j - shift) where shift = max_i(x_i). + >>> softMax = SoftMax() creating: createSoftMax ''' @@ -2218,6 +2363,7 @@ class SoftMin(Model): Softmin is defined as: f_i(x) = exp(-x_i - shift) / sum_j exp(-x_j - shift) where shift = max_i(-x_i). + >>> softMin = SoftMin() creating: createSoftMin ''' @@ -2233,7 +2379,9 @@ class SoftPlus(Model): Apply the SoftPlus function to an n-dimensional input tensor. SoftPlus function: f_i(x) = 1/beta * log(1 + exp(beta * x_i)) - :param beta Controls sharpness of transfer function + + :param beta: Controls sharpness of transfer function + >>> softPlus = SoftPlus(1e-5) creating: createSoftPlus @@ -2251,12 +2399,16 @@ class SoftShrink(Model): ''' Apply the soft shrinkage function element-wise to the input Tensor + SoftShrinkage operator: +``` | x - lambda, if x > lambda f(x) = | x + lambda, if x < -lambda | 0, otherwise +``` + + :param the_lambda: lambda, default is 0.5 - :param the_lambda lambda, default is 0.5 >>> softShrink = SoftShrink(1e-5) creating: createSoftShrink @@ -2274,8 +2426,10 @@ class SoftSign(Model): ''' Apply SoftSign function to an n-dimensional input Tensor. + SoftSign function: f_i(x) = x_i / (1+|x_i|) + >>> softSign = SoftSign() creating: createSoftSign ''' @@ -2290,15 +2444,19 @@ class SpatialDilatedConvolution(Model): ''' Apply a 2D dilated convolution over an input image. + The input tensor is expected to be a 3D or 4D(with batch) tensor. + If input is a 3D tensor nInputPlane x height x width, owidth = floor(width + 2 * padW - dilationW * (kW-1) - 1) / dW + 1 oheight = floor(height + 2 * padH - dilationH * (kH-1) - 1) / dH + 1 + Reference Paper: Yu F, Koltun V. Multi-scale context aggregation by dilated convolutions[J]. arXiv preprint arXiv:1511.07122, 2015. + :param n_input_plane: The number of expected input planes in the image given into forward(). :param n_output_plane: The number of output planes the convolution layer will produce. :param kw: The kernel width of the convolution. @@ -2311,6 +2469,7 @@ class SpatialDilatedConvolution(Model): :param dilation_h: The number of pixels to skip. Default is 1. :param init_method: Init method, Default, Xavier. + >>> spatialDilatedConvolution = SpatialDilatedConvolution(1, 1, 1, 1) creating: createSpatialDilatedConvolution ''' @@ -2356,30 +2515,35 @@ class SpatialFullConvolution(Model): adjH values used to construct the module). This module can be used without a bias by setting parameter noBias = true while constructing the module. + If input is a 3D tensor nInputPlane x height x width, owidth = (width - 1) * dW - 2*padW + kW + adjW oheight = (height - 1) * dH - 2*padH + kH + adjH + Other frameworks call this operation "In-network Upsampling", "Fractionally-strided convolution", "Backwards Convolution," "Deconvolution", or "Upconvolution." + Reference Paper: Long J, Shelhamer E, Darrell T. Fully convolutional networks for semantic segmentation[C]//Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition. 2015: 3431-3440. - :param nInputPlane The number of expected input planes in the image given into forward() - :param nOutputPlane The number of output planes the convolution layer will produce. - :param kW The kernel width of the convolution. - :param kH The kernel height of the convolution. - :param dW The step of the convolution in the width dimension. Default is 1. - :param dH The step of the convolution in the height dimension. Default is 1. - :param padW The additional zeros added per width to the input planes. Default is 0. - :param padH The additional zeros added per height to the input planes. Default is 0. - :param adjW Extra width to add to the output image. Default is 0. - :param adjH Extra height to add to the output image. Default is 0. - :param nGroup Kernel group number. - :param noBias If bias is needed. - :param initMethod Init method, Default, Xavier, Bilinear. + + :param nInputPlane: The number of expected input planes in the image given into forward() + :param nOutputPlane: The number of output planes the convolution layer will produce. + :param kW: The kernel width of the convolution. + :param kH: The kernel height of the convolution. + :param dW: The step of the convolution in the width dimension. Default is 1. + :param dH: The step of the convolution in the height dimension. Default is 1. + :param padW: The additional zeros added per width to the input planes. Default is 0. + :param padH: The additional zeros added per height to the input planes. Default is 0. + :param adjW: Extra width to add to the output image. Default is 0. + :param adjH: Extra height to add to the output image. Default is 0. + :param nGroup: Kernel group number. + :param noBias: If bias is needed. + :param initMethod: Init method, Default, Xavier, Bilinear. + >>> spatialFullConvolution = SpatialFullConvolution(1, 1, 1, 1) creating: createSpatialFullConvolution @@ -2422,6 +2586,7 @@ def set_init_method(self, weight_init_method = None, bias_init_method = None): class SpatialShareConvolution(Model): ''' + >>> spatialShareConvolution = SpatialShareConvolution(1, 1, 1, 1) creating: createSpatialShareConvolution ''' @@ -2461,20 +2626,21 @@ class VolumetricConvolution(Model): ''' Applies a 3D convolution over an input image composed of several input planes. The input tensor in forward(input) is expected to be a 4D tensor (nInputPlane x time x height x width). - :param n_input_plane The number of expected input planes in the image given into forward() - :param n_output_plane The number of output planes the convolution layer will produce. - :param k_t The kernel size of the convolution in time - :param k_w The kernel width of the convolution - :param k_h The kernel height of the convolution - :param d_t The step of the convolution in the time dimension. Default is 1 - :param d_w The step of the convolution in the width dimension. Default is 1 - :param d_h The step of the convolution in the height dimension. Default is 1 - :param pad_t Additional zeros added to the input plane data on both sides of time axis. - Default is 0. (kT-1)/2 is often used here. - :param pad_w The additional zeros added per width to the input planes. - :param pad_h The additional zeros added per height to the input planes. - :param with_bias whether with bias - :param init_method Init method, Default, Xavier, Bilinear. + + :param n_input_plane: The number of expected input planes in the image given into forward() + :param n_output_plane: The number of output planes the convolution layer will produce. + :param k_t: The kernel size of the convolution in time + :param k_w: The kernel width of the convolution + :param k_h: The kernel height of the convolution + :param d_t: The step of the convolution in the time dimension. Default is 1 + :param d_w: The step of the convolution in the width dimension. Default is 1 + :param d_h: The step of the convolution in the height dimension. Default is 1 + :param pad_t: Additional zeros added to the input plane data on both sides of time axis.Default is 0. (kT-1)/2 is often used here. + :param pad_w: The additional zeros added per width to the input planes. + :param pad_h: The additional zeros added per height to the input planes. + :param with_bias: whether with bias + :param init_method: Init method, Default, Xavier, Bilinear. + >>> volumetricConvolution = VolumetricConvolution(6, 12, 5, 5, 5, 1, 1, 1) creating: createVolumetricConvolution @@ -2522,15 +2688,17 @@ class VolumetricMaxPooling(Model): The number of output features is equal to the number of input planes / dT. The input can optionally be padded with zeros. Padding should be smaller than half of kernel size. That is, padT < kT/2, padW < kW/2 and padH < kH/2 - :param k_t The kernel size - :param k_w The kernel width - :param k_h The kernel height - :param d_t The step in the time dimension - :param d_w The step in the width dimension - :param d_h The step in the height dimension - :param pad_t The padding in the time dimension - :param pad_w The padding in the width dimension - :param pad_h The padding in the height dimension + + :param k_t: The kernel size + :param k_w: The kernel width + :param k_h: The kernel height + :param d_t: The step in the time dimension + :param d_w: The step in the width dimension + :param d_h: The step in the height dimension + :param pad_t: The padding in the time dimension + :param pad_w: The padding in the width dimension + :param pad_h: The padding in the height dimension + >>> volumetricMaxPooling = VolumetricMaxPooling(5, 5, 5, 1, 1, 1) creating: createVolumetricMaxPooling @@ -2564,11 +2732,13 @@ class SpatialZeroPadding(Model): ''' Each feature map of a given input is padded with specified number of zeros. If padding values are negative, then input is cropped. + :param padLeft: pad left position :param padRight: pad right position :param padTop: pad top position :param padBottom: pad bottom position + >>> spatialZeroPadding = SpatialZeroPadding(1, 1, 1, 1) creating: createSpatialZeroPadding ''' @@ -2593,15 +2763,16 @@ class SplitTable(Model): outputs several tables, splitting the Tensor along the specified dimension `dimension`. + The input to this layer is expected to be a tensor, or a batch of tensors; when using mini-batch, a batch of sample tensors will be passed to the layer and the user need to specify the number of dimensions of each sample tensor in a batch using `nInputDims`. + :param dimension: to be split along this dimension - :param n_input_dims: specify the number of dimensions that this module will receive - If it is more than the dimension of input tensors, the first dimension - would be considered as batch size + :param n_input_dims: specify the number of dimensions that this module will receiveIf it is more than the dimension of input tensors, the first dimensionwould be considered as batch size + >>> splitTable = SplitTable(1, 1) creating: createSplitTable @@ -2621,6 +2792,7 @@ class Sqrt(Model): ''' Apply an element-wise sqrt operation. + >>> sqrt = Sqrt() creating: createSqrt ''' @@ -2634,6 +2806,7 @@ class Square(Model): ''' Apply an element-wise square operation. + >>> square = Square() creating: createSquare ''' @@ -2648,8 +2821,11 @@ class Squeeze(Model): ''' Delete singleton all dimensions or a specific dim. - :param dim Optional. The dimension to be delete. Default: delete all dimensions. - :param num_input_dims Optional. If in a batch model, set to the inputDims. + + :param dim: Optional. The dimension to be delete. Default: delete all dimensions. + :param num_input_dims: Optional. If in a batch model, set to the inputDims. + + >>> squeeze = Squeeze(1) @@ -2676,11 +2852,11 @@ class Sum(Model): the user need to specify the number of dimensions of each sample tensor in the batch using `nInputDims`. - :param dimension the dimension to be applied sum operation - :param n_input_dims specify the number of dimensions that this module will receive - If it is more than the dimension of input tensors, the first dimension - would be considered as batch size - :param size_average default is false, if it is true, it will return the mean instead + + :param dimension: the dimension to be applied sum operation + :param n_input_dims: specify the number of dimensions that this module will receiveIf it is more than the dimension of input tensors, the first dimensionwould be considered as batch size + :param size_average: default is false, if it is true, it will return the mean instead + >>> sum = Sum(1, 1, True) creating: createSum @@ -2704,6 +2880,7 @@ class TanhShrink(Model): during the forward process: [f(x) = tanh(x) - 1] + >>> tanhShrink = TanhShrink() creating: createTanhShrink ''' @@ -2719,9 +2896,11 @@ class Threshold(Model): Threshold input Tensor. If values in the Tensor smaller than th, then replace it with v - :param th the threshold to compare with - :param v the value to replace with - :param ip inplace mode + + :param th: the threshold to compare with + :param v: the value to replace with + :param ip: inplace mode + >>> threshold = Threshold(1e-5, 1e-5, True) creating: createThreshold @@ -2745,8 +2924,10 @@ class Unsqueeze(Model): For an input with dim = input.dim(), there are dim + 1 possible positions to insert the singleton dimension. - :param pos The position will be insert singleton. - :param num_input_dims Optional. If in a batch model, set to the inputDim + + :param pos: The position will be insert singleton. + :param num_input_dims: Optional. If in a batch model, set to the inputDim + >>> unsqueeze = Unsqueeze(1, 1) creating: createUnsqueeze @@ -2766,8 +2947,10 @@ class Reshape(Model): The forward(input) reshape the input tensor into a size(0) * size(1) * ... tensor, taking the elements row-wise. + :param size: the reshape size + >>> reshape = Reshape([1, 28, 28]) creating: createReshape >>> reshape = Reshape([1, 28, 28], False) @@ -2782,8 +2965,10 @@ class BiRecurrent(Container): ''' Create a Bidirectional recurrent layer + :param merge: merge layer + >>> biRecurrent = BiRecurrent(CAddTable()) creating: createCAddTable creating: createBiRecurrent @@ -2802,9 +2987,11 @@ class ConcatTable(Container): ConcateTable is a container module like Concate. Applies an input to each member module, input can be a tensor or a table. + ConcateTable usually works with CAddTable and CMulTable to implement element wise add/multiply on outputs of two modules. + >>> concatTable = ConcatTable() creating: createConcatTable ''' @@ -2818,6 +3005,7 @@ class Identity(Model): Identity just return the input to output. It's useful in same parallel container to get an origin input. + >>> identity = Identity() creating: createIdentity ''' @@ -2832,7 +3020,9 @@ class Reverse(Model): Reverse the input w.r.t given dimension. The input can be a Tensor or Table. - :param dim + + :param dim: + >>> reverse = Reverse() creating: createReverse @@ -2849,7 +3039,9 @@ class Transpose(Model): ''' Transpose input along specified dimensions - :param permutations dimension pairs that need to swap + + :param permutations: dimension pairs that need to swap + >>> transpose = Transpose([(1,2)]) creating: createTranspose @@ -2866,10 +3058,12 @@ class SpatialContrastiveNormalization(Model): ''' Subtractive + divisive contrast normalization. - :param n_input_plane - :param kernel - :param threshold - :param thresval + + :param n_input_plane: + :param kernel: + :param threshold: + :param thresval: + >>> kernel = np.ones([9,9]).astype("float32") >>> spatialContrastiveNormalization = SpatialContrastiveNormalization(1, kernel) @@ -2897,6 +3091,7 @@ class SpatialConvolutionMap(Model): It uses a generic connection table between input and output features. The SpatialConvolution is equivalent to using a full connection table. + >>> ct = np.ones([9,9]).astype("float32") >>> spatialConvolutionMap = SpatialConvolutionMap(ct, 9, 9) creating: createSpatialConvolutionMap @@ -2929,19 +3124,23 @@ class SpatialDivisiveNormalization(Model): an input image, since there is only one feature, the region is only spatial. For an RGB image, the weighted average is taken over RGB channels and a spatial region. + If the kernel is 1D, then it will be used for constructing and separable 2D kernel. The operations will be much more efficient in this case. + The kernel is generally chosen as a gaussian when it is believed that the correlation of two pixel locations decrease with increasing distance. On the feature dimension, a uniform average is used since the weighting across features is not known. - :param nInputPlane number of input plane, default is 1. - :param kernel kernel tensor, default is a 9 x 9 tensor. - :param threshold threshold - :param thresval threshhold value to replace with - if data is smaller than theshold + + + :param nInputPlane: number of input plane, default is 1. + :param kernel: kernel tensor, default is a 9 x 9 tensor. + :param threshold: threshold + :param thresval: threshhold value to replace withif data is smaller than theshold + >>> kernel = np.ones([9,9]).astype("float32") >>> spatialDivisiveNormalization = SpatialDivisiveNormalization(2,kernel) @@ -2967,18 +3166,21 @@ class Graph(Model): A graph container. Each node can have multiple inputs. The output of the node should be a tensor. The output tensor can be connected to multiple nodes. So the module in each node can have a tensor or table input, and should have a tensor output. - - The graph container can have multiple inputs and multiple outputs. If there's one input, - the input data fed to the graph module should be a tensor. If there're multiple inputs, - the input data fed to the graph module should be a table, which is actually an sequence of - tensor. The order of the input tensors should be same with the order of the input nodes. + + + The graph container can have multiple inputs and multiple outputs. If there's one input, + the input data fed to the graph module should be a tensor. If there're multiple inputs, + the input data fed to the graph module should be a table, which is actually an sequence of + tensor. The order of the input tensors should be same with the order of the input nodes. This is also applied to the gradient from the module in the back propagation. - + + If there's one output, the module output is a tensor. If there're multiple outputs, the module output is a table, which is actually an sequence of tensor. The order of the output tensors is same with the order of the output modules. This is also applied to the gradient passed to the module in the back propagation. - + + All inputs should be able to connect to outputs through some paths in the graph. It is allowed that some successors of the inputs node are not connect to outputs. If so, these nodes will be excluded in the computation. @@ -2998,15 +3200,19 @@ class SpatialSubtractiveNormalization(Model): an input image, since there is only one feature, the region is only spatial. For an RGB image, the weighted average is taken over RGB channels and a spatial region. + If the kernel is 1D, then it will be used for constructing and separable 2D kernel. The operations will be much more efficient in this case. + The kernel is generally chosen as a gaussian when it is believed that the correlation of two pixel locations decrease with increasing distance. On the feature dimension, a uniform average is used since the weighting across features is not known. - :param n_input_plane number of input plane, default is 1. - :param kernel kernel tensor, default is a 9 x 9 tensor. + + :param n_input_plane: number of input plane, default is 1. + :param kernel: kernel tensor, default is a 9 x 9 tensor. + >>> kernel = np.ones([9,9]).astype("float32") >>> spatialSubtractiveNormalization = SpatialSubtractiveNormalization(2,kernel) diff --git a/python/dllib/src/bigdl/dllib/optim/optimizer.py b/python/dllib/src/bigdl/dllib/optim/optimizer.py index e868c1473bd..f4e095fb293 100644 --- a/python/dllib/src/bigdl/dllib/optim/optimizer.py +++ b/python/dllib/src/bigdl/dllib/optim/optimizer.py @@ -33,11 +33,12 @@ class MaxIteration(JavaValue): """ - A trigger specifies a timespot or several timespots during training, - and a corresponding action will be taken when the timespot(s) is reached. - MaxIteration is a trigger that triggers an action when training reaches - the number of iterations specified by "max". - Usually used as end_trigger when creating an Optimizer. + A trigger specifies a timespot or several timespots during training, + and a corresponding action will be taken when the timespot(s) is reached. + MaxIteration is a trigger that triggers an action when training reaches + the number of iterations specified by "max". + Usually used as end_trigger when creating an Optimizer. + >>> maxIteration = MaxIteration(20) creating: createMaxIteration @@ -46,6 +47,7 @@ def __init__(self, max, bigdl_type="float"): """ Create a MaxIteration trigger. + :param max: max """ JavaValue.__init__(self, None, bigdl_type, max) @@ -59,13 +61,15 @@ class MaxEpoch(JavaValue): the number of epochs specified by "max_epoch". Usually used as end_trigger when creating an Optimizer. + >>> maxEpoch = MaxEpoch(2) creating: createMaxEpoch """ def __init__(self, max_epoch, bigdl_type="float"): """ Create a MaxEpoch trigger. - + + :param max_epoch: max_epoch """ JavaValue.__init__(self, None, bigdl_type, max_epoch) @@ -79,12 +83,13 @@ class EveryEpoch(JavaValue): Could be used as trigger in setvalidation and setcheckpoint in Optimizer, and also in TrainSummary.set_summary_trigger. + >>> everyEpoch = EveryEpoch() creating: createEveryEpoch """ def __init__(self, bigdl_type="float"): """ - Create a EveryEpoch trigger. + Create a EveryEpoch trigger. """ JavaValue.__init__(self, None, bigdl_type) @@ -93,11 +98,12 @@ class SeveralIteration(JavaValue): """ A trigger specifies a timespot or several timespots during training, and a corresponding action will be taken when the timespot(s) is reached. - SeveralIteration is a trigger that triggers an action every "n" + SeveralIteration is a trigger that triggers an action every "n" iterations. Could be used as trigger in setvalidation and setcheckpoint in Optimizer, and also in TrainSummary.set_summary_trigger. + >>> serveralIteration = SeveralIteration(2) creating: createSeveralIteration """ @@ -105,20 +111,22 @@ def __init__(self, interval, bigdl_type="float"): """ Create a SeveralIteration trigger. - :param interval: interval is the "n" where an action is triggered - every "n" iterations + + :param interval: interval is the "n" where an action is triggeredevery "n" iterations """ JavaValue.__init__(self, None, bigdl_type, interval) class Poly(JavaValue): """ - A learning rate decay policy, where the effective learning rate - follows a polynomial decay, to be zero by the max_iteration. + A learning rate decay policy, where the effective learning rate + follows a polynomial decay, to be zero by the max_iteration. Calculation: base_lr (1 - iter/max_iteration) ^ (power) - - :param power - :param max_iteration + + + :param power: + :param max_iteration: + >>> poly = Poly(0.5, 2) creating: createPoly @@ -129,11 +137,13 @@ def __init__(self, power, max_iteration, bigdl_type="float"): class Step(JavaValue): """ - A learning rate decay policy, where the effective learning rate is + A learning rate decay policy, where the effective learning rate is calculated as base_lr * gamma ^ (floor(iter / step_size)) - - :param step_size - :param gamma + + + :param step_size: + :param gamma: + >>> step = Step(2, 0.3) creating: createStep @@ -321,8 +331,10 @@ class MultiStep(JavaValue): """ similar to step but it allows non uniform steps defined by stepSizes - :param step_size the series of step sizes used for lr decay - :param gamma coefficient of decay + + :param step_size: the series of step sizes used for lr decay + :param gamma: coefficient of decay + >>> step = MultiStep([2, 5], 0.3) creating: createMultiStep @@ -333,10 +345,10 @@ def __init__(self, step_sizes, gamma, bigdl_type="float"): class Optimizer(JavaValue): """ - An optimizer is in general to minimize any function with respect - to a set of parameters. In case of training a neural network, - an optimizer tries to minimize the loss of the neural net with - respect to its weights/biases, over the training set. + An optimizer is in general to minimize any function with respect + to a set of parameters. In case of training a neural network, + an optimizer tries to minimize the loss of the neural net with + respect to its weights/biases, over the training set. """ def __init__(self, model, @@ -349,34 +361,36 @@ def __init__(self, """ Create an optimizer. + :param model: the neural net model - :param traiing_rdd: the training dataset + :param traiing_rdd: the training dataset :param criterion: the loss function :param optim_method: the algorithm to use for optimization, e.g. SGD, Adagrad, etc. If optim_method is None, the default algorithm is SGD. :param end_trigger: when to end the optimization :param batch_size: training batch size - """ + """ JavaValue.__init__(self, None, bigdl_type, model.value, training_rdd, criterion, optim_method if optim_method else SGD(), end_trigger, batch_size) def set_validation(self, batch_size, val_rdd, trigger, val_method=["Top1Accuracy"]): """ - Configure validation settings. + Configure validation settings. + :param batch_size: validation batch size :param val_rdd: validation dataset :param trigger: validation interval - :param val_method: the ValidationMethod to use, - e.g. "Top1Accuracy", "Top5Accuracy", "Loss" + :param val_method: the ValidationMethod to use,e.g. "Top1Accuracy", "Top5Accuracy", "Loss" """ callBigDlFunc(self.bigdl_type, "setValidation", self.value, batch_size, trigger, val_rdd, val_method) def set_model(self, model): """ - Set model. + Set model. + :param model: new model """ @@ -385,12 +399,12 @@ def set_model(self, model): def set_checkpoint(self, checkpoint_trigger, checkpoint_path, isOverWrite=True): """ - Configure checkpoint settings. + Configure checkpoint settings. + :param checkpoint_trigger: the interval to write snapshots :param checkpoint_path: the path to write snapshots into - :param isOverWrite: whether to overwrite existing snapshots in path. - default is True + :param isOverWrite: whether to overwrite existing snapshots in path.default is True """ if not os.path.exists(checkpoint_path): mkpath(checkpoint_path) @@ -400,7 +414,7 @@ def set_checkpoint(self, checkpoint_trigger, # return a module def optimize(self): """ - Do an optimization. + Do an optimization. """ jmodel = callJavaFunc(get_spark_context(), self.value.optimize) from bigdl.nn.layer import Model @@ -408,11 +422,12 @@ def optimize(self): def set_train_summary(self, summary): """ - Set train summary. A TrainSummary object contains information - necesary for the optimizer to know how often the logs are recorded, - where to store the logs and how to retrieve them, etc. For details, + Set train summary. A TrainSummary object contains information + necesary for the optimizer to know how often the logs are recorded, + where to store the logs and how to retrieve them, etc. For details, refer to the docs of TrainSummary. + :param summary: a TrainSummary object """ callBigDlFunc(self.bigdl_type, "setTrainSummary", self.value, @@ -421,13 +436,15 @@ def set_train_summary(self, summary): def set_val_summary(self, summary): """ - Set validation summary. A ValidationSummary object contains information - necesary for the optimizer to know how often the logs are recorded, - where to store the logs and how to retrieve them, etc. For details, + Set validation summary. A ValidationSummary object contains information + necesary for the optimizer to know how often the logs are recorded, + where to store the logs and how to retrieve them, etc. For details, refer to the docs of ValidationSummary. + :param summary: a ValidationSummary object + """ callBigDlFunc(self.bigdl_type, "setValSummary", self.value, summary) @@ -436,7 +453,7 @@ def set_val_summary(self, summary): def prepare_input(self): """ Load input. Notebook user can call this method to seprate load data and - create optimizer time + create optimizer time """ print("Loading input ...") self.value.prepareInput() @@ -444,22 +461,24 @@ def prepare_input(self): class TrainSummary(JavaValue, ): """ - A logging facility which allows user to trace how indicators (e.g. - learning rate, training loss, throughput, etc.) change with iterations/time - in an optimization process. TrainSummary is for training indicators only - (check ValidationSummary for validation indicators). It contains necessary - information for the optimizer to know where to store the logs, how to - retrieve the logs, and so on. - The logs are written in tensorflow-compatible - format so that they can be visualized directly using tensorboard. Also the - logs can be retrieved as ndarrays and visualized using python libraries - such as matplotlib (in notebook, etc.). - - Use optimizer.setTrainSummary to enable train logger. + A logging facility which allows user to trace how indicators (e.g. + learning rate, training loss, throughput, etc.) change with iterations/time + in an optimization process. TrainSummary is for training indicators only + (check ValidationSummary for validation indicators). It contains necessary + information for the optimizer to know where to store the logs, how to + retrieve the logs, and so on. - The logs are written in tensorflow-compatible + format so that they can be visualized directly using tensorboard. Also the + logs can be retrieved as ndarrays and visualized using python libraries + such as matplotlib (in notebook, etc.). + + + Use optimizer.setTrainSummary to enable train logger. """ def __init__(self, log_dir, app_name, bigdl_type="float"): """ Create a TrainSummary. Logs will be saved to log_dir/app_name/train. + :param log_dir: the root dir to store the logs :param app_name: the application name """ @@ -467,28 +486,21 @@ def __init__(self, log_dir, app_name, bigdl_type="float"): def read_scalar(self, tag): """ - Retrieve train logs by type. Return an array of records in the format + Retrieve train logs by type. Return an array of records in the format (step,value,wallClockTime). - "Step" is the iteration count by default. - - :param tag: the type of the logs, Supported tags are: "LearningRate", - "Loss", "Throughput" + + + :param tag: the type of the logs, Supported tags are: "LearningRate","Loss", "Throughput" """ return callBigDlFunc(self.bigdl_type, "summaryReadScalar", self.value, tag) def set_summary_trigger(self, name, trigger): """ - Set the interval of recording for each indicator. - - :param tag: tag name. Supported tag names are "LearningRate", "Loss", - "Throughput", "Parameters". "Parameters" is an umbrella tag that - includes weight, bias, gradWeight, gradBias, and some running status - (eg. runningMean and runningVar in BatchNormalization). If you - didn't set any triggers, we will by default record Loss and Throughput - in each iteration, while *NOT* recording LearningRate and Parameters, - as recording parameters may introduce substantial overhead when the - model is very big, LearningRate is not a public attribute for all - OptimMethod. + Set the interval of recording for each indicator. + + + :param tag: tag name. Supported tag names are "LearningRate", "Loss","Throughput", "Parameters". "Parameters" is an umbrella tag thatincludes weight, bias, gradWeight, gradBias, and some running status(eg. runningMean and runningVar in BatchNormalization). If youdidn't set any triggers, we will by default record Loss and Throughputin each iteration, while *NOT* recording LearningRate and Parameters,as recording parameters may introduce substantial overhead when themodel is very big, LearningRate is not a public attribute for allOptimMethod. :param trigger: trigger """ return callBigDlFunc(self.bigdl_type, "summarySetTrigger", self.value, @@ -497,25 +509,27 @@ def set_summary_trigger(self, name, trigger): class ValidationSummary(JavaValue): """ - A logging facility which allows user to trace how indicators (e.g. - validation loss, top1 accuray, top5 accuracy etc.) change with - iterations/time in an optimization process. ValidationSummary is for - validation indicators only (check TrainSummary for train indicators). - It contains necessary information for the optimizer to know where to - store the logs, how to retrieve the logs, and so on. - The logs are - written in tensorflow-compatible format so that they can be visualized - directly using tensorboard. Also the logs can be retrieved as ndarrays - and visualized using python libraries such as matplotlib + A logging facility which allows user to trace how indicators (e.g. + validation loss, top1 accuray, top5 accuracy etc.) change with + iterations/time in an optimization process. ValidationSummary is for + validation indicators only (check TrainSummary for train indicators). + It contains necessary information for the optimizer to know where to + store the logs, how to retrieve the logs, and so on. - The logs are + written in tensorflow-compatible format so that they can be visualized + directly using tensorboard. Also the logs can be retrieved as ndarrays + and visualized using python libraries such as matplotlib (in notebook, etc.). - + + Use optimizer.setValidationSummary to enable validation logger. """ def __init__(self, log_dir, app_name, bigdl_type="float"): """ - Create a ValidationSummary. Logs will be saved to - log_dir/app_name/train. By default, all ValidationMethod set into - optimizer will be recorded and the recording interval is the same - as trigger of ValidationMethod in the optimizer. + Create a ValidationSummary. Logs will be saved to + log_dir/app_name/train. By default, all ValidationMethod set into + optimizer will be recorded and the recording interval is the same + as trigger of ValidationMethod in the optimizer. + :param log_dir: the root dir to store the logs :param app_name: the application name @@ -524,13 +538,12 @@ def __init__(self, log_dir, app_name, bigdl_type="float"): def read_scalar(self, tag): """ - Retrieve validation logs by type. Return an array of records in the - format (step,value,wallClockTime). - "Step" is the iteration count + Retrieve validation logs by type. Return an array of records in the + format (step,value,wallClockTime). - "Step" is the iteration count by default. - :param tag: the type of the logs. The tag should match the name of - the ValidationMethod set into the optimizer. e.g. - "Top1AccuracyLoss","Top1Accuracy" or "Top5Accuracy". + + :param tag: the type of the logs. The tag should match the name ofthe ValidationMethod set into the optimizer. e.g."Top1AccuracyLoss","Top1Accuracy" or "Top5Accuracy". """ return callBigDlFunc(self.bigdl_type, "summaryReadScalar", self.value, tag) diff --git a/python/dllib/src/bigdl/dllib/utils/common.py b/python/dllib/src/bigdl/dllib/utils/common.py index 1839a62d8b8..f19e15df134 100644 --- a/python/dllib/src/bigdl/dllib/utils/common.py +++ b/python/dllib/src/bigdl/dllib/utils/common.py @@ -90,6 +90,7 @@ class TestResult(): """ def __init__(self, result, total_num, method): """ + :param result: the validation result. i.e: top1 accuracy percentage. :param total_num: the total processed records. :param method: the validation method. i.e: Top1Accuracy @@ -110,6 +111,7 @@ class JTensor(object): """ A wrapper to easy our work when need to pass or return Tensor to/from Scala. + >>> import numpy as np >>> from bigdl.util.common import JTensor >>> np.random.seed(123) @@ -124,6 +126,7 @@ def __init__(self, storage, shape, bigdl_type="float"): def from_ndarray(cls, a_ndarray, bigdl_type="float"): """ Convert a ndarray to Tensor which would be used in Java side. + >>> import numpy as np >>> from bigdl.util.common import JTensor >>> from bigdl.util.common import callBigDlFunc @@ -153,7 +156,9 @@ def get_dtype(): def flatten_ndarray(cls, a_ndarray): """ Utility method to flatten a ndarray - :return (storage, shape) + + :return: (storage, shape) + >>> from bigdl.util.common import JTensor >>> np.random.seed(123) >>> data = np.random.uniform(0, 1, (2, 3)) @@ -335,6 +340,7 @@ def callJavaFunc(sc, func, *args): def _to_java_object_rdd(rdd): """ Return a JavaRDD of Object by unpickling + It will convert each Python object into Java object by Pyrolite, whenever the RDD is serialized in batch or not. """ From 18fb08073e2c51e4cb462cd56e56129047a6ac90 Mon Sep 17 00:00:00 2001 From: Yuchao-Tao Date: Fri, 9 Jun 2017 17:42:17 +0800 Subject: [PATCH 048/823] transform python files in respect to comments. fix the format problem of ":return:" (#873) --- .../src/bigdl/dllib/feature/dataset/base.py | 13 +- .../src/bigdl/dllib/feature/dataset/mnist.py | 10 +- .../src/bigdl/dllib/feature/dataset/news20.py | 2 + .../src/bigdl/dllib/models/lenet/lenet5.py | 1 + python/dllib/src/bigdl/dllib/nn/criterion.py | 95 ++- python/dllib/src/bigdl/dllib/nn/layer.py | 608 ++++++++++++------ .../dllib/src/bigdl/dllib/optim/optimizer.py | 187 +++--- python/dllib/src/bigdl/utils/common.py | 8 +- 8 files changed, 604 insertions(+), 320 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/feature/dataset/base.py b/python/dllib/src/bigdl/dllib/feature/dataset/base.py index f211ee5261c..771786841c1 100644 --- a/python/dllib/src/bigdl/dllib/feature/dataset/base.py +++ b/python/dllib/src/bigdl/dllib/feature/dataset/base.py @@ -28,8 +28,9 @@ class Progbar(object): def __init__(self, target, width=30, verbose=1, interval=0.01): ''' - @param target: total number of steps expected - @param interval: minimum visual progress update interval (in seconds) + + :param target: total number of steps expected + :param interval: minimum visual progress update interval (in seconds) ''' self.width = width self.target = target @@ -44,10 +45,10 @@ def __init__(self, target, width=30, verbose=1, interval=0.01): def update(self, current, values=[], force=False): ''' - @param current: index of current step - @param values: list of tuples (name, value_for_last_step). - The progress bar will display averages for these values. - @param force: force visual progress update + + :param current: index of current step + :param values: list of tuples (name, value_for_last_step).The progress bar will display averages for these values. + :param force: force visual progress update ''' for k, v in values: if k not in self.sum_values: diff --git a/python/dllib/src/bigdl/dllib/feature/dataset/mnist.py b/python/dllib/src/bigdl/dllib/feature/dataset/mnist.py index f23087e8736..0af0f70c51f 100644 --- a/python/dllib/src/bigdl/dllib/feature/dataset/mnist.py +++ b/python/dllib/src/bigdl/dllib/feature/dataset/mnist.py @@ -38,15 +38,19 @@ def _read32(bytestream): def extract_images(f): """Extract the images into a 4D uint8 numpy array [index, y, x, depth]. + Args: f: A file object that can be passed into a gzip reader. + Returns: data: A 4D unit8 numpy array [index, y, x, depth]. + Raises: ValueError: If the bytestream does not start with 2051. + """ print('Extracting', f.name) with gzip.GzipFile(fileobj=f) as bytestream: @@ -81,9 +85,9 @@ def extract_labels(f): def read_data_sets(train_dir, data_type="train"): """ Parse or download mnist data if train_dir is empty. + :param train_dir: The directory storing the mnist data - :param data_type: Reading training set or testing set. - It can be either "train" or "test" + :param data_type: Reading training set or testing set.It can be either "train" or "test" :return: (ndarray, ndarray) representing (features, labels) features is a 4D unit8 numpy array [index, y, x, depth] representing each pixel valued from 0 to 255. labels @@ -126,4 +130,4 @@ def read_data_sets(train_dir, data_type="train"): assert numpy.abs(numpy.mean(train) - TRAIN_MEAN) / TRAIN_MEAN < 1e-7 assert numpy.abs(numpy.std(train) - TRAIN_STD) / TRAIN_STD < 1e-7 assert numpy.abs(numpy.mean(test) - TEST_MEAN) / TEST_MEAN < 1e-7 - assert numpy.abs(numpy.std(test) - TEST_STD) / TEST_STD < 1e-7 \ No newline at end of file + assert numpy.abs(numpy.std(test) - TEST_STD) / TEST_STD < 1e-7 diff --git a/python/dllib/src/bigdl/dllib/feature/dataset/news20.py b/python/dllib/src/bigdl/dllib/feature/dataset/news20.py index 3d066ad3b52..e5aac8b02e8 100644 --- a/python/dllib/src/bigdl/dllib/feature/dataset/news20.py +++ b/python/dllib/src/bigdl/dllib/feature/dataset/news20.py @@ -53,6 +53,7 @@ def download_glove_w2v(dest_dir): def get_news20(source_dir="/tmp/news20/"): """ Parse or download news20 if source_dir is empty. + :param source_dir: The directory storing news data. :return: A list of (tokens, label) """ @@ -81,6 +82,7 @@ def get_news20(source_dir="/tmp/news20/"): def get_glove_w2v(source_dir="/tmp/news20/", dim=100): """ Parse or download the pre-trained glove word2vec if source_dir is empty. + :param source_dir: The directory storing the pre-trained word2vec :param dim: The dimension of a vector :return: A dict mapping from word to vector diff --git a/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py b/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py index b3fcba58c9c..68891fb8f09 100644 --- a/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py +++ b/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py @@ -46,6 +46,7 @@ def get_minst(sc, data_type="train", location="/tmp/mnist"): """ Get and normalize the mnist data. We would download it automatically if the data doesn't present at the specific location. + :param sc: SparkContext :param data_type: training data or testing data :param location: Location storing the mnist diff --git a/python/dllib/src/bigdl/dllib/nn/criterion.py b/python/dllib/src/bigdl/dllib/nn/criterion.py index 63ddf6a5bf1..83f5f4113a1 100644 --- a/python/dllib/src/bigdl/dllib/nn/criterion.py +++ b/python/dllib/src/bigdl/dllib/nn/criterion.py @@ -46,6 +46,7 @@ def forward(self, input, target): NB: It's for debug only, please use optimizer.optimize() in production. Takes an input object, and computes the corresponding loss of the criterion, compared with `target` + :param input: ndarray or list of ndarray :param target: ndarray or list of ndarray :return: value of loss @@ -61,6 +62,7 @@ def backward(self, input, target): """ NB: It's for debug only, please use optimizer.optimize() in production. Performs a back-propagation step through the criterion, with respect to the given input. + :param input: ndarray or list of ndarray :param target: ndarray or list of ndarray :return: ndarray @@ -76,6 +78,7 @@ def backward(self, input, target): def of(cls, jcriterion, bigdl_type="float"): """ Create a python Criterion by a java criterion object + :param jcriterion: A java criterion object which created by Py4j :return: a criterion. """ @@ -93,8 +96,10 @@ class ClassNLLCriterion(Criterion): If provided, the optional argument weights should be a 1D Tensor assigning weight to each of the classes. - :param weights weights of each class - :param size_average whether to average or not + + :param weights: weights of each class + :param size_average: whether to average or not + >>> np.random.seed(123) >>> weights = np.random.uniform(0, 1, (2,)).astype("float32") @@ -118,7 +123,10 @@ class MSECriterion(Criterion): ''' Creates a criterion that measures the mean squared error between n elements in the input x and output y: +``` loss(x, y) = 1/n \sum |x_i - y_i|^2 +``` + If x and y are d-dimensional Tensors with a total of n elements, the sum operation still operates over all the elements, and divides by n. @@ -127,6 +135,7 @@ class MSECriterion(Criterion): By default, the losses are averaged over observations for each minibatch. However, if the field sizeAverage is set to false, the losses are instead summed. + >>> mSECriterion = MSECriterion() creating: createMSECriterion ''' @@ -140,6 +149,7 @@ class AbsCriterion(Criterion): ''' measures the mean absolute value of the element-wise difference between input + >>> absCriterion = AbsCriterion(True) creating: createAbsCriterion ''' @@ -157,7 +167,9 @@ class ClassSimplexCriterion(Criterion): ClassSimplexCriterion implements a criterion for classification. It learns an embedding per class, where each class' embedding is a point on an (N-1)-dimensional simplex, where N is the number of classes. - :param nClasses the number of classes. + + :param nClasses: the number of classes. + >>> classSimplexCriterion = ClassSimplexCriterion(2) creating: createClassSimplexCriterion @@ -176,7 +188,9 @@ class CosineEmbeddingCriterion(Criterion): Creates a criterion that measures the loss given an input x = {x1, x2}, a table of two Tensors, and a Tensor label y with values 1 or -1. - :param margin a number from -1 to 1, 0 to 0.5 is suggested + + :param margin: a number from -1 to 1, 0 to 0.5 is suggested + >>> cosineEmbeddingCriterion = CosineEmbeddingCriterion(1e-5, True) creating: createCosineEmbeddingCriterion @@ -196,7 +210,9 @@ class DistKLDivCriterion(Criterion): ''' The Kullback-Leibler divergence criterion - :param sizeAverage + + :param sizeAverage: + >>> distKLDivCriterion = DistKLDivCriterion(True) creating: createDistKLDivCriterion @@ -219,11 +235,13 @@ class HingeEmbeddingCriterion(Criterion): e.g. using the L1 pairwise distance, and is typically used for learning nonlinear embeddings or semi-supervised learning. + If x and y are n-dimensional Tensors, the sum operation still operates over all the elements, and divides by n (this can be avoided if one sets the internal variable sizeAverage to false). The margin has a default value of 1, or can be set in the constructor. + >>> hingeEmbeddingCriterion = HingeEmbeddingCriterion(1e-5, True) creating: createHingeEmbeddingCriterion ''' @@ -243,7 +261,9 @@ class L1HingeEmbeddingCriterion(Criterion): Creates a criterion that measures the loss given an input x = {x1, x2}, a table of two Tensors, and a label y (1 or -1): - :param margin + + :param margin: + >>> l1HingeEmbeddingCriterion = L1HingeEmbeddingCriterion(1e-5) creating: createL1HingeEmbeddingCriterion @@ -262,9 +282,11 @@ class MarginCriterion(Criterion): Creates a criterion that optimizes a two-class classification hinge loss (margin-based loss) between input x (a Tensor of dimension 1) and output y. - :param margin if unspecified, is by default 1. + + :param margin: if unspecified, is by default 1. :param size_average: size average in a mini-batch + >>> marginCriterion = MarginCriterion(1e-5, True) creating: createMarginCriterion ''' @@ -288,7 +310,9 @@ class MarginRankingCriterion(Criterion): If y == 1 then it assumed the first input should be ranked higher (have a larger value) than the second input, and vice-versa for y == -1. - :param margin + + :param margin: + >>> marginRankingCriterion = MarginRankingCriterion(1e-5, True) creating: createMarginRankingCriterion @@ -308,6 +332,7 @@ class MultiCriterion(Criterion): ''' a weighted sum of other criterions each applied to the same input and target + >>> multiCriterion = MultiCriterion() creating: createMultiCriterion ''' @@ -323,8 +348,10 @@ class MultiLabelMarginCriterion(Criterion): Creates a criterion that optimizes a multi-class multi-classification hinge loss ( margin-based loss) between input x and output y (which is a Tensor of target class indices) + :param size_average: size average in a mini-batch + >>> multiLabelMarginCriterion = MultiLabelMarginCriterion(True) creating: createMultiLabelMarginCriterion ''' @@ -342,9 +369,12 @@ class ParallelCriterion(Criterion): ParallelCriterion is a weighted sum of other criterions each applied to a different input and target. Set repeatTarget = true to share the target for criterions. + Use add(criterion[, weight]) method to add criterion. Where weight is a scalar(default 1). - :param repeat_target Whether to share the target for all criterions. + + :param repeat_target: Whether to share the target for all criterions. + >>> parallelCriterion = ParallelCriterion(True) creating: createParallelCriterion @@ -364,14 +394,18 @@ class SmoothL1Criterion(Criterion): It uses a squared term if the absolute element-wise error falls below 1. It is less sensitive to outliers than the MSECriterion and in some cases prevents exploding gradients (e.g. see "Fast R-CNN" paper by Ross Girshick). +``` | 0.5 * (x_i - y_i)^2^, if |x_i - y_i| < 1 loss(x, y) = 1/n \sum | | |x_i - y_i| - 0.5, otherwise +``` If x and y are d-dimensional Tensors with a total of n elements, the sum operation still operates over all the elements, and divides by n. The division by n can be avoided if one sets the internal variable sizeAverage to false - :param size_average whether to average the loss + + :param size_average: whether to average the loss + >>> smoothL1Criterion = SmoothL1Criterion(True) creating: createSmoothL1Criterion @@ -392,11 +426,13 @@ class SmoothL1CriterionWithWeights(Criterion): It is less sensitive to outliers than the MSECriterion and in some cases prevents exploding gradients (e.g. see "Fast R-CNN" paper by Ross Girshick). +``` d = (x - y) * w_in loss(x, y, w_in, w_out) | 0.5 * (sigma * d_i)^2 * w_out if |d_i| < 1 / sigma / sigma = 1/n \sum | | (|d_i| - 0.5 / sigma / sigma) * w_out otherwise +``` >>> smoothL1CriterionWithWeights = SmoothL1CriterionWithWeights(1e-5, 1) creating: createSmoothL1CriterionWithWeights @@ -418,9 +454,10 @@ class SoftmaxWithCriterion(Criterion): passing real-valued predictions through a softmax to get a probability distribution over classes. It should be preferred over separate SoftmaxLayer + MultinomialLogisticLossLayer as its gradient computation is more numerically stable. - :param ignoreLabel (optional) Specify a label value that - should be ignored when computing the loss. - :param normalizeMode How to normalize the output loss. + + :param ignoreLabel: (optional) Specify a label value thatshould be ignored when computing the loss. + :param normalizeMode: How to normalize the output loss. + >>> softmaxWithCriterion = SoftmaxWithCriterion() creating: createSoftmaxWithCriterion @@ -442,9 +479,11 @@ class TimeDistributedCriterion(Criterion): This class is intended to support inputs with 3 or more dimensions. Apply Any Provided Criterion to every temporal slice of an input. + :param criterion: embedded criterion :param size_average: whether to divide the sequence length + >>> td = TimeDistributedCriterion(ClassNLLCriterion()) creating: createClassNLLCriterion creating: createTimeDistributedCriterion @@ -459,7 +498,9 @@ class CrossEntropyCriterion(Criterion): """ This criterion combines LogSoftMax and ClassNLLCriterion in one single class. - :param weights A tensor assigning weight to each of the classes + + :param weights: A tensor assigning weight to each of the classes + >>> np.random.seed(123) >>> weights = np.random.uniform(0, 1, (2,)).astype("float32") @@ -484,8 +525,10 @@ class BCECriterion(Criterion): Creates a criterion that measures the Binary Cross Entropy between the target and the output - :param weights weights for each class - :param sizeAverage whether to average the loss or not + + :param weights: weights for each class + :param sizeAverage: whether to average the loss or not + >>> np.random.seed(123) >>> weights = np.random.uniform(0, 1, (2,)).astype("float32") @@ -508,10 +551,14 @@ class MultiLabelSoftMarginCriterion(Criterion): ''' A MultiLabel multiclass criterion based on sigmoid: the loss is: +``` l(x,y) = - sum_i y[i] * log(p[i]) + (1 - y[i]) * log (1 - p[i]) +``` where p[i] = exp(x[i]) / (1 + exp(x[i])) and with weights: +``` l(x,y) = - sum_i weights[i] (y[i] * log(p[i]) + (1 - y[i]) * log (1 - p[i])) +``` >>> np.random.seed(123) >>> weights = np.random.uniform(0, 1, (2,)).astype("float32") @@ -535,10 +582,12 @@ class MultiMarginCriterion(Criterion): Creates a criterion that optimizes a multi-class classification hinge loss (margin-based loss) between input x and output y (which is a target class index). - :param p - :param weights - :param margin - :param size_average + + :param p: + :param weights: + :param margin: + :param size_average: + >>> np.random.seed(123) >>> weights = np.random.uniform(0, 1, (2,)).astype("float32") @@ -567,10 +616,12 @@ class SoftMarginCriterion(Criterion): between input x (a Tensor of dimension 1) and output y (which is a tensor containing either 1s or -1s). +``` loss(x, y) = sum_i (log(1 + exp(-y[i]*x[i]))) / x:nElement() +``` + + :param sizeaverage: The normalization by the number of elements in the inputcan be disabled by setting - :param sizeaverage The normalization by the number of elements in the input - can be disabled by setting >>> softMarginCriterion = SoftMarginCriterion(False) creating: createSoftMarginCriterion diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index ee48037af8c..5f3e157d0e1 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -90,6 +90,7 @@ def __call__(self, x=None): def of(cls, jmodel, bigdl_type="float"): """ Create a Python Model + :param jmodel: Java model create by Py4j :return: A Python Model """ @@ -113,6 +114,7 @@ def name(self): def set_seed(self, seed=123): """ You can control the random seed which used to init weights for this model. + :param seed: random seed :return: Model itself. """ @@ -150,6 +152,7 @@ def forward(self, input): """ NB: It's for debug only, please use optimizer.optimize() in production. Takes an input object, and computes the corresponding output of the module + :param input: ndarray or list of ndarray :return: ndarray or list of ndarray """ @@ -166,6 +169,7 @@ def backward(self, input, grad_output): general this method makes the assumption forward(input) has been called before, with the same input. This is necessary for optimization reasons. If you do not respect this rule, backward() will compute incorrect gradients. + :param input: ndarray or list of ndarray :param grad_output: ndarray or list of ndarray :return: ndarray or list of ndarray @@ -187,6 +191,7 @@ def reset(self): def parameters(self): """ Get the model parameters which containing: weight, bias, gradBias, gradWeight + :return: dict(layername -> dict(parametername -> ndarray)) """ name_to_params = callBigDlFunc(self.bigdl_type, @@ -207,6 +212,7 @@ def predict(self, data_rdd): Model inference base on the given data. You need to invoke collect() to trigger those action \ as the returning result is an RDD. + :param data_rdd: the data to be predict. :return: An RDD represent the predict result. """ @@ -217,10 +223,10 @@ def predict(self, data_rdd): def test(self, val_rdd, batch_size, val_methods): """ A method to benchmark the model quality. + :param val_rdd: the input data :param batch_size: batch size - :param val_methods: a list of validation methods. i.e: Top1Accuracy, - Top5Accuracy and Loss. + :param val_methods: a list of validation methods. i.e: Top1Accuracy,Top5Accuracy and Loss. :return: """ return callBigDlFunc(self.bigdl_type, @@ -231,8 +237,10 @@ def test(self, val_rdd, batch_size, val_methods): def set_weights(self, weights): """ Set weights for this layer + :param weights: a list of numpy arrays which represent weight and bias :return: + >>> linear = Linear(3,2) creating: createLinear >>> linear.set_weights([np.array([[1,2,3],[4,5,6]]), np.array([7,8])]) @@ -269,6 +277,7 @@ def set_weights(self, weights): def get_weights(self): """ Get weights for this layer + :return: list of numpy arrays which represent weight and bias """ tensorWeights = callBigDlFunc(self.bigdl_type, @@ -283,6 +292,7 @@ def get_weights(self): def load(path, bigdl_type="float"): """ Load a pre-trained Bigdl model. + :param path: The path containing the pre-trained model. :return: A pre-trained model. """ @@ -293,6 +303,7 @@ def load(path, bigdl_type="float"): def load_torch(path, bigdl_type="float"): """ Load a pre-trained Torch model. + :param path: The path containing the pre-trained model. :return: A pre-trained model. """ @@ -304,8 +315,8 @@ def load_caffe(model, defPath, modelPath, match_all=True, bigdl_type="float"): """ Load a pre-trained Caffe model. - :param model: A bigdl model definition \ - which equivalent to the pre-trained caffe model. + + :param model: A bigdl model definition \which equivalent to the pre-trained caffe model. :param defPath: The path containing the caffe model definition. :param modelPath: The path containing the pre-trained caffe model. :return: A pre-trained model. @@ -315,7 +326,7 @@ def load_caffe(model, defPath, modelPath, match_all=True, bigdl_type="float"): class Container(Model): ''' - [[Container]] is a sub-class of Model that declares methods defined in all containers. + [[Container]] is a sub-class of Model that declares methods defined in all containers. A container usually contain some other modules which can be added through the "add" method ''' @@ -339,12 +350,11 @@ class Linear(Model): an input sample of given batch (the number of rows means the batch size and the number of columns should be equal to the `inputSize`). - :param input_size the size the each input sample - :param output_size the size of the module output of each sample - :param init_method two initialized methods are supported here, which are [[Default]] - and [[Xavier]], where [[Xavier]] set bias to zero here. For more - detailed information about `initMethod`, please refer to - [[InitializationMethod]] + + :param input_size: the size the each input sample + :param output_size: the size of the module output of each sample + :param init_method: two initialized methods are supported here, which are [[Default]]and [[Xavier]], where [[Xavier]] set bias to zero here. For moredetailed information about `initMethod`, please refer to[[InitializationMethod]] + >>> linear = Linear(100, 10, "Xavier") creating: createLinear @@ -366,9 +376,11 @@ class ReLU(Model): Applies the rectified linear unit (ReLU) function element-wise to the input Tensor, thus outputting a Tensor of the same dimension. + ReLU is defined as: f(x) = max(0, x) Can optionally do its operation in-place without using extra state memory + >>> relu = ReLU() creating: createReLU ''' @@ -383,6 +395,7 @@ class Tanh(Model): Applies the Tanh function element-wise to the input Tensor, thus outputting a Tensor of the same dimension. Tanh is defined as f(x) = (exp(x)-exp(-x))/(exp(x)+exp(-x)). + >>> tanh = Tanh() creating: createTanh ''' @@ -397,6 +410,7 @@ class Echo(Model): This module is for debug purpose, which can print activation and gradient in your model topology + >>> echo = Echo() creating: createEcho ''' @@ -412,6 +426,7 @@ class LogSoftMax(Model): LogSoftmax is defined as: f_i(x) = log(1 / a exp(x_i)) where a = sum_j[exp(x_j)]. + >>> logSoftMax = LogSoftMax() creating: createLogSoftMax ''' @@ -426,6 +441,7 @@ class Sequential(Container): Sequential provides a means to plug layers together in a feed-forward fully connected manner. + >>> echo = Echo() creating: createEcho >>> s = Sequential() @@ -434,6 +450,7 @@ class Sequential(Container): >>> s = s.add(s) >>> s = s.add(echo) + ''' def __init__(self, bigdl_type="float"): @@ -447,17 +464,19 @@ class SpatialConvolution(Model): The input tensor in forward(input) is expected to be a 3D tensor (nInputPlane x height x width). - :param n_input_plane The number of expected input planes in the image given into forward() - :param n_output_plane The number of output planes the convolution layer will produce. - :param kernel_w The kernel width of the convolution - :param kernel_h The kernel height of the convolution - :param stride_w The step of the convolution in the width dimension. - :param stride_h The step of the convolution in the height dimension - :param pad_w The additional zeros added per width to the input planes. - :param pad_h The additional zeros added per height to the input planes. - :param n_group Kernel group number - :param propagate_back Propagate gradient back - :param init_method Initialization method to initialize bias and weight + + :param n_input_plane: The number of expected input planes in the image given into forward() + :param n_output_plane: The number of output planes the convolution layer will produce. + :param kernel_w: The kernel width of the convolution + :param kernel_h: The kernel height of the convolution + :param stride_w: The step of the convolution in the width dimension. + :param stride_h: The step of the convolution in the height dimension + :param pad_w: The additional zeros added per width to the input planes. + :param pad_h: The additional zeros added per height to the input planes. + :param n_group: Kernel group number + :param propagate_back: Propagate gradient back + :param init_method: Initialization method to initialize bias and weight + >>> spatialConvolution = SpatialConvolution(6, 12, 5, 5) creating: createSpatialConvolution @@ -504,12 +523,14 @@ class SpatialMaxPooling(Model): oheight = op((height + 2*padH - kH) / dH + 1) op is a rounding operator. By default, it is floor. It can be changed by calling :ceil() or :floor() methods. - :param kW kernel width - :param kH kernel height - :param dW step size in width - :param dH step size in height - :param padW padding in width - :param padH padding in height + + :param kW: kernel width + :param kH: kernel height + :param dW: step size in width + :param dH: step size in height + :param padW: padding in width + :param padH: padding in height + >>> spatialMaxPooling = SpatialMaxPooling(2, 2, 2, 2) creating: createSpatialMaxPooling ''' @@ -537,8 +558,10 @@ class Select(Model): ''' A Simple layer selecting an index of the input tensor in the given dimension - :param dimension the dimension to select - :param index the index of the dimension to be selected + + :param dimension: the dimension to select + :param index: the index of the dimension to be selected + >>> select = Select(1, 1) creating: createSelect @@ -552,6 +575,7 @@ class Recurrent(Container): Recurrent module is a container of rnn cells Different types of rnn cells can be added using add() function + >>> recurrent = Recurrent() creating: createRecurrent ''' @@ -562,22 +586,19 @@ def __init__(self, bigdl_type="float"): class LSTM(Model): ''' - Long Short Term Memory architecture. - Ref. - A.: http://arxiv.org/pdf/1303.5778v1 (blueprint for this module) - B. http://web.eecs.utk.edu/~itamar/courses/ECE-692/Bobby_paper1.pdf - C. http://arxiv.org/pdf/1503.04069v1.pdf - D. https://github.com/wojzaremba/lstm - E. https://github.com/Element-Research/rnn/blob/master/FastLSTM.lua +| Long Short Term Memory architecture. +| Ref. +| A.: http://arxiv.org/pdf/1303.5778v1 (blueprint for this module) +| B. http://web.eecs.utk.edu/~itamar/courses/ECE-692/Bobby_paper1.pdf +| C. http://arxiv.org/pdf/1503.04069v1.pdf +| D. https://github.com/wojzaremba/lstm +| E. https://github.com/Element-Research/rnn/blob/master/FastLSTM.lua + :param inputSize: the size of each input vector :param hiddenSize: Hidden unit size in the LSTM - :param p: is used for [[Dropout]] probability. For more details about - RNN dropouts, please refer to - [RnnDrop: A Novel Dropout for RNNs in ASR] - (http://www.stat.berkeley.edu/~tsmoon/files/Conference/asru2015.pdf) - [A Theoretically Grounded Application of Dropout in Recurrent Neural Networks] - (https://arxiv.org/pdf/1512.05287.pdf) + :param p: is used for [[Dropout]] probability. For more details aboutRNN dropouts, please refer to[RnnDrop: A Novel Dropout for RNNs in ASR](http://www.stat.berkeley.edu/~tsmoon/files/Conference/asru2015.pdf)[A Theoretically Grounded Application of Dropout in Recurrent Neural Networks](https://arxiv.org/pdf/1512.05287.pdf) + >>> lstm = LSTM(4, 3, 0.5) creating: createLSTM @@ -589,21 +610,18 @@ def __init__(self, input_size, hidden_size, p=0.0, bigdl_type="float"): class LSTMPeephole(Model): ''' - Long Short Term Memory architecture with peephole. - Ref. A.: http://arxiv.org/pdf/1303.5778v1 (blueprint for this module) - B. http://web.eecs.utk.edu/~itamar/courses/ECE-692/Bobby_paper1.pdf - C. http://arxiv.org/pdf/1503.04069v1.pdf - D. https://github.com/wojzaremba/lstm - E. https://github.com/Element-Research/rnn/blob/master/LSTM.lua +| Long Short Term Memory architecture with peephole. +| Ref. A.: http://arxiv.org/pdf/1303.5778v1 (blueprint for this module) +| B. http://web.eecs.utk.edu/~itamar/courses/ECE-692/Bobby_paper1.pdf +| C. http://arxiv.org/pdf/1503.04069v1.pdf +| D. https://github.com/wojzaremba/lstm +| E. https://github.com/Element-Research/rnn/blob/master/LSTM.lua + :param input_size: the size of each input vector :param hidden_size: Hidden unit size in the LSTM - :param p: is used for [[Dropout]] probability. For more details about - RNN dropouts, please refer to - [RnnDrop: A Novel Dropout for RNNs in ASR] - (http://www.stat.berkeley.edu/~tsmoon/files/Conference/asru2015.pdf) - [A Theoretically Grounded Application of Dropout in Recurrent Neural Networks] - (https://arxiv.org/pdf/1512.05287.pdf) + :param p: is used for [[Dropout]] probability. For more details aboutRNN dropouts, please refer to[RnnDrop: A Novel Dropout for RNNs in ASR](http://www.stat.berkeley.edu/~tsmoon/files/Conference/asru2015.pdf)[A Theoretically Grounded Application of Dropout in Recurrent Neural Networks](https://arxiv.org/pdf/1512.05287.pdf) + >>> lstm = LSTMPeephole(4, 3, 0.5) creating: createLSTMPeephole @@ -618,18 +636,16 @@ class GRU(Model): Gated Recurrent Units architecture. The first input in sequence uses zero value for cell and hidden state - Ref. - http://www.wildml.com/2015/10/recurrent-neural-network-tutorial-part-4-implementing-a-grulstm-rnn-with-python-and-theano/ - https://github.com/Element-Research/rnn/blob/master/GRU.lua + +| Ref. +| http://www.wildml.com/2015/10/recurrent-neural-network-tutorial-part-4-implementing-a-grulstm-rnn-with-python-and-theano/ +| https://github.com/Element-Research/rnn/blob/master/GRU.lua + :param input_size: the size of each input vector :param hidden_size: Hidden unit size in GRU - :param p: is used for [[Dropout]] probability. For more details about - RNN dropouts, please refer to - [RnnDrop: A Novel Dropout for RNNs in ASR] - (http://www.stat.berkeley.edu/~tsmoon/files/Conference/asru2015.pdf) - [A Theoretically Grounded Application of Dropout in Recurrent Neural Networks] - (https://arxiv.org/pdf/1512.05287.pdf) + :param p: is used for [[Dropout]] probability. For more details aboutRNN dropouts, please refer to[RnnDrop: A Novel Dropout for RNNs in ASR](http://www.stat.berkeley.edu/~tsmoon/files/Conference/asru2015.pdf)[A Theoretically Grounded Application of Dropout in Recurrent Neural Networks](https://arxiv.org/pdf/1512.05287.pdf) + >>> gru = GRU(4, 3, 0.5) creating: createGRU @@ -643,10 +659,12 @@ class RnnCell(Model): ''' It is a simple RNN. User can pass an activation function to the RNN. + :param input_size: the size of each input vector :param hidden_size: Hidden unit size in simple RNN :param activation: activation function + >>> reshape = RnnCell(4, 3, Tanh()) creating: createTanh creating: createRnnCell @@ -665,9 +683,11 @@ class TimeDistributed(Model): This layer is intended to apply contained layer to each temporal time slice of input tensor. + For instance, The TimeDistributed Layer can feed each time slice of input tensor to the Linear layer. + >>> td = TimeDistributed(Linear(2, 3)) creating: createLinear creating: createTimeDistributed @@ -683,6 +703,7 @@ class Concat(Container): Concat concatenates the output of one layer of "parallel" modules along the provided {@code dimension}: they take the same inputs, and their output is concatenated. +``` +-----------+ +----> module1 -----+ | | | | @@ -690,9 +711,11 @@ class Concat(Container): | | | | +----> module3 -----+ +-----------+ +``` :param dimension: dimension + >>> concat = Concat(2) creating: createConcat ''' @@ -710,16 +733,17 @@ class SpatialAveragePooling(Model): Applies 2D average-pooling operation in kWxkH regions by step size dWxdH steps. The number of output features is equal to the number of input planes. - :param kW kernel width - :param kH kernel height - :param dW step width - :param dH step height - :param padW padding width - :param padH padding height - :param ceilMode whether the output size is to be ceiled or floored - :param countIncludePad whether to include padding when dividing the - number of elements in pooling region - :param divide whether to do the averaging + + :param kW: kernel width + :param kH: kernel height + :param dW: step width + :param dH: step height + :param padW: padding width + :param padH: padding height + :param ceilMode: whether the output size is to be ceiled or floored + :param countIncludePad: whether to include padding when dividing thenumber of elements in pooling region + :param divide: whether to do the averaging + >>> spatialAveragePooling = SpatialAveragePooling(7,7) creating: createSpatialAveragePooling @@ -758,13 +782,16 @@ class SpatialBatchNormalization(Model): For non-convolutional layers, see [[BatchNormalization]] The operation implemented is: +``` ( x - mean(x) ) y = -------------------- * gamma + beta standard-deviation(x) +``` where gamma and beta are learnable parameters. The learning of gamma and beta is optional. + >>> spatialBatchNormalization = SpatialBatchNormalization(1) creating: createSpatialBatchNormalization ''' @@ -791,19 +818,22 @@ class SpatialCrossMapLRN(Model): ''' Applies Spatial Local Response Normalization between different feature maps. The operation implemented is: +``` x_f y_f = ------------------------------------------------- (k+(alpha/size)* sum_{l=l1 to l2} (x_l^2^))^beta^ +``` where x_f is the input at spatial locations h,w (not shown for simplicity) and feature map f, l1 corresponds to max(0,f-ceil(size/2)) and l2 to min(F, f-ceil(size/2) + size). Here, F is the number of feature maps. - :param size: the number of channels to sum over (for cross channel LRN) or the side length of - the square region to sum over (for within channel LRN) + + :param size: the number of channels to sum over (for cross channel LRN) or the side length ofthe square region to sum over (for within channel LRN) :param alpha: the scaling parameter :param beta: the exponent :param k: a constant + >>> spatialCrossMapLRN = SpatialCrossMapLRN() creating: createSpatialCrossMapLRN ''' @@ -829,10 +859,12 @@ class Dropout(Model): set, the outputs are scaled by a factor of 1/(1-initP) during training. During evaluating, output is the same as input. + :param initP: probability to be dropped :param inplace: inplace model :param scale: if scale by a factor of 1/(1-initP) + >>> dropout = Dropout(0.4) creating: createDropout ''' @@ -856,8 +888,10 @@ class View(Model): inputs of the modules. This makes it possible to use minibatch inputs when using a size -1 for one of the dimensions. + :param size: sizes use for creates a new view + >>> view = View([1024,2]) creating: createView ''' @@ -876,6 +910,7 @@ class Abs(Model): ''' an element-wise abs operation + >>> abs = Abs() creating: createAbs ''' @@ -889,7 +924,9 @@ class Add(Model): ''' adds a bias term to input data ; - :param input_size size of input data + + :param input_size: size of input data + >>> add = Add(1) creating: createAdd ''' @@ -909,8 +946,10 @@ class AddConstant(Model): ''' adding a constant - :param constant_scalar constant value - :param inplace Can optionally do its operation in-place without using extra state memory + + :param constant_scalar: constant value + :param inplace: Can optionally do its operation in-place without using extra state memory + >>> addConstant = AddConstant(1e-5, True) creating: createAddConstant @@ -932,20 +971,26 @@ class BatchNormalization(Model): Covariate Shift" by Sergey Ioffe, Christian Szegedy https://arxiv.org/abs/1502.03167 + This implementation is useful for inputs NOT coming from convolution layers. For convolution layers, use nn.SpatialBatchNormalization. + The operation implemented is: +``` ( x - mean(x) ) y = -------------------- * gamma + beta standard-deviation(x) +``` where gamma and beta are learnable parameters.The learning of gamma and beta is optional. + :param n_output: output feature map number :param eps: avoid divide zero :param momentum: momentum for weight update :param affine: affine operation on output or not + >>> batchNormalization = BatchNormalization(1, 1e-5, 1e-5, True) creating: createBatchNormalization ''' @@ -972,10 +1017,12 @@ class Bilinear(Model): The input tensor given in forward(input) is a table containing both inputs x_1 and x_2, which are tensors of size N x inputDimension1 and N x inputDimension2, respectively. - :param input_size1 input dimension of x_1 - :param input_size2 input dimension of x_2 - :param output_size output dimension - :param bias_res whether use bias + + :param input_size1: input dimension of x_1 + :param input_size2: input dimension of x_2 + :param output_size: output dimension + :param bias_res: whether use bias + >>> bilinear = Bilinear(1, 1, 1, True) creating: createBilinear @@ -1002,10 +1049,12 @@ class Bottle(Container): ''' Bottle allows varying dimensionality input to be forwarded through any module that accepts input of nInputDim dimensions, and generates output of nOutputDim dimensions. + :param module: transform module :param n_input_dim: nInputDim dimensions of module :param n_output_dim1: output of nOutputDim dimensions + >>> bottle = Bottle(Linear(100,10), 1, 1) creating: createLinear creating: createBottle @@ -1032,8 +1081,10 @@ class CAdd(Model): it will report an error). If the input is a batch, a singleton dimension will be add to the first dimension before the expand. + :param size: the size of the bias + >>> cAdd = CAdd([1,2]) creating: createCAdd ''' @@ -1055,8 +1106,10 @@ class CAddTable(Model): Merge the input tensors in the input table by element wise adding them together. The input table is actually an array of tensor with same size. + :param inplace: reuse the input memory + >>> cAddTable = CAddTable(True) creating: createCAddTable ''' @@ -1073,6 +1126,7 @@ class CDivTable(Model): ''' Takes a table with two Tensor and returns the component-wise division between them. + >>> cDivTable = CDivTable() creating: createCDivTable ''' @@ -1087,6 +1141,7 @@ class CMaxTable(Model): ''' Takes a table of Tensors and outputs the max of all of them. + >>> cMaxTable = CMaxTable() creating: createCMaxTable ''' @@ -1100,6 +1155,7 @@ class CMinTable(Model): ''' Takes a table of Tensors and outputs the min of all of them. + >>> cMinTable = CMinTable() creating: createCMinTable ''' @@ -1114,7 +1170,9 @@ class CMul(Model): ''' Applies a component-wise multiplication to the incoming data - :param size size of the data + + :param size: size of the data + >>> cMul = CMul([1,2]) creating: createCMul @@ -1136,6 +1194,7 @@ class CMulTable(Model): ''' Takes a table of Tensors and outputs the multiplication of all of them. + >>> cMulTable = CMulTable() creating: createCMulTable ''' @@ -1150,6 +1209,7 @@ class CSubTable(Model): ''' Takes a table with two Tensor and returns the component-wise subtraction between them. + >>> cSubTable = CSubTable() creating: createCSubTable ''' @@ -1167,8 +1227,10 @@ class Clamp(Model): otherwise elements less than min_value (or greater than max_value) are saturated to min_value (or max_value). - :param min - :param max + + :param min: + :param max: + >>> clamp = Clamp(1, 3) creating: createClamp @@ -1188,6 +1250,7 @@ class Contiguous(Model): ''' used to make input, grad_output both contiguous + >>> contiguous = Contiguous() creating: createContiguous ''' @@ -1206,9 +1269,11 @@ class Cosine(Model): an input sample of given batch (the number of rows means the batch size and the number of columns should be equal to the inputSize). + :param input_size: the size of each input sample :param output_size: the size of the module output of each sample + >>> cosine = Cosine(2,3) creating: createCosine ''' @@ -1230,6 +1295,7 @@ class CosineDistance(Model): ''' Outputs the cosine distance between inputs + >>> cosineDistance = CosineDistance() creating: createCosineDistance ''' @@ -1242,11 +1308,13 @@ class DiceCoefficientCriterion(Model): ''' The Dice-Coefficient criterion - input: Tensor, target: Tensor + input: Tensor,target: Tensor +``` return: 2 * (input intersection target) 1 - ---------------------------------- input union target +``` >>> diceCoefficientCriterion = DiceCoefficientCriterion(size_average = True, epsilon = 1.0) creating: createDiceCoefficientCriterion @@ -1266,6 +1334,7 @@ class DotProduct(Model): This is a simple table layer which takes a table of two tensors as input and calculate the dot product between them as outputs + >>> dotProduct = DotProduct() creating: createDotProduct ''' @@ -1282,6 +1351,7 @@ class ELU(Model): Fast and Accurate Deep Network Learning by Exponential Linear Units (ELUs) [http://arxiv.org/pdf/1511.07289.pdf] + >>> eLU = ELU(1e-5, True) creating: createELU ''' @@ -1299,9 +1369,11 @@ class Euclidean(Model): ''' Outputs the Euclidean distance of the input to outputSize centers - :param inputSize inputSize - :param outputSize outputSize - :param T Numeric type. Only support float/double now + + :param inputSize: inputSize + :param outputSize: outputSize + :param T: Numeric type. Only support float/double now + >>> euclidean = Euclidean(1, 1, True) creating: createEuclidean @@ -1328,6 +1400,7 @@ class Exp(Model): ''' Applies element-wise exp to input tensor. + >>> exp = Exp() creating: createExp ''' @@ -1344,6 +1417,7 @@ class FlattenTable(Model): (potentially nested) as input and a table of Tensors without any nested table will be produced + >>> flattenTable = FlattenTable() creating: createFlattenTable ''' @@ -1363,7 +1437,9 @@ class GradientReversal(Model): ["Domain-Adversarial Training of Neural Networks" (http://arxiv.org/abs/1505.07818)] - :param lambda hyper-parameter lambda can be set dynamically during training + + :param lambda: hyper-parameter lambda can be set dynamically during training + >>> gradientReversal = GradientReversal(1e-5) creating: createGradientReversal @@ -1382,12 +1458,15 @@ class HardShrink(Model): This is a transfer layer which applies the hard shrinkage function element-wise to the input Tensor. The parameter lambda is set to 0.5 by default +``` x, if x > lambda f(x) = x, if x < -lambda 0, otherwise +``` :param the_lambda: a threshold value whose default value is 0.5 + >>> hardShrink = HardShrink(1e-5) creating: createHardShrink ''' @@ -1403,12 +1482,15 @@ class HardTanh(Model): ''' Applies HardTanh to each element of input, HardTanh is defined: +``` | maxValue, if x > maxValue f(x) = | minValue, if x < minValue | x, otherwise - :param min_value minValue in f(x), default is -1. - :param max_value maxValue in f(x), default is 1. - :param inplace whether enable inplace model. +``` + :param min_value: minValue in f(x), default is -1. + :param max_value: maxValue in f(x), default is 1. + :param inplace: whether enable inplace model. + >>> hardTanh = HardTanh(1e-5, 1e5, True) creating: createHardTanh @@ -1430,7 +1512,9 @@ class Index(Model): ''' Applies the Tensor index operation along the given dimension. - :param dimension the dimension to be indexed + + :param dimension: the dimension to be indexed + >>> index = Index(1) creating: createIndex @@ -1458,8 +1542,10 @@ class InferReshape(Model): For example, (4, 5, 6, 7) -> InferReshape (4, 0, 3, -1) -> (4, 5, 3, 14) with 1st and 3rd dim same as given size, with 2nd dim same as input, and the infered dim is 14 - :param size the target tensor size - :param batch_mode whether in batch mode + + :param size: the target tensor size + :param batch_mode: whether in batch mode + >>> inferReshape = InferReshape([4, 0, 3, -1], False) creating: createInferReshape @@ -1480,15 +1566,16 @@ class JoinTable(Model): It is a table module which takes a table of Tensors as input and outputs a Tensor by joining them together along the dimension `dimension`. + The input to this layer is expected to be a tensor, or a batch of tensors; when using mini-batch, a batch of sample tensors will be passed to the layer and the user need to specify the number of dimensions of each sample tensor in the batch using `nInputDims`. - :param dimension to be join in this dimension - :param nInputDims specify the number of dimensions that this module will receive - If it is more than the dimension of input tensors, the first dimension - would be considered as batch size + + :param dimension: to be join in this dimension + :param nInputDims: specify the number of dimensions that this module will receiveIf it is more than the dimension of input tensors, the first dimensionwould be considered as batch size + >>> joinTable = JoinTable(1, 1) creating: createJoinTable @@ -1507,6 +1594,7 @@ class L1Cost(Model): ''' compute L1 norm for input, and sign of input + >>> l1Cost = L1Cost() creating: createL1Cost ''' @@ -1524,9 +1612,11 @@ class L1Penalty(Model): directly to the output, and computes an L1 loss of the latent state (input) and stores it in the module's loss field. During backward propagation: gradInput = gradOutput + gradLoss. - :param l1weight - :param sizeAverage - :param provideOutput + + :param l1weight: + :param sizeAverage: + :param provideOutput: + >>> l1Penalty = L1Penalty(1, True, True) creating: createL1Penalty @@ -1549,9 +1639,11 @@ class LeakyReLU(Model): It is a transfer module that applies LeakyReLU, which parameter negval sets the slope of the negative part: LeakyReLU is defined as: f(x) = max(0, x) + negval * min(0, x) + :param negval: sets the slope of the negative partl :param inplace: if it is true, doing the operation in-place without using extra state memory + >>> leakyReLU = LeakyReLU(1e-5, True) creating: createLeakyReLU ''' @@ -1571,6 +1663,7 @@ class Log(Model): Applies the log function element-wise to the input Tensor, thus outputting a Tensor of the same dimension. + >>> log = Log() creating: createLog ''' @@ -1586,6 +1679,7 @@ class LogSigmoid(Model): This class is a transform layer corresponding to the sigmoid function: f(x) = Log(1 / (1 + e ^^ (-x))) + >>> logSigmoid = LogSigmoid() creating: createLogSigmoid ''' @@ -1600,6 +1694,7 @@ class LookupTable(Model): ''' a convolution of width 1, commonly used for word embeddings + >>> lookupTable = LookupTable(1, 1, 1e-5, 1e-5, 1e-5, True) creating: createLookupTable ''' @@ -1629,9 +1724,11 @@ class MM(Model): ''' Module to perform matrix multiplication on two mini-batch inputs, producing a mini-batch. + :param trans_a: specifying whether or not transpose the first input matrix :param trans_b: specifying whether or not transpose the second input matrix + >>> mM = MM(True, True) creating: createMM ''' @@ -1650,7 +1747,9 @@ class MV(Model): It is a module to perform matrix vector multiplication on two mini-batch inputs, producing a mini-batch. - :param trans whether make matrix transpose before multiplication + + :param trans: whether make matrix transpose before multiplication + >>> mV = MV(True) creating: createMV @@ -1670,6 +1769,7 @@ class MapTable(Container): to all input elements. The member module is cloned as necessary to process all input elements. + >>> mapTable = MapTable(Linear(100,10)) creating: createLinear creating: createMapTable @@ -1686,6 +1786,7 @@ class MaskedSelect(Model): ''' Performs a torch.MaskedSelect on a Tensor. The mask is supplied as a tabular argument with the input on the forward and backward passes. + >>> maskedSelect = MaskedSelect() creating: createMaskedSelect ''' @@ -1700,8 +1801,10 @@ class Max(Model): ''' Applies a max operation over dimension `dim` - :param dim max along this dimension - :param num_input_dims Optional. If in a batch model, set to the inputDims. + + :param dim: max along this dimension + :param num_input_dims: Optional. If in a batch model, set to the inputDims. + >>> max = Max(1) creating: createMax @@ -1726,10 +1829,10 @@ class Mean(Model): user need to specify the number of dimensions of each sample tensor in the batch using nInputDims. + :param dimension: the dimension to be applied mean operation - :param n_input_dims: specify the number of dimensions that this module will receive - If it is more than the dimension of input tensors, the first dimension would be considered - as batch size + :param n_input_dims: specify the number of dimensions that this module will receiveIf it is more than the dimension of input tensors, the first dimension would be consideredas batch size + >>> mean = Mean(1, 1) creating: createMean @@ -1749,8 +1852,10 @@ class Min(Model): ''' Applies a min operation over dimension `dim`. - :param dim min along this dimension - :param num_input_dims Optional. If in a batch model, set to the input_dim. + + :param dim: min along this dimension + :param num_input_dims: Optional. If in a batch model, set to the input_dim. + >>> min = Min(1) creating: createMin @@ -1774,6 +1879,7 @@ class MixtureTable(Model): should take the form of a table of Tensors. This Module works for experts of dimension 1D or more, and for a 1D or 2D gater, i.e. for single examples or mini-batches. + >>> mixtureTable = MixtureTable() creating: createMixtureTable >>> mixtureTable = MixtureTable(10) @@ -1791,6 +1897,7 @@ class Mul(Model): ''' Multiply a single scalar factor to the incoming data + >>> mul = Mul() creating: createMul ''' @@ -1810,8 +1917,10 @@ class MulConstant(Model): Multiplies input Tensor by a (non-learnable) scalar constant. This module is sometimes useful for debugging purposes. - :param scalar scalar constant - :param inplace Can optionally do its operation in-place without using extra state memory + + :param scalar: scalar constant + :param inplace: Can optionally do its operation in-place without using extra state memory + >>> mulConstant = MulConstant(2.5) creating: createMulConstant @@ -1831,6 +1940,7 @@ class Narrow(Model): ''' Narrow is application of narrow operation in a module. The module further supports a negative length in order to handle inputs with an unknown size. + >>> narrow = Narrow(1, 1, 1) creating: createNarrow ''' @@ -1854,8 +1964,10 @@ class NarrowTable(Model): a table or a Tensor. If `length` is negative, it means selecting the elements from the offset to element which located at the abs(`length`) to the last element of the input. - :param offset the start index of table - :param length the length want to select + + :param offset: the start index of table + :param length: the length want to select + >>> narrowTable = NarrowTable(1, 1) creating: createNarrowTable @@ -1877,6 +1989,7 @@ class Normalize(Model): division by zero when the input contains all zero elements (default = 1e-10). p can be the max value of double + >>> normalize = Normalize(1e-5, 1e-5) creating: createNormalize ''' @@ -1895,14 +2008,19 @@ class PReLU(Model): ''' Applies parametric ReLU, which parameter varies the slope of the negative part. + PReLU: f(x) = max(0, x) + a * min(0, x) + nOutputPlane's default value is 0, that means using PReLU in shared version and has only one parameters. + Notice: Please don't use weight decay on this. - :param n_output_plane input map number. Default is 0. + + :param n_output_plane: input map number. Default is 0. + >>> pReLU = PReLU(1) creating: createPReLU @@ -1925,17 +2043,18 @@ class Padding(Model): This module adds pad units of padding to dimension dim of the input. If pad is negative, padding is added to the left, otherwise, it is added to the right of the dimension. + The input to this layer is expected to be a tensor, or a batch of tensors; when using mini-batch, a batch of sample tensors will be passed to the layer and the user need to specify the number of dimensions of each sample tensor in the batch using n_input_dim. - :param dim the dimension to be applied padding operation - :param pad num of the pad units - :param n_input_dim specify the number of dimensions that this module will receive - If it is more than the dimension of input tensors, the first dimension - would be considered as batch size - :param value padding value + + :param dim: the dimension to be applied padding operation + :param pad: num of the pad units + :param n_input_dim: specify the number of dimensions that this module will receiveIf it is more than the dimension of input tensors, the first dimensionwould be considered as batch size + :param value: padding value + >>> padding = Padding(1, 1, 1, 1e-5, 1) creating: createPadding @@ -1966,7 +2085,9 @@ class PairwiseDistance(Model): it must have the size of `inputSize`. If it is a matrix, then each row is assumed to be an input sample of the given batch (the number of rows means the batch size and the number of columns should be equal to the `inputSize`). - :param norm the norm of distance + + :param norm: the norm of distance + >>> pairwiseDistance = PairwiseDistance(2) creating: createPairwiseDistance @@ -1985,6 +2106,7 @@ class ParallelTable(Container): It is a container module that applies the i-th member module to the i-th input, and outputs an output in the form of Table + >>> parallelTable = ParallelTable() creating: createParallelTable ''' @@ -1999,10 +2121,12 @@ class Power(Model): ''' Apply an element-wise power operation with scale and shift. f(x) = (shift + scale * x)^power^ + :param power: the exponent. :param scale: Default is 1. :param shift: Default is 0. + >>> power = Power(1e-5) creating: createPower ''' @@ -2024,28 +2148,37 @@ class RReLU(Model): Applies the randomized leaky rectified linear unit (RReLU) element-wise to the input Tensor, thus outputting a Tensor of the same dimension. Informally the RReLU is also known as 'insanity' layer. RReLU is defined as: +``` f(x) = max(0,x) + a * min(0, x) where a ~ U(l, u). +``` In training mode negative inputs are multiplied by a factor a drawn from a uniform random distribution U(l, u). + In evaluation mode a RReLU behaves like a LeakyReLU with a constant mean factor a = (l + u) / 2. + By default, l = 1/8 and u = 1/3. If l == u a RReLU effectively becomes a LeakyReLU. + Regardless of operating in in-place mode a RReLU will internally allocate an input-sized noise tensor to store random factors for negative inputs. + The backward() operation assumes that forward() has been called before. + For reference see [Empirical Evaluation of Rectified Activations in Convolutional Network]( http://arxiv.org/abs/1505.00853). + :param lower: lower boundary of uniform random distribution :param upper: upper boundary of uniform random distribution :param inplace: optionally do its operation in-place without using extra state memory + >>> rReLU = RReLU(1e-5, 1e5, True) creating: createRReLU ''' @@ -2066,7 +2199,9 @@ class ReLU6(Model): ''' Same as ReLU except that the rectifying function f(x) saturates at x = 6 - :param inplace either True = in-place or False = keeping separate state + + :param inplace: either True = in-place or False = keeping separate state + >>> reLU6 = ReLU6(True) creating: createReLU6 @@ -2085,10 +2220,12 @@ class Replicate(Model): Replicate repeats input `nFeatures` times along its `dim` dimension. Notice: No memory copy, it set the stride along the `dim`-th dimension to zero. + :param n_features: replicate times. :param dim: dimension to be replicated. :param n_dim: specify the number of non-batch dimensions. + >>> replicate = Replicate(2) creating: createReplicate ''' @@ -2117,9 +2254,11 @@ class RoiPooling(Model): into the corresponding output grid cell. Pooling is applied independently to each feature map channel + :param pooled_w: spatial extent in width :param pooled_h: spatial extent in height - :param spatial_scale spatial scale + :param spatial_scale: spatial scale + >>> roiPooling = RoiPooling(1, 1, 1e-5) creating: createRoiPooling @@ -2144,7 +2283,9 @@ class Scale(Model): match the shape of the input. Similarly, perform a expand cdd bias and perform an elementwise add - :param size size of weight and bias + + :param size: size of weight and bias + >>> scale = Scale([1,2]) creating: createScale @@ -2166,7 +2307,9 @@ class SelectTable(Model): This is true regardless of the depth of the encapsulated Tensor as the function used internally to do so is recursive. - :param dimension the dimension to be selected + + :param dimension: the dimension to be selected + >>> selectTable = SelectTable(1) creating: createSelectTable @@ -2184,6 +2327,7 @@ class Sigmoid(Model): ''' Applies the Sigmoid function element-wise to the input Tensor, thus outputting a Tensor of the same dimension. + >>> sigmoid = Sigmoid() creating: createSigmoid ''' @@ -2201,6 +2345,7 @@ class SoftMax(Model): Softmax is defined as: f_i(x) = exp(x_i - shift) / sum_j exp(x_j - shift) where shift = max_i(x_i). + >>> softMax = SoftMax() creating: createSoftMax ''' @@ -2218,6 +2363,7 @@ class SoftMin(Model): Softmin is defined as: f_i(x) = exp(-x_i - shift) / sum_j exp(-x_j - shift) where shift = max_i(-x_i). + >>> softMin = SoftMin() creating: createSoftMin ''' @@ -2233,7 +2379,9 @@ class SoftPlus(Model): Apply the SoftPlus function to an n-dimensional input tensor. SoftPlus function: f_i(x) = 1/beta * log(1 + exp(beta * x_i)) - :param beta Controls sharpness of transfer function + + :param beta: Controls sharpness of transfer function + >>> softPlus = SoftPlus(1e-5) creating: createSoftPlus @@ -2251,12 +2399,16 @@ class SoftShrink(Model): ''' Apply the soft shrinkage function element-wise to the input Tensor + SoftShrinkage operator: +``` | x - lambda, if x > lambda f(x) = | x + lambda, if x < -lambda | 0, otherwise +``` + + :param the_lambda: lambda, default is 0.5 - :param the_lambda lambda, default is 0.5 >>> softShrink = SoftShrink(1e-5) creating: createSoftShrink @@ -2274,8 +2426,10 @@ class SoftSign(Model): ''' Apply SoftSign function to an n-dimensional input Tensor. + SoftSign function: f_i(x) = x_i / (1+|x_i|) + >>> softSign = SoftSign() creating: createSoftSign ''' @@ -2290,15 +2444,19 @@ class SpatialDilatedConvolution(Model): ''' Apply a 2D dilated convolution over an input image. + The input tensor is expected to be a 3D or 4D(with batch) tensor. + If input is a 3D tensor nInputPlane x height x width, owidth = floor(width + 2 * padW - dilationW * (kW-1) - 1) / dW + 1 oheight = floor(height + 2 * padH - dilationH * (kH-1) - 1) / dH + 1 + Reference Paper: Yu F, Koltun V. Multi-scale context aggregation by dilated convolutions[J]. arXiv preprint arXiv:1511.07122, 2015. + :param n_input_plane: The number of expected input planes in the image given into forward(). :param n_output_plane: The number of output planes the convolution layer will produce. :param kw: The kernel width of the convolution. @@ -2311,6 +2469,7 @@ class SpatialDilatedConvolution(Model): :param dilation_h: The number of pixels to skip. Default is 1. :param init_method: Init method, Default, Xavier. + >>> spatialDilatedConvolution = SpatialDilatedConvolution(1, 1, 1, 1) creating: createSpatialDilatedConvolution ''' @@ -2356,30 +2515,35 @@ class SpatialFullConvolution(Model): adjH values used to construct the module). This module can be used without a bias by setting parameter noBias = true while constructing the module. + If input is a 3D tensor nInputPlane x height x width, owidth = (width - 1) * dW - 2*padW + kW + adjW oheight = (height - 1) * dH - 2*padH + kH + adjH + Other frameworks call this operation "In-network Upsampling", "Fractionally-strided convolution", "Backwards Convolution," "Deconvolution", or "Upconvolution." + Reference Paper: Long J, Shelhamer E, Darrell T. Fully convolutional networks for semantic segmentation[C]//Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition. 2015: 3431-3440. - :param nInputPlane The number of expected input planes in the image given into forward() - :param nOutputPlane The number of output planes the convolution layer will produce. - :param kW The kernel width of the convolution. - :param kH The kernel height of the convolution. - :param dW The step of the convolution in the width dimension. Default is 1. - :param dH The step of the convolution in the height dimension. Default is 1. - :param padW The additional zeros added per width to the input planes. Default is 0. - :param padH The additional zeros added per height to the input planes. Default is 0. - :param adjW Extra width to add to the output image. Default is 0. - :param adjH Extra height to add to the output image. Default is 0. - :param nGroup Kernel group number. - :param noBias If bias is needed. - :param initMethod Init method, Default, Xavier, Bilinear. + + :param nInputPlane: The number of expected input planes in the image given into forward() + :param nOutputPlane: The number of output planes the convolution layer will produce. + :param kW: The kernel width of the convolution. + :param kH: The kernel height of the convolution. + :param dW: The step of the convolution in the width dimension. Default is 1. + :param dH: The step of the convolution in the height dimension. Default is 1. + :param padW: The additional zeros added per width to the input planes. Default is 0. + :param padH: The additional zeros added per height to the input planes. Default is 0. + :param adjW: Extra width to add to the output image. Default is 0. + :param adjH: Extra height to add to the output image. Default is 0. + :param nGroup: Kernel group number. + :param noBias: If bias is needed. + :param initMethod: Init method, Default, Xavier, Bilinear. + >>> spatialFullConvolution = SpatialFullConvolution(1, 1, 1, 1) creating: createSpatialFullConvolution @@ -2422,6 +2586,7 @@ def set_init_method(self, weight_init_method = None, bias_init_method = None): class SpatialShareConvolution(Model): ''' + >>> spatialShareConvolution = SpatialShareConvolution(1, 1, 1, 1) creating: createSpatialShareConvolution ''' @@ -2461,20 +2626,21 @@ class VolumetricConvolution(Model): ''' Applies a 3D convolution over an input image composed of several input planes. The input tensor in forward(input) is expected to be a 4D tensor (nInputPlane x time x height x width). - :param n_input_plane The number of expected input planes in the image given into forward() - :param n_output_plane The number of output planes the convolution layer will produce. - :param k_t The kernel size of the convolution in time - :param k_w The kernel width of the convolution - :param k_h The kernel height of the convolution - :param d_t The step of the convolution in the time dimension. Default is 1 - :param d_w The step of the convolution in the width dimension. Default is 1 - :param d_h The step of the convolution in the height dimension. Default is 1 - :param pad_t Additional zeros added to the input plane data on both sides of time axis. - Default is 0. (kT-1)/2 is often used here. - :param pad_w The additional zeros added per width to the input planes. - :param pad_h The additional zeros added per height to the input planes. - :param with_bias whether with bias - :param init_method Init method, Default, Xavier, Bilinear. + + :param n_input_plane: The number of expected input planes in the image given into forward() + :param n_output_plane: The number of output planes the convolution layer will produce. + :param k_t: The kernel size of the convolution in time + :param k_w: The kernel width of the convolution + :param k_h: The kernel height of the convolution + :param d_t: The step of the convolution in the time dimension. Default is 1 + :param d_w: The step of the convolution in the width dimension. Default is 1 + :param d_h: The step of the convolution in the height dimension. Default is 1 + :param pad_t: Additional zeros added to the input plane data on both sides of time axis.Default is 0. (kT-1)/2 is often used here. + :param pad_w: The additional zeros added per width to the input planes. + :param pad_h: The additional zeros added per height to the input planes. + :param with_bias: whether with bias + :param init_method: Init method, Default, Xavier, Bilinear. + >>> volumetricConvolution = VolumetricConvolution(6, 12, 5, 5, 5, 1, 1, 1) creating: createVolumetricConvolution @@ -2522,15 +2688,17 @@ class VolumetricMaxPooling(Model): The number of output features is equal to the number of input planes / dT. The input can optionally be padded with zeros. Padding should be smaller than half of kernel size. That is, padT < kT/2, padW < kW/2 and padH < kH/2 - :param k_t The kernel size - :param k_w The kernel width - :param k_h The kernel height - :param d_t The step in the time dimension - :param d_w The step in the width dimension - :param d_h The step in the height dimension - :param pad_t The padding in the time dimension - :param pad_w The padding in the width dimension - :param pad_h The padding in the height dimension + + :param k_t: The kernel size + :param k_w: The kernel width + :param k_h: The kernel height + :param d_t: The step in the time dimension + :param d_w: The step in the width dimension + :param d_h: The step in the height dimension + :param pad_t: The padding in the time dimension + :param pad_w: The padding in the width dimension + :param pad_h: The padding in the height dimension + >>> volumetricMaxPooling = VolumetricMaxPooling(5, 5, 5, 1, 1, 1) creating: createVolumetricMaxPooling @@ -2564,11 +2732,13 @@ class SpatialZeroPadding(Model): ''' Each feature map of a given input is padded with specified number of zeros. If padding values are negative, then input is cropped. + :param padLeft: pad left position :param padRight: pad right position :param padTop: pad top position :param padBottom: pad bottom position + >>> spatialZeroPadding = SpatialZeroPadding(1, 1, 1, 1) creating: createSpatialZeroPadding ''' @@ -2593,15 +2763,16 @@ class SplitTable(Model): outputs several tables, splitting the Tensor along the specified dimension `dimension`. + The input to this layer is expected to be a tensor, or a batch of tensors; when using mini-batch, a batch of sample tensors will be passed to the layer and the user need to specify the number of dimensions of each sample tensor in a batch using `nInputDims`. + :param dimension: to be split along this dimension - :param n_input_dims: specify the number of dimensions that this module will receive - If it is more than the dimension of input tensors, the first dimension - would be considered as batch size + :param n_input_dims: specify the number of dimensions that this module will receiveIf it is more than the dimension of input tensors, the first dimensionwould be considered as batch size + >>> splitTable = SplitTable(1, 1) creating: createSplitTable @@ -2621,6 +2792,7 @@ class Sqrt(Model): ''' Apply an element-wise sqrt operation. + >>> sqrt = Sqrt() creating: createSqrt ''' @@ -2634,6 +2806,7 @@ class Square(Model): ''' Apply an element-wise square operation. + >>> square = Square() creating: createSquare ''' @@ -2648,8 +2821,11 @@ class Squeeze(Model): ''' Delete singleton all dimensions or a specific dim. - :param dim Optional. The dimension to be delete. Default: delete all dimensions. - :param num_input_dims Optional. If in a batch model, set to the inputDims. + + :param dim: Optional. The dimension to be delete. Default: delete all dimensions. + :param num_input_dims: Optional. If in a batch model, set to the inputDims. + + >>> squeeze = Squeeze(1) @@ -2676,11 +2852,11 @@ class Sum(Model): the user need to specify the number of dimensions of each sample tensor in the batch using `nInputDims`. - :param dimension the dimension to be applied sum operation - :param n_input_dims specify the number of dimensions that this module will receive - If it is more than the dimension of input tensors, the first dimension - would be considered as batch size - :param size_average default is false, if it is true, it will return the mean instead + + :param dimension: the dimension to be applied sum operation + :param n_input_dims: specify the number of dimensions that this module will receiveIf it is more than the dimension of input tensors, the first dimensionwould be considered as batch size + :param size_average: default is false, if it is true, it will return the mean instead + >>> sum = Sum(1, 1, True) creating: createSum @@ -2704,6 +2880,7 @@ class TanhShrink(Model): during the forward process: [f(x) = tanh(x) - 1] + >>> tanhShrink = TanhShrink() creating: createTanhShrink ''' @@ -2719,9 +2896,11 @@ class Threshold(Model): Threshold input Tensor. If values in the Tensor smaller than th, then replace it with v - :param th the threshold to compare with - :param v the value to replace with - :param ip inplace mode + + :param th: the threshold to compare with + :param v: the value to replace with + :param ip: inplace mode + >>> threshold = Threshold(1e-5, 1e-5, True) creating: createThreshold @@ -2745,8 +2924,10 @@ class Unsqueeze(Model): For an input with dim = input.dim(), there are dim + 1 possible positions to insert the singleton dimension. - :param pos The position will be insert singleton. - :param num_input_dims Optional. If in a batch model, set to the inputDim + + :param pos: The position will be insert singleton. + :param num_input_dims: Optional. If in a batch model, set to the inputDim + >>> unsqueeze = Unsqueeze(1, 1) creating: createUnsqueeze @@ -2766,8 +2947,10 @@ class Reshape(Model): The forward(input) reshape the input tensor into a size(0) * size(1) * ... tensor, taking the elements row-wise. + :param size: the reshape size + >>> reshape = Reshape([1, 28, 28]) creating: createReshape >>> reshape = Reshape([1, 28, 28], False) @@ -2782,8 +2965,10 @@ class BiRecurrent(Container): ''' Create a Bidirectional recurrent layer + :param merge: merge layer + >>> biRecurrent = BiRecurrent(CAddTable()) creating: createCAddTable creating: createBiRecurrent @@ -2802,9 +2987,11 @@ class ConcatTable(Container): ConcateTable is a container module like Concate. Applies an input to each member module, input can be a tensor or a table. + ConcateTable usually works with CAddTable and CMulTable to implement element wise add/multiply on outputs of two modules. + >>> concatTable = ConcatTable() creating: createConcatTable ''' @@ -2818,6 +3005,7 @@ class Identity(Model): Identity just return the input to output. It's useful in same parallel container to get an origin input. + >>> identity = Identity() creating: createIdentity ''' @@ -2832,7 +3020,9 @@ class Reverse(Model): Reverse the input w.r.t given dimension. The input can be a Tensor or Table. - :param dim + + :param dim: + >>> reverse = Reverse() creating: createReverse @@ -2849,7 +3039,9 @@ class Transpose(Model): ''' Transpose input along specified dimensions - :param permutations dimension pairs that need to swap + + :param permutations: dimension pairs that need to swap + >>> transpose = Transpose([(1,2)]) creating: createTranspose @@ -2866,10 +3058,12 @@ class SpatialContrastiveNormalization(Model): ''' Subtractive + divisive contrast normalization. - :param n_input_plane - :param kernel - :param threshold - :param thresval + + :param n_input_plane: + :param kernel: + :param threshold: + :param thresval: + >>> kernel = np.ones([9,9]).astype("float32") >>> spatialContrastiveNormalization = SpatialContrastiveNormalization(1, kernel) @@ -2897,6 +3091,7 @@ class SpatialConvolutionMap(Model): It uses a generic connection table between input and output features. The SpatialConvolution is equivalent to using a full connection table. + >>> ct = np.ones([9,9]).astype("float32") >>> spatialConvolutionMap = SpatialConvolutionMap(ct, 9, 9) creating: createSpatialConvolutionMap @@ -2929,19 +3124,23 @@ class SpatialDivisiveNormalization(Model): an input image, since there is only one feature, the region is only spatial. For an RGB image, the weighted average is taken over RGB channels and a spatial region. + If the kernel is 1D, then it will be used for constructing and separable 2D kernel. The operations will be much more efficient in this case. + The kernel is generally chosen as a gaussian when it is believed that the correlation of two pixel locations decrease with increasing distance. On the feature dimension, a uniform average is used since the weighting across features is not known. - :param nInputPlane number of input plane, default is 1. - :param kernel kernel tensor, default is a 9 x 9 tensor. - :param threshold threshold - :param thresval threshhold value to replace with - if data is smaller than theshold + + + :param nInputPlane: number of input plane, default is 1. + :param kernel: kernel tensor, default is a 9 x 9 tensor. + :param threshold: threshold + :param thresval: threshhold value to replace withif data is smaller than theshold + >>> kernel = np.ones([9,9]).astype("float32") >>> spatialDivisiveNormalization = SpatialDivisiveNormalization(2,kernel) @@ -2967,18 +3166,21 @@ class Graph(Model): A graph container. Each node can have multiple inputs. The output of the node should be a tensor. The output tensor can be connected to multiple nodes. So the module in each node can have a tensor or table input, and should have a tensor output. - - The graph container can have multiple inputs and multiple outputs. If there's one input, - the input data fed to the graph module should be a tensor. If there're multiple inputs, - the input data fed to the graph module should be a table, which is actually an sequence of - tensor. The order of the input tensors should be same with the order of the input nodes. + + + The graph container can have multiple inputs and multiple outputs. If there's one input, + the input data fed to the graph module should be a tensor. If there're multiple inputs, + the input data fed to the graph module should be a table, which is actually an sequence of + tensor. The order of the input tensors should be same with the order of the input nodes. This is also applied to the gradient from the module in the back propagation. - + + If there's one output, the module output is a tensor. If there're multiple outputs, the module output is a table, which is actually an sequence of tensor. The order of the output tensors is same with the order of the output modules. This is also applied to the gradient passed to the module in the back propagation. - + + All inputs should be able to connect to outputs through some paths in the graph. It is allowed that some successors of the inputs node are not connect to outputs. If so, these nodes will be excluded in the computation. @@ -2998,15 +3200,19 @@ class SpatialSubtractiveNormalization(Model): an input image, since there is only one feature, the region is only spatial. For an RGB image, the weighted average is taken over RGB channels and a spatial region. + If the kernel is 1D, then it will be used for constructing and separable 2D kernel. The operations will be much more efficient in this case. + The kernel is generally chosen as a gaussian when it is believed that the correlation of two pixel locations decrease with increasing distance. On the feature dimension, a uniform average is used since the weighting across features is not known. - :param n_input_plane number of input plane, default is 1. - :param kernel kernel tensor, default is a 9 x 9 tensor. + + :param n_input_plane: number of input plane, default is 1. + :param kernel: kernel tensor, default is a 9 x 9 tensor. + >>> kernel = np.ones([9,9]).astype("float32") >>> spatialSubtractiveNormalization = SpatialSubtractiveNormalization(2,kernel) diff --git a/python/dllib/src/bigdl/dllib/optim/optimizer.py b/python/dllib/src/bigdl/dllib/optim/optimizer.py index e868c1473bd..f4e095fb293 100644 --- a/python/dllib/src/bigdl/dllib/optim/optimizer.py +++ b/python/dllib/src/bigdl/dllib/optim/optimizer.py @@ -33,11 +33,12 @@ class MaxIteration(JavaValue): """ - A trigger specifies a timespot or several timespots during training, - and a corresponding action will be taken when the timespot(s) is reached. - MaxIteration is a trigger that triggers an action when training reaches - the number of iterations specified by "max". - Usually used as end_trigger when creating an Optimizer. + A trigger specifies a timespot or several timespots during training, + and a corresponding action will be taken when the timespot(s) is reached. + MaxIteration is a trigger that triggers an action when training reaches + the number of iterations specified by "max". + Usually used as end_trigger when creating an Optimizer. + >>> maxIteration = MaxIteration(20) creating: createMaxIteration @@ -46,6 +47,7 @@ def __init__(self, max, bigdl_type="float"): """ Create a MaxIteration trigger. + :param max: max """ JavaValue.__init__(self, None, bigdl_type, max) @@ -59,13 +61,15 @@ class MaxEpoch(JavaValue): the number of epochs specified by "max_epoch". Usually used as end_trigger when creating an Optimizer. + >>> maxEpoch = MaxEpoch(2) creating: createMaxEpoch """ def __init__(self, max_epoch, bigdl_type="float"): """ Create a MaxEpoch trigger. - + + :param max_epoch: max_epoch """ JavaValue.__init__(self, None, bigdl_type, max_epoch) @@ -79,12 +83,13 @@ class EveryEpoch(JavaValue): Could be used as trigger in setvalidation and setcheckpoint in Optimizer, and also in TrainSummary.set_summary_trigger. + >>> everyEpoch = EveryEpoch() creating: createEveryEpoch """ def __init__(self, bigdl_type="float"): """ - Create a EveryEpoch trigger. + Create a EveryEpoch trigger. """ JavaValue.__init__(self, None, bigdl_type) @@ -93,11 +98,12 @@ class SeveralIteration(JavaValue): """ A trigger specifies a timespot or several timespots during training, and a corresponding action will be taken when the timespot(s) is reached. - SeveralIteration is a trigger that triggers an action every "n" + SeveralIteration is a trigger that triggers an action every "n" iterations. Could be used as trigger in setvalidation and setcheckpoint in Optimizer, and also in TrainSummary.set_summary_trigger. + >>> serveralIteration = SeveralIteration(2) creating: createSeveralIteration """ @@ -105,20 +111,22 @@ def __init__(self, interval, bigdl_type="float"): """ Create a SeveralIteration trigger. - :param interval: interval is the "n" where an action is triggered - every "n" iterations + + :param interval: interval is the "n" where an action is triggeredevery "n" iterations """ JavaValue.__init__(self, None, bigdl_type, interval) class Poly(JavaValue): """ - A learning rate decay policy, where the effective learning rate - follows a polynomial decay, to be zero by the max_iteration. + A learning rate decay policy, where the effective learning rate + follows a polynomial decay, to be zero by the max_iteration. Calculation: base_lr (1 - iter/max_iteration) ^ (power) - - :param power - :param max_iteration + + + :param power: + :param max_iteration: + >>> poly = Poly(0.5, 2) creating: createPoly @@ -129,11 +137,13 @@ def __init__(self, power, max_iteration, bigdl_type="float"): class Step(JavaValue): """ - A learning rate decay policy, where the effective learning rate is + A learning rate decay policy, where the effective learning rate is calculated as base_lr * gamma ^ (floor(iter / step_size)) - - :param step_size - :param gamma + + + :param step_size: + :param gamma: + >>> step = Step(2, 0.3) creating: createStep @@ -321,8 +331,10 @@ class MultiStep(JavaValue): """ similar to step but it allows non uniform steps defined by stepSizes - :param step_size the series of step sizes used for lr decay - :param gamma coefficient of decay + + :param step_size: the series of step sizes used for lr decay + :param gamma: coefficient of decay + >>> step = MultiStep([2, 5], 0.3) creating: createMultiStep @@ -333,10 +345,10 @@ def __init__(self, step_sizes, gamma, bigdl_type="float"): class Optimizer(JavaValue): """ - An optimizer is in general to minimize any function with respect - to a set of parameters. In case of training a neural network, - an optimizer tries to minimize the loss of the neural net with - respect to its weights/biases, over the training set. + An optimizer is in general to minimize any function with respect + to a set of parameters. In case of training a neural network, + an optimizer tries to minimize the loss of the neural net with + respect to its weights/biases, over the training set. """ def __init__(self, model, @@ -349,34 +361,36 @@ def __init__(self, """ Create an optimizer. + :param model: the neural net model - :param traiing_rdd: the training dataset + :param traiing_rdd: the training dataset :param criterion: the loss function :param optim_method: the algorithm to use for optimization, e.g. SGD, Adagrad, etc. If optim_method is None, the default algorithm is SGD. :param end_trigger: when to end the optimization :param batch_size: training batch size - """ + """ JavaValue.__init__(self, None, bigdl_type, model.value, training_rdd, criterion, optim_method if optim_method else SGD(), end_trigger, batch_size) def set_validation(self, batch_size, val_rdd, trigger, val_method=["Top1Accuracy"]): """ - Configure validation settings. + Configure validation settings. + :param batch_size: validation batch size :param val_rdd: validation dataset :param trigger: validation interval - :param val_method: the ValidationMethod to use, - e.g. "Top1Accuracy", "Top5Accuracy", "Loss" + :param val_method: the ValidationMethod to use,e.g. "Top1Accuracy", "Top5Accuracy", "Loss" """ callBigDlFunc(self.bigdl_type, "setValidation", self.value, batch_size, trigger, val_rdd, val_method) def set_model(self, model): """ - Set model. + Set model. + :param model: new model """ @@ -385,12 +399,12 @@ def set_model(self, model): def set_checkpoint(self, checkpoint_trigger, checkpoint_path, isOverWrite=True): """ - Configure checkpoint settings. + Configure checkpoint settings. + :param checkpoint_trigger: the interval to write snapshots :param checkpoint_path: the path to write snapshots into - :param isOverWrite: whether to overwrite existing snapshots in path. - default is True + :param isOverWrite: whether to overwrite existing snapshots in path.default is True """ if not os.path.exists(checkpoint_path): mkpath(checkpoint_path) @@ -400,7 +414,7 @@ def set_checkpoint(self, checkpoint_trigger, # return a module def optimize(self): """ - Do an optimization. + Do an optimization. """ jmodel = callJavaFunc(get_spark_context(), self.value.optimize) from bigdl.nn.layer import Model @@ -408,11 +422,12 @@ def optimize(self): def set_train_summary(self, summary): """ - Set train summary. A TrainSummary object contains information - necesary for the optimizer to know how often the logs are recorded, - where to store the logs and how to retrieve them, etc. For details, + Set train summary. A TrainSummary object contains information + necesary for the optimizer to know how often the logs are recorded, + where to store the logs and how to retrieve them, etc. For details, refer to the docs of TrainSummary. + :param summary: a TrainSummary object """ callBigDlFunc(self.bigdl_type, "setTrainSummary", self.value, @@ -421,13 +436,15 @@ def set_train_summary(self, summary): def set_val_summary(self, summary): """ - Set validation summary. A ValidationSummary object contains information - necesary for the optimizer to know how often the logs are recorded, - where to store the logs and how to retrieve them, etc. For details, + Set validation summary. A ValidationSummary object contains information + necesary for the optimizer to know how often the logs are recorded, + where to store the logs and how to retrieve them, etc. For details, refer to the docs of ValidationSummary. + :param summary: a ValidationSummary object + """ callBigDlFunc(self.bigdl_type, "setValSummary", self.value, summary) @@ -436,7 +453,7 @@ def set_val_summary(self, summary): def prepare_input(self): """ Load input. Notebook user can call this method to seprate load data and - create optimizer time + create optimizer time """ print("Loading input ...") self.value.prepareInput() @@ -444,22 +461,24 @@ def prepare_input(self): class TrainSummary(JavaValue, ): """ - A logging facility which allows user to trace how indicators (e.g. - learning rate, training loss, throughput, etc.) change with iterations/time - in an optimization process. TrainSummary is for training indicators only - (check ValidationSummary for validation indicators). It contains necessary - information for the optimizer to know where to store the logs, how to - retrieve the logs, and so on. - The logs are written in tensorflow-compatible - format so that they can be visualized directly using tensorboard. Also the - logs can be retrieved as ndarrays and visualized using python libraries - such as matplotlib (in notebook, etc.). - - Use optimizer.setTrainSummary to enable train logger. + A logging facility which allows user to trace how indicators (e.g. + learning rate, training loss, throughput, etc.) change with iterations/time + in an optimization process. TrainSummary is for training indicators only + (check ValidationSummary for validation indicators). It contains necessary + information for the optimizer to know where to store the logs, how to + retrieve the logs, and so on. - The logs are written in tensorflow-compatible + format so that they can be visualized directly using tensorboard. Also the + logs can be retrieved as ndarrays and visualized using python libraries + such as matplotlib (in notebook, etc.). + + + Use optimizer.setTrainSummary to enable train logger. """ def __init__(self, log_dir, app_name, bigdl_type="float"): """ Create a TrainSummary. Logs will be saved to log_dir/app_name/train. + :param log_dir: the root dir to store the logs :param app_name: the application name """ @@ -467,28 +486,21 @@ def __init__(self, log_dir, app_name, bigdl_type="float"): def read_scalar(self, tag): """ - Retrieve train logs by type. Return an array of records in the format + Retrieve train logs by type. Return an array of records in the format (step,value,wallClockTime). - "Step" is the iteration count by default. - - :param tag: the type of the logs, Supported tags are: "LearningRate", - "Loss", "Throughput" + + + :param tag: the type of the logs, Supported tags are: "LearningRate","Loss", "Throughput" """ return callBigDlFunc(self.bigdl_type, "summaryReadScalar", self.value, tag) def set_summary_trigger(self, name, trigger): """ - Set the interval of recording for each indicator. - - :param tag: tag name. Supported tag names are "LearningRate", "Loss", - "Throughput", "Parameters". "Parameters" is an umbrella tag that - includes weight, bias, gradWeight, gradBias, and some running status - (eg. runningMean and runningVar in BatchNormalization). If you - didn't set any triggers, we will by default record Loss and Throughput - in each iteration, while *NOT* recording LearningRate and Parameters, - as recording parameters may introduce substantial overhead when the - model is very big, LearningRate is not a public attribute for all - OptimMethod. + Set the interval of recording for each indicator. + + + :param tag: tag name. Supported tag names are "LearningRate", "Loss","Throughput", "Parameters". "Parameters" is an umbrella tag thatincludes weight, bias, gradWeight, gradBias, and some running status(eg. runningMean and runningVar in BatchNormalization). If youdidn't set any triggers, we will by default record Loss and Throughputin each iteration, while *NOT* recording LearningRate and Parameters,as recording parameters may introduce substantial overhead when themodel is very big, LearningRate is not a public attribute for allOptimMethod. :param trigger: trigger """ return callBigDlFunc(self.bigdl_type, "summarySetTrigger", self.value, @@ -497,25 +509,27 @@ def set_summary_trigger(self, name, trigger): class ValidationSummary(JavaValue): """ - A logging facility which allows user to trace how indicators (e.g. - validation loss, top1 accuray, top5 accuracy etc.) change with - iterations/time in an optimization process. ValidationSummary is for - validation indicators only (check TrainSummary for train indicators). - It contains necessary information for the optimizer to know where to - store the logs, how to retrieve the logs, and so on. - The logs are - written in tensorflow-compatible format so that they can be visualized - directly using tensorboard. Also the logs can be retrieved as ndarrays - and visualized using python libraries such as matplotlib + A logging facility which allows user to trace how indicators (e.g. + validation loss, top1 accuray, top5 accuracy etc.) change with + iterations/time in an optimization process. ValidationSummary is for + validation indicators only (check TrainSummary for train indicators). + It contains necessary information for the optimizer to know where to + store the logs, how to retrieve the logs, and so on. - The logs are + written in tensorflow-compatible format so that they can be visualized + directly using tensorboard. Also the logs can be retrieved as ndarrays + and visualized using python libraries such as matplotlib (in notebook, etc.). - + + Use optimizer.setValidationSummary to enable validation logger. """ def __init__(self, log_dir, app_name, bigdl_type="float"): """ - Create a ValidationSummary. Logs will be saved to - log_dir/app_name/train. By default, all ValidationMethod set into - optimizer will be recorded and the recording interval is the same - as trigger of ValidationMethod in the optimizer. + Create a ValidationSummary. Logs will be saved to + log_dir/app_name/train. By default, all ValidationMethod set into + optimizer will be recorded and the recording interval is the same + as trigger of ValidationMethod in the optimizer. + :param log_dir: the root dir to store the logs :param app_name: the application name @@ -524,13 +538,12 @@ def __init__(self, log_dir, app_name, bigdl_type="float"): def read_scalar(self, tag): """ - Retrieve validation logs by type. Return an array of records in the - format (step,value,wallClockTime). - "Step" is the iteration count + Retrieve validation logs by type. Return an array of records in the + format (step,value,wallClockTime). - "Step" is the iteration count by default. - :param tag: the type of the logs. The tag should match the name of - the ValidationMethod set into the optimizer. e.g. - "Top1AccuracyLoss","Top1Accuracy" or "Top5Accuracy". + + :param tag: the type of the logs. The tag should match the name ofthe ValidationMethod set into the optimizer. e.g."Top1AccuracyLoss","Top1Accuracy" or "Top5Accuracy". """ return callBigDlFunc(self.bigdl_type, "summaryReadScalar", self.value, tag) diff --git a/python/dllib/src/bigdl/utils/common.py b/python/dllib/src/bigdl/utils/common.py index 1839a62d8b8..f19e15df134 100644 --- a/python/dllib/src/bigdl/utils/common.py +++ b/python/dllib/src/bigdl/utils/common.py @@ -90,6 +90,7 @@ class TestResult(): """ def __init__(self, result, total_num, method): """ + :param result: the validation result. i.e: top1 accuracy percentage. :param total_num: the total processed records. :param method: the validation method. i.e: Top1Accuracy @@ -110,6 +111,7 @@ class JTensor(object): """ A wrapper to easy our work when need to pass or return Tensor to/from Scala. + >>> import numpy as np >>> from bigdl.util.common import JTensor >>> np.random.seed(123) @@ -124,6 +126,7 @@ def __init__(self, storage, shape, bigdl_type="float"): def from_ndarray(cls, a_ndarray, bigdl_type="float"): """ Convert a ndarray to Tensor which would be used in Java side. + >>> import numpy as np >>> from bigdl.util.common import JTensor >>> from bigdl.util.common import callBigDlFunc @@ -153,7 +156,9 @@ def get_dtype(): def flatten_ndarray(cls, a_ndarray): """ Utility method to flatten a ndarray - :return (storage, shape) + + :return: (storage, shape) + >>> from bigdl.util.common import JTensor >>> np.random.seed(123) >>> data = np.random.uniform(0, 1, (2, 3)) @@ -335,6 +340,7 @@ def callJavaFunc(sc, func, *args): def _to_java_object_rdd(rdd): """ Return a JavaRDD of Object by unpickling + It will convert each Python object into Java object by Pyrolite, whenever the RDD is serialized in batch or not. """ From c78c67a5db2aa123c3f1f487b9ca9e793eac2f15 Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Wed, 14 Jun 2017 09:25:47 +0800 Subject: [PATCH 049/823] Add local integration test (#994) * add local integration test * manually run local integration test --- .../src/bigdl/dllib/models/lenet/README.md | 8 ++++++++ .../src/bigdl/dllib/models/lenet/lenet5.py | 18 ++++++++++++++---- .../models/textclassifier/textclassifier.py | 3 ++- python/dllib/src/bigdl/dllib/nn/layer.py | 12 ++++++------ python/dllib/src/bigdl/dllib/utils/common.py | 6 +++--- 5 files changed, 33 insertions(+), 14 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/models/lenet/README.md b/python/dllib/src/bigdl/dllib/models/lenet/README.md index e537ccad99d..d1a79c3bfd5 100644 --- a/python/dllib/src/bigdl/dllib/models/lenet/README.md +++ b/python/dllib/src/bigdl/dllib/models/lenet/README.md @@ -53,6 +53,14 @@ We would train a LeNet model in spark local mode with the following commands and * ```--batchSize``` option can be used to set batch size, the default value is 128. +* ```--endTriggerType``` option can be used to control how to end the training process, the value can be "epoch" or "iteration" and default value is "epoch". + +* ```--endTriggerNum``` use together with ```endTriggerType```, the default value is 20. + +* ```--modelPath``` option can be used to set model path for testing, the default value is /tmp/lenet5/model.470. + +* ```--checkpointPath``` option can be used to set checkpoint path for saving model, the default value is /tmp/lenet5/. + To verify the accuracy, search "accuracy" from log: ``` diff --git a/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py b/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py index 68891fb8f09..b44e64a43e1 100644 --- a/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py +++ b/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py @@ -16,7 +16,6 @@ # Still in experimental stage! from optparse import OptionParser - from bigdl.dataset import mnist from bigdl.dataset.transformer import * from bigdl.nn.layer import * @@ -65,7 +64,10 @@ def get_minst(sc, data_type="train", location="/tmp/mnist"): parser = OptionParser() parser.add_option("-a", "--action", dest="action", default="train") parser.add_option("-b", "--batchSize", type=int, dest="batchSize", default="128") - parser.add_option("-m", "--modelPath", dest="modelPath", default="") + parser.add_option("-o", "--modelPath", dest="modelPath", default="/tmp/lenet5/model.470") + parser.add_option("-c", "--checkpointPath", dest="checkpointPath", default="/tmp/lenet5") + parser.add_option("-t", "--endTriggerType", dest="endTriggerType", default="epoch") + parser.add_option("-n", "--endTriggerNum", type=int, dest="endTriggerNum", default="20") (options, args) = parser.parse_args(sys.argv) @@ -73,16 +75,23 @@ def get_minst(sc, data_type="train", location="/tmp/mnist"): init_engine() if options.action == "train": + def get_end_trigger(): + if options.endTriggerType.lower() == "epoch": + return MaxEpoch(options.endTriggerNum) + else: + return MaxIteration(options.endTriggerNum) + train_data = get_minst(sc, "train").map( normalizer(mnist.TRAIN_MEAN, mnist.TRAIN_STD)) test_data = get_minst(sc, "test").map( normalizer(mnist.TEST_MEAN, mnist.TEST_STD)) + optimizer = Optimizer( model=build_model(10), training_rdd=train_data, criterion=ClassNLLCriterion(), optim_method=SGD(learningrate=0.01, learningrate_decay=0.0002), - end_trigger=MaxEpoch(20), + end_trigger=get_end_trigger(), batch_size=options.batchSize) optimizer.set_validation( batch_size=options.batchSize, @@ -90,7 +99,7 @@ def get_minst(sc, data_type="train", location="/tmp/mnist"): trigger=EveryEpoch(), val_method=["Top1Accuracy"] ) - optimizer.set_checkpoint(EveryEpoch(), "/tmp/lenet5/") + optimizer.set_checkpoint(EveryEpoch(), options.checkpointPath) trained_model = optimizer.optimize() parameters = trained_model.parameters() elif options.action == "test": @@ -101,3 +110,4 @@ def get_minst(sc, data_type="train", location="/tmp/mnist"): results = model.test(test_data, options.batchSize, ["Top1Accuracy"]) for result in results: print(result) + sc.stop() diff --git a/python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py b/python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py index 9c67606c852..c190f04222f 100644 --- a/python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py +++ b/python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py @@ -114,7 +114,7 @@ def train(sc, bword_to_ic = sc.broadcast(word_to_ic) w2v = news20.get_glove_w2v(dim=embedding_dim) - filtered_w2v = {w: v for w, v in w2v.items() if w in word_to_ic} + filtered_w2v = dict((w, v) for w, v in w2v.items() if w in word_to_ic) bfiltered_w2v = sc.broadcast(filtered_w2v) tokens_rdd = data_rdd.map(lambda text_label: @@ -173,5 +173,6 @@ def train(sc, train(sc, batch_size, sequence_len, max_words, embedding_dim, training_split) + sc.stop() elif options.action == "test": pass diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 5f3e157d0e1..58fe8ec276f 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -199,13 +199,13 @@ def parameters(self): self.value) def to_ndarray(params): - return { - param_name: np.array(values[0], - dtype=self.get_dtype()).reshape( - values[1]) for param_name, values in params.items()} + return dict((param_name, + np.array(values[0], dtype=self.get_dtype()).reshape( + values[1])) for param_name, values in + params.items()) - return {layer_name: to_ndarray(params) for layer_name, params in - name_to_params.items()} + return dict((layer_name, to_ndarray(params)) for layer_name, params in + name_to_params.items()) def predict(self, data_rdd): """ diff --git a/python/dllib/src/bigdl/dllib/utils/common.py b/python/dllib/src/bigdl/dllib/utils/common.py index f19e15df134..c51b41e1b08 100644 --- a/python/dllib/src/bigdl/dllib/utils/common.py +++ b/python/dllib/src/bigdl/dllib/utils/common.py @@ -277,13 +277,13 @@ def create_spark_conf(): return sparkConf -def get_spark_context(): +def get_spark_context(conf = SparkConf()): if "getOrCreate" in SparkContext.__dict__: - return SparkContext.getOrCreate() + return SparkContext.getOrCreate(conf) else: with SparkContext._lock: # Compatible with Spark1.5.1 if SparkContext._active_spark_context is None: - SparkContext(SparkConf()) + SparkContext(conf) return SparkContext._active_spark_context From ef040bdd6fcf5e3faf3dd4e98df95d6b42eb5be2 Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Wed, 14 Jun 2017 09:25:47 +0800 Subject: [PATCH 050/823] Add local integration test (#994) * add local integration test * manually run local integration test --- .../src/bigdl/dllib/models/lenet/README.md | 8 ++++++++ .../src/bigdl/dllib/models/lenet/lenet5.py | 18 ++++++++++++++---- .../models/textclassifier/textclassifier.py | 3 ++- python/dllib/src/bigdl/dllib/nn/layer.py | 12 ++++++------ python/dllib/src/bigdl/utils/common.py | 6 +++--- 5 files changed, 33 insertions(+), 14 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/models/lenet/README.md b/python/dllib/src/bigdl/dllib/models/lenet/README.md index e537ccad99d..d1a79c3bfd5 100644 --- a/python/dllib/src/bigdl/dllib/models/lenet/README.md +++ b/python/dllib/src/bigdl/dllib/models/lenet/README.md @@ -53,6 +53,14 @@ We would train a LeNet model in spark local mode with the following commands and * ```--batchSize``` option can be used to set batch size, the default value is 128. +* ```--endTriggerType``` option can be used to control how to end the training process, the value can be "epoch" or "iteration" and default value is "epoch". + +* ```--endTriggerNum``` use together with ```endTriggerType```, the default value is 20. + +* ```--modelPath``` option can be used to set model path for testing, the default value is /tmp/lenet5/model.470. + +* ```--checkpointPath``` option can be used to set checkpoint path for saving model, the default value is /tmp/lenet5/. + To verify the accuracy, search "accuracy" from log: ``` diff --git a/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py b/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py index 68891fb8f09..b44e64a43e1 100644 --- a/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py +++ b/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py @@ -16,7 +16,6 @@ # Still in experimental stage! from optparse import OptionParser - from bigdl.dataset import mnist from bigdl.dataset.transformer import * from bigdl.nn.layer import * @@ -65,7 +64,10 @@ def get_minst(sc, data_type="train", location="/tmp/mnist"): parser = OptionParser() parser.add_option("-a", "--action", dest="action", default="train") parser.add_option("-b", "--batchSize", type=int, dest="batchSize", default="128") - parser.add_option("-m", "--modelPath", dest="modelPath", default="") + parser.add_option("-o", "--modelPath", dest="modelPath", default="/tmp/lenet5/model.470") + parser.add_option("-c", "--checkpointPath", dest="checkpointPath", default="/tmp/lenet5") + parser.add_option("-t", "--endTriggerType", dest="endTriggerType", default="epoch") + parser.add_option("-n", "--endTriggerNum", type=int, dest="endTriggerNum", default="20") (options, args) = parser.parse_args(sys.argv) @@ -73,16 +75,23 @@ def get_minst(sc, data_type="train", location="/tmp/mnist"): init_engine() if options.action == "train": + def get_end_trigger(): + if options.endTriggerType.lower() == "epoch": + return MaxEpoch(options.endTriggerNum) + else: + return MaxIteration(options.endTriggerNum) + train_data = get_minst(sc, "train").map( normalizer(mnist.TRAIN_MEAN, mnist.TRAIN_STD)) test_data = get_minst(sc, "test").map( normalizer(mnist.TEST_MEAN, mnist.TEST_STD)) + optimizer = Optimizer( model=build_model(10), training_rdd=train_data, criterion=ClassNLLCriterion(), optim_method=SGD(learningrate=0.01, learningrate_decay=0.0002), - end_trigger=MaxEpoch(20), + end_trigger=get_end_trigger(), batch_size=options.batchSize) optimizer.set_validation( batch_size=options.batchSize, @@ -90,7 +99,7 @@ def get_minst(sc, data_type="train", location="/tmp/mnist"): trigger=EveryEpoch(), val_method=["Top1Accuracy"] ) - optimizer.set_checkpoint(EveryEpoch(), "/tmp/lenet5/") + optimizer.set_checkpoint(EveryEpoch(), options.checkpointPath) trained_model = optimizer.optimize() parameters = trained_model.parameters() elif options.action == "test": @@ -101,3 +110,4 @@ def get_minst(sc, data_type="train", location="/tmp/mnist"): results = model.test(test_data, options.batchSize, ["Top1Accuracy"]) for result in results: print(result) + sc.stop() diff --git a/python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py b/python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py index 9c67606c852..c190f04222f 100644 --- a/python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py +++ b/python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py @@ -114,7 +114,7 @@ def train(sc, bword_to_ic = sc.broadcast(word_to_ic) w2v = news20.get_glove_w2v(dim=embedding_dim) - filtered_w2v = {w: v for w, v in w2v.items() if w in word_to_ic} + filtered_w2v = dict((w, v) for w, v in w2v.items() if w in word_to_ic) bfiltered_w2v = sc.broadcast(filtered_w2v) tokens_rdd = data_rdd.map(lambda text_label: @@ -173,5 +173,6 @@ def train(sc, train(sc, batch_size, sequence_len, max_words, embedding_dim, training_split) + sc.stop() elif options.action == "test": pass diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 5f3e157d0e1..58fe8ec276f 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -199,13 +199,13 @@ def parameters(self): self.value) def to_ndarray(params): - return { - param_name: np.array(values[0], - dtype=self.get_dtype()).reshape( - values[1]) for param_name, values in params.items()} + return dict((param_name, + np.array(values[0], dtype=self.get_dtype()).reshape( + values[1])) for param_name, values in + params.items()) - return {layer_name: to_ndarray(params) for layer_name, params in - name_to_params.items()} + return dict((layer_name, to_ndarray(params)) for layer_name, params in + name_to_params.items()) def predict(self, data_rdd): """ diff --git a/python/dllib/src/bigdl/utils/common.py b/python/dllib/src/bigdl/utils/common.py index f19e15df134..c51b41e1b08 100644 --- a/python/dllib/src/bigdl/utils/common.py +++ b/python/dllib/src/bigdl/utils/common.py @@ -277,13 +277,13 @@ def create_spark_conf(): return sparkConf -def get_spark_context(): +def get_spark_context(conf = SparkConf()): if "getOrCreate" in SparkContext.__dict__: - return SparkContext.getOrCreate() + return SparkContext.getOrCreate(conf) else: with SparkContext._lock: # Compatible with Spark1.5.1 if SparkContext._active_spark_context is None: - SparkContext(SparkConf()) + SparkContext(conf) return SparkContext._active_spark_context From 1027ff7f8c1c5bfaf576c3178bf83cdae6a0b6bc Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Wed, 14 Jun 2017 09:25:47 +0800 Subject: [PATCH 051/823] Add local integration test (#994) * add local integration test * manually run local integration test --- python/dllib/test/dev/prepare_env.sh | 49 +++++++++++++++ python/dllib/test/dev/run-tests | 18 +----- ...n-local-functional_api_forward_backward.sh | 21 +++++++ .../commands/run-local-lenet.sh | 26 ++++++++ .../commands/run-local-textclassifier.sh | 19 ++++++ .../test/local_integration/run-all-local.sh | 63 +++++++++++++++++++ 6 files changed, 179 insertions(+), 17 deletions(-) create mode 100755 python/dllib/test/dev/prepare_env.sh create mode 100755 python/dllib/test/local_integration/commands/run-local-functional_api_forward_backward.sh create mode 100755 python/dllib/test/local_integration/commands/run-local-lenet.sh create mode 100755 python/dllib/test/local_integration/commands/run-local-textclassifier.sh create mode 100755 python/dllib/test/local_integration/run-all-local.sh diff --git a/python/dllib/test/dev/prepare_env.sh b/python/dllib/test/dev/prepare_env.sh new file mode 100755 index 00000000000..c33b0eb12f3 --- /dev/null +++ b/python/dllib/test/dev/prepare_env.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash + +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +SCRIPT_DIR=$(dirname ${BASH_SOURCE[0]}) +echo "SCRIPT_DIR": $SCRIPT_DIR +export DL_PYTHON_HOME="$(cd ${SCRIPT_DIR}/../../; pwd)" + +export BIGDL_HOME="$(cd ${SCRIPT_DIR}/../../..; pwd)" + +echo "BIGDL_HOME: $BIGDL_HOME" +echo "SPARK_HOME": $SPARK_HOME +echo "DL_PYTHON_HOME": $DL_PYTHON_HOME + +if [ -z ${SPARK_HOME+x} ]; then echo "SPARK_HOME is unset"; exit 1; else echo "SPARK_HOME is set to '$SPARK_HOME'"; fi + +export PYSPARK_ZIP=`find $SPARK_HOME/python/lib -type f -iname '*.zip' | tr "\n" ":"` + +export PYTHONPATH=$PYSPARK_ZIP:$DL_PYTHON_HOME:$DL_PYTHON_HOME/:$DL_PYTHON_HOME/test/dev:$BIGDL_HOME/spark/dl/src/main/resources/spark-bigdl.conf + +export SPARK_CLASSPATH=$(find $BIGDL_HOME/spark/dl/target/ -name "*with-dependencies.jar" | head -n 1) +echo "SPARK_CLASSPATH": $SPARK_CLASSPATH +source $BIGDL_HOME/scripts/bigdl.sh + +export PYTHON_EXECUTABLES=("python2" "python3") + +function run_notebook() { + notebook_path=$1 + target_notebook_path=${DL_PYTHON_HOME}/tmp_${PYTHON_EXECUTABLE}.ipynb + echo "Change kernel to $PYTHON_EXECUTABLE" + sed "s/\"python.\"/\"$PYTHON_EXECUTABLE\"/g" $notebook_path > ${target_notebook_path} + jupyter nbconvert --to notebook --execute \ + --ExecutePreprocessor.timeout=360 --output ${DL_PYTHON_HOME}/tmp_out.ipynb \ + $target_notebook_path +} + +export -f run_notebook diff --git a/python/dllib/test/dev/run-tests b/python/dllib/test/dev/run-tests index f812eacb8c5..f260b372dd4 100755 --- a/python/dllib/test/dev/run-tests +++ b/python/dllib/test/dev/run-tests @@ -16,23 +16,7 @@ # limitations under the License. # - -DL_PYTHON_HOME="$(cd "`dirname $0`"/../..; pwd)" - -BIGDL_HOME="$(cd "`dirname $0`"/../../..; pwd)" -BIGDL_VERSION=0.2.0-SNAPSHOT - -echo "BIGDL_HOME: $BIGDL_HOME" -echo "SPARK_HOME": $SPARK_HOME -echo "DL_PYTHON_HOME": $DL_PYTHON_HOME - -if [ -z ${SPARK_HOME+x} ]; then echo "SPARK_HOME is unset"; exit 1; else echo "SPARK_HOME is set to '$SPARK_HOME'"; fi - -PYSPARK_ZIP=`find $SPARK_HOME/python/lib -type f -iname '*.zip' | tr "\n" ":"` - -export PYTHONPATH=$PYSPARK_ZIP:$DL_PYTHON_HOME:$DL_PYTHON_HOME/:$DL_PYTHON_HOME/test/dev:$BIGDL_HOME/spark/dl/src/main/resources/spark-bigdl.conf - -export SPARK_CLASSPATH=$BIGDL_HOME/spark/dl/target/bigdl-$BIGDL_VERSION-jar-with-dependencies.jar +. `dirname $0`/prepare_env.sh cd "`dirname $0`" exec $BIGDL_HOME/scripts/bigdl.sh -- python -u ./run-tests.py "$@" diff --git a/python/dllib/test/local_integration/commands/run-local-functional_api_forward_backward.sh b/python/dllib/test/local_integration/commands/run-local-functional_api_forward_backward.sh new file mode 100755 index 00000000000..1368e1e164a --- /dev/null +++ b/python/dllib/test/local_integration/commands/run-local-functional_api_forward_backward.sh @@ -0,0 +1,21 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +cd "`dirname $0`" + +run_notebook $BIGDL_HOME/pyspark/example/functional_api_forward_backward.ipynb + + diff --git a/python/dllib/test/local_integration/commands/run-local-lenet.sh b/python/dllib/test/local_integration/commands/run-local-lenet.sh new file mode 100755 index 00000000000..8b69f03d993 --- /dev/null +++ b/python/dllib/test/local_integration/commands/run-local-lenet.sh @@ -0,0 +1,26 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +cd "`dirname $0`" + +($PYTHON_EXECUTABLE ${BIGDL_HOME}/pyspark/bigdl/models/lenet/lenet5.py \ + --action train \ + --endTriggerType epoch \ + --endTriggerNum 1) && \ + +($PYTHON_EXECUTABLE ${BIGDL_HOME}/pyspark/bigdl/models/lenet/lenet5.py \ + --action test \ + --modelPath $(find /tmp/lenet5 -name "model*" | head -n 1)) diff --git a/python/dllib/test/local_integration/commands/run-local-textclassifier.sh b/python/dllib/test/local_integration/commands/run-local-textclassifier.sh new file mode 100755 index 00000000000..7cbd26e3bf1 --- /dev/null +++ b/python/dllib/test/local_integration/commands/run-local-textclassifier.sh @@ -0,0 +1,19 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +$PYTHON_EXECUTABLE ${BIGDL_HOME}/pyspark/bigdl/models/textclassifier/textclassifier.py \ + --max_epoch 1 \ + --model cnn diff --git a/python/dllib/test/local_integration/run-all-local.sh b/python/dllib/test/local_integration/run-all-local.sh new file mode 100755 index 00000000000..fe992ea8943 --- /dev/null +++ b/python/dllib/test/local_integration/run-all-local.sh @@ -0,0 +1,63 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +#!/bin/bash +set -e +RUN_SCRIPT_DIR=$(dirname ${BASH_SOURCE[0]}) + +DEV_DIR="$(cd ${RUN_SCRIPT_DIR}/../dev; pwd)" +echo "DEV_DIR": $DEV_DIR +cd "`dirname $0`" + +. $DEV_DIR/prepare_env.sh + +red=`tput setaf 1` +green=`tput setaf 2` +cyan=`tput setaf 6` +reset=`tput sgr0` +PYTHON_EXECUTABLES=("python2" "python3") + +function cleanup { + find ${DL_PYTHON_HOME} -name "tmp_*.ipynb" -exec rm {} \; + tput sgr0 +} + +function start_integration () { + for command in $(find . -name "run-local*.sh") + do + printf "${green}Running : $command ${green}" + start=`date +%s` + $command > run-all-local.log + if (( $? )); then + printf " ${red}Failed. \nPlease check ${DEV_DIR}/../local_integration/run-all-local.log\n${reset}"; + exit 1; + fi + end=`date +%s` + printf " [Succeeded! Took $((end-start)) secs] \n" + done + + find ${RUN_SCRIPT_DIR} -name "run-all-local.log" -exec rm {} \; +} + +trap cleanup EXIT + +for p in ${PYTHON_EXECUTABLES[@]} +do + echo "${cyan}Using python version: $p${reset}" + export PYTHON_EXECUTABLE=$p + export PYSPARK_PYTHON=$p + start_integration +done From 3c43d377aeacb51eef27bb868670d82c34be0b0d Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Wed, 14 Jun 2017 10:19:55 +0800 Subject: [PATCH 052/823] Rename Graph to Model (#982) * Rename Graph to Model * fix model --- python/dllib/src/bigdl/dllib/nn/criterion.py | 12 +- python/dllib/src/bigdl/dllib/nn/layer.py | 327 +++++++++--------- .../dllib/src/bigdl/dllib/optim/optimizer.py | 4 +- 3 files changed, 174 insertions(+), 169 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/nn/criterion.py b/python/dllib/src/bigdl/dllib/nn/criterion.py index 83f5f4113a1..20c2e46350c 100644 --- a/python/dllib/src/bigdl/dllib/nn/criterion.py +++ b/python/dllib/src/bigdl/dllib/nn/criterion.py @@ -20,7 +20,7 @@ from bigdl.util.common import JavaValue from bigdl.util.common import callBigDlFunc from bigdl.util.common import JTensor -from bigdl.nn.layer import Model +from bigdl.nn.layer import Layer import numpy as np if sys.version >= '3': @@ -54,8 +54,8 @@ def forward(self, input, target): output = callBigDlFunc(self.bigdl_type, "criterionForward", self.value, - Model.check_input(input), - Model.check_input(target)) + Layer.check_input(input), + Layer.check_input(target)) return output def backward(self, input, target): @@ -70,9 +70,9 @@ def backward(self, input, target): output = callBigDlFunc(self.bigdl_type, "criterionBackward", self.value, - Model.check_input(input), - Model.check_input(target)) - return Model.convert_output(output) + Layer.check_input(input), + Layer.check_input(target)) + return Layer.convert_output(output) @classmethod def of(cls, jcriterion, bigdl_type="float"): diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 58fe8ec276f..aa57fd640f3 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -49,14 +49,14 @@ def of(cls, jvalue, bigdl_type="float"): return Node(jvalue, bigdl_type) def element(self): - return Model.of(self.value.element()) + return Layer.of(self.value.element()) -class Model(JavaValue): +class Layer(JavaValue): """ - Model is the basic component of a neural network + Layer is the basic component of a neural network and it's also the base class of layers. - Model can connect to others to construct a complex neural network. + Layer can connect to others to construct a complex neural network. """ def __init__(self, jvalue, bigdl_type, *args): @@ -87,14 +87,13 @@ def __call__(self, x=None): to_list(x))) @classmethod - def of(cls, jmodel, bigdl_type="float"): + def of(cls, jvalue, bigdl_type="float"): """ - Create a Python Model - - :param jmodel: Java model create by Py4j - :return: A Python Model + Create a Python Layer base on the given java value + :param jvalue: Java object create by Py4j + :return: A Python Layer """ - model = Model(jmodel,bigdl_type) + model = Layer(jvalue, bigdl_type) return model def set_name(self, name): @@ -126,6 +125,7 @@ def get_dtype(self): return "float32" else: return "float64" + @staticmethod def check_input(input): if type(input) is list: @@ -288,6 +288,55 @@ def get_weights(self): print("The layer does not have weight/bias") return None + def save(self, path, over_write = False): + callBigDlFunc(self.bigdl_type, "modelSave", self.value, path, + over_write) + + +class Container(Layer): + ''' + [[Container]] is a sub-class of Model that declares methods defined in all containers. + A container usually contain some other modules which can be added through the "add" method + ''' + + def __init__(self, jvalue, bigdl_type, *args): + super(Container, self).__init__(jvalue, bigdl_type, *args) + + def add(self, model): + self.value.add(model.value) + return self + + +class Model(Container): + """ + A graph container. Each node can have multiple inputs. The output of the node should be a + tensor. The output tensor can be connected to multiple nodes. So the module in each node can + have a tensor or table input, and should have a tensor output. + + The graph container can have multiple inputs and multiple outputs. If there's one input, + the input data fed to the graph module should be a tensor. If there're multiple inputs, + the input data fed to the graph module should be a table, which is actually an sequence of + tensor. The order of the input tensors should be same with the order of the input nodes. + This is also applied to the gradient from the module in the back propagation. + + If there's one output, the module output is a tensor. If there're multiple outputs, the module + output is a table, which is actually an sequence of tensor. The order of the output tensors is + same with the order of the output modules. This is also applied to the gradient passed to the + module in the back propagation. + + All inputs should be able to connect to outputs through some paths in the graph. + It is allowed that some successors of the inputs node are not connect to outputs. + If so, these nodes will be excluded in the computation. + """ + + def __init__(self, + inputs, + outputs, + bigdl_type="float"): + super(Model, self).__init__(None, bigdl_type, + to_list(inputs), + to_list(outputs)) + @staticmethod def load(path, bigdl_type="float"): """ @@ -297,7 +346,7 @@ def load(path, bigdl_type="float"): :return: A pre-trained model. """ jmodel = callBigDlFunc(bigdl_type, "loadBigDL", path) - return Model.of(jmodel) + return Layer.of(jmodel) @staticmethod def load_torch(path, bigdl_type="float"): @@ -308,7 +357,7 @@ def load_torch(path, bigdl_type="float"): :return: A pre-trained model. """ jmodel = callBigDlFunc(bigdl_type, "loadTorch", path) - return Model.of(jmodel) + return Layer.of(jmodel) @staticmethod def load_caffe(model, defPath, modelPath, match_all=True, bigdl_type="float"): @@ -322,25 +371,10 @@ def load_caffe(model, defPath, modelPath, match_all=True, bigdl_type="float"): :return: A pre-trained model. """ jmodel = callBigDlFunc(bigdl_type, "loadCaffe", model, defPath, modelPath, match_all) - return Model.of(jmodel) + return Layer.of(jmodel) -class Container(Model): - ''' - [[Container]] is a sub-class of Model that declares methods defined in all containers. - A container usually contain some other modules which can be added through the "add" method - ''' - - def __init__(self, jvalue, bigdl_type, *args): - super(Container, self).__init__(jvalue, bigdl_type, *args) - - def add(self, model): - self.value.add(model.value) - return self - - - -class Linear(Model): +class Linear(Layer): ''' The [[Linear]] module applies a linear transformation to the input data, @@ -370,7 +404,7 @@ def set_init_method(self, weight_init_method = None, bias_init_method = None): weight_init_method, bias_init_method) -class ReLU(Model): +class ReLU(Layer): ''' Applies the rectified linear unit (ReLU) function element-wise to the input Tensor, @@ -389,7 +423,7 @@ def __init__(self, ip=False, bigdl_type="float"): super(ReLU, self).__init__(None, bigdl_type, ip) -class Tanh(Model): +class Tanh(Layer): ''' Applies the Tanh function element-wise to the input Tensor, thus outputting a Tensor of the same @@ -404,7 +438,7 @@ def __init__(self, bigdl_type="float"): super(Tanh, self).__init__(None, bigdl_type) -class Echo(Model): +class Echo(Layer): ''' This module is for debug purpose, which can print activation and gradient in your model @@ -419,7 +453,7 @@ def __init__(self, bigdl_type="float"): super(Echo, self).__init__(None, bigdl_type) -class LogSoftMax(Model): +class LogSoftMax(Layer): ''' Applies the LogSoftMax function to an n-dimensional input Tensor. @@ -457,7 +491,7 @@ def __init__(self, bigdl_type="float"): super(Sequential, self).__init__(None, bigdl_type) -class SpatialConvolution(Model): +class SpatialConvolution(Layer): ''' Applies a 2D convolution over an input image composed of several input planes. @@ -512,7 +546,7 @@ def set_init_method(self, weight_init_method = None, bias_init_method = None): weight_init_method, bias_init_method) -class SpatialMaxPooling(Model): +class SpatialMaxPooling(Layer): ''' Applies 2D max-pooling operation in kWxkH regions by step size dWxdH steps. @@ -553,7 +587,7 @@ def __init__(self, kw, to_ceil) -class Select(Model): +class Select(Layer): ''' A Simple layer selecting an index of the input tensor in the given dimension @@ -584,7 +618,7 @@ def __init__(self, bigdl_type="float"): super(Recurrent, self).__init__(None, bigdl_type) -class LSTM(Model): +class LSTM(Layer): ''' | Long Short Term Memory architecture. | Ref. @@ -608,7 +642,7 @@ def __init__(self, input_size, hidden_size, p=0.0, bigdl_type="float"): super(LSTM, self).__init__(None, bigdl_type, input_size, hidden_size, p) -class LSTMPeephole(Model): +class LSTMPeephole(Layer): ''' | Long Short Term Memory architecture with peephole. | Ref. A.: http://arxiv.org/pdf/1303.5778v1 (blueprint for this module) @@ -631,7 +665,7 @@ def __init__(self, input_size, hidden_size, p=0.0, bigdl_type="float"): super(LSTMPeephole, self).__init__(None, bigdl_type, input_size, hidden_size, p) -class GRU(Model): +class GRU(Layer): ''' Gated Recurrent Units architecture. The first input in sequence uses zero value for cell and hidden state @@ -655,7 +689,7 @@ def __init__(self, input_size, hidden_size, p=0.0, bigdl_type="float"): super(GRU, self).__init__(None, bigdl_type, input_size, hidden_size, p) -class RnnCell(Model): +class RnnCell(Layer): ''' It is a simple RNN. User can pass an activation function to the RNN. @@ -678,7 +712,7 @@ def __init__(self, super(RnnCell, self).__init__(None, bigdl_type, input_size, hidden_size, activation) -class TimeDistributed(Model): +class TimeDistributed(Layer): ''' This layer is intended to apply contained layer to each temporal time slice of input tensor. @@ -727,7 +761,7 @@ def __init__(self, dimension) -class SpatialAveragePooling(Model): +class SpatialAveragePooling(Layer): ''' Applies 2D average-pooling operation in kWxkH regions by step size dWxdH steps. @@ -772,7 +806,7 @@ def __init__(self, divide) -class SpatialBatchNormalization(Model): +class SpatialBatchNormalization(Layer): ''' This file implements Batch Normalization as described in the paper: @@ -813,7 +847,7 @@ def set_init_method(self, weight_init_method = None, bias_init_method = None): weight_init_method, bias_init_method) -class SpatialCrossMapLRN(Model): +class SpatialCrossMapLRN(Layer): ''' Applies Spatial Local Response Normalization between different feature maps. @@ -851,7 +885,7 @@ def __init__(self, k) -class Dropout(Model): +class Dropout(Layer): ''' Dropout masks(set to zero) parts of input using a bernoulli distribution. @@ -880,7 +914,7 @@ def __init__(self, scale) -class View(Model): +class View(Layer): ''' This module creates a new view of the input tensor using the sizes passed to the constructor. @@ -905,7 +939,7 @@ def __init__(self, num_input_dims) -class Abs(Model): +class Abs(Layer): ''' an element-wise abs operation @@ -920,7 +954,7 @@ def __init__(self, super(Abs, self).__init__(None, bigdl_type) -class Add(Model): +class Add(Layer): ''' adds a bias term to input data ; @@ -941,7 +975,7 @@ def set_init_method(self, weight_init_method = None, bias_init_method = None): weight_init_method, bias_init_method) -class AddConstant(Model): +class AddConstant(Layer): ''' adding a constant @@ -963,7 +997,7 @@ def __init__(self, constant_scalar, inplace) -class BatchNormalization(Model): +class BatchNormalization(Layer): ''' This layer implements Batch Normalization as described in the paper: @@ -1010,7 +1044,7 @@ def set_init_method(self, weight_init_method = None, bias_init_method = None): weight_init_method, bias_init_method) -class Bilinear(Model): +class Bilinear(Layer): ''' a bilinear transformation with sparse inputs, @@ -1071,7 +1105,7 @@ def __init__(self, n_output_dim1) -class CAdd(Model): +class CAdd(Layer): ''' This layer has a bias tensor with given size. The bias will be added element wise to the input @@ -1100,7 +1134,7 @@ def set_init_method(self, weight_init_method = None, bias_init_method = None): weight_init_method, bias_init_method) -class CAddTable(Model): +class CAddTable(Layer): ''' Merge the input tensors in the input table by element wise adding them together. The input @@ -1121,7 +1155,7 @@ def __init__(self, inplace) -class CDivTable(Model): +class CDivTable(Layer): ''' Takes a table with two Tensor and returns the component-wise division between them. @@ -1136,7 +1170,7 @@ def __init__(self, super(CDivTable, self).__init__(None, bigdl_type) -class CMaxTable(Model): +class CMaxTable(Layer): ''' Takes a table of Tensors and outputs the max of all of them. @@ -1151,7 +1185,7 @@ def __init__(self, super(CMaxTable, self).__init__(None, bigdl_type) -class CMinTable(Model): +class CMinTable(Layer): ''' Takes a table of Tensors and outputs the min of all of them. @@ -1165,7 +1199,7 @@ def __init__(self, super(CMinTable, self).__init__(None, bigdl_type) -class CMul(Model): +class CMul(Layer): ''' Applies a component-wise multiplication to the incoming data @@ -1189,7 +1223,7 @@ def set_init_method(self, weight_init_method = None, bias_init_method = None): weight_init_method, bias_init_method) -class CMulTable(Model): +class CMulTable(Layer): ''' Takes a table of Tensors and outputs the multiplication of all of them. @@ -1204,7 +1238,7 @@ def __init__(self, super(CMulTable, self).__init__(None, bigdl_type) -class CSubTable(Model): +class CSubTable(Layer): ''' Takes a table with two Tensor and returns the component-wise subtraction between them. @@ -1219,7 +1253,7 @@ def __init__(self, super(CSubTable, self).__init__(None, bigdl_type) -class Clamp(Model): +class Clamp(Layer): ''' Clamps all elements into the range [min_value, max_value]. @@ -1245,7 +1279,7 @@ def __init__(self, max) -class Contiguous(Model): +class Contiguous(Layer): ''' used to make input, grad_output both contiguous @@ -1260,7 +1294,7 @@ def __init__(self, super(Contiguous, self).__init__(None, bigdl_type) -class Cosine(Model): +class Cosine(Layer): ''' Cosine calculates the cosine similarity of the input to k mean centers. The input given in @@ -1290,7 +1324,7 @@ def set_init_method(self, weight_init_method = None, bias_init_method = None): weight_init_method, bias_init_method) -class CosineDistance(Model): +class CosineDistance(Layer): ''' Outputs the cosine distance between inputs @@ -1304,7 +1338,7 @@ def __init__(self, bigdl_type="float"): super(CosineDistance, self).__init__(None, bigdl_type) -class DiceCoefficientCriterion(Model): +class DiceCoefficientCriterion(Layer): ''' The Dice-Coefficient criterion @@ -1328,7 +1362,7 @@ def __init__(self, size_average, epsilon) -class DotProduct(Model): +class DotProduct(Layer): ''' This is a simple table layer which takes a table of two tensors as input @@ -1344,7 +1378,7 @@ def __init__(self, super(DotProduct, self).__init__(None, bigdl_type) -class ELU(Model): +class ELU(Layer): ''' D-A Clevert, Thomas Unterthiner, Sepp Hochreiter @@ -1365,7 +1399,7 @@ def __init__(self, inplace) -class Euclidean(Model): +class Euclidean(Layer): ''' Outputs the Euclidean distance of the input to outputSize centers @@ -1396,7 +1430,7 @@ def set_init_method(self, weight_init_method = None, bias_init_method = None): weight_init_method, bias_init_method) -class Exp(Model): +class Exp(Layer): ''' Applies element-wise exp to input tensor. @@ -1410,7 +1444,7 @@ def __init__(self, super(Exp, self).__init__(None, bigdl_type) -class FlattenTable(Model): +class FlattenTable(Layer): ''' This is a table layer which takes an arbitrarily deep table of Tensors @@ -1427,7 +1461,7 @@ def __init__(self, super(FlattenTable, self).__init__(None, bigdl_type) -class GradientReversal(Model): +class GradientReversal(Layer): ''' It is a simple module preserves the input, but takes the @@ -1452,7 +1486,7 @@ def __init__(self, the_lambda) -class HardShrink(Model): +class HardShrink(Layer): ''' This is a transfer layer which applies the hard shrinkage function @@ -1478,7 +1512,7 @@ def __init__(self, the_lambda) -class HardTanh(Model): +class HardTanh(Layer): ''' Applies HardTanh to each element of input, HardTanh is defined: @@ -1507,7 +1541,7 @@ def __init__(self, inplace) -class Index(Model): +class Index(Layer): ''' Applies the Tensor index operation along the given dimension. @@ -1527,7 +1561,7 @@ def __init__(self, dimension) -class InferReshape(Model): +class InferReshape(Layer): ''' Reshape with the support of infered size, @@ -1560,7 +1594,7 @@ def __init__(self, batch_mode) -class JoinTable(Model): +class JoinTable(Layer): ''' It is a table module which takes a table of Tensors as input and @@ -1590,7 +1624,7 @@ def __init__(self, n_input_dims) -class L1Cost(Model): +class L1Cost(Layer): ''' compute L1 norm for input, and sign of input @@ -1604,7 +1638,7 @@ def __init__(self, super(L1Cost, self).__init__(None, bigdl_type) -class L1Penalty(Model): +class L1Penalty(Layer): ''' adds an L1 penalty to an input (for sparsity). @@ -1633,7 +1667,7 @@ def __init__(self, provide_output) -class LeakyReLU(Model): +class LeakyReLU(Layer): ''' It is a transfer module that applies LeakyReLU, which parameter negval sets the slope of the @@ -1657,7 +1691,7 @@ def __init__(self, inplace) -class Log(Model): +class Log(Layer): ''' Applies the log function element-wise to the input Tensor, @@ -1673,7 +1707,7 @@ def __init__(self, super(Log, self).__init__(None, bigdl_type) -class LogSigmoid(Model): +class LogSigmoid(Layer): ''' This class is a transform layer corresponding to the sigmoid function: @@ -1689,7 +1723,7 @@ def __init__(self, super(LogSigmoid, self).__init__(None, bigdl_type) -class LookupTable(Model): +class LookupTable(Layer): ''' a convolution of width 1, commonly used for word embeddings @@ -1719,7 +1753,7 @@ def set_init_method(self, weight_init_method = None, bias_init_method = None): weight_init_method, bias_init_method) -class MM(Model): +class MM(Layer): ''' Module to perform matrix multiplication on two mini-batch inputs, producing a mini-batch. @@ -1741,7 +1775,7 @@ def __init__(self, trans_b) -class MV(Model): +class MV(Layer): ''' It is a module to perform matrix vector multiplication on two mini-batch inputs, @@ -1781,7 +1815,7 @@ def __init__(self, super(MapTable, self).__init__(None, bigdl_type, module) -class MaskedSelect(Model): +class MaskedSelect(Layer): ''' Performs a torch.MaskedSelect on a Tensor. The mask is supplied as a tabular argument with @@ -1796,7 +1830,7 @@ def __init__(self, super(MaskedSelect, self).__init__(None, bigdl_type) -class Max(Model): +class Max(Layer): ''' Applies a max operation over dimension `dim` @@ -1819,7 +1853,7 @@ def __init__(self, num_input_dims) -class Mean(Model): +class Mean(Layer): ''' It is a simple layer which applies a mean operation over the given dimension. When nInputDims @@ -1847,7 +1881,7 @@ def __init__(self, n_input_dims) -class Min(Model): +class Min(Layer): ''' Applies a min operation over dimension `dim`. @@ -1870,7 +1904,7 @@ def __init__(self, num_input_dims) -class MixtureTable(Model): +class MixtureTable(Layer): ''' Creates a module that takes a table {gater, experts} as input and outputs the mixture of experts @@ -1892,7 +1926,7 @@ def __init__(self, super(MixtureTable, self).__init__(None, bigdl_type, dim) -class Mul(Model): +class Mul(Layer): ''' Multiply a single scalar factor to the incoming data @@ -1911,7 +1945,7 @@ def set_init_method(self, weight_init_method = None, bias_init_method = None): weight_init_method, bias_init_method) -class MulConstant(Model): +class MulConstant(Layer): ''' Multiplies input Tensor by a (non-learnable) scalar constant. @@ -1935,7 +1969,7 @@ def __init__(self, inplace) -class Narrow(Model): +class Narrow(Layer): ''' Narrow is application of narrow operation in a module. @@ -1956,7 +1990,7 @@ def __init__(self, length) -class NarrowTable(Model): +class NarrowTable(Layer): ''' Creates a module that takes a table as input and outputs the subtable starting at index @@ -1982,7 +2016,7 @@ def __init__(self, length) -class Normalize(Model): +class Normalize(Layer): ''' Normalizes the input Tensor to have unit L_p norm. The smoothing parameter eps prevents @@ -2003,7 +2037,7 @@ def __init__(self, eps) -class PReLU(Model): +class PReLU(Layer): ''' Applies parametric ReLU, which parameter varies the slope of the negative part. @@ -2037,7 +2071,7 @@ def set_init_method(self, weight_init_method = None, bias_init_method = None): weight_init_method, bias_init_method) -class Padding(Model): +class Padding(Layer): ''' This module adds pad units of padding to dimension dim of the input. If pad is negative, @@ -2075,7 +2109,7 @@ def __init__(self, n_index) -class PairwiseDistance(Model): +class PairwiseDistance(Layer): ''' It is a module that takes a table of two vectors as input and outputs @@ -2116,7 +2150,7 @@ def __init__(self, super(ParallelTable, self).__init__(None, bigdl_type) -class Power(Model): +class Power(Layer): ''' Apply an element-wise power operation with scale and shift. @@ -2142,7 +2176,7 @@ def __init__(self, shift) -class RReLU(Model): +class RReLU(Layer): ''' Applies the randomized leaky rectified linear unit (RReLU) element-wise to the input Tensor, @@ -2194,7 +2228,7 @@ def __init__(self, inplace) -class ReLU6(Model): +class ReLU6(Layer): ''' Same as ReLU except that the rectifying function f(x) saturates at x = 6 @@ -2214,7 +2248,7 @@ def __init__(self, inplace) -class Replicate(Model): +class Replicate(Layer): ''' Replicate repeats input `nFeatures` times along its `dim` dimension. @@ -2240,7 +2274,7 @@ def __init__(self, n_dim) -class RoiPooling(Model): +class RoiPooling(Layer): ''' Region of interest pooling @@ -2275,7 +2309,7 @@ def __init__(self, spatial_scale) -class Scale(Model): +class Scale(Layer): ''' Scale is the combination of CMul and CAdd @@ -2298,7 +2332,7 @@ def __init__(self, size) -class SelectTable(Model): +class SelectTable(Layer): ''' Creates a module that takes a table as input and outputs the element at index `index` @@ -2322,7 +2356,7 @@ def __init__(self, dimension) -class Sigmoid(Model): +class Sigmoid(Layer): ''' Applies the Sigmoid function element-wise to the input Tensor, @@ -2337,7 +2371,7 @@ def __init__(self, super(Sigmoid, self).__init__(None, bigdl_type) -class SoftMax(Model): +class SoftMax(Layer): ''' Applies the SoftMax function to an n-dimensional input Tensor, rescaling them so that the @@ -2355,7 +2389,7 @@ def __init__(self, super(SoftMax, self).__init__(None, bigdl_type) -class SoftMin(Model): +class SoftMin(Layer): ''' Applies the SoftMin function to an n-dimensional input Tensor, rescaling them so that the @@ -2373,7 +2407,7 @@ def __init__(self, super(SoftMin, self).__init__(None, bigdl_type) -class SoftPlus(Model): +class SoftPlus(Layer): ''' Apply the SoftPlus function to an n-dimensional input tensor. @@ -2394,7 +2428,7 @@ def __init__(self, beta) -class SoftShrink(Model): +class SoftShrink(Layer): ''' Apply the soft shrinkage function element-wise to the input Tensor @@ -2421,7 +2455,7 @@ def __init__(self, the_lambda) -class SoftSign(Model): +class SoftSign(Layer): ''' Apply SoftSign function to an n-dimensional input Tensor. @@ -2439,7 +2473,7 @@ def __init__(self, super(SoftSign, self).__init__(None, bigdl_type) -class SpatialDilatedConvolution(Model): +class SpatialDilatedConvolution(Layer): ''' Apply a 2D dilated convolution over an input image. @@ -2505,7 +2539,7 @@ def set_init_method(self, weight_init_method = None, bias_init_method = None): weight_init_method, bias_init_method) -class SpatialFullConvolution(Model): +class SpatialFullConvolution(Layer): ''' Apply a 2D full convolution over an input image. The input tensor is expected to be a 3D or 4D(with batch) tensor. Note that instead @@ -2583,7 +2617,7 @@ def set_init_method(self, weight_init_method = None, bias_init_method = None): weight_init_method, bias_init_method) -class SpatialShareConvolution(Model): +class SpatialShareConvolution(Layer): ''' @@ -2621,7 +2655,7 @@ def set_init_method(self, weight_init_method = None, bias_init_method = None): weight_init_method, bias_init_method) -class VolumetricConvolution(Model): +class VolumetricConvolution(Layer): ''' Applies a 3D convolution over an input image composed of several input planes. The input tensor @@ -2681,7 +2715,7 @@ def set_init_method(self, weight_init_method = None, bias_init_method = None): weight_init_method, bias_init_method) -class VolumetricMaxPooling(Model): +class VolumetricMaxPooling(Layer): ''' Applies 3D max-pooling operation in kTxkWxkH regions by step size dTxdWxdH steps. @@ -2727,7 +2761,7 @@ def __init__(self, pad_h) -class SpatialZeroPadding(Model): +class SpatialZeroPadding(Layer): ''' Each feature map of a given input is padded with specified number of zeros. @@ -2756,7 +2790,7 @@ def __init__(self, pad_bottom) -class SplitTable(Model): +class SplitTable(Layer): ''' Creates a module that takes a Tensor as input and @@ -2787,7 +2821,7 @@ def __init__(self, n_input_dims) -class Sqrt(Model): +class Sqrt(Layer): ''' Apply an element-wise sqrt operation. @@ -2802,7 +2836,7 @@ def __init__(self, super(Sqrt, self).__init__(None, bigdl_type) -class Square(Model): +class Square(Layer): ''' Apply an element-wise square operation. @@ -2816,7 +2850,7 @@ def __init__(self, super(Square, self).__init__(None, bigdl_type) -class Squeeze(Model): +class Squeeze(Layer): ''' Delete singleton all dimensions or a specific dim. @@ -2841,7 +2875,7 @@ def __init__(self, num_input_dims) -class Sum(Model): +class Sum(Layer): ''' It is a simple layer which applies a sum operation over the given dimension. @@ -2873,7 +2907,7 @@ def __init__(self, size_average) -class TanhShrink(Model): +class TanhShrink(Layer): ''' A simple layer for each element of the input tensor, do the following operation @@ -2890,7 +2924,7 @@ def __init__(self, super(TanhShrink, self).__init__(None, bigdl_type) -class Threshold(Model): +class Threshold(Layer): ''' Threshold input Tensor. @@ -2917,7 +2951,7 @@ def __init__(self, ip) -class Unsqueeze(Model): +class Unsqueeze(Layer): ''' Create an Unsqueeze layer. Insert singleton dim (i.e., dimension 1) at position pos. @@ -2942,7 +2976,7 @@ def __init__(self, num_input_dims) -class Reshape(Model): +class Reshape(Layer): ''' The forward(input) reshape the input tensor into a size(0) * size(1) * ... tensor, taking the elements row-wise. @@ -3000,7 +3034,8 @@ def __init__(self, bigdl_type="float"): super(ConcatTable, self).__init__(None, bigdl_type) -class Identity(Model): + +class Identity(Layer): ''' Identity just return the input to output. It's useful in same parallel container to get an origin input. @@ -3015,7 +3050,7 @@ def __init__(self, super(Identity, self).__init__(None, bigdl_type) -class Reverse(Model): +class Reverse(Layer): ''' Reverse the input w.r.t given dimension. The input can be a Tensor or Table. @@ -3035,7 +3070,7 @@ def __init__(self, dimension) -class Transpose(Model): +class Transpose(Layer): ''' Transpose input along specified dimensions @@ -3054,7 +3089,7 @@ def __init__(self, permutations) -class SpatialContrastiveNormalization(Model): +class SpatialContrastiveNormalization(Layer): ''' Subtractive + divisive contrast normalization. @@ -3085,7 +3120,7 @@ def __init__(self, thresval) -class SpatialConvolutionMap(Model): +class SpatialConvolutionMap(Layer): ''' This class is a generalization of SpatialConvolution. It uses a generic connection table between input and output features. @@ -3116,7 +3151,7 @@ def __init__(self, pad_h) -class SpatialDivisiveNormalization(Model): +class SpatialDivisiveNormalization(Layer): ''' Applies a spatial division operation on a series of 2D inputs using kernel for computing the weighted average in a neighborhood. The neighborhood is defined for @@ -3161,38 +3196,8 @@ def __init__(self, threshold, thresval) -class Graph(Model): - ''' - A graph container. Each node can have multiple inputs. The output of the node should be a - tensor. The output tensor can be connected to multiple nodes. So the module in each node can - have a tensor or table input, and should have a tensor output. - - - The graph container can have multiple inputs and multiple outputs. If there's one input, - the input data fed to the graph module should be a tensor. If there're multiple inputs, - the input data fed to the graph module should be a table, which is actually an sequence of - tensor. The order of the input tensors should be same with the order of the input nodes. - This is also applied to the gradient from the module in the back propagation. - - - If there's one output, the module output is a tensor. If there're multiple outputs, the module - output is a table, which is actually an sequence of tensor. The order of the output tensors is - same with the order of the output modules. This is also applied to the gradient passed to the - module in the back propagation. - - - All inputs should be able to connect to outputs through some paths in the graph. - It is allowed that some successors of the inputs node are not connect to outputs. - If so, these nodes will be excluded in the computation. - ''' - - def __init__(self, - input, - output, - bigdl_type="float"): - super(Graph, self).__init__(None, bigdl_type, input, output) -class SpatialSubtractiveNormalization(Model): +class SpatialSubtractiveNormalization(Layer): ''' Applies a spatial subtraction operation on a series of 2D inputs using kernel for computing the weighted average in a neighborhood. The neighborhood is defined for diff --git a/python/dllib/src/bigdl/dllib/optim/optimizer.py b/python/dllib/src/bigdl/dllib/optim/optimizer.py index f4e095fb293..90cbe249d66 100644 --- a/python/dllib/src/bigdl/dllib/optim/optimizer.py +++ b/python/dllib/src/bigdl/dllib/optim/optimizer.py @@ -417,8 +417,8 @@ def optimize(self): Do an optimization. """ jmodel = callJavaFunc(get_spark_context(), self.value.optimize) - from bigdl.nn.layer import Model - return Model.of(jmodel) + from bigdl.nn.layer import Layer + return Layer.of(jmodel) def set_train_summary(self, summary): """ From 0d74e0257e5a9dc69272462fd30824b38d295b4e Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Wed, 14 Jun 2017 10:19:55 +0800 Subject: [PATCH 053/823] Rename Graph to Model (#982) * Rename Graph to Model * fix model --- python/dllib/src/bigdl/dllib/nn/criterion.py | 12 +- python/dllib/src/bigdl/dllib/nn/layer.py | 327 +++++++++--------- .../dllib/src/bigdl/dllib/optim/optimizer.py | 4 +- 3 files changed, 174 insertions(+), 169 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/nn/criterion.py b/python/dllib/src/bigdl/dllib/nn/criterion.py index 83f5f4113a1..20c2e46350c 100644 --- a/python/dllib/src/bigdl/dllib/nn/criterion.py +++ b/python/dllib/src/bigdl/dllib/nn/criterion.py @@ -20,7 +20,7 @@ from bigdl.util.common import JavaValue from bigdl.util.common import callBigDlFunc from bigdl.util.common import JTensor -from bigdl.nn.layer import Model +from bigdl.nn.layer import Layer import numpy as np if sys.version >= '3': @@ -54,8 +54,8 @@ def forward(self, input, target): output = callBigDlFunc(self.bigdl_type, "criterionForward", self.value, - Model.check_input(input), - Model.check_input(target)) + Layer.check_input(input), + Layer.check_input(target)) return output def backward(self, input, target): @@ -70,9 +70,9 @@ def backward(self, input, target): output = callBigDlFunc(self.bigdl_type, "criterionBackward", self.value, - Model.check_input(input), - Model.check_input(target)) - return Model.convert_output(output) + Layer.check_input(input), + Layer.check_input(target)) + return Layer.convert_output(output) @classmethod def of(cls, jcriterion, bigdl_type="float"): diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 58fe8ec276f..aa57fd640f3 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -49,14 +49,14 @@ def of(cls, jvalue, bigdl_type="float"): return Node(jvalue, bigdl_type) def element(self): - return Model.of(self.value.element()) + return Layer.of(self.value.element()) -class Model(JavaValue): +class Layer(JavaValue): """ - Model is the basic component of a neural network + Layer is the basic component of a neural network and it's also the base class of layers. - Model can connect to others to construct a complex neural network. + Layer can connect to others to construct a complex neural network. """ def __init__(self, jvalue, bigdl_type, *args): @@ -87,14 +87,13 @@ def __call__(self, x=None): to_list(x))) @classmethod - def of(cls, jmodel, bigdl_type="float"): + def of(cls, jvalue, bigdl_type="float"): """ - Create a Python Model - - :param jmodel: Java model create by Py4j - :return: A Python Model + Create a Python Layer base on the given java value + :param jvalue: Java object create by Py4j + :return: A Python Layer """ - model = Model(jmodel,bigdl_type) + model = Layer(jvalue, bigdl_type) return model def set_name(self, name): @@ -126,6 +125,7 @@ def get_dtype(self): return "float32" else: return "float64" + @staticmethod def check_input(input): if type(input) is list: @@ -288,6 +288,55 @@ def get_weights(self): print("The layer does not have weight/bias") return None + def save(self, path, over_write = False): + callBigDlFunc(self.bigdl_type, "modelSave", self.value, path, + over_write) + + +class Container(Layer): + ''' + [[Container]] is a sub-class of Model that declares methods defined in all containers. + A container usually contain some other modules which can be added through the "add" method + ''' + + def __init__(self, jvalue, bigdl_type, *args): + super(Container, self).__init__(jvalue, bigdl_type, *args) + + def add(self, model): + self.value.add(model.value) + return self + + +class Model(Container): + """ + A graph container. Each node can have multiple inputs. The output of the node should be a + tensor. The output tensor can be connected to multiple nodes. So the module in each node can + have a tensor or table input, and should have a tensor output. + + The graph container can have multiple inputs and multiple outputs. If there's one input, + the input data fed to the graph module should be a tensor. If there're multiple inputs, + the input data fed to the graph module should be a table, which is actually an sequence of + tensor. The order of the input tensors should be same with the order of the input nodes. + This is also applied to the gradient from the module in the back propagation. + + If there's one output, the module output is a tensor. If there're multiple outputs, the module + output is a table, which is actually an sequence of tensor. The order of the output tensors is + same with the order of the output modules. This is also applied to the gradient passed to the + module in the back propagation. + + All inputs should be able to connect to outputs through some paths in the graph. + It is allowed that some successors of the inputs node are not connect to outputs. + If so, these nodes will be excluded in the computation. + """ + + def __init__(self, + inputs, + outputs, + bigdl_type="float"): + super(Model, self).__init__(None, bigdl_type, + to_list(inputs), + to_list(outputs)) + @staticmethod def load(path, bigdl_type="float"): """ @@ -297,7 +346,7 @@ def load(path, bigdl_type="float"): :return: A pre-trained model. """ jmodel = callBigDlFunc(bigdl_type, "loadBigDL", path) - return Model.of(jmodel) + return Layer.of(jmodel) @staticmethod def load_torch(path, bigdl_type="float"): @@ -308,7 +357,7 @@ def load_torch(path, bigdl_type="float"): :return: A pre-trained model. """ jmodel = callBigDlFunc(bigdl_type, "loadTorch", path) - return Model.of(jmodel) + return Layer.of(jmodel) @staticmethod def load_caffe(model, defPath, modelPath, match_all=True, bigdl_type="float"): @@ -322,25 +371,10 @@ def load_caffe(model, defPath, modelPath, match_all=True, bigdl_type="float"): :return: A pre-trained model. """ jmodel = callBigDlFunc(bigdl_type, "loadCaffe", model, defPath, modelPath, match_all) - return Model.of(jmodel) + return Layer.of(jmodel) -class Container(Model): - ''' - [[Container]] is a sub-class of Model that declares methods defined in all containers. - A container usually contain some other modules which can be added through the "add" method - ''' - - def __init__(self, jvalue, bigdl_type, *args): - super(Container, self).__init__(jvalue, bigdl_type, *args) - - def add(self, model): - self.value.add(model.value) - return self - - - -class Linear(Model): +class Linear(Layer): ''' The [[Linear]] module applies a linear transformation to the input data, @@ -370,7 +404,7 @@ def set_init_method(self, weight_init_method = None, bias_init_method = None): weight_init_method, bias_init_method) -class ReLU(Model): +class ReLU(Layer): ''' Applies the rectified linear unit (ReLU) function element-wise to the input Tensor, @@ -389,7 +423,7 @@ def __init__(self, ip=False, bigdl_type="float"): super(ReLU, self).__init__(None, bigdl_type, ip) -class Tanh(Model): +class Tanh(Layer): ''' Applies the Tanh function element-wise to the input Tensor, thus outputting a Tensor of the same @@ -404,7 +438,7 @@ def __init__(self, bigdl_type="float"): super(Tanh, self).__init__(None, bigdl_type) -class Echo(Model): +class Echo(Layer): ''' This module is for debug purpose, which can print activation and gradient in your model @@ -419,7 +453,7 @@ def __init__(self, bigdl_type="float"): super(Echo, self).__init__(None, bigdl_type) -class LogSoftMax(Model): +class LogSoftMax(Layer): ''' Applies the LogSoftMax function to an n-dimensional input Tensor. @@ -457,7 +491,7 @@ def __init__(self, bigdl_type="float"): super(Sequential, self).__init__(None, bigdl_type) -class SpatialConvolution(Model): +class SpatialConvolution(Layer): ''' Applies a 2D convolution over an input image composed of several input planes. @@ -512,7 +546,7 @@ def set_init_method(self, weight_init_method = None, bias_init_method = None): weight_init_method, bias_init_method) -class SpatialMaxPooling(Model): +class SpatialMaxPooling(Layer): ''' Applies 2D max-pooling operation in kWxkH regions by step size dWxdH steps. @@ -553,7 +587,7 @@ def __init__(self, kw, to_ceil) -class Select(Model): +class Select(Layer): ''' A Simple layer selecting an index of the input tensor in the given dimension @@ -584,7 +618,7 @@ def __init__(self, bigdl_type="float"): super(Recurrent, self).__init__(None, bigdl_type) -class LSTM(Model): +class LSTM(Layer): ''' | Long Short Term Memory architecture. | Ref. @@ -608,7 +642,7 @@ def __init__(self, input_size, hidden_size, p=0.0, bigdl_type="float"): super(LSTM, self).__init__(None, bigdl_type, input_size, hidden_size, p) -class LSTMPeephole(Model): +class LSTMPeephole(Layer): ''' | Long Short Term Memory architecture with peephole. | Ref. A.: http://arxiv.org/pdf/1303.5778v1 (blueprint for this module) @@ -631,7 +665,7 @@ def __init__(self, input_size, hidden_size, p=0.0, bigdl_type="float"): super(LSTMPeephole, self).__init__(None, bigdl_type, input_size, hidden_size, p) -class GRU(Model): +class GRU(Layer): ''' Gated Recurrent Units architecture. The first input in sequence uses zero value for cell and hidden state @@ -655,7 +689,7 @@ def __init__(self, input_size, hidden_size, p=0.0, bigdl_type="float"): super(GRU, self).__init__(None, bigdl_type, input_size, hidden_size, p) -class RnnCell(Model): +class RnnCell(Layer): ''' It is a simple RNN. User can pass an activation function to the RNN. @@ -678,7 +712,7 @@ def __init__(self, super(RnnCell, self).__init__(None, bigdl_type, input_size, hidden_size, activation) -class TimeDistributed(Model): +class TimeDistributed(Layer): ''' This layer is intended to apply contained layer to each temporal time slice of input tensor. @@ -727,7 +761,7 @@ def __init__(self, dimension) -class SpatialAveragePooling(Model): +class SpatialAveragePooling(Layer): ''' Applies 2D average-pooling operation in kWxkH regions by step size dWxdH steps. @@ -772,7 +806,7 @@ def __init__(self, divide) -class SpatialBatchNormalization(Model): +class SpatialBatchNormalization(Layer): ''' This file implements Batch Normalization as described in the paper: @@ -813,7 +847,7 @@ def set_init_method(self, weight_init_method = None, bias_init_method = None): weight_init_method, bias_init_method) -class SpatialCrossMapLRN(Model): +class SpatialCrossMapLRN(Layer): ''' Applies Spatial Local Response Normalization between different feature maps. @@ -851,7 +885,7 @@ def __init__(self, k) -class Dropout(Model): +class Dropout(Layer): ''' Dropout masks(set to zero) parts of input using a bernoulli distribution. @@ -880,7 +914,7 @@ def __init__(self, scale) -class View(Model): +class View(Layer): ''' This module creates a new view of the input tensor using the sizes passed to the constructor. @@ -905,7 +939,7 @@ def __init__(self, num_input_dims) -class Abs(Model): +class Abs(Layer): ''' an element-wise abs operation @@ -920,7 +954,7 @@ def __init__(self, super(Abs, self).__init__(None, bigdl_type) -class Add(Model): +class Add(Layer): ''' adds a bias term to input data ; @@ -941,7 +975,7 @@ def set_init_method(self, weight_init_method = None, bias_init_method = None): weight_init_method, bias_init_method) -class AddConstant(Model): +class AddConstant(Layer): ''' adding a constant @@ -963,7 +997,7 @@ def __init__(self, constant_scalar, inplace) -class BatchNormalization(Model): +class BatchNormalization(Layer): ''' This layer implements Batch Normalization as described in the paper: @@ -1010,7 +1044,7 @@ def set_init_method(self, weight_init_method = None, bias_init_method = None): weight_init_method, bias_init_method) -class Bilinear(Model): +class Bilinear(Layer): ''' a bilinear transformation with sparse inputs, @@ -1071,7 +1105,7 @@ def __init__(self, n_output_dim1) -class CAdd(Model): +class CAdd(Layer): ''' This layer has a bias tensor with given size. The bias will be added element wise to the input @@ -1100,7 +1134,7 @@ def set_init_method(self, weight_init_method = None, bias_init_method = None): weight_init_method, bias_init_method) -class CAddTable(Model): +class CAddTable(Layer): ''' Merge the input tensors in the input table by element wise adding them together. The input @@ -1121,7 +1155,7 @@ def __init__(self, inplace) -class CDivTable(Model): +class CDivTable(Layer): ''' Takes a table with two Tensor and returns the component-wise division between them. @@ -1136,7 +1170,7 @@ def __init__(self, super(CDivTable, self).__init__(None, bigdl_type) -class CMaxTable(Model): +class CMaxTable(Layer): ''' Takes a table of Tensors and outputs the max of all of them. @@ -1151,7 +1185,7 @@ def __init__(self, super(CMaxTable, self).__init__(None, bigdl_type) -class CMinTable(Model): +class CMinTable(Layer): ''' Takes a table of Tensors and outputs the min of all of them. @@ -1165,7 +1199,7 @@ def __init__(self, super(CMinTable, self).__init__(None, bigdl_type) -class CMul(Model): +class CMul(Layer): ''' Applies a component-wise multiplication to the incoming data @@ -1189,7 +1223,7 @@ def set_init_method(self, weight_init_method = None, bias_init_method = None): weight_init_method, bias_init_method) -class CMulTable(Model): +class CMulTable(Layer): ''' Takes a table of Tensors and outputs the multiplication of all of them. @@ -1204,7 +1238,7 @@ def __init__(self, super(CMulTable, self).__init__(None, bigdl_type) -class CSubTable(Model): +class CSubTable(Layer): ''' Takes a table with two Tensor and returns the component-wise subtraction between them. @@ -1219,7 +1253,7 @@ def __init__(self, super(CSubTable, self).__init__(None, bigdl_type) -class Clamp(Model): +class Clamp(Layer): ''' Clamps all elements into the range [min_value, max_value]. @@ -1245,7 +1279,7 @@ def __init__(self, max) -class Contiguous(Model): +class Contiguous(Layer): ''' used to make input, grad_output both contiguous @@ -1260,7 +1294,7 @@ def __init__(self, super(Contiguous, self).__init__(None, bigdl_type) -class Cosine(Model): +class Cosine(Layer): ''' Cosine calculates the cosine similarity of the input to k mean centers. The input given in @@ -1290,7 +1324,7 @@ def set_init_method(self, weight_init_method = None, bias_init_method = None): weight_init_method, bias_init_method) -class CosineDistance(Model): +class CosineDistance(Layer): ''' Outputs the cosine distance between inputs @@ -1304,7 +1338,7 @@ def __init__(self, bigdl_type="float"): super(CosineDistance, self).__init__(None, bigdl_type) -class DiceCoefficientCriterion(Model): +class DiceCoefficientCriterion(Layer): ''' The Dice-Coefficient criterion @@ -1328,7 +1362,7 @@ def __init__(self, size_average, epsilon) -class DotProduct(Model): +class DotProduct(Layer): ''' This is a simple table layer which takes a table of two tensors as input @@ -1344,7 +1378,7 @@ def __init__(self, super(DotProduct, self).__init__(None, bigdl_type) -class ELU(Model): +class ELU(Layer): ''' D-A Clevert, Thomas Unterthiner, Sepp Hochreiter @@ -1365,7 +1399,7 @@ def __init__(self, inplace) -class Euclidean(Model): +class Euclidean(Layer): ''' Outputs the Euclidean distance of the input to outputSize centers @@ -1396,7 +1430,7 @@ def set_init_method(self, weight_init_method = None, bias_init_method = None): weight_init_method, bias_init_method) -class Exp(Model): +class Exp(Layer): ''' Applies element-wise exp to input tensor. @@ -1410,7 +1444,7 @@ def __init__(self, super(Exp, self).__init__(None, bigdl_type) -class FlattenTable(Model): +class FlattenTable(Layer): ''' This is a table layer which takes an arbitrarily deep table of Tensors @@ -1427,7 +1461,7 @@ def __init__(self, super(FlattenTable, self).__init__(None, bigdl_type) -class GradientReversal(Model): +class GradientReversal(Layer): ''' It is a simple module preserves the input, but takes the @@ -1452,7 +1486,7 @@ def __init__(self, the_lambda) -class HardShrink(Model): +class HardShrink(Layer): ''' This is a transfer layer which applies the hard shrinkage function @@ -1478,7 +1512,7 @@ def __init__(self, the_lambda) -class HardTanh(Model): +class HardTanh(Layer): ''' Applies HardTanh to each element of input, HardTanh is defined: @@ -1507,7 +1541,7 @@ def __init__(self, inplace) -class Index(Model): +class Index(Layer): ''' Applies the Tensor index operation along the given dimension. @@ -1527,7 +1561,7 @@ def __init__(self, dimension) -class InferReshape(Model): +class InferReshape(Layer): ''' Reshape with the support of infered size, @@ -1560,7 +1594,7 @@ def __init__(self, batch_mode) -class JoinTable(Model): +class JoinTable(Layer): ''' It is a table module which takes a table of Tensors as input and @@ -1590,7 +1624,7 @@ def __init__(self, n_input_dims) -class L1Cost(Model): +class L1Cost(Layer): ''' compute L1 norm for input, and sign of input @@ -1604,7 +1638,7 @@ def __init__(self, super(L1Cost, self).__init__(None, bigdl_type) -class L1Penalty(Model): +class L1Penalty(Layer): ''' adds an L1 penalty to an input (for sparsity). @@ -1633,7 +1667,7 @@ def __init__(self, provide_output) -class LeakyReLU(Model): +class LeakyReLU(Layer): ''' It is a transfer module that applies LeakyReLU, which parameter negval sets the slope of the @@ -1657,7 +1691,7 @@ def __init__(self, inplace) -class Log(Model): +class Log(Layer): ''' Applies the log function element-wise to the input Tensor, @@ -1673,7 +1707,7 @@ def __init__(self, super(Log, self).__init__(None, bigdl_type) -class LogSigmoid(Model): +class LogSigmoid(Layer): ''' This class is a transform layer corresponding to the sigmoid function: @@ -1689,7 +1723,7 @@ def __init__(self, super(LogSigmoid, self).__init__(None, bigdl_type) -class LookupTable(Model): +class LookupTable(Layer): ''' a convolution of width 1, commonly used for word embeddings @@ -1719,7 +1753,7 @@ def set_init_method(self, weight_init_method = None, bias_init_method = None): weight_init_method, bias_init_method) -class MM(Model): +class MM(Layer): ''' Module to perform matrix multiplication on two mini-batch inputs, producing a mini-batch. @@ -1741,7 +1775,7 @@ def __init__(self, trans_b) -class MV(Model): +class MV(Layer): ''' It is a module to perform matrix vector multiplication on two mini-batch inputs, @@ -1781,7 +1815,7 @@ def __init__(self, super(MapTable, self).__init__(None, bigdl_type, module) -class MaskedSelect(Model): +class MaskedSelect(Layer): ''' Performs a torch.MaskedSelect on a Tensor. The mask is supplied as a tabular argument with @@ -1796,7 +1830,7 @@ def __init__(self, super(MaskedSelect, self).__init__(None, bigdl_type) -class Max(Model): +class Max(Layer): ''' Applies a max operation over dimension `dim` @@ -1819,7 +1853,7 @@ def __init__(self, num_input_dims) -class Mean(Model): +class Mean(Layer): ''' It is a simple layer which applies a mean operation over the given dimension. When nInputDims @@ -1847,7 +1881,7 @@ def __init__(self, n_input_dims) -class Min(Model): +class Min(Layer): ''' Applies a min operation over dimension `dim`. @@ -1870,7 +1904,7 @@ def __init__(self, num_input_dims) -class MixtureTable(Model): +class MixtureTable(Layer): ''' Creates a module that takes a table {gater, experts} as input and outputs the mixture of experts @@ -1892,7 +1926,7 @@ def __init__(self, super(MixtureTable, self).__init__(None, bigdl_type, dim) -class Mul(Model): +class Mul(Layer): ''' Multiply a single scalar factor to the incoming data @@ -1911,7 +1945,7 @@ def set_init_method(self, weight_init_method = None, bias_init_method = None): weight_init_method, bias_init_method) -class MulConstant(Model): +class MulConstant(Layer): ''' Multiplies input Tensor by a (non-learnable) scalar constant. @@ -1935,7 +1969,7 @@ def __init__(self, inplace) -class Narrow(Model): +class Narrow(Layer): ''' Narrow is application of narrow operation in a module. @@ -1956,7 +1990,7 @@ def __init__(self, length) -class NarrowTable(Model): +class NarrowTable(Layer): ''' Creates a module that takes a table as input and outputs the subtable starting at index @@ -1982,7 +2016,7 @@ def __init__(self, length) -class Normalize(Model): +class Normalize(Layer): ''' Normalizes the input Tensor to have unit L_p norm. The smoothing parameter eps prevents @@ -2003,7 +2037,7 @@ def __init__(self, eps) -class PReLU(Model): +class PReLU(Layer): ''' Applies parametric ReLU, which parameter varies the slope of the negative part. @@ -2037,7 +2071,7 @@ def set_init_method(self, weight_init_method = None, bias_init_method = None): weight_init_method, bias_init_method) -class Padding(Model): +class Padding(Layer): ''' This module adds pad units of padding to dimension dim of the input. If pad is negative, @@ -2075,7 +2109,7 @@ def __init__(self, n_index) -class PairwiseDistance(Model): +class PairwiseDistance(Layer): ''' It is a module that takes a table of two vectors as input and outputs @@ -2116,7 +2150,7 @@ def __init__(self, super(ParallelTable, self).__init__(None, bigdl_type) -class Power(Model): +class Power(Layer): ''' Apply an element-wise power operation with scale and shift. @@ -2142,7 +2176,7 @@ def __init__(self, shift) -class RReLU(Model): +class RReLU(Layer): ''' Applies the randomized leaky rectified linear unit (RReLU) element-wise to the input Tensor, @@ -2194,7 +2228,7 @@ def __init__(self, inplace) -class ReLU6(Model): +class ReLU6(Layer): ''' Same as ReLU except that the rectifying function f(x) saturates at x = 6 @@ -2214,7 +2248,7 @@ def __init__(self, inplace) -class Replicate(Model): +class Replicate(Layer): ''' Replicate repeats input `nFeatures` times along its `dim` dimension. @@ -2240,7 +2274,7 @@ def __init__(self, n_dim) -class RoiPooling(Model): +class RoiPooling(Layer): ''' Region of interest pooling @@ -2275,7 +2309,7 @@ def __init__(self, spatial_scale) -class Scale(Model): +class Scale(Layer): ''' Scale is the combination of CMul and CAdd @@ -2298,7 +2332,7 @@ def __init__(self, size) -class SelectTable(Model): +class SelectTable(Layer): ''' Creates a module that takes a table as input and outputs the element at index `index` @@ -2322,7 +2356,7 @@ def __init__(self, dimension) -class Sigmoid(Model): +class Sigmoid(Layer): ''' Applies the Sigmoid function element-wise to the input Tensor, @@ -2337,7 +2371,7 @@ def __init__(self, super(Sigmoid, self).__init__(None, bigdl_type) -class SoftMax(Model): +class SoftMax(Layer): ''' Applies the SoftMax function to an n-dimensional input Tensor, rescaling them so that the @@ -2355,7 +2389,7 @@ def __init__(self, super(SoftMax, self).__init__(None, bigdl_type) -class SoftMin(Model): +class SoftMin(Layer): ''' Applies the SoftMin function to an n-dimensional input Tensor, rescaling them so that the @@ -2373,7 +2407,7 @@ def __init__(self, super(SoftMin, self).__init__(None, bigdl_type) -class SoftPlus(Model): +class SoftPlus(Layer): ''' Apply the SoftPlus function to an n-dimensional input tensor. @@ -2394,7 +2428,7 @@ def __init__(self, beta) -class SoftShrink(Model): +class SoftShrink(Layer): ''' Apply the soft shrinkage function element-wise to the input Tensor @@ -2421,7 +2455,7 @@ def __init__(self, the_lambda) -class SoftSign(Model): +class SoftSign(Layer): ''' Apply SoftSign function to an n-dimensional input Tensor. @@ -2439,7 +2473,7 @@ def __init__(self, super(SoftSign, self).__init__(None, bigdl_type) -class SpatialDilatedConvolution(Model): +class SpatialDilatedConvolution(Layer): ''' Apply a 2D dilated convolution over an input image. @@ -2505,7 +2539,7 @@ def set_init_method(self, weight_init_method = None, bias_init_method = None): weight_init_method, bias_init_method) -class SpatialFullConvolution(Model): +class SpatialFullConvolution(Layer): ''' Apply a 2D full convolution over an input image. The input tensor is expected to be a 3D or 4D(with batch) tensor. Note that instead @@ -2583,7 +2617,7 @@ def set_init_method(self, weight_init_method = None, bias_init_method = None): weight_init_method, bias_init_method) -class SpatialShareConvolution(Model): +class SpatialShareConvolution(Layer): ''' @@ -2621,7 +2655,7 @@ def set_init_method(self, weight_init_method = None, bias_init_method = None): weight_init_method, bias_init_method) -class VolumetricConvolution(Model): +class VolumetricConvolution(Layer): ''' Applies a 3D convolution over an input image composed of several input planes. The input tensor @@ -2681,7 +2715,7 @@ def set_init_method(self, weight_init_method = None, bias_init_method = None): weight_init_method, bias_init_method) -class VolumetricMaxPooling(Model): +class VolumetricMaxPooling(Layer): ''' Applies 3D max-pooling operation in kTxkWxkH regions by step size dTxdWxdH steps. @@ -2727,7 +2761,7 @@ def __init__(self, pad_h) -class SpatialZeroPadding(Model): +class SpatialZeroPadding(Layer): ''' Each feature map of a given input is padded with specified number of zeros. @@ -2756,7 +2790,7 @@ def __init__(self, pad_bottom) -class SplitTable(Model): +class SplitTable(Layer): ''' Creates a module that takes a Tensor as input and @@ -2787,7 +2821,7 @@ def __init__(self, n_input_dims) -class Sqrt(Model): +class Sqrt(Layer): ''' Apply an element-wise sqrt operation. @@ -2802,7 +2836,7 @@ def __init__(self, super(Sqrt, self).__init__(None, bigdl_type) -class Square(Model): +class Square(Layer): ''' Apply an element-wise square operation. @@ -2816,7 +2850,7 @@ def __init__(self, super(Square, self).__init__(None, bigdl_type) -class Squeeze(Model): +class Squeeze(Layer): ''' Delete singleton all dimensions or a specific dim. @@ -2841,7 +2875,7 @@ def __init__(self, num_input_dims) -class Sum(Model): +class Sum(Layer): ''' It is a simple layer which applies a sum operation over the given dimension. @@ -2873,7 +2907,7 @@ def __init__(self, size_average) -class TanhShrink(Model): +class TanhShrink(Layer): ''' A simple layer for each element of the input tensor, do the following operation @@ -2890,7 +2924,7 @@ def __init__(self, super(TanhShrink, self).__init__(None, bigdl_type) -class Threshold(Model): +class Threshold(Layer): ''' Threshold input Tensor. @@ -2917,7 +2951,7 @@ def __init__(self, ip) -class Unsqueeze(Model): +class Unsqueeze(Layer): ''' Create an Unsqueeze layer. Insert singleton dim (i.e., dimension 1) at position pos. @@ -2942,7 +2976,7 @@ def __init__(self, num_input_dims) -class Reshape(Model): +class Reshape(Layer): ''' The forward(input) reshape the input tensor into a size(0) * size(1) * ... tensor, taking the elements row-wise. @@ -3000,7 +3034,8 @@ def __init__(self, bigdl_type="float"): super(ConcatTable, self).__init__(None, bigdl_type) -class Identity(Model): + +class Identity(Layer): ''' Identity just return the input to output. It's useful in same parallel container to get an origin input. @@ -3015,7 +3050,7 @@ def __init__(self, super(Identity, self).__init__(None, bigdl_type) -class Reverse(Model): +class Reverse(Layer): ''' Reverse the input w.r.t given dimension. The input can be a Tensor or Table. @@ -3035,7 +3070,7 @@ def __init__(self, dimension) -class Transpose(Model): +class Transpose(Layer): ''' Transpose input along specified dimensions @@ -3054,7 +3089,7 @@ def __init__(self, permutations) -class SpatialContrastiveNormalization(Model): +class SpatialContrastiveNormalization(Layer): ''' Subtractive + divisive contrast normalization. @@ -3085,7 +3120,7 @@ def __init__(self, thresval) -class SpatialConvolutionMap(Model): +class SpatialConvolutionMap(Layer): ''' This class is a generalization of SpatialConvolution. It uses a generic connection table between input and output features. @@ -3116,7 +3151,7 @@ def __init__(self, pad_h) -class SpatialDivisiveNormalization(Model): +class SpatialDivisiveNormalization(Layer): ''' Applies a spatial division operation on a series of 2D inputs using kernel for computing the weighted average in a neighborhood. The neighborhood is defined for @@ -3161,38 +3196,8 @@ def __init__(self, threshold, thresval) -class Graph(Model): - ''' - A graph container. Each node can have multiple inputs. The output of the node should be a - tensor. The output tensor can be connected to multiple nodes. So the module in each node can - have a tensor or table input, and should have a tensor output. - - - The graph container can have multiple inputs and multiple outputs. If there's one input, - the input data fed to the graph module should be a tensor. If there're multiple inputs, - the input data fed to the graph module should be a table, which is actually an sequence of - tensor. The order of the input tensors should be same with the order of the input nodes. - This is also applied to the gradient from the module in the back propagation. - - - If there's one output, the module output is a tensor. If there're multiple outputs, the module - output is a table, which is actually an sequence of tensor. The order of the output tensors is - same with the order of the output modules. This is also applied to the gradient passed to the - module in the back propagation. - - - All inputs should be able to connect to outputs through some paths in the graph. - It is allowed that some successors of the inputs node are not connect to outputs. - If so, these nodes will be excluded in the computation. - ''' - - def __init__(self, - input, - output, - bigdl_type="float"): - super(Graph, self).__init__(None, bigdl_type, input, output) -class SpatialSubtractiveNormalization(Model): +class SpatialSubtractiveNormalization(Layer): ''' Applies a spatial subtraction operation on a series of 2D inputs using kernel for computing the weighted average in a neighborhood. The neighborhood is defined for diff --git a/python/dllib/src/bigdl/dllib/optim/optimizer.py b/python/dllib/src/bigdl/dllib/optim/optimizer.py index f4e095fb293..90cbe249d66 100644 --- a/python/dllib/src/bigdl/dllib/optim/optimizer.py +++ b/python/dllib/src/bigdl/dllib/optim/optimizer.py @@ -417,8 +417,8 @@ def optimize(self): Do an optimization. """ jmodel = callJavaFunc(get_spark_context(), self.value.optimize) - from bigdl.nn.layer import Model - return Model.of(jmodel) + from bigdl.nn.layer import Layer + return Layer.of(jmodel) def set_train_summary(self, summary): """ From 5b85af38e1e7b15c32d4f6a1cd879522e56ae4ec Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Wed, 14 Jun 2017 10:19:55 +0800 Subject: [PATCH 054/823] Rename Graph to Model (#982) * Rename Graph to Model * fix model --- python/dllib/test/dev/diff.py | 20 +++++++++----------- simple_integration_test.py | 19 ++++++++++++++----- 2 files changed, 23 insertions(+), 16 deletions(-) diff --git a/python/dllib/test/dev/diff.py b/python/dllib/test/dev/diff.py index 0f17e8c5fef..caf4d9054fc 100755 --- a/python/dllib/test/dev/diff.py +++ b/python/dllib/test/dev/diff.py @@ -24,6 +24,8 @@ scala_nn_root = "./spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/" python_nn_root = "./pyspark/bigdl/nn/" +scala_to_python = {"Graph": "Model"} + def extract_scala_class(class_path): exclude_key_words = set(["*", "abstract"]) @@ -43,7 +45,7 @@ def get_scala_classes(scala_nn_root): def get_python_classes(nn_root): - exclude_classes = set(["Criterion", "Model"]) + exclude_classes = set(["Criterion"]) raw_classes = [] python_nn_layers = [join(nn_root, name) for name in os.listdir(nn_root) if isfile(join(nn_root, name)) and name.endswith('py') and "__" not in name] # noqa for p in python_nn_layers: @@ -59,17 +61,13 @@ def get_python_classes(nn_root): print "scala_layers: {0}, {1}".format(len(scala_layers), scala_layers) print("\n") print "python_layers: {0}, {1}".format(len(python_layers), python_layers) -print "\nIn scala, not in python: ", -for name in scala_layers: - if name not in python_layers: - print name, -print "\n" - -print "In python, not in scala: ", -for name in python_layers: - if name not in scala_layers: +print "In scala, not in python: ", +diff_count = 0 +for name in scala_layers: + if name not in python_layers and scala_to_python[name] not in python_layers: print name, + diff_count += 1 -if len(scala_layers - python_layers) > 0: +if diff_count > 0: raise Exception("There are difference between python and scala") diff --git a/simple_integration_test.py b/simple_integration_test.py index 8e7d55789fc..e30610930cf 100644 --- a/simple_integration_test.py +++ b/simple_integration_test.py @@ -35,16 +35,25 @@ def setUp(self): def tearDown(self): self.sc.stop() + def test_load_model(self): + fc1 = Linear(4, 2) + fc1.set_weights([np.ones((4, 2)), np.ones((2, ))]) + tmp_path = tempfile.mktemp() + fc1.save(tmp_path, True) + fc1_loaded = Model.load(tmp_path) + self.assertTrue(np.allclose(fc1_loaded.get_weights()[0], + fc1.get_weights()[0])) + def test_create_node(self): import numpy as np fc1 = Linear(4, 2)() fc2 = Linear(4, 2)() cadd = CAddTable()([fc1, fc2]) output1 = ReLU()(cadd) - graph = Graph([fc1, fc2], [output1]) + model = Model([fc1, fc2], [output1]) fc1.element().set_weights([np.ones((4, 2)), np.ones((2, ))]) fc2.element().set_weights([np.ones((4, 2)), np.ones((2, ))]) - output = graph.forward([np.array([0.1, 0.2, -0.3, -0.4]), + output = model.forward([np.array([0.1, 0.2, -0.3, -0.4]), np.array([0.5, 0.4, -0.2, -0.1])]) self.assertTrue(np.allclose(output, np.array([2.2, 2.2]))) @@ -55,12 +64,12 @@ def test_graph_backward(self): cadd = CAddTable()([fc1, fc2]) output1 = ReLU()(cadd) output2 = Threshold(10.0)(cadd) - graph = Graph([fc1, fc2], [output1, output2]) + model = Model([fc1, fc2], [output1, output2]) fc1.element().set_weights([np.ones((4, 2)), np.ones((2, ))]) fc2.element().set_weights([np.ones((4, 2)) * 2, np.ones((2, )) * 2]) - output = graph.forward([np.array([0.1, 0.2, -0.3, -0.4]), + output = model.forward([np.array([0.1, 0.2, -0.3, -0.4]), np.array([0.5, 0.4, -0.2, -0.1])]) - gradInput = graph.backward([np.array([0.1, 0.2, -0.3, -0.4]), + gradInput = model.backward([np.array([0.1, 0.2, -0.3, -0.4]), np.array([0.5, 0.4, -0.2, -0.1])], [np.array([1.0, 2.0]), np.array([3.0, 4.0])]) From 55d78276dfd021abfc28333543b5f4df55eab03d Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Thu, 15 Jun 2017 15:38:55 +0800 Subject: [PATCH 055/823] add default spark config (#1017) --- python/dllib/src/bigdl/dllib/utils/common.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/python/dllib/src/bigdl/dllib/utils/common.py b/python/dllib/src/bigdl/dllib/utils/common.py index c51b41e1b08..06f2c76325b 100644 --- a/python/dllib/src/bigdl/dllib/utils/common.py +++ b/python/dllib/src/bigdl/dllib/utils/common.py @@ -277,7 +277,14 @@ def create_spark_conf(): return sparkConf -def get_spark_context(conf = SparkConf()): +def get_spark_context(conf = None): + """ + Get the current active spark context and create one if no active instance + :param conf: combining bigdl configs into spark conf + :return: SparkContext + """ + if not conf: + conf = create_spark_conf() if "getOrCreate" in SparkContext.__dict__: return SparkContext.getOrCreate(conf) else: From 8dc82826ffbd48176e3931d92094ed6f1f9c34e9 Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Thu, 15 Jun 2017 15:38:55 +0800 Subject: [PATCH 056/823] add default spark config (#1017) --- python/dllib/src/bigdl/utils/common.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/python/dllib/src/bigdl/utils/common.py b/python/dllib/src/bigdl/utils/common.py index c51b41e1b08..06f2c76325b 100644 --- a/python/dllib/src/bigdl/utils/common.py +++ b/python/dllib/src/bigdl/utils/common.py @@ -277,7 +277,14 @@ def create_spark_conf(): return sparkConf -def get_spark_context(conf = SparkConf()): +def get_spark_context(conf = None): + """ + Get the current active spark context and create one if no active instance + :param conf: combining bigdl configs into spark conf + :return: SparkContext + """ + if not conf: + conf = create_spark_conf() if "getOrCreate" in SparkContext.__dict__: return SparkContext.getOrCreate(conf) else: From e098451d91e16dab54b59109b0a8a83701c80e02 Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Thu, 15 Jun 2017 16:22:07 +0800 Subject: [PATCH 057/823] Add zero_grad_parameters and update_parameters methods (#1018) * Add zero_grad_parameters and update_parameters methods * revert --- python/dllib/src/bigdl/dllib/nn/layer.py | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index aa57fd640f3..d6e7e646b56 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -181,6 +181,23 @@ def backward(self, input, grad_output): self.check_input(grad_output)) return self.convert_output(output) + def zero_grad_parameters(self): + """ + NB: It's for debug only, please use optimizer.optimize() in production. + If the module has parameters, this will zero the accumulation of the gradients with respect + to these parameters. Otherwise, it does nothing. + """ + callJavaFunc(get_spark_context(), self.value.zeroGradParameters) + + def update_parameters(self, learning_rate): + """ + NB: It's for debug only, please use optimizer.optimize() in production. + """ + callBigDlFunc(self.bigdl_type, + "updateParameters", + self.value, + learning_rate) + def reset(self): """ Initialize the model weights. @@ -270,8 +287,13 @@ def set_weights(self, weights): ... print(err.java_exception) ... java.lang.IllegalArgumentException: requirement failed: the number of input weight/bias is not consistant with number of weight/bias of this layer + >>> cAdd = CAdd([4, 1]) + creating: createCAdd + >>> cAdd.set_weights(np.ones([4, 1])) + >>> (cAdd.get_weights()[0] == np.ones([4, 1])).all() + True """ - tensors = [JTensor.from_ndarray(param, self.bigdl_type) for param in weights] + tensors = [JTensor.from_ndarray(param, self.bigdl_type) for param in to_list(weights)] callBigDlFunc(self.bigdl_type, "setWeights", self.value, tensors) def get_weights(self): From 9f86673ae7e7d8f54aceceadf3a338cbf991cbe7 Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Thu, 15 Jun 2017 16:22:07 +0800 Subject: [PATCH 058/823] Add zero_grad_parameters and update_parameters methods (#1018) * Add zero_grad_parameters and update_parameters methods * revert --- python/dllib/src/bigdl/dllib/nn/layer.py | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index aa57fd640f3..d6e7e646b56 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -181,6 +181,23 @@ def backward(self, input, grad_output): self.check_input(grad_output)) return self.convert_output(output) + def zero_grad_parameters(self): + """ + NB: It's for debug only, please use optimizer.optimize() in production. + If the module has parameters, this will zero the accumulation of the gradients with respect + to these parameters. Otherwise, it does nothing. + """ + callJavaFunc(get_spark_context(), self.value.zeroGradParameters) + + def update_parameters(self, learning_rate): + """ + NB: It's for debug only, please use optimizer.optimize() in production. + """ + callBigDlFunc(self.bigdl_type, + "updateParameters", + self.value, + learning_rate) + def reset(self): """ Initialize the model weights. @@ -270,8 +287,13 @@ def set_weights(self, weights): ... print(err.java_exception) ... java.lang.IllegalArgumentException: requirement failed: the number of input weight/bias is not consistant with number of weight/bias of this layer + >>> cAdd = CAdd([4, 1]) + creating: createCAdd + >>> cAdd.set_weights(np.ones([4, 1])) + >>> (cAdd.get_weights()[0] == np.ones([4, 1])).all() + True """ - tensors = [JTensor.from_ndarray(param, self.bigdl_type) for param in weights] + tensors = [JTensor.from_ndarray(param, self.bigdl_type) for param in to_list(weights)] callBigDlFunc(self.bigdl_type, "setWeights", self.value, tensors) def get_weights(self): From 63a0340250aa17b7d2657d4b586f162ce2e87a0b Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Thu, 15 Jun 2017 16:22:07 +0800 Subject: [PATCH 059/823] Add zero_grad_parameters and update_parameters methods (#1018) * Add zero_grad_parameters and update_parameters methods * revert --- simple_integration_test.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/simple_integration_test.py b/simple_integration_test.py index e30610930cf..d094f745a0c 100644 --- a/simple_integration_test.py +++ b/simple_integration_test.py @@ -35,6 +35,33 @@ def setUp(self): def tearDown(self): self.sc.stop() + def test_training(self): + cadd = CAdd([5, 1]) + y = np.ones([5, 4]) + bf = np.ones([5, 4]) + for i in range(y.shape[0]): + bf[i] = i + 1 + + def grad_update(mlp, x, y, criterion, learning_rate): + pred = mlp.forward(x) + err = criterion.forward(pred, y) + grad_criterion = criterion.backward(pred, y) + mlp.zero_grad_parameters() + mlp.backward(x, grad_criterion) + mlp.update_parameters(learning_rate) + return err + + mse = MSECriterion() + for i in range(0, 1000): + x = np.random.random((5, 4)) + y = x.copy() + y = y + bf + err = grad_update(cadd, x, y, mse, 0.01) + print(cadd.get_weights()[0]) + self.assertTrue(np.allclose(cadd.get_weights()[0], + np.array([1, 2, 3, 4, 5]).reshape((5, 1)), + rtol=1.e-1)) + def test_load_model(self): fc1 = Linear(4, 2) fc1.set_weights([np.ones((4, 2)), np.ones((2, ))]) From 57ee7afbcb0ae838d20846a3104d729b5f5166cc Mon Sep 17 00:00:00 2001 From: yunhuiguo Date: Mon, 19 Jun 2017 09:54:48 +0800 Subject: [PATCH 060/823] add movielens (#1027) add movielens update update update Update movielens.py update update update Update movielens.py update update --- .../bigdl/dllib/feature/dataset/movielens.py | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 python/dllib/src/bigdl/dllib/feature/dataset/movielens.py diff --git a/python/dllib/src/bigdl/dllib/feature/dataset/movielens.py b/python/dllib/src/bigdl/dllib/feature/dataset/movielens.py new file mode 100644 index 00000000000..251ac35e06c --- /dev/null +++ b/python/dllib/src/bigdl/dllib/feature/dataset/movielens.py @@ -0,0 +1,48 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os + +import zipfile +import numpy as np + +from bigdl.dataset import base + +SOURCE_URL = 'http://files.grouplens.org/datasets/movielens/' + +def read_data_sets(data_dir): + """ + Parse or download movielens 1m data if train_dir is empty. + + :param data_dir: The directory storing the movielens data + :return: a 2D numpy array with user index and item index in each row + """ + + WHOLE_DATA = 'ml-1m.zip' + local_file = base.maybe_download(WHOLE_DATA, data_dir, SOURCE_URL + WHOLE_DATA) + zip_ref = zipfile.ZipFile(local_file, 'r') + extracted_to = os.path.join(data_dir, "ml-1m") + if not os.path.exists(extracted_to): + print("Extracting %s to %s" % (local_file, data_dir)) + zip_ref.extractall(data_dir) + zip_ref.close() + rating_files = os.path.join(extracted_to,"ratings.dat") + rating_list = [i.strip().split("::")[:2] for i in open(rating_files,"r").readlines()] + movielens_data = np.array(rating_list).astype(int) + return movielens_data + +if __name__ == "__main__": + movielens_data = read_data_sets("/tmp/movielens/") From b0c2846016905f3264e0fac0d3bf8bd179d4f948 Mon Sep 17 00:00:00 2001 From: yunhuiguo Date: Mon, 19 Jun 2017 09:54:48 +0800 Subject: [PATCH 061/823] add movielens (#1027) add movielens update update update Update movielens.py update update update Update movielens.py update update --- .../bigdl/dllib/feature/dataset/movielens.py | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 python/dllib/src/bigdl/dllib/feature/dataset/movielens.py diff --git a/python/dllib/src/bigdl/dllib/feature/dataset/movielens.py b/python/dllib/src/bigdl/dllib/feature/dataset/movielens.py new file mode 100644 index 00000000000..251ac35e06c --- /dev/null +++ b/python/dllib/src/bigdl/dllib/feature/dataset/movielens.py @@ -0,0 +1,48 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os + +import zipfile +import numpy as np + +from bigdl.dataset import base + +SOURCE_URL = 'http://files.grouplens.org/datasets/movielens/' + +def read_data_sets(data_dir): + """ + Parse or download movielens 1m data if train_dir is empty. + + :param data_dir: The directory storing the movielens data + :return: a 2D numpy array with user index and item index in each row + """ + + WHOLE_DATA = 'ml-1m.zip' + local_file = base.maybe_download(WHOLE_DATA, data_dir, SOURCE_URL + WHOLE_DATA) + zip_ref = zipfile.ZipFile(local_file, 'r') + extracted_to = os.path.join(data_dir, "ml-1m") + if not os.path.exists(extracted_to): + print("Extracting %s to %s" % (local_file, data_dir)) + zip_ref.extractall(data_dir) + zip_ref.close() + rating_files = os.path.join(extracted_to,"ratings.dat") + rating_list = [i.strip().split("::")[:2] for i in open(rating_files,"r").readlines()] + movielens_data = np.array(rating_list).astype(int) + return movielens_data + +if __name__ == "__main__": + movielens_data = read_data_sets("/tmp/movielens/") From b36125031228d07743976c249ef6d346bb2bdfc8 Mon Sep 17 00:00:00 2001 From: yunhuiguo Date: Mon, 19 Jun 2017 09:54:48 +0800 Subject: [PATCH 062/823] add movielens (#1027) add movielens update update update Update movielens.py update update update Update movielens.py update update --- simple_integration_test.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/simple_integration_test.py b/simple_integration_test.py index d094f745a0c..aefc99f6593 100644 --- a/simple_integration_test.py +++ b/simple_integration_test.py @@ -20,6 +20,7 @@ from bigdl.optim.optimizer import * from bigdl.util.common import * from bigdl.nn.initialization_method import * +from bigdl.dataset import movielens import numpy as np import unittest import tempfile @@ -328,5 +329,11 @@ def test_rng(self): for i in range(0, 2): self.assertTrue(np.allclose(data[i], data2[i])) + def test_movielens(self): + movielens_data = movielens.read_data_sets("/tmp/movielens/") + ground_label = np.array([[1, 1193], [1, 661]]) + for i in range(0, 2): + self.assertTrue(np.allclose(movielens_data[i], ground_label[i], atol=1e-6, rtol=0)) + if __name__ == "__main__": unittest.main(failfast=True) From 5458210a7ea9d9e91a07b15350b56f7ed3a890b7 Mon Sep 17 00:00:00 2001 From: Yanzhang Wang Date: Mon, 19 Jun 2017 17:00:40 +0800 Subject: [PATCH 063/823] [FIX943] bash environment variables to java properties (#988) * feat: make all environment virables to java properties * fix: typo * refactor: add api for setting mkl environment * fix: get back the omp environments * fix: set mkl use one omp thread, which means sequential mode * fix: typo of waitPolicyPasssive * test: add tests for localmode and corenumber Converting localmode variable to method. The only method to make the mode bigdl running is through java propery `-Dbigdl.localmode=true`. The default value is false. * fix: set wrong mode in SummarySpec and change name of mkl properties * fix: delete the bigdl.sh in all documents * fix: code style exceeds 100 chars * refactor: move mkl environment variables setting to MKL. The previous patch will only effect the driver side, except executor side. All sides will load library and execute setMklEnv method after loading completed if we put these code snippets in MKL.java which is in BigDL-core project. * fix: bigdl-native back to 0.1.0 * fix: before Engine.init we should not call Engine.coreNumber * feat: We convert all environment variables to mkl/omp apis. Because we have modify the BigDL-core, which contains the new JNI methods called in MKL.java. So we should modify the bigdl.native to 0.2.0-SNAPSHOT in spark/dl/pom.xml. * fix: delete all bigdl.sh in shell scripts and documents --- python/dllib/src/bigdl/dllib/models/lenet/README.md | 4 ---- python/dllib/src/bigdl/dllib/models/textclassifier/README.md | 3 --- 2 files changed, 7 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/models/lenet/README.md b/python/dllib/src/bigdl/dllib/models/lenet/README.md index d1a79c3bfd5..96f324ff9c6 100644 --- a/python/dllib/src/bigdl/dllib/models/lenet/README.md +++ b/python/dllib/src/bigdl/dllib/models/lenet/README.md @@ -17,15 +17,11 @@ Program would download the minst data into ```/tmp/mnist``` automatically by def ``` -**Source bigdl.sh fisrt which would setup the essential environment for you, otherwise program would fail fast.** - We would train a LeNet model in spark local mode with the following commands and you can distribute it across cluster by modifying the spark master and the executor cores. ``` BigDL_HOME=... - source $BigDL_HOME/dist/bin/bigdl.sh - SPARK_HOME=... MASTER=local[*] PYTHON_API_ZIP_PATH=${BigDL_HOME}/dist/lib/bigdl-VERSION-python-api.zip diff --git a/python/dllib/src/bigdl/dllib/models/textclassifier/README.md b/python/dllib/src/bigdl/dllib/models/textclassifier/README.md index 6999bb5c3e4..39875980000 100644 --- a/python/dllib/src/bigdl/dllib/models/textclassifier/README.md +++ b/python/dllib/src/bigdl/dllib/models/textclassifier/README.md @@ -23,8 +23,6 @@ $ [/tmp/news20]$ tree . -L 1 then running the flowing script would automatically download the data during the first run. -bigdl.sh would setup the essential environment for you and it would accept a spark-submit command as an input parameter. - ```{r, engine='sh'} PYTHONHASHSEED=... BigDL_HOME=... @@ -33,7 +31,6 @@ bigdl.sh would setup the essential environment for you and it would accept a spa PYTHON_API_ZIP_PATH=${BigDL_HOME}/dist/lib/bigdl-VERSION-python-api.zip BigDL_JAR_PATH=${BigDL_HOME}/dist/lib/bigdl-VERSION-jar-with-dependencies.jar PYTHONPATH=${PYTHON_API_ZIP_PATH}:$PYTHONPATH - source ${BigDL_HOME}/dist/bin/bigdl.sh ${SPARK_HOME}/bin/spark-submit \ --master ${MASTER} \ From 72bc1e1fe281419e62b6123012b70419bbd696c2 Mon Sep 17 00:00:00 2001 From: Yanzhang Wang Date: Mon, 19 Jun 2017 17:00:40 +0800 Subject: [PATCH 064/823] [FIX943] bash environment variables to java properties (#988) * feat: make all environment virables to java properties * fix: typo * refactor: add api for setting mkl environment * fix: get back the omp environments * fix: set mkl use one omp thread, which means sequential mode * fix: typo of waitPolicyPasssive * test: add tests for localmode and corenumber Converting localmode variable to method. The only method to make the mode bigdl running is through java propery `-Dbigdl.localmode=true`. The default value is false. * fix: set wrong mode in SummarySpec and change name of mkl properties * fix: delete the bigdl.sh in all documents * fix: code style exceeds 100 chars * refactor: move mkl environment variables setting to MKL. The previous patch will only effect the driver side, except executor side. All sides will load library and execute setMklEnv method after loading completed if we put these code snippets in MKL.java which is in BigDL-core project. * fix: bigdl-native back to 0.1.0 * fix: before Engine.init we should not call Engine.coreNumber * feat: We convert all environment variables to mkl/omp apis. Because we have modify the BigDL-core, which contains the new JNI methods called in MKL.java. So we should modify the bigdl.native to 0.2.0-SNAPSHOT in spark/dl/pom.xml. * fix: delete all bigdl.sh in shell scripts and documents --- python/dllib/src/bigdl/dllib/models/lenet/README.md | 4 ---- python/dllib/src/bigdl/dllib/models/textclassifier/README.md | 3 --- 2 files changed, 7 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/models/lenet/README.md b/python/dllib/src/bigdl/dllib/models/lenet/README.md index d1a79c3bfd5..96f324ff9c6 100644 --- a/python/dllib/src/bigdl/dllib/models/lenet/README.md +++ b/python/dllib/src/bigdl/dllib/models/lenet/README.md @@ -17,15 +17,11 @@ Program would download the minst data into ```/tmp/mnist``` automatically by def ``` -**Source bigdl.sh fisrt which would setup the essential environment for you, otherwise program would fail fast.** - We would train a LeNet model in spark local mode with the following commands and you can distribute it across cluster by modifying the spark master and the executor cores. ``` BigDL_HOME=... - source $BigDL_HOME/dist/bin/bigdl.sh - SPARK_HOME=... MASTER=local[*] PYTHON_API_ZIP_PATH=${BigDL_HOME}/dist/lib/bigdl-VERSION-python-api.zip diff --git a/python/dllib/src/bigdl/dllib/models/textclassifier/README.md b/python/dllib/src/bigdl/dllib/models/textclassifier/README.md index 6999bb5c3e4..39875980000 100644 --- a/python/dllib/src/bigdl/dllib/models/textclassifier/README.md +++ b/python/dllib/src/bigdl/dllib/models/textclassifier/README.md @@ -23,8 +23,6 @@ $ [/tmp/news20]$ tree . -L 1 then running the flowing script would automatically download the data during the first run. -bigdl.sh would setup the essential environment for you and it would accept a spark-submit command as an input parameter. - ```{r, engine='sh'} PYTHONHASHSEED=... BigDL_HOME=... @@ -33,7 +31,6 @@ bigdl.sh would setup the essential environment for you and it would accept a spa PYTHON_API_ZIP_PATH=${BigDL_HOME}/dist/lib/bigdl-VERSION-python-api.zip BigDL_JAR_PATH=${BigDL_HOME}/dist/lib/bigdl-VERSION-jar-with-dependencies.jar PYTHONPATH=${PYTHON_API_ZIP_PATH}:$PYTHONPATH - source ${BigDL_HOME}/dist/bin/bigdl.sh ${SPARK_HOME}/bin/spark-submit \ --master ${MASTER} \ From 507d72ffd01739b46d654d82633caac9eac647eb Mon Sep 17 00:00:00 2001 From: Yanzhang Wang Date: Mon, 19 Jun 2017 17:00:40 +0800 Subject: [PATCH 065/823] [FIX943] bash environment variables to java properties (#988) * feat: make all environment virables to java properties * fix: typo * refactor: add api for setting mkl environment * fix: get back the omp environments * fix: set mkl use one omp thread, which means sequential mode * fix: typo of waitPolicyPasssive * test: add tests for localmode and corenumber Converting localmode variable to method. The only method to make the mode bigdl running is through java propery `-Dbigdl.localmode=true`. The default value is false. * fix: set wrong mode in SummarySpec and change name of mkl properties * fix: delete the bigdl.sh in all documents * fix: code style exceeds 100 chars * refactor: move mkl environment variables setting to MKL. The previous patch will only effect the driver side, except executor side. All sides will load library and execute setMklEnv method after loading completed if we put these code snippets in MKL.java which is in BigDL-core project. * fix: bigdl-native back to 0.1.0 * fix: before Engine.init we should not call Engine.coreNumber * feat: We convert all environment variables to mkl/omp apis. Because we have modify the BigDL-core, which contains the new JNI methods called in MKL.java. So we should modify the bigdl.native to 0.2.0-SNAPSHOT in spark/dl/pom.xml. * fix: delete all bigdl.sh in shell scripts and documents --- python/dllib/test/dev/prepare_env.sh | 1 - python/dllib/test/dev/run-tests | 2 +- resources/conf/test_spark-bigdl.conf | 12 +----------- 3 files changed, 2 insertions(+), 13 deletions(-) diff --git a/python/dllib/test/dev/prepare_env.sh b/python/dllib/test/dev/prepare_env.sh index c33b0eb12f3..b77cdd7d5a7 100755 --- a/python/dllib/test/dev/prepare_env.sh +++ b/python/dllib/test/dev/prepare_env.sh @@ -32,7 +32,6 @@ export PYTHONPATH=$PYSPARK_ZIP:$DL_PYTHON_HOME:$DL_PYTHON_HOME/:$DL_PYTHON_HOME/ export SPARK_CLASSPATH=$(find $BIGDL_HOME/spark/dl/target/ -name "*with-dependencies.jar" | head -n 1) echo "SPARK_CLASSPATH": $SPARK_CLASSPATH -source $BIGDL_HOME/scripts/bigdl.sh export PYTHON_EXECUTABLES=("python2" "python3") diff --git a/python/dllib/test/dev/run-tests b/python/dllib/test/dev/run-tests index f260b372dd4..d089568c143 100755 --- a/python/dllib/test/dev/run-tests +++ b/python/dllib/test/dev/run-tests @@ -19,4 +19,4 @@ . `dirname $0`/prepare_env.sh cd "`dirname $0`" -exec $BIGDL_HOME/scripts/bigdl.sh -- python -u ./run-tests.py "$@" +exec python -u ./run-tests.py "$@" diff --git a/resources/conf/test_spark-bigdl.conf b/resources/conf/test_spark-bigdl.conf index 790a9709ebc..9870f82a6bc 100644 --- a/resources/conf/test_spark-bigdl.conf +++ b/resources/conf/test_spark-bigdl.conf @@ -27,16 +27,6 @@ # https://github.com/intel-analytics/BigDL/wiki/Programming-Guide#engine # -spark.executorEnv.DL_ENGINE_TYPE mklblas -spark.executorEnv.MKL_DISABLE_FAST_MM 1 -spark.executorEnv.KMP_BLOCKTIME 0 -spark.executorEnv.OMP_WAIT_POLICY passive -spark.executorEnv.OMP_NUM_THREADS 1 -spark.yarn.appMasterEnv.DL_ENGINE_TYPE mklblas -spark.yarn.appMasterEnv.MKL_DISABLE_FAST_MM 1 -spark.yarn.appMasterEnv.KMP_BLOCKTIME 0 -spark.yarn.appMasterEnv.OMP_WAIT_POLICY passive -spark.yarn.appMasterEnv.OMP_NUM_THREADS 1 spark.shuffle.reduceLocality.enabled false spark.shuffle.blockTransferService nio -spark.scheduler.minRegisteredResourcesRatio 1.0 \ No newline at end of file +spark.scheduler.minRegisteredResourcesRatio 1.0 From 626740f0437083a62fcbf96cc5bd0c78824f18c2 Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Tue, 20 Jun 2017 12:32:59 +0800 Subject: [PATCH 066/823] fix default value and base class (#1037) --- python/dllib/src/bigdl/dllib/nn/criterion.py | 44 +++++++++++++++++- python/dllib/src/bigdl/dllib/nn/layer.py | 47 +++----------------- 2 files changed, 49 insertions(+), 42 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/nn/criterion.py b/python/dllib/src/bigdl/dllib/nn/criterion.py index 20c2e46350c..d3c31192020 100644 --- a/python/dllib/src/bigdl/dllib/nn/criterion.py +++ b/python/dllib/src/bigdl/dllib/nn/criterion.py @@ -247,7 +247,7 @@ class HingeEmbeddingCriterion(Criterion): ''' def __init__(self, - margin=1, + margin=1.0, size_average=True, bigdl_type="float"): super(HingeEmbeddingCriterion, self).__init__(None, bigdl_type, @@ -267,10 +267,12 @@ class L1HingeEmbeddingCriterion(Criterion): >>> l1HingeEmbeddingCriterion = L1HingeEmbeddingCriterion(1e-5) creating: createL1HingeEmbeddingCriterion + >>> l1HingeEmbeddingCriterion = L1HingeEmbeddingCriterion() + creating: createL1HingeEmbeddingCriterion ''' def __init__(self, - margin=1, + margin=1.0, bigdl_type="float"): super(L1HingeEmbeddingCriterion, self).__init__(None, bigdl_type, margin) @@ -634,6 +636,44 @@ def __init__(self, bigdl_type="float"): super(SoftMarginCriterion, self).__init__(None, bigdl_type, size_average) + +class DiceCoefficientCriterion(Criterion): + + ''' + The Dice-Coefficient criterion + input: Tensor,target: Tensor + +``` + return: 2 * (input intersection target) + 1 - ---------------------------------- + input union target +``` + + >>> diceCoefficientCriterion = DiceCoefficientCriterion(size_average = True, epsilon = 1.0) + creating: createDiceCoefficientCriterion + ''' + + def __init__(self, + size_average, + epsilon, + bigdl_type="float"): + super(DiceCoefficientCriterion, self).__init__(None, bigdl_type, + size_average, + epsilon) + +class L1Cost(Criterion): + + ''' + compute L1 norm for input, and sign of input + + >>> l1Cost = L1Cost() + creating: createL1Cost + ''' + + def __init__(self, + bigdl_type="float"): + super(L1Cost, self).__init__(None, bigdl_type) + def _test(): import doctest from pyspark import SparkContext diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index d6e7e646b56..ce1334e3c8c 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -1360,29 +1360,6 @@ def __init__(self, bigdl_type="float"): super(CosineDistance, self).__init__(None, bigdl_type) -class DiceCoefficientCriterion(Layer): - - ''' - The Dice-Coefficient criterion - input: Tensor,target: Tensor - -``` - return: 2 * (input intersection target) - 1 - ---------------------------------- - input union target -``` - - >>> diceCoefficientCriterion = DiceCoefficientCriterion(size_average = True, epsilon = 1.0) - creating: createDiceCoefficientCriterion - ''' - - def __init__(self, - size_average, - epsilon, - bigdl_type="float"): - super(DiceCoefficientCriterion, self).__init__(None, bigdl_type, - size_average, - epsilon) class DotProduct(Layer): @@ -1499,10 +1476,12 @@ class GradientReversal(Layer): >>> gradientReversal = GradientReversal(1e-5) creating: createGradientReversal + >>> gradientReversal = GradientReversal() + creating: createGradientReversal ''' def __init__(self, - the_lambda=1, + the_lambda=1.0, bigdl_type="float"): super(GradientReversal, self).__init__(None, bigdl_type, the_lambda) @@ -1550,11 +1529,13 @@ class HardTanh(Layer): >>> hardTanh = HardTanh(1e-5, 1e5, True) creating: createHardTanh + >>> hardTanh = HardTanh() + creating: createHardTanh ''' def __init__(self, - min_value=-1, - max_value=1, + min_value=-1.0, + max_value=1.0, inplace=False, bigdl_type="float"): super(HardTanh, self).__init__(None, bigdl_type, @@ -1646,20 +1627,6 @@ def __init__(self, n_input_dims) -class L1Cost(Layer): - - ''' - compute L1 norm for input, and sign of input - - >>> l1Cost = L1Cost() - creating: createL1Cost - ''' - - def __init__(self, - bigdl_type="float"): - super(L1Cost, self).__init__(None, bigdl_type) - - class L1Penalty(Layer): ''' From 3f9f73ea3cca2c1564507de488d54e55b2df22ae Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Tue, 20 Jun 2017 12:32:59 +0800 Subject: [PATCH 067/823] fix default value and base class (#1037) --- python/dllib/src/bigdl/dllib/nn/criterion.py | 44 +++++++++++++++++- python/dllib/src/bigdl/dllib/nn/layer.py | 47 +++----------------- 2 files changed, 49 insertions(+), 42 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/nn/criterion.py b/python/dllib/src/bigdl/dllib/nn/criterion.py index 20c2e46350c..d3c31192020 100644 --- a/python/dllib/src/bigdl/dllib/nn/criterion.py +++ b/python/dllib/src/bigdl/dllib/nn/criterion.py @@ -247,7 +247,7 @@ class HingeEmbeddingCriterion(Criterion): ''' def __init__(self, - margin=1, + margin=1.0, size_average=True, bigdl_type="float"): super(HingeEmbeddingCriterion, self).__init__(None, bigdl_type, @@ -267,10 +267,12 @@ class L1HingeEmbeddingCriterion(Criterion): >>> l1HingeEmbeddingCriterion = L1HingeEmbeddingCriterion(1e-5) creating: createL1HingeEmbeddingCriterion + >>> l1HingeEmbeddingCriterion = L1HingeEmbeddingCriterion() + creating: createL1HingeEmbeddingCriterion ''' def __init__(self, - margin=1, + margin=1.0, bigdl_type="float"): super(L1HingeEmbeddingCriterion, self).__init__(None, bigdl_type, margin) @@ -634,6 +636,44 @@ def __init__(self, bigdl_type="float"): super(SoftMarginCriterion, self).__init__(None, bigdl_type, size_average) + +class DiceCoefficientCriterion(Criterion): + + ''' + The Dice-Coefficient criterion + input: Tensor,target: Tensor + +``` + return: 2 * (input intersection target) + 1 - ---------------------------------- + input union target +``` + + >>> diceCoefficientCriterion = DiceCoefficientCriterion(size_average = True, epsilon = 1.0) + creating: createDiceCoefficientCriterion + ''' + + def __init__(self, + size_average, + epsilon, + bigdl_type="float"): + super(DiceCoefficientCriterion, self).__init__(None, bigdl_type, + size_average, + epsilon) + +class L1Cost(Criterion): + + ''' + compute L1 norm for input, and sign of input + + >>> l1Cost = L1Cost() + creating: createL1Cost + ''' + + def __init__(self, + bigdl_type="float"): + super(L1Cost, self).__init__(None, bigdl_type) + def _test(): import doctest from pyspark import SparkContext diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index d6e7e646b56..ce1334e3c8c 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -1360,29 +1360,6 @@ def __init__(self, bigdl_type="float"): super(CosineDistance, self).__init__(None, bigdl_type) -class DiceCoefficientCriterion(Layer): - - ''' - The Dice-Coefficient criterion - input: Tensor,target: Tensor - -``` - return: 2 * (input intersection target) - 1 - ---------------------------------- - input union target -``` - - >>> diceCoefficientCriterion = DiceCoefficientCriterion(size_average = True, epsilon = 1.0) - creating: createDiceCoefficientCriterion - ''' - - def __init__(self, - size_average, - epsilon, - bigdl_type="float"): - super(DiceCoefficientCriterion, self).__init__(None, bigdl_type, - size_average, - epsilon) class DotProduct(Layer): @@ -1499,10 +1476,12 @@ class GradientReversal(Layer): >>> gradientReversal = GradientReversal(1e-5) creating: createGradientReversal + >>> gradientReversal = GradientReversal() + creating: createGradientReversal ''' def __init__(self, - the_lambda=1, + the_lambda=1.0, bigdl_type="float"): super(GradientReversal, self).__init__(None, bigdl_type, the_lambda) @@ -1550,11 +1529,13 @@ class HardTanh(Layer): >>> hardTanh = HardTanh(1e-5, 1e5, True) creating: createHardTanh + >>> hardTanh = HardTanh() + creating: createHardTanh ''' def __init__(self, - min_value=-1, - max_value=1, + min_value=-1.0, + max_value=1.0, inplace=False, bigdl_type="float"): super(HardTanh, self).__init__(None, bigdl_type, @@ -1646,20 +1627,6 @@ def __init__(self, n_input_dims) -class L1Cost(Layer): - - ''' - compute L1 norm for input, and sign of input - - >>> l1Cost = L1Cost() - creating: createL1Cost - ''' - - def __init__(self, - bigdl_type="float"): - super(L1Cost, self).__init__(None, bigdl_type) - - class L1Penalty(Layer): ''' From bed40698e4c632cf82c31b1764a52c70b9ca5196 Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Tue, 20 Jun 2017 15:41:07 +0800 Subject: [PATCH 068/823] convert 1-dimension list into Table instead of Tensor (#1041) --- python/dllib/src/bigdl/dllib/nn/criterion.py | 24 +++++++++++++++----- python/dllib/src/bigdl/dllib/nn/layer.py | 22 ++++++++++++++---- 2 files changed, 35 insertions(+), 11 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/nn/criterion.py b/python/dllib/src/bigdl/dllib/nn/criterion.py index d3c31192020..884bba5124b 100644 --- a/python/dllib/src/bigdl/dllib/nn/criterion.py +++ b/python/dllib/src/bigdl/dllib/nn/criterion.py @@ -51,11 +51,15 @@ def forward(self, input, target): :param target: ndarray or list of ndarray :return: value of loss """ + jinput, input_is_table = Layer.check_input(input) + jtarget, target_is_table = Layer.check_input(target) output = callBigDlFunc(self.bigdl_type, "criterionForward", self.value, - Layer.check_input(input), - Layer.check_input(target)) + jinput, + input_is_table, + jtarget, + target_is_table) return output def backward(self, input, target): @@ -67,11 +71,15 @@ def backward(self, input, target): :param target: ndarray or list of ndarray :return: ndarray """ + jinput, input_is_table = Layer.check_input(input) + jtarget, target_is_table = Layer.check_input(target) output = callBigDlFunc(self.bigdl_type, "criterionBackward", self.value, - Layer.check_input(input), - Layer.check_input(target)) + jinput, + input_is_table, + jtarget, + target_is_table) return Layer.convert_output(output) @classmethod @@ -184,7 +192,7 @@ def __init__(self, class CosineEmbeddingCriterion(Criterion): - ''' + """ Creates a criterion that measures the loss given an input x = {x1, x2}, a table of two Tensors, and a Tensor label y with values 1 or -1. @@ -194,7 +202,11 @@ class CosineEmbeddingCriterion(Criterion): >>> cosineEmbeddingCriterion = CosineEmbeddingCriterion(1e-5, True) creating: createCosineEmbeddingCriterion - ''' + >>> cosineEmbeddingCriterion.forward([np.array([1.0, 2.0, 3.0, 4.0, 5.0]), + ... np.array([5.0, 4.0, 3.0, 2.0, 1.0])], + ... [np.ones(5)]) + 0.0 + """ def __init__(self, margin=0.0, diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index ce1334e3c8c..a69764a6d03 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -128,23 +128,29 @@ def get_dtype(self): @staticmethod def check_input(input): + """ + :param input: ndarray or list of ndarray + :return: (list of JTensor, isTable) + """ if type(input) is list: if len(input) == 0: raise Exception('Error when checking: empty input') if not hasattr(input[0], 'shape'): raise Exception( 'Error when checking: expecting list of ndarray') - return [JTensor.from_ndarray(i) for i in input] + return [JTensor.from_ndarray(i) for i in input], True else: if not hasattr(input, 'shape'): raise Exception( 'Error when checking: expecting list of ndarray') - return [JTensor.from_ndarray(input)] + return [JTensor.from_ndarray(input)], False @staticmethod def convert_output(output): if type(output) is JTensor: return output.to_ndarray() + elif(len(output) == 1): + return output[0].to_ndarray() else: return [x.to_ndarray() for x in output] @@ -156,10 +162,12 @@ def forward(self, input): :param input: ndarray or list of ndarray :return: ndarray or list of ndarray """ + jinput, input_is_table = self.check_input(input) output = callBigDlFunc(self.bigdl_type, "modelForward", self.value, - self.check_input(input)) + jinput, + input_is_table) return self.convert_output(output) def backward(self, input, grad_output): @@ -174,11 +182,15 @@ def backward(self, input, grad_output): :param grad_output: ndarray or list of ndarray :return: ndarray or list of ndarray """ + jinput, input_is_table = self.check_input(input) + jgrad_output, grad_output_is_table = self.check_input(grad_output) output = callBigDlFunc(self.bigdl_type, "modelBackward", self.value, - self.check_input(input), - self.check_input(grad_output)) + jinput, + input_is_table, + jgrad_output, + grad_output_is_table) return self.convert_output(output) def zero_grad_parameters(self): From 516bf508445100ec2fb75e46a215cae59c52fff7 Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Tue, 20 Jun 2017 15:41:07 +0800 Subject: [PATCH 069/823] convert 1-dimension list into Table instead of Tensor (#1041) --- python/dllib/src/bigdl/dllib/nn/criterion.py | 24 +++++++++++++++----- python/dllib/src/bigdl/dllib/nn/layer.py | 22 ++++++++++++++---- 2 files changed, 35 insertions(+), 11 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/nn/criterion.py b/python/dllib/src/bigdl/dllib/nn/criterion.py index d3c31192020..884bba5124b 100644 --- a/python/dllib/src/bigdl/dllib/nn/criterion.py +++ b/python/dllib/src/bigdl/dllib/nn/criterion.py @@ -51,11 +51,15 @@ def forward(self, input, target): :param target: ndarray or list of ndarray :return: value of loss """ + jinput, input_is_table = Layer.check_input(input) + jtarget, target_is_table = Layer.check_input(target) output = callBigDlFunc(self.bigdl_type, "criterionForward", self.value, - Layer.check_input(input), - Layer.check_input(target)) + jinput, + input_is_table, + jtarget, + target_is_table) return output def backward(self, input, target): @@ -67,11 +71,15 @@ def backward(self, input, target): :param target: ndarray or list of ndarray :return: ndarray """ + jinput, input_is_table = Layer.check_input(input) + jtarget, target_is_table = Layer.check_input(target) output = callBigDlFunc(self.bigdl_type, "criterionBackward", self.value, - Layer.check_input(input), - Layer.check_input(target)) + jinput, + input_is_table, + jtarget, + target_is_table) return Layer.convert_output(output) @classmethod @@ -184,7 +192,7 @@ def __init__(self, class CosineEmbeddingCriterion(Criterion): - ''' + """ Creates a criterion that measures the loss given an input x = {x1, x2}, a table of two Tensors, and a Tensor label y with values 1 or -1. @@ -194,7 +202,11 @@ class CosineEmbeddingCriterion(Criterion): >>> cosineEmbeddingCriterion = CosineEmbeddingCriterion(1e-5, True) creating: createCosineEmbeddingCriterion - ''' + >>> cosineEmbeddingCriterion.forward([np.array([1.0, 2.0, 3.0, 4.0, 5.0]), + ... np.array([5.0, 4.0, 3.0, 2.0, 1.0])], + ... [np.ones(5)]) + 0.0 + """ def __init__(self, margin=0.0, diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index ce1334e3c8c..a69764a6d03 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -128,23 +128,29 @@ def get_dtype(self): @staticmethod def check_input(input): + """ + :param input: ndarray or list of ndarray + :return: (list of JTensor, isTable) + """ if type(input) is list: if len(input) == 0: raise Exception('Error when checking: empty input') if not hasattr(input[0], 'shape'): raise Exception( 'Error when checking: expecting list of ndarray') - return [JTensor.from_ndarray(i) for i in input] + return [JTensor.from_ndarray(i) for i in input], True else: if not hasattr(input, 'shape'): raise Exception( 'Error when checking: expecting list of ndarray') - return [JTensor.from_ndarray(input)] + return [JTensor.from_ndarray(input)], False @staticmethod def convert_output(output): if type(output) is JTensor: return output.to_ndarray() + elif(len(output) == 1): + return output[0].to_ndarray() else: return [x.to_ndarray() for x in output] @@ -156,10 +162,12 @@ def forward(self, input): :param input: ndarray or list of ndarray :return: ndarray or list of ndarray """ + jinput, input_is_table = self.check_input(input) output = callBigDlFunc(self.bigdl_type, "modelForward", self.value, - self.check_input(input)) + jinput, + input_is_table) return self.convert_output(output) def backward(self, input, grad_output): @@ -174,11 +182,15 @@ def backward(self, input, grad_output): :param grad_output: ndarray or list of ndarray :return: ndarray or list of ndarray """ + jinput, input_is_table = self.check_input(input) + jgrad_output, grad_output_is_table = self.check_input(grad_output) output = callBigDlFunc(self.bigdl_type, "modelBackward", self.value, - self.check_input(input), - self.check_input(grad_output)) + jinput, + input_is_table, + jgrad_output, + grad_output_is_table) return self.convert_output(output) def zero_grad_parameters(self): From b2e4970acdc354622bd155c0223280eb9ec0e958 Mon Sep 17 00:00:00 2001 From: Xianyan Date: Wed, 21 Jun 2017 08:53:37 +0800 Subject: [PATCH 070/823] [Issue 1044] add method for MultiCriterion and ParallelCriterion (#1047) * add method for MultiCriterion and ParallelCriterion * fix parallelCriterion --- python/dllib/src/bigdl/dllib/nn/criterion.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/nn/criterion.py b/python/dllib/src/bigdl/dllib/nn/criterion.py index 884bba5124b..e740c1cd82f 100644 --- a/python/dllib/src/bigdl/dllib/nn/criterion.py +++ b/python/dllib/src/bigdl/dllib/nn/criterion.py @@ -349,12 +349,20 @@ class MultiCriterion(Criterion): >>> multiCriterion = MultiCriterion() creating: createMultiCriterion + >>> mSECriterion = MSECriterion() + creating: createMSECriterion + >>> multiCriterion = multiCriterion.add(mSECriterion) + >>> multiCriterion = multiCriterion.add(mSECriterion) ''' def __init__(self, bigdl_type="float"): super(MultiCriterion, self).__init__(None, bigdl_type) + def add(self, criterion, weight=1.0): + self.value.add(criterion.value, weight) + return self + class MultiLabelMarginCriterion(Criterion): @@ -392,6 +400,10 @@ class ParallelCriterion(Criterion): >>> parallelCriterion = ParallelCriterion(True) creating: createParallelCriterion + >>> mSECriterion = MSECriterion() + creating: createMSECriterion + >>> parallelCriterion = parallelCriterion.add(mSECriterion) + >>> parallelCriterion = parallelCriterion.add(mSECriterion) ''' def __init__(self, @@ -400,6 +412,10 @@ def __init__(self, super(ParallelCriterion, self).__init__(None, bigdl_type, repeat_target) + def add(self, criterion, weight=1.0): + self.value.add(criterion.value, weight) + return self + class SmoothL1Criterion(Criterion): From 090715254212236ab4bd24c7dbd646f2c4eb58f1 Mon Sep 17 00:00:00 2001 From: Xianyan Date: Wed, 21 Jun 2017 08:53:37 +0800 Subject: [PATCH 071/823] [Issue 1044] add method for MultiCriterion and ParallelCriterion (#1047) * add method for MultiCriterion and ParallelCriterion * fix parallelCriterion --- python/dllib/src/bigdl/dllib/nn/criterion.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/nn/criterion.py b/python/dllib/src/bigdl/dllib/nn/criterion.py index 884bba5124b..e740c1cd82f 100644 --- a/python/dllib/src/bigdl/dllib/nn/criterion.py +++ b/python/dllib/src/bigdl/dllib/nn/criterion.py @@ -349,12 +349,20 @@ class MultiCriterion(Criterion): >>> multiCriterion = MultiCriterion() creating: createMultiCriterion + >>> mSECriterion = MSECriterion() + creating: createMSECriterion + >>> multiCriterion = multiCriterion.add(mSECriterion) + >>> multiCriterion = multiCriterion.add(mSECriterion) ''' def __init__(self, bigdl_type="float"): super(MultiCriterion, self).__init__(None, bigdl_type) + def add(self, criterion, weight=1.0): + self.value.add(criterion.value, weight) + return self + class MultiLabelMarginCriterion(Criterion): @@ -392,6 +400,10 @@ class ParallelCriterion(Criterion): >>> parallelCriterion = ParallelCriterion(True) creating: createParallelCriterion + >>> mSECriterion = MSECriterion() + creating: createMSECriterion + >>> parallelCriterion = parallelCriterion.add(mSECriterion) + >>> parallelCriterion = parallelCriterion.add(mSECriterion) ''' def __init__(self, @@ -400,6 +412,10 @@ def __init__(self, super(ParallelCriterion, self).__init__(None, bigdl_type, repeat_target) + def add(self, criterion, weight=1.0): + self.value.add(criterion.value, weight) + return self + class SmoothL1Criterion(Criterion): From 56863cbccf5b7bb8af2b7e7e9e1e3c58f8b8cd14 Mon Sep 17 00:00:00 2001 From: Ian Wong Date: Wed, 21 Jun 2017 12:50:17 +0800 Subject: [PATCH 072/823] Support Tensorflow model file read/write (#800) * nn refactor * nn refactor * fix code style issue * change back the layers * code refactor * change tests to automatic test * add more test for model save * remove some useless unit test * add more save test * add more writer test * rnn test case automation * refine save test * refine save unit test * remove NHWC * meet code review * meet code review * use MulConst in MulTF * add a flatten node for tf 1.1 * fix code style and failed unit test * mv tf model layers to another package * add a python example to show how to define model in tensorflow and run in BigDL * move tf example python code to example folder * Add backward test in LeNet and AlexNet (#19) * add unit test of lenet backward * add some print * add backward test in lenet and alexnet * seperate testModel into forward and backward methods --- python/dllib/src/bigdl/dllib/nn/layer.py | 10 +++ .../dllib/src/bigdl/dllib/utils/tf_utils.py | 88 +++++++++++++++++++ 2 files changed, 98 insertions(+) create mode 100644 python/dllib/src/bigdl/dllib/utils/tf_utils.py diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index a69764a6d03..a6509ee1a46 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -407,6 +407,16 @@ def load_caffe(model, defPath, modelPath, match_all=True, bigdl_type="float"): jmodel = callBigDlFunc(bigdl_type, "loadCaffe", model, defPath, modelPath, match_all) return Layer.of(jmodel) + @staticmethod + def load_tensorflow(path, inputs, outputs, byte_order = "little_endian", bigdl_type="float"): + """ + Load a pre-trained Tensorflow model. + :param path: The path containing the pre-trained model. + :return: A pre-trained model. + """ + jmodel = callBigDlFunc(bigdl_type, "loadTF", path, inputs, outputs, byte_order) + return Model.of(jmodel) + class Linear(Layer): diff --git a/python/dllib/src/bigdl/dllib/utils/tf_utils.py b/python/dllib/src/bigdl/dllib/utils/tf_utils.py new file mode 100644 index 00000000000..ac57a52d862 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/utils/tf_utils.py @@ -0,0 +1,88 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import tempfile + +import tensorflow as tf + +from google.protobuf import text_format + +from tensorflow.core.framework import graph_pb2 +from tensorflow.python.client import session +from tensorflow.python.framework import graph_util +from tensorflow.python.framework import importer +from tensorflow.python.platform import gfile + +from bigdl.nn.layer import Model + +def convert(input_ops, output_ops, sess): + """ + Convert tensorflow model to bigdl model + :param input_ops: operation list used for input, should be placeholders + :param output_ops: operations list used for output + :param sess: current tensorflow session + :return: bigdl model + """ + input_names = map(lambda x: x.name.split(":")[0], input_ops) + output_names = map(lambda x: x.name.split(":")[0], output_ops) + temp = tempfile.mkdtemp() + + saver = tf.train.Saver() + saver.save(sess, temp + '/model.chkp') + tf.train.write_graph(sess.graph, temp, 'model.pbtxt') + + merge_checkpoint(temp + '/model.pbtxt', + temp + '/model.chkp', + output_names, + temp + '/model.pb', sess) + return Model.load_tensorflow(temp + '/model.pb', input_names, output_names) + +def merge_checkpoint(input_graph, + checkpoint, + output_node_names, + output_graph, + sess): + """ + Get the variable values from the checkpoint file, and merge them to the GraphDef file + Args: + input_graph: the GraphDef file, doesn't contain variable values + checkpoint: the checkpoint file + output_node_names: A list of string, the output names + output_graph: String of the location and the name of the + output graph + """ + restore_op_name = "save/restore_all" + filename_tensor_name = "save/Const:0" + + input_graph_def = graph_pb2.GraphDef() + with gfile.FastGFile(input_graph, "r") as f: + text_format.Merge(f.read().decode("utf-8"), input_graph_def) + + for node in input_graph_def.node: + node.device = "" + + importer.import_graph_def(input_graph_def, name="") + + sess.run([restore_op_name], {filename_tensor_name: checkpoint}) + output_graph_def = graph_util.convert_variables_to_constants( + sess, + input_graph_def, + output_node_names, + variable_names_blacklist="" + ) + + with gfile.GFile(output_graph, "wb") as f: + f.write(output_graph_def.SerializeToString()) \ No newline at end of file From db9b4320dbf063692a264b1c7df9320da46de80e Mon Sep 17 00:00:00 2001 From: Ian Wong Date: Wed, 21 Jun 2017 12:50:17 +0800 Subject: [PATCH 073/823] Support Tensorflow model file read/write (#800) * nn refactor * nn refactor * fix code style issue * change back the layers * code refactor * change tests to automatic test * add more test for model save * remove some useless unit test * add more save test * add more writer test * rnn test case automation * refine save test * refine save unit test * remove NHWC * meet code review * meet code review * use MulConst in MulTF * add a flatten node for tf 1.1 * fix code style and failed unit test * mv tf model layers to another package * add a python example to show how to define model in tensorflow and run in BigDL * move tf example python code to example folder * Add backward test in LeNet and AlexNet (#19) * add unit test of lenet backward * add some print * add backward test in lenet and alexnet * seperate testModel into forward and backward methods --- python/dllib/src/bigdl/dllib/nn/layer.py | 10 +++ .../dllib/src/bigdl/dllib/utils/tf_utils.py | 88 +++++++++++++++++++ 2 files changed, 98 insertions(+) create mode 100644 python/dllib/src/bigdl/dllib/utils/tf_utils.py diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index a69764a6d03..a6509ee1a46 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -407,6 +407,16 @@ def load_caffe(model, defPath, modelPath, match_all=True, bigdl_type="float"): jmodel = callBigDlFunc(bigdl_type, "loadCaffe", model, defPath, modelPath, match_all) return Layer.of(jmodel) + @staticmethod + def load_tensorflow(path, inputs, outputs, byte_order = "little_endian", bigdl_type="float"): + """ + Load a pre-trained Tensorflow model. + :param path: The path containing the pre-trained model. + :return: A pre-trained model. + """ + jmodel = callBigDlFunc(bigdl_type, "loadTF", path, inputs, outputs, byte_order) + return Model.of(jmodel) + class Linear(Layer): diff --git a/python/dllib/src/bigdl/dllib/utils/tf_utils.py b/python/dllib/src/bigdl/dllib/utils/tf_utils.py new file mode 100644 index 00000000000..ac57a52d862 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/utils/tf_utils.py @@ -0,0 +1,88 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import tempfile + +import tensorflow as tf + +from google.protobuf import text_format + +from tensorflow.core.framework import graph_pb2 +from tensorflow.python.client import session +from tensorflow.python.framework import graph_util +from tensorflow.python.framework import importer +from tensorflow.python.platform import gfile + +from bigdl.nn.layer import Model + +def convert(input_ops, output_ops, sess): + """ + Convert tensorflow model to bigdl model + :param input_ops: operation list used for input, should be placeholders + :param output_ops: operations list used for output + :param sess: current tensorflow session + :return: bigdl model + """ + input_names = map(lambda x: x.name.split(":")[0], input_ops) + output_names = map(lambda x: x.name.split(":")[0], output_ops) + temp = tempfile.mkdtemp() + + saver = tf.train.Saver() + saver.save(sess, temp + '/model.chkp') + tf.train.write_graph(sess.graph, temp, 'model.pbtxt') + + merge_checkpoint(temp + '/model.pbtxt', + temp + '/model.chkp', + output_names, + temp + '/model.pb', sess) + return Model.load_tensorflow(temp + '/model.pb', input_names, output_names) + +def merge_checkpoint(input_graph, + checkpoint, + output_node_names, + output_graph, + sess): + """ + Get the variable values from the checkpoint file, and merge them to the GraphDef file + Args: + input_graph: the GraphDef file, doesn't contain variable values + checkpoint: the checkpoint file + output_node_names: A list of string, the output names + output_graph: String of the location and the name of the + output graph + """ + restore_op_name = "save/restore_all" + filename_tensor_name = "save/Const:0" + + input_graph_def = graph_pb2.GraphDef() + with gfile.FastGFile(input_graph, "r") as f: + text_format.Merge(f.read().decode("utf-8"), input_graph_def) + + for node in input_graph_def.node: + node.device = "" + + importer.import_graph_def(input_graph_def, name="") + + sess.run([restore_op_name], {filename_tensor_name: checkpoint}) + output_graph_def = graph_util.convert_variables_to_constants( + sess, + input_graph_def, + output_node_names, + variable_names_blacklist="" + ) + + with gfile.GFile(output_graph, "wb") as f: + f.write(output_graph_def.SerializeToString()) \ No newline at end of file From a2e74db41513dd2ddea837e3b81e6c48bdbe8beb Mon Sep 17 00:00:00 2001 From: Ian Wong Date: Wed, 21 Jun 2017 12:50:17 +0800 Subject: [PATCH 074/823] Support Tensorflow model file read/write (#800) * nn refactor * nn refactor * fix code style issue * change back the layers * code refactor * change tests to automatic test * add more test for model save * remove some useless unit test * add more save test * add more writer test * rnn test case automation * refine save test * refine save unit test * remove NHWC * meet code review * meet code review * use MulConst in MulTF * add a flatten node for tf 1.1 * fix code style and failed unit test * mv tf model layers to another package * add a python example to show how to define model in tensorflow and run in BigDL * move tf example python code to example folder * Add backward test in LeNet and AlexNet (#19) * add unit test of lenet backward * add some print * add backward test in lenet and alexnet * seperate testModel into forward and backward methods --- .../commands/run-tf-example.sh | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100755 python/dllib/test/local_integration/commands/run-tf-example.sh diff --git a/python/dllib/test/local_integration/commands/run-tf-example.sh b/python/dllib/test/local_integration/commands/run-tf-example.sh new file mode 100755 index 00000000000..66f331a929b --- /dev/null +++ b/python/dllib/test/local_integration/commands/run-tf-example.sh @@ -0,0 +1,19 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +cd "`dirname $0`" + +$PYTHON_EXECUTABLE $BIGDL_HOME/pyspark/example/tf_example.py \ No newline at end of file From 78e728ecdf13a094b298998c542b03992bb3f873 Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Wed, 21 Jun 2017 13:10:26 +0800 Subject: [PATCH 075/823] remove tf layers from python api (#1045) --- python/dllib/src/bigdl/dllib/nn/layer.py | 46 +++--------------------- 1 file changed, 4 insertions(+), 42 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index a6509ee1a46..94756d76c29 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -3245,56 +3245,18 @@ def __init__(self, n_input_plane, JTensor.from_ndarray(kernel)) -class Const(Model): - ''' - Return a constant tensor defined by value when forward - ''' - - def __init__(self, value, bigdl_type="float"): - super(Const, self).__init__(None, bigdl_type, JTensor.from_ndarray(value)) -class Fill(Model): - ''' - Return a constant tensor defined by value when forward - ''' - - def __init__(self, value, bigdl_type="float"): - super(Fill, self).__init__(None, bigdl_type, value) - -class Pack(Model): +class Pack(Layer): ''' Stacks a list of n-dimensional tensors into one (n+1)-dimensional tensor. + + >>> layer = Pack(1) + creating: createPack ''' def __init__(self, dimension, bigdl_type="float"): super(Pack, self).__init__(None, bigdl_type, dimension) -class Shape(Model): - ''' - Given input, return the shape of this input as a 1-D tensor - ''' - - def __init__(self, bigdl_type="float"): - super(Shape, self).__init__(None, bigdl_type) - -class SplitAndSelect(Model): - ''' - First split the tensor along the [[dimension]] into [[numSplit]] sub tensors, - then select the [[index]]th one - ''' - - def __init__(self, dimension, index, num_split, bigdl_type="float"): - super(SplitAndSelect, self).__init__(None, bigdl_type, dimension, index, num_split) - -class StrideSlice(Model): - ''' - Extracts a strided slice from a tensor. - ''' - - def __init__(self): - raise Exception('StrideSlice is not supported in python yet') - - def _test(): import doctest From 577b56cca3040d16e81e8704810124278daac162 Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Wed, 21 Jun 2017 13:10:26 +0800 Subject: [PATCH 076/823] remove tf layers from python api (#1045) --- python/dllib/src/bigdl/dllib/nn/layer.py | 46 +++--------------------- 1 file changed, 4 insertions(+), 42 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index a6509ee1a46..94756d76c29 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -3245,56 +3245,18 @@ def __init__(self, n_input_plane, JTensor.from_ndarray(kernel)) -class Const(Model): - ''' - Return a constant tensor defined by value when forward - ''' - - def __init__(self, value, bigdl_type="float"): - super(Const, self).__init__(None, bigdl_type, JTensor.from_ndarray(value)) -class Fill(Model): - ''' - Return a constant tensor defined by value when forward - ''' - - def __init__(self, value, bigdl_type="float"): - super(Fill, self).__init__(None, bigdl_type, value) - -class Pack(Model): +class Pack(Layer): ''' Stacks a list of n-dimensional tensors into one (n+1)-dimensional tensor. + + >>> layer = Pack(1) + creating: createPack ''' def __init__(self, dimension, bigdl_type="float"): super(Pack, self).__init__(None, bigdl_type, dimension) -class Shape(Model): - ''' - Given input, return the shape of this input as a 1-D tensor - ''' - - def __init__(self, bigdl_type="float"): - super(Shape, self).__init__(None, bigdl_type) - -class SplitAndSelect(Model): - ''' - First split the tensor along the [[dimension]] into [[numSplit]] sub tensors, - then select the [[index]]th one - ''' - - def __init__(self, dimension, index, num_split, bigdl_type="float"): - super(SplitAndSelect, self).__init__(None, bigdl_type, dimension, index, num_split) - -class StrideSlice(Model): - ''' - Extracts a strided slice from a tensor. - ''' - - def __init__(self): - raise Exception('StrideSlice is not supported in python yet') - - def _test(): import doctest From 3e0e08100823031c0e04b5f356651c869e8fe836 Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Wed, 21 Jun 2017 13:10:26 +0800 Subject: [PATCH 077/823] remove tf layers from python api (#1045) --- python/dllib/test/dev/diff.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/dllib/test/dev/diff.py b/python/dllib/test/dev/diff.py index caf4d9054fc..f8b3d65b348 100755 --- a/python/dllib/test/dev/diff.py +++ b/python/dllib/test/dev/diff.py @@ -28,7 +28,8 @@ def extract_scala_class(class_path): - exclude_key_words = set(["*", "abstract"]) + exclude_key_words = set(["*", "abstract", "Const", "Fill", "Shape", + "SplitAndSelect", "StrideSlice"]) include_key_words = set(["Module", "Criterion", "Container", "Cell", "TensorNumeric"]) # noqa content = "\n".join([line for line in open(class_path).readlines() if all([key not in line for key in exclude_key_words])]) # noqa match = re.search(r"class ([\w]+)[^{]+", content) From 69242d345e7167cccf66f5b8d472e4421c7cc8df Mon Sep 17 00:00:00 2001 From: Xin Qiu Date: Wed, 21 Jun 2017 13:22:12 +0800 Subject: [PATCH 078/823] update some doc (#1050) --- python/dllib/src/bigdl/dllib/optim/optimizer.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/optim/optimizer.py b/python/dllib/src/bigdl/dllib/optim/optimizer.py index 90cbe249d66..1155d6b5639 100644 --- a/python/dllib/src/bigdl/dllib/optim/optimizer.py +++ b/python/dllib/src/bigdl/dllib/optim/optimizer.py @@ -363,7 +363,7 @@ def __init__(self, :param model: the neural net model - :param traiing_rdd: the training dataset + :param training_rdd: the training dataset :param criterion: the loss function :param optim_method: the algorithm to use for optimization, e.g. SGD, Adagrad, etc. If optim_method is None, the default algorithm is SGD. @@ -423,7 +423,7 @@ def optimize(self): def set_train_summary(self, summary): """ Set train summary. A TrainSummary object contains information - necesary for the optimizer to know how often the logs are recorded, + necessary for the optimizer to know how often the logs are recorded, where to store the logs and how to retrieve them, etc. For details, refer to the docs of TrainSummary. @@ -437,7 +437,7 @@ def set_train_summary(self, summary): def set_val_summary(self, summary): """ Set validation summary. A ValidationSummary object contains information - necesary for the optimizer to know how often the logs are recorded, + necessary for the optimizer to know how often the logs are recorded, where to store the logs and how to retrieve them, etc. For details, refer to the docs of ValidationSummary. From 05a59a6c139da4e5b174af700a0ba0722270b6cf Mon Sep 17 00:00:00 2001 From: Xin Qiu Date: Wed, 21 Jun 2017 13:22:12 +0800 Subject: [PATCH 079/823] update some doc (#1050) --- python/dllib/src/bigdl/dllib/optim/optimizer.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/optim/optimizer.py b/python/dllib/src/bigdl/dllib/optim/optimizer.py index 90cbe249d66..1155d6b5639 100644 --- a/python/dllib/src/bigdl/dllib/optim/optimizer.py +++ b/python/dllib/src/bigdl/dllib/optim/optimizer.py @@ -363,7 +363,7 @@ def __init__(self, :param model: the neural net model - :param traiing_rdd: the training dataset + :param training_rdd: the training dataset :param criterion: the loss function :param optim_method: the algorithm to use for optimization, e.g. SGD, Adagrad, etc. If optim_method is None, the default algorithm is SGD. @@ -423,7 +423,7 @@ def optimize(self): def set_train_summary(self, summary): """ Set train summary. A TrainSummary object contains information - necesary for the optimizer to know how often the logs are recorded, + necessary for the optimizer to know how often the logs are recorded, where to store the logs and how to retrieve them, etc. For details, refer to the docs of TrainSummary. @@ -437,7 +437,7 @@ def set_train_summary(self, summary): def set_val_summary(self, summary): """ Set validation summary. A ValidationSummary object contains information - necesary for the optimizer to know how often the logs are recorded, + necessary for the optimizer to know how often the logs are recorded, where to store the logs and how to retrieve them, etc. For details, refer to the docs of ValidationSummary. From 262a95019c3dbf55a0bb1599410ea0b7d55644f1 Mon Sep 17 00:00:00 2001 From: Yuan Liu Date: Wed, 21 Jun 2017 13:52:00 +0800 Subject: [PATCH 080/823] fix lenet example mnist typo (#1030) --- python/dllib/src/bigdl/dllib/models/lenet/README.md | 2 +- python/dllib/src/bigdl/dllib/models/lenet/lenet5.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/models/lenet/README.md b/python/dllib/src/bigdl/dllib/models/lenet/README.md index 96f324ff9c6..7472e743ef4 100644 --- a/python/dllib/src/bigdl/dllib/models/lenet/README.md +++ b/python/dllib/src/bigdl/dllib/models/lenet/README.md @@ -5,7 +5,7 @@ please refer to . ## How to run this example: -Program would download the minst data into ```/tmp/mnist``` automatically by default. +Program would download the mnist data into ```/tmp/mnist``` automatically by default. ``` /tmp/mnist$ tree . diff --git a/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py b/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py index b44e64a43e1..04804b61564 100644 --- a/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py +++ b/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py @@ -41,7 +41,7 @@ def build_model(class_num): return model -def get_minst(sc, data_type="train", location="/tmp/mnist"): +def get_mnist(sc, data_type="train", location="/tmp/mnist"): """ Get and normalize the mnist data. We would download it automatically if the data doesn't present at the specific location. @@ -81,9 +81,9 @@ def get_end_trigger(): else: return MaxIteration(options.endTriggerNum) - train_data = get_minst(sc, "train").map( + train_data = get_mnist(sc, "train").map( normalizer(mnist.TRAIN_MEAN, mnist.TRAIN_STD)) - test_data = get_minst(sc, "test").map( + test_data = get_mnist(sc, "test").map( normalizer(mnist.TEST_MEAN, mnist.TEST_STD)) optimizer = Optimizer( From aae0396dfef96ddfe46ae67a9f275c63e6b5dfe6 Mon Sep 17 00:00:00 2001 From: Yuan Liu Date: Wed, 21 Jun 2017 13:52:00 +0800 Subject: [PATCH 081/823] fix lenet example mnist typo (#1030) --- python/dllib/src/bigdl/dllib/models/lenet/README.md | 2 +- python/dllib/src/bigdl/dllib/models/lenet/lenet5.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/models/lenet/README.md b/python/dllib/src/bigdl/dllib/models/lenet/README.md index 96f324ff9c6..7472e743ef4 100644 --- a/python/dllib/src/bigdl/dllib/models/lenet/README.md +++ b/python/dllib/src/bigdl/dllib/models/lenet/README.md @@ -5,7 +5,7 @@ please refer to . ## How to run this example: -Program would download the minst data into ```/tmp/mnist``` automatically by default. +Program would download the mnist data into ```/tmp/mnist``` automatically by default. ``` /tmp/mnist$ tree . diff --git a/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py b/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py index b44e64a43e1..04804b61564 100644 --- a/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py +++ b/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py @@ -41,7 +41,7 @@ def build_model(class_num): return model -def get_minst(sc, data_type="train", location="/tmp/mnist"): +def get_mnist(sc, data_type="train", location="/tmp/mnist"): """ Get and normalize the mnist data. We would download it automatically if the data doesn't present at the specific location. @@ -81,9 +81,9 @@ def get_end_trigger(): else: return MaxIteration(options.endTriggerNum) - train_data = get_minst(sc, "train").map( + train_data = get_mnist(sc, "train").map( normalizer(mnist.TRAIN_MEAN, mnist.TRAIN_STD)) - test_data = get_minst(sc, "test").map( + test_data = get_mnist(sc, "test").map( normalizer(mnist.TEST_MEAN, mnist.TEST_STD)) optimizer = Optimizer( From ec4d73f611fe30ae70b8679b81de2d195fb1b555 Mon Sep 17 00:00:00 2001 From: Yuchao-Tao Date: Thu, 22 Jun 2017 09:24:02 +0800 Subject: [PATCH 082/823] Python regularizer (#1008) * # This is a combination of 5 commits. # The first commit's message is: add Regularizer to PythonBidgDL.scala and optimizer.py # This is the 2nd commit message: update regularizer in layer.py # This is the 3rd commit message: some fix # This is the 4th commit message: some fix 2 # This is the 5th commit message: adjust according to the test * Resolve Conflicts; fix; fix RNN; fix; pass compilation; fix; fix on the missmatch on the parameter list * fix style * fix style * fix style * fix style --- python/dllib/src/bigdl/dllib/nn/layer.py | 189 ++++++++++++------ .../dllib/src/bigdl/dllib/optim/optimizer.py | 36 +++- python/dllib/src/bigdl/dllib/utils/common.py | 4 + 3 files changed, 162 insertions(+), 67 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 94756d76c29..b7ae6e39a76 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -25,16 +25,13 @@ from bigdl.util.common import callJavaFunc from bigdl.util.common import get_spark_context from bigdl.util.common import to_list +from bigdl.util.common import INTMAX, INTMIN, DOUBLEMAX +from bigdl.optim.optimizer import L1Regularizer, L2Regularizer, L1L2Regularizer if sys.version >= '3': long = int unicode = str -INTMAX = 2147483647 -INTMIN = -2147483648 -DOUBLEMAX = 1.7976931348623157E308 - - class Node(JavaValue): """ Represent a node in a graph. The connections between nodes are directed. @@ -428,20 +425,23 @@ class Linear(Layer): an input sample of given batch (the number of rows means the batch size and the number of columns should be equal to the `inputSize`). - - :param input_size: the size the each input sample - :param output_size: the size of the module output of each sample + :param input_size the size the each input sample + :param output_size the size of the module output of each sample :param init_method: two initialized methods are supported here, which are [[Default]]and [[Xavier]], where [[Xavier]] set bias to zero here. For moredetailed information about `initMethod`, please refer to[[InitializationMethod]] + :param wRegularizer: instance of [[Regularizer]](eg. L1 or L2 regularization), applied to the input weights matrices. + :param bRegularizer: instance of [[Regularizer]]applied to the bias. - >>> linear = Linear(100, 10, "Xavier") + >>> linear = Linear(100, 10, "Xavier", True, L1Regularizer(0.5), L1Regularizer(0.5)) + creating: createL1Regularizer + creating: createL1Regularizer creating: createLinear ''' - def __init__(self, input_size, output_size, init_method="default", with_bias=True, + def __init__(self, input_size, output_size, init_method="default", with_bias=True, wRegularizer=None, bRegularizer=None, bigdl_type="float"): super(Linear, self).__init__(None, bigdl_type, input_size, output_size, - init_method, with_bias) + init_method, with_bias, wRegularizer, bRegularizer) def set_init_method(self, weight_init_method = None, bias_init_method = None): callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, @@ -542,19 +542,19 @@ class SpatialConvolution(Layer): The input tensor in forward(input) is expected to be a 3D tensor (nInputPlane x height x width). - - :param n_input_plane: The number of expected input planes in the image given into forward() - :param n_output_plane: The number of output planes the convolution layer will produce. - :param kernel_w: The kernel width of the convolution - :param kernel_h: The kernel height of the convolution - :param stride_w: The step of the convolution in the width dimension. - :param stride_h: The step of the convolution in the height dimension - :param pad_w: The additional zeros added per width to the input planes. - :param pad_h: The additional zeros added per height to the input planes. - :param n_group: Kernel group number - :param propagate_back: Propagate gradient back - :param init_method: Initialization method to initialize bias and weight - + :param n_input_plane The number of expected input planes in the image given into forward() + :param n_output_plane The number of output planes the convolution layer will produce. + :param kernel_w The kernel width of the convolution + :param kernel_h The kernel height of the convolution + :param stride_w The step of the convolution in the width dimension. + :param stride_h The step of the convolution in the height dimension + :param pad_w The additional zeros added per width to the input planes. + :param pad_h The additional zeros added per height to the input planes. + :param n_group Kernel group number + :param propagate_back Propagate gradient back + :param init_method Initialization method to initialize bias and weight + :param wRegularizer: instance of [[Regularizer]](eg. L1 or L2 regularization), applied to the input weights matrices. + :param bRegularizer: instance of [[Regularizer]]applied to the bias. >>> spatialConvolution = SpatialConvolution(6, 12, 5, 5) creating: createSpatialConvolution @@ -572,6 +572,8 @@ def __init__(self, n_group=1, propagate_back=True, init_method="default", + wRegularizer=None, + bRegularizer=None, bigdl_type="float"): super(SpatialConvolution, self).__init__(None, bigdl_type, n_input_plane, @@ -584,7 +586,9 @@ def __init__(self, pad_h, n_group, propagate_back, - init_method) + init_method, + wRegularizer, + bRegularizer) def set_init_method(self, weight_init_method = None, bias_init_method = None): callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, weight_init_method, bias_init_method) @@ -676,14 +680,20 @@ class LSTM(Layer): :param inputSize: the size of each input vector :param hiddenSize: Hidden unit size in the LSTM :param p: is used for [[Dropout]] probability. For more details aboutRNN dropouts, please refer to[RnnDrop: A Novel Dropout for RNNs in ASR](http://www.stat.berkeley.edu/~tsmoon/files/Conference/asru2015.pdf)[A Theoretically Grounded Application of Dropout in Recurrent Neural Networks](https://arxiv.org/pdf/1512.05287.pdf) + :param wRegularizer: instance of [[Regularizer]](eg. L1 or L2 regularization), applied to the input weights matrices. + :param uRegularizer: instance [[Regularizer]](eg. L1 or L2 regularization), applied to the recurrent weights matrices. + :param bRegularizer: instance of [[Regularizer]]applied to the bias. - >>> lstm = LSTM(4, 3, 0.5) + >>> lstm = LSTM(4, 3, 0.5, L1Regularizer(0.5), L1Regularizer(0.5), L1Regularizer(0.5)) + creating: createL1Regularizer + creating: createL1Regularizer + creating: createL1Regularizer creating: createLSTM ''' - def __init__(self, input_size, hidden_size, p=0.0, bigdl_type="float"): - super(LSTM, self).__init__(None, bigdl_type, input_size, hidden_size, p) + def __init__(self, input_size, hidden_size, p=0.0, wRegularizer=None, uRegularizer=None, bRegularizer=None, bigdl_type="float"): + super(LSTM, self).__init__(None, bigdl_type, input_size, hidden_size, p, wRegularizer, uRegularizer, bRegularizer) class LSTMPeephole(Layer): @@ -699,14 +709,19 @@ class LSTMPeephole(Layer): :param input_size: the size of each input vector :param hidden_size: Hidden unit size in the LSTM :param p: is used for [[Dropout]] probability. For more details aboutRNN dropouts, please refer to[RnnDrop: A Novel Dropout for RNNs in ASR](http://www.stat.berkeley.edu/~tsmoon/files/Conference/asru2015.pdf)[A Theoretically Grounded Application of Dropout in Recurrent Neural Networks](https://arxiv.org/pdf/1512.05287.pdf) - - - >>> lstm = LSTMPeephole(4, 3, 0.5) + :param wRegularizer: instance of [[Regularizer]](eg. L1 or L2 regularization), applied to the input weights matrices. + :param uRegularizer: instance [[Regularizer]](eg. L1 or L2 regularization), applied to the recurrent weights matrices. + :param bRegularizer: instance of [[Regularizer]]applied to the bias. + + >>> lstm = LSTMPeephole(4, 3, 0.5, L1Regularizer(0.5), L1Regularizer(0.5), L1Regularizer(0.5)) + creating: createL1Regularizer + creating: createL1Regularizer + creating: createL1Regularizer creating: createLSTMPeephole ''' - def __init__(self, input_size, hidden_size, p=0.0, bigdl_type="float"): - super(LSTMPeephole, self).__init__(None, bigdl_type, input_size, hidden_size, p) + def __init__(self, input_size, hidden_size, p=0.0, wRegularizer=None, uRegularizer=None, bRegularizer=None, bigdl_type="float"): + super(LSTMPeephole, self).__init__(None, bigdl_type, input_size, hidden_size, p, wRegularizer, uRegularizer, bRegularizer) class GRU(Layer): @@ -723,14 +738,21 @@ class GRU(Layer): :param input_size: the size of each input vector :param hidden_size: Hidden unit size in GRU :param p: is used for [[Dropout]] probability. For more details aboutRNN dropouts, please refer to[RnnDrop: A Novel Dropout for RNNs in ASR](http://www.stat.berkeley.edu/~tsmoon/files/Conference/asru2015.pdf)[A Theoretically Grounded Application of Dropout in Recurrent Neural Networks](https://arxiv.org/pdf/1512.05287.pdf) + :param wRegularizer: instance of [[Regularizer]](eg. L1 or L2 regularization), applied to the input weights matrices. + :param uRegularizer: instance [[Regularizer]](eg. L1 or L2 regularization), applied to the recurrent weights matrices. + :param bRegularizer: instance of [[Regularizer]]applied to the bias. + - >>> gru = GRU(4, 3, 0.5) + >>> gru = GRU(4, 3, 0.5, L1Regularizer(0.5), L1Regularizer(0.5), L1Regularizer(0.5)) + creating: createL1Regularizer + creating: createL1Regularizer + creating: createL1Regularizer creating: createGRU ''' - def __init__(self, input_size, hidden_size, p=0.0, bigdl_type="float"): - super(GRU, self).__init__(None, bigdl_type, input_size, hidden_size, p) + def __init__(self, input_size, hidden_size, p=0.0, wRegularizer=None, uRegularizer=None, bRegularizer=None, bigdl_type="float"): + super(GRU, self).__init__(None, bigdl_type, input_size, hidden_size, p, wRegularizer, uRegularizer, bRegularizer) class RnnCell(Layer): @@ -741,10 +763,16 @@ class RnnCell(Layer): :param input_size: the size of each input vector :param hidden_size: Hidden unit size in simple RNN :param activation: activation function + :param wRegularizer: instance of [[Regularizer]](eg. L1 or L2 regularization), applied to the input weights matrices. + :param uRegularizer: instance [[Regularizer]](eg. L1 or L2 regularization), applied to the recurrent weights matrices. + :param bRegularizer: instance of [[Regularizer]](../regularizers.md),applied to the bias. - >>> reshape = RnnCell(4, 3, Tanh()) + >>> reshape = RnnCell(4, 3, Tanh(), L1Regularizer(0.5), L1Regularizer(0.5), L1Regularizer(0.5)) creating: createTanh + creating: createL1Regularizer + creating: createL1Regularizer + creating: createL1Regularizer creating: createRnnCell ''' @@ -752,8 +780,11 @@ def __init__(self, input_size, hidden_size, activation, + wRegularizer=None, + uRegularizer=None, + bRegularizer=None, bigdl_type="float"): - super(RnnCell, self).__init__(None, bigdl_type, input_size, hidden_size, activation) + super(RnnCell, self).__init__(None, bigdl_type, input_size, hidden_size, activation, wRegularizer, uRegularizer, bRegularizer) class TimeDistributed(Layer): @@ -1095,14 +1126,15 @@ class Bilinear(Layer): The input tensor given in forward(input) is a table containing both inputs x_1 and x_2, which are tensors of size N x inputDimension1 and N x inputDimension2, respectively. + :param input_size1 input dimension of x_1 + :param input_size2 input dimension of x_2 + :param output_size output dimension + :param bias_res whether use bias + :param wRegularizer: instance of [[Regularizer]](eg. L1 or L2 regularization), applied to the input weights matrices. + :param bRegularizer: instance of [[Regularizer]]applied to the bias. - :param input_size1: input dimension of x_1 - :param input_size2: input dimension of x_2 - :param output_size: output dimension - :param bias_res: whether use bias - - - >>> bilinear = Bilinear(1, 1, 1, True) + >>> bilinear = Bilinear(1, 1, 1, True, L1Regularizer(0.5)) + creating: createL1Regularizer creating: createBilinear ''' @@ -1111,12 +1143,16 @@ def __init__(self, input_size2, output_size, bias_res=True, + wRegularizer=None, + bRegularizer=None, bigdl_type="float"): super(Bilinear, self).__init__(None, bigdl_type, input_size1, input_size2, output_size, - bias_res) + bias_res, + wRegularizer, + bRegularizer) def set_init_method(self, weight_init_method = None, bias_init_method = None): callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, weight_init_method, bias_init_method) @@ -1739,8 +1775,10 @@ class LookupTable(Layer): ''' a convolution of width 1, commonly used for word embeddings + :param wRegularizer: instance of [[Regularizer]](eg. L1 or L2 regularization), applied to the input weights matrices. - >>> lookupTable = LookupTable(1, 1, 1e-5, 1e-5, 1e-5, True) + >>> lookupTable = LookupTable(1, 1, 1e-5, 1e-5, 1e-5, True, L1Regularizer(0.5)) + creating: createL1Regularizer creating: createLookupTable ''' @@ -1751,6 +1789,7 @@ def __init__(self, max_norm=DOUBLEMAX, norm_type=2.0, should_scale_grad_by_freq=False, + wRegularizer=None, bigdl_type="float"): super(LookupTable, self).__init__(None, bigdl_type, n_index, @@ -1758,7 +1797,8 @@ def __init__(self, padding_value, max_norm, norm_type, - should_scale_grad_by_freq) + should_scale_grad_by_freq, + wRegularizer) def set_init_method(self, weight_init_method = None, bias_init_method = None): callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, weight_init_method, bias_init_method) @@ -2513,6 +2553,8 @@ class SpatialDilatedConvolution(Layer): :param dilation_w: The number of pixels to skip. Default is 1. :param dilation_h: The number of pixels to skip. Default is 1. :param init_method: Init method, Default, Xavier. + :param wRegularizer: instance of [[Regularizer]](eg. L1 or L2 regularization), applied to the input weights matrices. + :param bRegularizer: instance of [[Regularizer]]applied to the bias. >>> spatialDilatedConvolution = SpatialDilatedConvolution(1, 1, 1, 1) @@ -2531,6 +2573,8 @@ def __init__(self, dilation_w=1, dilation_h=1, init_method='default', + wRegularizer=None, + bRegularizer=None, bigdl_type="float"): super(SpatialDilatedConvolution, self).__init__(None, bigdl_type, n_input_plane, @@ -2543,8 +2587,10 @@ def __init__(self, pad_h, dilation_w, dilation_h, - init_method) - + init_method, + wRegularizer, + bRegularizer) + def set_init_method(self, weight_init_method = None, bias_init_method = None): callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, weight_init_method, bias_init_method) @@ -2574,20 +2620,21 @@ class SpatialFullConvolution(Layer): segmentation[C]//Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition. 2015: 3431-3440. - - :param nInputPlane: The number of expected input planes in the image given into forward() - :param nOutputPlane: The number of output planes the convolution layer will produce. - :param kW: The kernel width of the convolution. - :param kH: The kernel height of the convolution. - :param dW: The step of the convolution in the width dimension. Default is 1. - :param dH: The step of the convolution in the height dimension. Default is 1. - :param padW: The additional zeros added per width to the input planes. Default is 0. - :param padH: The additional zeros added per height to the input planes. Default is 0. - :param adjW: Extra width to add to the output image. Default is 0. - :param adjH: Extra height to add to the output image. Default is 0. - :param nGroup: Kernel group number. - :param noBias: If bias is needed. - :param initMethod: Init method, Default, Xavier, Bilinear. + :param nInputPlane The number of expected input planes in the image given into forward() + :param nOutputPlane The number of output planes the convolution layer will produce. + :param kW The kernel width of the convolution. + :param kH The kernel height of the convolution. + :param dW The step of the convolution in the width dimension. Default is 1. + :param dH The step of the convolution in the height dimension. Default is 1. + :param padW The additional zeros added per width to the input planes. Default is 0. + :param padH The additional zeros added per height to the input planes. Default is 0. + :param adjW Extra width to add to the output image. Default is 0. + :param adjH Extra height to add to the output image. Default is 0. + :param nGroup Kernel group number. + :param noBias If bias is needed. + :param initMethod Init method, Default, Xavier, Bilinear. + :param wRegularizer: instance of [[Regularizer]](eg. L1 or L2 regularization), applied to the input weights matrices. + :param bRegularizer: instance of [[Regularizer]]applied to the bias. >>> spatialFullConvolution = SpatialFullConvolution(1, 1, 1, 1) @@ -2608,6 +2655,8 @@ def __init__(self, n_group=1, no_bias=False, init_method='default', + wRegularizer=None, + bRegularizer=None, bigdl_type="float"): super(SpatialFullConvolution, self).__init__(None, bigdl_type, n_input_plane, @@ -2622,7 +2671,9 @@ def __init__(self, adj_h, n_group, no_bias, - init_method) + init_method, + wRegularizer, + bRegularizer) def set_init_method(self, weight_init_method = None, bias_init_method = None): callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, weight_init_method, bias_init_method) @@ -3137,6 +3188,8 @@ class SpatialConvolutionMap(Layer): It uses a generic connection table between input and output features. The SpatialConvolution is equivalent to using a full connection table. + :param wRegularizer: instance of [[Regularizer]](eg. L1 or L2 regularization), applied to the input weights matrices. + :param bRegularizer: instance of [[Regularizer]]applied to the bias. >>> ct = np.ones([9,9]).astype("float32") >>> spatialConvolutionMap = SpatialConvolutionMap(ct, 9, 9) @@ -3151,6 +3204,8 @@ def __init__(self, dh=1, pad_w=0, pad_h=0, + wRegularizer=None, + bRegularizer=None, bigdl_type="float"): super(SpatialConvolutionMap, self).__init__(None, bigdl_type, JTensor.from_ndarray(conn_table), @@ -3159,7 +3214,9 @@ def __init__(self, dw, dh, pad_w, - pad_h) + pad_h, + wRegularizer, + bRegularizer) class SpatialDivisiveNormalization(Layer): diff --git a/python/dllib/src/bigdl/dllib/optim/optimizer.py b/python/dllib/src/bigdl/dllib/optim/optimizer.py index 1155d6b5639..e2bf42c049a 100644 --- a/python/dllib/src/bigdl/dllib/optim/optimizer.py +++ b/python/dllib/src/bigdl/dllib/optim/optimizer.py @@ -19,7 +19,7 @@ import sys from distutils.dir_util import mkpath -from bigdl.nn.layer import DOUBLEMAX +from bigdl.util.common import DOUBLEMAX from bigdl.util.common import JTensor from bigdl.util.common import JavaValue from bigdl.util.common import callBigDlFunc @@ -549,6 +549,40 @@ def read_scalar(self, tag): tag) +class L1L2Regularizer(JavaValue): + """ + Apply both L1 and L2 regularization + + :param l1 l1 regularization rate + :param l2 l2 regularization rate + + """ + def __init__(self, l1, l2, bigdl_type="float"): + JavaValue.__init__(self, None, bigdl_type, l1, l2) + + +class L1Regularizer(JavaValue): + """ + Apply L1 regularization + + :param l1 l1 regularization rate + + """ + def __init__(self, l1, bigdl_type="float"): + JavaValue.__init__(self, None, bigdl_type, l1) + + +class L2Regularizer(JavaValue): + """ + Apply L2 regularization + + :param l2 l2 regularization rate + + """ + def __init__(self, l2, bigdl_type="float"): + JavaValue.__init__(self, None, bigdl_type, l2) + + def _test(): import doctest from pyspark import SparkContext diff --git a/python/dllib/src/bigdl/dllib/utils/common.py b/python/dllib/src/bigdl/dllib/utils/common.py index 06f2c76325b..9e2bb79845e 100644 --- a/python/dllib/src/bigdl/dllib/utils/common.py +++ b/python/dllib/src/bigdl/dllib/utils/common.py @@ -27,6 +27,10 @@ import numpy as np import threading +INTMAX = 2147483647 +INTMIN = -2147483648 +DOUBLEMAX = 1.7976931348623157E308 + if sys.version >= '3': long = int unicode = str From 855f6d23e4fdc205f58d1bad44294e1b4e5b5df4 Mon Sep 17 00:00:00 2001 From: Yuchao-Tao Date: Thu, 22 Jun 2017 09:24:02 +0800 Subject: [PATCH 083/823] Python regularizer (#1008) * # This is a combination of 5 commits. # The first commit's message is: add Regularizer to PythonBidgDL.scala and optimizer.py # This is the 2nd commit message: update regularizer in layer.py # This is the 3rd commit message: some fix # This is the 4th commit message: some fix 2 # This is the 5th commit message: adjust according to the test * Resolve Conflicts; fix; fix RNN; fix; pass compilation; fix; fix on the missmatch on the parameter list * fix style * fix style * fix style * fix style --- python/dllib/src/bigdl/dllib/nn/layer.py | 189 ++++++++++++------ .../dllib/src/bigdl/dllib/optim/optimizer.py | 36 +++- python/dllib/src/bigdl/utils/common.py | 4 + 3 files changed, 162 insertions(+), 67 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 94756d76c29..b7ae6e39a76 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -25,16 +25,13 @@ from bigdl.util.common import callJavaFunc from bigdl.util.common import get_spark_context from bigdl.util.common import to_list +from bigdl.util.common import INTMAX, INTMIN, DOUBLEMAX +from bigdl.optim.optimizer import L1Regularizer, L2Regularizer, L1L2Regularizer if sys.version >= '3': long = int unicode = str -INTMAX = 2147483647 -INTMIN = -2147483648 -DOUBLEMAX = 1.7976931348623157E308 - - class Node(JavaValue): """ Represent a node in a graph. The connections between nodes are directed. @@ -428,20 +425,23 @@ class Linear(Layer): an input sample of given batch (the number of rows means the batch size and the number of columns should be equal to the `inputSize`). - - :param input_size: the size the each input sample - :param output_size: the size of the module output of each sample + :param input_size the size the each input sample + :param output_size the size of the module output of each sample :param init_method: two initialized methods are supported here, which are [[Default]]and [[Xavier]], where [[Xavier]] set bias to zero here. For moredetailed information about `initMethod`, please refer to[[InitializationMethod]] + :param wRegularizer: instance of [[Regularizer]](eg. L1 or L2 regularization), applied to the input weights matrices. + :param bRegularizer: instance of [[Regularizer]]applied to the bias. - >>> linear = Linear(100, 10, "Xavier") + >>> linear = Linear(100, 10, "Xavier", True, L1Regularizer(0.5), L1Regularizer(0.5)) + creating: createL1Regularizer + creating: createL1Regularizer creating: createLinear ''' - def __init__(self, input_size, output_size, init_method="default", with_bias=True, + def __init__(self, input_size, output_size, init_method="default", with_bias=True, wRegularizer=None, bRegularizer=None, bigdl_type="float"): super(Linear, self).__init__(None, bigdl_type, input_size, output_size, - init_method, with_bias) + init_method, with_bias, wRegularizer, bRegularizer) def set_init_method(self, weight_init_method = None, bias_init_method = None): callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, @@ -542,19 +542,19 @@ class SpatialConvolution(Layer): The input tensor in forward(input) is expected to be a 3D tensor (nInputPlane x height x width). - - :param n_input_plane: The number of expected input planes in the image given into forward() - :param n_output_plane: The number of output planes the convolution layer will produce. - :param kernel_w: The kernel width of the convolution - :param kernel_h: The kernel height of the convolution - :param stride_w: The step of the convolution in the width dimension. - :param stride_h: The step of the convolution in the height dimension - :param pad_w: The additional zeros added per width to the input planes. - :param pad_h: The additional zeros added per height to the input planes. - :param n_group: Kernel group number - :param propagate_back: Propagate gradient back - :param init_method: Initialization method to initialize bias and weight - + :param n_input_plane The number of expected input planes in the image given into forward() + :param n_output_plane The number of output planes the convolution layer will produce. + :param kernel_w The kernel width of the convolution + :param kernel_h The kernel height of the convolution + :param stride_w The step of the convolution in the width dimension. + :param stride_h The step of the convolution in the height dimension + :param pad_w The additional zeros added per width to the input planes. + :param pad_h The additional zeros added per height to the input planes. + :param n_group Kernel group number + :param propagate_back Propagate gradient back + :param init_method Initialization method to initialize bias and weight + :param wRegularizer: instance of [[Regularizer]](eg. L1 or L2 regularization), applied to the input weights matrices. + :param bRegularizer: instance of [[Regularizer]]applied to the bias. >>> spatialConvolution = SpatialConvolution(6, 12, 5, 5) creating: createSpatialConvolution @@ -572,6 +572,8 @@ def __init__(self, n_group=1, propagate_back=True, init_method="default", + wRegularizer=None, + bRegularizer=None, bigdl_type="float"): super(SpatialConvolution, self).__init__(None, bigdl_type, n_input_plane, @@ -584,7 +586,9 @@ def __init__(self, pad_h, n_group, propagate_back, - init_method) + init_method, + wRegularizer, + bRegularizer) def set_init_method(self, weight_init_method = None, bias_init_method = None): callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, weight_init_method, bias_init_method) @@ -676,14 +680,20 @@ class LSTM(Layer): :param inputSize: the size of each input vector :param hiddenSize: Hidden unit size in the LSTM :param p: is used for [[Dropout]] probability. For more details aboutRNN dropouts, please refer to[RnnDrop: A Novel Dropout for RNNs in ASR](http://www.stat.berkeley.edu/~tsmoon/files/Conference/asru2015.pdf)[A Theoretically Grounded Application of Dropout in Recurrent Neural Networks](https://arxiv.org/pdf/1512.05287.pdf) + :param wRegularizer: instance of [[Regularizer]](eg. L1 or L2 regularization), applied to the input weights matrices. + :param uRegularizer: instance [[Regularizer]](eg. L1 or L2 regularization), applied to the recurrent weights matrices. + :param bRegularizer: instance of [[Regularizer]]applied to the bias. - >>> lstm = LSTM(4, 3, 0.5) + >>> lstm = LSTM(4, 3, 0.5, L1Regularizer(0.5), L1Regularizer(0.5), L1Regularizer(0.5)) + creating: createL1Regularizer + creating: createL1Regularizer + creating: createL1Regularizer creating: createLSTM ''' - def __init__(self, input_size, hidden_size, p=0.0, bigdl_type="float"): - super(LSTM, self).__init__(None, bigdl_type, input_size, hidden_size, p) + def __init__(self, input_size, hidden_size, p=0.0, wRegularizer=None, uRegularizer=None, bRegularizer=None, bigdl_type="float"): + super(LSTM, self).__init__(None, bigdl_type, input_size, hidden_size, p, wRegularizer, uRegularizer, bRegularizer) class LSTMPeephole(Layer): @@ -699,14 +709,19 @@ class LSTMPeephole(Layer): :param input_size: the size of each input vector :param hidden_size: Hidden unit size in the LSTM :param p: is used for [[Dropout]] probability. For more details aboutRNN dropouts, please refer to[RnnDrop: A Novel Dropout for RNNs in ASR](http://www.stat.berkeley.edu/~tsmoon/files/Conference/asru2015.pdf)[A Theoretically Grounded Application of Dropout in Recurrent Neural Networks](https://arxiv.org/pdf/1512.05287.pdf) - - - >>> lstm = LSTMPeephole(4, 3, 0.5) + :param wRegularizer: instance of [[Regularizer]](eg. L1 or L2 regularization), applied to the input weights matrices. + :param uRegularizer: instance [[Regularizer]](eg. L1 or L2 regularization), applied to the recurrent weights matrices. + :param bRegularizer: instance of [[Regularizer]]applied to the bias. + + >>> lstm = LSTMPeephole(4, 3, 0.5, L1Regularizer(0.5), L1Regularizer(0.5), L1Regularizer(0.5)) + creating: createL1Regularizer + creating: createL1Regularizer + creating: createL1Regularizer creating: createLSTMPeephole ''' - def __init__(self, input_size, hidden_size, p=0.0, bigdl_type="float"): - super(LSTMPeephole, self).__init__(None, bigdl_type, input_size, hidden_size, p) + def __init__(self, input_size, hidden_size, p=0.0, wRegularizer=None, uRegularizer=None, bRegularizer=None, bigdl_type="float"): + super(LSTMPeephole, self).__init__(None, bigdl_type, input_size, hidden_size, p, wRegularizer, uRegularizer, bRegularizer) class GRU(Layer): @@ -723,14 +738,21 @@ class GRU(Layer): :param input_size: the size of each input vector :param hidden_size: Hidden unit size in GRU :param p: is used for [[Dropout]] probability. For more details aboutRNN dropouts, please refer to[RnnDrop: A Novel Dropout for RNNs in ASR](http://www.stat.berkeley.edu/~tsmoon/files/Conference/asru2015.pdf)[A Theoretically Grounded Application of Dropout in Recurrent Neural Networks](https://arxiv.org/pdf/1512.05287.pdf) + :param wRegularizer: instance of [[Regularizer]](eg. L1 or L2 regularization), applied to the input weights matrices. + :param uRegularizer: instance [[Regularizer]](eg. L1 or L2 regularization), applied to the recurrent weights matrices. + :param bRegularizer: instance of [[Regularizer]]applied to the bias. + - >>> gru = GRU(4, 3, 0.5) + >>> gru = GRU(4, 3, 0.5, L1Regularizer(0.5), L1Regularizer(0.5), L1Regularizer(0.5)) + creating: createL1Regularizer + creating: createL1Regularizer + creating: createL1Regularizer creating: createGRU ''' - def __init__(self, input_size, hidden_size, p=0.0, bigdl_type="float"): - super(GRU, self).__init__(None, bigdl_type, input_size, hidden_size, p) + def __init__(self, input_size, hidden_size, p=0.0, wRegularizer=None, uRegularizer=None, bRegularizer=None, bigdl_type="float"): + super(GRU, self).__init__(None, bigdl_type, input_size, hidden_size, p, wRegularizer, uRegularizer, bRegularizer) class RnnCell(Layer): @@ -741,10 +763,16 @@ class RnnCell(Layer): :param input_size: the size of each input vector :param hidden_size: Hidden unit size in simple RNN :param activation: activation function + :param wRegularizer: instance of [[Regularizer]](eg. L1 or L2 regularization), applied to the input weights matrices. + :param uRegularizer: instance [[Regularizer]](eg. L1 or L2 regularization), applied to the recurrent weights matrices. + :param bRegularizer: instance of [[Regularizer]](../regularizers.md),applied to the bias. - >>> reshape = RnnCell(4, 3, Tanh()) + >>> reshape = RnnCell(4, 3, Tanh(), L1Regularizer(0.5), L1Regularizer(0.5), L1Regularizer(0.5)) creating: createTanh + creating: createL1Regularizer + creating: createL1Regularizer + creating: createL1Regularizer creating: createRnnCell ''' @@ -752,8 +780,11 @@ def __init__(self, input_size, hidden_size, activation, + wRegularizer=None, + uRegularizer=None, + bRegularizer=None, bigdl_type="float"): - super(RnnCell, self).__init__(None, bigdl_type, input_size, hidden_size, activation) + super(RnnCell, self).__init__(None, bigdl_type, input_size, hidden_size, activation, wRegularizer, uRegularizer, bRegularizer) class TimeDistributed(Layer): @@ -1095,14 +1126,15 @@ class Bilinear(Layer): The input tensor given in forward(input) is a table containing both inputs x_1 and x_2, which are tensors of size N x inputDimension1 and N x inputDimension2, respectively. + :param input_size1 input dimension of x_1 + :param input_size2 input dimension of x_2 + :param output_size output dimension + :param bias_res whether use bias + :param wRegularizer: instance of [[Regularizer]](eg. L1 or L2 regularization), applied to the input weights matrices. + :param bRegularizer: instance of [[Regularizer]]applied to the bias. - :param input_size1: input dimension of x_1 - :param input_size2: input dimension of x_2 - :param output_size: output dimension - :param bias_res: whether use bias - - - >>> bilinear = Bilinear(1, 1, 1, True) + >>> bilinear = Bilinear(1, 1, 1, True, L1Regularizer(0.5)) + creating: createL1Regularizer creating: createBilinear ''' @@ -1111,12 +1143,16 @@ def __init__(self, input_size2, output_size, bias_res=True, + wRegularizer=None, + bRegularizer=None, bigdl_type="float"): super(Bilinear, self).__init__(None, bigdl_type, input_size1, input_size2, output_size, - bias_res) + bias_res, + wRegularizer, + bRegularizer) def set_init_method(self, weight_init_method = None, bias_init_method = None): callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, weight_init_method, bias_init_method) @@ -1739,8 +1775,10 @@ class LookupTable(Layer): ''' a convolution of width 1, commonly used for word embeddings + :param wRegularizer: instance of [[Regularizer]](eg. L1 or L2 regularization), applied to the input weights matrices. - >>> lookupTable = LookupTable(1, 1, 1e-5, 1e-5, 1e-5, True) + >>> lookupTable = LookupTable(1, 1, 1e-5, 1e-5, 1e-5, True, L1Regularizer(0.5)) + creating: createL1Regularizer creating: createLookupTable ''' @@ -1751,6 +1789,7 @@ def __init__(self, max_norm=DOUBLEMAX, norm_type=2.0, should_scale_grad_by_freq=False, + wRegularizer=None, bigdl_type="float"): super(LookupTable, self).__init__(None, bigdl_type, n_index, @@ -1758,7 +1797,8 @@ def __init__(self, padding_value, max_norm, norm_type, - should_scale_grad_by_freq) + should_scale_grad_by_freq, + wRegularizer) def set_init_method(self, weight_init_method = None, bias_init_method = None): callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, weight_init_method, bias_init_method) @@ -2513,6 +2553,8 @@ class SpatialDilatedConvolution(Layer): :param dilation_w: The number of pixels to skip. Default is 1. :param dilation_h: The number of pixels to skip. Default is 1. :param init_method: Init method, Default, Xavier. + :param wRegularizer: instance of [[Regularizer]](eg. L1 or L2 regularization), applied to the input weights matrices. + :param bRegularizer: instance of [[Regularizer]]applied to the bias. >>> spatialDilatedConvolution = SpatialDilatedConvolution(1, 1, 1, 1) @@ -2531,6 +2573,8 @@ def __init__(self, dilation_w=1, dilation_h=1, init_method='default', + wRegularizer=None, + bRegularizer=None, bigdl_type="float"): super(SpatialDilatedConvolution, self).__init__(None, bigdl_type, n_input_plane, @@ -2543,8 +2587,10 @@ def __init__(self, pad_h, dilation_w, dilation_h, - init_method) - + init_method, + wRegularizer, + bRegularizer) + def set_init_method(self, weight_init_method = None, bias_init_method = None): callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, weight_init_method, bias_init_method) @@ -2574,20 +2620,21 @@ class SpatialFullConvolution(Layer): segmentation[C]//Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition. 2015: 3431-3440. - - :param nInputPlane: The number of expected input planes in the image given into forward() - :param nOutputPlane: The number of output planes the convolution layer will produce. - :param kW: The kernel width of the convolution. - :param kH: The kernel height of the convolution. - :param dW: The step of the convolution in the width dimension. Default is 1. - :param dH: The step of the convolution in the height dimension. Default is 1. - :param padW: The additional zeros added per width to the input planes. Default is 0. - :param padH: The additional zeros added per height to the input planes. Default is 0. - :param adjW: Extra width to add to the output image. Default is 0. - :param adjH: Extra height to add to the output image. Default is 0. - :param nGroup: Kernel group number. - :param noBias: If bias is needed. - :param initMethod: Init method, Default, Xavier, Bilinear. + :param nInputPlane The number of expected input planes in the image given into forward() + :param nOutputPlane The number of output planes the convolution layer will produce. + :param kW The kernel width of the convolution. + :param kH The kernel height of the convolution. + :param dW The step of the convolution in the width dimension. Default is 1. + :param dH The step of the convolution in the height dimension. Default is 1. + :param padW The additional zeros added per width to the input planes. Default is 0. + :param padH The additional zeros added per height to the input planes. Default is 0. + :param adjW Extra width to add to the output image. Default is 0. + :param adjH Extra height to add to the output image. Default is 0. + :param nGroup Kernel group number. + :param noBias If bias is needed. + :param initMethod Init method, Default, Xavier, Bilinear. + :param wRegularizer: instance of [[Regularizer]](eg. L1 or L2 regularization), applied to the input weights matrices. + :param bRegularizer: instance of [[Regularizer]]applied to the bias. >>> spatialFullConvolution = SpatialFullConvolution(1, 1, 1, 1) @@ -2608,6 +2655,8 @@ def __init__(self, n_group=1, no_bias=False, init_method='default', + wRegularizer=None, + bRegularizer=None, bigdl_type="float"): super(SpatialFullConvolution, self).__init__(None, bigdl_type, n_input_plane, @@ -2622,7 +2671,9 @@ def __init__(self, adj_h, n_group, no_bias, - init_method) + init_method, + wRegularizer, + bRegularizer) def set_init_method(self, weight_init_method = None, bias_init_method = None): callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, weight_init_method, bias_init_method) @@ -3137,6 +3188,8 @@ class SpatialConvolutionMap(Layer): It uses a generic connection table between input and output features. The SpatialConvolution is equivalent to using a full connection table. + :param wRegularizer: instance of [[Regularizer]](eg. L1 or L2 regularization), applied to the input weights matrices. + :param bRegularizer: instance of [[Regularizer]]applied to the bias. >>> ct = np.ones([9,9]).astype("float32") >>> spatialConvolutionMap = SpatialConvolutionMap(ct, 9, 9) @@ -3151,6 +3204,8 @@ def __init__(self, dh=1, pad_w=0, pad_h=0, + wRegularizer=None, + bRegularizer=None, bigdl_type="float"): super(SpatialConvolutionMap, self).__init__(None, bigdl_type, JTensor.from_ndarray(conn_table), @@ -3159,7 +3214,9 @@ def __init__(self, dw, dh, pad_w, - pad_h) + pad_h, + wRegularizer, + bRegularizer) class SpatialDivisiveNormalization(Layer): diff --git a/python/dllib/src/bigdl/dllib/optim/optimizer.py b/python/dllib/src/bigdl/dllib/optim/optimizer.py index 1155d6b5639..e2bf42c049a 100644 --- a/python/dllib/src/bigdl/dllib/optim/optimizer.py +++ b/python/dllib/src/bigdl/dllib/optim/optimizer.py @@ -19,7 +19,7 @@ import sys from distutils.dir_util import mkpath -from bigdl.nn.layer import DOUBLEMAX +from bigdl.util.common import DOUBLEMAX from bigdl.util.common import JTensor from bigdl.util.common import JavaValue from bigdl.util.common import callBigDlFunc @@ -549,6 +549,40 @@ def read_scalar(self, tag): tag) +class L1L2Regularizer(JavaValue): + """ + Apply both L1 and L2 regularization + + :param l1 l1 regularization rate + :param l2 l2 regularization rate + + """ + def __init__(self, l1, l2, bigdl_type="float"): + JavaValue.__init__(self, None, bigdl_type, l1, l2) + + +class L1Regularizer(JavaValue): + """ + Apply L1 regularization + + :param l1 l1 regularization rate + + """ + def __init__(self, l1, bigdl_type="float"): + JavaValue.__init__(self, None, bigdl_type, l1) + + +class L2Regularizer(JavaValue): + """ + Apply L2 regularization + + :param l2 l2 regularization rate + + """ + def __init__(self, l2, bigdl_type="float"): + JavaValue.__init__(self, None, bigdl_type, l2) + + def _test(): import doctest from pyspark import SparkContext diff --git a/python/dllib/src/bigdl/utils/common.py b/python/dllib/src/bigdl/utils/common.py index 06f2c76325b..9e2bb79845e 100644 --- a/python/dllib/src/bigdl/utils/common.py +++ b/python/dllib/src/bigdl/utils/common.py @@ -27,6 +27,10 @@ import numpy as np import threading +INTMAX = 2147483647 +INTMIN = -2147483648 +DOUBLEMAX = 1.7976931348623157E308 + if sys.version >= '3': long = int unicode = str From 1fb2b2a87625f81fb7d5bf4b79059dc69eb167c7 Mon Sep 17 00:00:00 2001 From: Xianyan Date: Thu, 22 Jun 2017 11:50:07 +0800 Subject: [PATCH 084/823] [Issue 1035] Allow layer regulazier set after model definition, add regulazier for cadd and cmul (#1048) * Allow layer regulazier set after model definition * add regulazier support for cadd and cmul * meet code review * add python set API, add regularizer to python cmul and cadd * add test for setBRegularizer * add parameters to create method, add doc --- python/dllib/src/bigdl/dllib/nn/layer.py | 28 +++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index b7ae6e39a76..ce3bf37636e 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -323,6 +323,22 @@ def save(self, path, over_write = False): callBigDlFunc(self.bigdl_type, "modelSave", self.value, path, over_write) + def setWRegularizer(self, wRegularizer): + ''' + set weight regularizer + :param wRegularizer: weight regularizer + :return: + ''' + self.value.wRegularizer = wRegularizer.value + + def setBRegularizer(self, bRegularizer): + ''' + set bias regularizer + :param wRegularizer: bias regularizer + :return: + ''' + self.value.bRegularizer = bRegularizer.value + class Container(Layer): ''' @@ -558,6 +574,10 @@ class SpatialConvolution(Layer): >>> spatialConvolution = SpatialConvolution(6, 12, 5, 5) creating: createSpatialConvolution + >>> spatialConvolution.setWRegularizer(L1Regularizer(0.5)) + creating: createL1Regularizer + >>> spatialConvolution.setBRegularizer(L1Regularizer(0.5)) + creating: createL1Regularizer ''' def __init__(self, @@ -1197,6 +1217,7 @@ class CAdd(Layer): :param size: the size of the bias + :param bRegularizer: instance of [[Regularizer]]applied to the bias. >>> cAdd = CAdd([1,2]) @@ -1204,10 +1225,10 @@ class CAdd(Layer): ''' def __init__(self, - size, + size, bRegularizer=None, bigdl_type="float"): super(CAdd, self).__init__(None, bigdl_type, - size) + size, bRegularizer) def set_init_method(self, weight_init_method = None, bias_init_method = None): callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, @@ -1294,9 +1315,10 @@ class CMul(Layer): def __init__(self, size, + wRegularizer=None, bigdl_type="float"): super(CMul, self).__init__(None, bigdl_type, - size) + size, wRegularizer) def set_init_method(self, weight_init_method = None, bias_init_method = None): callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, From 8b47f5ce3ddb7cda4c50fce537178e695214416c Mon Sep 17 00:00:00 2001 From: Xianyan Date: Thu, 22 Jun 2017 11:50:07 +0800 Subject: [PATCH 085/823] [Issue 1035] Allow layer regulazier set after model definition, add regulazier for cadd and cmul (#1048) * Allow layer regulazier set after model definition * add regulazier support for cadd and cmul * meet code review * add python set API, add regularizer to python cmul and cadd * add test for setBRegularizer * add parameters to create method, add doc --- python/dllib/src/bigdl/dllib/nn/layer.py | 28 +++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index b7ae6e39a76..ce3bf37636e 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -323,6 +323,22 @@ def save(self, path, over_write = False): callBigDlFunc(self.bigdl_type, "modelSave", self.value, path, over_write) + def setWRegularizer(self, wRegularizer): + ''' + set weight regularizer + :param wRegularizer: weight regularizer + :return: + ''' + self.value.wRegularizer = wRegularizer.value + + def setBRegularizer(self, bRegularizer): + ''' + set bias regularizer + :param wRegularizer: bias regularizer + :return: + ''' + self.value.bRegularizer = bRegularizer.value + class Container(Layer): ''' @@ -558,6 +574,10 @@ class SpatialConvolution(Layer): >>> spatialConvolution = SpatialConvolution(6, 12, 5, 5) creating: createSpatialConvolution + >>> spatialConvolution.setWRegularizer(L1Regularizer(0.5)) + creating: createL1Regularizer + >>> spatialConvolution.setBRegularizer(L1Regularizer(0.5)) + creating: createL1Regularizer ''' def __init__(self, @@ -1197,6 +1217,7 @@ class CAdd(Layer): :param size: the size of the bias + :param bRegularizer: instance of [[Regularizer]]applied to the bias. >>> cAdd = CAdd([1,2]) @@ -1204,10 +1225,10 @@ class CAdd(Layer): ''' def __init__(self, - size, + size, bRegularizer=None, bigdl_type="float"): super(CAdd, self).__init__(None, bigdl_type, - size) + size, bRegularizer) def set_init_method(self, weight_init_method = None, bias_init_method = None): callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, @@ -1294,9 +1315,10 @@ class CMul(Layer): def __init__(self, size, + wRegularizer=None, bigdl_type="float"): super(CMul, self).__init__(None, bigdl_type, - size) + size, wRegularizer) def set_init_method(self, weight_init_method = None, bias_init_method = None): callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, From 96239a535a58a4ff86c4dd8ac90fbad9ad77098d Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Fri, 23 Jun 2017 13:38:59 +0800 Subject: [PATCH 086/823] Fix type conversion error for RoiPooling.forward (#1078) * fix type * add forward * clean * add test back --- python/dllib/src/bigdl/dllib/nn/layer.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index ce3bf37636e..58287076c2c 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -2367,8 +2367,12 @@ class RoiPooling(Layer): :param spatial_scale: spatial scale - >>> roiPooling = RoiPooling(1, 1, 1e-5) + >>> import numpy as np + >>> input_data = np.random.rand(2,2,6,8) + >>> input_rois = np.array([0, 0, 0, 7, 5, 1, 6, 2, 7, 5, 1, 3, 1, 6, 4, 0, 3, 3, 3, 3],dtype='float64').reshape(4,5) + >>> m = RoiPooling(3,2,1.0) creating: createRoiPooling + >>> out = m.forward([input_data,input_rois]) ''' def __init__(self, From 2816e6f4bce8e179bd3fcda13691de564ffd92f1 Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Fri, 23 Jun 2017 13:38:59 +0800 Subject: [PATCH 087/823] Fix type conversion error for RoiPooling.forward (#1078) * fix type * add forward * clean * add test back --- python/dllib/src/bigdl/dllib/nn/layer.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index ce3bf37636e..58287076c2c 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -2367,8 +2367,12 @@ class RoiPooling(Layer): :param spatial_scale: spatial scale - >>> roiPooling = RoiPooling(1, 1, 1e-5) + >>> import numpy as np + >>> input_data = np.random.rand(2,2,6,8) + >>> input_rois = np.array([0, 0, 0, 7, 5, 1, 6, 2, 7, 5, 1, 3, 1, 6, 4, 0, 3, 3, 3, 3],dtype='float64').reshape(4,5) + >>> m = RoiPooling(3,2,1.0) creating: createRoiPooling + >>> out = m.forward([input_data,input_rois]) ''' def __init__(self, From 1fa755203a860dfac481a9735f1d905d35c0ddb5 Mon Sep 17 00:00:00 2001 From: zhangxiaoli73 <380761639@qq.com> Date: Fri, 23 Jun 2017 16:52:47 +0800 Subject: [PATCH 088/823] add L1HingeEmbeddingCriterion unit test for python (#1073) --- python/dllib/src/bigdl/dllib/nn/criterion.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/nn/criterion.py b/python/dllib/src/bigdl/dllib/nn/criterion.py index e740c1cd82f..218c23c5f8a 100644 --- a/python/dllib/src/bigdl/dllib/nn/criterion.py +++ b/python/dllib/src/bigdl/dllib/nn/criterion.py @@ -281,6 +281,13 @@ class L1HingeEmbeddingCriterion(Criterion): creating: createL1HingeEmbeddingCriterion >>> l1HingeEmbeddingCriterion = L1HingeEmbeddingCriterion() creating: createL1HingeEmbeddingCriterion + >>> input1 = np.array([2.1, -2.2]) + >>> input2 = np.array([-0.55, 0.298]) + >>> input = [input1, input2] + >>> target = np.array([1.0]) + >>> result = l1HingeEmbeddingCriterion.forward(input, target) + >>> (result == 5.148) + True ''' def __init__(self, From 034e241725494b7ed698bf0ae24168557126b9f3 Mon Sep 17 00:00:00 2001 From: zhangxiaoli73 <380761639@qq.com> Date: Fri, 23 Jun 2017 16:52:47 +0800 Subject: [PATCH 089/823] add L1HingeEmbeddingCriterion unit test for python (#1073) --- python/dllib/src/bigdl/dllib/nn/criterion.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/nn/criterion.py b/python/dllib/src/bigdl/dllib/nn/criterion.py index e740c1cd82f..218c23c5f8a 100644 --- a/python/dllib/src/bigdl/dllib/nn/criterion.py +++ b/python/dllib/src/bigdl/dllib/nn/criterion.py @@ -281,6 +281,13 @@ class L1HingeEmbeddingCriterion(Criterion): creating: createL1HingeEmbeddingCriterion >>> l1HingeEmbeddingCriterion = L1HingeEmbeddingCriterion() creating: createL1HingeEmbeddingCriterion + >>> input1 = np.array([2.1, -2.2]) + >>> input2 = np.array([-0.55, 0.298]) + >>> input = [input1, input2] + >>> target = np.array([1.0]) + >>> result = l1HingeEmbeddingCriterion.forward(input, target) + >>> (result == 5.148) + True ''' def __init__(self, From 5e78a5f422be8f862701931908b0f007e9a7505c Mon Sep 17 00:00:00 2001 From: Yao Zhang Date: Mon, 26 Jun 2017 14:07:58 +0800 Subject: [PATCH 090/823] Polish InferReshape documentation (#1087) * Polish InferReshape documentation * remove linearSpec unnecessary change * fix core conflict * meet code review --- python/dllib/src/bigdl/dllib/nn/layer.py | 36 ++++++++++++++---------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 58287076c2c..5e35024e6de 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -1647,21 +1647,27 @@ def __init__(self, class InferReshape(Layer): ''' - Reshape with the support of infered size, - Positive numbers are used directly, setting the corresponding dimension of the output tensor. - In addition, two special values are accepted: - 0 means "copy the respective dimension of the input". - i.e., if the input has 2 as its 1st dimension, - the output will have 2 as its 1st dimension as well - -1 stands for "infer this from the other dimensions" - this dimension is calculated to keep the overall element count the same as in the input. - At most one -1 can be used in a reshape operation. - For example, (4, 5, 6, 7) -> InferReshape (4, 0, 3, -1) -> (4, 5, 3, 14) - with 1st and 3rd dim same as given size, with 2nd dim same as input, and the infered dim is 14 - - - :param size: the target tensor size - :param batch_mode: whether in batch mode + Reshape the input tensor with automatic size inference support. + Positive numbers in the `size` argument are used to reshape the input to the + corresponding dimension size. + There are also two special values allowed in `size`: + a. `0` means keep the corresponding dimension size of the input unchanged. + i.e., if the 1st dimension size of the input is 2, + the 1st dimension size of output will be set as 2 as well. + b. `-1` means infer this dimension size from other dimensions. + This dimension size is calculated by keeping the amount of output elements + consistent with the input. + Only one `-1` is allowable in `size`. + + For example, + Input tensor with size: (4, 5, 6, 7) + -> InferReshape(Array(4, 0, 3, -1)) + Output tensor with size: (4, 5, 3, 14) + The 1st and 3rd dim are set to given sizes, keep the 2nd dim unchanged, + and inferred the last dim as 14. + + :param size: the target tensor size + :param batch_mode: whether in batch mode >>> inferReshape = InferReshape([4, 0, 3, -1], False) From fd7017772c38170b5cc68faac513338b4f00540e Mon Sep 17 00:00:00 2001 From: Yao Zhang Date: Mon, 26 Jun 2017 14:07:58 +0800 Subject: [PATCH 091/823] Polish InferReshape documentation (#1087) * Polish InferReshape documentation * remove linearSpec unnecessary change * fix core conflict * meet code review --- python/dllib/src/bigdl/dllib/nn/layer.py | 36 ++++++++++++++---------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 58287076c2c..5e35024e6de 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -1647,21 +1647,27 @@ def __init__(self, class InferReshape(Layer): ''' - Reshape with the support of infered size, - Positive numbers are used directly, setting the corresponding dimension of the output tensor. - In addition, two special values are accepted: - 0 means "copy the respective dimension of the input". - i.e., if the input has 2 as its 1st dimension, - the output will have 2 as its 1st dimension as well - -1 stands for "infer this from the other dimensions" - this dimension is calculated to keep the overall element count the same as in the input. - At most one -1 can be used in a reshape operation. - For example, (4, 5, 6, 7) -> InferReshape (4, 0, 3, -1) -> (4, 5, 3, 14) - with 1st and 3rd dim same as given size, with 2nd dim same as input, and the infered dim is 14 - - - :param size: the target tensor size - :param batch_mode: whether in batch mode + Reshape the input tensor with automatic size inference support. + Positive numbers in the `size` argument are used to reshape the input to the + corresponding dimension size. + There are also two special values allowed in `size`: + a. `0` means keep the corresponding dimension size of the input unchanged. + i.e., if the 1st dimension size of the input is 2, + the 1st dimension size of output will be set as 2 as well. + b. `-1` means infer this dimension size from other dimensions. + This dimension size is calculated by keeping the amount of output elements + consistent with the input. + Only one `-1` is allowable in `size`. + + For example, + Input tensor with size: (4, 5, 6, 7) + -> InferReshape(Array(4, 0, 3, -1)) + Output tensor with size: (4, 5, 3, 14) + The 1st and 3rd dim are set to given sizes, keep the 2nd dim unchanged, + and inferred the last dim as 14. + + :param size: the target tensor size + :param batch_mode: whether in batch mode >>> inferReshape = InferReshape([4, 0, 3, -1], False) From a52ecd66603c8d015fa7a56fec8b4ff47f2538ab Mon Sep 17 00:00:00 2001 From: Xianyan Date: Wed, 28 Jun 2017 13:44:44 +0800 Subject: [PATCH 092/823] add Exponential learning rate decay (#1103) * add Exponential learning rate decay * fix unit test --- python/dllib/src/bigdl/dllib/optim/optimizer.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/optim/optimizer.py b/python/dllib/src/bigdl/dllib/optim/optimizer.py index e2bf42c049a..436f21b44b1 100644 --- a/python/dllib/src/bigdl/dllib/optim/optimizer.py +++ b/python/dllib/src/bigdl/dllib/optim/optimizer.py @@ -135,6 +135,22 @@ def __init__(self, power, max_iteration, bigdl_type="float"): JavaValue.__init__(self, None, bigdl_type, power, max_iteration) +class Exponential(JavaValue): + """ + [[Exponential]] is a learning rate schedule, which rescale the learning rate by + lr_{n + 1} = lr * decayRate `^` (iter / decayStep) + :param decay_step the inteval for lr decay + :param decay_rate decay rate + :param stair_case if true, iter / decayStep is an integer division + and the decayed learning rate follows a staircase function. + + >>> exponential = Exponential(100, 0.1) + creating: createExponential + """ + def __init__(self, decay_step, decay_rate, stair_case=False, bigdl_type="float"): + JavaValue.__init__(self, None, bigdl_type, decay_step, decay_rate, stair_case) + + class Step(JavaValue): """ A learning rate decay policy, where the effective learning rate is From 5f6880836221c9273a842fa633b3a8b656eaac0a Mon Sep 17 00:00:00 2001 From: Xianyan Date: Wed, 28 Jun 2017 13:44:44 +0800 Subject: [PATCH 093/823] add Exponential learning rate decay (#1103) * add Exponential learning rate decay * fix unit test --- python/dllib/src/bigdl/dllib/optim/optimizer.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/optim/optimizer.py b/python/dllib/src/bigdl/dllib/optim/optimizer.py index e2bf42c049a..436f21b44b1 100644 --- a/python/dllib/src/bigdl/dllib/optim/optimizer.py +++ b/python/dllib/src/bigdl/dllib/optim/optimizer.py @@ -135,6 +135,22 @@ def __init__(self, power, max_iteration, bigdl_type="float"): JavaValue.__init__(self, None, bigdl_type, power, max_iteration) +class Exponential(JavaValue): + """ + [[Exponential]] is a learning rate schedule, which rescale the learning rate by + lr_{n + 1} = lr * decayRate `^` (iter / decayStep) + :param decay_step the inteval for lr decay + :param decay_rate decay rate + :param stair_case if true, iter / decayStep is an integer division + and the decayed learning rate follows a staircase function. + + >>> exponential = Exponential(100, 0.1) + creating: createExponential + """ + def __init__(self, decay_step, decay_rate, stair_case=False, bigdl_type="float"): + JavaValue.__init__(self, None, bigdl_type, decay_step, decay_rate, stair_case) + + class Step(JavaValue): """ A learning rate decay policy, where the effective learning rate is From ca2dfd4d05074a556e586131cde1a4bfc25b07f7 Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Thu, 29 Jun 2017 13:53:31 +0800 Subject: [PATCH 094/823] support pip install (#1113) --- .../src/bigdl/dllib/models/lenet/lenet5.py | 2 +- python/dllib/src/bigdl/dllib/utils/common.py | 18 +++--- python/dllib/src/bigdl/dllib/utils/engine.py | 62 +++++++++++++++++++ python/dllib/test/__init__.py | 3 + version.py | 17 +++++ 5 files changed, 91 insertions(+), 11 deletions(-) create mode 100644 python/dllib/src/bigdl/dllib/utils/engine.py create mode 100644 version.py diff --git a/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py b/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py index 04804b61564..57994858672 100644 --- a/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py +++ b/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py @@ -104,7 +104,7 @@ def get_end_trigger(): parameters = trained_model.parameters() elif options.action == "test": # Load a pre-trained model and then validate it through top1 accuracy. - test_data = get_minst(sc, "test").map( + test_data = get_mnist(sc, "test").map( normalizer(mnist.TEST_MEAN, mnist.TEST_STD)) model = Model.load(options.modelPath) results = model.test(test_data, options.batchSize, ["Top1Accuracy"]) diff --git a/python/dllib/src/bigdl/dllib/utils/common.py b/python/dllib/src/bigdl/dllib/utils/common.py index 9e2bb79845e..00b40feaa5b 100644 --- a/python/dllib/src/bigdl/dllib/utils/common.py +++ b/python/dllib/src/bigdl/dllib/utils/common.py @@ -14,7 +14,9 @@ # limitations under the License. # +import os import sys +import glob from py4j.protocol import Py4JJavaError from py4j.java_gateway import JavaObject from py4j.java_collections import ListConverter, JavaArray, JavaList, JavaMap @@ -26,6 +28,7 @@ from pyspark import SparkConf import numpy as np import threading +from bigdl.util.engine import prepare_env INTMAX = 2147483647 INTMIN = -2147483648 @@ -162,7 +165,7 @@ def flatten_ndarray(cls, a_ndarray): Utility method to flatten a ndarray :return: (storage, shape) - + >>> from bigdl.util.common import JTensor >>> np.random.seed(123) >>> data = np.random.uniform(0, 1, (2, 3)) @@ -287,15 +290,10 @@ def get_spark_context(conf = None): :param conf: combining bigdl configs into spark conf :return: SparkContext """ - if not conf: - conf = create_spark_conf() - if "getOrCreate" in SparkContext.__dict__: - return SparkContext.getOrCreate(conf) - else: - with SparkContext._lock: # Compatible with Spark1.5.1 - if SparkContext._active_spark_context is None: - SparkContext(conf) - return SparkContext._active_spark_context + with SparkContext._lock: # Compatible with Spark1.5.1 + if SparkContext._active_spark_context is None: + SparkContext(conf=conf or create_spark_conf()) + return SparkContext._active_spark_context def get_spark_sql_context(sc): diff --git a/python/dllib/src/bigdl/dllib/utils/engine.py b/python/dllib/src/bigdl/dllib/utils/engine.py new file mode 100644 index 00000000000..42a2455f378 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/utils/engine.py @@ -0,0 +1,62 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import sys +import os +import glob + +def __prepare_spark_env(): + modules = sys.modules + if "pyspark" not in modules or "py4j" not in modules: + spark_home = os.environ.get('SPARK_HOME', None) + if not spark_home: + raise ValueError( + """Could not find Spark. Pls make sure SPARK_HOME env is set: + export SPARK_HOME=path to your spark home directory""") + py4j = glob.glob(os.path.join(spark_home, 'python/lib', 'py4j-*.zip'))[0] + pyspark = glob.glob(os.path.join(spark_home, 'python/lib', 'pyspark*.zip'))[0] + sys.path.insert(0, py4j) + sys.path.insert(0, pyspark) + + +def __prepare_bigdl_env(): + import bigdl.nn.layer + jar_dir = os.path.abspath(bigdl.nn.layer.__file__ + "/../../") + jar_paths = glob.glob(os.path.join(jar_dir, "share/lib/*.jar")) + conf_paths = glob.glob(os.path.join(jar_dir, "share/conf/*.conf")) + + def append_path(env_var_name, path): + try: + os.environ[env_var_name] = path + ":" + os.environ[ + env_var_name] # noqa + except KeyError: + os.environ[env_var_name] = path + + if conf_paths and conf_paths: + assert len(conf_paths) == 1, "Expecting one jar: %s" % len(jar_paths) + assert len(conf_paths) == 1, "Expecting one conf: %s" % len(conf_paths) + print("Adding %s to spark.driver.extraClassPath" % jar_paths[0]) + print("Adding %s to spark.executor.extraClassPath" % jar_paths[0]) + append_path("spark.driver.extraClassPath", jar_paths[0]) + append_path("spark.executor.extraClassPath", jar_paths[0]) + append_path("SPARK_CLASSPATH", jar_paths[0]) + print("Prepending %s to sys.path" % conf_paths[0]) + sys.path.insert(0, conf_paths[0]) + + +def prepare_env(): + __prepare_spark_env() + __prepare_bigdl_env() \ No newline at end of file diff --git a/python/dllib/test/__init__.py b/python/dllib/test/__init__.py index 2151a805423..63e9a94acc5 100644 --- a/python/dllib/test/__init__.py +++ b/python/dllib/test/__init__.py @@ -13,3 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. # + +from bigdl.util.engine import prepare_env +prepare_env() diff --git a/version.py b/version.py new file mode 100644 index 00000000000..ddff350988c --- /dev/null +++ b/version.py @@ -0,0 +1,17 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +__version__ = "0.2.0.dev2" \ No newline at end of file From 4e45ca3aa9c55f0e47c7f28ac6fef9860e4abb2a Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Thu, 29 Jun 2017 13:53:31 +0800 Subject: [PATCH 095/823] support pip install (#1113) --- .../src/bigdl/dllib/models/lenet/lenet5.py | 2 +- python/dllib/src/bigdl/utils/common.py | 18 +++--- python/dllib/src/bigdl/utils/engine.py | 62 +++++++++++++++++++ python/dllib/test/__init__.py | 3 + version.py | 17 +++++ 5 files changed, 91 insertions(+), 11 deletions(-) create mode 100644 python/dllib/src/bigdl/utils/engine.py create mode 100644 version.py diff --git a/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py b/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py index 04804b61564..57994858672 100644 --- a/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py +++ b/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py @@ -104,7 +104,7 @@ def get_end_trigger(): parameters = trained_model.parameters() elif options.action == "test": # Load a pre-trained model and then validate it through top1 accuracy. - test_data = get_minst(sc, "test").map( + test_data = get_mnist(sc, "test").map( normalizer(mnist.TEST_MEAN, mnist.TEST_STD)) model = Model.load(options.modelPath) results = model.test(test_data, options.batchSize, ["Top1Accuracy"]) diff --git a/python/dllib/src/bigdl/utils/common.py b/python/dllib/src/bigdl/utils/common.py index 9e2bb79845e..00b40feaa5b 100644 --- a/python/dllib/src/bigdl/utils/common.py +++ b/python/dllib/src/bigdl/utils/common.py @@ -14,7 +14,9 @@ # limitations under the License. # +import os import sys +import glob from py4j.protocol import Py4JJavaError from py4j.java_gateway import JavaObject from py4j.java_collections import ListConverter, JavaArray, JavaList, JavaMap @@ -26,6 +28,7 @@ from pyspark import SparkConf import numpy as np import threading +from bigdl.util.engine import prepare_env INTMAX = 2147483647 INTMIN = -2147483648 @@ -162,7 +165,7 @@ def flatten_ndarray(cls, a_ndarray): Utility method to flatten a ndarray :return: (storage, shape) - + >>> from bigdl.util.common import JTensor >>> np.random.seed(123) >>> data = np.random.uniform(0, 1, (2, 3)) @@ -287,15 +290,10 @@ def get_spark_context(conf = None): :param conf: combining bigdl configs into spark conf :return: SparkContext """ - if not conf: - conf = create_spark_conf() - if "getOrCreate" in SparkContext.__dict__: - return SparkContext.getOrCreate(conf) - else: - with SparkContext._lock: # Compatible with Spark1.5.1 - if SparkContext._active_spark_context is None: - SparkContext(conf) - return SparkContext._active_spark_context + with SparkContext._lock: # Compatible with Spark1.5.1 + if SparkContext._active_spark_context is None: + SparkContext(conf=conf or create_spark_conf()) + return SparkContext._active_spark_context def get_spark_sql_context(sc): diff --git a/python/dllib/src/bigdl/utils/engine.py b/python/dllib/src/bigdl/utils/engine.py new file mode 100644 index 00000000000..42a2455f378 --- /dev/null +++ b/python/dllib/src/bigdl/utils/engine.py @@ -0,0 +1,62 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import sys +import os +import glob + +def __prepare_spark_env(): + modules = sys.modules + if "pyspark" not in modules or "py4j" not in modules: + spark_home = os.environ.get('SPARK_HOME', None) + if not spark_home: + raise ValueError( + """Could not find Spark. Pls make sure SPARK_HOME env is set: + export SPARK_HOME=path to your spark home directory""") + py4j = glob.glob(os.path.join(spark_home, 'python/lib', 'py4j-*.zip'))[0] + pyspark = glob.glob(os.path.join(spark_home, 'python/lib', 'pyspark*.zip'))[0] + sys.path.insert(0, py4j) + sys.path.insert(0, pyspark) + + +def __prepare_bigdl_env(): + import bigdl.nn.layer + jar_dir = os.path.abspath(bigdl.nn.layer.__file__ + "/../../") + jar_paths = glob.glob(os.path.join(jar_dir, "share/lib/*.jar")) + conf_paths = glob.glob(os.path.join(jar_dir, "share/conf/*.conf")) + + def append_path(env_var_name, path): + try: + os.environ[env_var_name] = path + ":" + os.environ[ + env_var_name] # noqa + except KeyError: + os.environ[env_var_name] = path + + if conf_paths and conf_paths: + assert len(conf_paths) == 1, "Expecting one jar: %s" % len(jar_paths) + assert len(conf_paths) == 1, "Expecting one conf: %s" % len(conf_paths) + print("Adding %s to spark.driver.extraClassPath" % jar_paths[0]) + print("Adding %s to spark.executor.extraClassPath" % jar_paths[0]) + append_path("spark.driver.extraClassPath", jar_paths[0]) + append_path("spark.executor.extraClassPath", jar_paths[0]) + append_path("SPARK_CLASSPATH", jar_paths[0]) + print("Prepending %s to sys.path" % conf_paths[0]) + sys.path.insert(0, conf_paths[0]) + + +def prepare_env(): + __prepare_spark_env() + __prepare_bigdl_env() \ No newline at end of file diff --git a/python/dllib/test/__init__.py b/python/dllib/test/__init__.py index 2151a805423..63e9a94acc5 100644 --- a/python/dllib/test/__init__.py +++ b/python/dllib/test/__init__.py @@ -13,3 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. # + +from bigdl.util.engine import prepare_env +prepare_env() diff --git a/version.py b/version.py new file mode 100644 index 00000000000..ddff350988c --- /dev/null +++ b/version.py @@ -0,0 +1,17 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +__version__ = "0.2.0.dev2" \ No newline at end of file From a8339ef6535339711940242f6309eb530d454141 Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Thu, 29 Jun 2017 15:58:12 +0800 Subject: [PATCH 096/823] add checking (#1126) --- python/dllib/src/bigdl/dllib/utils/common.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/utils/common.py b/python/dllib/src/bigdl/dllib/utils/common.py index 00b40feaa5b..be79aa570e0 100644 --- a/python/dllib/src/bigdl/dllib/utils/common.py +++ b/python/dllib/src/bigdl/dllib/utils/common.py @@ -258,10 +258,10 @@ def load_conf(conf_str): "#" not in line and line.strip()) for p in sys.path: - if bigdl_conf_file in p: + if bigdl_conf_file in p and os.path.isfile(p): with open(p) if sys.version_info < (3,) else open(p, encoding='latin-1') as conf_file: # noqa return load_conf(conf_file.read()) - if bigdl_python_wrapper in p: + if bigdl_python_wrapper in p and os.path.isfile(p): import zipfile with zipfile.ZipFile(p, 'r') as zip_conf: content = zip_conf.read(bigdl_conf_file) From 3b279dc7c62d682392f9388f1044f2f575c3e5f1 Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Thu, 29 Jun 2017 15:58:12 +0800 Subject: [PATCH 097/823] add checking (#1126) --- python/dllib/src/bigdl/utils/common.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/dllib/src/bigdl/utils/common.py b/python/dllib/src/bigdl/utils/common.py index 00b40feaa5b..be79aa570e0 100644 --- a/python/dllib/src/bigdl/utils/common.py +++ b/python/dllib/src/bigdl/utils/common.py @@ -258,10 +258,10 @@ def load_conf(conf_str): "#" not in line and line.strip()) for p in sys.path: - if bigdl_conf_file in p: + if bigdl_conf_file in p and os.path.isfile(p): with open(p) if sys.version_info < (3,) else open(p, encoding='latin-1') as conf_file: # noqa return load_conf(conf_file.read()) - if bigdl_python_wrapper in p: + if bigdl_python_wrapper in p and os.path.isfile(p): import zipfile with zipfile.ZipFile(p, 'r') as zip_conf: content = zip_conf.read(bigdl_conf_file) From ae66a1a8960ece16a9a1c40b911cbbcd6140830b Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Thu, 29 Jun 2017 15:58:12 +0800 Subject: [PATCH 098/823] add checking (#1126) --- simple_integration_test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/simple_integration_test.py b/simple_integration_test.py index aefc99f6593..ee8e472339b 100644 --- a/simple_integration_test.py +++ b/simple_integration_test.py @@ -111,6 +111,7 @@ def test_load_zip_conf(self): import sys sys.path = [path for path in sys.path if "spark-bigdl.conf" not in path] sys.path.insert(0, os.path.join(os.path.split(__file__)[0], "resources/conf/python-api.zip")) # noqa + sys.path.insert(0, os.path.join(os.path.split(__file__)[0], "resources/conf/invalid-python-api.zip")) # noqa result = get_bigdl_conf() self.assertTrue(result.get("spark.executorEnv.OMP_WAIT_POLICY"), "passive") From 4bfd7a4a540f3177def765b915bc9c2c46b557a2 Mon Sep 17 00:00:00 2001 From: Ian Wong Date: Tue, 4 Jul 2017 13:52:13 +0800 Subject: [PATCH 099/823] Checkin document pages (#1068) * checkin md doc files * add some docs * add more docs * change the folder location * fix unit test * fix typos * fix typo --- python/dllib/src/bigdl/dllib/nn/layer.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 5e35024e6de..14a90c84798 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -815,6 +815,9 @@ class TimeDistributed(Layer): For instance, The TimeDistributed Layer can feed each time slice of input tensor to the Linear layer. + + The input data format is [Batch, Time, Other dims]. For the contained layer, it must not change + the Other dims length. >>> td = TimeDistributed(Linear(2, 3)) @@ -2265,7 +2268,7 @@ class RReLU(Layer): f(x) = max(0,x) + a * min(0, x) where a ~ U(l, u). ``` - In training mode negative inputs are multiplied by a factor a drawn from a uniform random + In training mode negative inputs are multiplied by a factor drawn from a uniform random distribution U(l, u). @@ -2812,7 +2815,7 @@ def set_init_method(self, weight_init_method = None, bias_init_method = None): class VolumetricMaxPooling(Layer): ''' - Applies 3D max-pooling operation in kTxkWxkH regions by step size dTxdWxdH steps. + Applies 3D max-pooling operation in kTxkWxkH regions by step size dTxdWxdH. The number of output features is equal to the number of input planes / dT. The input can optionally be padded with zeros. Padding should be smaller than half of kernel size. That is, padT < kT/2, padW < kW/2 and padH < kH/2 @@ -2889,12 +2892,12 @@ class SplitTable(Layer): ''' Creates a module that takes a Tensor as input and outputs several tables, splitting the Tensor along - the specified dimension `dimension`. + the specified dimension `dimension`. Please note the dimension starts from 1. The input to this layer is expected to be a tensor, or a batch of tensors; when using mini-batch, a batch of sample tensors will be passed to the layer and - the user need to specify the number of dimensions of each sample tensor in a + the user needs to specify the number of dimensions of each sample tensor in a batch using `nInputDims`. @@ -2994,10 +2997,12 @@ def __init__(self, dimension=1, n_input_dims=-1, size_average=False, + squeeze=True, bigdl_type="float"): super(Sum, self).__init__(None, bigdl_type, dimension, n_input_dims, + squeeze, size_average) From 1abb067573ebac4d3fd5fd1cfc980c436dd9e107 Mon Sep 17 00:00:00 2001 From: Ian Wong Date: Tue, 4 Jul 2017 13:52:13 +0800 Subject: [PATCH 100/823] Checkin document pages (#1068) * checkin md doc files * add some docs * add more docs * change the folder location * fix unit test * fix typos * fix typo --- python/dllib/src/bigdl/dllib/nn/layer.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 5e35024e6de..14a90c84798 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -815,6 +815,9 @@ class TimeDistributed(Layer): For instance, The TimeDistributed Layer can feed each time slice of input tensor to the Linear layer. + + The input data format is [Batch, Time, Other dims]. For the contained layer, it must not change + the Other dims length. >>> td = TimeDistributed(Linear(2, 3)) @@ -2265,7 +2268,7 @@ class RReLU(Layer): f(x) = max(0,x) + a * min(0, x) where a ~ U(l, u). ``` - In training mode negative inputs are multiplied by a factor a drawn from a uniform random + In training mode negative inputs are multiplied by a factor drawn from a uniform random distribution U(l, u). @@ -2812,7 +2815,7 @@ def set_init_method(self, weight_init_method = None, bias_init_method = None): class VolumetricMaxPooling(Layer): ''' - Applies 3D max-pooling operation in kTxkWxkH regions by step size dTxdWxdH steps. + Applies 3D max-pooling operation in kTxkWxkH regions by step size dTxdWxdH. The number of output features is equal to the number of input planes / dT. The input can optionally be padded with zeros. Padding should be smaller than half of kernel size. That is, padT < kT/2, padW < kW/2 and padH < kH/2 @@ -2889,12 +2892,12 @@ class SplitTable(Layer): ''' Creates a module that takes a Tensor as input and outputs several tables, splitting the Tensor along - the specified dimension `dimension`. + the specified dimension `dimension`. Please note the dimension starts from 1. The input to this layer is expected to be a tensor, or a batch of tensors; when using mini-batch, a batch of sample tensors will be passed to the layer and - the user need to specify the number of dimensions of each sample tensor in a + the user needs to specify the number of dimensions of each sample tensor in a batch using `nInputDims`. @@ -2994,10 +2997,12 @@ def __init__(self, dimension=1, n_input_dims=-1, size_average=False, + squeeze=True, bigdl_type="float"): super(Sum, self).__init__(None, bigdl_type, dimension, n_input_dims, + squeeze, size_average) From b5d6d70e871aea21c6d46caacaaea81559e95d2b Mon Sep 17 00:00:00 2001 From: yangw Date: Tue, 4 Jul 2017 16:19:35 +0800 Subject: [PATCH 101/823] add initWeight .etc --- python/dllib/src/bigdl/dllib/nn/criterion.py | 6 +- python/dllib/src/bigdl/dllib/nn/layer.py | 60 ++++++++++++++++---- 2 files changed, 53 insertions(+), 13 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/nn/criterion.py b/python/dllib/src/bigdl/dllib/nn/criterion.py index 218c23c5f8a..f94fb2d11c7 100644 --- a/python/dllib/src/bigdl/dllib/nn/criterion.py +++ b/python/dllib/src/bigdl/dllib/nn/criterion.py @@ -686,11 +686,13 @@ class DiceCoefficientCriterion(Criterion): >>> diceCoefficientCriterion = DiceCoefficientCriterion(size_average = True, epsilon = 1.0) creating: createDiceCoefficientCriterion + >>> diceCoefficientCriterion = DiceCoefficientCriterion() + creating: createDiceCoefficientCriterion ''' def __init__(self, - size_average, - epsilon, + size_average=True, + epsilon=1.0, bigdl_type="float"): super(DiceCoefficientCriterion, self).__init__(None, bigdl_type, size_average, diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 14a90c84798..755723c13c2 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -443,25 +443,33 @@ class Linear(Layer): :param input_size the size the each input sample :param output_size the size of the module output of each sample - :param init_method: two initialized methods are supported here, which are [[Default]]and [[Xavier]], where [[Xavier]] set bias to zero here. For moredetailed information about `initMethod`, please refer to[[InitializationMethod]] :param wRegularizer: instance of [[Regularizer]](eg. L1 or L2 regularization), applied to the input weights matrices. :param bRegularizer: instance of [[Regularizer]]applied to the bias. + :param init_weight: the optional initial value for the weight + :param init_bias: the optional initial value for the bias + :param init_grad_weight: the optional initial value for the grad_weight + :param init_grad_bias: the optional initial value for the grad_bias - >>> linear = Linear(100, 10, "Xavier", True, L1Regularizer(0.5), L1Regularizer(0.5)) + >>> linear = Linear(100, 10, True, L1Regularizer(0.5), L1Regularizer(0.5)) creating: createL1Regularizer creating: createL1Regularizer creating: createLinear ''' - def __init__(self, input_size, output_size, init_method="default", with_bias=True, wRegularizer=None, bRegularizer=None, - bigdl_type="float"): + def __init__(self, input_size, output_size, with_bias=True, wRegularizer=None, bRegularizer=None, + init_weight=None, init_bias=None, init_grad_weight=None, init_grad_bias=None, bigdl_type="float"): super(Linear, self).__init__(None, bigdl_type, input_size, output_size, - init_method, with_bias, wRegularizer, bRegularizer) + with_bias, wRegularizer, bRegularizer, + JTensor.from_ndarray(init_weight), + JTensor.from_ndarray(init_bias), + JTensor.from_ndarray(init_grad_weight), + JTensor.from_ndarray(init_grad_bias)) def set_init_method(self, weight_init_method = None, bias_init_method = None): callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, weight_init_method, bias_init_method) + return self class ReLU(Layer): @@ -568,9 +576,12 @@ class SpatialConvolution(Layer): :param pad_h The additional zeros added per height to the input planes. :param n_group Kernel group number :param propagate_back Propagate gradient back - :param init_method Initialization method to initialize bias and weight :param wRegularizer: instance of [[Regularizer]](eg. L1 or L2 regularization), applied to the input weights matrices. :param bRegularizer: instance of [[Regularizer]]applied to the bias. + :param init_weight: the optional initial value for the weight + :param init_bias: the optional initial value for the bias + :param init_grad_weight: the optional initial value for the grad_weight + :param init_grad_bias: the optional initial value for the grad_bias >>> spatialConvolution = SpatialConvolution(6, 12, 5, 5) creating: createSpatialConvolution @@ -594,6 +605,10 @@ def __init__(self, init_method="default", wRegularizer=None, bRegularizer=None, + init_weight=None, + init_bias=None, + init_grad_weight=None, + init_grad_bias=None, bigdl_type="float"): super(SpatialConvolution, self).__init__(None, bigdl_type, n_input_plane, @@ -608,10 +623,15 @@ def __init__(self, propagate_back, init_method, wRegularizer, - bRegularizer) + bRegularizer, + JTensor.from_ndarray(init_weight), + JTensor.from_ndarray(init_bias), + JTensor.from_ndarray(init_grad_weight), + JTensor.from_ndarray(init_grad_bias)) def set_init_method(self, weight_init_method = None, bias_init_method = None): callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, weight_init_method, bias_init_method) + return self class SpatialMaxPooling(Layer): @@ -943,6 +963,7 @@ def __init__(self, def set_init_method(self, weight_init_method = None, bias_init_method = None): callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, weight_init_method, bias_init_method) + return self class SpatialCrossMapLRN(Layer): @@ -1071,6 +1092,7 @@ def __init__(self, def set_init_method(self, weight_init_method = None, bias_init_method = None): callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, weight_init_method, bias_init_method) + return self class AddConstant(Layer): @@ -1140,6 +1162,7 @@ def __init__(self, def set_init_method(self, weight_init_method = None, bias_init_method = None): callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, weight_init_method, bias_init_method) + return self class Bilinear(Layer): @@ -1179,6 +1202,7 @@ def __init__(self, def set_init_method(self, weight_init_method = None, bias_init_method = None): callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, weight_init_method, bias_init_method) + return self class Bottle(Container): @@ -1236,6 +1260,7 @@ def __init__(self, def set_init_method(self, weight_init_method = None, bias_init_method = None): callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, weight_init_method, bias_init_method) + return self class CAddTable(Layer): @@ -1326,6 +1351,7 @@ def __init__(self, def set_init_method(self, weight_init_method = None, bias_init_method = None): callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, weight_init_method, bias_init_method) + return self class CMulTable(Layer): @@ -1427,6 +1453,7 @@ def __init__(self, def set_init_method(self, weight_init_method = None, bias_init_method = None): callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, weight_init_method, bias_init_method) + return self class CosineDistance(Layer): @@ -1510,6 +1537,7 @@ def __init__(self, def set_init_method(self, weight_init_method = None, bias_init_method = None): callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, weight_init_method, bias_init_method) + return self class Exp(Layer): @@ -1833,6 +1861,7 @@ def __init__(self, def set_init_method(self, weight_init_method = None, bias_init_method = None): callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, weight_init_method, bias_init_method) + return self class MM(Layer): @@ -1948,19 +1977,21 @@ class Mean(Layer): :param dimension: the dimension to be applied mean operation :param n_input_dims: specify the number of dimensions that this module will receiveIf it is more than the dimension of input tensors, the first dimension would be consideredas batch size + :param squeeze: default is true, which will squeeze the sum dimension; set it to false to keep the sum dimension - - >>> mean = Mean(1, 1) + >>> mean = Mean(1, 1, True) creating: createMean ''' def __init__(self, dimension=1, n_input_dims=-1, + squeeze=True, bigdl_type="float"): super(Mean, self).__init__(None, bigdl_type, dimension, - n_input_dims) + n_input_dims, + squeeze) class Min(Layer): @@ -2025,6 +2056,7 @@ def __init__(self, def set_init_method(self, weight_init_method = None, bias_init_method = None): callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, weight_init_method, bias_init_method) + return self class MulConstant(Layer): @@ -2151,6 +2183,7 @@ def __init__(self, def set_init_method(self, weight_init_method = None, bias_init_method = None): callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, weight_init_method, bias_init_method) + return self class Padding(Layer): @@ -2629,6 +2662,7 @@ def __init__(self, def set_init_method(self, weight_init_method = None, bias_init_method = None): callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, weight_init_method, bias_init_method) + return self class SpatialFullConvolution(Layer): @@ -2712,6 +2746,7 @@ def __init__(self, def set_init_method(self, weight_init_method = None, bias_init_method = None): callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, weight_init_method, bias_init_method) + return self class SpatialShareConvolution(Layer): @@ -2750,6 +2785,7 @@ def __init__(self, def set_init_method(self, weight_init_method = None, bias_init_method = None): callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, weight_init_method, bias_init_method) + return self class VolumetricConvolution(Layer): @@ -2810,6 +2846,7 @@ def __init__(self, def set_init_method(self, weight_init_method = None, bias_init_method = None): callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, weight_init_method, bias_init_method) + return self class VolumetricMaxPooling(Layer): @@ -2987,9 +3024,10 @@ class Sum(Layer): :param dimension: the dimension to be applied sum operation :param n_input_dims: specify the number of dimensions that this module will receiveIf it is more than the dimension of input tensors, the first dimensionwould be considered as batch size :param size_average: default is false, if it is true, it will return the mean instead + :param squeeze: default is true, which will squeeze the sum dimension; set it to false to keep the sum dimension - >>> sum = Sum(1, 1, True) + >>> sum = Sum(1, 1, True, True) creating: createSum ''' From 2d0601e5991676172fb776e22d756cbb4dc77db9 Mon Sep 17 00:00:00 2001 From: yangw Date: Tue, 4 Jul 2017 16:19:35 +0800 Subject: [PATCH 102/823] add initWeight .etc --- python/dllib/src/bigdl/dllib/nn/criterion.py | 6 +- python/dllib/src/bigdl/dllib/nn/layer.py | 60 ++++++++++++++++---- 2 files changed, 53 insertions(+), 13 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/nn/criterion.py b/python/dllib/src/bigdl/dllib/nn/criterion.py index 218c23c5f8a..f94fb2d11c7 100644 --- a/python/dllib/src/bigdl/dllib/nn/criterion.py +++ b/python/dllib/src/bigdl/dllib/nn/criterion.py @@ -686,11 +686,13 @@ class DiceCoefficientCriterion(Criterion): >>> diceCoefficientCriterion = DiceCoefficientCriterion(size_average = True, epsilon = 1.0) creating: createDiceCoefficientCriterion + >>> diceCoefficientCriterion = DiceCoefficientCriterion() + creating: createDiceCoefficientCriterion ''' def __init__(self, - size_average, - epsilon, + size_average=True, + epsilon=1.0, bigdl_type="float"): super(DiceCoefficientCriterion, self).__init__(None, bigdl_type, size_average, diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 14a90c84798..755723c13c2 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -443,25 +443,33 @@ class Linear(Layer): :param input_size the size the each input sample :param output_size the size of the module output of each sample - :param init_method: two initialized methods are supported here, which are [[Default]]and [[Xavier]], where [[Xavier]] set bias to zero here. For moredetailed information about `initMethod`, please refer to[[InitializationMethod]] :param wRegularizer: instance of [[Regularizer]](eg. L1 or L2 regularization), applied to the input weights matrices. :param bRegularizer: instance of [[Regularizer]]applied to the bias. + :param init_weight: the optional initial value for the weight + :param init_bias: the optional initial value for the bias + :param init_grad_weight: the optional initial value for the grad_weight + :param init_grad_bias: the optional initial value for the grad_bias - >>> linear = Linear(100, 10, "Xavier", True, L1Regularizer(0.5), L1Regularizer(0.5)) + >>> linear = Linear(100, 10, True, L1Regularizer(0.5), L1Regularizer(0.5)) creating: createL1Regularizer creating: createL1Regularizer creating: createLinear ''' - def __init__(self, input_size, output_size, init_method="default", with_bias=True, wRegularizer=None, bRegularizer=None, - bigdl_type="float"): + def __init__(self, input_size, output_size, with_bias=True, wRegularizer=None, bRegularizer=None, + init_weight=None, init_bias=None, init_grad_weight=None, init_grad_bias=None, bigdl_type="float"): super(Linear, self).__init__(None, bigdl_type, input_size, output_size, - init_method, with_bias, wRegularizer, bRegularizer) + with_bias, wRegularizer, bRegularizer, + JTensor.from_ndarray(init_weight), + JTensor.from_ndarray(init_bias), + JTensor.from_ndarray(init_grad_weight), + JTensor.from_ndarray(init_grad_bias)) def set_init_method(self, weight_init_method = None, bias_init_method = None): callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, weight_init_method, bias_init_method) + return self class ReLU(Layer): @@ -568,9 +576,12 @@ class SpatialConvolution(Layer): :param pad_h The additional zeros added per height to the input planes. :param n_group Kernel group number :param propagate_back Propagate gradient back - :param init_method Initialization method to initialize bias and weight :param wRegularizer: instance of [[Regularizer]](eg. L1 or L2 regularization), applied to the input weights matrices. :param bRegularizer: instance of [[Regularizer]]applied to the bias. + :param init_weight: the optional initial value for the weight + :param init_bias: the optional initial value for the bias + :param init_grad_weight: the optional initial value for the grad_weight + :param init_grad_bias: the optional initial value for the grad_bias >>> spatialConvolution = SpatialConvolution(6, 12, 5, 5) creating: createSpatialConvolution @@ -594,6 +605,10 @@ def __init__(self, init_method="default", wRegularizer=None, bRegularizer=None, + init_weight=None, + init_bias=None, + init_grad_weight=None, + init_grad_bias=None, bigdl_type="float"): super(SpatialConvolution, self).__init__(None, bigdl_type, n_input_plane, @@ -608,10 +623,15 @@ def __init__(self, propagate_back, init_method, wRegularizer, - bRegularizer) + bRegularizer, + JTensor.from_ndarray(init_weight), + JTensor.from_ndarray(init_bias), + JTensor.from_ndarray(init_grad_weight), + JTensor.from_ndarray(init_grad_bias)) def set_init_method(self, weight_init_method = None, bias_init_method = None): callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, weight_init_method, bias_init_method) + return self class SpatialMaxPooling(Layer): @@ -943,6 +963,7 @@ def __init__(self, def set_init_method(self, weight_init_method = None, bias_init_method = None): callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, weight_init_method, bias_init_method) + return self class SpatialCrossMapLRN(Layer): @@ -1071,6 +1092,7 @@ def __init__(self, def set_init_method(self, weight_init_method = None, bias_init_method = None): callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, weight_init_method, bias_init_method) + return self class AddConstant(Layer): @@ -1140,6 +1162,7 @@ def __init__(self, def set_init_method(self, weight_init_method = None, bias_init_method = None): callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, weight_init_method, bias_init_method) + return self class Bilinear(Layer): @@ -1179,6 +1202,7 @@ def __init__(self, def set_init_method(self, weight_init_method = None, bias_init_method = None): callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, weight_init_method, bias_init_method) + return self class Bottle(Container): @@ -1236,6 +1260,7 @@ def __init__(self, def set_init_method(self, weight_init_method = None, bias_init_method = None): callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, weight_init_method, bias_init_method) + return self class CAddTable(Layer): @@ -1326,6 +1351,7 @@ def __init__(self, def set_init_method(self, weight_init_method = None, bias_init_method = None): callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, weight_init_method, bias_init_method) + return self class CMulTable(Layer): @@ -1427,6 +1453,7 @@ def __init__(self, def set_init_method(self, weight_init_method = None, bias_init_method = None): callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, weight_init_method, bias_init_method) + return self class CosineDistance(Layer): @@ -1510,6 +1537,7 @@ def __init__(self, def set_init_method(self, weight_init_method = None, bias_init_method = None): callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, weight_init_method, bias_init_method) + return self class Exp(Layer): @@ -1833,6 +1861,7 @@ def __init__(self, def set_init_method(self, weight_init_method = None, bias_init_method = None): callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, weight_init_method, bias_init_method) + return self class MM(Layer): @@ -1948,19 +1977,21 @@ class Mean(Layer): :param dimension: the dimension to be applied mean operation :param n_input_dims: specify the number of dimensions that this module will receiveIf it is more than the dimension of input tensors, the first dimension would be consideredas batch size + :param squeeze: default is true, which will squeeze the sum dimension; set it to false to keep the sum dimension - - >>> mean = Mean(1, 1) + >>> mean = Mean(1, 1, True) creating: createMean ''' def __init__(self, dimension=1, n_input_dims=-1, + squeeze=True, bigdl_type="float"): super(Mean, self).__init__(None, bigdl_type, dimension, - n_input_dims) + n_input_dims, + squeeze) class Min(Layer): @@ -2025,6 +2056,7 @@ def __init__(self, def set_init_method(self, weight_init_method = None, bias_init_method = None): callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, weight_init_method, bias_init_method) + return self class MulConstant(Layer): @@ -2151,6 +2183,7 @@ def __init__(self, def set_init_method(self, weight_init_method = None, bias_init_method = None): callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, weight_init_method, bias_init_method) + return self class Padding(Layer): @@ -2629,6 +2662,7 @@ def __init__(self, def set_init_method(self, weight_init_method = None, bias_init_method = None): callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, weight_init_method, bias_init_method) + return self class SpatialFullConvolution(Layer): @@ -2712,6 +2746,7 @@ def __init__(self, def set_init_method(self, weight_init_method = None, bias_init_method = None): callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, weight_init_method, bias_init_method) + return self class SpatialShareConvolution(Layer): @@ -2750,6 +2785,7 @@ def __init__(self, def set_init_method(self, weight_init_method = None, bias_init_method = None): callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, weight_init_method, bias_init_method) + return self class VolumetricConvolution(Layer): @@ -2810,6 +2846,7 @@ def __init__(self, def set_init_method(self, weight_init_method = None, bias_init_method = None): callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, weight_init_method, bias_init_method) + return self class VolumetricMaxPooling(Layer): @@ -2987,9 +3024,10 @@ class Sum(Layer): :param dimension: the dimension to be applied sum operation :param n_input_dims: specify the number of dimensions that this module will receiveIf it is more than the dimension of input tensors, the first dimensionwould be considered as batch size :param size_average: default is false, if it is true, it will return the mean instead + :param squeeze: default is true, which will squeeze the sum dimension; set it to false to keep the sum dimension - >>> sum = Sum(1, 1, True) + >>> sum = Sum(1, 1, True, True) creating: createSum ''' From 16fe960fdd33e70e81f072fa723261b7fb984f75 Mon Sep 17 00:00:00 2001 From: yangw Date: Tue, 4 Jul 2017 16:19:35 +0800 Subject: [PATCH 103/823] add initWeight .etc --- simple_integration_test.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/simple_integration_test.py b/simple_integration_test.py index ee8e472339b..9e2e1acfeab 100644 --- a/simple_integration_test.py +++ b/simple_integration_test.py @@ -16,6 +16,7 @@ # Still in experimental stage! from bigdl.nn.layer import * +from bigdl.nn.initialization_method import * from bigdl.nn.criterion import * from bigdl.optim.optimizer import * from bigdl.util.common import * @@ -116,8 +117,10 @@ def test_load_zip_conf(self): self.assertTrue(result.get("spark.executorEnv.OMP_WAIT_POLICY"), "passive") def test_set_seed(self): - l1 = Linear(10, 20, "Xavier").set_name("linear1").set_seed(1234).reset() # noqa - l2 = Linear(10, 20, "Xavier").set_name("linear2").set_seed(1234).reset() # noqa + w_init = Xavier() + b_init = Zeros() + l1 = Linear(10, 20).set_init_method(w_init, b_init).set_name("linear1").set_seed(1234).reset() # noqa + l2 = Linear(10, 20).set_init_method(w_init, b_init).set_name("linear2").set_seed(1234).reset() # noqa p1 = l1.parameters() p2 = l2.parameters() self.assertTrue((p1["linear1"]["weight"] == p2["linear2"]["weight"]).all()) # noqa @@ -137,13 +140,14 @@ def gen_rand_sample(): lambda i: gen_rand_sample()) model_test = Sequential() - l1_test = Linear(FEATURES_DIM, 1, "Xavier").set_name("linear1_test") + l1_test = Linear(FEATURES_DIM, 1).set_init_method(Xavier(), Zeros())\ + .set_name("linear1_test") self.assertEqual("linear1_test", l1_test.name()) model_test.add(l1_test) model_test.add(Sigmoid()) model = Sequential() - l1 = Linear(FEATURES_DIM, 1, "Xavier").set_name("linear1") + l1 = Linear(FEATURES_DIM, 1).set_init_method(Xavier(), Zeros()).set_name("linear1") self.assertEqual("linear1", l1.name()) model.add(l1) @@ -305,7 +309,8 @@ def test_predict(self): label = (features).sum() + 0.4 predict_data = self.sc.parallelize(range(0, total_length)).map( lambda i: Sample.from_ndarray(features[i], label)) - model = Linear(2, 1, "Xavier").set_name("linear1").set_seed(1234).reset() + model = Linear(2, 1).set_init_method(Xavier(), Zeros())\ + .set_name("linear1").set_seed(1234).reset() predict_result = model.predict(predict_data) p = predict_result.take(6) ground_label = np.array([[-0.47596836], [-0.37598032], [-0.00492062], From 63b50c9d99a08b09595d9fd3b94b7b5492460b5a Mon Sep 17 00:00:00 2001 From: jenniew Date: Tue, 4 Jul 2017 21:21:30 -0700 Subject: [PATCH 104/823] update layer docs --- python/dllib/src/bigdl/dllib/nn/layer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 755723c13c2..c98de4e129c 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -1921,7 +1921,7 @@ class MapTable(Container): ''' def __init__(self, - module, + module=None, bigdl_type="float"): super(MapTable, self).__init__(None, bigdl_type, module) From f0c217b8d16c4446bf86d72f2874f5212d5a326c Mon Sep 17 00:00:00 2001 From: jenniew Date: Tue, 4 Jul 2017 21:21:30 -0700 Subject: [PATCH 105/823] update layer docs --- python/dllib/src/bigdl/dllib/nn/layer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 755723c13c2..c98de4e129c 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -1921,7 +1921,7 @@ class MapTable(Container): ''' def __init__(self, - module, + module=None, bigdl_type="float"): super(MapTable, self).__init__(None, bigdl_type, module) From 7da78b06e6d39d05e6b77942abfb4a67cba13346 Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Wed, 5 Jul 2017 15:29:14 +0800 Subject: [PATCH 106/823] fix doc-gen (#1151) --- .../src/bigdl/dllib/feature/dataset/mnist.py | 34 ++++++++----------- 1 file changed, 15 insertions(+), 19 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/feature/dataset/mnist.py b/python/dllib/src/bigdl/dllib/feature/dataset/mnist.py index 0af0f70c51f..492882fd730 100644 --- a/python/dllib/src/bigdl/dllib/feature/dataset/mnist.py +++ b/python/dllib/src/bigdl/dllib/feature/dataset/mnist.py @@ -38,18 +38,9 @@ def _read32(bytestream): def extract_images(f): """Extract the images into a 4D uint8 numpy array [index, y, x, depth]. - - Args: - f: A file object that can be passed into a gzip reader. - - - Returns: - data: A 4D unit8 numpy array [index, y, x, depth]. - - - Raises: - ValueError: If the bytestream does not start with 2051. - + :param: f: A file object that can be passed into a gzip reader. + :return: data: A 4D unit8 numpy array [index, y, x, depth]. + :raise: ValueError: If the bytestream does not start with 2051. """ print('Extracting', f.name) @@ -86,13 +77,18 @@ def read_data_sets(train_dir, data_type="train"): """ Parse or download mnist data if train_dir is empty. - :param train_dir: The directory storing the mnist data - :param data_type: Reading training set or testing set.It can be either "train" or "test" - :return: (ndarray, ndarray) representing (features, labels) - features is a 4D unit8 numpy array [index, y, x, depth] - representing each pixel valued from 0 to 255. labels - is 1D unit8 nunpy array representing the label valued - from 0 to 9. + :param: train_dir: The directory storing the mnist data + + :param: data_type: Reading training set or testing set.It can be either "train" or "test" + + :return: + + ``` + (ndarray, ndarray) representing (features, labels) + features is a 4D unit8 numpy array [index, y, x, depth] representing each pixel valued from 0 to 255. + labels is 1D unit8 nunpy array representing the label valued from 0 to 9. + ``` + """ TRAIN_IMAGES = 'train-images-idx3-ubyte.gz' TRAIN_LABELS = 'train-labels-idx1-ubyte.gz' From 7168887f76b508d71cd8875da0a45dc25a693ea8 Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Wed, 5 Jul 2017 15:29:14 +0800 Subject: [PATCH 107/823] fix doc-gen (#1151) --- .../src/bigdl/dllib/feature/dataset/mnist.py | 34 ++++++++----------- 1 file changed, 15 insertions(+), 19 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/feature/dataset/mnist.py b/python/dllib/src/bigdl/dllib/feature/dataset/mnist.py index 0af0f70c51f..492882fd730 100644 --- a/python/dllib/src/bigdl/dllib/feature/dataset/mnist.py +++ b/python/dllib/src/bigdl/dllib/feature/dataset/mnist.py @@ -38,18 +38,9 @@ def _read32(bytestream): def extract_images(f): """Extract the images into a 4D uint8 numpy array [index, y, x, depth]. - - Args: - f: A file object that can be passed into a gzip reader. - - - Returns: - data: A 4D unit8 numpy array [index, y, x, depth]. - - - Raises: - ValueError: If the bytestream does not start with 2051. - + :param: f: A file object that can be passed into a gzip reader. + :return: data: A 4D unit8 numpy array [index, y, x, depth]. + :raise: ValueError: If the bytestream does not start with 2051. """ print('Extracting', f.name) @@ -86,13 +77,18 @@ def read_data_sets(train_dir, data_type="train"): """ Parse or download mnist data if train_dir is empty. - :param train_dir: The directory storing the mnist data - :param data_type: Reading training set or testing set.It can be either "train" or "test" - :return: (ndarray, ndarray) representing (features, labels) - features is a 4D unit8 numpy array [index, y, x, depth] - representing each pixel valued from 0 to 255. labels - is 1D unit8 nunpy array representing the label valued - from 0 to 9. + :param: train_dir: The directory storing the mnist data + + :param: data_type: Reading training set or testing set.It can be either "train" or "test" + + :return: + + ``` + (ndarray, ndarray) representing (features, labels) + features is a 4D unit8 numpy array [index, y, x, depth] representing each pixel valued from 0 to 255. + labels is 1D unit8 nunpy array representing the label valued from 0 to 9. + ``` + """ TRAIN_IMAGES = 'train-images-idx3-ubyte.gz' TRAIN_LABELS = 'train-labels-idx1-ubyte.gz' From 1328f0ee708ba10b2664fe00fca6d4b7cf742827 Mon Sep 17 00:00:00 2001 From: zhangxiaoli73 <380761639@qq.com> Date: Thu, 6 Jul 2017 09:47:21 +0800 Subject: [PATCH 108/823] add default value for python LSTMPeephole (#1147) --- python/dllib/src/bigdl/dllib/nn/layer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index c98de4e129c..db8b2791e5e 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -760,7 +760,7 @@ class LSTMPeephole(Layer): creating: createLSTMPeephole ''' - def __init__(self, input_size, hidden_size, p=0.0, wRegularizer=None, uRegularizer=None, bRegularizer=None, bigdl_type="float"): + def __init__(self, input_size=4, hidden_size=3, p=0.0, wRegularizer=None, uRegularizer=None, bRegularizer=None, bigdl_type="float"): super(LSTMPeephole, self).__init__(None, bigdl_type, input_size, hidden_size, p, wRegularizer, uRegularizer, bRegularizer) From e970bde26e094b68dd915cb0ed9ba557294c1ebf Mon Sep 17 00:00:00 2001 From: zhangxiaoli73 <380761639@qq.com> Date: Thu, 6 Jul 2017 09:47:21 +0800 Subject: [PATCH 109/823] add default value for python LSTMPeephole (#1147) --- python/dllib/src/bigdl/dllib/nn/layer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index c98de4e129c..db8b2791e5e 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -760,7 +760,7 @@ class LSTMPeephole(Layer): creating: createLSTMPeephole ''' - def __init__(self, input_size, hidden_size, p=0.0, wRegularizer=None, uRegularizer=None, bRegularizer=None, bigdl_type="float"): + def __init__(self, input_size=4, hidden_size=3, p=0.0, wRegularizer=None, uRegularizer=None, bRegularizer=None, bigdl_type="float"): super(LSTMPeephole, self).__init__(None, bigdl_type, input_size, hidden_size, p, wRegularizer, uRegularizer, bRegularizer) From d2a2fd91731c8b340f3b5a5f1f4f053f625ef94e Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Thu, 6 Jul 2017 12:44:22 +0800 Subject: [PATCH 110/823] Change type of ValidationMethod from string to ValidationMethod instance (#1149) * refactor ValidationMethod * add * minor * refactor --- .../src/bigdl/dllib/models/lenet/lenet5.py | 4 +- .../models/textclassifier/textclassifier.py | 2 +- .../dllib/src/bigdl/dllib/optim/optimizer.py | 50 ++++++++++++++++++- 3 files changed, 51 insertions(+), 5 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py b/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py index 57994858672..6de0b0124e1 100644 --- a/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py +++ b/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py @@ -97,7 +97,7 @@ def get_end_trigger(): batch_size=options.batchSize, val_rdd=test_data, trigger=EveryEpoch(), - val_method=["Top1Accuracy"] + val_method=[Top1Accuracy()] ) optimizer.set_checkpoint(EveryEpoch(), options.checkpointPath) trained_model = optimizer.optimize() @@ -107,7 +107,7 @@ def get_end_trigger(): test_data = get_mnist(sc, "test").map( normalizer(mnist.TEST_MEAN, mnist.TEST_STD)) model = Model.load(options.modelPath) - results = model.test(test_data, options.batchSize, ["Top1Accuracy"]) + results = model.test(test_data, options.batchSize, [Top1Accuracy()]) for result in results: print(result) sc.stop() diff --git a/python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py b/python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py index c190f04222f..9e235990efd 100644 --- a/python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py +++ b/python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py @@ -144,7 +144,7 @@ def train(sc, batch_size=batch_size, val_rdd=val_rdd, trigger=EveryEpoch(), - val_method=["Top1Accuracy"] + val_method=[Top1Accuracy()] ) train_model = optimizer.optimize() diff --git a/python/dllib/src/bigdl/dllib/optim/optimizer.py b/python/dllib/src/bigdl/dllib/optim/optimizer.py index 436f21b44b1..762fcf4090a 100644 --- a/python/dllib/src/bigdl/dllib/optim/optimizer.py +++ b/python/dllib/src/bigdl/dllib/optim/optimizer.py @@ -25,12 +25,56 @@ from bigdl.util.common import callBigDlFunc from bigdl.util.common import callJavaFunc from bigdl.util.common import get_spark_context +from bigdl.util.common import to_list + if sys.version >= '3': long = int unicode = str +class Top1Accuracy(JavaValue): + """ + Caculate the percentage that output's max probability index equals target. + + >>> top1 = Top1Accuracy() + creating: createTop1Accuracy + """ + def __init__(self, bigdl_type="float"): + JavaValue.__init__(self, None, bigdl_type) + + +class Top5Accuracy(JavaValue): + """ + Caculate the percentage that output's max probability index equals target. + + >>> top5 = Top5Accuracy() + creating: createTop5Accuracy + """ + def __init__(self, bigdl_type="float"): + JavaValue.__init__(self, None, bigdl_type) + + +class Loss(JavaValue): + + """ + This evaluation method is calculate loss of output with respect to target + >>> from bigdl.nn.criterion import ClassNLLCriterion + >>> loss = Loss() + creating: createClassNLLCriterion + creating: createLoss + + >>> loss = Loss(ClassNLLCriterion()) + creating: createClassNLLCriterion + creating: createLoss + """ + def __init__(self, cri=None, bigdl_type="float"): + from bigdl.nn.criterion import ClassNLLCriterion + if cri is None: + cri = ClassNLLCriterion() + JavaValue.__init__(self, None, bigdl_type, cri) + + class MaxIteration(JavaValue): """ A trigger specifies a timespot or several timespots during training, @@ -390,7 +434,7 @@ def __init__(self, training_rdd, criterion, optim_method if optim_method else SGD(), end_trigger, batch_size) - def set_validation(self, batch_size, val_rdd, trigger, val_method=["Top1Accuracy"]): + def set_validation(self, batch_size, val_rdd, trigger, val_method=None): """ Configure validation settings. @@ -400,8 +444,10 @@ def set_validation(self, batch_size, val_rdd, trigger, val_method=["Top1Accuracy :param trigger: validation interval :param val_method: the ValidationMethod to use,e.g. "Top1Accuracy", "Top5Accuracy", "Loss" """ + if val_method is None: + val_method = [Top1Accuracy()] callBigDlFunc(self.bigdl_type, "setValidation", self.value, batch_size, - trigger, val_rdd, val_method) + trigger, val_rdd, to_list(val_method)) def set_model(self, model): """ From 3a9b60a0ae73d3a30dec04a2d45a6cdcce9b80a6 Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Thu, 6 Jul 2017 12:44:22 +0800 Subject: [PATCH 111/823] Change type of ValidationMethod from string to ValidationMethod instance (#1149) * refactor ValidationMethod * add * minor * refactor --- .../src/bigdl/dllib/models/lenet/lenet5.py | 4 +- .../models/textclassifier/textclassifier.py | 2 +- .../dllib/src/bigdl/dllib/optim/optimizer.py | 50 ++++++++++++++++++- 3 files changed, 51 insertions(+), 5 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py b/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py index 57994858672..6de0b0124e1 100644 --- a/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py +++ b/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py @@ -97,7 +97,7 @@ def get_end_trigger(): batch_size=options.batchSize, val_rdd=test_data, trigger=EveryEpoch(), - val_method=["Top1Accuracy"] + val_method=[Top1Accuracy()] ) optimizer.set_checkpoint(EveryEpoch(), options.checkpointPath) trained_model = optimizer.optimize() @@ -107,7 +107,7 @@ def get_end_trigger(): test_data = get_mnist(sc, "test").map( normalizer(mnist.TEST_MEAN, mnist.TEST_STD)) model = Model.load(options.modelPath) - results = model.test(test_data, options.batchSize, ["Top1Accuracy"]) + results = model.test(test_data, options.batchSize, [Top1Accuracy()]) for result in results: print(result) sc.stop() diff --git a/python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py b/python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py index c190f04222f..9e235990efd 100644 --- a/python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py +++ b/python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py @@ -144,7 +144,7 @@ def train(sc, batch_size=batch_size, val_rdd=val_rdd, trigger=EveryEpoch(), - val_method=["Top1Accuracy"] + val_method=[Top1Accuracy()] ) train_model = optimizer.optimize() diff --git a/python/dllib/src/bigdl/dllib/optim/optimizer.py b/python/dllib/src/bigdl/dllib/optim/optimizer.py index 436f21b44b1..762fcf4090a 100644 --- a/python/dllib/src/bigdl/dllib/optim/optimizer.py +++ b/python/dllib/src/bigdl/dllib/optim/optimizer.py @@ -25,12 +25,56 @@ from bigdl.util.common import callBigDlFunc from bigdl.util.common import callJavaFunc from bigdl.util.common import get_spark_context +from bigdl.util.common import to_list + if sys.version >= '3': long = int unicode = str +class Top1Accuracy(JavaValue): + """ + Caculate the percentage that output's max probability index equals target. + + >>> top1 = Top1Accuracy() + creating: createTop1Accuracy + """ + def __init__(self, bigdl_type="float"): + JavaValue.__init__(self, None, bigdl_type) + + +class Top5Accuracy(JavaValue): + """ + Caculate the percentage that output's max probability index equals target. + + >>> top5 = Top5Accuracy() + creating: createTop5Accuracy + """ + def __init__(self, bigdl_type="float"): + JavaValue.__init__(self, None, bigdl_type) + + +class Loss(JavaValue): + + """ + This evaluation method is calculate loss of output with respect to target + >>> from bigdl.nn.criterion import ClassNLLCriterion + >>> loss = Loss() + creating: createClassNLLCriterion + creating: createLoss + + >>> loss = Loss(ClassNLLCriterion()) + creating: createClassNLLCriterion + creating: createLoss + """ + def __init__(self, cri=None, bigdl_type="float"): + from bigdl.nn.criterion import ClassNLLCriterion + if cri is None: + cri = ClassNLLCriterion() + JavaValue.__init__(self, None, bigdl_type, cri) + + class MaxIteration(JavaValue): """ A trigger specifies a timespot or several timespots during training, @@ -390,7 +434,7 @@ def __init__(self, training_rdd, criterion, optim_method if optim_method else SGD(), end_trigger, batch_size) - def set_validation(self, batch_size, val_rdd, trigger, val_method=["Top1Accuracy"]): + def set_validation(self, batch_size, val_rdd, trigger, val_method=None): """ Configure validation settings. @@ -400,8 +444,10 @@ def set_validation(self, batch_size, val_rdd, trigger, val_method=["Top1Accuracy :param trigger: validation interval :param val_method: the ValidationMethod to use,e.g. "Top1Accuracy", "Top5Accuracy", "Loss" """ + if val_method is None: + val_method = [Top1Accuracy()] callBigDlFunc(self.bigdl_type, "setValidation", self.value, batch_size, - trigger, val_rdd, val_method) + trigger, val_rdd, to_list(val_method)) def set_model(self, model): """ From ec21c6fdee72eaca13cfbe082ed98fefce4223d9 Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Thu, 6 Jul 2017 12:44:22 +0800 Subject: [PATCH 112/823] Change type of ValidationMethod from string to ValidationMethod instance (#1149) * refactor ValidationMethod * add * minor * refactor --- simple_integration_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/simple_integration_test.py b/simple_integration_test.py index 9e2e1acfeab..4efdd1eca88 100644 --- a/simple_integration_test.py +++ b/simple_integration_test.py @@ -165,7 +165,7 @@ def gen_rand_sample(): batch_size=batch_size, val_rdd=trainingData, trigger=EveryEpoch(), - val_method=["Top1Accuracy"] + val_method=[Top1Accuracy()] ) optimizer.optimize() @@ -197,7 +197,7 @@ def gen_rand_sample(): print(str(i) + "\n") print(len(p)) - test_results = trained_model.test(trainingData, 32, ["Top1Accuracy"]) + test_results = trained_model.test(trainingData, 32, [Top1Accuracy()]) for test_result in test_results: print(test_result) From 478f9345538d0114828dd7c8bc3094b2335a4ae4 Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Thu, 6 Jul 2017 03:01:09 -0500 Subject: [PATCH 113/823] fix python apis that are incompatible with scala apis (#1145) * fix python apis that are incompatible with scala apis * add test --- python/dllib/src/bigdl/dllib/nn/layer.py | 80 ++++++++++++++++++------ 1 file changed, 61 insertions(+), 19 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index db8b2791e5e..0f16c85af78 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -455,6 +455,15 @@ class Linear(Layer): creating: createL1Regularizer creating: createL1Regularizer creating: createLinear + >>> import numpy as np + >>> init_weight = np.random.randn(10, 100) + >>> init_bias = np.random.randn(10) + >>> init_grad_weight = np.zeros([10, 100]) + >>> init_grad_bias = np.zeros([10]) + >>> linear = Linear(100, 10, True, L1Regularizer(0.5), L1Regularizer(0.5), init_weight, init_bias, init_grad_weight, init_grad_bias) + creating: createL1Regularizer + creating: createL1Regularizer + creating: createLinear ''' def __init__(self, input_size, output_size, with_bias=True, wRegularizer=None, bRegularizer=None, @@ -589,6 +598,15 @@ class SpatialConvolution(Layer): creating: createL1Regularizer >>> spatialConvolution.setBRegularizer(L1Regularizer(0.5)) creating: createL1Regularizer + >>> import numpy as np + >>> init_weight = np.random.randn(1, 12, 6, 5, 5) + >>> init_bias = np.random.randn(12) + >>> init_grad_weight = np.zeros([1, 12, 6, 5, 5]) + >>> init_grad_bias = np.zeros([12]) + >>> spatialConvolution = SpatialConvolution(6, 12, 5, 5, 1, 1, 0, 0, 1, True, L1Regularizer(0.5), L1Regularizer(0.5), init_weight, init_bias, init_grad_weight, init_grad_bias) + creating: createL1Regularizer + creating: createL1Regularizer + creating: createSpatialConvolution ''' def __init__(self, @@ -602,7 +620,6 @@ def __init__(self, pad_h=0, n_group=1, propagate_back=True, - init_method="default", wRegularizer=None, bRegularizer=None, init_weight=None, @@ -621,7 +638,6 @@ def __init__(self, pad_h, n_group, propagate_back, - init_method, wRegularizer, bRegularizer, JTensor.from_ndarray(init_weight), @@ -946,6 +962,13 @@ class SpatialBatchNormalization(Layer): >>> spatialBatchNormalization = SpatialBatchNormalization(1) creating: createSpatialBatchNormalization + >>> import numpy as np + >>> init_weight = np.array([1.0]) + >>> init_grad_weight = np.array([0.0]) + >>> init_bias = np.array([0.0]) + >>> init_grad_bias = np.array([0.0]) + >>> spatialBatchNormalization = SpatialBatchNormalization(1, 1e-5, 0.1, True, init_weight, init_bias, init_grad_weight, init_grad_bias) + creating: createSpatialBatchNormalization ''' def __init__(self, @@ -953,12 +976,20 @@ def __init__(self, eps=1e-5, momentum=0.1, affine=True, + init_weight=None, + init_bias=None, + init_grad_weight=None, + init_grad_bias=None, bigdl_type="float"): super(SpatialBatchNormalization, self).__init__(None, bigdl_type, n_output, eps, momentum, - affine) + affine, + JTensor.from_ndarray(init_weight), + JTensor.from_ndarray(init_bias), + JTensor.from_ndarray(init_grad_weight), + JTensor.from_ndarray(init_grad_bias)) def set_init_method(self, weight_init_method = None, bias_init_method = None): callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, @@ -1147,18 +1178,33 @@ class BatchNormalization(Layer): >>> batchNormalization = BatchNormalization(1, 1e-5, 1e-5, True) creating: createBatchNormalization + >>> import numpy as np + >>> init_weight = np.random.randn(2) + >>> init_grad_weight = np.zeros([2]) + >>> init_bias = np.zeros([2]) + >>> init_grad_bias = np.zeros([2]) + >>> batchNormalization = BatchNormalization(2, 1e-5, 1e-5, True, init_weight, init_bias, init_grad_weight, init_grad_bias) + creating: createBatchNormalization ''' def __init__(self, n_output, eps=1e-5, momentum=0.1, affine=True, + init_weight=None, + init_bias=None, + init_grad_weight=None, + init_grad_bias=None, bigdl_type="float"): super(BatchNormalization, self).__init__(None, bigdl_type, n_output, eps, momentum, - affine) + affine, + JTensor.from_ndarray(init_weight), + JTensor.from_ndarray(init_bias), + JTensor.from_ndarray(init_grad_weight), + JTensor.from_ndarray(init_grad_bias)) def set_init_method(self, weight_init_method = None, bias_init_method = None): callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, weight_init_method, bias_init_method) @@ -2009,7 +2055,7 @@ class Min(Layer): ''' def __init__(self, - dim, + dim=1, num_input_dims=INTMIN, bigdl_type="float"): super(Min, self).__init__(None, bigdl_type, @@ -2461,7 +2507,7 @@ class SelectTable(Layer): internally to do so is recursive. - :param dimension: the dimension to be selected + :param index: the index to be selected >>> selectTable = SelectTable(1) @@ -2469,10 +2515,10 @@ class SelectTable(Layer): ''' def __init__(self, - dimension, + index, bigdl_type="float"): super(SelectTable, self).__init__(None, bigdl_type, - dimension) + index) class Sigmoid(Layer): @@ -2640,7 +2686,6 @@ def __init__(self, pad_h=0, dilation_w=1, dilation_h=1, - init_method='default', wRegularizer=None, bRegularizer=None, bigdl_type="float"): @@ -2655,7 +2700,6 @@ def __init__(self, pad_h, dilation_w, dilation_h, - init_method, wRegularizer, bRegularizer) @@ -2723,7 +2767,6 @@ def __init__(self, adj_h=0, n_group=1, no_bias=False, - init_method='default', wRegularizer=None, bRegularizer=None, bigdl_type="float"): @@ -2740,7 +2783,6 @@ def __init__(self, adj_h, n_group, no_bias, - init_method, wRegularizer, bRegularizer) def set_init_method(self, weight_init_method = None, bias_init_method = None): @@ -2768,7 +2810,6 @@ def __init__(self, pad_h=0, n_group=1, propagate_back=True, - init_method='default', bigdl_type="float"): super(SpatialShareConvolution, self).__init__(None, bigdl_type, n_input_plane, @@ -2780,8 +2821,7 @@ def __init__(self, pad_w, pad_h, n_group, - propagate_back, - init_method) + propagate_back) def set_init_method(self, weight_init_method = None, bias_init_method = None): callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, weight_init_method, bias_init_method) @@ -2826,7 +2866,6 @@ def __init__(self, pad_w=0, pad_h=0, with_bias=True, - init_method="default", bigdl_type="float"): super(VolumetricConvolution, self).__init__(None, bigdl_type, n_input_plane, @@ -2840,8 +2879,7 @@ def __init__(self, pad_t, pad_w, pad_h, - with_bias, - init_method) + with_bias) def set_init_method(self, weight_init_method = None, bias_init_method = None): callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, @@ -3198,13 +3236,17 @@ class Reverse(Layer): >>> reverse = Reverse() creating: createReverse + >>> reverse = Reverse(1, False) + creating: createReverse ''' def __init__(self, dimension=1, + is_inplace=False, bigdl_type="float"): super(Reverse, self).__init__(None, bigdl_type, - dimension) + dimension, + is_inplace) class Transpose(Layer): From 63adb1ced1e8d2662a596a58be8230429b6f09ab Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Thu, 6 Jul 2017 03:01:09 -0500 Subject: [PATCH 114/823] fix python apis that are incompatible with scala apis (#1145) * fix python apis that are incompatible with scala apis * add test --- python/dllib/src/bigdl/dllib/nn/layer.py | 80 ++++++++++++++++++------ 1 file changed, 61 insertions(+), 19 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index db8b2791e5e..0f16c85af78 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -455,6 +455,15 @@ class Linear(Layer): creating: createL1Regularizer creating: createL1Regularizer creating: createLinear + >>> import numpy as np + >>> init_weight = np.random.randn(10, 100) + >>> init_bias = np.random.randn(10) + >>> init_grad_weight = np.zeros([10, 100]) + >>> init_grad_bias = np.zeros([10]) + >>> linear = Linear(100, 10, True, L1Regularizer(0.5), L1Regularizer(0.5), init_weight, init_bias, init_grad_weight, init_grad_bias) + creating: createL1Regularizer + creating: createL1Regularizer + creating: createLinear ''' def __init__(self, input_size, output_size, with_bias=True, wRegularizer=None, bRegularizer=None, @@ -589,6 +598,15 @@ class SpatialConvolution(Layer): creating: createL1Regularizer >>> spatialConvolution.setBRegularizer(L1Regularizer(0.5)) creating: createL1Regularizer + >>> import numpy as np + >>> init_weight = np.random.randn(1, 12, 6, 5, 5) + >>> init_bias = np.random.randn(12) + >>> init_grad_weight = np.zeros([1, 12, 6, 5, 5]) + >>> init_grad_bias = np.zeros([12]) + >>> spatialConvolution = SpatialConvolution(6, 12, 5, 5, 1, 1, 0, 0, 1, True, L1Regularizer(0.5), L1Regularizer(0.5), init_weight, init_bias, init_grad_weight, init_grad_bias) + creating: createL1Regularizer + creating: createL1Regularizer + creating: createSpatialConvolution ''' def __init__(self, @@ -602,7 +620,6 @@ def __init__(self, pad_h=0, n_group=1, propagate_back=True, - init_method="default", wRegularizer=None, bRegularizer=None, init_weight=None, @@ -621,7 +638,6 @@ def __init__(self, pad_h, n_group, propagate_back, - init_method, wRegularizer, bRegularizer, JTensor.from_ndarray(init_weight), @@ -946,6 +962,13 @@ class SpatialBatchNormalization(Layer): >>> spatialBatchNormalization = SpatialBatchNormalization(1) creating: createSpatialBatchNormalization + >>> import numpy as np + >>> init_weight = np.array([1.0]) + >>> init_grad_weight = np.array([0.0]) + >>> init_bias = np.array([0.0]) + >>> init_grad_bias = np.array([0.0]) + >>> spatialBatchNormalization = SpatialBatchNormalization(1, 1e-5, 0.1, True, init_weight, init_bias, init_grad_weight, init_grad_bias) + creating: createSpatialBatchNormalization ''' def __init__(self, @@ -953,12 +976,20 @@ def __init__(self, eps=1e-5, momentum=0.1, affine=True, + init_weight=None, + init_bias=None, + init_grad_weight=None, + init_grad_bias=None, bigdl_type="float"): super(SpatialBatchNormalization, self).__init__(None, bigdl_type, n_output, eps, momentum, - affine) + affine, + JTensor.from_ndarray(init_weight), + JTensor.from_ndarray(init_bias), + JTensor.from_ndarray(init_grad_weight), + JTensor.from_ndarray(init_grad_bias)) def set_init_method(self, weight_init_method = None, bias_init_method = None): callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, @@ -1147,18 +1178,33 @@ class BatchNormalization(Layer): >>> batchNormalization = BatchNormalization(1, 1e-5, 1e-5, True) creating: createBatchNormalization + >>> import numpy as np + >>> init_weight = np.random.randn(2) + >>> init_grad_weight = np.zeros([2]) + >>> init_bias = np.zeros([2]) + >>> init_grad_bias = np.zeros([2]) + >>> batchNormalization = BatchNormalization(2, 1e-5, 1e-5, True, init_weight, init_bias, init_grad_weight, init_grad_bias) + creating: createBatchNormalization ''' def __init__(self, n_output, eps=1e-5, momentum=0.1, affine=True, + init_weight=None, + init_bias=None, + init_grad_weight=None, + init_grad_bias=None, bigdl_type="float"): super(BatchNormalization, self).__init__(None, bigdl_type, n_output, eps, momentum, - affine) + affine, + JTensor.from_ndarray(init_weight), + JTensor.from_ndarray(init_bias), + JTensor.from_ndarray(init_grad_weight), + JTensor.from_ndarray(init_grad_bias)) def set_init_method(self, weight_init_method = None, bias_init_method = None): callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, weight_init_method, bias_init_method) @@ -2009,7 +2055,7 @@ class Min(Layer): ''' def __init__(self, - dim, + dim=1, num_input_dims=INTMIN, bigdl_type="float"): super(Min, self).__init__(None, bigdl_type, @@ -2461,7 +2507,7 @@ class SelectTable(Layer): internally to do so is recursive. - :param dimension: the dimension to be selected + :param index: the index to be selected >>> selectTable = SelectTable(1) @@ -2469,10 +2515,10 @@ class SelectTable(Layer): ''' def __init__(self, - dimension, + index, bigdl_type="float"): super(SelectTable, self).__init__(None, bigdl_type, - dimension) + index) class Sigmoid(Layer): @@ -2640,7 +2686,6 @@ def __init__(self, pad_h=0, dilation_w=1, dilation_h=1, - init_method='default', wRegularizer=None, bRegularizer=None, bigdl_type="float"): @@ -2655,7 +2700,6 @@ def __init__(self, pad_h, dilation_w, dilation_h, - init_method, wRegularizer, bRegularizer) @@ -2723,7 +2767,6 @@ def __init__(self, adj_h=0, n_group=1, no_bias=False, - init_method='default', wRegularizer=None, bRegularizer=None, bigdl_type="float"): @@ -2740,7 +2783,6 @@ def __init__(self, adj_h, n_group, no_bias, - init_method, wRegularizer, bRegularizer) def set_init_method(self, weight_init_method = None, bias_init_method = None): @@ -2768,7 +2810,6 @@ def __init__(self, pad_h=0, n_group=1, propagate_back=True, - init_method='default', bigdl_type="float"): super(SpatialShareConvolution, self).__init__(None, bigdl_type, n_input_plane, @@ -2780,8 +2821,7 @@ def __init__(self, pad_w, pad_h, n_group, - propagate_back, - init_method) + propagate_back) def set_init_method(self, weight_init_method = None, bias_init_method = None): callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, weight_init_method, bias_init_method) @@ -2826,7 +2866,6 @@ def __init__(self, pad_w=0, pad_h=0, with_bias=True, - init_method="default", bigdl_type="float"): super(VolumetricConvolution, self).__init__(None, bigdl_type, n_input_plane, @@ -2840,8 +2879,7 @@ def __init__(self, pad_t, pad_w, pad_h, - with_bias, - init_method) + with_bias) def set_init_method(self, weight_init_method = None, bias_init_method = None): callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, @@ -3198,13 +3236,17 @@ class Reverse(Layer): >>> reverse = Reverse() creating: createReverse + >>> reverse = Reverse(1, False) + creating: createReverse ''' def __init__(self, dimension=1, + is_inplace=False, bigdl_type="float"): super(Reverse, self).__init__(None, bigdl_type, - dimension) + dimension, + is_inplace) class Transpose(Layer): From 6c52066df9c73a51952f0653a7c8108cd0400426 Mon Sep 17 00:00:00 2001 From: ding Date: Wed, 5 Jul 2017 11:56:46 -0400 Subject: [PATCH 115/823] support spatial convultion without bias --- python/dllib/src/bigdl/dllib/nn/layer.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 0f16c85af78..46932a40438 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -591,6 +591,7 @@ class SpatialConvolution(Layer): :param init_bias: the optional initial value for the bias :param init_grad_weight: the optional initial value for the grad_weight :param init_grad_bias: the optional initial value for the grad_bias + :param with_bias: the optional initial value for if need bias >>> spatialConvolution = SpatialConvolution(6, 12, 5, 5) creating: createSpatialConvolution @@ -626,6 +627,7 @@ def __init__(self, init_bias=None, init_grad_weight=None, init_grad_bias=None, + with_bias=True, bigdl_type="float"): super(SpatialConvolution, self).__init__(None, bigdl_type, n_input_plane, @@ -643,7 +645,8 @@ def __init__(self, JTensor.from_ndarray(init_weight), JTensor.from_ndarray(init_bias), JTensor.from_ndarray(init_grad_weight), - JTensor.from_ndarray(init_grad_bias)) + JTensor.from_ndarray(init_grad_bias), + with_bias) def set_init_method(self, weight_init_method = None, bias_init_method = None): callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, weight_init_method, bias_init_method) From f0c59af2249be6a27f1b1c59b421f7441da8fe44 Mon Sep 17 00:00:00 2001 From: ding Date: Wed, 5 Jul 2017 11:56:46 -0400 Subject: [PATCH 116/823] support spatial convultion without bias --- python/dllib/src/bigdl/dllib/nn/layer.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 0f16c85af78..46932a40438 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -591,6 +591,7 @@ class SpatialConvolution(Layer): :param init_bias: the optional initial value for the bias :param init_grad_weight: the optional initial value for the grad_weight :param init_grad_bias: the optional initial value for the grad_bias + :param with_bias: the optional initial value for if need bias >>> spatialConvolution = SpatialConvolution(6, 12, 5, 5) creating: createSpatialConvolution @@ -626,6 +627,7 @@ def __init__(self, init_bias=None, init_grad_weight=None, init_grad_bias=None, + with_bias=True, bigdl_type="float"): super(SpatialConvolution, self).__init__(None, bigdl_type, n_input_plane, @@ -643,7 +645,8 @@ def __init__(self, JTensor.from_ndarray(init_weight), JTensor.from_ndarray(init_bias), JTensor.from_ndarray(init_grad_weight), - JTensor.from_ndarray(init_grad_bias)) + JTensor.from_ndarray(init_grad_bias), + with_bias) def set_init_method(self, weight_init_method = None, bias_init_method = None): callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, weight_init_method, bias_init_method) From 5a85b29b2826f15640d8921527d7d28964b8cb50 Mon Sep 17 00:00:00 2001 From: Xianyan Date: Fri, 7 Jul 2017 11:17:18 +0800 Subject: [PATCH 117/823] Add plateau learning rate schedule, add trigger for loss and score (#1142) * add plateau learning rate schedule * add score trigger * Add plateau learning rate schedule, add trigger for loss and score * add some document and adjust code * update validate state * remove auto * update readme * add python API and update docs * fix python test --- .../dllib/src/bigdl/dllib/optim/optimizer.py | 70 +++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/optim/optimizer.py b/python/dllib/src/bigdl/dllib/optim/optimizer.py index 762fcf4090a..503d638a601 100644 --- a/python/dllib/src/bigdl/dllib/optim/optimizer.py +++ b/python/dllib/src/bigdl/dllib/optim/optimizer.py @@ -161,6 +161,42 @@ def __init__(self, interval, bigdl_type="float"): JavaValue.__init__(self, None, bigdl_type, interval) +class MaxScore(JavaValue): + """ + A trigger that triggers an action when validation score larger than "max" score + + + >>> maxScore = MaxScore(0.4) + creating: createMaxScore + """ + def __init__(self, max, bigdl_type="float"): + """ + Create a MaxScore trigger. + + + :param max: max score + """ + JavaValue.__init__(self, None, bigdl_type, max) + + +class MinLoss(JavaValue): + """ + A trigger that triggers an action when training loss less than "min" loss + + + >>> minLoss = MinLoss(0.1) + creating: createMinLoss + """ + def __init__(self, min, bigdl_type="float"): + """ + Create a MinLoss trigger. + + + :param min: min loss + """ + JavaValue.__init__(self, None, bigdl_type, min) + + class Poly(JavaValue): """ A learning rate decay policy, where the effective learning rate @@ -225,6 +261,40 @@ class Default(JavaValue): def __init__(self, bigdl_type="float"): JavaValue.__init__(self, None, bigdl_type) + +class Plateau(JavaValue): + """ + Plateau is the learning rate schedule when a metric has stopped improving. + Models often benefit from reducing the learning rate by a factor of 2-10 + once learning stagnates. It monitors a quantity and if no improvement + is seen for a 'patience' number of epochs, the learning rate is reduced. + + :param monitor quantity to be monitored, can be Loss or score + :param factor factor by which the learning rate will be reduced. new_lr = lr * factor + :param patience number of epochs with no improvement after which learning rate will be reduced. + :param mode one of {min, max}. + In min mode, lr will be reduced when the quantity monitored has stopped decreasing; + in max mode it will be reduced when the quantity monitored has stopped increasing + :param epsilon threshold for measuring the new optimum, to only focus on significant changes. + :param cooldown number of epochs to wait before resuming normal operation + after lr has been reduced. + :param min_lr lower bound on the learning rate. + + >>> plateau = Plateau("score") + creating: createPlateau + """ + def __init__(self, + monitor, + factor=0.1, + patience=10, + mode="min", + epsilon=1e-4, + cooldown=0, + min_lr=0.0, + bigdl_type="float"): + JavaValue.__init__(self, None, bigdl_type, monitor, factor, patience, mode, epsilon, + cooldown, min_lr) + class SGD(JavaValue): """ A plain implementation of SGD From 5189c2e25ff37f65ab50ef2bdb343df511f50de1 Mon Sep 17 00:00:00 2001 From: Xianyan Date: Fri, 7 Jul 2017 11:17:18 +0800 Subject: [PATCH 118/823] Add plateau learning rate schedule, add trigger for loss and score (#1142) * add plateau learning rate schedule * add score trigger * Add plateau learning rate schedule, add trigger for loss and score * add some document and adjust code * update validate state * remove auto * update readme * add python API and update docs * fix python test --- .../dllib/src/bigdl/dllib/optim/optimizer.py | 70 +++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/optim/optimizer.py b/python/dllib/src/bigdl/dllib/optim/optimizer.py index 762fcf4090a..503d638a601 100644 --- a/python/dllib/src/bigdl/dllib/optim/optimizer.py +++ b/python/dllib/src/bigdl/dllib/optim/optimizer.py @@ -161,6 +161,42 @@ def __init__(self, interval, bigdl_type="float"): JavaValue.__init__(self, None, bigdl_type, interval) +class MaxScore(JavaValue): + """ + A trigger that triggers an action when validation score larger than "max" score + + + >>> maxScore = MaxScore(0.4) + creating: createMaxScore + """ + def __init__(self, max, bigdl_type="float"): + """ + Create a MaxScore trigger. + + + :param max: max score + """ + JavaValue.__init__(self, None, bigdl_type, max) + + +class MinLoss(JavaValue): + """ + A trigger that triggers an action when training loss less than "min" loss + + + >>> minLoss = MinLoss(0.1) + creating: createMinLoss + """ + def __init__(self, min, bigdl_type="float"): + """ + Create a MinLoss trigger. + + + :param min: min loss + """ + JavaValue.__init__(self, None, bigdl_type, min) + + class Poly(JavaValue): """ A learning rate decay policy, where the effective learning rate @@ -225,6 +261,40 @@ class Default(JavaValue): def __init__(self, bigdl_type="float"): JavaValue.__init__(self, None, bigdl_type) + +class Plateau(JavaValue): + """ + Plateau is the learning rate schedule when a metric has stopped improving. + Models often benefit from reducing the learning rate by a factor of 2-10 + once learning stagnates. It monitors a quantity and if no improvement + is seen for a 'patience' number of epochs, the learning rate is reduced. + + :param monitor quantity to be monitored, can be Loss or score + :param factor factor by which the learning rate will be reduced. new_lr = lr * factor + :param patience number of epochs with no improvement after which learning rate will be reduced. + :param mode one of {min, max}. + In min mode, lr will be reduced when the quantity monitored has stopped decreasing; + in max mode it will be reduced when the quantity monitored has stopped increasing + :param epsilon threshold for measuring the new optimum, to only focus on significant changes. + :param cooldown number of epochs to wait before resuming normal operation + after lr has been reduced. + :param min_lr lower bound on the learning rate. + + >>> plateau = Plateau("score") + creating: createPlateau + """ + def __init__(self, + monitor, + factor=0.1, + patience=10, + mode="min", + epsilon=1e-4, + cooldown=0, + min_lr=0.0, + bigdl_type="float"): + JavaValue.__init__(self, None, bigdl_type, monitor, factor, patience, mode, epsilon, + cooldown, min_lr) + class SGD(JavaValue): """ A plain implementation of SGD From faa216a1b769ce2e0ea72354f318bd169bd971b0 Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Fri, 7 Jul 2017 13:22:18 +0800 Subject: [PATCH 119/823] Fix reference error and add script for packing pypi package (#1172) * cycle reference * script for packing whl --- python/dllib/src/bigdl/dllib/utils/engine.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/utils/engine.py b/python/dllib/src/bigdl/dllib/utils/engine.py index 42a2455f378..998887a8689 100644 --- a/python/dllib/src/bigdl/dllib/utils/engine.py +++ b/python/dllib/src/bigdl/dllib/utils/engine.py @@ -26,6 +26,7 @@ def __prepare_spark_env(): raise ValueError( """Could not find Spark. Pls make sure SPARK_HOME env is set: export SPARK_HOME=path to your spark home directory""") + print("Using %s" % spark_home) py4j = glob.glob(os.path.join(spark_home, 'python/lib', 'py4j-*.zip'))[0] pyspark = glob.glob(os.path.join(spark_home, 'python/lib', 'pyspark*.zip'))[0] sys.path.insert(0, py4j) @@ -33,13 +34,13 @@ def __prepare_spark_env(): def __prepare_bigdl_env(): - import bigdl.nn.layer - jar_dir = os.path.abspath(bigdl.nn.layer.__file__ + "/../../") + jar_dir = os.path.abspath(__file__ + "/../../") jar_paths = glob.glob(os.path.join(jar_dir, "share/lib/*.jar")) conf_paths = glob.glob(os.path.join(jar_dir, "share/conf/*.conf")) def append_path(env_var_name, path): try: + print("Adding %s to %s" % (jar_paths[0], env_var_name)) os.environ[env_var_name] = path + ":" + os.environ[ env_var_name] # noqa except KeyError: @@ -48,10 +49,6 @@ def append_path(env_var_name, path): if conf_paths and conf_paths: assert len(conf_paths) == 1, "Expecting one jar: %s" % len(jar_paths) assert len(conf_paths) == 1, "Expecting one conf: %s" % len(conf_paths) - print("Adding %s to spark.driver.extraClassPath" % jar_paths[0]) - print("Adding %s to spark.executor.extraClassPath" % jar_paths[0]) - append_path("spark.driver.extraClassPath", jar_paths[0]) - append_path("spark.executor.extraClassPath", jar_paths[0]) append_path("SPARK_CLASSPATH", jar_paths[0]) print("Prepending %s to sys.path" % conf_paths[0]) sys.path.insert(0, conf_paths[0]) From 0dbab68649c79f71c189ff58236d7b4ae0e5c736 Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Fri, 7 Jul 2017 13:22:18 +0800 Subject: [PATCH 120/823] Fix reference error and add script for packing pypi package (#1172) * cycle reference * script for packing whl --- python/dllib/src/bigdl/utils/engine.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/python/dllib/src/bigdl/utils/engine.py b/python/dllib/src/bigdl/utils/engine.py index 42a2455f378..998887a8689 100644 --- a/python/dllib/src/bigdl/utils/engine.py +++ b/python/dllib/src/bigdl/utils/engine.py @@ -26,6 +26,7 @@ def __prepare_spark_env(): raise ValueError( """Could not find Spark. Pls make sure SPARK_HOME env is set: export SPARK_HOME=path to your spark home directory""") + print("Using %s" % spark_home) py4j = glob.glob(os.path.join(spark_home, 'python/lib', 'py4j-*.zip'))[0] pyspark = glob.glob(os.path.join(spark_home, 'python/lib', 'pyspark*.zip'))[0] sys.path.insert(0, py4j) @@ -33,13 +34,13 @@ def __prepare_spark_env(): def __prepare_bigdl_env(): - import bigdl.nn.layer - jar_dir = os.path.abspath(bigdl.nn.layer.__file__ + "/../../") + jar_dir = os.path.abspath(__file__ + "/../../") jar_paths = glob.glob(os.path.join(jar_dir, "share/lib/*.jar")) conf_paths = glob.glob(os.path.join(jar_dir, "share/conf/*.conf")) def append_path(env_var_name, path): try: + print("Adding %s to %s" % (jar_paths[0], env_var_name)) os.environ[env_var_name] = path + ":" + os.environ[ env_var_name] # noqa except KeyError: @@ -48,10 +49,6 @@ def append_path(env_var_name, path): if conf_paths and conf_paths: assert len(conf_paths) == 1, "Expecting one jar: %s" % len(jar_paths) assert len(conf_paths) == 1, "Expecting one conf: %s" % len(conf_paths) - print("Adding %s to spark.driver.extraClassPath" % jar_paths[0]) - print("Adding %s to spark.executor.extraClassPath" % jar_paths[0]) - append_path("spark.driver.extraClassPath", jar_paths[0]) - append_path("spark.executor.extraClassPath", jar_paths[0]) append_path("SPARK_CLASSPATH", jar_paths[0]) print("Prepending %s to sys.path" % conf_paths[0]) sys.path.insert(0, conf_paths[0]) From 81f7a26b9bd7a50e60cddc26f8632169b41d4343 Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Fri, 7 Jul 2017 13:22:18 +0800 Subject: [PATCH 121/823] Fix reference error and add script for packing pypi package (#1172) * cycle reference * script for packing whl --- python/dllib/test/dev/release/release.sh | 71 ++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100755 python/dllib/test/dev/release/release.sh diff --git a/python/dllib/test/dev/release/release.sh b/python/dllib/test/dev/release/release.sh new file mode 100755 index 00000000000..c0b9b574795 --- /dev/null +++ b/python/dllib/test/dev/release/release.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash + +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -e +RUN_SCRIPT_DIR=$(cd $(dirname $0) ; pwd) +echo $RUN_SCRIPT_DIR +BIGDL_DIR="$(cd ${RUN_SCRIPT_DIR}/../../../../; pwd)" +echo $BIGDL_DIR +BIGDL_PYTHON_DIR="$(cd ${RUN_SCRIPT_DIR}/../../../../pyspark; pwd)" +echo $BIGDL_PYTHON_DIR + +if (( $# < 2)); then + echo "Bad parameters. Uasge: release.sh mac spark_2.x" + exit -1 +fi + +platform=$1 +spark_profile=$2 +quick=$3 + +bigdl_version=$(python -c "exec(open('$BIGDL_DIR/pyspark/bigdl/version.py').read()); print(__version__)") + +cd ${BIGDL_DIR} +if [ "$platform" == "mac" ]; then + echo "Building bigdl for mac system" + dist_profile="-P mac -P $spark_profile" + verbose_pname="macosx_10_11_x86_64" +elif [ "$platform" == "linux" ]; then + echo "Building bigdl for linux system" + dist_profile="-P $spark_profile" + verbose_pname="manylinux1_x86_64" +else + echo "unsupport platform" +fi + +bigdl_build_command="${BIGDL_DIR}/make-dist.sh ${dist_profile}" +if [ "$quick" == "true" ]; then + echo "Skip disting BigDL" +else + echo "Dist BigDL: $bigdl_build_command" + $bigdl_build_command +fi + +cd $BIGDL_PYTHON_DIR +sdist_command="python setup.py sdist" +echo "packing source code: ${sdist_command}" +$sdist_command + +wheel_command="python setup.py bdist_wheel --plat-name ${verbose_pname}" +echo "Packing python distribution: $wheel_command" +${wheel_command} + +upload_command="twine upload dist/BigDL-${bigdl_version}-py2.py3-none-${verbose_pname}.whl" +echo "Please manually upload with this command: $upload_command" + + From 66be3e088dc6f8bfe2ec21af127b0cfd88f68f7b Mon Sep 17 00:00:00 2001 From: yunhuiguo Date: Fri, 7 Jul 2017 15:59:50 +0800 Subject: [PATCH 122/823] Add MAE for validation and update the interface of movielens dataset module (#1148) * add MAE for validation upadte MAE add MAE test * upate movielens dataset module update movielens update movielens update update update update update update resolve conflicts update * add MAE in python validation method update update --- .../src/bigdl/dllib/feature/dataset/movielens.py | 14 +++++++++++--- python/dllib/src/bigdl/dllib/optim/optimizer.py | 9 +++++++++ 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/feature/dataset/movielens.py b/python/dllib/src/bigdl/dllib/feature/dataset/movielens.py index 251ac35e06c..93bcefc831f 100644 --- a/python/dllib/src/bigdl/dllib/feature/dataset/movielens.py +++ b/python/dllib/src/bigdl/dllib/feature/dataset/movielens.py @@ -22,7 +22,6 @@ from bigdl.dataset import base SOURCE_URL = 'http://files.grouplens.org/datasets/movielens/' - def read_data_sets(data_dir): """ Parse or download movielens 1m data if train_dir is empty. @@ -30,7 +29,6 @@ def read_data_sets(data_dir): :param data_dir: The directory storing the movielens data :return: a 2D numpy array with user index and item index in each row """ - WHOLE_DATA = 'ml-1m.zip' local_file = base.maybe_download(WHOLE_DATA, data_dir, SOURCE_URL + WHOLE_DATA) zip_ref = zipfile.ZipFile(local_file, 'r') @@ -40,9 +38,19 @@ def read_data_sets(data_dir): zip_ref.extractall(data_dir) zip_ref.close() rating_files = os.path.join(extracted_to,"ratings.dat") - rating_list = [i.strip().split("::")[:2] for i in open(rating_files,"r").readlines()] + + rating_list = [i.strip().split("::") for i in open(rating_files,"r").readlines()] movielens_data = np.array(rating_list).astype(int) return movielens_data +def get_id_pairs(data_dir): + movielens_data = read_data_sets(data_dir) + return movielens_data[:, 0:2] + +def get_id_ratings(data_dir): + movielens_data = read_data_sets(data_dir) + return movielens_data[:, 0:3] + + if __name__ == "__main__": movielens_data = read_data_sets("/tmp/movielens/") diff --git a/python/dllib/src/bigdl/dllib/optim/optimizer.py b/python/dllib/src/bigdl/dllib/optim/optimizer.py index 503d638a601..428fdcfbe83 100644 --- a/python/dllib/src/bigdl/dllib/optim/optimizer.py +++ b/python/dllib/src/bigdl/dllib/optim/optimizer.py @@ -74,6 +74,15 @@ def __init__(self, cri=None, bigdl_type="float"): cri = ClassNLLCriterion() JavaValue.__init__(self, None, bigdl_type, cri) +class MAE(JavaValue): + """ + This evaluation method calculates the mean absolute error of output with respect to target. + + >>> mae = MAE() + creating: createMAE + """ + def __init__(self, bigdl_type="float"): + JavaValue.__init__(self, None, bigdl_type) class MaxIteration(JavaValue): """ From 023c384ecbcb4cdb226ad27c06de440b76ead60c Mon Sep 17 00:00:00 2001 From: yunhuiguo Date: Fri, 7 Jul 2017 15:59:50 +0800 Subject: [PATCH 123/823] Add MAE for validation and update the interface of movielens dataset module (#1148) * add MAE for validation upadte MAE add MAE test * upate movielens dataset module update movielens update movielens update update update update update update resolve conflicts update * add MAE in python validation method update update --- .../src/bigdl/dllib/feature/dataset/movielens.py | 14 +++++++++++--- python/dllib/src/bigdl/dllib/optim/optimizer.py | 9 +++++++++ 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/feature/dataset/movielens.py b/python/dllib/src/bigdl/dllib/feature/dataset/movielens.py index 251ac35e06c..93bcefc831f 100644 --- a/python/dllib/src/bigdl/dllib/feature/dataset/movielens.py +++ b/python/dllib/src/bigdl/dllib/feature/dataset/movielens.py @@ -22,7 +22,6 @@ from bigdl.dataset import base SOURCE_URL = 'http://files.grouplens.org/datasets/movielens/' - def read_data_sets(data_dir): """ Parse or download movielens 1m data if train_dir is empty. @@ -30,7 +29,6 @@ def read_data_sets(data_dir): :param data_dir: The directory storing the movielens data :return: a 2D numpy array with user index and item index in each row """ - WHOLE_DATA = 'ml-1m.zip' local_file = base.maybe_download(WHOLE_DATA, data_dir, SOURCE_URL + WHOLE_DATA) zip_ref = zipfile.ZipFile(local_file, 'r') @@ -40,9 +38,19 @@ def read_data_sets(data_dir): zip_ref.extractall(data_dir) zip_ref.close() rating_files = os.path.join(extracted_to,"ratings.dat") - rating_list = [i.strip().split("::")[:2] for i in open(rating_files,"r").readlines()] + + rating_list = [i.strip().split("::") for i in open(rating_files,"r").readlines()] movielens_data = np.array(rating_list).astype(int) return movielens_data +def get_id_pairs(data_dir): + movielens_data = read_data_sets(data_dir) + return movielens_data[:, 0:2] + +def get_id_ratings(data_dir): + movielens_data = read_data_sets(data_dir) + return movielens_data[:, 0:3] + + if __name__ == "__main__": movielens_data = read_data_sets("/tmp/movielens/") diff --git a/python/dllib/src/bigdl/dllib/optim/optimizer.py b/python/dllib/src/bigdl/dllib/optim/optimizer.py index 503d638a601..428fdcfbe83 100644 --- a/python/dllib/src/bigdl/dllib/optim/optimizer.py +++ b/python/dllib/src/bigdl/dllib/optim/optimizer.py @@ -74,6 +74,15 @@ def __init__(self, cri=None, bigdl_type="float"): cri = ClassNLLCriterion() JavaValue.__init__(self, None, bigdl_type, cri) +class MAE(JavaValue): + """ + This evaluation method calculates the mean absolute error of output with respect to target. + + >>> mae = MAE() + creating: createMAE + """ + def __init__(self, bigdl_type="float"): + JavaValue.__init__(self, None, bigdl_type) class MaxIteration(JavaValue): """ From 6feeeb389effe28721e70f1550ae76425961c6db Mon Sep 17 00:00:00 2001 From: yunhuiguo Date: Fri, 7 Jul 2017 15:59:50 +0800 Subject: [PATCH 124/823] Add MAE for validation and update the interface of movielens dataset module (#1148) * add MAE for validation upadte MAE add MAE test * upate movielens dataset module update movielens update movielens update update update update update update resolve conflicts update * add MAE in python validation method update update --- simple_integration_test.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/simple_integration_test.py b/simple_integration_test.py index 4efdd1eca88..1470dd6b767 100644 --- a/simple_integration_test.py +++ b/simple_integration_test.py @@ -337,9 +337,16 @@ def test_rng(self): def test_movielens(self): movielens_data = movielens.read_data_sets("/tmp/movielens/") - ground_label = np.array([[1, 1193], [1, 661]]) + id_pairs = movielens.get_id_pairs("/tmp/movielens/") + id_ratings = movielens.get_id_ratings("/tmp/movielens/") + + ground_data = np.array([[1, 1193, 5, 978300760], [1, 661, 3, 978302109]]) + ground_pairs = np.array([[1, 1193], [1, 661]]) + ground_ratings = np.array([[1, 1193, 5], [1, 661, 3]]) for i in range(0, 2): - self.assertTrue(np.allclose(movielens_data[i], ground_label[i], atol=1e-6, rtol=0)) + self.assertTrue(np.allclose(movielens_data[i], ground_data[i], atol=1e-6, rtol=0)) + self.assertTrue(np.allclose(id_pairs[i], ground_pairs[i], atol=1e-6, rtol=0)) + self.assertTrue(np.allclose(id_ratings[i], ground_ratings[i], atol=1e-6, rtol=0)) if __name__ == "__main__": unittest.main(failfast=True) From e52b99a8f7a31257ca635e3407dc6ac3b2672005 Mon Sep 17 00:00:00 2001 From: ding Date: Mon, 26 Jun 2017 11:08:01 -0400 Subject: [PATCH 125/823] support convlstm and convolution without bias --- python/dllib/src/bigdl/dllib/nn/layer.py | 27 ++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 46932a40438..8e922c04f62 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -3434,6 +3434,33 @@ class Pack(Layer): def __init__(self, dimension, bigdl_type="float"): super(Pack, self).__init__(None, bigdl_type, dimension) +class ConvLSTMPeephole(Layer): + ''' + +| Convolution Long Short Term Memory architecture with peephole. +| Ref. A.: https://arxiv.org/abs/1506.04214 (blueprint for this module) +| B. https://github.com/viorik/ConvLSTM + + :param input_size: the size of each input + :param hidden_size: Hidden unit size in the LSTM + :param kernel_i Convolutional filter size to convolve input + :param kernel_c Convolutional filter size to convolve cell + :param stride The step of the convolution + :param wRegularizer: instance of [[Regularizer]](eg. L1 or L2 regularization), applied to the input weights matrices + :param uRegularizer: instance [[Regularizer]](eg. L1 or L2 regularization), applied to the recurrent weights matrices + :param bRegularizer: instance of [[Regularizer]]applied to the bias. + + >>> convlstm = ConvLSTMPeephole(4, 3, 3, 3, 1, L1Regularizer(0.5), L1Regularizer(0.5), L1Regularizer(0.5)) + creating: createL1Regularizer + creating: createL1Regularizer + creating: createL1Regularizer + creating: createConvLSTMPeephole + ''' + + def __init__(self, input_size, hidden_size, kernel_i, kernel_c, stride, wRegularizer=None, uRegularizer=None, + bRegularizer=None, with_peephole=False, bigdl_type="float"): + super(ConvLSTMPeephole, self).__init__(None, bigdl_type, input_size, hidden_size, kernel_i, kernel_c, stride, + wRegularizer, uRegularizer, bRegularizer, with_peephole) def _test(): import doctest From 3a3ee94a0baad26da86be7e4cd9fc6ac62a3a365 Mon Sep 17 00:00:00 2001 From: ding Date: Mon, 26 Jun 2017 11:08:01 -0400 Subject: [PATCH 126/823] support convlstm and convolution without bias --- python/dllib/src/bigdl/dllib/nn/layer.py | 27 ++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 46932a40438..8e922c04f62 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -3434,6 +3434,33 @@ class Pack(Layer): def __init__(self, dimension, bigdl_type="float"): super(Pack, self).__init__(None, bigdl_type, dimension) +class ConvLSTMPeephole(Layer): + ''' + +| Convolution Long Short Term Memory architecture with peephole. +| Ref. A.: https://arxiv.org/abs/1506.04214 (blueprint for this module) +| B. https://github.com/viorik/ConvLSTM + + :param input_size: the size of each input + :param hidden_size: Hidden unit size in the LSTM + :param kernel_i Convolutional filter size to convolve input + :param kernel_c Convolutional filter size to convolve cell + :param stride The step of the convolution + :param wRegularizer: instance of [[Regularizer]](eg. L1 or L2 regularization), applied to the input weights matrices + :param uRegularizer: instance [[Regularizer]](eg. L1 or L2 regularization), applied to the recurrent weights matrices + :param bRegularizer: instance of [[Regularizer]]applied to the bias. + + >>> convlstm = ConvLSTMPeephole(4, 3, 3, 3, 1, L1Regularizer(0.5), L1Regularizer(0.5), L1Regularizer(0.5)) + creating: createL1Regularizer + creating: createL1Regularizer + creating: createL1Regularizer + creating: createConvLSTMPeephole + ''' + + def __init__(self, input_size, hidden_size, kernel_i, kernel_c, stride, wRegularizer=None, uRegularizer=None, + bRegularizer=None, with_peephole=False, bigdl_type="float"): + super(ConvLSTMPeephole, self).__init__(None, bigdl_type, input_size, hidden_size, kernel_i, kernel_c, stride, + wRegularizer, uRegularizer, bRegularizer, with_peephole) def _test(): import doctest From e786319c6197f1d4e82c9ea9b2e7ea8a3e91acec Mon Sep 17 00:00:00 2001 From: ding Date: Mon, 26 Jun 2017 11:26:10 -0400 Subject: [PATCH 127/823] code rebase --- python/dllib/src/bigdl/dllib/nn/layer.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 8e922c04f62..ce87084f8ab 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -3449,7 +3449,8 @@ class ConvLSTMPeephole(Layer): :param wRegularizer: instance of [[Regularizer]](eg. L1 or L2 regularization), applied to the input weights matrices :param uRegularizer: instance [[Regularizer]](eg. L1 or L2 regularization), applied to the recurrent weights matrices :param bRegularizer: instance of [[Regularizer]]applied to the bias. - + :param with_peephold: whether use last cell status control a gate. + >>> convlstm = ConvLSTMPeephole(4, 3, 3, 3, 1, L1Regularizer(0.5), L1Regularizer(0.5), L1Regularizer(0.5)) creating: createL1Regularizer creating: createL1Regularizer From 73fba05f9e5a960e328f7cd9f34cf72146b0977c Mon Sep 17 00:00:00 2001 From: ding Date: Mon, 26 Jun 2017 11:26:10 -0400 Subject: [PATCH 128/823] code rebase --- python/dllib/src/bigdl/dllib/nn/layer.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 8e922c04f62..ce87084f8ab 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -3449,7 +3449,8 @@ class ConvLSTMPeephole(Layer): :param wRegularizer: instance of [[Regularizer]](eg. L1 or L2 regularization), applied to the input weights matrices :param uRegularizer: instance [[Regularizer]](eg. L1 or L2 regularization), applied to the recurrent weights matrices :param bRegularizer: instance of [[Regularizer]]applied to the bias. - + :param with_peephold: whether use last cell status control a gate. + >>> convlstm = ConvLSTMPeephole(4, 3, 3, 3, 1, L1Regularizer(0.5), L1Regularizer(0.5), L1Regularizer(0.5)) creating: createL1Regularizer creating: createL1Regularizer From 7af4524478eab8070c8c25da36a7a20811b032b0 Mon Sep 17 00:00:00 2001 From: ding Date: Wed, 28 Jun 2017 11:50:25 -0400 Subject: [PATCH 129/823] code clean --- .../bigdl/dllib/feature/dataset/sentence.py | 42 +++++ .../src/bigdl/dllib/models/rnn/README.md | 110 +++++++++++ .../src/bigdl/dllib/models/rnn/rnnexample.py | 171 ++++++++++++++++++ .../dllib/src/bigdl/dllib/optim/optimizer.py | 2 + 4 files changed, 325 insertions(+) create mode 100644 python/dllib/src/bigdl/dllib/feature/dataset/sentence.py create mode 100644 python/dllib/src/bigdl/dllib/models/rnn/README.md create mode 100644 python/dllib/src/bigdl/dllib/models/rnn/rnnexample.py diff --git a/python/dllib/src/bigdl/dllib/feature/dataset/sentence.py b/python/dllib/src/bigdl/dllib/feature/dataset/sentence.py new file mode 100644 index 00000000000..46627ab0746 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/feature/dataset/sentence.py @@ -0,0 +1,42 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import nltk +import os +import itertools +import sys +from nltk.tokenize import word_tokenize + +def read_localfile(fileName): + lines = [] + with open(fileName) as f: + for line in f: + lines.append(line) + f.close() + return lines + +def sentences_split(line): + nltk.data.path.append(os.environ.get('PWD')) + sent_tokenizer = nltk.tokenize.PunktSentenceTokenizer() + sentenized = sent_tokenizer.tokenize(line) + return sentenized + +def sentences_bipadding(sent): + return "SENTENCESTART " + sent + " SENTENCEEND" + +def sentence_tokenizer(sentences): + tokenized_sents = nltk.word_tokenize(sentences) + return tokenized_sents \ No newline at end of file diff --git a/python/dllib/src/bigdl/dllib/models/rnn/README.md b/python/dllib/src/bigdl/dllib/models/rnn/README.md new file mode 100644 index 00000000000..55bb8e314c0 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/models/rnn/README.md @@ -0,0 +1,110 @@ +# Recurrent Neural Network + +Model that supports sequence to sequence processing + +This is an implementation of Simple Recurrent Neural Networks for Language Modeling. Please refer to the [original paper](http://www.fit.vutbr.cz/research/groups/speech/publi/2010/mikolov_interspeech2010_IS100722.pdf) by Tomas Mikolov. + +The implementation of RNNs in this code is referred to in the [Keras Recurrent](https://keras.io/layers/recurrent/) documentation. + + +## Get the BigDL files + +Please build BigDL referring to [Build Page](https://github.com/intel-analytics/BigDL/wiki/Build-Page). + + +## Prepare the Input Data +You can download the Tiny Shakespeare Texts corpus from [here](https://raw.githubusercontent.com/karpathy/char-rnn/master/data/tinyshakespeare/input.txt). + +After downloading the text, separate it into train.txt and val.txt. In our case, we just select 80 percentage of the input to be train and remaining 20 percentage to be val. +Please upload the directory will contains train.txt and val.txt to hdfs. The program will later read in the original text file from hdfs. +```shell +export LANG=en_US.UTF-8 +head -n 8000 input.txt > val.txt +tail -n +8000 input.txt > train.txt +``` + +### Sample Text + +The input text may look as follows: + +``` + First Citizen: + Before we proceed any further, hear me speak. + + All: + Speak, speak. + + First Citizen: + You are all resolved rather to die than to famish? +``` +## Preprocessing + +The sentences_split, sentence_tokenizer use [NLTK Toolkit](http://www.nltk.org/). NLTK is included +in prepared virtual env. As Nltk expected english.pickle to tokenizer in the current location under +tokenizers/punkt/english.pickle. we need download punkt and navigate into the folder for packaging +``` +$python +>>> import nltk +>>> nltk.download('punkt') +``` +then you can find nltk_data folder, zip file with tokenizer.zip#tokenizer +``` +$ cd nltk_data/tokenizers/ +$ zip -r ../../tokenizers.zip * +$ cd ../../ +``` + +### Sample Sequence of Processed Data +``` + 3998,3875,3690,3999 + 3998,3171,3958,2390,3832,3202,3855,3983,3883,3999 + 3998,3667,3999 + 3998,3151,3883,3999 + 3998,3875,3690,3999 +``` + +## Train the Model +Example command: +```bash +./dist/bin/bigdl.sh -- \ + +BigDL_HOME=... +SPARK_HOME=... +MASTER=yarn +PYTHON_API_PATH=${BigDL_HOME}/dist/lib/bigdl-VERSION-python-api.zip +BigDL_JAR_PATH=${BigDL_HOME}/dist/lib/bigdl-VERSION-jar-with-dependencies.jar +PYTHONPATH=${PYTHON_API_PATH}:$PYTHONPATH +# set http_proxy if you need proxy to access internet +http_proxy=... +# Please refer ${BigDL_HOME}/pyspark/python_package/README.md to see how to create virtual env +PYSPARK_DRIVER_PYTHON=./venv/bin/python PYSPARK_PYTHON=./venv.zip/venv/bin/python ${SPARK_HOME}/bin/spark-submit \ + --master ${MASTER} \ + --deploy-mode client \ + --conf spark.executorEnv.http_proxy=${http_proxy} \ + --driver-memory 10g \ + --executor-cores 1 \ + --executor-memory 60g \ + --py-files ${PYTHON_API_PATH} \ + --properties-file ${BigDL_HOME}/dist/conf/spark-bigdl.conf \ + --jars ${BigDL_JAR_PATH} \ + --archives venv.zip,tokenizers.zip#tokenizers \ + --conf spark.driver.extraClassPath=${BigDL_JAR_PATH} \ + --conf spark.executor.extraClassPath=bigdl-version-jar-with-dependencies.jar \ + --conf spark.yarn.appMasterEnv.NLTK_DATA=./ \ + --num-executors 1 \ + ${BigDL_HOME}/pyspark/dl/models/rnn/rnnexample.py --folder hdfs://almaren-node-001:9000/rnn/ --batchSize 12 +``` + +* `--folder` hdfs directory where train.txt and val.txt are located + +* `--batchSize` option can be used to set batch size, the default value is 128. + +## Expected Training Output +Users can see the Loss of the model printed by the program. The Loss, in this case, is the perplexity of the language model. The lower, the better. +``` +INFO DistriOptimizer$:247 - [Epoch 1 0/6879][Iteration 1][Wall Clock 0.0s] Train 12 in 4.926679827seconds. Throughput is 2.4357176 records/second. Loss is 8.277311. Current learning rate is 0.1. +INFO DistriOptimizer$:247 - [Epoch 1 12/6879][Iteration 2][Wall Clock 4.926679827s] Train 12 in 2.622718594seconds. Throughput is 4.575405 records/second. Loss is 8.07377. Current learning rate is 0.1. +INFO DistriOptimizer$:247 - [Epoch 1 24/6879][Iteration 3][Wall Clock 7.549398421s] Train 12 in 2.478575083seconds. Throughput is 4.8414917 records/second. Loss is 7.8527904. Current learning rate is 0.1. +INFO DistriOptimizer$:247 - [Epoch 1 36/6879][Iteration 4][Wall Clock 10.027973504s] Train 12 in 2.475138056seconds. Throughput is 4.8482146 records/second. Loss is 7.581617. Current learning rate is 0.1. +... +``` \ No newline at end of file diff --git a/python/dllib/src/bigdl/dllib/models/rnn/rnnexample.py b/python/dllib/src/bigdl/dllib/models/rnn/rnnexample.py new file mode 100644 index 00000000000..a741d407391 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/models/rnn/rnnexample.py @@ -0,0 +1,171 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Still in experimental stage! + +import itertools +import re +from optparse import OptionParser + +from bigdl.dataset import sentence +from bigdl.nn.layer import * +from bigdl.nn.criterion import * +from bigdl.optim.optimizer import * +from bigdl.util.common import * +from bigdl.util.common import Sample + +def prepare_data(sc, folder, vocabsize): + train_file = folder + "train_test.txt" + val_file = folder + "val_test.txt" + + train_sentences_rdd = sc.textFile(train_file) \ + .map(lambda line: sentence.sentences_split(line)) + pad_sent = train_sentences_rdd.flatMap(lambda x: x). \ + map(lambda sent: sentence.sentences_bipadding(sent)) + tokens = pad_sent.map(lambda pad: sentence.sentence_tokenizer(pad)) + max_len = tokens.map(lambda x: len(x)).max() + print("max length %s" % max_len) + + words = tokens.flatMap(lambda x: x) + print("%s words and %s sentences processed in train data" % (words.count(), tokens.count())) + + val_sentences_rdd = sc.textFile(val_file) \ + .map(lambda line: sentence.sentences_split(line)) + val_pad_sent = val_sentences_rdd.flatMap(lambda x: x). \ + map(lambda sent: sentence.sentences_bipadding(sent)) + val_tokens = val_pad_sent.map(lambda pad: sentence.sentence_tokenizer(pad)) + val_max_len = val_tokens.map(lambda x: len(x)).max() + print("val max length %s" % val_max_len) + + val_words = val_tokens.flatMap(lambda x: x) + print("%s words and %s sentences processed in validation data" % (val_words.count(), val_tokens.count())) + + sort_words = words.map(lambda w: (w, 1)) \ + .reduceByKey(lambda a, b: a + b) \ + .sortBy(lambda w_c: w_c[1]) + vocabulary = np.array(sort_words.map(lambda w: w[0]).collect()) + + fre_len = vocabulary.size + if vocabsize > fre_len: + length = fre_len + else: + length = vocabsize + discard_vocab = vocabulary[: fre_len-length] + used_vocab = vocabulary[fre_len-length: fre_len] + used_vocab_size = used_vocab.size + index = np.arange(used_vocab_size) + index2word = dict(enumerate(used_vocab)) + word2index = dict(zip(used_vocab, index)) + total_vocab_len = used_vocab_size + 1 + startIdx = word2index.get("SENTENCESTART") + endIdx = word2index.get("SENTENCEEND") + + def text2labeled(sent): + indexes = [word2index.get(x, used_vocab_size) for x in sent] + data = indexes[0: -1] + label = indexes[1: len(indexes)] + return data, label + + def labeled2onehotformat(labeled_sent): + label = [x+1 for x in labeled_sent[1]] + size = len(labeled_sent[0]) + feature_onehot = np.zeros(size * total_vocab_len, dtype='int').reshape( + [size, total_vocab_len]) + for i, el in enumerate(labeled_sent[0]): + feature_onehot[i, el] = 1 + return feature_onehot, label + + def padding(features, label): + pad_len = max_len - len(label) + print("max_len: %s pad_len: %s" % (max_len, pad_len)) + padded_label = (label + [startIdx] * max_len)[:max_len] + feature_padding = np.zeros((pad_len, total_vocab_len), dtype=np.int) + feature_padding[:, endIdx + 1] = np.ones(pad_len) + padded_feautres = np.concatenate((features, feature_padding), axis=0) + return padded_feautres, padded_label + + sample_rdd = tokens.map(lambda sentence_te: text2labeled(sentence_te)) \ + .map(lambda labeled_sent: labeled2onehotformat(labeled_sent)) \ + .map(lambda x: padding(x[0], x[1])) \ + .map(lambda vectors_label: Sample.from_ndarray(vectors_label[0], np.array(vectors_label[1]))) + + val_sample_rdd = val_tokens.map(lambda sentence_t: text2labeled(sentence_t)) \ + .map(lambda labeled_sent: labeled2onehotformat(labeled_sent)) \ + .map(lambda x: padding(x[0], x[1])) \ + .map(lambda vectors_label: Sample.from_ndarray(vectors_label[0], np.array(vectors_label[1]))) + + return sample_rdd, val_sample_rdd, total_vocab_len + +def build_model(input_size, hidden_size, output_size): + model = Sequential() + model.add(Recurrent() + .add(RnnCell(input_size, hidden_size, Tanh())))\ + .add(TimeDistributed(Linear(hidden_size, output_size))) + model.reset() + return model + +if __name__ == "__main__": + parser = OptionParser() + + parser.add_option("-f", "--folder", dest="folder", default="./") + parser.add_option("-b", "--batchSize", dest="batchSize", default="128") + parser.add_option("--learningRate", dest="learningrate", default="0.1") + parser.add_option("--momentum", dest="momentum", default="0.0") + parser.add_option("--weightDecay", dest="weight_decay", default="0.0") + parser.add_option("--dampening", dest="dampening", default="0.0") + parser.add_option("--hiddenSize", dest="hidden_size", default="40") + parser.add_option("--vocabSize", dest="vob_size", default="4000") + parser.add_option("--maxEpoch", dest="max_epoch", default="30") + + (options, args) = parser.parse_args(sys.argv) + + batch_size = int(options.batchSize) + learningrate = float(options.learningrate) + momentum = float(options.momentum) + weight_decay = float(options.weight_decay) + dampening = float(options.dampening) + hidden_size = int(options.hidden_size) + vob_size = int(options.vob_size) + max_epoch = int(options.max_epoch) + folder = options.folder + + sc = SparkContext(appName="simplernn_example", + conf=create_spark_conf()) + init_engine() + + (train_rdd, val_rdd, vob_size) = prepare_data(sc, folder, vob_size) + + optimizer = Optimizer( + model=build_model(vob_size, hidden_size, vob_size), + training_rdd=train_rdd, + criterion=TimeDistributedCriterion(CrossEntropyCriterion(), size_average=True), + batch_size=batch_size, + optim_method=SGD(learningrate=learningrate, weightdecay=weight_decay, + momentum=momentum, dampening=dampening), + end_trigger=MaxEpoch(max_epoch) + ) + + optimizer.set_validation( + batch_size=batch_size, + val_rdd=val_rdd, + trigger=EveryEpoch(), + val_method=["Loss"], + criterion="TimeDistributedCriterion", + embedded_cri="CrossEntropyCriterion" + ) + + train_model = optimizer.optimize() + sc.stop() \ No newline at end of file diff --git a/python/dllib/src/bigdl/dllib/optim/optimizer.py b/python/dllib/src/bigdl/dllib/optim/optimizer.py index 428fdcfbe83..74de7ff5bd1 100644 --- a/python/dllib/src/bigdl/dllib/optim/optimizer.py +++ b/python/dllib/src/bigdl/dllib/optim/optimizer.py @@ -522,6 +522,8 @@ def set_validation(self, batch_size, val_rdd, trigger, val_method=None): :param val_rdd: validation dataset :param trigger: validation interval :param val_method: the ValidationMethod to use,e.g. "Top1Accuracy", "Top5Accuracy", "Loss" + :param criterion: used criterion + :param embedded_cri: embedded criterion """ if val_method is None: val_method = [Top1Accuracy()] From 99857efeb8240d6e95078a717ef97cdd583bfadc Mon Sep 17 00:00:00 2001 From: ding Date: Wed, 28 Jun 2017 11:50:25 -0400 Subject: [PATCH 130/823] code clean --- .../bigdl/dllib/feature/dataset/sentence.py | 42 +++++ .../src/bigdl/dllib/models/rnn/README.md | 110 +++++++++++ .../src/bigdl/dllib/models/rnn/rnnexample.py | 171 ++++++++++++++++++ .../dllib/src/bigdl/dllib/optim/optimizer.py | 2 + 4 files changed, 325 insertions(+) create mode 100644 python/dllib/src/bigdl/dllib/feature/dataset/sentence.py create mode 100644 python/dllib/src/bigdl/dllib/models/rnn/README.md create mode 100644 python/dllib/src/bigdl/dllib/models/rnn/rnnexample.py diff --git a/python/dllib/src/bigdl/dllib/feature/dataset/sentence.py b/python/dllib/src/bigdl/dllib/feature/dataset/sentence.py new file mode 100644 index 00000000000..46627ab0746 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/feature/dataset/sentence.py @@ -0,0 +1,42 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import nltk +import os +import itertools +import sys +from nltk.tokenize import word_tokenize + +def read_localfile(fileName): + lines = [] + with open(fileName) as f: + for line in f: + lines.append(line) + f.close() + return lines + +def sentences_split(line): + nltk.data.path.append(os.environ.get('PWD')) + sent_tokenizer = nltk.tokenize.PunktSentenceTokenizer() + sentenized = sent_tokenizer.tokenize(line) + return sentenized + +def sentences_bipadding(sent): + return "SENTENCESTART " + sent + " SENTENCEEND" + +def sentence_tokenizer(sentences): + tokenized_sents = nltk.word_tokenize(sentences) + return tokenized_sents \ No newline at end of file diff --git a/python/dllib/src/bigdl/dllib/models/rnn/README.md b/python/dllib/src/bigdl/dllib/models/rnn/README.md new file mode 100644 index 00000000000..55bb8e314c0 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/models/rnn/README.md @@ -0,0 +1,110 @@ +# Recurrent Neural Network + +Model that supports sequence to sequence processing + +This is an implementation of Simple Recurrent Neural Networks for Language Modeling. Please refer to the [original paper](http://www.fit.vutbr.cz/research/groups/speech/publi/2010/mikolov_interspeech2010_IS100722.pdf) by Tomas Mikolov. + +The implementation of RNNs in this code is referred to in the [Keras Recurrent](https://keras.io/layers/recurrent/) documentation. + + +## Get the BigDL files + +Please build BigDL referring to [Build Page](https://github.com/intel-analytics/BigDL/wiki/Build-Page). + + +## Prepare the Input Data +You can download the Tiny Shakespeare Texts corpus from [here](https://raw.githubusercontent.com/karpathy/char-rnn/master/data/tinyshakespeare/input.txt). + +After downloading the text, separate it into train.txt and val.txt. In our case, we just select 80 percentage of the input to be train and remaining 20 percentage to be val. +Please upload the directory will contains train.txt and val.txt to hdfs. The program will later read in the original text file from hdfs. +```shell +export LANG=en_US.UTF-8 +head -n 8000 input.txt > val.txt +tail -n +8000 input.txt > train.txt +``` + +### Sample Text + +The input text may look as follows: + +``` + First Citizen: + Before we proceed any further, hear me speak. + + All: + Speak, speak. + + First Citizen: + You are all resolved rather to die than to famish? +``` +## Preprocessing + +The sentences_split, sentence_tokenizer use [NLTK Toolkit](http://www.nltk.org/). NLTK is included +in prepared virtual env. As Nltk expected english.pickle to tokenizer in the current location under +tokenizers/punkt/english.pickle. we need download punkt and navigate into the folder for packaging +``` +$python +>>> import nltk +>>> nltk.download('punkt') +``` +then you can find nltk_data folder, zip file with tokenizer.zip#tokenizer +``` +$ cd nltk_data/tokenizers/ +$ zip -r ../../tokenizers.zip * +$ cd ../../ +``` + +### Sample Sequence of Processed Data +``` + 3998,3875,3690,3999 + 3998,3171,3958,2390,3832,3202,3855,3983,3883,3999 + 3998,3667,3999 + 3998,3151,3883,3999 + 3998,3875,3690,3999 +``` + +## Train the Model +Example command: +```bash +./dist/bin/bigdl.sh -- \ + +BigDL_HOME=... +SPARK_HOME=... +MASTER=yarn +PYTHON_API_PATH=${BigDL_HOME}/dist/lib/bigdl-VERSION-python-api.zip +BigDL_JAR_PATH=${BigDL_HOME}/dist/lib/bigdl-VERSION-jar-with-dependencies.jar +PYTHONPATH=${PYTHON_API_PATH}:$PYTHONPATH +# set http_proxy if you need proxy to access internet +http_proxy=... +# Please refer ${BigDL_HOME}/pyspark/python_package/README.md to see how to create virtual env +PYSPARK_DRIVER_PYTHON=./venv/bin/python PYSPARK_PYTHON=./venv.zip/venv/bin/python ${SPARK_HOME}/bin/spark-submit \ + --master ${MASTER} \ + --deploy-mode client \ + --conf spark.executorEnv.http_proxy=${http_proxy} \ + --driver-memory 10g \ + --executor-cores 1 \ + --executor-memory 60g \ + --py-files ${PYTHON_API_PATH} \ + --properties-file ${BigDL_HOME}/dist/conf/spark-bigdl.conf \ + --jars ${BigDL_JAR_PATH} \ + --archives venv.zip,tokenizers.zip#tokenizers \ + --conf spark.driver.extraClassPath=${BigDL_JAR_PATH} \ + --conf spark.executor.extraClassPath=bigdl-version-jar-with-dependencies.jar \ + --conf spark.yarn.appMasterEnv.NLTK_DATA=./ \ + --num-executors 1 \ + ${BigDL_HOME}/pyspark/dl/models/rnn/rnnexample.py --folder hdfs://almaren-node-001:9000/rnn/ --batchSize 12 +``` + +* `--folder` hdfs directory where train.txt and val.txt are located + +* `--batchSize` option can be used to set batch size, the default value is 128. + +## Expected Training Output +Users can see the Loss of the model printed by the program. The Loss, in this case, is the perplexity of the language model. The lower, the better. +``` +INFO DistriOptimizer$:247 - [Epoch 1 0/6879][Iteration 1][Wall Clock 0.0s] Train 12 in 4.926679827seconds. Throughput is 2.4357176 records/second. Loss is 8.277311. Current learning rate is 0.1. +INFO DistriOptimizer$:247 - [Epoch 1 12/6879][Iteration 2][Wall Clock 4.926679827s] Train 12 in 2.622718594seconds. Throughput is 4.575405 records/second. Loss is 8.07377. Current learning rate is 0.1. +INFO DistriOptimizer$:247 - [Epoch 1 24/6879][Iteration 3][Wall Clock 7.549398421s] Train 12 in 2.478575083seconds. Throughput is 4.8414917 records/second. Loss is 7.8527904. Current learning rate is 0.1. +INFO DistriOptimizer$:247 - [Epoch 1 36/6879][Iteration 4][Wall Clock 10.027973504s] Train 12 in 2.475138056seconds. Throughput is 4.8482146 records/second. Loss is 7.581617. Current learning rate is 0.1. +... +``` \ No newline at end of file diff --git a/python/dllib/src/bigdl/dllib/models/rnn/rnnexample.py b/python/dllib/src/bigdl/dllib/models/rnn/rnnexample.py new file mode 100644 index 00000000000..a741d407391 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/models/rnn/rnnexample.py @@ -0,0 +1,171 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Still in experimental stage! + +import itertools +import re +from optparse import OptionParser + +from bigdl.dataset import sentence +from bigdl.nn.layer import * +from bigdl.nn.criterion import * +from bigdl.optim.optimizer import * +from bigdl.util.common import * +from bigdl.util.common import Sample + +def prepare_data(sc, folder, vocabsize): + train_file = folder + "train_test.txt" + val_file = folder + "val_test.txt" + + train_sentences_rdd = sc.textFile(train_file) \ + .map(lambda line: sentence.sentences_split(line)) + pad_sent = train_sentences_rdd.flatMap(lambda x: x). \ + map(lambda sent: sentence.sentences_bipadding(sent)) + tokens = pad_sent.map(lambda pad: sentence.sentence_tokenizer(pad)) + max_len = tokens.map(lambda x: len(x)).max() + print("max length %s" % max_len) + + words = tokens.flatMap(lambda x: x) + print("%s words and %s sentences processed in train data" % (words.count(), tokens.count())) + + val_sentences_rdd = sc.textFile(val_file) \ + .map(lambda line: sentence.sentences_split(line)) + val_pad_sent = val_sentences_rdd.flatMap(lambda x: x). \ + map(lambda sent: sentence.sentences_bipadding(sent)) + val_tokens = val_pad_sent.map(lambda pad: sentence.sentence_tokenizer(pad)) + val_max_len = val_tokens.map(lambda x: len(x)).max() + print("val max length %s" % val_max_len) + + val_words = val_tokens.flatMap(lambda x: x) + print("%s words and %s sentences processed in validation data" % (val_words.count(), val_tokens.count())) + + sort_words = words.map(lambda w: (w, 1)) \ + .reduceByKey(lambda a, b: a + b) \ + .sortBy(lambda w_c: w_c[1]) + vocabulary = np.array(sort_words.map(lambda w: w[0]).collect()) + + fre_len = vocabulary.size + if vocabsize > fre_len: + length = fre_len + else: + length = vocabsize + discard_vocab = vocabulary[: fre_len-length] + used_vocab = vocabulary[fre_len-length: fre_len] + used_vocab_size = used_vocab.size + index = np.arange(used_vocab_size) + index2word = dict(enumerate(used_vocab)) + word2index = dict(zip(used_vocab, index)) + total_vocab_len = used_vocab_size + 1 + startIdx = word2index.get("SENTENCESTART") + endIdx = word2index.get("SENTENCEEND") + + def text2labeled(sent): + indexes = [word2index.get(x, used_vocab_size) for x in sent] + data = indexes[0: -1] + label = indexes[1: len(indexes)] + return data, label + + def labeled2onehotformat(labeled_sent): + label = [x+1 for x in labeled_sent[1]] + size = len(labeled_sent[0]) + feature_onehot = np.zeros(size * total_vocab_len, dtype='int').reshape( + [size, total_vocab_len]) + for i, el in enumerate(labeled_sent[0]): + feature_onehot[i, el] = 1 + return feature_onehot, label + + def padding(features, label): + pad_len = max_len - len(label) + print("max_len: %s pad_len: %s" % (max_len, pad_len)) + padded_label = (label + [startIdx] * max_len)[:max_len] + feature_padding = np.zeros((pad_len, total_vocab_len), dtype=np.int) + feature_padding[:, endIdx + 1] = np.ones(pad_len) + padded_feautres = np.concatenate((features, feature_padding), axis=0) + return padded_feautres, padded_label + + sample_rdd = tokens.map(lambda sentence_te: text2labeled(sentence_te)) \ + .map(lambda labeled_sent: labeled2onehotformat(labeled_sent)) \ + .map(lambda x: padding(x[0], x[1])) \ + .map(lambda vectors_label: Sample.from_ndarray(vectors_label[0], np.array(vectors_label[1]))) + + val_sample_rdd = val_tokens.map(lambda sentence_t: text2labeled(sentence_t)) \ + .map(lambda labeled_sent: labeled2onehotformat(labeled_sent)) \ + .map(lambda x: padding(x[0], x[1])) \ + .map(lambda vectors_label: Sample.from_ndarray(vectors_label[0], np.array(vectors_label[1]))) + + return sample_rdd, val_sample_rdd, total_vocab_len + +def build_model(input_size, hidden_size, output_size): + model = Sequential() + model.add(Recurrent() + .add(RnnCell(input_size, hidden_size, Tanh())))\ + .add(TimeDistributed(Linear(hidden_size, output_size))) + model.reset() + return model + +if __name__ == "__main__": + parser = OptionParser() + + parser.add_option("-f", "--folder", dest="folder", default="./") + parser.add_option("-b", "--batchSize", dest="batchSize", default="128") + parser.add_option("--learningRate", dest="learningrate", default="0.1") + parser.add_option("--momentum", dest="momentum", default="0.0") + parser.add_option("--weightDecay", dest="weight_decay", default="0.0") + parser.add_option("--dampening", dest="dampening", default="0.0") + parser.add_option("--hiddenSize", dest="hidden_size", default="40") + parser.add_option("--vocabSize", dest="vob_size", default="4000") + parser.add_option("--maxEpoch", dest="max_epoch", default="30") + + (options, args) = parser.parse_args(sys.argv) + + batch_size = int(options.batchSize) + learningrate = float(options.learningrate) + momentum = float(options.momentum) + weight_decay = float(options.weight_decay) + dampening = float(options.dampening) + hidden_size = int(options.hidden_size) + vob_size = int(options.vob_size) + max_epoch = int(options.max_epoch) + folder = options.folder + + sc = SparkContext(appName="simplernn_example", + conf=create_spark_conf()) + init_engine() + + (train_rdd, val_rdd, vob_size) = prepare_data(sc, folder, vob_size) + + optimizer = Optimizer( + model=build_model(vob_size, hidden_size, vob_size), + training_rdd=train_rdd, + criterion=TimeDistributedCriterion(CrossEntropyCriterion(), size_average=True), + batch_size=batch_size, + optim_method=SGD(learningrate=learningrate, weightdecay=weight_decay, + momentum=momentum, dampening=dampening), + end_trigger=MaxEpoch(max_epoch) + ) + + optimizer.set_validation( + batch_size=batch_size, + val_rdd=val_rdd, + trigger=EveryEpoch(), + val_method=["Loss"], + criterion="TimeDistributedCriterion", + embedded_cri="CrossEntropyCriterion" + ) + + train_model = optimizer.optimize() + sc.stop() \ No newline at end of file diff --git a/python/dllib/src/bigdl/dllib/optim/optimizer.py b/python/dllib/src/bigdl/dllib/optim/optimizer.py index 428fdcfbe83..74de7ff5bd1 100644 --- a/python/dllib/src/bigdl/dllib/optim/optimizer.py +++ b/python/dllib/src/bigdl/dllib/optim/optimizer.py @@ -522,6 +522,8 @@ def set_validation(self, batch_size, val_rdd, trigger, val_method=None): :param val_rdd: validation dataset :param trigger: validation interval :param val_method: the ValidationMethod to use,e.g. "Top1Accuracy", "Top5Accuracy", "Loss" + :param criterion: used criterion + :param embedded_cri: embedded criterion """ if val_method is None: val_method = [Top1Accuracy()] From a7705bc56fb645c37e4fbf59f37d45dcf625c620 Mon Sep 17 00:00:00 2001 From: ding Date: Thu, 29 Jun 2017 07:12:59 -0400 Subject: [PATCH 131/823] fix style check --- python/dllib/src/bigdl/dllib/feature/dataset/sentence.py | 2 +- python/dllib/src/bigdl/dllib/models/rnn/README.md | 4 +--- python/dllib/src/bigdl/dllib/models/rnn/rnnexample.py | 6 +++--- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/feature/dataset/sentence.py b/python/dllib/src/bigdl/dllib/feature/dataset/sentence.py index 46627ab0746..960c31bdf1d 100644 --- a/python/dllib/src/bigdl/dllib/feature/dataset/sentence.py +++ b/python/dllib/src/bigdl/dllib/feature/dataset/sentence.py @@ -39,4 +39,4 @@ def sentences_bipadding(sent): def sentence_tokenizer(sentences): tokenized_sents = nltk.word_tokenize(sentences) - return tokenized_sents \ No newline at end of file + return tokenized_sents diff --git a/python/dllib/src/bigdl/dllib/models/rnn/README.md b/python/dllib/src/bigdl/dllib/models/rnn/README.md index 55bb8e314c0..c888a07c9f7 100644 --- a/python/dllib/src/bigdl/dllib/models/rnn/README.md +++ b/python/dllib/src/bigdl/dllib/models/rnn/README.md @@ -40,8 +40,7 @@ The input text may look as follows: ## Preprocessing The sentences_split, sentence_tokenizer use [NLTK Toolkit](http://www.nltk.org/). NLTK is included -in prepared virtual env. As Nltk expected english.pickle to tokenizer in the current location under -tokenizers/punkt/english.pickle. we need download punkt and navigate into the folder for packaging +in created virtual env(see ${BigDL_HOME}/pyspark/python_package/README.md to see how to create virtual env). As NLTK expects english.pickle to tokenizer, we need download punkt and navigate into the folder for packaging ``` $python >>> import nltk @@ -76,7 +75,6 @@ BigDL_JAR_PATH=${BigDL_HOME}/dist/lib/bigdl-VERSION-jar-with-dependencies.jar PYTHONPATH=${PYTHON_API_PATH}:$PYTHONPATH # set http_proxy if you need proxy to access internet http_proxy=... -# Please refer ${BigDL_HOME}/pyspark/python_package/README.md to see how to create virtual env PYSPARK_DRIVER_PYTHON=./venv/bin/python PYSPARK_PYTHON=./venv.zip/venv/bin/python ${SPARK_HOME}/bin/spark-submit \ --master ${MASTER} \ --deploy-mode client \ diff --git a/python/dllib/src/bigdl/dllib/models/rnn/rnnexample.py b/python/dllib/src/bigdl/dllib/models/rnn/rnnexample.py index a741d407391..53e28a167c8 100644 --- a/python/dllib/src/bigdl/dllib/models/rnn/rnnexample.py +++ b/python/dllib/src/bigdl/dllib/models/rnn/rnnexample.py @@ -28,8 +28,8 @@ from bigdl.util.common import Sample def prepare_data(sc, folder, vocabsize): - train_file = folder + "train_test.txt" - val_file = folder + "val_test.txt" + train_file = folder + "train.txt" + val_file = folder + "val.txt" train_sentences_rdd = sc.textFile(train_file) \ .map(lambda line: sentence.sentences_split(line)) @@ -168,4 +168,4 @@ def build_model(input_size, hidden_size, output_size): ) train_model = optimizer.optimize() - sc.stop() \ No newline at end of file + sc.stop() From 72ca21a6c9d333e0c3ef575d523af77628792b37 Mon Sep 17 00:00:00 2001 From: ding Date: Thu, 29 Jun 2017 07:12:59 -0400 Subject: [PATCH 132/823] fix style check --- python/dllib/src/bigdl/dllib/feature/dataset/sentence.py | 2 +- python/dllib/src/bigdl/dllib/models/rnn/README.md | 4 +--- python/dllib/src/bigdl/dllib/models/rnn/rnnexample.py | 6 +++--- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/feature/dataset/sentence.py b/python/dllib/src/bigdl/dllib/feature/dataset/sentence.py index 46627ab0746..960c31bdf1d 100644 --- a/python/dllib/src/bigdl/dllib/feature/dataset/sentence.py +++ b/python/dllib/src/bigdl/dllib/feature/dataset/sentence.py @@ -39,4 +39,4 @@ def sentences_bipadding(sent): def sentence_tokenizer(sentences): tokenized_sents = nltk.word_tokenize(sentences) - return tokenized_sents \ No newline at end of file + return tokenized_sents diff --git a/python/dllib/src/bigdl/dllib/models/rnn/README.md b/python/dllib/src/bigdl/dllib/models/rnn/README.md index 55bb8e314c0..c888a07c9f7 100644 --- a/python/dllib/src/bigdl/dllib/models/rnn/README.md +++ b/python/dllib/src/bigdl/dllib/models/rnn/README.md @@ -40,8 +40,7 @@ The input text may look as follows: ## Preprocessing The sentences_split, sentence_tokenizer use [NLTK Toolkit](http://www.nltk.org/). NLTK is included -in prepared virtual env. As Nltk expected english.pickle to tokenizer in the current location under -tokenizers/punkt/english.pickle. we need download punkt and navigate into the folder for packaging +in created virtual env(see ${BigDL_HOME}/pyspark/python_package/README.md to see how to create virtual env). As NLTK expects english.pickle to tokenizer, we need download punkt and navigate into the folder for packaging ``` $python >>> import nltk @@ -76,7 +75,6 @@ BigDL_JAR_PATH=${BigDL_HOME}/dist/lib/bigdl-VERSION-jar-with-dependencies.jar PYTHONPATH=${PYTHON_API_PATH}:$PYTHONPATH # set http_proxy if you need proxy to access internet http_proxy=... -# Please refer ${BigDL_HOME}/pyspark/python_package/README.md to see how to create virtual env PYSPARK_DRIVER_PYTHON=./venv/bin/python PYSPARK_PYTHON=./venv.zip/venv/bin/python ${SPARK_HOME}/bin/spark-submit \ --master ${MASTER} \ --deploy-mode client \ diff --git a/python/dllib/src/bigdl/dllib/models/rnn/rnnexample.py b/python/dllib/src/bigdl/dllib/models/rnn/rnnexample.py index a741d407391..53e28a167c8 100644 --- a/python/dllib/src/bigdl/dllib/models/rnn/rnnexample.py +++ b/python/dllib/src/bigdl/dllib/models/rnn/rnnexample.py @@ -28,8 +28,8 @@ from bigdl.util.common import Sample def prepare_data(sc, folder, vocabsize): - train_file = folder + "train_test.txt" - val_file = folder + "val_test.txt" + train_file = folder + "train.txt" + val_file = folder + "val.txt" train_sentences_rdd = sc.textFile(train_file) \ .map(lambda line: sentence.sentences_split(line)) @@ -168,4 +168,4 @@ def build_model(input_size, hidden_size, output_size): ) train_model = optimizer.optimize() - sc.stop() \ No newline at end of file + sc.stop() From c7d68f1e7ec3b5ac46910c7b11e95299b5fc1a49 Mon Sep 17 00:00:00 2001 From: ding Date: Thu, 6 Jul 2017 06:22:55 -0400 Subject: [PATCH 133/823] fix comments --- .../src/bigdl/dllib/models/rnn/README.md | 13 ++----- .../src/bigdl/dllib/models/rnn/rnnexample.py | 36 +++++++++++-------- 2 files changed, 24 insertions(+), 25 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/models/rnn/README.md b/python/dllib/src/bigdl/dllib/models/rnn/README.md index c888a07c9f7..b381bcf56f1 100644 --- a/python/dllib/src/bigdl/dllib/models/rnn/README.md +++ b/python/dllib/src/bigdl/dllib/models/rnn/README.md @@ -15,13 +15,7 @@ Please build BigDL referring to [Build Page](https://github.com/intel-analytics/ ## Prepare the Input Data You can download the Tiny Shakespeare Texts corpus from [here](https://raw.githubusercontent.com/karpathy/char-rnn/master/data/tinyshakespeare/input.txt). -After downloading the text, separate it into train.txt and val.txt. In our case, we just select 80 percentage of the input to be train and remaining 20 percentage to be val. -Please upload the directory will contains train.txt and val.txt to hdfs. The program will later read in the original text file from hdfs. -```shell -export LANG=en_US.UTF-8 -head -n 8000 input.txt > val.txt -tail -n +8000 input.txt > train.txt -``` +If you run on spark local mode, you can skip this step, we will download the file for you. ### Sample Text @@ -63,9 +57,8 @@ $ cd ../../ ``` ## Train the Model -Example command: +Example command in yarn: ```bash -./dist/bin/bigdl.sh -- \ BigDL_HOME=... SPARK_HOME=... @@ -90,7 +83,7 @@ PYSPARK_DRIVER_PYTHON=./venv/bin/python PYSPARK_PYTHON=./venv.zip/venv/bin/pytho --conf spark.executor.extraClassPath=bigdl-version-jar-with-dependencies.jar \ --conf spark.yarn.appMasterEnv.NLTK_DATA=./ \ --num-executors 1 \ - ${BigDL_HOME}/pyspark/dl/models/rnn/rnnexample.py --folder hdfs://almaren-node-001:9000/rnn/ --batchSize 12 + ${BigDL_HOME}/pyspark/dl/models/rnn/rnnexample.py --folder hdfs://xxx:9000/rnn/ --batchSize 12 ``` * `--folder` hdfs directory where train.txt and val.txt are located diff --git a/python/dllib/src/bigdl/dllib/models/rnn/rnnexample.py b/python/dllib/src/bigdl/dllib/models/rnn/rnnexample.py index 53e28a167c8..30bd62a25dc 100644 --- a/python/dllib/src/bigdl/dllib/models/rnn/rnnexample.py +++ b/python/dllib/src/bigdl/dllib/models/rnn/rnnexample.py @@ -20,6 +20,7 @@ import re from optparse import OptionParser +from bigdl.dataset import base from bigdl.dataset import sentence from bigdl.nn.layer import * from bigdl.nn.criterion import * @@ -27,26 +28,30 @@ from bigdl.util.common import * from bigdl.util.common import Sample -def prepare_data(sc, folder, vocabsize): - train_file = folder + "train.txt" - val_file = folder + "val.txt" +def download_data(dest_dir): + TINYSHAKESPEARE_URL = 'https://raw.githubusercontent.com/karpathy/char-rnn/master/data/tinyshakespeare/input.txt' # noqa + file_name = "input.txt" + file_abs_path = base.maybe_download(file_name, dest_dir, TINYSHAKESPEARE_URL) + return file_abs_path - train_sentences_rdd = sc.textFile(train_file) \ +def prepare_data(sc, folder, vocabsize, training_split): + if not folder.startswith( 'hdfs://' ): + file = download_data(folder) + else: + file = folder + sentences_rdd = sc.textFile(file) \ .map(lambda line: sentence.sentences_split(line)) - pad_sent = train_sentences_rdd.flatMap(lambda x: x). \ + pad_sent = sentences_rdd.flatMap(lambda x: x). \ map(lambda sent: sentence.sentences_bipadding(sent)) tokens = pad_sent.map(lambda pad: sentence.sentence_tokenizer(pad)) - max_len = tokens.map(lambda x: len(x)).max() + train_tokens, val_tokens = tokens.randomSplit([training_split, 1 - training_split]) + + max_len = train_tokens.map(lambda x: len(x)).max() print("max length %s" % max_len) - words = tokens.flatMap(lambda x: x) - print("%s words and %s sentences processed in train data" % (words.count(), tokens.count())) + words = train_tokens.flatMap(lambda x: x) + print("%s words and %s sentences processed in train data" % (words.count(), train_tokens.count())) - val_sentences_rdd = sc.textFile(val_file) \ - .map(lambda line: sentence.sentences_split(line)) - val_pad_sent = val_sentences_rdd.flatMap(lambda x: x). \ - map(lambda sent: sentence.sentences_bipadding(sent)) - val_tokens = val_pad_sent.map(lambda pad: sentence.sentence_tokenizer(pad)) val_max_len = val_tokens.map(lambda x: len(x)).max() print("val max length %s" % val_max_len) @@ -120,7 +125,7 @@ def build_model(input_size, hidden_size, output_size): if __name__ == "__main__": parser = OptionParser() - parser.add_option("-f", "--folder", dest="folder", default="./") + parser.add_option("-f", "--folder", dest="folder", default="/tmp/rnn") parser.add_option("-b", "--batchSize", dest="batchSize", default="128") parser.add_option("--learningRate", dest="learningrate", default="0.1") parser.add_option("--momentum", dest="momentum", default="0.0") @@ -141,12 +146,13 @@ def build_model(input_size, hidden_size, output_size): vob_size = int(options.vob_size) max_epoch = int(options.max_epoch) folder = options.folder + training_split = 0.8 sc = SparkContext(appName="simplernn_example", conf=create_spark_conf()) init_engine() - (train_rdd, val_rdd, vob_size) = prepare_data(sc, folder, vob_size) + (train_rdd, val_rdd, vob_size) = prepare_data(sc, folder, vob_size, training_split) optimizer = Optimizer( model=build_model(vob_size, hidden_size, vob_size), From 8c913ed8bc1c49512f28072e59a38c66cafb16f2 Mon Sep 17 00:00:00 2001 From: ding Date: Thu, 6 Jul 2017 06:22:55 -0400 Subject: [PATCH 134/823] fix comments --- .../src/bigdl/dllib/models/rnn/README.md | 13 ++----- .../src/bigdl/dllib/models/rnn/rnnexample.py | 36 +++++++++++-------- 2 files changed, 24 insertions(+), 25 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/models/rnn/README.md b/python/dllib/src/bigdl/dllib/models/rnn/README.md index c888a07c9f7..b381bcf56f1 100644 --- a/python/dllib/src/bigdl/dllib/models/rnn/README.md +++ b/python/dllib/src/bigdl/dllib/models/rnn/README.md @@ -15,13 +15,7 @@ Please build BigDL referring to [Build Page](https://github.com/intel-analytics/ ## Prepare the Input Data You can download the Tiny Shakespeare Texts corpus from [here](https://raw.githubusercontent.com/karpathy/char-rnn/master/data/tinyshakespeare/input.txt). -After downloading the text, separate it into train.txt and val.txt. In our case, we just select 80 percentage of the input to be train and remaining 20 percentage to be val. -Please upload the directory will contains train.txt and val.txt to hdfs. The program will later read in the original text file from hdfs. -```shell -export LANG=en_US.UTF-8 -head -n 8000 input.txt > val.txt -tail -n +8000 input.txt > train.txt -``` +If you run on spark local mode, you can skip this step, we will download the file for you. ### Sample Text @@ -63,9 +57,8 @@ $ cd ../../ ``` ## Train the Model -Example command: +Example command in yarn: ```bash -./dist/bin/bigdl.sh -- \ BigDL_HOME=... SPARK_HOME=... @@ -90,7 +83,7 @@ PYSPARK_DRIVER_PYTHON=./venv/bin/python PYSPARK_PYTHON=./venv.zip/venv/bin/pytho --conf spark.executor.extraClassPath=bigdl-version-jar-with-dependencies.jar \ --conf spark.yarn.appMasterEnv.NLTK_DATA=./ \ --num-executors 1 \ - ${BigDL_HOME}/pyspark/dl/models/rnn/rnnexample.py --folder hdfs://almaren-node-001:9000/rnn/ --batchSize 12 + ${BigDL_HOME}/pyspark/dl/models/rnn/rnnexample.py --folder hdfs://xxx:9000/rnn/ --batchSize 12 ``` * `--folder` hdfs directory where train.txt and val.txt are located diff --git a/python/dllib/src/bigdl/dllib/models/rnn/rnnexample.py b/python/dllib/src/bigdl/dllib/models/rnn/rnnexample.py index 53e28a167c8..30bd62a25dc 100644 --- a/python/dllib/src/bigdl/dllib/models/rnn/rnnexample.py +++ b/python/dllib/src/bigdl/dllib/models/rnn/rnnexample.py @@ -20,6 +20,7 @@ import re from optparse import OptionParser +from bigdl.dataset import base from bigdl.dataset import sentence from bigdl.nn.layer import * from bigdl.nn.criterion import * @@ -27,26 +28,30 @@ from bigdl.util.common import * from bigdl.util.common import Sample -def prepare_data(sc, folder, vocabsize): - train_file = folder + "train.txt" - val_file = folder + "val.txt" +def download_data(dest_dir): + TINYSHAKESPEARE_URL = 'https://raw.githubusercontent.com/karpathy/char-rnn/master/data/tinyshakespeare/input.txt' # noqa + file_name = "input.txt" + file_abs_path = base.maybe_download(file_name, dest_dir, TINYSHAKESPEARE_URL) + return file_abs_path - train_sentences_rdd = sc.textFile(train_file) \ +def prepare_data(sc, folder, vocabsize, training_split): + if not folder.startswith( 'hdfs://' ): + file = download_data(folder) + else: + file = folder + sentences_rdd = sc.textFile(file) \ .map(lambda line: sentence.sentences_split(line)) - pad_sent = train_sentences_rdd.flatMap(lambda x: x). \ + pad_sent = sentences_rdd.flatMap(lambda x: x). \ map(lambda sent: sentence.sentences_bipadding(sent)) tokens = pad_sent.map(lambda pad: sentence.sentence_tokenizer(pad)) - max_len = tokens.map(lambda x: len(x)).max() + train_tokens, val_tokens = tokens.randomSplit([training_split, 1 - training_split]) + + max_len = train_tokens.map(lambda x: len(x)).max() print("max length %s" % max_len) - words = tokens.flatMap(lambda x: x) - print("%s words and %s sentences processed in train data" % (words.count(), tokens.count())) + words = train_tokens.flatMap(lambda x: x) + print("%s words and %s sentences processed in train data" % (words.count(), train_tokens.count())) - val_sentences_rdd = sc.textFile(val_file) \ - .map(lambda line: sentence.sentences_split(line)) - val_pad_sent = val_sentences_rdd.flatMap(lambda x: x). \ - map(lambda sent: sentence.sentences_bipadding(sent)) - val_tokens = val_pad_sent.map(lambda pad: sentence.sentence_tokenizer(pad)) val_max_len = val_tokens.map(lambda x: len(x)).max() print("val max length %s" % val_max_len) @@ -120,7 +125,7 @@ def build_model(input_size, hidden_size, output_size): if __name__ == "__main__": parser = OptionParser() - parser.add_option("-f", "--folder", dest="folder", default="./") + parser.add_option("-f", "--folder", dest="folder", default="/tmp/rnn") parser.add_option("-b", "--batchSize", dest="batchSize", default="128") parser.add_option("--learningRate", dest="learningrate", default="0.1") parser.add_option("--momentum", dest="momentum", default="0.0") @@ -141,12 +146,13 @@ def build_model(input_size, hidden_size, output_size): vob_size = int(options.vob_size) max_epoch = int(options.max_epoch) folder = options.folder + training_split = 0.8 sc = SparkContext(appName="simplernn_example", conf=create_spark_conf()) init_engine() - (train_rdd, val_rdd, vob_size) = prepare_data(sc, folder, vob_size) + (train_rdd, val_rdd, vob_size) = prepare_data(sc, folder, vob_size, training_split) optimizer = Optimizer( model=build_model(vob_size, hidden_size, vob_size), From 2a6325b4ce02a46035547111cb47032ce389ac61 Mon Sep 17 00:00:00 2001 From: ding Date: Thu, 6 Jul 2017 09:39:45 -0400 Subject: [PATCH 135/823] add rnn example in local_integration test --- python/dllib/src/bigdl/dllib/optim/optimizer.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/optim/optimizer.py b/python/dllib/src/bigdl/dllib/optim/optimizer.py index 74de7ff5bd1..428fdcfbe83 100644 --- a/python/dllib/src/bigdl/dllib/optim/optimizer.py +++ b/python/dllib/src/bigdl/dllib/optim/optimizer.py @@ -522,8 +522,6 @@ def set_validation(self, batch_size, val_rdd, trigger, val_method=None): :param val_rdd: validation dataset :param trigger: validation interval :param val_method: the ValidationMethod to use,e.g. "Top1Accuracy", "Top5Accuracy", "Loss" - :param criterion: used criterion - :param embedded_cri: embedded criterion """ if val_method is None: val_method = [Top1Accuracy()] From 5f125756b299714917e0d23a0527123a0720f419 Mon Sep 17 00:00:00 2001 From: ding Date: Thu, 6 Jul 2017 09:39:45 -0400 Subject: [PATCH 136/823] add rnn example in local_integration test --- python/dllib/src/bigdl/dllib/optim/optimizer.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/optim/optimizer.py b/python/dllib/src/bigdl/dllib/optim/optimizer.py index 74de7ff5bd1..428fdcfbe83 100644 --- a/python/dllib/src/bigdl/dllib/optim/optimizer.py +++ b/python/dllib/src/bigdl/dllib/optim/optimizer.py @@ -522,8 +522,6 @@ def set_validation(self, batch_size, val_rdd, trigger, val_method=None): :param val_rdd: validation dataset :param trigger: validation interval :param val_method: the ValidationMethod to use,e.g. "Top1Accuracy", "Top5Accuracy", "Loss" - :param criterion: used criterion - :param embedded_cri: embedded criterion """ if val_method is None: val_method = [Top1Accuracy()] From 7352655134260e08a044df3c289e5e014e6d43ea Mon Sep 17 00:00:00 2001 From: ding Date: Thu, 6 Jul 2017 09:39:45 -0400 Subject: [PATCH 137/823] add rnn example in local_integration test --- .../commands/run-local-rnn.sh | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100755 python/dllib/test/local_integration/commands/run-local-rnn.sh diff --git a/python/dllib/test/local_integration/commands/run-local-rnn.sh b/python/dllib/test/local_integration/commands/run-local-rnn.sh new file mode 100755 index 00000000000..23c61f60228 --- /dev/null +++ b/python/dllib/test/local_integration/commands/run-local-rnn.sh @@ -0,0 +1,18 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +$PYTHON_EXECUTABLE ${BIGDL_HOME}/pyspark/bigdl/models/rnn/rnnexample.py \ + --max_epoch 1 From b697f12dbd084ef0e43b7df2bf61e8db240ca57a Mon Sep 17 00:00:00 2001 From: ding Date: Fri, 7 Jul 2017 07:39:30 -0400 Subject: [PATCH 138/823] add prepare env script for rnn in local integration test --- .../dllib/src/bigdl/dllib/models/rnn/rnnexample.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/models/rnn/rnnexample.py b/python/dllib/src/bigdl/dllib/models/rnn/rnnexample.py index 30bd62a25dc..207af873acf 100644 --- a/python/dllib/src/bigdl/dllib/models/rnn/rnnexample.py +++ b/python/dllib/src/bigdl/dllib/models/rnn/rnnexample.py @@ -104,13 +104,15 @@ def padding(features, label): sample_rdd = tokens.map(lambda sentence_te: text2labeled(sentence_te)) \ .map(lambda labeled_sent: labeled2onehotformat(labeled_sent)) \ - .map(lambda x: padding(x[0], x[1])) \ - .map(lambda vectors_label: Sample.from_ndarray(vectors_label[0], np.array(vectors_label[1]))) + .map(lambda x: padding(x[0], x[1], train_max_len)) \ + .map(lambda vectors_label: Sample.from_ndarray(vectors_label[0], + np.array(vectors_label[1]))).cache() val_sample_rdd = val_tokens.map(lambda sentence_t: text2labeled(sentence_t)) \ .map(lambda labeled_sent: labeled2onehotformat(labeled_sent)) \ - .map(lambda x: padding(x[0], x[1])) \ - .map(lambda vectors_label: Sample.from_ndarray(vectors_label[0], np.array(vectors_label[1]))) + .map(lambda x: padding(x[0], x[1], val_max_len)) \ + .map(lambda vectors_label: Sample.from_ndarray(vectors_label[0], + np.array(vectors_label[1]))).cache() return sample_rdd, val_sample_rdd, total_vocab_len @@ -126,7 +128,7 @@ def build_model(input_size, hidden_size, output_size): parser = OptionParser() parser.add_option("-f", "--folder", dest="folder", default="/tmp/rnn") - parser.add_option("-b", "--batchSize", dest="batchSize", default="128") + parser.add_option("-b", "--batchSize", dest="batchSize", default="12") parser.add_option("--learningRate", dest="learningrate", default="0.1") parser.add_option("--momentum", dest="momentum", default="0.0") parser.add_option("--weightDecay", dest="weight_decay", default="0.0") From 626f65d92c3e471ceddf161ffde569c2d8521b0c Mon Sep 17 00:00:00 2001 From: ding Date: Fri, 7 Jul 2017 07:39:30 -0400 Subject: [PATCH 139/823] add prepare env script for rnn in local integration test --- .../dllib/src/bigdl/dllib/models/rnn/rnnexample.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/models/rnn/rnnexample.py b/python/dllib/src/bigdl/dllib/models/rnn/rnnexample.py index 30bd62a25dc..207af873acf 100644 --- a/python/dllib/src/bigdl/dllib/models/rnn/rnnexample.py +++ b/python/dllib/src/bigdl/dllib/models/rnn/rnnexample.py @@ -104,13 +104,15 @@ def padding(features, label): sample_rdd = tokens.map(lambda sentence_te: text2labeled(sentence_te)) \ .map(lambda labeled_sent: labeled2onehotformat(labeled_sent)) \ - .map(lambda x: padding(x[0], x[1])) \ - .map(lambda vectors_label: Sample.from_ndarray(vectors_label[0], np.array(vectors_label[1]))) + .map(lambda x: padding(x[0], x[1], train_max_len)) \ + .map(lambda vectors_label: Sample.from_ndarray(vectors_label[0], + np.array(vectors_label[1]))).cache() val_sample_rdd = val_tokens.map(lambda sentence_t: text2labeled(sentence_t)) \ .map(lambda labeled_sent: labeled2onehotformat(labeled_sent)) \ - .map(lambda x: padding(x[0], x[1])) \ - .map(lambda vectors_label: Sample.from_ndarray(vectors_label[0], np.array(vectors_label[1]))) + .map(lambda x: padding(x[0], x[1], val_max_len)) \ + .map(lambda vectors_label: Sample.from_ndarray(vectors_label[0], + np.array(vectors_label[1]))).cache() return sample_rdd, val_sample_rdd, total_vocab_len @@ -126,7 +128,7 @@ def build_model(input_size, hidden_size, output_size): parser = OptionParser() parser.add_option("-f", "--folder", dest="folder", default="/tmp/rnn") - parser.add_option("-b", "--batchSize", dest="batchSize", default="128") + parser.add_option("-b", "--batchSize", dest="batchSize", default="12") parser.add_option("--learningRate", dest="learningrate", default="0.1") parser.add_option("--momentum", dest="momentum", default="0.0") parser.add_option("--weightDecay", dest="weight_decay", default="0.0") From a9b59bea04d1c93dfeed238ff4b0291bc929c7bb Mon Sep 17 00:00:00 2001 From: ding Date: Fri, 7 Jul 2017 07:39:30 -0400 Subject: [PATCH 140/823] add prepare env script for rnn in local integration test --- .../commands/prepare_nltk.py | 19 +++++++++++++++++++ .../commands/run-local-rnn.sh | 4 +++- 2 files changed, 22 insertions(+), 1 deletion(-) create mode 100755 python/dllib/test/local_integration/commands/prepare_nltk.py diff --git a/python/dllib/test/local_integration/commands/prepare_nltk.py b/python/dllib/test/local_integration/commands/prepare_nltk.py new file mode 100755 index 00000000000..c9372e7d5cd --- /dev/null +++ b/python/dllib/test/local_integration/commands/prepare_nltk.py @@ -0,0 +1,19 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import nltk +nltk.download('punkt') + diff --git a/python/dllib/test/local_integration/commands/run-local-rnn.sh b/python/dllib/test/local_integration/commands/run-local-rnn.sh index 23c61f60228..cc9061b81cf 100755 --- a/python/dllib/test/local_integration/commands/run-local-rnn.sh +++ b/python/dllib/test/local_integration/commands/run-local-rnn.sh @@ -14,5 +14,7 @@ # limitations under the License. # +export SPARK_DRIVER_MEMORY=20g +$PYTHON_EXECUTABLE prepare_nltk.py $PYTHON_EXECUTABLE ${BIGDL_HOME}/pyspark/bigdl/models/rnn/rnnexample.py \ - --max_epoch 1 + --maxEpoch 1 From 48a1937ff60a8c4e2936931b8ddaf2888b1466d7 Mon Sep 17 00:00:00 2001 From: ding Date: Fri, 7 Jul 2017 07:57:38 -0400 Subject: [PATCH 141/823] rebase code --- .../src/bigdl/dllib/models/rnn/rnnexample.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/models/rnn/rnnexample.py b/python/dllib/src/bigdl/dllib/models/rnn/rnnexample.py index 207af873acf..da49781f472 100644 --- a/python/dllib/src/bigdl/dllib/models/rnn/rnnexample.py +++ b/python/dllib/src/bigdl/dllib/models/rnn/rnnexample.py @@ -45,9 +45,11 @@ def prepare_data(sc, folder, vocabsize, training_split): map(lambda sent: sentence.sentences_bipadding(sent)) tokens = pad_sent.map(lambda pad: sentence.sentence_tokenizer(pad)) train_tokens, val_tokens = tokens.randomSplit([training_split, 1 - training_split]) + train_tokens.cache() + val_tokens.cache() - max_len = train_tokens.map(lambda x: len(x)).max() - print("max length %s" % max_len) + train_max_len = train_tokens.map(lambda x: len(x)).max() + print("max length %s" % train_max_len) words = train_tokens.flatMap(lambda x: x) print("%s words and %s sentences processed in train data" % (words.count(), train_tokens.count())) @@ -93,16 +95,15 @@ def labeled2onehotformat(labeled_sent): feature_onehot[i, el] = 1 return feature_onehot, label - def padding(features, label): - pad_len = max_len - len(label) - print("max_len: %s pad_len: %s" % (max_len, pad_len)) - padded_label = (label + [startIdx] * max_len)[:max_len] + def padding(features, label, length): + pad_len = length - len(label) + padded_label = (label + [startIdx] * length)[:length] feature_padding = np.zeros((pad_len, total_vocab_len), dtype=np.int) feature_padding[:, endIdx + 1] = np.ones(pad_len) padded_feautres = np.concatenate((features, feature_padding), axis=0) return padded_feautres, padded_label - sample_rdd = tokens.map(lambda sentence_te: text2labeled(sentence_te)) \ + sample_rdd = train_tokens.map(lambda sentence_te: text2labeled(sentence_te)) \ .map(lambda labeled_sent: labeled2onehotformat(labeled_sent)) \ .map(lambda x: padding(x[0], x[1], train_max_len)) \ .map(lambda vectors_label: Sample.from_ndarray(vectors_label[0], @@ -170,9 +171,7 @@ def build_model(input_size, hidden_size, output_size): batch_size=batch_size, val_rdd=val_rdd, trigger=EveryEpoch(), - val_method=["Loss"], - criterion="TimeDistributedCriterion", - embedded_cri="CrossEntropyCriterion" + val_method=[Loss(TimeDistributedCriterion(CrossEntropyCriterion(), size_average=True))] ) train_model = optimizer.optimize() From 44df65a92d8c46f15707692b150cde1f2b77b3f1 Mon Sep 17 00:00:00 2001 From: ding Date: Fri, 7 Jul 2017 07:57:38 -0400 Subject: [PATCH 142/823] rebase code --- .../src/bigdl/dllib/models/rnn/rnnexample.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/models/rnn/rnnexample.py b/python/dllib/src/bigdl/dllib/models/rnn/rnnexample.py index 207af873acf..da49781f472 100644 --- a/python/dllib/src/bigdl/dllib/models/rnn/rnnexample.py +++ b/python/dllib/src/bigdl/dllib/models/rnn/rnnexample.py @@ -45,9 +45,11 @@ def prepare_data(sc, folder, vocabsize, training_split): map(lambda sent: sentence.sentences_bipadding(sent)) tokens = pad_sent.map(lambda pad: sentence.sentence_tokenizer(pad)) train_tokens, val_tokens = tokens.randomSplit([training_split, 1 - training_split]) + train_tokens.cache() + val_tokens.cache() - max_len = train_tokens.map(lambda x: len(x)).max() - print("max length %s" % max_len) + train_max_len = train_tokens.map(lambda x: len(x)).max() + print("max length %s" % train_max_len) words = train_tokens.flatMap(lambda x: x) print("%s words and %s sentences processed in train data" % (words.count(), train_tokens.count())) @@ -93,16 +95,15 @@ def labeled2onehotformat(labeled_sent): feature_onehot[i, el] = 1 return feature_onehot, label - def padding(features, label): - pad_len = max_len - len(label) - print("max_len: %s pad_len: %s" % (max_len, pad_len)) - padded_label = (label + [startIdx] * max_len)[:max_len] + def padding(features, label, length): + pad_len = length - len(label) + padded_label = (label + [startIdx] * length)[:length] feature_padding = np.zeros((pad_len, total_vocab_len), dtype=np.int) feature_padding[:, endIdx + 1] = np.ones(pad_len) padded_feautres = np.concatenate((features, feature_padding), axis=0) return padded_feautres, padded_label - sample_rdd = tokens.map(lambda sentence_te: text2labeled(sentence_te)) \ + sample_rdd = train_tokens.map(lambda sentence_te: text2labeled(sentence_te)) \ .map(lambda labeled_sent: labeled2onehotformat(labeled_sent)) \ .map(lambda x: padding(x[0], x[1], train_max_len)) \ .map(lambda vectors_label: Sample.from_ndarray(vectors_label[0], @@ -170,9 +171,7 @@ def build_model(input_size, hidden_size, output_size): batch_size=batch_size, val_rdd=val_rdd, trigger=EveryEpoch(), - val_method=["Loss"], - criterion="TimeDistributedCriterion", - embedded_cri="CrossEntropyCriterion" + val_method=[Loss(TimeDistributedCriterion(CrossEntropyCriterion(), size_average=True))] ) train_model = optimizer.optimize() From c4e0f6e9886e6890d771c3fbf3f5153f0689acbb Mon Sep 17 00:00:00 2001 From: ding Date: Fri, 7 Jul 2017 07:57:38 -0400 Subject: [PATCH 143/823] rebase code --- python/dllib/test/local_integration/commands/prepare_nltk.py | 1 - python/dllib/test/local_integration/commands/run-local-rnn.sh | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/python/dllib/test/local_integration/commands/prepare_nltk.py b/python/dllib/test/local_integration/commands/prepare_nltk.py index c9372e7d5cd..d7e25431264 100755 --- a/python/dllib/test/local_integration/commands/prepare_nltk.py +++ b/python/dllib/test/local_integration/commands/prepare_nltk.py @@ -16,4 +16,3 @@ import nltk nltk.download('punkt') - diff --git a/python/dllib/test/local_integration/commands/run-local-rnn.sh b/python/dllib/test/local_integration/commands/run-local-rnn.sh index cc9061b81cf..5659c69d026 100755 --- a/python/dllib/test/local_integration/commands/run-local-rnn.sh +++ b/python/dllib/test/local_integration/commands/run-local-rnn.sh @@ -14,7 +14,7 @@ # limitations under the License. # -export SPARK_DRIVER_MEMORY=20g +export SPARK_DRIVER_MEMORY=30g $PYTHON_EXECUTABLE prepare_nltk.py $PYTHON_EXECUTABLE ${BIGDL_HOME}/pyspark/bigdl/models/rnn/rnnexample.py \ --maxEpoch 1 From 68e1fbba2929694d42e435a897ec470afd17b163 Mon Sep 17 00:00:00 2001 From: ding Date: Mon, 10 Jul 2017 11:16:52 -0400 Subject: [PATCH 144/823] improve perf and add md --- python/dllib/src/bigdl/dllib/nn/layer.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index ce87084f8ab..cde433cbf5e 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -3441,8 +3441,8 @@ class ConvLSTMPeephole(Layer): | Ref. A.: https://arxiv.org/abs/1506.04214 (blueprint for this module) | B. https://github.com/viorik/ConvLSTM - :param input_size: the size of each input - :param hidden_size: Hidden unit size in the LSTM + :param input_size: number of input planes in the image given into forward() + :param output_size: number of output planes the convolution layer will produce :param kernel_i Convolutional filter size to convolve input :param kernel_c Convolutional filter size to convolve cell :param stride The step of the convolution @@ -3458,9 +3458,9 @@ class ConvLSTMPeephole(Layer): creating: createConvLSTMPeephole ''' - def __init__(self, input_size, hidden_size, kernel_i, kernel_c, stride, wRegularizer=None, uRegularizer=None, - bRegularizer=None, with_peephole=False, bigdl_type="float"): - super(ConvLSTMPeephole, self).__init__(None, bigdl_type, input_size, hidden_size, kernel_i, kernel_c, stride, + def __init__(self, input_size, output_size, kernel_i, kernel_c, stride, wRegularizer=None, uRegularizer=None, + bRegularizer=None, with_peephole=True, bigdl_type="float"): + super(ConvLSTMPeephole, self).__init__(None, bigdl_type, input_size, output_size, kernel_i, kernel_c, stride, wRegularizer, uRegularizer, bRegularizer, with_peephole) def _test(): From dc271aacc39dfde713944c9bb5c6aea59f99ef39 Mon Sep 17 00:00:00 2001 From: ding Date: Mon, 10 Jul 2017 11:16:52 -0400 Subject: [PATCH 145/823] improve perf and add md --- python/dllib/src/bigdl/dllib/nn/layer.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index ce87084f8ab..cde433cbf5e 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -3441,8 +3441,8 @@ class ConvLSTMPeephole(Layer): | Ref. A.: https://arxiv.org/abs/1506.04214 (blueprint for this module) | B. https://github.com/viorik/ConvLSTM - :param input_size: the size of each input - :param hidden_size: Hidden unit size in the LSTM + :param input_size: number of input planes in the image given into forward() + :param output_size: number of output planes the convolution layer will produce :param kernel_i Convolutional filter size to convolve input :param kernel_c Convolutional filter size to convolve cell :param stride The step of the convolution @@ -3458,9 +3458,9 @@ class ConvLSTMPeephole(Layer): creating: createConvLSTMPeephole ''' - def __init__(self, input_size, hidden_size, kernel_i, kernel_c, stride, wRegularizer=None, uRegularizer=None, - bRegularizer=None, with_peephole=False, bigdl_type="float"): - super(ConvLSTMPeephole, self).__init__(None, bigdl_type, input_size, hidden_size, kernel_i, kernel_c, stride, + def __init__(self, input_size, output_size, kernel_i, kernel_c, stride, wRegularizer=None, uRegularizer=None, + bRegularizer=None, with_peephole=True, bigdl_type="float"): + super(ConvLSTMPeephole, self).__init__(None, bigdl_type, input_size, output_size, kernel_i, kernel_c, stride, wRegularizer, uRegularizer, bRegularizer, with_peephole) def _test(): From 450524c3aa0d120634ed05bb9658b6aa540c46bd Mon Sep 17 00:00:00 2001 From: Yao Zhang Date: Wed, 12 Jul 2017 15:51:00 +0800 Subject: [PATCH 146/823] Add 1D Convolution (TemporalConvolution) (#1180) * add test * remove unnecessary change in LinearSpec * finish Temporal Convolution * add python API * meet code review * meet code review * fix typos * fix typos * meet code review --- python/dllib/src/bigdl/dllib/nn/layer.py | 59 ++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index cde433cbf5e..412a5fb3a9d 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -567,6 +567,65 @@ class Sequential(Container): def __init__(self, bigdl_type="float"): super(Sequential, self).__init__(None, bigdl_type) +class TemporalConvolution(Layer): + + ''' + Applies a 1D convolution over an input sequence composed of nInputFrame frames.. + The input tensor in `forward(input)` is expected to be a 2D tensor + (`nInputFrame` x `inputFrameSize`) or a 3D tensor + (`nBatchFrame` x `nInputFrame` x `inputFrameSize`). + + :param input_frame_size The input frame size expected in sequences given into `forward()` + :param output_frame_size The output frame size the convolution layer will produce. + :param kernel_w The kernel width of the convolution + :param stride_w The step of the convolution in the width dimension. + :param propagate_back Whether propagate gradient back, default is true. + :param weight_regularizer instance of [[Regularizer]] + (eg. L1 or L2 regularization), applied to the input weights matrices. + :param bias_regularizer instance of [[Regularizer]] + applied to the bias. + :param init_weight Initial weight + :param init_bias Initial bias + :param init_grad_weight Initial gradient weight + :param init_grad_bias Initial gradient bias + + >>> temporalConvolution = TemporalConvolution(6, 12, 5, 5) + creating: createTemporalConvolution + >>> temporalConvolution.setWRegularizer(L1Regularizer(0.5)) + creating: createL1Regularizer + >>> temporalConvolution.setBRegularizer(L1Regularizer(0.5)) + creating: createL1Regularizer + ''' + + def __init__(self, + input_frame_size, + output_frame_size, + kernel_w, + stride_w=1, + propagate_back=True, + weight_regularizer=None, + bias_regularizer=None, + init_weight=None, + init_bias=None, + init_grad_weight=None, + init_grad_bias=None, + bigdl_type="float"): + super(TemporalConvolution, self).__init__(None, bigdl_type, + input_frame_size, + output_frame_size, + kernel_w, + stride_w, + propagate_back, + weight_regularizer, + bias_regularizer, + JTensor.from_ndarray(init_weight), + JTensor.from_ndarray(init_bias), + JTensor.from_ndarray(init_grad_weight), + JTensor.from_ndarray(init_grad_bias)) + def set_init_method(self, weight_init_method = None, bias_init_method = None): + callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, + weight_init_method, bias_init_method) + return self class SpatialConvolution(Layer): From cdc9ba88dc236567b0331fc5e6fcaed06caac005 Mon Sep 17 00:00:00 2001 From: Yao Zhang Date: Wed, 12 Jul 2017 15:51:00 +0800 Subject: [PATCH 147/823] Add 1D Convolution (TemporalConvolution) (#1180) * add test * remove unnecessary change in LinearSpec * finish Temporal Convolution * add python API * meet code review * meet code review * fix typos * fix typos * meet code review --- python/dllib/src/bigdl/dllib/nn/layer.py | 59 ++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index cde433cbf5e..412a5fb3a9d 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -567,6 +567,65 @@ class Sequential(Container): def __init__(self, bigdl_type="float"): super(Sequential, self).__init__(None, bigdl_type) +class TemporalConvolution(Layer): + + ''' + Applies a 1D convolution over an input sequence composed of nInputFrame frames.. + The input tensor in `forward(input)` is expected to be a 2D tensor + (`nInputFrame` x `inputFrameSize`) or a 3D tensor + (`nBatchFrame` x `nInputFrame` x `inputFrameSize`). + + :param input_frame_size The input frame size expected in sequences given into `forward()` + :param output_frame_size The output frame size the convolution layer will produce. + :param kernel_w The kernel width of the convolution + :param stride_w The step of the convolution in the width dimension. + :param propagate_back Whether propagate gradient back, default is true. + :param weight_regularizer instance of [[Regularizer]] + (eg. L1 or L2 regularization), applied to the input weights matrices. + :param bias_regularizer instance of [[Regularizer]] + applied to the bias. + :param init_weight Initial weight + :param init_bias Initial bias + :param init_grad_weight Initial gradient weight + :param init_grad_bias Initial gradient bias + + >>> temporalConvolution = TemporalConvolution(6, 12, 5, 5) + creating: createTemporalConvolution + >>> temporalConvolution.setWRegularizer(L1Regularizer(0.5)) + creating: createL1Regularizer + >>> temporalConvolution.setBRegularizer(L1Regularizer(0.5)) + creating: createL1Regularizer + ''' + + def __init__(self, + input_frame_size, + output_frame_size, + kernel_w, + stride_w=1, + propagate_back=True, + weight_regularizer=None, + bias_regularizer=None, + init_weight=None, + init_bias=None, + init_grad_weight=None, + init_grad_bias=None, + bigdl_type="float"): + super(TemporalConvolution, self).__init__(None, bigdl_type, + input_frame_size, + output_frame_size, + kernel_w, + stride_w, + propagate_back, + weight_regularizer, + bias_regularizer, + JTensor.from_ndarray(init_weight), + JTensor.from_ndarray(init_bias), + JTensor.from_ndarray(init_grad_weight), + JTensor.from_ndarray(init_grad_bias)) + def set_init_method(self, weight_init_method = None, bias_init_method = None): + callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, + weight_init_method, bias_init_method) + return self class SpatialConvolution(Layer): From e0a96b2b761fb20c13553f0b6d2c003e85990985 Mon Sep 17 00:00:00 2001 From: Yao Zhang Date: Thu, 13 Jul 2017 06:23:49 +0800 Subject: [PATCH 148/823] Tree LSTM with Sentiment classification example (#1217) * finish treeLSTM with sentiment example * fix scala style * fix python error * fix a python bug * fix a typo * fix a typo --- python/dllib/src/bigdl/dllib/nn/criterion.py | 28 ++++++++++++++++--- python/dllib/src/bigdl/dllib/nn/layer.py | 24 ++++++++++++++++ .../dllib/src/bigdl/dllib/optim/optimizer.py | 9 ++++++ 3 files changed, 57 insertions(+), 4 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/nn/criterion.py b/python/dllib/src/bigdl/dllib/nn/criterion.py index f94fb2d11c7..38d9016ef1c 100644 --- a/python/dllib/src/bigdl/dllib/nn/criterion.py +++ b/python/dllib/src/bigdl/dllib/nn/criterion.py @@ -99,10 +99,30 @@ def of(cls, jcriterion, bigdl_type="float"): class ClassNLLCriterion(Criterion): ''' - The negative log likelihood criterion. - It is useful to train a classification problem with n classes. - If provided, the optional argument weights should be a 1D Tensor - assigning weight to each of the classes. + The negative log likelihood criterion. It is useful to train a classification problem with n + classes. If provided, the optional argument weights should be a 1D Tensor assigning weight to + each of the classes. This is particularly useful when you have an unbalanced training set. + + The input given through a forward() is expected to contain log-probabilities of each class: + input has to be a 1D Tensor of size n. Obtaining log-probabilities in a neural network is easily + achieved by adding a LogSoftMax layer in the last layer of your neural network. You may use + CrossEntropyCriterion instead, if you prefer not to add an extra layer to your network. This + criterion expects a class index (1 to the number of class) as target when calling + forward(input, target) and backward(input, target). + + The loss can be described as: + loss(x, class) = -x[class] + or in the case of the weights argument it is specified as follows: + loss(x, class) = -weights[class] * x[class] + Due to the behaviour of the backend code, it is necessary to set sizeAverage to false when + calculating losses in non-batch mode. + + Note that if the target is `-1`, the training process will skip this sample. + In other will, the forward process will return zero output and the backward process + will also return zero `gradInput`. + + By default, the losses are averaged over observations for each minibatch. However, if the field + sizeAverage is set to false, the losses are instead summed for each minibatch. :param weights: weights of each class diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 412a5fb3a9d..2244965535c 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -627,6 +627,30 @@ def set_init_method(self, weight_init_method = None, bias_init_method = None): weight_init_method, bias_init_method) return self +class BinaryTreeLSTM(Layer): + ''' + This class is an implementation of Binary TreeLSTM (Constituency Tree LSTM). + :param inputSize input units size + :param hiddenSize hidden units size + :param gateOutput whether gate output + :param withGraph whether create lstms with [[Graph]], the default value is true. + >>> treeLSTM = BinaryTreeLSTM(100, 200) + creating: createBinaryTreeLSTM + ''' + + def __init__(self, + input_size, + hidden_size, + gate_output=True, + with_graph=True, + bigdl_type="float"): + super(BinaryTreeLSTM, self).__init__(None, + bigdl_type, + input_size, + hidden_size, + gate_output, + with_graph) + class SpatialConvolution(Layer): ''' diff --git a/python/dllib/src/bigdl/dllib/optim/optimizer.py b/python/dllib/src/bigdl/dllib/optim/optimizer.py index 428fdcfbe83..b8f66179796 100644 --- a/python/dllib/src/bigdl/dllib/optim/optimizer.py +++ b/python/dllib/src/bigdl/dllib/optim/optimizer.py @@ -43,6 +43,15 @@ class Top1Accuracy(JavaValue): def __init__(self, bigdl_type="float"): JavaValue.__init__(self, None, bigdl_type) +class TreeNNAccuracy(JavaValue): + """ + Caculate the percentage that output's max probability index equals target. + + >>> top1 = TreeNNAccuracy() + creating: createTreeNNAccuracy + """ + def __init__(self, bigdl_type="float"): + JavaValue.__init__(self, None, bigdl_type) class Top5Accuracy(JavaValue): """ From 5729de9fe1fbceef2d2d5aa583d61f83114ac33a Mon Sep 17 00:00:00 2001 From: Yao Zhang Date: Thu, 13 Jul 2017 06:23:49 +0800 Subject: [PATCH 149/823] Tree LSTM with Sentiment classification example (#1217) * finish treeLSTM with sentiment example * fix scala style * fix python error * fix a python bug * fix a typo * fix a typo --- python/dllib/src/bigdl/dllib/nn/criterion.py | 28 ++++++++++++++++--- python/dllib/src/bigdl/dllib/nn/layer.py | 24 ++++++++++++++++ .../dllib/src/bigdl/dllib/optim/optimizer.py | 9 ++++++ 3 files changed, 57 insertions(+), 4 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/nn/criterion.py b/python/dllib/src/bigdl/dllib/nn/criterion.py index f94fb2d11c7..38d9016ef1c 100644 --- a/python/dllib/src/bigdl/dllib/nn/criterion.py +++ b/python/dllib/src/bigdl/dllib/nn/criterion.py @@ -99,10 +99,30 @@ def of(cls, jcriterion, bigdl_type="float"): class ClassNLLCriterion(Criterion): ''' - The negative log likelihood criterion. - It is useful to train a classification problem with n classes. - If provided, the optional argument weights should be a 1D Tensor - assigning weight to each of the classes. + The negative log likelihood criterion. It is useful to train a classification problem with n + classes. If provided, the optional argument weights should be a 1D Tensor assigning weight to + each of the classes. This is particularly useful when you have an unbalanced training set. + + The input given through a forward() is expected to contain log-probabilities of each class: + input has to be a 1D Tensor of size n. Obtaining log-probabilities in a neural network is easily + achieved by adding a LogSoftMax layer in the last layer of your neural network. You may use + CrossEntropyCriterion instead, if you prefer not to add an extra layer to your network. This + criterion expects a class index (1 to the number of class) as target when calling + forward(input, target) and backward(input, target). + + The loss can be described as: + loss(x, class) = -x[class] + or in the case of the weights argument it is specified as follows: + loss(x, class) = -weights[class] * x[class] + Due to the behaviour of the backend code, it is necessary to set sizeAverage to false when + calculating losses in non-batch mode. + + Note that if the target is `-1`, the training process will skip this sample. + In other will, the forward process will return zero output and the backward process + will also return zero `gradInput`. + + By default, the losses are averaged over observations for each minibatch. However, if the field + sizeAverage is set to false, the losses are instead summed for each minibatch. :param weights: weights of each class diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 412a5fb3a9d..2244965535c 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -627,6 +627,30 @@ def set_init_method(self, weight_init_method = None, bias_init_method = None): weight_init_method, bias_init_method) return self +class BinaryTreeLSTM(Layer): + ''' + This class is an implementation of Binary TreeLSTM (Constituency Tree LSTM). + :param inputSize input units size + :param hiddenSize hidden units size + :param gateOutput whether gate output + :param withGraph whether create lstms with [[Graph]], the default value is true. + >>> treeLSTM = BinaryTreeLSTM(100, 200) + creating: createBinaryTreeLSTM + ''' + + def __init__(self, + input_size, + hidden_size, + gate_output=True, + with_graph=True, + bigdl_type="float"): + super(BinaryTreeLSTM, self).__init__(None, + bigdl_type, + input_size, + hidden_size, + gate_output, + with_graph) + class SpatialConvolution(Layer): ''' diff --git a/python/dllib/src/bigdl/dllib/optim/optimizer.py b/python/dllib/src/bigdl/dllib/optim/optimizer.py index 428fdcfbe83..b8f66179796 100644 --- a/python/dllib/src/bigdl/dllib/optim/optimizer.py +++ b/python/dllib/src/bigdl/dllib/optim/optimizer.py @@ -43,6 +43,15 @@ class Top1Accuracy(JavaValue): def __init__(self, bigdl_type="float"): JavaValue.__init__(self, None, bigdl_type) +class TreeNNAccuracy(JavaValue): + """ + Caculate the percentage that output's max probability index equals target. + + >>> top1 = TreeNNAccuracy() + creating: createTreeNNAccuracy + """ + def __init__(self, bigdl_type="float"): + JavaValue.__init__(self, None, bigdl_type) class Top5Accuracy(JavaValue): """ From 6c69fffeceb5a3c4094abe70287e7eedd3792b7e Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Wed, 12 Jul 2017 19:52:10 -0700 Subject: [PATCH 150/823] support init graph using tensorflow model in python (#1212) * support init graph using tensorflow model * add doc * fix test * meet code review --- python/dllib/src/bigdl/dllib/nn/layer.py | 15 +++++++++++++-- .../dllib/src/bigdl/dllib/utils/tf_utils.py | 19 ++++++++++++++++--- 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 2244965535c..0c81697981c 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -374,15 +374,26 @@ class Model(Container): All inputs should be able to connect to outputs through some paths in the graph. It is allowed that some successors of the inputs node are not connect to outputs. If so, these nodes will be excluded in the computation. + + We also support initialzing a Graph directly from a tensorflow module. In this case, you should + pass your tensorflow nodes as inputs and outputs and also specify the byte_order parameter ("little_endian" + or "big_endian") and node_type parameter ("bigdl" or "tensorflow") + node_type parameter. """ def __init__(self, inputs, outputs, - bigdl_type="float"): - super(Model, self).__init__(None, bigdl_type, + bigdl_type="float", byte_order="little_endian", model_type="bigdl"): + if model_type == "bigdl": + super(Model, self).__init__(None, bigdl_type, to_list(inputs), to_list(outputs)) + else: + from bigdl.util.tf_utils import convert + model = convert(to_list(inputs), to_list(outputs), byte_order, bigdl_type) + super(Model, self).__init__(model, bigdl_type) + @staticmethod def load(path, bigdl_type="float"): diff --git a/python/dllib/src/bigdl/dllib/utils/tf_utils.py b/python/dllib/src/bigdl/dllib/utils/tf_utils.py index ac57a52d862..e2a7dc44fc8 100644 --- a/python/dllib/src/bigdl/dllib/utils/tf_utils.py +++ b/python/dllib/src/bigdl/dllib/utils/tf_utils.py @@ -17,6 +17,7 @@ import tempfile import tensorflow as tf +import shutil from google.protobuf import text_format @@ -25,10 +26,9 @@ from tensorflow.python.framework import graph_util from tensorflow.python.framework import importer from tensorflow.python.platform import gfile - from bigdl.nn.layer import Model -def convert(input_ops, output_ops, sess): +def convert(input_ops, output_ops, byte_order, bigdl_type): """ Convert tensorflow model to bigdl model :param input_ops: operation list used for input, should be placeholders @@ -36,6 +36,10 @@ def convert(input_ops, output_ops, sess): :param sess: current tensorflow session :return: bigdl model """ + sess = tf.Session() + init = tf.global_variables_initializer() + sess.run(init) + input_names = map(lambda x: x.name.split(":")[0], input_ops) output_names = map(lambda x: x.name.split(":")[0], output_ops) temp = tempfile.mkdtemp() @@ -48,7 +52,16 @@ def convert(input_ops, output_ops, sess): temp + '/model.chkp', output_names, temp + '/model.pb', sess) - return Model.load_tensorflow(temp + '/model.pb', input_names, output_names) + + model = Model.load_tensorflow(temp + '/model.pb', input_names, output_names, byte_order, bigdl_type) + + try: + shutil.rmtree(temp) + except OSError as e: + if e.errno != errno.ENOENT: + raise + + return model def merge_checkpoint(input_graph, checkpoint, From 5fc6b933bb69ca2ef60d716ed38124130b36e566 Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Wed, 12 Jul 2017 19:52:10 -0700 Subject: [PATCH 151/823] support init graph using tensorflow model in python (#1212) * support init graph using tensorflow model * add doc * fix test * meet code review --- python/dllib/src/bigdl/dllib/nn/layer.py | 15 +++++++++++++-- .../dllib/src/bigdl/dllib/utils/tf_utils.py | 19 ++++++++++++++++--- 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 2244965535c..0c81697981c 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -374,15 +374,26 @@ class Model(Container): All inputs should be able to connect to outputs through some paths in the graph. It is allowed that some successors of the inputs node are not connect to outputs. If so, these nodes will be excluded in the computation. + + We also support initialzing a Graph directly from a tensorflow module. In this case, you should + pass your tensorflow nodes as inputs and outputs and also specify the byte_order parameter ("little_endian" + or "big_endian") and node_type parameter ("bigdl" or "tensorflow") + node_type parameter. """ def __init__(self, inputs, outputs, - bigdl_type="float"): - super(Model, self).__init__(None, bigdl_type, + bigdl_type="float", byte_order="little_endian", model_type="bigdl"): + if model_type == "bigdl": + super(Model, self).__init__(None, bigdl_type, to_list(inputs), to_list(outputs)) + else: + from bigdl.util.tf_utils import convert + model = convert(to_list(inputs), to_list(outputs), byte_order, bigdl_type) + super(Model, self).__init__(model, bigdl_type) + @staticmethod def load(path, bigdl_type="float"): diff --git a/python/dllib/src/bigdl/dllib/utils/tf_utils.py b/python/dllib/src/bigdl/dllib/utils/tf_utils.py index ac57a52d862..e2a7dc44fc8 100644 --- a/python/dllib/src/bigdl/dllib/utils/tf_utils.py +++ b/python/dllib/src/bigdl/dllib/utils/tf_utils.py @@ -17,6 +17,7 @@ import tempfile import tensorflow as tf +import shutil from google.protobuf import text_format @@ -25,10 +26,9 @@ from tensorflow.python.framework import graph_util from tensorflow.python.framework import importer from tensorflow.python.platform import gfile - from bigdl.nn.layer import Model -def convert(input_ops, output_ops, sess): +def convert(input_ops, output_ops, byte_order, bigdl_type): """ Convert tensorflow model to bigdl model :param input_ops: operation list used for input, should be placeholders @@ -36,6 +36,10 @@ def convert(input_ops, output_ops, sess): :param sess: current tensorflow session :return: bigdl model """ + sess = tf.Session() + init = tf.global_variables_initializer() + sess.run(init) + input_names = map(lambda x: x.name.split(":")[0], input_ops) output_names = map(lambda x: x.name.split(":")[0], output_ops) temp = tempfile.mkdtemp() @@ -48,7 +52,16 @@ def convert(input_ops, output_ops, sess): temp + '/model.chkp', output_names, temp + '/model.pb', sess) - return Model.load_tensorflow(temp + '/model.pb', input_names, output_names) + + model = Model.load_tensorflow(temp + '/model.pb', input_names, output_names, byte_order, bigdl_type) + + try: + shutil.rmtree(temp) + except OSError as e: + if e.errno != errno.ENOENT: + raise + + return model def merge_checkpoint(input_graph, checkpoint, From a93aa4c72f05e6550fdddacc5d97ff45ac01c0b7 Mon Sep 17 00:00:00 2001 From: Xianyan Date: Tue, 18 Jul 2017 15:10:31 +0800 Subject: [PATCH 152/823] add load caffe model doc, remove useless parameter, add python api (#1266) * add load caffe model doc, remove useless parameter, add python api * update doc * add save caffe * remove empty lines * fix python test * Update caffe-support.md * fix python * update doc --- python/dllib/src/bigdl/dllib/nn/layer.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 0c81697981c..563cfbc2d3c 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -323,6 +323,10 @@ def save(self, path, over_write = False): callBigDlFunc(self.bigdl_type, "modelSave", self.value, path, over_write) + def save_caffe(self, prototxt_path, model_path, use_v2 = True, overwrite = False): + callBigDlFunc(self.bigdl_type, "saveCaffe", self.value, prototxt_path, + model_path, use_v2, overwrite) + def setWRegularizer(self, wRegularizer): ''' set weight regularizer @@ -431,6 +435,19 @@ def load_caffe(model, defPath, modelPath, match_all=True, bigdl_type="float"): jmodel = callBigDlFunc(bigdl_type, "loadCaffe", model, defPath, modelPath, match_all) return Layer.of(jmodel) + @staticmethod + def load_caffe_model(defPath, modelPath, bigdl_type="float"): + """ + Load a pre-trained Caffe model. + + + :param defPath: The path containing the caffe model definition. + :param modelPath: The path containing the pre-trained caffe model. + :return: A pre-trained model. + """ + jmodel = callBigDlFunc(bigdl_type, "loadCaffeModel", defPath, modelPath) + return Layer.of(jmodel) + @staticmethod def load_tensorflow(path, inputs, outputs, byte_order = "little_endian", bigdl_type="float"): """ From 7b60748a39a7984dba85851377546d40d1629c4d Mon Sep 17 00:00:00 2001 From: Xianyan Date: Tue, 18 Jul 2017 15:10:31 +0800 Subject: [PATCH 153/823] add load caffe model doc, remove useless parameter, add python api (#1266) * add load caffe model doc, remove useless parameter, add python api * update doc * add save caffe * remove empty lines * fix python test * Update caffe-support.md * fix python * update doc --- python/dllib/src/bigdl/dllib/nn/layer.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 0c81697981c..563cfbc2d3c 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -323,6 +323,10 @@ def save(self, path, over_write = False): callBigDlFunc(self.bigdl_type, "modelSave", self.value, path, over_write) + def save_caffe(self, prototxt_path, model_path, use_v2 = True, overwrite = False): + callBigDlFunc(self.bigdl_type, "saveCaffe", self.value, prototxt_path, + model_path, use_v2, overwrite) + def setWRegularizer(self, wRegularizer): ''' set weight regularizer @@ -431,6 +435,19 @@ def load_caffe(model, defPath, modelPath, match_all=True, bigdl_type="float"): jmodel = callBigDlFunc(bigdl_type, "loadCaffe", model, defPath, modelPath, match_all) return Layer.of(jmodel) + @staticmethod + def load_caffe_model(defPath, modelPath, bigdl_type="float"): + """ + Load a pre-trained Caffe model. + + + :param defPath: The path containing the caffe model definition. + :param modelPath: The path containing the pre-trained caffe model. + :return: A pre-trained model. + """ + jmodel = callBigDlFunc(bigdl_type, "loadCaffeModel", defPath, modelPath) + return Layer.of(jmodel) + @staticmethod def load_tensorflow(path, inputs, outputs, byte_order = "little_endian", bigdl_type="float"): """ From eb03c9a2aaeb9a3ee0e8c67e21c8419b6beb57cf Mon Sep 17 00:00:00 2001 From: Xianyan Date: Tue, 18 Jul 2017 15:10:31 +0800 Subject: [PATCH 154/823] add load caffe model doc, remove useless parameter, add python api (#1266) * add load caffe model doc, remove useless parameter, add python api * update doc * add save caffe * remove empty lines * fix python test * Update caffe-support.md * fix python * update doc --- load_caffe_test.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/load_caffe_test.py b/load_caffe_test.py index 6ac29171549..7cd6e50d292 100644 --- a/load_caffe_test.py +++ b/load_caffe_test.py @@ -42,7 +42,7 @@ def test_load_caffe(self): module = Sequential()\ .add(SpatialConvolution(3, 4, 2, 2).set_name("conv"))\ .add(SpatialConvolution(4, 3, 2, 2).set_name("conv2"))\ - .add(Linear(2, 27, with_bias=False).set_name("ip")) + .add(Linear(27, 2, with_bias=False).set_name("ip")) model = Model.load_caffe(module, proto_txt, model_path, bigdl_type="float") @@ -99,7 +99,7 @@ def test_load_caffe(self): -0.3027454615, 0.1254911423, 0.2114857137, 0.0392392874, 0.1668677032, 0.0506658256, 0.1139862537, 0.2874754369, -0.3273061812, 0.2115428150, -0.2002333999, -0.1621897519, - 0.0032395422, 0.2072965205]).astype("float").reshape((27, 2)) + 0.0032395422, 0.2072965205]).astype("float").reshape((2, 27)) self.assertTrue(np.allclose(parameters["conv"]["weight"], conv1_weight, atol=1e-6, rtol=0)) @@ -116,7 +116,7 @@ def test_load_caffe(self): module = Sequential()\ .add(SpatialConvolution(3, 4, 2, 2).set_name("conv"))\ .add(SpatialConvolution(4, 3, 2, 2).set_name("conv3"))\ - .add(Linear(2, 27, with_bias=False).set_name("ip")) + .add(Linear(27, 2, with_bias=False).set_name("ip")) model = Model.load_caffe(module, proto_txt, model_path, match_all=False) @@ -133,6 +133,20 @@ def test_load_caffe(self): self.assertTrue(np.allclose(parameters["ip"]["weight"], linear_weight, atol=1e-6, rtol=0)) + # test load caffe dynamically + model = Model.load_caffe_model(proto_txt, model_path, bigdl_type="float") + parameters = model.parameters() + self.assertTrue(np.allclose(parameters["conv"]["weight"], + conv1_weight, atol=1e-6, rtol=0)) + self.assertTrue(np.allclose(parameters["conv"]["bias"], + conv1_bias, atol=1e-6, rtol=0)) + self.assertTrue(np.allclose(parameters["conv2"]["weight"], + conv2_weight, atol=1e-6, rtol=0)) + self.assertTrue(np.allclose(parameters["conv2"]["bias"], + conv2_bias, atol=1e-6, rtol=0)) + self.assertTrue(np.allclose(parameters["ip"]["weight"], + linear_weight, atol=1e-6, rtol=0)) + if __name__ == "__main__": unittest.main() From 75c03b253b228cd10218d45793a49b9948580a2a Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Tue, 18 Jul 2017 00:56:14 -0700 Subject: [PATCH 155/823] refine tensorflow support doc (#1253) * refine tensorflow support * meet review * add tensorflow saver to python api and refine docs --- python/dllib/src/bigdl/dllib/nn/layer.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 563cfbc2d3c..833a16eb4c6 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -379,7 +379,7 @@ class Model(Container): It is allowed that some successors of the inputs node are not connect to outputs. If so, these nodes will be excluded in the computation. - We also support initialzing a Graph directly from a tensorflow module. In this case, you should + We also support initializing a Graph directly from a tensorflow module. In this case, you should pass your tensorflow nodes as inputs and outputs and also specify the byte_order parameter ("little_endian" or "big_endian") and node_type parameter ("bigdl" or "tensorflow") node_type parameter. @@ -458,6 +458,24 @@ def load_tensorflow(path, inputs, outputs, byte_order = "little_endian", bigdl_t jmodel = callBigDlFunc(bigdl_type, "loadTF", path, inputs, outputs, byte_order) return Model.of(jmodel) + @staticmethod + def save_tensorflow(model, inputs, path, byte_order="little_endian", data_format="nhwc"): + """ + Save a model to protobuf files so that it can be used in tensorflow inference. + + When saving the model, placeholders will be added to the tf model as input nodes. So + you need to pass in the names and shapes of the placeholders. BigDL model doesn't have + such information. The order of the placeholder information should be same as the inputs + of the graph model. + :param model: the model to be saved + :param inputs: placeholder information, should be an array of tuples (input_name, shape) + where 'input_name' is a string and shape is an array of integer + :param path: the path to be saved to + :param byte_order: model byte order + :param data_format: model data format, should be "nhwc" or "nchw" + """ + callBigDlFunc(model.bigdl_type, "saveTF", model, inputs, path, byte_order, data_format) + class Linear(Layer): From 016592743a0577de35083719901bf50492e230e9 Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Tue, 18 Jul 2017 00:56:14 -0700 Subject: [PATCH 156/823] refine tensorflow support doc (#1253) * refine tensorflow support * meet review * add tensorflow saver to python api and refine docs --- python/dllib/src/bigdl/dllib/nn/layer.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 563cfbc2d3c..833a16eb4c6 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -379,7 +379,7 @@ class Model(Container): It is allowed that some successors of the inputs node are not connect to outputs. If so, these nodes will be excluded in the computation. - We also support initialzing a Graph directly from a tensorflow module. In this case, you should + We also support initializing a Graph directly from a tensorflow module. In this case, you should pass your tensorflow nodes as inputs and outputs and also specify the byte_order parameter ("little_endian" or "big_endian") and node_type parameter ("bigdl" or "tensorflow") node_type parameter. @@ -458,6 +458,24 @@ def load_tensorflow(path, inputs, outputs, byte_order = "little_endian", bigdl_t jmodel = callBigDlFunc(bigdl_type, "loadTF", path, inputs, outputs, byte_order) return Model.of(jmodel) + @staticmethod + def save_tensorflow(model, inputs, path, byte_order="little_endian", data_format="nhwc"): + """ + Save a model to protobuf files so that it can be used in tensorflow inference. + + When saving the model, placeholders will be added to the tf model as input nodes. So + you need to pass in the names and shapes of the placeholders. BigDL model doesn't have + such information. The order of the placeholder information should be same as the inputs + of the graph model. + :param model: the model to be saved + :param inputs: placeholder information, should be an array of tuples (input_name, shape) + where 'input_name' is a string and shape is an array of integer + :param path: the path to be saved to + :param byte_order: model byte order + :param data_format: model data format, should be "nhwc" or "nchw" + """ + callBigDlFunc(model.bigdl_type, "saveTF", model, inputs, path, byte_order, data_format) + class Linear(Layer): From a622dc95926a951de8b6e797c07f71a2d7dca401 Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Tue, 18 Jul 2017 00:56:14 -0700 Subject: [PATCH 157/823] refine tensorflow support doc (#1253) * refine tensorflow support * meet review * add tensorflow saver to python api and refine docs --- python/dllib/test/dev/modules.py | 7 ++++ tensorflow_test.py | 57 ++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+) create mode 100644 tensorflow_test.py diff --git a/python/dllib/test/dev/modules.py b/python/dllib/test/dev/modules.py index 42849dab800..25313d95ecb 100644 --- a/python/dllib/test/dev/modules.py +++ b/python/dllib/test/dev/modules.py @@ -90,3 +90,10 @@ def __hash__(self): "test.load_caffe_test" ] ) + +test_tensorflow = Module( + name="tensorflow_test", + python_test_goals=[ + "test.tensorflow_test" + ] +) diff --git a/tensorflow_test.py b/tensorflow_test.py new file mode 100644 index 00000000000..dd450043c8d --- /dev/null +++ b/tensorflow_test.py @@ -0,0 +1,57 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Still in experimental stage! + +from bigdl.nn.layer import * +from bigdl.optim.optimizer import * +from bigdl.util.common import * +import numpy as np +import unittest +import shutil +import tempfile + + +class TestTensorflow(unittest.TestCase): + + def setUp(self): + sparkConf = create_spark_conf() + self.sc = SparkContext(master="local[4]", appName="test model", + conf=sparkConf) + init_engine() + + def tearDown(self): + self.sc.stop() + + def test_load_and_save(self): + linear = Linear(10, 2)() + sigmoid = Sigmoid()(linear) + softmax = SoftMax().set_name("output")(sigmoid) + model_original = Model([linear], [softmax]) + input = np.random.random((4, 10)) + + temp = tempfile.mkdtemp() + + Model.save_tensorflow(model_original, [("input", [4, 10])], temp + "/model.pb") + + model_loaded = Model.load_tensorflow(temp + "/model.pb", ["input"], ["output"]) + expected_output = model_original.forward(input) + output = model_loaded.forward(input) + self.assertTrue(np.allclose(output, + expected_output, atol=1e-6, rtol=0)) + + +if __name__ == "__main__": + unittest.main() From 66bf1a9a591f5cca19ccbb7c91d2e68bfa3b2789 Mon Sep 17 00:00:00 2001 From: Xin Qiu Date: Wed, 19 Jul 2017 08:39:30 +0800 Subject: [PATCH 158/823] update shared convolution (#1144) update shared convolution with latest SpatialConvolution. Add wRegularize, bRegularizer, initWeight, initBias, initGradWeight, initGradBias, withBias --- python/dllib/src/bigdl/dllib/nn/layer.py | 25 +++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 833a16eb4c6..07c8e3ff87b 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -2929,6 +2929,15 @@ class SpatialShareConvolution(Layer): >>> spatialShareConvolution = SpatialShareConvolution(1, 1, 1, 1) creating: createSpatialShareConvolution + >>> import numpy as np + >>> init_weight = np.random.randn(1, 12, 6, 5, 5) + >>> init_bias = np.random.randn(12) + >>> init_grad_weight = np.zeros([1, 12, 6, 5, 5]) + >>> init_grad_bias = np.zeros([12]) + >>> conv = SpatialShareConvolution(6, 12, 5, 5, 1, 1, 0, 0, 1, True, L1Regularizer(0.5), L1Regularizer(0.5), init_weight, init_bias, init_grad_weight, init_grad_bias) + creating: createL1Regularizer + creating: createL1Regularizer + creating: createSpatialShareConvolution ''' def __init__(self, @@ -2942,6 +2951,13 @@ def __init__(self, pad_h=0, n_group=1, propagate_back=True, + wRegularizer=None, + bRegularizer=None, + init_weight=None, + init_bias=None, + init_grad_weight=None, + init_grad_bias=None, + with_bias=True, bigdl_type="float"): super(SpatialShareConvolution, self).__init__(None, bigdl_type, n_input_plane, @@ -2953,7 +2969,14 @@ def __init__(self, pad_w, pad_h, n_group, - propagate_back) + propagate_back, + wRegularizer, + bRegularizer, + JTensor.from_ndarray(init_weight), + JTensor.from_ndarray(init_bias), + JTensor.from_ndarray(init_grad_weight), + JTensor.from_ndarray(init_grad_bias), + with_bias) def set_init_method(self, weight_init_method = None, bias_init_method = None): callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, weight_init_method, bias_init_method) From 55817c6c865dcc942c97cb59a1a29ec3d3c10e5a Mon Sep 17 00:00:00 2001 From: Xin Qiu Date: Wed, 19 Jul 2017 08:39:30 +0800 Subject: [PATCH 159/823] update shared convolution (#1144) update shared convolution with latest SpatialConvolution. Add wRegularize, bRegularizer, initWeight, initBias, initGradWeight, initGradBias, withBias --- python/dllib/src/bigdl/dllib/nn/layer.py | 25 +++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 833a16eb4c6..07c8e3ff87b 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -2929,6 +2929,15 @@ class SpatialShareConvolution(Layer): >>> spatialShareConvolution = SpatialShareConvolution(1, 1, 1, 1) creating: createSpatialShareConvolution + >>> import numpy as np + >>> init_weight = np.random.randn(1, 12, 6, 5, 5) + >>> init_bias = np.random.randn(12) + >>> init_grad_weight = np.zeros([1, 12, 6, 5, 5]) + >>> init_grad_bias = np.zeros([12]) + >>> conv = SpatialShareConvolution(6, 12, 5, 5, 1, 1, 0, 0, 1, True, L1Regularizer(0.5), L1Regularizer(0.5), init_weight, init_bias, init_grad_weight, init_grad_bias) + creating: createL1Regularizer + creating: createL1Regularizer + creating: createSpatialShareConvolution ''' def __init__(self, @@ -2942,6 +2951,13 @@ def __init__(self, pad_h=0, n_group=1, propagate_back=True, + wRegularizer=None, + bRegularizer=None, + init_weight=None, + init_bias=None, + init_grad_weight=None, + init_grad_bias=None, + with_bias=True, bigdl_type="float"): super(SpatialShareConvolution, self).__init__(None, bigdl_type, n_input_plane, @@ -2953,7 +2969,14 @@ def __init__(self, pad_w, pad_h, n_group, - propagate_back) + propagate_back, + wRegularizer, + bRegularizer, + JTensor.from_ndarray(init_weight), + JTensor.from_ndarray(init_bias), + JTensor.from_ndarray(init_grad_weight), + JTensor.from_ndarray(init_grad_bias), + with_bias) def set_init_method(self, weight_init_method = None, bias_init_method = None): callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, weight_init_method, bias_init_method) From 3782ca800236d49470d02e32e10560d5fa3a51f5 Mon Sep 17 00:00:00 2001 From: Ian Wong Date: Wed, 19 Jul 2017 00:19:56 -0500 Subject: [PATCH 160/823] Graph doc (#1275) * Add Input docs * fix bugs * meet code review --- python/dllib/src/bigdl/dllib/nn/layer.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 07c8e3ff87b..4a888152396 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -1648,6 +1648,27 @@ def __init__(self, bigdl_type="float"): super(CosineDistance, self).__init__(None, bigdl_type) +class Input(Node): + + ''' + Input layer do nothing to the input tensors, just passing them through. It is used as input to + the Graph container (add a link) when the first layer of the graph container accepts multiple + tensors as inputs. + + Each input node of the graph container should accept one tensor as input. If you want a module + accepting multiple tensors as input, you should add some Input module before it and connect + the outputs of the Input nodes to it. + + Please note that the return is not a layer but a Node containing input layer. + + >>> input = Input() + creating: createInput + ''' + + def __init__(self, + bigdl_type="float"): + super(Input, self).__init__(None, bigdl_type) + class DotProduct(Layer): From bca45c7333741f6766cdbee4622b278807937476 Mon Sep 17 00:00:00 2001 From: Ian Wong Date: Wed, 19 Jul 2017 00:19:56 -0500 Subject: [PATCH 161/823] Graph doc (#1275) * Add Input docs * fix bugs * meet code review --- python/dllib/src/bigdl/dllib/nn/layer.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 07c8e3ff87b..4a888152396 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -1648,6 +1648,27 @@ def __init__(self, bigdl_type="float"): super(CosineDistance, self).__init__(None, bigdl_type) +class Input(Node): + + ''' + Input layer do nothing to the input tensors, just passing them through. It is used as input to + the Graph container (add a link) when the first layer of the graph container accepts multiple + tensors as inputs. + + Each input node of the graph container should accept one tensor as input. If you want a module + accepting multiple tensors as input, you should add some Input module before it and connect + the outputs of the Input nodes to it. + + Please note that the return is not a layer but a Node containing input layer. + + >>> input = Input() + creating: createInput + ''' + + def __init__(self, + bigdl_type="float"): + super(Input, self).__init__(None, bigdl_type) + class DotProduct(Layer): From e6120b83d750d5dd3acb9bdf546b2e736a21abf0 Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Tue, 18 Jul 2017 22:45:28 -0700 Subject: [PATCH 162/823] Refine tensorflow doc (a follow up of pr #1253 to meet code review) (#1272) * meet code review * fix example * meet code review * refine title * make saveTF compatible with saveCaffe --- python/dllib/src/bigdl/dllib/nn/layer.py | 35 ++++++++++++------------ 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 4a888152396..04e3e477d67 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -327,6 +327,23 @@ def save_caffe(self, prototxt_path, model_path, use_v2 = True, overwrite = False callBigDlFunc(self.bigdl_type, "saveCaffe", self.value, prototxt_path, model_path, use_v2, overwrite) + def save_tensorflow(self, inputs, path, byte_order="little_endian", data_format="nhwc"): + """ + Save a model to protobuf files so that it can be used in tensorflow inference. + + When saving the model, placeholders will be added to the tf model as input nodes. So + you need to pass in the names and shapes of the placeholders. BigDL model doesn't have + such information. The order of the placeholder information should be same as the inputs + of the graph model. + :param inputs: placeholder information, should be an array of tuples (input_name, shape) + where 'input_name' is a string and shape is an array of integer + :param path: the path to be saved to + :param byte_order: model byte order + :param data_format: model data format, should be "nhwc" or "nchw" + """ + callBigDlFunc(self.bigdl_type, "saveTF", self.value, inputs, path, byte_order, data_format) + + def setWRegularizer(self, wRegularizer): ''' set weight regularizer @@ -458,24 +475,6 @@ def load_tensorflow(path, inputs, outputs, byte_order = "little_endian", bigdl_t jmodel = callBigDlFunc(bigdl_type, "loadTF", path, inputs, outputs, byte_order) return Model.of(jmodel) - @staticmethod - def save_tensorflow(model, inputs, path, byte_order="little_endian", data_format="nhwc"): - """ - Save a model to protobuf files so that it can be used in tensorflow inference. - - When saving the model, placeholders will be added to the tf model as input nodes. So - you need to pass in the names and shapes of the placeholders. BigDL model doesn't have - such information. The order of the placeholder information should be same as the inputs - of the graph model. - :param model: the model to be saved - :param inputs: placeholder information, should be an array of tuples (input_name, shape) - where 'input_name' is a string and shape is an array of integer - :param path: the path to be saved to - :param byte_order: model byte order - :param data_format: model data format, should be "nhwc" or "nchw" - """ - callBigDlFunc(model.bigdl_type, "saveTF", model, inputs, path, byte_order, data_format) - class Linear(Layer): From 8c13c40808a1db17b76629ad6582d49f2e96df33 Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Tue, 18 Jul 2017 22:45:28 -0700 Subject: [PATCH 163/823] Refine tensorflow doc (a follow up of pr #1253 to meet code review) (#1272) * meet code review * fix example * meet code review * refine title * make saveTF compatible with saveCaffe --- python/dllib/src/bigdl/dllib/nn/layer.py | 35 ++++++++++++------------ 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 4a888152396..04e3e477d67 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -327,6 +327,23 @@ def save_caffe(self, prototxt_path, model_path, use_v2 = True, overwrite = False callBigDlFunc(self.bigdl_type, "saveCaffe", self.value, prototxt_path, model_path, use_v2, overwrite) + def save_tensorflow(self, inputs, path, byte_order="little_endian", data_format="nhwc"): + """ + Save a model to protobuf files so that it can be used in tensorflow inference. + + When saving the model, placeholders will be added to the tf model as input nodes. So + you need to pass in the names and shapes of the placeholders. BigDL model doesn't have + such information. The order of the placeholder information should be same as the inputs + of the graph model. + :param inputs: placeholder information, should be an array of tuples (input_name, shape) + where 'input_name' is a string and shape is an array of integer + :param path: the path to be saved to + :param byte_order: model byte order + :param data_format: model data format, should be "nhwc" or "nchw" + """ + callBigDlFunc(self.bigdl_type, "saveTF", self.value, inputs, path, byte_order, data_format) + + def setWRegularizer(self, wRegularizer): ''' set weight regularizer @@ -458,24 +475,6 @@ def load_tensorflow(path, inputs, outputs, byte_order = "little_endian", bigdl_t jmodel = callBigDlFunc(bigdl_type, "loadTF", path, inputs, outputs, byte_order) return Model.of(jmodel) - @staticmethod - def save_tensorflow(model, inputs, path, byte_order="little_endian", data_format="nhwc"): - """ - Save a model to protobuf files so that it can be used in tensorflow inference. - - When saving the model, placeholders will be added to the tf model as input nodes. So - you need to pass in the names and shapes of the placeholders. BigDL model doesn't have - such information. The order of the placeholder information should be same as the inputs - of the graph model. - :param model: the model to be saved - :param inputs: placeholder information, should be an array of tuples (input_name, shape) - where 'input_name' is a string and shape is an array of integer - :param path: the path to be saved to - :param byte_order: model byte order - :param data_format: model data format, should be "nhwc" or "nchw" - """ - callBigDlFunc(model.bigdl_type, "saveTF", model, inputs, path, byte_order, data_format) - class Linear(Layer): From 62766cfcefef6fb497d7ea4ceefc1d324037e7b6 Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Tue, 18 Jul 2017 22:45:28 -0700 Subject: [PATCH 164/823] Refine tensorflow doc (a follow up of pr #1253 to meet code review) (#1272) * meet code review * fix example * meet code review * refine title * make saveTF compatible with saveCaffe --- tensorflow_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow_test.py b/tensorflow_test.py index dd450043c8d..0a0ea2ccd1d 100644 --- a/tensorflow_test.py +++ b/tensorflow_test.py @@ -44,7 +44,7 @@ def test_load_and_save(self): temp = tempfile.mkdtemp() - Model.save_tensorflow(model_original, [("input", [4, 10])], temp + "/model.pb") + model_original.save_tensorflow([("input", [4, 10])], temp + "/model.pb") model_loaded = Model.load_tensorflow(temp + "/model.pb", ["input"], ["output"]) expected_output = model_original.forward(input) From cc7437eb2323cce1cd01c47886ef320239b33383 Mon Sep 17 00:00:00 2001 From: Yan Wan Date: Wed, 19 Jul 2017 20:28:29 +0800 Subject: [PATCH 165/823] add CosineDistanceCriterion (#1195) * add CosineDistanceCriterion * add cosineDistanceCriterion * add CosineDistanceCriterion python test * add python test * add blank line in pythonapi * revise APIDocs --- python/dllib/src/bigdl/dllib/nn/criterion.py | 21 ++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/nn/criterion.py b/python/dllib/src/bigdl/dllib/nn/criterion.py index 38d9016ef1c..ebffa855939 100644 --- a/python/dllib/src/bigdl/dllib/nn/criterion.py +++ b/python/dllib/src/bigdl/dllib/nn/criterion.py @@ -210,6 +210,27 @@ def __init__(self, n_classes) +class CosineDistanceCriterion(Criterion): + + """ + Creates a criterion that measures the loss given an input and target, + Loss = 1 - cos(x, y) + + + >>> cosineDistanceCriterion = CosineDistanceCriterion(True) + creating: createCosineDistanceCriterion + >>> cosineDistanceCriterion.forward(np.array([1.0, 2.0, 3.0, 4.0, 5.0]), + ... np.array([5.0, 4.0, 3.0, 2.0, 1.0])) + 0.07272728 + """ + + def __init__(self, + size_average=True, + bigdl_type="float"): + super(CosineDistanceCriterion, self).__init__(None, bigdl_type, + size_average) + + class CosineEmbeddingCriterion(Criterion): """ From 36941dffa7e15bab0963751119007af626ca1a19 Mon Sep 17 00:00:00 2001 From: Yan Wan Date: Wed, 19 Jul 2017 20:28:29 +0800 Subject: [PATCH 166/823] add CosineDistanceCriterion (#1195) * add CosineDistanceCriterion * add cosineDistanceCriterion * add CosineDistanceCriterion python test * add python test * add blank line in pythonapi * revise APIDocs --- python/dllib/src/bigdl/dllib/nn/criterion.py | 21 ++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/nn/criterion.py b/python/dllib/src/bigdl/dllib/nn/criterion.py index 38d9016ef1c..ebffa855939 100644 --- a/python/dllib/src/bigdl/dllib/nn/criterion.py +++ b/python/dllib/src/bigdl/dllib/nn/criterion.py @@ -210,6 +210,27 @@ def __init__(self, n_classes) +class CosineDistanceCriterion(Criterion): + + """ + Creates a criterion that measures the loss given an input and target, + Loss = 1 - cos(x, y) + + + >>> cosineDistanceCriterion = CosineDistanceCriterion(True) + creating: createCosineDistanceCriterion + >>> cosineDistanceCriterion.forward(np.array([1.0, 2.0, 3.0, 4.0, 5.0]), + ... np.array([5.0, 4.0, 3.0, 2.0, 1.0])) + 0.07272728 + """ + + def __init__(self, + size_average=True, + bigdl_type="float"): + super(CosineDistanceCriterion, self).__init__(None, bigdl_type, + size_average) + + class CosineEmbeddingCriterion(Criterion): """ From 421b2601cab4fa9f814bd0eb00d1c67ed7c7c631 Mon Sep 17 00:00:00 2001 From: Ian Wong Date: Thu, 20 Jul 2017 03:09:02 -0500 Subject: [PATCH 167/823] update py lenet doc (#1312) --- python/dllib/src/bigdl/dllib/models/lenet/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/models/lenet/README.md b/python/dllib/src/bigdl/dllib/models/lenet/README.md index 7472e743ef4..9de42a34b7d 100644 --- a/python/dllib/src/bigdl/dllib/models/lenet/README.md +++ b/python/dllib/src/bigdl/dllib/models/lenet/README.md @@ -4,6 +4,8 @@ LeNet5 is a classical CNN model used in digital number classification. For detai please refer to . ## How to run this example: +Please note that due to some permission issue, this example **cannot** be run on Windows. + Program would download the mnist data into ```/tmp/mnist``` automatically by default. From a4345775b366664ba0f437fca7cdf3402cadae7e Mon Sep 17 00:00:00 2001 From: Ian Wong Date: Thu, 20 Jul 2017 03:09:02 -0500 Subject: [PATCH 168/823] update py lenet doc (#1312) --- python/dllib/src/bigdl/dllib/models/lenet/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/models/lenet/README.md b/python/dllib/src/bigdl/dllib/models/lenet/README.md index 7472e743ef4..9de42a34b7d 100644 --- a/python/dllib/src/bigdl/dllib/models/lenet/README.md +++ b/python/dllib/src/bigdl/dllib/models/lenet/README.md @@ -4,6 +4,8 @@ LeNet5 is a classical CNN model used in digital number classification. For detai please refer to . ## How to run this example: +Please note that due to some permission issue, this example **cannot** be run on Windows. + Program would download the mnist data into ```/tmp/mnist``` automatically by default. From 948499b81f7a56247fe2b811e813567c922d97ad Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Thu, 20 Jul 2017 16:26:04 +0800 Subject: [PATCH 169/823] remove eample folder (#1199) --- ...n-local-functional_api_forward_backward.sh | 21 ------------------- .../commands/run-tf-example.sh | 19 ----------------- 2 files changed, 40 deletions(-) delete mode 100755 python/dllib/test/local_integration/commands/run-local-functional_api_forward_backward.sh delete mode 100755 python/dllib/test/local_integration/commands/run-tf-example.sh diff --git a/python/dllib/test/local_integration/commands/run-local-functional_api_forward_backward.sh b/python/dllib/test/local_integration/commands/run-local-functional_api_forward_backward.sh deleted file mode 100755 index 1368e1e164a..00000000000 --- a/python/dllib/test/local_integration/commands/run-local-functional_api_forward_backward.sh +++ /dev/null @@ -1,21 +0,0 @@ -# -# Copyright 2016 The BigDL Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -cd "`dirname $0`" - -run_notebook $BIGDL_HOME/pyspark/example/functional_api_forward_backward.ipynb - - diff --git a/python/dllib/test/local_integration/commands/run-tf-example.sh b/python/dllib/test/local_integration/commands/run-tf-example.sh deleted file mode 100755 index 66f331a929b..00000000000 --- a/python/dllib/test/local_integration/commands/run-tf-example.sh +++ /dev/null @@ -1,19 +0,0 @@ -# -# Copyright 2016 The BigDL Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -cd "`dirname $0`" - -$PYTHON_EXECUTABLE $BIGDL_HOME/pyspark/example/tf_example.py \ No newline at end of file From a15b194b94124ca40289baba73d5ff6f29b2ade9 Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Thu, 20 Jul 2017 17:13:21 +0800 Subject: [PATCH 170/823] Update python doc (#1304) * update python doc * Update install-pre-built.md * update * update * Update install-from-pip.md * Update python-examples.md * update * update * update * Update install-without-pip.md * Update install-without-pip.md * update * update bigdl * Update python-resources.md --- python/dllib/src/bigdl/dllib/models/lenet/README.md | 4 +++- .../dllib/src/bigdl/dllib/models/textclassifier/README.md | 6 ++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/models/lenet/README.md b/python/dllib/src/bigdl/dllib/models/lenet/README.md index 9de42a34b7d..43aecbd5d34 100644 --- a/python/dllib/src/bigdl/dllib/models/lenet/README.md +++ b/python/dllib/src/bigdl/dllib/models/lenet/README.md @@ -3,6 +3,9 @@ LeNet5 is a classical CNN model used in digital number classification. For detail information, please refer to . +## Install dependencies + * [Install dependencies](../../../README.md#install.dependencies) + ## How to run this example: Please note that due to some permission issue, this example **cannot** be run on Windows. @@ -36,7 +39,6 @@ We would train a LeNet model in spark local mode with the following commands and --total-executor-cores 2 \ --executor-cores 2 \ --executor-memory 4g \ - --conf spark.akka.frameSize=64 \ --py-files ${PYTHON_API_ZIP_PATH},${BigDL_HOME}/pyspark/bigdl/models/lenet/lenet5.py \ --properties-file ${BigDL_HOME}/dist/conf/spark-bigdl.conf \ --jars ${BigDL_JAR_PATH} \ diff --git a/python/dllib/src/bigdl/dllib/models/textclassifier/README.md b/python/dllib/src/bigdl/dllib/models/textclassifier/README.md index 39875980000..5be8a7a101a 100644 --- a/python/dllib/src/bigdl/dllib/models/textclassifier/README.md +++ b/python/dllib/src/bigdl/dllib/models/textclassifier/README.md @@ -8,6 +8,9 @@ * Embedding: 100-dimensional pre-trained GloVe embeddings of 400k words which trained on a 2014 dump of English Wikipedia. * Training data: "20 Newsgroup dataset" which containing 20 categories and with totally 19997 texts. +## Install dependencies + * [Install dependencies](../../../README.md#install.dependencies) + ## How to run this example: If there is no [Pre-train GloVe word embeddings](http://nlp.stanford.edu/data/glove.6B.zip) @@ -24,7 +27,7 @@ $ [/tmp/news20]$ tree . -L 1 then running the flowing script would automatically download the data during the first run. ```{r, engine='sh'} - PYTHONHASHSEED=... + PYTHONHASHSEED=0 BigDL_HOME=... SPARK_HOME=... MASTER=... @@ -39,7 +42,6 @@ then running the flowing script would automatically download the data during the --total-executor-cores 4 \ --executor-cores 4 \ --executor-memory 20g \ - --conf spark.akka.frameSize=64 \ --py-files ${PYTHON_API_ZIP_PATH},${BigDL_HOME}/pyspark/bigdl/models/textclassifier/textclassifier.py \ --jars ${BigDL_JAR_PATH} \ --conf spark.driver.extraClassPath=${BigDL_JAR_PATH} \ From b104ae6041705562af9054554b355ddf007cae8d Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Thu, 20 Jul 2017 17:13:21 +0800 Subject: [PATCH 171/823] Update python doc (#1304) * update python doc * Update install-pre-built.md * update * update * Update install-from-pip.md * Update python-examples.md * update * update * update * Update install-without-pip.md * Update install-without-pip.md * update * update bigdl * Update python-resources.md --- python/dllib/src/bigdl/dllib/models/lenet/README.md | 4 +++- .../dllib/src/bigdl/dllib/models/textclassifier/README.md | 6 ++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/models/lenet/README.md b/python/dllib/src/bigdl/dllib/models/lenet/README.md index 9de42a34b7d..43aecbd5d34 100644 --- a/python/dllib/src/bigdl/dllib/models/lenet/README.md +++ b/python/dllib/src/bigdl/dllib/models/lenet/README.md @@ -3,6 +3,9 @@ LeNet5 is a classical CNN model used in digital number classification. For detail information, please refer to . +## Install dependencies + * [Install dependencies](../../../README.md#install.dependencies) + ## How to run this example: Please note that due to some permission issue, this example **cannot** be run on Windows. @@ -36,7 +39,6 @@ We would train a LeNet model in spark local mode with the following commands and --total-executor-cores 2 \ --executor-cores 2 \ --executor-memory 4g \ - --conf spark.akka.frameSize=64 \ --py-files ${PYTHON_API_ZIP_PATH},${BigDL_HOME}/pyspark/bigdl/models/lenet/lenet5.py \ --properties-file ${BigDL_HOME}/dist/conf/spark-bigdl.conf \ --jars ${BigDL_JAR_PATH} \ diff --git a/python/dllib/src/bigdl/dllib/models/textclassifier/README.md b/python/dllib/src/bigdl/dllib/models/textclassifier/README.md index 39875980000..5be8a7a101a 100644 --- a/python/dllib/src/bigdl/dllib/models/textclassifier/README.md +++ b/python/dllib/src/bigdl/dllib/models/textclassifier/README.md @@ -8,6 +8,9 @@ * Embedding: 100-dimensional pre-trained GloVe embeddings of 400k words which trained on a 2014 dump of English Wikipedia. * Training data: "20 Newsgroup dataset" which containing 20 categories and with totally 19997 texts. +## Install dependencies + * [Install dependencies](../../../README.md#install.dependencies) + ## How to run this example: If there is no [Pre-train GloVe word embeddings](http://nlp.stanford.edu/data/glove.6B.zip) @@ -24,7 +27,7 @@ $ [/tmp/news20]$ tree . -L 1 then running the flowing script would automatically download the data during the first run. ```{r, engine='sh'} - PYTHONHASHSEED=... + PYTHONHASHSEED=0 BigDL_HOME=... SPARK_HOME=... MASTER=... @@ -39,7 +42,6 @@ then running the flowing script would automatically download the data during the --total-executor-cores 4 \ --executor-cores 4 \ --executor-memory 20g \ - --conf spark.akka.frameSize=64 \ --py-files ${PYTHON_API_ZIP_PATH},${BigDL_HOME}/pyspark/bigdl/models/textclassifier/textclassifier.py \ --jars ${BigDL_JAR_PATH} \ --conf spark.driver.extraClassPath=${BigDL_JAR_PATH} \ From 351258f407a526584c1522d851de2d2d0b0ef92d Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Fri, 21 Jul 2017 10:53:39 +0800 Subject: [PATCH 172/823] Remove experimental statement from header of python api (#1334) * remove experimental statement from python api * clean --- python/dllib/src/bigdl/dllib/models/lenet/lenet5.py | 1 - python/dllib/src/bigdl/dllib/models/rnn/rnnexample.py | 1 - .../src/bigdl/dllib/models/textclassifier/textclassifier.py | 1 - 3 files changed, 3 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py b/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py index 6de0b0124e1..2544eb8c98c 100644 --- a/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py +++ b/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py @@ -13,7 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. # -# Still in experimental stage! from optparse import OptionParser from bigdl.dataset import mnist diff --git a/python/dllib/src/bigdl/dllib/models/rnn/rnnexample.py b/python/dllib/src/bigdl/dllib/models/rnn/rnnexample.py index da49781f472..dddaff1f61f 100644 --- a/python/dllib/src/bigdl/dllib/models/rnn/rnnexample.py +++ b/python/dllib/src/bigdl/dllib/models/rnn/rnnexample.py @@ -14,7 +14,6 @@ # limitations under the License. # -# Still in experimental stage! import itertools import re diff --git a/python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py b/python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py index 9e235990efd..1864fac9fce 100644 --- a/python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py +++ b/python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py @@ -14,7 +14,6 @@ # limitations under the License. # -# Still in experimental stage! import itertools import re From d94a1fac106dba842d43f24689c1b73c2eceafc0 Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Fri, 21 Jul 2017 10:53:39 +0800 Subject: [PATCH 173/823] Remove experimental statement from header of python api (#1334) * remove experimental statement from python api * clean --- python/dllib/src/bigdl/dllib/models/lenet/lenet5.py | 1 - python/dllib/src/bigdl/dllib/models/rnn/rnnexample.py | 1 - .../src/bigdl/dllib/models/textclassifier/textclassifier.py | 1 - 3 files changed, 3 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py b/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py index 6de0b0124e1..2544eb8c98c 100644 --- a/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py +++ b/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py @@ -13,7 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. # -# Still in experimental stage! from optparse import OptionParser from bigdl.dataset import mnist diff --git a/python/dllib/src/bigdl/dllib/models/rnn/rnnexample.py b/python/dllib/src/bigdl/dllib/models/rnn/rnnexample.py index da49781f472..dddaff1f61f 100644 --- a/python/dllib/src/bigdl/dllib/models/rnn/rnnexample.py +++ b/python/dllib/src/bigdl/dllib/models/rnn/rnnexample.py @@ -14,7 +14,6 @@ # limitations under the License. # -# Still in experimental stage! import itertools import re diff --git a/python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py b/python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py index 9e235990efd..1864fac9fce 100644 --- a/python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py +++ b/python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py @@ -14,7 +14,6 @@ # limitations under the License. # -# Still in experimental stage! import itertools import re From d746415bce9d0a0a69edfb9a15173afb83954468 Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Fri, 21 Jul 2017 10:53:39 +0800 Subject: [PATCH 174/823] Remove experimental statement from header of python api (#1334) * remove experimental statement from python api * clean --- load_caffe_test.py | 1 - simple_integration_test.py | 1 - tensorflow_test.py | 1 - 3 files changed, 3 deletions(-) diff --git a/load_caffe_test.py b/load_caffe_test.py index 7cd6e50d292..e3507d31770 100644 --- a/load_caffe_test.py +++ b/load_caffe_test.py @@ -13,7 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. # -# Still in experimental stage! from bigdl.nn.layer import * from bigdl.optim.optimizer import * diff --git a/simple_integration_test.py b/simple_integration_test.py index 1470dd6b767..506725f2d9e 100644 --- a/simple_integration_test.py +++ b/simple_integration_test.py @@ -13,7 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. # -# Still in experimental stage! from bigdl.nn.layer import * from bigdl.nn.initialization_method import * diff --git a/tensorflow_test.py b/tensorflow_test.py index 0a0ea2ccd1d..c20208842f3 100644 --- a/tensorflow_test.py +++ b/tensorflow_test.py @@ -13,7 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. # -# Still in experimental stage! from bigdl.nn.layer import * from bigdl.optim.optimizer import * From 17d1a44d4209ba1ad78ea03d4f043ef720ccf539 Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Fri, 21 Jul 2017 11:13:18 +0800 Subject: [PATCH 175/823] Update README.md --- .../src/bigdl/dllib/models/textclassifier/README.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/python/dllib/src/bigdl/dllib/models/textclassifier/README.md b/python/dllib/src/bigdl/dllib/models/textclassifier/README.md index 5be8a7a101a..440c746d2ca 100644 --- a/python/dllib/src/bigdl/dllib/models/textclassifier/README.md +++ b/python/dllib/src/bigdl/dllib/models/textclassifier/README.md @@ -23,8 +23,16 @@ $ [/tmp/news20]$ tree . -L 1 ├── 20news-19997.tar.gz └── glove.6B.zip ``` +- The example code would automatically download the data during the first run. +- If you install BigDL via pip, you can run this example locally by the following command: + - Check [Run after pip install](../../../../docs/docs/PythonSupport/run-from-pip.md) +``` +export SPARK_HOME=path to spark-1.6.3 +python ${BigDL_HOME}/pyspark/bigdl/models/textclassifier/textclassifier.py --max_epoch 3 --model cnn + +``` -then running the flowing script would automatically download the data during the first run. +- You can also use `spark-submit` to launch this example: ```{r, engine='sh'} PYTHONHASHSEED=0 From 2fcbfc31f20ed6a3c87c430d9071c419f829e60a Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Fri, 21 Jul 2017 11:13:18 +0800 Subject: [PATCH 176/823] Update README.md --- .../src/bigdl/dllib/models/textclassifier/README.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/python/dllib/src/bigdl/dllib/models/textclassifier/README.md b/python/dllib/src/bigdl/dllib/models/textclassifier/README.md index 5be8a7a101a..440c746d2ca 100644 --- a/python/dllib/src/bigdl/dllib/models/textclassifier/README.md +++ b/python/dllib/src/bigdl/dllib/models/textclassifier/README.md @@ -23,8 +23,16 @@ $ [/tmp/news20]$ tree . -L 1 ├── 20news-19997.tar.gz └── glove.6B.zip ``` +- The example code would automatically download the data during the first run. +- If you install BigDL via pip, you can run this example locally by the following command: + - Check [Run after pip install](../../../../docs/docs/PythonSupport/run-from-pip.md) +``` +export SPARK_HOME=path to spark-1.6.3 +python ${BigDL_HOME}/pyspark/bigdl/models/textclassifier/textclassifier.py --max_epoch 3 --model cnn + +``` -then running the flowing script would automatically download the data during the first run. +- You can also use `spark-submit` to launch this example: ```{r, engine='sh'} PYTHONHASHSEED=0 From 591d1d480a5045916924f1b4b0d097705ad2ec33 Mon Sep 17 00:00:00 2001 From: zhangxiaoli73 <380761639@qq.com> Date: Fri, 21 Jul 2017 12:56:07 +0800 Subject: [PATCH 177/823] add predictClass for python (#1335) --- python/dllib/src/bigdl/dllib/nn/layer.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 04e3e477d67..50a9a6cc010 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -246,6 +246,17 @@ def predict(self, data_rdd): "modelPredictRDD", self.value, data_rdd) return result.map(lambda data: data.to_ndarray()) + def predict_class(self, data_rdd): + """ + module predict, return the predict label + + :param data_rdd: the data to be predict. + :return: An RDD represent the predict label. + """ + result = callBigDlFunc(self.bigdl_type, + "modelPredictClass", self.value, data_rdd) + return result + def test(self, val_rdd, batch_size, val_methods): """ A method to benchmark the model quality. From a662f66ba89eeff04d738f0cb36474b2829037bd Mon Sep 17 00:00:00 2001 From: zhangxiaoli73 <380761639@qq.com> Date: Fri, 21 Jul 2017 12:56:07 +0800 Subject: [PATCH 178/823] add predictClass for python (#1335) --- python/dllib/src/bigdl/dllib/nn/layer.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 04e3e477d67..50a9a6cc010 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -246,6 +246,17 @@ def predict(self, data_rdd): "modelPredictRDD", self.value, data_rdd) return result.map(lambda data: data.to_ndarray()) + def predict_class(self, data_rdd): + """ + module predict, return the predict label + + :param data_rdd: the data to be predict. + :return: An RDD represent the predict label. + """ + result = callBigDlFunc(self.bigdl_type, + "modelPredictClass", self.value, data_rdd) + return result + def test(self, val_rdd, batch_size, val_methods): """ A method to benchmark the model quality. From 0aebc3f4fe5a0838e7b9be6a2577a8b448a4c21e Mon Sep 17 00:00:00 2001 From: zhangxiaoli73 <380761639@qq.com> Date: Fri, 21 Jul 2017 12:56:07 +0800 Subject: [PATCH 179/823] add predictClass for python (#1335) --- simple_integration_test.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/simple_integration_test.py b/simple_integration_test.py index 506725f2d9e..72392ec14a3 100644 --- a/simple_integration_test.py +++ b/simple_integration_test.py @@ -316,6 +316,10 @@ def test_predict(self): [-0.5906958], [-0.12307882], [-0.77907401]], dtype="float32") for i in range(0, total_length): self.assertTrue(np.allclose(p[i], ground_label[i], atol=1e-6, rtol=0)) + predict_class = model.predict_class(predict_data) + predict_labels = predict_class.take(6) + for i in range(0, total_length): + self.assertTrue(predict_labels[i] == 1) def test_rng(self): rng = RNG() From 27a5d49ba8498aca597a8ed790ac7bcdf383b40f Mon Sep 17 00:00:00 2001 From: jenniew Date: Thu, 20 Jul 2017 22:41:14 -0700 Subject: [PATCH 180/823] update python text classifier readme (#1342) --- python/dllib/src/bigdl/dllib/models/textclassifier/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/models/textclassifier/README.md b/python/dllib/src/bigdl/dllib/models/textclassifier/README.md index 440c746d2ca..f7c0b946c55 100644 --- a/python/dllib/src/bigdl/dllib/models/textclassifier/README.md +++ b/python/dllib/src/bigdl/dllib/models/textclassifier/README.md @@ -13,6 +13,8 @@ ## How to run this example: +Please note that due to some permission issue, this example **cannot** be run on Windows. + If there is no [Pre-train GloVe word embeddings](http://nlp.stanford.edu/data/glove.6B.zip) or [20 Newsgroup dataset](http://www.cs.cmu.edu/afs/cs.cmu.edu/project/theo-20/www/data/news20.html) in `/tmp/news20` directory with the following structure looks like: From ef863d2ccfbd7a2a788af4c0698b111294c5935c Mon Sep 17 00:00:00 2001 From: jenniew Date: Thu, 20 Jul 2017 22:41:14 -0700 Subject: [PATCH 181/823] update python text classifier readme (#1342) --- python/dllib/src/bigdl/dllib/models/textclassifier/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/models/textclassifier/README.md b/python/dllib/src/bigdl/dllib/models/textclassifier/README.md index 440c746d2ca..f7c0b946c55 100644 --- a/python/dllib/src/bigdl/dllib/models/textclassifier/README.md +++ b/python/dllib/src/bigdl/dllib/models/textclassifier/README.md @@ -13,6 +13,8 @@ ## How to run this example: +Please note that due to some permission issue, this example **cannot** be run on Windows. + If there is no [Pre-train GloVe word embeddings](http://nlp.stanford.edu/data/glove.6B.zip) or [20 Newsgroup dataset](http://www.cs.cmu.edu/afs/cs.cmu.edu/project/theo-20/www/data/news20.html) in `/tmp/news20` directory with the following structure looks like: From 9d4a33cd536751bca0390d8cd5410a7a8ecc2c76 Mon Sep 17 00:00:00 2001 From: dding3 Date: Tue, 25 Jul 2017 22:17:34 -0700 Subject: [PATCH 182/823] Convlstm3d (#1361) * support convlstm3d --- python/dllib/src/bigdl/dllib/nn/layer.py | 38 ++++++++++++++++++++---- 1 file changed, 32 insertions(+), 6 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 50a9a6cc010..19ab2061fff 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -3617,7 +3617,7 @@ class Pack(Layer): def __init__(self, dimension, bigdl_type="float"): super(Pack, self).__init__(None, bigdl_type, dimension) -class ConvLSTMPeephole(Layer): +class ConvLSTMPeephole2D(Layer): ''' | Convolution Long Short Term Memory architecture with peephole. @@ -3632,19 +3632,45 @@ class ConvLSTMPeephole(Layer): :param wRegularizer: instance of [[Regularizer]](eg. L1 or L2 regularization), applied to the input weights matrices :param uRegularizer: instance [[Regularizer]](eg. L1 or L2 regularization), applied to the recurrent weights matrices :param bRegularizer: instance of [[Regularizer]]applied to the bias. - :param with_peephold: whether use last cell status control a gate. + :param with_peephole: whether use last cell status control a gate. - >>> convlstm = ConvLSTMPeephole(4, 3, 3, 3, 1, L1Regularizer(0.5), L1Regularizer(0.5), L1Regularizer(0.5)) + >>> convlstm = ConvLSTMPeephole2D(4, 3, 3, 3, 1, L1Regularizer(0.5), L1Regularizer(0.5), L1Regularizer(0.5)) creating: createL1Regularizer creating: createL1Regularizer creating: createL1Regularizer - creating: createConvLSTMPeephole + creating: createConvLSTMPeephole2D ''' def __init__(self, input_size, output_size, kernel_i, kernel_c, stride, wRegularizer=None, uRegularizer=None, bRegularizer=None, with_peephole=True, bigdl_type="float"): - super(ConvLSTMPeephole, self).__init__(None, bigdl_type, input_size, output_size, kernel_i, kernel_c, stride, - wRegularizer, uRegularizer, bRegularizer, with_peephole) + super(ConvLSTMPeephole2D, self).__init__(None, bigdl_type, input_size, output_size, kernel_i, kernel_c, stride, + wRegularizer, uRegularizer, bRegularizer, with_peephole) + + +class ConvLSTMPeephole3D(Layer): + ''' + + :param input_size: number of input planes in the image given into forward() + :param output_size: number of output planes the convolution layer will produce + :param kernel_i Convolutional filter size to convolve input + :param kernel_c Convolutional filter size to convolve cell + :param stride The step of the convolution + :param wRegularizer: instance of [[Regularizer]](eg. L1 or L2 regularization), applied to the input weights matrices + :param uRegularizer: instance [[Regularizer]](eg. L1 or L2 regularization), applied to the recurrent weights matrices + :param bRegularizer: instance of [[Regularizer]]applied to the bias. + :param with_peephole: whether use last cell status control a gate. + + >>> convlstm = ConvLSTMPeephole3D(4, 3, 3, 3, 1, L1Regularizer(0.5), L1Regularizer(0.5), L1Regularizer(0.5)) + creating: createL1Regularizer + creating: createL1Regularizer + creating: createL1Regularizer + creating: createConvLSTMPeephole3D + ''' + + def __init__(self, input_size, output_size, kernel_i, kernel_c, stride, wRegularizer=None, uRegularizer=None, + bRegularizer=None, with_peephole=True, bigdl_type="float"): + super(ConvLSTMPeephole3D, self).__init__(None, bigdl_type, input_size, output_size, kernel_i, kernel_c, stride, + wRegularizer, uRegularizer, bRegularizer, with_peephole) def _test(): import doctest From da8b914c6e5fd0266e2395f7efece1bd6e339623 Mon Sep 17 00:00:00 2001 From: dding3 Date: Tue, 25 Jul 2017 22:17:34 -0700 Subject: [PATCH 183/823] Convlstm3d (#1361) * support convlstm3d --- python/dllib/src/bigdl/dllib/nn/layer.py | 38 ++++++++++++++++++++---- 1 file changed, 32 insertions(+), 6 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 50a9a6cc010..19ab2061fff 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -3617,7 +3617,7 @@ class Pack(Layer): def __init__(self, dimension, bigdl_type="float"): super(Pack, self).__init__(None, bigdl_type, dimension) -class ConvLSTMPeephole(Layer): +class ConvLSTMPeephole2D(Layer): ''' | Convolution Long Short Term Memory architecture with peephole. @@ -3632,19 +3632,45 @@ class ConvLSTMPeephole(Layer): :param wRegularizer: instance of [[Regularizer]](eg. L1 or L2 regularization), applied to the input weights matrices :param uRegularizer: instance [[Regularizer]](eg. L1 or L2 regularization), applied to the recurrent weights matrices :param bRegularizer: instance of [[Regularizer]]applied to the bias. - :param with_peephold: whether use last cell status control a gate. + :param with_peephole: whether use last cell status control a gate. - >>> convlstm = ConvLSTMPeephole(4, 3, 3, 3, 1, L1Regularizer(0.5), L1Regularizer(0.5), L1Regularizer(0.5)) + >>> convlstm = ConvLSTMPeephole2D(4, 3, 3, 3, 1, L1Regularizer(0.5), L1Regularizer(0.5), L1Regularizer(0.5)) creating: createL1Regularizer creating: createL1Regularizer creating: createL1Regularizer - creating: createConvLSTMPeephole + creating: createConvLSTMPeephole2D ''' def __init__(self, input_size, output_size, kernel_i, kernel_c, stride, wRegularizer=None, uRegularizer=None, bRegularizer=None, with_peephole=True, bigdl_type="float"): - super(ConvLSTMPeephole, self).__init__(None, bigdl_type, input_size, output_size, kernel_i, kernel_c, stride, - wRegularizer, uRegularizer, bRegularizer, with_peephole) + super(ConvLSTMPeephole2D, self).__init__(None, bigdl_type, input_size, output_size, kernel_i, kernel_c, stride, + wRegularizer, uRegularizer, bRegularizer, with_peephole) + + +class ConvLSTMPeephole3D(Layer): + ''' + + :param input_size: number of input planes in the image given into forward() + :param output_size: number of output planes the convolution layer will produce + :param kernel_i Convolutional filter size to convolve input + :param kernel_c Convolutional filter size to convolve cell + :param stride The step of the convolution + :param wRegularizer: instance of [[Regularizer]](eg. L1 or L2 regularization), applied to the input weights matrices + :param uRegularizer: instance [[Regularizer]](eg. L1 or L2 regularization), applied to the recurrent weights matrices + :param bRegularizer: instance of [[Regularizer]]applied to the bias. + :param with_peephole: whether use last cell status control a gate. + + >>> convlstm = ConvLSTMPeephole3D(4, 3, 3, 3, 1, L1Regularizer(0.5), L1Regularizer(0.5), L1Regularizer(0.5)) + creating: createL1Regularizer + creating: createL1Regularizer + creating: createL1Regularizer + creating: createConvLSTMPeephole3D + ''' + + def __init__(self, input_size, output_size, kernel_i, kernel_c, stride, wRegularizer=None, uRegularizer=None, + bRegularizer=None, with_peephole=True, bigdl_type="float"): + super(ConvLSTMPeephole3D, self).__init__(None, bigdl_type, input_size, output_size, kernel_i, kernel_c, stride, + wRegularizer, uRegularizer, bRegularizer, with_peephole) def _test(): import doctest From 68faa409cabde6e70788e57ba9c13c5fce277b83 Mon Sep 17 00:00:00 2001 From: Xianyan Date: Wed, 26 Jul 2017 15:35:22 +0800 Subject: [PATCH 184/823] Support global average pooling (#1365) * add global pooling * update doc and python * update doc --- python/dllib/src/bigdl/dllib/nn/layer.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 19ab2061fff..5eaebcc1577 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -1050,6 +1050,8 @@ class SpatialAveragePooling(Layer): :param dH: step height :param padW: padding width :param padH: padding height + :param global_pooling: If globalPooling then it will pool over the size of the input by doing + kH = input->height and kW = input->width :param ceilMode: whether the output size is to be ceiled or floored :param countIncludePad: whether to include padding when dividing thenumber of elements in pooling region :param divide: whether to do the averaging @@ -1066,6 +1068,7 @@ def __init__(self, dh=1, pad_w=0, pad_h=0, + global_pooling=False, ceil_mode=False, count_include_pad=True, divide=True, @@ -1077,6 +1080,7 @@ def __init__(self, dh, pad_w, pad_h, + global_pooling, ceil_mode, count_include_pad, divide) From 06dc8a5fa67ddad2db56537b3029990dd94e2872 Mon Sep 17 00:00:00 2001 From: Xianyan Date: Wed, 26 Jul 2017 15:35:22 +0800 Subject: [PATCH 185/823] Support global average pooling (#1365) * add global pooling * update doc and python * update doc --- python/dllib/src/bigdl/dllib/nn/layer.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 19ab2061fff..5eaebcc1577 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -1050,6 +1050,8 @@ class SpatialAveragePooling(Layer): :param dH: step height :param padW: padding width :param padH: padding height + :param global_pooling: If globalPooling then it will pool over the size of the input by doing + kH = input->height and kW = input->width :param ceilMode: whether the output size is to be ceiled or floored :param countIncludePad: whether to include padding when dividing thenumber of elements in pooling region :param divide: whether to do the averaging @@ -1066,6 +1068,7 @@ def __init__(self, dh=1, pad_w=0, pad_h=0, + global_pooling=False, ceil_mode=False, count_include_pad=True, divide=True, @@ -1077,6 +1080,7 @@ def __init__(self, dh, pad_w, pad_h, + global_pooling, ceil_mode, count_include_pad, divide) From d292a42b2f337f1019ce78814427be8376d43d4f Mon Sep 17 00:00:00 2001 From: Xianyan Date: Wed, 26 Jul 2017 16:18:23 +0800 Subject: [PATCH 186/823] add SpatialWithinChannelLRN, update the mapping of caffe lrn (#1364) * add SpatialWithinChannelLRN * add python * fix map error * fix python * meet code review * update doc --- python/dllib/src/bigdl/dllib/nn/layer.py | 27 +++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 5eaebcc1577..3ad0501a5ec 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -1158,7 +1158,7 @@ class SpatialCrossMapLRN(Layer): l1 corresponds to max(0,f-ceil(size/2)) and l2 to min(F, f-ceil(size/2) + size). Here, F is the number of feature maps. - :param size: the number of channels to sum over (for cross channel LRN) or the side length ofthe square region to sum over (for within channel LRN) + :param size: the number of channels to sum over :param alpha: the scaling parameter :param beta: the exponent :param k: a constant @@ -3610,6 +3610,31 @@ def __init__(self, JTensor.from_ndarray(kernel)) +class SpatialWithinChannelLRN(Layer): + ''' + The local response normalization layer performs a kind of lateral inhibition + by normalizing over local input regions. the local regions extend spatially, + in separate channels (i.e., they have shape 1 x local_size x local_size). + + :param size the side length of the square region to sum over + :param alpha the scaling parameter + :param beta the exponent + + + >>> layer = SpatialWithinChannelLRN() + creating: createSpatialWithinChannelLRN + ''' + + def __init__(self, + size=5, + alpha=1.0, + beta=0.75, + bigdl_type="float"): + super(SpatialWithinChannelLRN, self).__init__(None, bigdl_type, + size, + alpha, + beta) + class Pack(Layer): ''' Stacks a list of n-dimensional tensors into one (n+1)-dimensional tensor. From bb4d10bb8d17750dc20981ad7ff75e1ecbcbfd90 Mon Sep 17 00:00:00 2001 From: Xianyan Date: Wed, 26 Jul 2017 16:18:23 +0800 Subject: [PATCH 187/823] add SpatialWithinChannelLRN, update the mapping of caffe lrn (#1364) * add SpatialWithinChannelLRN * add python * fix map error * fix python * meet code review * update doc --- python/dllib/src/bigdl/dllib/nn/layer.py | 27 +++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 5eaebcc1577..3ad0501a5ec 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -1158,7 +1158,7 @@ class SpatialCrossMapLRN(Layer): l1 corresponds to max(0,f-ceil(size/2)) and l2 to min(F, f-ceil(size/2) + size). Here, F is the number of feature maps. - :param size: the number of channels to sum over (for cross channel LRN) or the side length ofthe square region to sum over (for within channel LRN) + :param size: the number of channels to sum over :param alpha: the scaling parameter :param beta: the exponent :param k: a constant @@ -3610,6 +3610,31 @@ def __init__(self, JTensor.from_ndarray(kernel)) +class SpatialWithinChannelLRN(Layer): + ''' + The local response normalization layer performs a kind of lateral inhibition + by normalizing over local input regions. the local regions extend spatially, + in separate channels (i.e., they have shape 1 x local_size x local_size). + + :param size the side length of the square region to sum over + :param alpha the scaling parameter + :param beta the exponent + + + >>> layer = SpatialWithinChannelLRN() + creating: createSpatialWithinChannelLRN + ''' + + def __init__(self, + size=5, + alpha=1.0, + beta=0.75, + bigdl_type="float"): + super(SpatialWithinChannelLRN, self).__init__(None, bigdl_type, + size, + alpha, + beta) + class Pack(Layer): ''' Stacks a list of n-dimensional tensors into one (n+1)-dimensional tensor. From 5b59f1d4935284dfc5e613ad92df9157e3bd4c92 Mon Sep 17 00:00:00 2001 From: dding3 Date: Wed, 26 Jul 2017 22:37:11 -0700 Subject: [PATCH 188/823] support get final state and cell (#1369) * support get final state and cell --- python/dllib/src/bigdl/dllib/nn/layer.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 3ad0501a5ec..c68b9564b29 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -863,6 +863,18 @@ class Recurrent(Container): def __init__(self, bigdl_type="float"): super(Recurrent, self).__init__(None, bigdl_type) + def get_finalState_cellStatus(self): + """ + get final state and cell status. + + :return: list of final state and cell status + """ + finalState_cellStatus = callBigDlFunc(self.bigdl_type, "getFinalStateAndCellStatus", self.value) + for idx, tensor in enumerate(finalState_cellStatus): + if tensor is not None: + finalState_cellStatus[idx] = tensor.to_ndarray() + + return finalState_cellStatus class LSTM(Layer): ''' From c17f4ed39e4d043f3078ffa54140bcd04e3b4d87 Mon Sep 17 00:00:00 2001 From: dding3 Date: Wed, 26 Jul 2017 22:37:11 -0700 Subject: [PATCH 189/823] support get final state and cell (#1369) * support get final state and cell --- python/dllib/src/bigdl/dllib/nn/layer.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 3ad0501a5ec..c68b9564b29 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -863,6 +863,18 @@ class Recurrent(Container): def __init__(self, bigdl_type="float"): super(Recurrent, self).__init__(None, bigdl_type) + def get_finalState_cellStatus(self): + """ + get final state and cell status. + + :return: list of final state and cell status + """ + finalState_cellStatus = callBigDlFunc(self.bigdl_type, "getFinalStateAndCellStatus", self.value) + for idx, tensor in enumerate(finalState_cellStatus): + if tensor is not None: + finalState_cellStatus[idx] = tensor.to_ndarray() + + return finalState_cellStatus class LSTM(Layer): ''' From 7d9ff54d422fc379a2c946ce4e871aaf6f9d5398 Mon Sep 17 00:00:00 2001 From: Xianyan Date: Thu, 27 Jul 2017 16:45:20 +0800 Subject: [PATCH 190/823] add regulaizer to volumetricconvolution (#1371) * add regulaizer to volumetricconvolution * fix name * update doc * update doc --- python/dllib/src/bigdl/dllib/nn/layer.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index c68b9564b29..f7532133782 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -3048,7 +3048,8 @@ class VolumetricConvolution(Layer): :param pad_w: The additional zeros added per width to the input planes. :param pad_h: The additional zeros added per height to the input planes. :param with_bias: whether with bias - :param init_method: Init method, Default, Xavier, Bilinear. + :param wRegularizer: instance of [[Regularizer]] (eg. L1 or L2 regularization), applied to the input weights matrices. + :param bRegularizer: instance of [[Regularizer]] applied to the bias. >>> volumetricConvolution = VolumetricConvolution(6, 12, 5, 5, 5, 1, 1, 1) @@ -3068,6 +3069,8 @@ def __init__(self, pad_w=0, pad_h=0, with_bias=True, + wRegularizer=None, + bRegularizer=None, bigdl_type="float"): super(VolumetricConvolution, self).__init__(None, bigdl_type, n_input_plane, @@ -3081,7 +3084,9 @@ def __init__(self, pad_t, pad_w, pad_h, - with_bias) + with_bias, + wRegularizer, + bRegularizer) def set_init_method(self, weight_init_method = None, bias_init_method = None): callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, From 05e265a8b956a41b1976ca96be0945ab695b36f9 Mon Sep 17 00:00:00 2001 From: Xianyan Date: Thu, 27 Jul 2017 16:45:20 +0800 Subject: [PATCH 191/823] add regulaizer to volumetricconvolution (#1371) * add regulaizer to volumetricconvolution * fix name * update doc * update doc --- python/dllib/src/bigdl/dllib/nn/layer.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index c68b9564b29..f7532133782 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -3048,7 +3048,8 @@ class VolumetricConvolution(Layer): :param pad_w: The additional zeros added per width to the input planes. :param pad_h: The additional zeros added per height to the input planes. :param with_bias: whether with bias - :param init_method: Init method, Default, Xavier, Bilinear. + :param wRegularizer: instance of [[Regularizer]] (eg. L1 or L2 regularization), applied to the input weights matrices. + :param bRegularizer: instance of [[Regularizer]] applied to the bias. >>> volumetricConvolution = VolumetricConvolution(6, 12, 5, 5, 5, 1, 1, 1) @@ -3068,6 +3069,8 @@ def __init__(self, pad_w=0, pad_h=0, with_bias=True, + wRegularizer=None, + bRegularizer=None, bigdl_type="float"): super(VolumetricConvolution, self).__init__(None, bigdl_type, n_input_plane, @@ -3081,7 +3084,9 @@ def __init__(self, pad_t, pad_w, pad_h, - with_bias) + with_bias, + wRegularizer, + bRegularizer) def set_init_method(self, weight_init_method = None, bias_init_method = None): callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, From 7e722731fed19a270d95cff8ab2fd4293b4f1d05 Mon Sep 17 00:00:00 2001 From: dding3 Date: Thu, 27 Jul 2017 23:10:48 -0700 Subject: [PATCH 192/823] add regularizer for convlstmpeephole3d (#1395) * add regularizer for convlstmpeephole3d * rename convlstmpeephole2d back to convlstmpeephole --- python/dllib/src/bigdl/dllib/nn/layer.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index f7532133782..a7e44a66dfa 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -3663,7 +3663,7 @@ class Pack(Layer): def __init__(self, dimension, bigdl_type="float"): super(Pack, self).__init__(None, bigdl_type, dimension) -class ConvLSTMPeephole2D(Layer): +class ConvLSTMPeephole(Layer): ''' | Convolution Long Short Term Memory architecture with peephole. @@ -3678,19 +3678,21 @@ class ConvLSTMPeephole2D(Layer): :param wRegularizer: instance of [[Regularizer]](eg. L1 or L2 regularization), applied to the input weights matrices :param uRegularizer: instance [[Regularizer]](eg. L1 or L2 regularization), applied to the recurrent weights matrices :param bRegularizer: instance of [[Regularizer]]applied to the bias. + :param cRegularizer: instance of [[Regularizer]]applied to peephole. :param with_peephole: whether use last cell status control a gate. - >>> convlstm = ConvLSTMPeephole2D(4, 3, 3, 3, 1, L1Regularizer(0.5), L1Regularizer(0.5), L1Regularizer(0.5)) + >>> convlstm = ConvLSTMPeephole(4, 3, 3, 3, 1, L1Regularizer(0.5), L1Regularizer(0.5), L1Regularizer(0.5), L1Regularizer(0.5)) creating: createL1Regularizer creating: createL1Regularizer creating: createL1Regularizer - creating: createConvLSTMPeephole2D + creating: createL1Regularizer + creating: createConvLSTMPeephole ''' def __init__(self, input_size, output_size, kernel_i, kernel_c, stride, wRegularizer=None, uRegularizer=None, - bRegularizer=None, with_peephole=True, bigdl_type="float"): - super(ConvLSTMPeephole2D, self).__init__(None, bigdl_type, input_size, output_size, kernel_i, kernel_c, stride, - wRegularizer, uRegularizer, bRegularizer, with_peephole) + bRegularizer=None, cRegularizer=None, with_peephole=True, bigdl_type="float"): + super(ConvLSTMPeephole, self).__init__(None, bigdl_type, input_size, output_size, kernel_i, kernel_c, stride, + wRegularizer, uRegularizer, bRegularizer, cRegularizer, with_peephole) class ConvLSTMPeephole3D(Layer): @@ -3704,9 +3706,11 @@ class ConvLSTMPeephole3D(Layer): :param wRegularizer: instance of [[Regularizer]](eg. L1 or L2 regularization), applied to the input weights matrices :param uRegularizer: instance [[Regularizer]](eg. L1 or L2 regularization), applied to the recurrent weights matrices :param bRegularizer: instance of [[Regularizer]]applied to the bias. + :param cRegularizer: instance of [[Regularizer]]applied to peephole. :param with_peephole: whether use last cell status control a gate. - >>> convlstm = ConvLSTMPeephole3D(4, 3, 3, 3, 1, L1Regularizer(0.5), L1Regularizer(0.5), L1Regularizer(0.5)) + >>> convlstm = ConvLSTMPeephole3D(4, 3, 3, 3, 1, L1Regularizer(0.5), L1Regularizer(0.5), L1Regularizer(0.5), L1Regularizer(0.5)) + creating: createL1Regularizer creating: createL1Regularizer creating: createL1Regularizer creating: createL1Regularizer @@ -3714,9 +3718,9 @@ class ConvLSTMPeephole3D(Layer): ''' def __init__(self, input_size, output_size, kernel_i, kernel_c, stride, wRegularizer=None, uRegularizer=None, - bRegularizer=None, with_peephole=True, bigdl_type="float"): + bRegularizer=None, cRegularizer=None, with_peephole=True, bigdl_type="float"): super(ConvLSTMPeephole3D, self).__init__(None, bigdl_type, input_size, output_size, kernel_i, kernel_c, stride, - wRegularizer, uRegularizer, bRegularizer, with_peephole) + wRegularizer, uRegularizer, bRegularizer, cRegularizer, with_peephole) def _test(): import doctest From 67f7d37c51f10b8311233943dcbebad63097d1a1 Mon Sep 17 00:00:00 2001 From: dding3 Date: Thu, 27 Jul 2017 23:10:48 -0700 Subject: [PATCH 193/823] add regularizer for convlstmpeephole3d (#1395) * add regularizer for convlstmpeephole3d * rename convlstmpeephole2d back to convlstmpeephole --- python/dllib/src/bigdl/dllib/nn/layer.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index f7532133782..a7e44a66dfa 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -3663,7 +3663,7 @@ class Pack(Layer): def __init__(self, dimension, bigdl_type="float"): super(Pack, self).__init__(None, bigdl_type, dimension) -class ConvLSTMPeephole2D(Layer): +class ConvLSTMPeephole(Layer): ''' | Convolution Long Short Term Memory architecture with peephole. @@ -3678,19 +3678,21 @@ class ConvLSTMPeephole2D(Layer): :param wRegularizer: instance of [[Regularizer]](eg. L1 or L2 regularization), applied to the input weights matrices :param uRegularizer: instance [[Regularizer]](eg. L1 or L2 regularization), applied to the recurrent weights matrices :param bRegularizer: instance of [[Regularizer]]applied to the bias. + :param cRegularizer: instance of [[Regularizer]]applied to peephole. :param with_peephole: whether use last cell status control a gate. - >>> convlstm = ConvLSTMPeephole2D(4, 3, 3, 3, 1, L1Regularizer(0.5), L1Regularizer(0.5), L1Regularizer(0.5)) + >>> convlstm = ConvLSTMPeephole(4, 3, 3, 3, 1, L1Regularizer(0.5), L1Regularizer(0.5), L1Regularizer(0.5), L1Regularizer(0.5)) creating: createL1Regularizer creating: createL1Regularizer creating: createL1Regularizer - creating: createConvLSTMPeephole2D + creating: createL1Regularizer + creating: createConvLSTMPeephole ''' def __init__(self, input_size, output_size, kernel_i, kernel_c, stride, wRegularizer=None, uRegularizer=None, - bRegularizer=None, with_peephole=True, bigdl_type="float"): - super(ConvLSTMPeephole2D, self).__init__(None, bigdl_type, input_size, output_size, kernel_i, kernel_c, stride, - wRegularizer, uRegularizer, bRegularizer, with_peephole) + bRegularizer=None, cRegularizer=None, with_peephole=True, bigdl_type="float"): + super(ConvLSTMPeephole, self).__init__(None, bigdl_type, input_size, output_size, kernel_i, kernel_c, stride, + wRegularizer, uRegularizer, bRegularizer, cRegularizer, with_peephole) class ConvLSTMPeephole3D(Layer): @@ -3704,9 +3706,11 @@ class ConvLSTMPeephole3D(Layer): :param wRegularizer: instance of [[Regularizer]](eg. L1 or L2 regularization), applied to the input weights matrices :param uRegularizer: instance [[Regularizer]](eg. L1 or L2 regularization), applied to the recurrent weights matrices :param bRegularizer: instance of [[Regularizer]]applied to the bias. + :param cRegularizer: instance of [[Regularizer]]applied to peephole. :param with_peephole: whether use last cell status control a gate. - >>> convlstm = ConvLSTMPeephole3D(4, 3, 3, 3, 1, L1Regularizer(0.5), L1Regularizer(0.5), L1Regularizer(0.5)) + >>> convlstm = ConvLSTMPeephole3D(4, 3, 3, 3, 1, L1Regularizer(0.5), L1Regularizer(0.5), L1Regularizer(0.5), L1Regularizer(0.5)) + creating: createL1Regularizer creating: createL1Regularizer creating: createL1Regularizer creating: createL1Regularizer @@ -3714,9 +3718,9 @@ class ConvLSTMPeephole3D(Layer): ''' def __init__(self, input_size, output_size, kernel_i, kernel_c, stride, wRegularizer=None, uRegularizer=None, - bRegularizer=None, with_peephole=True, bigdl_type="float"): + bRegularizer=None, cRegularizer=None, with_peephole=True, bigdl_type="float"): super(ConvLSTMPeephole3D, self).__init__(None, bigdl_type, input_size, output_size, kernel_i, kernel_c, stride, - wRegularizer, uRegularizer, bRegularizer, with_peephole) + wRegularizer, uRegularizer, bRegularizer, cRegularizer, with_peephole) def _test(): import doctest From cb6542097ae330c24f98798574eeb46e07986e24 Mon Sep 17 00:00:00 2001 From: zhangxiaoli73 <380761639@qq.com> Date: Wed, 2 Aug 2017 13:42:05 +0800 Subject: [PATCH 194/823] add load optimmethod for python (#1387) * add load optimmethod for python * update md --- python/dllib/src/bigdl/dllib/nn/layer.py | 9 ++- .../dllib/src/bigdl/dllib/optim/optimizer.py | 59 ++++++++++++++----- 2 files changed, 51 insertions(+), 17 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index a7e44a66dfa..db0f800e97c 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -27,6 +27,7 @@ from bigdl.util.common import to_list from bigdl.util.common import INTMAX, INTMIN, DOUBLEMAX from bigdl.optim.optimizer import L1Regularizer, L2Regularizer, L1L2Regularizer +from py4j.java_gateway import JavaObject if sys.version >= '3': long = int @@ -57,8 +58,12 @@ class Layer(JavaValue): """ def __init__(self, jvalue, bigdl_type, *args): - self.value = jvalue if jvalue else callBigDlFunc( - bigdl_type, JavaValue.jvm_class_constructor(self), *args) + if (jvalue): + assert(type(jvalue) == JavaObject) + self.value = jvalue + else: + self.value = callBigDlFunc( + bigdl_type, JavaValue.jvm_class_constructor(self), *args) self.bigdl_type = bigdl_type def __str__(self): diff --git a/python/dllib/src/bigdl/dllib/optim/optimizer.py b/python/dllib/src/bigdl/dllib/optim/optimizer.py index b8f66179796..93f6f0aa0e7 100644 --- a/python/dllib/src/bigdl/dllib/optim/optimizer.py +++ b/python/dllib/src/bigdl/dllib/optim/optimizer.py @@ -26,6 +26,7 @@ from bigdl.util.common import callJavaFunc from bigdl.util.common import get_spark_context from bigdl.util.common import to_list +from py4j.java_gateway import JavaObject if sys.version >= '3': @@ -313,7 +314,35 @@ def __init__(self, JavaValue.__init__(self, None, bigdl_type, monitor, factor, patience, mode, epsilon, cooldown, min_lr) -class SGD(JavaValue): +class OptimMethod(JavaValue): + + def __init__(self, jvalue, bigdl_type, *args): + if (jvalue): + assert(type(jvalue) == JavaObject) + self.value = jvalue + else: + self.value = callBigDlFunc( + bigdl_type, JavaValue.jvm_class_constructor(self), *args) + self.bigdl_type = bigdl_type + + @staticmethod + def load(path, bigdl_type="float"): + """ + load optim method + :param path: file path + """ + return callBigDlFunc(bigdl_type, "loadOptimMethod", path) + + def save(self, path, overWrite): + """ + save OptimMethod + :param path path + :param overWrite whether to overwrite + """ + method=self.value + return callBigDlFunc(self.bigdl_type, "saveOptimMethod", method, path, overWrite) + +class SGD(OptimMethod): """ A plain implementation of SGD @@ -340,12 +369,12 @@ def __init__(self, learningrates=None, weightdecays=None, bigdl_type="float"): - JavaValue.__init__(self, None, bigdl_type, learningrate, learningrate_decay, weightdecay, + super(SGD, self).__init__(None, bigdl_type, learningrate, learningrate_decay, weightdecay, momentum, dampening, nesterov, leaningrate_schedule if (leaningrate_schedule) else Default(), JTensor.from_ndarray(learningrates), JTensor.from_ndarray(weightdecays)) -class Adagrad(JavaValue): +class Adagrad(OptimMethod): """ An implementation of Adagrad. See the original paper: http://jmlr.org/papers/volume12/duchi11a/duchi11a.pdf @@ -361,9 +390,9 @@ def __init__(self, learningrate_decay=0.0, weightdecay=0.0, bigdl_type="float"): - JavaValue.__init__(self, None, bigdl_type, learningrate, learningrate_decay, weightdecay) + super(Adagrad, self).__init__(None, bigdl_type, learningrate, learningrate_decay, weightdecay) -class LBFGS(JavaValue): +class LBFGS(OptimMethod): """ This implementation of L-BFGS relies on a user-provided line search function (state.lineSearch). If this function is not @@ -400,10 +429,10 @@ def __init__(self, bigdl_type="float"): if linesearch or linesearch_options: raise ValueError('linesearch and linesearch_options must be None in LBFGS') - JavaValue.__init__(self, None, bigdl_type, max_iter, max_eval, tolfun, tolx, - ncorrection, learningrate, verbose, linesearch, linesearch_options) + super(LBFGS, self).__init__(None, bigdl_type, max_iter, max_eval, tolfun, tolx, + ncorrection, learningrate, verbose, linesearch, linesearch_options) -class Adadelta(JavaValue): +class Adadelta(OptimMethod): """ Adadelta implementation for SGD: http://arxiv.org/abs/1212.5701 @@ -416,9 +445,9 @@ def __init__(self, decayrate = 0.9, epsilon = 1e-10, bigdl_type="float"): - JavaValue.__init__(self, None, bigdl_type, decayrate, epsilon) + super(Adadelta, self).__init__(None, bigdl_type, decayrate, epsilon) -class Adam(JavaValue): +class Adam(OptimMethod): """ An implementation of Adam http://arxiv.org/pdf/1412.6980.pdf :param learningrate learning rate @@ -436,10 +465,10 @@ def __init__(self, beta2 = 0.999, epsilon = 1e-8, bigdl_type="float"): - JavaValue.__init__(self, None, bigdl_type, learningrate, learningrate_decay, + super(Adam, self).__init__(None, bigdl_type, learningrate, learningrate_decay, beta1, beta2, epsilon) -class Adamax(JavaValue): +class Adamax(OptimMethod): """ An implementation of Adamax http://arxiv.org/pdf/1412.6980.pdf :param learningrate learning rate @@ -455,9 +484,9 @@ def __init__(self, beta2 = 0.999, epsilon = 1e-38, bigdl_type="float"): - JavaValue.__init__(self, None, bigdl_type, learningrate, beta1, beta2, epsilon) + super(Adamax, self).__init__(None, bigdl_type, learningrate, beta1, beta2, epsilon) -class RMSprop(JavaValue): +class RMSprop(OptimMethod): """ An implementation of RMSprop :param learningrate learning rate @@ -473,7 +502,7 @@ def __init__(self, decayrate = 0.99, epsilon = 1e-8, bigdl_type="float"): - JavaValue.__init__(self, None, bigdl_type, learningrate, learningrate_decay, decayrate, epsilon) + super(RMSprop, self).__init__(None, bigdl_type, learningrate, learningrate_decay, decayrate, epsilon) class MultiStep(JavaValue): """ From 5400f6cc5a7739f5d2ee6987924af8a6797d97e3 Mon Sep 17 00:00:00 2001 From: zhangxiaoli73 <380761639@qq.com> Date: Wed, 2 Aug 2017 13:42:05 +0800 Subject: [PATCH 195/823] add load optimmethod for python (#1387) * add load optimmethod for python * update md --- python/dllib/src/bigdl/dllib/nn/layer.py | 9 ++- .../dllib/src/bigdl/dllib/optim/optimizer.py | 59 ++++++++++++++----- 2 files changed, 51 insertions(+), 17 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index a7e44a66dfa..db0f800e97c 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -27,6 +27,7 @@ from bigdl.util.common import to_list from bigdl.util.common import INTMAX, INTMIN, DOUBLEMAX from bigdl.optim.optimizer import L1Regularizer, L2Regularizer, L1L2Regularizer +from py4j.java_gateway import JavaObject if sys.version >= '3': long = int @@ -57,8 +58,12 @@ class Layer(JavaValue): """ def __init__(self, jvalue, bigdl_type, *args): - self.value = jvalue if jvalue else callBigDlFunc( - bigdl_type, JavaValue.jvm_class_constructor(self), *args) + if (jvalue): + assert(type(jvalue) == JavaObject) + self.value = jvalue + else: + self.value = callBigDlFunc( + bigdl_type, JavaValue.jvm_class_constructor(self), *args) self.bigdl_type = bigdl_type def __str__(self): diff --git a/python/dllib/src/bigdl/dllib/optim/optimizer.py b/python/dllib/src/bigdl/dllib/optim/optimizer.py index b8f66179796..93f6f0aa0e7 100644 --- a/python/dllib/src/bigdl/dllib/optim/optimizer.py +++ b/python/dllib/src/bigdl/dllib/optim/optimizer.py @@ -26,6 +26,7 @@ from bigdl.util.common import callJavaFunc from bigdl.util.common import get_spark_context from bigdl.util.common import to_list +from py4j.java_gateway import JavaObject if sys.version >= '3': @@ -313,7 +314,35 @@ def __init__(self, JavaValue.__init__(self, None, bigdl_type, monitor, factor, patience, mode, epsilon, cooldown, min_lr) -class SGD(JavaValue): +class OptimMethod(JavaValue): + + def __init__(self, jvalue, bigdl_type, *args): + if (jvalue): + assert(type(jvalue) == JavaObject) + self.value = jvalue + else: + self.value = callBigDlFunc( + bigdl_type, JavaValue.jvm_class_constructor(self), *args) + self.bigdl_type = bigdl_type + + @staticmethod + def load(path, bigdl_type="float"): + """ + load optim method + :param path: file path + """ + return callBigDlFunc(bigdl_type, "loadOptimMethod", path) + + def save(self, path, overWrite): + """ + save OptimMethod + :param path path + :param overWrite whether to overwrite + """ + method=self.value + return callBigDlFunc(self.bigdl_type, "saveOptimMethod", method, path, overWrite) + +class SGD(OptimMethod): """ A plain implementation of SGD @@ -340,12 +369,12 @@ def __init__(self, learningrates=None, weightdecays=None, bigdl_type="float"): - JavaValue.__init__(self, None, bigdl_type, learningrate, learningrate_decay, weightdecay, + super(SGD, self).__init__(None, bigdl_type, learningrate, learningrate_decay, weightdecay, momentum, dampening, nesterov, leaningrate_schedule if (leaningrate_schedule) else Default(), JTensor.from_ndarray(learningrates), JTensor.from_ndarray(weightdecays)) -class Adagrad(JavaValue): +class Adagrad(OptimMethod): """ An implementation of Adagrad. See the original paper: http://jmlr.org/papers/volume12/duchi11a/duchi11a.pdf @@ -361,9 +390,9 @@ def __init__(self, learningrate_decay=0.0, weightdecay=0.0, bigdl_type="float"): - JavaValue.__init__(self, None, bigdl_type, learningrate, learningrate_decay, weightdecay) + super(Adagrad, self).__init__(None, bigdl_type, learningrate, learningrate_decay, weightdecay) -class LBFGS(JavaValue): +class LBFGS(OptimMethod): """ This implementation of L-BFGS relies on a user-provided line search function (state.lineSearch). If this function is not @@ -400,10 +429,10 @@ def __init__(self, bigdl_type="float"): if linesearch or linesearch_options: raise ValueError('linesearch and linesearch_options must be None in LBFGS') - JavaValue.__init__(self, None, bigdl_type, max_iter, max_eval, tolfun, tolx, - ncorrection, learningrate, verbose, linesearch, linesearch_options) + super(LBFGS, self).__init__(None, bigdl_type, max_iter, max_eval, tolfun, tolx, + ncorrection, learningrate, verbose, linesearch, linesearch_options) -class Adadelta(JavaValue): +class Adadelta(OptimMethod): """ Adadelta implementation for SGD: http://arxiv.org/abs/1212.5701 @@ -416,9 +445,9 @@ def __init__(self, decayrate = 0.9, epsilon = 1e-10, bigdl_type="float"): - JavaValue.__init__(self, None, bigdl_type, decayrate, epsilon) + super(Adadelta, self).__init__(None, bigdl_type, decayrate, epsilon) -class Adam(JavaValue): +class Adam(OptimMethod): """ An implementation of Adam http://arxiv.org/pdf/1412.6980.pdf :param learningrate learning rate @@ -436,10 +465,10 @@ def __init__(self, beta2 = 0.999, epsilon = 1e-8, bigdl_type="float"): - JavaValue.__init__(self, None, bigdl_type, learningrate, learningrate_decay, + super(Adam, self).__init__(None, bigdl_type, learningrate, learningrate_decay, beta1, beta2, epsilon) -class Adamax(JavaValue): +class Adamax(OptimMethod): """ An implementation of Adamax http://arxiv.org/pdf/1412.6980.pdf :param learningrate learning rate @@ -455,9 +484,9 @@ def __init__(self, beta2 = 0.999, epsilon = 1e-38, bigdl_type="float"): - JavaValue.__init__(self, None, bigdl_type, learningrate, beta1, beta2, epsilon) + super(Adamax, self).__init__(None, bigdl_type, learningrate, beta1, beta2, epsilon) -class RMSprop(JavaValue): +class RMSprop(OptimMethod): """ An implementation of RMSprop :param learningrate learning rate @@ -473,7 +502,7 @@ def __init__(self, decayrate = 0.99, epsilon = 1e-8, bigdl_type="float"): - JavaValue.__init__(self, None, bigdl_type, learningrate, learningrate_decay, decayrate, epsilon) + super(RMSprop, self).__init__(None, bigdl_type, learningrate, learningrate_decay, decayrate, epsilon) class MultiStep(JavaValue): """ From 389ae9e339b8c6fbed5c6099bbb064201161e6b4 Mon Sep 17 00:00:00 2001 From: zhangxiaoli73 <380761639@qq.com> Date: Wed, 2 Aug 2017 13:42:05 +0800 Subject: [PATCH 196/823] add load optimmethod for python (#1387) * add load optimmethod for python * update md --- simple_integration_test.py | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/simple_integration_test.py b/simple_integration_test.py index 72392ec14a3..8fdf52ee3a4 100644 --- a/simple_integration_test.py +++ b/simple_integration_test.py @@ -72,6 +72,41 @@ def test_load_model(self): self.assertTrue(np.allclose(fc1_loaded.get_weights()[0], fc1.get_weights()[0])) + def test_load_optim_method(self): + FEATURES_DIM = 2 + data_len = 100 + batch_size = 32 + epoch_num = 5 + + def gen_rand_sample(): + features = np.random.uniform(0, 1, (FEATURES_DIM)) + label = (2 * features).sum() + 0.4 + return Sample.from_ndarray(features, label) + trainingData = self.sc.parallelize(range(0, data_len)).map(lambda i: gen_rand_sample()) + model = Sequential() + l1 = Linear(FEATURES_DIM, 1).set_init_method(Xavier(), Zeros()).set_name("linear1") + model.add(l1) + + sgd = SGD(learningrate=0.01, learningrate_decay=0.0002, weightdecay=0.0, + momentum=0.0, dampening=0.0, nesterov=False, + leaningrate_schedule=Poly(0.5, int((data_len/batch_size)*epoch_num))) + + tmp_path = tempfile.mktemp() + sgd.save(tmp_path, True) + optim_method = OptimMethod.load(tmp_path) + self.assertTrue(optim_method.learningRate() == sgd.value.learningRate()) + self.assertTrue(optim_method.momentum() == sgd.value.momentum()) + self.assertTrue(optim_method.nesterov() == sgd.value.nesterov()) + + optimizer = Optimizer( + model=model, + training_rdd=trainingData, + criterion=MSECriterion(), + optim_method=optim_method, + end_trigger=MaxEpoch(epoch_num), + batch_size=batch_size) + optimizer.optimize() + def test_create_node(self): import numpy as np fc1 = Linear(4, 2)() From 68ca8d2a0c408e497840be7e621e6af76ac2d88c Mon Sep 17 00:00:00 2001 From: Yao Zhang Date: Thu, 3 Aug 2017 11:04:21 +0800 Subject: [PATCH 197/823] Add 3D de-convolution (#1401) * finish volumetric full convolution * add documentation * reset core * meet code review * add serial version uid * meet code review --- python/dllib/src/bigdl/dllib/nn/layer.py | 92 ++++++++++++++++++++++++ 1 file changed, 92 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index db0f800e97c..519f23f76d3 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -2974,6 +2974,98 @@ def set_init_method(self, weight_init_method = None, bias_init_method = None): weight_init_method, bias_init_method) return self +class VolumetricFullConvolution(Layer): + ''' + Apply a 3D full convolution over an 3D input image, a sequence of images, or a video etc. + The input tensor is expected to be a 4D or 5D(with batch) tensor. Note that instead + of setting adjT, adjW and adjH, `VolumetricFullConvolution` also accepts a table input + with two tensors: T(convInput, sizeTensor) where convInput is the standard input tensor, + and the size of sizeTensor is used to set the size of the output (will ignore the adjT, adjW and + adjH values used to construct the module). This module can be used without a bias by setting + parameter noBias = true while constructing the module. + + + If input is a 4D tensor nInputPlane x depth x height x width, + odepth = (depth - 1) * dT - 2*padt + kT + adjT + owidth = (width - 1) * dW - 2*padW + kW + adjW + oheight = (height - 1) * dH - 2*padH + kH + adjH + + + Other frameworks call this operation "In-network Upsampling", "Fractionally-strided convolution", + "Backwards Convolution," "Deconvolution", or "Upconvolution." + + + Reference Paper: Long J, Shelhamer E, Darrell T. Fully convolutional networks for semantic + segmentation[C]//Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition. + 2015: 3431-3440. + + :param nInputPlane The number of expected input planes in the image given into forward() + :param nOutputPlane The number of output planes the convolution layer will produce. + :param kT The kernel depth of the convolution. + :param kW The kernel width of the convolution. + :param kH The kernel height of the convolution. + :param dT The step of the convolution in the depth dimension. Default is 1. + :param dW The step of the convolution in the width dimension. Default is 1. + :param dH The step of the convolution in the height dimension. Default is 1. + :param padT The additional zeros added per depth to the input planes. Default is 0. + :param padW The additional zeros added per width to the input planes. Default is 0. + :param padH The additional zeros added per height to the input planes. Default is 0. + :param adjT Extra depth to add to the output image. Default is 0. + :param adjW Extra width to add to the output image. Default is 0. + :param adjH Extra height to add to the output image. Default is 0. + :param nGroup Kernel group number. + :param noBias If bias is needed. + :param wRegularizer: instance of [[Regularizer]](eg. L1 or L2 regularization), applied to the input weights matrices. + :param bRegularizer: instance of [[Regularizer]]applied to the bias. + + + >>> volumetricFullConvolution = VolumetricFullConvolution(1, 1, 1, 1, 1, 1) + creating: createVolumetricFullConvolution + ''' + + def __init__(self, + n_input_plane, + n_output_plane, + kt, + kw, + kh, + dt=1, + dw=1, + dh=1, + pad_t=0, + pad_w=0, + pad_h=0, + adj_t=0, + adj_w=0, + adj_h=0, + n_group=1, + no_bias=False, + wRegularizer=None, + bRegularizer=None, + bigdl_type="float"): + super(VolumetricFullConvolution, self).__init__(None, bigdl_type, + n_input_plane, + n_output_plane, + kt, + kw, + kh, + dt, + dw, + dh, + pad_t, + pad_w, + pad_h, + adj_t, + adj_w, + adj_h, + n_group, + no_bias, + wRegularizer, + bRegularizer) + def set_init_method(self, weight_init_method = None, bias_init_method = None): + callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, + weight_init_method, bias_init_method) + return self class SpatialShareConvolution(Layer): From aec5d23e65731b745c05f624e08fd3ef0c3dd8ac Mon Sep 17 00:00:00 2001 From: Yao Zhang Date: Thu, 3 Aug 2017 11:04:21 +0800 Subject: [PATCH 198/823] Add 3D de-convolution (#1401) * finish volumetric full convolution * add documentation * reset core * meet code review * add serial version uid * meet code review --- python/dllib/src/bigdl/dllib/nn/layer.py | 92 ++++++++++++++++++++++++ 1 file changed, 92 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index db0f800e97c..519f23f76d3 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -2974,6 +2974,98 @@ def set_init_method(self, weight_init_method = None, bias_init_method = None): weight_init_method, bias_init_method) return self +class VolumetricFullConvolution(Layer): + ''' + Apply a 3D full convolution over an 3D input image, a sequence of images, or a video etc. + The input tensor is expected to be a 4D or 5D(with batch) tensor. Note that instead + of setting adjT, adjW and adjH, `VolumetricFullConvolution` also accepts a table input + with two tensors: T(convInput, sizeTensor) where convInput is the standard input tensor, + and the size of sizeTensor is used to set the size of the output (will ignore the adjT, adjW and + adjH values used to construct the module). This module can be used without a bias by setting + parameter noBias = true while constructing the module. + + + If input is a 4D tensor nInputPlane x depth x height x width, + odepth = (depth - 1) * dT - 2*padt + kT + adjT + owidth = (width - 1) * dW - 2*padW + kW + adjW + oheight = (height - 1) * dH - 2*padH + kH + adjH + + + Other frameworks call this operation "In-network Upsampling", "Fractionally-strided convolution", + "Backwards Convolution," "Deconvolution", or "Upconvolution." + + + Reference Paper: Long J, Shelhamer E, Darrell T. Fully convolutional networks for semantic + segmentation[C]//Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition. + 2015: 3431-3440. + + :param nInputPlane The number of expected input planes in the image given into forward() + :param nOutputPlane The number of output planes the convolution layer will produce. + :param kT The kernel depth of the convolution. + :param kW The kernel width of the convolution. + :param kH The kernel height of the convolution. + :param dT The step of the convolution in the depth dimension. Default is 1. + :param dW The step of the convolution in the width dimension. Default is 1. + :param dH The step of the convolution in the height dimension. Default is 1. + :param padT The additional zeros added per depth to the input planes. Default is 0. + :param padW The additional zeros added per width to the input planes. Default is 0. + :param padH The additional zeros added per height to the input planes. Default is 0. + :param adjT Extra depth to add to the output image. Default is 0. + :param adjW Extra width to add to the output image. Default is 0. + :param adjH Extra height to add to the output image. Default is 0. + :param nGroup Kernel group number. + :param noBias If bias is needed. + :param wRegularizer: instance of [[Regularizer]](eg. L1 or L2 regularization), applied to the input weights matrices. + :param bRegularizer: instance of [[Regularizer]]applied to the bias. + + + >>> volumetricFullConvolution = VolumetricFullConvolution(1, 1, 1, 1, 1, 1) + creating: createVolumetricFullConvolution + ''' + + def __init__(self, + n_input_plane, + n_output_plane, + kt, + kw, + kh, + dt=1, + dw=1, + dh=1, + pad_t=0, + pad_w=0, + pad_h=0, + adj_t=0, + adj_w=0, + adj_h=0, + n_group=1, + no_bias=False, + wRegularizer=None, + bRegularizer=None, + bigdl_type="float"): + super(VolumetricFullConvolution, self).__init__(None, bigdl_type, + n_input_plane, + n_output_plane, + kt, + kw, + kh, + dt, + dw, + dh, + pad_t, + pad_w, + pad_h, + adj_t, + adj_w, + adj_h, + n_group, + no_bias, + wRegularizer, + bRegularizer) + def set_init_method(self, weight_init_method = None, bias_init_method = None): + callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, + weight_init_method, bias_init_method) + return self class SpatialShareConvolution(Layer): From ff00358067e48acde6cd59d813d19202a5c2b4a1 Mon Sep 17 00:00:00 2001 From: dding3 Date: Thu, 10 Aug 2017 20:34:53 -0700 Subject: [PATCH 199/823] Set hiddenstate (#1439) * support set/get state in recurrent --- python/dllib/src/bigdl/dllib/nn/layer.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 519f23f76d3..23c2a9e563f 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -868,18 +868,24 @@ class Recurrent(Container): def __init__(self, bigdl_type="float"): super(Recurrent, self).__init__(None, bigdl_type) - def get_finalState_cellStatus(self): + def get_state(self): """ - get final state and cell status. + get hidden state and cell at last time step. - :return: list of final state and cell status + :return: list of hidden state and cell """ - finalState_cellStatus = callBigDlFunc(self.bigdl_type, "getFinalStateAndCellStatus", self.value) - for idx, tensor in enumerate(finalState_cellStatus): - if tensor is not None: - finalState_cellStatus[idx] = tensor.to_ndarray() + state = callBigDlFunc(self.bigdl_type, "getState", self.value) + for idx, tensor in enumerate(state): + state[idx] = tensor.to_ndarray() - return finalState_cellStatus + return state + + def set_state(self, state): + """ + set hidden state and cell at first time step. + """ + jstate, state_is_table = self.check_input(state) + callBigDlFunc(self.bigdl_type, "setState", self.value, jstate, state_is_table) class LSTM(Layer): ''' From 5397467061a8f0061747939043cfa83ab3c9803f Mon Sep 17 00:00:00 2001 From: dding3 Date: Thu, 10 Aug 2017 20:34:53 -0700 Subject: [PATCH 200/823] Set hiddenstate (#1439) * support set/get state in recurrent --- python/dllib/src/bigdl/dllib/nn/layer.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 519f23f76d3..23c2a9e563f 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -868,18 +868,24 @@ class Recurrent(Container): def __init__(self, bigdl_type="float"): super(Recurrent, self).__init__(None, bigdl_type) - def get_finalState_cellStatus(self): + def get_state(self): """ - get final state and cell status. + get hidden state and cell at last time step. - :return: list of final state and cell status + :return: list of hidden state and cell """ - finalState_cellStatus = callBigDlFunc(self.bigdl_type, "getFinalStateAndCellStatus", self.value) - for idx, tensor in enumerate(finalState_cellStatus): - if tensor is not None: - finalState_cellStatus[idx] = tensor.to_ndarray() + state = callBigDlFunc(self.bigdl_type, "getState", self.value) + for idx, tensor in enumerate(state): + state[idx] = tensor.to_ndarray() - return finalState_cellStatus + return state + + def set_state(self, state): + """ + set hidden state and cell at first time step. + """ + jstate, state_is_table = self.check_input(state) + callBigDlFunc(self.bigdl_type, "setState", self.value, jstate, state_is_table) class LSTM(Layer): ''' From 0f47447ec4444f7d5105e981dd7b060430914121 Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Mon, 14 Aug 2017 05:55:26 -0700 Subject: [PATCH 201/823] support nhwc format in SpatialConvolution (#1434) * support nhwc format in SpatialConvolution * add python api and doc * meet code review * fix bug and add tests * meet code review * fix bug --- python/dllib/src/bigdl/dllib/nn/layer.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 23c2a9e563f..931da1bd065 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -736,6 +736,9 @@ class SpatialConvolution(Layer): :param init_grad_weight: the optional initial value for the grad_weight :param init_grad_bias: the optional initial value for the grad_bias :param with_bias: the optional initial value for if need bias + :param data_format: a string value of "NHWC" or "NCHW" to specify the input data format of this layer. In "NHWC" format + data is stored in the order of [batch_size, height, width, channels], in "NCHW" format data is stored + in the order of [batch_size, channels, height, width]. >>> spatialConvolution = SpatialConvolution(6, 12, 5, 5) creating: createSpatialConvolution @@ -748,7 +751,7 @@ class SpatialConvolution(Layer): >>> init_bias = np.random.randn(12) >>> init_grad_weight = np.zeros([1, 12, 6, 5, 5]) >>> init_grad_bias = np.zeros([12]) - >>> spatialConvolution = SpatialConvolution(6, 12, 5, 5, 1, 1, 0, 0, 1, True, L1Regularizer(0.5), L1Regularizer(0.5), init_weight, init_bias, init_grad_weight, init_grad_bias) + >>> spatialConvolution = SpatialConvolution(6, 12, 5, 5, 1, 1, 0, 0, 1, True, L1Regularizer(0.5), L1Regularizer(0.5), init_weight, init_bias, init_grad_weight, init_grad_bias, True, "NCHW") creating: createL1Regularizer creating: createL1Regularizer creating: createSpatialConvolution @@ -772,6 +775,7 @@ def __init__(self, init_grad_weight=None, init_grad_bias=None, with_bias=True, + data_format="NCHW", bigdl_type="float"): super(SpatialConvolution, self).__init__(None, bigdl_type, n_input_plane, @@ -790,7 +794,8 @@ def __init__(self, JTensor.from_ndarray(init_bias), JTensor.from_ndarray(init_grad_weight), JTensor.from_ndarray(init_grad_bias), - with_bias) + with_bias, + data_format) def set_init_method(self, weight_init_method = None, bias_init_method = None): callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, weight_init_method, bias_init_method) From 3e1771d67791e900a9ec44f943fe1cbb57b9e28b Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Mon, 14 Aug 2017 05:55:26 -0700 Subject: [PATCH 202/823] support nhwc format in SpatialConvolution (#1434) * support nhwc format in SpatialConvolution * add python api and doc * meet code review * fix bug and add tests * meet code review * fix bug --- python/dllib/src/bigdl/dllib/nn/layer.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 23c2a9e563f..931da1bd065 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -736,6 +736,9 @@ class SpatialConvolution(Layer): :param init_grad_weight: the optional initial value for the grad_weight :param init_grad_bias: the optional initial value for the grad_bias :param with_bias: the optional initial value for if need bias + :param data_format: a string value of "NHWC" or "NCHW" to specify the input data format of this layer. In "NHWC" format + data is stored in the order of [batch_size, height, width, channels], in "NCHW" format data is stored + in the order of [batch_size, channels, height, width]. >>> spatialConvolution = SpatialConvolution(6, 12, 5, 5) creating: createSpatialConvolution @@ -748,7 +751,7 @@ class SpatialConvolution(Layer): >>> init_bias = np.random.randn(12) >>> init_grad_weight = np.zeros([1, 12, 6, 5, 5]) >>> init_grad_bias = np.zeros([12]) - >>> spatialConvolution = SpatialConvolution(6, 12, 5, 5, 1, 1, 0, 0, 1, True, L1Regularizer(0.5), L1Regularizer(0.5), init_weight, init_bias, init_grad_weight, init_grad_bias) + >>> spatialConvolution = SpatialConvolution(6, 12, 5, 5, 1, 1, 0, 0, 1, True, L1Regularizer(0.5), L1Regularizer(0.5), init_weight, init_bias, init_grad_weight, init_grad_bias, True, "NCHW") creating: createL1Regularizer creating: createL1Regularizer creating: createSpatialConvolution @@ -772,6 +775,7 @@ def __init__(self, init_grad_weight=None, init_grad_bias=None, with_bias=True, + data_format="NCHW", bigdl_type="float"): super(SpatialConvolution, self).__init__(None, bigdl_type, n_input_plane, @@ -790,7 +794,8 @@ def __init__(self, JTensor.from_ndarray(init_bias), JTensor.from_ndarray(init_grad_weight), JTensor.from_ndarray(init_grad_bias), - with_bias) + with_bias, + data_format) def set_init_method(self, weight_init_method = None, bias_init_method = None): callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, weight_init_method, bias_init_method) From 3874e08510da2a7533f836b9c8a58c2a1e2b89f8 Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Tue, 15 Aug 2017 09:03:28 +0800 Subject: [PATCH 203/823] Switch to pytest for python unittest (#1452) * switch to pytest * delete useless code * fix style and unittest * specify python version * ignore sentense.py --- python/dllib/src/bigdl/dllib/utils/common.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/utils/common.py b/python/dllib/src/bigdl/dllib/utils/common.py index be79aa570e0..8b153ce85cc 100644 --- a/python/dllib/src/bigdl/dllib/utils/common.py +++ b/python/dllib/src/bigdl/dllib/utils/common.py @@ -91,7 +91,7 @@ def __str__(self): return self.value.toString() -class TestResult(): +class EvaluatedResult(): """ A testing result used to benchmark the model quality. """ @@ -107,10 +107,10 @@ def __init__(self, result, total_num, method): self.method = method def __reduce__(self): - return (TestResult, (self.result, self.total_num, self.method)) + return (EvaluatedResult, (self.result, self.total_num, self.method)) def __str__(self): - return "Test result: %s, total_num: %s, method: %s" % ( + return "Evaluated result: %s, total_num: %s, method: %s" % ( self.result, self.total_num, self.method) @@ -240,7 +240,7 @@ def uniform(self, a, b, size): 'Rating', 'LabeledPoint', 'Sample', - 'TestResult', + 'EvaluatedResult', 'JTensor' ] From 2ff390dc13ac3799df6f1c2c5b69f31d1148ce95 Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Tue, 15 Aug 2017 09:03:28 +0800 Subject: [PATCH 204/823] Switch to pytest for python unittest (#1452) * switch to pytest * delete useless code * fix style and unittest * specify python version * ignore sentense.py --- python/dllib/src/bigdl/utils/common.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/python/dllib/src/bigdl/utils/common.py b/python/dllib/src/bigdl/utils/common.py index be79aa570e0..8b153ce85cc 100644 --- a/python/dllib/src/bigdl/utils/common.py +++ b/python/dllib/src/bigdl/utils/common.py @@ -91,7 +91,7 @@ def __str__(self): return self.value.toString() -class TestResult(): +class EvaluatedResult(): """ A testing result used to benchmark the model quality. """ @@ -107,10 +107,10 @@ def __init__(self, result, total_num, method): self.method = method def __reduce__(self): - return (TestResult, (self.result, self.total_num, self.method)) + return (EvaluatedResult, (self.result, self.total_num, self.method)) def __str__(self): - return "Test result: %s, total_num: %s, method: %s" % ( + return "Evaluated result: %s, total_num: %s, method: %s" % ( self.result, self.total_num, self.method) @@ -240,7 +240,7 @@ def uniform(self, a, b, size): 'Rating', 'LabeledPoint', 'Sample', - 'TestResult', + 'EvaluatedResult', 'JTensor' ] From 14504c99749a1f734c76135c5dcd3ba91c1932ad Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Tue, 15 Aug 2017 09:03:28 +0800 Subject: [PATCH 205/823] Switch to pytest for python unittest (#1452) * switch to pytest * delete useless code * fix style and unittest * specify python version * ignore sentense.py --- python/dllib/test/__init__.py | 15 --- .../test/bigdl/resources}/conf/python-api.zip | Bin .../resources}/conf/test_spark-bigdl.conf | 0 .../resources}/mnist-data/testing_data.pickle | 0 .../pre_trained_lenet/lenet-model.9381 | Bin .../pre_trained_lenet/lenet-state.9381 | Bin .../test/bigdl/resources}/test.caffemodel | Bin .../dllib/test/bigdl/resources}/test.prototxt | 0 .../dllib/test/bigdl/test_load_caffe.py | 104 +++++++-------- .../test/bigdl/test_simple_integration.py | 119 ++++++++++-------- .../dllib/test/bigdl/test_tensorflow.py | 15 +-- python/dllib/test/dev/modules.py | 99 --------------- python/dllib/test/dev/prepare_env.sh | 2 +- python/dllib/test/dev/run-tests | 14 ++- python/dllib/test/dev/run-tests.py | 36 +----- 15 files changed, 139 insertions(+), 265 deletions(-) delete mode 100644 python/dllib/test/__init__.py rename {resources => python/dllib/test/bigdl/resources}/conf/python-api.zip (100%) rename {resources => python/dllib/test/bigdl/resources}/conf/test_spark-bigdl.conf (100%) rename {resources => python/dllib/test/bigdl/resources}/mnist-data/testing_data.pickle (100%) rename {resources => python/dllib/test/bigdl/resources}/pre_trained_lenet/lenet-model.9381 (100%) rename {resources => python/dllib/test/bigdl/resources}/pre_trained_lenet/lenet-state.9381 (100%) rename {resources => python/dllib/test/bigdl/resources}/test.caffemodel (100%) rename {resources => python/dllib/test/bigdl/resources}/test.prototxt (100%) rename load_caffe_test.py => python/dllib/test/bigdl/test_load_caffe.py (67%) rename simple_integration_test.py => python/dllib/test/bigdl/test_simple_integration.py (79%) rename tensorflow_test.py => python/dllib/test/bigdl/test_tensorflow.py (76%) delete mode 100644 python/dllib/test/dev/modules.py diff --git a/python/dllib/test/__init__.py b/python/dllib/test/__init__.py deleted file mode 100644 index 2151a805423..00000000000 --- a/python/dllib/test/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -# -# Copyright 2016 The BigDL Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# diff --git a/resources/conf/python-api.zip b/python/dllib/test/bigdl/resources/conf/python-api.zip similarity index 100% rename from resources/conf/python-api.zip rename to python/dllib/test/bigdl/resources/conf/python-api.zip diff --git a/resources/conf/test_spark-bigdl.conf b/python/dllib/test/bigdl/resources/conf/test_spark-bigdl.conf similarity index 100% rename from resources/conf/test_spark-bigdl.conf rename to python/dllib/test/bigdl/resources/conf/test_spark-bigdl.conf diff --git a/resources/mnist-data/testing_data.pickle b/python/dllib/test/bigdl/resources/mnist-data/testing_data.pickle similarity index 100% rename from resources/mnist-data/testing_data.pickle rename to python/dllib/test/bigdl/resources/mnist-data/testing_data.pickle diff --git a/resources/pre_trained_lenet/lenet-model.9381 b/python/dllib/test/bigdl/resources/pre_trained_lenet/lenet-model.9381 similarity index 100% rename from resources/pre_trained_lenet/lenet-model.9381 rename to python/dllib/test/bigdl/resources/pre_trained_lenet/lenet-model.9381 diff --git a/resources/pre_trained_lenet/lenet-state.9381 b/python/dllib/test/bigdl/resources/pre_trained_lenet/lenet-state.9381 similarity index 100% rename from resources/pre_trained_lenet/lenet-state.9381 rename to python/dllib/test/bigdl/resources/pre_trained_lenet/lenet-state.9381 diff --git a/resources/test.caffemodel b/python/dllib/test/bigdl/resources/test.caffemodel similarity index 100% rename from resources/test.caffemodel rename to python/dllib/test/bigdl/resources/test.caffemodel diff --git a/resources/test.prototxt b/python/dllib/test/bigdl/resources/test.prototxt similarity index 100% rename from resources/test.prototxt rename to python/dllib/test/bigdl/resources/test.prototxt diff --git a/load_caffe_test.py b/python/dllib/test/bigdl/test_load_caffe.py similarity index 67% rename from load_caffe_test.py rename to python/dllib/test/bigdl/test_load_caffe.py index e3507d31770..129e94c91ea 100644 --- a/load_caffe_test.py +++ b/python/dllib/test/bigdl/test_load_caffe.py @@ -18,18 +18,24 @@ from bigdl.optim.optimizer import * from bigdl.util.common import * import numpy as np -import unittest +import pytest +from numpy.testing import assert_allclose -class TestLoadCaffe(unittest.TestCase): - - def setUp(self): +class TestLoadCaffe(): + def setup_method(self, method): + """ setup any state tied to the execution of the given method in a + class. setup_method is invoked for every test method of a class. + """ sparkConf = create_spark_conf() self.sc = SparkContext(master="local[4]", appName="test model", conf=sparkConf) init_engine() - def tearDown(self): + def teardown_method(self, method): + """ teardown any state that was previously setup with a setup_method + call. + """ self.sc.stop() def test_load_caffe(self): @@ -38,9 +44,9 @@ def test_load_caffe(self): proto_txt = os.path.join(resource_path, "test.prototxt") model_path = os.path.join(resource_path, "test.caffemodel") - module = Sequential()\ - .add(SpatialConvolution(3, 4, 2, 2).set_name("conv"))\ - .add(SpatialConvolution(4, 3, 2, 2).set_name("conv2"))\ + module = Sequential() \ + .add(SpatialConvolution(3, 4, 2, 2).set_name("conv")) \ + .add(SpatialConvolution(4, 3, 2, 2).set_name("conv2")) \ .add(Linear(27, 2, with_bias=False).set_name("ip")) model = Model.load_caffe(module, proto_txt, model_path, bigdl_type="float") @@ -59,11 +65,11 @@ def test_load_caffe(self): 0.1151610613, -0.4214298427, -0.4075299501, -0.1441932321, -0.3215276599, 0.4862193465, 0.0050434470, 0.4745523334, 0.3657383919, -0.2879499197, 0.3388324380, 0.3669666648, - -0.4454920888, -0.4200569391, -0.4690187573, -0.4590228796])\ - .astype("float")\ - .reshape((4, 3, 2, 2)) + -0.4454920888, -0.4200569391, -0.4690187573, -0.4590228796]) \ + .astype("float") \ + .reshape((1, 4, 3, 2, 2)) - conv1_bias = np.array([0.0458712392, -0.0029324144, -0.0251041390, 0.0052924110])\ + conv1_bias = np.array([0.0458712392, -0.0029324144, -0.0251041390, 0.0052924110]) \ .astype("float") conv2_weight = np.array([ @@ -78,9 +84,9 @@ def test_load_caffe(self): -0.0088762743, 0.0061115879, 0.0048167249, -0.0107875718, -0.0249741413, -0.0018652071, 0.0028419730, 0.0255292989, -0.0091862874, 0.0010728909, 0.0009157739, 0.0073709050, - -0.0088602817, -0.0093507599, 0.0070853345, -0.0074293613])\ - .astype("float")\ - .reshape((3, 4, 2, 2)) + -0.0088602817, -0.0093507599, 0.0070853345, -0.0074293613]) \ + .astype("float") \ + .reshape((1, 3, 4, 2, 2)) conv2_bias = np.array([0, 0, 0]).astype("float") @@ -100,52 +106,52 @@ def test_load_caffe(self): -0.3273061812, 0.2115428150, -0.2002333999, -0.1621897519, 0.0032395422, 0.2072965205]).astype("float").reshape((2, 27)) - self.assertTrue(np.allclose(parameters["conv"]["weight"], - conv1_weight, atol=1e-6, rtol=0)) - self.assertTrue(np.allclose(parameters["conv"]["bias"], - conv1_bias, atol=1e-6, rtol=0)) - self.assertTrue(np.allclose(parameters["conv2"]["weight"], - conv2_weight, atol=1e-6, rtol=0)) - self.assertTrue(np.allclose(parameters["conv2"]["bias"], - conv2_bias, atol=1e-6, rtol=0)) - self.assertTrue(np.allclose(parameters["ip"]["weight"], - linear_weight, atol=1e-6, rtol=0)) + assert_allclose(parameters["conv"]["weight"], + conv1_weight, atol=1e-6, rtol=0) + assert_allclose(parameters["conv"]["bias"], + conv1_bias, atol=1e-6, rtol=0) + assert_allclose(parameters["conv2"]["weight"], + conv2_weight, atol=1e-6, rtol=0) + assert_allclose(parameters["conv2"]["bias"], + conv2_bias, atol=1e-6, rtol=0) + assert_allclose(parameters["ip"]["weight"], + linear_weight, atol=1e-6, rtol=0) # test load caffe not match all parameters - module = Sequential()\ - .add(SpatialConvolution(3, 4, 2, 2).set_name("conv"))\ - .add(SpatialConvolution(4, 3, 2, 2).set_name("conv3"))\ + module = Sequential() \ + .add(SpatialConvolution(3, 4, 2, 2).set_name("conv")) \ + .add(SpatialConvolution(4, 3, 2, 2).set_name("conv3")) \ .add(Linear(27, 2, with_bias=False).set_name("ip")) model = Model.load_caffe(module, proto_txt, model_path, match_all=False) parameters = model.parameters() - self.assertTrue(np.allclose(parameters["conv"]["weight"], - conv1_weight, atol=1e-6, rtol=0)) - self.assertTrue(np.allclose(parameters["conv"]["bias"], - conv1_bias, atol=1e-6, rtol=0)) - self.assertFalse(np.allclose(parameters["conv3"]["weight"], - conv2_weight, atol=1e-6, rtol=0)) - self.assertFalse(np.allclose(parameters["conv3"]["bias"], - conv2_bias, atol=1e-6, rtol=0)) - self.assertTrue(np.allclose(parameters["ip"]["weight"], - linear_weight, atol=1e-6, rtol=0)) + assert_allclose(parameters["conv"]["weight"], + conv1_weight, atol=1e-6, rtol=0) + assert_allclose(parameters["conv"]["bias"], + conv1_bias, atol=1e-6, rtol=0) + assert not (np.allclose(parameters["conv3"]["weight"], + conv2_weight, atol=1e-6, rtol=0)) + assert not (np.allclose(parameters["conv3"]["bias"], + conv2_bias, atol=1e-6, rtol=0)) + assert_allclose(parameters["ip"]["weight"], + linear_weight, atol=1e-6, rtol=0) # test load caffe dynamically model = Model.load_caffe_model(proto_txt, model_path, bigdl_type="float") parameters = model.parameters() - self.assertTrue(np.allclose(parameters["conv"]["weight"], - conv1_weight, atol=1e-6, rtol=0)) - self.assertTrue(np.allclose(parameters["conv"]["bias"], - conv1_bias, atol=1e-6, rtol=0)) - self.assertTrue(np.allclose(parameters["conv2"]["weight"], - conv2_weight, atol=1e-6, rtol=0)) - self.assertTrue(np.allclose(parameters["conv2"]["bias"], - conv2_bias, atol=1e-6, rtol=0)) - self.assertTrue(np.allclose(parameters["ip"]["weight"], - linear_weight, atol=1e-6, rtol=0)) + assert_allclose(parameters["conv"]["weight"], + conv1_weight, atol=1e-6, rtol=0) + assert_allclose(parameters["conv"]["bias"], + conv1_bias, atol=1e-6, rtol=0) + assert_allclose(parameters["conv2"]["weight"], + conv2_weight, atol=1e-6, rtol=0) + assert_allclose(parameters["conv2"]["bias"], + conv2_bias, atol=1e-6, rtol=0) + assert_allclose(parameters["ip"]["weight"], + linear_weight, atol=1e-6, rtol=0) if __name__ == "__main__": - unittest.main() + pytest.main([__file__]) diff --git a/simple_integration_test.py b/python/dllib/test/bigdl/test_simple_integration.py similarity index 79% rename from simple_integration_test.py rename to python/dllib/test/bigdl/test_simple_integration.py index 8fdf52ee3a4..35e4d5e8216 100644 --- a/simple_integration_test.py +++ b/python/dllib/test/bigdl/test_simple_integration.py @@ -22,18 +22,25 @@ from bigdl.nn.initialization_method import * from bigdl.dataset import movielens import numpy as np -import unittest import tempfile +import pytest +from numpy.testing import assert_allclose -class TestWorkFlow(unittest.TestCase): - def setUp(self): +class TestSimple(): + def setup_method(self, method): + """ setup any state tied to the execution of the given method in a + class. setup_method is invoked for every test method of a class. + """ sparkConf = create_spark_conf() self.sc = SparkContext(master="local[4]", appName="test model", conf=sparkConf) init_engine() - def tearDown(self): + def teardown_method(self, method): + """ teardown any state that was previously setup with a setup_method + call. + """ self.sc.stop() def test_training(self): @@ -52,25 +59,25 @@ def grad_update(mlp, x, y, criterion, learning_rate): mlp.update_parameters(learning_rate) return err - mse = MSECriterion() + mse = MSECriterion(self) for i in range(0, 1000): x = np.random.random((5, 4)) y = x.copy() y = y + bf err = grad_update(cadd, x, y, mse, 0.01) print(cadd.get_weights()[0]) - self.assertTrue(np.allclose(cadd.get_weights()[0], - np.array([1, 2, 3, 4, 5]).reshape((5, 1)), - rtol=1.e-1)) + assert_allclose(cadd.get_weights()[0], + np.array([1, 2, 3, 4, 5]).reshape((5, 1)), + rtol=1.e-1) def test_load_model(self): fc1 = Linear(4, 2) - fc1.set_weights([np.ones((4, 2)), np.ones((2, ))]) + fc1.set_weights([np.ones((4, 2)), np.ones((2,))]) tmp_path = tempfile.mktemp() fc1.save(tmp_path, True) fc1_loaded = Model.load(tmp_path) - self.assertTrue(np.allclose(fc1_loaded.get_weights()[0], - fc1.get_weights()[0])) + assert_allclose(fc1_loaded.get_weights()[0], + fc1.get_weights()[0]) def test_load_optim_method(self): FEATURES_DIM = 2 @@ -82,6 +89,7 @@ def gen_rand_sample(): features = np.random.uniform(0, 1, (FEATURES_DIM)) label = (2 * features).sum() + 0.4 return Sample.from_ndarray(features, label) + trainingData = self.sc.parallelize(range(0, data_len)).map(lambda i: gen_rand_sample()) model = Sequential() l1 = Linear(FEATURES_DIM, 1).set_init_method(Xavier(), Zeros()).set_name("linear1") @@ -89,14 +97,14 @@ def gen_rand_sample(): sgd = SGD(learningrate=0.01, learningrate_decay=0.0002, weightdecay=0.0, momentum=0.0, dampening=0.0, nesterov=False, - leaningrate_schedule=Poly(0.5, int((data_len/batch_size)*epoch_num))) + leaningrate_schedule=Poly(0.5, int((data_len / batch_size) * epoch_num))) tmp_path = tempfile.mktemp() sgd.save(tmp_path, True) optim_method = OptimMethod.load(tmp_path) - self.assertTrue(optim_method.learningRate() == sgd.value.learningRate()) - self.assertTrue(optim_method.momentum() == sgd.value.momentum()) - self.assertTrue(optim_method.nesterov() == sgd.value.nesterov()) + assert optim_method.learningRate() == sgd.value.learningRate() + assert optim_method.momentum() == sgd.value.momentum() + assert optim_method.nesterov() == sgd.value.nesterov() optimizer = Optimizer( model=model, @@ -114,12 +122,12 @@ def test_create_node(self): cadd = CAddTable()([fc1, fc2]) output1 = ReLU()(cadd) model = Model([fc1, fc2], [output1]) - fc1.element().set_weights([np.ones((4, 2)), np.ones((2, ))]) - fc2.element().set_weights([np.ones((4, 2)), np.ones((2, ))]) + fc1.element().set_weights([np.ones((4, 2)), np.ones((2,))]) + fc2.element().set_weights([np.ones((4, 2)), np.ones((2,))]) output = model.forward([np.array([0.1, 0.2, -0.3, -0.4]), np.array([0.5, 0.4, -0.2, -0.1])]) - self.assertTrue(np.allclose(output, - np.array([2.2, 2.2]))) + assert_allclose(output, + np.array([2.2, 2.2])) def test_graph_backward(self): fc1 = Linear(4, 2)() @@ -128,36 +136,40 @@ def test_graph_backward(self): output1 = ReLU()(cadd) output2 = Threshold(10.0)(cadd) model = Model([fc1, fc2], [output1, output2]) - fc1.element().set_weights([np.ones((4, 2)), np.ones((2, ))]) - fc2.element().set_weights([np.ones((4, 2)) * 2, np.ones((2, )) * 2]) + fc1.element().set_weights([np.ones((4, 2)), np.ones((2,))]) + fc2.element().set_weights([np.ones((4, 2)) * 2, np.ones((2,)) * 2]) output = model.forward([np.array([0.1, 0.2, -0.3, -0.4]), np.array([0.5, 0.4, -0.2, -0.1])]) gradInput = model.backward([np.array([0.1, 0.2, -0.3, -0.4]), np.array([0.5, 0.4, -0.2, -0.1])], [np.array([1.0, 2.0]), np.array([3.0, 4.0])]) - self.assertTrue(np.allclose(gradInput[0], - np.array([3.0, 3.0, 3.0, 3.0]))) - self.assertTrue(np.allclose(gradInput[1], - np.array([6.0, 6.0, 6.0, 6.0]))) + assert_allclose(gradInput[0], + np.array([3.0, 3.0, 3.0, 3.0])) + assert_allclose(gradInput[1], + np.array([6.0, 6.0, 6.0, 6.0])) def test_load_zip_conf(self): from bigdl.util.common import get_bigdl_conf import sys sys.path = [path for path in sys.path if "spark-bigdl.conf" not in path] - sys.path.insert(0, os.path.join(os.path.split(__file__)[0], "resources/conf/python-api.zip")) # noqa - sys.path.insert(0, os.path.join(os.path.split(__file__)[0], "resources/conf/invalid-python-api.zip")) # noqa + sys.path.insert(0, os.path.join(os.path.split(__file__)[0], + "resources/conf/python-api.zip")) # noqa + sys.path.insert(0, os.path.join(os.path.split(__file__)[0], + "resources/conf/invalid-python-api.zip")) # noqa result = get_bigdl_conf() - self.assertTrue(result.get("spark.executorEnv.OMP_WAIT_POLICY"), "passive") + assert result.get("spark.executorEnv.OMP_WAIT_POLICY"), "passive" def test_set_seed(self): w_init = Xavier() b_init = Zeros() - l1 = Linear(10, 20).set_init_method(w_init, b_init).set_name("linear1").set_seed(1234).reset() # noqa - l2 = Linear(10, 20).set_init_method(w_init, b_init).set_name("linear2").set_seed(1234).reset() # noqa + l1 = Linear(10, 20).set_init_method(w_init, b_init).set_name("linear1").set_seed( + 1234).reset() # noqa + l2 = Linear(10, 20).set_init_method(w_init, b_init).set_name("linear2").set_seed( + 1234).reset() # noqa p1 = l1.parameters() p2 = l2.parameters() - self.assertTrue((p1["linear1"]["weight"] == p2["linear2"]["weight"]).all()) # noqa + assert (p1["linear1"]["weight"] == p2["linear2"]["weight"]).all() # noqa def test_simple_flow(self): FEATURES_DIM = 2 @@ -174,20 +186,20 @@ def gen_rand_sample(): lambda i: gen_rand_sample()) model_test = Sequential() - l1_test = Linear(FEATURES_DIM, 1).set_init_method(Xavier(), Zeros())\ + l1_test = Linear(FEATURES_DIM, 1).set_init_method(Xavier(), Zeros()) \ .set_name("linear1_test") - self.assertEqual("linear1_test", l1_test.name()) + assert "linear1_test" == l1_test.name() model_test.add(l1_test) model_test.add(Sigmoid()) model = Sequential() l1 = Linear(FEATURES_DIM, 1).set_init_method(Xavier(), Zeros()).set_name("linear1") - self.assertEqual("linear1", l1.name()) + assert "linear1" == l1.name() model.add(l1) optim_method = SGD(learningrate=0.01, learningrate_decay=0.0002, weightdecay=0.0, momentum=0.0, dampening=0.0, nesterov=False, - leaningrate_schedule=Poly(0.5, int((data_len/batch_size)*epoch_num))) + leaningrate_schedule=Poly(0.5, int((data_len / batch_size) * epoch_num))) optimizer = Optimizer( model=model_test, training_rdd=trainingData, @@ -222,7 +234,7 @@ def gen_rand_sample(): # TODO: add result validation parameters = trained_model.parameters() - self.assertIsNotNone(parameters["linear1"]) + assert parameters["linear1"] is not None print("parameters %s" % parameters["linear1"]) predict_result = trained_model.predict(trainingData) p = predict_result.take(2) @@ -243,13 +255,13 @@ def test_forward_backward(self): linear = Linear(4, 5) input = rng.uniform(0.0, 1.0, [4]) output = linear.forward(input) - self.assertTrue(np.allclose(output, - np.array([0.41366524, - 0.009532653, - -0.677581, - 0.07945433, - -0.5742568]), - atol=1e-6, rtol=0)) + assert_allclose(output, + np.array([0.41366524, + 0.009532653, + -0.677581, + 0.07945433, + -0.5742568]), + atol=1e-6, rtol=0) mse = MSECriterion() target = rng.uniform(0.0, 1.0, [5]) loss = mse.forward(output, target) @@ -343,18 +355,18 @@ def test_predict(self): label = (features).sum() + 0.4 predict_data = self.sc.parallelize(range(0, total_length)).map( lambda i: Sample.from_ndarray(features[i], label)) - model = Linear(2, 1).set_init_method(Xavier(), Zeros())\ + model = Linear(2, 1).set_init_method(Xavier(), Zeros()) \ .set_name("linear1").set_seed(1234).reset() predict_result = model.predict(predict_data) p = predict_result.take(6) ground_label = np.array([[-0.47596836], [-0.37598032], [-0.00492062], [-0.5906958], [-0.12307882], [-0.77907401]], dtype="float32") for i in range(0, total_length): - self.assertTrue(np.allclose(p[i], ground_label[i], atol=1e-6, rtol=0)) + assert_allclose(p[i], ground_label[i], atol=1e-6, rtol=0) predict_class = model.predict_class(predict_data) predict_labels = predict_class.take(6) for i in range(0, total_length): - self.assertTrue(predict_labels[i] == 1) + assert predict_labels[i] == 1 def test_rng(self): rng = RNG() @@ -362,16 +374,16 @@ def test_rng(self): result = rng.uniform(0.1, 0.2, [2, 3]) ground_label = np.array([[0.15434049, 0.16711557, 0.12783694], [0.14120464, 0.14245176, 0.15263824]]) - self.assertTrue(result.shape == (2, 3)) + assert result.shape == (2, 3) data = result for i in range(0, 2): - self.assertTrue(np.allclose(data[i], ground_label[i], atol=1e-6, rtol=0)) + assert_allclose(data[i], ground_label[i], atol=1e-6, rtol=0) rng.set_seed(100) result2 = rng.uniform(0.1, 0.2, [2, 3]) data2 = result2 for i in range(0, 2): - self.assertTrue(np.allclose(data[i], data2[i])) + assert_allclose(data[i], data2[i]) def test_movielens(self): movielens_data = movielens.read_data_sets("/tmp/movielens/") @@ -382,9 +394,10 @@ def test_movielens(self): ground_pairs = np.array([[1, 1193], [1, 661]]) ground_ratings = np.array([[1, 1193, 5], [1, 661, 3]]) for i in range(0, 2): - self.assertTrue(np.allclose(movielens_data[i], ground_data[i], atol=1e-6, rtol=0)) - self.assertTrue(np.allclose(id_pairs[i], ground_pairs[i], atol=1e-6, rtol=0)) - self.assertTrue(np.allclose(id_ratings[i], ground_ratings[i], atol=1e-6, rtol=0)) + assert_allclose(movielens_data[i], ground_data[i], atol=1e-6, rtol=0) + assert_allclose(id_pairs[i], ground_pairs[i], atol=1e-6, rtol=0) + assert_allclose(id_ratings[i], ground_ratings[i], atol=1e-6, rtol=0) + if __name__ == "__main__": - unittest.main(failfast=True) + pytest.main([__file__]) diff --git a/tensorflow_test.py b/python/dllib/test/bigdl/test_tensorflow.py similarity index 76% rename from tensorflow_test.py rename to python/dllib/test/bigdl/test_tensorflow.py index c20208842f3..70304617211 100644 --- a/tensorflow_test.py +++ b/python/dllib/test/bigdl/test_tensorflow.py @@ -21,18 +21,10 @@ import unittest import shutil import tempfile +from numpy.testing import assert_allclose -class TestTensorflow(unittest.TestCase): - - def setUp(self): - sparkConf = create_spark_conf() - self.sc = SparkContext(master="local[4]", appName="test model", - conf=sparkConf) - init_engine() - - def tearDown(self): - self.sc.stop() +class TestTensorflow(): def test_load_and_save(self): linear = Linear(10, 2)() @@ -48,8 +40,7 @@ def test_load_and_save(self): model_loaded = Model.load_tensorflow(temp + "/model.pb", ["input"], ["output"]) expected_output = model_original.forward(input) output = model_loaded.forward(input) - self.assertTrue(np.allclose(output, - expected_output, atol=1e-6, rtol=0)) + assert_allclose(output, expected_output, atol=1e-6, rtol=0) if __name__ == "__main__": diff --git a/python/dllib/test/dev/modules.py b/python/dllib/test/dev/modules.py deleted file mode 100644 index 25313d95ecb..00000000000 --- a/python/dllib/test/dev/modules.py +++ /dev/null @@ -1,99 +0,0 @@ -# -# Copyright 2016 The BigDL Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# Adopt from Spark and it might be refactored in the future - -from functools import total_ordering - -all_modules = [] - - -@total_ordering -class Module(object): - - def __init__(self, name, python_test_goals=()): - """ - Define a new module. - - :param name: A short module name, for display in logging and error messagesl - :param python_test_goals: A set of Python test goals for testing this module. - """ - self.name = name - self.python_test_goals = python_test_goals - - all_modules.append(self) - - def __repr__(self): - return "Module<%s>" % self.name - - def __lt__(self, other): - return self.name < other.name - - def __eq__(self, other): - return self.name == other.name - - def __ne__(self, other): - return not (self.name == other.name) - - def __hash__(self): - return hash(self.name) - - -bigdl_layer = Module( - name="bigdl_layer", - python_test_goals=[ - "bigdl.nn.layer" - ]) - -bigdl_layer = Module( - name="bigdl_criterion", - python_test_goals=[ - "bigdl.nn.criterion" - ]) - -bigdl_layer = Module( - name="bigdl_common", - python_test_goals=[ - "bigdl.util.common" - ]) - -bigdl_optimizer = Module( - name="bigdl_optimizer", - python_test_goals=[ - "bigdl.optim.optimizer", - ] -) - -test_simple_integration_test = Module( - name="simple_integration_test", - python_test_goals=[ - "test.simple_integration_test" - ] -) - -test_load_caffe = Module( - name="load_caffe_test", - python_test_goals=[ - "test.load_caffe_test" - ] -) - -test_tensorflow = Module( - name="tensorflow_test", - python_test_goals=[ - "test.tensorflow_test" - ] -) diff --git a/python/dllib/test/dev/prepare_env.sh b/python/dllib/test/dev/prepare_env.sh index b77cdd7d5a7..fbde2c55379 100755 --- a/python/dllib/test/dev/prepare_env.sh +++ b/python/dllib/test/dev/prepare_env.sh @@ -33,7 +33,7 @@ export PYTHONPATH=$PYSPARK_ZIP:$DL_PYTHON_HOME:$DL_PYTHON_HOME/:$DL_PYTHON_HOME/ export SPARK_CLASSPATH=$(find $BIGDL_HOME/spark/dl/target/ -name "*with-dependencies.jar" | head -n 1) echo "SPARK_CLASSPATH": $SPARK_CLASSPATH -export PYTHON_EXECUTABLES=("python2" "python3") +export PYTHON_EXECUTABLES=("python2.7" "python3.5") function run_notebook() { notebook_path=$1 diff --git a/python/dllib/test/dev/run-tests b/python/dllib/test/dev/run-tests index d089568c143..27853e22b03 100755 --- a/python/dllib/test/dev/run-tests +++ b/python/dllib/test/dev/run-tests @@ -19,4 +19,16 @@ . `dirname $0`/prepare_env.sh cd "`dirname $0`" -exec python -u ./run-tests.py "$@" + +export DL_CORE_NUMBER=4 + +for p in ${PYTHON_EXECUTABLES[@]} +do + echo "${cyan}Using python version: $p${reset}" + export PYTHON_EXECUTABLE=$p + export PYSPARK_PYTHON=$p + export PYSPARK_DRIVER_PYTHON=$p + $p -m pytest -v --doctest-modules ../../../pyspark/bigdl --ignore=../../../pyspark/bigdl/dataset/sentence.py + $p -m pytest -v -n 4 ../../../pyspark/test/ +done + diff --git a/python/dllib/test/dev/run-tests.py b/python/dllib/test/dev/run-tests.py index ea778236b09..b911c0c9552 100755 --- a/python/dllib/test/dev/run-tests.py +++ b/python/dllib/test/dev/run-tests.py @@ -91,11 +91,7 @@ def print_red(text): def run_individual_python_test(test_name, python_exec): - env = dict(os.environ) - env.update({ - 'DL_CORE_NUMBER': '4', - 'PYSPARK_PYTHON': python_exec - }) + LOGGER.debug("Starting test(%s): %s", python_exec, test_name) start_time = time.time() try: @@ -139,36 +135,6 @@ def get_default_python_executables(): return python_execs -def parse_opts(): - parser = OptionParser( - prog="run-tests" - ) - parser.add_option( - "--python-executables", type="string", default=','.join(get_default_python_executables()), - help="A comma-separated list of Python executables to test against (default: %default)" - ) - parser.add_option( - "--modules", type="string", - default=",".join(sorted(python_modules.keys())), - help="A comma-separated list of Python modules to test (default: %default)" - ) - parser.add_option( - "-p", "--parallelism", type="int", default=4, - help="The number of suites to test in parallel (default %default)" - ) - parser.add_option( - "--verbose", action="store_true", - help="Enable additional debug logging" - ) - - (opts, args) = parser.parse_args() - if args: - parser.error("Unsupported arguments: %s" % ' '.join(args)) - if opts.parallelism < 1: - parser.error("Parallelism cannot be less than 1") - return opts - - def main(): opts = parse_opts() if (opts.verbose): From 704b213fd645184a076290ce19e9aaacfc55e4b4 Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Wed, 16 Aug 2017 12:37:07 +0800 Subject: [PATCH 206/823] Ignore some folders from doctest scanning (#1465) * remove movielens downloading from unittest * fix spark1.5 Lock problem * fix return --- python/dllib/src/bigdl/dllib/utils/common.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/utils/common.py b/python/dllib/src/bigdl/dllib/utils/common.py index 8b153ce85cc..c0b4d0063c9 100644 --- a/python/dllib/src/bigdl/dllib/utils/common.py +++ b/python/dllib/src/bigdl/dllib/utils/common.py @@ -283,14 +283,17 @@ def create_spark_conf(): sparkConf.setAll(bigdl_conf.items()) return sparkConf - def get_spark_context(conf = None): """ Get the current active spark context and create one if no active instance :param conf: combining bigdl configs into spark conf :return: SparkContext """ - with SparkContext._lock: # Compatible with Spark1.5.1 + if hasattr(SparkContext, "getOrCreate"): + return SparkContext.getOrCreate(conf=conf or create_spark_conf()) + else: + # Might have threading issue but we cann't add _lock here + # as it's not RLock in spark1.5 if SparkContext._active_spark_context is None: SparkContext(conf=conf or create_spark_conf()) return SparkContext._active_spark_context From 88a988f49a942bd45778165ca0ecb232d47ce3e1 Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Wed, 16 Aug 2017 12:37:07 +0800 Subject: [PATCH 207/823] Ignore some folders from doctest scanning (#1465) * remove movielens downloading from unittest * fix spark1.5 Lock problem * fix return --- python/dllib/src/bigdl/utils/common.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/python/dllib/src/bigdl/utils/common.py b/python/dllib/src/bigdl/utils/common.py index 8b153ce85cc..c0b4d0063c9 100644 --- a/python/dllib/src/bigdl/utils/common.py +++ b/python/dllib/src/bigdl/utils/common.py @@ -283,14 +283,17 @@ def create_spark_conf(): sparkConf.setAll(bigdl_conf.items()) return sparkConf - def get_spark_context(conf = None): """ Get the current active spark context and create one if no active instance :param conf: combining bigdl configs into spark conf :return: SparkContext """ - with SparkContext._lock: # Compatible with Spark1.5.1 + if hasattr(SparkContext, "getOrCreate"): + return SparkContext.getOrCreate(conf=conf or create_spark_conf()) + else: + # Might have threading issue but we cann't add _lock here + # as it's not RLock in spark1.5 if SparkContext._active_spark_context is None: SparkContext(conf=conf or create_spark_conf()) return SparkContext._active_spark_context From 1a792842048da24cf4f91cb1a6c49820a9cbccce Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Wed, 16 Aug 2017 12:37:07 +0800 Subject: [PATCH 208/823] Ignore some folders from doctest scanning (#1465) * remove movielens downloading from unittest * fix spark1.5 Lock problem * fix return --- python/dllib/test/bigdl/test_simple_integration.py | 13 ------------- python/dllib/test/dev/run-tests | 6 +++++- 2 files changed, 5 insertions(+), 14 deletions(-) diff --git a/python/dllib/test/bigdl/test_simple_integration.py b/python/dllib/test/bigdl/test_simple_integration.py index 35e4d5e8216..9a75ba40d53 100644 --- a/python/dllib/test/bigdl/test_simple_integration.py +++ b/python/dllib/test/bigdl/test_simple_integration.py @@ -385,19 +385,6 @@ def test_rng(self): for i in range(0, 2): assert_allclose(data[i], data2[i]) - def test_movielens(self): - movielens_data = movielens.read_data_sets("/tmp/movielens/") - id_pairs = movielens.get_id_pairs("/tmp/movielens/") - id_ratings = movielens.get_id_ratings("/tmp/movielens/") - - ground_data = np.array([[1, 1193, 5, 978300760], [1, 661, 3, 978302109]]) - ground_pairs = np.array([[1, 1193], [1, 661]]) - ground_ratings = np.array([[1, 1193, 5], [1, 661, 3]]) - for i in range(0, 2): - assert_allclose(movielens_data[i], ground_data[i], atol=1e-6, rtol=0) - assert_allclose(id_pairs[i], ground_pairs[i], atol=1e-6, rtol=0) - assert_allclose(id_ratings[i], ground_ratings[i], atol=1e-6, rtol=0) - if __name__ == "__main__": pytest.main([__file__]) diff --git a/python/dllib/test/dev/run-tests b/python/dllib/test/dev/run-tests index 27853e22b03..39def31a75a 100755 --- a/python/dllib/test/dev/run-tests +++ b/python/dllib/test/dev/run-tests @@ -28,7 +28,11 @@ do export PYTHON_EXECUTABLE=$p export PYSPARK_PYTHON=$p export PYSPARK_DRIVER_PYTHON=$p - $p -m pytest -v --doctest-modules ../../../pyspark/bigdl --ignore=../../../pyspark/bigdl/dataset/sentence.py + $p -m pytest -v --doctest-modules ../../../pyspark/bigdl \ + --ignore=../../../pyspark/bigdl/dataset/ \ + --ignore=../../../pyspark/bigdl/util/ \ + --ignore=../../../pyspark/bigdl/models/ + $p -m pytest -v -n 4 ../../../pyspark/test/ done From 12b0e37bc38a94eca1bd882bc418d42359071f5e Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Sun, 20 Aug 2017 19:51:48 -0700 Subject: [PATCH 209/823] Support NHWC format in SpatialMaxPooling (#1444) * support nhwc in SpatialMaxPooling * add tests * fix bug and meet code review * add python api and doc --- python/dllib/src/bigdl/dllib/nn/layer.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 931da1bd065..42c9fecfec8 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -820,9 +820,12 @@ class SpatialMaxPooling(Layer): :param dH: step size in height :param padW: padding in width :param padH: padding in height + :param format: "NCHW" or "NHWC", indicating the input data format >>> spatialMaxPooling = SpatialMaxPooling(2, 2, 2, 2) creating: createSpatialMaxPooling + >>> spatialMaxPooling = SpatialMaxPooling(2, 2, 2, 2, -1, -1, True, "NHWC") + creating: createSpatialMaxPooling ''' # to_ceil: call floor() when False; call ceil() when True @@ -833,6 +836,7 @@ def __init__(self, kw, pad_w=0, pad_h=0, to_ceil=False, + format="NCHW", bigdl_type="float"): super(SpatialMaxPooling, self).__init__(None, bigdl_type, kw, kh, @@ -840,7 +844,8 @@ def __init__(self, kw, dh, pad_w, pad_h, - to_ceil) + to_ceil, + format) class Select(Layer): From b4204c511b037d82b401b6033d0e4638f4cba5de Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Sun, 20 Aug 2017 19:51:48 -0700 Subject: [PATCH 210/823] Support NHWC format in SpatialMaxPooling (#1444) * support nhwc in SpatialMaxPooling * add tests * fix bug and meet code review * add python api and doc --- python/dllib/src/bigdl/dllib/nn/layer.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 931da1bd065..42c9fecfec8 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -820,9 +820,12 @@ class SpatialMaxPooling(Layer): :param dH: step size in height :param padW: padding in width :param padH: padding in height + :param format: "NCHW" or "NHWC", indicating the input data format >>> spatialMaxPooling = SpatialMaxPooling(2, 2, 2, 2) creating: createSpatialMaxPooling + >>> spatialMaxPooling = SpatialMaxPooling(2, 2, 2, 2, -1, -1, True, "NHWC") + creating: createSpatialMaxPooling ''' # to_ceil: call floor() when False; call ceil() when True @@ -833,6 +836,7 @@ def __init__(self, kw, pad_w=0, pad_h=0, to_ceil=False, + format="NCHW", bigdl_type="float"): super(SpatialMaxPooling, self).__init__(None, bigdl_type, kw, kh, @@ -840,7 +844,8 @@ def __init__(self, kw, dh, pad_w, pad_h, - to_ceil) + to_ceil, + format) class Select(Layer): From 7385c69cf49f5c7232ea54afabc21bc39bd647ca Mon Sep 17 00:00:00 2001 From: Xianyan Date: Wed, 23 Aug 2017 10:44:11 +0800 Subject: [PATCH 211/823] [Issue #1033 #450] Disable backpropagation for general layers (#1181) * add propagateback and generate backward graph new unit test pass unit test except resnet pass unit test pass support rebuild support multiple inputs fix updateGradInput and acc fix unit test rename to trainable, add API to set trainable at graph unit test except inception and resnet pass unit test pass code clean fix conflicts add unit test for setTrainable, setFreeze, unFreeze use input buffer add getSubmodule in AbstractModule resolve conflicts update isInputNode update comment update setTrainable fix fix mixture of graph and container add setFreeze python API update python and add doc meet code review * meet code review * fix unit test * meet code review * fix code style * fix inception unit test * meet code review * meet code review * rm cloneNode * resolve conflicts * fix reset * meet code review * meet code review * fix --- python/dllib/src/bigdl/dllib/nn/layer.py | 49 ++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 42c9fecfec8..15754b1a890 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -376,6 +376,22 @@ def setBRegularizer(self, bRegularizer): ''' self.value.bRegularizer = bRegularizer.value + def freeze(self): + ''' + freeze layer + ''' + callBigDlFunc(self.bigdl_type, + "setLayerFreeze", self.value) + return self + + def unfreeze(self): + ''' + unfreeze layer + ''' + callBigDlFunc(self.bigdl_type, + "setLayerUnFreeze", self.value) + + class Container(Layer): ''' @@ -491,6 +507,39 @@ def load_tensorflow(path, inputs, outputs, byte_order = "little_endian", bigdl_t jmodel = callBigDlFunc(bigdl_type, "loadTF", path, inputs, outputs, byte_order) return Model.of(jmodel) + def freeze(self, freeze_layers, bigdl_type="float"): + """ + set an array of layers to be freezed + :param freeze_layers: an array of layer names + :param bigdl_type: + :return: + """ + callBigDlFunc(bigdl_type, "setFreeze", self.value, freeze_layers) + return self + + def unfreeze(self, bigdl_type="float"): + """ + set all layers to be trainable + :param bigdl_type: + :return: + """ + callBigDlFunc(bigdl_type, "unFreeze", self.value) + return self + + def stop_gradient(self, stop_layers, bigdl_type="float"): + """ + stop the input gradient of layers that match the given ```names``` + their input gradient are not computed. + And they will not contributed to the input gradient computation of + layers that depend on them. + :param stop_layers: an array of layer names + :param bigdl_type: + :return: + """ + callBigDlFunc(bigdl_type, "setStopGradient", self.value, stop_layers) + return self + + class Linear(Layer): From acd95515ca1f8a42627b36c5156d2e1bac7ea479 Mon Sep 17 00:00:00 2001 From: Xianyan Date: Wed, 23 Aug 2017 10:44:11 +0800 Subject: [PATCH 212/823] [Issue #1033 #450] Disable backpropagation for general layers (#1181) * add propagateback and generate backward graph new unit test pass unit test except resnet pass unit test pass support rebuild support multiple inputs fix updateGradInput and acc fix unit test rename to trainable, add API to set trainable at graph unit test except inception and resnet pass unit test pass code clean fix conflicts add unit test for setTrainable, setFreeze, unFreeze use input buffer add getSubmodule in AbstractModule resolve conflicts update isInputNode update comment update setTrainable fix fix mixture of graph and container add setFreeze python API update python and add doc meet code review * meet code review * fix unit test * meet code review * fix code style * fix inception unit test * meet code review * meet code review * rm cloneNode * resolve conflicts * fix reset * meet code review * meet code review * fix --- python/dllib/src/bigdl/dllib/nn/layer.py | 49 ++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 42c9fecfec8..15754b1a890 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -376,6 +376,22 @@ def setBRegularizer(self, bRegularizer): ''' self.value.bRegularizer = bRegularizer.value + def freeze(self): + ''' + freeze layer + ''' + callBigDlFunc(self.bigdl_type, + "setLayerFreeze", self.value) + return self + + def unfreeze(self): + ''' + unfreeze layer + ''' + callBigDlFunc(self.bigdl_type, + "setLayerUnFreeze", self.value) + + class Container(Layer): ''' @@ -491,6 +507,39 @@ def load_tensorflow(path, inputs, outputs, byte_order = "little_endian", bigdl_t jmodel = callBigDlFunc(bigdl_type, "loadTF", path, inputs, outputs, byte_order) return Model.of(jmodel) + def freeze(self, freeze_layers, bigdl_type="float"): + """ + set an array of layers to be freezed + :param freeze_layers: an array of layer names + :param bigdl_type: + :return: + """ + callBigDlFunc(bigdl_type, "setFreeze", self.value, freeze_layers) + return self + + def unfreeze(self, bigdl_type="float"): + """ + set all layers to be trainable + :param bigdl_type: + :return: + """ + callBigDlFunc(bigdl_type, "unFreeze", self.value) + return self + + def stop_gradient(self, stop_layers, bigdl_type="float"): + """ + stop the input gradient of layers that match the given ```names``` + their input gradient are not computed. + And they will not contributed to the input gradient computation of + layers that depend on them. + :param stop_layers: an array of layer names + :param bigdl_type: + :return: + """ + callBigDlFunc(bigdl_type, "setStopGradient", self.value, stop_layers) + return self + + class Linear(Layer): From ca8c59c31e87cc1c8eb1d2f63d284731399cad0e Mon Sep 17 00:00:00 2001 From: Ian Wong Date: Thu, 24 Aug 2017 13:59:13 +0800 Subject: [PATCH 213/823] Export tf checkpoint (#1498) * Support dump tensors from checkpoint and save as Java readable file * add unit test * meet code review --- python/dllib/src/bigdl/dllib/utils/common.py | 8 +- .../dllib/src/bigdl/dllib/utils/tf_utils.py | 83 ++++++++++++++++++- 2 files changed, 86 insertions(+), 5 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/utils/common.py b/python/dllib/src/bigdl/dllib/utils/common.py index c0b4d0063c9..4b3fb177da9 100644 --- a/python/dllib/src/bigdl/dllib/utils/common.py +++ b/python/dllib/src/bigdl/dllib/utils/common.py @@ -19,7 +19,7 @@ import glob from py4j.protocol import Py4JJavaError from py4j.java_gateway import JavaObject -from py4j.java_collections import ListConverter, JavaArray, JavaList, JavaMap +from py4j.java_collections import ListConverter, JavaArray, JavaList, JavaMap, MapConverter from pyspark import RDD, SparkContext from pyspark.serializers import PickleSerializer, AutoBatchedSerializer @@ -375,10 +375,10 @@ def _py2java(sc, obj): sc._gateway._gateway_client) elif isinstance(obj, dict): result = {} + print(obj.keys()) for (key, value) in obj.items(): - result[key] = _py2java(sc, value) if isinstance(value, JavaValue) else value # noqa - obj = result - + result[key] = _py2java(sc, value) + obj = MapConverter().convert(result, sc._gateway._gateway_client) elif isinstance(obj, JavaValue): obj = obj.value elif isinstance(obj, JavaObject): diff --git a/python/dllib/src/bigdl/dllib/utils/tf_utils.py b/python/dllib/src/bigdl/dllib/utils/tf_utils.py index e2a7dc44fc8..0e8696f2c62 100644 --- a/python/dllib/src/bigdl/dllib/utils/tf_utils.py +++ b/python/dllib/src/bigdl/dllib/utils/tf_utils.py @@ -27,6 +27,9 @@ from tensorflow.python.framework import importer from tensorflow.python.platform import gfile from bigdl.nn.layer import Model +from bigdl.util.common import JTensor +from bigdl.util.common import callBigDlFunc +import os def convert(input_ops, output_ops, byte_order, bigdl_type): """ @@ -63,6 +66,84 @@ def convert(input_ops, output_ops, byte_order, bigdl_type): return model + +def export_checkpoint(checkpoint_path): + """ + Export variable tensors from the checkpoint files. + + :param checkpoint_path: tensorflow checkpoint path + :return: dictionary of tensor. The key is the variable name and the value is the numpy + """ + reader = tf.train.NewCheckpointReader(checkpoint_path) + + # Get tensor name list + tensor_names = filter(lambda n: n!='global_step', + reader.get_variable_to_shape_map().keys()) + # Prepare key-value dictionary + tensors = {} + for tn in tensor_names: + tensors[tn] = reader.get_tensor(tn) + + return tensors + + +def save_variable_bigdl(tensors, target_path, bigdl_type="float"): + """ + Save a variable dictionary to a Java object file, so it can be read by BigDL + + :param tensors: tensor dictionary + :param target_path: where is the Java object file store + :param bigdl_type: model variable numeric type + :return: nothing + """ + jtensors = {} + for tn in tensors.keys(): + jtensors[tn] = JTensor.from_ndarray(tensors[tn]) + + callBigDlFunc(bigdl_type, "saveTensorDictionary", jtensors, target_path) + + +def dump_model(path, sess=None, graph=None, bigdl_type="float"): + """ + Dump a tensorflow model to files. The graph will be dumped to path/model.pb, and the checkpoint will + be dumped to path/model.bin + + :param path: dump folder path + :param sess: if user pass in session, we assume that the variable of the graph in the session + has been inited + :param graph: tensorflow graph. Default use the default graph of the session + :param bigdl_type: model variable numeric type + :return: nothing + """ + if not os.path.isdir(path): + print("Folder " + path + " does not exist") + raise + + if sess is None: + sess = tf.Session() + init = tf.global_variables_initializer() + sess.run(init) + + temp = tempfile.mkdtemp() + # dump checkpoint to temp files + checkpoint = temp + '/model.chkp' + saver = tf.train.Saver() + saver.save(sess, checkpoint) + + # generate bin files + tensors = export_checkpoint(checkpoint) + save_variable_bigdl(tensors, path + "/model.bin", bigdl_type) + + # dump grap to pb file + graph = sess.graph if graph is None else graph + tf.train.write_graph(graph, path, 'model.pb') + try: + shutil.rmtree(temp) + except OSError as e: + if e.errno != errno.ENOENT: + raise + + def merge_checkpoint(input_graph, checkpoint, output_node_names, @@ -98,4 +179,4 @@ def merge_checkpoint(input_graph, ) with gfile.GFile(output_graph, "wb") as f: - f.write(output_graph_def.SerializeToString()) \ No newline at end of file + f.write(output_graph_def.SerializeToString()) From 485996ce23ee396b3b6950dc9388e3ffd99d320f Mon Sep 17 00:00:00 2001 From: Ian Wong Date: Thu, 24 Aug 2017 13:59:13 +0800 Subject: [PATCH 214/823] Export tf checkpoint (#1498) * Support dump tensors from checkpoint and save as Java readable file * add unit test * meet code review --- .../dllib/src/bigdl/dllib/utils/tf_utils.py | 83 ++++++++++++++++++- python/dllib/src/bigdl/utils/common.py | 8 +- 2 files changed, 86 insertions(+), 5 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/utils/tf_utils.py b/python/dllib/src/bigdl/dllib/utils/tf_utils.py index e2a7dc44fc8..0e8696f2c62 100644 --- a/python/dllib/src/bigdl/dllib/utils/tf_utils.py +++ b/python/dllib/src/bigdl/dllib/utils/tf_utils.py @@ -27,6 +27,9 @@ from tensorflow.python.framework import importer from tensorflow.python.platform import gfile from bigdl.nn.layer import Model +from bigdl.util.common import JTensor +from bigdl.util.common import callBigDlFunc +import os def convert(input_ops, output_ops, byte_order, bigdl_type): """ @@ -63,6 +66,84 @@ def convert(input_ops, output_ops, byte_order, bigdl_type): return model + +def export_checkpoint(checkpoint_path): + """ + Export variable tensors from the checkpoint files. + + :param checkpoint_path: tensorflow checkpoint path + :return: dictionary of tensor. The key is the variable name and the value is the numpy + """ + reader = tf.train.NewCheckpointReader(checkpoint_path) + + # Get tensor name list + tensor_names = filter(lambda n: n!='global_step', + reader.get_variable_to_shape_map().keys()) + # Prepare key-value dictionary + tensors = {} + for tn in tensor_names: + tensors[tn] = reader.get_tensor(tn) + + return tensors + + +def save_variable_bigdl(tensors, target_path, bigdl_type="float"): + """ + Save a variable dictionary to a Java object file, so it can be read by BigDL + + :param tensors: tensor dictionary + :param target_path: where is the Java object file store + :param bigdl_type: model variable numeric type + :return: nothing + """ + jtensors = {} + for tn in tensors.keys(): + jtensors[tn] = JTensor.from_ndarray(tensors[tn]) + + callBigDlFunc(bigdl_type, "saveTensorDictionary", jtensors, target_path) + + +def dump_model(path, sess=None, graph=None, bigdl_type="float"): + """ + Dump a tensorflow model to files. The graph will be dumped to path/model.pb, and the checkpoint will + be dumped to path/model.bin + + :param path: dump folder path + :param sess: if user pass in session, we assume that the variable of the graph in the session + has been inited + :param graph: tensorflow graph. Default use the default graph of the session + :param bigdl_type: model variable numeric type + :return: nothing + """ + if not os.path.isdir(path): + print("Folder " + path + " does not exist") + raise + + if sess is None: + sess = tf.Session() + init = tf.global_variables_initializer() + sess.run(init) + + temp = tempfile.mkdtemp() + # dump checkpoint to temp files + checkpoint = temp + '/model.chkp' + saver = tf.train.Saver() + saver.save(sess, checkpoint) + + # generate bin files + tensors = export_checkpoint(checkpoint) + save_variable_bigdl(tensors, path + "/model.bin", bigdl_type) + + # dump grap to pb file + graph = sess.graph if graph is None else graph + tf.train.write_graph(graph, path, 'model.pb') + try: + shutil.rmtree(temp) + except OSError as e: + if e.errno != errno.ENOENT: + raise + + def merge_checkpoint(input_graph, checkpoint, output_node_names, @@ -98,4 +179,4 @@ def merge_checkpoint(input_graph, ) with gfile.GFile(output_graph, "wb") as f: - f.write(output_graph_def.SerializeToString()) \ No newline at end of file + f.write(output_graph_def.SerializeToString()) diff --git a/python/dllib/src/bigdl/utils/common.py b/python/dllib/src/bigdl/utils/common.py index c0b4d0063c9..4b3fb177da9 100644 --- a/python/dllib/src/bigdl/utils/common.py +++ b/python/dllib/src/bigdl/utils/common.py @@ -19,7 +19,7 @@ import glob from py4j.protocol import Py4JJavaError from py4j.java_gateway import JavaObject -from py4j.java_collections import ListConverter, JavaArray, JavaList, JavaMap +from py4j.java_collections import ListConverter, JavaArray, JavaList, JavaMap, MapConverter from pyspark import RDD, SparkContext from pyspark.serializers import PickleSerializer, AutoBatchedSerializer @@ -375,10 +375,10 @@ def _py2java(sc, obj): sc._gateway._gateway_client) elif isinstance(obj, dict): result = {} + print(obj.keys()) for (key, value) in obj.items(): - result[key] = _py2java(sc, value) if isinstance(value, JavaValue) else value # noqa - obj = result - + result[key] = _py2java(sc, value) + obj = MapConverter().convert(result, sc._gateway._gateway_client) elif isinstance(obj, JavaValue): obj = obj.value elif isinstance(obj, JavaObject): From 141a2bb215006e55a57a9c86db0b7436dcab1697 Mon Sep 17 00:00:00 2001 From: Ian Wong Date: Thu, 24 Aug 2017 13:59:13 +0800 Subject: [PATCH 215/823] Export tf checkpoint (#1498) * Support dump tensors from checkpoint and save as Java readable file * add unit test * meet code review --- python/dllib/test/bigdl/test_simple_integration.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/python/dllib/test/bigdl/test_simple_integration.py b/python/dllib/test/bigdl/test_simple_integration.py index 9a75ba40d53..0a98ef45750 100644 --- a/python/dllib/test/bigdl/test_simple_integration.py +++ b/python/dllib/test/bigdl/test_simple_integration.py @@ -19,6 +19,7 @@ from bigdl.nn.criterion import * from bigdl.optim.optimizer import * from bigdl.util.common import * +from bigdl.util.common import _py2java from bigdl.nn.initialization_method import * from bigdl.dataset import movielens import numpy as np @@ -385,6 +386,12 @@ def test_rng(self): for i in range(0, 2): assert_allclose(data[i], data2[i]) + def test_save_jtensor_dict(self): + tensors = {} + tensors["tensor1"] = JTensor.from_ndarray(np.random.rand(3, 2)) + tensors["tensor2"] = JTensor.from_ndarray(np.random.rand(3, 2)) + # in old impl, this will throw an exception + _py2java(self.sc, tensors) if __name__ == "__main__": pytest.main([__file__]) From b0740ef59f9a75197c496538e2a8d35ab6354e35 Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Thu, 24 Aug 2017 00:27:39 -0700 Subject: [PATCH 216/823] support NHWC format in SpatialAveragePooling (#1488) * support NHWC format in SpatialAveragePooling * fix style * meet code review --- python/dllib/src/bigdl/dllib/nn/layer.py | 44 ++++++++++++++++++++++-- 1 file changed, 42 insertions(+), 2 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 15754b1a890..272027b8a75 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -862,6 +862,18 @@ class SpatialMaxPooling(Layer): oheight = op((height + 2*padH - kH) / dH + 1) op is a rounding operator. By default, it is floor. It can be changed by calling :ceil() or :floor() methods. + + When padW and padH are both -1, we use a padding algorithm similar to the "SAME" + padding of tensorflow. That is + + outHeight = Math.ceil(inHeight.toFloat/strideH.toFloat) + outWidth = Math.ceil(inWidth.toFloat/strideW.toFloat) + + padAlongHeight = Math.max(0, (outHeight - 1) * strideH + kernelH - inHeight) + padAlongWidth = Math.max(0, (outWidth - 1) * strideW + kernelW - inWidth) + + padTop = padAlongHeight / 2 + padLeft = padAlongWidth / 2 :param kW: kernel width :param kH: kernel height @@ -1124,7 +1136,18 @@ class SpatialAveragePooling(Layer): ''' Applies 2D average-pooling operation in kWxkH regions by step size dWxdH steps. The number of output features is equal to the number of input planes. - + + When padW and padH are both -1, we use a padding algorithm similar to the "SAME" + padding of tensorflow. That is + + outHeight = Math.ceil(inHeight.toFloat/strideH.toFloat) + outWidth = Math.ceil(inWidth.toFloat/strideW.toFloat) + + padAlongHeight = Math.max(0, (outHeight - 1) * strideH + kernelH - inHeight) + padAlongWidth = Math.max(0, (outWidth - 1) * strideW + kernelW - inWidth) + + padTop = padAlongHeight / 2 + padLeft = padAlongWidth / 2 :param kW: kernel width :param kH: kernel height @@ -1137,10 +1160,13 @@ class SpatialAveragePooling(Layer): :param ceilMode: whether the output size is to be ceiled or floored :param countIncludePad: whether to include padding when dividing thenumber of elements in pooling region :param divide: whether to do the averaging + :param format: "NCHW" or "NHWC", indicating the input data format >>> spatialAveragePooling = SpatialAveragePooling(7,7) creating: createSpatialAveragePooling + >>> spatialAveragePooling = SpatialAveragePooling(2, 2, 2, 2, -1, -1, True, "NHWC") + creating: createSpatialAveragePooling ''' def __init__(self, @@ -1154,6 +1180,7 @@ def __init__(self, ceil_mode=False, count_include_pad=True, divide=True, + format="NCHW", bigdl_type="float"): super(SpatialAveragePooling, self).__init__(None, bigdl_type, kw, @@ -1165,7 +1192,8 @@ def __init__(self, global_pooling, ceil_mode, count_include_pad, - divide) + divide, + format) class SpatialBatchNormalization(Layer): @@ -3673,6 +3701,18 @@ class SpatialConvolutionMap(Layer): This class is a generalization of SpatialConvolution. It uses a generic connection table between input and output features. The SpatialConvolution is equivalent to using a full connection table. + + When padW and padH are both -1, we use a padding algorithm similar to the "SAME" + padding of tensorflow. That is + + outHeight = Math.ceil(inHeight.toFloat/strideH.toFloat) + outWidth = Math.ceil(inWidth.toFloat/strideW.toFloat) + + padAlongHeight = Math.max(0, (outHeight - 1) * strideH + kernelH - inHeight) + padAlongWidth = Math.max(0, (outWidth - 1) * strideW + kernelW - inWidth) + + padTop = padAlongHeight / 2 + padLeft = padAlongWidth / 2 :param wRegularizer: instance of [[Regularizer]](eg. L1 or L2 regularization), applied to the input weights matrices. :param bRegularizer: instance of [[Regularizer]]applied to the bias. From ee4d2cc227a58c28f005244f0f164cf394a3f088 Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Thu, 24 Aug 2017 00:27:39 -0700 Subject: [PATCH 217/823] support NHWC format in SpatialAveragePooling (#1488) * support NHWC format in SpatialAveragePooling * fix style * meet code review --- python/dllib/src/bigdl/dllib/nn/layer.py | 44 ++++++++++++++++++++++-- 1 file changed, 42 insertions(+), 2 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 15754b1a890..272027b8a75 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -862,6 +862,18 @@ class SpatialMaxPooling(Layer): oheight = op((height + 2*padH - kH) / dH + 1) op is a rounding operator. By default, it is floor. It can be changed by calling :ceil() or :floor() methods. + + When padW and padH are both -1, we use a padding algorithm similar to the "SAME" + padding of tensorflow. That is + + outHeight = Math.ceil(inHeight.toFloat/strideH.toFloat) + outWidth = Math.ceil(inWidth.toFloat/strideW.toFloat) + + padAlongHeight = Math.max(0, (outHeight - 1) * strideH + kernelH - inHeight) + padAlongWidth = Math.max(0, (outWidth - 1) * strideW + kernelW - inWidth) + + padTop = padAlongHeight / 2 + padLeft = padAlongWidth / 2 :param kW: kernel width :param kH: kernel height @@ -1124,7 +1136,18 @@ class SpatialAveragePooling(Layer): ''' Applies 2D average-pooling operation in kWxkH regions by step size dWxdH steps. The number of output features is equal to the number of input planes. - + + When padW and padH are both -1, we use a padding algorithm similar to the "SAME" + padding of tensorflow. That is + + outHeight = Math.ceil(inHeight.toFloat/strideH.toFloat) + outWidth = Math.ceil(inWidth.toFloat/strideW.toFloat) + + padAlongHeight = Math.max(0, (outHeight - 1) * strideH + kernelH - inHeight) + padAlongWidth = Math.max(0, (outWidth - 1) * strideW + kernelW - inWidth) + + padTop = padAlongHeight / 2 + padLeft = padAlongWidth / 2 :param kW: kernel width :param kH: kernel height @@ -1137,10 +1160,13 @@ class SpatialAveragePooling(Layer): :param ceilMode: whether the output size is to be ceiled or floored :param countIncludePad: whether to include padding when dividing thenumber of elements in pooling region :param divide: whether to do the averaging + :param format: "NCHW" or "NHWC", indicating the input data format >>> spatialAveragePooling = SpatialAveragePooling(7,7) creating: createSpatialAveragePooling + >>> spatialAveragePooling = SpatialAveragePooling(2, 2, 2, 2, -1, -1, True, "NHWC") + creating: createSpatialAveragePooling ''' def __init__(self, @@ -1154,6 +1180,7 @@ def __init__(self, ceil_mode=False, count_include_pad=True, divide=True, + format="NCHW", bigdl_type="float"): super(SpatialAveragePooling, self).__init__(None, bigdl_type, kw, @@ -1165,7 +1192,8 @@ def __init__(self, global_pooling, ceil_mode, count_include_pad, - divide) + divide, + format) class SpatialBatchNormalization(Layer): @@ -3673,6 +3701,18 @@ class SpatialConvolutionMap(Layer): This class is a generalization of SpatialConvolution. It uses a generic connection table between input and output features. The SpatialConvolution is equivalent to using a full connection table. + + When padW and padH are both -1, we use a padding algorithm similar to the "SAME" + padding of tensorflow. That is + + outHeight = Math.ceil(inHeight.toFloat/strideH.toFloat) + outWidth = Math.ceil(inWidth.toFloat/strideW.toFloat) + + padAlongHeight = Math.max(0, (outHeight - 1) * strideH + kernelH - inHeight) + padAlongWidth = Math.max(0, (outWidth - 1) * strideW + kernelW - inWidth) + + padTop = padAlongHeight / 2 + padLeft = padAlongWidth / 2 :param wRegularizer: instance of [[Regularizer]](eg. L1 or L2 regularization), applied to the input weights matrices. :param bRegularizer: instance of [[Regularizer]]applied to the bias. From 8d24225726343c0a039223ea578c83d0714e0d2b Mon Sep 17 00:00:00 2001 From: Jerry Wu Date: Fri, 25 Aug 2017 08:25:40 +0800 Subject: [PATCH 218/823] Serialization - version control & python API (#1491) * add version control * version support * add python api * add version control proto * add unit test * change function name * rename function api * per comments * refinement to make hierachy more cleaner * backward compatibility --- python/dllib/src/bigdl/dllib/nn/layer.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 272027b8a75..2e7fc455e4a 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -338,6 +338,9 @@ def get_weights(self): def save(self, path, over_write = False): callBigDlFunc(self.bigdl_type, "modelSave", self.value, path, over_write) + def saveModel(self, path, over_write = False): + callBigDlFunc(self.bigdl_type, "saveBigDLModule", self.value, path, + over_write) def save_caffe(self, prototxt_path, model_path, use_v2 = True, overwrite = False): callBigDlFunc(self.bigdl_type, "saveCaffe", self.value, prototxt_path, @@ -459,6 +462,17 @@ def load(path, bigdl_type="float"): jmodel = callBigDlFunc(bigdl_type, "loadBigDL", path) return Layer.of(jmodel) + @staticmethod + def loadModel(path, bigdl_type="float"): + """ + Load a pre-trained Bigdl model. + + :param path: The path containing the pre-trained model. + :return: A pre-trained model. + """ + jmodel = callBigDlFunc(bigdl_type, "loadBigDLModule", path) + return Layer.of(jmodel) + @staticmethod def load_torch(path, bigdl_type="float"): """ From 8771934df056a9b4b1131350856368ddc935f306 Mon Sep 17 00:00:00 2001 From: Jerry Wu Date: Fri, 25 Aug 2017 08:25:40 +0800 Subject: [PATCH 219/823] Serialization - version control & python API (#1491) * add version control * version support * add python api * add version control proto * add unit test * change function name * rename function api * per comments * refinement to make hierachy more cleaner * backward compatibility --- python/dllib/src/bigdl/dllib/nn/layer.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 272027b8a75..2e7fc455e4a 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -338,6 +338,9 @@ def get_weights(self): def save(self, path, over_write = False): callBigDlFunc(self.bigdl_type, "modelSave", self.value, path, over_write) + def saveModel(self, path, over_write = False): + callBigDlFunc(self.bigdl_type, "saveBigDLModule", self.value, path, + over_write) def save_caffe(self, prototxt_path, model_path, use_v2 = True, overwrite = False): callBigDlFunc(self.bigdl_type, "saveCaffe", self.value, prototxt_path, @@ -459,6 +462,17 @@ def load(path, bigdl_type="float"): jmodel = callBigDlFunc(bigdl_type, "loadBigDL", path) return Layer.of(jmodel) + @staticmethod + def loadModel(path, bigdl_type="float"): + """ + Load a pre-trained Bigdl model. + + :param path: The path containing the pre-trained model. + :return: A pre-trained model. + """ + jmodel = callBigDlFunc(bigdl_type, "loadBigDLModule", path) + return Layer.of(jmodel) + @staticmethod def load_torch(path, bigdl_type="float"): """ From faf4e0583e25de650ba317dcddf244e10c1b3047 Mon Sep 17 00:00:00 2001 From: Jerry Wu Date: Fri, 25 Aug 2017 08:25:40 +0800 Subject: [PATCH 220/823] Serialization - version control & python API (#1491) * add version control * version support * add python api * add version control proto * add unit test * change function name * rename function api * per comments * refinement to make hierachy more cleaner * backward compatibility --- python/dllib/test/bigdl/test_simple_integration.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/python/dllib/test/bigdl/test_simple_integration.py b/python/dllib/test/bigdl/test_simple_integration.py index 0a98ef45750..1a99af5da3e 100644 --- a/python/dllib/test/bigdl/test_simple_integration.py +++ b/python/dllib/test/bigdl/test_simple_integration.py @@ -80,6 +80,15 @@ def test_load_model(self): assert_allclose(fc1_loaded.get_weights()[0], fc1.get_weights()[0]) + def test_load_model_proto(self): + fc1 = Linear(4, 2) + fc1.set_weights([np.ones((4, 2)), np.ones((2,))]) + tmp_path = tempfile.mktemp() + fc1.saveModel(tmp_path, True) + fc1_loaded = Model.loadModel(tmp_path) + assert_allclose(fc1_loaded.get_weights()[0], + fc1.get_weights()[0]) + def test_load_optim_method(self): FEATURES_DIM = 2 data_len = 100 From 794a7dff79e220dc3cae1abf5e9e08ed399c5912 Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Fri, 25 Aug 2017 13:39:10 +0800 Subject: [PATCH 221/823] fix python unittest (#1508) --- python/dllib/src/bigdl/dllib/nn/layer.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 2e7fc455e4a..38d12059d90 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -311,7 +311,7 @@ def set_weights(self, weights): ... except Py4JJavaError as err: ... print(err.java_exception) ... - java.lang.IllegalArgumentException: requirement failed: the number of input weight/bias is not consistant with number of weight/bias of this layer + java.lang.IllegalArgumentException: requirement failed: the number of input weight/bias is not consistant with number of weight/bias of this layer, number of input 1, number of output 2 >>> cAdd = CAdd([4, 1]) creating: createCAdd >>> cAdd.set_weights(np.ones([4, 1])) @@ -1179,7 +1179,7 @@ class SpatialAveragePooling(Layer): >>> spatialAveragePooling = SpatialAveragePooling(7,7) creating: createSpatialAveragePooling - >>> spatialAveragePooling = SpatialAveragePooling(2, 2, 2, 2, -1, -1, True, "NHWC") + >>> spatialAveragePooling = SpatialAveragePooling(2, 2, 2, 2, -1, -1, True, format="NHWC") creating: createSpatialAveragePooling ''' @@ -1209,6 +1209,9 @@ def __init__(self, divide, format) + def set_weights(self, weights): + super(SpatialAveragePooling, self).set_weights(weights) + class SpatialBatchNormalization(Layer): From 64a49e64c3028ce3e287a351e2231dcee0e5e9b2 Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Fri, 25 Aug 2017 13:39:10 +0800 Subject: [PATCH 222/823] fix python unittest (#1508) --- python/dllib/src/bigdl/dllib/nn/layer.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 2e7fc455e4a..38d12059d90 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -311,7 +311,7 @@ def set_weights(self, weights): ... except Py4JJavaError as err: ... print(err.java_exception) ... - java.lang.IllegalArgumentException: requirement failed: the number of input weight/bias is not consistant with number of weight/bias of this layer + java.lang.IllegalArgumentException: requirement failed: the number of input weight/bias is not consistant with number of weight/bias of this layer, number of input 1, number of output 2 >>> cAdd = CAdd([4, 1]) creating: createCAdd >>> cAdd.set_weights(np.ones([4, 1])) @@ -1179,7 +1179,7 @@ class SpatialAveragePooling(Layer): >>> spatialAveragePooling = SpatialAveragePooling(7,7) creating: createSpatialAveragePooling - >>> spatialAveragePooling = SpatialAveragePooling(2, 2, 2, 2, -1, -1, True, "NHWC") + >>> spatialAveragePooling = SpatialAveragePooling(2, 2, 2, 2, -1, -1, True, format="NHWC") creating: createSpatialAveragePooling ''' @@ -1209,6 +1209,9 @@ def __init__(self, divide, format) + def set_weights(self, weights): + super(SpatialAveragePooling, self).set_weights(weights) + class SpatialBatchNormalization(Layer): From 26446c5025b3e82a42e6a351002cf9eee21ea457 Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Fri, 25 Aug 2017 13:39:10 +0800 Subject: [PATCH 223/823] fix python unittest (#1508) --- python/dllib/test/dev/run-tests | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/python/dllib/test/dev/run-tests b/python/dllib/test/dev/run-tests index 39def31a75a..471651e28a9 100755 --- a/python/dllib/test/dev/run-tests +++ b/python/dllib/test/dev/run-tests @@ -31,8 +31,11 @@ do $p -m pytest -v --doctest-modules ../../../pyspark/bigdl \ --ignore=../../../pyspark/bigdl/dataset/ \ --ignore=../../../pyspark/bigdl/util/ \ - --ignore=../../../pyspark/bigdl/models/ - + --ignore=../../../pyspark/bigdl/models/ && \ $p -m pytest -v -n 4 ../../../pyspark/test/ + if [ $? -ne 0 ]; + then + exit $? + fi done From bfe951208235729b500fe8a0f49dc54e551b1ecf Mon Sep 17 00:00:00 2001 From: Yan Wan Date: Mon, 28 Aug 2017 14:58:45 +0800 Subject: [PATCH 224/823] BifurcateSplitTable (#1513) * add BifurcateSplitTable * createBifurcateSplitTable * add buffer * clearState --- python/dllib/src/bigdl/dllib/nn/layer.py | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 38d12059d90..6d5fec21d97 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -1484,7 +1484,29 @@ def set_init_method(self, weight_init_method = None, bias_init_method = None): return self -class Bilinear(Layer): +class BifurcateSplitTable(Model): + ''' + Creates a module that takes a Tensor as input and + outputs two tables, splitting the Tensor along + the specified dimension `dimension`. + + The input to this layer is expected to be a tensor, or a batch of tensors; + + :param dimension to be split along this dimension + :param T Numeric type. Only support float/double now + + >>> bifurcateSplitTable = BifurcateSplitTable(1) + creating: createBifurcateSplitTable + ''' + + def __init__(self, + dimension, + bigdl_type="float"): + super(BifurcateSplitTable, self).__init__(None, bigdl_type, + dimension) + + +class Bilinear(Model): ''' a bilinear transformation with sparse inputs, From 1138e204ee42a1ad340da9232d86b0d1d92f89c5 Mon Sep 17 00:00:00 2001 From: Yan Wan Date: Mon, 28 Aug 2017 14:58:45 +0800 Subject: [PATCH 225/823] BifurcateSplitTable (#1513) * add BifurcateSplitTable * createBifurcateSplitTable * add buffer * clearState --- python/dllib/src/bigdl/dllib/nn/layer.py | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 38d12059d90..6d5fec21d97 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -1484,7 +1484,29 @@ def set_init_method(self, weight_init_method = None, bias_init_method = None): return self -class Bilinear(Layer): +class BifurcateSplitTable(Model): + ''' + Creates a module that takes a Tensor as input and + outputs two tables, splitting the Tensor along + the specified dimension `dimension`. + + The input to this layer is expected to be a tensor, or a batch of tensors; + + :param dimension to be split along this dimension + :param T Numeric type. Only support float/double now + + >>> bifurcateSplitTable = BifurcateSplitTable(1) + creating: createBifurcateSplitTable + ''' + + def __init__(self, + dimension, + bigdl_type="float"): + super(BifurcateSplitTable, self).__init__(None, bigdl_type, + dimension) + + +class Bilinear(Model): ''' a bilinear transformation with sparse inputs, From 0952edaf37a17e9eaef315fd939666df16b71aa7 Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Tue, 29 Aug 2017 10:49:49 +0800 Subject: [PATCH 226/823] Refactor the serde logic of JTensor and Sample (#1510) * update * fix * add bigdltype * update comment --- python/dllib/src/bigdl/dllib/utils/common.py | 106 +++++++++---------- 1 file changed, 53 insertions(+), 53 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/utils/common.py b/python/dllib/src/bigdl/dllib/utils/common.py index 4b3fb177da9..82e40e0cad3 100644 --- a/python/dllib/src/bigdl/dllib/utils/common.py +++ b/python/dllib/src/bigdl/dllib/utils/common.py @@ -113,20 +113,26 @@ def __str__(self): return "Evaluated result: %s, total_num: %s, method: %s" % ( self.result, self.total_num, self.method) +def get_dtype(bigdl_type): + # Always return float32 for now + return "float32" class JTensor(object): """ A wrapper to easy our work when need to pass or return Tensor to/from Scala. - >>> import numpy as np >>> from bigdl.util.common import JTensor >>> np.random.seed(123) >>> """ def __init__(self, storage, shape, bigdl_type="float"): - self.storage = storage - self.shape = shape + if isinstance(storage, bytes) and isinstance(shape, bytes): + self.storage = np.frombuffer(storage, dtype=get_dtype(bigdl_type)) + self.shape = np.frombuffer(shape, dtype=np.int32) + else: + self.storage = np.array(storage, dtype=get_dtype(bigdl_type)) + self.shape = np.array(shape, dtype=np.int32) self.bigdl_type = bigdl_type @classmethod @@ -148,75 +154,69 @@ def from_ndarray(cls, a_ndarray, bigdl_type="float"): >>> (array_from_tensor == data).all() True """ - return cls(*JTensor.flatten_ndarray(a_ndarray), - bigdl_type= bigdl_type) if a_ndarray is not None else None # noqa + if a_ndarray is None: + return None + assert isinstance(a_ndarray, np.ndarray), \ + "input should be a np.ndarray, not %s" % type(a_ndarray) + return cls(a_ndarray, + a_ndarray.shape if a_ndarray.shape else (a_ndarray.size), + bigdl_type= bigdl_type) def to_ndarray(self): - def get_dtype(): - if "float" == self.bigdl_type: - return "float32" - else: - return "float64" - return np.array(self.storage, dtype=get_dtype()).reshape(self.shape) # noqa - - @classmethod - def flatten_ndarray(cls, a_ndarray): - """ - Utility method to flatten a ndarray - - :return: (storage, shape) - - >>> from bigdl.util.common import JTensor - >>> np.random.seed(123) - >>> data = np.random.uniform(0, 1, (2, 3)) - >>> (storage, shape) = JTensor.flatten_ndarray(data) - >>> shape - [2, 3] - >>> (storage, shape) = JTensor.flatten_ndarray(np.array(2)) - >>> shape - [1] - """ - storage = [float(i) for i in a_ndarray.ravel()] - shape = list(a_ndarray.shape) if a_ndarray.shape else [a_ndarray.size] - return storage, shape + return np.array(self.storage, dtype=get_dtype(self.bigdl_type)).reshape(self.shape) # noqa def __reduce__(self): - return (JTensor, (self.storage, self.shape, self.bigdl_type)) + return JTensor, (self.storage.tostring(), self.shape.tostring(), self.bigdl_type) def __str__(self): - return "storage: %s, shape: %s," % (self.storage, self.storage) + return "JTensor: storage: %s, shape: %s" % (self.storage, self.shape) + + def __repr__(self): + return "JTensor: storage: %s, shape: %s" % (self.storage, self.shape) class Sample(object): - def __init__(self, features, label, features_shape, label_shape, - bigdl_type="float"): - def get_dtype(): - if "float" == bigdl_type: - return "float32" - else: - return "float64" - self.features = np.array(features, dtype=get_dtype()).reshape(features_shape) # noqa - self.label = np.array(label, dtype=get_dtype()).reshape(label_shape) + def __init__(self, features, label, bigdl_type="float"): + """ + User should always use Sample.from_ndarray to construct Sample. + :param features: a JTensor + :param label: a JTensor + :param bigdl_type: "double" or "float" + """ + self.features = features + self.label = label self.bigdl_type = bigdl_type @classmethod def from_ndarray(cls, features, label, bigdl_type="float"): + """ + Convert a ndarray of features and label to Sample, which would be used in Java side. + + >>> import numpy as np + >>> from bigdl.util.common import callBigDlFunc + >>> from numpy.testing import assert_allclose + >>> sample = Sample.from_ndarray(np.random.random((2,3)), np.random.random((2,3))) + >>> sample_back = callBigDlFunc("float", "testSample", sample) + >>> assert_allclose(sample.features.to_ndarray(), sample_back.features.to_ndarray()) + >>> assert_allclose(sample.label.to_ndarray(), sample_back.label.to_ndarray()) + """ + assert isinstance(features, np.ndarray), \ + "features should be a np.ndarray, not %s" % type(features) + if not isinstance(label, np.ndarray): # in case label is a scalar. + label = np.array(label) return cls( - features=[float(i) for i in features.ravel()], - label=[float(i) for i in label.ravel()], - features_shape=list(features.shape), - label_shape=list(label.shape) if label.shape else [label.size], + features=JTensor.from_ndarray(features), + label=JTensor.from_ndarray(label), bigdl_type=bigdl_type) def __reduce__(self): - (features_storage, features_shape) = JTensor.flatten_ndarray(self.features) - (label_storage, label_shape) = JTensor.flatten_ndarray(self.label) - return (Sample, ( - features_storage, label_storage, features_shape, label_shape, - self.bigdl_type)) + return Sample, (self.features, self.label, self.bigdl_type) def __str__(self): - return "features: %s, label: %s," % (self.features, self.label) + return "Sample: features: %s, label: %s," % (self.features, self.label) + + def __repr__(self): + return "Sample: features: %s, label: %s" % (self.storage, self.shape) class RNG(): """ From 56ab7bdda1ec989e6e0d6ee040d4518f712b0588 Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Tue, 29 Aug 2017 10:49:49 +0800 Subject: [PATCH 227/823] Refactor the serde logic of JTensor and Sample (#1510) * update * fix * add bigdltype * update comment --- python/dllib/src/bigdl/utils/common.py | 106 ++++++++++++------------- 1 file changed, 53 insertions(+), 53 deletions(-) diff --git a/python/dllib/src/bigdl/utils/common.py b/python/dllib/src/bigdl/utils/common.py index 4b3fb177da9..82e40e0cad3 100644 --- a/python/dllib/src/bigdl/utils/common.py +++ b/python/dllib/src/bigdl/utils/common.py @@ -113,20 +113,26 @@ def __str__(self): return "Evaluated result: %s, total_num: %s, method: %s" % ( self.result, self.total_num, self.method) +def get_dtype(bigdl_type): + # Always return float32 for now + return "float32" class JTensor(object): """ A wrapper to easy our work when need to pass or return Tensor to/from Scala. - >>> import numpy as np >>> from bigdl.util.common import JTensor >>> np.random.seed(123) >>> """ def __init__(self, storage, shape, bigdl_type="float"): - self.storage = storage - self.shape = shape + if isinstance(storage, bytes) and isinstance(shape, bytes): + self.storage = np.frombuffer(storage, dtype=get_dtype(bigdl_type)) + self.shape = np.frombuffer(shape, dtype=np.int32) + else: + self.storage = np.array(storage, dtype=get_dtype(bigdl_type)) + self.shape = np.array(shape, dtype=np.int32) self.bigdl_type = bigdl_type @classmethod @@ -148,75 +154,69 @@ def from_ndarray(cls, a_ndarray, bigdl_type="float"): >>> (array_from_tensor == data).all() True """ - return cls(*JTensor.flatten_ndarray(a_ndarray), - bigdl_type= bigdl_type) if a_ndarray is not None else None # noqa + if a_ndarray is None: + return None + assert isinstance(a_ndarray, np.ndarray), \ + "input should be a np.ndarray, not %s" % type(a_ndarray) + return cls(a_ndarray, + a_ndarray.shape if a_ndarray.shape else (a_ndarray.size), + bigdl_type= bigdl_type) def to_ndarray(self): - def get_dtype(): - if "float" == self.bigdl_type: - return "float32" - else: - return "float64" - return np.array(self.storage, dtype=get_dtype()).reshape(self.shape) # noqa - - @classmethod - def flatten_ndarray(cls, a_ndarray): - """ - Utility method to flatten a ndarray - - :return: (storage, shape) - - >>> from bigdl.util.common import JTensor - >>> np.random.seed(123) - >>> data = np.random.uniform(0, 1, (2, 3)) - >>> (storage, shape) = JTensor.flatten_ndarray(data) - >>> shape - [2, 3] - >>> (storage, shape) = JTensor.flatten_ndarray(np.array(2)) - >>> shape - [1] - """ - storage = [float(i) for i in a_ndarray.ravel()] - shape = list(a_ndarray.shape) if a_ndarray.shape else [a_ndarray.size] - return storage, shape + return np.array(self.storage, dtype=get_dtype(self.bigdl_type)).reshape(self.shape) # noqa def __reduce__(self): - return (JTensor, (self.storage, self.shape, self.bigdl_type)) + return JTensor, (self.storage.tostring(), self.shape.tostring(), self.bigdl_type) def __str__(self): - return "storage: %s, shape: %s," % (self.storage, self.storage) + return "JTensor: storage: %s, shape: %s" % (self.storage, self.shape) + + def __repr__(self): + return "JTensor: storage: %s, shape: %s" % (self.storage, self.shape) class Sample(object): - def __init__(self, features, label, features_shape, label_shape, - bigdl_type="float"): - def get_dtype(): - if "float" == bigdl_type: - return "float32" - else: - return "float64" - self.features = np.array(features, dtype=get_dtype()).reshape(features_shape) # noqa - self.label = np.array(label, dtype=get_dtype()).reshape(label_shape) + def __init__(self, features, label, bigdl_type="float"): + """ + User should always use Sample.from_ndarray to construct Sample. + :param features: a JTensor + :param label: a JTensor + :param bigdl_type: "double" or "float" + """ + self.features = features + self.label = label self.bigdl_type = bigdl_type @classmethod def from_ndarray(cls, features, label, bigdl_type="float"): + """ + Convert a ndarray of features and label to Sample, which would be used in Java side. + + >>> import numpy as np + >>> from bigdl.util.common import callBigDlFunc + >>> from numpy.testing import assert_allclose + >>> sample = Sample.from_ndarray(np.random.random((2,3)), np.random.random((2,3))) + >>> sample_back = callBigDlFunc("float", "testSample", sample) + >>> assert_allclose(sample.features.to_ndarray(), sample_back.features.to_ndarray()) + >>> assert_allclose(sample.label.to_ndarray(), sample_back.label.to_ndarray()) + """ + assert isinstance(features, np.ndarray), \ + "features should be a np.ndarray, not %s" % type(features) + if not isinstance(label, np.ndarray): # in case label is a scalar. + label = np.array(label) return cls( - features=[float(i) for i in features.ravel()], - label=[float(i) for i in label.ravel()], - features_shape=list(features.shape), - label_shape=list(label.shape) if label.shape else [label.size], + features=JTensor.from_ndarray(features), + label=JTensor.from_ndarray(label), bigdl_type=bigdl_type) def __reduce__(self): - (features_storage, features_shape) = JTensor.flatten_ndarray(self.features) - (label_storage, label_shape) = JTensor.flatten_ndarray(self.label) - return (Sample, ( - features_storage, label_storage, features_shape, label_shape, - self.bigdl_type)) + return Sample, (self.features, self.label, self.bigdl_type) def __str__(self): - return "features: %s, label: %s," % (self.features, self.label) + return "Sample: features: %s, label: %s," % (self.features, self.label) + + def __repr__(self): + return "Sample: features: %s, label: %s" % (self.storage, self.shape) class RNG(): """ From 90eba1e7a8c782724a8f437394ce2e561056bd95 Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Tue, 29 Aug 2017 10:49:49 +0800 Subject: [PATCH 228/823] Refactor the serde logic of JTensor and Sample (#1510) * update * fix * add bigdltype * update comment --- python/dllib/test/bigdl/test_simple_integration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/dllib/test/bigdl/test_simple_integration.py b/python/dllib/test/bigdl/test_simple_integration.py index 1a99af5da3e..51f2a1a2b15 100644 --- a/python/dllib/test/bigdl/test_simple_integration.py +++ b/python/dllib/test/bigdl/test_simple_integration.py @@ -189,7 +189,7 @@ def test_simple_flow(self): def gen_rand_sample(): features = np.random.uniform(0, 1, (FEATURES_DIM)) - label = (2 * features).sum() + 0.4 + label = np.array((2 * features).sum() + 0.4) return Sample.from_ndarray(features, label) trainingData = self.sc.parallelize(range(0, data_len)).map( From d05f293b788970b5fd6cefb91922833a700c2b31 Mon Sep 17 00:00:00 2001 From: Yan Wan Date: Wed, 30 Aug 2017 11:20:08 +0800 Subject: [PATCH 229/823] add isInputWithBias and isHiddenWithBias for RNNCell (#1515) * add isInputWithBias and isHiddenWithBias for RNNCell * default affine = true in BatchNormParams in recurrent layer * add serializableSpec for recurrent --- python/dllib/src/bigdl/dllib/nn/layer.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 6d5fec21d97..7402fad7b27 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -1069,12 +1069,15 @@ class RnnCell(Layer): :param input_size: the size of each input vector :param hidden_size: Hidden unit size in simple RNN :param activation: activation function + :param isInputWithBias: boolean + :param isHiddenWithBias: boolean + :param wRegularizer: instance of [[Regularizer]](eg. L1 or L2 regularization), applied to the input weights matrices. :param uRegularizer: instance [[Regularizer]](eg. L1 or L2 regularization), applied to the recurrent weights matrices. :param bRegularizer: instance of [[Regularizer]](../regularizers.md),applied to the bias. - >>> reshape = RnnCell(4, 3, Tanh(), L1Regularizer(0.5), L1Regularizer(0.5), L1Regularizer(0.5)) + >>> reshape = RnnCell(4, 3, Tanh(), True, True, L1Regularizer(0.5), L1Regularizer(0.5), L1Regularizer(0.5)) creating: createTanh creating: createL1Regularizer creating: createL1Regularizer From ae7a5a80795521a20508e871cd7b69cb920c0ef0 Mon Sep 17 00:00:00 2001 From: Yan Wan Date: Wed, 30 Aug 2017 11:20:08 +0800 Subject: [PATCH 230/823] add isInputWithBias and isHiddenWithBias for RNNCell (#1515) * add isInputWithBias and isHiddenWithBias for RNNCell * default affine = true in BatchNormParams in recurrent layer * add serializableSpec for recurrent --- python/dllib/src/bigdl/dllib/nn/layer.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 6d5fec21d97..7402fad7b27 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -1069,12 +1069,15 @@ class RnnCell(Layer): :param input_size: the size of each input vector :param hidden_size: Hidden unit size in simple RNN :param activation: activation function + :param isInputWithBias: boolean + :param isHiddenWithBias: boolean + :param wRegularizer: instance of [[Regularizer]](eg. L1 or L2 regularization), applied to the input weights matrices. :param uRegularizer: instance [[Regularizer]](eg. L1 or L2 regularization), applied to the recurrent weights matrices. :param bRegularizer: instance of [[Regularizer]](../regularizers.md),applied to the bias. - >>> reshape = RnnCell(4, 3, Tanh(), L1Regularizer(0.5), L1Regularizer(0.5), L1Regularizer(0.5)) + >>> reshape = RnnCell(4, 3, Tanh(), True, True, L1Regularizer(0.5), L1Regularizer(0.5), L1Regularizer(0.5)) creating: createTanh creating: createL1Regularizer creating: createL1Regularizer From 8b5f56190f88d6e415729558b1459a26ab1f8532 Mon Sep 17 00:00:00 2001 From: Yao Zhang Date: Fri, 1 Sep 2017 15:29:46 +0800 Subject: [PATCH 231/823] 1D Max Pooling (#1436) * finish 1D pooling * finish documentation * meet code review * finish 1d pooling * meet code review * fix scala style errors --- python/dllib/src/bigdl/dllib/nn/layer.py | 26 ++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 7402fad7b27..3026e227d32 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -865,6 +865,32 @@ def set_init_method(self, weight_init_method = None, bias_init_method = None): return self +class TemporalMaxPooling(Layer): + + ''' + Applies 1D max-pooling operation in kW regions by step size dW steps. + Input sequence composed of nInputFrame frames. + The input tensor in forward(input) is expected to be a 2D tensor (nInputFrame x inputFrameSize) + or a 3D tensor (nBatchFrame x nInputFrame x inputFrameSize). + + If the input sequence is a 2D tensor of dimension nInputFrame x inputFrameSize, + the output sequence will be nOutputFrame x inputFrameSize where + + nOutputFrame = (nInputFrame - k_w) / d_w + 1 + + :param k_w: kernel width + :param d_w: step size in width + + >>> temporalMaxPooling = TemporalMaxPooling(2, 2) + creating: createTemporalMaxPooling + ''' + + def __init__(self, + k_w, + d_w, + bigdl_type="float"): + super(TemporalMaxPooling, self).__init__(None, bigdl_type, k_w, + d_w) class SpatialMaxPooling(Layer): ''' From 2a4574dfb1198d94937e37b3bd6ddde908e4807a Mon Sep 17 00:00:00 2001 From: Yao Zhang Date: Fri, 1 Sep 2017 15:29:46 +0800 Subject: [PATCH 232/823] 1D Max Pooling (#1436) * finish 1D pooling * finish documentation * meet code review * finish 1d pooling * meet code review * fix scala style errors --- python/dllib/src/bigdl/dllib/nn/layer.py | 26 ++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 7402fad7b27..3026e227d32 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -865,6 +865,32 @@ def set_init_method(self, weight_init_method = None, bias_init_method = None): return self +class TemporalMaxPooling(Layer): + + ''' + Applies 1D max-pooling operation in kW regions by step size dW steps. + Input sequence composed of nInputFrame frames. + The input tensor in forward(input) is expected to be a 2D tensor (nInputFrame x inputFrameSize) + or a 3D tensor (nBatchFrame x nInputFrame x inputFrameSize). + + If the input sequence is a 2D tensor of dimension nInputFrame x inputFrameSize, + the output sequence will be nOutputFrame x inputFrameSize where + + nOutputFrame = (nInputFrame - k_w) / d_w + 1 + + :param k_w: kernel width + :param d_w: step size in width + + >>> temporalMaxPooling = TemporalMaxPooling(2, 2) + creating: createTemporalMaxPooling + ''' + + def __init__(self, + k_w, + d_w, + bigdl_type="float"): + super(TemporalMaxPooling, self).__init__(None, bigdl_type, k_w, + d_w) class SpatialMaxPooling(Layer): ''' From dd7ef201a6df6424923a1da2db716b91d1cd7ca9 Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Wed, 6 Sep 2017 16:31:56 +0800 Subject: [PATCH 233/823] fix sample (#1537) --- .../dllib/feature/dataset/transformer.py | 6 +++--- .../src/bigdl/dllib/models/lenet/lenet5.py | 20 ++++++++++--------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/feature/dataset/transformer.py b/python/dllib/src/bigdl/dllib/feature/dataset/transformer.py index 018d93c13c6..8af69d8c220 100644 --- a/python/dllib/src/bigdl/dllib/feature/dataset/transformer.py +++ b/python/dllib/src/bigdl/dllib/feature/dataset/transformer.py @@ -18,9 +18,9 @@ from bigdl.util.common import Sample -def normalizer(mean, std): +def normalizer(data, mean, std): """ Normalize features by standard deviation + data is a ndarray """ - return lambda sample: Sample.from_ndarray((sample.features - mean) / std, - sample.label, sample.bigdl_type) + return (data - mean) / std diff --git a/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py b/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py index 2544eb8c98c..70e4e417c13 100644 --- a/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py +++ b/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py @@ -48,14 +48,12 @@ def get_mnist(sc, data_type="train", location="/tmp/mnist"): :param sc: SparkContext :param data_type: training data or testing data :param location: Location storing the mnist - :return: A RDD of Sample + :return: A RDD of (features: Ndarray, label: Ndarray) """ (images, labels) = mnist.read_data_sets(location, data_type) images = sc.parallelize(images) - labels = sc.parallelize(labels) - # Target start from 1 in BigDL - record = images.zip(labels).map(lambda features_label: - Sample.from_ndarray(features_label[0], features_label[1] + 1)) + labels = sc.parallelize(labels + 1) # Target start from 1 in BigDL + record = images.zip(labels) return record @@ -80,10 +78,14 @@ def get_end_trigger(): else: return MaxIteration(options.endTriggerNum) - train_data = get_mnist(sc, "train").map( - normalizer(mnist.TRAIN_MEAN, mnist.TRAIN_STD)) - test_data = get_mnist(sc, "test").map( - normalizer(mnist.TEST_MEAN, mnist.TEST_STD)) + train_data = get_mnist(sc, "train")\ + .map(lambda rec_tuple: (normalizer(rec_tuple[0], mnist.TRAIN_MEAN, mnist.TRAIN_STD), + rec_tuple[1]))\ + .map(lambda t: Sample.from_ndarray(t[0], t[1])) + test_data = get_mnist(sc, "test")\ + .map(lambda rec_tuple: (normalizer(rec_tuple[0], mnist.TEST_MEAN, mnist.TEST_STD), + rec_tuple[1]))\ + .map(lambda t: Sample.from_ndarray(t[0], t[1])) optimizer = Optimizer( model=build_model(10), From 941067067a61f4cb882057f9bbca41f5572af7e4 Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Wed, 6 Sep 2017 16:31:56 +0800 Subject: [PATCH 234/823] fix sample (#1537) --- .../dllib/feature/dataset/transformer.py | 6 +++--- .../src/bigdl/dllib/models/lenet/lenet5.py | 20 ++++++++++--------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/feature/dataset/transformer.py b/python/dllib/src/bigdl/dllib/feature/dataset/transformer.py index 018d93c13c6..8af69d8c220 100644 --- a/python/dllib/src/bigdl/dllib/feature/dataset/transformer.py +++ b/python/dllib/src/bigdl/dllib/feature/dataset/transformer.py @@ -18,9 +18,9 @@ from bigdl.util.common import Sample -def normalizer(mean, std): +def normalizer(data, mean, std): """ Normalize features by standard deviation + data is a ndarray """ - return lambda sample: Sample.from_ndarray((sample.features - mean) / std, - sample.label, sample.bigdl_type) + return (data - mean) / std diff --git a/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py b/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py index 2544eb8c98c..70e4e417c13 100644 --- a/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py +++ b/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py @@ -48,14 +48,12 @@ def get_mnist(sc, data_type="train", location="/tmp/mnist"): :param sc: SparkContext :param data_type: training data or testing data :param location: Location storing the mnist - :return: A RDD of Sample + :return: A RDD of (features: Ndarray, label: Ndarray) """ (images, labels) = mnist.read_data_sets(location, data_type) images = sc.parallelize(images) - labels = sc.parallelize(labels) - # Target start from 1 in BigDL - record = images.zip(labels).map(lambda features_label: - Sample.from_ndarray(features_label[0], features_label[1] + 1)) + labels = sc.parallelize(labels + 1) # Target start from 1 in BigDL + record = images.zip(labels) return record @@ -80,10 +78,14 @@ def get_end_trigger(): else: return MaxIteration(options.endTriggerNum) - train_data = get_mnist(sc, "train").map( - normalizer(mnist.TRAIN_MEAN, mnist.TRAIN_STD)) - test_data = get_mnist(sc, "test").map( - normalizer(mnist.TEST_MEAN, mnist.TEST_STD)) + train_data = get_mnist(sc, "train")\ + .map(lambda rec_tuple: (normalizer(rec_tuple[0], mnist.TRAIN_MEAN, mnist.TRAIN_STD), + rec_tuple[1]))\ + .map(lambda t: Sample.from_ndarray(t[0], t[1])) + test_data = get_mnist(sc, "test")\ + .map(lambda rec_tuple: (normalizer(rec_tuple[0], mnist.TEST_MEAN, mnist.TEST_STD), + rec_tuple[1]))\ + .map(lambda t: Sample.from_ndarray(t[0], t[1])) optimizer = Optimizer( model=build_model(10), From 21ed49fbaa2432e6046a20faf0afa2c75ad50477 Mon Sep 17 00:00:00 2001 From: Ian Wong Date: Tue, 12 Sep 2017 11:32:51 +0800 Subject: [PATCH 235/823] Display bigdl model in tensorboard (#1545) * Display bigdl model in tensorboard * add document, python API and meet code review. * meet code review * meet code review --- python/dllib/test/bigdl/test_simple_integration.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/python/dllib/test/bigdl/test_simple_integration.py b/python/dllib/test/bigdl/test_simple_integration.py index 51f2a1a2b15..30f0363c430 100644 --- a/python/dllib/test/bigdl/test_simple_integration.py +++ b/python/dllib/test/bigdl/test_simple_integration.py @@ -159,6 +159,15 @@ def test_graph_backward(self): assert_allclose(gradInput[1], np.array([6.0, 6.0, 6.0, 6.0])) + def test_save_graph_topology(self): + fc1 = Linear(4, 2)() + fc2 = Linear(4, 2)() + cadd = CAddTable()([fc1, fc2]) + output1 = ReLU()(cadd) + output2 = Threshold(10.0)(cadd) + model = Model([fc1, fc2], [output1, output2]) + model.save_graph_topology(tempfile.mkdtemp) + def test_load_zip_conf(self): from bigdl.util.common import get_bigdl_conf import sys From c77d331f21d4bd986b6565e5a888c12fab97ade2 Mon Sep 17 00:00:00 2001 From: Ian Wong Date: Tue, 12 Sep 2017 11:32:51 +0800 Subject: [PATCH 236/823] Display bigdl model in tensorboard (#1545) * Display bigdl model in tensorboard * add document, python API and meet code review. * meet code review * meet code review --- python/dllib/src/bigdl/dllib/nn/layer.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 3026e227d32..2fee44d6316 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -553,7 +553,14 @@ def stop_gradient(self, stop_layers, bigdl_type="float"): callBigDlFunc(bigdl_type, "setStopGradient", self.value, stop_layers) return self - + def save_graph_topology(self, log_path, bigdl_type="float"): + """ + save current model graph to a folder, which can be display in tensorboard by running + tensorboard --logdir logPath + :param log_path: + :return: + """ + callBigDlFunc(bigdl_type, "saveGraphTopology", self.value, log_path) class Linear(Layer): From 5cccb0d5bc2ad597905da8948a28437b5e3f5b77 Mon Sep 17 00:00:00 2001 From: Ian Wong Date: Tue, 12 Sep 2017 11:32:51 +0800 Subject: [PATCH 237/823] Display bigdl model in tensorboard (#1545) * Display bigdl model in tensorboard * add document, python API and meet code review. * meet code review * meet code review --- python/dllib/src/bigdl/dllib/nn/layer.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 3026e227d32..2fee44d6316 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -553,7 +553,14 @@ def stop_gradient(self, stop_layers, bigdl_type="float"): callBigDlFunc(bigdl_type, "setStopGradient", self.value, stop_layers) return self - + def save_graph_topology(self, log_path, bigdl_type="float"): + """ + save current model graph to a folder, which can be display in tensorboard by running + tensorboard --logdir logPath + :param log_path: + :return: + """ + callBigDlFunc(bigdl_type, "saveGraphTopology", self.value, log_path) class Linear(Layer): From 39bb8b98c914c23023964fb5a7515f824b15efb6 Mon Sep 17 00:00:00 2001 From: Ian Wong Date: Tue, 12 Sep 2017 13:39:27 +0800 Subject: [PATCH 238/823] Fix operation logical related bugs (#1549) * Support control flow * Decouple input/output tensor numeric type from the module numeric type(parameter type). * FIx unit tests * while loop api refactor * fix unit test * make breeze version configurable as spark 2.1 bump breeze version * Add scheduler to exclude layers * remove control-flow change * remove Schduler and ControlOps * fix broken unit test --- python/dllib/test/dev/diff.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/dllib/test/dev/diff.py b/python/dllib/test/dev/diff.py index f8b3d65b348..8e417e8429c 100755 --- a/python/dllib/test/dev/diff.py +++ b/python/dllib/test/dev/diff.py @@ -29,7 +29,7 @@ def extract_scala_class(class_path): exclude_key_words = set(["*", "abstract", "Const", "Fill", "Shape", - "SplitAndSelect", "StrideSlice"]) + "SplitAndSelect", "StrideSlice", "Scheduler"]) include_key_words = set(["Module", "Criterion", "Container", "Cell", "TensorNumeric"]) # noqa content = "\n".join([line for line in open(class_path).readlines() if all([key not in line for key in exclude_key_words])]) # noqa match = re.search(r"class ([\w]+)[^{]+", content) From ea5c388abb1447d93856f6df4ba77441ef4f4891 Mon Sep 17 00:00:00 2001 From: Jerry Wu Date: Mon, 18 Sep 2017 12:54:27 +0800 Subject: [PATCH 239/823] expor evaluate api to python (#1560) --- python/dllib/src/bigdl/dllib/nn/layer.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 2fee44d6316..d740999aeae 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -238,6 +238,15 @@ def to_ndarray(params): return dict((layer_name, to_ndarray(params)) for layer_name, params in name_to_params.items()) + def evaluate(self): + """ + Evaluate the model to set train = false, useful when doing test/forward + :return: layer itself + """ + callBigDlFunc(self.bigdl_type, + "evaluate", self.value) + return self + def predict(self, data_rdd): """ Model inference base on the given data. From 18587bb2e3676795ac71ed016437db9bff19de34 Mon Sep 17 00:00:00 2001 From: Jerry Wu Date: Mon, 18 Sep 2017 12:54:27 +0800 Subject: [PATCH 240/823] expor evaluate api to python (#1560) --- python/dllib/src/bigdl/dllib/nn/layer.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 2fee44d6316..d740999aeae 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -238,6 +238,15 @@ def to_ndarray(params): return dict((layer_name, to_ndarray(params)) for layer_name, params in name_to_params.items()) + def evaluate(self): + """ + Evaluate the model to set train = false, useful when doing test/forward + :return: layer itself + """ + callBigDlFunc(self.bigdl_type, + "evaluate", self.value) + return self + def predict(self, data_rdd): """ Model inference base on the given data. From 74081c14da3db6acaf23f694893b542a736e4d87 Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Tue, 19 Sep 2017 16:28:37 +0800 Subject: [PATCH 241/823] Support running tensorflow model on BigDL (feeding data with placeholder, providing model output node) (#1534) * run tensorflow as spark local * remove useless code * linear without bias tf * add training mode * allow pass in session * optimizer * fix test * fix test --- python/dllib/src/bigdl/dllib/nn/layer.py | 45 +++++++++++++++++++ .../dllib/src/bigdl/dllib/utils/tf_utils.py | 21 +++++++++ 2 files changed, 66 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index d740999aeae..c44cc2d0607 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -403,6 +403,36 @@ def unfreeze(self): callBigDlFunc(self.bigdl_type, "setLayerUnFreeze", self.value) + def evaluate(self): + ''' + Set this layer in the evaluation mode + ''' + callJavaFunc(get_spark_context(), self.value.evaluate) + return self + + def training(self): + ''' + Set this layer in the training mode + ''' + callJavaFunc(get_spark_context(), self.value.training) + return self + + def is_training(self): + ''' + :return: Whether this layer is in the training mode + + >>> layer = Dropout() + creating: createDropout + >>> layer = layer.evaluate() + >>> layer.is_training() + False + >>> layer = layer.training() + >>> layer.is_training() + True + ''' + return callJavaFunc(get_spark_context(), self.value.isTraining) + + class Container(Layer): @@ -530,6 +560,21 @@ def load_tensorflow(path, inputs, outputs, byte_order = "little_endian", bigdl_t jmodel = callBigDlFunc(bigdl_type, "loadTF", path, inputs, outputs, byte_order) return Model.of(jmodel) + @staticmethod + def train(output, data, label, opt_method, criterion, batch_size, end_when, session=None, bigdl_type="float"): + from bigdl.util.tf_utils import get_path + from bigdl.util.common import Sample + output_name = output.name.split(":")[0] + path = get_path(output_name, session) + sc = get_spark_context() + rdd_train_images = sc.parallelize(data) + rdd_train_labels = sc.parallelize(label) + rdd_train_sample = rdd_train_images.zip(rdd_train_labels).map(lambda (features, label): + Sample.from_ndarray(features, label)) + jmodel = callBigDlFunc(bigdl_type, "trainTF", path, output_name, rdd_train_sample, opt_method, criterion, batch_size, end_when) + return Model.of(jmodel) + + def freeze(self, freeze_layers, bigdl_type="float"): """ set an array of layers to be freezed diff --git a/python/dllib/src/bigdl/dllib/utils/tf_utils.py b/python/dllib/src/bigdl/dllib/utils/tf_utils.py index 0e8696f2c62..5cd6377417d 100644 --- a/python/dllib/src/bigdl/dllib/utils/tf_utils.py +++ b/python/dllib/src/bigdl/dllib/utils/tf_utils.py @@ -31,6 +31,27 @@ from bigdl.util.common import callBigDlFunc import os +def get_path(output_name, sess=None): + + if sess is None: + sess = tf.Session() + init = tf.global_variables_initializer() + sess.run(init) + + temp = tempfile.mkdtemp() + + saver = tf.train.Saver() + saver.save(sess, temp + '/model.chkp') + tf.train.write_graph(sess.graph, temp, 'model.pbtxt') + + merge_checkpoint(temp + '/model.pbtxt', + temp + '/model.chkp', + [output_name], + temp + '/model.pb', sess) + return temp + '/model.pb' + + + def convert(input_ops, output_ops, byte_order, bigdl_type): """ Convert tensorflow model to bigdl model From e8206550781cc1fa828dfaa8892b496f266cf10c Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Tue, 19 Sep 2017 16:28:37 +0800 Subject: [PATCH 242/823] Support running tensorflow model on BigDL (feeding data with placeholder, providing model output node) (#1534) * run tensorflow as spark local * remove useless code * linear without bias tf * add training mode * allow pass in session * optimizer * fix test * fix test --- python/dllib/src/bigdl/dllib/nn/layer.py | 45 +++++++++++++++++++ .../dllib/src/bigdl/dllib/utils/tf_utils.py | 21 +++++++++ 2 files changed, 66 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index d740999aeae..c44cc2d0607 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -403,6 +403,36 @@ def unfreeze(self): callBigDlFunc(self.bigdl_type, "setLayerUnFreeze", self.value) + def evaluate(self): + ''' + Set this layer in the evaluation mode + ''' + callJavaFunc(get_spark_context(), self.value.evaluate) + return self + + def training(self): + ''' + Set this layer in the training mode + ''' + callJavaFunc(get_spark_context(), self.value.training) + return self + + def is_training(self): + ''' + :return: Whether this layer is in the training mode + + >>> layer = Dropout() + creating: createDropout + >>> layer = layer.evaluate() + >>> layer.is_training() + False + >>> layer = layer.training() + >>> layer.is_training() + True + ''' + return callJavaFunc(get_spark_context(), self.value.isTraining) + + class Container(Layer): @@ -530,6 +560,21 @@ def load_tensorflow(path, inputs, outputs, byte_order = "little_endian", bigdl_t jmodel = callBigDlFunc(bigdl_type, "loadTF", path, inputs, outputs, byte_order) return Model.of(jmodel) + @staticmethod + def train(output, data, label, opt_method, criterion, batch_size, end_when, session=None, bigdl_type="float"): + from bigdl.util.tf_utils import get_path + from bigdl.util.common import Sample + output_name = output.name.split(":")[0] + path = get_path(output_name, session) + sc = get_spark_context() + rdd_train_images = sc.parallelize(data) + rdd_train_labels = sc.parallelize(label) + rdd_train_sample = rdd_train_images.zip(rdd_train_labels).map(lambda (features, label): + Sample.from_ndarray(features, label)) + jmodel = callBigDlFunc(bigdl_type, "trainTF", path, output_name, rdd_train_sample, opt_method, criterion, batch_size, end_when) + return Model.of(jmodel) + + def freeze(self, freeze_layers, bigdl_type="float"): """ set an array of layers to be freezed diff --git a/python/dllib/src/bigdl/dllib/utils/tf_utils.py b/python/dllib/src/bigdl/dllib/utils/tf_utils.py index 0e8696f2c62..5cd6377417d 100644 --- a/python/dllib/src/bigdl/dllib/utils/tf_utils.py +++ b/python/dllib/src/bigdl/dllib/utils/tf_utils.py @@ -31,6 +31,27 @@ from bigdl.util.common import callBigDlFunc import os +def get_path(output_name, sess=None): + + if sess is None: + sess = tf.Session() + init = tf.global_variables_initializer() + sess.run(init) + + temp = tempfile.mkdtemp() + + saver = tf.train.Saver() + saver.save(sess, temp + '/model.chkp') + tf.train.write_graph(sess.graph, temp, 'model.pbtxt') + + merge_checkpoint(temp + '/model.pbtxt', + temp + '/model.chkp', + [output_name], + temp + '/model.pb', sess) + return temp + '/model.pb' + + + def convert(input_ops, output_ops, byte_order, bigdl_type): """ Convert tensorflow model to bigdl model From 94f442a35aeec57b50714d47cfb65590c0bb8a48 Mon Sep 17 00:00:00 2001 From: Ian Wong Date: Thu, 21 Sep 2017 11:53:21 +0800 Subject: [PATCH 243/823] Support load tf unet (#1575) * add deconv2d, resizebilinear and support broadcast in add * fix test * fix style error and meet code review * fix python failure * fix unit test --- python/dllib/src/bigdl/dllib/nn/layer.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index c44cc2d0607..1f7bda561d1 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -4053,6 +4053,20 @@ def __init__(self, input_size, output_size, kernel_i, kernel_c, stride, wRegular super(ConvLSTMPeephole3D, self).__init__(None, bigdl_type, input_size, output_size, kernel_i, kernel_c, stride, wRegularizer, uRegularizer, bRegularizer, cRegularizer, with_peephole) +class ResizeBilinear(Layer): + """ + Resize the input image with bilinear interpolation. The input image must be a float tensor with + NHWC layout + + :param output_height: output height + :param output_width: output width + :param align_corner: align corner or not + + >>> resizeBilinear = ResizeBilinear(10, 20, false) + """ + def __init__(self, output_height, output_width, align_corner): + super(ResizeBilinear, self).__init__(None, output_height, output_width, align_corner) + def _test(): import doctest from pyspark import SparkContext From 4b4bc854405c79a9f1775dbf78aa4156ed2effa9 Mon Sep 17 00:00:00 2001 From: Ian Wong Date: Thu, 21 Sep 2017 11:53:21 +0800 Subject: [PATCH 244/823] Support load tf unet (#1575) * add deconv2d, resizebilinear and support broadcast in add * fix test * fix style error and meet code review * fix python failure * fix unit test --- python/dllib/src/bigdl/dllib/nn/layer.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index c44cc2d0607..1f7bda561d1 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -4053,6 +4053,20 @@ def __init__(self, input_size, output_size, kernel_i, kernel_c, stride, wRegular super(ConvLSTMPeephole3D, self).__init__(None, bigdl_type, input_size, output_size, kernel_i, kernel_c, stride, wRegularizer, uRegularizer, bRegularizer, cRegularizer, with_peephole) +class ResizeBilinear(Layer): + """ + Resize the input image with bilinear interpolation. The input image must be a float tensor with + NHWC layout + + :param output_height: output height + :param output_width: output width + :param align_corner: align corner or not + + >>> resizeBilinear = ResizeBilinear(10, 20, false) + """ + def __init__(self, output_height, output_width, align_corner): + super(ResizeBilinear, self).__init__(None, output_height, output_width, align_corner) + def _test(): import doctest from pyspark import SparkContext From 2215dff8b78228bdf89cca12d999a0b449668efd Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Wed, 27 Sep 2017 17:16:16 +0800 Subject: [PATCH 245/823] Handle jar path different spark versions; Redirect Spark Logs in Python (#1543) * Use `spark.driver.extraClassPath` to get the jar for spark 2.2+. * Show BigDL INFO logs and redirect spark logs in python models. --- .../src/bigdl/dllib/models/lenet/lenet5.py | 3 +- .../src/bigdl/dllib/models/rnn/rnnexample.py | 2 + .../models/textclassifier/textclassifier.py | 2 + python/dllib/src/bigdl/dllib/utils/common.py | 27 +++++++- python/dllib/src/bigdl/dllib/utils/engine.py | 66 +++++++++++++++++-- 5 files changed, 93 insertions(+), 7 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py b/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py index 70e4e417c13..ac0e5eb7513 100644 --- a/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py +++ b/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py @@ -69,6 +69,8 @@ def get_mnist(sc, data_type="train", location="/tmp/mnist"): (options, args) = parser.parse_args(sys.argv) sc = SparkContext(appName="lenet5", conf=create_spark_conf()) + redire_spark_logs() + show_bigdl_info_logs() init_engine() if options.action == "train": @@ -86,7 +88,6 @@ def get_end_trigger(): .map(lambda rec_tuple: (normalizer(rec_tuple[0], mnist.TEST_MEAN, mnist.TEST_STD), rec_tuple[1]))\ .map(lambda t: Sample.from_ndarray(t[0], t[1])) - optimizer = Optimizer( model=build_model(10), training_rdd=train_data, diff --git a/python/dllib/src/bigdl/dllib/models/rnn/rnnexample.py b/python/dllib/src/bigdl/dllib/models/rnn/rnnexample.py index dddaff1f61f..b387928e71c 100644 --- a/python/dllib/src/bigdl/dllib/models/rnn/rnnexample.py +++ b/python/dllib/src/bigdl/dllib/models/rnn/rnnexample.py @@ -152,6 +152,8 @@ def build_model(input_size, hidden_size, output_size): sc = SparkContext(appName="simplernn_example", conf=create_spark_conf()) + redire_spark_logs() + show_bigdl_info_logs() init_engine() (train_rdd, val_rdd, vob_size) = prepare_data(sc, folder, vob_size, training_split) diff --git a/python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py b/python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py index 1864fac9fce..dd5a8e0bdae 100644 --- a/python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py +++ b/python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py @@ -168,6 +168,8 @@ def train(sc, training_split = 0.8 sc = SparkContext(appName="text_classifier", conf=create_spark_conf()) + redire_spark_logs() + show_bigdl_info_logs() init_engine() train(sc, batch_size, diff --git a/python/dllib/src/bigdl/dllib/utils/common.py b/python/dllib/src/bigdl/dllib/utils/common.py index 82e40e0cad3..7f33632ffd4 100644 --- a/python/dllib/src/bigdl/dllib/utils/common.py +++ b/python/dllib/src/bigdl/dllib/utils/common.py @@ -28,7 +28,7 @@ from pyspark import SparkConf import numpy as np import threading -from bigdl.util.engine import prepare_env +from bigdl.util.engine import get_bigdl_classpath, is_spark_below_2_2 INTMAX = 2147483647 INTMIN = -2147483648 @@ -249,6 +249,22 @@ def init_engine(bigdl_type="float"): callBigDlFunc(bigdl_type, "initEngine") +def redire_spark_logs(bigdl_type="float", logPath=os.getcwd()+"/bigdl.log"): + """ + Redirect spark logs to the specified path. + :param bigdl_type: "double" or "float" + :param logPath: the file path to be redirected to; the default file is under the current workspace named `bigdl.log`. + """ + callBigDlFunc(bigdl_type, "redirectSparkLogs", logPath) + +def show_bigdl_info_logs(bigdl_type="float"): + """ + Set BigDL log level to INFO. + :param bigdl_type: "double" or "float" + """ + callBigDlFunc(bigdl_type, "showBigDlInfoLogs") + + def get_bigdl_conf(): bigdl_conf_file = "spark-bigdl.conf" bigdl_python_wrapper = "python-api.zip" @@ -277,12 +293,21 @@ def to_list(a): return [a] +def extend_spark_driver_cp(sparkConf, path): + original_driver_classpath = ":" + sparkConf.get("spark.driver.extraClassPath") \ + if sparkConf.contains("spark.driver.extraClassPath") else "" + sparkConf.set("spark.driver.extraClassPath", path + original_driver_classpath) + + def create_spark_conf(): bigdl_conf = get_bigdl_conf() sparkConf = SparkConf() sparkConf.setAll(bigdl_conf.items()) + if not is_spark_below_2_2(): + extend_spark_driver_cp(sparkConf, get_bigdl_classpath()) return sparkConf + def get_spark_context(conf = None): """ Get the current active spark context and create one if no active instance diff --git a/python/dllib/src/bigdl/dllib/utils/engine.py b/python/dllib/src/bigdl/dllib/utils/engine.py index 998887a8689..948b8d0a888 100644 --- a/python/dllib/src/bigdl/dllib/utils/engine.py +++ b/python/dllib/src/bigdl/dllib/utils/engine.py @@ -35,24 +35,80 @@ def __prepare_spark_env(): def __prepare_bigdl_env(): jar_dir = os.path.abspath(__file__ + "/../../") - jar_paths = glob.glob(os.path.join(jar_dir, "share/lib/*.jar")) conf_paths = glob.glob(os.path.join(jar_dir, "share/conf/*.conf")) + bigdl_classpath = get_bigdl_classpath() def append_path(env_var_name, path): try: - print("Adding %s to %s" % (jar_paths[0], env_var_name)) + print("Adding %s to %s" % (path, env_var_name)) os.environ[env_var_name] = path + ":" + os.environ[ env_var_name] # noqa except KeyError: os.environ[env_var_name] = path - if conf_paths and conf_paths: - assert len(conf_paths) == 1, "Expecting one jar: %s" % len(jar_paths) + if conf_paths: assert len(conf_paths) == 1, "Expecting one conf: %s" % len(conf_paths) - append_path("SPARK_CLASSPATH", jar_paths[0]) print("Prepending %s to sys.path" % conf_paths[0]) sys.path.insert(0, conf_paths[0]) + if bigdl_classpath and is_spark_below_2_2(): + append_path("SPARK_CLASSPATH", bigdl_classpath) + + +def get_bigdl_classpath(): + """ + Get and return the jar path for bigdl if exists. + """ + if(os.getenv("BIGDL_CLASSPATH")): + return os.environ["BIGDL_CLASSPATH"] + jar_dir = os.path.abspath(__file__ + "/../../") + jar_paths = glob.glob(os.path.join(jar_dir, "share/lib/*.jar")) + if jar_paths: + assert len(jar_paths) == 1, "Expecting one jar: %s" % len(jar_paths) + return jar_paths[0] + return "" + + +def is_spark_below_2_2(): + """ + Check if spark version is below 2.2 + """ + import pyspark + if(hasattr(pyspark,"version")): + full_version = pyspark.version.__version__ + # We only need the general spark version (eg, 1.6, 2.2). + parts = full_version.split(".") + spark_version = parts[0] + "." + parts[1] + if(compare_version(spark_version, "2.2")>=0): + return False + return True + + +def compare_version(version1, version2): + """ + Compare version strings. + :param version1; + :param version2; + :return: 1 if version1 is after version2; -1 if version1 is before version2; 0 if two versions are the same. + """ + v1Arr = version1.split(".") + v2Arr = version2.split(".") + len1 = len(v1Arr) + len2 = len(v2Arr) + lenMax = max(len1, len2) + for x in range(lenMax): + v1Token = 0 + if x < len1: + v1Token = int(v1Arr[x]) + v2Token = 0 + if x < len2: + v2Token = int(v2Arr[x]) + if v1Token < v2Token: + return -1 + if v1Token > v2Token: + return 1 + return 0 + def prepare_env(): __prepare_spark_env() From df371b3860d15972380ea0a4a0d146c7b038cd37 Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Wed, 27 Sep 2017 17:16:16 +0800 Subject: [PATCH 246/823] Handle jar path different spark versions; Redirect Spark Logs in Python (#1543) * Use `spark.driver.extraClassPath` to get the jar for spark 2.2+. * Show BigDL INFO logs and redirect spark logs in python models. --- .../src/bigdl/dllib/models/lenet/lenet5.py | 3 +- .../src/bigdl/dllib/models/rnn/rnnexample.py | 2 + .../models/textclassifier/textclassifier.py | 2 + python/dllib/src/bigdl/utils/common.py | 27 +++++++- python/dllib/src/bigdl/utils/engine.py | 66 +++++++++++++++++-- 5 files changed, 93 insertions(+), 7 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py b/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py index 70e4e417c13..ac0e5eb7513 100644 --- a/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py +++ b/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py @@ -69,6 +69,8 @@ def get_mnist(sc, data_type="train", location="/tmp/mnist"): (options, args) = parser.parse_args(sys.argv) sc = SparkContext(appName="lenet5", conf=create_spark_conf()) + redire_spark_logs() + show_bigdl_info_logs() init_engine() if options.action == "train": @@ -86,7 +88,6 @@ def get_end_trigger(): .map(lambda rec_tuple: (normalizer(rec_tuple[0], mnist.TEST_MEAN, mnist.TEST_STD), rec_tuple[1]))\ .map(lambda t: Sample.from_ndarray(t[0], t[1])) - optimizer = Optimizer( model=build_model(10), training_rdd=train_data, diff --git a/python/dllib/src/bigdl/dllib/models/rnn/rnnexample.py b/python/dllib/src/bigdl/dllib/models/rnn/rnnexample.py index dddaff1f61f..b387928e71c 100644 --- a/python/dllib/src/bigdl/dllib/models/rnn/rnnexample.py +++ b/python/dllib/src/bigdl/dllib/models/rnn/rnnexample.py @@ -152,6 +152,8 @@ def build_model(input_size, hidden_size, output_size): sc = SparkContext(appName="simplernn_example", conf=create_spark_conf()) + redire_spark_logs() + show_bigdl_info_logs() init_engine() (train_rdd, val_rdd, vob_size) = prepare_data(sc, folder, vob_size, training_split) diff --git a/python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py b/python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py index 1864fac9fce..dd5a8e0bdae 100644 --- a/python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py +++ b/python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py @@ -168,6 +168,8 @@ def train(sc, training_split = 0.8 sc = SparkContext(appName="text_classifier", conf=create_spark_conf()) + redire_spark_logs() + show_bigdl_info_logs() init_engine() train(sc, batch_size, diff --git a/python/dllib/src/bigdl/utils/common.py b/python/dllib/src/bigdl/utils/common.py index 82e40e0cad3..7f33632ffd4 100644 --- a/python/dllib/src/bigdl/utils/common.py +++ b/python/dllib/src/bigdl/utils/common.py @@ -28,7 +28,7 @@ from pyspark import SparkConf import numpy as np import threading -from bigdl.util.engine import prepare_env +from bigdl.util.engine import get_bigdl_classpath, is_spark_below_2_2 INTMAX = 2147483647 INTMIN = -2147483648 @@ -249,6 +249,22 @@ def init_engine(bigdl_type="float"): callBigDlFunc(bigdl_type, "initEngine") +def redire_spark_logs(bigdl_type="float", logPath=os.getcwd()+"/bigdl.log"): + """ + Redirect spark logs to the specified path. + :param bigdl_type: "double" or "float" + :param logPath: the file path to be redirected to; the default file is under the current workspace named `bigdl.log`. + """ + callBigDlFunc(bigdl_type, "redirectSparkLogs", logPath) + +def show_bigdl_info_logs(bigdl_type="float"): + """ + Set BigDL log level to INFO. + :param bigdl_type: "double" or "float" + """ + callBigDlFunc(bigdl_type, "showBigDlInfoLogs") + + def get_bigdl_conf(): bigdl_conf_file = "spark-bigdl.conf" bigdl_python_wrapper = "python-api.zip" @@ -277,12 +293,21 @@ def to_list(a): return [a] +def extend_spark_driver_cp(sparkConf, path): + original_driver_classpath = ":" + sparkConf.get("spark.driver.extraClassPath") \ + if sparkConf.contains("spark.driver.extraClassPath") else "" + sparkConf.set("spark.driver.extraClassPath", path + original_driver_classpath) + + def create_spark_conf(): bigdl_conf = get_bigdl_conf() sparkConf = SparkConf() sparkConf.setAll(bigdl_conf.items()) + if not is_spark_below_2_2(): + extend_spark_driver_cp(sparkConf, get_bigdl_classpath()) return sparkConf + def get_spark_context(conf = None): """ Get the current active spark context and create one if no active instance diff --git a/python/dllib/src/bigdl/utils/engine.py b/python/dllib/src/bigdl/utils/engine.py index 998887a8689..948b8d0a888 100644 --- a/python/dllib/src/bigdl/utils/engine.py +++ b/python/dllib/src/bigdl/utils/engine.py @@ -35,24 +35,80 @@ def __prepare_spark_env(): def __prepare_bigdl_env(): jar_dir = os.path.abspath(__file__ + "/../../") - jar_paths = glob.glob(os.path.join(jar_dir, "share/lib/*.jar")) conf_paths = glob.glob(os.path.join(jar_dir, "share/conf/*.conf")) + bigdl_classpath = get_bigdl_classpath() def append_path(env_var_name, path): try: - print("Adding %s to %s" % (jar_paths[0], env_var_name)) + print("Adding %s to %s" % (path, env_var_name)) os.environ[env_var_name] = path + ":" + os.environ[ env_var_name] # noqa except KeyError: os.environ[env_var_name] = path - if conf_paths and conf_paths: - assert len(conf_paths) == 1, "Expecting one jar: %s" % len(jar_paths) + if conf_paths: assert len(conf_paths) == 1, "Expecting one conf: %s" % len(conf_paths) - append_path("SPARK_CLASSPATH", jar_paths[0]) print("Prepending %s to sys.path" % conf_paths[0]) sys.path.insert(0, conf_paths[0]) + if bigdl_classpath and is_spark_below_2_2(): + append_path("SPARK_CLASSPATH", bigdl_classpath) + + +def get_bigdl_classpath(): + """ + Get and return the jar path for bigdl if exists. + """ + if(os.getenv("BIGDL_CLASSPATH")): + return os.environ["BIGDL_CLASSPATH"] + jar_dir = os.path.abspath(__file__ + "/../../") + jar_paths = glob.glob(os.path.join(jar_dir, "share/lib/*.jar")) + if jar_paths: + assert len(jar_paths) == 1, "Expecting one jar: %s" % len(jar_paths) + return jar_paths[0] + return "" + + +def is_spark_below_2_2(): + """ + Check if spark version is below 2.2 + """ + import pyspark + if(hasattr(pyspark,"version")): + full_version = pyspark.version.__version__ + # We only need the general spark version (eg, 1.6, 2.2). + parts = full_version.split(".") + spark_version = parts[0] + "." + parts[1] + if(compare_version(spark_version, "2.2")>=0): + return False + return True + + +def compare_version(version1, version2): + """ + Compare version strings. + :param version1; + :param version2; + :return: 1 if version1 is after version2; -1 if version1 is before version2; 0 if two versions are the same. + """ + v1Arr = version1.split(".") + v2Arr = version2.split(".") + len1 = len(v1Arr) + len2 = len(v2Arr) + lenMax = max(len1, len2) + for x in range(lenMax): + v1Token = 0 + if x < len1: + v1Token = int(v1Arr[x]) + v2Token = 0 + if x < len2: + v2Token = int(v2Arr[x]) + if v1Token < v2Token: + return -1 + if v1Token > v2Token: + return 1 + return 0 + def prepare_env(): __prepare_spark_env() From 9cc6bfb84706b04ee2d137062785dc4adc8a3fa4 Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Wed, 27 Sep 2017 17:16:16 +0800 Subject: [PATCH 247/823] Handle jar path different spark versions; Redirect Spark Logs in Python (#1543) * Use `spark.driver.extraClassPath` to get the jar for spark 2.2+. * Show BigDL INFO logs and redirect spark logs in python models. --- python/dllib/test/bigdl/test_simple_integration.py | 9 +++++++++ python/dllib/test/dev/prepare_env.sh | 4 ++-- python/dllib/test/dev/release/release.sh | 2 +- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/python/dllib/test/bigdl/test_simple_integration.py b/python/dllib/test/bigdl/test_simple_integration.py index 30f0363c430..d9426bffe6f 100644 --- a/python/dllib/test/bigdl/test_simple_integration.py +++ b/python/dllib/test/bigdl/test_simple_integration.py @@ -26,6 +26,7 @@ import tempfile import pytest from numpy.testing import assert_allclose +from bigdl.util.engine import compare_version class TestSimple(): @@ -411,5 +412,13 @@ def test_save_jtensor_dict(self): # in old impl, this will throw an exception _py2java(self.sc, tensors) + def test_compare_version(self): + assert compare_version("2.1.1", "2.2.0") == -1 + assert compare_version("2.2.0", "1.6.2") == 1 + assert compare_version("2.2.0", "2.2.0") == 0 + assert compare_version("1.6.0", "2.1.0") == -1 + assert compare_version("2.1.0", "2.1.1") == -1 + assert compare_version("2.0.1", "1.5.2") == 1 + if __name__ == "__main__": pytest.main([__file__]) diff --git a/python/dllib/test/dev/prepare_env.sh b/python/dllib/test/dev/prepare_env.sh index fbde2c55379..a3a3825cbcf 100755 --- a/python/dllib/test/dev/prepare_env.sh +++ b/python/dllib/test/dev/prepare_env.sh @@ -30,8 +30,8 @@ export PYSPARK_ZIP=`find $SPARK_HOME/python/lib -type f -iname '*.zip' | tr "\n export PYTHONPATH=$PYSPARK_ZIP:$DL_PYTHON_HOME:$DL_PYTHON_HOME/:$DL_PYTHON_HOME/test/dev:$BIGDL_HOME/spark/dl/src/main/resources/spark-bigdl.conf -export SPARK_CLASSPATH=$(find $BIGDL_HOME/spark/dl/target/ -name "*with-dependencies.jar" | head -n 1) -echo "SPARK_CLASSPATH": $SPARK_CLASSPATH +export BIGDL_CLASSPATH=$(find $BIGDL_HOME/spark/dl/target/ -name "*with-dependencies.jar" | head -n 1) +echo "BIGDL_CLASSPATH": $BIGDL_CLASSPATH export PYTHON_EXECUTABLES=("python2.7" "python3.5") diff --git a/python/dllib/test/dev/release/release.sh b/python/dllib/test/dev/release/release.sh index c0b9b574795..15c39914c7e 100755 --- a/python/dllib/test/dev/release/release.sh +++ b/python/dllib/test/dev/release/release.sh @@ -25,7 +25,7 @@ BIGDL_PYTHON_DIR="$(cd ${RUN_SCRIPT_DIR}/../../../../pyspark; pwd)" echo $BIGDL_PYTHON_DIR if (( $# < 2)); then - echo "Bad parameters. Uasge: release.sh mac spark_2.x" + echo "Bad parameters. Usage: release.sh mac spark_2.x" exit -1 fi From 81785ed946599666f52cbea5733a4f2218b20459 Mon Sep 17 00:00:00 2001 From: dding3 Date: Wed, 27 Sep 2017 13:42:18 -0400 Subject: [PATCH 248/823] Support feed prior prediction back into recurrent (#1476) * Implement RecurrentDecoder * add ut for lstm with recurrent decoder --- python/dllib/src/bigdl/dllib/nn/layer.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 1f7bda561d1..7f57e72aa9d 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -1059,6 +1059,26 @@ def set_state(self, state): jstate, state_is_table = self.check_input(state) callBigDlFunc(self.bigdl_type, "setState", self.value, jstate, state_is_table) + +class RecurrentDecoder(Recurrent): + ''' + RecurrentDecoder module is a container of rnn cells which used to make + a prediction of the next timestep based on the prediction we made from + the previous timestep. Input for RecurrentDecoder is dynamically composed + during training. input at t(i) is output at t(i-1), input at t(0) is + user input, and user input has to be batch x ???(depends on cell type) + without time information. + + Different types of rnn cells can be added using add() function. Currently + only support lstmpeephole, convlstm, convlstm3D cell. + + >>> recurrent_decoder = RecurrentDecoder(output_length = 5) + creating: createRecurrentDecoder + ''' + + def __init__(self, output_length, bigdl_type="float"): + super(Recurrent, self).__init__(None, bigdl_type, output_length) + class LSTM(Layer): ''' | Long Short Term Memory architecture. From ace33bc607f47d11f41b46f5571bdb65ca7023e1 Mon Sep 17 00:00:00 2001 From: dding3 Date: Wed, 27 Sep 2017 13:42:18 -0400 Subject: [PATCH 249/823] Support feed prior prediction back into recurrent (#1476) * Implement RecurrentDecoder * add ut for lstm with recurrent decoder --- python/dllib/src/bigdl/dllib/nn/layer.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 1f7bda561d1..7f57e72aa9d 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -1059,6 +1059,26 @@ def set_state(self, state): jstate, state_is_table = self.check_input(state) callBigDlFunc(self.bigdl_type, "setState", self.value, jstate, state_is_table) + +class RecurrentDecoder(Recurrent): + ''' + RecurrentDecoder module is a container of rnn cells which used to make + a prediction of the next timestep based on the prediction we made from + the previous timestep. Input for RecurrentDecoder is dynamically composed + during training. input at t(i) is output at t(i-1), input at t(0) is + user input, and user input has to be batch x ???(depends on cell type) + without time information. + + Different types of rnn cells can be added using add() function. Currently + only support lstmpeephole, convlstm, convlstm3D cell. + + >>> recurrent_decoder = RecurrentDecoder(output_length = 5) + creating: createRecurrentDecoder + ''' + + def __init__(self, output_length, bigdl_type="float"): + super(Recurrent, self).__init__(None, bigdl_type, output_length) + class LSTM(Layer): ''' | Long Short Term Memory architecture. From eca44d02fbe3ff3fa8791d349078785b480a605a Mon Sep 17 00:00:00 2001 From: hkvision Date: Mon, 25 Sep 2017 11:35:53 +0800 Subject: [PATCH 250/823] Change model.test to model.evaluate in Python --- python/dllib/test/bigdl/test_simple_integration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/dllib/test/bigdl/test_simple_integration.py b/python/dllib/test/bigdl/test_simple_integration.py index d9426bffe6f..3d59cc50b0e 100644 --- a/python/dllib/test/bigdl/test_simple_integration.py +++ b/python/dllib/test/bigdl/test_simple_integration.py @@ -263,7 +263,7 @@ def gen_rand_sample(): print(str(i) + "\n") print(len(p)) - test_results = trained_model.test(trainingData, 32, [Top1Accuracy()]) + test_results = trained_model.evaluate(trainingData, 32, [Top1Accuracy()]) for test_result in test_results: print(test_result) From 5fa68011f53cfb72e20f767f2454f19123dd32f9 Mon Sep 17 00:00:00 2001 From: hkvision Date: Mon, 25 Sep 2017 11:35:53 +0800 Subject: [PATCH 251/823] Change model.test to model.evaluate in Python --- python/dllib/src/bigdl/dllib/models/lenet/lenet5.py | 2 +- python/dllib/src/bigdl/dllib/nn/layer.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py b/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py index ac0e5eb7513..f3a2fb1b123 100644 --- a/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py +++ b/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py @@ -109,7 +109,7 @@ def get_end_trigger(): test_data = get_mnist(sc, "test").map( normalizer(mnist.TEST_MEAN, mnist.TEST_STD)) model = Model.load(options.modelPath) - results = model.test(test_data, options.batchSize, [Top1Accuracy()]) + results = model.evaluate(test_data, options.batchSize, [Top1Accuracy()]) for result in results: print(result) sc.stop() diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 7f57e72aa9d..fe077ea8155 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -271,7 +271,7 @@ def predict_class(self, data_rdd): "modelPredictClass", self.value, data_rdd) return result - def test(self, val_rdd, batch_size, val_methods): + def evaluate(self, val_rdd, batch_size, val_methods): """ A method to benchmark the model quality. @@ -281,7 +281,7 @@ def test(self, val_rdd, batch_size, val_methods): :return: """ return callBigDlFunc(self.bigdl_type, - "modelTest", + "modelEvaluate", self.value, val_rdd, batch_size, val_methods) From b0164ad618820b34fc102164dce3e9d1a6029e08 Mon Sep 17 00:00:00 2001 From: hkvision Date: Mon, 25 Sep 2017 11:35:53 +0800 Subject: [PATCH 252/823] Change model.test to model.evaluate in Python --- python/dllib/src/bigdl/dllib/models/lenet/lenet5.py | 2 +- python/dllib/src/bigdl/dllib/nn/layer.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py b/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py index ac0e5eb7513..f3a2fb1b123 100644 --- a/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py +++ b/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py @@ -109,7 +109,7 @@ def get_end_trigger(): test_data = get_mnist(sc, "test").map( normalizer(mnist.TEST_MEAN, mnist.TEST_STD)) model = Model.load(options.modelPath) - results = model.test(test_data, options.batchSize, [Top1Accuracy()]) + results = model.evaluate(test_data, options.batchSize, [Top1Accuracy()]) for result in results: print(result) sc.stop() diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 7f57e72aa9d..fe077ea8155 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -271,7 +271,7 @@ def predict_class(self, data_rdd): "modelPredictClass", self.value, data_rdd) return result - def test(self, val_rdd, batch_size, val_methods): + def evaluate(self, val_rdd, batch_size, val_methods): """ A method to benchmark the model quality. @@ -281,7 +281,7 @@ def test(self, val_rdd, batch_size, val_methods): :return: """ return callBigDlFunc(self.bigdl_type, - "modelTest", + "modelEvaluate", self.value, val_rdd, batch_size, val_methods) From 2c667fcdf772f9d2a08ccc24f0caf0b98c25d890 Mon Sep 17 00:00:00 2001 From: helenlly Date: Mon, 9 Oct 2017 15:26:46 +0800 Subject: [PATCH 253/823] Revert "Change model.test to model.evaluate in Python" This reverts commit 5de5cd4fdc1d3c02147a2a47a6653f1b49f2fc21. --- python/dllib/test/bigdl/test_simple_integration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/dllib/test/bigdl/test_simple_integration.py b/python/dllib/test/bigdl/test_simple_integration.py index 3d59cc50b0e..d9426bffe6f 100644 --- a/python/dllib/test/bigdl/test_simple_integration.py +++ b/python/dllib/test/bigdl/test_simple_integration.py @@ -263,7 +263,7 @@ def gen_rand_sample(): print(str(i) + "\n") print(len(p)) - test_results = trained_model.evaluate(trainingData, 32, [Top1Accuracy()]) + test_results = trained_model.test(trainingData, 32, [Top1Accuracy()]) for test_result in test_results: print(test_result) From a49ea5232272f25c64b6f04078712a9882cb23f2 Mon Sep 17 00:00:00 2001 From: helenlly Date: Mon, 9 Oct 2017 15:26:46 +0800 Subject: [PATCH 254/823] Revert "Change model.test to model.evaluate in Python" This reverts commit 5de5cd4fdc1d3c02147a2a47a6653f1b49f2fc21. --- python/dllib/src/bigdl/dllib/models/lenet/lenet5.py | 2 +- python/dllib/src/bigdl/dllib/nn/layer.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py b/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py index f3a2fb1b123..ac0e5eb7513 100644 --- a/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py +++ b/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py @@ -109,7 +109,7 @@ def get_end_trigger(): test_data = get_mnist(sc, "test").map( normalizer(mnist.TEST_MEAN, mnist.TEST_STD)) model = Model.load(options.modelPath) - results = model.evaluate(test_data, options.batchSize, [Top1Accuracy()]) + results = model.test(test_data, options.batchSize, [Top1Accuracy()]) for result in results: print(result) sc.stop() diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index fe077ea8155..7f57e72aa9d 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -271,7 +271,7 @@ def predict_class(self, data_rdd): "modelPredictClass", self.value, data_rdd) return result - def evaluate(self, val_rdd, batch_size, val_methods): + def test(self, val_rdd, batch_size, val_methods): """ A method to benchmark the model quality. @@ -281,7 +281,7 @@ def evaluate(self, val_rdd, batch_size, val_methods): :return: """ return callBigDlFunc(self.bigdl_type, - "modelEvaluate", + "modelTest", self.value, val_rdd, batch_size, val_methods) From 75077a9f53f5c03327438a1dcb5be26a98088555 Mon Sep 17 00:00:00 2001 From: helenlly Date: Mon, 9 Oct 2017 15:26:46 +0800 Subject: [PATCH 255/823] Revert "Change model.test to model.evaluate in Python" This reverts commit 5de5cd4fdc1d3c02147a2a47a6653f1b49f2fc21. --- python/dllib/src/bigdl/dllib/models/lenet/lenet5.py | 2 +- python/dllib/src/bigdl/dllib/nn/layer.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py b/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py index f3a2fb1b123..ac0e5eb7513 100644 --- a/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py +++ b/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py @@ -109,7 +109,7 @@ def get_end_trigger(): test_data = get_mnist(sc, "test").map( normalizer(mnist.TEST_MEAN, mnist.TEST_STD)) model = Model.load(options.modelPath) - results = model.evaluate(test_data, options.batchSize, [Top1Accuracy()]) + results = model.test(test_data, options.batchSize, [Top1Accuracy()]) for result in results: print(result) sc.stop() diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index fe077ea8155..7f57e72aa9d 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -271,7 +271,7 @@ def predict_class(self, data_rdd): "modelPredictClass", self.value, data_rdd) return result - def evaluate(self, val_rdd, batch_size, val_methods): + def test(self, val_rdd, batch_size, val_methods): """ A method to benchmark the model quality. @@ -281,7 +281,7 @@ def evaluate(self, val_rdd, batch_size, val_methods): :return: """ return callBigDlFunc(self.bigdl_type, - "modelEvaluate", + "modelTest", self.value, val_rdd, batch_size, val_methods) From d99f0ee18a7f98b242194035359fdf09852ffcb6 Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Tue, 10 Oct 2017 10:01:21 +0800 Subject: [PATCH 256/823] Fix error in exit status when running python tests (#1631) * Correct command exit status --- python/dllib/test/dev/run-tests | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/python/dllib/test/dev/run-tests b/python/dllib/test/dev/run-tests index 471651e28a9..d142d25b31e 100755 --- a/python/dllib/test/dev/run-tests +++ b/python/dllib/test/dev/run-tests @@ -33,9 +33,10 @@ do --ignore=../../../pyspark/bigdl/util/ \ --ignore=../../../pyspark/bigdl/models/ && \ $p -m pytest -v -n 4 ../../../pyspark/test/ - if [ $? -ne 0 ]; + exit_status=$? + if [ $exit_status -ne 0 ]; then - exit $? + exit $exit_status fi done From 7339a791b97e96e027363527fdbf3c38fb1a86d0 Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Tue, 10 Oct 2017 10:02:15 +0800 Subject: [PATCH 257/823] Fix failures in python tests (#1623) * fix test failures in creating layers * Fix wrong file path in save_graph_topology * Fix lambda invalid syntax in python3.5 --- python/dllib/src/bigdl/dllib/nn/layer.py | 26 +++++++++++++++--------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 7f57e72aa9d..d1fbd47f7f2 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -569,8 +569,8 @@ def train(output, data, label, opt_method, criterion, batch_size, end_when, sess sc = get_spark_context() rdd_train_images = sc.parallelize(data) rdd_train_labels = sc.parallelize(label) - rdd_train_sample = rdd_train_images.zip(rdd_train_labels).map(lambda (features, label): - Sample.from_ndarray(features, label)) + rdd_train_sample = rdd_train_images.zip(rdd_train_labels).map(lambda input: + Sample.from_ndarray(input[0], input[1])) jmodel = callBigDlFunc(bigdl_type, "trainTF", path, output_name, rdd_train_sample, opt_method, criterion, batch_size, end_when) return Model.of(jmodel) @@ -611,10 +611,13 @@ def save_graph_topology(self, log_path, bigdl_type="float"): """ save current model graph to a folder, which can be display in tensorboard by running tensorboard --logdir logPath - :param log_path: - :return: + :param log_path: path to save the model graph + :param bigdl_type: + :return: """ callBigDlFunc(bigdl_type, "saveGraphTopology", self.value, log_path) + return self + class Linear(Layer): @@ -1196,11 +1199,13 @@ def __init__(self, input_size, hidden_size, activation, + isInputWithBias=True, + isHiddenWithBias=True, wRegularizer=None, uRegularizer=None, bRegularizer=None, bigdl_type="float"): - super(RnnCell, self).__init__(None, bigdl_type, input_size, hidden_size, activation, wRegularizer, uRegularizer, bRegularizer) + super(RnnCell, self).__init__(None, bigdl_type, input_size, hidden_size, activation, isInputWithBias, isHiddenWithBias, wRegularizer, uRegularizer, bRegularizer) class TimeDistributed(Layer): @@ -1594,7 +1599,7 @@ def set_init_method(self, weight_init_method = None, bias_init_method = None): return self -class BifurcateSplitTable(Model): +class BifurcateSplitTable(Layer): ''' Creates a module that takes a Tensor as input and outputs two tables, splitting the Tensor along @@ -1616,7 +1621,7 @@ def __init__(self, dimension) -class Bilinear(Model): +class Bilinear(Layer): ''' a bilinear transformation with sparse inputs, @@ -4082,10 +4087,11 @@ class ResizeBilinear(Layer): :param output_width: output width :param align_corner: align corner or not - >>> resizeBilinear = ResizeBilinear(10, 20, false) + >>> resizeBilinear = ResizeBilinear(10, 20, False) + creating: createResizeBilinear """ - def __init__(self, output_height, output_width, align_corner): - super(ResizeBilinear, self).__init__(None, output_height, output_width, align_corner) + def __init__(self, output_height, output_width, align_corner, bigdl_type="float"): + super(ResizeBilinear, self).__init__(None, bigdl_type, output_height, output_width, align_corner) def _test(): import doctest From c262c27f6d56eef10d1d41b6a7a08628a3df93d4 Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Tue, 10 Oct 2017 10:02:15 +0800 Subject: [PATCH 258/823] Fix failures in python tests (#1623) * fix test failures in creating layers * Fix wrong file path in save_graph_topology * Fix lambda invalid syntax in python3.5 --- python/dllib/src/bigdl/dllib/nn/layer.py | 26 +++++++++++++++--------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 7f57e72aa9d..d1fbd47f7f2 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -569,8 +569,8 @@ def train(output, data, label, opt_method, criterion, batch_size, end_when, sess sc = get_spark_context() rdd_train_images = sc.parallelize(data) rdd_train_labels = sc.parallelize(label) - rdd_train_sample = rdd_train_images.zip(rdd_train_labels).map(lambda (features, label): - Sample.from_ndarray(features, label)) + rdd_train_sample = rdd_train_images.zip(rdd_train_labels).map(lambda input: + Sample.from_ndarray(input[0], input[1])) jmodel = callBigDlFunc(bigdl_type, "trainTF", path, output_name, rdd_train_sample, opt_method, criterion, batch_size, end_when) return Model.of(jmodel) @@ -611,10 +611,13 @@ def save_graph_topology(self, log_path, bigdl_type="float"): """ save current model graph to a folder, which can be display in tensorboard by running tensorboard --logdir logPath - :param log_path: - :return: + :param log_path: path to save the model graph + :param bigdl_type: + :return: """ callBigDlFunc(bigdl_type, "saveGraphTopology", self.value, log_path) + return self + class Linear(Layer): @@ -1196,11 +1199,13 @@ def __init__(self, input_size, hidden_size, activation, + isInputWithBias=True, + isHiddenWithBias=True, wRegularizer=None, uRegularizer=None, bRegularizer=None, bigdl_type="float"): - super(RnnCell, self).__init__(None, bigdl_type, input_size, hidden_size, activation, wRegularizer, uRegularizer, bRegularizer) + super(RnnCell, self).__init__(None, bigdl_type, input_size, hidden_size, activation, isInputWithBias, isHiddenWithBias, wRegularizer, uRegularizer, bRegularizer) class TimeDistributed(Layer): @@ -1594,7 +1599,7 @@ def set_init_method(self, weight_init_method = None, bias_init_method = None): return self -class BifurcateSplitTable(Model): +class BifurcateSplitTable(Layer): ''' Creates a module that takes a Tensor as input and outputs two tables, splitting the Tensor along @@ -1616,7 +1621,7 @@ def __init__(self, dimension) -class Bilinear(Model): +class Bilinear(Layer): ''' a bilinear transformation with sparse inputs, @@ -4082,10 +4087,11 @@ class ResizeBilinear(Layer): :param output_width: output width :param align_corner: align corner or not - >>> resizeBilinear = ResizeBilinear(10, 20, false) + >>> resizeBilinear = ResizeBilinear(10, 20, False) + creating: createResizeBilinear """ - def __init__(self, output_height, output_width, align_corner): - super(ResizeBilinear, self).__init__(None, output_height, output_width, align_corner) + def __init__(self, output_height, output_width, align_corner, bigdl_type="float"): + super(ResizeBilinear, self).__init__(None, bigdl_type, output_height, output_width, align_corner) def _test(): import doctest From 7ba53d410bcbc06cc33e3a23f455b91ce826baf2 Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Tue, 10 Oct 2017 10:02:15 +0800 Subject: [PATCH 259/823] Fix failures in python tests (#1623) * fix test failures in creating layers * Fix wrong file path in save_graph_topology * Fix lambda invalid syntax in python3.5 --- python/dllib/test/bigdl/test_simple_integration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/dllib/test/bigdl/test_simple_integration.py b/python/dllib/test/bigdl/test_simple_integration.py index d9426bffe6f..9d806cf1908 100644 --- a/python/dllib/test/bigdl/test_simple_integration.py +++ b/python/dllib/test/bigdl/test_simple_integration.py @@ -167,7 +167,7 @@ def test_save_graph_topology(self): output1 = ReLU()(cadd) output2 = Threshold(10.0)(cadd) model = Model([fc1, fc2], [output1, output2]) - model.save_graph_topology(tempfile.mkdtemp) + model.save_graph_topology(tempfile.mkdtemp()) def test_load_zip_conf(self): from bigdl.util.common import get_bigdl_conf From 5d800996b7f9056652d4ca4e8ba0c6af48f89fc6 Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Wed, 11 Oct 2017 08:57:39 +0800 Subject: [PATCH 260/823] Refactor Python API: model test to model evaluation (#1641) * Change model.test to model.evaluate in Python * Add model evaluation code in docs and correct some typo * update docs * Remove duplicate evaluate(self) definition from previous two PRs * fix same function name but different arguments for evaluate() --- .../src/bigdl/dllib/models/lenet/lenet5.py | 2 +- python/dllib/src/bigdl/dllib/nn/layer.py | 47 +++++++++---------- 2 files changed, 23 insertions(+), 26 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py b/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py index ac0e5eb7513..f3a2fb1b123 100644 --- a/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py +++ b/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py @@ -109,7 +109,7 @@ def get_end_trigger(): test_data = get_mnist(sc, "test").map( normalizer(mnist.TEST_MEAN, mnist.TEST_STD)) model = Model.load(options.modelPath) - results = model.test(test_data, options.batchSize, [Top1Accuracy()]) + results = model.evaluate(test_data, options.batchSize, [Top1Accuracy()]) for result in results: print(result) sc.stop() diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index d1fbd47f7f2..becce958166 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -238,14 +238,32 @@ def to_ndarray(params): return dict((layer_name, to_ndarray(params)) for layer_name, params in name_to_params.items()) - def evaluate(self): + def evaluate(self, *args): """ + No argument passed in: Evaluate the model to set train = false, useful when doing test/forward :return: layer itself + + Three arguments passed in: + A method to benchmark the model quality. + + :param val_rdd: the input data + :param batch_size: batch size + :param val_methods: a list of validation methods. i.e: Top1Accuracy,Top5Accuracy and Loss. + :return: """ - callBigDlFunc(self.bigdl_type, - "evaluate", self.value) - return self + if len(args) == 0: + callBigDlFunc(self.bigdl_type, + "evaluate", self.value) + return self + elif len(args) == 3: + val_rdd, batch_size, val_methods = args + return callBigDlFunc(self.bigdl_type, + "modelEvaluate", + self.value, + val_rdd, batch_size, val_methods) + else: + raise Exception("Error when calling evaluate(): it takes no argument or exactly three arguments only") def predict(self, data_rdd): """ @@ -271,20 +289,6 @@ def predict_class(self, data_rdd): "modelPredictClass", self.value, data_rdd) return result - def test(self, val_rdd, batch_size, val_methods): - """ - A method to benchmark the model quality. - - :param val_rdd: the input data - :param batch_size: batch size - :param val_methods: a list of validation methods. i.e: Top1Accuracy,Top5Accuracy and Loss. - :return: - """ - return callBigDlFunc(self.bigdl_type, - "modelTest", - self.value, - val_rdd, batch_size, val_methods) - def set_weights(self, weights): """ Set weights for this layer @@ -403,13 +407,6 @@ def unfreeze(self): callBigDlFunc(self.bigdl_type, "setLayerUnFreeze", self.value) - def evaluate(self): - ''' - Set this layer in the evaluation mode - ''' - callJavaFunc(get_spark_context(), self.value.evaluate) - return self - def training(self): ''' Set this layer in the training mode From bd9c39f8098fd889e032fd68b774623322af2e98 Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Wed, 11 Oct 2017 08:57:39 +0800 Subject: [PATCH 261/823] Refactor Python API: model test to model evaluation (#1641) * Change model.test to model.evaluate in Python * Add model evaluation code in docs and correct some typo * update docs * Remove duplicate evaluate(self) definition from previous two PRs * fix same function name but different arguments for evaluate() --- .../src/bigdl/dllib/models/lenet/lenet5.py | 2 +- python/dllib/src/bigdl/dllib/nn/layer.py | 47 +++++++++---------- 2 files changed, 23 insertions(+), 26 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py b/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py index ac0e5eb7513..f3a2fb1b123 100644 --- a/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py +++ b/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py @@ -109,7 +109,7 @@ def get_end_trigger(): test_data = get_mnist(sc, "test").map( normalizer(mnist.TEST_MEAN, mnist.TEST_STD)) model = Model.load(options.modelPath) - results = model.test(test_data, options.batchSize, [Top1Accuracy()]) + results = model.evaluate(test_data, options.batchSize, [Top1Accuracy()]) for result in results: print(result) sc.stop() diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index d1fbd47f7f2..becce958166 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -238,14 +238,32 @@ def to_ndarray(params): return dict((layer_name, to_ndarray(params)) for layer_name, params in name_to_params.items()) - def evaluate(self): + def evaluate(self, *args): """ + No argument passed in: Evaluate the model to set train = false, useful when doing test/forward :return: layer itself + + Three arguments passed in: + A method to benchmark the model quality. + + :param val_rdd: the input data + :param batch_size: batch size + :param val_methods: a list of validation methods. i.e: Top1Accuracy,Top5Accuracy and Loss. + :return: """ - callBigDlFunc(self.bigdl_type, - "evaluate", self.value) - return self + if len(args) == 0: + callBigDlFunc(self.bigdl_type, + "evaluate", self.value) + return self + elif len(args) == 3: + val_rdd, batch_size, val_methods = args + return callBigDlFunc(self.bigdl_type, + "modelEvaluate", + self.value, + val_rdd, batch_size, val_methods) + else: + raise Exception("Error when calling evaluate(): it takes no argument or exactly three arguments only") def predict(self, data_rdd): """ @@ -271,20 +289,6 @@ def predict_class(self, data_rdd): "modelPredictClass", self.value, data_rdd) return result - def test(self, val_rdd, batch_size, val_methods): - """ - A method to benchmark the model quality. - - :param val_rdd: the input data - :param batch_size: batch size - :param val_methods: a list of validation methods. i.e: Top1Accuracy,Top5Accuracy and Loss. - :return: - """ - return callBigDlFunc(self.bigdl_type, - "modelTest", - self.value, - val_rdd, batch_size, val_methods) - def set_weights(self, weights): """ Set weights for this layer @@ -403,13 +407,6 @@ def unfreeze(self): callBigDlFunc(self.bigdl_type, "setLayerUnFreeze", self.value) - def evaluate(self): - ''' - Set this layer in the evaluation mode - ''' - callJavaFunc(get_spark_context(), self.value.evaluate) - return self - def training(self): ''' Set this layer in the training mode From 8940464f9814f32e4b828222045d7620886c5b14 Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Wed, 11 Oct 2017 08:57:39 +0800 Subject: [PATCH 262/823] Refactor Python API: model test to model evaluation (#1641) * Change model.test to model.evaluate in Python * Add model evaluation code in docs and correct some typo * update docs * Remove duplicate evaluate(self) definition from previous two PRs * fix same function name but different arguments for evaluate() --- python/dllib/test/bigdl/test_simple_integration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/dllib/test/bigdl/test_simple_integration.py b/python/dllib/test/bigdl/test_simple_integration.py index 9d806cf1908..f8ec31b5b54 100644 --- a/python/dllib/test/bigdl/test_simple_integration.py +++ b/python/dllib/test/bigdl/test_simple_integration.py @@ -263,7 +263,7 @@ def gen_rand_sample(): print(str(i) + "\n") print(len(p)) - test_results = trained_model.test(trainingData, 32, [Top1Accuracy()]) + test_results = trained_model.evaluate(trainingData, 32, [Top1Accuracy()]) for test_result in test_results: print(test_result) From c043b58982640297554dda60eebd5fa9c3d243cc Mon Sep 17 00:00:00 2001 From: Yanzhang Wang Date: Wed, 11 Oct 2017 03:16:48 -0400 Subject: [PATCH 263/823] feat: quantize a whole graph/modules (#1618) * feat: quantize a whole graph/modules * feat: python supports * fix: delete unusage --- python/dllib/src/bigdl/dllib/nn/layer.py | 81 +++++++++++++++++++++++- 1 file changed, 80 insertions(+), 1 deletion(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index becce958166..5d3edf94878 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -429,7 +429,87 @@ def is_training(self): ''' return callJavaFunc(get_spark_context(), self.value.isTraining) + def quantize(self): + ''' + Clone self and quantize it, at last return a new quantized model. + :return: A new quantized model. + >>> fc = Linear(4, 2) + creating: createLinear + >>> fc.set_weights([np.ones((4, 2)), np.ones((2,))]) + >>> input = np.ones((2, 4)) + >>> fc.forward(input) + array([[ 5., 5.], + [ 5., 5.]], dtype=float32) + >>> quantized_fc = fc.quantize() + >>> quantized_fc.forward(input) + array([[ 5., 5.], + [ 5., 5.]], dtype=float32) + + >>> assert("quantized.Linear" in quantized_fc.__str__()) + >>> conv = SpatialConvolution(1, 2, 3, 3) + creating: createSpatialConvolution + >>> conv.set_weights([np.ones((2, 1, 3, 3)), np.zeros((2,))]) + >>> input = np.ones((2, 1, 4, 4)) + >>> conv.forward(input) + array([[[[ 9., 9.], + [ 9., 9.]], + + [[ 9., 9.], + [ 9., 9.]]], + + + [[[ 9., 9.], + [ 9., 9.]], + + [[ 9., 9.], + [ 9., 9.]]]], dtype=float32) + >>> quantized_conv = conv.quantize() + >>> quantized_conv.forward(input) + array([[[[ 9., 9.], + [ 9., 9.]], + + [[ 9., 9.], + [ 9., 9.]]], + + + [[[ 9., 9.], + [ 9., 9.]], + + [[ 9., 9.], + [ 9., 9.]]]], dtype=float32) + >>> assert("quantized.SpatialConvolution" in quantized_conv.__str__()) + >>> seq = Sequential() + creating: createSequential + >>> seq = seq.add(conv) + >>> seq = seq.add(Reshape([8, 4], False)) + creating: createReshape + >>> seq = seq.add(fc) + >>> input = np.ones([1, 1, 6, 6]) + >>> seq.forward(input) + array([[ 37., 37.], + [ 37., 37.], + [ 37., 37.], + [ 37., 37.], + [ 37., 37.], + [ 37., 37.], + [ 37., 37.], + [ 37., 37.]], dtype=float32) + >>> quantized_seq = seq.quantize() + >>> quantized_seq.forward(input) + array([[ 37., 37.], + [ 37., 37.], + [ 37., 37.], + [ 37., 37.], + [ 37., 37.], + [ 37., 37.], + [ 37., 37.], + [ 37., 37.]], dtype=float32) + >>> assert("quantized.Linear" in quantized_seq.__str__()) + >>> assert("quantized.SpatialConvolution" in quantized_seq.__str__()) + ''' + quantized_model = callBigDlFunc(self.bigdl_type, "quantize", self.value) + return Layer.of(quantized_model) class Container(Layer): @@ -615,7 +695,6 @@ def save_graph_topology(self, log_path, bigdl_type="float"): callBigDlFunc(bigdl_type, "saveGraphTopology", self.value, log_path) return self - class Linear(Layer): ''' From 09faf23b1a2d8c73a7f127ea802a7f9c771a3019 Mon Sep 17 00:00:00 2001 From: Yanzhang Wang Date: Wed, 11 Oct 2017 03:16:48 -0400 Subject: [PATCH 264/823] feat: quantize a whole graph/modules (#1618) * feat: quantize a whole graph/modules * feat: python supports * fix: delete unusage --- python/dllib/src/bigdl/dllib/nn/layer.py | 81 +++++++++++++++++++++++- 1 file changed, 80 insertions(+), 1 deletion(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index becce958166..5d3edf94878 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -429,7 +429,87 @@ def is_training(self): ''' return callJavaFunc(get_spark_context(), self.value.isTraining) + def quantize(self): + ''' + Clone self and quantize it, at last return a new quantized model. + :return: A new quantized model. + >>> fc = Linear(4, 2) + creating: createLinear + >>> fc.set_weights([np.ones((4, 2)), np.ones((2,))]) + >>> input = np.ones((2, 4)) + >>> fc.forward(input) + array([[ 5., 5.], + [ 5., 5.]], dtype=float32) + >>> quantized_fc = fc.quantize() + >>> quantized_fc.forward(input) + array([[ 5., 5.], + [ 5., 5.]], dtype=float32) + + >>> assert("quantized.Linear" in quantized_fc.__str__()) + >>> conv = SpatialConvolution(1, 2, 3, 3) + creating: createSpatialConvolution + >>> conv.set_weights([np.ones((2, 1, 3, 3)), np.zeros((2,))]) + >>> input = np.ones((2, 1, 4, 4)) + >>> conv.forward(input) + array([[[[ 9., 9.], + [ 9., 9.]], + + [[ 9., 9.], + [ 9., 9.]]], + + + [[[ 9., 9.], + [ 9., 9.]], + + [[ 9., 9.], + [ 9., 9.]]]], dtype=float32) + >>> quantized_conv = conv.quantize() + >>> quantized_conv.forward(input) + array([[[[ 9., 9.], + [ 9., 9.]], + + [[ 9., 9.], + [ 9., 9.]]], + + + [[[ 9., 9.], + [ 9., 9.]], + + [[ 9., 9.], + [ 9., 9.]]]], dtype=float32) + >>> assert("quantized.SpatialConvolution" in quantized_conv.__str__()) + >>> seq = Sequential() + creating: createSequential + >>> seq = seq.add(conv) + >>> seq = seq.add(Reshape([8, 4], False)) + creating: createReshape + >>> seq = seq.add(fc) + >>> input = np.ones([1, 1, 6, 6]) + >>> seq.forward(input) + array([[ 37., 37.], + [ 37., 37.], + [ 37., 37.], + [ 37., 37.], + [ 37., 37.], + [ 37., 37.], + [ 37., 37.], + [ 37., 37.]], dtype=float32) + >>> quantized_seq = seq.quantize() + >>> quantized_seq.forward(input) + array([[ 37., 37.], + [ 37., 37.], + [ 37., 37.], + [ 37., 37.], + [ 37., 37.], + [ 37., 37.], + [ 37., 37.], + [ 37., 37.]], dtype=float32) + >>> assert("quantized.Linear" in quantized_seq.__str__()) + >>> assert("quantized.SpatialConvolution" in quantized_seq.__str__()) + ''' + quantized_model = callBigDlFunc(self.bigdl_type, "quantize", self.value) + return Layer.of(quantized_model) class Container(Layer): @@ -615,7 +695,6 @@ def save_graph_topology(self, log_path, bigdl_type="float"): callBigDlFunc(bigdl_type, "saveGraphTopology", self.value, log_path) return self - class Linear(Layer): ''' From 1fc417779f6d6bffdd6ff2c6a6b3e753a2e34946 Mon Sep 17 00:00:00 2001 From: zhangxiaoli73 <380761639@qq.com> Date: Thu, 12 Oct 2017 13:16:43 +0800 Subject: [PATCH 265/823] add Variational Auto Encoder (#1432) * vae * add python api and unit tests * add docs --- python/dllib/src/bigdl/dllib/nn/criterion.py | 22 ++++++++++++++++++++ python/dllib/src/bigdl/dllib/nn/layer.py | 9 ++++++++ 2 files changed, 31 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/nn/criterion.py b/python/dllib/src/bigdl/dllib/nn/criterion.py index ebffa855939..8f5aa3a3dcd 100644 --- a/python/dllib/src/bigdl/dllib/nn/criterion.py +++ b/python/dllib/src/bigdl/dllib/nn/criterion.py @@ -464,6 +464,28 @@ def add(self, criterion, weight=1.0): self.value.add(criterion.value, weight) return self +class KLDCriterion(Criterion): + + ''' + Computes the KL-divergence of the Gaussian distribution. + >>> KLDCriterion = KLDCriterion() + creating: createKLDCriterion + ''' + + def __init__(self, bigdl_type="float"): + super(KLDCriterion, self).__init__(None, bigdl_type) + + +class GaussianCriterion(Criterion): + + ''' + Computes the log-likelihood of a sample x given a Gaussian distribution p. + >>> GaussianCriterion = GaussianCriterion() + creating: createGaussianCriterion + ''' + + def __init__(self, bigdl_type="float"): + super(GaussianCriterion, self).__init__(None, bigdl_type) class SmoothL1Criterion(Criterion): diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 5d3edf94878..2398a11af2a 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -4169,6 +4169,15 @@ class ResizeBilinear(Layer): def __init__(self, output_height, output_width, align_corner, bigdl_type="float"): super(ResizeBilinear, self).__init__(None, bigdl_type, output_height, output_width, align_corner) +class GaussianSampler(Layer): + """ + Takes {mean, log_variance} as input and samples from the Gaussian distribution + >>> sampler = GaussianSampler() + creating: createGaussianSampler + """ + def __init__(self, bigdl_type="float"): + super(GaussianSampler, self).__init__(None, bigdl_type) + def _test(): import doctest from pyspark import SparkContext From ba091ef513c7361dd37625f9eaef8cd505951f81 Mon Sep 17 00:00:00 2001 From: zhangxiaoli73 <380761639@qq.com> Date: Thu, 12 Oct 2017 13:16:43 +0800 Subject: [PATCH 266/823] add Variational Auto Encoder (#1432) * vae * add python api and unit tests * add docs --- python/dllib/src/bigdl/dllib/nn/criterion.py | 22 ++++++++++++++++++++ python/dllib/src/bigdl/dllib/nn/layer.py | 9 ++++++++ 2 files changed, 31 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/nn/criterion.py b/python/dllib/src/bigdl/dllib/nn/criterion.py index ebffa855939..8f5aa3a3dcd 100644 --- a/python/dllib/src/bigdl/dllib/nn/criterion.py +++ b/python/dllib/src/bigdl/dllib/nn/criterion.py @@ -464,6 +464,28 @@ def add(self, criterion, weight=1.0): self.value.add(criterion.value, weight) return self +class KLDCriterion(Criterion): + + ''' + Computes the KL-divergence of the Gaussian distribution. + >>> KLDCriterion = KLDCriterion() + creating: createKLDCriterion + ''' + + def __init__(self, bigdl_type="float"): + super(KLDCriterion, self).__init__(None, bigdl_type) + + +class GaussianCriterion(Criterion): + + ''' + Computes the log-likelihood of a sample x given a Gaussian distribution p. + >>> GaussianCriterion = GaussianCriterion() + creating: createGaussianCriterion + ''' + + def __init__(self, bigdl_type="float"): + super(GaussianCriterion, self).__init__(None, bigdl_type) class SmoothL1Criterion(Criterion): diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 5d3edf94878..2398a11af2a 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -4169,6 +4169,15 @@ class ResizeBilinear(Layer): def __init__(self, output_height, output_width, align_corner, bigdl_type="float"): super(ResizeBilinear, self).__init__(None, bigdl_type, output_height, output_width, align_corner) +class GaussianSampler(Layer): + """ + Takes {mean, log_variance} as input and samples from the Gaussian distribution + >>> sampler = GaussianSampler() + creating: createGaussianSampler + """ + def __init__(self, bigdl_type="float"): + super(GaussianSampler, self).__init__(None, bigdl_type) + def _test(): import doctest from pyspark import SparkContext From 75e71be3b64ba0a9807f18d31911d656d07b5b63 Mon Sep 17 00:00:00 2001 From: dding3 Date: Thu, 12 Oct 2017 22:03:22 -0400 Subject: [PATCH 267/823] RecurrentDecoder enhancement (#1619) * RecurrentDecoder refactor --- python/dllib/src/bigdl/dllib/nn/layer.py | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 2398a11af2a..dda84786e28 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -1119,25 +1119,24 @@ class Recurrent(Container): def __init__(self, bigdl_type="float"): super(Recurrent, self).__init__(None, bigdl_type) - def get_state(self): + def get_hidden_state(self): """ get hidden state and cell at last time step. :return: list of hidden state and cell """ - state = callBigDlFunc(self.bigdl_type, "getState", self.value) + state = callBigDlFunc(self.bigdl_type, "getHiddenState", self.value) for idx, tensor in enumerate(state): - state[idx] = tensor.to_ndarray() + state[idx] = tensor.to_ndarray() return state - - def set_state(self, state): + + def set_hidden_state(self, states): """ set hidden state and cell at first time step. """ - jstate, state_is_table = self.check_input(state) - callBigDlFunc(self.bigdl_type, "setState", self.value, jstate, state_is_table) - + jstate, state_is_table = self.check_input(states) + callBigDlFunc(self.bigdl_type, "setHiddenState", self.value, jstate, state_is_table) class RecurrentDecoder(Recurrent): ''' @@ -1145,11 +1144,10 @@ class RecurrentDecoder(Recurrent): a prediction of the next timestep based on the prediction we made from the previous timestep. Input for RecurrentDecoder is dynamically composed during training. input at t(i) is output at t(i-1), input at t(0) is - user input, and user input has to be batch x ???(depends on cell type) - without time information. + user input, and user input has to be batch x stepShape(shape of the input + at a single time step). - Different types of rnn cells can be added using add() function. Currently - only support lstmpeephole, convlstm, convlstm3D cell. + Different types of rnn cells can be added using add() function. >>> recurrent_decoder = RecurrentDecoder(output_length = 5) creating: createRecurrentDecoder From b051c8054d6d335a49a3da697218993ae03a922a Mon Sep 17 00:00:00 2001 From: dding3 Date: Thu, 12 Oct 2017 22:03:22 -0400 Subject: [PATCH 268/823] RecurrentDecoder enhancement (#1619) * RecurrentDecoder refactor --- python/dllib/src/bigdl/dllib/nn/layer.py | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 2398a11af2a..dda84786e28 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -1119,25 +1119,24 @@ class Recurrent(Container): def __init__(self, bigdl_type="float"): super(Recurrent, self).__init__(None, bigdl_type) - def get_state(self): + def get_hidden_state(self): """ get hidden state and cell at last time step. :return: list of hidden state and cell """ - state = callBigDlFunc(self.bigdl_type, "getState", self.value) + state = callBigDlFunc(self.bigdl_type, "getHiddenState", self.value) for idx, tensor in enumerate(state): - state[idx] = tensor.to_ndarray() + state[idx] = tensor.to_ndarray() return state - - def set_state(self, state): + + def set_hidden_state(self, states): """ set hidden state and cell at first time step. """ - jstate, state_is_table = self.check_input(state) - callBigDlFunc(self.bigdl_type, "setState", self.value, jstate, state_is_table) - + jstate, state_is_table = self.check_input(states) + callBigDlFunc(self.bigdl_type, "setHiddenState", self.value, jstate, state_is_table) class RecurrentDecoder(Recurrent): ''' @@ -1145,11 +1144,10 @@ class RecurrentDecoder(Recurrent): a prediction of the next timestep based on the prediction we made from the previous timestep. Input for RecurrentDecoder is dynamically composed during training. input at t(i) is output at t(i-1), input at t(0) is - user input, and user input has to be batch x ???(depends on cell type) - without time information. + user input, and user input has to be batch x stepShape(shape of the input + at a single time step). - Different types of rnn cells can be added using add() function. Currently - only support lstmpeephole, convlstm, convlstm3D cell. + Different types of rnn cells can be added using add() function. >>> recurrent_decoder = RecurrentDecoder(output_length = 5) creating: createRecurrentDecoder From bb7c1b3619629e6e2faea9e607b68ac0c3e363f5 Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Fri, 13 Oct 2017 16:30:33 +0800 Subject: [PATCH 269/823] Support feature with one or more tensors for Sample in python (#1650) * feature: JTensor to list of JTensors * correct typo in creating sequential model for multiple inputs * add test for multiple inputs --- python/dllib/src/bigdl/dllib/utils/common.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/utils/common.py b/python/dllib/src/bigdl/dllib/utils/common.py index 7f33632ffd4..cdb387d4f6d 100644 --- a/python/dllib/src/bigdl/dllib/utils/common.py +++ b/python/dllib/src/bigdl/dllib/utils/common.py @@ -179,7 +179,7 @@ class Sample(object): def __init__(self, features, label, bigdl_type="float"): """ User should always use Sample.from_ndarray to construct Sample. - :param features: a JTensor + :param features: a list of JTensors :param label: a JTensor :param bigdl_type: "double" or "float" """ @@ -191,21 +191,27 @@ def __init__(self, features, label, bigdl_type="float"): def from_ndarray(cls, features, label, bigdl_type="float"): """ Convert a ndarray of features and label to Sample, which would be used in Java side. + :param features: an ndarray or a list of ndarrays + :param label: an ndarray or a scalar + :param bigdl_type: "double" or "float" >>> import numpy as np >>> from bigdl.util.common import callBigDlFunc >>> from numpy.testing import assert_allclose >>> sample = Sample.from_ndarray(np.random.random((2,3)), np.random.random((2,3))) >>> sample_back = callBigDlFunc("float", "testSample", sample) - >>> assert_allclose(sample.features.to_ndarray(), sample_back.features.to_ndarray()) + >>> assert_allclose(sample.features[0].to_ndarray(), sample_back.features[0].to_ndarray()) >>> assert_allclose(sample.label.to_ndarray(), sample_back.label.to_ndarray()) """ - assert isinstance(features, np.ndarray), \ - "features should be a np.ndarray, not %s" % type(features) + if isinstance(features, np.ndarray): + features = [features] + else: + assert all(isinstance(feature, np.ndarray) for feature in features), \ + "features should be a list of np.ndarray, not %s" % type(features) if not isinstance(label, np.ndarray): # in case label is a scalar. label = np.array(label) return cls( - features=JTensor.from_ndarray(features), + features=[JTensor.from_ndarray(f) for f in features], label=JTensor.from_ndarray(label), bigdl_type=bigdl_type) From 99d10d915ca772c43fbef00de19d2695ced75b99 Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Fri, 13 Oct 2017 16:30:33 +0800 Subject: [PATCH 270/823] Support feature with one or more tensors for Sample in python (#1650) * feature: JTensor to list of JTensors * correct typo in creating sequential model for multiple inputs * add test for multiple inputs --- python/dllib/src/bigdl/utils/common.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/python/dllib/src/bigdl/utils/common.py b/python/dllib/src/bigdl/utils/common.py index 7f33632ffd4..cdb387d4f6d 100644 --- a/python/dllib/src/bigdl/utils/common.py +++ b/python/dllib/src/bigdl/utils/common.py @@ -179,7 +179,7 @@ class Sample(object): def __init__(self, features, label, bigdl_type="float"): """ User should always use Sample.from_ndarray to construct Sample. - :param features: a JTensor + :param features: a list of JTensors :param label: a JTensor :param bigdl_type: "double" or "float" """ @@ -191,21 +191,27 @@ def __init__(self, features, label, bigdl_type="float"): def from_ndarray(cls, features, label, bigdl_type="float"): """ Convert a ndarray of features and label to Sample, which would be used in Java side. + :param features: an ndarray or a list of ndarrays + :param label: an ndarray or a scalar + :param bigdl_type: "double" or "float" >>> import numpy as np >>> from bigdl.util.common import callBigDlFunc >>> from numpy.testing import assert_allclose >>> sample = Sample.from_ndarray(np.random.random((2,3)), np.random.random((2,3))) >>> sample_back = callBigDlFunc("float", "testSample", sample) - >>> assert_allclose(sample.features.to_ndarray(), sample_back.features.to_ndarray()) + >>> assert_allclose(sample.features[0].to_ndarray(), sample_back.features[0].to_ndarray()) >>> assert_allclose(sample.label.to_ndarray(), sample_back.label.to_ndarray()) """ - assert isinstance(features, np.ndarray), \ - "features should be a np.ndarray, not %s" % type(features) + if isinstance(features, np.ndarray): + features = [features] + else: + assert all(isinstance(feature, np.ndarray) for feature in features), \ + "features should be a list of np.ndarray, not %s" % type(features) if not isinstance(label, np.ndarray): # in case label is a scalar. label = np.array(label) return cls( - features=JTensor.from_ndarray(features), + features=[JTensor.from_ndarray(f) for f in features], label=JTensor.from_ndarray(label), bigdl_type=bigdl_type) From 684ca3674d4f639fe25c0c08b4805365fda29cf3 Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Fri, 13 Oct 2017 16:30:33 +0800 Subject: [PATCH 271/823] Support feature with one or more tensors for Sample in python (#1650) * feature: JTensor to list of JTensors * correct typo in creating sequential model for multiple inputs * add test for multiple inputs --- .../test/bigdl/test_simple_integration.py | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/python/dllib/test/bigdl/test_simple_integration.py b/python/dllib/test/bigdl/test_simple_integration.py index f8ec31b5b54..c2bcf681a89 100644 --- a/python/dllib/test/bigdl/test_simple_integration.py +++ b/python/dllib/test/bigdl/test_simple_integration.py @@ -267,6 +267,51 @@ def gen_rand_sample(): for test_result in test_results: print(test_result) + def test_multiple_input(self): + """ + Test training data of samples with several tensors as feature + using a sequential model with multiple inputs. + """ + FEATURES_DIM = 2 + data_len = 100 + batch_size = 32 + epoch_num = 5 + + def gen_rand_sample(): + features1 = np.random.uniform(0, 1, (FEATURES_DIM)) + features2 = np.random.uniform(0, 1, (FEATURES_DIM)) + label = np.array((2 * (features1 + features2)).sum() + 0.4) + return Sample.from_ndarray([features1, features2], label) + + trainingData = self.sc.parallelize(range(0, data_len)).map( + lambda i: gen_rand_sample()) + + model_test = Sequential() + branches = ParallelTable() + branch1 = Sequential().add(Linear(FEATURES_DIM, 1)).add(ReLU()) + branch2 = Sequential().add(Linear(FEATURES_DIM, 1)).add(ReLU()) + branches.add(branch1).add(branch2) + model_test.add(branches).add(CAddTable()) + + optim_method = SGD(learningrate=0.01, learningrate_decay=0.0002, weightdecay=0.0, + momentum=0.0, dampening=0.0, nesterov=False, + leaningrate_schedule=Poly(0.5, int((data_len / batch_size) * epoch_num))) + optimizer = Optimizer( + model=model_test, + training_rdd=trainingData, + criterion=MSECriterion(), + optim_method=optim_method, + end_trigger=MaxEpoch(epoch_num), + batch_size=batch_size) + optimizer.set_validation( + batch_size=batch_size, + val_rdd=trainingData, + trigger=EveryEpoch(), + val_method=[Top1Accuracy()] + ) + + optimizer.optimize() + def test_forward_backward(self): from bigdl.nn.layer import Linear rng = RNG() From 35f401146b98c176469414468820488dfa1066a2 Mon Sep 17 00:00:00 2001 From: Xin Qiu Date: Fri, 13 Oct 2017 16:44:26 +0800 Subject: [PATCH 272/823] SparseLinear SparseJoinTable DenseToSparse (#1652) * SparseLinear SparseJoinTable DenseToSparse * Python api * add DenseToSparseSpec * update to upstream * add some method * meet code review * fix python unit test * fix python unit test --- python/dllib/src/bigdl/dllib/nn/layer.py | 90 ++++++++++++++++++++++++ 1 file changed, 90 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index dda84786e28..c5dfa479347 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -744,6 +744,74 @@ def set_init_method(self, weight_init_method = None, bias_init_method = None): weight_init_method, bias_init_method) return self +class SparseLinear(Layer): + + ''' + SparseLinear is the sparse version of module Linear. SparseLinear has two different from Linear: + firstly, SparseLinear's input Tensor is a SparseTensor. Secondly, SparseLinear doesn't backward + gradient to next layer in the backpropagation by default, as the gradInput of SparseLinear is + useless and very big in most cases. + + But, considering model like Wide&Deep, we provide backwardStart and backwardLength to backward + part of the gradient to next layer. + + :param input_size the size the each input sample + :param output_size the size of the module output of each sample + :param backwardStart backwardStart index, counting from 1 + :param backwardLength backward length + :param withBias if has bias + :param wRegularizer: instance of [[Regularizer]](eg. L1 or L2 regularization), applied to the input weights matrices. + :param bRegularizer: instance of [[Regularizer]]applied to the bias. + :param init_weight: the optional initial value for the weight + :param init_bias: the optional initial value for the bias + :param init_grad_weight: the optional initial value for the grad_weight + :param init_grad_bias: the optional initial value for the grad_bias + + + >>> sparselinear = SparseLinear(100, 10, True, wRegularizer=L1Regularizer(0.5), bRegularizer=L1Regularizer(0.5)) + creating: createL1Regularizer + creating: createL1Regularizer + creating: createSparseLinear + >>> import numpy as np + >>> init_weight = np.random.randn(10, 100) + >>> init_bias = np.random.randn(10) + >>> init_grad_weight = np.zeros([10, 100]) + >>> init_grad_bias = np.zeros([10]) + >>> sparselinear = SparseLinear(100, 10, True, 1, 5, L1Regularizer(0.5), L1Regularizer(0.5), init_weight, init_bias, init_grad_weight, init_grad_bias) + creating: createL1Regularizer + creating: createL1Regularizer + creating: createSparseLinear + ''' + + def __init__(self, input_size, output_size, with_bias=True, backwardStart=-1, backwardLength=-1, + wRegularizer=None, bRegularizer=None, init_weight=None, init_bias=None, + init_grad_weight=None, init_grad_bias=None, bigdl_type="float"): + super(SparseLinear, self).__init__(None, bigdl_type, input_size, output_size, + with_bias, backwardStart, backwardLength, + wRegularizer, bRegularizer, + JTensor.from_ndarray(init_weight), + JTensor.from_ndarray(init_bias), + JTensor.from_ndarray(init_grad_weight), + JTensor.from_ndarray(init_grad_bias)) + + def set_init_method(self, weight_init_method = None, bias_init_method = None): + callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, + weight_init_method, bias_init_method) + return self + +class DenseToSparse(Layer): + + ''' + Convert DenseTensor to SparseTensor. + + + >>> DenseToSparse = DenseToSparse() + creating: createDenseToSparse + ''' + + def __init__(self, + bigdl_type="float"): + super(DenseToSparse, self).__init__(None, bigdl_type) class ReLU(Layer): @@ -2294,6 +2362,28 @@ def __init__(self, dimension, n_input_dims) +class SparseJoinTable(Layer): + + ''' + :: Experimental :: + + Sparse version of JoinTable. Backward just pass the origin gradOutput back to + the next layers without split. So this layer may just works in Wide&Deep like models. + + + :param dimension: to be join in this dimension + + + >>> joinTable = SparseJoinTable(1) + creating: createSparseJoinTable + ''' + + def __init__(self, + dimension, + bigdl_type="float"): + super(SparseJoinTable, self).__init__(None, bigdl_type, + dimension) + class L1Penalty(Layer): From 0bc41c750fb76b291647266c8848912f645e097d Mon Sep 17 00:00:00 2001 From: Xin Qiu Date: Fri, 13 Oct 2017 16:44:26 +0800 Subject: [PATCH 273/823] SparseLinear SparseJoinTable DenseToSparse (#1652) * SparseLinear SparseJoinTable DenseToSparse * Python api * add DenseToSparseSpec * update to upstream * add some method * meet code review * fix python unit test * fix python unit test --- python/dllib/src/bigdl/dllib/nn/layer.py | 90 ++++++++++++++++++++++++ 1 file changed, 90 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index dda84786e28..c5dfa479347 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -744,6 +744,74 @@ def set_init_method(self, weight_init_method = None, bias_init_method = None): weight_init_method, bias_init_method) return self +class SparseLinear(Layer): + + ''' + SparseLinear is the sparse version of module Linear. SparseLinear has two different from Linear: + firstly, SparseLinear's input Tensor is a SparseTensor. Secondly, SparseLinear doesn't backward + gradient to next layer in the backpropagation by default, as the gradInput of SparseLinear is + useless and very big in most cases. + + But, considering model like Wide&Deep, we provide backwardStart and backwardLength to backward + part of the gradient to next layer. + + :param input_size the size the each input sample + :param output_size the size of the module output of each sample + :param backwardStart backwardStart index, counting from 1 + :param backwardLength backward length + :param withBias if has bias + :param wRegularizer: instance of [[Regularizer]](eg. L1 or L2 regularization), applied to the input weights matrices. + :param bRegularizer: instance of [[Regularizer]]applied to the bias. + :param init_weight: the optional initial value for the weight + :param init_bias: the optional initial value for the bias + :param init_grad_weight: the optional initial value for the grad_weight + :param init_grad_bias: the optional initial value for the grad_bias + + + >>> sparselinear = SparseLinear(100, 10, True, wRegularizer=L1Regularizer(0.5), bRegularizer=L1Regularizer(0.5)) + creating: createL1Regularizer + creating: createL1Regularizer + creating: createSparseLinear + >>> import numpy as np + >>> init_weight = np.random.randn(10, 100) + >>> init_bias = np.random.randn(10) + >>> init_grad_weight = np.zeros([10, 100]) + >>> init_grad_bias = np.zeros([10]) + >>> sparselinear = SparseLinear(100, 10, True, 1, 5, L1Regularizer(0.5), L1Regularizer(0.5), init_weight, init_bias, init_grad_weight, init_grad_bias) + creating: createL1Regularizer + creating: createL1Regularizer + creating: createSparseLinear + ''' + + def __init__(self, input_size, output_size, with_bias=True, backwardStart=-1, backwardLength=-1, + wRegularizer=None, bRegularizer=None, init_weight=None, init_bias=None, + init_grad_weight=None, init_grad_bias=None, bigdl_type="float"): + super(SparseLinear, self).__init__(None, bigdl_type, input_size, output_size, + with_bias, backwardStart, backwardLength, + wRegularizer, bRegularizer, + JTensor.from_ndarray(init_weight), + JTensor.from_ndarray(init_bias), + JTensor.from_ndarray(init_grad_weight), + JTensor.from_ndarray(init_grad_bias)) + + def set_init_method(self, weight_init_method = None, bias_init_method = None): + callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, + weight_init_method, bias_init_method) + return self + +class DenseToSparse(Layer): + + ''' + Convert DenseTensor to SparseTensor. + + + >>> DenseToSparse = DenseToSparse() + creating: createDenseToSparse + ''' + + def __init__(self, + bigdl_type="float"): + super(DenseToSparse, self).__init__(None, bigdl_type) class ReLU(Layer): @@ -2294,6 +2362,28 @@ def __init__(self, dimension, n_input_dims) +class SparseJoinTable(Layer): + + ''' + :: Experimental :: + + Sparse version of JoinTable. Backward just pass the origin gradOutput back to + the next layers without split. So this layer may just works in Wide&Deep like models. + + + :param dimension: to be join in this dimension + + + >>> joinTable = SparseJoinTable(1) + creating: createSparseJoinTable + ''' + + def __init__(self, + dimension, + bigdl_type="float"): + super(SparseJoinTable, self).__init__(None, bigdl_type, + dimension) + class L1Penalty(Layer): From 84a667fe703a3e7a73328559adca8005478c3d56 Mon Sep 17 00:00:00 2001 From: Xianyan Date: Mon, 16 Oct 2017 11:28:35 +0800 Subject: [PATCH 274/823] move model freeze to AbstractModule (#1647) * move model freeze to container * update * update freeze API * refactor freeze API * meet code review and add unit test * fix doc * meet code review --- python/dllib/src/bigdl/dllib/nn/layer.py | 48 ++++++++---------------- 1 file changed, 16 insertions(+), 32 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index c5dfa479347..e466d97a58a 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -392,20 +392,24 @@ def setBRegularizer(self, bRegularizer): ''' self.value.bRegularizer = bRegularizer.value - def freeze(self): - ''' - freeze layer - ''' - callBigDlFunc(self.bigdl_type, - "setLayerFreeze", self.value) + def freeze(self, names=None): + """ + freeze module, if names is not None, set an array of layers that match given names + to be freezed + :param names: an array of layer names + :return: + """ + callBigDlFunc(self.bigdl_type, "freeze", self.value, names) return self - def unfreeze(self): - ''' - unfreeze layer - ''' - callBigDlFunc(self.bigdl_type, - "setLayerUnFreeze", self.value) + def unfreeze(self, names=None): + """ + unfreeze module, if names is not None, unfreeze layers that match given names + :param names: an array of layer names + :return: + """ + callBigDlFunc(self.bigdl_type, "unFreeze", self.value, names) + return self def training(self): ''' @@ -651,26 +655,6 @@ def train(output, data, label, opt_method, criterion, batch_size, end_when, sess jmodel = callBigDlFunc(bigdl_type, "trainTF", path, output_name, rdd_train_sample, opt_method, criterion, batch_size, end_when) return Model.of(jmodel) - - def freeze(self, freeze_layers, bigdl_type="float"): - """ - set an array of layers to be freezed - :param freeze_layers: an array of layer names - :param bigdl_type: - :return: - """ - callBigDlFunc(bigdl_type, "setFreeze", self.value, freeze_layers) - return self - - def unfreeze(self, bigdl_type="float"): - """ - set all layers to be trainable - :param bigdl_type: - :return: - """ - callBigDlFunc(bigdl_type, "unFreeze", self.value) - return self - def stop_gradient(self, stop_layers, bigdl_type="float"): """ stop the input gradient of layers that match the given ```names``` From e53bbaf3393463b6259ebec8b2409c5b6fbe7e2b Mon Sep 17 00:00:00 2001 From: Xianyan Date: Mon, 16 Oct 2017 11:28:35 +0800 Subject: [PATCH 275/823] move model freeze to AbstractModule (#1647) * move model freeze to container * update * update freeze API * refactor freeze API * meet code review and add unit test * fix doc * meet code review --- python/dllib/src/bigdl/dllib/nn/layer.py | 48 ++++++++---------------- 1 file changed, 16 insertions(+), 32 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index c5dfa479347..e466d97a58a 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -392,20 +392,24 @@ def setBRegularizer(self, bRegularizer): ''' self.value.bRegularizer = bRegularizer.value - def freeze(self): - ''' - freeze layer - ''' - callBigDlFunc(self.bigdl_type, - "setLayerFreeze", self.value) + def freeze(self, names=None): + """ + freeze module, if names is not None, set an array of layers that match given names + to be freezed + :param names: an array of layer names + :return: + """ + callBigDlFunc(self.bigdl_type, "freeze", self.value, names) return self - def unfreeze(self): - ''' - unfreeze layer - ''' - callBigDlFunc(self.bigdl_type, - "setLayerUnFreeze", self.value) + def unfreeze(self, names=None): + """ + unfreeze module, if names is not None, unfreeze layers that match given names + :param names: an array of layer names + :return: + """ + callBigDlFunc(self.bigdl_type, "unFreeze", self.value, names) + return self def training(self): ''' @@ -651,26 +655,6 @@ def train(output, data, label, opt_method, criterion, batch_size, end_when, sess jmodel = callBigDlFunc(bigdl_type, "trainTF", path, output_name, rdd_train_sample, opt_method, criterion, batch_size, end_when) return Model.of(jmodel) - - def freeze(self, freeze_layers, bigdl_type="float"): - """ - set an array of layers to be freezed - :param freeze_layers: an array of layer names - :param bigdl_type: - :return: - """ - callBigDlFunc(bigdl_type, "setFreeze", self.value, freeze_layers) - return self - - def unfreeze(self, bigdl_type="float"): - """ - set all layers to be trainable - :param bigdl_type: - :return: - """ - callBigDlFunc(bigdl_type, "unFreeze", self.value) - return self - def stop_gradient(self, stop_layers, bigdl_type="float"): """ stop the input gradient of layers that match the given ```names``` From f44a2c4e262d3ffe0d82556c973960d963f2b590 Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Tue, 17 Oct 2017 08:36:52 +0800 Subject: [PATCH 276/823] Refactor pip install and update docs (#1669) * refactor pip install * update docs for python * correct typo in docs --- .../src/bigdl/dllib/models/lenet/README.md | 5 ++++ python/dllib/src/bigdl/dllib/utils/common.py | 6 ++--- python/dllib/src/bigdl/dllib/utils/engine.py | 23 +++++++++++++++---- 3 files changed, 26 insertions(+), 8 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/models/lenet/README.md b/python/dllib/src/bigdl/dllib/models/lenet/README.md index 43aecbd5d34..7f7d156555e 100644 --- a/python/dllib/src/bigdl/dllib/models/lenet/README.md +++ b/python/dllib/src/bigdl/dllib/models/lenet/README.md @@ -70,3 +70,8 @@ INFO DistriOptimizer$:522 - Top1Accuracy is Accuracy(correct: 9572, count: 1000 ``` + +Or you can train a LeNet model directly in shell after installing BigDL from pip: +``` +python .../models/lenet/lenet5.py +``` \ No newline at end of file diff --git a/python/dllib/src/bigdl/dllib/utils/common.py b/python/dllib/src/bigdl/dllib/utils/common.py index cdb387d4f6d..8be622303f0 100644 --- a/python/dllib/src/bigdl/dllib/utils/common.py +++ b/python/dllib/src/bigdl/dllib/utils/common.py @@ -255,13 +255,13 @@ def init_engine(bigdl_type="float"): callBigDlFunc(bigdl_type, "initEngine") -def redire_spark_logs(bigdl_type="float", logPath=os.getcwd()+"/bigdl.log"): +def redire_spark_logs(bigdl_type="float", log_path=os.getcwd()+"/bigdl.log"): """ Redirect spark logs to the specified path. :param bigdl_type: "double" or "float" - :param logPath: the file path to be redirected to; the default file is under the current workspace named `bigdl.log`. + :param log_path: the file path to be redirected to; the default file is under the current workspace named `bigdl.log`. """ - callBigDlFunc(bigdl_type, "redirectSparkLogs", logPath) + callBigDlFunc(bigdl_type, "redirectSparkLogs", log_path) def show_bigdl_info_logs(bigdl_type="float"): """ diff --git a/python/dllib/src/bigdl/dllib/utils/engine.py b/python/dllib/src/bigdl/dllib/utils/engine.py index 948b8d0a888..556e1e2e1cb 100644 --- a/python/dllib/src/bigdl/dllib/utils/engine.py +++ b/python/dllib/src/bigdl/dllib/utils/engine.py @@ -18,14 +18,27 @@ import os import glob +def exist_pyspark(): + try: + import pyspark + return True + except ImportError: + return False + def __prepare_spark_env(): - modules = sys.modules - if "pyspark" not in modules or "py4j" not in modules: - spark_home = os.environ.get('SPARK_HOME', None) + spark_home = os.environ.get('SPARK_HOME', None) + if exist_pyspark(): + import pyspark + if spark_home and not pyspark.__file__.startswith(spark_home): + raise Exception( + """Find two unmatched spark sources. pyspark is found in """ + + pyspark.__file__ + ", while SPARK_HOME env is set to " + spark_home + + ". Please use one spark source only. For example, you can unset SPARK_HOME and use pyspark only.") + else: if not spark_home: raise ValueError( - """Could not find Spark. Pls make sure SPARK_HOME env is set: - export SPARK_HOME=path to your spark home directory""") + """Could not find Spark. Please make sure SPARK_HOME env is set: + export SPARK_HOME=path to your spark home directory.""") print("Using %s" % spark_home) py4j = glob.glob(os.path.join(spark_home, 'python/lib', 'py4j-*.zip'))[0] pyspark = glob.glob(os.path.join(spark_home, 'python/lib', 'pyspark*.zip'))[0] From ac5532635c6a2fdf4d2e734fbc66f616080ee288 Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Tue, 17 Oct 2017 08:36:52 +0800 Subject: [PATCH 277/823] Refactor pip install and update docs (#1669) * refactor pip install * update docs for python * correct typo in docs --- .../src/bigdl/dllib/models/lenet/README.md | 5 ++++ python/dllib/src/bigdl/utils/common.py | 6 ++--- python/dllib/src/bigdl/utils/engine.py | 23 +++++++++++++++---- 3 files changed, 26 insertions(+), 8 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/models/lenet/README.md b/python/dllib/src/bigdl/dllib/models/lenet/README.md index 43aecbd5d34..7f7d156555e 100644 --- a/python/dllib/src/bigdl/dllib/models/lenet/README.md +++ b/python/dllib/src/bigdl/dllib/models/lenet/README.md @@ -70,3 +70,8 @@ INFO DistriOptimizer$:522 - Top1Accuracy is Accuracy(correct: 9572, count: 1000 ``` + +Or you can train a LeNet model directly in shell after installing BigDL from pip: +``` +python .../models/lenet/lenet5.py +``` \ No newline at end of file diff --git a/python/dllib/src/bigdl/utils/common.py b/python/dllib/src/bigdl/utils/common.py index cdb387d4f6d..8be622303f0 100644 --- a/python/dllib/src/bigdl/utils/common.py +++ b/python/dllib/src/bigdl/utils/common.py @@ -255,13 +255,13 @@ def init_engine(bigdl_type="float"): callBigDlFunc(bigdl_type, "initEngine") -def redire_spark_logs(bigdl_type="float", logPath=os.getcwd()+"/bigdl.log"): +def redire_spark_logs(bigdl_type="float", log_path=os.getcwd()+"/bigdl.log"): """ Redirect spark logs to the specified path. :param bigdl_type: "double" or "float" - :param logPath: the file path to be redirected to; the default file is under the current workspace named `bigdl.log`. + :param log_path: the file path to be redirected to; the default file is under the current workspace named `bigdl.log`. """ - callBigDlFunc(bigdl_type, "redirectSparkLogs", logPath) + callBigDlFunc(bigdl_type, "redirectSparkLogs", log_path) def show_bigdl_info_logs(bigdl_type="float"): """ diff --git a/python/dllib/src/bigdl/utils/engine.py b/python/dllib/src/bigdl/utils/engine.py index 948b8d0a888..556e1e2e1cb 100644 --- a/python/dllib/src/bigdl/utils/engine.py +++ b/python/dllib/src/bigdl/utils/engine.py @@ -18,14 +18,27 @@ import os import glob +def exist_pyspark(): + try: + import pyspark + return True + except ImportError: + return False + def __prepare_spark_env(): - modules = sys.modules - if "pyspark" not in modules or "py4j" not in modules: - spark_home = os.environ.get('SPARK_HOME', None) + spark_home = os.environ.get('SPARK_HOME', None) + if exist_pyspark(): + import pyspark + if spark_home and not pyspark.__file__.startswith(spark_home): + raise Exception( + """Find two unmatched spark sources. pyspark is found in """ + + pyspark.__file__ + ", while SPARK_HOME env is set to " + spark_home + + ". Please use one spark source only. For example, you can unset SPARK_HOME and use pyspark only.") + else: if not spark_home: raise ValueError( - """Could not find Spark. Pls make sure SPARK_HOME env is set: - export SPARK_HOME=path to your spark home directory""") + """Could not find Spark. Please make sure SPARK_HOME env is set: + export SPARK_HOME=path to your spark home directory.""") print("Using %s" % spark_home) py4j = glob.glob(os.path.join(spark_home, 'python/lib', 'py4j-*.zip'))[0] pyspark = glob.glob(os.path.join(spark_home, 'python/lib', 'pyspark*.zip'))[0] From b6caf629bff39371b8cbdacfdf897ebd6fd8fc82 Mon Sep 17 00:00:00 2001 From: Xin Qiu Date: Wed, 18 Oct 2017 14:49:01 +0800 Subject: [PATCH 278/823] Sparse Sample&MiniBatch + python API (#1668) --- python/dllib/src/bigdl/dllib/utils/common.py | 110 ++++++++++++++++++- 1 file changed, 104 insertions(+), 6 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/utils/common.py b/python/dllib/src/bigdl/dllib/utils/common.py index 8be622303f0..1eaf0b17c10 100644 --- a/python/dllib/src/bigdl/dllib/utils/common.py +++ b/python/dllib/src/bigdl/dllib/utils/common.py @@ -126,19 +126,31 @@ class JTensor(object): >>> np.random.seed(123) >>> """ - def __init__(self, storage, shape, bigdl_type="float"): + def __init__(self, storage, shape, bigdl_type="float", indices=None): + """ + + :param storage: values in this tensor + :param shape: shape of this tensor + :param bigdl_type: numeric type + :param indices: if indices is provided, means this is a SparseTensor; + if not provided, means this is a DenseTensor + """ if isinstance(storage, bytes) and isinstance(shape, bytes): self.storage = np.frombuffer(storage, dtype=get_dtype(bigdl_type)) self.shape = np.frombuffer(shape, dtype=np.int32) else: self.storage = np.array(storage, dtype=get_dtype(bigdl_type)) self.shape = np.array(shape, dtype=np.int32) + if indices is not None: + self.indices = np.array(indices, dtype=np.int32) + else: + self.indices = None self.bigdl_type = bigdl_type @classmethod def from_ndarray(cls, a_ndarray, bigdl_type="float"): """ - Convert a ndarray to Tensor which would be used in Java side. + Convert a ndarray to a DenseTensor which would be used in Java side. >>> import numpy as np >>> from bigdl.util.common import JTensor @@ -160,19 +172,77 @@ def from_ndarray(cls, a_ndarray, bigdl_type="float"): "input should be a np.ndarray, not %s" % type(a_ndarray) return cls(a_ndarray, a_ndarray.shape if a_ndarray.shape else (a_ndarray.size), - bigdl_type= bigdl_type) + bigdl_type=bigdl_type) + + @classmethod + def sparse(cls, a_ndarray, i_ndarray, shape, bigdl_type="float"): + """ + Convert a three ndarray to SparseTensor which would be used in Java side. + For example: + a_ndarray = [1, 3, 2, 4] + i_ndarray = [[0, 0, 1, 2], + [0, 3, 2, 1]] + shape = [3, 4] + Present a dense tensor + [[ 1, 0, 0, 3], + [ 0, 0, 2, 0], + [ 0, 4, 0, 0]] + + :param a_ndarray non-zero elements in this SparseTensor + :param i_ndarray zero-based indices for non-zero element + i_ndarray's shape should be (shape.size, a_ndarray.size) + And the i-th non-zero elements indices is i_ndarray[:, 1] + :param shape shape as a DenseTensor. + + >>> import numpy as np + >>> from bigdl.util.common import JTensor + >>> from bigdl.util.common import callBigDlFunc + >>> np.random.seed(123) + >>> data = np.arrange(1, 7).astype("float32") + >>> indices = np.arange(1, 7) + >>> shape = np.array([10]) + >>> result = JTensor.sparse(data, indices, shape) + >>> tensor1 = callBigDlFunc("float", "testTensor", result) # noqa + >>> array_from_tensor = tensor1.to_ndarray() + >>> expected_ndarray = np.array([0, 1, 2, 3, 4, 5, 6, 7, 0, 0]) + >>> (array_from_tensor == expected_ndarray).all() + True + """ + if a_ndarray is None: + return None + assert isinstance(a_ndarray, np.ndarray), \ + "values array should be a np.ndarray, not %s" % type(a_ndarray) + assert isinstance(i_ndarray, np.ndarray), \ + "indices array should be a np.ndarray, not %s" % type(a_ndarray) + assert i_ndarray.size == a_ndarray.size * shape.size, \ + "size of values and indices should match." + return cls(a_ndarray, + shape, + i_ndarray, + bigdl_type= bigdl_type) def to_ndarray(self): + """ + Transfer JTensor to ndarray. + As SparseTensor may generate an very big ndarray, so we don't support this function for SparseTensor. + :return: a ndarray + """ + assert self.indices is None, "sparseTensor to ndarray is not supported" return np.array(self.storage, dtype=get_dtype(self.bigdl_type)).reshape(self.shape) # noqa def __reduce__(self): - return JTensor, (self.storage.tostring(), self.shape.tostring(), self.bigdl_type) + if self.indices is None: + return JTensor, (self.storage.tostring(), self.shape.tostring(), self.bigdl_type) + else: + return JTensor, (self.storage.tostring(), self.shape.tostring(), self.bigdl_type, self.indices.tostring()) def __str__(self): - return "JTensor: storage: %s, shape: %s" % (self.storage, self.shape) + indices = "" if self.indices is None else ",indices %s" % self.indices + return "JTensor: storage: %s, shape: %s %s" % (self.storage, self.shape, self.shape, indices) def __repr__(self): - return "JTensor: storage: %s, shape: %s" % (self.storage, self.shape) + indices = "" if self.indices is None else ",indices %s" % self.indices + return "JTensor: storage: %s, shape: %s %s" % (self.storage, self.shape, self.shape, indices) class Sample(object): @@ -215,6 +285,34 @@ def from_ndarray(cls, features, label, bigdl_type="float"): label=JTensor.from_ndarray(label), bigdl_type=bigdl_type) + @classmethod + def from_jtensor(cls, features, label, bigdl_type="float"): + """ + Convert a sequence of JTensor to Sample, which would be used in Java side. + :param features: an JTensor or a list of JTensor + :param label: an JTensor or a scalar + :param bigdl_type: "double" or "float" + + >>> import numpy as np + >>> data = np.random.uniform(0, 1, (6)).astype("float32") + >>> indices = np.arange(1, 7) + >>> shape = np.array([10]) + >>> feature0 = JTensor.from_ndarray(data, indices, shape) + >>> feature1 = JTensor.from_ndarray(np.random((2, 3))) + >>> sample = Sample.from_jtensor([feature0, feature1], 1) + """ + if isinstance(features, JTensor): + features = [features] + else: + assert all(isinstance(feature, JTensor) for feature in features), \ + "features should be a list of JTensor, not %s" % type(features) + if not isinstance(label, JTensor): # in case label is a scalar. + label = JTensor.from_ndarray(np.array(label)) + return cls( + features=features, + label=label, + bigdl_type=bigdl_type) + def __reduce__(self): return Sample, (self.features, self.label, self.bigdl_type) From 7b4051a1b96b0b9958d1985024c8e9001a6e9609 Mon Sep 17 00:00:00 2001 From: Xin Qiu Date: Wed, 18 Oct 2017 14:49:01 +0800 Subject: [PATCH 279/823] Sparse Sample&MiniBatch + python API (#1668) --- python/dllib/src/bigdl/utils/common.py | 110 +++++++++++++++++++++++-- 1 file changed, 104 insertions(+), 6 deletions(-) diff --git a/python/dllib/src/bigdl/utils/common.py b/python/dllib/src/bigdl/utils/common.py index 8be622303f0..1eaf0b17c10 100644 --- a/python/dllib/src/bigdl/utils/common.py +++ b/python/dllib/src/bigdl/utils/common.py @@ -126,19 +126,31 @@ class JTensor(object): >>> np.random.seed(123) >>> """ - def __init__(self, storage, shape, bigdl_type="float"): + def __init__(self, storage, shape, bigdl_type="float", indices=None): + """ + + :param storage: values in this tensor + :param shape: shape of this tensor + :param bigdl_type: numeric type + :param indices: if indices is provided, means this is a SparseTensor; + if not provided, means this is a DenseTensor + """ if isinstance(storage, bytes) and isinstance(shape, bytes): self.storage = np.frombuffer(storage, dtype=get_dtype(bigdl_type)) self.shape = np.frombuffer(shape, dtype=np.int32) else: self.storage = np.array(storage, dtype=get_dtype(bigdl_type)) self.shape = np.array(shape, dtype=np.int32) + if indices is not None: + self.indices = np.array(indices, dtype=np.int32) + else: + self.indices = None self.bigdl_type = bigdl_type @classmethod def from_ndarray(cls, a_ndarray, bigdl_type="float"): """ - Convert a ndarray to Tensor which would be used in Java side. + Convert a ndarray to a DenseTensor which would be used in Java side. >>> import numpy as np >>> from bigdl.util.common import JTensor @@ -160,19 +172,77 @@ def from_ndarray(cls, a_ndarray, bigdl_type="float"): "input should be a np.ndarray, not %s" % type(a_ndarray) return cls(a_ndarray, a_ndarray.shape if a_ndarray.shape else (a_ndarray.size), - bigdl_type= bigdl_type) + bigdl_type=bigdl_type) + + @classmethod + def sparse(cls, a_ndarray, i_ndarray, shape, bigdl_type="float"): + """ + Convert a three ndarray to SparseTensor which would be used in Java side. + For example: + a_ndarray = [1, 3, 2, 4] + i_ndarray = [[0, 0, 1, 2], + [0, 3, 2, 1]] + shape = [3, 4] + Present a dense tensor + [[ 1, 0, 0, 3], + [ 0, 0, 2, 0], + [ 0, 4, 0, 0]] + + :param a_ndarray non-zero elements in this SparseTensor + :param i_ndarray zero-based indices for non-zero element + i_ndarray's shape should be (shape.size, a_ndarray.size) + And the i-th non-zero elements indices is i_ndarray[:, 1] + :param shape shape as a DenseTensor. + + >>> import numpy as np + >>> from bigdl.util.common import JTensor + >>> from bigdl.util.common import callBigDlFunc + >>> np.random.seed(123) + >>> data = np.arrange(1, 7).astype("float32") + >>> indices = np.arange(1, 7) + >>> shape = np.array([10]) + >>> result = JTensor.sparse(data, indices, shape) + >>> tensor1 = callBigDlFunc("float", "testTensor", result) # noqa + >>> array_from_tensor = tensor1.to_ndarray() + >>> expected_ndarray = np.array([0, 1, 2, 3, 4, 5, 6, 7, 0, 0]) + >>> (array_from_tensor == expected_ndarray).all() + True + """ + if a_ndarray is None: + return None + assert isinstance(a_ndarray, np.ndarray), \ + "values array should be a np.ndarray, not %s" % type(a_ndarray) + assert isinstance(i_ndarray, np.ndarray), \ + "indices array should be a np.ndarray, not %s" % type(a_ndarray) + assert i_ndarray.size == a_ndarray.size * shape.size, \ + "size of values and indices should match." + return cls(a_ndarray, + shape, + i_ndarray, + bigdl_type= bigdl_type) def to_ndarray(self): + """ + Transfer JTensor to ndarray. + As SparseTensor may generate an very big ndarray, so we don't support this function for SparseTensor. + :return: a ndarray + """ + assert self.indices is None, "sparseTensor to ndarray is not supported" return np.array(self.storage, dtype=get_dtype(self.bigdl_type)).reshape(self.shape) # noqa def __reduce__(self): - return JTensor, (self.storage.tostring(), self.shape.tostring(), self.bigdl_type) + if self.indices is None: + return JTensor, (self.storage.tostring(), self.shape.tostring(), self.bigdl_type) + else: + return JTensor, (self.storage.tostring(), self.shape.tostring(), self.bigdl_type, self.indices.tostring()) def __str__(self): - return "JTensor: storage: %s, shape: %s" % (self.storage, self.shape) + indices = "" if self.indices is None else ",indices %s" % self.indices + return "JTensor: storage: %s, shape: %s %s" % (self.storage, self.shape, self.shape, indices) def __repr__(self): - return "JTensor: storage: %s, shape: %s" % (self.storage, self.shape) + indices = "" if self.indices is None else ",indices %s" % self.indices + return "JTensor: storage: %s, shape: %s %s" % (self.storage, self.shape, self.shape, indices) class Sample(object): @@ -215,6 +285,34 @@ def from_ndarray(cls, features, label, bigdl_type="float"): label=JTensor.from_ndarray(label), bigdl_type=bigdl_type) + @classmethod + def from_jtensor(cls, features, label, bigdl_type="float"): + """ + Convert a sequence of JTensor to Sample, which would be used in Java side. + :param features: an JTensor or a list of JTensor + :param label: an JTensor or a scalar + :param bigdl_type: "double" or "float" + + >>> import numpy as np + >>> data = np.random.uniform(0, 1, (6)).astype("float32") + >>> indices = np.arange(1, 7) + >>> shape = np.array([10]) + >>> feature0 = JTensor.from_ndarray(data, indices, shape) + >>> feature1 = JTensor.from_ndarray(np.random((2, 3))) + >>> sample = Sample.from_jtensor([feature0, feature1], 1) + """ + if isinstance(features, JTensor): + features = [features] + else: + assert all(isinstance(feature, JTensor) for feature in features), \ + "features should be a list of JTensor, not %s" % type(features) + if not isinstance(label, JTensor): # in case label is a scalar. + label = JTensor.from_ndarray(np.array(label)) + return cls( + features=features, + label=label, + bigdl_type=bigdl_type) + def __reduce__(self): return Sample, (self.features, self.label, self.bigdl_type) From fa046db7636e482620ab5834da4f23d9f7ec56c2 Mon Sep 17 00:00:00 2001 From: Ian Wong Date: Thu, 19 Oct 2017 09:56:14 +0800 Subject: [PATCH 280/823] add more layers and operations (#1678) * add more layers and operations * fix compile error * fix compile error --- python/dllib/src/bigdl/dllib/nn/layer.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index e466d97a58a..212fdc5ea5a 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -3824,6 +3824,23 @@ def __init__(self, v, ip) +class Negative(Layer): + + ''' + Create an Negative layer. Computing negative value of each element of input tensor + + :param inplace: if output tensor reuse input tensor storage. Default value is false + + + >>> negative = Negative(False) + creating: createNegative + ''' + + def __init__(self, + inplace = False, + bigdl_type="float"): + super(Negative, self).__init__(None, bigdl_type, inplace) + class Unsqueeze(Layer): From bc22d4e5f4b2aea9b617cd29298c71a8747eaae2 Mon Sep 17 00:00:00 2001 From: Ian Wong Date: Thu, 19 Oct 2017 09:56:14 +0800 Subject: [PATCH 281/823] add more layers and operations (#1678) * add more layers and operations * fix compile error * fix compile error --- python/dllib/src/bigdl/dllib/nn/layer.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index e466d97a58a..212fdc5ea5a 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -3824,6 +3824,23 @@ def __init__(self, v, ip) +class Negative(Layer): + + ''' + Create an Negative layer. Computing negative value of each element of input tensor + + :param inplace: if output tensor reuse input tensor storage. Default value is false + + + >>> negative = Negative(False) + creating: createNegative + ''' + + def __init__(self, + inplace = False, + bigdl_type="float"): + super(Negative, self).__init__(None, bigdl_type, inplace) + class Unsqueeze(Layer): From 57d097359c2be3c31b600b1d058f08693ec135be Mon Sep 17 00:00:00 2001 From: Ian Wong Date: Thu, 19 Oct 2017 14:35:17 +0800 Subject: [PATCH 282/823] Support load more tensorflow operations (#1656) * Support load more tf operation * refine session * fix style issue * meet code reivew --- python/dllib/src/bigdl/dllib/utils/tf_utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/dllib/src/bigdl/dllib/utils/tf_utils.py b/python/dllib/src/bigdl/dllib/utils/tf_utils.py index 5cd6377417d..7e8d071daca 100644 --- a/python/dllib/src/bigdl/dllib/utils/tf_utils.py +++ b/python/dllib/src/bigdl/dllib/utils/tf_utils.py @@ -157,7 +157,8 @@ def dump_model(path, sess=None, graph=None, bigdl_type="float"): # dump grap to pb file graph = sess.graph if graph is None else graph - tf.train.write_graph(graph, path, 'model.pb') + with gfile.GFile(path + "/model.pb", "wb") as f: + f.write(graph.as_graph_def().SerializeToString()) try: shutil.rmtree(temp) except OSError as e: From f8748673c0680f8a312a49f3b3c5f0870f2b0e6c Mon Sep 17 00:00:00 2001 From: Ian Wong Date: Thu, 19 Oct 2017 14:35:17 +0800 Subject: [PATCH 283/823] Support load more tensorflow operations (#1656) * Support load more tf operation * refine session * fix style issue * meet code reivew --- python/dllib/src/bigdl/dllib/utils/tf_utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/dllib/src/bigdl/dllib/utils/tf_utils.py b/python/dllib/src/bigdl/dllib/utils/tf_utils.py index 5cd6377417d..7e8d071daca 100644 --- a/python/dllib/src/bigdl/dllib/utils/tf_utils.py +++ b/python/dllib/src/bigdl/dllib/utils/tf_utils.py @@ -157,7 +157,8 @@ def dump_model(path, sess=None, graph=None, bigdl_type="float"): # dump grap to pb file graph = sess.graph if graph is None else graph - tf.train.write_graph(graph, path, 'model.pb') + with gfile.GFile(path + "/model.pb", "wb") as f: + f.write(graph.as_graph_def().SerializeToString()) try: shutil.rmtree(temp) except OSError as e: From 950149f13cc8458e6567916e264d017c98cb064c Mon Sep 17 00:00:00 2001 From: Ian Wong Date: Thu, 19 Oct 2017 17:02:21 +0800 Subject: [PATCH 284/823] populate and check spark.speculation to spark conf (#1683) * populate and check spark.speculation to spark conf * fix an error in pytest --- python/dllib/test/bigdl/test_simple_integration.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python/dllib/test/bigdl/test_simple_integration.py b/python/dllib/test/bigdl/test_simple_integration.py index c2bcf681a89..ada34dcde4b 100644 --- a/python/dllib/test/bigdl/test_simple_integration.py +++ b/python/dllib/test/bigdl/test_simple_integration.py @@ -172,6 +172,7 @@ def test_save_graph_topology(self): def test_load_zip_conf(self): from bigdl.util.common import get_bigdl_conf import sys + sys_path_back = sys.path sys.path = [path for path in sys.path if "spark-bigdl.conf" not in path] sys.path.insert(0, os.path.join(os.path.split(__file__)[0], "resources/conf/python-api.zip")) # noqa @@ -179,6 +180,7 @@ def test_load_zip_conf(self): "resources/conf/invalid-python-api.zip")) # noqa result = get_bigdl_conf() assert result.get("spark.executorEnv.OMP_WAIT_POLICY"), "passive" + sys.path = sys_path_back def test_set_seed(self): w_init = Xavier() From d52181888ab2f9f832a5fd360526897fe3f72d92 Mon Sep 17 00:00:00 2001 From: Xianyan Date: Fri, 20 Oct 2017 13:44:15 +0800 Subject: [PATCH 285/823] fix 'Sample' object has no attribute 'storage' (#1686) --- python/dllib/src/bigdl/dllib/utils/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/dllib/src/bigdl/dllib/utils/common.py b/python/dllib/src/bigdl/dllib/utils/common.py index 1eaf0b17c10..7fa9fc085fc 100644 --- a/python/dllib/src/bigdl/dllib/utils/common.py +++ b/python/dllib/src/bigdl/dllib/utils/common.py @@ -320,7 +320,7 @@ def __str__(self): return "Sample: features: %s, label: %s," % (self.features, self.label) def __repr__(self): - return "Sample: features: %s, label: %s" % (self.storage, self.shape) + return "Sample: features: %s, label: %s" % (self.features, self.label) class RNG(): """ From 17d03246b6b41147d79942122ba03eadb4e8a75d Mon Sep 17 00:00:00 2001 From: Xianyan Date: Fri, 20 Oct 2017 13:44:15 +0800 Subject: [PATCH 286/823] fix 'Sample' object has no attribute 'storage' (#1686) --- python/dllib/src/bigdl/utils/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/dllib/src/bigdl/utils/common.py b/python/dllib/src/bigdl/utils/common.py index 1eaf0b17c10..7fa9fc085fc 100644 --- a/python/dllib/src/bigdl/utils/common.py +++ b/python/dllib/src/bigdl/utils/common.py @@ -320,7 +320,7 @@ def __str__(self): return "Sample: features: %s, label: %s," % (self.features, self.label) def __repr__(self): - return "Sample: features: %s, label: %s" % (self.storage, self.shape) + return "Sample: features: %s, label: %s" % (self.features, self.label) class RNG(): """ From 16a2f8dfb6e6b2f47ef7141f29d2ef82cd1f127d Mon Sep 17 00:00:00 2001 From: megaSpoon Date: Mon, 23 Oct 2017 03:05:47 -0700 Subject: [PATCH 287/823] feat: ML Pipeline: (#1654) * DLEstimator, DLClassifier, DLModel, DLClassiferModel API * add unit test test_dl_classifier --- .../dllib/models/ml_pipeline/__init__.py | 0 .../dllib/models/ml_pipeline/dl_classifier.py | 150 ++++++++++++++++++ 2 files changed, 150 insertions(+) create mode 100644 python/dllib/src/bigdl/dllib/models/ml_pipeline/__init__.py create mode 100644 python/dllib/src/bigdl/dllib/models/ml_pipeline/dl_classifier.py diff --git a/python/dllib/src/bigdl/dllib/models/ml_pipeline/__init__.py b/python/dllib/src/bigdl/dllib/models/ml_pipeline/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/python/dllib/src/bigdl/dllib/models/ml_pipeline/dl_classifier.py b/python/dllib/src/bigdl/dllib/models/ml_pipeline/dl_classifier.py new file mode 100644 index 00000000000..4dc4cd96248 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/models/ml_pipeline/dl_classifier.py @@ -0,0 +1,150 @@ +from pyspark.ml.pipeline import Estimator, Model +from pyspark.ml.param.shared import * +from bigdl.util.common import * + + +if sys.version >= '3': + long = int + unicode = str + + +class HasBatchSize(Params): + """ + Mixin for param batchSize: batch size. + """ + + # a placeholder to make it appear in the generated doc + batchSize = Param(Params._dummy(), "batchSize", "batchSize") + + def __init__(self): + super(HasBatchSize, self).__init__() + #: param for batch size. + self.batchSize = Param(self, "batchSize", "batchSize") + self._setDefault(batchSize=1) + + def setBatchSize(self, val): + """ + Sets the value of :py:attr:`batchSize`. + """ + self._paramMap[self.batchSize] = val + pythonBigDL_method_name = "setBatchSize" + self.__class__.__name__ + callBigDlFunc(self.bigdl_type, pythonBigDL_method_name, self.value, val) + return self + + def getBatchSize(self): + """ + Gets the value of batchSize or its default value. + """ + return self.getOrDefault(self.batchSize) + +class HasMaxEpoch(Params): + maxEpoch = Param(Params._dummy(), "maxEpoch", "number of max Epoch") + + def __init__(self): + super(HasMaxEpoch, self).__init__() + self.maxEpoch = Param(self, "maxEpoch", "maxEpoch") + self._setDefault(maxEpoch=100) + + def setMaxEpoch(self, val): + self._paramMap[self.maxEpoch] = val + pythonBigDL_method_name = "setMaxEpoch" + self.__class__.__name__ + callBigDlFunc(self.bigdl_type, pythonBigDL_method_name, self.value, val) + return self + + def getMaxEpoch(self): + """ + Gets the value of maxEpoch or its default value. + """ + return self.getOrDefault(self.maxEpoch) + +class HasFeatureSize(Params): + featureSize = Param(Params._dummy(), "featureSize", "size of the feature") + + def __init__(self): + super(HasFeatureSize, self).__init__() + self.featureSize = Param(self, "featureSize", "featureSize") + self._setDefault(featureSize=None) + + def setFeatureSize(self, val): + self._paramMap[self.featureSize] = val + pythonBigDL_mehtod_name = "setFeatureSize" + self.__class__.__name__ + callBigDlFunc(self.bigdl_type, pythonBigDL_mehtod_name, self.value, val) + return self + + def getFeatureSize(self): + return self.getOrDefault(self.featureSize) + +class HasLearningRate(Params): + learningRate = Param(Params._dummy(), "learningRate", "learning rate") + + def __init__(self): + super(HasLearningRate, self).__init__() + self.learningRate = Param(self, "learningRate", "learning rate") + self._setDefault(learningRate=100) + + def setLearningRate(self, val): + self._paramMap[self.learningRate] = val + pythonBigDL_method_name = "setLearningRate" + self.__class__.__name__ + callBigDlFunc(self.bigdl_type, pythonBigDL_method_name, self.value, val) + return self + + def getLearningRate(self): + """ + Gets the value of maxEpoch or its default value. + """ + return self.getOrDefault(self.learningRate) + +class DLEstimator(Estimator, HasFeaturesCol, HasLabelCol, HasPredictionCol, HasBatchSize, HasMaxEpoch, HasLearningRate, JavaValue): + + def __init__(self, model, criterion, feature_size, label_size, jvalue=None, bigdl_type="float"): + super(DLEstimator, self).__init__() + self.value = jvalue if jvalue else callBigDlFunc( + bigdl_type, self.jvm_class_constructor(), model, criterion, feature_size, label_size) + self.bigdl_type = bigdl_type + self.featureSize = feature_size + + def _fit(self, dataset): + #self._transfer_params_to_java() + jmodel = callBigDlFunc(self.bigdl_type, "fitEstimator", self.value, dataset) + model = DLModel.of(jmodel, self.featureSize, self.bigdl_type) + return model + + +class DLModel(Model, HasFeaturesCol, HasPredictionCol, HasBatchSize, HasFeatureSize, JavaValue): + def __init__(self, model, featureSize, jvalue=None, bigdl_type="float"): + super(DLModel, self).__init__() + self.value = jvalue if jvalue else callBigDlFunc( + bigdl_type, self.jvm_class_constructor(), model, featureSize) + self.bigdl_type = bigdl_type + self.setFeatureSize(featureSize) + + def _transform(self, dataset): + return callBigDlFunc(self.bigdl_type, "dlModelTransform", self.value, dataset) + + @classmethod + def of(self, jvalue, feature_size=None, bigdl_type="float"): + model = DLModel(model=None, featureSize=feature_size, jvalue=jvalue, bigdl_type=bigdl_type) + return model + + +class DLClassifier(DLEstimator): + def __init__(self, model, criterion, feature_size, bigdl_type="float"): + super(DLClassifier, self).__init__(model, criterion, feature_size, [1], None, bigdl_type) + + def _fit(self, dataset): + jmodel = callBigDlFunc(self.bigdl_type, "fitClassifier", self.value, dataset) + model = DLClassifierModel.of(jmodel, self.featureSize, self.bigdl_type) + return model + + +class DLClassifierModel(DLModel): + def __init__(self, model, featureSize, jvalue=None, bigdl_type="float"): + super(DLClassifierModel, self).__init__(model, featureSize, jvalue, bigdl_type) + + def _transform(self, dataset): + return callBigDlFunc(self.bigdl_type, "dlClassifierModelTransform", self.value, dataset) + + @classmethod + def of(self, jvalue, feature_size=None, bigdl_type="float"): + model = DLClassifierModel(model=None, featureSize=feature_size, jvalue=jvalue, bigdl_type=bigdl_type) + return model \ No newline at end of file From 4ec5adfc65b0ee489ef225e255de0cacff3b9256 Mon Sep 17 00:00:00 2001 From: megaSpoon Date: Mon, 23 Oct 2017 03:05:47 -0700 Subject: [PATCH 288/823] feat: ML Pipeline: (#1654) * DLEstimator, DLClassifier, DLModel, DLClassiferModel API * add unit test test_dl_classifier --- .../dllib/models/ml_pipeline/__init__.py | 0 .../dllib/models/ml_pipeline/dl_classifier.py | 150 ++++++++++++++++++ 2 files changed, 150 insertions(+) create mode 100644 python/dllib/src/bigdl/dllib/models/ml_pipeline/__init__.py create mode 100644 python/dllib/src/bigdl/dllib/models/ml_pipeline/dl_classifier.py diff --git a/python/dllib/src/bigdl/dllib/models/ml_pipeline/__init__.py b/python/dllib/src/bigdl/dllib/models/ml_pipeline/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/python/dllib/src/bigdl/dllib/models/ml_pipeline/dl_classifier.py b/python/dllib/src/bigdl/dllib/models/ml_pipeline/dl_classifier.py new file mode 100644 index 00000000000..4dc4cd96248 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/models/ml_pipeline/dl_classifier.py @@ -0,0 +1,150 @@ +from pyspark.ml.pipeline import Estimator, Model +from pyspark.ml.param.shared import * +from bigdl.util.common import * + + +if sys.version >= '3': + long = int + unicode = str + + +class HasBatchSize(Params): + """ + Mixin for param batchSize: batch size. + """ + + # a placeholder to make it appear in the generated doc + batchSize = Param(Params._dummy(), "batchSize", "batchSize") + + def __init__(self): + super(HasBatchSize, self).__init__() + #: param for batch size. + self.batchSize = Param(self, "batchSize", "batchSize") + self._setDefault(batchSize=1) + + def setBatchSize(self, val): + """ + Sets the value of :py:attr:`batchSize`. + """ + self._paramMap[self.batchSize] = val + pythonBigDL_method_name = "setBatchSize" + self.__class__.__name__ + callBigDlFunc(self.bigdl_type, pythonBigDL_method_name, self.value, val) + return self + + def getBatchSize(self): + """ + Gets the value of batchSize or its default value. + """ + return self.getOrDefault(self.batchSize) + +class HasMaxEpoch(Params): + maxEpoch = Param(Params._dummy(), "maxEpoch", "number of max Epoch") + + def __init__(self): + super(HasMaxEpoch, self).__init__() + self.maxEpoch = Param(self, "maxEpoch", "maxEpoch") + self._setDefault(maxEpoch=100) + + def setMaxEpoch(self, val): + self._paramMap[self.maxEpoch] = val + pythonBigDL_method_name = "setMaxEpoch" + self.__class__.__name__ + callBigDlFunc(self.bigdl_type, pythonBigDL_method_name, self.value, val) + return self + + def getMaxEpoch(self): + """ + Gets the value of maxEpoch or its default value. + """ + return self.getOrDefault(self.maxEpoch) + +class HasFeatureSize(Params): + featureSize = Param(Params._dummy(), "featureSize", "size of the feature") + + def __init__(self): + super(HasFeatureSize, self).__init__() + self.featureSize = Param(self, "featureSize", "featureSize") + self._setDefault(featureSize=None) + + def setFeatureSize(self, val): + self._paramMap[self.featureSize] = val + pythonBigDL_mehtod_name = "setFeatureSize" + self.__class__.__name__ + callBigDlFunc(self.bigdl_type, pythonBigDL_mehtod_name, self.value, val) + return self + + def getFeatureSize(self): + return self.getOrDefault(self.featureSize) + +class HasLearningRate(Params): + learningRate = Param(Params._dummy(), "learningRate", "learning rate") + + def __init__(self): + super(HasLearningRate, self).__init__() + self.learningRate = Param(self, "learningRate", "learning rate") + self._setDefault(learningRate=100) + + def setLearningRate(self, val): + self._paramMap[self.learningRate] = val + pythonBigDL_method_name = "setLearningRate" + self.__class__.__name__ + callBigDlFunc(self.bigdl_type, pythonBigDL_method_name, self.value, val) + return self + + def getLearningRate(self): + """ + Gets the value of maxEpoch or its default value. + """ + return self.getOrDefault(self.learningRate) + +class DLEstimator(Estimator, HasFeaturesCol, HasLabelCol, HasPredictionCol, HasBatchSize, HasMaxEpoch, HasLearningRate, JavaValue): + + def __init__(self, model, criterion, feature_size, label_size, jvalue=None, bigdl_type="float"): + super(DLEstimator, self).__init__() + self.value = jvalue if jvalue else callBigDlFunc( + bigdl_type, self.jvm_class_constructor(), model, criterion, feature_size, label_size) + self.bigdl_type = bigdl_type + self.featureSize = feature_size + + def _fit(self, dataset): + #self._transfer_params_to_java() + jmodel = callBigDlFunc(self.bigdl_type, "fitEstimator", self.value, dataset) + model = DLModel.of(jmodel, self.featureSize, self.bigdl_type) + return model + + +class DLModel(Model, HasFeaturesCol, HasPredictionCol, HasBatchSize, HasFeatureSize, JavaValue): + def __init__(self, model, featureSize, jvalue=None, bigdl_type="float"): + super(DLModel, self).__init__() + self.value = jvalue if jvalue else callBigDlFunc( + bigdl_type, self.jvm_class_constructor(), model, featureSize) + self.bigdl_type = bigdl_type + self.setFeatureSize(featureSize) + + def _transform(self, dataset): + return callBigDlFunc(self.bigdl_type, "dlModelTransform", self.value, dataset) + + @classmethod + def of(self, jvalue, feature_size=None, bigdl_type="float"): + model = DLModel(model=None, featureSize=feature_size, jvalue=jvalue, bigdl_type=bigdl_type) + return model + + +class DLClassifier(DLEstimator): + def __init__(self, model, criterion, feature_size, bigdl_type="float"): + super(DLClassifier, self).__init__(model, criterion, feature_size, [1], None, bigdl_type) + + def _fit(self, dataset): + jmodel = callBigDlFunc(self.bigdl_type, "fitClassifier", self.value, dataset) + model = DLClassifierModel.of(jmodel, self.featureSize, self.bigdl_type) + return model + + +class DLClassifierModel(DLModel): + def __init__(self, model, featureSize, jvalue=None, bigdl_type="float"): + super(DLClassifierModel, self).__init__(model, featureSize, jvalue, bigdl_type) + + def _transform(self, dataset): + return callBigDlFunc(self.bigdl_type, "dlClassifierModelTransform", self.value, dataset) + + @classmethod + def of(self, jvalue, feature_size=None, bigdl_type="float"): + model = DLClassifierModel(model=None, featureSize=feature_size, jvalue=jvalue, bigdl_type=bigdl_type) + return model \ No newline at end of file From 0229a26e6fda94d9e4378abb9df68f5545b694ce Mon Sep 17 00:00:00 2001 From: megaSpoon Date: Mon, 23 Oct 2017 03:05:47 -0700 Subject: [PATCH 289/823] feat: ML Pipeline: (#1654) * DLEstimator, DLClassifier, DLModel, DLClassiferModel API * add unit test test_dl_classifier --- python/dllib/test/bigdl/test_dl_classifier.py | 170 ++++++++++++++++++ 1 file changed, 170 insertions(+) create mode 100644 python/dllib/test/bigdl/test_dl_classifier.py diff --git a/python/dllib/test/bigdl/test_dl_classifier.py b/python/dllib/test/bigdl/test_dl_classifier.py new file mode 100644 index 00000000000..2514cca1a77 --- /dev/null +++ b/python/dllib/test/bigdl/test_dl_classifier.py @@ -0,0 +1,170 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from bigdl.nn.layer import * +from bigdl.nn.criterion import * +from bigdl.util.common import * +from bigdl.models.ml_pipeline.dl_classifier import * +from pyspark.sql.types import * +from pyspark.context import SparkContext +import numpy as np +import pytest +from numpy.testing import assert_allclose + + +class TestDLClassifer(): + def setup_method(self, method): + """ setup any state tied to the execution of the given method in a + class. setup_method is invoked for every test method of a class. + """ + sparkConf = create_spark_conf() + self.sc = SparkContext(master="local[1]", appName="test model", + conf=sparkConf) + self.sqlContext = SQLContext(self.sc) + init_engine() + + def teardown_method(self, method): + """ teardown any state that was previously setup with a setup_method + call. + """ + self.sc.stop() + + def test_all_set_get_methods(self): + """ run tests for all the set and get methods involved in DLEstimator, DLModel, + DLClassifier, DLClassifierModel + """ + + ''' + use linear model and MSE criterion to test the DLEstimator and DLModel + ''' + linear_model = Sequential().add(Linear(2, 2)) + mse_criterion = MSECriterion() + + ''' + initialize a DLEstimator to test setter, getter for + batchSize, maxEpoch, learningRate in DLEstimator + ''' + estimator = DLEstimator(model=linear_model, criterion=mse_criterion, + feature_size=[2], label_size=[2]) + + # check the set and get methods for batchSize + assert estimator.setBatchSize(30).getBatchSize() == 30 + # check the set and get methods for maxEpoch + assert estimator.setMaxEpoch(40).getMaxEpoch() == 40 + # check the set and get methods for learningRate + assert estimator.setLearningRate(1e-4).getLearningRate() == 1e-4 + + ''' + initialize a DLModel to test setter, getter for featureSize, batchSize in DLModel + ''' + dl_model = DLClassifierModel(model=linear_model, featureSize=[1]) + + # check the set and get methods for featureSize in DLModel + assert dl_model.setFeatureSize([2]).getFeatureSize() == [2] + # check the set and get methods for batchSize in DLModel + assert dl_model.setBatchSize((20)).getBatchSize() == 20 + + ''' + use linear model and ClassNLL criterion to test the DLClassifier and DLClassifierModel + ''' + linear_model = Sequential().add(Linear(2, 2)) + classNLL_criterion = ClassNLLCriterion() + + ''' + initialize a DLClassifier to test setter, getter for + batchSize, maxEpoch, learningRate in DLClassifier + ''' + classifier = DLClassifier(model=linear_model, criterion=classNLL_criterion, + feature_size=[2]) + + # check the set and get methods for batchSize + assert classifier.setBatchSize(20).getBatchSize() == 20 + # check the set and get methods for maxEpoch + assert classifier.setMaxEpoch(50).getMaxEpoch() == 50 + # check the set and get methods for learningRate + assert classifier.setLearningRate(1e-5).getLearningRate() == 1e-5 + ''' + initialize a DLClassifierModel to test setter, getter + for featureSize, batchSize in DLClassifierModel + ''' + dl_classifier_model = DLClassifierModel(model=linear_model, featureSize=[1]) + + # check the set and get methods for featureSize + assert dl_classifier_model.setFeatureSize([2]).getFeatureSize() == [2] + + # check the set and get methods for batchSize + assert dl_classifier_model.setBatchSize((20)).getBatchSize() == 20 + + def test_dlestimator_fit_dlmodel_transform(self): + model = Sequential().add(Linear(2, 2)) + criterion = MSECriterion() + estimator = DLEstimator(model, criterion, [2], [2]).setBatchSize(4)\ + .setLearningRate(0.01).setMaxEpoch(1000) + + data = self.sc.parallelize([ + ((2.0, 1.0), (1.0, 2.0)), + ((1.0, 2.0), (2.0, 1.0)), + ((2.0, 1.0), (1.0, 2.0)), + ((1.0, 2.0), (2.0, 1.0))]) + + schema = StructType([ + StructField("features", ArrayType(DoubleType(), False), False), + StructField("label", ArrayType(DoubleType(), False), False)]) + df = self.sqlContext.createDataFrame(data, schema) + dlModel = estimator.fit(df) + + dlModel.transform(df).createTempView("dlModelDF") + results = self.sqlContext.table("dlModelDF") + + count = results.rdd.count() + data = results.rdd.collect() + + for i in range(count): + row_label = data[i]['label'] + row_prediction = data[i]['prediction'] + assert_allclose(row_label[0], row_prediction[0], atol=0.01, rtol=0) + assert_allclose(row_label[1], row_prediction[1], atol=0.01, rtol=0) + + def test_dlclassifier_fit_dlclassifiermodel_transform(self): + model = Sequential().add(Linear(2, 2)) + criterion = ClassNLLCriterion() + classifier = DLClassifier(model, criterion, [2]).setBatchSize(4)\ + .setLearningRate(0.01).setMaxEpoch(1000) + data = self.sc.parallelize([ + ((2.0, 1.0), 1.0), + ((1.0, 2.0), 2.0), + ((2.0, 1.0), 1.0), + ((1.0, 2.0), 2.0)]) + + schema = StructType([ + StructField("features", ArrayType(DoubleType(), False), False), + StructField("label", DoubleType(), False)]) + df = self.sqlContext.createDataFrame(data, schema) + dlClassifierModel = classifier.fit(df) + + dlClassifierModel.transform(df).createTempView("dlClassifierModelDF") + results = self.sqlContext.table("dlClassifierModelDF") + + count = results.rdd.count() + data = results.rdd.collect() + + for i in range(count): + row_label = data[i]['label'] + row_prediction = data[i]['prediction'] + assert row_label == row_prediction + +if __name__ == "__main__": + pytest.main() From 4747539466991650d60fd53398acf891638e8fd3 Mon Sep 17 00:00:00 2001 From: Xin Qiu Date: Mon, 23 Oct 2017 18:31:42 +0800 Subject: [PATCH 290/823] fix jtensor (#1706) --- python/dllib/src/bigdl/dllib/utils/common.py | 34 +++++++++++++------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/utils/common.py b/python/dllib/src/bigdl/dllib/utils/common.py index 7fa9fc085fc..86f0fb06fba 100644 --- a/python/dllib/src/bigdl/dllib/utils/common.py +++ b/python/dllib/src/bigdl/dllib/utils/common.py @@ -158,6 +158,12 @@ def from_ndarray(cls, a_ndarray, bigdl_type="float"): >>> np.random.seed(123) >>> data = np.random.uniform(0, 1, (2, 3)).astype("float32") >>> result = JTensor.from_ndarray(data) + >>> print(result) + JTensor: storage: [[ 0.69646919 0.28613934 0.22685145] + [ 0.55131477 0.71946895 0.42310646]], shape: [2 3], float + >>> result + JTensor: storage: [[ 0.69646919 0.28613934 0.22685145] + [ 0.55131477 0.71946895 0.42310646]], shape: [2 3], float >>> data_back = result.to_ndarray() >>> (data == data_back).all() True @@ -172,7 +178,7 @@ def from_ndarray(cls, a_ndarray, bigdl_type="float"): "input should be a np.ndarray, not %s" % type(a_ndarray) return cls(a_ndarray, a_ndarray.shape if a_ndarray.shape else (a_ndarray.size), - bigdl_type=bigdl_type) + bigdl_type) @classmethod def sparse(cls, a_ndarray, i_ndarray, shape, bigdl_type="float"): @@ -198,13 +204,15 @@ def sparse(cls, a_ndarray, i_ndarray, shape, bigdl_type="float"): >>> from bigdl.util.common import JTensor >>> from bigdl.util.common import callBigDlFunc >>> np.random.seed(123) - >>> data = np.arrange(1, 7).astype("float32") + >>> data = np.arange(1, 7).astype("float32") >>> indices = np.arange(1, 7) >>> shape = np.array([10]) >>> result = JTensor.sparse(data, indices, shape) + >>> result + JTensor: storage: [ 1. 2. 3. 4. 5. 6.], shape: [10] ,indices [1 2 3 4 5 6], float >>> tensor1 = callBigDlFunc("float", "testTensor", result) # noqa >>> array_from_tensor = tensor1.to_ndarray() - >>> expected_ndarray = np.array([0, 1, 2, 3, 4, 5, 6, 7, 0, 0]) + >>> expected_ndarray = np.array([0, 1, 2, 3, 4, 5, 6, 0, 0, 0]) >>> (array_from_tensor == expected_ndarray).all() True """ @@ -218,8 +226,8 @@ def sparse(cls, a_ndarray, i_ndarray, shape, bigdl_type="float"): "size of values and indices should match." return cls(a_ndarray, shape, - i_ndarray, - bigdl_type= bigdl_type) + bigdl_type, + i_ndarray) def to_ndarray(self): """ @@ -237,12 +245,11 @@ def __reduce__(self): return JTensor, (self.storage.tostring(), self.shape.tostring(), self.bigdl_type, self.indices.tostring()) def __str__(self): - indices = "" if self.indices is None else ",indices %s" % self.indices - return "JTensor: storage: %s, shape: %s %s" % (self.storage, self.shape, self.shape, indices) + return self.__repr__() def __repr__(self): - indices = "" if self.indices is None else ",indices %s" % self.indices - return "JTensor: storage: %s, shape: %s %s" % (self.storage, self.shape, self.shape, indices) + indices = "" if self.indices is None else " ,indices %s" % str(self.indices) + return "JTensor: storage: %s, shape: %s%s, %s" % (str(self.storage), str(self.shape), indices, self.bigdl_type) class Sample(object): @@ -268,10 +275,15 @@ def from_ndarray(cls, features, label, bigdl_type="float"): >>> import numpy as np >>> from bigdl.util.common import callBigDlFunc >>> from numpy.testing import assert_allclose + >>> np.random.seed(123) >>> sample = Sample.from_ndarray(np.random.random((2,3)), np.random.random((2,3))) >>> sample_back = callBigDlFunc("float", "testSample", sample) >>> assert_allclose(sample.features[0].to_ndarray(), sample_back.features[0].to_ndarray()) >>> assert_allclose(sample.label.to_ndarray(), sample_back.label.to_ndarray()) + >>> print(sample) + Sample: features: [JTensor: storage: [[ 0.69646919 0.28613934 0.22685145] + [ 0.55131477 0.71946895 0.42310646]], shape: [2 3], float], label: JTensor: storage: [[ 0.98076421 0.68482971 0.48093191] + [ 0.39211753 0.343178 0.72904968]], shape: [2 3], float, """ if isinstance(features, np.ndarray): features = [features] @@ -297,8 +309,8 @@ def from_jtensor(cls, features, label, bigdl_type="float"): >>> data = np.random.uniform(0, 1, (6)).astype("float32") >>> indices = np.arange(1, 7) >>> shape = np.array([10]) - >>> feature0 = JTensor.from_ndarray(data, indices, shape) - >>> feature1 = JTensor.from_ndarray(np.random((2, 3))) + >>> feature0 = JTensor.sparse(data, indices, shape) + >>> feature1 = JTensor.from_ndarray(np.random.uniform(0, 1, (2, 3)).astype("float32")) >>> sample = Sample.from_jtensor([feature0, feature1], 1) """ if isinstance(features, JTensor): From 5f142526440c771c13980234f7e1f68d08986deb Mon Sep 17 00:00:00 2001 From: Xin Qiu Date: Mon, 23 Oct 2017 18:31:42 +0800 Subject: [PATCH 291/823] fix jtensor (#1706) --- python/dllib/src/bigdl/utils/common.py | 34 +++++++++++++++++--------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/python/dllib/src/bigdl/utils/common.py b/python/dllib/src/bigdl/utils/common.py index 7fa9fc085fc..86f0fb06fba 100644 --- a/python/dllib/src/bigdl/utils/common.py +++ b/python/dllib/src/bigdl/utils/common.py @@ -158,6 +158,12 @@ def from_ndarray(cls, a_ndarray, bigdl_type="float"): >>> np.random.seed(123) >>> data = np.random.uniform(0, 1, (2, 3)).astype("float32") >>> result = JTensor.from_ndarray(data) + >>> print(result) + JTensor: storage: [[ 0.69646919 0.28613934 0.22685145] + [ 0.55131477 0.71946895 0.42310646]], shape: [2 3], float + >>> result + JTensor: storage: [[ 0.69646919 0.28613934 0.22685145] + [ 0.55131477 0.71946895 0.42310646]], shape: [2 3], float >>> data_back = result.to_ndarray() >>> (data == data_back).all() True @@ -172,7 +178,7 @@ def from_ndarray(cls, a_ndarray, bigdl_type="float"): "input should be a np.ndarray, not %s" % type(a_ndarray) return cls(a_ndarray, a_ndarray.shape if a_ndarray.shape else (a_ndarray.size), - bigdl_type=bigdl_type) + bigdl_type) @classmethod def sparse(cls, a_ndarray, i_ndarray, shape, bigdl_type="float"): @@ -198,13 +204,15 @@ def sparse(cls, a_ndarray, i_ndarray, shape, bigdl_type="float"): >>> from bigdl.util.common import JTensor >>> from bigdl.util.common import callBigDlFunc >>> np.random.seed(123) - >>> data = np.arrange(1, 7).astype("float32") + >>> data = np.arange(1, 7).astype("float32") >>> indices = np.arange(1, 7) >>> shape = np.array([10]) >>> result = JTensor.sparse(data, indices, shape) + >>> result + JTensor: storage: [ 1. 2. 3. 4. 5. 6.], shape: [10] ,indices [1 2 3 4 5 6], float >>> tensor1 = callBigDlFunc("float", "testTensor", result) # noqa >>> array_from_tensor = tensor1.to_ndarray() - >>> expected_ndarray = np.array([0, 1, 2, 3, 4, 5, 6, 7, 0, 0]) + >>> expected_ndarray = np.array([0, 1, 2, 3, 4, 5, 6, 0, 0, 0]) >>> (array_from_tensor == expected_ndarray).all() True """ @@ -218,8 +226,8 @@ def sparse(cls, a_ndarray, i_ndarray, shape, bigdl_type="float"): "size of values and indices should match." return cls(a_ndarray, shape, - i_ndarray, - bigdl_type= bigdl_type) + bigdl_type, + i_ndarray) def to_ndarray(self): """ @@ -237,12 +245,11 @@ def __reduce__(self): return JTensor, (self.storage.tostring(), self.shape.tostring(), self.bigdl_type, self.indices.tostring()) def __str__(self): - indices = "" if self.indices is None else ",indices %s" % self.indices - return "JTensor: storage: %s, shape: %s %s" % (self.storage, self.shape, self.shape, indices) + return self.__repr__() def __repr__(self): - indices = "" if self.indices is None else ",indices %s" % self.indices - return "JTensor: storage: %s, shape: %s %s" % (self.storage, self.shape, self.shape, indices) + indices = "" if self.indices is None else " ,indices %s" % str(self.indices) + return "JTensor: storage: %s, shape: %s%s, %s" % (str(self.storage), str(self.shape), indices, self.bigdl_type) class Sample(object): @@ -268,10 +275,15 @@ def from_ndarray(cls, features, label, bigdl_type="float"): >>> import numpy as np >>> from bigdl.util.common import callBigDlFunc >>> from numpy.testing import assert_allclose + >>> np.random.seed(123) >>> sample = Sample.from_ndarray(np.random.random((2,3)), np.random.random((2,3))) >>> sample_back = callBigDlFunc("float", "testSample", sample) >>> assert_allclose(sample.features[0].to_ndarray(), sample_back.features[0].to_ndarray()) >>> assert_allclose(sample.label.to_ndarray(), sample_back.label.to_ndarray()) + >>> print(sample) + Sample: features: [JTensor: storage: [[ 0.69646919 0.28613934 0.22685145] + [ 0.55131477 0.71946895 0.42310646]], shape: [2 3], float], label: JTensor: storage: [[ 0.98076421 0.68482971 0.48093191] + [ 0.39211753 0.343178 0.72904968]], shape: [2 3], float, """ if isinstance(features, np.ndarray): features = [features] @@ -297,8 +309,8 @@ def from_jtensor(cls, features, label, bigdl_type="float"): >>> data = np.random.uniform(0, 1, (6)).astype("float32") >>> indices = np.arange(1, 7) >>> shape = np.array([10]) - >>> feature0 = JTensor.from_ndarray(data, indices, shape) - >>> feature1 = JTensor.from_ndarray(np.random((2, 3))) + >>> feature0 = JTensor.sparse(data, indices, shape) + >>> feature1 = JTensor.from_ndarray(np.random.uniform(0, 1, (2, 3)).astype("float32")) >>> sample = Sample.from_jtensor([feature0, feature1], 1) """ if isinstance(features, JTensor): From 04985b79f634997cf20e32ee9388e3230d78402a Mon Sep 17 00:00:00 2001 From: Xin Qiu Date: Mon, 23 Oct 2017 18:31:42 +0800 Subject: [PATCH 292/823] fix jtensor (#1706) --- python/dllib/test/dev/run-tests | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/dllib/test/dev/run-tests b/python/dllib/test/dev/run-tests index d142d25b31e..33fd95db966 100755 --- a/python/dllib/test/dev/run-tests +++ b/python/dllib/test/dev/run-tests @@ -30,7 +30,7 @@ do export PYSPARK_DRIVER_PYTHON=$p $p -m pytest -v --doctest-modules ../../../pyspark/bigdl \ --ignore=../../../pyspark/bigdl/dataset/ \ - --ignore=../../../pyspark/bigdl/util/ \ + --ignore=../../../pyspark/bigdl/util/tf_utils.py \ --ignore=../../../pyspark/bigdl/models/ && \ $p -m pytest -v -n 4 ../../../pyspark/test/ exit_status=$? From 6ed12e84a49010c9c66806d3ad818946b8638369 Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Tue, 24 Oct 2017 13:31:15 +0800 Subject: [PATCH 293/823] fix test for dlclassifier (#1715) * fix test * style * precision --- python/dllib/src/bigdl/dllib/utils/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/dllib/src/bigdl/dllib/utils/common.py b/python/dllib/src/bigdl/dllib/utils/common.py index 86f0fb06fba..06afc457fcd 100644 --- a/python/dllib/src/bigdl/dllib/utils/common.py +++ b/python/dllib/src/bigdl/dllib/utils/common.py @@ -442,7 +442,7 @@ def get_spark_context(conf = None): def get_spark_sql_context(sc): if "getOrCreate" in SQLContext.__dict__: - return SQLContext.getOrCreate() + return SQLContext.getOrCreate(sc) else: return SQLContext(sc) # Compatible with Spark1.5.1 From 6ef4c89b9bd76cd83fc514afcdb463add963b756 Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Tue, 24 Oct 2017 13:31:15 +0800 Subject: [PATCH 294/823] fix test for dlclassifier (#1715) * fix test * style * precision --- python/dllib/src/bigdl/utils/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/dllib/src/bigdl/utils/common.py b/python/dllib/src/bigdl/utils/common.py index 86f0fb06fba..06afc457fcd 100644 --- a/python/dllib/src/bigdl/utils/common.py +++ b/python/dllib/src/bigdl/utils/common.py @@ -442,7 +442,7 @@ def get_spark_context(conf = None): def get_spark_sql_context(sc): if "getOrCreate" in SQLContext.__dict__: - return SQLContext.getOrCreate() + return SQLContext.getOrCreate(sc) else: return SQLContext(sc) # Compatible with Spark1.5.1 From 96a08e3d9ff7541720f71fe0d641c00466ad5cc3 Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Tue, 24 Oct 2017 13:31:15 +0800 Subject: [PATCH 295/823] fix test for dlclassifier (#1715) * fix test * style * precision --- python/dllib/test/bigdl/test_dl_classifier.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/python/dllib/test/bigdl/test_dl_classifier.py b/python/dllib/test/bigdl/test_dl_classifier.py index 2514cca1a77..07e04f818d8 100644 --- a/python/dllib/test/bigdl/test_dl_classifier.py +++ b/python/dllib/test/bigdl/test_dl_classifier.py @@ -126,7 +126,7 @@ def test_dlestimator_fit_dlmodel_transform(self): df = self.sqlContext.createDataFrame(data, schema) dlModel = estimator.fit(df) - dlModel.transform(df).createTempView("dlModelDF") + dlModel.transform(df).registerTempTable("dlModelDF") # Compatible with spark 1.6 results = self.sqlContext.table("dlModelDF") count = results.rdd.count() @@ -135,8 +135,8 @@ def test_dlestimator_fit_dlmodel_transform(self): for i in range(count): row_label = data[i]['label'] row_prediction = data[i]['prediction'] - assert_allclose(row_label[0], row_prediction[0], atol=0.01, rtol=0) - assert_allclose(row_label[1], row_prediction[1], atol=0.01, rtol=0) + assert_allclose(row_label[0], row_prediction[0], atol=0, rtol=1e-1) + assert_allclose(row_label[1], row_prediction[1], atol=0, rtol=1e-1) def test_dlclassifier_fit_dlclassifiermodel_transform(self): model = Sequential().add(Linear(2, 2)) @@ -155,7 +155,7 @@ def test_dlclassifier_fit_dlclassifiermodel_transform(self): df = self.sqlContext.createDataFrame(data, schema) dlClassifierModel = classifier.fit(df) - dlClassifierModel.transform(df).createTempView("dlClassifierModelDF") + dlClassifierModel.transform(df).registerTempTable("dlClassifierModelDF") results = self.sqlContext.table("dlClassifierModelDF") count = results.rdd.count() From 8bbc4d6187171cf97bb17041c6da728867c51ccc Mon Sep 17 00:00:00 2001 From: Xin Qiu Date: Wed, 25 Oct 2017 07:37:54 +0800 Subject: [PATCH 296/823] fix jtensor (#1720) --- python/dllib/src/bigdl/dllib/utils/common.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/utils/common.py b/python/dllib/src/bigdl/dllib/utils/common.py index 06afc457fcd..83292e055cd 100644 --- a/python/dllib/src/bigdl/dllib/utils/common.py +++ b/python/dllib/src/bigdl/dllib/utils/common.py @@ -141,10 +141,14 @@ def __init__(self, storage, shape, bigdl_type="float", indices=None): else: self.storage = np.array(storage, dtype=get_dtype(bigdl_type)) self.shape = np.array(shape, dtype=np.int32) - if indices is not None: - self.indices = np.array(indices, dtype=np.int32) - else: + if indices is None: self.indices = None + elif isinstance(indices, bytes): + self.indices = np.frombuffer(indices, dtype=np.int32) + else: + assert isinstance(indices, np.ndarray), \ + "indices should be a np.ndarray, not %s, %s" % (type(a_ndarray), str(indices)) + self.indices = np.array(indices, dtype=np.int32) self.bigdl_type = bigdl_type @classmethod From 5b91b37a3a3d5ebe15c41f2aea319457857312da Mon Sep 17 00:00:00 2001 From: Xin Qiu Date: Wed, 25 Oct 2017 07:37:54 +0800 Subject: [PATCH 297/823] fix jtensor (#1720) --- python/dllib/src/bigdl/utils/common.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/python/dllib/src/bigdl/utils/common.py b/python/dllib/src/bigdl/utils/common.py index 06afc457fcd..83292e055cd 100644 --- a/python/dllib/src/bigdl/utils/common.py +++ b/python/dllib/src/bigdl/utils/common.py @@ -141,10 +141,14 @@ def __init__(self, storage, shape, bigdl_type="float", indices=None): else: self.storage = np.array(storage, dtype=get_dtype(bigdl_type)) self.shape = np.array(shape, dtype=np.int32) - if indices is not None: - self.indices = np.array(indices, dtype=np.int32) - else: + if indices is None: self.indices = None + elif isinstance(indices, bytes): + self.indices = np.frombuffer(indices, dtype=np.int32) + else: + assert isinstance(indices, np.ndarray), \ + "indices should be a np.ndarray, not %s, %s" % (type(a_ndarray), str(indices)) + self.indices = np.array(indices, dtype=np.int32) self.bigdl_type = bigdl_type @classmethod From 42df19cb84052db8c2b139320e1615752ff137ab Mon Sep 17 00:00:00 2001 From: Ian Wong Date: Wed, 25 Oct 2017 13:57:33 +0800 Subject: [PATCH 298/823] update python version and bigdl-version-info on branch-0.3 (#1730) (#1735) --- version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.py b/version.py index ddff350988c..13932794d54 100644 --- a/version.py +++ b/version.py @@ -14,4 +14,4 @@ # limitations under the License. # -__version__ = "0.2.0.dev2" \ No newline at end of file +__version__ = "0.3.0.dev0" \ No newline at end of file From 62f3d0aaa082b890d10d8694c12b3bdec05505ca Mon Sep 17 00:00:00 2001 From: Ian Wong Date: Wed, 25 Oct 2017 13:57:33 +0800 Subject: [PATCH 299/823] update python version and bigdl-version-info on branch-0.3 (#1730) (#1735) --- version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.py b/version.py index ddff350988c..13932794d54 100644 --- a/version.py +++ b/version.py @@ -14,4 +14,4 @@ # limitations under the License. # -__version__ = "0.2.0.dev2" \ No newline at end of file +__version__ = "0.3.0.dev0" \ No newline at end of file From 816ff00ec0ecf8b8b5a93f7ba4e12af3ad7ae1bf Mon Sep 17 00:00:00 2001 From: Xin Qiu Date: Thu, 26 Oct 2017 09:28:27 +0800 Subject: [PATCH 300/823] Sparse md file (#1738) --- python/dllib/src/bigdl/dllib/nn/layer.py | 33 ++++++++++++++++-------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 212fdc5ea5a..b103135843d 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -131,21 +131,22 @@ def get_dtype(self): @staticmethod def check_input(input): """ - :param input: ndarray or list of ndarray + :param input: ndarray or list of ndarray or JTensor or list of JTensor. :return: (list of JTensor, isTable) """ + def to_jtensor(i): + if isinstance(i, np.ndarray): + return JTensor.from_ndarray(i) + elif isinstance(i, JTensor): + return i + else: + raise Exception("Error unknown input type %s" % type(i)) if type(input) is list: if len(input) == 0: raise Exception('Error when checking: empty input') - if not hasattr(input[0], 'shape'): - raise Exception( - 'Error when checking: expecting list of ndarray') - return [JTensor.from_ndarray(i) for i in input], True + return list(map(lambda i: to_jtensor(i), input)), True else: - if not hasattr(input, 'shape'): - raise Exception( - 'Error when checking: expecting list of ndarray') - return [JTensor.from_ndarray(input)], False + return [to_jtensor(input)], False @staticmethod def convert_output(output): @@ -162,6 +163,7 @@ def forward(self, input): Takes an input object, and computes the corresponding output of the module :param input: ndarray or list of ndarray + :param input: ndarray or list of ndarray or JTensor or list of JTensor. :return: ndarray or list of ndarray """ jinput, input_is_table = self.check_input(input) @@ -180,8 +182,8 @@ def backward(self, input, grad_output): input. This is necessary for optimization reasons. If you do not respect this rule, backward() will compute incorrect gradients. - :param input: ndarray or list of ndarray - :param grad_output: ndarray or list of ndarray + :param input: ndarray or list of ndarray or JTensor or list of JTensor. + :param grad_output: ndarray or list of ndarray or JTensor or list of JTensor. :return: ndarray or list of ndarray """ jinput, input_is_table = self.check_input(input) @@ -765,6 +767,15 @@ class SparseLinear(Layer): creating: createL1Regularizer creating: createL1Regularizer creating: createSparseLinear + >>> np.random.seed(123) + >>> init_weight = np.random.randn(5, 1000) + >>> init_bias = np.random.randn(5) + >>> sparselinear = SparseLinear(1000, 5, init_weight=init_weight, init_bias=init_bias) + creating: createSparseLinear + >>> input = JTensor.sparse(np.array([1, 3, 5, 2, 4, 6]), np.array([0, 0, 0, 1, 1, 1, 1, 5, 300, 2, 100, 500]), np.array([2, 1000])) + >>> print(sparselinear.forward(input)) + [[ 10.09569263 -10.94844246 -4.1086688 1.02527523 11.80737209] + [ 7.9651413 9.7131443 -10.22719955 0.02345783 -3.74368906]] ''' def __init__(self, input_size, output_size, with_bias=True, backwardStart=-1, backwardLength=-1, From 1b6c4c7e9ebf0f1465d08f50aee1c0da894af441 Mon Sep 17 00:00:00 2001 From: Xin Qiu Date: Thu, 26 Oct 2017 09:28:27 +0800 Subject: [PATCH 301/823] Sparse md file (#1738) --- python/dllib/src/bigdl/dllib/nn/layer.py | 33 ++++++++++++++++-------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 212fdc5ea5a..b103135843d 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -131,21 +131,22 @@ def get_dtype(self): @staticmethod def check_input(input): """ - :param input: ndarray or list of ndarray + :param input: ndarray or list of ndarray or JTensor or list of JTensor. :return: (list of JTensor, isTable) """ + def to_jtensor(i): + if isinstance(i, np.ndarray): + return JTensor.from_ndarray(i) + elif isinstance(i, JTensor): + return i + else: + raise Exception("Error unknown input type %s" % type(i)) if type(input) is list: if len(input) == 0: raise Exception('Error when checking: empty input') - if not hasattr(input[0], 'shape'): - raise Exception( - 'Error when checking: expecting list of ndarray') - return [JTensor.from_ndarray(i) for i in input], True + return list(map(lambda i: to_jtensor(i), input)), True else: - if not hasattr(input, 'shape'): - raise Exception( - 'Error when checking: expecting list of ndarray') - return [JTensor.from_ndarray(input)], False + return [to_jtensor(input)], False @staticmethod def convert_output(output): @@ -162,6 +163,7 @@ def forward(self, input): Takes an input object, and computes the corresponding output of the module :param input: ndarray or list of ndarray + :param input: ndarray or list of ndarray or JTensor or list of JTensor. :return: ndarray or list of ndarray """ jinput, input_is_table = self.check_input(input) @@ -180,8 +182,8 @@ def backward(self, input, grad_output): input. This is necessary for optimization reasons. If you do not respect this rule, backward() will compute incorrect gradients. - :param input: ndarray or list of ndarray - :param grad_output: ndarray or list of ndarray + :param input: ndarray or list of ndarray or JTensor or list of JTensor. + :param grad_output: ndarray or list of ndarray or JTensor or list of JTensor. :return: ndarray or list of ndarray """ jinput, input_is_table = self.check_input(input) @@ -765,6 +767,15 @@ class SparseLinear(Layer): creating: createL1Regularizer creating: createL1Regularizer creating: createSparseLinear + >>> np.random.seed(123) + >>> init_weight = np.random.randn(5, 1000) + >>> init_bias = np.random.randn(5) + >>> sparselinear = SparseLinear(1000, 5, init_weight=init_weight, init_bias=init_bias) + creating: createSparseLinear + >>> input = JTensor.sparse(np.array([1, 3, 5, 2, 4, 6]), np.array([0, 0, 0, 1, 1, 1, 1, 5, 300, 2, 100, 500]), np.array([2, 1000])) + >>> print(sparselinear.forward(input)) + [[ 10.09569263 -10.94844246 -4.1086688 1.02527523 11.80737209] + [ 7.9651413 9.7131443 -10.22719955 0.02345783 -3.74368906]] ''' def __init__(self, input_size, output_size, with_bias=True, backwardStart=-1, backwardLength=-1, From 5d94ac7a27cebf4bfe40ad2bd73f896392d83161 Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Thu, 26 Oct 2017 10:44:14 +0800 Subject: [PATCH 302/823] Fix unmatched spark sources on master (#1758) * fix unmatched spark sources --- python/dllib/src/bigdl/dllib/utils/engine.py | 21 +++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/utils/engine.py b/python/dllib/src/bigdl/dllib/utils/engine.py index 556e1e2e1cb..2b278cd3edd 100644 --- a/python/dllib/src/bigdl/dllib/utils/engine.py +++ b/python/dllib/src/bigdl/dllib/utils/engine.py @@ -17,24 +17,35 @@ import sys import os import glob +import warnings def exist_pyspark(): + # check whether pyspark package exists try: import pyspark return True except ImportError: return False +def check_spark_source_conflict(spark_home, pyspark_path): + # check if both spark_home env var and pyspark package exist + # trigger a warning if two spark sources don't match + if spark_home and not pyspark_path.startswith(spark_home): + warning_msg = "Find both SPARK_HOME and pyspark. You may need to check whether they " + \ + "match with each other. SPARK_HOME environment variable is set to: " + spark_home + \ + ", and pyspark is found in: " + pyspark_path + ". If they are unmatched, " + \ + "please use one source only to avoid conflict. " + \ + "For example, you can unset SPARK_HOME and use pyspark only." + warnings.warn(warning_msg) + def __prepare_spark_env(): spark_home = os.environ.get('SPARK_HOME', None) if exist_pyspark(): + # use pyspark as the spark source import pyspark - if spark_home and not pyspark.__file__.startswith(spark_home): - raise Exception( - """Find two unmatched spark sources. pyspark is found in """ - + pyspark.__file__ + ", while SPARK_HOME env is set to " + spark_home - + ". Please use one spark source only. For example, you can unset SPARK_HOME and use pyspark only.") + check_spark_source_conflict(spark_home, pyspark.__file__) else: + # use SPARK_HOME as the spark source if not spark_home: raise ValueError( """Could not find Spark. Please make sure SPARK_HOME env is set: From 2627738aeb0fd62a495896cc5650674236870f5d Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Thu, 26 Oct 2017 10:44:14 +0800 Subject: [PATCH 303/823] Fix unmatched spark sources on master (#1758) * fix unmatched spark sources --- python/dllib/src/bigdl/utils/engine.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/python/dllib/src/bigdl/utils/engine.py b/python/dllib/src/bigdl/utils/engine.py index 556e1e2e1cb..2b278cd3edd 100644 --- a/python/dllib/src/bigdl/utils/engine.py +++ b/python/dllib/src/bigdl/utils/engine.py @@ -17,24 +17,35 @@ import sys import os import glob +import warnings def exist_pyspark(): + # check whether pyspark package exists try: import pyspark return True except ImportError: return False +def check_spark_source_conflict(spark_home, pyspark_path): + # check if both spark_home env var and pyspark package exist + # trigger a warning if two spark sources don't match + if spark_home and not pyspark_path.startswith(spark_home): + warning_msg = "Find both SPARK_HOME and pyspark. You may need to check whether they " + \ + "match with each other. SPARK_HOME environment variable is set to: " + spark_home + \ + ", and pyspark is found in: " + pyspark_path + ". If they are unmatched, " + \ + "please use one source only to avoid conflict. " + \ + "For example, you can unset SPARK_HOME and use pyspark only." + warnings.warn(warning_msg) + def __prepare_spark_env(): spark_home = os.environ.get('SPARK_HOME', None) if exist_pyspark(): + # use pyspark as the spark source import pyspark - if spark_home and not pyspark.__file__.startswith(spark_home): - raise Exception( - """Find two unmatched spark sources. pyspark is found in """ - + pyspark.__file__ + ", while SPARK_HOME env is set to " + spark_home - + ". Please use one spark source only. For example, you can unset SPARK_HOME and use pyspark only.") + check_spark_source_conflict(spark_home, pyspark.__file__) else: + # use SPARK_HOME as the spark source if not spark_home: raise ValueError( """Could not find Spark. Please make sure SPARK_HOME env is set: From ce3f79e5ebb2d931f283ce56e19e82c7e0e6f407 Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Thu, 26 Oct 2017 15:02:07 +0800 Subject: [PATCH 304/823] Update README.md --- python/dllib/src/bigdl/dllib/models/textclassifier/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/dllib/src/bigdl/dllib/models/textclassifier/README.md b/python/dllib/src/bigdl/dllib/models/textclassifier/README.md index f7c0b946c55..36f319baa2a 100644 --- a/python/dllib/src/bigdl/dllib/models/textclassifier/README.md +++ b/python/dllib/src/bigdl/dllib/models/textclassifier/README.md @@ -27,7 +27,7 @@ $ [/tmp/news20]$ tree . -L 1 ``` - The example code would automatically download the data during the first run. - If you install BigDL via pip, you can run this example locally by the following command: - - Check [Run after pip install](../../../../docs/docs/PythonSupport/run-from-pip.md) + - Check [Run after pip install](../../../../docs/docs/PythonUserGuide/run-from-pip.md) ``` export SPARK_HOME=path to spark-1.6.3 python ${BigDL_HOME}/pyspark/bigdl/models/textclassifier/textclassifier.py --max_epoch 3 --model cnn From 7d645a78425f03adb58231ee2d51696d88525c7b Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Thu, 26 Oct 2017 15:02:07 +0800 Subject: [PATCH 305/823] Update README.md --- python/dllib/src/bigdl/dllib/models/textclassifier/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/dllib/src/bigdl/dllib/models/textclassifier/README.md b/python/dllib/src/bigdl/dllib/models/textclassifier/README.md index f7c0b946c55..36f319baa2a 100644 --- a/python/dllib/src/bigdl/dllib/models/textclassifier/README.md +++ b/python/dllib/src/bigdl/dllib/models/textclassifier/README.md @@ -27,7 +27,7 @@ $ [/tmp/news20]$ tree . -L 1 ``` - The example code would automatically download the data during the first run. - If you install BigDL via pip, you can run this example locally by the following command: - - Check [Run after pip install](../../../../docs/docs/PythonSupport/run-from-pip.md) + - Check [Run after pip install](../../../../docs/docs/PythonUserGuide/run-from-pip.md) ``` export SPARK_HOME=path to spark-1.6.3 python ${BigDL_HOME}/pyspark/bigdl/models/textclassifier/textclassifier.py --max_epoch 3 --model cnn From 267ed41cf85f2f81d1ac9ecb842b5fd76892d004 Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Fri, 27 Oct 2017 08:48:28 +0800 Subject: [PATCH 306/823] update python rnn readme; add ipython==5.0 to venv requirements (#1769) --- .../src/bigdl/dllib/models/rnn/README.md | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/models/rnn/README.md b/python/dllib/src/bigdl/dllib/models/rnn/README.md index b381bcf56f1..2e21a8b6582 100644 --- a/python/dllib/src/bigdl/dllib/models/rnn/README.md +++ b/python/dllib/src/bigdl/dllib/models/rnn/README.md @@ -15,6 +15,13 @@ Please build BigDL referring to [Build Page](https://github.com/intel-analytics/ ## Prepare the Input Data You can download the Tiny Shakespeare Texts corpus from [here](https://raw.githubusercontent.com/karpathy/char-rnn/master/data/tinyshakespeare/input.txt). +After downloading the text, please place it into an appropriate directory (e.g `/opt/text/input.txt`). Please separate it into `train.txt` and `val.txt`. In our case, we just select 80 percentage of the input to be train and remaining 20 percentage to be val. The program will later read in the original text file from this directory. +```shell +export LANG=en_US.UTF-8 +head -n 8000 input.txt > val.txt +tail -n +8000 input.txt > train.txt +``` + If you run on spark local mode, you can skip this step, we will download the file for you. ### Sample Text @@ -71,10 +78,10 @@ http_proxy=... PYSPARK_DRIVER_PYTHON=./venv/bin/python PYSPARK_PYTHON=./venv.zip/venv/bin/python ${SPARK_HOME}/bin/spark-submit \ --master ${MASTER} \ --deploy-mode client \ - --conf spark.executorEnv.http_proxy=${http_proxy} \ - --driver-memory 10g \ - --executor-cores 1 \ - --executor-memory 60g \ + --conf spark.executorEnv.http_proxy=${http_proxy} \ + --driver-memory 10g \ + --executor-cores 1 \ + --executor-memory 60g \ --py-files ${PYTHON_API_PATH} \ --properties-file ${BigDL_HOME}/dist/conf/spark-bigdl.conf \ --jars ${BigDL_JAR_PATH} \ @@ -83,10 +90,10 @@ PYSPARK_DRIVER_PYTHON=./venv/bin/python PYSPARK_PYTHON=./venv.zip/venv/bin/pytho --conf spark.executor.extraClassPath=bigdl-version-jar-with-dependencies.jar \ --conf spark.yarn.appMasterEnv.NLTK_DATA=./ \ --num-executors 1 \ - ${BigDL_HOME}/pyspark/dl/models/rnn/rnnexample.py --folder hdfs://xxx:9000/rnn/ --batchSize 12 + ${BigDL_HOME}/pyspark/bigdl/models/rnn/rnnexample.py --folder hdfs://xxx:9000/rnn/ --batchSize 12 ``` -* `--folder` hdfs directory where train.txt and val.txt are located +* `--folder` hdfs directory where `train.txt` and `val.txt` are located. * `--batchSize` option can be used to set batch size, the default value is 128. From 12ca148c898df50c2d2a8fde971bfa2bf1691acd Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Fri, 27 Oct 2017 08:48:28 +0800 Subject: [PATCH 307/823] update python rnn readme; add ipython==5.0 to venv requirements (#1769) --- .../src/bigdl/dllib/models/rnn/README.md | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/models/rnn/README.md b/python/dllib/src/bigdl/dllib/models/rnn/README.md index b381bcf56f1..2e21a8b6582 100644 --- a/python/dllib/src/bigdl/dllib/models/rnn/README.md +++ b/python/dllib/src/bigdl/dllib/models/rnn/README.md @@ -15,6 +15,13 @@ Please build BigDL referring to [Build Page](https://github.com/intel-analytics/ ## Prepare the Input Data You can download the Tiny Shakespeare Texts corpus from [here](https://raw.githubusercontent.com/karpathy/char-rnn/master/data/tinyshakespeare/input.txt). +After downloading the text, please place it into an appropriate directory (e.g `/opt/text/input.txt`). Please separate it into `train.txt` and `val.txt`. In our case, we just select 80 percentage of the input to be train and remaining 20 percentage to be val. The program will later read in the original text file from this directory. +```shell +export LANG=en_US.UTF-8 +head -n 8000 input.txt > val.txt +tail -n +8000 input.txt > train.txt +``` + If you run on spark local mode, you can skip this step, we will download the file for you. ### Sample Text @@ -71,10 +78,10 @@ http_proxy=... PYSPARK_DRIVER_PYTHON=./venv/bin/python PYSPARK_PYTHON=./venv.zip/venv/bin/python ${SPARK_HOME}/bin/spark-submit \ --master ${MASTER} \ --deploy-mode client \ - --conf spark.executorEnv.http_proxy=${http_proxy} \ - --driver-memory 10g \ - --executor-cores 1 \ - --executor-memory 60g \ + --conf spark.executorEnv.http_proxy=${http_proxy} \ + --driver-memory 10g \ + --executor-cores 1 \ + --executor-memory 60g \ --py-files ${PYTHON_API_PATH} \ --properties-file ${BigDL_HOME}/dist/conf/spark-bigdl.conf \ --jars ${BigDL_JAR_PATH} \ @@ -83,10 +90,10 @@ PYSPARK_DRIVER_PYTHON=./venv/bin/python PYSPARK_PYTHON=./venv.zip/venv/bin/pytho --conf spark.executor.extraClassPath=bigdl-version-jar-with-dependencies.jar \ --conf spark.yarn.appMasterEnv.NLTK_DATA=./ \ --num-executors 1 \ - ${BigDL_HOME}/pyspark/dl/models/rnn/rnnexample.py --folder hdfs://xxx:9000/rnn/ --batchSize 12 + ${BigDL_HOME}/pyspark/bigdl/models/rnn/rnnexample.py --folder hdfs://xxx:9000/rnn/ --batchSize 12 ``` -* `--folder` hdfs directory where train.txt and val.txt are located +* `--folder` hdfs directory where `train.txt` and `val.txt` are located. * `--batchSize` option can be used to set batch size, the default value is 128. From 82f5717a06a65eac8ee7142b638c6989f5dc751a Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Fri, 27 Oct 2017 09:42:11 +0800 Subject: [PATCH 308/823] Update README.md --- python/dllib/src/bigdl/dllib/models/textclassifier/README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/models/textclassifier/README.md b/python/dllib/src/bigdl/dllib/models/textclassifier/README.md index 36f319baa2a..a1eb781a4e3 100644 --- a/python/dllib/src/bigdl/dllib/models/textclassifier/README.md +++ b/python/dllib/src/bigdl/dllib/models/textclassifier/README.md @@ -29,9 +29,8 @@ $ [/tmp/news20]$ tree . -L 1 - If you install BigDL via pip, you can run this example locally by the following command: - Check [Run after pip install](../../../../docs/docs/PythonUserGuide/run-from-pip.md) ``` -export SPARK_HOME=path to spark-1.6.3 python ${BigDL_HOME}/pyspark/bigdl/models/textclassifier/textclassifier.py --max_epoch 3 --model cnn - + ``` - You can also use `spark-submit` to launch this example: From ef9b550303ca25cb7dfb370f84701efbaa199b79 Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Fri, 27 Oct 2017 09:42:11 +0800 Subject: [PATCH 309/823] Update README.md --- python/dllib/src/bigdl/dllib/models/textclassifier/README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/models/textclassifier/README.md b/python/dllib/src/bigdl/dllib/models/textclassifier/README.md index 36f319baa2a..a1eb781a4e3 100644 --- a/python/dllib/src/bigdl/dllib/models/textclassifier/README.md +++ b/python/dllib/src/bigdl/dllib/models/textclassifier/README.md @@ -29,9 +29,8 @@ $ [/tmp/news20]$ tree . -L 1 - If you install BigDL via pip, you can run this example locally by the following command: - Check [Run after pip install](../../../../docs/docs/PythonUserGuide/run-from-pip.md) ``` -export SPARK_HOME=path to spark-1.6.3 python ${BigDL_HOME}/pyspark/bigdl/models/textclassifier/textclassifier.py --max_epoch 3 --model cnn - + ``` - You can also use `spark-submit` to launch this example: From 381bf57ba3cb6db4b66d98482c3eaf31c8a588a2 Mon Sep 17 00:00:00 2001 From: megaSpoon Date: Thu, 26 Oct 2017 21:17:30 -0700 Subject: [PATCH 310/823] update unittest for dl_classifier (#1774) solution explanation: dataframe 1.5 in spark-1.5x does not support accessing column by its name so we access it by index number. --- python/dllib/test/bigdl/test_dl_classifier.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/python/dllib/test/bigdl/test_dl_classifier.py b/python/dllib/test/bigdl/test_dl_classifier.py index 07e04f818d8..99cef1ede55 100644 --- a/python/dllib/test/bigdl/test_dl_classifier.py +++ b/python/dllib/test/bigdl/test_dl_classifier.py @@ -133,8 +133,8 @@ def test_dlestimator_fit_dlmodel_transform(self): data = results.rdd.collect() for i in range(count): - row_label = data[i]['label'] - row_prediction = data[i]['prediction'] + row_label = data[i][1] + row_prediction = data[i][2] assert_allclose(row_label[0], row_prediction[0], atol=0, rtol=1e-1) assert_allclose(row_label[1], row_prediction[1], atol=0, rtol=1e-1) @@ -162,8 +162,8 @@ def test_dlclassifier_fit_dlclassifiermodel_transform(self): data = results.rdd.collect() for i in range(count): - row_label = data[i]['label'] - row_prediction = data[i]['prediction'] + row_label = data[i][1] + row_prediction = data[i][2] assert row_label == row_prediction if __name__ == "__main__": From dc364aa818eaf80b43eafc7331e5a111fb30d576 Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Fri, 27 Oct 2017 14:11:49 +0800 Subject: [PATCH 311/823] enable setting path for data in python examples on master (#1777) * enable users to set path for downloading data in lenet, textclassifier python --- python/dllib/src/bigdl/dllib/models/lenet/README.md | 9 +++++---- python/dllib/src/bigdl/dllib/models/lenet/lenet5.py | 5 +++-- .../src/bigdl/dllib/models/textclassifier/README.md | 10 ++++++---- .../dllib/models/textclassifier/textclassifier.py | 8 +++++--- 4 files changed, 19 insertions(+), 13 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/models/lenet/README.md b/python/dllib/src/bigdl/dllib/models/lenet/README.md index 7f7d156555e..111268b0315 100644 --- a/python/dllib/src/bigdl/dllib/models/lenet/README.md +++ b/python/dllib/src/bigdl/dllib/models/lenet/README.md @@ -26,7 +26,6 @@ We would train a LeNet model in spark local mode with the following commands and ``` BigDL_HOME=... - SPARK_HOME=... MASTER=local[*] PYTHON_API_ZIP_PATH=${BigDL_HOME}/dist/lib/bigdl-VERSION-python-api.zip @@ -45,12 +44,15 @@ We would train a LeNet model in spark local mode with the following commands and --conf spark.driver.extraClassPath=${BigDL_JAR_PATH} \ --conf spark.executor.extraClassPath=bigdl-VERSION-jar-with-dependencies.jar \ ${BigDL_HOME}/pyspark/bigdl/models/lenet/lenet5.py \ - --action train + --action train \ + --dataPath /tmp/mnist ``` * ```--action``` it can be train or test. +* ```--dataPath``` option can be used to set the path for downloading mnist data, the default value is /tmp/mnist. Make sure that you have write permission to the specified path. + * ```--batchSize``` option can be used to set batch size, the default value is 128. * ```--endTriggerType``` option can be used to control how to end the training process, the value can be "epoch" or "iteration" and default value is "epoch". @@ -68,10 +70,9 @@ INFO DistriOptimizer$:247 - [Epoch 1 0/60000][Iteration 1][Wall Clock 0.0s] Tra INFO DistriOptimizer$:522 - Top1Accuracy is Accuracy(correct: 9572, count: 10000, accuracy: 0.9572) - ``` Or you can train a LeNet model directly in shell after installing BigDL from pip: ``` -python .../models/lenet/lenet5.py +python ${BigDL_HOME}/pyspark/bigdl/models/lenet/lenet5.py ``` \ No newline at end of file diff --git a/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py b/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py index f3a2fb1b123..92a7074a08a 100644 --- a/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py +++ b/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py @@ -65,6 +65,7 @@ def get_mnist(sc, data_type="train", location="/tmp/mnist"): parser.add_option("-c", "--checkpointPath", dest="checkpointPath", default="/tmp/lenet5") parser.add_option("-t", "--endTriggerType", dest="endTriggerType", default="epoch") parser.add_option("-n", "--endTriggerNum", type=int, dest="endTriggerNum", default="20") + parser.add_option("-d", "--dataPath", dest="dataPath", default="/tmp/mnist") (options, args) = parser.parse_args(sys.argv) @@ -80,11 +81,11 @@ def get_end_trigger(): else: return MaxIteration(options.endTriggerNum) - train_data = get_mnist(sc, "train")\ + train_data = get_mnist(sc, "train", options.dataPath)\ .map(lambda rec_tuple: (normalizer(rec_tuple[0], mnist.TRAIN_MEAN, mnist.TRAIN_STD), rec_tuple[1]))\ .map(lambda t: Sample.from_ndarray(t[0], t[1])) - test_data = get_mnist(sc, "test")\ + test_data = get_mnist(sc, "test", options.dataPath)\ .map(lambda rec_tuple: (normalizer(rec_tuple[0], mnist.TEST_MEAN, mnist.TEST_STD), rec_tuple[1]))\ .map(lambda t: Sample.from_ndarray(t[0], t[1])) diff --git a/python/dllib/src/bigdl/dllib/models/textclassifier/README.md b/python/dllib/src/bigdl/dllib/models/textclassifier/README.md index a1eb781a4e3..f431a54163e 100644 --- a/python/dllib/src/bigdl/dllib/models/textclassifier/README.md +++ b/python/dllib/src/bigdl/dllib/models/textclassifier/README.md @@ -57,14 +57,16 @@ python ${BigDL_HOME}/pyspark/bigdl/models/textclassifier/textclassifier.py --max --conf spark.executor.extraClassPath=bigdl-VERSION-jar-with-dependencies.jar \ --conf spark.executorEnv.PYTHONHASHSEED=${PYTHONHASHSEED} \ ${BigDL_HOME}/pyspark/bigdl/models/textclassifier/textclassifier.py \ - --max_epoch 3 - --model cnn + --data_path /tmp/news20/ \ + --max_epoch 3 \ + --model cnn ``` +* `--data_path` option can be used to set the path for downloading news20 data, the default value is /tmp/news20. Make sure that you have write permission to the specified path. -* `--max_epoch` option can be used to set how many epochs the model to be trained +* `--max_epoch` option can be used to set how many epochs the model to be trained. * `--model` option can be used to choose a model to be trained, three models are supported in this example, -which are `cnn`, `lstm` and `gru`, default is `cnn` +which are `cnn`, `lstm` and `gru`, default is `cnn`. * `--batchSize` option can be used to set batch size, the default value is 128. diff --git a/python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py b/python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py index dd5a8e0bdae..922ac4a2aef 100644 --- a/python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py +++ b/python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py @@ -99,11 +99,11 @@ def build_model(class_num): return model -def train(sc, +def train(sc, data_path, batch_size, sequence_len, max_words, embedding_dim, training_split): print('Processing text dataset') - texts = news20.get_news20() + texts = news20.get_news20(source_dir=data_path) data_rdd = sc.parallelize(texts, 2) word_to_ic = analyze_texts(data_rdd) @@ -155,6 +155,7 @@ def train(sc, parser.add_option("-m", "--max_epoch", dest="max_epoch", default="15") parser.add_option("--model", dest="model_type", default="cnn") parser.add_option("-p", "--p", dest="p", default="0.0") + parser.add_option("-d", "--data_path", dest="data_path", default="/tmp/news20/") (options, args) = parser.parse_args(sys.argv) if options.action == "train": @@ -168,10 +169,11 @@ def train(sc, training_split = 0.8 sc = SparkContext(appName="text_classifier", conf=create_spark_conf()) + data_path = options.data_path redire_spark_logs() show_bigdl_info_logs() init_engine() - train(sc, + train(sc, data_path, batch_size, sequence_len, max_words, embedding_dim, training_split) sc.stop() From 84abf2d9b3729be4783ad720f86bd58cdcd71055 Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Fri, 27 Oct 2017 14:11:49 +0800 Subject: [PATCH 312/823] enable setting path for data in python examples on master (#1777) * enable users to set path for downloading data in lenet, textclassifier python --- python/dllib/src/bigdl/dllib/models/lenet/README.md | 9 +++++---- python/dllib/src/bigdl/dllib/models/lenet/lenet5.py | 5 +++-- .../src/bigdl/dllib/models/textclassifier/README.md | 10 ++++++---- .../dllib/models/textclassifier/textclassifier.py | 8 +++++--- 4 files changed, 19 insertions(+), 13 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/models/lenet/README.md b/python/dllib/src/bigdl/dllib/models/lenet/README.md index 7f7d156555e..111268b0315 100644 --- a/python/dllib/src/bigdl/dllib/models/lenet/README.md +++ b/python/dllib/src/bigdl/dllib/models/lenet/README.md @@ -26,7 +26,6 @@ We would train a LeNet model in spark local mode with the following commands and ``` BigDL_HOME=... - SPARK_HOME=... MASTER=local[*] PYTHON_API_ZIP_PATH=${BigDL_HOME}/dist/lib/bigdl-VERSION-python-api.zip @@ -45,12 +44,15 @@ We would train a LeNet model in spark local mode with the following commands and --conf spark.driver.extraClassPath=${BigDL_JAR_PATH} \ --conf spark.executor.extraClassPath=bigdl-VERSION-jar-with-dependencies.jar \ ${BigDL_HOME}/pyspark/bigdl/models/lenet/lenet5.py \ - --action train + --action train \ + --dataPath /tmp/mnist ``` * ```--action``` it can be train or test. +* ```--dataPath``` option can be used to set the path for downloading mnist data, the default value is /tmp/mnist. Make sure that you have write permission to the specified path. + * ```--batchSize``` option can be used to set batch size, the default value is 128. * ```--endTriggerType``` option can be used to control how to end the training process, the value can be "epoch" or "iteration" and default value is "epoch". @@ -68,10 +70,9 @@ INFO DistriOptimizer$:247 - [Epoch 1 0/60000][Iteration 1][Wall Clock 0.0s] Tra INFO DistriOptimizer$:522 - Top1Accuracy is Accuracy(correct: 9572, count: 10000, accuracy: 0.9572) - ``` Or you can train a LeNet model directly in shell after installing BigDL from pip: ``` -python .../models/lenet/lenet5.py +python ${BigDL_HOME}/pyspark/bigdl/models/lenet/lenet5.py ``` \ No newline at end of file diff --git a/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py b/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py index f3a2fb1b123..92a7074a08a 100644 --- a/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py +++ b/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py @@ -65,6 +65,7 @@ def get_mnist(sc, data_type="train", location="/tmp/mnist"): parser.add_option("-c", "--checkpointPath", dest="checkpointPath", default="/tmp/lenet5") parser.add_option("-t", "--endTriggerType", dest="endTriggerType", default="epoch") parser.add_option("-n", "--endTriggerNum", type=int, dest="endTriggerNum", default="20") + parser.add_option("-d", "--dataPath", dest="dataPath", default="/tmp/mnist") (options, args) = parser.parse_args(sys.argv) @@ -80,11 +81,11 @@ def get_end_trigger(): else: return MaxIteration(options.endTriggerNum) - train_data = get_mnist(sc, "train")\ + train_data = get_mnist(sc, "train", options.dataPath)\ .map(lambda rec_tuple: (normalizer(rec_tuple[0], mnist.TRAIN_MEAN, mnist.TRAIN_STD), rec_tuple[1]))\ .map(lambda t: Sample.from_ndarray(t[0], t[1])) - test_data = get_mnist(sc, "test")\ + test_data = get_mnist(sc, "test", options.dataPath)\ .map(lambda rec_tuple: (normalizer(rec_tuple[0], mnist.TEST_MEAN, mnist.TEST_STD), rec_tuple[1]))\ .map(lambda t: Sample.from_ndarray(t[0], t[1])) diff --git a/python/dllib/src/bigdl/dllib/models/textclassifier/README.md b/python/dllib/src/bigdl/dllib/models/textclassifier/README.md index a1eb781a4e3..f431a54163e 100644 --- a/python/dllib/src/bigdl/dllib/models/textclassifier/README.md +++ b/python/dllib/src/bigdl/dllib/models/textclassifier/README.md @@ -57,14 +57,16 @@ python ${BigDL_HOME}/pyspark/bigdl/models/textclassifier/textclassifier.py --max --conf spark.executor.extraClassPath=bigdl-VERSION-jar-with-dependencies.jar \ --conf spark.executorEnv.PYTHONHASHSEED=${PYTHONHASHSEED} \ ${BigDL_HOME}/pyspark/bigdl/models/textclassifier/textclassifier.py \ - --max_epoch 3 - --model cnn + --data_path /tmp/news20/ \ + --max_epoch 3 \ + --model cnn ``` +* `--data_path` option can be used to set the path for downloading news20 data, the default value is /tmp/news20. Make sure that you have write permission to the specified path. -* `--max_epoch` option can be used to set how many epochs the model to be trained +* `--max_epoch` option can be used to set how many epochs the model to be trained. * `--model` option can be used to choose a model to be trained, three models are supported in this example, -which are `cnn`, `lstm` and `gru`, default is `cnn` +which are `cnn`, `lstm` and `gru`, default is `cnn`. * `--batchSize` option can be used to set batch size, the default value is 128. diff --git a/python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py b/python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py index dd5a8e0bdae..922ac4a2aef 100644 --- a/python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py +++ b/python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py @@ -99,11 +99,11 @@ def build_model(class_num): return model -def train(sc, +def train(sc, data_path, batch_size, sequence_len, max_words, embedding_dim, training_split): print('Processing text dataset') - texts = news20.get_news20() + texts = news20.get_news20(source_dir=data_path) data_rdd = sc.parallelize(texts, 2) word_to_ic = analyze_texts(data_rdd) @@ -155,6 +155,7 @@ def train(sc, parser.add_option("-m", "--max_epoch", dest="max_epoch", default="15") parser.add_option("--model", dest="model_type", default="cnn") parser.add_option("-p", "--p", dest="p", default="0.0") + parser.add_option("-d", "--data_path", dest="data_path", default="/tmp/news20/") (options, args) = parser.parse_args(sys.argv) if options.action == "train": @@ -168,10 +169,11 @@ def train(sc, training_split = 0.8 sc = SparkContext(appName="text_classifier", conf=create_spark_conf()) + data_path = options.data_path redire_spark_logs() show_bigdl_info_logs() init_engine() - train(sc, + train(sc, data_path, batch_size, sequence_len, max_words, embedding_dim, training_split) sc.stop() From 827b92f22f97653daac35ce2da15dc5aa56b8313 Mon Sep 17 00:00:00 2001 From: Jerry Wu Date: Mon, 30 Oct 2017 10:11:40 +0800 Subject: [PATCH 313/823] Add caffe unit test jobs and test cases framework (#1733) * add run caffe jobs * fix style * fix typo * chmod * add test info * refinement * fix import * refinement * refinement to get exit status * fix typo * fix path --- python/dllib/test/bigdl/caffe/caffe_layers.py | 40 ++++++++++++ .../test/bigdl/caffe/test_caffe_layers.py | 64 +++++++++++++++++++ .../test/bigdl/{ => caffe}/test_load_caffe.py | 16 +---- python/dllib/test/dev/prepare_env.sh | 2 +- python/dllib/test/dev/run-caffe.sh | 38 +++++++++++ python/dllib/test/dev/run-tests | 3 +- 6 files changed, 146 insertions(+), 17 deletions(-) create mode 100644 python/dllib/test/bigdl/caffe/caffe_layers.py create mode 100644 python/dllib/test/bigdl/caffe/test_caffe_layers.py rename python/dllib/test/bigdl/{ => caffe}/test_load_caffe.py (92%) create mode 100755 python/dllib/test/dev/run-caffe.sh diff --git a/python/dllib/test/bigdl/caffe/caffe_layers.py b/python/dllib/test/bigdl/caffe/caffe_layers.py new file mode 100644 index 00000000000..b0614301d75 --- /dev/null +++ b/python/dllib/test/bigdl/caffe/caffe_layers.py @@ -0,0 +1,40 @@ +# Test data for convolution +convolutionDefinition = """ +name : "ConvolutionTest" +input : "data" +input_shape {dim:1 dim :3 dim :5 dim :5} +layer { + name : "convolution" + type : "Convolution" + bottom : "data" + top : "convolution" + convolution_param { + num_output : 4 + kernel_size: 2 + weight_filler { + type: "xavier" + } + bias_filler { + type: "gaussian" + std: 0.02 + } + } + } +""" +convolutionShapes = [{"data": (1, 3, 5, 5)}] +convolutionName = "convolution" +# End layer definitions +testlayers = [] + + +class caffe_test_layer(): + def __init__(self, name, definition, shapes): + self.name = name + self.definition = definition + self.shapes = shapes + + +def registerTestLayer(name, definition, shapes): + layer = caffe_test_layer(name, definition, shapes) + testlayers.append(layer) +registerTestLayer(convolutionName, convolutionDefinition, convolutionShapes) diff --git a/python/dllib/test/bigdl/caffe/test_caffe_layers.py b/python/dllib/test/bigdl/caffe/test_caffe_layers.py new file mode 100644 index 00000000000..6623addf0d6 --- /dev/null +++ b/python/dllib/test/bigdl/caffe/test_caffe_layers.py @@ -0,0 +1,64 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from bigdl.nn.layer import * +from bigdl.optim.optimizer import * +from bigdl.util.common import * +import numpy as np +import pytest +import tempfile +from numpy import random +from numpy.testing import assert_allclose + +import caffe +from caffe_layers import testlayers + + +class TestCaffeLayers(): + + def test_caffe_layers(self): + temp = tempfile.mkdtemp() + for testlayer in testlayers: + name = testlayer.name + definition = testlayer.definition + shapes = testlayer.shapes + prototxtfile = temp + name + ".prototxt" + weightfile = temp + name + ".caffemodel" + prototxt = open(prototxtfile, 'w') + prototxt.write(definition) + prototxt.close() + caffe.set_mode_cpu() + caffe.set_random_seed(100) + net = caffe.Net(prototxtfile, caffe.TEST) + inputs = [] + for shape in shapes: + (inputName, size) = shape.items()[0] + input = random.uniform(size=size) + net.blobs[inputName].data[...] = input + inputs.append(input) + cafferesult = net.forward().get(name) + net.save(weightfile) + model = Model.load_caffe_model(prototxtfile, weightfile, bigdl_type="float") + model.set_seed(100) + if len(inputs) == 1: + inputs = inputs[0] + bigdlResult = model.forward(inputs) + print cafferesult + print bigdlResult + assert_allclose(cafferesult, bigdlResult, atol=1e-4, rtol=0) + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/python/dllib/test/bigdl/test_load_caffe.py b/python/dllib/test/bigdl/caffe/test_load_caffe.py similarity index 92% rename from python/dllib/test/bigdl/test_load_caffe.py rename to python/dllib/test/bigdl/caffe/test_load_caffe.py index 129e94c91ea..6067082c695 100644 --- a/python/dllib/test/bigdl/test_load_caffe.py +++ b/python/dllib/test/bigdl/caffe/test_load_caffe.py @@ -23,24 +23,10 @@ class TestLoadCaffe(): - def setup_method(self, method): - """ setup any state tied to the execution of the given method in a - class. setup_method is invoked for every test method of a class. - """ - sparkConf = create_spark_conf() - self.sc = SparkContext(master="local[4]", appName="test model", - conf=sparkConf) - init_engine() - - def teardown_method(self, method): - """ teardown any state that was previously setup with a setup_method - call. - """ - self.sc.stop() def test_load_caffe(self): # test load caffe with match all parameters - resource_path = os.path.join(os.path.split(__file__)[0], "resources") + resource_path = os.path.join(os.path.split(__file__)[0], "../resources") proto_txt = os.path.join(resource_path, "test.prototxt") model_path = os.path.join(resource_path, "test.caffemodel") diff --git a/python/dllib/test/dev/prepare_env.sh b/python/dllib/test/dev/prepare_env.sh index a3a3825cbcf..27c0900d881 100755 --- a/python/dllib/test/dev/prepare_env.sh +++ b/python/dllib/test/dev/prepare_env.sh @@ -28,7 +28,7 @@ if [ -z ${SPARK_HOME+x} ]; then echo "SPARK_HOME is unset"; exit 1; else echo "S export PYSPARK_ZIP=`find $SPARK_HOME/python/lib -type f -iname '*.zip' | tr "\n" ":"` -export PYTHONPATH=$PYSPARK_ZIP:$DL_PYTHON_HOME:$DL_PYTHON_HOME/:$DL_PYTHON_HOME/test/dev:$BIGDL_HOME/spark/dl/src/main/resources/spark-bigdl.conf +export PYTHONPATH=$PYTHONPATH:$PYSPARK_ZIP:$DL_PYTHON_HOME:$DL_PYTHON_HOME/:$DL_PYTHON_HOME/test/dev:$BIGDL_HOME/spark/dl/src/main/resources/spark-bigdl.conf export BIGDL_CLASSPATH=$(find $BIGDL_HOME/spark/dl/target/ -name "*with-dependencies.jar" | head -n 1) echo "BIGDL_CLASSPATH": $BIGDL_CLASSPATH diff --git a/python/dllib/test/dev/run-caffe.sh b/python/dllib/test/dev/run-caffe.sh new file mode 100755 index 00000000000..8234eb90d9f --- /dev/null +++ b/python/dllib/test/dev/run-caffe.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash + +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +. `dirname $0`/prepare_env.sh + +cd "`dirname $0`" + +export DL_CORE_NUMBER=4 + +p="python2.7" +echo "${cyan}Using python version: $p${reset}" +export PYTHON_EXECUTABLE=$p +export PYSPARK_PYTHON=$p +export PYSPARK_DRIVER_PYTHON=$p +$p -m pytest -v ../../../pyspark/test/bigdl/caffe/ \ +--ignore=../../../pyspark/test/bigdl/caffe/caffe_layers.py +exit_status=$? +echo "running caffe layer unit tests" +if [ $exit_status -ne 0 ]; +then + exit $exit_status +fi \ No newline at end of file diff --git a/python/dllib/test/dev/run-tests b/python/dllib/test/dev/run-tests index 33fd95db966..62bc3802820 100755 --- a/python/dllib/test/dev/run-tests +++ b/python/dllib/test/dev/run-tests @@ -32,7 +32,8 @@ do --ignore=../../../pyspark/bigdl/dataset/ \ --ignore=../../../pyspark/bigdl/util/tf_utils.py \ --ignore=../../../pyspark/bigdl/models/ && \ - $p -m pytest -v -n 4 ../../../pyspark/test/ + $p -m pytest -v -n 4 ../../../pyspark/test/ \ + --ignore=../../../pyspark/test/bigdl/caffe/ exit_status=$? if [ $exit_status -ne 0 ]; then From 83e49c0e074bb4786d44b9c5d10968bdcb6bd9b2 Mon Sep 17 00:00:00 2001 From: Xin Qiu Date: Tue, 31 Oct 2017 10:00:11 +0800 Subject: [PATCH 314/823] MSRA filler (#1802) --- python/dllib/src/bigdl/dllib/nn/initialization_method.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/nn/initialization_method.py b/python/dllib/src/bigdl/dllib/nn/initialization_method.py index e8c9ecd29db..aaddee5597f 100644 --- a/python/dllib/src/bigdl/dllib/nn/initialization_method.py +++ b/python/dllib/src/bigdl/dllib/nn/initialization_method.py @@ -81,6 +81,14 @@ class Xavier(InitializationMethod): def __init__(self, bigdl_type="float"): JavaValue.__init__(self, None, bigdl_type) +class MsraFiller(InitializationMethod): + """ + MsraFiller Initializer. + See https://www.cv-foundation.org/openaccess/content_iccv_2015/papers/He_Delving_Deep_into_ICCV_2015_paper.pdf + """ + def __init__(self, varianceNormAverage=True, bigdl_type="float"): + JavaValue.__init__(self, None, bigdl_type, varianceNormAverage) + class BilinearFiller(InitializationMethod): """ Initialize the weight with coefficients for bilinear interpolation. From a645bd50a0b75e44796b0975383cc7480f3dc044 Mon Sep 17 00:00:00 2001 From: Xin Qiu Date: Tue, 31 Oct 2017 10:00:11 +0800 Subject: [PATCH 315/823] MSRA filler (#1802) --- python/dllib/src/bigdl/dllib/nn/initialization_method.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/nn/initialization_method.py b/python/dllib/src/bigdl/dllib/nn/initialization_method.py index e8c9ecd29db..aaddee5597f 100644 --- a/python/dllib/src/bigdl/dllib/nn/initialization_method.py +++ b/python/dllib/src/bigdl/dllib/nn/initialization_method.py @@ -81,6 +81,14 @@ class Xavier(InitializationMethod): def __init__(self, bigdl_type="float"): JavaValue.__init__(self, None, bigdl_type) +class MsraFiller(InitializationMethod): + """ + MsraFiller Initializer. + See https://www.cv-foundation.org/openaccess/content_iccv_2015/papers/He_Delving_Deep_into_ICCV_2015_paper.pdf + """ + def __init__(self, varianceNormAverage=True, bigdl_type="float"): + JavaValue.__init__(self, None, bigdl_type, varianceNormAverage) + class BilinearFiller(InitializationMethod): """ Initialize the weight with coefficients for bilinear interpolation. From 2cf6b7970afc7e0c91cec8cf3ac80f0e74abfed8 Mon Sep 17 00:00:00 2001 From: Xin Qiu Date: Tue, 31 Oct 2017 10:00:11 +0800 Subject: [PATCH 316/823] MSRA filler (#1802) --- python/dllib/test/bigdl/test_simple_integration.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/dllib/test/bigdl/test_simple_integration.py b/python/dllib/test/bigdl/test_simple_integration.py index ada34dcde4b..c842e1f76a0 100644 --- a/python/dllib/test/bigdl/test_simple_integration.py +++ b/python/dllib/test/bigdl/test_simple_integration.py @@ -366,6 +366,7 @@ def test_init_method(self): None ] special_initializers = [ + MsraFiller(False), Xavier(), RandomUniform(), ] From 09eb6e6b3f66bb8646508da6f78988ffed634cd5 Mon Sep 17 00:00:00 2001 From: Ian Wong Date: Tue, 31 Oct 2017 11:16:36 +0800 Subject: [PATCH 317/823] bump bigdl version to 0.4.0-SNAPSHOT (#1807) --- version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.py b/version.py index 13932794d54..2be0f54013f 100644 --- a/version.py +++ b/version.py @@ -14,4 +14,4 @@ # limitations under the License. # -__version__ = "0.3.0.dev0" \ No newline at end of file +__version__ = "0.4.0.dev0" \ No newline at end of file From b0e65c611c39a626876d003ba72c3c0ea78efdcb Mon Sep 17 00:00:00 2001 From: Ian Wong Date: Tue, 31 Oct 2017 11:16:36 +0800 Subject: [PATCH 318/823] bump bigdl version to 0.4.0-SNAPSHOT (#1807) --- version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.py b/version.py index 13932794d54..2be0f54013f 100644 --- a/version.py +++ b/version.py @@ -14,4 +14,4 @@ # limitations under the License. # -__version__ = "0.3.0.dev0" \ No newline at end of file +__version__ = "0.4.0.dev0" \ No newline at end of file From d207bbc7570292817ba1c28376784bfb73946814 Mon Sep 17 00:00:00 2001 From: Jerry Wu Date: Wed, 1 Nov 2017 15:37:25 +0800 Subject: [PATCH 319/823] Caffe layer wise unit test (#1810) * add run caffe jobs * fix style * fix typo * chmod * add test info * refinement * fix import * refinement * refinement to get exit status * fix typo * fix path * add layers unit tests * add python interface * fix style --- python/dllib/src/bigdl/dllib/nn/layer.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index b103135843d..ce0c4b71919 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -4226,6 +4226,24 @@ def __init__(self, input_size, output_size, kernel_i, kernel_c, stride, wRegular super(ConvLSTMPeephole, self).__init__(None, bigdl_type, input_size, output_size, kernel_i, kernel_c, stride, wRegularizer, uRegularizer, bRegularizer, cRegularizer, with_peephole) +class Tile(Layer): + ''' + Replicate 'copies' copy along 'dim' dimension + + >>> layer = Tile(1, 2) + creating: createTile + ''' + def __init__(self, dim = 1, copies = 2, bigdl_type="float"): + super(Tile, self).__init__(None, bigdl_type, dim, copies) + +class BinaryThreshold(Layer): + ''' + Binary threshold, 1 if value > th, 0 otherwise + >>> layer = BinaryThreshold(0.1, False) + creating: createBinaryThreshold + ''' + def __init__(self, th=1e-6, ip = False, bigdl_type="float"): + super(BinaryThreshold, self).__init__(None, bigdl_type, th, ip) class ConvLSTMPeephole3D(Layer): ''' From 451b9fda6b242c5e91f91da32ee1550fe8b0bb94 Mon Sep 17 00:00:00 2001 From: Jerry Wu Date: Wed, 1 Nov 2017 15:37:25 +0800 Subject: [PATCH 320/823] Caffe layer wise unit test (#1810) * add run caffe jobs * fix style * fix typo * chmod * add test info * refinement * fix import * refinement * refinement to get exit status * fix typo * fix path * add layers unit tests * add python interface * fix style --- python/dllib/src/bigdl/dllib/nn/layer.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index b103135843d..ce0c4b71919 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -4226,6 +4226,24 @@ def __init__(self, input_size, output_size, kernel_i, kernel_c, stride, wRegular super(ConvLSTMPeephole, self).__init__(None, bigdl_type, input_size, output_size, kernel_i, kernel_c, stride, wRegularizer, uRegularizer, bRegularizer, cRegularizer, with_peephole) +class Tile(Layer): + ''' + Replicate 'copies' copy along 'dim' dimension + + >>> layer = Tile(1, 2) + creating: createTile + ''' + def __init__(self, dim = 1, copies = 2, bigdl_type="float"): + super(Tile, self).__init__(None, bigdl_type, dim, copies) + +class BinaryThreshold(Layer): + ''' + Binary threshold, 1 if value > th, 0 otherwise + >>> layer = BinaryThreshold(0.1, False) + creating: createBinaryThreshold + ''' + def __init__(self, th=1e-6, ip = False, bigdl_type="float"): + super(BinaryThreshold, self).__init__(None, bigdl_type, th, ip) class ConvLSTMPeephole3D(Layer): ''' From 0c7c6363869469dc96387acdb65d1945bdb42488 Mon Sep 17 00:00:00 2001 From: Jerry Wu Date: Wed, 1 Nov 2017 15:37:25 +0800 Subject: [PATCH 321/823] Caffe layer wise unit test (#1810) * add run caffe jobs * fix style * fix typo * chmod * add test info * refinement * fix import * refinement * refinement to get exit status * fix typo * fix path * add layers unit tests * add python interface * fix style --- python/dllib/test/bigdl/caffe/caffe_layers.py | 586 ++++++++++++++++++ 1 file changed, 586 insertions(+) diff --git a/python/dllib/test/bigdl/caffe/caffe_layers.py b/python/dllib/test/bigdl/caffe/caffe_layers.py index b0614301d75..5c07c64bffe 100644 --- a/python/dllib/test/bigdl/caffe/caffe_layers.py +++ b/python/dllib/test/bigdl/caffe/caffe_layers.py @@ -23,6 +23,564 @@ """ convolutionShapes = [{"data": (1, 3, 5, 5)}] convolutionName = "convolution" + + +# Test data for Relu +reluDefinition = """ +name : "ReluTest" +input : "data" +input_shape{dim:2 dim :2} + layer { + name: "relu" + type: "ReLU" + bottom: "data" + top: "relu" +} +""" +reluShapes = [{"data": (2, 2)}] +reluName = "relu" + + +# Test Data for SpatialCrossMapLRN +crossMapLrnDefinition = """ +name : "SpatialCrossMapLRNTest" +input : "data" +input_shape{dim:1 dim :3 dim:224 dim :224} +layer { + name: "crossMapLrn" + type: "LRN" + bottom: "data" + top: "crossMapLrn" + lrn_param { + local_size: 5 + alpha: 1.0E-4 + beta: 0.75 + k: 1.0 + } +} +""" +crossMapLrnShapes = [{"data": (1, 3, 224, 224)}] +crossMapLrnName = "crossMapLrn" + + +# Test Data for SpatialWithinChannelLRN +withinChannelLRNDefinition = """ +name : "SpatialWithinChannelLRNTest" +input : "data" +input_shape{dim:1 dim :3 dim:224 dim :224} +layer { + name: "withinChannelLRN" + type: "LRN" + bottom: "data" + top: "withinChannelLRN" + lrn_param { + local_size: 5 + alpha: 1.0E-4 + beta: 0.75 + k: 1.0 + norm_region : WITHIN_CHANNEL + } +} +""" +withinChannelLRNShapes = [{"data": (1, 3, 224, 224)}] +withinChannelLRNName = "withinChannelLRN" + +# Test data for Inner product +innerProductDefinition = """ +name : "InnerProductTest" +input : "data" +input_shape{dim: 2 dim: 10} +layer { + name: "innerProduct" + type: "InnerProduct" + bottom: "data" + top: "innerProduct" + inner_product_param { + num_output: 10 + } +} +""" + +innerProductShapes = [{"data": (2, 10)}] +innerProductName = "innerProduct" + +# Test data for max pooling +maxpoolingDefinition = """ +name : "MaxpoolingTest" +input : "data" +input_shape{dim: 1 dim: 3 dim: 3 dim: 3} +layer { + name: "maxpooling" + type: "Pooling" + bottom: "data" + top: "maxpooling" + pooling_param { + pool: MAX + kernel_size: 2 + stride: 2 + } +} +""" +maxpoolingShapes = [{"data": (1, 3, 3, 3)}] +maxpoolingName = "maxpooling" + +# Test data for average pooling +avepoolingDefinition = """ +name : "AvepoolingTest" +input : "data" +input_shape{dim: 1 dim: 3 dim: 3 dim: 3} +layer { + name: "avepooling" + type: "Pooling" + bottom: "data" + top: "avepooling" + pooling_param { + pool: AVE + kernel_size: 2 + stride: 2 + } +} +""" +avepoolingShapes = [{"data": (1, 3, 3, 3)}] +avepoolingName = "avepooling" + +# Test data for SoftMax +softMaxDefinition = """ +name : "SoftMaxTest" +input : "data" +input_shape{dim: 2 dim: 2} +layer { + name: "softMax" + type: "Softmax" + bottom: "data" + top: "softMax" +} +""" +softMaxShapes = [{"data": (2, 2)}] +softMaxName = "softMax" + +# Test data for Tanh +tanhDefinition = """ +name : "TanhTest" +input : "data" +input_shape{dim: 2 dim: 2} +layer { + name: "tanh" + type: "TanH" + bottom: "data" + top: "tanh" +} +""" +tanhShapes = [{"data": (2, 2)}] +tanhName = "tanh" + +# Test data for Sigmoid +sigmoidDefinition = """ +name : "SigmoidTest" +input : "data" +input_shape{dim: 2 dim: 2} +layer { + name: "sigmoid" + type: "Sigmoid" + bottom: "data" + top: "sigmoid" +} +""" +sigmoidShapes = [{"data": (2, 2)}] +sigmoidName = "sigmoid" + +# Test data for Abs +absDefinition = """ +name : "AbsTest" +input : "data" +input_shape{dim: 2 dim: 2} +layer { + name: "abs" + type: "AbsVal" + bottom: "data" + top: "abs" +} +""" +absShapes = [{"data": (2, 2)}] +absName = "abs" + +# Test data for BatchNormalization +batchNormDefinition = """ +name : "BatchNormTest" +input: "data" +input_dim: 1 +input_dim: 3 +input_dim: 224 +input_dim: 224 + +layer { + bottom: "data" + top: "conv1" + name: "conv1" + type: "Convolution" + convolution_param { + num_output: 64 + kernel_size: 7 + pad: 3 + stride: 2 + } +} + +layer { + bottom: "conv1" + top: "batchNorm" + name: "batchNorm" + type: "BatchNorm" + batch_norm_param { + use_global_stats: true + } +} +""" +batchNormShapes = [{"data": (1, 3, 224, 224)}] +batchNormName = "batchNorm" + +# Test data for Concat +concatDefinition = """ +name : "ConcatTest" +input : "data1" +input_shape{dim: 2 dim: 2} +input : "data2" +input_shape{dim: 2 dim: 2} +layer { + name: "abs" + type: "AbsVal" + bottom: "data1" + top: "abs" +} +layer { + name: "sigmoid" + type: "Sigmoid" + bottom: "data2" + top: "sigmoid" +} +layer { + name: "concat" + type: "Concat" + bottom: "abs" + bottom: "sigmoid" + top: "concat" +} +""" +concatShapes = [{"data1": (2, 2)}, {"data2": (2, 2)}] +concatName = "concat" + +# Test data for Elu +eluDefinition = """ +name : "EluTest" +input : "data" +input_shape{dim: 2 dim: 2} +layer { + name: "elu" + type: "ELU" + bottom: "data" + top: "elu" +} +""" +eluShapes = [{"data": (2, 2)}] +eluName = "elu" + +# Test data for Flattern +flattenDefinition = """ +name : "FlattenTest" +input : "data" +input_shape{dim: 2 dim: 2} +layer { + name: "flatten" + type: "Flatten" + bottom: "data" + top: "flatten" +} +""" +flattenShapes = [{"data": (2, 2)}] +flattenName = "flatten" + +# Test data for Log +logDefinition = """ +name : "LogTest" +input : "data" +input_shape{dim: 2 dim: 2} +layer { + name: "log" + type: "Log" + bottom: "data" + top: "log" +} +""" +logShapes = [{"data": (2, 2)}] +logName = "log" + +# Test data for Power +powerDefinition = """ +name : "PowerTest" +input : "data" +input_shape{dim: 2 dim: 2} +layer { + name: "power" + type: "Power" + bottom: "data" + top: "power" +} +""" +powerShapes = [{"data": (2, 2)}] +powerName = "power" + +# Test data for PReLU +preluDefinition = """ +name : "PReLUTest" +input : "data" +input_shape{dim: 2 dim: 5} +layer { + name: "prelu" + type: "PReLU" + bottom: "data" + top: "prelu" +} +""" +preluShapes = [{"data": (2, 5)}] +preluName = "prelu" + +# Test data for Reshape +reshapeDefinition = """ +name : "ReshapeTest" +input : "data" +input_shape{dim: 2 dim: 8} +layer { + name: "reshape" + type: "Reshape" + bottom: "data" + top: "reshape" + reshape_param { shape { dim: 0 dim: -1 dim: 4 } } +} +""" +reshapeShapes = [{"data": (2, 8)}] +reshapeName = "reshape" + +# Test data for Scale +scaleDefinition = """ +name : "ScaleTest" +input : "data" +input_shape{dim: 2 dim: 2} +layer { + name: "scale" + type: "Scale" + bottom: "data" + top: "scale" +} +""" +scaleShapes = [{"data": (2, 2)}] +scaleName = "scale" + +# Test data for Bias +biasDefinition = """ +name : "BiasTest" +input : "data" +input_shape{dim: 2 dim: 2} +layer { + name: "bias" + type: "Bias" + bottom: "data" + top: "bias" +} +""" +biasShapes = [{"data": (2, 2)}] +biasName = "bias" + +# Test data for Threshold +thresholdDefinition = """ +name : "ThresholdTest" +input : "data" +input_shape{dim: 2 dim: 2} +layer { + name: "threshold" + type: "Threshold" + bottom: "data" + top: "threshold" + threshold_param { + threshold : 0.5 + } +} +""" +thresholdShapes = [{"data": (2, 2)}] +thresholdName = "threshold" + +# Test data for Exp +expDefinition = """ +name : "ExpTest" +input : "data" +input_shape{dim: 2 dim: 2} +layer { + name: "exp" + type: "Exp" + bottom: "data" + top: "exp" +} +""" +expShapes = [{"data": (2, 2)}] +expName = "exp" + +# Test data for Slice +sliceDefinition = """ +name : "SliceTest" +input : "data" +input_shape{dim: 2 dim: 2} +layer { + name: "slice" + type: "Slice" + bottom: "data" + top: "slice" +} +""" +sliceShapes = [{"data": (2, 2)}] +sliceName = "slice" + +# Test data for Tile +tileDefinition = """ +name : "TileTest" +input : "data" +input_shape{dim: 2 dim : 2} +layer { + name: "tile" + type: "Tile" + bottom: "data" + top: "tile" + tile_param { + axis : 1 + tiles : 2 + } +} +""" +tileShapes = [{"data": (2, 2)}] +tileName = "tile" + +# Test data for Eltwise MAX +eltwiseMaxDefinition = """ +name : "EltwiseMaxTest" +input : "data1" +input_shape{dim: 2 dim: 2} +input : "data2" +input_shape{dim: 2 dim: 2} +layer { + name: "abs" + type: "AbsVal" + bottom: "data1" + top: "abs" +} +layer { + name: "sigmoid" + type: "Sigmoid" + bottom: "data2" + top: "sigmoid" +} +layer { + name: "eltwiseMax" + type: "Eltwise" + bottom: "abs" + bottom: "sigmoid" + top: "eltwiseMax" + eltwise_param { + operation : MAX + } +} +""" +eltwiseMaxShapes = [{"data1": (2, 2)}, {"data2": (2, 2)}] +eltwiseMaxName = "eltwiseMax" + +# Test data for Eltwise Prod +eltwiseProdDefinition = """ +name : "EltwiseProdTest" +input : "data1" +input_shape{dim: 2 dim: 2} +input : "data2" +input_shape{dim: 2 dim: 2} +layer { + name: "abs" + type: "AbsVal" + bottom: "data1" + top: "abs" +} +layer { + name: "sigmoid" + type: "Sigmoid" + bottom: "data2" + top: "sigmoid" +} +layer { + name: "eltwiseProd" + type: "Eltwise" + bottom: "abs" + bottom: "sigmoid" + top: "eltwiseProd" + eltwise_param { + operation : PROD + } +} +""" +eltwiseProdShapes = [{"data1": (2, 2)}, {"data2": (2, 2)}] +eltwiseProdName = "eltwiseProd" + +# Test data for Eltwise SUM +eltwiseSUMDefinition = """ +name : "EltwiseSUMTest" +input : "data1" +input_shape{dim: 2 dim: 2} +input : "data2" +input_shape{dim: 2 dim: 2} +layer { + name: "abs1" + type: "AbsVal" + bottom: "data1" + top: "abs1" +} +layer { + name: "abs2" + type: "AbsVal" + bottom: "data2" + top: "abs2" +} +layer { + name: "eltwiseSUM" + type: "Eltwise" + bottom: "abs1" + bottom: "abs2" + top: "eltwiseSUM" + eltwise_param { + operation : SUM + coeff: [0.5 , 1.0] + } +} +""" +eltwiseSUMShapes = [{"data1": (2, 2)}, {"data2": (2, 2)}] +eltwiseSUMName = "eltwiseSUM" + +deconvolutionDefinition = """ +name : "deconvolution" +input : "data" +input_shape {dim:1 dim :3 dim :5 dim :5} +layer { + name: "deconvolution" + type: "Deconvolution" + bottom: "data" + top: "deconvolution" + convolution_param { + num_output: 4 + pad: 0 + kernel_size: 2 + stride: 2 + weight_filler { + type: "xavier" + } + } +} + +""" +deconvolutionShapes = [{"data": (1, 3, 5, 5)}] +deconvolutionName = "deconvolution" + # End layer definitions testlayers = [] @@ -38,3 +596,31 @@ def registerTestLayer(name, definition, shapes): layer = caffe_test_layer(name, definition, shapes) testlayers.append(layer) registerTestLayer(convolutionName, convolutionDefinition, convolutionShapes) +registerTestLayer(reluName, reluDefinition, reluShapes) +registerTestLayer(crossMapLrnName, crossMapLrnDefinition, crossMapLrnShapes) +registerTestLayer(withinChannelLRNName, withinChannelLRNDefinition, withinChannelLRNShapes) +registerTestLayer(innerProductName, innerProductDefinition, innerProductShapes) +registerTestLayer(maxpoolingName, maxpoolingDefinition, maxpoolingShapes) +registerTestLayer(avepoolingName, avepoolingDefinition, avepoolingShapes) +registerTestLayer(softMaxName, softMaxDefinition, softMaxShapes) +registerTestLayer(tanhName, tanhDefinition, tanhShapes) +registerTestLayer(sigmoidName, sigmoidDefinition, sigmoidShapes) +registerTestLayer(absName, absDefinition, absShapes) +registerTestLayer(batchNormName, batchNormDefinition, batchNormShapes) +registerTestLayer(concatName, concatDefinition, concatShapes) +registerTestLayer(eluName, eluDefinition, eluShapes) +registerTestLayer(flattenName, flattenDefinition, flattenShapes) +registerTestLayer(logName, logDefinition, logShapes) +registerTestLayer(powerName, powerDefinition, powerShapes) +registerTestLayer(preluName, preluDefinition, preluShapes) +registerTestLayer(reshapeName, reshapeDefinition, reshapeShapes) +registerTestLayer(scaleName, scaleDefinition, scaleShapes) +registerTestLayer(biasName, biasDefinition, biasShapes) +registerTestLayer(thresholdName, thresholdDefinition, thresholdShapes) +registerTestLayer(expName, expDefinition, expShapes) +registerTestLayer(sliceName, sliceDefinition, sliceShapes) +registerTestLayer(tileName, tileDefinition, tileShapes) +registerTestLayer(eltwiseMaxName, eltwiseMaxDefinition, eltwiseMaxShapes) +registerTestLayer(eltwiseProdName, eltwiseProdDefinition, eltwiseProdShapes) +registerTestLayer(eltwiseSUMName, eltwiseSUMDefinition, eltwiseSUMShapes) +registerTestLayer(deconvolutionName, deconvolutionDefinition, deconvolutionShapes) From e93858cfe2295a5e82d7d8a3d19c4cbc38b201c7 Mon Sep 17 00:00:00 2001 From: Jerry Wu Date: Mon, 6 Nov 2017 10:55:00 +0800 Subject: [PATCH 322/823] Fix load bigdl_conf_file exception (#1831) * update bigdl download url * fix (#1719) * fix jtensor (#1722) * [Backport 0.3 PR1728]Bump script (#1729) * checkin bump-version script * add comments * check if zip file contains the conf --- python/dllib/src/bigdl/dllib/utils/common.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/utils/common.py b/python/dllib/src/bigdl/dllib/utils/common.py index 83292e055cd..5fa51973447 100644 --- a/python/dllib/src/bigdl/dllib/utils/common.py +++ b/python/dllib/src/bigdl/dllib/utils/common.py @@ -400,10 +400,11 @@ def load_conf(conf_str): if bigdl_python_wrapper in p and os.path.isfile(p): import zipfile with zipfile.ZipFile(p, 'r') as zip_conf: - content = zip_conf.read(bigdl_conf_file) - if sys.version_info >= (3,): - content = str(content, 'latin-1') - return load_conf(content) + if bigdl_conf_file in zip_conf.namelist(): + content = zip_conf.read(bigdl_conf_file) + if sys.version_info >= (3,): + content = str(content, 'latin-1') + return load_conf(content) raise Exception("Cannot find spark-bigdl.conf.Pls add it to PYTHONPATH.") From 6da5d25282f2162c2fa2f02d0190403aad0b2e1f Mon Sep 17 00:00:00 2001 From: Jerry Wu Date: Mon, 6 Nov 2017 10:55:00 +0800 Subject: [PATCH 323/823] Fix load bigdl_conf_file exception (#1831) * update bigdl download url * fix (#1719) * fix jtensor (#1722) * [Backport 0.3 PR1728]Bump script (#1729) * checkin bump-version script * add comments * check if zip file contains the conf --- python/dllib/src/bigdl/utils/common.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/python/dllib/src/bigdl/utils/common.py b/python/dllib/src/bigdl/utils/common.py index 83292e055cd..5fa51973447 100644 --- a/python/dllib/src/bigdl/utils/common.py +++ b/python/dllib/src/bigdl/utils/common.py @@ -400,10 +400,11 @@ def load_conf(conf_str): if bigdl_python_wrapper in p and os.path.isfile(p): import zipfile with zipfile.ZipFile(p, 'r') as zip_conf: - content = zip_conf.read(bigdl_conf_file) - if sys.version_info >= (3,): - content = str(content, 'latin-1') - return load_conf(content) + if bigdl_conf_file in zip_conf.namelist(): + content = zip_conf.read(bigdl_conf_file) + if sys.version_info >= (3,): + content = str(content, 'latin-1') + return load_conf(content) raise Exception("Cannot find spark-bigdl.conf.Pls add it to PYTHONPATH.") From 26f06d67f1dbf32c57387635b16b4bef22b405c2 Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Tue, 7 Nov 2017 20:37:22 -0600 Subject: [PATCH 324/823] Expose local optimizer and predictor (#1689) * local refactor update update * nupdate * support list of input * add unit-test * update * style * compile * fix * fix --- python/dllib/src/bigdl/dllib/nn/layer.py | 37 ++++++ .../dllib/src/bigdl/dllib/optim/optimizer.py | 113 +++++++++++++++--- 2 files changed, 133 insertions(+), 17 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index ce0c4b71919..67b97eee19e 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -267,6 +267,43 @@ def evaluate(self, *args): else: raise Exception("Error when calling evaluate(): it takes no argument or exactly three arguments only") + def _to_jtensors(self, x): + x = to_list(x) + if isinstance(x[0], np.ndarray): + return [JTensor.from_ndarray(i) for i in x] + elif isinstance(x[0], JTensor): + return x + else: + raise Exception("Not supported type: %s" % type(x[0])) + + + def predict_local(self, X): + """ + :param X: X can be a ndarray or list of ndarray if the model has multiple inputs. + The first dimension of X should be batch. + :return: a ndarray as the prediction result. + """ + + jresults = callBigDlFunc(self.bigdl_type, + "predictLocal", + self.value, + self._to_jtensors(X)) + + return np.stack([j.to_ndarray()for j in jresults]) + + def predict_local_class(self, X): + """ + + :param X: X can be a ndarray or list of ndarray if the model has multiple inputs. + The first dimension of X should be batch. + :return: a ndarray as the prediction result. + """ + result = callBigDlFunc(self.bigdl_type, + "predictLocalClass", + self.value, + self._to_jtensors(X)) + return np.stack(result) + def predict(self, data_rdd): """ Model inference base on the given data. diff --git a/python/dllib/src/bigdl/dllib/optim/optimizer.py b/python/dllib/src/bigdl/dllib/optim/optimizer.py index 93f6f0aa0e7..a817dd006cf 100644 --- a/python/dllib/src/bigdl/dllib/optim/optimizer.py +++ b/python/dllib/src/bigdl/dllib/optim/optimizer.py @@ -27,7 +27,7 @@ from bigdl.util.common import get_spark_context from bigdl.util.common import to_list from py4j.java_gateway import JavaObject - +import multiprocessing if sys.version >= '3': long = int @@ -520,10 +520,81 @@ def __init__(self, step_sizes, gamma, bigdl_type="float"): JavaValue.__init__(self, None, bigdl_type, step_sizes, gamma) +class DistriOptimizer(JavaValue): + def __init__(self, + model, + training_rdd, + criterion, + end_trigger, + batch_size, + optim_method=None, + bigdl_type="float"): + """ + Create an optimizer. + + + :param model: the neural net model + :param training_data: the training dataset + :param criterion: the loss function + :param optim_method: the algorithm to use for optimization, + e.g. SGD, Adagrad, etc. If optim_method is None, the default algorithm is SGD. + :param end_trigger: when to end the optimization + :param batch_size: training batch size + """ + JavaValue.__init__(self, None, bigdl_type, model.value, + training_rdd, criterion, + optim_method if optim_method else SGD(), end_trigger, batch_size) + + +class LocalOptimizer(JavaValue): + """ + Create an optimizer. + + + :param model: the neural net model + :param X: the training features which is an ndarray or list of ndarray + :param Y: the training label which is an ndarray + :param criterion: the loss function + :param optim_method: the algorithm to use for optimization, + e.g. SGD, Adagrad, etc. If optim_method is None, the default algorithm is SGD. + :param end_trigger: when to end the optimization + :param batch_size: training batch size + :param cores: by default is the total physical cores. + """ + def __init__(self, + X, + y, + model, + criterion, + end_trigger, + batch_size, + optim_method=None, + cores=None, + bigdl_type="float"): + if cores is None: + cores = multiprocessing.cpu_count() + JavaValue.__init__(self, None, bigdl_type, + [JTensor.from_ndarray(X) for X in to_list(X)], + JTensor.from_ndarray(y), + model.value, + criterion, + optim_method if optim_method else SGD(), end_trigger, batch_size, cores) + + def optimize(self): + """ + Do an optimization. + """ + jmodel = callJavaFunc(get_spark_context(), self.value.optimize) + from bigdl.nn.layer import Layer + return Layer.of(jmodel) + + class Optimizer(JavaValue): """ + This is a distrubuted optimizer. An optimizer is in general to minimize any function with respect - to a set of parameters. In case of training a neural network, + to a set of parameters. + In case of training a neural network, an optimizer tries to minimize the loss of the neural net with respect to its weights/biases, over the training set. """ @@ -535,21 +606,29 @@ def __init__(self, batch_size, optim_method=None, bigdl_type="float"): - """ - Create an optimizer. - - - :param model: the neural net model - :param training_rdd: the training dataset - :param criterion: the loss function - :param optim_method: the algorithm to use for optimization, - e.g. SGD, Adagrad, etc. If optim_method is None, the default algorithm is SGD. - :param end_trigger: when to end the optimization - :param batch_size: training batch size - """ - JavaValue.__init__(self, None, bigdl_type, model.value, - training_rdd, criterion, - optim_method if optim_method else SGD(), end_trigger, batch_size) + """ + Create an optimizer. + + + :param model: the neural net model + :param training_rdd: the training dataset + :param criterion: the loss function + :param optim_method: the algorithm to use for optimization, + e.g. SGD, Adagrad, etc. If optim_method is None, the default algorithm is SGD. + :param end_trigger: when to end the optimization + :param batch_size: training batch size + """ + self.pvalue = DistriOptimizer(model, + training_rdd, + criterion, + end_trigger, + batch_size, + optim_method, + bigdl_type) + self.value = self.pvalue.value + self.bigdl_type = self.pvalue.bigdl_type + + def set_validation(self, batch_size, val_rdd, trigger, val_method=None): """ From 3e7247fa1933c511f9d24008e5f8165112e89539 Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Tue, 7 Nov 2017 20:37:22 -0600 Subject: [PATCH 325/823] Expose local optimizer and predictor (#1689) * local refactor update update * nupdate * support list of input * add unit-test * update * style * compile * fix * fix --- python/dllib/src/bigdl/dllib/nn/layer.py | 37 ++++++ .../dllib/src/bigdl/dllib/optim/optimizer.py | 113 +++++++++++++++--- 2 files changed, 133 insertions(+), 17 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index ce0c4b71919..67b97eee19e 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -267,6 +267,43 @@ def evaluate(self, *args): else: raise Exception("Error when calling evaluate(): it takes no argument or exactly three arguments only") + def _to_jtensors(self, x): + x = to_list(x) + if isinstance(x[0], np.ndarray): + return [JTensor.from_ndarray(i) for i in x] + elif isinstance(x[0], JTensor): + return x + else: + raise Exception("Not supported type: %s" % type(x[0])) + + + def predict_local(self, X): + """ + :param X: X can be a ndarray or list of ndarray if the model has multiple inputs. + The first dimension of X should be batch. + :return: a ndarray as the prediction result. + """ + + jresults = callBigDlFunc(self.bigdl_type, + "predictLocal", + self.value, + self._to_jtensors(X)) + + return np.stack([j.to_ndarray()for j in jresults]) + + def predict_local_class(self, X): + """ + + :param X: X can be a ndarray or list of ndarray if the model has multiple inputs. + The first dimension of X should be batch. + :return: a ndarray as the prediction result. + """ + result = callBigDlFunc(self.bigdl_type, + "predictLocalClass", + self.value, + self._to_jtensors(X)) + return np.stack(result) + def predict(self, data_rdd): """ Model inference base on the given data. diff --git a/python/dllib/src/bigdl/dllib/optim/optimizer.py b/python/dllib/src/bigdl/dllib/optim/optimizer.py index 93f6f0aa0e7..a817dd006cf 100644 --- a/python/dllib/src/bigdl/dllib/optim/optimizer.py +++ b/python/dllib/src/bigdl/dllib/optim/optimizer.py @@ -27,7 +27,7 @@ from bigdl.util.common import get_spark_context from bigdl.util.common import to_list from py4j.java_gateway import JavaObject - +import multiprocessing if sys.version >= '3': long = int @@ -520,10 +520,81 @@ def __init__(self, step_sizes, gamma, bigdl_type="float"): JavaValue.__init__(self, None, bigdl_type, step_sizes, gamma) +class DistriOptimizer(JavaValue): + def __init__(self, + model, + training_rdd, + criterion, + end_trigger, + batch_size, + optim_method=None, + bigdl_type="float"): + """ + Create an optimizer. + + + :param model: the neural net model + :param training_data: the training dataset + :param criterion: the loss function + :param optim_method: the algorithm to use for optimization, + e.g. SGD, Adagrad, etc. If optim_method is None, the default algorithm is SGD. + :param end_trigger: when to end the optimization + :param batch_size: training batch size + """ + JavaValue.__init__(self, None, bigdl_type, model.value, + training_rdd, criterion, + optim_method if optim_method else SGD(), end_trigger, batch_size) + + +class LocalOptimizer(JavaValue): + """ + Create an optimizer. + + + :param model: the neural net model + :param X: the training features which is an ndarray or list of ndarray + :param Y: the training label which is an ndarray + :param criterion: the loss function + :param optim_method: the algorithm to use for optimization, + e.g. SGD, Adagrad, etc. If optim_method is None, the default algorithm is SGD. + :param end_trigger: when to end the optimization + :param batch_size: training batch size + :param cores: by default is the total physical cores. + """ + def __init__(self, + X, + y, + model, + criterion, + end_trigger, + batch_size, + optim_method=None, + cores=None, + bigdl_type="float"): + if cores is None: + cores = multiprocessing.cpu_count() + JavaValue.__init__(self, None, bigdl_type, + [JTensor.from_ndarray(X) for X in to_list(X)], + JTensor.from_ndarray(y), + model.value, + criterion, + optim_method if optim_method else SGD(), end_trigger, batch_size, cores) + + def optimize(self): + """ + Do an optimization. + """ + jmodel = callJavaFunc(get_spark_context(), self.value.optimize) + from bigdl.nn.layer import Layer + return Layer.of(jmodel) + + class Optimizer(JavaValue): """ + This is a distrubuted optimizer. An optimizer is in general to minimize any function with respect - to a set of parameters. In case of training a neural network, + to a set of parameters. + In case of training a neural network, an optimizer tries to minimize the loss of the neural net with respect to its weights/biases, over the training set. """ @@ -535,21 +606,29 @@ def __init__(self, batch_size, optim_method=None, bigdl_type="float"): - """ - Create an optimizer. - - - :param model: the neural net model - :param training_rdd: the training dataset - :param criterion: the loss function - :param optim_method: the algorithm to use for optimization, - e.g. SGD, Adagrad, etc. If optim_method is None, the default algorithm is SGD. - :param end_trigger: when to end the optimization - :param batch_size: training batch size - """ - JavaValue.__init__(self, None, bigdl_type, model.value, - training_rdd, criterion, - optim_method if optim_method else SGD(), end_trigger, batch_size) + """ + Create an optimizer. + + + :param model: the neural net model + :param training_rdd: the training dataset + :param criterion: the loss function + :param optim_method: the algorithm to use for optimization, + e.g. SGD, Adagrad, etc. If optim_method is None, the default algorithm is SGD. + :param end_trigger: when to end the optimization + :param batch_size: training batch size + """ + self.pvalue = DistriOptimizer(model, + training_rdd, + criterion, + end_trigger, + batch_size, + optim_method, + bigdl_type) + self.value = self.pvalue.value + self.bigdl_type = self.pvalue.bigdl_type + + def set_validation(self, batch_size, val_rdd, trigger, val_method=None): """ From e509fc72c8f7f6682298b321817230e44e1d4fcb Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Tue, 7 Nov 2017 20:37:22 -0600 Subject: [PATCH 326/823] Expose local optimizer and predictor (#1689) * local refactor update update * nupdate * support list of input * add unit-test * update * style * compile * fix * fix --- .../test/bigdl/test_simple_integration.py | 62 ++++++++++++++++++- 1 file changed, 61 insertions(+), 1 deletion(-) diff --git a/python/dllib/test/bigdl/test_simple_integration.py b/python/dllib/test/bigdl/test_simple_integration.py index c842e1f76a0..cdfd7aaab9b 100644 --- a/python/dllib/test/bigdl/test_simple_integration.py +++ b/python/dllib/test/bigdl/test_simple_integration.py @@ -25,8 +25,9 @@ import numpy as np import tempfile import pytest -from numpy.testing import assert_allclose +from numpy.testing import assert_allclose, assert_array_equal from bigdl.util.engine import compare_version +np.random.seed(1337) # for reproducibility class TestSimple(): @@ -468,5 +469,64 @@ def test_compare_version(self): assert compare_version("2.1.0", "2.1.1") == -1 assert compare_version("2.0.1", "1.5.2") == 1 + def test_local_optimizer_predict(self): + feature_num = 2 + data_len = 1000 + batch_size = 32 + epoch_num = 500 + + X_ = np.random.uniform(0, 1, (data_len, feature_num)) + y_ = (2 * X_).sum(1) + 0.4 + model = Sequential() + l1 = Linear(feature_num, 1) + model.add(l1) + + localOptimizer = LocalOptimizer( + model=model, + X=X_, + y=y_, + criterion=MSECriterion(), + optim_method=SGD(learningrate=1e-2), + end_trigger=MaxEpoch(epoch_num), + batch_size=batch_size) + trained_model = localOptimizer.optimize() + trained_model = model + w = trained_model.get_weights() + assert_allclose(w[0], np.array([2, 2]).reshape([1, 2]), rtol=1e-1) + assert_allclose(w[1], np.array([0.4]), rtol=1e-1) + + predict_result = trained_model.predict_local(X_) + assert_allclose(y_, predict_result.reshape((data_len,)), rtol=1e-1) + + def test_local_predict_class(self): + feature_num = 2 + data_len = 3 + X_ = np.random.uniform(-1, 1, (data_len, feature_num)) + model = Sequential() + l1 = Linear(feature_num, 1) + model.add(l1) + model.add(Sigmoid()) + model.set_seed(1234).reset() + predict_result = model.predict_local_class(X_) + assert_array_equal(predict_result, np.ones([3])) + + def test_local_predict_multiple_input(self): + l1 = Linear(3, 2)() + l2 = Linear(3, 3)() + joinTable = JoinTable(dimension=1, n_input_dims=1)([l1, l2]) + model = Model(inputs=[l1, l2], outputs=joinTable) + result = model.predict_local([np.ones([4, 3]), np.ones([4, 3])]) + assert result.shape == (4, 5) + result2 = model.predict_local_class([np.ones([4, 3]), np.ones([4, 3])]) + assert result2.shape == (4,) + + result3 = model.predict_local([JTensor.from_ndarray(np.ones([4, 3])), + JTensor.from_ndarray(np.ones([4, 3]))]) + assert result3.shape == (4, 5) + result4 = model.predict_local_class([JTensor.from_ndarray(np.ones([4, 3])), + JTensor.from_ndarray(np.ones([4, 3]))]) + assert result4.shape == (4,) + + if __name__ == "__main__": pytest.main([__file__]) From 43165587d03c44511dc59598549a36ed2aa86c7b Mon Sep 17 00:00:00 2001 From: dding3 Date: Fri, 10 Nov 2017 15:16:15 -0500 Subject: [PATCH 327/823] support SAME padding in 3d conv and allows user config padding size in convlstm and convlstm3d (#1862) * support SAME padding in 3d conv * allows user config padding size in convlstm and convlstm3d --- python/dllib/src/bigdl/dllib/nn/layer.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 67b97eee19e..9fade510de9 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -4243,14 +4243,15 @@ class ConvLSTMPeephole(Layer): :param output_size: number of output planes the convolution layer will produce :param kernel_i Convolutional filter size to convolve input :param kernel_c Convolutional filter size to convolve cell - :param stride The step of the convolution + :param stride The step of the convolution, default is 1 + :param padding The additional zeros added, default is -1 :param wRegularizer: instance of [[Regularizer]](eg. L1 or L2 regularization), applied to the input weights matrices :param uRegularizer: instance [[Regularizer]](eg. L1 or L2 regularization), applied to the recurrent weights matrices :param bRegularizer: instance of [[Regularizer]]applied to the bias. :param cRegularizer: instance of [[Regularizer]]applied to peephole. :param with_peephole: whether use last cell status control a gate. - >>> convlstm = ConvLSTMPeephole(4, 3, 3, 3, 1, L1Regularizer(0.5), L1Regularizer(0.5), L1Regularizer(0.5), L1Regularizer(0.5)) + >>> convlstm = ConvLSTMPeephole(4, 3, 3, 3, 1, -1, L1Regularizer(0.5), L1Regularizer(0.5), L1Regularizer(0.5), L1Regularizer(0.5)) creating: createL1Regularizer creating: createL1Regularizer creating: createL1Regularizer @@ -4258,10 +4259,10 @@ class ConvLSTMPeephole(Layer): creating: createConvLSTMPeephole ''' - def __init__(self, input_size, output_size, kernel_i, kernel_c, stride, wRegularizer=None, uRegularizer=None, + def __init__(self, input_size, output_size, kernel_i, kernel_c, stride=1, padding=-1, wRegularizer=None, uRegularizer=None, bRegularizer=None, cRegularizer=None, with_peephole=True, bigdl_type="float"): super(ConvLSTMPeephole, self).__init__(None, bigdl_type, input_size, output_size, kernel_i, kernel_c, stride, - wRegularizer, uRegularizer, bRegularizer, cRegularizer, with_peephole) + padding, wRegularizer, uRegularizer, bRegularizer, cRegularizer, with_peephole) class Tile(Layer): ''' @@ -4290,13 +4291,14 @@ class ConvLSTMPeephole3D(Layer): :param kernel_i Convolutional filter size to convolve input :param kernel_c Convolutional filter size to convolve cell :param stride The step of the convolution + :param padding The additional zeros added :param wRegularizer: instance of [[Regularizer]](eg. L1 or L2 regularization), applied to the input weights matrices :param uRegularizer: instance [[Regularizer]](eg. L1 or L2 regularization), applied to the recurrent weights matrices :param bRegularizer: instance of [[Regularizer]]applied to the bias. :param cRegularizer: instance of [[Regularizer]]applied to peephole. :param with_peephole: whether use last cell status control a gate. - >>> convlstm = ConvLSTMPeephole3D(4, 3, 3, 3, 1, L1Regularizer(0.5), L1Regularizer(0.5), L1Regularizer(0.5), L1Regularizer(0.5)) + >>> convlstm = ConvLSTMPeephole3D(4, 3, 3, 3, 1, -1, L1Regularizer(0.5), L1Regularizer(0.5), L1Regularizer(0.5), L1Regularizer(0.5)) creating: createL1Regularizer creating: createL1Regularizer creating: createL1Regularizer @@ -4304,10 +4306,10 @@ class ConvLSTMPeephole3D(Layer): creating: createConvLSTMPeephole3D ''' - def __init__(self, input_size, output_size, kernel_i, kernel_c, stride, wRegularizer=None, uRegularizer=None, + def __init__(self, input_size, output_size, kernel_i, kernel_c, stride=1, padding=-1, wRegularizer=None, uRegularizer=None, bRegularizer=None, cRegularizer=None, with_peephole=True, bigdl_type="float"): super(ConvLSTMPeephole3D, self).__init__(None, bigdl_type, input_size, output_size, kernel_i, kernel_c, stride, - wRegularizer, uRegularizer, bRegularizer, cRegularizer, with_peephole) + padding, wRegularizer, uRegularizer, bRegularizer, cRegularizer, with_peephole) class ResizeBilinear(Layer): """ From 455434edc672f876df32dd9f60f5cb7ba98b4f34 Mon Sep 17 00:00:00 2001 From: dding3 Date: Fri, 10 Nov 2017 15:16:15 -0500 Subject: [PATCH 328/823] support SAME padding in 3d conv and allows user config padding size in convlstm and convlstm3d (#1862) * support SAME padding in 3d conv * allows user config padding size in convlstm and convlstm3d --- python/dllib/src/bigdl/dllib/nn/layer.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 67b97eee19e..9fade510de9 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -4243,14 +4243,15 @@ class ConvLSTMPeephole(Layer): :param output_size: number of output planes the convolution layer will produce :param kernel_i Convolutional filter size to convolve input :param kernel_c Convolutional filter size to convolve cell - :param stride The step of the convolution + :param stride The step of the convolution, default is 1 + :param padding The additional zeros added, default is -1 :param wRegularizer: instance of [[Regularizer]](eg. L1 or L2 regularization), applied to the input weights matrices :param uRegularizer: instance [[Regularizer]](eg. L1 or L2 regularization), applied to the recurrent weights matrices :param bRegularizer: instance of [[Regularizer]]applied to the bias. :param cRegularizer: instance of [[Regularizer]]applied to peephole. :param with_peephole: whether use last cell status control a gate. - >>> convlstm = ConvLSTMPeephole(4, 3, 3, 3, 1, L1Regularizer(0.5), L1Regularizer(0.5), L1Regularizer(0.5), L1Regularizer(0.5)) + >>> convlstm = ConvLSTMPeephole(4, 3, 3, 3, 1, -1, L1Regularizer(0.5), L1Regularizer(0.5), L1Regularizer(0.5), L1Regularizer(0.5)) creating: createL1Regularizer creating: createL1Regularizer creating: createL1Regularizer @@ -4258,10 +4259,10 @@ class ConvLSTMPeephole(Layer): creating: createConvLSTMPeephole ''' - def __init__(self, input_size, output_size, kernel_i, kernel_c, stride, wRegularizer=None, uRegularizer=None, + def __init__(self, input_size, output_size, kernel_i, kernel_c, stride=1, padding=-1, wRegularizer=None, uRegularizer=None, bRegularizer=None, cRegularizer=None, with_peephole=True, bigdl_type="float"): super(ConvLSTMPeephole, self).__init__(None, bigdl_type, input_size, output_size, kernel_i, kernel_c, stride, - wRegularizer, uRegularizer, bRegularizer, cRegularizer, with_peephole) + padding, wRegularizer, uRegularizer, bRegularizer, cRegularizer, with_peephole) class Tile(Layer): ''' @@ -4290,13 +4291,14 @@ class ConvLSTMPeephole3D(Layer): :param kernel_i Convolutional filter size to convolve input :param kernel_c Convolutional filter size to convolve cell :param stride The step of the convolution + :param padding The additional zeros added :param wRegularizer: instance of [[Regularizer]](eg. L1 or L2 regularization), applied to the input weights matrices :param uRegularizer: instance [[Regularizer]](eg. L1 or L2 regularization), applied to the recurrent weights matrices :param bRegularizer: instance of [[Regularizer]]applied to the bias. :param cRegularizer: instance of [[Regularizer]]applied to peephole. :param with_peephole: whether use last cell status control a gate. - >>> convlstm = ConvLSTMPeephole3D(4, 3, 3, 3, 1, L1Regularizer(0.5), L1Regularizer(0.5), L1Regularizer(0.5), L1Regularizer(0.5)) + >>> convlstm = ConvLSTMPeephole3D(4, 3, 3, 3, 1, -1, L1Regularizer(0.5), L1Regularizer(0.5), L1Regularizer(0.5), L1Regularizer(0.5)) creating: createL1Regularizer creating: createL1Regularizer creating: createL1Regularizer @@ -4304,10 +4306,10 @@ class ConvLSTMPeephole3D(Layer): creating: createConvLSTMPeephole3D ''' - def __init__(self, input_size, output_size, kernel_i, kernel_c, stride, wRegularizer=None, uRegularizer=None, + def __init__(self, input_size, output_size, kernel_i, kernel_c, stride=1, padding=-1, wRegularizer=None, uRegularizer=None, bRegularizer=None, cRegularizer=None, with_peephole=True, bigdl_type="float"): super(ConvLSTMPeephole3D, self).__init__(None, bigdl_type, input_size, output_size, kernel_i, kernel_c, stride, - wRegularizer, uRegularizer, bRegularizer, cRegularizer, with_peephole) + padding, wRegularizer, uRegularizer, bRegularizer, cRegularizer, with_peephole) class ResizeBilinear(Layer): """ From a4cb1a478fbd807b386a504ddd08a10f7a5751b7 Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Mon, 13 Nov 2017 17:00:30 +0800 Subject: [PATCH 329/823] fix squeeze bug and some other refinement (#1874) --- python/dllib/src/bigdl/dllib/utils/tf_utils.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/python/dllib/src/bigdl/dllib/utils/tf_utils.py b/python/dllib/src/bigdl/dllib/utils/tf_utils.py index 7e8d071daca..f1d294cb368 100644 --- a/python/dllib/src/bigdl/dllib/utils/tf_utils.py +++ b/python/dllib/src/bigdl/dllib/utils/tf_utils.py @@ -117,9 +117,14 @@ def save_variable_bigdl(tensors, target_path, bigdl_type="float"): :param bigdl_type: model variable numeric type :return: nothing """ + import numpy as np jtensors = {} for tn in tensors.keys(): - jtensors[tn] = JTensor.from_ndarray(tensors[tn]) + if not isinstance(tensors[tn], np.ndarray): + value = np.array(tensors[tn]) + else: + value = tensors[tn] + jtensors[tn] = JTensor.from_ndarray(value) callBigDlFunc(bigdl_type, "saveTensorDictionary", jtensors, target_path) From a65b2b8604818a3459dc1fe80c2b33b343b24c9b Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Mon, 13 Nov 2017 17:00:30 +0800 Subject: [PATCH 330/823] fix squeeze bug and some other refinement (#1874) --- python/dllib/src/bigdl/dllib/utils/tf_utils.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/python/dllib/src/bigdl/dllib/utils/tf_utils.py b/python/dllib/src/bigdl/dllib/utils/tf_utils.py index 7e8d071daca..f1d294cb368 100644 --- a/python/dllib/src/bigdl/dllib/utils/tf_utils.py +++ b/python/dllib/src/bigdl/dllib/utils/tf_utils.py @@ -117,9 +117,14 @@ def save_variable_bigdl(tensors, target_path, bigdl_type="float"): :param bigdl_type: model variable numeric type :return: nothing """ + import numpy as np jtensors = {} for tn in tensors.keys(): - jtensors[tn] = JTensor.from_ndarray(tensors[tn]) + if not isinstance(tensors[tn], np.ndarray): + value = np.array(tensors[tn]) + else: + value = tensors[tn] + jtensors[tn] = JTensor.from_ndarray(value) callBigDlFunc(bigdl_type, "saveTensorDictionary", jtensors, target_path) From 26f289ec743b122b51d22174a4fd7b4e98acc414 Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Tue, 14 Nov 2017 12:45:08 +0800 Subject: [PATCH 331/823] Support keras squared_hinge and sparse_categorial_crossentropy (#1865) * add squared hinge loss * classNLL support prob as input * add python api * meet code review * fix python tests * more doc * more doc --- python/dllib/src/bigdl/dllib/nn/criterion.py | 33 +++++++++++++------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/nn/criterion.py b/python/dllib/src/bigdl/dllib/nn/criterion.py index 8f5aa3a3dcd..8f94a03f88b 100644 --- a/python/dllib/src/bigdl/dllib/nn/criterion.py +++ b/python/dllib/src/bigdl/dllib/nn/criterion.py @@ -103,13 +103,14 @@ class ClassNLLCriterion(Criterion): classes. If provided, the optional argument weights should be a 1D Tensor assigning weight to each of the classes. This is particularly useful when you have an unbalanced training set. - The input given through a forward() is expected to contain log-probabilities of each class: - input has to be a 1D Tensor of size n. Obtaining log-probabilities in a neural network is easily - achieved by adding a LogSoftMax layer in the last layer of your neural network. You may use - CrossEntropyCriterion instead, if you prefer not to add an extra layer to your network. This - criterion expects a class index (1 to the number of class) as target when calling - forward(input, target) and backward(input, target). - + The input given through a forward() is expected to contain log-probabilities/probabilities of + each class: input has to be a 1D Tensor of size n. Obtaining log-probabilities/probabilities + in a neural network is easily achieved by adding a LogSoftMax/SoftMax layer in the last layer + of your neural network. You may use CrossEntropyCriterion instead, if you prefer not to add an + extra layer to your network. This criterion expects a class index (1 to the number of class) as + target when calling forward(input, target) and backward(input, target). + + In the log-probabilities case, The loss can be described as: loss(x, class) = -x[class] or in the case of the weights argument it is specified as follows: @@ -124,14 +125,18 @@ class ClassNLLCriterion(Criterion): By default, the losses are averaged over observations for each minibatch. However, if the field sizeAverage is set to false, the losses are instead summed for each minibatch. + In particular, when weights=None, size_average=True and logProbAsInput=False, this is same as + `sparse_categorical_crossentropy` loss in keras. + :param weights: weights of each class :param size_average: whether to average or not + :param logProbAsInput: indicating whether to accept log-probabilities or probabilities as input. >>> np.random.seed(123) >>> weights = np.random.uniform(0, 1, (2,)).astype("float32") - >>> classNLLCriterion = ClassNLLCriterion(weights,True) + >>> classNLLCriterion = ClassNLLCriterion(weights, True, True) creating: createClassNLLCriterion >>> classNLLCriterion = ClassNLLCriterion() creating: createClassNLLCriterion @@ -140,10 +145,11 @@ class ClassNLLCriterion(Criterion): def __init__(self, weights=None, size_average=True, + logProbAsInput=True, bigdl_type="float"): super(ClassNLLCriterion, self).__init__(None, bigdl_type, JTensor.from_ndarray(weights), - size_average) + size_average, logProbAsInput) class MSECriterion(Criterion): @@ -344,22 +350,27 @@ class MarginCriterion(Criterion): Creates a criterion that optimizes a two-class classification hinge loss (margin-based loss) between input x (a Tensor of dimension 1) and output y. + When margin = 1, size_average = True and squared = False, this is the same as hinge loss in keras; + When margin = 1, size_average = False and squared = True, this is the same as squared_hinge loss in keras. :param margin: if unspecified, is by default 1. :param size_average: size average in a mini-batch + :param squared: whether to calculate the squared hinge loss - >>> marginCriterion = MarginCriterion(1e-5, True) + >>> marginCriterion = MarginCriterion(1e-5, True, False) creating: createMarginCriterion ''' def __init__(self, margin=1.0, size_average=True, + squared=False, bigdl_type="float"): super(MarginCriterion, self).__init__(None, bigdl_type, margin, - size_average) + size_average, + squared) class MarginRankingCriterion(Criterion): From 88ffd25174fb69b27d58e1b28083f56d49241b53 Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Tue, 14 Nov 2017 12:45:08 +0800 Subject: [PATCH 332/823] Support keras squared_hinge and sparse_categorial_crossentropy (#1865) * add squared hinge loss * classNLL support prob as input * add python api * meet code review * fix python tests * more doc * more doc --- python/dllib/src/bigdl/dllib/nn/criterion.py | 33 +++++++++++++------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/nn/criterion.py b/python/dllib/src/bigdl/dllib/nn/criterion.py index 8f5aa3a3dcd..8f94a03f88b 100644 --- a/python/dllib/src/bigdl/dllib/nn/criterion.py +++ b/python/dllib/src/bigdl/dllib/nn/criterion.py @@ -103,13 +103,14 @@ class ClassNLLCriterion(Criterion): classes. If provided, the optional argument weights should be a 1D Tensor assigning weight to each of the classes. This is particularly useful when you have an unbalanced training set. - The input given through a forward() is expected to contain log-probabilities of each class: - input has to be a 1D Tensor of size n. Obtaining log-probabilities in a neural network is easily - achieved by adding a LogSoftMax layer in the last layer of your neural network. You may use - CrossEntropyCriterion instead, if you prefer not to add an extra layer to your network. This - criterion expects a class index (1 to the number of class) as target when calling - forward(input, target) and backward(input, target). - + The input given through a forward() is expected to contain log-probabilities/probabilities of + each class: input has to be a 1D Tensor of size n. Obtaining log-probabilities/probabilities + in a neural network is easily achieved by adding a LogSoftMax/SoftMax layer in the last layer + of your neural network. You may use CrossEntropyCriterion instead, if you prefer not to add an + extra layer to your network. This criterion expects a class index (1 to the number of class) as + target when calling forward(input, target) and backward(input, target). + + In the log-probabilities case, The loss can be described as: loss(x, class) = -x[class] or in the case of the weights argument it is specified as follows: @@ -124,14 +125,18 @@ class ClassNLLCriterion(Criterion): By default, the losses are averaged over observations for each minibatch. However, if the field sizeAverage is set to false, the losses are instead summed for each minibatch. + In particular, when weights=None, size_average=True and logProbAsInput=False, this is same as + `sparse_categorical_crossentropy` loss in keras. + :param weights: weights of each class :param size_average: whether to average or not + :param logProbAsInput: indicating whether to accept log-probabilities or probabilities as input. >>> np.random.seed(123) >>> weights = np.random.uniform(0, 1, (2,)).astype("float32") - >>> classNLLCriterion = ClassNLLCriterion(weights,True) + >>> classNLLCriterion = ClassNLLCriterion(weights, True, True) creating: createClassNLLCriterion >>> classNLLCriterion = ClassNLLCriterion() creating: createClassNLLCriterion @@ -140,10 +145,11 @@ class ClassNLLCriterion(Criterion): def __init__(self, weights=None, size_average=True, + logProbAsInput=True, bigdl_type="float"): super(ClassNLLCriterion, self).__init__(None, bigdl_type, JTensor.from_ndarray(weights), - size_average) + size_average, logProbAsInput) class MSECriterion(Criterion): @@ -344,22 +350,27 @@ class MarginCriterion(Criterion): Creates a criterion that optimizes a two-class classification hinge loss (margin-based loss) between input x (a Tensor of dimension 1) and output y. + When margin = 1, size_average = True and squared = False, this is the same as hinge loss in keras; + When margin = 1, size_average = False and squared = True, this is the same as squared_hinge loss in keras. :param margin: if unspecified, is by default 1. :param size_average: size average in a mini-batch + :param squared: whether to calculate the squared hinge loss - >>> marginCriterion = MarginCriterion(1e-5, True) + >>> marginCriterion = MarginCriterion(1e-5, True, False) creating: createMarginCriterion ''' def __init__(self, margin=1.0, size_average=True, + squared=False, bigdl_type="float"): super(MarginCriterion, self).__init__(None, bigdl_type, margin, - size_average) + size_average, + squared) class MarginRankingCriterion(Criterion): From a2e9da22a112199e719649db7debf88f26c02a53 Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Wed, 15 Nov 2017 09:15:24 -0600 Subject: [PATCH 333/823] Support keras model loading (#1827) * keras load model and api wrapper * fix batchnorm * fix averagepooling * remove function wrapper and clean code * add unit test * remove backend * fix and clean * update * separate unittest * x * remove -n * ignore application * separate test * pass parameter * try to fix test * disable shape checking * ignore --- keras1/__init__.py | 15 + keras1/converter.py | 1111 ++++++++++++++++++ python/dllib/src/bigdl/dllib/nn/layer.py | 84 +- python/dllib/src/bigdl/dllib/utils/common.py | 11 +- 4 files changed, 1208 insertions(+), 13 deletions(-) create mode 100644 keras1/__init__.py create mode 100644 keras1/converter.py diff --git a/keras1/__init__.py b/keras1/__init__.py new file mode 100644 index 00000000000..2151a805423 --- /dev/null +++ b/keras1/__init__.py @@ -0,0 +1,15 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/keras1/converter.py b/keras1/converter.py new file mode 100644 index 00000000000..90f65991249 --- /dev/null +++ b/keras1/converter.py @@ -0,0 +1,1111 @@ + +import bigdl.nn.initialization_method as BInit +import numpy as np +import bigdl.nn.layer as BLayer +import bigdl.optim.optimizer as boptimizer +import bigdl.nn.criterion as bcriterion +import bigdl.util.common as bcommon +import keras.optimizers as koptimizers +from keras.models import model_from_json +from keras.models import Sequential, Model +import keras +import warnings + + +def unsupport_exp(name): + raise Exception("We don't support %s for now" % name) + + +class WeightLoader: + + @staticmethod + def __load_weights_by_execution_seq(bmodel, kmodel): + blayers = [l for l in bmodel.layers if l.is_with_weights()] + klayers = [l for l in kmodel.layers if l.get_weights()] + if len(blayers) != len(klayers): + raise Exception( + "keras with %s layers but bigdl with %s layers" % (len(klayers), len(blayers))) + for b, k in zip(blayers, klayers): + if b.name() != k.name: + raise Exception("Found different layer in execution order, bigdl:%s, keras: %s" % (b.name(), k.name)) # noqa + bigdl_weights = WeightsConverter.get_bigdl_weigths_from_keras(k) + b.set_weights(bigdl_weights) + + # TODO: add more unitest + @staticmethod + def __load_weights_by_name(bmodel, kmodel, by_name=False): + keras_name_to_layer = WeightLoader.__keras_name_to_Layers(kmodel, with_weights=True) + bigdl_name_to_layer = WeightLoader.__bigdl_name_to_Layers(bmodel, with_weights=True) + layers_not_in_keras = set(bigdl_name_to_layer.keys()) - set(keras_name_to_layer.keys()) + if layers_not_in_keras: + raise Exception("Layers %s can be found in bigdl, but not in keras" % repr(layers_not_in_keras)) # noqa + layers_not_in_bigdl = set(keras_name_to_layer.keys()) - set(bigdl_name_to_layer.keys()) + if layers_not_in_bigdl: + if by_name: + warnings.warn("Ignore weight of layers %s as it cannot be found in bigdl" % repr(layers_not_in_bigdl)) # noqa + else: + raise Exception("Layers %s can be found in bigdl, but not in keras" % repr(layers_not_in_keras)) # noqa + for blayer in bigdl_name_to_layer.values(): + if blayer.name() in keras_name_to_layer: + klayer = keras_name_to_layer[blayer.name()] + bigdl_weights = WeightsConverter.get_bigdl_weigths_from_keras(klayer) + blayer.set_weights(bigdl_weights) + if isinstance(klayer, keras.layers.BatchNormalization): + blayer.set_running_mean(keras.backend.eval(klayer.running_mean)) + blayer.set_running_std(keras.backend.eval(klayer.running_std)) + + @staticmethod + def load_weights_from_kmodel(bmodel, kmodel, by_name=False): + """ + Load weights from kmodel to bmodel + """ + if by_name: + WeightLoader.__load_weights_by_name(bmodel, kmodel) + else: + WeightLoader.__load_weights_by_execution_seq(bmodel, kmodel) + + @staticmethod + def load_weights_from_json_hdf5(def_json, weights_hdf5, by_name=False): + with open(def_json, "r") as jp: + kmodel = model_from_json(jp.read()) + bmodel = DefinitionLoader.from_json_path(def_json) + WeightLoader.load_weights_from_hdf5(bmodel, kmodel, weights_hdf5, by_name) + return bmodel + + @staticmethod + def load_weights_from_hdf5(bmodel, kmodel, filepath, by_name=False): + '''Loads all layer weights from a HDF5 save file. + + If `by_name` is False (default) weights are loaded + based on the network's execution order topology, + meaning layers in the execution seq should be exactly the same + the architecture + + If `by_name` is True, weights are loaded into layers + only if they share the same name. This is useful + for fine-tuning or transfer-learning models where + some of the layers have changed. + ''' + kmodel.load_weights(filepath=filepath, by_name=by_name) + WeightLoader.load_weights_from_kmodel(bmodel, kmodel, by_name=by_name) + + @staticmethod + def __keras_name_to_Layers(model, with_weights=False): + if with_weights: + layers = [l for l in model.layers if l.get_weights()] + else: + layers = [l for l in model.layers] + + return dict([(layer.name, layer) for layer in layers]) + + @staticmethod + def __bigdl_name_to_Layers(model, with_weights=False): + if with_weights: + layers = [l for l in model.layers if l.is_with_weights()] + else: + layers = [l for l in model.layers] + + return dict([(layer.name(), layer) for layer in layers]) + + +class WeightsConverter: + """ + Convert keras weights to bigdl weights + The shape of weights would be changed if using different backend, + so we only test against TensorFlow backend. + TODO: Support th backend as well. + """ + + @staticmethod + def get_converter(class_name): + function_name = "convert_" + class_name.lower() + if not hasattr(WeightsConverter, function_name): + raise unsupport_exp(class_name) + converter = getattr(WeightsConverter, function_name) + return converter + + @staticmethod + # weights is a list of ndarray or a ndarray + # convert keras weights per layer to bigdl format + def to_bigdl_weights(class_name, weights): + return WeightsConverter.get_converter(class_name)(weights) + + @staticmethod + def get_bigdl_weigths_from_keras(k): + if isinstance(k, keras.engine.Model): + return WeightsConverter.get_weights_from_kmodel(k) + elif isinstance(k, keras.engine.Layer): + return WeightsConverter.get_bigdl_weights_from_klayer(k) + else: + raise Exception("Unsupport type: %s", k) + + @staticmethod + def get_bigdl_weights_from_klayer(klayer): + # we should use get_weights instead of klayer.weights + return WeightsConverter.to_bigdl_weights(klayer.__class__.__name__, klayer.get_weights()) + + @staticmethod + def get_weights_from_kmodel(kmodel): + """ + Convert kmodel's weights to bigdl format. + We are supposing the order is the same as the execution order. + :param kmodel: keras model + :return: list of ndarray + """ + layers_with_weights = [layer for layer in kmodel.layers if layer.weights] + bweights = [] + for klayer in layers_with_weights: + # bws would be [weiths, bias] or [weights] + bws = WeightsConverter.get_bigdl_weights_from_klayer(klayer) + for w in bws: + bweights.append(w) + return bweights + + @staticmethod + def convert_dense(weights): + return [np.transpose(weights[0]), weights[1]] + + @staticmethod + def convert_timedistributeddense(weights): + return [np.transpose(weights[0]), weights[1]] + + @staticmethod + def convert_batchnormalization(weights): + gamma = weights[0] + beta = weights[1] + return [gamma, beta] + + @staticmethod + def convert_convolution2d(weights): + weight = np.expand_dims(weights[0], 0) # bigdl has a leading dim with value 1 + if len(weights) > 1: + return [weight, weights[1]] + else: + return [weight] + + @staticmethod + def convert_convolution1d(weights): + return WeightsConverter.convert_convolution2d(weights) + + @staticmethod + def convert_embedding(weights): + return weights + + @staticmethod + def convert_simplernn(weights): + return [np.transpose(weights[0]), np.transpose(weights[1]), weights[2]] + + @staticmethod + def convert_lstm(weights): + w1 = np.concatenate((weights[0].T, weights[3].T, weights[6].T, weights[9].T)) + w2 = np.concatenate((weights[2], weights[5], weights[8], weights[11])) + w3 = np.concatenate((weights[1].T, weights[4].T, weights[7].T, weights[10].T)) + return [w1, w2, w3] + + @staticmethod + def convert_gru(weights): + w1 = np.concatenate((weights[3].T, weights[0].T, weights[6].T)) + w2 = np.concatenate((weights[2], weights[5], weights[8])) + w3 = np.concatenate((weights[4].T, weights[1].T)) + w4 = weights[7].T + return [w1, w2, w3, w4] + + +class DefinitionLoader: + + def __init__(self, kmodel): + self.node_id_to_instance = {} + self.node_id_to_layer = {} + self.node_id_to_config_layer = {} + self.kmodel = kmodel + self.kconfig = self.kmodel.get_config() + + for layer in self.kmodel.layers: + self.node_id_to_layer[layer.name] = layer + + if isinstance(self.kmodel, Sequential): + for layer_config in self.kmodel.get_config(): + layer_name = layer_config["config"]["name"] + self.node_id_to_config_layer[layer_name] = layer_config + else: + for layerConfig in self.kconfig["layers"]: + self.node_id_to_config_layer[layerConfig["name"]] = layerConfig + + def __to_bigdl(self): + if isinstance(self.kmodel, Sequential): + bmodel = self._construct_bigdl_sequence() + elif isinstance(self.kmodel, Model): + bmodel = self._construct_bigdl_model() + return bmodel + + @classmethod + def from_kmodel(cls, kmodel): + return cls(kmodel).__to_bigdl() + + @classmethod + def from_json_path(cls, json_path): + with open(json_path, "r") as jp: + return DefinitionLoader.from_json_str(jp.read()) + + @classmethod + def from_json_str(cls, json_str): + kmodel = model_from_json(json_str) + return DefinitionLoader.from_kmodel(kmodel) + + def _do_create_node(self, layer, clayer): + if clayer["class_name"] == "InputLayer": + input = BLayer.Input() + input.element().set_name(layer.name) # cannot set name for node? + self.node_id_to_instance[layer.name] = input + return input + bigdl_in_nodes = [] + for node in clayer["inbound_nodes"]: + for out in node: + out_name = out[0] + out_index = out[1] + out_tensor_index = out[2] + if out_name not in self.node_id_to_instance: + self._do_create_node(self.node_id_to_layer[out_name], + self.node_id_to_config_layer[out_name]) + bigdl_in_nodes.append(self.node_id_to_instance[out_name]) + + blayer = LayerConverter().create(layer, clayer) + new_bnode = blayer(bigdl_in_nodes) + self.node_id_to_instance[layer.name] = new_bnode + return new_bnode + + def _construct_bigdl_model(self): + for clayer in self.kconfig["layers"]: + if clayer["name"] not in self.node_id_to_instance: + + self._do_create_node(self.node_id_to_layer[clayer["name"]], + clayer) + ins = [] + for input_layer in self.kconfig["input_layers"]: + name = input_layer[0] + ins.append(self.node_id_to_instance[name]) + outs = [] + for output_layer in self.kconfig["output_layers"]: + name = output_layer[0] + outs.append(self.node_id_to_instance[name]) + return BLayer.Model(inputs=ins, outputs=outs) + + def _construct_bigdl_sequence(self): + bseq = BLayer.Sequential() + layerConverter = LayerConverter() + for layer in self.kmodel.layers: + blayer = layerConverter.create(layer, self.node_id_to_config_layer[layer.name]) + bseq.add(blayer) + return bseq + +class LayerConverter: + + def __check_is_share_weights(self, kclayer): + # For Merge layer len(kclayer["inbound_nodes"]) is equal to 1 + # "inbound_nodes": [ + # [ + # [ + # "batchnormalization_194", + # 0, + # 0 + # ], + # [ + # "batchnormalization_196", + # 0, + # 0 + # ], + # [ + # "batchnormalization_199", + # 0, + # 0 + # ], + # [ + # "batchnormalization_200", + # 0, + # 0 + # ] + # ] + # ], + if "inbound_nodes" in kclayer and len(kclayer["inbound_nodes"]) > 1: + raise Exception( + "%s doesn't support multiple inputs with shared weights" % kclayer["class_name"]) + + def create(self, klayer, kclayer): + class_name = kclayer["class_name"] + + self.__check_is_share_weights(kclayer) + + if (hasattr(klayer, "b_constraint") and klayer.b_constraint) or \ + (hasattr(klayer, "W_constraint") and klayer.W_constraint): + raise Exception("We don't support constraint for now") + + if (hasattr(klayer, "activity_regularizer") and klayer.activity_regularizer): + raise Exception("We don't support activity_regularizer for now") + + function_name = "create_" + class_name.lower() + if not hasattr(self, function_name): + raise Exception("We don't support layer: %s for now" % class_name ) + + api = getattr(self, function_name) + blayer = api(klayer, kclayer) + return blayer.set_name(klayer.name) + + def create_model(self, klayer, kclyer): + return DefinitionLoader.from_kmodel(klayer) + + def create_inputlayer(self, klayer, kclyer): + return BLayer.Identity() + + def create_dense(self, klayer, kclayer): + config = kclayer["config"] + # Multiple inputs should share the same input_dim for Dense layer + # We don't need to respect the tensor index for method `get_input_shape_at` + # which is internal implementation and `get_input_shape_at` has hided that for us, + # What we need to use is the input index, not node index, not tensor index. + input_shape = klayer.get_input_shape_at(0) + blayer = BLayer.Linear( + input_size=input_shape[1], + output_size=config["output_dim"], + with_bias=config["bias"], + wRegularizer=self.to_bigdl_reg(config["W_regularizer"]), + bRegularizer=self.to_bigdl_reg(config["b_regularizer"]) + ) + return self.combo_parameter_layer(blayer, config) + + def create_timedistributeddense(self, klayer, kclayer): + config = kclayer["config"] + input_shape = klayer.get_input_shape_at(0) + print(input_shape) + print(config["output_dim"]) + blayer = BLayer.TimeDistributed(BLayer.Linear( + input_size=input_shape[2], + output_size=config["output_dim"], + with_bias=config["bias"], + wRegularizer=self.to_bigdl_reg(config["W_regularizer"]), + bRegularizer=self.to_bigdl_reg(config["b_regularizer"]) + )) + return self.combo_parameter_layer(blayer, config) + + def create_embedding(self, klayer, kclayer): + config = kclayer["config"] + input_shape = klayer.get_input_shape_at(0) # batch, seq_len + seq_len = input_shape[1] + if klayer.input_length and klayer.input_length != seq_len: + raise Exception( + "The input_length doesn't match: %s vs %s" % (seq_len, klayer.input_length)) + + if (hasattr(klayer, "dropout") and klayer.dropout != 0): + raise Exception("We don't support dropout for now") + + if (hasattr(klayer, "mask_zero") and klayer.mask_zero != False): + raise Exception("We don't support mask_zero for now") + + bseq = BLayer.Sequential() + blayer = BLayer.LookupTable( + n_index = klayer.input_dim, + n_output = klayer.output_dim, + padding_value=0.0, + norm_type=2.0, + should_scale_grad_by_freq=False, + wRegularizer= self.to_bigdl_reg(config["W_regularizer"]), + bigdl_type="float") + bseq.add(BLayer.AddConstant(1.0, inplace=True)) # Add 1 as BigDL is one-based index + bseq.add(blayer) + return bseq + + def create_activation(self, klayer, kclayer): + config = kclayer["config"] + return self.to_bigdl_activation(config["activation"], klayer.name) + + def create_dropout(self, klayer, kclayer): + return BLayer.Dropout(klayer.p) + + def create_flatten(self, klayer, kclayer): + self.__check_is_share_weights(kclayer) + input_shape = klayer.input_shape + blayer = BLayer.Reshape([np.prod(input_shape[1:])], None) + return blayer + + def create_reshape(self, klayer, kclayer): + self.__check_is_share_weights(kclayer) + blayer = BLayer.Reshape(klayer.target_shape, None) + return blayer + + def create_repeatvector(self, klayer, kclayer): + return BLayer.Replicate(n_features=klayer.n, + n_dim=1, + bigdl_type="float") + + def create_merge(self, klayer, kclayer): + self.__check_is_share_weights(kclayer) + input_shape = klayer.get_input_shape_at(0) + #TODO: dot_axes, node_indices, tensor_indices not supported. + if klayer.output_shape and not isinstance(klayer.output_shape, tuple): + raise Exception("Only output_shape=None or a shape tuple is supported for now.") + if klayer.output_mask: + raise Exception("Argument `output_mask` is not supported for now.") + if klayer.mode == "concat": + blayer = BLayer.JoinTable( + dimension=klayer.concat_axis, + n_input_dims=len(input_shape[0]) - 1, + bigdl_type="float") + elif klayer.mode == "sum": + blayer = BLayer.CAddTable( + inplace=False, + bigdl_type="float") + elif klayer.mode == "mul": + blayer = BLayer.CMulTable(bigdl_type="float") + elif klayer.mode == "max": + blayer = BLayer.CMaxTable(bigdl_type="float") + elif klayer.mode == "dot": + if len(input_shape[0]) >= 3: + raise Exception("Merge mode `dot` doesn't support 3D input or above for now.") + if input_shape[0][0] == None: + raise Exception("For merge mode `dot`, please specify `batch_input_shape`.") + model = BLayer.Sequential() + blayer = model.add(BLayer.DotProduct(bigdl_type="float"))\ + .add(BLayer.Reshape([input_shape[0][0], 1])) + elif klayer.mode in ['ave', 'cos']: + raise Exception("Merge mode `%s` not supported for now" % klayer.mode) # TODO: whether to support cosine and average + else: # invalid mode or lambda functions + raise Exception("Invalid merge mode: `%s`. Lambda/function as merge mode is not supported for now." % klayer.mode) + return blayer + + def create_elu(self, klayer, kclayer): + return BLayer.ELU(alpha=float(klayer.alpha), + inplace=False, + bigdl_type="float") + + def create_prelu(self, klayer, kclayer): + return BLayer.PReLU(n_output_plane=0, + bigdl_type="float") + + def create_leakyrelu(self, klayer, kclayer): + return BLayer.LeakyReLU(negval=float(klayer.alpha), + inplace=False, + bigdl_type="float") + + def create_parametricsoftplus(self, klayer, kclayer): + alpha = float(klayer.alpha_init) + beta = float(klayer.beta_init) + if round(alpha * beta, 4) == 1.0: + return BLayer.SoftPlus(beta=beta, + bigdl_type="float") + else: + raise Exception("Only alpha_init = 1/beta_init is supported for now") + + def create_thresholdedrelu(self, klayer, kclayer): + return BLayer.Threshold(th=float(klayer.theta), + v=0.0, + ip=False, + bigdl_type="float") + + def __generate_zeropadding1d(self, pad_top, pad_bottom): + return BLayer.SpatialZeroPadding(pad_left=0, + pad_right=0, + pad_top=pad_top, + pad_bottom=pad_bottom, + bigdl_type="float") + + def create_zeropadding1d(self, klayer, kclayer): + padding = klayer.padding + if isinstance(padding, int): + return self.__generate_zeropadding1d(padding, padding) + elif isinstance(padding, dict): + return self.__generate_zeropadding1d(padding.get('left_pad', 0), padding.get('right_pad', 0)) + else: # tuple of int (length 2) + padding = tuple(padding) + return self.__generate_zeropadding1d(padding[0], padding[1]) + + def __generate_zeropadding2d(self, dim1, dim2, n_input_dim, pad1, pad2, pad3, pad4): + model = BLayer.Sequential() + paddinglayer1 = BLayer.Padding(dim=dim1, + pad=pad1, + n_input_dim=n_input_dim, + value=0.0, + n_index=1, + bigdl_type="float") + paddinglayer2 = BLayer.Padding(dim=dim1, + pad=pad2, + n_input_dim=n_input_dim, + value=0.0, + n_index=1, + bigdl_type="float") + paddinglayer3 = BLayer.Padding(dim=dim2, + pad=pad3, + n_input_dim=n_input_dim, + value=0.0, + n_index=1, + bigdl_type="float") + paddinglayer4 = BLayer.Padding(dim=dim2, + pad=pad4, + n_input_dim=n_input_dim, + value=0.0, + n_index=1, + bigdl_type="float") + model.add(paddinglayer1) + model.add(paddinglayer2) + model.add(paddinglayer3) + model.add(paddinglayer4) + return model + + # NB: zeropadding doesn't serialize dim_ording to jason file + def create_zeropadding2d(self, klayer, kclayer): + padding = klayer.padding + input_shape = klayer.get_input_shape_at(0) + dim1 = 1 + dim2 = 2 + if klayer.dim_ordering == "th": + dim1 = 2 + dim2 = 3 + if isinstance(padding, dict): # dictionary + return self.__generate_zeropadding2d(dim1, dim2, len(input_shape) - 1, + -padding.get('top_pad', 0), padding.get('bottom_pad', 0), + -padding.get('left_pad', 0), padding.get('right_pad', 0)) + else: # tuple of int + padding = tuple(padding) + if len(padding) == 2: + return self.__generate_zeropadding2d(dim1, dim2, len(input_shape) - 1, + -padding[0], padding[0], -padding[1], padding[1]) + elif len(padding) == 4: + return self.__generate_zeropadding2d(dim1, dim2, len(input_shape) - 1, + -padding[0], padding[1], -padding[2], padding[3]) + + # NB: zeropadding doesn't serialize dim_ording to jason file + def create_zeropadding3d(self, klayer, kclayer): + padding = tuple(klayer.padding) + input_shape = klayer.get_input_shape_at(0) + model = BLayer.Sequential() + paddinglayer1 = BLayer.Padding(dim=2, + pad=-padding[0], + n_input_dim=len(input_shape) - 1, + value=0.0, + n_index=1, + bigdl_type="float") + paddinglayer2 = BLayer.Padding(dim=2, + pad=padding[0], + n_input_dim=len(input_shape) - 1, + value=0.0, + n_index=1, + bigdl_type="float") + paddinglayer3 = BLayer.Padding(dim=3, + pad=-padding[1], + n_input_dim=len(input_shape) - 1, + value=0.0, + n_index=1, + bigdl_type="float") + paddinglayer4 = BLayer.Padding(dim=3, + pad=padding[1], + n_input_dim=len(input_shape) - 1, + value=0.0, + n_index=1, + bigdl_type="float") + paddinglayer5 = BLayer.Padding(dim=4, + pad=-padding[2], + n_input_dim=len(input_shape) - 1, + value=0.0, + n_index=1, + bigdl_type="float") + paddinglayer6 = BLayer.Padding(dim=4, + pad=padding[2], + n_input_dim=len(input_shape) - 1, + value=0.0, + n_index=1, + bigdl_type="float") + model.add(paddinglayer1) + model.add(paddinglayer2) + model.add(paddinglayer3) + model.add(paddinglayer4) + model.add(paddinglayer5) + model.add(paddinglayer6) + return model + + def create_cropping1d(self, klayer, kclayer): + cropping = tuple(klayer.cropping) + return BLayer.SpatialZeroPadding(0, 0, -cropping[0], -cropping[1]) + + def __return_sequences(self, return_sequences, blayer): + # For recurrent layers, handle whether to return the last output sentence or the full sequence. + if return_sequences: + return blayer + else: + model = BLayer.Sequential() + model.add(blayer) + model.add(BLayer.Select(2, -1)) + return model + + def create_simplernn(self, klayer, kclayer): + rec = BLayer.Recurrent() + input_shape = klayer.get_input_shape_at(0) + config = kclayer["config"] + self.check_constraint_in_config(config) + activation = self.to_bigdl_activation(config["activation"], + "%s_%s" % (config["name"], config["activation"])) + # TODO: throw exception for dropout + rnn = BLayer.RnnCell(input_size=input_shape[2], + hidden_size=klayer.output_dim, + activation=activation, + isInputWithBias=False, + wRegularizer=self.to_bigdl_reg(config["W_regularizer"]), + uRegularizer=self.to_bigdl_reg(config["U_regularizer"]), + bRegularizer=self.to_bigdl_reg(config["b_regularizer"]), + bigdl_type="float") + return self.__return_sequences(klayer.return_sequences, rec.add(rnn)) + + def create_lstm(self, klayer, kclayer): + rec = BLayer.Recurrent() + input_shape = klayer.get_input_shape_at(0) + config = kclayer["config"] + self.check_constraint_in_config(config) + activation = self.to_bigdl_activation(config["activation"], + "%s_%s" % (config["name"], config["activation"])) + if not isinstance(activation, BLayer.Tanh): + raise Exception("For activation, only `tanh` is supported for now.") + inner_activation = self.to_bigdl_activation(config["inner_activation"], + "%s_%s" % (config["name"], config["inner_activation"])) + if not isinstance(inner_activation, BLayer.Sigmoid): + raise Exception("For inner_activation, only `sigmond` is supported for now.") + # TODO: throw exception for dropout + lstm = BLayer.LSTM(input_size=input_shape[2], + hidden_size=klayer.output_dim, + p=klayer.dropout_W, + wRegularizer=self.to_bigdl_reg(config["W_regularizer"]), + uRegularizer=self.to_bigdl_reg(config["U_regularizer"]), + bRegularizer=self.to_bigdl_reg(config["b_regularizer"]), + bigdl_type="float") + return self.__return_sequences(klayer.return_sequences, rec.add(lstm)) + + def create_gru(self, klayer, kclayer): + rec = BLayer.Recurrent() + input_shape = klayer.get_input_shape_at(0) + config = kclayer["config"] + self.check_constraint_in_config(config) + activation = self.to_bigdl_activation(config["activation"], + "%s_%s" % (config["name"], config["activation"])) + if not isinstance(activation, BLayer.Tanh): + raise Exception("For activation, only `tanh` is supported for now.") + inner_activation = self.to_bigdl_activation(config["inner_activation"], + "%s_%s" % (config["name"], config["inner_activation"])) + if not isinstance(inner_activation, BLayer.Sigmoid): + raise Exception("For inner_activation, only `sigmond` is supported for now.") + # TODO: throw exception for dropout + gru = BLayer.GRU(input_size=input_shape[2], + hidden_size=klayer.output_dim, + p=klayer.dropout_W, + wRegularizer=self.to_bigdl_reg(config["W_regularizer"]), + uRegularizer=self.to_bigdl_reg(config["U_regularizer"]), + bRegularizer=self.to_bigdl_reg(config["b_regularizer"]), + bigdl_type="float") + return self.__return_sequences(klayer.return_sequences, rec.add(gru)) + + def create_batchnormalization(self, klayer, kclayer): + config = kclayer["config"] + + self.__check_is_share_weights(kclayer) + if keras.backend.image_dim_ordering() != "th" or klayer.axis != 1: + raise Exception("""We only support th image order for BatchNormalization \n + which meaning NCHW for now. but the current order is %s and axis is %s + """ % (keras.backend.image_dim_ordering(), klayer.axis)) # noqa + if klayer.mode != 0: + raise Exception( + "Only support mode = 0 for now, but the current mode is: %s", klayer.mode) + + if config["gamma_regularizer"]: + raise Exception("We don't support gamma_regularizer for now") + + if config["beta_regularizer"]: + raise Exception("We don't support beta_regularizer for now") + + input_shape = klayer.get_input_shape_at(0) + n_input_channel = input_shape[klayer.axis] # default is -1 which is channel-last + + # init gamma and beta + # TODO: replace this with to_bigdl_init in the future + gamma = self.get_value_from_init(klayer.gamma_init.func_name, (n_input_channel,)) + beta = self.get_value_from_init(klayer.beta_init.func_name, (n_input_channel,)) + + blayer = BLayer.SpatialBatchNormalization( + n_output=n_input_channel, + eps=klayer.epsilon, + momentum=klayer.momentum, + affine=True, + init_weight=gamma, + init_bias=beta, + init_grad_weight=None, + init_grad_bias=None, + bigdl_type="float") + + k_running_mean = keras.backend.eval(klayer.running_mean) + k_running_std = keras.backend.eval(klayer.running_std) + blayer.set_running_mean(k_running_mean) + blayer.set_running_std(k_running_std) + return blayer + + def get_bdim_order(self, kclayer): + return self.to_bigdl_2d_ordering(self.get_kdim_order(kclayer)) + + def get_kdim_order(self, kclayer): + config = kclayer["config"] + if "dim_ordering" in config: + return config["dim_ordering"] + else: + warnings.warn("Cannot find dim_ordering from json definition. Use default instead.") + return keras.backend.image_dim_ordering() + + def to_bigdl_2d_ordering(self, order): + if order == "tf": + return "NHWC" + elif order == "th": + return "NCHW" + else: + raise Exception("Unsupport ordering: %s" % order) + + def to_bigdl_2d_padding(self, border_mode): + if border_mode == "same": + return (-1, -1) + elif border_mode == "valid": + return (0, 0) + else: + raise Exception("Unsupported border mode: %s" % border_mode) + + def to_bigdl_1d_padding(self, border_mode, kernel_w): + if border_mode == "same": + raise Exception("We don't support padding for now") + # TODO: support padding + # return int((kernel_w -1) / 2) + elif border_mode == "valid": + return 0 + else: + raise Exception("Unsupported border mode: %s" % border_mode) + +################# Layers with weights ############################# noqa + + def create_convolution1d(self, klayer, kclayer): + config = kclayer["config"] + input_shape = klayer.get_input_shape_at(0) + # batch, steps, dim, batch is None here, so you cannot use it directly. + stack_size = input_shape[2] + + bpadW, bpadH = self.to_bigdl_2d_padding(klayer.border_mode) + seq = BLayer.Sequential() + seq.add(BLayer.Reshape([input_shape[1], 1, input_shape[2]], True)) + blayer = BLayer.SpatialConvolution( + n_input_plane = stack_size, + n_output_plane = klayer.nb_filter, + kernel_w = 1, + kernel_h = klayer.filter_length, + stride_w= 1, + stride_h= klayer.subsample_length, + pad_w= bpadW, + pad_h= bpadH, + n_group=1, + propagate_back=True, + wRegularizer = self.to_bigdl_reg(config["W_regularizer"]), + bRegularizer=self.to_bigdl_reg(config["b_regularizer"]), + init_weight=None, + init_bias=None, + init_grad_weight=None, + init_grad_bias=None, + with_bias=config["bias"], + data_format="NHWC", + bigdl_type="float") + seq.add(blayer) + seq.add(BLayer.Squeeze(3)) + return self.combo_parameter_layer(seq, config) + + def create_convolution2d(self, klayer, kclayer): + config = kclayer["config"] + bigdl_order = self.get_bdim_order(kclayer) + input_shape = klayer.get_input_shape_at(0) + + if bigdl_order == "NCHW": + stack_size = input_shape[1] + elif bigdl_order == "NHWC": + stack_size = input_shape[3] + + bpadW, bpadH = self.to_bigdl_2d_padding(klayer.border_mode) + blayer = BLayer.SpatialConvolution( + n_input_plane = stack_size, + n_output_plane = klayer.nb_filter, + kernel_w = klayer.nb_col, + kernel_h = klayer.nb_row, + stride_w= klayer.subsample[0], + stride_h= klayer.subsample[1], + pad_w= bpadW, + pad_h= bpadH, + n_group=1, + propagate_back=True, + wRegularizer = self.to_bigdl_reg(config["W_regularizer"]), + bRegularizer=self.to_bigdl_reg(config["b_regularizer"]), + init_weight=None, + init_bias=None, + init_grad_weight=None, + init_grad_bias=None, + with_bias=config["bias"], + data_format=bigdl_order, + bigdl_type="float") + + return self.combo_parameter_layer(blayer, config) + + def create_maxpooling2d(self, klayer, kclayer): + bigdl_order = self.get_bdim_order(kclayer) + bpadW, bpadH = self.to_bigdl_2d_padding(klayer.border_mode) + blayer = BLayer.SpatialMaxPooling( + kw = klayer.pool_size[0], + kh = klayer.pool_size[1], + dw = klayer.strides[0], + dh = klayer.strides[1], + pad_w=bpadW, + pad_h=bpadH, + to_ceil=False, + format=bigdl_order, + bigdl_type="float") + return blayer + + def create_averagepooling2d(self, klayer, kclayer): + bigdl_order = self.get_bdim_order(kclayer) + bpadW, bpadH = self.to_bigdl_2d_padding(klayer.border_mode) + blayer = BLayer.SpatialAveragePooling( + kw=klayer.pool_size[0], + kh=klayer.pool_size[1], + dw=klayer.strides[0], + dh=klayer.strides[1], + pad_w=bpadW, + pad_h=bpadH, + global_pooling=False, + ceil_mode=False, + count_include_pad=False, + divide=True, + format=bigdl_order, + bigdl_type="float" + ) + return blayer + + def create_globalmaxpooling2d(self, klayer, kclayer): + bigdl_order = self.get_bdim_order(kclayer) + input_shape = klayer.get_input_shape_at(0) + if bigdl_order == "NCHW": + b_kw = input_shape[3] + b_kh = input_shape[2] + else: + b_kw = input_shape[2] + b_kh = input_shape[1] + + seq = BLayer.Sequential() + blayer = BLayer.SpatialMaxPooling( + kw=b_kw, + kh=b_kh, + dw=b_kw, + dh=b_kh, + pad_w=0, + pad_h=0, + to_ceil=False, + format=bigdl_order, + bigdl_type="float" + ) + seq.add(blayer) + if bigdl_order == "NCHW": + seq.add(BLayer.Squeeze(3, num_input_dims=3)) + seq.add(BLayer.Squeeze(2, num_input_dims=2)) + else: + seq.add(BLayer.Squeeze(2, num_input_dims=3)) + seq.add(BLayer.Squeeze(1, num_input_dims=2)) + return seq + + def create_globalmaxpooling1d(self, klayer, kclayer): + input_shape = klayer.get_input_shape_at(0) # batch, step, dim + b_kw = 1 + b_kh = input_shape[1] + + seq = BLayer.Sequential() + seq.add(BLayer.View([input_shape[1], 1, input_shape[2]], num_input_dims=2)) + blayer = BLayer.SpatialMaxPooling( + kw=b_kw, + kh=b_kh, + dw=0, + dh=0, + pad_w=0, + pad_h=0, + to_ceil=False, + format="NHWC", + bigdl_type="float" + ) + seq.add(blayer) + seq.add(BLayer.Squeeze(2, num_input_dims=2)) + seq.add(BLayer.Squeeze(1, num_input_dims=1)) + return seq + + def create_globalaveragepooling1d(self, klayer, kclayer): + input_shape = klayer.get_input_shape_at(0) # batch, step, dim + b_kw = 1 + b_kh = input_shape[1] + + seq = BLayer.Sequential() + seq.add(BLayer.View([input_shape[1], 1, input_shape[2]], num_input_dims=2)) + blayer = BLayer.SpatialAveragePooling( + kw=b_kw, + kh=b_kh, + dw=0, + dh=0, + pad_w=0, + pad_h=0, + global_pooling=False, + ceil_mode=False, + count_include_pad=False, + divide=True, + format="NHWC", + bigdl_type="float" + ) + seq.add(blayer) + seq.add(BLayer.Squeeze(2, num_input_dims=2)) # the index start from one but without batch + seq.add(BLayer.Squeeze(1, num_input_dims=1)) + + return seq + + def create_maxpooling1d(self, klayer, kclayer): + input_shape = klayer.get_input_shape_at(0) # batch, steps, dim + bpadW, bpadH = self.to_bigdl_2d_padding(klayer.border_mode) + + seq = BLayer.Sequential() + seq.add(BLayer.View([1, input_shape[1], input_shape[2]], num_input_dims=3)) + blayer = BLayer.SpatialMaxPooling( + kw=klayer.pool_length, + kh=1, + dw=klayer.stride, + dh=1, + pad_w=bpadW, + pad_h=bpadH, + to_ceil=False, + format="NHWC", + bigdl_type="float" + ) + seq.add(blayer) + return seq + + def create_averagepooling1d(self, klayer, kclayer): + input_shape = klayer.get_input_shape_at(0) # batch, steps, dim + bpadW, bpadH = self.to_bigdl_2d_padding(klayer.border_mode) + + seq = BLayer.Sequential() + seq.add(BLayer.View([1, input_shape[1], input_shape[2]], num_input_dims=3)) + blayer = BLayer.SpatialAveragePooling( + kw=klayer.pool_length, + kh=1, + dw=klayer.stride, + dh=1, + pad_w=bpadW, + pad_h=bpadH, + global_pooling=False, + ceil_mode=False, + count_include_pad=False, + divide=True, + format="NHWC", + bigdl_type="float" + ) + seq.add(blayer) + return seq + + def create_globalaveragepooling2d(self, klayer, kclayer): + bigdl_order = self.get_bdim_order(kclayer) + input_shape = klayer.get_input_shape_at(0) + if bigdl_order == "NCHW": + b_kw = input_shape[3] + b_kh = input_shape[2] + else: + b_kw = input_shape[2] + b_kh = input_shape[1] + + seq = BLayer.Sequential() + blayer = BLayer.SpatialAveragePooling( + kw=b_kw, + kh=b_kh, + dw=b_kw, + dh=b_kh, + pad_w=0, + pad_h=0, + global_pooling=False, + ceil_mode=False, + count_include_pad=False, + divide=True, + format=bigdl_order, + bigdl_type="float" + ) + seq.add(blayer) + if bigdl_order == "NCHW": + seq.add(BLayer.Squeeze(3, num_input_dims=3)) + seq.add(BLayer.Squeeze(2, num_input_dims=2)) + else: + seq.add(BLayer.Squeeze(2, num_input_dims=3)) + seq.add(BLayer.Squeeze(1, num_input_dims=2)) + return seq + + def check_constraint_in_config(self, config): + if "W_constraint" in config: + if config["W_constraint"]: + raise Exception("W_constraint is not supported for now") + if "b_constraint" in config: + if config["b_constraint"]: + raise Exception("b_constraint is not supported for now") + + def combo_parameter_layer(self, blayer, config): + self.check_constraint_in_config(config) + + blayer.set_name(config["name"]) + if hasattr(blayer, "set_init_method"): + blayer.set_init_method(self.to_bigdl_init(config["init"]), + BInit.Zeros()) # Keras always set this to be zeros + # "linear" meaning do nothing + if config["activation"] != "linear" : + activation = self.to_bigdl_activation(config["activation"], + "%s_%s" % (config["name"], config["activation"])) + return self.fuse(blayer, activation) + else: + return blayer + + def to_bigdl_activation(self, activation_name, activation_id): + activation = None + if activation_name == "tanh": + activation = BLayer.Tanh() + elif activation_name == "sigmoid": + activation = BLayer.Sigmoid() + elif activation_name == "relu": + activation = BLayer.ReLU() + elif activation_name == "softmax": + activation = BLayer.SoftMax() + else: + raise Exception("Unsupported activation type: %s" % activation_name) + activation.set_name(activation_id) + return activation + + def get_value_from_init(self, kinit_method, shape): + if kinit_method == "zero": + return np.zeros(shape) + elif kinit_method == "one": + return np.ones(shape) + else: + raise Exception("We don't support % for now", kinit_method) + + def to_bigdl_init(self, kinit_method): # kinit_method is a string + init = None + if kinit_method == "glorot_uniform": + init = BInit.Xavier() + elif kinit_method == "one": + init = BInit.Ones() + elif kinit_method == "zero": + init = BInit.Zeros() + else: + raise Exception("Unsupported init type: %s" % init) + return init + + def to_bigdl_reg(self, reg): # reg is a dict + if reg: + raise Exception("will support reg very soon") + else: + return None + + def fuse(self, src_blayer, activation): # activation is a string + seq = BLayer.Sequential() + seq.add(src_blayer) + seq.add(activation) + seq.set_name(src_blayer.name()) + return seq \ No newline at end of file diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 9fade510de9..4e8ba20864a 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -66,6 +66,22 @@ def __init__(self, jvalue, bigdl_type, *args): bigdl_type, JavaValue.jvm_class_constructor(self), *args) self.bigdl_type = bigdl_type + def set_running_mean(self, running_mean): + """ + :param running_mean: a ndarray + """ + callBigDlFunc(self.bigdl_type, "setRunningMean", + self.value, JTensor.from_ndarray(running_mean)) + return self + + def set_running_std(self, running_std): + """ + :param running_mean: a ndarray + """ + callBigDlFunc(self.bigdl_type, "setRunningStd", + self.value, JTensor.from_ndarray(running_std)) + return self + def __str__(self): """ >>> conv2 = SpatialConvolution(6, 12, 5, 5).set_name("conv2") @@ -387,6 +403,10 @@ def get_weights(self): print("The layer does not have weight/bias") return None + def is_with_weights(self): + return callBigDlFunc(self.bigdl_type, + "isWithWeights", self.value) + def save(self, path, over_write = False): callBigDlFunc(self.bigdl_type, "modelSave", self.value, path, over_write) @@ -450,17 +470,20 @@ def unfreeze(self, names=None): callBigDlFunc(self.bigdl_type, "unFreeze", self.value, names) return self - def training(self): + def training(self, is_training=True): ''' - Set this layer in the training mode + Set this layer in the training mode or in predition mode if is_training=False ''' - callJavaFunc(get_spark_context(), self.value.training) + if is_training: + callJavaFunc(get_spark_context(), self.value.training) + else: + callJavaFunc(get_spark_context(), self.value.evaluate) return self def is_training(self): ''' :return: Whether this layer is in the training mode - + >>> layer = Dropout() creating: createDropout >>> layer = layer.evaluate() @@ -479,7 +502,7 @@ def quantize(self): >>> fc = Linear(4, 2) creating: createLinear - >>> fc.set_weights([np.ones((4, 2)), np.ones((2,))]) + >>> fc.set_weights([np.ones((2, 4)), np.ones((2,))]) >>> input = np.ones((2, 4)) >>> fc.forward(input) array([[ 5., 5.], @@ -568,6 +591,12 @@ def add(self, model): self.value.add(model.value) return self + @property + def layers(self): + jlayers = callBigDlFunc(self.bigdl_type, "getContainerModules" , self) + layers = [Layer.of(jlayer) for jlayer in jlayers] + return layers + class Model(Container): """ @@ -599,8 +628,12 @@ class Model(Container): def __init__(self, inputs, outputs, + jvalue=None, bigdl_type="float", byte_order="little_endian", model_type="bigdl"): - if model_type == "bigdl": + if jvalue: + self.value = jvalue + self.bigdl_type = bigdl_type + elif model_type == "bigdl": super(Model, self).__init__(None, bigdl_type, to_list(inputs), to_list(outputs)) @@ -610,6 +643,20 @@ def __init__(self, super(Model, self).__init__(model, bigdl_type) + @staticmethod + def from_jvalue(jvalue, bigdl_type="float"): + """ + Create a Python Model base on the given java value + :param jvalue: Java object create by Py4j + :return: A Python Model + """ + model = Model([], [], jvalue=jvalue) + model.value = jvalue + return model + + def __str__(self): + return "->".join(self.layers()) + @staticmethod def load(path, bigdl_type="float"): """ @@ -643,6 +690,22 @@ def load_torch(path, bigdl_type="float"): jmodel = callBigDlFunc(bigdl_type, "loadTorch", path) return Layer.of(jmodel) + @staticmethod + def load_keras(def_path, weights_path=None, by_name=False): + """ + Load a pre-trained Keras model. + + :param def_path: The json path containing the keras model definition. + :param weights_path: The HDF5 path containing the pre-trained keras model weights. + :return: A pre-trained model. + """ + from bigdl.keras1.converter import DefinitionLoader, WeightLoader + if weights_path: + return WeightLoader.load_weights_from_json_hdf5(def_path, weights_path, by_name=by_name) + else: + return DefinitionLoader.from_json_path(def_path) + return bmodel + @staticmethod def load_caffe(model, defPath, modelPath, match_all=True, bigdl_type="float"): """ @@ -1225,12 +1288,12 @@ def get_hidden_state(self): :return: list of hidden state and cell """ - state = callBigDlFunc(self.bigdl_type, "getHiddenState", self.value) + state = callBigDlFunc(self.bigdl_type, "getHiddenState", self.value) for idx, tensor in enumerate(state): state[idx] = tensor.to_ndarray() return state - + def set_hidden_state(self, states): """ set hidden state and cell at first time step. @@ -1767,6 +1830,7 @@ def __init__(self, JTensor.from_ndarray(init_bias), JTensor.from_ndarray(init_grad_weight), JTensor.from_ndarray(init_grad_bias)) + def set_init_method(self, weight_init_method = None, bias_init_method = None): callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, weight_init_method, bias_init_method) @@ -4315,11 +4379,11 @@ class ResizeBilinear(Layer): """ Resize the input image with bilinear interpolation. The input image must be a float tensor with NHWC layout - + :param output_height: output height :param output_width: output width :param align_corner: align corner or not - + >>> resizeBilinear = ResizeBilinear(10, 20, False) creating: createResizeBilinear """ diff --git a/python/dllib/src/bigdl/dllib/utils/common.py b/python/dllib/src/bigdl/dllib/utils/common.py index 5fa51973447..428a109215c 100644 --- a/python/dllib/src/bigdl/dllib/utils/common.py +++ b/python/dllib/src/bigdl/dllib/utils/common.py @@ -28,7 +28,8 @@ from pyspark import SparkConf import numpy as np import threading -from bigdl.util.engine import get_bigdl_classpath, is_spark_below_2_2 +import tempfile +from bigdl.util.engine import prepare_env, get_bigdl_classpath, is_spark_below_2_2 INTMAX = 2147483647 INTMIN = -2147483648 @@ -439,7 +440,7 @@ def get_spark_context(conf = None): return SparkContext.getOrCreate(conf=conf or create_spark_conf()) else: # Might have threading issue but we cann't add _lock here - # as it's not RLock in spark1.5 + # as it's not RLock in spark1.5; if SparkContext._active_spark_context is None: SparkContext(conf=conf or create_spark_conf()) return SparkContext._active_spark_context @@ -458,7 +459,6 @@ def callBigDlFunc(bigdl_type, name, *args): api = getattr(jinstance, name) return callJavaFunc(sc, api, *args) - def _java2py(sc, r, encoding="bytes"): if isinstance(r, JavaObject): clsName = r.getClass().getSimpleName() @@ -536,6 +536,11 @@ def _py2java(sc, obj): obj = sc._jvm.org.apache.spark.bigdl.api.python.BigDLSerDe.loads(data) return obj +def create_tmp_path(): + tmp_file = tempfile.NamedTemporaryFile(prefix="bigdl") + tmp_file.close() + return tmp_file.name + def _test(): import doctest From 10bfd54ed424aa3355f7b814e48da59e4129377e Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Wed, 15 Nov 2017 09:15:24 -0600 Subject: [PATCH 334/823] Support keras model loading (#1827) * keras load model and api wrapper * fix batchnorm * fix averagepooling * remove function wrapper and clean code * add unit test * remove backend * fix and clean * update * separate unittest * x * remove -n * ignore application * separate test * pass parameter * try to fix test * disable shape checking * ignore --- keras1/__init__.py | 15 + keras1/converter.py | 1111 ++++++++++++++++++++++ python/dllib/src/bigdl/dllib/nn/layer.py | 84 +- python/dllib/src/bigdl/utils/common.py | 11 +- 4 files changed, 1208 insertions(+), 13 deletions(-) create mode 100644 keras1/__init__.py create mode 100644 keras1/converter.py diff --git a/keras1/__init__.py b/keras1/__init__.py new file mode 100644 index 00000000000..2151a805423 --- /dev/null +++ b/keras1/__init__.py @@ -0,0 +1,15 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/keras1/converter.py b/keras1/converter.py new file mode 100644 index 00000000000..90f65991249 --- /dev/null +++ b/keras1/converter.py @@ -0,0 +1,1111 @@ + +import bigdl.nn.initialization_method as BInit +import numpy as np +import bigdl.nn.layer as BLayer +import bigdl.optim.optimizer as boptimizer +import bigdl.nn.criterion as bcriterion +import bigdl.util.common as bcommon +import keras.optimizers as koptimizers +from keras.models import model_from_json +from keras.models import Sequential, Model +import keras +import warnings + + +def unsupport_exp(name): + raise Exception("We don't support %s for now" % name) + + +class WeightLoader: + + @staticmethod + def __load_weights_by_execution_seq(bmodel, kmodel): + blayers = [l for l in bmodel.layers if l.is_with_weights()] + klayers = [l for l in kmodel.layers if l.get_weights()] + if len(blayers) != len(klayers): + raise Exception( + "keras with %s layers but bigdl with %s layers" % (len(klayers), len(blayers))) + for b, k in zip(blayers, klayers): + if b.name() != k.name: + raise Exception("Found different layer in execution order, bigdl:%s, keras: %s" % (b.name(), k.name)) # noqa + bigdl_weights = WeightsConverter.get_bigdl_weigths_from_keras(k) + b.set_weights(bigdl_weights) + + # TODO: add more unitest + @staticmethod + def __load_weights_by_name(bmodel, kmodel, by_name=False): + keras_name_to_layer = WeightLoader.__keras_name_to_Layers(kmodel, with_weights=True) + bigdl_name_to_layer = WeightLoader.__bigdl_name_to_Layers(bmodel, with_weights=True) + layers_not_in_keras = set(bigdl_name_to_layer.keys()) - set(keras_name_to_layer.keys()) + if layers_not_in_keras: + raise Exception("Layers %s can be found in bigdl, but not in keras" % repr(layers_not_in_keras)) # noqa + layers_not_in_bigdl = set(keras_name_to_layer.keys()) - set(bigdl_name_to_layer.keys()) + if layers_not_in_bigdl: + if by_name: + warnings.warn("Ignore weight of layers %s as it cannot be found in bigdl" % repr(layers_not_in_bigdl)) # noqa + else: + raise Exception("Layers %s can be found in bigdl, but not in keras" % repr(layers_not_in_keras)) # noqa + for blayer in bigdl_name_to_layer.values(): + if blayer.name() in keras_name_to_layer: + klayer = keras_name_to_layer[blayer.name()] + bigdl_weights = WeightsConverter.get_bigdl_weigths_from_keras(klayer) + blayer.set_weights(bigdl_weights) + if isinstance(klayer, keras.layers.BatchNormalization): + blayer.set_running_mean(keras.backend.eval(klayer.running_mean)) + blayer.set_running_std(keras.backend.eval(klayer.running_std)) + + @staticmethod + def load_weights_from_kmodel(bmodel, kmodel, by_name=False): + """ + Load weights from kmodel to bmodel + """ + if by_name: + WeightLoader.__load_weights_by_name(bmodel, kmodel) + else: + WeightLoader.__load_weights_by_execution_seq(bmodel, kmodel) + + @staticmethod + def load_weights_from_json_hdf5(def_json, weights_hdf5, by_name=False): + with open(def_json, "r") as jp: + kmodel = model_from_json(jp.read()) + bmodel = DefinitionLoader.from_json_path(def_json) + WeightLoader.load_weights_from_hdf5(bmodel, kmodel, weights_hdf5, by_name) + return bmodel + + @staticmethod + def load_weights_from_hdf5(bmodel, kmodel, filepath, by_name=False): + '''Loads all layer weights from a HDF5 save file. + + If `by_name` is False (default) weights are loaded + based on the network's execution order topology, + meaning layers in the execution seq should be exactly the same + the architecture + + If `by_name` is True, weights are loaded into layers + only if they share the same name. This is useful + for fine-tuning or transfer-learning models where + some of the layers have changed. + ''' + kmodel.load_weights(filepath=filepath, by_name=by_name) + WeightLoader.load_weights_from_kmodel(bmodel, kmodel, by_name=by_name) + + @staticmethod + def __keras_name_to_Layers(model, with_weights=False): + if with_weights: + layers = [l for l in model.layers if l.get_weights()] + else: + layers = [l for l in model.layers] + + return dict([(layer.name, layer) for layer in layers]) + + @staticmethod + def __bigdl_name_to_Layers(model, with_weights=False): + if with_weights: + layers = [l for l in model.layers if l.is_with_weights()] + else: + layers = [l for l in model.layers] + + return dict([(layer.name(), layer) for layer in layers]) + + +class WeightsConverter: + """ + Convert keras weights to bigdl weights + The shape of weights would be changed if using different backend, + so we only test against TensorFlow backend. + TODO: Support th backend as well. + """ + + @staticmethod + def get_converter(class_name): + function_name = "convert_" + class_name.lower() + if not hasattr(WeightsConverter, function_name): + raise unsupport_exp(class_name) + converter = getattr(WeightsConverter, function_name) + return converter + + @staticmethod + # weights is a list of ndarray or a ndarray + # convert keras weights per layer to bigdl format + def to_bigdl_weights(class_name, weights): + return WeightsConverter.get_converter(class_name)(weights) + + @staticmethod + def get_bigdl_weigths_from_keras(k): + if isinstance(k, keras.engine.Model): + return WeightsConverter.get_weights_from_kmodel(k) + elif isinstance(k, keras.engine.Layer): + return WeightsConverter.get_bigdl_weights_from_klayer(k) + else: + raise Exception("Unsupport type: %s", k) + + @staticmethod + def get_bigdl_weights_from_klayer(klayer): + # we should use get_weights instead of klayer.weights + return WeightsConverter.to_bigdl_weights(klayer.__class__.__name__, klayer.get_weights()) + + @staticmethod + def get_weights_from_kmodel(kmodel): + """ + Convert kmodel's weights to bigdl format. + We are supposing the order is the same as the execution order. + :param kmodel: keras model + :return: list of ndarray + """ + layers_with_weights = [layer for layer in kmodel.layers if layer.weights] + bweights = [] + for klayer in layers_with_weights: + # bws would be [weiths, bias] or [weights] + bws = WeightsConverter.get_bigdl_weights_from_klayer(klayer) + for w in bws: + bweights.append(w) + return bweights + + @staticmethod + def convert_dense(weights): + return [np.transpose(weights[0]), weights[1]] + + @staticmethod + def convert_timedistributeddense(weights): + return [np.transpose(weights[0]), weights[1]] + + @staticmethod + def convert_batchnormalization(weights): + gamma = weights[0] + beta = weights[1] + return [gamma, beta] + + @staticmethod + def convert_convolution2d(weights): + weight = np.expand_dims(weights[0], 0) # bigdl has a leading dim with value 1 + if len(weights) > 1: + return [weight, weights[1]] + else: + return [weight] + + @staticmethod + def convert_convolution1d(weights): + return WeightsConverter.convert_convolution2d(weights) + + @staticmethod + def convert_embedding(weights): + return weights + + @staticmethod + def convert_simplernn(weights): + return [np.transpose(weights[0]), np.transpose(weights[1]), weights[2]] + + @staticmethod + def convert_lstm(weights): + w1 = np.concatenate((weights[0].T, weights[3].T, weights[6].T, weights[9].T)) + w2 = np.concatenate((weights[2], weights[5], weights[8], weights[11])) + w3 = np.concatenate((weights[1].T, weights[4].T, weights[7].T, weights[10].T)) + return [w1, w2, w3] + + @staticmethod + def convert_gru(weights): + w1 = np.concatenate((weights[3].T, weights[0].T, weights[6].T)) + w2 = np.concatenate((weights[2], weights[5], weights[8])) + w3 = np.concatenate((weights[4].T, weights[1].T)) + w4 = weights[7].T + return [w1, w2, w3, w4] + + +class DefinitionLoader: + + def __init__(self, kmodel): + self.node_id_to_instance = {} + self.node_id_to_layer = {} + self.node_id_to_config_layer = {} + self.kmodel = kmodel + self.kconfig = self.kmodel.get_config() + + for layer in self.kmodel.layers: + self.node_id_to_layer[layer.name] = layer + + if isinstance(self.kmodel, Sequential): + for layer_config in self.kmodel.get_config(): + layer_name = layer_config["config"]["name"] + self.node_id_to_config_layer[layer_name] = layer_config + else: + for layerConfig in self.kconfig["layers"]: + self.node_id_to_config_layer[layerConfig["name"]] = layerConfig + + def __to_bigdl(self): + if isinstance(self.kmodel, Sequential): + bmodel = self._construct_bigdl_sequence() + elif isinstance(self.kmodel, Model): + bmodel = self._construct_bigdl_model() + return bmodel + + @classmethod + def from_kmodel(cls, kmodel): + return cls(kmodel).__to_bigdl() + + @classmethod + def from_json_path(cls, json_path): + with open(json_path, "r") as jp: + return DefinitionLoader.from_json_str(jp.read()) + + @classmethod + def from_json_str(cls, json_str): + kmodel = model_from_json(json_str) + return DefinitionLoader.from_kmodel(kmodel) + + def _do_create_node(self, layer, clayer): + if clayer["class_name"] == "InputLayer": + input = BLayer.Input() + input.element().set_name(layer.name) # cannot set name for node? + self.node_id_to_instance[layer.name] = input + return input + bigdl_in_nodes = [] + for node in clayer["inbound_nodes"]: + for out in node: + out_name = out[0] + out_index = out[1] + out_tensor_index = out[2] + if out_name not in self.node_id_to_instance: + self._do_create_node(self.node_id_to_layer[out_name], + self.node_id_to_config_layer[out_name]) + bigdl_in_nodes.append(self.node_id_to_instance[out_name]) + + blayer = LayerConverter().create(layer, clayer) + new_bnode = blayer(bigdl_in_nodes) + self.node_id_to_instance[layer.name] = new_bnode + return new_bnode + + def _construct_bigdl_model(self): + for clayer in self.kconfig["layers"]: + if clayer["name"] not in self.node_id_to_instance: + + self._do_create_node(self.node_id_to_layer[clayer["name"]], + clayer) + ins = [] + for input_layer in self.kconfig["input_layers"]: + name = input_layer[0] + ins.append(self.node_id_to_instance[name]) + outs = [] + for output_layer in self.kconfig["output_layers"]: + name = output_layer[0] + outs.append(self.node_id_to_instance[name]) + return BLayer.Model(inputs=ins, outputs=outs) + + def _construct_bigdl_sequence(self): + bseq = BLayer.Sequential() + layerConverter = LayerConverter() + for layer in self.kmodel.layers: + blayer = layerConverter.create(layer, self.node_id_to_config_layer[layer.name]) + bseq.add(blayer) + return bseq + +class LayerConverter: + + def __check_is_share_weights(self, kclayer): + # For Merge layer len(kclayer["inbound_nodes"]) is equal to 1 + # "inbound_nodes": [ + # [ + # [ + # "batchnormalization_194", + # 0, + # 0 + # ], + # [ + # "batchnormalization_196", + # 0, + # 0 + # ], + # [ + # "batchnormalization_199", + # 0, + # 0 + # ], + # [ + # "batchnormalization_200", + # 0, + # 0 + # ] + # ] + # ], + if "inbound_nodes" in kclayer and len(kclayer["inbound_nodes"]) > 1: + raise Exception( + "%s doesn't support multiple inputs with shared weights" % kclayer["class_name"]) + + def create(self, klayer, kclayer): + class_name = kclayer["class_name"] + + self.__check_is_share_weights(kclayer) + + if (hasattr(klayer, "b_constraint") and klayer.b_constraint) or \ + (hasattr(klayer, "W_constraint") and klayer.W_constraint): + raise Exception("We don't support constraint for now") + + if (hasattr(klayer, "activity_regularizer") and klayer.activity_regularizer): + raise Exception("We don't support activity_regularizer for now") + + function_name = "create_" + class_name.lower() + if not hasattr(self, function_name): + raise Exception("We don't support layer: %s for now" % class_name ) + + api = getattr(self, function_name) + blayer = api(klayer, kclayer) + return blayer.set_name(klayer.name) + + def create_model(self, klayer, kclyer): + return DefinitionLoader.from_kmodel(klayer) + + def create_inputlayer(self, klayer, kclyer): + return BLayer.Identity() + + def create_dense(self, klayer, kclayer): + config = kclayer["config"] + # Multiple inputs should share the same input_dim for Dense layer + # We don't need to respect the tensor index for method `get_input_shape_at` + # which is internal implementation and `get_input_shape_at` has hided that for us, + # What we need to use is the input index, not node index, not tensor index. + input_shape = klayer.get_input_shape_at(0) + blayer = BLayer.Linear( + input_size=input_shape[1], + output_size=config["output_dim"], + with_bias=config["bias"], + wRegularizer=self.to_bigdl_reg(config["W_regularizer"]), + bRegularizer=self.to_bigdl_reg(config["b_regularizer"]) + ) + return self.combo_parameter_layer(blayer, config) + + def create_timedistributeddense(self, klayer, kclayer): + config = kclayer["config"] + input_shape = klayer.get_input_shape_at(0) + print(input_shape) + print(config["output_dim"]) + blayer = BLayer.TimeDistributed(BLayer.Linear( + input_size=input_shape[2], + output_size=config["output_dim"], + with_bias=config["bias"], + wRegularizer=self.to_bigdl_reg(config["W_regularizer"]), + bRegularizer=self.to_bigdl_reg(config["b_regularizer"]) + )) + return self.combo_parameter_layer(blayer, config) + + def create_embedding(self, klayer, kclayer): + config = kclayer["config"] + input_shape = klayer.get_input_shape_at(0) # batch, seq_len + seq_len = input_shape[1] + if klayer.input_length and klayer.input_length != seq_len: + raise Exception( + "The input_length doesn't match: %s vs %s" % (seq_len, klayer.input_length)) + + if (hasattr(klayer, "dropout") and klayer.dropout != 0): + raise Exception("We don't support dropout for now") + + if (hasattr(klayer, "mask_zero") and klayer.mask_zero != False): + raise Exception("We don't support mask_zero for now") + + bseq = BLayer.Sequential() + blayer = BLayer.LookupTable( + n_index = klayer.input_dim, + n_output = klayer.output_dim, + padding_value=0.0, + norm_type=2.0, + should_scale_grad_by_freq=False, + wRegularizer= self.to_bigdl_reg(config["W_regularizer"]), + bigdl_type="float") + bseq.add(BLayer.AddConstant(1.0, inplace=True)) # Add 1 as BigDL is one-based index + bseq.add(blayer) + return bseq + + def create_activation(self, klayer, kclayer): + config = kclayer["config"] + return self.to_bigdl_activation(config["activation"], klayer.name) + + def create_dropout(self, klayer, kclayer): + return BLayer.Dropout(klayer.p) + + def create_flatten(self, klayer, kclayer): + self.__check_is_share_weights(kclayer) + input_shape = klayer.input_shape + blayer = BLayer.Reshape([np.prod(input_shape[1:])], None) + return blayer + + def create_reshape(self, klayer, kclayer): + self.__check_is_share_weights(kclayer) + blayer = BLayer.Reshape(klayer.target_shape, None) + return blayer + + def create_repeatvector(self, klayer, kclayer): + return BLayer.Replicate(n_features=klayer.n, + n_dim=1, + bigdl_type="float") + + def create_merge(self, klayer, kclayer): + self.__check_is_share_weights(kclayer) + input_shape = klayer.get_input_shape_at(0) + #TODO: dot_axes, node_indices, tensor_indices not supported. + if klayer.output_shape and not isinstance(klayer.output_shape, tuple): + raise Exception("Only output_shape=None or a shape tuple is supported for now.") + if klayer.output_mask: + raise Exception("Argument `output_mask` is not supported for now.") + if klayer.mode == "concat": + blayer = BLayer.JoinTable( + dimension=klayer.concat_axis, + n_input_dims=len(input_shape[0]) - 1, + bigdl_type="float") + elif klayer.mode == "sum": + blayer = BLayer.CAddTable( + inplace=False, + bigdl_type="float") + elif klayer.mode == "mul": + blayer = BLayer.CMulTable(bigdl_type="float") + elif klayer.mode == "max": + blayer = BLayer.CMaxTable(bigdl_type="float") + elif klayer.mode == "dot": + if len(input_shape[0]) >= 3: + raise Exception("Merge mode `dot` doesn't support 3D input or above for now.") + if input_shape[0][0] == None: + raise Exception("For merge mode `dot`, please specify `batch_input_shape`.") + model = BLayer.Sequential() + blayer = model.add(BLayer.DotProduct(bigdl_type="float"))\ + .add(BLayer.Reshape([input_shape[0][0], 1])) + elif klayer.mode in ['ave', 'cos']: + raise Exception("Merge mode `%s` not supported for now" % klayer.mode) # TODO: whether to support cosine and average + else: # invalid mode or lambda functions + raise Exception("Invalid merge mode: `%s`. Lambda/function as merge mode is not supported for now." % klayer.mode) + return blayer + + def create_elu(self, klayer, kclayer): + return BLayer.ELU(alpha=float(klayer.alpha), + inplace=False, + bigdl_type="float") + + def create_prelu(self, klayer, kclayer): + return BLayer.PReLU(n_output_plane=0, + bigdl_type="float") + + def create_leakyrelu(self, klayer, kclayer): + return BLayer.LeakyReLU(negval=float(klayer.alpha), + inplace=False, + bigdl_type="float") + + def create_parametricsoftplus(self, klayer, kclayer): + alpha = float(klayer.alpha_init) + beta = float(klayer.beta_init) + if round(alpha * beta, 4) == 1.0: + return BLayer.SoftPlus(beta=beta, + bigdl_type="float") + else: + raise Exception("Only alpha_init = 1/beta_init is supported for now") + + def create_thresholdedrelu(self, klayer, kclayer): + return BLayer.Threshold(th=float(klayer.theta), + v=0.0, + ip=False, + bigdl_type="float") + + def __generate_zeropadding1d(self, pad_top, pad_bottom): + return BLayer.SpatialZeroPadding(pad_left=0, + pad_right=0, + pad_top=pad_top, + pad_bottom=pad_bottom, + bigdl_type="float") + + def create_zeropadding1d(self, klayer, kclayer): + padding = klayer.padding + if isinstance(padding, int): + return self.__generate_zeropadding1d(padding, padding) + elif isinstance(padding, dict): + return self.__generate_zeropadding1d(padding.get('left_pad', 0), padding.get('right_pad', 0)) + else: # tuple of int (length 2) + padding = tuple(padding) + return self.__generate_zeropadding1d(padding[0], padding[1]) + + def __generate_zeropadding2d(self, dim1, dim2, n_input_dim, pad1, pad2, pad3, pad4): + model = BLayer.Sequential() + paddinglayer1 = BLayer.Padding(dim=dim1, + pad=pad1, + n_input_dim=n_input_dim, + value=0.0, + n_index=1, + bigdl_type="float") + paddinglayer2 = BLayer.Padding(dim=dim1, + pad=pad2, + n_input_dim=n_input_dim, + value=0.0, + n_index=1, + bigdl_type="float") + paddinglayer3 = BLayer.Padding(dim=dim2, + pad=pad3, + n_input_dim=n_input_dim, + value=0.0, + n_index=1, + bigdl_type="float") + paddinglayer4 = BLayer.Padding(dim=dim2, + pad=pad4, + n_input_dim=n_input_dim, + value=0.0, + n_index=1, + bigdl_type="float") + model.add(paddinglayer1) + model.add(paddinglayer2) + model.add(paddinglayer3) + model.add(paddinglayer4) + return model + + # NB: zeropadding doesn't serialize dim_ording to jason file + def create_zeropadding2d(self, klayer, kclayer): + padding = klayer.padding + input_shape = klayer.get_input_shape_at(0) + dim1 = 1 + dim2 = 2 + if klayer.dim_ordering == "th": + dim1 = 2 + dim2 = 3 + if isinstance(padding, dict): # dictionary + return self.__generate_zeropadding2d(dim1, dim2, len(input_shape) - 1, + -padding.get('top_pad', 0), padding.get('bottom_pad', 0), + -padding.get('left_pad', 0), padding.get('right_pad', 0)) + else: # tuple of int + padding = tuple(padding) + if len(padding) == 2: + return self.__generate_zeropadding2d(dim1, dim2, len(input_shape) - 1, + -padding[0], padding[0], -padding[1], padding[1]) + elif len(padding) == 4: + return self.__generate_zeropadding2d(dim1, dim2, len(input_shape) - 1, + -padding[0], padding[1], -padding[2], padding[3]) + + # NB: zeropadding doesn't serialize dim_ording to jason file + def create_zeropadding3d(self, klayer, kclayer): + padding = tuple(klayer.padding) + input_shape = klayer.get_input_shape_at(0) + model = BLayer.Sequential() + paddinglayer1 = BLayer.Padding(dim=2, + pad=-padding[0], + n_input_dim=len(input_shape) - 1, + value=0.0, + n_index=1, + bigdl_type="float") + paddinglayer2 = BLayer.Padding(dim=2, + pad=padding[0], + n_input_dim=len(input_shape) - 1, + value=0.0, + n_index=1, + bigdl_type="float") + paddinglayer3 = BLayer.Padding(dim=3, + pad=-padding[1], + n_input_dim=len(input_shape) - 1, + value=0.0, + n_index=1, + bigdl_type="float") + paddinglayer4 = BLayer.Padding(dim=3, + pad=padding[1], + n_input_dim=len(input_shape) - 1, + value=0.0, + n_index=1, + bigdl_type="float") + paddinglayer5 = BLayer.Padding(dim=4, + pad=-padding[2], + n_input_dim=len(input_shape) - 1, + value=0.0, + n_index=1, + bigdl_type="float") + paddinglayer6 = BLayer.Padding(dim=4, + pad=padding[2], + n_input_dim=len(input_shape) - 1, + value=0.0, + n_index=1, + bigdl_type="float") + model.add(paddinglayer1) + model.add(paddinglayer2) + model.add(paddinglayer3) + model.add(paddinglayer4) + model.add(paddinglayer5) + model.add(paddinglayer6) + return model + + def create_cropping1d(self, klayer, kclayer): + cropping = tuple(klayer.cropping) + return BLayer.SpatialZeroPadding(0, 0, -cropping[0], -cropping[1]) + + def __return_sequences(self, return_sequences, blayer): + # For recurrent layers, handle whether to return the last output sentence or the full sequence. + if return_sequences: + return blayer + else: + model = BLayer.Sequential() + model.add(blayer) + model.add(BLayer.Select(2, -1)) + return model + + def create_simplernn(self, klayer, kclayer): + rec = BLayer.Recurrent() + input_shape = klayer.get_input_shape_at(0) + config = kclayer["config"] + self.check_constraint_in_config(config) + activation = self.to_bigdl_activation(config["activation"], + "%s_%s" % (config["name"], config["activation"])) + # TODO: throw exception for dropout + rnn = BLayer.RnnCell(input_size=input_shape[2], + hidden_size=klayer.output_dim, + activation=activation, + isInputWithBias=False, + wRegularizer=self.to_bigdl_reg(config["W_regularizer"]), + uRegularizer=self.to_bigdl_reg(config["U_regularizer"]), + bRegularizer=self.to_bigdl_reg(config["b_regularizer"]), + bigdl_type="float") + return self.__return_sequences(klayer.return_sequences, rec.add(rnn)) + + def create_lstm(self, klayer, kclayer): + rec = BLayer.Recurrent() + input_shape = klayer.get_input_shape_at(0) + config = kclayer["config"] + self.check_constraint_in_config(config) + activation = self.to_bigdl_activation(config["activation"], + "%s_%s" % (config["name"], config["activation"])) + if not isinstance(activation, BLayer.Tanh): + raise Exception("For activation, only `tanh` is supported for now.") + inner_activation = self.to_bigdl_activation(config["inner_activation"], + "%s_%s" % (config["name"], config["inner_activation"])) + if not isinstance(inner_activation, BLayer.Sigmoid): + raise Exception("For inner_activation, only `sigmond` is supported for now.") + # TODO: throw exception for dropout + lstm = BLayer.LSTM(input_size=input_shape[2], + hidden_size=klayer.output_dim, + p=klayer.dropout_W, + wRegularizer=self.to_bigdl_reg(config["W_regularizer"]), + uRegularizer=self.to_bigdl_reg(config["U_regularizer"]), + bRegularizer=self.to_bigdl_reg(config["b_regularizer"]), + bigdl_type="float") + return self.__return_sequences(klayer.return_sequences, rec.add(lstm)) + + def create_gru(self, klayer, kclayer): + rec = BLayer.Recurrent() + input_shape = klayer.get_input_shape_at(0) + config = kclayer["config"] + self.check_constraint_in_config(config) + activation = self.to_bigdl_activation(config["activation"], + "%s_%s" % (config["name"], config["activation"])) + if not isinstance(activation, BLayer.Tanh): + raise Exception("For activation, only `tanh` is supported for now.") + inner_activation = self.to_bigdl_activation(config["inner_activation"], + "%s_%s" % (config["name"], config["inner_activation"])) + if not isinstance(inner_activation, BLayer.Sigmoid): + raise Exception("For inner_activation, only `sigmond` is supported for now.") + # TODO: throw exception for dropout + gru = BLayer.GRU(input_size=input_shape[2], + hidden_size=klayer.output_dim, + p=klayer.dropout_W, + wRegularizer=self.to_bigdl_reg(config["W_regularizer"]), + uRegularizer=self.to_bigdl_reg(config["U_regularizer"]), + bRegularizer=self.to_bigdl_reg(config["b_regularizer"]), + bigdl_type="float") + return self.__return_sequences(klayer.return_sequences, rec.add(gru)) + + def create_batchnormalization(self, klayer, kclayer): + config = kclayer["config"] + + self.__check_is_share_weights(kclayer) + if keras.backend.image_dim_ordering() != "th" or klayer.axis != 1: + raise Exception("""We only support th image order for BatchNormalization \n + which meaning NCHW for now. but the current order is %s and axis is %s + """ % (keras.backend.image_dim_ordering(), klayer.axis)) # noqa + if klayer.mode != 0: + raise Exception( + "Only support mode = 0 for now, but the current mode is: %s", klayer.mode) + + if config["gamma_regularizer"]: + raise Exception("We don't support gamma_regularizer for now") + + if config["beta_regularizer"]: + raise Exception("We don't support beta_regularizer for now") + + input_shape = klayer.get_input_shape_at(0) + n_input_channel = input_shape[klayer.axis] # default is -1 which is channel-last + + # init gamma and beta + # TODO: replace this with to_bigdl_init in the future + gamma = self.get_value_from_init(klayer.gamma_init.func_name, (n_input_channel,)) + beta = self.get_value_from_init(klayer.beta_init.func_name, (n_input_channel,)) + + blayer = BLayer.SpatialBatchNormalization( + n_output=n_input_channel, + eps=klayer.epsilon, + momentum=klayer.momentum, + affine=True, + init_weight=gamma, + init_bias=beta, + init_grad_weight=None, + init_grad_bias=None, + bigdl_type="float") + + k_running_mean = keras.backend.eval(klayer.running_mean) + k_running_std = keras.backend.eval(klayer.running_std) + blayer.set_running_mean(k_running_mean) + blayer.set_running_std(k_running_std) + return blayer + + def get_bdim_order(self, kclayer): + return self.to_bigdl_2d_ordering(self.get_kdim_order(kclayer)) + + def get_kdim_order(self, kclayer): + config = kclayer["config"] + if "dim_ordering" in config: + return config["dim_ordering"] + else: + warnings.warn("Cannot find dim_ordering from json definition. Use default instead.") + return keras.backend.image_dim_ordering() + + def to_bigdl_2d_ordering(self, order): + if order == "tf": + return "NHWC" + elif order == "th": + return "NCHW" + else: + raise Exception("Unsupport ordering: %s" % order) + + def to_bigdl_2d_padding(self, border_mode): + if border_mode == "same": + return (-1, -1) + elif border_mode == "valid": + return (0, 0) + else: + raise Exception("Unsupported border mode: %s" % border_mode) + + def to_bigdl_1d_padding(self, border_mode, kernel_w): + if border_mode == "same": + raise Exception("We don't support padding for now") + # TODO: support padding + # return int((kernel_w -1) / 2) + elif border_mode == "valid": + return 0 + else: + raise Exception("Unsupported border mode: %s" % border_mode) + +################# Layers with weights ############################# noqa + + def create_convolution1d(self, klayer, kclayer): + config = kclayer["config"] + input_shape = klayer.get_input_shape_at(0) + # batch, steps, dim, batch is None here, so you cannot use it directly. + stack_size = input_shape[2] + + bpadW, bpadH = self.to_bigdl_2d_padding(klayer.border_mode) + seq = BLayer.Sequential() + seq.add(BLayer.Reshape([input_shape[1], 1, input_shape[2]], True)) + blayer = BLayer.SpatialConvolution( + n_input_plane = stack_size, + n_output_plane = klayer.nb_filter, + kernel_w = 1, + kernel_h = klayer.filter_length, + stride_w= 1, + stride_h= klayer.subsample_length, + pad_w= bpadW, + pad_h= bpadH, + n_group=1, + propagate_back=True, + wRegularizer = self.to_bigdl_reg(config["W_regularizer"]), + bRegularizer=self.to_bigdl_reg(config["b_regularizer"]), + init_weight=None, + init_bias=None, + init_grad_weight=None, + init_grad_bias=None, + with_bias=config["bias"], + data_format="NHWC", + bigdl_type="float") + seq.add(blayer) + seq.add(BLayer.Squeeze(3)) + return self.combo_parameter_layer(seq, config) + + def create_convolution2d(self, klayer, kclayer): + config = kclayer["config"] + bigdl_order = self.get_bdim_order(kclayer) + input_shape = klayer.get_input_shape_at(0) + + if bigdl_order == "NCHW": + stack_size = input_shape[1] + elif bigdl_order == "NHWC": + stack_size = input_shape[3] + + bpadW, bpadH = self.to_bigdl_2d_padding(klayer.border_mode) + blayer = BLayer.SpatialConvolution( + n_input_plane = stack_size, + n_output_plane = klayer.nb_filter, + kernel_w = klayer.nb_col, + kernel_h = klayer.nb_row, + stride_w= klayer.subsample[0], + stride_h= klayer.subsample[1], + pad_w= bpadW, + pad_h= bpadH, + n_group=1, + propagate_back=True, + wRegularizer = self.to_bigdl_reg(config["W_regularizer"]), + bRegularizer=self.to_bigdl_reg(config["b_regularizer"]), + init_weight=None, + init_bias=None, + init_grad_weight=None, + init_grad_bias=None, + with_bias=config["bias"], + data_format=bigdl_order, + bigdl_type="float") + + return self.combo_parameter_layer(blayer, config) + + def create_maxpooling2d(self, klayer, kclayer): + bigdl_order = self.get_bdim_order(kclayer) + bpadW, bpadH = self.to_bigdl_2d_padding(klayer.border_mode) + blayer = BLayer.SpatialMaxPooling( + kw = klayer.pool_size[0], + kh = klayer.pool_size[1], + dw = klayer.strides[0], + dh = klayer.strides[1], + pad_w=bpadW, + pad_h=bpadH, + to_ceil=False, + format=bigdl_order, + bigdl_type="float") + return blayer + + def create_averagepooling2d(self, klayer, kclayer): + bigdl_order = self.get_bdim_order(kclayer) + bpadW, bpadH = self.to_bigdl_2d_padding(klayer.border_mode) + blayer = BLayer.SpatialAveragePooling( + kw=klayer.pool_size[0], + kh=klayer.pool_size[1], + dw=klayer.strides[0], + dh=klayer.strides[1], + pad_w=bpadW, + pad_h=bpadH, + global_pooling=False, + ceil_mode=False, + count_include_pad=False, + divide=True, + format=bigdl_order, + bigdl_type="float" + ) + return blayer + + def create_globalmaxpooling2d(self, klayer, kclayer): + bigdl_order = self.get_bdim_order(kclayer) + input_shape = klayer.get_input_shape_at(0) + if bigdl_order == "NCHW": + b_kw = input_shape[3] + b_kh = input_shape[2] + else: + b_kw = input_shape[2] + b_kh = input_shape[1] + + seq = BLayer.Sequential() + blayer = BLayer.SpatialMaxPooling( + kw=b_kw, + kh=b_kh, + dw=b_kw, + dh=b_kh, + pad_w=0, + pad_h=0, + to_ceil=False, + format=bigdl_order, + bigdl_type="float" + ) + seq.add(blayer) + if bigdl_order == "NCHW": + seq.add(BLayer.Squeeze(3, num_input_dims=3)) + seq.add(BLayer.Squeeze(2, num_input_dims=2)) + else: + seq.add(BLayer.Squeeze(2, num_input_dims=3)) + seq.add(BLayer.Squeeze(1, num_input_dims=2)) + return seq + + def create_globalmaxpooling1d(self, klayer, kclayer): + input_shape = klayer.get_input_shape_at(0) # batch, step, dim + b_kw = 1 + b_kh = input_shape[1] + + seq = BLayer.Sequential() + seq.add(BLayer.View([input_shape[1], 1, input_shape[2]], num_input_dims=2)) + blayer = BLayer.SpatialMaxPooling( + kw=b_kw, + kh=b_kh, + dw=0, + dh=0, + pad_w=0, + pad_h=0, + to_ceil=False, + format="NHWC", + bigdl_type="float" + ) + seq.add(blayer) + seq.add(BLayer.Squeeze(2, num_input_dims=2)) + seq.add(BLayer.Squeeze(1, num_input_dims=1)) + return seq + + def create_globalaveragepooling1d(self, klayer, kclayer): + input_shape = klayer.get_input_shape_at(0) # batch, step, dim + b_kw = 1 + b_kh = input_shape[1] + + seq = BLayer.Sequential() + seq.add(BLayer.View([input_shape[1], 1, input_shape[2]], num_input_dims=2)) + blayer = BLayer.SpatialAveragePooling( + kw=b_kw, + kh=b_kh, + dw=0, + dh=0, + pad_w=0, + pad_h=0, + global_pooling=False, + ceil_mode=False, + count_include_pad=False, + divide=True, + format="NHWC", + bigdl_type="float" + ) + seq.add(blayer) + seq.add(BLayer.Squeeze(2, num_input_dims=2)) # the index start from one but without batch + seq.add(BLayer.Squeeze(1, num_input_dims=1)) + + return seq + + def create_maxpooling1d(self, klayer, kclayer): + input_shape = klayer.get_input_shape_at(0) # batch, steps, dim + bpadW, bpadH = self.to_bigdl_2d_padding(klayer.border_mode) + + seq = BLayer.Sequential() + seq.add(BLayer.View([1, input_shape[1], input_shape[2]], num_input_dims=3)) + blayer = BLayer.SpatialMaxPooling( + kw=klayer.pool_length, + kh=1, + dw=klayer.stride, + dh=1, + pad_w=bpadW, + pad_h=bpadH, + to_ceil=False, + format="NHWC", + bigdl_type="float" + ) + seq.add(blayer) + return seq + + def create_averagepooling1d(self, klayer, kclayer): + input_shape = klayer.get_input_shape_at(0) # batch, steps, dim + bpadW, bpadH = self.to_bigdl_2d_padding(klayer.border_mode) + + seq = BLayer.Sequential() + seq.add(BLayer.View([1, input_shape[1], input_shape[2]], num_input_dims=3)) + blayer = BLayer.SpatialAveragePooling( + kw=klayer.pool_length, + kh=1, + dw=klayer.stride, + dh=1, + pad_w=bpadW, + pad_h=bpadH, + global_pooling=False, + ceil_mode=False, + count_include_pad=False, + divide=True, + format="NHWC", + bigdl_type="float" + ) + seq.add(blayer) + return seq + + def create_globalaveragepooling2d(self, klayer, kclayer): + bigdl_order = self.get_bdim_order(kclayer) + input_shape = klayer.get_input_shape_at(0) + if bigdl_order == "NCHW": + b_kw = input_shape[3] + b_kh = input_shape[2] + else: + b_kw = input_shape[2] + b_kh = input_shape[1] + + seq = BLayer.Sequential() + blayer = BLayer.SpatialAveragePooling( + kw=b_kw, + kh=b_kh, + dw=b_kw, + dh=b_kh, + pad_w=0, + pad_h=0, + global_pooling=False, + ceil_mode=False, + count_include_pad=False, + divide=True, + format=bigdl_order, + bigdl_type="float" + ) + seq.add(blayer) + if bigdl_order == "NCHW": + seq.add(BLayer.Squeeze(3, num_input_dims=3)) + seq.add(BLayer.Squeeze(2, num_input_dims=2)) + else: + seq.add(BLayer.Squeeze(2, num_input_dims=3)) + seq.add(BLayer.Squeeze(1, num_input_dims=2)) + return seq + + def check_constraint_in_config(self, config): + if "W_constraint" in config: + if config["W_constraint"]: + raise Exception("W_constraint is not supported for now") + if "b_constraint" in config: + if config["b_constraint"]: + raise Exception("b_constraint is not supported for now") + + def combo_parameter_layer(self, blayer, config): + self.check_constraint_in_config(config) + + blayer.set_name(config["name"]) + if hasattr(blayer, "set_init_method"): + blayer.set_init_method(self.to_bigdl_init(config["init"]), + BInit.Zeros()) # Keras always set this to be zeros + # "linear" meaning do nothing + if config["activation"] != "linear" : + activation = self.to_bigdl_activation(config["activation"], + "%s_%s" % (config["name"], config["activation"])) + return self.fuse(blayer, activation) + else: + return blayer + + def to_bigdl_activation(self, activation_name, activation_id): + activation = None + if activation_name == "tanh": + activation = BLayer.Tanh() + elif activation_name == "sigmoid": + activation = BLayer.Sigmoid() + elif activation_name == "relu": + activation = BLayer.ReLU() + elif activation_name == "softmax": + activation = BLayer.SoftMax() + else: + raise Exception("Unsupported activation type: %s" % activation_name) + activation.set_name(activation_id) + return activation + + def get_value_from_init(self, kinit_method, shape): + if kinit_method == "zero": + return np.zeros(shape) + elif kinit_method == "one": + return np.ones(shape) + else: + raise Exception("We don't support % for now", kinit_method) + + def to_bigdl_init(self, kinit_method): # kinit_method is a string + init = None + if kinit_method == "glorot_uniform": + init = BInit.Xavier() + elif kinit_method == "one": + init = BInit.Ones() + elif kinit_method == "zero": + init = BInit.Zeros() + else: + raise Exception("Unsupported init type: %s" % init) + return init + + def to_bigdl_reg(self, reg): # reg is a dict + if reg: + raise Exception("will support reg very soon") + else: + return None + + def fuse(self, src_blayer, activation): # activation is a string + seq = BLayer.Sequential() + seq.add(src_blayer) + seq.add(activation) + seq.set_name(src_blayer.name()) + return seq \ No newline at end of file diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 9fade510de9..4e8ba20864a 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -66,6 +66,22 @@ def __init__(self, jvalue, bigdl_type, *args): bigdl_type, JavaValue.jvm_class_constructor(self), *args) self.bigdl_type = bigdl_type + def set_running_mean(self, running_mean): + """ + :param running_mean: a ndarray + """ + callBigDlFunc(self.bigdl_type, "setRunningMean", + self.value, JTensor.from_ndarray(running_mean)) + return self + + def set_running_std(self, running_std): + """ + :param running_mean: a ndarray + """ + callBigDlFunc(self.bigdl_type, "setRunningStd", + self.value, JTensor.from_ndarray(running_std)) + return self + def __str__(self): """ >>> conv2 = SpatialConvolution(6, 12, 5, 5).set_name("conv2") @@ -387,6 +403,10 @@ def get_weights(self): print("The layer does not have weight/bias") return None + def is_with_weights(self): + return callBigDlFunc(self.bigdl_type, + "isWithWeights", self.value) + def save(self, path, over_write = False): callBigDlFunc(self.bigdl_type, "modelSave", self.value, path, over_write) @@ -450,17 +470,20 @@ def unfreeze(self, names=None): callBigDlFunc(self.bigdl_type, "unFreeze", self.value, names) return self - def training(self): + def training(self, is_training=True): ''' - Set this layer in the training mode + Set this layer in the training mode or in predition mode if is_training=False ''' - callJavaFunc(get_spark_context(), self.value.training) + if is_training: + callJavaFunc(get_spark_context(), self.value.training) + else: + callJavaFunc(get_spark_context(), self.value.evaluate) return self def is_training(self): ''' :return: Whether this layer is in the training mode - + >>> layer = Dropout() creating: createDropout >>> layer = layer.evaluate() @@ -479,7 +502,7 @@ def quantize(self): >>> fc = Linear(4, 2) creating: createLinear - >>> fc.set_weights([np.ones((4, 2)), np.ones((2,))]) + >>> fc.set_weights([np.ones((2, 4)), np.ones((2,))]) >>> input = np.ones((2, 4)) >>> fc.forward(input) array([[ 5., 5.], @@ -568,6 +591,12 @@ def add(self, model): self.value.add(model.value) return self + @property + def layers(self): + jlayers = callBigDlFunc(self.bigdl_type, "getContainerModules" , self) + layers = [Layer.of(jlayer) for jlayer in jlayers] + return layers + class Model(Container): """ @@ -599,8 +628,12 @@ class Model(Container): def __init__(self, inputs, outputs, + jvalue=None, bigdl_type="float", byte_order="little_endian", model_type="bigdl"): - if model_type == "bigdl": + if jvalue: + self.value = jvalue + self.bigdl_type = bigdl_type + elif model_type == "bigdl": super(Model, self).__init__(None, bigdl_type, to_list(inputs), to_list(outputs)) @@ -610,6 +643,20 @@ def __init__(self, super(Model, self).__init__(model, bigdl_type) + @staticmethod + def from_jvalue(jvalue, bigdl_type="float"): + """ + Create a Python Model base on the given java value + :param jvalue: Java object create by Py4j + :return: A Python Model + """ + model = Model([], [], jvalue=jvalue) + model.value = jvalue + return model + + def __str__(self): + return "->".join(self.layers()) + @staticmethod def load(path, bigdl_type="float"): """ @@ -643,6 +690,22 @@ def load_torch(path, bigdl_type="float"): jmodel = callBigDlFunc(bigdl_type, "loadTorch", path) return Layer.of(jmodel) + @staticmethod + def load_keras(def_path, weights_path=None, by_name=False): + """ + Load a pre-trained Keras model. + + :param def_path: The json path containing the keras model definition. + :param weights_path: The HDF5 path containing the pre-trained keras model weights. + :return: A pre-trained model. + """ + from bigdl.keras1.converter import DefinitionLoader, WeightLoader + if weights_path: + return WeightLoader.load_weights_from_json_hdf5(def_path, weights_path, by_name=by_name) + else: + return DefinitionLoader.from_json_path(def_path) + return bmodel + @staticmethod def load_caffe(model, defPath, modelPath, match_all=True, bigdl_type="float"): """ @@ -1225,12 +1288,12 @@ def get_hidden_state(self): :return: list of hidden state and cell """ - state = callBigDlFunc(self.bigdl_type, "getHiddenState", self.value) + state = callBigDlFunc(self.bigdl_type, "getHiddenState", self.value) for idx, tensor in enumerate(state): state[idx] = tensor.to_ndarray() return state - + def set_hidden_state(self, states): """ set hidden state and cell at first time step. @@ -1767,6 +1830,7 @@ def __init__(self, JTensor.from_ndarray(init_bias), JTensor.from_ndarray(init_grad_weight), JTensor.from_ndarray(init_grad_bias)) + def set_init_method(self, weight_init_method = None, bias_init_method = None): callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, weight_init_method, bias_init_method) @@ -4315,11 +4379,11 @@ class ResizeBilinear(Layer): """ Resize the input image with bilinear interpolation. The input image must be a float tensor with NHWC layout - + :param output_height: output height :param output_width: output width :param align_corner: align corner or not - + >>> resizeBilinear = ResizeBilinear(10, 20, False) creating: createResizeBilinear """ diff --git a/python/dllib/src/bigdl/utils/common.py b/python/dllib/src/bigdl/utils/common.py index 5fa51973447..428a109215c 100644 --- a/python/dllib/src/bigdl/utils/common.py +++ b/python/dllib/src/bigdl/utils/common.py @@ -28,7 +28,8 @@ from pyspark import SparkConf import numpy as np import threading -from bigdl.util.engine import get_bigdl_classpath, is_spark_below_2_2 +import tempfile +from bigdl.util.engine import prepare_env, get_bigdl_classpath, is_spark_below_2_2 INTMAX = 2147483647 INTMIN = -2147483648 @@ -439,7 +440,7 @@ def get_spark_context(conf = None): return SparkContext.getOrCreate(conf=conf or create_spark_conf()) else: # Might have threading issue but we cann't add _lock here - # as it's not RLock in spark1.5 + # as it's not RLock in spark1.5; if SparkContext._active_spark_context is None: SparkContext(conf=conf or create_spark_conf()) return SparkContext._active_spark_context @@ -458,7 +459,6 @@ def callBigDlFunc(bigdl_type, name, *args): api = getattr(jinstance, name) return callJavaFunc(sc, api, *args) - def _java2py(sc, r, encoding="bytes"): if isinstance(r, JavaObject): clsName = r.getClass().getSimpleName() @@ -536,6 +536,11 @@ def _py2java(sc, obj): obj = sc._jvm.org.apache.spark.bigdl.api.python.BigDLSerDe.loads(data) return obj +def create_tmp_path(): + tmp_file = tempfile.NamedTemporaryFile(prefix="bigdl") + tmp_file.close() + return tmp_file.name + def _test(): import doctest From fbc7cc08d9fcc5774eae2616306a57bb858ddef3 Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Wed, 15 Nov 2017 09:15:24 -0600 Subject: [PATCH 335/823] Support keras model loading (#1827) * keras load model and api wrapper * fix batchnorm * fix averagepooling * remove function wrapper and clean code * add unit test * remove backend * fix and clean * update * separate unittest * x * remove -n * ignore application * separate test * pass parameter * try to fix test * disable shape checking * ignore --- python/dllib/test/__init__.py | 0 python/dllib/test/bigdl/__init__.py | 0 .../test/bigdl/keras1/test_application.py | 156 ++++++++ python/dllib/test/bigdl/keras1/test_layer.py | 347 ++++++++++++++++++ .../test/bigdl/keras1/test_load_model.py | 68 ++++ python/dllib/test/bigdl/test_utils.py | 275 ++++++++++++++ python/dllib/test/dev/run-keras.sh | 43 +++ python/dllib/test/dev/run-tests | 7 +- python/dllib/test/dev/run-tests.py | 198 ---------- 9 files changed, 895 insertions(+), 199 deletions(-) create mode 100644 python/dllib/test/__init__.py create mode 100644 python/dllib/test/bigdl/__init__.py create mode 100644 python/dllib/test/bigdl/keras1/test_application.py create mode 100644 python/dllib/test/bigdl/keras1/test_layer.py create mode 100644 python/dllib/test/bigdl/keras1/test_load_model.py create mode 100644 python/dllib/test/bigdl/test_utils.py create mode 100755 python/dllib/test/dev/run-keras.sh delete mode 100755 python/dllib/test/dev/run-tests.py diff --git a/python/dllib/test/__init__.py b/python/dllib/test/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/python/dllib/test/bigdl/__init__.py b/python/dllib/test/bigdl/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/python/dllib/test/bigdl/keras1/test_application.py b/python/dllib/test/bigdl/keras1/test_application.py new file mode 100644 index 00000000000..37eacea8be5 --- /dev/null +++ b/python/dllib/test/bigdl/keras1/test_application.py @@ -0,0 +1,156 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from __future__ import print_function + +from keras.layers import * + +np.random.seed(1337) # for reproducibility +import pytest +from keras.applications import * +from bigdl.keras1.converter import * + +from test.bigdl.test_utils import BigDLTestCase, TestModels + + +class TestApplication(BigDLTestCase): + + def assert_model(self, input_data, kmodel, rtol=1e-5, atol=1e-5): + bmodel = DefinitionLoader.from_kmodel(kmodel) + WeightLoader.load_weights_from_kmodel(bmodel, kmodel, by_name=True) + + keras_output = kmodel.predict(input_data) + bmodel.training(is_training=False) + bigdl_output = bmodel.forward(input_data) + + self.assert_allclose(keras_output, bigdl_output, rtol=rtol, atol=atol) + + def test_lenet(self): + kmodel, input_data, output_data = TestModels.kmodel_seq_lenet_mnist() + self.modelTest(input_data, kmodel, dump_weights=True) + + def test_text_classification(self): + '''This example demonstrates the use of Convolution1D for text classification. + This example is from Keras + ''' + + import numpy as np + np.random.seed(1337) # for reproducibility + + from keras.preprocessing import sequence + from keras.models import Sequential + from keras.layers import Dense, Dropout, Activation + from keras.layers import Embedding + from keras.layers import Convolution1D + from keras.datasets import imdb + + # set parameters: + max_features = 5000 + maxlen = 400 + batch_size = 32 + embedding_dims = 50 + nb_filter = 250 + filter_length = 3 + hidden_dims = 250 + nb_epoch = 2 + + print('Loading data...') + (X_train, y_train), (X_test, y_test) = imdb.load_data(nb_words=max_features) + print(len(X_train), 'train sequences') + print(len(X_test), 'test sequences') + + print('Pad sequences (samples x time)') + X_train = sequence.pad_sequences(X_train, maxlen=maxlen) + X_test = sequence.pad_sequences(X_test, maxlen=maxlen) + print('X_train shape:', X_train.shape) + print('X_test shape:', X_test.shape) + + print('Build model...') + model = Sequential() + + model.add(Embedding(max_features, + embedding_dims, + input_length=maxlen)) # Exception if specify Dropout dropout=0.2 + + # we add a Convolution1D, which will learn nb_filter + # word group filters of size filter_length: + model.add(Convolution1D(nb_filter=nb_filter, + filter_length=filter_length, + border_mode='valid', + activation='relu', + subsample_length=1)) + # we use max pooling: + model.add(GlobalMaxPooling1D()) + # model.add(GlobalAveragePooling1D()) + + # We add a vanilla hidden layer: + model.add(Dense(hidden_dims)) + model.add(Dropout(0.2)) + model.add(Activation('relu')) + + # We project onto a single unit output layer, and squash it with a sigmoid: + model.add(Dense(1)) + model.add(Activation('sigmoid')) + + model.compile(loss='binary_crossentropy', + optimizer='adam', + metrics=['accuracy']) + model = use_bigdl_backend(model) + + model.fit(X_train, y_train, + batch_size=batch_size, + nb_epoch=nb_epoch, + validation_data=(X_test, y_test)) + # 2017-09-22 15:53:45 INFO DistriOptimizer$:657 + # - Top1Accuracy is Accuracy(correct: 21557, count: 25000, accuracy: 0.86228) + # this result is from GlobalAveragePooling not GlobalMaxPooling. + model.predict(X_test) + model.evaluate(X_test, y_test) + print(model) + + def test_resnet50(self): + keras.backend.set_image_dim_ordering("th") + kmodel = resnet50.ResNet50(include_top=False, input_shape=(3, 224, 224)) + input_data = np.random.random([2, 3, 224, 224]) + self.assert_model(input_data, kmodel) + + def test_vgg16(self): + keras.backend.set_image_dim_ordering("th") + kmodel = vgg16.VGG16(include_top=False, input_shape=(3, 224, 224)) + input_data = np.random.random([2, 3, 224, 224]) + self.assert_model(input_data, kmodel) + + def test_vgg19(self): + keras.backend.set_image_dim_ordering("th") + kmodel = vgg19.VGG19(include_top=False, input_shape=(3, 224, 224)) + input_data = np.random.random([2, 3, 224, 224]) + self.assert_model(input_data, kmodel) + + def test_inception_v3(self): + keras.backend.set_image_dim_ordering("th") + kmodel = inception_v3.InceptionV3(include_top=False, input_shape=(3, 299, 299)) + input_data = np.random.random([2, 3, 299, 299]) + + bmodel = DefinitionLoader.from_kmodel(kmodel) + WeightLoader.load_weights_from_kmodel(bmodel, kmodel, by_name=True) + + keras_output = kmodel.predict(input_data) + bmodel.training(is_training=False) + bigdl_output = bmodel.forward(input_data) + + self.assert_allclose(keras_output, bigdl_output, rtol=1e-3, atol=1e-3) + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/python/dllib/test/bigdl/keras1/test_layer.py b/python/dllib/test/bigdl/keras1/test_layer.py new file mode 100644 index 00000000000..a766c203dc9 --- /dev/null +++ b/python/dllib/test/bigdl/keras1/test_layer.py @@ -0,0 +1,347 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from __future__ import print_function + +import pytest +from keras.layers import * + +np.random.seed(1337) # for reproducibility +from keras.layers.core import * +from keras.layers.convolutional import * +from keras.layers import Dense, Input +from bigdl.keras1.converter import * +from test.bigdl.test_utils import BigDLTestCase + + +class TestLayer(BigDLTestCase): + + def test_dense(self): + input_data = np.random.random_sample([1, 10]) + dense = Dense(2, init='one', activation="relu", input_shape=(10, )) + self.modelTestSingleLayer(input_data, dense, dump_weights=True) + + def test_timedistributeddense(self): + input_data = np.random.random_sample([2, 4, 5]) + layer = TimeDistributedDense(6, input_shape=(4, 5)) + self.modelTestSingleLayer(input_data, layer, dump_weights=True) + + def test_embedding(self): + # Test index start from 0 + input_data = np.array([[0, 1, 2, 99], [0, 4, 5, 99]]) + layer = Embedding(input_dim=100, # index [0,99] + output_dim=64, # vector dim + init='uniform', + input_length=None, + W_regularizer=None, activity_regularizer=None, + W_constraint=None, + mask_zero=False, + weights=None, dropout=0.) + self.modelTestSingleLayer(input_data, layer, dump_weights=True) + # Random input + input_data2 = np.random.randint(100, size=(10, 128)) # batch: 20, seqlen 128 + self.modelTestSingleLayer(input_data2, layer, dump_weights=True) + + # TODO: add test that exception would be raised if input_lenght == 6 + with pytest.raises(Exception) as excinfo: + layer = Embedding(input_dim=100, # index [0,99] + output_dim=64, # vector dim + init='uniform', + input_length=111, + W_regularizer=None, activity_regularizer=None, + W_constraint=None, + mask_zero=False, + weights=None, dropout=0.) + input_data = np.random.randint(100, size=(10, 128)) # batch: 20, seqlen 128 + self.modelTestSingleLayer(input_data, layer, dump_weights=True) + assert str(excinfo.value) == """The input_length doesn't match: 128 vs 111""" + + # TODO: Add more test case? activation + def test_conv1D(self): + input_data = np.random.random_sample([1, 10, 32]) + layer = lambda: Convolution1D(64, 3, border_mode='valid', input_shape=(10, 32)) + self.modelTestSingleLayerWithOrdersModes(input_data, layer, + dump_weights=True, dim_orderings=["tf"]) + + layer2 = lambda: Convolution1D(64, 3, border_mode='same', + input_shape=(10, 32)) + self.modelTestSingleLayerWithOrdersModes(input_data, layer2, + dump_weights=True, dim_orderings=["tf"]) + + layer3 = lambda: Convolution1D(64, 3, border_mode='same', + activation="relu", input_shape=(10, 32)) + self.modelTestSingleLayerWithOrdersModes(input_data, layer3, + dump_weights=True, dim_orderings=["tf"]) + + def _load_keras(self, json_path, hdf5_path): + with open(json_path, "r") as jp: + kmodel = model_from_json(jp.read()) + kmodel.load_weights_from_hdf5(hdf5_path) + bmodel = DefinitionLoader.from_json_path(json_path) + WeightLoader.load_weights_from_hdf5(bmodel, kmodel, hdf5_path) + return kmodel, bmodel + + def test_conv2D(self): + input_data = np.random.random_sample([1, 3, 128, 128]) + layer = lambda: Convolution2D(64, 1, 20, input_shape=(3, 128, 128)) + self.modelTestSingleLayerWithOrdersModes(input_data, + layer, + dump_weights=True, rtol=1e-5, atol=1e-5) + # Test if alias works or not + layer = lambda: Conv2D(64, 3, 1, input_shape=(3, 128, 128)) + self.modelTestSingleLayerWithOrdersModes(input_data, + layer, + dump_weights=True, rtol=1e-5, atol=1e-5) + + def test_maxpooling2d(self): + input_data = np.random.random_sample([1, 3, 20, 20]) + layer = lambda: MaxPooling2D(pool_size=[3, 3], strides=[2, 2], + border_mode="valid", input_shape=(3, 20, 20)) + self.modelTestSingleLayerWithOrdersModes(input_data, layer) + + def test_maxpooling1d(self): + input_data = np.random.random_sample([1, 3, 20]) + layer = MaxPooling1D(pool_length=2, stride=None, + border_mode='valid', + input_shape=(3, 20),) + self.modelTestSingleLayer(input_data, layer) + + def test_globalmaxpooling2d(self): + input_data = np.random.random_sample([1, 3, 20, 20]) + layer = lambda: GlobalMaxPooling2D(input_shape=(3, 20, 20)) + self.modelTestSingleLayerWithOrdersModes(input_data, layer, + border_modes=[None]) + + def test_globalmaxpooling1d(self): + input_data = np.random.random_sample([1, 3, 20]) + layer = GlobalMaxPooling1D(input_shape=(3, 20)) + self.modelTestSingleLayer(input_data, layer) + + def test_averagepooling2d(self): + input_data = np.random.random_sample([1, 3, 20, 20]) + layer = lambda: AveragePooling2D(pool_size=[3, 3], strides=[2, 2], + border_mode="valid", input_shape=(3, 20, 20)) + self.modelTestSingleLayerWithOrdersModes(input_data, layer) + + def test_averagepooling1d(self): + input_data = np.random.random_sample([1, 3, 20]) + layer = lambda: AveragePooling1D(pool_length=2, stride=None, + border_mode='valid', input_shape=(3, 20)) + self.modelTestSingleLayerWithOrdersModes(input_data, layer, dim_orderings=["tf"]) + + def test_globalaveragepooling2d(self): + input_data = np.random.random_sample([1, 3, 20, 20]) + layer = lambda: GlobalAveragePooling2D(input_shape=(3, 20, 20)) + self.modelTestSingleLayerWithOrdersModes(input_data, layer, rtol=1e-6, atol=1e-6, + border_modes=[None]) + + def test_globalaveragepooling1d(self): + input_data = np.random.random_sample([1, 3, 20]) + layer = GlobalAveragePooling1D(input_shape=(3, 20)) + self.modelTestSingleLayer(input_data, layer) + + def test_batchnormalization(self): + input_data = np.random.random_sample([2, 6, 128, 128]) + layer = BatchNormalization(input_shape=(6, 128, 128), axis=1) + self.modelTestSingleLayer(input_data, layer, + dump_weights=True) + + def test_flatten(self): + input_data = np.random.random_sample([1, 2, 3]) + layer = Flatten(input_shape=(2, 3)) + self.modelTestSingleLayer(input_data, layer) + + def test_reshape(self): + input_data = np.random.random_sample([1, 3, 5, 4]) + layer = Reshape(target_shape=(3, 20), input_shape=(3, 5, 4)) + self.modelTestSingleLayer(input_data, layer) + + def test_repeatvector(self): + input_data = np.random.random_sample([2, 3]) + layer = RepeatVector(4, input_shape=(3, )) + self.modelTestSingleLayer(input_data, layer) + + def test_merge_concat(self): + # input_data1 = np.random.random_sample([2, 3, 5]) + # input_data2 = np.random.random_sample([2, 3, 6]) + # model1 = Sequential() + # model1.add(Dense(20, input_dim=2)) + # model1.add(Dense(20, input_dim=2)) + # + # model2 = Sequential() + # model2.add(Input(input_dim=32)) + # + # merged_model = Sequential() + # merged_model.add(Merge([model1, model2], mode='concat', concat_axis=0)) + + inputLayer1 = InputLayer(input_shape=(3, 6, 7)) + inputLayer2 = InputLayer(input_shape=(3, 6, 8)) + inputLayer3 = InputLayer(input_shape=(3, 6, 9)) + + layer = Merge([inputLayer1, inputLayer2, inputLayer3], mode='concat', concat_axis=3) + # the index including batch and start from zero which is the index to be merge + input_data = [np.random.random_sample([2, 3, 6, 7]), + np.random.random([2, 3, 6, 8]), + np.random.random([2, 3, 6, 9])] + self.modelTestSingleLayer(input_data, layer, functional_apis=[False]) + + def test_merge_sum(self): + inputLayer1 = InputLayer(input_shape=(3, 6, 7)) + inputLayer2 = InputLayer(input_shape=(3, 6, 7)) + inputLayer3 = InputLayer(input_shape=(3, 6, 7)) + + layer = Merge([inputLayer1, inputLayer2, inputLayer3], mode='sum') + # the index including batch and start from zero, and it's the index to be merge + input_data = [np.random.random_sample([2, 3, 6, 7]), + np.random.random([2, 3, 6, 7]), + np.random.random([2, 3, 6, 7])] + self.modelTestSingleLayer(input_data, layer, functional_apis=[False]) + + def test_merge_mul(self): + inputLayer1 = InputLayer(input_shape=(3, 6, 7)) + inputLayer2 = InputLayer(input_shape=(3, 6, 7)) + inputLayer3 = InputLayer(input_shape=(3, 6, 7)) + + layer = Merge([inputLayer1, inputLayer2, inputLayer3], mode='mul') + input_data = [np.random.random([2, 3, 6, 7]), + np.random.random([2, 3, 6, 7]), + np.random.random([2, 3, 6, 7])] + self.modelTestSingleLayer(input_data, layer, functional_apis=[False]) + + def test_merge_max(self): + inputLayer1 = InputLayer(input_shape=(3, 6, 7)) + inputLayer2 = InputLayer(input_shape=(3, 6, 7)) + inputLayer3 = InputLayer(input_shape=(3, 6, 7)) + + layer = Merge([inputLayer1, inputLayer2, inputLayer3], mode='max') + input_data = [np.random.random([2, 3, 6, 7]), + np.random.random([2, 3, 6, 7]), + np.random.random([2, 3, 6, 7])] + self.modelTestSingleLayer(input_data, layer, functional_apis=[False]) + + def test_merge_dot(self): + # use batch_input_shape for merge dot + inputLayer1 = InputLayer(batch_input_shape=(2, 3)) + inputLayer2 = InputLayer(batch_input_shape=(2, 3)) + + layer = Merge([inputLayer1, inputLayer2], mode='dot') + input_data = [np.random.random([2, 3]), + np.random.random([2, 3])] + self.modelTestSingleLayer(input_data, layer, functional_apis=[False]) + + def test_elu(self): + input_data = np.random.random_sample([10, 2, 3, 4]) + layer = ELU(alpha=1.0, input_shape=(2, 3, 4)) + self.modelTestSingleLayer(input_data, layer) + + def test_prelu(self): + input_data = np.random.random_sample([1, 2, 3, 4]) + layer = PReLU(input_shape=(2, 3, 4)) + self.modelTestSingleLayer(input_data, layer) + + def test_leakyrelu(self): + input_data = np.random.random_sample([1, 2, 3]) + layer = LeakyReLU(alpha=0.5, input_shape=(2, 3)) + self.modelTestSingleLayer(input_data, layer) + + def test_thresholdedrelu(self): + input_data = np.random.random_sample([1, 2, 3]) + layer = ThresholdedReLU(theta=0.2, input_shape=(2, 3)) + self.modelTestSingleLayer(input_data, layer) + + def test_parametricsoftplus(self): + input_data = np.random.random_sample([1, 2, 3]) + layer = ParametricSoftplus(alpha_init=0.4, beta_init=2.5, input_shape=(2, 3)) + self.modelTestSingleLayer(input_data, layer) + + def test_zeropadding1d(self): + input_data = np.random.uniform(0, 1, [3, 2, 3]) + layer1 = ZeroPadding1D(padding=3, input_shape=(2, 3)) + self.modelTestSingleLayer(input_data, layer1) + layer2 = ZeroPadding1D(padding=(2, 3), input_shape=(2, 3)) + self.modelTestSingleLayer(input_data, layer2) + layer3 = ZeroPadding1D(padding={'left_pad': 1, 'right_pad': 2}, input_shape=(2, 3)) + self.modelTestSingleLayer(input_data, layer3) + + def test_zeropadding2d(self): + input_data = np.random.uniform(0, 1, [1, 2, 3, 4]) + layer1 = ZeroPadding2D(padding=(2, 3), input_shape=(2, 3, 4)) + self.modelTestSingleLayer(input_data, layer1) + layer2 = ZeroPadding2D(padding=(2, 3, 4, 1), input_shape=(2, 3, 4)) + self.modelTestSingleLayer(input_data, layer2) + layer3 = ZeroPadding2D( + padding={'top_pad': 1, 'bottom_pad': 2, 'left_pad': 3, 'right_pad': 4}, + input_shape=(2, 3, 4)) + self.modelTestSingleLayer(input_data, layer3) + + def test_zeropadding3d(self): + input_data = np.random.uniform(0, 1, [3, 2, 4, 1, 5]) + layer = ZeroPadding3D(padding=(1, 2, 3), input_shape=(2, 4, 1, 5)) + self.modelTestSingleLayer(input_data, layer) + + def test_cropping1d(self): + input_data = np.random.uniform(0, 1, [3, 10, 10]) + layer = Cropping1D(cropping=(1, 2)) + self.modelTestSingleLayer(input_data, layer) + + def test_simplernn(self): + input_data = np.random.random([3, 4, 5]) + layer = SimpleRNN(5, input_shape=(4, 5), return_sequences=True) + self.modelTestSingleLayer(input_data, layer, + dump_weights=True, rtol=1e-6, atol=1e-6) + + layer2 = SimpleRNN(3, input_shape=(4, 5), return_sequences=False) + self.modelTestSingleLayer(input_data, layer2, + dump_weights=True, rtol=1e-6, atol=1e-6) + layer3 = SimpleRNN(3, input_shape=(4, 5), activation='relu') + self.modelTestSingleLayer(input_data, layer3, + dump_weights=True, rtol=1e-6, atol=1e-6) + + def test_lstm(self): + input_data = np.random.random([3, 4, 5]) + layer = LSTM(5, input_shape=(4, 5), return_sequences=True, inner_activation='sigmoid') + self.modelTestSingleLayer(input_data, layer, dump_weights=True) + layer2 = LSTM(3, input_shape=(4, 5), return_sequences=False, inner_activation='sigmoid') + self.modelTestSingleLayer(input_data, layer2, dump_weights=True) + + def test_gru(self): + input_data = np.random.random([3, 4, 5]) + layer = GRU(4, input_shape=(4, 5), return_sequences=True, inner_activation='sigmoid') + self.modelTestSingleLayer(input_data, layer, dump_weights=True) + layer2 = GRU(8, input_shape=(4, 5), return_sequences=False, inner_activation='sigmoid') + self.modelTestSingleLayer(input_data, layer2, dump_weights=True) + + # TODO: Support share weights training. + def test_multiple_inputs_share_weights(self): + with pytest.raises(Exception) as excinfo: + input_node1 = Input(shape=[3, 16, 16]) + input_node2 = Input(shape=[3, 32, 32]) + conv2d = Convolution2D(5, 3, 3, + border_mode='same') + conv1 = conv2d(input_node1) + conv2 = conv2d(input_node2) + out1 = Flatten()(conv1) + out2 = Flatten()(conv2) + model1 = Model(input=[input_node1, input_node2], output=[out1, out2]) + tensor1, tensor2 = model1([input_node1, input_node2]) + out3 = Dense(7)(tensor1) + out4 = Dense(8)(tensor2) + model2 = Model(input=[input_node1, input_node2], output=[out3, out4]) + def_path, w_path = self._dump_keras(model2) + bigdl_model = DefinitionLoader.from_json_path(def_path) + assert str(excinfo.value) == """Convolution2D doesn't support multiple inputs with shared weights""" # noqa + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/python/dllib/test/bigdl/keras1/test_load_model.py b/python/dllib/test/bigdl/keras1/test_load_model.py new file mode 100644 index 00000000000..a93e675edd7 --- /dev/null +++ b/python/dllib/test/bigdl/keras1/test_load_model.py @@ -0,0 +1,68 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from __future__ import print_function + +import numpy as np +import pytest +from numpy.testing import assert_allclose + +import bigdl.nn.layer as BLayer +from bigdl.keras1.converter import WeightLoader +from bigdl.keras1.converter import DefinitionLoader + +np.random.seed(1337) # for reproducibility +from test.bigdl.test_utils import BigDLTestCase, TestModels + + +class TestLoadModel(BigDLTestCase): + + def __kmodel_load_def_weight_test(self, kmodel, input_data): + keras_model_path_json, keras_model_path_hdf5 = self._dump_keras(kmodel, dump_weights=True) + bmodel = DefinitionLoader.from_json_path(keras_model_path_json) + WeightLoader.load_weights_from_hdf5(bmodel, + kmodel, + keras_model_path_hdf5) + bmodel.training(False) + boutput = bmodel.forward(input_data) + koutput = kmodel.predict(input_data) + assert_allclose(boutput, koutput, rtol=1e-5) + + def test_load_api_with_hdf5(self): + kmodel, input_data, output_data = TestModels.kmodel_graph_1_layer() + keras_model_json_path, keras_model_hdf5_path = self._dump_keras(kmodel, dump_weights=True) + bmodel = BLayer.Model.load_keras(keras_model_json_path, keras_model_hdf5_path) + self.assert_allclose(kmodel.predict(input_data), + bmodel.forward(input_data)) + + def test_load_api_no_hdf5(self): + kmodel, input_data, output_data = TestModels.kmodel_graph_1_layer() + keras_model_json_path, keras_model_hdf5_path = self._dump_keras(kmodel, dump_weights=True) + bmodel = BLayer.Model.load_keras(keras_model_json_path) + + def test_load_def_weights_graph_1_layer(self): + kmodel, input_data, output_data = TestModels.kmodel_graph_1_layer() + self.__kmodel_load_def_weight_test(kmodel, input_data) + + def test_load_def_weights_graph_activation(self): + kmodel, input_data, output_data = TestModels.kmodel_graph_activation_is_layer() + self.__kmodel_load_def_weight_test(kmodel, input_data) + + def test_load_def_weights_kmodel_seq_lenet_mnist(self): + kmodel, input_data, output_data = TestModels.kmodel_seq_lenet_mnist() + self.__kmodel_load_def_weight_test(kmodel, input_data) + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/python/dllib/test/bigdl/test_utils.py b/python/dllib/test/bigdl/test_utils.py new file mode 100644 index 00000000000..bceff4b1fb4 --- /dev/null +++ b/python/dllib/test/bigdl/test_utils.py @@ -0,0 +1,275 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from __future__ import print_function +import numpy as np +from keras.models import Sequential, Model +from bigdl.keras1.converter import WeightLoader, WeightsConverter +np.random.seed(1337) # for reproducibility +from keras.layers.core import * +from keras.layers.convolutional import * +from keras.layers import Dense, Dropout, Input +from keras.models import model_from_json +from keras.optimizers import RMSprop +from bigdl.util.common import create_tmp_path +from bigdl.keras1.converter import DefinitionLoader +import numpy as np +from unittest import TestCase +import keras + + +class TestModels: + + @staticmethod + def kmodel_graph_1_layer(): + input1 = Input(shape=(3,)) + dense = Dense(2)(input1) + kmodel = Model(input=input1, output=dense) + kmodel.compile(loss='categorical_crossentropy', + optimizer=RMSprop(), + metrics=['accuracy']) + input_data = np.random.sample([1, 3]) + output_data = np.random.sample([1, 2]) + return kmodel, input_data, output_data + + @staticmethod + def kmodel_graph_activation_is_layer(): + input1 = Input(shape=(20,)) + dense = Dense(10)(input1) + activation = Activation('relu')(dense) + dense2 = Dense(10, activation='relu')(activation) + dense3 = Dense(5)(dense2) + # activation2 = Activation('softmax')(dense3) + kmodel = Model(input=input1, output=dense3) + kmodel.compile(loss='categorical_crossentropy', + optimizer=RMSprop(), + metrics=['accuracy']) + input_data = np.random.sample([1, 20]) + output_data = np.random.sample([1, 5]) + return kmodel, input_data, output_data + + @staticmethod + def kmodel_seq_lenet_mnist(): + # Part of the code is from keras example + # assuming channel first + input_shape = [1, 28, 28] + b_input_shape = input_shape[:] + nb_samples = 4 + b_input_shape.insert(0, nb_samples) + nb_classes = 10 + + # input image dimensions + img_rows, img_cols = 28, 28 + # number of convolutional filters to use + nb_filters = 32 + # size of pooling area for max pooling + pool_size = (2, 2) + # convolution kernel size + kernel_size = (3, 3) + + model = Sequential() + + model.add(Convolution2D(nb_filters, kernel_size[0], kernel_size[1], + border_mode='valid', + input_shape=input_shape)) + model.add(Activation('relu')) + model.add(Convolution2D(nb_filters, kernel_size[0], kernel_size[1])) + model.add(Activation('relu')) + model.add(MaxPooling2D(pool_size=pool_size)) + model.add(Dropout(0.)) + + model.add(Flatten()) + model.add(Dense(128)) + model.add(Activation('relu')) + model.add(Dropout(0.)) + model.add(Dense(nb_classes)) + model.add(Activation('softmax')) + + model.compile(loss='categorical_crossentropy', + optimizer='Adagrad', + metrics=['accuracy']) + input_data = np.random.random_sample(b_input_shape) + output_data = np.random.randint(1, 5, [nb_samples, 1]) + return model, input_data, output_data + + +class BigDLTestCase(TestCase): + + def __generate_model(self, input_data, output_layer): + def without_batch(batch_shape): + return batch_shape[1:] + if isinstance(output_layer, keras.engine.Merge): # it's a list in case of Merge Layer + assert isinstance(input_data, list) + input_tensor = [Input(shape=without_batch(i.shape)) for i in input_data] + else: + input_tensor = Input(shape=without_batch(input_data.shape)) + out_tensor = output_layer(input_tensor) + return Model(input=input_tensor, output=out_tensor) + + def __generate_sequence(self, input_data, output_layer): + seq = Sequential() + if not isinstance(output_layer, keras.engine.Merge): + seq.add(keras.layers.InputLayer(batch_input_shape=input_data.shape)) + seq.add(output_layer) + return seq + + def _load_keras(self, json_path, hdf5_path): + with open(json_path, "r") as jp: + kmodel = model_from_json(jp.read()) + kmodel.load_weights_from_hdf5(hdf5_path) + bmodel = DefinitionLoader.from_json_path(json_path) + WeightLoader.load_weights_from_hdf5(bmodel, kmodel, hdf5_path) + return kmodel, bmodel + + def _dump_keras(self, keras_model, dump_weights=False): + keras_model_path = create_tmp_path() + keras_model_json_path = keras_model_path + ".json" + keras_model_hdf5_path = keras_model_path + ".hdf5" + with open(keras_model_json_path, "w") as json_file: + json_file.write(keras_model.to_json()) + print("json path: " + keras_model_json_path) + if dump_weights: + keras_model.save(keras_model_hdf5_path) + print("hdf5 path: " + keras_model_hdf5_path) + return keras_model_json_path, keras_model_hdf5_path + + def assert_allclose(self, a, b, rtol=1e-6, atol=1e-6, msg=None): + # from tensorflow + self.assertEqual(a.shape, b.shape, "Shape mismatch: expected %s, got %s." % + (a.shape, b.shape)) + if not np.allclose(a, b, rtol=rtol, atol=atol): + cond = np.logical_or( + np.abs(a - b) > atol + rtol * np.abs(b), np.isnan(a) != np.isnan(b)) + if a.ndim: + x = a[np.where(cond)] + y = b[np.where(cond)] + print("not close where = ", np.where(cond)) + else: + # np.where is broken for scalars + x, y = a, b + print("not close lhs = ", x) + print("not close rhs = ", y) + print("not close dif = ", np.abs(x - y)) + print("not close tol = ", atol + rtol * np.abs(y)) + print("dtype = %s, shape = %s" % (a.dtype, a.shape)) + np.testing.assert_allclose(a, b, rtol=rtol, atol=atol, err_msg=msg) + + def __generate_keras_model(self, functional_api, input_data, output_layer): + if functional_api: + keras_model = self.__generate_model(input_data, output_layer) + else: + keras_model = self.__generate_sequence(input_data, output_layer) + return keras_model + + def modelTest(self, + input_data, + keras_model, + dump_weights=False, + is_training=False, + rtol=1e-7, + atol=1e-7): + # weight_converter is a function keras [ndarray]-> bigdl [ndarray] + keras_model_json_path, keras_model_hdf5_path = self._dump_keras(keras_model, dump_weights) + bigdl_model = DefinitionLoader.from_json_path(keras_model_json_path) + bigdl_model.training(is_training) + bigdl_output = bigdl_model.forward(input_data) + keras_output = keras_model.predict(input_data) + # TODO: we should verify bigdl_output and keras_output here + # init result is not the same, so we disable the verification for now + # self.assert_allclose(bigdl_output, + # keras_output, + # rtol=rtol, + # atol=atol) + if dump_weights: # load weights if possible + WeightLoader.load_weights_from_hdf5(bigdl_model, keras_model, keras_model_hdf5_path) + bweights = bigdl_model.get_weights() + bweights_from_keras = WeightsConverter.get_bigdl_weigths_from_keras(keras_model) + + # bweights and bweights_from_keras are all list + assert isinstance(bweights, list) + assert len(bweights) == len(bweights_from_keras) + for i in range(len(bweights)): + self.assert_allclose(bweights[i], bweights_from_keras[i], rtol, atol) + + bigdl_output2 = bigdl_model.forward(input_data) + self.assert_allclose(bigdl_output2, + keras_output, + rtol=rtol, + atol=atol) + + def modelTestSingleLayerWithOrdersModes(self, + input_data, + output_layer_creator, # a keras layer + dim_orderings=["tf", "th"], + border_modes=["valid", "same"], + dump_weights=False, + is_training=False, + rtol=1e-7, + atol=1e-7): + for dim_ordering in dim_orderings: + print("Testing with dim_ordering %s" % dim_ordering) + keras.backend.set_image_dim_ordering(dim_ordering) + for border_mode in border_modes: + print("Testing with border_mode %s" % border_mode) + output_layer = output_layer_creator() + if not hasattr(output_layer, "dim_ordering") and "1D" not in output_layer.__class__.__name__: # noqa + raise Exception("cannot set dim order for %s" % output_layer) + output_layer.dim_ordering = dim_ordering + if hasattr(output_layer, "border_mode"): + output_layer.border_mode = border_mode + elif border_mode is not None: + raise Exception("cannot set border_mode for %s" % output_layer) + self.modelTestSingleLayer(input_data, + output_layer, # a keras layer + dump_weights, + is_training, + rtol, + atol) + + def modelTestSingleLayer(self, + input_data, + output_layer, # a keras layer + dump_weights=False, + is_training=False, + rtol=1e-7, + atol=1e-7, + functional_apis=[True, False]): + for api in functional_apis: + self._do_modelTestSingleLayer( + input_data, + output_layer, # a keras layer + functional_api=api, + dump_weights=dump_weights, + is_training=is_training, + rtol=rtol, + atol=atol) + + def _do_modelTestSingleLayer(self, + input_data, + output_layer, # a keras layer + functional_api=True, + dump_weights=False, + is_training=False, + rtol=1e-7, + atol=1e-7): + keras_model = self.__generate_keras_model(functional_api=functional_api, + input_data=input_data, + output_layer=output_layer) + self.modelTest(input_data, + keras_model, + dump_weights=dump_weights, + is_training=is_training, + rtol=rtol, + atol=atol) diff --git a/python/dllib/test/dev/run-keras.sh b/python/dllib/test/dev/run-keras.sh new file mode 100755 index 00000000000..2e7b2cf9aab --- /dev/null +++ b/python/dllib/test/dev/run-keras.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash + +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# run: run-keras.sh python2.7 +. `dirname $0`/prepare_env.sh + +if (( $# < 1)); then + echo "Bad parameters. Usage: run-keras.sh python2.7" + exit -1 +fi + +cd "`dirname $0`" + +export DL_CORE_NUMBER=4 + +echo "${cyan}Using python version: $p${reset}" +export PYTHON_EXECUTABLE=$p +export PYSPARK_PYTHON=$p +export PYSPARK_DRIVER_PYTHON=$p +$1 -m pytest -v ../../../pyspark/test/bigdl/keras1 \ + --ignore=../../../pyspark/test/bigdl/keras1/test_application.py + +exit_status=$? +if [ $exit_status -ne 0 ]; +then + exit $exit_status +fi + diff --git a/python/dllib/test/dev/run-tests b/python/dllib/test/dev/run-tests index 62bc3802820..1e5aceb6d62 100755 --- a/python/dllib/test/dev/run-tests +++ b/python/dllib/test/dev/run-tests @@ -31,9 +31,14 @@ do $p -m pytest -v --doctest-modules ../../../pyspark/bigdl \ --ignore=../../../pyspark/bigdl/dataset/ \ --ignore=../../../pyspark/bigdl/util/tf_utils.py \ + --ignore=../../../pyspark/bigdl/keras1 \ + --ignore=../../../pyspark/test/bigdl/keras1/test_application.py \ --ignore=../../../pyspark/bigdl/models/ && \ $p -m pytest -v -n 4 ../../../pyspark/test/ \ - --ignore=../../../pyspark/test/bigdl/caffe/ + --ignore=../../../pyspark/test/bigdl/caffe/ \ + --ignore=../../../pyspark/test/bigdl/keras1/ \ + --ignore=../../../pyspark/test/bigdl/test_utils.py + exit_status=$? if [ $exit_status -ne 0 ]; then diff --git a/python/dllib/test/dev/run-tests.py b/python/dllib/test/dev/run-tests.py deleted file mode 100755 index b911c0c9552..00000000000 --- a/python/dllib/test/dev/run-tests.py +++ /dev/null @@ -1,198 +0,0 @@ -#!/usr/bin/env python - -# -# Copyright 2016 The BigDL Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# Adopt from Spark and it might be refactored in the future - - -from __future__ import print_function - -import logging -import os -import re -import subprocess -import sys -import tempfile -import time -from optparse import OptionParser -from os import path -from threading import Thread, Lock -from modules import all_modules # noqa -if sys.version < '3': - import Queue -else: - import queue as Queue - -if sys.version_info >= (2, 7): - subprocess_check_output = subprocess.check_output - subprocess_check_call = subprocess.check_call -else: - raise Exception("only support version >= 2,7") - - -def is_exe(path): - """ - Check if a given path is an executable file. - From: http://stackoverflow.com/a/377028 - """ - return os.path.isfile(path) and os.access(path, os.X_OK) - - -def which(program): - """ - Find and return the given program by its absolute path or 'None' if the program cannot be found. - From: http://stackoverflow.com/a/377028 - """ - - fpath = os.path.split(program)[0] - - if fpath: - if is_exe(program): - return program - else: - for path in os.environ.get("PATH").split(os.pathsep): - path = path.strip('"') - exe_file = os.path.join(path, program) - if is_exe(exe_file): - return exe_file - return None - - -current_dir = path.dirname(path.realpath(__file__)) -bigdl_python_home = path.abspath(path.join(current_dir, '..')) -sys.path.append(bigdl_python_home) - -SPARK_HOME = os.getenv('SPARK_HOME') - -python_modules = dict((m.name, m) for m in all_modules if m.python_test_goals if m.name != 'root') - - -def print_red(text): - print('\033[31m' + text + '\033[0m') - - -LOG_FILE = os.path.join(current_dir, "./unit-tests.log") -FAILURE_REPORTING_LOCK = Lock() -LOGGER = logging.getLogger() - - -def run_individual_python_test(test_name, python_exec): - - LOGGER.debug("Starting test(%s): %s", python_exec, test_name) - start_time = time.time() - try: - per_test_output = tempfile.TemporaryFile() - retcode = subprocess.Popen( - [python_exec, "-m", test_name], - stderr=per_test_output, stdout=per_test_output, env=env).wait() - except: - LOGGER.exception("Got exception while running %s with %s", test_name, python_exec) - # Here, we use os._exit() instead of sys.exit() in order to force Python to exit even if - # this code is invoked from a thread other than the main thread. - os._exit(1) - duration = time.time() - start_time - # Exit on the first failure. - if retcode != 0: - try: - with FAILURE_REPORTING_LOCK: - with open(LOG_FILE, 'ab') as log_file: - per_test_output.seek(0) - log_file.writelines(per_test_output) - per_test_output.seek(0) - for line in per_test_output: - decoded_line = line.decode() - if not re.match('[0-9]+', decoded_line): - print(decoded_line, end='') - per_test_output.close() - except: - LOGGER.exception("Got an exception while trying to print failed test output") - finally: - print_red("\nHad test failures in %s with %s; see logs." % (test_name, python_exec)) - # Here, we use os._exit() instead of sys.exit() in order to force Python to exit even if - # this code is invoked from a thread other than the main thread. - os._exit(-1) - else: - per_test_output.close() - LOGGER.info("Finished test(%s): %s (%is)", python_exec, test_name, duration) - - -def get_default_python_executables(): - python_execs = ["python2.7", "python3.5"] - return python_execs - - -def main(): - opts = parse_opts() - if (opts.verbose): - log_level = logging.DEBUG - else: - log_level = logging.INFO - logging.basicConfig(stream=sys.stdout, level=log_level, format="%(message)s") - LOGGER.info("Running BigDL python tests. Output is in %s", LOG_FILE) - if os.path.exists(LOG_FILE): - os.remove(LOG_FILE) - python_execs = opts.python_executables.split(',') - modules_to_test = [] - for module_name in opts.modules.split(','): - if module_name in python_modules: - modules_to_test.append(python_modules[module_name]) - else: - print("Error: unrecognized module '%s'. Supported modules: %s" % - (module_name, ", ".join(python_modules))) - sys.exit(-1) - LOGGER.info("Will test against the following Python executables: %s", python_execs) - LOGGER.info("Will test the following Python modules: %s", [x.name for x in modules_to_test]) - - task_queue = Queue.PriorityQueue() - for python_exec in python_execs: - LOGGER.debug("%s version is: %s", python_exec, subprocess_check_output( - [python_exec, "--version"], stderr=subprocess.STDOUT, universal_newlines=True).strip()) - for module in modules_to_test: - for test_goal in module.python_test_goals: - if test_goal in ('nn.layer'): - priority = 0 - else: - priority = 100 - task_queue.put((priority, (python_exec, test_goal))) - - def process_queue(task_queue): - while True: - try: - (priority, (python_exec, test_goal)) = task_queue.get_nowait() - except Queue.Empty: - break - try: - run_individual_python_test(test_goal, python_exec) - finally: - task_queue.task_done() - - start_time = time.time() - for _ in range(opts.parallelism): - worker = Thread(target=process_queue, args=(task_queue,)) - worker.daemon = True - worker.start() - try: - task_queue.join() - except (KeyboardInterrupt, SystemExit): - print_red("Exiting due to interrupt") - sys.exit(-1) - total_duration = time.time() - start_time - LOGGER.info("Tests passed in %i seconds", total_duration) - - -if __name__ == "__main__": - main() From 3e29a97c710b714c244ba58fb6e351781fa8b84c Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Thu, 16 Nov 2017 00:56:52 -0600 Subject: [PATCH 336/823] fix py3 syntax (#1888) --- keras1/converter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/keras1/converter.py b/keras1/converter.py index 90f65991249..f4f272fc177 100644 --- a/keras1/converter.py +++ b/keras1/converter.py @@ -721,8 +721,8 @@ def create_batchnormalization(self, klayer, kclayer): # init gamma and beta # TODO: replace this with to_bigdl_init in the future - gamma = self.get_value_from_init(klayer.gamma_init.func_name, (n_input_channel,)) - beta = self.get_value_from_init(klayer.beta_init.func_name, (n_input_channel,)) + gamma = self.get_value_from_init(klayer.gamma_init.__name__, (n_input_channel,)) + beta = self.get_value_from_init(klayer.beta_init.__name__, (n_input_channel,)) blayer = BLayer.SpatialBatchNormalization( n_output=n_input_channel, From 38d90b86d8a2f2e36bd6665a20c3b161c4ce102c Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Thu, 16 Nov 2017 00:56:52 -0600 Subject: [PATCH 337/823] fix py3 syntax (#1888) --- keras1/converter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/keras1/converter.py b/keras1/converter.py index 90f65991249..f4f272fc177 100644 --- a/keras1/converter.py +++ b/keras1/converter.py @@ -721,8 +721,8 @@ def create_batchnormalization(self, klayer, kclayer): # init gamma and beta # TODO: replace this with to_bigdl_init in the future - gamma = self.get_value_from_init(klayer.gamma_init.func_name, (n_input_channel,)) - beta = self.get_value_from_init(klayer.beta_init.func_name, (n_input_channel,)) + gamma = self.get_value_from_init(klayer.gamma_init.__name__, (n_input_channel,)) + beta = self.get_value_from_init(klayer.beta_init.__name__, (n_input_channel,)) blayer = BLayer.SpatialBatchNormalization( n_output=n_input_channel, From 882255e1f12a87b77ce7c9f638e35dc6aeb06b75 Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Thu, 16 Nov 2017 07:56:02 -0600 Subject: [PATCH 338/823] Rename keras1 to keras (#1893) * rename keras1 to keras * rename test * fix py35 --- .../dllib/bigdlkeras/layers}/__init__.py | 0 .../dllib/src/bigdl/dllib/keras}/converter.py | 52 +++++++++---------- python/dllib/src/bigdl/dllib/nn/layer.py | 2 +- 3 files changed, 27 insertions(+), 27 deletions(-) rename {keras1 => python/dllib/src/bigdl/dllib/bigdlkeras/layers}/__init__.py (100%) rename {keras1 => python/dllib/src/bigdl/dllib/keras}/converter.py (96%) diff --git a/keras1/__init__.py b/python/dllib/src/bigdl/dllib/bigdlkeras/layers/__init__.py similarity index 100% rename from keras1/__init__.py rename to python/dllib/src/bigdl/dllib/bigdlkeras/layers/__init__.py diff --git a/keras1/converter.py b/python/dllib/src/bigdl/dllib/keras/converter.py similarity index 96% rename from keras1/converter.py rename to python/dllib/src/bigdl/dllib/keras/converter.py index f4f272fc177..40baa27813c 100644 --- a/keras1/converter.py +++ b/python/dllib/src/bigdl/dllib/keras/converter.py @@ -364,7 +364,7 @@ def create_dense(self, klayer, kclayer): # What we need to use is the input index, not node index, not tensor index. input_shape = klayer.get_input_shape_at(0) blayer = BLayer.Linear( - input_size=input_shape[1], + input_size=int(input_shape[1]), output_size=config["output_dim"], with_bias=config["bias"], wRegularizer=self.to_bigdl_reg(config["W_regularizer"]), @@ -378,7 +378,7 @@ def create_timedistributeddense(self, klayer, kclayer): print(input_shape) print(config["output_dim"]) blayer = BLayer.TimeDistributed(BLayer.Linear( - input_size=input_shape[2], + input_size=int(input_shape[2]), output_size=config["output_dim"], with_bias=config["bias"], wRegularizer=self.to_bigdl_reg(config["W_regularizer"]), @@ -389,7 +389,7 @@ def create_timedistributeddense(self, klayer, kclayer): def create_embedding(self, klayer, kclayer): config = kclayer["config"] input_shape = klayer.get_input_shape_at(0) # batch, seq_len - seq_len = input_shape[1] + seq_len = int(input_shape[1]) if klayer.input_length and klayer.input_length != seq_len: raise Exception( "The input_length doesn't match: %s vs %s" % (seq_len, klayer.input_length)) @@ -423,7 +423,7 @@ def create_dropout(self, klayer, kclayer): def create_flatten(self, klayer, kclayer): self.__check_is_share_weights(kclayer) input_shape = klayer.input_shape - blayer = BLayer.Reshape([np.prod(input_shape[1:])], None) + blayer = BLayer.Reshape([int(np.prod(input_shape[1:]))], None) return blayer def create_reshape(self, klayer, kclayer): @@ -642,7 +642,7 @@ def create_simplernn(self, klayer, kclayer): activation = self.to_bigdl_activation(config["activation"], "%s_%s" % (config["name"], config["activation"])) # TODO: throw exception for dropout - rnn = BLayer.RnnCell(input_size=input_shape[2], + rnn = BLayer.RnnCell(input_size=int(input_shape[2]), hidden_size=klayer.output_dim, activation=activation, isInputWithBias=False, @@ -666,7 +666,7 @@ def create_lstm(self, klayer, kclayer): if not isinstance(inner_activation, BLayer.Sigmoid): raise Exception("For inner_activation, only `sigmond` is supported for now.") # TODO: throw exception for dropout - lstm = BLayer.LSTM(input_size=input_shape[2], + lstm = BLayer.LSTM(input_size=int(input_shape[2]), hidden_size=klayer.output_dim, p=klayer.dropout_W, wRegularizer=self.to_bigdl_reg(config["W_regularizer"]), @@ -689,7 +689,7 @@ def create_gru(self, klayer, kclayer): if not isinstance(inner_activation, BLayer.Sigmoid): raise Exception("For inner_activation, only `sigmond` is supported for now.") # TODO: throw exception for dropout - gru = BLayer.GRU(input_size=input_shape[2], + gru = BLayer.GRU(input_size=int(input_shape[2]), hidden_size=klayer.output_dim, p=klayer.dropout_W, wRegularizer=self.to_bigdl_reg(config["W_regularizer"]), @@ -717,7 +717,7 @@ def create_batchnormalization(self, klayer, kclayer): raise Exception("We don't support beta_regularizer for now") input_shape = klayer.get_input_shape_at(0) - n_input_channel = input_shape[klayer.axis] # default is -1 which is channel-last + n_input_channel = int(input_shape[klayer.axis]) # default is -1 which is channel-last # init gamma and beta # TODO: replace this with to_bigdl_init in the future @@ -784,11 +784,11 @@ def create_convolution1d(self, klayer, kclayer): config = kclayer["config"] input_shape = klayer.get_input_shape_at(0) # batch, steps, dim, batch is None here, so you cannot use it directly. - stack_size = input_shape[2] + stack_size = int(input_shape[2]) bpadW, bpadH = self.to_bigdl_2d_padding(klayer.border_mode) seq = BLayer.Sequential() - seq.add(BLayer.Reshape([input_shape[1], 1, input_shape[2]], True)) + seq.add(BLayer.Reshape([int(input_shape[1]), 1, int(input_shape[2])], True)) blayer = BLayer.SpatialConvolution( n_input_plane = stack_size, n_output_plane = klayer.nb_filter, @@ -819,9 +819,9 @@ def create_convolution2d(self, klayer, kclayer): input_shape = klayer.get_input_shape_at(0) if bigdl_order == "NCHW": - stack_size = input_shape[1] + stack_size = int(input_shape[1]) elif bigdl_order == "NHWC": - stack_size = input_shape[3] + stack_size = int(input_shape[3]) bpadW, bpadH = self.to_bigdl_2d_padding(klayer.border_mode) blayer = BLayer.SpatialConvolution( @@ -885,11 +885,11 @@ def create_globalmaxpooling2d(self, klayer, kclayer): bigdl_order = self.get_bdim_order(kclayer) input_shape = klayer.get_input_shape_at(0) if bigdl_order == "NCHW": - b_kw = input_shape[3] - b_kh = input_shape[2] + b_kw = int(input_shape[3]) + b_kh = int(input_shape[2]) else: - b_kw = input_shape[2] - b_kh = input_shape[1] + b_kw = int(input_shape[2]) + b_kh = int(input_shape[1]) seq = BLayer.Sequential() blayer = BLayer.SpatialMaxPooling( @@ -915,10 +915,10 @@ def create_globalmaxpooling2d(self, klayer, kclayer): def create_globalmaxpooling1d(self, klayer, kclayer): input_shape = klayer.get_input_shape_at(0) # batch, step, dim b_kw = 1 - b_kh = input_shape[1] + b_kh = int(input_shape[1]) seq = BLayer.Sequential() - seq.add(BLayer.View([input_shape[1], 1, input_shape[2]], num_input_dims=2)) + seq.add(BLayer.View([int(input_shape[1]), 1, int(input_shape[2])], num_input_dims=2)) blayer = BLayer.SpatialMaxPooling( kw=b_kw, kh=b_kh, @@ -938,10 +938,10 @@ def create_globalmaxpooling1d(self, klayer, kclayer): def create_globalaveragepooling1d(self, klayer, kclayer): input_shape = klayer.get_input_shape_at(0) # batch, step, dim b_kw = 1 - b_kh = input_shape[1] + b_kh = int(input_shape[1]) seq = BLayer.Sequential() - seq.add(BLayer.View([input_shape[1], 1, input_shape[2]], num_input_dims=2)) + seq.add(BLayer.View([int(input_shape[1]), 1, int(input_shape[2])], num_input_dims=2)) blayer = BLayer.SpatialAveragePooling( kw=b_kw, kh=b_kh, @@ -967,7 +967,7 @@ def create_maxpooling1d(self, klayer, kclayer): bpadW, bpadH = self.to_bigdl_2d_padding(klayer.border_mode) seq = BLayer.Sequential() - seq.add(BLayer.View([1, input_shape[1], input_shape[2]], num_input_dims=3)) + seq.add(BLayer.View([1, int(input_shape[1]), int(input_shape[2])], num_input_dims=3)) blayer = BLayer.SpatialMaxPooling( kw=klayer.pool_length, kh=1, @@ -987,7 +987,7 @@ def create_averagepooling1d(self, klayer, kclayer): bpadW, bpadH = self.to_bigdl_2d_padding(klayer.border_mode) seq = BLayer.Sequential() - seq.add(BLayer.View([1, input_shape[1], input_shape[2]], num_input_dims=3)) + seq.add(BLayer.View([1, int(input_shape[1]), int(input_shape[2])], num_input_dims=3)) blayer = BLayer.SpatialAveragePooling( kw=klayer.pool_length, kh=1, @@ -1009,11 +1009,11 @@ def create_globalaveragepooling2d(self, klayer, kclayer): bigdl_order = self.get_bdim_order(kclayer) input_shape = klayer.get_input_shape_at(0) if bigdl_order == "NCHW": - b_kw = input_shape[3] - b_kh = input_shape[2] + b_kw = int(input_shape[3]) + b_kh = int(input_shape[2]) else: - b_kw = input_shape[2] - b_kh = input_shape[1] + b_kw = int(input_shape[2]) + b_kh = int(input_shape[1]) seq = BLayer.Sequential() blayer = BLayer.SpatialAveragePooling( diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 4e8ba20864a..1e1a31169d1 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -699,7 +699,7 @@ def load_keras(def_path, weights_path=None, by_name=False): :param weights_path: The HDF5 path containing the pre-trained keras model weights. :return: A pre-trained model. """ - from bigdl.keras1.converter import DefinitionLoader, WeightLoader + from bigdl.keras.converter import DefinitionLoader, WeightLoader if weights_path: return WeightLoader.load_weights_from_json_hdf5(def_path, weights_path, by_name=by_name) else: From 926d5b47838f83788df34033446ed0feef0e6c12 Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Thu, 16 Nov 2017 07:56:02 -0600 Subject: [PATCH 339/823] Rename keras1 to keras (#1893) * rename keras1 to keras * rename test * fix py35 --- .../dllib/bigdlkeras/layers}/__init__.py | 0 .../dllib/src/bigdl/dllib/keras}/converter.py | 52 +++++++++---------- python/dllib/src/bigdl/dllib/nn/layer.py | 2 +- 3 files changed, 27 insertions(+), 27 deletions(-) rename {keras1 => python/dllib/src/bigdl/dllib/bigdlkeras/layers}/__init__.py (100%) rename {keras1 => python/dllib/src/bigdl/dllib/keras}/converter.py (96%) diff --git a/keras1/__init__.py b/python/dllib/src/bigdl/dllib/bigdlkeras/layers/__init__.py similarity index 100% rename from keras1/__init__.py rename to python/dllib/src/bigdl/dllib/bigdlkeras/layers/__init__.py diff --git a/keras1/converter.py b/python/dllib/src/bigdl/dllib/keras/converter.py similarity index 96% rename from keras1/converter.py rename to python/dllib/src/bigdl/dllib/keras/converter.py index f4f272fc177..40baa27813c 100644 --- a/keras1/converter.py +++ b/python/dllib/src/bigdl/dllib/keras/converter.py @@ -364,7 +364,7 @@ def create_dense(self, klayer, kclayer): # What we need to use is the input index, not node index, not tensor index. input_shape = klayer.get_input_shape_at(0) blayer = BLayer.Linear( - input_size=input_shape[1], + input_size=int(input_shape[1]), output_size=config["output_dim"], with_bias=config["bias"], wRegularizer=self.to_bigdl_reg(config["W_regularizer"]), @@ -378,7 +378,7 @@ def create_timedistributeddense(self, klayer, kclayer): print(input_shape) print(config["output_dim"]) blayer = BLayer.TimeDistributed(BLayer.Linear( - input_size=input_shape[2], + input_size=int(input_shape[2]), output_size=config["output_dim"], with_bias=config["bias"], wRegularizer=self.to_bigdl_reg(config["W_regularizer"]), @@ -389,7 +389,7 @@ def create_timedistributeddense(self, klayer, kclayer): def create_embedding(self, klayer, kclayer): config = kclayer["config"] input_shape = klayer.get_input_shape_at(0) # batch, seq_len - seq_len = input_shape[1] + seq_len = int(input_shape[1]) if klayer.input_length and klayer.input_length != seq_len: raise Exception( "The input_length doesn't match: %s vs %s" % (seq_len, klayer.input_length)) @@ -423,7 +423,7 @@ def create_dropout(self, klayer, kclayer): def create_flatten(self, klayer, kclayer): self.__check_is_share_weights(kclayer) input_shape = klayer.input_shape - blayer = BLayer.Reshape([np.prod(input_shape[1:])], None) + blayer = BLayer.Reshape([int(np.prod(input_shape[1:]))], None) return blayer def create_reshape(self, klayer, kclayer): @@ -642,7 +642,7 @@ def create_simplernn(self, klayer, kclayer): activation = self.to_bigdl_activation(config["activation"], "%s_%s" % (config["name"], config["activation"])) # TODO: throw exception for dropout - rnn = BLayer.RnnCell(input_size=input_shape[2], + rnn = BLayer.RnnCell(input_size=int(input_shape[2]), hidden_size=klayer.output_dim, activation=activation, isInputWithBias=False, @@ -666,7 +666,7 @@ def create_lstm(self, klayer, kclayer): if not isinstance(inner_activation, BLayer.Sigmoid): raise Exception("For inner_activation, only `sigmond` is supported for now.") # TODO: throw exception for dropout - lstm = BLayer.LSTM(input_size=input_shape[2], + lstm = BLayer.LSTM(input_size=int(input_shape[2]), hidden_size=klayer.output_dim, p=klayer.dropout_W, wRegularizer=self.to_bigdl_reg(config["W_regularizer"]), @@ -689,7 +689,7 @@ def create_gru(self, klayer, kclayer): if not isinstance(inner_activation, BLayer.Sigmoid): raise Exception("For inner_activation, only `sigmond` is supported for now.") # TODO: throw exception for dropout - gru = BLayer.GRU(input_size=input_shape[2], + gru = BLayer.GRU(input_size=int(input_shape[2]), hidden_size=klayer.output_dim, p=klayer.dropout_W, wRegularizer=self.to_bigdl_reg(config["W_regularizer"]), @@ -717,7 +717,7 @@ def create_batchnormalization(self, klayer, kclayer): raise Exception("We don't support beta_regularizer for now") input_shape = klayer.get_input_shape_at(0) - n_input_channel = input_shape[klayer.axis] # default is -1 which is channel-last + n_input_channel = int(input_shape[klayer.axis]) # default is -1 which is channel-last # init gamma and beta # TODO: replace this with to_bigdl_init in the future @@ -784,11 +784,11 @@ def create_convolution1d(self, klayer, kclayer): config = kclayer["config"] input_shape = klayer.get_input_shape_at(0) # batch, steps, dim, batch is None here, so you cannot use it directly. - stack_size = input_shape[2] + stack_size = int(input_shape[2]) bpadW, bpadH = self.to_bigdl_2d_padding(klayer.border_mode) seq = BLayer.Sequential() - seq.add(BLayer.Reshape([input_shape[1], 1, input_shape[2]], True)) + seq.add(BLayer.Reshape([int(input_shape[1]), 1, int(input_shape[2])], True)) blayer = BLayer.SpatialConvolution( n_input_plane = stack_size, n_output_plane = klayer.nb_filter, @@ -819,9 +819,9 @@ def create_convolution2d(self, klayer, kclayer): input_shape = klayer.get_input_shape_at(0) if bigdl_order == "NCHW": - stack_size = input_shape[1] + stack_size = int(input_shape[1]) elif bigdl_order == "NHWC": - stack_size = input_shape[3] + stack_size = int(input_shape[3]) bpadW, bpadH = self.to_bigdl_2d_padding(klayer.border_mode) blayer = BLayer.SpatialConvolution( @@ -885,11 +885,11 @@ def create_globalmaxpooling2d(self, klayer, kclayer): bigdl_order = self.get_bdim_order(kclayer) input_shape = klayer.get_input_shape_at(0) if bigdl_order == "NCHW": - b_kw = input_shape[3] - b_kh = input_shape[2] + b_kw = int(input_shape[3]) + b_kh = int(input_shape[2]) else: - b_kw = input_shape[2] - b_kh = input_shape[1] + b_kw = int(input_shape[2]) + b_kh = int(input_shape[1]) seq = BLayer.Sequential() blayer = BLayer.SpatialMaxPooling( @@ -915,10 +915,10 @@ def create_globalmaxpooling2d(self, klayer, kclayer): def create_globalmaxpooling1d(self, klayer, kclayer): input_shape = klayer.get_input_shape_at(0) # batch, step, dim b_kw = 1 - b_kh = input_shape[1] + b_kh = int(input_shape[1]) seq = BLayer.Sequential() - seq.add(BLayer.View([input_shape[1], 1, input_shape[2]], num_input_dims=2)) + seq.add(BLayer.View([int(input_shape[1]), 1, int(input_shape[2])], num_input_dims=2)) blayer = BLayer.SpatialMaxPooling( kw=b_kw, kh=b_kh, @@ -938,10 +938,10 @@ def create_globalmaxpooling1d(self, klayer, kclayer): def create_globalaveragepooling1d(self, klayer, kclayer): input_shape = klayer.get_input_shape_at(0) # batch, step, dim b_kw = 1 - b_kh = input_shape[1] + b_kh = int(input_shape[1]) seq = BLayer.Sequential() - seq.add(BLayer.View([input_shape[1], 1, input_shape[2]], num_input_dims=2)) + seq.add(BLayer.View([int(input_shape[1]), 1, int(input_shape[2])], num_input_dims=2)) blayer = BLayer.SpatialAveragePooling( kw=b_kw, kh=b_kh, @@ -967,7 +967,7 @@ def create_maxpooling1d(self, klayer, kclayer): bpadW, bpadH = self.to_bigdl_2d_padding(klayer.border_mode) seq = BLayer.Sequential() - seq.add(BLayer.View([1, input_shape[1], input_shape[2]], num_input_dims=3)) + seq.add(BLayer.View([1, int(input_shape[1]), int(input_shape[2])], num_input_dims=3)) blayer = BLayer.SpatialMaxPooling( kw=klayer.pool_length, kh=1, @@ -987,7 +987,7 @@ def create_averagepooling1d(self, klayer, kclayer): bpadW, bpadH = self.to_bigdl_2d_padding(klayer.border_mode) seq = BLayer.Sequential() - seq.add(BLayer.View([1, input_shape[1], input_shape[2]], num_input_dims=3)) + seq.add(BLayer.View([1, int(input_shape[1]), int(input_shape[2])], num_input_dims=3)) blayer = BLayer.SpatialAveragePooling( kw=klayer.pool_length, kh=1, @@ -1009,11 +1009,11 @@ def create_globalaveragepooling2d(self, klayer, kclayer): bigdl_order = self.get_bdim_order(kclayer) input_shape = klayer.get_input_shape_at(0) if bigdl_order == "NCHW": - b_kw = input_shape[3] - b_kh = input_shape[2] + b_kw = int(input_shape[3]) + b_kh = int(input_shape[2]) else: - b_kw = input_shape[2] - b_kh = input_shape[1] + b_kw = int(input_shape[2]) + b_kh = int(input_shape[1]) seq = BLayer.Sequential() blayer = BLayer.SpatialAveragePooling( diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 4e8ba20864a..1e1a31169d1 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -699,7 +699,7 @@ def load_keras(def_path, weights_path=None, by_name=False): :param weights_path: The HDF5 path containing the pre-trained keras model weights. :return: A pre-trained model. """ - from bigdl.keras1.converter import DefinitionLoader, WeightLoader + from bigdl.keras.converter import DefinitionLoader, WeightLoader if weights_path: return WeightLoader.load_weights_from_json_hdf5(def_path, weights_path, by_name=by_name) else: From bfb4ad70b521c31fe3018a17b9ee58f87173c24e Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Thu, 16 Nov 2017 07:56:02 -0600 Subject: [PATCH 340/823] Rename keras1 to keras (#1893) * rename keras1 to keras * rename test * fix py35 --- .../dllib/test/bigdl/{keras1 => keras}/test_application.py | 2 +- python/dllib/test/bigdl/{keras1 => keras}/test_layer.py | 2 +- .../dllib/test/bigdl/{keras1 => keras}/test_load_model.py | 4 ++-- python/dllib/test/bigdl/test_utils.py | 4 ++-- python/dllib/test/dev/run-keras.sh | 4 ++-- python/dllib/test/dev/run-tests | 6 +++--- 6 files changed, 11 insertions(+), 11 deletions(-) rename python/dllib/test/bigdl/{keras1 => keras}/test_application.py (99%) rename python/dllib/test/bigdl/{keras1 => keras}/test_layer.py (99%) rename python/dllib/test/bigdl/{keras1 => keras}/test_load_model.py (96%) diff --git a/python/dllib/test/bigdl/keras1/test_application.py b/python/dllib/test/bigdl/keras/test_application.py similarity index 99% rename from python/dllib/test/bigdl/keras1/test_application.py rename to python/dllib/test/bigdl/keras/test_application.py index 37eacea8be5..ce60c7838fc 100644 --- a/python/dllib/test/bigdl/keras1/test_application.py +++ b/python/dllib/test/bigdl/keras/test_application.py @@ -20,7 +20,7 @@ np.random.seed(1337) # for reproducibility import pytest from keras.applications import * -from bigdl.keras1.converter import * +from bigdl.keras.converter import * from test.bigdl.test_utils import BigDLTestCase, TestModels diff --git a/python/dllib/test/bigdl/keras1/test_layer.py b/python/dllib/test/bigdl/keras/test_layer.py similarity index 99% rename from python/dllib/test/bigdl/keras1/test_layer.py rename to python/dllib/test/bigdl/keras/test_layer.py index a766c203dc9..bdc312f1d46 100644 --- a/python/dllib/test/bigdl/keras1/test_layer.py +++ b/python/dllib/test/bigdl/keras/test_layer.py @@ -22,7 +22,7 @@ from keras.layers.core import * from keras.layers.convolutional import * from keras.layers import Dense, Input -from bigdl.keras1.converter import * +from bigdl.keras.converter import * from test.bigdl.test_utils import BigDLTestCase diff --git a/python/dllib/test/bigdl/keras1/test_load_model.py b/python/dllib/test/bigdl/keras/test_load_model.py similarity index 96% rename from python/dllib/test/bigdl/keras1/test_load_model.py rename to python/dllib/test/bigdl/keras/test_load_model.py index a93e675edd7..6b9d72d0c61 100644 --- a/python/dllib/test/bigdl/keras1/test_load_model.py +++ b/python/dllib/test/bigdl/keras/test_load_model.py @@ -20,8 +20,8 @@ from numpy.testing import assert_allclose import bigdl.nn.layer as BLayer -from bigdl.keras1.converter import WeightLoader -from bigdl.keras1.converter import DefinitionLoader +from bigdl.keras.converter import WeightLoader +from bigdl.keras.converter import DefinitionLoader np.random.seed(1337) # for reproducibility from test.bigdl.test_utils import BigDLTestCase, TestModels diff --git a/python/dllib/test/bigdl/test_utils.py b/python/dllib/test/bigdl/test_utils.py index bceff4b1fb4..5a0ade9b422 100644 --- a/python/dllib/test/bigdl/test_utils.py +++ b/python/dllib/test/bigdl/test_utils.py @@ -16,7 +16,7 @@ from __future__ import print_function import numpy as np from keras.models import Sequential, Model -from bigdl.keras1.converter import WeightLoader, WeightsConverter +from bigdl.keras.converter import WeightLoader, WeightsConverter np.random.seed(1337) # for reproducibility from keras.layers.core import * from keras.layers.convolutional import * @@ -24,7 +24,7 @@ from keras.models import model_from_json from keras.optimizers import RMSprop from bigdl.util.common import create_tmp_path -from bigdl.keras1.converter import DefinitionLoader +from bigdl.keras.converter import DefinitionLoader import numpy as np from unittest import TestCase import keras diff --git a/python/dllib/test/dev/run-keras.sh b/python/dllib/test/dev/run-keras.sh index 2e7b2cf9aab..de5c6d1401f 100755 --- a/python/dllib/test/dev/run-keras.sh +++ b/python/dllib/test/dev/run-keras.sh @@ -32,8 +32,8 @@ echo "${cyan}Using python version: $p${reset}" export PYTHON_EXECUTABLE=$p export PYSPARK_PYTHON=$p export PYSPARK_DRIVER_PYTHON=$p -$1 -m pytest -v ../../../pyspark/test/bigdl/keras1 \ - --ignore=../../../pyspark/test/bigdl/keras1/test_application.py +$1 -m pytest -v ../../../pyspark/test/bigdl/keras \ + --ignore=../../../pyspark/test/bigdl/keras/test_application.py exit_status=$? if [ $exit_status -ne 0 ]; diff --git a/python/dllib/test/dev/run-tests b/python/dllib/test/dev/run-tests index 1e5aceb6d62..41374c43e46 100755 --- a/python/dllib/test/dev/run-tests +++ b/python/dllib/test/dev/run-tests @@ -31,12 +31,12 @@ do $p -m pytest -v --doctest-modules ../../../pyspark/bigdl \ --ignore=../../../pyspark/bigdl/dataset/ \ --ignore=../../../pyspark/bigdl/util/tf_utils.py \ - --ignore=../../../pyspark/bigdl/keras1 \ - --ignore=../../../pyspark/test/bigdl/keras1/test_application.py \ + --ignore=../../../pyspark/bigdl/keras \ + --ignore=../../../pyspark/test/bigdl/keras/test_application.py \ --ignore=../../../pyspark/bigdl/models/ && \ $p -m pytest -v -n 4 ../../../pyspark/test/ \ --ignore=../../../pyspark/test/bigdl/caffe/ \ - --ignore=../../../pyspark/test/bigdl/keras1/ \ + --ignore=../../../pyspark/test/bigdl/keras/ \ --ignore=../../../pyspark/test/bigdl/test_utils.py exit_status=$? From 6a4ba28c15a0bcf670f8a677049b81e40ed2e6c3 Mon Sep 17 00:00:00 2001 From: Jerry Wu Date: Fri, 17 Nov 2017 00:09:00 -0600 Subject: [PATCH 341/823] Keras support - add Hard-sigmoid layer (#1897) * add hard sigmoid * add python * add python api * fix typo * add doc --- python/dllib/src/bigdl/dllib/nn/layer.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 1e1a31169d1..dcb4ed4f2ea 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -4399,6 +4399,20 @@ class GaussianSampler(Layer): def __init__(self, bigdl_type="float"): super(GaussianSampler, self).__init__(None, bigdl_type) +class HardSigmoid(Layer): + """ + Apply Hard-sigmoid function +``` + | 0, if x < -2.5 + f(x) = | 1, if x > 2.5 + | 0.2 * x + 0.5, otherwise +``` + >>> hardSigmoid = HardSigmoid() + creating: createHardSigmoid + """ + def __init__(self, bigdl_type="float"): + super(HardSigmoid, self).__init__(None, bigdl_type) + def _test(): import doctest from pyspark import SparkContext From 9ef727c83fee7a76a4bacb0ed9b7a33ca5932fe7 Mon Sep 17 00:00:00 2001 From: Jerry Wu Date: Fri, 17 Nov 2017 00:09:00 -0600 Subject: [PATCH 342/823] Keras support - add Hard-sigmoid layer (#1897) * add hard sigmoid * add python * add python api * fix typo * add doc --- python/dllib/src/bigdl/dllib/nn/layer.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 1e1a31169d1..dcb4ed4f2ea 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -4399,6 +4399,20 @@ class GaussianSampler(Layer): def __init__(self, bigdl_type="float"): super(GaussianSampler, self).__init__(None, bigdl_type) +class HardSigmoid(Layer): + """ + Apply Hard-sigmoid function +``` + | 0, if x < -2.5 + f(x) = | 1, if x > 2.5 + | 0.2 * x + 0.5, otherwise +``` + >>> hardSigmoid = HardSigmoid() + creating: createHardSigmoid + """ + def __init__(self, bigdl_type="float"): + super(HardSigmoid, self).__init__(None, bigdl_type) + def _test(): import doctest from pyspark import SparkContext From def24b1bd599f775fbcc066a77e2370c6ffeb105 Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Mon, 20 Nov 2017 04:23:48 -0600 Subject: [PATCH 343/823] Ignore keras test if environment is not ready and add test method for loss (#1899) * add check method for loss * clean * clean * comments * checking * fix * rebase * rebase --- python/dllib/test/bigdl/keras/test_layer.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/dllib/test/bigdl/keras/test_layer.py b/python/dllib/test/bigdl/keras/test_layer.py index bdc312f1d46..6035c587211 100644 --- a/python/dllib/test/bigdl/keras/test_layer.py +++ b/python/dllib/test/bigdl/keras/test_layer.py @@ -24,6 +24,7 @@ from keras.layers import Dense, Input from bigdl.keras.converter import * from test.bigdl.test_utils import BigDLTestCase +from keras.metrics import * class TestLayer(BigDLTestCase): From ee1adefe2270a14e569c8fc6dd71dd429ad17b4b Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Tue, 21 Nov 2017 11:09:43 +0800 Subject: [PATCH 344/823] Add more keras layers and test (#1891) * add, update and fix * add hard_sigmoid * fix python3.5 lambda in permute * fix syntax for python3.5 * fix pooling and conv layers * fix syntax * fix permute for py35 --- .../dllib/src/bigdl/dllib/keras/converter.py | 496 +++++++++++++++--- python/dllib/src/bigdl/dllib/nn/layer.py | 71 +++ 2 files changed, 486 insertions(+), 81 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/keras/converter.py b/python/dllib/src/bigdl/dllib/keras/converter.py index 40baa27813c..ab35e86a72f 100644 --- a/python/dllib/src/bigdl/dllib/keras/converter.py +++ b/python/dllib/src/bigdl/dllib/keras/converter.py @@ -2,6 +2,7 @@ import bigdl.nn.initialization_method as BInit import numpy as np import bigdl.nn.layer as BLayer +from bigdl.optim.optimizer import L1L2Regularizer as BRegularizer import bigdl.optim.optimizer as boptimizer import bigdl.nn.criterion as bcriterion import bigdl.util.common as bcommon @@ -175,6 +176,23 @@ def convert_batchnormalization(weights): beta = weights[1] return [gamma, beta] + @staticmethod + def convert_atrousconvolution2d(weights): + return weights + + @staticmethod + def convert_atrousconvolution1d(weights): + return [np.transpose(weights[0], (3, 2, 0, 1)), weights[1]] + + @staticmethod + def convert_deconvolution2d(weights): + w = np.transpose(weights[0], (1, 0, 2, 3)) + weight = np.expand_dims(w, 0) + if len(weights) > 1: + return [weight, weights[1]] + else: + return [weight] + @staticmethod def convert_convolution2d(weights): weight = np.expand_dims(weights[0], 0) # bigdl has a leading dim with value 1 @@ -187,6 +205,10 @@ def convert_convolution2d(weights): def convert_convolution1d(weights): return WeightsConverter.convert_convolution2d(weights) + @staticmethod + def convert_convolution3d(weights): + return weights + @staticmethod def convert_embedding(weights): return weights @@ -202,10 +224,17 @@ def convert_lstm(weights): w3 = np.concatenate((weights[1].T, weights[4].T, weights[7].T, weights[10].T)) return [w1, w2, w3] + @staticmethod + def convert_convlstm2d(weights): + return [np.expand_dims(weights[6], 0), weights[8], np.expand_dims(weights[7], 0), + np.expand_dims(weights[0], 0), weights[2], np.expand_dims(weights[1], 0), + np.expand_dims(weights[3], 0), weights[5], np.expand_dims(weights[4], 0), + np.expand_dims(weights[9], 0), weights[11], np.expand_dims(weights[10], 0)] + @staticmethod def convert_gru(weights): w1 = np.concatenate((weights[3].T, weights[0].T, weights[6].T)) - w2 = np.concatenate((weights[2], weights[5], weights[8])) + w2 = np.concatenate((weights[5], weights[2], weights[8])) w3 = np.concatenate((weights[4].T, weights[1].T)) w4 = weights[7].T return [w1, w2, w3, w4] @@ -375,8 +404,6 @@ def create_dense(self, klayer, kclayer): def create_timedistributeddense(self, klayer, kclayer): config = kclayer["config"] input_shape = klayer.get_input_shape_at(0) - print(input_shape) - print(config["output_dim"]) blayer = BLayer.TimeDistributed(BLayer.Linear( input_size=int(input_shape[2]), output_size=config["output_dim"], @@ -426,6 +453,45 @@ def create_flatten(self, klayer, kclayer): blayer = BLayer.Reshape([int(np.prod(input_shape[1:]))], None) return blayer + def create_permute(self, klayer, kclayer): + swaps = self.__perm_to_pair(list(klayer.dims)) + swaps.reverse() + swaps = map(lambda pair: (pair[0]+1, pair[1]+1), swaps) + return BLayer.Transpose(list(swaps)) + + def __perm_to_pair(self, perm): + # perm: a list as a permutation of [1..n], eg [3, 1, 2] for n=3. + # return a list of tuples that needs to be swapped to obtain the input `perm`. + pairs = [] + + def sort(arr, low, high): + i = low + j = high + pivot = arr[low + int((high - low) / 2)] + while i <= j: + while arr[i] < pivot: + i += 1 + while arr[j] > pivot: + j -= 1 + if i <= j: + exchangeNumbers(arr, i, j) + i += 1 + j -= 1 + if low < j: + sort(arr, low, j) + if i < high: + sort(arr, i, high) + + def exchangeNumbers(arr, i, j): + temp = arr[i] + arr[i] = arr[j] + arr[j] = temp + pairs.append((i + 1, j + 1)) + + sort(perm, 0, len(perm) - 1) + + return list(filter(lambda pair: pair[0] != pair[1], pairs)) + def create_reshape(self, klayer, kclayer): self.__check_is_share_weights(kclayer) blayer = BLayer.Reshape(klayer.target_shape, None) @@ -439,11 +505,12 @@ def create_repeatvector(self, klayer, kclayer): def create_merge(self, klayer, kclayer): self.__check_is_share_weights(kclayer) input_shape = klayer.get_input_shape_at(0) - #TODO: dot_axes, node_indices, tensor_indices not supported. if klayer.output_shape and not isinstance(klayer.output_shape, tuple): - raise Exception("Only output_shape=None or a shape tuple is supported for now.") + raise Exception("Only output_shape=None or a shape tuple is supported for now") + if klayer.node_indices: + unsupport_exp("node_indices") if klayer.output_mask: - raise Exception("Argument `output_mask` is not supported for now.") + unsupport_exp("output_mask") if klayer.mode == "concat": blayer = BLayer.JoinTable( dimension=klayer.concat_axis, @@ -459,14 +526,18 @@ def create_merge(self, klayer, kclayer): blayer = BLayer.CMaxTable(bigdl_type="float") elif klayer.mode == "dot": if len(input_shape[0]) >= 3: - raise Exception("Merge mode `dot` doesn't support 3D input or above for now.") - if input_shape[0][0] == None: - raise Exception("For merge mode `dot`, please specify `batch_input_shape`.") + raise Exception("For merge mode dot, 3D input or above is not supported for now.") + if klayer.dot_axes != [1, 1]: + raise Exception("For merge mode dot, only dot_axes=1 is supported for now.") model = BLayer.Sequential() blayer = model.add(BLayer.DotProduct(bigdl_type="float"))\ - .add(BLayer.Reshape([input_shape[0][0], 1])) - elif klayer.mode in ['ave', 'cos']: - raise Exception("Merge mode `%s` not supported for now" % klayer.mode) # TODO: whether to support cosine and average + .add(BLayer.Reshape([1], True)) + elif klayer.mode == "ave": + blayer = BLayer.CAveTable( + inplace=False, + bigdl_type="float") + elif klayer.mode in ['cos']: + raise Exception("Merge mode `%s` not supported for now" % klayer.mode) else: # invalid mode or lambda functions raise Exception("Invalid merge mode: `%s`. Lambda/function as merge mode is not supported for now." % klayer.mode) return blayer @@ -488,6 +559,8 @@ def create_leakyrelu(self, klayer, kclayer): def create_parametricsoftplus(self, klayer, kclayer): alpha = float(klayer.alpha_init) beta = float(klayer.beta_init) + if klayer.shared_axes != [None]: + unsupport_exp("shared_axes") if round(alpha * beta, 4) == 1.0: return BLayer.SoftPlus(beta=beta, bigdl_type="float") @@ -549,64 +622,65 @@ def __generate_zeropadding2d(self, dim1, dim2, n_input_dim, pad1, pad2, pad3, pa model.add(paddinglayer4) return model - # NB: zeropadding doesn't serialize dim_ording to jason file + # NB: zeropadding doesn't serialize dim_ording to json file def create_zeropadding2d(self, klayer, kclayer): padding = klayer.padding input_shape = klayer.get_input_shape_at(0) - dim1 = 1 - dim2 = 2 + dim = 1 if klayer.dim_ordering == "th": - dim1 = 2 - dim2 = 3 + dim = 2 if isinstance(padding, dict): # dictionary - return self.__generate_zeropadding2d(dim1, dim2, len(input_shape) - 1, + return self.__generate_zeropadding2d(dim, dim+1, len(input_shape) - 1, -padding.get('top_pad', 0), padding.get('bottom_pad', 0), -padding.get('left_pad', 0), padding.get('right_pad', 0)) else: # tuple of int padding = tuple(padding) if len(padding) == 2: - return self.__generate_zeropadding2d(dim1, dim2, len(input_shape) - 1, + return self.__generate_zeropadding2d(dim, dim+1, len(input_shape) - 1, -padding[0], padding[0], -padding[1], padding[1]) elif len(padding) == 4: - return self.__generate_zeropadding2d(dim1, dim2, len(input_shape) - 1, + return self.__generate_zeropadding2d(dim, dim+1, len(input_shape) - 1, -padding[0], padding[1], -padding[2], padding[3]) - # NB: zeropadding doesn't serialize dim_ording to jason file + # NB: zeropadding doesn't serialize dim_ording to json file def create_zeropadding3d(self, klayer, kclayer): padding = tuple(klayer.padding) input_shape = klayer.get_input_shape_at(0) + dim = 1 + if klayer.dim_ordering == "th": + dim = 2 model = BLayer.Sequential() - paddinglayer1 = BLayer.Padding(dim=2, + paddinglayer1 = BLayer.Padding(dim=dim, pad=-padding[0], n_input_dim=len(input_shape) - 1, value=0.0, n_index=1, bigdl_type="float") - paddinglayer2 = BLayer.Padding(dim=2, + paddinglayer2 = BLayer.Padding(dim=dim, pad=padding[0], n_input_dim=len(input_shape) - 1, value=0.0, n_index=1, bigdl_type="float") - paddinglayer3 = BLayer.Padding(dim=3, + paddinglayer3 = BLayer.Padding(dim=dim+1, pad=-padding[1], n_input_dim=len(input_shape) - 1, value=0.0, n_index=1, bigdl_type="float") - paddinglayer4 = BLayer.Padding(dim=3, + paddinglayer4 = BLayer.Padding(dim=dim+1, pad=padding[1], n_input_dim=len(input_shape) - 1, value=0.0, n_index=1, bigdl_type="float") - paddinglayer5 = BLayer.Padding(dim=4, + paddinglayer5 = BLayer.Padding(dim=dim+2, pad=-padding[2], n_input_dim=len(input_shape) - 1, value=0.0, n_index=1, bigdl_type="float") - paddinglayer6 = BLayer.Padding(dim=4, + paddinglayer6 = BLayer.Padding(dim=dim+2, pad=padding[2], n_input_dim=len(input_shape) - 1, value=0.0, @@ -641,7 +715,6 @@ def create_simplernn(self, klayer, kclayer): self.check_constraint_in_config(config) activation = self.to_bigdl_activation(config["activation"], "%s_%s" % (config["name"], config["activation"])) - # TODO: throw exception for dropout rnn = BLayer.RnnCell(input_size=int(input_shape[2]), hidden_size=klayer.output_dim, activation=activation, @@ -660,21 +733,60 @@ def create_lstm(self, klayer, kclayer): activation = self.to_bigdl_activation(config["activation"], "%s_%s" % (config["name"], config["activation"])) if not isinstance(activation, BLayer.Tanh): - raise Exception("For activation, only `tanh` is supported for now.") + raise Exception("For activation, only tanh is supported for now.") inner_activation = self.to_bigdl_activation(config["inner_activation"], "%s_%s" % (config["name"], config["inner_activation"])) if not isinstance(inner_activation, BLayer.Sigmoid): - raise Exception("For inner_activation, only `sigmond` is supported for now.") - # TODO: throw exception for dropout + raise Exception("For inner_activation, only sigmond is supported for now.") lstm = BLayer.LSTM(input_size=int(input_shape[2]), hidden_size=klayer.output_dim, - p=klayer.dropout_W, + p=0.0, wRegularizer=self.to_bigdl_reg(config["W_regularizer"]), uRegularizer=self.to_bigdl_reg(config["U_regularizer"]), bRegularizer=self.to_bigdl_reg(config["b_regularizer"]), bigdl_type="float") return self.__return_sequences(klayer.return_sequences, rec.add(lstm)) + def create_convlstm2d(self, klayer, kclayer): + rec = BLayer.Recurrent() + input_shape = klayer.get_input_shape_at(0) + config = kclayer["config"] + self.check_constraint_in_config(config) + activation = self.to_bigdl_activation(config["activation"], + "%s_%s" % (config["name"], config["activation"])) + if not isinstance(activation, BLayer.Tanh): + raise Exception("For activation, only tanh is supported for now.") + inner_activation = self.to_bigdl_activation(config["inner_activation"], + "%s_%s" % (config["name"], config["inner_activation"])) + if not isinstance(inner_activation, BLayer.Sigmoid): + raise Exception("For inner_activation, only sigmond is supported for now.") + + #TODO: border_mode = 'valid' + if config["border_mode"] != 'same': + raise Exception("Unsupported border_mode: valid") + + if config["nb_row"] != config["nb_col"]: + raise Exception("Only square kernel is supported for now. Please set nb_row=nb_col.") + if klayer.subsample[0] != klayer.subsample[1]: + raise Exception("Only equal stride is supported for now. " + "Please set subsample to be a tuple with equal values.") + + blayer = BLayer.ConvLSTMPeephole(input_size=int(input_shape[2]), + output_size=config["nb_filter"], + kernel_i=config["nb_col"], + kernel_c=config["nb_row"], + # NB: ConvLSTM doesn't serialize subsample to json file + stride=klayer.subsample[0], + padding=-1, + # NB: ConvLSTM doesn't serialize regularizers to json file + # wRegularizer=self.to_bigdl_reg(config["W_regularizer"]), + # uRegularizer=self.to_bigdl_reg(config["U_regularizer"]), + # bRegularizer=self.to_bigdl_reg(config["b_regularizer"]), + cRegularizer=None, + with_peephole=False, + bigdl_type="float") + return self.__return_sequences(klayer.return_sequences, rec.add(blayer)) + def create_gru(self, klayer, kclayer): rec = BLayer.Recurrent() input_shape = klayer.get_input_shape_at(0) @@ -688,10 +800,9 @@ def create_gru(self, klayer, kclayer): "%s_%s" % (config["name"], config["inner_activation"])) if not isinstance(inner_activation, BLayer.Sigmoid): raise Exception("For inner_activation, only `sigmond` is supported for now.") - # TODO: throw exception for dropout gru = BLayer.GRU(input_size=int(input_shape[2]), hidden_size=klayer.output_dim, - p=klayer.dropout_W, + p=0.0, wRegularizer=self.to_bigdl_reg(config["W_regularizer"]), uRegularizer=self.to_bigdl_reg(config["U_regularizer"]), bRegularizer=self.to_bigdl_reg(config["b_regularizer"]), @@ -703,8 +814,8 @@ def create_batchnormalization(self, klayer, kclayer): self.__check_is_share_weights(kclayer) if keras.backend.image_dim_ordering() != "th" or klayer.axis != 1: - raise Exception("""We only support th image order for BatchNormalization \n - which meaning NCHW for now. but the current order is %s and axis is %s + raise Exception("""For BatchNormalization, we only support th image ordering (i.e. NCHW) """ + + """with axis = 1 for now, but the current order is %s and axis is %s """ % (keras.backend.image_dim_ordering(), klayer.axis)) # noqa if klayer.mode != 0: raise Exception( @@ -760,11 +871,18 @@ def to_bigdl_2d_ordering(self, order): else: raise Exception("Unsupport ordering: %s" % order) + def to_bigdl_3d_padding(self, border_mode): + if border_mode == "valid": + return 0, 0, 0 + # TODO: border_mode=`same` + else: + raise Exception("Unsupported border mode: %s" % border_mode) + def to_bigdl_2d_padding(self, border_mode): if border_mode == "same": - return (-1, -1) + return -1, -1 elif border_mode == "valid": - return (0, 0) + return 0, 0 else: raise Exception("Unsupported border mode: %s" % border_mode) @@ -778,8 +896,6 @@ def to_bigdl_1d_padding(self, border_mode, kernel_w): else: raise Exception("Unsupported border mode: %s" % border_mode) -################# Layers with weights ############################# noqa - def create_convolution1d(self, klayer, kclayer): config = kclayer["config"] input_shape = klayer.get_input_shape_at(0) @@ -790,17 +906,17 @@ def create_convolution1d(self, klayer, kclayer): seq = BLayer.Sequential() seq.add(BLayer.Reshape([int(input_shape[1]), 1, int(input_shape[2])], True)) blayer = BLayer.SpatialConvolution( - n_input_plane = stack_size, - n_output_plane = klayer.nb_filter, - kernel_w = 1, - kernel_h = klayer.filter_length, - stride_w= 1, - stride_h= klayer.subsample_length, - pad_w= bpadW, - pad_h= bpadH, + n_input_plane=stack_size, + n_output_plane=klayer.nb_filter, + kernel_w=1, + kernel_h=klayer.filter_length, + stride_w=1, + stride_h=klayer.subsample_length, + pad_w=bpadW, + pad_h=bpadH, n_group=1, propagate_back=True, - wRegularizer = self.to_bigdl_reg(config["W_regularizer"]), + wRegularizer=self.to_bigdl_reg(config["W_regularizer"]), bRegularizer=self.to_bigdl_reg(config["b_regularizer"]), init_weight=None, init_bias=None, @@ -825,17 +941,17 @@ def create_convolution2d(self, klayer, kclayer): bpadW, bpadH = self.to_bigdl_2d_padding(klayer.border_mode) blayer = BLayer.SpatialConvolution( - n_input_plane = stack_size, - n_output_plane = klayer.nb_filter, - kernel_w = klayer.nb_col, - kernel_h = klayer.nb_row, - stride_w= klayer.subsample[0], - stride_h= klayer.subsample[1], - pad_w= bpadW, - pad_h= bpadH, + n_input_plane=stack_size, + n_output_plane=klayer.nb_filter, + kernel_w=klayer.nb_col, + kernel_h=klayer.nb_row, + stride_w=klayer.subsample[0], + stride_h=klayer.subsample[1], + pad_w=bpadW, + pad_h=bpadH, n_group=1, propagate_back=True, - wRegularizer = self.to_bigdl_reg(config["W_regularizer"]), + wRegularizer=self.to_bigdl_reg(config["W_regularizer"]), bRegularizer=self.to_bigdl_reg(config["b_regularizer"]), init_weight=None, init_bias=None, @@ -847,14 +963,145 @@ def create_convolution2d(self, klayer, kclayer): return self.combo_parameter_layer(blayer, config) + def create_convolution3d(self, klayer, kclayer): + config = kclayer["config"] + if klayer.dim_ordering != "th": + raise Exception("Please use `th` for `dim_ordering`. `%s` is not supported for now." % klayer.dim_ordering) + input_shape = klayer.get_input_shape_at(0) + + bpadT, bpadW, bpadH = self.to_bigdl_3d_padding(klayer.border_mode) + blayer = BLayer.VolumetricConvolution( + n_input_plane=int(input_shape[1]), + n_output_plane=klayer.nb_filter, + k_t=klayer.kernel_dim1, + k_w=klayer.kernel_dim3, + k_h=klayer.kernel_dim2, + d_t=klayer.subsample[0], + d_w=klayer.subsample[2], + d_h=klayer.subsample[1], + pad_t=bpadT, + pad_w=bpadW, + pad_h=bpadH, + with_bias=config["bias"], + wRegularizer=self.to_bigdl_reg(config["W_regularizer"]), + bRegularizer=self.to_bigdl_reg(config["b_regularizer"]), + bigdl_type="float") + + return self.combo_parameter_layer(blayer, config) + + def create_atrousconvolution1d(self, klayer, kclayer): + config = kclayer["config"] + if not config["bias"]: + raise Exception("Please set `bias=True` for AtrousConvolution1D") + input_shape = klayer.get_input_shape_at(0) + + # TODO: border_mode=`same` + if klayer.border_mode == "same": + raise Exception("Unsupported border mode: %s" % klayer.border_mode) + bpadW, bpadH = self.to_bigdl_2d_padding(klayer.border_mode) + seq = BLayer.Sequential() + seq.add(BLayer.Transpose([(2, 3)])) + seq.add(BLayer.Reshape([int(input_shape[2]), int(input_shape[1]), 1], True)) + blayer = BLayer.SpatialDilatedConvolution( + n_input_plane=int(input_shape[2]), + n_output_plane=config["nb_filter"], + kw=1, + kh=config["filter_length"], + dw=1, + dh=config["subsample_length"], + pad_w=bpadW, + pad_h=bpadH, + dilation_w=1, + dilation_h=config["atrous_rate"], + wRegularizer=self.to_bigdl_reg(config["W_regularizer"]), + bRegularizer=self.to_bigdl_reg(config["b_regularizer"]), + bigdl_type="float") + + seq.add(blayer) + seq.add(BLayer.Transpose([(2, 3)])) + seq.add(BLayer.Squeeze(4)) + return self.combo_parameter_layer(seq, config) + + def create_atrousconvolution2d(self, klayer, kclayer): + config = kclayer["config"] + if klayer.dim_ordering != "th": + raise Exception("Please use `th` for `dim_ordering`. `%s` is not supported for now." % klayer.dim_ordering) + if not config["bias"]: + raise Exception("Please set `bias=True` for AtrousConvolution2D") + input_shape = klayer.get_input_shape_at(0) + + # TODO: border_mode=`same` + if klayer.border_mode == "same": + raise Exception("Unsupported border mode: %s" % klayer.border_mode) + bpadW, bpadH = self.to_bigdl_2d_padding(klayer.border_mode) + blayer = BLayer.SpatialDilatedConvolution( + n_input_plane=int(input_shape[1]), + n_output_plane=config["nb_filter"], + kw=config["nb_col"], + kh=config["nb_row"], + dw=config["subsample"][1], + dh=config["subsample"][0], + pad_w=bpadW, + pad_h=bpadH, + dilation_w=config["atrous_rate"][1], + dilation_h=config["atrous_rate"][0], + wRegularizer=self.to_bigdl_reg(config["W_regularizer"]), + bRegularizer=self.to_bigdl_reg(config["b_regularizer"]), + bigdl_type="float") + + return self.combo_parameter_layer(blayer, config) + + def create_deconvolution2d(self, klayer, kclayer): + config = kclayer["config"] + if klayer.dim_ordering != "th": + raise Exception("Please use `th` for `dim_ordering`. `%s` is not supported for now." % klayer.dim_ordering) + input_shape = klayer.get_input_shape_at(0) + + bpadW, bpadH = self.to_bigdl_2d_padding(klayer.border_mode) + blayer = BLayer.SpatialFullConvolution( + n_input_plane=int(input_shape[1]), + n_output_plane=klayer.nb_filter, + kw=klayer.nb_col, + kh=klayer.nb_row, + dw=klayer.subsample[1], + dh=klayer.subsample[0], + pad_w=bpadW, + pad_h=bpadH, + adj_w=0, + adj_h=0, + n_group=1, + no_bias=False, + wRegularizer=self.to_bigdl_reg(config["W_regularizer"]), + bRegularizer=self.to_bigdl_reg(config["b_regularizer"]), + bigdl_type="float") + + return self.combo_parameter_layer(blayer, config) + + def create_maxpooling3d(self, klayer, kclayer): + if klayer.dim_ordering != "th": + raise Exception("Please use `th` for `dim_ordering`. `%s` is not supported for now." % klayer.dim_ordering) + bpadT, bpadW, bpadH = self.to_bigdl_3d_padding(klayer.border_mode) + blayer = BLayer.VolumetricMaxPooling( + k_t=klayer.pool_size[0], + k_w=klayer.pool_size[2], + k_h=klayer.pool_size[1], + d_t=klayer.strides[0], + d_w=klayer.strides[2], + d_h=klayer.strides[1], + pad_t=bpadT, + pad_w=bpadW, + pad_h=bpadH, + bigdl_type="float") + return blayer + def create_maxpooling2d(self, klayer, kclayer): bigdl_order = self.get_bdim_order(kclayer) bpadW, bpadH = self.to_bigdl_2d_padding(klayer.border_mode) blayer = BLayer.SpatialMaxPooling( - kw = klayer.pool_size[0], - kh = klayer.pool_size[1], - dw = klayer.strides[0], - dh = klayer.strides[1], + kw=klayer.pool_size[1], + kh=klayer.pool_size[0], + dw=klayer.strides[1], + dh=klayer.strides[0], pad_w=bpadW, pad_h=bpadH, to_ceil=False, @@ -862,14 +1109,73 @@ def create_maxpooling2d(self, klayer, kclayer): bigdl_type="float") return blayer + def create_globalmaxpooling3d(self, klayer, kclayer): + input_shape = klayer.get_input_shape_at(0) + if klayer.dim_ordering == "th": + b_kt = int(input_shape[2]) + b_kw = int(input_shape[4]) + b_kh = int(input_shape[3]) + else: + raise Exception("Please use `th` for dim_ordering. `%s` is not supported for now." % klayer.dim_ordering) + + seq = BLayer.Sequential() + blayer = BLayer.VolumetricMaxPooling( + k_t=b_kt, + k_w=b_kw, + k_h=b_kh, + d_t=1, + d_w=1, + d_h=1, + pad_t=0, + pad_w=0, + pad_h=0, + bigdl_type="float" + ) + seq.add(blayer) + seq.add(BLayer.Squeeze(5)) + seq.add(BLayer.Squeeze(4)) + seq.add(BLayer.Squeeze(3)) + + return seq + + def create_globalaveragepooling3d(self, klayer, kclayer): + input_shape = klayer.get_input_shape_at(0) + if klayer.dim_ordering == "th": + b_kt = int(input_shape[2]) + b_kw = int(input_shape[4]) + b_kh = int(input_shape[3]) + else: + raise Exception("Please use `th` for dim_ordering. `%s` is not supported for now." % klayer.dim_ordering) + + seq = BLayer.Sequential() + blayer = BLayer.VolumetricAveragePooling( + k_t=b_kt, + k_w=b_kw, + k_h=b_kh, + d_t=1, + d_w=1, + d_h=1, + pad_t=0, + pad_w=0, + pad_h=0, + count_include_pad=False, + bigdl_type="float" + ) + seq.add(blayer) + seq.add(BLayer.Squeeze(5)) + seq.add(BLayer.Squeeze(4)) + seq.add(BLayer.Squeeze(3)) + + return seq + def create_averagepooling2d(self, klayer, kclayer): bigdl_order = self.get_bdim_order(kclayer) bpadW, bpadH = self.to_bigdl_2d_padding(klayer.border_mode) blayer = BLayer.SpatialAveragePooling( - kw=klayer.pool_size[0], - kh=klayer.pool_size[1], - dw=klayer.strides[0], - dh=klayer.strides[1], + kw=klayer.pool_size[1], + kh=klayer.pool_size[0], + dw=klayer.strides[1], + dh=klayer.strides[0], pad_w=bpadW, pad_h=bpadH, global_pooling=False, @@ -881,6 +1187,24 @@ def create_averagepooling2d(self, klayer, kclayer): ) return blayer + def create_averagepooling3d(self, klayer, kclayer): + if klayer.dim_ordering != "th": + raise Exception("Please use `th` for `dim_ordering`. `%s` is not supported for now." % klayer.dim_ordering) + bpadT, bpadW, bpadH = self.to_bigdl_3d_padding(klayer.border_mode) + blayer = BLayer.VolumetricAveragePooling( + k_t=klayer.pool_size[0], + k_w=klayer.pool_size[2], + k_h=klayer.pool_size[1], + d_t=klayer.strides[0], + d_w=klayer.strides[2], + d_h=klayer.strides[1], + pad_t=bpadT, + pad_w=bpadW, + pad_h=bpadH, + count_include_pad=False, + bigdl_type="float") + return blayer + def create_globalmaxpooling2d(self, klayer, kclayer): bigdl_order = self.get_bdim_order(kclayer) input_shape = klayer.get_input_shape_at(0) @@ -967,12 +1291,12 @@ def create_maxpooling1d(self, klayer, kclayer): bpadW, bpadH = self.to_bigdl_2d_padding(klayer.border_mode) seq = BLayer.Sequential() - seq.add(BLayer.View([1, int(input_shape[1]), int(input_shape[2])], num_input_dims=3)) + seq.add(BLayer.Reshape([int(input_shape[1]), 1, int(input_shape[2])], True)) blayer = BLayer.SpatialMaxPooling( - kw=klayer.pool_length, - kh=1, - dw=klayer.stride, - dh=1, + kw=1, + kh=klayer.pool_length, + dw=1, + dh=klayer.stride, pad_w=bpadW, pad_h=bpadH, to_ceil=False, @@ -980,6 +1304,7 @@ def create_maxpooling1d(self, klayer, kclayer): bigdl_type="float" ) seq.add(blayer) + seq.add(BLayer.Squeeze(3)) return seq def create_averagepooling1d(self, klayer, kclayer): @@ -987,12 +1312,12 @@ def create_averagepooling1d(self, klayer, kclayer): bpadW, bpadH = self.to_bigdl_2d_padding(klayer.border_mode) seq = BLayer.Sequential() - seq.add(BLayer.View([1, int(input_shape[1]), int(input_shape[2])], num_input_dims=3)) + seq.add(BLayer.Reshape([int(input_shape[1]), 1, int(input_shape[2])], True)) blayer = BLayer.SpatialAveragePooling( - kw=klayer.pool_length, - kh=1, - dw=klayer.stride, - dh=1, + kw=1, + kh=klayer.pool_length, + dw=1, + dh=klayer.stride, pad_w=bpadW, pad_h=bpadH, global_pooling=False, @@ -1003,6 +1328,7 @@ def create_averagepooling1d(self, klayer, kclayer): bigdl_type="float" ) seq.add(blayer) + seq.add(BLayer.Squeeze(3)) return seq def create_globalaveragepooling2d(self, klayer, kclayer): @@ -1053,9 +1379,9 @@ def combo_parameter_layer(self, blayer, config): blayer.set_name(config["name"]) if hasattr(blayer, "set_init_method"): blayer.set_init_method(self.to_bigdl_init(config["init"]), - BInit.Zeros()) # Keras always set this to be zeros + BInit.Zeros()) # Keras always set this to be zeros # "linear" meaning do nothing - if config["activation"] != "linear" : + if config["activation"] != "linear": activation = self.to_bigdl_activation(config["activation"], "%s_%s" % (config["name"], config["activation"])) return self.fuse(blayer, activation) @@ -1068,10 +1394,18 @@ def to_bigdl_activation(self, activation_name, activation_id): activation = BLayer.Tanh() elif activation_name == "sigmoid": activation = BLayer.Sigmoid() + elif activation_name == "hard_sigmoid": + activation = BLayer.HardSigmoid() elif activation_name == "relu": activation = BLayer.ReLU() elif activation_name == "softmax": activation = BLayer.SoftMax() + elif activation_name == "softplus": + activation = BLayer.SoftPlus(beta=1.0) + elif activation_name == "softsign": + activation = BLayer.SoftSign() + elif activation_name == "linear": + activation = BLayer.Identity() else: raise Exception("Unsupported activation type: %s" % activation_name) activation.set_name(activation_id) @@ -1085,7 +1419,7 @@ def get_value_from_init(self, kinit_method, shape): else: raise Exception("We don't support % for now", kinit_method) - def to_bigdl_init(self, kinit_method): # kinit_method is a string + def to_bigdl_init(self, kinit_method): # kinit_method is a string init = None if kinit_method == "glorot_uniform": init = BInit.Xavier() @@ -1099,11 +1433,11 @@ def to_bigdl_init(self, kinit_method): # kinit_method is a string def to_bigdl_reg(self, reg): # reg is a dict if reg: - raise Exception("will support reg very soon") + return BRegularizer(reg['l1'], reg['l2']) else: return None - def fuse(self, src_blayer, activation): # activation is a string + def fuse(self, src_blayer, activation): # activation is a layer seq = BLayer.Sequential() seq.add(src_blayer) seq.add(activation) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index dcb4ed4f2ea..6f45aeb38bd 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -1977,6 +1977,26 @@ def __init__(self, super(CAddTable, self).__init__(None, bigdl_type, inplace) +class CAveTable(Layer): + + ''' + Merge the input tensors in the input table by element wise taking the average. The input + table is actually an array of tensor with same size. + + + :param inplace: reuse the input memory + + + >>> cAveTable = CAveTable(True) + creating: createCAveTable + ''' + + def __init__(self, + inplace=False, + bigdl_type="float"): + super(CAveTable, self).__init__(None, bigdl_type, + inplace) + class CDivTable(Layer): @@ -3744,6 +3764,57 @@ def __init__(self, pad_h) +class VolumetricAveragePooling(Layer): + + ''' + Applies 3D average-pooling operation in kTxkWxkH regions by step size dTxdWxdH. + The number of output features is equal to the number of input planes / dT. + The input can optionally be padded with zeros. Padding should be smaller than + half of kernel size. That is, padT < kT/2, padW < kW/2 and padH < kH/2 + + :param k_t: The kernel size + :param k_w: The kernel width + :param k_h: The kernel height + :param d_t: The step in the time dimension + :param d_w: The step in the width dimension + :param d_h: The step in the height dimension + :param pad_t: The padding in the time dimension + :param pad_w: The padding in the width dimension + :param pad_h: The padding in the height dimension + :param count_include_pad: whether to include padding when dividing the number of elements in pooling region + :param ceil_mode: whether the output size is to be ceiled or floored + + + >>> volumetricAveragePooling = VolumetricAveragePooling(5, 5, 5, 1, 1, 1) + creating: createVolumetricAveragePooling + ''' + + def __init__(self, + k_t, + k_w, + k_h, + d_t, + d_w, + d_h, + pad_t=0, + pad_w=0, + pad_h=0, + count_include_pad=True, + ceil_mode=False, + bigdl_type="float"): + super(VolumetricAveragePooling, self).__init__(None, bigdl_type, + k_t, + k_w, + k_h, + d_t, + d_w, + d_h, + pad_t, + pad_w, + pad_h, + count_include_pad, + ceil_mode) + class SpatialZeroPadding(Layer): ''' From 6ea8cd941f54fb4dd7d736f4e69fb470e0b3f4f7 Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Tue, 21 Nov 2017 11:09:43 +0800 Subject: [PATCH 345/823] Add more keras layers and test (#1891) * add, update and fix * add hard_sigmoid * fix python3.5 lambda in permute * fix syntax for python3.5 * fix pooling and conv layers * fix syntax * fix permute for py35 --- .../dllib/src/bigdl/dllib/keras/converter.py | 496 +++++++++++++++--- python/dllib/src/bigdl/dllib/nn/layer.py | 71 +++ 2 files changed, 486 insertions(+), 81 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/keras/converter.py b/python/dllib/src/bigdl/dllib/keras/converter.py index 40baa27813c..ab35e86a72f 100644 --- a/python/dllib/src/bigdl/dllib/keras/converter.py +++ b/python/dllib/src/bigdl/dllib/keras/converter.py @@ -2,6 +2,7 @@ import bigdl.nn.initialization_method as BInit import numpy as np import bigdl.nn.layer as BLayer +from bigdl.optim.optimizer import L1L2Regularizer as BRegularizer import bigdl.optim.optimizer as boptimizer import bigdl.nn.criterion as bcriterion import bigdl.util.common as bcommon @@ -175,6 +176,23 @@ def convert_batchnormalization(weights): beta = weights[1] return [gamma, beta] + @staticmethod + def convert_atrousconvolution2d(weights): + return weights + + @staticmethod + def convert_atrousconvolution1d(weights): + return [np.transpose(weights[0], (3, 2, 0, 1)), weights[1]] + + @staticmethod + def convert_deconvolution2d(weights): + w = np.transpose(weights[0], (1, 0, 2, 3)) + weight = np.expand_dims(w, 0) + if len(weights) > 1: + return [weight, weights[1]] + else: + return [weight] + @staticmethod def convert_convolution2d(weights): weight = np.expand_dims(weights[0], 0) # bigdl has a leading dim with value 1 @@ -187,6 +205,10 @@ def convert_convolution2d(weights): def convert_convolution1d(weights): return WeightsConverter.convert_convolution2d(weights) + @staticmethod + def convert_convolution3d(weights): + return weights + @staticmethod def convert_embedding(weights): return weights @@ -202,10 +224,17 @@ def convert_lstm(weights): w3 = np.concatenate((weights[1].T, weights[4].T, weights[7].T, weights[10].T)) return [w1, w2, w3] + @staticmethod + def convert_convlstm2d(weights): + return [np.expand_dims(weights[6], 0), weights[8], np.expand_dims(weights[7], 0), + np.expand_dims(weights[0], 0), weights[2], np.expand_dims(weights[1], 0), + np.expand_dims(weights[3], 0), weights[5], np.expand_dims(weights[4], 0), + np.expand_dims(weights[9], 0), weights[11], np.expand_dims(weights[10], 0)] + @staticmethod def convert_gru(weights): w1 = np.concatenate((weights[3].T, weights[0].T, weights[6].T)) - w2 = np.concatenate((weights[2], weights[5], weights[8])) + w2 = np.concatenate((weights[5], weights[2], weights[8])) w3 = np.concatenate((weights[4].T, weights[1].T)) w4 = weights[7].T return [w1, w2, w3, w4] @@ -375,8 +404,6 @@ def create_dense(self, klayer, kclayer): def create_timedistributeddense(self, klayer, kclayer): config = kclayer["config"] input_shape = klayer.get_input_shape_at(0) - print(input_shape) - print(config["output_dim"]) blayer = BLayer.TimeDistributed(BLayer.Linear( input_size=int(input_shape[2]), output_size=config["output_dim"], @@ -426,6 +453,45 @@ def create_flatten(self, klayer, kclayer): blayer = BLayer.Reshape([int(np.prod(input_shape[1:]))], None) return blayer + def create_permute(self, klayer, kclayer): + swaps = self.__perm_to_pair(list(klayer.dims)) + swaps.reverse() + swaps = map(lambda pair: (pair[0]+1, pair[1]+1), swaps) + return BLayer.Transpose(list(swaps)) + + def __perm_to_pair(self, perm): + # perm: a list as a permutation of [1..n], eg [3, 1, 2] for n=3. + # return a list of tuples that needs to be swapped to obtain the input `perm`. + pairs = [] + + def sort(arr, low, high): + i = low + j = high + pivot = arr[low + int((high - low) / 2)] + while i <= j: + while arr[i] < pivot: + i += 1 + while arr[j] > pivot: + j -= 1 + if i <= j: + exchangeNumbers(arr, i, j) + i += 1 + j -= 1 + if low < j: + sort(arr, low, j) + if i < high: + sort(arr, i, high) + + def exchangeNumbers(arr, i, j): + temp = arr[i] + arr[i] = arr[j] + arr[j] = temp + pairs.append((i + 1, j + 1)) + + sort(perm, 0, len(perm) - 1) + + return list(filter(lambda pair: pair[0] != pair[1], pairs)) + def create_reshape(self, klayer, kclayer): self.__check_is_share_weights(kclayer) blayer = BLayer.Reshape(klayer.target_shape, None) @@ -439,11 +505,12 @@ def create_repeatvector(self, klayer, kclayer): def create_merge(self, klayer, kclayer): self.__check_is_share_weights(kclayer) input_shape = klayer.get_input_shape_at(0) - #TODO: dot_axes, node_indices, tensor_indices not supported. if klayer.output_shape and not isinstance(klayer.output_shape, tuple): - raise Exception("Only output_shape=None or a shape tuple is supported for now.") + raise Exception("Only output_shape=None or a shape tuple is supported for now") + if klayer.node_indices: + unsupport_exp("node_indices") if klayer.output_mask: - raise Exception("Argument `output_mask` is not supported for now.") + unsupport_exp("output_mask") if klayer.mode == "concat": blayer = BLayer.JoinTable( dimension=klayer.concat_axis, @@ -459,14 +526,18 @@ def create_merge(self, klayer, kclayer): blayer = BLayer.CMaxTable(bigdl_type="float") elif klayer.mode == "dot": if len(input_shape[0]) >= 3: - raise Exception("Merge mode `dot` doesn't support 3D input or above for now.") - if input_shape[0][0] == None: - raise Exception("For merge mode `dot`, please specify `batch_input_shape`.") + raise Exception("For merge mode dot, 3D input or above is not supported for now.") + if klayer.dot_axes != [1, 1]: + raise Exception("For merge mode dot, only dot_axes=1 is supported for now.") model = BLayer.Sequential() blayer = model.add(BLayer.DotProduct(bigdl_type="float"))\ - .add(BLayer.Reshape([input_shape[0][0], 1])) - elif klayer.mode in ['ave', 'cos']: - raise Exception("Merge mode `%s` not supported for now" % klayer.mode) # TODO: whether to support cosine and average + .add(BLayer.Reshape([1], True)) + elif klayer.mode == "ave": + blayer = BLayer.CAveTable( + inplace=False, + bigdl_type="float") + elif klayer.mode in ['cos']: + raise Exception("Merge mode `%s` not supported for now" % klayer.mode) else: # invalid mode or lambda functions raise Exception("Invalid merge mode: `%s`. Lambda/function as merge mode is not supported for now." % klayer.mode) return blayer @@ -488,6 +559,8 @@ def create_leakyrelu(self, klayer, kclayer): def create_parametricsoftplus(self, klayer, kclayer): alpha = float(klayer.alpha_init) beta = float(klayer.beta_init) + if klayer.shared_axes != [None]: + unsupport_exp("shared_axes") if round(alpha * beta, 4) == 1.0: return BLayer.SoftPlus(beta=beta, bigdl_type="float") @@ -549,64 +622,65 @@ def __generate_zeropadding2d(self, dim1, dim2, n_input_dim, pad1, pad2, pad3, pa model.add(paddinglayer4) return model - # NB: zeropadding doesn't serialize dim_ording to jason file + # NB: zeropadding doesn't serialize dim_ording to json file def create_zeropadding2d(self, klayer, kclayer): padding = klayer.padding input_shape = klayer.get_input_shape_at(0) - dim1 = 1 - dim2 = 2 + dim = 1 if klayer.dim_ordering == "th": - dim1 = 2 - dim2 = 3 + dim = 2 if isinstance(padding, dict): # dictionary - return self.__generate_zeropadding2d(dim1, dim2, len(input_shape) - 1, + return self.__generate_zeropadding2d(dim, dim+1, len(input_shape) - 1, -padding.get('top_pad', 0), padding.get('bottom_pad', 0), -padding.get('left_pad', 0), padding.get('right_pad', 0)) else: # tuple of int padding = tuple(padding) if len(padding) == 2: - return self.__generate_zeropadding2d(dim1, dim2, len(input_shape) - 1, + return self.__generate_zeropadding2d(dim, dim+1, len(input_shape) - 1, -padding[0], padding[0], -padding[1], padding[1]) elif len(padding) == 4: - return self.__generate_zeropadding2d(dim1, dim2, len(input_shape) - 1, + return self.__generate_zeropadding2d(dim, dim+1, len(input_shape) - 1, -padding[0], padding[1], -padding[2], padding[3]) - # NB: zeropadding doesn't serialize dim_ording to jason file + # NB: zeropadding doesn't serialize dim_ording to json file def create_zeropadding3d(self, klayer, kclayer): padding = tuple(klayer.padding) input_shape = klayer.get_input_shape_at(0) + dim = 1 + if klayer.dim_ordering == "th": + dim = 2 model = BLayer.Sequential() - paddinglayer1 = BLayer.Padding(dim=2, + paddinglayer1 = BLayer.Padding(dim=dim, pad=-padding[0], n_input_dim=len(input_shape) - 1, value=0.0, n_index=1, bigdl_type="float") - paddinglayer2 = BLayer.Padding(dim=2, + paddinglayer2 = BLayer.Padding(dim=dim, pad=padding[0], n_input_dim=len(input_shape) - 1, value=0.0, n_index=1, bigdl_type="float") - paddinglayer3 = BLayer.Padding(dim=3, + paddinglayer3 = BLayer.Padding(dim=dim+1, pad=-padding[1], n_input_dim=len(input_shape) - 1, value=0.0, n_index=1, bigdl_type="float") - paddinglayer4 = BLayer.Padding(dim=3, + paddinglayer4 = BLayer.Padding(dim=dim+1, pad=padding[1], n_input_dim=len(input_shape) - 1, value=0.0, n_index=1, bigdl_type="float") - paddinglayer5 = BLayer.Padding(dim=4, + paddinglayer5 = BLayer.Padding(dim=dim+2, pad=-padding[2], n_input_dim=len(input_shape) - 1, value=0.0, n_index=1, bigdl_type="float") - paddinglayer6 = BLayer.Padding(dim=4, + paddinglayer6 = BLayer.Padding(dim=dim+2, pad=padding[2], n_input_dim=len(input_shape) - 1, value=0.0, @@ -641,7 +715,6 @@ def create_simplernn(self, klayer, kclayer): self.check_constraint_in_config(config) activation = self.to_bigdl_activation(config["activation"], "%s_%s" % (config["name"], config["activation"])) - # TODO: throw exception for dropout rnn = BLayer.RnnCell(input_size=int(input_shape[2]), hidden_size=klayer.output_dim, activation=activation, @@ -660,21 +733,60 @@ def create_lstm(self, klayer, kclayer): activation = self.to_bigdl_activation(config["activation"], "%s_%s" % (config["name"], config["activation"])) if not isinstance(activation, BLayer.Tanh): - raise Exception("For activation, only `tanh` is supported for now.") + raise Exception("For activation, only tanh is supported for now.") inner_activation = self.to_bigdl_activation(config["inner_activation"], "%s_%s" % (config["name"], config["inner_activation"])) if not isinstance(inner_activation, BLayer.Sigmoid): - raise Exception("For inner_activation, only `sigmond` is supported for now.") - # TODO: throw exception for dropout + raise Exception("For inner_activation, only sigmond is supported for now.") lstm = BLayer.LSTM(input_size=int(input_shape[2]), hidden_size=klayer.output_dim, - p=klayer.dropout_W, + p=0.0, wRegularizer=self.to_bigdl_reg(config["W_regularizer"]), uRegularizer=self.to_bigdl_reg(config["U_regularizer"]), bRegularizer=self.to_bigdl_reg(config["b_regularizer"]), bigdl_type="float") return self.__return_sequences(klayer.return_sequences, rec.add(lstm)) + def create_convlstm2d(self, klayer, kclayer): + rec = BLayer.Recurrent() + input_shape = klayer.get_input_shape_at(0) + config = kclayer["config"] + self.check_constraint_in_config(config) + activation = self.to_bigdl_activation(config["activation"], + "%s_%s" % (config["name"], config["activation"])) + if not isinstance(activation, BLayer.Tanh): + raise Exception("For activation, only tanh is supported for now.") + inner_activation = self.to_bigdl_activation(config["inner_activation"], + "%s_%s" % (config["name"], config["inner_activation"])) + if not isinstance(inner_activation, BLayer.Sigmoid): + raise Exception("For inner_activation, only sigmond is supported for now.") + + #TODO: border_mode = 'valid' + if config["border_mode"] != 'same': + raise Exception("Unsupported border_mode: valid") + + if config["nb_row"] != config["nb_col"]: + raise Exception("Only square kernel is supported for now. Please set nb_row=nb_col.") + if klayer.subsample[0] != klayer.subsample[1]: + raise Exception("Only equal stride is supported for now. " + "Please set subsample to be a tuple with equal values.") + + blayer = BLayer.ConvLSTMPeephole(input_size=int(input_shape[2]), + output_size=config["nb_filter"], + kernel_i=config["nb_col"], + kernel_c=config["nb_row"], + # NB: ConvLSTM doesn't serialize subsample to json file + stride=klayer.subsample[0], + padding=-1, + # NB: ConvLSTM doesn't serialize regularizers to json file + # wRegularizer=self.to_bigdl_reg(config["W_regularizer"]), + # uRegularizer=self.to_bigdl_reg(config["U_regularizer"]), + # bRegularizer=self.to_bigdl_reg(config["b_regularizer"]), + cRegularizer=None, + with_peephole=False, + bigdl_type="float") + return self.__return_sequences(klayer.return_sequences, rec.add(blayer)) + def create_gru(self, klayer, kclayer): rec = BLayer.Recurrent() input_shape = klayer.get_input_shape_at(0) @@ -688,10 +800,9 @@ def create_gru(self, klayer, kclayer): "%s_%s" % (config["name"], config["inner_activation"])) if not isinstance(inner_activation, BLayer.Sigmoid): raise Exception("For inner_activation, only `sigmond` is supported for now.") - # TODO: throw exception for dropout gru = BLayer.GRU(input_size=int(input_shape[2]), hidden_size=klayer.output_dim, - p=klayer.dropout_W, + p=0.0, wRegularizer=self.to_bigdl_reg(config["W_regularizer"]), uRegularizer=self.to_bigdl_reg(config["U_regularizer"]), bRegularizer=self.to_bigdl_reg(config["b_regularizer"]), @@ -703,8 +814,8 @@ def create_batchnormalization(self, klayer, kclayer): self.__check_is_share_weights(kclayer) if keras.backend.image_dim_ordering() != "th" or klayer.axis != 1: - raise Exception("""We only support th image order for BatchNormalization \n - which meaning NCHW for now. but the current order is %s and axis is %s + raise Exception("""For BatchNormalization, we only support th image ordering (i.e. NCHW) """ + + """with axis = 1 for now, but the current order is %s and axis is %s """ % (keras.backend.image_dim_ordering(), klayer.axis)) # noqa if klayer.mode != 0: raise Exception( @@ -760,11 +871,18 @@ def to_bigdl_2d_ordering(self, order): else: raise Exception("Unsupport ordering: %s" % order) + def to_bigdl_3d_padding(self, border_mode): + if border_mode == "valid": + return 0, 0, 0 + # TODO: border_mode=`same` + else: + raise Exception("Unsupported border mode: %s" % border_mode) + def to_bigdl_2d_padding(self, border_mode): if border_mode == "same": - return (-1, -1) + return -1, -1 elif border_mode == "valid": - return (0, 0) + return 0, 0 else: raise Exception("Unsupported border mode: %s" % border_mode) @@ -778,8 +896,6 @@ def to_bigdl_1d_padding(self, border_mode, kernel_w): else: raise Exception("Unsupported border mode: %s" % border_mode) -################# Layers with weights ############################# noqa - def create_convolution1d(self, klayer, kclayer): config = kclayer["config"] input_shape = klayer.get_input_shape_at(0) @@ -790,17 +906,17 @@ def create_convolution1d(self, klayer, kclayer): seq = BLayer.Sequential() seq.add(BLayer.Reshape([int(input_shape[1]), 1, int(input_shape[2])], True)) blayer = BLayer.SpatialConvolution( - n_input_plane = stack_size, - n_output_plane = klayer.nb_filter, - kernel_w = 1, - kernel_h = klayer.filter_length, - stride_w= 1, - stride_h= klayer.subsample_length, - pad_w= bpadW, - pad_h= bpadH, + n_input_plane=stack_size, + n_output_plane=klayer.nb_filter, + kernel_w=1, + kernel_h=klayer.filter_length, + stride_w=1, + stride_h=klayer.subsample_length, + pad_w=bpadW, + pad_h=bpadH, n_group=1, propagate_back=True, - wRegularizer = self.to_bigdl_reg(config["W_regularizer"]), + wRegularizer=self.to_bigdl_reg(config["W_regularizer"]), bRegularizer=self.to_bigdl_reg(config["b_regularizer"]), init_weight=None, init_bias=None, @@ -825,17 +941,17 @@ def create_convolution2d(self, klayer, kclayer): bpadW, bpadH = self.to_bigdl_2d_padding(klayer.border_mode) blayer = BLayer.SpatialConvolution( - n_input_plane = stack_size, - n_output_plane = klayer.nb_filter, - kernel_w = klayer.nb_col, - kernel_h = klayer.nb_row, - stride_w= klayer.subsample[0], - stride_h= klayer.subsample[1], - pad_w= bpadW, - pad_h= bpadH, + n_input_plane=stack_size, + n_output_plane=klayer.nb_filter, + kernel_w=klayer.nb_col, + kernel_h=klayer.nb_row, + stride_w=klayer.subsample[0], + stride_h=klayer.subsample[1], + pad_w=bpadW, + pad_h=bpadH, n_group=1, propagate_back=True, - wRegularizer = self.to_bigdl_reg(config["W_regularizer"]), + wRegularizer=self.to_bigdl_reg(config["W_regularizer"]), bRegularizer=self.to_bigdl_reg(config["b_regularizer"]), init_weight=None, init_bias=None, @@ -847,14 +963,145 @@ def create_convolution2d(self, klayer, kclayer): return self.combo_parameter_layer(blayer, config) + def create_convolution3d(self, klayer, kclayer): + config = kclayer["config"] + if klayer.dim_ordering != "th": + raise Exception("Please use `th` for `dim_ordering`. `%s` is not supported for now." % klayer.dim_ordering) + input_shape = klayer.get_input_shape_at(0) + + bpadT, bpadW, bpadH = self.to_bigdl_3d_padding(klayer.border_mode) + blayer = BLayer.VolumetricConvolution( + n_input_plane=int(input_shape[1]), + n_output_plane=klayer.nb_filter, + k_t=klayer.kernel_dim1, + k_w=klayer.kernel_dim3, + k_h=klayer.kernel_dim2, + d_t=klayer.subsample[0], + d_w=klayer.subsample[2], + d_h=klayer.subsample[1], + pad_t=bpadT, + pad_w=bpadW, + pad_h=bpadH, + with_bias=config["bias"], + wRegularizer=self.to_bigdl_reg(config["W_regularizer"]), + bRegularizer=self.to_bigdl_reg(config["b_regularizer"]), + bigdl_type="float") + + return self.combo_parameter_layer(blayer, config) + + def create_atrousconvolution1d(self, klayer, kclayer): + config = kclayer["config"] + if not config["bias"]: + raise Exception("Please set `bias=True` for AtrousConvolution1D") + input_shape = klayer.get_input_shape_at(0) + + # TODO: border_mode=`same` + if klayer.border_mode == "same": + raise Exception("Unsupported border mode: %s" % klayer.border_mode) + bpadW, bpadH = self.to_bigdl_2d_padding(klayer.border_mode) + seq = BLayer.Sequential() + seq.add(BLayer.Transpose([(2, 3)])) + seq.add(BLayer.Reshape([int(input_shape[2]), int(input_shape[1]), 1], True)) + blayer = BLayer.SpatialDilatedConvolution( + n_input_plane=int(input_shape[2]), + n_output_plane=config["nb_filter"], + kw=1, + kh=config["filter_length"], + dw=1, + dh=config["subsample_length"], + pad_w=bpadW, + pad_h=bpadH, + dilation_w=1, + dilation_h=config["atrous_rate"], + wRegularizer=self.to_bigdl_reg(config["W_regularizer"]), + bRegularizer=self.to_bigdl_reg(config["b_regularizer"]), + bigdl_type="float") + + seq.add(blayer) + seq.add(BLayer.Transpose([(2, 3)])) + seq.add(BLayer.Squeeze(4)) + return self.combo_parameter_layer(seq, config) + + def create_atrousconvolution2d(self, klayer, kclayer): + config = kclayer["config"] + if klayer.dim_ordering != "th": + raise Exception("Please use `th` for `dim_ordering`. `%s` is not supported for now." % klayer.dim_ordering) + if not config["bias"]: + raise Exception("Please set `bias=True` for AtrousConvolution2D") + input_shape = klayer.get_input_shape_at(0) + + # TODO: border_mode=`same` + if klayer.border_mode == "same": + raise Exception("Unsupported border mode: %s" % klayer.border_mode) + bpadW, bpadH = self.to_bigdl_2d_padding(klayer.border_mode) + blayer = BLayer.SpatialDilatedConvolution( + n_input_plane=int(input_shape[1]), + n_output_plane=config["nb_filter"], + kw=config["nb_col"], + kh=config["nb_row"], + dw=config["subsample"][1], + dh=config["subsample"][0], + pad_w=bpadW, + pad_h=bpadH, + dilation_w=config["atrous_rate"][1], + dilation_h=config["atrous_rate"][0], + wRegularizer=self.to_bigdl_reg(config["W_regularizer"]), + bRegularizer=self.to_bigdl_reg(config["b_regularizer"]), + bigdl_type="float") + + return self.combo_parameter_layer(blayer, config) + + def create_deconvolution2d(self, klayer, kclayer): + config = kclayer["config"] + if klayer.dim_ordering != "th": + raise Exception("Please use `th` for `dim_ordering`. `%s` is not supported for now." % klayer.dim_ordering) + input_shape = klayer.get_input_shape_at(0) + + bpadW, bpadH = self.to_bigdl_2d_padding(klayer.border_mode) + blayer = BLayer.SpatialFullConvolution( + n_input_plane=int(input_shape[1]), + n_output_plane=klayer.nb_filter, + kw=klayer.nb_col, + kh=klayer.nb_row, + dw=klayer.subsample[1], + dh=klayer.subsample[0], + pad_w=bpadW, + pad_h=bpadH, + adj_w=0, + adj_h=0, + n_group=1, + no_bias=False, + wRegularizer=self.to_bigdl_reg(config["W_regularizer"]), + bRegularizer=self.to_bigdl_reg(config["b_regularizer"]), + bigdl_type="float") + + return self.combo_parameter_layer(blayer, config) + + def create_maxpooling3d(self, klayer, kclayer): + if klayer.dim_ordering != "th": + raise Exception("Please use `th` for `dim_ordering`. `%s` is not supported for now." % klayer.dim_ordering) + bpadT, bpadW, bpadH = self.to_bigdl_3d_padding(klayer.border_mode) + blayer = BLayer.VolumetricMaxPooling( + k_t=klayer.pool_size[0], + k_w=klayer.pool_size[2], + k_h=klayer.pool_size[1], + d_t=klayer.strides[0], + d_w=klayer.strides[2], + d_h=klayer.strides[1], + pad_t=bpadT, + pad_w=bpadW, + pad_h=bpadH, + bigdl_type="float") + return blayer + def create_maxpooling2d(self, klayer, kclayer): bigdl_order = self.get_bdim_order(kclayer) bpadW, bpadH = self.to_bigdl_2d_padding(klayer.border_mode) blayer = BLayer.SpatialMaxPooling( - kw = klayer.pool_size[0], - kh = klayer.pool_size[1], - dw = klayer.strides[0], - dh = klayer.strides[1], + kw=klayer.pool_size[1], + kh=klayer.pool_size[0], + dw=klayer.strides[1], + dh=klayer.strides[0], pad_w=bpadW, pad_h=bpadH, to_ceil=False, @@ -862,14 +1109,73 @@ def create_maxpooling2d(self, klayer, kclayer): bigdl_type="float") return blayer + def create_globalmaxpooling3d(self, klayer, kclayer): + input_shape = klayer.get_input_shape_at(0) + if klayer.dim_ordering == "th": + b_kt = int(input_shape[2]) + b_kw = int(input_shape[4]) + b_kh = int(input_shape[3]) + else: + raise Exception("Please use `th` for dim_ordering. `%s` is not supported for now." % klayer.dim_ordering) + + seq = BLayer.Sequential() + blayer = BLayer.VolumetricMaxPooling( + k_t=b_kt, + k_w=b_kw, + k_h=b_kh, + d_t=1, + d_w=1, + d_h=1, + pad_t=0, + pad_w=0, + pad_h=0, + bigdl_type="float" + ) + seq.add(blayer) + seq.add(BLayer.Squeeze(5)) + seq.add(BLayer.Squeeze(4)) + seq.add(BLayer.Squeeze(3)) + + return seq + + def create_globalaveragepooling3d(self, klayer, kclayer): + input_shape = klayer.get_input_shape_at(0) + if klayer.dim_ordering == "th": + b_kt = int(input_shape[2]) + b_kw = int(input_shape[4]) + b_kh = int(input_shape[3]) + else: + raise Exception("Please use `th` for dim_ordering. `%s` is not supported for now." % klayer.dim_ordering) + + seq = BLayer.Sequential() + blayer = BLayer.VolumetricAveragePooling( + k_t=b_kt, + k_w=b_kw, + k_h=b_kh, + d_t=1, + d_w=1, + d_h=1, + pad_t=0, + pad_w=0, + pad_h=0, + count_include_pad=False, + bigdl_type="float" + ) + seq.add(blayer) + seq.add(BLayer.Squeeze(5)) + seq.add(BLayer.Squeeze(4)) + seq.add(BLayer.Squeeze(3)) + + return seq + def create_averagepooling2d(self, klayer, kclayer): bigdl_order = self.get_bdim_order(kclayer) bpadW, bpadH = self.to_bigdl_2d_padding(klayer.border_mode) blayer = BLayer.SpatialAveragePooling( - kw=klayer.pool_size[0], - kh=klayer.pool_size[1], - dw=klayer.strides[0], - dh=klayer.strides[1], + kw=klayer.pool_size[1], + kh=klayer.pool_size[0], + dw=klayer.strides[1], + dh=klayer.strides[0], pad_w=bpadW, pad_h=bpadH, global_pooling=False, @@ -881,6 +1187,24 @@ def create_averagepooling2d(self, klayer, kclayer): ) return blayer + def create_averagepooling3d(self, klayer, kclayer): + if klayer.dim_ordering != "th": + raise Exception("Please use `th` for `dim_ordering`. `%s` is not supported for now." % klayer.dim_ordering) + bpadT, bpadW, bpadH = self.to_bigdl_3d_padding(klayer.border_mode) + blayer = BLayer.VolumetricAveragePooling( + k_t=klayer.pool_size[0], + k_w=klayer.pool_size[2], + k_h=klayer.pool_size[1], + d_t=klayer.strides[0], + d_w=klayer.strides[2], + d_h=klayer.strides[1], + pad_t=bpadT, + pad_w=bpadW, + pad_h=bpadH, + count_include_pad=False, + bigdl_type="float") + return blayer + def create_globalmaxpooling2d(self, klayer, kclayer): bigdl_order = self.get_bdim_order(kclayer) input_shape = klayer.get_input_shape_at(0) @@ -967,12 +1291,12 @@ def create_maxpooling1d(self, klayer, kclayer): bpadW, bpadH = self.to_bigdl_2d_padding(klayer.border_mode) seq = BLayer.Sequential() - seq.add(BLayer.View([1, int(input_shape[1]), int(input_shape[2])], num_input_dims=3)) + seq.add(BLayer.Reshape([int(input_shape[1]), 1, int(input_shape[2])], True)) blayer = BLayer.SpatialMaxPooling( - kw=klayer.pool_length, - kh=1, - dw=klayer.stride, - dh=1, + kw=1, + kh=klayer.pool_length, + dw=1, + dh=klayer.stride, pad_w=bpadW, pad_h=bpadH, to_ceil=False, @@ -980,6 +1304,7 @@ def create_maxpooling1d(self, klayer, kclayer): bigdl_type="float" ) seq.add(blayer) + seq.add(BLayer.Squeeze(3)) return seq def create_averagepooling1d(self, klayer, kclayer): @@ -987,12 +1312,12 @@ def create_averagepooling1d(self, klayer, kclayer): bpadW, bpadH = self.to_bigdl_2d_padding(klayer.border_mode) seq = BLayer.Sequential() - seq.add(BLayer.View([1, int(input_shape[1]), int(input_shape[2])], num_input_dims=3)) + seq.add(BLayer.Reshape([int(input_shape[1]), 1, int(input_shape[2])], True)) blayer = BLayer.SpatialAveragePooling( - kw=klayer.pool_length, - kh=1, - dw=klayer.stride, - dh=1, + kw=1, + kh=klayer.pool_length, + dw=1, + dh=klayer.stride, pad_w=bpadW, pad_h=bpadH, global_pooling=False, @@ -1003,6 +1328,7 @@ def create_averagepooling1d(self, klayer, kclayer): bigdl_type="float" ) seq.add(blayer) + seq.add(BLayer.Squeeze(3)) return seq def create_globalaveragepooling2d(self, klayer, kclayer): @@ -1053,9 +1379,9 @@ def combo_parameter_layer(self, blayer, config): blayer.set_name(config["name"]) if hasattr(blayer, "set_init_method"): blayer.set_init_method(self.to_bigdl_init(config["init"]), - BInit.Zeros()) # Keras always set this to be zeros + BInit.Zeros()) # Keras always set this to be zeros # "linear" meaning do nothing - if config["activation"] != "linear" : + if config["activation"] != "linear": activation = self.to_bigdl_activation(config["activation"], "%s_%s" % (config["name"], config["activation"])) return self.fuse(blayer, activation) @@ -1068,10 +1394,18 @@ def to_bigdl_activation(self, activation_name, activation_id): activation = BLayer.Tanh() elif activation_name == "sigmoid": activation = BLayer.Sigmoid() + elif activation_name == "hard_sigmoid": + activation = BLayer.HardSigmoid() elif activation_name == "relu": activation = BLayer.ReLU() elif activation_name == "softmax": activation = BLayer.SoftMax() + elif activation_name == "softplus": + activation = BLayer.SoftPlus(beta=1.0) + elif activation_name == "softsign": + activation = BLayer.SoftSign() + elif activation_name == "linear": + activation = BLayer.Identity() else: raise Exception("Unsupported activation type: %s" % activation_name) activation.set_name(activation_id) @@ -1085,7 +1419,7 @@ def get_value_from_init(self, kinit_method, shape): else: raise Exception("We don't support % for now", kinit_method) - def to_bigdl_init(self, kinit_method): # kinit_method is a string + def to_bigdl_init(self, kinit_method): # kinit_method is a string init = None if kinit_method == "glorot_uniform": init = BInit.Xavier() @@ -1099,11 +1433,11 @@ def to_bigdl_init(self, kinit_method): # kinit_method is a string def to_bigdl_reg(self, reg): # reg is a dict if reg: - raise Exception("will support reg very soon") + return BRegularizer(reg['l1'], reg['l2']) else: return None - def fuse(self, src_blayer, activation): # activation is a string + def fuse(self, src_blayer, activation): # activation is a layer seq = BLayer.Sequential() seq.add(src_blayer) seq.add(activation) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index dcb4ed4f2ea..6f45aeb38bd 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -1977,6 +1977,26 @@ def __init__(self, super(CAddTable, self).__init__(None, bigdl_type, inplace) +class CAveTable(Layer): + + ''' + Merge the input tensors in the input table by element wise taking the average. The input + table is actually an array of tensor with same size. + + + :param inplace: reuse the input memory + + + >>> cAveTable = CAveTable(True) + creating: createCAveTable + ''' + + def __init__(self, + inplace=False, + bigdl_type="float"): + super(CAveTable, self).__init__(None, bigdl_type, + inplace) + class CDivTable(Layer): @@ -3744,6 +3764,57 @@ def __init__(self, pad_h) +class VolumetricAveragePooling(Layer): + + ''' + Applies 3D average-pooling operation in kTxkWxkH regions by step size dTxdWxdH. + The number of output features is equal to the number of input planes / dT. + The input can optionally be padded with zeros. Padding should be smaller than + half of kernel size. That is, padT < kT/2, padW < kW/2 and padH < kH/2 + + :param k_t: The kernel size + :param k_w: The kernel width + :param k_h: The kernel height + :param d_t: The step in the time dimension + :param d_w: The step in the width dimension + :param d_h: The step in the height dimension + :param pad_t: The padding in the time dimension + :param pad_w: The padding in the width dimension + :param pad_h: The padding in the height dimension + :param count_include_pad: whether to include padding when dividing the number of elements in pooling region + :param ceil_mode: whether the output size is to be ceiled or floored + + + >>> volumetricAveragePooling = VolumetricAveragePooling(5, 5, 5, 1, 1, 1) + creating: createVolumetricAveragePooling + ''' + + def __init__(self, + k_t, + k_w, + k_h, + d_t, + d_w, + d_h, + pad_t=0, + pad_w=0, + pad_h=0, + count_include_pad=True, + ceil_mode=False, + bigdl_type="float"): + super(VolumetricAveragePooling, self).__init__(None, bigdl_type, + k_t, + k_w, + k_h, + d_t, + d_w, + d_h, + pad_t, + pad_w, + pad_h, + count_include_pad, + ceil_mode) + class SpatialZeroPadding(Layer): ''' From ef9ff32a8c8dcba9f9f79e019a3fbdead0eea4eb Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Tue, 21 Nov 2017 11:09:43 +0800 Subject: [PATCH 346/823] Add more keras layers and test (#1891) * add, update and fix * add hard_sigmoid * fix python3.5 lambda in permute * fix syntax for python3.5 * fix pooling and conv layers * fix syntax * fix permute for py35 --- .../test/bigdl/keras/test_application.py | 17 ++ python/dllib/test/bigdl/keras/test_layer.py | 147 ++++++++++++++---- python/dllib/test/bigdl/test_utils.py | 36 ++++- 3 files changed, 166 insertions(+), 34 deletions(-) diff --git a/python/dllib/test/bigdl/keras/test_application.py b/python/dllib/test/bigdl/keras/test_application.py index ce60c7838fc..4c1ffa139c7 100644 --- a/python/dllib/test/bigdl/keras/test_application.py +++ b/python/dllib/test/bigdl/keras/test_application.py @@ -21,6 +21,7 @@ import pytest from keras.applications import * from bigdl.keras.converter import * +from keras.applications.music_tagger_crnn import MusicTaggerCRNN from test.bigdl.test_utils import BigDLTestCase, TestModels @@ -138,6 +139,22 @@ def test_vgg19(self): input_data = np.random.random([2, 3, 224, 224]) self.assert_model(input_data, kmodel) + def test_music_tagger_crnn(self): + # Remove the first BatchNormalization layer in the model as we don't support `axis=3` + # Set `inner_activation` in GRU to be `sigmoid` + keras.backend.set_image_dim_ordering("th") + kmodel = MusicTaggerCRNN(include_top=False) + input_data = np.random.random([2, 1, 96, 1366]) + + bmodel = DefinitionLoader.from_kmodel(kmodel) + WeightLoader.load_weights_from_kmodel(bmodel, kmodel, by_name=True) + + keras_output = kmodel.predict(input_data) + bmodel.training(is_training=False) + bigdl_output = bmodel.forward(input_data) + + self.assert_allclose(keras_output, bigdl_output, rtol=1e-6, atol=1e-6) + def test_inception_v3(self): keras.backend.set_image_dim_ordering("th") kmodel = inception_v3.InceptionV3(include_top=False, input_shape=(3, 299, 299)) diff --git a/python/dllib/test/bigdl/keras/test_layer.py b/python/dllib/test/bigdl/keras/test_layer.py index 6035c587211..a928bd827d5 100644 --- a/python/dllib/test/bigdl/keras/test_layer.py +++ b/python/dllib/test/bigdl/keras/test_layer.py @@ -24,15 +24,24 @@ from keras.layers import Dense, Input from bigdl.keras.converter import * from test.bigdl.test_utils import BigDLTestCase -from keras.metrics import * +from keras.regularizers import l1, l2, l1l2 class TestLayer(BigDLTestCase): def test_dense(self): input_data = np.random.random_sample([1, 10]) - dense = Dense(2, init='one', activation="relu", input_shape=(10, )) - self.modelTestSingleLayer(input_data, dense, dump_weights=True) + layer = Dense(2, init='one', activation="relu", + input_shape=(10, ), W_regularizer=l1l2(l1=0.01, l2=0.02)) + self.modelTestSingleLayer(input_data, layer, dump_weights=True) + layer2 = Dense(2, init='one', activation="softplus", + input_shape=(10, ), b_regularizer=l2(0.02)) + self.modelTestSingleLayer(input_data, layer2, dump_weights=True) + layer3 = Dense(2, init='one', input_shape=(10, ), + W_regularizer=keras.regularizers.WeightRegularizer(l1=0.1)) + self.modelTestSingleLayer(input_data, layer3, dump_weights=True) + layer4 = Dense(2, init='glorot_uniform', activation="hard_sigmoid", input_shape=(10, )) + self.modelTestSingleLayer(input_data, layer4, dump_weights=True) def test_timedistributeddense(self): input_data = np.random.random_sample([2, 4, 5]) @@ -106,17 +115,67 @@ def test_conv2D(self): layer, dump_weights=True, rtol=1e-5, atol=1e-5) + def test_conv3D(self): + input_data = np.random.random_sample([1, 3, 32, 32, 32]) + layer = Convolution3D(12, 5, 3, 4, dim_ordering="th", + border_mode="valid", subsample=(1, 1, 2), + input_shape=(3, 32, 32, 32)) + self.modelTestSingleLayer(input_data, layer, + dump_weights=True, rtol=1e-5, atol=1e-5) + + def test_atrousconvolution1d(self): + input_data = np.random.random_sample([2, 10, 32]) + layer = AtrousConvolution1D(64, 3, atrous_rate=2, + border_mode="valid", activation='relu', + input_shape=(10, 32)) + self.modelTestSingleLayer(input_data, layer, dump_weights=True) + + def test_atrousconvolution2d(self): + input_data = np.random.random_sample([1, 3, 128, 128]) + layer = AtrousConvolution2D(64, 3, 4, atrous_rate=(2, 2), dim_ordering="th", + border_mode="valid", activation='tanh', + input_shape=(3, 128, 128)) + self.modelTestSingleLayer(input_data, layer, dump_weights=True) + + def test_deconvolution2d(self): + input_data = np.random.random_sample([32, 3, 12, 12]) + layer = Deconvolution2D(3, 3, 3, output_shape=(None, 3, 14, 14), + border_mode="valid", dim_ordering="th", + input_shape=(3, 12, 12)) + self.modelTestSingleLayer(input_data, layer, dump_weights=True) + layer2 = Deconvolution2D(3, 3, 3, output_shape=(None, 3, 25, 25), + border_mode="valid", subsample=(2, 2), + dim_ordering="th", input_shape=(3, 12, 12)) + self.modelTestSingleLayer(input_data, layer2, dump_weights=True) + + def test_maxpooling3d(self): + input_data = np.random.random_sample([1, 3, 20, 15, 35]) + layer = MaxPooling3D(pool_size=(2, 2, 4), strides=(3, 1, 5), dim_ordering="th", + border_mode="valid", input_shape=(3, 20, 15, 35)) + self.modelTestSingleLayer(input_data, layer) + def test_maxpooling2d(self): input_data = np.random.random_sample([1, 3, 20, 20]) - layer = lambda: MaxPooling2D(pool_size=[3, 3], strides=[2, 2], + layer = lambda: MaxPooling2D(pool_size=[2, 3], strides=[4, 2], border_mode="valid", input_shape=(3, 20, 20)) self.modelTestSingleLayerWithOrdersModes(input_data, layer) + layer2 = lambda: MaxPooling2D(pool_size=[1, 1], strides=[2, 2], + border_mode="valid", input_shape=(3, 20, 20)) + self.modelTestSingleLayerWithOrdersModes(input_data, layer2) def test_maxpooling1d(self): - input_data = np.random.random_sample([1, 3, 20]) - layer = MaxPooling1D(pool_length=2, stride=None, - border_mode='valid', - input_shape=(3, 20),) + input_data = np.random.random_sample([5, 96, 64]) + layer = MaxPooling1D(pool_length=4, stride=None, + border_mode='valid', input_shape=(96, 64)) + self.modelTestSingleLayer(input_data, layer) + input_data2 = np.random.random_sample([1, 3, 20]) + layer2 = MaxPooling1D(pool_length=2, stride=None, + border_mode='valid', input_shape=(3, 20)) + self.modelTestSingleLayer(input_data2, layer2) + + def test_globalmaxpooling3d(self): + input_data = np.random.random_sample([1, 5, 20, 25, 35]) + layer = GlobalMaxPooling3D(dim_ordering="th", input_shape=(5, 20, 25, 35)) self.modelTestSingleLayer(input_data, layer) def test_globalmaxpooling2d(self): @@ -130,22 +189,36 @@ def test_globalmaxpooling1d(self): layer = GlobalMaxPooling1D(input_shape=(3, 20)) self.modelTestSingleLayer(input_data, layer) + def test_averagepooling3d(self): + input_data = np.random.random_sample([2, 6, 20, 15, 35]) + layer = AveragePooling3D(pool_size=(2, 3, 4), strides=(3, 1, 5), dim_ordering="th", + border_mode="valid", input_shape=(3, 20, 15, 35)) + self.modelTestSingleLayer(input_data, layer) + def test_averagepooling2d(self): input_data = np.random.random_sample([1, 3, 20, 20]) - layer = lambda: AveragePooling2D(pool_size=[3, 3], strides=[2, 2], + layer = lambda: AveragePooling2D(pool_size=[2, 3], strides=[4, 2], border_mode="valid", input_shape=(3, 20, 20)) self.modelTestSingleLayerWithOrdersModes(input_data, layer) + layer2 = lambda: AveragePooling2D(pool_size=[1, 1], strides=[2, 2], + border_mode="valid", input_shape=(3, 20, 20)) + self.modelTestSingleLayerWithOrdersModes(input_data, layer2) def test_averagepooling1d(self): - input_data = np.random.random_sample([1, 3, 20]) - layer = lambda: AveragePooling1D(pool_length=2, stride=None, - border_mode='valid', input_shape=(3, 20)) + input_data = np.random.random_sample([5, 96, 64]) + layer = lambda: AveragePooling1D(pool_length=4, stride=None, + border_mode='valid', input_shape=(96, 64)) self.modelTestSingleLayerWithOrdersModes(input_data, layer, dim_orderings=["tf"]) + def test_globalaveragepooling3d(self): + input_data = np.random.random_sample([1, 5, 20, 25, 35]) + layer = GlobalAveragePooling3D(dim_ordering="th", input_shape=(5, 20, 25, 35)) + self.modelTestSingleLayer(input_data, layer, rtol=1e-5, atol=1e-5) + def test_globalaveragepooling2d(self): input_data = np.random.random_sample([1, 3, 20, 20]) layer = lambda: GlobalAveragePooling2D(input_shape=(3, 20, 20)) - self.modelTestSingleLayerWithOrdersModes(input_data, layer, rtol=1e-6, atol=1e-6, + self.modelTestSingleLayerWithOrdersModes(input_data, layer, border_modes=[None]) def test_globalaveragepooling1d(self): @@ -156,14 +229,24 @@ def test_globalaveragepooling1d(self): def test_batchnormalization(self): input_data = np.random.random_sample([2, 6, 128, 128]) layer = BatchNormalization(input_shape=(6, 128, 128), axis=1) - self.modelTestSingleLayer(input_data, layer, - dump_weights=True) + self.modelTestSingleLayer(input_data, layer, dump_weights=True, random_weights=False) def test_flatten(self): input_data = np.random.random_sample([1, 2, 3]) layer = Flatten(input_shape=(2, 3)) self.modelTestSingleLayer(input_data, layer) + def test_permute(self): + input_data = np.random.random_sample([5, 4, 3, 2, 6]) + layer1 = Permute((4, 1, 2, 3), input_shape=(4, 3, 2, 6)) + self.modelTestSingleLayer(input_data, layer1) + layer2 = Permute((4, 3, 2, 1), input_shape=(4, 3, 2, 6)) + self.modelTestSingleLayer(input_data, layer2) + layer3 = Permute((1, 2, 3, 4), input_shape=(4, 3, 2, 6)) + self.modelTestSingleLayer(input_data, layer3) + layer4 = Permute((1, 4, 2, 3), input_shape=(4, 3, 2, 6)) + self.modelTestSingleLayer(input_data, layer4) + def test_reshape(self): input_data = np.random.random_sample([1, 3, 5, 4]) layer = Reshape(target_shape=(3, 20), input_shape=(3, 5, 4)) @@ -232,10 +315,20 @@ def test_merge_max(self): np.random.random([2, 3, 6, 7])] self.modelTestSingleLayer(input_data, layer, functional_apis=[False]) + def test_merge_ave(self): + inputLayer1 = InputLayer(input_shape=(3, 6, 7)) + inputLayer2 = InputLayer(input_shape=(3, 6, 7)) + inputLayer3 = InputLayer(input_shape=(3, 6, 7)) + + layer = Merge([inputLayer1, inputLayer2, inputLayer3], mode='ave') + input_data = [np.random.random([2, 3, 6, 7]), + np.random.random([2, 3, 6, 7]), + np.random.random([2, 3, 6, 7])] + self.modelTestSingleLayer(input_data, layer, functional_apis=[False]) + def test_merge_dot(self): - # use batch_input_shape for merge dot - inputLayer1 = InputLayer(batch_input_shape=(2, 3)) - inputLayer2 = InputLayer(batch_input_shape=(2, 3)) + inputLayer1 = InputLayer(input_shape=(3, )) + inputLayer2 = InputLayer(input_shape=(3, )) layer = Merge([inputLayer1, inputLayer2], mode='dot') input_data = [np.random.random([2, 3]), @@ -265,7 +358,7 @@ def test_thresholdedrelu(self): def test_parametricsoftplus(self): input_data = np.random.random_sample([1, 2, 3]) layer = ParametricSoftplus(alpha_init=0.4, beta_init=2.5, input_shape=(2, 3)) - self.modelTestSingleLayer(input_data, layer) + self.modelTestSingleLayer(input_data, layer, random_weights=False) def test_zeropadding1d(self): input_data = np.random.uniform(0, 1, [3, 2, 3]) @@ -300,15 +393,11 @@ def test_cropping1d(self): def test_simplernn(self): input_data = np.random.random([3, 4, 5]) layer = SimpleRNN(5, input_shape=(4, 5), return_sequences=True) - self.modelTestSingleLayer(input_data, layer, - dump_weights=True, rtol=1e-6, atol=1e-6) - + self.modelTestSingleLayer(input_data, layer, dump_weights=True) layer2 = SimpleRNN(3, input_shape=(4, 5), return_sequences=False) - self.modelTestSingleLayer(input_data, layer2, - dump_weights=True, rtol=1e-6, atol=1e-6) + self.modelTestSingleLayer(input_data, layer2, dump_weights=True) layer3 = SimpleRNN(3, input_shape=(4, 5), activation='relu') - self.modelTestSingleLayer(input_data, layer3, - dump_weights=True, rtol=1e-6, atol=1e-6) + self.modelTestSingleLayer(input_data, layer3, dump_weights=True) def test_lstm(self): input_data = np.random.random([3, 4, 5]) @@ -317,6 +406,12 @@ def test_lstm(self): layer2 = LSTM(3, input_shape=(4, 5), return_sequences=False, inner_activation='sigmoid') self.modelTestSingleLayer(input_data, layer2, dump_weights=True) + def test_convlstm2d(self): + input_data = np.random.random_sample([4, 8, 40, 40, 32]) + layer = ConvLSTM2D(32, 4, 4, input_shape=(8, 40, 40, 32), return_sequences=True, + inner_activation='sigmoid', border_mode='same') + self.modelTestSingleLayer(input_data, layer, dump_weights=True, random_weights=True) + def test_gru(self): input_data = np.random.random([3, 4, 5]) layer = GRU(4, input_shape=(4, 5), return_sequences=True, inner_activation='sigmoid') diff --git a/python/dllib/test/bigdl/test_utils.py b/python/dllib/test/bigdl/test_utils.py index 5a0ade9b422..8676d8a86f7 100644 --- a/python/dllib/test/bigdl/test_utils.py +++ b/python/dllib/test/bigdl/test_utils.py @@ -173,13 +173,27 @@ def __generate_keras_model(self, functional_api, input_data, output_layer): keras_model = self.__generate_sequence(input_data, output_layer) return keras_model + def __generate_random_weights(self, weights): + # weights: a list of ndarray; weights from keras + # Randomly generate a new list with the same shape to serve as the new weights for testing + new_weights = [] + for weight in weights: + new_weights.append(np.random.random(list(weight.shape))) + return new_weights + def modelTest(self, input_data, keras_model, + random_weights=True, dump_weights=False, is_training=False, - rtol=1e-7, - atol=1e-7): + rtol=1e-6, + atol=1e-6): + if random_weights: + # Randomly generate weights instead of using initial weights + kweights = keras_model.get_weights() + new_kweights = self.__generate_random_weights(kweights) + keras_model.set_weights(new_kweights) # weight_converter is a function keras [ndarray]-> bigdl [ndarray] keras_model_json_path, keras_model_hdf5_path = self._dump_keras(keras_model, dump_weights) bigdl_model = DefinitionLoader.from_json_path(keras_model_json_path) @@ -214,10 +228,11 @@ def modelTestSingleLayerWithOrdersModes(self, output_layer_creator, # a keras layer dim_orderings=["tf", "th"], border_modes=["valid", "same"], + random_weights=True, dump_weights=False, is_training=False, - rtol=1e-7, - atol=1e-7): + rtol=1e-6, + atol=1e-6): for dim_ordering in dim_orderings: print("Testing with dim_ordering %s" % dim_ordering) keras.backend.set_image_dim_ordering(dim_ordering) @@ -233,6 +248,7 @@ def modelTestSingleLayerWithOrdersModes(self, raise Exception("cannot set border_mode for %s" % output_layer) self.modelTestSingleLayer(input_data, output_layer, # a keras layer + random_weights, dump_weights, is_training, rtol, @@ -241,16 +257,18 @@ def modelTestSingleLayerWithOrdersModes(self, def modelTestSingleLayer(self, input_data, output_layer, # a keras layer + random_weights=True, dump_weights=False, is_training=False, - rtol=1e-7, - atol=1e-7, + rtol=1e-6, + atol=1e-6, functional_apis=[True, False]): for api in functional_apis: self._do_modelTestSingleLayer( input_data, output_layer, # a keras layer functional_api=api, + random_weights=random_weights, dump_weights=dump_weights, is_training=is_training, rtol=rtol, @@ -260,15 +278,17 @@ def _do_modelTestSingleLayer(self, input_data, output_layer, # a keras layer functional_api=True, + random_weights=True, dump_weights=False, is_training=False, - rtol=1e-7, - atol=1e-7): + rtol=1e-6, + atol=1e-6): keras_model = self.__generate_keras_model(functional_api=functional_api, input_data=input_data, output_layer=output_layer) self.modelTest(input_data, keras_model, + random_weights=random_weights, dump_weights=dump_weights, is_training=is_training, rtol=rtol, From 60e067bbd0363daf446420d3abab85b7d5fb267c Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Thu, 23 Nov 2017 15:12:44 +0800 Subject: [PATCH 347/823] Add tensorflow transfer learning example (#1895) * add transfer learning example * add readme * rm tensorflow files * make getRDD public * meet code review * update doc * fix tests * add a link in document * refine doc --- python/dllib/src/bigdl/dllib/nn/layer.py | 11 +++- .../dllib/src/bigdl/dllib/utils/tf_utils.py | 59 ++++++++----------- 2 files changed, 34 insertions(+), 36 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 6f45aeb38bd..06d2772643d 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -640,7 +640,7 @@ def __init__(self, else: from bigdl.util.tf_utils import convert model = convert(to_list(inputs), to_list(outputs), byte_order, bigdl_type) - super(Model, self).__init__(model, bigdl_type) + super(Model, self).__init__(model.value, bigdl_type) @staticmethod @@ -734,13 +734,18 @@ def load_caffe_model(defPath, modelPath, bigdl_type="float"): return Layer.of(jmodel) @staticmethod - def load_tensorflow(path, inputs, outputs, byte_order = "little_endian", bigdl_type="float"): + def load_tensorflow(path, inputs, outputs, byte_order = "little_endian", + bin_file = None, bigdl_type="float"): """ Load a pre-trained Tensorflow model. :param path: The path containing the pre-trained model. + :param inputs: The input node of this graph + :param outputs: The output node of this graph + :param byte_order: byte_order of the file, `little_endian` or `big_endian` + :param bin_file: the optional bin file produced by bigdl dump_model util function to store the weights :return: A pre-trained model. """ - jmodel = callBigDlFunc(bigdl_type, "loadTF", path, inputs, outputs, byte_order) + jmodel = callBigDlFunc(bigdl_type, "loadTF", path, inputs, outputs, byte_order, bin_file) return Model.of(jmodel) @staticmethod diff --git a/python/dllib/src/bigdl/dllib/utils/tf_utils.py b/python/dllib/src/bigdl/dllib/utils/tf_utils.py index f1d294cb368..c19a508d79e 100644 --- a/python/dllib/src/bigdl/dllib/utils/tf_utils.py +++ b/python/dllib/src/bigdl/dllib/utils/tf_utils.py @@ -57,27 +57,19 @@ def convert(input_ops, output_ops, byte_order, bigdl_type): Convert tensorflow model to bigdl model :param input_ops: operation list used for input, should be placeholders :param output_ops: operations list used for output - :param sess: current tensorflow session :return: bigdl model """ - sess = tf.Session() - init = tf.global_variables_initializer() - sess.run(init) input_names = map(lambda x: x.name.split(":")[0], input_ops) output_names = map(lambda x: x.name.split(":")[0], output_ops) temp = tempfile.mkdtemp() - saver = tf.train.Saver() - saver.save(sess, temp + '/model.chkp') - tf.train.write_graph(sess.graph, temp, 'model.pbtxt') - - merge_checkpoint(temp + '/model.pbtxt', - temp + '/model.chkp', - output_names, - temp + '/model.pb', sess) + dump_model(path=temp) + model_path = temp + '/model.pb' + bin_path = temp + '/model.bin' - model = Model.load_tensorflow(temp + '/model.pb', input_names, output_names, byte_order, bigdl_type) + model = Model.load_tensorflow(model_path, input_names, output_names, + byte_order, bin_path, bigdl_type) try: shutil.rmtree(temp) @@ -129,7 +121,7 @@ def save_variable_bigdl(tensors, target_path, bigdl_type="float"): callBigDlFunc(bigdl_type, "saveTensorDictionary", jtensors, target_path) -def dump_model(path, sess=None, graph=None, bigdl_type="float"): +def dump_model(path, graph=None, sess=None, ckpt_file=None, bigdl_type="float"): """ Dump a tensorflow model to files. The graph will be dumped to path/model.pb, and the checkpoint will be dumped to path/model.bin @@ -142,33 +134,34 @@ def dump_model(path, sess=None, graph=None, bigdl_type="float"): :return: nothing """ if not os.path.isdir(path): - print("Folder " + path + " does not exist") - raise - - if sess is None: - sess = tf.Session() - init = tf.global_variables_initializer() - sess.run(init) - - temp = tempfile.mkdtemp() - # dump checkpoint to temp files - checkpoint = temp + '/model.chkp' - saver = tf.train.Saver() - saver.save(sess, checkpoint) + raise ValueError("Folder " + path + " does not exist") + + temp = None + if ckpt_file is None: + if sess is None: + sess = tf.Session() + init = tf.global_variables_initializer() + sess.run(init) + temp = tempfile.mkdtemp() + ckpt_file = temp + # dump checkpoint to temp files + saver = tf.train.Saver() + saver.save(sess, ckpt_file) # generate bin files - tensors = export_checkpoint(checkpoint) + tensors = export_checkpoint(ckpt_file) save_variable_bigdl(tensors, path + "/model.bin", bigdl_type) # dump grap to pb file graph = sess.graph if graph is None else graph with gfile.GFile(path + "/model.pb", "wb") as f: f.write(graph.as_graph_def().SerializeToString()) - try: - shutil.rmtree(temp) - except OSError as e: - if e.errno != errno.ENOENT: - raise + if temp is not None: + try: + shutil.rmtree(temp) + except OSError as e: + if e.errno != errno.ENOENT: + raise def merge_checkpoint(input_graph, From 8e6f4ca4ead009f7cef206f05a84bb4a4ae14872 Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Thu, 23 Nov 2017 15:12:44 +0800 Subject: [PATCH 348/823] Add tensorflow transfer learning example (#1895) * add transfer learning example * add readme * rm tensorflow files * make getRDD public * meet code review * update doc * fix tests * add a link in document * refine doc --- python/dllib/src/bigdl/dllib/nn/layer.py | 11 +++- .../dllib/src/bigdl/dllib/utils/tf_utils.py | 59 ++++++++----------- 2 files changed, 34 insertions(+), 36 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 6f45aeb38bd..06d2772643d 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -640,7 +640,7 @@ def __init__(self, else: from bigdl.util.tf_utils import convert model = convert(to_list(inputs), to_list(outputs), byte_order, bigdl_type) - super(Model, self).__init__(model, bigdl_type) + super(Model, self).__init__(model.value, bigdl_type) @staticmethod @@ -734,13 +734,18 @@ def load_caffe_model(defPath, modelPath, bigdl_type="float"): return Layer.of(jmodel) @staticmethod - def load_tensorflow(path, inputs, outputs, byte_order = "little_endian", bigdl_type="float"): + def load_tensorflow(path, inputs, outputs, byte_order = "little_endian", + bin_file = None, bigdl_type="float"): """ Load a pre-trained Tensorflow model. :param path: The path containing the pre-trained model. + :param inputs: The input node of this graph + :param outputs: The output node of this graph + :param byte_order: byte_order of the file, `little_endian` or `big_endian` + :param bin_file: the optional bin file produced by bigdl dump_model util function to store the weights :return: A pre-trained model. """ - jmodel = callBigDlFunc(bigdl_type, "loadTF", path, inputs, outputs, byte_order) + jmodel = callBigDlFunc(bigdl_type, "loadTF", path, inputs, outputs, byte_order, bin_file) return Model.of(jmodel) @staticmethod diff --git a/python/dllib/src/bigdl/dllib/utils/tf_utils.py b/python/dllib/src/bigdl/dllib/utils/tf_utils.py index f1d294cb368..c19a508d79e 100644 --- a/python/dllib/src/bigdl/dllib/utils/tf_utils.py +++ b/python/dllib/src/bigdl/dllib/utils/tf_utils.py @@ -57,27 +57,19 @@ def convert(input_ops, output_ops, byte_order, bigdl_type): Convert tensorflow model to bigdl model :param input_ops: operation list used for input, should be placeholders :param output_ops: operations list used for output - :param sess: current tensorflow session :return: bigdl model """ - sess = tf.Session() - init = tf.global_variables_initializer() - sess.run(init) input_names = map(lambda x: x.name.split(":")[0], input_ops) output_names = map(lambda x: x.name.split(":")[0], output_ops) temp = tempfile.mkdtemp() - saver = tf.train.Saver() - saver.save(sess, temp + '/model.chkp') - tf.train.write_graph(sess.graph, temp, 'model.pbtxt') - - merge_checkpoint(temp + '/model.pbtxt', - temp + '/model.chkp', - output_names, - temp + '/model.pb', sess) + dump_model(path=temp) + model_path = temp + '/model.pb' + bin_path = temp + '/model.bin' - model = Model.load_tensorflow(temp + '/model.pb', input_names, output_names, byte_order, bigdl_type) + model = Model.load_tensorflow(model_path, input_names, output_names, + byte_order, bin_path, bigdl_type) try: shutil.rmtree(temp) @@ -129,7 +121,7 @@ def save_variable_bigdl(tensors, target_path, bigdl_type="float"): callBigDlFunc(bigdl_type, "saveTensorDictionary", jtensors, target_path) -def dump_model(path, sess=None, graph=None, bigdl_type="float"): +def dump_model(path, graph=None, sess=None, ckpt_file=None, bigdl_type="float"): """ Dump a tensorflow model to files. The graph will be dumped to path/model.pb, and the checkpoint will be dumped to path/model.bin @@ -142,33 +134,34 @@ def dump_model(path, sess=None, graph=None, bigdl_type="float"): :return: nothing """ if not os.path.isdir(path): - print("Folder " + path + " does not exist") - raise - - if sess is None: - sess = tf.Session() - init = tf.global_variables_initializer() - sess.run(init) - - temp = tempfile.mkdtemp() - # dump checkpoint to temp files - checkpoint = temp + '/model.chkp' - saver = tf.train.Saver() - saver.save(sess, checkpoint) + raise ValueError("Folder " + path + " does not exist") + + temp = None + if ckpt_file is None: + if sess is None: + sess = tf.Session() + init = tf.global_variables_initializer() + sess.run(init) + temp = tempfile.mkdtemp() + ckpt_file = temp + # dump checkpoint to temp files + saver = tf.train.Saver() + saver.save(sess, ckpt_file) # generate bin files - tensors = export_checkpoint(checkpoint) + tensors = export_checkpoint(ckpt_file) save_variable_bigdl(tensors, path + "/model.bin", bigdl_type) # dump grap to pb file graph = sess.graph if graph is None else graph with gfile.GFile(path + "/model.pb", "wb") as f: f.write(graph.as_graph_def().SerializeToString()) - try: - shutil.rmtree(temp) - except OSError as e: - if e.errno != errno.ENOENT: - raise + if temp is not None: + try: + shutil.rmtree(temp) + except OSError as e: + if e.errno != errno.ENOENT: + raise def merge_checkpoint(input_graph, From cb379ab2ad9351307e93c6392ac210fc9035b5e4 Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Fri, 24 Nov 2017 02:13:19 -0600 Subject: [PATCH 349/823] Support nested Model and enable test_application (#1927) * good * update * up * goodgood * clean code * fix and enable application test --- .../dllib/src/bigdl/dllib/keras/converter.py | 163 ++++++++++-------- python/dllib/src/bigdl/dllib/nn/layer.py | 6 + 2 files changed, 93 insertions(+), 76 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/keras/converter.py b/python/dllib/src/bigdl/dllib/keras/converter.py index ab35e86a72f..238441b98d4 100644 --- a/python/dllib/src/bigdl/dllib/keras/converter.py +++ b/python/dllib/src/bigdl/dllib/keras/converter.py @@ -8,7 +8,7 @@ import bigdl.util.common as bcommon import keras.optimizers as koptimizers from keras.models import model_from_json -from keras.models import Sequential, Model +from keras.models import Sequential, Model, Layer import keras import warnings @@ -19,51 +19,24 @@ def unsupport_exp(name): class WeightLoader: - @staticmethod - def __load_weights_by_execution_seq(bmodel, kmodel): - blayers = [l for l in bmodel.layers if l.is_with_weights()] - klayers = [l for l in kmodel.layers if l.get_weights()] - if len(blayers) != len(klayers): - raise Exception( - "keras with %s layers but bigdl with %s layers" % (len(klayers), len(blayers))) - for b, k in zip(blayers, klayers): - if b.name() != k.name: - raise Exception("Found different layer in execution order, bigdl:%s, keras: %s" % (b.name(), k.name)) # noqa - bigdl_weights = WeightsConverter.get_bigdl_weigths_from_keras(k) - b.set_weights(bigdl_weights) - # TODO: add more unitest + # bmodel and kmodel should have the same layers. + # and this method should only be called when bmodel is generated by kmodel @staticmethod - def __load_weights_by_name(bmodel, kmodel, by_name=False): + def load_weights_from_kmodel(bmodel, kmodel): keras_name_to_layer = WeightLoader.__keras_name_to_Layers(kmodel, with_weights=True) bigdl_name_to_layer = WeightLoader.__bigdl_name_to_Layers(bmodel, with_weights=True) - layers_not_in_keras = set(bigdl_name_to_layer.keys()) - set(keras_name_to_layer.keys()) - if layers_not_in_keras: - raise Exception("Layers %s can be found in bigdl, but not in keras" % repr(layers_not_in_keras)) # noqa - layers_not_in_bigdl = set(keras_name_to_layer.keys()) - set(bigdl_name_to_layer.keys()) - if layers_not_in_bigdl: - if by_name: - warnings.warn("Ignore weight of layers %s as it cannot be found in bigdl" % repr(layers_not_in_bigdl)) # noqa - else: - raise Exception("Layers %s can be found in bigdl, but not in keras" % repr(layers_not_in_keras)) # noqa - for blayer in bigdl_name_to_layer.values(): - if blayer.name() in keras_name_to_layer: - klayer = keras_name_to_layer[blayer.name()] - bigdl_weights = WeightsConverter.get_bigdl_weigths_from_keras(klayer) + # klayer should be just a layer, not seq, not Model + for klayer in keras_name_to_layer.values(): + if klayer.name in bigdl_name_to_layer: + blayer = bigdl_name_to_layer[klayer.name] + bigdl_weights = WeightsConverter.get_bigdl_weights_from_klayer(klayer) blayer.set_weights(bigdl_weights) if isinstance(klayer, keras.layers.BatchNormalization): blayer.set_running_mean(keras.backend.eval(klayer.running_mean)) blayer.set_running_std(keras.backend.eval(klayer.running_std)) - - @staticmethod - def load_weights_from_kmodel(bmodel, kmodel, by_name=False): - """ - Load weights from kmodel to bmodel - """ - if by_name: - WeightLoader.__load_weights_by_name(bmodel, kmodel) - else: - WeightLoader.__load_weights_by_execution_seq(bmodel, kmodel) + else: + raise Exception("should not enter here, klayer: %s", klayer) @staticmethod def load_weights_from_json_hdf5(def_json, weights_hdf5, by_name=False): @@ -88,23 +61,29 @@ def load_weights_from_hdf5(bmodel, kmodel, filepath, by_name=False): some of the layers have changed. ''' kmodel.load_weights(filepath=filepath, by_name=by_name) - WeightLoader.load_weights_from_kmodel(bmodel, kmodel, by_name=by_name) + WeightLoader.load_weights_from_kmodel(bmodel, kmodel) @staticmethod def __keras_name_to_Layers(model, with_weights=False): + total_layers = DefinitionLoader(model).node_id_to_layer.values() + if with_weights: - layers = [l for l in model.layers if l.get_weights()] + layers = [l for l in total_layers + if l.get_weights() and not isinstance(l, Model) and not isinstance(l, Sequential)] # noqa else: - layers = [l for l in model.layers] + layers = [l for l in total_layers if not isinstance(l, Model) and not isinstance(l, Sequential)] # noqa return dict([(layer.name, layer) for layer in layers]) @staticmethod def __bigdl_name_to_Layers(model, with_weights=False): + # NB: Container in BigDL is_with_weights() is true if one of the nested layer with_weights + # but in Keras container get_weights() return false even if the nested layer with_weights + all_layers = model.layers + model.flattened_layers if with_weights: - layers = [l for l in model.layers if l.is_with_weights()] + layers = [l for l in all_layers if l.is_with_weights()] else: - layers = [l for l in model.layers] + layers = all_layers return dict([(layer.name(), layer) for layer in layers]) @@ -131,15 +110,6 @@ def get_converter(class_name): def to_bigdl_weights(class_name, weights): return WeightsConverter.get_converter(class_name)(weights) - @staticmethod - def get_bigdl_weigths_from_keras(k): - if isinstance(k, keras.engine.Model): - return WeightsConverter.get_weights_from_kmodel(k) - elif isinstance(k, keras.engine.Layer): - return WeightsConverter.get_bigdl_weights_from_klayer(k) - else: - raise Exception("Unsupport type: %s", k) - @staticmethod def get_bigdl_weights_from_klayer(klayer): # we should use get_weights instead of klayer.weights @@ -242,6 +212,39 @@ def convert_gru(weights): class DefinitionLoader: + @staticmethod + def __build_node_id_2_klayer(kmodel, node_id_to_config_layer): + """ + The result would contain all of the layers including nested layers. + :param kmodel: a keras model which can be Sequential or Model + :param node_id_to_config_layer: a container to store the result + """ + node_id_to_config_layer[kmodel.name] = kmodel # include itself as well + def gather_result(layers): + if layers: # layers maybe None here. + for layer in layers: + if layer.name not in node_id_to_config_layer: + node_id_to_config_layer[layer.name] = layer + DefinitionLoader.__build_node_id_2_klayer(layer, node_id_to_config_layer) + if hasattr(kmodel, "layers"): + gather_result(kmodel.layers) + if hasattr(kmodel, "flattened_layers"): + gather_result(kmodel.flattened_layers) # it's a expensive operation + + @staticmethod + def __build_node_id_2_kclayer(kmodel, node_id_to_config_layer): + if isinstance(kmodel, Sequential): + for layer_config in kmodel.get_config(): + layer_name = layer_config["config"]["name"] + node_id_to_config_layer[layer_name] = layer_config + elif isinstance(kmodel, Model): + for layerConfig in kmodel.get_config()["layers"]: + node_id_to_config_layer[layerConfig["name"]] = layerConfig + elif isinstance(kmodel, Layer): + node_id_to_config_layer[kmodel.name] = kmodel.get_config() + else: + raise Exception("should not enter here: %s" % kmodel) + def __init__(self, kmodel): self.node_id_to_instance = {} self.node_id_to_layer = {} @@ -249,23 +252,20 @@ def __init__(self, kmodel): self.kmodel = kmodel self.kconfig = self.kmodel.get_config() - for layer in self.kmodel.layers: - self.node_id_to_layer[layer.name] = layer - - if isinstance(self.kmodel, Sequential): - for layer_config in self.kmodel.get_config(): - layer_name = layer_config["config"]["name"] - self.node_id_to_config_layer[layer_name] = layer_config - else: - for layerConfig in self.kconfig["layers"]: - self.node_id_to_config_layer[layerConfig["name"]] = layerConfig + DefinitionLoader.__build_node_id_2_klayer(kmodel, self.node_id_to_layer) + DefinitionLoader.__build_node_id_2_kclayer(kmodel, self.node_id_to_config_layer) def __to_bigdl(self): if isinstance(self.kmodel, Sequential): - bmodel = self._construct_bigdl_sequence() + bigdlmodel = self._construct_bigdl_sequence() elif isinstance(self.kmodel, Model): - bmodel = self._construct_bigdl_model() - return bmodel + bigdlmodel = self._construct_bigdl_model() + elif isinstance(self.kmodel, Layer): + bigdlmodel = LayerConverter().create(self.kmodel, + self.node_id_to_config_layer[self.kmodel.name]) + else: + raise Exception("Should not enter here: %s" % self.kmodel) + return bigdlmodel @classmethod def from_kmodel(cls, kmodel): @@ -306,7 +306,6 @@ def _do_create_node(self, layer, clayer): def _construct_bigdl_model(self): for clayer in self.kconfig["layers"]: if clayer["name"] not in self.node_id_to_instance: - self._do_create_node(self.node_id_to_layer[clayer["name"]], clayer) ins = [] @@ -323,6 +322,7 @@ def _construct_bigdl_sequence(self): bseq = BLayer.Sequential() layerConverter = LayerConverter() for layer in self.kmodel.layers: + # recursive logic is within create method. blayer = layerConverter.create(layer, self.node_id_to_config_layer[layer.name]) bseq.add(blayer) return bseq @@ -360,7 +360,7 @@ def __check_is_share_weights(self, kclayer): "%s doesn't support multiple inputs with shared weights" % kclayer["class_name"]) def create(self, klayer, kclayer): - class_name = kclayer["class_name"] + class_name = klayer.__class__.__name__ self.__check_is_share_weights(kclayer) @@ -379,10 +379,13 @@ def create(self, klayer, kclayer): blayer = api(klayer, kclayer) return blayer.set_name(klayer.name) - def create_model(self, klayer, kclyer): + def create_model(self, klayer, kclayer): + return DefinitionLoader.from_kmodel(klayer) + + def create_sequential(self, klayer, kclayer): return DefinitionLoader.from_kmodel(klayer) - def create_inputlayer(self, klayer, kclyer): + def create_inputlayer(self, klayer, kclayer): return BLayer.Identity() def create_dense(self, klayer, kclayer): @@ -448,7 +451,6 @@ def create_dropout(self, klayer, kclayer): return BLayer.Dropout(klayer.p) def create_flatten(self, klayer, kclayer): - self.__check_is_share_weights(kclayer) input_shape = klayer.input_shape blayer = BLayer.Reshape([int(np.prod(input_shape[1:]))], None) return blayer @@ -493,7 +495,6 @@ def exchangeNumbers(arr, i, j): return list(filter(lambda pair: pair[0] != pair[1], pairs)) def create_reshape(self, klayer, kclayer): - self.__check_is_share_weights(kclayer) blayer = BLayer.Reshape(klayer.target_shape, None) return blayer @@ -502,12 +503,14 @@ def create_repeatvector(self, klayer, kclayer): n_dim=1, bigdl_type="float") + def __is_from_sequential(self, klayer, kclayer): + return "layers" in kclayer["config"] and hasattr(klayer, "layers") and klayer.layers is not None # noqa + def create_merge(self, klayer, kclayer): - self.__check_is_share_weights(kclayer) input_shape = klayer.get_input_shape_at(0) if klayer.output_shape and not isinstance(klayer.output_shape, tuple): raise Exception("Only output_shape=None or a shape tuple is supported for now") - if klayer.node_indices: + if klayer.node_indices and not all(0 == i for i in klayer.node_indices): unsupport_exp("node_indices") if klayer.output_mask: unsupport_exp("output_mask") @@ -540,7 +543,17 @@ def create_merge(self, klayer, kclayer): raise Exception("Merge mode `%s` not supported for now" % klayer.mode) else: # invalid mode or lambda functions raise Exception("Invalid merge mode: `%s`. Lambda/function as merge mode is not supported for now." % klayer.mode) - return blayer + if self.__is_from_sequential(klayer, kclayer): + bseq = BLayer.Sequential() + parallel_table = BLayer.ParallelTable() + for l in klayer.layers: + bl = DefinitionLoader.from_kmodel(l) + parallel_table.add(bl) + bseq.add(parallel_table) + bseq.add(blayer) + return bseq + else: + return blayer def create_elu(self, klayer, kclayer): return BLayer.ELU(alpha=float(klayer.alpha), @@ -811,8 +824,6 @@ def create_gru(self, klayer, kclayer): def create_batchnormalization(self, klayer, kclayer): config = kclayer["config"] - - self.__check_is_share_weights(kclayer) if keras.backend.image_dim_ordering() != "th" or klayer.axis != 1: raise Exception("""For BatchNormalization, we only support th image ordering (i.e. NCHW) """ + """with axis = 1 for now, but the current order is %s and axis is %s diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 06d2772643d..79bbfe20779 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -597,6 +597,12 @@ def layers(self): layers = [Layer.of(jlayer) for jlayer in jlayers] return layers + @property + def flattened_layers(self): + jlayers = callBigDlFunc(self.bigdl_type, "getFlattenModules", self) + layers = [Layer.of(jlayer) for jlayer in jlayers] + return layers + class Model(Container): """ From 57dfed24ee206973713b6117b80e4e7514376fc3 Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Fri, 24 Nov 2017 02:13:19 -0600 Subject: [PATCH 350/823] Support nested Model and enable test_application (#1927) * good * update * up * goodgood * clean code * fix and enable application test --- .../dllib/src/bigdl/dllib/keras/converter.py | 163 ++++++++++-------- python/dllib/src/bigdl/dllib/nn/layer.py | 6 + 2 files changed, 93 insertions(+), 76 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/keras/converter.py b/python/dllib/src/bigdl/dllib/keras/converter.py index ab35e86a72f..238441b98d4 100644 --- a/python/dllib/src/bigdl/dllib/keras/converter.py +++ b/python/dllib/src/bigdl/dllib/keras/converter.py @@ -8,7 +8,7 @@ import bigdl.util.common as bcommon import keras.optimizers as koptimizers from keras.models import model_from_json -from keras.models import Sequential, Model +from keras.models import Sequential, Model, Layer import keras import warnings @@ -19,51 +19,24 @@ def unsupport_exp(name): class WeightLoader: - @staticmethod - def __load_weights_by_execution_seq(bmodel, kmodel): - blayers = [l for l in bmodel.layers if l.is_with_weights()] - klayers = [l for l in kmodel.layers if l.get_weights()] - if len(blayers) != len(klayers): - raise Exception( - "keras with %s layers but bigdl with %s layers" % (len(klayers), len(blayers))) - for b, k in zip(blayers, klayers): - if b.name() != k.name: - raise Exception("Found different layer in execution order, bigdl:%s, keras: %s" % (b.name(), k.name)) # noqa - bigdl_weights = WeightsConverter.get_bigdl_weigths_from_keras(k) - b.set_weights(bigdl_weights) - # TODO: add more unitest + # bmodel and kmodel should have the same layers. + # and this method should only be called when bmodel is generated by kmodel @staticmethod - def __load_weights_by_name(bmodel, kmodel, by_name=False): + def load_weights_from_kmodel(bmodel, kmodel): keras_name_to_layer = WeightLoader.__keras_name_to_Layers(kmodel, with_weights=True) bigdl_name_to_layer = WeightLoader.__bigdl_name_to_Layers(bmodel, with_weights=True) - layers_not_in_keras = set(bigdl_name_to_layer.keys()) - set(keras_name_to_layer.keys()) - if layers_not_in_keras: - raise Exception("Layers %s can be found in bigdl, but not in keras" % repr(layers_not_in_keras)) # noqa - layers_not_in_bigdl = set(keras_name_to_layer.keys()) - set(bigdl_name_to_layer.keys()) - if layers_not_in_bigdl: - if by_name: - warnings.warn("Ignore weight of layers %s as it cannot be found in bigdl" % repr(layers_not_in_bigdl)) # noqa - else: - raise Exception("Layers %s can be found in bigdl, but not in keras" % repr(layers_not_in_keras)) # noqa - for blayer in bigdl_name_to_layer.values(): - if blayer.name() in keras_name_to_layer: - klayer = keras_name_to_layer[blayer.name()] - bigdl_weights = WeightsConverter.get_bigdl_weigths_from_keras(klayer) + # klayer should be just a layer, not seq, not Model + for klayer in keras_name_to_layer.values(): + if klayer.name in bigdl_name_to_layer: + blayer = bigdl_name_to_layer[klayer.name] + bigdl_weights = WeightsConverter.get_bigdl_weights_from_klayer(klayer) blayer.set_weights(bigdl_weights) if isinstance(klayer, keras.layers.BatchNormalization): blayer.set_running_mean(keras.backend.eval(klayer.running_mean)) blayer.set_running_std(keras.backend.eval(klayer.running_std)) - - @staticmethod - def load_weights_from_kmodel(bmodel, kmodel, by_name=False): - """ - Load weights from kmodel to bmodel - """ - if by_name: - WeightLoader.__load_weights_by_name(bmodel, kmodel) - else: - WeightLoader.__load_weights_by_execution_seq(bmodel, kmodel) + else: + raise Exception("should not enter here, klayer: %s", klayer) @staticmethod def load_weights_from_json_hdf5(def_json, weights_hdf5, by_name=False): @@ -88,23 +61,29 @@ def load_weights_from_hdf5(bmodel, kmodel, filepath, by_name=False): some of the layers have changed. ''' kmodel.load_weights(filepath=filepath, by_name=by_name) - WeightLoader.load_weights_from_kmodel(bmodel, kmodel, by_name=by_name) + WeightLoader.load_weights_from_kmodel(bmodel, kmodel) @staticmethod def __keras_name_to_Layers(model, with_weights=False): + total_layers = DefinitionLoader(model).node_id_to_layer.values() + if with_weights: - layers = [l for l in model.layers if l.get_weights()] + layers = [l for l in total_layers + if l.get_weights() and not isinstance(l, Model) and not isinstance(l, Sequential)] # noqa else: - layers = [l for l in model.layers] + layers = [l for l in total_layers if not isinstance(l, Model) and not isinstance(l, Sequential)] # noqa return dict([(layer.name, layer) for layer in layers]) @staticmethod def __bigdl_name_to_Layers(model, with_weights=False): + # NB: Container in BigDL is_with_weights() is true if one of the nested layer with_weights + # but in Keras container get_weights() return false even if the nested layer with_weights + all_layers = model.layers + model.flattened_layers if with_weights: - layers = [l for l in model.layers if l.is_with_weights()] + layers = [l for l in all_layers if l.is_with_weights()] else: - layers = [l for l in model.layers] + layers = all_layers return dict([(layer.name(), layer) for layer in layers]) @@ -131,15 +110,6 @@ def get_converter(class_name): def to_bigdl_weights(class_name, weights): return WeightsConverter.get_converter(class_name)(weights) - @staticmethod - def get_bigdl_weigths_from_keras(k): - if isinstance(k, keras.engine.Model): - return WeightsConverter.get_weights_from_kmodel(k) - elif isinstance(k, keras.engine.Layer): - return WeightsConverter.get_bigdl_weights_from_klayer(k) - else: - raise Exception("Unsupport type: %s", k) - @staticmethod def get_bigdl_weights_from_klayer(klayer): # we should use get_weights instead of klayer.weights @@ -242,6 +212,39 @@ def convert_gru(weights): class DefinitionLoader: + @staticmethod + def __build_node_id_2_klayer(kmodel, node_id_to_config_layer): + """ + The result would contain all of the layers including nested layers. + :param kmodel: a keras model which can be Sequential or Model + :param node_id_to_config_layer: a container to store the result + """ + node_id_to_config_layer[kmodel.name] = kmodel # include itself as well + def gather_result(layers): + if layers: # layers maybe None here. + for layer in layers: + if layer.name not in node_id_to_config_layer: + node_id_to_config_layer[layer.name] = layer + DefinitionLoader.__build_node_id_2_klayer(layer, node_id_to_config_layer) + if hasattr(kmodel, "layers"): + gather_result(kmodel.layers) + if hasattr(kmodel, "flattened_layers"): + gather_result(kmodel.flattened_layers) # it's a expensive operation + + @staticmethod + def __build_node_id_2_kclayer(kmodel, node_id_to_config_layer): + if isinstance(kmodel, Sequential): + for layer_config in kmodel.get_config(): + layer_name = layer_config["config"]["name"] + node_id_to_config_layer[layer_name] = layer_config + elif isinstance(kmodel, Model): + for layerConfig in kmodel.get_config()["layers"]: + node_id_to_config_layer[layerConfig["name"]] = layerConfig + elif isinstance(kmodel, Layer): + node_id_to_config_layer[kmodel.name] = kmodel.get_config() + else: + raise Exception("should not enter here: %s" % kmodel) + def __init__(self, kmodel): self.node_id_to_instance = {} self.node_id_to_layer = {} @@ -249,23 +252,20 @@ def __init__(self, kmodel): self.kmodel = kmodel self.kconfig = self.kmodel.get_config() - for layer in self.kmodel.layers: - self.node_id_to_layer[layer.name] = layer - - if isinstance(self.kmodel, Sequential): - for layer_config in self.kmodel.get_config(): - layer_name = layer_config["config"]["name"] - self.node_id_to_config_layer[layer_name] = layer_config - else: - for layerConfig in self.kconfig["layers"]: - self.node_id_to_config_layer[layerConfig["name"]] = layerConfig + DefinitionLoader.__build_node_id_2_klayer(kmodel, self.node_id_to_layer) + DefinitionLoader.__build_node_id_2_kclayer(kmodel, self.node_id_to_config_layer) def __to_bigdl(self): if isinstance(self.kmodel, Sequential): - bmodel = self._construct_bigdl_sequence() + bigdlmodel = self._construct_bigdl_sequence() elif isinstance(self.kmodel, Model): - bmodel = self._construct_bigdl_model() - return bmodel + bigdlmodel = self._construct_bigdl_model() + elif isinstance(self.kmodel, Layer): + bigdlmodel = LayerConverter().create(self.kmodel, + self.node_id_to_config_layer[self.kmodel.name]) + else: + raise Exception("Should not enter here: %s" % self.kmodel) + return bigdlmodel @classmethod def from_kmodel(cls, kmodel): @@ -306,7 +306,6 @@ def _do_create_node(self, layer, clayer): def _construct_bigdl_model(self): for clayer in self.kconfig["layers"]: if clayer["name"] not in self.node_id_to_instance: - self._do_create_node(self.node_id_to_layer[clayer["name"]], clayer) ins = [] @@ -323,6 +322,7 @@ def _construct_bigdl_sequence(self): bseq = BLayer.Sequential() layerConverter = LayerConverter() for layer in self.kmodel.layers: + # recursive logic is within create method. blayer = layerConverter.create(layer, self.node_id_to_config_layer[layer.name]) bseq.add(blayer) return bseq @@ -360,7 +360,7 @@ def __check_is_share_weights(self, kclayer): "%s doesn't support multiple inputs with shared weights" % kclayer["class_name"]) def create(self, klayer, kclayer): - class_name = kclayer["class_name"] + class_name = klayer.__class__.__name__ self.__check_is_share_weights(kclayer) @@ -379,10 +379,13 @@ def create(self, klayer, kclayer): blayer = api(klayer, kclayer) return blayer.set_name(klayer.name) - def create_model(self, klayer, kclyer): + def create_model(self, klayer, kclayer): + return DefinitionLoader.from_kmodel(klayer) + + def create_sequential(self, klayer, kclayer): return DefinitionLoader.from_kmodel(klayer) - def create_inputlayer(self, klayer, kclyer): + def create_inputlayer(self, klayer, kclayer): return BLayer.Identity() def create_dense(self, klayer, kclayer): @@ -448,7 +451,6 @@ def create_dropout(self, klayer, kclayer): return BLayer.Dropout(klayer.p) def create_flatten(self, klayer, kclayer): - self.__check_is_share_weights(kclayer) input_shape = klayer.input_shape blayer = BLayer.Reshape([int(np.prod(input_shape[1:]))], None) return blayer @@ -493,7 +495,6 @@ def exchangeNumbers(arr, i, j): return list(filter(lambda pair: pair[0] != pair[1], pairs)) def create_reshape(self, klayer, kclayer): - self.__check_is_share_weights(kclayer) blayer = BLayer.Reshape(klayer.target_shape, None) return blayer @@ -502,12 +503,14 @@ def create_repeatvector(self, klayer, kclayer): n_dim=1, bigdl_type="float") + def __is_from_sequential(self, klayer, kclayer): + return "layers" in kclayer["config"] and hasattr(klayer, "layers") and klayer.layers is not None # noqa + def create_merge(self, klayer, kclayer): - self.__check_is_share_weights(kclayer) input_shape = klayer.get_input_shape_at(0) if klayer.output_shape and not isinstance(klayer.output_shape, tuple): raise Exception("Only output_shape=None or a shape tuple is supported for now") - if klayer.node_indices: + if klayer.node_indices and not all(0 == i for i in klayer.node_indices): unsupport_exp("node_indices") if klayer.output_mask: unsupport_exp("output_mask") @@ -540,7 +543,17 @@ def create_merge(self, klayer, kclayer): raise Exception("Merge mode `%s` not supported for now" % klayer.mode) else: # invalid mode or lambda functions raise Exception("Invalid merge mode: `%s`. Lambda/function as merge mode is not supported for now." % klayer.mode) - return blayer + if self.__is_from_sequential(klayer, kclayer): + bseq = BLayer.Sequential() + parallel_table = BLayer.ParallelTable() + for l in klayer.layers: + bl = DefinitionLoader.from_kmodel(l) + parallel_table.add(bl) + bseq.add(parallel_table) + bseq.add(blayer) + return bseq + else: + return blayer def create_elu(self, klayer, kclayer): return BLayer.ELU(alpha=float(klayer.alpha), @@ -811,8 +824,6 @@ def create_gru(self, klayer, kclayer): def create_batchnormalization(self, klayer, kclayer): config = kclayer["config"] - - self.__check_is_share_weights(kclayer) if keras.backend.image_dim_ordering() != "th" or klayer.axis != 1: raise Exception("""For BatchNormalization, we only support th image ordering (i.e. NCHW) """ + """with axis = 1 for now, but the current order is %s and axis is %s diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 06d2772643d..79bbfe20779 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -597,6 +597,12 @@ def layers(self): layers = [Layer.of(jlayer) for jlayer in jlayers] return layers + @property + def flattened_layers(self): + jlayers = callBigDlFunc(self.bigdl_type, "getFlattenModules", self) + layers = [Layer.of(jlayer) for jlayer in jlayers] + return layers + class Model(Container): """ From ee41483da9723d872d33460a6734afdbbea26872 Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Fri, 24 Nov 2017 02:13:19 -0600 Subject: [PATCH 351/823] Support nested Model and enable test_application (#1927) * good * update * up * goodgood * clean code * fix and enable application test --- .../test/bigdl/keras/test_application.py | 32 ++-- python/dllib/test/bigdl/keras/test_layer.py | 137 ++++++++++++++++-- python/dllib/test/bigdl/test_utils.py | 11 +- python/dllib/test/dev/run-keras.sh | 3 +- 4 files changed, 149 insertions(+), 34 deletions(-) diff --git a/python/dllib/test/bigdl/keras/test_application.py b/python/dllib/test/bigdl/keras/test_application.py index 4c1ffa139c7..75f082e9a8a 100644 --- a/python/dllib/test/bigdl/keras/test_application.py +++ b/python/dllib/test/bigdl/keras/test_application.py @@ -30,7 +30,7 @@ class TestApplication(BigDLTestCase): def assert_model(self, input_data, kmodel, rtol=1e-5, atol=1e-5): bmodel = DefinitionLoader.from_kmodel(kmodel) - WeightLoader.load_weights_from_kmodel(bmodel, kmodel, by_name=True) + WeightLoader.load_weights_from_kmodel(bmodel, kmodel) keras_output = kmodel.predict(input_data) bmodel.training(is_training=False) @@ -42,11 +42,14 @@ def test_lenet(self): kmodel, input_data, output_data = TestModels.kmodel_seq_lenet_mnist() self.modelTest(input_data, kmodel, dump_weights=True) + @pytest.mark.skip(reason="need to fix todo before running the test") def test_text_classification(self): '''This example demonstrates the use of Convolution1D for text classification. This example is from Keras ''' + # TODO: support backend support + import numpy as np np.random.seed(1337) # for reproducibility @@ -123,31 +126,38 @@ def test_text_classification(self): def test_resnet50(self): keras.backend.set_image_dim_ordering("th") - kmodel = resnet50.ResNet50(include_top=False, input_shape=(3, 224, 224)) + kmodel = resnet50.ResNet50(include_top=False, + input_shape=(3, 224, 224), + weights=None) input_data = np.random.random([2, 3, 224, 224]) self.assert_model(input_data, kmodel) def test_vgg16(self): keras.backend.set_image_dim_ordering("th") - kmodel = vgg16.VGG16(include_top=False, input_shape=(3, 224, 224)) + kmodel = vgg16.VGG16(include_top=False, + input_shape=(3, 224, 224), + weights=None) input_data = np.random.random([2, 3, 224, 224]) self.assert_model(input_data, kmodel) def test_vgg19(self): keras.backend.set_image_dim_ordering("th") - kmodel = vgg19.VGG19(include_top=False, input_shape=(3, 224, 224)) + kmodel = vgg19.VGG19(include_top=False, + input_shape=(3, 224, 224), + weights=None) input_data = np.random.random([2, 3, 224, 224]) self.assert_model(input_data, kmodel) + @pytest.mark.skip(reason="need to fix todo before running the test") def test_music_tagger_crnn(self): - # Remove the first BatchNormalization layer in the model as we don't support `axis=3` - # Set `inner_activation` in GRU to be `sigmoid` + # TODO: Remove the first BatchNormalization layer in the model as we don't support `axis=3` + # TODO: Set `inner_activation` in GRU to be `sigmoid` keras.backend.set_image_dim_ordering("th") - kmodel = MusicTaggerCRNN(include_top=False) + kmodel = MusicTaggerCRNN(include_top=False, weights=None) input_data = np.random.random([2, 1, 96, 1366]) bmodel = DefinitionLoader.from_kmodel(kmodel) - WeightLoader.load_weights_from_kmodel(bmodel, kmodel, by_name=True) + WeightLoader.load_weights_from_kmodel(bmodel, kmodel) keras_output = kmodel.predict(input_data) bmodel.training(is_training=False) @@ -157,11 +167,13 @@ def test_music_tagger_crnn(self): def test_inception_v3(self): keras.backend.set_image_dim_ordering("th") - kmodel = inception_v3.InceptionV3(include_top=False, input_shape=(3, 299, 299)) + kmodel = inception_v3.InceptionV3(include_top=False, + input_shape=(3, 299, 299), + weights=None) input_data = np.random.random([2, 3, 299, 299]) bmodel = DefinitionLoader.from_kmodel(kmodel) - WeightLoader.load_weights_from_kmodel(bmodel, kmodel, by_name=True) + WeightLoader.load_weights_from_kmodel(bmodel, kmodel) keras_output = kmodel.predict(input_data) bmodel.training(is_training=False) diff --git a/python/dllib/test/bigdl/keras/test_layer.py b/python/dllib/test/bigdl/keras/test_layer.py index a928bd827d5..debe82cba8d 100644 --- a/python/dllib/test/bigdl/keras/test_layer.py +++ b/python/dllib/test/bigdl/keras/test_layer.py @@ -257,19 +257,132 @@ def test_repeatvector(self): layer = RepeatVector(4, input_shape=(3, )) self.modelTestSingleLayer(input_data, layer) - def test_merge_concat(self): - # input_data1 = np.random.random_sample([2, 3, 5]) - # input_data2 = np.random.random_sample([2, 3, 6]) - # model1 = Sequential() - # model1.add(Dense(20, input_dim=2)) - # model1.add(Dense(20, input_dim=2)) - # - # model2 = Sequential() - # model2.add(Input(input_dim=32)) - # - # merged_model = Sequential() - # merged_model.add(Merge([model1, model2], mode='concat', concat_axis=0)) + def test_nested_model_seq_concat(self): + input_data1 = np.random.random_sample([2, 3]) + input1 = Input((3,)) + out1 = Dense(3)(input1) + out1_1 = Dense(3)(out1) + + branch1 = Model(input=[input1], output=out1_1) + + branch2 = Sequential() + branch2.add(Dense(3, input_shape=[3])) + branch2.add(Dense(3)) + branch2.add(branch1) + branch2_tensor = branch2(input1) + + kmodel = Model(input=[input1], output=branch2_tensor) + kmodel.predict([input_data1]) + self.modelTest(input_data1, + kmodel, + random_weights=False, + dump_weights=True, + is_training=False) + + def test_merge_method_concat(self): + input_data1 = np.random.random_sample([2, 4]) + input_data2 = np.random.random_sample([2, 3]) + input1 = Input((4,)) + input2 = Input((3,)) + out1 = Dense(4)(input1) + out2 = Dense(3)(input2) + from keras.engine import merge + m = merge([out1, out2], mode="concat", concat_axis=1) + kmodel = Model(input=[input1, input2], output=m) + + self.modelTest([input_data1, input_data2], + kmodel, + random_weights=False, + dump_weights=True, + is_training=False) + + def test_merge_method_mix_concat(self): + input_data1 = np.random.random_sample([2, 4]) + input_data2 = np.random.random_sample([2, 3]) + input1 = Input((4,)) + input2 = Input((3,)) + out1 = Dense(4)(input1) + branch1 = Model(input1, out1)(input1) + branch2 = Dense(3)(input2) + from keras.engine import merge + m = merge([branch1, branch2], mode="concat", concat_axis=1) + kmodel = Model(input=[input1, input2], output=m) + + self.modelTest([input_data1, input_data2], + kmodel, + random_weights=False, + dump_weights=True, + is_training=False) + + def test_merge_model_seq_concat(self): + input_data1 = np.random.random_sample([2, 4]) + input_data2 = np.random.random_sample([2, 3]) + input1 = Input((4,)) + input2 = Input((3,)) + out1 = Dense(4)(input1) + out1_1 = Dense(4)(out1) + + branch1 = Model(input=[input1], output=out1_1) + branch2 = Sequential() + branch2.add(Dense(3, input_shape=[3])) + branch2.add(Dense(3)) + branch1_tensor = branch1(input1) + branch2_tensor = branch2(input2) + + from keras.engine import merge + m = merge([branch1_tensor, branch2_tensor], mode="concat", concat_axis=1) + kmodel = Model(input=[input1, input2], output=m) + kmodel.predict([input_data1, input_data2]) + self.modelTest([input_data1, input_data2], + kmodel, + random_weights=False, + dump_weights=True, + is_training=False) + + def test_merge_model_model_concat(self): + input_data1 = np.random.random_sample([2, 4]) + input_data2 = np.random.random_sample([2, 3]) + input1 = Input((4,)) + input2 = Input((3,)) + out1 = Dense(4)(input1) + out1_1 = Dense(4)(out1) + + out2 = Dense(3)(input2) + out2_1 = Dense(3)(out2) + + branch1 = Model(input=[input1], output=out1_1) + branch2 = Model(input=[input2], output=out2_1) + branch1_tensor = branch1(input1) + branch2_tensor = branch2(input2) + + from keras.engine import merge + m = merge([branch1_tensor, branch2_tensor], mode="concat", concat_axis=1) + kmodel = Model(input=[input1, input2], output=m) + + self.modelTest([input_data1, input_data2], + kmodel, + random_weights=False, + dump_weights=True, + is_training=False) + + def test_merge_seq_seq_concat(self): + input_data1 = np.random.random_sample([2, 4]) + input_data2 = np.random.random_sample([2, 3]) + branch1 = Sequential() + branch1.add(Dense(20, input_shape=[4])) + + branch2 = Sequential() + branch2.add(Dense(10, input_shape=[3])) + + merged_model = Sequential() + merged_model.add(Merge([branch1, branch2], mode='concat', concat_axis=1)) + + self.modelTestSingleLayer([input_data1, input_data2], + Merge([branch1, branch2], mode='concat', concat_axis=1), + dump_weights=True, + functional_apis=[False]) + def test_merge_concat(self): inputLayer1 = InputLayer(input_shape=(3, 6, 7)) inputLayer2 = InputLayer(input_shape=(3, 6, 8)) inputLayer3 = InputLayer(input_shape=(3, 6, 9)) diff --git a/python/dllib/test/bigdl/test_utils.py b/python/dllib/test/bigdl/test_utils.py index 8676d8a86f7..68f83d619e5 100644 --- a/python/dllib/test/bigdl/test_utils.py +++ b/python/dllib/test/bigdl/test_utils.py @@ -21,10 +21,9 @@ from keras.layers.core import * from keras.layers.convolutional import * from keras.layers import Dense, Dropout, Input -from keras.models import model_from_json from keras.optimizers import RMSprop from bigdl.util.common import create_tmp_path -from bigdl.keras.converter import DefinitionLoader +from bigdl.keras.converter import * import numpy as np from unittest import TestCase import keras @@ -208,14 +207,6 @@ def modelTest(self, # atol=atol) if dump_weights: # load weights if possible WeightLoader.load_weights_from_hdf5(bigdl_model, keras_model, keras_model_hdf5_path) - bweights = bigdl_model.get_weights() - bweights_from_keras = WeightsConverter.get_bigdl_weigths_from_keras(keras_model) - - # bweights and bweights_from_keras are all list - assert isinstance(bweights, list) - assert len(bweights) == len(bweights_from_keras) - for i in range(len(bweights)): - self.assert_allclose(bweights[i], bweights_from_keras[i], rtol, atol) bigdl_output2 = bigdl_model.forward(input_data) self.assert_allclose(bigdl_output2, diff --git a/python/dllib/test/dev/run-keras.sh b/python/dllib/test/dev/run-keras.sh index de5c6d1401f..811dce9ba73 100755 --- a/python/dllib/test/dev/run-keras.sh +++ b/python/dllib/test/dev/run-keras.sh @@ -32,8 +32,7 @@ echo "${cyan}Using python version: $p${reset}" export PYTHON_EXECUTABLE=$p export PYSPARK_PYTHON=$p export PYSPARK_DRIVER_PYTHON=$p -$1 -m pytest -v ../../../pyspark/test/bigdl/keras \ - --ignore=../../../pyspark/test/bigdl/keras/test_application.py +$1 -m pytest -v ../../../pyspark/test/bigdl/keras exit_status=$? if [ $exit_status -ne 0 ]; From f40b56b0ac539363691e2251df6168ae825487c4 Mon Sep 17 00:00:00 2001 From: Hawkwood <2041829103@qq.com> Date: Mon, 27 Nov 2017 15:23:50 +0800 Subject: [PATCH 352/823] enable auto release (#1920) * recover * update optimization fixed #1521 * solve conflict * Update version.py * Update release.sh * Update version.py * Update version.py * Update version.py * Update version.py * Update version.py * Update version.py * Update version.py * Update version.py * secure --- version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.py b/version.py index 2be0f54013f..a143c2249e8 100644 --- a/version.py +++ b/version.py @@ -14,4 +14,4 @@ # limitations under the License. # -__version__ = "0.4.0.dev0" \ No newline at end of file +__version__ = "0.4.0.dev0" From f6e013de4c8c8e3b8b376604700ec6617804c5b7 Mon Sep 17 00:00:00 2001 From: Hawkwood <2041829103@qq.com> Date: Mon, 27 Nov 2017 15:23:50 +0800 Subject: [PATCH 353/823] enable auto release (#1920) * recover * update optimization fixed #1521 * solve conflict * Update version.py * Update release.sh * Update version.py * Update version.py * Update version.py * Update version.py * Update version.py * Update version.py * Update version.py * Update version.py * secure --- version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.py b/version.py index 2be0f54013f..a143c2249e8 100644 --- a/version.py +++ b/version.py @@ -14,4 +14,4 @@ # limitations under the License. # -__version__ = "0.4.0.dev0" \ No newline at end of file +__version__ = "0.4.0.dev0" From b7a8c8bc49cd7a5e0edb88aeeb68a29b65f0414d Mon Sep 17 00:00:00 2001 From: Hawkwood <2041829103@qq.com> Date: Mon, 27 Nov 2017 15:23:50 +0800 Subject: [PATCH 354/823] enable auto release (#1920) * recover * update optimization fixed #1521 * solve conflict * Update version.py * Update release.sh * Update version.py * Update version.py * Update version.py * Update version.py * Update version.py * Update version.py * Update version.py * Update version.py * secure --- python/dllib/test/dev/release/release.sh | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/python/dllib/test/dev/release/release.sh b/python/dllib/test/dev/release/release.sh index 15c39914c7e..79dcba2498e 100755 --- a/python/dllib/test/dev/release/release.sh +++ b/python/dllib/test/dev/release/release.sh @@ -32,9 +32,14 @@ fi platform=$1 spark_profile=$2 quick=$3 - +input_version=$4 bigdl_version=$(python -c "exec(open('$BIGDL_DIR/pyspark/bigdl/version.py').read()); print(__version__)") +if [ "$input_version" != "$bigdl_version" ]; then + echo "Not the proposed version" + exit -1 +fi + cd ${BIGDL_DIR} if [ "$platform" == "mac" ]; then echo "Building bigdl for mac system" @@ -61,6 +66,13 @@ sdist_command="python setup.py sdist" echo "packing source code: ${sdist_command}" $sdist_command +if [-d "${BIGDL_DIR}/pyspark/build" ]; then + rm -r ${BIGDL_DIR}/pyspark/build +fi + +if [-d "${BIGDL_DIR}/pyspark/dist" ]; then + rm -r ${BIGDL_DIR}/pyspark/dist +fi wheel_command="python setup.py bdist_wheel --plat-name ${verbose_pname}" echo "Packing python distribution: $wheel_command" ${wheel_command} @@ -68,4 +80,4 @@ ${wheel_command} upload_command="twine upload dist/BigDL-${bigdl_version}-py2.py3-none-${verbose_pname}.whl" echo "Please manually upload with this command: $upload_command" - +$upload_command From 0d49b87193e8d0522201df5233b1230dd05f7715 Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Mon, 27 Nov 2017 16:54:13 +0800 Subject: [PATCH 355/823] add activation for recurrent layers (#1909) --- .../dllib/src/bigdl/dllib/keras/converter.py | 63 +++------ python/dllib/src/bigdl/dllib/nn/layer.py | 122 +++++++++++++----- python/dllib/src/bigdl/dllib/utils/common.py | 29 +++++ 3 files changed, 133 insertions(+), 81 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/keras/converter.py b/python/dllib/src/bigdl/dllib/keras/converter.py index 238441b98d4..13951403a62 100644 --- a/python/dllib/src/bigdl/dllib/keras/converter.py +++ b/python/dllib/src/bigdl/dllib/keras/converter.py @@ -5,7 +5,7 @@ from bigdl.optim.optimizer import L1L2Regularizer as BRegularizer import bigdl.optim.optimizer as boptimizer import bigdl.nn.criterion as bcriterion -import bigdl.util.common as bcommon +from bigdl.util.common import get_activation_by_name import keras.optimizers as koptimizers from keras.models import model_from_json from keras.models import Sequential, Model, Layer @@ -445,7 +445,7 @@ def create_embedding(self, klayer, kclayer): def create_activation(self, klayer, kclayer): config = kclayer["config"] - return self.to_bigdl_activation(config["activation"], klayer.name) + return get_activation_by_name(config["activation"], klayer.name) def create_dropout(self, klayer, kclayer): return BLayer.Dropout(klayer.p) @@ -726,7 +726,7 @@ def create_simplernn(self, klayer, kclayer): input_shape = klayer.get_input_shape_at(0) config = kclayer["config"] self.check_constraint_in_config(config) - activation = self.to_bigdl_activation(config["activation"], + activation = get_activation_by_name(config["activation"], "%s_%s" % (config["name"], config["activation"])) rnn = BLayer.RnnCell(input_size=int(input_shape[2]), hidden_size=klayer.output_dim, @@ -743,17 +743,15 @@ def create_lstm(self, klayer, kclayer): input_shape = klayer.get_input_shape_at(0) config = kclayer["config"] self.check_constraint_in_config(config) - activation = self.to_bigdl_activation(config["activation"], + activation = get_activation_by_name(config["activation"], "%s_%s" % (config["name"], config["activation"])) - if not isinstance(activation, BLayer.Tanh): - raise Exception("For activation, only tanh is supported for now.") - inner_activation = self.to_bigdl_activation(config["inner_activation"], + inner_activation = get_activation_by_name(config["inner_activation"], "%s_%s" % (config["name"], config["inner_activation"])) - if not isinstance(inner_activation, BLayer.Sigmoid): - raise Exception("For inner_activation, only sigmond is supported for now.") lstm = BLayer.LSTM(input_size=int(input_shape[2]), hidden_size=klayer.output_dim, p=0.0, + activation=activation, + inner_activation=inner_activation, wRegularizer=self.to_bigdl_reg(config["W_regularizer"]), uRegularizer=self.to_bigdl_reg(config["U_regularizer"]), bRegularizer=self.to_bigdl_reg(config["b_regularizer"]), @@ -765,14 +763,10 @@ def create_convlstm2d(self, klayer, kclayer): input_shape = klayer.get_input_shape_at(0) config = kclayer["config"] self.check_constraint_in_config(config) - activation = self.to_bigdl_activation(config["activation"], + activation = get_activation_by_name(config["activation"], "%s_%s" % (config["name"], config["activation"])) - if not isinstance(activation, BLayer.Tanh): - raise Exception("For activation, only tanh is supported for now.") - inner_activation = self.to_bigdl_activation(config["inner_activation"], + inner_activation = get_activation_by_name(config["inner_activation"], "%s_%s" % (config["name"], config["inner_activation"])) - if not isinstance(inner_activation, BLayer.Sigmoid): - raise Exception("For inner_activation, only sigmond is supported for now.") #TODO: border_mode = 'valid' if config["border_mode"] != 'same': @@ -791,6 +785,8 @@ def create_convlstm2d(self, klayer, kclayer): # NB: ConvLSTM doesn't serialize subsample to json file stride=klayer.subsample[0], padding=-1, + activation=activation, + inner_activation=inner_activation, # NB: ConvLSTM doesn't serialize regularizers to json file # wRegularizer=self.to_bigdl_reg(config["W_regularizer"]), # uRegularizer=self.to_bigdl_reg(config["U_regularizer"]), @@ -805,17 +801,15 @@ def create_gru(self, klayer, kclayer): input_shape = klayer.get_input_shape_at(0) config = kclayer["config"] self.check_constraint_in_config(config) - activation = self.to_bigdl_activation(config["activation"], + activation = get_activation_by_name(config["activation"], "%s_%s" % (config["name"], config["activation"])) - if not isinstance(activation, BLayer.Tanh): - raise Exception("For activation, only `tanh` is supported for now.") - inner_activation = self.to_bigdl_activation(config["inner_activation"], + inner_activation = get_activation_by_name(config["inner_activation"], "%s_%s" % (config["name"], config["inner_activation"])) - if not isinstance(inner_activation, BLayer.Sigmoid): - raise Exception("For inner_activation, only `sigmond` is supported for now.") gru = BLayer.GRU(input_size=int(input_shape[2]), hidden_size=klayer.output_dim, p=0.0, + activation=activation, + inner_activation=inner_activation, wRegularizer=self.to_bigdl_reg(config["W_regularizer"]), uRegularizer=self.to_bigdl_reg(config["U_regularizer"]), bRegularizer=self.to_bigdl_reg(config["b_regularizer"]), @@ -1391,37 +1385,14 @@ def combo_parameter_layer(self, blayer, config): if hasattr(blayer, "set_init_method"): blayer.set_init_method(self.to_bigdl_init(config["init"]), BInit.Zeros()) # Keras always set this to be zeros - # "linear" meaning do nothing + # "linear" means doing nothing if config["activation"] != "linear": - activation = self.to_bigdl_activation(config["activation"], + activation = get_activation_by_name(config["activation"], "%s_%s" % (config["name"], config["activation"])) return self.fuse(blayer, activation) else: return blayer - def to_bigdl_activation(self, activation_name, activation_id): - activation = None - if activation_name == "tanh": - activation = BLayer.Tanh() - elif activation_name == "sigmoid": - activation = BLayer.Sigmoid() - elif activation_name == "hard_sigmoid": - activation = BLayer.HardSigmoid() - elif activation_name == "relu": - activation = BLayer.ReLU() - elif activation_name == "softmax": - activation = BLayer.SoftMax() - elif activation_name == "softplus": - activation = BLayer.SoftPlus(beta=1.0) - elif activation_name == "softsign": - activation = BLayer.SoftSign() - elif activation_name == "linear": - activation = BLayer.Identity() - else: - raise Exception("Unsupported activation type: %s" % activation_name) - activation.set_name(activation_id) - return activation - def get_value_from_init(self, kinit_method, shape): if kinit_method == "zero": return np.zeros(shape) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 79bbfe20779..e9c53e33f7a 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -18,6 +18,7 @@ import sys import numpy as np +import six from bigdl.util.common import JTensor from bigdl.util.common import JavaValue @@ -26,6 +27,7 @@ from bigdl.util.common import get_spark_context from bigdl.util.common import to_list from bigdl.util.common import INTMAX, INTMIN, DOUBLEMAX +from bigdl.util.common import get_activation_by_name from bigdl.optim.optimizer import L1Regularizer, L2Regularizer, L1L2Regularizer from py4j.java_gateway import JavaObject @@ -953,6 +955,21 @@ def __init__(self, bigdl_type="float"): super(Tanh, self).__init__(None, bigdl_type) +class Sigmoid(Layer): + + ''' + Applies the Sigmoid function element-wise to the input Tensor, + thus outputting a Tensor of the same dimension. + + >>> sigmoid = Sigmoid() + creating: createSigmoid + ''' + + def __init__(self, + bigdl_type="float"): + super(Sigmoid, self).__init__(None, bigdl_type) + + class Echo(Layer): ''' @@ -1343,21 +1360,37 @@ class LSTM(Layer): :param inputSize: the size of each input vector :param hiddenSize: Hidden unit size in the LSTM - :param p: is used for [[Dropout]] probability. For more details aboutRNN dropouts, please refer to[RnnDrop: A Novel Dropout for RNNs in ASR](http://www.stat.berkeley.edu/~tsmoon/files/Conference/asru2015.pdf)[A Theoretically Grounded Application of Dropout in Recurrent Neural Networks](https://arxiv.org/pdf/1512.05287.pdf) + :param p: is used for [[Dropout]] probability. For more details aboutRNN dropouts, please refer to[RnnDrop: A Novel Dropout for RNNs in ASR](http://www.stat.berkeley.edu/~tsmoon/files/Conference/asru2015.pdf)[A Theoretically Grounded Application of Dropout in Recurrent Neural Networks](https://arxiv.org/pdf/1512.05287.pdf) + :param activation: activation function, by default to be Tanh if not specified. + It can also be the name of an existing activation as a string. + :param inner_activation: activation function for the inner cells, by default to be Sigmoid if not specified. + It can also be the name of an existing activation as a string. :param wRegularizer: instance of [[Regularizer]](eg. L1 or L2 regularization), applied to the input weights matrices. :param uRegularizer: instance [[Regularizer]](eg. L1 or L2 regularization), applied to the recurrent weights matrices. :param bRegularizer: instance of [[Regularizer]]applied to the bias. - >>> lstm = LSTM(4, 3, 0.5, L1Regularizer(0.5), L1Regularizer(0.5), L1Regularizer(0.5)) + >>> lstm = LSTM(4, 3, 0.5, 'tanh', Sigmoid(), L1Regularizer(0.5), L1Regularizer(0.5), L1Regularizer(0.5)) + creating: createSigmoid creating: createL1Regularizer creating: createL1Regularizer creating: createL1Regularizer + creating: createTanh creating: createLSTM ''' - def __init__(self, input_size, hidden_size, p=0.0, wRegularizer=None, uRegularizer=None, bRegularizer=None, bigdl_type="float"): - super(LSTM, self).__init__(None, bigdl_type, input_size, hidden_size, p, wRegularizer, uRegularizer, bRegularizer) + def __init__(self, input_size, hidden_size, p=0.0, activation=None, inner_activation=None, + wRegularizer=None, uRegularizer=None, bRegularizer=None, bigdl_type="float"): + if not activation: + activation = Tanh() + if not inner_activation: + inner_activation = Sigmoid() + if isinstance(activation, six.string_types): + activation = get_activation_by_name(activation) + if isinstance(inner_activation, six.string_types): + inner_activation = get_activation_by_name(inner_activation) + super(LSTM, self).__init__(None, bigdl_type, input_size, hidden_size, p, + activation, inner_activation, wRegularizer, uRegularizer, bRegularizer) class LSTMPeephole(Layer): @@ -1401,22 +1434,38 @@ class GRU(Layer): :param input_size: the size of each input vector :param hidden_size: Hidden unit size in GRU - :param p: is used for [[Dropout]] probability. For more details aboutRNN dropouts, please refer to[RnnDrop: A Novel Dropout for RNNs in ASR](http://www.stat.berkeley.edu/~tsmoon/files/Conference/asru2015.pdf)[A Theoretically Grounded Application of Dropout in Recurrent Neural Networks](https://arxiv.org/pdf/1512.05287.pdf) + :param p: is used for [[Dropout]] probability. For more details aboutRNN dropouts, please refer to[RnnDrop: A Novel Dropout for RNNs in ASR](http://www.stat.berkeley.edu/~tsmoon/files/Conference/asru2015.pdf)[A Theoretically Grounded Application of Dropout in Recurrent Neural Networks](https://arxiv.org/pdf/1512.05287.pdf) + :param activation: activation function, by default to be Tanh if not specified. + It can also be the name of an existing activation as a string. + :param inner_activation: activation function for the inner cells, by default to be Sigmoid if not specified. + It can also be the name of an existing activation as a string. :param wRegularizer: instance of [[Regularizer]](eg. L1 or L2 regularization), applied to the input weights matrices. :param uRegularizer: instance [[Regularizer]](eg. L1 or L2 regularization), applied to the recurrent weights matrices. :param bRegularizer: instance of [[Regularizer]]applied to the bias. - >>> gru = GRU(4, 3, 0.5, L1Regularizer(0.5), L1Regularizer(0.5), L1Regularizer(0.5)) + >>> gru = GRU(4, 3, 0.5, Tanh(), Sigmoid(), L1Regularizer(0.5), L1Regularizer(0.5), L1Regularizer(0.5)) + creating: createTanh + creating: createSigmoid creating: createL1Regularizer creating: createL1Regularizer creating: createL1Regularizer creating: createGRU ''' - def __init__(self, input_size, hidden_size, p=0.0, wRegularizer=None, uRegularizer=None, bRegularizer=None, bigdl_type="float"): - super(GRU, self).__init__(None, bigdl_type, input_size, hidden_size, p, wRegularizer, uRegularizer, bRegularizer) + def __init__(self, input_size, hidden_size, p=0.0, activation=None, inner_activation=None, + wRegularizer=None, uRegularizer=None, bRegularizer=None, bigdl_type="float"): + if not activation: + activation = Tanh() + if not inner_activation: + inner_activation = Sigmoid() + if isinstance(activation, six.string_types): + activation = get_activation_by_name(activation) + if isinstance(inner_activation, six.string_types): + inner_activation = get_activation_by_name(inner_activation) + super(GRU, self).__init__(None, bigdl_type, input_size, hidden_size, p, activation, inner_activation, + wRegularizer, uRegularizer, bRegularizer) class RnnCell(Layer): @@ -1426,16 +1475,15 @@ class RnnCell(Layer): :param input_size: the size of each input vector :param hidden_size: Hidden unit size in simple RNN - :param activation: activation function + :param activation: activation function. It can also be the name of an existing activation as a string. :param isInputWithBias: boolean :param isHiddenWithBias: boolean - :param wRegularizer: instance of [[Regularizer]](eg. L1 or L2 regularization), applied to the input weights matrices. :param uRegularizer: instance [[Regularizer]](eg. L1 or L2 regularization), applied to the recurrent weights matrices. :param bRegularizer: instance of [[Regularizer]](../regularizers.md),applied to the bias. - >>> reshape = RnnCell(4, 3, Tanh(), True, True, L1Regularizer(0.5), L1Regularizer(0.5), L1Regularizer(0.5)) + >>> rnn = RnnCell(4, 3, Tanh(), True, True, L1Regularizer(0.5), L1Regularizer(0.5), L1Regularizer(0.5)) creating: createTanh creating: createL1Regularizer creating: createL1Regularizer @@ -1453,6 +1501,8 @@ def __init__(self, uRegularizer=None, bRegularizer=None, bigdl_type="float"): + if isinstance(activation, six.string_types): + activation = get_activation_by_name(activation) super(RnnCell, self).__init__(None, bigdl_type, input_size, hidden_size, activation, isInputWithBias, isHiddenWithBias, wRegularizer, uRegularizer, bRegularizer) @@ -3243,21 +3293,6 @@ def __init__(self, index) -class Sigmoid(Layer): - - ''' - Applies the Sigmoid function element-wise to the input Tensor, - thus outputting a Tensor of the same dimension. - - >>> sigmoid = Sigmoid() - creating: createSigmoid - ''' - - def __init__(self, - bigdl_type="float"): - super(Sigmoid, self).__init__(None, bigdl_type) - - class SoftMax(Layer): ''' @@ -4387,17 +4422,23 @@ class ConvLSTMPeephole(Layer): :param input_size: number of input planes in the image given into forward() :param output_size: number of output planes the convolution layer will produce - :param kernel_i Convolutional filter size to convolve input - :param kernel_c Convolutional filter size to convolve cell - :param stride The step of the convolution, default is 1 - :param padding The additional zeros added, default is -1 + :param kernel_i: Convolutional filter size to convolve input + :param kernel_c: Convolutional filter size to convolve cell + :param stride: The step of the convolution, default is 1 + :param padding: The additional zeros added, default is -1 + :param activation: activation function, by default to be Tanh if not specified. + It can also be the name of an existing activation as a string. + :param inner_activation: activation function for the inner cells, by default to be Sigmoid if not specified. + It can also be the name of an existing activation as a string. :param wRegularizer: instance of [[Regularizer]](eg. L1 or L2 regularization), applied to the input weights matrices :param uRegularizer: instance [[Regularizer]](eg. L1 or L2 regularization), applied to the recurrent weights matrices :param bRegularizer: instance of [[Regularizer]]applied to the bias. :param cRegularizer: instance of [[Regularizer]]applied to peephole. :param with_peephole: whether use last cell status control a gate. - >>> convlstm = ConvLSTMPeephole(4, 3, 3, 3, 1, -1, L1Regularizer(0.5), L1Regularizer(0.5), L1Regularizer(0.5), L1Regularizer(0.5)) + >>> convlstm = ConvLSTMPeephole(4, 3, 3, 3, 1, -1, Tanh(), HardSigmoid(), L1Regularizer(0.5), L1Regularizer(0.5), L1Regularizer(0.5), L1Regularizer(0.5)) + creating: createTanh + creating: createHardSigmoid creating: createL1Regularizer creating: createL1Regularizer creating: createL1Regularizer @@ -4405,10 +4446,21 @@ class ConvLSTMPeephole(Layer): creating: createConvLSTMPeephole ''' - def __init__(self, input_size, output_size, kernel_i, kernel_c, stride=1, padding=-1, wRegularizer=None, uRegularizer=None, - bRegularizer=None, cRegularizer=None, with_peephole=True, bigdl_type="float"): - super(ConvLSTMPeephole, self).__init__(None, bigdl_type, input_size, output_size, kernel_i, kernel_c, stride, - padding, wRegularizer, uRegularizer, bRegularizer, cRegularizer, with_peephole) + def __init__(self, input_size, output_size, kernel_i, kernel_c, stride=1, padding=-1, + activation=None, inner_activation=None, + wRegularizer=None, uRegularizer=None, bRegularizer=None, cRegularizer=None, + with_peephole=True, bigdl_type="float"): + if not activation: + activation = Tanh() + if not inner_activation: + inner_activation = Sigmoid() + if isinstance(activation, six.string_types): + activation = get_activation_by_name(activation) + if isinstance(inner_activation, six.string_types): + inner_activation = get_activation_by_name(inner_activation) + super(ConvLSTMPeephole, self).__init__(None, bigdl_type, input_size, output_size, kernel_i, kernel_c, + stride, padding, activation, inner_activation, + wRegularizer, uRegularizer, bRegularizer, cRegularizer, with_peephole) class Tile(Layer): ''' diff --git a/python/dllib/src/bigdl/dllib/utils/common.py b/python/dllib/src/bigdl/dllib/utils/common.py index 428a109215c..5680c2539e7 100644 --- a/python/dllib/src/bigdl/dllib/utils/common.py +++ b/python/dllib/src/bigdl/dllib/utils/common.py @@ -536,12 +536,41 @@ def _py2java(sc, obj): obj = sc._jvm.org.apache.spark.bigdl.api.python.BigDLSerDe.loads(data) return obj + def create_tmp_path(): tmp_file = tempfile.NamedTemporaryFile(prefix="bigdl") tmp_file.close() return tmp_file.name +def get_activation_by_name(activation_name, activation_id=None): + """ Convert to a bigdl activation layer + given the name of the activation as a string """ + import bigdl.nn.layer as BLayer + activation = None + activation_name = activation_name.lower() + if activation_name == "tanh": + activation = BLayer.Tanh() + elif activation_name == "sigmoid": + activation = BLayer.Sigmoid() + elif activation_name == "hard_sigmoid": + activation = BLayer.HardSigmoid() + elif activation_name == "relu": + activation = BLayer.ReLU() + elif activation_name == "softmax": + activation = BLayer.SoftMax() + elif activation_name == "softplus": + activation = BLayer.SoftPlus(beta=1.0) + elif activation_name == "softsign": + activation = BLayer.SoftSign() + elif activation_name == "linear": + activation = BLayer.Identity() + else: + raise Exception("Unsupported activation type: %s" % activation_name) + if not activation_id: + activation.set_name(activation_id) + return activation + def _test(): import doctest from pyspark import SparkContext From 4d872e40731e53d922aedd88958cfad30c68c277 Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Mon, 27 Nov 2017 16:54:13 +0800 Subject: [PATCH 356/823] add activation for recurrent layers (#1909) --- .../dllib/src/bigdl/dllib/keras/converter.py | 63 +++------ python/dllib/src/bigdl/dllib/nn/layer.py | 122 +++++++++++++----- python/dllib/src/bigdl/utils/common.py | 29 +++++ 3 files changed, 133 insertions(+), 81 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/keras/converter.py b/python/dllib/src/bigdl/dllib/keras/converter.py index 238441b98d4..13951403a62 100644 --- a/python/dllib/src/bigdl/dllib/keras/converter.py +++ b/python/dllib/src/bigdl/dllib/keras/converter.py @@ -5,7 +5,7 @@ from bigdl.optim.optimizer import L1L2Regularizer as BRegularizer import bigdl.optim.optimizer as boptimizer import bigdl.nn.criterion as bcriterion -import bigdl.util.common as bcommon +from bigdl.util.common import get_activation_by_name import keras.optimizers as koptimizers from keras.models import model_from_json from keras.models import Sequential, Model, Layer @@ -445,7 +445,7 @@ def create_embedding(self, klayer, kclayer): def create_activation(self, klayer, kclayer): config = kclayer["config"] - return self.to_bigdl_activation(config["activation"], klayer.name) + return get_activation_by_name(config["activation"], klayer.name) def create_dropout(self, klayer, kclayer): return BLayer.Dropout(klayer.p) @@ -726,7 +726,7 @@ def create_simplernn(self, klayer, kclayer): input_shape = klayer.get_input_shape_at(0) config = kclayer["config"] self.check_constraint_in_config(config) - activation = self.to_bigdl_activation(config["activation"], + activation = get_activation_by_name(config["activation"], "%s_%s" % (config["name"], config["activation"])) rnn = BLayer.RnnCell(input_size=int(input_shape[2]), hidden_size=klayer.output_dim, @@ -743,17 +743,15 @@ def create_lstm(self, klayer, kclayer): input_shape = klayer.get_input_shape_at(0) config = kclayer["config"] self.check_constraint_in_config(config) - activation = self.to_bigdl_activation(config["activation"], + activation = get_activation_by_name(config["activation"], "%s_%s" % (config["name"], config["activation"])) - if not isinstance(activation, BLayer.Tanh): - raise Exception("For activation, only tanh is supported for now.") - inner_activation = self.to_bigdl_activation(config["inner_activation"], + inner_activation = get_activation_by_name(config["inner_activation"], "%s_%s" % (config["name"], config["inner_activation"])) - if not isinstance(inner_activation, BLayer.Sigmoid): - raise Exception("For inner_activation, only sigmond is supported for now.") lstm = BLayer.LSTM(input_size=int(input_shape[2]), hidden_size=klayer.output_dim, p=0.0, + activation=activation, + inner_activation=inner_activation, wRegularizer=self.to_bigdl_reg(config["W_regularizer"]), uRegularizer=self.to_bigdl_reg(config["U_regularizer"]), bRegularizer=self.to_bigdl_reg(config["b_regularizer"]), @@ -765,14 +763,10 @@ def create_convlstm2d(self, klayer, kclayer): input_shape = klayer.get_input_shape_at(0) config = kclayer["config"] self.check_constraint_in_config(config) - activation = self.to_bigdl_activation(config["activation"], + activation = get_activation_by_name(config["activation"], "%s_%s" % (config["name"], config["activation"])) - if not isinstance(activation, BLayer.Tanh): - raise Exception("For activation, only tanh is supported for now.") - inner_activation = self.to_bigdl_activation(config["inner_activation"], + inner_activation = get_activation_by_name(config["inner_activation"], "%s_%s" % (config["name"], config["inner_activation"])) - if not isinstance(inner_activation, BLayer.Sigmoid): - raise Exception("For inner_activation, only sigmond is supported for now.") #TODO: border_mode = 'valid' if config["border_mode"] != 'same': @@ -791,6 +785,8 @@ def create_convlstm2d(self, klayer, kclayer): # NB: ConvLSTM doesn't serialize subsample to json file stride=klayer.subsample[0], padding=-1, + activation=activation, + inner_activation=inner_activation, # NB: ConvLSTM doesn't serialize regularizers to json file # wRegularizer=self.to_bigdl_reg(config["W_regularizer"]), # uRegularizer=self.to_bigdl_reg(config["U_regularizer"]), @@ -805,17 +801,15 @@ def create_gru(self, klayer, kclayer): input_shape = klayer.get_input_shape_at(0) config = kclayer["config"] self.check_constraint_in_config(config) - activation = self.to_bigdl_activation(config["activation"], + activation = get_activation_by_name(config["activation"], "%s_%s" % (config["name"], config["activation"])) - if not isinstance(activation, BLayer.Tanh): - raise Exception("For activation, only `tanh` is supported for now.") - inner_activation = self.to_bigdl_activation(config["inner_activation"], + inner_activation = get_activation_by_name(config["inner_activation"], "%s_%s" % (config["name"], config["inner_activation"])) - if not isinstance(inner_activation, BLayer.Sigmoid): - raise Exception("For inner_activation, only `sigmond` is supported for now.") gru = BLayer.GRU(input_size=int(input_shape[2]), hidden_size=klayer.output_dim, p=0.0, + activation=activation, + inner_activation=inner_activation, wRegularizer=self.to_bigdl_reg(config["W_regularizer"]), uRegularizer=self.to_bigdl_reg(config["U_regularizer"]), bRegularizer=self.to_bigdl_reg(config["b_regularizer"]), @@ -1391,37 +1385,14 @@ def combo_parameter_layer(self, blayer, config): if hasattr(blayer, "set_init_method"): blayer.set_init_method(self.to_bigdl_init(config["init"]), BInit.Zeros()) # Keras always set this to be zeros - # "linear" meaning do nothing + # "linear" means doing nothing if config["activation"] != "linear": - activation = self.to_bigdl_activation(config["activation"], + activation = get_activation_by_name(config["activation"], "%s_%s" % (config["name"], config["activation"])) return self.fuse(blayer, activation) else: return blayer - def to_bigdl_activation(self, activation_name, activation_id): - activation = None - if activation_name == "tanh": - activation = BLayer.Tanh() - elif activation_name == "sigmoid": - activation = BLayer.Sigmoid() - elif activation_name == "hard_sigmoid": - activation = BLayer.HardSigmoid() - elif activation_name == "relu": - activation = BLayer.ReLU() - elif activation_name == "softmax": - activation = BLayer.SoftMax() - elif activation_name == "softplus": - activation = BLayer.SoftPlus(beta=1.0) - elif activation_name == "softsign": - activation = BLayer.SoftSign() - elif activation_name == "linear": - activation = BLayer.Identity() - else: - raise Exception("Unsupported activation type: %s" % activation_name) - activation.set_name(activation_id) - return activation - def get_value_from_init(self, kinit_method, shape): if kinit_method == "zero": return np.zeros(shape) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 79bbfe20779..e9c53e33f7a 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -18,6 +18,7 @@ import sys import numpy as np +import six from bigdl.util.common import JTensor from bigdl.util.common import JavaValue @@ -26,6 +27,7 @@ from bigdl.util.common import get_spark_context from bigdl.util.common import to_list from bigdl.util.common import INTMAX, INTMIN, DOUBLEMAX +from bigdl.util.common import get_activation_by_name from bigdl.optim.optimizer import L1Regularizer, L2Regularizer, L1L2Regularizer from py4j.java_gateway import JavaObject @@ -953,6 +955,21 @@ def __init__(self, bigdl_type="float"): super(Tanh, self).__init__(None, bigdl_type) +class Sigmoid(Layer): + + ''' + Applies the Sigmoid function element-wise to the input Tensor, + thus outputting a Tensor of the same dimension. + + >>> sigmoid = Sigmoid() + creating: createSigmoid + ''' + + def __init__(self, + bigdl_type="float"): + super(Sigmoid, self).__init__(None, bigdl_type) + + class Echo(Layer): ''' @@ -1343,21 +1360,37 @@ class LSTM(Layer): :param inputSize: the size of each input vector :param hiddenSize: Hidden unit size in the LSTM - :param p: is used for [[Dropout]] probability. For more details aboutRNN dropouts, please refer to[RnnDrop: A Novel Dropout for RNNs in ASR](http://www.stat.berkeley.edu/~tsmoon/files/Conference/asru2015.pdf)[A Theoretically Grounded Application of Dropout in Recurrent Neural Networks](https://arxiv.org/pdf/1512.05287.pdf) + :param p: is used for [[Dropout]] probability. For more details aboutRNN dropouts, please refer to[RnnDrop: A Novel Dropout for RNNs in ASR](http://www.stat.berkeley.edu/~tsmoon/files/Conference/asru2015.pdf)[A Theoretically Grounded Application of Dropout in Recurrent Neural Networks](https://arxiv.org/pdf/1512.05287.pdf) + :param activation: activation function, by default to be Tanh if not specified. + It can also be the name of an existing activation as a string. + :param inner_activation: activation function for the inner cells, by default to be Sigmoid if not specified. + It can also be the name of an existing activation as a string. :param wRegularizer: instance of [[Regularizer]](eg. L1 or L2 regularization), applied to the input weights matrices. :param uRegularizer: instance [[Regularizer]](eg. L1 or L2 regularization), applied to the recurrent weights matrices. :param bRegularizer: instance of [[Regularizer]]applied to the bias. - >>> lstm = LSTM(4, 3, 0.5, L1Regularizer(0.5), L1Regularizer(0.5), L1Regularizer(0.5)) + >>> lstm = LSTM(4, 3, 0.5, 'tanh', Sigmoid(), L1Regularizer(0.5), L1Regularizer(0.5), L1Regularizer(0.5)) + creating: createSigmoid creating: createL1Regularizer creating: createL1Regularizer creating: createL1Regularizer + creating: createTanh creating: createLSTM ''' - def __init__(self, input_size, hidden_size, p=0.0, wRegularizer=None, uRegularizer=None, bRegularizer=None, bigdl_type="float"): - super(LSTM, self).__init__(None, bigdl_type, input_size, hidden_size, p, wRegularizer, uRegularizer, bRegularizer) + def __init__(self, input_size, hidden_size, p=0.0, activation=None, inner_activation=None, + wRegularizer=None, uRegularizer=None, bRegularizer=None, bigdl_type="float"): + if not activation: + activation = Tanh() + if not inner_activation: + inner_activation = Sigmoid() + if isinstance(activation, six.string_types): + activation = get_activation_by_name(activation) + if isinstance(inner_activation, six.string_types): + inner_activation = get_activation_by_name(inner_activation) + super(LSTM, self).__init__(None, bigdl_type, input_size, hidden_size, p, + activation, inner_activation, wRegularizer, uRegularizer, bRegularizer) class LSTMPeephole(Layer): @@ -1401,22 +1434,38 @@ class GRU(Layer): :param input_size: the size of each input vector :param hidden_size: Hidden unit size in GRU - :param p: is used for [[Dropout]] probability. For more details aboutRNN dropouts, please refer to[RnnDrop: A Novel Dropout for RNNs in ASR](http://www.stat.berkeley.edu/~tsmoon/files/Conference/asru2015.pdf)[A Theoretically Grounded Application of Dropout in Recurrent Neural Networks](https://arxiv.org/pdf/1512.05287.pdf) + :param p: is used for [[Dropout]] probability. For more details aboutRNN dropouts, please refer to[RnnDrop: A Novel Dropout for RNNs in ASR](http://www.stat.berkeley.edu/~tsmoon/files/Conference/asru2015.pdf)[A Theoretically Grounded Application of Dropout in Recurrent Neural Networks](https://arxiv.org/pdf/1512.05287.pdf) + :param activation: activation function, by default to be Tanh if not specified. + It can also be the name of an existing activation as a string. + :param inner_activation: activation function for the inner cells, by default to be Sigmoid if not specified. + It can also be the name of an existing activation as a string. :param wRegularizer: instance of [[Regularizer]](eg. L1 or L2 regularization), applied to the input weights matrices. :param uRegularizer: instance [[Regularizer]](eg. L1 or L2 regularization), applied to the recurrent weights matrices. :param bRegularizer: instance of [[Regularizer]]applied to the bias. - >>> gru = GRU(4, 3, 0.5, L1Regularizer(0.5), L1Regularizer(0.5), L1Regularizer(0.5)) + >>> gru = GRU(4, 3, 0.5, Tanh(), Sigmoid(), L1Regularizer(0.5), L1Regularizer(0.5), L1Regularizer(0.5)) + creating: createTanh + creating: createSigmoid creating: createL1Regularizer creating: createL1Regularizer creating: createL1Regularizer creating: createGRU ''' - def __init__(self, input_size, hidden_size, p=0.0, wRegularizer=None, uRegularizer=None, bRegularizer=None, bigdl_type="float"): - super(GRU, self).__init__(None, bigdl_type, input_size, hidden_size, p, wRegularizer, uRegularizer, bRegularizer) + def __init__(self, input_size, hidden_size, p=0.0, activation=None, inner_activation=None, + wRegularizer=None, uRegularizer=None, bRegularizer=None, bigdl_type="float"): + if not activation: + activation = Tanh() + if not inner_activation: + inner_activation = Sigmoid() + if isinstance(activation, six.string_types): + activation = get_activation_by_name(activation) + if isinstance(inner_activation, six.string_types): + inner_activation = get_activation_by_name(inner_activation) + super(GRU, self).__init__(None, bigdl_type, input_size, hidden_size, p, activation, inner_activation, + wRegularizer, uRegularizer, bRegularizer) class RnnCell(Layer): @@ -1426,16 +1475,15 @@ class RnnCell(Layer): :param input_size: the size of each input vector :param hidden_size: Hidden unit size in simple RNN - :param activation: activation function + :param activation: activation function. It can also be the name of an existing activation as a string. :param isInputWithBias: boolean :param isHiddenWithBias: boolean - :param wRegularizer: instance of [[Regularizer]](eg. L1 or L2 regularization), applied to the input weights matrices. :param uRegularizer: instance [[Regularizer]](eg. L1 or L2 regularization), applied to the recurrent weights matrices. :param bRegularizer: instance of [[Regularizer]](../regularizers.md),applied to the bias. - >>> reshape = RnnCell(4, 3, Tanh(), True, True, L1Regularizer(0.5), L1Regularizer(0.5), L1Regularizer(0.5)) + >>> rnn = RnnCell(4, 3, Tanh(), True, True, L1Regularizer(0.5), L1Regularizer(0.5), L1Regularizer(0.5)) creating: createTanh creating: createL1Regularizer creating: createL1Regularizer @@ -1453,6 +1501,8 @@ def __init__(self, uRegularizer=None, bRegularizer=None, bigdl_type="float"): + if isinstance(activation, six.string_types): + activation = get_activation_by_name(activation) super(RnnCell, self).__init__(None, bigdl_type, input_size, hidden_size, activation, isInputWithBias, isHiddenWithBias, wRegularizer, uRegularizer, bRegularizer) @@ -3243,21 +3293,6 @@ def __init__(self, index) -class Sigmoid(Layer): - - ''' - Applies the Sigmoid function element-wise to the input Tensor, - thus outputting a Tensor of the same dimension. - - >>> sigmoid = Sigmoid() - creating: createSigmoid - ''' - - def __init__(self, - bigdl_type="float"): - super(Sigmoid, self).__init__(None, bigdl_type) - - class SoftMax(Layer): ''' @@ -4387,17 +4422,23 @@ class ConvLSTMPeephole(Layer): :param input_size: number of input planes in the image given into forward() :param output_size: number of output planes the convolution layer will produce - :param kernel_i Convolutional filter size to convolve input - :param kernel_c Convolutional filter size to convolve cell - :param stride The step of the convolution, default is 1 - :param padding The additional zeros added, default is -1 + :param kernel_i: Convolutional filter size to convolve input + :param kernel_c: Convolutional filter size to convolve cell + :param stride: The step of the convolution, default is 1 + :param padding: The additional zeros added, default is -1 + :param activation: activation function, by default to be Tanh if not specified. + It can also be the name of an existing activation as a string. + :param inner_activation: activation function for the inner cells, by default to be Sigmoid if not specified. + It can also be the name of an existing activation as a string. :param wRegularizer: instance of [[Regularizer]](eg. L1 or L2 regularization), applied to the input weights matrices :param uRegularizer: instance [[Regularizer]](eg. L1 or L2 regularization), applied to the recurrent weights matrices :param bRegularizer: instance of [[Regularizer]]applied to the bias. :param cRegularizer: instance of [[Regularizer]]applied to peephole. :param with_peephole: whether use last cell status control a gate. - >>> convlstm = ConvLSTMPeephole(4, 3, 3, 3, 1, -1, L1Regularizer(0.5), L1Regularizer(0.5), L1Regularizer(0.5), L1Regularizer(0.5)) + >>> convlstm = ConvLSTMPeephole(4, 3, 3, 3, 1, -1, Tanh(), HardSigmoid(), L1Regularizer(0.5), L1Regularizer(0.5), L1Regularizer(0.5), L1Regularizer(0.5)) + creating: createTanh + creating: createHardSigmoid creating: createL1Regularizer creating: createL1Regularizer creating: createL1Regularizer @@ -4405,10 +4446,21 @@ class ConvLSTMPeephole(Layer): creating: createConvLSTMPeephole ''' - def __init__(self, input_size, output_size, kernel_i, kernel_c, stride=1, padding=-1, wRegularizer=None, uRegularizer=None, - bRegularizer=None, cRegularizer=None, with_peephole=True, bigdl_type="float"): - super(ConvLSTMPeephole, self).__init__(None, bigdl_type, input_size, output_size, kernel_i, kernel_c, stride, - padding, wRegularizer, uRegularizer, bRegularizer, cRegularizer, with_peephole) + def __init__(self, input_size, output_size, kernel_i, kernel_c, stride=1, padding=-1, + activation=None, inner_activation=None, + wRegularizer=None, uRegularizer=None, bRegularizer=None, cRegularizer=None, + with_peephole=True, bigdl_type="float"): + if not activation: + activation = Tanh() + if not inner_activation: + inner_activation = Sigmoid() + if isinstance(activation, six.string_types): + activation = get_activation_by_name(activation) + if isinstance(inner_activation, six.string_types): + inner_activation = get_activation_by_name(inner_activation) + super(ConvLSTMPeephole, self).__init__(None, bigdl_type, input_size, output_size, kernel_i, kernel_c, + stride, padding, activation, inner_activation, + wRegularizer, uRegularizer, bRegularizer, cRegularizer, with_peephole) class Tile(Layer): ''' diff --git a/python/dllib/src/bigdl/utils/common.py b/python/dllib/src/bigdl/utils/common.py index 428a109215c..5680c2539e7 100644 --- a/python/dllib/src/bigdl/utils/common.py +++ b/python/dllib/src/bigdl/utils/common.py @@ -536,12 +536,41 @@ def _py2java(sc, obj): obj = sc._jvm.org.apache.spark.bigdl.api.python.BigDLSerDe.loads(data) return obj + def create_tmp_path(): tmp_file = tempfile.NamedTemporaryFile(prefix="bigdl") tmp_file.close() return tmp_file.name +def get_activation_by_name(activation_name, activation_id=None): + """ Convert to a bigdl activation layer + given the name of the activation as a string """ + import bigdl.nn.layer as BLayer + activation = None + activation_name = activation_name.lower() + if activation_name == "tanh": + activation = BLayer.Tanh() + elif activation_name == "sigmoid": + activation = BLayer.Sigmoid() + elif activation_name == "hard_sigmoid": + activation = BLayer.HardSigmoid() + elif activation_name == "relu": + activation = BLayer.ReLU() + elif activation_name == "softmax": + activation = BLayer.SoftMax() + elif activation_name == "softplus": + activation = BLayer.SoftPlus(beta=1.0) + elif activation_name == "softsign": + activation = BLayer.SoftSign() + elif activation_name == "linear": + activation = BLayer.Identity() + else: + raise Exception("Unsupported activation type: %s" % activation_name) + if not activation_id: + activation.set_name(activation_id) + return activation + def _test(): import doctest from pyspark import SparkContext From afae62778e3a7f51dacf70a96f2620b2cc1330e7 Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Mon, 27 Nov 2017 16:54:13 +0800 Subject: [PATCH 357/823] add activation for recurrent layers (#1909) --- .../test/bigdl/keras/test_application.py | 3 +-- python/dllib/test/bigdl/keras/test_layer.py | 19 +++++++++++++------ 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/python/dllib/test/bigdl/keras/test_application.py b/python/dllib/test/bigdl/keras/test_application.py index 75f082e9a8a..f5214cf1fea 100644 --- a/python/dllib/test/bigdl/keras/test_application.py +++ b/python/dllib/test/bigdl/keras/test_application.py @@ -150,8 +150,7 @@ def test_vgg19(self): @pytest.mark.skip(reason="need to fix todo before running the test") def test_music_tagger_crnn(self): - # TODO: Remove the first BatchNormalization layer in the model as we don't support `axis=3` - # TODO: Set `inner_activation` in GRU to be `sigmoid` + # TODO: For the first BatchNormalization layer in the model, we don't support `axis=3` keras.backend.set_image_dim_ordering("th") kmodel = MusicTaggerCRNN(include_top=False, weights=None) input_data = np.random.random([2, 1, 96, 1366]) diff --git a/python/dllib/test/bigdl/keras/test_layer.py b/python/dllib/test/bigdl/keras/test_layer.py index debe82cba8d..cb67fc46d98 100644 --- a/python/dllib/test/bigdl/keras/test_layer.py +++ b/python/dllib/test/bigdl/keras/test_layer.py @@ -514,22 +514,28 @@ def test_simplernn(self): def test_lstm(self): input_data = np.random.random([3, 4, 5]) - layer = LSTM(5, input_shape=(4, 5), return_sequences=True, inner_activation='sigmoid') + layer = LSTM(5, input_shape=(4, 5), return_sequences=True) self.modelTestSingleLayer(input_data, layer, dump_weights=True) - layer2 = LSTM(3, input_shape=(4, 5), return_sequences=False, inner_activation='sigmoid') + layer2 = LSTM(3, input_shape=(4, 5), return_sequences=False, + activation='relu', inner_activation='sigmoid') self.modelTestSingleLayer(input_data, layer2, dump_weights=True) def test_convlstm2d(self): input_data = np.random.random_sample([4, 8, 40, 40, 32]) - layer = ConvLSTM2D(32, 4, 4, input_shape=(8, 40, 40, 32), return_sequences=True, - inner_activation='sigmoid', border_mode='same') + layer = ConvLSTM2D(32, 4, 4, input_shape=(8, 40, 40, 32), + return_sequences=True, border_mode='same') self.modelTestSingleLayer(input_data, layer, dump_weights=True, random_weights=True) + layer2 = ConvLSTM2D(32, 4, 4, input_shape=(8, 40, 40, 32), return_sequences=True, + activation='relu', inner_activation='sigmoid', border_mode='same') + self.modelTestSingleLayer(input_data, layer2, dump_weights=True, + random_weights=True, rtol=1e-5, atol=1e-5) def test_gru(self): input_data = np.random.random([3, 4, 5]) - layer = GRU(4, input_shape=(4, 5), return_sequences=True, inner_activation='sigmoid') + layer = GRU(4, input_shape=(4, 5), return_sequences=True) self.modelTestSingleLayer(input_data, layer, dump_weights=True) - layer2 = GRU(8, input_shape=(4, 5), return_sequences=False, inner_activation='sigmoid') + layer2 = GRU(8, input_shape=(4, 5), return_sequences=False, + activation='relu', inner_activation='sigmoid') self.modelTestSingleLayer(input_data, layer2, dump_weights=True) # TODO: Support share weights training. @@ -552,5 +558,6 @@ def test_multiple_inputs_share_weights(self): bigdl_model = DefinitionLoader.from_json_path(def_path) assert str(excinfo.value) == """Convolution2D doesn't support multiple inputs with shared weights""" # noqa + if __name__ == "__main__": pytest.main([__file__]) From e36bc6ddd1406a58a718429a1c66087c547b112f Mon Sep 17 00:00:00 2001 From: Shane Huang Date: Mon, 27 Nov 2017 17:47:46 +0800 Subject: [PATCH 358/823] add GaussianDropout, GaussianNoise layers (#1894) * add GaussianDropout, GaussianNoise layers, unit tests, python wrapper, module serializer unit test and docs --- python/dllib/src/bigdl/dllib/nn/layer.py | 43 ++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index e9c53e33f7a..6fb3dc11793 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -1750,6 +1750,49 @@ def __init__(self, scale) +class GaussianDropout(Layer): + + ''' + Apply multiplicative 1-centered Gaussian noise. + The multiplicative noise will have standard deviation `sqrt(rate / (1 - rate)). + + As it is a regularization layer, it is only active at training time. + + :param rate: drop probability (as with `Dropout`). + + + >>> GaussianDropout = GaussianDropout(0.5) + creating: createGaussianDropout + ''' + + def __init__(self, + rate, + bigdl_type="float"): + super(GaussianDropout, self).__init__(None, bigdl_type, + rate) + + +class GaussianNoise(Layer): + + ''' + Apply additive zero-centered Gaussian noise. + This is useful to mitigate overfitting + (you could see it as a form of random data augmentation). + Gaussian Noise (GS) is a natural choice as corruption process for real valued inputs. + + As it is a regularization layer, it is only active at training time. + + :param stdev: standard deviation of the noise distribution + + >>> GaussianNoise = GaussianNoise(0.5) + creating: createGaussianNoise + ''' + def __init__(self, + stddev, + bigdl_type="float"): + super(GaussianNoise, self).__init__(None, bigdl_type, + stddev) + class View(Layer): ''' From c0c33beeb75f095d37568e3f29d341c0d69e5320 Mon Sep 17 00:00:00 2001 From: Shane Huang Date: Mon, 27 Nov 2017 17:47:46 +0800 Subject: [PATCH 359/823] add GaussianDropout, GaussianNoise layers (#1894) * add GaussianDropout, GaussianNoise layers, unit tests, python wrapper, module serializer unit test and docs --- python/dllib/src/bigdl/dllib/nn/layer.py | 43 ++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index e9c53e33f7a..6fb3dc11793 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -1750,6 +1750,49 @@ def __init__(self, scale) +class GaussianDropout(Layer): + + ''' + Apply multiplicative 1-centered Gaussian noise. + The multiplicative noise will have standard deviation `sqrt(rate / (1 - rate)). + + As it is a regularization layer, it is only active at training time. + + :param rate: drop probability (as with `Dropout`). + + + >>> GaussianDropout = GaussianDropout(0.5) + creating: createGaussianDropout + ''' + + def __init__(self, + rate, + bigdl_type="float"): + super(GaussianDropout, self).__init__(None, bigdl_type, + rate) + + +class GaussianNoise(Layer): + + ''' + Apply additive zero-centered Gaussian noise. + This is useful to mitigate overfitting + (you could see it as a form of random data augmentation). + Gaussian Noise (GS) is a natural choice as corruption process for real valued inputs. + + As it is a regularization layer, it is only active at training time. + + :param stdev: standard deviation of the noise distribution + + >>> GaussianNoise = GaussianNoise(0.5) + creating: createGaussianNoise + ''' + def __init__(self, + stddev, + bigdl_type="float"): + super(GaussianNoise, self).__init__(None, bigdl_type, + stddev) + class View(Layer): ''' From 6e9a9c3bdb247b12d8aaf56593803974240d55cb Mon Sep 17 00:00:00 2001 From: Jerry Wu Date: Mon, 27 Nov 2017 19:09:19 -0600 Subject: [PATCH 360/823] Keras support - add cosine proximity criterion (#1902) * keras support to add CosineProximityCriterion * fix criterion * add unit test * fix typo * add comment --- python/dllib/src/bigdl/dllib/nn/criterion.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/nn/criterion.py b/python/dllib/src/bigdl/dllib/nn/criterion.py index 8f94a03f88b..7dfc41227fd 100644 --- a/python/dllib/src/bigdl/dllib/nn/criterion.py +++ b/python/dllib/src/bigdl/dllib/nn/criterion.py @@ -785,6 +785,24 @@ def __init__(self, bigdl_type="float"): super(L1Cost, self).__init__(None, bigdl_type) +class CosineProximityCriterion(Criterion): + + ''' + compute the negative of the mean cosine proximity between predictions and targets. +``` + x'(i) = x(i) / sqrt(max(sum(x(i)^2), 1e-12)) + y'(i) = y(i) / sqrt(max(sum(x(i)^2), 1e-12)) + cosine_proximity(x, y) = sum_i(-1 * x'(i) * y'(i)) +``` + + >>> cosineProximityCriterion = CosineProximityCriterion() + creating: createCosineProximityCriterion + ''' + + def __init__(self, + bigdl_type="float"): + super(CosineProximityCriterion, self).__init__(None, bigdl_type) + def _test(): import doctest from pyspark import SparkContext From deb04ea4fe0daf4c899fdf1962205dbec3482830 Mon Sep 17 00:00:00 2001 From: Jerry Wu Date: Mon, 27 Nov 2017 19:09:19 -0600 Subject: [PATCH 361/823] Keras support - add cosine proximity criterion (#1902) * keras support to add CosineProximityCriterion * fix criterion * add unit test * fix typo * add comment --- python/dllib/src/bigdl/dllib/nn/criterion.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/nn/criterion.py b/python/dllib/src/bigdl/dllib/nn/criterion.py index 8f94a03f88b..7dfc41227fd 100644 --- a/python/dllib/src/bigdl/dllib/nn/criterion.py +++ b/python/dllib/src/bigdl/dllib/nn/criterion.py @@ -785,6 +785,24 @@ def __init__(self, bigdl_type="float"): super(L1Cost, self).__init__(None, bigdl_type) +class CosineProximityCriterion(Criterion): + + ''' + compute the negative of the mean cosine proximity between predictions and targets. +``` + x'(i) = x(i) / sqrt(max(sum(x(i)^2), 1e-12)) + y'(i) = y(i) / sqrt(max(sum(x(i)^2), 1e-12)) + cosine_proximity(x, y) = sum_i(-1 * x'(i) * y'(i)) +``` + + >>> cosineProximityCriterion = CosineProximityCriterion() + creating: createCosineProximityCriterion + ''' + + def __init__(self, + bigdl_type="float"): + super(CosineProximityCriterion, self).__init__(None, bigdl_type) + def _test(): import doctest from pyspark import SparkContext From d4c2aa2517a6a016e81f300002b5a1b23f6894f9 Mon Sep 17 00:00:00 2001 From: Xianyan Date: Tue, 28 Nov 2017 19:05:11 +0800 Subject: [PATCH 362/823] Add keras highway (#1913) * Add highway * Fix unit test * Add python support * fix ut * add module seri * fix ut * Add API doc * Add Regularizer --- python/dllib/src/bigdl/dllib/nn/layer.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 6fb3dc11793..4d2feebf194 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -4590,6 +4590,23 @@ class HardSigmoid(Layer): def __init__(self, bigdl_type="float"): super(HardSigmoid, self).__init__(None, bigdl_type) +class Highway(Layer): + """ + Densely connected highway network. + Highway layers are a natural extension of LSTMs to feedforward networks. + + :param size input size + :param with_bias whether to include a bias + :param activation name of activation function to use + :param wRegularizer: instance of [[Regularizer]](eg. L1 or L2 regularization), applied to the input weights matrices. + :param bRegularizer: instance of [[Regularizer]]applied to the bias. + + >>> highway = Highway(2) + creating: createHighway + """ + def __init__(self, size, with_bias=True, activation = None, wRegularizer=None, bRegularizer=None, bigdl_type="float"): + super(Highway, self).__init__(None, bigdl_type, size, with_bias, activation, wRegularizer, bRegularizer) + def _test(): import doctest from pyspark import SparkContext From 4c58ffb3dce4d405a2549aaa6dd734056d9e668d Mon Sep 17 00:00:00 2001 From: Xianyan Date: Tue, 28 Nov 2017 19:05:11 +0800 Subject: [PATCH 363/823] Add keras highway (#1913) * Add highway * Fix unit test * Add python support * fix ut * add module seri * fix ut * Add API doc * Add Regularizer --- python/dllib/src/bigdl/dllib/nn/layer.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 6fb3dc11793..4d2feebf194 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -4590,6 +4590,23 @@ class HardSigmoid(Layer): def __init__(self, bigdl_type="float"): super(HardSigmoid, self).__init__(None, bigdl_type) +class Highway(Layer): + """ + Densely connected highway network. + Highway layers are a natural extension of LSTMs to feedforward networks. + + :param size input size + :param with_bias whether to include a bias + :param activation name of activation function to use + :param wRegularizer: instance of [[Regularizer]](eg. L1 or L2 regularization), applied to the input weights matrices. + :param bRegularizer: instance of [[Regularizer]]applied to the bias. + + >>> highway = Highway(2) + creating: createHighway + """ + def __init__(self, size, with_bias=True, activation = None, wRegularizer=None, bRegularizer=None, bigdl_type="float"): + super(Highway, self).__init__(None, bigdl_type, size, with_bias, activation, wRegularizer, bRegularizer) + def _test(): import doctest from pyspark import SparkContext From fd1552460d7097e1780668f54a7735fe7fcae162 Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Wed, 29 Nov 2017 15:12:28 +0800 Subject: [PATCH 364/823] Always averaged the returning value (#1951) * always averaged the returning value * try to fix random err on jenkins --- python/dllib/test/bigdl/test_dl_classifier.py | 5 ++--- python/dllib/test/bigdl/test_simple_integration.py | 5 ++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/python/dllib/test/bigdl/test_dl_classifier.py b/python/dllib/test/bigdl/test_dl_classifier.py index 99cef1ede55..792b61f18fb 100644 --- a/python/dllib/test/bigdl/test_dl_classifier.py +++ b/python/dllib/test/bigdl/test_dl_classifier.py @@ -30,9 +30,8 @@ def setup_method(self, method): """ setup any state tied to the execution of the given method in a class. setup_method is invoked for every test method of a class. """ - sparkConf = create_spark_conf() - self.sc = SparkContext(master="local[1]", appName="test model", - conf=sparkConf) + sparkConf = create_spark_conf().setMaster("local[1]").setAppName("test model") + self.sc = get_spark_context(sparkConf) self.sqlContext = SQLContext(self.sc) init_engine() diff --git a/python/dllib/test/bigdl/test_simple_integration.py b/python/dllib/test/bigdl/test_simple_integration.py index cdfd7aaab9b..23460b41df0 100644 --- a/python/dllib/test/bigdl/test_simple_integration.py +++ b/python/dllib/test/bigdl/test_simple_integration.py @@ -35,9 +35,8 @@ def setup_method(self, method): """ setup any state tied to the execution of the given method in a class. setup_method is invoked for every test method of a class. """ - sparkConf = create_spark_conf() - self.sc = SparkContext(master="local[4]", appName="test model", - conf=sparkConf) + sparkConf = create_spark_conf().setMaster("local[4]").setAppName("test model") + self.sc = get_spark_context(sparkConf) init_engine() def teardown_method(self, method): From 4bffdbde623dcbe8dd5061620ff390b50e24b7ea Mon Sep 17 00:00:00 2001 From: zhangxiaoli73 <380761639@qq.com> Date: Thu, 30 Nov 2017 09:32:30 +0800 Subject: [PATCH 365/823] fix python DataFrame (#1883) --- python/dllib/src/bigdl/dllib/utils/common.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/utils/common.py b/python/dllib/src/bigdl/dllib/utils/common.py index 5680c2539e7..dcc681ab303 100644 --- a/python/dllib/src/bigdl/dllib/utils/common.py +++ b/python/dllib/src/bigdl/dllib/utils/common.py @@ -474,6 +474,9 @@ def _java2py(sc, r, encoding="bytes"): if clsName == 'DataFrame': return DataFrame(r, get_spark_sql_context(sc)) + if clsName == 'Dataset': + return DataFrame(r, get_spark_sql_context(sc)) + if clsName in _picklable_classes: r = sc._jvm.org.apache.spark.bigdl.api.python.BigDLSerDe.dumps(r) elif isinstance(r, (JavaArray, JavaList, JavaMap)): From 45a02977f4d5913cea8abb062ec0a41ea9e776e9 Mon Sep 17 00:00:00 2001 From: zhangxiaoli73 <380761639@qq.com> Date: Thu, 30 Nov 2017 09:32:30 +0800 Subject: [PATCH 366/823] fix python DataFrame (#1883) --- python/dllib/src/bigdl/utils/common.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/python/dllib/src/bigdl/utils/common.py b/python/dllib/src/bigdl/utils/common.py index 5680c2539e7..dcc681ab303 100644 --- a/python/dllib/src/bigdl/utils/common.py +++ b/python/dllib/src/bigdl/utils/common.py @@ -474,6 +474,9 @@ def _java2py(sc, r, encoding="bytes"): if clsName == 'DataFrame': return DataFrame(r, get_spark_sql_context(sc)) + if clsName == 'Dataset': + return DataFrame(r, get_spark_sql_context(sc)) + if clsName in _picklable_classes: r = sc._jvm.org.apache.spark.bigdl.api.python.BigDLSerDe.dumps(r) elif isinstance(r, (JavaArray, JavaList, JavaMap)): From 291489dd558229edb984064cf7bf2ed9b2d5416a Mon Sep 17 00:00:00 2001 From: zhangxiaoli73 <380761639@qq.com> Date: Thu, 30 Nov 2017 09:32:30 +0800 Subject: [PATCH 367/823] fix python DataFrame (#1883) --- python/dllib/test/bigdl/test_dl_classifier.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/python/dllib/test/bigdl/test_dl_classifier.py b/python/dllib/test/bigdl/test_dl_classifier.py index 792b61f18fb..9bedbce686f 100644 --- a/python/dllib/test/bigdl/test_dl_classifier.py +++ b/python/dllib/test/bigdl/test_dl_classifier.py @@ -125,7 +125,9 @@ def test_dlestimator_fit_dlmodel_transform(self): df = self.sqlContext.createDataFrame(data, schema) dlModel = estimator.fit(df) - dlModel.transform(df).registerTempTable("dlModelDF") # Compatible with spark 1.6 + res = dlModel.transform(df) + assert type(res).__name__ == 'DataFrame' + res.registerTempTable("dlModelDF") # Compatible with spark 1.6 results = self.sqlContext.table("dlModelDF") count = results.rdd.count() @@ -154,7 +156,9 @@ def test_dlclassifier_fit_dlclassifiermodel_transform(self): df = self.sqlContext.createDataFrame(data, schema) dlClassifierModel = classifier.fit(df) - dlClassifierModel.transform(df).registerTempTable("dlClassifierModelDF") + res = dlClassifierModel.transform(df) + assert type(res).__name__ == 'DataFrame' + res.registerTempTable("dlClassifierModelDF") results = self.sqlContext.table("dlClassifierModelDF") count = results.rdd.count() From 5d99f7bfaf62e7ad10c8bf0da05c96e5bed3acaa Mon Sep 17 00:00:00 2001 From: Xianyan Date: Thu, 30 Nov 2017 13:26:32 +0800 Subject: [PATCH 368/823] Add Upsample3d (#1911) * add * Add upsample3d * add SerialVersionUID * add python * Fix ut * add saveModule test * add updample3d serializer and markdown doc * meet code review * meet code review * fix ut * fix ut * resolve conflicts --- python/dllib/src/bigdl/dllib/nn/layer.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 4d2feebf194..949c02c0101 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -4607,6 +4607,20 @@ class Highway(Layer): def __init__(self, size, with_bias=True, activation = None, wRegularizer=None, bRegularizer=None, bigdl_type="float"): super(Highway, self).__init__(None, bigdl_type, size, with_bias, activation, wRegularizer, bRegularizer) +class UpSampling3D(Layer): + """ + Upsampling layer for 3D inputs. + Repeats the 1st, 2nd and 3rd dimensions + of the data by size[0], size[1] and size[2] respectively. + The input data is assumed to be of the form `minibatch x channels x depth x height x width`. + + :param size Repeats the depth, height, width dimensions of the data by + >>> upsample3d = UpSampling3D([1, 2, 3]) + creating: createUpSampling3D + """ + def __init__(self, size, bigdl_type="float"): + super(UpSampling3D, self).__init__(None, bigdl_type, size) + def _test(): import doctest from pyspark import SparkContext From de106198b0a0b02c15d5b87d2f123029f1827149 Mon Sep 17 00:00:00 2001 From: Xianyan Date: Thu, 30 Nov 2017 13:26:32 +0800 Subject: [PATCH 369/823] Add Upsample3d (#1911) * add * Add upsample3d * add SerialVersionUID * add python * Fix ut * add saveModule test * add updample3d serializer and markdown doc * meet code review * meet code review * fix ut * fix ut * resolve conflicts --- python/dllib/src/bigdl/dllib/nn/layer.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 4d2feebf194..949c02c0101 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -4607,6 +4607,20 @@ class Highway(Layer): def __init__(self, size, with_bias=True, activation = None, wRegularizer=None, bRegularizer=None, bigdl_type="float"): super(Highway, self).__init__(None, bigdl_type, size, with_bias, activation, wRegularizer, bRegularizer) +class UpSampling3D(Layer): + """ + Upsampling layer for 3D inputs. + Repeats the 1st, 2nd and 3rd dimensions + of the data by size[0], size[1] and size[2] respectively. + The input data is assumed to be of the form `minibatch x channels x depth x height x width`. + + :param size Repeats the depth, height, width dimensions of the data by + >>> upsample3d = UpSampling3D([1, 2, 3]) + creating: createUpSampling3D + """ + def __init__(self, size, bigdl_type="float"): + super(UpSampling3D, self).__init__(None, bigdl_type, size) + def _test(): import doctest from pyspark import SparkContext From 99f6edefaadb67186dc1fe4ec03d9dda62c8701f Mon Sep 17 00:00:00 2001 From: Xianyan Date: Thu, 30 Nov 2017 13:31:52 +0800 Subject: [PATCH 370/823] Add OpenCV Vision Transformer Structure (#1954) * Add OpenCV Vision Transformer * Add more unit test * Add scala docs and python docs * Fix python spark context * update image --- .../bigdl/dllib/feature/transform/__init__.py | 15 + .../feature/transform/vision/__init__.py | 15 + .../dllib/feature/transform/vision/image.py | 266 ++++++++++++++++++ 3 files changed, 296 insertions(+) create mode 100644 python/dllib/src/bigdl/dllib/feature/transform/__init__.py create mode 100644 python/dllib/src/bigdl/dllib/feature/transform/vision/__init__.py create mode 100644 python/dllib/src/bigdl/dllib/feature/transform/vision/image.py diff --git a/python/dllib/src/bigdl/dllib/feature/transform/__init__.py b/python/dllib/src/bigdl/dllib/feature/transform/__init__.py new file mode 100644 index 00000000000..2151a805423 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/feature/transform/__init__.py @@ -0,0 +1,15 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/python/dllib/src/bigdl/dllib/feature/transform/vision/__init__.py b/python/dllib/src/bigdl/dllib/feature/transform/vision/__init__.py new file mode 100644 index 00000000000..2151a805423 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/feature/transform/vision/__init__.py @@ -0,0 +1,15 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py b/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py new file mode 100644 index 00000000000..48111234c60 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py @@ -0,0 +1,266 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import sys +from bigdl.util.common import JavaValue +from bigdl.util.common import callBigDlFunc +from bigdl.util.common import * + +if sys.version >= '3': + long = int + unicode = str + + +class FeatureTransformer(JavaValue): + """ + FeatureTransformer is a transformer that transform ImageFeature + """ + + def __init__(self, bigdl_type="float", *args): + self.value = callBigDlFunc( + bigdl_type, JavaValue.jvm_class_constructor(self), *args) + + def transform(self, image_feature, bigdl_type="float"): + """ + transform ImageFeature + """ + callBigDlFunc(bigdl_type, "transformImageFeature", self.value, image_feature) + return image_feature + + def __call__(self, image_frame, bigdl_type="float"): + """ + transform ImageFrame + """ + jframe = callBigDlFunc(bigdl_type, + "transformImageFrame", self.value, image_frame) + return ImageFrame(jvalue=jframe) + +class Pipeline(FeatureTransformer): + """ + Pipeline of FeatureTransformer + """ + + def __init__(self, transformers, bigdl_type="float"): + for transfomer in transformers: + assert transfomer.__class__.__bases__[0].__name__ == "FeatureTransformer", "the transformer should be " \ + "subclass of FeatureTransformer" + super(Pipeline, self).__init__(bigdl_type, transformers) + +class ImageFeature(JavaValue): + """ + Each ImageFeature keeps information about single image, + it can include various status of an image, + e.g. original bytes read from image file, an opencv mat, + pixels in float array, image label, meta data and so on. + it uses HashMap to store all these data, + the key is string that identify the corresponding value + """ + + def __init__(self, image=None, label=None, path=None, bigdl_type="float"): + image_tensor = JTensor.from_ndarray(image) if image is not None else None + label_tensor = JTensor.from_ndarray(label) if label is not None else None + self.bigdl_type = bigdl_type + self.value = callBigDlFunc( + bigdl_type, JavaValue.jvm_class_constructor(self), image_tensor, label_tensor, path) + + def to_sample(self, float_key="floats", to_chw=True, with_im_info=False): + """ + ImageFeature to sample + """ + return callBigDlFunc(self.bigdl_type, "imageFeatureToSample", self.value, float_key, to_chw, with_im_info) + + def get_image(self, float_key="floats", to_chw=True): + """ + get image as ndarray from ImageFeature + """ + tensor = callBigDlFunc(self.bigdl_type, "imageFeatureToImageTensor", self.value, + float_key, to_chw) + return tensor.to_ndarray() + + def get_label(self): + """ + get label as ndarray from ImageFeature + """ + label = callBigDlFunc(self.bigdl_type, "imageFeatureToLabelTensor", self.value) + return label.to_ndarray() + + def keys(self): + """ + get key set from ImageFeature + """ + return callBigDlFunc(self.bigdl_type, "imageFeatureGetKeys", self.value) + +class ImageFrame(JavaValue): + """ + ImageFrame wraps a set of ImageFeature + """ + + def __init__(self, jvalue, bigdl_type="float"): + self.value = jvalue + self.bigdl_type = bigdl_type + if self.is_local(): + self.image_frame = LocalImageFrame(jvalue=self.value) + else: + self.image_frame = DistributedImageFrame(jvalue=self.value) + + + @classmethod + def read(cls, path, sc=None, bigdl_type="float"): + """ + Read images as Image Frame + if sc is defined, Read image as DistributedImageFrame from local file system or HDFS + if sc is null, Read image as LocalImageFrame from local file system + :param path path to read images + if sc is defined, path can be local or HDFS. Wildcard character are supported. + if sc is null, path is local directory/image file/image file with wildcard character + :param sc SparkContext + :return ImageFrame + """ + return ImageFrame(jvalue=callBigDlFunc(bigdl_type, "read", path, sc)) + + @classmethod + def readParquet(cls, path, sql_context, bigdl_type="float"): + """ + Read parquet file as DistributedImageFrame + """ + return DistributedImageFrame(jvalue=callBigDlFunc(bigdl_type, "readParquet", path, sql_context)) + + def is_local(self): + """ + whether this is a LocalImageFrame + """ + return callBigDlFunc(self.bigdl_type, "isLocal", self.value) + + def is_distributed(self): + """ + whether this is a DistributedImageFrame + """ + return callBigDlFunc(self.bigdl_type, "isDistributed", self.value) + + def transform(self, transformer, bigdl_type="float"): + """ + transformImageFrame + """ + self.value = callBigDlFunc(bigdl_type, + "transformImageFrame", transformer, self.value) + return self + + def get_image(self, float_key="floats", to_chw=True): + """ + get image from ImageFrame + """ + return self.image_frame.get_image(float_key, to_chw) + + def get_label(self): + """ + get label from ImageFrame + """ + return self.image_frame.get_label() + + def to_sample(self, float_key="floats", to_chw=True, with_im_info=False): + """ + ImageFrame toSample + """ + return self.image_frame.to_sample(float_key, to_chw, with_im_info) + + +class LocalImageFrame(ImageFrame): + """ + LocalImageFrame wraps a list of ImageFeature + """ + def __init__(self, image_list=None, label_list=None, jvalue=None, bigdl_type="float"): + assert jvalue or image_list, "jvalue and image_list cannot be None in the same time" + if jvalue: + self.value = jvalue + else: + # init from image ndarray list and label rdd(optional) + image_tensor_list = image_list.map(lambda image: JTensor.from_ndarray(image)) + label_tensor_list = label_list.map(lambda label: JTensor.from_ndarray(label)) if label_list else None + self.value = callBigDlFunc(bigdl_type, JavaValue.jvm_class_constructor(self), + image_tensor_list, label_tensor_list) + + self.bigdl_type = bigdl_type + + def get_image(self, float_key="floats", to_chw=True): + """ + get image list from ImageFrame + """ + tensors = callBigDlFunc(self.bigdl_type, + "localImageFrameToImageTensor", self.value, float_key, to_chw) + return map(lambda tensor: tensor.to_ndarray(), tensors) + + def get_label(self): + """ + get label list from ImageFrame + """ + labels = callBigDlFunc(self.bigdl_type, "localImageFrameToLabelTensor", self.value) + return map(lambda tensor: tensor.to_ndarray(), labels) + + def to_sample(self, float_key="floats", to_chw=True, with_im_info=False): + """ + to sample list + """ + return callBigDlFunc(self.bigdl_type, + "localImageFrameToSample", self.value, float_key, to_chw, with_im_info) + + + +class DistributedImageFrame(ImageFrame): + """ + DistributedImageFrame wraps an RDD of ImageFeature + """ + + def __init__(self, image_rdd=None, label_rdd=None, jvalue=None, bigdl_type="float"): + assert jvalue or image_rdd, "jvalue and image_rdd cannot be None in the same time" + if jvalue: + self.value = jvalue + else: + # init from image ndarray rdd and label rdd(optional) + image_tensor_rdd = image_rdd.map(lambda image: JTensor.from_ndarray(image)) + label_tensor_rdd = label_rdd.map(lambda label: JTensor.from_ndarray(label)) if label_rdd else None + self.value = callBigDlFunc(bigdl_type, JavaValue.jvm_class_constructor(self), + image_tensor_rdd, label_tensor_rdd) + + self.bigdl_type = bigdl_type + + def get_image(self, float_key="floats", to_chw=True): + """ + get image rdd from ImageFrame + """ + tensor_rdd = callBigDlFunc(self.bigdl_type, + "distributedImageFrameToImageTensorRdd", self.value, float_key, to_chw) + return tensor_rdd.map(lambda tensor: tensor.to_ndarray()) + + def get_label(self): + """ + get label rdd from ImageFrame + """ + tensor_rdd = callBigDlFunc(self.bigdl_type, "distributedImageFrameToLabelTensorRdd", self.value) + return tensor_rdd.map(lambda tensor: tensor.to_ndarray()) + + def to_sample(self, float_key="floats", to_chw=True, with_im_info=False): + """ + to sample rdd + """ + return callBigDlFunc(self.bigdl_type, + "distributedImageFrameToSampleRdd", self.value, float_key, to_chw, with_im_info) + +class HFlip(FeatureTransformer): + """ + Flip the image horizontally + """ + def __init__(self, bigdl_type="float"): + super(HFlip, self).__init__(bigdl_type) From ec48bee89462467d9de13e8638f4755e932ad1da Mon Sep 17 00:00:00 2001 From: Xianyan Date: Thu, 30 Nov 2017 13:31:52 +0800 Subject: [PATCH 371/823] Add OpenCV Vision Transformer Structure (#1954) * Add OpenCV Vision Transformer * Add more unit test * Add scala docs and python docs * Fix python spark context * update image --- .../bigdl/dllib/feature/transform/__init__.py | 15 + .../feature/transform/vision/__init__.py | 15 + .../dllib/feature/transform/vision/image.py | 266 ++++++++++++++++++ 3 files changed, 296 insertions(+) create mode 100644 python/dllib/src/bigdl/dllib/feature/transform/__init__.py create mode 100644 python/dllib/src/bigdl/dllib/feature/transform/vision/__init__.py create mode 100644 python/dllib/src/bigdl/dllib/feature/transform/vision/image.py diff --git a/python/dllib/src/bigdl/dllib/feature/transform/__init__.py b/python/dllib/src/bigdl/dllib/feature/transform/__init__.py new file mode 100644 index 00000000000..2151a805423 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/feature/transform/__init__.py @@ -0,0 +1,15 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/python/dllib/src/bigdl/dllib/feature/transform/vision/__init__.py b/python/dllib/src/bigdl/dllib/feature/transform/vision/__init__.py new file mode 100644 index 00000000000..2151a805423 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/feature/transform/vision/__init__.py @@ -0,0 +1,15 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py b/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py new file mode 100644 index 00000000000..48111234c60 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py @@ -0,0 +1,266 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import sys +from bigdl.util.common import JavaValue +from bigdl.util.common import callBigDlFunc +from bigdl.util.common import * + +if sys.version >= '3': + long = int + unicode = str + + +class FeatureTransformer(JavaValue): + """ + FeatureTransformer is a transformer that transform ImageFeature + """ + + def __init__(self, bigdl_type="float", *args): + self.value = callBigDlFunc( + bigdl_type, JavaValue.jvm_class_constructor(self), *args) + + def transform(self, image_feature, bigdl_type="float"): + """ + transform ImageFeature + """ + callBigDlFunc(bigdl_type, "transformImageFeature", self.value, image_feature) + return image_feature + + def __call__(self, image_frame, bigdl_type="float"): + """ + transform ImageFrame + """ + jframe = callBigDlFunc(bigdl_type, + "transformImageFrame", self.value, image_frame) + return ImageFrame(jvalue=jframe) + +class Pipeline(FeatureTransformer): + """ + Pipeline of FeatureTransformer + """ + + def __init__(self, transformers, bigdl_type="float"): + for transfomer in transformers: + assert transfomer.__class__.__bases__[0].__name__ == "FeatureTransformer", "the transformer should be " \ + "subclass of FeatureTransformer" + super(Pipeline, self).__init__(bigdl_type, transformers) + +class ImageFeature(JavaValue): + """ + Each ImageFeature keeps information about single image, + it can include various status of an image, + e.g. original bytes read from image file, an opencv mat, + pixels in float array, image label, meta data and so on. + it uses HashMap to store all these data, + the key is string that identify the corresponding value + """ + + def __init__(self, image=None, label=None, path=None, bigdl_type="float"): + image_tensor = JTensor.from_ndarray(image) if image is not None else None + label_tensor = JTensor.from_ndarray(label) if label is not None else None + self.bigdl_type = bigdl_type + self.value = callBigDlFunc( + bigdl_type, JavaValue.jvm_class_constructor(self), image_tensor, label_tensor, path) + + def to_sample(self, float_key="floats", to_chw=True, with_im_info=False): + """ + ImageFeature to sample + """ + return callBigDlFunc(self.bigdl_type, "imageFeatureToSample", self.value, float_key, to_chw, with_im_info) + + def get_image(self, float_key="floats", to_chw=True): + """ + get image as ndarray from ImageFeature + """ + tensor = callBigDlFunc(self.bigdl_type, "imageFeatureToImageTensor", self.value, + float_key, to_chw) + return tensor.to_ndarray() + + def get_label(self): + """ + get label as ndarray from ImageFeature + """ + label = callBigDlFunc(self.bigdl_type, "imageFeatureToLabelTensor", self.value) + return label.to_ndarray() + + def keys(self): + """ + get key set from ImageFeature + """ + return callBigDlFunc(self.bigdl_type, "imageFeatureGetKeys", self.value) + +class ImageFrame(JavaValue): + """ + ImageFrame wraps a set of ImageFeature + """ + + def __init__(self, jvalue, bigdl_type="float"): + self.value = jvalue + self.bigdl_type = bigdl_type + if self.is_local(): + self.image_frame = LocalImageFrame(jvalue=self.value) + else: + self.image_frame = DistributedImageFrame(jvalue=self.value) + + + @classmethod + def read(cls, path, sc=None, bigdl_type="float"): + """ + Read images as Image Frame + if sc is defined, Read image as DistributedImageFrame from local file system or HDFS + if sc is null, Read image as LocalImageFrame from local file system + :param path path to read images + if sc is defined, path can be local or HDFS. Wildcard character are supported. + if sc is null, path is local directory/image file/image file with wildcard character + :param sc SparkContext + :return ImageFrame + """ + return ImageFrame(jvalue=callBigDlFunc(bigdl_type, "read", path, sc)) + + @classmethod + def readParquet(cls, path, sql_context, bigdl_type="float"): + """ + Read parquet file as DistributedImageFrame + """ + return DistributedImageFrame(jvalue=callBigDlFunc(bigdl_type, "readParquet", path, sql_context)) + + def is_local(self): + """ + whether this is a LocalImageFrame + """ + return callBigDlFunc(self.bigdl_type, "isLocal", self.value) + + def is_distributed(self): + """ + whether this is a DistributedImageFrame + """ + return callBigDlFunc(self.bigdl_type, "isDistributed", self.value) + + def transform(self, transformer, bigdl_type="float"): + """ + transformImageFrame + """ + self.value = callBigDlFunc(bigdl_type, + "transformImageFrame", transformer, self.value) + return self + + def get_image(self, float_key="floats", to_chw=True): + """ + get image from ImageFrame + """ + return self.image_frame.get_image(float_key, to_chw) + + def get_label(self): + """ + get label from ImageFrame + """ + return self.image_frame.get_label() + + def to_sample(self, float_key="floats", to_chw=True, with_im_info=False): + """ + ImageFrame toSample + """ + return self.image_frame.to_sample(float_key, to_chw, with_im_info) + + +class LocalImageFrame(ImageFrame): + """ + LocalImageFrame wraps a list of ImageFeature + """ + def __init__(self, image_list=None, label_list=None, jvalue=None, bigdl_type="float"): + assert jvalue or image_list, "jvalue and image_list cannot be None in the same time" + if jvalue: + self.value = jvalue + else: + # init from image ndarray list and label rdd(optional) + image_tensor_list = image_list.map(lambda image: JTensor.from_ndarray(image)) + label_tensor_list = label_list.map(lambda label: JTensor.from_ndarray(label)) if label_list else None + self.value = callBigDlFunc(bigdl_type, JavaValue.jvm_class_constructor(self), + image_tensor_list, label_tensor_list) + + self.bigdl_type = bigdl_type + + def get_image(self, float_key="floats", to_chw=True): + """ + get image list from ImageFrame + """ + tensors = callBigDlFunc(self.bigdl_type, + "localImageFrameToImageTensor", self.value, float_key, to_chw) + return map(lambda tensor: tensor.to_ndarray(), tensors) + + def get_label(self): + """ + get label list from ImageFrame + """ + labels = callBigDlFunc(self.bigdl_type, "localImageFrameToLabelTensor", self.value) + return map(lambda tensor: tensor.to_ndarray(), labels) + + def to_sample(self, float_key="floats", to_chw=True, with_im_info=False): + """ + to sample list + """ + return callBigDlFunc(self.bigdl_type, + "localImageFrameToSample", self.value, float_key, to_chw, with_im_info) + + + +class DistributedImageFrame(ImageFrame): + """ + DistributedImageFrame wraps an RDD of ImageFeature + """ + + def __init__(self, image_rdd=None, label_rdd=None, jvalue=None, bigdl_type="float"): + assert jvalue or image_rdd, "jvalue and image_rdd cannot be None in the same time" + if jvalue: + self.value = jvalue + else: + # init from image ndarray rdd and label rdd(optional) + image_tensor_rdd = image_rdd.map(lambda image: JTensor.from_ndarray(image)) + label_tensor_rdd = label_rdd.map(lambda label: JTensor.from_ndarray(label)) if label_rdd else None + self.value = callBigDlFunc(bigdl_type, JavaValue.jvm_class_constructor(self), + image_tensor_rdd, label_tensor_rdd) + + self.bigdl_type = bigdl_type + + def get_image(self, float_key="floats", to_chw=True): + """ + get image rdd from ImageFrame + """ + tensor_rdd = callBigDlFunc(self.bigdl_type, + "distributedImageFrameToImageTensorRdd", self.value, float_key, to_chw) + return tensor_rdd.map(lambda tensor: tensor.to_ndarray()) + + def get_label(self): + """ + get label rdd from ImageFrame + """ + tensor_rdd = callBigDlFunc(self.bigdl_type, "distributedImageFrameToLabelTensorRdd", self.value) + return tensor_rdd.map(lambda tensor: tensor.to_ndarray()) + + def to_sample(self, float_key="floats", to_chw=True, with_im_info=False): + """ + to sample rdd + """ + return callBigDlFunc(self.bigdl_type, + "distributedImageFrameToSampleRdd", self.value, float_key, to_chw, with_im_info) + +class HFlip(FeatureTransformer): + """ + Flip the image horizontally + """ + def __init__(self, bigdl_type="float"): + super(HFlip, self).__init__(bigdl_type) From 7aede85e21685f3974cec1a503f67f869f7813a7 Mon Sep 17 00:00:00 2001 From: Xianyan Date: Thu, 30 Nov 2017 13:31:52 +0800 Subject: [PATCH 372/823] Add OpenCV Vision Transformer Structure (#1954) * Add OpenCV Vision Transformer * Add more unit test * Add scala docs and python docs * Fix python spark context * update image --- .../test/bigdl/resources/pascal/000025.jpg | Bin 0 -> 95959 bytes python/dllib/test/bigdl/transform/__init__.py | 15 ++++ .../dllib/test/bigdl/transform/test_image.py | 82 ++++++++++++++++++ 3 files changed, 97 insertions(+) create mode 100644 python/dllib/test/bigdl/resources/pascal/000025.jpg create mode 100644 python/dllib/test/bigdl/transform/__init__.py create mode 100644 python/dllib/test/bigdl/transform/test_image.py diff --git a/python/dllib/test/bigdl/resources/pascal/000025.jpg b/python/dllib/test/bigdl/resources/pascal/000025.jpg new file mode 100644 index 0000000000000000000000000000000000000000..b1b0359c9aea428d497108f161e31c9c16a913e2 GIT binary patch literal 95959 zcmbTdWmH>F)c+gYiWe!~VuciUFQr8b6e+a0yK8WFhXO5HN-0p>-5rV)OL3AED4rl8 zKnQ<+&$HIOZ|;kG?_`}>$vSi9?C&`vdp>(||L1-iK%uUzrVPNs0syccK7ji*fD!;3 z>%a70j{RT4#rZGC!^OqH#mB?P|DS_^h!CHEhyWj-kc5zk_`meiPGgR%dcu&{A( z@$d;A%piFv&_eNGJ`T=<(RdFQKa>u6I1j+3#G`sDq=-+gV@dGL^RaM5MhPMNtA+s@ z-QREy5i74qB4S!P`X>yWT-?ujctyp;B_yS!m0rJ5R#8<`*VBJ*VEEyqk+qGjoxOvj zledqrpFb!dFe*AGHZDFPF*EB^c24f+FL~ceznA?euc-Xl*woz8+ScCDIXE;tGCDRs zF*(1mxU>xZv$6_-!v5~;?)}?8K%AXlTq3Vf*Ej#^!UEv@udIjs|4|p^gD&j<-T}dX zy0EbQ9}XN!T)d}3_*9BI1eTuE&x9igAHT{dX&4}47tw{&Sb6;>rsWWYa3cPb_CL!0 z-wBKS|E28z683-VS_M49!FqUkIFtYdz?sBdX{fw8mR)ezvoG>yvi)jfwWrIyl6U2@ zB(NI2F+QX_;o;TI=GRF7Oql>T=B3Xm!{M*)i1y8$CJ-XqDiIKs{9z-~=5Xwv3)S;` zPsVh;b(~q7-J?(Sof!0+)??pTb9myefAyKzqZu>sVSatxY+RH>LXtQ#s#Bt+o0HE+ zbuRCA4~TR|liYm`WyZ|W8qJQlV;$1U6L&;fSBBKurU@V4tr1??6zo;}b3SBEFe5{V z_duhn9a#9p{U}>;L!BFJH|8oNWew90i(C2vAXPfGs9XI7-{Vd4N@>%5MoX4u#EuJz7w+a082b#PZkTw->3&0H4Q9@~v2RGim{0m# z`{S#~2FFgX0`gk>9n{31`tBy6b|}5N+NzMBA}1 z44J6O2suuTvV4BMvqVV6dyDNw4lyJc6i76R`K|^FOZhQ3%RdX$NT?T?bKAD-p{Xkq zKFLwFmK@*QRaVHscP1Gt;##0qYHwM*RDG6WZ3Wr;kc2qfzSJcnd&B$b`svKyjqYiO zK1UtJvf4NaNZ(2yq_)}oc2l&am#u@0Q2L8XCpw8>q-Jwdl_)Pv3e}&u~;e1#4QN%jR_$@j=acv2WJ)TZP0jk7Tdy#3jy zt+5H7EPh5A;U7{dqADM5`NoSO_c&Ho#%a17bugqI=f~t&|I|(LFjflfJkw2Zi{qFl zb`MzVeTsGlUtX{8%&$^Dwz#Ex;fbO7XU1}ucMq`F##XJd-!?W*(AC_z3~DV*3JY?S zoSoBOycv6)TjkQ;UQix?4{+0<-)q&eg$g`2-8lVVm0LVy`9zJzv% z^x-&FK{^nYfo+Y|^A>AVM0gnrRbL%vR5qKF!(c7vp*_NVNf`ET^^!}+-%Y##=V4|o7pyVV z!*rVj4cd=JTscqwd=o5NpomWWTC}d0-%Ne>TG83MB0a6Kb$ySGIFmb-t1S;Skiy7c zUSf9rWAnolVBYfHg!}%^yoyrT~s-wR_r7ZPis|>Ws)a z9FUjlgeyeB&c8Mksd1^+BuDNVaJyLeD+tnG)|K?qVJ6K0C`utY=FQ?&(98On_@Mn2iB{6qZ4rI-~ z@)wOefkmACHJn@JtoDPit_?TT`u$ZF{953{v>jwv8WmNpHJ1B%T0buvJbvvxsJ7$q z4L{@a2(0V(tjiey3GHoe0{aM%2D2=dEY==B4GBGCmJi_P)VFQJ34-KfVM*dO_0vFD z@b12Z@*m`YBQ`A{kch6`jKcj7Ty-&(j8Na7uNduVf}A zuMqACkh)hS{pA%iaI9OgQlm=S3mF*Xh%^|8Tj5)C<{vZNNCmN7Zu`>j_Q+`8W%{1R zvG5>9MvxWH;Gy~6&m5*Xp##k8#eqT;GivXj8p|?aZMHKrFw3Zd^VWvxfVh!f(>vp{ z9B-BIRdltdb_zS*sk->$cUO$?c5U6DXM-rkp*?G1W_DLYQ(dOSPlELv%kBZ?Do;Vt z>d6`Fw!{%<$pwHhdU{wTo4*2SH-S4;)9-wYg=?u+;74zcnlEr=T{gUaeQGZAnj;6c zGj8^+BP@=EWx3nb+oNei$-UfI-Ks55d?SG-T-WdKcIMam1M@?(gc?+yK%4?OM|1Jp zj@Aqdhr*}8j?tr7GHodfHFl|fChNxr6$?AsdO4r0?sh@N$9&KvbGlji!ZJs`R{9(n z2h&O3Y&)-?Q6x=PXbqo($|96VV8hznNpTj{#pj z!E@c76&j9Tzcd4HgMxxgeQi@*w>b;0`G1k=+n=7#Zs?&9;{I0{A~4^j44>7R>hX7L zPq%{-qfM9W6fk$NOEk|ITSht=k&n%ykKKDZ(H4PVX;czI4>H=s(zCsWda*mM$gozK z`ko}Rx>~#qz;vQ3T}g8p7S-pbfNzRc(54E0)sdyr8$)bv+-zH_f*U-<6eC4Yr0fj6 z2PCkrn3K1txw2|^cP1TY4Y}+d@%G0XQv#}5QiXE4IczMvjl*y+H3V4AivcEw{K#TRJJKHPC*jTd# zL0eN)Yjnzj`5elFN)L-yx(47NrEN^^quhxAe^4wGb0YXM$G|#WWP%u=b5VP@^jZk9 zPL`bkza44>>GiKF#M!ZSH!hT>T9rm|36EiJxLR$Mg+_^sw$xY2_Oiq*If%{WJZG1? zytC_`-Zc!3!KXBHB)?38zRkuQPOzWVe(n;r=8Nye zS*J(qKnWKkQr;E#0^AhJ0*{0e*j(#VhAmr*LcCKiVSTvi<*C8SaR5@qR5^L4CWJq- z9qvA8YK)=0m6K|5MwuExfdSdC**j0zBL5sIOYPB2G57-Cdqj53)zEJgVck0Hd!xdT zu}k*3#%DSAfVXWR+yu@1 zsFSRjZR=~%1N^5DpHhjIUa;c<%)x5n+^-j_dsDnWhA0ymj-n@FL769T6*0m=hp+rqcjY)~otUO{I%h;#_2 zjk{fTZ?%U2_QFVG&Ge7Vc;ibV@qhmw-va_tAAgeE17=G~k>tS(LpdA@Pr2HQpmx!EdJ$G}N%aC#&ZzZAe*lVm+ap$>x_5iL8 z%qbZ=)a8+nl(ayq>v6Y22yL-!q)egK`p}O2u2O4oW+5{2%PPU98JY7mzi*AYXwLHL zMejUKa(`2~)GE<_5f-j@#H>;Nki=?j!_P4%!kCg+M_PT;UZZMFj#On;1zx6lsqGM8 ziEe9~RGIF<*t$Yf-7z2Rlmg{6G}wqbf0wws{3j&E_4ETlN~i%&0lFPGta0lo@3=fC z=tt)6u0a5Y)y#?4hpZ{;^~<{WK6_RsCu#2F=9M2-k*iSl=YM$&B(2k~ya;E5&l7Ls ze=>T#l+%*W{s-agHv4z(NqJTKF6qw6&4uMlT@HqIc4vSbK!ezrqrU+!R<#|cSN;u% zVqfmnfxoq*@Bc?IY&w_AEf#{xX{)l6Jh4-<%uwrhDjWDe&D5c`gq&UKNXB4&4zP9D)B=02|0WBc~;afC+hss7wKA*@oeKvp067VjyJ%Ee=OmkqMP``Bq zU2ZLge|EeXi!X0#qa;aDM-6kbR5xrhY*Kvvy1v=NcMrgg;|?JsfA=uueFG5;3AL6$ z>SB#*tI2{pxK?m20wXfFV!U$t+6ayd5z057aP4ieb&pA{71)B^m-?`^kg7Z2{yl&Q zEsNrYuZ}xn2vMGpJ27_ijcY$=j_^BU&uQ0bSZ4f$JMzmjE%DcpoH#6n!0Z7jg@Bej zKi;Ejdu%C^E~pWd!b@i2*2zr>OCG#jc$ooXzI-UHeW8OpN1gKn?u%hiSXnQCM}MZ`oCV8!an&)bSE#t!M5xP{?bVfv?dC?_ z?Vud7^RPG!Q|OjCt-IYKe{-R^_Cl__W2qs%rAy!TXLH8xuxj}pgfsNy zOW7zaUnXF;H(TN2@$9w}M7K=EmhODcr z=O{%SSuUZUmO0+U z)con}(bPq}t0jZ(82WQF2^^YUBPgR>3qLKyw^1~#^gq+=i33=52qgD%EqAlRm&52m z-ioOh>k7V#wj6^ON`jBBnLUVLt0e=^4}py^9_}Dk4p5}j!dMtZn{b-A%*QW=U!T31 zU|9$ME+*n8-98q!xCaRTskkZ1KHo|j%SBL+{VtMlfXn}0h|{~V>#2hD#W&YILi4sD zxcVwoTy0&;(oTb*X1D}q;?EPz4@Z2K?+e|8{}%^&*!`HTlJoPG4I=P{S-j;gAB`WS;<+IPjhHDt#C4J~f9t)C+ z6cC0&_4Ua}T;7w+ZUzGgB%jpjBHTx7?E91KAsh9^Jv|+*FX#m&uglVd8n6_PBTNd% zMnV(}dV;3Sl@d5wEf4~1ljA=VqA>lzu-lEmg>s@d>&xCr2C`&V`isHbH8da0o|IZO zW*cBJrmHX$4zTK&e>I0|o~9wt`tax?T13h>jVe^=`*`zlZ_Zy-JVGMa(v`|4silsx zdkUN=C5RJ$5e1e?chL1KTZ`O@QQo6GYrKo-scIKP06c`%gREuXpu^W(X z+jSkoHE#|TJ4()QVrK}%y0XB9#9vrv$~Bm2BB;%KmqRI0jZmJe!q>)YAH_jfv+q>E z#btq~gQUE21cTxXWO*icd8?8eLsT950GM3+;e5N16Sr6PUmaA+t<=AJ0K?jdys-3d zjbb841Akg0&Vnv8%ttQ2JY}7XmS=BIT&psbz0H{pFw6)k1kv2-sezstqOC1&{B)h; zJ>R*9O~yOi8IQFH$=x0p^7CDJx9xXS@$fyMac{sU5qJ-v9I<>Dtj(GTNnaQoKD!oI z4|4bVwf&doJ@bC=WB5{TOei19iBEl=Cv#1sT|PJikmivbuhiuc<+!eGn)veY*ZWV7 zx;-uY1@;DIb!AgV?@K|#W7=tI_L-Zk&sHJSEH0Tc!U;qs$!<8f_9rYIkBgbye9S5T znC`a;G6Qw9sovyBSi8UY^4?K(^3z5{Mjx{C!)lAcquQ8qIUlE0;cfWpg^tmGwWjR#A=ZUq7cHNc z79Sj%XHZ%u@?W>fhh-=luC=Salq3i&w;&THr?0-#9$|G^GPOcSv zB^~(^@Q6ZjtspA#ABKiR^VtCm`Z!mxTAbF)*f_0C#ujkSQ^Rlr!`38xQa-=8TzQO0@EF}h9abq^rf zYi&Squ&^z<`4ozNR9+a*3w~=u`R#0mgt_c<%sZ`*`CoO?XX*bfccv81wIE)2DuNh#xWzB!|!6GYr0*B4M z&mv&|8*IGvm)BzQdTps+p!OI!U9ja`ul$ztra-qNp8$o*V`R?sBd=A}L?T^%d|O=V zNM(JP)U@wb!y<;oo%x{E>mWV<^yt$0xhi1liFXHhE296h76%!J=o>C{*G|$0`#y0K zW#wR015@z|1CnchjYKDI0m;M*JR^ah7rXiDP~MP&FbdJ~962+_cz#`t0QF{r{jTT4 zMgG7B`M#JyeZi)m7X}L+qfpD|@`gcuDDTs`I6E>Gz(x7dgLi$wI@9|c4nEdKqzy*0 z+rx+iT>icRSuawkV`rob2hi^?_i(?tYw>jr(lln)R%&hQ5Plw1!C_Grgf1^pFj|_c zQkPgi`&25PeH4OM=`78k|1|a1ur=Z4ILB8nZM1t9S&|T}1Ycg=(#(<4a9*yBv8Cq~ z$trd_4~y?3YpzQYPl8dx@)IVM8fPVK<)eAfRTH5Cn$X!e+E!NUW&2yP?`)rI`JXf; zTY46W6kSeeKbhq>sGjNAma_g;_ZYh}gw1iSq8qm=e+D@BeXwdoLQ$lt|aTX>uff~D-o($w|e|)UFBEBWtM|*}hX6x?#c?Z@z5c zAxQU3Cn#Z~nSPJb+YZqfWVowKsH?S-(Z9Y4+H3BuhY6GbP6*XO5BKu8z3j^7h}iI5BUDn_;Yc7Z5QkY}vPJWX$Tl2hyZPRMVATZly5!n+WeA(3^He8S-jlLUshhztu`2C>}B#DbzdHyyFF_wVMsSSloX;@QU;>`sWx@ai-5 z0#?yd9JV9>(GCRz-H^k`B{UDDAivAG!{S`=Y~}lUg-9}8KX?0eV~F$b6ockWS?N#f zX373Tg}RcpWvNTdru`IxJC^JHGSmeYUwI`b>oBR*sgiCe#V&UHqyZ7(eCli@IXZTAxemJ(8 zYEj!Tw?MNMVfG+)F=XY=?s!v<^^Big%BWNT@Bciefl0m;0!z&*nbJF@`vu0aZnet**ZuDc_*}=}9ANjU$Oi*7-HmTgk zHF0>l1l~LE%`Sm8R%m&0FA)ejG}KGo8@ai84^V9DQ$%VDo6hsA&c2konD)`CHh4yv zc~y9fGWonvv9DJCt>qq|^v?j5nPvnR_S{y3~EJBcv7Lz$p$}QJb?Jx88 zoq=_F->E?IP?>Vp$TrjfJ9mt%3@pS?jg0Q-CO8KPYumJ_N)6m?xky8Cr5#5h@S)5q zW7Lie(o$U7$I}ntm>^uuF@+Q#FU=bBOVkrH?k?0C^5(8MSdMk&B$QK8ZtO}l$uIp* z`0G7D`drf*L;MjwW4zV0AIh?HlI17A5DXPP1V7tHK36NiK$%5@D^+;vXfe65uNJ zO!I-QT88tS_Ov3d!J7=CtYoqRuB36%7tAapmZljh<@@965py_^o+z-6^*6LSf+a&C z{(=?z(0|pn6>kL5YKdzB7w)$(G_cp>A5mD>^DwKq2bc#;$CM|PGrlhtxoEq*Vx=$& z8<4&SIA}8-WH@8w5Pn(T-#iK*bFMfL@H;52cq%yyGfUVuM{(^9%8}@YSPz?t^ex)9 zH)U>~fYqBmJoEJ&=WGnkKIo(Dl|o5C7+!BmJ6Czj{&1#2_&NK+ z5w+^Vcb`A}90u)v+mFJBALZ?ALBozG5XNsKnc3A4rckrV(N$L&H3zFNSVsqfrR#a~ zG>d(4tb?PP%KX3Xuo2oL8`v`fE#@X1PJt7Z=07b$MDotH=|jORc<^h7wXxM;myXF5 zWm`A7>$)to$OD_WzH91suc&Q<%1HVQ7e;l0@;^?zWlFAip&58m_;9|dW$_!GyaZ~4 zv#Tr0pEkzoV*w0LY22d}eo@1^Y^=^>JbPoF-tkRgWUH+>@OfwBv!nkfP!$HfGB~BtxtEWEm&1Z$-~USi+x=I&Yz+ZYe^4F=&-M%o47}gXVO> zZx)Yf7>pS|>~>`eoP2pLW1Y;>?uUEBaB~RRLR0&j&&e~bg3=&yg9>E(5@#>%+{r&W zHjuOvaPZ`&&nDa0c#k^sJ5-*{{OHwCvQ_VjlTMA>HIMOqAZlP(0i(@4S{**k`zdpI z_9!Ew$nXVST5#UY)le~Mu(tE9@V{PsgjuD%YmJd)&H=Ei2}oV!Wyn!HT8_Fy7!HRu z<*gf|^-yXsV@Ez@%&lJz65b`^O3Z0lUtCh1Miu-9f!o(qcskO<__l39;MJ~-@!xZ7 zd|7u{?^!pX@qD7+vxSvnF|is0u&`g$#2K59XVf_!SNkx8gxxvfEW zl3=+7^6W>gyHGSvv4pkj8~L>aNn~vmRCD+o?Y0c<%nYG- zbSl5+P~}jO3BHzYBAT3h+}pS1ZZBU*0W!7da4*NJO`Y@~b1GyWY}vS~$Ecrwvc3mk z1s1D5kR`5Og5snw@=&&^ibF3&!o_M=Q3&;kpb2h(*{hDpSc5mmU57 zzP9+wKYU;fQrhy+@iPueSHJT_wii?3+LJ_kOa|m=N~K{s79ms|?Dm*@5Gk!y{Nta2 zn!BxycuwZWt*crsjr4|e66%z(<=!$>*sa)_LmzeUN9ZZtubWD+FOU2{IZ&L6%S)Fc zfUQZ;Oq%$5-A~wKRI`UaJJ-GarkfU@M)1=z((TmL_xv_(+I{u#0gZ^dKXK}e`H`2) z5Rx@Sv3SJ%z0D@)DTgr7m*g1@y9YCQ8K@LBY52aW%l+DExUAocMsl(snuiMuP8eD;_XZ&giqvQ`vAIYiUg5x4n_vYOEataPf8LN5cu2wy zQ3W@KE9UhI5iR7gTR1JViGj6gylPPY4;K@D_kgKtyiejnzrWkbL*4}XGU#8(8aYnZ zme+&K6SK95d=M1Jg$aU8r^y=1Iwa5iR^ zLH$sH{I42u)%qdWTV7C#A7Ayjzjh27mRJHWj@8p@v6z1n$-l7la$P|e7^zaKb=3j8 z2ap95(WW50M{Y3f#SXolovCMze{9=mLq2mCc&I+vw(4PrfFr7VUj!>Pz{lb_9gb^! zjJx8qW{5{U5WTRm=`PL>WIqANCbL6-yL%CIi2|1r?a4@;G=a%InxzLMG~7 z-{p4;m7#8tCGfB)Uy{SFppdr*;^)&AJ9dglL*9QX%7%Z2WU1O^so!ny$(z-L#;#8a z?izF76cQw|>ce-Nd|f?W*N{)}Mtb#okP*BqrDuu0G0Xw2LB#5YN5vObr{o}yT=M`V z!|N=7BV~E>v1CKwt#Po{UAH_j+NTF@gZS5Z71vy-Hb08lADpUKt8#r*niG?cD{LFt z=2xZATpSVR;6HzHXANmhs~GJ>etCrKbWNEGnE(FsJMiZ*N9ANa^CH4TepG)|x3$;4 z{5V^6lV^o#FYz8=Lk;P9W~wo~$=hI?AYroBv*^sWjqV7&OC!kG?4C7+uLGYsR=H!Z z8b!W1f++61rjMLHw`p*})}Z=u7Krsh+#IXVg@`lNoC4K0nxp$7rBJ9jU_ch$FKnrz zr4lU+4wnu2zM=91w0rw5Z%>la?Avu5{0)KppPIX5wmP4gkKUSSzQ6+U7Wn+w zaA)$nal18k$RDY76$7ao|NZxuS(ffqZ9|3J8e4A5PBA)v)Z7L|8sPYZ@2a8}Z44Wo z7`U>3I37Jsy5kvLVXa6}Xz#7w_;7)f`W-eU|I|qr#jXkte(%XbuR32Hy*)Wuq9Z7O zRxviUkX*2rjW?sdQwBszl9;om(K;8gq4?^Ya2*0qs)!0Hl7|AQe%1`zL^ZcP{{xBL z=!c2T2S4d+PwT&8Azb=c%lEbD^#Z?%&9&u<5#0qlZt5_TVLV8E^V}F=o4wT8d^Z06 z&oGfN^SV7wt?CnGUAa6x%J|kEx~M?@0_)bG)8I_7%rDl@Qpm&u&~GJ;&&{%>6kXg{ zQMpwD)4X+Pb9MKg{5nrNBU6a)A!O^d&*;BaP=QX-t1rx85%psBP7T}!FSD{LY!|Pl z4PGvxUEN{Pz0IAGJV)%b`TmtA;^#$#Y(CIl3xq_RSQzPX-{J)@$>;Xf&zbSy4xNYr zLsp&|+vk(}BB?%mT7ji)clol^2xFLk$`<4vFqC2j{K#b^k(Pt-3bM)aZ1u^D&87O7bvcy`t^f6ooWdLDA*5mB~Gz`LXW&_ze@O zG&$dH*7DK$9SIa14TEQaf>Dv1^CvVQyn@p>oQ!Z8KWJ|t;-tm4Vr?deE#_=EIpr#Q z$TDt$t$4H^tp=4sYu0TzXHA`JKXnNwzzGMPj@S-(d)!348bOb&`b)eEc|b?EE6_4o z8fi_JO!SN3{K(X2-r9aMtFy->F7t2L##@D-KLlq4w?kvQp3|?d^387(y%Qw%5>yVW z@sfBq0;LhL`ytRdN1MZ)JPqZp@e6zHvlN=W&s|BtrR0Q zI_~Pi?2;Hmw4q_~0xx8NP~ZkXDYbgHxT{>zANyFI@4lRG&3t|W)~m}@-Ne&yB+H-t z*3y~~S)2H&PP0Q_)tXj@p~kE;_z#;p8^viS`WJ0Z8<9vPe_`EaYm6B`P?X#sQ$Hh* zXNA9Gr2zp)&mJBU$W11Sq;a*jh7s3yJC_;oH*Wa z)NeFr6krqneH39Xx=0Il?zpV9a#G~Bxd+rKO^Ak}cPThKz+ zu19sv#plyrMbaol02GynJl9RRI6kuOp#l=He=E_+kz%t1J^5R)+Z(lb57^GzB9_(s zPAZ{+a^H^k0qZo{!+Sc?xiqjpCOnPfmc85)+m(N443R;Y83p1YZGXAT#ovIZi6pIw z95@^mZ?N?33$Tpe%%#)>6)cpeEK7f6l?T;*h%+T&82p*s@#^ayM;PU-d0D_!d@s|# z&Z7Mmpt5^$!sCdD8SixMx(CW$g`n2@edO!k07u)pQtu?F0@A#Kz(O9++G1+mvh$u( zbX7aIXTJmCpwj4ZI>gq3UZ3%*d^zy+a5j=abLa)jL~nc-Pv`4NcBjhsyZd7|K;UOA zH};0U?qTs_L(0>l5H2m-mY3Jv>jHiE*(#+k#`>@s=K<&)n{$Cw`VkJf)S%g6i7TNe zWtkus)1mlB`Sr}_$v+D${QIBOY@xWhk4fqj5jdl)WT$RM8m@237MoFjWtE<(HHS-E zj2dE4H2RL^GGk?Z)WwWCyu9Qgn7<(<0^t`hzb5UIk&z-jM7Qs%=h}h(OD{W zpp+l0#KGDe$0jy3a?T5xPJ96@bgL?0KMWCkAex$S{NO8v-Hha^Ze2{bc=ehOA{(}w zUSP=VZX;SuWd>@}42nN-*U|l#AM3FGd=MSCqbv84rj$G3()~r-XRoj>^x>?&{rUki z{m920p5ov@Bu*%_z)>*gBS&fYY?4j-qSnFrW(}~57;_;`)wKWcDO-YXeBhn4>Rhg; zbi^l?CjOGzJ>VnfK?AEvhWT@qTf)$V&dr?F?DE*^GPQEod5x;) zUV(didn}g^8~L{LSQ%SB8wt>Y`d{M#vL&Aq((+^c$$_T#cI zUSU=DE>{uN1qr}Oj*z=fQ3{HFB6|y_{8^lQCEZP>g6M-3(C$l*FQ2-~h)&W|CC_Y! z_gkb3V{tXfKLoW0Bzl?Ae2CO9u;VZ&ep}7velCwUO+|!Ni@#ao_TR=^yPgH8K@2== zZ1sV~%DTc~Lt#z}+h?wC>$xbs3y!y`XLW%B_L2v?z1zD^A#LCiF%Qdf%;?oX zC_TKVIL@uPVrPlJc{}Yf6&vwEwO4?G`i%;P7^^y180CITa9B(f%7j*&KNdSScy8U& z(Ab_(WMS%00LUzLkdk{@H!VPpe!bu`oHi&=gZInTjnmb|7AsC{K&v8)8zoq>%s zI{wG#TFMtPJXWWpoh9uyqGDdqhf7VHvlnummTuj2{H%o-@u|GOVrCr@AOz7V!AVaC z`;w;Jp{^wQ_2fa0_04mpnZu1z5LH;QZvF>9s1?6FHfGIfK$^+ z>WY>}V>7SXQ;_W5at#J--mH??9OP;|5J~~wZCJpkI=hhx$$cfx@j_!_^)%8;`neb) zwFFk9q}`PX8@k~zcnx{GQ2qr-IjSKMipHAu(kkP+)^~S9N{3{tpCg;*@6zV9Ba7JX zB2Ilj_?*VdGs8Qb2>bqx-V7w4j=h(_;kc8k3z_fUcx&`$s&@6m3X?0>j;>_x#=F1d z7C^)@7#f#S{g({vHu8bOI=#e%DN<-lYPV+#>2ufRhM_$a2P;te#CN|b86P&28LSC& z&{j@a!+t4!q;f3`+Wl3V?>CbjCK>Y*NB8 zpHHoW97bw=gL=wc1WRE%q~U6}V-a(C&|p`4c-9CDe*GL;4qB)(&>4NcKctUf>0C2d z7CARldu+ixqB8jy{vnykaB35HSAw>ODv+xnNM7B2t#J$Hl$#DL9QJ@CM7I)ht{7Ajz9pWK zBJf!U%U{aFC)4K zn5pb248t*x5VxQvokd7zN84|Pfn`;}4-KQx?qpXci-KF?>nkW(PhGGMBG`;|E;Ovk z?W1hJ`WME3(y&Bi@v8su!#(|Z=J!Unzr+r$NKgopzkTFsE`cHheS6p{OK^)V+^8u< zb{40a9vN!qJGyRjRndQEO{oa3xcl4OJ2f-W_|ru7AkNr9GO;a0DZYlDO4& zVz4V=iygcNsAUd)#ETugCU;Y*+`0<%n9jRiw#D>Mo(=`98K`8X zbTU}>=%ykf{oW{?K>J27-RgRZXN;uas!qJy$r{0{v-Gcd^&X%gvN13t`9gqGtFwUa zSnQV7v9m3{46zmA3v^&LNU`l|?A>Ji#lHJ=2ISVx&i2;}IOHpU;_4d{pg?Jzj!63t zY3jWzG^xg4cw6-Dcg5sSns+~iGPU(a!c>X~?y~yqidRmDC=WDB9UHYDJ2MrrfobeJ zI(?Zw@iRJdXE14`6EcCP!`kSM`L^?)|Cnp8i`|q2|JKQ{UI=C06l>sMX)+CXPGF{Z zY_1VC;VGDi%oaN{M2Bs&N~KK0mpIm+h8emKVTjNe^wa-*IndN_@sk?aCkPGYB~fN^ zsXS9NjeK{W91h)Mh9>(xH-KD(^=i)oG742%g0jMbw+-}B9M_26iLc5ye5+qO8?6ct z$LR-wniEHo9Vb^)O=8ud! zr}s=xV^v#iCWO=JdH({YyOI;HRiIbVzG?$9YB% zJfjhIBXCqU9rLQ^NcOUVyS{(VXUpBEQB-T3JPi7Q(Nfm7YT5Tc33R?kfel5Gl0i_3yos=n?U)z(`t z@L7g)I5kxVV-HDL$k@-9=%h#&w2D`mdBwd*lcV*|Ol$nHZFA~TV2;$qwIx1kYB0>o9|_$CWaAcJ8W`!5+tu5YwmLm&#g#q9gKXUQgK zVTcIGSoL`Z+`ETQuWfU{pws`CUmLg*QzE^z$ktc8mz7*e?33?u&ANRLi1JMl7+!rV z&uyPzN9|5hYwL9Q*vO!wwY6-)A!ogH zQ2TbpI!&1+`*@447vCJ!29+4v@E-_eNwj5X&hy3OJWT5Q{Lw%d+p{-R6vLMBp8(#8 zaQ9Vzq}!r`U2%%IbZRc{Im+<;B5ep6Mz8$`{S!V;_v?KYFQT3FD&LJX|6B1g>C9^6 z{ATi5J*u61V)4%St~6M|xjOh!NMoF;g>Tx0%ach426KI)+yLj0rL-2`MO`$ zr@egDZbmTQm7<5-c<^f72oueY=7Ll1y)txaNb1`E2KU2|fe8ryR=ZPf}B zYl;6x2I0(ica4TU4p!}~LFSfkD0f|{h)&)PPkLt(-4r=@!$Zpp&5m9VtGW^M%rHvz z?;R$d&$2l9or5BL8TAGEpslF8FU*WKf`vHEK}!kN{z^frQQkunVXq7uek0s? zBHB>YZ~P_9C%5EL>n|+KGemO$bMPE+;FrT>E3KLj_(U$bv2zG!%?;M>zB7jJ9V>44 z5ST8@zJcZhSTB^EmD0Pv0G6J2C<69yn5Yv`3dn@JZy3r079Mbm%(S&`8<;(o9kQ5u z;6pppVX;QxI@LNy)A&*QC6U;LUkXBC+kD_*eXx2J)hmA(r&u zw*ca)7IpMM6os<$4v{<^W}=7=6}>tZA1do)#x`$ucubq)hhZwL{SlwOX>u|~JQVXH zqEU*`du;l*&flw|xV6c$k1fwRtxw3K?g4b&irhndIPbvt!%NK${M#AIN?)EC05h(L z8`vVi?o5XHp})=K7g9b&s!ho++hJYVT#g1^)IRJV9Z*!Tn|&fyJk&>MW@`SnqIG%I zps@6p832Fj{mXI-09K|lV7ChW_5pJBh4(?YUJU|MMkFLSv+KS07tX68bWc__=4L64 zHyoX2i-J`^xlMA5Qh50h>yMyKnl0_~dmW2hXC)sn{;S4J7yW;;7H}2~4pD&t{FA&5 zni~Y(2TpZK&2^2Aj^&M1HXb%Q|Hw{4noM0+iXPW_Phtd}z^$9~oeRz7b{|TK9%aE7 z)Th9#gES;8-*a&047kzMr$Z8k=FMrVd~_6R|E|1ejs(S<6+J?PhaUwlX(uvFpVYwu ze0h%uI5D&if1M*6B6H~iwwAeXHI{_g}~j%=;Xzg#j5z+CPA#~eKwJ5Y^&cztR2`Z z=n0Af>gX1BIzoa*kjrQPhoZ9%Yx--$I0}k%cPb!Ccb6dD4H8otCfzW)y96WzmCn)K zDcxNgAq^WK3>f>pd;i~d?K(UAp7Y$#eSe8k8KZzyD zZ1b~H;X^7vhyHx_XH)Rr_@_GbbGxGpIpKHoyG~cc%Y;{BH_aw>*NiQOv+_0J3#RoO#j8Q-aTTkm{KlBVl#UEgZ7H2p+Io%&PC=Xv~M;cYmA1{SM~ zF0uU{=67^jV}&>09mGp|eAwudSu|#?7s-k_rRF7q3G;nBuyoceUO(H>PCMu!e9G&2 z=Ep-_E6mMbOUbNmj)<{H*B%P*2ZG_E!y#dXSV9{#%|QxU3oN~%6Qv227eCclDsX2={a|=$eMAQy~1=jI<&UsR&=rhB%YSFEnf8Zq-C) z0jZ*k7MjF)e$*I1xB?B%EF<#W|Y4NqXc|z9k@r& zJ4h5(z`r~c8q4D%wC9LNvPq+sA51>61-PAu3sNj}y@u?(8gPG{GSdZCjD)P}ybPh; zg8OvBqjxOpTM%Y27X1R6Gwbi0P7+NbEk_6b_|#e0?K;hR3(&%I$=$%xUbuD=@--BK z6rvPqbP_0ks8^hUC_1fp-}NG_C5u>tr+S&B%s4EN+E11L$bhM~3rOKI?go-gSMkr% zr&SrZ<_KUH(QXnS0-HF}zRhvn_r zC7W9@{*O%1=#De1cxLgmmwX5ok@C3xa4L?xninjM*M^|_{>=6o1b zWytqE=Y3~Yp|#RC{x%i0_{NqF$KYPU-INGG17i-{^)?t@Hqwst<;nJj0yD!W)HDUm zN_$cV|5Ci}G7r^Nf zg$6#y!~^YP;m|gB-8>STDOQT_f+q`ZAyxg{V-@2iB#}|Df7pYmsTYanG2HGxWg<0g zz2Qu^AD3!{j?VDcgGQsyR}3O4HNJgYzi`+$heo_R(89wVMA+`p$;QgO>Ve%>6wffM z-0Cze)|4sGZH&=ubOOI*Hb%^PgPW%-9v$Vopg-|T&E-y#yXG22$6Gs}@W zjoAk6jimP^Ha)wW9s9%d=@c0V&y`?Y#KWHE zys)F{VvS9bP4f@_q)T)N`Lqw8Z~LigNA_z7;vH2AiqFq(6fM2();xdM9nMu}kY^CJ zU60oPCS&p#!xAo0egn<`Q2?EP%jQT=j9Ob+^<&%W8>~F*)ng6bOPSh930lhftrODHazCSJ1v? z@IAxD<{8>4|1y9;$yDnF8#Ya5%%t{DG31mvoD7i%^$iuGn|gUAW^9zK*b@acrLmI} z<+G9y@l0L?yFL91vEf$jp*BIfAp%|La}7Gm8&+yaX;`91AmtAE>dk3Vvp&gkn}!Xuvb(H=n86@p4FgPxH)VLT&#$C~9lsR|}} zsty)%kinnFlBkWV1X9fpS#nD%w%D^N(_}8%;6Hd;%p`OLiWE=-02_ntEu*+%tQi~a z!ce9r;~@gcfot~LlA5N48Ir4?-wU_VCP-%bwAtN`*A$Z)1u?^g_q0CgZb^J!!MPh7WsqoZO6%_ad$pTc4)h$bMm~ z_vCyp_V|MTmom}9wmt$5rTN)7L4ukj3dj>IVg z$sr1h98mAu({)4q@W&&((nsPgzL70>RG8mR2O8TJ>G6;z|8ViB5Sz%6)~0&OmlE7{ z4clqlYK`gAb){%2D;2)pRCIlNbSE@DQ3uat4k!A>a5@t#j+Dzrt913*aa_xKtuk-m zTt2UcQq{hQqNTae5Qzt(+lYcwi|k=hTE$4+oMxPDPQM<*czzKT44+pLfyTpyt#7N) z_?HQELQ!TJ%h!l%eG?XOO)&p0G3}z55qJO>Q`H&1>Mp37y>;BRmf{2k748SboC)Hw zpP_}+00Y;yw(cKhuB=iuDTDD3U=)t1T`Fp>KpNIPjBJ7L98tEW-{WMZDhrm*2fgwv zEC*-HTKh4nBcA#YtG~%U>#0oE865kt zGB6Xf?i=ZKNw5z1?Nq?I=zPKIEU|2LhIl65grG5raB=QSugU_~qF#jb^U7NDuC8dL z?0Nx+^oK1u<_2bfyM@#OTUxCOFId9~nVThH##UK6*wflmmbJ9J6fth))maI>fBp^$ z%U82$8T)Adhmrkn7HNqXd*mm2ff&E;zhxp8z89X7xg76?9mX|b8KwyS|+?ZXjFU`fqsTh(cRoTp8Wk1C- z(nB6RR!HVOL!b=TK)k(>7C~+Oo&#S#AEck<#JD8!z>9Xn;CM9v3s_=GkgfAgly%U? z0f2}K`+OHRPl+WR)0JIwxNCopB_HGTF`Dgpl9E^0vf~5)5r$xw#b2_)4`Z8()4Vof ztEEJt8{StnP$Ncjr{}P=a?Aq;3O8|zBSuPuXtAHGYHw4lN|JP3_RyzOe$@-|iubsg z_rDXbHpr7Ie|k*io|$A7*){Qb>H8{Qp@~xpaH>caS3<<^{&QhT9Gki&=X_6sNew+k zS!E;4MdMs2+tMszEgzDF?OxYj$ZsZ?eDu?=rW*BK-+pOV6v+<#&I##u3c<-t9)lV` zss<~RGX5a-rS&zzq|)-b3epuZ0b|-@KfBu#mwEvogicpMe;C?1XRZHep|sU+YSQDe z&Jlb3Gl>mi1Sx0relJd0!X;D%53>s z#R}GsTdxx(uBZz(vh$yumeXv@P6S=whtHF>dD9;X&ET#%S#mA@OS!?ZTn~J2^nOIX zambJPhCM%6xbIT)`j+5EYW={uDC4WKoj=ZtA2JYWc!BC2Snnx$G(wm+rGqgHj^Buf`cM?9OJ#mly>am zN8_UykC1r1#Gm)J9c-#v<`Wg&AzsUWpkW+?<)`^Z6yU_>89YR*He5E^o3d%qD{r;- z%w<>H!9H&JqbmZhTmt}EN{F1ay9_px7Jsy9{4C;TeIh1s?n7SvHr~Q zOnTZRKjpJ*e98KH@@1aF?cS{?VgquyVSM{3d)Fn$5h%2f88P(mVo_bfWcy8YI+YSO z$X+rvZPKNhZ@kYxC6*aM^K&vcsQ0Yc+bqEp7o2umSXVE`%pdQq(?!yBPR)w$G6G=A zmvKyFsOLxr0Qu^1ZqG014*8BLA|Qm&Q9^%f3m6c!8TQp^{l$GrmayRDhQ|Q8H*=0P z!9xlhZ?7~1);qFNh>J9X{rkM=wIBgFZVm8HR=3msc7PkIB9nt1j&lm&2M};c+f#$2 zrMgnO#;!%p@FqlRJA`^}K%LMZ%ELX}iy`$OLFlO_$1u6ef|PVwO?N&^Lu*!LO>$YS zHu%#DqINwv4O&I?+~nk?qe7#(^VdE0zk97yg_A3;yE};ihjBtV+)^yXLCG|jm_D~C z6PlC0ch6EjW2{SS`Vpm|*!^Vt-q?06?e_GFPddKoBwP5)!W_g|xHdJBD?hT!Qte;6 zqqFjQPKviI;dG!XBDE$l14XgVC<@P6Jj{Vi6EQ{U!j*@TTt_8}G^JJ*?kIv!Qsg+X z^;(y_O+y-*ILSP{52)PpSvj}%tp}J#^^e?h*GW9>Q$%faXQ7G5GH>B7jn9Q-9g;Vc zk}^EaEu85F2#XWn?1wb3(mv!t=yecgP|65syO9`Mc;Dqmh-9N+yDsXQX^>(7iT{)< zOa%k_4GyJs5A9->Aw1q&bG*rdG)(yEc8Cldc>b}}q7LmDbcb|GQ_gYM{FpTx0;Nu> z6*>1a2SA8}Jb=f)vejA`G^>P~kpE#Y#>^NEy<0l4n9R#-$+{w*)UszP!Z)6h$WiH1 z-Ya1Wkd5W1znRuqnN8iyi>r3U$W)k?^=0m75+MvB_6K9aBe~IQ9;+LLTd|j5$JoXZ zu)>-Zd6Fv=zp$;;DnFAr(iMRG4K9vy?y(+G{ZmQ%wfy0$fb+~n1~%$B$D;Zp*m^}; zB4C9gPuMJNYy9#x$7m}5y}q;ERVk^ttD9hIy0iGGP9VvmLB^X)c87h;mE2gR%JDty z^!LIg2_*C9b zSLAg(dGwexo4)6LC%~^GO?C0YpD*XPJ7s!;Qlvl-T+){{*wHHFS%5iX~%?yhWXC z7NcDza{wgP0gs91DvjRJjoy9B@+Y#zAo;)_k__<%x5+}{duZpn0Puw&Ms7qgx6QIp z-mAia38?}8uhY{F0!ms81v>-0XM30Z${%4Io9lMZGR#%ifjA z2*~IeHvDo(mavIHc}U(uM3Y@7{jaqCFo}@wdU*Sbk9F_F6!)B3=rC+>#}K`hs99Q1 zM9M(N^f$J7FT{TsyUxNwdwhl|EV%2c`{+4uAwgTo~i_nAMDnsBBO z-q<+E-mZEMPqFz2-ykow@$#2$g8A>H?8~PDxOQ-bHbO|Z*8J{vE1Q3GM|!MTI8Jm< zwTVyKYN?E1SZ-FtAMl)2b*J%GR2j z06GjPv3VYl7URD=%%|;xxx~Q(heC*p5V|ns{#M$QlxFpXFJFF|jbo*2Hs>TB#I50f zru4C*FjPGabsM&IY5LY)G_$as(7Wf^$um_FnGm7be{vJxf93+xLQp`p6Mc0Z3V5Ub zsqEQ?*>{G0OrKU(`7@zJp~OdoD-K4hL6A&9o&5mQ>P8=J@457Dg2Qe{z6y%2ZNJK=bsXx<_Hb1~TQ8 z2D<1!y5-A8dg_{Aw&Y#523Ozwn}VzH8O3iuBF6zuzDNqE$KiQI#Gq_r?g^j8_9mSu zym=tXHxgfP1FU0~sV9fqD}uwqgatBLx-ko)JROmkt|1pqGn?UV9-jNwym-{trFE3( zu}{(-yS~aA)|R5tHvhRHoX$dqp||S~>po`2$_?^#9L0Z>YUz-B7ioc=SDlmMXC=N@j$ort?#L zDp=OlUSWw)7uCHst)qS}QirxcDJvnee0L=a$jHD+q?OLE~6?b{HHF0iHWGfFD$B?ZJ7%h1AOvWt~nS?BJl zJ5=sDF3w2mH|zAYZ))fN!+?g(>v@*#aBfU7^Nn%_!?iY@`ypiEzyd@dqzW8GMHS4q z;VT??VY%b5TI^J6?HLBBmt;>pGlb2lUZiw$CsLZJpvOQ8KjBGl%34f-i>iE{K=5K6;be(3IVoACKt|H=JeR!8G8`1*=U-1x>QvzN|G z%yt&`#quLbrhy)zr%oC49(dGR_0yYw9B-JbM#iM8st^ek5$=Zz$lZJRq1k2Ew;gCa zvp7|v6y7ek7H7BU$3^4^FYou~ytFj3jb{gRd7NX{ewKU&{4`U+K2B&pglo?`kjG+= znBV*}>P2s(?+Qd0%gdWBltQChxS{VxrN|aD%oZ~D6aO~7Lv)@6gv9?NcoVp~u5$R9 zBzazZFJb|HX`$T*pjkaCB)*s?7t5SbUTtzhXEJ#^Z(v3_G3>lPUwMIHvjLcib+ev?< z=pvwyxGuciiN8HvE$QQzdJg9jtuia^+EQ0OailL43PFo?G>C>mX{}7rl z=QZr4>9qWRsiwf~fH^MIgn6&yqJ53yq6V}#uC=X~!SP9IYQ$Hv>cP|q>w#cG*b`c@ zVDbx~Baj$2Dfz}RK1V6;o8ilK=5H^~rRCvQ8NyiWj8-|(mSo?BFG8I-wX?*8e?Qvy zVTYx+&6vCKTUciQ+OQmRtlFYcU-OS%k{T$dxhjdJ-jq)#rw=vBx5ol0pqF6bsrqIn zwD36@)B>EeHwRLH;|dlzh)RBwdg=c=IQ1QW(9Q^$6dQ9V(Dm?#f_q1S3Ne%)@N!I6=x4#ATaxt$Rg7uc1{ID3xDfMPg%4 zp$jNNv$W4uVTCB%6d0i^H6pMtE~wN2#nm(G-1KBQh4~6X(%p6RQiNrg`E|C#JyTdy z7=j-hJig6?XnfSwgegD@lVzKmru5CQ`z2}VmO z+^bjT#qY2vXNSnmK4KWe8M;y7Vhn^*T_k7jhcvmK3!sB-CM|PZJ&aqY*$)K?&({^5 zW$^vd?Kqn_1qDe7$;pxj%5uiGl`2I+_0pJ12v2*vBHhI|_iFL-ae*{&H-9* zA2@-QtQoNoyhhn;N8utgCa!XUS;rEPF__kgmOfu|$kZU3ZR*TlliH7m;t*LKf{%wm znb4g$8TN*!;%0`oY*BACm6mbt?C)C4A_Ova@==!m7PL+Buhkw3qYGQvkt9{O2~Wvf z!bu_8=|mI=SCwQ{J2+*83vBavJVFxuH9@eCKipO+i}aP`B8epl1VE)(4Mbbon`^51 zzjyJv;}1~RrJD=7vM2wC@oww__~8E3+Jonp0f2oBA(5fu;y*D`fn!x|G ztc8P5T(9tFqeldbV)S|#zS1Z2@+n(}YjaDX%)+q)d);>N=27gn@r_gzLHu)3{&;!$ zLIYu{^uLJIj~L_tBiVqBwl!_SnCU zII9p-;1m+SWoH$6V%s!nwHP+fi)Z~0#>`i-2Y(pq5sNw*`w7-rUyy%r~jVPt(7HdE>-#kR+}gI`>xxxkzSOyrd6YoU7l z>D7N}ner%xngBG#oY%l?eDqVkFK0rKo;@_Md(*@Y0yY?+c1mB{7?fJhy$<24xNrhJ8i2Y&U+z7EPT*ba^ zfBG#I;tJu#MB<{kYJoHfdR-m!W$~PnK7%wCLo&SaHnaz_xFj!5N%?1F!OvDzdLhFbJ(Dy{0b|lHj8Lhp(B!?dYI1-R=yU{Z#={rTu#QV7s-|knvO5yJ@S7zw|);w5FW> zo1nl@ni&gLQer{(qX%lF7q+s~P}>UAS6iuVA|i_=Scyb_u=SW#|5Ev;Ud~qjRlJg& zvibSo%h3tTs~+0IkzAt`&uXo?i*{3Mc^o@NpR3WwpXGfs)&>6x;jX(Wp*{Erc5_4_ zoYO=iIHBY|o61x9>-G457=G0{gVgo~M$DBXp^c0fBPy3AYC^|XH)z!aI(Z;2UfPPP z4&joKfb)(pFb%Kw4um9ACQ>iFQNdw{7S(}iRGp|Uy0y_*bni> zWmIod&XuH3^*%V%x5D*cxh3+HC^YyWXR_jwuKx$do1Bx$#KH--@!q z6i&6|X*?YRF9lZHc(7~g`%(Gkes#`)mF%X&JIm3cxFIXP2Cqu_1+$%B_fe1Z=qoN2 ziUl>xD^deS9NcIVnPh3I56S<*^RwX`a>tgeW!qe*{u&TrV-zLa-tpEIDe z33_ORpVq}wcIqwA)2&^($JunT`Q1P6=8l&rQbH6{`gE5T& zNcXN*{e$@%XQ=P%|J<@jB{x6G0{ldE;8?Ws=Igbm-9Af+E*mC;m0=Pcf(;g?MC!Jav=dvxjYMl0J1J2 zy)_lIxZbeE9R(eEIp&j;C^wVIY>IuStv@*PEG7zzJ{fp;qU3cj!Di>hZ8%rR3Zzd| zp=bQLsPAmFS{xBErTv}-OZ>z$rN+3&cm@;G@rc8~O`#m3 z3d++7L;3Fv$xyD`gwAL1rK**i^}cJ9)d9|cylo)>634>T+@_T)0o$6foV$;!BL{>5 zY5jY-PqB+2;g=2D<}KyuRKkon#^)gTeAneJ)fpTDh)oYl87d^YNodxH%H5svWxi#> zVj5{J_+nJeAW_rSy>uf%iM=+am1k_JyvNuHI~_#NdH9(yG`DKL`D&0Y@-bHAs`InB zITB)zT`Sr+f`V%7n*LWt{AHg;uC3S(w>|EP<&hlH;l7 zt1DaE=Q=aLhT&yh4pMUANq?+-4-3zHdm8ZE<(j7@?CEc|KVxB~atP2X&evAr3g1Pr z-5q`Nibr{azQ9~HBMj>2i{^9Zm*+{y`N}5?d6l>q4}VcSp!0fInNM{+oU2H&#b9hS zXy#=R9McjO_Tg~-r0&zFRZ|thKee@QSvkXB85r7aEaDr_eXbWeQO=a?IZTQlR3j=_ zwn%Y9@6r~eHH-a0>Rt;K{cd?*FB8kpqF9N4$pLpBCSdfq=($U668gTxRCusbu@kH6vBzS(GewOal44r!s+3SVWqBoB{42i-+_?B7-Y;5FsCn>G5*^)-OWu2^^;6=L*3f}|w zxPwmth}wtJBL1wv&M`(iD|N}9-J$%OG2t8I^D_A15P+(O1>m=5SvcXC|J_;L*12X6 zdptBqFymyX;NKNNOQ-No@;Q43JK2S|G(uK5SUGICA5-fZGXwr%c*#+QB1sL?Z*d(i~zC7bB!Wd+4VW7fp*)wl@nF*9G zD9k|4EyX5yH65c7#Qr0#RVE5fw-V{ftgp${$@}=SGd8Y{|BOx*&UV|(VEMzp9D_7i!Qx;EIAKjC zLIP=N?w{VrCSx(r!j8S^B^|Ib``OV@&pomx_y^O!y8TMxUFYp>WI{Gcg%-VO3>uclv%R%j=0$g}--cOLI3_(jP z{AFig*}6HR;0o&E^3pX^VLg$fZ;CGJbt~mN0k>7~F`XvtJZ4}9dlkeF`&3ez@oS@| zBoc=ji(}u#Wr~Hqk$=q23b-*YV5`?rW9mrwW9OcC^12a!a82jd>%f<%JKI&i)ofzR z1G%*=`e$0Bb}9;&P|N?Y`!n(rVi^7O#RZ%!r}XI4y^4ppe08fTSW=qS)I*8H=q&x< zX3_n7MyH;9^6H;dNR(SgR<<+Tc=x%91*n53FDpZQC@=V$Ns&QtGnw;jqy1W$|Cb{H z>>4^%n9{?1Lg&py4lFZS&YOC7HH@$5T%FeNN~6aldE44O%rXL|4!KOUMfek?2-SSx z5hfyYB->uwz#A>NIqin#*xTv({wE{Tr~lwBYv;NHFppB7`N`YY{%haMB~ zMd%oR%J1U)7@eV(FWHWGALfGajg2f`m*%4hYMBqAHm-cNm3yeylX_uh=B7R+Sa@rY zNO)%83#ND~^CrSai(8c7Y7m|kx0i*?{+gqMfchw*j>;h-C(o1rLfhs2a2r^Utz!S^ zWXH*hXv<2v&X;Hr_oOC#Y%hZs)@B5#pqLy5zF`eZi}t`_UQs9>WAQ7U+QAG;>b%)c z|D}DxS($O<*g`oSK{}pGPjP z2`!bmR28wx`*d(d+=4jQdRR6o;V;%l)92043@tq{Fqz`O%soAgh-SF!;DhE)>~v<) zu?%a<$E9WqzE?b@qym>iRd<=5QTVUuRG)X{2N~lvJ^WauWiqLn*av0)2GRZBrJSdQ z4p5yMW|%KGZ#c84#W5YEpX}O(GZU5G0S0=Tv?F?7NpU~QfFxkPeZ0>V4vs7_Va}yd z#kJT|1l76&)DR5QMBDFoT-pIBPub7BtJm6y1oSDhjGkPy(BE+uu$=(`lkz4uI~xa# z&zd@ZsihT<$Eab^Xqk(~@Dw;q1I!0!`>aSH>xJ@=O&`++?b&@NW#xfI*lc>~-%s^2 zapxnp7+k*3jpEeLOkrR=!t+ry=t2@=jAT%S>+i}UMV!I?(l6}`)s}n6k2!hrC<}Pl z@@p&n#Yw)XS2*f1Y@57@binC*-W$+#Oc!z+A!cAFV;4lzAZu=n=20Rk%5GZ_-jC>n zPQOtX-Cki*553tAHlh)#(Xu`~T=G5Z;br~eL`9?vKp527o*-wcz)Nq8I}%H?Zbh!GsqiRYLKx1qb|q`R6^qKxc795xAeZuoBF`O-aMU- zD_WC%!!NN|lIjGLohJJ~&{QSF+y5{I9(NG$wa>S9Q#KY@Nf_F+h z6#%_<&R~KQOX7s-1Z_A&6Qzfo=iiZJsWk!F$TwsKc?Jdlj(W+1g+ba+WiA&oM8T5} zpow21Usj|H-yZ$SB*OnKFQbwoK4O#cQ>Tm_2t}tfF@vW9SdO=;zd04K_rxle1Jj4Y zE8V~UvIMmuX!aXFyNF@n#<^~Ox(KjOLfXodfqE!sX%Y$;23=OA1y3!AqMO8|go_FG z?uu<%#HXx4957FlVfI*Wc}qt=gI4p0rQSG8*;ypf@0N<1t*;u9Nf;B7jL*Y1bE~-1 ze%y!?#rm@AkhXp$y9ffJOKB<~I_|cvH2Mjj@_k7>XZkCQiqDuf52R(+U@-6H<-cB{ zA1(wU4p7$Ttn~Rem`KiD^vh39rgugt8;=^t+~&+xT8*>dS>bE5QGUK@EvklS>v|+- zbpJij$;@QhVx|qG?OF;r`ec+=xy@xqXP3FFhr?xzm$7=Cip?GoDh?iSMeyvLl|c%l zeb()V7=*X)9;u8ZYO5HA%MDspFDz+T67f-odG=TtO%?iKExpzuwn<7GqdU*LWOwN* zircya-@oZ?Z_D{{@Po}JZHzsAGKall3WSa%y~sgOl_EwVtL+^PrbZ~dnl}(pCG~O| zL+z)gSIWO=$`AAsD|&txj9aAqH6EGWO8Dafe!Y4fofiNA z`YFk~7N$>dRKCPnI#Odclf+^{n0RomoJ=3kH?^0GA^ns{ZuHlDiO8%%EugPKy-~Sn zu|PSL*Jyr2sg0}7uh7hfklXy%i<jVao}<$`SZAK@TziZl&i1)-)dYdWv`o4r*O#fMf=>%pW0{P7OIh& zZbjn8^6Jcb#iYSTdkm+=u%*Nd7R$}|8!R@AhcX4D1x}Z7WlfE9F6>8ix9fdW>3zU(*@0kMVhg8&*G&Yz9$B{{vtxf%uKtY2SJhenWhHbY_fVm@ z7eue5+fz#q8%VWvsf!NkN?<-#>eyr2+gP9MZ#CC+Ofv^|DT^~;&E}?w6t^WA@q@OC z&i2A|FE>2Zp3v;cV!vrWtFA@#T!#`^OVVE~Dk^oF=gOm;SQ^&8U&h8Ok<+h`>S$}X zYhe=Ks$Vz*VAt}Ozx{43ThkIqRBGsW(05QyFk5Y%k&D@4NHOu`NxQW#q0jQd`Q z022_Q(z3`?dL2LE`Stqu7O3Bpt<`laORhwmE;{5xucPdS;SlA%FAk)Oe4`dc(>odTj07;}O= z$zNOkhhZ4+$eOd6u8KXw9+)mKug*wdssC<*=xjx+M_H8YbLAKKc=gUm#`4`9o4wC= zu6J-*49FU>@n{gzuiEq86v+!C3a8VY2fI=q1rsjR()@}uOvuX{+j;4c!!_Tty2lp( zCL`+!|0%84dM9QFro9ufa3taK+dG9EQF}!NCv4o00*+bfRp2XB_9fcdVyBze_lq&? zoP|9yAee5AVft*n8Q!%gBt?iAOPmOPOT{kJ17?L;Z@c@vAHTf2OyTU1sjMzEBEdy?CGw*PMHY1 zTG@&>$D8?T!(?G z*P8F8vd}R*&v^2V^c97Ew0WFs0i^!L-7&DN-&@i;fI5?faoq`?+~aufL$Z^=)j9L{1sIn<6qU21+h$`D86bqDi&#dp-G?NI!oiC#2u87r5m)BqN!9>j2vyA)r&_SFe^5 zxb}53c2mM^u%Y*54KVHtA^L~c_OJbQGN99oEmCYoj2UhzA}CTpugu2%```5isD+9r z^61)N>nteb@txo3fW!=voM5Nl3zD_9COTHsloSROzhvAMQQwGi)&GN<0T8=(3BsYm-2tkq zv%(3=+2>lS(OdEyzqtS47KcjWKFK0#b{~CVkB&V8jde>%mfGW9yPn3jXWs@n&ben^ zm@o~3bUYN1a0=)7w$Fs~<#f{gX)|wnmC*(vbXe>eIG9pwevtD?D zKiO0^^z%zw9c_)mmSD-?*#}e+`gu6t)cI8Ct^W)7U{O+&+T5Z`I|3JoORWBWRpcgM zj(`3&BG=1tAWJYEr+nChO``Y^LO#Jjn5Djyq3sgE|&$4l{p{m5>o5ynv6 zs!cy*qd;Eb*g5V8`$M5anExH@rJ9H|kdhXmD*K9#3*)}-J-6_vQ`<(hqsMRa|MH?Tm zQ8$s}*?rNJuk$(}LJBEZ!s~d6wzxU;oNE{M+H1ZJ0Wxh^4O+rVv#!^UI}$|#gzM$q zCC4f3n7s}KG7>R<_oB}U*BJ(fUP2Vz5mTdfIH}lBUCl*#s?tkt=f$&4s?Lt4L~V9E z?N4{-o|-VC(2P%r3P|rW2P4{W2Z(mXx}(l0*Houdl|(t~yCWR%Okr{esS7Bi0%VC; zsM^9&OpfjlNgfSl&HdPFQMB?or~cB+{@<6#2}4}VOOb*!)z&Bf?+0E70O}J1-Myb3H*=@bTOmD&uW*ONxOWXgw9ac zl=Xs?z(c2HBeZVAcf23vM)acPX+Gcq^BmBC4*qB(i3_3j=qXNXo)&P|(f=?=o!tB~ zMgpYg5!R~pT_hi%;HHh3maGFRZlRI_aSaVaJyF~bF3QluVQ`cs4}uP^V1WR@iIjsl zhh%7%4i~&@IF6I8+M>ogB9Q87nJ$UNe31AjPRvZDEVA@2{mzB%jsRH&DF0gGVVr)^ z+nPH7xZB~ZwR5UuS#AqQQ0%QRR5I64kyre1 zgbX3ZX~EY0!!vWbOB|3d%=g^A z)Dca6#@>C@COty2d>gedcXToN5{%TS##DH7_eaQ;%kPk4PLTk{v0K~hyfh0kL zHb5d1d3Do9!tsmJ1S=bRho4*&rTolXm7;?7O#C`UC7_7MfRO${*F6B4nIGrvXV^%# zJ5-y%ggZJ(2(D2-ManTkGZ^JzO7<6*9Le3fDLH3}9pk?~>vZjdoQ*fZ8nl{&m zmg3hdFEEKGhuwT4OaP3%5J<>I?`_1~8(41~8*C~%%~?p>xzw@4D}mlAwfc*UJzF-j z7=vN-ORD($Ml&>Bf2{>zxrZRB1`CC>(=&<3w=z+kVM!c7?K5 zMAdHZ^X-z61pkV)^?V_xj;;ni{cr*LhWd5458w(}TXfwku(L;5X8jwet4!HgwqED| zMYDdpkM1Cq2a+Qw7V0r0K!yrmBb?=e6eBlu8*6!~}{2uNyEtzpChS zOZGpC&cdz9_l@Ex-5`yWq|zWEIROy?afEbCX&Bux7$Mytol57VySuv^BqcU-z~KAa z?=RT3>wT~1dGB+d^Ep`6HOvGbo-S5sfT#HJ7fh7VOsgHSyCo(;F{@1*D!(I;cz8W1TCxM4?#_ng0gSUxt&vh@~@`tI_{3y>l|`f=`A zh1>j-+%WlEm9EaDQx}JE6|mo_F8#J3MBcPqN^XtLquuI1==D;Tpct$?XIJSKR6jIvveh-ula<~ zGM7H`T*96R{-J+z3GRua6u8Mj8gF&dY}$&61K=^pluea+URCV+wAIxXBbDdL*1s(- z=@`nI)zjBm7Y|ihjgC!O{QufUTpvTV?sYj3NxR~~PegJyL^t*o$V@c3;=@!Oql|*_ zX6w72ApgILU877S8_(Ec(5nC!Z}0hN$KX!eYfxVqqZjT^fBA*??^=U?vo?40 z`r^`Li7oYb6*5$T!D>I2V##B7G4e+yKdP17Hlx2Nu0Scz3-~!`U<7L!yS+e_!MO?$ zqKg4Fn1Iv#cxf^}{x~hn%79+eXV{67(*7g-Zkc1m(P5g8OSNgRw1l*r;Nzd9eJ}op zQO)LWh%*ETvtgoyKW@NE)c!6VEyT(622PW!is8BjI7wyPz5R$RuJoUM5{IXTyQ@jt zw8*T__Yc)k+~s^AOT`#k!t|iv`omQ^0G|I3!;YCwCazn}e`ZCq(%EKDjMVZQ9;*Cu z3}wFv;ehX)_ei?0O{tlctQ+=yH!X5YZ{+1-s0lLXl`6q(7yZq_lWTj_NiPbB^||SE z-g*cw^1|)mItkrbk6g384H3sn7!>+QR{jqP$ahmk_E zd&)sMBga$i?Qj8qybWVRn~l#NpV-666IK35wFBqo733|d`W=y+xsf@*(EIbyDho?V zla6tEiZ12fO1rB*&G$(Ln{f4W;{IL7WmuoiAEg#wLuYTtwQ~9HK;{6^9%#2nm4fYt z=xm%v^-rSfp)}|#lgkHrLp?O&exdH}mCjqev0UMU^H;ccxg%Ae{$?IM=Ap?`t8H48 zzylDuDPJ4T4uvO7O5-nULcZkrni@QXLAU15(GKM>fYs#E%H+*3GGzOm;UI`QI#m*T zeg4no6^z83oxD%)MJ`zAWf=EI(jzorzYBUiMj8?6D4GVC_sxeK`8NU$LC&FUy$xw_QFzp%kuFL1UaY~W@?q^xDo)*QfOyKaX!|5<3Rba)8|M!**ppGP&LOxu=P zeyV2>kHPl<&`fTz%}MX6uF#vevbrt@MQ_>(Kenf^|G`Uulm;+F(+n!~LoU%m5&w$zM|j3x@*+Un9! zwAT1+cXCCyuVpd{ty}@iH;|s6?G;pJ^6NMF4a6Q#PEJM=ct6plI^QkJmsrx4o&Uen zrr$|<2j8C)>HF};-hY)~XxA~4NeF(> zl^c-5*t8%Gg00o3WEZXUO{d|}y`WQmX8M(sJ@ExKyZo4UKV;k>u_I#w=qvsR?QS%? zhVZ^Nd)+*p%ki2$F>%^u{W!brY<1n_o(a7^!Rkj!ch(CGLa|YA;I~~bjlrNA1Y`2) zL^D&VOKFi!gge+7T4u!X-J^h@q^6etXn|H6(;j;CWwmF8b#oB5aTKrYi8;2g_~Ii` ze1w-%aam9kE~66FF5nL-isEuB?t~QnOPUS7+Oy#NucffQ9NF6^M3ZQ5aP}IG@7Yb3 zPjj)WUM5b>O&0;PU98Ibz_&9wj;5p}UrG2=>-hjPz1>dt(0>CUFd;PO+>~-ylm`Ph zthUbIfy&Nd=GD@2<;Rla_tq;f@#YYGy_{Q}CnZm8_XHOL%qS+f-}&HZHfDc|G>s10 z6t^=+t(y4xx9|78Pgwax1qBJ!1WPY{bQ_VbA3t$Em~Ro~$SA{jAL~WerMg={VI6F} z6{`xqd0^)nJ?Y~bB)pn9My)l+$0S~#PbqI!wnouji!YMye@vJF|$ z&+fcL4!3yn%g|@RR)>mSo~NFOKVF_HOjOgw^KkL!~2Y;lKB1w{Ii;U-O-~Mj87Pnn~*mO|WlRurmoY{i36zous;4-;B(ku}qY> z9@??MNKVn#e0JsvDgres#mVA(;ojJ_A{FwjC0%VBeJl!eJbur8`tymhurF8W-Oz!v z#CjvHgeVe-K36D_Ru#t{21>rM%VJlTba@g4a$}VG()rXjPFAzI zy`#+;zBO&-pv&l}F5V7&D;@9An)iOY5c`tiby0qCnQZ-I0YtwXB8TXx-aU0WQ$D-4 zH`w3TX9o!Jy;_m^(`4jBa0$Y;CC*1q@Tq;fq+arF#b@K~6lDG!c`o~wa{8=z7H0Ns z%6LdDVRC~Feh3~r+4Bu^YgGQ6Dy-;8(374J)nxSg?28j_B}5HZ0w(6yyB7^fSb=aO z%geoXcm12IkiF{q90+MmzaphB94AkHq1ioJ6wj5)?2dmr;^hq(zMtw`ser!93J7fu zA{w#HM}9hQ(qb~lcy)Hip;_tYA;$fO+^RlVvN>iwAmy;)Jbu9lj{n1LG)g$o)x zp50d(tFxjv#RG)QL1G}1+#Vm&psQ@o8Cu{HzuV~7n7nt}ff|>=-4YY7AgiByqBGf6 z9F~<1r_4MVW`~n0Svo7ZsU4!om|ZCV+632=ZiZ7`3^Ey{s&Dx9a9x+2Md|DLaM*Q0 zlSOVI4k85DwgXfbAaCH8Xu`KxQaLl{)5e)-C>`Zrq0zFRm|0sHm$@rEy*d2V$Vm(P ztpgetLtBJ#RjD~Dec`>C`m+(UApiR7*%jo-#f;QC2*`b+Z6SW^-J39p*`Qpx7p!9I zn%p=um`gvS?i|^UnoNP8nmMV&LHYO*&ab<3~IG6%XQWEI5v9t_gVBp9z8i zFAjc^0-?UmK&;Sp6IR=+LuV+N(#JX_+I2kg+9HxYEMYP)9+P4BZDUJD`ulygFYEj) z1ZI(owB5LgpCp*(2jWP^ODDm*YHaW4-OIzxd}VZ1f0*jxCOhF{gg_{sMwwNbwR<%( z>|5`uKT;hqQim-=F*~2&yhn1{IpCX18;ZvlTSnn*Mul$9ZR9e93{ z^QQ%yc{+qPU+=!&U_NgFt#tOQA3o6|?YH$faW<8ZP)8?<0OIl8x`|KMUCaSg zy9nx6b3sX-w<73X@j}Ppx64CE20Bpm_*+o^WOqk+6SBxPC#}lSUYD-Hi96nXx(KhB z#MxXq5)-nOEW9u@{5V5blpOE@D*&Gzoee`T(IZb#?3c)3pA5T^{63#7Cl3R8BJLE)LBgcgzX6`X5n;)@pOl$0Qy!EUpTKvo4-89^B$Epjx9*Qxc$=wZ$fW%=d zR`$E^c%M}4wF)>=I9dm>rkL)@lS$%o)k8FF3O2^wI9gf>#x{k+{vrl_u5yW2ejmMy zGF7X4)}U4p7=*t|JLnH|1sN-aI#JpVcqY|nXIK1(K^&J=n!C%bUA6Kq*ulNrK`;M~N+aza$OMpt$*l+hcO@^#~)#UQ_>mPU7zoE--pqe#!Mj z$cK2i`Gz(l1wy_vH;|reZVS9PqR+Y!(nr0Y5*>p=i&kMZWh(@>JsL zKMd)N9Tc#LkobEQTQ5&Fo9!0G{i5)`+V89SNwdcv0LSwcvzzD0d76I5<-vZzmCY|f z#7atYr~b3W==zHfu05arl8KE(*Az?XJ(qhZ-yK9TG9xB;269$pQy=(|x}Pr1v>^N_ zpr7-;a$s9_DIKPVN6?q~tuwSW?%R=*^*Bb8JE34iTjO7Q2)g-rQ0vKAOTnzbD`>~i zEqi=Piv=W*BBd>#w7D;Ful+{l2HQMN^|1(G2V;0zFk$iY>_@xW&|TxMJg=2z?5!@T zMF+^FM!JYF6Lqr^!9!RcmAp3|>SDA9*tLy_t9}RYimI2&Fg@>C+Ll!by3q3Sx*R|( zEu66z`-=gu!==cRx|d4k{pl*qAel9zy71{3x`!eEr3Xl#R1sLXjcO2 zJrZ*#j_}{TTZ0Q-g}drtbxY6@-V1}4nws-&`5GQ7?ykgUSSm!ZSQUHsxvcTpw{PSo z{~joz-5;N1{QEM#6r-0Yt}CLkg)aPUrIi#A&zJJlq|xzOY*@Q2W=&#il|_FGn(Jj; z4bg8KIeM8W0>EqRv*+1ruN)oZp_~=je6gbHTF3M}&xtn2YlxP?N)&Bez(p~a89+YI zQg5mzP8?76VcDi&dkhz^eZccdhXxaoNXC+)jj3`)&~7I;N3y5kCtk#D6n{Y`sJb<1 zRuNunT#~Y*%*M|EDK@nrIl?f-Wk8p77Q&VKm?X&%$tHe;!V#`3`aZ&WV0d*tH!bZpf{8&1BPfVdYTq!zGNx6DzK#84v`*d16L6 zAdj|yVJn3l)lh*5h--ljkSHSxYgy)syqV>OVC>Ts6-XKRumys_dLPJzBq(qXJFmi#b&ZE%9Oep|)hJbBvMx&$%<;0S9+o}I~uD{m2r=FC)kzV9b zb2e#^Dv{c*xM!a4XwO%uI?5P`vnMENs&9%U8tF9MceW0gt z7G)^qd6X0!D&I=34C%k4)8=3GOjzS)Yut{#J_8XxF3`$VMK zQ?+KcO~7>2irv*naJ1zQ2W}oCI3AJO*Dlc0AW#bSd`@K--U~N+ASr5V+yV5D5`rU` zRF^rE&MKVa-ycZe(&5G#nYK+DAe(pBJ-*u%;*I$RXt%O{dnOZXAy;OAS#5SRioE`9 zvx#|4Y@wBtvF`LgRf-<(qySIyghMqgWcqfvC@|C~EKJr^ zpwfg@ZuI-l;NXA3_cI1$)0es^3eB}&2w=EgNvzE&-*m))2V1S#QJcT}>1G{&DFYL3 z2TK<3hh|M-^hyvU+W$1_&oB#lvu(J}^Bx@_Av9LY8^yf*gHij!TN;t;XgmK{cr7y9 zL1#-*`x;(I&$rWor z*4QDicEZNhbwpK$Xqvir80wib_?B zmgMvaburRu=>1)}PON2?${IHj%hBN7ez~D?%K-cE=tsK8;e)%39DFKPx-Etz_4#R4 z=MMJvyOTOMHuuw+U)@jSm*U%9%rQ{DT(~+c9G#(V?sNpvLh#{6K5n&MNjvT*Xrp`C zaU~JGk7A}rLL)OD#c%*M#DOP-jsvY<0&9$-CaH=_*_b)IY2fUr!7JFF=Kuhw3Pt|# z9by4we3z;Y-|WlwR1D-dRsYjVwHC|q*NjvAGt=mVH;hZvA%#wK_VUor6Ytw@npv@& z4QFz$dG_i$=-1NH)5!he$tHs8i5S@`m)e&j7P>Gn*xT|YjN6vF@pkdcZ`Qys4isjl z@94CnYcP&~=`EgdpB#YgC#lJ0zxR)pw@riSrR z#&Gtxz6fKsc7EJ{sU$6KaN}J+g>1DcJpfU5I^wMywQ<0?lq9S2PWP1j?N_w15Dz7( z`|)UVnN!C=a~#1_Ka{3Je!+c(j4C|X(QsQ{*OY4%7lfk$uwp^B8u{QTA@AJX! z?hJFX5i}Y5HdA-i7^>QQJ7T6t(3-aE+{Liv221Xs;Lm?hMERxm`}D;;rsfMa#doqa zdXJ#biWJ5YZV$l?c@xrYQLF zfaV1W(?hGcddJBxJwtCiW_pm@vN#qZ(iv;NX&oa-Yu7`6I{Po z@rt4-Hudcio8OdKbT9ejH$oh)KRCtKF%Ev7IFT8PeSp)~e9{kUbjf@y8hwkjk1I)2 z*c#1o-*79P01deVmPo5@FJ85S75A5klR|NUCwcAzPzsYt>kQ98X|elY6%U1%A})Wr zlUGyI{=;bVVhUzTzSytdGB(`p?7M3N@06swt?EOkPII%jeO>@sHyP3S1J$ODgMLt* zct{)C*iFhW%Lh?t27nhFsflG5)D?cyY_>9DtVlxZLC>C6Ul^~Ta1c!SHY_Wt9&nRa z%sygS7*LsfI8*&S2z{tYA>89o92tOKVxhSh$!E@P5z};`RQ|1UT*o|@`8T=Ms^`G~ zGCKzh^QVE9_%{c{FL_J+f5p=;sB6D&8tNWRvB?*>QAJo!by}U1=yk@OT=m|_ptr6Q zq`c^AS7;#>90Darfh_E)0vEXSPIoW16s1|8I{~t(?i$!w`@X{7T=E%0<9=O>)Jh5# z(=Hz-JfOXb*nLQwty9chvD{tg^JN9ojHe8K7+-#{ZRkflBeQ7Xgf7O~3Ggmp`F?|$ z66H88Qcb*6J?p6WczRle`*SKA5o(Hu(Zt}+CVG1?NNCJ+Q~mhUp8;)q*eQJMFDeGx za2qS*Ox=oYM7~FY)I|;q`z1sxf=)$q#;vP$uJJmQr44i!#ylkvzdQq{;=RYDVip<^xM66I1M_mKzjcpd3kA3Fi z9Li?u$|foGjktfPOCJ zq?B?a8e2BdkSNs}rL)SwJ?1=W5oLy9m8WscGcMY>T6^EA-v8a*Qug z!tiYs0g^s#_>E#+yIp!ak;*SZeL(W$V$^x~Z*aWM#!IgZ1`$)cGl#)A;6igQ1>}8w zL!#j(8A?SjZ#9_CYbzSvS>)sX4-{_zJ7q(x>U*L`;>eX%$A@kK6-*>QPv$(tcn3|c zKFzkhIY~-%a9F>9Zhe^1XQww>fX3|`j&Y+WIkxK=0kr;1&;~3U{_O?qzt?^$X3V5Z znb|h~`M$tByFHLC>7N0V>4{?vEGy6eAB+rEC|R)Biti&7h=vAkbs3iw{kjY zLm7qt?Z|{ANh55<4Lpi^)XGvOZDXS`7!p_{h7bM9Ptd9UUR6(c@3z1oG5?C2J+KE; zb>4x%p4iofjobYG?t|agH?Z^xr!v~~9)S<}clGxt17-Gqm#WOEX*$MEm-2VO*3a3Fw&F9nxOmG3xG1A=xErV4S3G)Gz%7*wv`uSi!}y-PFbFyV6P zKDI%)VAI25wB%>}XEP35Oc|g*VA99EGvfSrTDU%(c1!3T-3>Enp$z=GNON3BZ!RIH z;E_k_K}{pFuSpx=Wgd9VRSHw&oE|vwW&YuKF0EM9o_(6OG=%Z}@`dwMkT(P7N^@s8 zIWkufo>jd*7R1>scjHaEN%Eq6%pdSDXuokd~L7H@$|D?Xm@3N-B7M324$@cGvp zwd%0tM+?W*#Sp^0)UHHvcqINkTM1x?=f{Hd}$5{Q_#9HyIQ;p0gj#=_l zfA$H2EVI%R=U}bMHD%X5Ao=P(25qdz!~N~O z?aQr)ON}rOyrCUG$pp9D|INlW;bzR5erl5`vy~-8e4t-QaF00U)X@~|PMx}8333H} zM$5AMr4P!;Muqvh7&(zubdp>mZ{v>Kpe(E7l9Apo8#`uiu&p+KS z(crH!3nhh`>QTm%hXPuU<+2hGFSuAH3`GK8@3Fr1u_NU}xhT;`WQbl;{qVijoB0xTIsVyMPe+d%WHZTS!)9&^cGkMFN){vTfi{@Nfr~3YGG%WPgV1l{^LQ=2_M7;F#{r(r~Xh~bD{J3 z;F*3HQ$T}%{RslkG|kA^Ac)Nf!>}4%(H4Lt-$u4cY-5(E1vIsN#rb&bl$Vry8&UZP z?Gv>3z>Tz+;mbMOPR6H=a87hRGLhQNwgMpvxBtXy zIg(tq$gTMSRR$5aeSZGRQA#!hYsyp7`yc5J^GI-Kn!Wk`UmfF>RgZlcODv79zP~4! zwo_Hh{DZ=ua?~$qosUY{TQ6WBl{?Y|bdiQN4F6nMofv#B(L8yervoDH5JW~mGX1{n zP&x?bC!k-gs?PV1aVMu0FG0w#oos`x0*e$pM|#1*hEHW3zn9?9)%~XQGNr}#Ph^x|4KYFo(E1TRBKn)B<8Fpib97t&~^jaaN(6?SZ zrKfI<@-{Q;S{c9igpY;uQR0e`=Lp@FN1b}_4c252?{8-h4<*7IVq=W)M|)|yY5v2o zh`~ks|LEJ45W52z*!558+cB@)V)$q7=PiParNLrBX2}IWoVB&{*#n8;3#@eNz0QL)*0)QA1Zw5A66}TMwu7F#KoY~X zW&J~Cslcz9W-dBLI>%PuIEig)Xj@ z!kDAJ*T73>Mwd?NdDiWO(sjk?uw@54`5>jMIXP3t1vs9&kfb|jvbw#OJ*U8Bo0MAx z&PIKVLc_}DJ=i}PmSE;w*VFT0>%3EMHaj)?tcz9ovkWD~D4RSF^e~+@cM?|Fa-CgB zBG(#kXfh1FRH+*??_!kOjVo5TzJ*1gdCYN4#E4rI;$r>~CcL#`D2Ekw+%%_%*E1Rr>&_C|ylwPjgCJZeIX!P|z8J^Fxzus=x>dsUP(#tZsmZvx=?x`+51!Xd26nJQK-QhxwqT|AQ%0$dxQ8^}DSNnzZEZ$ExX9m8 zgad^1J5Wo|as%jx&Mkmkmct|lI7mOmd2yi{EKw6vou>Jb%Pj`wLPCmHY%SFuJMp*w z;ul+k?LxU0^~};D{3+%a5~Lg^2j57tDPA%x(c`W7rk5Y>%N_9ceD1dRSa*bNlq4Hl zQyiVR@bSqQoxA=UqpC7kDQeFDqh+JP4$Vz)=X{%;_n;ezvP4Fdh{A3>(g-JA`pv$| z;8vQV?2xyf&b!aHjX4j?41;Qg1KuAEE^;x4=*)=n7%rt8%XL@+(^nI5B%#-(Ye~s) zXzxw^0!e3CW4g;^;$V(nd#mn@EXFL_XzL~j92pB}WiA4bh~yIwloMh5fB1G9i##<+ zLW|F20-FyhpU>U7e z=%IKU;<_X|2aF8_Hjj1fi;wog)*GXFRC?Ff$_^f?bIjYn^h;FnWQZ>=^g}OBu%9D~t>9<7y-$)YEy9KhJEzhnifrcIge+!AA_@?<{|umE zA!dhk#pKjY`FGo_8+afRpJiZC$|o4Pm_3TeK{Kl`leLf? zc_+#TvKVVPTT{RyI6E0EUWbiOuA(1g0D$sIz-xBLBf=|XLHF&K6!PWo(sOb(!q1~? z#op#zU-!cXjdi~uDG)rnIr&c2O5kms>AKP=r(4fxvJjb(BISO6sF$Z^oL0)QjmBs4 zqM^pC6Ik=Qt^p~Z%S-B*6V*bgfPNa3>sYb#_~eq-1P0PhG_ST9bXzB z>qNQRf3I;Tq-gyxG8(59i{B5t?~mI|0fcwJsOIv~Yw;eWg6H;RSO?#Xv^_YqrS-X# z{zn3-RY4-DAfcgq-6F629j?e?q(=GEIFs1N=9GyGVucUQ@dqaZqDv#iEiII6h!SZ` z@0|x2jmwSFsU8aTRjeyP>oVrY=91j|8&XQoYYt5STYvB!|s?e#Q7V_VOv11QuZ z`>UwgA{!mu-+5)MTeX3!(o1uSm}7J=SXpg(YLGGNsCca`?+ZGEE`;r)Eo7F#pw{%i zMuSd62?y)|I@)6&X4|nDPJ14?colv6t)y3$EM^T*{*Io8HYfXQDCOq*=7- zHCqnCVRo1G@WC6%^b^OC*4~H!Rk}EHHk$)9J93ayhBv7p!1qO52`q zC(KAo<%6|MsXqGv;F|J!S3VX?18!P|}+ zBqQBgS>dm#bNuGktO))^oB*9F+Dc+(s_a1bkEnLwqvRLd_3hmb+dOTGk1szSxAKW% zCl1*6H?2M;5?aNVePX!;)pf$1(7w5re>WqK$^6pbEk+Wl+;9+K$N((w&U(OJWtglX zznF8LV?{fHBf|NqW&Qns7h;lq*ta{wFzC zFlR&coC7a^LwUca4IBRRngBV%pe{kI=!1O@?=HT%U!gLxJVPA13iK+>9;(k3pd9Cu zFwyaUF1GYO6*}VB(lzf_@Xdmw8GINH{==x^)+s4{s7&nNg3>{>5Oq%dRW=ui(TYPY z+`rV#tcAHz2h-I_bJ2?wJB( z>#w=HO!ca0lxz@Zi*Kd8zQRaoFA?z*BQGphD51M;pw^eWqkPtd^7WXMy>~j) zSt7%m`=p^NC4_}J<#Uj*c&}6X`-`snG>os_y(XJBg5#C);oCuz?3Tq%0Q^gCSiB+9 zP;L<=SEf{CvsdW(X_r$?-jlb|z17Gy;Z<(GG2fnH1Cz{i3JJn&j9wJDa9*6`if<8?DrlKti22=|Af<7308ws-s|$ZxZb8V z9!`eJ3g|&T!A&6+XlT=_pXs>3@0QDaQ+kOP9~|FIH-FTWuMwkDX8edB66=*RbLcsV z0UzNv2F|G|^`|f~&zc3z2Q<`0E72oTO`U`#&2#>*naLnH3wPU1HhZ3EBkwkJSa){>kKDEEzQD>cGN0xX}i^$$o6}r}Cfh*!}QeuA4Cy9~L{#@imsp zF2si6aV_2nM{oDtpeRg8OvYDlu8j_U`zc8u!Er@hEguq*5YR=1fSoF!KUSVCLJYkH zg8lJU{m(gGs^?hUk^8jkUqvhL>()S5)Hv7tlkj0 z62q45=_);M?yd`hRdxm@IdI{f~O_ zzx==72gnAsjqVf=OyWb7W^j8uDN*{Zv)hSBCi)`wdMjf{ENqTNFj3Y|F5Kt8D)DUA z?m+|e2R*)(Vrt?889B*(30SI*CEhNIv0M2;cqnf4A^{KzC3Ks_Y(lRX(LgiX3njmo z`c-p3u?Sz5oU}@)3koh{W*}97OR&DDw+n7V5GsWHWM78pQ$CjXK4OKZ)zH#i7xG+t zNWRa^hTTo{9*}1v>7QHFU3An??g+La=yiMnXN#d^eSM>i;ZG&`+qlA2QoEHi-Q9-u zqJX6R6Ycodg~|p9Aan`CV`yozXg6{hAy&LCKq?(29?7ezoQgx;q~3LZU2$rAJaE~y z?uj=(9AV};0&COpQGn+Y` zWBMp`r>j(~>?@njkYa0|zQZcA9y4PWn6tKXkI_ z&?$9wDliD8)6LLAnL|dxC8QIgQ<%CR-uVya;-Y1WbxP|^;wujD(JR&m-L}WQvAsXE zIHn%2y6)s8FrFJeZztT=?X~CHw)(NC82L5vY3F0EX11ZnUGBaE5CqJl%IDqYPYX<3 z1_V@`WHu`=MfVLIld#@Bv25`a8xW4t z&o@B6?T&OUG5(Nx%l-8+k+e;}`8i3I2kh^od4@ z4MQOXq1|YLfb4wg*3sX@#}j@A^G>3TY$t4E*vs&?{h01xkzB55!k*hA$21YJd zi1gT0NOUR{c*}4V<79Zb4W)+_*}7VbYivl%I;Om(bHJ0`nBPL568-iD#?NmJJh>B8 zsS2#Pm}M^ji0#R3Wsq~Br`b>R+FW&WrSoREdWy=}O$KK|wOw;xHr{{@`x~iM!se*S zogF$t4Vxb2LBlb6hQBp7YEZtR?fG4{t*!K}41-w(iALe)W25s;=nsgbkF$_<$39)S zsz^V+vt>eKVmiaF@6cbj(KtDfi47AeGW1)FLmmMxmyD8V#@T4nq5ZK-Se@C?Ff6=t zt1w4Q5*lg-|EmRA)OztB##sqq7i6o3eO;Ipe@wk+{59)F^sXFzD3T(G&%-+e)@(G& zA2SBVbr=pv#uNMegX#m5mhK3yQ|y(Ol0tZ)^mn&6BiZ);&%9<_@?RVQ9V9Sb+>)0t zBayH?E~3=8cL6vr10@bIU_8j`Uq?AC@PX0-aLV+!!)&S7zESB)_kdE-9`3)7{KfO| zq>{y+%!h=K=KWOHOWA+D^ixiVC6(?0If6BpkEETHx@$_k8^rDXvo;x~xER0do49Cq zB2nj$-_YonR4eFwq3ii3lBaxK4PNi(df|WNBUU-M=){Q+4`i>&3-*f25M$cHhm2s>)&f* zNd6grz{c{zc^tSSw*{ua{x?XJI4P)1EA|dEpeq)G4wm)hjedR+94udh4DNPS6J3b$ z=dY3vB=UR_-WQF_E?~zH8Oi4`U&VBGaZF_qqPep=!7$_-B$=S^1B!T}2PZPh&UH1a zfBJ&l?e#m$mQ}|+FckCA4He}xQsgYOa+&=soM(>_nBB+pb^t)Yp3gWd=T-mfxD#!| zlIml^TO3aw`Eh9zTFbnw{@ zc01z^>#;W-)XeQ$NQv+JR2#8J7H|5>oXwgRizw@Iq3!Z!1@V~N5mJCu0yZmVpGrTZ zA+lJj@m@~34_4t8`Yzka)O~0E=VHbGY3)v7&@*Ue&BiH(|8Ly{Dbe83AXR^F(FT}4 z%fLR(jCV4mYgl$mR}FTaL(`Hj?w#2nh!Kz<0PVdj?O^m$&~HL8(6227y`yjynQ559 z5yWG3P2@RYt(&@hDIyKnQfzG^EFn-F_R3oFw&!_lz|qBk~P19;YU<>O9wtH`tFg4(JuOKu|RrKQjowpX}APEECKVmUSX-UN$ghHHXWF?ZX;(dSy9K#W21Kzx+}BYhu7WrY$Jq`I#R zqnmAwf#FhpQF5y8&ZPM2;o&Pfrc(nt&4*VVbV2Fb_v3p~-??wm*XlV*SCp9ro{b;L%bs~F2yKuR@IUnR$vt~n30 zY;Eb1ZUI+#23M%kXHMEYs|*y_y<+S|ZFZ!5?7zW6H6hONyp9y=d@r&rh+M1SBhFK|xLJ=p_de)>)beTh zCbtoAqP@)=DVb9evr}#?<2K2#+DHAy{u-=aa#{9VrC)qlvU6Q(h}rok^lb(#nD^w2 zP486OU<;!#c8ipjtqGHM@H2UAvPsXu&UvqzrVM!dAo5y`+RCDTBC2BC=9N&W1;?)Y zuumv_SEmhYu0i=Kxio$g8MhZFg9|og8_2H~QZ(M{y!7b}+Q;R6M1*5vl zrLEUrL$!s07$;cMkPb~86uhOezO`1vMJbD!L!+6)Uw3FwV!-)O(I+k(JuT+}8XdzS zgypC=$hZeZWO+Xm1A@3Sx#BQ~x8rH1u~~zAB(7oWEqmm)vAKLzMLre@&k)j!0L6Y+nZr!Y6o@&d-_4^C|1TEb{UEDB|ltPy_u~ITVvKC z!G$s|E&G*dXNgTEC9_SeOeH5?vY>YOJ8oYW9IuMBX@_5lFE6U&S4V?hzLR*z;5_B) zqg`Nm_O-gHH59LMC!sRgQ@Di$&s36BZc=vsng&Jj@(F+Y|MY7C#6EK<6WLr$ScqFU z@n)%8L<+r=y&ef{9IY;^EGCokS6 zhjlv0`m$mArj&0j;``4Rz;t?5`oDqA@;RU0J-$P~l~vGgZQ1AOC6@&16>)4Qg9#Th zq_U4=N}8~EOvoUbi?LEXP`=9?vmusg zSgk9JPW*_HWK!x-GkYs7v#RS8{9@xBnKQVjY^-|hyjIPM*M7A@B zi5k1ioCBMt-!(PP)0W93t`%KOeO|*(5&hmg?=;dEH=#!Vuj`2f{?tfP{A-2w%cN=jYFQaI?lRh&!Xt_bXda89&%yB_?ZV*M=vAJ` zY0Z7BZD2s*bqQ+foGECOoHe9-7i;tTnPl|e;I6-auVKtHvNWg{sXMa8J8={!HTYt9 z{iD1P*d9v1O(I=r+9kcLe`f+4Wr2@UMiXvMalpwr72@72{j@wM zqHEVz9urGT?MipI^5>Q2U$cncc2x#RIRNvwBz5Axeer*Se`+s^-X1>`ykwH-S{>{U zc@^fjez3FKpP0%Jkt6OIBWi`gUUr?;@>lH*`*D8Czq8H1_GgE_Jb!8HO$$qDbEjE{ zi_DHr=oqq>0EUi61z8ww!zrz7lGj3ZKCt+!;%#H%zliKSGo*d9Q`I#sLw&#z7LsXL zg;feOl1U*z{{R~IFWEcbC&TXwe$Y0y9yq^CjdI6Ckbh*@$^>R+b@L;VKn~!D32nFq zvNFT3)_ zm>i#kzwlqZYvIqtM);+mcu!E&ukEewAo6Y|QRl$wLlRp!c5)8vWkN4f+&5YYSpu_q zpVD`S>^>erH`!zGmYr=0Ohq)ds^GRW_n02s^G)&B!{3IUG|}eKJWJrM2L8d&myYjG zg_9WM=Wa*T5Nq+b_FMk|gPy(pF#=g>M{|wpt9AW?M+E7^!)pEK3Y0$`q0m zvmBlPsqy#rit(1S;r1RGy_-$3m57O8IFd!r-ecFEz;qv8Gkj;Q>8TyRhb<;H`WBH0 zM2(*&Rvk+d_lq2}6UijkDts65Q{z6R68Jm9H$zT<;yam_$$@4(cbaz^%Ej z;9uEG$A1%iIcws*SH?P3te0^KRx5!t(gk6a-c(>55OI@?`q#PqANX0U{4DrCsCYA1 z)a|a^YBJozEw1399aDlF0uEH1aycB%BJC%576wck-dgT#^RvgQxPX2l4D^N=FzZ1AQ5;|>BifMeu3~GN zxs$|xO~#*ZDGUT=YebU=xdf>>q}H#Bb<3-m65>cBWdQM>yi?T*<;(v77HGE?(?xfsIYd<(;-3d6 zt^*F$;vcjJ!#y+OXM!)hA)xECY1bE$+xZt4_Fimpv~vgl0G4hK%%O&H#_&#YitW5l z;#+I*x_p*~6>JA+9|VEV7(ddhM}4H(eVXPJoFZLEyvn!)ZW+hnnsL4L3)$Tqf9yB= zWBd^KH{r|Q9BU`UCM&yJTPr{8uMFK@BvItEIg%t~P`O1Rm^U2_edplsiCTA!{5@;p z&k5_RsA>189tnZU$Oz*Z0IBRj0D=f0bg!Jh;F)?i!>xvDUa$c2jJ^=xnlt$u6#dYOD#t+us$X0n#w@;dd-NF?K4kR0PIu~ z&>w%&ytCpCo|=Z2uj#T*VFucAvYtG)Ro)wqanH7E*nS|wA6&Y-Ot0$v0;3{VY!NUrX+9?qAK5^HMM0K(UO+UG;_PDs~#Vs+iBBjaLGT5A%iM*BY;la zoE(EiTaUor57DjS)BG*q^?NwtQ5!tW*&tw?`W$ERqQ4rwK8c@0?)iQ{cerfCp>?n*3hz2AMV2i4Cllk+g>Bq|&?agC~wS!R~&Q z?&olq)qSiUv44Zp{EjNys}-ZNW`@-!!DP2jFypD(Gy2yf@fYEIo;%iTZmso6;*QX| zG%?GxO3lb8o`ag~wY%lCy^`2QR3QM8dI4C8son!6&Ypt}xEUB^t~-JKYp*rLdknTx zgVxLQMkzC#PlK#1lGjzbmRKyN`AkiPF5G}|&HxATrSWILoi|g_ZLG9?eQmDpq}vhHM{DNc`&UuZXm3qiJaH!f`SzKWR8* zbyCC+W<5Js!b2U3Ra3kD29sN!PX7P~v`Yx3xf4nD)e7vWSjJo^T(B4s{uCMMOl1MEr09GI~GUTcQmr>gU zfDJm|P_(%*-7M3}eErD)8&#vu3F+AID|qHOl<#ul`BukM`!IYj(taK5J|%1K8r^D= z-)K_947WJgBZftjAPj(Uo-l7_;;z#q&S9qh*}V%D#~O#S%~U#q>d}+4J%GvPV7x`?e=)cc45Z$=ogWb?nZD4 zue9OHlf=+=MCFLiHdvPtx;Q8FHR6W8iNJd@A^5 zXYud$nXSIoD7NpQ%akNYHZpYkY;niQK5-AySw+?@5#IjQYT zg|ed@^~Nf?K_U#Y5zaXn#ZLonV5P7>yayGb2EB#Mk&a&=FfD>c3Bb-rIq6%PKZx4G zWR@>64$Yv0*!Ac7Xq6M@xz&0lYb5NcM;u|QE(vYTHk)MEhj00Yl|$B!X8qapJnZb#iD z)Uj&P1vrs{j=b;()~euoF0raXbQjNtJOTpYuv9AJ1oiFsR&+MH7O5<&Z*lf@FO^1J z*yk7-J-9!K#dB+{rPvQCX2}7w&N%D%a(_CrZLHYXmzD_gD}jc|BRKqr*Z%;mPV5c4 z+dX0pmbUi`EJZQ3oD!sv8y$fl4w&hV)tPS^+E2Q8)WmoxDf^`V00{Qv9Gqv9_0ZBj zC0w(uoz<*&G0%vZ$Q#rSha`KRYh7(L&1&5q86%ERz=Z3T1m&^NXCVIoz;JqCmrYm? z*f01s5wjV8;gS)lfyDws^Qhj zpSyqHr{C~Ue+qxWK7VBGOXAksYu^=XMYN9q=yyoX4c?+--er|kkMwwB{o{bj8{||f z&T;(z0RG4S0Jj(I8~X!zu1$O4uZZ63L!FCi9vIXvCHoD^76sW*@QyYQnL!MAVA$h- z{tC1Eed=HEQ6G#tSB~u3>dV9WFNW=O?Gr}2m5i3I$R)d#ha)oF6Z3UsiXwTb1mkTV;F`;clo0EM$kYQRp1Ys87CdhX&1=6h<@?2 z;Y}CDemm6cv<*HP9$V=ILfYRElqWb~NdugB`@+7<{{VtZ>0b~21blMvKgTZ|X^56K zo-44_bs29EJ1x4)gm(-%*cqnAI3at0Yvi8`_@7+U^%vGIEFnMH_P^R9E)>R9en4~B z5*Uw8JBs?#;r^R!JUs*^M%olFJ@5b=XWON8QjA==kfk{#WBLr&JXPU{{8eM(%{yGO zxrHuQ?Nh~bkdd-HGE0!^pagK+*p*cbz!qh%JsOvYJ~;S>=-ca-+DC@%Z(0F;Ji%J& zmA0@4&J^16T&U{m(Mg5)OZ7${7;XFeFuITNA ze6IMOK1*Rj`FUV7UzWcdzu>Nawr9m=veESKiuc|h(X^W>EVRulMw?=#yB=0YBQhKS zGC({7_p--2adfIlHKW)3%_TJ97u5csd~fk1PWWlzD{q8fv!$!+J|@>>bbE_dSCInz zuI$V}OlN|`D9+=@74z@MJrlq`vFC?hn{vy;wFezgP19o_7*&RvT~fZMqzBa@uxjQVqrrCGj`-g|NOcvW^TI42qZ z06f+%a(juC1d@Z8+utIt=(?4}7x6p}41tItR1$OiIPacnuA6w;cBKvcQHE$GR{(%P z!vlgFf(|jwe$D>Pf3t7wZ~J=w%>EzOq3~Um-TZJuW^4FN7uJl8^FV|Q`Fn$HP|T-1 z5I7l5s-~{T7`eM6@~h!jz+a2M9Q;o_8*@A0PtITj|qGw{jR(r@xS&@x3tnV zj|=&C7nWETrWcL4X2*(>ZRUl`KM>?245)4bSLv!N0Q~gES{iR`OKT zu6T5o}9)qqc@F(IA{2m$ckHfzXG@Td0U$L)+ z$BElZPwfqFTk#FV)-P}rD>OEcqA)4;D{4Nve}dJzuADAQP=%stPiIZ+bW38$Mn(rVE+KYOn+cc*bBq=H~#>% zkL?ek*lKq0G>~{tOuJj%X7PaeQY%P_e6ipc+JlnJ03?1oe0~1_gS-57;vb09&+sqy zZ?e*~Qqn`d{`Fe=<^le!y8y@>ba?3R5{UYjVOGzR3 zVoM*#rDiUm!a$&@0Ouze{OcDzZc=@Z+TVa*@O0y8_c1~HL3|6H?IgP9>p;CyAdl3e zozgcvoxpYjj(u0a zs>P2u2EUYhtBLMFJKNI(p{qI_#rT*>X(5P2z_h4#l_2CC_BrO9Yf1GLtbb&>Zm+4$ zdmW|9JZxNLWn2()E4$K>t?i|NxyDIedJ6nL{{Vw|{{Y~te-!*v@m9n38U3LAX?Nlq zJzCaL;5~0unWek3xq$}&>Nk&iuZEO$QT^|1c?E*-aP3*h@oSfE?mCy{k zj#<3*fk2VtP`DjKWLGWYSjDxX*=gtGwj1ZU##cBYSKRI(vJP;`G@@r{a*t^vbcg11XID73-EU zq#9w3M=Ur1a%;){8tImwYBw;KLh|IJu{h^FdYblaKGH|CSzQ6be+l-WT@Nhrq*2&8 zn`qo*C5=O>J-(yi7_S!%x;?}J&>FY#H3GHT;|#G7x$m6UJK>why6}?PZ8ARHhz5RR zpXE&m&(gGmcwX_8g+cO?*vajmYId=r-pgkc6YVAC7}zicY}i@@8re?<;HM-GIjv>W z4b9v!?Hf@`oc{m_ALP++Tb=>?7Wki8(Ek8vKZw5yE;T2HI6QB6E}jE%AKGDp^)NjN zmBAkM^?$_gixB7*iKg977f>-Mo;c6kC2~)&J%1YTkB7gr)rY~)+7rg#wI`1?WtI;S zOQ+oUN5c_hxVVDNAd>duJSq~2<;UEsJ8`v6M~EfyH;VPk&k<_3a;x54G_J=vU`}|? zIPLVPj)>Bwo6z*%_)LD^auH_SNF3(?{(oA}@x7cH#hbi%X4*%}dSqkyX1w`sJVSKt z2AYoDx9!Gw$JV+J4cqJXQSH=>j-+9?4tc<;Pg|Mld=c?3%vpGwP_Z$P682OJ-vNjf z;$A9(Is8?3VCqUuAVQ=PNM#Fx2VT7^>2KQy!)@Tdi(0j?i_1x#JBZiRO1aPG4Sdh4 zO*O}f;ksL)3q0sV<#!0q>;@yaUJg6s6}5Rz1*bK+-)wVCYn%NU}DRuZgpScW+u1|U0S128NQ{m{TsUxj^U zJ&&t+I(Ogt`_HAm6n@Z`pR_;hrKWiA;tES^b*Nk2L3AyT?vPu%MYNu|Apt(-zdF2a z;(H5!6YA@y#;p~_{DJ~smr%q71au_x_*d#Li*)Yr^w#Uzhw9Dga=j8#~Z&RBv*Jw5B|^LW&j zAKBerR=>+%#hy(`MoAul;-uEEH0ynK)*=#kw}5#im*pV)@zdV3%=76!8<$kMF{H1? z4&`J7{KOvP1ZKS(z@8bo*MDM<4SZ15B4K@X<6UaP*xOfi{3Bs z1*eOfLa?)v<~vwW2zKQ{u+Q@c@uM3;txUEFzSsM|;fz%5k3pAMdsrg=%eaz8M=Gr6 z5lEp%NC)Q2WAOsAW#j89(k)|hv=-?s zgrQ*=Zh<=%+m_EjD#ekyxYe}l-3L&FM58ts%V$ac%(H=I!0LC(|Q}!}x-hR+^=}H=ZJsFwB^>f0)~>YlR?= znIN2d{VO=pjH)GleF}1M^eq0=8pW(BW1-r`Dm~2Jb;1w`a0V0(a-egNE6Ojm87!>U z-bqq6Jjl(@&4M`t(>UAo&sxy&-myNZ;=5;_;iX8U07%51r*S^G=hm=q-EVh7$m*ux z?^PKCBP5g8u^omw^smw~oIlkv`f&2Q`RYyiq}kH^HSm_X@iWDmZ-smtrp+-xBmbICAHJP;FUB!)>;i~#`9yu_t0$* zaG(ip0U>@>VpcF0Y1+O1V_%%#@K28v_=ijVtGp-w00>NW%czY?B(}P`fm$mSb_(}d zV}Y>?V2n2)43@$AZ~hK1`0hM!@%zKS26W`SYfl6CaU_=F=#^RLoM5uX0A3Y9XE`ib zsK8PWe71`2E7`Z${Ic%x+VOJ+PB(Kwx?N_`^(hJrb zxa0$YlaGG?0H^C&+I`WsoH8GBF_H8mztXoxO&j;RG_gh%U_W#>eBp*e6OKUyWAm(e z?PQGcAIb>;fzr+KHU%dvwa4x5QN zIq#fx>4E7*?84p7c35PDZe$0Hk=Gn?(yOx&L3YXY2fb`v*t$sKNYuI#3dlg)gV=S* z#b@1FTyG#QFrc$zY3Ka(suL=+3}Zj2;*|+fIrhh1Dbs~QW86tzxgGuKrz1OZK=sCH zp%Nh=WSnuwIjay)=59j6pa(7d^Hff8kbSf7OCDI187DrU)}6rUygnBC`Wss-2A)3hr;bzEi!50TiKS-H5~Yr+)oy z+8d8BH_RAr3`1vj*TLF;Q_mT}u8+k30PsgI*_TqkgT(sxhpnZ%k8nO!u!rW}9T*t) zk%r)ogl8i)_srTEk8r>WcnOX%T(6Db()4{gZ9`FzExc%2LfnCYo_{Lgc)1;m4HNV8 z<2U>h_x4`+Wv5xkb@2B_k58KLUdJ2|q+%}I5JAS{g&m0e-ui# z&Z8~8nV6C5x!Hm2PZilhGLq(JC03&=Bhb7lp=vsv?2y57ix#Ps)#P>Mt-~v{VMxd= zv@dK7{VVDZ2l(33%EDVI;%8wCINQiv4D*4;F<%*I{{RuDx0*Subwj>2CUWB;nSnfx z=e9xqAADDB;m_IwTGAEQS&l`HYk3FVAG`!(Z)^^yjCHQaIYLGaUvq)|qJB8|&*Jxr z=DF8z9y>b+du=|(Ha=8QhRHboCp}q@T=9SauQGRC-e+;^THY?ciS-GsZP9$cFuJfD zp1k{TYYfSf5DbrBT1~d@F~Vk6-LXjb99FRLMdkcf+HK#IkXuAzg(HXzDPr87t@9Fj z^&_Vhj@%Eu)AXi>A-8ARkQ&G!Hwq$lZgO%s$Mo%58ikI?ltP=73=&U6>7M<$tSei? z1e@2B+Pxpc8hxLLJR;W(F6UOsj1Wdfc){oL?^Ce)9(w9`5Ldl@IsX6z(fz#qY5O7k zU$xTp4-?*h!X2#KEIOTxlW&62*x+2MH{2sQVx@AucVG(n-^4m_v$=TiPBDfceweJJ zW#km?j2}uIV>sD@ZY=(hw7qBk5^W{CXQg*w@4#@KE21 zUjnq>g?=T`HEnB5pI4UJ7_{vzV@VoG*@Uks0f7fCxtWgD`F5)_aq&m&v;P1EG5Fd0 zI`}^C;Z~ujNv(Khdv=c2()L2qSrVjejrxerdK~S*#w*3O--!CJjyzv?;w@%*C7Sx_ zrje&|*et}1dI6B24(cnKjvv`cn^dZ%k*}@US!?=QTiuT^w2a7O0er8QA&2BfJMGEp zD~;5&>+KSCv$$eof0`LFfaB90i0DOjb~n2%S5+{!;}-Z=dV)iJ&`&t*Z~zB9bSA4@ zY0+NkFCFYk!dr$e1C}AU+Q6QNlhdzC>{_D(lCratP`XQaBTUG^95@(0hj9RYHG9SW z5xVfD)KFaEgz;}iRcrzd8GDAo$0|taT~CER9o-vob8TTJpsf)tDnQuZjFkr;c|hn- zIOCIEovwb$I{yHTeg@oXJ`V7_<5=+Y@=hTr#!W`wV0jibz{`apl|aeZ$}+?X@bP)I z3?3pr)?Be}&i;K~x@-B|-V~hT?6p2`fytL_nJP{Q2Nl!lmSQyCkq}Db~Hut5B6KW@%Qbgr~GBrbW6FkpAXnwm}XUFGF&iKEQH})F7-wumGc7vFl+h( z@V=jUp?Hc&4ck0V@}rjop6ip)k`2o_;F+5EiE-@?V}pb zrE6{r-rbbL$XtNOKY-&uPiNL1-KU3Ew2Ue=pb$C^-`2K*X%%e=FXW0ja!D8mEIZeo zd|KOhx%*AYRgZaSNKj7-p!$sGr@eXxw(AUgV~=|>{sZ2;`^Tx}+so!y7G;SJ@(uw% zHahm)IK@~tL-4}JI0*At@OqP8zhu5vh$rYaf-*YSlKdQn?(ZEXl(|Mby#_zdE7`B% zTWtXV82Lf{DZRmJc=yD1v$k8i?MTZnng%0{7_q@T40P|!asCR_4ENu-v@5sn#EN!6 zob^7w=cy*UuNlcWynA?Lb$1w$spvg8>r`O#CV*K8@VFV;0MFMH5>mLI%9Xr;Pd_#W z2{qRIB+##XWvX1oV`R58K@gMd?A#C-1qAwTJoaEOCKc#*^`- zzh@s8TX^33Gd`a@a!09Iq50*u$Sld9<5B}C=ysL#5UTFi>}zk zd82BUl9r0*7hHw}AAAsb1Q0+Vigo9UWwB!w+%g5mM48;AeS1-DE=|reO#PN@wDj7u zSuZ`99FM?Oi+l?3hM0zZYU$J- zNys(g9xL$z_omO#n_HcABPk{9 z?}j*bV#Tx0bKjC^sdV2MU8bK4SUt3nykUV@!vXD+%@z2UFLZjUGPdXGcZK{rKDGY< z1X}og9i65mo8f(=u_n`jlq^3WYw)*I_^YFMOUD|mmG+1Jl?~h5hn7r)3d|YU9)yGP zfzrRDe-K#cUk|@zuMGSeOIcbC6T#M24>KSE_P~nGkM@Xf*1rZmI{wa{1=apF_=Ch2 z)^`rqTAMtUW*n*=#Z=*bVh8}>cQyC9tT5im%B}3X8Ojpmiaw$69kz|)Z}=r|hJ0b9 z+ZfVsi<-1>Wy#*6*%Y==_2(d1$A+o#ht_a(2x zohQH_@JGJ_L!izgOK%@vTuW@Z1}2^5QV2Y$!j8D=E99>f>G11Ni>cyiWVL@X7|;UF zNdTzNADLob+d~dk@90gxW3Tg3aMgE+}KTx>8jo#EMH{_JP9^PvARO zk$98A+P{YGuB@)K*`yIhUF1b9paaViy@w;}Yl8zkzO*@G6rX3goTDyc<9ufEB(^`< za@|cLvxym-lji3;$EhCmhpKq{UhxK}G?HFBG=-8<9|V((ll3(X#l6mzdNli+7g=IZ zLhapvMtL8NM{l8D-p>KEu$8BtUKJOTLw(`KJLLLTrB4rqr|&4DedoE+_=8oB&i??+ zL?l6NpP9b#+kj86WkL zrH&0|2_=nYV5F-SR&o^NE0A{dW-1BBYu`WMw@Y;w?YpVo-d?TY)OBEBwQq8gwEhU0@&_Hbe@%Pv_8C;?TDOLBes_4?H(p)WxeDCLiHJ_jV6)H0w$xmj>}V<6OxJ21lI z9SZ@DeX26szGIaHcJ!tq5*UnU004f!g>@eY{vO@@O7W$qh2VK37ZQbep(tK3ar10u z8-_?9QaK#gI=0snzRP%-_MTcWTO{=9U#LIuZ|Cd_;t!1f02O2SqvEX){{Up^60DJ4 z6O~Z}w;Xo;0~5wr9V-g=CgltzwG+ z_5lz8GER9TImLaEbpl(*xxm^!>FJ8I;lB>r=z2_Ab)+jCF(Rs>4io|DS$-qAl1WPL z+ZtqNkD9!ivU8QsLJ{PNkFIM`*`v)7&U1nZAbxfE-TwducK-l^j(9iV2f_Vk!O-}j zKesg)8r5{$Y1$jOq(3rA83Q@o6@z`!IRtm@zu>Q*@KT)*_80g@x`SzlQr5Mpn$tql zQsBkr$^d4QdgQ90!DEAifsj~N^SApze$gKlKWT4>S6>nSEqJEqQPd`4B3Wf)6|`jK zVu%uUs*XKRQ(Dl&I**y`MOyrd%X66VXNt977Hc6F($#Fc=OCu^p99HnlE4vV?v9TwL z=h7IboXrStDV!L|yJ?qu{ z7ox+f$sU^|WrVp|P>u@s80p)er?qt-A3xxo{{XZf!S96r4*jDv{{RGEYhMpMeWaa3 z!~P=DplgJSWTpvVxCjY`M44Vcm;l9iG_S3PZ4UsIi8u%HrV-Pg)nOH7WKw~6b$()dS5Gi!|o(aR3 z#=;252PZsmdeSVZyAE;Nr7`2!;ewol>DH`Ab);Rxa}x=sStf8i$blFHt~1ks0Sk~u z0-SM9p8M>|B@p5{6Ut^`$zkjO#|IeqKU$@rz-~0MUJ&6Yl zQOf7kSJgkTKZtKr;$MbzZB4K3AxQ7djoem@VHZ+cLhi0lB191dV!d8QUUh0xrBt+w%Z}4jp4%ra%lRy&!sKvI9FPu8eZBiXL-3cwwl>=CgQJ6S86q3U zyqZ>SndRMxAmn6V;E;L@e13V6yy4cN8L0ECUe~j6YW*+2=Dj@}Jf}hts@9+IPBZ=r zZTlbDYTvbHgQ)ly;oXmiZFS!VjZehd&X;T;XmwO;jbH~g{+RbtLoaM_qxQks%|bqUda?nWIT|m9suJRug#ws_?K1q ztMOAYLnQYNB1IH2DPrsZR76pdbH)@7N$HYn_gD5z@EQ19@E5_~4(;rPt7%#kBI*fQ zOGi7enJS!cNRi(>o<(~6JC)+He6p4jy!Nb{{ULP26(S!@qk-xF(C&SPyiiI73QC_ede5W_^ocsLF5Z~KYJ6P$X(L0 z+yMK;fIWw;dLN863tO))X)Z#EWNm;DKu|NzI$(F8Ep&Nyqh&SKxu0Vh6{NJP+| zF;x7-kTKe}Zm;cb<3kLjB-8DWj#za+raAYmF9-NC>&5z{N@iFgWG=h8c0hjcUqU$y za0tN}6vbv@x=S=Me`qspb_or&`+ ze+m7lvdbY1WqYug1wbGyO2t@`U!YU$o&oUBhBVib?$b<*?a^j9t|h}uxA?GsfFDK2 zzH6eeZ|vln*eFGLcc@X&5rh1yTDfXmt3#O7d=KGY3iwjn!(I&V&7P5`T0~;EvD0l5 z2rby4k85d!*S>=$LtjEth#$1ag8m42Zs%R`rnLk%_Uy=sB%d^`)G7|F zPoVU#&F|X__Q}@&0BH{#jcRQzbp0yLOnPmELy!sPM?DLkK7)^1iOLkkx;%H{Mf~@d zuXS+Yq?MWEIqJmZJJP#=t(#~;a@SuIOm4TeVlijwfwh# ze#B8+)vws$Ei?<&ghXJww`k7Ri6A5c^~Dw3Cxz@(6M1uYJGtH-Fu(xAA$s=CD6h^x zcJ*xZ)w`dpUm84LFO9wsY1+n&wv$CZiEn(@Z+2rslA&10$T$E2^BVk|`1j$D5_nJJ z*0bTuRgTi}ZdAi6%C3b-1d!vXV0g!VYc4O^U*grLg!L~3N8{V+G~;fzj|I)c7|aP6 zj6wnAfB?cTd}Fz;Li@)0ua9*m)I3qXTdyiKi6mB1k)mu4z#hN?FhCvpSE-K9>0>b5 ziY_$^$?j~2gNlrwK7y7WI0!SGo z9OESSs%vdB#+rrIo6D!O7#=kzm2Z_tI3VK$^~MK0))e!}#|`y$?<-li*URoETim68 zZk}J0cN9w#J4qBXJ8nX&@~-tiF+X^Q=8Hi#rxce_Xz`<|MKk$i#uo!BrBs$D1mU?E z=Cs2AtH0xHrvC(05wl>Wr)!s-^xq;!h z3y=U9?0By=Vk6s7y`HBtsrVww=4(xA9WgHNhSZHwK4*71Wi9BxboLlD`!5UF=rI2b)~f_VJ0W=NFw@dkjrm~8yGI`2=aK-&99QZ2b$_hjY1Hc0zNWcg)5VeIvn9gBd%&&wDKWnge6KJm?;<-=nttUzd@e$Ok3Ot-a{cEkC-S8kbQfB z&tK_D42tQM;}OO{B#uW;dXtW)`Bird9(EKe0m&fpJwKIMmOFF|qacpwIUxFx$4vTm z>zan;#OGq6+6EIX$RFqRs}<2&KpzG+!j6Oh2{``%BAP*z{_ui({W?^hNGEZ~$3c#2 z?vJD0YL`)1+r(a5hQvy%lEm~F{=GZXY@VmE{=lEI)~EYfe$bjn!c8~TuPR)#`DM&29fqE zsVqi#PzL~D3|GZp@NZ}Q6OQZld-y279KIsycJb?eC5c~5i7W&{+TD~EWys#_*v9s3 zZQK-KmT%K&?UE~BGy)X?AY-r$@!L8);yEI&PH3D)s?xHVmF@SIUbb-)U7)ZK5lVjyC8>YY~%n>IT^-%s?321-hds)Kd(3fq7aDGl6vvQO#u%O4Im_EI6QUZ`O~Zw&Dx48L@{0uk`s|)l^ai$bP|_#$^yp?`(47R$IPo8+F+5v1RUbOoSRsd z4++5vh}6v}J5L0FF~)j~oS$D|U$?*TX@|xuTfcz+02jUlT;4$)=8@w24-s0$=C_#k z*Os4Tk@k^|t1X;kE8b8Dr=jy)tDxj&yb5J?n`9I8Q5K^^P*5B~szuYYE#_3wng4}WVt z4@!BY@UFQphx|b-G2{(DQ41x!u5*R$sim9T;s&h zvm}Xvt%5xbX!up&i^O)95?wSRIm3xj%CZs2AdC<}0|a1$$f{{uQdZ`tk2TE~QgUtE z6YU;nl8YLb=tj^w0uNq!6@#j2w|abVwd71b(E`g0l{j=GADAN;^%dyRYb|T49~E7A zy7N%Gw30b(?yi|xvWIf+EEu*(1Dxiwelz?-)qW!D(rUNbO38b5Ae!P>)Djq|$ik=q zoMnguB$9dNp*^fWQ_mIKZ#_MID$H%?C>#(t`qS4hMO=|c< z$cg;930b1&A$T|=jQ%IvfmD1;rKj5r@D3jizmH?wem|{MwFA>W6hJiZ1_>72DY%`b zE;^CXN6-#G6I=Rk#Qy*jd_22tV@1B&ym|KW5@L1FIT`DY`7@`s2g%|@6cwwH^UDl{+jleKou#KY;r*C zefh5>_*diaho2X1^xqnIcTIV{64IATl^7D$Cq^f-)=Cd>|2aMGV1#<*<7i zyXZ*!D4A|&fx2%1{#DNS>qRVD?NEKgoPR3ov*GErOIEeE zx8E!Lr~uD3Q4`q0{hD;&x7gjqGwc?(Yyjgek&r(uS7Y%@Le%bb2`qHGt2m{2of2!T z6@)OsVgMwZ9sn7rJ{x>DpW)S=?cM&SXOi}C{zOUgKmfoQ>csUYfIDZcOYy75J_+zo zf)9%PO|4r%ktSe5S)NAeff|rp`YuLKQU*x3Yutl^i@=@__)FnvykX;?5=p52jxa6l zi?akKj@c$QBY=>8<}#q4Kw?};adIT^BADC39)&rAj5Pj!k*@#!nkAmGLvc_bqZ2Zgt7@_Q~olEo4>y0LRud zM%t3K8T&(gPVqE4ME)J{-mtdMG7&4N$;YSCzE9Tm{{V~{HSmW=)nU7Ty9`8NmM4}Z zeb6)5a%&IH>io6k^YMv$3BmzA;)#ZDGF$mKh z5f_h`?j!Nfr71BU3Hwd_NV@T_#9NIv`&4Vi(t);yLm_C)ihp{^*k$qo>OzX}uNiob z%S~S{R~~a5Btw#0*mX6x;|~zaaiSN|&`GHmK4kN@QZ?t0qXo}W21mVfS}%t5+jx$z zc=r!A$yj8)yNF2P*zQsQAykK#hD zgZEE=UDR~lAzoceN>^zTSMs7~&gBD^$3j~qWSr+Bq|}xc(}X%ycGj*Gf`N9k?qvW0 zpW$FR>T(Bq`3hW7nnJv!+LR^nLPq*go0EIJk> zfH(k*QCd99h2-xn1h3^*NqmHMusN<5I$G>wFc zFh(#*90GcJRhPYr3*WVA_l7B6WJ7TtT^&Hf5C#c64n{c1=DtD4nqKR%=_HyYUk@x$ z!>3;@!b2P@q z_RU&*=d{*iy@?5$))n0+oP4#9892$xz6N@58y#vJg^N(X{?l0AX%!Tt&ma#Tae#7r zf_iXj=+lyVvh95hDYQ)oRPrFywX{cj{fjh(N`trM1RuT><0Gb63aP1DUF*?acw zp^cVBHw|tGz{uSzAscg$M;rlx#b;=i5nkT4)ToPW2{Jscv7}NCK^wWt01nZN)HZtU z!p(cF>Dq&W;hr{l%#j7)WDnv1k(0?M*jB0z*ZGE*)Pq-+;tfetL5@XRdtz0VEuuzN z`^0ooG5{l>7&WT}*NHB)Ynyw>+Uk4GtPo-#2GZFBAQtW1j8;aWty;sWU1^$p(@!O! zkL}V+A&@&VlFO6GEI>FPbYN7zANY%P;#~#cZ6i?9nptFr%v*C7^a=vXpy!3jT=CB} zx76#(Dobv=diN`l?{f#@M}tR*yg4S1sK}yo1D%n5iF|$I zYj21C7|^bD4O>yYTT2@#B55Rf*zHwS8Nn@qo`X3jgI_`4!E@pFn^4p4W181ZjZ%4~ zPy&N6C@7?_5_}A8Z{t$v9HvI zzuOMilWKqgC!hEZK7*xxLjM54&i*#Jn&aV*fuaY>5$adrOPrP_2)7>C!60X;$QASv zo0Vm$yN@g^elqx=Pv@+=1~PgnIQAg*CZy3|c&&}>va>R&3JQ=^9OvAhfByhoUDs5} zBA+c#Z6^#fk@U|QIiwL>y{L*-XN&~_Vp0Zu4?s!Jt$NFA97PooJWp_7R*gthU~_}@ z`h9WfRVQXOdC#ci3b%bE_Q%PZLKh{Nk_jDn$FHD2TB3~kRyqXp zEHTW)G>kG3cVY)54Dtp^8NfKje^P(&O78>s2jTw!?9rh3cfguphSuKMM3PAgtdhyd zEa5oZ$8pE-@;`>6tej(XR2MXlr#v~ML!?^U>DDobh--HTYhbw}0dhIyoN?N` z^SRk8AIvZO7Z3KK@lVHZ+tza|_4vNSKIV6t4v_zG!gK#s9F*xiGt|^wX65zQw7$1ge zTgJ;|@CT!FkMXS^hMpRq#(xnsKMKbd;eDz>9Dwu@h79EU1q^=*TaL$H`#{fkVd1X; zcqA>vQE6TSw3bMvEwoK}G`9}w4;W=JoQ|X)!-~@23p%qVR1h!@GwIv>^sQfs{wULY zb@3lr@kWQ$|Pj977GGtroC`9&CV?9>tC{8k`h8YKOU*zv*p=&IO;u?Okinn@DD&E>PDoNxJ6m;F5xyT%Hde<|jTx`3JTZTJ0z{YZG zdT$U|#e5ga4yuG?_hL_eyyrO`4N?(pG)9W%DPju#RGUaLoyA5!p&v{VIHv2mUZpg# zyik(7sN`gf@yD+Rk58q2q71zET+Dj`I5j(@v1#sE^L{fZQ@PxLSBV8l`~hXF zXi`J=T$WM{V3G=+xcuwStqxkQUT3|Bi>Zf-@-ly8yLfJX2kCxu0g3LxBeDBTmG$St zX^o%59XDmlf%aT61pVU8#e9|fBI&KB_zLsF(}TNB)-PgVj&bKR0(~+V8v0Yh&lUdw z!d1{s)50gVXvDEG4<6t^VS)}ueRK4$S`9jJ^gM~d#;ZrV{>xt-FTM--t_?!s5qV*A zd9wP?&BplIa9DqKMc{&Z;|9M={s(Jb9`O#1ABa3XtNp4gxWl0wl+W;y)Dh5%{Ac)U zZE@kG@~>VQq4UEuc2Ke;BWkDsGch?Ju_X0k0QLO@{eV6$TKIp(_Ff#n)cpN3T8KuN zKXTw;rZdxbD1F#dS8>$NNc%p|Prtd;hxTQoODTkwIKWm7#!qe!>0K?ZhDE=bdofo* zhFDGuH=xffJ@Jb8kHJ6iQSXfY3%PjwOJ$~Oc2nN7`8Rg|0B4%f8b&)rkpZ|WkhpbX z0P9-bIsLPKFn-bh021`yfqw(t^fRj>OR4I%zr2>t%%Z%qmpecTje*b>Wf|t8h9aF? z^1X|#LZu|Rx}QvVqvBd={uR`5&1NZSr1B#Fv&< z<8v1zk_Iw(#sz$@bE{t1X-3A%H=g$G_98|y(mNfhpm!vY2e}oyZRlX5W9SdqpTIHv zUHHp&@he}q4WaleW7?#PnB~+TCLnte8iOGp-VyrOt@!ifHOGy9DBJiyM!J8oTg*hE z8<`G!ElWnDr$CZAZjj{wbmFbykb(M}^@_G1;{-?Kild|~l* zt`2_BYnMpFa^$u*6Uiss(t07iS@k8&<;I?E!h)Gn2y@6Ba%;rDYY!A$K|h7HOG(cA z9~9Y79^X}&fBj#LdsmC(c&yZI0f@)Uc@^^q?G59<3B%(*4rx9y*X`_d=)6H^E}?gG zC`jOeW>Eh$fktCP>YY)0{=m{sEYUDp^U)oZCjei$>LE}v<$ZfCn zDPGPuI1$0P><8a(IPH=v&HOc^Yo0XKR?f|pq?Ps~NZ1m}e(WjOyB+}Po~I;d%*A6> zILWP*?);YGin+CSu35B61?-YX=1+p|xCHIU$Ed*{kJhcrsrhmTTfZ~Rxy+l?g$%fG zM{HyaV;yl^*M&S?ajI%Mj+-`?Aq#LLHtbtzAHw*?6@dh5=6r-)KE}}HO zQ&YLUfHTC?u!)hE1xFqE&Oy(;6_I@YA)ivZmr84K16;X6*vn+(o_)#b#TAovZ=o$K zo*RFn>M>vW7CM<0(p)Rb#a2SdSe&q9jC98y=C%^T@;C>HbbH9|A%OyIobCtcE1_AdCwq2Zh*>R48lQ?^!~Q6k>bEyB*-DX%h)e*9PSLea z4nSf_+z$f=wXbhtxzyG*ntaT|%v`Fx5tPn%3}EB#oCDArcZu(0*EG#D#51jhy}%1> zG}y4p%FJ0NjkAxKlB^Hik$&*w{?3M%VT zlW#&6jaL0&aWbq)w%u5Oh02k<1NWP0BR#=A>aL@%e`V>hL1Umr6tM+iBE)e&QM3`$ zgVnn6TNWA#(zK{G8+*qynX(dFtERKrCqM+q07!;1NgZ%N2h%6(S^BhFHNkVIh~-1&n_1mhDxtwA za3qiq1B~?tHK}o?+UuGWdS%{~1&{Bfh(R+kRbiH81RP^HD~|7uD>)?n_4gELH*geVgUKfYrGB7)!2*0Yb*B6t@$ZeU-QBg%7etoQ zGzkz9#cG?#1CR?jI6eAT<(bnD+ml1nU@R_zglW-31`WF^1_}}eG0)Uz75fwa00f27 z^&b!Z$r^5>d<(9#;vG)z5Ktil%~aY3dInd=OablB(BzMntlH}z)eb2(kFalLcz$A3 zC_7a*WS^(G&+|3-&Hn%gC0<=zd}#0~XeMNj!t;cbpeWKc$R$_YU=n@To-6hfR@E(< z5G9e7&pQx+Sm2Hipy)oCugD+xIOpuo;}3@S?Q+G=nSL1%vtdKV6+!9OJPaN?=Dw2+ zFZP(5)sOC5KPxq&#&FUwI5=~_0QJDmITdKeW3t|=#kvK_2iF||{{Re{FKp(P_%Q`= zdW6O~!Nz^Nex|J5My()B1qcQmn2<&~@!LF+*Pp_^g&t|j-UN;ZZ%ph+?;7je?xN zCm<3sM{dOb0F6CtE18!TZwj>d`FCW>KVG@1A5Zf9$Qjk7W-Tr_!8yPO&|@dl@}p3>q43XHhr@m~ zv(+ym-wnK~Wr{PnIRr8xAH+9>Bp#(kb6?X}{1MZ}`mc=t0A{^6R=>2_8t}7j3}=0+ z3mkpWpcPf@27X-aAIz_WG`maR5L??_+^jIPswH>a0|H0O^5BBQjtb+RM^XJl{{X=# zvp`?8i6*larE1YUOn?mw%9q z4p=S;9Ag9VKb3FjG8tu)%^&LlP!2!_R$+&2>fLxcw(F5#AA*c|rA>HZbOX%15ABny+~42ow~7Pl@3a+ z)FhUQMg>OZLxplj?mz9#@W=LV{{Vu!{=@n&#P5avCDZi{BgGnSG!1$3 z0F3?+_~%&oRq+Q#Z8Kf))}+>YotX`jDiMZ2>Ip_xMpj@qD}#ZW!b{egAF~kP%IpH; zJC7r;Qgg{Cfr0Ix*+2L%C;StV*W+J=G@sj__I`s-mj2gGw2#C_IRJ-O(qsYPzVlc2 zOXVZRz?F!!G7=TG17D81w}*944eOU0PPe8@1^unrg6`6A$t*Id!y|*jkidcue0Acf z*iPmJ;=Po{D9FZe#YewS^TkDVrorS3sQ&56TrXcFLt2{AHxab!h6V8V$Ho zIopnR40pf>IpA~r0=2#&c>e&w-wS*hsr*3rjpCg*LeO+;IWD!0V^+P6-qz0DS1lxx zxdA}nWS&VK&MWZ0{tB!A00%7i$KocTdGKTY3HNiTcxS`OG?8C?OPV!JPgg}@fn^h^ z`xFeox0=At>f~ZDd6=guSsFH@8=po00Ks5%{{Vu}cybR6Yd;jcH=}s=+Euymwu67G zLlyR;CK*fu5XIC=i0Y-3F7;8nfnS}T8u8m|>E=lrd2vc1edShX&3zC1H+)rx;!lU~V(~Nio+Hp~ zRyVnk`P6OHuttz~bxf&lq^LRP6*#(Wr8U!Y(aLd9jGZZU*!8c4uVtkCOw}N>P^)zu z6U!LsyzWwQ>Ov37zJu^~nV{eJHqzoXXys@yS9ci@}F!*7v^q-73GfwJZywmMo z)jCAA*;rv^r{>ol3)*Hg!0{FGA=<~CLg6qY$kjZH)ZctgC z+$^z=m;gdHNnex(`F%U~YWM@K_trQG#7GT0O0_;!As&)B-M+RXCP5&&iBr z0$2iejH>2XN-%P6_UN|?7z+4_sKr|SdL1{!uMGHx$HYuy(&iECRzKQ0TO@44TY1@t zDtqm4@t;OQ4SVP8WhaL|AN(7*)i3-R_D>W{=F3~UmQeCfZwUVYM*!sPO^iN4kC|DB zCkD8fttYyM2!7C6Euj&@k`^lxDZQfDW?%cckO?ajC&xaE=&V1PJ1d5SIwm8(B zk}^r)f}osaw;gMf{e?UsZ{x4pAnO{1iduP_8*5lqq>+yYaSQUGQU@p3j%#=J99qW=Jbg?wF$M9^(~7iBvArutlYW1XWJEJk<&xR3ZJ z6{NcV0FS&!c*V&-qkmD6IqkD)wqa*)F{ z_2jX^1TmJDS)3}Ofs#oi1poqac)+j7Z}=$B$E_3MSB$RoUj^%_cc!%4eCR`Ll~J-; zOMT@m00$i4<08A?+xz|sFRT1%*6)58d^^6CJT-3YmtSWkf3zGQy3PLpJi#A#_=x8? z#e9h-k9n+KHlwM#OEis=NZ<~@4#0M)la&3~>_cV(X^Hkje|W@Oq=6PB>g1ksdkh2b zT>#VJzS7>qa7b7t##Ut6hEl8we9A!?1$oHgpz&-8X&tth02l^W1B_?oAJ(7Y$>WkI z)8%cWZc)MH4&Z+}<;7v+92HoltBk)UVCLmfHeT_bnQ3|9!{a{<+dAr&j1ydxxeFm# zR}25Q^EAfIH?DTwD0U#wFo0y%|P2c zNo$hL5s0qRK*j(hVUJuKjw!`nHVx5VK-H~{k)K*ixTcmu!40630{|1aoSd*@&pgpi zuxl+w#v7ri!3&tjn-sSN8C%>0J+aPt_UT1r=@|n!Dtgm@;T)P3xSLnjVo2=aR+>wq0hx)$*vgUt z=O7X9oL232Gruy>gz9T|p+@Pa2qLza+}ym4g3OJb$e<7l0tOd@#cud-QPQsLpw+@W zhfkHIh0H`m^ReBuIXwUvAcKZH zJo=TT{{WM3B;s$fUa!arK&5h-8xN^DVT|%BWf)Cp^(oyaq0xAs;tjRc)!wo!?P0g_ ziopcad5W?Catq@u;YmDTaI4KzpWq&02UBuD( zY{5vkPE;`Kf;k(pz`@6QsY;C{X7Ar#<4Q?f=ln%^b)iq>-DI`Bu~}xxVAw6lPK}NL z+B;7$&P{U})%Gd;T&(ziJM$*qxvx@fbTtji0CfXZ` zVkC?JqYMBbXO1|>TBWFe!ar|p_Bvg(k$KGuO1D$6Gq>GP4n9Ggu*L!7ttxIZcS`U6 z2(2dWjn57v+`YBtjR<+65>0O`gcgaxDqMy;aB@Ky;NbgL?jP*!;3z&1{>xtv{1%83 zeJ{fnhUA9M`Q=YKW+$r>@*JO9{36lxJy!n!#aexYUuG>1qY_6fl6ho;>K#K2j{K?X z+*kAs;%#2Y=&5YiY6?OE4Zz{B6<4o+LD*N&WbL@lPkldM_yE|IK2kPAcsC$@3G=O7dORr(L& z8xIodcd=UNXr&}6JY}TE%2@9szBnKbbJD*vf8e|tH;FVqA9yN15e>>TTh_M2lZNw< zF~bg{j(O?mF<$-_cg4k9`muE`sI&9h=6GS5Of8TA1;80*2N)f|13fZ&^r&?SM&TaJ zxqZwLcV~_O&rE)R`{ty&g=H%;##BhBEWnW2{t?eO@6+C*x-ukDq_dFVe8X-Ta!AH| zk_XrM*U}@*W^B;H_e`d8WPYQs93FV>f_Uxu(oY<2WpY)3WhxhvPaJvy?fx| zXajLoE$`1Kx3^55^p|o*AuP_UcCiYhI0WDvl5j{l$E7=ya-=r#n37XD8(6xKoRA0^ z9s7VibHyvhWGvCexr>pN>5PIvKcPP3>r%@!!O%8cNIO|qaRG2R?0LuKnq)8KN9Eu& zfP9%hDrc|aGC2PL3VMxL(eUN(_BV)Z^;QIpGH|M+X*{wh7!85AhHv5( zh9gRPn$lmqKB3eviL!aTvPq2KbH`us{BvKQM!6a{dT z_=;E&6-XjDRbo{+$2g4Sv?x3ggbe7FNSKaM~Zr>5?b5bmcDHgKv4T;r+F1A+&!KBk+lSpA|YBMHgeK)~tg z_~Y=$HQwH*Dpoj&ZLyqVzF7YNpP;8(B1df_9ow=tfywLt0M@H>+W9x40(K1G<#-*= zar8Amg8U!i4;pxm+rgd>(Qb7eS5cTnWn*ZYo>%+bH~<9%u{c##EW`ySpOBKXIqRt8 zdx9bbk3pWK=cRPt6nt5zYdSxMJP&Z?qVSf4bmSXG1HIJu50i}NGOF{_ZaY`#uY-T! zmVXDoW^dU;S@`YoM#?{kpAa=a55o_H?E#f^jW1KbWVN&@676fZNiGq@vf@3;HpTZ@ zzWUOC;FEvwP4A3f25kNX{6hVi{5@}}>DnwdBgVcNfN8qiw-G|Iz3I7&iRNbwD|zcG zvoYMH8r@K-YcgDuKb)NrPxh<7TpZIU2nRgoQ4nLuOV z(#j;(o~b?^6BqOf>s-C?ztXO@B*$hybir|v>B#*nf9(#oi*FfJE0Co@`==i< zKg^Hus#2Jvk$m)7;xc~nIY9wG0ne`<^lT??hwQif5Fh>ub>V-Dp9o_8gMVmG5Lpd( z!2L(${yXYM5ZFANB#;${{RG3hvJup8{&`by`$To z66n!+f7r0bvHt*SSR$8+S(Q)=h>&?pxmHP0V+1kzkL1t8j|b>FmZ@#xKZM>izwrM6 zi8M!w_Rjv&@s*&tMhuS|3_MPx0v%5cfHE>`^(*!Y{{Vv7&t(m-!e84`Q=#iHTbo7k zoGicD*P|hRaDXkOj5_YbB!a{-DuX#Yy)-T|(#-s*@lS+2LEx=w_rl&R@ZIK_s9DU> zUs>PV7^a3!K1M9bYBCgkie34Hegj z7%Tmt?l{G)EF65Ktk)2*mS)0_Ez|9@L?^j?X{c8Q0nZHf@joBVteUW zpc2;R8TMitwg4|Y*bghq4tl0Qrk<@;*rT_V6n&Y;O!g?0odSV4sq@V+W!Ew zm+eLHL-s%TEvtNW`2FKpbUzPh@NS-RJjt#lCu&1)68zIl%s^Pw5=qJ0s?0poP59}d z-0IiT!Fy_LG;P9bkF+dN$FWIvc4k$K0y$&CfN_z{e?DLES^og|D9?<)Zok?xYg;?W z)4{$Inrkl#TU#;NDBZS8=OhB}AqQz7ZHy>fZenwcrBZyT*GNX}U-&CG{1yk}cl;Eu z$8Be+Jl-4lbEIF${u%I1tmJ?L5SLf1r)|Z~(s?6fL5i+%y}mBg@6E(WNoO0ezmjoI zVE1w@pc%;f=Qt$#kHWNdn~ftwu`w7#jpe8dv;)u_XSdg{TI*JVvPKeemgP?m>#=E4 ziP069blgtU-?z9G)%aK9HOGRyNoC`Y49jlQo$%b-yJ9(`U_+w;%7UkYI0OPc4o)3ZMI{hhpR@T=lihHZR7YL>V6x_$-J8i9LxY=d-(+YFA3M&&1lCm08T^rw!$ zXkQn6SMXM!XRUle8jYrbV{dPzTWWGM+S#HsAplgJMns8uXROJtHd zlhmA^waF>_DRM<4)Nm539 zz8L(f0OWEB=D$-c6GHG7hil;-I%W~*_AuGS8#_FKVnsy+u_cMX$vNboO8h7A{-di+ z;$+n{80NN^-HW6vJ4nec;Oa}97AipJj-9LYv-TwYqIGYNJ_OW0Blx>qw?($Sk_k~J zCFPkoVhP9IW0mc+f(h$iJ($(0R$Rx{Wqh%NhuS`-)jlA$kEcr{dt<`nyqqW;X9pQ4 zjyhMD_`Bnm*(XhNO}Xo{EP`)ovuWzJ(=Jvy?_O;?|LA;GJVr zku)7W+T-l80UQcg_{3Y=F#eWcDoqibnIPnLGE$x|Ny14N6jWyN0MS&`>Dj`r#4l%ocezo@4 zO47tri@xXQ_^J+tc|)<}@!9HrFY!ZPt+N1fp;yoWp(%{ozwbS=AUAm|Zuqey3Ib)DH+;BgNsQ&=NH~!H&jqroT9|*in zrs<0yF7H0^tzogO2-k5Sg{FA)5mX2|`O zcprh}{xsbid>86m5}UwrfJh%h!TMDxNmOmQ#{;csSjeq0NgDx%GI;>wy-R|e9s3-Y z)Pna)(_Yg2vrOJ?w9H8u5lgrgV5gNJkU$3kNyy@?+xTxnhr@Ck>y@~LXNiW%6iTtW zjoZ2$2mHoxeJI#&@g(=kHBgg;NQraFkwa}Do~6Qt9fnCg1x2lRcT&=H_^iL-AeE$P zU&@MOhKL+K6otd|IRp#?_}Ac$N!sVIn^8BRwU0~JZS5ZVRhDUP0EP~C;BUYMj!xVUz~pLL4vcmC z@-=dr)RRK6xWKup;fEba2OxJjNNn>|)q|EnvMTNBTTxtopPYVZN6;9$o&elRlCpaW}3hjhdH+1a$t>3Tp z)tWe~lE6rYem}G)z0JvLn{#${T8-c+AK?HSW{Au>Orh{Xp z!*P5`k!^60!x{37>^9){1dKBBaskN$iicWt5vq)NY*kGr`$Be})=3SNIF8~ZY>SPg z?8Y0yWG>Abir3v7#00J?LXC&8DE@C#3+*;j; zoo$TiD4Q)TodG~U)scV?865Pic{M`!X{k#*>?eOIqDKG~E#>4BhS&&V2suADJxxub zwUW4U#ygz+8VR}7pwo4$=&YcP+Ue$mjixn5EUE(hxpR;d0(uckqG;N>X=&n3AH<7w z8!?Abz%uz^31U@>1CoOoBLR=faV{u_`A^C7JVOA~3%I7~( zUx~6yplKpgs$N`4tWR#sYheYmERo0xTN*zm#4saBA8o`Tqc!ttS}D$oECkw40{1 zNZcaX#!5<&oDewL2YyH(duG2ZKj7f$ZM82S_(uN#PK9>b=`+J7B_t_lR#gM|0RZ6V zErZxs>g|q~Yj0&Ko_{q2f4hJ=4T3tK!yiigwf_Ku=4oeN(X{KkJ2=ui3(zB26rLL@ zSdd5qBq(E!P6m0epu%a1gYz`5{l<^Wp?x8^{rtp`fb9%FbTh)>e_oh1UgAjSRtoDF zZNawnAd&Ke>To^rMdWE5$I8hYuAlrp8c`s;wlL$}17b9as*)AM@0Y zq&%`YXPM$A$kLnGwcpK@tS*z)<;?Jt3tlL<{$iB$L#h=D(R=*zdvmhr|!sFT*|= zU)?3%uM~2IlrAF-cjx$tIUofGcR#`_`fm7kEi_*SS!s5mBeSryOL$`(0ICA21r*HE+6KWVQYYIeI` z3A|~3=^1^*DY$mzjtJa9KZ&pCWA^p^sBHfLX3yE{Q}O<@daJ%bkh2|%S=a<##Eqb_ z1Cx~=%D$gp1XcUQ0wfLSK-) zl7iV{SRSrMatR!fjP&d2SL8!=W-d3b=5_&841vbc#yI|G>sekKoHhKhrN9w|Zz+P{ zvBo;_`E~8@K28l_yhJ~KeUI$T^(lepTwIMmq}ZM$M$!J*|8iz4pr|qnVq_rObftg4f#EL z;aIh8X3I&ulVa*xg}tet7~34PA~R(BKk_yGC|O%t_-;1v-jQaO_L^{8D<~pOs}vA{ z6lwtA1w>E)=mmFSAe_0o7}R|a4gII>d}pI-{tHhE{2#dSHi4#VgHhHjbbII$dt(A2 zvrz<_OscbDKvbQk-9cZOSZVls_KxvnH%i(U?Dg>?d*F8@5@`|Js<*tZZnc;OL5ssZPUAiqu*SY!C{{RKa{g^y|@oV;9 zv;DQT&kWjGc&9+|o#XhsOR~BXHJ*!q_IPc9br^9B@FwC87wu;}WDm$m6sGkn^9KW- z--Uloe-$)O3HbNG8t20=8|m$LqIi=^xYIR@J;e;xvmr+Sj#Yp=l>Yz=DxRXhov+&8 z_IL3&?6vXVUidZQS(@NjP9@Uy7DA%_$;mL>4|Yx%e+#Rx@YkP-g{RHj(i(Yi(ZpO_oN6;nA1?UN#_ukUb5@k;_Zpl7Rx-Ob_zSAAJ@K@Y$Ds5T#}e~5IM1y|B*C0QyeQ-G z?rGUJ2kpPaKiRKP{fa+mUjcZl_BZ&maiDl3#dex){{W2q5q*Bwnsk;8Ia!`GlO`8v z_c|0*4A>aj8BekNa{mB=oLcFZRyv=fJ=2i;%r zP^+sS4tyGt(ikA~bsNYu>p*^U6etT2rMiCY2;dB4`qr3?1ZyQI%a=~YLaTyk{IvL~ z{{RIS_?_TChu$IhF{x=9hPIw1vARpWLBY4#Jj=ARvk#dR<+d+=xX7=Nrqcfav~+ki z3)#ZS_F&4uXU*pUi9EMMz|T=%d}=n{6xMYOTjA%9tRlM7^?59AA+(s08{NXkU~mu!g}x}w2pI|;4^_}h7GoSrzz{{RZW)$XLdg|Ow+KU35W zK;pV@ik<`Ur-M96Cx-k*raU)D<$(kgm=bofF~)e$1OtlXQaiMnVG&3PU@#DpPq6R) zHR{4qoK&LjnaeB1*_{V~Y`jr^B)@}EUn0^y-d)AP$pZuAUUCO~;A7tvr>DP&{6Q4+ z>AHW(9Eli`IJ3T2Y$^<+|9QACIj!r$1bQ@Ig)xDX8 zh%ts|IZ|6Yv&Tb|{Og)ga+|Snmnuh;UU+9l(d{JD^xJ0pJn@x%`OeY^Do+G)k?UD{ zUcck-hc+iq_-(ICr^RsKU4U4tr~?3?A#z6>k05lfMbz{ZL$)}08Q}ZYRK6C9=|~HX zc@@n(GHTX3spBfbeAi?CAAZt$mxB`TLTQcN)!e2=mMeFECCY_hSp2x-pd_izdg8q^ zz&{Y|?0g|Usdi1Y*Rng^-OQO{6ce%~9J>%R$4_oZ^8H0Gv~LcLHt_qjsF0~e}j5fs@@RML^5lO2kz|UhbZjV z(&Pr)xPurZp@b@+M!E zTecB){F`gwU-%<0#vNPXx5b@D_DS(=xViApj9Tkj&@3)tX(6!IQYDgj$Sf5jk~o2F z%E!))N6WdZ@6Xwi&q?@8}V!Y=2R@8MHi@WLW5Xm55+k1120qtJ@07K7h8Ga)1eeRK> zPkF3(hzMYehK1Q&k`(mLNE~&q$FpdrG( z1pYK?VOClcX3GEv8NvN2_cu$YSX^99D#pH10hz!7n1x)Ez~zTDy9jTULE|-3#W(hL z9v_xXPS?%0ml0h>u(5cU{K0Su&Ibnvn$I0CVVBjKdZ@q1=cgWOiFuOfdiIy5>pG>5 zxuZnLV^Y!B&XCBmFu`r@^A=GMe5gA^6N+@x__EpyX|=ogS<9Q409~)q&}Cd>cmNOq z!w;DC#c^I2m&Vo_)$8iE6KWCaF6hm0icD5pGF?dMHmE|`1b5-9rSRUl;T!obF7NeS zw$@r?&vg`x+vH@8z?3NdHE8yX9x_HN@T!!QFMWTn)b=!MR$tL2(tJZ`_ILB!TRKe* z!I_B;2GHS1Jeb``QciH#$5JVo}fABg<7}U7-mZpE7_HWZ@W| z1xKPZ+I_*3Q<~__0kyk-J|&P(8ti5nP)I5WCNskB2OVn0g{WG~X?1yhDx#f1+}vl( zGO{x4%Mc2ayc{3lBw%9|q$#+|U$|-->h>~qF{WAH%Wr2ST6(Dgt)9pdtP}!syP?KM zwkWHfF41+(M_rQo=UtmlxRw}R>ODFr+8G?ghkBB8f-pJsLZnj4&=`k8MnaLfv5+_=K{z8dso^bWRkyUawwFzfEiOXMERq#z z({TYxfN($|agMn?jZ?SLyftyFTu*hPq|?mJELN(LAGO^he4E{f&T;cbGR#zwfOA## zTLrz+%UgJcf3{pk5pU$S`MkY=EM8guHg_r*0Ju;FM>V{?n}Uu106+LY=lGV|TBCOJ z#2Rj=;o$eOMy5BFMB_Od*lv6raf}iD&`IK~{{X^K_V{fi)MN{DEJ8gxX<}I9E{q6l z;4a3_6rMBflU&rvp?!u1(d8F5(Lpt|l1U&hwiQg5Jps!VVbM(j%TLv{(=-z_R@X5p zP_v0*K(T-j5J4jY7-BHk#xYn$MPIew{4tu8ld@);U8Uxkc{}Kq$d>Xf_Yp8K`7<%t zlBonM9Go585^_dJt9Sk#5vH4Q4yMl{LhW#{jI@fZue2ipw+cwZ3=VQTRM3q>Le(ua z-CtFnHMOtsDC|8VC z?frTRFR5zRShUctZS||&1=iwaX1Bb(jfTZjyKwn{W652^ch2V>EAFrOBZrT!wHNVs z!aJ=ZPt`qi7Gu~i~NZ9mL=urQnsn@OktUaF>u)nD`&O#6vA91v zl3VvtaCzD~s}r6v^*{U)m&6lK`)+BPev0hYllV_ku(}qHn?QaH@CZ zc{S)|)lLm#tVjT`7(F@82+t?;ug}l;DUXKJ z#lHq`yftItNYeQnPXn-N8Qn7bK+iY>x!N#z>5sg;b#r&4OBKeRcf4S^44a3`$xtu{ zpwB&duZ_QL&lKq|Wfm@#eBe#Prg4@IImS*8KsW@0p0)L`e3cZt8%^^jkH+0+OaB0g zgIo)p$QK)ml1m;~{v`DtzSWM#XclG?p;OGImucEZ0QUnND)l+(Uaj$Gz_%RjUTLhr>3Jk_>@W#Q^4J1T83d8kQ8$?^%KKTQ05gGrF_Fee!8yR{a4KnLkr`*Z z3mjpHMQyBHD&QP?1A~qSPPnR#J4%iAgSRS32Wf1c?cXGf_0OR6tGg3xkzKT-DHYBNi>FKbLNmjGcGyKGC>D{&m43ggmOp)%DR?qs6rx;O14K* zI-jRZCoXj)S&nzjBg35xF<0DV6k-bI}E zA}yoIVzLG~RpbIQ&V4=2d3EaZ&c|gKa>D23hx{Cg@p4T&!9EB0HD@t*z0_M#yD|^~ zSmP*w^(1a$dSiiKk&imX9^)xz3{Ef!cR0W~>;T6d0r#)bKlnI?$54aguZ4B{n>SH$ zr%4JhVs zrEJsBwF@v~(*tfn$6?nW*0*j~%VM!eyR*LxGlA=l$G<${xjSt}B)X1YEH=r5%KdPA z52(Q-@vg5?K_rhX#S$zi-d{a&*ByO6rnY;Mwk&vpJB?4n_Vd}yzi1JVj#C93a*j{8 z1QW+Rbg$?y;6KLgBjcCA{{RU5Y0_j?i^YB#v((53J0_AP^U(XO;#5B9Yx(`Pcb7tz zD~R&j2GQ5cUs5}f+uVf! zS_#AH&9yHsW^g-!qMv9 z8@yX%BFFx|&Sv?PoR7NraBy+Cx^APw7Jmr8;FX`V4UfegRc!oW+8B6d|fmm_jA*BeuO{+QCNPK@GI@ zW7FI8u0EyiRJ_y+ODiz|0iiEm&<&`idki1ffPK%>vdfa^c204@=nX8_ODlPV06Hj8 zJ!y*Il16dfn1X4Hao}||SR5SFrI5(`zba@MIG_*IZ}>Mx-&FB$#L0A-q7P@`%c*s* zFtY%Ru#{2q^kXBTT=3XnbJD)&{{Vu7c!qnW(X`3tGDUBA&e_ilP_BOfYvND%BoByY z{gS*EzB!*mTj%i=mfQGbkf{+nD}slnM$iasjHx58Yv|wFv*QPcJ}o|#sd!dWZT7QV zqYb7f#sU%QIXN9`(5D$FqbW%=&zt;y>8AWd(=Qhw?}fKUxndSw%snx;IX-}o;;&Be zCA9Y!PA-$qjbxH2x2VXBp#U6ubo{HyJ~7QC6WiW>p+clt2*j{0yagnG5af-$$z1Ra zdmg7Xz|bwVsbgKgXtH^g#&WKTt)EUvug~+_OA{|CHTS5?^*!9B-V$DigZ|JS6Yy@M zqAsJY=}}w-Fv9+1O68tbMci@;!h@XkJv-NlXdVFYHnL^C9u%`#AitGk^5Ot4(grdJ z#t6q9d9R>8Cg{fY!hf}y8ru5YM9bVCH9!CkgO9ENuQ2#`pytBm*4hfF#Wua-vaxzDMr`~4uratg0Q>0ehGQAqKxbDH>np&L&TTf0Y<7fY2b zg+E_za(p=+A7T6ELfPic9oY0nwe(*Da!bw*Ii!rs;v zf+d0iaUt^@DBZfgI@WaE&%imr00H0p;=T9wZ1}gQ{?EQY@F$6H>{{+E9$9ZS8+JQN z#VlssuzNbA%2Z^4M>!{s991a$T5HVe!^x=2(EShaKfwDNui9Ho`0zCQ$>i{tgmq{p zzO;@QEknr~Tg7b;m=#Ce8K@|V~xNZE#8UYo6il&q3Sx;m3DO~AidWvHF@~5(@}_tQqfVB z00fBq(gQq<_+$5S^&MkJ*FF?@pZ1YsWSVuKinLjLGjnfkDw%EVH7Fx4T5LNX-pvw_ z6%|aILEOxFaP;jZZiiJ#T^`-ydx;lLA0feBxav)LrMi8dJLh*?Zej@;InS+nUyhPE zG?^w=zy}fF@qkTuZ^U02T6kMWxYVz1mMDrAc4oi;Wb%Is`UvcBGvNOK_$rUZsC3Z> zgKQ>Ny1JE);6ZG1x%vT<^#Z;kg;FSEhmVz4Zy?}~$Mde!_MP~zd+~?kPN(9z=Phct z4Hd8_fS~ptKtC$u%kT1@NR@`pI}^`ZsQc15>b*-4!1Cr)2b>?mpJe_Vk5i6+6-y7< zAdyLe;2o2($l7t}2_SKj1`b6cU5MdnQ|_qA=ck~o4S&NHz9@TtwaafbQD1pC*6Pe9 z7VfHi%#pMflPatZS7^v>)!=bnI-Vac_6hsdKP9{UK4Zf|PNn-6pZf>H`nH2(t6jYF z+XEDn`J24uSO{5szDNfnC2|Q)M+8=9i0w419~K=CN=a<*Ju&ED*c8D0B!ckIb=wY2bCY*?Dn zRra)xCguc!7?&jHl1_Wz6h^91N!=8jdY(Putv zNd-e-93H)MF1@7LL3unnPN#btz^XL0-mI*}Lu{AjCn{J40FFQy=BKgHf3jk@f49ka zuuBmU%FGEov#4?y05S@aM;YTJRws)s%ri^k3s|))KegQ~b3>+Dsg6yjcGhVa;NxQ* zxj5u7JPw$-)x;#{eJ$U$wB2>+OOs0HS7{a2o8kLm9+7zZgGi$KNP?7zgyEeqs^EYF zF!`~JuNkd9C&V5A@cyfD;oTLkEpJl)00vzwhK1NJ({L6=CN?0Q(WzwtazfVwbl(v* zKMZO*=ZHKlCab12zI>0T#-Hfqu^+tN11MvH&Y*z7SLMjgro9D*zjLYh;tPwdGTP>5 zI$xg|R+zRGn4$#`f&k!lw2d3=_@{Gmd*$@y3^^UM!l_cFl2m@Wmh5CXo?U8xd_#FhB&4 zEpRy}o~DK7p?P2fUu(}~m#Zvo1Iop=gurl6_R0Vk8-_;U1zotXmIHSNpJ=m3Iz|Q~ zP-F_BnWH0i3lea1_f(!S#Kw&n#XYa`{C};9Qo7vH)BH~-iLIXRL(^2-M|98&xh*`E zl~gLCfN~fslg@B+oMSberFgz;JsxY%6L^PHgY2%#k-?`Mlt>j=x0XYxL%ZhacWxsi zsl8#Xz3a*08$ay$E$nROk!=JJ+rexUHW=Ur1d>MJ+llH<4RiW#lc*-ACG7f@(wofM zS(}S>!W0|ik+yNP0EgxWJA!~s6!5g7mN|jz8d>o)gr3N#F_m6X9DMsNcjIs5*|THL15_V{HaJ!l&k4 z&ps3bp6!ep``6*Uf_ya6t#rGMcw8b(B8UXecd;RN4haDB*BIoQ`V0jsSZY`0vwwe* z{;l#lDMIS!Me#<{P`T6Y=lf^tuBo_)?eE8wr&2GTh!R@sRyD3W`J zazH1`-XjOo9eZSBy?0F`W`2BpVYp2~3x5*a$jvGbDoEU9`DnXUh{s?~GI{E8-@*K0 zVCyr;;If7!#;5@q9AtsWJn^1?O8SQDNB+}_?@XBvE)`>xBAkWXGH?MWrU2j%bAet1 z@&5q9+P}h&8Q*w2U4m$DEhH%MFbvyDhDHQ@xGG0XjxpA~OdVA^X+6$Lu(P@3*X$DD zO7JOmeSr%xU;ycoG3)*VAXHZp-z`{(BP%+O@@6eOzWYjjv!pFa2fxVj~ zl5vclx##rs=9=8c6u)YD$zU?e)rLqOyc`UFj%(0N$8y{gi0~%_;0!J}Z1=|;=dtc{ z&3z~T00i>=oIG>;K>UBv^iK}yO(vUc!hLGxqaIQ~j0rEESZ)XvIy+!7Ay`+0{21_& z@%O|{J4A-eJP&dvx{SrWnlvD{2aIjWB=9{BKtH@+v2X0};opEh1k-elBg2Vi9@05A z6SouGNwfr64l*!FY;m_Z$2dHAyh@=Iy0dCD{iKhlJ{Rd$-WAeex3^^$`hAkdySWOA z1~N|swh2DD9<|UzdnTi&nQonmNh+B*;HvZ)9fk)T&*fZ1dd1ATc(i5mt=d3kR2X8Q z@Stam2L$IF`Rt?zV^INBkUL zX=`z>_&dZYZ}Q1=JX7(KnNUF8oN~OmHZOz=l5sMRyd0=#JkmvdP{y6Fu?G?-@@RcPO0fyc}=bn1^`sdcY zj2zUdTg>9Fnw`qF&6zi35{ga_89Wcik6xA5+%KP_Lvar0W(pOQ5xD2C&}WaOWAf&l#Mr2U@$I%?mu7sn3|{A$tB^^;o=lXJ1mN(eBLE+u=z5d&sy-yhog!CqS%=EqNX~L`j!t|304!6CoL$h` zO&_y=wCC*g@b~uO{hdyLb*ppsn|Sqa1o)$FR@QfjJTvjhhFIBFRUJ|D_mOe=$MCP= zH-$fE{{Vtt6})ZxT7Jx5+P793&6kEeFq*cfEDGAnOw&(o5RVMdszj1YbsK!p<`qA` zEWby8!5)8W7x7oayZ->$U&Y#(ZA-$dCEtZvn3Hj(v+kPQXQ{W9_bC4Wo>Sm)5ywye z00naJKCk2d0NG*n?IB)Y_(ZkaZC>I9$tKfemh-S4h$dMNqN^U&>B&Z|KWP({Hy6s! z&)?cJ_Pg=-?NR>#2&MwC)(M zM);TD-+*5YKWv|dx-ab4EvwnyK(K1bW2&{}a;(Nk(pGZqi6w!HvVsQ++gCWJc+=vy z?CEv?00i>TeivvbP5%Idr+gygOpe@rwk=U^w_IOJ+jFs^1yze-mDK?GNL-JC{{RAq z!2bXPSpM3cG@30}NU+iqP|-9kCO1olx}A->8QIrlQO52`buE%rvDo16eRbKD(DFap z+rh8kkBu|l`Ao3iD#%!Nalt>32lK8o#Z7G{m1`stZu24}k_hfb4|B-QewFWEwf>`_ zd}R1rp!{j@-Hev--$LSBc}Op9JZx4-i2w#Fz<@v*QVs@iK6lhV(mXY5H=UikhTLER z81O$1e_F+eYp3`< z;CsJ_-xBnH2wGb-pKHBADzE(W2Xi0$wwZRkag)msMnP;4J7Io?oNdgmC$aT*iEC-${{Ro#=@wD# zirU>{Obl=r1NnZHC9TE7NX;wd2qQS@P}DWAVP3r%4bNk?AdSBbHy3 z`Ncsw=mrlw_s7kO@~c1E3r%kT&8QZh9}-w!o4XB4(lzq#5?hVe5YEg;?u=nD03&j# zz$=>k{{V<-&0^h_^1uE|75X#Q%kc_=yZ*o6_C3SK+V+!e;X6r5U5fFAVBoPndHHeF z@z=N&#QZMM=Y1GN@#aZ&3!+yY+U7%n_z_-D;!llt+O)T#JBTMsq>g2YEsSdvm@48R z#O*s+FW~42Gj$mO2pN)TwHDmft-T=uJ?t1%%u z9+mbfT30;YmN@&pEL+5vL&!~zujX72mOOQAdV$b-;=3fYN$umhT%GZQmK>fsk=UNU zjZ(eR1iD%M)J3DishE7Bl!6KRV>P*hUNb_H-jQ#9(&r$7;*sf_O99OC? zH{>ubZ3uEUoYZY=v|x{?*0t_!yvu^ydVZ9DWKB3L=0>FC0MCNNbx=KfA9Fjon0Oa$IpPhQQ>`(CP#-Fqo?H#7i;eP_Iy=!Hx!bPU56yMqD zZEn~PbtENLD`jw9zG1lG2I9p}8)e{IYryGqs?RUlK?;^3lW@j1Z`w)|+%oaoIOE-Q zZ`;G+hwR<^J89pu=j_4Y2rWEQccsB?;%K!$4BE>S*8yEs=b5K+-d~tPqaiG$oFQUs z%daU)vy${jT|QL{A7TF1zqEd_;d?1Q82B3dSeL|?_c05dYe|;Q2=zObm6f3p+wYM= zkil0mvZ={p%q!AAX{{FT!k+>B1^X;`e&9Z<;$34;)4aAgS$wBi?hBGh+Gljh$vEqt zYv7OAlf(_+?~C3U_^Ga1+Fng#tU)i9_UaJjf-@ox7a1<2_*g0AjMwRJ#r4zmkJ`84 zeuH7BrPiaRT5Gz@m;{?>1PN~olhc_=13g&@?OM^K;~T!Cl=+#{d|jIF!oCzX*At|> z?nsC)@JEx3p8QwmfBY2h_J$r1@C3dM(XUtSnv0Yeag|<&ABo4{YwmyApT$4$k9c0% z*6?|AvNO85=X!bz4i36a{mUlb$+rS^_^KaNU5t@!Pd@P>STa z!%_RhcM=GdrdSb#AQs6ByC)k-&jT4b>N?|~s4cb0TGHxmHEdauHHs-ha8;cdROgaG zBq`{-xHtlm?kR5VcCDF}JoT1V!D5?1*gyw!HtYpe=XMC`SY8*@;J?z^^48{HkwDL< ztdW(JH<^V;9Do&oBo4blA2H+dt~acwnX5mBEAegqO&xe^#s@=gELz?@I^CTvtu9t1 zGB(}9B*wsgV4!c1kO?HV?B@owbZZpQG~FuF>gL!Bo3VYYTq`Q7BeO_|NysB`Md`Rk zt?S1j;=NkiO4DwxJUwudNRT4i+D7dmd)HOmq;tP&hjtvO$OkMiEO>ukxf=6ovxiMN zW`*u8B!C8<;l5uifw-!(1P2(xh6g<3$&ECyC+<80w~7t4aG1~#5P zZZ>scd9G>AtxKv#_g0tp5j+#ZRzVYnF=<(E{cLF&E~w}HUE)m^-&>O2Uk2IVssJLM zTVl)<@>HFqoH1d^By`UdS7G6sZ8yaBGF@u+=ECL+hK5zOf0j}(!TM)7V5dC%o|vMr zRe4pl``HNa9};*AOnXlf&8F+qFPpk}u5EWKN>D0<8gT zAZ0+?(~_X_#GKYY#QF7|CvD}8m96A6e)CV&Q&nYYj_r>-mQk4qWEf=t@r;p??ZQ>7 z+6jJNefxi12caxFzNp%b!XGZ;Kb_^1PuwIbNXir?NFaskrAf);WYauhr%!s4+Uh!m z+qInbFL2Vzzz^J&Ry+k6Vxfm(++gui=pHe(w9=DN)1C{c8ZS2H?)mnr%;*&eB!a*o zAgZ2ECp(UB+}-IuAWKgX-&|bkkiMgFJh59{h*#|aS&`VX`K4oGfbRi$01ifUr7Eh? zwXE&7zt?TM`5GwR$ryUihxD%u30ucHADHk&*FR{tc@lICTtv)4BMcOrZ6u6m9Gafj z#52zwmX+aMI^?FWG%~?&rf!m0!zn}rV3sEaO``{H=57sAj#%~RH1@n#iYB&4n$lR7 zXx9yxLNdPRET;tt4VD0&NUrxv(l2z2ElS@`@jP+OX=nCJ`Qx!mes^vVFa%*0K~_;N zPD-;9)J_qSy-T-!y*~cB?nOwhZ4PGs;tM;ge-QYSLv)(OWw*UaBmV$j248QKS0^Pg zg8UW*hxbow@FT&K`1i(Ex*n&Zf5JUy4T8^o9P$}ubwC|q4gm^^q?S26HZWMItjN4E zV;_fR(vs%ld6!#+9a-!HBzCCI%_=)%W8}rO9#xR-<2CJnv*&^|kA>Q{rST`nn!VN1 z-03SS$0W<5@Iy{le$s(rUtkEpGp+Dh0+ zzCpRM(Q*JI9N_veua`V2`$u?}Uhs#8g|3|jpBAIy8LgTNP}^Y}Rd$9v%O)6;BWIFA zjNn()9vL1S@THZ$lW7Y_YiDq$%P04@E_a1zIL=rua&kb%Fc4#RNA$X>fR}P zd3-uGF(EGTvMh^&0T@PXdW;d1j!DIQWBXiM*neZ)7%kJN>V&FSgv;Lw_p5u$Oi}DUj}D z89m1Dm=9s;itbIy352bE501Pwp`A;`QRz`iA#16lj5CvjEXu0K9fmKSmbK-Tk!%C9RFht58AGOFeov!g1n1OX)-;ys2uy_nMj=Vn*(IG0&&QKk!b^guW2?Vep-F?+VWq&Zj)d z=SeVS+Zr*+$>moqg-@66le7@TeP7{wdmS@SxYcz%R9sy;sES!w5P*OVz%mpbilB5I zI5ptr6=znyn24;P#JBDu1jtt>ImsZl6cT&mfs9vCr)t`F*k)Up zon2D_V##gnGQ{>I00*scPosDa1K--Oma_72V+_u7uM|VtbE7R zVo>suRxVmkQJ+qkJY@d3)00lduEk%9KL&JPh~5>t@cyB0ml4et`C+8#Gb>D2j$YZui6v#efUTFMf@VY_;urt3t8RT$rs7y;JQnLwUmLC zBZki$XCAfLMzp9@=0r*gq%40k=E0ioPbA_l-!g(ueNRD;>0JekO$DFt8))0#B$eyC zj+x`~75YQsZ}=cIR^w2U!yggaX`1FY(xTKMK2V1^WoBQQm<$j`00i)G4Sc)%cmBp# zKe8u_G(9dYZcErTTUS}b!?cM}P?k_wu?IiI1_-ZCo;wd(Y7u&zj}KM~?_#*50*07#s&}OE2x`jYy3I*qvHPng5MIfUyB|A)V|NF z=r_xEX)MQbN$s?(B2^ub#{;Zb!=YXFX_NQ?~fC{{RH-weTm2FFaU& zE@=7>!?v~=R8H0(M;O6g{~V`w6%?o z@6i#G;yt5=e<-sl>KQN(%yaX1!5`UQz+beVz&Wq{39Sty$2u;dExp=aUo@7Qq)xtc zEK`<*n8{u7f}q4pDGiaWMaerw8AaYVWoGcdg?wx9E8;$p@ZkiS9+UA8P;U6eyiI}4~{zk|w` z$&i+hDalWl2MvY<;eaCp8ElGMoHIHWEP#bkg&)?T11A-GPtxvn8>qC{5VVs>6;)J{ zN%j@oc*o#&sqpVZU2nuLRLQhJgB+4G&!-t5`M@1mbY6Ou zsdbNtdY8ppDE0gKQrz6YL^jJKG*+RG2^l1Ua2bgE$It*OiI-23Sjq+63~YhX$pkK0 zdi~?Ll5x~GKHn*iuPja>%1dTF5=m@`;N5>SEMt0XD4Wp z5DpD|L#}vMSZ0;_D0ZM>C9X1Yjv6D!_AsYv^GXb7f_3quR#8LQYJV3>1Q{@5*h+Awu#%$;Dc4hF%`FzI{q9 zL2cRiacwNI2Rsr=D8_OR&5Vx3Ceeu@J;X8!$|S}0K^NAh+ZwxBD&IiJgMZv za+yhQoP!=mM)K4d2s~t_L9e=ed9Fp_{RVj&OkqhZPqlS&r0v+K?0hZ%00m$8{SSjY zEva4k#Uzpzf&;te=R6O7aroEdDoZu{TO3h9}WvmxCX`oq643s=9h5bgI|c}Sjb}YI<>ktrn1DvCCk}DoXl=yZ_)xDp3r`atU6=1g`02YYlih!<0IR%)q z{Mp1;dY$%#WSW7wk~^4T0d18UHHDgDV<3M9(~uh+`M-MrR}9?web@g04vW3DJ9FZQ zwAmxKy-CVGqD!zD5uLI|RI&r+GjC0*K-#A{`Egh}?z833dfIlZ8WfYrgt``#hH^2T zxMgM}DLi0zbOwB5>(}LpW*3{XsjEVbxT{x-L;!bb(&_9IIph*I8t$t zN##xn9+}5bXsr34^uOSf%#SCVPST~-FE8z25B6})b9E_U=CZ$*xLlH5o>e5V+BrBq ztB(cvmiFnqb9;9jt>NuGEvECfxOq&9U(8kq%nrC>bNpLQalu7;k*iXTxnK2WY?5-Z z&ujh`yq8P5zVOZRMQe4NqG8NH$k-90pcn`XkCgBN^(65|R`HgJtJ!!i(#+i0TE!K( zlG&E1tbc#*>TueC8zOx5=;!zE0-DADbq>~Vjxx9p2`;|~aGA0GACgq9W;a#?BC z!ek$6ybId(jyN3nUyS?-pkMq~y1Ma5`#V}*0B@}!Up4`}C|L2(oB%rx zq@Lv!ue6*YCZ58n73|lU^dIc);%^1~HSoyLyeR~;#{?f}aOHxuvLD^94mW(8Sdsu7 z@K~Dk{{V;o02=SLty}EuC03W_d0YmsTZ2D#+lh4Cj^o662%7?_2a0>@OQ_%D{XS}-L9@m+OQJ4L^2o?#(5wd z90ddncH<=#lhNGP=cQ<06s>hTRKAY-S)D;BlSWr|ah#Eya?6v~u+Jv0L-9gGcr0wL zL&I(fx{$<4Vn#x!Y<3yR>F?5tD!JvLE$mpd_^i{+EoYBqxniju z>fSL8zzUelg>01s9DPnY)(q;tdBkcyZV&{0pr4xxjuWJ13zTlW58dw{=cmFdr(^+ z4ES7qw$vjlHN+Z>GI=&zda8lv24Xl^7aXv`MS3Z@(~Eu1J*4b?KdktA(@_yy%WRJE zTqKZgA97HBM+dP4z4`d`BT0$qRv#flZ={^oR%-2fZ|&IxaPanz!| za}^a%q?+p4C#HVOo)FOddHXY4W63(Loj1${6{byUt zWu$2{+(B-rKGMjDqmP#gKMVjr3Mj8vDT-Kj{K78gJ-{~+OMO4g!#jg>`B$9$R?}me z>dNiaaM7~31E}l$XrjGB(dT~@w5#~%rjcy1<^RjjvvC1ZQ zDB6Rr1#_M&{{Vt|X;<0`>M?lQT{`6<0j<8qvqvM~3@HqyM+dHGqPcM1@@@AvtgO;U zhI~o=jJ#vvQ*zc9_Sf;+#L`bDM0-F0oCab~72#Lj99E3sfdqo2_7qWFiCpBChGOV( zqizIs>HdDT@87b=!Ch;xog8@s67gzehEox>+^qKeiMlaBkamd%uPTc^63^G>i+rb?F)$9n`gDCK}F zDz7RC40eIH7(zfht11y`sA@CY#!QjRI;2s%5#@zqRgV}YRd*Z?OJH}Rio#NG)e!e4 xZEC|%V{v=pdpPCK&mN_3a9s~v0}x Date: Fri, 1 Dec 2017 16:51:13 +0800 Subject: [PATCH 373/823] Add more image transformers (#1965) * Add OpenCV Vision Transformer * Add more transformers * update image * Add more transformers * Add more test * Refine documents * update * Add visualize test * Make BoundingBox Serializable * Fix ut * Add more python test * remove head from imageframe --- .../dllib/feature/transform/vision/image.py | 300 ++++++++++++++++++ 1 file changed, 300 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py b/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py index 48111234c60..81f76e00c24 100644 --- a/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py +++ b/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py @@ -262,5 +262,305 @@ class HFlip(FeatureTransformer): """ Flip the image horizontally """ + def __init__(self, bigdl_type="float"): super(HFlip, self).__init__(bigdl_type) + +class Resize(FeatureTransformer): + """ + Resize image + :param resize_h height after resize + :param resize_w width after resize + :param resize_mode if resizeMode = -1, random select a mode from (Imgproc.INTER_LINEAR, + Imgproc.INTER_CUBIC, Imgproc.INTER_AREA, Imgproc.INTER_NEAREST, Imgproc.INTER_LANCZOS4) + """ + + def __init__(self, resize_h, resize_w, resize_mode = 1, bigdl_type="float"): + super(Resize, self).__init__(bigdl_type, resize_h, resize_w, resize_mode) + +class Brightness(FeatureTransformer): + """ + adjust the image brightness + :param deltaLow brightness parameter: low bound + :param deltaHigh brightness parameter: high bound + """ + + def __init__(self, delta_low, delta_high, bigdl_type="float"): + super(Brightness, self).__init__(bigdl_type, delta_low, delta_high) + +class ChannelOrder(FeatureTransformer): + """ + random change the channel of an image + """ + + def __init__(self, bigdl_type="float"): + super(ChannelOrder, self).__init__(bigdl_type) + +class Contrast(FeatureTransformer): + """ + Adjust the image contrast + :param delta_low contrast parameter low bound + :param delta_high contrast parameter high bound + """ + + def __init__(self, delta_low, delta_high, bigdl_type="float"): + super(Contrast, self).__init__(bigdl_type, delta_low, delta_high) + +class Saturation(FeatureTransformer): + """ + Adjust image saturation + """ + + def __init__(self, delta_low, delta_high, bigdl_type="float"): + super(Saturation, self).__init__(bigdl_type, delta_low, delta_high) + +class Hue(FeatureTransformer): + """ + Adjust image hue + :param delta_low hue parameter: low bound + :param delta_high hue parameter: high bound + """ + + def __init__(self, delta_low, delta_high, bigdl_type="float"): + super(Hue, self).__init__(bigdl_type, delta_low, delta_high) + +class ChannelNormalize(FeatureTransformer): + """ + image channel normalize + :param mean_r mean value in R channel + :param mean_g mean value in G channel + :param meanB_b mean value in B channel + :param std_r std value in R channel + :param std_g std value in G channel + :param std_b std value in B channel + """ + def __init__(self, mean_r, mean_b, mean_g, std_r=1.0, std_g=1.0, std_b=1.0, bigdl_type="float"): + super(ChannelNormalize, self).__init__(bigdl_type, mean_r, mean_g, mean_b, std_r, std_g, std_b) + +class PixelNormalize(FeatureTransformer): + """ + Pixel level normalizer, data(i) = data(i) - mean(i) + + :param means pixel level mean, following H * W * C order + """ + + def __init__(self, means, bigdl_type="float"): + super(PixelNormalize, self).__init__(bigdl_type, means) + + +class RandomCrop(FeatureTransformer): + """ + Random crop a `cropWidth` x `cropHeight` patch from an image. + The patch size should be less than the image size. + + :param crop_width width after crop + :param crop_height height after crop + :param is_clip whether to clip the roi to image boundaries + """ + + def __init__(self, crop_width, crop_height, is_clip=True, bigdl_type="float"): + super(RandomCrop, self).__init__(bigdl_type, crop_width, crop_height, is_clip) + +class CenterCrop(FeatureTransformer): + """ + Crop a `cropWidth` x `cropHeight` patch from center of image. + The patch size should be less than the image size. + :param crop_width width after crop + :param crop_height height after crop + :param is_clip clip cropping box boundary + """ + + def __init__(self, crop_width, crop_height, is_clip=True, bigdl_type="float"): + super(CenterCrop, self).__init__(bigdl_type, crop_width, crop_height, is_clip) + +class FixedCrop(FeatureTransformer): + """ + Crop a fixed area of image + + :param x1 start in width + :param y1 start in height + :param x2 end in width + :param y2 end in height + :param normalized whether args are normalized, i.e. in range [0, 1] + :param is_clip whether to clip the roi to image boundaries + """ + + def __init__(self, x1, y1, x2, y2, normalized=True, is_clip=True, bigdl_type="float"): + super(FixedCrop, self).__init__(bigdl_type, x1, y1, x2, y2, normalized, is_clip) + +class DetectionCrop(FeatureTransformer): + """ + Crop from object detections, each image should has a tensor detection, + which is stored in ImageFeature + :param roi_key key that map a tensor detection + :param normalized whether is detection is normalized, i.e. in range [0, 1] + """ + + def __init__(self, roi_key, normalized=True, bigdl_type="float"): + super(DetectionCrop, self).__init__(bigdl_type, roi_key, normalized) + + +class Expand(FeatureTransformer): + """ + expand image, fill the blank part with the meanR, meanG, meanB + + :param means_r means in R channel + :param means_g means in G channel + :param means_b means in B channel + :param min_expand_ratio min expand ratio + :param max_expand_ratio max expand ratio + """ + + def __init__(self, means_r=123, means_g=117, means_b=104, + min_expand_ratio=1.0, + max_expand_ratio=4.0, bigdl_type="float"): + super(Expand, self).__init__(bigdl_type, means_r, means_g, means_b, + min_expand_ratio, max_expand_ratio) + +class Filler(FeatureTransformer): + """ + Fill part of image with certain pixel value + :param start_x start x ratio + :param start_y start y ratio + :param end_x end x ratio + :param end_y end y ratio + :param value filling value + """ + + def __init__(self, start_x, start_y, end_x, end_y, value = 255, bigdl_type="float"): + super(Filler, self).__init__(bigdl_type, start_x, + start_y, + end_x, + end_y, + value) + +class RandomTransformer(FeatureTransformer): + """ + It is a wrapper for transformers to control the transform probability + :param transformer transformer to apply randomness + :param prob max prob + """ + + def __init__(self, transformer, prob, bigdl_type="float"): + super(RandomTransformer, self).__init__(bigdl_type, transformer, prob) + + +class ColorJitter(FeatureTransformer): + """ + Random adjust brightness, contrast, hue, saturation + :param brightness_prob probability to adjust brightness + :param brightness_delta brightness parameter + :param contrast_prob probability to adjust contrast + :param contrast_lower contrast lower parameter + :param contrast_upper contrast upper parameter + :param hue_prob probability to adjust hue + :param hue_delta hue parameter + :param saturation_prob probability to adjust saturation + :param saturation_lower saturation lower parameter + :param saturation_upper saturation upper parameter + :param random_order_prob random order for different operation + :param shuffle shuffle the transformers + """ + def __init__(self, brightness_prob = 0.5, + brightness_delta = 32.0, + contrast_prob = 0.5, + contrast_lower = 0.5, + contrast_upper = 1.5, + hue_prob = 0.5, + hue_delta = 18.0, + saturation_prob = 0.5, + saturation_lower = 0.5, + saturation_upper = 1.5, + random_order_prob = 0.0, + shuffle = False, + bigdl_type="float"): + super(ColorJitter, self).__init__(bigdl_type, brightness_prob, + brightness_delta, + contrast_prob, + contrast_lower, + contrast_upper, + hue_prob, + hue_delta, + saturation_prob, + saturation_lower, + saturation_upper, + random_order_prob, + shuffle) + +class RandomSampler(FeatureTransformer): + """ + Random sample a bounding box given some constraints and crop the image + This is used in SSD training augmentation + """ + + def __init__(self): + super(RandomSampler, self).__init__(bigdl_type) + +class RoiProject(FeatureTransformer): + """ + Project gt boxes onto the coordinate system defined by image boundary + :param need_meet_center_constraint whether need to meet center constraint, i.e., the center of gt box need be within image boundary + """ + + def __init__(self, need_meet_center_constraint, bigdl_type="float"): + super(RoiProject, self).__init__(bigdl_type, need_meet_center_constraint) + +class RoiHFlip(FeatureTransformer): + """ + horizontally flip the roi + :param normalized whether the roi is normalized, i.e. in range [0, 1] + """ + + def __init__(self, normalized=True, bigdl_type="float"): + super(RoiHFlip, self).__init__(bigdl_type, normalized) + +class RoiResize(FeatureTransformer): + """ + resize the roi according to scale + :param normalized whether the roi is normalized, i.e. in range [0, 1] + """ + def __init__(self, normalized=True, bigdl_type="float"): + super(RoiResize, self).__init__(bigdl_type, normalized) + +class RoiNormalize(FeatureTransformer): + """ + Normalize Roi to [0, 1] + """ + + def __init__(self, bigdl_type="float"): + super(RoiNormalize, self).__init__(bigdl_type) + +class MatToFloats(FeatureTransformer): + + def __init__(self, valid_height=300, valid_width=300, valid_channel=300, + mean_r=-1.0, mean_g=-1.0, mean_b=-1.0, out_key = "floats", bigdl_type="float"): + super(MatToFloats, self).__init__(bigdl_type, valid_height, valid_width, valid_channel, + mean_r, mean_g, mean_b, out_key) +class AspectScale(FeatureTransformer): + """ + Resize the image, keep the aspect ratio. scale according to the short edge + :param scale scale size, apply to short edge + :param scale_multiple_of make the scaled size multiple of some value + :param max_size max size after scale + """ + + def __init__(self, scale, scale_multiple_of = 1, max_size = 1000, bigdl_type="float"): + super(AspectScale, self).__init__(bigdl_type, scale, scale_multiple_of, max_size) + +class RandomAspectScale(FeatureTransformer): + """ + resize the image by randomly choosing a scale + :param scales array of scale options that for random choice + :param scaleMultipleOf Resize test images so that its width and height are multiples of + :param maxSize Max pixel size of the longest side of a scaled input image + """ + def __init__(self, scales, scale_multiple_of = 1, max_size = 1000, bigdl_type="float"): + super(RandomAspectScale, self).__init__(bigdl_type, scales, scale_multiple_of, max_size) + +class BytesToMat(FeatureTransformer): + """ + Transform byte array(original image file in byte) to OpenCVMat + """ + def __init__(self, bigdl_type="float"): + super(BytesToMat, self).__init__(bigdl_type) + From a622a27bfe609e637f85e950ae786b6d67c3539a Mon Sep 17 00:00:00 2001 From: Xianyan Date: Fri, 1 Dec 2017 16:51:13 +0800 Subject: [PATCH 374/823] Add more image transformers (#1965) * Add OpenCV Vision Transformer * Add more transformers * update image * Add more transformers * Add more test * Refine documents * update * Add visualize test * Make BoundingBox Serializable * Fix ut * Add more python test * remove head from imageframe --- .../dllib/feature/transform/vision/image.py | 300 ++++++++++++++++++ 1 file changed, 300 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py b/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py index 48111234c60..81f76e00c24 100644 --- a/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py +++ b/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py @@ -262,5 +262,305 @@ class HFlip(FeatureTransformer): """ Flip the image horizontally """ + def __init__(self, bigdl_type="float"): super(HFlip, self).__init__(bigdl_type) + +class Resize(FeatureTransformer): + """ + Resize image + :param resize_h height after resize + :param resize_w width after resize + :param resize_mode if resizeMode = -1, random select a mode from (Imgproc.INTER_LINEAR, + Imgproc.INTER_CUBIC, Imgproc.INTER_AREA, Imgproc.INTER_NEAREST, Imgproc.INTER_LANCZOS4) + """ + + def __init__(self, resize_h, resize_w, resize_mode = 1, bigdl_type="float"): + super(Resize, self).__init__(bigdl_type, resize_h, resize_w, resize_mode) + +class Brightness(FeatureTransformer): + """ + adjust the image brightness + :param deltaLow brightness parameter: low bound + :param deltaHigh brightness parameter: high bound + """ + + def __init__(self, delta_low, delta_high, bigdl_type="float"): + super(Brightness, self).__init__(bigdl_type, delta_low, delta_high) + +class ChannelOrder(FeatureTransformer): + """ + random change the channel of an image + """ + + def __init__(self, bigdl_type="float"): + super(ChannelOrder, self).__init__(bigdl_type) + +class Contrast(FeatureTransformer): + """ + Adjust the image contrast + :param delta_low contrast parameter low bound + :param delta_high contrast parameter high bound + """ + + def __init__(self, delta_low, delta_high, bigdl_type="float"): + super(Contrast, self).__init__(bigdl_type, delta_low, delta_high) + +class Saturation(FeatureTransformer): + """ + Adjust image saturation + """ + + def __init__(self, delta_low, delta_high, bigdl_type="float"): + super(Saturation, self).__init__(bigdl_type, delta_low, delta_high) + +class Hue(FeatureTransformer): + """ + Adjust image hue + :param delta_low hue parameter: low bound + :param delta_high hue parameter: high bound + """ + + def __init__(self, delta_low, delta_high, bigdl_type="float"): + super(Hue, self).__init__(bigdl_type, delta_low, delta_high) + +class ChannelNormalize(FeatureTransformer): + """ + image channel normalize + :param mean_r mean value in R channel + :param mean_g mean value in G channel + :param meanB_b mean value in B channel + :param std_r std value in R channel + :param std_g std value in G channel + :param std_b std value in B channel + """ + def __init__(self, mean_r, mean_b, mean_g, std_r=1.0, std_g=1.0, std_b=1.0, bigdl_type="float"): + super(ChannelNormalize, self).__init__(bigdl_type, mean_r, mean_g, mean_b, std_r, std_g, std_b) + +class PixelNormalize(FeatureTransformer): + """ + Pixel level normalizer, data(i) = data(i) - mean(i) + + :param means pixel level mean, following H * W * C order + """ + + def __init__(self, means, bigdl_type="float"): + super(PixelNormalize, self).__init__(bigdl_type, means) + + +class RandomCrop(FeatureTransformer): + """ + Random crop a `cropWidth` x `cropHeight` patch from an image. + The patch size should be less than the image size. + + :param crop_width width after crop + :param crop_height height after crop + :param is_clip whether to clip the roi to image boundaries + """ + + def __init__(self, crop_width, crop_height, is_clip=True, bigdl_type="float"): + super(RandomCrop, self).__init__(bigdl_type, crop_width, crop_height, is_clip) + +class CenterCrop(FeatureTransformer): + """ + Crop a `cropWidth` x `cropHeight` patch from center of image. + The patch size should be less than the image size. + :param crop_width width after crop + :param crop_height height after crop + :param is_clip clip cropping box boundary + """ + + def __init__(self, crop_width, crop_height, is_clip=True, bigdl_type="float"): + super(CenterCrop, self).__init__(bigdl_type, crop_width, crop_height, is_clip) + +class FixedCrop(FeatureTransformer): + """ + Crop a fixed area of image + + :param x1 start in width + :param y1 start in height + :param x2 end in width + :param y2 end in height + :param normalized whether args are normalized, i.e. in range [0, 1] + :param is_clip whether to clip the roi to image boundaries + """ + + def __init__(self, x1, y1, x2, y2, normalized=True, is_clip=True, bigdl_type="float"): + super(FixedCrop, self).__init__(bigdl_type, x1, y1, x2, y2, normalized, is_clip) + +class DetectionCrop(FeatureTransformer): + """ + Crop from object detections, each image should has a tensor detection, + which is stored in ImageFeature + :param roi_key key that map a tensor detection + :param normalized whether is detection is normalized, i.e. in range [0, 1] + """ + + def __init__(self, roi_key, normalized=True, bigdl_type="float"): + super(DetectionCrop, self).__init__(bigdl_type, roi_key, normalized) + + +class Expand(FeatureTransformer): + """ + expand image, fill the blank part with the meanR, meanG, meanB + + :param means_r means in R channel + :param means_g means in G channel + :param means_b means in B channel + :param min_expand_ratio min expand ratio + :param max_expand_ratio max expand ratio + """ + + def __init__(self, means_r=123, means_g=117, means_b=104, + min_expand_ratio=1.0, + max_expand_ratio=4.0, bigdl_type="float"): + super(Expand, self).__init__(bigdl_type, means_r, means_g, means_b, + min_expand_ratio, max_expand_ratio) + +class Filler(FeatureTransformer): + """ + Fill part of image with certain pixel value + :param start_x start x ratio + :param start_y start y ratio + :param end_x end x ratio + :param end_y end y ratio + :param value filling value + """ + + def __init__(self, start_x, start_y, end_x, end_y, value = 255, bigdl_type="float"): + super(Filler, self).__init__(bigdl_type, start_x, + start_y, + end_x, + end_y, + value) + +class RandomTransformer(FeatureTransformer): + """ + It is a wrapper for transformers to control the transform probability + :param transformer transformer to apply randomness + :param prob max prob + """ + + def __init__(self, transformer, prob, bigdl_type="float"): + super(RandomTransformer, self).__init__(bigdl_type, transformer, prob) + + +class ColorJitter(FeatureTransformer): + """ + Random adjust brightness, contrast, hue, saturation + :param brightness_prob probability to adjust brightness + :param brightness_delta brightness parameter + :param contrast_prob probability to adjust contrast + :param contrast_lower contrast lower parameter + :param contrast_upper contrast upper parameter + :param hue_prob probability to adjust hue + :param hue_delta hue parameter + :param saturation_prob probability to adjust saturation + :param saturation_lower saturation lower parameter + :param saturation_upper saturation upper parameter + :param random_order_prob random order for different operation + :param shuffle shuffle the transformers + """ + def __init__(self, brightness_prob = 0.5, + brightness_delta = 32.0, + contrast_prob = 0.5, + contrast_lower = 0.5, + contrast_upper = 1.5, + hue_prob = 0.5, + hue_delta = 18.0, + saturation_prob = 0.5, + saturation_lower = 0.5, + saturation_upper = 1.5, + random_order_prob = 0.0, + shuffle = False, + bigdl_type="float"): + super(ColorJitter, self).__init__(bigdl_type, brightness_prob, + brightness_delta, + contrast_prob, + contrast_lower, + contrast_upper, + hue_prob, + hue_delta, + saturation_prob, + saturation_lower, + saturation_upper, + random_order_prob, + shuffle) + +class RandomSampler(FeatureTransformer): + """ + Random sample a bounding box given some constraints and crop the image + This is used in SSD training augmentation + """ + + def __init__(self): + super(RandomSampler, self).__init__(bigdl_type) + +class RoiProject(FeatureTransformer): + """ + Project gt boxes onto the coordinate system defined by image boundary + :param need_meet_center_constraint whether need to meet center constraint, i.e., the center of gt box need be within image boundary + """ + + def __init__(self, need_meet_center_constraint, bigdl_type="float"): + super(RoiProject, self).__init__(bigdl_type, need_meet_center_constraint) + +class RoiHFlip(FeatureTransformer): + """ + horizontally flip the roi + :param normalized whether the roi is normalized, i.e. in range [0, 1] + """ + + def __init__(self, normalized=True, bigdl_type="float"): + super(RoiHFlip, self).__init__(bigdl_type, normalized) + +class RoiResize(FeatureTransformer): + """ + resize the roi according to scale + :param normalized whether the roi is normalized, i.e. in range [0, 1] + """ + def __init__(self, normalized=True, bigdl_type="float"): + super(RoiResize, self).__init__(bigdl_type, normalized) + +class RoiNormalize(FeatureTransformer): + """ + Normalize Roi to [0, 1] + """ + + def __init__(self, bigdl_type="float"): + super(RoiNormalize, self).__init__(bigdl_type) + +class MatToFloats(FeatureTransformer): + + def __init__(self, valid_height=300, valid_width=300, valid_channel=300, + mean_r=-1.0, mean_g=-1.0, mean_b=-1.0, out_key = "floats", bigdl_type="float"): + super(MatToFloats, self).__init__(bigdl_type, valid_height, valid_width, valid_channel, + mean_r, mean_g, mean_b, out_key) +class AspectScale(FeatureTransformer): + """ + Resize the image, keep the aspect ratio. scale according to the short edge + :param scale scale size, apply to short edge + :param scale_multiple_of make the scaled size multiple of some value + :param max_size max size after scale + """ + + def __init__(self, scale, scale_multiple_of = 1, max_size = 1000, bigdl_type="float"): + super(AspectScale, self).__init__(bigdl_type, scale, scale_multiple_of, max_size) + +class RandomAspectScale(FeatureTransformer): + """ + resize the image by randomly choosing a scale + :param scales array of scale options that for random choice + :param scaleMultipleOf Resize test images so that its width and height are multiples of + :param maxSize Max pixel size of the longest side of a scaled input image + """ + def __init__(self, scales, scale_multiple_of = 1, max_size = 1000, bigdl_type="float"): + super(RandomAspectScale, self).__init__(bigdl_type, scales, scale_multiple_of, max_size) + +class BytesToMat(FeatureTransformer): + """ + Transform byte array(original image file in byte) to OpenCVMat + """ + def __init__(self, bigdl_type="float"): + super(BytesToMat, self).__init__(bigdl_type) + From a4197b9e5805e597f4f5ab883bfc0f4a4d217bc9 Mon Sep 17 00:00:00 2001 From: Xianyan Date: Fri, 1 Dec 2017 16:51:13 +0800 Subject: [PATCH 375/823] Add more image transformers (#1965) * Add OpenCV Vision Transformer * Add more transformers * update image * Add more transformers * Add more test * Refine documents * update * Add visualize test * Make BoundingBox Serializable * Fix ut * Add more python test * remove head from imageframe --- .../dllib/test/bigdl/transform/test_image.py | 78 +++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/python/dllib/test/bigdl/transform/test_image.py b/python/dllib/test/bigdl/transform/test_image.py index 29011603278..bb18cc9af4f 100644 --- a/python/dllib/test/bigdl/transform/test_image.py +++ b/python/dllib/test/bigdl/transform/test_image.py @@ -78,5 +78,83 @@ def test_hflip(self): transformer = HFlip() self.transformer_test(transformer) + def test_colorjitter(self): + color = ColorJitter(random_order_prob=1.0, shuffle=True) + self.transformer_test(color) + + def test_resize(self): + resize = Resize(200, 200, 1) + self.transformer_test(resize) + + def test_brightness(self): + brightness = Brightness(0.0, 32.0) + self.transformer_test(brightness) + + def test_channel_order(self): + transformer = ChannelOrder() + self.transformer_test(transformer) + + def test_aspect_scale(self): + transformer = AspectScale(300) + self.transformer_test(transformer) + + def test_random_aspect_scale(self): + transformer = RandomAspectScale([300, 400]) + self.transformer_test(transformer) + + def test_contrast(self): + transformer = Contrast(0.5, 1.5) + self.transformer_test(transformer) + + def test_saturation(self): + transformer = Saturation(0.5, 1.5) + self.transformer_test(transformer) + + def test_hue(self): + transformer = Hue(0.5, 1.5) + self.transformer_test(transformer) + + def test_channel_normalize(self): + transformer = ChannelNormalize(100.0, 200.0, 300.0, 2.0, 2.0, 2.0) + self.transformer_test(transformer) + + def test_pixel_normalize(self): + means = [2.0] * 3 * 500 * 375 + transformer = PixelNormalize(means) + self.transformer_test(transformer) + + def test_fixed_crop_norm(self): + crop = FixedCrop(0.0, 0.0, 0.5, 1.0) + self.transformer_test(crop) + + def test_fixed_crop_unnorm(self): + crop = FixedCrop(0.0, 0.0, 200.0, 200., False) + self.transformer_test(crop) + + def test_center_crop(self): + crop = CenterCrop(200, 200) + self.transformer_test(crop) + + def test_random_crop(self): + crop = RandomCrop(200, 200) + self.transformer_test(crop) + + def test_filler(self): + filler = Filler(0.0, 0.0, 0.1, 0.2) + self.transformer_test(filler) + + def test_expand(self): + expand = Expand(means_r=123, means_g=117, means_b=104, + max_expand_ratio=2.0) + self.transformer_test(expand) + + def test_random_transformer(self): + transformer = RandomTransformer(HFlip(), 0.5) + self.transformer_test(transformer) + + def test_pipeline(self): + transformer = Pipeline([ColorJitter(), HFlip(), Resize(200, 200, 1)]) + self.transformer_test(transformer) + if __name__ == "__main__": pytest.main([__file__]) From 4b711dba3da5e1ff45a16fb1015f3333fee420ef Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Fri, 1 Dec 2017 23:36:36 +0800 Subject: [PATCH 376/823] Do not check spark-bigdl.conf if sc exists (#1966) * do not check spark-bigdl.conf if sc exists recover syspath print warning instead exception if bigdl.conf doesn't exist enhance unitest remove warning update test try to fix jenkins update * delete * fix --- python/dllib/src/bigdl/dllib/utils/common.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/utils/common.py b/python/dllib/src/bigdl/dllib/utils/common.py index dcc681ab303..0c2f73a5aea 100644 --- a/python/dllib/src/bigdl/dllib/utils/common.py +++ b/python/dllib/src/bigdl/dllib/utils/common.py @@ -406,7 +406,7 @@ def load_conf(conf_str): if sys.version_info >= (3,): content = str(content, 'latin-1') return load_conf(content) - raise Exception("Cannot find spark-bigdl.conf.Pls add it to PYTHONPATH.") + return {} def to_list(a): @@ -437,13 +437,21 @@ def get_spark_context(conf = None): :return: SparkContext """ if hasattr(SparkContext, "getOrCreate"): - return SparkContext.getOrCreate(conf=conf or create_spark_conf()) + with SparkContext._lock: + if SparkContext._active_spark_context is None: + spark_conf = create_spark_conf() if conf is None else conf + return SparkContext.getOrCreate(spark_conf) + else: + return SparkContext.getOrCreate() + else: # Might have threading issue but we cann't add _lock here # as it's not RLock in spark1.5; if SparkContext._active_spark_context is None: - SparkContext(conf=conf or create_spark_conf()) - return SparkContext._active_spark_context + spark_conf = create_spark_conf() if conf is None else conf + return SparkContext(conf=spark_conf) + else: + return SparkContext._active_spark_context def get_spark_sql_context(sc): From 901abbd30205dc3c712166bb1f269a29224b93fd Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Fri, 1 Dec 2017 23:36:36 +0800 Subject: [PATCH 377/823] Do not check spark-bigdl.conf if sc exists (#1966) * do not check spark-bigdl.conf if sc exists recover syspath print warning instead exception if bigdl.conf doesn't exist enhance unitest remove warning update test try to fix jenkins update * delete * fix --- python/dllib/src/bigdl/utils/common.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/python/dllib/src/bigdl/utils/common.py b/python/dllib/src/bigdl/utils/common.py index dcc681ab303..0c2f73a5aea 100644 --- a/python/dllib/src/bigdl/utils/common.py +++ b/python/dllib/src/bigdl/utils/common.py @@ -406,7 +406,7 @@ def load_conf(conf_str): if sys.version_info >= (3,): content = str(content, 'latin-1') return load_conf(content) - raise Exception("Cannot find spark-bigdl.conf.Pls add it to PYTHONPATH.") + return {} def to_list(a): @@ -437,13 +437,21 @@ def get_spark_context(conf = None): :return: SparkContext """ if hasattr(SparkContext, "getOrCreate"): - return SparkContext.getOrCreate(conf=conf or create_spark_conf()) + with SparkContext._lock: + if SparkContext._active_spark_context is None: + spark_conf = create_spark_conf() if conf is None else conf + return SparkContext.getOrCreate(spark_conf) + else: + return SparkContext.getOrCreate() + else: # Might have threading issue but we cann't add _lock here # as it's not RLock in spark1.5; if SparkContext._active_spark_context is None: - SparkContext(conf=conf or create_spark_conf()) - return SparkContext._active_spark_context + spark_conf = create_spark_conf() if conf is None else conf + return SparkContext(conf=spark_conf) + else: + return SparkContext._active_spark_context def get_spark_sql_context(sc): From 4e8efd17f9d8ec383f0d83e720bef58c9780dacd Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Fri, 1 Dec 2017 23:36:36 +0800 Subject: [PATCH 378/823] Do not check spark-bigdl.conf if sc exists (#1966) * do not check spark-bigdl.conf if sc exists recover syspath print warning instead exception if bigdl.conf doesn't exist enhance unitest remove warning update test try to fix jenkins update * delete * fix --- python/dllib/test/dev/run-tests | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/dllib/test/dev/run-tests b/python/dllib/test/dev/run-tests index 41374c43e46..2c3e83113b2 100755 --- a/python/dllib/test/dev/run-tests +++ b/python/dllib/test/dev/run-tests @@ -34,7 +34,7 @@ do --ignore=../../../pyspark/bigdl/keras \ --ignore=../../../pyspark/test/bigdl/keras/test_application.py \ --ignore=../../../pyspark/bigdl/models/ && \ - $p -m pytest -v -n 4 ../../../pyspark/test/ \ + $p -m pytest -v ../../../pyspark/test/ \ --ignore=../../../pyspark/test/bigdl/caffe/ \ --ignore=../../../pyspark/test/bigdl/keras/ \ --ignore=../../../pyspark/test/bigdl/test_utils.py From 5ae4617812d5d38487b785fc37db2208a6514fab Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Mon, 4 Dec 2017 13:12:38 +0800 Subject: [PATCH 379/823] Add include_container option for flattened_layers (#1941) * update * style --- python/dllib/src/bigdl/dllib/keras/converter.py | 2 +- python/dllib/src/bigdl/dllib/nn/layer.py | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/keras/converter.py b/python/dllib/src/bigdl/dllib/keras/converter.py index 13951403a62..e948a1d53b3 100644 --- a/python/dllib/src/bigdl/dllib/keras/converter.py +++ b/python/dllib/src/bigdl/dllib/keras/converter.py @@ -79,7 +79,7 @@ def __keras_name_to_Layers(model, with_weights=False): def __bigdl_name_to_Layers(model, with_weights=False): # NB: Container in BigDL is_with_weights() is true if one of the nested layer with_weights # but in Keras container get_weights() return false even if the nested layer with_weights - all_layers = model.layers + model.flattened_layers + all_layers = model.flattened_layers(include_container=True) if with_weights: layers = [l for l in all_layers if l.is_with_weights()] else: diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 949c02c0101..04148373d6c 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -599,9 +599,8 @@ def layers(self): layers = [Layer.of(jlayer) for jlayer in jlayers] return layers - @property - def flattened_layers(self): - jlayers = callBigDlFunc(self.bigdl_type, "getFlattenModules", self) + def flattened_layers(self, include_container=False): + jlayers = callBigDlFunc(self.bigdl_type, "getFlattenModules", self, include_container) layers = [Layer.of(jlayer) for jlayer in jlayers] return layers From cb5efb189d35383fa8662a0069a431eaf37668da Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Mon, 4 Dec 2017 13:12:38 +0800 Subject: [PATCH 380/823] Add include_container option for flattened_layers (#1941) * update * style --- python/dllib/src/bigdl/dllib/keras/converter.py | 2 +- python/dllib/src/bigdl/dllib/nn/layer.py | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/keras/converter.py b/python/dllib/src/bigdl/dllib/keras/converter.py index 13951403a62..e948a1d53b3 100644 --- a/python/dllib/src/bigdl/dllib/keras/converter.py +++ b/python/dllib/src/bigdl/dllib/keras/converter.py @@ -79,7 +79,7 @@ def __keras_name_to_Layers(model, with_weights=False): def __bigdl_name_to_Layers(model, with_weights=False): # NB: Container in BigDL is_with_weights() is true if one of the nested layer with_weights # but in Keras container get_weights() return false even if the nested layer with_weights - all_layers = model.layers + model.flattened_layers + all_layers = model.flattened_layers(include_container=True) if with_weights: layers = [l for l in all_layers if l.is_with_weights()] else: diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 949c02c0101..04148373d6c 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -599,9 +599,8 @@ def layers(self): layers = [Layer.of(jlayer) for jlayer in jlayers] return layers - @property - def flattened_layers(self): - jlayers = callBigDlFunc(self.bigdl_type, "getFlattenModules", self) + def flattened_layers(self, include_container=False): + jlayers = callBigDlFunc(self.bigdl_type, "getFlattenModules", self, include_container) layers = [Layer.of(jlayer) for jlayer in jlayers] return layers From 02f4cfd63f1cb91ba7a1796977fafb04cd62f6bc Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Mon, 4 Dec 2017 13:12:38 +0800 Subject: [PATCH 381/823] Add include_container option for flattened_layers (#1941) * update * style --- python/dllib/test/bigdl/keras/test_layer.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/python/dllib/test/bigdl/keras/test_layer.py b/python/dllib/test/bigdl/keras/test_layer.py index cb67fc46d98..2d812ba706a 100644 --- a/python/dllib/test/bigdl/keras/test_layer.py +++ b/python/dllib/test/bigdl/keras/test_layer.py @@ -296,6 +296,20 @@ def test_merge_method_concat(self): dump_weights=True, is_training=False) + def test_nested_with_combo_bigdl_layer_lstm(self): + branch1 = Sequential() + branch1.add(LSTM(64, input_dim=10, input_length=10, return_sequences=True, + inner_activation='sigmoid')) + branch2 = Sequential() + branch2.add(Reshape((10, 2), input_shape=(20, ))) + + input_data = [np.random.random([3, 10, 10]), np.random.random([3, 20])] + + kmodel = Sequential() + kmodel.add(Merge([branch1, branch2], mode='concat')) + kmodel.add(Activation('sigmoid')) + self.modelTest(input_data, kmodel, dump_weights=True) + def test_merge_method_mix_concat(self): input_data1 = np.random.random_sample([2, 4]) input_data2 = np.random.random_sample([2, 3]) From 0049f45aa6efd7aef86101a8f2c1d2156a801c68 Mon Sep 17 00:00:00 2001 From: Ian Wong Date: Mon, 4 Dec 2017 16:31:47 +0800 Subject: [PATCH 382/823] Add an API to get node from graph model with given name (#1871) * get node from graph * add unit test * add Graph.node api in python --- python/dllib/src/bigdl/dllib/nn/layer.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 04148373d6c..37139b8fe36 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -782,6 +782,17 @@ def stop_gradient(self, stop_layers, bigdl_type="float"): callBigDlFunc(bigdl_type, "setStopGradient", self.value, stop_layers) return self + def node(self, name, bigdl_type="float"): + """ + Return the corresponding node has the given name. If the given name doesn't match any node, + an exception will be thrown + :param name: node name + :param bigdl_type: + :return: + """ + jnode = callBigDlFunc(bigdl_type, "findGraphNode", self.value, name) + return Node.of(jnode) + def save_graph_topology(self, log_path, bigdl_type="float"): """ save current model graph to a folder, which can be display in tensorboard by running From 5a74072f4d8b84ff6f22b97961babc283402d497 Mon Sep 17 00:00:00 2001 From: Ian Wong Date: Mon, 4 Dec 2017 16:31:47 +0800 Subject: [PATCH 383/823] Add an API to get node from graph model with given name (#1871) * get node from graph * add unit test * add Graph.node api in python --- python/dllib/src/bigdl/dllib/nn/layer.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 04148373d6c..37139b8fe36 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -782,6 +782,17 @@ def stop_gradient(self, stop_layers, bigdl_type="float"): callBigDlFunc(bigdl_type, "setStopGradient", self.value, stop_layers) return self + def node(self, name, bigdl_type="float"): + """ + Return the corresponding node has the given name. If the given name doesn't match any node, + an exception will be thrown + :param name: node name + :param bigdl_type: + :return: + """ + jnode = callBigDlFunc(bigdl_type, "findGraphNode", self.value, name) + return Node.of(jnode) + def save_graph_topology(self, log_path, bigdl_type="float"): """ save current model graph to a folder, which can be display in tensorboard by running From 7b149b22a0d93f7b59e6d7a5405d61b439e35eda Mon Sep 17 00:00:00 2001 From: Ian Wong Date: Mon, 4 Dec 2017 16:31:47 +0800 Subject: [PATCH 384/823] Add an API to get node from graph model with given name (#1871) * get node from graph * add unit test * add Graph.node api in python --- python/dllib/test/bigdl/test_simple_integration.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/python/dllib/test/bigdl/test_simple_integration.py b/python/dllib/test/bigdl/test_simple_integration.py index 23460b41df0..c0a83483613 100644 --- a/python/dllib/test/bigdl/test_simple_integration.py +++ b/python/dllib/test/bigdl/test_simple_integration.py @@ -160,6 +160,16 @@ def test_graph_backward(self): assert_allclose(gradInput[1], np.array([6.0, 6.0, 6.0, 6.0])) + def test_get_node(self): + fc1 = Linear(4, 2)() + fc2 = Linear(4, 2)() + fc1.element().set_name("fc1") + cadd = CAddTable()([fc1, fc2]) + output1 = ReLU()(cadd) + model = Model([fc1, fc2], [output1]) + res = model.node("fc1") + assert res.element().name() == "fc1" + def test_save_graph_topology(self): fc1 = Linear(4, 2)() fc2 = Linear(4, 2)() From bb3d5a4e99dc87d8e666358af85a438abff8b261 Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Tue, 5 Dec 2017 09:15:49 +0800 Subject: [PATCH 385/823] An Activity wrapper for python to simplify the returning value (#1962) * update add test remove jtable clean clean clean * sort list * clean * update and use toSeq * add toSeqActivity clean * use D --- python/dllib/src/bigdl/dllib/utils/common.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/python/dllib/src/bigdl/dllib/utils/common.py b/python/dllib/src/bigdl/dllib/utils/common.py index 0c2f73a5aea..93fa4f9fc90 100644 --- a/python/dllib/src/bigdl/dllib/utils/common.py +++ b/python/dllib/src/bigdl/dllib/utils/common.py @@ -118,6 +118,13 @@ def get_dtype(bigdl_type): # Always return float32 for now return "float32" + +class JActivity(object): + + def __init__(self, value): + self.value = value + + class JTensor(object): """ A wrapper to easy our work when need to pass or return Tensor to/from Scala. @@ -362,7 +369,8 @@ def uniform(self, a, b, size): 'LabeledPoint', 'Sample', 'EvaluatedResult', - 'JTensor' + 'JTensor', + 'JActivity' ] From bcd6e62edad222bdc1a0a29f16203172c6131986 Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Tue, 5 Dec 2017 09:15:49 +0800 Subject: [PATCH 386/823] An Activity wrapper for python to simplify the returning value (#1962) * update add test remove jtable clean clean clean * sort list * clean * update and use toSeq * add toSeqActivity clean * use D --- python/dllib/src/bigdl/utils/common.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/python/dllib/src/bigdl/utils/common.py b/python/dllib/src/bigdl/utils/common.py index 0c2f73a5aea..93fa4f9fc90 100644 --- a/python/dllib/src/bigdl/utils/common.py +++ b/python/dllib/src/bigdl/utils/common.py @@ -118,6 +118,13 @@ def get_dtype(bigdl_type): # Always return float32 for now return "float32" + +class JActivity(object): + + def __init__(self, value): + self.value = value + + class JTensor(object): """ A wrapper to easy our work when need to pass or return Tensor to/from Scala. @@ -362,7 +369,8 @@ def uniform(self, a, b, size): 'LabeledPoint', 'Sample', 'EvaluatedResult', - 'JTensor' + 'JTensor', + 'JActivity' ] From 1370bbaa6448202f11bc0e297024e74e7041cea0 Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Tue, 5 Dec 2017 09:15:49 +0800 Subject: [PATCH 387/823] An Activity wrapper for python to simplify the returning value (#1962) * update add test remove jtable clean clean clean * sort list * clean * update and use toSeq * add toSeqActivity clean * use D --- python/dllib/test/bigdl/test_pickler.py | 67 +++++++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 python/dllib/test/bigdl/test_pickler.py diff --git a/python/dllib/test/bigdl/test_pickler.py b/python/dllib/test/bigdl/test_pickler.py new file mode 100644 index 00000000000..8697775fb67 --- /dev/null +++ b/python/dllib/test/bigdl/test_pickler.py @@ -0,0 +1,67 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from bigdl.nn.layer import * +from bigdl.nn.initialization_method import * +from bigdl.nn.criterion import * +from bigdl.optim.optimizer import * +from bigdl.util.common import * +from bigdl.util.common import _py2java +from bigdl.nn.initialization_method import * +from bigdl.dataset import movielens +import numpy as np +import tempfile +import pytest +from numpy.testing import assert_allclose, assert_array_equal +from bigdl.util.engine import compare_version +np.random.seed(1337) # for reproducibility + + +class TestPickler(): + def setup_method(self, method): + """ setup any state tied to the execution of the given method in a + class. setup_method is invoked for every test method of a class. + """ + JavaCreator.set_creator_class("com.intel.analytics.bigdl.python.api.PythonBigDLValidator") + sparkConf = create_spark_conf().setMaster("local[4]").setAppName("test model") + self.sc = get_spark_context(sparkConf) + init_engine() + + def teardown_method(self, method): + """ teardown any state that was previously setup with a setup_method + call. + """ + self.sc.stop() + + def test_activity_with_jtensor(self): + back = callBigDlFunc("float", "testActivityWithTensor") + assert isinstance(back.value, JTensor) + + def test_activity_with_table_of_tensor(self): + back = callBigDlFunc("float", "testActivityWithTableOfTensor") + assert isinstance(back.value, list) + assert isinstance(back.value[0], JTensor) + assert back.value[0].to_ndarray()[0] < back.value[1].to_ndarray()[0] + assert back.value[1].to_ndarray()[0] < back.value[2].to_ndarray()[0] + + def test_activity_with_table_of_table(self): + back = callBigDlFunc("float", "testActivityWithTableOfTable") + assert isinstance(back.value, list) + assert isinstance(back.value[0], list) + assert isinstance(back.value[0][0], JTensor) + +if __name__ == "__main__": + pytest.main([__file__]) From 01a34fb9dd9360fa49e152236a3a46cc351a32bc Mon Sep 17 00:00:00 2001 From: Ian Wong Date: Tue, 5 Dec 2017 15:07:59 +0800 Subject: [PATCH 388/823] Support NHWC for LRN and batch normalization, and optimize batch normalization speed (#1936) * add LRN and LRNGrad * add LRNGrad * support bn with NHWC * support bn with NHWC * add batch norm grad * support tf bnv2/bngradv2 * fix uts * fix uts * fix ut * fix ut * fix ut * fix ut * fix style issue * fix ut * fix ut * fix ut * update python api and docs * meet code review * meet code review * fix ut * fix ut * fix ut --- python/dllib/src/bigdl/dllib/nn/layer.py | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 37139b8fe36..fdf43bea166 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -1654,6 +1654,14 @@ class SpatialBatchNormalization(Layer): where gamma and beta are learnable parameters. The learning of gamma and beta is optional. + + :param n_output: output feature map number + :param eps: avoid divide zero + :param momentum: momentum for weight update + :param affine: affine operation on output or not + :param data_format a string value (or DataFormat Object in Scala) of "NHWC" or "NCHW" to specify the input data format of this layer. In "NHWC" format + data is stored in the order of [batch_size, height, width, channels], in "NCHW" format data is stored + in the order of [batch_size, channels, height, width]. >>> spatialBatchNormalization = SpatialBatchNormalization(1) @@ -1665,6 +1673,8 @@ class SpatialBatchNormalization(Layer): >>> init_grad_bias = np.array([0.0]) >>> spatialBatchNormalization = SpatialBatchNormalization(1, 1e-5, 0.1, True, init_weight, init_bias, init_grad_weight, init_grad_bias) creating: createSpatialBatchNormalization + >>> spatialBatchNormalization = SpatialBatchNormalization(1, 1e-5, 0.1, True, init_weight, init_bias, init_grad_weight, init_grad_bias, "NHWC") + creating: createSpatialBatchNormalization ''' def __init__(self, @@ -1676,6 +1686,7 @@ def __init__(self, init_bias=None, init_grad_weight=None, init_grad_bias=None, + data_format="NCHW", bigdl_type="float"): super(SpatialBatchNormalization, self).__init__(None, bigdl_type, n_output, @@ -1685,7 +1696,8 @@ def __init__(self, JTensor.from_ndarray(init_weight), JTensor.from_ndarray(init_bias), JTensor.from_ndarray(init_grad_weight), - JTensor.from_ndarray(init_grad_bias)) + JTensor.from_ndarray(init_grad_bias), + data_format) def set_init_method(self, weight_init_method = None, bias_init_method = None): callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, @@ -1712,10 +1724,15 @@ class SpatialCrossMapLRN(Layer): :param alpha: the scaling parameter :param beta: the exponent :param k: a constant + :param data_format a string value (or DataFormat Object in Scala) of "NHWC" or "NCHW" to specify the input data format of this layer. In "NHWC" format + data is stored in the order of [batch_size, height, width, channels], in "NCHW" format data is stored + in the order of [batch_size, channels, height, width] >>> spatialCrossMapLRN = SpatialCrossMapLRN() creating: createSpatialCrossMapLRN + >>> spatialCrossMapLRN = SpatialCrossMapLRN(5, 1.0, 0.75, 1.0, "NHWC") + creating: createSpatialCrossMapLRN ''' def __init__(self, @@ -1723,12 +1740,13 @@ def __init__(self, alpha=1.0, beta=0.75, k=1.0, + data_format="NCHW", bigdl_type="float"): super(SpatialCrossMapLRN, self).__init__(None, bigdl_type, size, alpha, beta, - k) + k, data_format) class Dropout(Layer): From b822257a48ad8cbd8216cbc5d62d1f850638fa2b Mon Sep 17 00:00:00 2001 From: Ian Wong Date: Tue, 5 Dec 2017 15:07:59 +0800 Subject: [PATCH 389/823] Support NHWC for LRN and batch normalization, and optimize batch normalization speed (#1936) * add LRN and LRNGrad * add LRNGrad * support bn with NHWC * support bn with NHWC * add batch norm grad * support tf bnv2/bngradv2 * fix uts * fix uts * fix ut * fix ut * fix ut * fix ut * fix style issue * fix ut * fix ut * fix ut * update python api and docs * meet code review * meet code review * fix ut * fix ut * fix ut --- python/dllib/src/bigdl/dllib/nn/layer.py | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 37139b8fe36..fdf43bea166 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -1654,6 +1654,14 @@ class SpatialBatchNormalization(Layer): where gamma and beta are learnable parameters. The learning of gamma and beta is optional. + + :param n_output: output feature map number + :param eps: avoid divide zero + :param momentum: momentum for weight update + :param affine: affine operation on output or not + :param data_format a string value (or DataFormat Object in Scala) of "NHWC" or "NCHW" to specify the input data format of this layer. In "NHWC" format + data is stored in the order of [batch_size, height, width, channels], in "NCHW" format data is stored + in the order of [batch_size, channels, height, width]. >>> spatialBatchNormalization = SpatialBatchNormalization(1) @@ -1665,6 +1673,8 @@ class SpatialBatchNormalization(Layer): >>> init_grad_bias = np.array([0.0]) >>> spatialBatchNormalization = SpatialBatchNormalization(1, 1e-5, 0.1, True, init_weight, init_bias, init_grad_weight, init_grad_bias) creating: createSpatialBatchNormalization + >>> spatialBatchNormalization = SpatialBatchNormalization(1, 1e-5, 0.1, True, init_weight, init_bias, init_grad_weight, init_grad_bias, "NHWC") + creating: createSpatialBatchNormalization ''' def __init__(self, @@ -1676,6 +1686,7 @@ def __init__(self, init_bias=None, init_grad_weight=None, init_grad_bias=None, + data_format="NCHW", bigdl_type="float"): super(SpatialBatchNormalization, self).__init__(None, bigdl_type, n_output, @@ -1685,7 +1696,8 @@ def __init__(self, JTensor.from_ndarray(init_weight), JTensor.from_ndarray(init_bias), JTensor.from_ndarray(init_grad_weight), - JTensor.from_ndarray(init_grad_bias)) + JTensor.from_ndarray(init_grad_bias), + data_format) def set_init_method(self, weight_init_method = None, bias_init_method = None): callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, @@ -1712,10 +1724,15 @@ class SpatialCrossMapLRN(Layer): :param alpha: the scaling parameter :param beta: the exponent :param k: a constant + :param data_format a string value (or DataFormat Object in Scala) of "NHWC" or "NCHW" to specify the input data format of this layer. In "NHWC" format + data is stored in the order of [batch_size, height, width, channels], in "NCHW" format data is stored + in the order of [batch_size, channels, height, width] >>> spatialCrossMapLRN = SpatialCrossMapLRN() creating: createSpatialCrossMapLRN + >>> spatialCrossMapLRN = SpatialCrossMapLRN(5, 1.0, 0.75, 1.0, "NHWC") + creating: createSpatialCrossMapLRN ''' def __init__(self, @@ -1723,12 +1740,13 @@ def __init__(self, alpha=1.0, beta=0.75, k=1.0, + data_format="NCHW", bigdl_type="float"): super(SpatialCrossMapLRN, self).__init__(None, bigdl_type, size, alpha, beta, - k) + k, data_format) class Dropout(Layer): From 24018515afc9ae4059282ec3437e2b3f50f1e005 Mon Sep 17 00:00:00 2001 From: Jerry Wu Date: Tue, 5 Dec 2017 02:05:32 -0600 Subject: [PATCH 390/823] Pb support bigmodel (#1976) * support new format * add unit test * add unit test * refinement * fix typo * fix unit test --- python/dllib/src/bigdl/dllib/nn/layer.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index fdf43bea166..60214316131 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -412,9 +412,9 @@ def is_with_weights(self): def save(self, path, over_write = False): callBigDlFunc(self.bigdl_type, "modelSave", self.value, path, over_write) - def saveModel(self, path, over_write = False): - callBigDlFunc(self.bigdl_type, "saveBigDLModule", self.value, path, - over_write) + def saveModel(self, modelPath, weightPath = None, over_write = False): + callBigDlFunc(self.bigdl_type, "saveBigDLModule", self.value, modelPath, + weightPath, over_write) def save_caffe(self, prototxt_path, model_path, use_v2 = True, overwrite = False): callBigDlFunc(self.bigdl_type, "saveCaffe", self.value, prototxt_path, @@ -676,14 +676,14 @@ def load(path, bigdl_type="float"): return Layer.of(jmodel) @staticmethod - def loadModel(path, bigdl_type="float"): + def loadModel(modelPath, weightPath =None, bigdl_type="float"): """ Load a pre-trained Bigdl model. :param path: The path containing the pre-trained model. :return: A pre-trained model. """ - jmodel = callBigDlFunc(bigdl_type, "loadBigDLModule", path) + jmodel = callBigDlFunc(bigdl_type, "loadBigDLModule", modelPath, weightPath) return Layer.of(jmodel) @staticmethod From 290b9b14df40f8e6134733b51dfa9eb86560f557 Mon Sep 17 00:00:00 2001 From: Jerry Wu Date: Tue, 5 Dec 2017 02:05:32 -0600 Subject: [PATCH 391/823] Pb support bigmodel (#1976) * support new format * add unit test * add unit test * refinement * fix typo * fix unit test --- python/dllib/src/bigdl/dllib/nn/layer.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index fdf43bea166..60214316131 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -412,9 +412,9 @@ def is_with_weights(self): def save(self, path, over_write = False): callBigDlFunc(self.bigdl_type, "modelSave", self.value, path, over_write) - def saveModel(self, path, over_write = False): - callBigDlFunc(self.bigdl_type, "saveBigDLModule", self.value, path, - over_write) + def saveModel(self, modelPath, weightPath = None, over_write = False): + callBigDlFunc(self.bigdl_type, "saveBigDLModule", self.value, modelPath, + weightPath, over_write) def save_caffe(self, prototxt_path, model_path, use_v2 = True, overwrite = False): callBigDlFunc(self.bigdl_type, "saveCaffe", self.value, prototxt_path, @@ -676,14 +676,14 @@ def load(path, bigdl_type="float"): return Layer.of(jmodel) @staticmethod - def loadModel(path, bigdl_type="float"): + def loadModel(modelPath, weightPath =None, bigdl_type="float"): """ Load a pre-trained Bigdl model. :param path: The path containing the pre-trained model. :return: A pre-trained model. """ - jmodel = callBigDlFunc(bigdl_type, "loadBigDLModule", path) + jmodel = callBigDlFunc(bigdl_type, "loadBigDLModule", modelPath, weightPath) return Layer.of(jmodel) @staticmethod From 4d7b2116a61434a456c5ff73e7b94fb6b44f97a0 Mon Sep 17 00:00:00 2001 From: Jerry Wu Date: Tue, 5 Dec 2017 02:05:32 -0600 Subject: [PATCH 392/823] Pb support bigmodel (#1976) * support new format * add unit test * add unit test * refinement * fix typo * fix unit test --- python/dllib/test/bigdl/test_simple_integration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/dllib/test/bigdl/test_simple_integration.py b/python/dllib/test/bigdl/test_simple_integration.py index c0a83483613..68c205f7f07 100644 --- a/python/dllib/test/bigdl/test_simple_integration.py +++ b/python/dllib/test/bigdl/test_simple_integration.py @@ -85,7 +85,7 @@ def test_load_model_proto(self): fc1 = Linear(4, 2) fc1.set_weights([np.ones((4, 2)), np.ones((2,))]) tmp_path = tempfile.mktemp() - fc1.saveModel(tmp_path, True) + fc1.saveModel(tmp_path, None, True) fc1_loaded = Model.loadModel(tmp_path) assert_allclose(fc1_loaded.get_weights()[0], fc1.get_weights()[0]) From 8038c1c515e978bc66cb298d1400ba098bff941f Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Tue, 5 Dec 2017 17:09:34 +0800 Subject: [PATCH 393/823] Add cos for merge (#1981) --- python/dllib/src/bigdl/dllib/keras/converter.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/python/dllib/src/bigdl/dllib/keras/converter.py b/python/dllib/src/bigdl/dllib/keras/converter.py index e948a1d53b3..0061eefd97a 100644 --- a/python/dllib/src/bigdl/dllib/keras/converter.py +++ b/python/dllib/src/bigdl/dllib/keras/converter.py @@ -540,7 +540,12 @@ def create_merge(self, klayer, kclayer): inplace=False, bigdl_type="float") elif klayer.mode in ['cos']: - raise Exception("Merge mode `%s` not supported for now" % klayer.mode) + if len(input_shape[0]) >= 3: + raise Exception("For merge mode cos, 3D input or above is not supported for now.") + if klayer.dot_axes != [1, 1]: + raise Exception("For merge mode cos, only dot_axes=1 is supported for now.") + blayer = BLayer.Sequential() + blayer.add(BLayer.CosineDistance(bigdl_type="float")).add(BLayer.Reshape([1, 1], True)) else: # invalid mode or lambda functions raise Exception("Invalid merge mode: `%s`. Lambda/function as merge mode is not supported for now." % klayer.mode) if self.__is_from_sequential(klayer, kclayer): From 3b080c7ce4554ae4f582b86063f22889fe11ebb3 Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Tue, 5 Dec 2017 17:09:34 +0800 Subject: [PATCH 394/823] Add cos for merge (#1981) --- python/dllib/src/bigdl/dllib/keras/converter.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/python/dllib/src/bigdl/dllib/keras/converter.py b/python/dllib/src/bigdl/dllib/keras/converter.py index e948a1d53b3..0061eefd97a 100644 --- a/python/dllib/src/bigdl/dllib/keras/converter.py +++ b/python/dllib/src/bigdl/dllib/keras/converter.py @@ -540,7 +540,12 @@ def create_merge(self, klayer, kclayer): inplace=False, bigdl_type="float") elif klayer.mode in ['cos']: - raise Exception("Merge mode `%s` not supported for now" % klayer.mode) + if len(input_shape[0]) >= 3: + raise Exception("For merge mode cos, 3D input or above is not supported for now.") + if klayer.dot_axes != [1, 1]: + raise Exception("For merge mode cos, only dot_axes=1 is supported for now.") + blayer = BLayer.Sequential() + blayer.add(BLayer.CosineDistance(bigdl_type="float")).add(BLayer.Reshape([1, 1], True)) else: # invalid mode or lambda functions raise Exception("Invalid merge mode: `%s`. Lambda/function as merge mode is not supported for now." % klayer.mode) if self.__is_from_sequential(klayer, kclayer): From 3211d0302e1e1b7325e905bd4f7b9e149674b9e2 Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Tue, 5 Dec 2017 17:09:34 +0800 Subject: [PATCH 395/823] Add cos for merge (#1981) --- python/dllib/test/bigdl/keras/test_layer.py | 18 +++++++++++++++++- python/dllib/test/bigdl/test_utils.py | 2 +- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/python/dllib/test/bigdl/keras/test_layer.py b/python/dllib/test/bigdl/keras/test_layer.py index 2d812ba706a..5b9a44f4571 100644 --- a/python/dllib/test/bigdl/keras/test_layer.py +++ b/python/dllib/test/bigdl/keras/test_layer.py @@ -22,9 +22,9 @@ from keras.layers.core import * from keras.layers.convolutional import * from keras.layers import Dense, Input +from keras.regularizers import l1, l2, l1l2 from bigdl.keras.converter import * from test.bigdl.test_utils import BigDLTestCase -from keras.regularizers import l1, l2, l1l2 class TestLayer(BigDLTestCase): @@ -279,6 +279,22 @@ def test_nested_model_seq_concat(self): dump_weights=True, is_training=False) + def test_merge_method_cos(self): + input_data1 = np.random.random_sample([2, 4]) + input_data2 = np.random.random_sample([2, 4]) + input1 = Input((4,)) + input2 = Input((4,)) + out1 = Dense(4)(input1) + out2 = Dense(4)(input2) + from keras.engine import merge + m = merge([out1, out2], mode="cos", dot_axes=1) + kmodel = Model(input=[input1, input2], output=m) + self.modelTest([input_data1, input_data2], + kmodel, + random_weights=False, + dump_weights=True, + is_training=False) + def test_merge_method_concat(self): input_data1 = np.random.random_sample([2, 4]) input_data2 = np.random.random_sample([2, 3]) diff --git a/python/dllib/test/bigdl/test_utils.py b/python/dllib/test/bigdl/test_utils.py index 68f83d619e5..c808ab64b72 100644 --- a/python/dllib/test/bigdl/test_utils.py +++ b/python/dllib/test/bigdl/test_utils.py @@ -16,7 +16,6 @@ from __future__ import print_function import numpy as np from keras.models import Sequential, Model -from bigdl.keras.converter import WeightLoader, WeightsConverter np.random.seed(1337) # for reproducibility from keras.layers.core import * from keras.layers.convolutional import * @@ -24,6 +23,7 @@ from keras.optimizers import RMSprop from bigdl.util.common import create_tmp_path from bigdl.keras.converter import * +from bigdl.keras.converter import WeightLoader, WeightsConverter import numpy as np from unittest import TestCase import keras From 2beaf51f8f8a33aec14a0c3da0e8d0c2697a1d58 Mon Sep 17 00:00:00 2001 From: Yanzhang Wang Date: Tue, 5 Dec 2017 17:42:49 +0800 Subject: [PATCH 396/823] feat: SReLU and ActivityRegularization support (#1917) * feat: SReLU and ActivityRegularization support * fix: update doc * fix: refactor the weights and gradWeights of SReLU * fix: serialization for srelu * fix: backward computing errors * fix: ActivityRegularization to layer * fix: change test location * fix: add python api * fix: shared axes have not consider enough --- python/dllib/src/bigdl/dllib/nn/layer.py | 59 +++++++++++++++++++ .../dllib/src/bigdl/dllib/optim/optimizer.py | 10 ++++ 2 files changed, 69 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 60214316131..5e418952c63 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -3251,6 +3251,65 @@ def __init__(self, super(ReLU6, self).__init__(None, bigdl_type, inplace) +class SReLU(Layer): + + '''S-shaped Rectified Linear Unit. + + It follows: + `f(x) = t^r + a^r(x - t^r) for x >= t^r`, + `f(x) = x for t^r > x > t^l`, + `f(x) = t^l + a^l(x - t^l) for x <= t^l`. + + # References + - [Deep Learning with S-shaped Rectified Linear Activation Units](http://arxiv.org/abs/1512.07030) + + + + :param shared_axes: the axes along which to share learnable + parameters for the activation function. + For example, if the incoming feature maps + are from a 2D convolution + with output shape `(batch, height, width, channels)`, + and you wish to share parameters across space + so that each filter only has one set of parameters, + set `shared_axes=[1, 2]`. + + >>> srelu = SReLU() + creating: createSReLU + >>> srelu = SReLU((1, 2)) + creating: createSReLU + ''' + + def __init__(self, + share_axes=None, + bigdl_type="float"): + super(SReLU, self).__init__(None, bigdl_type, + share_axes) + + def set_init_method(self, tLeftInit = None, aLeftInit = None, + tRightInit = None, aRightInit = None): + callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, + tLeftInit, aLeftInit, tRightInit, aRightInit) + return self + +class ActivityRegularization(Layer): + + ''' + Layer that applies an update to the cost function based input activity. + + :param l1: L1 regularization factor (positive float). + :param l2: L2 regularization factor (positive float). + + + >>> ar = ActivityRegularization(0.1, 0.02) + creating: createActivityRegularization + ''' + + def __init__(self, + l1=0.0, + l2=0.0, + bigdl_type="float"): + super(ActivityRegularization, self).__init__(None, bigdl_type, l1, l2) class Replicate(Layer): diff --git a/python/dllib/src/bigdl/dllib/optim/optimizer.py b/python/dllib/src/bigdl/dllib/optim/optimizer.py index a817dd006cf..0b18022a994 100644 --- a/python/dllib/src/bigdl/dllib/optim/optimizer.py +++ b/python/dllib/src/bigdl/dllib/optim/optimizer.py @@ -818,6 +818,16 @@ class L1L2Regularizer(JavaValue): def __init__(self, l1, l2, bigdl_type="float"): JavaValue.__init__(self, None, bigdl_type, l1, l2) +class ActivityRegularization(JavaValue): + """ + Apply both L1 and L2 regularization + + :param l1 l1 regularization rate + :param l2 l2 regularization rate + + """ + def __init__(self, l1, l2, bigdl_type="float"): + JavaValue.__init__(self, None, bigdl_type, l1, l2) class L1Regularizer(JavaValue): """ From 60345ea6e3c730e35c50c3d96c502a2b4002467f Mon Sep 17 00:00:00 2001 From: Yanzhang Wang Date: Tue, 5 Dec 2017 17:42:49 +0800 Subject: [PATCH 397/823] feat: SReLU and ActivityRegularization support (#1917) * feat: SReLU and ActivityRegularization support * fix: update doc * fix: refactor the weights and gradWeights of SReLU * fix: serialization for srelu * fix: backward computing errors * fix: ActivityRegularization to layer * fix: change test location * fix: add python api * fix: shared axes have not consider enough --- python/dllib/src/bigdl/dllib/nn/layer.py | 59 +++++++++++++++++++ .../dllib/src/bigdl/dllib/optim/optimizer.py | 10 ++++ 2 files changed, 69 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 60214316131..5e418952c63 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -3251,6 +3251,65 @@ def __init__(self, super(ReLU6, self).__init__(None, bigdl_type, inplace) +class SReLU(Layer): + + '''S-shaped Rectified Linear Unit. + + It follows: + `f(x) = t^r + a^r(x - t^r) for x >= t^r`, + `f(x) = x for t^r > x > t^l`, + `f(x) = t^l + a^l(x - t^l) for x <= t^l`. + + # References + - [Deep Learning with S-shaped Rectified Linear Activation Units](http://arxiv.org/abs/1512.07030) + + + + :param shared_axes: the axes along which to share learnable + parameters for the activation function. + For example, if the incoming feature maps + are from a 2D convolution + with output shape `(batch, height, width, channels)`, + and you wish to share parameters across space + so that each filter only has one set of parameters, + set `shared_axes=[1, 2]`. + + >>> srelu = SReLU() + creating: createSReLU + >>> srelu = SReLU((1, 2)) + creating: createSReLU + ''' + + def __init__(self, + share_axes=None, + bigdl_type="float"): + super(SReLU, self).__init__(None, bigdl_type, + share_axes) + + def set_init_method(self, tLeftInit = None, aLeftInit = None, + tRightInit = None, aRightInit = None): + callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, + tLeftInit, aLeftInit, tRightInit, aRightInit) + return self + +class ActivityRegularization(Layer): + + ''' + Layer that applies an update to the cost function based input activity. + + :param l1: L1 regularization factor (positive float). + :param l2: L2 regularization factor (positive float). + + + >>> ar = ActivityRegularization(0.1, 0.02) + creating: createActivityRegularization + ''' + + def __init__(self, + l1=0.0, + l2=0.0, + bigdl_type="float"): + super(ActivityRegularization, self).__init__(None, bigdl_type, l1, l2) class Replicate(Layer): diff --git a/python/dllib/src/bigdl/dllib/optim/optimizer.py b/python/dllib/src/bigdl/dllib/optim/optimizer.py index a817dd006cf..0b18022a994 100644 --- a/python/dllib/src/bigdl/dllib/optim/optimizer.py +++ b/python/dllib/src/bigdl/dllib/optim/optimizer.py @@ -818,6 +818,16 @@ class L1L2Regularizer(JavaValue): def __init__(self, l1, l2, bigdl_type="float"): JavaValue.__init__(self, None, bigdl_type, l1, l2) +class ActivityRegularization(JavaValue): + """ + Apply both L1 and L2 regularization + + :param l1 l1 regularization rate + :param l2 l2 regularization rate + + """ + def __init__(self, l1, l2, bigdl_type="float"): + JavaValue.__init__(self, None, bigdl_type, l1, l2) class L1Regularizer(JavaValue): """ From 8e0586faffc0829877d8bd26d08cfb77d481c781 Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Wed, 6 Dec 2017 08:52:43 +0800 Subject: [PATCH 398/823] add keras wrapper; code refactor (#1953) * TimeDistributed & Bidirectional * Code refactor of LayerConverter --- .../dllib/src/bigdl/dllib/keras/converter.py | 842 +++++++++--------- 1 file changed, 443 insertions(+), 399 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/keras/converter.py b/python/dllib/src/bigdl/dllib/keras/converter.py index 0061eefd97a..8626497a8a2 100644 --- a/python/dllib/src/bigdl/dllib/keras/converter.py +++ b/python/dllib/src/bigdl/dllib/keras/converter.py @@ -107,13 +107,13 @@ def get_converter(class_name): @staticmethod # weights is a list of ndarray or a ndarray # convert keras weights per layer to bigdl format - def to_bigdl_weights(class_name, weights): - return WeightsConverter.get_converter(class_name)(weights) + def to_bigdl_weights(klayer, weights): + return WeightsConverter.get_converter(klayer.__class__.__name__)(klayer, weights) @staticmethod def get_bigdl_weights_from_klayer(klayer): # we should use get_weights instead of klayer.weights - return WeightsConverter.to_bigdl_weights(klayer.__class__.__name__, klayer.get_weights()) + return WeightsConverter.to_bigdl_weights(klayer, klayer.get_weights()) @staticmethod def get_weights_from_kmodel(kmodel): @@ -126,36 +126,48 @@ def get_weights_from_kmodel(kmodel): layers_with_weights = [layer for layer in kmodel.layers if layer.weights] bweights = [] for klayer in layers_with_weights: - # bws would be [weiths, bias] or [weights] + # bws would be [weights, bias] or [weights] bws = WeightsConverter.get_bigdl_weights_from_klayer(klayer) for w in bws: bweights.append(w) return bweights @staticmethod - def convert_dense(weights): + def convert_timedistributed(klayer, weights): + return WeightsConverter.to_bigdl_weights(klayer.layer, weights) + + @staticmethod + def convert_bidirectional(klayer, weights): + kweights_forward = weights[:int(len(weights) / 2)] + kweights_backward = weights[int(len(weights) / 2):] + bweights_forward = WeightsConverter.to_bigdl_weights(klayer.layer, kweights_forward) + bweights_backward = WeightsConverter.to_bigdl_weights(klayer.layer, kweights_backward) + return bweights_forward + bweights_backward + + @staticmethod + def convert_dense(klayer, weights): return [np.transpose(weights[0]), weights[1]] @staticmethod - def convert_timedistributeddense(weights): + def convert_timedistributeddense(klayer, weights): return [np.transpose(weights[0]), weights[1]] @staticmethod - def convert_batchnormalization(weights): + def convert_batchnormalization(klayer, weights): gamma = weights[0] beta = weights[1] return [gamma, beta] @staticmethod - def convert_atrousconvolution2d(weights): + def convert_atrousconvolution2d(klayer, weights): return weights @staticmethod - def convert_atrousconvolution1d(weights): + def convert_atrousconvolution1d(klayer, weights): return [np.transpose(weights[0], (3, 2, 0, 1)), weights[1]] @staticmethod - def convert_deconvolution2d(weights): + def convert_deconvolution2d(klayer, weights): w = np.transpose(weights[0], (1, 0, 2, 3)) weight = np.expand_dims(w, 0) if len(weights) > 1: @@ -164,7 +176,7 @@ def convert_deconvolution2d(weights): return [weight] @staticmethod - def convert_convolution2d(weights): + def convert_convolution2d(klayer, weights): weight = np.expand_dims(weights[0], 0) # bigdl has a leading dim with value 1 if len(weights) > 1: return [weight, weights[1]] @@ -172,37 +184,37 @@ def convert_convolution2d(weights): return [weight] @staticmethod - def convert_convolution1d(weights): - return WeightsConverter.convert_convolution2d(weights) + def convert_convolution1d(klayer, weights): + return WeightsConverter.convert_convolution2d(klayer, weights) @staticmethod - def convert_convolution3d(weights): + def convert_convolution3d(klayer, weights): return weights @staticmethod - def convert_embedding(weights): + def convert_embedding(klayer, weights): return weights @staticmethod - def convert_simplernn(weights): + def convert_simplernn(klayer, weights): return [np.transpose(weights[0]), np.transpose(weights[1]), weights[2]] @staticmethod - def convert_lstm(weights): + def convert_lstm(klayer, weights): w1 = np.concatenate((weights[0].T, weights[3].T, weights[6].T, weights[9].T)) w2 = np.concatenate((weights[2], weights[5], weights[8], weights[11])) w3 = np.concatenate((weights[1].T, weights[4].T, weights[7].T, weights[10].T)) return [w1, w2, w3] @staticmethod - def convert_convlstm2d(weights): + def convert_convlstm2d(klayer, weights): return [np.expand_dims(weights[6], 0), weights[8], np.expand_dims(weights[7], 0), np.expand_dims(weights[0], 0), weights[2], np.expand_dims(weights[1], 0), np.expand_dims(weights[3], 0), weights[5], np.expand_dims(weights[4], 0), np.expand_dims(weights[9], 0), weights[11], np.expand_dims(weights[10], 0)] @staticmethod - def convert_gru(weights): + def convert_gru(klayer, weights): w1 = np.concatenate((weights[3].T, weights[0].T, weights[6].T)) w2 = np.concatenate((weights[5], weights[2], weights[8])) w3 = np.concatenate((weights[4].T, weights[1].T)) @@ -261,8 +273,8 @@ def __to_bigdl(self): elif isinstance(self.kmodel, Model): bigdlmodel = self._construct_bigdl_model() elif isinstance(self.kmodel, Layer): - bigdlmodel = LayerConverter().create(self.kmodel, - self.node_id_to_config_layer[self.kmodel.name]) + bigdlmodel = LayerConverter(self.kmodel, + self.node_id_to_config_layer[self.kmodel.name]).create() else: raise Exception("Should not enter here: %s" % self.kmodel) return bigdlmodel @@ -298,7 +310,7 @@ def _do_create_node(self, layer, clayer): self.node_id_to_config_layer[out_name]) bigdl_in_nodes.append(self.node_id_to_instance[out_name]) - blayer = LayerConverter().create(layer, clayer) + blayer = LayerConverter(layer, clayer).create() new_bnode = blayer(bigdl_in_nodes) self.node_id_to_instance[layer.name] = new_bnode return new_bnode @@ -320,16 +332,27 @@ def _construct_bigdl_model(self): def _construct_bigdl_sequence(self): bseq = BLayer.Sequential() - layerConverter = LayerConverter() for layer in self.kmodel.layers: # recursive logic is within create method. - blayer = layerConverter.create(layer, self.node_id_to_config_layer[layer.name]) + blayer = LayerConverter(layer, self.node_id_to_config_layer[layer.name]).create() bseq.add(blayer) return bseq class LayerConverter: - def __check_is_share_weights(self, kclayer): + def __init__(self, klayer, kclayer, input_shape=None): + self.klayer = klayer + self.kclayer = kclayer + if "config" in kclayer: + self.config = kclayer["config"] + else: + self.config = {} + if not input_shape: + self.input_shape = klayer.get_input_shape_at(0) + else: + self.input_shape = input_shape + + def __check_is_share_weights(self): # For Merge layer len(kclayer["inbound_nodes"]) is equal to 1 # "inbound_nodes": [ # [ @@ -355,108 +378,127 @@ def __check_is_share_weights(self, kclayer): # ] # ] # ], - if "inbound_nodes" in kclayer and len(kclayer["inbound_nodes"]) > 1: + if "inbound_nodes" in self.kclayer and len(self.kclayer["inbound_nodes"]) > 1: raise Exception( - "%s doesn't support multiple inputs with shared weights" % kclayer["class_name"]) + "%s doesn't support multiple inputs with shared weights" % self.kclayer["class_name"]) - def create(self, klayer, kclayer): - class_name = klayer.__class__.__name__ + def create(self): + class_name = self.klayer.__class__.__name__ - self.__check_is_share_weights(kclayer) + self.__check_is_share_weights() - if (hasattr(klayer, "b_constraint") and klayer.b_constraint) or \ - (hasattr(klayer, "W_constraint") and klayer.W_constraint): + if (hasattr(self.klayer, "b_constraint") and self.klayer.b_constraint) or \ + (hasattr(self.klayer, "W_constraint") and self.klayer.W_constraint): raise Exception("We don't support constraint for now") - if (hasattr(klayer, "activity_regularizer") and klayer.activity_regularizer): + if (hasattr(self.klayer, "activity_regularizer") and self.klayer.activity_regularizer): raise Exception("We don't support activity_regularizer for now") function_name = "create_" + class_name.lower() if not hasattr(self, function_name): raise Exception("We don't support layer: %s for now" % class_name ) - api = getattr(self, function_name) - blayer = api(klayer, kclayer) - return blayer.set_name(klayer.name) + blayer_creator = getattr(self, function_name) + blayer = blayer_creator() + return blayer.set_name(self.klayer.name) - def create_model(self, klayer, kclayer): - return DefinitionLoader.from_kmodel(klayer) + def create_model(self): + return DefinitionLoader.from_kmodel(self.klayer) - def create_sequential(self, klayer, kclayer): - return DefinitionLoader.from_kmodel(klayer) + def create_sequential(self): + return DefinitionLoader.from_kmodel(self.klayer) - def create_inputlayer(self, klayer, kclayer): + def create_inputlayer(self): return BLayer.Identity() - def create_dense(self, klayer, kclayer): - config = kclayer["config"] + def create_dense(self): # Multiple inputs should share the same input_dim for Dense layer # We don't need to respect the tensor index for method `get_input_shape_at` # which is internal implementation and `get_input_shape_at` has hided that for us, # What we need to use is the input index, not node index, not tensor index. - input_shape = klayer.get_input_shape_at(0) + if len(self.input_shape) > 2: + raise Exception("Input for Dense must be 1D or 2D") blayer = BLayer.Linear( - input_size=int(input_shape[1]), - output_size=config["output_dim"], - with_bias=config["bias"], - wRegularizer=self.to_bigdl_reg(config["W_regularizer"]), - bRegularizer=self.to_bigdl_reg(config["b_regularizer"]) + input_size=int(self.input_shape[-1]), # last dim is input_dim + output_size=self.config["output_dim"], + with_bias=self.config["bias"], + wRegularizer=self.to_bigdl_reg(self.config["W_regularizer"]), + bRegularizer=self.to_bigdl_reg(self.config["b_regularizer"]) ) - return self.combo_parameter_layer(blayer, config) + return self.combo_parameter_layer(blayer, self.config) - def create_timedistributeddense(self, klayer, kclayer): - config = kclayer["config"] - input_shape = klayer.get_input_shape_at(0) + def create_timedistributeddense(self): blayer = BLayer.TimeDistributed(BLayer.Linear( - input_size=int(input_shape[2]), - output_size=config["output_dim"], - with_bias=config["bias"], - wRegularizer=self.to_bigdl_reg(config["W_regularizer"]), - bRegularizer=self.to_bigdl_reg(config["b_regularizer"]) + input_size=int(self.input_shape[-1]), + output_size=self.config["output_dim"], + with_bias=self.config["bias"], + wRegularizer=self.to_bigdl_reg(self.config["W_regularizer"]), + bRegularizer=self.to_bigdl_reg(self.config["b_regularizer"]) )) - return self.combo_parameter_layer(blayer, config) + return self.combo_parameter_layer(blayer, self.config) + + def create_timedistributed(self): + # input_shape is (batch, time, other dims) + inner_input_shape = (self.input_shape[0], ) + self.input_shape[2:] + blayer = LayerConverter(self.klayer.layer, self.config['layer'], inner_input_shape).create() + return BLayer.TimeDistributed(blayer) + + def create_bidirectional(self): + if not self.klayer.layer.return_sequences: + raise Exception("Only return_sequences=True is supported for RNNs for now") + recurrent_name = "generate_" + self.klayer.layer.__class__.__name__.lower() + "_cell" + + recurrent_creator = getattr(self, recurrent_name) + recurrent = recurrent_creator(self.klayer.layer, self.config['layer'], self.input_shape) + if self.klayer.merge_mode == "concat": + merge = BLayer.JoinTable(len(self.input_shape) - 1, len(self.input_shape) - 1) + elif self.klayer.merge_mode == "sum": + merge = BLayer.CAddTable() + elif self.klayer.merge_mode == "mul": + merge = BLayer.CMulTable() + elif self.klayer.merge_mode == "ave": + merge = BLayer.CAveTable() + else: + raise Exception("Invalid merge mode: %s" % self.klayer.merge_mode) + blayer = BLayer.BiRecurrent(merge).add(recurrent) + return blayer - def create_embedding(self, klayer, kclayer): - config = kclayer["config"] - input_shape = klayer.get_input_shape_at(0) # batch, seq_len - seq_len = int(input_shape[1]) - if klayer.input_length and klayer.input_length != seq_len: + def create_embedding(self): + seq_len = int(self.input_shape[1]) + if self.klayer.input_length and self.klayer.input_length != seq_len: raise Exception( - "The input_length doesn't match: %s vs %s" % (seq_len, klayer.input_length)) + "The input_length doesn't match: %s vs %s" % (seq_len, self.klayer.input_length)) - if (hasattr(klayer, "dropout") and klayer.dropout != 0): + if (hasattr(self.klayer, "dropout") and self.klayer.dropout != 0): raise Exception("We don't support dropout for now") - if (hasattr(klayer, "mask_zero") and klayer.mask_zero != False): + if (hasattr(self.klayer, "mask_zero") and self.klayer.mask_zero != False): raise Exception("We don't support mask_zero for now") bseq = BLayer.Sequential() blayer = BLayer.LookupTable( - n_index = klayer.input_dim, - n_output = klayer.output_dim, + n_index=self.klayer.input_dim, + n_output=self.klayer.output_dim, padding_value=0.0, norm_type=2.0, should_scale_grad_by_freq=False, - wRegularizer= self.to_bigdl_reg(config["W_regularizer"]), + wRegularizer=self.to_bigdl_reg(self.config["W_regularizer"]), bigdl_type="float") - bseq.add(BLayer.AddConstant(1.0, inplace=True)) # Add 1 as BigDL is one-based index + bseq.add(BLayer.AddConstant(1.0, inplace=True)) # Add 1 as BigDL is one-based index bseq.add(blayer) return bseq - def create_activation(self, klayer, kclayer): - config = kclayer["config"] - return get_activation_by_name(config["activation"], klayer.name) + def create_activation(self): + return get_activation_by_name(self.config["activation"], self.klayer.name) - def create_dropout(self, klayer, kclayer): - return BLayer.Dropout(klayer.p) + def create_dropout(self): + return BLayer.Dropout(self.klayer.p) - def create_flatten(self, klayer, kclayer): - input_shape = klayer.input_shape - blayer = BLayer.Reshape([int(np.prod(input_shape[1:]))], None) - return blayer + def create_flatten(self): + return BLayer.Reshape([int(np.prod(self.input_shape[1:]))], None) - def create_permute(self, klayer, kclayer): - swaps = self.__perm_to_pair(list(klayer.dims)) + def create_permute(self): + swaps = self.__perm_to_pair(list(self.klayer.dims)) swaps.reverse() swaps = map(lambda pair: (pair[0]+1, pair[1]+1), swaps) return BLayer.Transpose(list(swaps)) @@ -494,64 +536,64 @@ def exchangeNumbers(arr, i, j): return list(filter(lambda pair: pair[0] != pair[1], pairs)) - def create_reshape(self, klayer, kclayer): - blayer = BLayer.Reshape(klayer.target_shape, None) + def create_reshape(self): + blayer = BLayer.Reshape(self.klayer.target_shape, None) return blayer - def create_repeatvector(self, klayer, kclayer): - return BLayer.Replicate(n_features=klayer.n, + def create_repeatvector(self): + return BLayer.Replicate(n_features=self.klayer.n, n_dim=1, bigdl_type="float") - def __is_from_sequential(self, klayer, kclayer): - return "layers" in kclayer["config"] and hasattr(klayer, "layers") and klayer.layers is not None # noqa + def __is_from_sequential(self): + return "layers" in self.kclayer["config"] and hasattr(self.klayer, "layers") and self.klayer.layers is not None # noqa - def create_merge(self, klayer, kclayer): - input_shape = klayer.get_input_shape_at(0) - if klayer.output_shape and not isinstance(klayer.output_shape, tuple): + def create_merge(self): + if self.klayer.output_shape and not isinstance(self.klayer.output_shape, tuple): raise Exception("Only output_shape=None or a shape tuple is supported for now") - if klayer.node_indices and not all(0 == i for i in klayer.node_indices): + if self.klayer.node_indices and not all(0 == i for i in self.klayer.node_indices): unsupport_exp("node_indices") - if klayer.output_mask: + if self.klayer.output_mask: unsupport_exp("output_mask") - if klayer.mode == "concat": + if self.klayer.mode == "concat": blayer = BLayer.JoinTable( - dimension=klayer.concat_axis, - n_input_dims=len(input_shape[0]) - 1, + dimension=self.klayer.concat_axis, + n_input_dims=len(self.input_shape[0]) - 1, bigdl_type="float") - elif klayer.mode == "sum": + elif self.klayer.mode == "sum": blayer = BLayer.CAddTable( inplace=False, bigdl_type="float") - elif klayer.mode == "mul": + elif self.klayer.mode == "mul": blayer = BLayer.CMulTable(bigdl_type="float") - elif klayer.mode == "max": + elif self.klayer.mode == "max": blayer = BLayer.CMaxTable(bigdl_type="float") - elif klayer.mode == "dot": - if len(input_shape[0]) >= 3: + elif self.klayer.mode == "dot": + if len(self.input_shape[0]) >= 3: raise Exception("For merge mode dot, 3D input or above is not supported for now.") - if klayer.dot_axes != [1, 1]: + if self.klayer.dot_axes != [1, 1]: raise Exception("For merge mode dot, only dot_axes=1 is supported for now.") model = BLayer.Sequential() blayer = model.add(BLayer.DotProduct(bigdl_type="float"))\ .add(BLayer.Reshape([1], True)) - elif klayer.mode == "ave": + elif self.klayer.mode == "ave": blayer = BLayer.CAveTable( inplace=False, bigdl_type="float") - elif klayer.mode in ['cos']: - if len(input_shape[0]) >= 3: + elif self.klayer.mode in ['cos']: + if len(self.input_shape[0]) >= 3: raise Exception("For merge mode cos, 3D input or above is not supported for now.") - if klayer.dot_axes != [1, 1]: + if self.klayer.dot_axes != [1, 1]: raise Exception("For merge mode cos, only dot_axes=1 is supported for now.") blayer = BLayer.Sequential() blayer.add(BLayer.CosineDistance(bigdl_type="float")).add(BLayer.Reshape([1, 1], True)) else: # invalid mode or lambda functions - raise Exception("Invalid merge mode: `%s`. Lambda/function as merge mode is not supported for now." % klayer.mode) - if self.__is_from_sequential(klayer, kclayer): + raise Exception("Invalid merge mode: `%s`. Lambda/function as merge mode is not supported for now." + % self.klayer.mode) + if self.__is_from_sequential(): bseq = BLayer.Sequential() parallel_table = BLayer.ParallelTable() - for l in klayer.layers: + for l in self.klayer.layers: bl = DefinitionLoader.from_kmodel(l) parallel_table.add(bl) bseq.add(parallel_table) @@ -560,24 +602,24 @@ def create_merge(self, klayer, kclayer): else: return blayer - def create_elu(self, klayer, kclayer): - return BLayer.ELU(alpha=float(klayer.alpha), + def create_elu(self): + return BLayer.ELU(alpha=float(self.klayer.alpha), inplace=False, bigdl_type="float") - def create_prelu(self, klayer, kclayer): + def create_prelu(self): return BLayer.PReLU(n_output_plane=0, bigdl_type="float") - def create_leakyrelu(self, klayer, kclayer): - return BLayer.LeakyReLU(negval=float(klayer.alpha), + def create_leakyrelu(self): + return BLayer.LeakyReLU(negval=float(self.klayer.alpha), inplace=False, bigdl_type="float") - def create_parametricsoftplus(self, klayer, kclayer): - alpha = float(klayer.alpha_init) - beta = float(klayer.beta_init) - if klayer.shared_axes != [None]: + def create_parametricsoftplus(self): + alpha = float(self.klayer.alpha_init) + beta = float(self.klayer.beta_init) + if self.klayer.shared_axes != [None]: unsupport_exp("shared_axes") if round(alpha * beta, 4) == 1.0: return BLayer.SoftPlus(beta=beta, @@ -585,8 +627,8 @@ def create_parametricsoftplus(self, klayer, kclayer): else: raise Exception("Only alpha_init = 1/beta_init is supported for now") - def create_thresholdedrelu(self, klayer, kclayer): - return BLayer.Threshold(th=float(klayer.theta), + def create_thresholdedrelu(self): + return BLayer.Threshold(th=float(self.klayer.theta), v=0.0, ip=False, bigdl_type="float") @@ -598,8 +640,8 @@ def __generate_zeropadding1d(self, pad_top, pad_bottom): pad_bottom=pad_bottom, bigdl_type="float") - def create_zeropadding1d(self, klayer, kclayer): - padding = klayer.padding + def create_zeropadding1d(self): + padding = self.klayer.padding if isinstance(padding, int): return self.__generate_zeropadding1d(padding, padding) elif isinstance(padding, dict): @@ -641,66 +683,64 @@ def __generate_zeropadding2d(self, dim1, dim2, n_input_dim, pad1, pad2, pad3, pa return model # NB: zeropadding doesn't serialize dim_ording to json file - def create_zeropadding2d(self, klayer, kclayer): - padding = klayer.padding - input_shape = klayer.get_input_shape_at(0) + def create_zeropadding2d(self): + padding = self.klayer.padding dim = 1 - if klayer.dim_ordering == "th": + if self.klayer.dim_ordering == "th": dim = 2 if isinstance(padding, dict): # dictionary - return self.__generate_zeropadding2d(dim, dim+1, len(input_shape) - 1, + return self.__generate_zeropadding2d(dim, dim+1, len(self.input_shape) - 1, -padding.get('top_pad', 0), padding.get('bottom_pad', 0), -padding.get('left_pad', 0), padding.get('right_pad', 0)) else: # tuple of int padding = tuple(padding) if len(padding) == 2: - return self.__generate_zeropadding2d(dim, dim+1, len(input_shape) - 1, + return self.__generate_zeropadding2d(dim, dim+1, len(self.input_shape) - 1, -padding[0], padding[0], -padding[1], padding[1]) elif len(padding) == 4: - return self.__generate_zeropadding2d(dim, dim+1, len(input_shape) - 1, + return self.__generate_zeropadding2d(dim, dim+1, len(self.input_shape) - 1, -padding[0], padding[1], -padding[2], padding[3]) # NB: zeropadding doesn't serialize dim_ording to json file - def create_zeropadding3d(self, klayer, kclayer): - padding = tuple(klayer.padding) - input_shape = klayer.get_input_shape_at(0) + def create_zeropadding3d(self): + padding = tuple(self.klayer.padding) dim = 1 - if klayer.dim_ordering == "th": + if self.klayer.dim_ordering == "th": dim = 2 model = BLayer.Sequential() paddinglayer1 = BLayer.Padding(dim=dim, pad=-padding[0], - n_input_dim=len(input_shape) - 1, + n_input_dim=len(self.input_shape) - 1, value=0.0, n_index=1, bigdl_type="float") paddinglayer2 = BLayer.Padding(dim=dim, pad=padding[0], - n_input_dim=len(input_shape) - 1, + n_input_dim=len(self.input_shape) - 1, value=0.0, n_index=1, bigdl_type="float") paddinglayer3 = BLayer.Padding(dim=dim+1, pad=-padding[1], - n_input_dim=len(input_shape) - 1, + n_input_dim=len(self.input_shape) - 1, value=0.0, n_index=1, bigdl_type="float") paddinglayer4 = BLayer.Padding(dim=dim+1, pad=padding[1], - n_input_dim=len(input_shape) - 1, + n_input_dim=len(self.input_shape) - 1, value=0.0, n_index=1, bigdl_type="float") paddinglayer5 = BLayer.Padding(dim=dim+2, pad=-padding[2], - n_input_dim=len(input_shape) - 1, + n_input_dim=len(self.input_shape) - 1, value=0.0, n_index=1, bigdl_type="float") paddinglayer6 = BLayer.Padding(dim=dim+2, pad=padding[2], - n_input_dim=len(input_shape) - 1, + n_input_dim=len(self.input_shape) - 1, value=0.0, n_index=1, bigdl_type="float") @@ -712,27 +752,34 @@ def create_zeropadding3d(self, klayer, kclayer): model.add(paddinglayer6) return model - def create_cropping1d(self, klayer, kclayer): - cropping = tuple(klayer.cropping) + def create_cropping1d(self): + cropping = tuple(self.klayer.cropping) return BLayer.SpatialZeroPadding(0, 0, -cropping[0], -cropping[1]) - def __return_sequences(self, return_sequences, blayer): - # For recurrent layers, handle whether to return the last output sentence or the full sequence. - if return_sequences: - return blayer - else: - model = BLayer.Sequential() - model.add(blayer) + def __check_recurrent_parameters(self, klayer): + if klayer.stateful: + raise Exception("Only stateful=False for recurrent layers is supported for now") + if hasattr(klayer, "consume_less") and klayer.consume_less == "gpu": + raise Exception("consume_less=gpu is not supported for now") + + def __process_recurrent_layer(self, return_sequences, go_backwards, blayer): + # For recurrent layers, + # handle whether to return the last output sentence or the full sequence; + # handle whether the input will go backwards + model = BLayer.Sequential() + if go_backwards: + model.add(BLayer.Reverse(2)) + model.add(blayer) + if not return_sequences: model.add(BLayer.Select(2, -1)) - return model + return model - def create_simplernn(self, klayer, kclayer): - rec = BLayer.Recurrent() - input_shape = klayer.get_input_shape_at(0) + def generate_simplernn_cell(self, klayer, kclayer, input_shape): # create a simplernn cell only + self.__check_recurrent_parameters(klayer) config = kclayer["config"] self.check_constraint_in_config(config) activation = get_activation_by_name(config["activation"], - "%s_%s" % (config["name"], config["activation"])) + "%s_%s" % (config["name"], config["activation"])) rnn = BLayer.RnnCell(input_size=int(input_shape[2]), hidden_size=klayer.output_dim, activation=activation, @@ -741,17 +788,22 @@ def create_simplernn(self, klayer, kclayer): uRegularizer=self.to_bigdl_reg(config["U_regularizer"]), bRegularizer=self.to_bigdl_reg(config["b_regularizer"]), bigdl_type="float") - return self.__return_sequences(klayer.return_sequences, rec.add(rnn)) + return rnn - def create_lstm(self, klayer, kclayer): + def create_simplernn(self): rec = BLayer.Recurrent() - input_shape = klayer.get_input_shape_at(0) + rnn = self.generate_simplernn_cell(self.klayer, self.kclayer, self.input_shape) + return self.__process_recurrent_layer(self.klayer.return_sequences, + self.klayer.go_backwards, rec.add(rnn)) + + def generate_lstm_cell(self, klayer, kclayer, input_shape): # create a lstm cell only + self.__check_recurrent_parameters(klayer) config = kclayer["config"] self.check_constraint_in_config(config) activation = get_activation_by_name(config["activation"], - "%s_%s" % (config["name"], config["activation"])) + "%s_%s" % (config["name"], config["activation"])) inner_activation = get_activation_by_name(config["inner_activation"], - "%s_%s" % (config["name"], config["inner_activation"])) + "%s_%s" % (config["name"], config["inner_activation"])) lstm = BLayer.LSTM(input_size=int(input_shape[2]), hidden_size=klayer.output_dim, p=0.0, @@ -761,55 +813,66 @@ def create_lstm(self, klayer, kclayer): uRegularizer=self.to_bigdl_reg(config["U_regularizer"]), bRegularizer=self.to_bigdl_reg(config["b_regularizer"]), bigdl_type="float") - return self.__return_sequences(klayer.return_sequences, rec.add(lstm)) + return lstm - def create_convlstm2d(self, klayer, kclayer): + def create_lstm(self): rec = BLayer.Recurrent() - input_shape = klayer.get_input_shape_at(0) + lstm = self.generate_lstm_cell(self.klayer, self.kclayer, self.input_shape) + return self.__process_recurrent_layer(self.klayer.return_sequences, + self.klayer.go_backwards, rec.add(lstm)) + + def generate_convlstm2d_cell(self, klayer, kclayer, input_shape): # create a convlstm2d cell only + self.__check_recurrent_parameters(klayer) config = kclayer["config"] self.check_constraint_in_config(config) activation = get_activation_by_name(config["activation"], - "%s_%s" % (config["name"], config["activation"])) + "%s_%s" % (config["name"], config["activation"])) inner_activation = get_activation_by_name(config["inner_activation"], - "%s_%s" % (config["name"], config["inner_activation"])) - - #TODO: border_mode = 'valid' - if config["border_mode"] != 'same': + "%s_%s" % (config["name"], config["inner_activation"])) + + convlstm = BLayer.ConvLSTMPeephole(input_size=int(input_shape[2]), + output_size=config["nb_filter"], + kernel_i=config["nb_col"], + kernel_c=config["nb_row"], + # NB: ConvLSTM doesn't serialize subsample to json file + stride=klayer.subsample[0], + padding=-1, + activation=activation, + inner_activation=inner_activation, + # NB: ConvLSTM doesn't serialize regularizers to json file + # wRegularizer=self.to_bigdl_reg(config["W_regularizer"]), + # uRegularizer=self.to_bigdl_reg(config["U_regularizer"]), + # bRegularizer=self.to_bigdl_reg(config["b_regularizer"]), + cRegularizer=None, + with_peephole=False, + bigdl_type="float") + return convlstm + + def create_convlstm2d(self): + # TODO: border_mode = 'valid' + if self.config["border_mode"] != 'same': raise Exception("Unsupported border_mode: valid") - if config["nb_row"] != config["nb_col"]: + if self.config["nb_row"] != self.config["nb_col"]: raise Exception("Only square kernel is supported for now. Please set nb_row=nb_col.") - if klayer.subsample[0] != klayer.subsample[1]: + if self.klayer.subsample[0] != self.klayer.subsample[1]: raise Exception("Only equal stride is supported for now. " "Please set subsample to be a tuple with equal values.") - blayer = BLayer.ConvLSTMPeephole(input_size=int(input_shape[2]), - output_size=config["nb_filter"], - kernel_i=config["nb_col"], - kernel_c=config["nb_row"], - # NB: ConvLSTM doesn't serialize subsample to json file - stride=klayer.subsample[0], - padding=-1, - activation=activation, - inner_activation=inner_activation, - # NB: ConvLSTM doesn't serialize regularizers to json file - # wRegularizer=self.to_bigdl_reg(config["W_regularizer"]), - # uRegularizer=self.to_bigdl_reg(config["U_regularizer"]), - # bRegularizer=self.to_bigdl_reg(config["b_regularizer"]), - cRegularizer=None, - with_peephole=False, - bigdl_type="float") - return self.__return_sequences(klayer.return_sequences, rec.add(blayer)) - - def create_gru(self, klayer, kclayer): rec = BLayer.Recurrent() - input_shape = klayer.get_input_shape_at(0) + convlstm = self.generate_convlstm2d_cell(self.klayer, + self.kclayer, self.input_shape) + return self.__process_recurrent_layer(self.klayer.return_sequences, + self.klayer.go_backwards, rec.add(convlstm)) + + def generate_gru_cell(self, klayer, kclayer, input_shape): # create a gru cell only + self.__check_recurrent_parameters(klayer) config = kclayer["config"] self.check_constraint_in_config(config) activation = get_activation_by_name(config["activation"], - "%s_%s" % (config["name"], config["activation"])) + "%s_%s" % (config["name"], config["activation"])) inner_activation = get_activation_by_name(config["inner_activation"], - "%s_%s" % (config["name"], config["inner_activation"])) + "%s_%s" % (config["name"], config["inner_activation"])) gru = BLayer.GRU(input_size=int(input_shape[2]), hidden_size=klayer.output_dim, p=0.0, @@ -819,36 +882,40 @@ def create_gru(self, klayer, kclayer): uRegularizer=self.to_bigdl_reg(config["U_regularizer"]), bRegularizer=self.to_bigdl_reg(config["b_regularizer"]), bigdl_type="float") - return self.__return_sequences(klayer.return_sequences, rec.add(gru)) + return gru - def create_batchnormalization(self, klayer, kclayer): - config = kclayer["config"] - if keras.backend.image_dim_ordering() != "th" or klayer.axis != 1: + def create_gru(self): + rec = BLayer.Recurrent() + gru = self.generate_gru_cell(self.klayer, self.kclayer, self.input_shape) + return self.__process_recurrent_layer(self.klayer.return_sequences, + self.klayer.go_backwards, rec.add(gru)) + + def create_batchnormalization(self): + if keras.backend.image_dim_ordering() != "th" or self.klayer.axis != 1: raise Exception("""For BatchNormalization, we only support th image ordering (i.e. NCHW) """ + """with axis = 1 for now, but the current order is %s and axis is %s - """ % (keras.backend.image_dim_ordering(), klayer.axis)) # noqa - if klayer.mode != 0: + """ % (keras.backend.image_dim_ordering(), self.klayer.axis)) # noqa + if self.klayer.mode != 0: raise Exception( - "Only support mode = 0 for now, but the current mode is: %s", klayer.mode) + "Only support mode = 0 for now, but the current mode is: %s", self.klayer.mode) - if config["gamma_regularizer"]: + if self.config["gamma_regularizer"]: raise Exception("We don't support gamma_regularizer for now") - if config["beta_regularizer"]: + if self.config["beta_regularizer"]: raise Exception("We don't support beta_regularizer for now") - input_shape = klayer.get_input_shape_at(0) - n_input_channel = int(input_shape[klayer.axis]) # default is -1 which is channel-last + n_input_channel = int(self.input_shape[self.klayer.axis]) # default is -1 which is channel-last # init gamma and beta # TODO: replace this with to_bigdl_init in the future - gamma = self.get_value_from_init(klayer.gamma_init.__name__, (n_input_channel,)) - beta = self.get_value_from_init(klayer.beta_init.__name__, (n_input_channel,)) + gamma = self.get_value_from_init(self.klayer.gamma_init.__name__, (n_input_channel,)) + beta = self.get_value_from_init(self.klayer.beta_init.__name__, (n_input_channel,)) blayer = BLayer.SpatialBatchNormalization( n_output=n_input_channel, - eps=klayer.epsilon, - momentum=klayer.momentum, + eps=self.klayer.epsilon, + momentum=self.klayer.momentum, affine=True, init_weight=gamma, init_bias=beta, @@ -856,22 +923,19 @@ def create_batchnormalization(self, klayer, kclayer): init_grad_bias=None, bigdl_type="float") - k_running_mean = keras.backend.eval(klayer.running_mean) - k_running_std = keras.backend.eval(klayer.running_std) + k_running_mean = keras.backend.eval(self.klayer.running_mean) + k_running_std = keras.backend.eval(self.klayer.running_std) blayer.set_running_mean(k_running_mean) blayer.set_running_std(k_running_std) return blayer - def get_bdim_order(self, kclayer): - return self.to_bigdl_2d_ordering(self.get_kdim_order(kclayer)) - - def get_kdim_order(self, kclayer): - config = kclayer["config"] - if "dim_ordering" in config: - return config["dim_ordering"] + def get_bdim_order(self): # get bigdl dim_ordering from keras dim_ordering + if "dim_ordering" in self.config: + order = self.config["dim_ordering"] else: - warnings.warn("Cannot find dim_ordering from json definition. Use default instead.") - return keras.backend.image_dim_ordering() + warnings.warn("Cannot find dim_ordering from json definition. Using the default instead.") + order = keras.backend.image_dim_ordering() + return self.to_bigdl_2d_ordering(order) def to_bigdl_2d_ordering(self, order): if order == "tf": @@ -879,7 +943,7 @@ def to_bigdl_2d_ordering(self, order): elif order == "th": return "NCHW" else: - raise Exception("Unsupport ordering: %s" % order) + raise Exception("Unsupported dim_ordering: %s" % order) def to_bigdl_3d_padding(self, border_mode): if border_mode == "valid": @@ -906,212 +970,200 @@ def to_bigdl_1d_padding(self, border_mode, kernel_w): else: raise Exception("Unsupported border mode: %s" % border_mode) - def create_convolution1d(self, klayer, kclayer): - config = kclayer["config"] - input_shape = klayer.get_input_shape_at(0) + def create_convolution1d(self): # batch, steps, dim, batch is None here, so you cannot use it directly. - stack_size = int(input_shape[2]) + stack_size = int(self.input_shape[2]) - bpadW, bpadH = self.to_bigdl_2d_padding(klayer.border_mode) + bpadW, bpadH = self.to_bigdl_2d_padding(self.klayer.border_mode) seq = BLayer.Sequential() - seq.add(BLayer.Reshape([int(input_shape[1]), 1, int(input_shape[2])], True)) + seq.add(BLayer.Reshape([int(self.input_shape[1]), 1, int(self.input_shape[2])], True)) blayer = BLayer.SpatialConvolution( n_input_plane=stack_size, - n_output_plane=klayer.nb_filter, + n_output_plane=self.klayer.nb_filter, kernel_w=1, - kernel_h=klayer.filter_length, + kernel_h=self.klayer.filter_length, stride_w=1, - stride_h=klayer.subsample_length, + stride_h=self.klayer.subsample_length, pad_w=bpadW, pad_h=bpadH, n_group=1, propagate_back=True, - wRegularizer=self.to_bigdl_reg(config["W_regularizer"]), - bRegularizer=self.to_bigdl_reg(config["b_regularizer"]), + wRegularizer=self.to_bigdl_reg(self.config["W_regularizer"]), + bRegularizer=self.to_bigdl_reg(self.config["b_regularizer"]), init_weight=None, init_bias=None, init_grad_weight=None, init_grad_bias=None, - with_bias=config["bias"], + with_bias=self.config["bias"], data_format="NHWC", bigdl_type="float") seq.add(blayer) seq.add(BLayer.Squeeze(3)) - return self.combo_parameter_layer(seq, config) + return self.combo_parameter_layer(seq, self.config) - def create_convolution2d(self, klayer, kclayer): - config = kclayer["config"] - bigdl_order = self.get_bdim_order(kclayer) - input_shape = klayer.get_input_shape_at(0) + def create_convolution2d(self): + bigdl_order = self.get_bdim_order() if bigdl_order == "NCHW": - stack_size = int(input_shape[1]) + stack_size = int(self.input_shape[1]) elif bigdl_order == "NHWC": - stack_size = int(input_shape[3]) + stack_size = int(self.input_shape[3]) - bpadW, bpadH = self.to_bigdl_2d_padding(klayer.border_mode) + bpadW, bpadH = self.to_bigdl_2d_padding(self.klayer.border_mode) blayer = BLayer.SpatialConvolution( n_input_plane=stack_size, - n_output_plane=klayer.nb_filter, - kernel_w=klayer.nb_col, - kernel_h=klayer.nb_row, - stride_w=klayer.subsample[0], - stride_h=klayer.subsample[1], + n_output_plane=self.klayer.nb_filter, + kernel_w=self.klayer.nb_col, + kernel_h=self.klayer.nb_row, + stride_w=self.klayer.subsample[0], + stride_h=self.klayer.subsample[1], pad_w=bpadW, pad_h=bpadH, n_group=1, propagate_back=True, - wRegularizer=self.to_bigdl_reg(config["W_regularizer"]), - bRegularizer=self.to_bigdl_reg(config["b_regularizer"]), + wRegularizer=self.to_bigdl_reg(self.config["W_regularizer"]), + bRegularizer=self.to_bigdl_reg(self.config["b_regularizer"]), init_weight=None, init_bias=None, init_grad_weight=None, init_grad_bias=None, - with_bias=config["bias"], + with_bias=self.config["bias"], data_format=bigdl_order, bigdl_type="float") - return self.combo_parameter_layer(blayer, config) + return self.combo_parameter_layer(blayer, self.config) - def create_convolution3d(self, klayer, kclayer): - config = kclayer["config"] - if klayer.dim_ordering != "th": - raise Exception("Please use `th` for `dim_ordering`. `%s` is not supported for now." % klayer.dim_ordering) - input_shape = klayer.get_input_shape_at(0) + def create_convolution3d(self): + if self.klayer.dim_ordering != "th": + raise Exception("Please use `th` for `dim_ordering`. `%s` is not supported for now." % self.klayer.dim_ordering) - bpadT, bpadW, bpadH = self.to_bigdl_3d_padding(klayer.border_mode) + bpadT, bpadW, bpadH = self.to_bigdl_3d_padding(self.klayer.border_mode) blayer = BLayer.VolumetricConvolution( - n_input_plane=int(input_shape[1]), - n_output_plane=klayer.nb_filter, - k_t=klayer.kernel_dim1, - k_w=klayer.kernel_dim3, - k_h=klayer.kernel_dim2, - d_t=klayer.subsample[0], - d_w=klayer.subsample[2], - d_h=klayer.subsample[1], + n_input_plane=int(self.input_shape[1]), + n_output_plane=self.klayer.nb_filter, + k_t=self.klayer.kernel_dim1, + k_w=self.klayer.kernel_dim3, + k_h=self.klayer.kernel_dim2, + d_t=self.klayer.subsample[0], + d_w=self.klayer.subsample[2], + d_h=self.klayer.subsample[1], pad_t=bpadT, pad_w=bpadW, pad_h=bpadH, - with_bias=config["bias"], - wRegularizer=self.to_bigdl_reg(config["W_regularizer"]), - bRegularizer=self.to_bigdl_reg(config["b_regularizer"]), + with_bias=self.config["bias"], + wRegularizer=self.to_bigdl_reg(self.config["W_regularizer"]), + bRegularizer=self.to_bigdl_reg(self.config["b_regularizer"]), bigdl_type="float") - return self.combo_parameter_layer(blayer, config) + return self.combo_parameter_layer(blayer, self.config) - def create_atrousconvolution1d(self, klayer, kclayer): - config = kclayer["config"] - if not config["bias"]: + def create_atrousconvolution1d(self): + if not self.config["bias"]: raise Exception("Please set `bias=True` for AtrousConvolution1D") - input_shape = klayer.get_input_shape_at(0) # TODO: border_mode=`same` - if klayer.border_mode == "same": - raise Exception("Unsupported border mode: %s" % klayer.border_mode) - bpadW, bpadH = self.to_bigdl_2d_padding(klayer.border_mode) + if self.klayer.border_mode == "same": + raise Exception("Unsupported border mode: %s" % self.klayer.border_mode) + bpadW, bpadH = self.to_bigdl_2d_padding(self.klayer.border_mode) seq = BLayer.Sequential() seq.add(BLayer.Transpose([(2, 3)])) - seq.add(BLayer.Reshape([int(input_shape[2]), int(input_shape[1]), 1], True)) + seq.add(BLayer.Reshape([int(self.input_shape[2]), int(self.input_shape[1]), 1], True)) blayer = BLayer.SpatialDilatedConvolution( - n_input_plane=int(input_shape[2]), - n_output_plane=config["nb_filter"], + n_input_plane=int(self.input_shape[2]), + n_output_plane=self.config["nb_filter"], kw=1, - kh=config["filter_length"], + kh=self.config["filter_length"], dw=1, - dh=config["subsample_length"], + dh=self.config["subsample_length"], pad_w=bpadW, pad_h=bpadH, dilation_w=1, - dilation_h=config["atrous_rate"], - wRegularizer=self.to_bigdl_reg(config["W_regularizer"]), - bRegularizer=self.to_bigdl_reg(config["b_regularizer"]), + dilation_h=self.config["atrous_rate"], + wRegularizer=self.to_bigdl_reg(self.config["W_regularizer"]), + bRegularizer=self.to_bigdl_reg(self.config["b_regularizer"]), bigdl_type="float") seq.add(blayer) seq.add(BLayer.Transpose([(2, 3)])) seq.add(BLayer.Squeeze(4)) - return self.combo_parameter_layer(seq, config) + return self.combo_parameter_layer(seq, self.config) - def create_atrousconvolution2d(self, klayer, kclayer): - config = kclayer["config"] - if klayer.dim_ordering != "th": - raise Exception("Please use `th` for `dim_ordering`. `%s` is not supported for now." % klayer.dim_ordering) - if not config["bias"]: + def create_atrousconvolution2d(self): + if self.klayer.dim_ordering != "th": + raise Exception("Please use `th` for `dim_ordering`. `%s` is not supported for now." % self.klayer.dim_ordering) + if not self.config["bias"]: raise Exception("Please set `bias=True` for AtrousConvolution2D") - input_shape = klayer.get_input_shape_at(0) # TODO: border_mode=`same` - if klayer.border_mode == "same": - raise Exception("Unsupported border mode: %s" % klayer.border_mode) - bpadW, bpadH = self.to_bigdl_2d_padding(klayer.border_mode) + if self.klayer.border_mode == "same": + raise Exception("Unsupported border mode: %s" % self.klayer.border_mode) + bpadW, bpadH = self.to_bigdl_2d_padding(self.klayer.border_mode) blayer = BLayer.SpatialDilatedConvolution( - n_input_plane=int(input_shape[1]), - n_output_plane=config["nb_filter"], - kw=config["nb_col"], - kh=config["nb_row"], - dw=config["subsample"][1], - dh=config["subsample"][0], + n_input_plane=int(self.input_shape[1]), + n_output_plane=self.config["nb_filter"], + kw=self.config["nb_col"], + kh=self.config["nb_row"], + dw=self.config["subsample"][1], + dh=self.config["subsample"][0], pad_w=bpadW, pad_h=bpadH, - dilation_w=config["atrous_rate"][1], - dilation_h=config["atrous_rate"][0], - wRegularizer=self.to_bigdl_reg(config["W_regularizer"]), - bRegularizer=self.to_bigdl_reg(config["b_regularizer"]), + dilation_w=self.config["atrous_rate"][1], + dilation_h=self.config["atrous_rate"][0], + wRegularizer=self.to_bigdl_reg(self.config["W_regularizer"]), + bRegularizer=self.to_bigdl_reg(self.config["b_regularizer"]), bigdl_type="float") - return self.combo_parameter_layer(blayer, config) + return self.combo_parameter_layer(blayer, self.config) - def create_deconvolution2d(self, klayer, kclayer): - config = kclayer["config"] - if klayer.dim_ordering != "th": - raise Exception("Please use `th` for `dim_ordering`. `%s` is not supported for now." % klayer.dim_ordering) - input_shape = klayer.get_input_shape_at(0) + def create_deconvolution2d(self): + if self.klayer.dim_ordering != "th": + raise Exception("Please use `th` for `dim_ordering`. `%s` is not supported for now." % self.klayer.dim_ordering) - bpadW, bpadH = self.to_bigdl_2d_padding(klayer.border_mode) + bpadW, bpadH = self.to_bigdl_2d_padding(self.klayer.border_mode) blayer = BLayer.SpatialFullConvolution( - n_input_plane=int(input_shape[1]), - n_output_plane=klayer.nb_filter, - kw=klayer.nb_col, - kh=klayer.nb_row, - dw=klayer.subsample[1], - dh=klayer.subsample[0], + n_input_plane=int(self.input_shape[1]), + n_output_plane=self.klayer.nb_filter, + kw=self.klayer.nb_col, + kh=self.klayer.nb_row, + dw=self.klayer.subsample[1], + dh=self.klayer.subsample[0], pad_w=bpadW, pad_h=bpadH, adj_w=0, adj_h=0, n_group=1, no_bias=False, - wRegularizer=self.to_bigdl_reg(config["W_regularizer"]), - bRegularizer=self.to_bigdl_reg(config["b_regularizer"]), + wRegularizer=self.to_bigdl_reg(self.config["W_regularizer"]), + bRegularizer=self.to_bigdl_reg(self.config["b_regularizer"]), bigdl_type="float") - return self.combo_parameter_layer(blayer, config) + return self.combo_parameter_layer(blayer, self.config) - def create_maxpooling3d(self, klayer, kclayer): - if klayer.dim_ordering != "th": - raise Exception("Please use `th` for `dim_ordering`. `%s` is not supported for now." % klayer.dim_ordering) - bpadT, bpadW, bpadH = self.to_bigdl_3d_padding(klayer.border_mode) + def create_maxpooling3d(self): + if self.klayer.dim_ordering != "th": + raise Exception("Please use `th` for `dim_ordering`. `%s` is not supported for now." % self.klayer.dim_ordering) + bpadT, bpadW, bpadH = self.to_bigdl_3d_padding(self.klayer.border_mode) blayer = BLayer.VolumetricMaxPooling( - k_t=klayer.pool_size[0], - k_w=klayer.pool_size[2], - k_h=klayer.pool_size[1], - d_t=klayer.strides[0], - d_w=klayer.strides[2], - d_h=klayer.strides[1], + k_t=self.klayer.pool_size[0], + k_w=self.klayer.pool_size[2], + k_h=self.klayer.pool_size[1], + d_t=self.klayer.strides[0], + d_w=self.klayer.strides[2], + d_h=self.klayer.strides[1], pad_t=bpadT, pad_w=bpadW, pad_h=bpadH, bigdl_type="float") return blayer - def create_maxpooling2d(self, klayer, kclayer): - bigdl_order = self.get_bdim_order(kclayer) - bpadW, bpadH = self.to_bigdl_2d_padding(klayer.border_mode) + def create_maxpooling2d(self): + bigdl_order = self.get_bdim_order() + bpadW, bpadH = self.to_bigdl_2d_padding(self.klayer.border_mode) blayer = BLayer.SpatialMaxPooling( - kw=klayer.pool_size[1], - kh=klayer.pool_size[0], - dw=klayer.strides[1], - dh=klayer.strides[0], + kw=self.klayer.pool_size[1], + kh=self.klayer.pool_size[0], + dw=self.klayer.strides[1], + dh=self.klayer.strides[0], pad_w=bpadW, pad_h=bpadH, to_ceil=False, @@ -1119,14 +1171,13 @@ def create_maxpooling2d(self, klayer, kclayer): bigdl_type="float") return blayer - def create_globalmaxpooling3d(self, klayer, kclayer): - input_shape = klayer.get_input_shape_at(0) - if klayer.dim_ordering == "th": - b_kt = int(input_shape[2]) - b_kw = int(input_shape[4]) - b_kh = int(input_shape[3]) + def create_globalmaxpooling3d(self): + if self.klayer.dim_ordering == "th": + b_kt = int(self.input_shape[2]) + b_kw = int(self.input_shape[4]) + b_kh = int(self.input_shape[3]) else: - raise Exception("Please use `th` for dim_ordering. `%s` is not supported for now." % klayer.dim_ordering) + raise Exception("Please use `th` for dim_ordering. `%s` is not supported for now." % self.klayer.dim_ordering) seq = BLayer.Sequential() blayer = BLayer.VolumetricMaxPooling( @@ -1148,14 +1199,13 @@ def create_globalmaxpooling3d(self, klayer, kclayer): return seq - def create_globalaveragepooling3d(self, klayer, kclayer): - input_shape = klayer.get_input_shape_at(0) - if klayer.dim_ordering == "th": - b_kt = int(input_shape[2]) - b_kw = int(input_shape[4]) - b_kh = int(input_shape[3]) + def create_globalaveragepooling3d(self): + if self.klayer.dim_ordering == "th": + b_kt = int(self.input_shape[2]) + b_kw = int(self.input_shape[4]) + b_kh = int(self.input_shape[3]) else: - raise Exception("Please use `th` for dim_ordering. `%s` is not supported for now." % klayer.dim_ordering) + raise Exception("Please use `th` for dim_ordering. `%s` is not supported for now." % self.klayer.dim_ordering) seq = BLayer.Sequential() blayer = BLayer.VolumetricAveragePooling( @@ -1178,14 +1228,14 @@ def create_globalaveragepooling3d(self, klayer, kclayer): return seq - def create_averagepooling2d(self, klayer, kclayer): - bigdl_order = self.get_bdim_order(kclayer) - bpadW, bpadH = self.to_bigdl_2d_padding(klayer.border_mode) + def create_averagepooling2d(self): + bigdl_order = self.get_bdim_order() + bpadW, bpadH = self.to_bigdl_2d_padding(self.klayer.border_mode) blayer = BLayer.SpatialAveragePooling( - kw=klayer.pool_size[1], - kh=klayer.pool_size[0], - dw=klayer.strides[1], - dh=klayer.strides[0], + kw=self.klayer.pool_size[1], + kh=self.klayer.pool_size[0], + dw=self.klayer.strides[1], + dh=self.klayer.strides[0], pad_w=bpadW, pad_h=bpadH, global_pooling=False, @@ -1197,17 +1247,17 @@ def create_averagepooling2d(self, klayer, kclayer): ) return blayer - def create_averagepooling3d(self, klayer, kclayer): - if klayer.dim_ordering != "th": - raise Exception("Please use `th` for `dim_ordering`. `%s` is not supported for now." % klayer.dim_ordering) - bpadT, bpadW, bpadH = self.to_bigdl_3d_padding(klayer.border_mode) + def create_averagepooling3d(self): + if self.klayer.dim_ordering != "th": + raise Exception("Please use `th` for `dim_ordering`. `%s` is not supported for now." % self.klayer.dim_ordering) + bpadT, bpadW, bpadH = self.to_bigdl_3d_padding(self.klayer.border_mode) blayer = BLayer.VolumetricAveragePooling( - k_t=klayer.pool_size[0], - k_w=klayer.pool_size[2], - k_h=klayer.pool_size[1], - d_t=klayer.strides[0], - d_w=klayer.strides[2], - d_h=klayer.strides[1], + k_t=self.klayer.pool_size[0], + k_w=self.klayer.pool_size[2], + k_h=self.klayer.pool_size[1], + d_t=self.klayer.strides[0], + d_w=self.klayer.strides[2], + d_h=self.klayer.strides[1], pad_t=bpadT, pad_w=bpadW, pad_h=bpadH, @@ -1215,15 +1265,14 @@ def create_averagepooling3d(self, klayer, kclayer): bigdl_type="float") return blayer - def create_globalmaxpooling2d(self, klayer, kclayer): - bigdl_order = self.get_bdim_order(kclayer) - input_shape = klayer.get_input_shape_at(0) + def create_globalmaxpooling2d(self): + bigdl_order = self.get_bdim_order() if bigdl_order == "NCHW": - b_kw = int(input_shape[3]) - b_kh = int(input_shape[2]) + b_kw = int(self.input_shape[3]) + b_kh = int(self.input_shape[2]) else: - b_kw = int(input_shape[2]) - b_kh = int(input_shape[1]) + b_kw = int(self.input_shape[2]) + b_kh = int(self.input_shape[1]) seq = BLayer.Sequential() blayer = BLayer.SpatialMaxPooling( @@ -1246,13 +1295,12 @@ def create_globalmaxpooling2d(self, klayer, kclayer): seq.add(BLayer.Squeeze(1, num_input_dims=2)) return seq - def create_globalmaxpooling1d(self, klayer, kclayer): - input_shape = klayer.get_input_shape_at(0) # batch, step, dim + def create_globalmaxpooling1d(self): b_kw = 1 - b_kh = int(input_shape[1]) + b_kh = int(self.input_shape[1]) seq = BLayer.Sequential() - seq.add(BLayer.View([int(input_shape[1]), 1, int(input_shape[2])], num_input_dims=2)) + seq.add(BLayer.View([int(self.input_shape[1]), 1, int(self.input_shape[2])], num_input_dims=2)) blayer = BLayer.SpatialMaxPooling( kw=b_kw, kh=b_kh, @@ -1269,13 +1317,12 @@ def create_globalmaxpooling1d(self, klayer, kclayer): seq.add(BLayer.Squeeze(1, num_input_dims=1)) return seq - def create_globalaveragepooling1d(self, klayer, kclayer): - input_shape = klayer.get_input_shape_at(0) # batch, step, dim + def create_globalaveragepooling1d(self): b_kw = 1 - b_kh = int(input_shape[1]) + b_kh = int(self.input_shape[1]) seq = BLayer.Sequential() - seq.add(BLayer.View([int(input_shape[1]), 1, int(input_shape[2])], num_input_dims=2)) + seq.add(BLayer.View([int(self.input_shape[1]), 1, int(self.input_shape[2])], num_input_dims=2)) blayer = BLayer.SpatialAveragePooling( kw=b_kw, kh=b_kh, @@ -1291,22 +1338,21 @@ def create_globalaveragepooling1d(self, klayer, kclayer): bigdl_type="float" ) seq.add(blayer) - seq.add(BLayer.Squeeze(2, num_input_dims=2)) # the index start from one but without batch + seq.add(BLayer.Squeeze(2, num_input_dims=2)) # the index start from one but without batch seq.add(BLayer.Squeeze(1, num_input_dims=1)) return seq - def create_maxpooling1d(self, klayer, kclayer): - input_shape = klayer.get_input_shape_at(0) # batch, steps, dim - bpadW, bpadH = self.to_bigdl_2d_padding(klayer.border_mode) + def create_maxpooling1d(self): + bpadW, bpadH = self.to_bigdl_2d_padding(self.klayer.border_mode) seq = BLayer.Sequential() - seq.add(BLayer.Reshape([int(input_shape[1]), 1, int(input_shape[2])], True)) + seq.add(BLayer.Reshape([int(self.input_shape[1]), 1, int(self.input_shape[2])], True)) blayer = BLayer.SpatialMaxPooling( kw=1, - kh=klayer.pool_length, + kh=self.klayer.pool_length, dw=1, - dh=klayer.stride, + dh=self.klayer.stride, pad_w=bpadW, pad_h=bpadH, to_ceil=False, @@ -1317,17 +1363,16 @@ def create_maxpooling1d(self, klayer, kclayer): seq.add(BLayer.Squeeze(3)) return seq - def create_averagepooling1d(self, klayer, kclayer): - input_shape = klayer.get_input_shape_at(0) # batch, steps, dim - bpadW, bpadH = self.to_bigdl_2d_padding(klayer.border_mode) + def create_averagepooling1d(self): + bpadW, bpadH = self.to_bigdl_2d_padding(self.klayer.border_mode) seq = BLayer.Sequential() - seq.add(BLayer.Reshape([int(input_shape[1]), 1, int(input_shape[2])], True)) + seq.add(BLayer.Reshape([int(self.input_shape[1]), 1, int(self.input_shape[2])], True)) blayer = BLayer.SpatialAveragePooling( kw=1, - kh=klayer.pool_length, + kh=self.klayer.pool_length, dw=1, - dh=klayer.stride, + dh=self.klayer.stride, pad_w=bpadW, pad_h=bpadH, global_pooling=False, @@ -1341,15 +1386,14 @@ def create_averagepooling1d(self, klayer, kclayer): seq.add(BLayer.Squeeze(3)) return seq - def create_globalaveragepooling2d(self, klayer, kclayer): - bigdl_order = self.get_bdim_order(kclayer) - input_shape = klayer.get_input_shape_at(0) + def create_globalaveragepooling2d(self): + bigdl_order = self.get_bdim_order() if bigdl_order == "NCHW": - b_kw = int(input_shape[3]) - b_kh = int(input_shape[2]) + b_kw = int(self.input_shape[3]) + b_kh = int(self.input_shape[2]) else: - b_kw = int(input_shape[2]) - b_kh = int(input_shape[1]) + b_kw = int(self.input_shape[2]) + b_kh = int(self.input_shape[1]) seq = BLayer.Sequential() blayer = BLayer.SpatialAveragePooling( @@ -1415,7 +1459,7 @@ def to_bigdl_init(self, kinit_method): # kinit_method is a string elif kinit_method == "zero": init = BInit.Zeros() else: - raise Exception("Unsupported init type: %s" % init) + raise Exception("Unsupported init type: %s" % kinit_method) return init def to_bigdl_reg(self, reg): # reg is a dict From 67db91d12453e110a28d1abfc8754b34e6f6ac11 Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Wed, 6 Dec 2017 08:52:43 +0800 Subject: [PATCH 399/823] add keras wrapper; code refactor (#1953) * TimeDistributed & Bidirectional * Code refactor of LayerConverter --- .../dllib/src/bigdl/dllib/keras/converter.py | 842 +++++++++--------- 1 file changed, 443 insertions(+), 399 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/keras/converter.py b/python/dllib/src/bigdl/dllib/keras/converter.py index 0061eefd97a..8626497a8a2 100644 --- a/python/dllib/src/bigdl/dllib/keras/converter.py +++ b/python/dllib/src/bigdl/dllib/keras/converter.py @@ -107,13 +107,13 @@ def get_converter(class_name): @staticmethod # weights is a list of ndarray or a ndarray # convert keras weights per layer to bigdl format - def to_bigdl_weights(class_name, weights): - return WeightsConverter.get_converter(class_name)(weights) + def to_bigdl_weights(klayer, weights): + return WeightsConverter.get_converter(klayer.__class__.__name__)(klayer, weights) @staticmethod def get_bigdl_weights_from_klayer(klayer): # we should use get_weights instead of klayer.weights - return WeightsConverter.to_bigdl_weights(klayer.__class__.__name__, klayer.get_weights()) + return WeightsConverter.to_bigdl_weights(klayer, klayer.get_weights()) @staticmethod def get_weights_from_kmodel(kmodel): @@ -126,36 +126,48 @@ def get_weights_from_kmodel(kmodel): layers_with_weights = [layer for layer in kmodel.layers if layer.weights] bweights = [] for klayer in layers_with_weights: - # bws would be [weiths, bias] or [weights] + # bws would be [weights, bias] or [weights] bws = WeightsConverter.get_bigdl_weights_from_klayer(klayer) for w in bws: bweights.append(w) return bweights @staticmethod - def convert_dense(weights): + def convert_timedistributed(klayer, weights): + return WeightsConverter.to_bigdl_weights(klayer.layer, weights) + + @staticmethod + def convert_bidirectional(klayer, weights): + kweights_forward = weights[:int(len(weights) / 2)] + kweights_backward = weights[int(len(weights) / 2):] + bweights_forward = WeightsConverter.to_bigdl_weights(klayer.layer, kweights_forward) + bweights_backward = WeightsConverter.to_bigdl_weights(klayer.layer, kweights_backward) + return bweights_forward + bweights_backward + + @staticmethod + def convert_dense(klayer, weights): return [np.transpose(weights[0]), weights[1]] @staticmethod - def convert_timedistributeddense(weights): + def convert_timedistributeddense(klayer, weights): return [np.transpose(weights[0]), weights[1]] @staticmethod - def convert_batchnormalization(weights): + def convert_batchnormalization(klayer, weights): gamma = weights[0] beta = weights[1] return [gamma, beta] @staticmethod - def convert_atrousconvolution2d(weights): + def convert_atrousconvolution2d(klayer, weights): return weights @staticmethod - def convert_atrousconvolution1d(weights): + def convert_atrousconvolution1d(klayer, weights): return [np.transpose(weights[0], (3, 2, 0, 1)), weights[1]] @staticmethod - def convert_deconvolution2d(weights): + def convert_deconvolution2d(klayer, weights): w = np.transpose(weights[0], (1, 0, 2, 3)) weight = np.expand_dims(w, 0) if len(weights) > 1: @@ -164,7 +176,7 @@ def convert_deconvolution2d(weights): return [weight] @staticmethod - def convert_convolution2d(weights): + def convert_convolution2d(klayer, weights): weight = np.expand_dims(weights[0], 0) # bigdl has a leading dim with value 1 if len(weights) > 1: return [weight, weights[1]] @@ -172,37 +184,37 @@ def convert_convolution2d(weights): return [weight] @staticmethod - def convert_convolution1d(weights): - return WeightsConverter.convert_convolution2d(weights) + def convert_convolution1d(klayer, weights): + return WeightsConverter.convert_convolution2d(klayer, weights) @staticmethod - def convert_convolution3d(weights): + def convert_convolution3d(klayer, weights): return weights @staticmethod - def convert_embedding(weights): + def convert_embedding(klayer, weights): return weights @staticmethod - def convert_simplernn(weights): + def convert_simplernn(klayer, weights): return [np.transpose(weights[0]), np.transpose(weights[1]), weights[2]] @staticmethod - def convert_lstm(weights): + def convert_lstm(klayer, weights): w1 = np.concatenate((weights[0].T, weights[3].T, weights[6].T, weights[9].T)) w2 = np.concatenate((weights[2], weights[5], weights[8], weights[11])) w3 = np.concatenate((weights[1].T, weights[4].T, weights[7].T, weights[10].T)) return [w1, w2, w3] @staticmethod - def convert_convlstm2d(weights): + def convert_convlstm2d(klayer, weights): return [np.expand_dims(weights[6], 0), weights[8], np.expand_dims(weights[7], 0), np.expand_dims(weights[0], 0), weights[2], np.expand_dims(weights[1], 0), np.expand_dims(weights[3], 0), weights[5], np.expand_dims(weights[4], 0), np.expand_dims(weights[9], 0), weights[11], np.expand_dims(weights[10], 0)] @staticmethod - def convert_gru(weights): + def convert_gru(klayer, weights): w1 = np.concatenate((weights[3].T, weights[0].T, weights[6].T)) w2 = np.concatenate((weights[5], weights[2], weights[8])) w3 = np.concatenate((weights[4].T, weights[1].T)) @@ -261,8 +273,8 @@ def __to_bigdl(self): elif isinstance(self.kmodel, Model): bigdlmodel = self._construct_bigdl_model() elif isinstance(self.kmodel, Layer): - bigdlmodel = LayerConverter().create(self.kmodel, - self.node_id_to_config_layer[self.kmodel.name]) + bigdlmodel = LayerConverter(self.kmodel, + self.node_id_to_config_layer[self.kmodel.name]).create() else: raise Exception("Should not enter here: %s" % self.kmodel) return bigdlmodel @@ -298,7 +310,7 @@ def _do_create_node(self, layer, clayer): self.node_id_to_config_layer[out_name]) bigdl_in_nodes.append(self.node_id_to_instance[out_name]) - blayer = LayerConverter().create(layer, clayer) + blayer = LayerConverter(layer, clayer).create() new_bnode = blayer(bigdl_in_nodes) self.node_id_to_instance[layer.name] = new_bnode return new_bnode @@ -320,16 +332,27 @@ def _construct_bigdl_model(self): def _construct_bigdl_sequence(self): bseq = BLayer.Sequential() - layerConverter = LayerConverter() for layer in self.kmodel.layers: # recursive logic is within create method. - blayer = layerConverter.create(layer, self.node_id_to_config_layer[layer.name]) + blayer = LayerConverter(layer, self.node_id_to_config_layer[layer.name]).create() bseq.add(blayer) return bseq class LayerConverter: - def __check_is_share_weights(self, kclayer): + def __init__(self, klayer, kclayer, input_shape=None): + self.klayer = klayer + self.kclayer = kclayer + if "config" in kclayer: + self.config = kclayer["config"] + else: + self.config = {} + if not input_shape: + self.input_shape = klayer.get_input_shape_at(0) + else: + self.input_shape = input_shape + + def __check_is_share_weights(self): # For Merge layer len(kclayer["inbound_nodes"]) is equal to 1 # "inbound_nodes": [ # [ @@ -355,108 +378,127 @@ def __check_is_share_weights(self, kclayer): # ] # ] # ], - if "inbound_nodes" in kclayer and len(kclayer["inbound_nodes"]) > 1: + if "inbound_nodes" in self.kclayer and len(self.kclayer["inbound_nodes"]) > 1: raise Exception( - "%s doesn't support multiple inputs with shared weights" % kclayer["class_name"]) + "%s doesn't support multiple inputs with shared weights" % self.kclayer["class_name"]) - def create(self, klayer, kclayer): - class_name = klayer.__class__.__name__ + def create(self): + class_name = self.klayer.__class__.__name__ - self.__check_is_share_weights(kclayer) + self.__check_is_share_weights() - if (hasattr(klayer, "b_constraint") and klayer.b_constraint) or \ - (hasattr(klayer, "W_constraint") and klayer.W_constraint): + if (hasattr(self.klayer, "b_constraint") and self.klayer.b_constraint) or \ + (hasattr(self.klayer, "W_constraint") and self.klayer.W_constraint): raise Exception("We don't support constraint for now") - if (hasattr(klayer, "activity_regularizer") and klayer.activity_regularizer): + if (hasattr(self.klayer, "activity_regularizer") and self.klayer.activity_regularizer): raise Exception("We don't support activity_regularizer for now") function_name = "create_" + class_name.lower() if not hasattr(self, function_name): raise Exception("We don't support layer: %s for now" % class_name ) - api = getattr(self, function_name) - blayer = api(klayer, kclayer) - return blayer.set_name(klayer.name) + blayer_creator = getattr(self, function_name) + blayer = blayer_creator() + return blayer.set_name(self.klayer.name) - def create_model(self, klayer, kclayer): - return DefinitionLoader.from_kmodel(klayer) + def create_model(self): + return DefinitionLoader.from_kmodel(self.klayer) - def create_sequential(self, klayer, kclayer): - return DefinitionLoader.from_kmodel(klayer) + def create_sequential(self): + return DefinitionLoader.from_kmodel(self.klayer) - def create_inputlayer(self, klayer, kclayer): + def create_inputlayer(self): return BLayer.Identity() - def create_dense(self, klayer, kclayer): - config = kclayer["config"] + def create_dense(self): # Multiple inputs should share the same input_dim for Dense layer # We don't need to respect the tensor index for method `get_input_shape_at` # which is internal implementation and `get_input_shape_at` has hided that for us, # What we need to use is the input index, not node index, not tensor index. - input_shape = klayer.get_input_shape_at(0) + if len(self.input_shape) > 2: + raise Exception("Input for Dense must be 1D or 2D") blayer = BLayer.Linear( - input_size=int(input_shape[1]), - output_size=config["output_dim"], - with_bias=config["bias"], - wRegularizer=self.to_bigdl_reg(config["W_regularizer"]), - bRegularizer=self.to_bigdl_reg(config["b_regularizer"]) + input_size=int(self.input_shape[-1]), # last dim is input_dim + output_size=self.config["output_dim"], + with_bias=self.config["bias"], + wRegularizer=self.to_bigdl_reg(self.config["W_regularizer"]), + bRegularizer=self.to_bigdl_reg(self.config["b_regularizer"]) ) - return self.combo_parameter_layer(blayer, config) + return self.combo_parameter_layer(blayer, self.config) - def create_timedistributeddense(self, klayer, kclayer): - config = kclayer["config"] - input_shape = klayer.get_input_shape_at(0) + def create_timedistributeddense(self): blayer = BLayer.TimeDistributed(BLayer.Linear( - input_size=int(input_shape[2]), - output_size=config["output_dim"], - with_bias=config["bias"], - wRegularizer=self.to_bigdl_reg(config["W_regularizer"]), - bRegularizer=self.to_bigdl_reg(config["b_regularizer"]) + input_size=int(self.input_shape[-1]), + output_size=self.config["output_dim"], + with_bias=self.config["bias"], + wRegularizer=self.to_bigdl_reg(self.config["W_regularizer"]), + bRegularizer=self.to_bigdl_reg(self.config["b_regularizer"]) )) - return self.combo_parameter_layer(blayer, config) + return self.combo_parameter_layer(blayer, self.config) + + def create_timedistributed(self): + # input_shape is (batch, time, other dims) + inner_input_shape = (self.input_shape[0], ) + self.input_shape[2:] + blayer = LayerConverter(self.klayer.layer, self.config['layer'], inner_input_shape).create() + return BLayer.TimeDistributed(blayer) + + def create_bidirectional(self): + if not self.klayer.layer.return_sequences: + raise Exception("Only return_sequences=True is supported for RNNs for now") + recurrent_name = "generate_" + self.klayer.layer.__class__.__name__.lower() + "_cell" + + recurrent_creator = getattr(self, recurrent_name) + recurrent = recurrent_creator(self.klayer.layer, self.config['layer'], self.input_shape) + if self.klayer.merge_mode == "concat": + merge = BLayer.JoinTable(len(self.input_shape) - 1, len(self.input_shape) - 1) + elif self.klayer.merge_mode == "sum": + merge = BLayer.CAddTable() + elif self.klayer.merge_mode == "mul": + merge = BLayer.CMulTable() + elif self.klayer.merge_mode == "ave": + merge = BLayer.CAveTable() + else: + raise Exception("Invalid merge mode: %s" % self.klayer.merge_mode) + blayer = BLayer.BiRecurrent(merge).add(recurrent) + return blayer - def create_embedding(self, klayer, kclayer): - config = kclayer["config"] - input_shape = klayer.get_input_shape_at(0) # batch, seq_len - seq_len = int(input_shape[1]) - if klayer.input_length and klayer.input_length != seq_len: + def create_embedding(self): + seq_len = int(self.input_shape[1]) + if self.klayer.input_length and self.klayer.input_length != seq_len: raise Exception( - "The input_length doesn't match: %s vs %s" % (seq_len, klayer.input_length)) + "The input_length doesn't match: %s vs %s" % (seq_len, self.klayer.input_length)) - if (hasattr(klayer, "dropout") and klayer.dropout != 0): + if (hasattr(self.klayer, "dropout") and self.klayer.dropout != 0): raise Exception("We don't support dropout for now") - if (hasattr(klayer, "mask_zero") and klayer.mask_zero != False): + if (hasattr(self.klayer, "mask_zero") and self.klayer.mask_zero != False): raise Exception("We don't support mask_zero for now") bseq = BLayer.Sequential() blayer = BLayer.LookupTable( - n_index = klayer.input_dim, - n_output = klayer.output_dim, + n_index=self.klayer.input_dim, + n_output=self.klayer.output_dim, padding_value=0.0, norm_type=2.0, should_scale_grad_by_freq=False, - wRegularizer= self.to_bigdl_reg(config["W_regularizer"]), + wRegularizer=self.to_bigdl_reg(self.config["W_regularizer"]), bigdl_type="float") - bseq.add(BLayer.AddConstant(1.0, inplace=True)) # Add 1 as BigDL is one-based index + bseq.add(BLayer.AddConstant(1.0, inplace=True)) # Add 1 as BigDL is one-based index bseq.add(blayer) return bseq - def create_activation(self, klayer, kclayer): - config = kclayer["config"] - return get_activation_by_name(config["activation"], klayer.name) + def create_activation(self): + return get_activation_by_name(self.config["activation"], self.klayer.name) - def create_dropout(self, klayer, kclayer): - return BLayer.Dropout(klayer.p) + def create_dropout(self): + return BLayer.Dropout(self.klayer.p) - def create_flatten(self, klayer, kclayer): - input_shape = klayer.input_shape - blayer = BLayer.Reshape([int(np.prod(input_shape[1:]))], None) - return blayer + def create_flatten(self): + return BLayer.Reshape([int(np.prod(self.input_shape[1:]))], None) - def create_permute(self, klayer, kclayer): - swaps = self.__perm_to_pair(list(klayer.dims)) + def create_permute(self): + swaps = self.__perm_to_pair(list(self.klayer.dims)) swaps.reverse() swaps = map(lambda pair: (pair[0]+1, pair[1]+1), swaps) return BLayer.Transpose(list(swaps)) @@ -494,64 +536,64 @@ def exchangeNumbers(arr, i, j): return list(filter(lambda pair: pair[0] != pair[1], pairs)) - def create_reshape(self, klayer, kclayer): - blayer = BLayer.Reshape(klayer.target_shape, None) + def create_reshape(self): + blayer = BLayer.Reshape(self.klayer.target_shape, None) return blayer - def create_repeatvector(self, klayer, kclayer): - return BLayer.Replicate(n_features=klayer.n, + def create_repeatvector(self): + return BLayer.Replicate(n_features=self.klayer.n, n_dim=1, bigdl_type="float") - def __is_from_sequential(self, klayer, kclayer): - return "layers" in kclayer["config"] and hasattr(klayer, "layers") and klayer.layers is not None # noqa + def __is_from_sequential(self): + return "layers" in self.kclayer["config"] and hasattr(self.klayer, "layers") and self.klayer.layers is not None # noqa - def create_merge(self, klayer, kclayer): - input_shape = klayer.get_input_shape_at(0) - if klayer.output_shape and not isinstance(klayer.output_shape, tuple): + def create_merge(self): + if self.klayer.output_shape and not isinstance(self.klayer.output_shape, tuple): raise Exception("Only output_shape=None or a shape tuple is supported for now") - if klayer.node_indices and not all(0 == i for i in klayer.node_indices): + if self.klayer.node_indices and not all(0 == i for i in self.klayer.node_indices): unsupport_exp("node_indices") - if klayer.output_mask: + if self.klayer.output_mask: unsupport_exp("output_mask") - if klayer.mode == "concat": + if self.klayer.mode == "concat": blayer = BLayer.JoinTable( - dimension=klayer.concat_axis, - n_input_dims=len(input_shape[0]) - 1, + dimension=self.klayer.concat_axis, + n_input_dims=len(self.input_shape[0]) - 1, bigdl_type="float") - elif klayer.mode == "sum": + elif self.klayer.mode == "sum": blayer = BLayer.CAddTable( inplace=False, bigdl_type="float") - elif klayer.mode == "mul": + elif self.klayer.mode == "mul": blayer = BLayer.CMulTable(bigdl_type="float") - elif klayer.mode == "max": + elif self.klayer.mode == "max": blayer = BLayer.CMaxTable(bigdl_type="float") - elif klayer.mode == "dot": - if len(input_shape[0]) >= 3: + elif self.klayer.mode == "dot": + if len(self.input_shape[0]) >= 3: raise Exception("For merge mode dot, 3D input or above is not supported for now.") - if klayer.dot_axes != [1, 1]: + if self.klayer.dot_axes != [1, 1]: raise Exception("For merge mode dot, only dot_axes=1 is supported for now.") model = BLayer.Sequential() blayer = model.add(BLayer.DotProduct(bigdl_type="float"))\ .add(BLayer.Reshape([1], True)) - elif klayer.mode == "ave": + elif self.klayer.mode == "ave": blayer = BLayer.CAveTable( inplace=False, bigdl_type="float") - elif klayer.mode in ['cos']: - if len(input_shape[0]) >= 3: + elif self.klayer.mode in ['cos']: + if len(self.input_shape[0]) >= 3: raise Exception("For merge mode cos, 3D input or above is not supported for now.") - if klayer.dot_axes != [1, 1]: + if self.klayer.dot_axes != [1, 1]: raise Exception("For merge mode cos, only dot_axes=1 is supported for now.") blayer = BLayer.Sequential() blayer.add(BLayer.CosineDistance(bigdl_type="float")).add(BLayer.Reshape([1, 1], True)) else: # invalid mode or lambda functions - raise Exception("Invalid merge mode: `%s`. Lambda/function as merge mode is not supported for now." % klayer.mode) - if self.__is_from_sequential(klayer, kclayer): + raise Exception("Invalid merge mode: `%s`. Lambda/function as merge mode is not supported for now." + % self.klayer.mode) + if self.__is_from_sequential(): bseq = BLayer.Sequential() parallel_table = BLayer.ParallelTable() - for l in klayer.layers: + for l in self.klayer.layers: bl = DefinitionLoader.from_kmodel(l) parallel_table.add(bl) bseq.add(parallel_table) @@ -560,24 +602,24 @@ def create_merge(self, klayer, kclayer): else: return blayer - def create_elu(self, klayer, kclayer): - return BLayer.ELU(alpha=float(klayer.alpha), + def create_elu(self): + return BLayer.ELU(alpha=float(self.klayer.alpha), inplace=False, bigdl_type="float") - def create_prelu(self, klayer, kclayer): + def create_prelu(self): return BLayer.PReLU(n_output_plane=0, bigdl_type="float") - def create_leakyrelu(self, klayer, kclayer): - return BLayer.LeakyReLU(negval=float(klayer.alpha), + def create_leakyrelu(self): + return BLayer.LeakyReLU(negval=float(self.klayer.alpha), inplace=False, bigdl_type="float") - def create_parametricsoftplus(self, klayer, kclayer): - alpha = float(klayer.alpha_init) - beta = float(klayer.beta_init) - if klayer.shared_axes != [None]: + def create_parametricsoftplus(self): + alpha = float(self.klayer.alpha_init) + beta = float(self.klayer.beta_init) + if self.klayer.shared_axes != [None]: unsupport_exp("shared_axes") if round(alpha * beta, 4) == 1.0: return BLayer.SoftPlus(beta=beta, @@ -585,8 +627,8 @@ def create_parametricsoftplus(self, klayer, kclayer): else: raise Exception("Only alpha_init = 1/beta_init is supported for now") - def create_thresholdedrelu(self, klayer, kclayer): - return BLayer.Threshold(th=float(klayer.theta), + def create_thresholdedrelu(self): + return BLayer.Threshold(th=float(self.klayer.theta), v=0.0, ip=False, bigdl_type="float") @@ -598,8 +640,8 @@ def __generate_zeropadding1d(self, pad_top, pad_bottom): pad_bottom=pad_bottom, bigdl_type="float") - def create_zeropadding1d(self, klayer, kclayer): - padding = klayer.padding + def create_zeropadding1d(self): + padding = self.klayer.padding if isinstance(padding, int): return self.__generate_zeropadding1d(padding, padding) elif isinstance(padding, dict): @@ -641,66 +683,64 @@ def __generate_zeropadding2d(self, dim1, dim2, n_input_dim, pad1, pad2, pad3, pa return model # NB: zeropadding doesn't serialize dim_ording to json file - def create_zeropadding2d(self, klayer, kclayer): - padding = klayer.padding - input_shape = klayer.get_input_shape_at(0) + def create_zeropadding2d(self): + padding = self.klayer.padding dim = 1 - if klayer.dim_ordering == "th": + if self.klayer.dim_ordering == "th": dim = 2 if isinstance(padding, dict): # dictionary - return self.__generate_zeropadding2d(dim, dim+1, len(input_shape) - 1, + return self.__generate_zeropadding2d(dim, dim+1, len(self.input_shape) - 1, -padding.get('top_pad', 0), padding.get('bottom_pad', 0), -padding.get('left_pad', 0), padding.get('right_pad', 0)) else: # tuple of int padding = tuple(padding) if len(padding) == 2: - return self.__generate_zeropadding2d(dim, dim+1, len(input_shape) - 1, + return self.__generate_zeropadding2d(dim, dim+1, len(self.input_shape) - 1, -padding[0], padding[0], -padding[1], padding[1]) elif len(padding) == 4: - return self.__generate_zeropadding2d(dim, dim+1, len(input_shape) - 1, + return self.__generate_zeropadding2d(dim, dim+1, len(self.input_shape) - 1, -padding[0], padding[1], -padding[2], padding[3]) # NB: zeropadding doesn't serialize dim_ording to json file - def create_zeropadding3d(self, klayer, kclayer): - padding = tuple(klayer.padding) - input_shape = klayer.get_input_shape_at(0) + def create_zeropadding3d(self): + padding = tuple(self.klayer.padding) dim = 1 - if klayer.dim_ordering == "th": + if self.klayer.dim_ordering == "th": dim = 2 model = BLayer.Sequential() paddinglayer1 = BLayer.Padding(dim=dim, pad=-padding[0], - n_input_dim=len(input_shape) - 1, + n_input_dim=len(self.input_shape) - 1, value=0.0, n_index=1, bigdl_type="float") paddinglayer2 = BLayer.Padding(dim=dim, pad=padding[0], - n_input_dim=len(input_shape) - 1, + n_input_dim=len(self.input_shape) - 1, value=0.0, n_index=1, bigdl_type="float") paddinglayer3 = BLayer.Padding(dim=dim+1, pad=-padding[1], - n_input_dim=len(input_shape) - 1, + n_input_dim=len(self.input_shape) - 1, value=0.0, n_index=1, bigdl_type="float") paddinglayer4 = BLayer.Padding(dim=dim+1, pad=padding[1], - n_input_dim=len(input_shape) - 1, + n_input_dim=len(self.input_shape) - 1, value=0.0, n_index=1, bigdl_type="float") paddinglayer5 = BLayer.Padding(dim=dim+2, pad=-padding[2], - n_input_dim=len(input_shape) - 1, + n_input_dim=len(self.input_shape) - 1, value=0.0, n_index=1, bigdl_type="float") paddinglayer6 = BLayer.Padding(dim=dim+2, pad=padding[2], - n_input_dim=len(input_shape) - 1, + n_input_dim=len(self.input_shape) - 1, value=0.0, n_index=1, bigdl_type="float") @@ -712,27 +752,34 @@ def create_zeropadding3d(self, klayer, kclayer): model.add(paddinglayer6) return model - def create_cropping1d(self, klayer, kclayer): - cropping = tuple(klayer.cropping) + def create_cropping1d(self): + cropping = tuple(self.klayer.cropping) return BLayer.SpatialZeroPadding(0, 0, -cropping[0], -cropping[1]) - def __return_sequences(self, return_sequences, blayer): - # For recurrent layers, handle whether to return the last output sentence or the full sequence. - if return_sequences: - return blayer - else: - model = BLayer.Sequential() - model.add(blayer) + def __check_recurrent_parameters(self, klayer): + if klayer.stateful: + raise Exception("Only stateful=False for recurrent layers is supported for now") + if hasattr(klayer, "consume_less") and klayer.consume_less == "gpu": + raise Exception("consume_less=gpu is not supported for now") + + def __process_recurrent_layer(self, return_sequences, go_backwards, blayer): + # For recurrent layers, + # handle whether to return the last output sentence or the full sequence; + # handle whether the input will go backwards + model = BLayer.Sequential() + if go_backwards: + model.add(BLayer.Reverse(2)) + model.add(blayer) + if not return_sequences: model.add(BLayer.Select(2, -1)) - return model + return model - def create_simplernn(self, klayer, kclayer): - rec = BLayer.Recurrent() - input_shape = klayer.get_input_shape_at(0) + def generate_simplernn_cell(self, klayer, kclayer, input_shape): # create a simplernn cell only + self.__check_recurrent_parameters(klayer) config = kclayer["config"] self.check_constraint_in_config(config) activation = get_activation_by_name(config["activation"], - "%s_%s" % (config["name"], config["activation"])) + "%s_%s" % (config["name"], config["activation"])) rnn = BLayer.RnnCell(input_size=int(input_shape[2]), hidden_size=klayer.output_dim, activation=activation, @@ -741,17 +788,22 @@ def create_simplernn(self, klayer, kclayer): uRegularizer=self.to_bigdl_reg(config["U_regularizer"]), bRegularizer=self.to_bigdl_reg(config["b_regularizer"]), bigdl_type="float") - return self.__return_sequences(klayer.return_sequences, rec.add(rnn)) + return rnn - def create_lstm(self, klayer, kclayer): + def create_simplernn(self): rec = BLayer.Recurrent() - input_shape = klayer.get_input_shape_at(0) + rnn = self.generate_simplernn_cell(self.klayer, self.kclayer, self.input_shape) + return self.__process_recurrent_layer(self.klayer.return_sequences, + self.klayer.go_backwards, rec.add(rnn)) + + def generate_lstm_cell(self, klayer, kclayer, input_shape): # create a lstm cell only + self.__check_recurrent_parameters(klayer) config = kclayer["config"] self.check_constraint_in_config(config) activation = get_activation_by_name(config["activation"], - "%s_%s" % (config["name"], config["activation"])) + "%s_%s" % (config["name"], config["activation"])) inner_activation = get_activation_by_name(config["inner_activation"], - "%s_%s" % (config["name"], config["inner_activation"])) + "%s_%s" % (config["name"], config["inner_activation"])) lstm = BLayer.LSTM(input_size=int(input_shape[2]), hidden_size=klayer.output_dim, p=0.0, @@ -761,55 +813,66 @@ def create_lstm(self, klayer, kclayer): uRegularizer=self.to_bigdl_reg(config["U_regularizer"]), bRegularizer=self.to_bigdl_reg(config["b_regularizer"]), bigdl_type="float") - return self.__return_sequences(klayer.return_sequences, rec.add(lstm)) + return lstm - def create_convlstm2d(self, klayer, kclayer): + def create_lstm(self): rec = BLayer.Recurrent() - input_shape = klayer.get_input_shape_at(0) + lstm = self.generate_lstm_cell(self.klayer, self.kclayer, self.input_shape) + return self.__process_recurrent_layer(self.klayer.return_sequences, + self.klayer.go_backwards, rec.add(lstm)) + + def generate_convlstm2d_cell(self, klayer, kclayer, input_shape): # create a convlstm2d cell only + self.__check_recurrent_parameters(klayer) config = kclayer["config"] self.check_constraint_in_config(config) activation = get_activation_by_name(config["activation"], - "%s_%s" % (config["name"], config["activation"])) + "%s_%s" % (config["name"], config["activation"])) inner_activation = get_activation_by_name(config["inner_activation"], - "%s_%s" % (config["name"], config["inner_activation"])) - - #TODO: border_mode = 'valid' - if config["border_mode"] != 'same': + "%s_%s" % (config["name"], config["inner_activation"])) + + convlstm = BLayer.ConvLSTMPeephole(input_size=int(input_shape[2]), + output_size=config["nb_filter"], + kernel_i=config["nb_col"], + kernel_c=config["nb_row"], + # NB: ConvLSTM doesn't serialize subsample to json file + stride=klayer.subsample[0], + padding=-1, + activation=activation, + inner_activation=inner_activation, + # NB: ConvLSTM doesn't serialize regularizers to json file + # wRegularizer=self.to_bigdl_reg(config["W_regularizer"]), + # uRegularizer=self.to_bigdl_reg(config["U_regularizer"]), + # bRegularizer=self.to_bigdl_reg(config["b_regularizer"]), + cRegularizer=None, + with_peephole=False, + bigdl_type="float") + return convlstm + + def create_convlstm2d(self): + # TODO: border_mode = 'valid' + if self.config["border_mode"] != 'same': raise Exception("Unsupported border_mode: valid") - if config["nb_row"] != config["nb_col"]: + if self.config["nb_row"] != self.config["nb_col"]: raise Exception("Only square kernel is supported for now. Please set nb_row=nb_col.") - if klayer.subsample[0] != klayer.subsample[1]: + if self.klayer.subsample[0] != self.klayer.subsample[1]: raise Exception("Only equal stride is supported for now. " "Please set subsample to be a tuple with equal values.") - blayer = BLayer.ConvLSTMPeephole(input_size=int(input_shape[2]), - output_size=config["nb_filter"], - kernel_i=config["nb_col"], - kernel_c=config["nb_row"], - # NB: ConvLSTM doesn't serialize subsample to json file - stride=klayer.subsample[0], - padding=-1, - activation=activation, - inner_activation=inner_activation, - # NB: ConvLSTM doesn't serialize regularizers to json file - # wRegularizer=self.to_bigdl_reg(config["W_regularizer"]), - # uRegularizer=self.to_bigdl_reg(config["U_regularizer"]), - # bRegularizer=self.to_bigdl_reg(config["b_regularizer"]), - cRegularizer=None, - with_peephole=False, - bigdl_type="float") - return self.__return_sequences(klayer.return_sequences, rec.add(blayer)) - - def create_gru(self, klayer, kclayer): rec = BLayer.Recurrent() - input_shape = klayer.get_input_shape_at(0) + convlstm = self.generate_convlstm2d_cell(self.klayer, + self.kclayer, self.input_shape) + return self.__process_recurrent_layer(self.klayer.return_sequences, + self.klayer.go_backwards, rec.add(convlstm)) + + def generate_gru_cell(self, klayer, kclayer, input_shape): # create a gru cell only + self.__check_recurrent_parameters(klayer) config = kclayer["config"] self.check_constraint_in_config(config) activation = get_activation_by_name(config["activation"], - "%s_%s" % (config["name"], config["activation"])) + "%s_%s" % (config["name"], config["activation"])) inner_activation = get_activation_by_name(config["inner_activation"], - "%s_%s" % (config["name"], config["inner_activation"])) + "%s_%s" % (config["name"], config["inner_activation"])) gru = BLayer.GRU(input_size=int(input_shape[2]), hidden_size=klayer.output_dim, p=0.0, @@ -819,36 +882,40 @@ def create_gru(self, klayer, kclayer): uRegularizer=self.to_bigdl_reg(config["U_regularizer"]), bRegularizer=self.to_bigdl_reg(config["b_regularizer"]), bigdl_type="float") - return self.__return_sequences(klayer.return_sequences, rec.add(gru)) + return gru - def create_batchnormalization(self, klayer, kclayer): - config = kclayer["config"] - if keras.backend.image_dim_ordering() != "th" or klayer.axis != 1: + def create_gru(self): + rec = BLayer.Recurrent() + gru = self.generate_gru_cell(self.klayer, self.kclayer, self.input_shape) + return self.__process_recurrent_layer(self.klayer.return_sequences, + self.klayer.go_backwards, rec.add(gru)) + + def create_batchnormalization(self): + if keras.backend.image_dim_ordering() != "th" or self.klayer.axis != 1: raise Exception("""For BatchNormalization, we only support th image ordering (i.e. NCHW) """ + """with axis = 1 for now, but the current order is %s and axis is %s - """ % (keras.backend.image_dim_ordering(), klayer.axis)) # noqa - if klayer.mode != 0: + """ % (keras.backend.image_dim_ordering(), self.klayer.axis)) # noqa + if self.klayer.mode != 0: raise Exception( - "Only support mode = 0 for now, but the current mode is: %s", klayer.mode) + "Only support mode = 0 for now, but the current mode is: %s", self.klayer.mode) - if config["gamma_regularizer"]: + if self.config["gamma_regularizer"]: raise Exception("We don't support gamma_regularizer for now") - if config["beta_regularizer"]: + if self.config["beta_regularizer"]: raise Exception("We don't support beta_regularizer for now") - input_shape = klayer.get_input_shape_at(0) - n_input_channel = int(input_shape[klayer.axis]) # default is -1 which is channel-last + n_input_channel = int(self.input_shape[self.klayer.axis]) # default is -1 which is channel-last # init gamma and beta # TODO: replace this with to_bigdl_init in the future - gamma = self.get_value_from_init(klayer.gamma_init.__name__, (n_input_channel,)) - beta = self.get_value_from_init(klayer.beta_init.__name__, (n_input_channel,)) + gamma = self.get_value_from_init(self.klayer.gamma_init.__name__, (n_input_channel,)) + beta = self.get_value_from_init(self.klayer.beta_init.__name__, (n_input_channel,)) blayer = BLayer.SpatialBatchNormalization( n_output=n_input_channel, - eps=klayer.epsilon, - momentum=klayer.momentum, + eps=self.klayer.epsilon, + momentum=self.klayer.momentum, affine=True, init_weight=gamma, init_bias=beta, @@ -856,22 +923,19 @@ def create_batchnormalization(self, klayer, kclayer): init_grad_bias=None, bigdl_type="float") - k_running_mean = keras.backend.eval(klayer.running_mean) - k_running_std = keras.backend.eval(klayer.running_std) + k_running_mean = keras.backend.eval(self.klayer.running_mean) + k_running_std = keras.backend.eval(self.klayer.running_std) blayer.set_running_mean(k_running_mean) blayer.set_running_std(k_running_std) return blayer - def get_bdim_order(self, kclayer): - return self.to_bigdl_2d_ordering(self.get_kdim_order(kclayer)) - - def get_kdim_order(self, kclayer): - config = kclayer["config"] - if "dim_ordering" in config: - return config["dim_ordering"] + def get_bdim_order(self): # get bigdl dim_ordering from keras dim_ordering + if "dim_ordering" in self.config: + order = self.config["dim_ordering"] else: - warnings.warn("Cannot find dim_ordering from json definition. Use default instead.") - return keras.backend.image_dim_ordering() + warnings.warn("Cannot find dim_ordering from json definition. Using the default instead.") + order = keras.backend.image_dim_ordering() + return self.to_bigdl_2d_ordering(order) def to_bigdl_2d_ordering(self, order): if order == "tf": @@ -879,7 +943,7 @@ def to_bigdl_2d_ordering(self, order): elif order == "th": return "NCHW" else: - raise Exception("Unsupport ordering: %s" % order) + raise Exception("Unsupported dim_ordering: %s" % order) def to_bigdl_3d_padding(self, border_mode): if border_mode == "valid": @@ -906,212 +970,200 @@ def to_bigdl_1d_padding(self, border_mode, kernel_w): else: raise Exception("Unsupported border mode: %s" % border_mode) - def create_convolution1d(self, klayer, kclayer): - config = kclayer["config"] - input_shape = klayer.get_input_shape_at(0) + def create_convolution1d(self): # batch, steps, dim, batch is None here, so you cannot use it directly. - stack_size = int(input_shape[2]) + stack_size = int(self.input_shape[2]) - bpadW, bpadH = self.to_bigdl_2d_padding(klayer.border_mode) + bpadW, bpadH = self.to_bigdl_2d_padding(self.klayer.border_mode) seq = BLayer.Sequential() - seq.add(BLayer.Reshape([int(input_shape[1]), 1, int(input_shape[2])], True)) + seq.add(BLayer.Reshape([int(self.input_shape[1]), 1, int(self.input_shape[2])], True)) blayer = BLayer.SpatialConvolution( n_input_plane=stack_size, - n_output_plane=klayer.nb_filter, + n_output_plane=self.klayer.nb_filter, kernel_w=1, - kernel_h=klayer.filter_length, + kernel_h=self.klayer.filter_length, stride_w=1, - stride_h=klayer.subsample_length, + stride_h=self.klayer.subsample_length, pad_w=bpadW, pad_h=bpadH, n_group=1, propagate_back=True, - wRegularizer=self.to_bigdl_reg(config["W_regularizer"]), - bRegularizer=self.to_bigdl_reg(config["b_regularizer"]), + wRegularizer=self.to_bigdl_reg(self.config["W_regularizer"]), + bRegularizer=self.to_bigdl_reg(self.config["b_regularizer"]), init_weight=None, init_bias=None, init_grad_weight=None, init_grad_bias=None, - with_bias=config["bias"], + with_bias=self.config["bias"], data_format="NHWC", bigdl_type="float") seq.add(blayer) seq.add(BLayer.Squeeze(3)) - return self.combo_parameter_layer(seq, config) + return self.combo_parameter_layer(seq, self.config) - def create_convolution2d(self, klayer, kclayer): - config = kclayer["config"] - bigdl_order = self.get_bdim_order(kclayer) - input_shape = klayer.get_input_shape_at(0) + def create_convolution2d(self): + bigdl_order = self.get_bdim_order() if bigdl_order == "NCHW": - stack_size = int(input_shape[1]) + stack_size = int(self.input_shape[1]) elif bigdl_order == "NHWC": - stack_size = int(input_shape[3]) + stack_size = int(self.input_shape[3]) - bpadW, bpadH = self.to_bigdl_2d_padding(klayer.border_mode) + bpadW, bpadH = self.to_bigdl_2d_padding(self.klayer.border_mode) blayer = BLayer.SpatialConvolution( n_input_plane=stack_size, - n_output_plane=klayer.nb_filter, - kernel_w=klayer.nb_col, - kernel_h=klayer.nb_row, - stride_w=klayer.subsample[0], - stride_h=klayer.subsample[1], + n_output_plane=self.klayer.nb_filter, + kernel_w=self.klayer.nb_col, + kernel_h=self.klayer.nb_row, + stride_w=self.klayer.subsample[0], + stride_h=self.klayer.subsample[1], pad_w=bpadW, pad_h=bpadH, n_group=1, propagate_back=True, - wRegularizer=self.to_bigdl_reg(config["W_regularizer"]), - bRegularizer=self.to_bigdl_reg(config["b_regularizer"]), + wRegularizer=self.to_bigdl_reg(self.config["W_regularizer"]), + bRegularizer=self.to_bigdl_reg(self.config["b_regularizer"]), init_weight=None, init_bias=None, init_grad_weight=None, init_grad_bias=None, - with_bias=config["bias"], + with_bias=self.config["bias"], data_format=bigdl_order, bigdl_type="float") - return self.combo_parameter_layer(blayer, config) + return self.combo_parameter_layer(blayer, self.config) - def create_convolution3d(self, klayer, kclayer): - config = kclayer["config"] - if klayer.dim_ordering != "th": - raise Exception("Please use `th` for `dim_ordering`. `%s` is not supported for now." % klayer.dim_ordering) - input_shape = klayer.get_input_shape_at(0) + def create_convolution3d(self): + if self.klayer.dim_ordering != "th": + raise Exception("Please use `th` for `dim_ordering`. `%s` is not supported for now." % self.klayer.dim_ordering) - bpadT, bpadW, bpadH = self.to_bigdl_3d_padding(klayer.border_mode) + bpadT, bpadW, bpadH = self.to_bigdl_3d_padding(self.klayer.border_mode) blayer = BLayer.VolumetricConvolution( - n_input_plane=int(input_shape[1]), - n_output_plane=klayer.nb_filter, - k_t=klayer.kernel_dim1, - k_w=klayer.kernel_dim3, - k_h=klayer.kernel_dim2, - d_t=klayer.subsample[0], - d_w=klayer.subsample[2], - d_h=klayer.subsample[1], + n_input_plane=int(self.input_shape[1]), + n_output_plane=self.klayer.nb_filter, + k_t=self.klayer.kernel_dim1, + k_w=self.klayer.kernel_dim3, + k_h=self.klayer.kernel_dim2, + d_t=self.klayer.subsample[0], + d_w=self.klayer.subsample[2], + d_h=self.klayer.subsample[1], pad_t=bpadT, pad_w=bpadW, pad_h=bpadH, - with_bias=config["bias"], - wRegularizer=self.to_bigdl_reg(config["W_regularizer"]), - bRegularizer=self.to_bigdl_reg(config["b_regularizer"]), + with_bias=self.config["bias"], + wRegularizer=self.to_bigdl_reg(self.config["W_regularizer"]), + bRegularizer=self.to_bigdl_reg(self.config["b_regularizer"]), bigdl_type="float") - return self.combo_parameter_layer(blayer, config) + return self.combo_parameter_layer(blayer, self.config) - def create_atrousconvolution1d(self, klayer, kclayer): - config = kclayer["config"] - if not config["bias"]: + def create_atrousconvolution1d(self): + if not self.config["bias"]: raise Exception("Please set `bias=True` for AtrousConvolution1D") - input_shape = klayer.get_input_shape_at(0) # TODO: border_mode=`same` - if klayer.border_mode == "same": - raise Exception("Unsupported border mode: %s" % klayer.border_mode) - bpadW, bpadH = self.to_bigdl_2d_padding(klayer.border_mode) + if self.klayer.border_mode == "same": + raise Exception("Unsupported border mode: %s" % self.klayer.border_mode) + bpadW, bpadH = self.to_bigdl_2d_padding(self.klayer.border_mode) seq = BLayer.Sequential() seq.add(BLayer.Transpose([(2, 3)])) - seq.add(BLayer.Reshape([int(input_shape[2]), int(input_shape[1]), 1], True)) + seq.add(BLayer.Reshape([int(self.input_shape[2]), int(self.input_shape[1]), 1], True)) blayer = BLayer.SpatialDilatedConvolution( - n_input_plane=int(input_shape[2]), - n_output_plane=config["nb_filter"], + n_input_plane=int(self.input_shape[2]), + n_output_plane=self.config["nb_filter"], kw=1, - kh=config["filter_length"], + kh=self.config["filter_length"], dw=1, - dh=config["subsample_length"], + dh=self.config["subsample_length"], pad_w=bpadW, pad_h=bpadH, dilation_w=1, - dilation_h=config["atrous_rate"], - wRegularizer=self.to_bigdl_reg(config["W_regularizer"]), - bRegularizer=self.to_bigdl_reg(config["b_regularizer"]), + dilation_h=self.config["atrous_rate"], + wRegularizer=self.to_bigdl_reg(self.config["W_regularizer"]), + bRegularizer=self.to_bigdl_reg(self.config["b_regularizer"]), bigdl_type="float") seq.add(blayer) seq.add(BLayer.Transpose([(2, 3)])) seq.add(BLayer.Squeeze(4)) - return self.combo_parameter_layer(seq, config) + return self.combo_parameter_layer(seq, self.config) - def create_atrousconvolution2d(self, klayer, kclayer): - config = kclayer["config"] - if klayer.dim_ordering != "th": - raise Exception("Please use `th` for `dim_ordering`. `%s` is not supported for now." % klayer.dim_ordering) - if not config["bias"]: + def create_atrousconvolution2d(self): + if self.klayer.dim_ordering != "th": + raise Exception("Please use `th` for `dim_ordering`. `%s` is not supported for now." % self.klayer.dim_ordering) + if not self.config["bias"]: raise Exception("Please set `bias=True` for AtrousConvolution2D") - input_shape = klayer.get_input_shape_at(0) # TODO: border_mode=`same` - if klayer.border_mode == "same": - raise Exception("Unsupported border mode: %s" % klayer.border_mode) - bpadW, bpadH = self.to_bigdl_2d_padding(klayer.border_mode) + if self.klayer.border_mode == "same": + raise Exception("Unsupported border mode: %s" % self.klayer.border_mode) + bpadW, bpadH = self.to_bigdl_2d_padding(self.klayer.border_mode) blayer = BLayer.SpatialDilatedConvolution( - n_input_plane=int(input_shape[1]), - n_output_plane=config["nb_filter"], - kw=config["nb_col"], - kh=config["nb_row"], - dw=config["subsample"][1], - dh=config["subsample"][0], + n_input_plane=int(self.input_shape[1]), + n_output_plane=self.config["nb_filter"], + kw=self.config["nb_col"], + kh=self.config["nb_row"], + dw=self.config["subsample"][1], + dh=self.config["subsample"][0], pad_w=bpadW, pad_h=bpadH, - dilation_w=config["atrous_rate"][1], - dilation_h=config["atrous_rate"][0], - wRegularizer=self.to_bigdl_reg(config["W_regularizer"]), - bRegularizer=self.to_bigdl_reg(config["b_regularizer"]), + dilation_w=self.config["atrous_rate"][1], + dilation_h=self.config["atrous_rate"][0], + wRegularizer=self.to_bigdl_reg(self.config["W_regularizer"]), + bRegularizer=self.to_bigdl_reg(self.config["b_regularizer"]), bigdl_type="float") - return self.combo_parameter_layer(blayer, config) + return self.combo_parameter_layer(blayer, self.config) - def create_deconvolution2d(self, klayer, kclayer): - config = kclayer["config"] - if klayer.dim_ordering != "th": - raise Exception("Please use `th` for `dim_ordering`. `%s` is not supported for now." % klayer.dim_ordering) - input_shape = klayer.get_input_shape_at(0) + def create_deconvolution2d(self): + if self.klayer.dim_ordering != "th": + raise Exception("Please use `th` for `dim_ordering`. `%s` is not supported for now." % self.klayer.dim_ordering) - bpadW, bpadH = self.to_bigdl_2d_padding(klayer.border_mode) + bpadW, bpadH = self.to_bigdl_2d_padding(self.klayer.border_mode) blayer = BLayer.SpatialFullConvolution( - n_input_plane=int(input_shape[1]), - n_output_plane=klayer.nb_filter, - kw=klayer.nb_col, - kh=klayer.nb_row, - dw=klayer.subsample[1], - dh=klayer.subsample[0], + n_input_plane=int(self.input_shape[1]), + n_output_plane=self.klayer.nb_filter, + kw=self.klayer.nb_col, + kh=self.klayer.nb_row, + dw=self.klayer.subsample[1], + dh=self.klayer.subsample[0], pad_w=bpadW, pad_h=bpadH, adj_w=0, adj_h=0, n_group=1, no_bias=False, - wRegularizer=self.to_bigdl_reg(config["W_regularizer"]), - bRegularizer=self.to_bigdl_reg(config["b_regularizer"]), + wRegularizer=self.to_bigdl_reg(self.config["W_regularizer"]), + bRegularizer=self.to_bigdl_reg(self.config["b_regularizer"]), bigdl_type="float") - return self.combo_parameter_layer(blayer, config) + return self.combo_parameter_layer(blayer, self.config) - def create_maxpooling3d(self, klayer, kclayer): - if klayer.dim_ordering != "th": - raise Exception("Please use `th` for `dim_ordering`. `%s` is not supported for now." % klayer.dim_ordering) - bpadT, bpadW, bpadH = self.to_bigdl_3d_padding(klayer.border_mode) + def create_maxpooling3d(self): + if self.klayer.dim_ordering != "th": + raise Exception("Please use `th` for `dim_ordering`. `%s` is not supported for now." % self.klayer.dim_ordering) + bpadT, bpadW, bpadH = self.to_bigdl_3d_padding(self.klayer.border_mode) blayer = BLayer.VolumetricMaxPooling( - k_t=klayer.pool_size[0], - k_w=klayer.pool_size[2], - k_h=klayer.pool_size[1], - d_t=klayer.strides[0], - d_w=klayer.strides[2], - d_h=klayer.strides[1], + k_t=self.klayer.pool_size[0], + k_w=self.klayer.pool_size[2], + k_h=self.klayer.pool_size[1], + d_t=self.klayer.strides[0], + d_w=self.klayer.strides[2], + d_h=self.klayer.strides[1], pad_t=bpadT, pad_w=bpadW, pad_h=bpadH, bigdl_type="float") return blayer - def create_maxpooling2d(self, klayer, kclayer): - bigdl_order = self.get_bdim_order(kclayer) - bpadW, bpadH = self.to_bigdl_2d_padding(klayer.border_mode) + def create_maxpooling2d(self): + bigdl_order = self.get_bdim_order() + bpadW, bpadH = self.to_bigdl_2d_padding(self.klayer.border_mode) blayer = BLayer.SpatialMaxPooling( - kw=klayer.pool_size[1], - kh=klayer.pool_size[0], - dw=klayer.strides[1], - dh=klayer.strides[0], + kw=self.klayer.pool_size[1], + kh=self.klayer.pool_size[0], + dw=self.klayer.strides[1], + dh=self.klayer.strides[0], pad_w=bpadW, pad_h=bpadH, to_ceil=False, @@ -1119,14 +1171,13 @@ def create_maxpooling2d(self, klayer, kclayer): bigdl_type="float") return blayer - def create_globalmaxpooling3d(self, klayer, kclayer): - input_shape = klayer.get_input_shape_at(0) - if klayer.dim_ordering == "th": - b_kt = int(input_shape[2]) - b_kw = int(input_shape[4]) - b_kh = int(input_shape[3]) + def create_globalmaxpooling3d(self): + if self.klayer.dim_ordering == "th": + b_kt = int(self.input_shape[2]) + b_kw = int(self.input_shape[4]) + b_kh = int(self.input_shape[3]) else: - raise Exception("Please use `th` for dim_ordering. `%s` is not supported for now." % klayer.dim_ordering) + raise Exception("Please use `th` for dim_ordering. `%s` is not supported for now." % self.klayer.dim_ordering) seq = BLayer.Sequential() blayer = BLayer.VolumetricMaxPooling( @@ -1148,14 +1199,13 @@ def create_globalmaxpooling3d(self, klayer, kclayer): return seq - def create_globalaveragepooling3d(self, klayer, kclayer): - input_shape = klayer.get_input_shape_at(0) - if klayer.dim_ordering == "th": - b_kt = int(input_shape[2]) - b_kw = int(input_shape[4]) - b_kh = int(input_shape[3]) + def create_globalaveragepooling3d(self): + if self.klayer.dim_ordering == "th": + b_kt = int(self.input_shape[2]) + b_kw = int(self.input_shape[4]) + b_kh = int(self.input_shape[3]) else: - raise Exception("Please use `th` for dim_ordering. `%s` is not supported for now." % klayer.dim_ordering) + raise Exception("Please use `th` for dim_ordering. `%s` is not supported for now." % self.klayer.dim_ordering) seq = BLayer.Sequential() blayer = BLayer.VolumetricAveragePooling( @@ -1178,14 +1228,14 @@ def create_globalaveragepooling3d(self, klayer, kclayer): return seq - def create_averagepooling2d(self, klayer, kclayer): - bigdl_order = self.get_bdim_order(kclayer) - bpadW, bpadH = self.to_bigdl_2d_padding(klayer.border_mode) + def create_averagepooling2d(self): + bigdl_order = self.get_bdim_order() + bpadW, bpadH = self.to_bigdl_2d_padding(self.klayer.border_mode) blayer = BLayer.SpatialAveragePooling( - kw=klayer.pool_size[1], - kh=klayer.pool_size[0], - dw=klayer.strides[1], - dh=klayer.strides[0], + kw=self.klayer.pool_size[1], + kh=self.klayer.pool_size[0], + dw=self.klayer.strides[1], + dh=self.klayer.strides[0], pad_w=bpadW, pad_h=bpadH, global_pooling=False, @@ -1197,17 +1247,17 @@ def create_averagepooling2d(self, klayer, kclayer): ) return blayer - def create_averagepooling3d(self, klayer, kclayer): - if klayer.dim_ordering != "th": - raise Exception("Please use `th` for `dim_ordering`. `%s` is not supported for now." % klayer.dim_ordering) - bpadT, bpadW, bpadH = self.to_bigdl_3d_padding(klayer.border_mode) + def create_averagepooling3d(self): + if self.klayer.dim_ordering != "th": + raise Exception("Please use `th` for `dim_ordering`. `%s` is not supported for now." % self.klayer.dim_ordering) + bpadT, bpadW, bpadH = self.to_bigdl_3d_padding(self.klayer.border_mode) blayer = BLayer.VolumetricAveragePooling( - k_t=klayer.pool_size[0], - k_w=klayer.pool_size[2], - k_h=klayer.pool_size[1], - d_t=klayer.strides[0], - d_w=klayer.strides[2], - d_h=klayer.strides[1], + k_t=self.klayer.pool_size[0], + k_w=self.klayer.pool_size[2], + k_h=self.klayer.pool_size[1], + d_t=self.klayer.strides[0], + d_w=self.klayer.strides[2], + d_h=self.klayer.strides[1], pad_t=bpadT, pad_w=bpadW, pad_h=bpadH, @@ -1215,15 +1265,14 @@ def create_averagepooling3d(self, klayer, kclayer): bigdl_type="float") return blayer - def create_globalmaxpooling2d(self, klayer, kclayer): - bigdl_order = self.get_bdim_order(kclayer) - input_shape = klayer.get_input_shape_at(0) + def create_globalmaxpooling2d(self): + bigdl_order = self.get_bdim_order() if bigdl_order == "NCHW": - b_kw = int(input_shape[3]) - b_kh = int(input_shape[2]) + b_kw = int(self.input_shape[3]) + b_kh = int(self.input_shape[2]) else: - b_kw = int(input_shape[2]) - b_kh = int(input_shape[1]) + b_kw = int(self.input_shape[2]) + b_kh = int(self.input_shape[1]) seq = BLayer.Sequential() blayer = BLayer.SpatialMaxPooling( @@ -1246,13 +1295,12 @@ def create_globalmaxpooling2d(self, klayer, kclayer): seq.add(BLayer.Squeeze(1, num_input_dims=2)) return seq - def create_globalmaxpooling1d(self, klayer, kclayer): - input_shape = klayer.get_input_shape_at(0) # batch, step, dim + def create_globalmaxpooling1d(self): b_kw = 1 - b_kh = int(input_shape[1]) + b_kh = int(self.input_shape[1]) seq = BLayer.Sequential() - seq.add(BLayer.View([int(input_shape[1]), 1, int(input_shape[2])], num_input_dims=2)) + seq.add(BLayer.View([int(self.input_shape[1]), 1, int(self.input_shape[2])], num_input_dims=2)) blayer = BLayer.SpatialMaxPooling( kw=b_kw, kh=b_kh, @@ -1269,13 +1317,12 @@ def create_globalmaxpooling1d(self, klayer, kclayer): seq.add(BLayer.Squeeze(1, num_input_dims=1)) return seq - def create_globalaveragepooling1d(self, klayer, kclayer): - input_shape = klayer.get_input_shape_at(0) # batch, step, dim + def create_globalaveragepooling1d(self): b_kw = 1 - b_kh = int(input_shape[1]) + b_kh = int(self.input_shape[1]) seq = BLayer.Sequential() - seq.add(BLayer.View([int(input_shape[1]), 1, int(input_shape[2])], num_input_dims=2)) + seq.add(BLayer.View([int(self.input_shape[1]), 1, int(self.input_shape[2])], num_input_dims=2)) blayer = BLayer.SpatialAveragePooling( kw=b_kw, kh=b_kh, @@ -1291,22 +1338,21 @@ def create_globalaveragepooling1d(self, klayer, kclayer): bigdl_type="float" ) seq.add(blayer) - seq.add(BLayer.Squeeze(2, num_input_dims=2)) # the index start from one but without batch + seq.add(BLayer.Squeeze(2, num_input_dims=2)) # the index start from one but without batch seq.add(BLayer.Squeeze(1, num_input_dims=1)) return seq - def create_maxpooling1d(self, klayer, kclayer): - input_shape = klayer.get_input_shape_at(0) # batch, steps, dim - bpadW, bpadH = self.to_bigdl_2d_padding(klayer.border_mode) + def create_maxpooling1d(self): + bpadW, bpadH = self.to_bigdl_2d_padding(self.klayer.border_mode) seq = BLayer.Sequential() - seq.add(BLayer.Reshape([int(input_shape[1]), 1, int(input_shape[2])], True)) + seq.add(BLayer.Reshape([int(self.input_shape[1]), 1, int(self.input_shape[2])], True)) blayer = BLayer.SpatialMaxPooling( kw=1, - kh=klayer.pool_length, + kh=self.klayer.pool_length, dw=1, - dh=klayer.stride, + dh=self.klayer.stride, pad_w=bpadW, pad_h=bpadH, to_ceil=False, @@ -1317,17 +1363,16 @@ def create_maxpooling1d(self, klayer, kclayer): seq.add(BLayer.Squeeze(3)) return seq - def create_averagepooling1d(self, klayer, kclayer): - input_shape = klayer.get_input_shape_at(0) # batch, steps, dim - bpadW, bpadH = self.to_bigdl_2d_padding(klayer.border_mode) + def create_averagepooling1d(self): + bpadW, bpadH = self.to_bigdl_2d_padding(self.klayer.border_mode) seq = BLayer.Sequential() - seq.add(BLayer.Reshape([int(input_shape[1]), 1, int(input_shape[2])], True)) + seq.add(BLayer.Reshape([int(self.input_shape[1]), 1, int(self.input_shape[2])], True)) blayer = BLayer.SpatialAveragePooling( kw=1, - kh=klayer.pool_length, + kh=self.klayer.pool_length, dw=1, - dh=klayer.stride, + dh=self.klayer.stride, pad_w=bpadW, pad_h=bpadH, global_pooling=False, @@ -1341,15 +1386,14 @@ def create_averagepooling1d(self, klayer, kclayer): seq.add(BLayer.Squeeze(3)) return seq - def create_globalaveragepooling2d(self, klayer, kclayer): - bigdl_order = self.get_bdim_order(kclayer) - input_shape = klayer.get_input_shape_at(0) + def create_globalaveragepooling2d(self): + bigdl_order = self.get_bdim_order() if bigdl_order == "NCHW": - b_kw = int(input_shape[3]) - b_kh = int(input_shape[2]) + b_kw = int(self.input_shape[3]) + b_kh = int(self.input_shape[2]) else: - b_kw = int(input_shape[2]) - b_kh = int(input_shape[1]) + b_kw = int(self.input_shape[2]) + b_kh = int(self.input_shape[1]) seq = BLayer.Sequential() blayer = BLayer.SpatialAveragePooling( @@ -1415,7 +1459,7 @@ def to_bigdl_init(self, kinit_method): # kinit_method is a string elif kinit_method == "zero": init = BInit.Zeros() else: - raise Exception("Unsupported init type: %s" % init) + raise Exception("Unsupported init type: %s" % kinit_method) return init def to_bigdl_reg(self, reg): # reg is a dict From 3f4a656cfd2975efc59930fc30f15960b862af3a Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Wed, 6 Dec 2017 08:52:43 +0800 Subject: [PATCH 400/823] add keras wrapper; code refactor (#1953) * TimeDistributed & Bidirectional * Code refactor of LayerConverter --- python/dllib/test/bigdl/keras/test_layer.py | 45 +++++++++++++++++---- 1 file changed, 38 insertions(+), 7 deletions(-) diff --git a/python/dllib/test/bigdl/keras/test_layer.py b/python/dllib/test/bigdl/keras/test_layer.py index 5b9a44f4571..48e9ffc9d45 100644 --- a/python/dllib/test/bigdl/keras/test_layer.py +++ b/python/dllib/test/bigdl/keras/test_layer.py @@ -537,7 +537,7 @@ def test_simplernn(self): input_data = np.random.random([3, 4, 5]) layer = SimpleRNN(5, input_shape=(4, 5), return_sequences=True) self.modelTestSingleLayer(input_data, layer, dump_weights=True) - layer2 = SimpleRNN(3, input_shape=(4, 5), return_sequences=False) + layer2 = SimpleRNN(3, input_shape=(4, 5), go_backwards=True) self.modelTestSingleLayer(input_data, layer2, dump_weights=True) layer3 = SimpleRNN(3, input_shape=(4, 5), activation='relu') self.modelTestSingleLayer(input_data, layer3, dump_weights=True) @@ -546,27 +546,28 @@ def test_lstm(self): input_data = np.random.random([3, 4, 5]) layer = LSTM(5, input_shape=(4, 5), return_sequences=True) self.modelTestSingleLayer(input_data, layer, dump_weights=True) - layer2 = LSTM(3, input_shape=(4, 5), return_sequences=False, + layer2 = LSTM(3, input_shape=(4, 5), go_backwards=True, activation='relu', inner_activation='sigmoid') self.modelTestSingleLayer(input_data, layer2, dump_weights=True) def test_convlstm2d(self): input_data = np.random.random_sample([4, 8, 40, 40, 32]) layer = ConvLSTM2D(32, 4, 4, input_shape=(8, 40, 40, 32), - return_sequences=True, border_mode='same') - self.modelTestSingleLayer(input_data, layer, dump_weights=True, random_weights=True) + border_mode='same', go_backwards=True) + self.modelTestSingleLayer(input_data, layer, dump_weights=True) layer2 = ConvLSTM2D(32, 4, 4, input_shape=(8, 40, 40, 32), return_sequences=True, activation='relu', inner_activation='sigmoid', border_mode='same') - self.modelTestSingleLayer(input_data, layer2, dump_weights=True, - random_weights=True, rtol=1e-5, atol=1e-5) + self.modelTestSingleLayer(input_data, layer2, dump_weights=True, rtol=1e-5, atol=1e-5) def test_gru(self): input_data = np.random.random([3, 4, 5]) layer = GRU(4, input_shape=(4, 5), return_sequences=True) self.modelTestSingleLayer(input_data, layer, dump_weights=True) - layer2 = GRU(8, input_shape=(4, 5), return_sequences=False, + layer2 = GRU(8, input_shape=(4, 5), go_backwards=True, activation='relu', inner_activation='sigmoid') self.modelTestSingleLayer(input_data, layer2, dump_weights=True) + layer3 = GRU(512, input_shape=(4, 5), go_backwards=True, return_sequences=True) + self.modelTestSingleLayer(input_data, layer3, dump_weights=True) # TODO: Support share weights training. def test_multiple_inputs_share_weights(self): @@ -588,6 +589,36 @@ def test_multiple_inputs_share_weights(self): bigdl_model = DefinitionLoader.from_json_path(def_path) assert str(excinfo.value) == """Convolution2D doesn't support multiple inputs with shared weights""" # noqa + def test_wrapper_timedistributed(self): + input_data = np.random.random_sample([3, 32, 64]) + layer = TimeDistributed(Dense(6), input_shape=(32, 64)) + self.modelTestSingleLayer(input_data, layer, dump_weights=True) + + input_data2 = np.random.random_sample([2, 10, 3, 32, 32]) + layer2 = TimeDistributed(Convolution2D(64, 3, 3), input_shape=(10, 3, 32, 32)) + self.modelTestSingleLayer(input_data2, layer2, dump_weights=True) + + def test_wrapper_bidirectional(self): + input_data = np.random.random([5, 32, 64]) + layer = Bidirectional(SimpleRNN(12, return_sequences=True), + input_shape=(32, 64)) + self.modelTestSingleLayer(input_data, layer, dump_weights=True) + layer2 = Bidirectional(LSTM(8, return_sequences=True), + input_shape=(32, 64), merge_mode='sum') + self.modelTestSingleLayer(input_data, layer2, dump_weights=True) + layer3 = Bidirectional(GRU(12, return_sequences=True), + input_shape=(32, 64), merge_mode='mul') + self.modelTestSingleLayer(input_data, layer3, dump_weights=True) + layer4 = Bidirectional(LSTM(64, return_sequences=True), + input_shape=(32, 64)) + self.modelTestSingleLayer(input_data, layer4, dump_weights=True) + + input_data2 = np.random.random_sample([4, 8, 40, 40, 32]) + layer5 = Bidirectional(ConvLSTM2D(32, 4, 4, border_mode='same', + return_sequences=True), + input_shape=(8, 40, 40, 32), merge_mode='ave') + self.modelTestSingleLayer(input_data2, layer5, dump_weights=True) + if __name__ == "__main__": pytest.main([__file__]) From 36f0e110ba6bb6fc941a140bc3003aecdf716499 Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Wed, 6 Dec 2017 10:45:08 +0800 Subject: [PATCH 401/823] Support table for sample-label (#1977) * support table for label * fix ut * add unit test * add doc * update --- python/dllib/src/bigdl/dllib/utils/common.py | 21 ++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/utils/common.py b/python/dllib/src/bigdl/dllib/utils/common.py index 93fa4f9fc90..6d8d806d5fd 100644 --- a/python/dllib/src/bigdl/dllib/utils/common.py +++ b/python/dllib/src/bigdl/dllib/utils/common.py @@ -269,7 +269,7 @@ def __init__(self, features, label, bigdl_type="float"): """ User should always use Sample.from_ndarray to construct Sample. :param features: a list of JTensors - :param label: a JTensor + :param label: a list of JTensors :param bigdl_type: "double" or "float" """ self.features = features @@ -281,7 +281,7 @@ def from_ndarray(cls, features, label, bigdl_type="float"): """ Convert a ndarray of features and label to Sample, which would be used in Java side. :param features: an ndarray or a list of ndarrays - :param label: an ndarray or a scalar + :param label: an ndarray or a list of ndarrays or a scalar :param bigdl_type: "double" or "float" >>> import numpy as np @@ -291,22 +291,27 @@ def from_ndarray(cls, features, label, bigdl_type="float"): >>> sample = Sample.from_ndarray(np.random.random((2,3)), np.random.random((2,3))) >>> sample_back = callBigDlFunc("float", "testSample", sample) >>> assert_allclose(sample.features[0].to_ndarray(), sample_back.features[0].to_ndarray()) - >>> assert_allclose(sample.label.to_ndarray(), sample_back.label.to_ndarray()) + >>> assert_allclose(sample.label[0].to_ndarray(), sample_back.label[0].to_ndarray()) >>> print(sample) Sample: features: [JTensor: storage: [[ 0.69646919 0.28613934 0.22685145] - [ 0.55131477 0.71946895 0.42310646]], shape: [2 3], float], label: JTensor: storage: [[ 0.98076421 0.68482971 0.48093191] - [ 0.39211753 0.343178 0.72904968]], shape: [2 3], float, + [ 0.55131477 0.71946895 0.42310646]], shape: [2 3], float], label: [JTensor: storage: [[ 0.98076421 0.68482971 0.48093191] + [ 0.39211753 0.343178 0.72904968]], shape: [2 3], float], """ if isinstance(features, np.ndarray): features = [features] else: assert all(isinstance(feature, np.ndarray) for feature in features), \ "features should be a list of np.ndarray, not %s" % type(features) - if not isinstance(label, np.ndarray): # in case label is a scalar. - label = np.array(label) + if np.isscalar(label): # in case label is a scalar. + label = [np.array(label)] + elif isinstance(label, np.ndarray): + label = [label] + else: + assert all(isinstance(l, np.ndarray) for l in label), \ + "label should be a list of np.ndarray, not %s" % type(label) return cls( features=[JTensor.from_ndarray(f) for f in features], - label=JTensor.from_ndarray(label), + label=[JTensor.from_ndarray(l) for l in label], bigdl_type=bigdl_type) @classmethod From 7d0c3d48b55d047beeeec0b278e1b7a3d0d5f346 Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Wed, 6 Dec 2017 10:45:08 +0800 Subject: [PATCH 402/823] Support table for sample-label (#1977) * support table for label * fix ut * add unit test * add doc * update --- python/dllib/src/bigdl/utils/common.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/python/dllib/src/bigdl/utils/common.py b/python/dllib/src/bigdl/utils/common.py index 93fa4f9fc90..6d8d806d5fd 100644 --- a/python/dllib/src/bigdl/utils/common.py +++ b/python/dllib/src/bigdl/utils/common.py @@ -269,7 +269,7 @@ def __init__(self, features, label, bigdl_type="float"): """ User should always use Sample.from_ndarray to construct Sample. :param features: a list of JTensors - :param label: a JTensor + :param label: a list of JTensors :param bigdl_type: "double" or "float" """ self.features = features @@ -281,7 +281,7 @@ def from_ndarray(cls, features, label, bigdl_type="float"): """ Convert a ndarray of features and label to Sample, which would be used in Java side. :param features: an ndarray or a list of ndarrays - :param label: an ndarray or a scalar + :param label: an ndarray or a list of ndarrays or a scalar :param bigdl_type: "double" or "float" >>> import numpy as np @@ -291,22 +291,27 @@ def from_ndarray(cls, features, label, bigdl_type="float"): >>> sample = Sample.from_ndarray(np.random.random((2,3)), np.random.random((2,3))) >>> sample_back = callBigDlFunc("float", "testSample", sample) >>> assert_allclose(sample.features[0].to_ndarray(), sample_back.features[0].to_ndarray()) - >>> assert_allclose(sample.label.to_ndarray(), sample_back.label.to_ndarray()) + >>> assert_allclose(sample.label[0].to_ndarray(), sample_back.label[0].to_ndarray()) >>> print(sample) Sample: features: [JTensor: storage: [[ 0.69646919 0.28613934 0.22685145] - [ 0.55131477 0.71946895 0.42310646]], shape: [2 3], float], label: JTensor: storage: [[ 0.98076421 0.68482971 0.48093191] - [ 0.39211753 0.343178 0.72904968]], shape: [2 3], float, + [ 0.55131477 0.71946895 0.42310646]], shape: [2 3], float], label: [JTensor: storage: [[ 0.98076421 0.68482971 0.48093191] + [ 0.39211753 0.343178 0.72904968]], shape: [2 3], float], """ if isinstance(features, np.ndarray): features = [features] else: assert all(isinstance(feature, np.ndarray) for feature in features), \ "features should be a list of np.ndarray, not %s" % type(features) - if not isinstance(label, np.ndarray): # in case label is a scalar. - label = np.array(label) + if np.isscalar(label): # in case label is a scalar. + label = [np.array(label)] + elif isinstance(label, np.ndarray): + label = [label] + else: + assert all(isinstance(l, np.ndarray) for l in label), \ + "label should be a list of np.ndarray, not %s" % type(label) return cls( features=[JTensor.from_ndarray(f) for f in features], - label=JTensor.from_ndarray(label), + label=[JTensor.from_ndarray(l) for l in label], bigdl_type=bigdl_type) @classmethod From feaf1738579f40192d84de21e41385def53dc754 Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Wed, 6 Dec 2017 10:45:08 +0800 Subject: [PATCH 403/823] Support table for sample-label (#1977) * support table for label * fix ut * add unit test * add doc * update --- .../test/bigdl/test_simple_integration.py | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/python/dllib/test/bigdl/test_simple_integration.py b/python/dllib/test/bigdl/test_simple_integration.py index 68c205f7f07..9c8a3e23378 100644 --- a/python/dllib/test/bigdl/test_simple_integration.py +++ b/python/dllib/test/bigdl/test_simple_integration.py @@ -324,6 +324,36 @@ def gen_rand_sample(): optimizer.optimize() + def test_table_label(self): + """ + Test for table as label in Sample. + For test purpose only. + """ + def gen_rand_sample(): + features1 = np.random.uniform(0, 1, 3) + features2 = np.random.uniform(0, 1, 3) + label = np.array((2 * (features1 + features2)).sum() + 0.4) + return Sample.from_ndarray([features1, features2], [label, label]) + + training_data = self.sc.parallelize(range(0, 50)).map( + lambda i: gen_rand_sample()) + + model_test = Sequential() + branches = ParallelTable() + branch1 = Sequential().add(Linear(3, 1)).add(Tanh()) + branch2 = Sequential().add(Linear(3, 1)).add(ReLU()) + branches.add(branch1).add(branch2) + model_test.add(branches) + + optimizer = Optimizer( + model=model_test, + training_rdd=training_data, + criterion=MarginRankingCriterion(), + optim_method=SGD(), + end_trigger=MaxEpoch(5), + batch_size=32) + optimizer.optimize() + def test_forward_backward(self): from bigdl.nn.layer import Linear rng = RNG() From aeb5f1fa193064359fe5a67684e7cb7cf571626b Mon Sep 17 00:00:00 2001 From: dding3 Date: Wed, 6 Dec 2017 14:17:03 -0500 Subject: [PATCH 404/823] Support Keras masking and maxoutdense (#1918) * support keras Masking layer * support keras maxoutdense --- .../dllib/src/bigdl/dllib/keras/converter.py | 8 +++ python/dllib/src/bigdl/dllib/nn/layer.py | 51 +++++++++++++++++++ 2 files changed, 59 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/keras/converter.py b/python/dllib/src/bigdl/dllib/keras/converter.py index 8626497a8a2..8ac62033eb4 100644 --- a/python/dllib/src/bigdl/dllib/keras/converter.py +++ b/python/dllib/src/bigdl/dllib/keras/converter.py @@ -221,6 +221,14 @@ def convert_gru(klayer, weights): w4 = weights[7].T return [w1, w2, w3, w4] + @staticmethod + def convert_maxoutdense(weights): + k_weights = weights[0] + b_weights = k_weights[0].T + for i in range(1, k_weights.shape[0]): + b_weights = np.concatenate((b_weights, k_weights[i].T)) + return [b_weights, weights[1]] + class DefinitionLoader: diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 5e418952c63..e3b7755a348 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -4663,6 +4663,57 @@ class GaussianSampler(Layer): def __init__(self, bigdl_type="float"): super(GaussianSampler, self).__init__(None, bigdl_type) +class Masking(Layer): + + ''' + Use a mask value to skip timesteps for a sequence + ``` + :param mask_value: mask value + + >>> masking = Masking(0.0) + creating: createMasking + ''' + + def __init__(self, + mask_value, + bigdl_type="float"): + super(Masking, self).__init__(None, bigdl_type, + mask_value) + +class Maxout(Layer): + + ''' + A linear maxout layer Maxout layer select the element-wise maximum value of + maxoutNumber Linear(inputSize, outputSize) layers + ``` + :param input_size: the size the each input sample + :param output_size: the size of the module output of each sample + :param maxout_number: number of Linear layers to use + :param with_bias: whether use bias in Linear + :param w_regularizer: instance of [[Regularizer]] + (eg. L1 or L2 regularization), applied to the input weights matrices. + :param b_regularizer: instance of [[Regularizer]] + applied to the bias. + :param init_weight: initial weight + :param init_bias: initial bias + + >>> maxout = Maxout(2, 5, 3) + creating: createMaxout + ''' + def __init__(self, + input_size, + output_size, + maxout_number, + with_bias = True, + w_regularizer=None, + b_regularizer=None, + init_weight=None, + init_bias=None, + bigdl_type="float"): + super(Maxout, self).__init__(None, bigdl_type, + input_size, output_size, maxout_number, with_bias, + w_regularizer, b_regularizer, init_weight, init_bias) + class HardSigmoid(Layer): """ Apply Hard-sigmoid function From 899b2299abf92ea82b2cb76408d04be652ab3ad8 Mon Sep 17 00:00:00 2001 From: dding3 Date: Wed, 6 Dec 2017 14:17:03 -0500 Subject: [PATCH 405/823] Support Keras masking and maxoutdense (#1918) * support keras Masking layer * support keras maxoutdense --- .../dllib/src/bigdl/dllib/keras/converter.py | 8 +++ python/dllib/src/bigdl/dllib/nn/layer.py | 51 +++++++++++++++++++ 2 files changed, 59 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/keras/converter.py b/python/dllib/src/bigdl/dllib/keras/converter.py index 8626497a8a2..8ac62033eb4 100644 --- a/python/dllib/src/bigdl/dllib/keras/converter.py +++ b/python/dllib/src/bigdl/dllib/keras/converter.py @@ -221,6 +221,14 @@ def convert_gru(klayer, weights): w4 = weights[7].T return [w1, w2, w3, w4] + @staticmethod + def convert_maxoutdense(weights): + k_weights = weights[0] + b_weights = k_weights[0].T + for i in range(1, k_weights.shape[0]): + b_weights = np.concatenate((b_weights, k_weights[i].T)) + return [b_weights, weights[1]] + class DefinitionLoader: diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 5e418952c63..e3b7755a348 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -4663,6 +4663,57 @@ class GaussianSampler(Layer): def __init__(self, bigdl_type="float"): super(GaussianSampler, self).__init__(None, bigdl_type) +class Masking(Layer): + + ''' + Use a mask value to skip timesteps for a sequence + ``` + :param mask_value: mask value + + >>> masking = Masking(0.0) + creating: createMasking + ''' + + def __init__(self, + mask_value, + bigdl_type="float"): + super(Masking, self).__init__(None, bigdl_type, + mask_value) + +class Maxout(Layer): + + ''' + A linear maxout layer Maxout layer select the element-wise maximum value of + maxoutNumber Linear(inputSize, outputSize) layers + ``` + :param input_size: the size the each input sample + :param output_size: the size of the module output of each sample + :param maxout_number: number of Linear layers to use + :param with_bias: whether use bias in Linear + :param w_regularizer: instance of [[Regularizer]] + (eg. L1 or L2 regularization), applied to the input weights matrices. + :param b_regularizer: instance of [[Regularizer]] + applied to the bias. + :param init_weight: initial weight + :param init_bias: initial bias + + >>> maxout = Maxout(2, 5, 3) + creating: createMaxout + ''' + def __init__(self, + input_size, + output_size, + maxout_number, + with_bias = True, + w_regularizer=None, + b_regularizer=None, + init_weight=None, + init_bias=None, + bigdl_type="float"): + super(Maxout, self).__init__(None, bigdl_type, + input_size, output_size, maxout_number, with_bias, + w_regularizer, b_regularizer, init_weight, init_bias) + class HardSigmoid(Layer): """ Apply Hard-sigmoid function From b250c296d281d45a4a599cfc7dfef556dd70c951 Mon Sep 17 00:00:00 2001 From: dding3 Date: Thu, 7 Dec 2017 15:15:46 -0500 Subject: [PATCH 406/823] Add multiple rnn cell and support get/setHiddenState in Recurrent(#1591) * add multicell * add get/setHiddenStates --- python/dllib/src/bigdl/dllib/nn/layer.py | 31 +++++++++++++++--------- 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index e3b7755a348..739ed2d67e9 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -1326,18 +1326,8 @@ def get_hidden_state(self): :return: list of hidden state and cell """ - state = callBigDlFunc(self.bigdl_type, "getHiddenState", self.value) - for idx, tensor in enumerate(state): - state[idx] = tensor.to_ndarray() - - return state - - def set_hidden_state(self, states): - """ - set hidden state and cell at first time step. - """ - jstate, state_is_table = self.check_input(states) - callBigDlFunc(self.bigdl_type, "setHiddenState", self.value, jstate, state_is_table) + states = callBigDlFunc(self.bigdl_type, "getHiddenState", self.value) + return states class RecurrentDecoder(Recurrent): ''' @@ -4639,6 +4629,23 @@ def __init__(self, input_size, output_size, kernel_i, kernel_c, stride=1, paddin super(ConvLSTMPeephole3D, self).__init__(None, bigdl_type, input_size, output_size, kernel_i, kernel_c, stride, padding, wRegularizer, uRegularizer, bRegularizer, cRegularizer, with_peephole) + +class MultiRNNCell(Layer): + ''' + A cell that enables stack multiple simple rnn cells + + >>> cells = [] + >>> cells.append(ConvLSTMPeephole3D(4, 3, 3, 3, 1)) + creating: createConvLSTMPeephole3D + >>> cells.append(ConvLSTMPeephole3D(4, 3, 3, 3, 1)) + creating: createConvLSTMPeephole3D + >>> stacked_convlstm = MultiRNNCell(cells) + creating: createMultiRNNCell + ''' + + def __init__(self, cells, bigdl_type="float"): + super(MultiRNNCell, self).__init__(None, bigdl_type, cells) + class ResizeBilinear(Layer): """ Resize the input image with bilinear interpolation. The input image must be a float tensor with From 3321e07f1332c436d73dc46dec6a5b2f652e8b43 Mon Sep 17 00:00:00 2001 From: dding3 Date: Thu, 7 Dec 2017 15:15:46 -0500 Subject: [PATCH 407/823] Add multiple rnn cell and support get/setHiddenState in Recurrent(#1591) * add multicell * add get/setHiddenStates --- python/dllib/src/bigdl/dllib/nn/layer.py | 31 +++++++++++++++--------- 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index e3b7755a348..739ed2d67e9 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -1326,18 +1326,8 @@ def get_hidden_state(self): :return: list of hidden state and cell """ - state = callBigDlFunc(self.bigdl_type, "getHiddenState", self.value) - for idx, tensor in enumerate(state): - state[idx] = tensor.to_ndarray() - - return state - - def set_hidden_state(self, states): - """ - set hidden state and cell at first time step. - """ - jstate, state_is_table = self.check_input(states) - callBigDlFunc(self.bigdl_type, "setHiddenState", self.value, jstate, state_is_table) + states = callBigDlFunc(self.bigdl_type, "getHiddenState", self.value) + return states class RecurrentDecoder(Recurrent): ''' @@ -4639,6 +4629,23 @@ def __init__(self, input_size, output_size, kernel_i, kernel_c, stride=1, paddin super(ConvLSTMPeephole3D, self).__init__(None, bigdl_type, input_size, output_size, kernel_i, kernel_c, stride, padding, wRegularizer, uRegularizer, bRegularizer, cRegularizer, with_peephole) + +class MultiRNNCell(Layer): + ''' + A cell that enables stack multiple simple rnn cells + + >>> cells = [] + >>> cells.append(ConvLSTMPeephole3D(4, 3, 3, 3, 1)) + creating: createConvLSTMPeephole3D + >>> cells.append(ConvLSTMPeephole3D(4, 3, 3, 3, 1)) + creating: createConvLSTMPeephole3D + >>> stacked_convlstm = MultiRNNCell(cells) + creating: createMultiRNNCell + ''' + + def __init__(self, cells, bigdl_type="float"): + super(MultiRNNCell, self).__init__(None, bigdl_type, cells) + class ResizeBilinear(Layer): """ Resize the input image with bilinear interpolation. The input image must be a float tensor with From 339b6acae664a9afae7469d35a42107887b81146 Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Fri, 8 Dec 2017 09:21:07 +0800 Subject: [PATCH 408/823] Support Python 3.6 (#1979) * test python 3.6 * update * check spark version * update docs --- python/dllib/test/dev/diff.py | 8 ++++---- python/dllib/test/dev/prepare_env.sh | 6 +++++- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/python/dllib/test/dev/diff.py b/python/dllib/test/dev/diff.py index 8e417e8429c..2ad47ab2e36 100755 --- a/python/dllib/test/dev/diff.py +++ b/python/dllib/test/dev/diff.py @@ -59,15 +59,15 @@ def get_python_classes(nn_root): scala_layers = get_scala_classes(scala_nn_root) python_layers = get_python_classes(python_nn_root) -print "scala_layers: {0}, {1}".format(len(scala_layers), scala_layers) +print("scala_layers: {0}, {1}".format(len(scala_layers), scala_layers)) print("\n") -print "python_layers: {0}, {1}".format(len(python_layers), python_layers) +print("python_layers: {0}, {1}".format(len(python_layers), python_layers)) -print "In scala, not in python: ", +print("In scala, not in python: "), diff_count = 0 for name in scala_layers: if name not in python_layers and scala_to_python[name] not in python_layers: - print name, + print(name), diff_count += 1 if diff_count > 0: diff --git a/python/dllib/test/dev/prepare_env.sh b/python/dllib/test/dev/prepare_env.sh index 27c0900d881..8e61a0f0025 100755 --- a/python/dllib/test/dev/prepare_env.sh +++ b/python/dllib/test/dev/prepare_env.sh @@ -33,7 +33,11 @@ export PYTHONPATH=$PYTHONPATH:$PYSPARK_ZIP:$DL_PYTHON_HOME:$DL_PYTHON_HOME/:$DL_ export BIGDL_CLASSPATH=$(find $BIGDL_HOME/spark/dl/target/ -name "*with-dependencies.jar" | head -n 1) echo "BIGDL_CLASSPATH": $BIGDL_CLASSPATH -export PYTHON_EXECUTABLES=("python2.7" "python3.5") +if [[ ($SPARK_HOME == *"2.2.0"*) || ($SPARK_HOME == *"2.1.1"*) || ($SPARK_HOME == *"1.6.4"*) ]]; then + export PYTHON_EXECUTABLES=("python2.7" "python3.5" "python3.6") +else + export PYTHON_EXECUTABLES=("python2.7" "python3.5") +fi function run_notebook() { notebook_path=$1 From 6463910800c087ced0fa214fdd97abc11855f6bf Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Fri, 8 Dec 2017 14:12:41 +0800 Subject: [PATCH 409/823] Fix sample table-label for tutorial nightly build (#1991) * fix * add feature=features[0] * update docs --- python/dllib/src/bigdl/dllib/utils/common.py | 55 +++++++++++--------- 1 file changed, 31 insertions(+), 24 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/utils/common.py b/python/dllib/src/bigdl/dllib/utils/common.py index 6d8d806d5fd..8c4378468e5 100644 --- a/python/dllib/src/bigdl/dllib/utils/common.py +++ b/python/dllib/src/bigdl/dllib/utils/common.py @@ -265,23 +265,25 @@ def __repr__(self): class Sample(object): - def __init__(self, features, label, bigdl_type="float"): + def __init__(self, features, labels, bigdl_type="float"): """ User should always use Sample.from_ndarray to construct Sample. :param features: a list of JTensors - :param label: a list of JTensors + :param labels: a list of JTensors :param bigdl_type: "double" or "float" """ + self.feature = features[0] self.features = features - self.label = label + self.label = labels[0] self.bigdl_type = bigdl_type + self.labels = labels @classmethod - def from_ndarray(cls, features, label, bigdl_type="float"): + def from_ndarray(cls, features, labels, bigdl_type="float"): """ - Convert a ndarray of features and label to Sample, which would be used in Java side. + Convert a ndarray of features and labels to Sample, which would be used in Java side. :param features: an ndarray or a list of ndarrays - :param label: an ndarray or a list of ndarrays or a scalar + :param labels: an ndarray or a list of ndarrays or a scalar :param bigdl_type: "double" or "float" >>> import numpy as np @@ -291,10 +293,10 @@ def from_ndarray(cls, features, label, bigdl_type="float"): >>> sample = Sample.from_ndarray(np.random.random((2,3)), np.random.random((2,3))) >>> sample_back = callBigDlFunc("float", "testSample", sample) >>> assert_allclose(sample.features[0].to_ndarray(), sample_back.features[0].to_ndarray()) - >>> assert_allclose(sample.label[0].to_ndarray(), sample_back.label[0].to_ndarray()) + >>> assert_allclose(sample.label.to_ndarray(), sample_back.label.to_ndarray()) >>> print(sample) Sample: features: [JTensor: storage: [[ 0.69646919 0.28613934 0.22685145] - [ 0.55131477 0.71946895 0.42310646]], shape: [2 3], float], label: [JTensor: storage: [[ 0.98076421 0.68482971 0.48093191] + [ 0.55131477 0.71946895 0.42310646]], shape: [2 3], float], labels: [JTensor: storage: [[ 0.98076421 0.68482971 0.48093191] [ 0.39211753 0.343178 0.72904968]], shape: [2 3], float], """ if isinstance(features, np.ndarray): @@ -302,24 +304,24 @@ def from_ndarray(cls, features, label, bigdl_type="float"): else: assert all(isinstance(feature, np.ndarray) for feature in features), \ "features should be a list of np.ndarray, not %s" % type(features) - if np.isscalar(label): # in case label is a scalar. - label = [np.array(label)] - elif isinstance(label, np.ndarray): - label = [label] + if np.isscalar(labels): # in case labels is a scalar. + labels = [np.array(labels)] + elif isinstance(labels, np.ndarray): + labels = [labels] else: - assert all(isinstance(l, np.ndarray) for l in label), \ - "label should be a list of np.ndarray, not %s" % type(label) + assert all(isinstance(label, np.ndarray) for label in labels), \ + "labels should be a list of np.ndarray, not %s" % type(labels) return cls( - features=[JTensor.from_ndarray(f) for f in features], - label=[JTensor.from_ndarray(l) for l in label], + features=[JTensor.from_ndarray(feature) for feature in features], + labels=[JTensor.from_ndarray(label) for label in labels], bigdl_type=bigdl_type) @classmethod - def from_jtensor(cls, features, label, bigdl_type="float"): + def from_jtensor(cls, features, labels, bigdl_type="float"): """ Convert a sequence of JTensor to Sample, which would be used in Java side. :param features: an JTensor or a list of JTensor - :param label: an JTensor or a scalar + :param labels: an JTensor or a list of JTensor or a scalar :param bigdl_type: "double" or "float" >>> import numpy as np @@ -335,21 +337,26 @@ def from_jtensor(cls, features, label, bigdl_type="float"): else: assert all(isinstance(feature, JTensor) for feature in features), \ "features should be a list of JTensor, not %s" % type(features) - if not isinstance(label, JTensor): # in case label is a scalar. - label = JTensor.from_ndarray(np.array(label)) + if np.isscalar(labels): # in case labels is a scalar. + labels = [JTensor.from_ndarray(np.array(labels))] + elif isinstance(labels, JTensor): + labels = [labels] + else: + assert all(isinstance(label, JTensor) for label in labels), \ + "labels should be a list of np.ndarray, not %s" % type(labels) return cls( features=features, - label=label, + labels=labels, bigdl_type=bigdl_type) def __reduce__(self): - return Sample, (self.features, self.label, self.bigdl_type) + return Sample, (self.features, self.labels, self.bigdl_type) def __str__(self): - return "Sample: features: %s, label: %s," % (self.features, self.label) + return "Sample: features: %s, labels: %s," % (self.features, self.labels) def __repr__(self): - return "Sample: features: %s, label: %s" % (self.features, self.label) + return "Sample: features: %s, labels: %s" % (self.features, self.labels) class RNG(): """ From 163e786d7bbbaa0507524fae3175b9127ab64a18 Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Fri, 8 Dec 2017 14:12:41 +0800 Subject: [PATCH 410/823] Fix sample table-label for tutorial nightly build (#1991) * fix * add feature=features[0] * update docs --- python/dllib/src/bigdl/utils/common.py | 55 +++++++++++++++----------- 1 file changed, 31 insertions(+), 24 deletions(-) diff --git a/python/dllib/src/bigdl/utils/common.py b/python/dllib/src/bigdl/utils/common.py index 6d8d806d5fd..8c4378468e5 100644 --- a/python/dllib/src/bigdl/utils/common.py +++ b/python/dllib/src/bigdl/utils/common.py @@ -265,23 +265,25 @@ def __repr__(self): class Sample(object): - def __init__(self, features, label, bigdl_type="float"): + def __init__(self, features, labels, bigdl_type="float"): """ User should always use Sample.from_ndarray to construct Sample. :param features: a list of JTensors - :param label: a list of JTensors + :param labels: a list of JTensors :param bigdl_type: "double" or "float" """ + self.feature = features[0] self.features = features - self.label = label + self.label = labels[0] self.bigdl_type = bigdl_type + self.labels = labels @classmethod - def from_ndarray(cls, features, label, bigdl_type="float"): + def from_ndarray(cls, features, labels, bigdl_type="float"): """ - Convert a ndarray of features and label to Sample, which would be used in Java side. + Convert a ndarray of features and labels to Sample, which would be used in Java side. :param features: an ndarray or a list of ndarrays - :param label: an ndarray or a list of ndarrays or a scalar + :param labels: an ndarray or a list of ndarrays or a scalar :param bigdl_type: "double" or "float" >>> import numpy as np @@ -291,10 +293,10 @@ def from_ndarray(cls, features, label, bigdl_type="float"): >>> sample = Sample.from_ndarray(np.random.random((2,3)), np.random.random((2,3))) >>> sample_back = callBigDlFunc("float", "testSample", sample) >>> assert_allclose(sample.features[0].to_ndarray(), sample_back.features[0].to_ndarray()) - >>> assert_allclose(sample.label[0].to_ndarray(), sample_back.label[0].to_ndarray()) + >>> assert_allclose(sample.label.to_ndarray(), sample_back.label.to_ndarray()) >>> print(sample) Sample: features: [JTensor: storage: [[ 0.69646919 0.28613934 0.22685145] - [ 0.55131477 0.71946895 0.42310646]], shape: [2 3], float], label: [JTensor: storage: [[ 0.98076421 0.68482971 0.48093191] + [ 0.55131477 0.71946895 0.42310646]], shape: [2 3], float], labels: [JTensor: storage: [[ 0.98076421 0.68482971 0.48093191] [ 0.39211753 0.343178 0.72904968]], shape: [2 3], float], """ if isinstance(features, np.ndarray): @@ -302,24 +304,24 @@ def from_ndarray(cls, features, label, bigdl_type="float"): else: assert all(isinstance(feature, np.ndarray) for feature in features), \ "features should be a list of np.ndarray, not %s" % type(features) - if np.isscalar(label): # in case label is a scalar. - label = [np.array(label)] - elif isinstance(label, np.ndarray): - label = [label] + if np.isscalar(labels): # in case labels is a scalar. + labels = [np.array(labels)] + elif isinstance(labels, np.ndarray): + labels = [labels] else: - assert all(isinstance(l, np.ndarray) for l in label), \ - "label should be a list of np.ndarray, not %s" % type(label) + assert all(isinstance(label, np.ndarray) for label in labels), \ + "labels should be a list of np.ndarray, not %s" % type(labels) return cls( - features=[JTensor.from_ndarray(f) for f in features], - label=[JTensor.from_ndarray(l) for l in label], + features=[JTensor.from_ndarray(feature) for feature in features], + labels=[JTensor.from_ndarray(label) for label in labels], bigdl_type=bigdl_type) @classmethod - def from_jtensor(cls, features, label, bigdl_type="float"): + def from_jtensor(cls, features, labels, bigdl_type="float"): """ Convert a sequence of JTensor to Sample, which would be used in Java side. :param features: an JTensor or a list of JTensor - :param label: an JTensor or a scalar + :param labels: an JTensor or a list of JTensor or a scalar :param bigdl_type: "double" or "float" >>> import numpy as np @@ -335,21 +337,26 @@ def from_jtensor(cls, features, label, bigdl_type="float"): else: assert all(isinstance(feature, JTensor) for feature in features), \ "features should be a list of JTensor, not %s" % type(features) - if not isinstance(label, JTensor): # in case label is a scalar. - label = JTensor.from_ndarray(np.array(label)) + if np.isscalar(labels): # in case labels is a scalar. + labels = [JTensor.from_ndarray(np.array(labels))] + elif isinstance(labels, JTensor): + labels = [labels] + else: + assert all(isinstance(label, JTensor) for label in labels), \ + "labels should be a list of np.ndarray, not %s" % type(labels) return cls( features=features, - label=label, + labels=labels, bigdl_type=bigdl_type) def __reduce__(self): - return Sample, (self.features, self.label, self.bigdl_type) + return Sample, (self.features, self.labels, self.bigdl_type) def __str__(self): - return "Sample: features: %s, label: %s," % (self.features, self.label) + return "Sample: features: %s, labels: %s," % (self.features, self.labels) def __repr__(self): - return "Sample: features: %s, label: %s" % (self.features, self.label) + return "Sample: features: %s, labels: %s" % (self.features, self.labels) class RNG(): """ From 3a26a0943cb6e41cd3818193ca4dac3627e881f2 Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Fri, 8 Dec 2017 18:13:02 +0800 Subject: [PATCH 411/823] Map newly added keras layers on python side (#1990) * update * add maxoutdense * add masking * add srelu * highway activation as string; update docs * code clean --- .../dllib/src/bigdl/dllib/keras/converter.py | 86 +++++++++++++------ python/dllib/src/bigdl/dllib/nn/layer.py | 12 +-- 2 files changed, 68 insertions(+), 30 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/keras/converter.py b/python/dllib/src/bigdl/dllib/keras/converter.py index 8ac62033eb4..71a72941413 100644 --- a/python/dllib/src/bigdl/dllib/keras/converter.py +++ b/python/dllib/src/bigdl/dllib/keras/converter.py @@ -222,13 +222,23 @@ def convert_gru(klayer, weights): return [w1, w2, w3, w4] @staticmethod - def convert_maxoutdense(weights): + def convert_highway(klayer, weights): + if len(weights) == 2: # if without bias + return [weights[1].T, weights[0].T] + return [weights[1].T, weights[3], weights[0].T, weights[2]] + + @staticmethod + def convert_maxoutdense(klayer, weights): k_weights = weights[0] b_weights = k_weights[0].T for i in range(1, k_weights.shape[0]): b_weights = np.concatenate((b_weights, k_weights[i].T)) return [b_weights, weights[1]] + @staticmethod + def convert_srelu(klayer, weights): + return weights + class DefinitionLoader: @@ -694,6 +704,7 @@ def __generate_zeropadding2d(self, dim1, dim2, n_input_dim, pad1, pad2, pad3, pa def create_zeropadding2d(self): padding = self.klayer.padding dim = 1 + warnings.warn("Cannot find dim_ordering from json definition. Using the default instead.") if self.klayer.dim_ordering == "th": dim = 2 if isinstance(padding, dict): # dictionary @@ -713,6 +724,7 @@ def create_zeropadding2d(self): def create_zeropadding3d(self): padding = tuple(self.klayer.padding) dim = 1 + warnings.warn("Cannot find dim_ordering from json definition. Using the default instead.") if self.klayer.dim_ordering == "th": dim = 2 model = BLayer.Sequential() @@ -785,7 +797,6 @@ def __process_recurrent_layer(self, return_sequences, go_backwards, blayer): def generate_simplernn_cell(self, klayer, kclayer, input_shape): # create a simplernn cell only self.__check_recurrent_parameters(klayer) config = kclayer["config"] - self.check_constraint_in_config(config) activation = get_activation_by_name(config["activation"], "%s_%s" % (config["name"], config["activation"])) rnn = BLayer.RnnCell(input_size=int(input_shape[2]), @@ -807,7 +818,6 @@ def create_simplernn(self): def generate_lstm_cell(self, klayer, kclayer, input_shape): # create a lstm cell only self.__check_recurrent_parameters(klayer) config = kclayer["config"] - self.check_constraint_in_config(config) activation = get_activation_by_name(config["activation"], "%s_%s" % (config["name"], config["activation"])) inner_activation = get_activation_by_name(config["inner_activation"], @@ -832,7 +842,6 @@ def create_lstm(self): def generate_convlstm2d_cell(self, klayer, kclayer, input_shape): # create a convlstm2d cell only self.__check_recurrent_parameters(klayer) config = kclayer["config"] - self.check_constraint_in_config(config) activation = get_activation_by_name(config["activation"], "%s_%s" % (config["name"], config["activation"])) inner_activation = get_activation_by_name(config["inner_activation"], @@ -876,7 +885,6 @@ def create_convlstm2d(self): def generate_gru_cell(self, klayer, kclayer, input_shape): # create a gru cell only self.__check_recurrent_parameters(klayer) config = kclayer["config"] - self.check_constraint_in_config(config) activation = get_activation_by_name(config["activation"], "%s_%s" % (config["name"], config["activation"])) inner_activation = get_activation_by_name(config["inner_activation"], @@ -968,16 +976,6 @@ def to_bigdl_2d_padding(self, border_mode): else: raise Exception("Unsupported border mode: %s" % border_mode) - def to_bigdl_1d_padding(self, border_mode, kernel_w): - if border_mode == "same": - raise Exception("We don't support padding for now") - # TODO: support padding - # return int((kernel_w -1) / 2) - elif border_mode == "valid": - return 0 - else: - raise Exception("Unsupported border mode: %s" % border_mode) - def create_convolution1d(self): # batch, steps, dim, batch is None here, so you cannot use it directly. stack_size = int(self.input_shape[2]) @@ -1427,17 +1425,55 @@ def create_globalaveragepooling2d(self): seq.add(BLayer.Squeeze(1, num_input_dims=2)) return seq - def check_constraint_in_config(self, config): - if "W_constraint" in config: - if config["W_constraint"]: - raise Exception("W_constraint is not supported for now") - if "b_constraint" in config: - if config["b_constraint"]: - raise Exception("b_constraint is not supported for now") + def create_upsampling3d(self): + if self.klayer.dim_ordering != "th": + raise Exception("Please use th for dim_ordering. %s is not supported for now." % self.klayer.dim_ordering) + warnings.warn("Cannot find dim_ordering from json definition. Using the default instead." + "We only support th for now.") + return BLayer.UpSampling3D(self.klayer.size) - def combo_parameter_layer(self, blayer, config): - self.check_constraint_in_config(config) + def create_gaussiannoise(self): + return BLayer.GaussianNoise(float(self.klayer.sigma)) + def create_gaussiandropout(self): + return BLayer.GaussianDropout(float(self.klayer.p)) + + def create_highway(self): + if self.config["activation"] == 'linear': + activation = None + else: + activation = get_activation_by_name(self.config["activation"], + "%s_%s" % (self.config["name"], self.config["activation"])) + blayer = BLayer.Highway(size=self.input_shape[1], + with_bias=self.klayer.bias, + activation=activation, + wRegularizer=self.to_bigdl_reg(self.config["W_regularizer"]), + bRegularizer=self.to_bigdl_reg(self.config["b_regularizer"])) + return blayer + + def create_maxoutdense(self): + if self.config["W_regularizer"]: + raise Exception("W_regularizer is not supported for MaxoutDense") + if self.config["b_regularizer"]: + raise Exception("b_regularizer is not supported for MaxoutDense") + if not self.config["bias"]: + raise Exception("Please set bias=True for MaxoutDense") + blayer = BLayer.Maxout(input_size=self.input_shape[1], + output_size=self.klayer.output_dim, + maxout_number=self.klayer.nb_feature) + return blayer + + def create_masking(self): + return BLayer.Masking(float(self.klayer.mask_value)) + + def create_srelu(self): + warnings.warn("Cannot find shared_axes from json definition. Using shared_axes=None instead.") + if self.klayer.shared_axes == [None]: + return BLayer.SReLU() + else: + return BLayer.SReLU(self.klayer.shared_axes) + + def combo_parameter_layer(self, blayer, config): blayer.set_name(config["name"]) if hasattr(blayer, "set_init_method"): blayer.set_init_method(self.to_bigdl_init(config["init"]), @@ -1445,7 +1481,7 @@ def combo_parameter_layer(self, blayer, config): # "linear" means doing nothing if config["activation"] != "linear": activation = get_activation_by_name(config["activation"], - "%s_%s" % (config["name"], config["activation"])) + "%s_%s" % (config["name"], config["activation"])) return self.fuse(blayer, activation) else: return blayer diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 739ed2d67e9..8af6f320bab 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -3276,8 +3276,8 @@ def __init__(self, super(SReLU, self).__init__(None, bigdl_type, share_axes) - def set_init_method(self, tLeftInit = None, aLeftInit = None, - tRightInit = None, aRightInit = None): + def set_init_method(self, tLeftInit=None, aLeftInit=None, + tRightInit=None, aRightInit=None): callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, tLeftInit, aLeftInit, tRightInit, aRightInit) return self @@ -4742,14 +4742,16 @@ class Highway(Layer): :param size input size :param with_bias whether to include a bias - :param activation name of activation function to use + :param activation activation function. It can also be the name of an existing activation as a string. :param wRegularizer: instance of [[Regularizer]](eg. L1 or L2 regularization), applied to the input weights matrices. - :param bRegularizer: instance of [[Regularizer]]applied to the bias. + :param bRegularizer: instance of [[Regularizer]], applied to the bias. >>> highway = Highway(2) creating: createHighway """ - def __init__(self, size, with_bias=True, activation = None, wRegularizer=None, bRegularizer=None, bigdl_type="float"): + def __init__(self, size, with_bias=True, activation=None, wRegularizer=None, bRegularizer=None, bigdl_type="float"): + if isinstance(activation, six.string_types): + activation = get_activation_by_name(activation) super(Highway, self).__init__(None, bigdl_type, size, with_bias, activation, wRegularizer, bRegularizer) class UpSampling3D(Layer): From 6ddd7c19575cc535a0b983648db06f7a2a6d03a1 Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Fri, 8 Dec 2017 18:13:02 +0800 Subject: [PATCH 412/823] Map newly added keras layers on python side (#1990) * update * add maxoutdense * add masking * add srelu * highway activation as string; update docs * code clean --- .../dllib/src/bigdl/dllib/keras/converter.py | 86 +++++++++++++------ python/dllib/src/bigdl/dllib/nn/layer.py | 12 +-- 2 files changed, 68 insertions(+), 30 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/keras/converter.py b/python/dllib/src/bigdl/dllib/keras/converter.py index 8ac62033eb4..71a72941413 100644 --- a/python/dllib/src/bigdl/dllib/keras/converter.py +++ b/python/dllib/src/bigdl/dllib/keras/converter.py @@ -222,13 +222,23 @@ def convert_gru(klayer, weights): return [w1, w2, w3, w4] @staticmethod - def convert_maxoutdense(weights): + def convert_highway(klayer, weights): + if len(weights) == 2: # if without bias + return [weights[1].T, weights[0].T] + return [weights[1].T, weights[3], weights[0].T, weights[2]] + + @staticmethod + def convert_maxoutdense(klayer, weights): k_weights = weights[0] b_weights = k_weights[0].T for i in range(1, k_weights.shape[0]): b_weights = np.concatenate((b_weights, k_weights[i].T)) return [b_weights, weights[1]] + @staticmethod + def convert_srelu(klayer, weights): + return weights + class DefinitionLoader: @@ -694,6 +704,7 @@ def __generate_zeropadding2d(self, dim1, dim2, n_input_dim, pad1, pad2, pad3, pa def create_zeropadding2d(self): padding = self.klayer.padding dim = 1 + warnings.warn("Cannot find dim_ordering from json definition. Using the default instead.") if self.klayer.dim_ordering == "th": dim = 2 if isinstance(padding, dict): # dictionary @@ -713,6 +724,7 @@ def create_zeropadding2d(self): def create_zeropadding3d(self): padding = tuple(self.klayer.padding) dim = 1 + warnings.warn("Cannot find dim_ordering from json definition. Using the default instead.") if self.klayer.dim_ordering == "th": dim = 2 model = BLayer.Sequential() @@ -785,7 +797,6 @@ def __process_recurrent_layer(self, return_sequences, go_backwards, blayer): def generate_simplernn_cell(self, klayer, kclayer, input_shape): # create a simplernn cell only self.__check_recurrent_parameters(klayer) config = kclayer["config"] - self.check_constraint_in_config(config) activation = get_activation_by_name(config["activation"], "%s_%s" % (config["name"], config["activation"])) rnn = BLayer.RnnCell(input_size=int(input_shape[2]), @@ -807,7 +818,6 @@ def create_simplernn(self): def generate_lstm_cell(self, klayer, kclayer, input_shape): # create a lstm cell only self.__check_recurrent_parameters(klayer) config = kclayer["config"] - self.check_constraint_in_config(config) activation = get_activation_by_name(config["activation"], "%s_%s" % (config["name"], config["activation"])) inner_activation = get_activation_by_name(config["inner_activation"], @@ -832,7 +842,6 @@ def create_lstm(self): def generate_convlstm2d_cell(self, klayer, kclayer, input_shape): # create a convlstm2d cell only self.__check_recurrent_parameters(klayer) config = kclayer["config"] - self.check_constraint_in_config(config) activation = get_activation_by_name(config["activation"], "%s_%s" % (config["name"], config["activation"])) inner_activation = get_activation_by_name(config["inner_activation"], @@ -876,7 +885,6 @@ def create_convlstm2d(self): def generate_gru_cell(self, klayer, kclayer, input_shape): # create a gru cell only self.__check_recurrent_parameters(klayer) config = kclayer["config"] - self.check_constraint_in_config(config) activation = get_activation_by_name(config["activation"], "%s_%s" % (config["name"], config["activation"])) inner_activation = get_activation_by_name(config["inner_activation"], @@ -968,16 +976,6 @@ def to_bigdl_2d_padding(self, border_mode): else: raise Exception("Unsupported border mode: %s" % border_mode) - def to_bigdl_1d_padding(self, border_mode, kernel_w): - if border_mode == "same": - raise Exception("We don't support padding for now") - # TODO: support padding - # return int((kernel_w -1) / 2) - elif border_mode == "valid": - return 0 - else: - raise Exception("Unsupported border mode: %s" % border_mode) - def create_convolution1d(self): # batch, steps, dim, batch is None here, so you cannot use it directly. stack_size = int(self.input_shape[2]) @@ -1427,17 +1425,55 @@ def create_globalaveragepooling2d(self): seq.add(BLayer.Squeeze(1, num_input_dims=2)) return seq - def check_constraint_in_config(self, config): - if "W_constraint" in config: - if config["W_constraint"]: - raise Exception("W_constraint is not supported for now") - if "b_constraint" in config: - if config["b_constraint"]: - raise Exception("b_constraint is not supported for now") + def create_upsampling3d(self): + if self.klayer.dim_ordering != "th": + raise Exception("Please use th for dim_ordering. %s is not supported for now." % self.klayer.dim_ordering) + warnings.warn("Cannot find dim_ordering from json definition. Using the default instead." + "We only support th for now.") + return BLayer.UpSampling3D(self.klayer.size) - def combo_parameter_layer(self, blayer, config): - self.check_constraint_in_config(config) + def create_gaussiannoise(self): + return BLayer.GaussianNoise(float(self.klayer.sigma)) + def create_gaussiandropout(self): + return BLayer.GaussianDropout(float(self.klayer.p)) + + def create_highway(self): + if self.config["activation"] == 'linear': + activation = None + else: + activation = get_activation_by_name(self.config["activation"], + "%s_%s" % (self.config["name"], self.config["activation"])) + blayer = BLayer.Highway(size=self.input_shape[1], + with_bias=self.klayer.bias, + activation=activation, + wRegularizer=self.to_bigdl_reg(self.config["W_regularizer"]), + bRegularizer=self.to_bigdl_reg(self.config["b_regularizer"])) + return blayer + + def create_maxoutdense(self): + if self.config["W_regularizer"]: + raise Exception("W_regularizer is not supported for MaxoutDense") + if self.config["b_regularizer"]: + raise Exception("b_regularizer is not supported for MaxoutDense") + if not self.config["bias"]: + raise Exception("Please set bias=True for MaxoutDense") + blayer = BLayer.Maxout(input_size=self.input_shape[1], + output_size=self.klayer.output_dim, + maxout_number=self.klayer.nb_feature) + return blayer + + def create_masking(self): + return BLayer.Masking(float(self.klayer.mask_value)) + + def create_srelu(self): + warnings.warn("Cannot find shared_axes from json definition. Using shared_axes=None instead.") + if self.klayer.shared_axes == [None]: + return BLayer.SReLU() + else: + return BLayer.SReLU(self.klayer.shared_axes) + + def combo_parameter_layer(self, blayer, config): blayer.set_name(config["name"]) if hasattr(blayer, "set_init_method"): blayer.set_init_method(self.to_bigdl_init(config["init"]), @@ -1445,7 +1481,7 @@ def combo_parameter_layer(self, blayer, config): # "linear" means doing nothing if config["activation"] != "linear": activation = get_activation_by_name(config["activation"], - "%s_%s" % (config["name"], config["activation"])) + "%s_%s" % (config["name"], config["activation"])) return self.fuse(blayer, activation) else: return blayer diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 739ed2d67e9..8af6f320bab 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -3276,8 +3276,8 @@ def __init__(self, super(SReLU, self).__init__(None, bigdl_type, share_axes) - def set_init_method(self, tLeftInit = None, aLeftInit = None, - tRightInit = None, aRightInit = None): + def set_init_method(self, tLeftInit=None, aLeftInit=None, + tRightInit=None, aRightInit=None): callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, tLeftInit, aLeftInit, tRightInit, aRightInit) return self @@ -4742,14 +4742,16 @@ class Highway(Layer): :param size input size :param with_bias whether to include a bias - :param activation name of activation function to use + :param activation activation function. It can also be the name of an existing activation as a string. :param wRegularizer: instance of [[Regularizer]](eg. L1 or L2 regularization), applied to the input weights matrices. - :param bRegularizer: instance of [[Regularizer]]applied to the bias. + :param bRegularizer: instance of [[Regularizer]], applied to the bias. >>> highway = Highway(2) creating: createHighway """ - def __init__(self, size, with_bias=True, activation = None, wRegularizer=None, bRegularizer=None, bigdl_type="float"): + def __init__(self, size, with_bias=True, activation=None, wRegularizer=None, bRegularizer=None, bigdl_type="float"): + if isinstance(activation, six.string_types): + activation = get_activation_by_name(activation) super(Highway, self).__init__(None, bigdl_type, size, with_bias, activation, wRegularizer, bRegularizer) class UpSampling3D(Layer): From de63ede7f805b7433d9da71958b2b0cb109d979b Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Fri, 8 Dec 2017 18:13:02 +0800 Subject: [PATCH 413/823] Map newly added keras layers on python side (#1990) * update * add maxoutdense * add masking * add srelu * highway activation as string; update docs * code clean --- python/dllib/test/bigdl/keras/test_layer.py | 28 +++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/python/dllib/test/bigdl/keras/test_layer.py b/python/dllib/test/bigdl/keras/test_layer.py index 48e9ffc9d45..bba9db09e4d 100644 --- a/python/dllib/test/bigdl/keras/test_layer.py +++ b/python/dllib/test/bigdl/keras/test_layer.py @@ -619,6 +619,34 @@ def test_wrapper_bidirectional(self): input_shape=(8, 40, 40, 32), merge_mode='ave') self.modelTestSingleLayer(input_data2, layer5, dump_weights=True) + def test_upsampling3d(self): + input_data = np.random.random([2, 5, 12, 12, 12]) + layer1 = UpSampling3D(input_shape=(5, 12, 12, 12)) + self.modelTestSingleLayer(input_data, layer1) + layer2 = UpSampling3D(size=(1, 2, 4), input_shape=(5, 12, 12, 12)) + self.modelTestSingleLayer(input_data, layer2) + + def test_highway(self): + input_data = np.random.random([4, 6]) + layer = Highway(input_shape=(6, )) + self.modelTestSingleLayer(input_data, layer, dump_weights=True) + layer2 = Highway(activation='sigmoid', bias=False, input_shape=(6, )) + self.modelTestSingleLayer(input_data, layer2, dump_weights=True) + + def test_maxoutdense(self): + input_data = np.random.random([4, 6]) + layer = MaxoutDense(3, 5) + self.modelTestSingleLayer(input_data, layer, dump_weights=True) + + def test_masking(self): + input_data = np.array([[[0, 1, 2], [-1, 1, 0], [3, 4, 1], [0, 0, 0]]]) + layer = Masking(-1, input_shape=(4, 3)) + self.modelTestSingleLayer(input_data, layer) + + def test_srelu(self): + input_data = np.random.random_sample([2, 4, 6]) + layer = SReLU(input_shape=(4, 6)) + self.modelTestSingleLayer(input_data, layer, dump_weights=True) if __name__ == "__main__": pytest.main([__file__]) From 54a0cd3d01a451b3af82d7da5ce1826e495745a7 Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Mon, 11 Dec 2017 10:17:27 +0800 Subject: [PATCH 414/823] A keras backend wrapper (#1945) * good * backend * add unitest * update doc * style * update test * fix test * test --- .../src/bigdl/dllib/bigdlkeras/backend.py | 181 ++++++++++++++++++ .../bigdl/dllib/bigdlkeras/optimization.py | 63 ++++++ .../dllib/src/bigdl/dllib/keras/converter.py | 15 ++ python/dllib/src/bigdl/dllib/nn/layer.py | 2 +- python/dllib/src/bigdl/dllib/utils/common.py | 15 ++ 5 files changed, 275 insertions(+), 1 deletion(-) create mode 100644 python/dllib/src/bigdl/dllib/bigdlkeras/backend.py create mode 100644 python/dllib/src/bigdl/dllib/bigdlkeras/optimization.py diff --git a/python/dllib/src/bigdl/dllib/bigdlkeras/backend.py b/python/dllib/src/bigdl/dllib/bigdlkeras/backend.py new file mode 100644 index 00000000000..02a4abf771d --- /dev/null +++ b/python/dllib/src/bigdl/dllib/bigdlkeras/backend.py @@ -0,0 +1,181 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import numpy as np +from pyspark.rdd import RDD + +from bigdl.keras.optimization import * +from bigdl.util.common import get_spark_context +from bigdl.util.common import to_sample_rdd + + +class KerasModelWrapper(): + + def __init__(self, kmodel): + self.bmodel = DefinitionLoader.from_kmodel(kmodel) + WeightLoader.load_weights_from_kmodel(self.bmodel, kmodel) # share the same weight. + self.criterion = OptimConverter.to_bigdl_criterion(kmodel.loss) + self.optim_method = OptimConverter.to_bigdl_optim_method(kmodel.optimizer) + self.metrics = OptimConverter.to_bigdl_metrics(kmodel.metrics) if kmodel.metrics else None + + def evaluate(self, x, y, batch_size=32, sample_weight=None, is_distributed=False): + """ + Evaluate a model by the given metrics. + :param x: ndarray or list of ndarray for local mode. + RDD[Sample] for distributed mode + :param y: ndarray or list of ndarray for local mode and would be None for cluster mode. + :param batch_size + :param is_distributed: run in local mode or distributed mode. + NB: if is_distributed=true, x should be RDD[Sample] and y should be None + :return: + """ + if sample_weight: + unsupport_exp("sample_weight") + if is_distributed: + if isinstance(x, np.ndarray): + input = to_sample_rdd(x, y) + elif isinstance(x, RDD): + input = x + if self.metrics: + sc = get_spark_context() + return [r.result for r in + self.bmodel.evaluate(input, batch_size, self.metrics)] + else: + raise Exception("No Metrics found.") + + raise Exception("not supported operation: %s", is_distributed) + + def predict(self, x, batch_size=None, verbose=None, is_distributed=False): + """Generates output predictions for the input samples, + processing the samples in a batched way. + + # Arguments + x: the input data, as a Numpy array or list of Numpy array for local mode. + as RDD[Sample] for distributed mode + is_distributed: used to control run in local or cluster. the default value is False + # Returns + A Numpy array or RDD[Sample] of predictions. + """ + if batch_size or verbose: + raise Exception("we don't support batch_size or verbose for now") + if is_distributed: + if isinstance(x, np.ndarray): + input = to_sample_rdd(x, np.zeros([x.shape[0]])) + # np.asarray(self.bmodel.predict(x_rdd).collect()) + elif isinstance(x, RDD): + input = x + return self.bmodel.predict(input) + else: + if isinstance(x, np.ndarray): + return self.bmodel.predict_local(x) + raise Exception("not supported type: %s" % x) + + def fit(self, x, y=None, batch_size=32, nb_epoch=10, verbose=1, callbacks=None, + validation_split=0., validation_data=None, shuffle=True, + class_weight=None, sample_weight=None, initial_epoch=0, is_distributed=False): + """Optimize the model by the given options + + :param x: ndarray or list of ndarray for local mode. + RDD[Sample] for distributed mode + :param y: ndarray or list of ndarray for local mode and would be None for cluster mode. + is_distributed: used to control run in local or cluster. the default value is False. + NB: if is_distributed=true, x should be RDD[Sample] and y should be None + :return: + A Numpy array or RDD[Sample] of predictions. + """ + if callbacks: + raise Exception("We don't support callbacks in fit for now") + if class_weight: + unsupport_exp("class_weight") + if sample_weight: + unsupport_exp("sample_weight") + if initial_epoch != 0: + unsupport_exp("initial_epoch") + if shuffle != True: + unsupport_exp("shuffle") + if validation_split != 0.: + unsupport_exp("validation_split") + bopt = self.__create_optimizer(x=x, + y=y, + batch_size=batch_size, + nb_epoch=nb_epoch, + validation_data=validation_data, + is_distributed=is_distributed) + bopt.optimize() + + def __create_optimizer(self, x=None, y=None, batch_size=32, nb_epoch=10, + validation_data=None, is_distributed=False): + if is_distributed: + if isinstance(x, np.ndarray): + input = to_sample_rdd(x, y) + validation_data_rdd = to_sample_rdd(*validation_data) + elif isinstance(x, RDD): + input = x + validation_data_rdd = validation_data + return self.__create_distributed_optimizer(training_rdd=input, + batch_size=batch_size, + nb_epoch=nb_epoch, + validation_data=validation_data_rdd) + else: + if isinstance(x, np.ndarray): + return self.__create_local_optimizer(x, y, + batch_size=batch_size, + nb_epoch=nb_epoch, + validation_data=validation_data) + raise Exception("not supported type: %s" % x) + + def __create_local_optimizer(self, x, y, batch_size=32, nb_epoch=10, validation_data=None): + if validation_data: + raise unsupport_exp("validation_data") + bopt = boptimizer.LocalOptimizer( + X=x, + y=y, + model=self.bmodel, + criterion=self.criterion, + end_trigger=boptimizer.MaxEpoch(nb_epoch), + batch_size=batch_size, + optim_method=self.optim_method, + cores=None + ) + # TODO: enable validation for local optimizer. + return bopt + + def __create_distributed_optimizer(self, training_rdd, + batch_size=32, + nb_epoch=10, + validation_data=None): + sc = get_spark_context() + bopt = boptimizer.Optimizer( + model=self.bmodel, + training_rdd=training_rdd, + criterion=self.criterion, + end_trigger=boptimizer.MaxEpoch(nb_epoch), + batch_size=batch_size, + optim_method=self.optim_method + ) + if validation_data: + bopt.set_validation(batch_size, + val_rdd=validation_data, + # TODO: check if keras use the same strategy + trigger=boptimizer.EveryEpoch(), + val_method=self.metrics) + return bopt + + +def with_bigdl_backend(kmodel): + bcommon.init_engine() + return KerasModelWrapper(kmodel) + diff --git a/python/dllib/src/bigdl/dllib/bigdlkeras/optimization.py b/python/dllib/src/bigdl/dllib/bigdlkeras/optimization.py new file mode 100644 index 00000000000..362603a13d3 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/bigdlkeras/optimization.py @@ -0,0 +1,63 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import keras.optimizers as koptimizers + +import bigdl.nn.criterion as bcriterion +import bigdl.optim.optimizer as boptimizer +import bigdl.util.common as bcommon +from bigdl.keras.converter import * + + +class OptimConverter: + + @staticmethod + def to_bigdl_metrics(metrics): + metrics = bcommon.to_list(metrics) + bmetrics = [] + for metric in metrics: + if metric == "accuracy": + bmetrics.append(boptimizer.Top1Accuracy()) + else: + unsupport_exp(metric) + # TODO: add more metrics + return bmetrics + + @staticmethod + def to_bigdl_criterion(kloss): + # TODO: it may pass in an object and with parameters + if kloss == "categorical_crossentropy": + return bcriterion.ClassNLLCriterion() + elif kloss == "mse" or kloss == "mean_squared_error": + return bcriterion.MSECriterion() + elif kloss == "binary_crossentropy": + return bcriterion.BCECriterion() + elif kloss == "mae" or kloss == "mean_absolute_error": + return bcriterion.AbsCriterion() + else: + raise Exception("Not supported type: %s" % kloss) + + @staticmethod + def to_bigdl_optim_method(koptim_method): + # This is always be an object + if isinstance(koptim_method, koptimizers.Adagrad): + return boptimizer.Adagrad() + elif isinstance(koptim_method, koptimizers.SGD): + return boptimizer.SGD(learningrate=0.01) # TODO: enrich parameters. ie: lr + elif isinstance(koptim_method, koptimizers.Adam): + return boptimizer.Adam() + else: + unsupport_exp(koptim_method) \ No newline at end of file diff --git a/python/dllib/src/bigdl/dllib/keras/converter.py b/python/dllib/src/bigdl/dllib/keras/converter.py index 71a72941413..9a52680c43f 100644 --- a/python/dllib/src/bigdl/dllib/keras/converter.py +++ b/python/dllib/src/bigdl/dllib/keras/converter.py @@ -1,3 +1,18 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# import bigdl.nn.initialization_method as BInit import numpy as np diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 8af6f320bab..3ccd7f2101f 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -270,7 +270,7 @@ def evaluate(self, *args): :param val_rdd: the input data :param batch_size: batch size :param val_methods: a list of validation methods. i.e: Top1Accuracy,Top5Accuracy and Loss. - :return: + :return: a list of the metrics result """ if len(args) == 0: callBigDlFunc(self.bigdl_type, diff --git a/python/dllib/src/bigdl/dllib/utils/common.py b/python/dllib/src/bigdl/dllib/utils/common.py index 8c4378468e5..70331c9ef67 100644 --- a/python/dllib/src/bigdl/dllib/utils/common.py +++ b/python/dllib/src/bigdl/dllib/utils/common.py @@ -435,6 +435,21 @@ def to_list(a): return [a] +def to_sample_rdd(x, y, numSlices=None): + """ + Conver x and y into RDD[Sample] + :param x: ndarray and the first dimension should be batch + :param y: ndarray and the first dimension should be batch + :param numSlices: + :return: + """ + sc = get_spark_context() + from bigdl.util.common import Sample + x_rdd = sc.parallelize(x, numSlices) + y_rdd = sc.parallelize(y, numSlices) + return x_rdd.zip(y_rdd).map(lambda item: Sample.from_ndarray(item[0], item[1])) + + def extend_spark_driver_cp(sparkConf, path): original_driver_classpath = ":" + sparkConf.get("spark.driver.extraClassPath") \ if sparkConf.contains("spark.driver.extraClassPath") else "" From 5adb944b41475789d0c998bad31f36f1212e9939 Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Mon, 11 Dec 2017 10:17:27 +0800 Subject: [PATCH 415/823] A keras backend wrapper (#1945) * good * backend * add unitest * update doc * style * update test * fix test * test --- .../src/bigdl/dllib/bigdlkeras/backend.py | 181 ++++++++++++++++++ .../bigdl/dllib/bigdlkeras/optimization.py | 63 ++++++ .../dllib/src/bigdl/dllib/keras/converter.py | 15 ++ python/dllib/src/bigdl/dllib/nn/layer.py | 2 +- python/dllib/src/bigdl/utils/common.py | 15 ++ 5 files changed, 275 insertions(+), 1 deletion(-) create mode 100644 python/dllib/src/bigdl/dllib/bigdlkeras/backend.py create mode 100644 python/dllib/src/bigdl/dllib/bigdlkeras/optimization.py diff --git a/python/dllib/src/bigdl/dllib/bigdlkeras/backend.py b/python/dllib/src/bigdl/dllib/bigdlkeras/backend.py new file mode 100644 index 00000000000..02a4abf771d --- /dev/null +++ b/python/dllib/src/bigdl/dllib/bigdlkeras/backend.py @@ -0,0 +1,181 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import numpy as np +from pyspark.rdd import RDD + +from bigdl.keras.optimization import * +from bigdl.util.common import get_spark_context +from bigdl.util.common import to_sample_rdd + + +class KerasModelWrapper(): + + def __init__(self, kmodel): + self.bmodel = DefinitionLoader.from_kmodel(kmodel) + WeightLoader.load_weights_from_kmodel(self.bmodel, kmodel) # share the same weight. + self.criterion = OptimConverter.to_bigdl_criterion(kmodel.loss) + self.optim_method = OptimConverter.to_bigdl_optim_method(kmodel.optimizer) + self.metrics = OptimConverter.to_bigdl_metrics(kmodel.metrics) if kmodel.metrics else None + + def evaluate(self, x, y, batch_size=32, sample_weight=None, is_distributed=False): + """ + Evaluate a model by the given metrics. + :param x: ndarray or list of ndarray for local mode. + RDD[Sample] for distributed mode + :param y: ndarray or list of ndarray for local mode and would be None for cluster mode. + :param batch_size + :param is_distributed: run in local mode or distributed mode. + NB: if is_distributed=true, x should be RDD[Sample] and y should be None + :return: + """ + if sample_weight: + unsupport_exp("sample_weight") + if is_distributed: + if isinstance(x, np.ndarray): + input = to_sample_rdd(x, y) + elif isinstance(x, RDD): + input = x + if self.metrics: + sc = get_spark_context() + return [r.result for r in + self.bmodel.evaluate(input, batch_size, self.metrics)] + else: + raise Exception("No Metrics found.") + + raise Exception("not supported operation: %s", is_distributed) + + def predict(self, x, batch_size=None, verbose=None, is_distributed=False): + """Generates output predictions for the input samples, + processing the samples in a batched way. + + # Arguments + x: the input data, as a Numpy array or list of Numpy array for local mode. + as RDD[Sample] for distributed mode + is_distributed: used to control run in local or cluster. the default value is False + # Returns + A Numpy array or RDD[Sample] of predictions. + """ + if batch_size or verbose: + raise Exception("we don't support batch_size or verbose for now") + if is_distributed: + if isinstance(x, np.ndarray): + input = to_sample_rdd(x, np.zeros([x.shape[0]])) + # np.asarray(self.bmodel.predict(x_rdd).collect()) + elif isinstance(x, RDD): + input = x + return self.bmodel.predict(input) + else: + if isinstance(x, np.ndarray): + return self.bmodel.predict_local(x) + raise Exception("not supported type: %s" % x) + + def fit(self, x, y=None, batch_size=32, nb_epoch=10, verbose=1, callbacks=None, + validation_split=0., validation_data=None, shuffle=True, + class_weight=None, sample_weight=None, initial_epoch=0, is_distributed=False): + """Optimize the model by the given options + + :param x: ndarray or list of ndarray for local mode. + RDD[Sample] for distributed mode + :param y: ndarray or list of ndarray for local mode and would be None for cluster mode. + is_distributed: used to control run in local or cluster. the default value is False. + NB: if is_distributed=true, x should be RDD[Sample] and y should be None + :return: + A Numpy array or RDD[Sample] of predictions. + """ + if callbacks: + raise Exception("We don't support callbacks in fit for now") + if class_weight: + unsupport_exp("class_weight") + if sample_weight: + unsupport_exp("sample_weight") + if initial_epoch != 0: + unsupport_exp("initial_epoch") + if shuffle != True: + unsupport_exp("shuffle") + if validation_split != 0.: + unsupport_exp("validation_split") + bopt = self.__create_optimizer(x=x, + y=y, + batch_size=batch_size, + nb_epoch=nb_epoch, + validation_data=validation_data, + is_distributed=is_distributed) + bopt.optimize() + + def __create_optimizer(self, x=None, y=None, batch_size=32, nb_epoch=10, + validation_data=None, is_distributed=False): + if is_distributed: + if isinstance(x, np.ndarray): + input = to_sample_rdd(x, y) + validation_data_rdd = to_sample_rdd(*validation_data) + elif isinstance(x, RDD): + input = x + validation_data_rdd = validation_data + return self.__create_distributed_optimizer(training_rdd=input, + batch_size=batch_size, + nb_epoch=nb_epoch, + validation_data=validation_data_rdd) + else: + if isinstance(x, np.ndarray): + return self.__create_local_optimizer(x, y, + batch_size=batch_size, + nb_epoch=nb_epoch, + validation_data=validation_data) + raise Exception("not supported type: %s" % x) + + def __create_local_optimizer(self, x, y, batch_size=32, nb_epoch=10, validation_data=None): + if validation_data: + raise unsupport_exp("validation_data") + bopt = boptimizer.LocalOptimizer( + X=x, + y=y, + model=self.bmodel, + criterion=self.criterion, + end_trigger=boptimizer.MaxEpoch(nb_epoch), + batch_size=batch_size, + optim_method=self.optim_method, + cores=None + ) + # TODO: enable validation for local optimizer. + return bopt + + def __create_distributed_optimizer(self, training_rdd, + batch_size=32, + nb_epoch=10, + validation_data=None): + sc = get_spark_context() + bopt = boptimizer.Optimizer( + model=self.bmodel, + training_rdd=training_rdd, + criterion=self.criterion, + end_trigger=boptimizer.MaxEpoch(nb_epoch), + batch_size=batch_size, + optim_method=self.optim_method + ) + if validation_data: + bopt.set_validation(batch_size, + val_rdd=validation_data, + # TODO: check if keras use the same strategy + trigger=boptimizer.EveryEpoch(), + val_method=self.metrics) + return bopt + + +def with_bigdl_backend(kmodel): + bcommon.init_engine() + return KerasModelWrapper(kmodel) + diff --git a/python/dllib/src/bigdl/dllib/bigdlkeras/optimization.py b/python/dllib/src/bigdl/dllib/bigdlkeras/optimization.py new file mode 100644 index 00000000000..362603a13d3 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/bigdlkeras/optimization.py @@ -0,0 +1,63 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import keras.optimizers as koptimizers + +import bigdl.nn.criterion as bcriterion +import bigdl.optim.optimizer as boptimizer +import bigdl.util.common as bcommon +from bigdl.keras.converter import * + + +class OptimConverter: + + @staticmethod + def to_bigdl_metrics(metrics): + metrics = bcommon.to_list(metrics) + bmetrics = [] + for metric in metrics: + if metric == "accuracy": + bmetrics.append(boptimizer.Top1Accuracy()) + else: + unsupport_exp(metric) + # TODO: add more metrics + return bmetrics + + @staticmethod + def to_bigdl_criterion(kloss): + # TODO: it may pass in an object and with parameters + if kloss == "categorical_crossentropy": + return bcriterion.ClassNLLCriterion() + elif kloss == "mse" or kloss == "mean_squared_error": + return bcriterion.MSECriterion() + elif kloss == "binary_crossentropy": + return bcriterion.BCECriterion() + elif kloss == "mae" or kloss == "mean_absolute_error": + return bcriterion.AbsCriterion() + else: + raise Exception("Not supported type: %s" % kloss) + + @staticmethod + def to_bigdl_optim_method(koptim_method): + # This is always be an object + if isinstance(koptim_method, koptimizers.Adagrad): + return boptimizer.Adagrad() + elif isinstance(koptim_method, koptimizers.SGD): + return boptimizer.SGD(learningrate=0.01) # TODO: enrich parameters. ie: lr + elif isinstance(koptim_method, koptimizers.Adam): + return boptimizer.Adam() + else: + unsupport_exp(koptim_method) \ No newline at end of file diff --git a/python/dllib/src/bigdl/dllib/keras/converter.py b/python/dllib/src/bigdl/dllib/keras/converter.py index 71a72941413..9a52680c43f 100644 --- a/python/dllib/src/bigdl/dllib/keras/converter.py +++ b/python/dllib/src/bigdl/dllib/keras/converter.py @@ -1,3 +1,18 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# import bigdl.nn.initialization_method as BInit import numpy as np diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 8af6f320bab..3ccd7f2101f 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -270,7 +270,7 @@ def evaluate(self, *args): :param val_rdd: the input data :param batch_size: batch size :param val_methods: a list of validation methods. i.e: Top1Accuracy,Top5Accuracy and Loss. - :return: + :return: a list of the metrics result """ if len(args) == 0: callBigDlFunc(self.bigdl_type, diff --git a/python/dllib/src/bigdl/utils/common.py b/python/dllib/src/bigdl/utils/common.py index 8c4378468e5..70331c9ef67 100644 --- a/python/dllib/src/bigdl/utils/common.py +++ b/python/dllib/src/bigdl/utils/common.py @@ -435,6 +435,21 @@ def to_list(a): return [a] +def to_sample_rdd(x, y, numSlices=None): + """ + Conver x and y into RDD[Sample] + :param x: ndarray and the first dimension should be batch + :param y: ndarray and the first dimension should be batch + :param numSlices: + :return: + """ + sc = get_spark_context() + from bigdl.util.common import Sample + x_rdd = sc.parallelize(x, numSlices) + y_rdd = sc.parallelize(y, numSlices) + return x_rdd.zip(y_rdd).map(lambda item: Sample.from_ndarray(item[0], item[1])) + + def extend_spark_driver_cp(sparkConf, path): original_driver_classpath = ":" + sparkConf.get("spark.driver.extraClassPath") \ if sparkConf.contains("spark.driver.extraClassPath") else "" From dd1b9dbea9e4884c039a86f2d877c7379a0b6fff Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Mon, 11 Dec 2017 10:17:27 +0800 Subject: [PATCH 416/823] A keras backend wrapper (#1945) * good * backend * add unitest * update doc * style * update test * fix test * test --- python/dllib/test/bigdl/keras/test_backend.py | 177 ++++++++++++++++++ python/dllib/test/bigdl/test_utils.py | 17 +- python/dllib/test/dev/run-keras.sh | 4 +- 3 files changed, 195 insertions(+), 3 deletions(-) create mode 100644 python/dllib/test/bigdl/keras/test_backend.py diff --git a/python/dllib/test/bigdl/keras/test_backend.py b/python/dllib/test/bigdl/keras/test_backend.py new file mode 100644 index 00000000000..40e6ce9c5fb --- /dev/null +++ b/python/dllib/test/bigdl/keras/test_backend.py @@ -0,0 +1,177 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from __future__ import print_function + +from keras.layers import * + +np.random.seed(1337) # for reproducibility +import pytest +from keras.applications import * +from bigdl.keras.converter import * +from bigdl.keras.backend import * +from test.bigdl.test_utils import BigDLTestCase, TestModels + + +class TestBackend(BigDLTestCase): + + def assert_model(self, input_data, kmodel, rtol=1e-5, atol=1e-5): + bmodel = DefinitionLoader.from_kmodel(kmodel) + WeightLoader.load_weights_from_kmodel(bmodel, kmodel) + + keras_output = kmodel.predict(input_data) + bmodel.training(is_training=False) + bigdl_output = bmodel.forward(input_data) + + self.assert_allclose(keras_output, bigdl_output, rtol=rtol, atol=atol) + + def test_lenet_local(self): + kmodel, X_train, y_train = TestModels.kmodel_seq_lenet_mnist() + self.modelTest(X_train, kmodel, dump_weights=True) + kmodel.compile(loss='categorical_crossentropy', + optimizer='adam', + metrics=['accuracy']) + model = with_bigdl_backend(kmodel) + + model.fit(X_train, y_train, + batch_size=4, + nb_epoch=2) + model.predict(X_train) + # TODO: support evaluate for local mode. + # model.evaluate(X_train, y_train) + print(model) + + def test_lenet_distributed_ndarray(self): + kmodel, X_train, y_train = TestModels.kmodel_seq_lenet_mnist() + self.modelTest(X_train, kmodel, dump_weights=True) + kmodel.compile(loss='categorical_crossentropy', + optimizer='adam', + metrics=['accuracy']) + model = with_bigdl_backend(kmodel) + + model.fit(X_train, y_train, + batch_size=4, + nb_epoch=2, + validation_data=(X_train, y_train), is_distributed=True) + model.predict(X_train, is_distributed=True).collect() + model.evaluate(X_train, y_train, is_distributed=True) + print(model) + + def test_lenet_distributed_rdd(self): + kmodel, X_train, y_train = TestModels.kmodel_seq_lenet_mnist() + sc = get_spark_context() + from bigdl.util.common import Sample + from bigdl.util.common import to_sample_rdd + training_rdd = to_sample_rdd(X_train, y_train) + + self.modelTest(X_train, kmodel, dump_weights=True) + kmodel.compile(loss='categorical_crossentropy', + optimizer='adam', + metrics=['accuracy']) + model = with_bigdl_backend(kmodel) + + model.fit(training_rdd, + batch_size=4, + nb_epoch=2, + validation_data=training_rdd, is_distributed=True) + model.predict(X_train, is_distributed=True).collect() + model.evaluate(X_train, y_train, is_distributed=True) + print(model) + + @pytest.mark.skip(reason="need to support evaluate locally before running the test") + def test_text_classification(self): + '''This example demonstrates the use of Convolution1D for text classification. + This example is from Keras + ''' + + # TODO: support backend support + + import numpy as np + np.random.seed(1337) # for reproducibility + + from keras.preprocessing import sequence + from keras.models import Sequential + from keras.layers import Dense, Dropout, Activation + from keras.layers import Embedding + from keras.layers import Convolution1D + from keras.datasets import imdb + + # set parameters: + max_features = 5000 + maxlen = 400 + batch_size = 32 + embedding_dims = 50 + nb_filter = 250 + filter_length = 3 + hidden_dims = 250 + nb_epoch = 2 + + print('Loading data...') + (X_train, y_train), (X_test, y_test) = imdb.load_data(nb_words=max_features) + print(len(X_train), 'train sequences') + print(len(X_test), 'test sequences') + + print('Pad sequences (samples x time)') + X_train = sequence.pad_sequences(X_train, maxlen=maxlen) + X_test = sequence.pad_sequences(X_test, maxlen=maxlen) + print('X_train shape:', X_train.shape) + print('X_test shape:', X_test.shape) + + print('Build model...') + model = Sequential() + + model.add(Embedding(max_features, + embedding_dims, + input_length=maxlen)) # Exception if specify Dropout dropout=0.2 + + # we add a Convolution1D, which will learn nb_filter + # word group filters of size filter_length: + model.add(Convolution1D(nb_filter=nb_filter, + filter_length=filter_length, + border_mode='valid', + activation='relu', + subsample_length=1)) + # we use max pooling: + model.add(GlobalMaxPooling1D()) + # model.add(GlobalAveragePooling1D()) + + # We add a vanilla hidden layer: + model.add(Dense(hidden_dims)) + model.add(Dropout(0.2)) + model.add(Activation('relu')) + + # We project onto a single unit output layer, and squash it with a sigmoid: + model.add(Dense(1)) + model.add(Activation('sigmoid')) + + model.compile(loss='binary_crossentropy', + optimizer='adam', + metrics=['accuracy']) + model = with_bigdl_backend(model) + + model.fit(X_train, y_train, + batch_size=batch_size, + nb_epoch=nb_epoch, + validation_data=(X_test, y_test)) + # 2017-09-22 15:53:45 INFO DistriOptimizer$:657 + # - Top1Accuracy is Accuracy(correct: 21557, count: 25000, accuracy: 0.86228) + # this result is from GlobalAveragePooling not GlobalMaxPooling. + model.predict(X_test) + model.evaluate(X_test, y_test) + print(model) + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/python/dllib/test/bigdl/test_utils.py b/python/dllib/test/bigdl/test_utils.py index c808ab64b72..384c9e8ba43 100644 --- a/python/dllib/test/bigdl/test_utils.py +++ b/python/dllib/test/bigdl/test_utils.py @@ -21,7 +21,7 @@ from keras.layers.convolutional import * from keras.layers import Dense, Dropout, Input from keras.optimizers import RMSprop -from bigdl.util.common import create_tmp_path +from bigdl.util.common import * from bigdl.keras.converter import * from bigdl.keras.converter import WeightLoader, WeightsConverter import numpy as np @@ -106,6 +106,21 @@ def kmodel_seq_lenet_mnist(): class BigDLTestCase(TestCase): + def setup_method(self, method): + """ setup any state tied to the execution of the given method in a + class. setup_method is invoked for every test method of a class. + """ + sparkConf = create_spark_conf().setMaster("local[4]").setAppName("test model") + self.sc = get_spark_context(sparkConf) + self.sqlContext = SQLContext(self.sc) + init_engine() + + def teardown_method(self, method): + """ teardown any state that was previously setup with a setup_method + call. + """ + self.sc.stop() + def __generate_model(self, input_data, output_layer): def without_batch(batch_shape): return batch_shape[1:] diff --git a/python/dllib/test/dev/run-keras.sh b/python/dllib/test/dev/run-keras.sh index 811dce9ba73..87219f64127 100755 --- a/python/dllib/test/dev/run-keras.sh +++ b/python/dllib/test/dev/run-keras.sh @@ -25,14 +25,14 @@ if (( $# < 1)); then fi cd "`dirname $0`" - +p=$1 # executable python export DL_CORE_NUMBER=4 echo "${cyan}Using python version: $p${reset}" export PYTHON_EXECUTABLE=$p export PYSPARK_PYTHON=$p export PYSPARK_DRIVER_PYTHON=$p -$1 -m pytest -v ../../../pyspark/test/bigdl/keras +$p -m pytest -v ../../../pyspark/test/bigdl/keras exit_status=$? if [ $exit_status -ne 0 ]; From a090f9106cb9cc7ea184e3d5542f3f310de62db2 Mon Sep 17 00:00:00 2001 From: zhangxiaoli73 <380761639@qq.com> Date: Mon, 11 Dec 2017 16:49:59 +0800 Subject: [PATCH 417/823] add MeanAbsolutePercentageCriterion & MeanSquaredLogarithmicCriterion (#1916) * add MeanAbsolutePercentageCriterion & MeanSquaredLogarithmicCriterion * add docs * meet pr comments * meet pr comments * exchange keras input target * update readme --- python/dllib/src/bigdl/dllib/nn/criterion.py | 29 ++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/nn/criterion.py b/python/dllib/src/bigdl/dllib/nn/criterion.py index 7dfc41227fd..c8e30c5a08c 100644 --- a/python/dllib/src/bigdl/dllib/nn/criterion.py +++ b/python/dllib/src/bigdl/dllib/nn/criterion.py @@ -803,6 +803,35 @@ def __init__(self, bigdl_type="float"): super(CosineProximityCriterion, self).__init__(None, bigdl_type) +class MeanAbsolutePercentageCriterion(Criterion): + + ''' + This method is same as `mean_absolute_percentage_error` loss in keras. + It caculates diff = K.abs((y - x) / K.clip(K.abs(y), K.epsilon(), Double.MaxValue)) + and return 100 * K.mean(diff) as outpout. Here, the x and y can have or not have a batch. + >>> error = MeanAbsolutePercentageCriterion() + creating: createMeanAbsolutePercentageCriterion + ''' + + def __init__(self, + bigdl_type="float"): + super(MeanAbsolutePercentageCriterion, self).__init__(None, bigdl_type) + +class MeanSquaredLogarithmicCriterion(Criterion): + + ''' + This method is same as `mean_squared_logarithmic_error` loss in keras. + It calculates: first_log = K.log(K.clip(y, K.epsilon(), Double.MaxValue) + 1.) + second_log = K.log(K.clip(x, K.epsilon(), Double.MaxValue) + 1.) + and output K.mean(K.square(first_log - second_log)). Here, the x and y can have or not have a batch. + >>> error = MeanSquaredLogarithmicCriterion() + creating: createMeanSquaredLogarithmicCriterion + ''' + + def __init__(self, + bigdl_type="float"): + super(MeanSquaredLogarithmicCriterion, self).__init__(None, bigdl_type) + def _test(): import doctest from pyspark import SparkContext From ad142d142f46238e08a046dab4df18c0731d1959 Mon Sep 17 00:00:00 2001 From: zhangxiaoli73 <380761639@qq.com> Date: Mon, 11 Dec 2017 16:49:59 +0800 Subject: [PATCH 418/823] add MeanAbsolutePercentageCriterion & MeanSquaredLogarithmicCriterion (#1916) * add MeanAbsolutePercentageCriterion & MeanSquaredLogarithmicCriterion * add docs * meet pr comments * meet pr comments * exchange keras input target * update readme --- python/dllib/src/bigdl/dllib/nn/criterion.py | 29 ++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/nn/criterion.py b/python/dllib/src/bigdl/dllib/nn/criterion.py index 7dfc41227fd..c8e30c5a08c 100644 --- a/python/dllib/src/bigdl/dllib/nn/criterion.py +++ b/python/dllib/src/bigdl/dllib/nn/criterion.py @@ -803,6 +803,35 @@ def __init__(self, bigdl_type="float"): super(CosineProximityCriterion, self).__init__(None, bigdl_type) +class MeanAbsolutePercentageCriterion(Criterion): + + ''' + This method is same as `mean_absolute_percentage_error` loss in keras. + It caculates diff = K.abs((y - x) / K.clip(K.abs(y), K.epsilon(), Double.MaxValue)) + and return 100 * K.mean(diff) as outpout. Here, the x and y can have or not have a batch. + >>> error = MeanAbsolutePercentageCriterion() + creating: createMeanAbsolutePercentageCriterion + ''' + + def __init__(self, + bigdl_type="float"): + super(MeanAbsolutePercentageCriterion, self).__init__(None, bigdl_type) + +class MeanSquaredLogarithmicCriterion(Criterion): + + ''' + This method is same as `mean_squared_logarithmic_error` loss in keras. + It calculates: first_log = K.log(K.clip(y, K.epsilon(), Double.MaxValue) + 1.) + second_log = K.log(K.clip(x, K.epsilon(), Double.MaxValue) + 1.) + and output K.mean(K.square(first_log - second_log)). Here, the x and y can have or not have a batch. + >>> error = MeanSquaredLogarithmicCriterion() + creating: createMeanSquaredLogarithmicCriterion + ''' + + def __init__(self, + bigdl_type="float"): + super(MeanSquaredLogarithmicCriterion, self).__init__(None, bigdl_type) + def _test(): import doctest from pyspark import SparkContext From 4177f3585b4bbb105a570025e44eccbbf165b652 Mon Sep 17 00:00:00 2001 From: Yuhao Yang Date: Mon, 11 Dec 2017 18:21:56 -0800 Subject: [PATCH 419/823] [keras][loss] kullback_leibler_divergence and poisson (#1938) * add kdl and poisson * add extra unittest * poisson update * kullback * update kld * kld update * style fix * add transient * add extra ut * init with null * resolve conflict --- python/dllib/src/bigdl/dllib/nn/criterion.py | 30 ++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/nn/criterion.py b/python/dllib/src/bigdl/dllib/nn/criterion.py index c8e30c5a08c..d1ab2669fe5 100644 --- a/python/dllib/src/bigdl/dllib/nn/criterion.py +++ b/python/dllib/src/bigdl/dllib/nn/criterion.py @@ -832,6 +832,36 @@ def __init__(self, bigdl_type="float"): super(MeanSquaredLogarithmicCriterion, self).__init__(None, bigdl_type) +class KullbackLeiblerDivergenceCriterion(Criterion): + + ''' + compute Kullback Leibler DivergenceCriterion error for intput and target + This method is same as `kullback_leibler_divergence` loss in keras. Loss calculated as: + y_true = K.clip(input, K.epsilon(), 1) + y_pred = K.clip(target, K.epsilon(), 1) + and output K.sum(y_true * K.log(y_true / y_pred), axis=-1) + + >>> error = KullbackLeiblerDivergenceCriterion() + creating: createKullbackLeiblerDivergenceCriterion + ''' + + def __init__(self, + bigdl_type="float"): + super(KullbackLeiblerDivergenceCriterion, self).__init__(None, bigdl_type) + +class PoissonCriterion(Criterion): + + ''' + compute Poisson error for input and target, loss calculated as: + mean(input - target * K.log(input + K.epsilon()), axis=-1) + >>> error = PoissonCriterion() + creating: createPoissonCriterion + ''' + + def __init__(self, + bigdl_type="float"): + super(PoissonCriterion, self).__init__(None, bigdl_type) + def _test(): import doctest from pyspark import SparkContext From a8402a8e3f7a21158489265ce8e5f24cff101b21 Mon Sep 17 00:00:00 2001 From: Yuhao Yang Date: Mon, 11 Dec 2017 18:21:56 -0800 Subject: [PATCH 420/823] [keras][loss] kullback_leibler_divergence and poisson (#1938) * add kdl and poisson * add extra unittest * poisson update * kullback * update kld * kld update * style fix * add transient * add extra ut * init with null * resolve conflict --- python/dllib/src/bigdl/dllib/nn/criterion.py | 30 ++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/nn/criterion.py b/python/dllib/src/bigdl/dllib/nn/criterion.py index c8e30c5a08c..d1ab2669fe5 100644 --- a/python/dllib/src/bigdl/dllib/nn/criterion.py +++ b/python/dllib/src/bigdl/dllib/nn/criterion.py @@ -832,6 +832,36 @@ def __init__(self, bigdl_type="float"): super(MeanSquaredLogarithmicCriterion, self).__init__(None, bigdl_type) +class KullbackLeiblerDivergenceCriterion(Criterion): + + ''' + compute Kullback Leibler DivergenceCriterion error for intput and target + This method is same as `kullback_leibler_divergence` loss in keras. Loss calculated as: + y_true = K.clip(input, K.epsilon(), 1) + y_pred = K.clip(target, K.epsilon(), 1) + and output K.sum(y_true * K.log(y_true / y_pred), axis=-1) + + >>> error = KullbackLeiblerDivergenceCriterion() + creating: createKullbackLeiblerDivergenceCriterion + ''' + + def __init__(self, + bigdl_type="float"): + super(KullbackLeiblerDivergenceCriterion, self).__init__(None, bigdl_type) + +class PoissonCriterion(Criterion): + + ''' + compute Poisson error for input and target, loss calculated as: + mean(input - target * K.log(input + K.epsilon()), axis=-1) + >>> error = PoissonCriterion() + creating: createPoissonCriterion + ''' + + def __init__(self, + bigdl_type="float"): + super(PoissonCriterion, self).__init__(None, bigdl_type) + def _test(): import doctest from pyspark import SparkContext From 5a83b3aa1f4585c6a446d837e5ba8f9d0fda5e5d Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Tue, 12 Dec 2017 10:56:56 +0800 Subject: [PATCH 421/823] support hdfs and s3 (#2007) --- .../dllib/src/bigdl/dllib/keras/converter.py | 24 ++++++++++++++----- python/dllib/src/bigdl/dllib/utils/common.py | 19 +++++++++++++++ 2 files changed, 37 insertions(+), 6 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/keras/converter.py b/python/dllib/src/bigdl/dllib/keras/converter.py index 9a52680c43f..3c4083870e1 100644 --- a/python/dllib/src/bigdl/dllib/keras/converter.py +++ b/python/dllib/src/bigdl/dllib/keras/converter.py @@ -20,6 +20,7 @@ from bigdl.optim.optimizer import L1L2Regularizer as BRegularizer import bigdl.optim.optimizer as boptimizer import bigdl.nn.criterion as bcriterion +import bigdl.util.common as BCommon from bigdl.util.common import get_activation_by_name import keras.optimizers as koptimizers from keras.models import model_from_json @@ -55,16 +56,22 @@ def load_weights_from_kmodel(bmodel, kmodel): @staticmethod def load_weights_from_json_hdf5(def_json, weights_hdf5, by_name=False): - with open(def_json, "r") as jp: - kmodel = model_from_json(jp.read()) + """ + The file path can be stored in a local file system, HDFS, S3, + or any Hadoop-supported file system. + """ bmodel = DefinitionLoader.from_json_path(def_json) + def_value = BCommon.text_from_path(def_json) + kmodel = model_from_json(def_value) WeightLoader.load_weights_from_hdf5(bmodel, kmodel, weights_hdf5, by_name) return bmodel + @staticmethod def load_weights_from_hdf5(bmodel, kmodel, filepath, by_name=False): '''Loads all layer weights from a HDF5 save file. - + filepath can be stored in a local file system, HDFS, S3, + or any Hadoop-supported file system. If `by_name` is False (default) weights are loaded based on the network's execution order topology, meaning layers in the execution seq should be exactly the same @@ -75,7 +82,8 @@ def load_weights_from_hdf5(bmodel, kmodel, filepath, by_name=False): for fine-tuning or transfer-learning models where some of the layers have changed. ''' - kmodel.load_weights(filepath=filepath, by_name=by_name) + local_file_path = BCommon.get_local_file(filepath) + kmodel.load_weights(filepath=local_file_path, by_name=by_name) WeightLoader.load_weights_from_kmodel(bmodel, kmodel) @staticmethod @@ -318,8 +326,12 @@ def from_kmodel(cls, kmodel): @classmethod def from_json_path(cls, json_path): - with open(json_path, "r") as jp: - return DefinitionLoader.from_json_str(jp.read()) + """ + :param json_path: definition path which can be stored in a local file system, HDFS, S3, or any Hadoop-supported file system. + :return: BigDL Model + """ + json_str = BCommon.text_from_path(json_path) + return DefinitionLoader.from_json_str(json_str) @classmethod def from_json_str(cls, json_str): diff --git a/python/dllib/src/bigdl/dllib/utils/common.py b/python/dllib/src/bigdl/dllib/utils/common.py index 70331c9ef67..a8cd977c89f 100644 --- a/python/dllib/src/bigdl/dllib/utils/common.py +++ b/python/dllib/src/bigdl/dllib/utils/common.py @@ -589,6 +589,25 @@ def create_tmp_path(): return tmp_file.name +def text_from_path(path): + sc = get_spark_context() + return sc.textFile(path).collect()[0] + + +def get_local_file(a_path): + if not is_distributed(a_path): + return a_path + path, data = get_spark_context().binaryFiles(a_path).collect()[0] + local_file_path = create_tmp_path() + with open(local_file_path, 'w') as local_file: + local_file.write(data) + return local_file_path + + +def is_distributed(path): + return "://" in path + + def get_activation_by_name(activation_name, activation_id=None): """ Convert to a bigdl activation layer given the name of the activation as a string """ From 5f56e834bc71c4589555634e0acab536948254db Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Tue, 12 Dec 2017 10:56:56 +0800 Subject: [PATCH 422/823] support hdfs and s3 (#2007) --- .../dllib/src/bigdl/dllib/keras/converter.py | 24 ++++++++++++++----- python/dllib/src/bigdl/utils/common.py | 19 +++++++++++++++ 2 files changed, 37 insertions(+), 6 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/keras/converter.py b/python/dllib/src/bigdl/dllib/keras/converter.py index 9a52680c43f..3c4083870e1 100644 --- a/python/dllib/src/bigdl/dllib/keras/converter.py +++ b/python/dllib/src/bigdl/dllib/keras/converter.py @@ -20,6 +20,7 @@ from bigdl.optim.optimizer import L1L2Regularizer as BRegularizer import bigdl.optim.optimizer as boptimizer import bigdl.nn.criterion as bcriterion +import bigdl.util.common as BCommon from bigdl.util.common import get_activation_by_name import keras.optimizers as koptimizers from keras.models import model_from_json @@ -55,16 +56,22 @@ def load_weights_from_kmodel(bmodel, kmodel): @staticmethod def load_weights_from_json_hdf5(def_json, weights_hdf5, by_name=False): - with open(def_json, "r") as jp: - kmodel = model_from_json(jp.read()) + """ + The file path can be stored in a local file system, HDFS, S3, + or any Hadoop-supported file system. + """ bmodel = DefinitionLoader.from_json_path(def_json) + def_value = BCommon.text_from_path(def_json) + kmodel = model_from_json(def_value) WeightLoader.load_weights_from_hdf5(bmodel, kmodel, weights_hdf5, by_name) return bmodel + @staticmethod def load_weights_from_hdf5(bmodel, kmodel, filepath, by_name=False): '''Loads all layer weights from a HDF5 save file. - + filepath can be stored in a local file system, HDFS, S3, + or any Hadoop-supported file system. If `by_name` is False (default) weights are loaded based on the network's execution order topology, meaning layers in the execution seq should be exactly the same @@ -75,7 +82,8 @@ def load_weights_from_hdf5(bmodel, kmodel, filepath, by_name=False): for fine-tuning or transfer-learning models where some of the layers have changed. ''' - kmodel.load_weights(filepath=filepath, by_name=by_name) + local_file_path = BCommon.get_local_file(filepath) + kmodel.load_weights(filepath=local_file_path, by_name=by_name) WeightLoader.load_weights_from_kmodel(bmodel, kmodel) @staticmethod @@ -318,8 +326,12 @@ def from_kmodel(cls, kmodel): @classmethod def from_json_path(cls, json_path): - with open(json_path, "r") as jp: - return DefinitionLoader.from_json_str(jp.read()) + """ + :param json_path: definition path which can be stored in a local file system, HDFS, S3, or any Hadoop-supported file system. + :return: BigDL Model + """ + json_str = BCommon.text_from_path(json_path) + return DefinitionLoader.from_json_str(json_str) @classmethod def from_json_str(cls, json_str): diff --git a/python/dllib/src/bigdl/utils/common.py b/python/dllib/src/bigdl/utils/common.py index 70331c9ef67..a8cd977c89f 100644 --- a/python/dllib/src/bigdl/utils/common.py +++ b/python/dllib/src/bigdl/utils/common.py @@ -589,6 +589,25 @@ def create_tmp_path(): return tmp_file.name +def text_from_path(path): + sc = get_spark_context() + return sc.textFile(path).collect()[0] + + +def get_local_file(a_path): + if not is_distributed(a_path): + return a_path + path, data = get_spark_context().binaryFiles(a_path).collect()[0] + local_file_path = create_tmp_path() + with open(local_file_path, 'w') as local_file: + local_file.write(data) + return local_file_path + + +def is_distributed(path): + return "://" in path + + def get_activation_by_name(activation_name, activation_id=None): """ Convert to a bigdl activation layer given the name of the activation as a string """ From 293b64516e3e20d1d7b5117dab0532b6e739e0a6 Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Tue, 12 Dec 2017 10:56:56 +0800 Subject: [PATCH 423/823] support hdfs and s3 (#2007) --- python/dllib/test/bigdl/keras/test_load_model.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/python/dllib/test/bigdl/keras/test_load_model.py b/python/dllib/test/bigdl/keras/test_load_model.py index 6b9d72d0c61..8a8156993ea 100644 --- a/python/dllib/test/bigdl/keras/test_load_model.py +++ b/python/dllib/test/bigdl/keras/test_load_model.py @@ -64,5 +64,21 @@ def test_load_def_weights_kmodel_seq_lenet_mnist(self): kmodel, input_data, output_data = TestModels.kmodel_seq_lenet_mnist() self.__kmodel_load_def_weight_test(kmodel, input_data) + def test_load_definition(self): + kmodel, input_data, output_data = TestModels.kmodel_seq_lenet_mnist() + keras_model_json_path, keras_model_hdf5_path = self._dump_keras(kmodel, dump_weights=True) + bmodel = DefinitionLoader.from_json_path(keras_model_json_path) + WeightLoader.load_weights_from_kmodel(bmodel, kmodel) + self.assert_allclose(bmodel.forward(input_data), kmodel.predict(input_data)) + + def test_load_weights(self): + kmodel, input_data, output_data = TestModels.kmodel_graph_1_layer() + keras_model_json_path, keras_model_hdf5_path = self._dump_keras(kmodel, dump_weights=True) + bmodel = DefinitionLoader.from_json_path(keras_model_json_path) + kmodel.set_weights([kmodel.get_weights()[0] + 100, kmodel.get_weights()[1]]) + WeightLoader.load_weights_from_hdf5(bmodel, kmodel, filepath=keras_model_hdf5_path) + self.assert_allclose(bmodel.forward(input_data), kmodel.predict(input_data)) + + if __name__ == "__main__": pytest.main([__file__]) From a91374c401e9c51972a999844ddacda2c1b57b3a Mon Sep 17 00:00:00 2001 From: Ian Wong Date: Tue, 12 Dec 2017 15:06:53 +0800 Subject: [PATCH 424/823] Support SperableConv2D and one hot crossentropy (#1944) * Support SperableConv2D and one hot crossentropy * fix compile error * add python wrapper * add new ut * add ut * fix unit test * fix ut * refine depthwiseConv2D * fix seperable conv2d unit tests * support DepthWiseConv2dBackpropInput and DepthWiseConv2DBackpropFilter * refine the code * add more test * fix failed test and style issue * fix inconsistant API with keras1 * add serialization unit test * fix failed unit test --- python/dllib/src/bigdl/dllib/nn/criterion.py | 9 +++ python/dllib/src/bigdl/dllib/nn/layer.py | 70 ++++++++++++++++++++ 2 files changed, 79 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/nn/criterion.py b/python/dllib/src/bigdl/dllib/nn/criterion.py index d1ab2669fe5..d48eaf48a39 100644 --- a/python/dllib/src/bigdl/dllib/nn/criterion.py +++ b/python/dllib/src/bigdl/dllib/nn/criterion.py @@ -283,6 +283,15 @@ def __init__(self, super(DistKLDivCriterion, self).__init__(None, bigdl_type, size_average) +class CategoricalCrossEntropy(Criterion): + """ + This criterion is same with cross entropy criterion, except it takes a one-hot format target + tensor + >>> cce = CategoricalCrossEntropy() + creating: createCategoricalCrossEntropy + """ + def __init__(self, bigdl_type="float"): + super(CategoricalCrossEntropy, self).__init__(None, bigdl_type) class HingeEmbeddingCriterion(Criterion): diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 3ccd7f2101f..e59f20a054c 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -3221,6 +3221,76 @@ def __init__(self, upper, inplace) +class SpatialSeperableConvolution(Layer): + + ''' + Separable convolutions consist in first performing a depthwise spatial convolution (which acts + on each input channel separately) followed by a pointwise convolution which mixes together the + resulting output channels. The depth_multiplier argument controls how many output channels are + generated per input channel in the depthwise step. + + :param n_input_channel The number of expected input planes in the image given into forward() + :param n_output_channel The number of output planes the convolution layer will produce. + :param depth_multiplier how many internal channels are generated per input channel + :param kernel_w The kernel width of the convolution + :param kernel_h The kernel height of the convolution + :param stride_w The step of the convolution in the width dimension. + :param stride_h The step of the convolution in the height dimension + :param pad_w The additional zeros added per width to the input planes. + :param pad_h The additional zeros added per height to the input planes. + :param with_bias: the optional initial value for if need bias + :param data_format: a string value of "NHWC" or "NCHW" to specify the input data format of this layer. In "NHWC" format + data is stored in the order of [batch_size, height, width, channels], in "NCHW" format data is stored + in the order of [batch_size, channels, height, width]. + :param wRegularizer: instance of [[Regularizer]](eg. L1 or L2 regularization), applied to the depth weights matrices. + :param bRegularizer: instance of [[Regularizer]]applied to the pointwise bias. + :param pRegularizer: instance of [[Regularizer]]applied to the pointwise weights. + + >>> conv = SpatialSeperableConvolution(6, 12, 1, 5, 5) + creating: createSpatialSeperableConvolution + >>> conv.setWRegularizer(L1Regularizer(0.5)) + creating: createL1Regularizer + >>> conv.setBRegularizer(L1Regularizer(0.5)) + creating: createL1Regularizer + >>> conv = SpatialSeperableConvolution(6, 12, 1, 5, 5, 1, 1, 0, 0, True, "NCHW", L1Regularizer(0.5), L1Regularizer(0.5), L1Regularizer(0.5)) + creating: createL1Regularizer + creating: createL1Regularizer + creating: createL1Regularizer + creating: createSpatialSeperableConvolution + ''' + + def __init__(self, + n_input_channel, + n_output_channel, + depth_multiplier, + kernel_w, + kernel_h, + stride_w=1, + stride_h=1, + pad_w=0, + pad_h=0, + with_bias=True, + data_format="NCHW", + w_regularizer=None, + b_regularizer=None, + p_regularizer=None, + bigdl_type="float"): + super(SpatialSeperableConvolution, self).__init__(None, bigdl_type, + n_input_channel, + n_output_channel, + depth_multiplier, + kernel_w, + kernel_h, + stride_w, + stride_h, + pad_w, + pad_h, + with_bias, + data_format, + w_regularizer, + b_regularizer, + p_regularizer, + ) class ReLU6(Layer): From 66863fb5cf5552c0817b4e65b00133161b6e6fa7 Mon Sep 17 00:00:00 2001 From: Ian Wong Date: Tue, 12 Dec 2017 15:06:53 +0800 Subject: [PATCH 425/823] Support SperableConv2D and one hot crossentropy (#1944) * Support SperableConv2D and one hot crossentropy * fix compile error * add python wrapper * add new ut * add ut * fix unit test * fix ut * refine depthwiseConv2D * fix seperable conv2d unit tests * support DepthWiseConv2dBackpropInput and DepthWiseConv2DBackpropFilter * refine the code * add more test * fix failed test and style issue * fix inconsistant API with keras1 * add serialization unit test * fix failed unit test --- python/dllib/src/bigdl/dllib/nn/criterion.py | 9 +++ python/dllib/src/bigdl/dllib/nn/layer.py | 70 ++++++++++++++++++++ 2 files changed, 79 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/nn/criterion.py b/python/dllib/src/bigdl/dllib/nn/criterion.py index d1ab2669fe5..d48eaf48a39 100644 --- a/python/dllib/src/bigdl/dllib/nn/criterion.py +++ b/python/dllib/src/bigdl/dllib/nn/criterion.py @@ -283,6 +283,15 @@ def __init__(self, super(DistKLDivCriterion, self).__init__(None, bigdl_type, size_average) +class CategoricalCrossEntropy(Criterion): + """ + This criterion is same with cross entropy criterion, except it takes a one-hot format target + tensor + >>> cce = CategoricalCrossEntropy() + creating: createCategoricalCrossEntropy + """ + def __init__(self, bigdl_type="float"): + super(CategoricalCrossEntropy, self).__init__(None, bigdl_type) class HingeEmbeddingCriterion(Criterion): diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 3ccd7f2101f..e59f20a054c 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -3221,6 +3221,76 @@ def __init__(self, upper, inplace) +class SpatialSeperableConvolution(Layer): + + ''' + Separable convolutions consist in first performing a depthwise spatial convolution (which acts + on each input channel separately) followed by a pointwise convolution which mixes together the + resulting output channels. The depth_multiplier argument controls how many output channels are + generated per input channel in the depthwise step. + + :param n_input_channel The number of expected input planes in the image given into forward() + :param n_output_channel The number of output planes the convolution layer will produce. + :param depth_multiplier how many internal channels are generated per input channel + :param kernel_w The kernel width of the convolution + :param kernel_h The kernel height of the convolution + :param stride_w The step of the convolution in the width dimension. + :param stride_h The step of the convolution in the height dimension + :param pad_w The additional zeros added per width to the input planes. + :param pad_h The additional zeros added per height to the input planes. + :param with_bias: the optional initial value for if need bias + :param data_format: a string value of "NHWC" or "NCHW" to specify the input data format of this layer. In "NHWC" format + data is stored in the order of [batch_size, height, width, channels], in "NCHW" format data is stored + in the order of [batch_size, channels, height, width]. + :param wRegularizer: instance of [[Regularizer]](eg. L1 or L2 regularization), applied to the depth weights matrices. + :param bRegularizer: instance of [[Regularizer]]applied to the pointwise bias. + :param pRegularizer: instance of [[Regularizer]]applied to the pointwise weights. + + >>> conv = SpatialSeperableConvolution(6, 12, 1, 5, 5) + creating: createSpatialSeperableConvolution + >>> conv.setWRegularizer(L1Regularizer(0.5)) + creating: createL1Regularizer + >>> conv.setBRegularizer(L1Regularizer(0.5)) + creating: createL1Regularizer + >>> conv = SpatialSeperableConvolution(6, 12, 1, 5, 5, 1, 1, 0, 0, True, "NCHW", L1Regularizer(0.5), L1Regularizer(0.5), L1Regularizer(0.5)) + creating: createL1Regularizer + creating: createL1Regularizer + creating: createL1Regularizer + creating: createSpatialSeperableConvolution + ''' + + def __init__(self, + n_input_channel, + n_output_channel, + depth_multiplier, + kernel_w, + kernel_h, + stride_w=1, + stride_h=1, + pad_w=0, + pad_h=0, + with_bias=True, + data_format="NCHW", + w_regularizer=None, + b_regularizer=None, + p_regularizer=None, + bigdl_type="float"): + super(SpatialSeperableConvolution, self).__init__(None, bigdl_type, + n_input_channel, + n_output_channel, + depth_multiplier, + kernel_w, + kernel_h, + stride_w, + stride_h, + pad_w, + pad_h, + with_bias, + data_format, + w_regularizer, + b_regularizer, + p_regularizer, + ) class ReLU6(Layer): From 40eca056e6aca89ff60016b9b9f213cf2cf719bb Mon Sep 17 00:00:00 2001 From: Xianyan Date: Tue, 12 Dec 2017 16:54:13 +0800 Subject: [PATCH 426/823] Add object detection related layers for model zoo and keras support (#2015) * Add object detection model zoo support * Add unit test * Add more doc and python wrapper * revert some and add createProposal * Fix ut --- python/dllib/src/bigdl/dllib/nn/layer.py | 141 +++++++++++++++++++++++ 1 file changed, 141 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index e59f20a054c..6efbff8d3ba 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -4838,6 +4838,147 @@ class UpSampling3D(Layer): def __init__(self, size, bigdl_type="float"): super(UpSampling3D, self).__init__(None, bigdl_type, size) +class PriorBox(Layer): + """ + Generate the prior boxes of designated sizes and aspect ratios across + all dimensions (H * W) + Intended for use with MultiBox detection method to generate prior + :param min_sizes minimum box size in pixels. can be multiple. required! + :param max_sizes maximum box size in pixels. can be ignored or same as the # of min_size. + :param aspect_ratios optional aspect ratios of the boxes. can be multiple + :param is_flip optional bool, default true. if set, flip the aspect ratio. + :param is_clip whether to clip the prior's coordidate such that it is within [0, 1] + >>> layer = PriorBox([0.1]) + creating: createPriorBox + """ + def __init__(self, min_sizes, + max_sizes=None, + aspect_ratios=None, + is_flip=True, + is_clip=False, + variances=None, + offset = 0.5, + img_h=0, + img_w=0, + img_size=0, + step_h=0.0, + step_w=0.0, + step=0.0, + bigdl_type="float"): + super(PriorBox, self).__init__(None, bigdl_type, + min_sizes, + max_sizes, + aspect_ratios, + is_flip, + is_clip, + variances, + offset, + img_h, + img_w, + img_size, + step_h, + step_w, + step) + +class NormalizeScale(Layer): + """ + NormalizeScale is conposed of normalize and scale, this is equal to caffe Normalize layer + :param p L_p norm + :param eps smoothing parameter + :param scale scale parameter + :param size size of scale input + :param w_regularizer weight regularizer + >>> layer = NormalizeScale(2.0, scale = 20.0, size = [1, 5, 1, 1]) + creating: createNormalizeScale + """ + def __init__(self, p, scale, size, w_regularizer=None, eps=1e-10, + bigdl_type="float"): + super(NormalizeScale, self).__init__(None, bigdl_type, p, eps, scale, size, w_regularizer) + +class Proposal(Layer): + """ + Outputs object detection proposals by applying estimated bounding-box + transformations to a set of regular boxes (called "anchors"). + rois: holds R regions of interest, each is a 5-tuple + (n, x1, y1, x2, y2) specifying an image batch index n and a rectangle (x1, y1, x2, y2) + scores: holds scores for R regions of interest + >>> layer = Proposal(1000, 200, [0.1, 0.2], [2.0, 3.0]) + creating: createProposal + """ + def __init__(self, pre_nms_topn, post_nms_topn, ratios, scales, + rpn_pre_nms_topn_train=12000, rpn_post_nms_topn_train=2000, + bigdl_type="float"): + super(Proposal, self).__init__(None, bigdl_type, + pre_nms_topn, + post_nms_topn, + ratios, + scales, + rpn_pre_nms_topn_train, + rpn_post_nms_topn_train) + +class DetectionOutputSSD(Layer): + """ + Layer to Post-process SSD output + :param n_classes number of classes + :param share_location whether to share location, default is true + :param bg_label background label + :param nms_thresh nms threshold + :param nms_topk nms topk + :param keep_top_k result topk + :param conf_thresh confidence threshold + :param variance_encoded_in_target if variance is encoded in target, + we simply need to retore the offset predictions, + else if variance is encoded in bbox, + we need to scale the offset accordingly. + :param conf_post_process whether add some additional post process to confidence prediction + >>> layer = DetectionOutputSSD() + creating: createDetectionOutputSSD + """ + + def __init__(self, n_classes=21, + share_location=True, + bg_label=0, + nms_thresh=0.45, + nms_topk=400, + keep_top_k=200, + conf_thresh=0.01, + variance_encoded_in_target=False, + conf_post_process=True, + bigdl_type="float"): + super(DetectionOutputSSD, self).__init__(None, + bigdl_type, + n_classes, + share_location, + bg_label, + nms_thresh, + nms_topk, + keep_top_k, + conf_thresh, + variance_encoded_in_target, + conf_post_process) + +class DetectionOutputFrcnn(Layer): + """ + Post process Faster-RCNN models + :param nms_thresh nms threshold + :param n_classes number of classes + :param bbox_vote whether to vote for detections + :param max_per_image limit max number of detections per image + :param thresh score threshold + >>> layer = DetectionOutputFrcnn(21, True) + creating: createDetectionOutputFrcnn + """ + + def __init__(self, n_classes, bbox_vote, nms_thresh = 0.3, + max_per_image=100, thresh=0.05, + bigdl_type="float"): + super(DetectionOutputFrcnn, self).__init__(None, bigdl_type, nms_thresh, + n_classes, + bbox_vote, + max_per_image, + thresh) + + def _test(): import doctest from pyspark import SparkContext From e31fed254b3fa2ab3f80b081600edafe5964479c Mon Sep 17 00:00:00 2001 From: Xianyan Date: Tue, 12 Dec 2017 16:54:13 +0800 Subject: [PATCH 427/823] Add object detection related layers for model zoo and keras support (#2015) * Add object detection model zoo support * Add unit test * Add more doc and python wrapper * revert some and add createProposal * Fix ut --- python/dllib/src/bigdl/dllib/nn/layer.py | 141 +++++++++++++++++++++++ 1 file changed, 141 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index e59f20a054c..6efbff8d3ba 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -4838,6 +4838,147 @@ class UpSampling3D(Layer): def __init__(self, size, bigdl_type="float"): super(UpSampling3D, self).__init__(None, bigdl_type, size) +class PriorBox(Layer): + """ + Generate the prior boxes of designated sizes and aspect ratios across + all dimensions (H * W) + Intended for use with MultiBox detection method to generate prior + :param min_sizes minimum box size in pixels. can be multiple. required! + :param max_sizes maximum box size in pixels. can be ignored or same as the # of min_size. + :param aspect_ratios optional aspect ratios of the boxes. can be multiple + :param is_flip optional bool, default true. if set, flip the aspect ratio. + :param is_clip whether to clip the prior's coordidate such that it is within [0, 1] + >>> layer = PriorBox([0.1]) + creating: createPriorBox + """ + def __init__(self, min_sizes, + max_sizes=None, + aspect_ratios=None, + is_flip=True, + is_clip=False, + variances=None, + offset = 0.5, + img_h=0, + img_w=0, + img_size=0, + step_h=0.0, + step_w=0.0, + step=0.0, + bigdl_type="float"): + super(PriorBox, self).__init__(None, bigdl_type, + min_sizes, + max_sizes, + aspect_ratios, + is_flip, + is_clip, + variances, + offset, + img_h, + img_w, + img_size, + step_h, + step_w, + step) + +class NormalizeScale(Layer): + """ + NormalizeScale is conposed of normalize and scale, this is equal to caffe Normalize layer + :param p L_p norm + :param eps smoothing parameter + :param scale scale parameter + :param size size of scale input + :param w_regularizer weight regularizer + >>> layer = NormalizeScale(2.0, scale = 20.0, size = [1, 5, 1, 1]) + creating: createNormalizeScale + """ + def __init__(self, p, scale, size, w_regularizer=None, eps=1e-10, + bigdl_type="float"): + super(NormalizeScale, self).__init__(None, bigdl_type, p, eps, scale, size, w_regularizer) + +class Proposal(Layer): + """ + Outputs object detection proposals by applying estimated bounding-box + transformations to a set of regular boxes (called "anchors"). + rois: holds R regions of interest, each is a 5-tuple + (n, x1, y1, x2, y2) specifying an image batch index n and a rectangle (x1, y1, x2, y2) + scores: holds scores for R regions of interest + >>> layer = Proposal(1000, 200, [0.1, 0.2], [2.0, 3.0]) + creating: createProposal + """ + def __init__(self, pre_nms_topn, post_nms_topn, ratios, scales, + rpn_pre_nms_topn_train=12000, rpn_post_nms_topn_train=2000, + bigdl_type="float"): + super(Proposal, self).__init__(None, bigdl_type, + pre_nms_topn, + post_nms_topn, + ratios, + scales, + rpn_pre_nms_topn_train, + rpn_post_nms_topn_train) + +class DetectionOutputSSD(Layer): + """ + Layer to Post-process SSD output + :param n_classes number of classes + :param share_location whether to share location, default is true + :param bg_label background label + :param nms_thresh nms threshold + :param nms_topk nms topk + :param keep_top_k result topk + :param conf_thresh confidence threshold + :param variance_encoded_in_target if variance is encoded in target, + we simply need to retore the offset predictions, + else if variance is encoded in bbox, + we need to scale the offset accordingly. + :param conf_post_process whether add some additional post process to confidence prediction + >>> layer = DetectionOutputSSD() + creating: createDetectionOutputSSD + """ + + def __init__(self, n_classes=21, + share_location=True, + bg_label=0, + nms_thresh=0.45, + nms_topk=400, + keep_top_k=200, + conf_thresh=0.01, + variance_encoded_in_target=False, + conf_post_process=True, + bigdl_type="float"): + super(DetectionOutputSSD, self).__init__(None, + bigdl_type, + n_classes, + share_location, + bg_label, + nms_thresh, + nms_topk, + keep_top_k, + conf_thresh, + variance_encoded_in_target, + conf_post_process) + +class DetectionOutputFrcnn(Layer): + """ + Post process Faster-RCNN models + :param nms_thresh nms threshold + :param n_classes number of classes + :param bbox_vote whether to vote for detections + :param max_per_image limit max number of detections per image + :param thresh score threshold + >>> layer = DetectionOutputFrcnn(21, True) + creating: createDetectionOutputFrcnn + """ + + def __init__(self, n_classes, bbox_vote, nms_thresh = 0.3, + max_per_image=100, thresh=0.05, + bigdl_type="float"): + super(DetectionOutputFrcnn, self).__init__(None, bigdl_type, nms_thresh, + n_classes, + bbox_vote, + max_per_image, + thresh) + + def _test(): import doctest from pyspark import SparkContext From 227a05a412e479cec64d02b83ea7d94456b1acdd Mon Sep 17 00:00:00 2001 From: Xianyan Date: Tue, 12 Dec 2017 18:04:07 +0800 Subject: [PATCH 428/823] Add image frame predict (#1972) * Add VisionPredictor * share convolution * inception example can work * move predictImage To Predictor add matToTensor, remove postPorcessor add python Add more doc * meet code review and fix unit test * fix test * remove unnecessary ClassTag in ImageFrame --- .../dllib/feature/transform/vision/image.py | 62 +++++++++++++------ python/dllib/src/bigdl/dllib/nn/layer.py | 21 +++++++ 2 files changed, 64 insertions(+), 19 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py b/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py index 81f76e00c24..86e3b26eec8 100644 --- a/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py +++ b/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py @@ -76,12 +76,6 @@ def __init__(self, image=None, label=None, path=None, bigdl_type="float"): self.value = callBigDlFunc( bigdl_type, JavaValue.jvm_class_constructor(self), image_tensor, label_tensor, path) - def to_sample(self, float_key="floats", to_chw=True, with_im_info=False): - """ - ImageFeature to sample - """ - return callBigDlFunc(self.bigdl_type, "imageFeatureToSample", self.value, float_key, to_chw, with_im_info) - def get_image(self, float_key="floats", to_chw=True): """ get image as ndarray from ImageFeature @@ -170,11 +164,11 @@ def get_label(self): """ return self.image_frame.get_label() - def to_sample(self, float_key="floats", to_chw=True, with_im_info=False): + def get_predict(self, key="predict"): """ - ImageFrame toSample + get prediction from ImageFrame """ - return self.image_frame.to_sample(float_key, to_chw, with_im_info) + return self.image_frame.get_predict(key) class LocalImageFrame(ImageFrame): @@ -209,12 +203,12 @@ def get_label(self): labels = callBigDlFunc(self.bigdl_type, "localImageFrameToLabelTensor", self.value) return map(lambda tensor: tensor.to_ndarray(), labels) - def to_sample(self, float_key="floats", to_chw=True, with_im_info=False): + def get_predict(self, key="predict"): """ - to sample list + get prediction list from ImageFrame """ - return callBigDlFunc(self.bigdl_type, - "localImageFrameToSample", self.value, float_key, to_chw, with_im_info) + predicts = callBigDlFunc(self.bigdl_type, "localImageFrameToPredict", self.value, key) + return map(lambda predict: (predict[0], predict[1].to_ndarray()), predicts) @@ -251,12 +245,13 @@ def get_label(self): tensor_rdd = callBigDlFunc(self.bigdl_type, "distributedImageFrameToLabelTensorRdd", self.value) return tensor_rdd.map(lambda tensor: tensor.to_ndarray()) - def to_sample(self, float_key="floats", to_chw=True, with_im_info=False): + def get_predict(self, key="predict"): """ - to sample rdd + get prediction rdd from ImageFrame """ - return callBigDlFunc(self.bigdl_type, - "distributedImageFrameToSampleRdd", self.value, float_key, to_chw, with_im_info) + predicts = callBigDlFunc(self.bigdl_type, "distributedImageFrameToPredict", self.value, key) + return predicts.map(lambda predict: (predict[0], predict[1].to_ndarray())) + class HFlip(FeatureTransformer): """ @@ -531,11 +526,29 @@ def __init__(self, bigdl_type="float"): super(RoiNormalize, self).__init__(bigdl_type) class MatToFloats(FeatureTransformer): + """ + Transform OpenCVMat to float array, note that in this transformer, the mat is released + :param valid_height valid height in case the mat is invalid + :param valid_width valid width in case the mat is invalid + :param valid_channel valid channel in case the mat is invalid + :param out_key key to store float array + """ def __init__(self, valid_height=300, valid_width=300, valid_channel=300, - mean_r=-1.0, mean_g=-1.0, mean_b=-1.0, out_key = "floats", bigdl_type="float"): + out_key = "floats", bigdl_type="float"): super(MatToFloats, self).__init__(bigdl_type, valid_height, valid_width, valid_channel, - mean_r, mean_g, mean_b, out_key) + out_key) + +class MatToTensor(FeatureTransformer): + """ + transform opencv mat to tensor + :param to_rgb BGR to RGB (default is BGR) + :param tensor_key key to store transformed tensor + """ + + def __init__(self, to_rgb=False, tensor_key="imageTensor", bigdl_type="float"): + super(MatToTensor, self).__init__(bigdl_type, to_rgb, tensor_key) + class AspectScale(FeatureTransformer): """ Resize the image, keep the aspect ratio. scale according to the short edge @@ -564,3 +577,14 @@ class BytesToMat(FeatureTransformer): def __init__(self, bigdl_type="float"): super(BytesToMat, self).__init__(bigdl_type) +class ImageFrameToSample(FeatureTransformer): + """ + transform imageframe to samples + :param input_keys keys that maps inputs (each input should be a tensor) + :param target_keys keys that maps targets (each target should be a tensor) + :param sample_key key to store sample + """ + def __init__(self, input_keys=["imageTensor"], target_keys=None, + sample_key="sample", bigdl_type="float"): + super(ImageFrameToSample, self).__init__(bigdl_type, input_keys, target_keys, sample_key) + diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 6efbff8d3ba..d176d013257 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -30,6 +30,7 @@ from bigdl.util.common import get_activation_by_name from bigdl.optim.optimizer import L1Regularizer, L2Regularizer, L1L2Regularizer from py4j.java_gateway import JavaObject +from bigdl.transform.vision.image import ImageFrame if sys.version >= '3': long = int @@ -346,6 +347,26 @@ def predict_class(self, data_rdd): "modelPredictClass", self.value, data_rdd) return result + def predict_image(self, image_frame, output_layer=None, share_buffer=False, + batch_per_partition=4, predict_key="predict"): + """ + model predict images, return imageFrame with predicted tensor + :param image_frame imageFrame that contains images + :param output_layer if output_layer is not null, the output of layer that matches + output_layer will be used as predicted output + :param share_buffer whether to share same memory for each batch predict results + :param batch_per_partition batch size per partition, default is 4 + :param predict_key key to store predicted results + """ + + image_frame = callBigDlFunc(self.bigdl_type, "modelPredictImage", self.value, + image_frame, + output_layer, + share_buffer, + batch_per_partition, + predict_key) + return ImageFrame(image_frame) + def set_weights(self, weights): """ Set weights for this layer From 22aa9ae2f08a2e86bb7098015b9f9767aff2f08a Mon Sep 17 00:00:00 2001 From: Xianyan Date: Tue, 12 Dec 2017 18:04:07 +0800 Subject: [PATCH 429/823] Add image frame predict (#1972) * Add VisionPredictor * share convolution * inception example can work * move predictImage To Predictor add matToTensor, remove postPorcessor add python Add more doc * meet code review and fix unit test * fix test * remove unnecessary ClassTag in ImageFrame --- .../dllib/feature/transform/vision/image.py | 62 +++++++++++++------ python/dllib/src/bigdl/dllib/nn/layer.py | 21 +++++++ 2 files changed, 64 insertions(+), 19 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py b/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py index 81f76e00c24..86e3b26eec8 100644 --- a/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py +++ b/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py @@ -76,12 +76,6 @@ def __init__(self, image=None, label=None, path=None, bigdl_type="float"): self.value = callBigDlFunc( bigdl_type, JavaValue.jvm_class_constructor(self), image_tensor, label_tensor, path) - def to_sample(self, float_key="floats", to_chw=True, with_im_info=False): - """ - ImageFeature to sample - """ - return callBigDlFunc(self.bigdl_type, "imageFeatureToSample", self.value, float_key, to_chw, with_im_info) - def get_image(self, float_key="floats", to_chw=True): """ get image as ndarray from ImageFeature @@ -170,11 +164,11 @@ def get_label(self): """ return self.image_frame.get_label() - def to_sample(self, float_key="floats", to_chw=True, with_im_info=False): + def get_predict(self, key="predict"): """ - ImageFrame toSample + get prediction from ImageFrame """ - return self.image_frame.to_sample(float_key, to_chw, with_im_info) + return self.image_frame.get_predict(key) class LocalImageFrame(ImageFrame): @@ -209,12 +203,12 @@ def get_label(self): labels = callBigDlFunc(self.bigdl_type, "localImageFrameToLabelTensor", self.value) return map(lambda tensor: tensor.to_ndarray(), labels) - def to_sample(self, float_key="floats", to_chw=True, with_im_info=False): + def get_predict(self, key="predict"): """ - to sample list + get prediction list from ImageFrame """ - return callBigDlFunc(self.bigdl_type, - "localImageFrameToSample", self.value, float_key, to_chw, with_im_info) + predicts = callBigDlFunc(self.bigdl_type, "localImageFrameToPredict", self.value, key) + return map(lambda predict: (predict[0], predict[1].to_ndarray()), predicts) @@ -251,12 +245,13 @@ def get_label(self): tensor_rdd = callBigDlFunc(self.bigdl_type, "distributedImageFrameToLabelTensorRdd", self.value) return tensor_rdd.map(lambda tensor: tensor.to_ndarray()) - def to_sample(self, float_key="floats", to_chw=True, with_im_info=False): + def get_predict(self, key="predict"): """ - to sample rdd + get prediction rdd from ImageFrame """ - return callBigDlFunc(self.bigdl_type, - "distributedImageFrameToSampleRdd", self.value, float_key, to_chw, with_im_info) + predicts = callBigDlFunc(self.bigdl_type, "distributedImageFrameToPredict", self.value, key) + return predicts.map(lambda predict: (predict[0], predict[1].to_ndarray())) + class HFlip(FeatureTransformer): """ @@ -531,11 +526,29 @@ def __init__(self, bigdl_type="float"): super(RoiNormalize, self).__init__(bigdl_type) class MatToFloats(FeatureTransformer): + """ + Transform OpenCVMat to float array, note that in this transformer, the mat is released + :param valid_height valid height in case the mat is invalid + :param valid_width valid width in case the mat is invalid + :param valid_channel valid channel in case the mat is invalid + :param out_key key to store float array + """ def __init__(self, valid_height=300, valid_width=300, valid_channel=300, - mean_r=-1.0, mean_g=-1.0, mean_b=-1.0, out_key = "floats", bigdl_type="float"): + out_key = "floats", bigdl_type="float"): super(MatToFloats, self).__init__(bigdl_type, valid_height, valid_width, valid_channel, - mean_r, mean_g, mean_b, out_key) + out_key) + +class MatToTensor(FeatureTransformer): + """ + transform opencv mat to tensor + :param to_rgb BGR to RGB (default is BGR) + :param tensor_key key to store transformed tensor + """ + + def __init__(self, to_rgb=False, tensor_key="imageTensor", bigdl_type="float"): + super(MatToTensor, self).__init__(bigdl_type, to_rgb, tensor_key) + class AspectScale(FeatureTransformer): """ Resize the image, keep the aspect ratio. scale according to the short edge @@ -564,3 +577,14 @@ class BytesToMat(FeatureTransformer): def __init__(self, bigdl_type="float"): super(BytesToMat, self).__init__(bigdl_type) +class ImageFrameToSample(FeatureTransformer): + """ + transform imageframe to samples + :param input_keys keys that maps inputs (each input should be a tensor) + :param target_keys keys that maps targets (each target should be a tensor) + :param sample_key key to store sample + """ + def __init__(self, input_keys=["imageTensor"], target_keys=None, + sample_key="sample", bigdl_type="float"): + super(ImageFrameToSample, self).__init__(bigdl_type, input_keys, target_keys, sample_key) + diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 6efbff8d3ba..d176d013257 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -30,6 +30,7 @@ from bigdl.util.common import get_activation_by_name from bigdl.optim.optimizer import L1Regularizer, L2Regularizer, L1L2Regularizer from py4j.java_gateway import JavaObject +from bigdl.transform.vision.image import ImageFrame if sys.version >= '3': long = int @@ -346,6 +347,26 @@ def predict_class(self, data_rdd): "modelPredictClass", self.value, data_rdd) return result + def predict_image(self, image_frame, output_layer=None, share_buffer=False, + batch_per_partition=4, predict_key="predict"): + """ + model predict images, return imageFrame with predicted tensor + :param image_frame imageFrame that contains images + :param output_layer if output_layer is not null, the output of layer that matches + output_layer will be used as predicted output + :param share_buffer whether to share same memory for each batch predict results + :param batch_per_partition batch size per partition, default is 4 + :param predict_key key to store predicted results + """ + + image_frame = callBigDlFunc(self.bigdl_type, "modelPredictImage", self.value, + image_frame, + output_layer, + share_buffer, + batch_per_partition, + predict_key) + return ImageFrame(image_frame) + def set_weights(self, weights): """ Set weights for this layer From 2af8a721f122105b2eb9afd041773e4372e76815 Mon Sep 17 00:00:00 2001 From: Xianyan Date: Tue, 12 Dec 2017 18:04:07 +0800 Subject: [PATCH 430/823] Add image frame predict (#1972) * Add VisionPredictor * share convolution * inception example can work * move predictImage To Predictor add matToTensor, remove postPorcessor add python Add more doc * meet code review and fix unit test * fix test * remove unnecessary ClassTag in ImageFrame --- .../test/bigdl/test_simple_integration.py | 18 +++++++++ .../dllib/test/bigdl/transform/test_image.py | 37 ++++++++++++++----- 2 files changed, 46 insertions(+), 9 deletions(-) diff --git a/python/dllib/test/bigdl/test_simple_integration.py b/python/dllib/test/bigdl/test_simple_integration.py index 9c8a3e23378..3dcb1f76d64 100644 --- a/python/dllib/test/bigdl/test_simple_integration.py +++ b/python/dllib/test/bigdl/test_simple_integration.py @@ -27,6 +27,7 @@ import pytest from numpy.testing import assert_allclose, assert_array_equal from bigdl.util.engine import compare_version +from bigdl.transform.vision.image import * np.random.seed(1337) # for reproducibility @@ -476,6 +477,23 @@ def test_predict(self): for i in range(0, total_length): assert predict_labels[i] == 1 + def test_predict_image(self): + resource_path = os.path.join(os.path.split(__file__)[0], "resources") + image_path = os.path.join(resource_path, "pascal/000025.jpg") + image_frame = ImageFrame.read(image_path, self.sc) + transformer = Pipeline([Resize(256, 256), CenterCrop(224, 224), + ChannelNormalize(0.485, 0.456, 0.406, 0.229, 0.224, 0.225), + MatToTensor(), ImageFrameToSample()]) + image_frame.transform(transformer) + + model = Sequential() + model.add(SpatialConvolution(3, 6, 5, 5)) + model.add(Tanh()) + + image_frame = model.predict_image(image_frame) + predicts = image_frame.get_predict() + predicts.collect() + def test_rng(self): rng = RNG() rng.set_seed(100) diff --git a/python/dllib/test/bigdl/transform/test_image.py b/python/dllib/test/bigdl/transform/test_image.py index bb18cc9af4f..595c3b57c93 100644 --- a/python/dllib/test/bigdl/transform/test_image.py +++ b/python/dllib/test/bigdl/transform/test_image.py @@ -41,14 +41,12 @@ def teardown_method(self, method): def transformer_test(self, transformer): image_frame = ImageFrame.read(self.image_path) transformer(image_frame) - image_frame.transform(transformer) - image_frame.to_sample() + image_frame.get_image() image_frame = ImageFrame.read(self.image_path, self.sc) transformer(image_frame) - image_frame.transform(transformer) - sample = image_frame.to_sample() - sample.count() + images = image_frame.get_image() + images.count() def test_get_image(self): image_frame = ImageFrame.read(self.image_path) @@ -58,10 +56,6 @@ def test_get_label(self): image_frame = ImageFrame.read(self.image_path) image_frame.get_label() - def test_to_sample(self): - image_frame = ImageFrame.read(self.image_path) - image_frame.to_sample() - def test_is_local(self): image_frame = ImageFrame.read(self.image_path) assert image_frame.is_local() is True @@ -156,5 +150,30 @@ def test_pipeline(self): transformer = Pipeline([ColorJitter(), HFlip(), Resize(200, 200, 1)]) self.transformer_test(transformer) + def test_inception_preprocess(self): + transformer = Pipeline([Resize(256, 256), CenterCrop(224, 224), + ChannelNormalize(0.485, 0.456, 0.406, 0.229, 0.224, 0.225), + MatToTensor(), ImageFrameToSample()]) + self.transformer_test(transformer) + + def test_mat_to_floats(self): + transformer = MatToFloats() + self.transformer_test(transformer) + + def test_mat_to_tensor(self): + transformer = MatToTensor() + self.transformer_test(transformer) + + def testImageFrameToSample(self): + transformer = Pipeline([MatToTensor(), ImageFrameToSample()]) + self.transformer_test(transformer) + + def test_image_frame_transform(self): + transformer = MatToTensor() + image_frame = ImageFrame.read(self.image_path) + image_frame.transform(transformer) + image_frame.get_image() + + if __name__ == "__main__": pytest.main([__file__]) From e7ef75961dc59e891cf629af76aa66fd3031175a Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Tue, 12 Dec 2017 18:17:15 +0800 Subject: [PATCH 431/823] Support same padding for keras atrousconv1d,2d and conv3d (#1925) * add same padding for conv layers * update --- .../dllib/src/bigdl/dllib/keras/converter.py | 116 +++++++++++++----- 1 file changed, 85 insertions(+), 31 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/keras/converter.py b/python/dllib/src/bigdl/dllib/keras/converter.py index 3c4083870e1..bf0cfb9ff94 100644 --- a/python/dllib/src/bigdl/dllib/keras/converter.py +++ b/python/dllib/src/bigdl/dllib/keras/converter.py @@ -27,6 +27,7 @@ from keras.models import Sequential, Model, Layer import keras import warnings +from math import ceil def unsupport_exp(name): @@ -255,7 +256,7 @@ def convert_maxoutdense(klayer, weights): k_weights = weights[0] b_weights = k_weights[0].T for i in range(1, k_weights.shape[0]): - b_weights = np.concatenate((b_weights, k_weights[i].T)) + b_weights = np.concatenate((b_weights, k_weights[i].T)) return [b_weights, weights[1]] @staticmethod @@ -991,13 +992,28 @@ def to_bigdl_2d_ordering(self, order): def to_bigdl_3d_padding(self, border_mode): if border_mode == "valid": return 0, 0, 0 - # TODO: border_mode=`same` + elif border_mode == "same": + return -1, -1, -1 else: raise Exception("Unsupported border mode: %s" % border_mode) - def to_bigdl_2d_padding(self, border_mode): + def __calculate_2d_same_padding(self, x, kx, dx, dilation_x): + return int(ceil((x * (dx - 1) + dilation_x * (kx - 1) - dx + 1) / 2)) + + def to_bigdl_2d_padding(self, border_mode, *args): if border_mode == "same": - return -1, -1 + if len(args) == 0: # if -1 for same padding is supported + return -1, -1 + # calculate padding by given parameters + elif len(args) == 4: # used by 1d layers constructed from 2d, just need one pad + h, kh, dh, dilation_h = args + pad_h = self.__calculate_2d_same_padding(h, kh, dh, dilation_h) + return pad_h, 0 + elif len(args) == 8: + h, kh, dh, dilation_h, w, kw, dw, dilation_w = args + pad_h = self.__calculate_2d_same_padding(h, kh, dh, dilation_h) + pad_w = self.__calculate_2d_same_padding(w, kw, dw, dilation_w) + return pad_h, pad_w elif border_mode == "valid": return 0, 0 else: @@ -1094,10 +1110,11 @@ def create_atrousconvolution1d(self): if not self.config["bias"]: raise Exception("Please set `bias=True` for AtrousConvolution1D") - # TODO: border_mode=`same` - if self.klayer.border_mode == "same": - raise Exception("Unsupported border mode: %s" % self.klayer.border_mode) - bpadW, bpadH = self.to_bigdl_2d_padding(self.klayer.border_mode) + h = self.input_shape[2] + kh = self.config["filter_length"] + dh = self.config["subsample_length"] + dilation_h = self.config["atrous_rate"] + pad_h, pad_w = self.to_bigdl_2d_padding(self.config["border_mode"], h, kh, dh, dilation_h) seq = BLayer.Sequential() seq.add(BLayer.Transpose([(2, 3)])) seq.add(BLayer.Reshape([int(self.input_shape[2]), int(self.input_shape[1]), 1], True)) @@ -1105,13 +1122,13 @@ def create_atrousconvolution1d(self): n_input_plane=int(self.input_shape[2]), n_output_plane=self.config["nb_filter"], kw=1, - kh=self.config["filter_length"], + kh=kh, dw=1, - dh=self.config["subsample_length"], - pad_w=bpadW, - pad_h=bpadH, + dh=dh, + pad_w=pad_w, + pad_h=pad_h, dilation_w=1, - dilation_h=self.config["atrous_rate"], + dilation_h=dilation_h, wRegularizer=self.to_bigdl_reg(self.config["W_regularizer"]), bRegularizer=self.to_bigdl_reg(self.config["b_regularizer"]), bigdl_type="float") @@ -1127,21 +1144,27 @@ def create_atrousconvolution2d(self): if not self.config["bias"]: raise Exception("Please set `bias=True` for AtrousConvolution2D") - # TODO: border_mode=`same` - if self.klayer.border_mode == "same": - raise Exception("Unsupported border mode: %s" % self.klayer.border_mode) - bpadW, bpadH = self.to_bigdl_2d_padding(self.klayer.border_mode) + h = self.input_shape[2] + w = self.input_shape[3] + kh = self.config["nb_row"] + kw = self.config["nb_col"] + dh = self.config["subsample"][0] + dw = self.config["subsample"][1] + dilation_h = self.config["atrous_rate"][0] + dilation_w = self.config["atrous_rate"][1] + pad_h, pad_w = self.to_bigdl_2d_padding(self.config["border_mode"], h, kh, dh, dilation_h, + w, kw, dw, dilation_w) blayer = BLayer.SpatialDilatedConvolution( n_input_plane=int(self.input_shape[1]), n_output_plane=self.config["nb_filter"], - kw=self.config["nb_col"], - kh=self.config["nb_row"], - dw=self.config["subsample"][1], - dh=self.config["subsample"][0], - pad_w=bpadW, - pad_h=bpadH, - dilation_w=self.config["atrous_rate"][1], - dilation_h=self.config["atrous_rate"][0], + kw=kw, + kh=kh, + dw=dw, + dh=dh, + pad_w=pad_w, + pad_h=pad_h, + dilation_w=dilation_w, + dilation_h=dilation_h, wRegularizer=self.to_bigdl_reg(self.config["W_regularizer"]), bRegularizer=self.to_bigdl_reg(self.config["b_regularizer"]), bigdl_type="float") @@ -1151,8 +1174,31 @@ def create_atrousconvolution2d(self): def create_deconvolution2d(self): if self.klayer.dim_ordering != "th": raise Exception("Please use `th` for `dim_ordering`. `%s` is not supported for now." % self.klayer.dim_ordering) - - bpadW, bpadH = self.to_bigdl_2d_padding(self.klayer.border_mode) + output_shape = self.config["output_shape"] + + h = self.input_shape[2] + w = self.input_shape[3] + kh = self.config["nb_row"] + kw = self.config["nb_col"] + dh = self.config["subsample"][0] + dw = self.config["subsample"][1] + output_h = output_shape[2] + output_w = output_shape[3] + pad_w = 0 + pad_h = 0 + if self.config["border_mode"] == "same": + two_pad_h = (h - 1) * dh + kh - output_h # 2 times pad_h + two_pad_w = (w - 1) * dw + kw - output_w # 2 times pad_w + if two_pad_h % 2 == 0: # we only support pad_h as an int + pad_h = int(two_pad_h / 2) + else: + raise Exception("For same padding, we only support padding on both sides for now. " + "Please make `(input_row - 1) * subsample[0] + nb_row - output_row` an even integer.") + if two_pad_w % 2 == 0: # we only support pad_w as an int + pad_w = int(two_pad_w / 2) + else: + raise Exception("For same padding, we only support padding on both sides for now. " + "Please make `(input_col - 1) * subsample[1] + nb_col - output_col` an even integer.") blayer = BLayer.SpatialFullConvolution( n_input_plane=int(self.input_shape[1]), n_output_plane=self.klayer.nb_filter, @@ -1160,8 +1206,8 @@ def create_deconvolution2d(self): kh=self.klayer.nb_row, dw=self.klayer.subsample[1], dh=self.klayer.subsample[0], - pad_w=bpadW, - pad_h=bpadH, + pad_w=pad_w, + pad_h=pad_h, adj_w=0, adj_h=0, n_group=1, @@ -1174,7 +1220,11 @@ def create_deconvolution2d(self): def create_maxpooling3d(self): if self.klayer.dim_ordering != "th": - raise Exception("Please use `th` for `dim_ordering`. `%s` is not supported for now." % self.klayer.dim_ordering) + raise Exception("Please use `th` for `dim_ordering`. `%s` is not supported for now." % klayer.dim_ordering) + # TODO: border_mode = 'same' + if self.klayer.border_mode == 'same': + raise Exception("Unsupported border_mode: same") + bpadT, bpadW, bpadH = self.to_bigdl_3d_padding(self.klayer.border_mode) blayer = BLayer.VolumetricMaxPooling( k_t=self.klayer.pool_size[0], @@ -1282,7 +1332,11 @@ def create_averagepooling2d(self): def create_averagepooling3d(self): if self.klayer.dim_ordering != "th": - raise Exception("Please use `th` for `dim_ordering`. `%s` is not supported for now." % self.klayer.dim_ordering) + raise Exception("Please use `th` for `dim_ordering`. `%s` is not supported for now." % klayer.dim_ordering) + # TODO: border_mode = 'same' + if self.klayer.border_mode == 'same': + raise Exception("Unsupported border_mode: same") + bpadT, bpadW, bpadH = self.to_bigdl_3d_padding(self.klayer.border_mode) blayer = BLayer.VolumetricAveragePooling( k_t=self.klayer.pool_size[0], From be7f5d61e6f02ba03477ea57329d63d99067ce50 Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Tue, 12 Dec 2017 18:17:15 +0800 Subject: [PATCH 432/823] Support same padding for keras atrousconv1d,2d and conv3d (#1925) * add same padding for conv layers * update --- .../dllib/src/bigdl/dllib/keras/converter.py | 116 +++++++++++++----- 1 file changed, 85 insertions(+), 31 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/keras/converter.py b/python/dllib/src/bigdl/dllib/keras/converter.py index 3c4083870e1..bf0cfb9ff94 100644 --- a/python/dllib/src/bigdl/dllib/keras/converter.py +++ b/python/dllib/src/bigdl/dllib/keras/converter.py @@ -27,6 +27,7 @@ from keras.models import Sequential, Model, Layer import keras import warnings +from math import ceil def unsupport_exp(name): @@ -255,7 +256,7 @@ def convert_maxoutdense(klayer, weights): k_weights = weights[0] b_weights = k_weights[0].T for i in range(1, k_weights.shape[0]): - b_weights = np.concatenate((b_weights, k_weights[i].T)) + b_weights = np.concatenate((b_weights, k_weights[i].T)) return [b_weights, weights[1]] @staticmethod @@ -991,13 +992,28 @@ def to_bigdl_2d_ordering(self, order): def to_bigdl_3d_padding(self, border_mode): if border_mode == "valid": return 0, 0, 0 - # TODO: border_mode=`same` + elif border_mode == "same": + return -1, -1, -1 else: raise Exception("Unsupported border mode: %s" % border_mode) - def to_bigdl_2d_padding(self, border_mode): + def __calculate_2d_same_padding(self, x, kx, dx, dilation_x): + return int(ceil((x * (dx - 1) + dilation_x * (kx - 1) - dx + 1) / 2)) + + def to_bigdl_2d_padding(self, border_mode, *args): if border_mode == "same": - return -1, -1 + if len(args) == 0: # if -1 for same padding is supported + return -1, -1 + # calculate padding by given parameters + elif len(args) == 4: # used by 1d layers constructed from 2d, just need one pad + h, kh, dh, dilation_h = args + pad_h = self.__calculate_2d_same_padding(h, kh, dh, dilation_h) + return pad_h, 0 + elif len(args) == 8: + h, kh, dh, dilation_h, w, kw, dw, dilation_w = args + pad_h = self.__calculate_2d_same_padding(h, kh, dh, dilation_h) + pad_w = self.__calculate_2d_same_padding(w, kw, dw, dilation_w) + return pad_h, pad_w elif border_mode == "valid": return 0, 0 else: @@ -1094,10 +1110,11 @@ def create_atrousconvolution1d(self): if not self.config["bias"]: raise Exception("Please set `bias=True` for AtrousConvolution1D") - # TODO: border_mode=`same` - if self.klayer.border_mode == "same": - raise Exception("Unsupported border mode: %s" % self.klayer.border_mode) - bpadW, bpadH = self.to_bigdl_2d_padding(self.klayer.border_mode) + h = self.input_shape[2] + kh = self.config["filter_length"] + dh = self.config["subsample_length"] + dilation_h = self.config["atrous_rate"] + pad_h, pad_w = self.to_bigdl_2d_padding(self.config["border_mode"], h, kh, dh, dilation_h) seq = BLayer.Sequential() seq.add(BLayer.Transpose([(2, 3)])) seq.add(BLayer.Reshape([int(self.input_shape[2]), int(self.input_shape[1]), 1], True)) @@ -1105,13 +1122,13 @@ def create_atrousconvolution1d(self): n_input_plane=int(self.input_shape[2]), n_output_plane=self.config["nb_filter"], kw=1, - kh=self.config["filter_length"], + kh=kh, dw=1, - dh=self.config["subsample_length"], - pad_w=bpadW, - pad_h=bpadH, + dh=dh, + pad_w=pad_w, + pad_h=pad_h, dilation_w=1, - dilation_h=self.config["atrous_rate"], + dilation_h=dilation_h, wRegularizer=self.to_bigdl_reg(self.config["W_regularizer"]), bRegularizer=self.to_bigdl_reg(self.config["b_regularizer"]), bigdl_type="float") @@ -1127,21 +1144,27 @@ def create_atrousconvolution2d(self): if not self.config["bias"]: raise Exception("Please set `bias=True` for AtrousConvolution2D") - # TODO: border_mode=`same` - if self.klayer.border_mode == "same": - raise Exception("Unsupported border mode: %s" % self.klayer.border_mode) - bpadW, bpadH = self.to_bigdl_2d_padding(self.klayer.border_mode) + h = self.input_shape[2] + w = self.input_shape[3] + kh = self.config["nb_row"] + kw = self.config["nb_col"] + dh = self.config["subsample"][0] + dw = self.config["subsample"][1] + dilation_h = self.config["atrous_rate"][0] + dilation_w = self.config["atrous_rate"][1] + pad_h, pad_w = self.to_bigdl_2d_padding(self.config["border_mode"], h, kh, dh, dilation_h, + w, kw, dw, dilation_w) blayer = BLayer.SpatialDilatedConvolution( n_input_plane=int(self.input_shape[1]), n_output_plane=self.config["nb_filter"], - kw=self.config["nb_col"], - kh=self.config["nb_row"], - dw=self.config["subsample"][1], - dh=self.config["subsample"][0], - pad_w=bpadW, - pad_h=bpadH, - dilation_w=self.config["atrous_rate"][1], - dilation_h=self.config["atrous_rate"][0], + kw=kw, + kh=kh, + dw=dw, + dh=dh, + pad_w=pad_w, + pad_h=pad_h, + dilation_w=dilation_w, + dilation_h=dilation_h, wRegularizer=self.to_bigdl_reg(self.config["W_regularizer"]), bRegularizer=self.to_bigdl_reg(self.config["b_regularizer"]), bigdl_type="float") @@ -1151,8 +1174,31 @@ def create_atrousconvolution2d(self): def create_deconvolution2d(self): if self.klayer.dim_ordering != "th": raise Exception("Please use `th` for `dim_ordering`. `%s` is not supported for now." % self.klayer.dim_ordering) - - bpadW, bpadH = self.to_bigdl_2d_padding(self.klayer.border_mode) + output_shape = self.config["output_shape"] + + h = self.input_shape[2] + w = self.input_shape[3] + kh = self.config["nb_row"] + kw = self.config["nb_col"] + dh = self.config["subsample"][0] + dw = self.config["subsample"][1] + output_h = output_shape[2] + output_w = output_shape[3] + pad_w = 0 + pad_h = 0 + if self.config["border_mode"] == "same": + two_pad_h = (h - 1) * dh + kh - output_h # 2 times pad_h + two_pad_w = (w - 1) * dw + kw - output_w # 2 times pad_w + if two_pad_h % 2 == 0: # we only support pad_h as an int + pad_h = int(two_pad_h / 2) + else: + raise Exception("For same padding, we only support padding on both sides for now. " + "Please make `(input_row - 1) * subsample[0] + nb_row - output_row` an even integer.") + if two_pad_w % 2 == 0: # we only support pad_w as an int + pad_w = int(two_pad_w / 2) + else: + raise Exception("For same padding, we only support padding on both sides for now. " + "Please make `(input_col - 1) * subsample[1] + nb_col - output_col` an even integer.") blayer = BLayer.SpatialFullConvolution( n_input_plane=int(self.input_shape[1]), n_output_plane=self.klayer.nb_filter, @@ -1160,8 +1206,8 @@ def create_deconvolution2d(self): kh=self.klayer.nb_row, dw=self.klayer.subsample[1], dh=self.klayer.subsample[0], - pad_w=bpadW, - pad_h=bpadH, + pad_w=pad_w, + pad_h=pad_h, adj_w=0, adj_h=0, n_group=1, @@ -1174,7 +1220,11 @@ def create_deconvolution2d(self): def create_maxpooling3d(self): if self.klayer.dim_ordering != "th": - raise Exception("Please use `th` for `dim_ordering`. `%s` is not supported for now." % self.klayer.dim_ordering) + raise Exception("Please use `th` for `dim_ordering`. `%s` is not supported for now." % klayer.dim_ordering) + # TODO: border_mode = 'same' + if self.klayer.border_mode == 'same': + raise Exception("Unsupported border_mode: same") + bpadT, bpadW, bpadH = self.to_bigdl_3d_padding(self.klayer.border_mode) blayer = BLayer.VolumetricMaxPooling( k_t=self.klayer.pool_size[0], @@ -1282,7 +1332,11 @@ def create_averagepooling2d(self): def create_averagepooling3d(self): if self.klayer.dim_ordering != "th": - raise Exception("Please use `th` for `dim_ordering`. `%s` is not supported for now." % self.klayer.dim_ordering) + raise Exception("Please use `th` for `dim_ordering`. `%s` is not supported for now." % klayer.dim_ordering) + # TODO: border_mode = 'same' + if self.klayer.border_mode == 'same': + raise Exception("Unsupported border_mode: same") + bpadT, bpadW, bpadH = self.to_bigdl_3d_padding(self.klayer.border_mode) blayer = BLayer.VolumetricAveragePooling( k_t=self.klayer.pool_size[0], From e5ee71e669d3a084dc8eef0daf4e0a0203b98b90 Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Tue, 12 Dec 2017 18:17:15 +0800 Subject: [PATCH 433/823] Support same padding for keras atrousconv1d,2d and conv3d (#1925) * add same padding for conv layers * update --- python/dllib/test/bigdl/keras/test_layer.py | 29 +++++++++++---------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/python/dllib/test/bigdl/keras/test_layer.py b/python/dllib/test/bigdl/keras/test_layer.py index bba9db09e4d..5baeb7c017f 100644 --- a/python/dllib/test/bigdl/keras/test_layer.py +++ b/python/dllib/test/bigdl/keras/test_layer.py @@ -117,25 +117,22 @@ def test_conv2D(self): def test_conv3D(self): input_data = np.random.random_sample([1, 3, 32, 32, 32]) - layer = Convolution3D(12, 5, 3, 4, dim_ordering="th", - border_mode="valid", subsample=(1, 1, 2), - input_shape=(3, 32, 32, 32)) - self.modelTestSingleLayer(input_data, layer, - dump_weights=True, rtol=1e-5, atol=1e-5) + layer = lambda: Convolution3D(12, 5, 3, 4, dim_ordering="th", subsample=(1, 2, 3), + input_shape=(3, 32, 32, 32)) + self.modelTestSingleLayerWithOrdersModes(input_data, layer, dim_orderings=["th"], + dump_weights=True, rtol=1e-5, atol=1e-5) def test_atrousconvolution1d(self): input_data = np.random.random_sample([2, 10, 32]) - layer = AtrousConvolution1D(64, 3, atrous_rate=2, - border_mode="valid", activation='relu', - input_shape=(10, 32)) - self.modelTestSingleLayer(input_data, layer, dump_weights=True) + layer = lambda: AtrousConvolution1D(64, 3, atrous_rate=2, input_shape=(10, 32)) + self.modelTestSingleLayerWithOrdersModes(input_data, layer, dump_weights=True) def test_atrousconvolution2d(self): - input_data = np.random.random_sample([1, 3, 128, 128]) - layer = AtrousConvolution2D(64, 3, 4, atrous_rate=(2, 2), dim_ordering="th", - border_mode="valid", activation='tanh', - input_shape=(3, 128, 128)) - self.modelTestSingleLayer(input_data, layer, dump_weights=True) + input_data = np.random.random([1, 3, 128, 128]) + layer = lambda: AtrousConvolution2D(64, 5, 7, atrous_rate=(2, 2), + dim_ordering="th", input_shape=(3, 128, 128)) + self.modelTestSingleLayerWithOrdersModes(input_data, layer, dim_orderings=["th"], + dump_weights=True) def test_deconvolution2d(self): input_data = np.random.random_sample([32, 3, 12, 12]) @@ -147,6 +144,10 @@ def test_deconvolution2d(self): border_mode="valid", subsample=(2, 2), dim_ordering="th", input_shape=(3, 12, 12)) self.modelTestSingleLayer(input_data, layer2, dump_weights=True) + layer3 = Deconvolution2D(3, 4, 4, output_shape=(None, 3, 24, 24), + border_mode="same", subsample=(2, 2), + dim_ordering="th", input_shape=(3, 12, 12)) + self.modelTestSingleLayer(input_data, layer3, dump_weights=True) def test_maxpooling3d(self): input_data = np.random.random_sample([1, 3, 20, 15, 35]) From 2028a9ea4fe6aeb5f968dada1dce30ad6aa0fc88 Mon Sep 17 00:00:00 2001 From: Shane Huang Date: Wed, 13 Dec 2017 14:31:08 +0800 Subject: [PATCH 434/823] add setTrainData and setCriterion for optimizer reuse (#1986) * add setTrainData and setCriterion for optimizer reuse, both scala and python api * refactor the apply of Optimizer and setTrainData, changed docs --- .../dllib/src/bigdl/dllib/optim/optimizer.py | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/optim/optimizer.py b/python/dllib/src/bigdl/dllib/optim/optimizer.py index 0b18022a994..8917b73c233 100644 --- a/python/dllib/src/bigdl/dllib/optim/optimizer.py +++ b/python/dllib/src/bigdl/dllib/optim/optimizer.py @@ -654,6 +654,27 @@ def set_model(self, model): """ self.value.setModel(model.value) + def set_traindata(self, training_rdd, batch_size): + """ + Set new training dataset, for optimizer reuse + + :param training_rdd: the training dataset + :param batch_size: training batch size + :return: + """ + callBigDlFunc(self.bigdl_type, "setTrainData", self.value, + training_rdd, batch_size) + + def set_criterion(self, criterion): + """ + set new criterion, for optimizer reuse + + :param criterion: new criterion + :return: + """ + callBigDlFunc(self.bigdl_type, "setCriterion", self.value, + criterion) + def set_checkpoint(self, checkpoint_trigger, checkpoint_path, isOverWrite=True): """ From 467ccadf24cd856fd9f577fa7861f7736ba00679 Mon Sep 17 00:00:00 2001 From: Shane Huang Date: Wed, 13 Dec 2017 14:31:08 +0800 Subject: [PATCH 435/823] add setTrainData and setCriterion for optimizer reuse (#1986) * add setTrainData and setCriterion for optimizer reuse, both scala and python api * refactor the apply of Optimizer and setTrainData, changed docs --- .../dllib/src/bigdl/dllib/optim/optimizer.py | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/optim/optimizer.py b/python/dllib/src/bigdl/dllib/optim/optimizer.py index 0b18022a994..8917b73c233 100644 --- a/python/dllib/src/bigdl/dllib/optim/optimizer.py +++ b/python/dllib/src/bigdl/dllib/optim/optimizer.py @@ -654,6 +654,27 @@ def set_model(self, model): """ self.value.setModel(model.value) + def set_traindata(self, training_rdd, batch_size): + """ + Set new training dataset, for optimizer reuse + + :param training_rdd: the training dataset + :param batch_size: training batch size + :return: + """ + callBigDlFunc(self.bigdl_type, "setTrainData", self.value, + training_rdd, batch_size) + + def set_criterion(self, criterion): + """ + set new criterion, for optimizer reuse + + :param criterion: new criterion + :return: + """ + callBigDlFunc(self.bigdl_type, "setCriterion", self.value, + criterion) + def set_checkpoint(self, checkpoint_trigger, checkpoint_path, isOverWrite=True): """ From e53aa80dd33a35253ad80a4c756939f58d0af558 Mon Sep 17 00:00:00 2001 From: Xianyan Date: Wed, 13 Dec 2017 15:10:53 +0800 Subject: [PATCH 436/823] Add partition number option for ImageFrame read (#2022) * add partition number * update * Fix ut * rename partitionNum to minPartitions --- .../src/bigdl/dllib/feature/transform/vision/image.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py b/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py index 86e3b26eec8..4797b27ff5a 100644 --- a/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py +++ b/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py @@ -112,7 +112,7 @@ def __init__(self, jvalue, bigdl_type="float"): @classmethod - def read(cls, path, sc=None, bigdl_type="float"): + def read(cls, path, sc=None, min_partitions=1, bigdl_type="float"): """ Read images as Image Frame if sc is defined, Read image as DistributedImageFrame from local file system or HDFS @@ -121,12 +121,13 @@ def read(cls, path, sc=None, bigdl_type="float"): if sc is defined, path can be local or HDFS. Wildcard character are supported. if sc is null, path is local directory/image file/image file with wildcard character :param sc SparkContext + :param min_partitions A suggestion value of the minimal splitting number for input data. :return ImageFrame """ - return ImageFrame(jvalue=callBigDlFunc(bigdl_type, "read", path, sc)) + return ImageFrame(jvalue=callBigDlFunc(bigdl_type, "read", path, sc, min_partitions)) @classmethod - def readParquet(cls, path, sql_context, bigdl_type="float"): + def read_parquet(cls, path, sql_context, bigdl_type="float"): """ Read parquet file as DistributedImageFrame """ From 0bb2fb3d3d50c3bb23bc3ab6112ee3bbfb02235f Mon Sep 17 00:00:00 2001 From: Xianyan Date: Wed, 13 Dec 2017 15:10:53 +0800 Subject: [PATCH 437/823] Add partition number option for ImageFrame read (#2022) * add partition number * update * Fix ut * rename partitionNum to minPartitions --- .../src/bigdl/dllib/feature/transform/vision/image.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py b/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py index 86e3b26eec8..4797b27ff5a 100644 --- a/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py +++ b/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py @@ -112,7 +112,7 @@ def __init__(self, jvalue, bigdl_type="float"): @classmethod - def read(cls, path, sc=None, bigdl_type="float"): + def read(cls, path, sc=None, min_partitions=1, bigdl_type="float"): """ Read images as Image Frame if sc is defined, Read image as DistributedImageFrame from local file system or HDFS @@ -121,12 +121,13 @@ def read(cls, path, sc=None, bigdl_type="float"): if sc is defined, path can be local or HDFS. Wildcard character are supported. if sc is null, path is local directory/image file/image file with wildcard character :param sc SparkContext + :param min_partitions A suggestion value of the minimal splitting number for input data. :return ImageFrame """ - return ImageFrame(jvalue=callBigDlFunc(bigdl_type, "read", path, sc)) + return ImageFrame(jvalue=callBigDlFunc(bigdl_type, "read", path, sc, min_partitions)) @classmethod - def readParquet(cls, path, sql_context, bigdl_type="float"): + def read_parquet(cls, path, sql_context, bigdl_type="float"): """ Read parquet file as DistributedImageFrame """ From 530221205c89bb65b5d48b270bbc0bdefaeac05c Mon Sep 17 00:00:00 2001 From: Xin Qiu Date: Wed, 13 Dec 2017 16:13:58 +0800 Subject: [PATCH 438/823] UpSampling1D UpSampling2D (#1928) --- python/dllib/src/bigdl/dllib/nn/layer.py | 33 ++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index d176d013257..d5fe25dc036 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -2327,6 +2327,39 @@ def __init__(self, bigdl_type="float"): super(CosineDistance, self).__init__(None, bigdl_type) + +class UpSampling2D(Layer): + """ + Upsampling layer for 2D inputs. + Repeats the heights and widths of the data by size[0] and size[1] respectively. + + If input's dataformat is NCHW, then the size of output is (N, C, H * size[0], W * size[1]) + + :param size tuple of 2 integers. The upsampling factors for heights and widths. + :param format DataFormat, NCHW or NHWC + + >>> upsampled2d = UpSampling2D([2, 3]) + creating: createUpSampling2D + """ + def __init__(self, size, data_format="nchw", bigdl_type="float"): + super(UpSampling2D, self).__init__(None, bigdl_type, size, data_format) + + +class UpSampling1D(Layer): + """ + Upsampling layer for 1D inputs. + Repeats each temporal step length times along the time axis. + + If input's size is (batch, steps, features), + then the output's size is (batch, steps * length, features) + + :param length integer, upsampling factor. + >>> upsampled1d = UpSampling1D(2) + creating: createUpSampling1D + """ + def __init__(self, length, bigdl_type="float"): + super(UpSampling1D, self).__init__(None, bigdl_type, length) + class Input(Node): ''' From 613dfd489922517e5beaa1ad43ce6c3d04bc2312 Mon Sep 17 00:00:00 2001 From: Xin Qiu Date: Wed, 13 Dec 2017 16:13:58 +0800 Subject: [PATCH 439/823] UpSampling1D UpSampling2D (#1928) --- python/dllib/src/bigdl/dllib/nn/layer.py | 33 ++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index d176d013257..d5fe25dc036 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -2327,6 +2327,39 @@ def __init__(self, bigdl_type="float"): super(CosineDistance, self).__init__(None, bigdl_type) + +class UpSampling2D(Layer): + """ + Upsampling layer for 2D inputs. + Repeats the heights and widths of the data by size[0] and size[1] respectively. + + If input's dataformat is NCHW, then the size of output is (N, C, H * size[0], W * size[1]) + + :param size tuple of 2 integers. The upsampling factors for heights and widths. + :param format DataFormat, NCHW or NHWC + + >>> upsampled2d = UpSampling2D([2, 3]) + creating: createUpSampling2D + """ + def __init__(self, size, data_format="nchw", bigdl_type="float"): + super(UpSampling2D, self).__init__(None, bigdl_type, size, data_format) + + +class UpSampling1D(Layer): + """ + Upsampling layer for 1D inputs. + Repeats each temporal step length times along the time axis. + + If input's size is (batch, steps, features), + then the output's size is (batch, steps * length, features) + + :param length integer, upsampling factor. + >>> upsampled1d = UpSampling1D(2) + creating: createUpSampling1D + """ + def __init__(self, length, bigdl_type="float"): + super(UpSampling1D, self).__init__(None, bigdl_type, length) + class Input(Node): ''' From 67a51478c77b4710ad341df1d94c56f0a21fdc6c Mon Sep 17 00:00:00 2001 From: Ian Wong Date: Thu, 14 Dec 2017 14:09:49 +0800 Subject: [PATCH 440/823] Fix 1923: Use static graph execution when there's no control operation in graph container (#2008) * checkin dynamic graph and static graph * fix compile error * refine graph * meet code review * fix serialize/load test * fix failed unit test * add unit test for dynamic graph * merge duplicate code in graph * fix generateback issue * fix python test * fix style issue * meet code review * use array to cache output instead of hashmap * meet code review * fix a confused part mentioned in code review --- python/dllib/test/dev/diff.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/dllib/test/dev/diff.py b/python/dllib/test/dev/diff.py index 2ad47ab2e36..781812ec4cd 100755 --- a/python/dllib/test/dev/diff.py +++ b/python/dllib/test/dev/diff.py @@ -29,7 +29,7 @@ def extract_scala_class(class_path): exclude_key_words = set(["*", "abstract", "Const", "Fill", "Shape", - "SplitAndSelect", "StrideSlice", "Scheduler"]) + "SplitAndSelect", "StrideSlice", "Scheduler", "StaticGraph", "DynamicGraph"]) # noqa include_key_words = set(["Module", "Criterion", "Container", "Cell", "TensorNumeric"]) # noqa content = "\n".join([line for line in open(class_path).readlines() if all([key not in line for key in exclude_key_words])]) # noqa match = re.search(r"class ([\w]+)[^{]+", content) From c07e78260cfa2ce25fed65b35a5273843f6c2285 Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Thu, 14 Dec 2017 16:40:23 +0800 Subject: [PATCH 441/823] Support BigDL for Spark on k8s (#2023) * maybe some refactor * handle k8s master url * k8s * add doc * add more doc * add doc --- python/dllib/src/bigdl/dllib/utils/common.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/utils/common.py b/python/dllib/src/bigdl/dllib/utils/common.py index a8cd977c89f..384b8c7427b 100644 --- a/python/dllib/src/bigdl/dllib/utils/common.py +++ b/python/dllib/src/bigdl/dllib/utils/common.py @@ -462,6 +462,17 @@ def create_spark_conf(): sparkConf.setAll(bigdl_conf.items()) if not is_spark_below_2_2(): extend_spark_driver_cp(sparkConf, get_bigdl_classpath()) + + # add content in PYSPARK_FILES in spark.submit.pyFiles + # This is a workaround for current Spark on k8s + python_lib = os.environ.get('PYSPARK_FILES', None) + if python_lib: + existing_py_files = sparkConf.get("spark.submit.pyFiles") + if existing_py_files: + sparkConf.set(key="spark.submit.pyFiles", value="%s,%s" % (python_lib, existing_py_files)) + else: + sparkConf.set(key="spark.submit.pyFiles", value=python_lib) + return sparkConf From dbad3d4a2e9f0fc0cc36ec468420027b1b7d3b28 Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Thu, 14 Dec 2017 16:40:23 +0800 Subject: [PATCH 442/823] Support BigDL for Spark on k8s (#2023) * maybe some refactor * handle k8s master url * k8s * add doc * add more doc * add doc --- python/dllib/src/bigdl/utils/common.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/python/dllib/src/bigdl/utils/common.py b/python/dllib/src/bigdl/utils/common.py index a8cd977c89f..384b8c7427b 100644 --- a/python/dllib/src/bigdl/utils/common.py +++ b/python/dllib/src/bigdl/utils/common.py @@ -462,6 +462,17 @@ def create_spark_conf(): sparkConf.setAll(bigdl_conf.items()) if not is_spark_below_2_2(): extend_spark_driver_cp(sparkConf, get_bigdl_classpath()) + + # add content in PYSPARK_FILES in spark.submit.pyFiles + # This is a workaround for current Spark on k8s + python_lib = os.environ.get('PYSPARK_FILES', None) + if python_lib: + existing_py_files = sparkConf.get("spark.submit.pyFiles") + if existing_py_files: + sparkConf.set(key="spark.submit.pyFiles", value="%s,%s" % (python_lib, existing_py_files)) + else: + sparkConf.set(key="spark.submit.pyFiles", value=python_lib) + return sparkConf From d3841a585be516761e6d1212b0af63b55cecc25a Mon Sep 17 00:00:00 2001 From: Xianyan Date: Thu, 14 Dec 2017 17:56:32 +0800 Subject: [PATCH 443/823] Add local model predictImage (#2026) * Add local model predictImage --- .../dllib/test/bigdl/test_simple_integration.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/python/dllib/test/bigdl/test_simple_integration.py b/python/dllib/test/bigdl/test_simple_integration.py index 3dcb1f76d64..0f73eead379 100644 --- a/python/dllib/test/bigdl/test_simple_integration.py +++ b/python/dllib/test/bigdl/test_simple_integration.py @@ -494,6 +494,22 @@ def test_predict_image(self): predicts = image_frame.get_predict() predicts.collect() + def test_predict_image_local(self): + resource_path = os.path.join(os.path.split(__file__)[0], "resources") + image_path = os.path.join(resource_path, "pascal/000025.jpg") + image_frame = ImageFrame.read(image_path) + transformer = Pipeline([Resize(256, 256), CenterCrop(224, 224), + ChannelNormalize(0.485, 0.456, 0.406, 0.229, 0.224, 0.225), + MatToTensor(), ImageFrameToSample()]) + image_frame.transform(transformer) + + model = Sequential() + model.add(SpatialConvolution(3, 6, 5, 5)) + model.add(Tanh()) + + image_frame = model.predict_image(image_frame) + predicts = image_frame.get_predict() + def test_rng(self): rng = RNG() rng.set_seed(100) From c1d2124ff2b5f7e3676357fc430745f3dfc2de32 Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Fri, 15 Dec 2017 12:44:46 +0800 Subject: [PATCH 444/823] Add documentation for loading keras models (#2005) * draft * update * minor update * add limitation * minor update * add dependencies * add example * update * add fs doc * update * add example * update docs * correct grammar * update * update * fix style * move example * test loading model with theano backend * check backend * update docs * fix ut --- .../src/bigdl/dllib/examples/__init__.py | 15 +++ .../bigdl/dllib/examples/keras/__init__.py | 15 +++ .../src/bigdl/dllib/examples/keras/cnn.py | 105 ++++++++++++++++++ python/dllib/src/bigdl/dllib/nn/layer.py | 7 +- 4 files changed, 141 insertions(+), 1 deletion(-) create mode 100644 python/dllib/src/bigdl/dllib/examples/__init__.py create mode 100644 python/dllib/src/bigdl/dllib/examples/keras/__init__.py create mode 100644 python/dllib/src/bigdl/dllib/examples/keras/cnn.py diff --git a/python/dllib/src/bigdl/dllib/examples/__init__.py b/python/dllib/src/bigdl/dllib/examples/__init__.py new file mode 100644 index 00000000000..2151a805423 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/examples/__init__.py @@ -0,0 +1,15 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/python/dllib/src/bigdl/dllib/examples/keras/__init__.py b/python/dllib/src/bigdl/dllib/examples/keras/__init__.py new file mode 100644 index 00000000000..2151a805423 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/examples/keras/__init__.py @@ -0,0 +1,15 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/python/dllib/src/bigdl/dllib/examples/keras/cnn.py b/python/dllib/src/bigdl/dllib/examples/keras/cnn.py new file mode 100644 index 00000000000..40f6e7bdce4 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/examples/keras/cnn.py @@ -0,0 +1,105 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +def get_mnist(sc, data_type="train", location="/tmp/mnist"): + """ + Download or load MNIST dataset. + Normalize and transform input data into an RDD of Sample + """ + from bigdl.dataset import mnist + from bigdl.dataset.transformer import normalizer + (images, labels) = mnist.read_data_sets(location, data_type) + images = images.reshape(images.shape[0], 1, 28, 28) + images = sc.parallelize(images) + labels = sc.parallelize(labels + 1) # Target start from 1 in BigDL + record = images.zip(labels).map(lambda rec_tuple: (normalizer(rec_tuple[0], mnist.TRAIN_MEAN, mnist.TRAIN_STD), + rec_tuple[1])) \ + .map(lambda t: Sample.from_ndarray(t[0], t[1])) + return record + + +def build_keras_model(): + """ + Define a convnet model in Keras + """ + from keras.models import Sequential + from keras.layers import Dense, Dropout, Activation, Flatten + from keras.layers import Convolution2D, MaxPooling2D + + keras_model = Sequential() + keras_model.add(Convolution2D(32, 3, 3, border_mode='valid', + input_shape=(1, 28, 28))) + keras_model.add(Activation('relu')) + keras_model.add(Convolution2D(32, 3, 3)) + keras_model.add(Activation('relu')) + keras_model.add(MaxPooling2D(pool_size=(2, 2))) + keras_model.add(Dropout(0.25)) + + keras_model.add(Flatten()) + keras_model.add(Dense(128)) + keras_model.add(Activation('relu')) + keras_model.add(Dropout(0.5)) + keras_model.add(Dense(10)) + keras_model.add(Activation('softmax')) + + return keras_model + + +def save_keras_model(keras_model, path): + """ + Save a Keras model to JSON with given path + """ + model_json = keras_model.to_json() + with open(path, "w") as json_file: + json_file.write(model_json) + + +if __name__ == "__main__": + keras_model = build_keras_model() + def_path = "/tmp/lenet.json" + save_keras_model(keras_model, def_path) + + from bigdl.util.common import * + from bigdl.nn.layer import * + from bigdl.optim.optimizer import * + from bigdl.nn.criterion import * + + # Load the JSON file to a BigDL model + bigdl_model = Model.load_keras(def_path=def_path) + + sc = get_spark_context(conf=create_spark_conf()) + redire_spark_logs() + show_bigdl_info_logs() + init_engine() + + train_data = get_mnist(sc, "train") + test_data = get_mnist(sc, "test") + + optimizer = Optimizer( + model=bigdl_model, + training_rdd=train_data, + criterion=ClassNLLCriterion(logProbAsInput=False), + optim_method=Adadelta(), + end_trigger=MaxEpoch(12), + batch_size=128) + optimizer.set_validation( + batch_size=128, + val_rdd=test_data, + trigger=EveryEpoch(), + val_method=[Top1Accuracy()] + ) + optimizer.optimize() diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index d5fe25dc036..07ac1059251 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -727,12 +727,17 @@ def load_keras(def_path, weights_path=None, by_name=False): :param weights_path: The HDF5 path containing the pre-trained keras model weights. :return: A pre-trained model. """ + import os + try: + import tensorflow + except ImportError: + os.environ['KERAS_BACKEND'] = "theano" + from theano import ifelse from bigdl.keras.converter import DefinitionLoader, WeightLoader if weights_path: return WeightLoader.load_weights_from_json_hdf5(def_path, weights_path, by_name=by_name) else: return DefinitionLoader.from_json_path(def_path) - return bmodel @staticmethod def load_caffe(model, defPath, modelPath, match_all=True, bigdl_type="float"): From 4874b085a2bb6ebd2e9bb36b46d2587cd0c9c554 Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Fri, 15 Dec 2017 12:44:46 +0800 Subject: [PATCH 445/823] Add documentation for loading keras models (#2005) * draft * update * minor update * add limitation * minor update * add dependencies * add example * update * add fs doc * update * add example * update docs * correct grammar * update * update * fix style * move example * test loading model with theano backend * check backend * update docs * fix ut --- .../src/bigdl/dllib/examples/__init__.py | 15 +++ .../bigdl/dllib/examples/keras/__init__.py | 15 +++ .../src/bigdl/dllib/examples/keras/cnn.py | 105 ++++++++++++++++++ python/dllib/src/bigdl/dllib/nn/layer.py | 7 +- 4 files changed, 141 insertions(+), 1 deletion(-) create mode 100644 python/dllib/src/bigdl/dllib/examples/__init__.py create mode 100644 python/dllib/src/bigdl/dllib/examples/keras/__init__.py create mode 100644 python/dllib/src/bigdl/dllib/examples/keras/cnn.py diff --git a/python/dllib/src/bigdl/dllib/examples/__init__.py b/python/dllib/src/bigdl/dllib/examples/__init__.py new file mode 100644 index 00000000000..2151a805423 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/examples/__init__.py @@ -0,0 +1,15 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/python/dllib/src/bigdl/dllib/examples/keras/__init__.py b/python/dllib/src/bigdl/dllib/examples/keras/__init__.py new file mode 100644 index 00000000000..2151a805423 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/examples/keras/__init__.py @@ -0,0 +1,15 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/python/dllib/src/bigdl/dllib/examples/keras/cnn.py b/python/dllib/src/bigdl/dllib/examples/keras/cnn.py new file mode 100644 index 00000000000..40f6e7bdce4 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/examples/keras/cnn.py @@ -0,0 +1,105 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +def get_mnist(sc, data_type="train", location="/tmp/mnist"): + """ + Download or load MNIST dataset. + Normalize and transform input data into an RDD of Sample + """ + from bigdl.dataset import mnist + from bigdl.dataset.transformer import normalizer + (images, labels) = mnist.read_data_sets(location, data_type) + images = images.reshape(images.shape[0], 1, 28, 28) + images = sc.parallelize(images) + labels = sc.parallelize(labels + 1) # Target start from 1 in BigDL + record = images.zip(labels).map(lambda rec_tuple: (normalizer(rec_tuple[0], mnist.TRAIN_MEAN, mnist.TRAIN_STD), + rec_tuple[1])) \ + .map(lambda t: Sample.from_ndarray(t[0], t[1])) + return record + + +def build_keras_model(): + """ + Define a convnet model in Keras + """ + from keras.models import Sequential + from keras.layers import Dense, Dropout, Activation, Flatten + from keras.layers import Convolution2D, MaxPooling2D + + keras_model = Sequential() + keras_model.add(Convolution2D(32, 3, 3, border_mode='valid', + input_shape=(1, 28, 28))) + keras_model.add(Activation('relu')) + keras_model.add(Convolution2D(32, 3, 3)) + keras_model.add(Activation('relu')) + keras_model.add(MaxPooling2D(pool_size=(2, 2))) + keras_model.add(Dropout(0.25)) + + keras_model.add(Flatten()) + keras_model.add(Dense(128)) + keras_model.add(Activation('relu')) + keras_model.add(Dropout(0.5)) + keras_model.add(Dense(10)) + keras_model.add(Activation('softmax')) + + return keras_model + + +def save_keras_model(keras_model, path): + """ + Save a Keras model to JSON with given path + """ + model_json = keras_model.to_json() + with open(path, "w") as json_file: + json_file.write(model_json) + + +if __name__ == "__main__": + keras_model = build_keras_model() + def_path = "/tmp/lenet.json" + save_keras_model(keras_model, def_path) + + from bigdl.util.common import * + from bigdl.nn.layer import * + from bigdl.optim.optimizer import * + from bigdl.nn.criterion import * + + # Load the JSON file to a BigDL model + bigdl_model = Model.load_keras(def_path=def_path) + + sc = get_spark_context(conf=create_spark_conf()) + redire_spark_logs() + show_bigdl_info_logs() + init_engine() + + train_data = get_mnist(sc, "train") + test_data = get_mnist(sc, "test") + + optimizer = Optimizer( + model=bigdl_model, + training_rdd=train_data, + criterion=ClassNLLCriterion(logProbAsInput=False), + optim_method=Adadelta(), + end_trigger=MaxEpoch(12), + batch_size=128) + optimizer.set_validation( + batch_size=128, + val_rdd=test_data, + trigger=EveryEpoch(), + val_method=[Top1Accuracy()] + ) + optimizer.optimize() diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index d5fe25dc036..07ac1059251 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -727,12 +727,17 @@ def load_keras(def_path, weights_path=None, by_name=False): :param weights_path: The HDF5 path containing the pre-trained keras model weights. :return: A pre-trained model. """ + import os + try: + import tensorflow + except ImportError: + os.environ['KERAS_BACKEND'] = "theano" + from theano import ifelse from bigdl.keras.converter import DefinitionLoader, WeightLoader if weights_path: return WeightLoader.load_weights_from_json_hdf5(def_path, weights_path, by_name=by_name) else: return DefinitionLoader.from_json_path(def_path) - return bmodel @staticmethod def load_caffe(model, defPath, modelPath, match_all=True, bigdl_type="float"): From b6f6ee8bd44fe2b39025823d3a17e6a568498733 Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Fri, 15 Dec 2017 12:44:46 +0800 Subject: [PATCH 446/823] Add documentation for loading keras models (#2005) * draft * update * minor update * add limitation * minor update * add dependencies * add example * update * add fs doc * update * add example * update docs * correct grammar * update * update * fix style * move example * test loading model with theano backend * check backend * update docs * fix ut --- python/dllib/test/bigdl/keras/test_layer.py | 1 + python/dllib/test/bigdl/test_utils.py | 16 ++++++++++++++++ 2 files changed, 17 insertions(+) diff --git a/python/dllib/test/bigdl/keras/test_layer.py b/python/dllib/test/bigdl/keras/test_layer.py index 5baeb7c017f..392bd5a68d0 100644 --- a/python/dllib/test/bigdl/keras/test_layer.py +++ b/python/dllib/test/bigdl/keras/test_layer.py @@ -649,5 +649,6 @@ def test_srelu(self): layer = SReLU(input_shape=(4, 6)) self.modelTestSingleLayer(input_data, layer, dump_weights=True) + if __name__ == "__main__": pytest.main([__file__]) diff --git a/python/dllib/test/bigdl/test_utils.py b/python/dllib/test/bigdl/test_utils.py index 384c9e8ba43..83425d99114 100644 --- a/python/dllib/test/bigdl/test_utils.py +++ b/python/dllib/test/bigdl/test_utils.py @@ -210,9 +210,15 @@ def modelTest(self, keras_model.set_weights(new_kweights) # weight_converter is a function keras [ndarray]-> bigdl [ndarray] keras_model_json_path, keras_model_hdf5_path = self._dump_keras(keras_model, dump_weights) + + # Use Theano backend to load as a bigdl model + self.__set_keras_backend("theano") bigdl_model = DefinitionLoader.from_json_path(keras_model_json_path) bigdl_model.training(is_training) bigdl_output = bigdl_model.forward(input_data) + + # Use TensorFlow backend to compare results + self.__set_keras_backend("tensorflow") keras_output = keras_model.predict(input_data) # TODO: we should verify bigdl_output and keras_output here # init result is not the same, so we disable the verification for now @@ -289,6 +295,7 @@ def _do_modelTestSingleLayer(self, is_training=False, rtol=1e-6, atol=1e-6): + self.__set_keras_backend("tensorflow") keras_model = self.__generate_keras_model(functional_api=functional_api, input_data=input_data, output_layer=output_layer) @@ -299,3 +306,12 @@ def _do_modelTestSingleLayer(self, is_training=is_training, rtol=rtol, atol=atol) + + def __set_keras_backend(self, backend): + if K.backend() != backend: + os.environ['KERAS_BACKEND'] = backend + from six.moves import reload_module + reload_module(K) + assert K.backend() == backend + if backend == "theano": + from theano import ifelse From 449d57a099f34d91d681ca7a40a950e9de023b2b Mon Sep 17 00:00:00 2001 From: Yuhao Yang Date: Thu, 14 Dec 2017 23:29:51 -0800 Subject: [PATCH 447/823] Add Cropping2D and Cropping3D (#1992) * add cropping * serialization test * equal and hash * style fix * python doc --- python/dllib/src/bigdl/dllib/nn/layer.py | 50 ++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 07ac1059251..d449546795e 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -5037,6 +5037,56 @@ def __init__(self, n_classes, bbox_vote, nms_thresh = 0.3, max_per_image, thresh) +class Cropping2D(Layer): + """ + Cropping layer for 2D input (e.g. picture). + It crops along spatial dimensions, i.e. width and height. + + # Input shape + 4D tensor with shape: + `(batchSize, channels, first_axis_to_crop, second_axis_to_crop)` + + # Output shape + 4D tensor with shape: + `(batchSize, channels, first_cropped_axis, second_cropped_axis)` + + :param heightCrop Array of length 2. How many units should be trimmed off at the beginning + and end of the height dimension. + :param widthCrop Array of length 2. How many units should be trimmed off at the beginning + and end of the width dimension. + :param data_format a string value (or DataFormat Object in Scala) of "NHWC" or "NCHW" to specify the input data format of this layer. In "NHWC" format + data is stored in the order of [batch_size, height, width, channels], in "NCHW" format data is stored + in the order of [batch_size, channels, height, width]. + >>> cropping2D = Cropping2D([1, 1], [2, 2]) + creating: createCropping2D + """ + def __init__(self, heightCrop, widthCrop, data_format="NCHW", bigdl_type="float"): + super(Cropping2D, self).__init__(None, bigdl_type, heightCrop, widthCrop, data_format) + +class Cropping3D(Layer): + """ + Cropping layer for 3D data (e.g. spatial or spatio-temporal). + + # Input shape + 5D tensor with shape: + `(batchSize, channels, first_axis_to_crop, second_axis_to_crop, third_axis_to_crop)` + + # Output shape + 5D tensor with shape: + `(batchSize, channels, first_cropped_axis, second_cropped_axis, third_cropped_axis)` + + :param dim1Crop Array of length 2. How many units should be trimmed off at the beginning + and end of the first dimension. + :param dim2Crop Array of length 2. How many units should be trimmed off at the beginning + and end of the second dimension. + :param dim3Crop Array of length 2. How many units should be trimmed off at the beginning + and end of the third dimension. + :param data_format a string value. "channel_first" or "channel_last" + >>> cropping3D = Cropping3D([1, 1], [2, 2], [1, 1]) + creating: createCropping3D + """ + def __init__(self, dim1Crop, dim2Crop, dim3Crop, data_format="channel_first", bigdl_type="float"): + super(Cropping3D, self).__init__(None, bigdl_type, dim1Crop, dim2Crop, dim3Crop, data_format) def _test(): import doctest From 31eb5982c24573b214166d025804a25621bcc1fa Mon Sep 17 00:00:00 2001 From: Yuhao Yang Date: Thu, 14 Dec 2017 23:29:51 -0800 Subject: [PATCH 448/823] Add Cropping2D and Cropping3D (#1992) * add cropping * serialization test * equal and hash * style fix * python doc --- python/dllib/src/bigdl/dllib/nn/layer.py | 50 ++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 07ac1059251..d449546795e 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -5037,6 +5037,56 @@ def __init__(self, n_classes, bbox_vote, nms_thresh = 0.3, max_per_image, thresh) +class Cropping2D(Layer): + """ + Cropping layer for 2D input (e.g. picture). + It crops along spatial dimensions, i.e. width and height. + + # Input shape + 4D tensor with shape: + `(batchSize, channels, first_axis_to_crop, second_axis_to_crop)` + + # Output shape + 4D tensor with shape: + `(batchSize, channels, first_cropped_axis, second_cropped_axis)` + + :param heightCrop Array of length 2. How many units should be trimmed off at the beginning + and end of the height dimension. + :param widthCrop Array of length 2. How many units should be trimmed off at the beginning + and end of the width dimension. + :param data_format a string value (or DataFormat Object in Scala) of "NHWC" or "NCHW" to specify the input data format of this layer. In "NHWC" format + data is stored in the order of [batch_size, height, width, channels], in "NCHW" format data is stored + in the order of [batch_size, channels, height, width]. + >>> cropping2D = Cropping2D([1, 1], [2, 2]) + creating: createCropping2D + """ + def __init__(self, heightCrop, widthCrop, data_format="NCHW", bigdl_type="float"): + super(Cropping2D, self).__init__(None, bigdl_type, heightCrop, widthCrop, data_format) + +class Cropping3D(Layer): + """ + Cropping layer for 3D data (e.g. spatial or spatio-temporal). + + # Input shape + 5D tensor with shape: + `(batchSize, channels, first_axis_to_crop, second_axis_to_crop, third_axis_to_crop)` + + # Output shape + 5D tensor with shape: + `(batchSize, channels, first_cropped_axis, second_cropped_axis, third_cropped_axis)` + + :param dim1Crop Array of length 2. How many units should be trimmed off at the beginning + and end of the first dimension. + :param dim2Crop Array of length 2. How many units should be trimmed off at the beginning + and end of the second dimension. + :param dim3Crop Array of length 2. How many units should be trimmed off at the beginning + and end of the third dimension. + :param data_format a string value. "channel_first" or "channel_last" + >>> cropping3D = Cropping3D([1, 1], [2, 2], [1, 1]) + creating: createCropping3D + """ + def __init__(self, dim1Crop, dim2Crop, dim3Crop, data_format="channel_first", bigdl_type="float"): + super(Cropping3D, self).__init__(None, bigdl_type, dim1Crop, dim2Crop, dim3Crop, data_format) def _test(): import doctest From 599d39fd61da6ee1fa8a7b4434172da8c6859dbf Mon Sep 17 00:00:00 2001 From: Yao Zhang Date: Fri, 15 Dec 2017 17:23:55 +0800 Subject: [PATCH 449/823] Refine textclassifier with data without headers and refactor to temporal convolution (#1707) * refine text classifier * add more comments to TemporalMaxPooling * update document * refine textclassification scala code --- .../src/bigdl/dllib/feature/dataset/news20.py | 14 ++--- .../dllib/models/textclassifier/README.md | 25 +++++---- .../models/textclassifier/textclassifier.py | 53 +++++++++++-------- python/dllib/src/bigdl/dllib/nn/layer.py | 4 +- 4 files changed, 52 insertions(+), 44 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/feature/dataset/news20.py b/python/dllib/src/bigdl/dllib/feature/dataset/news20.py index e5aac8b02e8..7bdd1c977d5 100644 --- a/python/dllib/src/bigdl/dllib/feature/dataset/news20.py +++ b/python/dllib/src/bigdl/dllib/feature/dataset/news20.py @@ -19,17 +19,17 @@ import os import sys -NEWS20_URL = 'http://qwone.com/~jason/20Newsgroups/20news-19997.tar.gz' # noqa +NEWS20_URL = 'http://qwone.com/~jason/20Newsgroups/20news-18828.tar.gz' # noqa GLOVE_URL = 'http://nlp.stanford.edu/data/glove.6B.zip' # noqa CLASS_NUM = 20 def download_news20(dest_dir): - file_name = "20news-19997.tar.gz" + file_name = "20news-18828.tar.gz" file_abs_path = base.maybe_download(file_name, dest_dir, NEWS20_URL) tar = tarfile.open(file_abs_path, "r:gz") - extracted_to = os.path.join(dest_dir, "20_newsgroups") + extracted_to = os.path.join(dest_dir, "20news-18828") if not os.path.exists(extracted_to): print("Extracting %s to %s" % (file_abs_path, extracted_to)) tar.extractall(dest_dir) @@ -50,7 +50,7 @@ def download_glove_w2v(dest_dir): return extracted_to -def get_news20(source_dir="/tmp/news20/"): +def get_news20(source_dir="./data/news20/"): """ Parse or download news20 if source_dir is empty. @@ -79,7 +79,7 @@ def get_news20(source_dir="/tmp/news20/"): return texts -def get_glove_w2v(source_dir="/tmp/news20/", dim=100): +def get_glove_w2v(source_dir="./data/news20/", dim=100): """ Parse or download the pre-trained glove word2vec if source_dir is empty. @@ -102,5 +102,5 @@ def get_glove_w2v(source_dir="/tmp/news20/", dim=100): if __name__ == "__main__": - get_news20("/tmp/news20/") - get_glove_w2v("/tmp/news20/") + get_news20("./data/news20/") + get_glove_w2v("./data/news20/") diff --git a/python/dllib/src/bigdl/dllib/models/textclassifier/README.md b/python/dllib/src/bigdl/dllib/models/textclassifier/README.md index f431a54163e..2abb081469f 100644 --- a/python/dllib/src/bigdl/dllib/models/textclassifier/README.md +++ b/python/dllib/src/bigdl/dllib/models/textclassifier/README.md @@ -1,11 +1,11 @@ ## Summary This example use a (pre-trained GloVe embedding) to convert word to vector, and uses it to train a CNN, LSTM or GRU text classification model on a 20 Newsgroup dataset - with 20 different categories. CNN model can achieve around 96% accuracy after 2 or 3 epochs training. + with 20 different categories. CNN model can achieve around 85% accuracy after 20 epochs training. LSTM and GRU are a little difficult to train, which need more epochs to achieve the equivalent result. (It was first described in: https://blog.keras.io/using-pre-trained-word-embeddings-in-a-keras-model.html) ## Data -* Embedding: 100-dimensional pre-trained GloVe embeddings of 400k words which trained on a 2014 dump of English Wikipedia. +* Embedding: 200-dimensional pre-trained GloVe embeddings of 400k words which trained on a 2014 dump of English Wikipedia. * Training data: "20 Newsgroup dataset" which containing 20 categories and with totally 19997 texts. ## Install dependencies @@ -17,12 +17,12 @@ Please note that due to some permission issue, this example **cannot** be run on If there is no [Pre-train GloVe word embeddings](http://nlp.stanford.edu/data/glove.6B.zip) or [20 Newsgroup dataset](http://www.cs.cmu.edu/afs/cs.cmu.edu/project/theo-20/www/data/news20.html) in -`/tmp/news20` directory with the following structure looks like: +`./data/news20` directory with the following structure looks like: ```{r, engine='sh'} $ [/tmp/news20]$ tree . -L 1 . - ├── 20news-19997.tar.gz + ├── 20news-18828.tar.gz └── glove.6B.zip ``` - The example code would automatically download the data during the first run. @@ -57,26 +57,25 @@ python ${BigDL_HOME}/pyspark/bigdl/models/textclassifier/textclassifier.py --max --conf spark.executor.extraClassPath=bigdl-VERSION-jar-with-dependencies.jar \ --conf spark.executorEnv.PYTHONHASHSEED=${PYTHONHASHSEED} \ ${BigDL_HOME}/pyspark/bigdl/models/textclassifier/textclassifier.py \ - --data_path /tmp/news20/ \ - --max_epoch 3 \ - --model cnn + --model cnn ``` * `--data_path` option can be used to set the path for downloading news20 data, the default value is /tmp/news20. Make sure that you have write permission to the specified path. -* `--max_epoch` option can be used to set how many epochs the model to be trained. +* `--max_epoch` option can be used to set how many epochs the model to be trained * `--model` option can be used to choose a model to be trained, three models are supported in this example, -which are `cnn`, `lstm` and `gru`, default is `cnn`. +which are `cnn`, `lstm` and `gru`, default is `cnn` * `--batchSize` option can be used to set batch size, the default value is 128. -* `embedding_dim` option can be used to set the embedding size of word vector, the default value is 50. +* `--embedding_dim` option can be used to set the embedding size of word vector, the default value is 200. + +* `--learning_rate` option can be used to set learning rate, default is 0.05. To verify the accuracy, search "accuracy" from log: ```{r, engine='sh'} - [Epoch 1 0/15964][Iteration 1][Wall Clock 0.0s] + [Epoch 20 15104/15005][Iteration 2360] - top1 accuracy is Accuracy(correct: 14749, count: 15964, accuracy: 0.9238912 - 553244801) + top1 accuracy is Accuracy(correct: 3239, count: 3823, accuracy: 0.84724) ``` diff --git a/python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py b/python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py index 922ac4a2aef..314e3e4146f 100644 --- a/python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py +++ b/python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py @@ -25,6 +25,7 @@ from bigdl.optim.optimizer import * from bigdl.util.common import * from bigdl.util.common import Sample +import datetime as dt def text_to_words(review_text): @@ -65,8 +66,6 @@ def to_sample(vectors, label, embedding_dim): features = np.array(flatten_features, dtype='float').reshape( [sequence_len, embedding_dim]) - if model_type.lower() == "cnn": - features = features.transpose(1, 0) return Sample.from_ndarray(features, np.array(label)) @@ -74,28 +73,25 @@ def build_model(class_num): model = Sequential() if model_type.lower() == "cnn": - model.add(Reshape([embedding_dim, 1, sequence_len])) - model.add(SpatialConvolution(embedding_dim, 128, 5, 1)) - model.add(ReLU()) - model.add(SpatialMaxPooling(5, 1, 5, 1)) - model.add(SpatialConvolution(128, 128, 5, 1)) - model.add(ReLU()) - model.add(SpatialMaxPooling(5, 1, 5, 1)) - model.add(Reshape([128])) + model.add(TemporalConvolution(embedding_dim, 256, 5)) \ + .add(ReLU()) \ + .add(TemporalMaxPooling(sequence_len - 5 + 1)) \ + .add(Squeeze(2)) elif model_type.lower() == "lstm": model.add(Recurrent() - .add(LSTM(embedding_dim, 128, p))) + .add(LSTM(embedding_dim, 256, p))) model.add(Select(2, -1)) elif model_type.lower() == "gru": model.add(Recurrent() - .add(GRU(embedding_dim, 128, p))) + .add(GRU(embedding_dim, 256, p))) model.add(Select(2, -1)) - else: - raise ValueError('model can only be cnn, lstm, or gru') - model.add(Linear(128, 100)) - model.add(Linear(100, class_num)) - model.add(LogSoftMax()) + model.add(Linear(256, 128)) \ + .add(Dropout(0.2)) \ + .add(ReLU()) \ + .add(Linear(128, class_num)) \ + .add(LogSoftMax()) + return model @@ -137,7 +133,7 @@ def train(sc, data_path, criterion=ClassNLLCriterion(), end_trigger=MaxEpoch(max_epoch), batch_size=batch_size, - optim_method=Adagrad(learningrate=0.01, learningrate_decay=0.0002)) + optim_method=Adagrad(learningrate=learning_rate, learningrate_decay=0.001)) optimizer.set_validation( batch_size=batch_size, @@ -145,14 +141,26 @@ def train(sc, data_path, trigger=EveryEpoch(), val_method=[Top1Accuracy()] ) + + logdir = '/tmp/.bigdl/' + app_name = 'adam-' + dt.datetime.now().strftime("%Y%m%d-%H%M%S") + + train_summary = TrainSummary(log_dir=logdir, app_name=app_name) + train_summary.set_summary_trigger("Parameters", SeveralIteration(50)) + val_summary = ValidationSummary(log_dir=logdir, app_name=app_name) + optimizer.set_train_summary(train_summary) + optimizer.set_val_summary(val_summary) + train_model = optimizer.optimize() + if __name__ == "__main__": parser = OptionParser() parser.add_option("-a", "--action", dest="action", default="train") + parser.add_option("-l", "--learning_rate", dest="learning_rate", default="0.05") parser.add_option("-b", "--batchSize", dest="batchSize", default="128") - parser.add_option("-e", "--embedding_dim", dest="embedding_dim", default="50") # noqa - parser.add_option("-m", "--max_epoch", dest="max_epoch", default="15") + parser.add_option("-e", "--embedding_dim", dest="embedding_dim", default="300") # noqa + parser.add_option("-m", "--max_epoch", dest="max_epoch", default="30") parser.add_option("--model", dest="model_type", default="cnn") parser.add_option("-p", "--p", dest="p", default="0.0") parser.add_option("-d", "--data_path", dest="data_path", default="/tmp/news20/") @@ -161,11 +169,12 @@ def train(sc, data_path, if options.action == "train": batch_size = int(options.batchSize) embedding_dim = int(options.embedding_dim) + learning_rate = float(options.learning_rate) max_epoch = int(options.max_epoch) p = float(options.p) model_type = options.model_type - sequence_len = 50 - max_words = 1000 + sequence_len = 500 + max_words = 5000 training_split = 0.8 sc = SparkContext(appName="text_classifier", conf=create_spark_conf()) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index d449546795e..a8702448bc4 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -1246,7 +1246,7 @@ class TemporalMaxPooling(Layer): nOutputFrame = (nInputFrame - k_w) / d_w + 1 :param k_w: kernel width - :param d_w: step size in width + :param d_w: step size in width, default is -1, means the `d_w` equals `k_w` >>> temporalMaxPooling = TemporalMaxPooling(2, 2) creating: createTemporalMaxPooling @@ -1254,7 +1254,7 @@ class TemporalMaxPooling(Layer): def __init__(self, k_w, - d_w, + d_w=-1, bigdl_type="float"): super(TemporalMaxPooling, self).__init__(None, bigdl_type, k_w, d_w) From a4227b51796187e86584ddf579be1b9656a7798b Mon Sep 17 00:00:00 2001 From: Yao Zhang Date: Fri, 15 Dec 2017 17:23:55 +0800 Subject: [PATCH 450/823] Refine textclassifier with data without headers and refactor to temporal convolution (#1707) * refine text classifier * add more comments to TemporalMaxPooling * update document * refine textclassification scala code --- .../src/bigdl/dllib/feature/dataset/news20.py | 14 ++--- .../dllib/models/textclassifier/README.md | 25 +++++---- .../models/textclassifier/textclassifier.py | 53 +++++++++++-------- python/dllib/src/bigdl/dllib/nn/layer.py | 4 +- 4 files changed, 52 insertions(+), 44 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/feature/dataset/news20.py b/python/dllib/src/bigdl/dllib/feature/dataset/news20.py index e5aac8b02e8..7bdd1c977d5 100644 --- a/python/dllib/src/bigdl/dllib/feature/dataset/news20.py +++ b/python/dllib/src/bigdl/dllib/feature/dataset/news20.py @@ -19,17 +19,17 @@ import os import sys -NEWS20_URL = 'http://qwone.com/~jason/20Newsgroups/20news-19997.tar.gz' # noqa +NEWS20_URL = 'http://qwone.com/~jason/20Newsgroups/20news-18828.tar.gz' # noqa GLOVE_URL = 'http://nlp.stanford.edu/data/glove.6B.zip' # noqa CLASS_NUM = 20 def download_news20(dest_dir): - file_name = "20news-19997.tar.gz" + file_name = "20news-18828.tar.gz" file_abs_path = base.maybe_download(file_name, dest_dir, NEWS20_URL) tar = tarfile.open(file_abs_path, "r:gz") - extracted_to = os.path.join(dest_dir, "20_newsgroups") + extracted_to = os.path.join(dest_dir, "20news-18828") if not os.path.exists(extracted_to): print("Extracting %s to %s" % (file_abs_path, extracted_to)) tar.extractall(dest_dir) @@ -50,7 +50,7 @@ def download_glove_w2v(dest_dir): return extracted_to -def get_news20(source_dir="/tmp/news20/"): +def get_news20(source_dir="./data/news20/"): """ Parse or download news20 if source_dir is empty. @@ -79,7 +79,7 @@ def get_news20(source_dir="/tmp/news20/"): return texts -def get_glove_w2v(source_dir="/tmp/news20/", dim=100): +def get_glove_w2v(source_dir="./data/news20/", dim=100): """ Parse or download the pre-trained glove word2vec if source_dir is empty. @@ -102,5 +102,5 @@ def get_glove_w2v(source_dir="/tmp/news20/", dim=100): if __name__ == "__main__": - get_news20("/tmp/news20/") - get_glove_w2v("/tmp/news20/") + get_news20("./data/news20/") + get_glove_w2v("./data/news20/") diff --git a/python/dllib/src/bigdl/dllib/models/textclassifier/README.md b/python/dllib/src/bigdl/dllib/models/textclassifier/README.md index f431a54163e..2abb081469f 100644 --- a/python/dllib/src/bigdl/dllib/models/textclassifier/README.md +++ b/python/dllib/src/bigdl/dllib/models/textclassifier/README.md @@ -1,11 +1,11 @@ ## Summary This example use a (pre-trained GloVe embedding) to convert word to vector, and uses it to train a CNN, LSTM or GRU text classification model on a 20 Newsgroup dataset - with 20 different categories. CNN model can achieve around 96% accuracy after 2 or 3 epochs training. + with 20 different categories. CNN model can achieve around 85% accuracy after 20 epochs training. LSTM and GRU are a little difficult to train, which need more epochs to achieve the equivalent result. (It was first described in: https://blog.keras.io/using-pre-trained-word-embeddings-in-a-keras-model.html) ## Data -* Embedding: 100-dimensional pre-trained GloVe embeddings of 400k words which trained on a 2014 dump of English Wikipedia. +* Embedding: 200-dimensional pre-trained GloVe embeddings of 400k words which trained on a 2014 dump of English Wikipedia. * Training data: "20 Newsgroup dataset" which containing 20 categories and with totally 19997 texts. ## Install dependencies @@ -17,12 +17,12 @@ Please note that due to some permission issue, this example **cannot** be run on If there is no [Pre-train GloVe word embeddings](http://nlp.stanford.edu/data/glove.6B.zip) or [20 Newsgroup dataset](http://www.cs.cmu.edu/afs/cs.cmu.edu/project/theo-20/www/data/news20.html) in -`/tmp/news20` directory with the following structure looks like: +`./data/news20` directory with the following structure looks like: ```{r, engine='sh'} $ [/tmp/news20]$ tree . -L 1 . - ├── 20news-19997.tar.gz + ├── 20news-18828.tar.gz └── glove.6B.zip ``` - The example code would automatically download the data during the first run. @@ -57,26 +57,25 @@ python ${BigDL_HOME}/pyspark/bigdl/models/textclassifier/textclassifier.py --max --conf spark.executor.extraClassPath=bigdl-VERSION-jar-with-dependencies.jar \ --conf spark.executorEnv.PYTHONHASHSEED=${PYTHONHASHSEED} \ ${BigDL_HOME}/pyspark/bigdl/models/textclassifier/textclassifier.py \ - --data_path /tmp/news20/ \ - --max_epoch 3 \ - --model cnn + --model cnn ``` * `--data_path` option can be used to set the path for downloading news20 data, the default value is /tmp/news20. Make sure that you have write permission to the specified path. -* `--max_epoch` option can be used to set how many epochs the model to be trained. +* `--max_epoch` option can be used to set how many epochs the model to be trained * `--model` option can be used to choose a model to be trained, three models are supported in this example, -which are `cnn`, `lstm` and `gru`, default is `cnn`. +which are `cnn`, `lstm` and `gru`, default is `cnn` * `--batchSize` option can be used to set batch size, the default value is 128. -* `embedding_dim` option can be used to set the embedding size of word vector, the default value is 50. +* `--embedding_dim` option can be used to set the embedding size of word vector, the default value is 200. + +* `--learning_rate` option can be used to set learning rate, default is 0.05. To verify the accuracy, search "accuracy" from log: ```{r, engine='sh'} - [Epoch 1 0/15964][Iteration 1][Wall Clock 0.0s] + [Epoch 20 15104/15005][Iteration 2360] - top1 accuracy is Accuracy(correct: 14749, count: 15964, accuracy: 0.9238912 - 553244801) + top1 accuracy is Accuracy(correct: 3239, count: 3823, accuracy: 0.84724) ``` diff --git a/python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py b/python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py index 922ac4a2aef..314e3e4146f 100644 --- a/python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py +++ b/python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py @@ -25,6 +25,7 @@ from bigdl.optim.optimizer import * from bigdl.util.common import * from bigdl.util.common import Sample +import datetime as dt def text_to_words(review_text): @@ -65,8 +66,6 @@ def to_sample(vectors, label, embedding_dim): features = np.array(flatten_features, dtype='float').reshape( [sequence_len, embedding_dim]) - if model_type.lower() == "cnn": - features = features.transpose(1, 0) return Sample.from_ndarray(features, np.array(label)) @@ -74,28 +73,25 @@ def build_model(class_num): model = Sequential() if model_type.lower() == "cnn": - model.add(Reshape([embedding_dim, 1, sequence_len])) - model.add(SpatialConvolution(embedding_dim, 128, 5, 1)) - model.add(ReLU()) - model.add(SpatialMaxPooling(5, 1, 5, 1)) - model.add(SpatialConvolution(128, 128, 5, 1)) - model.add(ReLU()) - model.add(SpatialMaxPooling(5, 1, 5, 1)) - model.add(Reshape([128])) + model.add(TemporalConvolution(embedding_dim, 256, 5)) \ + .add(ReLU()) \ + .add(TemporalMaxPooling(sequence_len - 5 + 1)) \ + .add(Squeeze(2)) elif model_type.lower() == "lstm": model.add(Recurrent() - .add(LSTM(embedding_dim, 128, p))) + .add(LSTM(embedding_dim, 256, p))) model.add(Select(2, -1)) elif model_type.lower() == "gru": model.add(Recurrent() - .add(GRU(embedding_dim, 128, p))) + .add(GRU(embedding_dim, 256, p))) model.add(Select(2, -1)) - else: - raise ValueError('model can only be cnn, lstm, or gru') - model.add(Linear(128, 100)) - model.add(Linear(100, class_num)) - model.add(LogSoftMax()) + model.add(Linear(256, 128)) \ + .add(Dropout(0.2)) \ + .add(ReLU()) \ + .add(Linear(128, class_num)) \ + .add(LogSoftMax()) + return model @@ -137,7 +133,7 @@ def train(sc, data_path, criterion=ClassNLLCriterion(), end_trigger=MaxEpoch(max_epoch), batch_size=batch_size, - optim_method=Adagrad(learningrate=0.01, learningrate_decay=0.0002)) + optim_method=Adagrad(learningrate=learning_rate, learningrate_decay=0.001)) optimizer.set_validation( batch_size=batch_size, @@ -145,14 +141,26 @@ def train(sc, data_path, trigger=EveryEpoch(), val_method=[Top1Accuracy()] ) + + logdir = '/tmp/.bigdl/' + app_name = 'adam-' + dt.datetime.now().strftime("%Y%m%d-%H%M%S") + + train_summary = TrainSummary(log_dir=logdir, app_name=app_name) + train_summary.set_summary_trigger("Parameters", SeveralIteration(50)) + val_summary = ValidationSummary(log_dir=logdir, app_name=app_name) + optimizer.set_train_summary(train_summary) + optimizer.set_val_summary(val_summary) + train_model = optimizer.optimize() + if __name__ == "__main__": parser = OptionParser() parser.add_option("-a", "--action", dest="action", default="train") + parser.add_option("-l", "--learning_rate", dest="learning_rate", default="0.05") parser.add_option("-b", "--batchSize", dest="batchSize", default="128") - parser.add_option("-e", "--embedding_dim", dest="embedding_dim", default="50") # noqa - parser.add_option("-m", "--max_epoch", dest="max_epoch", default="15") + parser.add_option("-e", "--embedding_dim", dest="embedding_dim", default="300") # noqa + parser.add_option("-m", "--max_epoch", dest="max_epoch", default="30") parser.add_option("--model", dest="model_type", default="cnn") parser.add_option("-p", "--p", dest="p", default="0.0") parser.add_option("-d", "--data_path", dest="data_path", default="/tmp/news20/") @@ -161,11 +169,12 @@ def train(sc, data_path, if options.action == "train": batch_size = int(options.batchSize) embedding_dim = int(options.embedding_dim) + learning_rate = float(options.learning_rate) max_epoch = int(options.max_epoch) p = float(options.p) model_type = options.model_type - sequence_len = 50 - max_words = 1000 + sequence_len = 500 + max_words = 5000 training_split = 0.8 sc = SparkContext(appName="text_classifier", conf=create_spark_conf()) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index d449546795e..a8702448bc4 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -1246,7 +1246,7 @@ class TemporalMaxPooling(Layer): nOutputFrame = (nInputFrame - k_w) / d_w + 1 :param k_w: kernel width - :param d_w: step size in width + :param d_w: step size in width, default is -1, means the `d_w` equals `k_w` >>> temporalMaxPooling = TemporalMaxPooling(2, 2) creating: createTemporalMaxPooling @@ -1254,7 +1254,7 @@ class TemporalMaxPooling(Layer): def __init__(self, k_w, - d_w, + d_w=-1, bigdl_type="float"): super(TemporalMaxPooling, self).__init__(None, bigdl_type, k_w, d_w) From 94dd0036d1fba1e0b23e1f297dd3dfc3a1b832d8 Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Fri, 15 Dec 2017 18:36:33 +0800 Subject: [PATCH 451/823] Newly added Keras Layers python side (#2024) * add new layers * update * fix ut --- .../dllib/src/bigdl/dllib/keras/converter.py | 95 +++++++++++++++---- python/dllib/src/bigdl/dllib/nn/layer.py | 6 +- 2 files changed, 82 insertions(+), 19 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/keras/converter.py b/python/dllib/src/bigdl/dllib/keras/converter.py index bf0cfb9ff94..f1cb4990d18 100644 --- a/python/dllib/src/bigdl/dllib/keras/converter.py +++ b/python/dllib/src/bigdl/dllib/keras/converter.py @@ -257,12 +257,22 @@ def convert_maxoutdense(klayer, weights): b_weights = k_weights[0].T for i in range(1, k_weights.shape[0]): b_weights = np.concatenate((b_weights, k_weights[i].T)) - return [b_weights, weights[1]] + return [b_weights, weights[1].reshape(k_weights.shape[0]*k_weights.shape[2], )] @staticmethod def convert_srelu(klayer, weights): return weights + @staticmethod + def convert_separableconvolution2d(klayer, weights): + if len(weights) == 2: # if without bias + if klayer.dim_ordering == "th": + bias = weights[1].shape[0] + else: + bias = weights[1].shape[3] + return [weights[0], weights[1], np.zeros(bias, )] + return weights + class DefinitionLoader: @@ -732,7 +742,8 @@ def __generate_zeropadding2d(self, dim1, dim2, n_input_dim, pad1, pad2, pad3, pa def create_zeropadding2d(self): padding = self.klayer.padding dim = 1 - warnings.warn("Cannot find dim_ordering from json definition. Using the default instead.") + if "dim_ordering" not in self.config: + warnings.warn("Cannot find dim_ordering from json definition. Using the default instead.") if self.klayer.dim_ordering == "th": dim = 2 if isinstance(padding, dict): # dictionary @@ -752,7 +763,8 @@ def create_zeropadding2d(self): def create_zeropadding3d(self): padding = tuple(self.klayer.padding) dim = 1 - warnings.warn("Cannot find dim_ordering from json definition. Using the default instead.") + if "dim_ordering" not in self.config: + warnings.warn("Cannot find dim_ordering from json definition. Using the default instead.") if self.klayer.dim_ordering == "th": dim = 2 model = BLayer.Sequential() @@ -935,10 +947,14 @@ def create_gru(self): self.klayer.go_backwards, rec.add(gru)) def create_batchnormalization(self): - if keras.backend.image_dim_ordering() != "th" or self.klayer.axis != 1: - raise Exception("""For BatchNormalization, we only support th image ordering (i.e. NCHW) """ + - """with axis = 1 for now, but the current order is %s and axis is %s - """ % (keras.backend.image_dim_ordering(), self.klayer.axis)) # noqa + if keras.backend.image_dim_ordering() == "th" and self.klayer.axis != 1: + raise Exception("""For BatchNormalization with th image ordering, we only support """ + + """axis = 1 for now, but the current axis is %s + """ % self.klayer.axis) # noqa + if keras.backend.image_dim_ordering() == "tf" and self.klayer.axis != -1: + raise Exception("""For BatchNormalization with tf image ordering, we only support """ + + """axis = -1 for now, but the current axis is %s + """ % self.klayer.axis) if self.klayer.mode != 0: raise Exception( "Only support mode = 0 for now, but the current mode is: %s", self.klayer.mode) @@ -949,7 +965,8 @@ def create_batchnormalization(self): if self.config["beta_regularizer"]: raise Exception("We don't support beta_regularizer for now") - n_input_channel = int(self.input_shape[self.klayer.axis]) # default is -1 which is channel-last + bigdl_order = self.to_bigdl_2d_ordering(keras.backend.image_dim_ordering()) + n_input_channel = int(self.input_shape[self.klayer.axis]) # init gamma and beta # TODO: replace this with to_bigdl_init in the future @@ -965,6 +982,7 @@ def create_batchnormalization(self): init_bias=beta, init_grad_weight=None, init_grad_bias=None, + data_format=bigdl_order, bigdl_type="float") k_running_mean = keras.backend.eval(self.klayer.running_mean) @@ -1064,8 +1082,8 @@ def create_convolution2d(self): n_output_plane=self.klayer.nb_filter, kernel_w=self.klayer.nb_col, kernel_h=self.klayer.nb_row, - stride_w=self.klayer.subsample[0], - stride_h=self.klayer.subsample[1], + stride_w=self.klayer.subsample[1], + stride_h=self.klayer.subsample[0], pad_w=bpadW, pad_h=bpadH, n_group=1, @@ -1108,7 +1126,7 @@ def create_convolution3d(self): def create_atrousconvolution1d(self): if not self.config["bias"]: - raise Exception("Please set `bias=True` for AtrousConvolution1D") + raise Exception("Only bias=True is supported for AtrousConvolution1D") h = self.input_shape[2] kh = self.config["filter_length"] @@ -1142,7 +1160,7 @@ def create_atrousconvolution2d(self): if self.klayer.dim_ordering != "th": raise Exception("Please use `th` for `dim_ordering`. `%s` is not supported for now." % self.klayer.dim_ordering) if not self.config["bias"]: - raise Exception("Please set `bias=True` for AtrousConvolution2D") + raise Exception("Only bias=True is supported for AtrousConvolution2D") h = self.input_shape[2] w = self.input_shape[3] @@ -1506,11 +1524,21 @@ def create_globalaveragepooling2d(self): seq.add(BLayer.Squeeze(1, num_input_dims=2)) return seq + def create_upsampling1d(self): + return BLayer.UpSampling1D(self.klayer.length) + + def create_upsampling2d(self): + if "dim_ordering" not in self.config: + warnings.warn("Cannot find dim_ordering from json definition. Using the default instead.") + bigdl_order = self.get_bdim_order() + return BLayer.UpSampling2D(self.klayer.size, bigdl_order) + def create_upsampling3d(self): if self.klayer.dim_ordering != "th": raise Exception("Please use th for dim_ordering. %s is not supported for now." % self.klayer.dim_ordering) - warnings.warn("Cannot find dim_ordering from json definition. Using the default instead." - "We only support th for now.") + if "dim_ordering" not in self.config: + warnings.warn("Cannot find dim_ordering from json definition. Using the default instead." + "We only support th for now.") return BLayer.UpSampling3D(self.klayer.size) def create_gaussiannoise(self): @@ -1538,7 +1566,7 @@ def create_maxoutdense(self): if self.config["b_regularizer"]: raise Exception("b_regularizer is not supported for MaxoutDense") if not self.config["bias"]: - raise Exception("Please set bias=True for MaxoutDense") + raise Exception("Only bias=True is supported for MaxoutDense") blayer = BLayer.Maxout(input_size=self.input_shape[1], output_size=self.klayer.output_dim, maxout_number=self.klayer.nb_feature) @@ -1548,12 +1576,47 @@ def create_masking(self): return BLayer.Masking(float(self.klayer.mask_value)) def create_srelu(self): - warnings.warn("Cannot find shared_axes from json definition. Using shared_axes=None instead.") + if "shared_axes" not in self.config: + warnings.warn("Cannot find shared_axes from json definition. Using shared_axes=None instead.") if self.klayer.shared_axes == [None]: return BLayer.SReLU() else: return BLayer.SReLU(self.klayer.shared_axes) + def create_separableconvolution2d(self): + if keras.backend.backend() != 'tensorflow': + raise Exception('Please use tensorflow backend for keras 1.2.2 ' + 'if you want to load SeparableConv2D') + bigdl_order = self.get_bdim_order() + + if bigdl_order == "NCHW": + stack_size = int(self.input_shape[1]) + elif bigdl_order == "NHWC": + stack_size = int(self.input_shape[3]) + + bpadW, bpadH = self.to_bigdl_2d_padding(self.klayer.border_mode) + blayer = BLayer.SpatialSeperableConvolution( + n_input_channel=stack_size, + n_output_channel=self.klayer.nb_filter, + depth_multiplier=self.klayer.depth_multiplier, + kernel_w=self.klayer.nb_col, + kernel_h=self.klayer.nb_row, + stride_w=self.klayer.subsample[1], + stride_h=self.klayer.subsample[0], + pad_w=bpadW, + pad_h=bpadH, + with_bias=self.klayer.bias, + data_format=bigdl_order, + w_regularizer=self.to_bigdl_reg(self.config["depthwise_regularizer"]), + b_regularizer=self.to_bigdl_reg(self.config["b_regularizer"]), + p_regularizer=self.to_bigdl_reg(self.config["pointwise_regularizer"]) + ) + + return self.combo_parameter_layer(blayer, self.config) + + def create_activityregularization(self): + return BLayer.ActivityRegularization(l1=self.klayer.l1, l2=self.klayer.l2) + def combo_parameter_layer(self, blayer, config): blayer.set_name(config["name"]) if hasattr(blayer, "set_init_method"): diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index a8702448bc4..7153a5fda5e 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -3301,9 +3301,9 @@ class SpatialSeperableConvolution(Layer): :param data_format: a string value of "NHWC" or "NCHW" to specify the input data format of this layer. In "NHWC" format data is stored in the order of [batch_size, height, width, channels], in "NCHW" format data is stored in the order of [batch_size, channels, height, width]. - :param wRegularizer: instance of [[Regularizer]](eg. L1 or L2 regularization), applied to the depth weights matrices. - :param bRegularizer: instance of [[Regularizer]]applied to the pointwise bias. - :param pRegularizer: instance of [[Regularizer]]applied to the pointwise weights. + :param w_regularizer: instance of [[Regularizer]](eg. L1 or L2 regularization), applied to the depth weights matrices. + :param b_regularizer: instance of [[Regularizer]]applied to the pointwise bias. + :param p_regularizer: instance of [[Regularizer]]applied to the pointwise weights. >>> conv = SpatialSeperableConvolution(6, 12, 1, 5, 5) creating: createSpatialSeperableConvolution From 225f7120152d094dfa968746cfdc128211e36661 Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Fri, 15 Dec 2017 18:36:33 +0800 Subject: [PATCH 452/823] Newly added Keras Layers python side (#2024) * add new layers * update * fix ut --- .../dllib/src/bigdl/dllib/keras/converter.py | 95 +++++++++++++++---- python/dllib/src/bigdl/dllib/nn/layer.py | 6 +- 2 files changed, 82 insertions(+), 19 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/keras/converter.py b/python/dllib/src/bigdl/dllib/keras/converter.py index bf0cfb9ff94..f1cb4990d18 100644 --- a/python/dllib/src/bigdl/dllib/keras/converter.py +++ b/python/dllib/src/bigdl/dllib/keras/converter.py @@ -257,12 +257,22 @@ def convert_maxoutdense(klayer, weights): b_weights = k_weights[0].T for i in range(1, k_weights.shape[0]): b_weights = np.concatenate((b_weights, k_weights[i].T)) - return [b_weights, weights[1]] + return [b_weights, weights[1].reshape(k_weights.shape[0]*k_weights.shape[2], )] @staticmethod def convert_srelu(klayer, weights): return weights + @staticmethod + def convert_separableconvolution2d(klayer, weights): + if len(weights) == 2: # if without bias + if klayer.dim_ordering == "th": + bias = weights[1].shape[0] + else: + bias = weights[1].shape[3] + return [weights[0], weights[1], np.zeros(bias, )] + return weights + class DefinitionLoader: @@ -732,7 +742,8 @@ def __generate_zeropadding2d(self, dim1, dim2, n_input_dim, pad1, pad2, pad3, pa def create_zeropadding2d(self): padding = self.klayer.padding dim = 1 - warnings.warn("Cannot find dim_ordering from json definition. Using the default instead.") + if "dim_ordering" not in self.config: + warnings.warn("Cannot find dim_ordering from json definition. Using the default instead.") if self.klayer.dim_ordering == "th": dim = 2 if isinstance(padding, dict): # dictionary @@ -752,7 +763,8 @@ def create_zeropadding2d(self): def create_zeropadding3d(self): padding = tuple(self.klayer.padding) dim = 1 - warnings.warn("Cannot find dim_ordering from json definition. Using the default instead.") + if "dim_ordering" not in self.config: + warnings.warn("Cannot find dim_ordering from json definition. Using the default instead.") if self.klayer.dim_ordering == "th": dim = 2 model = BLayer.Sequential() @@ -935,10 +947,14 @@ def create_gru(self): self.klayer.go_backwards, rec.add(gru)) def create_batchnormalization(self): - if keras.backend.image_dim_ordering() != "th" or self.klayer.axis != 1: - raise Exception("""For BatchNormalization, we only support th image ordering (i.e. NCHW) """ + - """with axis = 1 for now, but the current order is %s and axis is %s - """ % (keras.backend.image_dim_ordering(), self.klayer.axis)) # noqa + if keras.backend.image_dim_ordering() == "th" and self.klayer.axis != 1: + raise Exception("""For BatchNormalization with th image ordering, we only support """ + + """axis = 1 for now, but the current axis is %s + """ % self.klayer.axis) # noqa + if keras.backend.image_dim_ordering() == "tf" and self.klayer.axis != -1: + raise Exception("""For BatchNormalization with tf image ordering, we only support """ + + """axis = -1 for now, but the current axis is %s + """ % self.klayer.axis) if self.klayer.mode != 0: raise Exception( "Only support mode = 0 for now, but the current mode is: %s", self.klayer.mode) @@ -949,7 +965,8 @@ def create_batchnormalization(self): if self.config["beta_regularizer"]: raise Exception("We don't support beta_regularizer for now") - n_input_channel = int(self.input_shape[self.klayer.axis]) # default is -1 which is channel-last + bigdl_order = self.to_bigdl_2d_ordering(keras.backend.image_dim_ordering()) + n_input_channel = int(self.input_shape[self.klayer.axis]) # init gamma and beta # TODO: replace this with to_bigdl_init in the future @@ -965,6 +982,7 @@ def create_batchnormalization(self): init_bias=beta, init_grad_weight=None, init_grad_bias=None, + data_format=bigdl_order, bigdl_type="float") k_running_mean = keras.backend.eval(self.klayer.running_mean) @@ -1064,8 +1082,8 @@ def create_convolution2d(self): n_output_plane=self.klayer.nb_filter, kernel_w=self.klayer.nb_col, kernel_h=self.klayer.nb_row, - stride_w=self.klayer.subsample[0], - stride_h=self.klayer.subsample[1], + stride_w=self.klayer.subsample[1], + stride_h=self.klayer.subsample[0], pad_w=bpadW, pad_h=bpadH, n_group=1, @@ -1108,7 +1126,7 @@ def create_convolution3d(self): def create_atrousconvolution1d(self): if not self.config["bias"]: - raise Exception("Please set `bias=True` for AtrousConvolution1D") + raise Exception("Only bias=True is supported for AtrousConvolution1D") h = self.input_shape[2] kh = self.config["filter_length"] @@ -1142,7 +1160,7 @@ def create_atrousconvolution2d(self): if self.klayer.dim_ordering != "th": raise Exception("Please use `th` for `dim_ordering`. `%s` is not supported for now." % self.klayer.dim_ordering) if not self.config["bias"]: - raise Exception("Please set `bias=True` for AtrousConvolution2D") + raise Exception("Only bias=True is supported for AtrousConvolution2D") h = self.input_shape[2] w = self.input_shape[3] @@ -1506,11 +1524,21 @@ def create_globalaveragepooling2d(self): seq.add(BLayer.Squeeze(1, num_input_dims=2)) return seq + def create_upsampling1d(self): + return BLayer.UpSampling1D(self.klayer.length) + + def create_upsampling2d(self): + if "dim_ordering" not in self.config: + warnings.warn("Cannot find dim_ordering from json definition. Using the default instead.") + bigdl_order = self.get_bdim_order() + return BLayer.UpSampling2D(self.klayer.size, bigdl_order) + def create_upsampling3d(self): if self.klayer.dim_ordering != "th": raise Exception("Please use th for dim_ordering. %s is not supported for now." % self.klayer.dim_ordering) - warnings.warn("Cannot find dim_ordering from json definition. Using the default instead." - "We only support th for now.") + if "dim_ordering" not in self.config: + warnings.warn("Cannot find dim_ordering from json definition. Using the default instead." + "We only support th for now.") return BLayer.UpSampling3D(self.klayer.size) def create_gaussiannoise(self): @@ -1538,7 +1566,7 @@ def create_maxoutdense(self): if self.config["b_regularizer"]: raise Exception("b_regularizer is not supported for MaxoutDense") if not self.config["bias"]: - raise Exception("Please set bias=True for MaxoutDense") + raise Exception("Only bias=True is supported for MaxoutDense") blayer = BLayer.Maxout(input_size=self.input_shape[1], output_size=self.klayer.output_dim, maxout_number=self.klayer.nb_feature) @@ -1548,12 +1576,47 @@ def create_masking(self): return BLayer.Masking(float(self.klayer.mask_value)) def create_srelu(self): - warnings.warn("Cannot find shared_axes from json definition. Using shared_axes=None instead.") + if "shared_axes" not in self.config: + warnings.warn("Cannot find shared_axes from json definition. Using shared_axes=None instead.") if self.klayer.shared_axes == [None]: return BLayer.SReLU() else: return BLayer.SReLU(self.klayer.shared_axes) + def create_separableconvolution2d(self): + if keras.backend.backend() != 'tensorflow': + raise Exception('Please use tensorflow backend for keras 1.2.2 ' + 'if you want to load SeparableConv2D') + bigdl_order = self.get_bdim_order() + + if bigdl_order == "NCHW": + stack_size = int(self.input_shape[1]) + elif bigdl_order == "NHWC": + stack_size = int(self.input_shape[3]) + + bpadW, bpadH = self.to_bigdl_2d_padding(self.klayer.border_mode) + blayer = BLayer.SpatialSeperableConvolution( + n_input_channel=stack_size, + n_output_channel=self.klayer.nb_filter, + depth_multiplier=self.klayer.depth_multiplier, + kernel_w=self.klayer.nb_col, + kernel_h=self.klayer.nb_row, + stride_w=self.klayer.subsample[1], + stride_h=self.klayer.subsample[0], + pad_w=bpadW, + pad_h=bpadH, + with_bias=self.klayer.bias, + data_format=bigdl_order, + w_regularizer=self.to_bigdl_reg(self.config["depthwise_regularizer"]), + b_regularizer=self.to_bigdl_reg(self.config["b_regularizer"]), + p_regularizer=self.to_bigdl_reg(self.config["pointwise_regularizer"]) + ) + + return self.combo_parameter_layer(blayer, self.config) + + def create_activityregularization(self): + return BLayer.ActivityRegularization(l1=self.klayer.l1, l2=self.klayer.l2) + def combo_parameter_layer(self, blayer, config): blayer.set_name(config["name"]) if hasattr(blayer, "set_init_method"): diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index a8702448bc4..7153a5fda5e 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -3301,9 +3301,9 @@ class SpatialSeperableConvolution(Layer): :param data_format: a string value of "NHWC" or "NCHW" to specify the input data format of this layer. In "NHWC" format data is stored in the order of [batch_size, height, width, channels], in "NCHW" format data is stored in the order of [batch_size, channels, height, width]. - :param wRegularizer: instance of [[Regularizer]](eg. L1 or L2 regularization), applied to the depth weights matrices. - :param bRegularizer: instance of [[Regularizer]]applied to the pointwise bias. - :param pRegularizer: instance of [[Regularizer]]applied to the pointwise weights. + :param w_regularizer: instance of [[Regularizer]](eg. L1 or L2 regularization), applied to the depth weights matrices. + :param b_regularizer: instance of [[Regularizer]]applied to the pointwise bias. + :param p_regularizer: instance of [[Regularizer]]applied to the pointwise weights. >>> conv = SpatialSeperableConvolution(6, 12, 1, 5, 5) creating: createSpatialSeperableConvolution From f33cb70d56c747460cdc6fbe4b6665f591a23a16 Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Fri, 15 Dec 2017 18:36:33 +0800 Subject: [PATCH 453/823] Newly added Keras Layers python side (#2024) * add new layers * update * fix ut --- .../test/bigdl/keras/test_application.py | 1 + python/dllib/test/bigdl/keras/test_layer.py | 85 +++++++++++++++---- .../dllib/test/bigdl/keras/test_load_model.py | 8 ++ 3 files changed, 77 insertions(+), 17 deletions(-) diff --git a/python/dllib/test/bigdl/keras/test_application.py b/python/dllib/test/bigdl/keras/test_application.py index f5214cf1fea..5e1384ae8a4 100644 --- a/python/dllib/test/bigdl/keras/test_application.py +++ b/python/dllib/test/bigdl/keras/test_application.py @@ -39,6 +39,7 @@ def assert_model(self, input_data, kmodel, rtol=1e-5, atol=1e-5): self.assert_allclose(keras_output, bigdl_output, rtol=rtol, atol=atol) def test_lenet(self): + K.set_image_dim_ordering("th") kmodel, input_data, output_data = TestModels.kmodel_seq_lenet_mnist() self.modelTest(input_data, kmodel, dump_weights=True) diff --git a/python/dllib/test/bigdl/keras/test_layer.py b/python/dllib/test/bigdl/keras/test_layer.py index 392bd5a68d0..a5b3b6fea7a 100644 --- a/python/dllib/test/bigdl/keras/test_layer.py +++ b/python/dllib/test/bigdl/keras/test_layer.py @@ -29,6 +29,41 @@ class TestLayer(BigDLTestCase): + def test_relu(self): + input_data = np.random.random_sample([2, 3, 5]) + layer = Activation('relu') + self.modelTestSingleLayer(input_data, layer) + + def test_tanh(self): + input_data = np.random.random_sample([2, 3, 5]) + layer = Activation('tanh') + self.modelTestSingleLayer(input_data, layer) + + def test_sigmoid(self): + input_data = np.random.random_sample([2, 3, 5]) + layer = Activation('sigmoid') + self.modelTestSingleLayer(input_data, layer) + + def test_hard_sigmoid(self): + input_data = np.random.random_sample([2, 3, 5]) + layer = Activation('hard_sigmoid') + self.modelTestSingleLayer(input_data, layer) + + def test_softmax(self): + input_data = np.random.random_sample([5, 6]) + layer = Activation('softmax') + self.modelTestSingleLayer(input_data, layer) + + def test_softplus(self): + input_data = np.random.random_sample([2, 3, 5]) + layer = Activation('softplus') + self.modelTestSingleLayer(input_data, layer) + + def test_softsign(self): + input_data = np.random.random_sample([2, 3, 5]) + layer = Activation('softsign') + self.modelTestSingleLayer(input_data, layer) + def test_dense(self): input_data = np.random.random_sample([1, 10]) layer = Dense(2, init='one', activation="relu", @@ -82,18 +117,16 @@ def test_embedding(self): def test_conv1D(self): input_data = np.random.random_sample([1, 10, 32]) layer = lambda: Convolution1D(64, 3, border_mode='valid', input_shape=(10, 32)) - self.modelTestSingleLayerWithOrdersModes(input_data, layer, - dump_weights=True, dim_orderings=["tf"]) - + self.modelTestSingleLayerWithOrdersModes(input_data, layer, dump_weights=True) layer2 = lambda: Convolution1D(64, 3, border_mode='same', input_shape=(10, 32)) - self.modelTestSingleLayerWithOrdersModes(input_data, layer2, - dump_weights=True, dim_orderings=["tf"]) - + self.modelTestSingleLayerWithOrdersModes(input_data, layer2, dump_weights=True) layer3 = lambda: Convolution1D(64, 3, border_mode='same', activation="relu", input_shape=(10, 32)) - self.modelTestSingleLayerWithOrdersModes(input_data, layer3, - dump_weights=True, dim_orderings=["tf"]) + self.modelTestSingleLayerWithOrdersModes(input_data, layer3, dump_weights=True) + layer4 = lambda: Convolution1D(32, 4, border_mode='same', + bias=False, input_shape=(10, 32)) + self.modelTestSingleLayerWithOrdersModes(input_data, layer4, dump_weights=True) def _load_keras(self, json_path, hdf5_path): with open(json_path, "r") as jp: @@ -105,14 +138,18 @@ def _load_keras(self, json_path, hdf5_path): def test_conv2D(self): input_data = np.random.random_sample([1, 3, 128, 128]) - layer = lambda: Convolution2D(64, 1, 20, input_shape=(3, 128, 128)) - self.modelTestSingleLayerWithOrdersModes(input_data, - layer, - dump_weights=True, rtol=1e-5, atol=1e-5) - # Test if alias works or not - layer = lambda: Conv2D(64, 3, 1, input_shape=(3, 128, 128)) - self.modelTestSingleLayerWithOrdersModes(input_data, - layer, + layer1 = lambda: Convolution2D(64, 1, 20, input_shape=(3, 128, 128)) + self.modelTestSingleLayerWithOrdersModes(input_data, layer1, dump_weights=True) + layer2 = lambda: Convolution2D(64, 1, 20, subsample=(2, 3), + input_shape=(3, 128, 128)) + self.modelTestSingleLayerWithOrdersModes(input_data, layer2, dump_weights=True) + layer3 = lambda: Convolution2D(32, 3, 3, activation='sigmoid', + bias=False, input_shape=(3, 128, 128)) + self.modelTestSingleLayerWithOrdersModes(input_data, layer3, + dump_weights=True) + # # Test if alias works or not + layer4 = lambda: Conv2D(64, 3, 1, input_shape=(3, 128, 128)) + self.modelTestSingleLayerWithOrdersModes(input_data, layer4, dump_weights=True, rtol=1e-5, atol=1e-5) def test_conv3D(self): @@ -597,7 +634,7 @@ def test_wrapper_timedistributed(self): input_data2 = np.random.random_sample([2, 10, 3, 32, 32]) layer2 = TimeDistributed(Convolution2D(64, 3, 3), input_shape=(10, 3, 32, 32)) - self.modelTestSingleLayer(input_data2, layer2, dump_weights=True) + self.modelTestSingleLayer(input_data2, layer2, dump_weights=True, rtol=1e-5, atol=1e-5) def test_wrapper_bidirectional(self): input_data = np.random.random([5, 32, 64]) @@ -620,6 +657,20 @@ def test_wrapper_bidirectional(self): input_shape=(8, 40, 40, 32), merge_mode='ave') self.modelTestSingleLayer(input_data2, layer5, dump_weights=True) + def test_upsampling1d(self): + input_data = np.random.random([2, 5, 8]) + layer1 = UpSampling1D(input_shape=(5, 8)) + self.modelTestSingleLayer(input_data, layer1) + layer2 = UpSampling1D(length=3, input_shape=(5, 8)) + self.modelTestSingleLayer(input_data, layer2) + + def test_upsampling2d(self): + input_data = np.random.random([2, 5, 6, 8]) + layer1 = UpSampling2D(input_shape=(5, 6, 8)) + self.modelTestSingleLayer(input_data, layer1) + layer2 = UpSampling2D(size=(1, 3), input_shape=(5, 6, 8)) + self.modelTestSingleLayer(input_data, layer2) + def test_upsampling3d(self): input_data = np.random.random([2, 5, 12, 12, 12]) layer1 = UpSampling3D(input_shape=(5, 12, 12, 12)) diff --git a/python/dllib/test/bigdl/keras/test_load_model.py b/python/dllib/test/bigdl/keras/test_load_model.py index 8a8156993ea..1d607bd7518 100644 --- a/python/dllib/test/bigdl/keras/test_load_model.py +++ b/python/dllib/test/bigdl/keras/test_load_model.py @@ -25,6 +25,7 @@ np.random.seed(1337) # for reproducibility from test.bigdl.test_utils import BigDLTestCase, TestModels +import keras.backend as K class TestLoadModel(BigDLTestCase): @@ -41,6 +42,7 @@ def __kmodel_load_def_weight_test(self, kmodel, input_data): assert_allclose(boutput, koutput, rtol=1e-5) def test_load_api_with_hdf5(self): + K.set_image_dim_ordering("th") kmodel, input_data, output_data = TestModels.kmodel_graph_1_layer() keras_model_json_path, keras_model_hdf5_path = self._dump_keras(kmodel, dump_weights=True) bmodel = BLayer.Model.load_keras(keras_model_json_path, keras_model_hdf5_path) @@ -48,23 +50,28 @@ def test_load_api_with_hdf5(self): bmodel.forward(input_data)) def test_load_api_no_hdf5(self): + K.set_image_dim_ordering("th") kmodel, input_data, output_data = TestModels.kmodel_graph_1_layer() keras_model_json_path, keras_model_hdf5_path = self._dump_keras(kmodel, dump_weights=True) bmodel = BLayer.Model.load_keras(keras_model_json_path) def test_load_def_weights_graph_1_layer(self): + K.set_image_dim_ordering("th") kmodel, input_data, output_data = TestModels.kmodel_graph_1_layer() self.__kmodel_load_def_weight_test(kmodel, input_data) def test_load_def_weights_graph_activation(self): + K.set_image_dim_ordering("th") kmodel, input_data, output_data = TestModels.kmodel_graph_activation_is_layer() self.__kmodel_load_def_weight_test(kmodel, input_data) def test_load_def_weights_kmodel_seq_lenet_mnist(self): + K.set_image_dim_ordering("th") kmodel, input_data, output_data = TestModels.kmodel_seq_lenet_mnist() self.__kmodel_load_def_weight_test(kmodel, input_data) def test_load_definition(self): + K.set_image_dim_ordering("th") kmodel, input_data, output_data = TestModels.kmodel_seq_lenet_mnist() keras_model_json_path, keras_model_hdf5_path = self._dump_keras(kmodel, dump_weights=True) bmodel = DefinitionLoader.from_json_path(keras_model_json_path) @@ -72,6 +79,7 @@ def test_load_definition(self): self.assert_allclose(bmodel.forward(input_data), kmodel.predict(input_data)) def test_load_weights(self): + K.set_image_dim_ordering("th") kmodel, input_data, output_data = TestModels.kmodel_graph_1_layer() keras_model_json_path, keras_model_hdf5_path = self._dump_keras(kmodel, dump_weights=True) bmodel = DefinitionLoader.from_json_path(keras_model_json_path) From 927c8a00b5a532caac84db24f35a52dcc9f4285a Mon Sep 17 00:00:00 2001 From: Yao Zhang Date: Fri, 15 Dec 2017 19:26:09 +0800 Subject: [PATCH 454/823] Add keras SpatialDropout1D, SpatialDropout2D, SpatialDropout3D (#1898) * add comments * meet review * add python code * add documentation * add serialzation test --- python/dllib/src/bigdl/dllib/nn/layer.py | 60 ++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 7153a5fda5e..2091c915344 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -1763,7 +1763,67 @@ def __init__(self, alpha, beta, k, data_format) +class SpatialDropout3D(Layer): + ''' + This version performs the same function as Dropout, however it drops + entire 3D feature maps instead of individual elements. If adjacent voxels + within feature maps are strongly correlated (as is normally the case in + early convolution layers) then regular dropout will not regularize the + activations and will otherwise just result in an effective learning rate + decrease. In this case, SpatialDropout3D will help promote independence + between feature maps and should be used instead. + + :param initP the probability p + :param format 'NCHW' or 'NHWC'. + In 'NCHW' mode, the channels dimension (the depth) + is at index 1, in 'NHWC' mode is it at index 4. + ''' + def __init__(self, + init_p=0.5, + data_format="NCHW", + bigdl_type="float"): + super(Dropout, self).__init__(None, bigdl_type, data_format, + init_p) + +class SpatialDropout2D(Layer): + ''' + This version performs the same function as Dropout, however it drops + entire 2D feature maps instead of individual elements. If adjacent pixels + within feature maps are strongly correlated (as is normally the case in + early convolution layers) then regular dropout will not regularize the + activations and will otherwise just result in an effective learning rate + decrease. In this case, SpatialDropout2D will help promote independence + between feature maps and should be used instead. + + :param initP the probability p + :param format 'NCHW' or 'NHWC'. + In 'NCHW' mode, the channels dimension (the depth) + is at index 1, in 'NHWC' mode is it at index 4. + ''' + def __init__(self, + init_p=0.5, + data_format="NCHW", + bigdl_type="float"): + super(Dropout, self).__init__(None, bigdl_type, data_format, + init_p) + +class SpatialDropout1D(Layer): + ''' + This version performs the same function as Dropout, however it drops + entire 1D feature maps instead of individual elements. If adjacent frames + within feature maps are strongly correlated (as is normally the case in + early convolution layers) then regular dropout will not regularize the + activations and will otherwise just result in an effective learning rate + decrease. In this case, SpatialDropout1D will help promote independence + between feature maps and should be used instead. + :param initP the probability p + ''' + def __init__(self, + init_p=0.5, + bigdl_type="float"): + super(Dropout, self).__init__(None, bigdl_type, + init_p) class Dropout(Layer): From b170bd851312564bddc028d1b0f78887fdb789d6 Mon Sep 17 00:00:00 2001 From: Yao Zhang Date: Fri, 15 Dec 2017 19:26:09 +0800 Subject: [PATCH 455/823] Add keras SpatialDropout1D, SpatialDropout2D, SpatialDropout3D (#1898) * add comments * meet review * add python code * add documentation * add serialzation test --- python/dllib/src/bigdl/dllib/nn/layer.py | 60 ++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 7153a5fda5e..2091c915344 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -1763,7 +1763,67 @@ def __init__(self, alpha, beta, k, data_format) +class SpatialDropout3D(Layer): + ''' + This version performs the same function as Dropout, however it drops + entire 3D feature maps instead of individual elements. If adjacent voxels + within feature maps are strongly correlated (as is normally the case in + early convolution layers) then regular dropout will not regularize the + activations and will otherwise just result in an effective learning rate + decrease. In this case, SpatialDropout3D will help promote independence + between feature maps and should be used instead. + + :param initP the probability p + :param format 'NCHW' or 'NHWC'. + In 'NCHW' mode, the channels dimension (the depth) + is at index 1, in 'NHWC' mode is it at index 4. + ''' + def __init__(self, + init_p=0.5, + data_format="NCHW", + bigdl_type="float"): + super(Dropout, self).__init__(None, bigdl_type, data_format, + init_p) + +class SpatialDropout2D(Layer): + ''' + This version performs the same function as Dropout, however it drops + entire 2D feature maps instead of individual elements. If adjacent pixels + within feature maps are strongly correlated (as is normally the case in + early convolution layers) then regular dropout will not regularize the + activations and will otherwise just result in an effective learning rate + decrease. In this case, SpatialDropout2D will help promote independence + between feature maps and should be used instead. + + :param initP the probability p + :param format 'NCHW' or 'NHWC'. + In 'NCHW' mode, the channels dimension (the depth) + is at index 1, in 'NHWC' mode is it at index 4. + ''' + def __init__(self, + init_p=0.5, + data_format="NCHW", + bigdl_type="float"): + super(Dropout, self).__init__(None, bigdl_type, data_format, + init_p) + +class SpatialDropout1D(Layer): + ''' + This version performs the same function as Dropout, however it drops + entire 1D feature maps instead of individual elements. If adjacent frames + within feature maps are strongly correlated (as is normally the case in + early convolution layers) then regular dropout will not regularize the + activations and will otherwise just result in an effective learning rate + decrease. In this case, SpatialDropout1D will help promote independence + between feature maps and should be used instead. + :param initP the probability p + ''' + def __init__(self, + init_p=0.5, + bigdl_type="float"): + super(Dropout, self).__init__(None, bigdl_type, + init_p) class Dropout(Layer): From 7fb4133007f227e5ba14c5533e717d343e3a39b6 Mon Sep 17 00:00:00 2001 From: dding3 Date: Fri, 15 Dec 2017 18:03:58 -0500 Subject: [PATCH 456/823] Support gradient clipping (#2001) support constant clip and l2norm clipping --- .../dllib/src/bigdl/dllib/optim/optimizer.py | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/optim/optimizer.py b/python/dllib/src/bigdl/dllib/optim/optimizer.py index 8917b73c233..c0398206998 100644 --- a/python/dllib/src/bigdl/dllib/optim/optimizer.py +++ b/python/dllib/src/bigdl/dllib/optim/optimizer.py @@ -690,6 +690,31 @@ def set_checkpoint(self, checkpoint_trigger, callBigDlFunc(self.bigdl_type, "setCheckPoint", self.value, checkpoint_trigger, checkpoint_path, isOverWrite) + def set_gradclip_const(self, min_value, max_value): + """ + Configure constant clipping settings. + + + :param min_value: the minimum value to clip by + :param max_value: the maxmimum value to clip by + """ + callBigDlFunc(self.bigdl_type, "setConstantClip", self.value, min_value, max_value) + + def set_gradclip_l2norm(self, clip_norm): + """ + Configure L2 norm clipping settings. + + + :param clip_norm: gradient L2-Norm threshold + """ + callBigDlFunc(self.bigdl_type, "setL2NormClip", self.value, clip_norm) + + def disable_gradclip(self): + """ + disable clipping. + """ + callBigDlFunc(self.bigdl_type, "disableClip", self.value) + # return a module def optimize(self): """ From c95c018f94078261bb40627ee0ef64ab2519dd96 Mon Sep 17 00:00:00 2001 From: dding3 Date: Fri, 15 Dec 2017 18:03:58 -0500 Subject: [PATCH 457/823] Support gradient clipping (#2001) support constant clip and l2norm clipping --- .../dllib/src/bigdl/dllib/optim/optimizer.py | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/optim/optimizer.py b/python/dllib/src/bigdl/dllib/optim/optimizer.py index 8917b73c233..c0398206998 100644 --- a/python/dllib/src/bigdl/dllib/optim/optimizer.py +++ b/python/dllib/src/bigdl/dllib/optim/optimizer.py @@ -690,6 +690,31 @@ def set_checkpoint(self, checkpoint_trigger, callBigDlFunc(self.bigdl_type, "setCheckPoint", self.value, checkpoint_trigger, checkpoint_path, isOverWrite) + def set_gradclip_const(self, min_value, max_value): + """ + Configure constant clipping settings. + + + :param min_value: the minimum value to clip by + :param max_value: the maxmimum value to clip by + """ + callBigDlFunc(self.bigdl_type, "setConstantClip", self.value, min_value, max_value) + + def set_gradclip_l2norm(self, clip_norm): + """ + Configure L2 norm clipping settings. + + + :param clip_norm: gradient L2-Norm threshold + """ + callBigDlFunc(self.bigdl_type, "setL2NormClip", self.value, clip_norm) + + def disable_gradclip(self): + """ + disable clipping. + """ + callBigDlFunc(self.bigdl_type, "disableClip", self.value) + # return a module def optimize(self): """ From a1e172287de480582d3be60d04b6b2e2bbee0fc5 Mon Sep 17 00:00:00 2001 From: Xin Qiu Date: Mon, 18 Dec 2017 09:35:05 +0800 Subject: [PATCH 458/823] Lookup table for multivalue (#2016) Layer LookupTableSparse --- python/dllib/src/bigdl/dllib/nn/layer.py | 50 ++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 2091c915344..90ab6e3eef5 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -2862,6 +2862,56 @@ def set_init_method(self, weight_init_method = None, bias_init_method = None): return self +class LookupTableSparse(Layer): + + ''' + LookupTable for multi-values. + Also called embedding_lookup_sparse in TensorFlow. + + The input of LookupTableSparse should be a 2D SparseTensor or two 2D SparseTensors. + If the input is a SparseTensor, the values are positive integer ids, + values in each row of this SparseTensor will be turned into a dense vector. + If the input is two SparseTensors, the first tensor should be the integer ids, just + like the SparseTensor input. And the second tensor is the corresponding + weights of the integer ids. + + :param wRegularizer: instance of [[Regularizer]](eg. L1 or L2 regularization), applied to the input weights matrices. + + >>> lookupTableSparse = LookupTableSparse(20, 5, "mean", 2, L1Regularizer(0.5)) + creating: createL1Regularizer + creating: createLookupTableSparse + >>> indices = np.array([[0, 0, 1, 2], [0, 1, 0, 3]]) + >>> values = np.array([2, 4, 1, 2]) + >>> weightValues = np.array([2, 0.5, 1, 3]) + >>> input = JTensor.sparse(values, indices, np.array([3, 4])) + >>> weight = JTensor.sparse(weightValues, indices, np.array([3, 4])) + >>> layer1 = LookupTableSparse(10, 4, "mean") + creating: createLookupTableSparse + >>> layer1.set_weights(np.arange(1, 41, 1).reshape(10, 4)) # set weight to 1 to 40 + >>> layer1.forward([input, weight]) + array([[ 6.5999999 , 7.60000038, 8.60000038, 9.60000038], + [ 1. , 2. , 3. , 4. ], + [ 5. , 6. , 7. , 8. ]], dtype=float32) + ''' + + def __init__(self, + n_index, + n_output, + combiner="sum", + max_norm=-1.0, + wRegularizer=None, + bigdl_type="float"): + super(LookupTableSparse, self).__init__(None, bigdl_type, + n_index, + n_output, + combiner, + max_norm + 0.0, + wRegularizer) + def set_init_method(self, weight_init_method = None, bias_init_method = None): + callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, + weight_init_method, bias_init_method) + return self + class MM(Layer): ''' From b686cc2ea8ec135add83736ddaacb3e3cb658ac5 Mon Sep 17 00:00:00 2001 From: Xin Qiu Date: Mon, 18 Dec 2017 09:35:05 +0800 Subject: [PATCH 459/823] Lookup table for multivalue (#2016) Layer LookupTableSparse --- python/dllib/src/bigdl/dllib/nn/layer.py | 50 ++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 2091c915344..90ab6e3eef5 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -2862,6 +2862,56 @@ def set_init_method(self, weight_init_method = None, bias_init_method = None): return self +class LookupTableSparse(Layer): + + ''' + LookupTable for multi-values. + Also called embedding_lookup_sparse in TensorFlow. + + The input of LookupTableSparse should be a 2D SparseTensor or two 2D SparseTensors. + If the input is a SparseTensor, the values are positive integer ids, + values in each row of this SparseTensor will be turned into a dense vector. + If the input is two SparseTensors, the first tensor should be the integer ids, just + like the SparseTensor input. And the second tensor is the corresponding + weights of the integer ids. + + :param wRegularizer: instance of [[Regularizer]](eg. L1 or L2 regularization), applied to the input weights matrices. + + >>> lookupTableSparse = LookupTableSparse(20, 5, "mean", 2, L1Regularizer(0.5)) + creating: createL1Regularizer + creating: createLookupTableSparse + >>> indices = np.array([[0, 0, 1, 2], [0, 1, 0, 3]]) + >>> values = np.array([2, 4, 1, 2]) + >>> weightValues = np.array([2, 0.5, 1, 3]) + >>> input = JTensor.sparse(values, indices, np.array([3, 4])) + >>> weight = JTensor.sparse(weightValues, indices, np.array([3, 4])) + >>> layer1 = LookupTableSparse(10, 4, "mean") + creating: createLookupTableSparse + >>> layer1.set_weights(np.arange(1, 41, 1).reshape(10, 4)) # set weight to 1 to 40 + >>> layer1.forward([input, weight]) + array([[ 6.5999999 , 7.60000038, 8.60000038, 9.60000038], + [ 1. , 2. , 3. , 4. ], + [ 5. , 6. , 7. , 8. ]], dtype=float32) + ''' + + def __init__(self, + n_index, + n_output, + combiner="sum", + max_norm=-1.0, + wRegularizer=None, + bigdl_type="float"): + super(LookupTableSparse, self).__init__(None, bigdl_type, + n_index, + n_output, + combiner, + max_norm + 0.0, + wRegularizer) + def set_init_method(self, weight_init_method = None, bias_init_method = None): + callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, + weight_init_method, bias_init_method) + return self + class MM(Layer): ''' From b1500f6d067c90ef4d25e958655768e5bf0cf28e Mon Sep 17 00:00:00 2001 From: Yao Zhang Date: Mon, 18 Dec 2017 12:16:35 +0800 Subject: [PATCH 460/823] Add locallyconnected2d layer (#2037) * finish locallyconnected2d * add locallyconnected2d * add serializer test * fix python test failed * meet code review * change spaticalConvolutionSpec * remove a test --- python/dllib/src/bigdl/dllib/nn/layer.py | 83 ++++++++++++++++++++++++ 1 file changed, 83 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 90ab6e3eef5..6037d452e4e 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -1142,6 +1142,89 @@ def __init__(self, gate_output, with_graph) +class LocallyConnected2D(Layer): + + ''' + The LocallyConnected2D layer works similarly to the [[SpatialConvolution]] layer, + except that weights are unshared, that is, a different set of filters + is applied at each different patch of the input. + + :param n_input_plane The number of expected input planes in the image given into forward() + :param input_width The expected width of input + :param input_height The expected height of input + :param n_output_plane The number of output planes the convolution layer will produce. + :param kernel_w The kernel width of the convolution + :param kernel_h The kernel height of the convolution + :param stride_w The step of the convolution in the width dimension. + :param stride_h The step of the convolution in the height dimension + :param pad_w The additional zeros added per width to the input planes. + :param pad_h The additional zeros added per height to the input planes. + :param propagate_back Propagate gradient back + :param wRegularizer: instance of [[Regularizer]](eg. L1 or L2 regularization), applied to the input weights matrices. + :param bRegularizer: instance of [[Regularizer]]applied to the bias. + :param init_weight: the optional initial value for the weight + :param init_bias: the optional initial value for the bias + :param init_grad_weight: the optional initial value for the grad_weight + :param init_grad_bias: the optional initial value for the grad_bias + :param with_bias: the optional initial value for if need bias + :param data_format: a string value of "NHWC" or "NCHW" to specify the input data format of this layer. In "NHWC" format + data is stored in the order of [batch_size, height, width, channels], in "NCHW" format data is stored + in the order of [batch_size, channels, height, width]. + + >>> locallyConnected2D = LocallyConnected2D(6, 2, 4, 12, 5, 5) + creating: createLocallyConnected2D + >>> locallyConnected2D.setWRegularizer(L1Regularizer(0.5)) + creating: createL1Regularizer + >>> locallyConnected2D.setBRegularizer(L1Regularizer(0.5)) + creating: createL1Regularizer + ''' + + def __init__(self, + n_input_plane, + input_width, + input_height, + n_output_plane, + kernel_w, + kernel_h, + stride_w=1, + stride_h=1, + pad_w=0, + pad_h=0, + propagate_back=True, + wRegularizer=None, + bRegularizer=None, + init_weight=None, + init_bias=None, + init_grad_weight=None, + init_grad_bias=None, + with_bias=True, + data_format="NCHW", + bigdl_type="float"): + super(LocallyConnected2D, self).__init__(None, bigdl_type, + n_input_plane, + input_width, + input_height, + n_output_plane, + kernel_w, + kernel_h, + stride_w, + stride_h, + pad_w, + pad_h, + propagate_back, + wRegularizer, + bRegularizer, + JTensor.from_ndarray(init_weight), + JTensor.from_ndarray(init_bias), + JTensor.from_ndarray(init_grad_weight), + JTensor.from_ndarray(init_grad_bias), + with_bias, + data_format) + def set_init_method(self, weight_init_method = None, bias_init_method = None): + callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, + weight_init_method, bias_init_method) + return self + class SpatialConvolution(Layer): ''' From 035547b10d150d57a2e9f745471565ed65cff6dc Mon Sep 17 00:00:00 2001 From: Yao Zhang Date: Mon, 18 Dec 2017 12:16:35 +0800 Subject: [PATCH 461/823] Add locallyconnected2d layer (#2037) * finish locallyconnected2d * add locallyconnected2d * add serializer test * fix python test failed * meet code review * change spaticalConvolutionSpec * remove a test --- python/dllib/src/bigdl/dllib/nn/layer.py | 83 ++++++++++++++++++++++++ 1 file changed, 83 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 90ab6e3eef5..6037d452e4e 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -1142,6 +1142,89 @@ def __init__(self, gate_output, with_graph) +class LocallyConnected2D(Layer): + + ''' + The LocallyConnected2D layer works similarly to the [[SpatialConvolution]] layer, + except that weights are unshared, that is, a different set of filters + is applied at each different patch of the input. + + :param n_input_plane The number of expected input planes in the image given into forward() + :param input_width The expected width of input + :param input_height The expected height of input + :param n_output_plane The number of output planes the convolution layer will produce. + :param kernel_w The kernel width of the convolution + :param kernel_h The kernel height of the convolution + :param stride_w The step of the convolution in the width dimension. + :param stride_h The step of the convolution in the height dimension + :param pad_w The additional zeros added per width to the input planes. + :param pad_h The additional zeros added per height to the input planes. + :param propagate_back Propagate gradient back + :param wRegularizer: instance of [[Regularizer]](eg. L1 or L2 regularization), applied to the input weights matrices. + :param bRegularizer: instance of [[Regularizer]]applied to the bias. + :param init_weight: the optional initial value for the weight + :param init_bias: the optional initial value for the bias + :param init_grad_weight: the optional initial value for the grad_weight + :param init_grad_bias: the optional initial value for the grad_bias + :param with_bias: the optional initial value for if need bias + :param data_format: a string value of "NHWC" or "NCHW" to specify the input data format of this layer. In "NHWC" format + data is stored in the order of [batch_size, height, width, channels], in "NCHW" format data is stored + in the order of [batch_size, channels, height, width]. + + >>> locallyConnected2D = LocallyConnected2D(6, 2, 4, 12, 5, 5) + creating: createLocallyConnected2D + >>> locallyConnected2D.setWRegularizer(L1Regularizer(0.5)) + creating: createL1Regularizer + >>> locallyConnected2D.setBRegularizer(L1Regularizer(0.5)) + creating: createL1Regularizer + ''' + + def __init__(self, + n_input_plane, + input_width, + input_height, + n_output_plane, + kernel_w, + kernel_h, + stride_w=1, + stride_h=1, + pad_w=0, + pad_h=0, + propagate_back=True, + wRegularizer=None, + bRegularizer=None, + init_weight=None, + init_bias=None, + init_grad_weight=None, + init_grad_bias=None, + with_bias=True, + data_format="NCHW", + bigdl_type="float"): + super(LocallyConnected2D, self).__init__(None, bigdl_type, + n_input_plane, + input_width, + input_height, + n_output_plane, + kernel_w, + kernel_h, + stride_w, + stride_h, + pad_w, + pad_h, + propagate_back, + wRegularizer, + bRegularizer, + JTensor.from_ndarray(init_weight), + JTensor.from_ndarray(init_bias), + JTensor.from_ndarray(init_grad_weight), + JTensor.from_ndarray(init_grad_bias), + with_bias, + data_format) + def set_init_method(self, weight_init_method = None, bias_init_method = None): + callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, + weight_init_method, bias_init_method) + return self + class SpatialConvolution(Layer): ''' From fea873a5ab22a4f4c8dc0ee002a7d96dae1b1efd Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Mon, 18 Dec 2017 15:22:10 +0800 Subject: [PATCH 462/823] Add remaining keras layer mappings (#2043) * add cropping2d3d * add spatialdropout * refactor 3d dim ordering * add locallyconnected2d * fix softmax * clean --- .../dllib/src/bigdl/dllib/keras/converter.py | 101 +++++++++++++++++- python/dllib/src/bigdl/dllib/nn/layer.py | 21 ++-- 2 files changed, 111 insertions(+), 11 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/keras/converter.py b/python/dllib/src/bigdl/dllib/keras/converter.py index f1cb4990d18..9b9a1de378f 100644 --- a/python/dllib/src/bigdl/dllib/keras/converter.py +++ b/python/dllib/src/bigdl/dllib/keras/converter.py @@ -273,6 +273,14 @@ def convert_separableconvolution2d(klayer, weights): return [weights[0], weights[1], np.zeros(bias, )] return weights + @staticmethod + def convert_locallyconnected2d(klayer, weights): + bweights1 = np.transpose(weights[0], (0, 2, 1)) + if len(weights) == 1: # if without bias + return [bweights1] + bweights2 = weights[1].reshape(weights[1].shape[0]*weights[1].shape[1], weights[1].shape[2]) + return[bweights1, bweights2] + class DefinitionLoader: @@ -545,7 +553,16 @@ def create_embedding(self): return bseq def create_activation(self): - return get_activation_by_name(self.config["activation"], self.klayer.name) + blayer = get_activation_by_name(self.config["activation"], self.klayer.name) + + # SoftMax is different between Keras and BigDL for 3D inputs + if self.config["activation"] == "softmax" and len(self.input_shape) == 3: + model = BLayer.Sequential() + model.add(BLayer.Transpose([(1, 3)])) + model.add(blayer) + model.add(BLayer.Transpose([(1, 3)])) + return model + return blayer def create_dropout(self): return BLayer.Dropout(self.klayer.p) @@ -816,6 +833,21 @@ def create_cropping1d(self): cropping = tuple(self.klayer.cropping) return BLayer.SpatialZeroPadding(0, 0, -cropping[0], -cropping[1]) + def create_cropping2d(self): + bigdl_order = self.get_bdim_order() + blayer = BLayer.Cropping2D(heightCrop=self.klayer.cropping[0], + widthCrop=self.klayer.cropping[1], + data_format=bigdl_order) + return blayer + + def create_cropping3d(self): + bigdl_order = self.get_bdim_order("3D") + blayer = BLayer.Cropping3D(dim1Crop=self.klayer.cropping[0], + dim2Crop=self.klayer.cropping[1], + dim3Crop=self.klayer.cropping[2], + data_format=bigdl_order) + return blayer + def __check_recurrent_parameters(self, klayer): if klayer.stateful: raise Exception("Only stateful=False for recurrent layers is supported for now") @@ -991,12 +1023,14 @@ def create_batchnormalization(self): blayer.set_running_std(k_running_std) return blayer - def get_bdim_order(self): # get bigdl dim_ordering from keras dim_ordering + def get_bdim_order(self, dim="2D"): # get bigdl dim_ordering from keras dim_ordering if "dim_ordering" in self.config: order = self.config["dim_ordering"] else: warnings.warn("Cannot find dim_ordering from json definition. Using the default instead.") order = keras.backend.image_dim_ordering() + if dim == "3D": + return self.to_bigdl_3d_ordering(order) return self.to_bigdl_2d_ordering(order) def to_bigdl_2d_ordering(self, order): @@ -1007,6 +1041,14 @@ def to_bigdl_2d_ordering(self, order): else: raise Exception("Unsupported dim_ordering: %s" % order) + def to_bigdl_3d_ordering(self, order): + if order == "tf": + return "channel_last" + elif order == "th": + return "channel_first" + else: + raise Exception("Unsupported dim_ordering: %s" % order) + def to_bigdl_3d_padding(self, border_mode): if border_mode == "valid": return 0, 0, 0 @@ -1528,10 +1570,9 @@ def create_upsampling1d(self): return BLayer.UpSampling1D(self.klayer.length) def create_upsampling2d(self): - if "dim_ordering" not in self.config: - warnings.warn("Cannot find dim_ordering from json definition. Using the default instead.") bigdl_order = self.get_bdim_order() - return BLayer.UpSampling2D(self.klayer.size, bigdl_order) + return BLayer.UpSampling2D(size=self.klayer.size, + data_format=bigdl_order) def create_upsampling3d(self): if self.klayer.dim_ordering != "th": @@ -1617,6 +1658,56 @@ def create_separableconvolution2d(self): def create_activityregularization(self): return BLayer.ActivityRegularization(l1=self.klayer.l1, l2=self.klayer.l2) + def create_spatialdropout1d(self): + return BLayer.SpatialDropout1D(init_p=float(self.klayer.p)) + + def create_spatialdropout2d(self): + bigdl_order = self.get_bdim_order() + blayer = BLayer.SpatialDropout2D(init_p=float(self.klayer.p), + data_format=bigdl_order) + return blayer + + def create_spatialdropout3d(self): + bigdl_order = self.get_bdim_order() + blayer = BLayer.SpatialDropout3D(init_p=float(self.klayer.p), + data_format=bigdl_order) + return blayer + + def create_locallyconnected2d(self): + bigdl_order = self.get_bdim_order() + + if bigdl_order == "NCHW": + stack_size = int(self.input_shape[1]) + input_width = int(self.input_shape[3]) + input_height = int(self.input_shape[2]) + elif bigdl_order == "NHWC": + stack_size = int(self.input_shape[3]) + input_width = int(self.input_shape[2]) + input_height = int(self.input_shape[1]) + + bpadW, bpadH = self.to_bigdl_2d_padding(self.klayer.border_mode) + blayer = BLayer.LocallyConnected2D(n_input_plane=stack_size, + input_width=input_width, + input_height=input_height, + n_output_plane=self.klayer.nb_filter, + kernel_w=self.klayer.nb_col, + kernel_h=self.klayer.nb_row, + stride_w=self.klayer.subsample[1], + stride_h=self.klayer.subsample[0], + pad_w=bpadW, + pad_h=bpadH, + wRegularizer=self.to_bigdl_reg(self.config["W_regularizer"]), + bRegularizer=self.to_bigdl_reg(self.config["b_regularizer"]), + with_bias=self.klayer.bias, + data_format=bigdl_order) + + if self.config["activation"] != "linear": + activation = get_activation_by_name(self.config["activation"], + "%s_%s" % (self.config["name"], self.config["activation"])) + return self.fuse(blayer, activation) + else: + return blayer + def combo_parameter_layer(self, blayer, config): blayer.set_name(config["name"]) if hasattr(blayer, "set_init_method"): diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 6037d452e4e..8f6c838277f 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -1860,13 +1860,16 @@ class SpatialDropout3D(Layer): :param format 'NCHW' or 'NHWC'. In 'NCHW' mode, the channels dimension (the depth) is at index 1, in 'NHWC' mode is it at index 4. + + >>> dropout = SpatialDropout3D(0.5, "NHWC") + creating: createSpatialDropout3D ''' def __init__(self, init_p=0.5, data_format="NCHW", bigdl_type="float"): - super(Dropout, self).__init__(None, bigdl_type, data_format, - init_p) + super(SpatialDropout3D, self).__init__(None, bigdl_type, + init_p, data_format) class SpatialDropout2D(Layer): ''' @@ -1882,13 +1885,16 @@ class SpatialDropout2D(Layer): :param format 'NCHW' or 'NHWC'. In 'NCHW' mode, the channels dimension (the depth) is at index 1, in 'NHWC' mode is it at index 4. + + >>> dropout = SpatialDropout2D(0.4, "NHWC") + creating: createSpatialDropout2D ''' def __init__(self, init_p=0.5, data_format="NCHW", bigdl_type="float"): - super(Dropout, self).__init__(None, bigdl_type, data_format, - init_p) + super(SpatialDropout2D, self).__init__(None, bigdl_type, + init_p, data_format) class SpatialDropout1D(Layer): ''' @@ -1901,12 +1907,15 @@ class SpatialDropout1D(Layer): between feature maps and should be used instead. :param initP the probability p + + >>> dropout = SpatialDropout1D(0.4) + creating: createSpatialDropout1D ''' def __init__(self, init_p=0.5, bigdl_type="float"): - super(Dropout, self).__init__(None, bigdl_type, - init_p) + super(SpatialDropout1D, self).__init__(None, bigdl_type, + init_p) class Dropout(Layer): From 6bae53c8641400f053565cbf6bc94ff47d5b88dd Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Mon, 18 Dec 2017 15:22:10 +0800 Subject: [PATCH 463/823] Add remaining keras layer mappings (#2043) * add cropping2d3d * add spatialdropout * refactor 3d dim ordering * add locallyconnected2d * fix softmax * clean --- .../dllib/src/bigdl/dllib/keras/converter.py | 101 +++++++++++++++++- python/dllib/src/bigdl/dllib/nn/layer.py | 21 ++-- 2 files changed, 111 insertions(+), 11 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/keras/converter.py b/python/dllib/src/bigdl/dllib/keras/converter.py index f1cb4990d18..9b9a1de378f 100644 --- a/python/dllib/src/bigdl/dllib/keras/converter.py +++ b/python/dllib/src/bigdl/dllib/keras/converter.py @@ -273,6 +273,14 @@ def convert_separableconvolution2d(klayer, weights): return [weights[0], weights[1], np.zeros(bias, )] return weights + @staticmethod + def convert_locallyconnected2d(klayer, weights): + bweights1 = np.transpose(weights[0], (0, 2, 1)) + if len(weights) == 1: # if without bias + return [bweights1] + bweights2 = weights[1].reshape(weights[1].shape[0]*weights[1].shape[1], weights[1].shape[2]) + return[bweights1, bweights2] + class DefinitionLoader: @@ -545,7 +553,16 @@ def create_embedding(self): return bseq def create_activation(self): - return get_activation_by_name(self.config["activation"], self.klayer.name) + blayer = get_activation_by_name(self.config["activation"], self.klayer.name) + + # SoftMax is different between Keras and BigDL for 3D inputs + if self.config["activation"] == "softmax" and len(self.input_shape) == 3: + model = BLayer.Sequential() + model.add(BLayer.Transpose([(1, 3)])) + model.add(blayer) + model.add(BLayer.Transpose([(1, 3)])) + return model + return blayer def create_dropout(self): return BLayer.Dropout(self.klayer.p) @@ -816,6 +833,21 @@ def create_cropping1d(self): cropping = tuple(self.klayer.cropping) return BLayer.SpatialZeroPadding(0, 0, -cropping[0], -cropping[1]) + def create_cropping2d(self): + bigdl_order = self.get_bdim_order() + blayer = BLayer.Cropping2D(heightCrop=self.klayer.cropping[0], + widthCrop=self.klayer.cropping[1], + data_format=bigdl_order) + return blayer + + def create_cropping3d(self): + bigdl_order = self.get_bdim_order("3D") + blayer = BLayer.Cropping3D(dim1Crop=self.klayer.cropping[0], + dim2Crop=self.klayer.cropping[1], + dim3Crop=self.klayer.cropping[2], + data_format=bigdl_order) + return blayer + def __check_recurrent_parameters(self, klayer): if klayer.stateful: raise Exception("Only stateful=False for recurrent layers is supported for now") @@ -991,12 +1023,14 @@ def create_batchnormalization(self): blayer.set_running_std(k_running_std) return blayer - def get_bdim_order(self): # get bigdl dim_ordering from keras dim_ordering + def get_bdim_order(self, dim="2D"): # get bigdl dim_ordering from keras dim_ordering if "dim_ordering" in self.config: order = self.config["dim_ordering"] else: warnings.warn("Cannot find dim_ordering from json definition. Using the default instead.") order = keras.backend.image_dim_ordering() + if dim == "3D": + return self.to_bigdl_3d_ordering(order) return self.to_bigdl_2d_ordering(order) def to_bigdl_2d_ordering(self, order): @@ -1007,6 +1041,14 @@ def to_bigdl_2d_ordering(self, order): else: raise Exception("Unsupported dim_ordering: %s" % order) + def to_bigdl_3d_ordering(self, order): + if order == "tf": + return "channel_last" + elif order == "th": + return "channel_first" + else: + raise Exception("Unsupported dim_ordering: %s" % order) + def to_bigdl_3d_padding(self, border_mode): if border_mode == "valid": return 0, 0, 0 @@ -1528,10 +1570,9 @@ def create_upsampling1d(self): return BLayer.UpSampling1D(self.klayer.length) def create_upsampling2d(self): - if "dim_ordering" not in self.config: - warnings.warn("Cannot find dim_ordering from json definition. Using the default instead.") bigdl_order = self.get_bdim_order() - return BLayer.UpSampling2D(self.klayer.size, bigdl_order) + return BLayer.UpSampling2D(size=self.klayer.size, + data_format=bigdl_order) def create_upsampling3d(self): if self.klayer.dim_ordering != "th": @@ -1617,6 +1658,56 @@ def create_separableconvolution2d(self): def create_activityregularization(self): return BLayer.ActivityRegularization(l1=self.klayer.l1, l2=self.klayer.l2) + def create_spatialdropout1d(self): + return BLayer.SpatialDropout1D(init_p=float(self.klayer.p)) + + def create_spatialdropout2d(self): + bigdl_order = self.get_bdim_order() + blayer = BLayer.SpatialDropout2D(init_p=float(self.klayer.p), + data_format=bigdl_order) + return blayer + + def create_spatialdropout3d(self): + bigdl_order = self.get_bdim_order() + blayer = BLayer.SpatialDropout3D(init_p=float(self.klayer.p), + data_format=bigdl_order) + return blayer + + def create_locallyconnected2d(self): + bigdl_order = self.get_bdim_order() + + if bigdl_order == "NCHW": + stack_size = int(self.input_shape[1]) + input_width = int(self.input_shape[3]) + input_height = int(self.input_shape[2]) + elif bigdl_order == "NHWC": + stack_size = int(self.input_shape[3]) + input_width = int(self.input_shape[2]) + input_height = int(self.input_shape[1]) + + bpadW, bpadH = self.to_bigdl_2d_padding(self.klayer.border_mode) + blayer = BLayer.LocallyConnected2D(n_input_plane=stack_size, + input_width=input_width, + input_height=input_height, + n_output_plane=self.klayer.nb_filter, + kernel_w=self.klayer.nb_col, + kernel_h=self.klayer.nb_row, + stride_w=self.klayer.subsample[1], + stride_h=self.klayer.subsample[0], + pad_w=bpadW, + pad_h=bpadH, + wRegularizer=self.to_bigdl_reg(self.config["W_regularizer"]), + bRegularizer=self.to_bigdl_reg(self.config["b_regularizer"]), + with_bias=self.klayer.bias, + data_format=bigdl_order) + + if self.config["activation"] != "linear": + activation = get_activation_by_name(self.config["activation"], + "%s_%s" % (self.config["name"], self.config["activation"])) + return self.fuse(blayer, activation) + else: + return blayer + def combo_parameter_layer(self, blayer, config): blayer.set_name(config["name"]) if hasattr(blayer, "set_init_method"): diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 6037d452e4e..8f6c838277f 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -1860,13 +1860,16 @@ class SpatialDropout3D(Layer): :param format 'NCHW' or 'NHWC'. In 'NCHW' mode, the channels dimension (the depth) is at index 1, in 'NHWC' mode is it at index 4. + + >>> dropout = SpatialDropout3D(0.5, "NHWC") + creating: createSpatialDropout3D ''' def __init__(self, init_p=0.5, data_format="NCHW", bigdl_type="float"): - super(Dropout, self).__init__(None, bigdl_type, data_format, - init_p) + super(SpatialDropout3D, self).__init__(None, bigdl_type, + init_p, data_format) class SpatialDropout2D(Layer): ''' @@ -1882,13 +1885,16 @@ class SpatialDropout2D(Layer): :param format 'NCHW' or 'NHWC'. In 'NCHW' mode, the channels dimension (the depth) is at index 1, in 'NHWC' mode is it at index 4. + + >>> dropout = SpatialDropout2D(0.4, "NHWC") + creating: createSpatialDropout2D ''' def __init__(self, init_p=0.5, data_format="NCHW", bigdl_type="float"): - super(Dropout, self).__init__(None, bigdl_type, data_format, - init_p) + super(SpatialDropout2D, self).__init__(None, bigdl_type, + init_p, data_format) class SpatialDropout1D(Layer): ''' @@ -1901,12 +1907,15 @@ class SpatialDropout1D(Layer): between feature maps and should be used instead. :param initP the probability p + + >>> dropout = SpatialDropout1D(0.4) + creating: createSpatialDropout1D ''' def __init__(self, init_p=0.5, bigdl_type="float"): - super(Dropout, self).__init__(None, bigdl_type, - init_p) + super(SpatialDropout1D, self).__init__(None, bigdl_type, + init_p) class Dropout(Layer): From e2a0bf8f9f1062d39b92e0999a102042d3947793 Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Mon, 18 Dec 2017 15:22:10 +0800 Subject: [PATCH 464/823] Add remaining keras layer mappings (#2043) * add cropping2d3d * add spatialdropout * refactor 3d dim ordering * add locallyconnected2d * fix softmax * clean --- python/dllib/test/bigdl/keras/test_layer.py | 27 ++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/python/dllib/test/bigdl/keras/test_layer.py b/python/dllib/test/bigdl/keras/test_layer.py index a5b3b6fea7a..39b64949b25 100644 --- a/python/dllib/test/bigdl/keras/test_layer.py +++ b/python/dllib/test/bigdl/keras/test_layer.py @@ -50,7 +50,7 @@ def test_hard_sigmoid(self): self.modelTestSingleLayer(input_data, layer) def test_softmax(self): - input_data = np.random.random_sample([5, 6]) + input_data = np.random.random_sample([2, 3, 5]) layer = Activation('softmax') self.modelTestSingleLayer(input_data, layer) @@ -571,6 +571,20 @@ def test_cropping1d(self): layer = Cropping1D(cropping=(1, 2)) self.modelTestSingleLayer(input_data, layer) + def test_cropping2d(self): + input_data = np.random.random([2, 3, 28, 28]) + layer1 = Cropping2D(cropping=((2, 2), (4, 4))) + self.modelTestSingleLayer(input_data, layer1) + layer2 = Cropping2D(cropping=((0, 2), (3, 1))) + self.modelTestSingleLayer(input_data, layer2) + + def test_cropping3d(self): + input_data = np.random.random([2, 10, 28, 28, 32]) + layer1 = Cropping3D(cropping=((1, 1), (2, 2), (4, 4))) + self.modelTestSingleLayer(input_data, layer1) + layer2 = Cropping3D(cropping=((0, 2), (3, 1), (2, 3))) + self.modelTestSingleLayer(input_data, layer2) + def test_simplernn(self): input_data = np.random.random([3, 4, 5]) layer = SimpleRNN(5, input_shape=(4, 5), return_sequences=True) @@ -700,6 +714,17 @@ def test_srelu(self): layer = SReLU(input_shape=(4, 6)) self.modelTestSingleLayer(input_data, layer, dump_weights=True) + def test_locallyconnected2d(self): + input_data = np.random.random_sample([2, 3, 6, 8]) + layer1 = LocallyConnected2D(3, 1, 2, input_shape=(3, 6, 8)) + self.modelTestSingleLayer(input_data, layer1, dump_weights=True) + layer2 = LocallyConnected2D(4, 2, 1, activation='sigmoid', input_shape=(3, 6, 8)) + self.modelTestSingleLayer(input_data, layer2, dump_weights=True) + layer3 = LocallyConnected2D(2, 2, 1, bias=False, input_shape=(3, 6, 8)) + self.modelTestSingleLayer(input_data, layer3, dump_weights=True) + layer4 = LocallyConnected2D(4, 2, 2, dim_ordering="tf", input_shape=(3, 6, 8)) + self.modelTestSingleLayer(input_data, layer4, dump_weights=True) + if __name__ == "__main__": pytest.main([__file__]) From f276e22715053bbcf4cc3644902fbdd776787803 Mon Sep 17 00:00:00 2001 From: Ian Wong Date: Mon, 18 Dec 2017 16:34:26 +0800 Subject: [PATCH 465/823] fix stale wiki links (#2046) --- python/dllib/src/bigdl/dllib/models/rnn/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/dllib/src/bigdl/dllib/models/rnn/README.md b/python/dllib/src/bigdl/dllib/models/rnn/README.md index 2e21a8b6582..2c58cc18a72 100644 --- a/python/dllib/src/bigdl/dllib/models/rnn/README.md +++ b/python/dllib/src/bigdl/dllib/models/rnn/README.md @@ -9,7 +9,7 @@ The implementation of RNNs in this code is referred to in the [Keras Recurrent]( ## Get the BigDL files -Please build BigDL referring to [Build Page](https://github.com/intel-analytics/BigDL/wiki/Build-Page). +Please build BigDL referring to [Build Page](https://bigdl-project.github.io/master/#ScalaUserGuide/install-build-src/). ## Prepare the Input Data From c09e51cad7e364d1378f2582ea74eab31c9c6aac Mon Sep 17 00:00:00 2001 From: Ian Wong Date: Mon, 18 Dec 2017 16:34:26 +0800 Subject: [PATCH 466/823] fix stale wiki links (#2046) --- python/dllib/src/bigdl/dllib/models/rnn/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/dllib/src/bigdl/dllib/models/rnn/README.md b/python/dllib/src/bigdl/dllib/models/rnn/README.md index 2e21a8b6582..2c58cc18a72 100644 --- a/python/dllib/src/bigdl/dllib/models/rnn/README.md +++ b/python/dllib/src/bigdl/dllib/models/rnn/README.md @@ -9,7 +9,7 @@ The implementation of RNNs in this code is referred to in the [Keras Recurrent]( ## Get the BigDL files -Please build BigDL referring to [Build Page](https://github.com/intel-analytics/BigDL/wiki/Build-Page). +Please build BigDL referring to [Build Page](https://bigdl-project.github.io/master/#ScalaUserGuide/install-build-src/). ## Prepare the Input Data From 8bc11a145112ecf300f82a0c7cae56153153d66f Mon Sep 17 00:00:00 2001 From: Ian Wong Date: Mon, 18 Dec 2017 16:34:26 +0800 Subject: [PATCH 467/823] fix stale wiki links (#2046) --- python/dllib/test/bigdl/resources/conf/test_spark-bigdl.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/dllib/test/bigdl/resources/conf/test_spark-bigdl.conf b/python/dllib/test/bigdl/resources/conf/test_spark-bigdl.conf index 9870f82a6bc..ed1e17a8d20 100644 --- a/python/dllib/test/bigdl/resources/conf/test_spark-bigdl.conf +++ b/python/dllib/test/bigdl/resources/conf/test_spark-bigdl.conf @@ -24,7 +24,7 @@ # in your spark conf file. # # For more details, please refer -# https://github.com/intel-analytics/BigDL/wiki/Programming-Guide#engine +# https://bigdl-project.github.io/master/#APIGuide/Engine/ # spark.shuffle.reduceLocality.enabled false From 4d7f4e66baad3a65c0d4d36748526d027cf661c2 Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Mon, 18 Dec 2017 16:50:08 +0800 Subject: [PATCH 468/823] Unify optimizer creation for local and distributed mode. (#2013) * add example for pure local of python api update doc doc update * update * doc * doc * doc * clean * style * reserve the previous API * clean --- .../src/bigdl/dllib/bigdlkeras/backend.py | 2 +- .../src/bigdl/dllib/models/lenet/README.md | 2 +- .../bigdl/dllib/models/local_lenet/README.md | 18 + .../dllib/models/local_lenet/__init__.py | 0 .../dllib/models/local_lenet/local_lenet.py | 78 ++++ python/dllib/src/bigdl/dllib/nn/layer.py | 31 +- .../dllib/src/bigdl/dllib/optim/optimizer.py | 333 ++++++++++-------- 7 files changed, 318 insertions(+), 146 deletions(-) create mode 100644 python/dllib/src/bigdl/dllib/models/local_lenet/README.md create mode 100644 python/dllib/src/bigdl/dllib/models/local_lenet/__init__.py create mode 100644 python/dllib/src/bigdl/dllib/models/local_lenet/local_lenet.py diff --git a/python/dllib/src/bigdl/dllib/bigdlkeras/backend.py b/python/dllib/src/bigdl/dllib/bigdlkeras/backend.py index 02a4abf771d..087cd50380c 100644 --- a/python/dllib/src/bigdl/dllib/bigdlkeras/backend.py +++ b/python/dllib/src/bigdl/dllib/bigdlkeras/backend.py @@ -142,7 +142,7 @@ def __create_local_optimizer(self, x, y, batch_size=32, nb_epoch=10, validation_ raise unsupport_exp("validation_data") bopt = boptimizer.LocalOptimizer( X=x, - y=y, + Y=y, model=self.bmodel, criterion=self.criterion, end_trigger=boptimizer.MaxEpoch(nb_epoch), diff --git a/python/dllib/src/bigdl/dllib/models/lenet/README.md b/python/dllib/src/bigdl/dllib/models/lenet/README.md index 111268b0315..271cd49a0df 100644 --- a/python/dllib/src/bigdl/dllib/models/lenet/README.md +++ b/python/dllib/src/bigdl/dllib/models/lenet/README.md @@ -5,7 +5,7 @@ please refer to . ## Install dependencies * [Install dependencies](../../../README.md#install.dependencies) - + ## How to run this example: Please note that due to some permission issue, this example **cannot** be run on Windows. diff --git a/python/dllib/src/bigdl/dllib/models/local_lenet/README.md b/python/dllib/src/bigdl/dllib/models/local_lenet/README.md new file mode 100644 index 00000000000..765ff4ea04c --- /dev/null +++ b/python/dllib/src/bigdl/dllib/models/local_lenet/README.md @@ -0,0 +1,18 @@ +# LeNet5 Model on MNIST + +LeNet5 is a classical CNN model used in digital number classification. For detailed information, +please refer to . + +This example would show how to train and inference a LeNet model in pure local mode without using +Spark local or Spark distributed cluster. + +## How to run this example: + +``` +pip install BigDL +``` + +``` +export SPARK_DRIVER_MEMORY=2g +python ${BigDL_HOME}/pyspark/bigdl/models/local_lenet/local_lenet.py +``` \ No newline at end of file diff --git a/python/dllib/src/bigdl/dllib/models/local_lenet/__init__.py b/python/dllib/src/bigdl/dllib/models/local_lenet/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/python/dllib/src/bigdl/dllib/models/local_lenet/local_lenet.py b/python/dllib/src/bigdl/dllib/models/local_lenet/local_lenet.py new file mode 100644 index 00000000000..1a8f91f2c1f --- /dev/null +++ b/python/dllib/src/bigdl/dllib/models/local_lenet/local_lenet.py @@ -0,0 +1,78 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from optparse import OptionParser +from bigdl.dataset import mnist +from bigdl.dataset.transformer import * +from bigdl.nn.layer import * +from bigdl.nn.criterion import * +from bigdl.optim.optimizer import * +from bigdl.util.common import * + + +def build_model(class_num): + model = Sequential() + model.add(Reshape([1, 28, 28])) + model.add(SpatialConvolution(1, 6, 5, 5)) + model.add(Tanh()) + model.add(SpatialMaxPooling(2, 2, 2, 2)) + model.add(Tanh()) + model.add(SpatialConvolution(6, 12, 5, 5)) + model.add(SpatialMaxPooling(2, 2, 2, 2)) + model.add(Reshape([12 * 4 * 4])) + model.add(Linear(12 * 4 * 4, 100)) + model.add(Tanh()) + model.add(Linear(100, class_num)) + model.add(LogSoftMax()) + return model + + +def get_mnist(data_type="train", location="/tmp/mnist"): + """ + Get and normalize the mnist data. We would download it automatically + if the data doesn't present at the specific location. + + :param data_type: training data or testing data + :param location: Location storing the mnist + :return: (features: Ndarray, label: Ndarray) + """ + X, Y = mnist.read_data_sets(location, data_type) + return X, Y + 1 # The label of ClassNLLCriterion starts from 1 instead of 0 + + +if __name__ == "__main__": + redire_spark_logs() + show_bigdl_info_logs() + init_engine() + (X_train, Y_train) = get_mnist("train") + (X_test, Y_test) = get_mnist("test") + optimizer = Optimizer.create( + model=build_model(10), + training_set=(X_train, Y_train), + criterion=ClassNLLCriterion(), + optim_method=SGD(learningrate=0.01, learningrate_decay=0.0002), + end_trigger=MaxEpoch(20), + batch_size=128) + optimizer.set_validation( + batch_size=128, + X_val = X_test, + Y_val = Y_test, + trigger=EveryEpoch(), + val_method=[Top1Accuracy()] + ) + trained_model = optimizer.optimize() + predict_result = trained_model.predict_class(X_test) + print(predict_result) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 8f6c838277f..eab65817470 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -30,6 +30,7 @@ from bigdl.util.common import get_activation_by_name from bigdl.optim.optimizer import L1Regularizer, L2Regularizer, L1L2Regularizer from py4j.java_gateway import JavaObject +from pyspark.rdd import RDD from bigdl.transform.vision.image import ImageFrame if sys.version >= '3': @@ -310,7 +311,7 @@ def predict_local(self, X): return np.stack([j.to_ndarray()for j in jresults]) - def predict_local_class(self, X): + def predict_class_local(self, X): """ :param X: X can be a ndarray or list of ndarray if the model has multiple inputs. @@ -323,7 +324,31 @@ def predict_local_class(self, X): self._to_jtensors(X)) return np.stack(result) - def predict(self, data_rdd): + def predict(self, features): + """ + Model inference base on the given data. + :param features: it can be a ndarray or list of ndarray for locally inference + or RDD[Sample] for running in distributed fashion + :return: ndarray or RDD[Sample] depend on the the type of features. + """ + if isinstance(features, RDD): + return self.predict_distributed(features) + else: + return self.predict_local(features) + + def predict_class(self, features): + """ + Model inference base on the given data which returning label + :param features: it can be a ndarray or list of ndarray for locally inference + or RDD[Sample] for running in distributed fashion + :return: ndarray or RDD[Sample] depend on the the type of features. + """ + if isinstance(features, RDD): + return self.predict_class_distributed(features) + else: + return self.predict_class_local(features) + + def predict_distributed(self, data_rdd): """ Model inference base on the given data. You need to invoke collect() to trigger those action \ @@ -336,7 +361,7 @@ def predict(self, data_rdd): "modelPredictRDD", self.value, data_rdd) return result.map(lambda data: data.to_ndarray()) - def predict_class(self, data_rdd): + def predict_class_distributed(self, data_rdd): """ module predict, return the predict label diff --git a/python/dllib/src/bigdl/dllib/optim/optimizer.py b/python/dllib/src/bigdl/dllib/optim/optimizer.py index c0398206998..4051472957a 100644 --- a/python/dllib/src/bigdl/dllib/optim/optimizer.py +++ b/python/dllib/src/bigdl/dllib/optim/optimizer.py @@ -15,10 +15,14 @@ # +import multiprocessing import os import sys from distutils.dir_util import mkpath +from py4j.java_gateway import JavaObject +from pyspark.rdd import RDD + from bigdl.util.common import DOUBLEMAX from bigdl.util.common import JTensor from bigdl.util.common import JavaValue @@ -26,8 +30,6 @@ from bigdl.util.common import callJavaFunc from bigdl.util.common import get_spark_context from bigdl.util.common import to_list -from py4j.java_gateway import JavaObject -import multiprocessing if sys.version >= '3': long = int @@ -520,130 +522,7 @@ def __init__(self, step_sizes, gamma, bigdl_type="float"): JavaValue.__init__(self, None, bigdl_type, step_sizes, gamma) -class DistriOptimizer(JavaValue): - def __init__(self, - model, - training_rdd, - criterion, - end_trigger, - batch_size, - optim_method=None, - bigdl_type="float"): - """ - Create an optimizer. - - - :param model: the neural net model - :param training_data: the training dataset - :param criterion: the loss function - :param optim_method: the algorithm to use for optimization, - e.g. SGD, Adagrad, etc. If optim_method is None, the default algorithm is SGD. - :param end_trigger: when to end the optimization - :param batch_size: training batch size - """ - JavaValue.__init__(self, None, bigdl_type, model.value, - training_rdd, criterion, - optim_method if optim_method else SGD(), end_trigger, batch_size) - - -class LocalOptimizer(JavaValue): - """ - Create an optimizer. - - - :param model: the neural net model - :param X: the training features which is an ndarray or list of ndarray - :param Y: the training label which is an ndarray - :param criterion: the loss function - :param optim_method: the algorithm to use for optimization, - e.g. SGD, Adagrad, etc. If optim_method is None, the default algorithm is SGD. - :param end_trigger: when to end the optimization - :param batch_size: training batch size - :param cores: by default is the total physical cores. - """ - def __init__(self, - X, - y, - model, - criterion, - end_trigger, - batch_size, - optim_method=None, - cores=None, - bigdl_type="float"): - if cores is None: - cores = multiprocessing.cpu_count() - JavaValue.__init__(self, None, bigdl_type, - [JTensor.from_ndarray(X) for X in to_list(X)], - JTensor.from_ndarray(y), - model.value, - criterion, - optim_method if optim_method else SGD(), end_trigger, batch_size, cores) - - def optimize(self): - """ - Do an optimization. - """ - jmodel = callJavaFunc(get_spark_context(), self.value.optimize) - from bigdl.nn.layer import Layer - return Layer.of(jmodel) - - -class Optimizer(JavaValue): - """ - This is a distrubuted optimizer. - An optimizer is in general to minimize any function with respect - to a set of parameters. - In case of training a neural network, - an optimizer tries to minimize the loss of the neural net with - respect to its weights/biases, over the training set. - """ - def __init__(self, - model, - training_rdd, - criterion, - end_trigger, - batch_size, - optim_method=None, - bigdl_type="float"): - """ - Create an optimizer. - - - :param model: the neural net model - :param training_rdd: the training dataset - :param criterion: the loss function - :param optim_method: the algorithm to use for optimization, - e.g. SGD, Adagrad, etc. If optim_method is None, the default algorithm is SGD. - :param end_trigger: when to end the optimization - :param batch_size: training batch size - """ - self.pvalue = DistriOptimizer(model, - training_rdd, - criterion, - end_trigger, - batch_size, - optim_method, - bigdl_type) - self.value = self.pvalue.value - self.bigdl_type = self.pvalue.bigdl_type - - - - def set_validation(self, batch_size, val_rdd, trigger, val_method=None): - """ - Configure validation settings. - - - :param batch_size: validation batch size - :param val_rdd: validation dataset - :param trigger: validation interval - :param val_method: the ValidationMethod to use,e.g. "Top1Accuracy", "Top5Accuracy", "Loss" - """ - if val_method is None: - val_method = [Top1Accuracy()] - callBigDlFunc(self.bigdl_type, "setValidation", self.value, batch_size, - trigger, val_rdd, to_list(val_method)) +class BaseOptimizer(JavaValue): def set_model(self, model): """ @@ -654,17 +533,6 @@ def set_model(self, model): """ self.value.setModel(model.value) - def set_traindata(self, training_rdd, batch_size): - """ - Set new training dataset, for optimizer reuse - - :param training_rdd: the training dataset - :param batch_size: training batch size - :return: - """ - callBigDlFunc(self.bigdl_type, "setTrainData", self.value, - training_rdd, batch_size) - def set_criterion(self, criterion): """ set new criterion, for optimizer reuse @@ -676,7 +544,7 @@ def set_criterion(self, criterion): criterion) def set_checkpoint(self, checkpoint_trigger, - checkpoint_path, isOverWrite=True): + checkpoint_path, isOverWrite=True): """ Configure checkpoint settings. @@ -703,9 +571,9 @@ def set_gradclip_const(self, min_value, max_value): def set_gradclip_l2norm(self, clip_norm): """ Configure L2 norm clipping settings. - - - :param clip_norm: gradient L2-Norm threshold + + + :param clip_norm: gradient L2-Norm threshold """ callBigDlFunc(self.bigdl_type, "setL2NormClip", self.value, clip_norm) @@ -763,6 +631,189 @@ def prepare_input(self): self.value.prepareInput() +class Optimizer(BaseOptimizer): + + # NOTE: This is a deprecated method, you should use `create` method instead. + def __init__(self, + model, + training_rdd, + criterion, + end_trigger, + batch_size, + optim_method=None, + bigdl_type="float"): + """ + Create a distributed optimizer. + + + :param model: the neural net model + :param training_rdd: the training dataset + :param criterion: the loss function + :param optim_method: the algorithm to use for optimization, + e.g. SGD, Adagrad, etc. If optim_method is None, the default algorithm is SGD. + :param end_trigger: when to end the optimization + :param batch_size: training batch size + """ + self.pvalue = DistriOptimizer(model, + training_rdd, + criterion, + end_trigger, + batch_size, + optim_method, + bigdl_type) + self.value = self.pvalue.value + self.bigdl_type = self.pvalue.bigdl_type + + @staticmethod + def create(model, + training_set, + criterion, + end_trigger=MaxEpoch(1), + batch_size=32, + optim_method=SGD(), + cores=None, + bigdl_type="float"): + """ + Create an optimizer. + Depend on the input type, the returning optimizer can be a local optimizer \ + or a distributed optimizer. + + :param model: the neural net model + :param training_set: (features, label) for local mode. RDD[Sample] for distributed mode. + :param criterion: the loss function + :param optim_method: the algorithm to use for optimization, + e.g. SGD, Adagrad, etc. If optim_method is None, the default algorithm is SGD. + :param end_trigger: when to end the optimization + :param batch_size: training batch size + :param cores: This is for local optimizer only and use total physical cores as the default value + """ + if isinstance(training_set, RDD): + return DistriOptimizer(model=model, + training_rdd=training_set, + criterion=criterion, + end_trigger=end_trigger, + batch_size=batch_size, + optim_method=optim_method, + bigdl_type=bigdl_type) + elif isinstance(training_set, tuple) and len(training_set) == 2: + x, y = training_set + return LocalOptimizer(X=x, + Y=y, + model=model, + criterion=criterion, + end_trigger=end_trigger, + batch_size=batch_size, + optim_method=optim_method, + cores=cores, + bigdl_type="float") + else: + raise Exception("Not supported training set: %s" % type(training_set)) + + def set_validation(self, batch_size, val_rdd, trigger, val_method=None): + """ + Configure validation settings. + + + :param batch_size: validation batch size + :param val_rdd: validation dataset + :param trigger: validation interval + :param val_method: the ValidationMethod to use,e.g. "Top1Accuracy", "Top5Accuracy", "Loss" + """ + if val_method is None: + val_method = [Top1Accuracy()] + callBigDlFunc(self.bigdl_type, "setValidation", self.value, batch_size, + trigger, val_rdd, to_list(val_method)) + + def set_traindata(self, training_rdd, batch_size): + """ + Set new training dataset, for optimizer reuse + + :param training_rdd: the training dataset + :param batch_size: training batch size + :return: + """ + callBigDlFunc(self.bigdl_type, "setTrainData", self.value, + training_rdd, batch_size) + + + +class DistriOptimizer(Optimizer): + def __init__(self, + model, + training_rdd, + criterion, + end_trigger, + batch_size, + optim_method=None, + bigdl_type="float"): + """ + Create an optimizer. + + + :param model: the neural net model + :param training_data: the training dataset + :param criterion: the loss function + :param optim_method: the algorithm to use for optimization, + e.g. SGD, Adagrad, etc. If optim_method is None, the default algorithm is SGD. + :param end_trigger: when to end the optimization + :param batch_size: training batch size + """ + JavaValue.__init__(self, None, bigdl_type, model.value, + training_rdd, criterion, + optim_method if optim_method else SGD(), end_trigger, batch_size) + + +class LocalOptimizer(BaseOptimizer): + """ + Create an optimizer. + + + :param model: the neural net model + :param X: the training features which is an ndarray or list of ndarray + :param Y: the training label which is an ndarray + :param criterion: the loss function + :param optim_method: the algorithm to use for optimization, + e.g. SGD, Adagrad, etc. If optim_method is None, the default algorithm is SGD. + :param end_trigger: when to end the optimization + :param batch_size: training batch size + :param cores: by default is the total physical cores. + """ + def __init__(self, + X, + Y, + model, + criterion, + end_trigger, + batch_size, + optim_method=None, + cores=None, + bigdl_type="float"): + if cores is None: + cores = multiprocessing.cpu_count() + JavaValue.__init__(self, None, bigdl_type, + [JTensor.from_ndarray(X) for X in to_list(X)], + JTensor.from_ndarray(Y), + model.value, + criterion, + optim_method if optim_method else SGD(), end_trigger, batch_size, cores) + + def set_validation(self, batch_size, X_val, Y_val, trigger, val_method=None): + """ + Configure validation settings. + + :param batch_size: validation batch size + :param X_val: features of validation dataset + :param Y_val: label of validation dataset + :param trigger: validation interval + :param val_method: the ValidationMethod to use,e.g. "Top1Accuracy", "Top5Accuracy", "Loss" + """ + if val_method is None: + val_method = [Top1Accuracy()] + callBigDlFunc(self.bigdl_type, "setValidation", self.value, batch_size, + trigger, [JTensor.from_ndarray(X) for X in to_list(X_val)], + JTensor.from_ndarray(Y_val), to_list(val_method)) + + class TrainSummary(JavaValue, ): """ A logging facility which allows user to trace how indicators (e.g. From 186ffa6c4b08cbf567bee0d1e020b1f956e7ffed Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Mon, 18 Dec 2017 16:50:08 +0800 Subject: [PATCH 469/823] Unify optimizer creation for local and distributed mode. (#2013) * add example for pure local of python api update doc doc update * update * doc * doc * doc * clean * style * reserve the previous API * clean --- .../src/bigdl/dllib/bigdlkeras/backend.py | 2 +- .../src/bigdl/dllib/models/lenet/README.md | 2 +- .../bigdl/dllib/models/local_lenet/README.md | 18 + .../dllib/models/local_lenet/__init__.py | 0 .../dllib/models/local_lenet/local_lenet.py | 78 ++++ python/dllib/src/bigdl/dllib/nn/layer.py | 31 +- .../dllib/src/bigdl/dllib/optim/optimizer.py | 333 ++++++++++-------- 7 files changed, 318 insertions(+), 146 deletions(-) create mode 100644 python/dllib/src/bigdl/dllib/models/local_lenet/README.md create mode 100644 python/dllib/src/bigdl/dllib/models/local_lenet/__init__.py create mode 100644 python/dllib/src/bigdl/dllib/models/local_lenet/local_lenet.py diff --git a/python/dllib/src/bigdl/dllib/bigdlkeras/backend.py b/python/dllib/src/bigdl/dllib/bigdlkeras/backend.py index 02a4abf771d..087cd50380c 100644 --- a/python/dllib/src/bigdl/dllib/bigdlkeras/backend.py +++ b/python/dllib/src/bigdl/dllib/bigdlkeras/backend.py @@ -142,7 +142,7 @@ def __create_local_optimizer(self, x, y, batch_size=32, nb_epoch=10, validation_ raise unsupport_exp("validation_data") bopt = boptimizer.LocalOptimizer( X=x, - y=y, + Y=y, model=self.bmodel, criterion=self.criterion, end_trigger=boptimizer.MaxEpoch(nb_epoch), diff --git a/python/dllib/src/bigdl/dllib/models/lenet/README.md b/python/dllib/src/bigdl/dllib/models/lenet/README.md index 111268b0315..271cd49a0df 100644 --- a/python/dllib/src/bigdl/dllib/models/lenet/README.md +++ b/python/dllib/src/bigdl/dllib/models/lenet/README.md @@ -5,7 +5,7 @@ please refer to . ## Install dependencies * [Install dependencies](../../../README.md#install.dependencies) - + ## How to run this example: Please note that due to some permission issue, this example **cannot** be run on Windows. diff --git a/python/dllib/src/bigdl/dllib/models/local_lenet/README.md b/python/dllib/src/bigdl/dllib/models/local_lenet/README.md new file mode 100644 index 00000000000..765ff4ea04c --- /dev/null +++ b/python/dllib/src/bigdl/dllib/models/local_lenet/README.md @@ -0,0 +1,18 @@ +# LeNet5 Model on MNIST + +LeNet5 is a classical CNN model used in digital number classification. For detailed information, +please refer to . + +This example would show how to train and inference a LeNet model in pure local mode without using +Spark local or Spark distributed cluster. + +## How to run this example: + +``` +pip install BigDL +``` + +``` +export SPARK_DRIVER_MEMORY=2g +python ${BigDL_HOME}/pyspark/bigdl/models/local_lenet/local_lenet.py +``` \ No newline at end of file diff --git a/python/dllib/src/bigdl/dllib/models/local_lenet/__init__.py b/python/dllib/src/bigdl/dllib/models/local_lenet/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/python/dllib/src/bigdl/dllib/models/local_lenet/local_lenet.py b/python/dllib/src/bigdl/dllib/models/local_lenet/local_lenet.py new file mode 100644 index 00000000000..1a8f91f2c1f --- /dev/null +++ b/python/dllib/src/bigdl/dllib/models/local_lenet/local_lenet.py @@ -0,0 +1,78 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from optparse import OptionParser +from bigdl.dataset import mnist +from bigdl.dataset.transformer import * +from bigdl.nn.layer import * +from bigdl.nn.criterion import * +from bigdl.optim.optimizer import * +from bigdl.util.common import * + + +def build_model(class_num): + model = Sequential() + model.add(Reshape([1, 28, 28])) + model.add(SpatialConvolution(1, 6, 5, 5)) + model.add(Tanh()) + model.add(SpatialMaxPooling(2, 2, 2, 2)) + model.add(Tanh()) + model.add(SpatialConvolution(6, 12, 5, 5)) + model.add(SpatialMaxPooling(2, 2, 2, 2)) + model.add(Reshape([12 * 4 * 4])) + model.add(Linear(12 * 4 * 4, 100)) + model.add(Tanh()) + model.add(Linear(100, class_num)) + model.add(LogSoftMax()) + return model + + +def get_mnist(data_type="train", location="/tmp/mnist"): + """ + Get and normalize the mnist data. We would download it automatically + if the data doesn't present at the specific location. + + :param data_type: training data or testing data + :param location: Location storing the mnist + :return: (features: Ndarray, label: Ndarray) + """ + X, Y = mnist.read_data_sets(location, data_type) + return X, Y + 1 # The label of ClassNLLCriterion starts from 1 instead of 0 + + +if __name__ == "__main__": + redire_spark_logs() + show_bigdl_info_logs() + init_engine() + (X_train, Y_train) = get_mnist("train") + (X_test, Y_test) = get_mnist("test") + optimizer = Optimizer.create( + model=build_model(10), + training_set=(X_train, Y_train), + criterion=ClassNLLCriterion(), + optim_method=SGD(learningrate=0.01, learningrate_decay=0.0002), + end_trigger=MaxEpoch(20), + batch_size=128) + optimizer.set_validation( + batch_size=128, + X_val = X_test, + Y_val = Y_test, + trigger=EveryEpoch(), + val_method=[Top1Accuracy()] + ) + trained_model = optimizer.optimize() + predict_result = trained_model.predict_class(X_test) + print(predict_result) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 8f6c838277f..eab65817470 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -30,6 +30,7 @@ from bigdl.util.common import get_activation_by_name from bigdl.optim.optimizer import L1Regularizer, L2Regularizer, L1L2Regularizer from py4j.java_gateway import JavaObject +from pyspark.rdd import RDD from bigdl.transform.vision.image import ImageFrame if sys.version >= '3': @@ -310,7 +311,7 @@ def predict_local(self, X): return np.stack([j.to_ndarray()for j in jresults]) - def predict_local_class(self, X): + def predict_class_local(self, X): """ :param X: X can be a ndarray or list of ndarray if the model has multiple inputs. @@ -323,7 +324,31 @@ def predict_local_class(self, X): self._to_jtensors(X)) return np.stack(result) - def predict(self, data_rdd): + def predict(self, features): + """ + Model inference base on the given data. + :param features: it can be a ndarray or list of ndarray for locally inference + or RDD[Sample] for running in distributed fashion + :return: ndarray or RDD[Sample] depend on the the type of features. + """ + if isinstance(features, RDD): + return self.predict_distributed(features) + else: + return self.predict_local(features) + + def predict_class(self, features): + """ + Model inference base on the given data which returning label + :param features: it can be a ndarray or list of ndarray for locally inference + or RDD[Sample] for running in distributed fashion + :return: ndarray or RDD[Sample] depend on the the type of features. + """ + if isinstance(features, RDD): + return self.predict_class_distributed(features) + else: + return self.predict_class_local(features) + + def predict_distributed(self, data_rdd): """ Model inference base on the given data. You need to invoke collect() to trigger those action \ @@ -336,7 +361,7 @@ def predict(self, data_rdd): "modelPredictRDD", self.value, data_rdd) return result.map(lambda data: data.to_ndarray()) - def predict_class(self, data_rdd): + def predict_class_distributed(self, data_rdd): """ module predict, return the predict label diff --git a/python/dllib/src/bigdl/dllib/optim/optimizer.py b/python/dllib/src/bigdl/dllib/optim/optimizer.py index c0398206998..4051472957a 100644 --- a/python/dllib/src/bigdl/dllib/optim/optimizer.py +++ b/python/dllib/src/bigdl/dllib/optim/optimizer.py @@ -15,10 +15,14 @@ # +import multiprocessing import os import sys from distutils.dir_util import mkpath +from py4j.java_gateway import JavaObject +from pyspark.rdd import RDD + from bigdl.util.common import DOUBLEMAX from bigdl.util.common import JTensor from bigdl.util.common import JavaValue @@ -26,8 +30,6 @@ from bigdl.util.common import callJavaFunc from bigdl.util.common import get_spark_context from bigdl.util.common import to_list -from py4j.java_gateway import JavaObject -import multiprocessing if sys.version >= '3': long = int @@ -520,130 +522,7 @@ def __init__(self, step_sizes, gamma, bigdl_type="float"): JavaValue.__init__(self, None, bigdl_type, step_sizes, gamma) -class DistriOptimizer(JavaValue): - def __init__(self, - model, - training_rdd, - criterion, - end_trigger, - batch_size, - optim_method=None, - bigdl_type="float"): - """ - Create an optimizer. - - - :param model: the neural net model - :param training_data: the training dataset - :param criterion: the loss function - :param optim_method: the algorithm to use for optimization, - e.g. SGD, Adagrad, etc. If optim_method is None, the default algorithm is SGD. - :param end_trigger: when to end the optimization - :param batch_size: training batch size - """ - JavaValue.__init__(self, None, bigdl_type, model.value, - training_rdd, criterion, - optim_method if optim_method else SGD(), end_trigger, batch_size) - - -class LocalOptimizer(JavaValue): - """ - Create an optimizer. - - - :param model: the neural net model - :param X: the training features which is an ndarray or list of ndarray - :param Y: the training label which is an ndarray - :param criterion: the loss function - :param optim_method: the algorithm to use for optimization, - e.g. SGD, Adagrad, etc. If optim_method is None, the default algorithm is SGD. - :param end_trigger: when to end the optimization - :param batch_size: training batch size - :param cores: by default is the total physical cores. - """ - def __init__(self, - X, - y, - model, - criterion, - end_trigger, - batch_size, - optim_method=None, - cores=None, - bigdl_type="float"): - if cores is None: - cores = multiprocessing.cpu_count() - JavaValue.__init__(self, None, bigdl_type, - [JTensor.from_ndarray(X) for X in to_list(X)], - JTensor.from_ndarray(y), - model.value, - criterion, - optim_method if optim_method else SGD(), end_trigger, batch_size, cores) - - def optimize(self): - """ - Do an optimization. - """ - jmodel = callJavaFunc(get_spark_context(), self.value.optimize) - from bigdl.nn.layer import Layer - return Layer.of(jmodel) - - -class Optimizer(JavaValue): - """ - This is a distrubuted optimizer. - An optimizer is in general to minimize any function with respect - to a set of parameters. - In case of training a neural network, - an optimizer tries to minimize the loss of the neural net with - respect to its weights/biases, over the training set. - """ - def __init__(self, - model, - training_rdd, - criterion, - end_trigger, - batch_size, - optim_method=None, - bigdl_type="float"): - """ - Create an optimizer. - - - :param model: the neural net model - :param training_rdd: the training dataset - :param criterion: the loss function - :param optim_method: the algorithm to use for optimization, - e.g. SGD, Adagrad, etc. If optim_method is None, the default algorithm is SGD. - :param end_trigger: when to end the optimization - :param batch_size: training batch size - """ - self.pvalue = DistriOptimizer(model, - training_rdd, - criterion, - end_trigger, - batch_size, - optim_method, - bigdl_type) - self.value = self.pvalue.value - self.bigdl_type = self.pvalue.bigdl_type - - - - def set_validation(self, batch_size, val_rdd, trigger, val_method=None): - """ - Configure validation settings. - - - :param batch_size: validation batch size - :param val_rdd: validation dataset - :param trigger: validation interval - :param val_method: the ValidationMethod to use,e.g. "Top1Accuracy", "Top5Accuracy", "Loss" - """ - if val_method is None: - val_method = [Top1Accuracy()] - callBigDlFunc(self.bigdl_type, "setValidation", self.value, batch_size, - trigger, val_rdd, to_list(val_method)) +class BaseOptimizer(JavaValue): def set_model(self, model): """ @@ -654,17 +533,6 @@ def set_model(self, model): """ self.value.setModel(model.value) - def set_traindata(self, training_rdd, batch_size): - """ - Set new training dataset, for optimizer reuse - - :param training_rdd: the training dataset - :param batch_size: training batch size - :return: - """ - callBigDlFunc(self.bigdl_type, "setTrainData", self.value, - training_rdd, batch_size) - def set_criterion(self, criterion): """ set new criterion, for optimizer reuse @@ -676,7 +544,7 @@ def set_criterion(self, criterion): criterion) def set_checkpoint(self, checkpoint_trigger, - checkpoint_path, isOverWrite=True): + checkpoint_path, isOverWrite=True): """ Configure checkpoint settings. @@ -703,9 +571,9 @@ def set_gradclip_const(self, min_value, max_value): def set_gradclip_l2norm(self, clip_norm): """ Configure L2 norm clipping settings. - - - :param clip_norm: gradient L2-Norm threshold + + + :param clip_norm: gradient L2-Norm threshold """ callBigDlFunc(self.bigdl_type, "setL2NormClip", self.value, clip_norm) @@ -763,6 +631,189 @@ def prepare_input(self): self.value.prepareInput() +class Optimizer(BaseOptimizer): + + # NOTE: This is a deprecated method, you should use `create` method instead. + def __init__(self, + model, + training_rdd, + criterion, + end_trigger, + batch_size, + optim_method=None, + bigdl_type="float"): + """ + Create a distributed optimizer. + + + :param model: the neural net model + :param training_rdd: the training dataset + :param criterion: the loss function + :param optim_method: the algorithm to use for optimization, + e.g. SGD, Adagrad, etc. If optim_method is None, the default algorithm is SGD. + :param end_trigger: when to end the optimization + :param batch_size: training batch size + """ + self.pvalue = DistriOptimizer(model, + training_rdd, + criterion, + end_trigger, + batch_size, + optim_method, + bigdl_type) + self.value = self.pvalue.value + self.bigdl_type = self.pvalue.bigdl_type + + @staticmethod + def create(model, + training_set, + criterion, + end_trigger=MaxEpoch(1), + batch_size=32, + optim_method=SGD(), + cores=None, + bigdl_type="float"): + """ + Create an optimizer. + Depend on the input type, the returning optimizer can be a local optimizer \ + or a distributed optimizer. + + :param model: the neural net model + :param training_set: (features, label) for local mode. RDD[Sample] for distributed mode. + :param criterion: the loss function + :param optim_method: the algorithm to use for optimization, + e.g. SGD, Adagrad, etc. If optim_method is None, the default algorithm is SGD. + :param end_trigger: when to end the optimization + :param batch_size: training batch size + :param cores: This is for local optimizer only and use total physical cores as the default value + """ + if isinstance(training_set, RDD): + return DistriOptimizer(model=model, + training_rdd=training_set, + criterion=criterion, + end_trigger=end_trigger, + batch_size=batch_size, + optim_method=optim_method, + bigdl_type=bigdl_type) + elif isinstance(training_set, tuple) and len(training_set) == 2: + x, y = training_set + return LocalOptimizer(X=x, + Y=y, + model=model, + criterion=criterion, + end_trigger=end_trigger, + batch_size=batch_size, + optim_method=optim_method, + cores=cores, + bigdl_type="float") + else: + raise Exception("Not supported training set: %s" % type(training_set)) + + def set_validation(self, batch_size, val_rdd, trigger, val_method=None): + """ + Configure validation settings. + + + :param batch_size: validation batch size + :param val_rdd: validation dataset + :param trigger: validation interval + :param val_method: the ValidationMethod to use,e.g. "Top1Accuracy", "Top5Accuracy", "Loss" + """ + if val_method is None: + val_method = [Top1Accuracy()] + callBigDlFunc(self.bigdl_type, "setValidation", self.value, batch_size, + trigger, val_rdd, to_list(val_method)) + + def set_traindata(self, training_rdd, batch_size): + """ + Set new training dataset, for optimizer reuse + + :param training_rdd: the training dataset + :param batch_size: training batch size + :return: + """ + callBigDlFunc(self.bigdl_type, "setTrainData", self.value, + training_rdd, batch_size) + + + +class DistriOptimizer(Optimizer): + def __init__(self, + model, + training_rdd, + criterion, + end_trigger, + batch_size, + optim_method=None, + bigdl_type="float"): + """ + Create an optimizer. + + + :param model: the neural net model + :param training_data: the training dataset + :param criterion: the loss function + :param optim_method: the algorithm to use for optimization, + e.g. SGD, Adagrad, etc. If optim_method is None, the default algorithm is SGD. + :param end_trigger: when to end the optimization + :param batch_size: training batch size + """ + JavaValue.__init__(self, None, bigdl_type, model.value, + training_rdd, criterion, + optim_method if optim_method else SGD(), end_trigger, batch_size) + + +class LocalOptimizer(BaseOptimizer): + """ + Create an optimizer. + + + :param model: the neural net model + :param X: the training features which is an ndarray or list of ndarray + :param Y: the training label which is an ndarray + :param criterion: the loss function + :param optim_method: the algorithm to use for optimization, + e.g. SGD, Adagrad, etc. If optim_method is None, the default algorithm is SGD. + :param end_trigger: when to end the optimization + :param batch_size: training batch size + :param cores: by default is the total physical cores. + """ + def __init__(self, + X, + Y, + model, + criterion, + end_trigger, + batch_size, + optim_method=None, + cores=None, + bigdl_type="float"): + if cores is None: + cores = multiprocessing.cpu_count() + JavaValue.__init__(self, None, bigdl_type, + [JTensor.from_ndarray(X) for X in to_list(X)], + JTensor.from_ndarray(Y), + model.value, + criterion, + optim_method if optim_method else SGD(), end_trigger, batch_size, cores) + + def set_validation(self, batch_size, X_val, Y_val, trigger, val_method=None): + """ + Configure validation settings. + + :param batch_size: validation batch size + :param X_val: features of validation dataset + :param Y_val: label of validation dataset + :param trigger: validation interval + :param val_method: the ValidationMethod to use,e.g. "Top1Accuracy", "Top5Accuracy", "Loss" + """ + if val_method is None: + val_method = [Top1Accuracy()] + callBigDlFunc(self.bigdl_type, "setValidation", self.value, batch_size, + trigger, [JTensor.from_ndarray(X) for X in to_list(X_val)], + JTensor.from_ndarray(Y_val), to_list(val_method)) + + class TrainSummary(JavaValue, ): """ A logging facility which allows user to trace how indicators (e.g. From 2a83aadf9b0d753a6e0c86808394cb47fe7827fe Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Mon, 18 Dec 2017 16:50:08 +0800 Subject: [PATCH 470/823] Unify optimizer creation for local and distributed mode. (#2013) * add example for pure local of python api update doc doc update * update * doc * doc * doc * clean * style * reserve the previous API * clean --- .../test/bigdl/test_simple_integration.py | 26 +++++++++---------- python/dllib/test/bigdl/test_utils.py | 2 ++ 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/python/dllib/test/bigdl/test_simple_integration.py b/python/dllib/test/bigdl/test_simple_integration.py index 0f73eead379..92a5e9db9f1 100644 --- a/python/dllib/test/bigdl/test_simple_integration.py +++ b/python/dllib/test/bigdl/test_simple_integration.py @@ -233,9 +233,9 @@ def gen_rand_sample(): optim_method = SGD(learningrate=0.01, learningrate_decay=0.0002, weightdecay=0.0, momentum=0.0, dampening=0.0, nesterov=False, leaningrate_schedule=Poly(0.5, int((data_len / batch_size) * epoch_num))) - optimizer = Optimizer( + optimizer = Optimizer.create( model=model_test, - training_rdd=trainingData, + training_set=trainingData, criterion=MSECriterion(), optim_method=optim_method, end_trigger=MaxEpoch(epoch_num), @@ -309,9 +309,9 @@ def gen_rand_sample(): optim_method = SGD(learningrate=0.01, learningrate_decay=0.0002, weightdecay=0.0, momentum=0.0, dampening=0.0, nesterov=False, leaningrate_schedule=Poly(0.5, int((data_len / batch_size) * epoch_num))) - optimizer = Optimizer( + optimizer = Optimizer.create( model=model_test, - training_rdd=trainingData, + training_set=trainingData, criterion=MSECriterion(), optim_method=optim_method, end_trigger=MaxEpoch(epoch_num), @@ -346,9 +346,9 @@ def gen_rand_sample(): branches.add(branch1).add(branch2) model_test.add(branches) - optimizer = Optimizer( + optimizer = Optimizer.create( model=model_test, - training_rdd=training_data, + training_set=training_data, criterion=MarginRankingCriterion(), optim_method=SGD(), end_trigger=MaxEpoch(5), @@ -554,14 +554,14 @@ def test_local_optimizer_predict(self): l1 = Linear(feature_num, 1) model.add(l1) - localOptimizer = LocalOptimizer( + localOptimizer = Optimizer.create( model=model, - X=X_, - y=y_, + training_set=(X_, y_), criterion=MSECriterion(), optim_method=SGD(learningrate=1e-2), end_trigger=MaxEpoch(epoch_num), batch_size=batch_size) + trained_model = localOptimizer.optimize() trained_model = model w = trained_model.get_weights() @@ -580,7 +580,7 @@ def test_local_predict_class(self): model.add(l1) model.add(Sigmoid()) model.set_seed(1234).reset() - predict_result = model.predict_local_class(X_) + predict_result = model.predict_class(X_) assert_array_equal(predict_result, np.ones([3])) def test_local_predict_multiple_input(self): @@ -590,14 +590,14 @@ def test_local_predict_multiple_input(self): model = Model(inputs=[l1, l2], outputs=joinTable) result = model.predict_local([np.ones([4, 3]), np.ones([4, 3])]) assert result.shape == (4, 5) - result2 = model.predict_local_class([np.ones([4, 3]), np.ones([4, 3])]) + result2 = model.predict_class([np.ones([4, 3]), np.ones([4, 3])]) assert result2.shape == (4,) result3 = model.predict_local([JTensor.from_ndarray(np.ones([4, 3])), JTensor.from_ndarray(np.ones([4, 3]))]) assert result3.shape == (4, 5) - result4 = model.predict_local_class([JTensor.from_ndarray(np.ones([4, 3])), - JTensor.from_ndarray(np.ones([4, 3]))]) + result4 = model.predict_class([JTensor.from_ndarray(np.ones([4, 3])), + JTensor.from_ndarray(np.ones([4, 3]))]) assert result4.shape == (4,) diff --git a/python/dllib/test/bigdl/test_utils.py b/python/dllib/test/bigdl/test_utils.py index 83425d99114..e29b41fe211 100644 --- a/python/dllib/test/bigdl/test_utils.py +++ b/python/dllib/test/bigdl/test_utils.py @@ -110,6 +110,7 @@ def setup_method(self, method): """ setup any state tied to the execution of the given method in a class. setup_method is invoked for every test method of a class. """ + keras.backend.set_image_dim_ordering("th") sparkConf = create_spark_conf().setMaster("local[4]").setAppName("test model") self.sc = get_spark_context(sparkConf) self.sqlContext = SQLContext(self.sc) @@ -119,6 +120,7 @@ def teardown_method(self, method): """ teardown any state that was previously setup with a setup_method call. """ + keras.backend.set_image_dim_ordering("th") self.sc.stop() def __generate_model(self, input_data, output_layer): From f5549fc88c108529d5f550b382a72b596d914091 Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Mon, 18 Dec 2017 19:34:38 +0800 Subject: [PATCH 471/823] Add keras imdb example (#2039) * add imdb example; refactor load keras API * update yml * meet review --- .../src/bigdl/dllib/examples/keras/README.md | 11 +++ .../dllib/examples/keras/imdb_cnn_lstm.py | 97 +++++++++++++++++++ .../bigdl/dllib/examples/keras/keras_utils.py | 40 ++++++++ .../examples/keras/{cnn.py => mnist_cnn.py} | 27 +++--- .../dllib/src/bigdl/dllib/keras/converter.py | 13 ++- python/dllib/src/bigdl/dllib/nn/layer.py | 26 +++-- 6 files changed, 189 insertions(+), 25 deletions(-) create mode 100644 python/dllib/src/bigdl/dllib/examples/keras/README.md create mode 100644 python/dllib/src/bigdl/dllib/examples/keras/imdb_cnn_lstm.py create mode 100644 python/dllib/src/bigdl/dllib/examples/keras/keras_utils.py rename python/dllib/src/bigdl/dllib/examples/keras/{cnn.py => mnist_cnn.py} (86%) diff --git a/python/dllib/src/bigdl/dllib/examples/keras/README.md b/python/dllib/src/bigdl/dllib/examples/keras/README.md new file mode 100644 index 00000000000..2dd3e9138d9 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/examples/keras/README.md @@ -0,0 +1,11 @@ +# **Keras Examples** + +We provide several simple examples here to show how to load a Keras model into BigDL and running the model in a distributed fashion. + +Note that the Keras version we support and test is [__Keras 1.2.2__](https://faroit.github.io/keras-docs/1.2.2/) with TensorFlow backend. + +For the sake of illustration, in these examples, we first define a model in Keras, then save it as a JSON/HDF5 file and load it into BigDL. + +You can directly run these examples if you [install BigDL from pip](../../../../docs/docs/PythonUserGuide/install-from-pip.md). After the training, good accuracy can be achieved. + +In the future, we are going to provide more examples for users to try out. \ No newline at end of file diff --git a/python/dllib/src/bigdl/dllib/examples/keras/imdb_cnn_lstm.py b/python/dllib/src/bigdl/dllib/examples/keras/imdb_cnn_lstm.py new file mode 100644 index 00000000000..0ef776d4e04 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/examples/keras/imdb_cnn_lstm.py @@ -0,0 +1,97 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +# IMDB sentiment classification using a recurrent convolutional network on BigDL +# Reference: https://github.com/fchollet/keras/blob/1.2.2/examples/imdb_cnn_lstm.py +# The Keras version we support and test is Keras 1.2.2 with TensorFlow backend. + +from bigdl.examples.keras.keras_utils import * + + +def load_imdb(): + """ + Load IMDB dataset + Transform input data into an RDD of Sample + """ + from keras.preprocessing import sequence + from keras.datasets import imdb + (X_train, y_train), (X_test, y_test) = imdb.load_data(nb_words=20000) + X_train = sequence.pad_sequences(X_train, maxlen=100) + X_test = sequence.pad_sequences(X_test, maxlen=100) + return X_train, y_train, X_test, y_test + + +def build_keras_model(): + """ + Define a recurrent convolutional model in Keras 1.2.2 + """ + from keras.models import Sequential + from keras.layers import Dense, Dropout, Activation + from keras.layers import Embedding + from keras.layers import LSTM + from keras.layers import Convolution1D, MaxPooling1D + keras_model = Sequential() + keras_model.add(Embedding(20000, 128, input_length=100)) + keras_model.add(Dropout(0.25)) + keras_model.add(Convolution1D(nb_filter=64, + filter_length=5, + border_mode='valid', + activation='relu', + subsample_length=1)) + keras_model.add(MaxPooling1D(pool_length=4)) + keras_model.add(LSTM(70)) + keras_model.add(Dense(1)) + keras_model.add(Activation('sigmoid')) + return keras_model + + +if __name__ == "__main__": + keras_model = build_keras_model() + hdf5_path = "/tmp/imdb.h5" + keras_model.save(hdf5_path) + + from bigdl.util.common import * + from bigdl.nn.layer import * + from bigdl.optim.optimizer import * + from bigdl.nn.criterion import * + + # Load the HDF5 file with weights to a BigDL model + bigdl_model = Model.load_keras(hdf5_path=hdf5_path) + + sc = get_spark_context(conf=create_spark_conf()) + redire_spark_logs() + show_bigdl_info_logs() + init_engine() + + X_train, y_train, X_test, y_test = load_imdb() + train_data = to_sample_rdd(X_train, y_train) + test_data = to_sample_rdd(X_test, y_test) + + optimizer = Optimizer( + model=bigdl_model, + training_rdd=train_data, + criterion=BCECriterion(), + optim_method=Adam(), + end_trigger=MaxEpoch(2), + batch_size=32) + optimizer.set_validation( + batch_size=32, + val_rdd=test_data, + trigger=EveryEpoch(), + val_method=[Top1Accuracy()] + ) + optimizer.optimize() \ No newline at end of file diff --git a/python/dllib/src/bigdl/dllib/examples/keras/keras_utils.py b/python/dllib/src/bigdl/dllib/examples/keras/keras_utils.py new file mode 100644 index 00000000000..89d8a8b46cd --- /dev/null +++ b/python/dllib/src/bigdl/dllib/examples/keras/keras_utils.py @@ -0,0 +1,40 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from bigdl.util.common import * + + +def save_keras_definition(keras_model, path): + """ + Save a Keras model definition to JSON with given path + """ + model_json = keras_model.to_json() + with open(path, "w") as json_file: + json_file.write(model_json) + + +def dump_keras(keras_model, json_path=None, hdf5_path=None, dump_weights=False): + tmp_path = create_tmp_path() + if not json_path: + json_path = tmp_path + ".json" + if not hdf5_path: + hdf5_path = tmp_path + ".hdf5" + save_keras_definition(keras_model, json_path) + print("json path: " + json_path) + if dump_weights: + keras_model.save(hdf5_path) + print("hdf5 path: " + hdf5_path) + return json_path, hdf5_path diff --git a/python/dllib/src/bigdl/dllib/examples/keras/cnn.py b/python/dllib/src/bigdl/dllib/examples/keras/mnist_cnn.py similarity index 86% rename from python/dllib/src/bigdl/dllib/examples/keras/cnn.py rename to python/dllib/src/bigdl/dllib/examples/keras/mnist_cnn.py index 40f6e7bdce4..855e62fdd59 100644 --- a/python/dllib/src/bigdl/dllib/examples/keras/cnn.py +++ b/python/dllib/src/bigdl/dllib/examples/keras/mnist_cnn.py @@ -15,6 +15,14 @@ # +# MNIST CNN Example on BigDL +# Reference: https://github.com/fchollet/keras/blob/1.2.2/examples/mnist_cnn.py +# ../../models/lenet/lenet5.py +# The Keras version we support and test is Keras 1.2.2 with TensorFlow backend. + +from bigdl.examples.keras.keras_utils import * + + def get_mnist(sc, data_type="train", location="/tmp/mnist"): """ Download or load MNIST dataset. @@ -34,7 +42,7 @@ def get_mnist(sc, data_type="train", location="/tmp/mnist"): def build_keras_model(): """ - Define a convnet model in Keras + Define a convnet model in Keras 1.2.2 """ from keras.models import Sequential from keras.layers import Dense, Dropout, Activation, Flatten @@ -48,30 +56,19 @@ def build_keras_model(): keras_model.add(Activation('relu')) keras_model.add(MaxPooling2D(pool_size=(2, 2))) keras_model.add(Dropout(0.25)) - keras_model.add(Flatten()) keras_model.add(Dense(128)) keras_model.add(Activation('relu')) keras_model.add(Dropout(0.5)) keras_model.add(Dense(10)) keras_model.add(Activation('softmax')) - return keras_model -def save_keras_model(keras_model, path): - """ - Save a Keras model to JSON with given path - """ - model_json = keras_model.to_json() - with open(path, "w") as json_file: - json_file.write(model_json) - - if __name__ == "__main__": keras_model = build_keras_model() - def_path = "/tmp/lenet.json" - save_keras_model(keras_model, def_path) + json_path = "/tmp/lenet.json" + save_keras_definition(keras_model, json_path) from bigdl.util.common import * from bigdl.nn.layer import * @@ -79,7 +76,7 @@ def save_keras_model(keras_model, path): from bigdl.nn.criterion import * # Load the JSON file to a BigDL model - bigdl_model = Model.load_keras(def_path=def_path) + bigdl_model = Model.load_keras(json_path=json_path) sc = get_spark_context(conf=create_spark_conf()) redire_spark_logs() diff --git a/python/dllib/src/bigdl/dllib/keras/converter.py b/python/dllib/src/bigdl/dllib/keras/converter.py index 9b9a1de378f..cb7c0df6b25 100644 --- a/python/dllib/src/bigdl/dllib/keras/converter.py +++ b/python/dllib/src/bigdl/dllib/keras/converter.py @@ -343,10 +343,21 @@ def __to_bigdl(self): def from_kmodel(cls, kmodel): return cls(kmodel).__to_bigdl() + @classmethod + def from_hdf5_path(cls, hdf5_path): + """ + :param hdf5_path: hdf5 path which can be stored in a local file system, HDFS, S3, or any Hadoop-supported file system. + :return: BigDL Model + """ + from keras.models import load_model + hdf5_local_path = BCommon.get_local_file(hdf5_path) + kmodel = load_model(hdf5_local_path) + return kmodel, DefinitionLoader.from_kmodel(kmodel) + @classmethod def from_json_path(cls, json_path): """ - :param json_path: definition path which can be stored in a local file system, HDFS, S3, or any Hadoop-supported file system. + :param json_path: definition path which can be stored in a local file system, HDFS, S3, or any Hadoop-supported file system. :return: BigDL Model """ json_str = BCommon.text_from_path(json_path) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index eab65817470..157a254432b 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -744,25 +744,33 @@ def load_torch(path, bigdl_type="float"): return Layer.of(jmodel) @staticmethod - def load_keras(def_path, weights_path=None, by_name=False): + def load_keras(json_path=None, hdf5_path=None, by_name=False): """ Load a pre-trained Keras model. - :param def_path: The json path containing the keras model definition. - :param weights_path: The HDF5 path containing the pre-trained keras model weights. - :return: A pre-trained model. + :param json_path: The json path containing the keras model definition. + :param hdf5_path: The HDF5 path containing the pre-trained keras model weights with or without the model architecture. + :return: A bigdl model. """ import os try: import tensorflow except ImportError: os.environ['KERAS_BACKEND'] = "theano" - from theano import ifelse + try: + from theano import ifelse + except ImportError: + raise Exception("No backend is found for Keras. " + "Please install either tensorflow or theano.") from bigdl.keras.converter import DefinitionLoader, WeightLoader - if weights_path: - return WeightLoader.load_weights_from_json_hdf5(def_path, weights_path, by_name=by_name) - else: - return DefinitionLoader.from_json_path(def_path) + if json_path and not hdf5_path: + return DefinitionLoader.from_json_path(json_path) + elif json_path and hdf5_path: + return WeightLoader.load_weights_from_json_hdf5(json_path, hdf5_path, by_name=by_name) + elif hdf5_path and not json_path: + kmodel, bmodel = DefinitionLoader.from_hdf5_path(hdf5_path) + WeightLoader.load_weights_from_kmodel(bmodel, kmodel) + return bmodel @staticmethod def load_caffe(model, defPath, modelPath, match_all=True, bigdl_type="float"): From 37a99a71557e217112c38627db0d8a5645542e8e Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Mon, 18 Dec 2017 19:34:38 +0800 Subject: [PATCH 472/823] Add keras imdb example (#2039) * add imdb example; refactor load keras API * update yml * meet review --- .../src/bigdl/dllib/examples/keras/README.md | 11 +++ .../dllib/examples/keras/imdb_cnn_lstm.py | 97 +++++++++++++++++++ .../bigdl/dllib/examples/keras/keras_utils.py | 40 ++++++++ .../examples/keras/{cnn.py => mnist_cnn.py} | 27 +++--- .../dllib/src/bigdl/dllib/keras/converter.py | 13 ++- python/dllib/src/bigdl/dllib/nn/layer.py | 26 +++-- 6 files changed, 189 insertions(+), 25 deletions(-) create mode 100644 python/dllib/src/bigdl/dllib/examples/keras/README.md create mode 100644 python/dllib/src/bigdl/dllib/examples/keras/imdb_cnn_lstm.py create mode 100644 python/dllib/src/bigdl/dllib/examples/keras/keras_utils.py rename python/dllib/src/bigdl/dllib/examples/keras/{cnn.py => mnist_cnn.py} (86%) diff --git a/python/dllib/src/bigdl/dllib/examples/keras/README.md b/python/dllib/src/bigdl/dllib/examples/keras/README.md new file mode 100644 index 00000000000..2dd3e9138d9 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/examples/keras/README.md @@ -0,0 +1,11 @@ +# **Keras Examples** + +We provide several simple examples here to show how to load a Keras model into BigDL and running the model in a distributed fashion. + +Note that the Keras version we support and test is [__Keras 1.2.2__](https://faroit.github.io/keras-docs/1.2.2/) with TensorFlow backend. + +For the sake of illustration, in these examples, we first define a model in Keras, then save it as a JSON/HDF5 file and load it into BigDL. + +You can directly run these examples if you [install BigDL from pip](../../../../docs/docs/PythonUserGuide/install-from-pip.md). After the training, good accuracy can be achieved. + +In the future, we are going to provide more examples for users to try out. \ No newline at end of file diff --git a/python/dllib/src/bigdl/dllib/examples/keras/imdb_cnn_lstm.py b/python/dllib/src/bigdl/dllib/examples/keras/imdb_cnn_lstm.py new file mode 100644 index 00000000000..0ef776d4e04 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/examples/keras/imdb_cnn_lstm.py @@ -0,0 +1,97 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +# IMDB sentiment classification using a recurrent convolutional network on BigDL +# Reference: https://github.com/fchollet/keras/blob/1.2.2/examples/imdb_cnn_lstm.py +# The Keras version we support and test is Keras 1.2.2 with TensorFlow backend. + +from bigdl.examples.keras.keras_utils import * + + +def load_imdb(): + """ + Load IMDB dataset + Transform input data into an RDD of Sample + """ + from keras.preprocessing import sequence + from keras.datasets import imdb + (X_train, y_train), (X_test, y_test) = imdb.load_data(nb_words=20000) + X_train = sequence.pad_sequences(X_train, maxlen=100) + X_test = sequence.pad_sequences(X_test, maxlen=100) + return X_train, y_train, X_test, y_test + + +def build_keras_model(): + """ + Define a recurrent convolutional model in Keras 1.2.2 + """ + from keras.models import Sequential + from keras.layers import Dense, Dropout, Activation + from keras.layers import Embedding + from keras.layers import LSTM + from keras.layers import Convolution1D, MaxPooling1D + keras_model = Sequential() + keras_model.add(Embedding(20000, 128, input_length=100)) + keras_model.add(Dropout(0.25)) + keras_model.add(Convolution1D(nb_filter=64, + filter_length=5, + border_mode='valid', + activation='relu', + subsample_length=1)) + keras_model.add(MaxPooling1D(pool_length=4)) + keras_model.add(LSTM(70)) + keras_model.add(Dense(1)) + keras_model.add(Activation('sigmoid')) + return keras_model + + +if __name__ == "__main__": + keras_model = build_keras_model() + hdf5_path = "/tmp/imdb.h5" + keras_model.save(hdf5_path) + + from bigdl.util.common import * + from bigdl.nn.layer import * + from bigdl.optim.optimizer import * + from bigdl.nn.criterion import * + + # Load the HDF5 file with weights to a BigDL model + bigdl_model = Model.load_keras(hdf5_path=hdf5_path) + + sc = get_spark_context(conf=create_spark_conf()) + redire_spark_logs() + show_bigdl_info_logs() + init_engine() + + X_train, y_train, X_test, y_test = load_imdb() + train_data = to_sample_rdd(X_train, y_train) + test_data = to_sample_rdd(X_test, y_test) + + optimizer = Optimizer( + model=bigdl_model, + training_rdd=train_data, + criterion=BCECriterion(), + optim_method=Adam(), + end_trigger=MaxEpoch(2), + batch_size=32) + optimizer.set_validation( + batch_size=32, + val_rdd=test_data, + trigger=EveryEpoch(), + val_method=[Top1Accuracy()] + ) + optimizer.optimize() \ No newline at end of file diff --git a/python/dllib/src/bigdl/dllib/examples/keras/keras_utils.py b/python/dllib/src/bigdl/dllib/examples/keras/keras_utils.py new file mode 100644 index 00000000000..89d8a8b46cd --- /dev/null +++ b/python/dllib/src/bigdl/dllib/examples/keras/keras_utils.py @@ -0,0 +1,40 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from bigdl.util.common import * + + +def save_keras_definition(keras_model, path): + """ + Save a Keras model definition to JSON with given path + """ + model_json = keras_model.to_json() + with open(path, "w") as json_file: + json_file.write(model_json) + + +def dump_keras(keras_model, json_path=None, hdf5_path=None, dump_weights=False): + tmp_path = create_tmp_path() + if not json_path: + json_path = tmp_path + ".json" + if not hdf5_path: + hdf5_path = tmp_path + ".hdf5" + save_keras_definition(keras_model, json_path) + print("json path: " + json_path) + if dump_weights: + keras_model.save(hdf5_path) + print("hdf5 path: " + hdf5_path) + return json_path, hdf5_path diff --git a/python/dllib/src/bigdl/dllib/examples/keras/cnn.py b/python/dllib/src/bigdl/dllib/examples/keras/mnist_cnn.py similarity index 86% rename from python/dllib/src/bigdl/dllib/examples/keras/cnn.py rename to python/dllib/src/bigdl/dllib/examples/keras/mnist_cnn.py index 40f6e7bdce4..855e62fdd59 100644 --- a/python/dllib/src/bigdl/dllib/examples/keras/cnn.py +++ b/python/dllib/src/bigdl/dllib/examples/keras/mnist_cnn.py @@ -15,6 +15,14 @@ # +# MNIST CNN Example on BigDL +# Reference: https://github.com/fchollet/keras/blob/1.2.2/examples/mnist_cnn.py +# ../../models/lenet/lenet5.py +# The Keras version we support and test is Keras 1.2.2 with TensorFlow backend. + +from bigdl.examples.keras.keras_utils import * + + def get_mnist(sc, data_type="train", location="/tmp/mnist"): """ Download or load MNIST dataset. @@ -34,7 +42,7 @@ def get_mnist(sc, data_type="train", location="/tmp/mnist"): def build_keras_model(): """ - Define a convnet model in Keras + Define a convnet model in Keras 1.2.2 """ from keras.models import Sequential from keras.layers import Dense, Dropout, Activation, Flatten @@ -48,30 +56,19 @@ def build_keras_model(): keras_model.add(Activation('relu')) keras_model.add(MaxPooling2D(pool_size=(2, 2))) keras_model.add(Dropout(0.25)) - keras_model.add(Flatten()) keras_model.add(Dense(128)) keras_model.add(Activation('relu')) keras_model.add(Dropout(0.5)) keras_model.add(Dense(10)) keras_model.add(Activation('softmax')) - return keras_model -def save_keras_model(keras_model, path): - """ - Save a Keras model to JSON with given path - """ - model_json = keras_model.to_json() - with open(path, "w") as json_file: - json_file.write(model_json) - - if __name__ == "__main__": keras_model = build_keras_model() - def_path = "/tmp/lenet.json" - save_keras_model(keras_model, def_path) + json_path = "/tmp/lenet.json" + save_keras_definition(keras_model, json_path) from bigdl.util.common import * from bigdl.nn.layer import * @@ -79,7 +76,7 @@ def save_keras_model(keras_model, path): from bigdl.nn.criterion import * # Load the JSON file to a BigDL model - bigdl_model = Model.load_keras(def_path=def_path) + bigdl_model = Model.load_keras(json_path=json_path) sc = get_spark_context(conf=create_spark_conf()) redire_spark_logs() diff --git a/python/dllib/src/bigdl/dllib/keras/converter.py b/python/dllib/src/bigdl/dllib/keras/converter.py index 9b9a1de378f..cb7c0df6b25 100644 --- a/python/dllib/src/bigdl/dllib/keras/converter.py +++ b/python/dllib/src/bigdl/dllib/keras/converter.py @@ -343,10 +343,21 @@ def __to_bigdl(self): def from_kmodel(cls, kmodel): return cls(kmodel).__to_bigdl() + @classmethod + def from_hdf5_path(cls, hdf5_path): + """ + :param hdf5_path: hdf5 path which can be stored in a local file system, HDFS, S3, or any Hadoop-supported file system. + :return: BigDL Model + """ + from keras.models import load_model + hdf5_local_path = BCommon.get_local_file(hdf5_path) + kmodel = load_model(hdf5_local_path) + return kmodel, DefinitionLoader.from_kmodel(kmodel) + @classmethod def from_json_path(cls, json_path): """ - :param json_path: definition path which can be stored in a local file system, HDFS, S3, or any Hadoop-supported file system. + :param json_path: definition path which can be stored in a local file system, HDFS, S3, or any Hadoop-supported file system. :return: BigDL Model """ json_str = BCommon.text_from_path(json_path) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index eab65817470..157a254432b 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -744,25 +744,33 @@ def load_torch(path, bigdl_type="float"): return Layer.of(jmodel) @staticmethod - def load_keras(def_path, weights_path=None, by_name=False): + def load_keras(json_path=None, hdf5_path=None, by_name=False): """ Load a pre-trained Keras model. - :param def_path: The json path containing the keras model definition. - :param weights_path: The HDF5 path containing the pre-trained keras model weights. - :return: A pre-trained model. + :param json_path: The json path containing the keras model definition. + :param hdf5_path: The HDF5 path containing the pre-trained keras model weights with or without the model architecture. + :return: A bigdl model. """ import os try: import tensorflow except ImportError: os.environ['KERAS_BACKEND'] = "theano" - from theano import ifelse + try: + from theano import ifelse + except ImportError: + raise Exception("No backend is found for Keras. " + "Please install either tensorflow or theano.") from bigdl.keras.converter import DefinitionLoader, WeightLoader - if weights_path: - return WeightLoader.load_weights_from_json_hdf5(def_path, weights_path, by_name=by_name) - else: - return DefinitionLoader.from_json_path(def_path) + if json_path and not hdf5_path: + return DefinitionLoader.from_json_path(json_path) + elif json_path and hdf5_path: + return WeightLoader.load_weights_from_json_hdf5(json_path, hdf5_path, by_name=by_name) + elif hdf5_path and not json_path: + kmodel, bmodel = DefinitionLoader.from_hdf5_path(hdf5_path) + WeightLoader.load_weights_from_kmodel(bmodel, kmodel) + return bmodel @staticmethod def load_caffe(model, defPath, modelPath, match_all=True, bigdl_type="float"): From 54b4b506f5180433bed48721f54814bf666f964f Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Mon, 18 Dec 2017 19:34:38 +0800 Subject: [PATCH 473/823] Add keras imdb example (#2039) * add imdb example; refactor load keras API * update yml * meet review --- python/dllib/test/bigdl/keras/test_layer.py | 3 ++- .../dllib/test/bigdl/keras/test_load_model.py | 23 +++++++++++++------ python/dllib/test/bigdl/test_utils.py | 16 +++---------- 3 files changed, 21 insertions(+), 21 deletions(-) diff --git a/python/dllib/test/bigdl/keras/test_layer.py b/python/dllib/test/bigdl/keras/test_layer.py index 39b64949b25..fe65b294260 100644 --- a/python/dllib/test/bigdl/keras/test_layer.py +++ b/python/dllib/test/bigdl/keras/test_layer.py @@ -25,6 +25,7 @@ from keras.regularizers import l1, l2, l1l2 from bigdl.keras.converter import * from test.bigdl.test_utils import BigDLTestCase +from bigdl.examples.keras.keras_utils import * class TestLayer(BigDLTestCase): @@ -637,7 +638,7 @@ def test_multiple_inputs_share_weights(self): out3 = Dense(7)(tensor1) out4 = Dense(8)(tensor2) model2 = Model(input=[input_node1, input_node2], output=[out3, out4]) - def_path, w_path = self._dump_keras(model2) + def_path, w_path = dump_keras(model2) bigdl_model = DefinitionLoader.from_json_path(def_path) assert str(excinfo.value) == """Convolution2D doesn't support multiple inputs with shared weights""" # noqa diff --git a/python/dllib/test/bigdl/keras/test_load_model.py b/python/dllib/test/bigdl/keras/test_load_model.py index 1d607bd7518..0be15f22b02 100644 --- a/python/dllib/test/bigdl/keras/test_load_model.py +++ b/python/dllib/test/bigdl/keras/test_load_model.py @@ -25,13 +25,14 @@ np.random.seed(1337) # for reproducibility from test.bigdl.test_utils import BigDLTestCase, TestModels +from bigdl.examples.keras.keras_utils import * import keras.backend as K class TestLoadModel(BigDLTestCase): def __kmodel_load_def_weight_test(self, kmodel, input_data): - keras_model_path_json, keras_model_path_hdf5 = self._dump_keras(kmodel, dump_weights=True) + keras_model_path_json, keras_model_path_hdf5 = dump_keras(kmodel, dump_weights=True) bmodel = DefinitionLoader.from_json_path(keras_model_path_json) WeightLoader.load_weights_from_hdf5(bmodel, kmodel, @@ -44,16 +45,24 @@ def __kmodel_load_def_weight_test(self, kmodel, input_data): def test_load_api_with_hdf5(self): K.set_image_dim_ordering("th") kmodel, input_data, output_data = TestModels.kmodel_graph_1_layer() - keras_model_json_path, keras_model_hdf5_path = self._dump_keras(kmodel, dump_weights=True) - bmodel = BLayer.Model.load_keras(keras_model_json_path, keras_model_hdf5_path) + keras_model_json_path, keras_model_hdf5_path = dump_keras(kmodel, dump_weights=True) + bmodel = BLayer.Model.load_keras(json_path=keras_model_json_path, + hdf5_path=keras_model_hdf5_path) + self.assert_allclose(kmodel.predict(input_data), + bmodel.forward(input_data)) + + def test_load_model_with_hdf5_with_definition(self): + kmodel, input_data, output_data = TestModels.kmodel_graph_1_layer() + keras_model_json_path, keras_model_hdf5_path = dump_keras(kmodel, dump_weights=True) + bmodel = BLayer.Model.load_keras(hdf5_path=keras_model_hdf5_path) self.assert_allclose(kmodel.predict(input_data), bmodel.forward(input_data)) def test_load_api_no_hdf5(self): K.set_image_dim_ordering("th") kmodel, input_data, output_data = TestModels.kmodel_graph_1_layer() - keras_model_json_path, keras_model_hdf5_path = self._dump_keras(kmodel, dump_weights=True) - bmodel = BLayer.Model.load_keras(keras_model_json_path) + keras_model_json_path, keras_model_hdf5_path = dump_keras(kmodel, dump_weights=True) + bmodel = BLayer.Model.load_keras(json_path=keras_model_json_path) def test_load_def_weights_graph_1_layer(self): K.set_image_dim_ordering("th") @@ -73,7 +82,7 @@ def test_load_def_weights_kmodel_seq_lenet_mnist(self): def test_load_definition(self): K.set_image_dim_ordering("th") kmodel, input_data, output_data = TestModels.kmodel_seq_lenet_mnist() - keras_model_json_path, keras_model_hdf5_path = self._dump_keras(kmodel, dump_weights=True) + keras_model_json_path, keras_model_hdf5_path = dump_keras(kmodel, dump_weights=True) bmodel = DefinitionLoader.from_json_path(keras_model_json_path) WeightLoader.load_weights_from_kmodel(bmodel, kmodel) self.assert_allclose(bmodel.forward(input_data), kmodel.predict(input_data)) @@ -81,7 +90,7 @@ def test_load_definition(self): def test_load_weights(self): K.set_image_dim_ordering("th") kmodel, input_data, output_data = TestModels.kmodel_graph_1_layer() - keras_model_json_path, keras_model_hdf5_path = self._dump_keras(kmodel, dump_weights=True) + keras_model_json_path, keras_model_hdf5_path = dump_keras(kmodel, dump_weights=True) bmodel = DefinitionLoader.from_json_path(keras_model_json_path) kmodel.set_weights([kmodel.get_weights()[0] + 100, kmodel.get_weights()[1]]) WeightLoader.load_weights_from_hdf5(bmodel, kmodel, filepath=keras_model_hdf5_path) diff --git a/python/dllib/test/bigdl/test_utils.py b/python/dllib/test/bigdl/test_utils.py index e29b41fe211..fc69aba96b1 100644 --- a/python/dllib/test/bigdl/test_utils.py +++ b/python/dllib/test/bigdl/test_utils.py @@ -27,6 +27,7 @@ import numpy as np from unittest import TestCase import keras +from bigdl.examples.keras.keras_utils import * class TestModels: @@ -149,18 +150,6 @@ def _load_keras(self, json_path, hdf5_path): WeightLoader.load_weights_from_hdf5(bmodel, kmodel, hdf5_path) return kmodel, bmodel - def _dump_keras(self, keras_model, dump_weights=False): - keras_model_path = create_tmp_path() - keras_model_json_path = keras_model_path + ".json" - keras_model_hdf5_path = keras_model_path + ".hdf5" - with open(keras_model_json_path, "w") as json_file: - json_file.write(keras_model.to_json()) - print("json path: " + keras_model_json_path) - if dump_weights: - keras_model.save(keras_model_hdf5_path) - print("hdf5 path: " + keras_model_hdf5_path) - return keras_model_json_path, keras_model_hdf5_path - def assert_allclose(self, a, b, rtol=1e-6, atol=1e-6, msg=None): # from tensorflow self.assertEqual(a.shape, b.shape, "Shape mismatch: expected %s, got %s." % @@ -211,7 +200,8 @@ def modelTest(self, new_kweights = self.__generate_random_weights(kweights) keras_model.set_weights(new_kweights) # weight_converter is a function keras [ndarray]-> bigdl [ndarray] - keras_model_json_path, keras_model_hdf5_path = self._dump_keras(keras_model, dump_weights) + keras_model_json_path, keras_model_hdf5_path = dump_keras(keras_model, + dump_weights=dump_weights) # Use Theano backend to load as a bigdl model self.__set_keras_backend("theano") From 29c3e4d62dd83c8b3404b94a6caf85a83108427a Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Mon, 18 Dec 2017 19:44:38 +0800 Subject: [PATCH 474/823] Support n-dimensional Dense (#2048) * update * nD dense --- .../dllib/src/bigdl/dllib/keras/converter.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/keras/converter.py b/python/dllib/src/bigdl/dllib/keras/converter.py index cb7c0df6b25..4a0e6f20468 100644 --- a/python/dllib/src/bigdl/dllib/keras/converter.py +++ b/python/dllib/src/bigdl/dllib/keras/converter.py @@ -491,16 +491,25 @@ def create_dense(self): # We don't need to respect the tensor index for method `get_input_shape_at` # which is internal implementation and `get_input_shape_at` has hided that for us, # What we need to use is the input index, not node index, not tensor index. - if len(self.input_shape) > 2: - raise Exception("Input for Dense must be 1D or 2D") + + out_dim = self.config["output_dim"] + in_dim = int(self.input_shape[-1]) blayer = BLayer.Linear( - input_size=int(self.input_shape[-1]), # last dim is input_dim - output_size=self.config["output_dim"], + input_size=in_dim, + output_size=out_dim, with_bias=self.config["bias"], wRegularizer=self.to_bigdl_reg(self.config["W_regularizer"]), bRegularizer=self.to_bigdl_reg(self.config["b_regularizer"]) ) - return self.combo_parameter_layer(blayer, self.config) + + if len(self.input_shape) <= 2: + return self.combo_parameter_layer(blayer, self.config) + else: + seq = BLayer.Sequential() + seq.add(BLayer.InferReshape([-1, out_dim], False)) + seq.add(blayer) + seq.add(BLayer.InferReshape([-1] + list(self.input_shape[1:-1]) + [out_dim], False)) + return self.combo_parameter_layer(seq, self.config) def create_timedistributeddense(self): blayer = BLayer.TimeDistributed(BLayer.Linear( From a7f353bb45e975b1d4e8c83b5953ad730c40c403 Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Mon, 18 Dec 2017 19:44:38 +0800 Subject: [PATCH 475/823] Support n-dimensional Dense (#2048) * update * nD dense --- .../dllib/src/bigdl/dllib/keras/converter.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/keras/converter.py b/python/dllib/src/bigdl/dllib/keras/converter.py index cb7c0df6b25..4a0e6f20468 100644 --- a/python/dllib/src/bigdl/dllib/keras/converter.py +++ b/python/dllib/src/bigdl/dllib/keras/converter.py @@ -491,16 +491,25 @@ def create_dense(self): # We don't need to respect the tensor index for method `get_input_shape_at` # which is internal implementation and `get_input_shape_at` has hided that for us, # What we need to use is the input index, not node index, not tensor index. - if len(self.input_shape) > 2: - raise Exception("Input for Dense must be 1D or 2D") + + out_dim = self.config["output_dim"] + in_dim = int(self.input_shape[-1]) blayer = BLayer.Linear( - input_size=int(self.input_shape[-1]), # last dim is input_dim - output_size=self.config["output_dim"], + input_size=in_dim, + output_size=out_dim, with_bias=self.config["bias"], wRegularizer=self.to_bigdl_reg(self.config["W_regularizer"]), bRegularizer=self.to_bigdl_reg(self.config["b_regularizer"]) ) - return self.combo_parameter_layer(blayer, self.config) + + if len(self.input_shape) <= 2: + return self.combo_parameter_layer(blayer, self.config) + else: + seq = BLayer.Sequential() + seq.add(BLayer.InferReshape([-1, out_dim], False)) + seq.add(blayer) + seq.add(BLayer.InferReshape([-1] + list(self.input_shape[1:-1]) + [out_dim], False)) + return self.combo_parameter_layer(seq, self.config) def create_timedistributeddense(self): blayer = BLayer.TimeDistributed(BLayer.Linear( From f43d2c5440d4688c4e790e15bd528d0e630d31ab Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Mon, 18 Dec 2017 19:44:38 +0800 Subject: [PATCH 476/823] Support n-dimensional Dense (#2048) * update * nD dense --- python/dllib/test/bigdl/keras/test_layer.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/dllib/test/bigdl/keras/test_layer.py b/python/dllib/test/bigdl/keras/test_layer.py index fe65b294260..d8b021cae15 100644 --- a/python/dllib/test/bigdl/keras/test_layer.py +++ b/python/dllib/test/bigdl/keras/test_layer.py @@ -66,9 +66,10 @@ def test_softsign(self): self.modelTestSingleLayer(input_data, layer) def test_dense(self): - input_data = np.random.random_sample([1, 10]) + input_data = np.random.random_sample([2, 10, 5, 7]) layer = Dense(2, init='one', activation="relu", input_shape=(10, ), W_regularizer=l1l2(l1=0.01, l2=0.02)) + input_data = np.random.random_sample([2, 10]) self.modelTestSingleLayer(input_data, layer, dump_weights=True) layer2 = Dense(2, init='one', activation="softplus", input_shape=(10, ), b_regularizer=l2(0.02)) From a15ff1b519444c7e62cc23cb46835761501ddb5b Mon Sep 17 00:00:00 2001 From: Ian Wong Date: Mon, 18 Dec 2017 20:17:29 +0800 Subject: [PATCH 477/823] bump bigdl version (#2049) --- version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.py b/version.py index a143c2249e8..81df44d6a63 100644 --- a/version.py +++ b/version.py @@ -14,4 +14,4 @@ # limitations under the License. # -__version__ = "0.4.0.dev0" +__version__ = "0.5.0.dev0" From fd1e9f899c39c8abeff75da559e8ecab8ebcc71c Mon Sep 17 00:00:00 2001 From: Ian Wong Date: Mon, 18 Dec 2017 20:17:29 +0800 Subject: [PATCH 478/823] bump bigdl version (#2049) --- version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.py b/version.py index a143c2249e8..81df44d6a63 100644 --- a/version.py +++ b/version.py @@ -14,4 +14,4 @@ # limitations under the License. # -__version__ = "0.4.0.dev0" +__version__ = "0.5.0.dev0" From 24951d8ad807de01a960454cb80cdbf002b5cb2a Mon Sep 17 00:00:00 2001 From: Guoqiong Song Date: Mon, 18 Dec 2017 12:54:21 -0800 Subject: [PATCH 479/823] Locallyconnected1D (#1964) * add LocallyConnected1D, address review comments, resolve conflicts, add serializer test --- python/dllib/src/bigdl/dllib/nn/layer.py | 64 ++++++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 157a254432b..39f3282517c 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -1151,6 +1151,70 @@ def set_init_method(self, weight_init_method = None, bias_init_method = None): weight_init_method, bias_init_method) return self +class LocallyConnected1D(Layer): + ''' + The `LocallyConnected1D` layer works similarly to + the `TemporalConvolution` layer, except that weights are unshared, + that is, a different set of filters is applied at each different patch + of the input. + The input tensor in `forward(input)` is expected to be a 2D tensor + (`nInputFrame` x `inputFrameSize`) or a 3D tensor + (`nBatchFrame` x `nInputFrame` x `inputFrameSize`). + :param nInputFrame the input frame channel + :param input_frame_size The input frame size expected in sequences given into `forward()` + :param output_frame_size The output frame size the convolution layer will produce. + :param kernel_w The kernel width of the convolution + :param stride_w The step of the convolution in the width dimension. + :param propagate_back Whether propagate gradient back, default is true. + :param weight_regularizer instance of [[Regularizer]] + (eg. L1 or L2 regularization), applied to the input weights matrices. + :param bias_regularizer instance of [[Regularizer]] + applied to the bias. + :param init_weight Initial weight + :param init_bias Initial bias + :param init_grad_weight Initial gradient weight + :param init_grad_bias Initial gradient bias + >>> locallyConnected1D = LocallyConnected1D(10, 6, 12, 5, 5) + creating: createLocallyConnected1D + >>> locallyConnected1D.setWRegularizer(L1Regularizer(0.5)) + creating: createL1Regularizer + >>> locallyConnected1D.setBRegularizer(L1Regularizer(0.5)) + creating: createL1Regularizer + ''' + + def __init__(self, + n_input_frame, + input_frame_size, + output_frame_size, + kernel_w, + stride_w=1, + propagate_back=True, + weight_regularizer=None, + bias_regularizer=None, + init_weight=None, + init_bias=None, + init_grad_weight=None, + init_grad_bias=None, + bigdl_type="float"): + super(LocallyConnected1D, self).__init__(None, bigdl_type, + n_input_frame, + input_frame_size, + output_frame_size, + kernel_w, + stride_w, + propagate_back, + weight_regularizer, + bias_regularizer, + JTensor.from_ndarray(init_weight), + JTensor.from_ndarray(init_bias), + JTensor.from_ndarray(init_grad_weight), + JTensor.from_ndarray(init_grad_bias)) + + def set_init_method(self, weight_init_method=None, bias_init_method=None): + callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, + weight_init_method, bias_init_method) + return self + class BinaryTreeLSTM(Layer): ''' This class is an implementation of Binary TreeLSTM (Constituency Tree LSTM). From a74db8fa59fdb0e6a6f154032d18a9567a022f0a Mon Sep 17 00:00:00 2001 From: Guoqiong Song Date: Mon, 18 Dec 2017 12:54:21 -0800 Subject: [PATCH 480/823] Locallyconnected1D (#1964) * add LocallyConnected1D, address review comments, resolve conflicts, add serializer test --- python/dllib/src/bigdl/dllib/nn/layer.py | 64 ++++++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 157a254432b..39f3282517c 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -1151,6 +1151,70 @@ def set_init_method(self, weight_init_method = None, bias_init_method = None): weight_init_method, bias_init_method) return self +class LocallyConnected1D(Layer): + ''' + The `LocallyConnected1D` layer works similarly to + the `TemporalConvolution` layer, except that weights are unshared, + that is, a different set of filters is applied at each different patch + of the input. + The input tensor in `forward(input)` is expected to be a 2D tensor + (`nInputFrame` x `inputFrameSize`) or a 3D tensor + (`nBatchFrame` x `nInputFrame` x `inputFrameSize`). + :param nInputFrame the input frame channel + :param input_frame_size The input frame size expected in sequences given into `forward()` + :param output_frame_size The output frame size the convolution layer will produce. + :param kernel_w The kernel width of the convolution + :param stride_w The step of the convolution in the width dimension. + :param propagate_back Whether propagate gradient back, default is true. + :param weight_regularizer instance of [[Regularizer]] + (eg. L1 or L2 regularization), applied to the input weights matrices. + :param bias_regularizer instance of [[Regularizer]] + applied to the bias. + :param init_weight Initial weight + :param init_bias Initial bias + :param init_grad_weight Initial gradient weight + :param init_grad_bias Initial gradient bias + >>> locallyConnected1D = LocallyConnected1D(10, 6, 12, 5, 5) + creating: createLocallyConnected1D + >>> locallyConnected1D.setWRegularizer(L1Regularizer(0.5)) + creating: createL1Regularizer + >>> locallyConnected1D.setBRegularizer(L1Regularizer(0.5)) + creating: createL1Regularizer + ''' + + def __init__(self, + n_input_frame, + input_frame_size, + output_frame_size, + kernel_w, + stride_w=1, + propagate_back=True, + weight_regularizer=None, + bias_regularizer=None, + init_weight=None, + init_bias=None, + init_grad_weight=None, + init_grad_bias=None, + bigdl_type="float"): + super(LocallyConnected1D, self).__init__(None, bigdl_type, + n_input_frame, + input_frame_size, + output_frame_size, + kernel_w, + stride_w, + propagate_back, + weight_regularizer, + bias_regularizer, + JTensor.from_ndarray(init_weight), + JTensor.from_ndarray(init_bias), + JTensor.from_ndarray(init_grad_weight), + JTensor.from_ndarray(init_grad_bias)) + + def set_init_method(self, weight_init_method=None, bias_init_method=None): + callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, + weight_init_method, bias_init_method) + return self + class BinaryTreeLSTM(Layer): ''' This class is an implementation of Binary TreeLSTM (Constituency Tree LSTM). From 28db7c44cb66f30b85600f695a1a2e54f8a412eb Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Wed, 20 Dec 2017 18:01:29 +0800 Subject: [PATCH 481/823] Add Keras unsupported argument list doc (#2065) * add unsupported list * meet review * rename and update * meet review * minor update * fix style --- python/dllib/src/bigdl/dllib/keras/converter.py | 15 +++++++-------- python/dllib/src/bigdl/dllib/nn/layer.py | 2 +- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/keras/converter.py b/python/dllib/src/bigdl/dllib/keras/converter.py index 4a0e6f20468..8db1917d255 100644 --- a/python/dllib/src/bigdl/dllib/keras/converter.py +++ b/python/dllib/src/bigdl/dllib/keras/converter.py @@ -257,6 +257,8 @@ def convert_maxoutdense(klayer, weights): b_weights = k_weights[0].T for i in range(1, k_weights.shape[0]): b_weights = np.concatenate((b_weights, k_weights[i].T)) + if len(weights) == 1: # if without bias + return [b_weights] return [b_weights, weights[1].reshape(k_weights.shape[0]*k_weights.shape[2], )] @staticmethod @@ -1291,7 +1293,7 @@ def create_deconvolution2d(self): adj_w=0, adj_h=0, n_group=1, - no_bias=False, + no_bias=not self.klayer.bias, wRegularizer=self.to_bigdl_reg(self.config["W_regularizer"]), bRegularizer=self.to_bigdl_reg(self.config["b_regularizer"]), bigdl_type="float") @@ -1622,15 +1624,12 @@ def create_highway(self): return blayer def create_maxoutdense(self): - if self.config["W_regularizer"]: - raise Exception("W_regularizer is not supported for MaxoutDense") - if self.config["b_regularizer"]: - raise Exception("b_regularizer is not supported for MaxoutDense") - if not self.config["bias"]: - raise Exception("Only bias=True is supported for MaxoutDense") blayer = BLayer.Maxout(input_size=self.input_shape[1], output_size=self.klayer.output_dim, - maxout_number=self.klayer.nb_feature) + maxout_number=self.klayer.nb_feature, + with_bias=self.klayer.bias, + w_regularizer=self.to_bigdl_reg(self.config["W_regularizer"]), + b_regularizer=self.to_bigdl_reg(self.config["b_regularizer"])) return blayer def create_masking(self): diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 39f3282517c..41d38acf14c 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -5139,7 +5139,7 @@ def __init__(self, input_size, output_size, maxout_number, - with_bias = True, + with_bias=True, w_regularizer=None, b_regularizer=None, init_weight=None, From b2569616ecd6c8e7905a02956857f524c2230979 Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Wed, 20 Dec 2017 18:01:29 +0800 Subject: [PATCH 482/823] Add Keras unsupported argument list doc (#2065) * add unsupported list * meet review * rename and update * meet review * minor update * fix style --- python/dllib/src/bigdl/dllib/keras/converter.py | 15 +++++++-------- python/dllib/src/bigdl/dllib/nn/layer.py | 2 +- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/keras/converter.py b/python/dllib/src/bigdl/dllib/keras/converter.py index 4a0e6f20468..8db1917d255 100644 --- a/python/dllib/src/bigdl/dllib/keras/converter.py +++ b/python/dllib/src/bigdl/dllib/keras/converter.py @@ -257,6 +257,8 @@ def convert_maxoutdense(klayer, weights): b_weights = k_weights[0].T for i in range(1, k_weights.shape[0]): b_weights = np.concatenate((b_weights, k_weights[i].T)) + if len(weights) == 1: # if without bias + return [b_weights] return [b_weights, weights[1].reshape(k_weights.shape[0]*k_weights.shape[2], )] @staticmethod @@ -1291,7 +1293,7 @@ def create_deconvolution2d(self): adj_w=0, adj_h=0, n_group=1, - no_bias=False, + no_bias=not self.klayer.bias, wRegularizer=self.to_bigdl_reg(self.config["W_regularizer"]), bRegularizer=self.to_bigdl_reg(self.config["b_regularizer"]), bigdl_type="float") @@ -1622,15 +1624,12 @@ def create_highway(self): return blayer def create_maxoutdense(self): - if self.config["W_regularizer"]: - raise Exception("W_regularizer is not supported for MaxoutDense") - if self.config["b_regularizer"]: - raise Exception("b_regularizer is not supported for MaxoutDense") - if not self.config["bias"]: - raise Exception("Only bias=True is supported for MaxoutDense") blayer = BLayer.Maxout(input_size=self.input_shape[1], output_size=self.klayer.output_dim, - maxout_number=self.klayer.nb_feature) + maxout_number=self.klayer.nb_feature, + with_bias=self.klayer.bias, + w_regularizer=self.to_bigdl_reg(self.config["W_regularizer"]), + b_regularizer=self.to_bigdl_reg(self.config["b_regularizer"])) return blayer def create_masking(self): diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 39f3282517c..41d38acf14c 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -5139,7 +5139,7 @@ def __init__(self, input_size, output_size, maxout_number, - with_bias = True, + with_bias=True, w_regularizer=None, b_regularizer=None, init_weight=None, From 9d350db5723b58d5e52255c242465b419683a9ab Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Wed, 20 Dec 2017 18:01:29 +0800 Subject: [PATCH 483/823] Add Keras unsupported argument list doc (#2065) * add unsupported list * meet review * rename and update * meet review * minor update * fix style --- python/dllib/test/bigdl/keras/test_layer.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/python/dllib/test/bigdl/keras/test_layer.py b/python/dllib/test/bigdl/keras/test_layer.py index d8b021cae15..667c6f0d644 100644 --- a/python/dllib/test/bigdl/keras/test_layer.py +++ b/python/dllib/test/bigdl/keras/test_layer.py @@ -160,6 +160,14 @@ def test_conv3D(self): input_shape=(3, 32, 32, 32)) self.modelTestSingleLayerWithOrdersModes(input_data, layer, dim_orderings=["th"], dump_weights=True, rtol=1e-5, atol=1e-5) + layer2 = lambda: Convolution3D(8, 6, 4, 2, dim_ordering="th", activation='sigmoid', + input_shape=(3, 32, 32, 32)) + self.modelTestSingleLayerWithOrdersModes(input_data, layer2, dim_orderings=["th"], + dump_weights=True, rtol=1e-5, atol=1e-5) + layer3 = lambda: Convolution3D(16, 2, 2, 2, dim_ordering="th", bias=False, + input_shape=(3, 32, 32, 32)) + self.modelTestSingleLayerWithOrdersModes(input_data, layer3, dim_orderings=["th"], + dump_weights=True, rtol=1e-5, atol=1e-5) def test_atrousconvolution1d(self): input_data = np.random.random_sample([2, 10, 32]) @@ -187,6 +195,10 @@ def test_deconvolution2d(self): border_mode="same", subsample=(2, 2), dim_ordering="th", input_shape=(3, 12, 12)) self.modelTestSingleLayer(input_data, layer3, dump_weights=True) + layer4 = Deconvolution2D(3, 3, 3, output_shape=(None, 3, 14, 14), + border_mode="valid", dim_ordering="th", + bias=False, activation='relu', input_shape=(3, 12, 12)) + self.modelTestSingleLayer(input_data, layer4, dump_weights=True) def test_maxpooling3d(self): input_data = np.random.random_sample([1, 3, 20, 15, 35]) @@ -705,6 +717,8 @@ def test_maxoutdense(self): input_data = np.random.random([4, 6]) layer = MaxoutDense(3, 5) self.modelTestSingleLayer(input_data, layer, dump_weights=True) + layer2 = MaxoutDense(4, 2, bias=False) + self.modelTestSingleLayer(input_data, layer2, dump_weights=True) def test_masking(self): input_data = np.array([[[0, 1, 2], [-1, 1, 0], [3, 4, 1], [0, 0, 0]]]) From ff101adbef9d75e5c0b0b93f9f6aff43147dbbcb Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Wed, 20 Dec 2017 18:55:08 +0800 Subject: [PATCH 484/823] fix (#2073) --- python/dllib/src/bigdl/dllib/optim/optimizer.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/optim/optimizer.py b/python/dllib/src/bigdl/dllib/optim/optimizer.py index 4051472957a..a1bfcaaa9ad 100644 --- a/python/dllib/src/bigdl/dllib/optim/optimizer.py +++ b/python/dllib/src/bigdl/dllib/optim/optimizer.py @@ -668,9 +668,9 @@ def __init__(self, def create(model, training_set, criterion, - end_trigger=MaxEpoch(1), + end_trigger=None, batch_size=32, - optim_method=SGD(), + optim_method=None, cores=None, bigdl_type="float"): """ @@ -683,10 +683,14 @@ def create(model, :param criterion: the loss function :param optim_method: the algorithm to use for optimization, e.g. SGD, Adagrad, etc. If optim_method is None, the default algorithm is SGD. - :param end_trigger: when to end the optimization + :param end_trigger: when to end the optimization. default value is MapEpoch(1) :param batch_size: training batch size :param cores: This is for local optimizer only and use total physical cores as the default value """ + if not end_trigger: + end_trigger = MaxEpoch(1) + if not optim_method: + optim_method = SGD() if isinstance(training_set, RDD): return DistriOptimizer(model=model, training_rdd=training_set, From 119250cc9b1a89b9317ef44b482841d95dec17dd Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Wed, 20 Dec 2017 18:55:08 +0800 Subject: [PATCH 485/823] fix (#2073) --- python/dllib/src/bigdl/dllib/optim/optimizer.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/optim/optimizer.py b/python/dllib/src/bigdl/dllib/optim/optimizer.py index 4051472957a..a1bfcaaa9ad 100644 --- a/python/dllib/src/bigdl/dllib/optim/optimizer.py +++ b/python/dllib/src/bigdl/dllib/optim/optimizer.py @@ -668,9 +668,9 @@ def __init__(self, def create(model, training_set, criterion, - end_trigger=MaxEpoch(1), + end_trigger=None, batch_size=32, - optim_method=SGD(), + optim_method=None, cores=None, bigdl_type="float"): """ @@ -683,10 +683,14 @@ def create(model, :param criterion: the loss function :param optim_method: the algorithm to use for optimization, e.g. SGD, Adagrad, etc. If optim_method is None, the default algorithm is SGD. - :param end_trigger: when to end the optimization + :param end_trigger: when to end the optimization. default value is MapEpoch(1) :param batch_size: training batch size :param cores: This is for local optimizer only and use total physical cores as the default value """ + if not end_trigger: + end_trigger = MaxEpoch(1) + if not optim_method: + optim_method = SGD() if isinstance(training_set, RDD): return DistriOptimizer(model=model, training_rdd=training_set, From f62598ecfa9f98cd2e273795bf19b21832238e76 Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Thu, 21 Dec 2017 11:17:15 +0800 Subject: [PATCH 486/823] Map Keras LocallyConnected1D (#2060) * add locallyconnected1d * fix style --- .../dllib/src/bigdl/dllib/keras/converter.py | 53 +++++++++++++++---- 1 file changed, 43 insertions(+), 10 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/keras/converter.py b/python/dllib/src/bigdl/dllib/keras/converter.py index 8db1917d255..bde8cd59d8f 100644 --- a/python/dllib/src/bigdl/dllib/keras/converter.py +++ b/python/dllib/src/bigdl/dllib/keras/converter.py @@ -275,6 +275,13 @@ def convert_separableconvolution2d(klayer, weights): return [weights[0], weights[1], np.zeros(bias, )] return weights + @staticmethod + def convert_locallyconnected1d(klayer, weights): + bweights1 = np.transpose(weights[0], (0, 2, 1)) + if len(weights) == 1: # if without bias + return [bweights1] + return[bweights1, weights[1]] + @staticmethod def convert_locallyconnected2d(klayer, weights): bweights1 = np.transpose(weights[0], (0, 2, 1)) @@ -468,7 +475,7 @@ def create(self): (hasattr(self.klayer, "W_constraint") and self.klayer.W_constraint): raise Exception("We don't support constraint for now") - if (hasattr(self.klayer, "activity_regularizer") and self.klayer.activity_regularizer): + if hasattr(self.klayer, "activity_regularizer") and self.klayer.activity_regularizer: raise Exception("We don't support activity_regularizer for now") function_name = "create_" + class_name.lower() @@ -555,10 +562,10 @@ def create_embedding(self): raise Exception( "The input_length doesn't match: %s vs %s" % (seq_len, self.klayer.input_length)) - if (hasattr(self.klayer, "dropout") and self.klayer.dropout != 0): + if hasattr(self.klayer, "dropout") and self.klayer.dropout != 0: raise Exception("We don't support dropout for now") - if (hasattr(self.klayer, "mask_zero") and self.klayer.mask_zero != False): + if hasattr(self.klayer, "mask_zero") and self.klayer.mask_zero != False: raise Exception("We don't support mask_zero for now") bseq = BLayer.Sequential() @@ -1192,7 +1199,7 @@ def create_atrousconvolution1d(self): if not self.config["bias"]: raise Exception("Only bias=True is supported for AtrousConvolution1D") - h = self.input_shape[2] + h = int(self.input_shape[1]) kh = self.config["filter_length"] dh = self.config["subsample_length"] dilation_h = self.config["atrous_rate"] @@ -1226,8 +1233,8 @@ def create_atrousconvolution2d(self): if not self.config["bias"]: raise Exception("Only bias=True is supported for AtrousConvolution2D") - h = self.input_shape[2] - w = self.input_shape[3] + h = int(self.input_shape[2]) + w = int(self.input_shape[3]) kh = self.config["nb_row"] kw = self.config["nb_col"] dh = self.config["subsample"][0] @@ -1258,8 +1265,8 @@ def create_deconvolution2d(self): raise Exception("Please use `th` for `dim_ordering`. `%s` is not supported for now." % self.klayer.dim_ordering) output_shape = self.config["output_shape"] - h = self.input_shape[2] - w = self.input_shape[3] + h = int(self.input_shape[2]) + w = int(self.input_shape[3]) kh = self.config["nb_row"] kw = self.config["nb_col"] dh = self.config["subsample"][0] @@ -1616,7 +1623,7 @@ def create_highway(self): else: activation = get_activation_by_name(self.config["activation"], "%s_%s" % (self.config["name"], self.config["activation"])) - blayer = BLayer.Highway(size=self.input_shape[1], + blayer = BLayer.Highway(size=int(self.input_shape[1]), with_bias=self.klayer.bias, activation=activation, wRegularizer=self.to_bigdl_reg(self.config["W_regularizer"]), @@ -1624,7 +1631,7 @@ def create_highway(self): return blayer def create_maxoutdense(self): - blayer = BLayer.Maxout(input_size=self.input_shape[1], + blayer = BLayer.Maxout(input_size=int(self.input_shape[1]), output_size=self.klayer.output_dim, maxout_number=self.klayer.nb_feature, with_bias=self.klayer.bias, @@ -1692,6 +1699,32 @@ def create_spatialdropout3d(self): data_format=bigdl_order) return blayer + def create_locallyconnected1d(self): + seq = BLayer.Sequential() + seq.add(BLayer.Reshape([int(self.input_shape[1]), 1, int(self.input_shape[2])], True)) + blayer = BLayer.LocallyConnected2D(n_input_plane=int(self.input_shape[2]), + input_width=1, + input_height=int(self.input_shape[1]), + n_output_plane=self.klayer.nb_filter, + kernel_w=1, + kernel_h=self.klayer.filter_length, + stride_w=1, + stride_h=self.klayer.subsample_length, + pad_w=0, + pad_h=0, + wRegularizer=self.to_bigdl_reg(self.config["W_regularizer"]), + bRegularizer=self.to_bigdl_reg(self.config["b_regularizer"]), + with_bias=self.klayer.bias, + data_format="NHWC") + seq.add(blayer) + seq.add(BLayer.Squeeze(3)) + if self.config["activation"] != "linear": + activation = get_activation_by_name(self.config["activation"], + "%s_%s" % (self.config["name"], self.config["activation"])) + return self.fuse(seq, activation) + else: + return seq + def create_locallyconnected2d(self): bigdl_order = self.get_bdim_order() From e06273c716de4da70b0fb4c4936c840726e3bb0c Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Thu, 21 Dec 2017 11:17:15 +0800 Subject: [PATCH 487/823] Map Keras LocallyConnected1D (#2060) * add locallyconnected1d * fix style --- .../dllib/src/bigdl/dllib/keras/converter.py | 53 +++++++++++++++---- 1 file changed, 43 insertions(+), 10 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/keras/converter.py b/python/dllib/src/bigdl/dllib/keras/converter.py index 8db1917d255..bde8cd59d8f 100644 --- a/python/dllib/src/bigdl/dllib/keras/converter.py +++ b/python/dllib/src/bigdl/dllib/keras/converter.py @@ -275,6 +275,13 @@ def convert_separableconvolution2d(klayer, weights): return [weights[0], weights[1], np.zeros(bias, )] return weights + @staticmethod + def convert_locallyconnected1d(klayer, weights): + bweights1 = np.transpose(weights[0], (0, 2, 1)) + if len(weights) == 1: # if without bias + return [bweights1] + return[bweights1, weights[1]] + @staticmethod def convert_locallyconnected2d(klayer, weights): bweights1 = np.transpose(weights[0], (0, 2, 1)) @@ -468,7 +475,7 @@ def create(self): (hasattr(self.klayer, "W_constraint") and self.klayer.W_constraint): raise Exception("We don't support constraint for now") - if (hasattr(self.klayer, "activity_regularizer") and self.klayer.activity_regularizer): + if hasattr(self.klayer, "activity_regularizer") and self.klayer.activity_regularizer: raise Exception("We don't support activity_regularizer for now") function_name = "create_" + class_name.lower() @@ -555,10 +562,10 @@ def create_embedding(self): raise Exception( "The input_length doesn't match: %s vs %s" % (seq_len, self.klayer.input_length)) - if (hasattr(self.klayer, "dropout") and self.klayer.dropout != 0): + if hasattr(self.klayer, "dropout") and self.klayer.dropout != 0: raise Exception("We don't support dropout for now") - if (hasattr(self.klayer, "mask_zero") and self.klayer.mask_zero != False): + if hasattr(self.klayer, "mask_zero") and self.klayer.mask_zero != False: raise Exception("We don't support mask_zero for now") bseq = BLayer.Sequential() @@ -1192,7 +1199,7 @@ def create_atrousconvolution1d(self): if not self.config["bias"]: raise Exception("Only bias=True is supported for AtrousConvolution1D") - h = self.input_shape[2] + h = int(self.input_shape[1]) kh = self.config["filter_length"] dh = self.config["subsample_length"] dilation_h = self.config["atrous_rate"] @@ -1226,8 +1233,8 @@ def create_atrousconvolution2d(self): if not self.config["bias"]: raise Exception("Only bias=True is supported for AtrousConvolution2D") - h = self.input_shape[2] - w = self.input_shape[3] + h = int(self.input_shape[2]) + w = int(self.input_shape[3]) kh = self.config["nb_row"] kw = self.config["nb_col"] dh = self.config["subsample"][0] @@ -1258,8 +1265,8 @@ def create_deconvolution2d(self): raise Exception("Please use `th` for `dim_ordering`. `%s` is not supported for now." % self.klayer.dim_ordering) output_shape = self.config["output_shape"] - h = self.input_shape[2] - w = self.input_shape[3] + h = int(self.input_shape[2]) + w = int(self.input_shape[3]) kh = self.config["nb_row"] kw = self.config["nb_col"] dh = self.config["subsample"][0] @@ -1616,7 +1623,7 @@ def create_highway(self): else: activation = get_activation_by_name(self.config["activation"], "%s_%s" % (self.config["name"], self.config["activation"])) - blayer = BLayer.Highway(size=self.input_shape[1], + blayer = BLayer.Highway(size=int(self.input_shape[1]), with_bias=self.klayer.bias, activation=activation, wRegularizer=self.to_bigdl_reg(self.config["W_regularizer"]), @@ -1624,7 +1631,7 @@ def create_highway(self): return blayer def create_maxoutdense(self): - blayer = BLayer.Maxout(input_size=self.input_shape[1], + blayer = BLayer.Maxout(input_size=int(self.input_shape[1]), output_size=self.klayer.output_dim, maxout_number=self.klayer.nb_feature, with_bias=self.klayer.bias, @@ -1692,6 +1699,32 @@ def create_spatialdropout3d(self): data_format=bigdl_order) return blayer + def create_locallyconnected1d(self): + seq = BLayer.Sequential() + seq.add(BLayer.Reshape([int(self.input_shape[1]), 1, int(self.input_shape[2])], True)) + blayer = BLayer.LocallyConnected2D(n_input_plane=int(self.input_shape[2]), + input_width=1, + input_height=int(self.input_shape[1]), + n_output_plane=self.klayer.nb_filter, + kernel_w=1, + kernel_h=self.klayer.filter_length, + stride_w=1, + stride_h=self.klayer.subsample_length, + pad_w=0, + pad_h=0, + wRegularizer=self.to_bigdl_reg(self.config["W_regularizer"]), + bRegularizer=self.to_bigdl_reg(self.config["b_regularizer"]), + with_bias=self.klayer.bias, + data_format="NHWC") + seq.add(blayer) + seq.add(BLayer.Squeeze(3)) + if self.config["activation"] != "linear": + activation = get_activation_by_name(self.config["activation"], + "%s_%s" % (self.config["name"], self.config["activation"])) + return self.fuse(seq, activation) + else: + return seq + def create_locallyconnected2d(self): bigdl_order = self.get_bdim_order() From 55ad3084af7927aa487d190fd9a3418dd4f7df6e Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Thu, 21 Dec 2017 11:17:15 +0800 Subject: [PATCH 488/823] Map Keras LocallyConnected1D (#2060) * add locallyconnected1d * fix style --- python/dllib/test/bigdl/keras/test_layer.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/python/dllib/test/bigdl/keras/test_layer.py b/python/dllib/test/bigdl/keras/test_layer.py index 667c6f0d644..4bc1f9a1dd2 100644 --- a/python/dllib/test/bigdl/keras/test_layer.py +++ b/python/dllib/test/bigdl/keras/test_layer.py @@ -730,6 +730,17 @@ def test_srelu(self): layer = SReLU(input_shape=(4, 6)) self.modelTestSingleLayer(input_data, layer, dump_weights=True) + def test_locallyconnected1d(self): + input_data = np.random.random_sample([3, 10, 32]) + layer1 = LocallyConnected1D(64, 3, input_shape=(10, 32)) + self.modelTestSingleLayer(input_data, layer1, dump_weights=True) + layer2 = LocallyConnected1D(64, 5, activation='sigmoid', input_shape=(10, 32)) + self.modelTestSingleLayer(input_data, layer2, dump_weights=True) + layer3 = LocallyConnected1D(32, 4, subsample_length=2, input_shape=(10, 32)) + self.modelTestSingleLayer(input_data, layer3, dump_weights=True) + layer4 = LocallyConnected1D(32, 4, bias=False, input_shape=(10, 32)) + self.modelTestSingleLayer(input_data, layer4, dump_weights=True) + def test_locallyconnected2d(self): input_data = np.random.random_sample([2, 3, 6, 8]) layer1 = LocallyConnected2D(3, 1, 2, input_shape=(3, 6, 8)) From b005fe5213d4cdd8d547da4ad2ca70192947f526 Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Thu, 21 Dec 2017 12:19:05 +0800 Subject: [PATCH 489/823] Support tf dim_ordering for Keras mnist rnn example and update readme (#2072) * support tf dim_ordering * minor update * update readme * fix style * ignore --- .../src/bigdl/dllib/examples/keras/README.md | 42 ++++++++++++++++++- .../dllib/examples/keras/imdb_cnn_lstm.py | 3 +- .../bigdl/dllib/examples/keras/mnist_cnn.py | 11 ++++- 3 files changed, 50 insertions(+), 6 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/examples/keras/README.md b/python/dllib/src/bigdl/dllib/examples/keras/README.md index 2dd3e9138d9..9f03b3cb9ea 100644 --- a/python/dllib/src/bigdl/dllib/examples/keras/README.md +++ b/python/dllib/src/bigdl/dllib/examples/keras/README.md @@ -1,4 +1,4 @@ -# **Keras Examples** +# **Examples for Keras Model Loading** We provide several simple examples here to show how to load a Keras model into BigDL and running the model in a distributed fashion. @@ -8,4 +8,42 @@ For the sake of illustration, in these examples, we first define a model in Kera You can directly run these examples if you [install BigDL from pip](../../../../docs/docs/PythonUserGuide/install-from-pip.md). After the training, good accuracy can be achieved. -In the future, we are going to provide more examples for users to try out. \ No newline at end of file +In the future, we are going to provide more examples for users to try out. + +# **How to run these examples** + +Here we take `minst_cnn.py` to illustrate. + +* Run in command line directly using `python`: + +``` +source BigDL/pyspark/test/dev/prepare_env.sh +python BigDL/pyspark/bigdl/examples/keras/mnist_cnn.py +``` + +* Run using `spark-submit` in local mode: + +``` +BigDL_HOME=... +SPARK_HOME=... +BIGDL_VERSION=... +MASTER=local[*] +PYTHON_API_ZIP_PATH=${BigDL_HOME}/dist/lib/bigdl-${BIGDL_VERSION}-python-api.zip +BigDL_JAR_PATH=${BigDL_HOME}/dist/lib/bigdl-${BIGDL_VERSION}-jar-with-dependencies.jar +PYTHONPATH=${PYTHON_API_ZIP_PATH}:$PYTHONPATH + +${SPARK_HOME}/bin/spark-submit \ + --master ${MASTER} \ + --driver-cores 5 \ + --driver-memory 10g \ + --total-executor-cores 80 \ + --executor-cores 10 \ + --executor-memory 20g \ + --py-files ${PYTHON_API_ZIP_PATH},${BigDL_HOME}/pyspark/bigdl/examples/keras/mnist_cnn.py \ + --properties-file ${BigDL_HOME}/dist/conf/spark-bigdl.conf \ + --jars ${BigDL_JAR_PATH} \ + --conf spark.driver.extraClassPath=${BigDL_JAR_PATH} \ + --conf spark.executor.extraClassPath=bigdl-${BIGDL_VERSION}-jar-with-dependencies.jar \ +${BigDL_HOME}/pyspark/bigdl/examples/keras/mnist_cnn.py + +``` \ No newline at end of file diff --git a/python/dllib/src/bigdl/dllib/examples/keras/imdb_cnn_lstm.py b/python/dllib/src/bigdl/dllib/examples/keras/imdb_cnn_lstm.py index 0ef776d4e04..3d8d6c5748d 100644 --- a/python/dllib/src/bigdl/dllib/examples/keras/imdb_cnn_lstm.py +++ b/python/dllib/src/bigdl/dllib/examples/keras/imdb_cnn_lstm.py @@ -18,8 +18,7 @@ # IMDB sentiment classification using a recurrent convolutional network on BigDL # Reference: https://github.com/fchollet/keras/blob/1.2.2/examples/imdb_cnn_lstm.py # The Keras version we support and test is Keras 1.2.2 with TensorFlow backend. - -from bigdl.examples.keras.keras_utils import * +# See README.md for how to run this example. def load_imdb(): diff --git a/python/dllib/src/bigdl/dllib/examples/keras/mnist_cnn.py b/python/dllib/src/bigdl/dllib/examples/keras/mnist_cnn.py index 855e62fdd59..8fe82bfe092 100644 --- a/python/dllib/src/bigdl/dllib/examples/keras/mnist_cnn.py +++ b/python/dllib/src/bigdl/dllib/examples/keras/mnist_cnn.py @@ -19,9 +19,16 @@ # Reference: https://github.com/fchollet/keras/blob/1.2.2/examples/mnist_cnn.py # ../../models/lenet/lenet5.py # The Keras version we support and test is Keras 1.2.2 with TensorFlow backend. +# See README.md for how to run this example. from bigdl.examples.keras.keras_utils import * +import keras.backend +if keras.backend.image_dim_ordering() == "th": + input_shape = (1, 28, 28) +else: + input_shape = (28, 28, 1) + def get_mnist(sc, data_type="train", location="/tmp/mnist"): """ @@ -31,7 +38,7 @@ def get_mnist(sc, data_type="train", location="/tmp/mnist"): from bigdl.dataset import mnist from bigdl.dataset.transformer import normalizer (images, labels) = mnist.read_data_sets(location, data_type) - images = images.reshape(images.shape[0], 1, 28, 28) + images = images.reshape((images.shape[0], ) + input_shape) images = sc.parallelize(images) labels = sc.parallelize(labels + 1) # Target start from 1 in BigDL record = images.zip(labels).map(lambda rec_tuple: (normalizer(rec_tuple[0], mnist.TRAIN_MEAN, mnist.TRAIN_STD), @@ -50,7 +57,7 @@ def build_keras_model(): keras_model = Sequential() keras_model.add(Convolution2D(32, 3, 3, border_mode='valid', - input_shape=(1, 28, 28))) + input_shape=input_shape)) keras_model.add(Activation('relu')) keras_model.add(Convolution2D(32, 3, 3)) keras_model.add(Activation('relu')) From c639cf48380a02a5cbed53736cd8618c640d50f9 Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Thu, 21 Dec 2017 12:19:05 +0800 Subject: [PATCH 490/823] Support tf dim_ordering for Keras mnist rnn example and update readme (#2072) * support tf dim_ordering * minor update * update readme * fix style * ignore --- .../src/bigdl/dllib/examples/keras/README.md | 42 ++++++++++++++++++- .../dllib/examples/keras/imdb_cnn_lstm.py | 3 +- .../bigdl/dllib/examples/keras/mnist_cnn.py | 11 ++++- 3 files changed, 50 insertions(+), 6 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/examples/keras/README.md b/python/dllib/src/bigdl/dllib/examples/keras/README.md index 2dd3e9138d9..9f03b3cb9ea 100644 --- a/python/dllib/src/bigdl/dllib/examples/keras/README.md +++ b/python/dllib/src/bigdl/dllib/examples/keras/README.md @@ -1,4 +1,4 @@ -# **Keras Examples** +# **Examples for Keras Model Loading** We provide several simple examples here to show how to load a Keras model into BigDL and running the model in a distributed fashion. @@ -8,4 +8,42 @@ For the sake of illustration, in these examples, we first define a model in Kera You can directly run these examples if you [install BigDL from pip](../../../../docs/docs/PythonUserGuide/install-from-pip.md). After the training, good accuracy can be achieved. -In the future, we are going to provide more examples for users to try out. \ No newline at end of file +In the future, we are going to provide more examples for users to try out. + +# **How to run these examples** + +Here we take `minst_cnn.py` to illustrate. + +* Run in command line directly using `python`: + +``` +source BigDL/pyspark/test/dev/prepare_env.sh +python BigDL/pyspark/bigdl/examples/keras/mnist_cnn.py +``` + +* Run using `spark-submit` in local mode: + +``` +BigDL_HOME=... +SPARK_HOME=... +BIGDL_VERSION=... +MASTER=local[*] +PYTHON_API_ZIP_PATH=${BigDL_HOME}/dist/lib/bigdl-${BIGDL_VERSION}-python-api.zip +BigDL_JAR_PATH=${BigDL_HOME}/dist/lib/bigdl-${BIGDL_VERSION}-jar-with-dependencies.jar +PYTHONPATH=${PYTHON_API_ZIP_PATH}:$PYTHONPATH + +${SPARK_HOME}/bin/spark-submit \ + --master ${MASTER} \ + --driver-cores 5 \ + --driver-memory 10g \ + --total-executor-cores 80 \ + --executor-cores 10 \ + --executor-memory 20g \ + --py-files ${PYTHON_API_ZIP_PATH},${BigDL_HOME}/pyspark/bigdl/examples/keras/mnist_cnn.py \ + --properties-file ${BigDL_HOME}/dist/conf/spark-bigdl.conf \ + --jars ${BigDL_JAR_PATH} \ + --conf spark.driver.extraClassPath=${BigDL_JAR_PATH} \ + --conf spark.executor.extraClassPath=bigdl-${BIGDL_VERSION}-jar-with-dependencies.jar \ +${BigDL_HOME}/pyspark/bigdl/examples/keras/mnist_cnn.py + +``` \ No newline at end of file diff --git a/python/dllib/src/bigdl/dllib/examples/keras/imdb_cnn_lstm.py b/python/dllib/src/bigdl/dllib/examples/keras/imdb_cnn_lstm.py index 0ef776d4e04..3d8d6c5748d 100644 --- a/python/dllib/src/bigdl/dllib/examples/keras/imdb_cnn_lstm.py +++ b/python/dllib/src/bigdl/dllib/examples/keras/imdb_cnn_lstm.py @@ -18,8 +18,7 @@ # IMDB sentiment classification using a recurrent convolutional network on BigDL # Reference: https://github.com/fchollet/keras/blob/1.2.2/examples/imdb_cnn_lstm.py # The Keras version we support and test is Keras 1.2.2 with TensorFlow backend. - -from bigdl.examples.keras.keras_utils import * +# See README.md for how to run this example. def load_imdb(): diff --git a/python/dllib/src/bigdl/dllib/examples/keras/mnist_cnn.py b/python/dllib/src/bigdl/dllib/examples/keras/mnist_cnn.py index 855e62fdd59..8fe82bfe092 100644 --- a/python/dllib/src/bigdl/dllib/examples/keras/mnist_cnn.py +++ b/python/dllib/src/bigdl/dllib/examples/keras/mnist_cnn.py @@ -19,9 +19,16 @@ # Reference: https://github.com/fchollet/keras/blob/1.2.2/examples/mnist_cnn.py # ../../models/lenet/lenet5.py # The Keras version we support and test is Keras 1.2.2 with TensorFlow backend. +# See README.md for how to run this example. from bigdl.examples.keras.keras_utils import * +import keras.backend +if keras.backend.image_dim_ordering() == "th": + input_shape = (1, 28, 28) +else: + input_shape = (28, 28, 1) + def get_mnist(sc, data_type="train", location="/tmp/mnist"): """ @@ -31,7 +38,7 @@ def get_mnist(sc, data_type="train", location="/tmp/mnist"): from bigdl.dataset import mnist from bigdl.dataset.transformer import normalizer (images, labels) = mnist.read_data_sets(location, data_type) - images = images.reshape(images.shape[0], 1, 28, 28) + images = images.reshape((images.shape[0], ) + input_shape) images = sc.parallelize(images) labels = sc.parallelize(labels + 1) # Target start from 1 in BigDL record = images.zip(labels).map(lambda rec_tuple: (normalizer(rec_tuple[0], mnist.TRAIN_MEAN, mnist.TRAIN_STD), @@ -50,7 +57,7 @@ def build_keras_model(): keras_model = Sequential() keras_model.add(Convolution2D(32, 3, 3, border_mode='valid', - input_shape=(1, 28, 28))) + input_shape=input_shape)) keras_model.add(Activation('relu')) keras_model.add(Convolution2D(32, 3, 3)) keras_model.add(Activation('relu')) From 05bef3bd0bff2311dfd3f2ac24cd92165be95785 Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Thu, 21 Dec 2017 12:19:05 +0800 Subject: [PATCH 491/823] Support tf dim_ordering for Keras mnist rnn example and update readme (#2072) * support tf dim_ordering * minor update * update readme * fix style * ignore --- python/dllib/test/dev/run-tests | 1 + 1 file changed, 1 insertion(+) diff --git a/python/dllib/test/dev/run-tests b/python/dllib/test/dev/run-tests index 2c3e83113b2..15237a53a20 100755 --- a/python/dllib/test/dev/run-tests +++ b/python/dllib/test/dev/run-tests @@ -33,6 +33,7 @@ do --ignore=../../../pyspark/bigdl/util/tf_utils.py \ --ignore=../../../pyspark/bigdl/keras \ --ignore=../../../pyspark/test/bigdl/keras/test_application.py \ + --ignore=../../../pyspark/bigdl/examples/ \ --ignore=../../../pyspark/bigdl/models/ && \ $p -m pytest -v ../../../pyspark/test/ \ --ignore=../../../pyspark/test/bigdl/caffe/ \ From 6d6f42478a23710e1a5eeecec923d147d9cc5a55 Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Thu, 21 Dec 2017 15:30:50 +0800 Subject: [PATCH 492/823] Fix KLDCriterion forward (#2078) * fix KLDCriterion * add python doc * add unit tests * add api doc --- python/dllib/src/bigdl/dllib/nn/criterion.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/python/dllib/src/bigdl/dllib/nn/criterion.py b/python/dllib/src/bigdl/dllib/nn/criterion.py index d48eaf48a39..fd3d817de3f 100644 --- a/python/dllib/src/bigdl/dllib/nn/criterion.py +++ b/python/dllib/src/bigdl/dllib/nn/criterion.py @@ -487,7 +487,10 @@ def add(self, criterion, weight=1.0): class KLDCriterion(Criterion): ''' - Computes the KL-divergence of the Gaussian distribution. + Computes the KL-divergence of the input normal distribution to a standard normal distribution. + The input has to be a table. The first element of input is the mean of the distribution, + the second element of input is the log_variance of the distribution. The input distribution is + assumed to be diagonal. >>> KLDCriterion = KLDCriterion() creating: createKLDCriterion ''' From 43edc17e29a105acaf085b40e10e3ac1c6f9a023 Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Thu, 21 Dec 2017 15:30:50 +0800 Subject: [PATCH 493/823] Fix KLDCriterion forward (#2078) * fix KLDCriterion * add python doc * add unit tests * add api doc --- python/dllib/src/bigdl/dllib/nn/criterion.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/python/dllib/src/bigdl/dllib/nn/criterion.py b/python/dllib/src/bigdl/dllib/nn/criterion.py index d48eaf48a39..fd3d817de3f 100644 --- a/python/dllib/src/bigdl/dllib/nn/criterion.py +++ b/python/dllib/src/bigdl/dllib/nn/criterion.py @@ -487,7 +487,10 @@ def add(self, criterion, weight=1.0): class KLDCriterion(Criterion): ''' - Computes the KL-divergence of the Gaussian distribution. + Computes the KL-divergence of the input normal distribution to a standard normal distribution. + The input has to be a table. The first element of input is the mean of the distribution, + the second element of input is the log_variance of the distribution. The input distribution is + assumed to be diagonal. >>> KLDCriterion = KLDCriterion() creating: createKLDCriterion ''' From bfceb69ef45cbbe2505d50461cb67c7d645fef5d Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Thu, 21 Dec 2017 16:53:50 +0800 Subject: [PATCH 494/823] minor update on keras docs and readme (#2085) --- python/dllib/src/bigdl/dllib/examples/keras/README.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/examples/keras/README.md b/python/dllib/src/bigdl/dllib/examples/keras/README.md index 9f03b3cb9ea..d44603e3c7b 100644 --- a/python/dllib/src/bigdl/dllib/examples/keras/README.md +++ b/python/dllib/src/bigdl/dllib/examples/keras/README.md @@ -12,9 +12,9 @@ In the future, we are going to provide more examples for users to try out. # **How to run these examples** -Here we take `minst_cnn.py` to illustrate. +Here we take `minst_cnn.py` to show how to run the examples under this directory. You can run other examples in a similar way as the following: -* Run in command line directly using `python`: +* Run in the command line directly using `python`: ``` source BigDL/pyspark/test/dev/prepare_env.sh @@ -45,5 +45,4 @@ ${SPARK_HOME}/bin/spark-submit \ --conf spark.driver.extraClassPath=${BigDL_JAR_PATH} \ --conf spark.executor.extraClassPath=bigdl-${BIGDL_VERSION}-jar-with-dependencies.jar \ ${BigDL_HOME}/pyspark/bigdl/examples/keras/mnist_cnn.py - ``` \ No newline at end of file From 67632e3ab34416b1d6a7d6d6d2ec90fee1565a8a Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Thu, 21 Dec 2017 16:53:50 +0800 Subject: [PATCH 495/823] minor update on keras docs and readme (#2085) --- python/dllib/src/bigdl/dllib/examples/keras/README.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/examples/keras/README.md b/python/dllib/src/bigdl/dllib/examples/keras/README.md index 9f03b3cb9ea..d44603e3c7b 100644 --- a/python/dllib/src/bigdl/dllib/examples/keras/README.md +++ b/python/dllib/src/bigdl/dllib/examples/keras/README.md @@ -12,9 +12,9 @@ In the future, we are going to provide more examples for users to try out. # **How to run these examples** -Here we take `minst_cnn.py` to illustrate. +Here we take `minst_cnn.py` to show how to run the examples under this directory. You can run other examples in a similar way as the following: -* Run in command line directly using `python`: +* Run in the command line directly using `python`: ``` source BigDL/pyspark/test/dev/prepare_env.sh @@ -45,5 +45,4 @@ ${SPARK_HOME}/bin/spark-submit \ --conf spark.driver.extraClassPath=${BigDL_JAR_PATH} \ --conf spark.executor.extraClassPath=bigdl-${BIGDL_VERSION}-jar-with-dependencies.jar \ ${BigDL_HOME}/pyspark/bigdl/examples/keras/mnist_cnn.py - ``` \ No newline at end of file From 8010afcefe9efe6d1ce84a122d4950d67d746b6b Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Thu, 21 Dec 2017 18:05:58 +0800 Subject: [PATCH 496/823] update keras example readme (#2087) --- .../src/bigdl/dllib/examples/keras/README.md | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/examples/keras/README.md b/python/dllib/src/bigdl/dllib/examples/keras/README.md index d44603e3c7b..0bf7d8026f4 100644 --- a/python/dllib/src/bigdl/dllib/examples/keras/README.md +++ b/python/dllib/src/bigdl/dllib/examples/keras/README.md @@ -33,16 +33,16 @@ BigDL_JAR_PATH=${BigDL_HOME}/dist/lib/bigdl-${BIGDL_VERSION}-jar-with-dependenci PYTHONPATH=${PYTHON_API_ZIP_PATH}:$PYTHONPATH ${SPARK_HOME}/bin/spark-submit \ - --master ${MASTER} \ - --driver-cores 5 \ - --driver-memory 10g \ - --total-executor-cores 80 \ - --executor-cores 10 \ - --executor-memory 20g \ - --py-files ${PYTHON_API_ZIP_PATH},${BigDL_HOME}/pyspark/bigdl/examples/keras/mnist_cnn.py \ - --properties-file ${BigDL_HOME}/dist/conf/spark-bigdl.conf \ - --jars ${BigDL_JAR_PATH} \ - --conf spark.driver.extraClassPath=${BigDL_JAR_PATH} \ - --conf spark.executor.extraClassPath=bigdl-${BIGDL_VERSION}-jar-with-dependencies.jar \ -${BigDL_HOME}/pyspark/bigdl/examples/keras/mnist_cnn.py + --master ${MASTER} \ + --driver-cores 5 \ + --driver-memory 10g \ + --total-executor-cores 80 \ + --executor-cores 10 \ + --executor-memory 20g \ + --py-files ${PYTHON_API_ZIP_PATH},${BigDL_HOME}/pyspark/bigdl/examples/keras/mnist_cnn.py \ + --properties-file ${BigDL_HOME}/dist/conf/spark-bigdl.conf \ + --jars ${BigDL_JAR_PATH} \ + --conf spark.driver.extraClassPath=${BigDL_JAR_PATH} \ + --conf spark.executor.extraClassPath=bigdl-${BIGDL_VERSION}-jar-with-dependencies.jar \ + ${BigDL_HOME}/pyspark/bigdl/examples/keras/mnist_cnn.py ``` \ No newline at end of file From 1b4cf0f96b0b185aa92e8e98f4759b523f4c24d6 Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Thu, 21 Dec 2017 18:05:58 +0800 Subject: [PATCH 497/823] update keras example readme (#2087) --- .../src/bigdl/dllib/examples/keras/README.md | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/examples/keras/README.md b/python/dllib/src/bigdl/dllib/examples/keras/README.md index d44603e3c7b..0bf7d8026f4 100644 --- a/python/dllib/src/bigdl/dllib/examples/keras/README.md +++ b/python/dllib/src/bigdl/dllib/examples/keras/README.md @@ -33,16 +33,16 @@ BigDL_JAR_PATH=${BigDL_HOME}/dist/lib/bigdl-${BIGDL_VERSION}-jar-with-dependenci PYTHONPATH=${PYTHON_API_ZIP_PATH}:$PYTHONPATH ${SPARK_HOME}/bin/spark-submit \ - --master ${MASTER} \ - --driver-cores 5 \ - --driver-memory 10g \ - --total-executor-cores 80 \ - --executor-cores 10 \ - --executor-memory 20g \ - --py-files ${PYTHON_API_ZIP_PATH},${BigDL_HOME}/pyspark/bigdl/examples/keras/mnist_cnn.py \ - --properties-file ${BigDL_HOME}/dist/conf/spark-bigdl.conf \ - --jars ${BigDL_JAR_PATH} \ - --conf spark.driver.extraClassPath=${BigDL_JAR_PATH} \ - --conf spark.executor.extraClassPath=bigdl-${BIGDL_VERSION}-jar-with-dependencies.jar \ -${BigDL_HOME}/pyspark/bigdl/examples/keras/mnist_cnn.py + --master ${MASTER} \ + --driver-cores 5 \ + --driver-memory 10g \ + --total-executor-cores 80 \ + --executor-cores 10 \ + --executor-memory 20g \ + --py-files ${PYTHON_API_ZIP_PATH},${BigDL_HOME}/pyspark/bigdl/examples/keras/mnist_cnn.py \ + --properties-file ${BigDL_HOME}/dist/conf/spark-bigdl.conf \ + --jars ${BigDL_JAR_PATH} \ + --conf spark.driver.extraClassPath=${BigDL_JAR_PATH} \ + --conf spark.executor.extraClassPath=bigdl-${BIGDL_VERSION}-jar-with-dependencies.jar \ + ${BigDL_HOME}/pyspark/bigdl/examples/keras/mnist_cnn.py ``` \ No newline at end of file From a1392595c7cc9b75e47b1c2b60349a5302ca9961 Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Fri, 22 Dec 2017 14:04:39 +0800 Subject: [PATCH 498/823] Support adding customized jars after pip install; fix release related (#2083) * support add jars * meet review * fix * append sys.path * update * refactor and fix * add docs --- python/dllib/src/bigdl/dllib/utils/common.py | 46 ++++++++++++++++++-- python/dllib/src/bigdl/dllib/utils/engine.py | 8 +++- 2 files changed, 50 insertions(+), 4 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/utils/common.py b/python/dllib/src/bigdl/dllib/utils/common.py index 384b8c7427b..cfc4cf9fcda 100644 --- a/python/dllib/src/bigdl/dllib/utils/common.py +++ b/python/dllib/src/bigdl/dllib/utils/common.py @@ -29,7 +29,7 @@ import numpy as np import threading import tempfile -from bigdl.util.engine import prepare_env, get_bigdl_classpath, is_spark_below_2_2 +from bigdl.util.engine import get_bigdl_classpath, is_spark_below_2_2 INTMAX = 2147483647 INTMIN = -2147483648 @@ -39,6 +39,7 @@ long = int unicode = str + class SingletonMixin(object): _lock = threading.RLock() _instance = None @@ -52,6 +53,7 @@ def instance(cls, cls._instance = cls(bigdl_type) return cls._instance + class JavaCreator(SingletonMixin): __creator_class="com.intel.analytics.bigdl.python.api.PythonBigDL" @@ -114,11 +116,43 @@ def __str__(self): return "Evaluated result: %s, total_num: %s, method: %s" % ( self.result, self.total_num, self.method) + def get_dtype(bigdl_type): # Always return float32 for now return "float32" +class Configuration(object): + __bigdl_jars = [get_bigdl_classpath()] + + @staticmethod + def add_extra_jars(jars): + """ + Add extra jars to classpath + :param jars: a string or a list of strings as jar paths + """ + import six + if isinstance(jars, six.string_types): + jars = [jars] + Configuration.__bigdl_jars += jars + + @staticmethod + def add_extra_python_modules(packages): + """ + Add extra python modules to sys.path + :param packages: a string or a list of strings as python package paths + """ + import six + if isinstance(packages, six.string_types): + packages = [packages] + for package in packages: + sys.path.insert(0, package) + + @staticmethod + def get_bigdl_jars(): + return Configuration.__bigdl_jars + + class JActivity(object): def __init__(self, value): @@ -358,6 +392,7 @@ def __str__(self): def __repr__(self): return "Sample: features: %s, labels: %s" % (self.features, self.labels) + class RNG(): """ generate tensor data with seed @@ -398,6 +433,7 @@ def redire_spark_logs(bigdl_type="float", log_path=os.getcwd()+"/bigdl.log"): """ callBigDlFunc(bigdl_type, "redirectSparkLogs", log_path) + def show_bigdl_info_logs(bigdl_type="float"): """ Set BigDL log level to INFO. @@ -461,7 +497,8 @@ def create_spark_conf(): sparkConf = SparkConf() sparkConf.setAll(bigdl_conf.items()) if not is_spark_below_2_2(): - extend_spark_driver_cp(sparkConf, get_bigdl_classpath()) + for jar in Configuration.get_bigdl_jars(): + extend_spark_driver_cp(sparkConf, jar) # add content in PYSPARK_FILES in spark.submit.pyFiles # This is a workaround for current Spark on k8s @@ -476,7 +513,7 @@ def create_spark_conf(): return sparkConf -def get_spark_context(conf = None): +def get_spark_context(conf=None): """ Get the current active spark context and create one if no active instance :param conf: combining bigdl configs into spark conf @@ -506,6 +543,7 @@ def get_spark_sql_context(sc): else: return SQLContext(sc) # Compatible with Spark1.5.1 + def callBigDlFunc(bigdl_type, name, *args): """ Call API in PythonBigDL """ jinstance = JavaCreator.instance(bigdl_type=bigdl_type).value @@ -513,6 +551,7 @@ def callBigDlFunc(bigdl_type, name, *args): api = getattr(jinstance, name) return callJavaFunc(sc, api, *args) + def _java2py(sc, r, encoding="bytes"): if isinstance(r, JavaObject): clsName = r.getClass().getSimpleName() @@ -647,6 +686,7 @@ def get_activation_by_name(activation_name, activation_id=None): activation.set_name(activation_id) return activation + def _test(): import doctest from pyspark import SparkContext diff --git a/python/dllib/src/bigdl/dllib/utils/engine.py b/python/dllib/src/bigdl/dllib/utils/engine.py index 2b278cd3edd..e6ea5cad3c9 100644 --- a/python/dllib/src/bigdl/dllib/utils/engine.py +++ b/python/dllib/src/bigdl/dllib/utils/engine.py @@ -19,6 +19,7 @@ import glob import warnings + def exist_pyspark(): # check whether pyspark package exists try: @@ -27,6 +28,7 @@ def exist_pyspark(): except ImportError: return False + def check_spark_source_conflict(spark_home, pyspark_path): # check if both spark_home env var and pyspark package exist # trigger a warning if two spark sources don't match @@ -38,6 +40,7 @@ def check_spark_source_conflict(spark_home, pyspark_path): "For example, you can unset SPARK_HOME and use pyspark only." warnings.warn(warning_msg) + def __prepare_spark_env(): spark_home = os.environ.get('SPARK_HOME', None) if exist_pyspark(): @@ -77,13 +80,16 @@ def append_path(env_var_name, path): if bigdl_classpath and is_spark_below_2_2(): append_path("SPARK_CLASSPATH", bigdl_classpath) + from bigdl.util.common import Configuration + for jar in Configuration.get_bigdl_jars(): + append_path("SPARK_CLASSPATH", jar) def get_bigdl_classpath(): """ Get and return the jar path for bigdl if exists. """ - if(os.getenv("BIGDL_CLASSPATH")): + if os.getenv("BIGDL_CLASSPATH"): return os.environ["BIGDL_CLASSPATH"] jar_dir = os.path.abspath(__file__ + "/../../") jar_paths = glob.glob(os.path.join(jar_dir, "share/lib/*.jar")) From 27c7abf8132089005ddde4d9e9ccf2a7c653a441 Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Fri, 22 Dec 2017 14:04:39 +0800 Subject: [PATCH 499/823] Support adding customized jars after pip install; fix release related (#2083) * support add jars * meet review * fix * append sys.path * update * refactor and fix * add docs --- python/dllib/src/bigdl/utils/common.py | 46 ++++++++++++++++++++++++-- python/dllib/src/bigdl/utils/engine.py | 8 ++++- 2 files changed, 50 insertions(+), 4 deletions(-) diff --git a/python/dllib/src/bigdl/utils/common.py b/python/dllib/src/bigdl/utils/common.py index 384b8c7427b..cfc4cf9fcda 100644 --- a/python/dllib/src/bigdl/utils/common.py +++ b/python/dllib/src/bigdl/utils/common.py @@ -29,7 +29,7 @@ import numpy as np import threading import tempfile -from bigdl.util.engine import prepare_env, get_bigdl_classpath, is_spark_below_2_2 +from bigdl.util.engine import get_bigdl_classpath, is_spark_below_2_2 INTMAX = 2147483647 INTMIN = -2147483648 @@ -39,6 +39,7 @@ long = int unicode = str + class SingletonMixin(object): _lock = threading.RLock() _instance = None @@ -52,6 +53,7 @@ def instance(cls, cls._instance = cls(bigdl_type) return cls._instance + class JavaCreator(SingletonMixin): __creator_class="com.intel.analytics.bigdl.python.api.PythonBigDL" @@ -114,11 +116,43 @@ def __str__(self): return "Evaluated result: %s, total_num: %s, method: %s" % ( self.result, self.total_num, self.method) + def get_dtype(bigdl_type): # Always return float32 for now return "float32" +class Configuration(object): + __bigdl_jars = [get_bigdl_classpath()] + + @staticmethod + def add_extra_jars(jars): + """ + Add extra jars to classpath + :param jars: a string or a list of strings as jar paths + """ + import six + if isinstance(jars, six.string_types): + jars = [jars] + Configuration.__bigdl_jars += jars + + @staticmethod + def add_extra_python_modules(packages): + """ + Add extra python modules to sys.path + :param packages: a string or a list of strings as python package paths + """ + import six + if isinstance(packages, six.string_types): + packages = [packages] + for package in packages: + sys.path.insert(0, package) + + @staticmethod + def get_bigdl_jars(): + return Configuration.__bigdl_jars + + class JActivity(object): def __init__(self, value): @@ -358,6 +392,7 @@ def __str__(self): def __repr__(self): return "Sample: features: %s, labels: %s" % (self.features, self.labels) + class RNG(): """ generate tensor data with seed @@ -398,6 +433,7 @@ def redire_spark_logs(bigdl_type="float", log_path=os.getcwd()+"/bigdl.log"): """ callBigDlFunc(bigdl_type, "redirectSparkLogs", log_path) + def show_bigdl_info_logs(bigdl_type="float"): """ Set BigDL log level to INFO. @@ -461,7 +497,8 @@ def create_spark_conf(): sparkConf = SparkConf() sparkConf.setAll(bigdl_conf.items()) if not is_spark_below_2_2(): - extend_spark_driver_cp(sparkConf, get_bigdl_classpath()) + for jar in Configuration.get_bigdl_jars(): + extend_spark_driver_cp(sparkConf, jar) # add content in PYSPARK_FILES in spark.submit.pyFiles # This is a workaround for current Spark on k8s @@ -476,7 +513,7 @@ def create_spark_conf(): return sparkConf -def get_spark_context(conf = None): +def get_spark_context(conf=None): """ Get the current active spark context and create one if no active instance :param conf: combining bigdl configs into spark conf @@ -506,6 +543,7 @@ def get_spark_sql_context(sc): else: return SQLContext(sc) # Compatible with Spark1.5.1 + def callBigDlFunc(bigdl_type, name, *args): """ Call API in PythonBigDL """ jinstance = JavaCreator.instance(bigdl_type=bigdl_type).value @@ -513,6 +551,7 @@ def callBigDlFunc(bigdl_type, name, *args): api = getattr(jinstance, name) return callJavaFunc(sc, api, *args) + def _java2py(sc, r, encoding="bytes"): if isinstance(r, JavaObject): clsName = r.getClass().getSimpleName() @@ -647,6 +686,7 @@ def get_activation_by_name(activation_name, activation_id=None): activation.set_name(activation_id) return activation + def _test(): import doctest from pyspark import SparkContext diff --git a/python/dllib/src/bigdl/utils/engine.py b/python/dllib/src/bigdl/utils/engine.py index 2b278cd3edd..e6ea5cad3c9 100644 --- a/python/dllib/src/bigdl/utils/engine.py +++ b/python/dllib/src/bigdl/utils/engine.py @@ -19,6 +19,7 @@ import glob import warnings + def exist_pyspark(): # check whether pyspark package exists try: @@ -27,6 +28,7 @@ def exist_pyspark(): except ImportError: return False + def check_spark_source_conflict(spark_home, pyspark_path): # check if both spark_home env var and pyspark package exist # trigger a warning if two spark sources don't match @@ -38,6 +40,7 @@ def check_spark_source_conflict(spark_home, pyspark_path): "For example, you can unset SPARK_HOME and use pyspark only." warnings.warn(warning_msg) + def __prepare_spark_env(): spark_home = os.environ.get('SPARK_HOME', None) if exist_pyspark(): @@ -77,13 +80,16 @@ def append_path(env_var_name, path): if bigdl_classpath and is_spark_below_2_2(): append_path("SPARK_CLASSPATH", bigdl_classpath) + from bigdl.util.common import Configuration + for jar in Configuration.get_bigdl_jars(): + append_path("SPARK_CLASSPATH", jar) def get_bigdl_classpath(): """ Get and return the jar path for bigdl if exists. """ - if(os.getenv("BIGDL_CLASSPATH")): + if os.getenv("BIGDL_CLASSPATH"): return os.environ["BIGDL_CLASSPATH"] jar_dir = os.path.abspath(__file__ + "/../../") jar_paths = glob.glob(os.path.join(jar_dir, "share/lib/*.jar")) From e7e6916b9a01048ac8906e0ea42321de8cf0d278 Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Fri, 22 Dec 2017 14:04:39 +0800 Subject: [PATCH 500/823] Support adding customized jars after pip install; fix release related (#2083) * support add jars * meet review * fix * append sys.path * update * refactor and fix * add docs --- python/dllib/test/dev/release/release.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/dllib/test/dev/release/release.sh b/python/dllib/test/dev/release/release.sh index 79dcba2498e..caba1f7fc1c 100755 --- a/python/dllib/test/dev/release/release.sh +++ b/python/dllib/test/dev/release/release.sh @@ -66,11 +66,11 @@ sdist_command="python setup.py sdist" echo "packing source code: ${sdist_command}" $sdist_command -if [-d "${BIGDL_DIR}/pyspark/build" ]; then +if [ -d "${BIGDL_DIR}/pyspark/build" ]; then rm -r ${BIGDL_DIR}/pyspark/build fi -if [-d "${BIGDL_DIR}/pyspark/dist" ]; then +if [ -d "${BIGDL_DIR}/pyspark/dist" ]; then rm -r ${BIGDL_DIR}/pyspark/dist fi wheel_command="python setup.py bdist_wheel --plat-name ${verbose_pname}" From b477edc12616adf22c92e807ebe9f3b8450d7ee7 Mon Sep 17 00:00:00 2001 From: Xianyan Date: Fri, 22 Dec 2017 19:52:52 +0800 Subject: [PATCH 501/823] Add transformer doc and some fix (#2092) --- .../dllib/feature/transform/vision/image.py | 27 ++++++++++++------- python/dllib/src/bigdl/dllib/utils/common.py | 1 - 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py b/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py index 4797b27ff5a..e3c14ebf129 100644 --- a/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py +++ b/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py @@ -182,8 +182,8 @@ def __init__(self, image_list=None, label_list=None, jvalue=None, bigdl_type="fl self.value = jvalue else: # init from image ndarray list and label rdd(optional) - image_tensor_list = image_list.map(lambda image: JTensor.from_ndarray(image)) - label_tensor_list = label_list.map(lambda label: JTensor.from_ndarray(label)) if label_list else None + image_tensor_list = map(lambda image: JTensor.from_ndarray(image), image_list) + label_tensor_list = map(lambda label: JTensor.from_ndarray(label), label_list) if label_list else None self.value = callBigDlFunc(bigdl_type, JavaValue.jvm_class_constructor(self), image_tensor_list, label_tensor_list) @@ -209,7 +209,7 @@ def get_predict(self, key="predict"): get prediction list from ImageFrame """ predicts = callBigDlFunc(self.bigdl_type, "localImageFrameToPredict", self.value, key) - return map(lambda predict: (predict[0], predict[1].to_ndarray()), predicts) + return map(lambda predict: (predict[0], predict[1].to_ndarray()) if predict[1] else (predict[0], None), predicts) @@ -251,7 +251,7 @@ def get_predict(self, key="predict"): get prediction rdd from ImageFrame """ predicts = callBigDlFunc(self.bigdl_type, "distributedImageFrameToPredict", self.value, key) - return predicts.map(lambda predict: (predict[0], predict[1].to_ndarray())) + return predicts.map(lambda predict: (predict[0], predict[1].to_ndarray()) if predict[1] else (predict[0], None)) class HFlip(FeatureTransformer): @@ -269,10 +269,15 @@ class Resize(FeatureTransformer): :param resize_w width after resize :param resize_mode if resizeMode = -1, random select a mode from (Imgproc.INTER_LINEAR, Imgproc.INTER_CUBIC, Imgproc.INTER_AREA, Imgproc.INTER_NEAREST, Imgproc.INTER_LANCZOS4) + :param use_scale_factor if true, scale factor fx and fy is used, fx = fy = 0 + note that the result of the following are different + Imgproc.resize(mat, mat, new Size(resizeWH, resizeWH), 0, 0, Imgproc.INTER_LINEAR) + Imgproc.resize(mat, mat, new Size(resizeWH, resizeWH)) """ - def __init__(self, resize_h, resize_w, resize_mode = 1, bigdl_type="float"): - super(Resize, self).__init__(bigdl_type, resize_h, resize_w, resize_mode) + def __init__(self, resize_h, resize_w, resize_mode = 1, use_scale_factor=True, + bigdl_type="float"): + super(Resize, self).__init__(bigdl_type, resize_h, resize_w, resize_mode, use_scale_factor) class Brightness(FeatureTransformer): """ @@ -533,12 +538,13 @@ class MatToFloats(FeatureTransformer): :param valid_width valid width in case the mat is invalid :param valid_channel valid channel in case the mat is invalid :param out_key key to store float array + :param share_buffer share buffer of output """ def __init__(self, valid_height=300, valid_width=300, valid_channel=300, - out_key = "floats", bigdl_type="float"): + out_key = "floats", share_buffer=True, bigdl_type="float"): super(MatToFloats, self).__init__(bigdl_type, valid_height, valid_width, valid_channel, - out_key) + out_key, share_buffer) class MatToTensor(FeatureTransformer): """ @@ -574,9 +580,10 @@ def __init__(self, scales, scale_multiple_of = 1, max_size = 1000, bigdl_type="f class BytesToMat(FeatureTransformer): """ Transform byte array(original image file in byte) to OpenCVMat + :param byte_key key that maps byte array """ - def __init__(self, bigdl_type="float"): - super(BytesToMat, self).__init__(bigdl_type) + def __init__(self, byte_key = "bytes", bigdl_type="float"): + super(BytesToMat, self).__init__(bigdl_type, byte_key) class ImageFrameToSample(FeatureTransformer): """ diff --git a/python/dllib/src/bigdl/dllib/utils/common.py b/python/dllib/src/bigdl/dllib/utils/common.py index cfc4cf9fcda..4c0522a7d8a 100644 --- a/python/dllib/src/bigdl/dllib/utils/common.py +++ b/python/dllib/src/bigdl/dllib/utils/common.py @@ -617,7 +617,6 @@ def _py2java(sc, obj): sc._gateway._gateway_client) elif isinstance(obj, dict): result = {} - print(obj.keys()) for (key, value) in obj.items(): result[key] = _py2java(sc, value) obj = MapConverter().convert(result, sc._gateway._gateway_client) From b43c3dd3571f6d15972145f636d8db51ad567536 Mon Sep 17 00:00:00 2001 From: Xianyan Date: Fri, 22 Dec 2017 19:52:52 +0800 Subject: [PATCH 502/823] Add transformer doc and some fix (#2092) --- .../dllib/feature/transform/vision/image.py | 27 ++++++++++++------- python/dllib/src/bigdl/utils/common.py | 1 - 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py b/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py index 4797b27ff5a..e3c14ebf129 100644 --- a/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py +++ b/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py @@ -182,8 +182,8 @@ def __init__(self, image_list=None, label_list=None, jvalue=None, bigdl_type="fl self.value = jvalue else: # init from image ndarray list and label rdd(optional) - image_tensor_list = image_list.map(lambda image: JTensor.from_ndarray(image)) - label_tensor_list = label_list.map(lambda label: JTensor.from_ndarray(label)) if label_list else None + image_tensor_list = map(lambda image: JTensor.from_ndarray(image), image_list) + label_tensor_list = map(lambda label: JTensor.from_ndarray(label), label_list) if label_list else None self.value = callBigDlFunc(bigdl_type, JavaValue.jvm_class_constructor(self), image_tensor_list, label_tensor_list) @@ -209,7 +209,7 @@ def get_predict(self, key="predict"): get prediction list from ImageFrame """ predicts = callBigDlFunc(self.bigdl_type, "localImageFrameToPredict", self.value, key) - return map(lambda predict: (predict[0], predict[1].to_ndarray()), predicts) + return map(lambda predict: (predict[0], predict[1].to_ndarray()) if predict[1] else (predict[0], None), predicts) @@ -251,7 +251,7 @@ def get_predict(self, key="predict"): get prediction rdd from ImageFrame """ predicts = callBigDlFunc(self.bigdl_type, "distributedImageFrameToPredict", self.value, key) - return predicts.map(lambda predict: (predict[0], predict[1].to_ndarray())) + return predicts.map(lambda predict: (predict[0], predict[1].to_ndarray()) if predict[1] else (predict[0], None)) class HFlip(FeatureTransformer): @@ -269,10 +269,15 @@ class Resize(FeatureTransformer): :param resize_w width after resize :param resize_mode if resizeMode = -1, random select a mode from (Imgproc.INTER_LINEAR, Imgproc.INTER_CUBIC, Imgproc.INTER_AREA, Imgproc.INTER_NEAREST, Imgproc.INTER_LANCZOS4) + :param use_scale_factor if true, scale factor fx and fy is used, fx = fy = 0 + note that the result of the following are different + Imgproc.resize(mat, mat, new Size(resizeWH, resizeWH), 0, 0, Imgproc.INTER_LINEAR) + Imgproc.resize(mat, mat, new Size(resizeWH, resizeWH)) """ - def __init__(self, resize_h, resize_w, resize_mode = 1, bigdl_type="float"): - super(Resize, self).__init__(bigdl_type, resize_h, resize_w, resize_mode) + def __init__(self, resize_h, resize_w, resize_mode = 1, use_scale_factor=True, + bigdl_type="float"): + super(Resize, self).__init__(bigdl_type, resize_h, resize_w, resize_mode, use_scale_factor) class Brightness(FeatureTransformer): """ @@ -533,12 +538,13 @@ class MatToFloats(FeatureTransformer): :param valid_width valid width in case the mat is invalid :param valid_channel valid channel in case the mat is invalid :param out_key key to store float array + :param share_buffer share buffer of output """ def __init__(self, valid_height=300, valid_width=300, valid_channel=300, - out_key = "floats", bigdl_type="float"): + out_key = "floats", share_buffer=True, bigdl_type="float"): super(MatToFloats, self).__init__(bigdl_type, valid_height, valid_width, valid_channel, - out_key) + out_key, share_buffer) class MatToTensor(FeatureTransformer): """ @@ -574,9 +580,10 @@ def __init__(self, scales, scale_multiple_of = 1, max_size = 1000, bigdl_type="f class BytesToMat(FeatureTransformer): """ Transform byte array(original image file in byte) to OpenCVMat + :param byte_key key that maps byte array """ - def __init__(self, bigdl_type="float"): - super(BytesToMat, self).__init__(bigdl_type) + def __init__(self, byte_key = "bytes", bigdl_type="float"): + super(BytesToMat, self).__init__(bigdl_type, byte_key) class ImageFrameToSample(FeatureTransformer): """ diff --git a/python/dllib/src/bigdl/utils/common.py b/python/dllib/src/bigdl/utils/common.py index cfc4cf9fcda..4c0522a7d8a 100644 --- a/python/dllib/src/bigdl/utils/common.py +++ b/python/dllib/src/bigdl/utils/common.py @@ -617,7 +617,6 @@ def _py2java(sc, obj): sc._gateway._gateway_client) elif isinstance(obj, dict): result = {} - print(obj.keys()) for (key, value) in obj.items(): result[key] = _py2java(sc, value) obj = MapConverter().convert(result, sc._gateway._gateway_client) From e34f2eb5ae391ec65f52f4bd7e7ca46032319a02 Mon Sep 17 00:00:00 2001 From: Xianyan Date: Fri, 22 Dec 2017 19:52:52 +0800 Subject: [PATCH 503/823] Add transformer doc and some fix (#2092) --- python/dllib/test/bigdl/transform/test_image.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/python/dllib/test/bigdl/transform/test_image.py b/python/dllib/test/bigdl/transform/test_image.py index 595c3b57c93..d27a9567e38 100644 --- a/python/dllib/test/bigdl/transform/test_image.py +++ b/python/dllib/test/bigdl/transform/test_image.py @@ -160,6 +160,10 @@ def test_mat_to_floats(self): transformer = MatToFloats() self.transformer_test(transformer) + def test_mat_to_floats_no_share(self): + transformer = MatToFloats(share_buffer=False) + self.transformer_test(transformer) + def test_mat_to_tensor(self): transformer = MatToTensor() self.transformer_test(transformer) @@ -174,6 +178,14 @@ def test_image_frame_transform(self): image_frame.transform(transformer) image_frame.get_image() + def test_empty_get_predict_local(self): + image_frame = ImageFrame.read(self.image_path) + image_frame.get_predict() + + def test_empty_get_predict_distributed(self): + image_frame = ImageFrame.read(self.image_path, self.sc) + image_frame.get_predict() + if __name__ == "__main__": pytest.main([__file__]) From 42c6ec113a2b0efa443728697241bb6ce8dbbca6 Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Mon, 25 Dec 2017 17:03:03 +0800 Subject: [PATCH 504/823] add batchsize option for keras examples (#2099) * add batchsize option * update * update --- .../src/bigdl/dllib/examples/keras/README.md | 7 +++++-- .../bigdl/dllib/examples/keras/imdb_cnn_lstm.py | 11 +++++++++-- .../src/bigdl/dllib/examples/keras/mnist_cnn.py | 16 +++++++++++----- 3 files changed, 25 insertions(+), 9 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/examples/keras/README.md b/python/dllib/src/bigdl/dllib/examples/keras/README.md index 0bf7d8026f4..6907bd9dfbb 100644 --- a/python/dllib/src/bigdl/dllib/examples/keras/README.md +++ b/python/dllib/src/bigdl/dllib/examples/keras/README.md @@ -2,9 +2,11 @@ We provide several simple examples here to show how to load a Keras model into BigDL and running the model in a distributed fashion. +You may need to see the page [Keras Support](../../../../docs/docs/ProgrammingGuide/keras-support.md) first before going into these examples. + Note that the Keras version we support and test is [__Keras 1.2.2__](https://faroit.github.io/keras-docs/1.2.2/) with TensorFlow backend. -For the sake of illustration, in these examples, we first define a model in Keras, then save it as a JSON/HDF5 file and load it into BigDL. +For the sake of illustration, in these examples, we first define a model in Keras 1.2.2, then save it as a JSON/HDF5 file and load it into BigDL. You can directly run these examples if you [install BigDL from pip](../../../../docs/docs/PythonUserGuide/install-from-pip.md). After the training, good accuracy can be achieved. @@ -45,4 +47,5 @@ ${SPARK_HOME}/bin/spark-submit \ --conf spark.driver.extraClassPath=${BigDL_JAR_PATH} \ --conf spark.executor.extraClassPath=bigdl-${BIGDL_VERSION}-jar-with-dependencies.jar \ ${BigDL_HOME}/pyspark/bigdl/examples/keras/mnist_cnn.py -``` \ No newline at end of file +``` +* ```--batchSize``` an option that can be used to set batch size. \ No newline at end of file diff --git a/python/dllib/src/bigdl/dllib/examples/keras/imdb_cnn_lstm.py b/python/dllib/src/bigdl/dllib/examples/keras/imdb_cnn_lstm.py index 3d8d6c5748d..945166c213c 100644 --- a/python/dllib/src/bigdl/dllib/examples/keras/imdb_cnn_lstm.py +++ b/python/dllib/src/bigdl/dllib/examples/keras/imdb_cnn_lstm.py @@ -20,6 +20,9 @@ # The Keras version we support and test is Keras 1.2.2 with TensorFlow backend. # See README.md for how to run this example. +from optparse import OptionParser +import sys + def load_imdb(): """ @@ -59,6 +62,10 @@ def build_keras_model(): if __name__ == "__main__": + parser = OptionParser() + parser.add_option("-b", "--batchSize", type=int, dest="batchSize", default="32") + (options, args) = parser.parse_args(sys.argv) + keras_model = build_keras_model() hdf5_path = "/tmp/imdb.h5" keras_model.save(hdf5_path) @@ -86,9 +93,9 @@ def build_keras_model(): criterion=BCECriterion(), optim_method=Adam(), end_trigger=MaxEpoch(2), - batch_size=32) + batch_size=options.batchSize) optimizer.set_validation( - batch_size=32, + batch_size=options.batchSize, val_rdd=test_data, trigger=EveryEpoch(), val_method=[Top1Accuracy()] diff --git a/python/dllib/src/bigdl/dllib/examples/keras/mnist_cnn.py b/python/dllib/src/bigdl/dllib/examples/keras/mnist_cnn.py index 8fe82bfe092..a257592239c 100644 --- a/python/dllib/src/bigdl/dllib/examples/keras/mnist_cnn.py +++ b/python/dllib/src/bigdl/dllib/examples/keras/mnist_cnn.py @@ -21,6 +21,7 @@ # The Keras version we support and test is Keras 1.2.2 with TensorFlow backend. # See README.md for how to run this example. +from optparse import OptionParser from bigdl.examples.keras.keras_utils import * import keras.backend @@ -32,7 +33,7 @@ def get_mnist(sc, data_type="train", location="/tmp/mnist"): """ - Download or load MNIST dataset. + Download or load MNIST dataset to/from the specified path. Normalize and transform input data into an RDD of Sample """ from bigdl.dataset import mnist @@ -73,6 +74,11 @@ def build_keras_model(): if __name__ == "__main__": + parser = OptionParser() + parser.add_option("-b", "--batchSize", type=int, dest="batchSize", default="128") + parser.add_option("-d", "--dataPath", dest="dataPath", default="/tmp/mnist") + (options, args) = parser.parse_args(sys.argv) + keras_model = build_keras_model() json_path = "/tmp/lenet.json" save_keras_definition(keras_model, json_path) @@ -90,8 +96,8 @@ def build_keras_model(): show_bigdl_info_logs() init_engine() - train_data = get_mnist(sc, "train") - test_data = get_mnist(sc, "test") + train_data = get_mnist(sc, "train", options.dataPath) + test_data = get_mnist(sc, "test", options.dataPath) optimizer = Optimizer( model=bigdl_model, @@ -99,9 +105,9 @@ def build_keras_model(): criterion=ClassNLLCriterion(logProbAsInput=False), optim_method=Adadelta(), end_trigger=MaxEpoch(12), - batch_size=128) + batch_size=options.batchSize) optimizer.set_validation( - batch_size=128, + batch_size=options.batchSize, val_rdd=test_data, trigger=EveryEpoch(), val_method=[Top1Accuracy()] From 1d0f75f6cd036d5c6fe951168bcdb9d5ef3f91ed Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Mon, 25 Dec 2017 17:03:03 +0800 Subject: [PATCH 505/823] add batchsize option for keras examples (#2099) * add batchsize option * update * update --- .../src/bigdl/dllib/examples/keras/README.md | 7 +++++-- .../bigdl/dllib/examples/keras/imdb_cnn_lstm.py | 11 +++++++++-- .../src/bigdl/dllib/examples/keras/mnist_cnn.py | 16 +++++++++++----- 3 files changed, 25 insertions(+), 9 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/examples/keras/README.md b/python/dllib/src/bigdl/dllib/examples/keras/README.md index 0bf7d8026f4..6907bd9dfbb 100644 --- a/python/dllib/src/bigdl/dllib/examples/keras/README.md +++ b/python/dllib/src/bigdl/dllib/examples/keras/README.md @@ -2,9 +2,11 @@ We provide several simple examples here to show how to load a Keras model into BigDL and running the model in a distributed fashion. +You may need to see the page [Keras Support](../../../../docs/docs/ProgrammingGuide/keras-support.md) first before going into these examples. + Note that the Keras version we support and test is [__Keras 1.2.2__](https://faroit.github.io/keras-docs/1.2.2/) with TensorFlow backend. -For the sake of illustration, in these examples, we first define a model in Keras, then save it as a JSON/HDF5 file and load it into BigDL. +For the sake of illustration, in these examples, we first define a model in Keras 1.2.2, then save it as a JSON/HDF5 file and load it into BigDL. You can directly run these examples if you [install BigDL from pip](../../../../docs/docs/PythonUserGuide/install-from-pip.md). After the training, good accuracy can be achieved. @@ -45,4 +47,5 @@ ${SPARK_HOME}/bin/spark-submit \ --conf spark.driver.extraClassPath=${BigDL_JAR_PATH} \ --conf spark.executor.extraClassPath=bigdl-${BIGDL_VERSION}-jar-with-dependencies.jar \ ${BigDL_HOME}/pyspark/bigdl/examples/keras/mnist_cnn.py -``` \ No newline at end of file +``` +* ```--batchSize``` an option that can be used to set batch size. \ No newline at end of file diff --git a/python/dllib/src/bigdl/dllib/examples/keras/imdb_cnn_lstm.py b/python/dllib/src/bigdl/dllib/examples/keras/imdb_cnn_lstm.py index 3d8d6c5748d..945166c213c 100644 --- a/python/dllib/src/bigdl/dllib/examples/keras/imdb_cnn_lstm.py +++ b/python/dllib/src/bigdl/dllib/examples/keras/imdb_cnn_lstm.py @@ -20,6 +20,9 @@ # The Keras version we support and test is Keras 1.2.2 with TensorFlow backend. # See README.md for how to run this example. +from optparse import OptionParser +import sys + def load_imdb(): """ @@ -59,6 +62,10 @@ def build_keras_model(): if __name__ == "__main__": + parser = OptionParser() + parser.add_option("-b", "--batchSize", type=int, dest="batchSize", default="32") + (options, args) = parser.parse_args(sys.argv) + keras_model = build_keras_model() hdf5_path = "/tmp/imdb.h5" keras_model.save(hdf5_path) @@ -86,9 +93,9 @@ def build_keras_model(): criterion=BCECriterion(), optim_method=Adam(), end_trigger=MaxEpoch(2), - batch_size=32) + batch_size=options.batchSize) optimizer.set_validation( - batch_size=32, + batch_size=options.batchSize, val_rdd=test_data, trigger=EveryEpoch(), val_method=[Top1Accuracy()] diff --git a/python/dllib/src/bigdl/dllib/examples/keras/mnist_cnn.py b/python/dllib/src/bigdl/dllib/examples/keras/mnist_cnn.py index 8fe82bfe092..a257592239c 100644 --- a/python/dllib/src/bigdl/dllib/examples/keras/mnist_cnn.py +++ b/python/dllib/src/bigdl/dllib/examples/keras/mnist_cnn.py @@ -21,6 +21,7 @@ # The Keras version we support and test is Keras 1.2.2 with TensorFlow backend. # See README.md for how to run this example. +from optparse import OptionParser from bigdl.examples.keras.keras_utils import * import keras.backend @@ -32,7 +33,7 @@ def get_mnist(sc, data_type="train", location="/tmp/mnist"): """ - Download or load MNIST dataset. + Download or load MNIST dataset to/from the specified path. Normalize and transform input data into an RDD of Sample """ from bigdl.dataset import mnist @@ -73,6 +74,11 @@ def build_keras_model(): if __name__ == "__main__": + parser = OptionParser() + parser.add_option("-b", "--batchSize", type=int, dest="batchSize", default="128") + parser.add_option("-d", "--dataPath", dest="dataPath", default="/tmp/mnist") + (options, args) = parser.parse_args(sys.argv) + keras_model = build_keras_model() json_path = "/tmp/lenet.json" save_keras_definition(keras_model, json_path) @@ -90,8 +96,8 @@ def build_keras_model(): show_bigdl_info_logs() init_engine() - train_data = get_mnist(sc, "train") - test_data = get_mnist(sc, "test") + train_data = get_mnist(sc, "train", options.dataPath) + test_data = get_mnist(sc, "test", options.dataPath) optimizer = Optimizer( model=bigdl_model, @@ -99,9 +105,9 @@ def build_keras_model(): criterion=ClassNLLCriterion(logProbAsInput=False), optim_method=Adadelta(), end_trigger=MaxEpoch(12), - batch_size=128) + batch_size=options.batchSize) optimizer.set_validation( - batch_size=128, + batch_size=options.batchSize, val_rdd=test_data, trigger=EveryEpoch(), val_method=[Top1Accuracy()] From d2a2db60750df71a69bdc9dd3443c1417e7bfb9c Mon Sep 17 00:00:00 2001 From: Yao Zhang Date: Wed, 27 Dec 2017 09:45:54 +0800 Subject: [PATCH 506/823] Chagne textclassifier readme (#2105) * chagne textclassifier readme * update core --- python/dllib/src/bigdl/dllib/models/textclassifier/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/dllib/src/bigdl/dllib/models/textclassifier/README.md b/python/dllib/src/bigdl/dllib/models/textclassifier/README.md index 2abb081469f..80d44b382c5 100644 --- a/python/dllib/src/bigdl/dllib/models/textclassifier/README.md +++ b/python/dllib/src/bigdl/dllib/models/textclassifier/README.md @@ -47,7 +47,7 @@ python ${BigDL_HOME}/pyspark/bigdl/models/textclassifier/textclassifier.py --max ${SPARK_HOME}/bin/spark-submit \ --master ${MASTER} \ --driver-cores 4 \ - --driver-memory 10g \ + --driver-memory 25g \ --total-executor-cores 4 \ --executor-cores 4 \ --executor-memory 20g \ From 6acf86da90b54997d5066300ac01ab50d473522e Mon Sep 17 00:00:00 2001 From: Yao Zhang Date: Wed, 27 Dec 2017 09:45:54 +0800 Subject: [PATCH 507/823] Chagne textclassifier readme (#2105) * chagne textclassifier readme * update core --- python/dllib/src/bigdl/dllib/models/textclassifier/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/dllib/src/bigdl/dllib/models/textclassifier/README.md b/python/dllib/src/bigdl/dllib/models/textclassifier/README.md index 2abb081469f..80d44b382c5 100644 --- a/python/dllib/src/bigdl/dllib/models/textclassifier/README.md +++ b/python/dllib/src/bigdl/dllib/models/textclassifier/README.md @@ -47,7 +47,7 @@ python ${BigDL_HOME}/pyspark/bigdl/models/textclassifier/textclassifier.py --max ${SPARK_HOME}/bin/spark-submit \ --master ${MASTER} \ --driver-cores 4 \ - --driver-memory 10g \ + --driver-memory 25g \ --total-executor-cores 4 \ --executor-cores 4 \ --executor-memory 20g \ From c0956f1bec8282f8619b1c6dae925ed7df2388fb Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Thu, 28 Dec 2017 10:12:51 +0800 Subject: [PATCH 508/823] add max_epoch option (#2114) --- .../src/bigdl/dllib/examples/keras/README.md | 3 ++- .../dllib/examples/keras/imdb_cnn_lstm.py | 3 ++- .../bigdl/dllib/examples/keras/mnist_cnn.py | 3 ++- .../bigdl/dllib/models/local_lenet/README.md | 5 ++++- .../dllib/models/local_lenet/local_lenet.py | 18 +++++++++++++----- 5 files changed, 23 insertions(+), 9 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/examples/keras/README.md b/python/dllib/src/bigdl/dllib/examples/keras/README.md index 6907bd9dfbb..86fa98e5797 100644 --- a/python/dllib/src/bigdl/dllib/examples/keras/README.md +++ b/python/dllib/src/bigdl/dllib/examples/keras/README.md @@ -48,4 +48,5 @@ ${SPARK_HOME}/bin/spark-submit \ --conf spark.executor.extraClassPath=bigdl-${BIGDL_VERSION}-jar-with-dependencies.jar \ ${BigDL_HOME}/pyspark/bigdl/examples/keras/mnist_cnn.py ``` -* ```--batchSize``` an option that can be used to set batch size. \ No newline at end of file +* ```--batchSize``` an option that can be used to set batch size. +* ```--max_epoch``` an option that can be used to set how many epochs for which the model is to be trained. \ No newline at end of file diff --git a/python/dllib/src/bigdl/dllib/examples/keras/imdb_cnn_lstm.py b/python/dllib/src/bigdl/dllib/examples/keras/imdb_cnn_lstm.py index 945166c213c..b7ce5fa4478 100644 --- a/python/dllib/src/bigdl/dllib/examples/keras/imdb_cnn_lstm.py +++ b/python/dllib/src/bigdl/dllib/examples/keras/imdb_cnn_lstm.py @@ -64,6 +64,7 @@ def build_keras_model(): if __name__ == "__main__": parser = OptionParser() parser.add_option("-b", "--batchSize", type=int, dest="batchSize", default="32") + parser.add_option("-m", "--max_epoch", type=int, dest="max_epoch", default="2") (options, args) = parser.parse_args(sys.argv) keras_model = build_keras_model() @@ -92,7 +93,7 @@ def build_keras_model(): training_rdd=train_data, criterion=BCECriterion(), optim_method=Adam(), - end_trigger=MaxEpoch(2), + end_trigger=MaxEpoch(options.max_epoch), batch_size=options.batchSize) optimizer.set_validation( batch_size=options.batchSize, diff --git a/python/dllib/src/bigdl/dllib/examples/keras/mnist_cnn.py b/python/dllib/src/bigdl/dllib/examples/keras/mnist_cnn.py index a257592239c..eacb408b1ac 100644 --- a/python/dllib/src/bigdl/dllib/examples/keras/mnist_cnn.py +++ b/python/dllib/src/bigdl/dllib/examples/keras/mnist_cnn.py @@ -76,6 +76,7 @@ def build_keras_model(): if __name__ == "__main__": parser = OptionParser() parser.add_option("-b", "--batchSize", type=int, dest="batchSize", default="128") + parser.add_option("-m", "--max_epoch", type=int, dest="max_epoch", default="12") parser.add_option("-d", "--dataPath", dest="dataPath", default="/tmp/mnist") (options, args) = parser.parse_args(sys.argv) @@ -104,7 +105,7 @@ def build_keras_model(): training_rdd=train_data, criterion=ClassNLLCriterion(logProbAsInput=False), optim_method=Adadelta(), - end_trigger=MaxEpoch(12), + end_trigger=MaxEpoch(options.max_epoch), batch_size=options.batchSize) optimizer.set_validation( batch_size=options.batchSize, diff --git a/python/dllib/src/bigdl/dllib/models/local_lenet/README.md b/python/dllib/src/bigdl/dllib/models/local_lenet/README.md index 765ff4ea04c..b9f9f6ab1e6 100644 --- a/python/dllib/src/bigdl/dllib/models/local_lenet/README.md +++ b/python/dllib/src/bigdl/dllib/models/local_lenet/README.md @@ -15,4 +15,7 @@ pip install BigDL ``` export SPARK_DRIVER_MEMORY=2g python ${BigDL_HOME}/pyspark/bigdl/models/local_lenet/local_lenet.py -``` \ No newline at end of file +``` +* ```--batchSize``` an option that can be used to set batch size. +* ```--max_epoch``` an option that can be used to set how many epochs for which the model is to be trained. +* ```--dataPath``` an option that can be used to set the path for downloading mnist data, the default value is /tmp/mnist. Make sure that you have write permission to the specified path. \ No newline at end of file diff --git a/python/dllib/src/bigdl/dllib/models/local_lenet/local_lenet.py b/python/dllib/src/bigdl/dllib/models/local_lenet/local_lenet.py index 1a8f91f2c1f..0fb96befdab 100644 --- a/python/dllib/src/bigdl/dllib/models/local_lenet/local_lenet.py +++ b/python/dllib/src/bigdl/dllib/models/local_lenet/local_lenet.py @@ -54,20 +54,28 @@ def get_mnist(data_type="train", location="/tmp/mnist"): if __name__ == "__main__": + parser = OptionParser() + parser.add_option("-b", "--batchSize", type=int, dest="batchSize", default="128") + parser.add_option("-m", "--max_epoch", type=int, dest="max_epoch", default="20") + parser.add_option("-d", "--dataPath", dest="dataPath", default="/tmp/mnist") + (options, args) = parser.parse_args(sys.argv) + redire_spark_logs() show_bigdl_info_logs() init_engine() - (X_train, Y_train) = get_mnist("train") - (X_test, Y_test) = get_mnist("test") + + (X_train, Y_train) = get_mnist("train", options.dataPath) + (X_test, Y_test) = get_mnist("test", options.dataPath) + optimizer = Optimizer.create( model=build_model(10), training_set=(X_train, Y_train), criterion=ClassNLLCriterion(), optim_method=SGD(learningrate=0.01, learningrate_decay=0.0002), - end_trigger=MaxEpoch(20), - batch_size=128) + end_trigger=MaxEpoch(options.max_epoch), + batch_size=options.batchSize) optimizer.set_validation( - batch_size=128, + batch_size=options.batchSize, X_val = X_test, Y_val = Y_test, trigger=EveryEpoch(), From bad6bd8c6efce30e333976c285c645f1cc527dec Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Thu, 28 Dec 2017 10:12:51 +0800 Subject: [PATCH 509/823] add max_epoch option (#2114) --- .../src/bigdl/dllib/examples/keras/README.md | 3 ++- .../dllib/examples/keras/imdb_cnn_lstm.py | 3 ++- .../bigdl/dllib/examples/keras/mnist_cnn.py | 3 ++- .../bigdl/dllib/models/local_lenet/README.md | 5 ++++- .../dllib/models/local_lenet/local_lenet.py | 18 +++++++++++++----- 5 files changed, 23 insertions(+), 9 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/examples/keras/README.md b/python/dllib/src/bigdl/dllib/examples/keras/README.md index 6907bd9dfbb..86fa98e5797 100644 --- a/python/dllib/src/bigdl/dllib/examples/keras/README.md +++ b/python/dllib/src/bigdl/dllib/examples/keras/README.md @@ -48,4 +48,5 @@ ${SPARK_HOME}/bin/spark-submit \ --conf spark.executor.extraClassPath=bigdl-${BIGDL_VERSION}-jar-with-dependencies.jar \ ${BigDL_HOME}/pyspark/bigdl/examples/keras/mnist_cnn.py ``` -* ```--batchSize``` an option that can be used to set batch size. \ No newline at end of file +* ```--batchSize``` an option that can be used to set batch size. +* ```--max_epoch``` an option that can be used to set how many epochs for which the model is to be trained. \ No newline at end of file diff --git a/python/dllib/src/bigdl/dllib/examples/keras/imdb_cnn_lstm.py b/python/dllib/src/bigdl/dllib/examples/keras/imdb_cnn_lstm.py index 945166c213c..b7ce5fa4478 100644 --- a/python/dllib/src/bigdl/dllib/examples/keras/imdb_cnn_lstm.py +++ b/python/dllib/src/bigdl/dllib/examples/keras/imdb_cnn_lstm.py @@ -64,6 +64,7 @@ def build_keras_model(): if __name__ == "__main__": parser = OptionParser() parser.add_option("-b", "--batchSize", type=int, dest="batchSize", default="32") + parser.add_option("-m", "--max_epoch", type=int, dest="max_epoch", default="2") (options, args) = parser.parse_args(sys.argv) keras_model = build_keras_model() @@ -92,7 +93,7 @@ def build_keras_model(): training_rdd=train_data, criterion=BCECriterion(), optim_method=Adam(), - end_trigger=MaxEpoch(2), + end_trigger=MaxEpoch(options.max_epoch), batch_size=options.batchSize) optimizer.set_validation( batch_size=options.batchSize, diff --git a/python/dllib/src/bigdl/dllib/examples/keras/mnist_cnn.py b/python/dllib/src/bigdl/dllib/examples/keras/mnist_cnn.py index a257592239c..eacb408b1ac 100644 --- a/python/dllib/src/bigdl/dllib/examples/keras/mnist_cnn.py +++ b/python/dllib/src/bigdl/dllib/examples/keras/mnist_cnn.py @@ -76,6 +76,7 @@ def build_keras_model(): if __name__ == "__main__": parser = OptionParser() parser.add_option("-b", "--batchSize", type=int, dest="batchSize", default="128") + parser.add_option("-m", "--max_epoch", type=int, dest="max_epoch", default="12") parser.add_option("-d", "--dataPath", dest="dataPath", default="/tmp/mnist") (options, args) = parser.parse_args(sys.argv) @@ -104,7 +105,7 @@ def build_keras_model(): training_rdd=train_data, criterion=ClassNLLCriterion(logProbAsInput=False), optim_method=Adadelta(), - end_trigger=MaxEpoch(12), + end_trigger=MaxEpoch(options.max_epoch), batch_size=options.batchSize) optimizer.set_validation( batch_size=options.batchSize, diff --git a/python/dllib/src/bigdl/dllib/models/local_lenet/README.md b/python/dllib/src/bigdl/dllib/models/local_lenet/README.md index 765ff4ea04c..b9f9f6ab1e6 100644 --- a/python/dllib/src/bigdl/dllib/models/local_lenet/README.md +++ b/python/dllib/src/bigdl/dllib/models/local_lenet/README.md @@ -15,4 +15,7 @@ pip install BigDL ``` export SPARK_DRIVER_MEMORY=2g python ${BigDL_HOME}/pyspark/bigdl/models/local_lenet/local_lenet.py -``` \ No newline at end of file +``` +* ```--batchSize``` an option that can be used to set batch size. +* ```--max_epoch``` an option that can be used to set how many epochs for which the model is to be trained. +* ```--dataPath``` an option that can be used to set the path for downloading mnist data, the default value is /tmp/mnist. Make sure that you have write permission to the specified path. \ No newline at end of file diff --git a/python/dllib/src/bigdl/dllib/models/local_lenet/local_lenet.py b/python/dllib/src/bigdl/dllib/models/local_lenet/local_lenet.py index 1a8f91f2c1f..0fb96befdab 100644 --- a/python/dllib/src/bigdl/dllib/models/local_lenet/local_lenet.py +++ b/python/dllib/src/bigdl/dllib/models/local_lenet/local_lenet.py @@ -54,20 +54,28 @@ def get_mnist(data_type="train", location="/tmp/mnist"): if __name__ == "__main__": + parser = OptionParser() + parser.add_option("-b", "--batchSize", type=int, dest="batchSize", default="128") + parser.add_option("-m", "--max_epoch", type=int, dest="max_epoch", default="20") + parser.add_option("-d", "--dataPath", dest="dataPath", default="/tmp/mnist") + (options, args) = parser.parse_args(sys.argv) + redire_spark_logs() show_bigdl_info_logs() init_engine() - (X_train, Y_train) = get_mnist("train") - (X_test, Y_test) = get_mnist("test") + + (X_train, Y_train) = get_mnist("train", options.dataPath) + (X_test, Y_test) = get_mnist("test", options.dataPath) + optimizer = Optimizer.create( model=build_model(10), training_set=(X_train, Y_train), criterion=ClassNLLCriterion(), optim_method=SGD(learningrate=0.01, learningrate_decay=0.0002), - end_trigger=MaxEpoch(20), - batch_size=128) + end_trigger=MaxEpoch(options.max_epoch), + batch_size=options.batchSize) optimizer.set_validation( - batch_size=128, + batch_size=options.batchSize, X_val = X_test, Y_val = Y_test, trigger=EveryEpoch(), From 07b7c08eeb210cee04a3ed3aec76ac565c90c1c4 Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Thu, 28 Dec 2017 15:07:13 +0800 Subject: [PATCH 510/823] Refine KLDCriterion (#2113) * optimize KLDCriterion * address comments * fix tests --- python/dllib/src/bigdl/dllib/nn/criterion.py | 6 +++--- python/dllib/src/bigdl/dllib/nn/layer.py | 7 +++++++ 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/nn/criterion.py b/python/dllib/src/bigdl/dllib/nn/criterion.py index fd3d817de3f..82b7e1affcb 100644 --- a/python/dllib/src/bigdl/dllib/nn/criterion.py +++ b/python/dllib/src/bigdl/dllib/nn/criterion.py @@ -491,12 +491,12 @@ class KLDCriterion(Criterion): The input has to be a table. The first element of input is the mean of the distribution, the second element of input is the log_variance of the distribution. The input distribution is assumed to be diagonal. - >>> KLDCriterion = KLDCriterion() + >>> KLDCriterion = KLDCriterion(True) creating: createKLDCriterion ''' - def __init__(self, bigdl_type="float"): - super(KLDCriterion, self).__init__(None, bigdl_type) + def __init__(self, size_average=True, bigdl_type="float"): + super(KLDCriterion, self).__init__(None, bigdl_type, size_average) class GaussianCriterion(Criterion): diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 41d38acf14c..fec515ddec4 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -53,6 +53,13 @@ def of(cls, jvalue, bigdl_type="float"): def element(self): return Layer.of(self.value.element()) + def remove_pre_edges(self): + callJavaFunc(get_spark_context(), self.value.removePreEdges) + + def remove_next_edges(self): + callJavaFunc(get_spark_context(), self.value.removeNextEdges) + + class Layer(JavaValue): """ From 04f383b95802e8b0570e62529f342f94e072080e Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Thu, 28 Dec 2017 15:07:13 +0800 Subject: [PATCH 511/823] Refine KLDCriterion (#2113) * optimize KLDCriterion * address comments * fix tests --- python/dllib/src/bigdl/dllib/nn/criterion.py | 6 +++--- python/dllib/src/bigdl/dllib/nn/layer.py | 7 +++++++ 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/nn/criterion.py b/python/dllib/src/bigdl/dllib/nn/criterion.py index fd3d817de3f..82b7e1affcb 100644 --- a/python/dllib/src/bigdl/dllib/nn/criterion.py +++ b/python/dllib/src/bigdl/dllib/nn/criterion.py @@ -491,12 +491,12 @@ class KLDCriterion(Criterion): The input has to be a table. The first element of input is the mean of the distribution, the second element of input is the log_variance of the distribution. The input distribution is assumed to be diagonal. - >>> KLDCriterion = KLDCriterion() + >>> KLDCriterion = KLDCriterion(True) creating: createKLDCriterion ''' - def __init__(self, bigdl_type="float"): - super(KLDCriterion, self).__init__(None, bigdl_type) + def __init__(self, size_average=True, bigdl_type="float"): + super(KLDCriterion, self).__init__(None, bigdl_type, size_average) class GaussianCriterion(Criterion): diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 41d38acf14c..fec515ddec4 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -53,6 +53,13 @@ def of(cls, jvalue, bigdl_type="float"): def element(self): return Layer.of(self.value.element()) + def remove_pre_edges(self): + callJavaFunc(get_spark_context(), self.value.removePreEdges) + + def remove_next_edges(self): + callJavaFunc(get_spark_context(), self.value.removeNextEdges) + + class Layer(JavaValue): """ From f1be0ae7069f9efd1f1b06e8d48cf726644d7685 Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Thu, 28 Dec 2017 16:16:05 +0800 Subject: [PATCH 512/823] fix keras dense for nD input (#2125) --- python/dllib/src/bigdl/dllib/keras/converter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/dllib/src/bigdl/dllib/keras/converter.py b/python/dllib/src/bigdl/dllib/keras/converter.py index bde8cd59d8f..d95cb1acc04 100644 --- a/python/dllib/src/bigdl/dllib/keras/converter.py +++ b/python/dllib/src/bigdl/dllib/keras/converter.py @@ -515,7 +515,7 @@ def create_dense(self): return self.combo_parameter_layer(blayer, self.config) else: seq = BLayer.Sequential() - seq.add(BLayer.InferReshape([-1, out_dim], False)) + seq.add(BLayer.InferReshape([-1, in_dim], False)) seq.add(blayer) seq.add(BLayer.InferReshape([-1] + list(self.input_shape[1:-1]) + [out_dim], False)) return self.combo_parameter_layer(seq, self.config) From b621ee912ebd3275e0bd3d9e9cc8dde344fb9cc8 Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Thu, 28 Dec 2017 16:16:05 +0800 Subject: [PATCH 513/823] fix keras dense for nD input (#2125) --- python/dllib/src/bigdl/dllib/keras/converter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/dllib/src/bigdl/dllib/keras/converter.py b/python/dllib/src/bigdl/dllib/keras/converter.py index bde8cd59d8f..d95cb1acc04 100644 --- a/python/dllib/src/bigdl/dllib/keras/converter.py +++ b/python/dllib/src/bigdl/dllib/keras/converter.py @@ -515,7 +515,7 @@ def create_dense(self): return self.combo_parameter_layer(blayer, self.config) else: seq = BLayer.Sequential() - seq.add(BLayer.InferReshape([-1, out_dim], False)) + seq.add(BLayer.InferReshape([-1, in_dim], False)) seq.add(blayer) seq.add(BLayer.InferReshape([-1] + list(self.input_shape[1:-1]) + [out_dim], False)) return self.combo_parameter_layer(seq, self.config) From 87f115cd78cb415bc0fd2b1eb25658a7920ef2a4 Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Thu, 28 Dec 2017 16:16:05 +0800 Subject: [PATCH 514/823] fix keras dense for nD input (#2125) --- python/dllib/test/bigdl/keras/test_layer.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/python/dllib/test/bigdl/keras/test_layer.py b/python/dllib/test/bigdl/keras/test_layer.py index 4bc1f9a1dd2..7282d8174fe 100644 --- a/python/dllib/test/bigdl/keras/test_layer.py +++ b/python/dllib/test/bigdl/keras/test_layer.py @@ -68,17 +68,17 @@ def test_softsign(self): def test_dense(self): input_data = np.random.random_sample([2, 10, 5, 7]) layer = Dense(2, init='one', activation="relu", - input_shape=(10, ), W_regularizer=l1l2(l1=0.01, l2=0.02)) - input_data = np.random.random_sample([2, 10]) + input_shape=(10, 5, 7), W_regularizer=l1l2(l1=0.01, l2=0.02)) self.modelTestSingleLayer(input_data, layer, dump_weights=True) + input_data2 = np.random.random_sample([2, 10]) layer2 = Dense(2, init='one', activation="softplus", input_shape=(10, ), b_regularizer=l2(0.02)) - self.modelTestSingleLayer(input_data, layer2, dump_weights=True) + self.modelTestSingleLayer(input_data2, layer2, dump_weights=True) layer3 = Dense(2, init='one', input_shape=(10, ), W_regularizer=keras.regularizers.WeightRegularizer(l1=0.1)) - self.modelTestSingleLayer(input_data, layer3, dump_weights=True) + self.modelTestSingleLayer(input_data2, layer3, dump_weights=True) layer4 = Dense(2, init='glorot_uniform', activation="hard_sigmoid", input_shape=(10, )) - self.modelTestSingleLayer(input_data, layer4, dump_weights=True) + self.modelTestSingleLayer(input_data2, layer4, dump_weights=True) def test_timedistributeddense(self): input_data = np.random.random_sample([2, 4, 5]) From 28a688d7213e0ca3e2db9b2c7dc8adc78f41c834 Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Thu, 28 Dec 2017 18:08:27 +0800 Subject: [PATCH 515/823] fix keras init_method (#2128) --- python/dllib/src/bigdl/dllib/keras/converter.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/keras/converter.py b/python/dllib/src/bigdl/dllib/keras/converter.py index d95cb1acc04..9511fd6fed9 100644 --- a/python/dllib/src/bigdl/dllib/keras/converter.py +++ b/python/dllib/src/bigdl/dllib/keras/converter.py @@ -1763,8 +1763,13 @@ def create_locallyconnected2d(self): def combo_parameter_layer(self, blayer, config): blayer.set_name(config["name"]) if hasattr(blayer, "set_init_method"): - blayer.set_init_method(self.to_bigdl_init(config["init"]), - BInit.Zeros()) # Keras always set this to be zeros + try: + blayer.set_init_method(self.to_bigdl_init(config["init"]), + BInit.Zeros()) # Keras always set this to be zeros + except Exception: + warning_msg = "We don't support initialization " + config["init"] + " for now. " \ + + "Using the default instead." + warnings.warn(warning_msg) # "linear" means doing nothing if config["activation"] != "linear": activation = get_activation_by_name(config["activation"], @@ -1789,6 +1794,10 @@ def to_bigdl_init(self, kinit_method): # kinit_method is a string init = BInit.Ones() elif kinit_method == "zero": init = BInit.Zeros() + elif kinit_method == "uniform": + init = BInit.RandomUniform(lower=-0.05, upper=0.05) + elif kinit_method == "normal": + init = BInit.RandomNormal(mean=0.0, stdv=0.05) else: raise Exception("Unsupported init type: %s" % kinit_method) return init From 08e19f5f27164a8d985643ec79a0f6ab2d1e1947 Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Thu, 28 Dec 2017 18:08:27 +0800 Subject: [PATCH 516/823] fix keras init_method (#2128) --- python/dllib/src/bigdl/dllib/keras/converter.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/keras/converter.py b/python/dllib/src/bigdl/dllib/keras/converter.py index d95cb1acc04..9511fd6fed9 100644 --- a/python/dllib/src/bigdl/dllib/keras/converter.py +++ b/python/dllib/src/bigdl/dllib/keras/converter.py @@ -1763,8 +1763,13 @@ def create_locallyconnected2d(self): def combo_parameter_layer(self, blayer, config): blayer.set_name(config["name"]) if hasattr(blayer, "set_init_method"): - blayer.set_init_method(self.to_bigdl_init(config["init"]), - BInit.Zeros()) # Keras always set this to be zeros + try: + blayer.set_init_method(self.to_bigdl_init(config["init"]), + BInit.Zeros()) # Keras always set this to be zeros + except Exception: + warning_msg = "We don't support initialization " + config["init"] + " for now. " \ + + "Using the default instead." + warnings.warn(warning_msg) # "linear" means doing nothing if config["activation"] != "linear": activation = get_activation_by_name(config["activation"], @@ -1789,6 +1794,10 @@ def to_bigdl_init(self, kinit_method): # kinit_method is a string init = BInit.Ones() elif kinit_method == "zero": init = BInit.Zeros() + elif kinit_method == "uniform": + init = BInit.RandomUniform(lower=-0.05, upper=0.05) + elif kinit_method == "normal": + init = BInit.RandomNormal(mean=0.0, stdv=0.05) else: raise Exception("Unsupported init type: %s" % kinit_method) return init From 57600afaece40601b15a1ff305ca7680d7273394 Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Thu, 28 Dec 2017 18:08:27 +0800 Subject: [PATCH 517/823] fix keras init_method (#2128) --- python/dllib/test/bigdl/keras/test_layer.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/python/dllib/test/bigdl/keras/test_layer.py b/python/dllib/test/bigdl/keras/test_layer.py index 7282d8174fe..3b963905571 100644 --- a/python/dllib/test/bigdl/keras/test_layer.py +++ b/python/dllib/test/bigdl/keras/test_layer.py @@ -79,6 +79,9 @@ def test_dense(self): self.modelTestSingleLayer(input_data2, layer3, dump_weights=True) layer4 = Dense(2, init='glorot_uniform', activation="hard_sigmoid", input_shape=(10, )) self.modelTestSingleLayer(input_data2, layer4, dump_weights=True) + # Test for unsupported init_method. Should get a warning not an exception. + layer5 = Dense(4, init='he_uniform', input_shape=(10, )) + self.modelTestSingleLayer(input_data2, layer5, dump_weights=True) def test_timedistributeddense(self): input_data = np.random.random_sample([2, 4, 5]) From db95f6baef34190d0f68383aa36bbcccbb912ab4 Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Tue, 2 Jan 2018 13:52:38 +0800 Subject: [PATCH 518/823] Add TransformerCriterion (#2097) * add TransformerCriterion * fix style * fix python tests * meet review --- python/dllib/src/bigdl/dllib/nn/criterion.py | 29 ++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/nn/criterion.py b/python/dllib/src/bigdl/dllib/nn/criterion.py index 82b7e1affcb..7f46189d063 100644 --- a/python/dllib/src/bigdl/dllib/nn/criterion.py +++ b/python/dllib/src/bigdl/dllib/nn/criterion.py @@ -874,6 +874,35 @@ def __init__(self, bigdl_type="float"): super(PoissonCriterion, self).__init__(None, bigdl_type) +class TransformerCriterion(Criterion): + ''' + The criterion that takes two modules to transform input and target, and take + one criterion to compute the loss with the transformed input and target. + + This criterion can be used to construct complex criterion. For example, the + `inputTransformer` and `targetTransformer` can be pre-trained CNN networks, + and we can use the networks' output to compute the high-level feature + reconstruction loss, which is commonly used in areas like neural style transfer + (https://arxiv.org/abs/1508.06576), texture synthesis (https://arxiv.org/abs/1505.07376), + .etc. + + >>> trans = TransformerCriterion(MSECriterion()) + creating: createMSECriterion + creating: createTransformerCriterion + ''' + + def __init__(self, + criterion, + input_transformer = None, + target_transformer = None, + bigdl_type="float"): + super(TransformerCriterion, self).__init__(None, + bigdl_type, + criterion, + input_transformer, + target_transformer) + + def _test(): import doctest from pyspark import SparkContext From 7324ee434768f67edce1b1ef6e6bac1d7d0bba32 Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Tue, 2 Jan 2018 13:52:38 +0800 Subject: [PATCH 519/823] Add TransformerCriterion (#2097) * add TransformerCriterion * fix style * fix python tests * meet review --- python/dllib/src/bigdl/dllib/nn/criterion.py | 29 ++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/nn/criterion.py b/python/dllib/src/bigdl/dllib/nn/criterion.py index 82b7e1affcb..7f46189d063 100644 --- a/python/dllib/src/bigdl/dllib/nn/criterion.py +++ b/python/dllib/src/bigdl/dllib/nn/criterion.py @@ -874,6 +874,35 @@ def __init__(self, bigdl_type="float"): super(PoissonCriterion, self).__init__(None, bigdl_type) +class TransformerCriterion(Criterion): + ''' + The criterion that takes two modules to transform input and target, and take + one criterion to compute the loss with the transformed input and target. + + This criterion can be used to construct complex criterion. For example, the + `inputTransformer` and `targetTransformer` can be pre-trained CNN networks, + and we can use the networks' output to compute the high-level feature + reconstruction loss, which is commonly used in areas like neural style transfer + (https://arxiv.org/abs/1508.06576), texture synthesis (https://arxiv.org/abs/1505.07376), + .etc. + + >>> trans = TransformerCriterion(MSECriterion()) + creating: createMSECriterion + creating: createTransformerCriterion + ''' + + def __init__(self, + criterion, + input_transformer = None, + target_transformer = None, + bigdl_type="float"): + super(TransformerCriterion, self).__init__(None, + bigdl_type, + criterion, + input_transformer, + target_transformer) + + def _test(): import doctest from pyspark import SparkContext From e72cc7aaf7e40d8bbc53487de62df8189ba04252 Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Thu, 11 Jan 2018 18:09:31 +0800 Subject: [PATCH 520/823] Fix and refine keras tests (#2176) * test * fix * update * fix style * update tests * update upsampling2d test * meet review --- python/dllib/src/bigdl/dllib/nn/layer.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index fec515ddec4..f0956d8970b 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -765,6 +765,7 @@ def load_keras(json_path=None, hdf5_path=None, by_name=False): except ImportError: os.environ['KERAS_BACKEND'] = "theano" try: + # Make theano backend compatible with Python3 from theano import ifelse except ImportError: raise Exception("No backend is found for Keras. " From d69cd32e3098adac70c2cedeab7b7944e43d08f4 Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Thu, 11 Jan 2018 18:09:31 +0800 Subject: [PATCH 521/823] Fix and refine keras tests (#2176) * test * fix * update * fix style * update tests * update upsampling2d test * meet review --- python/dllib/src/bigdl/dllib/nn/layer.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index fec515ddec4..f0956d8970b 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -765,6 +765,7 @@ def load_keras(json_path=None, hdf5_path=None, by_name=False): except ImportError: os.environ['KERAS_BACKEND'] = "theano" try: + # Make theano backend compatible with Python3 from theano import ifelse except ImportError: raise Exception("No backend is found for Keras. " From bf141c0af261fd82b6bdb3b396b72d3b560e2f0c Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Thu, 11 Jan 2018 18:09:31 +0800 Subject: [PATCH 522/823] Fix and refine keras tests (#2176) * test * fix * update * fix style * update tests * update upsampling2d test * meet review --- .../test/bigdl/keras/test_application.py | 1 + python/dllib/test/bigdl/keras/test_layer.py | 57 ++++++++++++------- python/dllib/test/bigdl/test_utils.py | 5 ++ 3 files changed, 41 insertions(+), 22 deletions(-) diff --git a/python/dllib/test/bigdl/keras/test_application.py b/python/dllib/test/bigdl/keras/test_application.py index 5e1384ae8a4..9095c4cf448 100644 --- a/python/dllib/test/bigdl/keras/test_application.py +++ b/python/dllib/test/bigdl/keras/test_application.py @@ -181,5 +181,6 @@ def test_inception_v3(self): self.assert_allclose(keras_output, bigdl_output, rtol=1e-3, atol=1e-3) + if __name__ == "__main__": pytest.main([__file__]) diff --git a/python/dllib/test/bigdl/keras/test_layer.py b/python/dllib/test/bigdl/keras/test_layer.py index 3b963905571..6beb0198fda 100644 --- a/python/dllib/test/bigdl/keras/test_layer.py +++ b/python/dllib/test/bigdl/keras/test_layer.py @@ -284,7 +284,10 @@ def test_globalaveragepooling1d(self): def test_batchnormalization(self): input_data = np.random.random_sample([2, 6, 128, 128]) layer = BatchNormalization(input_shape=(6, 128, 128), axis=1) - self.modelTestSingleLayer(input_data, layer, dump_weights=True, random_weights=False) + self.modelTestSingleLayer(input_data, layer, dump_weights=True) + K.set_image_dim_ordering("tf") + layer2 = BatchNormalization(input_shape=(6, 128, 128), axis=-1) + self.modelTestSingleLayer(input_data, layer2, dump_weights=True) def test_flatten(self): input_data = np.random.random_sample([1, 2, 3]) @@ -569,19 +572,23 @@ def test_zeropadding1d(self): def test_zeropadding2d(self): input_data = np.random.uniform(0, 1, [1, 2, 3, 4]) - layer1 = ZeroPadding2D(padding=(2, 3), input_shape=(2, 3, 4)) - self.modelTestSingleLayer(input_data, layer1) - layer2 = ZeroPadding2D(padding=(2, 3, 4, 1), input_shape=(2, 3, 4)) - self.modelTestSingleLayer(input_data, layer2) - layer3 = ZeroPadding2D( + layer1 = lambda: ZeroPadding2D(padding=(2, 3), input_shape=(2, 3, 4)) + self.modelTestSingleLayerWithOrdersModes(input_data, layer1, + border_modes=[None]) + layer2 = lambda: ZeroPadding2D(padding=(2, 3, 4, 1), input_shape=(2, 3, 4)) + self.modelTestSingleLayerWithOrdersModes(input_data, layer2, + border_modes=[None]) + layer3 = lambda: ZeroPadding2D( padding={'top_pad': 1, 'bottom_pad': 2, 'left_pad': 3, 'right_pad': 4}, input_shape=(2, 3, 4)) - self.modelTestSingleLayer(input_data, layer3) + self.modelTestSingleLayerWithOrdersModes(input_data, layer3, + border_modes=[None]) def test_zeropadding3d(self): input_data = np.random.uniform(0, 1, [3, 2, 4, 1, 5]) - layer = ZeroPadding3D(padding=(1, 2, 3), input_shape=(2, 4, 1, 5)) - self.modelTestSingleLayer(input_data, layer) + layer = lambda: ZeroPadding3D(padding=(1, 2, 3), input_shape=(2, 4, 1, 5)) + self.modelTestSingleLayerWithOrdersModes(input_data, layer, + border_modes=[None]) def test_cropping1d(self): input_data = np.random.uniform(0, 1, [3, 10, 10]) @@ -589,18 +596,22 @@ def test_cropping1d(self): self.modelTestSingleLayer(input_data, layer) def test_cropping2d(self): - input_data = np.random.random([2, 3, 28, 28]) - layer1 = Cropping2D(cropping=((2, 2), (4, 4))) - self.modelTestSingleLayer(input_data, layer1) - layer2 = Cropping2D(cropping=((0, 2), (3, 1))) - self.modelTestSingleLayer(input_data, layer2) + input_data = np.random.random([2, 5, 28, 28]) + layer1 = lambda: Cropping2D(cropping=((2, 2), (4, 4))) + self.modelTestSingleLayerWithOrdersModes(input_data, layer1, + border_modes=[None]) + layer2 = lambda: Cropping2D(cropping=((0, 2), (3, 1))) + self.modelTestSingleLayerWithOrdersModes(input_data, layer2, + border_modes=[None]) def test_cropping3d(self): input_data = np.random.random([2, 10, 28, 28, 32]) - layer1 = Cropping3D(cropping=((1, 1), (2, 2), (4, 4))) - self.modelTestSingleLayer(input_data, layer1) - layer2 = Cropping3D(cropping=((0, 2), (3, 1), (2, 3))) - self.modelTestSingleLayer(input_data, layer2) + layer1 = lambda: Cropping3D(cropping=((1, 1), (2, 2), (4, 4))) + self.modelTestSingleLayerWithOrdersModes(input_data, layer1, + border_modes=[None]) + layer2 = lambda: Cropping3D(cropping=((0, 2), (3, 1), (2, 3))) + self.modelTestSingleLayerWithOrdersModes(input_data, layer2, + border_modes=[None]) def test_simplernn(self): input_data = np.random.random([3, 4, 5]) @@ -697,10 +708,12 @@ def test_upsampling1d(self): def test_upsampling2d(self): input_data = np.random.random([2, 5, 6, 8]) - layer1 = UpSampling2D(input_shape=(5, 6, 8)) - self.modelTestSingleLayer(input_data, layer1) - layer2 = UpSampling2D(size=(1, 3), input_shape=(5, 6, 8)) - self.modelTestSingleLayer(input_data, layer2) + layer1 = lambda: UpSampling2D(input_shape=(5, 6, 8)) + self.modelTestSingleLayerWithOrdersModes(input_data, layer1, + border_modes=[None]) + layer2 = lambda: UpSampling2D(size=(1, 3), input_shape=(5, 6, 8)) + self.modelTestSingleLayerWithOrdersModes(input_data, layer2, + border_modes=[None]) def test_upsampling3d(self): input_data = np.random.random([2, 5, 12, 12, 12]) diff --git a/python/dllib/test/bigdl/test_utils.py b/python/dllib/test/bigdl/test_utils.py index fc69aba96b1..ba360cba379 100644 --- a/python/dllib/test/bigdl/test_utils.py +++ b/python/dllib/test/bigdl/test_utils.py @@ -299,11 +299,16 @@ def _do_modelTestSingleLayer(self, rtol=rtol, atol=atol) + # Set Keras Backend without affecting the current dim_ordering def __set_keras_backend(self, backend): if K.backend() != backend: + current_dim = K.image_dim_ordering() os.environ['KERAS_BACKEND'] = backend from six.moves import reload_module reload_module(K) + K.set_image_dim_ordering(current_dim) assert K.backend() == backend + assert K.image_dim_ordering() == current_dim + # Make theano backend compatible with Python3 if backend == "theano": from theano import ifelse From 509f128febd6d14a7a78b99e9a807bd5552c7b8d Mon Sep 17 00:00:00 2001 From: Xianyan Date: Mon, 15 Jan 2018 18:20:03 +0800 Subject: [PATCH 523/823] Load imagenet sequence file to ImageFrame (#2184) * Load imagenet dataset to ImageFrame * Refactor and add python * meet code review --- .../dllib/feature/transform/vision/image.py | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py b/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py index e3c14ebf129..f851fa8632d 100644 --- a/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py +++ b/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py @@ -596,3 +596,37 @@ def __init__(self, input_keys=["imageTensor"], target_keys=None, sample_key="sample", bigdl_type="float"): super(ImageFrameToSample, self).__init__(bigdl_type, input_keys, target_keys, sample_key) +class PixelBytesToMat(FeatureTransformer): + """ + Transform byte array(pixels in byte) to OpenCVMat + :param byte_key key that maps byte array + """ + def __init__(self, byte_key = "bytes", bigdl_type="float"): + super(PixelBytesToMat, self).__init__(bigdl_type, byte_key) + + +class SeqFileFolder(JavaValue): + + @classmethod + def files_to_image_frame(cls, + url, + sc, + class_num, + partition_num=-1, + bigdl_type="float"): + """ + Extract hadoop sequence files from an HDFS path as ImageFrame + :param url: sequence files folder path + :param sc: spark context + :param class_num: class number of data + :param partition_num: partition number, default: Engine.nodeNumber() * Engine.coreNumber() + """ + jvalue = callBigDlFunc(bigdl_type, + "seqFilesToImageFrame", + url, + sc, + class_num, + partition_num) + return ImageFrame(jvalue=jvalue) + + From a235d2583c589febd7a7ba48409e03953f758347 Mon Sep 17 00:00:00 2001 From: Xianyan Date: Mon, 15 Jan 2018 18:20:03 +0800 Subject: [PATCH 524/823] Load imagenet sequence file to ImageFrame (#2184) * Load imagenet dataset to ImageFrame * Refactor and add python * meet code review --- .../dllib/feature/transform/vision/image.py | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py b/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py index e3c14ebf129..f851fa8632d 100644 --- a/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py +++ b/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py @@ -596,3 +596,37 @@ def __init__(self, input_keys=["imageTensor"], target_keys=None, sample_key="sample", bigdl_type="float"): super(ImageFrameToSample, self).__init__(bigdl_type, input_keys, target_keys, sample_key) +class PixelBytesToMat(FeatureTransformer): + """ + Transform byte array(pixels in byte) to OpenCVMat + :param byte_key key that maps byte array + """ + def __init__(self, byte_key = "bytes", bigdl_type="float"): + super(PixelBytesToMat, self).__init__(bigdl_type, byte_key) + + +class SeqFileFolder(JavaValue): + + @classmethod + def files_to_image_frame(cls, + url, + sc, + class_num, + partition_num=-1, + bigdl_type="float"): + """ + Extract hadoop sequence files from an HDFS path as ImageFrame + :param url: sequence files folder path + :param sc: spark context + :param class_num: class number of data + :param partition_num: partition number, default: Engine.nodeNumber() * Engine.coreNumber() + """ + jvalue = callBigDlFunc(bigdl_type, + "seqFilesToImageFrame", + url, + sc, + class_num, + partition_num) + return ImageFrame(jvalue=jvalue) + + From e0ed6f2d5e24ee16236c01186a21b950c3ac93e1 Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Tue, 16 Jan 2018 10:57:32 +0800 Subject: [PATCH 525/823] Fix optimizer state messed up when calling optimize() multiple times in DistriOptimizer (#2178) * support continue training * meet code review * add unit tests * fix set model --- python/dllib/src/bigdl/dllib/optim/optimizer.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/optim/optimizer.py b/python/dllib/src/bigdl/dllib/optim/optimizer.py index a1bfcaaa9ad..050830c8b7f 100644 --- a/python/dllib/src/bigdl/dllib/optim/optimizer.py +++ b/python/dllib/src/bigdl/dllib/optim/optimizer.py @@ -630,6 +630,13 @@ def prepare_input(self): print("Loading input ...") self.value.prepareInput() + def set_end_when(self, end_when): + """ + When to stop, passed in a [[Trigger]] + """ + self.value.setEndWhen(end_when.value) + return self + class Optimizer(BaseOptimizer): From f69b57ee160fcf35499f05233766528336e1a344 Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Tue, 16 Jan 2018 10:57:32 +0800 Subject: [PATCH 526/823] Fix optimizer state messed up when calling optimize() multiple times in DistriOptimizer (#2178) * support continue training * meet code review * add unit tests * fix set model --- python/dllib/src/bigdl/dllib/optim/optimizer.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/optim/optimizer.py b/python/dllib/src/bigdl/dllib/optim/optimizer.py index a1bfcaaa9ad..050830c8b7f 100644 --- a/python/dllib/src/bigdl/dllib/optim/optimizer.py +++ b/python/dllib/src/bigdl/dllib/optim/optimizer.py @@ -630,6 +630,13 @@ def prepare_input(self): print("Loading input ...") self.value.prepareInput() + def set_end_when(self, end_when): + """ + When to stop, passed in a [[Trigger]] + """ + self.value.setEndWhen(end_when.value) + return self + class Optimizer(BaseOptimizer): From 01ded6856cd6f1a99e68f1d1da5bd3686f7d3c08 Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Tue, 16 Jan 2018 10:57:32 +0800 Subject: [PATCH 527/823] Fix optimizer state messed up when calling optimize() multiple times in DistriOptimizer (#2178) * support continue training * meet code review * add unit tests * fix set model --- python/dllib/test/bigdl/test_simple_integration.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/dllib/test/bigdl/test_simple_integration.py b/python/dllib/test/bigdl/test_simple_integration.py index 92a5e9db9f1..778c278393d 100644 --- a/python/dllib/test/bigdl/test_simple_integration.py +++ b/python/dllib/test/bigdl/test_simple_integration.py @@ -259,6 +259,7 @@ def gen_rand_sample(): app_name="run1") optimizer.set_train_summary(train_summary) optimizer.set_val_summary(val_summary) + optimizer.set_end_when(MaxEpoch(epoch_num * 2)) trained_model = optimizer.optimize() lr_result = train_summary.read_scalar("LearningRate") From 52532c879e18bcc1ab9f5a400fede9fc83b8b147 Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Mon, 22 Jan 2018 13:11:17 +0800 Subject: [PATCH 528/823] Add NCHW support for resizebilinear (#2205) * add nchw support for resizebilinear * fix style * fixed indent * test cover more cases * remove double code and python interface * add doc --- python/dllib/src/bigdl/dllib/nn/layer.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index f0956d8970b..935b9cad0db 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -5085,17 +5085,19 @@ def __init__(self, cells, bigdl_type="float"): class ResizeBilinear(Layer): """ Resize the input image with bilinear interpolation. The input image must be a float tensor with - NHWC layout + NHWC or NCHW layout :param output_height: output height :param output_width: output width :param align_corner: align corner or not + :param data_format: the data format of the input image, NHWC or NCHW - >>> resizeBilinear = ResizeBilinear(10, 20, False) + >>> resizeBilinear = ResizeBilinear(10, 20, False, "NCHW") creating: createResizeBilinear """ - def __init__(self, output_height, output_width, align_corner, bigdl_type="float"): - super(ResizeBilinear, self).__init__(None, bigdl_type, output_height, output_width, align_corner) + def __init__(self, output_height, output_width, align_corner=False, data_format="NCHW", bigdl_type="float"): + super(ResizeBilinear, self).__init__(None, bigdl_type, output_height, + output_width, align_corner, data_format) class GaussianSampler(Layer): """ From 978c930ba172bc6e959b9ac1a11573677f74c157 Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Mon, 22 Jan 2018 13:11:17 +0800 Subject: [PATCH 529/823] Add NCHW support for resizebilinear (#2205) * add nchw support for resizebilinear * fix style * fixed indent * test cover more cases * remove double code and python interface * add doc --- python/dllib/src/bigdl/dllib/nn/layer.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index f0956d8970b..935b9cad0db 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -5085,17 +5085,19 @@ def __init__(self, cells, bigdl_type="float"): class ResizeBilinear(Layer): """ Resize the input image with bilinear interpolation. The input image must be a float tensor with - NHWC layout + NHWC or NCHW layout :param output_height: output height :param output_width: output width :param align_corner: align corner or not + :param data_format: the data format of the input image, NHWC or NCHW - >>> resizeBilinear = ResizeBilinear(10, 20, False) + >>> resizeBilinear = ResizeBilinear(10, 20, False, "NCHW") creating: createResizeBilinear """ - def __init__(self, output_height, output_width, align_corner, bigdl_type="float"): - super(ResizeBilinear, self).__init__(None, bigdl_type, output_height, output_width, align_corner) + def __init__(self, output_height, output_width, align_corner=False, data_format="NCHW", bigdl_type="float"): + super(ResizeBilinear, self).__init__(None, bigdl_type, output_height, + output_width, align_corner, data_format) class GaussianSampler(Layer): """ From 62561241eda6f6003bb9be15bf566a469986b89a Mon Sep 17 00:00:00 2001 From: Ian Wong Date: Wed, 24 Jan 2018 10:24:30 +0800 Subject: [PATCH 530/823] Refine container (#2166) * move add method to subclass of container * meet code review * fix conflict code --- python/dllib/test/dev/diff.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/dllib/test/dev/diff.py b/python/dllib/test/dev/diff.py index 781812ec4cd..e0bca163fa2 100755 --- a/python/dllib/test/dev/diff.py +++ b/python/dllib/test/dev/diff.py @@ -29,7 +29,7 @@ def extract_scala_class(class_path): exclude_key_words = set(["*", "abstract", "Const", "Fill", "Shape", - "SplitAndSelect", "StrideSlice", "Scheduler", "StaticGraph", "DynamicGraph"]) # noqa + "SplitAndSelect", "StrideSlice", "Scheduler", "StaticGraph", "DynamicGraph", "DynamicContainer"]) # noqa include_key_words = set(["Module", "Criterion", "Container", "Cell", "TensorNumeric"]) # noqa content = "\n".join([line for line in open(class_path).readlines() if all([key not in line for key in exclude_key_words])]) # noqa match = re.search(r"class ([\w]+)[^{]+", content) From d4c2a8a6494f1ca4df989b0f58b3fb502639e718 Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Thu, 25 Jan 2018 16:15:10 +0800 Subject: [PATCH 531/823] Fix python ut with wrong format and precision (#2225) * fix * more fix * support conda env * update --- python/dllib/src/bigdl/dllib/nn/layer.py | 87 ++++++-------------- python/dllib/src/bigdl/dllib/utils/common.py | 30 ++++--- 2 files changed, 44 insertions(+), 73 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 935b9cad0db..d353534c92c 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -412,10 +412,8 @@ def set_weights(self, weights): >>> weights = linear.get_weights() >>> weights[0].shape == (2,3) True - >>> weights[0][0] - array([ 1., 2., 3.], dtype=float32) - >>> weights[1] - array([ 7., 8.], dtype=float32) + >>> np.testing.assert_allclose(weights[0][0], np.array([1., 2., 3.])) + >>> np.testing.assert_allclose(weights[1], np.array([7., 8.])) >>> relu = ReLU() creating: createReLU >>> from py4j.protocol import Py4JJavaError @@ -559,46 +557,26 @@ def quantize(self): creating: createLinear >>> fc.set_weights([np.ones((2, 4)), np.ones((2,))]) >>> input = np.ones((2, 4)) - >>> fc.forward(input) - array([[ 5., 5.], - [ 5., 5.]], dtype=float32) + >>> output = fc.forward(input) + >>> expected_output = np.array([[5., 5.], [5., 5.]]) + >>> np.testing.assert_allclose(output, expected_output) >>> quantized_fc = fc.quantize() - >>> quantized_fc.forward(input) - array([[ 5., 5.], - [ 5., 5.]], dtype=float32) + >>> quantized_output = quantized_fc.forward(input) + >>> expected_quantized_output = np.array([[5., 5.], [5., 5.]]) + >>> np.testing.assert_allclose(quantized_output, expected_quantized_output) >>> assert("quantized.Linear" in quantized_fc.__str__()) >>> conv = SpatialConvolution(1, 2, 3, 3) creating: createSpatialConvolution >>> conv.set_weights([np.ones((2, 1, 3, 3)), np.zeros((2,))]) >>> input = np.ones((2, 1, 4, 4)) - >>> conv.forward(input) - array([[[[ 9., 9.], - [ 9., 9.]], - - [[ 9., 9.], - [ 9., 9.]]], - - - [[[ 9., 9.], - [ 9., 9.]], - - [[ 9., 9.], - [ 9., 9.]]]], dtype=float32) + >>> output = conv.forward(input) + >>> expected_output = np.array([[[[9., 9.], [9., 9.]], [[9., 9.], [9., 9.]]], [[[9., 9.], [9., 9.]], [[9., 9.], [9., 9.]]]]) + >>> np.testing.assert_allclose(output, expected_output) >>> quantized_conv = conv.quantize() - >>> quantized_conv.forward(input) - array([[[[ 9., 9.], - [ 9., 9.]], - - [[ 9., 9.], - [ 9., 9.]]], - - - [[[ 9., 9.], - [ 9., 9.]], - - [[ 9., 9.], - [ 9., 9.]]]], dtype=float32) + >>> quantized_output = quantized_conv.forward(input) + >>> expected_quantized_output = np.array([[[[9., 9.], [9., 9.]], [[9., 9.], [9., 9.]]], [[[9., 9.], [9., 9.]], [[9., 9.], [9., 9.]]]]) + >>> np.testing.assert_allclose(quantized_output, expected_quantized_output) >>> assert("quantized.SpatialConvolution" in quantized_conv.__str__()) >>> seq = Sequential() creating: createSequential @@ -607,25 +585,13 @@ def quantize(self): creating: createReshape >>> seq = seq.add(fc) >>> input = np.ones([1, 1, 6, 6]) - >>> seq.forward(input) - array([[ 37., 37.], - [ 37., 37.], - [ 37., 37.], - [ 37., 37.], - [ 37., 37.], - [ 37., 37.], - [ 37., 37.], - [ 37., 37.]], dtype=float32) + >>> output = seq.forward(input) + >>> expected_output = np.array([[37., 37.], [37., 37.], [37., 37.], [37., 37.], [37., 37.], [37., 37.], [37., 37.], [37., 37.]]) + >>> np.testing.assert_allclose(output, expected_output) >>> quantized_seq = seq.quantize() - >>> quantized_seq.forward(input) - array([[ 37., 37.], - [ 37., 37.], - [ 37., 37.], - [ 37., 37.], - [ 37., 37.], - [ 37., 37.], - [ 37., 37.], - [ 37., 37.]], dtype=float32) + >>> quantized_output = quantized_seq.forward(input) + >>> expected_quantized_output = np.array([[37., 37.], [37., 37.], [37., 37.], [37., 37.], [37., 37.], [37., 37.], [37., 37.], [37., 37.]]) + >>> np.testing.assert_allclose(quantized_output, expected_quantized_output) >>> assert("quantized.Linear" in quantized_seq.__str__()) >>> assert("quantized.SpatialConvolution" in quantized_seq.__str__()) ''' @@ -963,9 +929,9 @@ class SparseLinear(Layer): >>> sparselinear = SparseLinear(1000, 5, init_weight=init_weight, init_bias=init_bias) creating: createSparseLinear >>> input = JTensor.sparse(np.array([1, 3, 5, 2, 4, 6]), np.array([0, 0, 0, 1, 1, 1, 1, 5, 300, 2, 100, 500]), np.array([2, 1000])) - >>> print(sparselinear.forward(input)) - [[ 10.09569263 -10.94844246 -4.1086688 1.02527523 11.80737209] - [ 7.9651413 9.7131443 -10.22719955 0.02345783 -3.74368906]] + >>> output = sparselinear.forward(input) + >>> expected_output = np.array([[10.09569263, -10.94844246, -4.1086688, 1.02527523, 11.80737209], [7.9651413, 9.7131443, -10.22719955, 0.02345783, -3.74368906]]) + >>> np.testing.assert_allclose(output, expected_output, rtol=1e-6, atol=1e-6) ''' def __init__(self, input_size, output_size, with_bias=True, backwardStart=-1, backwardLength=-1, @@ -3085,10 +3051,9 @@ class LookupTableSparse(Layer): >>> layer1 = LookupTableSparse(10, 4, "mean") creating: createLookupTableSparse >>> layer1.set_weights(np.arange(1, 41, 1).reshape(10, 4)) # set weight to 1 to 40 - >>> layer1.forward([input, weight]) - array([[ 6.5999999 , 7.60000038, 8.60000038, 9.60000038], - [ 1. , 2. , 3. , 4. ], - [ 5. , 6. , 7. , 8. ]], dtype=float32) + >>> output = layer1.forward([input, weight]) + >>> expected_output = np.array([[6.5999999 , 7.60000038, 8.60000038, 9.60000038],[ 1., 2., 3., 4.], [5., 6., 7., 8.]]) + >>> np.testing.assert_allclose(output, expected_output, rtol=1e-6, atol=1e-6) ''' def __init__(self, diff --git a/python/dllib/src/bigdl/dllib/utils/common.py b/python/dllib/src/bigdl/dllib/utils/common.py index 4c0522a7d8a..25248e630af 100644 --- a/python/dllib/src/bigdl/dllib/utils/common.py +++ b/python/dllib/src/bigdl/dllib/utils/common.py @@ -204,12 +204,10 @@ def from_ndarray(cls, a_ndarray, bigdl_type="float"): >>> np.random.seed(123) >>> data = np.random.uniform(0, 1, (2, 3)).astype("float32") >>> result = JTensor.from_ndarray(data) - >>> print(result) - JTensor: storage: [[ 0.69646919 0.28613934 0.22685145] - [ 0.55131477 0.71946895 0.42310646]], shape: [2 3], float - >>> result - JTensor: storage: [[ 0.69646919 0.28613934 0.22685145] - [ 0.55131477 0.71946895 0.42310646]], shape: [2 3], float + >>> expected_storage = np.array([[0.69646919, 0.28613934, 0.22685145], [0.55131477, 0.71946895, 0.42310646]]) + >>> expected_shape = np.array([2, 3]) + >>> np.testing.assert_allclose(result.storage, expected_storage, rtol=1e-6, atol=1e-6) + >>> np.testing.assert_allclose(result.shape, expected_shape) >>> data_back = result.to_ndarray() >>> (data == data_back).all() True @@ -254,8 +252,12 @@ def sparse(cls, a_ndarray, i_ndarray, shape, bigdl_type="float"): >>> indices = np.arange(1, 7) >>> shape = np.array([10]) >>> result = JTensor.sparse(data, indices, shape) - >>> result - JTensor: storage: [ 1. 2. 3. 4. 5. 6.], shape: [10] ,indices [1 2 3 4 5 6], float + >>> expected_storage = np.array([1., 2., 3., 4., 5., 6.]) + >>> expected_shape = np.array([10]) + >>> expected_indices = np.array([1, 2, 3, 4, 5, 6]) + >>> np.testing.assert_allclose(result.storage, expected_storage) + >>> np.testing.assert_allclose(result.shape, expected_shape) + >>> np.testing.assert_allclose(result.indices, expected_indices) >>> tensor1 = callBigDlFunc("float", "testTensor", result) # noqa >>> array_from_tensor = tensor1.to_ndarray() >>> expected_ndarray = np.array([0, 1, 2, 3, 4, 5, 6, 0, 0, 0]) @@ -328,10 +330,14 @@ def from_ndarray(cls, features, labels, bigdl_type="float"): >>> sample_back = callBigDlFunc("float", "testSample", sample) >>> assert_allclose(sample.features[0].to_ndarray(), sample_back.features[0].to_ndarray()) >>> assert_allclose(sample.label.to_ndarray(), sample_back.label.to_ndarray()) - >>> print(sample) - Sample: features: [JTensor: storage: [[ 0.69646919 0.28613934 0.22685145] - [ 0.55131477 0.71946895 0.42310646]], shape: [2 3], float], labels: [JTensor: storage: [[ 0.98076421 0.68482971 0.48093191] - [ 0.39211753 0.343178 0.72904968]], shape: [2 3], float], + >>> expected_feature_storage = np.array(([[0.69646919, 0.28613934, 0.22685145], [0.55131477, 0.71946895, 0.42310646]])) + >>> expected_feature_shape = np.array([2, 3]) + >>> expected_label_storage = np.array(([[0.98076421, 0.68482971, 0.48093191], [0.39211753, 0.343178, 0.72904968]])) + >>> expected_label_shape = np.array([2, 3]) + >>> assert_allclose(sample.features[0].storage, expected_feature_storage, rtol=1e-6, atol=1e-6) + >>> assert_allclose(sample.features[0].shape, expected_feature_shape) + >>> assert_allclose(sample.labels[0].storage, expected_label_storage, rtol=1e-6, atol=1e-6) + >>> assert_allclose(sample.labels[0].shape, expected_label_shape) """ if isinstance(features, np.ndarray): features = [features] From 7dd2d669700bcc743c778a1496ac9b8697d9b6bb Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Thu, 25 Jan 2018 16:15:10 +0800 Subject: [PATCH 532/823] Fix python ut with wrong format and precision (#2225) * fix * more fix * support conda env * update --- python/dllib/src/bigdl/dllib/nn/layer.py | 87 +++++++----------------- python/dllib/src/bigdl/utils/common.py | 30 ++++---- 2 files changed, 44 insertions(+), 73 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 935b9cad0db..d353534c92c 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -412,10 +412,8 @@ def set_weights(self, weights): >>> weights = linear.get_weights() >>> weights[0].shape == (2,3) True - >>> weights[0][0] - array([ 1., 2., 3.], dtype=float32) - >>> weights[1] - array([ 7., 8.], dtype=float32) + >>> np.testing.assert_allclose(weights[0][0], np.array([1., 2., 3.])) + >>> np.testing.assert_allclose(weights[1], np.array([7., 8.])) >>> relu = ReLU() creating: createReLU >>> from py4j.protocol import Py4JJavaError @@ -559,46 +557,26 @@ def quantize(self): creating: createLinear >>> fc.set_weights([np.ones((2, 4)), np.ones((2,))]) >>> input = np.ones((2, 4)) - >>> fc.forward(input) - array([[ 5., 5.], - [ 5., 5.]], dtype=float32) + >>> output = fc.forward(input) + >>> expected_output = np.array([[5., 5.], [5., 5.]]) + >>> np.testing.assert_allclose(output, expected_output) >>> quantized_fc = fc.quantize() - >>> quantized_fc.forward(input) - array([[ 5., 5.], - [ 5., 5.]], dtype=float32) + >>> quantized_output = quantized_fc.forward(input) + >>> expected_quantized_output = np.array([[5., 5.], [5., 5.]]) + >>> np.testing.assert_allclose(quantized_output, expected_quantized_output) >>> assert("quantized.Linear" in quantized_fc.__str__()) >>> conv = SpatialConvolution(1, 2, 3, 3) creating: createSpatialConvolution >>> conv.set_weights([np.ones((2, 1, 3, 3)), np.zeros((2,))]) >>> input = np.ones((2, 1, 4, 4)) - >>> conv.forward(input) - array([[[[ 9., 9.], - [ 9., 9.]], - - [[ 9., 9.], - [ 9., 9.]]], - - - [[[ 9., 9.], - [ 9., 9.]], - - [[ 9., 9.], - [ 9., 9.]]]], dtype=float32) + >>> output = conv.forward(input) + >>> expected_output = np.array([[[[9., 9.], [9., 9.]], [[9., 9.], [9., 9.]]], [[[9., 9.], [9., 9.]], [[9., 9.], [9., 9.]]]]) + >>> np.testing.assert_allclose(output, expected_output) >>> quantized_conv = conv.quantize() - >>> quantized_conv.forward(input) - array([[[[ 9., 9.], - [ 9., 9.]], - - [[ 9., 9.], - [ 9., 9.]]], - - - [[[ 9., 9.], - [ 9., 9.]], - - [[ 9., 9.], - [ 9., 9.]]]], dtype=float32) + >>> quantized_output = quantized_conv.forward(input) + >>> expected_quantized_output = np.array([[[[9., 9.], [9., 9.]], [[9., 9.], [9., 9.]]], [[[9., 9.], [9., 9.]], [[9., 9.], [9., 9.]]]]) + >>> np.testing.assert_allclose(quantized_output, expected_quantized_output) >>> assert("quantized.SpatialConvolution" in quantized_conv.__str__()) >>> seq = Sequential() creating: createSequential @@ -607,25 +585,13 @@ def quantize(self): creating: createReshape >>> seq = seq.add(fc) >>> input = np.ones([1, 1, 6, 6]) - >>> seq.forward(input) - array([[ 37., 37.], - [ 37., 37.], - [ 37., 37.], - [ 37., 37.], - [ 37., 37.], - [ 37., 37.], - [ 37., 37.], - [ 37., 37.]], dtype=float32) + >>> output = seq.forward(input) + >>> expected_output = np.array([[37., 37.], [37., 37.], [37., 37.], [37., 37.], [37., 37.], [37., 37.], [37., 37.], [37., 37.]]) + >>> np.testing.assert_allclose(output, expected_output) >>> quantized_seq = seq.quantize() - >>> quantized_seq.forward(input) - array([[ 37., 37.], - [ 37., 37.], - [ 37., 37.], - [ 37., 37.], - [ 37., 37.], - [ 37., 37.], - [ 37., 37.], - [ 37., 37.]], dtype=float32) + >>> quantized_output = quantized_seq.forward(input) + >>> expected_quantized_output = np.array([[37., 37.], [37., 37.], [37., 37.], [37., 37.], [37., 37.], [37., 37.], [37., 37.], [37., 37.]]) + >>> np.testing.assert_allclose(quantized_output, expected_quantized_output) >>> assert("quantized.Linear" in quantized_seq.__str__()) >>> assert("quantized.SpatialConvolution" in quantized_seq.__str__()) ''' @@ -963,9 +929,9 @@ class SparseLinear(Layer): >>> sparselinear = SparseLinear(1000, 5, init_weight=init_weight, init_bias=init_bias) creating: createSparseLinear >>> input = JTensor.sparse(np.array([1, 3, 5, 2, 4, 6]), np.array([0, 0, 0, 1, 1, 1, 1, 5, 300, 2, 100, 500]), np.array([2, 1000])) - >>> print(sparselinear.forward(input)) - [[ 10.09569263 -10.94844246 -4.1086688 1.02527523 11.80737209] - [ 7.9651413 9.7131443 -10.22719955 0.02345783 -3.74368906]] + >>> output = sparselinear.forward(input) + >>> expected_output = np.array([[10.09569263, -10.94844246, -4.1086688, 1.02527523, 11.80737209], [7.9651413, 9.7131443, -10.22719955, 0.02345783, -3.74368906]]) + >>> np.testing.assert_allclose(output, expected_output, rtol=1e-6, atol=1e-6) ''' def __init__(self, input_size, output_size, with_bias=True, backwardStart=-1, backwardLength=-1, @@ -3085,10 +3051,9 @@ class LookupTableSparse(Layer): >>> layer1 = LookupTableSparse(10, 4, "mean") creating: createLookupTableSparse >>> layer1.set_weights(np.arange(1, 41, 1).reshape(10, 4)) # set weight to 1 to 40 - >>> layer1.forward([input, weight]) - array([[ 6.5999999 , 7.60000038, 8.60000038, 9.60000038], - [ 1. , 2. , 3. , 4. ], - [ 5. , 6. , 7. , 8. ]], dtype=float32) + >>> output = layer1.forward([input, weight]) + >>> expected_output = np.array([[6.5999999 , 7.60000038, 8.60000038, 9.60000038],[ 1., 2., 3., 4.], [5., 6., 7., 8.]]) + >>> np.testing.assert_allclose(output, expected_output, rtol=1e-6, atol=1e-6) ''' def __init__(self, diff --git a/python/dllib/src/bigdl/utils/common.py b/python/dllib/src/bigdl/utils/common.py index 4c0522a7d8a..25248e630af 100644 --- a/python/dllib/src/bigdl/utils/common.py +++ b/python/dllib/src/bigdl/utils/common.py @@ -204,12 +204,10 @@ def from_ndarray(cls, a_ndarray, bigdl_type="float"): >>> np.random.seed(123) >>> data = np.random.uniform(0, 1, (2, 3)).astype("float32") >>> result = JTensor.from_ndarray(data) - >>> print(result) - JTensor: storage: [[ 0.69646919 0.28613934 0.22685145] - [ 0.55131477 0.71946895 0.42310646]], shape: [2 3], float - >>> result - JTensor: storage: [[ 0.69646919 0.28613934 0.22685145] - [ 0.55131477 0.71946895 0.42310646]], shape: [2 3], float + >>> expected_storage = np.array([[0.69646919, 0.28613934, 0.22685145], [0.55131477, 0.71946895, 0.42310646]]) + >>> expected_shape = np.array([2, 3]) + >>> np.testing.assert_allclose(result.storage, expected_storage, rtol=1e-6, atol=1e-6) + >>> np.testing.assert_allclose(result.shape, expected_shape) >>> data_back = result.to_ndarray() >>> (data == data_back).all() True @@ -254,8 +252,12 @@ def sparse(cls, a_ndarray, i_ndarray, shape, bigdl_type="float"): >>> indices = np.arange(1, 7) >>> shape = np.array([10]) >>> result = JTensor.sparse(data, indices, shape) - >>> result - JTensor: storage: [ 1. 2. 3. 4. 5. 6.], shape: [10] ,indices [1 2 3 4 5 6], float + >>> expected_storage = np.array([1., 2., 3., 4., 5., 6.]) + >>> expected_shape = np.array([10]) + >>> expected_indices = np.array([1, 2, 3, 4, 5, 6]) + >>> np.testing.assert_allclose(result.storage, expected_storage) + >>> np.testing.assert_allclose(result.shape, expected_shape) + >>> np.testing.assert_allclose(result.indices, expected_indices) >>> tensor1 = callBigDlFunc("float", "testTensor", result) # noqa >>> array_from_tensor = tensor1.to_ndarray() >>> expected_ndarray = np.array([0, 1, 2, 3, 4, 5, 6, 0, 0, 0]) @@ -328,10 +330,14 @@ def from_ndarray(cls, features, labels, bigdl_type="float"): >>> sample_back = callBigDlFunc("float", "testSample", sample) >>> assert_allclose(sample.features[0].to_ndarray(), sample_back.features[0].to_ndarray()) >>> assert_allclose(sample.label.to_ndarray(), sample_back.label.to_ndarray()) - >>> print(sample) - Sample: features: [JTensor: storage: [[ 0.69646919 0.28613934 0.22685145] - [ 0.55131477 0.71946895 0.42310646]], shape: [2 3], float], labels: [JTensor: storage: [[ 0.98076421 0.68482971 0.48093191] - [ 0.39211753 0.343178 0.72904968]], shape: [2 3], float], + >>> expected_feature_storage = np.array(([[0.69646919, 0.28613934, 0.22685145], [0.55131477, 0.71946895, 0.42310646]])) + >>> expected_feature_shape = np.array([2, 3]) + >>> expected_label_storage = np.array(([[0.98076421, 0.68482971, 0.48093191], [0.39211753, 0.343178, 0.72904968]])) + >>> expected_label_shape = np.array([2, 3]) + >>> assert_allclose(sample.features[0].storage, expected_feature_storage, rtol=1e-6, atol=1e-6) + >>> assert_allclose(sample.features[0].shape, expected_feature_shape) + >>> assert_allclose(sample.labels[0].storage, expected_label_storage, rtol=1e-6, atol=1e-6) + >>> assert_allclose(sample.labels[0].shape, expected_label_shape) """ if isinstance(features, np.ndarray): features = [features] From 9ddfb4d87af16ed02269b4d3512a95d8c440eaf4 Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Thu, 25 Jan 2018 16:15:10 +0800 Subject: [PATCH 533/823] Fix python ut with wrong format and precision (#2225) * fix * more fix * support conda env * update --- python/dllib/test/dev/run-tests | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/python/dllib/test/dev/run-tests b/python/dllib/test/dev/run-tests index 15237a53a20..72448292867 100755 --- a/python/dllib/test/dev/run-tests +++ b/python/dllib/test/dev/run-tests @@ -22,6 +22,11 @@ cd "`dirname $0`" export DL_CORE_NUMBER=4 +if [ -n $CONDA_DEFAULT_ENV ]; +then + export PYTHON_EXECUTABLES=("python") +fi + for p in ${PYTHON_EXECUTABLES[@]} do echo "${cyan}Using python version: $p${reset}" From fe5a92ee1868fae5207bbdc5334f843390c903a8 Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Thu, 25 Jan 2018 19:38:07 +0800 Subject: [PATCH 534/823] Keras Bigdl backend example (#2164) * update loss * add loss, bigdl backend example * add test loss * update * style * support parameters for optim methods * add ut for loss * meet review * fix ut * print evaluate and predict result * fix ut * update * clean --- .../src/bigdl/dllib/bigdlkeras/backend.py | 34 ++++---- .../bigdl/dllib/bigdlkeras/optimization.py | 80 ++++++++++++++++--- .../examples/keras/imdb_bigdl_backend.py | 43 ++++++++++ .../dllib/src/bigdl/dllib/keras/converter.py | 21 +++-- python/dllib/src/bigdl/dllib/nn/criterion.py | 2 +- 5 files changed, 141 insertions(+), 39 deletions(-) create mode 100644 python/dllib/src/bigdl/dllib/examples/keras/imdb_bigdl_backend.py diff --git a/python/dllib/src/bigdl/dllib/bigdlkeras/backend.py b/python/dllib/src/bigdl/dllib/bigdlkeras/backend.py index 087cd50380c..91be1a1a528 100644 --- a/python/dllib/src/bigdl/dllib/bigdlkeras/backend.py +++ b/python/dllib/src/bigdl/dllib/bigdlkeras/backend.py @@ -20,11 +20,14 @@ from bigdl.keras.optimization import * from bigdl.util.common import get_spark_context from bigdl.util.common import to_sample_rdd +from bigdl.util.common import redire_spark_logs, show_bigdl_info_logs -class KerasModelWrapper(): +class KerasModelWrapper: def __init__(self, kmodel): + redire_spark_logs() + show_bigdl_info_logs() self.bmodel = DefinitionLoader.from_kmodel(kmodel) WeightLoader.load_weights_from_kmodel(self.bmodel, kmodel) # share the same weight. self.criterion = OptimConverter.to_bigdl_criterion(kmodel.loss) @@ -55,8 +58,8 @@ def evaluate(self, x, y, batch_size=32, sample_weight=None, is_distributed=False self.bmodel.evaluate(input, batch_size, self.metrics)] else: raise Exception("No Metrics found.") - - raise Exception("not supported operation: %s", is_distributed) + else: + raise Exception("We only support evaluation in distributed mode") def predict(self, x, batch_size=None, verbose=None, is_distributed=False): """Generates output predictions for the input samples, @@ -93,6 +96,7 @@ def fit(self, x, y=None, batch_size=32, nb_epoch=10, verbose=1, callbacks=None, :param y: ndarray or list of ndarray for local mode and would be None for cluster mode. is_distributed: used to control run in local or cluster. the default value is False. NB: if is_distributed=true, x should be RDD[Sample] and y should be None + :param is_distributed: Whether to train in local mode or distributed mode :return: A Numpy array or RDD[Sample] of predictions. """ @@ -109,15 +113,15 @@ def fit(self, x, y=None, batch_size=32, nb_epoch=10, verbose=1, callbacks=None, if validation_split != 0.: unsupport_exp("validation_split") bopt = self.__create_optimizer(x=x, - y=y, - batch_size=batch_size, - nb_epoch=nb_epoch, - validation_data=validation_data, - is_distributed=is_distributed) + y=y, + batch_size=batch_size, + nb_epoch=nb_epoch, + validation_data=validation_data, + is_distributed=is_distributed) bopt.optimize() def __create_optimizer(self, x=None, y=None, batch_size=32, nb_epoch=10, - validation_data=None, is_distributed=False): + validation_data=None, is_distributed=False): if is_distributed: if isinstance(x, np.ndarray): input = to_sample_rdd(x, y) @@ -126,15 +130,15 @@ def __create_optimizer(self, x=None, y=None, batch_size=32, nb_epoch=10, input = x validation_data_rdd = validation_data return self.__create_distributed_optimizer(training_rdd=input, - batch_size=batch_size, - nb_epoch=nb_epoch, - validation_data=validation_data_rdd) + batch_size=batch_size, + nb_epoch=nb_epoch, + validation_data=validation_data_rdd) else: if isinstance(x, np.ndarray): return self.__create_local_optimizer(x, y, - batch_size=batch_size, - nb_epoch=nb_epoch, - validation_data=validation_data) + batch_size=batch_size, + nb_epoch=nb_epoch, + validation_data=validation_data) raise Exception("not supported type: %s" % x) def __create_local_optimizer(self, x, y, batch_size=32, nb_epoch=10, validation_data=None): diff --git a/python/dllib/src/bigdl/dllib/bigdlkeras/optimization.py b/python/dllib/src/bigdl/dllib/bigdlkeras/optimization.py index 362603a13d3..250f3d4f886 100644 --- a/python/dllib/src/bigdl/dllib/bigdlkeras/optimization.py +++ b/python/dllib/src/bigdl/dllib/bigdlkeras/optimization.py @@ -20,6 +20,8 @@ import bigdl.optim.optimizer as boptimizer import bigdl.util.common as bcommon from bigdl.keras.converter import * +from keras.objectives import * +import six class OptimConverter: @@ -38,26 +40,80 @@ def to_bigdl_metrics(metrics): @staticmethod def to_bigdl_criterion(kloss): - # TODO: it may pass in an object and with parameters - if kloss == "categorical_crossentropy": - return bcriterion.ClassNLLCriterion() - elif kloss == "mse" or kloss == "mean_squared_error": + if isinstance(kloss, six.string_types): + kloss = kloss.lower() + + if kloss == "categorical_crossentropy" or kloss == categorical_crossentropy: + return bcriterion.CategoricalCrossEntropy() + elif kloss == "mse" or kloss == "mean_squared_error" or kloss == mse: return bcriterion.MSECriterion() - elif kloss == "binary_crossentropy": + elif kloss == "binary_crossentropy" or kloss == binary_crossentropy: return bcriterion.BCECriterion() - elif kloss == "mae" or kloss == "mean_absolute_error": + elif kloss == "mae" or kloss == "mean_absolute_error" or kloss == mae: return bcriterion.AbsCriterion() + elif kloss == "hinge" or kloss == hinge: + return bcriterion.MarginCriterion() + elif kloss == "mean_absolute_percentage_error" or \ + kloss == "mape" or kloss == mean_absolute_percentage_error: + return bcriterion.MeanAbsolutePercentageCriterion() + elif kloss == "mean_squared_logarithmic_error" or \ + kloss == "msle" or kloss == mean_squared_logarithmic_error: + return bcriterion.MeanSquaredLogarithmicCriterion() + elif kloss == "squared_hinge" or kloss == squared_hinge: + return bcriterion.MarginCriterion(squared=True) + elif kloss == "sparse_categorical_crossentropy" or \ + kloss == sparse_categorical_crossentropy: + return bcriterion.ClassNLLCriterion(logProbAsInput=False) + elif kloss == "kullback_leibler_divergence" or \ + kloss == "kld" or kloss == kullback_leibler_divergence: + return bcriterion.KullbackLeiblerDivergenceCriterion() + elif kloss == "poisson" or kloss == poisson: + return bcriterion.PoissonCriterion() + elif kloss == "cosine_proximity" or kloss == "cosine" or kloss == cosine_proximity: + return bcriterion.CosineProximityCriterion() else: - raise Exception("Not supported type: %s" % kloss) + raise Exception("Not supported loss: %s" % kloss) @staticmethod def to_bigdl_optim_method(koptim_method): - # This is always be an object + # koptim_method is always an object + lr = float(K.eval(koptim_method.lr)) + decay = float(K.eval(koptim_method.decay)) if isinstance(koptim_method, koptimizers.Adagrad): - return boptimizer.Adagrad() + warnings.warn("For Adagrad, we don't support epsilon for now") + return boptimizer.Adagrad(learningrate=lr, + learningrate_decay=decay) elif isinstance(koptim_method, koptimizers.SGD): - return boptimizer.SGD(learningrate=0.01) # TODO: enrich parameters. ie: lr + momentum = float(K.eval(koptim_method.momentum)) + return boptimizer.SGD(learningrate=lr, + learningrate_decay=decay, + momentum=momentum, + nesterov=koptim_method.nesterov) elif isinstance(koptim_method, koptimizers.Adam): - return boptimizer.Adam() + beta1 = float(K.eval(koptim_method.beta_1)) + beta2 = float(K.eval(koptim_method.beta_2)) + return boptimizer.Adam(learningrate=lr, + learningrate_decay=decay, + beta1=beta1, + beta2=beta2, + epsilon=koptim_method.epsilon) + elif isinstance(koptim_method, koptimizers.RMSprop): + rho = float(K.eval(koptim_method.rho)) + return boptimizer.RMSprop(learningrate=lr, + learningrate_decay=decay, + decayrate=rho, + epsilon=koptim_method.epsilon) + elif isinstance(koptim_method, koptimizers.Adadelta): + warnings.warn("For Adadelta, we don't support learning rate and learning rate decay for now") + return boptimizer.Adadelta(decayrate=koptim_method.rho, + epsilon=koptim_method.epsilon) + elif isinstance(koptim_method, koptimizers.Adamax): + beta1 = float(K.eval(koptim_method.beta_1)) + beta2 = float(K.eval(koptim_method.beta_2)) + warnings.warn("For Adamax, we don't support learning rate decay for now") + return boptimizer.Adamax(learningrate=lr, + beta1=beta1, + beta2=beta2, + epsilon=koptim_method.epsilon) else: - unsupport_exp(koptim_method) \ No newline at end of file + unsupport_exp(koptim_method) diff --git a/python/dllib/src/bigdl/dllib/examples/keras/imdb_bigdl_backend.py b/python/dllib/src/bigdl/dllib/examples/keras/imdb_bigdl_backend.py new file mode 100644 index 00000000000..ee80c97a04c --- /dev/null +++ b/python/dllib/src/bigdl/dllib/examples/keras/imdb_bigdl_backend.py @@ -0,0 +1,43 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +# Alternative of the example imdb_cnn_lstm.py using BigDL backend +# You can essentially employ the same code written in Keras 1.2.2 and add the following line +# after building, compiling the model and before the training. +# model = with_bigdl_backend(model) + +from bigdl.keras.backend import * +from bigdl.examples.keras.imdb_cnn_lstm import * + +X_train, y_train, X_test, y_test = load_imdb() + +# Define a model in Keras 1.2.2 and compile +keras_model = build_keras_model() +keras_model.compile(loss='binary_crossentropy', + optimizer='adam', + metrics=['accuracy']) + +# Add this line of code to use BigDL backend alternatively +model = with_bigdl_backend(keras_model) + +model.fit(X_train, y_train, batch_size=32, nb_epoch=1, + validation_data=(X_test, y_test), is_distributed=True) +# You may need to use data different from validation to evaluate or predict the model. +accuracy = model.evaluate(X_test, y_test, batch_size=32, is_distributed=True) +print("Top1Accuracy: ", accuracy[0]) +results = model.predict(X_test, is_distributed=True) +print("First predict result: ", results.take(1)) diff --git a/python/dllib/src/bigdl/dllib/keras/converter.py b/python/dllib/src/bigdl/dllib/keras/converter.py index 9511fd6fed9..272c6923240 100644 --- a/python/dllib/src/bigdl/dllib/keras/converter.py +++ b/python/dllib/src/bigdl/dllib/keras/converter.py @@ -1476,12 +1476,12 @@ def create_globalmaxpooling1d(self): b_kh = int(self.input_shape[1]) seq = BLayer.Sequential() - seq.add(BLayer.View([int(self.input_shape[1]), 1, int(self.input_shape[2])], num_input_dims=2)) + seq.add(BLayer.Reshape([int(self.input_shape[1]), 1, int(self.input_shape[2])], True)) blayer = BLayer.SpatialMaxPooling( kw=b_kw, kh=b_kh, - dw=0, - dh=0, + dw=1, + dh=1, pad_w=0, pad_h=0, to_ceil=False, @@ -1489,8 +1489,8 @@ def create_globalmaxpooling1d(self): bigdl_type="float" ) seq.add(blayer) - seq.add(BLayer.Squeeze(2, num_input_dims=2)) - seq.add(BLayer.Squeeze(1, num_input_dims=1)) + seq.add(BLayer.Squeeze(3)) + seq.add(BLayer.Squeeze(2)) return seq def create_globalaveragepooling1d(self): @@ -1498,12 +1498,12 @@ def create_globalaveragepooling1d(self): b_kh = int(self.input_shape[1]) seq = BLayer.Sequential() - seq.add(BLayer.View([int(self.input_shape[1]), 1, int(self.input_shape[2])], num_input_dims=2)) + seq.add(BLayer.Reshape([int(self.input_shape[1]), 1, int(self.input_shape[2])], True)) blayer = BLayer.SpatialAveragePooling( kw=b_kw, kh=b_kh, - dw=0, - dh=0, + dw=1, + dh=1, pad_w=0, pad_h=0, global_pooling=False, @@ -1514,9 +1514,8 @@ def create_globalaveragepooling1d(self): bigdl_type="float" ) seq.add(blayer) - seq.add(BLayer.Squeeze(2, num_input_dims=2)) # the index start from one but without batch - seq.add(BLayer.Squeeze(1, num_input_dims=1)) - + seq.add(BLayer.Squeeze(3)) + seq.add(BLayer.Squeeze(2)) return seq def create_maxpooling1d(self): diff --git a/python/dllib/src/bigdl/dllib/nn/criterion.py b/python/dllib/src/bigdl/dllib/nn/criterion.py index 7f46189d063..8af46365b8d 100644 --- a/python/dllib/src/bigdl/dllib/nn/criterion.py +++ b/python/dllib/src/bigdl/dllib/nn/criterion.py @@ -820,7 +820,7 @@ class MeanAbsolutePercentageCriterion(Criterion): ''' This method is same as `mean_absolute_percentage_error` loss in keras. It caculates diff = K.abs((y - x) / K.clip(K.abs(y), K.epsilon(), Double.MaxValue)) - and return 100 * K.mean(diff) as outpout. Here, the x and y can have or not have a batch. + and return 100 * K.mean(diff) as output. Here, the x and y can have or not have a batch. >>> error = MeanAbsolutePercentageCriterion() creating: createMeanAbsolutePercentageCriterion ''' From 9ab83c0a0a1eeee34da6eb899b7003aea814280c Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Thu, 25 Jan 2018 19:38:07 +0800 Subject: [PATCH 535/823] Keras Bigdl backend example (#2164) * update loss * add loss, bigdl backend example * add test loss * update * style * support parameters for optim methods * add ut for loss * meet review * fix ut * print evaluate and predict result * fix ut * update * clean --- .../src/bigdl/dllib/bigdlkeras/backend.py | 34 ++++---- .../bigdl/dllib/bigdlkeras/optimization.py | 80 ++++++++++++++++--- .../examples/keras/imdb_bigdl_backend.py | 43 ++++++++++ .../dllib/src/bigdl/dllib/keras/converter.py | 21 +++-- python/dllib/src/bigdl/dllib/nn/criterion.py | 2 +- 5 files changed, 141 insertions(+), 39 deletions(-) create mode 100644 python/dllib/src/bigdl/dllib/examples/keras/imdb_bigdl_backend.py diff --git a/python/dllib/src/bigdl/dllib/bigdlkeras/backend.py b/python/dllib/src/bigdl/dllib/bigdlkeras/backend.py index 087cd50380c..91be1a1a528 100644 --- a/python/dllib/src/bigdl/dllib/bigdlkeras/backend.py +++ b/python/dllib/src/bigdl/dllib/bigdlkeras/backend.py @@ -20,11 +20,14 @@ from bigdl.keras.optimization import * from bigdl.util.common import get_spark_context from bigdl.util.common import to_sample_rdd +from bigdl.util.common import redire_spark_logs, show_bigdl_info_logs -class KerasModelWrapper(): +class KerasModelWrapper: def __init__(self, kmodel): + redire_spark_logs() + show_bigdl_info_logs() self.bmodel = DefinitionLoader.from_kmodel(kmodel) WeightLoader.load_weights_from_kmodel(self.bmodel, kmodel) # share the same weight. self.criterion = OptimConverter.to_bigdl_criterion(kmodel.loss) @@ -55,8 +58,8 @@ def evaluate(self, x, y, batch_size=32, sample_weight=None, is_distributed=False self.bmodel.evaluate(input, batch_size, self.metrics)] else: raise Exception("No Metrics found.") - - raise Exception("not supported operation: %s", is_distributed) + else: + raise Exception("We only support evaluation in distributed mode") def predict(self, x, batch_size=None, verbose=None, is_distributed=False): """Generates output predictions for the input samples, @@ -93,6 +96,7 @@ def fit(self, x, y=None, batch_size=32, nb_epoch=10, verbose=1, callbacks=None, :param y: ndarray or list of ndarray for local mode and would be None for cluster mode. is_distributed: used to control run in local or cluster. the default value is False. NB: if is_distributed=true, x should be RDD[Sample] and y should be None + :param is_distributed: Whether to train in local mode or distributed mode :return: A Numpy array or RDD[Sample] of predictions. """ @@ -109,15 +113,15 @@ def fit(self, x, y=None, batch_size=32, nb_epoch=10, verbose=1, callbacks=None, if validation_split != 0.: unsupport_exp("validation_split") bopt = self.__create_optimizer(x=x, - y=y, - batch_size=batch_size, - nb_epoch=nb_epoch, - validation_data=validation_data, - is_distributed=is_distributed) + y=y, + batch_size=batch_size, + nb_epoch=nb_epoch, + validation_data=validation_data, + is_distributed=is_distributed) bopt.optimize() def __create_optimizer(self, x=None, y=None, batch_size=32, nb_epoch=10, - validation_data=None, is_distributed=False): + validation_data=None, is_distributed=False): if is_distributed: if isinstance(x, np.ndarray): input = to_sample_rdd(x, y) @@ -126,15 +130,15 @@ def __create_optimizer(self, x=None, y=None, batch_size=32, nb_epoch=10, input = x validation_data_rdd = validation_data return self.__create_distributed_optimizer(training_rdd=input, - batch_size=batch_size, - nb_epoch=nb_epoch, - validation_data=validation_data_rdd) + batch_size=batch_size, + nb_epoch=nb_epoch, + validation_data=validation_data_rdd) else: if isinstance(x, np.ndarray): return self.__create_local_optimizer(x, y, - batch_size=batch_size, - nb_epoch=nb_epoch, - validation_data=validation_data) + batch_size=batch_size, + nb_epoch=nb_epoch, + validation_data=validation_data) raise Exception("not supported type: %s" % x) def __create_local_optimizer(self, x, y, batch_size=32, nb_epoch=10, validation_data=None): diff --git a/python/dllib/src/bigdl/dllib/bigdlkeras/optimization.py b/python/dllib/src/bigdl/dllib/bigdlkeras/optimization.py index 362603a13d3..250f3d4f886 100644 --- a/python/dllib/src/bigdl/dllib/bigdlkeras/optimization.py +++ b/python/dllib/src/bigdl/dllib/bigdlkeras/optimization.py @@ -20,6 +20,8 @@ import bigdl.optim.optimizer as boptimizer import bigdl.util.common as bcommon from bigdl.keras.converter import * +from keras.objectives import * +import six class OptimConverter: @@ -38,26 +40,80 @@ def to_bigdl_metrics(metrics): @staticmethod def to_bigdl_criterion(kloss): - # TODO: it may pass in an object and with parameters - if kloss == "categorical_crossentropy": - return bcriterion.ClassNLLCriterion() - elif kloss == "mse" or kloss == "mean_squared_error": + if isinstance(kloss, six.string_types): + kloss = kloss.lower() + + if kloss == "categorical_crossentropy" or kloss == categorical_crossentropy: + return bcriterion.CategoricalCrossEntropy() + elif kloss == "mse" or kloss == "mean_squared_error" or kloss == mse: return bcriterion.MSECriterion() - elif kloss == "binary_crossentropy": + elif kloss == "binary_crossentropy" or kloss == binary_crossentropy: return bcriterion.BCECriterion() - elif kloss == "mae" or kloss == "mean_absolute_error": + elif kloss == "mae" or kloss == "mean_absolute_error" or kloss == mae: return bcriterion.AbsCriterion() + elif kloss == "hinge" or kloss == hinge: + return bcriterion.MarginCriterion() + elif kloss == "mean_absolute_percentage_error" or \ + kloss == "mape" or kloss == mean_absolute_percentage_error: + return bcriterion.MeanAbsolutePercentageCriterion() + elif kloss == "mean_squared_logarithmic_error" or \ + kloss == "msle" or kloss == mean_squared_logarithmic_error: + return bcriterion.MeanSquaredLogarithmicCriterion() + elif kloss == "squared_hinge" or kloss == squared_hinge: + return bcriterion.MarginCriterion(squared=True) + elif kloss == "sparse_categorical_crossentropy" or \ + kloss == sparse_categorical_crossentropy: + return bcriterion.ClassNLLCriterion(logProbAsInput=False) + elif kloss == "kullback_leibler_divergence" or \ + kloss == "kld" or kloss == kullback_leibler_divergence: + return bcriterion.KullbackLeiblerDivergenceCriterion() + elif kloss == "poisson" or kloss == poisson: + return bcriterion.PoissonCriterion() + elif kloss == "cosine_proximity" or kloss == "cosine" or kloss == cosine_proximity: + return bcriterion.CosineProximityCriterion() else: - raise Exception("Not supported type: %s" % kloss) + raise Exception("Not supported loss: %s" % kloss) @staticmethod def to_bigdl_optim_method(koptim_method): - # This is always be an object + # koptim_method is always an object + lr = float(K.eval(koptim_method.lr)) + decay = float(K.eval(koptim_method.decay)) if isinstance(koptim_method, koptimizers.Adagrad): - return boptimizer.Adagrad() + warnings.warn("For Adagrad, we don't support epsilon for now") + return boptimizer.Adagrad(learningrate=lr, + learningrate_decay=decay) elif isinstance(koptim_method, koptimizers.SGD): - return boptimizer.SGD(learningrate=0.01) # TODO: enrich parameters. ie: lr + momentum = float(K.eval(koptim_method.momentum)) + return boptimizer.SGD(learningrate=lr, + learningrate_decay=decay, + momentum=momentum, + nesterov=koptim_method.nesterov) elif isinstance(koptim_method, koptimizers.Adam): - return boptimizer.Adam() + beta1 = float(K.eval(koptim_method.beta_1)) + beta2 = float(K.eval(koptim_method.beta_2)) + return boptimizer.Adam(learningrate=lr, + learningrate_decay=decay, + beta1=beta1, + beta2=beta2, + epsilon=koptim_method.epsilon) + elif isinstance(koptim_method, koptimizers.RMSprop): + rho = float(K.eval(koptim_method.rho)) + return boptimizer.RMSprop(learningrate=lr, + learningrate_decay=decay, + decayrate=rho, + epsilon=koptim_method.epsilon) + elif isinstance(koptim_method, koptimizers.Adadelta): + warnings.warn("For Adadelta, we don't support learning rate and learning rate decay for now") + return boptimizer.Adadelta(decayrate=koptim_method.rho, + epsilon=koptim_method.epsilon) + elif isinstance(koptim_method, koptimizers.Adamax): + beta1 = float(K.eval(koptim_method.beta_1)) + beta2 = float(K.eval(koptim_method.beta_2)) + warnings.warn("For Adamax, we don't support learning rate decay for now") + return boptimizer.Adamax(learningrate=lr, + beta1=beta1, + beta2=beta2, + epsilon=koptim_method.epsilon) else: - unsupport_exp(koptim_method) \ No newline at end of file + unsupport_exp(koptim_method) diff --git a/python/dllib/src/bigdl/dllib/examples/keras/imdb_bigdl_backend.py b/python/dllib/src/bigdl/dllib/examples/keras/imdb_bigdl_backend.py new file mode 100644 index 00000000000..ee80c97a04c --- /dev/null +++ b/python/dllib/src/bigdl/dllib/examples/keras/imdb_bigdl_backend.py @@ -0,0 +1,43 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +# Alternative of the example imdb_cnn_lstm.py using BigDL backend +# You can essentially employ the same code written in Keras 1.2.2 and add the following line +# after building, compiling the model and before the training. +# model = with_bigdl_backend(model) + +from bigdl.keras.backend import * +from bigdl.examples.keras.imdb_cnn_lstm import * + +X_train, y_train, X_test, y_test = load_imdb() + +# Define a model in Keras 1.2.2 and compile +keras_model = build_keras_model() +keras_model.compile(loss='binary_crossentropy', + optimizer='adam', + metrics=['accuracy']) + +# Add this line of code to use BigDL backend alternatively +model = with_bigdl_backend(keras_model) + +model.fit(X_train, y_train, batch_size=32, nb_epoch=1, + validation_data=(X_test, y_test), is_distributed=True) +# You may need to use data different from validation to evaluate or predict the model. +accuracy = model.evaluate(X_test, y_test, batch_size=32, is_distributed=True) +print("Top1Accuracy: ", accuracy[0]) +results = model.predict(X_test, is_distributed=True) +print("First predict result: ", results.take(1)) diff --git a/python/dllib/src/bigdl/dllib/keras/converter.py b/python/dllib/src/bigdl/dllib/keras/converter.py index 9511fd6fed9..272c6923240 100644 --- a/python/dllib/src/bigdl/dllib/keras/converter.py +++ b/python/dllib/src/bigdl/dllib/keras/converter.py @@ -1476,12 +1476,12 @@ def create_globalmaxpooling1d(self): b_kh = int(self.input_shape[1]) seq = BLayer.Sequential() - seq.add(BLayer.View([int(self.input_shape[1]), 1, int(self.input_shape[2])], num_input_dims=2)) + seq.add(BLayer.Reshape([int(self.input_shape[1]), 1, int(self.input_shape[2])], True)) blayer = BLayer.SpatialMaxPooling( kw=b_kw, kh=b_kh, - dw=0, - dh=0, + dw=1, + dh=1, pad_w=0, pad_h=0, to_ceil=False, @@ -1489,8 +1489,8 @@ def create_globalmaxpooling1d(self): bigdl_type="float" ) seq.add(blayer) - seq.add(BLayer.Squeeze(2, num_input_dims=2)) - seq.add(BLayer.Squeeze(1, num_input_dims=1)) + seq.add(BLayer.Squeeze(3)) + seq.add(BLayer.Squeeze(2)) return seq def create_globalaveragepooling1d(self): @@ -1498,12 +1498,12 @@ def create_globalaveragepooling1d(self): b_kh = int(self.input_shape[1]) seq = BLayer.Sequential() - seq.add(BLayer.View([int(self.input_shape[1]), 1, int(self.input_shape[2])], num_input_dims=2)) + seq.add(BLayer.Reshape([int(self.input_shape[1]), 1, int(self.input_shape[2])], True)) blayer = BLayer.SpatialAveragePooling( kw=b_kw, kh=b_kh, - dw=0, - dh=0, + dw=1, + dh=1, pad_w=0, pad_h=0, global_pooling=False, @@ -1514,9 +1514,8 @@ def create_globalaveragepooling1d(self): bigdl_type="float" ) seq.add(blayer) - seq.add(BLayer.Squeeze(2, num_input_dims=2)) # the index start from one but without batch - seq.add(BLayer.Squeeze(1, num_input_dims=1)) - + seq.add(BLayer.Squeeze(3)) + seq.add(BLayer.Squeeze(2)) return seq def create_maxpooling1d(self): diff --git a/python/dllib/src/bigdl/dllib/nn/criterion.py b/python/dllib/src/bigdl/dllib/nn/criterion.py index 7f46189d063..8af46365b8d 100644 --- a/python/dllib/src/bigdl/dllib/nn/criterion.py +++ b/python/dllib/src/bigdl/dllib/nn/criterion.py @@ -820,7 +820,7 @@ class MeanAbsolutePercentageCriterion(Criterion): ''' This method is same as `mean_absolute_percentage_error` loss in keras. It caculates diff = K.abs((y - x) / K.clip(K.abs(y), K.epsilon(), Double.MaxValue)) - and return 100 * K.mean(diff) as outpout. Here, the x and y can have or not have a batch. + and return 100 * K.mean(diff) as output. Here, the x and y can have or not have a batch. >>> error = MeanAbsolutePercentageCriterion() creating: createMeanAbsolutePercentageCriterion ''' From 49ca201462330ca978436f9993637d835fc187ea Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Thu, 25 Jan 2018 19:38:07 +0800 Subject: [PATCH 536/823] Keras Bigdl backend example (#2164) * update loss * add loss, bigdl backend example * add test loss * update * style * support parameters for optim methods * add ut for loss * meet review * fix ut * print evaluate and predict result * fix ut * update * clean --- .../test/bigdl/keras/test_application.py | 22 ++-- python/dllib/test/bigdl/keras/test_backend.py | 12 +- python/dllib/test/bigdl/keras/test_loss.py | 120 ++++++++++++++++++ python/dllib/test/bigdl/test_utils.py | 6 + 4 files changed, 139 insertions(+), 21 deletions(-) create mode 100644 python/dllib/test/bigdl/keras/test_loss.py diff --git a/python/dllib/test/bigdl/keras/test_application.py b/python/dllib/test/bigdl/keras/test_application.py index 9095c4cf448..36d2926bffb 100644 --- a/python/dllib/test/bigdl/keras/test_application.py +++ b/python/dllib/test/bigdl/keras/test_application.py @@ -24,6 +24,7 @@ from keras.applications.music_tagger_crnn import MusicTaggerCRNN from test.bigdl.test_utils import BigDLTestCase, TestModels +from bigdl.keras.backend import * class TestApplication(BigDLTestCase): @@ -43,14 +44,10 @@ def test_lenet(self): kmodel, input_data, output_data = TestModels.kmodel_seq_lenet_mnist() self.modelTest(input_data, kmodel, dump_weights=True) - @pytest.mark.skip(reason="need to fix todo before running the test") def test_text_classification(self): - '''This example demonstrates the use of Convolution1D for text classification. - This example is from Keras - ''' - - # TODO: support backend support - + # This example demonstrates the use of Convolution1D for text classification. + # This example is from Keras + K.set_image_dim_ordering("th") import numpy as np np.random.seed(1337) # for reproducibility @@ -69,7 +66,7 @@ def test_text_classification(self): nb_filter = 250 filter_length = 3 hidden_dims = 250 - nb_epoch = 2 + nb_epoch = 1 print('Loading data...') (X_train, y_train), (X_test, y_test) = imdb.load_data(nb_words=max_features) @@ -112,17 +109,18 @@ def test_text_classification(self): model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy']) - model = use_bigdl_backend(model) + model = with_bigdl_backend(model) model.fit(X_train, y_train, batch_size=batch_size, nb_epoch=nb_epoch, - validation_data=(X_test, y_test)) + validation_data=(X_test, y_test), + is_distributed=True) # 2017-09-22 15:53:45 INFO DistriOptimizer$:657 # - Top1Accuracy is Accuracy(correct: 21557, count: 25000, accuracy: 0.86228) # this result is from GlobalAveragePooling not GlobalMaxPooling. - model.predict(X_test) - model.evaluate(X_test, y_test) + model.predict(X_test, is_distributed=True) + model.evaluate(X_test, y_test, is_distributed=True) print(model) def test_resnet50(self): diff --git a/python/dllib/test/bigdl/keras/test_backend.py b/python/dllib/test/bigdl/keras/test_backend.py index 40e6ce9c5fb..d855981acb9 100644 --- a/python/dllib/test/bigdl/keras/test_backend.py +++ b/python/dllib/test/bigdl/keras/test_backend.py @@ -45,9 +45,7 @@ def test_lenet_local(self): metrics=['accuracy']) model = with_bigdl_backend(kmodel) - model.fit(X_train, y_train, - batch_size=4, - nb_epoch=2) + model.fit(X_train, y_train, batch_size=4, nb_epoch=2) model.predict(X_train) # TODO: support evaluate for local mode. # model.evaluate(X_train, y_train) @@ -61,9 +59,7 @@ def test_lenet_distributed_ndarray(self): metrics=['accuracy']) model = with_bigdl_backend(kmodel) - model.fit(X_train, y_train, - batch_size=4, - nb_epoch=2, + model.fit(X_train, y_train, batch_size=4, nb_epoch=2, validation_data=(X_train, y_train), is_distributed=True) model.predict(X_train, is_distributed=True).collect() model.evaluate(X_train, y_train, is_distributed=True) @@ -82,9 +78,7 @@ def test_lenet_distributed_rdd(self): metrics=['accuracy']) model = with_bigdl_backend(kmodel) - model.fit(training_rdd, - batch_size=4, - nb_epoch=2, + model.fit(training_rdd, batch_size=4, nb_epoch=2, validation_data=training_rdd, is_distributed=True) model.predict(X_train, is_distributed=True).collect() model.evaluate(X_train, y_train, is_distributed=True) diff --git a/python/dllib/test/bigdl/keras/test_loss.py b/python/dllib/test/bigdl/keras/test_loss.py new file mode 100644 index 00000000000..c06ad5ef9a0 --- /dev/null +++ b/python/dllib/test/bigdl/keras/test_loss.py @@ -0,0 +1,120 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from __future__ import print_function + +import numpy as np +import pytest +from keras import objectives + +from bigdl.keras.optimization import OptimConverter +from test.bigdl.test_utils import BigDLTestCase +np.random.seed(1337) # for reproducibility + + +class TestLoss(BigDLTestCase): + + def test_mse(self): + y_a = np.random.random([2, 3, 4]) + y_b = np.random.random([2, 3, 4]) + kloss = objectives.mean_squared_error + bloss = OptimConverter.to_bigdl_criterion(objectives.mean_squared_error) + self.compare_loss(y_a, y_b, kloss, bloss) + + def test_msle(self): + y_a = np.random.random([2, 3, 4]) + y_b = np.random.random([2, 3, 4]) + kloss = objectives.mean_squared_logarithmic_error + bloss = OptimConverter.to_bigdl_criterion("msle") + self.compare_loss(y_a, y_b, kloss, bloss) + + def test_mae(self): + y_a = np.random.random([5, 6, 7]) + y_b = np.random.random([5, 6, 7]) + kloss = objectives.mean_absolute_error + bloss = OptimConverter.to_bigdl_criterion("mean_absolute_error") + self.compare_loss(y_a, y_b, kloss, bloss) + + def test_mape(self): + y_a = np.random.random([5, 6, 7]) + y_b = np.random.random([5, 6, 7]) + kloss = objectives.mean_absolute_percentage_error + bloss = OptimConverter.to_bigdl_criterion("mape") + self.compare_loss(y_a, y_b, kloss, bloss) + + def test_binary_crossentropy(self): + y_a = np.random.random([5, 6, 7]) + y_b = np.random.random([5, 6, 7]) + kloss = objectives.binary_crossentropy + bloss = OptimConverter.to_bigdl_criterion("binary_crossentropy") + self.compare_loss(y_a, y_b, kloss, bloss) + + def test_categorical_crossentropy(self): + y_a = np.random.random([2, 3]) + y_b = np.array([[0, 1, 0], [0, 0, 1]]) + kloss = objectives.categorical_crossentropy + bloss = OptimConverter.to_bigdl_criterion("categorical_crossentropy") + self.compare_loss(y_a, y_b, kloss, bloss) + + def test_sparse_categorical_crossentropy(self): + import keras.backend as K + y_a = np.array([0.12, 0.22, 0.30, 0.17, 0.19]) + # index starts from 1 in BigDL but starts from 0 in Keras + y_b_bigdl = np.array([2]) + y_b_keras = np.array([1]) + kloss = objectives.sparse_categorical_crossentropy + bloss = OptimConverter.to_bigdl_criterion("sparse_categorical_crossentropy") + bigdl_output = bloss.forward(y_a, y_b_bigdl) + keras_output = np.mean(K.eval(kloss(K.variable(y_b_keras), K.variable(y_a)))) + np.testing.assert_allclose(bigdl_output, keras_output) + + def test_hinge(self): + y_a = np.random.random([2, 3, 4]) + y_b = np.random.random([2, 3, 4]) + kloss = objectives.hinge + bloss = OptimConverter.to_bigdl_criterion("hinge") + self.compare_loss(y_a, y_b, kloss, bloss) + + def test_squared_hinge(self): + y_a = np.random.random([2, 3, 4]) + y_b = np.random.random([2, 3, 4]) + kloss = objectives.squared_hinge + bloss = OptimConverter.to_bigdl_criterion("squared_hinge") + self.compare_loss(y_a, y_b, kloss, bloss) + + def test_poisson(self): + y_a = np.random.random([2, 3, 4]) + y_b = np.random.random([2, 3, 4]) + kloss = objectives.poisson + bloss = OptimConverter.to_bigdl_criterion("poisson") + self.compare_loss(y_a, y_b, kloss, bloss) + + def test_cosine_proximity(self): + y_a = np.random.random([2, 3, 4]) + y_b = np.random.random([2, 3, 4]) + kloss = objectives.cosine_proximity + bloss = OptimConverter.to_bigdl_criterion("cosine_proximity") + self.compare_loss(y_a, y_b, kloss, bloss) + + def test_kld(self): + y_a = np.random.random([4, 5]) + y_b = np.random.random([4, 5]) + kloss = objectives.kullback_leibler_divergence + bloss = OptimConverter.to_bigdl_criterion("kld") + self.compare_loss(y_a, y_b, kloss, bloss) + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/python/dllib/test/bigdl/test_utils.py b/python/dllib/test/bigdl/test_utils.py index ba360cba379..c38f9da49ee 100644 --- a/python/dllib/test/bigdl/test_utils.py +++ b/python/dllib/test/bigdl/test_utils.py @@ -312,3 +312,9 @@ def __set_keras_backend(self, backend): # Make theano backend compatible with Python3 if backend == "theano": from theano import ifelse + + def compare_loss(self, y_a, y_b, kloss, bloss, rtol=1e-6, atol=1e-6): + # y_a: input/y_pred; y_b: target/y_true + keras_output = np.mean(K.eval(kloss(K.variable(y_b), K.variable(y_a)))) + bigdl_output = bloss.forward(y_a, y_b) + np.testing.assert_allclose(bigdl_output, keras_output, rtol=rtol, atol=atol) From e32f977c3b2586f767f7d909e04a16b4db1b50da Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Mon, 29 Jan 2018 22:32:05 +0800 Subject: [PATCH 537/823] Add PGCriterion to compute the negative policy gradient given action distribution, sampled action and reward (#2197) * add pgcriterion * fix style * fix style and python test * add doc * mv to nn --- python/dllib/src/bigdl/dllib/nn/criterion.py | 49 ++++++++++++++++++++ python/dllib/src/bigdl/dllib/nn/layer.py | 25 ++++++++++ 2 files changed, 74 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/nn/criterion.py b/python/dllib/src/bigdl/dllib/nn/criterion.py index 8af46365b8d..47ecc77736a 100644 --- a/python/dllib/src/bigdl/dllib/nn/criterion.py +++ b/python/dllib/src/bigdl/dllib/nn/criterion.py @@ -902,6 +902,55 @@ def __init__(self, input_transformer, target_transformer) +class DotProductCriterion(Criterion): + ''' + Compute the dot product of input and target tensor. + Input and target are required to have the same size. + :param size_average: whether to average over each observations in the same batch + + >>> dp =DotProductCriterion(False) + creating: createDotProductCriterion + ''' + + def __init__(self, + size_average = False, + bigdl_type="float"): + super(DotProductCriterion, self).__init__(None, + bigdl_type, + size_average) + +class PGCriterion(Criterion): + ''' + The Criterion to compute the negative policy gradient given a + multinomial distribution and the sampled action and reward. + + The input to this criterion should be a 2-D tensor representing + a batch of multinomial distribution, the target should also be + a 2-D tensor with the same size of input, representing the sampled + action and reward/advantage with the index of non-zero element in the vector + represents the sampled action and the non-zero element itself represents + the reward. If the action is space is large, you should consider using + SparseTensor for target. + + The loss computed is simple the standard policy gradient, + + loss = - 1/n * sum(R_{n} dot_product log(P_{n})) + + where R_{n} is the reward vector, and P_{n} is the input distribution. + + :param sizeAverage whether to average over each observations in the same batch + + >>> pg = PGCriterion() + creating: createPGCriterion + ''' + + def __init__(self, + sizeAverage = False, + bigdl_type="float"): + super(PGCriterion, self).__init__(None, + bigdl_type, + sizeAverage) + def _test(): import doctest diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index d353534c92c..6b62044cfbb 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -2933,6 +2933,31 @@ def __init__(self, size_average, provide_output) +class NegativeEntropyPenalty(Layer): + ''' + Penalize the input multinomial distribution if it has low entropy. + The input to this layer should be a batch of vector each representing a + multinomial distribution. The input is typically the output of a softmax layer. + + For forward, the output is the same as input and a NegativeEntropy loss of + the latent state will be calculated each time. For backward, + gradInput = gradOutput + gradLoss + + This can be used in reinforcement learning to discourage the policy from + collapsing to a single action for a given state, which improves exploration. + See the A3C paper for more detail (https://arxiv.org/pdf/1602.01783.pdf). + + >>> ne = NegativeEntropyPenalty(0.01) + creating: createNegativeEntropyPenalty + + :param beta penalty coefficient + ''' + + def __init__(self, beta=0.01, bigdl_type="float"): + super(NegativeEntropyPenalty, self).__init__(None, + bigdl_type, + beta) + class LeakyReLU(Layer): From f5f8b9ae3e19e933dba994a8f46247ca9128c850 Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Mon, 29 Jan 2018 22:32:05 +0800 Subject: [PATCH 538/823] Add PGCriterion to compute the negative policy gradient given action distribution, sampled action and reward (#2197) * add pgcriterion * fix style * fix style and python test * add doc * mv to nn --- python/dllib/src/bigdl/dllib/nn/criterion.py | 49 ++++++++++++++++++++ python/dllib/src/bigdl/dllib/nn/layer.py | 25 ++++++++++ 2 files changed, 74 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/nn/criterion.py b/python/dllib/src/bigdl/dllib/nn/criterion.py index 8af46365b8d..47ecc77736a 100644 --- a/python/dllib/src/bigdl/dllib/nn/criterion.py +++ b/python/dllib/src/bigdl/dllib/nn/criterion.py @@ -902,6 +902,55 @@ def __init__(self, input_transformer, target_transformer) +class DotProductCriterion(Criterion): + ''' + Compute the dot product of input and target tensor. + Input and target are required to have the same size. + :param size_average: whether to average over each observations in the same batch + + >>> dp =DotProductCriterion(False) + creating: createDotProductCriterion + ''' + + def __init__(self, + size_average = False, + bigdl_type="float"): + super(DotProductCriterion, self).__init__(None, + bigdl_type, + size_average) + +class PGCriterion(Criterion): + ''' + The Criterion to compute the negative policy gradient given a + multinomial distribution and the sampled action and reward. + + The input to this criterion should be a 2-D tensor representing + a batch of multinomial distribution, the target should also be + a 2-D tensor with the same size of input, representing the sampled + action and reward/advantage with the index of non-zero element in the vector + represents the sampled action and the non-zero element itself represents + the reward. If the action is space is large, you should consider using + SparseTensor for target. + + The loss computed is simple the standard policy gradient, + + loss = - 1/n * sum(R_{n} dot_product log(P_{n})) + + where R_{n} is the reward vector, and P_{n} is the input distribution. + + :param sizeAverage whether to average over each observations in the same batch + + >>> pg = PGCriterion() + creating: createPGCriterion + ''' + + def __init__(self, + sizeAverage = False, + bigdl_type="float"): + super(PGCriterion, self).__init__(None, + bigdl_type, + sizeAverage) + def _test(): import doctest diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index d353534c92c..6b62044cfbb 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -2933,6 +2933,31 @@ def __init__(self, size_average, provide_output) +class NegativeEntropyPenalty(Layer): + ''' + Penalize the input multinomial distribution if it has low entropy. + The input to this layer should be a batch of vector each representing a + multinomial distribution. The input is typically the output of a softmax layer. + + For forward, the output is the same as input and a NegativeEntropy loss of + the latent state will be calculated each time. For backward, + gradInput = gradOutput + gradLoss + + This can be used in reinforcement learning to discourage the policy from + collapsing to a single action for a given state, which improves exploration. + See the A3C paper for more detail (https://arxiv.org/pdf/1602.01783.pdf). + + >>> ne = NegativeEntropyPenalty(0.01) + creating: createNegativeEntropyPenalty + + :param beta penalty coefficient + ''' + + def __init__(self, beta=0.01, bigdl_type="float"): + super(NegativeEntropyPenalty, self).__init__(None, + bigdl_type, + beta) + class LeakyReLU(Layer): From bddfc9090970dea513f0b79de569c521604a2233 Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Wed, 31 Jan 2018 09:04:11 +0800 Subject: [PATCH 539/823] Add python api for keras layer (#2242) * firtst try * update * update Dense * clean * style * style --- .../bigdl/dllib/bigdlkeras/layers/layer.py | 69 +++++++ .../src/bigdl/dllib/keras/ToBigDLHelper.py | 93 +++++++++ .../dllib/src/bigdl/dllib/keras/converter.py | 183 ++++++------------ .../src/bigdl/dllib/nn/keras/__init__.py | 0 python/dllib/src/bigdl/dllib/utils/common.py | 44 +++-- 5 files changed, 251 insertions(+), 138 deletions(-) create mode 100644 python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py create mode 100644 python/dllib/src/bigdl/dllib/keras/ToBigDLHelper.py create mode 100644 python/dllib/src/bigdl/dllib/nn/keras/__init__.py diff --git a/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py b/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py new file mode 100644 index 00000000000..b52756e99b6 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py @@ -0,0 +1,69 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import sys + +from bigdl.keras.ToBigDLHelper import to_bigdl_reg, to_bigdl_init +from bigdl.nn.layer import Layer +from bigdl.util.common import get_activation_by_name + + +if sys.version >= '3': + long = int + unicode = str + +class Dense(Layer): + """Just your regular densely-connected NN layer. + + # Example + + # Arguments + output_dim: int > 0. + init: name of initialization function for the weights of the layer + activation: name of activation function to use + W_regularizer: instance of regularizer (eg. L1 or L2 regularization), + applied to the main weights matrix. + b_regularizer: nstance of regularizer (eg. L1 or L2 regularization), + applied to bias + bias: whether to include a bias + (i.e. make the layer affine rather than linear). + input_shape: is required when using this layer as the first layer in a model. + + # Input shape + nD tensor with shape: `(nb_samples, ..., input_dim)`. + The most common situation would be + a 2D input with shape `(nb_samples, input_dim)`. + + # Output shape + nD tensor with shape: `(nb_samples, ..., output_dim)`. + For instance, for a 2D input with shape `(nb_samples, input_dim)`, + the output would have shape `(nb_samples, output_dim)`. + >>> dense = Dense(10, input_shape=(3, 4)) + creating: createXavier + creating: createDense + """ + def __init__(self, output_dim, init='glorot_uniform', + activation=None, + W_regularizer=None, b_regularizer=None, + bias=True, input_shape=None, bigdl_type="float"): + super(Dense, self).__init__(None, bigdl_type, + output_dim, + to_bigdl_init(init), + get_activation_by_name(activation) if activation else None, # noqa + to_bigdl_reg(W_regularizer), + to_bigdl_reg(b_regularizer), + bias, + list(input_shape) if input_shape else None) diff --git a/python/dllib/src/bigdl/dllib/keras/ToBigDLHelper.py b/python/dllib/src/bigdl/dllib/keras/ToBigDLHelper.py new file mode 100644 index 00000000000..01b81de7cd3 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/keras/ToBigDLHelper.py @@ -0,0 +1,93 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from math import ceil + +import bigdl.nn.initialization_method as BInit +from bigdl.optim.optimizer import L1L2Regularizer as BRegularizer + + +def to_bigdl_2d_ordering(order): + if order == "tf": + return "NHWC" + elif order == "th": + return "NCHW" + else: + raise Exception("Unsupported dim_ordering: %s" % order) + +def to_bigdl_3d_ordering(order): + if order == "tf": + return "channel_last" + elif order == "th": + return "channel_first" + else: + raise Exception("Unsupported dim_ordering: %s" % order) + +def to_bigdl_3d_padding(border_mode): + if border_mode == "valid": + return 0, 0, 0 + elif border_mode == "same": + return -1, -1, -1 + else: + raise Exception("Unsupported border mode: %s" % border_mode) + + +def __calculate_2d_same_padding(x, kx, dx, dilation_x): + return int(ceil((x * (dx - 1) + dilation_x * (kx - 1) - dx + 1) / 2)) + + +def to_bigdl_2d_padding(border_mode, *args): + if border_mode == "same": + if len(args) == 0: # if -1 for same padding is supported + return -1, -1 + # calculate padding by given parameters + elif len(args) == 4: # used by 1d layers constructed from 2d, just need one pad + h, kh, dh, dilation_h = args + pad_h = __calculate_2d_same_padding(h, kh, dh, dilation_h) + return pad_h, 0 + elif len(args) == 8: + h, kh, dh, dilation_h, w, kw, dw, dilation_w = args + pad_h = __calculate_2d_same_padding(h, kh, dh, dilation_h) + pad_w = __calculate_2d_same_padding(w, kw, dw, dilation_w) + return pad_h, pad_w + elif border_mode == "valid": + return 0, 0 + else: + raise Exception("Unsupported border mode: %s" % border_mode) + + +def to_bigdl_init(kinit_method): # kinit_method is a string + init = None + if kinit_method == "glorot_uniform": + init = BInit.Xavier() + elif kinit_method == "one": + init = BInit.Ones() + elif kinit_method == "zero": + init = BInit.Zeros() + elif kinit_method == "uniform": + init = BInit.RandomUniform(lower=-0.05, upper=0.05) + elif kinit_method == "normal": + init = BInit.RandomNormal(mean=0.0, stdv=0.05) + else: + raise Exception("Unsupported init type: %s" % kinit_method) + return init + + +def to_bigdl_reg(reg): # reg is a dict + if reg: + return BRegularizer(reg['l1'], reg['l2']) + else: + return None \ No newline at end of file diff --git a/python/dllib/src/bigdl/dllib/keras/converter.py b/python/dllib/src/bigdl/dllib/keras/converter.py index 272c6923240..df870f8ffca 100644 --- a/python/dllib/src/bigdl/dllib/keras/converter.py +++ b/python/dllib/src/bigdl/dllib/keras/converter.py @@ -28,6 +28,7 @@ import keras import warnings from math import ceil +from bigdl.keras.ToBigDLHelper import * def unsupport_exp(name): @@ -507,8 +508,8 @@ def create_dense(self): input_size=in_dim, output_size=out_dim, with_bias=self.config["bias"], - wRegularizer=self.to_bigdl_reg(self.config["W_regularizer"]), - bRegularizer=self.to_bigdl_reg(self.config["b_regularizer"]) + wRegularizer=to_bigdl_reg(self.config["W_regularizer"]), + bRegularizer=to_bigdl_reg(self.config["b_regularizer"]) ) if len(self.input_shape) <= 2: @@ -525,8 +526,8 @@ def create_timedistributeddense(self): input_size=int(self.input_shape[-1]), output_size=self.config["output_dim"], with_bias=self.config["bias"], - wRegularizer=self.to_bigdl_reg(self.config["W_regularizer"]), - bRegularizer=self.to_bigdl_reg(self.config["b_regularizer"]) + wRegularizer=to_bigdl_reg(self.config["W_regularizer"]), + bRegularizer=to_bigdl_reg(self.config["b_regularizer"]) )) return self.combo_parameter_layer(blayer, self.config) @@ -575,7 +576,7 @@ def create_embedding(self): padding_value=0.0, norm_type=2.0, should_scale_grad_by_freq=False, - wRegularizer=self.to_bigdl_reg(self.config["W_regularizer"]), + wRegularizer=to_bigdl_reg(self.config["W_regularizer"]), bigdl_type="float") bseq.add(BLayer.AddConstant(1.0, inplace=True)) # Add 1 as BigDL is one-based index bseq.add(blayer) @@ -904,9 +905,9 @@ def generate_simplernn_cell(self, klayer, kclayer, input_shape): # create a sim hidden_size=klayer.output_dim, activation=activation, isInputWithBias=False, - wRegularizer=self.to_bigdl_reg(config["W_regularizer"]), - uRegularizer=self.to_bigdl_reg(config["U_regularizer"]), - bRegularizer=self.to_bigdl_reg(config["b_regularizer"]), + wRegularizer=to_bigdl_reg(config["W_regularizer"]), + uRegularizer=to_bigdl_reg(config["U_regularizer"]), + bRegularizer=to_bigdl_reg(config["b_regularizer"]), bigdl_type="float") return rnn @@ -928,9 +929,9 @@ def generate_lstm_cell(self, klayer, kclayer, input_shape): # create a lstm cel p=0.0, activation=activation, inner_activation=inner_activation, - wRegularizer=self.to_bigdl_reg(config["W_regularizer"]), - uRegularizer=self.to_bigdl_reg(config["U_regularizer"]), - bRegularizer=self.to_bigdl_reg(config["b_regularizer"]), + wRegularizer=to_bigdl_reg(config["W_regularizer"]), + uRegularizer=to_bigdl_reg(config["U_regularizer"]), + bRegularizer=to_bigdl_reg(config["b_regularizer"]), bigdl_type="float") return lstm @@ -958,9 +959,9 @@ def generate_convlstm2d_cell(self, klayer, kclayer, input_shape): # create a co activation=activation, inner_activation=inner_activation, # NB: ConvLSTM doesn't serialize regularizers to json file - # wRegularizer=self.to_bigdl_reg(config["W_regularizer"]), - # uRegularizer=self.to_bigdl_reg(config["U_regularizer"]), - # bRegularizer=self.to_bigdl_reg(config["b_regularizer"]), + # wRegularizer=to_bigdl_reg(config["W_regularizer"]), + # uRegularizer=to_bigdl_reg(config["U_regularizer"]), + # bRegularizer=to_bigdl_reg(config["b_regularizer"]), cRegularizer=None, with_peephole=False, bigdl_type="float") @@ -995,9 +996,9 @@ def generate_gru_cell(self, klayer, kclayer, input_shape): # create a gru cell p=0.0, activation=activation, inner_activation=inner_activation, - wRegularizer=self.to_bigdl_reg(config["W_regularizer"]), - uRegularizer=self.to_bigdl_reg(config["U_regularizer"]), - bRegularizer=self.to_bigdl_reg(config["b_regularizer"]), + wRegularizer=to_bigdl_reg(config["W_regularizer"]), + uRegularizer=to_bigdl_reg(config["U_regularizer"]), + bRegularizer=to_bigdl_reg(config["b_regularizer"]), bigdl_type="float") return gru @@ -1026,7 +1027,7 @@ def create_batchnormalization(self): if self.config["beta_regularizer"]: raise Exception("We don't support beta_regularizer for now") - bigdl_order = self.to_bigdl_2d_ordering(keras.backend.image_dim_ordering()) + bigdl_order = to_bigdl_2d_ordering(keras.backend.image_dim_ordering()) n_input_channel = int(self.input_shape[self.klayer.axis]) # init gamma and beta @@ -1059,60 +1060,14 @@ def get_bdim_order(self, dim="2D"): # get bigdl dim_ordering from keras dim_ord warnings.warn("Cannot find dim_ordering from json definition. Using the default instead.") order = keras.backend.image_dim_ordering() if dim == "3D": - return self.to_bigdl_3d_ordering(order) - return self.to_bigdl_2d_ordering(order) - - def to_bigdl_2d_ordering(self, order): - if order == "tf": - return "NHWC" - elif order == "th": - return "NCHW" - else: - raise Exception("Unsupported dim_ordering: %s" % order) - - def to_bigdl_3d_ordering(self, order): - if order == "tf": - return "channel_last" - elif order == "th": - return "channel_first" - else: - raise Exception("Unsupported dim_ordering: %s" % order) - - def to_bigdl_3d_padding(self, border_mode): - if border_mode == "valid": - return 0, 0, 0 - elif border_mode == "same": - return -1, -1, -1 - else: - raise Exception("Unsupported border mode: %s" % border_mode) - - def __calculate_2d_same_padding(self, x, kx, dx, dilation_x): - return int(ceil((x * (dx - 1) + dilation_x * (kx - 1) - dx + 1) / 2)) - - def to_bigdl_2d_padding(self, border_mode, *args): - if border_mode == "same": - if len(args) == 0: # if -1 for same padding is supported - return -1, -1 - # calculate padding by given parameters - elif len(args) == 4: # used by 1d layers constructed from 2d, just need one pad - h, kh, dh, dilation_h = args - pad_h = self.__calculate_2d_same_padding(h, kh, dh, dilation_h) - return pad_h, 0 - elif len(args) == 8: - h, kh, dh, dilation_h, w, kw, dw, dilation_w = args - pad_h = self.__calculate_2d_same_padding(h, kh, dh, dilation_h) - pad_w = self.__calculate_2d_same_padding(w, kw, dw, dilation_w) - return pad_h, pad_w - elif border_mode == "valid": - return 0, 0 - else: - raise Exception("Unsupported border mode: %s" % border_mode) + return to_bigdl_3d_ordering(order) + return to_bigdl_2d_ordering(order) def create_convolution1d(self): # batch, steps, dim, batch is None here, so you cannot use it directly. stack_size = int(self.input_shape[2]) - bpadW, bpadH = self.to_bigdl_2d_padding(self.klayer.border_mode) + bpadW, bpadH = to_bigdl_2d_padding(self.klayer.border_mode) seq = BLayer.Sequential() seq.add(BLayer.Reshape([int(self.input_shape[1]), 1, int(self.input_shape[2])], True)) blayer = BLayer.SpatialConvolution( @@ -1126,8 +1081,8 @@ def create_convolution1d(self): pad_h=bpadH, n_group=1, propagate_back=True, - wRegularizer=self.to_bigdl_reg(self.config["W_regularizer"]), - bRegularizer=self.to_bigdl_reg(self.config["b_regularizer"]), + wRegularizer=to_bigdl_reg(self.config["W_regularizer"]), + bRegularizer=to_bigdl_reg(self.config["b_regularizer"]), init_weight=None, init_bias=None, init_grad_weight=None, @@ -1147,7 +1102,7 @@ def create_convolution2d(self): elif bigdl_order == "NHWC": stack_size = int(self.input_shape[3]) - bpadW, bpadH = self.to_bigdl_2d_padding(self.klayer.border_mode) + bpadW, bpadH = to_bigdl_2d_padding(self.klayer.border_mode) blayer = BLayer.SpatialConvolution( n_input_plane=stack_size, n_output_plane=self.klayer.nb_filter, @@ -1159,8 +1114,8 @@ def create_convolution2d(self): pad_h=bpadH, n_group=1, propagate_back=True, - wRegularizer=self.to_bigdl_reg(self.config["W_regularizer"]), - bRegularizer=self.to_bigdl_reg(self.config["b_regularizer"]), + wRegularizer=to_bigdl_reg(self.config["W_regularizer"]), + bRegularizer=to_bigdl_reg(self.config["b_regularizer"]), init_weight=None, init_bias=None, init_grad_weight=None, @@ -1175,7 +1130,7 @@ def create_convolution3d(self): if self.klayer.dim_ordering != "th": raise Exception("Please use `th` for `dim_ordering`. `%s` is not supported for now." % self.klayer.dim_ordering) - bpadT, bpadW, bpadH = self.to_bigdl_3d_padding(self.klayer.border_mode) + bpadT, bpadW, bpadH = to_bigdl_3d_padding(self.klayer.border_mode) blayer = BLayer.VolumetricConvolution( n_input_plane=int(self.input_shape[1]), n_output_plane=self.klayer.nb_filter, @@ -1189,8 +1144,8 @@ def create_convolution3d(self): pad_w=bpadW, pad_h=bpadH, with_bias=self.config["bias"], - wRegularizer=self.to_bigdl_reg(self.config["W_regularizer"]), - bRegularizer=self.to_bigdl_reg(self.config["b_regularizer"]), + wRegularizer=to_bigdl_reg(self.config["W_regularizer"]), + bRegularizer=to_bigdl_reg(self.config["b_regularizer"]), bigdl_type="float") return self.combo_parameter_layer(blayer, self.config) @@ -1203,7 +1158,7 @@ def create_atrousconvolution1d(self): kh = self.config["filter_length"] dh = self.config["subsample_length"] dilation_h = self.config["atrous_rate"] - pad_h, pad_w = self.to_bigdl_2d_padding(self.config["border_mode"], h, kh, dh, dilation_h) + pad_h, pad_w = to_bigdl_2d_padding(self.config["border_mode"], h, kh, dh, dilation_h) seq = BLayer.Sequential() seq.add(BLayer.Transpose([(2, 3)])) seq.add(BLayer.Reshape([int(self.input_shape[2]), int(self.input_shape[1]), 1], True)) @@ -1218,8 +1173,8 @@ def create_atrousconvolution1d(self): pad_h=pad_h, dilation_w=1, dilation_h=dilation_h, - wRegularizer=self.to_bigdl_reg(self.config["W_regularizer"]), - bRegularizer=self.to_bigdl_reg(self.config["b_regularizer"]), + wRegularizer=to_bigdl_reg(self.config["W_regularizer"]), + bRegularizer=to_bigdl_reg(self.config["b_regularizer"]), bigdl_type="float") seq.add(blayer) @@ -1241,7 +1196,7 @@ def create_atrousconvolution2d(self): dw = self.config["subsample"][1] dilation_h = self.config["atrous_rate"][0] dilation_w = self.config["atrous_rate"][1] - pad_h, pad_w = self.to_bigdl_2d_padding(self.config["border_mode"], h, kh, dh, dilation_h, + pad_h, pad_w = to_bigdl_2d_padding(self.config["border_mode"], h, kh, dh, dilation_h, w, kw, dw, dilation_w) blayer = BLayer.SpatialDilatedConvolution( n_input_plane=int(self.input_shape[1]), @@ -1254,8 +1209,8 @@ def create_atrousconvolution2d(self): pad_h=pad_h, dilation_w=dilation_w, dilation_h=dilation_h, - wRegularizer=self.to_bigdl_reg(self.config["W_regularizer"]), - bRegularizer=self.to_bigdl_reg(self.config["b_regularizer"]), + wRegularizer=to_bigdl_reg(self.config["W_regularizer"]), + bRegularizer=to_bigdl_reg(self.config["b_regularizer"]), bigdl_type="float") return self.combo_parameter_layer(blayer, self.config) @@ -1301,8 +1256,8 @@ def create_deconvolution2d(self): adj_h=0, n_group=1, no_bias=not self.klayer.bias, - wRegularizer=self.to_bigdl_reg(self.config["W_regularizer"]), - bRegularizer=self.to_bigdl_reg(self.config["b_regularizer"]), + wRegularizer=to_bigdl_reg(self.config["W_regularizer"]), + bRegularizer=to_bigdl_reg(self.config["b_regularizer"]), bigdl_type="float") return self.combo_parameter_layer(blayer, self.config) @@ -1314,7 +1269,7 @@ def create_maxpooling3d(self): if self.klayer.border_mode == 'same': raise Exception("Unsupported border_mode: same") - bpadT, bpadW, bpadH = self.to_bigdl_3d_padding(self.klayer.border_mode) + bpadT, bpadW, bpadH = to_bigdl_3d_padding(self.klayer.border_mode) blayer = BLayer.VolumetricMaxPooling( k_t=self.klayer.pool_size[0], k_w=self.klayer.pool_size[2], @@ -1330,7 +1285,7 @@ def create_maxpooling3d(self): def create_maxpooling2d(self): bigdl_order = self.get_bdim_order() - bpadW, bpadH = self.to_bigdl_2d_padding(self.klayer.border_mode) + bpadW, bpadH = to_bigdl_2d_padding(self.klayer.border_mode) blayer = BLayer.SpatialMaxPooling( kw=self.klayer.pool_size[1], kh=self.klayer.pool_size[0], @@ -1402,7 +1357,7 @@ def create_globalaveragepooling3d(self): def create_averagepooling2d(self): bigdl_order = self.get_bdim_order() - bpadW, bpadH = self.to_bigdl_2d_padding(self.klayer.border_mode) + bpadW, bpadH = to_bigdl_2d_padding(self.klayer.border_mode) blayer = BLayer.SpatialAveragePooling( kw=self.klayer.pool_size[1], kh=self.klayer.pool_size[0], @@ -1426,7 +1381,7 @@ def create_averagepooling3d(self): if self.klayer.border_mode == 'same': raise Exception("Unsupported border_mode: same") - bpadT, bpadW, bpadH = self.to_bigdl_3d_padding(self.klayer.border_mode) + bpadT, bpadW, bpadH = to_bigdl_3d_padding(self.klayer.border_mode) blayer = BLayer.VolumetricAveragePooling( k_t=self.klayer.pool_size[0], k_w=self.klayer.pool_size[2], @@ -1519,7 +1474,7 @@ def create_globalaveragepooling1d(self): return seq def create_maxpooling1d(self): - bpadW, bpadH = self.to_bigdl_2d_padding(self.klayer.border_mode) + bpadW, bpadH = to_bigdl_2d_padding(self.klayer.border_mode) seq = BLayer.Sequential() seq.add(BLayer.Reshape([int(self.input_shape[1]), 1, int(self.input_shape[2])], True)) @@ -1539,7 +1494,7 @@ def create_maxpooling1d(self): return seq def create_averagepooling1d(self): - bpadW, bpadH = self.to_bigdl_2d_padding(self.klayer.border_mode) + bpadW, bpadH = to_bigdl_2d_padding(self.klayer.border_mode) seq = BLayer.Sequential() seq.add(BLayer.Reshape([int(self.input_shape[1]), 1, int(self.input_shape[2])], True)) @@ -1625,8 +1580,8 @@ def create_highway(self): blayer = BLayer.Highway(size=int(self.input_shape[1]), with_bias=self.klayer.bias, activation=activation, - wRegularizer=self.to_bigdl_reg(self.config["W_regularizer"]), - bRegularizer=self.to_bigdl_reg(self.config["b_regularizer"])) + wRegularizer=to_bigdl_reg(self.config["W_regularizer"]), + bRegularizer=to_bigdl_reg(self.config["b_regularizer"])) return blayer def create_maxoutdense(self): @@ -1634,8 +1589,8 @@ def create_maxoutdense(self): output_size=self.klayer.output_dim, maxout_number=self.klayer.nb_feature, with_bias=self.klayer.bias, - w_regularizer=self.to_bigdl_reg(self.config["W_regularizer"]), - b_regularizer=self.to_bigdl_reg(self.config["b_regularizer"])) + w_regularizer=to_bigdl_reg(self.config["W_regularizer"]), + b_regularizer=to_bigdl_reg(self.config["b_regularizer"])) return blayer def create_masking(self): @@ -1660,7 +1615,7 @@ def create_separableconvolution2d(self): elif bigdl_order == "NHWC": stack_size = int(self.input_shape[3]) - bpadW, bpadH = self.to_bigdl_2d_padding(self.klayer.border_mode) + bpadW, bpadH = to_bigdl_2d_padding(self.klayer.border_mode) blayer = BLayer.SpatialSeperableConvolution( n_input_channel=stack_size, n_output_channel=self.klayer.nb_filter, @@ -1673,9 +1628,9 @@ def create_separableconvolution2d(self): pad_h=bpadH, with_bias=self.klayer.bias, data_format=bigdl_order, - w_regularizer=self.to_bigdl_reg(self.config["depthwise_regularizer"]), - b_regularizer=self.to_bigdl_reg(self.config["b_regularizer"]), - p_regularizer=self.to_bigdl_reg(self.config["pointwise_regularizer"]) + w_regularizer=to_bigdl_reg(self.config["depthwise_regularizer"]), + b_regularizer=to_bigdl_reg(self.config["b_regularizer"]), + p_regularizer=to_bigdl_reg(self.config["pointwise_regularizer"]) ) return self.combo_parameter_layer(blayer, self.config) @@ -1711,8 +1666,8 @@ def create_locallyconnected1d(self): stride_h=self.klayer.subsample_length, pad_w=0, pad_h=0, - wRegularizer=self.to_bigdl_reg(self.config["W_regularizer"]), - bRegularizer=self.to_bigdl_reg(self.config["b_regularizer"]), + wRegularizer=to_bigdl_reg(self.config["W_regularizer"]), + bRegularizer=to_bigdl_reg(self.config["b_regularizer"]), with_bias=self.klayer.bias, data_format="NHWC") seq.add(blayer) @@ -1736,7 +1691,7 @@ def create_locallyconnected2d(self): input_width = int(self.input_shape[2]) input_height = int(self.input_shape[1]) - bpadW, bpadH = self.to_bigdl_2d_padding(self.klayer.border_mode) + bpadW, bpadH = to_bigdl_2d_padding(self.klayer.border_mode) blayer = BLayer.LocallyConnected2D(n_input_plane=stack_size, input_width=input_width, input_height=input_height, @@ -1747,8 +1702,8 @@ def create_locallyconnected2d(self): stride_h=self.klayer.subsample[0], pad_w=bpadW, pad_h=bpadH, - wRegularizer=self.to_bigdl_reg(self.config["W_regularizer"]), - bRegularizer=self.to_bigdl_reg(self.config["b_regularizer"]), + wRegularizer=to_bigdl_reg(self.config["W_regularizer"]), + bRegularizer=to_bigdl_reg(self.config["b_regularizer"]), with_bias=self.klayer.bias, data_format=bigdl_order) @@ -1763,7 +1718,7 @@ def combo_parameter_layer(self, blayer, config): blayer.set_name(config["name"]) if hasattr(blayer, "set_init_method"): try: - blayer.set_init_method(self.to_bigdl_init(config["init"]), + blayer.set_init_method(to_bigdl_init(config["init"]), BInit.Zeros()) # Keras always set this to be zeros except Exception: warning_msg = "We don't support initialization " + config["init"] + " for now. " \ @@ -1785,28 +1740,6 @@ def get_value_from_init(self, kinit_method, shape): else: raise Exception("We don't support % for now", kinit_method) - def to_bigdl_init(self, kinit_method): # kinit_method is a string - init = None - if kinit_method == "glorot_uniform": - init = BInit.Xavier() - elif kinit_method == "one": - init = BInit.Ones() - elif kinit_method == "zero": - init = BInit.Zeros() - elif kinit_method == "uniform": - init = BInit.RandomUniform(lower=-0.05, upper=0.05) - elif kinit_method == "normal": - init = BInit.RandomNormal(mean=0.0, stdv=0.05) - else: - raise Exception("Unsupported init type: %s" % kinit_method) - return init - - def to_bigdl_reg(self, reg): # reg is a dict - if reg: - return BRegularizer(reg['l1'], reg['l2']) - else: - return None - def fuse(self, src_blayer, activation): # activation is a layer seq = BLayer.Sequential() seq.add(src_blayer) diff --git a/python/dllib/src/bigdl/dllib/nn/keras/__init__.py b/python/dllib/src/bigdl/dllib/nn/keras/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/python/dllib/src/bigdl/dllib/utils/common.py b/python/dllib/src/bigdl/dllib/utils/common.py index 25248e630af..5e3756fd6ec 100644 --- a/python/dllib/src/bigdl/dllib/utils/common.py +++ b/python/dllib/src/bigdl/dllib/utils/common.py @@ -55,7 +55,14 @@ def instance(cls, class JavaCreator(SingletonMixin): - __creator_class="com.intel.analytics.bigdl.python.api.PythonBigDL" + __creator_class=["com.intel.analytics.bigdl.python.api.PythonBigDL", + "com.intel.analytics.bigdl.python.api.PythonBigDLKeras"] + + @classmethod + def add_creator_class(cls, jinvoker): + with JavaCreator._lock: + JavaCreator.__creator_class.append(jinvoker) + JavaCreator._instance = None @classmethod def get_creator_class(cls): @@ -70,13 +77,15 @@ def set_creator_class(cls, cclass): def __init__(self, bigdl_type): sc = get_spark_context() - jclass = getattr(sc._jvm, JavaCreator.get_creator_class()) - if bigdl_type == "float": - self.value = getattr(jclass, "ofFloat")() - elif bigdl_type == "double": - self.value = getattr(jclass, "ofDouble")() - else: - raise Exception("Not supported bigdl_type: %s" % bigdl_type) + self.value = [] + for creator_class in JavaCreator.get_creator_class(): + jclass = getattr(sc._jvm, creator_class) + if bigdl_type == "float": + self.value.append(getattr(jclass, "ofFloat")()) + elif bigdl_type == "double": + self.value.append(getattr(jclass, "ofDouble")()) + else: + raise Exception("Not supported bigdl_type: %s" % bigdl_type) class JavaValue(object): @@ -398,7 +407,6 @@ def __str__(self): def __repr__(self): return "Sample: features: %s, labels: %s" % (self.features, self.labels) - class RNG(): """ generate tensor data with seed @@ -549,13 +557,23 @@ def get_spark_sql_context(sc): else: return SQLContext(sc) # Compatible with Spark1.5.1 - def callBigDlFunc(bigdl_type, name, *args): """ Call API in PythonBigDL """ - jinstance = JavaCreator.instance(bigdl_type=bigdl_type).value sc = get_spark_context() - api = getattr(jinstance, name) - return callJavaFunc(sc, api, *args) + error = Exception("Cannot find function: %s" % name) + for jinvoker in JavaCreator.instance(bigdl_type=bigdl_type).value: + # hasattr(jinvoker, name) always return true here, + # so you need to invoke the method to check if it exist or not + try: + api = getattr(jinvoker, name) + result = callJavaFunc(sc, api, *args) + except Exception as e: + error = e + if "does not exist" not in e.message: + raise e + else: + return result + raise error def _java2py(sc, r, encoding="bytes"): From b33c7cd2b1c2e31332323599defa3741b55e30d0 Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Wed, 31 Jan 2018 09:04:11 +0800 Subject: [PATCH 540/823] Add python api for keras layer (#2242) * firtst try * update * update Dense * clean * style * style --- .../bigdl/dllib/bigdlkeras/layers/layer.py | 69 +++++++ .../src/bigdl/dllib/keras/ToBigDLHelper.py | 93 +++++++++ .../dllib/src/bigdl/dllib/keras/converter.py | 183 ++++++------------ .../src/bigdl/dllib/nn/keras/__init__.py | 0 python/dllib/src/bigdl/utils/common.py | 44 +++-- 5 files changed, 251 insertions(+), 138 deletions(-) create mode 100644 python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py create mode 100644 python/dllib/src/bigdl/dllib/keras/ToBigDLHelper.py create mode 100644 python/dllib/src/bigdl/dllib/nn/keras/__init__.py diff --git a/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py b/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py new file mode 100644 index 00000000000..b52756e99b6 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py @@ -0,0 +1,69 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import sys + +from bigdl.keras.ToBigDLHelper import to_bigdl_reg, to_bigdl_init +from bigdl.nn.layer import Layer +from bigdl.util.common import get_activation_by_name + + +if sys.version >= '3': + long = int + unicode = str + +class Dense(Layer): + """Just your regular densely-connected NN layer. + + # Example + + # Arguments + output_dim: int > 0. + init: name of initialization function for the weights of the layer + activation: name of activation function to use + W_regularizer: instance of regularizer (eg. L1 or L2 regularization), + applied to the main weights matrix. + b_regularizer: nstance of regularizer (eg. L1 or L2 regularization), + applied to bias + bias: whether to include a bias + (i.e. make the layer affine rather than linear). + input_shape: is required when using this layer as the first layer in a model. + + # Input shape + nD tensor with shape: `(nb_samples, ..., input_dim)`. + The most common situation would be + a 2D input with shape `(nb_samples, input_dim)`. + + # Output shape + nD tensor with shape: `(nb_samples, ..., output_dim)`. + For instance, for a 2D input with shape `(nb_samples, input_dim)`, + the output would have shape `(nb_samples, output_dim)`. + >>> dense = Dense(10, input_shape=(3, 4)) + creating: createXavier + creating: createDense + """ + def __init__(self, output_dim, init='glorot_uniform', + activation=None, + W_regularizer=None, b_regularizer=None, + bias=True, input_shape=None, bigdl_type="float"): + super(Dense, self).__init__(None, bigdl_type, + output_dim, + to_bigdl_init(init), + get_activation_by_name(activation) if activation else None, # noqa + to_bigdl_reg(W_regularizer), + to_bigdl_reg(b_regularizer), + bias, + list(input_shape) if input_shape else None) diff --git a/python/dllib/src/bigdl/dllib/keras/ToBigDLHelper.py b/python/dllib/src/bigdl/dllib/keras/ToBigDLHelper.py new file mode 100644 index 00000000000..01b81de7cd3 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/keras/ToBigDLHelper.py @@ -0,0 +1,93 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from math import ceil + +import bigdl.nn.initialization_method as BInit +from bigdl.optim.optimizer import L1L2Regularizer as BRegularizer + + +def to_bigdl_2d_ordering(order): + if order == "tf": + return "NHWC" + elif order == "th": + return "NCHW" + else: + raise Exception("Unsupported dim_ordering: %s" % order) + +def to_bigdl_3d_ordering(order): + if order == "tf": + return "channel_last" + elif order == "th": + return "channel_first" + else: + raise Exception("Unsupported dim_ordering: %s" % order) + +def to_bigdl_3d_padding(border_mode): + if border_mode == "valid": + return 0, 0, 0 + elif border_mode == "same": + return -1, -1, -1 + else: + raise Exception("Unsupported border mode: %s" % border_mode) + + +def __calculate_2d_same_padding(x, kx, dx, dilation_x): + return int(ceil((x * (dx - 1) + dilation_x * (kx - 1) - dx + 1) / 2)) + + +def to_bigdl_2d_padding(border_mode, *args): + if border_mode == "same": + if len(args) == 0: # if -1 for same padding is supported + return -1, -1 + # calculate padding by given parameters + elif len(args) == 4: # used by 1d layers constructed from 2d, just need one pad + h, kh, dh, dilation_h = args + pad_h = __calculate_2d_same_padding(h, kh, dh, dilation_h) + return pad_h, 0 + elif len(args) == 8: + h, kh, dh, dilation_h, w, kw, dw, dilation_w = args + pad_h = __calculate_2d_same_padding(h, kh, dh, dilation_h) + pad_w = __calculate_2d_same_padding(w, kw, dw, dilation_w) + return pad_h, pad_w + elif border_mode == "valid": + return 0, 0 + else: + raise Exception("Unsupported border mode: %s" % border_mode) + + +def to_bigdl_init(kinit_method): # kinit_method is a string + init = None + if kinit_method == "glorot_uniform": + init = BInit.Xavier() + elif kinit_method == "one": + init = BInit.Ones() + elif kinit_method == "zero": + init = BInit.Zeros() + elif kinit_method == "uniform": + init = BInit.RandomUniform(lower=-0.05, upper=0.05) + elif kinit_method == "normal": + init = BInit.RandomNormal(mean=0.0, stdv=0.05) + else: + raise Exception("Unsupported init type: %s" % kinit_method) + return init + + +def to_bigdl_reg(reg): # reg is a dict + if reg: + return BRegularizer(reg['l1'], reg['l2']) + else: + return None \ No newline at end of file diff --git a/python/dllib/src/bigdl/dllib/keras/converter.py b/python/dllib/src/bigdl/dllib/keras/converter.py index 272c6923240..df870f8ffca 100644 --- a/python/dllib/src/bigdl/dllib/keras/converter.py +++ b/python/dllib/src/bigdl/dllib/keras/converter.py @@ -28,6 +28,7 @@ import keras import warnings from math import ceil +from bigdl.keras.ToBigDLHelper import * def unsupport_exp(name): @@ -507,8 +508,8 @@ def create_dense(self): input_size=in_dim, output_size=out_dim, with_bias=self.config["bias"], - wRegularizer=self.to_bigdl_reg(self.config["W_regularizer"]), - bRegularizer=self.to_bigdl_reg(self.config["b_regularizer"]) + wRegularizer=to_bigdl_reg(self.config["W_regularizer"]), + bRegularizer=to_bigdl_reg(self.config["b_regularizer"]) ) if len(self.input_shape) <= 2: @@ -525,8 +526,8 @@ def create_timedistributeddense(self): input_size=int(self.input_shape[-1]), output_size=self.config["output_dim"], with_bias=self.config["bias"], - wRegularizer=self.to_bigdl_reg(self.config["W_regularizer"]), - bRegularizer=self.to_bigdl_reg(self.config["b_regularizer"]) + wRegularizer=to_bigdl_reg(self.config["W_regularizer"]), + bRegularizer=to_bigdl_reg(self.config["b_regularizer"]) )) return self.combo_parameter_layer(blayer, self.config) @@ -575,7 +576,7 @@ def create_embedding(self): padding_value=0.0, norm_type=2.0, should_scale_grad_by_freq=False, - wRegularizer=self.to_bigdl_reg(self.config["W_regularizer"]), + wRegularizer=to_bigdl_reg(self.config["W_regularizer"]), bigdl_type="float") bseq.add(BLayer.AddConstant(1.0, inplace=True)) # Add 1 as BigDL is one-based index bseq.add(blayer) @@ -904,9 +905,9 @@ def generate_simplernn_cell(self, klayer, kclayer, input_shape): # create a sim hidden_size=klayer.output_dim, activation=activation, isInputWithBias=False, - wRegularizer=self.to_bigdl_reg(config["W_regularizer"]), - uRegularizer=self.to_bigdl_reg(config["U_regularizer"]), - bRegularizer=self.to_bigdl_reg(config["b_regularizer"]), + wRegularizer=to_bigdl_reg(config["W_regularizer"]), + uRegularizer=to_bigdl_reg(config["U_regularizer"]), + bRegularizer=to_bigdl_reg(config["b_regularizer"]), bigdl_type="float") return rnn @@ -928,9 +929,9 @@ def generate_lstm_cell(self, klayer, kclayer, input_shape): # create a lstm cel p=0.0, activation=activation, inner_activation=inner_activation, - wRegularizer=self.to_bigdl_reg(config["W_regularizer"]), - uRegularizer=self.to_bigdl_reg(config["U_regularizer"]), - bRegularizer=self.to_bigdl_reg(config["b_regularizer"]), + wRegularizer=to_bigdl_reg(config["W_regularizer"]), + uRegularizer=to_bigdl_reg(config["U_regularizer"]), + bRegularizer=to_bigdl_reg(config["b_regularizer"]), bigdl_type="float") return lstm @@ -958,9 +959,9 @@ def generate_convlstm2d_cell(self, klayer, kclayer, input_shape): # create a co activation=activation, inner_activation=inner_activation, # NB: ConvLSTM doesn't serialize regularizers to json file - # wRegularizer=self.to_bigdl_reg(config["W_regularizer"]), - # uRegularizer=self.to_bigdl_reg(config["U_regularizer"]), - # bRegularizer=self.to_bigdl_reg(config["b_regularizer"]), + # wRegularizer=to_bigdl_reg(config["W_regularizer"]), + # uRegularizer=to_bigdl_reg(config["U_regularizer"]), + # bRegularizer=to_bigdl_reg(config["b_regularizer"]), cRegularizer=None, with_peephole=False, bigdl_type="float") @@ -995,9 +996,9 @@ def generate_gru_cell(self, klayer, kclayer, input_shape): # create a gru cell p=0.0, activation=activation, inner_activation=inner_activation, - wRegularizer=self.to_bigdl_reg(config["W_regularizer"]), - uRegularizer=self.to_bigdl_reg(config["U_regularizer"]), - bRegularizer=self.to_bigdl_reg(config["b_regularizer"]), + wRegularizer=to_bigdl_reg(config["W_regularizer"]), + uRegularizer=to_bigdl_reg(config["U_regularizer"]), + bRegularizer=to_bigdl_reg(config["b_regularizer"]), bigdl_type="float") return gru @@ -1026,7 +1027,7 @@ def create_batchnormalization(self): if self.config["beta_regularizer"]: raise Exception("We don't support beta_regularizer for now") - bigdl_order = self.to_bigdl_2d_ordering(keras.backend.image_dim_ordering()) + bigdl_order = to_bigdl_2d_ordering(keras.backend.image_dim_ordering()) n_input_channel = int(self.input_shape[self.klayer.axis]) # init gamma and beta @@ -1059,60 +1060,14 @@ def get_bdim_order(self, dim="2D"): # get bigdl dim_ordering from keras dim_ord warnings.warn("Cannot find dim_ordering from json definition. Using the default instead.") order = keras.backend.image_dim_ordering() if dim == "3D": - return self.to_bigdl_3d_ordering(order) - return self.to_bigdl_2d_ordering(order) - - def to_bigdl_2d_ordering(self, order): - if order == "tf": - return "NHWC" - elif order == "th": - return "NCHW" - else: - raise Exception("Unsupported dim_ordering: %s" % order) - - def to_bigdl_3d_ordering(self, order): - if order == "tf": - return "channel_last" - elif order == "th": - return "channel_first" - else: - raise Exception("Unsupported dim_ordering: %s" % order) - - def to_bigdl_3d_padding(self, border_mode): - if border_mode == "valid": - return 0, 0, 0 - elif border_mode == "same": - return -1, -1, -1 - else: - raise Exception("Unsupported border mode: %s" % border_mode) - - def __calculate_2d_same_padding(self, x, kx, dx, dilation_x): - return int(ceil((x * (dx - 1) + dilation_x * (kx - 1) - dx + 1) / 2)) - - def to_bigdl_2d_padding(self, border_mode, *args): - if border_mode == "same": - if len(args) == 0: # if -1 for same padding is supported - return -1, -1 - # calculate padding by given parameters - elif len(args) == 4: # used by 1d layers constructed from 2d, just need one pad - h, kh, dh, dilation_h = args - pad_h = self.__calculate_2d_same_padding(h, kh, dh, dilation_h) - return pad_h, 0 - elif len(args) == 8: - h, kh, dh, dilation_h, w, kw, dw, dilation_w = args - pad_h = self.__calculate_2d_same_padding(h, kh, dh, dilation_h) - pad_w = self.__calculate_2d_same_padding(w, kw, dw, dilation_w) - return pad_h, pad_w - elif border_mode == "valid": - return 0, 0 - else: - raise Exception("Unsupported border mode: %s" % border_mode) + return to_bigdl_3d_ordering(order) + return to_bigdl_2d_ordering(order) def create_convolution1d(self): # batch, steps, dim, batch is None here, so you cannot use it directly. stack_size = int(self.input_shape[2]) - bpadW, bpadH = self.to_bigdl_2d_padding(self.klayer.border_mode) + bpadW, bpadH = to_bigdl_2d_padding(self.klayer.border_mode) seq = BLayer.Sequential() seq.add(BLayer.Reshape([int(self.input_shape[1]), 1, int(self.input_shape[2])], True)) blayer = BLayer.SpatialConvolution( @@ -1126,8 +1081,8 @@ def create_convolution1d(self): pad_h=bpadH, n_group=1, propagate_back=True, - wRegularizer=self.to_bigdl_reg(self.config["W_regularizer"]), - bRegularizer=self.to_bigdl_reg(self.config["b_regularizer"]), + wRegularizer=to_bigdl_reg(self.config["W_regularizer"]), + bRegularizer=to_bigdl_reg(self.config["b_regularizer"]), init_weight=None, init_bias=None, init_grad_weight=None, @@ -1147,7 +1102,7 @@ def create_convolution2d(self): elif bigdl_order == "NHWC": stack_size = int(self.input_shape[3]) - bpadW, bpadH = self.to_bigdl_2d_padding(self.klayer.border_mode) + bpadW, bpadH = to_bigdl_2d_padding(self.klayer.border_mode) blayer = BLayer.SpatialConvolution( n_input_plane=stack_size, n_output_plane=self.klayer.nb_filter, @@ -1159,8 +1114,8 @@ def create_convolution2d(self): pad_h=bpadH, n_group=1, propagate_back=True, - wRegularizer=self.to_bigdl_reg(self.config["W_regularizer"]), - bRegularizer=self.to_bigdl_reg(self.config["b_regularizer"]), + wRegularizer=to_bigdl_reg(self.config["W_regularizer"]), + bRegularizer=to_bigdl_reg(self.config["b_regularizer"]), init_weight=None, init_bias=None, init_grad_weight=None, @@ -1175,7 +1130,7 @@ def create_convolution3d(self): if self.klayer.dim_ordering != "th": raise Exception("Please use `th` for `dim_ordering`. `%s` is not supported for now." % self.klayer.dim_ordering) - bpadT, bpadW, bpadH = self.to_bigdl_3d_padding(self.klayer.border_mode) + bpadT, bpadW, bpadH = to_bigdl_3d_padding(self.klayer.border_mode) blayer = BLayer.VolumetricConvolution( n_input_plane=int(self.input_shape[1]), n_output_plane=self.klayer.nb_filter, @@ -1189,8 +1144,8 @@ def create_convolution3d(self): pad_w=bpadW, pad_h=bpadH, with_bias=self.config["bias"], - wRegularizer=self.to_bigdl_reg(self.config["W_regularizer"]), - bRegularizer=self.to_bigdl_reg(self.config["b_regularizer"]), + wRegularizer=to_bigdl_reg(self.config["W_regularizer"]), + bRegularizer=to_bigdl_reg(self.config["b_regularizer"]), bigdl_type="float") return self.combo_parameter_layer(blayer, self.config) @@ -1203,7 +1158,7 @@ def create_atrousconvolution1d(self): kh = self.config["filter_length"] dh = self.config["subsample_length"] dilation_h = self.config["atrous_rate"] - pad_h, pad_w = self.to_bigdl_2d_padding(self.config["border_mode"], h, kh, dh, dilation_h) + pad_h, pad_w = to_bigdl_2d_padding(self.config["border_mode"], h, kh, dh, dilation_h) seq = BLayer.Sequential() seq.add(BLayer.Transpose([(2, 3)])) seq.add(BLayer.Reshape([int(self.input_shape[2]), int(self.input_shape[1]), 1], True)) @@ -1218,8 +1173,8 @@ def create_atrousconvolution1d(self): pad_h=pad_h, dilation_w=1, dilation_h=dilation_h, - wRegularizer=self.to_bigdl_reg(self.config["W_regularizer"]), - bRegularizer=self.to_bigdl_reg(self.config["b_regularizer"]), + wRegularizer=to_bigdl_reg(self.config["W_regularizer"]), + bRegularizer=to_bigdl_reg(self.config["b_regularizer"]), bigdl_type="float") seq.add(blayer) @@ -1241,7 +1196,7 @@ def create_atrousconvolution2d(self): dw = self.config["subsample"][1] dilation_h = self.config["atrous_rate"][0] dilation_w = self.config["atrous_rate"][1] - pad_h, pad_w = self.to_bigdl_2d_padding(self.config["border_mode"], h, kh, dh, dilation_h, + pad_h, pad_w = to_bigdl_2d_padding(self.config["border_mode"], h, kh, dh, dilation_h, w, kw, dw, dilation_w) blayer = BLayer.SpatialDilatedConvolution( n_input_plane=int(self.input_shape[1]), @@ -1254,8 +1209,8 @@ def create_atrousconvolution2d(self): pad_h=pad_h, dilation_w=dilation_w, dilation_h=dilation_h, - wRegularizer=self.to_bigdl_reg(self.config["W_regularizer"]), - bRegularizer=self.to_bigdl_reg(self.config["b_regularizer"]), + wRegularizer=to_bigdl_reg(self.config["W_regularizer"]), + bRegularizer=to_bigdl_reg(self.config["b_regularizer"]), bigdl_type="float") return self.combo_parameter_layer(blayer, self.config) @@ -1301,8 +1256,8 @@ def create_deconvolution2d(self): adj_h=0, n_group=1, no_bias=not self.klayer.bias, - wRegularizer=self.to_bigdl_reg(self.config["W_regularizer"]), - bRegularizer=self.to_bigdl_reg(self.config["b_regularizer"]), + wRegularizer=to_bigdl_reg(self.config["W_regularizer"]), + bRegularizer=to_bigdl_reg(self.config["b_regularizer"]), bigdl_type="float") return self.combo_parameter_layer(blayer, self.config) @@ -1314,7 +1269,7 @@ def create_maxpooling3d(self): if self.klayer.border_mode == 'same': raise Exception("Unsupported border_mode: same") - bpadT, bpadW, bpadH = self.to_bigdl_3d_padding(self.klayer.border_mode) + bpadT, bpadW, bpadH = to_bigdl_3d_padding(self.klayer.border_mode) blayer = BLayer.VolumetricMaxPooling( k_t=self.klayer.pool_size[0], k_w=self.klayer.pool_size[2], @@ -1330,7 +1285,7 @@ def create_maxpooling3d(self): def create_maxpooling2d(self): bigdl_order = self.get_bdim_order() - bpadW, bpadH = self.to_bigdl_2d_padding(self.klayer.border_mode) + bpadW, bpadH = to_bigdl_2d_padding(self.klayer.border_mode) blayer = BLayer.SpatialMaxPooling( kw=self.klayer.pool_size[1], kh=self.klayer.pool_size[0], @@ -1402,7 +1357,7 @@ def create_globalaveragepooling3d(self): def create_averagepooling2d(self): bigdl_order = self.get_bdim_order() - bpadW, bpadH = self.to_bigdl_2d_padding(self.klayer.border_mode) + bpadW, bpadH = to_bigdl_2d_padding(self.klayer.border_mode) blayer = BLayer.SpatialAveragePooling( kw=self.klayer.pool_size[1], kh=self.klayer.pool_size[0], @@ -1426,7 +1381,7 @@ def create_averagepooling3d(self): if self.klayer.border_mode == 'same': raise Exception("Unsupported border_mode: same") - bpadT, bpadW, bpadH = self.to_bigdl_3d_padding(self.klayer.border_mode) + bpadT, bpadW, bpadH = to_bigdl_3d_padding(self.klayer.border_mode) blayer = BLayer.VolumetricAveragePooling( k_t=self.klayer.pool_size[0], k_w=self.klayer.pool_size[2], @@ -1519,7 +1474,7 @@ def create_globalaveragepooling1d(self): return seq def create_maxpooling1d(self): - bpadW, bpadH = self.to_bigdl_2d_padding(self.klayer.border_mode) + bpadW, bpadH = to_bigdl_2d_padding(self.klayer.border_mode) seq = BLayer.Sequential() seq.add(BLayer.Reshape([int(self.input_shape[1]), 1, int(self.input_shape[2])], True)) @@ -1539,7 +1494,7 @@ def create_maxpooling1d(self): return seq def create_averagepooling1d(self): - bpadW, bpadH = self.to_bigdl_2d_padding(self.klayer.border_mode) + bpadW, bpadH = to_bigdl_2d_padding(self.klayer.border_mode) seq = BLayer.Sequential() seq.add(BLayer.Reshape([int(self.input_shape[1]), 1, int(self.input_shape[2])], True)) @@ -1625,8 +1580,8 @@ def create_highway(self): blayer = BLayer.Highway(size=int(self.input_shape[1]), with_bias=self.klayer.bias, activation=activation, - wRegularizer=self.to_bigdl_reg(self.config["W_regularizer"]), - bRegularizer=self.to_bigdl_reg(self.config["b_regularizer"])) + wRegularizer=to_bigdl_reg(self.config["W_regularizer"]), + bRegularizer=to_bigdl_reg(self.config["b_regularizer"])) return blayer def create_maxoutdense(self): @@ -1634,8 +1589,8 @@ def create_maxoutdense(self): output_size=self.klayer.output_dim, maxout_number=self.klayer.nb_feature, with_bias=self.klayer.bias, - w_regularizer=self.to_bigdl_reg(self.config["W_regularizer"]), - b_regularizer=self.to_bigdl_reg(self.config["b_regularizer"])) + w_regularizer=to_bigdl_reg(self.config["W_regularizer"]), + b_regularizer=to_bigdl_reg(self.config["b_regularizer"])) return blayer def create_masking(self): @@ -1660,7 +1615,7 @@ def create_separableconvolution2d(self): elif bigdl_order == "NHWC": stack_size = int(self.input_shape[3]) - bpadW, bpadH = self.to_bigdl_2d_padding(self.klayer.border_mode) + bpadW, bpadH = to_bigdl_2d_padding(self.klayer.border_mode) blayer = BLayer.SpatialSeperableConvolution( n_input_channel=stack_size, n_output_channel=self.klayer.nb_filter, @@ -1673,9 +1628,9 @@ def create_separableconvolution2d(self): pad_h=bpadH, with_bias=self.klayer.bias, data_format=bigdl_order, - w_regularizer=self.to_bigdl_reg(self.config["depthwise_regularizer"]), - b_regularizer=self.to_bigdl_reg(self.config["b_regularizer"]), - p_regularizer=self.to_bigdl_reg(self.config["pointwise_regularizer"]) + w_regularizer=to_bigdl_reg(self.config["depthwise_regularizer"]), + b_regularizer=to_bigdl_reg(self.config["b_regularizer"]), + p_regularizer=to_bigdl_reg(self.config["pointwise_regularizer"]) ) return self.combo_parameter_layer(blayer, self.config) @@ -1711,8 +1666,8 @@ def create_locallyconnected1d(self): stride_h=self.klayer.subsample_length, pad_w=0, pad_h=0, - wRegularizer=self.to_bigdl_reg(self.config["W_regularizer"]), - bRegularizer=self.to_bigdl_reg(self.config["b_regularizer"]), + wRegularizer=to_bigdl_reg(self.config["W_regularizer"]), + bRegularizer=to_bigdl_reg(self.config["b_regularizer"]), with_bias=self.klayer.bias, data_format="NHWC") seq.add(blayer) @@ -1736,7 +1691,7 @@ def create_locallyconnected2d(self): input_width = int(self.input_shape[2]) input_height = int(self.input_shape[1]) - bpadW, bpadH = self.to_bigdl_2d_padding(self.klayer.border_mode) + bpadW, bpadH = to_bigdl_2d_padding(self.klayer.border_mode) blayer = BLayer.LocallyConnected2D(n_input_plane=stack_size, input_width=input_width, input_height=input_height, @@ -1747,8 +1702,8 @@ def create_locallyconnected2d(self): stride_h=self.klayer.subsample[0], pad_w=bpadW, pad_h=bpadH, - wRegularizer=self.to_bigdl_reg(self.config["W_regularizer"]), - bRegularizer=self.to_bigdl_reg(self.config["b_regularizer"]), + wRegularizer=to_bigdl_reg(self.config["W_regularizer"]), + bRegularizer=to_bigdl_reg(self.config["b_regularizer"]), with_bias=self.klayer.bias, data_format=bigdl_order) @@ -1763,7 +1718,7 @@ def combo_parameter_layer(self, blayer, config): blayer.set_name(config["name"]) if hasattr(blayer, "set_init_method"): try: - blayer.set_init_method(self.to_bigdl_init(config["init"]), + blayer.set_init_method(to_bigdl_init(config["init"]), BInit.Zeros()) # Keras always set this to be zeros except Exception: warning_msg = "We don't support initialization " + config["init"] + " for now. " \ @@ -1785,28 +1740,6 @@ def get_value_from_init(self, kinit_method, shape): else: raise Exception("We don't support % for now", kinit_method) - def to_bigdl_init(self, kinit_method): # kinit_method is a string - init = None - if kinit_method == "glorot_uniform": - init = BInit.Xavier() - elif kinit_method == "one": - init = BInit.Ones() - elif kinit_method == "zero": - init = BInit.Zeros() - elif kinit_method == "uniform": - init = BInit.RandomUniform(lower=-0.05, upper=0.05) - elif kinit_method == "normal": - init = BInit.RandomNormal(mean=0.0, stdv=0.05) - else: - raise Exception("Unsupported init type: %s" % kinit_method) - return init - - def to_bigdl_reg(self, reg): # reg is a dict - if reg: - return BRegularizer(reg['l1'], reg['l2']) - else: - return None - def fuse(self, src_blayer, activation): # activation is a layer seq = BLayer.Sequential() seq.add(src_blayer) diff --git a/python/dllib/src/bigdl/dllib/nn/keras/__init__.py b/python/dllib/src/bigdl/dllib/nn/keras/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/python/dllib/src/bigdl/utils/common.py b/python/dllib/src/bigdl/utils/common.py index 25248e630af..5e3756fd6ec 100644 --- a/python/dllib/src/bigdl/utils/common.py +++ b/python/dllib/src/bigdl/utils/common.py @@ -55,7 +55,14 @@ def instance(cls, class JavaCreator(SingletonMixin): - __creator_class="com.intel.analytics.bigdl.python.api.PythonBigDL" + __creator_class=["com.intel.analytics.bigdl.python.api.PythonBigDL", + "com.intel.analytics.bigdl.python.api.PythonBigDLKeras"] + + @classmethod + def add_creator_class(cls, jinvoker): + with JavaCreator._lock: + JavaCreator.__creator_class.append(jinvoker) + JavaCreator._instance = None @classmethod def get_creator_class(cls): @@ -70,13 +77,15 @@ def set_creator_class(cls, cclass): def __init__(self, bigdl_type): sc = get_spark_context() - jclass = getattr(sc._jvm, JavaCreator.get_creator_class()) - if bigdl_type == "float": - self.value = getattr(jclass, "ofFloat")() - elif bigdl_type == "double": - self.value = getattr(jclass, "ofDouble")() - else: - raise Exception("Not supported bigdl_type: %s" % bigdl_type) + self.value = [] + for creator_class in JavaCreator.get_creator_class(): + jclass = getattr(sc._jvm, creator_class) + if bigdl_type == "float": + self.value.append(getattr(jclass, "ofFloat")()) + elif bigdl_type == "double": + self.value.append(getattr(jclass, "ofDouble")()) + else: + raise Exception("Not supported bigdl_type: %s" % bigdl_type) class JavaValue(object): @@ -398,7 +407,6 @@ def __str__(self): def __repr__(self): return "Sample: features: %s, labels: %s" % (self.features, self.labels) - class RNG(): """ generate tensor data with seed @@ -549,13 +557,23 @@ def get_spark_sql_context(sc): else: return SQLContext(sc) # Compatible with Spark1.5.1 - def callBigDlFunc(bigdl_type, name, *args): """ Call API in PythonBigDL """ - jinstance = JavaCreator.instance(bigdl_type=bigdl_type).value sc = get_spark_context() - api = getattr(jinstance, name) - return callJavaFunc(sc, api, *args) + error = Exception("Cannot find function: %s" % name) + for jinvoker in JavaCreator.instance(bigdl_type=bigdl_type).value: + # hasattr(jinvoker, name) always return true here, + # so you need to invoke the method to check if it exist or not + try: + api = getattr(jinvoker, name) + result = callJavaFunc(sc, api, *args) + except Exception as e: + error = e + if "does not exist" not in e.message: + raise e + else: + return result + raise error def _java2py(sc, r, encoding="bytes"): From a78499ad5a21a0df042fbea37af3122312cb38d1 Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Wed, 31 Jan 2018 09:04:11 +0800 Subject: [PATCH 541/823] Add python api for keras layer (#2242) * firtst try * update * update Dense * clean * style * style --- python/dllib/test/bigdl/test_pickler.py | 2 +- python/dllib/test/bigdl/test_simple_integration.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/python/dllib/test/bigdl/test_pickler.py b/python/dllib/test/bigdl/test_pickler.py index 8697775fb67..a3e41eec34f 100644 --- a/python/dllib/test/bigdl/test_pickler.py +++ b/python/dllib/test/bigdl/test_pickler.py @@ -35,7 +35,7 @@ def setup_method(self, method): """ setup any state tied to the execution of the given method in a class. setup_method is invoked for every test method of a class. """ - JavaCreator.set_creator_class("com.intel.analytics.bigdl.python.api.PythonBigDLValidator") + JavaCreator.add_creator_class("com.intel.analytics.bigdl.python.api.PythonBigDLValidator") sparkConf = create_spark_conf().setMaster("local[4]").setAppName("test model") self.sc = get_spark_context(sparkConf) init_engine() diff --git a/python/dllib/test/bigdl/test_simple_integration.py b/python/dllib/test/bigdl/test_simple_integration.py index 778c278393d..500c8455f9a 100644 --- a/python/dllib/test/bigdl/test_simple_integration.py +++ b/python/dllib/test/bigdl/test_simple_integration.py @@ -601,6 +601,5 @@ def test_local_predict_multiple_input(self): JTensor.from_ndarray(np.ones([4, 3]))]) assert result4.shape == (4,) - if __name__ == "__main__": pytest.main([__file__]) From 51718414830c170202322f23828b3a7819897a88 Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Wed, 31 Jan 2018 15:07:49 +0800 Subject: [PATCH 542/823] fix syntax (#2252) --- python/dllib/src/bigdl/dllib/utils/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/dllib/src/bigdl/dllib/utils/common.py b/python/dllib/src/bigdl/dllib/utils/common.py index 5e3756fd6ec..1b3962f840f 100644 --- a/python/dllib/src/bigdl/dllib/utils/common.py +++ b/python/dllib/src/bigdl/dllib/utils/common.py @@ -569,7 +569,7 @@ def callBigDlFunc(bigdl_type, name, *args): result = callJavaFunc(sc, api, *args) except Exception as e: error = e - if "does not exist" not in e.message: + if "does not exist" not in str(e): raise e else: return result From c1cd09aee0afd60707f83c1501e8f36a09f0796f Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Wed, 31 Jan 2018 15:07:49 +0800 Subject: [PATCH 543/823] fix syntax (#2252) --- python/dllib/src/bigdl/utils/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/dllib/src/bigdl/utils/common.py b/python/dllib/src/bigdl/utils/common.py index 5e3756fd6ec..1b3962f840f 100644 --- a/python/dllib/src/bigdl/utils/common.py +++ b/python/dllib/src/bigdl/utils/common.py @@ -569,7 +569,7 @@ def callBigDlFunc(bigdl_type, name, *args): result = callJavaFunc(sc, api, *args) except Exception as e: error = e - if "does not exist" not in e.message: + if "does not exist" not in str(e): raise e else: return result From b4dfcab3e1683ecc3deb308a07c9b13818205dc5 Mon Sep 17 00:00:00 2001 From: dding3 Date: Wed, 31 Jan 2018 19:30:46 -0800 Subject: [PATCH 544/823] Support gradual increase learning rate and SequentialSchedule in learningratescheduler (#2093) * support SequentialSchedule and WarmupSchedule --- .../dllib/src/bigdl/dllib/optim/optimizer.py | 46 +++++++++++++++++-- 1 file changed, 43 insertions(+), 3 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/optim/optimizer.py b/python/dllib/src/bigdl/dllib/optim/optimizer.py index 050830c8b7f..6ebf76c5a05 100644 --- a/python/dllib/src/bigdl/dllib/optim/optimizer.py +++ b/python/dllib/src/bigdl/dllib/optim/optimizer.py @@ -225,9 +225,8 @@ class Poly(JavaValue): Calculation: base_lr (1 - iter/max_iteration) ^ (power) - :param power: - :param max_iteration: - + :param power: coeffient of decay, refer to calculation formula + :param max_iteration: max iteration when lr becomes zero >>> poly = Poly(0.5, 2) creating: createPoly @@ -316,6 +315,47 @@ def __init__(self, JavaValue.__init__(self, None, bigdl_type, monitor, factor, patience, mode, epsilon, cooldown, min_lr) +class Warmup(JavaValue): + """ + A learning rate gradual increase policy, where the effective learning rate + increase delta after each iteration. + Calculation: base_lr + delta * iteration + + :param delta: increase amount after each iteration + + >>> warmup = Warmup(0.05) + creating: createWarmup + """ + def __init__(self, delta, bigdl_type="float"): + JavaValue.__init__(self, None, bigdl_type, delta) + +class SequentialSchedule(JavaValue): + """ + Stack several learning rate schedulers. + + :param iterationPerEpoch: iteration numbers per epoch + + >>> sequentialSchedule = SequentialSchedule(5) + creating: createSequentialSchedule + >>> poly = Poly(0.5, 2) + creating: createPoly + >>> test = sequentialSchedule.add(poly, 5) + + + + """ + def __init__(self, iteration_per_epoch, bigdl_type="float"): + JavaValue.__init__(self, None, bigdl_type, iteration_per_epoch) + + def add(self, scheduler, max_iteration, bigdl_type="float"): + """ + Add a learning rate scheduler to the contained `schedules` + + :param scheduler: learning rate scheduler to be add + :param max_iteration: iteration numbers this scheduler will run + """ + return callBigDlFunc(bigdl_type, "addScheduler", self.value, scheduler, max_iteration) + class OptimMethod(JavaValue): def __init__(self, jvalue, bigdl_type, *args): From 85916aed0a6a2e28fb0de7ed90600f86162303e1 Mon Sep 17 00:00:00 2001 From: dding3 Date: Wed, 31 Jan 2018 19:30:46 -0800 Subject: [PATCH 545/823] Support gradual increase learning rate and SequentialSchedule in learningratescheduler (#2093) * support SequentialSchedule and WarmupSchedule --- .../dllib/src/bigdl/dllib/optim/optimizer.py | 46 +++++++++++++++++-- 1 file changed, 43 insertions(+), 3 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/optim/optimizer.py b/python/dllib/src/bigdl/dllib/optim/optimizer.py index 050830c8b7f..6ebf76c5a05 100644 --- a/python/dllib/src/bigdl/dllib/optim/optimizer.py +++ b/python/dllib/src/bigdl/dllib/optim/optimizer.py @@ -225,9 +225,8 @@ class Poly(JavaValue): Calculation: base_lr (1 - iter/max_iteration) ^ (power) - :param power: - :param max_iteration: - + :param power: coeffient of decay, refer to calculation formula + :param max_iteration: max iteration when lr becomes zero >>> poly = Poly(0.5, 2) creating: createPoly @@ -316,6 +315,47 @@ def __init__(self, JavaValue.__init__(self, None, bigdl_type, monitor, factor, patience, mode, epsilon, cooldown, min_lr) +class Warmup(JavaValue): + """ + A learning rate gradual increase policy, where the effective learning rate + increase delta after each iteration. + Calculation: base_lr + delta * iteration + + :param delta: increase amount after each iteration + + >>> warmup = Warmup(0.05) + creating: createWarmup + """ + def __init__(self, delta, bigdl_type="float"): + JavaValue.__init__(self, None, bigdl_type, delta) + +class SequentialSchedule(JavaValue): + """ + Stack several learning rate schedulers. + + :param iterationPerEpoch: iteration numbers per epoch + + >>> sequentialSchedule = SequentialSchedule(5) + creating: createSequentialSchedule + >>> poly = Poly(0.5, 2) + creating: createPoly + >>> test = sequentialSchedule.add(poly, 5) + + + + """ + def __init__(self, iteration_per_epoch, bigdl_type="float"): + JavaValue.__init__(self, None, bigdl_type, iteration_per_epoch) + + def add(self, scheduler, max_iteration, bigdl_type="float"): + """ + Add a learning rate scheduler to the contained `schedules` + + :param scheduler: learning rate scheduler to be add + :param max_iteration: iteration numbers this scheduler will run + """ + return callBigDlFunc(bigdl_type, "addScheduler", self.value, scheduler, max_iteration) + class OptimMethod(JavaValue): def __init__(self, jvalue, bigdl_type, *args): From cbd0920fe56830b698fd07a4b4ccff91759250e4 Mon Sep 17 00:00:00 2001 From: Yanzhang Wang Date: Thu, 1 Feb 2018 02:23:34 -0500 Subject: [PATCH 546/823] fix: initializing of weights for SReLU should not in forward. (#2257) * fix: initializing of weights for SReLU should not in forward. Otherwise, the Keras will not convert successfully. * fix: add exception for SReLU when call wrong initMethod * fix: python style * fix: self.config err --- python/dllib/src/bigdl/dllib/keras/converter.py | 12 ++++++++++-- python/dllib/src/bigdl/dllib/nn/layer.py | 17 +++++++++++------ 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/keras/converter.py b/python/dllib/src/bigdl/dllib/keras/converter.py index df870f8ffca..e71d30b62e7 100644 --- a/python/dllib/src/bigdl/dllib/keras/converter.py +++ b/python/dllib/src/bigdl/dllib/keras/converter.py @@ -1599,10 +1599,18 @@ def create_masking(self): def create_srelu(self): if "shared_axes" not in self.config: warnings.warn("Cannot find shared_axes from json definition. Using shared_axes=None instead.") + shape = self.input_shape[1:] + t_left_init = to_bigdl_init(self.config["t_left_init"]) + a_left_init = to_bigdl_init(self.config["a_left_init"]) + t_right_init = to_bigdl_init(self.config["t_right_init"]) + a_right_init = to_bigdl_init(self.config["a_right_init"]) if self.klayer.shared_axes == [None]: - return BLayer.SReLU() + srelu = BLayer.SReLU(shape) else: - return BLayer.SReLU(self.klayer.shared_axes) + srelu = BLayer.SReLU(shape, self.klayer.shared_axes) + + srelu.set_init_method(t_left_init, a_left_init, t_right_init, a_right_init) + return srelu def create_separableconvolution2d(self): if keras.backend.backend() != 'tensorflow': diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 6b62044cfbb..34791b68d87 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -3679,8 +3679,8 @@ class SReLU(Layer): # References - [Deep Learning with S-shaped Rectified Linear Activation Units](http://arxiv.org/abs/1512.07030) - - + :param input_shape: shape for tleft, aleft, tright, aright. + E.g. for a 4-D input, the shape is the last 3-D :param shared_axes: the axes along which to share learnable parameters for the activation function. For example, if the incoming feature maps @@ -3690,22 +3690,27 @@ class SReLU(Layer): so that each filter only has one set of parameters, set `shared_axes=[1, 2]`. - >>> srelu = SReLU() + >>> srelu = SReLU((2, 3)) creating: createSReLU - >>> srelu = SReLU((1, 2)) + >>> srelu = SReLU((2, 2), (1, 2)) creating: createSReLU + >>> from bigdl.nn.initialization_method import Xavier + >>> init = Xavier() + creating: createXavier + >>> srelu = srelu.set_init_method(tLeftInit=init, aLeftInit=init, tRightInit=init, aRightInit=init) ''' def __init__(self, + input_shape, share_axes=None, bigdl_type="float"): - super(SReLU, self).__init__(None, bigdl_type, + super(SReLU, self).__init__(None, bigdl_type, input_shape, share_axes) def set_init_method(self, tLeftInit=None, aLeftInit=None, tRightInit=None, aRightInit=None): callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, - tLeftInit, aLeftInit, tRightInit, aRightInit) + [tLeftInit, aLeftInit, tRightInit, aRightInit]) return self class ActivityRegularization(Layer): From d9724ebdc8dae4f09f53f4b9297e2fa0e8a79f2e Mon Sep 17 00:00:00 2001 From: Yanzhang Wang Date: Thu, 1 Feb 2018 02:23:34 -0500 Subject: [PATCH 547/823] fix: initializing of weights for SReLU should not in forward. (#2257) * fix: initializing of weights for SReLU should not in forward. Otherwise, the Keras will not convert successfully. * fix: add exception for SReLU when call wrong initMethod * fix: python style * fix: self.config err --- python/dllib/src/bigdl/dllib/keras/converter.py | 12 ++++++++++-- python/dllib/src/bigdl/dllib/nn/layer.py | 17 +++++++++++------ 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/keras/converter.py b/python/dllib/src/bigdl/dllib/keras/converter.py index df870f8ffca..e71d30b62e7 100644 --- a/python/dllib/src/bigdl/dllib/keras/converter.py +++ b/python/dllib/src/bigdl/dllib/keras/converter.py @@ -1599,10 +1599,18 @@ def create_masking(self): def create_srelu(self): if "shared_axes" not in self.config: warnings.warn("Cannot find shared_axes from json definition. Using shared_axes=None instead.") + shape = self.input_shape[1:] + t_left_init = to_bigdl_init(self.config["t_left_init"]) + a_left_init = to_bigdl_init(self.config["a_left_init"]) + t_right_init = to_bigdl_init(self.config["t_right_init"]) + a_right_init = to_bigdl_init(self.config["a_right_init"]) if self.klayer.shared_axes == [None]: - return BLayer.SReLU() + srelu = BLayer.SReLU(shape) else: - return BLayer.SReLU(self.klayer.shared_axes) + srelu = BLayer.SReLU(shape, self.klayer.shared_axes) + + srelu.set_init_method(t_left_init, a_left_init, t_right_init, a_right_init) + return srelu def create_separableconvolution2d(self): if keras.backend.backend() != 'tensorflow': diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 6b62044cfbb..34791b68d87 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -3679,8 +3679,8 @@ class SReLU(Layer): # References - [Deep Learning with S-shaped Rectified Linear Activation Units](http://arxiv.org/abs/1512.07030) - - + :param input_shape: shape for tleft, aleft, tright, aright. + E.g. for a 4-D input, the shape is the last 3-D :param shared_axes: the axes along which to share learnable parameters for the activation function. For example, if the incoming feature maps @@ -3690,22 +3690,27 @@ class SReLU(Layer): so that each filter only has one set of parameters, set `shared_axes=[1, 2]`. - >>> srelu = SReLU() + >>> srelu = SReLU((2, 3)) creating: createSReLU - >>> srelu = SReLU((1, 2)) + >>> srelu = SReLU((2, 2), (1, 2)) creating: createSReLU + >>> from bigdl.nn.initialization_method import Xavier + >>> init = Xavier() + creating: createXavier + >>> srelu = srelu.set_init_method(tLeftInit=init, aLeftInit=init, tRightInit=init, aRightInit=init) ''' def __init__(self, + input_shape, share_axes=None, bigdl_type="float"): - super(SReLU, self).__init__(None, bigdl_type, + super(SReLU, self).__init__(None, bigdl_type, input_shape, share_axes) def set_init_method(self, tLeftInit=None, aLeftInit=None, tRightInit=None, aRightInit=None): callBigDlFunc(self.bigdl_type, "setInitMethod", self.value, - tLeftInit, aLeftInit, tRightInit, aRightInit) + [tLeftInit, aLeftInit, tRightInit, aRightInit]) return self class ActivityRegularization(Layer): From 1ba84b7c815562af53f9bb43016e4cc8bc9078f6 Mon Sep 17 00:00:00 2001 From: Xianyan Date: Fri, 2 Feb 2018 09:48:48 +0800 Subject: [PATCH 548/823] Add FixExpand and add more options to AspectScale (#2253) * Add FixExpand and add more options to AspectScale * fix ut --- .../dllib/feature/transform/vision/image.py | 23 ++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py b/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py index f851fa8632d..2ebab444e49 100644 --- a/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py +++ b/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py @@ -559,13 +559,21 @@ def __init__(self, to_rgb=False, tensor_key="imageTensor", bigdl_type="float"): class AspectScale(FeatureTransformer): """ Resize the image, keep the aspect ratio. scale according to the short edge - :param scale scale size, apply to short edge + :param min_size scale size, apply to short edge :param scale_multiple_of make the scaled size multiple of some value :param max_size max size after scale + :param resize_mode if resizeMode = -1, random select a mode from + (Imgproc.INTER_LINEAR, Imgproc.INTER_CUBIC, Imgproc.INTER_AREA, + Imgproc.INTER_NEAREST, Imgproc.INTER_LANCZOS4) + :param use_scale_factor if true, scale factor fx and fy is used, fx = fy = 0 + :aram min_scale control the minimum scale up for image """ - def __init__(self, scale, scale_multiple_of = 1, max_size = 1000, bigdl_type="float"): - super(AspectScale, self).__init__(bigdl_type, scale, scale_multiple_of, max_size) + def __init__(self, min_size, scale_multiple_of = 1, max_size = 1000, + resize_mode = 1, use_scale_factor=True, min_scale=-1.0, + bigdl_type="float"): + super(AspectScale, self).__init__(bigdl_type, min_size, scale_multiple_of, max_size, + resize_mode, use_scale_factor, min_scale) class RandomAspectScale(FeatureTransformer): """ @@ -604,6 +612,15 @@ class PixelBytesToMat(FeatureTransformer): def __init__(self, byte_key = "bytes", bigdl_type="float"): super(PixelBytesToMat, self).__init__(bigdl_type, byte_key) +class FixExpand(FeatureTransformer): + """ + Expand image with given expandHeight and expandWidth, + put the original image to the center of expanded image + :param expand_height height expand to + :param expand_width width expand to + """ + def __init__(self, expand_height, expand_width, bigdl_type="float"): + super(FixExpand, self).__init__(bigdl_type, expand_height, expand_width) class SeqFileFolder(JavaValue): From 2938bdb2076805dd3be800bc145a675dec7383b1 Mon Sep 17 00:00:00 2001 From: Xianyan Date: Fri, 2 Feb 2018 09:48:48 +0800 Subject: [PATCH 549/823] Add FixExpand and add more options to AspectScale (#2253) * Add FixExpand and add more options to AspectScale * fix ut --- .../dllib/feature/transform/vision/image.py | 23 ++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py b/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py index f851fa8632d..2ebab444e49 100644 --- a/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py +++ b/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py @@ -559,13 +559,21 @@ def __init__(self, to_rgb=False, tensor_key="imageTensor", bigdl_type="float"): class AspectScale(FeatureTransformer): """ Resize the image, keep the aspect ratio. scale according to the short edge - :param scale scale size, apply to short edge + :param min_size scale size, apply to short edge :param scale_multiple_of make the scaled size multiple of some value :param max_size max size after scale + :param resize_mode if resizeMode = -1, random select a mode from + (Imgproc.INTER_LINEAR, Imgproc.INTER_CUBIC, Imgproc.INTER_AREA, + Imgproc.INTER_NEAREST, Imgproc.INTER_LANCZOS4) + :param use_scale_factor if true, scale factor fx and fy is used, fx = fy = 0 + :aram min_scale control the minimum scale up for image """ - def __init__(self, scale, scale_multiple_of = 1, max_size = 1000, bigdl_type="float"): - super(AspectScale, self).__init__(bigdl_type, scale, scale_multiple_of, max_size) + def __init__(self, min_size, scale_multiple_of = 1, max_size = 1000, + resize_mode = 1, use_scale_factor=True, min_scale=-1.0, + bigdl_type="float"): + super(AspectScale, self).__init__(bigdl_type, min_size, scale_multiple_of, max_size, + resize_mode, use_scale_factor, min_scale) class RandomAspectScale(FeatureTransformer): """ @@ -604,6 +612,15 @@ class PixelBytesToMat(FeatureTransformer): def __init__(self, byte_key = "bytes", bigdl_type="float"): super(PixelBytesToMat, self).__init__(bigdl_type, byte_key) +class FixExpand(FeatureTransformer): + """ + Expand image with given expandHeight and expandWidth, + put the original image to the center of expanded image + :param expand_height height expand to + :param expand_width width expand to + """ + def __init__(self, expand_height, expand_width, bigdl_type="float"): + super(FixExpand, self).__init__(bigdl_type, expand_height, expand_width) class SeqFileFolder(JavaValue): From be99543565f59d1f591ac8d0a164d643c1a59df9 Mon Sep 17 00:00:00 2001 From: Xianyan Date: Fri, 2 Feb 2018 09:48:48 +0800 Subject: [PATCH 550/823] Add FixExpand and add more options to AspectScale (#2253) * Add FixExpand and add more options to AspectScale * fix ut --- python/dllib/test/bigdl/transform/test_image.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/python/dllib/test/bigdl/transform/test_image.py b/python/dllib/test/bigdl/transform/test_image.py index d27a9567e38..159526f1963 100644 --- a/python/dllib/test/bigdl/transform/test_image.py +++ b/python/dllib/test/bigdl/transform/test_image.py @@ -142,6 +142,10 @@ def test_expand(self): max_expand_ratio=2.0) self.transformer_test(expand) + def test_fix_expand(self): + expand = FixExpand(1000, 1000) + self.transformer_test(expand) + def test_random_transformer(self): transformer = RandomTransformer(HFlip(), 0.5) self.transformer_test(transformer) From 8fe5396a01183ca606afd81b6b7d48f876203cfa Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Fri, 2 Feb 2018 21:28:47 +0800 Subject: [PATCH 551/823] Keras-like API Some layers and LeNet example (#2214) * update dense * style * doc for conv2d * add error msg * add ser test * fix softmax * readme for lenet * remove * fix style * softmax 2d ut * add ser test for softmax * fix grammar * review and update * scala doc * fix --- python/dllib/src/bigdl/dllib/keras/converter.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/python/dllib/src/bigdl/dllib/keras/converter.py b/python/dllib/src/bigdl/dllib/keras/converter.py index e71d30b62e7..5fad1fc2ddf 100644 --- a/python/dllib/src/bigdl/dllib/keras/converter.py +++ b/python/dllib/src/bigdl/dllib/keras/converter.py @@ -640,7 +640,10 @@ def exchangeNumbers(arr, i, j): return list(filter(lambda pair: pair[0] != pair[1], pairs)) def create_reshape(self): - blayer = BLayer.Reshape(self.klayer.target_shape, None) + if -1 in self.klayer.target_shape: + blayer = BLayer.InferReshape(self.klayer.target_shape, True) + else: + blayer = BLayer.Reshape(self.klayer.target_shape, None) return blayer def create_repeatvector(self): From 80d2c9bcd9d19a3d65103bebf984231a7f392cc6 Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Fri, 2 Feb 2018 21:28:47 +0800 Subject: [PATCH 552/823] Keras-like API Some layers and LeNet example (#2214) * update dense * style * doc for conv2d * add error msg * add ser test * fix softmax * readme for lenet * remove * fix style * softmax 2d ut * add ser test for softmax * fix grammar * review and update * scala doc * fix --- python/dllib/src/bigdl/dllib/keras/converter.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/python/dllib/src/bigdl/dllib/keras/converter.py b/python/dllib/src/bigdl/dllib/keras/converter.py index e71d30b62e7..5fad1fc2ddf 100644 --- a/python/dllib/src/bigdl/dllib/keras/converter.py +++ b/python/dllib/src/bigdl/dllib/keras/converter.py @@ -640,7 +640,10 @@ def exchangeNumbers(arr, i, j): return list(filter(lambda pair: pair[0] != pair[1], pairs)) def create_reshape(self): - blayer = BLayer.Reshape(self.klayer.target_shape, None) + if -1 in self.klayer.target_shape: + blayer = BLayer.InferReshape(self.klayer.target_shape, True) + else: + blayer = BLayer.Reshape(self.klayer.target_shape, None) return blayer def create_repeatvector(self): From cc9d6624d79aafca4e54984628a0174193dd229c Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Fri, 2 Feb 2018 21:28:47 +0800 Subject: [PATCH 553/823] Keras-like API Some layers and LeNet example (#2214) * update dense * style * doc for conv2d * add error msg * add ser test * fix softmax * readme for lenet * remove * fix style * softmax 2d ut * add ser test for softmax * fix grammar * review and update * scala doc * fix --- python/dllib/test/bigdl/keras/test_layer.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python/dllib/test/bigdl/keras/test_layer.py b/python/dllib/test/bigdl/keras/test_layer.py index 6beb0198fda..18a38849a60 100644 --- a/python/dllib/test/bigdl/keras/test_layer.py +++ b/python/dllib/test/bigdl/keras/test_layer.py @@ -309,6 +309,8 @@ def test_reshape(self): input_data = np.random.random_sample([1, 3, 5, 4]) layer = Reshape(target_shape=(3, 20), input_shape=(3, 5, 4)) self.modelTestSingleLayer(input_data, layer) + layer2 = Reshape(target_shape=(-1, 2, 3), input_shape=(3, 5, 4)) + self.modelTestSingleLayer(input_data, layer2) def test_repeatvector(self): input_data = np.random.random_sample([2, 3]) From 331e58a2154feed9ab4697bbbfa0af875c726bee Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Tue, 6 Feb 2018 19:50:03 +0800 Subject: [PATCH 554/823] Refactor layer creation for keras (#2277) * keras * update * fix --- .../dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py | 11 +++++++++-- python/dllib/src/bigdl/dllib/nn/layer.py | 4 ++-- python/dllib/src/bigdl/dllib/utils/common.py | 3 +-- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py b/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py index b52756e99b6..2d9b151d156 100644 --- a/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py +++ b/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py @@ -25,7 +25,14 @@ long = int unicode = str -class Dense(Layer): +class KerasLayer(Layer): + def jvm_class_constructor(self): + name = "createKeras" + self.__class__.__name__ + print("creating: " + name) + return name + + +class Dense(KerasLayer): """Just your regular densely-connected NN layer. # Example @@ -53,7 +60,7 @@ class Dense(Layer): the output would have shape `(nb_samples, output_dim)`. >>> dense = Dense(10, input_shape=(3, 4)) creating: createXavier - creating: createDense + creating: createKerasDense """ def __init__(self, output_dim, init='glorot_uniform', activation=None, diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 34791b68d87..b04276e58d9 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -43,7 +43,7 @@ class Node(JavaValue): """ def __init__(self, jvalue, bigdl_type, *args): self.value = jvalue if jvalue else callBigDlFunc( - bigdl_type, JavaValue.jvm_class_constructor(self), *args) + bigdl_type, self.jvm_class_constructor(), *args) self.bigdl_type = bigdl_type @classmethod @@ -74,7 +74,7 @@ def __init__(self, jvalue, bigdl_type, *args): self.value = jvalue else: self.value = callBigDlFunc( - bigdl_type, JavaValue.jvm_class_constructor(self), *args) + bigdl_type, self.jvm_class_constructor(), *args) self.bigdl_type = bigdl_type def set_running_mean(self, running_mean): diff --git a/python/dllib/src/bigdl/dllib/utils/common.py b/python/dllib/src/bigdl/dllib/utils/common.py index 1b3962f840f..3c19256864c 100644 --- a/python/dllib/src/bigdl/dllib/utils/common.py +++ b/python/dllib/src/bigdl/dllib/utils/common.py @@ -55,8 +55,7 @@ def instance(cls, class JavaCreator(SingletonMixin): - __creator_class=["com.intel.analytics.bigdl.python.api.PythonBigDL", - "com.intel.analytics.bigdl.python.api.PythonBigDLKeras"] + __creator_class=["com.intel.analytics.bigdl.python.api.PythonBigDLKeras"] @classmethod def add_creator_class(cls, jinvoker): From 45548b9514ac52f687d7d86080f843dd67ae29c1 Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Tue, 6 Feb 2018 19:50:03 +0800 Subject: [PATCH 555/823] Refactor layer creation for keras (#2277) * keras * update * fix --- .../dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py | 11 +++++++++-- python/dllib/src/bigdl/dllib/nn/layer.py | 4 ++-- python/dllib/src/bigdl/utils/common.py | 3 +-- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py b/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py index b52756e99b6..2d9b151d156 100644 --- a/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py +++ b/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py @@ -25,7 +25,14 @@ long = int unicode = str -class Dense(Layer): +class KerasLayer(Layer): + def jvm_class_constructor(self): + name = "createKeras" + self.__class__.__name__ + print("creating: " + name) + return name + + +class Dense(KerasLayer): """Just your regular densely-connected NN layer. # Example @@ -53,7 +60,7 @@ class Dense(Layer): the output would have shape `(nb_samples, output_dim)`. >>> dense = Dense(10, input_shape=(3, 4)) creating: createXavier - creating: createDense + creating: createKerasDense """ def __init__(self, output_dim, init='glorot_uniform', activation=None, diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 34791b68d87..b04276e58d9 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -43,7 +43,7 @@ class Node(JavaValue): """ def __init__(self, jvalue, bigdl_type, *args): self.value = jvalue if jvalue else callBigDlFunc( - bigdl_type, JavaValue.jvm_class_constructor(self), *args) + bigdl_type, self.jvm_class_constructor(), *args) self.bigdl_type = bigdl_type @classmethod @@ -74,7 +74,7 @@ def __init__(self, jvalue, bigdl_type, *args): self.value = jvalue else: self.value = callBigDlFunc( - bigdl_type, JavaValue.jvm_class_constructor(self), *args) + bigdl_type, self.jvm_class_constructor(), *args) self.bigdl_type = bigdl_type def set_running_mean(self, running_mean): diff --git a/python/dllib/src/bigdl/utils/common.py b/python/dllib/src/bigdl/utils/common.py index 1b3962f840f..3c19256864c 100644 --- a/python/dllib/src/bigdl/utils/common.py +++ b/python/dllib/src/bigdl/utils/common.py @@ -55,8 +55,7 @@ def instance(cls, class JavaCreator(SingletonMixin): - __creator_class=["com.intel.analytics.bigdl.python.api.PythonBigDL", - "com.intel.analytics.bigdl.python.api.PythonBigDLKeras"] + __creator_class=["com.intel.analytics.bigdl.python.api.PythonBigDLKeras"] @classmethod def add_creator_class(cls, jinvoker): From 24b85df4e3ee2b1e91cc8f6bac78fdcb905616da Mon Sep 17 00:00:00 2001 From: Xu Xiao Date: Wed, 7 Feb 2018 19:51:00 +0800 Subject: [PATCH 556/823] add CrossProduct Layer (#2273) * add CrossProduct Module * update scaladoc & unit tests * make some modifications * add python support * fix bug in python wrapper --- python/dllib/src/bigdl/dllib/nn/layer.py | 29 ++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index b04276e58d9..7b5357124f2 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -2556,6 +2556,35 @@ def __init__(self, super(CosineDistance, self).__init__(None, bigdl_type) +class CrossProduct(Layer): + + """ + A layer which takes a table of multiple tensors(n >= 2) as input + and calculate to dot product for `all combinations of pairs` among input tensors. + + Dot-product outputs are ordered according to orders of pairs in input Table. + For instance, input (Table) is T(A, B, C), output (Tensor) will be [A.*B, A.*C, B.*C]. + + Dimensions of input' Tensors could be one or two, if two, first dimension is `batchSize`. + For convenience, output is 2-dim Tensor regardless of input' dims. + + Table size checking and Tensor size checking will be execute before each forward, + when [[numTensor]] and [[embeddingSize]] are set values greater than zero. + + :param numTensor (for checking)number of Tensor input Table contains, :default 0(won't check) + :param embeddingSize (for checking)vector length of dot product, :default 0(won't check) + + >>> crossProduct = CrossProduct() + creating: createCrossProduct + """ + + def __init__(self, + numTensor=0, + embeddingSize=0, + bigdl_type="float"): + super(CrossProduct, self).__init__(None, bigdl_type, numTensor, embeddingSize) + + class UpSampling2D(Layer): """ Upsampling layer for 2D inputs. From 69ac392dcac598dcfb670efe173f3d7e3e61dcf1 Mon Sep 17 00:00:00 2001 From: Xu Xiao Date: Wed, 7 Feb 2018 19:51:00 +0800 Subject: [PATCH 557/823] add CrossProduct Layer (#2273) * add CrossProduct Module * update scaladoc & unit tests * make some modifications * add python support * fix bug in python wrapper --- python/dllib/src/bigdl/dllib/nn/layer.py | 29 ++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index b04276e58d9..7b5357124f2 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -2556,6 +2556,35 @@ def __init__(self, super(CosineDistance, self).__init__(None, bigdl_type) +class CrossProduct(Layer): + + """ + A layer which takes a table of multiple tensors(n >= 2) as input + and calculate to dot product for `all combinations of pairs` among input tensors. + + Dot-product outputs are ordered according to orders of pairs in input Table. + For instance, input (Table) is T(A, B, C), output (Tensor) will be [A.*B, A.*C, B.*C]. + + Dimensions of input' Tensors could be one or two, if two, first dimension is `batchSize`. + For convenience, output is 2-dim Tensor regardless of input' dims. + + Table size checking and Tensor size checking will be execute before each forward, + when [[numTensor]] and [[embeddingSize]] are set values greater than zero. + + :param numTensor (for checking)number of Tensor input Table contains, :default 0(won't check) + :param embeddingSize (for checking)vector length of dot product, :default 0(won't check) + + >>> crossProduct = CrossProduct() + creating: createCrossProduct + """ + + def __init__(self, + numTensor=0, + embeddingSize=0, + bigdl_type="float"): + super(CrossProduct, self).__init__(None, bigdl_type, numTensor, embeddingSize) + + class UpSampling2D(Layer): """ Upsampling layer for 2D inputs. From d7d1b1970574d54d721cb02c7d1207910a4480e5 Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Thu, 8 Feb 2018 15:01:48 +0800 Subject: [PATCH 558/823] Keras-like API AtrousConv1D2D, LocallyConnected1D, deconv, convlstm (#2247) * layers * meet review * update * add ser test * style and fix * refactor and update * style * fix * refine --- python/dllib/src/bigdl/dllib/keras/converter.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/keras/converter.py b/python/dllib/src/bigdl/dllib/keras/converter.py index 5fad1fc2ddf..7bbbca734e5 100644 --- a/python/dllib/src/bigdl/dllib/keras/converter.py +++ b/python/dllib/src/bigdl/dllib/keras/converter.py @@ -975,6 +975,8 @@ def create_convlstm2d(self): if self.config["border_mode"] != 'same': raise Exception("Unsupported border_mode: valid") + if self.klayer.dim_ordering != "th": + raise Exception("Please use `th` for `dim_ordering`. `%s` is not supported for now." % klayer.dim_ordering) if self.config["nb_row"] != self.config["nb_col"]: raise Exception("Only square kernel is supported for now. Please set nb_row=nb_col.") if self.klayer.subsample[0] != self.klayer.subsample[1]: From 931b37f217590aa5f5de82594847106c27c983f5 Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Thu, 8 Feb 2018 15:01:48 +0800 Subject: [PATCH 559/823] Keras-like API AtrousConv1D2D, LocallyConnected1D, deconv, convlstm (#2247) * layers * meet review * update * add ser test * style and fix * refactor and update * style * fix * refine --- python/dllib/src/bigdl/dllib/keras/converter.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/keras/converter.py b/python/dllib/src/bigdl/dllib/keras/converter.py index 5fad1fc2ddf..7bbbca734e5 100644 --- a/python/dllib/src/bigdl/dllib/keras/converter.py +++ b/python/dllib/src/bigdl/dllib/keras/converter.py @@ -975,6 +975,8 @@ def create_convlstm2d(self): if self.config["border_mode"] != 'same': raise Exception("Unsupported border_mode: valid") + if self.klayer.dim_ordering != "th": + raise Exception("Please use `th` for `dim_ordering`. `%s` is not supported for now." % klayer.dim_ordering) if self.config["nb_row"] != self.config["nb_col"]: raise Exception("Only square kernel is supported for now. Please set nb_row=nb_col.") if self.klayer.subsample[0] != self.klayer.subsample[1]: From 9f09a347528c5cffc8745bc28239fefed0ec2596 Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Thu, 8 Feb 2018 19:51:24 +0800 Subject: [PATCH 560/823] Keras-like API embedding, batchnorm, padding, upsampling layers (#2245) * layers * update * clean * clean * wrap * update * finish batchnorm * update batchnorm * update type * update upsampling3d * add ser test * fix style * fix * fix ut * update * update --- .../bigdl/dllib/bigdlkeras/layers/layer.py | 127 ++++++++++++------ .../dllib/src/bigdl/dllib/keras/converter.py | 9 +- python/dllib/src/bigdl/dllib/nn/layer.py | 2 +- 3 files changed, 96 insertions(+), 42 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py b/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py index 2d9b151d156..25b199ae9b5 100644 --- a/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py +++ b/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py @@ -16,15 +16,14 @@ import sys -from bigdl.keras.ToBigDLHelper import to_bigdl_reg, to_bigdl_init -from bigdl.nn.layer import Layer -from bigdl.util.common import get_activation_by_name - +from bigdl.nn.layer import Layer, Container +from bigdl.util.common import callBigDlFunc, JTensor if sys.version >= '3': long = int unicode = str + class KerasLayer(Layer): def jvm_class_constructor(self): name = "createKeras" + self.__class__.__name__ @@ -32,45 +31,95 @@ def jvm_class_constructor(self): return name +class Sequential(Container): + def __init__(self, bigdl_type="float"): + super(Sequential, self).__init__(None, bigdl_type, True) + + +class InputLayer(KerasLayer): + def __init__(self, input_shape=None, bigdl_type="float"): + super(InputLayer, self).__init__(None, bigdl_type, + list(input_shape) if input_shape else None) + + class Dense(KerasLayer): - """Just your regular densely-connected NN layer. - - # Example - - # Arguments - output_dim: int > 0. - init: name of initialization function for the weights of the layer - activation: name of activation function to use - W_regularizer: instance of regularizer (eg. L1 or L2 regularization), - applied to the main weights matrix. - b_regularizer: nstance of regularizer (eg. L1 or L2 regularization), - applied to bias - bias: whether to include a bias - (i.e. make the layer affine rather than linear). - input_shape: is required when using this layer as the first layer in a model. - - # Input shape - nD tensor with shape: `(nb_samples, ..., input_dim)`. - The most common situation would be - a 2D input with shape `(nb_samples, input_dim)`. - - # Output shape - nD tensor with shape: `(nb_samples, ..., output_dim)`. - For instance, for a 2D input with shape `(nb_samples, input_dim)`, - the output would have shape `(nb_samples, output_dim)`. - >>> dense = Dense(10, input_shape=(3, 4)) - creating: createXavier - creating: createKerasDense - """ - def __init__(self, output_dim, init='glorot_uniform', - activation=None, + """ + A densely-connected NN layer. + When you use this layer as the first layer of a model, you need to provide the argument + inputShape (a Single Shape, does not include the batch dimension). + The most common input is 2D. + + # Arguments + output_dim: The size of output dimension. + init: String representations of initialization method for the weights of the layer. + Default is 'glorot_uniform'. + activation: String representations of activation function to use (such as 'relu' or 'sigmoid'). + Default is None. + W_regularizer: An instance of [[Regularizer]], (eg. L1 or L2 regularization), + applied to the input weights matrices. Default is None. + b_regularizer: An instance of [[Regularizer]], applied to the bias. Default is None. + bias: Whether to include a bias (i.e. make the layer affine rather than linear). Default is True. + + >>> dense = Dense(10, input_shape=(3, 4)) + creating: createKerasDense + """ + def __init__(self, output_dim, init='glorot_uniform', activation=None, W_regularizer=None, b_regularizer=None, bias=True, input_shape=None, bigdl_type="float"): super(Dense, self).__init__(None, bigdl_type, output_dim, - to_bigdl_init(init), - get_activation_by_name(activation) if activation else None, # noqa - to_bigdl_reg(W_regularizer), - to_bigdl_reg(b_regularizer), + init, + activation, + W_regularizer, + b_regularizer, bias, list(input_shape) if input_shape else None) + + +class Embedding(KerasLayer): + """ + >>> embedding = Embedding(1000, 32, input_shape=(10, )) + creating: createKerasEmbedding + """ + def __init__(self, input_dim, output_dim, init='uniform', + W_regularizer=None, input_shape=None, bigdl_type="float"): + super(Embedding, self).__init__(None, bigdl_type, + input_dim, + output_dim, + init, + W_regularizer, + list(input_shape) if input_shape else None) + + +class BatchNormalization(KerasLayer): + """ + >>> batchNormalization = BatchNormalization(input_shape=(3, 12, 12)) + creating: createKerasBatchNormalization + """ + def __init__(self, epsilon=0.001, momentum=0.99, beta_init='zero', gamma_init='one', + dim_ordering="th", input_shape=None, bigdl_type="float"): + super(BatchNormalization, self).__init__(None, bigdl_type, + epsilon, + momentum, + beta_init, + gamma_init, + dim_ordering, + list(input_shape) if input_shape else None) + + def set_running_mean(self, running_mean): + callBigDlFunc(self.bigdl_type, "setKerasRunningMean", + self.value, JTensor.from_ndarray(running_mean)) + return self + + def set_running_std(self, running_std): + callBigDlFunc(self.bigdl_type, "setKerasRunningStd", + self.value, JTensor.from_ndarray(running_std)) + return self + + def get_running_mean(self): + return callBigDlFunc(self.bigdl_type, "getKerasRunningMean", + self.value).to_ndarray() + + def get_running_std(self): + return callBigDlFunc(self.bigdl_type, "getKerasRunningStd", + self.value).to_ndarray() diff --git a/python/dllib/src/bigdl/dllib/keras/converter.py b/python/dllib/src/bigdl/dllib/keras/converter.py index 7bbbca734e5..a012fead569 100644 --- a/python/dllib/src/bigdl/dllib/keras/converter.py +++ b/python/dllib/src/bigdl/dllib/keras/converter.py @@ -578,8 +578,9 @@ def create_embedding(self): should_scale_grad_by_freq=False, wRegularizer=to_bigdl_reg(self.config["W_regularizer"]), bigdl_type="float") - bseq.add(BLayer.AddConstant(1.0, inplace=True)) # Add 1 as BigDL is one-based index + bseq.add(BLayer.AddConstant(1.0)) # Add 1 as BigDL is one-based index bseq.add(blayer) + blayer.set_init_method(to_bigdl_init(self.config["init"])) return bseq def create_activation(self): @@ -976,7 +977,8 @@ def create_convlstm2d(self): raise Exception("Unsupported border_mode: valid") if self.klayer.dim_ordering != "th": - raise Exception("Please use `th` for `dim_ordering`. `%s` is not supported for now." % klayer.dim_ordering) + raise Exception("Please use `th` for `dim_ordering`. `%s` is not supported for now." + % self.klayer.dim_ordering) if self.config["nb_row"] != self.config["nb_col"]: raise Exception("Only square kernel is supported for now. Please set nb_row=nb_col.") if self.klayer.subsample[0] != self.klayer.subsample[1]: @@ -1014,6 +1016,9 @@ def create_gru(self): self.klayer.go_backwards, rec.add(gru)) def create_batchnormalization(self): + if len(self.input_shape) != 4: + raise Exception("Only 4D input is supported for now, but the current input dim is %s", + len(self.input_shape)) if keras.backend.image_dim_ordering() == "th" and self.klayer.axis != 1: raise Exception("""For BatchNormalization with th image ordering, we only support """ + """axis = 1 for now, but the current axis is %s diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 7b5357124f2..7f3f6d55dd3 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -1063,7 +1063,7 @@ class Sequential(Container): ''' def __init__(self, bigdl_type="float"): - super(Sequential, self).__init__(None, bigdl_type) + super(Sequential, self).__init__(None, bigdl_type, False) class TemporalConvolution(Layer): From d0d2757da1d9621d7140cd4eae706ee059e12e5e Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Thu, 8 Feb 2018 19:51:24 +0800 Subject: [PATCH 561/823] Keras-like API embedding, batchnorm, padding, upsampling layers (#2245) * layers * update * clean * clean * wrap * update * finish batchnorm * update batchnorm * update type * update upsampling3d * add ser test * fix style * fix * fix ut * update * update --- .../bigdl/dllib/bigdlkeras/layers/layer.py | 127 ++++++++++++------ .../dllib/src/bigdl/dllib/keras/converter.py | 9 +- python/dllib/src/bigdl/dllib/nn/layer.py | 2 +- 3 files changed, 96 insertions(+), 42 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py b/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py index 2d9b151d156..25b199ae9b5 100644 --- a/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py +++ b/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py @@ -16,15 +16,14 @@ import sys -from bigdl.keras.ToBigDLHelper import to_bigdl_reg, to_bigdl_init -from bigdl.nn.layer import Layer -from bigdl.util.common import get_activation_by_name - +from bigdl.nn.layer import Layer, Container +from bigdl.util.common import callBigDlFunc, JTensor if sys.version >= '3': long = int unicode = str + class KerasLayer(Layer): def jvm_class_constructor(self): name = "createKeras" + self.__class__.__name__ @@ -32,45 +31,95 @@ def jvm_class_constructor(self): return name +class Sequential(Container): + def __init__(self, bigdl_type="float"): + super(Sequential, self).__init__(None, bigdl_type, True) + + +class InputLayer(KerasLayer): + def __init__(self, input_shape=None, bigdl_type="float"): + super(InputLayer, self).__init__(None, bigdl_type, + list(input_shape) if input_shape else None) + + class Dense(KerasLayer): - """Just your regular densely-connected NN layer. - - # Example - - # Arguments - output_dim: int > 0. - init: name of initialization function for the weights of the layer - activation: name of activation function to use - W_regularizer: instance of regularizer (eg. L1 or L2 regularization), - applied to the main weights matrix. - b_regularizer: nstance of regularizer (eg. L1 or L2 regularization), - applied to bias - bias: whether to include a bias - (i.e. make the layer affine rather than linear). - input_shape: is required when using this layer as the first layer in a model. - - # Input shape - nD tensor with shape: `(nb_samples, ..., input_dim)`. - The most common situation would be - a 2D input with shape `(nb_samples, input_dim)`. - - # Output shape - nD tensor with shape: `(nb_samples, ..., output_dim)`. - For instance, for a 2D input with shape `(nb_samples, input_dim)`, - the output would have shape `(nb_samples, output_dim)`. - >>> dense = Dense(10, input_shape=(3, 4)) - creating: createXavier - creating: createKerasDense - """ - def __init__(self, output_dim, init='glorot_uniform', - activation=None, + """ + A densely-connected NN layer. + When you use this layer as the first layer of a model, you need to provide the argument + inputShape (a Single Shape, does not include the batch dimension). + The most common input is 2D. + + # Arguments + output_dim: The size of output dimension. + init: String representations of initialization method for the weights of the layer. + Default is 'glorot_uniform'. + activation: String representations of activation function to use (such as 'relu' or 'sigmoid'). + Default is None. + W_regularizer: An instance of [[Regularizer]], (eg. L1 or L2 regularization), + applied to the input weights matrices. Default is None. + b_regularizer: An instance of [[Regularizer]], applied to the bias. Default is None. + bias: Whether to include a bias (i.e. make the layer affine rather than linear). Default is True. + + >>> dense = Dense(10, input_shape=(3, 4)) + creating: createKerasDense + """ + def __init__(self, output_dim, init='glorot_uniform', activation=None, W_regularizer=None, b_regularizer=None, bias=True, input_shape=None, bigdl_type="float"): super(Dense, self).__init__(None, bigdl_type, output_dim, - to_bigdl_init(init), - get_activation_by_name(activation) if activation else None, # noqa - to_bigdl_reg(W_regularizer), - to_bigdl_reg(b_regularizer), + init, + activation, + W_regularizer, + b_regularizer, bias, list(input_shape) if input_shape else None) + + +class Embedding(KerasLayer): + """ + >>> embedding = Embedding(1000, 32, input_shape=(10, )) + creating: createKerasEmbedding + """ + def __init__(self, input_dim, output_dim, init='uniform', + W_regularizer=None, input_shape=None, bigdl_type="float"): + super(Embedding, self).__init__(None, bigdl_type, + input_dim, + output_dim, + init, + W_regularizer, + list(input_shape) if input_shape else None) + + +class BatchNormalization(KerasLayer): + """ + >>> batchNormalization = BatchNormalization(input_shape=(3, 12, 12)) + creating: createKerasBatchNormalization + """ + def __init__(self, epsilon=0.001, momentum=0.99, beta_init='zero', gamma_init='one', + dim_ordering="th", input_shape=None, bigdl_type="float"): + super(BatchNormalization, self).__init__(None, bigdl_type, + epsilon, + momentum, + beta_init, + gamma_init, + dim_ordering, + list(input_shape) if input_shape else None) + + def set_running_mean(self, running_mean): + callBigDlFunc(self.bigdl_type, "setKerasRunningMean", + self.value, JTensor.from_ndarray(running_mean)) + return self + + def set_running_std(self, running_std): + callBigDlFunc(self.bigdl_type, "setKerasRunningStd", + self.value, JTensor.from_ndarray(running_std)) + return self + + def get_running_mean(self): + return callBigDlFunc(self.bigdl_type, "getKerasRunningMean", + self.value).to_ndarray() + + def get_running_std(self): + return callBigDlFunc(self.bigdl_type, "getKerasRunningStd", + self.value).to_ndarray() diff --git a/python/dllib/src/bigdl/dllib/keras/converter.py b/python/dllib/src/bigdl/dllib/keras/converter.py index 7bbbca734e5..a012fead569 100644 --- a/python/dllib/src/bigdl/dllib/keras/converter.py +++ b/python/dllib/src/bigdl/dllib/keras/converter.py @@ -578,8 +578,9 @@ def create_embedding(self): should_scale_grad_by_freq=False, wRegularizer=to_bigdl_reg(self.config["W_regularizer"]), bigdl_type="float") - bseq.add(BLayer.AddConstant(1.0, inplace=True)) # Add 1 as BigDL is one-based index + bseq.add(BLayer.AddConstant(1.0)) # Add 1 as BigDL is one-based index bseq.add(blayer) + blayer.set_init_method(to_bigdl_init(self.config["init"])) return bseq def create_activation(self): @@ -976,7 +977,8 @@ def create_convlstm2d(self): raise Exception("Unsupported border_mode: valid") if self.klayer.dim_ordering != "th": - raise Exception("Please use `th` for `dim_ordering`. `%s` is not supported for now." % klayer.dim_ordering) + raise Exception("Please use `th` for `dim_ordering`. `%s` is not supported for now." + % self.klayer.dim_ordering) if self.config["nb_row"] != self.config["nb_col"]: raise Exception("Only square kernel is supported for now. Please set nb_row=nb_col.") if self.klayer.subsample[0] != self.klayer.subsample[1]: @@ -1014,6 +1016,9 @@ def create_gru(self): self.klayer.go_backwards, rec.add(gru)) def create_batchnormalization(self): + if len(self.input_shape) != 4: + raise Exception("Only 4D input is supported for now, but the current input dim is %s", + len(self.input_shape)) if keras.backend.image_dim_ordering() == "th" and self.klayer.axis != 1: raise Exception("""For BatchNormalization with th image ordering, we only support """ + """axis = 1 for now, but the current axis is %s diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 7b5357124f2..7f3f6d55dd3 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -1063,7 +1063,7 @@ class Sequential(Container): ''' def __init__(self, bigdl_type="float"): - super(Sequential, self).__init__(None, bigdl_type) + super(Sequential, self).__init__(None, bigdl_type, False) class TemporalConvolution(Layer): From b32ee418fddad5362d4244abba4c1510e1ef0b9d Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Thu, 8 Feb 2018 19:51:24 +0800 Subject: [PATCH 562/823] Keras-like API embedding, batchnorm, padding, upsampling layers (#2245) * layers * update * clean * clean * wrap * update * finish batchnorm * update batchnorm * update type * update upsampling3d * add ser test * fix style * fix * fix ut * update * update --- python/dllib/test/bigdl/keras/test_newapi.py | 54 ++++++++++++++++++++ python/dllib/test/bigdl/test_utils.py | 21 ++++++++ 2 files changed, 75 insertions(+) create mode 100644 python/dllib/test/bigdl/keras/test_newapi.py diff --git a/python/dllib/test/bigdl/keras/test_newapi.py b/python/dllib/test/bigdl/keras/test_newapi.py new file mode 100644 index 00000000000..22dc5b6e2d8 --- /dev/null +++ b/python/dllib/test/bigdl/keras/test_newapi.py @@ -0,0 +1,54 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest + +from test.bigdl.test_utils import BigDLTestCase +import bigdl.nn.keras.layer as BLayer +import keras.layers as KLayer +import keras.backend as K +import numpy as np +from bigdl.keras.converter import WeightsConverter + +np.random.seed(1337) # for reproducibility + + +class TestLayer(BigDLTestCase): + + def test_embedding(self): + input_data = np.random.randint(1000, size=(32, 10)) + blayer = BLayer.Embedding(1000, 64, input_shape=(10, )) + klayer = KLayer.Embedding(1000, 64, input_length=10) + self.compare_newapi(klayer, blayer, input_data, + WeightsConverter.convert_embedding) + + def test_batchnormalization(self): + K.set_image_dim_ordering("th") + input_data = np.random.random_sample([2, 5, 32, 32]) + blayer = BLayer.BatchNormalization(input_shape=(5, 32, 32)) + klayer = KLayer.BatchNormalization(axis=1, input_shape=(5, 32, 32)) + self.compare_newapi(klayer, blayer, input_data, + WeightsConverter.convert_batchnormalization) + K.set_image_dim_ordering("tf") + input_data2 = np.random.random_sample([2, 32, 32, 4]) + blayer = BLayer.BatchNormalization(dim_ordering="tf", input_shape=(32, 32, 4)) + klayer = KLayer.BatchNormalization(axis=-1, input_shape=(32, 32, 4)) + self.compare_newapi(klayer, blayer, input_data2, + WeightsConverter.convert_batchnormalization) + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/python/dllib/test/bigdl/test_utils.py b/python/dllib/test/bigdl/test_utils.py index c38f9da49ee..2ef0b91f4ec 100644 --- a/python/dllib/test/bigdl/test_utils.py +++ b/python/dllib/test/bigdl/test_utils.py @@ -318,3 +318,24 @@ def compare_loss(self, y_a, y_b, kloss, bloss, rtol=1e-6, atol=1e-6): keras_output = np.mean(K.eval(kloss(K.variable(y_b), K.variable(y_a)))) bigdl_output = bloss.forward(y_a, y_b) np.testing.assert_allclose(bigdl_output, keras_output, rtol=rtol, atol=atol) + + # Compare forward results with Keras for new Keras-like API layers. + def compare_newapi(self, klayer, blayer, input_data, weight_converter, + is_training=False, rtol=1e-6, atol=1e-6): + from keras.models import Sequential as KSequential + from bigdl.nn.keras.layer import Sequential as BSequential + bmodel = BSequential() + bmodel.add(blayer) + kmodel = KSequential() + kmodel.add(klayer) + koutput = kmodel.predict(input_data) + if isinstance(blayer, BLayer.BatchNormalization): + k_running_mean = K.eval(klayer.running_mean) + k_running_std = K.eval(klayer.running_std) + blayer.set_running_mean(k_running_mean) + blayer.set_running_std(k_running_std) + if kmodel.get_weights(): + bmodel.set_weights(weight_converter(klayer, kmodel.get_weights())) + bmodel.training(is_training) + boutput = bmodel.forward(input_data) + self.assert_allclose(boutput, koutput, rtol=rtol, atol=atol) From 0a9a14366f1d7145921988f203753b37f0c594dc Mon Sep 17 00:00:00 2001 From: Yuhao Yang Date: Thu, 8 Feb 2018 13:21:05 -0800 Subject: [PATCH 563/823] move DF API to dlframe (#2269) * move DF API to dlframe * remove extra lines * deprecate * extra line --- dlframes/__init__.py | 0 dlframes/dl_classifier.py | 150 ++++++++++++++++++ .../dllib/models/ml_pipeline/dl_classifier.py | 19 +++ 3 files changed, 169 insertions(+) create mode 100644 dlframes/__init__.py create mode 100644 dlframes/dl_classifier.py diff --git a/dlframes/__init__.py b/dlframes/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/dlframes/dl_classifier.py b/dlframes/dl_classifier.py new file mode 100644 index 00000000000..34cc298b641 --- /dev/null +++ b/dlframes/dl_classifier.py @@ -0,0 +1,150 @@ +from pyspark.ml.pipeline import Estimator, Model +from pyspark.ml.param.shared import * +from bigdl.util.common import * + + +if sys.version >= '3': + long = int + unicode = str + + +class HasBatchSize(Params): + """ + Mixin for param batchSize: batch size. + """ + + # a placeholder to make it appear in the generated doc + batchSize = Param(Params._dummy(), "batchSize", "batchSize") + + def __init__(self): + super(HasBatchSize, self).__init__() + #: param for batch size. + self.batchSize = Param(self, "batchSize", "batchSize") + self._setDefault(batchSize=1) + + def setBatchSize(self, val): + """ + Sets the value of :py:attr:`batchSize`. + """ + self._paramMap[self.batchSize] = val + pythonBigDL_method_name = "setBatchSize" + self.__class__.__name__ + callBigDlFunc(self.bigdl_type, pythonBigDL_method_name, self.value, val) + return self + + def getBatchSize(self): + """ + Gets the value of batchSize or its default value. + """ + return self.getOrDefault(self.batchSize) + +class HasMaxEpoch(Params): + maxEpoch = Param(Params._dummy(), "maxEpoch", "number of max Epoch") + + def __init__(self): + super(HasMaxEpoch, self).__init__() + self.maxEpoch = Param(self, "maxEpoch", "maxEpoch") + self._setDefault(maxEpoch=100) + + def setMaxEpoch(self, val): + self._paramMap[self.maxEpoch] = val + pythonBigDL_method_name = "setMaxEpoch" + self.__class__.__name__ + callBigDlFunc(self.bigdl_type, pythonBigDL_method_name, self.value, val) + return self + + def getMaxEpoch(self): + """ + Gets the value of maxEpoch or its default value. + """ + return self.getOrDefault(self.maxEpoch) + +class HasFeatureSize(Params): + featureSize = Param(Params._dummy(), "featureSize", "size of the feature") + + def __init__(self): + super(HasFeatureSize, self).__init__() + self.featureSize = Param(self, "featureSize", "featureSize") + self._setDefault(featureSize=None) + + def setFeatureSize(self, val): + self._paramMap[self.featureSize] = val + pythonBigDL_mehtod_name = "setFeatureSize" + self.__class__.__name__ + callBigDlFunc(self.bigdl_type, pythonBigDL_mehtod_name, self.value, val) + return self + + def getFeatureSize(self): + return self.getOrDefault(self.featureSize) + +class HasLearningRate(Params): + learningRate = Param(Params._dummy(), "learningRate", "learning rate") + + def __init__(self): + super(HasLearningRate, self).__init__() + self.learningRate = Param(self, "learningRate", "learning rate") + self._setDefault(learningRate=100) + + def setLearningRate(self, val): + self._paramMap[self.learningRate] = val + pythonBigDL_method_name = "setLearningRate" + self.__class__.__name__ + callBigDlFunc(self.bigdl_type, pythonBigDL_method_name, self.value, val) + return self + + def getLearningRate(self): + """ + Gets the value of maxEpoch or its default value. + """ + return self.getOrDefault(self.learningRate) + +class DLEstimator(Estimator, HasFeaturesCol, HasLabelCol, HasPredictionCol, HasBatchSize, HasMaxEpoch, HasLearningRate, JavaValue): + + def __init__(self, model, criterion, feature_size, label_size, jvalue=None, bigdl_type="float"): + super(DLEstimator, self).__init__() + self.value = jvalue if jvalue else callBigDlFunc( + bigdl_type, self.jvm_class_constructor(), model, criterion, feature_size, label_size) + self.bigdl_type = bigdl_type + self.featureSize = feature_size + + def _fit(self, dataset): + #self._transfer_params_to_java() + jmodel = callBigDlFunc(self.bigdl_type, "fitEstimator", self.value, dataset) + model = DLModel.of(jmodel, self.featureSize, self.bigdl_type) + return model + + +class DLModel(Model, HasFeaturesCol, HasPredictionCol, HasBatchSize, HasFeatureSize, JavaValue): + def __init__(self, model, featureSize, jvalue=None, bigdl_type="float"): + super(DLModel, self).__init__() + self.value = jvalue if jvalue else callBigDlFunc( + bigdl_type, self.jvm_class_constructor(), model, featureSize) + self.bigdl_type = bigdl_type + self.setFeatureSize(featureSize) + + def _transform(self, dataset): + return callBigDlFunc(self.bigdl_type, "dlModelTransform", self.value, dataset) + + @classmethod + def of(self, jvalue, feature_size=None, bigdl_type="float"): + model = DLModel(model=None, featureSize=feature_size, jvalue=jvalue, bigdl_type=bigdl_type) + return model + + +class DLClassifier(DLEstimator): + def __init__(self, model, criterion, feature_size, bigdl_type="float"): + super(DLClassifier, self).__init__(model, criterion, feature_size, [1], None, bigdl_type) + + def _fit(self, dataset): + jmodel = callBigDlFunc(self.bigdl_type, "fitClassifier", self.value, dataset) + model = DLClassifierModel.of(jmodel, self.featureSize, self.bigdl_type) + return model + + +class DLClassifierModel(DLModel): + def __init__(self, model, featureSize, jvalue=None, bigdl_type="float"): + super(DLClassifierModel, self).__init__(model, featureSize, jvalue, bigdl_type) + + def _transform(self, dataset): + return callBigDlFunc(self.bigdl_type, "dlClassifierModelTransform", self.value, dataset) + + @classmethod + def of(self, jvalue, feature_size=None, bigdl_type="float"): + model = DLClassifierModel(model=None, featureSize=feature_size, jvalue=jvalue, bigdl_type=bigdl_type) + return model diff --git a/python/dllib/src/bigdl/dllib/models/ml_pipeline/dl_classifier.py b/python/dllib/src/bigdl/dllib/models/ml_pipeline/dl_classifier.py index 4dc4cd96248..dd0833fa6f6 100644 --- a/python/dllib/src/bigdl/dllib/models/ml_pipeline/dl_classifier.py +++ b/python/dllib/src/bigdl/dllib/models/ml_pipeline/dl_classifier.py @@ -95,7 +95,11 @@ def getLearningRate(self): return self.getOrDefault(self.learningRate) class DLEstimator(Estimator, HasFeaturesCol, HasLabelCol, HasPredictionCol, HasBatchSize, HasMaxEpoch, HasLearningRate, JavaValue): + """ + .. note:: Deprecated in 0.5.0. `DLEstimator` has been migrated to package + `bigdl.dlframes`. This will be removed in BigDL 0.6. + """ def __init__(self, model, criterion, feature_size, label_size, jvalue=None, bigdl_type="float"): super(DLEstimator, self).__init__() self.value = jvalue if jvalue else callBigDlFunc( @@ -111,6 +115,11 @@ def _fit(self, dataset): class DLModel(Model, HasFeaturesCol, HasPredictionCol, HasBatchSize, HasFeatureSize, JavaValue): + """ + .. note:: Deprecated in 0.5.0. `DLModel` has been migrated to package + `bigdl.dlframes`. This will be removed in BigDL 0.6. + + """ def __init__(self, model, featureSize, jvalue=None, bigdl_type="float"): super(DLModel, self).__init__() self.value = jvalue if jvalue else callBigDlFunc( @@ -128,6 +137,11 @@ def of(self, jvalue, feature_size=None, bigdl_type="float"): class DLClassifier(DLEstimator): + """ + .. note:: Deprecated in 0.5.0. `DLClassifier` has been migrated to package + `bigdl.dlframes`. This will be removed in BigDL 0.6. + + """ def __init__(self, model, criterion, feature_size, bigdl_type="float"): super(DLClassifier, self).__init__(model, criterion, feature_size, [1], None, bigdl_type) @@ -138,6 +152,11 @@ def _fit(self, dataset): class DLClassifierModel(DLModel): + """ + .. note:: Deprecated in 0.5.0. `DLClassifierModel` has been migrated to package + `bigdl.dlframes`. This will be removed in BigDL 0.6. + + """ def __init__(self, model, featureSize, jvalue=None, bigdl_type="float"): super(DLClassifierModel, self).__init__(model, featureSize, jvalue, bigdl_type) From f249d46e431781c3460391f9ab5d4584410db415 Mon Sep 17 00:00:00 2001 From: Yuhao Yang Date: Thu, 8 Feb 2018 13:21:05 -0800 Subject: [PATCH 564/823] move DF API to dlframe (#2269) * move DF API to dlframe * remove extra lines * deprecate * extra line --- dlframes/__init__.py | 0 dlframes/dl_classifier.py | 150 ++++++++++++++++++ .../dllib/models/ml_pipeline/dl_classifier.py | 19 +++ 3 files changed, 169 insertions(+) create mode 100644 dlframes/__init__.py create mode 100644 dlframes/dl_classifier.py diff --git a/dlframes/__init__.py b/dlframes/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/dlframes/dl_classifier.py b/dlframes/dl_classifier.py new file mode 100644 index 00000000000..34cc298b641 --- /dev/null +++ b/dlframes/dl_classifier.py @@ -0,0 +1,150 @@ +from pyspark.ml.pipeline import Estimator, Model +from pyspark.ml.param.shared import * +from bigdl.util.common import * + + +if sys.version >= '3': + long = int + unicode = str + + +class HasBatchSize(Params): + """ + Mixin for param batchSize: batch size. + """ + + # a placeholder to make it appear in the generated doc + batchSize = Param(Params._dummy(), "batchSize", "batchSize") + + def __init__(self): + super(HasBatchSize, self).__init__() + #: param for batch size. + self.batchSize = Param(self, "batchSize", "batchSize") + self._setDefault(batchSize=1) + + def setBatchSize(self, val): + """ + Sets the value of :py:attr:`batchSize`. + """ + self._paramMap[self.batchSize] = val + pythonBigDL_method_name = "setBatchSize" + self.__class__.__name__ + callBigDlFunc(self.bigdl_type, pythonBigDL_method_name, self.value, val) + return self + + def getBatchSize(self): + """ + Gets the value of batchSize or its default value. + """ + return self.getOrDefault(self.batchSize) + +class HasMaxEpoch(Params): + maxEpoch = Param(Params._dummy(), "maxEpoch", "number of max Epoch") + + def __init__(self): + super(HasMaxEpoch, self).__init__() + self.maxEpoch = Param(self, "maxEpoch", "maxEpoch") + self._setDefault(maxEpoch=100) + + def setMaxEpoch(self, val): + self._paramMap[self.maxEpoch] = val + pythonBigDL_method_name = "setMaxEpoch" + self.__class__.__name__ + callBigDlFunc(self.bigdl_type, pythonBigDL_method_name, self.value, val) + return self + + def getMaxEpoch(self): + """ + Gets the value of maxEpoch or its default value. + """ + return self.getOrDefault(self.maxEpoch) + +class HasFeatureSize(Params): + featureSize = Param(Params._dummy(), "featureSize", "size of the feature") + + def __init__(self): + super(HasFeatureSize, self).__init__() + self.featureSize = Param(self, "featureSize", "featureSize") + self._setDefault(featureSize=None) + + def setFeatureSize(self, val): + self._paramMap[self.featureSize] = val + pythonBigDL_mehtod_name = "setFeatureSize" + self.__class__.__name__ + callBigDlFunc(self.bigdl_type, pythonBigDL_mehtod_name, self.value, val) + return self + + def getFeatureSize(self): + return self.getOrDefault(self.featureSize) + +class HasLearningRate(Params): + learningRate = Param(Params._dummy(), "learningRate", "learning rate") + + def __init__(self): + super(HasLearningRate, self).__init__() + self.learningRate = Param(self, "learningRate", "learning rate") + self._setDefault(learningRate=100) + + def setLearningRate(self, val): + self._paramMap[self.learningRate] = val + pythonBigDL_method_name = "setLearningRate" + self.__class__.__name__ + callBigDlFunc(self.bigdl_type, pythonBigDL_method_name, self.value, val) + return self + + def getLearningRate(self): + """ + Gets the value of maxEpoch or its default value. + """ + return self.getOrDefault(self.learningRate) + +class DLEstimator(Estimator, HasFeaturesCol, HasLabelCol, HasPredictionCol, HasBatchSize, HasMaxEpoch, HasLearningRate, JavaValue): + + def __init__(self, model, criterion, feature_size, label_size, jvalue=None, bigdl_type="float"): + super(DLEstimator, self).__init__() + self.value = jvalue if jvalue else callBigDlFunc( + bigdl_type, self.jvm_class_constructor(), model, criterion, feature_size, label_size) + self.bigdl_type = bigdl_type + self.featureSize = feature_size + + def _fit(self, dataset): + #self._transfer_params_to_java() + jmodel = callBigDlFunc(self.bigdl_type, "fitEstimator", self.value, dataset) + model = DLModel.of(jmodel, self.featureSize, self.bigdl_type) + return model + + +class DLModel(Model, HasFeaturesCol, HasPredictionCol, HasBatchSize, HasFeatureSize, JavaValue): + def __init__(self, model, featureSize, jvalue=None, bigdl_type="float"): + super(DLModel, self).__init__() + self.value = jvalue if jvalue else callBigDlFunc( + bigdl_type, self.jvm_class_constructor(), model, featureSize) + self.bigdl_type = bigdl_type + self.setFeatureSize(featureSize) + + def _transform(self, dataset): + return callBigDlFunc(self.bigdl_type, "dlModelTransform", self.value, dataset) + + @classmethod + def of(self, jvalue, feature_size=None, bigdl_type="float"): + model = DLModel(model=None, featureSize=feature_size, jvalue=jvalue, bigdl_type=bigdl_type) + return model + + +class DLClassifier(DLEstimator): + def __init__(self, model, criterion, feature_size, bigdl_type="float"): + super(DLClassifier, self).__init__(model, criterion, feature_size, [1], None, bigdl_type) + + def _fit(self, dataset): + jmodel = callBigDlFunc(self.bigdl_type, "fitClassifier", self.value, dataset) + model = DLClassifierModel.of(jmodel, self.featureSize, self.bigdl_type) + return model + + +class DLClassifierModel(DLModel): + def __init__(self, model, featureSize, jvalue=None, bigdl_type="float"): + super(DLClassifierModel, self).__init__(model, featureSize, jvalue, bigdl_type) + + def _transform(self, dataset): + return callBigDlFunc(self.bigdl_type, "dlClassifierModelTransform", self.value, dataset) + + @classmethod + def of(self, jvalue, feature_size=None, bigdl_type="float"): + model = DLClassifierModel(model=None, featureSize=feature_size, jvalue=jvalue, bigdl_type=bigdl_type) + return model diff --git a/python/dllib/src/bigdl/dllib/models/ml_pipeline/dl_classifier.py b/python/dllib/src/bigdl/dllib/models/ml_pipeline/dl_classifier.py index 4dc4cd96248..dd0833fa6f6 100644 --- a/python/dllib/src/bigdl/dllib/models/ml_pipeline/dl_classifier.py +++ b/python/dllib/src/bigdl/dllib/models/ml_pipeline/dl_classifier.py @@ -95,7 +95,11 @@ def getLearningRate(self): return self.getOrDefault(self.learningRate) class DLEstimator(Estimator, HasFeaturesCol, HasLabelCol, HasPredictionCol, HasBatchSize, HasMaxEpoch, HasLearningRate, JavaValue): + """ + .. note:: Deprecated in 0.5.0. `DLEstimator` has been migrated to package + `bigdl.dlframes`. This will be removed in BigDL 0.6. + """ def __init__(self, model, criterion, feature_size, label_size, jvalue=None, bigdl_type="float"): super(DLEstimator, self).__init__() self.value = jvalue if jvalue else callBigDlFunc( @@ -111,6 +115,11 @@ def _fit(self, dataset): class DLModel(Model, HasFeaturesCol, HasPredictionCol, HasBatchSize, HasFeatureSize, JavaValue): + """ + .. note:: Deprecated in 0.5.0. `DLModel` has been migrated to package + `bigdl.dlframes`. This will be removed in BigDL 0.6. + + """ def __init__(self, model, featureSize, jvalue=None, bigdl_type="float"): super(DLModel, self).__init__() self.value = jvalue if jvalue else callBigDlFunc( @@ -128,6 +137,11 @@ def of(self, jvalue, feature_size=None, bigdl_type="float"): class DLClassifier(DLEstimator): + """ + .. note:: Deprecated in 0.5.0. `DLClassifier` has been migrated to package + `bigdl.dlframes`. This will be removed in BigDL 0.6. + + """ def __init__(self, model, criterion, feature_size, bigdl_type="float"): super(DLClassifier, self).__init__(model, criterion, feature_size, [1], None, bigdl_type) @@ -138,6 +152,11 @@ def _fit(self, dataset): class DLClassifierModel(DLModel): + """ + .. note:: Deprecated in 0.5.0. `DLClassifierModel` has been migrated to package + `bigdl.dlframes`. This will be removed in BigDL 0.6. + + """ def __init__(self, model, featureSize, jvalue=None, bigdl_type="float"): super(DLClassifierModel, self).__init__(model, featureSize, jvalue, bigdl_type) From e52a5cc628e7cd22dfc33b1dda7890c54b5235a2 Mon Sep 17 00:00:00 2001 From: Yuhao Yang Date: Thu, 8 Feb 2018 13:21:05 -0800 Subject: [PATCH 565/823] move DF API to dlframe (#2269) * move DF API to dlframe * remove extra lines * deprecate * extra line --- .../test/bigdl/dlframes/test_dl_classifier.py | 173 ++++++++++++++++++ 1 file changed, 173 insertions(+) create mode 100644 python/dllib/test/bigdl/dlframes/test_dl_classifier.py diff --git a/python/dllib/test/bigdl/dlframes/test_dl_classifier.py b/python/dllib/test/bigdl/dlframes/test_dl_classifier.py new file mode 100644 index 00000000000..4325abe63e8 --- /dev/null +++ b/python/dllib/test/bigdl/dlframes/test_dl_classifier.py @@ -0,0 +1,173 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from bigdl.nn.layer import * +from bigdl.nn.criterion import * +from bigdl.util.common import * +from bigdl.dlframes.dl_classifier import * +from pyspark.sql.types import * +from pyspark.context import SparkContext +import numpy as np +import pytest +from numpy.testing import assert_allclose + + +class TestDLClassifer(): + def setup_method(self, method): + """ setup any state tied to the execution of the given method in a + class. setup_method is invoked for every test method of a class. + """ + sparkConf = create_spark_conf().setMaster("local[1]").setAppName("test model") + self.sc = get_spark_context(sparkConf) + self.sqlContext = SQLContext(self.sc) + init_engine() + + def teardown_method(self, method): + """ teardown any state that was previously setup with a setup_method + call. + """ + self.sc.stop() + + def test_all_set_get_methods(self): + """ run tests for all the set and get methods involved in DLEstimator, DLModel, + DLClassifier, DLClassifierModel + """ + + ''' + use linear model and MSE criterion to test the DLEstimator and DLModel + ''' + linear_model = Sequential().add(Linear(2, 2)) + mse_criterion = MSECriterion() + + ''' + initialize a DLEstimator to test setter, getter for + batchSize, maxEpoch, learningRate in DLEstimator + ''' + estimator = DLEstimator(model=linear_model, criterion=mse_criterion, + feature_size=[2], label_size=[2]) + + # check the set and get methods for batchSize + assert estimator.setBatchSize(30).getBatchSize() == 30 + # check the set and get methods for maxEpoch + assert estimator.setMaxEpoch(40).getMaxEpoch() == 40 + # check the set and get methods for learningRate + assert estimator.setLearningRate(1e-4).getLearningRate() == 1e-4 + + ''' + initialize a DLModel to test setter, getter for featureSize, batchSize in DLModel + ''' + dl_model = DLClassifierModel(model=linear_model, featureSize=[1]) + + # check the set and get methods for featureSize in DLModel + assert dl_model.setFeatureSize([2]).getFeatureSize() == [2] + # check the set and get methods for batchSize in DLModel + assert dl_model.setBatchSize((20)).getBatchSize() == 20 + + ''' + use linear model and ClassNLL criterion to test the DLClassifier and DLClassifierModel + ''' + linear_model = Sequential().add(Linear(2, 2)) + classNLL_criterion = ClassNLLCriterion() + + ''' + initialize a DLClassifier to test setter, getter for + batchSize, maxEpoch, learningRate in DLClassifier + ''' + classifier = DLClassifier(model=linear_model, criterion=classNLL_criterion, + feature_size=[2]) + + # check the set and get methods for batchSize + assert classifier.setBatchSize(20).getBatchSize() == 20 + # check the set and get methods for maxEpoch + assert classifier.setMaxEpoch(50).getMaxEpoch() == 50 + # check the set and get methods for learningRate + assert classifier.setLearningRate(1e-5).getLearningRate() == 1e-5 + ''' + initialize a DLClassifierModel to test setter, getter + for featureSize, batchSize in DLClassifierModel + ''' + dl_classifier_model = DLClassifierModel(model=linear_model, featureSize=[1]) + + # check the set and get methods for featureSize + assert dl_classifier_model.setFeatureSize([2]).getFeatureSize() == [2] + + # check the set and get methods for batchSize + assert dl_classifier_model.setBatchSize((20)).getBatchSize() == 20 + + def test_dlestimator_fit_dlmodel_transform(self): + model = Sequential().add(Linear(2, 2)) + criterion = MSECriterion() + estimator = DLEstimator(model, criterion, [2], [2]).setBatchSize(4)\ + .setLearningRate(0.01).setMaxEpoch(1000) + + data = self.sc.parallelize([ + ((2.0, 1.0), (1.0, 2.0)), + ((1.0, 2.0), (2.0, 1.0)), + ((2.0, 1.0), (1.0, 2.0)), + ((1.0, 2.0), (2.0, 1.0))]) + + schema = StructType([ + StructField("features", ArrayType(DoubleType(), False), False), + StructField("label", ArrayType(DoubleType(), False), False)]) + df = self.sqlContext.createDataFrame(data, schema) + dlModel = estimator.fit(df) + + res = dlModel.transform(df) + assert type(res).__name__ == 'DataFrame' + res.registerTempTable("dlModelDF") # Compatible with spark 1.6 + results = self.sqlContext.table("dlModelDF") + + count = results.rdd.count() + data = results.rdd.collect() + + for i in range(count): + row_label = data[i][1] + row_prediction = data[i][2] + assert_allclose(row_label[0], row_prediction[0], atol=0, rtol=1e-1) + assert_allclose(row_label[1], row_prediction[1], atol=0, rtol=1e-1) + + def test_dlclassifier_fit_dlclassifiermodel_transform(self): + model = Sequential().add(Linear(2, 2)) + criterion = ClassNLLCriterion() + classifier = DLClassifier(model, criterion, [2]).setBatchSize(4)\ + .setLearningRate(0.01).setMaxEpoch(1000) + data = self.sc.parallelize([ + ((2.0, 1.0), 1.0), + ((1.0, 2.0), 2.0), + ((2.0, 1.0), 1.0), + ((1.0, 2.0), 2.0)]) + + schema = StructType([ + StructField("features", ArrayType(DoubleType(), False), False), + StructField("label", DoubleType(), False)]) + df = self.sqlContext.createDataFrame(data, schema) + dlClassifierModel = classifier.fit(df) + + res = dlClassifierModel.transform(df) + assert type(res).__name__ == 'DataFrame' + res.registerTempTable("dlClassifierModelDF") + results = self.sqlContext.table("dlClassifierModelDF") + + count = results.rdd.count() + data = results.rdd.collect() + + for i in range(count): + row_label = data[i][1] + row_prediction = data[i][2] + assert row_label == row_prediction + +if __name__ == "__main__": + pytest.main() From 12f1787f279881246ca8f84a44658791c5488b9f Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Fri, 9 Feb 2018 10:22:57 +0800 Subject: [PATCH 566/823] Support Calling Java Function in Python Executor and ModelBroadcast in Python (#2284) * support call java function in executor * fix test * fix style * address comments * add parallelism * address comments * fix partition num * address comments * fix typo * fix typo * add integration test * address comments --- .../src/bigdl/dllib/models/utils/__init__.py | 0 .../dllib/models/utils/model_broadcast.py | 72 ++++++++++++++++++ python/dllib/src/bigdl/dllib/nn/layer.py | 18 ++--- .../dllib/src/bigdl/dllib/optim/optimizer.py | 2 +- python/dllib/src/bigdl/dllib/utils/common.py | 76 +++++++++++++------ 5 files changed, 133 insertions(+), 35 deletions(-) create mode 100644 python/dllib/src/bigdl/dllib/models/utils/__init__.py create mode 100644 python/dllib/src/bigdl/dllib/models/utils/model_broadcast.py diff --git a/python/dllib/src/bigdl/dllib/models/utils/__init__.py b/python/dllib/src/bigdl/dllib/models/utils/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/python/dllib/src/bigdl/dllib/models/utils/model_broadcast.py b/python/dllib/src/bigdl/dllib/models/utils/model_broadcast.py new file mode 100644 index 00000000000..4b88ae1a3fb --- /dev/null +++ b/python/dllib/src/bigdl/dllib/models/utils/model_broadcast.py @@ -0,0 +1,72 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os +import sys +import gc +from tempfile import NamedTemporaryFile + +from pyspark.cloudpickle import print_exec +from pyspark.broadcast import Broadcast +from pyspark.broadcast import _from_id +from bigdl.nn.layer import Model + +def _from_id_and_type(bid, bigdl_type): + result = _from_id(bid) + return ModelBroadcast(path=result._path, bigdl_type=bigdl_type) + +def broadcastModel(sc, layer): + return ModelBroadcast(sc, layer, sc._pickled_broadcast_vars) + +class ModelBroadcast(Broadcast): + + def __init__(self, sc=None, layer=None, pickle_registry=None, path=None, bigdl_type="float"): + """ + Should not be called directly by users -- use L{SparkContext.broadcast()} + instead. + """ + if layer is not None: + self.bigdl_type = layer.bigdl_type + else: + self.bigdl_type = bigdl_type + super(ModelBroadcast, self).__init__(sc, layer, pickle_registry, path) + + def dump(self, value, f): + try: + value.saveModel(f.name, over_write=True) + except Exception as e: + msg = "Could not serialize broadcast: %s" % e.__class__.__name__ + print_exec(sys.stderr) + raise ValueError(msg) + f.close() + return f.name + + def _load(self, path): + return Model.loadModel(path, bigdl_type=self.bigdl_type) + + @property + def value(self): + """ Return the broadcasted value + """ + if not hasattr(self, "_value") and self._path is not None: + self._value = self._load(self._path) + return self._value + + def __reduce__(self): + if self._jbroadcast is None: + raise Exception("Broadcast can only be serialized in driver") + self._pickle_registry.add(self) + return _from_id_and_type, (self._jbroadcast.id(), self.bigdl_type) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 7f3f6d55dd3..856dca0aee0 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -54,10 +54,10 @@ def element(self): return Layer.of(self.value.element()) def remove_pre_edges(self): - callJavaFunc(get_spark_context(), self.value.removePreEdges) + callJavaFunc(self.value.removePreEdges) def remove_next_edges(self): - callJavaFunc(get_spark_context(), self.value.removeNextEdges) + callJavaFunc(self.value.removeNextEdges) @@ -130,14 +130,14 @@ def set_name(self, name): Give this model a name. There would be a generated name consist of class name and UUID if user doesn't set it. """ - callJavaFunc(get_spark_context(), self.value.setName, name) + callJavaFunc(self.value.setName, name) return self def name(self): """ Name of this layer """ - return callJavaFunc(get_spark_context(), self.value.getName) + return callJavaFunc(self.value.getName) def set_seed(self, seed=123): """ @@ -230,7 +230,7 @@ def zero_grad_parameters(self): If the module has parameters, this will zero the accumulation of the gradients with respect to these parameters. Otherwise, it does nothing. """ - callJavaFunc(get_spark_context(), self.value.zeroGradParameters) + callJavaFunc(self.value.zeroGradParameters) def update_parameters(self, learning_rate): """ @@ -245,7 +245,7 @@ def reset(self): """ Initialize the model weights. """ - callJavaFunc(get_spark_context(), self.value.reset) + callJavaFunc(self.value.reset) return self def parameters(self): @@ -528,9 +528,9 @@ def training(self, is_training=True): Set this layer in the training mode or in predition mode if is_training=False ''' if is_training: - callJavaFunc(get_spark_context(), self.value.training) + callJavaFunc(self.value.training) else: - callJavaFunc(get_spark_context(), self.value.evaluate) + callJavaFunc(self.value.evaluate) return self def is_training(self): @@ -546,7 +546,7 @@ def is_training(self): >>> layer.is_training() True ''' - return callJavaFunc(get_spark_context(), self.value.isTraining) + return callJavaFunc(self.value.isTraining) def quantize(self): ''' diff --git a/python/dllib/src/bigdl/dllib/optim/optimizer.py b/python/dllib/src/bigdl/dllib/optim/optimizer.py index 6ebf76c5a05..0e966264811 100644 --- a/python/dllib/src/bigdl/dllib/optim/optimizer.py +++ b/python/dllib/src/bigdl/dllib/optim/optimizer.py @@ -628,7 +628,7 @@ def optimize(self): """ Do an optimization. """ - jmodel = callJavaFunc(get_spark_context(), self.value.optimize) + jmodel = callJavaFunc(self.value.optimize) from bigdl.nn.layer import Layer return Layer.of(jmodel) diff --git a/python/dllib/src/bigdl/dllib/utils/common.py b/python/dllib/src/bigdl/dllib/utils/common.py index 3c19256864c..6097e525cf3 100644 --- a/python/dllib/src/bigdl/dllib/utils/common.py +++ b/python/dllib/src/bigdl/dllib/utils/common.py @@ -16,16 +16,17 @@ import os import sys -import glob from py4j.protocol import Py4JJavaError from py4j.java_gateway import JavaObject from py4j.java_collections import ListConverter, JavaArray, JavaList, JavaMap, MapConverter +from py4j.java_gateway import JavaGateway, GatewayClient from pyspark import RDD, SparkContext from pyspark.serializers import PickleSerializer, AutoBatchedSerializer from pyspark.sql import DataFrame, SQLContext from pyspark.mllib.common import callJavaFunc from pyspark import SparkConf +from pyspark.files import SparkFiles import numpy as np import threading import tempfile @@ -46,13 +47,18 @@ class SingletonMixin(object): @classmethod def instance(cls, - bigdl_type="float"): + bigdl_type, *args): if not cls._instance: with cls._lock: if not cls._instance: - cls._instance = cls(bigdl_type) + cls._instance = cls(bigdl_type, *args) return cls._instance +class GatewayWrapper(SingletonMixin): + + def __init__(self, bigdl_type, port=25333): + self.value = JavaGateway(GatewayClient(port=port), auto_convert=True) + class JavaCreator(SingletonMixin): __creator_class=["com.intel.analytics.bigdl.python.api.PythonBigDLKeras"] @@ -74,11 +80,10 @@ def set_creator_class(cls, cclass): JavaCreator.__creator_class = cclass JavaCreator._instance = None - def __init__(self, bigdl_type): - sc = get_spark_context() + def __init__(self, bigdl_type, gateway): self.value = [] for creator_class in JavaCreator.get_creator_class(): - jclass = getattr(sc._jvm, creator_class) + jclass = getattr(gateway.jvm, creator_class) if bigdl_type == "float": self.value.append(getattr(jclass, "ofFloat")()) elif bigdl_type == "double": @@ -437,6 +442,9 @@ def uniform(self, a, b, size): def init_engine(bigdl_type="float"): callBigDlFunc(bigdl_type, "initEngine") +def init_executor_gateway(sc, bigdl_type="float"): + callBigDlFunc(bigdl_type, "initExecutorGateway", sc, sc._gateway._gateway_client.port) + def redire_spark_logs(bigdl_type="float", log_path=os.getcwd()+"/bigdl.log"): """ @@ -556,16 +564,33 @@ def get_spark_sql_context(sc): else: return SQLContext(sc) # Compatible with Spark1.5.1 +def _get_port(): + root_dir = SparkFiles.getRootDirectory() + path = os.path.join(root_dir, "gateway_port") + f = open(path) + port = int(f.readline()) + return port + +def _get_gateway(): + if SparkFiles._is_running_on_worker: + gateway_port = _get_port() + gateway = GatewayWrapper.instance(None, gateway_port).value + else: + sc = get_spark_context() + gateway = sc._gateway + return gateway + + def callBigDlFunc(bigdl_type, name, *args): """ Call API in PythonBigDL """ - sc = get_spark_context() + gateway = _get_gateway() error = Exception("Cannot find function: %s" % name) - for jinvoker in JavaCreator.instance(bigdl_type=bigdl_type).value: + for jinvoker in JavaCreator.instance(bigdl_type, gateway).value: # hasattr(jinvoker, name) always return true here, # so you need to invoke the method to check if it exist or not try: api = getattr(jinvoker, name) - result = callJavaFunc(sc, api, *args) + result = callJavaFunc(api, *args) except Exception as e: error = e if "does not exist" not in str(e): @@ -575,7 +600,7 @@ def callBigDlFunc(bigdl_type, name, *args): raise error -def _java2py(sc, r, encoding="bytes"): +def _java2py(gateway, r, encoding="bytes"): if isinstance(r, JavaObject): clsName = r.getClass().getSimpleName() # convert RDD into JavaRDD @@ -584,20 +609,20 @@ def _java2py(sc, r, encoding="bytes"): clsName = 'JavaRDD' if clsName == 'JavaRDD': - jrdd = sc._jvm.SerDe.javaToPython(r) - return RDD(jrdd, sc) + jrdd = gateway.jvm.SerDe.javaToPython(r) + return RDD(jrdd, get_spark_context()) if clsName == 'DataFrame': - return DataFrame(r, get_spark_sql_context(sc)) + return DataFrame(r, get_spark_sql_context(get_spark_context())) if clsName == 'Dataset': - return DataFrame(r, get_spark_sql_context(sc)) + return DataFrame(r, get_spark_sql_context(get_spark_context())) if clsName in _picklable_classes: - r = sc._jvm.org.apache.spark.bigdl.api.python.BigDLSerDe.dumps(r) + r = gateway.jvm.org.apache.spark.bigdl.api.python.BigDLSerDe.dumps(r) elif isinstance(r, (JavaArray, JavaList, JavaMap)): try: - r = sc._jvm.org.apache.spark.bigdl.api.python.BigDLSerDe.dumps( + r = gateway.jvm.org.apache.spark.bigdl.api.python.BigDLSerDe.dumps( r) except Py4JJavaError: pass # not pickable @@ -607,11 +632,12 @@ def _java2py(sc, r, encoding="bytes"): return r -def callJavaFunc(sc, func, *args): +def callJavaFunc(func, *args): """ Call Java Function """ - args = [_py2java(sc, a) for a in args] + gateway = _get_gateway() + args = [_py2java(gateway, a) for a in args] result = func(*args) - return _java2py(sc, result) + return _java2py(gateway, result) def _to_java_object_rdd(rdd): @@ -627,7 +653,7 @@ def _to_java_object_rdd(rdd): rdd._jrdd, True) -def _py2java(sc, obj): +def _py2java(gateway, obj): """ Convert Python object into Java """ if isinstance(obj, RDD): obj = _to_java_object_rdd(obj) @@ -636,13 +662,13 @@ def _py2java(sc, obj): elif isinstance(obj, SparkContext): obj = obj._jsc elif isinstance(obj, (list, tuple)): - obj = ListConverter().convert([_py2java(sc, x) for x in obj], - sc._gateway._gateway_client) + obj = ListConverter().convert([_py2java(gateway, x) for x in obj], + gateway._gateway_client) elif isinstance(obj, dict): result = {} for (key, value) in obj.items(): - result[key] = _py2java(sc, value) - obj = MapConverter().convert(result, sc._gateway._gateway_client) + result[key] = _py2java(gateway, value) + obj = MapConverter().convert(result, gateway._gateway_client) elif isinstance(obj, JavaValue): obj = obj.value elif isinstance(obj, JavaObject): @@ -651,7 +677,7 @@ def _py2java(sc, obj): pass else: data = bytearray(PickleSerializer().dumps(obj)) - obj = sc._jvm.org.apache.spark.bigdl.api.python.BigDLSerDe.loads(data) + obj = gateway.jvm.org.apache.spark.bigdl.api.python.BigDLSerDe.loads(data) return obj From 59ba09fb9d469a5ed7908d8562a701ac9c8f9353 Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Fri, 9 Feb 2018 10:22:57 +0800 Subject: [PATCH 567/823] Support Calling Java Function in Python Executor and ModelBroadcast in Python (#2284) * support call java function in executor * fix test * fix style * address comments * add parallelism * address comments * fix partition num * address comments * fix typo * fix typo * add integration test * address comments --- .../src/bigdl/dllib/models/utils/__init__.py | 0 .../dllib/models/utils/model_broadcast.py | 72 ++++++++++++++++++ python/dllib/src/bigdl/dllib/nn/layer.py | 18 ++--- .../dllib/src/bigdl/dllib/optim/optimizer.py | 2 +- python/dllib/src/bigdl/utils/common.py | 76 +++++++++++++------ 5 files changed, 133 insertions(+), 35 deletions(-) create mode 100644 python/dllib/src/bigdl/dllib/models/utils/__init__.py create mode 100644 python/dllib/src/bigdl/dllib/models/utils/model_broadcast.py diff --git a/python/dllib/src/bigdl/dllib/models/utils/__init__.py b/python/dllib/src/bigdl/dllib/models/utils/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/python/dllib/src/bigdl/dllib/models/utils/model_broadcast.py b/python/dllib/src/bigdl/dllib/models/utils/model_broadcast.py new file mode 100644 index 00000000000..4b88ae1a3fb --- /dev/null +++ b/python/dllib/src/bigdl/dllib/models/utils/model_broadcast.py @@ -0,0 +1,72 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os +import sys +import gc +from tempfile import NamedTemporaryFile + +from pyspark.cloudpickle import print_exec +from pyspark.broadcast import Broadcast +from pyspark.broadcast import _from_id +from bigdl.nn.layer import Model + +def _from_id_and_type(bid, bigdl_type): + result = _from_id(bid) + return ModelBroadcast(path=result._path, bigdl_type=bigdl_type) + +def broadcastModel(sc, layer): + return ModelBroadcast(sc, layer, sc._pickled_broadcast_vars) + +class ModelBroadcast(Broadcast): + + def __init__(self, sc=None, layer=None, pickle_registry=None, path=None, bigdl_type="float"): + """ + Should not be called directly by users -- use L{SparkContext.broadcast()} + instead. + """ + if layer is not None: + self.bigdl_type = layer.bigdl_type + else: + self.bigdl_type = bigdl_type + super(ModelBroadcast, self).__init__(sc, layer, pickle_registry, path) + + def dump(self, value, f): + try: + value.saveModel(f.name, over_write=True) + except Exception as e: + msg = "Could not serialize broadcast: %s" % e.__class__.__name__ + print_exec(sys.stderr) + raise ValueError(msg) + f.close() + return f.name + + def _load(self, path): + return Model.loadModel(path, bigdl_type=self.bigdl_type) + + @property + def value(self): + """ Return the broadcasted value + """ + if not hasattr(self, "_value") and self._path is not None: + self._value = self._load(self._path) + return self._value + + def __reduce__(self): + if self._jbroadcast is None: + raise Exception("Broadcast can only be serialized in driver") + self._pickle_registry.add(self) + return _from_id_and_type, (self._jbroadcast.id(), self.bigdl_type) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 7f3f6d55dd3..856dca0aee0 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -54,10 +54,10 @@ def element(self): return Layer.of(self.value.element()) def remove_pre_edges(self): - callJavaFunc(get_spark_context(), self.value.removePreEdges) + callJavaFunc(self.value.removePreEdges) def remove_next_edges(self): - callJavaFunc(get_spark_context(), self.value.removeNextEdges) + callJavaFunc(self.value.removeNextEdges) @@ -130,14 +130,14 @@ def set_name(self, name): Give this model a name. There would be a generated name consist of class name and UUID if user doesn't set it. """ - callJavaFunc(get_spark_context(), self.value.setName, name) + callJavaFunc(self.value.setName, name) return self def name(self): """ Name of this layer """ - return callJavaFunc(get_spark_context(), self.value.getName) + return callJavaFunc(self.value.getName) def set_seed(self, seed=123): """ @@ -230,7 +230,7 @@ def zero_grad_parameters(self): If the module has parameters, this will zero the accumulation of the gradients with respect to these parameters. Otherwise, it does nothing. """ - callJavaFunc(get_spark_context(), self.value.zeroGradParameters) + callJavaFunc(self.value.zeroGradParameters) def update_parameters(self, learning_rate): """ @@ -245,7 +245,7 @@ def reset(self): """ Initialize the model weights. """ - callJavaFunc(get_spark_context(), self.value.reset) + callJavaFunc(self.value.reset) return self def parameters(self): @@ -528,9 +528,9 @@ def training(self, is_training=True): Set this layer in the training mode or in predition mode if is_training=False ''' if is_training: - callJavaFunc(get_spark_context(), self.value.training) + callJavaFunc(self.value.training) else: - callJavaFunc(get_spark_context(), self.value.evaluate) + callJavaFunc(self.value.evaluate) return self def is_training(self): @@ -546,7 +546,7 @@ def is_training(self): >>> layer.is_training() True ''' - return callJavaFunc(get_spark_context(), self.value.isTraining) + return callJavaFunc(self.value.isTraining) def quantize(self): ''' diff --git a/python/dllib/src/bigdl/dllib/optim/optimizer.py b/python/dllib/src/bigdl/dllib/optim/optimizer.py index 6ebf76c5a05..0e966264811 100644 --- a/python/dllib/src/bigdl/dllib/optim/optimizer.py +++ b/python/dllib/src/bigdl/dllib/optim/optimizer.py @@ -628,7 +628,7 @@ def optimize(self): """ Do an optimization. """ - jmodel = callJavaFunc(get_spark_context(), self.value.optimize) + jmodel = callJavaFunc(self.value.optimize) from bigdl.nn.layer import Layer return Layer.of(jmodel) diff --git a/python/dllib/src/bigdl/utils/common.py b/python/dllib/src/bigdl/utils/common.py index 3c19256864c..6097e525cf3 100644 --- a/python/dllib/src/bigdl/utils/common.py +++ b/python/dllib/src/bigdl/utils/common.py @@ -16,16 +16,17 @@ import os import sys -import glob from py4j.protocol import Py4JJavaError from py4j.java_gateway import JavaObject from py4j.java_collections import ListConverter, JavaArray, JavaList, JavaMap, MapConverter +from py4j.java_gateway import JavaGateway, GatewayClient from pyspark import RDD, SparkContext from pyspark.serializers import PickleSerializer, AutoBatchedSerializer from pyspark.sql import DataFrame, SQLContext from pyspark.mllib.common import callJavaFunc from pyspark import SparkConf +from pyspark.files import SparkFiles import numpy as np import threading import tempfile @@ -46,13 +47,18 @@ class SingletonMixin(object): @classmethod def instance(cls, - bigdl_type="float"): + bigdl_type, *args): if not cls._instance: with cls._lock: if not cls._instance: - cls._instance = cls(bigdl_type) + cls._instance = cls(bigdl_type, *args) return cls._instance +class GatewayWrapper(SingletonMixin): + + def __init__(self, bigdl_type, port=25333): + self.value = JavaGateway(GatewayClient(port=port), auto_convert=True) + class JavaCreator(SingletonMixin): __creator_class=["com.intel.analytics.bigdl.python.api.PythonBigDLKeras"] @@ -74,11 +80,10 @@ def set_creator_class(cls, cclass): JavaCreator.__creator_class = cclass JavaCreator._instance = None - def __init__(self, bigdl_type): - sc = get_spark_context() + def __init__(self, bigdl_type, gateway): self.value = [] for creator_class in JavaCreator.get_creator_class(): - jclass = getattr(sc._jvm, creator_class) + jclass = getattr(gateway.jvm, creator_class) if bigdl_type == "float": self.value.append(getattr(jclass, "ofFloat")()) elif bigdl_type == "double": @@ -437,6 +442,9 @@ def uniform(self, a, b, size): def init_engine(bigdl_type="float"): callBigDlFunc(bigdl_type, "initEngine") +def init_executor_gateway(sc, bigdl_type="float"): + callBigDlFunc(bigdl_type, "initExecutorGateway", sc, sc._gateway._gateway_client.port) + def redire_spark_logs(bigdl_type="float", log_path=os.getcwd()+"/bigdl.log"): """ @@ -556,16 +564,33 @@ def get_spark_sql_context(sc): else: return SQLContext(sc) # Compatible with Spark1.5.1 +def _get_port(): + root_dir = SparkFiles.getRootDirectory() + path = os.path.join(root_dir, "gateway_port") + f = open(path) + port = int(f.readline()) + return port + +def _get_gateway(): + if SparkFiles._is_running_on_worker: + gateway_port = _get_port() + gateway = GatewayWrapper.instance(None, gateway_port).value + else: + sc = get_spark_context() + gateway = sc._gateway + return gateway + + def callBigDlFunc(bigdl_type, name, *args): """ Call API in PythonBigDL """ - sc = get_spark_context() + gateway = _get_gateway() error = Exception("Cannot find function: %s" % name) - for jinvoker in JavaCreator.instance(bigdl_type=bigdl_type).value: + for jinvoker in JavaCreator.instance(bigdl_type, gateway).value: # hasattr(jinvoker, name) always return true here, # so you need to invoke the method to check if it exist or not try: api = getattr(jinvoker, name) - result = callJavaFunc(sc, api, *args) + result = callJavaFunc(api, *args) except Exception as e: error = e if "does not exist" not in str(e): @@ -575,7 +600,7 @@ def callBigDlFunc(bigdl_type, name, *args): raise error -def _java2py(sc, r, encoding="bytes"): +def _java2py(gateway, r, encoding="bytes"): if isinstance(r, JavaObject): clsName = r.getClass().getSimpleName() # convert RDD into JavaRDD @@ -584,20 +609,20 @@ def _java2py(sc, r, encoding="bytes"): clsName = 'JavaRDD' if clsName == 'JavaRDD': - jrdd = sc._jvm.SerDe.javaToPython(r) - return RDD(jrdd, sc) + jrdd = gateway.jvm.SerDe.javaToPython(r) + return RDD(jrdd, get_spark_context()) if clsName == 'DataFrame': - return DataFrame(r, get_spark_sql_context(sc)) + return DataFrame(r, get_spark_sql_context(get_spark_context())) if clsName == 'Dataset': - return DataFrame(r, get_spark_sql_context(sc)) + return DataFrame(r, get_spark_sql_context(get_spark_context())) if clsName in _picklable_classes: - r = sc._jvm.org.apache.spark.bigdl.api.python.BigDLSerDe.dumps(r) + r = gateway.jvm.org.apache.spark.bigdl.api.python.BigDLSerDe.dumps(r) elif isinstance(r, (JavaArray, JavaList, JavaMap)): try: - r = sc._jvm.org.apache.spark.bigdl.api.python.BigDLSerDe.dumps( + r = gateway.jvm.org.apache.spark.bigdl.api.python.BigDLSerDe.dumps( r) except Py4JJavaError: pass # not pickable @@ -607,11 +632,12 @@ def _java2py(sc, r, encoding="bytes"): return r -def callJavaFunc(sc, func, *args): +def callJavaFunc(func, *args): """ Call Java Function """ - args = [_py2java(sc, a) for a in args] + gateway = _get_gateway() + args = [_py2java(gateway, a) for a in args] result = func(*args) - return _java2py(sc, result) + return _java2py(gateway, result) def _to_java_object_rdd(rdd): @@ -627,7 +653,7 @@ def _to_java_object_rdd(rdd): rdd._jrdd, True) -def _py2java(sc, obj): +def _py2java(gateway, obj): """ Convert Python object into Java """ if isinstance(obj, RDD): obj = _to_java_object_rdd(obj) @@ -636,13 +662,13 @@ def _py2java(sc, obj): elif isinstance(obj, SparkContext): obj = obj._jsc elif isinstance(obj, (list, tuple)): - obj = ListConverter().convert([_py2java(sc, x) for x in obj], - sc._gateway._gateway_client) + obj = ListConverter().convert([_py2java(gateway, x) for x in obj], + gateway._gateway_client) elif isinstance(obj, dict): result = {} for (key, value) in obj.items(): - result[key] = _py2java(sc, value) - obj = MapConverter().convert(result, sc._gateway._gateway_client) + result[key] = _py2java(gateway, value) + obj = MapConverter().convert(result, gateway._gateway_client) elif isinstance(obj, JavaValue): obj = obj.value elif isinstance(obj, JavaObject): @@ -651,7 +677,7 @@ def _py2java(sc, obj): pass else: data = bytearray(PickleSerializer().dumps(obj)) - obj = sc._jvm.org.apache.spark.bigdl.api.python.BigDLSerDe.loads(data) + obj = gateway.jvm.org.apache.spark.bigdl.api.python.BigDLSerDe.loads(data) return obj From 2273ab36fcdebcfc369da09f8310cf34f3d20b96 Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Fri, 9 Feb 2018 10:22:57 +0800 Subject: [PATCH 568/823] Support Calling Java Function in Python Executor and ModelBroadcast in Python (#2284) * support call java function in executor * fix test * fix style * address comments * add parallelism * address comments * fix partition num * address comments * fix typo * fix typo * add integration test * address comments --- .../dllib/test/bigdl/test_simple_integration.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/python/dllib/test/bigdl/test_simple_integration.py b/python/dllib/test/bigdl/test_simple_integration.py index 500c8455f9a..651a3ce24ca 100644 --- a/python/dllib/test/bigdl/test_simple_integration.py +++ b/python/dllib/test/bigdl/test_simple_integration.py @@ -28,6 +28,7 @@ from numpy.testing import assert_allclose, assert_array_equal from bigdl.util.engine import compare_version from bigdl.transform.vision.image import * +from bigdl.models.utils.model_broadcast import broadcastModel np.random.seed(1337) # for reproducibility @@ -533,7 +534,7 @@ def test_save_jtensor_dict(self): tensors["tensor1"] = JTensor.from_ndarray(np.random.rand(3, 2)) tensors["tensor2"] = JTensor.from_ndarray(np.random.rand(3, 2)) # in old impl, this will throw an exception - _py2java(self.sc, tensors) + _py2java(self.sc._gateway, tensors) def test_compare_version(self): assert compare_version("2.1.1", "2.2.0") == -1 @@ -601,5 +602,17 @@ def test_local_predict_multiple_input(self): JTensor.from_ndarray(np.ones([4, 3]))]) assert result4.shape == (4,) + def test_model_broadcast(self): + + init_executor_gateway(self.sc) + model = Linear(3, 2) + broadcasted = broadcastModel(self.sc, model) + input_data = np.random.rand(3) + output = self.sc.parallelize([input_data], 1)\ + .map(lambda x: broadcasted.value.forward(x)).first() + expected = model.forward(input_data) + + assert_allclose(output, expected) + if __name__ == "__main__": pytest.main([__file__]) From 5825871e8fa24cf8d121c1f736603563f3af7da5 Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Fri, 9 Feb 2018 18:42:19 +0800 Subject: [PATCH 569/823] Keras-like API Merge and Wrappers (#2263) * merge * update * timedistributed * bidirectional * update * update * update bidirectional * style * update merge * refine and add ser test * refine timedistributed * fix * fix merge * createmerge * python merge ut * meet review * add doc * fix style * update --- .../bigdl/dllib/bigdlkeras/layers/layer.py | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py b/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py index 25b199ae9b5..a50c14c54c1 100644 --- a/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py +++ b/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py @@ -32,11 +32,19 @@ def jvm_class_constructor(self): class Sequential(Container): + """ + >>> sequential = Sequential() + creating: createSequential + """ def __init__(self, bigdl_type="float"): super(Sequential, self).__init__(None, bigdl_type, True) class InputLayer(KerasLayer): + """ + >>> inputLayer = InputLayer(input_shape=(3, 5)) + creating: createKerasInputLayer + """ def __init__(self, input_shape=None, bigdl_type="float"): super(InputLayer, self).__init__(None, bigdl_type, list(input_shape) if input_shape else None) @@ -123,3 +131,21 @@ def get_running_mean(self): def get_running_std(self): return callBigDlFunc(self.bigdl_type, "getKerasRunningStd", self.value).to_ndarray() + + +class Merge(KerasLayer): + """ + >>> l1 = InputLayer(input_shape=(3, 5)) + creating: createKerasInputLayer + >>> l2 = InputLayer(input_shape=(3, 5)) + creating: createKerasInputLayer + >>> merge = Merge(layers=[l1, l2], mode='sum') + creating: createKerasMerge + """ + def __init__(self, layers=None, mode='sum', concat_axis=-1, + input_shape=None, bigdl_type="float"): + super(Merge, self).__init__(None, bigdl_type, + list(layers) if layers else None, + mode, + concat_axis, + input_shape) From cc87f9f3718d2f4bf65c0b9352daecb18fcd5311 Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Fri, 9 Feb 2018 18:42:19 +0800 Subject: [PATCH 570/823] Keras-like API Merge and Wrappers (#2263) * merge * update * timedistributed * bidirectional * update * update * update bidirectional * style * update merge * refine and add ser test * refine timedistributed * fix * fix merge * createmerge * python merge ut * meet review * add doc * fix style * update --- .../bigdl/dllib/bigdlkeras/layers/layer.py | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py b/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py index 25b199ae9b5..a50c14c54c1 100644 --- a/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py +++ b/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py @@ -32,11 +32,19 @@ def jvm_class_constructor(self): class Sequential(Container): + """ + >>> sequential = Sequential() + creating: createSequential + """ def __init__(self, bigdl_type="float"): super(Sequential, self).__init__(None, bigdl_type, True) class InputLayer(KerasLayer): + """ + >>> inputLayer = InputLayer(input_shape=(3, 5)) + creating: createKerasInputLayer + """ def __init__(self, input_shape=None, bigdl_type="float"): super(InputLayer, self).__init__(None, bigdl_type, list(input_shape) if input_shape else None) @@ -123,3 +131,21 @@ def get_running_mean(self): def get_running_std(self): return callBigDlFunc(self.bigdl_type, "getKerasRunningStd", self.value).to_ndarray() + + +class Merge(KerasLayer): + """ + >>> l1 = InputLayer(input_shape=(3, 5)) + creating: createKerasInputLayer + >>> l2 = InputLayer(input_shape=(3, 5)) + creating: createKerasInputLayer + >>> merge = Merge(layers=[l1, l2], mode='sum') + creating: createKerasMerge + """ + def __init__(self, layers=None, mode='sum', concat_axis=-1, + input_shape=None, bigdl_type="float"): + super(Merge, self).__init__(None, bigdl_type, + list(layers) if layers else None, + mode, + concat_axis, + input_shape) From 92018d1afa7d367e40314332daa0525dccd3fa60 Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Fri, 9 Feb 2018 18:42:19 +0800 Subject: [PATCH 571/823] Keras-like API Merge and Wrappers (#2263) * merge * update * timedistributed * bidirectional * update * update * update bidirectional * style * update merge * refine and add ser test * refine timedistributed * fix * fix merge * createmerge * python merge ut * meet review * add doc * fix style * update --- python/dllib/test/bigdl/keras/test_newapi.py | 70 ++++++++++++++++++++ python/dllib/test/bigdl/test_utils.py | 2 +- 2 files changed, 71 insertions(+), 1 deletion(-) diff --git a/python/dllib/test/bigdl/keras/test_newapi.py b/python/dllib/test/bigdl/keras/test_newapi.py index 22dc5b6e2d8..f5c41164d99 100644 --- a/python/dllib/test/bigdl/keras/test_newapi.py +++ b/python/dllib/test/bigdl/keras/test_newapi.py @@ -49,6 +49,76 @@ def test_batchnormalization(self): self.compare_newapi(klayer, blayer, input_data2, WeightsConverter.convert_batchnormalization) + def test_merge_sum(self): + b1 = BLayer.InputLayer(input_shape=(3, 5)) + b2 = BLayer.InputLayer(input_shape=(3, 5)) + blayer = BLayer.Merge(layers=[b1, b2], mode="sum") + k1 = KLayer.InputLayer(input_shape=(3, 5)) + k2 = KLayer.InputLayer(input_shape=(3, 5)) + klayer = KLayer.Merge(layers=[k1, k2], mode="sum") + input_data = [np.random.random([2, 3, 5]), np.random.random([2, 3, 5])] + self.compare_newapi(klayer, blayer, input_data) + + def test_merge_mul(self): + b1 = BLayer.InputLayer(input_shape=(3, 5)) + b2 = BLayer.InputLayer(input_shape=(3, 5)) + blayer = BLayer.Merge(layers=[b1, b2], mode="mul") + k1 = KLayer.InputLayer(input_shape=(3, 5)) + k2 = KLayer.InputLayer(input_shape=(3, 5)) + klayer = KLayer.Merge(layers=[k1, k2], mode="mul") + input_data = [np.random.random([2, 3, 5]), np.random.random([2, 3, 5])] + self.compare_newapi(klayer, blayer, input_data) + + def test_merge_ave(self): + b1 = BLayer.InputLayer(input_shape=(2, 5, 8)) + b2 = BLayer.InputLayer(input_shape=(2, 5, 8)) + blayer = BLayer.Merge(layers=[b1, b2], mode="ave") + k1 = KLayer.InputLayer(input_shape=(2, 5, 8)) + k2 = KLayer.InputLayer(input_shape=(2, 5, 8)) + klayer = KLayer.Merge(layers=[k1, k2], mode="ave") + input_data = [np.random.random([3, 2, 5, 8]), np.random.random([3, 2, 5, 8])] + self.compare_newapi(klayer, blayer, input_data) + + def test_merge_max(self): + b1 = BLayer.InputLayer(input_shape=(2, 5, 8)) + b2 = BLayer.InputLayer(input_shape=(2, 5, 8)) + blayer = BLayer.Merge(layers=[b1, b2], mode="max") + k1 = KLayer.InputLayer(input_shape=(2, 5, 8)) + k2 = KLayer.InputLayer(input_shape=(2, 5, 8)) + klayer = KLayer.Merge(layers=[k1, k2], mode="max") + input_data = [np.random.random([3, 2, 5, 8]), np.random.random([3, 2, 5, 8])] + self.compare_newapi(klayer, blayer, input_data) + + def test_merge_concat(self): + b1 = BLayer.InputLayer(input_shape=(2, 5, 11)) + b2 = BLayer.InputLayer(input_shape=(2, 5, 8)) + blayer = BLayer.Merge(layers=[b1, b2], mode="concat") + k1 = KLayer.InputLayer(input_shape=(2, 5, 11)) + k2 = KLayer.InputLayer(input_shape=(2, 5, 8)) + klayer = KLayer.Merge(layers=[k1, k2], mode="concat") + input_data = [np.random.random([3, 2, 5, 11]), np.random.random([3, 2, 5, 8])] + self.compare_newapi(klayer, blayer, input_data) + + def test_merge_dot(self): + b1 = BLayer.InputLayer(input_shape=(4, )) + b2 = BLayer.InputLayer(input_shape=(4, )) + blayer = BLayer.Merge(layers=[b1, b2], mode="dot") + k1 = KLayer.InputLayer(input_shape=(4, )) + k2 = KLayer.InputLayer(input_shape=(4, )) + klayer = KLayer.Merge(layers=[k1, k2], mode="dot") + input_data = [np.random.random([2, 4]), np.random.random([2, 4])] + self.compare_newapi(klayer, blayer, input_data) + + def test_merge_cos(self): + b1 = BLayer.InputLayer(input_shape=(3, )) + b2 = BLayer.InputLayer(input_shape=(3, )) + blayer = BLayer.Merge(layers=[b1, b2], mode="cos") + k1 = KLayer.InputLayer(input_shape=(3, )) + k2 = KLayer.InputLayer(input_shape=(3, )) + klayer = KLayer.Merge(layers=[k1, k2], mode="cos") + input_data = [np.random.random([2, 3]), np.random.random([2, 3])] + self.compare_newapi(klayer, blayer, input_data) + if __name__ == "__main__": pytest.main([__file__]) diff --git a/python/dllib/test/bigdl/test_utils.py b/python/dllib/test/bigdl/test_utils.py index 2ef0b91f4ec..04f9869aa08 100644 --- a/python/dllib/test/bigdl/test_utils.py +++ b/python/dllib/test/bigdl/test_utils.py @@ -320,7 +320,7 @@ def compare_loss(self, y_a, y_b, kloss, bloss, rtol=1e-6, atol=1e-6): np.testing.assert_allclose(bigdl_output, keras_output, rtol=rtol, atol=atol) # Compare forward results with Keras for new Keras-like API layers. - def compare_newapi(self, klayer, blayer, input_data, weight_converter, + def compare_newapi(self, klayer, blayer, input_data, weight_converter=None, is_training=False, rtol=1e-6, atol=1e-6): from keras.models import Sequential as KSequential from bigdl.nn.keras.layer import Sequential as BSequential From 0d21d83d310b256162e8d5a0c7435f011dfda625 Mon Sep 17 00:00:00 2001 From: Xianyan Date: Mon, 12 Feb 2018 14:18:09 +0800 Subject: [PATCH 572/823] support imageframe in python optimizer (#2207) * ImageFrame to Python Sample * fix * support imageframe in python optimizer * clean * fix style * update * fix style * fix style * fix test * meet code review * train inception with imageframe interface * return new ImageFeature in PixelBytesToMat * update * some more wrapper * update * update * update * support imageframe in python optimizer * python can train * clean * update * update * revert rename * update API * fix ut --- .../bigdl/dllib/feature/dataset/dataset.py | 42 +++++++++++++++++++ .../dllib/src/bigdl/dllib/optim/optimizer.py | 23 +++++++--- 2 files changed, 59 insertions(+), 6 deletions(-) create mode 100644 python/dllib/src/bigdl/dllib/feature/dataset/dataset.py diff --git a/python/dllib/src/bigdl/dllib/feature/dataset/dataset.py b/python/dllib/src/bigdl/dllib/feature/dataset/dataset.py new file mode 100644 index 00000000000..a6e65a795b4 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/feature/dataset/dataset.py @@ -0,0 +1,42 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import sys +from bigdl.util.common import JavaValue +from bigdl.util.common import callBigDlFunc +from bigdl.util.common import * +from bigdl.transform.vision.image import * + +if sys.version >= '3': + long = int + unicode = str + +class DataSet(JavaValue): + + def __init__(self, jvalue=None, bigdl_type="float"): + self.bigdl_type = bigdl_type + if jvalue: + self.value = jvalue + + @classmethod + def image_frame(cls, image_frame, bigdl_type="float"): + jvalue = callBigDlFunc(bigdl_type, "createDatasetFromImageFrame", image_frame) + return DataSet(jvalue=jvalue) + + def transform(self, transformer): + if isinstance(transformer, FeatureTransformer): + jvalue = callBigDlFunc(self.bigdl_type, "featureTransformDataset", self.value, transformer) + return DataSet(jvalue=jvalue) \ No newline at end of file diff --git a/python/dllib/src/bigdl/dllib/optim/optimizer.py b/python/dllib/src/bigdl/dllib/optim/optimizer.py index 0e966264811..0f26db3165c 100644 --- a/python/dllib/src/bigdl/dllib/optim/optimizer.py +++ b/python/dllib/src/bigdl/dllib/optim/optimizer.py @@ -30,6 +30,7 @@ from bigdl.util.common import callJavaFunc from bigdl.util.common import get_spark_context from bigdl.util.common import to_list +from bigdl.dataset.dataset import * if sys.version >= '3': long = int @@ -340,7 +341,7 @@ class SequentialSchedule(JavaValue): >>> poly = Poly(0.5, 2) creating: createPoly >>> test = sequentialSchedule.add(poly, 5) - + """ @@ -738,7 +739,7 @@ def create(model, end_trigger = MaxEpoch(1) if not optim_method: optim_method = SGD() - if isinstance(training_set, RDD): + if isinstance(training_set, RDD) or isinstance(training_set, DataSet): return DistriOptimizer(model=model, training_rdd=training_set, criterion=criterion, @@ -772,7 +773,10 @@ def set_validation(self, batch_size, val_rdd, trigger, val_method=None): """ if val_method is None: val_method = [Top1Accuracy()] - callBigDlFunc(self.bigdl_type, "setValidation", self.value, batch_size, + func_name = "setValidation" + if isinstance(val_rdd, DataSet): + func_name = "setValidationFromDataSet" + callBigDlFunc(self.bigdl_type, func_name, self.value, batch_size, trigger, val_rdd, to_list(val_method)) def set_traindata(self, training_rdd, batch_size): @@ -809,9 +813,16 @@ def __init__(self, :param end_trigger: when to end the optimization :param batch_size: training batch size """ - JavaValue.__init__(self, None, bigdl_type, model.value, - training_rdd, criterion, - optim_method if optim_method else SGD(), end_trigger, batch_size) + if isinstance(training_rdd, RDD): + JavaValue.__init__(self, None, bigdl_type, model.value, + training_rdd, criterion, + optim_method if optim_method else SGD(), end_trigger, batch_size) + elif isinstance(training_rdd, DataSet): + self.bigdl_type = bigdl_type + self.value = callBigDlFunc(self.bigdl_type, "createDistriOptimizerFromDataSet", + model.value, training_rdd, criterion, + optim_method if optim_method else SGD(), + end_trigger, batch_size) class LocalOptimizer(BaseOptimizer): From 4aa25d123bd37761d10f264d31683f31587f616d Mon Sep 17 00:00:00 2001 From: Xianyan Date: Mon, 12 Feb 2018 14:18:09 +0800 Subject: [PATCH 573/823] support imageframe in python optimizer (#2207) * ImageFrame to Python Sample * fix * support imageframe in python optimizer * clean * fix style * update * fix style * fix style * fix test * meet code review * train inception with imageframe interface * return new ImageFeature in PixelBytesToMat * update * some more wrapper * update * update * update * support imageframe in python optimizer * python can train * clean * update * update * revert rename * update API * fix ut --- .../bigdl/dllib/feature/dataset/dataset.py | 42 +++++++++++++++++++ .../dllib/src/bigdl/dllib/optim/optimizer.py | 23 +++++++--- 2 files changed, 59 insertions(+), 6 deletions(-) create mode 100644 python/dllib/src/bigdl/dllib/feature/dataset/dataset.py diff --git a/python/dllib/src/bigdl/dllib/feature/dataset/dataset.py b/python/dllib/src/bigdl/dllib/feature/dataset/dataset.py new file mode 100644 index 00000000000..a6e65a795b4 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/feature/dataset/dataset.py @@ -0,0 +1,42 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import sys +from bigdl.util.common import JavaValue +from bigdl.util.common import callBigDlFunc +from bigdl.util.common import * +from bigdl.transform.vision.image import * + +if sys.version >= '3': + long = int + unicode = str + +class DataSet(JavaValue): + + def __init__(self, jvalue=None, bigdl_type="float"): + self.bigdl_type = bigdl_type + if jvalue: + self.value = jvalue + + @classmethod + def image_frame(cls, image_frame, bigdl_type="float"): + jvalue = callBigDlFunc(bigdl_type, "createDatasetFromImageFrame", image_frame) + return DataSet(jvalue=jvalue) + + def transform(self, transformer): + if isinstance(transformer, FeatureTransformer): + jvalue = callBigDlFunc(self.bigdl_type, "featureTransformDataset", self.value, transformer) + return DataSet(jvalue=jvalue) \ No newline at end of file diff --git a/python/dllib/src/bigdl/dllib/optim/optimizer.py b/python/dllib/src/bigdl/dllib/optim/optimizer.py index 0e966264811..0f26db3165c 100644 --- a/python/dllib/src/bigdl/dllib/optim/optimizer.py +++ b/python/dllib/src/bigdl/dllib/optim/optimizer.py @@ -30,6 +30,7 @@ from bigdl.util.common import callJavaFunc from bigdl.util.common import get_spark_context from bigdl.util.common import to_list +from bigdl.dataset.dataset import * if sys.version >= '3': long = int @@ -340,7 +341,7 @@ class SequentialSchedule(JavaValue): >>> poly = Poly(0.5, 2) creating: createPoly >>> test = sequentialSchedule.add(poly, 5) - + """ @@ -738,7 +739,7 @@ def create(model, end_trigger = MaxEpoch(1) if not optim_method: optim_method = SGD() - if isinstance(training_set, RDD): + if isinstance(training_set, RDD) or isinstance(training_set, DataSet): return DistriOptimizer(model=model, training_rdd=training_set, criterion=criterion, @@ -772,7 +773,10 @@ def set_validation(self, batch_size, val_rdd, trigger, val_method=None): """ if val_method is None: val_method = [Top1Accuracy()] - callBigDlFunc(self.bigdl_type, "setValidation", self.value, batch_size, + func_name = "setValidation" + if isinstance(val_rdd, DataSet): + func_name = "setValidationFromDataSet" + callBigDlFunc(self.bigdl_type, func_name, self.value, batch_size, trigger, val_rdd, to_list(val_method)) def set_traindata(self, training_rdd, batch_size): @@ -809,9 +813,16 @@ def __init__(self, :param end_trigger: when to end the optimization :param batch_size: training batch size """ - JavaValue.__init__(self, None, bigdl_type, model.value, - training_rdd, criterion, - optim_method if optim_method else SGD(), end_trigger, batch_size) + if isinstance(training_rdd, RDD): + JavaValue.__init__(self, None, bigdl_type, model.value, + training_rdd, criterion, + optim_method if optim_method else SGD(), end_trigger, batch_size) + elif isinstance(training_rdd, DataSet): + self.bigdl_type = bigdl_type + self.value = callBigDlFunc(self.bigdl_type, "createDistriOptimizerFromDataSet", + model.value, training_rdd, criterion, + optim_method if optim_method else SGD(), + end_trigger, batch_size) class LocalOptimizer(BaseOptimizer): From e6c87a148bd0f7278bbf798e62e4297f968b0b89 Mon Sep 17 00:00:00 2001 From: Xianyan Date: Mon, 12 Feb 2018 14:18:09 +0800 Subject: [PATCH 574/823] support imageframe in python optimizer (#2207) * ImageFrame to Python Sample * fix * support imageframe in python optimizer * clean * fix style * update * fix style * fix style * fix test * meet code review * train inception with imageframe interface * return new ImageFeature in PixelBytesToMat * update * some more wrapper * update * update * update * support imageframe in python optimizer * python can train * clean * update * update * revert rename * update API * fix ut --- .../test/bigdl/test_simple_integration.py | 46 +++++++++++++++++++ .../dllib/test/bigdl/transform/test_image.py | 12 ++--- 2 files changed, 52 insertions(+), 6 deletions(-) diff --git a/python/dllib/test/bigdl/test_simple_integration.py b/python/dllib/test/bigdl/test_simple_integration.py index 651a3ce24ca..db27bb23ddc 100644 --- a/python/dllib/test/bigdl/test_simple_integration.py +++ b/python/dllib/test/bigdl/test_simple_integration.py @@ -29,6 +29,7 @@ from bigdl.util.engine import compare_version from bigdl.transform.vision.image import * from bigdl.models.utils.model_broadcast import broadcastModel +from bigdl.dataset.dataset import * np.random.seed(1337) # for reproducibility @@ -614,5 +615,50 @@ def test_model_broadcast(self): assert_allclose(output, expected) + def test_train_DataSet(self): + batch_size = 8 + epoch_num = 5 + images = [] + labels = [] + for i in range(0, 8): + features = np.random.uniform(0, 1, (200, 200, 3)) + label = np.array([2]) + images.append(features) + labels.append(label) + + image_frame = DistributedImageFrame(self.sc.parallelize(images), + self.sc.parallelize(labels)) + + transformer = Pipeline([BytesToMat(), Resize(256, 256), CenterCrop(224, 224), + ChannelNormalize(0.485, 0.456, 0.406, 0.229, 0.224, 0.225), + MatToTensor(), ImageFrameToSample(target_keys=['label'])]) + data_set = DataSet.image_frame(image_frame).transform(transformer) + + model = Sequential() + model.add(SpatialConvolution(3, 1, 5, 5)) + model.add(View([1 * 220 * 220])) + model.add(Linear(1 * 220 * 220, 20)) + model.add(LogSoftMax()) + optim_method = SGD(learningrate=0.01) + optimizer = Optimizer.create( + model=model, + training_set=data_set, + criterion=ClassNLLCriterion(), + optim_method=optim_method, + end_trigger=MaxEpoch(epoch_num), + batch_size=batch_size) + optimizer.set_validation( + batch_size=batch_size, + val_rdd=data_set, + trigger=EveryEpoch(), + val_method=[Top1Accuracy()] + ) + + trained_model = optimizer.optimize() + + predict_result = trained_model.predict_image(image_frame.transform(transformer)) + assert_allclose(predict_result.get_predict().count(), 8) + + if __name__ == "__main__": pytest.main([__file__]) diff --git a/python/dllib/test/bigdl/transform/test_image.py b/python/dllib/test/bigdl/transform/test_image.py index 159526f1963..da5af4e7cf2 100644 --- a/python/dllib/test/bigdl/transform/test_image.py +++ b/python/dllib/test/bigdl/transform/test_image.py @@ -40,12 +40,12 @@ def teardown_method(self, method): def transformer_test(self, transformer): image_frame = ImageFrame.read(self.image_path) - transformer(image_frame) - image_frame.get_image() + transformed = transformer(image_frame) + transformed.get_image() image_frame = ImageFrame.read(self.image_path, self.sc) - transformer(image_frame) - images = image_frame.get_image() + transformed = transformer(image_frame) + images = transformed.get_image() images.count() def test_get_image(self): @@ -179,8 +179,8 @@ def testImageFrameToSample(self): def test_image_frame_transform(self): transformer = MatToTensor() image_frame = ImageFrame.read(self.image_path) - image_frame.transform(transformer) - image_frame.get_image() + transformed = image_frame.transform(transformer) + transformed.get_image() def test_empty_get_predict_local(self): image_frame = ImageFrame.read(self.image_path) From b53b05d48b5d62aa6a07119e5c2cbca8a6047db2 Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Wed, 14 Feb 2018 15:43:30 +0800 Subject: [PATCH 575/823] Keras-like API Python Wrapper for layers + lenet example (#2296) * python lenet * docs * style * wrap getoutputshape * update input output shape * update * python layers * clean * fix ut * more layers * fix ut * more layers * fix ut * more layers * wrapper * remove * style * style and srelu * style * refine lenet * update * add ut for input output shape * fix float * refine lenet * update * meet review * clean * todo * update * style * readme * style * style * fix * style --- .../bigdl/dllib/bigdlkeras/layers/layer.py | 1819 ++++++++++++++++- .../src/bigdl/dllib/examples/lenet/README.md | 75 + .../bigdl/dllib/examples/lenet/__init__.py | 15 + .../src/bigdl/dllib/examples/lenet/lenet.py | 66 + .../src/bigdl/dllib/models/lenet/lenet5.py | 43 +- .../src/bigdl/dllib/models/lenet/utils.py | 75 + .../bigdl/dllib/models/local_lenet/README.md | 2 + .../dllib/models/local_lenet/local_lenet.py | 35 +- python/dllib/src/bigdl/dllib/nn/layer.py | 12 +- 9 files changed, 2057 insertions(+), 85 deletions(-) create mode 100644 python/dllib/src/bigdl/dllib/examples/lenet/README.md create mode 100644 python/dllib/src/bigdl/dllib/examples/lenet/__init__.py create mode 100644 python/dllib/src/bigdl/dllib/examples/lenet/lenet.py create mode 100644 python/dllib/src/bigdl/dllib/models/lenet/utils.py diff --git a/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py b/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py index a50c14c54c1..567006329f9 100644 --- a/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py +++ b/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py @@ -16,14 +16,47 @@ import sys -from bigdl.nn.layer import Layer, Container -from bigdl.util.common import callBigDlFunc, JTensor +from bigdl.nn.layer import Layer, Sequential as TSequential, Model as TModel +from bigdl.util.common import callBigDlFunc, JTensor, JavaValue, to_list if sys.version >= '3': long = int unicode = str +class InferShape(JavaValue): + def __init__(self, bigdl_type="float"): + self.bigdl_type = bigdl_type + + @classmethod + def __to_keras_shape(cls, shape): + return tuple([None] + shape[1:]) + + def __process_shape(self, shape): + if len(shape) == 1: + return self.__to_keras_shape(shape[0]) + else: + return [self.__to_keras_shape(s) for s in shape] + + def get_input_shape(self): + """ + Return a list of shape tuples if there are multiple inputs. + Return one shape tuple otherwise. + """ + input = callBigDlFunc(self.bigdl_type, "getInputShape", + self.value) + return self.__process_shape(input) + + def get_output_shape(self): + """ + Return a list of shape tuples if there are multiple outputs. + Return one shape tuple otherwise. + """ + output = callBigDlFunc(self.bigdl_type, "getOutputShape", + self.value) + return self.__process_shape(output) + + class KerasLayer(Layer): def jvm_class_constructor(self): name = "createKeras" + self.__class__.__name__ @@ -31,18 +64,40 @@ def jvm_class_constructor(self): return name -class Sequential(Container): +class Sequential(TSequential, InferShape): """ + Container for a Sequential model. + >>> sequential = Sequential() creating: createSequential """ def __init__(self, bigdl_type="float"): - super(Sequential, self).__init__(None, bigdl_type, True) + super(Sequential, self).__init__(bigdl_type, True) + + +class Model(TModel, InferShape): + def __init__(self, input, output, bigdl_type="float"): + super(Model, self).__init__(to_list(input), + to_list(output), + is_keras=True, + bigdl_type=bigdl_type) + + +class Input(KerasLayer): + def __init__(self, name=None, input_shape=None, bigdl_type="float"): + super(Input, self).__init__(None, bigdl_type, + name, + list(input_shape) if input_shape else None) class InputLayer(KerasLayer): """ - >>> inputLayer = InputLayer(input_shape=(3, 5)) + Layer to be used as an entry point into a model. + + # Arguments + input_shape: A shape tuple, not including the batch axis. + + >>> inputlayer = InputLayer(input_shape=(3, 5)) creating: createKerasInputLayer """ def __init__(self, input_shape=None, bigdl_type="float"): @@ -53,10 +108,11 @@ def __init__(self, input_shape=None, bigdl_type="float"): class Dense(KerasLayer): """ A densely-connected NN layer. - When you use this layer as the first layer of a model, you need to provide the argument - inputShape (a Single Shape, does not include the batch dimension). The most common input is 2D. + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + # Arguments output_dim: The size of output dimension. init: String representations of initialization method for the weights of the layer. @@ -67,11 +123,12 @@ class Dense(KerasLayer): applied to the input weights matrices. Default is None. b_regularizer: An instance of [[Regularizer]], applied to the bias. Default is None. bias: Whether to include a bias (i.e. make the layer affine rather than linear). Default is True. + input_shape: A shape tuple, not including batch. >>> dense = Dense(10, input_shape=(3, 4)) creating: createKerasDense """ - def __init__(self, output_dim, init='glorot_uniform', activation=None, + def __init__(self, output_dim, init="glorot_uniform", activation=None, W_regularizer=None, b_regularizer=None, bias=True, input_shape=None, bigdl_type="float"): super(Dense, self).__init__(None, bigdl_type, @@ -84,12 +141,59 @@ def __init__(self, output_dim, init='glorot_uniform', activation=None, list(input_shape) if input_shape else None) +class MaxoutDense(KerasLayer): + """ + A dense maxout layer that takes the element-wise maximum of nbFeature, Dense(inputDim, outputDim) linear layers. + This allows the layer to learn a convex, piecewise linear activation function over the inputs. + The input of this layer should be 2D. + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + + # Arguments + output_dim: The size of output dimension. + nb_feature: Number of Dense layers to use internally. Int. Default is 4. + W_regularizer: An instance of [[Regularizer]], (eg. L1 or L2 regularization), + applied to the input weights matrices. Default is None. + b_regularizer: An instance of [[Regularizer]], applied to the bias. Default is None. + bias: Whether to include a bias (i.e. make the layer affine rather than linear). Default is True. + input_shape: A shape tuple, not including batch. + + >>> maxoutdense = MaxoutDense(6, input_shape=(10, )) + creating: createKerasMaxoutDense + """ + def __init__(self, output_dim, nb_feature=4, W_regularizer=None, + b_regularizer=None, bias=True, input_shape=None, bigdl_type="float"): + super(MaxoutDense, self).__init__(None, bigdl_type, + output_dim, + nb_feature, + W_regularizer, + b_regularizer, + bias, + list(input_shape) if input_shape else None) + + class Embedding(KerasLayer): """ + Turn positive integers (indexes) into dense vectors of fixed size. + The input of this layer should be 2D. + + This layer can only be used as the first layer in a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + + # Arguments + input_dim: Size of the vocabulary. Int > 0. + output_dim: Dimension of the dense embedding. Int >= 0. + init: String representations of initialization method for the weights of the layer. + Default is 'uniform'. + W_regularizer: An instance of [[Regularizer]], (eg. L1 or L2 regularization), + applied to the embedding matrix. Default is None. + input_shape: A shape tuple, not including batch. + >>> embedding = Embedding(1000, 32, input_shape=(10, )) creating: createKerasEmbedding """ - def __init__(self, input_dim, output_dim, init='uniform', + def __init__(self, input_dim, output_dim, init="uniform", W_regularizer=None, input_shape=None, bigdl_type="float"): super(Embedding, self).__init__(None, bigdl_type, input_dim, @@ -101,14 +205,33 @@ def __init__(self, input_dim, output_dim, init='uniform', class BatchNormalization(KerasLayer): """ - >>> batchNormalization = BatchNormalization(input_shape=(3, 12, 12)) + Batch normalization layer. + Normalize the activations of the previous layer at each batch, i.e. applies a transformation + that maintains the mean activation close to 0 and the activation standard deviation close to 1. + It is a feature-wise normalization, each feature map in the input will be normalized separately. + The input of this layer should be 4D. + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + + # Arguments + epsilon: Small float > 0. Fuzz parameter. Default is 0.001. + momentum: Float. Momentum in the computation of the exponential average of the mean and + standard deviation of the data, for feature-wise normalization. Default is 0.99. + beta_init: Name of initialization function for shift parameter. Default is 'zero'. + gamma_init: Name of initialization function for scale parameter. Default is 'one'. + dim_ordering: Format of input data. Either 'th' (Channel First) or 'tf' (Channel Last). Default is 'th'. + For 'th', axis along which to normalize is 1. For 'tf', axis is 3. + input_shape: A shape tuple, not including batch. + + >>> batchnormalization = BatchNormalization(input_shape=(3, 12, 12)) creating: createKerasBatchNormalization """ - def __init__(self, epsilon=0.001, momentum=0.99, beta_init='zero', gamma_init='one', + def __init__(self, epsilon=0.001, momentum=0.99, beta_init="zero", gamma_init="one", dim_ordering="th", input_shape=None, bigdl_type="float"): super(BatchNormalization, self).__init__(None, bigdl_type, - epsilon, - momentum, + float(epsilon), + float(momentum), beta_init, gamma_init, dim_ordering, @@ -130,11 +253,25 @@ def get_running_mean(self): def get_running_std(self): return callBigDlFunc(self.bigdl_type, "getKerasRunningStd", - self.value).to_ndarray() + self.value).to_ndarray() class Merge(KerasLayer): """ + Used to merge a list of tensors into a single tensor, following some merge mode. + Merge must have at least two input layers. + + When using this layer as the first layer in a model, you need to provide the argument + input_shape for input layers (a list of shape tuples, does not include the batch dimension). + + # Arguments + layers: A list of layer instances. Must be more than one layer. + mode: Merge mode. String, must be one of: 'sum', 'mul', 'concat', 'ave', 'cos', + 'dot', 'max'. Default is 'sum'. + concat_axis: Int, axis to use in mode concat. Only specify this when mode is 'concat'. + Default is -1, meaning the last axis of the input. + input_shape: A list of shape tuples, each not including batch. + >>> l1 = InputLayer(input_shape=(3, 5)) creating: createKerasInputLayer >>> l2 = InputLayer(input_shape=(3, 5)) @@ -142,10 +279,1662 @@ class Merge(KerasLayer): >>> merge = Merge(layers=[l1, l2], mode='sum') creating: createKerasMerge """ - def __init__(self, layers=None, mode='sum', concat_axis=-1, + def __init__(self, layers=None, mode="sum", concat_axis=-1, input_shape=None, bigdl_type="float"): super(Merge, self).__init__(None, bigdl_type, list(layers) if layers else None, mode, concat_axis, input_shape) + + +class Dropout(KerasLayer): + """ + Applies Dropout to the input by randomly setting a fraction 'p' of input units to 0 at each + update during training time in order to prevent overfitting. + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + + # Arguments + p: Fraction of the input units to drop. Float between 0 and 1. + input_shape: A shape tuple, not including batch. + + >>> dropout = Dropout(0.25, input_shape=(2, 3)) + creating: createKerasDropout + """ + def __init__(self, p, input_shape=None, bigdl_type="float"): + super(Dropout, self).__init__(None, bigdl_type, + float(p), + list(input_shape) if input_shape else None) + + +class Flatten(KerasLayer): + """ + Flattens the input without affecting the batch size. + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + + # Arguments + input_shape: A shape tuple, not including batch. + + >>> flatten = Flatten(input_shape=(3, 10, 2)) + creating: createKerasFlatten + """ + def __init__(self, input_shape=None, bigdl_type="float"): + super(Flatten, self).__init__(None, bigdl_type, + list(input_shape) if input_shape else None) + + +class Reshape(KerasLayer): + """ + Reshapes an output to a certain shape. + Supports shape inference by allowing one -1 in the target shape. + For example, if input_shape = (2, 3, 4), target_shape = (3, -1), + then output_shape will be (3, 8). + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + + # Arguments + target_shape: A shape tuple. The target shape that you desire to have. Batch dimension should be excluded. + input_shape: A shape tuple, not including batch. + + >>> reshape = Reshape((2, 10), input_shape=(5, 4)) + creating: createKerasReshape + """ + def __init__(self, target_shape, input_shape=None, bigdl_type="float"): + super(Reshape, self).__init__(None, bigdl_type, + target_shape, + list(input_shape) if input_shape else None) + + +class Activation(KerasLayer): + """ + Simple activation function to be applied to the output. + Available activations: 'tanh', 'relu', 'sigmoid', 'softmax', 'softplus', 'softsign', 'hard_sigmoid'. + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + + # Arguments + activation: Name of the activation function as string. + input_shape: A shape tuple, not including batch. + + >>> activation = Activation("relu", input_shape=(3, 4)) + creating: createKerasActivation + """ + def __init__(self, activation, input_shape=None, bigdl_type="float"): + super(Activation, self).__init__(None, bigdl_type, + activation, + list(input_shape) if input_shape else None) + + +class RepeatVector(KerasLayer): + """ + Repeats the input n times. + The input of this layer should be 2D. + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + + # Arguments + n: Repetition factor. Int. + input_shape: A shape tuple, not including batch. + + >>> repeatvector = RepeatVector(5, input_shape=(3, )) + creating: createKerasRepeatVector + """ + def __init__(self, n, input_shape=None, bigdl_type="float"): + super(RepeatVector, self).__init__(None, bigdl_type, + n, + list(input_shape) if input_shape else None) + + +class Permute(KerasLayer): + """ + Permutes the dimensions of the input according to a given pattern. + Useful for connecting RNNs and convnets together. + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + + # Arguments + dims: Tuple of int. Permutation pattern, does not include the samples dimension. Indexing starts at 1. + input_shape: A shape tuple, not including batch. + + >>> permute = Permute((2, 1, 3), input_shape=(3, 4, 5)) + creating: createKerasPermute + """ + def __init__(self, dims, input_shape=None, bigdl_type="float"): + super(Permute, self).__init__(None, bigdl_type, + dims, + list(input_shape) if input_shape else None) + + +class Highway(KerasLayer): + """ + Densely connected highway network. Highway layers are a natural extension of LSTMs to feedforward networks. + The input of this layer should be 2D, i.e. (batch, input dim). + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + + # Arguments + activation: String representations of activation function to use (such as 'relu' or 'sigmoid'). + Default is None. + W_regularizer: An instance of [[Regularizer]], (eg. L1 or L2 regularization), + applied to the input weights matrices. Default is None. + b_regularizer: An instance of [[Regularizer]], applied to the bias. Default is None. + bias: Whether to include a bias (i.e. make the layer affine rather than linear). + Default is True. + input_shape: A shape tuple, not including batch. + + >>> highway = Highway(activation='relu', input_shape=(8, )) + creating: createKerasHighway + """ + def __init__(self, activation=None, W_regularizer=None, b_regularizer=None, + bias=True, input_shape=None, bigdl_type="float"): + super(Highway, self).__init__(None, bigdl_type, + activation, + W_regularizer, + b_regularizer, + bias, + list(input_shape) if input_shape else None) + + +class Convolution1D(KerasLayer): + """ + Applies convolution operator for filtering neighborhoods of 1D inputs. + You can also use Conv1D as an alias of this layer. + The input of this layer should be 3D. + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + + # Arguments + nb_filter: Number of convolution filters to use. + filter_length: The extension (spatial or temporal) of each filter. + init: String representations of initialization method for the weights of the layer. + Default is 'glorot_uniform'. + activation: String representations of activation function to use (such as 'relu' or 'sigmoid'). + Default is None. + border_mode: Either 'valid' or 'same'. Default is 'valid'. + subsample_length: Factor by which to subsample output. Int. Default is 1. + W_regularizer: An instance of [[Regularizer]], (eg. L1 or L2 regularization), + applied to the input weights matrices. Default is None. + b_regularizer: An instance of [[Regularizer]], applied to the bias. Default is None. + bias: Whether to include a bias (i.e. make the layer affine rather than linear). + Default is True. + input_shape: A shape tuple, not including batch. + + >>> conv1d = Convolution1D(12, 4, input_shape=(3, 16)) + creating: createKerasConvolution1D + """ + def __init__(self, nb_filter, filter_length, init="glorot_uniform", + activation=None, border_mode="valid", subsample_length=1, + W_regularizer=None, b_regularizer=None, bias=True, + input_shape=None, bigdl_type="float"): + super(Convolution1D, self).__init__(None, bigdl_type, + nb_filter, + filter_length, + init, + activation, + border_mode, + subsample_length, + W_regularizer, + b_regularizer, + bias, + list(input_shape) if input_shape else None) + + +class Convolution2D(KerasLayer): + """ + Applies a 2D convolution over an input image composed of several input planes. + You can also use Conv2D as an alias of this layer. + The input of this layer should be 4D. + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + e.g. input_shape=(3, 128, 128) for 128x128 RGB pictures. + + # Arguments + nb_filter: Number of convolution filters to use. + nb_row: Number of rows in the convolution kernel. + nb_col: Number of cols in the convolution kernel. + init: String representations of initialization method for the weights of the layer. + Default is 'glorot_uniform'. + activation: String representations of activation function to use (such as 'relu' or 'sigmoid'). + Default is None. + border_mode: Either 'valid' or 'same'. Default is 'valid'. + subsample: Int tuple of length 2 corresponding to the step of the convolution in the + height and width dimension. Also called strides elsewhere. Default is (1, 1). + dim_ordering: Format of input data. Either 'th' (Channel First) or 'tf' (Channel Last). Default is 'th'. + W_regularizer: An instance of [[Regularizer]], (eg. L1 or L2 regularization), + applied to the input weights matrices. Default is None. + b_regularizer: An instance of [[Regularizer]], applied to the bias. Default is None. + bias: Whether to include a bias (i.e. make the layer affine rather than linear). + Default is True. + input_shape: A shape tuple, not including batch. + + >>> conv2d = Convolution2D(32, 3, 3, input_shape=(3, 128, 128)) + creating: createKerasConvolution2D + """ + def __init__(self, nb_filter, nb_row, nb_col, + init="glorot_uniform", activation=None, + border_mode="valid", subsample=(1, 1), dim_ordering="th", + W_regularizer=None, b_regularizer=None, bias=True, + input_shape=None, bigdl_type="float"): + super(Convolution2D, self).__init__(None, bigdl_type, + nb_filter, + nb_row, + nb_col, + init, + activation, + border_mode, + subsample, + dim_ordering, + W_regularizer, + b_regularizer, + bias, + list(input_shape) if input_shape else None) + + +class Convolution3D(KerasLayer): + """ + Applies convolution operator for filtering windows of three-dimensional inputs. + You can also use Conv3D as an alias of this layer. + Data format currently supported for this layer is dim_ordering='th' (Channel First). + The input of this layer should be 5D. + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + + # Arguments + nb_filter: Number of convolution filters to use. + kernel_dim1: Length of the first dimension in the convolution kernel. + kernel_dim2: Length of the second dimension in the convolution kernel. + kernel_dim3: Length of the third dimension in the convolution kernel. + init: String representations of initialization method for the weights of the layer. + Default is 'glorot_uniform'. + activation: String representations of activation function to use (such as 'relu' or 'sigmoid'). + Default is None. + border_mode: Either 'valid' or 'same'. Default is 'valid'. + subsample: Int tuple of length 3. Factor by which to subsample output. + Also called strides elsewhere. Default is (1, 1, 1). + dim_ordering: Format of input data. Only 'th' (Channel First) is supported for now. + W_regularizer: An instance of [[Regularizer]], (eg. L1 or L2 regularization), + applied to the input weights matrices. Default is None. + b_regularizer: An instance of [[Regularizer]], applied to the bias. Default is None. + bias: Whether to include a bias (i.e. make the layer affine rather than linear). + Default is True. + input_shape: A shape tuple, not including batch. + + >>> conv3d = Convolution3D(32, 3, 4, 5, input_shape=(3, 64, 64, 64)) + creating: createKerasConvolution3D + """ + def __init__(self, nb_filter, kernel_dim1, kernel_dim2, kernel_dim3, + init="glorot_uniform", activation=None, border_mode="valid", + subsample=(1, 1, 1), dim_ordering="th", W_regularizer=None, + b_regularizer=None, bias=True, input_shape=None, bigdl_type="float"): + super(Convolution3D, self).__init__(None, bigdl_type, + nb_filter, + kernel_dim1, + kernel_dim2, + kernel_dim3, + init, + activation, + border_mode, + subsample, + dim_ordering, + W_regularizer, + b_regularizer, + bias, + list(input_shape) if input_shape else None) + + +class AtrousConvolution1D(KerasLayer): + """ + Applies an atrous convolution operator for filtering neighborhoods of 1D inputs. + A.k.a dilated convolution or convolution with holes. + Border mode currently supported for this layer is 'valid'. + Bias will be included in this layer. + You can also use AtrousConv1D as an alias of this layer. + The input of this layer should be 3D. + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + + # Arguments + nb_filter: Number of convolution filters to use. + filter_length: The extension (spatial or temporal) of each filter. + init: String representations of initialization method for the weights of the layer. + Default is 'glorot_uniform'. + activation: String representations of activation function to use (such as 'relu' or 'sigmoid'). + Default is None. + border_mode: Only 'valid' is supported for now. + subsample_length: Factor by which to subsample output. Int. Default is 1. + atrous_rate: Factor for kernel dilation. Also called filter_dilation elsewhere. Int. Default is 1. + W_regularizer: An instance of [[Regularizer]], (eg. L1 or L2 regularization), + applied to the input weights matrices. Default is None. + b_regularizer: An instance of [[Regularizer]], applied to the bias. Default is None. + bias: Only 'True' is supported for now. + input_shape: A shape tuple, not including batch. + + >>> atrousconv1d = AtrousConvolution1D(8, 3, input_shape=(3, 12)) + creating: createKerasAtrousConvolution1D + """ + def __init__(self, nb_filter, filter_length, init="glorot_uniform", + activation=None, border_mode='valid', subsample_length=1, atrous_rate=1, + W_regularizer=None, b_regularizer=None, bias=True, input_shape=None, bigdl_type="float"): + if border_mode != "valid": + raise ValueError("For AtrousConvolution1D, only border_mode='valid' is supported for now") + if not bias: + raise ValueError("For AtrousConvolution1D, only bias=True is supported for now") + super(AtrousConvolution1D, self).__init__(None, bigdl_type, + nb_filter, + filter_length, + init, + activation, + subsample_length, + atrous_rate, + W_regularizer, + b_regularizer, + list(input_shape) if input_shape else None) + + +class AtrousConvolution2D(KerasLayer): + """ + Applies an atrous Convolution operator for filtering windows of 2D inputs. + A.k.a dilated convolution or convolution with holes. + Data format currently supported for this layer is dim_ordering='th' (Channel First). + Border mode currently supported for this layer is 'valid'. + Bias will be included in this layer. + You can also use AtrousConv2D as an alias of this layer. + The input of this layer should be 4D. + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + e.g. input_shape=(3, 128, 128) for 128x128 RGB pictures. + + # Arguments + nb_filter: Number of convolution filters to use. + nb_row: Number of rows in the convolution kernel. + nb_col: Number of cols in the convolution kernel. + init: String representations of initialization method for the weights of the layer. + Default is 'glorot_uniform'. + activation: String representations of activation function to use (such as 'relu' or 'sigmoid'). + Default is None. + border_mode: Only 'valid' is supported for now. + subsample: Int tuple of length 2 corresponding to the step of the convolution in the + height and width dimension. Also called strides elsewhere. Default is (1, 1). + atrous_rate: Int tuple of length 2. Factor for kernel dilation. + Also called filter_dilation elsewhere. Default is (1, 1). + dim_ordering: Format of input data. Only 'th' (Channel First) is supported for now. + W_regularizer: An instance of [[Regularizer]], (eg. L1 or L2 regularization), + applied to the input weights matrices. Default is None. + b_regularizer: An instance of [[Regularizer]], applied to the bias. Default is None. + bias: Only 'True' is supported for now. + input_shape: A shape tuple, not including batch. + + >>> atrousconv2d = AtrousConvolution2D(12, 4, 3, input_shape=(3, 64, 64)) + creating: createKerasAtrousConvolution2D + """ + def __init__(self, nb_filter, nb_row, nb_col, init="glorot_uniform", + activation=None, border_mode="valid", subsample=(1, 1), + atrous_rate=(1, 1), dim_ordering="th", W_regularizer=None, + b_regularizer=None, bias=True, input_shape=None, bigdl_type="float"): + if border_mode != "valid": + raise ValueError("For AtrousConvolution2D, only border_mode='valid' is supported for now") + if not bias: + raise ValueError("For AtrousConvolution2D, only bias=True is supported for now") + super(AtrousConvolution2D, self).__init__(None, bigdl_type, + nb_filter, + nb_row, + nb_col, + init, + activation, + subsample, + atrous_rate, + dim_ordering, + W_regularizer, + b_regularizer, + list(input_shape) if input_shape else None) + + +class Deconvolution2D(KerasLayer): + """ + Transposed convolution operator for filtering windows of 2D inputs. + The need for transposed convolutions generally arises from the desire to use a transformation + going in the opposite direction of a normal convolution, i.e., from something that has + the shape of the output of some convolution to something that has the shape of its input + while maintaining a connectivity pattern that is compatible with said convolution. + Data format currently supported for this layer is dim_ordering='th' (Channel First). + Border mode currently supported for this layer is 'valid'. + You can also use Deconv2D as an alias of this layer. + The input of this layer should be 4D. + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + e.g. input_shape=(3, 128, 128) for 128x128 RGB pictures. + + # Arguments + nb_filter: Number of transposed convolution filters to use. + nb_row: Number of rows in the convolution kernel. + nb_col: Number of cols in the convolution kernel. + output_shape: Output shape of the transposed convolution operation. Tuple of int. + init: String representations of initialization method for the weights of the layer. + Default is 'glorot_uniform'. + activation: String representations of activation function to use (such as 'relu' or 'sigmoid'). + Default is None. + border_mode: Only 'valid' is supported for now. + subsample: Int tuple of length 2 corresponding to the step of the convolution in the + height and width dimension. Also called strides elsewhere. Default is (1, 1). + dim_ordering: Format of input data. Only 'th' (Channel First) is supported for now. + W_regularizer: An instance of [[Regularizer]], (eg. L1 or L2 regularization), + applied to the input weights matrices. Default is None. + b_regularizer: An instance of [[Regularizer]], applied to the bias. Default is None. + bias: Whether to include a bias (i.e. make the layer affine rather than linear). + Default is True. + input_shape: A shape tuple, not including batch. + + >>> deconv2d = Deconvolution2D(3, 3, 3, output_shape=(None, 3, 14, 14), input_shape=(3, 12, 12)) + creating: createKerasDeconvolution2D + """ + def __init__(self, nb_filter, nb_row, nb_col, output_shape, init="glorot_uniform", + activation=None, border_mode="valid", subsample=(1, 1), dim_ordering="th", + W_regularizer=None, b_regularizer=None, bias=True, input_shape=None, bigdl_type="float"): + if border_mode != "valid": + raise ValueError("For Deconvolution2D, only border_mode='valid' is supported for now") + super(Deconvolution2D, self).__init__(None, bigdl_type, + nb_filter, + nb_row, + nb_col, + init, + activation, + subsample, + dim_ordering, + W_regularizer, + b_regularizer, + bias, + list(input_shape) if input_shape else None) + + +class SeparableConvolution2D(KerasLayer): + """ + Applies separable convolution operator for 2D inputs. + Separable convolutions consist in first performing a depthwise spatial convolution (which acts + on each input channel separately) followed by a pointwise convolution which mixes together the + resulting output channels. The depthMultiplier argument controls how many output channels are + generated per input channel in the depthwise step. + You can also use SeparableConv2D as an alias of this layer. + The input of this layer should be 4D. + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + e.g. input_shape=(3, 128, 128) for 128x128 RGB pictures. + + # Arguments + nb_filter: Number of convolution filters to use. + nb_row: Number of rows in the convolution kernel. + nb_col: Number of cols in the convolution kernel. + init: String representations of initialization method for the weights of the layer. + Default is 'glorot_uniform'. + activation: String representations of activation function to use (such as 'relu' or 'sigmoid'). + Default is None. + border_mode: Either 'valid' or 'same'. Default is 'valid'. + subsample: Int tuple of length 2 corresponding to the step of the convolution in the + height and width dimension. Also called strides elsewhere. Default is (1, 1). + depth_multiplier: How many output channel to use per input channel for the depthwise convolution step. + Int. Default is 1. + dim_ordering: Format of input data. Either 'th' (Channel First) or 'tf' (Channel Last). Default is 'th'. + depthwise_regularizer: An instance of [[Regularizer]], (eg. L1 or L2 regularization), + applied to the depthwise weights matrices. Default is None. + pointwise_regularizer: An instance of [[Regularizer]], applied to the pointwise weights matrices. + Default is None. + b_regularizer: An instance of [[Regularizer]], applied to the bias. Default is None. + bias: Whether to include a bias (i.e. make the layer affine rather than linear). + Default is True. + input_shape: A shape tuple, not including batch. + + >>> separableconv2d = SeparableConvolution2D(12, 3, 4, input_shape=(3, 32, 32)) + creating: createKerasSeparableConvolution2D + """ + def __init__(self, nb_filter, nb_row, nb_col, init="glorot_uniform", + activation=None, border_mode="valid", subsample=(1, 1), depth_multiplier=1, + dim_ordering="th", depthwise_regularizer=None, pointwise_regularizer=None, + b_regularizer=None, bias=True, input_shape=None, bigdl_type="float"): + super(SeparableConvolution2D, self).__init__(None, bigdl_type, + nb_filter, + nb_row, + nb_col, + init, + activation, + border_mode, + subsample, + depth_multiplier, + dim_ordering, + depthwise_regularizer, + pointwise_regularizer, + b_regularizer, + bias, + list(input_shape) if input_shape else None) + + +Conv1D = Convolution1D +Conv2D = Convolution2D +Conv3D = Convolution3D +Deconv2D = Deconvolution2D +AtrousConv1D = AtrousConvolution1D +AtrousConv2D = AtrousConvolution2D +SeparableConv2D = SeparableConvolution2D + + +class Cropping1D(KerasLayer): + """ + Cropping layer for 1D input (e.g. temporal sequence). + The input of this layer should be 3D. + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + + # Arguments + cropping: Int tuple of length 2. How many units should be trimmed off at the beginning and + end of the cropping dimension. Default is (1, 1). + input_shape: A shape tuple, not including batch. + + >>> cropping1d = Cropping1D(cropping=(1, 2), input_shape=(8, 8)) + creating: createKerasCropping1D + """ + def __init__(self, cropping=(1, 1), input_shape=None, bigdl_type="float"): + super(Cropping1D, self).__init__(None, bigdl_type, + cropping, + list(input_shape) if input_shape else None) + + +class Cropping2D(KerasLayer): + """ + Cropping layer for 2D input (e.g. picture). + The input of this layer should be 4D. + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + + # Arguments + cropping: Int tuple of tuple of length 2. How many units should be trimmed off at the beginning and + end of the 2 cropping dimensions (i.e. height and width). Default is ((0, 0), (0, 0)). + input_shape: A shape tuple, not including batch. + + >>> cropping2d = Cropping2D(cropping=((1, 2), (0, 1)), input_shape=(12, 12, 12)) + creating: createKerasCropping2D + """ + def __init__(self, cropping=((0, 0), (0, 0)), dim_ordering="th", + input_shape=None, bigdl_type="float"): + super(Cropping2D, self).__init__(None, bigdl_type, + cropping[0], + cropping[1], + dim_ordering, + list(input_shape) if input_shape else None) + + +class Cropping3D(KerasLayer): + """ + Cropping layer for 3D data (e.g. spatial or spatio-temporal). + The input of this layer should be 5D. + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + + # Arguments + cropping: Int tuple of tuple of length 3. How many units should be trimmed off at the beginning and + end of the 3 cropping dimensions (i.e. kernel_dim1, kernel_dim2 and kernel_dim3). + Default is ((1, 1), (1, 1), (1, 1)). + input_shape: A shape tuple, not including batch. + + >>> cropping3d = Cropping3D(cropping=((0, 2), (1, 1), (3, 1)), input_shape=(4, 12, 12, 16)) + creating: createKerasCropping3D + """ + def __init__(self, cropping=((1, 1), (1, 1), (1, 1)), dim_ordering="th", + input_shape=None, bigdl_type="float"): + super(Cropping3D, self).__init__(None, bigdl_type, + cropping[0], + cropping[1], + cropping[2], + dim_ordering, + list(input_shape) if input_shape else None) + + +class UpSampling1D(KerasLayer): + """ + UpSampling layer for 1D inputs. + Repeats each temporal step 'length' times along the time axis. + The input of this layer should be 3D. + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + + # Arguments + length: Int. UpSampling factor. Default is 2. + input_shape: A shape tuple, not including batch. + + >>> upsampling1d = UpSampling1D(length=3, input_shape=(3, 12)) + creating: createKerasUpSampling1D + """ + def __init__(self, length=2, input_shape=None, bigdl_type="float"): + super(UpSampling1D, self).__init__(None, bigdl_type, + length, + list(input_shape) if input_shape else None) + + +class UpSampling2D(KerasLayer): + """ + UpSampling layer for 2D inputs. + Repeats the rows and columns of the data by size[0] and size[1] respectively. + The input of this layer should be 4D. + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + + # Arguments + size: Int tuple of length 2. UpSampling factors for rows and columns. Default is (2, 2). + dim_ordering: Format of input data. Either 'th' (Channel First) or 'tf' (Channel Last). Default is 'th'. + input_shape: A shape tuple, not including batch. + + >>> upsampling2d = UpSampling2D(size=(1, 3), input_shape=(3, 16, 16)) + creating: createKerasUpSampling2D + """ + def __init__(self, size=(2, 2), dim_ordering="th", input_shape=None, bigdl_type="float"): + super(UpSampling2D, self).__init__(None, bigdl_type, + size, + dim_ordering, + list(input_shape) if input_shape else None) + + +class UpSampling3D(KerasLayer): + """ + UpSampling layer for 2D inputs. + Repeats the 1st, 2nd and 3rd dimensions of the data by size[0], size[1] and size[2] respectively. + Data format currently supported for this layer is dim_ordering='th' (Channel First). + The input of this layer should be 5D. + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + + # Arguments + size: Int tuple of length 3. UpSampling factors for dim1, dim2 and dim3. Default is (2, 2, 2). + dim_ordering: Format of input data. Only 'th' (Channel First) is supported for now. + input_shape: A shape tuple, not including batch. + + >>> upsampling3d = UpSampling3D(size=(1, 2, 3), input_shape=(3, 16, 16, 16)) + creating: createKerasUpSampling3D + """ + def __init__(self, size=(2, 2, 2), dim_ordering="th", input_shape=None, bigdl_type="float"): + super(UpSampling3D, self).__init__(None, bigdl_type, + size, + dim_ordering, + list(input_shape) if input_shape else None) + + +class ZeroPadding1D(KerasLayer): + """ + Zero-padding layer for 1D input (e.g. temporal sequence). + The input of this layer should be 3D. + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + + # Arguments + padding: Int or int tuple of length 2. + If int, how many zeros to add both at the beginning and at the end of the padding dimension. + If tuple of length 2, how many zeros to add in the order '(left_pad, right_pad)'. + Default is 1. + input_shape: A shape tuple, not including batch. + + >>> zeropadding1d = ZeroPadding1D(padding=2, input_shape=(3, 6)) + creating: createKerasZeroPadding1D + """ + def __init__(self, padding=1, input_shape=None, bigdl_type="float"): + if isinstance(padding, int): + padding = (padding, padding) + super(ZeroPadding1D, self).__init__(None, bigdl_type, + padding, + list(input_shape) if input_shape else None) + + +class ZeroPadding2D(KerasLayer): + """ + Zero-padding layer for 2D input (e.g. picture). + The input of this layer should be 4D. + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + + # Arguments + padding: Int tuple of length 2 or length 4. + If tuple of length 2, how many zeros to add both at the beginning and at the end of rows and cols. + If tuple of length 4, how many zeros to add in the order '(top_pad, bottom_pad, left_pad, right_pad)'. + Default is (1, 1). + dim_ordering: Format of input data. Either 'th' (Channel First) or 'tf' (Channel Last). Default is 'th'. + input_shape: A shape tuple, not including batch. + + >>> zeropadding2d = ZeroPadding2D(padding=(2, 1), input_shape=(2, 8, 8)) + creating: createKerasZeroPadding2D + """ + def __init__(self, padding=(1, 1), dim_ordering="th", input_shape=None, bigdl_type="float"): + if len(padding) == 2: + padding = (padding[0], padding[0], padding[1], padding[1]) + super(ZeroPadding2D, self).__init__(None, bigdl_type, + padding, + dim_ordering, + list(input_shape) if input_shape else None) + + +class ZeroPadding3D(KerasLayer): + """ + Zero-padding layer for 3D data (spatial or spatio-temporal). + The input of this layer should be 5D. + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + + # Arguments + padding: Int tuple of length 3. How many zeros to add at the beginning and at the end of the 3 padding dimensions. + Symmetric padding will be applied to each dimension. Default is (1, 1, 1). + dim_ordering: Format of input data. Either 'th' (Channel First) or 'tf' (Channel Last). Default is 'th'. + input_shape: A shape tuple, not including batch. + + >>> zeropadding3d = ZeroPadding3D(padding=(2, 1, 2), input_shape=(2, 8, 8, 10)) + creating: createKerasZeroPadding3D + """ + def __init__(self, padding=(1, 1, 1), dim_ordering="th", input_shape=None, bigdl_type="float"): + super(ZeroPadding3D, self).__init__(None, bigdl_type, + padding, + dim_ordering, + list(input_shape) if input_shape else None) + + +class MaxPooling1D(KerasLayer): + """ + Applies max pooling operation for temporal data. + The input of this layer should be 3D. + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + + # Arguments + pool_length: Size of the region to which max pooling is applied. + strides: Factor by which to downscale. 2 will halve the input. + Default is None, and in this case it will be equal to pool_length.. + border_mode: Either 'valid' or 'same'. Default is 'valid'. + input_shape: A shape tuple, not including batch. + + >>> maxpooling1d = MaxPooling1D(3, input_shape=(3, 24)) + creating: createKerasMaxPooling1D + """ + def __init__(self, pool_length=2, stride=None, border_mode="valid", + input_shape=None, bigdl_type="float"): + if not stride: + stride = -1 + super(MaxPooling1D, self).__init__(None, bigdl_type, + pool_length, + stride, + border_mode, + list(input_shape) if input_shape else None) + + +class MaxPooling2D(KerasLayer): + """ + Applies max pooling operation for spatial data. + The input of this layer should be 4D. + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + + # Arguments + pool_size: Int tuple of length 2 corresponding to the downscale vertically and horizontally. + Default is (2, 2), which will halve the image in each dimension. + strides: Int tuple of length 2. Stride values. Default is None, and in this case it will be equal to pool_size. + border_mode: Either 'valid' or 'same'. Default is 'valid'. + dim_ordering: Format of input data. Either 'th' (Channel First) or 'tf' (Channel Last). Default is 'th'. + input_shape: A shape tuple, not including batch. + + >>> maxpooling2d = MaxPooling2D((2, 2), input_shape=(3, 32, 32)) + creating: createKerasMaxPooling2D + """ + def __init__(self, pool_size=(2, 2), strides=None, + border_mode='valid', dim_ordering='th', + input_shape=None, bigdl_type="float"): + super(MaxPooling2D, self).__init__(None, bigdl_type, + pool_size, + strides, + border_mode, + dim_ordering, + list(input_shape) if input_shape else None) + + +class MaxPooling3D(KerasLayer): + """ + Applies max pooling operation for 3D data (spatial or spatio-temporal). + Data format currently supported for this layer is dim_ordering='th' (Channel First). + Border mode currently supported for this layer is 'valid'. + The input of this layer should be 5D. + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + + # Arguments + pool_size: Int tuple of length 3. Factors by which to downscale (dim1, dim2, dim3). + Default is (2, 2, 2), which will halve the image in each dimension. + strides: Int tuple of length 3. Stride values. Default is None, and in this case it will be equal to pool_size. + border_mode: Only 'valid' is supported for now. + dim_ordering: Format of input data. Only 'th' (Channel First) is supported for now. + input_shape: A shape tuple, not including batch. + + >>> maxpooling3d = MaxPooling3D((2, 1, 3), input_shape=(3, 32, 32, 32)) + creating: createKerasMaxPooling3D + """ + def __init__(self, pool_size=(2, 2, 2), strides=None, border_mode="valid", + dim_ordering="th", input_shape=None, bigdl_type="float"): + if border_mode != "valid": + raise ValueError("For MaxPooling3D, only border_mode='valid' is supported for now") + super(MaxPooling3D, self).__init__(None, bigdl_type, + pool_size, + strides, + dim_ordering, + list(input_shape) if input_shape else None) + + +class AveragePooling1D(KerasLayer): + """ + Applies average pooling operation for temporal data. + The input of this layer should be 3D. + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + + # Arguments + pool_length: Size of the region to which max pooling is applied. + strides: Factor by which to downscale. 2 will halve the input. + Default is None, and in this case it will be equal to pool_length.. + border_mode: Either 'valid' or 'same'. Default is 'valid'. + input_shape: A shape tuple, not including batch. + + >>> averagepooling1d = AveragePooling1D(input_shape=(3, 24)) + creating: createKerasAveragePooling1D + """ + def __init__(self, pool_length=2, stride=None, border_mode="valid", + input_shape=None, bigdl_type="float"): + if not stride: + stride = -1 + super(AveragePooling1D, self).__init__(None, bigdl_type, + pool_length, + stride, + border_mode, + list(input_shape) if input_shape else None) + + +class AveragePooling2D(KerasLayer): + """ + Applies average pooling operation for spatial data. + The input of this layer should be 4D. + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + + # Arguments + pool_size: Int tuple of length 2 corresponding to the downscale vertically and horizontally. + Default is (2, 2), which will halve the image in each dimension. + strides: Int tuple of length 2. Stride values. Default is None, and in this case it will be equal to pool_size. + border_mode: Either 'valid' or 'same'. Default is 'valid'. + dim_ordering: Format of input data. Either 'th' (Channel First) or 'tf' (Channel Last). Default is 'th'. + input_shape: A shape tuple, not including batch. + + >>> averagepooling2d = AveragePooling2D((1, 2), input_shape=(2, 28, 32)) + creating: createKerasAveragePooling2D + """ + def __init__(self, pool_size=(2, 2), strides=None, border_mode="valid", + dim_ordering="th", input_shape=None, bigdl_type="float"): + super(AveragePooling2D, self).__init__(None, bigdl_type, + pool_size, + strides, + border_mode, + dim_ordering, + list(input_shape) if input_shape else None) + + +class AveragePooling3D(KerasLayer): + """ + Applies average pooling operation for 3D data (spatial or spatio-temporal). + Data format currently supported for this layer is dim_ordering='th' (Channel First). + Border mode currently supported for this layer is 'valid'. + The input of this layer should be 5D. + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + + # Arguments + pool_size: Int tuple of length 3. Factors by which to downscale (dim1, dim2, dim3). + Default is (2, 2, 2), which will halve the image in each dimension. + strides: Int tuple of length 3. Stride values. Default is None, and in this case it will be equal to pool_size. + border_mode: Only 'valid' is supported for now. + dim_ordering: Format of input data. Only 'th' (Channel First) is supported for now. + input_shape: A shape tuple, not including batch. + + >>> averagepooling3d = AveragePooling3D((1, 1, 2), input_shape=(3, 28, 32, 36)) + creating: createKerasAveragePooling3D + """ + def __init__(self, pool_size=(2, 2, 2), strides=None, border_mode="valid", + dim_ordering="th", input_shape=None, bigdl_type="float"): + if border_mode != "valid": + raise ValueError("For AveragePooling3D, only border_mode='valid' is supported for now") + super(AveragePooling3D, self).__init__(None, bigdl_type, + pool_size, + strides, + dim_ordering, + list(input_shape) if input_shape else None) + + +class GlobalMaxPooling1D(KerasLayer): + """ + Applies global max pooling operation for temporal data. + The input of this layer should be 3D. + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + + # Arguments + input_shape: A shape tuple, not including batch. + + >>> globalmaxpooling1d = GlobalMaxPooling1D(input_shape=(4, 8)) + creating: createKerasGlobalMaxPooling1D + """ + def __init__(self, input_shape=None, bigdl_type="float"): + super(GlobalMaxPooling1D, self).__init__(None, bigdl_type, + list(input_shape) if input_shape else None) + + +class GlobalAveragePooling1D(KerasLayer): + """ + Applies global average pooling operation for temporal data. + The input of this layer should be 3D. + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + + # Arguments + input_shape: A shape tuple, not including batch. + + >>> globalaveragepooling1d = GlobalAveragePooling1D(input_shape=(12, 12)) + creating: createKerasGlobalAveragePooling1D + """ + def __init__(self, input_shape=None, bigdl_type="float"): + super(GlobalAveragePooling1D, self).__init__(None, bigdl_type, + list(input_shape) if input_shape else None) + + +class GlobalMaxPooling2D(KerasLayer): + """ + Applies global max pooling operation for spatial data. + The input of this layer should be 4D. + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + + # Arguments + dim_ordering: Format of input data. Either 'th' (Channel First) or 'tf' (Channel Last). Default is 'th'. + input_shape: A shape tuple, not including batch. + + >>> globalmaxpooling2d = GlobalMaxPooling2D(input_shape=(4, 32, 32)) + creating: createKerasGlobalMaxPooling2D + """ + def __init__(self, dim_ordering="th", input_shape=None, bigdl_type="float"): + super(GlobalMaxPooling2D, self).__init__(None, bigdl_type, + dim_ordering, + list(input_shape) if input_shape else None) + + +class GlobalAveragePooling2D(KerasLayer): + """ + Applies global average pooling operation for spatial data. + The input of this layer should be 4D. + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + + # Arguments + dim_ordering: Format of input data. Either 'th' (Channel First) or 'tf' (Channel Last). Default is 'th'. + input_shape: A shape tuple, not including batch. + + >>> globalaveragepooling2d = GlobalAveragePooling2D(input_shape=(4, 32, 32)) + creating: createKerasGlobalAveragePooling2D + """ + def __init__(self, dim_ordering="th", input_shape=None, bigdl_type="float"): + super(GlobalAveragePooling2D, self).__init__(None, bigdl_type, + dim_ordering, + list(input_shape) if input_shape else None) + + +class GlobalMaxPooling3D(KerasLayer): + """ + Applies global max pooling operation for 3D data. + Data format currently supported for this layer is dim_ordering='th' (Channel First). + Border mode currently supported for this layer is 'valid'. + The input of this layer should be 5D. + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + + # Arguments + dim_ordering: Format of input data. Only 'th' (Channel First) is supported for now. + input_shape: A shape tuple, not including batch. + + >>> globalmaxpooling3d = GlobalMaxPooling3D(input_shape=(4, 32, 32, 32)) + creating: createKerasGlobalMaxPooling3D + """ + def __init__(self, dim_ordering="th", input_shape=None, bigdl_type="float"): + super(GlobalMaxPooling3D, self).__init__(None, bigdl_type, + dim_ordering, + list(input_shape) if input_shape else None) + + +class GlobalAveragePooling3D(KerasLayer): + """ + Applies global average pooling operation for 3D data. + Data format currently supported for this layer is dim_ordering='th' (Channel First). + Border mode currently supported for this layer is 'valid'. + The input of this layer should be 5D. + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + + # Arguments + dim_ordering: Format of input data. Only 'th' (Channel First) is supported for now. + input_shape: A shape tuple, not including batch. + + >>> globalaveragepooling3d = GlobalAveragePooling3D(input_shape=(4, 16, 16, 20)) + creating: createKerasGlobalAveragePooling3D + """ + def __init__(self, dim_ordering="th", input_shape=None, bigdl_type="float"): + super(GlobalAveragePooling3D, self).__init__(None, bigdl_type, + dim_ordering, + list(input_shape) if input_shape else None) + + +class SimpleRNN(KerasLayer): + """ + A fully-connected recurrent neural network cell. The output is to be fed back to input. + The input of this layer should be 3D, i.e. (batch, time steps, input dim). + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + + # Arguments + output_dim: Hidden unit size. Dimension of internal projections and final output. + activation: String representations of activation function to use (such as 'relu' or 'sigmoid'). + Default is 'tanh'. + return_sequences: Whether to return the full sequence or only return the last output in the output sequence. + Default is False. + go_backwards: Whether the input sequence will be processed backwards. Default is False. + W_regularizer: An instance of [[Regularizer]], (eg. L1 or L2 regularization), + applied to the input weights matrices. Default is None. + U_regularizer: An instance of [[Regularizer]], applied the recurrent weights matrices. Default is None. + b_regularizer: An instance of [[Regularizer]], applied to the bias. Default is None. + input_shape: A shape tuple, not including batch. + + >>> simplernn = SimpleRNN(16, input_shape=(3, 32)) + creating: createKerasSimpleRNN + """ + def __init__(self, output_dim, activation="tanh", return_sequences=False, + go_backwards=False, W_regularizer=None, U_regularizer=None, + b_regularizer=None, input_shape=None, bigdl_type="float"): + super(SimpleRNN, self).__init__(None, bigdl_type, + output_dim, + activation, + return_sequences, + go_backwards, + W_regularizer, + U_regularizer, + b_regularizer, + list(input_shape) if input_shape else None) + + +class LSTM(KerasLayer): + """ + Long Short Term Memory unit architecture. + The input of this layer should be 3D, i.e. (batch, time steps, input dim). + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + + # Arguments + output_dim: Hidden unit size. Dimension of internal projections and final output. + activation: String representations of activation function to use (such as 'relu' or 'sigmoid'). + Default is 'tanh'. + inner_activation: String representations of activation function for inner cells. Default is 'hard_sigmoid'. + return_sequences: Whether to return the full sequence or only return the last output in the output sequence. + Default is False. + go_backwards: Whether the input sequence will be processed backwards. Default is False. + W_regularizer: An instance of [[Regularizer]], (eg. L1 or L2 regularization), + applied to the input weights matrices. Default is None. + U_regularizer: An instance of [[Regularizer]], applied the recurrent weights matrices. Default is None. + b_regularizer: An instance of [[Regularizer]], applied to the bias. Default is None. + input_shape: A shape tuple, not including batch. + + >>> lstm = LSTM(32, input_shape=(8, 16)) + creating: createKerasLSTM + """ + def __init__(self, output_dim, activation="tanh", inner_activation="hard_sigmoid", + return_sequences=False, go_backwards=False, W_regularizer=None, + U_regularizer=None, b_regularizer=None, input_shape=None, bigdl_type="float"): + super(LSTM, self).__init__(None, bigdl_type, + output_dim, + activation, + inner_activation, + return_sequences, + go_backwards, + W_regularizer, + U_regularizer, + b_regularizer, + list(input_shape) if input_shape else None) + + +class GRU(KerasLayer): + """ + Gated Recurrent Unit architecture. + The input of this layer should be 3D, i.e. (batch, time steps, input dim). + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + + # Arguments + output_dim: Hidden unit size. Dimension of internal projections and final output. + activation: String representations of activation function to use (such as 'relu' or 'sigmoid'). + Default is 'tanh'. + inner_activation: String representations of activation function for inner cells. Default is 'hard_sigmoid'. + return_sequences: Whether to return the full sequence or only return the last output in the output sequence. + Default is False. + go_backwards: Whether the input sequence will be processed backwards. Default is False. + W_regularizer: An instance of [[Regularizer]], (eg. L1 or L2 regularization), + applied to the input weights matrices. Default is None. + U_regularizer: An instance of [[Regularizer]], applied the recurrent weights matrices. Default is None. + b_regularizer: An instance of [[Regularizer]], applied to the bias. Default is None. + input_shape: A shape tuple, not including batch. + + >>> gru = GRU(24, input_shape=(32, 32)) + creating: createKerasGRU + """ + def __init__(self, output_dim, activation="tanh", inner_activation="hard_sigmoid", + return_sequences=False, go_backwards=False, W_regularizer=None, + U_regularizer=None, b_regularizer=None, input_shape=None, bigdl_type="float"): + super(GRU, self).__init__(None, bigdl_type, + output_dim, + activation, + inner_activation, + return_sequences, + go_backwards, + W_regularizer, + U_regularizer, + b_regularizer, + list(input_shape) if input_shape else None) + + +class ConvLSTM2D(KerasLayer): + """ + Convolutional LSTM. + Data format currently supported for this layer is dim_ordering='th' (Channel First). + Border mode currently supported for this layer is 'same'. + The convolution kernel for this layer is a square kernel with equal strides 'subsample'. + The input of this layer should be 5D. + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + + # Arguments + nb_filter: Number of convolution filters to use. + nb_row: Number of rows in the convolution kernel. + nb_col: Number of cols in the convolution kernel. Should be equal to nb_row as for a square kernel. + activation: String representations of activation function to use (such as 'relu' or 'sigmoid'). + Default is 'tanh'. + inner_activation: String representations of activation function for inner cells. Default is 'hard_sigmoid'. + dim_ordering: Format of input data. Only 'th' (Channel First) is supported for now. + border_mode: Only 'same' is supported for now. + subsample: Tuple of length 2. Factor by which to subsample output. Also called strides elsewhere. + Only support subsample[0] equal to subsample[1] for now. Default is (1, 1). + W_regularizer: An instance of [[Regularizer]], (eg. L1 or L2 regularization), + applied to the input weights matrices. Default is None. + U_regularizer: An instance of [[Regularizer]], applied the recurrent weights matrices. Default is None. + b_regularizer: An instance of [[Regularizer]], applied to the bias. Default is None. + return_sequences: Whether to return the full sequence or only return the last output in the output sequence. + Default is False. + go_backwards: Whether the input sequence will be processed backwards. Default is False. + input_shape: A shape tuple, not including batch. + + >>> convlstm2d = ConvLSTM2D(24, 3, 3, input_shape=(4, 32, 32, 32)) + creating: createKerasConvLSTM2D + """ + def __init__(self, nb_filter, nb_row, nb_col, activation="tanh", + inner_activation="hard_sigmoid", dim_ordering="th", border_mode="same", + subsample=(1, 1), W_regularizer=None, U_regularizer=None, b_regularizer=None, + return_sequences=False, go_backwards=False, input_shape=None, bigdl_type="float"): + if nb_row != nb_col: + raise ValueError("For ConvLSTM2D, only square kernel is supported for now") + if border_mode != "same": + raise ValueError("For ConvLSTM2D, only border_mode='same' is supported for now") + if subsample[0] != subsample[1]: + raise ValueError("For ConvLSTM2D, only equal strides is supported for now") + super(ConvLSTM2D, self).__init__(None, bigdl_type, + nb_filter, + nb_row, + activation, + inner_activation, + dim_ordering, + subsample[0], + W_regularizer, + U_regularizer, + b_regularizer, + return_sequences, + go_backwards, + list(input_shape) if input_shape else None) + + +class LocallyConnected1D(KerasLayer): + """ + Locally-connected layer for 1D inputs which works similarly to the TemporalConvolution layer, except that + weights are unshared, that is, a different set of filters is applied at each different patch of the input. + Border mode currently supported for this layer is 'valid'. + The input of this layer should be 3D. + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + + # Arguments + nb_filter: Dimensionality of the output. + filter_length: The extension (spatial or temporal) of each filter. + activation: String representations of activation function to use (such as 'relu' or 'sigmoid'). + Default is None. + border_mode: Only 'valid' is supported for now. + subsample_length: Factor by which to subsample output. Int. Default is 1. + W_regularizer: An instance of [[Regularizer]], (eg. L1 or L2 regularization), + applied to the input weights matrices. Default is None. + b_regularizer: An instance of [[Regularizer]], applied to the bias. Default is None. + bias: Whether to include a bias (i.e. make the layer affine rather than linear). + Default is True. + input_shape: A shape tuple, not including batch. + + >>> locallyconnected1d = LocallyConnected1D(6, 3, input_shape=(8, 12)) + creating: createKerasLocallyConnected1D + """ + def __init__(self, nb_filter, filter_length, activation=None, border_mode="valid", + subsample_length=1, W_regularizer=None, b_regularizer=None, + bias=True, input_shape=None, bigdl_type="float"): + if border_mode != "valid": + raise ValueError("For LocallyConnected1D, only border_mode='valid' is supported for now") + super(LocallyConnected1D, self).__init__(None, bigdl_type, + nb_filter, + filter_length, + activation, + subsample_length, + W_regularizer, + b_regularizer, + bias, + list(input_shape) if input_shape else None) + + +class LocallyConnected2D(KerasLayer): + """ + Locally-connected layer for 2D inputs that works similarly to the SpatialConvolution layer, except that + weights are unshared, that is, a different set of filters is applied at each different patch of the input. + The input of this layer should be 4D. + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + + # Arguments + nb_filter: Number of convolution filters to use. + nb_row: Number of rows in the convolution kernel. + nb_col: Number of cols in the convolution kernel. + activation: String representations of activation function to use (such as 'relu' or 'sigmoid'). + Default is None. + border_mode: Either 'valid' or 'same'. Default is 'valid'. + subsample: Int tuple of length 2 corresponding to the step of the convolution in the + height and width dimension. Also called strides elsewhere. Default is (1, 1). + dim_ordering: Format of input data. Either 'th' (Channel First) or 'tf' (Channel Last). Default is 'th'. + W_regularizer: An instance of [[Regularizer]], (eg. L1 or L2 regularization), + applied to the input weights matrices. Default is None. + b_regularizer: An instance of [[Regularizer]], applied to the bias. Default is None. + bias: Whether to include a bias (i.e. make the layer affine rather than linear). + Default is True. + input_shape: A shape tuple, not including batch. + + >>> locallyconnected2d = LocallyConnected2D(12, 3, 4, input_shape=(3, 128, 128)) + creating: createKerasLocallyConnected2D + """ + def __init__(self, nb_filter, nb_row, nb_col, activation=None, + border_mode="valid", subsample=(1, 1), dim_ordering="th", + W_regularizer=None, b_regularizer=None, bias=True, + input_shape=None, bigdl_type="float"): + super(LocallyConnected2D, self).__init__(None, bigdl_type, + nb_filter, + nb_row, + nb_col, + activation, + border_mode, + subsample, + dim_ordering, + W_regularizer, + b_regularizer, + bias, + list(input_shape) if input_shape else None) + + +class SpatialDropout1D(KerasLayer): + """ + Spatial 1D version of Dropout. + This version performs the same function as Dropout, however it drops entire 1D feature maps + instead of individual elements. If adjacent frames within feature maps are strongly correlated + (as is normally the case in early convolution layers) then regular dropout will not regularize the + activations and will otherwise just result in an effective learning rate decrease. + In this case, SpatialDropout1D will help promote independence between feature maps and should be used instead. + The input of this layer should be 3D. + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + + # Arguments + p: Fraction of the input units to drop. Float between 0 and 1. + input_shape: A shape tuple, not including batch. + + >>> spatialdropout1d = SpatialDropout1D(0.4, input_shape=(10, 12)) + creating: createKerasSpatialDropout1D + """ + def __init__(self, p=0.5, input_shape=None, bigdl_type="float"): + super(SpatialDropout1D, self).__init__(None, bigdl_type, + float(p), + list(input_shape) if input_shape else None) + + +class SpatialDropout2D(KerasLayer): + """ + Spatial 2D version of Dropout. + This version performs the same function as Dropout, however it drops entire 2D feature maps + instead of individual elements. If adjacent pixels within feature maps are strongly correlated + (as is normally the case in early convolution layers) then regular dropout will not regularize the + activations and will otherwise just result in an effective learning rate decrease. + In this case, SpatialDropout2D will help promote independence between feature maps and should be used instead. + The input of this layer should be 4D. + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + + # Arguments + p: Fraction of the input units to drop. Float between 0 and 1. + dim_ordering: Format of input data. Either 'th' (Channel First) or 'tf' (Channel Last). Default is 'th'. + input_shape: A shape tuple, not including batch. + + >>> spatialdropout2d = SpatialDropout2D(0.25, input_shape=(5, 12, 12)) + creating: createKerasSpatialDropout2D + """ + def __init__(self, p=0.5, dim_ordering="th", input_shape=None, bigdl_type="float"): + super(SpatialDropout2D, self).__init__(None, bigdl_type, + float(p), + dim_ordering, + list(input_shape) if input_shape else None) + + +class SpatialDropout3D(KerasLayer): + """ + Spatial 3D version of Dropout. + This version performs the same function as Dropout, however it drops entire 3D feature maps + instead of individual elements. If adjacent voxels within feature maps are strongly correlated + (as is normally the case in early convolution layers) then regular dropout will not regularize the + activations and will otherwise just result in an effective learning rate decrease. + In this case, SpatialDropout3D will help promote independence between feature maps and should be used instead. + The input of this layer should be 5D. + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + + # Arguments + p: Fraction of the input units to drop. Float between 0 and 1. + dim_ordering: Format of input data. Either 'th' (Channel First) or 'tf' (Channel Last). Default is 'th'. + input_shape: A shape tuple, not including batch. + + >>> spatialdropout3d = SpatialDropout3D(0.6, input_shape=(4, 12, 12, 16)) + creating: createKerasSpatialDropout3D + """ + def __init__(self, p=0.5, dim_ordering="th", input_shape=None, bigdl_type="float"): + super(SpatialDropout3D, self).__init__(None, bigdl_type, + float(p), + dim_ordering, + list(input_shape) if input_shape else None) + + +class GaussianDropout(KerasLayer): + """ + Apply multiplicative 1-centered Gaussian noise. + As it is a regularization layer, it is only active at training time. + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + + # Arguments + p: Drop probability. Float between 0 and 1. + The multiplicative noise will have standard deviation 'sqrt(p/(1-p))'. + input_shape: A shape tuple, not including batch. + + >>> gaussiandropout = GaussianDropout(0.45, input_shape=(4, 8)) + creating: createKerasGaussianDropout + """ + def __init__(self, p, input_shape=None, bigdl_type="float"): + super(GaussianDropout, self).__init__(None, bigdl_type, + float(p), + list(input_shape) if input_shape else None) + + +class GaussianNoise(KerasLayer): + """ + Apply additive zero-centered Gaussian noise. + This is useful to mitigate overfitting (you could see it as a form of random data augmentation). + Gaussian Noise is a natural choice as corruption process for real valued inputs. + As it is a regularization layer, it is only active at training time. + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + + # Arguments + sigma: Float, standard deviation of the noise distribution. + input_shape: A shape tuple, not including batch. + + >>> gaussiannoise = GaussianNoise(0.45, input_shape=(3, 4, 5)) + creating: createKerasGaussianNoise + """ + def __init__(self, sigma, input_shape=None, bigdl_type="float"): + super(GaussianNoise, self).__init__(None, bigdl_type, + float(sigma), + list(input_shape) if input_shape else None) + + +class Masking(KerasLayer): + """ + Use a mask value to skip timesteps for a sequence. + Masks a sequence by using a mask value to skip timesteps. + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + + # Arguments + mask_value: Float, mask value. For each timestep in the input tensor (dimension #1 in the tensor), + if all values in the input tensor at that timestep are equal to `mask_value`, + then the timestep will masked (skipped) in all downstream layers. + input_shape: A shape tuple, not including batch. + + >>> masking = Masking(0.3, input_shape=(6, 8)) + creating: createKerasMasking + """ + def __init__(self, mask_value=0.0, input_shape=None, bigdl_type="float"): + super(Masking, self).__init__(None, bigdl_type, + float(mask_value), + list(input_shape) if input_shape else None) + + +class SReLU(KerasLayer): + """ + S-shaped Rectified Linear Unit. + It follows: + f(x) = t^r + a^r(x - t^r) for x >= t^r + f(x) = x for t^r > x > t^l, + f(x) = t^l + a^l(x - t^l) for x <= t^l + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + + # Arguments + t_left_init: String representations of initialization method for the left part intercept. + Default is 'zero'. + a_left_init: String representations of initialization method for the left part slope. + Default is 'glorot_uniform'. + t_right_init: String representations of initialization method for the right part intercept. + Default is 'glorot_uniform'. + a_right_init: String representations of initialization method for the right part slope. + Default is 'one'. + shared_axes: Int tuple. The axes along which to share learnable parameters for the activation function. + Default is None. + For example, if the incoming feature maps are from a 2D convolution with output shape + (batch, height, width, channels), and you wish to share parameters across space so that + each filter only has one set of parameters, set 'SharedAxes=(1,2)'. + input_shape: A shape tuple, not including batch. + + >>> srelu = SReLU(input_shape=(4, 5)) + creating: createKerasSReLU + """ + def __init__(self, t_left_init='zero', a_left_init='glorot_uniform', + t_right_init='glorot_uniform', a_right_init='one', + shared_axes=None, input_shape=None, bigdl_type="float"): + super(SReLU, self).__init__(None, bigdl_type, + t_left_init, + a_left_init, + t_right_init, + a_right_init, + shared_axes, + list(input_shape) if input_shape else None) + + +class ELU(KerasLayer): + """ + Exponential Linear Unit. + It follows: + f(x) = alpha * (exp(x) - 1.) for x < 0, + f(x) = x for x >= 0. + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + + # Arguments + alpha: Float, scale for the negative factor. + input_shape: A shape tuple, not including batch. + + >>> elu = ELU(1.2, input_shape=(4, 5)) + creating: createKerasELU + """ + def __init__(self, alpha=1.0, input_shape=None, bigdl_type="float"): + super(ELU, self).__init__(None, bigdl_type, + float(alpha), + list(input_shape) if input_shape else None) + + +class LeakyReLU(KerasLayer): + """ + Leaky version of a Rectified Linear Unit. + It allows a small gradient when the unit is not active: + f(x) = alpha * x for x < 0, + f(x) = x for x >= 0. + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + + # Arguments + alpha: Float >= 0. Negative slope coefficient. + input_shape: A shape tuple, not including batch. + + >>> leakyrelu = LeakyReLU(0.02, input_shape=(4, 5)) + creating: createKerasLeakyReLU + """ + def __init__(self, alpha=0.01, input_shape=None, bigdl_type="float"): + super(LeakyReLU, self).__init__(None, bigdl_type, + float(alpha), + list(input_shape) if input_shape else None) + + +class ThresholdedReLU(KerasLayer): + """ + Thresholded Rectified Linear Unit. + It follows: + f(x) = x for x > theta, + f(x) = 0 otherwise. + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + + # Arguments + theta: Float >= 0. Threshold location of activation. + input_shape: A shape tuple, not including batch. + + >>> thresholdedrelu = ThresholdedReLU(input_shape=(10, 12)) + creating: createKerasThresholdedReLU + """ + def __init__(self, theta=1.0, input_shape=None, bigdl_type="float"): + super(ThresholdedReLU, self).__init__(None, bigdl_type, + float(theta), + list(input_shape) if input_shape else None) + + +class TimeDistributed(KerasLayer): + """ + TimeDistributed wrapper. + Apply a layer to every temporal slice of an input. + The input should be at least 3D, and the dimension of index one will be considered to be the temporal dimension. + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + + # Arguments + layer: A layer instance. + input_shape: A shape tuple, not including batch. + + >>> timedistributed = TimeDistributed(Dense(8), input_shape=(10, 12)) + creating: createKerasDense + creating: createKerasTimeDistributed + """ + def __init__(self, layer, input_shape=None, bigdl_type="float"): + super(TimeDistributed, self).__init__(None, bigdl_type, + layer, + list(input_shape) if input_shape else None) + + +class Bidirectional(KerasLayer): + """ + Bidirectional wrapper for RNNs. + Bidirectional currently requires RNNs to return the full sequence, i.e. return_sequences = True. + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + + Example of creating a bidirectional LSTM: + Bidirectiona(LSTM(12, return_sequences=True), merge_mode="sum", input_shape=(32, 32)) + + # Arguments + layer: An instance of a recurrent layer. + merge_mode: Mode by which outputs of the forward and backward RNNs will be combined. + Must be one of: 'sum', 'mul', 'concat', 'ave'. Default is 'concat'. + input_shape: A shape tuple, not including batch. + + >>> bidiretional = Bidirectional(LSTM(10, return_sequences=True), input_shape=(12, 16)) + creating: createKerasLSTM + creating: createKerasBidirectional + """ + def __init__(self, layer, merge_mode="concat", input_shape=None, bigdl_type="float"): + super(Bidirectional, self).__init__(None, bigdl_type, + layer, + merge_mode, + list(input_shape) if input_shape else None) \ No newline at end of file diff --git a/python/dllib/src/bigdl/dllib/examples/lenet/README.md b/python/dllib/src/bigdl/dllib/examples/lenet/README.md new file mode 100644 index 00000000000..4eeeccd1e7b --- /dev/null +++ b/python/dllib/src/bigdl/dllib/examples/lenet/README.md @@ -0,0 +1,75 @@ +# LeNet5 Model on MNIST with new API + +This example defines a classical CNN model used in digital number classification with the new set of API in BigDL. For detailed information with regard to LeNet, please refer to . + +This example is the same as [../../model/lenet/lenet5.py](../../models/lenet/lenet5.py), except that this example uses new API for model definition. + +## Install dependencies + * [Install dependencies](../../../README.md#install.dependencies) + +## How to run this example: +Please note that due to some permission issue, this example **cannot** be run on Windows. + + +Program would download the mnist data into ```/tmp/mnist``` automatically by default. + +``` +/tmp/mnist$ tree . +. +├── t10k-images-idx3-ubyte.gz +├── t10k-labels-idx1-ubyte.gz +├── train-images-idx3-ubyte.gz +└── train-labels-idx1-ubyte.gz + +``` + +We would train a LeNet model in spark local mode with the following commands and you can distribute it across cluster by modifying the spark master and the executor cores. + +``` + BigDL_HOME=... + SPARK_HOME=... + MASTER=local[*] + PYTHON_API_ZIP_PATH=${BigDL_HOME}/dist/lib/bigdl-VERSION-python-api.zip + BigDL_JAR_PATH=${BigDL_HOME}/dist/lib/bigdl-VERSION-jar-with-dependencies.jar + PYTHONPATH=${PYTHON_API_ZIP_PATH}:$PYTHONPATH + ${SPARK_HOME}/bin/spark-submit \ + --master ${MASTER} \ + --driver-cores 2 \ + --driver-memory 2g \ + --total-executor-cores 2 \ + --executor-cores 2 \ + --executor-memory 4g \ + --py-files ${PYTHON_API_ZIP_PATH},${BigDL_HOME}/pyspark/bigdl/models/lenet/lenet5.py \ + --properties-file ${BigDL_HOME}/dist/conf/spark-bigdl.conf \ + --jars ${BigDL_JAR_PATH} \ + --conf spark.driver.extraClassPath=${BigDL_JAR_PATH} \ + --conf spark.executor.extraClassPath=bigdl-VERSION-jar-with-dependencies.jar \ + ${BigDL_HOME}/pyspark/bigdl/models/lenet/lenet5.py \ + --dataPath /tmp/mnist + ``` + +* ```--dataPath``` option can be used to set the path for downloading mnist data, the default value is /tmp/mnist. Make sure that you have write permission to the specified path. + +* ```--batchSize``` option can be used to set batch size, the default value is 128. + +* ```--endTriggerType``` option can be used to control how to end the training process, the value can be "epoch" or "iteration" and default value is "epoch". + +* ```--endTriggerNum``` use together with ```endTriggerType```, the default value is 20. + +* ```--modelPath``` option can be used to set model path for testing, the default value is /tmp/lenet5/model.470. + +* ```--checkpointPath``` option can be used to set checkpoint path for saving model, the default value is /tmp/lenet5/. + +To verify the accuracy, search "accuracy" from log: + +``` +INFO DistriOptimizer$:247 - [Epoch 1 0/60000][Iteration 1][Wall Clock 0.0s] Train 128 in xx seconds. Throughput is xx records/second. + +INFO DistriOptimizer$:522 - Top1Accuracy is Accuracy(correct: 9572, count: 10000, accuracy: 0.9572) + +``` + +Or you can train a LeNet model directly in shell after installing BigDL from pip: +``` +python ${BigDL_HOME}/pyspark/bigdl/examples/lenet/lenet.py +``` \ No newline at end of file diff --git a/python/dllib/src/bigdl/dllib/examples/lenet/__init__.py b/python/dllib/src/bigdl/dllib/examples/lenet/__init__.py new file mode 100644 index 00000000000..2151a805423 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/examples/lenet/__init__.py @@ -0,0 +1,15 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/python/dllib/src/bigdl/dllib/examples/lenet/lenet.py b/python/dllib/src/bigdl/dllib/examples/lenet/lenet.py new file mode 100644 index 00000000000..ba76772d1a3 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/examples/lenet/lenet.py @@ -0,0 +1,66 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from optparse import OptionParser +from bigdl.models.lenet.utils import * +from bigdl.nn.keras.layer import * +from bigdl.nn.criterion import * +from bigdl.optim.optimizer import * +from bigdl.util.common import * + + +def build_model(class_num): + model = Sequential() + model.add(Reshape((1, 28, 28), input_shape=(28, 28, 1))) + model.add(Convolution2D(32, 3, 3, activation="relu")) + model.add(Convolution2D(32, 3, 3, activation="relu")) + model.add(MaxPooling2D(pool_size=(2, 2))) + model.add(Dropout(0.25)) + model.add(Flatten()) + model.add(Dense(128, activation="relu")) + model.add(Dropout(0.5)) + model.add(Dense(class_num, activation="softmax")) + return model + + +if __name__ == "__main__": + parser = OptionParser() + parser.add_option("-b", "--batchSize", type=int, dest="batchSize", default="128") + parser.add_option("-c", "--checkpointPath", dest="checkpointPath", default="/tmp/lenet5") + parser.add_option("-t", "--endTriggerType", dest="endTriggerType", default="epoch") + parser.add_option("-n", "--endTriggerNum", type=int, dest="endTriggerNum", default="20") + parser.add_option("-d", "--dataPath", dest="dataPath", default="/tmp/mnist") + + (options, args) = parser.parse_args(sys.argv) + + sc = SparkContext(appName="lenet5", conf=create_spark_conf()) + redire_spark_logs() + show_bigdl_info_logs() + init_engine() + + (train_data, test_data) = preprocess_mnist(sc, options) + + optimizer = Optimizer( + model=build_model(10), + training_rdd=train_data, + criterion=ClassNLLCriterion(logProbAsInput=False), + optim_method=SGD(learningrate=0.01, learningrate_decay=0.0002), + end_trigger=get_end_trigger(options), + batch_size=options.batchSize) + validate_optimizer(optimizer, test_data, options) + trained_model = optimizer.optimize() + parameters = trained_model.parameters() + sc.stop() diff --git a/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py b/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py index 92a7074a08a..e6e28b7d65e 100644 --- a/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py +++ b/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py @@ -15,7 +15,7 @@ # from optparse import OptionParser -from bigdl.dataset import mnist +from utils import * from bigdl.dataset.transformer import * from bigdl.nn.layer import * from bigdl.nn.criterion import * @@ -40,23 +40,6 @@ def build_model(class_num): return model -def get_mnist(sc, data_type="train", location="/tmp/mnist"): - """ - Get and normalize the mnist data. We would download it automatically - if the data doesn't present at the specific location. - - :param sc: SparkContext - :param data_type: training data or testing data - :param location: Location storing the mnist - :return: A RDD of (features: Ndarray, label: Ndarray) - """ - (images, labels) = mnist.read_data_sets(location, data_type) - images = sc.parallelize(images) - labels = sc.parallelize(labels + 1) # Target start from 1 in BigDL - record = images.zip(labels) - return record - - if __name__ == "__main__": parser = OptionParser() parser.add_option("-a", "--action", dest="action", default="train") @@ -75,34 +58,16 @@ def get_mnist(sc, data_type="train", location="/tmp/mnist"): init_engine() if options.action == "train": - def get_end_trigger(): - if options.endTriggerType.lower() == "epoch": - return MaxEpoch(options.endTriggerNum) - else: - return MaxIteration(options.endTriggerNum) + (train_data, test_data) = preprocess_mnist(sc, options) - train_data = get_mnist(sc, "train", options.dataPath)\ - .map(lambda rec_tuple: (normalizer(rec_tuple[0], mnist.TRAIN_MEAN, mnist.TRAIN_STD), - rec_tuple[1]))\ - .map(lambda t: Sample.from_ndarray(t[0], t[1])) - test_data = get_mnist(sc, "test", options.dataPath)\ - .map(lambda rec_tuple: (normalizer(rec_tuple[0], mnist.TEST_MEAN, mnist.TEST_STD), - rec_tuple[1]))\ - .map(lambda t: Sample.from_ndarray(t[0], t[1])) optimizer = Optimizer( model=build_model(10), training_rdd=train_data, criterion=ClassNLLCriterion(), optim_method=SGD(learningrate=0.01, learningrate_decay=0.0002), - end_trigger=get_end_trigger(), + end_trigger=get_end_trigger(options), batch_size=options.batchSize) - optimizer.set_validation( - batch_size=options.batchSize, - val_rdd=test_data, - trigger=EveryEpoch(), - val_method=[Top1Accuracy()] - ) - optimizer.set_checkpoint(EveryEpoch(), options.checkpointPath) + validate_optimizer(optimizer, test_data, options) trained_model = optimizer.optimize() parameters = trained_model.parameters() elif options.action == "test": diff --git a/python/dllib/src/bigdl/dllib/models/lenet/utils.py b/python/dllib/src/bigdl/dllib/models/lenet/utils.py new file mode 100644 index 00000000000..e90444fda33 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/models/lenet/utils.py @@ -0,0 +1,75 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from bigdl.dataset import mnist +from bigdl.dataset.transformer import * +from bigdl.optim.optimizer import * + + +def get_mnist(sc, data_type="train", location="/tmp/mnist"): + """ + Get mnist dataset and parallelize into RDDs. + Data would be downloaded automatically if it doesn't present at the specific location. + + :param sc: SparkContext. + :param data_type: "train" for training data and "test" for testing data. + :param location: Location to store mnist dataset. + :return: RDD of (features: ndarray, label: ndarray). + """ + (images, labels) = mnist.read_data_sets(location, data_type) + images = sc.parallelize(images) + labels = sc.parallelize(labels + 1) # Target start from 1 in BigDL + record = images.zip(labels) + return record + + +def preprocess_mnist(sc, options): + """ + Preprocess mnist dataset. + Normalize and transform into Sample of RDDs. + """ + train_data = get_mnist(sc, "train", options.dataPath)\ + .map(lambda rec_tuple: (normalizer(rec_tuple[0], mnist.TRAIN_MEAN, mnist.TRAIN_STD), + rec_tuple[1]))\ + .map(lambda t: Sample.from_ndarray(t[0], t[1])) + test_data = get_mnist(sc, "test", options.dataPath)\ + .map(lambda rec_tuple: (normalizer(rec_tuple[0], mnist.TEST_MEAN, mnist.TEST_STD), + rec_tuple[1]))\ + .map(lambda t: Sample.from_ndarray(t[0], t[1])) + return train_data, test_data + + +def get_end_trigger(options): + """ + When to end the optimization based on input option. + """ + if options.endTriggerType.lower() == "epoch": + return MaxEpoch(options.endTriggerNum) + else: + return MaxIteration(options.endTriggerNum) + + +def validate_optimizer(optimizer, test_data, options): + """ + Set validation and checkpoint for distributed optimizer. + """ + optimizer.set_validation( + batch_size=options.batchSize, + val_rdd=test_data, + trigger=EveryEpoch(), + val_method=[Top1Accuracy()] + ) + optimizer.set_checkpoint(EveryEpoch(), options.checkpointPath) diff --git a/python/dllib/src/bigdl/dllib/models/local_lenet/README.md b/python/dllib/src/bigdl/dllib/models/local_lenet/README.md index b9f9f6ab1e6..a1691532cca 100644 --- a/python/dllib/src/bigdl/dllib/models/local_lenet/README.md +++ b/python/dllib/src/bigdl/dllib/models/local_lenet/README.md @@ -3,6 +3,8 @@ LeNet5 is a classical CNN model used in digital number classification. For detailed information, please refer to . +The model used here is exactly the same as the model in [../lenet/lenet5.py](../lenet/lenet5.py). + This example would show how to train and inference a LeNet model in pure local mode without using Spark local or Spark distributed cluster. diff --git a/python/dllib/src/bigdl/dllib/models/local_lenet/local_lenet.py b/python/dllib/src/bigdl/dllib/models/local_lenet/local_lenet.py index 0fb96befdab..cc264abacba 100644 --- a/python/dllib/src/bigdl/dllib/models/local_lenet/local_lenet.py +++ b/python/dllib/src/bigdl/dllib/models/local_lenet/local_lenet.py @@ -16,38 +16,20 @@ from optparse import OptionParser from bigdl.dataset import mnist -from bigdl.dataset.transformer import * -from bigdl.nn.layer import * +from bigdl.models.lenet.lenet5 import build_model from bigdl.nn.criterion import * from bigdl.optim.optimizer import * from bigdl.util.common import * -def build_model(class_num): - model = Sequential() - model.add(Reshape([1, 28, 28])) - model.add(SpatialConvolution(1, 6, 5, 5)) - model.add(Tanh()) - model.add(SpatialMaxPooling(2, 2, 2, 2)) - model.add(Tanh()) - model.add(SpatialConvolution(6, 12, 5, 5)) - model.add(SpatialMaxPooling(2, 2, 2, 2)) - model.add(Reshape([12 * 4 * 4])) - model.add(Linear(12 * 4 * 4, 100)) - model.add(Tanh()) - model.add(Linear(100, class_num)) - model.add(LogSoftMax()) - return model - - def get_mnist(data_type="train", location="/tmp/mnist"): """ - Get and normalize the mnist data. We would download it automatically - if the data doesn't present at the specific location. + Get mnist dataset with features and label as ndarray. + Data would be downloaded automatically if it doesn't present at the specific location. - :param data_type: training data or testing data - :param location: Location storing the mnist - :return: (features: Ndarray, label: Ndarray) + :param data_type: "train" for training data and "test" for testing data. + :param location: Location to store mnist dataset. + :return: (features: ndarray, label: ndarray) """ X, Y = mnist.read_data_sets(location, data_type) return X, Y + 1 # The label of ClassNLLCriterion starts from 1 instead of 0 @@ -67,6 +49,7 @@ def get_mnist(data_type="train", location="/tmp/mnist"): (X_train, Y_train) = get_mnist("train", options.dataPath) (X_test, Y_test) = get_mnist("test", options.dataPath) + # The model used here is exactly the same as the model in ../lenet/lenet5.py optimizer = Optimizer.create( model=build_model(10), training_set=(X_train, Y_train), @@ -76,8 +59,8 @@ def get_mnist(data_type="train", location="/tmp/mnist"): batch_size=options.batchSize) optimizer.set_validation( batch_size=options.batchSize, - X_val = X_test, - Y_val = Y_test, + X_val=X_test, + Y_val=Y_test, trigger=EveryEpoch(), val_method=[Top1Accuracy()] ) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 856dca0aee0..5de2162a4d5 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -614,7 +614,7 @@ def add(self, model): @property def layers(self): - jlayers = callBigDlFunc(self.bigdl_type, "getContainerModules" , self) + jlayers = callBigDlFunc(self.bigdl_type, "getContainerModules", self) layers = [Layer.of(jlayer) for jlayer in jlayers] return layers @@ -654,6 +654,7 @@ class Model(Container): def __init__(self, inputs, outputs, + is_keras=False, jvalue=None, bigdl_type="float", byte_order="little_endian", model_type="bigdl"): if jvalue: @@ -661,8 +662,9 @@ def __init__(self, self.bigdl_type = bigdl_type elif model_type == "bigdl": super(Model, self).__init__(None, bigdl_type, - to_list(inputs), - to_list(outputs)) + to_list(inputs), + to_list(outputs), + is_keras) else: from bigdl.util.tf_utils import convert model = convert(to_list(inputs), to_list(outputs), byte_order, bigdl_type) @@ -1062,8 +1064,8 @@ class Sequential(Container): ''' - def __init__(self, bigdl_type="float"): - super(Sequential, self).__init__(None, bigdl_type, False) + def __init__(self, bigdl_type="float", is_keras=False): + super(Sequential, self).__init__(None, bigdl_type, is_keras) class TemporalConvolution(Layer): From 7c0e43b40a282d3f4ae3f368def7161b913086a3 Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Wed, 14 Feb 2018 15:43:30 +0800 Subject: [PATCH 576/823] Keras-like API Python Wrapper for layers + lenet example (#2296) * python lenet * docs * style * wrap getoutputshape * update input output shape * update * python layers * clean * fix ut * more layers * fix ut * more layers * fix ut * more layers * wrapper * remove * style * style and srelu * style * refine lenet * update * add ut for input output shape * fix float * refine lenet * update * meet review * clean * todo * update * style * readme * style * style * fix * style --- .../bigdl/dllib/bigdlkeras/layers/layer.py | 1819 ++++++++++++++++- .../src/bigdl/dllib/examples/lenet/README.md | 75 + .../bigdl/dllib/examples/lenet/__init__.py | 15 + .../src/bigdl/dllib/examples/lenet/lenet.py | 66 + .../src/bigdl/dllib/models/lenet/lenet5.py | 43 +- .../src/bigdl/dllib/models/lenet/utils.py | 75 + .../bigdl/dllib/models/local_lenet/README.md | 2 + .../dllib/models/local_lenet/local_lenet.py | 35 +- python/dllib/src/bigdl/dllib/nn/layer.py | 12 +- 9 files changed, 2057 insertions(+), 85 deletions(-) create mode 100644 python/dllib/src/bigdl/dllib/examples/lenet/README.md create mode 100644 python/dllib/src/bigdl/dllib/examples/lenet/__init__.py create mode 100644 python/dllib/src/bigdl/dllib/examples/lenet/lenet.py create mode 100644 python/dllib/src/bigdl/dllib/models/lenet/utils.py diff --git a/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py b/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py index a50c14c54c1..567006329f9 100644 --- a/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py +++ b/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py @@ -16,14 +16,47 @@ import sys -from bigdl.nn.layer import Layer, Container -from bigdl.util.common import callBigDlFunc, JTensor +from bigdl.nn.layer import Layer, Sequential as TSequential, Model as TModel +from bigdl.util.common import callBigDlFunc, JTensor, JavaValue, to_list if sys.version >= '3': long = int unicode = str +class InferShape(JavaValue): + def __init__(self, bigdl_type="float"): + self.bigdl_type = bigdl_type + + @classmethod + def __to_keras_shape(cls, shape): + return tuple([None] + shape[1:]) + + def __process_shape(self, shape): + if len(shape) == 1: + return self.__to_keras_shape(shape[0]) + else: + return [self.__to_keras_shape(s) for s in shape] + + def get_input_shape(self): + """ + Return a list of shape tuples if there are multiple inputs. + Return one shape tuple otherwise. + """ + input = callBigDlFunc(self.bigdl_type, "getInputShape", + self.value) + return self.__process_shape(input) + + def get_output_shape(self): + """ + Return a list of shape tuples if there are multiple outputs. + Return one shape tuple otherwise. + """ + output = callBigDlFunc(self.bigdl_type, "getOutputShape", + self.value) + return self.__process_shape(output) + + class KerasLayer(Layer): def jvm_class_constructor(self): name = "createKeras" + self.__class__.__name__ @@ -31,18 +64,40 @@ def jvm_class_constructor(self): return name -class Sequential(Container): +class Sequential(TSequential, InferShape): """ + Container for a Sequential model. + >>> sequential = Sequential() creating: createSequential """ def __init__(self, bigdl_type="float"): - super(Sequential, self).__init__(None, bigdl_type, True) + super(Sequential, self).__init__(bigdl_type, True) + + +class Model(TModel, InferShape): + def __init__(self, input, output, bigdl_type="float"): + super(Model, self).__init__(to_list(input), + to_list(output), + is_keras=True, + bigdl_type=bigdl_type) + + +class Input(KerasLayer): + def __init__(self, name=None, input_shape=None, bigdl_type="float"): + super(Input, self).__init__(None, bigdl_type, + name, + list(input_shape) if input_shape else None) class InputLayer(KerasLayer): """ - >>> inputLayer = InputLayer(input_shape=(3, 5)) + Layer to be used as an entry point into a model. + + # Arguments + input_shape: A shape tuple, not including the batch axis. + + >>> inputlayer = InputLayer(input_shape=(3, 5)) creating: createKerasInputLayer """ def __init__(self, input_shape=None, bigdl_type="float"): @@ -53,10 +108,11 @@ def __init__(self, input_shape=None, bigdl_type="float"): class Dense(KerasLayer): """ A densely-connected NN layer. - When you use this layer as the first layer of a model, you need to provide the argument - inputShape (a Single Shape, does not include the batch dimension). The most common input is 2D. + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + # Arguments output_dim: The size of output dimension. init: String representations of initialization method for the weights of the layer. @@ -67,11 +123,12 @@ class Dense(KerasLayer): applied to the input weights matrices. Default is None. b_regularizer: An instance of [[Regularizer]], applied to the bias. Default is None. bias: Whether to include a bias (i.e. make the layer affine rather than linear). Default is True. + input_shape: A shape tuple, not including batch. >>> dense = Dense(10, input_shape=(3, 4)) creating: createKerasDense """ - def __init__(self, output_dim, init='glorot_uniform', activation=None, + def __init__(self, output_dim, init="glorot_uniform", activation=None, W_regularizer=None, b_regularizer=None, bias=True, input_shape=None, bigdl_type="float"): super(Dense, self).__init__(None, bigdl_type, @@ -84,12 +141,59 @@ def __init__(self, output_dim, init='glorot_uniform', activation=None, list(input_shape) if input_shape else None) +class MaxoutDense(KerasLayer): + """ + A dense maxout layer that takes the element-wise maximum of nbFeature, Dense(inputDim, outputDim) linear layers. + This allows the layer to learn a convex, piecewise linear activation function over the inputs. + The input of this layer should be 2D. + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + + # Arguments + output_dim: The size of output dimension. + nb_feature: Number of Dense layers to use internally. Int. Default is 4. + W_regularizer: An instance of [[Regularizer]], (eg. L1 or L2 regularization), + applied to the input weights matrices. Default is None. + b_regularizer: An instance of [[Regularizer]], applied to the bias. Default is None. + bias: Whether to include a bias (i.e. make the layer affine rather than linear). Default is True. + input_shape: A shape tuple, not including batch. + + >>> maxoutdense = MaxoutDense(6, input_shape=(10, )) + creating: createKerasMaxoutDense + """ + def __init__(self, output_dim, nb_feature=4, W_regularizer=None, + b_regularizer=None, bias=True, input_shape=None, bigdl_type="float"): + super(MaxoutDense, self).__init__(None, bigdl_type, + output_dim, + nb_feature, + W_regularizer, + b_regularizer, + bias, + list(input_shape) if input_shape else None) + + class Embedding(KerasLayer): """ + Turn positive integers (indexes) into dense vectors of fixed size. + The input of this layer should be 2D. + + This layer can only be used as the first layer in a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + + # Arguments + input_dim: Size of the vocabulary. Int > 0. + output_dim: Dimension of the dense embedding. Int >= 0. + init: String representations of initialization method for the weights of the layer. + Default is 'uniform'. + W_regularizer: An instance of [[Regularizer]], (eg. L1 or L2 regularization), + applied to the embedding matrix. Default is None. + input_shape: A shape tuple, not including batch. + >>> embedding = Embedding(1000, 32, input_shape=(10, )) creating: createKerasEmbedding """ - def __init__(self, input_dim, output_dim, init='uniform', + def __init__(self, input_dim, output_dim, init="uniform", W_regularizer=None, input_shape=None, bigdl_type="float"): super(Embedding, self).__init__(None, bigdl_type, input_dim, @@ -101,14 +205,33 @@ def __init__(self, input_dim, output_dim, init='uniform', class BatchNormalization(KerasLayer): """ - >>> batchNormalization = BatchNormalization(input_shape=(3, 12, 12)) + Batch normalization layer. + Normalize the activations of the previous layer at each batch, i.e. applies a transformation + that maintains the mean activation close to 0 and the activation standard deviation close to 1. + It is a feature-wise normalization, each feature map in the input will be normalized separately. + The input of this layer should be 4D. + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + + # Arguments + epsilon: Small float > 0. Fuzz parameter. Default is 0.001. + momentum: Float. Momentum in the computation of the exponential average of the mean and + standard deviation of the data, for feature-wise normalization. Default is 0.99. + beta_init: Name of initialization function for shift parameter. Default is 'zero'. + gamma_init: Name of initialization function for scale parameter. Default is 'one'. + dim_ordering: Format of input data. Either 'th' (Channel First) or 'tf' (Channel Last). Default is 'th'. + For 'th', axis along which to normalize is 1. For 'tf', axis is 3. + input_shape: A shape tuple, not including batch. + + >>> batchnormalization = BatchNormalization(input_shape=(3, 12, 12)) creating: createKerasBatchNormalization """ - def __init__(self, epsilon=0.001, momentum=0.99, beta_init='zero', gamma_init='one', + def __init__(self, epsilon=0.001, momentum=0.99, beta_init="zero", gamma_init="one", dim_ordering="th", input_shape=None, bigdl_type="float"): super(BatchNormalization, self).__init__(None, bigdl_type, - epsilon, - momentum, + float(epsilon), + float(momentum), beta_init, gamma_init, dim_ordering, @@ -130,11 +253,25 @@ def get_running_mean(self): def get_running_std(self): return callBigDlFunc(self.bigdl_type, "getKerasRunningStd", - self.value).to_ndarray() + self.value).to_ndarray() class Merge(KerasLayer): """ + Used to merge a list of tensors into a single tensor, following some merge mode. + Merge must have at least two input layers. + + When using this layer as the first layer in a model, you need to provide the argument + input_shape for input layers (a list of shape tuples, does not include the batch dimension). + + # Arguments + layers: A list of layer instances. Must be more than one layer. + mode: Merge mode. String, must be one of: 'sum', 'mul', 'concat', 'ave', 'cos', + 'dot', 'max'. Default is 'sum'. + concat_axis: Int, axis to use in mode concat. Only specify this when mode is 'concat'. + Default is -1, meaning the last axis of the input. + input_shape: A list of shape tuples, each not including batch. + >>> l1 = InputLayer(input_shape=(3, 5)) creating: createKerasInputLayer >>> l2 = InputLayer(input_shape=(3, 5)) @@ -142,10 +279,1662 @@ class Merge(KerasLayer): >>> merge = Merge(layers=[l1, l2], mode='sum') creating: createKerasMerge """ - def __init__(self, layers=None, mode='sum', concat_axis=-1, + def __init__(self, layers=None, mode="sum", concat_axis=-1, input_shape=None, bigdl_type="float"): super(Merge, self).__init__(None, bigdl_type, list(layers) if layers else None, mode, concat_axis, input_shape) + + +class Dropout(KerasLayer): + """ + Applies Dropout to the input by randomly setting a fraction 'p' of input units to 0 at each + update during training time in order to prevent overfitting. + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + + # Arguments + p: Fraction of the input units to drop. Float between 0 and 1. + input_shape: A shape tuple, not including batch. + + >>> dropout = Dropout(0.25, input_shape=(2, 3)) + creating: createKerasDropout + """ + def __init__(self, p, input_shape=None, bigdl_type="float"): + super(Dropout, self).__init__(None, bigdl_type, + float(p), + list(input_shape) if input_shape else None) + + +class Flatten(KerasLayer): + """ + Flattens the input without affecting the batch size. + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + + # Arguments + input_shape: A shape tuple, not including batch. + + >>> flatten = Flatten(input_shape=(3, 10, 2)) + creating: createKerasFlatten + """ + def __init__(self, input_shape=None, bigdl_type="float"): + super(Flatten, self).__init__(None, bigdl_type, + list(input_shape) if input_shape else None) + + +class Reshape(KerasLayer): + """ + Reshapes an output to a certain shape. + Supports shape inference by allowing one -1 in the target shape. + For example, if input_shape = (2, 3, 4), target_shape = (3, -1), + then output_shape will be (3, 8). + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + + # Arguments + target_shape: A shape tuple. The target shape that you desire to have. Batch dimension should be excluded. + input_shape: A shape tuple, not including batch. + + >>> reshape = Reshape((2, 10), input_shape=(5, 4)) + creating: createKerasReshape + """ + def __init__(self, target_shape, input_shape=None, bigdl_type="float"): + super(Reshape, self).__init__(None, bigdl_type, + target_shape, + list(input_shape) if input_shape else None) + + +class Activation(KerasLayer): + """ + Simple activation function to be applied to the output. + Available activations: 'tanh', 'relu', 'sigmoid', 'softmax', 'softplus', 'softsign', 'hard_sigmoid'. + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + + # Arguments + activation: Name of the activation function as string. + input_shape: A shape tuple, not including batch. + + >>> activation = Activation("relu", input_shape=(3, 4)) + creating: createKerasActivation + """ + def __init__(self, activation, input_shape=None, bigdl_type="float"): + super(Activation, self).__init__(None, bigdl_type, + activation, + list(input_shape) if input_shape else None) + + +class RepeatVector(KerasLayer): + """ + Repeats the input n times. + The input of this layer should be 2D. + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + + # Arguments + n: Repetition factor. Int. + input_shape: A shape tuple, not including batch. + + >>> repeatvector = RepeatVector(5, input_shape=(3, )) + creating: createKerasRepeatVector + """ + def __init__(self, n, input_shape=None, bigdl_type="float"): + super(RepeatVector, self).__init__(None, bigdl_type, + n, + list(input_shape) if input_shape else None) + + +class Permute(KerasLayer): + """ + Permutes the dimensions of the input according to a given pattern. + Useful for connecting RNNs and convnets together. + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + + # Arguments + dims: Tuple of int. Permutation pattern, does not include the samples dimension. Indexing starts at 1. + input_shape: A shape tuple, not including batch. + + >>> permute = Permute((2, 1, 3), input_shape=(3, 4, 5)) + creating: createKerasPermute + """ + def __init__(self, dims, input_shape=None, bigdl_type="float"): + super(Permute, self).__init__(None, bigdl_type, + dims, + list(input_shape) if input_shape else None) + + +class Highway(KerasLayer): + """ + Densely connected highway network. Highway layers are a natural extension of LSTMs to feedforward networks. + The input of this layer should be 2D, i.e. (batch, input dim). + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + + # Arguments + activation: String representations of activation function to use (such as 'relu' or 'sigmoid'). + Default is None. + W_regularizer: An instance of [[Regularizer]], (eg. L1 or L2 regularization), + applied to the input weights matrices. Default is None. + b_regularizer: An instance of [[Regularizer]], applied to the bias. Default is None. + bias: Whether to include a bias (i.e. make the layer affine rather than linear). + Default is True. + input_shape: A shape tuple, not including batch. + + >>> highway = Highway(activation='relu', input_shape=(8, )) + creating: createKerasHighway + """ + def __init__(self, activation=None, W_regularizer=None, b_regularizer=None, + bias=True, input_shape=None, bigdl_type="float"): + super(Highway, self).__init__(None, bigdl_type, + activation, + W_regularizer, + b_regularizer, + bias, + list(input_shape) if input_shape else None) + + +class Convolution1D(KerasLayer): + """ + Applies convolution operator for filtering neighborhoods of 1D inputs. + You can also use Conv1D as an alias of this layer. + The input of this layer should be 3D. + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + + # Arguments + nb_filter: Number of convolution filters to use. + filter_length: The extension (spatial or temporal) of each filter. + init: String representations of initialization method for the weights of the layer. + Default is 'glorot_uniform'. + activation: String representations of activation function to use (such as 'relu' or 'sigmoid'). + Default is None. + border_mode: Either 'valid' or 'same'. Default is 'valid'. + subsample_length: Factor by which to subsample output. Int. Default is 1. + W_regularizer: An instance of [[Regularizer]], (eg. L1 or L2 regularization), + applied to the input weights matrices. Default is None. + b_regularizer: An instance of [[Regularizer]], applied to the bias. Default is None. + bias: Whether to include a bias (i.e. make the layer affine rather than linear). + Default is True. + input_shape: A shape tuple, not including batch. + + >>> conv1d = Convolution1D(12, 4, input_shape=(3, 16)) + creating: createKerasConvolution1D + """ + def __init__(self, nb_filter, filter_length, init="glorot_uniform", + activation=None, border_mode="valid", subsample_length=1, + W_regularizer=None, b_regularizer=None, bias=True, + input_shape=None, bigdl_type="float"): + super(Convolution1D, self).__init__(None, bigdl_type, + nb_filter, + filter_length, + init, + activation, + border_mode, + subsample_length, + W_regularizer, + b_regularizer, + bias, + list(input_shape) if input_shape else None) + + +class Convolution2D(KerasLayer): + """ + Applies a 2D convolution over an input image composed of several input planes. + You can also use Conv2D as an alias of this layer. + The input of this layer should be 4D. + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + e.g. input_shape=(3, 128, 128) for 128x128 RGB pictures. + + # Arguments + nb_filter: Number of convolution filters to use. + nb_row: Number of rows in the convolution kernel. + nb_col: Number of cols in the convolution kernel. + init: String representations of initialization method for the weights of the layer. + Default is 'glorot_uniform'. + activation: String representations of activation function to use (such as 'relu' or 'sigmoid'). + Default is None. + border_mode: Either 'valid' or 'same'. Default is 'valid'. + subsample: Int tuple of length 2 corresponding to the step of the convolution in the + height and width dimension. Also called strides elsewhere. Default is (1, 1). + dim_ordering: Format of input data. Either 'th' (Channel First) or 'tf' (Channel Last). Default is 'th'. + W_regularizer: An instance of [[Regularizer]], (eg. L1 or L2 regularization), + applied to the input weights matrices. Default is None. + b_regularizer: An instance of [[Regularizer]], applied to the bias. Default is None. + bias: Whether to include a bias (i.e. make the layer affine rather than linear). + Default is True. + input_shape: A shape tuple, not including batch. + + >>> conv2d = Convolution2D(32, 3, 3, input_shape=(3, 128, 128)) + creating: createKerasConvolution2D + """ + def __init__(self, nb_filter, nb_row, nb_col, + init="glorot_uniform", activation=None, + border_mode="valid", subsample=(1, 1), dim_ordering="th", + W_regularizer=None, b_regularizer=None, bias=True, + input_shape=None, bigdl_type="float"): + super(Convolution2D, self).__init__(None, bigdl_type, + nb_filter, + nb_row, + nb_col, + init, + activation, + border_mode, + subsample, + dim_ordering, + W_regularizer, + b_regularizer, + bias, + list(input_shape) if input_shape else None) + + +class Convolution3D(KerasLayer): + """ + Applies convolution operator for filtering windows of three-dimensional inputs. + You can also use Conv3D as an alias of this layer. + Data format currently supported for this layer is dim_ordering='th' (Channel First). + The input of this layer should be 5D. + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + + # Arguments + nb_filter: Number of convolution filters to use. + kernel_dim1: Length of the first dimension in the convolution kernel. + kernel_dim2: Length of the second dimension in the convolution kernel. + kernel_dim3: Length of the third dimension in the convolution kernel. + init: String representations of initialization method for the weights of the layer. + Default is 'glorot_uniform'. + activation: String representations of activation function to use (such as 'relu' or 'sigmoid'). + Default is None. + border_mode: Either 'valid' or 'same'. Default is 'valid'. + subsample: Int tuple of length 3. Factor by which to subsample output. + Also called strides elsewhere. Default is (1, 1, 1). + dim_ordering: Format of input data. Only 'th' (Channel First) is supported for now. + W_regularizer: An instance of [[Regularizer]], (eg. L1 or L2 regularization), + applied to the input weights matrices. Default is None. + b_regularizer: An instance of [[Regularizer]], applied to the bias. Default is None. + bias: Whether to include a bias (i.e. make the layer affine rather than linear). + Default is True. + input_shape: A shape tuple, not including batch. + + >>> conv3d = Convolution3D(32, 3, 4, 5, input_shape=(3, 64, 64, 64)) + creating: createKerasConvolution3D + """ + def __init__(self, nb_filter, kernel_dim1, kernel_dim2, kernel_dim3, + init="glorot_uniform", activation=None, border_mode="valid", + subsample=(1, 1, 1), dim_ordering="th", W_regularizer=None, + b_regularizer=None, bias=True, input_shape=None, bigdl_type="float"): + super(Convolution3D, self).__init__(None, bigdl_type, + nb_filter, + kernel_dim1, + kernel_dim2, + kernel_dim3, + init, + activation, + border_mode, + subsample, + dim_ordering, + W_regularizer, + b_regularizer, + bias, + list(input_shape) if input_shape else None) + + +class AtrousConvolution1D(KerasLayer): + """ + Applies an atrous convolution operator for filtering neighborhoods of 1D inputs. + A.k.a dilated convolution or convolution with holes. + Border mode currently supported for this layer is 'valid'. + Bias will be included in this layer. + You can also use AtrousConv1D as an alias of this layer. + The input of this layer should be 3D. + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + + # Arguments + nb_filter: Number of convolution filters to use. + filter_length: The extension (spatial or temporal) of each filter. + init: String representations of initialization method for the weights of the layer. + Default is 'glorot_uniform'. + activation: String representations of activation function to use (such as 'relu' or 'sigmoid'). + Default is None. + border_mode: Only 'valid' is supported for now. + subsample_length: Factor by which to subsample output. Int. Default is 1. + atrous_rate: Factor for kernel dilation. Also called filter_dilation elsewhere. Int. Default is 1. + W_regularizer: An instance of [[Regularizer]], (eg. L1 or L2 regularization), + applied to the input weights matrices. Default is None. + b_regularizer: An instance of [[Regularizer]], applied to the bias. Default is None. + bias: Only 'True' is supported for now. + input_shape: A shape tuple, not including batch. + + >>> atrousconv1d = AtrousConvolution1D(8, 3, input_shape=(3, 12)) + creating: createKerasAtrousConvolution1D + """ + def __init__(self, nb_filter, filter_length, init="glorot_uniform", + activation=None, border_mode='valid', subsample_length=1, atrous_rate=1, + W_regularizer=None, b_regularizer=None, bias=True, input_shape=None, bigdl_type="float"): + if border_mode != "valid": + raise ValueError("For AtrousConvolution1D, only border_mode='valid' is supported for now") + if not bias: + raise ValueError("For AtrousConvolution1D, only bias=True is supported for now") + super(AtrousConvolution1D, self).__init__(None, bigdl_type, + nb_filter, + filter_length, + init, + activation, + subsample_length, + atrous_rate, + W_regularizer, + b_regularizer, + list(input_shape) if input_shape else None) + + +class AtrousConvolution2D(KerasLayer): + """ + Applies an atrous Convolution operator for filtering windows of 2D inputs. + A.k.a dilated convolution or convolution with holes. + Data format currently supported for this layer is dim_ordering='th' (Channel First). + Border mode currently supported for this layer is 'valid'. + Bias will be included in this layer. + You can also use AtrousConv2D as an alias of this layer. + The input of this layer should be 4D. + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + e.g. input_shape=(3, 128, 128) for 128x128 RGB pictures. + + # Arguments + nb_filter: Number of convolution filters to use. + nb_row: Number of rows in the convolution kernel. + nb_col: Number of cols in the convolution kernel. + init: String representations of initialization method for the weights of the layer. + Default is 'glorot_uniform'. + activation: String representations of activation function to use (such as 'relu' or 'sigmoid'). + Default is None. + border_mode: Only 'valid' is supported for now. + subsample: Int tuple of length 2 corresponding to the step of the convolution in the + height and width dimension. Also called strides elsewhere. Default is (1, 1). + atrous_rate: Int tuple of length 2. Factor for kernel dilation. + Also called filter_dilation elsewhere. Default is (1, 1). + dim_ordering: Format of input data. Only 'th' (Channel First) is supported for now. + W_regularizer: An instance of [[Regularizer]], (eg. L1 or L2 regularization), + applied to the input weights matrices. Default is None. + b_regularizer: An instance of [[Regularizer]], applied to the bias. Default is None. + bias: Only 'True' is supported for now. + input_shape: A shape tuple, not including batch. + + >>> atrousconv2d = AtrousConvolution2D(12, 4, 3, input_shape=(3, 64, 64)) + creating: createKerasAtrousConvolution2D + """ + def __init__(self, nb_filter, nb_row, nb_col, init="glorot_uniform", + activation=None, border_mode="valid", subsample=(1, 1), + atrous_rate=(1, 1), dim_ordering="th", W_regularizer=None, + b_regularizer=None, bias=True, input_shape=None, bigdl_type="float"): + if border_mode != "valid": + raise ValueError("For AtrousConvolution2D, only border_mode='valid' is supported for now") + if not bias: + raise ValueError("For AtrousConvolution2D, only bias=True is supported for now") + super(AtrousConvolution2D, self).__init__(None, bigdl_type, + nb_filter, + nb_row, + nb_col, + init, + activation, + subsample, + atrous_rate, + dim_ordering, + W_regularizer, + b_regularizer, + list(input_shape) if input_shape else None) + + +class Deconvolution2D(KerasLayer): + """ + Transposed convolution operator for filtering windows of 2D inputs. + The need for transposed convolutions generally arises from the desire to use a transformation + going in the opposite direction of a normal convolution, i.e., from something that has + the shape of the output of some convolution to something that has the shape of its input + while maintaining a connectivity pattern that is compatible with said convolution. + Data format currently supported for this layer is dim_ordering='th' (Channel First). + Border mode currently supported for this layer is 'valid'. + You can also use Deconv2D as an alias of this layer. + The input of this layer should be 4D. + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + e.g. input_shape=(3, 128, 128) for 128x128 RGB pictures. + + # Arguments + nb_filter: Number of transposed convolution filters to use. + nb_row: Number of rows in the convolution kernel. + nb_col: Number of cols in the convolution kernel. + output_shape: Output shape of the transposed convolution operation. Tuple of int. + init: String representations of initialization method for the weights of the layer. + Default is 'glorot_uniform'. + activation: String representations of activation function to use (such as 'relu' or 'sigmoid'). + Default is None. + border_mode: Only 'valid' is supported for now. + subsample: Int tuple of length 2 corresponding to the step of the convolution in the + height and width dimension. Also called strides elsewhere. Default is (1, 1). + dim_ordering: Format of input data. Only 'th' (Channel First) is supported for now. + W_regularizer: An instance of [[Regularizer]], (eg. L1 or L2 regularization), + applied to the input weights matrices. Default is None. + b_regularizer: An instance of [[Regularizer]], applied to the bias. Default is None. + bias: Whether to include a bias (i.e. make the layer affine rather than linear). + Default is True. + input_shape: A shape tuple, not including batch. + + >>> deconv2d = Deconvolution2D(3, 3, 3, output_shape=(None, 3, 14, 14), input_shape=(3, 12, 12)) + creating: createKerasDeconvolution2D + """ + def __init__(self, nb_filter, nb_row, nb_col, output_shape, init="glorot_uniform", + activation=None, border_mode="valid", subsample=(1, 1), dim_ordering="th", + W_regularizer=None, b_regularizer=None, bias=True, input_shape=None, bigdl_type="float"): + if border_mode != "valid": + raise ValueError("For Deconvolution2D, only border_mode='valid' is supported for now") + super(Deconvolution2D, self).__init__(None, bigdl_type, + nb_filter, + nb_row, + nb_col, + init, + activation, + subsample, + dim_ordering, + W_regularizer, + b_regularizer, + bias, + list(input_shape) if input_shape else None) + + +class SeparableConvolution2D(KerasLayer): + """ + Applies separable convolution operator for 2D inputs. + Separable convolutions consist in first performing a depthwise spatial convolution (which acts + on each input channel separately) followed by a pointwise convolution which mixes together the + resulting output channels. The depthMultiplier argument controls how many output channels are + generated per input channel in the depthwise step. + You can also use SeparableConv2D as an alias of this layer. + The input of this layer should be 4D. + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + e.g. input_shape=(3, 128, 128) for 128x128 RGB pictures. + + # Arguments + nb_filter: Number of convolution filters to use. + nb_row: Number of rows in the convolution kernel. + nb_col: Number of cols in the convolution kernel. + init: String representations of initialization method for the weights of the layer. + Default is 'glorot_uniform'. + activation: String representations of activation function to use (such as 'relu' or 'sigmoid'). + Default is None. + border_mode: Either 'valid' or 'same'. Default is 'valid'. + subsample: Int tuple of length 2 corresponding to the step of the convolution in the + height and width dimension. Also called strides elsewhere. Default is (1, 1). + depth_multiplier: How many output channel to use per input channel for the depthwise convolution step. + Int. Default is 1. + dim_ordering: Format of input data. Either 'th' (Channel First) or 'tf' (Channel Last). Default is 'th'. + depthwise_regularizer: An instance of [[Regularizer]], (eg. L1 or L2 regularization), + applied to the depthwise weights matrices. Default is None. + pointwise_regularizer: An instance of [[Regularizer]], applied to the pointwise weights matrices. + Default is None. + b_regularizer: An instance of [[Regularizer]], applied to the bias. Default is None. + bias: Whether to include a bias (i.e. make the layer affine rather than linear). + Default is True. + input_shape: A shape tuple, not including batch. + + >>> separableconv2d = SeparableConvolution2D(12, 3, 4, input_shape=(3, 32, 32)) + creating: createKerasSeparableConvolution2D + """ + def __init__(self, nb_filter, nb_row, nb_col, init="glorot_uniform", + activation=None, border_mode="valid", subsample=(1, 1), depth_multiplier=1, + dim_ordering="th", depthwise_regularizer=None, pointwise_regularizer=None, + b_regularizer=None, bias=True, input_shape=None, bigdl_type="float"): + super(SeparableConvolution2D, self).__init__(None, bigdl_type, + nb_filter, + nb_row, + nb_col, + init, + activation, + border_mode, + subsample, + depth_multiplier, + dim_ordering, + depthwise_regularizer, + pointwise_regularizer, + b_regularizer, + bias, + list(input_shape) if input_shape else None) + + +Conv1D = Convolution1D +Conv2D = Convolution2D +Conv3D = Convolution3D +Deconv2D = Deconvolution2D +AtrousConv1D = AtrousConvolution1D +AtrousConv2D = AtrousConvolution2D +SeparableConv2D = SeparableConvolution2D + + +class Cropping1D(KerasLayer): + """ + Cropping layer for 1D input (e.g. temporal sequence). + The input of this layer should be 3D. + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + + # Arguments + cropping: Int tuple of length 2. How many units should be trimmed off at the beginning and + end of the cropping dimension. Default is (1, 1). + input_shape: A shape tuple, not including batch. + + >>> cropping1d = Cropping1D(cropping=(1, 2), input_shape=(8, 8)) + creating: createKerasCropping1D + """ + def __init__(self, cropping=(1, 1), input_shape=None, bigdl_type="float"): + super(Cropping1D, self).__init__(None, bigdl_type, + cropping, + list(input_shape) if input_shape else None) + + +class Cropping2D(KerasLayer): + """ + Cropping layer for 2D input (e.g. picture). + The input of this layer should be 4D. + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + + # Arguments + cropping: Int tuple of tuple of length 2. How many units should be trimmed off at the beginning and + end of the 2 cropping dimensions (i.e. height and width). Default is ((0, 0), (0, 0)). + input_shape: A shape tuple, not including batch. + + >>> cropping2d = Cropping2D(cropping=((1, 2), (0, 1)), input_shape=(12, 12, 12)) + creating: createKerasCropping2D + """ + def __init__(self, cropping=((0, 0), (0, 0)), dim_ordering="th", + input_shape=None, bigdl_type="float"): + super(Cropping2D, self).__init__(None, bigdl_type, + cropping[0], + cropping[1], + dim_ordering, + list(input_shape) if input_shape else None) + + +class Cropping3D(KerasLayer): + """ + Cropping layer for 3D data (e.g. spatial or spatio-temporal). + The input of this layer should be 5D. + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + + # Arguments + cropping: Int tuple of tuple of length 3. How many units should be trimmed off at the beginning and + end of the 3 cropping dimensions (i.e. kernel_dim1, kernel_dim2 and kernel_dim3). + Default is ((1, 1), (1, 1), (1, 1)). + input_shape: A shape tuple, not including batch. + + >>> cropping3d = Cropping3D(cropping=((0, 2), (1, 1), (3, 1)), input_shape=(4, 12, 12, 16)) + creating: createKerasCropping3D + """ + def __init__(self, cropping=((1, 1), (1, 1), (1, 1)), dim_ordering="th", + input_shape=None, bigdl_type="float"): + super(Cropping3D, self).__init__(None, bigdl_type, + cropping[0], + cropping[1], + cropping[2], + dim_ordering, + list(input_shape) if input_shape else None) + + +class UpSampling1D(KerasLayer): + """ + UpSampling layer for 1D inputs. + Repeats each temporal step 'length' times along the time axis. + The input of this layer should be 3D. + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + + # Arguments + length: Int. UpSampling factor. Default is 2. + input_shape: A shape tuple, not including batch. + + >>> upsampling1d = UpSampling1D(length=3, input_shape=(3, 12)) + creating: createKerasUpSampling1D + """ + def __init__(self, length=2, input_shape=None, bigdl_type="float"): + super(UpSampling1D, self).__init__(None, bigdl_type, + length, + list(input_shape) if input_shape else None) + + +class UpSampling2D(KerasLayer): + """ + UpSampling layer for 2D inputs. + Repeats the rows and columns of the data by size[0] and size[1] respectively. + The input of this layer should be 4D. + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + + # Arguments + size: Int tuple of length 2. UpSampling factors for rows and columns. Default is (2, 2). + dim_ordering: Format of input data. Either 'th' (Channel First) or 'tf' (Channel Last). Default is 'th'. + input_shape: A shape tuple, not including batch. + + >>> upsampling2d = UpSampling2D(size=(1, 3), input_shape=(3, 16, 16)) + creating: createKerasUpSampling2D + """ + def __init__(self, size=(2, 2), dim_ordering="th", input_shape=None, bigdl_type="float"): + super(UpSampling2D, self).__init__(None, bigdl_type, + size, + dim_ordering, + list(input_shape) if input_shape else None) + + +class UpSampling3D(KerasLayer): + """ + UpSampling layer for 2D inputs. + Repeats the 1st, 2nd and 3rd dimensions of the data by size[0], size[1] and size[2] respectively. + Data format currently supported for this layer is dim_ordering='th' (Channel First). + The input of this layer should be 5D. + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + + # Arguments + size: Int tuple of length 3. UpSampling factors for dim1, dim2 and dim3. Default is (2, 2, 2). + dim_ordering: Format of input data. Only 'th' (Channel First) is supported for now. + input_shape: A shape tuple, not including batch. + + >>> upsampling3d = UpSampling3D(size=(1, 2, 3), input_shape=(3, 16, 16, 16)) + creating: createKerasUpSampling3D + """ + def __init__(self, size=(2, 2, 2), dim_ordering="th", input_shape=None, bigdl_type="float"): + super(UpSampling3D, self).__init__(None, bigdl_type, + size, + dim_ordering, + list(input_shape) if input_shape else None) + + +class ZeroPadding1D(KerasLayer): + """ + Zero-padding layer for 1D input (e.g. temporal sequence). + The input of this layer should be 3D. + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + + # Arguments + padding: Int or int tuple of length 2. + If int, how many zeros to add both at the beginning and at the end of the padding dimension. + If tuple of length 2, how many zeros to add in the order '(left_pad, right_pad)'. + Default is 1. + input_shape: A shape tuple, not including batch. + + >>> zeropadding1d = ZeroPadding1D(padding=2, input_shape=(3, 6)) + creating: createKerasZeroPadding1D + """ + def __init__(self, padding=1, input_shape=None, bigdl_type="float"): + if isinstance(padding, int): + padding = (padding, padding) + super(ZeroPadding1D, self).__init__(None, bigdl_type, + padding, + list(input_shape) if input_shape else None) + + +class ZeroPadding2D(KerasLayer): + """ + Zero-padding layer for 2D input (e.g. picture). + The input of this layer should be 4D. + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + + # Arguments + padding: Int tuple of length 2 or length 4. + If tuple of length 2, how many zeros to add both at the beginning and at the end of rows and cols. + If tuple of length 4, how many zeros to add in the order '(top_pad, bottom_pad, left_pad, right_pad)'. + Default is (1, 1). + dim_ordering: Format of input data. Either 'th' (Channel First) or 'tf' (Channel Last). Default is 'th'. + input_shape: A shape tuple, not including batch. + + >>> zeropadding2d = ZeroPadding2D(padding=(2, 1), input_shape=(2, 8, 8)) + creating: createKerasZeroPadding2D + """ + def __init__(self, padding=(1, 1), dim_ordering="th", input_shape=None, bigdl_type="float"): + if len(padding) == 2: + padding = (padding[0], padding[0], padding[1], padding[1]) + super(ZeroPadding2D, self).__init__(None, bigdl_type, + padding, + dim_ordering, + list(input_shape) if input_shape else None) + + +class ZeroPadding3D(KerasLayer): + """ + Zero-padding layer for 3D data (spatial or spatio-temporal). + The input of this layer should be 5D. + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + + # Arguments + padding: Int tuple of length 3. How many zeros to add at the beginning and at the end of the 3 padding dimensions. + Symmetric padding will be applied to each dimension. Default is (1, 1, 1). + dim_ordering: Format of input data. Either 'th' (Channel First) or 'tf' (Channel Last). Default is 'th'. + input_shape: A shape tuple, not including batch. + + >>> zeropadding3d = ZeroPadding3D(padding=(2, 1, 2), input_shape=(2, 8, 8, 10)) + creating: createKerasZeroPadding3D + """ + def __init__(self, padding=(1, 1, 1), dim_ordering="th", input_shape=None, bigdl_type="float"): + super(ZeroPadding3D, self).__init__(None, bigdl_type, + padding, + dim_ordering, + list(input_shape) if input_shape else None) + + +class MaxPooling1D(KerasLayer): + """ + Applies max pooling operation for temporal data. + The input of this layer should be 3D. + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + + # Arguments + pool_length: Size of the region to which max pooling is applied. + strides: Factor by which to downscale. 2 will halve the input. + Default is None, and in this case it will be equal to pool_length.. + border_mode: Either 'valid' or 'same'. Default is 'valid'. + input_shape: A shape tuple, not including batch. + + >>> maxpooling1d = MaxPooling1D(3, input_shape=(3, 24)) + creating: createKerasMaxPooling1D + """ + def __init__(self, pool_length=2, stride=None, border_mode="valid", + input_shape=None, bigdl_type="float"): + if not stride: + stride = -1 + super(MaxPooling1D, self).__init__(None, bigdl_type, + pool_length, + stride, + border_mode, + list(input_shape) if input_shape else None) + + +class MaxPooling2D(KerasLayer): + """ + Applies max pooling operation for spatial data. + The input of this layer should be 4D. + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + + # Arguments + pool_size: Int tuple of length 2 corresponding to the downscale vertically and horizontally. + Default is (2, 2), which will halve the image in each dimension. + strides: Int tuple of length 2. Stride values. Default is None, and in this case it will be equal to pool_size. + border_mode: Either 'valid' or 'same'. Default is 'valid'. + dim_ordering: Format of input data. Either 'th' (Channel First) or 'tf' (Channel Last). Default is 'th'. + input_shape: A shape tuple, not including batch. + + >>> maxpooling2d = MaxPooling2D((2, 2), input_shape=(3, 32, 32)) + creating: createKerasMaxPooling2D + """ + def __init__(self, pool_size=(2, 2), strides=None, + border_mode='valid', dim_ordering='th', + input_shape=None, bigdl_type="float"): + super(MaxPooling2D, self).__init__(None, bigdl_type, + pool_size, + strides, + border_mode, + dim_ordering, + list(input_shape) if input_shape else None) + + +class MaxPooling3D(KerasLayer): + """ + Applies max pooling operation for 3D data (spatial or spatio-temporal). + Data format currently supported for this layer is dim_ordering='th' (Channel First). + Border mode currently supported for this layer is 'valid'. + The input of this layer should be 5D. + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + + # Arguments + pool_size: Int tuple of length 3. Factors by which to downscale (dim1, dim2, dim3). + Default is (2, 2, 2), which will halve the image in each dimension. + strides: Int tuple of length 3. Stride values. Default is None, and in this case it will be equal to pool_size. + border_mode: Only 'valid' is supported for now. + dim_ordering: Format of input data. Only 'th' (Channel First) is supported for now. + input_shape: A shape tuple, not including batch. + + >>> maxpooling3d = MaxPooling3D((2, 1, 3), input_shape=(3, 32, 32, 32)) + creating: createKerasMaxPooling3D + """ + def __init__(self, pool_size=(2, 2, 2), strides=None, border_mode="valid", + dim_ordering="th", input_shape=None, bigdl_type="float"): + if border_mode != "valid": + raise ValueError("For MaxPooling3D, only border_mode='valid' is supported for now") + super(MaxPooling3D, self).__init__(None, bigdl_type, + pool_size, + strides, + dim_ordering, + list(input_shape) if input_shape else None) + + +class AveragePooling1D(KerasLayer): + """ + Applies average pooling operation for temporal data. + The input of this layer should be 3D. + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + + # Arguments + pool_length: Size of the region to which max pooling is applied. + strides: Factor by which to downscale. 2 will halve the input. + Default is None, and in this case it will be equal to pool_length.. + border_mode: Either 'valid' or 'same'. Default is 'valid'. + input_shape: A shape tuple, not including batch. + + >>> averagepooling1d = AveragePooling1D(input_shape=(3, 24)) + creating: createKerasAveragePooling1D + """ + def __init__(self, pool_length=2, stride=None, border_mode="valid", + input_shape=None, bigdl_type="float"): + if not stride: + stride = -1 + super(AveragePooling1D, self).__init__(None, bigdl_type, + pool_length, + stride, + border_mode, + list(input_shape) if input_shape else None) + + +class AveragePooling2D(KerasLayer): + """ + Applies average pooling operation for spatial data. + The input of this layer should be 4D. + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + + # Arguments + pool_size: Int tuple of length 2 corresponding to the downscale vertically and horizontally. + Default is (2, 2), which will halve the image in each dimension. + strides: Int tuple of length 2. Stride values. Default is None, and in this case it will be equal to pool_size. + border_mode: Either 'valid' or 'same'. Default is 'valid'. + dim_ordering: Format of input data. Either 'th' (Channel First) or 'tf' (Channel Last). Default is 'th'. + input_shape: A shape tuple, not including batch. + + >>> averagepooling2d = AveragePooling2D((1, 2), input_shape=(2, 28, 32)) + creating: createKerasAveragePooling2D + """ + def __init__(self, pool_size=(2, 2), strides=None, border_mode="valid", + dim_ordering="th", input_shape=None, bigdl_type="float"): + super(AveragePooling2D, self).__init__(None, bigdl_type, + pool_size, + strides, + border_mode, + dim_ordering, + list(input_shape) if input_shape else None) + + +class AveragePooling3D(KerasLayer): + """ + Applies average pooling operation for 3D data (spatial or spatio-temporal). + Data format currently supported for this layer is dim_ordering='th' (Channel First). + Border mode currently supported for this layer is 'valid'. + The input of this layer should be 5D. + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + + # Arguments + pool_size: Int tuple of length 3. Factors by which to downscale (dim1, dim2, dim3). + Default is (2, 2, 2), which will halve the image in each dimension. + strides: Int tuple of length 3. Stride values. Default is None, and in this case it will be equal to pool_size. + border_mode: Only 'valid' is supported for now. + dim_ordering: Format of input data. Only 'th' (Channel First) is supported for now. + input_shape: A shape tuple, not including batch. + + >>> averagepooling3d = AveragePooling3D((1, 1, 2), input_shape=(3, 28, 32, 36)) + creating: createKerasAveragePooling3D + """ + def __init__(self, pool_size=(2, 2, 2), strides=None, border_mode="valid", + dim_ordering="th", input_shape=None, bigdl_type="float"): + if border_mode != "valid": + raise ValueError("For AveragePooling3D, only border_mode='valid' is supported for now") + super(AveragePooling3D, self).__init__(None, bigdl_type, + pool_size, + strides, + dim_ordering, + list(input_shape) if input_shape else None) + + +class GlobalMaxPooling1D(KerasLayer): + """ + Applies global max pooling operation for temporal data. + The input of this layer should be 3D. + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + + # Arguments + input_shape: A shape tuple, not including batch. + + >>> globalmaxpooling1d = GlobalMaxPooling1D(input_shape=(4, 8)) + creating: createKerasGlobalMaxPooling1D + """ + def __init__(self, input_shape=None, bigdl_type="float"): + super(GlobalMaxPooling1D, self).__init__(None, bigdl_type, + list(input_shape) if input_shape else None) + + +class GlobalAveragePooling1D(KerasLayer): + """ + Applies global average pooling operation for temporal data. + The input of this layer should be 3D. + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + + # Arguments + input_shape: A shape tuple, not including batch. + + >>> globalaveragepooling1d = GlobalAveragePooling1D(input_shape=(12, 12)) + creating: createKerasGlobalAveragePooling1D + """ + def __init__(self, input_shape=None, bigdl_type="float"): + super(GlobalAveragePooling1D, self).__init__(None, bigdl_type, + list(input_shape) if input_shape else None) + + +class GlobalMaxPooling2D(KerasLayer): + """ + Applies global max pooling operation for spatial data. + The input of this layer should be 4D. + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + + # Arguments + dim_ordering: Format of input data. Either 'th' (Channel First) or 'tf' (Channel Last). Default is 'th'. + input_shape: A shape tuple, not including batch. + + >>> globalmaxpooling2d = GlobalMaxPooling2D(input_shape=(4, 32, 32)) + creating: createKerasGlobalMaxPooling2D + """ + def __init__(self, dim_ordering="th", input_shape=None, bigdl_type="float"): + super(GlobalMaxPooling2D, self).__init__(None, bigdl_type, + dim_ordering, + list(input_shape) if input_shape else None) + + +class GlobalAveragePooling2D(KerasLayer): + """ + Applies global average pooling operation for spatial data. + The input of this layer should be 4D. + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + + # Arguments + dim_ordering: Format of input data. Either 'th' (Channel First) or 'tf' (Channel Last). Default is 'th'. + input_shape: A shape tuple, not including batch. + + >>> globalaveragepooling2d = GlobalAveragePooling2D(input_shape=(4, 32, 32)) + creating: createKerasGlobalAveragePooling2D + """ + def __init__(self, dim_ordering="th", input_shape=None, bigdl_type="float"): + super(GlobalAveragePooling2D, self).__init__(None, bigdl_type, + dim_ordering, + list(input_shape) if input_shape else None) + + +class GlobalMaxPooling3D(KerasLayer): + """ + Applies global max pooling operation for 3D data. + Data format currently supported for this layer is dim_ordering='th' (Channel First). + Border mode currently supported for this layer is 'valid'. + The input of this layer should be 5D. + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + + # Arguments + dim_ordering: Format of input data. Only 'th' (Channel First) is supported for now. + input_shape: A shape tuple, not including batch. + + >>> globalmaxpooling3d = GlobalMaxPooling3D(input_shape=(4, 32, 32, 32)) + creating: createKerasGlobalMaxPooling3D + """ + def __init__(self, dim_ordering="th", input_shape=None, bigdl_type="float"): + super(GlobalMaxPooling3D, self).__init__(None, bigdl_type, + dim_ordering, + list(input_shape) if input_shape else None) + + +class GlobalAveragePooling3D(KerasLayer): + """ + Applies global average pooling operation for 3D data. + Data format currently supported for this layer is dim_ordering='th' (Channel First). + Border mode currently supported for this layer is 'valid'. + The input of this layer should be 5D. + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + + # Arguments + dim_ordering: Format of input data. Only 'th' (Channel First) is supported for now. + input_shape: A shape tuple, not including batch. + + >>> globalaveragepooling3d = GlobalAveragePooling3D(input_shape=(4, 16, 16, 20)) + creating: createKerasGlobalAveragePooling3D + """ + def __init__(self, dim_ordering="th", input_shape=None, bigdl_type="float"): + super(GlobalAveragePooling3D, self).__init__(None, bigdl_type, + dim_ordering, + list(input_shape) if input_shape else None) + + +class SimpleRNN(KerasLayer): + """ + A fully-connected recurrent neural network cell. The output is to be fed back to input. + The input of this layer should be 3D, i.e. (batch, time steps, input dim). + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + + # Arguments + output_dim: Hidden unit size. Dimension of internal projections and final output. + activation: String representations of activation function to use (such as 'relu' or 'sigmoid'). + Default is 'tanh'. + return_sequences: Whether to return the full sequence or only return the last output in the output sequence. + Default is False. + go_backwards: Whether the input sequence will be processed backwards. Default is False. + W_regularizer: An instance of [[Regularizer]], (eg. L1 or L2 regularization), + applied to the input weights matrices. Default is None. + U_regularizer: An instance of [[Regularizer]], applied the recurrent weights matrices. Default is None. + b_regularizer: An instance of [[Regularizer]], applied to the bias. Default is None. + input_shape: A shape tuple, not including batch. + + >>> simplernn = SimpleRNN(16, input_shape=(3, 32)) + creating: createKerasSimpleRNN + """ + def __init__(self, output_dim, activation="tanh", return_sequences=False, + go_backwards=False, W_regularizer=None, U_regularizer=None, + b_regularizer=None, input_shape=None, bigdl_type="float"): + super(SimpleRNN, self).__init__(None, bigdl_type, + output_dim, + activation, + return_sequences, + go_backwards, + W_regularizer, + U_regularizer, + b_regularizer, + list(input_shape) if input_shape else None) + + +class LSTM(KerasLayer): + """ + Long Short Term Memory unit architecture. + The input of this layer should be 3D, i.e. (batch, time steps, input dim). + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + + # Arguments + output_dim: Hidden unit size. Dimension of internal projections and final output. + activation: String representations of activation function to use (such as 'relu' or 'sigmoid'). + Default is 'tanh'. + inner_activation: String representations of activation function for inner cells. Default is 'hard_sigmoid'. + return_sequences: Whether to return the full sequence or only return the last output in the output sequence. + Default is False. + go_backwards: Whether the input sequence will be processed backwards. Default is False. + W_regularizer: An instance of [[Regularizer]], (eg. L1 or L2 regularization), + applied to the input weights matrices. Default is None. + U_regularizer: An instance of [[Regularizer]], applied the recurrent weights matrices. Default is None. + b_regularizer: An instance of [[Regularizer]], applied to the bias. Default is None. + input_shape: A shape tuple, not including batch. + + >>> lstm = LSTM(32, input_shape=(8, 16)) + creating: createKerasLSTM + """ + def __init__(self, output_dim, activation="tanh", inner_activation="hard_sigmoid", + return_sequences=False, go_backwards=False, W_regularizer=None, + U_regularizer=None, b_regularizer=None, input_shape=None, bigdl_type="float"): + super(LSTM, self).__init__(None, bigdl_type, + output_dim, + activation, + inner_activation, + return_sequences, + go_backwards, + W_regularizer, + U_regularizer, + b_regularizer, + list(input_shape) if input_shape else None) + + +class GRU(KerasLayer): + """ + Gated Recurrent Unit architecture. + The input of this layer should be 3D, i.e. (batch, time steps, input dim). + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + + # Arguments + output_dim: Hidden unit size. Dimension of internal projections and final output. + activation: String representations of activation function to use (such as 'relu' or 'sigmoid'). + Default is 'tanh'. + inner_activation: String representations of activation function for inner cells. Default is 'hard_sigmoid'. + return_sequences: Whether to return the full sequence or only return the last output in the output sequence. + Default is False. + go_backwards: Whether the input sequence will be processed backwards. Default is False. + W_regularizer: An instance of [[Regularizer]], (eg. L1 or L2 regularization), + applied to the input weights matrices. Default is None. + U_regularizer: An instance of [[Regularizer]], applied the recurrent weights matrices. Default is None. + b_regularizer: An instance of [[Regularizer]], applied to the bias. Default is None. + input_shape: A shape tuple, not including batch. + + >>> gru = GRU(24, input_shape=(32, 32)) + creating: createKerasGRU + """ + def __init__(self, output_dim, activation="tanh", inner_activation="hard_sigmoid", + return_sequences=False, go_backwards=False, W_regularizer=None, + U_regularizer=None, b_regularizer=None, input_shape=None, bigdl_type="float"): + super(GRU, self).__init__(None, bigdl_type, + output_dim, + activation, + inner_activation, + return_sequences, + go_backwards, + W_regularizer, + U_regularizer, + b_regularizer, + list(input_shape) if input_shape else None) + + +class ConvLSTM2D(KerasLayer): + """ + Convolutional LSTM. + Data format currently supported for this layer is dim_ordering='th' (Channel First). + Border mode currently supported for this layer is 'same'. + The convolution kernel for this layer is a square kernel with equal strides 'subsample'. + The input of this layer should be 5D. + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + + # Arguments + nb_filter: Number of convolution filters to use. + nb_row: Number of rows in the convolution kernel. + nb_col: Number of cols in the convolution kernel. Should be equal to nb_row as for a square kernel. + activation: String representations of activation function to use (such as 'relu' or 'sigmoid'). + Default is 'tanh'. + inner_activation: String representations of activation function for inner cells. Default is 'hard_sigmoid'. + dim_ordering: Format of input data. Only 'th' (Channel First) is supported for now. + border_mode: Only 'same' is supported for now. + subsample: Tuple of length 2. Factor by which to subsample output. Also called strides elsewhere. + Only support subsample[0] equal to subsample[1] for now. Default is (1, 1). + W_regularizer: An instance of [[Regularizer]], (eg. L1 or L2 regularization), + applied to the input weights matrices. Default is None. + U_regularizer: An instance of [[Regularizer]], applied the recurrent weights matrices. Default is None. + b_regularizer: An instance of [[Regularizer]], applied to the bias. Default is None. + return_sequences: Whether to return the full sequence or only return the last output in the output sequence. + Default is False. + go_backwards: Whether the input sequence will be processed backwards. Default is False. + input_shape: A shape tuple, not including batch. + + >>> convlstm2d = ConvLSTM2D(24, 3, 3, input_shape=(4, 32, 32, 32)) + creating: createKerasConvLSTM2D + """ + def __init__(self, nb_filter, nb_row, nb_col, activation="tanh", + inner_activation="hard_sigmoid", dim_ordering="th", border_mode="same", + subsample=(1, 1), W_regularizer=None, U_regularizer=None, b_regularizer=None, + return_sequences=False, go_backwards=False, input_shape=None, bigdl_type="float"): + if nb_row != nb_col: + raise ValueError("For ConvLSTM2D, only square kernel is supported for now") + if border_mode != "same": + raise ValueError("For ConvLSTM2D, only border_mode='same' is supported for now") + if subsample[0] != subsample[1]: + raise ValueError("For ConvLSTM2D, only equal strides is supported for now") + super(ConvLSTM2D, self).__init__(None, bigdl_type, + nb_filter, + nb_row, + activation, + inner_activation, + dim_ordering, + subsample[0], + W_regularizer, + U_regularizer, + b_regularizer, + return_sequences, + go_backwards, + list(input_shape) if input_shape else None) + + +class LocallyConnected1D(KerasLayer): + """ + Locally-connected layer for 1D inputs which works similarly to the TemporalConvolution layer, except that + weights are unshared, that is, a different set of filters is applied at each different patch of the input. + Border mode currently supported for this layer is 'valid'. + The input of this layer should be 3D. + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + + # Arguments + nb_filter: Dimensionality of the output. + filter_length: The extension (spatial or temporal) of each filter. + activation: String representations of activation function to use (such as 'relu' or 'sigmoid'). + Default is None. + border_mode: Only 'valid' is supported for now. + subsample_length: Factor by which to subsample output. Int. Default is 1. + W_regularizer: An instance of [[Regularizer]], (eg. L1 or L2 regularization), + applied to the input weights matrices. Default is None. + b_regularizer: An instance of [[Regularizer]], applied to the bias. Default is None. + bias: Whether to include a bias (i.e. make the layer affine rather than linear). + Default is True. + input_shape: A shape tuple, not including batch. + + >>> locallyconnected1d = LocallyConnected1D(6, 3, input_shape=(8, 12)) + creating: createKerasLocallyConnected1D + """ + def __init__(self, nb_filter, filter_length, activation=None, border_mode="valid", + subsample_length=1, W_regularizer=None, b_regularizer=None, + bias=True, input_shape=None, bigdl_type="float"): + if border_mode != "valid": + raise ValueError("For LocallyConnected1D, only border_mode='valid' is supported for now") + super(LocallyConnected1D, self).__init__(None, bigdl_type, + nb_filter, + filter_length, + activation, + subsample_length, + W_regularizer, + b_regularizer, + bias, + list(input_shape) if input_shape else None) + + +class LocallyConnected2D(KerasLayer): + """ + Locally-connected layer for 2D inputs that works similarly to the SpatialConvolution layer, except that + weights are unshared, that is, a different set of filters is applied at each different patch of the input. + The input of this layer should be 4D. + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + + # Arguments + nb_filter: Number of convolution filters to use. + nb_row: Number of rows in the convolution kernel. + nb_col: Number of cols in the convolution kernel. + activation: String representations of activation function to use (such as 'relu' or 'sigmoid'). + Default is None. + border_mode: Either 'valid' or 'same'. Default is 'valid'. + subsample: Int tuple of length 2 corresponding to the step of the convolution in the + height and width dimension. Also called strides elsewhere. Default is (1, 1). + dim_ordering: Format of input data. Either 'th' (Channel First) or 'tf' (Channel Last). Default is 'th'. + W_regularizer: An instance of [[Regularizer]], (eg. L1 or L2 regularization), + applied to the input weights matrices. Default is None. + b_regularizer: An instance of [[Regularizer]], applied to the bias. Default is None. + bias: Whether to include a bias (i.e. make the layer affine rather than linear). + Default is True. + input_shape: A shape tuple, not including batch. + + >>> locallyconnected2d = LocallyConnected2D(12, 3, 4, input_shape=(3, 128, 128)) + creating: createKerasLocallyConnected2D + """ + def __init__(self, nb_filter, nb_row, nb_col, activation=None, + border_mode="valid", subsample=(1, 1), dim_ordering="th", + W_regularizer=None, b_regularizer=None, bias=True, + input_shape=None, bigdl_type="float"): + super(LocallyConnected2D, self).__init__(None, bigdl_type, + nb_filter, + nb_row, + nb_col, + activation, + border_mode, + subsample, + dim_ordering, + W_regularizer, + b_regularizer, + bias, + list(input_shape) if input_shape else None) + + +class SpatialDropout1D(KerasLayer): + """ + Spatial 1D version of Dropout. + This version performs the same function as Dropout, however it drops entire 1D feature maps + instead of individual elements. If adjacent frames within feature maps are strongly correlated + (as is normally the case in early convolution layers) then regular dropout will not regularize the + activations and will otherwise just result in an effective learning rate decrease. + In this case, SpatialDropout1D will help promote independence between feature maps and should be used instead. + The input of this layer should be 3D. + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + + # Arguments + p: Fraction of the input units to drop. Float between 0 and 1. + input_shape: A shape tuple, not including batch. + + >>> spatialdropout1d = SpatialDropout1D(0.4, input_shape=(10, 12)) + creating: createKerasSpatialDropout1D + """ + def __init__(self, p=0.5, input_shape=None, bigdl_type="float"): + super(SpatialDropout1D, self).__init__(None, bigdl_type, + float(p), + list(input_shape) if input_shape else None) + + +class SpatialDropout2D(KerasLayer): + """ + Spatial 2D version of Dropout. + This version performs the same function as Dropout, however it drops entire 2D feature maps + instead of individual elements. If adjacent pixels within feature maps are strongly correlated + (as is normally the case in early convolution layers) then regular dropout will not regularize the + activations and will otherwise just result in an effective learning rate decrease. + In this case, SpatialDropout2D will help promote independence between feature maps and should be used instead. + The input of this layer should be 4D. + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + + # Arguments + p: Fraction of the input units to drop. Float between 0 and 1. + dim_ordering: Format of input data. Either 'th' (Channel First) or 'tf' (Channel Last). Default is 'th'. + input_shape: A shape tuple, not including batch. + + >>> spatialdropout2d = SpatialDropout2D(0.25, input_shape=(5, 12, 12)) + creating: createKerasSpatialDropout2D + """ + def __init__(self, p=0.5, dim_ordering="th", input_shape=None, bigdl_type="float"): + super(SpatialDropout2D, self).__init__(None, bigdl_type, + float(p), + dim_ordering, + list(input_shape) if input_shape else None) + + +class SpatialDropout3D(KerasLayer): + """ + Spatial 3D version of Dropout. + This version performs the same function as Dropout, however it drops entire 3D feature maps + instead of individual elements. If adjacent voxels within feature maps are strongly correlated + (as is normally the case in early convolution layers) then regular dropout will not regularize the + activations and will otherwise just result in an effective learning rate decrease. + In this case, SpatialDropout3D will help promote independence between feature maps and should be used instead. + The input of this layer should be 5D. + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + + # Arguments + p: Fraction of the input units to drop. Float between 0 and 1. + dim_ordering: Format of input data. Either 'th' (Channel First) or 'tf' (Channel Last). Default is 'th'. + input_shape: A shape tuple, not including batch. + + >>> spatialdropout3d = SpatialDropout3D(0.6, input_shape=(4, 12, 12, 16)) + creating: createKerasSpatialDropout3D + """ + def __init__(self, p=0.5, dim_ordering="th", input_shape=None, bigdl_type="float"): + super(SpatialDropout3D, self).__init__(None, bigdl_type, + float(p), + dim_ordering, + list(input_shape) if input_shape else None) + + +class GaussianDropout(KerasLayer): + """ + Apply multiplicative 1-centered Gaussian noise. + As it is a regularization layer, it is only active at training time. + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + + # Arguments + p: Drop probability. Float between 0 and 1. + The multiplicative noise will have standard deviation 'sqrt(p/(1-p))'. + input_shape: A shape tuple, not including batch. + + >>> gaussiandropout = GaussianDropout(0.45, input_shape=(4, 8)) + creating: createKerasGaussianDropout + """ + def __init__(self, p, input_shape=None, bigdl_type="float"): + super(GaussianDropout, self).__init__(None, bigdl_type, + float(p), + list(input_shape) if input_shape else None) + + +class GaussianNoise(KerasLayer): + """ + Apply additive zero-centered Gaussian noise. + This is useful to mitigate overfitting (you could see it as a form of random data augmentation). + Gaussian Noise is a natural choice as corruption process for real valued inputs. + As it is a regularization layer, it is only active at training time. + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + + # Arguments + sigma: Float, standard deviation of the noise distribution. + input_shape: A shape tuple, not including batch. + + >>> gaussiannoise = GaussianNoise(0.45, input_shape=(3, 4, 5)) + creating: createKerasGaussianNoise + """ + def __init__(self, sigma, input_shape=None, bigdl_type="float"): + super(GaussianNoise, self).__init__(None, bigdl_type, + float(sigma), + list(input_shape) if input_shape else None) + + +class Masking(KerasLayer): + """ + Use a mask value to skip timesteps for a sequence. + Masks a sequence by using a mask value to skip timesteps. + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + + # Arguments + mask_value: Float, mask value. For each timestep in the input tensor (dimension #1 in the tensor), + if all values in the input tensor at that timestep are equal to `mask_value`, + then the timestep will masked (skipped) in all downstream layers. + input_shape: A shape tuple, not including batch. + + >>> masking = Masking(0.3, input_shape=(6, 8)) + creating: createKerasMasking + """ + def __init__(self, mask_value=0.0, input_shape=None, bigdl_type="float"): + super(Masking, self).__init__(None, bigdl_type, + float(mask_value), + list(input_shape) if input_shape else None) + + +class SReLU(KerasLayer): + """ + S-shaped Rectified Linear Unit. + It follows: + f(x) = t^r + a^r(x - t^r) for x >= t^r + f(x) = x for t^r > x > t^l, + f(x) = t^l + a^l(x - t^l) for x <= t^l + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + + # Arguments + t_left_init: String representations of initialization method for the left part intercept. + Default is 'zero'. + a_left_init: String representations of initialization method for the left part slope. + Default is 'glorot_uniform'. + t_right_init: String representations of initialization method for the right part intercept. + Default is 'glorot_uniform'. + a_right_init: String representations of initialization method for the right part slope. + Default is 'one'. + shared_axes: Int tuple. The axes along which to share learnable parameters for the activation function. + Default is None. + For example, if the incoming feature maps are from a 2D convolution with output shape + (batch, height, width, channels), and you wish to share parameters across space so that + each filter only has one set of parameters, set 'SharedAxes=(1,2)'. + input_shape: A shape tuple, not including batch. + + >>> srelu = SReLU(input_shape=(4, 5)) + creating: createKerasSReLU + """ + def __init__(self, t_left_init='zero', a_left_init='glorot_uniform', + t_right_init='glorot_uniform', a_right_init='one', + shared_axes=None, input_shape=None, bigdl_type="float"): + super(SReLU, self).__init__(None, bigdl_type, + t_left_init, + a_left_init, + t_right_init, + a_right_init, + shared_axes, + list(input_shape) if input_shape else None) + + +class ELU(KerasLayer): + """ + Exponential Linear Unit. + It follows: + f(x) = alpha * (exp(x) - 1.) for x < 0, + f(x) = x for x >= 0. + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + + # Arguments + alpha: Float, scale for the negative factor. + input_shape: A shape tuple, not including batch. + + >>> elu = ELU(1.2, input_shape=(4, 5)) + creating: createKerasELU + """ + def __init__(self, alpha=1.0, input_shape=None, bigdl_type="float"): + super(ELU, self).__init__(None, bigdl_type, + float(alpha), + list(input_shape) if input_shape else None) + + +class LeakyReLU(KerasLayer): + """ + Leaky version of a Rectified Linear Unit. + It allows a small gradient when the unit is not active: + f(x) = alpha * x for x < 0, + f(x) = x for x >= 0. + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + + # Arguments + alpha: Float >= 0. Negative slope coefficient. + input_shape: A shape tuple, not including batch. + + >>> leakyrelu = LeakyReLU(0.02, input_shape=(4, 5)) + creating: createKerasLeakyReLU + """ + def __init__(self, alpha=0.01, input_shape=None, bigdl_type="float"): + super(LeakyReLU, self).__init__(None, bigdl_type, + float(alpha), + list(input_shape) if input_shape else None) + + +class ThresholdedReLU(KerasLayer): + """ + Thresholded Rectified Linear Unit. + It follows: + f(x) = x for x > theta, + f(x) = 0 otherwise. + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + + # Arguments + theta: Float >= 0. Threshold location of activation. + input_shape: A shape tuple, not including batch. + + >>> thresholdedrelu = ThresholdedReLU(input_shape=(10, 12)) + creating: createKerasThresholdedReLU + """ + def __init__(self, theta=1.0, input_shape=None, bigdl_type="float"): + super(ThresholdedReLU, self).__init__(None, bigdl_type, + float(theta), + list(input_shape) if input_shape else None) + + +class TimeDistributed(KerasLayer): + """ + TimeDistributed wrapper. + Apply a layer to every temporal slice of an input. + The input should be at least 3D, and the dimension of index one will be considered to be the temporal dimension. + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + + # Arguments + layer: A layer instance. + input_shape: A shape tuple, not including batch. + + >>> timedistributed = TimeDistributed(Dense(8), input_shape=(10, 12)) + creating: createKerasDense + creating: createKerasTimeDistributed + """ + def __init__(self, layer, input_shape=None, bigdl_type="float"): + super(TimeDistributed, self).__init__(None, bigdl_type, + layer, + list(input_shape) if input_shape else None) + + +class Bidirectional(KerasLayer): + """ + Bidirectional wrapper for RNNs. + Bidirectional currently requires RNNs to return the full sequence, i.e. return_sequences = True. + + When you use this layer as the first layer of a model, you need to provide the argument + input_shape (a shape tuple, does not include the batch dimension). + + Example of creating a bidirectional LSTM: + Bidirectiona(LSTM(12, return_sequences=True), merge_mode="sum", input_shape=(32, 32)) + + # Arguments + layer: An instance of a recurrent layer. + merge_mode: Mode by which outputs of the forward and backward RNNs will be combined. + Must be one of: 'sum', 'mul', 'concat', 'ave'. Default is 'concat'. + input_shape: A shape tuple, not including batch. + + >>> bidiretional = Bidirectional(LSTM(10, return_sequences=True), input_shape=(12, 16)) + creating: createKerasLSTM + creating: createKerasBidirectional + """ + def __init__(self, layer, merge_mode="concat", input_shape=None, bigdl_type="float"): + super(Bidirectional, self).__init__(None, bigdl_type, + layer, + merge_mode, + list(input_shape) if input_shape else None) \ No newline at end of file diff --git a/python/dllib/src/bigdl/dllib/examples/lenet/README.md b/python/dllib/src/bigdl/dllib/examples/lenet/README.md new file mode 100644 index 00000000000..4eeeccd1e7b --- /dev/null +++ b/python/dllib/src/bigdl/dllib/examples/lenet/README.md @@ -0,0 +1,75 @@ +# LeNet5 Model on MNIST with new API + +This example defines a classical CNN model used in digital number classification with the new set of API in BigDL. For detailed information with regard to LeNet, please refer to . + +This example is the same as [../../model/lenet/lenet5.py](../../models/lenet/lenet5.py), except that this example uses new API for model definition. + +## Install dependencies + * [Install dependencies](../../../README.md#install.dependencies) + +## How to run this example: +Please note that due to some permission issue, this example **cannot** be run on Windows. + + +Program would download the mnist data into ```/tmp/mnist``` automatically by default. + +``` +/tmp/mnist$ tree . +. +├── t10k-images-idx3-ubyte.gz +├── t10k-labels-idx1-ubyte.gz +├── train-images-idx3-ubyte.gz +└── train-labels-idx1-ubyte.gz + +``` + +We would train a LeNet model in spark local mode with the following commands and you can distribute it across cluster by modifying the spark master and the executor cores. + +``` + BigDL_HOME=... + SPARK_HOME=... + MASTER=local[*] + PYTHON_API_ZIP_PATH=${BigDL_HOME}/dist/lib/bigdl-VERSION-python-api.zip + BigDL_JAR_PATH=${BigDL_HOME}/dist/lib/bigdl-VERSION-jar-with-dependencies.jar + PYTHONPATH=${PYTHON_API_ZIP_PATH}:$PYTHONPATH + ${SPARK_HOME}/bin/spark-submit \ + --master ${MASTER} \ + --driver-cores 2 \ + --driver-memory 2g \ + --total-executor-cores 2 \ + --executor-cores 2 \ + --executor-memory 4g \ + --py-files ${PYTHON_API_ZIP_PATH},${BigDL_HOME}/pyspark/bigdl/models/lenet/lenet5.py \ + --properties-file ${BigDL_HOME}/dist/conf/spark-bigdl.conf \ + --jars ${BigDL_JAR_PATH} \ + --conf spark.driver.extraClassPath=${BigDL_JAR_PATH} \ + --conf spark.executor.extraClassPath=bigdl-VERSION-jar-with-dependencies.jar \ + ${BigDL_HOME}/pyspark/bigdl/models/lenet/lenet5.py \ + --dataPath /tmp/mnist + ``` + +* ```--dataPath``` option can be used to set the path for downloading mnist data, the default value is /tmp/mnist. Make sure that you have write permission to the specified path. + +* ```--batchSize``` option can be used to set batch size, the default value is 128. + +* ```--endTriggerType``` option can be used to control how to end the training process, the value can be "epoch" or "iteration" and default value is "epoch". + +* ```--endTriggerNum``` use together with ```endTriggerType```, the default value is 20. + +* ```--modelPath``` option can be used to set model path for testing, the default value is /tmp/lenet5/model.470. + +* ```--checkpointPath``` option can be used to set checkpoint path for saving model, the default value is /tmp/lenet5/. + +To verify the accuracy, search "accuracy" from log: + +``` +INFO DistriOptimizer$:247 - [Epoch 1 0/60000][Iteration 1][Wall Clock 0.0s] Train 128 in xx seconds. Throughput is xx records/second. + +INFO DistriOptimizer$:522 - Top1Accuracy is Accuracy(correct: 9572, count: 10000, accuracy: 0.9572) + +``` + +Or you can train a LeNet model directly in shell after installing BigDL from pip: +``` +python ${BigDL_HOME}/pyspark/bigdl/examples/lenet/lenet.py +``` \ No newline at end of file diff --git a/python/dllib/src/bigdl/dllib/examples/lenet/__init__.py b/python/dllib/src/bigdl/dllib/examples/lenet/__init__.py new file mode 100644 index 00000000000..2151a805423 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/examples/lenet/__init__.py @@ -0,0 +1,15 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/python/dllib/src/bigdl/dllib/examples/lenet/lenet.py b/python/dllib/src/bigdl/dllib/examples/lenet/lenet.py new file mode 100644 index 00000000000..ba76772d1a3 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/examples/lenet/lenet.py @@ -0,0 +1,66 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from optparse import OptionParser +from bigdl.models.lenet.utils import * +from bigdl.nn.keras.layer import * +from bigdl.nn.criterion import * +from bigdl.optim.optimizer import * +from bigdl.util.common import * + + +def build_model(class_num): + model = Sequential() + model.add(Reshape((1, 28, 28), input_shape=(28, 28, 1))) + model.add(Convolution2D(32, 3, 3, activation="relu")) + model.add(Convolution2D(32, 3, 3, activation="relu")) + model.add(MaxPooling2D(pool_size=(2, 2))) + model.add(Dropout(0.25)) + model.add(Flatten()) + model.add(Dense(128, activation="relu")) + model.add(Dropout(0.5)) + model.add(Dense(class_num, activation="softmax")) + return model + + +if __name__ == "__main__": + parser = OptionParser() + parser.add_option("-b", "--batchSize", type=int, dest="batchSize", default="128") + parser.add_option("-c", "--checkpointPath", dest="checkpointPath", default="/tmp/lenet5") + parser.add_option("-t", "--endTriggerType", dest="endTriggerType", default="epoch") + parser.add_option("-n", "--endTriggerNum", type=int, dest="endTriggerNum", default="20") + parser.add_option("-d", "--dataPath", dest="dataPath", default="/tmp/mnist") + + (options, args) = parser.parse_args(sys.argv) + + sc = SparkContext(appName="lenet5", conf=create_spark_conf()) + redire_spark_logs() + show_bigdl_info_logs() + init_engine() + + (train_data, test_data) = preprocess_mnist(sc, options) + + optimizer = Optimizer( + model=build_model(10), + training_rdd=train_data, + criterion=ClassNLLCriterion(logProbAsInput=False), + optim_method=SGD(learningrate=0.01, learningrate_decay=0.0002), + end_trigger=get_end_trigger(options), + batch_size=options.batchSize) + validate_optimizer(optimizer, test_data, options) + trained_model = optimizer.optimize() + parameters = trained_model.parameters() + sc.stop() diff --git a/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py b/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py index 92a7074a08a..e6e28b7d65e 100644 --- a/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py +++ b/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py @@ -15,7 +15,7 @@ # from optparse import OptionParser -from bigdl.dataset import mnist +from utils import * from bigdl.dataset.transformer import * from bigdl.nn.layer import * from bigdl.nn.criterion import * @@ -40,23 +40,6 @@ def build_model(class_num): return model -def get_mnist(sc, data_type="train", location="/tmp/mnist"): - """ - Get and normalize the mnist data. We would download it automatically - if the data doesn't present at the specific location. - - :param sc: SparkContext - :param data_type: training data or testing data - :param location: Location storing the mnist - :return: A RDD of (features: Ndarray, label: Ndarray) - """ - (images, labels) = mnist.read_data_sets(location, data_type) - images = sc.parallelize(images) - labels = sc.parallelize(labels + 1) # Target start from 1 in BigDL - record = images.zip(labels) - return record - - if __name__ == "__main__": parser = OptionParser() parser.add_option("-a", "--action", dest="action", default="train") @@ -75,34 +58,16 @@ def get_mnist(sc, data_type="train", location="/tmp/mnist"): init_engine() if options.action == "train": - def get_end_trigger(): - if options.endTriggerType.lower() == "epoch": - return MaxEpoch(options.endTriggerNum) - else: - return MaxIteration(options.endTriggerNum) + (train_data, test_data) = preprocess_mnist(sc, options) - train_data = get_mnist(sc, "train", options.dataPath)\ - .map(lambda rec_tuple: (normalizer(rec_tuple[0], mnist.TRAIN_MEAN, mnist.TRAIN_STD), - rec_tuple[1]))\ - .map(lambda t: Sample.from_ndarray(t[0], t[1])) - test_data = get_mnist(sc, "test", options.dataPath)\ - .map(lambda rec_tuple: (normalizer(rec_tuple[0], mnist.TEST_MEAN, mnist.TEST_STD), - rec_tuple[1]))\ - .map(lambda t: Sample.from_ndarray(t[0], t[1])) optimizer = Optimizer( model=build_model(10), training_rdd=train_data, criterion=ClassNLLCriterion(), optim_method=SGD(learningrate=0.01, learningrate_decay=0.0002), - end_trigger=get_end_trigger(), + end_trigger=get_end_trigger(options), batch_size=options.batchSize) - optimizer.set_validation( - batch_size=options.batchSize, - val_rdd=test_data, - trigger=EveryEpoch(), - val_method=[Top1Accuracy()] - ) - optimizer.set_checkpoint(EveryEpoch(), options.checkpointPath) + validate_optimizer(optimizer, test_data, options) trained_model = optimizer.optimize() parameters = trained_model.parameters() elif options.action == "test": diff --git a/python/dllib/src/bigdl/dllib/models/lenet/utils.py b/python/dllib/src/bigdl/dllib/models/lenet/utils.py new file mode 100644 index 00000000000..e90444fda33 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/models/lenet/utils.py @@ -0,0 +1,75 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from bigdl.dataset import mnist +from bigdl.dataset.transformer import * +from bigdl.optim.optimizer import * + + +def get_mnist(sc, data_type="train", location="/tmp/mnist"): + """ + Get mnist dataset and parallelize into RDDs. + Data would be downloaded automatically if it doesn't present at the specific location. + + :param sc: SparkContext. + :param data_type: "train" for training data and "test" for testing data. + :param location: Location to store mnist dataset. + :return: RDD of (features: ndarray, label: ndarray). + """ + (images, labels) = mnist.read_data_sets(location, data_type) + images = sc.parallelize(images) + labels = sc.parallelize(labels + 1) # Target start from 1 in BigDL + record = images.zip(labels) + return record + + +def preprocess_mnist(sc, options): + """ + Preprocess mnist dataset. + Normalize and transform into Sample of RDDs. + """ + train_data = get_mnist(sc, "train", options.dataPath)\ + .map(lambda rec_tuple: (normalizer(rec_tuple[0], mnist.TRAIN_MEAN, mnist.TRAIN_STD), + rec_tuple[1]))\ + .map(lambda t: Sample.from_ndarray(t[0], t[1])) + test_data = get_mnist(sc, "test", options.dataPath)\ + .map(lambda rec_tuple: (normalizer(rec_tuple[0], mnist.TEST_MEAN, mnist.TEST_STD), + rec_tuple[1]))\ + .map(lambda t: Sample.from_ndarray(t[0], t[1])) + return train_data, test_data + + +def get_end_trigger(options): + """ + When to end the optimization based on input option. + """ + if options.endTriggerType.lower() == "epoch": + return MaxEpoch(options.endTriggerNum) + else: + return MaxIteration(options.endTriggerNum) + + +def validate_optimizer(optimizer, test_data, options): + """ + Set validation and checkpoint for distributed optimizer. + """ + optimizer.set_validation( + batch_size=options.batchSize, + val_rdd=test_data, + trigger=EveryEpoch(), + val_method=[Top1Accuracy()] + ) + optimizer.set_checkpoint(EveryEpoch(), options.checkpointPath) diff --git a/python/dllib/src/bigdl/dllib/models/local_lenet/README.md b/python/dllib/src/bigdl/dllib/models/local_lenet/README.md index b9f9f6ab1e6..a1691532cca 100644 --- a/python/dllib/src/bigdl/dllib/models/local_lenet/README.md +++ b/python/dllib/src/bigdl/dllib/models/local_lenet/README.md @@ -3,6 +3,8 @@ LeNet5 is a classical CNN model used in digital number classification. For detailed information, please refer to . +The model used here is exactly the same as the model in [../lenet/lenet5.py](../lenet/lenet5.py). + This example would show how to train and inference a LeNet model in pure local mode without using Spark local or Spark distributed cluster. diff --git a/python/dllib/src/bigdl/dllib/models/local_lenet/local_lenet.py b/python/dllib/src/bigdl/dllib/models/local_lenet/local_lenet.py index 0fb96befdab..cc264abacba 100644 --- a/python/dllib/src/bigdl/dllib/models/local_lenet/local_lenet.py +++ b/python/dllib/src/bigdl/dllib/models/local_lenet/local_lenet.py @@ -16,38 +16,20 @@ from optparse import OptionParser from bigdl.dataset import mnist -from bigdl.dataset.transformer import * -from bigdl.nn.layer import * +from bigdl.models.lenet.lenet5 import build_model from bigdl.nn.criterion import * from bigdl.optim.optimizer import * from bigdl.util.common import * -def build_model(class_num): - model = Sequential() - model.add(Reshape([1, 28, 28])) - model.add(SpatialConvolution(1, 6, 5, 5)) - model.add(Tanh()) - model.add(SpatialMaxPooling(2, 2, 2, 2)) - model.add(Tanh()) - model.add(SpatialConvolution(6, 12, 5, 5)) - model.add(SpatialMaxPooling(2, 2, 2, 2)) - model.add(Reshape([12 * 4 * 4])) - model.add(Linear(12 * 4 * 4, 100)) - model.add(Tanh()) - model.add(Linear(100, class_num)) - model.add(LogSoftMax()) - return model - - def get_mnist(data_type="train", location="/tmp/mnist"): """ - Get and normalize the mnist data. We would download it automatically - if the data doesn't present at the specific location. + Get mnist dataset with features and label as ndarray. + Data would be downloaded automatically if it doesn't present at the specific location. - :param data_type: training data or testing data - :param location: Location storing the mnist - :return: (features: Ndarray, label: Ndarray) + :param data_type: "train" for training data and "test" for testing data. + :param location: Location to store mnist dataset. + :return: (features: ndarray, label: ndarray) """ X, Y = mnist.read_data_sets(location, data_type) return X, Y + 1 # The label of ClassNLLCriterion starts from 1 instead of 0 @@ -67,6 +49,7 @@ def get_mnist(data_type="train", location="/tmp/mnist"): (X_train, Y_train) = get_mnist("train", options.dataPath) (X_test, Y_test) = get_mnist("test", options.dataPath) + # The model used here is exactly the same as the model in ../lenet/lenet5.py optimizer = Optimizer.create( model=build_model(10), training_set=(X_train, Y_train), @@ -76,8 +59,8 @@ def get_mnist(data_type="train", location="/tmp/mnist"): batch_size=options.batchSize) optimizer.set_validation( batch_size=options.batchSize, - X_val = X_test, - Y_val = Y_test, + X_val=X_test, + Y_val=Y_test, trigger=EveryEpoch(), val_method=[Top1Accuracy()] ) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 856dca0aee0..5de2162a4d5 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -614,7 +614,7 @@ def add(self, model): @property def layers(self): - jlayers = callBigDlFunc(self.bigdl_type, "getContainerModules" , self) + jlayers = callBigDlFunc(self.bigdl_type, "getContainerModules", self) layers = [Layer.of(jlayer) for jlayer in jlayers] return layers @@ -654,6 +654,7 @@ class Model(Container): def __init__(self, inputs, outputs, + is_keras=False, jvalue=None, bigdl_type="float", byte_order="little_endian", model_type="bigdl"): if jvalue: @@ -661,8 +662,9 @@ def __init__(self, self.bigdl_type = bigdl_type elif model_type == "bigdl": super(Model, self).__init__(None, bigdl_type, - to_list(inputs), - to_list(outputs)) + to_list(inputs), + to_list(outputs), + is_keras) else: from bigdl.util.tf_utils import convert model = convert(to_list(inputs), to_list(outputs), byte_order, bigdl_type) @@ -1062,8 +1064,8 @@ class Sequential(Container): ''' - def __init__(self, bigdl_type="float"): - super(Sequential, self).__init__(None, bigdl_type, False) + def __init__(self, bigdl_type="float", is_keras=False): + super(Sequential, self).__init__(None, bigdl_type, is_keras) class TemporalConvolution(Layer): From 9c1de596ec384fe1c9dfd79d4263251fb2da2bb5 Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Wed, 14 Feb 2018 15:43:30 +0800 Subject: [PATCH 577/823] Keras-like API Python Wrapper for layers + lenet example (#2296) * python lenet * docs * style * wrap getoutputshape * update input output shape * update * python layers * clean * fix ut * more layers * fix ut * more layers * fix ut * more layers * wrapper * remove * style * style and srelu * style * refine lenet * update * add ut for input output shape * fix float * refine lenet * update * meet review * clean * todo * update * style * readme * style * style * fix * style --- python/dllib/test/bigdl/keras/test_newapi.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/python/dllib/test/bigdl/keras/test_newapi.py b/python/dllib/test/bigdl/keras/test_newapi.py index f5c41164d99..6f47fd23852 100644 --- a/python/dllib/test/bigdl/keras/test_newapi.py +++ b/python/dllib/test/bigdl/keras/test_newapi.py @@ -119,6 +119,24 @@ def test_merge_cos(self): input_data = [np.random.random([2, 3]), np.random.random([2, 3])] self.compare_newapi(klayer, blayer, input_data) + def test_lenet_shape(self): + from bigdl.examples.lenet.lenet import build_model + model = build_model(10) + input_shape = model.get_input_shape() + np.testing.assert_allclose((28, 28, 1), input_shape[1:]) + output_shape = model.get_output_shape() + np.testing.assert_allclose((10, ), output_shape[1:]) + + def test_graph(self): + x1 = BLayer.Input(input_shape=(8, )) + x2 = BLayer.Input(input_shape=(6, )) + y1 = BLayer.Dense(10)(x1) + y2 = BLayer.Dense(10)(x2) + model = BLayer.Model([x1, x2], [y1, y2]) + input_shapes = model.get_input_shape() + np.testing.assert_allclose((8, ), input_shapes[0][1:]) + np.testing.assert_allclose((6, ), input_shapes[1][1:]) + if __name__ == "__main__": pytest.main([__file__]) From 586fc1133b212887130de0d7fb89b8eab940d038 Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Thu, 22 Feb 2018 14:36:45 +0800 Subject: [PATCH 578/823] add meaningful error message for calling java code on executor side error (#2311) * add meaningful error message for calling java code on executor side * adjust for python3 --- python/dllib/src/bigdl/dllib/utils/common.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/utils/common.py b/python/dllib/src/bigdl/dllib/utils/common.py index 6097e525cf3..501178dd7aa 100644 --- a/python/dllib/src/bigdl/dllib/utils/common.py +++ b/python/dllib/src/bigdl/dllib/utils/common.py @@ -30,6 +30,7 @@ import numpy as np import threading import tempfile +import traceback from bigdl.util.engine import get_bigdl_classpath, is_spark_below_2_2 INTMAX = 2147483647 @@ -567,8 +568,15 @@ def get_spark_sql_context(sc): def _get_port(): root_dir = SparkFiles.getRootDirectory() path = os.path.join(root_dir, "gateway_port") - f = open(path) - port = int(f.readline()) + try: + with open(path) as f: + port = int(f.readline()) + except IOError as e: + traceback.print_exc() + raise RuntimeError("Could not open the file %s, which contains the listening port of" + " local Java Gateway, please make sure the init_executor_gateway()" + " function is called before any call of java function on the" + " executor side." % e.filename) return port def _get_gateway(): From 7509252bda927163bf8f1bb51b96a7fe7fc0b1b1 Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Thu, 22 Feb 2018 14:36:45 +0800 Subject: [PATCH 579/823] add meaningful error message for calling java code on executor side error (#2311) * add meaningful error message for calling java code on executor side * adjust for python3 --- python/dllib/src/bigdl/utils/common.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/python/dllib/src/bigdl/utils/common.py b/python/dllib/src/bigdl/utils/common.py index 6097e525cf3..501178dd7aa 100644 --- a/python/dllib/src/bigdl/utils/common.py +++ b/python/dllib/src/bigdl/utils/common.py @@ -30,6 +30,7 @@ import numpy as np import threading import tempfile +import traceback from bigdl.util.engine import get_bigdl_classpath, is_spark_below_2_2 INTMAX = 2147483647 @@ -567,8 +568,15 @@ def get_spark_sql_context(sc): def _get_port(): root_dir = SparkFiles.getRootDirectory() path = os.path.join(root_dir, "gateway_port") - f = open(path) - port = int(f.readline()) + try: + with open(path) as f: + port = int(f.readline()) + except IOError as e: + traceback.print_exc() + raise RuntimeError("Could not open the file %s, which contains the listening port of" + " local Java Gateway, please make sure the init_executor_gateway()" + " function is called before any call of java function on the" + " executor side." % e.filename) return port def _get_gateway(): From 172c0ad33f7f693cc97ecbf128ba45d7b3260736 Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Wed, 28 Feb 2018 13:52:16 +0800 Subject: [PATCH 580/823] support getting node and core number in Python (#2333) * support getting node and core number on python * python style * address comments --- python/dllib/src/bigdl/dllib/models/utils/model_broadcast.py | 2 +- python/dllib/src/bigdl/dllib/utils/common.py | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/python/dllib/src/bigdl/dllib/models/utils/model_broadcast.py b/python/dllib/src/bigdl/dllib/models/utils/model_broadcast.py index 4b88ae1a3fb..d333cb1772f 100644 --- a/python/dllib/src/bigdl/dllib/models/utils/model_broadcast.py +++ b/python/dllib/src/bigdl/dllib/models/utils/model_broadcast.py @@ -28,7 +28,7 @@ def _from_id_and_type(bid, bigdl_type): result = _from_id(bid) return ModelBroadcast(path=result._path, bigdl_type=bigdl_type) -def broadcastModel(sc, layer): +def broadcast_model(sc, layer): return ModelBroadcast(sc, layer, sc._pickled_broadcast_vars) class ModelBroadcast(Broadcast): diff --git a/python/dllib/src/bigdl/dllib/utils/common.py b/python/dllib/src/bigdl/dllib/utils/common.py index 501178dd7aa..03428ba9901 100644 --- a/python/dllib/src/bigdl/dllib/utils/common.py +++ b/python/dllib/src/bigdl/dllib/utils/common.py @@ -446,6 +446,9 @@ def init_engine(bigdl_type="float"): def init_executor_gateway(sc, bigdl_type="float"): callBigDlFunc(bigdl_type, "initExecutorGateway", sc, sc._gateway._gateway_client.port) +def get_node_and_core_number(bigdl_type="float"): + result = callBigDlFunc(bigdl_type, "getNodeAndCoreNumber") + return result[0], result[1] def redire_spark_logs(bigdl_type="float", log_path=os.getcwd()+"/bigdl.log"): """ From a6c4e55dba8e76367427b9b74475c9d5550b064d Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Wed, 28 Feb 2018 13:52:16 +0800 Subject: [PATCH 581/823] support getting node and core number in Python (#2333) * support getting node and core number on python * python style * address comments --- python/dllib/src/bigdl/dllib/models/utils/model_broadcast.py | 2 +- python/dllib/src/bigdl/utils/common.py | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/python/dllib/src/bigdl/dllib/models/utils/model_broadcast.py b/python/dllib/src/bigdl/dllib/models/utils/model_broadcast.py index 4b88ae1a3fb..d333cb1772f 100644 --- a/python/dllib/src/bigdl/dllib/models/utils/model_broadcast.py +++ b/python/dllib/src/bigdl/dllib/models/utils/model_broadcast.py @@ -28,7 +28,7 @@ def _from_id_and_type(bid, bigdl_type): result = _from_id(bid) return ModelBroadcast(path=result._path, bigdl_type=bigdl_type) -def broadcastModel(sc, layer): +def broadcast_model(sc, layer): return ModelBroadcast(sc, layer, sc._pickled_broadcast_vars) class ModelBroadcast(Broadcast): diff --git a/python/dllib/src/bigdl/utils/common.py b/python/dllib/src/bigdl/utils/common.py index 501178dd7aa..03428ba9901 100644 --- a/python/dllib/src/bigdl/utils/common.py +++ b/python/dllib/src/bigdl/utils/common.py @@ -446,6 +446,9 @@ def init_engine(bigdl_type="float"): def init_executor_gateway(sc, bigdl_type="float"): callBigDlFunc(bigdl_type, "initExecutorGateway", sc, sc._gateway._gateway_client.port) +def get_node_and_core_number(bigdl_type="float"): + result = callBigDlFunc(bigdl_type, "getNodeAndCoreNumber") + return result[0], result[1] def redire_spark_logs(bigdl_type="float", log_path=os.getcwd()+"/bigdl.log"): """ From 0d44c246a2e0be45f3022a5a00d7e226a26b439e Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Wed, 28 Feb 2018 13:52:16 +0800 Subject: [PATCH 582/823] support getting node and core number in Python (#2333) * support getting node and core number on python * python style * address comments --- python/dllib/test/bigdl/test_simple_integration.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/python/dllib/test/bigdl/test_simple_integration.py b/python/dllib/test/bigdl/test_simple_integration.py index db27bb23ddc..ba24f6abaca 100644 --- a/python/dllib/test/bigdl/test_simple_integration.py +++ b/python/dllib/test/bigdl/test_simple_integration.py @@ -28,7 +28,7 @@ from numpy.testing import assert_allclose, assert_array_equal from bigdl.util.engine import compare_version from bigdl.transform.vision.image import * -from bigdl.models.utils.model_broadcast import broadcastModel +from bigdl.models.utils.model_broadcast import broadcast_model from bigdl.dataset.dataset import * np.random.seed(1337) # for reproducibility @@ -607,7 +607,7 @@ def test_model_broadcast(self): init_executor_gateway(self.sc) model = Linear(3, 2) - broadcasted = broadcastModel(self.sc, model) + broadcasted = broadcast_model(self.sc, model) input_data = np.random.rand(3) output = self.sc.parallelize([input_data], 1)\ .map(lambda x: broadcasted.value.forward(x)).first() @@ -659,6 +659,12 @@ def test_train_DataSet(self): predict_result = trained_model.predict_image(image_frame.transform(transformer)) assert_allclose(predict_result.get_predict().count(), 8) + def test_get_node_and_core_num(self): + node, core = get_node_and_core_number() + + assert node == 1 + assert core == 4 + if __name__ == "__main__": pytest.main([__file__]) From 51156ab81bb7c9607b47285c674e4e6372210428 Mon Sep 17 00:00:00 2001 From: Yao Zhang Date: Thu, 1 Mar 2018 16:01:36 +0800 Subject: [PATCH 583/823] Optimize padding mechanism (#2324) * optimize padding options * restore classNLLCriterion comments * meet code review * add more tests * meet code review * meet code review * meet code review * Add python api * fix a bug * fix a bug --- python/dllib/src/bigdl/dllib/nn/criterion.py | 23 ++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/nn/criterion.py b/python/dllib/src/bigdl/dllib/nn/criterion.py index 47ecc77736a..f2076860bf3 100644 --- a/python/dllib/src/bigdl/dllib/nn/criterion.py +++ b/python/dllib/src/bigdl/dllib/nn/criterion.py @@ -596,6 +596,29 @@ def __init__(self, ignore_label, normalize_mode) +class TimeDistributedMaskCriterion(Criterion): + ''' + This class is intended to support inputs with 3 or more dimensions. + Apply Any Provided Criterion to every temporal slice of an input. + In addition, it supports padding mask. + + eg. if the target is [ [-1, 1, 2, 3, -1], [5, 4, 3, -1, -1] ], + and set the paddingValue property to -1, then the loss of -1 would not + be accumulated and the loss is only divided by 6 (ont including the amount of + -1, in this case, we are only interested in 1, 2, 3, 5, 4, 3) + + :param criterion: embedded criterion + :param padding_value: padding value + + + >>> td = TimeDistributedMaskCriterion(ClassNLLCriterion()) + creating: createClassNLLCriterion + creating: createTimeDistributedMaskCriterion + ''' + + def __init__(self, criterion, padding_value=0, bigdl_type="float"): + super(TimeDistributedMaskCriterion, self).__init__( + None, bigdl_type, criterion, padding_value) class TimeDistributedCriterion(Criterion): ''' From 1e176afd398f1941ec24018e30e83e1b347e150d Mon Sep 17 00:00:00 2001 From: Yao Zhang Date: Thu, 1 Mar 2018 16:01:36 +0800 Subject: [PATCH 584/823] Optimize padding mechanism (#2324) * optimize padding options * restore classNLLCriterion comments * meet code review * add more tests * meet code review * meet code review * meet code review * Add python api * fix a bug * fix a bug --- python/dllib/src/bigdl/dllib/nn/criterion.py | 23 ++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/nn/criterion.py b/python/dllib/src/bigdl/dllib/nn/criterion.py index 47ecc77736a..f2076860bf3 100644 --- a/python/dllib/src/bigdl/dllib/nn/criterion.py +++ b/python/dllib/src/bigdl/dllib/nn/criterion.py @@ -596,6 +596,29 @@ def __init__(self, ignore_label, normalize_mode) +class TimeDistributedMaskCriterion(Criterion): + ''' + This class is intended to support inputs with 3 or more dimensions. + Apply Any Provided Criterion to every temporal slice of an input. + In addition, it supports padding mask. + + eg. if the target is [ [-1, 1, 2, 3, -1], [5, 4, 3, -1, -1] ], + and set the paddingValue property to -1, then the loss of -1 would not + be accumulated and the loss is only divided by 6 (ont including the amount of + -1, in this case, we are only interested in 1, 2, 3, 5, 4, 3) + + :param criterion: embedded criterion + :param padding_value: padding value + + + >>> td = TimeDistributedMaskCriterion(ClassNLLCriterion()) + creating: createClassNLLCriterion + creating: createTimeDistributedMaskCriterion + ''' + + def __init__(self, criterion, padding_value=0, bigdl_type="float"): + super(TimeDistributedMaskCriterion, self).__init__( + None, bigdl_type, criterion, padding_value) class TimeDistributedCriterion(Criterion): ''' From 05a369a9e81df488e8508805fe7a390064e07a42 Mon Sep 17 00:00:00 2001 From: Ian Wong Date: Wed, 7 Mar 2018 14:18:41 +0800 Subject: [PATCH 585/823] [Enhancement] Check duplicate layers in the container (#2351) * check duplicate layers in the container * add more unit ttest * fix unit test * fix test * meet code review --- python/dllib/src/bigdl/dllib/nn/layer.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 5de2162a4d5..b6a02021834 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -1058,8 +1058,6 @@ class Sequential(Container): >>> s = Sequential() creating: createSequential >>> s = s.add(echo) - >>> s = s.add(s) - >>> s = s.add(echo) ''' From 82e00fc7a7e7a733ed15a400e6b42e79401e6234 Mon Sep 17 00:00:00 2001 From: Ian Wong Date: Wed, 7 Mar 2018 14:18:41 +0800 Subject: [PATCH 586/823] [Enhancement] Check duplicate layers in the container (#2351) * check duplicate layers in the container * add more unit ttest * fix unit test * fix test * meet code review --- python/dllib/src/bigdl/dllib/nn/layer.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 5de2162a4d5..b6a02021834 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -1058,8 +1058,6 @@ class Sequential(Container): >>> s = Sequential() creating: createSequential >>> s = s.add(echo) - >>> s = s.add(s) - >>> s = s.add(echo) ''' From 9e4413b04d621ed0d87b3b2858186e1bef9875f6 Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Thu, 8 Mar 2018 12:26:49 +0800 Subject: [PATCH 587/823] Fix and refactor Python adding extra jars; fix doc issues (#2358) * refactor adding extra jars * update * update docs * refactor python faq * update docs for script --- python/dllib/src/bigdl/dllib/utils/common.py | 44 +++++--------------- python/dllib/src/bigdl/dllib/utils/engine.py | 16 ++++--- 2 files changed, 21 insertions(+), 39 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/utils/common.py b/python/dllib/src/bigdl/dllib/utils/common.py index 03428ba9901..7747796cff9 100644 --- a/python/dllib/src/bigdl/dllib/utils/common.py +++ b/python/dllib/src/bigdl/dllib/utils/common.py @@ -16,6 +16,7 @@ import os import sys +import six from py4j.protocol import Py4JJavaError from py4j.java_gateway import JavaObject from py4j.java_collections import ListConverter, JavaArray, JavaList, JavaMap, MapConverter @@ -55,6 +56,7 @@ def instance(cls, cls._instance = cls(bigdl_type, *args) return cls._instance + class GatewayWrapper(SingletonMixin): def __init__(self, bigdl_type, port=25333): @@ -77,6 +79,8 @@ def get_creator_class(cls): @classmethod def set_creator_class(cls, cclass): + if isinstance(cclass, six.string_types): + cclass = [cclass] with JavaCreator._lock: JavaCreator.__creator_class = cclass JavaCreator._instance = None @@ -136,37 +140,6 @@ def get_dtype(bigdl_type): return "float32" -class Configuration(object): - __bigdl_jars = [get_bigdl_classpath()] - - @staticmethod - def add_extra_jars(jars): - """ - Add extra jars to classpath - :param jars: a string or a list of strings as jar paths - """ - import six - if isinstance(jars, six.string_types): - jars = [jars] - Configuration.__bigdl_jars += jars - - @staticmethod - def add_extra_python_modules(packages): - """ - Add extra python modules to sys.path - :param packages: a string or a list of strings as python package paths - """ - import six - if isinstance(packages, six.string_types): - packages = [packages] - for package in packages: - sys.path.insert(0, package) - - @staticmethod - def get_bigdl_jars(): - return Configuration.__bigdl_jars - - class JActivity(object): def __init__(self, value): @@ -443,13 +416,16 @@ def uniform(self, a, b, size): def init_engine(bigdl_type="float"): callBigDlFunc(bigdl_type, "initEngine") + def init_executor_gateway(sc, bigdl_type="float"): callBigDlFunc(bigdl_type, "initExecutorGateway", sc, sc._gateway._gateway_client.port) + def get_node_and_core_number(bigdl_type="float"): result = callBigDlFunc(bigdl_type, "getNodeAndCoreNumber") return result[0], result[1] + def redire_spark_logs(bigdl_type="float", log_path=os.getcwd()+"/bigdl.log"): """ Redirect spark logs to the specified path. @@ -521,8 +497,8 @@ def create_spark_conf(): bigdl_conf = get_bigdl_conf() sparkConf = SparkConf() sparkConf.setAll(bigdl_conf.items()) - if not is_spark_below_2_2(): - for jar in Configuration.get_bigdl_jars(): + if os.environ.get("BIGDL_JARS", None) and not is_spark_below_2_2(): + for jar in os.environ["BIGDL_JARS"].split(":"): extend_spark_driver_cp(sparkConf, jar) # add content in PYSPARK_FILES in spark.submit.pyFiles @@ -568,6 +544,7 @@ def get_spark_sql_context(sc): else: return SQLContext(sc) # Compatible with Spark1.5.1 + def _get_port(): root_dir = SparkFiles.getRootDirectory() path = os.path.join(root_dir, "gateway_port") @@ -582,6 +559,7 @@ def _get_port(): " executor side." % e.filename) return port + def _get_gateway(): if SparkFiles._is_running_on_worker: gateway_port = _get_port() diff --git a/python/dllib/src/bigdl/dllib/utils/engine.py b/python/dllib/src/bigdl/dllib/utils/engine.py index e6ea5cad3c9..05d7b239d1a 100644 --- a/python/dllib/src/bigdl/dllib/utils/engine.py +++ b/python/dllib/src/bigdl/dllib/utils/engine.py @@ -68,22 +68,26 @@ def __prepare_bigdl_env(): def append_path(env_var_name, path): try: print("Adding %s to %s" % (path, env_var_name)) - os.environ[env_var_name] = path + ":" + os.environ[ - env_var_name] # noqa + os.environ[env_var_name] = path + ":" + os.environ[env_var_name] # noqa except KeyError: os.environ[env_var_name] = path + if bigdl_classpath: + append_path("BIGDL_JARS", bigdl_classpath) + if conf_paths: assert len(conf_paths) == 1, "Expecting one conf: %s" % len(conf_paths) print("Prepending %s to sys.path" % conf_paths[0]) sys.path.insert(0, conf_paths[0]) - if bigdl_classpath and is_spark_below_2_2(): - append_path("SPARK_CLASSPATH", bigdl_classpath) - from bigdl.util.common import Configuration - for jar in Configuration.get_bigdl_jars(): + if os.environ.get("BIGDL_JARS", None) and is_spark_below_2_2(): + for jar in os.environ["BIGDL_JARS"].split(":"): append_path("SPARK_CLASSPATH", jar) + if os.environ.get("BIGDL_PACKAGES", None): + for package in os.environ["BIGDL_PACKAGES"].split(":"): + sys.path.insert(0, package) + def get_bigdl_classpath(): """ From 4243846f1369faa034869ed5c38a8e17bfb746fb Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Thu, 8 Mar 2018 12:26:49 +0800 Subject: [PATCH 588/823] Fix and refactor Python adding extra jars; fix doc issues (#2358) * refactor adding extra jars * update * update docs * refactor python faq * update docs for script --- python/dllib/src/bigdl/utils/common.py | 44 +++++++------------------- python/dllib/src/bigdl/utils/engine.py | 16 ++++++---- 2 files changed, 21 insertions(+), 39 deletions(-) diff --git a/python/dllib/src/bigdl/utils/common.py b/python/dllib/src/bigdl/utils/common.py index 03428ba9901..7747796cff9 100644 --- a/python/dllib/src/bigdl/utils/common.py +++ b/python/dllib/src/bigdl/utils/common.py @@ -16,6 +16,7 @@ import os import sys +import six from py4j.protocol import Py4JJavaError from py4j.java_gateway import JavaObject from py4j.java_collections import ListConverter, JavaArray, JavaList, JavaMap, MapConverter @@ -55,6 +56,7 @@ def instance(cls, cls._instance = cls(bigdl_type, *args) return cls._instance + class GatewayWrapper(SingletonMixin): def __init__(self, bigdl_type, port=25333): @@ -77,6 +79,8 @@ def get_creator_class(cls): @classmethod def set_creator_class(cls, cclass): + if isinstance(cclass, six.string_types): + cclass = [cclass] with JavaCreator._lock: JavaCreator.__creator_class = cclass JavaCreator._instance = None @@ -136,37 +140,6 @@ def get_dtype(bigdl_type): return "float32" -class Configuration(object): - __bigdl_jars = [get_bigdl_classpath()] - - @staticmethod - def add_extra_jars(jars): - """ - Add extra jars to classpath - :param jars: a string or a list of strings as jar paths - """ - import six - if isinstance(jars, six.string_types): - jars = [jars] - Configuration.__bigdl_jars += jars - - @staticmethod - def add_extra_python_modules(packages): - """ - Add extra python modules to sys.path - :param packages: a string or a list of strings as python package paths - """ - import six - if isinstance(packages, six.string_types): - packages = [packages] - for package in packages: - sys.path.insert(0, package) - - @staticmethod - def get_bigdl_jars(): - return Configuration.__bigdl_jars - - class JActivity(object): def __init__(self, value): @@ -443,13 +416,16 @@ def uniform(self, a, b, size): def init_engine(bigdl_type="float"): callBigDlFunc(bigdl_type, "initEngine") + def init_executor_gateway(sc, bigdl_type="float"): callBigDlFunc(bigdl_type, "initExecutorGateway", sc, sc._gateway._gateway_client.port) + def get_node_and_core_number(bigdl_type="float"): result = callBigDlFunc(bigdl_type, "getNodeAndCoreNumber") return result[0], result[1] + def redire_spark_logs(bigdl_type="float", log_path=os.getcwd()+"/bigdl.log"): """ Redirect spark logs to the specified path. @@ -521,8 +497,8 @@ def create_spark_conf(): bigdl_conf = get_bigdl_conf() sparkConf = SparkConf() sparkConf.setAll(bigdl_conf.items()) - if not is_spark_below_2_2(): - for jar in Configuration.get_bigdl_jars(): + if os.environ.get("BIGDL_JARS", None) and not is_spark_below_2_2(): + for jar in os.environ["BIGDL_JARS"].split(":"): extend_spark_driver_cp(sparkConf, jar) # add content in PYSPARK_FILES in spark.submit.pyFiles @@ -568,6 +544,7 @@ def get_spark_sql_context(sc): else: return SQLContext(sc) # Compatible with Spark1.5.1 + def _get_port(): root_dir = SparkFiles.getRootDirectory() path = os.path.join(root_dir, "gateway_port") @@ -582,6 +559,7 @@ def _get_port(): " executor side." % e.filename) return port + def _get_gateway(): if SparkFiles._is_running_on_worker: gateway_port = _get_port() diff --git a/python/dllib/src/bigdl/utils/engine.py b/python/dllib/src/bigdl/utils/engine.py index e6ea5cad3c9..05d7b239d1a 100644 --- a/python/dllib/src/bigdl/utils/engine.py +++ b/python/dllib/src/bigdl/utils/engine.py @@ -68,22 +68,26 @@ def __prepare_bigdl_env(): def append_path(env_var_name, path): try: print("Adding %s to %s" % (path, env_var_name)) - os.environ[env_var_name] = path + ":" + os.environ[ - env_var_name] # noqa + os.environ[env_var_name] = path + ":" + os.environ[env_var_name] # noqa except KeyError: os.environ[env_var_name] = path + if bigdl_classpath: + append_path("BIGDL_JARS", bigdl_classpath) + if conf_paths: assert len(conf_paths) == 1, "Expecting one conf: %s" % len(conf_paths) print("Prepending %s to sys.path" % conf_paths[0]) sys.path.insert(0, conf_paths[0]) - if bigdl_classpath and is_spark_below_2_2(): - append_path("SPARK_CLASSPATH", bigdl_classpath) - from bigdl.util.common import Configuration - for jar in Configuration.get_bigdl_jars(): + if os.environ.get("BIGDL_JARS", None) and is_spark_below_2_2(): + for jar in os.environ["BIGDL_JARS"].split(":"): append_path("SPARK_CLASSPATH", jar) + if os.environ.get("BIGDL_PACKAGES", None): + for package in os.environ["BIGDL_PACKAGES"].split(":"): + sys.path.insert(0, package) + def get_bigdl_classpath(): """ From 6623d4446f23ee130283dd8cce68fb4cf01f86a6 Mon Sep 17 00:00:00 2001 From: Jerry Wu Date: Mon, 12 Mar 2018 16:24:23 +0800 Subject: [PATCH 589/823] Register picklers on engine initialization (#2375) * merge upstream * add test --- python/dllib/src/bigdl/dllib/utils/common.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/utils/common.py b/python/dllib/src/bigdl/dllib/utils/common.py index 7747796cff9..1cbffc0e144 100644 --- a/python/dllib/src/bigdl/dllib/utils/common.py +++ b/python/dllib/src/bigdl/dllib/utils/common.py @@ -415,6 +415,8 @@ def uniform(self, a, b, size): def init_engine(bigdl_type="float"): callBigDlFunc(bigdl_type, "initEngine") + # Spark context is supposed to have been created when init_engine is called + get_spark_context()._jvm.org.apache.spark.bigdl.api.python.BigDLSerDe.initialize() def init_executor_gateway(sc, bigdl_type="float"): From ccd4d3af74f8d45fd5d6f13fc7721ee6eb73f849 Mon Sep 17 00:00:00 2001 From: Jerry Wu Date: Mon, 12 Mar 2018 16:24:23 +0800 Subject: [PATCH 590/823] Register picklers on engine initialization (#2375) * merge upstream * add test --- python/dllib/src/bigdl/utils/common.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python/dllib/src/bigdl/utils/common.py b/python/dllib/src/bigdl/utils/common.py index 7747796cff9..1cbffc0e144 100644 --- a/python/dllib/src/bigdl/utils/common.py +++ b/python/dllib/src/bigdl/utils/common.py @@ -415,6 +415,8 @@ def uniform(self, a, b, size): def init_engine(bigdl_type="float"): callBigDlFunc(bigdl_type, "initEngine") + # Spark context is supposed to have been created when init_engine is called + get_spark_context()._jvm.org.apache.spark.bigdl.api.python.BigDLSerDe.initialize() def init_executor_gateway(sc, bigdl_type="float"): From 374f4ce616ea18b71e5dbff749f9b82e418501c6 Mon Sep 17 00:00:00 2001 From: Jerry Wu Date: Mon, 12 Mar 2018 16:24:23 +0800 Subject: [PATCH 591/823] Register picklers on engine initialization (#2375) * merge upstream * add test --- python/dllib/test/bigdl/test_simple_integration.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/python/dllib/test/bigdl/test_simple_integration.py b/python/dllib/test/bigdl/test_simple_integration.py index ba24f6abaca..6e80fc05e45 100644 --- a/python/dllib/test/bigdl/test_simple_integration.py +++ b/python/dllib/test/bigdl/test_simple_integration.py @@ -665,6 +665,13 @@ def test_get_node_and_core_num(self): assert node == 1 assert core == 4 + def tes_read_image_frame(self): + init_engine() + resource_path = os.path.join(os.path.split(__file__)[0], "resources") + image_path = os.path.join(resource_path, "pascal/000025.jpg") + image_frame = ImageFrame.read(image_path, self.sc) + count = image_frame.get_image().count() + assert count == 1 if __name__ == "__main__": pytest.main([__file__]) From 90e2c98d2035b5a13df450236d42bfcdf4725550 Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Tue, 13 Mar 2018 14:18:03 +0800 Subject: [PATCH 592/823] Enhance and refactor the logic of InferShape (#2293) * refactor keras api style and imports fix * fix warning * fix python * fix ser test * fix test * fix test * remove checking for weight sharing * fix unittest --- .../bigdl/dllib/bigdlkeras/layers/layer.py | 36 ++++++++++++------- python/dllib/src/bigdl/dllib/nn/layer.py | 8 ++--- python/dllib/src/bigdl/dllib/utils/common.py | 2 +- 3 files changed, 27 insertions(+), 19 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py b/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py index 567006329f9..120a8b3aa3b 100644 --- a/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py +++ b/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py @@ -16,7 +16,7 @@ import sys -from bigdl.nn.layer import Layer, Sequential as TSequential, Model as TModel +from bigdl.nn.layer import Layer, Node from bigdl.util.common import callBigDlFunc, JTensor, JavaValue, to_list if sys.version >= '3': @@ -56,34 +56,44 @@ def get_output_shape(self): self.value) return self.__process_shape(output) - -class KerasLayer(Layer): +class KerasCreator(JavaValue): def jvm_class_constructor(self): name = "createKeras" + self.__class__.__name__ print("creating: " + name) return name +class KerasLayer(Layer, InferShape, KerasCreator): + pass + +class KerasModel(KerasLayer): + # TODO: enrich the KerasModel related API here. + pass + -class Sequential(TSequential, InferShape): +class Sequential(KerasModel): """ Container for a Sequential model. - - >>> sequential = Sequential() - creating: createSequential + >>> import bigdl.nn.keras.layer + >>> sequential = bigdl.nn.keras.layer.Sequential() + creating: createKerasSequential """ def __init__(self, bigdl_type="float"): - super(Sequential, self).__init__(bigdl_type, True) + super(Sequential, self).__init__(None, bigdl_type=bigdl_type) + + def add(self, model): + self.value.add(model.value) + return self -class Model(TModel, InferShape): +class Model(KerasModel): def __init__(self, input, output, bigdl_type="float"): - super(Model, self).__init__(to_list(input), + super(Model, self).__init__(None, bigdl_type, + to_list(input), to_list(output), - is_keras=True, - bigdl_type=bigdl_type) + ) -class Input(KerasLayer): +class Input(Node, KerasCreator): def __init__(self, name=None, input_shape=None, bigdl_type="float"): super(Input, self).__init__(None, bigdl_type, name, diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index b6a02021834..e5bb601c40d 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -654,7 +654,6 @@ class Model(Container): def __init__(self, inputs, outputs, - is_keras=False, jvalue=None, bigdl_type="float", byte_order="little_endian", model_type="bigdl"): if jvalue: @@ -663,8 +662,7 @@ def __init__(self, elif model_type == "bigdl": super(Model, self).__init__(None, bigdl_type, to_list(inputs), - to_list(outputs), - is_keras) + to_list(outputs)) else: from bigdl.util.tf_utils import convert model = convert(to_list(inputs), to_list(outputs), byte_order, bigdl_type) @@ -1062,8 +1060,8 @@ class Sequential(Container): ''' - def __init__(self, bigdl_type="float", is_keras=False): - super(Sequential, self).__init__(None, bigdl_type, is_keras) + def __init__(self, bigdl_type="float"): + super(Sequential, self).__init__(None, bigdl_type) class TemporalConvolution(Layer): diff --git a/python/dllib/src/bigdl/dllib/utils/common.py b/python/dllib/src/bigdl/dllib/utils/common.py index 1cbffc0e144..d05f69a19b5 100644 --- a/python/dllib/src/bigdl/dllib/utils/common.py +++ b/python/dllib/src/bigdl/dllib/utils/common.py @@ -82,7 +82,7 @@ def set_creator_class(cls, cclass): if isinstance(cclass, six.string_types): cclass = [cclass] with JavaCreator._lock: - JavaCreator.__creator_class = cclass + JavaCreator.__creator_class = [cclass] JavaCreator._instance = None def __init__(self, bigdl_type, gateway): From b6b6361e774a8212800c4fe4374f021b2b95ffea Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Tue, 13 Mar 2018 14:18:03 +0800 Subject: [PATCH 593/823] Enhance and refactor the logic of InferShape (#2293) * refactor keras api style and imports fix * fix warning * fix python * fix ser test * fix test * fix test * remove checking for weight sharing * fix unittest --- .../bigdl/dllib/bigdlkeras/layers/layer.py | 36 ++++++++++++------- python/dllib/src/bigdl/dllib/nn/layer.py | 8 ++--- python/dllib/src/bigdl/utils/common.py | 2 +- 3 files changed, 27 insertions(+), 19 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py b/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py index 567006329f9..120a8b3aa3b 100644 --- a/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py +++ b/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py @@ -16,7 +16,7 @@ import sys -from bigdl.nn.layer import Layer, Sequential as TSequential, Model as TModel +from bigdl.nn.layer import Layer, Node from bigdl.util.common import callBigDlFunc, JTensor, JavaValue, to_list if sys.version >= '3': @@ -56,34 +56,44 @@ def get_output_shape(self): self.value) return self.__process_shape(output) - -class KerasLayer(Layer): +class KerasCreator(JavaValue): def jvm_class_constructor(self): name = "createKeras" + self.__class__.__name__ print("creating: " + name) return name +class KerasLayer(Layer, InferShape, KerasCreator): + pass + +class KerasModel(KerasLayer): + # TODO: enrich the KerasModel related API here. + pass + -class Sequential(TSequential, InferShape): +class Sequential(KerasModel): """ Container for a Sequential model. - - >>> sequential = Sequential() - creating: createSequential + >>> import bigdl.nn.keras.layer + >>> sequential = bigdl.nn.keras.layer.Sequential() + creating: createKerasSequential """ def __init__(self, bigdl_type="float"): - super(Sequential, self).__init__(bigdl_type, True) + super(Sequential, self).__init__(None, bigdl_type=bigdl_type) + + def add(self, model): + self.value.add(model.value) + return self -class Model(TModel, InferShape): +class Model(KerasModel): def __init__(self, input, output, bigdl_type="float"): - super(Model, self).__init__(to_list(input), + super(Model, self).__init__(None, bigdl_type, + to_list(input), to_list(output), - is_keras=True, - bigdl_type=bigdl_type) + ) -class Input(KerasLayer): +class Input(Node, KerasCreator): def __init__(self, name=None, input_shape=None, bigdl_type="float"): super(Input, self).__init__(None, bigdl_type, name, diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index b6a02021834..e5bb601c40d 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -654,7 +654,6 @@ class Model(Container): def __init__(self, inputs, outputs, - is_keras=False, jvalue=None, bigdl_type="float", byte_order="little_endian", model_type="bigdl"): if jvalue: @@ -663,8 +662,7 @@ def __init__(self, elif model_type == "bigdl": super(Model, self).__init__(None, bigdl_type, to_list(inputs), - to_list(outputs), - is_keras) + to_list(outputs)) else: from bigdl.util.tf_utils import convert model = convert(to_list(inputs), to_list(outputs), byte_order, bigdl_type) @@ -1062,8 +1060,8 @@ class Sequential(Container): ''' - def __init__(self, bigdl_type="float", is_keras=False): - super(Sequential, self).__init__(None, bigdl_type, is_keras) + def __init__(self, bigdl_type="float"): + super(Sequential, self).__init__(None, bigdl_type) class TemporalConvolution(Layer): diff --git a/python/dllib/src/bigdl/utils/common.py b/python/dllib/src/bigdl/utils/common.py index 1cbffc0e144..d05f69a19b5 100644 --- a/python/dllib/src/bigdl/utils/common.py +++ b/python/dllib/src/bigdl/utils/common.py @@ -82,7 +82,7 @@ def set_creator_class(cls, cclass): if isinstance(cclass, six.string_types): cclass = [cclass] with JavaCreator._lock: - JavaCreator.__creator_class = cclass + JavaCreator.__creator_class = [cclass] JavaCreator._instance = None def __init__(self, bigdl_type, gateway): From fa04cbee201bd460910e024e63d9dd58b56327f6 Mon Sep 17 00:00:00 2001 From: Jerry Wu Date: Tue, 13 Mar 2018 14:42:23 +0800 Subject: [PATCH 594/823] fix java to python (#2379) --- python/dllib/src/bigdl/dllib/utils/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/dllib/src/bigdl/dllib/utils/common.py b/python/dllib/src/bigdl/dllib/utils/common.py index d05f69a19b5..98ce9efd46c 100644 --- a/python/dllib/src/bigdl/dllib/utils/common.py +++ b/python/dllib/src/bigdl/dllib/utils/common.py @@ -600,7 +600,7 @@ def _java2py(gateway, r, encoding="bytes"): clsName = 'JavaRDD' if clsName == 'JavaRDD': - jrdd = gateway.jvm.SerDe.javaToPython(r) + jrdd = gateway.jvm.org.apache.spark.bigdl.api.python.BigDLSerDe.javaToPython(r) return RDD(jrdd, get_spark_context()) if clsName == 'DataFrame': From b833dda62fecd412c81f9ef6cdbdf0490bfa2722 Mon Sep 17 00:00:00 2001 From: Jerry Wu Date: Tue, 13 Mar 2018 14:42:23 +0800 Subject: [PATCH 595/823] fix java to python (#2379) --- python/dllib/src/bigdl/utils/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/dllib/src/bigdl/utils/common.py b/python/dllib/src/bigdl/utils/common.py index d05f69a19b5..98ce9efd46c 100644 --- a/python/dllib/src/bigdl/utils/common.py +++ b/python/dllib/src/bigdl/utils/common.py @@ -600,7 +600,7 @@ def _java2py(gateway, r, encoding="bytes"): clsName = 'JavaRDD' if clsName == 'JavaRDD': - jrdd = gateway.jvm.SerDe.javaToPython(r) + jrdd = gateway.jvm.org.apache.spark.bigdl.api.python.BigDLSerDe.javaToPython(r) return RDD(jrdd, get_spark_context()) if clsName == 'DataFrame': From 56250bd74412156983035161bc0d99cd618484f3 Mon Sep 17 00:00:00 2001 From: Yuhao Yang Date: Tue, 13 Mar 2018 14:07:45 -0700 Subject: [PATCH 596/823] DataFrame-based image reader and transformer (#2325) * DataFrame image support * comment update * sql adapter * rename ut * remove a ut * improve python ut * bypass 1.5 * add ut * import existential * clone transformer --- .../bigdl/dlframes/test_dl_image_reader.py | 58 +++++++++++++++ .../dlframes/test_dl_image_transformer.py | 71 +++++++++++++++++++ 2 files changed, 129 insertions(+) create mode 100644 python/dllib/test/bigdl/dlframes/test_dl_image_reader.py create mode 100644 python/dllib/test/bigdl/dlframes/test_dl_image_transformer.py diff --git a/python/dllib/test/bigdl/dlframes/test_dl_image_reader.py b/python/dllib/test/bigdl/dlframes/test_dl_image_reader.py new file mode 100644 index 00000000000..2a40e7cf267 --- /dev/null +++ b/python/dllib/test/bigdl/dlframes/test_dl_image_reader.py @@ -0,0 +1,58 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import os +from bigdl.dlframes.dl_image_reader import * +from bigdl.transform.vision.image import * +from bigdl.util.common import * +from pyspark.sql.types import * + + +class TestDLImageReader(): + + def setup_method(self, method): + """ + setup any state tied to the execution of the given method in a + class. setup_method is invoked for every test method of a class. + """ + sparkConf = create_spark_conf().setMaster("local[1]").setAppName("test model") + self.sc = get_spark_context(sparkConf) + init_engine() + self.resource_path = os.path.join(os.path.split(__file__)[0], "../resources") + + def teardown_method(self, method): + """ + teardown any state that was previously setup with a setup_method + call. + """ + self.sc.stop() + + def test_get_pascal_image(self): + image_path = os.path.join(self.resource_path, "pascal/000025.jpg") + image_frame = DLImageReader.readImages(image_path, self.sc) + assert image_frame.count() == 1 + assert type(image_frame).__name__ == 'DataFrame' + first_row = image_frame.take(1)[0][0] + assert first_row[0].endswith("pascal/000025.jpg") + assert first_row[1] == 375 + assert first_row[2] == 500 + assert first_row[3] == 3 + assert first_row[4] == 16 + assert len(first_row[5]) == 95959 + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/python/dllib/test/bigdl/dlframes/test_dl_image_transformer.py b/python/dllib/test/bigdl/dlframes/test_dl_image_transformer.py new file mode 100644 index 00000000000..23e1d7c61a6 --- /dev/null +++ b/python/dllib/test/bigdl/dlframes/test_dl_image_transformer.py @@ -0,0 +1,71 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import os +from bigdl.dlframes.dl_image_reader import * +from bigdl.dlframes.dl_image_transformer import * +from bigdl.util.common import * +from bigdl.transform.vision.image import * + + +class TestDLImageTransformer(): + + def setup_method(self, method): + """ + setup any state tied to the execution of the given method in a + class. setup_method is invoked for every test method of a class. + """ + sparkConf = create_spark_conf().setMaster("local[1]").setAppName("test model") + self.sc = get_spark_context(sparkConf) + init_engine() + resource_path = os.path.join(os.path.split(__file__)[0], "../resources") + self.image_path = os.path.join(resource_path, "pascal/000025.jpg") + + def teardown_method(self, method): + """ + teardown any state that was previously setup with a setup_method + call. + """ + self.sc.stop() + + def test_transform_image(self): + # Current the unit test does not work under Spark 1.5, The error message: + # "java.lang.IllegalStateException: Cannot call methods on a stopped SparkContext" + # Yet the transformer works during manual test in 1.5, thus we bypass the 1.5 unit + # test, and withhold the support for Spark 1.5, until the unit test failure reason + # is clarified. + + if not self.sc.version.startswith("1.5"): + image_frame = DLImageReader.readImages(self.image_path, self.sc) + transformer = DLImageTransformer( + Pipeline([Resize(256, 256), CenterCrop(224, 224), + ChannelNormalize(0.485, 0.456, 0.406, 0.229, 0.224, 0.225), + MatToTensor()]) + ).setInputCol("image").setOutputCol("output") + + result = transformer.transform(image_frame) + assert(result.count() == 1) + first_row = result.select("output").take(1)[0][0] + assert first_row[0].endswith("pascal/000025.jpg") + assert first_row[1] == 224 + assert first_row[2] == 224 + assert first_row[3] == 3 + assert first_row[4] == 21 + assert len(first_row[5]) == 224 * 224 * 3 + +if __name__ == "__main__": + pytest.main([__file__]) From 2fe824213ac7d65597aff4784dd30a6a42c58dca Mon Sep 17 00:00:00 2001 From: Yuhao Yang Date: Tue, 13 Mar 2018 14:07:45 -0700 Subject: [PATCH 597/823] DataFrame-based image reader and transformer (#2325) * DataFrame image support * comment update * sql adapter * rename ut * remove a ut * improve python ut * bypass 1.5 * add ut * import existential * clone transformer --- dlframes/dl_image_reader.py | 42 ++++++++++++++++++++++++++ dlframes/dl_image_transformer.py | 52 ++++++++++++++++++++++++++++++++ 2 files changed, 94 insertions(+) create mode 100644 dlframes/dl_image_reader.py create mode 100644 dlframes/dl_image_transformer.py diff --git a/dlframes/dl_image_reader.py b/dlframes/dl_image_reader.py new file mode 100644 index 00000000000..96c8c9bd71b --- /dev/null +++ b/dlframes/dl_image_reader.py @@ -0,0 +1,42 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import sys +from bigdl.util.common import callBigDlFunc + +if sys.version >= '3': + long = int + unicode = str + +class DLImageReader: + """ + Primary DataFrame-based image loading interface, defining API to read images from files + to DataFrame. + """ + + @staticmethod + def readImages(path, sc=None, minParitions = 1, bigdl_type="float"): + """ + Read the directory of images into DataFrame from the local or remote source. + :param path Directory to the input data files, the path can be comma separated paths as the + list of inputs. Wildcards path are supported similarly to sc.binaryFiles(path). + :param min_partitions A suggestion value of the minimal splitting number for input data. + :return DataFrame with a single column "image"; Each record in the column represents one image + record: Row (uri, height, width, channels, CvType, bytes) + """ + df = callBigDlFunc(bigdl_type, "dlReadImage", path, sc, minParitions) + df._sc._jsc = sc._jsc + return df diff --git a/dlframes/dl_image_transformer.py b/dlframes/dl_image_transformer.py new file mode 100644 index 00000000000..e4fe70a0e7a --- /dev/null +++ b/dlframes/dl_image_transformer.py @@ -0,0 +1,52 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import sys +from bigdl.util.common import JavaValue, callBigDlFunc +from pyspark.ml.param.shared import * +from pyspark.ml.wrapper import JavaTransformer + +if sys.version >= '3': + long = int + unicode = str + +class DLImageTransformer(JavaTransformer, HasInputCol, HasOutputCol, JavaValue): + """ + Provides DataFrame-based API for image pre-processing and feature transformation. + DLImageTransformer follows the Spark Transformer API pattern and can be used as one stage + in Spark ML pipeline. + + The input column can be either DLImageSchema.byteSchema or DLImageSchema.floatSchema. If + using DLImageReader, the default format is DLImageSchema.byteSchema + The output column is always DLImageSchema.floatSchema. + """ + + def __init__(self, transformer, jvalue=None, bigdl_type="float"): + super(DLImageTransformer, self).__init__() + self.value = jvalue if jvalue else callBigDlFunc( + bigdl_type, self.jvm_class_constructor(), transformer) + self._java_obj = self.value + self.bigdl_type = bigdl_type + + def transform(self, dataset): + """ + Apply the transformer to the images in "inputCol" and store the transformed result + into "outputCols" + """ + self._transfer_params_to_java() + return callBigDlFunc(self.bigdl_type, "dlImageTransform", self.value, dataset) + + From 46b7ae106aeb9c20bbc4b2939045ecfb12a35a17 Mon Sep 17 00:00:00 2001 From: Yuhao Yang Date: Tue, 13 Mar 2018 14:07:45 -0700 Subject: [PATCH 598/823] DataFrame-based image reader and transformer (#2325) * DataFrame image support * comment update * sql adapter * rename ut * remove a ut * improve python ut * bypass 1.5 * add ut * import existential * clone transformer --- dlframes/dl_image_reader.py | 42 ++++++++++++++++++++++++++ dlframes/dl_image_transformer.py | 52 ++++++++++++++++++++++++++++++++ 2 files changed, 94 insertions(+) create mode 100644 dlframes/dl_image_reader.py create mode 100644 dlframes/dl_image_transformer.py diff --git a/dlframes/dl_image_reader.py b/dlframes/dl_image_reader.py new file mode 100644 index 00000000000..96c8c9bd71b --- /dev/null +++ b/dlframes/dl_image_reader.py @@ -0,0 +1,42 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import sys +from bigdl.util.common import callBigDlFunc + +if sys.version >= '3': + long = int + unicode = str + +class DLImageReader: + """ + Primary DataFrame-based image loading interface, defining API to read images from files + to DataFrame. + """ + + @staticmethod + def readImages(path, sc=None, minParitions = 1, bigdl_type="float"): + """ + Read the directory of images into DataFrame from the local or remote source. + :param path Directory to the input data files, the path can be comma separated paths as the + list of inputs. Wildcards path are supported similarly to sc.binaryFiles(path). + :param min_partitions A suggestion value of the minimal splitting number for input data. + :return DataFrame with a single column "image"; Each record in the column represents one image + record: Row (uri, height, width, channels, CvType, bytes) + """ + df = callBigDlFunc(bigdl_type, "dlReadImage", path, sc, minParitions) + df._sc._jsc = sc._jsc + return df diff --git a/dlframes/dl_image_transformer.py b/dlframes/dl_image_transformer.py new file mode 100644 index 00000000000..e4fe70a0e7a --- /dev/null +++ b/dlframes/dl_image_transformer.py @@ -0,0 +1,52 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import sys +from bigdl.util.common import JavaValue, callBigDlFunc +from pyspark.ml.param.shared import * +from pyspark.ml.wrapper import JavaTransformer + +if sys.version >= '3': + long = int + unicode = str + +class DLImageTransformer(JavaTransformer, HasInputCol, HasOutputCol, JavaValue): + """ + Provides DataFrame-based API for image pre-processing and feature transformation. + DLImageTransformer follows the Spark Transformer API pattern and can be used as one stage + in Spark ML pipeline. + + The input column can be either DLImageSchema.byteSchema or DLImageSchema.floatSchema. If + using DLImageReader, the default format is DLImageSchema.byteSchema + The output column is always DLImageSchema.floatSchema. + """ + + def __init__(self, transformer, jvalue=None, bigdl_type="float"): + super(DLImageTransformer, self).__init__() + self.value = jvalue if jvalue else callBigDlFunc( + bigdl_type, self.jvm_class_constructor(), transformer) + self._java_obj = self.value + self.bigdl_type = bigdl_type + + def transform(self, dataset): + """ + Apply the transformer to the images in "inputCol" and store the transformed result + into "outputCols" + """ + self._transfer_params_to_java() + return callBigDlFunc(self.bigdl_type, "dlImageTransform", self.value, dataset) + + From 1866c4ed17cce6f2830d3e5c84d203eceafc918e Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Wed, 14 Mar 2018 20:11:19 +0800 Subject: [PATCH 599/823] Keras-like API for training and evaluation (#2306) * update scala compile fit * update * python topology * refactor lenet * update * refactor fit * add python ut * ut for image dataset * fix ut * fix * update docs * update readme * fix --- python/dllib/test/bigdl/keras/test_newapi.py | 40 +++++++++++++++++++- python/dllib/test/bigdl/test_utils.py | 2 +- 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/python/dllib/test/bigdl/keras/test_newapi.py b/python/dllib/test/bigdl/keras/test_newapi.py index 6f47fd23852..3be161ddf61 100644 --- a/python/dllib/test/bigdl/keras/test_newapi.py +++ b/python/dllib/test/bigdl/keras/test_newapi.py @@ -20,8 +20,8 @@ import bigdl.nn.keras.layer as BLayer import keras.layers as KLayer import keras.backend as K -import numpy as np from bigdl.keras.converter import WeightsConverter +from bigdl.dataset.dataset import * np.random.seed(1337) # for reproducibility @@ -128,15 +128,51 @@ def test_lenet_shape(self): np.testing.assert_allclose((10, ), output_shape[1:]) def test_graph(self): + from bigdl.nn.keras.topology import Model as BModel x1 = BLayer.Input(input_shape=(8, )) x2 = BLayer.Input(input_shape=(6, )) y1 = BLayer.Dense(10)(x1) y2 = BLayer.Dense(10)(x2) - model = BLayer.Model([x1, x2], [y1, y2]) + model = BModel([x1, x2], [y1, y2]) input_shapes = model.get_input_shape() np.testing.assert_allclose((8, ), input_shapes[0][1:]) np.testing.assert_allclose((6, ), input_shapes[1][1:]) + def test_train(self): + from bigdl.nn.keras.topology import Sequential as BSequential + x = np.random.random([32, 10]) + y = np.random.random([32, ]) + model = BSequential() + model.add(BLayer.Dense(5, input_shape=(10, ))) + model.compile(optimizer="sgd", loss="mse", metrics=["accuracy"]) + model.fit(x, y, batch_size=8, nb_epoch=2, validation_data=(x, y)) + model.evaluate(x, y, batch_size=8) + model.predict(x) + + def test_train_dataset(self): + images = [] + labels = [] + for i in range(0, 8): + features = np.random.uniform(0, 1, (200, 200, 3)) + label = np.array([2]) + images.append(features) + labels.append(label) + image_frame = DistributedImageFrame(self.sc.parallelize(images), + self.sc.parallelize(labels)) + + transformer = Pipeline([BytesToMat(), Resize(256, 256), CenterCrop(224, 224), + ChannelNormalize(0.485, 0.456, 0.406, 0.229, 0.224, 0.225), + MatToTensor(), ImageFrameToSample(target_keys=['label'])]) + data_set = DataSet.image_frame(image_frame).transform(transformer) + + from bigdl.nn.keras.topology import Sequential as BSequential + model = BSequential() + model.add(BLayer.Convolution2D(1, 5, 5, input_shape=(3, 224, 224))) + model.add(BLayer.Reshape((1*220*220, ))) + model.add(BLayer.Dense(20, activation="softmax")) + model.compile(optimizer="sgd", loss="sparse_categorical_crossentropy", metrics=["accuracy"]) + model.fit(data_set, batch_size=8, nb_epoch=2, validation_data=data_set) + if __name__ == "__main__": pytest.main([__file__]) diff --git a/python/dllib/test/bigdl/test_utils.py b/python/dllib/test/bigdl/test_utils.py index 04f9869aa08..2f8b2821bd0 100644 --- a/python/dllib/test/bigdl/test_utils.py +++ b/python/dllib/test/bigdl/test_utils.py @@ -323,7 +323,7 @@ def compare_loss(self, y_a, y_b, kloss, bloss, rtol=1e-6, atol=1e-6): def compare_newapi(self, klayer, blayer, input_data, weight_converter=None, is_training=False, rtol=1e-6, atol=1e-6): from keras.models import Sequential as KSequential - from bigdl.nn.keras.layer import Sequential as BSequential + from bigdl.nn.keras.topology import Sequential as BSequential bmodel = BSequential() bmodel.add(blayer) kmodel = KSequential() From b8061023ea5aa55ec9342aae9f0710729871f72f Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Wed, 14 Mar 2018 20:11:19 +0800 Subject: [PATCH 600/823] Keras-like API for training and evaluation (#2306) * update scala compile fit * update * python topology * refactor lenet * update * refactor fit * add python ut * ut for image dataset * fix ut * fix * update docs * update readme * fix --- .../src/bigdl/dllib/bigdlkeras/backend.py | 9 +- .../bigdl/dllib/bigdlkeras/layers/layer.py | 31 +--- .../bigdl/dllib/bigdlkeras/layers/topology.py | 155 ++++++++++++++++++ .../bigdl/dllib/bigdlkeras/optimization.py | 100 ++++++----- .../src/bigdl/dllib/examples/lenet/README.md | 19 +-- .../src/bigdl/dllib/examples/lenet/lenet.py | 35 ++-- .../src/bigdl/dllib/feature/dataset/mnist.py | 11 ++ .../src/bigdl/dllib/keras/ToBigDLHelper.py | 2 + .../dllib/src/bigdl/dllib/keras/converter.py | 6 - .../dllib/models/local_lenet/local_lenet.py | 3 +- .../src/bigdl/dllib/nn/keras/__init__.py | 20 +++ python/dllib/src/bigdl/dllib/nn/layer.py | 3 +- 12 files changed, 269 insertions(+), 125 deletions(-) create mode 100644 python/dllib/src/bigdl/dllib/bigdlkeras/layers/topology.py diff --git a/python/dllib/src/bigdl/dllib/bigdlkeras/backend.py b/python/dllib/src/bigdl/dllib/bigdlkeras/backend.py index 91be1a1a528..8b250484626 100644 --- a/python/dllib/src/bigdl/dllib/bigdlkeras/backend.py +++ b/python/dllib/src/bigdl/dllib/bigdlkeras/backend.py @@ -14,13 +14,8 @@ # limitations under the License. # -import numpy as np -from pyspark.rdd import RDD - from bigdl.keras.optimization import * -from bigdl.util.common import get_spark_context -from bigdl.util.common import to_sample_rdd -from bigdl.util.common import redire_spark_logs, show_bigdl_info_logs +from bigdl.util.common import * class KerasModelWrapper: @@ -180,6 +175,6 @@ def __create_distributed_optimizer(self, training_rdd, def with_bigdl_backend(kmodel): - bcommon.init_engine() + init_engine() return KerasModelWrapper(kmodel) diff --git a/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py b/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py index 120a8b3aa3b..31261882ebb 100644 --- a/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py +++ b/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py @@ -17,7 +17,7 @@ import sys from bigdl.nn.layer import Layer, Node -from bigdl.util.common import callBigDlFunc, JTensor, JavaValue, to_list +from bigdl.util.common import callBigDlFunc, JTensor, JavaValue if sys.version >= '3': long = int @@ -56,43 +56,18 @@ def get_output_shape(self): self.value) return self.__process_shape(output) + class KerasCreator(JavaValue): def jvm_class_constructor(self): name = "createKeras" + self.__class__.__name__ print("creating: " + name) return name -class KerasLayer(Layer, InferShape, KerasCreator): - pass -class KerasModel(KerasLayer): - # TODO: enrich the KerasModel related API here. +class KerasLayer(Layer, InferShape, KerasCreator): pass -class Sequential(KerasModel): - """ - Container for a Sequential model. - >>> import bigdl.nn.keras.layer - >>> sequential = bigdl.nn.keras.layer.Sequential() - creating: createKerasSequential - """ - def __init__(self, bigdl_type="float"): - super(Sequential, self).__init__(None, bigdl_type=bigdl_type) - - def add(self, model): - self.value.add(model.value) - return self - - -class Model(KerasModel): - def __init__(self, input, output, bigdl_type="float"): - super(Model, self).__init__(None, bigdl_type, - to_list(input), - to_list(output), - ) - - class Input(Node, KerasCreator): def __init__(self, name=None, input_shape=None, bigdl_type="float"): super(Input, self).__init__(None, bigdl_type, diff --git a/python/dllib/src/bigdl/dllib/bigdlkeras/layers/topology.py b/python/dllib/src/bigdl/dllib/bigdlkeras/layers/topology.py new file mode 100644 index 00000000000..1e7867901cc --- /dev/null +++ b/python/dllib/src/bigdl/dllib/bigdlkeras/layers/topology.py @@ -0,0 +1,155 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from bigdl.nn.keras.layer import KerasLayer +from bigdl.dataset.dataset import * +from bigdl.keras.optimization import OptimConverter +import multiprocessing + + +class KerasModel(KerasLayer): + def compile(self, optimizer, loss, metrics=None): + """ + Configures the learning process. Must be called before fit. + + # Arguments + optimizer: Optimization method to be used. One can alternatively pass in the corresponding + string representation, such as 'sgd'. + loss: Criterion to be used. One can alternatively pass in the corresponding string + representation, such as 'mse'. + metrics: List of validation methods to be used. Default is None. One can alternatively use ['accuracy']. + """ + if isinstance(optimizer, six.string_types): + optimizer = OptimConverter.to_bigdl_optim_method(optimizer) + if isinstance(loss, six.string_types): + loss = OptimConverter.to_bigdl_criterion(loss) + if all(isinstance(metric, six.string_types) for metric in metrics): + metrics = OptimConverter.to_bigdl_metrics(metrics) + callBigDlFunc(self.bigdl_type, "compile", + self.value, + optimizer, + loss, + metrics) + + def fit(self, x, y=None, batch_size=32, nb_epoch=10, validation_data=None, distributed=True): + """ + Train a model for a fixed number of epochs on a dataset. + + # Arguments + x: Input data. A Numpy array or RDD of Sample or Image DataSet. + y: Labels. A Numpy array. Default is None if x is already RDD of Sample or Image DataSet. + batch_size: Number of samples per gradient update. + nb_epoch: Number of iterations to train. + validation_data: Tuple (x_val, y_val) where x_val and y_val are both Numpy arrays. + Or RDD of Sample. Default is None if no validation is involved. + distributed: Boolean. Whether to train the model in distributed mode or local mode. + Default is True. In local mode, x and y must both be Numpy arrays. + """ + if distributed: + if isinstance(x, np.ndarray) and isinstance(y, np.ndarray): + training_data = to_sample_rdd(x, y) + if validation_data: + validation_data = to_sample_rdd(*validation_data) + elif (isinstance(x, RDD) or isinstance(x, DataSet)) and not y: + training_data = x + else: + raise TypeError("Unsupported training data type: %s" % type(x)) + callBigDlFunc(self.bigdl_type, "fit", + self.value, + training_data, + batch_size, + nb_epoch, + validation_data) + else: + if validation_data: + val_x = [JTensor.from_ndarray(x) for x in to_list(validation_data[0])] + val_y = JTensor.from_ndarray(validation_data[1]) + else: + val_x, val_y = None, None + callBigDlFunc(self.bigdl_type, "fit", + self.value, + [JTensor.from_ndarray(x) for x in to_list(x)], + JTensor.from_ndarray(y), + batch_size, + nb_epoch, + val_x, + val_y, + multiprocessing.cpu_count()) + + def evaluate(self, x, y=None, batch_size=32): + """ + Evaluate a model on a given dataset in distributed mode. + + # Arguments + x: Input data. ANumpy array or RDD of Sample. + y: Labels. A Numpy array. Default is None if x is already RDD of Sample. + batch_size: Number of samples per gradient update. + """ + if isinstance(x, np.ndarray) and isinstance(y, np.ndarray): + evaluation_data = to_sample_rdd(x, y) + elif isinstance(x, RDD) and not y: + evaluation_data = x + else: + raise TypeError("Unsupported evaluation data type: %s" % type(x)) + return callBigDlFunc(self.bigdl_type, "evaluate", + self.value, + evaluation_data, + batch_size) + + def predict(self, x, distributed=True): + """ + Use a model to do prediction. + + # Arguments + x: Input data. A Numpy array or RDD of Sample. + distributed: Boolean. Whether to do prediction in distributed mode or local mode. + Default is True. In local mode, x must be a Numpy array. + """ + if is_distributed: + if isinstance(x, np.ndarray): + features = to_sample_rdd(x, np.zeros([x.shape[0]])) + elif isinstance(x, RDD): + features = x + else: + raise TypeError("Unsupported prediction data type: %s" % type(x)) + return self.predict_distributed(features) + else: + if isinstance(x, np.ndarray): + return self.predict_local(x) + else: + raise TypeError("Unsupported prediction data type: %s" % type(x)) + + +class Sequential(KerasModel): + """ + Container for a Sequential model. + + >>> sequential = Sequential() + creating: createKerasSequential + """ + def __init__(self, bigdl_type="float"): + super(Sequential, self).__init__(None, bigdl_type=bigdl_type) + + def add(self, model): + self.value.add(model.value) + return self + + +class Model(KerasModel): + def __init__(self, input, output, bigdl_type="float"): + super(Model, self).__init__(None, bigdl_type, + to_list(input), + to_list(output)) \ No newline at end of file diff --git a/python/dllib/src/bigdl/dllib/bigdlkeras/optimization.py b/python/dllib/src/bigdl/dllib/bigdlkeras/optimization.py index 250f3d4f886..3bced61db49 100644 --- a/python/dllib/src/bigdl/dllib/bigdlkeras/optimization.py +++ b/python/dllib/src/bigdl/dllib/bigdlkeras/optimization.py @@ -18,7 +18,7 @@ import bigdl.nn.criterion as bcriterion import bigdl.optim.optimizer as boptimizer -import bigdl.util.common as bcommon +from bigdl.util.common import to_list from bigdl.keras.converter import * from keras.objectives import * import six @@ -28,7 +28,7 @@ class OptimConverter: @staticmethod def to_bigdl_metrics(metrics): - metrics = bcommon.to_list(metrics) + metrics = to_list(metrics) bmetrics = [] for metric in metrics: if metric == "accuracy": @@ -42,7 +42,6 @@ def to_bigdl_metrics(metrics): def to_bigdl_criterion(kloss): if isinstance(kloss, six.string_types): kloss = kloss.lower() - if kloss == "categorical_crossentropy" or kloss == categorical_crossentropy: return bcriterion.CategoricalCrossEntropy() elif kloss == "mse" or kloss == "mean_squared_error" or kloss == mse: @@ -76,44 +75,61 @@ def to_bigdl_criterion(kloss): @staticmethod def to_bigdl_optim_method(koptim_method): - # koptim_method is always an object - lr = float(K.eval(koptim_method.lr)) - decay = float(K.eval(koptim_method.decay)) - if isinstance(koptim_method, koptimizers.Adagrad): - warnings.warn("For Adagrad, we don't support epsilon for now") - return boptimizer.Adagrad(learningrate=lr, - learningrate_decay=decay) - elif isinstance(koptim_method, koptimizers.SGD): - momentum = float(K.eval(koptim_method.momentum)) - return boptimizer.SGD(learningrate=lr, - learningrate_decay=decay, - momentum=momentum, - nesterov=koptim_method.nesterov) - elif isinstance(koptim_method, koptimizers.Adam): - beta1 = float(K.eval(koptim_method.beta_1)) - beta2 = float(K.eval(koptim_method.beta_2)) - return boptimizer.Adam(learningrate=lr, - learningrate_decay=decay, - beta1=beta1, - beta2=beta2, - epsilon=koptim_method.epsilon) - elif isinstance(koptim_method, koptimizers.RMSprop): - rho = float(K.eval(koptim_method.rho)) - return boptimizer.RMSprop(learningrate=lr, + if isinstance(koptim_method, six.string_types): + koptim_method = koptim_method.lower() + if koptim_method == "adagrad": + return boptimizer.Adagrad(learningrate=0.01) + elif koptim_method == "sgd": + return boptimizer.SGD(learningrate=0.01) + elif koptim_method == "adam": + return boptimizer.Adam() + elif koptim_method == "rmsprop": + return boptimizer.RMSprop(learningrate=0.001, decayrate=0.9) + elif koptim_method == "adadelta": + return boptimizer.Adadelta(decayrate=0.95, epsilon=1e-8) + elif koptim_method == "adamax": + return boptimizer.Adamax(epsilon=1e-8) + else: + unsupport_exp(koptim_method) + else: + # koptim_method a Keras object + lr = float(K.eval(koptim_method.lr)) + decay = float(K.eval(koptim_method.decay)) + if isinstance(koptim_method, koptimizers.Adagrad): + warnings.warn("For Adagrad, we don't support epsilon for now") + return boptimizer.Adagrad(learningrate=lr, + learningrate_decay=decay) + elif isinstance(koptim_method, koptimizers.SGD): + momentum = float(K.eval(koptim_method.momentum)) + return boptimizer.SGD(learningrate=lr, learningrate_decay=decay, - decayrate=rho, - epsilon=koptim_method.epsilon) - elif isinstance(koptim_method, koptimizers.Adadelta): - warnings.warn("For Adadelta, we don't support learning rate and learning rate decay for now") - return boptimizer.Adadelta(decayrate=koptim_method.rho, + momentum=momentum, + nesterov=koptim_method.nesterov) + elif isinstance(koptim_method, koptimizers.Adam): + beta1 = float(K.eval(koptim_method.beta_1)) + beta2 = float(K.eval(koptim_method.beta_2)) + return boptimizer.Adam(learningrate=lr, + learningrate_decay=decay, + beta1=beta1, + beta2=beta2, epsilon=koptim_method.epsilon) - elif isinstance(koptim_method, koptimizers.Adamax): - beta1 = float(K.eval(koptim_method.beta_1)) - beta2 = float(K.eval(koptim_method.beta_2)) - warnings.warn("For Adamax, we don't support learning rate decay for now") - return boptimizer.Adamax(learningrate=lr, - beta1=beta1, - beta2=beta2, - epsilon=koptim_method.epsilon) - else: - unsupport_exp(koptim_method) + elif isinstance(koptim_method, koptimizers.RMSprop): + rho = float(K.eval(koptim_method.rho)) + return boptimizer.RMSprop(learningrate=lr, + learningrate_decay=decay, + decayrate=rho, + epsilon=koptim_method.epsilon) + elif isinstance(koptim_method, koptimizers.Adadelta): + warnings.warn("For Adadelta, we don't support learning rate and learning rate decay for now") + return boptimizer.Adadelta(decayrate=koptim_method.rho, + epsilon=koptim_method.epsilon) + elif isinstance(koptim_method, koptimizers.Adamax): + beta1 = float(K.eval(koptim_method.beta_1)) + beta2 = float(K.eval(koptim_method.beta_2)) + warnings.warn("For Adamax, we don't support learning rate decay for now") + return boptimizer.Adamax(learningrate=lr, + beta1=beta1, + beta2=beta2, + epsilon=koptim_method.epsilon) + else: + unsupport_exp(koptim_method) diff --git a/python/dllib/src/bigdl/dllib/examples/lenet/README.md b/python/dllib/src/bigdl/dllib/examples/lenet/README.md index 4eeeccd1e7b..c6661674425 100644 --- a/python/dllib/src/bigdl/dllib/examples/lenet/README.md +++ b/python/dllib/src/bigdl/dllib/examples/lenet/README.md @@ -1,8 +1,8 @@ # LeNet5 Model on MNIST with new API -This example defines a classical CNN model used in digital number classification with the new set of API in BigDL. For detailed information with regard to LeNet, please refer to . +This example defines a classical CNN model used in digital number classification with the new set of Keras-Style API in BigDL, which is more user-friendly. For detailed information with regard to LeNet, please refer to . -This example is the same as [../../model/lenet/lenet5.py](../../models/lenet/lenet5.py), except that this example uses new API for model definition. +This example is the same as [../../model/lenet/lenet5.py](../../models/lenet/lenet5.py), except that here we use the new Keras-Style API for model definition and training. ## Install dependencies * [Install dependencies](../../../README.md#install.dependencies) @@ -44,21 +44,14 @@ We would train a LeNet model in spark local mode with the following commands and --jars ${BigDL_JAR_PATH} \ --conf spark.driver.extraClassPath=${BigDL_JAR_PATH} \ --conf spark.executor.extraClassPath=bigdl-VERSION-jar-with-dependencies.jar \ - ${BigDL_HOME}/pyspark/bigdl/models/lenet/lenet5.py \ - --dataPath /tmp/mnist + ${BigDL_HOME}/pyspark/bigdl/models/lenet/lenet5.py ``` -* ```--dataPath``` option can be used to set the path for downloading mnist data, the default value is /tmp/mnist. Make sure that you have write permission to the specified path. +* ```--dataPath``` an option to set the path for downloading mnist data, the default value is /tmp/mnist. Make sure that you have write permission to the specified path. -* ```--batchSize``` option can be used to set batch size, the default value is 128. +* ```--batchSize``` an option to set the batch size, the default value is 128. -* ```--endTriggerType``` option can be used to control how to end the training process, the value can be "epoch" or "iteration" and default value is "epoch". - -* ```--endTriggerNum``` use together with ```endTriggerType```, the default value is 20. - -* ```--modelPath``` option can be used to set model path for testing, the default value is /tmp/lenet5/model.470. - -* ```--checkpointPath``` option can be used to set checkpoint path for saving model, the default value is /tmp/lenet5/. +* ```--maxEpoch``` an option to set the number of epochs to train the model, the default value is 20. To verify the accuracy, search "accuracy" from log: diff --git a/python/dllib/src/bigdl/dllib/examples/lenet/lenet.py b/python/dllib/src/bigdl/dllib/examples/lenet/lenet.py index ba76772d1a3..433be38325b 100644 --- a/python/dllib/src/bigdl/dllib/examples/lenet/lenet.py +++ b/python/dllib/src/bigdl/dllib/examples/lenet/lenet.py @@ -15,11 +15,9 @@ # from optparse import OptionParser -from bigdl.models.lenet.utils import * +from bigdl.nn.keras.topology import Sequential from bigdl.nn.keras.layer import * -from bigdl.nn.criterion import * -from bigdl.optim.optimizer import * -from bigdl.util.common import * +from bigdl.dataset import mnist def build_model(class_num): @@ -39,28 +37,15 @@ def build_model(class_num): if __name__ == "__main__": parser = OptionParser() parser.add_option("-b", "--batchSize", type=int, dest="batchSize", default="128") - parser.add_option("-c", "--checkpointPath", dest="checkpointPath", default="/tmp/lenet5") - parser.add_option("-t", "--endTriggerType", dest="endTriggerType", default="epoch") - parser.add_option("-n", "--endTriggerNum", type=int, dest="endTriggerNum", default="20") + parser.add_option("-n", "--maxEpoch", type=int, dest="maxEpoch", default="20") parser.add_option("-d", "--dataPath", dest="dataPath", default="/tmp/mnist") - (options, args) = parser.parse_args(sys.argv) - sc = SparkContext(appName="lenet5", conf=create_spark_conf()) - redire_spark_logs() - show_bigdl_info_logs() - init_engine() - - (train_data, test_data) = preprocess_mnist(sc, options) + (X_train, Y_train), (X_test, Y_test) = mnist.load_data(options.dataPath) - optimizer = Optimizer( - model=build_model(10), - training_rdd=train_data, - criterion=ClassNLLCriterion(logProbAsInput=False), - optim_method=SGD(learningrate=0.01, learningrate_decay=0.0002), - end_trigger=get_end_trigger(options), - batch_size=options.batchSize) - validate_optimizer(optimizer, test_data, options) - trained_model = optimizer.optimize() - parameters = trained_model.parameters() - sc.stop() + model = build_model(10) + model.compile(loss='sparse_categorical_crossentropy', + optimizer='adadelta', + metrics=['accuracy']) + model.fit(X_train, Y_train, batch_size=options.batchSize, nb_epoch=options.maxEpoch, + validation_data=(X_test, Y_test)) diff --git a/python/dllib/src/bigdl/dllib/feature/dataset/mnist.py b/python/dllib/src/bigdl/dllib/feature/dataset/mnist.py index 492882fd730..881d8db4a95 100644 --- a/python/dllib/src/bigdl/dllib/feature/dataset/mnist.py +++ b/python/dllib/src/bigdl/dllib/feature/dataset/mnist.py @@ -21,6 +21,7 @@ import numpy from bigdl.dataset import base +from bigdl.dataset.transformer import * SOURCE_URL = 'http://yann.lecun.com/exdb/mnist/' @@ -120,6 +121,16 @@ def read_data_sets(train_dir, data_type="train"): return test_images, test_labels +def load_data(location="/tmp/mnist"): + (train_images, train_labels) = read_data_sets(location, "train") + (test_images, test_labels) = read_data_sets(location, "test") + X_train = normalizer(train_images, TRAIN_MEAN, TRAIN_STD) + X_test = normalizer(test_images, TRAIN_MEAN, TRAIN_STD) + Y_train = train_labels + 1 + Y_test = test_labels + 1 + return (X_train, Y_train), (X_test, Y_test) + + if __name__ == "__main__": train, _ = read_data_sets("/tmp/mnist/", "train") test, _ = read_data_sets("/tmp/mnist", "test") diff --git a/python/dllib/src/bigdl/dllib/keras/ToBigDLHelper.py b/python/dllib/src/bigdl/dllib/keras/ToBigDLHelper.py index 01b81de7cd3..6e0484eec4c 100644 --- a/python/dllib/src/bigdl/dllib/keras/ToBigDLHelper.py +++ b/python/dllib/src/bigdl/dllib/keras/ToBigDLHelper.py @@ -28,6 +28,7 @@ def to_bigdl_2d_ordering(order): else: raise Exception("Unsupported dim_ordering: %s" % order) + def to_bigdl_3d_ordering(order): if order == "tf": return "channel_last" @@ -36,6 +37,7 @@ def to_bigdl_3d_ordering(order): else: raise Exception("Unsupported dim_ordering: %s" % order) + def to_bigdl_3d_padding(border_mode): if border_mode == "valid": return 0, 0, 0 diff --git a/python/dllib/src/bigdl/dllib/keras/converter.py b/python/dllib/src/bigdl/dllib/keras/converter.py index a012fead569..b6df571fab2 100644 --- a/python/dllib/src/bigdl/dllib/keras/converter.py +++ b/python/dllib/src/bigdl/dllib/keras/converter.py @@ -14,20 +14,14 @@ # limitations under the License. # -import bigdl.nn.initialization_method as BInit import numpy as np import bigdl.nn.layer as BLayer -from bigdl.optim.optimizer import L1L2Regularizer as BRegularizer -import bigdl.optim.optimizer as boptimizer -import bigdl.nn.criterion as bcriterion import bigdl.util.common as BCommon from bigdl.util.common import get_activation_by_name -import keras.optimizers as koptimizers from keras.models import model_from_json from keras.models import Sequential, Model, Layer import keras import warnings -from math import ceil from bigdl.keras.ToBigDLHelper import * diff --git a/python/dllib/src/bigdl/dllib/models/local_lenet/local_lenet.py b/python/dllib/src/bigdl/dllib/models/local_lenet/local_lenet.py index cc264abacba..125f060ff32 100644 --- a/python/dllib/src/bigdl/dllib/models/local_lenet/local_lenet.py +++ b/python/dllib/src/bigdl/dllib/models/local_lenet/local_lenet.py @@ -46,8 +46,7 @@ def get_mnist(data_type="train", location="/tmp/mnist"): show_bigdl_info_logs() init_engine() - (X_train, Y_train) = get_mnist("train", options.dataPath) - (X_test, Y_test) = get_mnist("test", options.dataPath) + (X_train, Y_train), (X_test, Y_test) = mnist.load_data(options.dataPath) # The model used here is exactly the same as the model in ../lenet/lenet5.py optimizer = Optimizer.create( diff --git a/python/dllib/src/bigdl/dllib/nn/keras/__init__.py b/python/dllib/src/bigdl/dllib/nn/keras/__init__.py index e69de29bb2d..086e4e1f057 100644 --- a/python/dllib/src/bigdl/dllib/nn/keras/__init__.py +++ b/python/dllib/src/bigdl/dllib/nn/keras/__init__.py @@ -0,0 +1,20 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from bigdl.util.common import * + +init_engine() +redire_spark_logs() +show_bigdl_info_logs() diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index e5bb601c40d..1224a23efdc 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -60,7 +60,6 @@ def remove_next_edges(self): callJavaFunc(self.value.removeNextEdges) - class Layer(JavaValue): """ Layer is the basic component of a neural network @@ -365,7 +364,7 @@ def predict_distributed(self, data_rdd): :return: An RDD represent the predict result. """ result = callBigDlFunc(self.bigdl_type, - "modelPredictRDD", self.value, data_rdd) + "modelPredictRDD", self.value, data_rdd) return result.map(lambda data: data.to_ndarray()) def predict_class_distributed(self, data_rdd): From f6249c04975277c4dab73dcfa5acf3b4b23a94a6 Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Wed, 14 Mar 2018 20:11:19 +0800 Subject: [PATCH 601/823] Keras-like API for training and evaluation (#2306) * update scala compile fit * update * python topology * refactor lenet * update * refactor fit * add python ut * ut for image dataset * fix ut * fix * update docs * update readme * fix --- .../src/bigdl/dllib/bigdlkeras/backend.py | 9 +- .../bigdl/dllib/bigdlkeras/layers/layer.py | 31 +--- .../bigdl/dllib/bigdlkeras/layers/topology.py | 155 ++++++++++++++++++ .../bigdl/dllib/bigdlkeras/optimization.py | 100 ++++++----- .../src/bigdl/dllib/examples/lenet/README.md | 19 +-- .../src/bigdl/dllib/examples/lenet/lenet.py | 35 ++-- .../src/bigdl/dllib/feature/dataset/mnist.py | 11 ++ .../src/bigdl/dllib/keras/ToBigDLHelper.py | 2 + .../dllib/src/bigdl/dllib/keras/converter.py | 6 - .../dllib/models/local_lenet/local_lenet.py | 3 +- .../src/bigdl/dllib/nn/keras/__init__.py | 20 +++ python/dllib/src/bigdl/dllib/nn/layer.py | 3 +- 12 files changed, 269 insertions(+), 125 deletions(-) create mode 100644 python/dllib/src/bigdl/dllib/bigdlkeras/layers/topology.py diff --git a/python/dllib/src/bigdl/dllib/bigdlkeras/backend.py b/python/dllib/src/bigdl/dllib/bigdlkeras/backend.py index 91be1a1a528..8b250484626 100644 --- a/python/dllib/src/bigdl/dllib/bigdlkeras/backend.py +++ b/python/dllib/src/bigdl/dllib/bigdlkeras/backend.py @@ -14,13 +14,8 @@ # limitations under the License. # -import numpy as np -from pyspark.rdd import RDD - from bigdl.keras.optimization import * -from bigdl.util.common import get_spark_context -from bigdl.util.common import to_sample_rdd -from bigdl.util.common import redire_spark_logs, show_bigdl_info_logs +from bigdl.util.common import * class KerasModelWrapper: @@ -180,6 +175,6 @@ def __create_distributed_optimizer(self, training_rdd, def with_bigdl_backend(kmodel): - bcommon.init_engine() + init_engine() return KerasModelWrapper(kmodel) diff --git a/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py b/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py index 120a8b3aa3b..31261882ebb 100644 --- a/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py +++ b/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py @@ -17,7 +17,7 @@ import sys from bigdl.nn.layer import Layer, Node -from bigdl.util.common import callBigDlFunc, JTensor, JavaValue, to_list +from bigdl.util.common import callBigDlFunc, JTensor, JavaValue if sys.version >= '3': long = int @@ -56,43 +56,18 @@ def get_output_shape(self): self.value) return self.__process_shape(output) + class KerasCreator(JavaValue): def jvm_class_constructor(self): name = "createKeras" + self.__class__.__name__ print("creating: " + name) return name -class KerasLayer(Layer, InferShape, KerasCreator): - pass -class KerasModel(KerasLayer): - # TODO: enrich the KerasModel related API here. +class KerasLayer(Layer, InferShape, KerasCreator): pass -class Sequential(KerasModel): - """ - Container for a Sequential model. - >>> import bigdl.nn.keras.layer - >>> sequential = bigdl.nn.keras.layer.Sequential() - creating: createKerasSequential - """ - def __init__(self, bigdl_type="float"): - super(Sequential, self).__init__(None, bigdl_type=bigdl_type) - - def add(self, model): - self.value.add(model.value) - return self - - -class Model(KerasModel): - def __init__(self, input, output, bigdl_type="float"): - super(Model, self).__init__(None, bigdl_type, - to_list(input), - to_list(output), - ) - - class Input(Node, KerasCreator): def __init__(self, name=None, input_shape=None, bigdl_type="float"): super(Input, self).__init__(None, bigdl_type, diff --git a/python/dllib/src/bigdl/dllib/bigdlkeras/layers/topology.py b/python/dllib/src/bigdl/dllib/bigdlkeras/layers/topology.py new file mode 100644 index 00000000000..1e7867901cc --- /dev/null +++ b/python/dllib/src/bigdl/dllib/bigdlkeras/layers/topology.py @@ -0,0 +1,155 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from bigdl.nn.keras.layer import KerasLayer +from bigdl.dataset.dataset import * +from bigdl.keras.optimization import OptimConverter +import multiprocessing + + +class KerasModel(KerasLayer): + def compile(self, optimizer, loss, metrics=None): + """ + Configures the learning process. Must be called before fit. + + # Arguments + optimizer: Optimization method to be used. One can alternatively pass in the corresponding + string representation, such as 'sgd'. + loss: Criterion to be used. One can alternatively pass in the corresponding string + representation, such as 'mse'. + metrics: List of validation methods to be used. Default is None. One can alternatively use ['accuracy']. + """ + if isinstance(optimizer, six.string_types): + optimizer = OptimConverter.to_bigdl_optim_method(optimizer) + if isinstance(loss, six.string_types): + loss = OptimConverter.to_bigdl_criterion(loss) + if all(isinstance(metric, six.string_types) for metric in metrics): + metrics = OptimConverter.to_bigdl_metrics(metrics) + callBigDlFunc(self.bigdl_type, "compile", + self.value, + optimizer, + loss, + metrics) + + def fit(self, x, y=None, batch_size=32, nb_epoch=10, validation_data=None, distributed=True): + """ + Train a model for a fixed number of epochs on a dataset. + + # Arguments + x: Input data. A Numpy array or RDD of Sample or Image DataSet. + y: Labels. A Numpy array. Default is None if x is already RDD of Sample or Image DataSet. + batch_size: Number of samples per gradient update. + nb_epoch: Number of iterations to train. + validation_data: Tuple (x_val, y_val) where x_val and y_val are both Numpy arrays. + Or RDD of Sample. Default is None if no validation is involved. + distributed: Boolean. Whether to train the model in distributed mode or local mode. + Default is True. In local mode, x and y must both be Numpy arrays. + """ + if distributed: + if isinstance(x, np.ndarray) and isinstance(y, np.ndarray): + training_data = to_sample_rdd(x, y) + if validation_data: + validation_data = to_sample_rdd(*validation_data) + elif (isinstance(x, RDD) or isinstance(x, DataSet)) and not y: + training_data = x + else: + raise TypeError("Unsupported training data type: %s" % type(x)) + callBigDlFunc(self.bigdl_type, "fit", + self.value, + training_data, + batch_size, + nb_epoch, + validation_data) + else: + if validation_data: + val_x = [JTensor.from_ndarray(x) for x in to_list(validation_data[0])] + val_y = JTensor.from_ndarray(validation_data[1]) + else: + val_x, val_y = None, None + callBigDlFunc(self.bigdl_type, "fit", + self.value, + [JTensor.from_ndarray(x) for x in to_list(x)], + JTensor.from_ndarray(y), + batch_size, + nb_epoch, + val_x, + val_y, + multiprocessing.cpu_count()) + + def evaluate(self, x, y=None, batch_size=32): + """ + Evaluate a model on a given dataset in distributed mode. + + # Arguments + x: Input data. ANumpy array or RDD of Sample. + y: Labels. A Numpy array. Default is None if x is already RDD of Sample. + batch_size: Number of samples per gradient update. + """ + if isinstance(x, np.ndarray) and isinstance(y, np.ndarray): + evaluation_data = to_sample_rdd(x, y) + elif isinstance(x, RDD) and not y: + evaluation_data = x + else: + raise TypeError("Unsupported evaluation data type: %s" % type(x)) + return callBigDlFunc(self.bigdl_type, "evaluate", + self.value, + evaluation_data, + batch_size) + + def predict(self, x, distributed=True): + """ + Use a model to do prediction. + + # Arguments + x: Input data. A Numpy array or RDD of Sample. + distributed: Boolean. Whether to do prediction in distributed mode or local mode. + Default is True. In local mode, x must be a Numpy array. + """ + if is_distributed: + if isinstance(x, np.ndarray): + features = to_sample_rdd(x, np.zeros([x.shape[0]])) + elif isinstance(x, RDD): + features = x + else: + raise TypeError("Unsupported prediction data type: %s" % type(x)) + return self.predict_distributed(features) + else: + if isinstance(x, np.ndarray): + return self.predict_local(x) + else: + raise TypeError("Unsupported prediction data type: %s" % type(x)) + + +class Sequential(KerasModel): + """ + Container for a Sequential model. + + >>> sequential = Sequential() + creating: createKerasSequential + """ + def __init__(self, bigdl_type="float"): + super(Sequential, self).__init__(None, bigdl_type=bigdl_type) + + def add(self, model): + self.value.add(model.value) + return self + + +class Model(KerasModel): + def __init__(self, input, output, bigdl_type="float"): + super(Model, self).__init__(None, bigdl_type, + to_list(input), + to_list(output)) \ No newline at end of file diff --git a/python/dllib/src/bigdl/dllib/bigdlkeras/optimization.py b/python/dllib/src/bigdl/dllib/bigdlkeras/optimization.py index 250f3d4f886..3bced61db49 100644 --- a/python/dllib/src/bigdl/dllib/bigdlkeras/optimization.py +++ b/python/dllib/src/bigdl/dllib/bigdlkeras/optimization.py @@ -18,7 +18,7 @@ import bigdl.nn.criterion as bcriterion import bigdl.optim.optimizer as boptimizer -import bigdl.util.common as bcommon +from bigdl.util.common import to_list from bigdl.keras.converter import * from keras.objectives import * import six @@ -28,7 +28,7 @@ class OptimConverter: @staticmethod def to_bigdl_metrics(metrics): - metrics = bcommon.to_list(metrics) + metrics = to_list(metrics) bmetrics = [] for metric in metrics: if metric == "accuracy": @@ -42,7 +42,6 @@ def to_bigdl_metrics(metrics): def to_bigdl_criterion(kloss): if isinstance(kloss, six.string_types): kloss = kloss.lower() - if kloss == "categorical_crossentropy" or kloss == categorical_crossentropy: return bcriterion.CategoricalCrossEntropy() elif kloss == "mse" or kloss == "mean_squared_error" or kloss == mse: @@ -76,44 +75,61 @@ def to_bigdl_criterion(kloss): @staticmethod def to_bigdl_optim_method(koptim_method): - # koptim_method is always an object - lr = float(K.eval(koptim_method.lr)) - decay = float(K.eval(koptim_method.decay)) - if isinstance(koptim_method, koptimizers.Adagrad): - warnings.warn("For Adagrad, we don't support epsilon for now") - return boptimizer.Adagrad(learningrate=lr, - learningrate_decay=decay) - elif isinstance(koptim_method, koptimizers.SGD): - momentum = float(K.eval(koptim_method.momentum)) - return boptimizer.SGD(learningrate=lr, - learningrate_decay=decay, - momentum=momentum, - nesterov=koptim_method.nesterov) - elif isinstance(koptim_method, koptimizers.Adam): - beta1 = float(K.eval(koptim_method.beta_1)) - beta2 = float(K.eval(koptim_method.beta_2)) - return boptimizer.Adam(learningrate=lr, - learningrate_decay=decay, - beta1=beta1, - beta2=beta2, - epsilon=koptim_method.epsilon) - elif isinstance(koptim_method, koptimizers.RMSprop): - rho = float(K.eval(koptim_method.rho)) - return boptimizer.RMSprop(learningrate=lr, + if isinstance(koptim_method, six.string_types): + koptim_method = koptim_method.lower() + if koptim_method == "adagrad": + return boptimizer.Adagrad(learningrate=0.01) + elif koptim_method == "sgd": + return boptimizer.SGD(learningrate=0.01) + elif koptim_method == "adam": + return boptimizer.Adam() + elif koptim_method == "rmsprop": + return boptimizer.RMSprop(learningrate=0.001, decayrate=0.9) + elif koptim_method == "adadelta": + return boptimizer.Adadelta(decayrate=0.95, epsilon=1e-8) + elif koptim_method == "adamax": + return boptimizer.Adamax(epsilon=1e-8) + else: + unsupport_exp(koptim_method) + else: + # koptim_method a Keras object + lr = float(K.eval(koptim_method.lr)) + decay = float(K.eval(koptim_method.decay)) + if isinstance(koptim_method, koptimizers.Adagrad): + warnings.warn("For Adagrad, we don't support epsilon for now") + return boptimizer.Adagrad(learningrate=lr, + learningrate_decay=decay) + elif isinstance(koptim_method, koptimizers.SGD): + momentum = float(K.eval(koptim_method.momentum)) + return boptimizer.SGD(learningrate=lr, learningrate_decay=decay, - decayrate=rho, - epsilon=koptim_method.epsilon) - elif isinstance(koptim_method, koptimizers.Adadelta): - warnings.warn("For Adadelta, we don't support learning rate and learning rate decay for now") - return boptimizer.Adadelta(decayrate=koptim_method.rho, + momentum=momentum, + nesterov=koptim_method.nesterov) + elif isinstance(koptim_method, koptimizers.Adam): + beta1 = float(K.eval(koptim_method.beta_1)) + beta2 = float(K.eval(koptim_method.beta_2)) + return boptimizer.Adam(learningrate=lr, + learningrate_decay=decay, + beta1=beta1, + beta2=beta2, epsilon=koptim_method.epsilon) - elif isinstance(koptim_method, koptimizers.Adamax): - beta1 = float(K.eval(koptim_method.beta_1)) - beta2 = float(K.eval(koptim_method.beta_2)) - warnings.warn("For Adamax, we don't support learning rate decay for now") - return boptimizer.Adamax(learningrate=lr, - beta1=beta1, - beta2=beta2, - epsilon=koptim_method.epsilon) - else: - unsupport_exp(koptim_method) + elif isinstance(koptim_method, koptimizers.RMSprop): + rho = float(K.eval(koptim_method.rho)) + return boptimizer.RMSprop(learningrate=lr, + learningrate_decay=decay, + decayrate=rho, + epsilon=koptim_method.epsilon) + elif isinstance(koptim_method, koptimizers.Adadelta): + warnings.warn("For Adadelta, we don't support learning rate and learning rate decay for now") + return boptimizer.Adadelta(decayrate=koptim_method.rho, + epsilon=koptim_method.epsilon) + elif isinstance(koptim_method, koptimizers.Adamax): + beta1 = float(K.eval(koptim_method.beta_1)) + beta2 = float(K.eval(koptim_method.beta_2)) + warnings.warn("For Adamax, we don't support learning rate decay for now") + return boptimizer.Adamax(learningrate=lr, + beta1=beta1, + beta2=beta2, + epsilon=koptim_method.epsilon) + else: + unsupport_exp(koptim_method) diff --git a/python/dllib/src/bigdl/dllib/examples/lenet/README.md b/python/dllib/src/bigdl/dllib/examples/lenet/README.md index 4eeeccd1e7b..c6661674425 100644 --- a/python/dllib/src/bigdl/dllib/examples/lenet/README.md +++ b/python/dllib/src/bigdl/dllib/examples/lenet/README.md @@ -1,8 +1,8 @@ # LeNet5 Model on MNIST with new API -This example defines a classical CNN model used in digital number classification with the new set of API in BigDL. For detailed information with regard to LeNet, please refer to . +This example defines a classical CNN model used in digital number classification with the new set of Keras-Style API in BigDL, which is more user-friendly. For detailed information with regard to LeNet, please refer to . -This example is the same as [../../model/lenet/lenet5.py](../../models/lenet/lenet5.py), except that this example uses new API for model definition. +This example is the same as [../../model/lenet/lenet5.py](../../models/lenet/lenet5.py), except that here we use the new Keras-Style API for model definition and training. ## Install dependencies * [Install dependencies](../../../README.md#install.dependencies) @@ -44,21 +44,14 @@ We would train a LeNet model in spark local mode with the following commands and --jars ${BigDL_JAR_PATH} \ --conf spark.driver.extraClassPath=${BigDL_JAR_PATH} \ --conf spark.executor.extraClassPath=bigdl-VERSION-jar-with-dependencies.jar \ - ${BigDL_HOME}/pyspark/bigdl/models/lenet/lenet5.py \ - --dataPath /tmp/mnist + ${BigDL_HOME}/pyspark/bigdl/models/lenet/lenet5.py ``` -* ```--dataPath``` option can be used to set the path for downloading mnist data, the default value is /tmp/mnist. Make sure that you have write permission to the specified path. +* ```--dataPath``` an option to set the path for downloading mnist data, the default value is /tmp/mnist. Make sure that you have write permission to the specified path. -* ```--batchSize``` option can be used to set batch size, the default value is 128. +* ```--batchSize``` an option to set the batch size, the default value is 128. -* ```--endTriggerType``` option can be used to control how to end the training process, the value can be "epoch" or "iteration" and default value is "epoch". - -* ```--endTriggerNum``` use together with ```endTriggerType```, the default value is 20. - -* ```--modelPath``` option can be used to set model path for testing, the default value is /tmp/lenet5/model.470. - -* ```--checkpointPath``` option can be used to set checkpoint path for saving model, the default value is /tmp/lenet5/. +* ```--maxEpoch``` an option to set the number of epochs to train the model, the default value is 20. To verify the accuracy, search "accuracy" from log: diff --git a/python/dllib/src/bigdl/dllib/examples/lenet/lenet.py b/python/dllib/src/bigdl/dllib/examples/lenet/lenet.py index ba76772d1a3..433be38325b 100644 --- a/python/dllib/src/bigdl/dllib/examples/lenet/lenet.py +++ b/python/dllib/src/bigdl/dllib/examples/lenet/lenet.py @@ -15,11 +15,9 @@ # from optparse import OptionParser -from bigdl.models.lenet.utils import * +from bigdl.nn.keras.topology import Sequential from bigdl.nn.keras.layer import * -from bigdl.nn.criterion import * -from bigdl.optim.optimizer import * -from bigdl.util.common import * +from bigdl.dataset import mnist def build_model(class_num): @@ -39,28 +37,15 @@ def build_model(class_num): if __name__ == "__main__": parser = OptionParser() parser.add_option("-b", "--batchSize", type=int, dest="batchSize", default="128") - parser.add_option("-c", "--checkpointPath", dest="checkpointPath", default="/tmp/lenet5") - parser.add_option("-t", "--endTriggerType", dest="endTriggerType", default="epoch") - parser.add_option("-n", "--endTriggerNum", type=int, dest="endTriggerNum", default="20") + parser.add_option("-n", "--maxEpoch", type=int, dest="maxEpoch", default="20") parser.add_option("-d", "--dataPath", dest="dataPath", default="/tmp/mnist") - (options, args) = parser.parse_args(sys.argv) - sc = SparkContext(appName="lenet5", conf=create_spark_conf()) - redire_spark_logs() - show_bigdl_info_logs() - init_engine() - - (train_data, test_data) = preprocess_mnist(sc, options) + (X_train, Y_train), (X_test, Y_test) = mnist.load_data(options.dataPath) - optimizer = Optimizer( - model=build_model(10), - training_rdd=train_data, - criterion=ClassNLLCriterion(logProbAsInput=False), - optim_method=SGD(learningrate=0.01, learningrate_decay=0.0002), - end_trigger=get_end_trigger(options), - batch_size=options.batchSize) - validate_optimizer(optimizer, test_data, options) - trained_model = optimizer.optimize() - parameters = trained_model.parameters() - sc.stop() + model = build_model(10) + model.compile(loss='sparse_categorical_crossentropy', + optimizer='adadelta', + metrics=['accuracy']) + model.fit(X_train, Y_train, batch_size=options.batchSize, nb_epoch=options.maxEpoch, + validation_data=(X_test, Y_test)) diff --git a/python/dllib/src/bigdl/dllib/feature/dataset/mnist.py b/python/dllib/src/bigdl/dllib/feature/dataset/mnist.py index 492882fd730..881d8db4a95 100644 --- a/python/dllib/src/bigdl/dllib/feature/dataset/mnist.py +++ b/python/dllib/src/bigdl/dllib/feature/dataset/mnist.py @@ -21,6 +21,7 @@ import numpy from bigdl.dataset import base +from bigdl.dataset.transformer import * SOURCE_URL = 'http://yann.lecun.com/exdb/mnist/' @@ -120,6 +121,16 @@ def read_data_sets(train_dir, data_type="train"): return test_images, test_labels +def load_data(location="/tmp/mnist"): + (train_images, train_labels) = read_data_sets(location, "train") + (test_images, test_labels) = read_data_sets(location, "test") + X_train = normalizer(train_images, TRAIN_MEAN, TRAIN_STD) + X_test = normalizer(test_images, TRAIN_MEAN, TRAIN_STD) + Y_train = train_labels + 1 + Y_test = test_labels + 1 + return (X_train, Y_train), (X_test, Y_test) + + if __name__ == "__main__": train, _ = read_data_sets("/tmp/mnist/", "train") test, _ = read_data_sets("/tmp/mnist", "test") diff --git a/python/dllib/src/bigdl/dllib/keras/ToBigDLHelper.py b/python/dllib/src/bigdl/dllib/keras/ToBigDLHelper.py index 01b81de7cd3..6e0484eec4c 100644 --- a/python/dllib/src/bigdl/dllib/keras/ToBigDLHelper.py +++ b/python/dllib/src/bigdl/dllib/keras/ToBigDLHelper.py @@ -28,6 +28,7 @@ def to_bigdl_2d_ordering(order): else: raise Exception("Unsupported dim_ordering: %s" % order) + def to_bigdl_3d_ordering(order): if order == "tf": return "channel_last" @@ -36,6 +37,7 @@ def to_bigdl_3d_ordering(order): else: raise Exception("Unsupported dim_ordering: %s" % order) + def to_bigdl_3d_padding(border_mode): if border_mode == "valid": return 0, 0, 0 diff --git a/python/dllib/src/bigdl/dllib/keras/converter.py b/python/dllib/src/bigdl/dllib/keras/converter.py index a012fead569..b6df571fab2 100644 --- a/python/dllib/src/bigdl/dllib/keras/converter.py +++ b/python/dllib/src/bigdl/dllib/keras/converter.py @@ -14,20 +14,14 @@ # limitations under the License. # -import bigdl.nn.initialization_method as BInit import numpy as np import bigdl.nn.layer as BLayer -from bigdl.optim.optimizer import L1L2Regularizer as BRegularizer -import bigdl.optim.optimizer as boptimizer -import bigdl.nn.criterion as bcriterion import bigdl.util.common as BCommon from bigdl.util.common import get_activation_by_name -import keras.optimizers as koptimizers from keras.models import model_from_json from keras.models import Sequential, Model, Layer import keras import warnings -from math import ceil from bigdl.keras.ToBigDLHelper import * diff --git a/python/dllib/src/bigdl/dllib/models/local_lenet/local_lenet.py b/python/dllib/src/bigdl/dllib/models/local_lenet/local_lenet.py index cc264abacba..125f060ff32 100644 --- a/python/dllib/src/bigdl/dllib/models/local_lenet/local_lenet.py +++ b/python/dllib/src/bigdl/dllib/models/local_lenet/local_lenet.py @@ -46,8 +46,7 @@ def get_mnist(data_type="train", location="/tmp/mnist"): show_bigdl_info_logs() init_engine() - (X_train, Y_train) = get_mnist("train", options.dataPath) - (X_test, Y_test) = get_mnist("test", options.dataPath) + (X_train, Y_train), (X_test, Y_test) = mnist.load_data(options.dataPath) # The model used here is exactly the same as the model in ../lenet/lenet5.py optimizer = Optimizer.create( diff --git a/python/dllib/src/bigdl/dllib/nn/keras/__init__.py b/python/dllib/src/bigdl/dllib/nn/keras/__init__.py index e69de29bb2d..086e4e1f057 100644 --- a/python/dllib/src/bigdl/dllib/nn/keras/__init__.py +++ b/python/dllib/src/bigdl/dllib/nn/keras/__init__.py @@ -0,0 +1,20 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from bigdl.util.common import * + +init_engine() +redire_spark_logs() +show_bigdl_info_logs() diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index e5bb601c40d..1224a23efdc 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -60,7 +60,6 @@ def remove_next_edges(self): callJavaFunc(self.value.removeNextEdges) - class Layer(JavaValue): """ Layer is the basic component of a neural network @@ -365,7 +364,7 @@ def predict_distributed(self, data_rdd): :return: An RDD represent the predict result. """ result = callBigDlFunc(self.bigdl_type, - "modelPredictRDD", self.value, data_rdd) + "modelPredictRDD", self.value, data_rdd) return result.map(lambda data: data.to_ndarray()) def predict_class_distributed(self, data_rdd): From 73f387955ac71bea5e2ff7ad2efea3d4ce64d859 Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Fri, 16 Mar 2018 13:04:28 +0800 Subject: [PATCH 602/823] Keras-like API functional merge and some fix (#2313) * resolve conflicts * update merge * update merge * update doc * refactor python * fix * style * meet review --- .../test/bigdl/keras/test_application.py | 8 +- python/dllib/test/bigdl/keras/test_newapi.py | 87 +++++++++++++++++-- python/dllib/test/bigdl/test_utils.py | 10 ++- 3 files changed, 89 insertions(+), 16 deletions(-) diff --git a/python/dllib/test/bigdl/keras/test_application.py b/python/dllib/test/bigdl/keras/test_application.py index 36d2926bffb..0d9a4587608 100644 --- a/python/dllib/test/bigdl/keras/test_application.py +++ b/python/dllib/test/bigdl/keras/test_application.py @@ -31,13 +31,7 @@ class TestApplication(BigDLTestCase): def assert_model(self, input_data, kmodel, rtol=1e-5, atol=1e-5): bmodel = DefinitionLoader.from_kmodel(kmodel) - WeightLoader.load_weights_from_kmodel(bmodel, kmodel) - - keras_output = kmodel.predict(input_data) - bmodel.training(is_training=False) - bigdl_output = bmodel.forward(input_data) - - self.assert_allclose(keras_output, bigdl_output, rtol=rtol, atol=atol) + self.compare_model(bmodel, kmodel, input_data, rtol, atol) def test_lenet(self): K.set_image_dim_ordering("th") diff --git a/python/dllib/test/bigdl/keras/test_newapi.py b/python/dllib/test/bigdl/keras/test_newapi.py index 3be161ddf61..55a7a6689ed 100644 --- a/python/dllib/test/bigdl/keras/test_newapi.py +++ b/python/dllib/test/bigdl/keras/test_newapi.py @@ -22,11 +22,15 @@ import keras.backend as K from bigdl.keras.converter import WeightsConverter from bigdl.dataset.dataset import * +from bigdl.nn.keras.topology import Model as BModel +from bigdl.nn.keras.topology import Sequential as BSequential +from keras.engine import merge as kmerge, Model as KModel +from keras.models import Sequential as KSequential np.random.seed(1337) # for reproducibility -class TestLayer(BigDLTestCase): +class TestNewAPI(BigDLTestCase): def test_embedding(self): input_data = np.random.randint(1000, size=(32, 10)) @@ -38,13 +42,13 @@ def test_embedding(self): def test_batchnormalization(self): K.set_image_dim_ordering("th") input_data = np.random.random_sample([2, 5, 32, 32]) - blayer = BLayer.BatchNormalization(input_shape=(5, 32, 32)) + blayer = BLayer.BatchNormalization(axis=1, input_shape=(5, 32, 32)) klayer = KLayer.BatchNormalization(axis=1, input_shape=(5, 32, 32)) self.compare_newapi(klayer, blayer, input_data, WeightsConverter.convert_batchnormalization) K.set_image_dim_ordering("tf") input_data2 = np.random.random_sample([2, 32, 32, 4]) - blayer = BLayer.BatchNormalization(dim_ordering="tf", input_shape=(32, 32, 4)) + blayer = BLayer.BatchNormalization(axis=-1, dim_ordering="tf", input_shape=(32, 32, 4)) klayer = KLayer.BatchNormalization(axis=-1, input_shape=(32, 32, 4)) self.compare_newapi(klayer, blayer, input_data2, WeightsConverter.convert_batchnormalization) @@ -128,18 +132,19 @@ def test_lenet_shape(self): np.testing.assert_allclose((10, ), output_shape[1:]) def test_graph(self): - from bigdl.nn.keras.topology import Model as BModel - x1 = BLayer.Input(input_shape=(8, )) - x2 = BLayer.Input(input_shape=(6, )) + x1 = BLayer.Input(shape=(8, )) + x2 = BLayer.Input(shape=(6, )) y1 = BLayer.Dense(10)(x1) y2 = BLayer.Dense(10)(x2) model = BModel([x1, x2], [y1, y2]) input_shapes = model.get_input_shape() + output_shapes = model.get_output_shape() np.testing.assert_allclose((8, ), input_shapes[0][1:]) np.testing.assert_allclose((6, ), input_shapes[1][1:]) + np.testing.assert_allclose((10, ), output_shapes[0][1:]) + np.testing.assert_allclose((10, ), output_shapes[1][1:]) def test_train(self): - from bigdl.nn.keras.topology import Sequential as BSequential x = np.random.random([32, 10]) y = np.random.random([32, ]) model = BSequential() @@ -165,7 +170,6 @@ def test_train_dataset(self): MatToTensor(), ImageFrameToSample(target_keys=['label'])]) data_set = DataSet.image_frame(image_frame).transform(transformer) - from bigdl.nn.keras.topology import Sequential as BSequential model = BSequential() model.add(BLayer.Convolution2D(1, 5, 5, input_shape=(3, 224, 224))) model.add(BLayer.Reshape((1*220*220, ))) @@ -173,6 +177,73 @@ def test_train_dataset(self): model.compile(optimizer="sgd", loss="sparse_categorical_crossentropy", metrics=["accuracy"]) model.fit(data_set, batch_size=8, nb_epoch=2, validation_data=data_set) + def convert_two_dense_model(self, kmodel, weights): + return [weights[2].T, weights[3], weights[0].T, weights[1]] + + def test_merge_method_sum(self): + bx1 = BLayer.Input(shape=(8, )) + bx2 = BLayer.Input(shape=(6, )) + by1 = BLayer.Dense(10)(bx1) + by2 = BLayer.Dense(10)(bx2) + bz = BLayer.merge([by1, by2], mode="sum") + bmodel = BModel([bx1, bx2], bz, name="graph1") + + kx1 = KLayer.Input(shape=(8, )) + kx2 = KLayer.Input(shape=(6, )) + ky1 = KLayer.Dense(10)(kx1) + ky2 = KLayer.Dense(10)(kx2) + kz = kmerge([ky1, ky2], mode="sum") + kmodel = KModel([kx1, kx2], kz) + + input_data = [np.random.random([2, 8]), np.random.random([2, 6])] + self.compare_newapi(kmodel, bmodel, input_data, self.convert_two_dense_model) + + def test_merge_method_model_concat(self): + bx1 = BLayer.Input(shape=(4, )) + bx1_1 = BLayer.Input(shape=(4, )) + bx2 = BLayer.Input(shape=(5, )) + by1 = BLayer.Dense(6, activation="sigmoid")(bx1) + bbranch1 = BModel(bx1, by1)(bx1_1) + bbranch2 = BLayer.Dense(8)(bx2) + bz = BLayer.merge([bbranch1, bbranch2], mode="concat") + bmodel = BModel([bx1_1, bx2], bz) + + kx1 = KLayer.Input(shape=(4, )) + kx2 = KLayer.Input(shape=(5, )) + ky1 = KLayer.Dense(6, activation="sigmoid")(kx1) + kbranch1 = KModel(kx1, ky1)(kx1) + kbranch2 = KLayer.Dense(8)(kx2) + kz = KLayer.merge([kbranch1, kbranch2], mode="concat") + kmodel = KModel([kx1, kx2], kz) + + input_data = [np.random.random([2, 4]), np.random.random([2, 5])] + self.compare_newapi(kmodel, bmodel, input_data, self.convert_two_dense_model) + + def test_merge_method_seq_concat(self): + bx1 = BLayer.Input(shape=(10, )) + bx1_1 = BLayer.Input(shape=(10, )) + bx2 = BLayer.Input(shape=(10, )) + by1 = BLayer.Dense(12, activation="sigmoid")(bx1) + bbranch1_node = BModel(bx1, by1)(bx1_1) + bbranch2 = BSequential() + bbranch2.add(BLayer.Dense(12, input_dim=10)) + bbranch2_node = bbranch2(bx2) + bz = BLayer.merge([bbranch1_node, bbranch2_node], mode="concat") + bmodel = BModel([bx1_1, bx2], bz) + + kx1 = KLayer.Input(shape=(10, )) + kx2 = KLayer.Input(shape=(10, )) + ky1 = KLayer.Dense(12, activation="sigmoid")(kx1) + kbranch1_node = KModel(kx1, ky1)(kx1) + kbranch2 = KSequential() + kbranch2.add(KLayer.Dense(12, input_dim=10)) + kbranch2_node = kbranch2(kx2) + kz = KLayer.merge([kbranch1_node, kbranch2_node], mode="concat") + kmodel = KModel([kx1, kx2], kz) + + input_data = [np.random.random([2, 10]), np.random.random([2, 10])] + self.compare_newapi(kmodel, bmodel, input_data, self.convert_two_dense_model) + if __name__ == "__main__": pytest.main([__file__]) diff --git a/python/dllib/test/bigdl/test_utils.py b/python/dllib/test/bigdl/test_utils.py index 2f8b2821bd0..4eadd7bec69 100644 --- a/python/dllib/test/bigdl/test_utils.py +++ b/python/dllib/test/bigdl/test_utils.py @@ -329,7 +329,8 @@ def compare_newapi(self, klayer, blayer, input_data, weight_converter=None, kmodel = KSequential() kmodel.add(klayer) koutput = kmodel.predict(input_data) - if isinstance(blayer, BLayer.BatchNormalization): + from bigdl.nn.keras.layer import BatchNormalization + if isinstance(blayer, BatchNormalization): k_running_mean = K.eval(klayer.running_mean) k_running_std = K.eval(klayer.running_std) blayer.set_running_mean(k_running_mean) @@ -339,3 +340,10 @@ def compare_newapi(self, klayer, blayer, input_data, weight_converter=None, bmodel.training(is_training) boutput = bmodel.forward(input_data) self.assert_allclose(boutput, koutput, rtol=rtol, atol=atol) + + def compare_model(self, bmodel, kmodel, input_data, rtol=1e-5, atol=1e-5): + WeightLoader.load_weights_from_kmodel(bmodel, kmodel) + bmodel.training(is_training=False) + bigdl_output = bmodel.forward(input_data) + keras_output = kmodel.predict(input_data) + self.assert_allclose(bigdl_output, keras_output, rtol=rtol, atol=atol) From 3759ee332c37d7402bce92c7349a9629436f82a9 Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Fri, 16 Mar 2018 13:04:28 +0800 Subject: [PATCH 603/823] Keras-like API functional merge and some fix (#2313) * resolve conflicts * update merge * update merge * update doc * refactor python * fix * style * meet review --- .../bigdl/dllib/bigdlkeras/layers/layer.py | 665 +++++++++++------- .../bigdl/dllib/bigdlkeras/layers/topology.py | 26 +- python/dllib/src/bigdl/dllib/nn/layer.py | 8 +- 3 files changed, 454 insertions(+), 245 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py b/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py index 31261882ebb..8b84dc0cdd3 100644 --- a/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py +++ b/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py @@ -65,29 +65,52 @@ def jvm_class_constructor(self): class KerasLayer(Layer, InferShape, KerasCreator): - pass + def __init__(self, jvalue, *args, **kwargs): + allowed_kwargs = {"name", "bigdl_type"} + for kwarg in kwargs.keys(): + if kwarg not in allowed_kwargs: + raise TypeError("Wrong argument for the layer:", kwarg) + bigdl_type = kwargs.get("bigdl_type") + if not bigdl_type: + bigdl_type = "float" + super(KerasCreator, self).__init__(jvalue, bigdl_type, *args) + name = kwargs.get("name") + if name: + self.set_name(name) class Input(Node, KerasCreator): - def __init__(self, name=None, input_shape=None, bigdl_type="float"): + """ + Used to instantiate an input node. + + # Arguments + shape: A shape tuple, not including batch. + name: String to set the name of the input node. If not specified, its name will by default to be a generated string. + + >>> input = Input(name="input1", shape=(3, 5)) + creating: createKerasInput + """ + def __init__(self, shape=None, name=None, bigdl_type="float"): super(Input, self).__init__(None, bigdl_type, name, - list(input_shape) if input_shape else None) + list(shape) if shape else None) class InputLayer(KerasLayer): """ - Layer to be used as an entry point into a model. + Used as an entry point into a model. # Arguments - input_shape: A shape tuple, not including the batch axis. + input_shape: A shape tuple, not including batch. + name: String to set the name of the input layer. If not specified, its name will by default to be a generated string. >>> inputlayer = InputLayer(input_shape=(3, 5)) creating: createKerasInputLayer """ - def __init__(self, input_shape=None, bigdl_type="float"): - super(InputLayer, self).__init__(None, bigdl_type, - list(input_shape) if input_shape else None) + def __init__(self, input_shape=None, **kwargs): + super(InputLayer, self).__init__(None, + list(input_shape) if input_shape else None, + **kwargs) class Dense(KerasLayer): @@ -100,30 +123,36 @@ class Dense(KerasLayer): # Arguments output_dim: The size of output dimension. - init: String representations of initialization method for the weights of the layer. + init: String representation of the initialization method for the weights of the layer. Default is 'glorot_uniform'. - activation: String representations of activation function to use (such as 'relu' or 'sigmoid'). + activation: String representation of the activation function to use (such as 'relu' or 'sigmoid'). Default is None. W_regularizer: An instance of [[Regularizer]], (eg. L1 or L2 regularization), applied to the input weights matrices. Default is None. b_regularizer: An instance of [[Regularizer]], applied to the bias. Default is None. bias: Whether to include a bias (i.e. make the layer affine rather than linear). Default is True. + input_dim: Dimensionality of the input for 2D input. For nD input, you can alternatively specify + 'input_shape' when using this layer as the first layer. input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. - >>> dense = Dense(10, input_shape=(3, 4)) + >>> dense = Dense(10, input_dim=8, name="dense1") creating: createKerasDense """ def __init__(self, output_dim, init="glorot_uniform", activation=None, W_regularizer=None, b_regularizer=None, - bias=True, input_shape=None, bigdl_type="float"): - super(Dense, self).__init__(None, bigdl_type, + bias=True, input_dim=None, input_shape=None, **kwargs): + if input_dim: + input_shape = (input_dim, ) + super(Dense, self).__init__(None, output_dim, init, activation, W_regularizer, b_regularizer, bias, - list(input_shape) if input_shape else None) + list(input_shape) if input_shape else None, + **kwargs) class MaxoutDense(KerasLayer): @@ -142,20 +171,26 @@ class MaxoutDense(KerasLayer): applied to the input weights matrices. Default is None. b_regularizer: An instance of [[Regularizer]], applied to the bias. Default is None. bias: Whether to include a bias (i.e. make the layer affine rather than linear). Default is True. + input_dim: Dimensionality of the input. Alternatively, you can specify 'input_shape' + when using this layer as the first layer. input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. >>> maxoutdense = MaxoutDense(6, input_shape=(10, )) creating: createKerasMaxoutDense """ - def __init__(self, output_dim, nb_feature=4, W_regularizer=None, - b_regularizer=None, bias=True, input_shape=None, bigdl_type="float"): - super(MaxoutDense, self).__init__(None, bigdl_type, + def __init__(self, output_dim, nb_feature=4, W_regularizer=None, b_regularizer=None, + bias=True, input_dim=None, input_shape=None, **kwargs): + if input_dim: + input_shape = (input_dim, ) + super(MaxoutDense, self).__init__(None, output_dim, nb_feature, W_regularizer, b_regularizer, bias, - list(input_shape) if input_shape else None) + list(input_shape) if input_shape else None, + **kwargs) class Embedding(KerasLayer): @@ -169,23 +204,25 @@ class Embedding(KerasLayer): # Arguments input_dim: Size of the vocabulary. Int > 0. output_dim: Dimension of the dense embedding. Int >= 0. - init: String representations of initialization method for the weights of the layer. + init: String representation of the initialization method for the weights of the layer. Default is 'uniform'. W_regularizer: An instance of [[Regularizer]], (eg. L1 or L2 regularization), applied to the embedding matrix. Default is None. input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. - >>> embedding = Embedding(1000, 32, input_shape=(10, )) + >>> embedding = Embedding(1000, 32, input_shape=(10, ), name="embedding1") creating: createKerasEmbedding """ - def __init__(self, input_dim, output_dim, init="uniform", - W_regularizer=None, input_shape=None, bigdl_type="float"): - super(Embedding, self).__init__(None, bigdl_type, + def __init__(self, input_dim, output_dim, init="uniform", W_regularizer=None, + input_shape=None, **kwargs): + super(Embedding, self).__init__(None, input_dim, output_dim, init, W_regularizer, - list(input_shape) if input_shape else None) + list(input_shape) if input_shape else None, + **kwargs) class BatchNormalization(KerasLayer): @@ -203,47 +240,69 @@ class BatchNormalization(KerasLayer): epsilon: Small float > 0. Fuzz parameter. Default is 0.001. momentum: Float. Momentum in the computation of the exponential average of the mean and standard deviation of the data, for feature-wise normalization. Default is 0.99. - beta_init: Name of initialization function for shift parameter. Default is 'zero'. - gamma_init: Name of initialization function for scale parameter. Default is 'one'. + beta_init: Name of the initialization function for shift parameter. Default is 'zero'. + gamma_init: Name of the initialization function for scale parameter. Default is 'one'. dim_ordering: Format of input data. Either 'th' (Channel First) or 'tf' (Channel Last). Default is 'th'. For 'th', axis along which to normalize is 1. For 'tf', axis is 3. input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. - >>> batchnormalization = BatchNormalization(input_shape=(3, 12, 12)) + >>> batchnormalization = BatchNormalization(input_shape=(3, 12, 12), name="bn1") creating: createKerasBatchNormalization """ - def __init__(self, epsilon=0.001, momentum=0.99, beta_init="zero", gamma_init="one", - dim_ordering="th", input_shape=None, bigdl_type="float"): - super(BatchNormalization, self).__init__(None, bigdl_type, + def __init__(self, epsilon=0.001, mode=0, axis=1, momentum=0.99, beta_init="zero", gamma_init="one", + dim_ordering="th", input_shape=None, **kwargs): + if mode != 0: + raise ValueError("For BatchNormalization, only mode=0 is supported for now") + if dim_ordering == "th" and axis != 1: + raise ValueError("For BatchNormalization with th dim ordering, only axis=1 is supported for now") + if dim_ordering == "tf" and axis != -1 and axis != 3: + raise ValueError("For BatchNormalization with tf dim ordering, only axis=-1 is supported for now") + super(BatchNormalization, self).__init__(None, float(epsilon), float(momentum), beta_init, gamma_init, dim_ordering, - list(input_shape) if input_shape else None) + list(input_shape) if input_shape else None, + **kwargs) def set_running_mean(self, running_mean): - callBigDlFunc(self.bigdl_type, "setKerasRunningMean", + """ + Set the running mean of the BatchNormalization layer. + :param running_mean: a Numpy array. + """ + callBigDlFunc(self.bigdl_type, "setRunningMean", self.value, JTensor.from_ndarray(running_mean)) return self def set_running_std(self, running_std): - callBigDlFunc(self.bigdl_type, "setKerasRunningStd", + """ + Set the running variance of the BatchNormalization layer. + :param running_std: a Numpy array. + """ + callBigDlFunc(self.bigdl_type, "setRunningStd", self.value, JTensor.from_ndarray(running_std)) return self def get_running_mean(self): - return callBigDlFunc(self.bigdl_type, "getKerasRunningMean", + """ + Get the running meaning of the BatchNormalization layer. + """ + return callBigDlFunc(self.bigdl_type, "getRunningMean", self.value).to_ndarray() def get_running_std(self): - return callBigDlFunc(self.bigdl_type, "getKerasRunningStd", + """ + Get the running variance of the BatchNormalization layer. + """ + return callBigDlFunc(self.bigdl_type, "getRunningStd", self.value).to_ndarray() class Merge(KerasLayer): """ - Used to merge a list of tensors into a single tensor, following some merge mode. + Used to merge a list of inputs into a single output, following some merge mode. Merge must have at least two input layers. When using this layer as the first layer in a model, you need to provide the argument @@ -253,24 +312,43 @@ class Merge(KerasLayer): layers: A list of layer instances. Must be more than one layer. mode: Merge mode. String, must be one of: 'sum', 'mul', 'concat', 'ave', 'cos', 'dot', 'max'. Default is 'sum'. - concat_axis: Int, axis to use in mode concat. Only specify this when mode is 'concat'. + concat_axis: Int, axis to use when concatenating layers. Only specify this when merge mode is 'concat'. Default is -1, meaning the last axis of the input. input_shape: A list of shape tuples, each not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. >>> l1 = InputLayer(input_shape=(3, 5)) creating: createKerasInputLayer >>> l2 = InputLayer(input_shape=(3, 5)) creating: createKerasInputLayer - >>> merge = Merge(layers=[l1, l2], mode='sum') + >>> merge = Merge(layers=[l1, l2], mode='sum', name="merge1") creating: createKerasMerge """ def __init__(self, layers=None, mode="sum", concat_axis=-1, - input_shape=None, bigdl_type="float"): - super(Merge, self).__init__(None, bigdl_type, + input_shape=None, **kwargs): + super(Merge, self).__init__(None, list(layers) if layers else None, mode, concat_axis, - input_shape) + input_shape, + **kwargs) + + +def merge(inputs, mode="sum", concat_axis=-1, name=None): + """ + Functional merge. Only use this method if you are defining a graph model. + Used to merge a list of input nodes into a single output node (NOT layers!), + following some merge mode. + + # Arguments + inputs: A list of node instances. Must be more than one node. + mode: Merge mode. String, must be one of: 'sum', 'mul', 'concat', 'ave', 'cos', + 'dot', 'max'. Default is 'sum'. + concat_axis: Int, axis to use when concatenating nodes. Only specify this when merge mode is 'concat'. + Default is -1, meaning the last axis of the input. + name: String to set the name of the merge. If not specified, its name will by default to be a generated string. + """ + return Merge(mode=mode, concat_axis=concat_axis, name=name)(list(inputs)) class Dropout(KerasLayer): @@ -284,14 +362,16 @@ class Dropout(KerasLayer): # Arguments p: Fraction of the input units to drop. Float between 0 and 1. input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. >>> dropout = Dropout(0.25, input_shape=(2, 3)) creating: createKerasDropout """ - def __init__(self, p, input_shape=None, bigdl_type="float"): - super(Dropout, self).__init__(None, bigdl_type, + def __init__(self, p, input_shape=None, **kwargs): + super(Dropout, self).__init__(None, float(p), - list(input_shape) if input_shape else None) + list(input_shape) if input_shape else None, + **kwargs) class Flatten(KerasLayer): @@ -303,13 +383,15 @@ class Flatten(KerasLayer): # Arguments input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. >>> flatten = Flatten(input_shape=(3, 10, 2)) creating: createKerasFlatten """ - def __init__(self, input_shape=None, bigdl_type="float"): - super(Flatten, self).__init__(None, bigdl_type, - list(input_shape) if input_shape else None) + def __init__(self, input_shape=None, **kwargs): + super(Flatten, self).__init__(None, + list(input_shape) if input_shape else None, + **kwargs) class Reshape(KerasLayer): @@ -325,14 +407,16 @@ class Reshape(KerasLayer): # Arguments target_shape: A shape tuple. The target shape that you desire to have. Batch dimension should be excluded. input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. >>> reshape = Reshape((2, 10), input_shape=(5, 4)) creating: createKerasReshape """ - def __init__(self, target_shape, input_shape=None, bigdl_type="float"): - super(Reshape, self).__init__(None, bigdl_type, + def __init__(self, target_shape, input_shape=None, **kwargs): + super(Reshape, self).__init__(None, target_shape, - list(input_shape) if input_shape else None) + list(input_shape) if input_shape else None, + **kwargs) class Activation(KerasLayer): @@ -346,14 +430,16 @@ class Activation(KerasLayer): # Arguments activation: Name of the activation function as string. input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. >>> activation = Activation("relu", input_shape=(3, 4)) creating: createKerasActivation """ - def __init__(self, activation, input_shape=None, bigdl_type="float"): - super(Activation, self).__init__(None, bigdl_type, + def __init__(self, activation, input_shape=None, **kwargs): + super(Activation, self).__init__(None, activation, - list(input_shape) if input_shape else None) + list(input_shape) if input_shape else None, + **kwargs) class RepeatVector(KerasLayer): @@ -366,15 +452,21 @@ class RepeatVector(KerasLayer): # Arguments n: Repetition factor. Int. + input_dim: Dimensionality of the input. Alternatively, you can specify 'input_shape' + when using this layer as the first layer. input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. >>> repeatvector = RepeatVector(5, input_shape=(3, )) creating: createKerasRepeatVector """ - def __init__(self, n, input_shape=None, bigdl_type="float"): - super(RepeatVector, self).__init__(None, bigdl_type, + def __init__(self, n, input_dim=None, input_shape=None, **kwargs): + if input_dim: + input_shape = (input_dim, ) + super(RepeatVector, self).__init__(None, n, - list(input_shape) if input_shape else None) + list(input_shape) if input_shape else None, + **kwargs) class Permute(KerasLayer): @@ -388,14 +480,16 @@ class Permute(KerasLayer): # Arguments dims: Tuple of int. Permutation pattern, does not include the samples dimension. Indexing starts at 1. input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. >>> permute = Permute((2, 1, 3), input_shape=(3, 4, 5)) creating: createKerasPermute """ - def __init__(self, dims, input_shape=None, bigdl_type="float"): - super(Permute, self).__init__(None, bigdl_type, + def __init__(self, dims, input_shape=None, **kwargs): + super(Permute, self).__init__(None, dims, - list(input_shape) if input_shape else None) + list(input_shape) if input_shape else None, + **kwargs) class Highway(KerasLayer): @@ -407,26 +501,32 @@ class Highway(KerasLayer): input_shape (a shape tuple, does not include the batch dimension). # Arguments - activation: String representations of activation function to use (such as 'relu' or 'sigmoid'). + activation: String representation of the activation function to use (such as 'relu' or 'sigmoid'). Default is None. W_regularizer: An instance of [[Regularizer]], (eg. L1 or L2 regularization), applied to the input weights matrices. Default is None. b_regularizer: An instance of [[Regularizer]], applied to the bias. Default is None. bias: Whether to include a bias (i.e. make the layer affine rather than linear). Default is True. + input_dim: Dimensionality of the input. Alternatively, you can specify 'input_shape' + when using this layer as the first layer. input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. >>> highway = Highway(activation='relu', input_shape=(8, )) creating: createKerasHighway """ def __init__(self, activation=None, W_regularizer=None, b_regularizer=None, - bias=True, input_shape=None, bigdl_type="float"): - super(Highway, self).__init__(None, bigdl_type, + bias=True, input_dim=None, input_shape=None, **kwargs): + if input_dim: + input_shape = (input_dim, ) + super(Highway, self).__init__(None, activation, W_regularizer, b_regularizer, bias, - list(input_shape) if input_shape else None) + list(input_shape) if input_shape else None, + **kwargs) class Convolution1D(KerasLayer): @@ -441,9 +541,9 @@ class Convolution1D(KerasLayer): # Arguments nb_filter: Number of convolution filters to use. filter_length: The extension (spatial or temporal) of each filter. - init: String representations of initialization method for the weights of the layer. + init: String representation of the initialization method for the weights of the layer. Default is 'glorot_uniform'. - activation: String representations of activation function to use (such as 'relu' or 'sigmoid'). + activation: String representation of the activation function to use (such as 'relu' or 'sigmoid'). Default is None. border_mode: Either 'valid' or 'same'. Default is 'valid'. subsample_length: Factor by which to subsample output. Int. Default is 1. @@ -453,6 +553,7 @@ class Convolution1D(KerasLayer): bias: Whether to include a bias (i.e. make the layer affine rather than linear). Default is True. input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. >>> conv1d = Convolution1D(12, 4, input_shape=(3, 16)) creating: createKerasConvolution1D @@ -460,8 +561,8 @@ class Convolution1D(KerasLayer): def __init__(self, nb_filter, filter_length, init="glorot_uniform", activation=None, border_mode="valid", subsample_length=1, W_regularizer=None, b_regularizer=None, bias=True, - input_shape=None, bigdl_type="float"): - super(Convolution1D, self).__init__(None, bigdl_type, + input_shape=None, **kwargs): + super(Convolution1D, self).__init__(None, nb_filter, filter_length, init, @@ -471,7 +572,8 @@ def __init__(self, nb_filter, filter_length, init="glorot_uniform", W_regularizer, b_regularizer, bias, - list(input_shape) if input_shape else None) + list(input_shape) if input_shape else None, + **kwargs) class Convolution2D(KerasLayer): @@ -488,9 +590,9 @@ class Convolution2D(KerasLayer): nb_filter: Number of convolution filters to use. nb_row: Number of rows in the convolution kernel. nb_col: Number of cols in the convolution kernel. - init: String representations of initialization method for the weights of the layer. + init: String representation of the initialization method for the weights of the layer. Default is 'glorot_uniform'. - activation: String representations of activation function to use (such as 'relu' or 'sigmoid'). + activation: String representation of the activation function to use (such as 'relu' or 'sigmoid'). Default is None. border_mode: Either 'valid' or 'same'. Default is 'valid'. subsample: Int tuple of length 2 corresponding to the step of the convolution in the @@ -502,16 +604,17 @@ class Convolution2D(KerasLayer): bias: Whether to include a bias (i.e. make the layer affine rather than linear). Default is True. input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. - >>> conv2d = Convolution2D(32, 3, 3, input_shape=(3, 128, 128)) + >>> conv2d = Convolution2D(32, 3, 3, input_shape=(3, 128, 128), name="convolution2d_1") creating: createKerasConvolution2D """ def __init__(self, nb_filter, nb_row, nb_col, init="glorot_uniform", activation=None, border_mode="valid", subsample=(1, 1), dim_ordering="th", W_regularizer=None, b_regularizer=None, bias=True, - input_shape=None, bigdl_type="float"): - super(Convolution2D, self).__init__(None, bigdl_type, + input_shape=None, **kwargs): + super(Convolution2D, self).__init__(None, nb_filter, nb_row, nb_col, @@ -523,7 +626,8 @@ def __init__(self, nb_filter, nb_row, nb_col, W_regularizer, b_regularizer, bias, - list(input_shape) if input_shape else None) + list(input_shape) if input_shape else None, + **kwargs) class Convolution3D(KerasLayer): @@ -541,9 +645,9 @@ class Convolution3D(KerasLayer): kernel_dim1: Length of the first dimension in the convolution kernel. kernel_dim2: Length of the second dimension in the convolution kernel. kernel_dim3: Length of the third dimension in the convolution kernel. - init: String representations of initialization method for the weights of the layer. + init: String representation of the initialization method for the weights of the layer. Default is 'glorot_uniform'. - activation: String representations of activation function to use (such as 'relu' or 'sigmoid'). + activation: String representation of the activation function to use (such as 'relu' or 'sigmoid'). Default is None. border_mode: Either 'valid' or 'same'. Default is 'valid'. subsample: Int tuple of length 3. Factor by which to subsample output. @@ -555,6 +659,7 @@ class Convolution3D(KerasLayer): bias: Whether to include a bias (i.e. make the layer affine rather than linear). Default is True. input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. >>> conv3d = Convolution3D(32, 3, 4, 5, input_shape=(3, 64, 64, 64)) creating: createKerasConvolution3D @@ -562,8 +667,8 @@ class Convolution3D(KerasLayer): def __init__(self, nb_filter, kernel_dim1, kernel_dim2, kernel_dim3, init="glorot_uniform", activation=None, border_mode="valid", subsample=(1, 1, 1), dim_ordering="th", W_regularizer=None, - b_regularizer=None, bias=True, input_shape=None, bigdl_type="float"): - super(Convolution3D, self).__init__(None, bigdl_type, + b_regularizer=None, bias=True, input_shape=None, **kwargs): + super(Convolution3D, self).__init__(None, nb_filter, kernel_dim1, kernel_dim2, @@ -576,7 +681,8 @@ def __init__(self, nb_filter, kernel_dim1, kernel_dim2, kernel_dim3, W_regularizer, b_regularizer, bias, - list(input_shape) if input_shape else None) + list(input_shape) if input_shape else None, + **kwargs) class AtrousConvolution1D(KerasLayer): @@ -594,9 +700,9 @@ class AtrousConvolution1D(KerasLayer): # Arguments nb_filter: Number of convolution filters to use. filter_length: The extension (spatial or temporal) of each filter. - init: String representations of initialization method for the weights of the layer. + init: String representation of the initialization method for the weights of the layer. Default is 'glorot_uniform'. - activation: String representations of activation function to use (such as 'relu' or 'sigmoid'). + activation: String representation of the activation function to use (such as 'relu' or 'sigmoid'). Default is None. border_mode: Only 'valid' is supported for now. subsample_length: Factor by which to subsample output. Int. Default is 1. @@ -606,18 +712,19 @@ class AtrousConvolution1D(KerasLayer): b_regularizer: An instance of [[Regularizer]], applied to the bias. Default is None. bias: Only 'True' is supported for now. input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. >>> atrousconv1d = AtrousConvolution1D(8, 3, input_shape=(3, 12)) creating: createKerasAtrousConvolution1D """ - def __init__(self, nb_filter, filter_length, init="glorot_uniform", - activation=None, border_mode='valid', subsample_length=1, atrous_rate=1, - W_regularizer=None, b_regularizer=None, bias=True, input_shape=None, bigdl_type="float"): + def __init__(self, nb_filter, filter_length, init="glorot_uniform", activation=None, + border_mode='valid', subsample_length=1, atrous_rate=1, W_regularizer=None, + b_regularizer=None, bias=True, input_shape=None, **kwargs): if border_mode != "valid": raise ValueError("For AtrousConvolution1D, only border_mode='valid' is supported for now") if not bias: raise ValueError("For AtrousConvolution1D, only bias=True is supported for now") - super(AtrousConvolution1D, self).__init__(None, bigdl_type, + super(AtrousConvolution1D, self).__init__(None, nb_filter, filter_length, init, @@ -626,7 +733,8 @@ def __init__(self, nb_filter, filter_length, init="glorot_uniform", atrous_rate, W_regularizer, b_regularizer, - list(input_shape) if input_shape else None) + list(input_shape) if input_shape else None, + **kwargs) class AtrousConvolution2D(KerasLayer): @@ -647,9 +755,9 @@ class AtrousConvolution2D(KerasLayer): nb_filter: Number of convolution filters to use. nb_row: Number of rows in the convolution kernel. nb_col: Number of cols in the convolution kernel. - init: String representations of initialization method for the weights of the layer. + init: String representation of the initialization method for the weights of the layer. Default is 'glorot_uniform'. - activation: String representations of activation function to use (such as 'relu' or 'sigmoid'). + activation: String representation of the activation function to use (such as 'relu' or 'sigmoid'). Default is None. border_mode: Only 'valid' is supported for now. subsample: Int tuple of length 2 corresponding to the step of the convolution in the @@ -662,6 +770,7 @@ class AtrousConvolution2D(KerasLayer): b_regularizer: An instance of [[Regularizer]], applied to the bias. Default is None. bias: Only 'True' is supported for now. input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. >>> atrousconv2d = AtrousConvolution2D(12, 4, 3, input_shape=(3, 64, 64)) creating: createKerasAtrousConvolution2D @@ -669,12 +778,12 @@ class AtrousConvolution2D(KerasLayer): def __init__(self, nb_filter, nb_row, nb_col, init="glorot_uniform", activation=None, border_mode="valid", subsample=(1, 1), atrous_rate=(1, 1), dim_ordering="th", W_regularizer=None, - b_regularizer=None, bias=True, input_shape=None, bigdl_type="float"): + b_regularizer=None, bias=True, input_shape=None, **kwargs): if border_mode != "valid": raise ValueError("For AtrousConvolution2D, only border_mode='valid' is supported for now") if not bias: raise ValueError("For AtrousConvolution2D, only bias=True is supported for now") - super(AtrousConvolution2D, self).__init__(None, bigdl_type, + super(AtrousConvolution2D, self).__init__(None, nb_filter, nb_row, nb_col, @@ -685,7 +794,8 @@ def __init__(self, nb_filter, nb_row, nb_col, init="glorot_uniform", dim_ordering, W_regularizer, b_regularizer, - list(input_shape) if input_shape else None) + list(input_shape) if input_shape else None, + **kwargs) class Deconvolution2D(KerasLayer): @@ -709,9 +819,9 @@ class Deconvolution2D(KerasLayer): nb_row: Number of rows in the convolution kernel. nb_col: Number of cols in the convolution kernel. output_shape: Output shape of the transposed convolution operation. Tuple of int. - init: String representations of initialization method for the weights of the layer. + init: String representation of the initialization method for the weights of the layer. Default is 'glorot_uniform'. - activation: String representations of activation function to use (such as 'relu' or 'sigmoid'). + activation: String representation of the activation function to use (such as 'relu' or 'sigmoid'). Default is None. border_mode: Only 'valid' is supported for now. subsample: Int tuple of length 2 corresponding to the step of the convolution in the @@ -723,16 +833,17 @@ class Deconvolution2D(KerasLayer): bias: Whether to include a bias (i.e. make the layer affine rather than linear). Default is True. input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. >>> deconv2d = Deconvolution2D(3, 3, 3, output_shape=(None, 3, 14, 14), input_shape=(3, 12, 12)) creating: createKerasDeconvolution2D """ def __init__(self, nb_filter, nb_row, nb_col, output_shape, init="glorot_uniform", activation=None, border_mode="valid", subsample=(1, 1), dim_ordering="th", - W_regularizer=None, b_regularizer=None, bias=True, input_shape=None, bigdl_type="float"): + W_regularizer=None, b_regularizer=None, bias=True, input_shape=None, **kwargs): if border_mode != "valid": raise ValueError("For Deconvolution2D, only border_mode='valid' is supported for now") - super(Deconvolution2D, self).__init__(None, bigdl_type, + super(Deconvolution2D, self).__init__(None, nb_filter, nb_row, nb_col, @@ -743,7 +854,8 @@ def __init__(self, nb_filter, nb_row, nb_col, output_shape, init="glorot_uniform W_regularizer, b_regularizer, bias, - list(input_shape) if input_shape else None) + list(input_shape) if input_shape else None, + **kwargs) class SeparableConvolution2D(KerasLayer): @@ -764,9 +876,9 @@ class SeparableConvolution2D(KerasLayer): nb_filter: Number of convolution filters to use. nb_row: Number of rows in the convolution kernel. nb_col: Number of cols in the convolution kernel. - init: String representations of initialization method for the weights of the layer. + init: String representation of the initialization method for the weights of the layer. Default is 'glorot_uniform'. - activation: String representations of activation function to use (such as 'relu' or 'sigmoid'). + activation: String representation of the activation function to use (such as 'relu' or 'sigmoid'). Default is None. border_mode: Either 'valid' or 'same'. Default is 'valid'. subsample: Int tuple of length 2 corresponding to the step of the convolution in the @@ -782,6 +894,7 @@ class SeparableConvolution2D(KerasLayer): bias: Whether to include a bias (i.e. make the layer affine rather than linear). Default is True. input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. >>> separableconv2d = SeparableConvolution2D(12, 3, 4, input_shape=(3, 32, 32)) creating: createKerasSeparableConvolution2D @@ -789,8 +902,8 @@ class SeparableConvolution2D(KerasLayer): def __init__(self, nb_filter, nb_row, nb_col, init="glorot_uniform", activation=None, border_mode="valid", subsample=(1, 1), depth_multiplier=1, dim_ordering="th", depthwise_regularizer=None, pointwise_regularizer=None, - b_regularizer=None, bias=True, input_shape=None, bigdl_type="float"): - super(SeparableConvolution2D, self).__init__(None, bigdl_type, + b_regularizer=None, bias=True, input_shape=None, **kwargs): + super(SeparableConvolution2D, self).__init__(None, nb_filter, nb_row, nb_col, @@ -804,7 +917,8 @@ def __init__(self, nb_filter, nb_row, nb_col, init="glorot_uniform", pointwise_regularizer, b_regularizer, bias, - list(input_shape) if input_shape else None) + list(input_shape) if input_shape else None, + **kwargs) Conv1D = Convolution1D @@ -828,14 +942,16 @@ class Cropping1D(KerasLayer): cropping: Int tuple of length 2. How many units should be trimmed off at the beginning and end of the cropping dimension. Default is (1, 1). input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. >>> cropping1d = Cropping1D(cropping=(1, 2), input_shape=(8, 8)) creating: createKerasCropping1D """ - def __init__(self, cropping=(1, 1), input_shape=None, bigdl_type="float"): - super(Cropping1D, self).__init__(None, bigdl_type, + def __init__(self, cropping=(1, 1), input_shape=None, **kwargs): + super(Cropping1D, self).__init__(None, cropping, - list(input_shape) if input_shape else None) + list(input_shape) if input_shape else None, + **kwargs) class Cropping2D(KerasLayer): @@ -850,17 +966,19 @@ class Cropping2D(KerasLayer): cropping: Int tuple of tuple of length 2. How many units should be trimmed off at the beginning and end of the 2 cropping dimensions (i.e. height and width). Default is ((0, 0), (0, 0)). input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. >>> cropping2d = Cropping2D(cropping=((1, 2), (0, 1)), input_shape=(12, 12, 12)) creating: createKerasCropping2D """ def __init__(self, cropping=((0, 0), (0, 0)), dim_ordering="th", - input_shape=None, bigdl_type="float"): - super(Cropping2D, self).__init__(None, bigdl_type, + input_shape=None, **kwargs): + super(Cropping2D, self).__init__(None, cropping[0], cropping[1], dim_ordering, - list(input_shape) if input_shape else None) + list(input_shape) if input_shape else None, + **kwargs) class Cropping3D(KerasLayer): @@ -876,18 +994,20 @@ class Cropping3D(KerasLayer): end of the 3 cropping dimensions (i.e. kernel_dim1, kernel_dim2 and kernel_dim3). Default is ((1, 1), (1, 1), (1, 1)). input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. >>> cropping3d = Cropping3D(cropping=((0, 2), (1, 1), (3, 1)), input_shape=(4, 12, 12, 16)) creating: createKerasCropping3D """ def __init__(self, cropping=((1, 1), (1, 1), (1, 1)), dim_ordering="th", - input_shape=None, bigdl_type="float"): - super(Cropping3D, self).__init__(None, bigdl_type, + input_shape=None, **kwargs): + super(Cropping3D, self).__init__(None, cropping[0], cropping[1], cropping[2], dim_ordering, - list(input_shape) if input_shape else None) + list(input_shape) if input_shape else None, + **kwargs) class UpSampling1D(KerasLayer): @@ -902,14 +1022,16 @@ class UpSampling1D(KerasLayer): # Arguments length: Int. UpSampling factor. Default is 2. input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. >>> upsampling1d = UpSampling1D(length=3, input_shape=(3, 12)) creating: createKerasUpSampling1D """ - def __init__(self, length=2, input_shape=None, bigdl_type="float"): - super(UpSampling1D, self).__init__(None, bigdl_type, + def __init__(self, length=2, input_shape=None, **kwargs): + super(UpSampling1D, self).__init__(None, length, - list(input_shape) if input_shape else None) + list(input_shape) if input_shape else None, + **kwargs) class UpSampling2D(KerasLayer): @@ -925,15 +1047,17 @@ class UpSampling2D(KerasLayer): size: Int tuple of length 2. UpSampling factors for rows and columns. Default is (2, 2). dim_ordering: Format of input data. Either 'th' (Channel First) or 'tf' (Channel Last). Default is 'th'. input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. >>> upsampling2d = UpSampling2D(size=(1, 3), input_shape=(3, 16, 16)) creating: createKerasUpSampling2D """ - def __init__(self, size=(2, 2), dim_ordering="th", input_shape=None, bigdl_type="float"): - super(UpSampling2D, self).__init__(None, bigdl_type, + def __init__(self, size=(2, 2), dim_ordering="th", input_shape=None, **kwargs): + super(UpSampling2D, self).__init__(None, size, dim_ordering, - list(input_shape) if input_shape else None) + list(input_shape) if input_shape else None, + **kwargs) class UpSampling3D(KerasLayer): @@ -950,15 +1074,17 @@ class UpSampling3D(KerasLayer): size: Int tuple of length 3. UpSampling factors for dim1, dim2 and dim3. Default is (2, 2, 2). dim_ordering: Format of input data. Only 'th' (Channel First) is supported for now. input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. >>> upsampling3d = UpSampling3D(size=(1, 2, 3), input_shape=(3, 16, 16, 16)) creating: createKerasUpSampling3D """ - def __init__(self, size=(2, 2, 2), dim_ordering="th", input_shape=None, bigdl_type="float"): - super(UpSampling3D, self).__init__(None, bigdl_type, + def __init__(self, size=(2, 2, 2), dim_ordering="th", input_shape=None, **kwargs): + super(UpSampling3D, self).__init__(None, size, dim_ordering, - list(input_shape) if input_shape else None) + list(input_shape) if input_shape else None, + **kwargs) class ZeroPadding1D(KerasLayer): @@ -975,16 +1101,18 @@ class ZeroPadding1D(KerasLayer): If tuple of length 2, how many zeros to add in the order '(left_pad, right_pad)'. Default is 1. input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. >>> zeropadding1d = ZeroPadding1D(padding=2, input_shape=(3, 6)) creating: createKerasZeroPadding1D """ - def __init__(self, padding=1, input_shape=None, bigdl_type="float"): + def __init__(self, padding=1, input_shape=None, **kwargs): if isinstance(padding, int): padding = (padding, padding) - super(ZeroPadding1D, self).__init__(None, bigdl_type, + super(ZeroPadding1D, self).__init__(None, padding, - list(input_shape) if input_shape else None) + list(input_shape) if input_shape else None, + **kwargs) class ZeroPadding2D(KerasLayer): @@ -1002,17 +1130,19 @@ class ZeroPadding2D(KerasLayer): Default is (1, 1). dim_ordering: Format of input data. Either 'th' (Channel First) or 'tf' (Channel Last). Default is 'th'. input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. >>> zeropadding2d = ZeroPadding2D(padding=(2, 1), input_shape=(2, 8, 8)) creating: createKerasZeroPadding2D """ - def __init__(self, padding=(1, 1), dim_ordering="th", input_shape=None, bigdl_type="float"): + def __init__(self, padding=(1, 1), dim_ordering="th", input_shape=None, **kwargs): if len(padding) == 2: padding = (padding[0], padding[0], padding[1], padding[1]) - super(ZeroPadding2D, self).__init__(None, bigdl_type, + super(ZeroPadding2D, self).__init__(None, padding, dim_ordering, - list(input_shape) if input_shape else None) + list(input_shape) if input_shape else None, + **kwargs) class ZeroPadding3D(KerasLayer): @@ -1028,15 +1158,17 @@ class ZeroPadding3D(KerasLayer): Symmetric padding will be applied to each dimension. Default is (1, 1, 1). dim_ordering: Format of input data. Either 'th' (Channel First) or 'tf' (Channel Last). Default is 'th'. input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. >>> zeropadding3d = ZeroPadding3D(padding=(2, 1, 2), input_shape=(2, 8, 8, 10)) creating: createKerasZeroPadding3D """ - def __init__(self, padding=(1, 1, 1), dim_ordering="th", input_shape=None, bigdl_type="float"): - super(ZeroPadding3D, self).__init__(None, bigdl_type, + def __init__(self, padding=(1, 1, 1), dim_ordering="th", input_shape=None, **kwargs): + super(ZeroPadding3D, self).__init__(None, padding, dim_ordering, - list(input_shape) if input_shape else None) + list(input_shape) if input_shape else None, + **kwargs) class MaxPooling1D(KerasLayer): @@ -1053,19 +1185,21 @@ class MaxPooling1D(KerasLayer): Default is None, and in this case it will be equal to pool_length.. border_mode: Either 'valid' or 'same'. Default is 'valid'. input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. >>> maxpooling1d = MaxPooling1D(3, input_shape=(3, 24)) creating: createKerasMaxPooling1D """ def __init__(self, pool_length=2, stride=None, border_mode="valid", - input_shape=None, bigdl_type="float"): + input_shape=None, **kwargs): if not stride: stride = -1 - super(MaxPooling1D, self).__init__(None, bigdl_type, + super(MaxPooling1D, self).__init__(None, pool_length, stride, border_mode, - list(input_shape) if input_shape else None) + list(input_shape) if input_shape else None, + **kwargs) class MaxPooling2D(KerasLayer): @@ -1083,19 +1217,21 @@ class MaxPooling2D(KerasLayer): border_mode: Either 'valid' or 'same'. Default is 'valid'. dim_ordering: Format of input data. Either 'th' (Channel First) or 'tf' (Channel Last). Default is 'th'. input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. - >>> maxpooling2d = MaxPooling2D((2, 2), input_shape=(3, 32, 32)) + >>> maxpooling2d = MaxPooling2D((2, 2), input_shape=(3, 32, 32), name="maxpooling2d_1") creating: createKerasMaxPooling2D """ def __init__(self, pool_size=(2, 2), strides=None, border_mode='valid', dim_ordering='th', - input_shape=None, bigdl_type="float"): - super(MaxPooling2D, self).__init__(None, bigdl_type, + input_shape=None, **kwargs): + super(MaxPooling2D, self).__init__(None, pool_size, strides, border_mode, dim_ordering, - list(input_shape) if input_shape else None) + list(input_shape) if input_shape else None, + **kwargs) class MaxPooling3D(KerasLayer): @@ -1115,19 +1251,21 @@ class MaxPooling3D(KerasLayer): border_mode: Only 'valid' is supported for now. dim_ordering: Format of input data. Only 'th' (Channel First) is supported for now. input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. >>> maxpooling3d = MaxPooling3D((2, 1, 3), input_shape=(3, 32, 32, 32)) creating: createKerasMaxPooling3D """ def __init__(self, pool_size=(2, 2, 2), strides=None, border_mode="valid", - dim_ordering="th", input_shape=None, bigdl_type="float"): + dim_ordering="th", input_shape=None, **kwargs): if border_mode != "valid": raise ValueError("For MaxPooling3D, only border_mode='valid' is supported for now") - super(MaxPooling3D, self).__init__(None, bigdl_type, + super(MaxPooling3D, self).__init__(None, pool_size, strides, dim_ordering, - list(input_shape) if input_shape else None) + list(input_shape) if input_shape else None, + **kwargs) class AveragePooling1D(KerasLayer): @@ -1144,19 +1282,21 @@ class AveragePooling1D(KerasLayer): Default is None, and in this case it will be equal to pool_length.. border_mode: Either 'valid' or 'same'. Default is 'valid'. input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. >>> averagepooling1d = AveragePooling1D(input_shape=(3, 24)) creating: createKerasAveragePooling1D """ def __init__(self, pool_length=2, stride=None, border_mode="valid", - input_shape=None, bigdl_type="float"): + input_shape=None, **kwargs): if not stride: stride = -1 - super(AveragePooling1D, self).__init__(None, bigdl_type, + super(AveragePooling1D, self).__init__(None, pool_length, stride, border_mode, - list(input_shape) if input_shape else None) + list(input_shape) if input_shape else None, + **kwargs) class AveragePooling2D(KerasLayer): @@ -1174,18 +1314,20 @@ class AveragePooling2D(KerasLayer): border_mode: Either 'valid' or 'same'. Default is 'valid'. dim_ordering: Format of input data. Either 'th' (Channel First) or 'tf' (Channel Last). Default is 'th'. input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. >>> averagepooling2d = AveragePooling2D((1, 2), input_shape=(2, 28, 32)) creating: createKerasAveragePooling2D """ def __init__(self, pool_size=(2, 2), strides=None, border_mode="valid", - dim_ordering="th", input_shape=None, bigdl_type="float"): - super(AveragePooling2D, self).__init__(None, bigdl_type, + dim_ordering="th", input_shape=None, **kwargs): + super(AveragePooling2D, self).__init__(None, pool_size, strides, border_mode, dim_ordering, - list(input_shape) if input_shape else None) + list(input_shape) if input_shape else None, + **kwargs) class AveragePooling3D(KerasLayer): @@ -1205,19 +1347,21 @@ class AveragePooling3D(KerasLayer): border_mode: Only 'valid' is supported for now. dim_ordering: Format of input data. Only 'th' (Channel First) is supported for now. input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. >>> averagepooling3d = AveragePooling3D((1, 1, 2), input_shape=(3, 28, 32, 36)) creating: createKerasAveragePooling3D """ def __init__(self, pool_size=(2, 2, 2), strides=None, border_mode="valid", - dim_ordering="th", input_shape=None, bigdl_type="float"): + dim_ordering="th", input_shape=None, **kwargs): if border_mode != "valid": raise ValueError("For AveragePooling3D, only border_mode='valid' is supported for now") - super(AveragePooling3D, self).__init__(None, bigdl_type, + super(AveragePooling3D, self).__init__(None, pool_size, strides, dim_ordering, - list(input_shape) if input_shape else None) + list(input_shape) if input_shape else None, + **kwargs) class GlobalMaxPooling1D(KerasLayer): @@ -1230,13 +1374,15 @@ class GlobalMaxPooling1D(KerasLayer): # Arguments input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. >>> globalmaxpooling1d = GlobalMaxPooling1D(input_shape=(4, 8)) creating: createKerasGlobalMaxPooling1D """ - def __init__(self, input_shape=None, bigdl_type="float"): - super(GlobalMaxPooling1D, self).__init__(None, bigdl_type, - list(input_shape) if input_shape else None) + def __init__(self, input_shape=None, **kwargs): + super(GlobalMaxPooling1D, self).__init__(None, + list(input_shape) if input_shape else None, + **kwargs) class GlobalAveragePooling1D(KerasLayer): @@ -1249,13 +1395,15 @@ class GlobalAveragePooling1D(KerasLayer): # Arguments input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. >>> globalaveragepooling1d = GlobalAveragePooling1D(input_shape=(12, 12)) creating: createKerasGlobalAveragePooling1D """ - def __init__(self, input_shape=None, bigdl_type="float"): - super(GlobalAveragePooling1D, self).__init__(None, bigdl_type, - list(input_shape) if input_shape else None) + def __init__(self, input_shape=None, **kwargs): + super(GlobalAveragePooling1D, self).__init__(None, + list(input_shape) if input_shape else None, + **kwargs) class GlobalMaxPooling2D(KerasLayer): @@ -1269,14 +1417,16 @@ class GlobalMaxPooling2D(KerasLayer): # Arguments dim_ordering: Format of input data. Either 'th' (Channel First) or 'tf' (Channel Last). Default is 'th'. input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. >>> globalmaxpooling2d = GlobalMaxPooling2D(input_shape=(4, 32, 32)) creating: createKerasGlobalMaxPooling2D """ - def __init__(self, dim_ordering="th", input_shape=None, bigdl_type="float"): - super(GlobalMaxPooling2D, self).__init__(None, bigdl_type, + def __init__(self, dim_ordering="th", input_shape=None, **kwargs): + super(GlobalMaxPooling2D, self).__init__(None, dim_ordering, - list(input_shape) if input_shape else None) + list(input_shape) if input_shape else None, + **kwargs) class GlobalAveragePooling2D(KerasLayer): @@ -1290,14 +1440,16 @@ class GlobalAveragePooling2D(KerasLayer): # Arguments dim_ordering: Format of input data. Either 'th' (Channel First) or 'tf' (Channel Last). Default is 'th'. input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. >>> globalaveragepooling2d = GlobalAveragePooling2D(input_shape=(4, 32, 32)) creating: createKerasGlobalAveragePooling2D """ - def __init__(self, dim_ordering="th", input_shape=None, bigdl_type="float"): - super(GlobalAveragePooling2D, self).__init__(None, bigdl_type, + def __init__(self, dim_ordering="th", input_shape=None, **kwargs): + super(GlobalAveragePooling2D, self).__init__(None, dim_ordering, - list(input_shape) if input_shape else None) + list(input_shape) if input_shape else None, + **kwargs) class GlobalMaxPooling3D(KerasLayer): @@ -1313,14 +1465,16 @@ class GlobalMaxPooling3D(KerasLayer): # Arguments dim_ordering: Format of input data. Only 'th' (Channel First) is supported for now. input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. >>> globalmaxpooling3d = GlobalMaxPooling3D(input_shape=(4, 32, 32, 32)) creating: createKerasGlobalMaxPooling3D """ - def __init__(self, dim_ordering="th", input_shape=None, bigdl_type="float"): - super(GlobalMaxPooling3D, self).__init__(None, bigdl_type, + def __init__(self, dim_ordering="th", input_shape=None, **kwargs): + super(GlobalMaxPooling3D, self).__init__(None, dim_ordering, - list(input_shape) if input_shape else None) + list(input_shape) if input_shape else None, + **kwargs) class GlobalAveragePooling3D(KerasLayer): @@ -1336,14 +1490,16 @@ class GlobalAveragePooling3D(KerasLayer): # Arguments dim_ordering: Format of input data. Only 'th' (Channel First) is supported for now. input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. >>> globalaveragepooling3d = GlobalAveragePooling3D(input_shape=(4, 16, 16, 20)) creating: createKerasGlobalAveragePooling3D """ - def __init__(self, dim_ordering="th", input_shape=None, bigdl_type="float"): - super(GlobalAveragePooling3D, self).__init__(None, bigdl_type, + def __init__(self, dim_ordering="th", input_shape=None, **kwargs): + super(GlobalAveragePooling3D, self).__init__(None, dim_ordering, - list(input_shape) if input_shape else None) + list(input_shape) if input_shape else None, + **kwargs) class SimpleRNN(KerasLayer): @@ -1356,7 +1512,7 @@ class SimpleRNN(KerasLayer): # Arguments output_dim: Hidden unit size. Dimension of internal projections and final output. - activation: String representations of activation function to use (such as 'relu' or 'sigmoid'). + activation: String representation of the activation function to use (such as 'relu' or 'sigmoid'). Default is 'tanh'. return_sequences: Whether to return the full sequence or only return the last output in the output sequence. Default is False. @@ -1366,14 +1522,15 @@ class SimpleRNN(KerasLayer): U_regularizer: An instance of [[Regularizer]], applied the recurrent weights matrices. Default is None. b_regularizer: An instance of [[Regularizer]], applied to the bias. Default is None. input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. >>> simplernn = SimpleRNN(16, input_shape=(3, 32)) creating: createKerasSimpleRNN """ def __init__(self, output_dim, activation="tanh", return_sequences=False, go_backwards=False, W_regularizer=None, U_regularizer=None, - b_regularizer=None, input_shape=None, bigdl_type="float"): - super(SimpleRNN, self).__init__(None, bigdl_type, + b_regularizer=None, input_shape=None, **kwargs): + super(SimpleRNN, self).__init__(None, output_dim, activation, return_sequences, @@ -1381,7 +1538,8 @@ def __init__(self, output_dim, activation="tanh", return_sequences=False, W_regularizer, U_regularizer, b_regularizer, - list(input_shape) if input_shape else None) + list(input_shape) if input_shape else None, + **kwargs) class LSTM(KerasLayer): @@ -1394,9 +1552,9 @@ class LSTM(KerasLayer): # Arguments output_dim: Hidden unit size. Dimension of internal projections and final output. - activation: String representations of activation function to use (such as 'relu' or 'sigmoid'). + activation: String representation of the activation function to use (such as 'relu' or 'sigmoid'). Default is 'tanh'. - inner_activation: String representations of activation function for inner cells. Default is 'hard_sigmoid'. + inner_activation: String representation of the activation function for inner cells. Default is 'hard_sigmoid'. return_sequences: Whether to return the full sequence or only return the last output in the output sequence. Default is False. go_backwards: Whether the input sequence will be processed backwards. Default is False. @@ -1405,14 +1563,15 @@ class LSTM(KerasLayer): U_regularizer: An instance of [[Regularizer]], applied the recurrent weights matrices. Default is None. b_regularizer: An instance of [[Regularizer]], applied to the bias. Default is None. input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. - >>> lstm = LSTM(32, input_shape=(8, 16)) + >>> lstm = LSTM(32, input_shape=(8, 16), name="lstm1") creating: createKerasLSTM """ def __init__(self, output_dim, activation="tanh", inner_activation="hard_sigmoid", return_sequences=False, go_backwards=False, W_regularizer=None, - U_regularizer=None, b_regularizer=None, input_shape=None, bigdl_type="float"): - super(LSTM, self).__init__(None, bigdl_type, + U_regularizer=None, b_regularizer=None, input_shape=None, **kwargs): + super(LSTM, self).__init__(None, output_dim, activation, inner_activation, @@ -1421,7 +1580,8 @@ def __init__(self, output_dim, activation="tanh", inner_activation="hard_sigmoid W_regularizer, U_regularizer, b_regularizer, - list(input_shape) if input_shape else None) + list(input_shape) if input_shape else None, + **kwargs) class GRU(KerasLayer): @@ -1434,9 +1594,9 @@ class GRU(KerasLayer): # Arguments output_dim: Hidden unit size. Dimension of internal projections and final output. - activation: String representations of activation function to use (such as 'relu' or 'sigmoid'). + activation: String representation of the activation function to use (such as 'relu' or 'sigmoid'). Default is 'tanh'. - inner_activation: String representations of activation function for inner cells. Default is 'hard_sigmoid'. + inner_activation: String representation of the activation function for inner cells. Default is 'hard_sigmoid'. return_sequences: Whether to return the full sequence or only return the last output in the output sequence. Default is False. go_backwards: Whether the input sequence will be processed backwards. Default is False. @@ -1445,14 +1605,15 @@ class GRU(KerasLayer): U_regularizer: An instance of [[Regularizer]], applied the recurrent weights matrices. Default is None. b_regularizer: An instance of [[Regularizer]], applied to the bias. Default is None. input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. >>> gru = GRU(24, input_shape=(32, 32)) creating: createKerasGRU """ def __init__(self, output_dim, activation="tanh", inner_activation="hard_sigmoid", return_sequences=False, go_backwards=False, W_regularizer=None, - U_regularizer=None, b_regularizer=None, input_shape=None, bigdl_type="float"): - super(GRU, self).__init__(None, bigdl_type, + U_regularizer=None, b_regularizer=None, input_shape=None, **kwargs): + super(GRU, self).__init__(None, output_dim, activation, inner_activation, @@ -1461,7 +1622,8 @@ def __init__(self, output_dim, activation="tanh", inner_activation="hard_sigmoid W_regularizer, U_regularizer, b_regularizer, - list(input_shape) if input_shape else None) + list(input_shape) if input_shape else None, + **kwargs) class ConvLSTM2D(KerasLayer): @@ -1479,9 +1641,9 @@ class ConvLSTM2D(KerasLayer): nb_filter: Number of convolution filters to use. nb_row: Number of rows in the convolution kernel. nb_col: Number of cols in the convolution kernel. Should be equal to nb_row as for a square kernel. - activation: String representations of activation function to use (such as 'relu' or 'sigmoid'). + activation: String representation of the activation function to use (such as 'relu' or 'sigmoid'). Default is 'tanh'. - inner_activation: String representations of activation function for inner cells. Default is 'hard_sigmoid'. + inner_activation: String representation of the activation function for inner cells. Default is 'hard_sigmoid'. dim_ordering: Format of input data. Only 'th' (Channel First) is supported for now. border_mode: Only 'same' is supported for now. subsample: Tuple of length 2. Factor by which to subsample output. Also called strides elsewhere. @@ -1494,6 +1656,7 @@ class ConvLSTM2D(KerasLayer): Default is False. go_backwards: Whether the input sequence will be processed backwards. Default is False. input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. >>> convlstm2d = ConvLSTM2D(24, 3, 3, input_shape=(4, 32, 32, 32)) creating: createKerasConvLSTM2D @@ -1501,14 +1664,14 @@ class ConvLSTM2D(KerasLayer): def __init__(self, nb_filter, nb_row, nb_col, activation="tanh", inner_activation="hard_sigmoid", dim_ordering="th", border_mode="same", subsample=(1, 1), W_regularizer=None, U_regularizer=None, b_regularizer=None, - return_sequences=False, go_backwards=False, input_shape=None, bigdl_type="float"): + return_sequences=False, go_backwards=False, input_shape=None, **kwargs): if nb_row != nb_col: raise ValueError("For ConvLSTM2D, only square kernel is supported for now") if border_mode != "same": raise ValueError("For ConvLSTM2D, only border_mode='same' is supported for now") if subsample[0] != subsample[1]: raise ValueError("For ConvLSTM2D, only equal strides is supported for now") - super(ConvLSTM2D, self).__init__(None, bigdl_type, + super(ConvLSTM2D, self).__init__(None, nb_filter, nb_row, activation, @@ -1520,7 +1683,8 @@ def __init__(self, nb_filter, nb_row, nb_col, activation="tanh", b_regularizer, return_sequences, go_backwards, - list(input_shape) if input_shape else None) + list(input_shape) if input_shape else None, + **kwargs) class LocallyConnected1D(KerasLayer): @@ -1536,7 +1700,7 @@ class LocallyConnected1D(KerasLayer): # Arguments nb_filter: Dimensionality of the output. filter_length: The extension (spatial or temporal) of each filter. - activation: String representations of activation function to use (such as 'relu' or 'sigmoid'). + activation: String representation of the activation function to use (such as 'relu' or 'sigmoid'). Default is None. border_mode: Only 'valid' is supported for now. subsample_length: Factor by which to subsample output. Int. Default is 1. @@ -1546,16 +1710,17 @@ class LocallyConnected1D(KerasLayer): bias: Whether to include a bias (i.e. make the layer affine rather than linear). Default is True. input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. >>> locallyconnected1d = LocallyConnected1D(6, 3, input_shape=(8, 12)) creating: createKerasLocallyConnected1D """ def __init__(self, nb_filter, filter_length, activation=None, border_mode="valid", subsample_length=1, W_regularizer=None, b_regularizer=None, - bias=True, input_shape=None, bigdl_type="float"): + bias=True, input_shape=None, **kwargs): if border_mode != "valid": raise ValueError("For LocallyConnected1D, only border_mode='valid' is supported for now") - super(LocallyConnected1D, self).__init__(None, bigdl_type, + super(LocallyConnected1D, self).__init__(None, nb_filter, filter_length, activation, @@ -1563,7 +1728,8 @@ def __init__(self, nb_filter, filter_length, activation=None, border_mode="valid W_regularizer, b_regularizer, bias, - list(input_shape) if input_shape else None) + list(input_shape) if input_shape else None, + **kwargs) class LocallyConnected2D(KerasLayer): @@ -1579,7 +1745,7 @@ class LocallyConnected2D(KerasLayer): nb_filter: Number of convolution filters to use. nb_row: Number of rows in the convolution kernel. nb_col: Number of cols in the convolution kernel. - activation: String representations of activation function to use (such as 'relu' or 'sigmoid'). + activation: String representation of the activation function to use (such as 'relu' or 'sigmoid'). Default is None. border_mode: Either 'valid' or 'same'. Default is 'valid'. subsample: Int tuple of length 2 corresponding to the step of the convolution in the @@ -1591,6 +1757,7 @@ class LocallyConnected2D(KerasLayer): bias: Whether to include a bias (i.e. make the layer affine rather than linear). Default is True. input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. >>> locallyconnected2d = LocallyConnected2D(12, 3, 4, input_shape=(3, 128, 128)) creating: createKerasLocallyConnected2D @@ -1598,8 +1765,8 @@ class LocallyConnected2D(KerasLayer): def __init__(self, nb_filter, nb_row, nb_col, activation=None, border_mode="valid", subsample=(1, 1), dim_ordering="th", W_regularizer=None, b_regularizer=None, bias=True, - input_shape=None, bigdl_type="float"): - super(LocallyConnected2D, self).__init__(None, bigdl_type, + input_shape=None, **kwargs): + super(LocallyConnected2D, self).__init__(None, nb_filter, nb_row, nb_col, @@ -1610,7 +1777,8 @@ def __init__(self, nb_filter, nb_row, nb_col, activation=None, W_regularizer, b_regularizer, bias, - list(input_shape) if input_shape else None) + list(input_shape) if input_shape else None, + **kwargs) class SpatialDropout1D(KerasLayer): @@ -1629,14 +1797,16 @@ class SpatialDropout1D(KerasLayer): # Arguments p: Fraction of the input units to drop. Float between 0 and 1. input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. >>> spatialdropout1d = SpatialDropout1D(0.4, input_shape=(10, 12)) creating: createKerasSpatialDropout1D """ - def __init__(self, p=0.5, input_shape=None, bigdl_type="float"): - super(SpatialDropout1D, self).__init__(None, bigdl_type, + def __init__(self, p=0.5, input_shape=None, **kwargs): + super(SpatialDropout1D, self).__init__(None, float(p), - list(input_shape) if input_shape else None) + list(input_shape) if input_shape else None, + **kwargs) class SpatialDropout2D(KerasLayer): @@ -1656,15 +1826,17 @@ class SpatialDropout2D(KerasLayer): p: Fraction of the input units to drop. Float between 0 and 1. dim_ordering: Format of input data. Either 'th' (Channel First) or 'tf' (Channel Last). Default is 'th'. input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. >>> spatialdropout2d = SpatialDropout2D(0.25, input_shape=(5, 12, 12)) creating: createKerasSpatialDropout2D """ - def __init__(self, p=0.5, dim_ordering="th", input_shape=None, bigdl_type="float"): - super(SpatialDropout2D, self).__init__(None, bigdl_type, + def __init__(self, p=0.5, dim_ordering="th", input_shape=None, **kwargs): + super(SpatialDropout2D, self).__init__(None, float(p), dim_ordering, - list(input_shape) if input_shape else None) + list(input_shape) if input_shape else None, + **kwargs) class SpatialDropout3D(KerasLayer): @@ -1684,15 +1856,17 @@ class SpatialDropout3D(KerasLayer): p: Fraction of the input units to drop. Float between 0 and 1. dim_ordering: Format of input data. Either 'th' (Channel First) or 'tf' (Channel Last). Default is 'th'. input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. >>> spatialdropout3d = SpatialDropout3D(0.6, input_shape=(4, 12, 12, 16)) creating: createKerasSpatialDropout3D """ - def __init__(self, p=0.5, dim_ordering="th", input_shape=None, bigdl_type="float"): - super(SpatialDropout3D, self).__init__(None, bigdl_type, + def __init__(self, p=0.5, dim_ordering="th", input_shape=None, **kwargs): + super(SpatialDropout3D, self).__init__(None, float(p), dim_ordering, - list(input_shape) if input_shape else None) + list(input_shape) if input_shape else None, + **kwargs) class GaussianDropout(KerasLayer): @@ -1707,14 +1881,16 @@ class GaussianDropout(KerasLayer): p: Drop probability. Float between 0 and 1. The multiplicative noise will have standard deviation 'sqrt(p/(1-p))'. input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. >>> gaussiandropout = GaussianDropout(0.45, input_shape=(4, 8)) creating: createKerasGaussianDropout """ - def __init__(self, p, input_shape=None, bigdl_type="float"): - super(GaussianDropout, self).__init__(None, bigdl_type, + def __init__(self, p, input_shape=None, **kwargs): + super(GaussianDropout, self).__init__(None, float(p), - list(input_shape) if input_shape else None) + list(input_shape) if input_shape else None, + **kwargs) class GaussianNoise(KerasLayer): @@ -1730,14 +1906,16 @@ class GaussianNoise(KerasLayer): # Arguments sigma: Float, standard deviation of the noise distribution. input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. - >>> gaussiannoise = GaussianNoise(0.45, input_shape=(3, 4, 5)) + >>> gaussiannoise = GaussianNoise(0.45, input_shape=(3, 4, 5), name="gaussiannoise1") creating: createKerasGaussianNoise """ - def __init__(self, sigma, input_shape=None, bigdl_type="float"): - super(GaussianNoise, self).__init__(None, bigdl_type, + def __init__(self, sigma, input_shape=None, **kwargs): + super(GaussianNoise, self).__init__(None, float(sigma), - list(input_shape) if input_shape else None) + list(input_shape) if input_shape else None, + **kwargs) class Masking(KerasLayer): @@ -1753,14 +1931,16 @@ class Masking(KerasLayer): if all values in the input tensor at that timestep are equal to `mask_value`, then the timestep will masked (skipped) in all downstream layers. input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. >>> masking = Masking(0.3, input_shape=(6, 8)) creating: createKerasMasking """ - def __init__(self, mask_value=0.0, input_shape=None, bigdl_type="float"): - super(Masking, self).__init__(None, bigdl_type, + def __init__(self, mask_value=0.0, input_shape=None, **kwargs): + super(Masking, self).__init__(None, float(mask_value), - list(input_shape) if input_shape else None) + list(input_shape) if input_shape else None, + **kwargs) class SReLU(KerasLayer): @@ -1775,13 +1955,13 @@ class SReLU(KerasLayer): input_shape (a shape tuple, does not include the batch dimension). # Arguments - t_left_init: String representations of initialization method for the left part intercept. + t_left_init: String representation of the initialization method for the left part intercept. Default is 'zero'. - a_left_init: String representations of initialization method for the left part slope. + a_left_init: String representation of the initialization method for the left part slope. Default is 'glorot_uniform'. - t_right_init: String representations of initialization method for the right part intercept. + t_right_init: String representation of ithe nitialization method for the right part intercept. Default is 'glorot_uniform'. - a_right_init: String representations of initialization method for the right part slope. + a_right_init: String representation of the initialization method for the right part slope. Default is 'one'. shared_axes: Int tuple. The axes along which to share learnable parameters for the activation function. Default is None. @@ -1789,20 +1969,22 @@ class SReLU(KerasLayer): (batch, height, width, channels), and you wish to share parameters across space so that each filter only has one set of parameters, set 'SharedAxes=(1,2)'. input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. >>> srelu = SReLU(input_shape=(4, 5)) creating: createKerasSReLU """ def __init__(self, t_left_init='zero', a_left_init='glorot_uniform', t_right_init='glorot_uniform', a_right_init='one', - shared_axes=None, input_shape=None, bigdl_type="float"): - super(SReLU, self).__init__(None, bigdl_type, + shared_axes=None, input_shape=None, **kwargs): + super(SReLU, self).__init__(None, t_left_init, a_left_init, t_right_init, a_right_init, shared_axes, - list(input_shape) if input_shape else None) + list(input_shape) if input_shape else None, + **kwargs) class ELU(KerasLayer): @@ -1818,14 +2000,16 @@ class ELU(KerasLayer): # Arguments alpha: Float, scale for the negative factor. input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. >>> elu = ELU(1.2, input_shape=(4, 5)) creating: createKerasELU """ - def __init__(self, alpha=1.0, input_shape=None, bigdl_type="float"): - super(ELU, self).__init__(None, bigdl_type, + def __init__(self, alpha=1.0, input_shape=None, **kwargs): + super(ELU, self).__init__(None, float(alpha), - list(input_shape) if input_shape else None) + list(input_shape) if input_shape else None, + **kwargs) class LeakyReLU(KerasLayer): @@ -1841,14 +2025,16 @@ class LeakyReLU(KerasLayer): # Arguments alpha: Float >= 0. Negative slope coefficient. input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. >>> leakyrelu = LeakyReLU(0.02, input_shape=(4, 5)) creating: createKerasLeakyReLU """ - def __init__(self, alpha=0.01, input_shape=None, bigdl_type="float"): - super(LeakyReLU, self).__init__(None, bigdl_type, + def __init__(self, alpha=0.01, input_shape=None, **kwargs): + super(LeakyReLU, self).__init__(None, float(alpha), - list(input_shape) if input_shape else None) + list(input_shape) if input_shape else None, + **kwargs) class ThresholdedReLU(KerasLayer): @@ -1864,14 +2050,16 @@ class ThresholdedReLU(KerasLayer): # Arguments theta: Float >= 0. Threshold location of activation. input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. >>> thresholdedrelu = ThresholdedReLU(input_shape=(10, 12)) creating: createKerasThresholdedReLU """ - def __init__(self, theta=1.0, input_shape=None, bigdl_type="float"): - super(ThresholdedReLU, self).__init__(None, bigdl_type, + def __init__(self, theta=1.0, input_shape=None, **kwargs): + super(ThresholdedReLU, self).__init__(None, float(theta), - list(input_shape) if input_shape else None) + list(input_shape) if input_shape else None, + **kwargs) class TimeDistributed(KerasLayer): @@ -1882,19 +2070,22 @@ class TimeDistributed(KerasLayer): When you use this layer as the first layer of a model, you need to provide the argument input_shape (a shape tuple, does not include the batch dimension). + name: String to specify the name of the wrapper. Default is None. # Arguments layer: A layer instance. input_shape: A shape tuple, not including batch. + name: String to set the name of the wrapper. If not specified, its name will by default to be a generated string. - >>> timedistributed = TimeDistributed(Dense(8), input_shape=(10, 12)) + >>> timedistributed = TimeDistributed(Dense(8), input_shape=(10, 12), name="timedistributeddense") creating: createKerasDense creating: createKerasTimeDistributed """ - def __init__(self, layer, input_shape=None, bigdl_type="float"): - super(TimeDistributed, self).__init__(None, bigdl_type, + def __init__(self, layer, input_shape=None, **kwargs): + super(TimeDistributed, self).__init__(None, layer, - list(input_shape) if input_shape else None) + list(input_shape) if input_shape else None, + **kwargs) class Bidirectional(KerasLayer): @@ -1913,13 +2104,15 @@ class Bidirectional(KerasLayer): merge_mode: Mode by which outputs of the forward and backward RNNs will be combined. Must be one of: 'sum', 'mul', 'concat', 'ave'. Default is 'concat'. input_shape: A shape tuple, not including batch. + name: String to set the name of the wrapper. If not specified, its name will by default to be a generated string. - >>> bidiretional = Bidirectional(LSTM(10, return_sequences=True), input_shape=(12, 16)) + >>> bidiretional = Bidirectional(LSTM(10, return_sequences=True), input_shape=(12, 16), name="bidirectionallstm") creating: createKerasLSTM creating: createKerasBidirectional """ - def __init__(self, layer, merge_mode="concat", input_shape=None, bigdl_type="float"): - super(Bidirectional, self).__init__(None, bigdl_type, + def __init__(self, layer, merge_mode="concat", input_shape=None, **kwargs): + super(Bidirectional, self).__init__(None, layer, merge_mode, - list(input_shape) if input_shape else None) \ No newline at end of file + list(input_shape) if input_shape else None, + **kwargs) diff --git a/python/dllib/src/bigdl/dllib/bigdlkeras/layers/topology.py b/python/dllib/src/bigdl/dllib/bigdlkeras/layers/topology.py index 1e7867901cc..4caca14278f 100644 --- a/python/dllib/src/bigdl/dllib/bigdlkeras/layers/topology.py +++ b/python/dllib/src/bigdl/dllib/bigdlkeras/layers/topology.py @@ -135,13 +135,16 @@ def predict(self, x, distributed=True): class Sequential(KerasModel): """ - Container for a Sequential model. + Container for a sequential model. - >>> sequential = Sequential() + # Arguments + name: String to specify the name of the sequential model. Default is None. + + >>> sequential = Sequential(name="seq1") creating: createKerasSequential """ - def __init__(self, bigdl_type="float"): - super(Sequential, self).__init__(None, bigdl_type=bigdl_type) + def __init__(self, **kwargs): + super(Sequential, self).__init__(None, **kwargs) def add(self, model): self.value.add(model.value) @@ -149,7 +152,16 @@ def add(self, model): class Model(KerasModel): - def __init__(self, input, output, bigdl_type="float"): - super(Model, self).__init__(None, bigdl_type, + """ + Container for a graph model. + + # Arguments + input: An input node or a list of input nodes. + output: An output node or a list of output nodes. + name: String to specify the name of the graph model. Default is None. + """ + def __init__(self, input, output, **kwargs): + super(Model, self).__init__(None, to_list(input), - to_list(output)) \ No newline at end of file + to_list(output), + **kwargs) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 1224a23efdc..f9f4031f835 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -78,7 +78,9 @@ def __init__(self, jvalue, bigdl_type, *args): def set_running_mean(self, running_mean): """ - :param running_mean: a ndarray + Set the running mean of the layer. + Only use this method for a BatchNormalization layer. + :param running_mean: a Numpy array. """ callBigDlFunc(self.bigdl_type, "setRunningMean", self.value, JTensor.from_ndarray(running_mean)) @@ -86,7 +88,9 @@ def set_running_mean(self, running_mean): def set_running_std(self, running_std): """ - :param running_mean: a ndarray + Set the running variance of the layer. + Only use this method for a BatchNormalization layer. + :param running_std: a Numpy array. """ callBigDlFunc(self.bigdl_type, "setRunningStd", self.value, JTensor.from_ndarray(running_std)) From f7062728bceaea7e5d907a2a68faa3b9018db1c5 Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Fri, 16 Mar 2018 13:04:28 +0800 Subject: [PATCH 604/823] Keras-like API functional merge and some fix (#2313) * resolve conflicts * update merge * update merge * update doc * refactor python * fix * style * meet review --- .../bigdl/dllib/bigdlkeras/layers/layer.py | 665 +++++++++++------- .../bigdl/dllib/bigdlkeras/layers/topology.py | 26 +- python/dllib/src/bigdl/dllib/nn/layer.py | 8 +- 3 files changed, 454 insertions(+), 245 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py b/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py index 31261882ebb..8b84dc0cdd3 100644 --- a/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py +++ b/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py @@ -65,29 +65,52 @@ def jvm_class_constructor(self): class KerasLayer(Layer, InferShape, KerasCreator): - pass + def __init__(self, jvalue, *args, **kwargs): + allowed_kwargs = {"name", "bigdl_type"} + for kwarg in kwargs.keys(): + if kwarg not in allowed_kwargs: + raise TypeError("Wrong argument for the layer:", kwarg) + bigdl_type = kwargs.get("bigdl_type") + if not bigdl_type: + bigdl_type = "float" + super(KerasCreator, self).__init__(jvalue, bigdl_type, *args) + name = kwargs.get("name") + if name: + self.set_name(name) class Input(Node, KerasCreator): - def __init__(self, name=None, input_shape=None, bigdl_type="float"): + """ + Used to instantiate an input node. + + # Arguments + shape: A shape tuple, not including batch. + name: String to set the name of the input node. If not specified, its name will by default to be a generated string. + + >>> input = Input(name="input1", shape=(3, 5)) + creating: createKerasInput + """ + def __init__(self, shape=None, name=None, bigdl_type="float"): super(Input, self).__init__(None, bigdl_type, name, - list(input_shape) if input_shape else None) + list(shape) if shape else None) class InputLayer(KerasLayer): """ - Layer to be used as an entry point into a model. + Used as an entry point into a model. # Arguments - input_shape: A shape tuple, not including the batch axis. + input_shape: A shape tuple, not including batch. + name: String to set the name of the input layer. If not specified, its name will by default to be a generated string. >>> inputlayer = InputLayer(input_shape=(3, 5)) creating: createKerasInputLayer """ - def __init__(self, input_shape=None, bigdl_type="float"): - super(InputLayer, self).__init__(None, bigdl_type, - list(input_shape) if input_shape else None) + def __init__(self, input_shape=None, **kwargs): + super(InputLayer, self).__init__(None, + list(input_shape) if input_shape else None, + **kwargs) class Dense(KerasLayer): @@ -100,30 +123,36 @@ class Dense(KerasLayer): # Arguments output_dim: The size of output dimension. - init: String representations of initialization method for the weights of the layer. + init: String representation of the initialization method for the weights of the layer. Default is 'glorot_uniform'. - activation: String representations of activation function to use (such as 'relu' or 'sigmoid'). + activation: String representation of the activation function to use (such as 'relu' or 'sigmoid'). Default is None. W_regularizer: An instance of [[Regularizer]], (eg. L1 or L2 regularization), applied to the input weights matrices. Default is None. b_regularizer: An instance of [[Regularizer]], applied to the bias. Default is None. bias: Whether to include a bias (i.e. make the layer affine rather than linear). Default is True. + input_dim: Dimensionality of the input for 2D input. For nD input, you can alternatively specify + 'input_shape' when using this layer as the first layer. input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. - >>> dense = Dense(10, input_shape=(3, 4)) + >>> dense = Dense(10, input_dim=8, name="dense1") creating: createKerasDense """ def __init__(self, output_dim, init="glorot_uniform", activation=None, W_regularizer=None, b_regularizer=None, - bias=True, input_shape=None, bigdl_type="float"): - super(Dense, self).__init__(None, bigdl_type, + bias=True, input_dim=None, input_shape=None, **kwargs): + if input_dim: + input_shape = (input_dim, ) + super(Dense, self).__init__(None, output_dim, init, activation, W_regularizer, b_regularizer, bias, - list(input_shape) if input_shape else None) + list(input_shape) if input_shape else None, + **kwargs) class MaxoutDense(KerasLayer): @@ -142,20 +171,26 @@ class MaxoutDense(KerasLayer): applied to the input weights matrices. Default is None. b_regularizer: An instance of [[Regularizer]], applied to the bias. Default is None. bias: Whether to include a bias (i.e. make the layer affine rather than linear). Default is True. + input_dim: Dimensionality of the input. Alternatively, you can specify 'input_shape' + when using this layer as the first layer. input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. >>> maxoutdense = MaxoutDense(6, input_shape=(10, )) creating: createKerasMaxoutDense """ - def __init__(self, output_dim, nb_feature=4, W_regularizer=None, - b_regularizer=None, bias=True, input_shape=None, bigdl_type="float"): - super(MaxoutDense, self).__init__(None, bigdl_type, + def __init__(self, output_dim, nb_feature=4, W_regularizer=None, b_regularizer=None, + bias=True, input_dim=None, input_shape=None, **kwargs): + if input_dim: + input_shape = (input_dim, ) + super(MaxoutDense, self).__init__(None, output_dim, nb_feature, W_regularizer, b_regularizer, bias, - list(input_shape) if input_shape else None) + list(input_shape) if input_shape else None, + **kwargs) class Embedding(KerasLayer): @@ -169,23 +204,25 @@ class Embedding(KerasLayer): # Arguments input_dim: Size of the vocabulary. Int > 0. output_dim: Dimension of the dense embedding. Int >= 0. - init: String representations of initialization method for the weights of the layer. + init: String representation of the initialization method for the weights of the layer. Default is 'uniform'. W_regularizer: An instance of [[Regularizer]], (eg. L1 or L2 regularization), applied to the embedding matrix. Default is None. input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. - >>> embedding = Embedding(1000, 32, input_shape=(10, )) + >>> embedding = Embedding(1000, 32, input_shape=(10, ), name="embedding1") creating: createKerasEmbedding """ - def __init__(self, input_dim, output_dim, init="uniform", - W_regularizer=None, input_shape=None, bigdl_type="float"): - super(Embedding, self).__init__(None, bigdl_type, + def __init__(self, input_dim, output_dim, init="uniform", W_regularizer=None, + input_shape=None, **kwargs): + super(Embedding, self).__init__(None, input_dim, output_dim, init, W_regularizer, - list(input_shape) if input_shape else None) + list(input_shape) if input_shape else None, + **kwargs) class BatchNormalization(KerasLayer): @@ -203,47 +240,69 @@ class BatchNormalization(KerasLayer): epsilon: Small float > 0. Fuzz parameter. Default is 0.001. momentum: Float. Momentum in the computation of the exponential average of the mean and standard deviation of the data, for feature-wise normalization. Default is 0.99. - beta_init: Name of initialization function for shift parameter. Default is 'zero'. - gamma_init: Name of initialization function for scale parameter. Default is 'one'. + beta_init: Name of the initialization function for shift parameter. Default is 'zero'. + gamma_init: Name of the initialization function for scale parameter. Default is 'one'. dim_ordering: Format of input data. Either 'th' (Channel First) or 'tf' (Channel Last). Default is 'th'. For 'th', axis along which to normalize is 1. For 'tf', axis is 3. input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. - >>> batchnormalization = BatchNormalization(input_shape=(3, 12, 12)) + >>> batchnormalization = BatchNormalization(input_shape=(3, 12, 12), name="bn1") creating: createKerasBatchNormalization """ - def __init__(self, epsilon=0.001, momentum=0.99, beta_init="zero", gamma_init="one", - dim_ordering="th", input_shape=None, bigdl_type="float"): - super(BatchNormalization, self).__init__(None, bigdl_type, + def __init__(self, epsilon=0.001, mode=0, axis=1, momentum=0.99, beta_init="zero", gamma_init="one", + dim_ordering="th", input_shape=None, **kwargs): + if mode != 0: + raise ValueError("For BatchNormalization, only mode=0 is supported for now") + if dim_ordering == "th" and axis != 1: + raise ValueError("For BatchNormalization with th dim ordering, only axis=1 is supported for now") + if dim_ordering == "tf" and axis != -1 and axis != 3: + raise ValueError("For BatchNormalization with tf dim ordering, only axis=-1 is supported for now") + super(BatchNormalization, self).__init__(None, float(epsilon), float(momentum), beta_init, gamma_init, dim_ordering, - list(input_shape) if input_shape else None) + list(input_shape) if input_shape else None, + **kwargs) def set_running_mean(self, running_mean): - callBigDlFunc(self.bigdl_type, "setKerasRunningMean", + """ + Set the running mean of the BatchNormalization layer. + :param running_mean: a Numpy array. + """ + callBigDlFunc(self.bigdl_type, "setRunningMean", self.value, JTensor.from_ndarray(running_mean)) return self def set_running_std(self, running_std): - callBigDlFunc(self.bigdl_type, "setKerasRunningStd", + """ + Set the running variance of the BatchNormalization layer. + :param running_std: a Numpy array. + """ + callBigDlFunc(self.bigdl_type, "setRunningStd", self.value, JTensor.from_ndarray(running_std)) return self def get_running_mean(self): - return callBigDlFunc(self.bigdl_type, "getKerasRunningMean", + """ + Get the running meaning of the BatchNormalization layer. + """ + return callBigDlFunc(self.bigdl_type, "getRunningMean", self.value).to_ndarray() def get_running_std(self): - return callBigDlFunc(self.bigdl_type, "getKerasRunningStd", + """ + Get the running variance of the BatchNormalization layer. + """ + return callBigDlFunc(self.bigdl_type, "getRunningStd", self.value).to_ndarray() class Merge(KerasLayer): """ - Used to merge a list of tensors into a single tensor, following some merge mode. + Used to merge a list of inputs into a single output, following some merge mode. Merge must have at least two input layers. When using this layer as the first layer in a model, you need to provide the argument @@ -253,24 +312,43 @@ class Merge(KerasLayer): layers: A list of layer instances. Must be more than one layer. mode: Merge mode. String, must be one of: 'sum', 'mul', 'concat', 'ave', 'cos', 'dot', 'max'. Default is 'sum'. - concat_axis: Int, axis to use in mode concat. Only specify this when mode is 'concat'. + concat_axis: Int, axis to use when concatenating layers. Only specify this when merge mode is 'concat'. Default is -1, meaning the last axis of the input. input_shape: A list of shape tuples, each not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. >>> l1 = InputLayer(input_shape=(3, 5)) creating: createKerasInputLayer >>> l2 = InputLayer(input_shape=(3, 5)) creating: createKerasInputLayer - >>> merge = Merge(layers=[l1, l2], mode='sum') + >>> merge = Merge(layers=[l1, l2], mode='sum', name="merge1") creating: createKerasMerge """ def __init__(self, layers=None, mode="sum", concat_axis=-1, - input_shape=None, bigdl_type="float"): - super(Merge, self).__init__(None, bigdl_type, + input_shape=None, **kwargs): + super(Merge, self).__init__(None, list(layers) if layers else None, mode, concat_axis, - input_shape) + input_shape, + **kwargs) + + +def merge(inputs, mode="sum", concat_axis=-1, name=None): + """ + Functional merge. Only use this method if you are defining a graph model. + Used to merge a list of input nodes into a single output node (NOT layers!), + following some merge mode. + + # Arguments + inputs: A list of node instances. Must be more than one node. + mode: Merge mode. String, must be one of: 'sum', 'mul', 'concat', 'ave', 'cos', + 'dot', 'max'. Default is 'sum'. + concat_axis: Int, axis to use when concatenating nodes. Only specify this when merge mode is 'concat'. + Default is -1, meaning the last axis of the input. + name: String to set the name of the merge. If not specified, its name will by default to be a generated string. + """ + return Merge(mode=mode, concat_axis=concat_axis, name=name)(list(inputs)) class Dropout(KerasLayer): @@ -284,14 +362,16 @@ class Dropout(KerasLayer): # Arguments p: Fraction of the input units to drop. Float between 0 and 1. input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. >>> dropout = Dropout(0.25, input_shape=(2, 3)) creating: createKerasDropout """ - def __init__(self, p, input_shape=None, bigdl_type="float"): - super(Dropout, self).__init__(None, bigdl_type, + def __init__(self, p, input_shape=None, **kwargs): + super(Dropout, self).__init__(None, float(p), - list(input_shape) if input_shape else None) + list(input_shape) if input_shape else None, + **kwargs) class Flatten(KerasLayer): @@ -303,13 +383,15 @@ class Flatten(KerasLayer): # Arguments input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. >>> flatten = Flatten(input_shape=(3, 10, 2)) creating: createKerasFlatten """ - def __init__(self, input_shape=None, bigdl_type="float"): - super(Flatten, self).__init__(None, bigdl_type, - list(input_shape) if input_shape else None) + def __init__(self, input_shape=None, **kwargs): + super(Flatten, self).__init__(None, + list(input_shape) if input_shape else None, + **kwargs) class Reshape(KerasLayer): @@ -325,14 +407,16 @@ class Reshape(KerasLayer): # Arguments target_shape: A shape tuple. The target shape that you desire to have. Batch dimension should be excluded. input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. >>> reshape = Reshape((2, 10), input_shape=(5, 4)) creating: createKerasReshape """ - def __init__(self, target_shape, input_shape=None, bigdl_type="float"): - super(Reshape, self).__init__(None, bigdl_type, + def __init__(self, target_shape, input_shape=None, **kwargs): + super(Reshape, self).__init__(None, target_shape, - list(input_shape) if input_shape else None) + list(input_shape) if input_shape else None, + **kwargs) class Activation(KerasLayer): @@ -346,14 +430,16 @@ class Activation(KerasLayer): # Arguments activation: Name of the activation function as string. input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. >>> activation = Activation("relu", input_shape=(3, 4)) creating: createKerasActivation """ - def __init__(self, activation, input_shape=None, bigdl_type="float"): - super(Activation, self).__init__(None, bigdl_type, + def __init__(self, activation, input_shape=None, **kwargs): + super(Activation, self).__init__(None, activation, - list(input_shape) if input_shape else None) + list(input_shape) if input_shape else None, + **kwargs) class RepeatVector(KerasLayer): @@ -366,15 +452,21 @@ class RepeatVector(KerasLayer): # Arguments n: Repetition factor. Int. + input_dim: Dimensionality of the input. Alternatively, you can specify 'input_shape' + when using this layer as the first layer. input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. >>> repeatvector = RepeatVector(5, input_shape=(3, )) creating: createKerasRepeatVector """ - def __init__(self, n, input_shape=None, bigdl_type="float"): - super(RepeatVector, self).__init__(None, bigdl_type, + def __init__(self, n, input_dim=None, input_shape=None, **kwargs): + if input_dim: + input_shape = (input_dim, ) + super(RepeatVector, self).__init__(None, n, - list(input_shape) if input_shape else None) + list(input_shape) if input_shape else None, + **kwargs) class Permute(KerasLayer): @@ -388,14 +480,16 @@ class Permute(KerasLayer): # Arguments dims: Tuple of int. Permutation pattern, does not include the samples dimension. Indexing starts at 1. input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. >>> permute = Permute((2, 1, 3), input_shape=(3, 4, 5)) creating: createKerasPermute """ - def __init__(self, dims, input_shape=None, bigdl_type="float"): - super(Permute, self).__init__(None, bigdl_type, + def __init__(self, dims, input_shape=None, **kwargs): + super(Permute, self).__init__(None, dims, - list(input_shape) if input_shape else None) + list(input_shape) if input_shape else None, + **kwargs) class Highway(KerasLayer): @@ -407,26 +501,32 @@ class Highway(KerasLayer): input_shape (a shape tuple, does not include the batch dimension). # Arguments - activation: String representations of activation function to use (such as 'relu' or 'sigmoid'). + activation: String representation of the activation function to use (such as 'relu' or 'sigmoid'). Default is None. W_regularizer: An instance of [[Regularizer]], (eg. L1 or L2 regularization), applied to the input weights matrices. Default is None. b_regularizer: An instance of [[Regularizer]], applied to the bias. Default is None. bias: Whether to include a bias (i.e. make the layer affine rather than linear). Default is True. + input_dim: Dimensionality of the input. Alternatively, you can specify 'input_shape' + when using this layer as the first layer. input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. >>> highway = Highway(activation='relu', input_shape=(8, )) creating: createKerasHighway """ def __init__(self, activation=None, W_regularizer=None, b_regularizer=None, - bias=True, input_shape=None, bigdl_type="float"): - super(Highway, self).__init__(None, bigdl_type, + bias=True, input_dim=None, input_shape=None, **kwargs): + if input_dim: + input_shape = (input_dim, ) + super(Highway, self).__init__(None, activation, W_regularizer, b_regularizer, bias, - list(input_shape) if input_shape else None) + list(input_shape) if input_shape else None, + **kwargs) class Convolution1D(KerasLayer): @@ -441,9 +541,9 @@ class Convolution1D(KerasLayer): # Arguments nb_filter: Number of convolution filters to use. filter_length: The extension (spatial or temporal) of each filter. - init: String representations of initialization method for the weights of the layer. + init: String representation of the initialization method for the weights of the layer. Default is 'glorot_uniform'. - activation: String representations of activation function to use (such as 'relu' or 'sigmoid'). + activation: String representation of the activation function to use (such as 'relu' or 'sigmoid'). Default is None. border_mode: Either 'valid' or 'same'. Default is 'valid'. subsample_length: Factor by which to subsample output. Int. Default is 1. @@ -453,6 +553,7 @@ class Convolution1D(KerasLayer): bias: Whether to include a bias (i.e. make the layer affine rather than linear). Default is True. input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. >>> conv1d = Convolution1D(12, 4, input_shape=(3, 16)) creating: createKerasConvolution1D @@ -460,8 +561,8 @@ class Convolution1D(KerasLayer): def __init__(self, nb_filter, filter_length, init="glorot_uniform", activation=None, border_mode="valid", subsample_length=1, W_regularizer=None, b_regularizer=None, bias=True, - input_shape=None, bigdl_type="float"): - super(Convolution1D, self).__init__(None, bigdl_type, + input_shape=None, **kwargs): + super(Convolution1D, self).__init__(None, nb_filter, filter_length, init, @@ -471,7 +572,8 @@ def __init__(self, nb_filter, filter_length, init="glorot_uniform", W_regularizer, b_regularizer, bias, - list(input_shape) if input_shape else None) + list(input_shape) if input_shape else None, + **kwargs) class Convolution2D(KerasLayer): @@ -488,9 +590,9 @@ class Convolution2D(KerasLayer): nb_filter: Number of convolution filters to use. nb_row: Number of rows in the convolution kernel. nb_col: Number of cols in the convolution kernel. - init: String representations of initialization method for the weights of the layer. + init: String representation of the initialization method for the weights of the layer. Default is 'glorot_uniform'. - activation: String representations of activation function to use (such as 'relu' or 'sigmoid'). + activation: String representation of the activation function to use (such as 'relu' or 'sigmoid'). Default is None. border_mode: Either 'valid' or 'same'. Default is 'valid'. subsample: Int tuple of length 2 corresponding to the step of the convolution in the @@ -502,16 +604,17 @@ class Convolution2D(KerasLayer): bias: Whether to include a bias (i.e. make the layer affine rather than linear). Default is True. input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. - >>> conv2d = Convolution2D(32, 3, 3, input_shape=(3, 128, 128)) + >>> conv2d = Convolution2D(32, 3, 3, input_shape=(3, 128, 128), name="convolution2d_1") creating: createKerasConvolution2D """ def __init__(self, nb_filter, nb_row, nb_col, init="glorot_uniform", activation=None, border_mode="valid", subsample=(1, 1), dim_ordering="th", W_regularizer=None, b_regularizer=None, bias=True, - input_shape=None, bigdl_type="float"): - super(Convolution2D, self).__init__(None, bigdl_type, + input_shape=None, **kwargs): + super(Convolution2D, self).__init__(None, nb_filter, nb_row, nb_col, @@ -523,7 +626,8 @@ def __init__(self, nb_filter, nb_row, nb_col, W_regularizer, b_regularizer, bias, - list(input_shape) if input_shape else None) + list(input_shape) if input_shape else None, + **kwargs) class Convolution3D(KerasLayer): @@ -541,9 +645,9 @@ class Convolution3D(KerasLayer): kernel_dim1: Length of the first dimension in the convolution kernel. kernel_dim2: Length of the second dimension in the convolution kernel. kernel_dim3: Length of the third dimension in the convolution kernel. - init: String representations of initialization method for the weights of the layer. + init: String representation of the initialization method for the weights of the layer. Default is 'glorot_uniform'. - activation: String representations of activation function to use (such as 'relu' or 'sigmoid'). + activation: String representation of the activation function to use (such as 'relu' or 'sigmoid'). Default is None. border_mode: Either 'valid' or 'same'. Default is 'valid'. subsample: Int tuple of length 3. Factor by which to subsample output. @@ -555,6 +659,7 @@ class Convolution3D(KerasLayer): bias: Whether to include a bias (i.e. make the layer affine rather than linear). Default is True. input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. >>> conv3d = Convolution3D(32, 3, 4, 5, input_shape=(3, 64, 64, 64)) creating: createKerasConvolution3D @@ -562,8 +667,8 @@ class Convolution3D(KerasLayer): def __init__(self, nb_filter, kernel_dim1, kernel_dim2, kernel_dim3, init="glorot_uniform", activation=None, border_mode="valid", subsample=(1, 1, 1), dim_ordering="th", W_regularizer=None, - b_regularizer=None, bias=True, input_shape=None, bigdl_type="float"): - super(Convolution3D, self).__init__(None, bigdl_type, + b_regularizer=None, bias=True, input_shape=None, **kwargs): + super(Convolution3D, self).__init__(None, nb_filter, kernel_dim1, kernel_dim2, @@ -576,7 +681,8 @@ def __init__(self, nb_filter, kernel_dim1, kernel_dim2, kernel_dim3, W_regularizer, b_regularizer, bias, - list(input_shape) if input_shape else None) + list(input_shape) if input_shape else None, + **kwargs) class AtrousConvolution1D(KerasLayer): @@ -594,9 +700,9 @@ class AtrousConvolution1D(KerasLayer): # Arguments nb_filter: Number of convolution filters to use. filter_length: The extension (spatial or temporal) of each filter. - init: String representations of initialization method for the weights of the layer. + init: String representation of the initialization method for the weights of the layer. Default is 'glorot_uniform'. - activation: String representations of activation function to use (such as 'relu' or 'sigmoid'). + activation: String representation of the activation function to use (such as 'relu' or 'sigmoid'). Default is None. border_mode: Only 'valid' is supported for now. subsample_length: Factor by which to subsample output. Int. Default is 1. @@ -606,18 +712,19 @@ class AtrousConvolution1D(KerasLayer): b_regularizer: An instance of [[Regularizer]], applied to the bias. Default is None. bias: Only 'True' is supported for now. input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. >>> atrousconv1d = AtrousConvolution1D(8, 3, input_shape=(3, 12)) creating: createKerasAtrousConvolution1D """ - def __init__(self, nb_filter, filter_length, init="glorot_uniform", - activation=None, border_mode='valid', subsample_length=1, atrous_rate=1, - W_regularizer=None, b_regularizer=None, bias=True, input_shape=None, bigdl_type="float"): + def __init__(self, nb_filter, filter_length, init="glorot_uniform", activation=None, + border_mode='valid', subsample_length=1, atrous_rate=1, W_regularizer=None, + b_regularizer=None, bias=True, input_shape=None, **kwargs): if border_mode != "valid": raise ValueError("For AtrousConvolution1D, only border_mode='valid' is supported for now") if not bias: raise ValueError("For AtrousConvolution1D, only bias=True is supported for now") - super(AtrousConvolution1D, self).__init__(None, bigdl_type, + super(AtrousConvolution1D, self).__init__(None, nb_filter, filter_length, init, @@ -626,7 +733,8 @@ def __init__(self, nb_filter, filter_length, init="glorot_uniform", atrous_rate, W_regularizer, b_regularizer, - list(input_shape) if input_shape else None) + list(input_shape) if input_shape else None, + **kwargs) class AtrousConvolution2D(KerasLayer): @@ -647,9 +755,9 @@ class AtrousConvolution2D(KerasLayer): nb_filter: Number of convolution filters to use. nb_row: Number of rows in the convolution kernel. nb_col: Number of cols in the convolution kernel. - init: String representations of initialization method for the weights of the layer. + init: String representation of the initialization method for the weights of the layer. Default is 'glorot_uniform'. - activation: String representations of activation function to use (such as 'relu' or 'sigmoid'). + activation: String representation of the activation function to use (such as 'relu' or 'sigmoid'). Default is None. border_mode: Only 'valid' is supported for now. subsample: Int tuple of length 2 corresponding to the step of the convolution in the @@ -662,6 +770,7 @@ class AtrousConvolution2D(KerasLayer): b_regularizer: An instance of [[Regularizer]], applied to the bias. Default is None. bias: Only 'True' is supported for now. input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. >>> atrousconv2d = AtrousConvolution2D(12, 4, 3, input_shape=(3, 64, 64)) creating: createKerasAtrousConvolution2D @@ -669,12 +778,12 @@ class AtrousConvolution2D(KerasLayer): def __init__(self, nb_filter, nb_row, nb_col, init="glorot_uniform", activation=None, border_mode="valid", subsample=(1, 1), atrous_rate=(1, 1), dim_ordering="th", W_regularizer=None, - b_regularizer=None, bias=True, input_shape=None, bigdl_type="float"): + b_regularizer=None, bias=True, input_shape=None, **kwargs): if border_mode != "valid": raise ValueError("For AtrousConvolution2D, only border_mode='valid' is supported for now") if not bias: raise ValueError("For AtrousConvolution2D, only bias=True is supported for now") - super(AtrousConvolution2D, self).__init__(None, bigdl_type, + super(AtrousConvolution2D, self).__init__(None, nb_filter, nb_row, nb_col, @@ -685,7 +794,8 @@ def __init__(self, nb_filter, nb_row, nb_col, init="glorot_uniform", dim_ordering, W_regularizer, b_regularizer, - list(input_shape) if input_shape else None) + list(input_shape) if input_shape else None, + **kwargs) class Deconvolution2D(KerasLayer): @@ -709,9 +819,9 @@ class Deconvolution2D(KerasLayer): nb_row: Number of rows in the convolution kernel. nb_col: Number of cols in the convolution kernel. output_shape: Output shape of the transposed convolution operation. Tuple of int. - init: String representations of initialization method for the weights of the layer. + init: String representation of the initialization method for the weights of the layer. Default is 'glorot_uniform'. - activation: String representations of activation function to use (such as 'relu' or 'sigmoid'). + activation: String representation of the activation function to use (such as 'relu' or 'sigmoid'). Default is None. border_mode: Only 'valid' is supported for now. subsample: Int tuple of length 2 corresponding to the step of the convolution in the @@ -723,16 +833,17 @@ class Deconvolution2D(KerasLayer): bias: Whether to include a bias (i.e. make the layer affine rather than linear). Default is True. input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. >>> deconv2d = Deconvolution2D(3, 3, 3, output_shape=(None, 3, 14, 14), input_shape=(3, 12, 12)) creating: createKerasDeconvolution2D """ def __init__(self, nb_filter, nb_row, nb_col, output_shape, init="glorot_uniform", activation=None, border_mode="valid", subsample=(1, 1), dim_ordering="th", - W_regularizer=None, b_regularizer=None, bias=True, input_shape=None, bigdl_type="float"): + W_regularizer=None, b_regularizer=None, bias=True, input_shape=None, **kwargs): if border_mode != "valid": raise ValueError("For Deconvolution2D, only border_mode='valid' is supported for now") - super(Deconvolution2D, self).__init__(None, bigdl_type, + super(Deconvolution2D, self).__init__(None, nb_filter, nb_row, nb_col, @@ -743,7 +854,8 @@ def __init__(self, nb_filter, nb_row, nb_col, output_shape, init="glorot_uniform W_regularizer, b_regularizer, bias, - list(input_shape) if input_shape else None) + list(input_shape) if input_shape else None, + **kwargs) class SeparableConvolution2D(KerasLayer): @@ -764,9 +876,9 @@ class SeparableConvolution2D(KerasLayer): nb_filter: Number of convolution filters to use. nb_row: Number of rows in the convolution kernel. nb_col: Number of cols in the convolution kernel. - init: String representations of initialization method for the weights of the layer. + init: String representation of the initialization method for the weights of the layer. Default is 'glorot_uniform'. - activation: String representations of activation function to use (such as 'relu' or 'sigmoid'). + activation: String representation of the activation function to use (such as 'relu' or 'sigmoid'). Default is None. border_mode: Either 'valid' or 'same'. Default is 'valid'. subsample: Int tuple of length 2 corresponding to the step of the convolution in the @@ -782,6 +894,7 @@ class SeparableConvolution2D(KerasLayer): bias: Whether to include a bias (i.e. make the layer affine rather than linear). Default is True. input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. >>> separableconv2d = SeparableConvolution2D(12, 3, 4, input_shape=(3, 32, 32)) creating: createKerasSeparableConvolution2D @@ -789,8 +902,8 @@ class SeparableConvolution2D(KerasLayer): def __init__(self, nb_filter, nb_row, nb_col, init="glorot_uniform", activation=None, border_mode="valid", subsample=(1, 1), depth_multiplier=1, dim_ordering="th", depthwise_regularizer=None, pointwise_regularizer=None, - b_regularizer=None, bias=True, input_shape=None, bigdl_type="float"): - super(SeparableConvolution2D, self).__init__(None, bigdl_type, + b_regularizer=None, bias=True, input_shape=None, **kwargs): + super(SeparableConvolution2D, self).__init__(None, nb_filter, nb_row, nb_col, @@ -804,7 +917,8 @@ def __init__(self, nb_filter, nb_row, nb_col, init="glorot_uniform", pointwise_regularizer, b_regularizer, bias, - list(input_shape) if input_shape else None) + list(input_shape) if input_shape else None, + **kwargs) Conv1D = Convolution1D @@ -828,14 +942,16 @@ class Cropping1D(KerasLayer): cropping: Int tuple of length 2. How many units should be trimmed off at the beginning and end of the cropping dimension. Default is (1, 1). input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. >>> cropping1d = Cropping1D(cropping=(1, 2), input_shape=(8, 8)) creating: createKerasCropping1D """ - def __init__(self, cropping=(1, 1), input_shape=None, bigdl_type="float"): - super(Cropping1D, self).__init__(None, bigdl_type, + def __init__(self, cropping=(1, 1), input_shape=None, **kwargs): + super(Cropping1D, self).__init__(None, cropping, - list(input_shape) if input_shape else None) + list(input_shape) if input_shape else None, + **kwargs) class Cropping2D(KerasLayer): @@ -850,17 +966,19 @@ class Cropping2D(KerasLayer): cropping: Int tuple of tuple of length 2. How many units should be trimmed off at the beginning and end of the 2 cropping dimensions (i.e. height and width). Default is ((0, 0), (0, 0)). input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. >>> cropping2d = Cropping2D(cropping=((1, 2), (0, 1)), input_shape=(12, 12, 12)) creating: createKerasCropping2D """ def __init__(self, cropping=((0, 0), (0, 0)), dim_ordering="th", - input_shape=None, bigdl_type="float"): - super(Cropping2D, self).__init__(None, bigdl_type, + input_shape=None, **kwargs): + super(Cropping2D, self).__init__(None, cropping[0], cropping[1], dim_ordering, - list(input_shape) if input_shape else None) + list(input_shape) if input_shape else None, + **kwargs) class Cropping3D(KerasLayer): @@ -876,18 +994,20 @@ class Cropping3D(KerasLayer): end of the 3 cropping dimensions (i.e. kernel_dim1, kernel_dim2 and kernel_dim3). Default is ((1, 1), (1, 1), (1, 1)). input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. >>> cropping3d = Cropping3D(cropping=((0, 2), (1, 1), (3, 1)), input_shape=(4, 12, 12, 16)) creating: createKerasCropping3D """ def __init__(self, cropping=((1, 1), (1, 1), (1, 1)), dim_ordering="th", - input_shape=None, bigdl_type="float"): - super(Cropping3D, self).__init__(None, bigdl_type, + input_shape=None, **kwargs): + super(Cropping3D, self).__init__(None, cropping[0], cropping[1], cropping[2], dim_ordering, - list(input_shape) if input_shape else None) + list(input_shape) if input_shape else None, + **kwargs) class UpSampling1D(KerasLayer): @@ -902,14 +1022,16 @@ class UpSampling1D(KerasLayer): # Arguments length: Int. UpSampling factor. Default is 2. input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. >>> upsampling1d = UpSampling1D(length=3, input_shape=(3, 12)) creating: createKerasUpSampling1D """ - def __init__(self, length=2, input_shape=None, bigdl_type="float"): - super(UpSampling1D, self).__init__(None, bigdl_type, + def __init__(self, length=2, input_shape=None, **kwargs): + super(UpSampling1D, self).__init__(None, length, - list(input_shape) if input_shape else None) + list(input_shape) if input_shape else None, + **kwargs) class UpSampling2D(KerasLayer): @@ -925,15 +1047,17 @@ class UpSampling2D(KerasLayer): size: Int tuple of length 2. UpSampling factors for rows and columns. Default is (2, 2). dim_ordering: Format of input data. Either 'th' (Channel First) or 'tf' (Channel Last). Default is 'th'. input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. >>> upsampling2d = UpSampling2D(size=(1, 3), input_shape=(3, 16, 16)) creating: createKerasUpSampling2D """ - def __init__(self, size=(2, 2), dim_ordering="th", input_shape=None, bigdl_type="float"): - super(UpSampling2D, self).__init__(None, bigdl_type, + def __init__(self, size=(2, 2), dim_ordering="th", input_shape=None, **kwargs): + super(UpSampling2D, self).__init__(None, size, dim_ordering, - list(input_shape) if input_shape else None) + list(input_shape) if input_shape else None, + **kwargs) class UpSampling3D(KerasLayer): @@ -950,15 +1074,17 @@ class UpSampling3D(KerasLayer): size: Int tuple of length 3. UpSampling factors for dim1, dim2 and dim3. Default is (2, 2, 2). dim_ordering: Format of input data. Only 'th' (Channel First) is supported for now. input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. >>> upsampling3d = UpSampling3D(size=(1, 2, 3), input_shape=(3, 16, 16, 16)) creating: createKerasUpSampling3D """ - def __init__(self, size=(2, 2, 2), dim_ordering="th", input_shape=None, bigdl_type="float"): - super(UpSampling3D, self).__init__(None, bigdl_type, + def __init__(self, size=(2, 2, 2), dim_ordering="th", input_shape=None, **kwargs): + super(UpSampling3D, self).__init__(None, size, dim_ordering, - list(input_shape) if input_shape else None) + list(input_shape) if input_shape else None, + **kwargs) class ZeroPadding1D(KerasLayer): @@ -975,16 +1101,18 @@ class ZeroPadding1D(KerasLayer): If tuple of length 2, how many zeros to add in the order '(left_pad, right_pad)'. Default is 1. input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. >>> zeropadding1d = ZeroPadding1D(padding=2, input_shape=(3, 6)) creating: createKerasZeroPadding1D """ - def __init__(self, padding=1, input_shape=None, bigdl_type="float"): + def __init__(self, padding=1, input_shape=None, **kwargs): if isinstance(padding, int): padding = (padding, padding) - super(ZeroPadding1D, self).__init__(None, bigdl_type, + super(ZeroPadding1D, self).__init__(None, padding, - list(input_shape) if input_shape else None) + list(input_shape) if input_shape else None, + **kwargs) class ZeroPadding2D(KerasLayer): @@ -1002,17 +1130,19 @@ class ZeroPadding2D(KerasLayer): Default is (1, 1). dim_ordering: Format of input data. Either 'th' (Channel First) or 'tf' (Channel Last). Default is 'th'. input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. >>> zeropadding2d = ZeroPadding2D(padding=(2, 1), input_shape=(2, 8, 8)) creating: createKerasZeroPadding2D """ - def __init__(self, padding=(1, 1), dim_ordering="th", input_shape=None, bigdl_type="float"): + def __init__(self, padding=(1, 1), dim_ordering="th", input_shape=None, **kwargs): if len(padding) == 2: padding = (padding[0], padding[0], padding[1], padding[1]) - super(ZeroPadding2D, self).__init__(None, bigdl_type, + super(ZeroPadding2D, self).__init__(None, padding, dim_ordering, - list(input_shape) if input_shape else None) + list(input_shape) if input_shape else None, + **kwargs) class ZeroPadding3D(KerasLayer): @@ -1028,15 +1158,17 @@ class ZeroPadding3D(KerasLayer): Symmetric padding will be applied to each dimension. Default is (1, 1, 1). dim_ordering: Format of input data. Either 'th' (Channel First) or 'tf' (Channel Last). Default is 'th'. input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. >>> zeropadding3d = ZeroPadding3D(padding=(2, 1, 2), input_shape=(2, 8, 8, 10)) creating: createKerasZeroPadding3D """ - def __init__(self, padding=(1, 1, 1), dim_ordering="th", input_shape=None, bigdl_type="float"): - super(ZeroPadding3D, self).__init__(None, bigdl_type, + def __init__(self, padding=(1, 1, 1), dim_ordering="th", input_shape=None, **kwargs): + super(ZeroPadding3D, self).__init__(None, padding, dim_ordering, - list(input_shape) if input_shape else None) + list(input_shape) if input_shape else None, + **kwargs) class MaxPooling1D(KerasLayer): @@ -1053,19 +1185,21 @@ class MaxPooling1D(KerasLayer): Default is None, and in this case it will be equal to pool_length.. border_mode: Either 'valid' or 'same'. Default is 'valid'. input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. >>> maxpooling1d = MaxPooling1D(3, input_shape=(3, 24)) creating: createKerasMaxPooling1D """ def __init__(self, pool_length=2, stride=None, border_mode="valid", - input_shape=None, bigdl_type="float"): + input_shape=None, **kwargs): if not stride: stride = -1 - super(MaxPooling1D, self).__init__(None, bigdl_type, + super(MaxPooling1D, self).__init__(None, pool_length, stride, border_mode, - list(input_shape) if input_shape else None) + list(input_shape) if input_shape else None, + **kwargs) class MaxPooling2D(KerasLayer): @@ -1083,19 +1217,21 @@ class MaxPooling2D(KerasLayer): border_mode: Either 'valid' or 'same'. Default is 'valid'. dim_ordering: Format of input data. Either 'th' (Channel First) or 'tf' (Channel Last). Default is 'th'. input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. - >>> maxpooling2d = MaxPooling2D((2, 2), input_shape=(3, 32, 32)) + >>> maxpooling2d = MaxPooling2D((2, 2), input_shape=(3, 32, 32), name="maxpooling2d_1") creating: createKerasMaxPooling2D """ def __init__(self, pool_size=(2, 2), strides=None, border_mode='valid', dim_ordering='th', - input_shape=None, bigdl_type="float"): - super(MaxPooling2D, self).__init__(None, bigdl_type, + input_shape=None, **kwargs): + super(MaxPooling2D, self).__init__(None, pool_size, strides, border_mode, dim_ordering, - list(input_shape) if input_shape else None) + list(input_shape) if input_shape else None, + **kwargs) class MaxPooling3D(KerasLayer): @@ -1115,19 +1251,21 @@ class MaxPooling3D(KerasLayer): border_mode: Only 'valid' is supported for now. dim_ordering: Format of input data. Only 'th' (Channel First) is supported for now. input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. >>> maxpooling3d = MaxPooling3D((2, 1, 3), input_shape=(3, 32, 32, 32)) creating: createKerasMaxPooling3D """ def __init__(self, pool_size=(2, 2, 2), strides=None, border_mode="valid", - dim_ordering="th", input_shape=None, bigdl_type="float"): + dim_ordering="th", input_shape=None, **kwargs): if border_mode != "valid": raise ValueError("For MaxPooling3D, only border_mode='valid' is supported for now") - super(MaxPooling3D, self).__init__(None, bigdl_type, + super(MaxPooling3D, self).__init__(None, pool_size, strides, dim_ordering, - list(input_shape) if input_shape else None) + list(input_shape) if input_shape else None, + **kwargs) class AveragePooling1D(KerasLayer): @@ -1144,19 +1282,21 @@ class AveragePooling1D(KerasLayer): Default is None, and in this case it will be equal to pool_length.. border_mode: Either 'valid' or 'same'. Default is 'valid'. input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. >>> averagepooling1d = AveragePooling1D(input_shape=(3, 24)) creating: createKerasAveragePooling1D """ def __init__(self, pool_length=2, stride=None, border_mode="valid", - input_shape=None, bigdl_type="float"): + input_shape=None, **kwargs): if not stride: stride = -1 - super(AveragePooling1D, self).__init__(None, bigdl_type, + super(AveragePooling1D, self).__init__(None, pool_length, stride, border_mode, - list(input_shape) if input_shape else None) + list(input_shape) if input_shape else None, + **kwargs) class AveragePooling2D(KerasLayer): @@ -1174,18 +1314,20 @@ class AveragePooling2D(KerasLayer): border_mode: Either 'valid' or 'same'. Default is 'valid'. dim_ordering: Format of input data. Either 'th' (Channel First) or 'tf' (Channel Last). Default is 'th'. input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. >>> averagepooling2d = AveragePooling2D((1, 2), input_shape=(2, 28, 32)) creating: createKerasAveragePooling2D """ def __init__(self, pool_size=(2, 2), strides=None, border_mode="valid", - dim_ordering="th", input_shape=None, bigdl_type="float"): - super(AveragePooling2D, self).__init__(None, bigdl_type, + dim_ordering="th", input_shape=None, **kwargs): + super(AveragePooling2D, self).__init__(None, pool_size, strides, border_mode, dim_ordering, - list(input_shape) if input_shape else None) + list(input_shape) if input_shape else None, + **kwargs) class AveragePooling3D(KerasLayer): @@ -1205,19 +1347,21 @@ class AveragePooling3D(KerasLayer): border_mode: Only 'valid' is supported for now. dim_ordering: Format of input data. Only 'th' (Channel First) is supported for now. input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. >>> averagepooling3d = AveragePooling3D((1, 1, 2), input_shape=(3, 28, 32, 36)) creating: createKerasAveragePooling3D """ def __init__(self, pool_size=(2, 2, 2), strides=None, border_mode="valid", - dim_ordering="th", input_shape=None, bigdl_type="float"): + dim_ordering="th", input_shape=None, **kwargs): if border_mode != "valid": raise ValueError("For AveragePooling3D, only border_mode='valid' is supported for now") - super(AveragePooling3D, self).__init__(None, bigdl_type, + super(AveragePooling3D, self).__init__(None, pool_size, strides, dim_ordering, - list(input_shape) if input_shape else None) + list(input_shape) if input_shape else None, + **kwargs) class GlobalMaxPooling1D(KerasLayer): @@ -1230,13 +1374,15 @@ class GlobalMaxPooling1D(KerasLayer): # Arguments input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. >>> globalmaxpooling1d = GlobalMaxPooling1D(input_shape=(4, 8)) creating: createKerasGlobalMaxPooling1D """ - def __init__(self, input_shape=None, bigdl_type="float"): - super(GlobalMaxPooling1D, self).__init__(None, bigdl_type, - list(input_shape) if input_shape else None) + def __init__(self, input_shape=None, **kwargs): + super(GlobalMaxPooling1D, self).__init__(None, + list(input_shape) if input_shape else None, + **kwargs) class GlobalAveragePooling1D(KerasLayer): @@ -1249,13 +1395,15 @@ class GlobalAveragePooling1D(KerasLayer): # Arguments input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. >>> globalaveragepooling1d = GlobalAveragePooling1D(input_shape=(12, 12)) creating: createKerasGlobalAveragePooling1D """ - def __init__(self, input_shape=None, bigdl_type="float"): - super(GlobalAveragePooling1D, self).__init__(None, bigdl_type, - list(input_shape) if input_shape else None) + def __init__(self, input_shape=None, **kwargs): + super(GlobalAveragePooling1D, self).__init__(None, + list(input_shape) if input_shape else None, + **kwargs) class GlobalMaxPooling2D(KerasLayer): @@ -1269,14 +1417,16 @@ class GlobalMaxPooling2D(KerasLayer): # Arguments dim_ordering: Format of input data. Either 'th' (Channel First) or 'tf' (Channel Last). Default is 'th'. input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. >>> globalmaxpooling2d = GlobalMaxPooling2D(input_shape=(4, 32, 32)) creating: createKerasGlobalMaxPooling2D """ - def __init__(self, dim_ordering="th", input_shape=None, bigdl_type="float"): - super(GlobalMaxPooling2D, self).__init__(None, bigdl_type, + def __init__(self, dim_ordering="th", input_shape=None, **kwargs): + super(GlobalMaxPooling2D, self).__init__(None, dim_ordering, - list(input_shape) if input_shape else None) + list(input_shape) if input_shape else None, + **kwargs) class GlobalAveragePooling2D(KerasLayer): @@ -1290,14 +1440,16 @@ class GlobalAveragePooling2D(KerasLayer): # Arguments dim_ordering: Format of input data. Either 'th' (Channel First) or 'tf' (Channel Last). Default is 'th'. input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. >>> globalaveragepooling2d = GlobalAveragePooling2D(input_shape=(4, 32, 32)) creating: createKerasGlobalAveragePooling2D """ - def __init__(self, dim_ordering="th", input_shape=None, bigdl_type="float"): - super(GlobalAveragePooling2D, self).__init__(None, bigdl_type, + def __init__(self, dim_ordering="th", input_shape=None, **kwargs): + super(GlobalAveragePooling2D, self).__init__(None, dim_ordering, - list(input_shape) if input_shape else None) + list(input_shape) if input_shape else None, + **kwargs) class GlobalMaxPooling3D(KerasLayer): @@ -1313,14 +1465,16 @@ class GlobalMaxPooling3D(KerasLayer): # Arguments dim_ordering: Format of input data. Only 'th' (Channel First) is supported for now. input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. >>> globalmaxpooling3d = GlobalMaxPooling3D(input_shape=(4, 32, 32, 32)) creating: createKerasGlobalMaxPooling3D """ - def __init__(self, dim_ordering="th", input_shape=None, bigdl_type="float"): - super(GlobalMaxPooling3D, self).__init__(None, bigdl_type, + def __init__(self, dim_ordering="th", input_shape=None, **kwargs): + super(GlobalMaxPooling3D, self).__init__(None, dim_ordering, - list(input_shape) if input_shape else None) + list(input_shape) if input_shape else None, + **kwargs) class GlobalAveragePooling3D(KerasLayer): @@ -1336,14 +1490,16 @@ class GlobalAveragePooling3D(KerasLayer): # Arguments dim_ordering: Format of input data. Only 'th' (Channel First) is supported for now. input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. >>> globalaveragepooling3d = GlobalAveragePooling3D(input_shape=(4, 16, 16, 20)) creating: createKerasGlobalAveragePooling3D """ - def __init__(self, dim_ordering="th", input_shape=None, bigdl_type="float"): - super(GlobalAveragePooling3D, self).__init__(None, bigdl_type, + def __init__(self, dim_ordering="th", input_shape=None, **kwargs): + super(GlobalAveragePooling3D, self).__init__(None, dim_ordering, - list(input_shape) if input_shape else None) + list(input_shape) if input_shape else None, + **kwargs) class SimpleRNN(KerasLayer): @@ -1356,7 +1512,7 @@ class SimpleRNN(KerasLayer): # Arguments output_dim: Hidden unit size. Dimension of internal projections and final output. - activation: String representations of activation function to use (such as 'relu' or 'sigmoid'). + activation: String representation of the activation function to use (such as 'relu' or 'sigmoid'). Default is 'tanh'. return_sequences: Whether to return the full sequence or only return the last output in the output sequence. Default is False. @@ -1366,14 +1522,15 @@ class SimpleRNN(KerasLayer): U_regularizer: An instance of [[Regularizer]], applied the recurrent weights matrices. Default is None. b_regularizer: An instance of [[Regularizer]], applied to the bias. Default is None. input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. >>> simplernn = SimpleRNN(16, input_shape=(3, 32)) creating: createKerasSimpleRNN """ def __init__(self, output_dim, activation="tanh", return_sequences=False, go_backwards=False, W_regularizer=None, U_regularizer=None, - b_regularizer=None, input_shape=None, bigdl_type="float"): - super(SimpleRNN, self).__init__(None, bigdl_type, + b_regularizer=None, input_shape=None, **kwargs): + super(SimpleRNN, self).__init__(None, output_dim, activation, return_sequences, @@ -1381,7 +1538,8 @@ def __init__(self, output_dim, activation="tanh", return_sequences=False, W_regularizer, U_regularizer, b_regularizer, - list(input_shape) if input_shape else None) + list(input_shape) if input_shape else None, + **kwargs) class LSTM(KerasLayer): @@ -1394,9 +1552,9 @@ class LSTM(KerasLayer): # Arguments output_dim: Hidden unit size. Dimension of internal projections and final output. - activation: String representations of activation function to use (such as 'relu' or 'sigmoid'). + activation: String representation of the activation function to use (such as 'relu' or 'sigmoid'). Default is 'tanh'. - inner_activation: String representations of activation function for inner cells. Default is 'hard_sigmoid'. + inner_activation: String representation of the activation function for inner cells. Default is 'hard_sigmoid'. return_sequences: Whether to return the full sequence or only return the last output in the output sequence. Default is False. go_backwards: Whether the input sequence will be processed backwards. Default is False. @@ -1405,14 +1563,15 @@ class LSTM(KerasLayer): U_regularizer: An instance of [[Regularizer]], applied the recurrent weights matrices. Default is None. b_regularizer: An instance of [[Regularizer]], applied to the bias. Default is None. input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. - >>> lstm = LSTM(32, input_shape=(8, 16)) + >>> lstm = LSTM(32, input_shape=(8, 16), name="lstm1") creating: createKerasLSTM """ def __init__(self, output_dim, activation="tanh", inner_activation="hard_sigmoid", return_sequences=False, go_backwards=False, W_regularizer=None, - U_regularizer=None, b_regularizer=None, input_shape=None, bigdl_type="float"): - super(LSTM, self).__init__(None, bigdl_type, + U_regularizer=None, b_regularizer=None, input_shape=None, **kwargs): + super(LSTM, self).__init__(None, output_dim, activation, inner_activation, @@ -1421,7 +1580,8 @@ def __init__(self, output_dim, activation="tanh", inner_activation="hard_sigmoid W_regularizer, U_regularizer, b_regularizer, - list(input_shape) if input_shape else None) + list(input_shape) if input_shape else None, + **kwargs) class GRU(KerasLayer): @@ -1434,9 +1594,9 @@ class GRU(KerasLayer): # Arguments output_dim: Hidden unit size. Dimension of internal projections and final output. - activation: String representations of activation function to use (such as 'relu' or 'sigmoid'). + activation: String representation of the activation function to use (such as 'relu' or 'sigmoid'). Default is 'tanh'. - inner_activation: String representations of activation function for inner cells. Default is 'hard_sigmoid'. + inner_activation: String representation of the activation function for inner cells. Default is 'hard_sigmoid'. return_sequences: Whether to return the full sequence or only return the last output in the output sequence. Default is False. go_backwards: Whether the input sequence will be processed backwards. Default is False. @@ -1445,14 +1605,15 @@ class GRU(KerasLayer): U_regularizer: An instance of [[Regularizer]], applied the recurrent weights matrices. Default is None. b_regularizer: An instance of [[Regularizer]], applied to the bias. Default is None. input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. >>> gru = GRU(24, input_shape=(32, 32)) creating: createKerasGRU """ def __init__(self, output_dim, activation="tanh", inner_activation="hard_sigmoid", return_sequences=False, go_backwards=False, W_regularizer=None, - U_regularizer=None, b_regularizer=None, input_shape=None, bigdl_type="float"): - super(GRU, self).__init__(None, bigdl_type, + U_regularizer=None, b_regularizer=None, input_shape=None, **kwargs): + super(GRU, self).__init__(None, output_dim, activation, inner_activation, @@ -1461,7 +1622,8 @@ def __init__(self, output_dim, activation="tanh", inner_activation="hard_sigmoid W_regularizer, U_regularizer, b_regularizer, - list(input_shape) if input_shape else None) + list(input_shape) if input_shape else None, + **kwargs) class ConvLSTM2D(KerasLayer): @@ -1479,9 +1641,9 @@ class ConvLSTM2D(KerasLayer): nb_filter: Number of convolution filters to use. nb_row: Number of rows in the convolution kernel. nb_col: Number of cols in the convolution kernel. Should be equal to nb_row as for a square kernel. - activation: String representations of activation function to use (such as 'relu' or 'sigmoid'). + activation: String representation of the activation function to use (such as 'relu' or 'sigmoid'). Default is 'tanh'. - inner_activation: String representations of activation function for inner cells. Default is 'hard_sigmoid'. + inner_activation: String representation of the activation function for inner cells. Default is 'hard_sigmoid'. dim_ordering: Format of input data. Only 'th' (Channel First) is supported for now. border_mode: Only 'same' is supported for now. subsample: Tuple of length 2. Factor by which to subsample output. Also called strides elsewhere. @@ -1494,6 +1656,7 @@ class ConvLSTM2D(KerasLayer): Default is False. go_backwards: Whether the input sequence will be processed backwards. Default is False. input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. >>> convlstm2d = ConvLSTM2D(24, 3, 3, input_shape=(4, 32, 32, 32)) creating: createKerasConvLSTM2D @@ -1501,14 +1664,14 @@ class ConvLSTM2D(KerasLayer): def __init__(self, nb_filter, nb_row, nb_col, activation="tanh", inner_activation="hard_sigmoid", dim_ordering="th", border_mode="same", subsample=(1, 1), W_regularizer=None, U_regularizer=None, b_regularizer=None, - return_sequences=False, go_backwards=False, input_shape=None, bigdl_type="float"): + return_sequences=False, go_backwards=False, input_shape=None, **kwargs): if nb_row != nb_col: raise ValueError("For ConvLSTM2D, only square kernel is supported for now") if border_mode != "same": raise ValueError("For ConvLSTM2D, only border_mode='same' is supported for now") if subsample[0] != subsample[1]: raise ValueError("For ConvLSTM2D, only equal strides is supported for now") - super(ConvLSTM2D, self).__init__(None, bigdl_type, + super(ConvLSTM2D, self).__init__(None, nb_filter, nb_row, activation, @@ -1520,7 +1683,8 @@ def __init__(self, nb_filter, nb_row, nb_col, activation="tanh", b_regularizer, return_sequences, go_backwards, - list(input_shape) if input_shape else None) + list(input_shape) if input_shape else None, + **kwargs) class LocallyConnected1D(KerasLayer): @@ -1536,7 +1700,7 @@ class LocallyConnected1D(KerasLayer): # Arguments nb_filter: Dimensionality of the output. filter_length: The extension (spatial or temporal) of each filter. - activation: String representations of activation function to use (such as 'relu' or 'sigmoid'). + activation: String representation of the activation function to use (such as 'relu' or 'sigmoid'). Default is None. border_mode: Only 'valid' is supported for now. subsample_length: Factor by which to subsample output. Int. Default is 1. @@ -1546,16 +1710,17 @@ class LocallyConnected1D(KerasLayer): bias: Whether to include a bias (i.e. make the layer affine rather than linear). Default is True. input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. >>> locallyconnected1d = LocallyConnected1D(6, 3, input_shape=(8, 12)) creating: createKerasLocallyConnected1D """ def __init__(self, nb_filter, filter_length, activation=None, border_mode="valid", subsample_length=1, W_regularizer=None, b_regularizer=None, - bias=True, input_shape=None, bigdl_type="float"): + bias=True, input_shape=None, **kwargs): if border_mode != "valid": raise ValueError("For LocallyConnected1D, only border_mode='valid' is supported for now") - super(LocallyConnected1D, self).__init__(None, bigdl_type, + super(LocallyConnected1D, self).__init__(None, nb_filter, filter_length, activation, @@ -1563,7 +1728,8 @@ def __init__(self, nb_filter, filter_length, activation=None, border_mode="valid W_regularizer, b_regularizer, bias, - list(input_shape) if input_shape else None) + list(input_shape) if input_shape else None, + **kwargs) class LocallyConnected2D(KerasLayer): @@ -1579,7 +1745,7 @@ class LocallyConnected2D(KerasLayer): nb_filter: Number of convolution filters to use. nb_row: Number of rows in the convolution kernel. nb_col: Number of cols in the convolution kernel. - activation: String representations of activation function to use (such as 'relu' or 'sigmoid'). + activation: String representation of the activation function to use (such as 'relu' or 'sigmoid'). Default is None. border_mode: Either 'valid' or 'same'. Default is 'valid'. subsample: Int tuple of length 2 corresponding to the step of the convolution in the @@ -1591,6 +1757,7 @@ class LocallyConnected2D(KerasLayer): bias: Whether to include a bias (i.e. make the layer affine rather than linear). Default is True. input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. >>> locallyconnected2d = LocallyConnected2D(12, 3, 4, input_shape=(3, 128, 128)) creating: createKerasLocallyConnected2D @@ -1598,8 +1765,8 @@ class LocallyConnected2D(KerasLayer): def __init__(self, nb_filter, nb_row, nb_col, activation=None, border_mode="valid", subsample=(1, 1), dim_ordering="th", W_regularizer=None, b_regularizer=None, bias=True, - input_shape=None, bigdl_type="float"): - super(LocallyConnected2D, self).__init__(None, bigdl_type, + input_shape=None, **kwargs): + super(LocallyConnected2D, self).__init__(None, nb_filter, nb_row, nb_col, @@ -1610,7 +1777,8 @@ def __init__(self, nb_filter, nb_row, nb_col, activation=None, W_regularizer, b_regularizer, bias, - list(input_shape) if input_shape else None) + list(input_shape) if input_shape else None, + **kwargs) class SpatialDropout1D(KerasLayer): @@ -1629,14 +1797,16 @@ class SpatialDropout1D(KerasLayer): # Arguments p: Fraction of the input units to drop. Float between 0 and 1. input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. >>> spatialdropout1d = SpatialDropout1D(0.4, input_shape=(10, 12)) creating: createKerasSpatialDropout1D """ - def __init__(self, p=0.5, input_shape=None, bigdl_type="float"): - super(SpatialDropout1D, self).__init__(None, bigdl_type, + def __init__(self, p=0.5, input_shape=None, **kwargs): + super(SpatialDropout1D, self).__init__(None, float(p), - list(input_shape) if input_shape else None) + list(input_shape) if input_shape else None, + **kwargs) class SpatialDropout2D(KerasLayer): @@ -1656,15 +1826,17 @@ class SpatialDropout2D(KerasLayer): p: Fraction of the input units to drop. Float between 0 and 1. dim_ordering: Format of input data. Either 'th' (Channel First) or 'tf' (Channel Last). Default is 'th'. input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. >>> spatialdropout2d = SpatialDropout2D(0.25, input_shape=(5, 12, 12)) creating: createKerasSpatialDropout2D """ - def __init__(self, p=0.5, dim_ordering="th", input_shape=None, bigdl_type="float"): - super(SpatialDropout2D, self).__init__(None, bigdl_type, + def __init__(self, p=0.5, dim_ordering="th", input_shape=None, **kwargs): + super(SpatialDropout2D, self).__init__(None, float(p), dim_ordering, - list(input_shape) if input_shape else None) + list(input_shape) if input_shape else None, + **kwargs) class SpatialDropout3D(KerasLayer): @@ -1684,15 +1856,17 @@ class SpatialDropout3D(KerasLayer): p: Fraction of the input units to drop. Float between 0 and 1. dim_ordering: Format of input data. Either 'th' (Channel First) or 'tf' (Channel Last). Default is 'th'. input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. >>> spatialdropout3d = SpatialDropout3D(0.6, input_shape=(4, 12, 12, 16)) creating: createKerasSpatialDropout3D """ - def __init__(self, p=0.5, dim_ordering="th", input_shape=None, bigdl_type="float"): - super(SpatialDropout3D, self).__init__(None, bigdl_type, + def __init__(self, p=0.5, dim_ordering="th", input_shape=None, **kwargs): + super(SpatialDropout3D, self).__init__(None, float(p), dim_ordering, - list(input_shape) if input_shape else None) + list(input_shape) if input_shape else None, + **kwargs) class GaussianDropout(KerasLayer): @@ -1707,14 +1881,16 @@ class GaussianDropout(KerasLayer): p: Drop probability. Float between 0 and 1. The multiplicative noise will have standard deviation 'sqrt(p/(1-p))'. input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. >>> gaussiandropout = GaussianDropout(0.45, input_shape=(4, 8)) creating: createKerasGaussianDropout """ - def __init__(self, p, input_shape=None, bigdl_type="float"): - super(GaussianDropout, self).__init__(None, bigdl_type, + def __init__(self, p, input_shape=None, **kwargs): + super(GaussianDropout, self).__init__(None, float(p), - list(input_shape) if input_shape else None) + list(input_shape) if input_shape else None, + **kwargs) class GaussianNoise(KerasLayer): @@ -1730,14 +1906,16 @@ class GaussianNoise(KerasLayer): # Arguments sigma: Float, standard deviation of the noise distribution. input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. - >>> gaussiannoise = GaussianNoise(0.45, input_shape=(3, 4, 5)) + >>> gaussiannoise = GaussianNoise(0.45, input_shape=(3, 4, 5), name="gaussiannoise1") creating: createKerasGaussianNoise """ - def __init__(self, sigma, input_shape=None, bigdl_type="float"): - super(GaussianNoise, self).__init__(None, bigdl_type, + def __init__(self, sigma, input_shape=None, **kwargs): + super(GaussianNoise, self).__init__(None, float(sigma), - list(input_shape) if input_shape else None) + list(input_shape) if input_shape else None, + **kwargs) class Masking(KerasLayer): @@ -1753,14 +1931,16 @@ class Masking(KerasLayer): if all values in the input tensor at that timestep are equal to `mask_value`, then the timestep will masked (skipped) in all downstream layers. input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. >>> masking = Masking(0.3, input_shape=(6, 8)) creating: createKerasMasking """ - def __init__(self, mask_value=0.0, input_shape=None, bigdl_type="float"): - super(Masking, self).__init__(None, bigdl_type, + def __init__(self, mask_value=0.0, input_shape=None, **kwargs): + super(Masking, self).__init__(None, float(mask_value), - list(input_shape) if input_shape else None) + list(input_shape) if input_shape else None, + **kwargs) class SReLU(KerasLayer): @@ -1775,13 +1955,13 @@ class SReLU(KerasLayer): input_shape (a shape tuple, does not include the batch dimension). # Arguments - t_left_init: String representations of initialization method for the left part intercept. + t_left_init: String representation of the initialization method for the left part intercept. Default is 'zero'. - a_left_init: String representations of initialization method for the left part slope. + a_left_init: String representation of the initialization method for the left part slope. Default is 'glorot_uniform'. - t_right_init: String representations of initialization method for the right part intercept. + t_right_init: String representation of ithe nitialization method for the right part intercept. Default is 'glorot_uniform'. - a_right_init: String representations of initialization method for the right part slope. + a_right_init: String representation of the initialization method for the right part slope. Default is 'one'. shared_axes: Int tuple. The axes along which to share learnable parameters for the activation function. Default is None. @@ -1789,20 +1969,22 @@ class SReLU(KerasLayer): (batch, height, width, channels), and you wish to share parameters across space so that each filter only has one set of parameters, set 'SharedAxes=(1,2)'. input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. >>> srelu = SReLU(input_shape=(4, 5)) creating: createKerasSReLU """ def __init__(self, t_left_init='zero', a_left_init='glorot_uniform', t_right_init='glorot_uniform', a_right_init='one', - shared_axes=None, input_shape=None, bigdl_type="float"): - super(SReLU, self).__init__(None, bigdl_type, + shared_axes=None, input_shape=None, **kwargs): + super(SReLU, self).__init__(None, t_left_init, a_left_init, t_right_init, a_right_init, shared_axes, - list(input_shape) if input_shape else None) + list(input_shape) if input_shape else None, + **kwargs) class ELU(KerasLayer): @@ -1818,14 +2000,16 @@ class ELU(KerasLayer): # Arguments alpha: Float, scale for the negative factor. input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. >>> elu = ELU(1.2, input_shape=(4, 5)) creating: createKerasELU """ - def __init__(self, alpha=1.0, input_shape=None, bigdl_type="float"): - super(ELU, self).__init__(None, bigdl_type, + def __init__(self, alpha=1.0, input_shape=None, **kwargs): + super(ELU, self).__init__(None, float(alpha), - list(input_shape) if input_shape else None) + list(input_shape) if input_shape else None, + **kwargs) class LeakyReLU(KerasLayer): @@ -1841,14 +2025,16 @@ class LeakyReLU(KerasLayer): # Arguments alpha: Float >= 0. Negative slope coefficient. input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. >>> leakyrelu = LeakyReLU(0.02, input_shape=(4, 5)) creating: createKerasLeakyReLU """ - def __init__(self, alpha=0.01, input_shape=None, bigdl_type="float"): - super(LeakyReLU, self).__init__(None, bigdl_type, + def __init__(self, alpha=0.01, input_shape=None, **kwargs): + super(LeakyReLU, self).__init__(None, float(alpha), - list(input_shape) if input_shape else None) + list(input_shape) if input_shape else None, + **kwargs) class ThresholdedReLU(KerasLayer): @@ -1864,14 +2050,16 @@ class ThresholdedReLU(KerasLayer): # Arguments theta: Float >= 0. Threshold location of activation. input_shape: A shape tuple, not including batch. + name: String to set the name of the layer. If not specified, its name will by default to be a generated string. >>> thresholdedrelu = ThresholdedReLU(input_shape=(10, 12)) creating: createKerasThresholdedReLU """ - def __init__(self, theta=1.0, input_shape=None, bigdl_type="float"): - super(ThresholdedReLU, self).__init__(None, bigdl_type, + def __init__(self, theta=1.0, input_shape=None, **kwargs): + super(ThresholdedReLU, self).__init__(None, float(theta), - list(input_shape) if input_shape else None) + list(input_shape) if input_shape else None, + **kwargs) class TimeDistributed(KerasLayer): @@ -1882,19 +2070,22 @@ class TimeDistributed(KerasLayer): When you use this layer as the first layer of a model, you need to provide the argument input_shape (a shape tuple, does not include the batch dimension). + name: String to specify the name of the wrapper. Default is None. # Arguments layer: A layer instance. input_shape: A shape tuple, not including batch. + name: String to set the name of the wrapper. If not specified, its name will by default to be a generated string. - >>> timedistributed = TimeDistributed(Dense(8), input_shape=(10, 12)) + >>> timedistributed = TimeDistributed(Dense(8), input_shape=(10, 12), name="timedistributeddense") creating: createKerasDense creating: createKerasTimeDistributed """ - def __init__(self, layer, input_shape=None, bigdl_type="float"): - super(TimeDistributed, self).__init__(None, bigdl_type, + def __init__(self, layer, input_shape=None, **kwargs): + super(TimeDistributed, self).__init__(None, layer, - list(input_shape) if input_shape else None) + list(input_shape) if input_shape else None, + **kwargs) class Bidirectional(KerasLayer): @@ -1913,13 +2104,15 @@ class Bidirectional(KerasLayer): merge_mode: Mode by which outputs of the forward and backward RNNs will be combined. Must be one of: 'sum', 'mul', 'concat', 'ave'. Default is 'concat'. input_shape: A shape tuple, not including batch. + name: String to set the name of the wrapper. If not specified, its name will by default to be a generated string. - >>> bidiretional = Bidirectional(LSTM(10, return_sequences=True), input_shape=(12, 16)) + >>> bidiretional = Bidirectional(LSTM(10, return_sequences=True), input_shape=(12, 16), name="bidirectionallstm") creating: createKerasLSTM creating: createKerasBidirectional """ - def __init__(self, layer, merge_mode="concat", input_shape=None, bigdl_type="float"): - super(Bidirectional, self).__init__(None, bigdl_type, + def __init__(self, layer, merge_mode="concat", input_shape=None, **kwargs): + super(Bidirectional, self).__init__(None, layer, merge_mode, - list(input_shape) if input_shape else None) \ No newline at end of file + list(input_shape) if input_shape else None, + **kwargs) diff --git a/python/dllib/src/bigdl/dllib/bigdlkeras/layers/topology.py b/python/dllib/src/bigdl/dllib/bigdlkeras/layers/topology.py index 1e7867901cc..4caca14278f 100644 --- a/python/dllib/src/bigdl/dllib/bigdlkeras/layers/topology.py +++ b/python/dllib/src/bigdl/dllib/bigdlkeras/layers/topology.py @@ -135,13 +135,16 @@ def predict(self, x, distributed=True): class Sequential(KerasModel): """ - Container for a Sequential model. + Container for a sequential model. - >>> sequential = Sequential() + # Arguments + name: String to specify the name of the sequential model. Default is None. + + >>> sequential = Sequential(name="seq1") creating: createKerasSequential """ - def __init__(self, bigdl_type="float"): - super(Sequential, self).__init__(None, bigdl_type=bigdl_type) + def __init__(self, **kwargs): + super(Sequential, self).__init__(None, **kwargs) def add(self, model): self.value.add(model.value) @@ -149,7 +152,16 @@ def add(self, model): class Model(KerasModel): - def __init__(self, input, output, bigdl_type="float"): - super(Model, self).__init__(None, bigdl_type, + """ + Container for a graph model. + + # Arguments + input: An input node or a list of input nodes. + output: An output node or a list of output nodes. + name: String to specify the name of the graph model. Default is None. + """ + def __init__(self, input, output, **kwargs): + super(Model, self).__init__(None, to_list(input), - to_list(output)) \ No newline at end of file + to_list(output), + **kwargs) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 1224a23efdc..f9f4031f835 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -78,7 +78,9 @@ def __init__(self, jvalue, bigdl_type, *args): def set_running_mean(self, running_mean): """ - :param running_mean: a ndarray + Set the running mean of the layer. + Only use this method for a BatchNormalization layer. + :param running_mean: a Numpy array. """ callBigDlFunc(self.bigdl_type, "setRunningMean", self.value, JTensor.from_ndarray(running_mean)) @@ -86,7 +88,9 @@ def set_running_mean(self, running_mean): def set_running_std(self, running_std): """ - :param running_mean: a ndarray + Set the running variance of the layer. + Only use this method for a BatchNormalization layer. + :param running_std: a Numpy array. """ callBigDlFunc(self.bigdl_type, "setRunningStd", self.value, JTensor.from_ndarray(running_std)) From 99182567cd685df648cfa863f30b2b60cc6017c9 Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Fri, 16 Mar 2018 15:27:46 +0800 Subject: [PATCH 605/823] Fix reload model in python (#2382) * fix * fix default --- python/dllib/test/bigdl/keras/test_backend.py | 5 ++ .../test/bigdl/test_simple_integration.py | 48 +++++++++++++++++++ 2 files changed, 53 insertions(+) diff --git a/python/dllib/test/bigdl/keras/test_backend.py b/python/dllib/test/bigdl/keras/test_backend.py index d855981acb9..96fddbb927a 100644 --- a/python/dllib/test/bigdl/keras/test_backend.py +++ b/python/dllib/test/bigdl/keras/test_backend.py @@ -37,6 +37,11 @@ def assert_model(self, input_data, kmodel, rtol=1e-5, atol=1e-5): self.assert_allclose(keras_output, bigdl_output, rtol=rtol, atol=atol) + def test_lenet_local_predict(self): + kmodel, X_train, y_train = TestModels.kmodel_seq_lenet_mnist() + model = with_bigdl_backend(kmodel) + model.predict(X_train) + def test_lenet_local(self): kmodel, X_train, y_train = TestModels.kmodel_seq_lenet_mnist() self.modelTest(X_train, kmodel, dump_weights=True) diff --git a/python/dllib/test/bigdl/test_simple_integration.py b/python/dllib/test/bigdl/test_simple_integration.py index 6e80fc05e45..f9ba1acc9c9 100644 --- a/python/dllib/test/bigdl/test_simple_integration.py +++ b/python/dllib/test/bigdl/test_simple_integration.py @@ -73,8 +73,56 @@ def grad_update(mlp, x, y, criterion, learning_rate): print(cadd.get_weights()[0]) assert_allclose(cadd.get_weights()[0], np.array([1, 2, 3, 4, 5]).reshape((5, 1)), + rtol=1.e-1) + def test_load_keras_model_of(self): + from bigdl.nn.keras.topology import Model as KModel + from bigdl.nn.keras.layer import Input as KInput + from bigdl.nn.keras.layer import Dense + + input = KInput(shape=[2, 3]) + fc1 = Dense(2)(input) + model = KModel(input, fc1) + tmp_path = tempfile.mktemp() + model.save(tmp_path, True) + model_loaded = KModel.load(tmp_path) + assert "bigdl.nn.keras.topology.Model" in str(type(model_loaded)) + assert len(model_loaded.layers) == 2 + + def test_load_keras_seq_of(self): + from bigdl.nn.keras.topology import Sequential as KSequential + from bigdl.nn.keras.layer import Dense + + model = KSequential() + fc1 = Dense(2, input_shape=[2, 3]) + model.add(fc1) + tmp_path = tempfile.mktemp() + model.save(tmp_path, True) + model_loaded = KSequential.load(tmp_path) + assert "bigdl.nn.keras.topology.Sequential" in str(type(model_loaded)) + assert len(model_loaded.layers) == 1 + + def test_load_model_of(self): + input = Input() + fc1 = Linear(4, 2)(input) + model = Model(input, fc1) + tmp_path = tempfile.mktemp() + model.save(tmp_path, True) + model_loaded = Model.load(tmp_path) + assert "Model" in str(type(model_loaded)) + assert len(model_loaded.layers) == 2 + + def test_load_sequential_of(self): + fc1 = Linear(4, 2) + model = Sequential() + model.add(fc1) + tmp_path = tempfile.mktemp() + model.save(tmp_path, True) + model_loaded = Model.load(tmp_path) + assert "Sequential" in str(type(model_loaded)) + assert len(model_loaded.layers) == 1 + def test_load_model(self): fc1 = Linear(4, 2) fc1.set_weights([np.ones((4, 2)), np.ones((2,))]) From d493c5ee216deb9e950520b8d0a760de3b2b3d08 Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Fri, 16 Mar 2018 15:27:46 +0800 Subject: [PATCH 606/823] Fix reload model in python (#2382) * fix * fix default --- .../src/bigdl/dllib/bigdlkeras/backend.py | 5 +- .../bigdl/dllib/bigdlkeras/layers/layer.py | 2 +- .../bigdl/dllib/bigdlkeras/layers/topology.py | 33 ++++++- python/dllib/src/bigdl/dllib/nn/layer.py | 92 +++++++++++++++---- 4 files changed, 106 insertions(+), 26 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/bigdlkeras/backend.py b/python/dllib/src/bigdl/dllib/bigdlkeras/backend.py index 8b250484626..4c08a792c7b 100644 --- a/python/dllib/src/bigdl/dllib/bigdlkeras/backend.py +++ b/python/dllib/src/bigdl/dllib/bigdlkeras/backend.py @@ -25,8 +25,9 @@ def __init__(self, kmodel): show_bigdl_info_logs() self.bmodel = DefinitionLoader.from_kmodel(kmodel) WeightLoader.load_weights_from_kmodel(self.bmodel, kmodel) # share the same weight. - self.criterion = OptimConverter.to_bigdl_criterion(kmodel.loss) - self.optim_method = OptimConverter.to_bigdl_optim_method(kmodel.optimizer) + self.criterion = OptimConverter.to_bigdl_criterion(kmodel.loss) if kmodel.loss else None + self.optim_method =\ + OptimConverter.to_bigdl_optim_method(kmodel.optimizer) if kmodel.optimizer else None self.metrics = OptimConverter.to_bigdl_metrics(kmodel.metrics) if kmodel.metrics else None def evaluate(self, x, y, batch_size=32, sample_weight=None, is_distributed=False): diff --git a/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py b/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py index 8b84dc0cdd3..6a0eeb916cf 100644 --- a/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py +++ b/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py @@ -16,7 +16,7 @@ import sys -from bigdl.nn.layer import Layer, Node +from bigdl.nn.layer import Layer, Node, SharedStaticUtils, Container from bigdl.util.common import callBigDlFunc, JTensor, JavaValue if sys.version >= '3': diff --git a/python/dllib/src/bigdl/dllib/bigdlkeras/layers/topology.py b/python/dllib/src/bigdl/dllib/bigdlkeras/layers/topology.py index 4caca14278f..dccaafb9c18 100644 --- a/python/dllib/src/bigdl/dllib/bigdlkeras/layers/topology.py +++ b/python/dllib/src/bigdl/dllib/bigdlkeras/layers/topology.py @@ -19,8 +19,10 @@ from bigdl.keras.optimization import OptimConverter import multiprocessing +from bigdl.nn.layer import SharedStaticUtils, Container -class KerasModel(KerasLayer): + +class KerasModel(KerasLayer, Container, SharedStaticUtils): def compile(self, optimizer, loss, metrics=None): """ Configures the learning process. Must be called before fit. @@ -143,8 +145,19 @@ class Sequential(KerasModel): >>> sequential = Sequential(name="seq1") creating: createKerasSequential """ - def __init__(self, **kwargs): - super(Sequential, self).__init__(None, **kwargs) + def __init__(self, jvalue=None, **kwargs): + super(Sequential, self).__init__(jvalue, **kwargs) + + @staticmethod + def from_jvalue(jvalue, bigdl_type="float"): + """ + Create a Python Model base on the given java value + :param jvalue: Java object create by Py4j + :return: A Python Model + """ + model = Sequential(jvalue=jvalue) + model.value = jvalue + return model def add(self, model): self.value.add(model.value) @@ -160,8 +173,18 @@ class Model(KerasModel): output: An output node or a list of output nodes. name: String to specify the name of the graph model. Default is None. """ - def __init__(self, input, output, **kwargs): - super(Model, self).__init__(None, + def __init__(self, input, output, jvalue=None, **kwargs): + super(Model, self).__init__(jvalue, to_list(input), to_list(output), **kwargs) + @staticmethod + def from_jvalue(jvalue, bigdl_type="float"): + """ + Create a Python Model base on the given java value + :param jvalue: Java object create by Py4j + :return: A Python Model + """ + model = Model([], [], jvalue=jvalue) + model.value = jvalue + return model diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index f9f4031f835..5a20823efe1 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -16,6 +16,7 @@ import sys +import importlib import numpy as np import six @@ -60,7 +61,60 @@ def remove_next_edges(self): callJavaFunc(self.value.removeNextEdges) -class Layer(JavaValue): +class SharedStaticUtils(): + + @staticmethod + def load(path, bigdl_type="float"): + """ + Load a pre-trained Bigdl model. + + :param path: The path containing the pre-trained model. + :return: A pre-trained model. + """ + jmodel = callBigDlFunc(bigdl_type, "loadBigDL", path) + return Layer.of(jmodel) + + + @staticmethod + def of(jvalue, bigdl_type="float"): + """ + Create a Python Layer base on the given java value and the real type. + :param jvalue: Java object create by Py4j + :return: A Python Layer + """ + def get_py_name(jclass_name): + if jclass_name == "StaticGraph" or jclass_name == "DynamicGraph": + return "Model" + elif jclass_name == "Input": + return "Layer" + else: + return jclass_name + + jname = callBigDlFunc(bigdl_type, + "getRealClassNameOfJValue", + jvalue) + + jpackage_name = ".".join(jname.split(".")[:-1]) + pclass_name = get_py_name(jname.split(".")[-1]) + + if "com.intel.analytics.bigdl.nn.keras.Model" == jname or \ + "com.intel.analytics.bigdl.nn.keras.Sequential" == jname: + base_module = importlib.import_module('bigdl.nn.keras.topology') + elif "com.intel.analytics.bigdl.nn.keras" == jpackage_name: + base_module = importlib.import_module('bigdl.nn.keras.layer') + else: + base_module = importlib.import_module('bigdl.nn.layer') + + realClassName = "Layer" # The top base class + if pclass_name in dir(base_module): + realClassName = pclass_name + module = getattr(base_module, realClassName) + jvalue_creator = getattr(module, "from_jvalue") + model = jvalue_creator(jvalue, bigdl_type) + return model + + +class Layer(JavaValue, SharedStaticUtils): """ Layer is the basic component of a neural network and it's also the base class of layers. @@ -118,14 +172,15 @@ def __call__(self, x=None): self, to_list(x))) - @classmethod - def of(cls, jvalue, bigdl_type="float"): + @staticmethod + def from_jvalue(jvalue, bigdl_type="float"): """ - Create a Python Layer base on the given java value + Create a Python Model base on the given java value :param jvalue: Java object create by Py4j - :return: A Python Layer + :return: A Python Model """ - model = Layer(jvalue, bigdl_type) + model = Layer(jvalue=jvalue, bigdl_type=bigdl_type) + model.value = jvalue return model def set_name(self, name): @@ -686,16 +741,6 @@ def from_jvalue(jvalue, bigdl_type="float"): def __str__(self): return "->".join(self.layers()) - @staticmethod - def load(path, bigdl_type="float"): - """ - Load a pre-trained Bigdl model. - - :param path: The path containing the pre-trained model. - :return: A pre-trained model. - """ - jmodel = callBigDlFunc(bigdl_type, "loadBigDL", path) - return Layer.of(jmodel) @staticmethod def loadModel(modelPath, weightPath =None, bigdl_type="float"): @@ -1063,8 +1108,19 @@ class Sequential(Container): ''' - def __init__(self, bigdl_type="float"): - super(Sequential, self).__init__(None, bigdl_type) + def __init__(self, jvalue=None, bigdl_type="float"): + super(Sequential, self).__init__(jvalue, bigdl_type) + + @staticmethod + def from_jvalue(jvalue, bigdl_type="float"): + """ + Create a Python Model base on the given java value + :param jvalue: Java object create by Py4j + :return: A Python Model + """ + model = Sequential(jvalue=jvalue) + model.value = jvalue + return model class TemporalConvolution(Layer): From eac66b50c21d5d60cce70bf812549bdd9d3b4b82 Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Fri, 16 Mar 2018 15:27:46 +0800 Subject: [PATCH 607/823] Fix reload model in python (#2382) * fix * fix default --- .../src/bigdl/dllib/bigdlkeras/backend.py | 5 +- .../bigdl/dllib/bigdlkeras/layers/layer.py | 2 +- .../bigdl/dllib/bigdlkeras/layers/topology.py | 33 ++++++- python/dllib/src/bigdl/dllib/nn/layer.py | 92 +++++++++++++++---- 4 files changed, 106 insertions(+), 26 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/bigdlkeras/backend.py b/python/dllib/src/bigdl/dllib/bigdlkeras/backend.py index 8b250484626..4c08a792c7b 100644 --- a/python/dllib/src/bigdl/dllib/bigdlkeras/backend.py +++ b/python/dllib/src/bigdl/dllib/bigdlkeras/backend.py @@ -25,8 +25,9 @@ def __init__(self, kmodel): show_bigdl_info_logs() self.bmodel = DefinitionLoader.from_kmodel(kmodel) WeightLoader.load_weights_from_kmodel(self.bmodel, kmodel) # share the same weight. - self.criterion = OptimConverter.to_bigdl_criterion(kmodel.loss) - self.optim_method = OptimConverter.to_bigdl_optim_method(kmodel.optimizer) + self.criterion = OptimConverter.to_bigdl_criterion(kmodel.loss) if kmodel.loss else None + self.optim_method =\ + OptimConverter.to_bigdl_optim_method(kmodel.optimizer) if kmodel.optimizer else None self.metrics = OptimConverter.to_bigdl_metrics(kmodel.metrics) if kmodel.metrics else None def evaluate(self, x, y, batch_size=32, sample_weight=None, is_distributed=False): diff --git a/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py b/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py index 8b84dc0cdd3..6a0eeb916cf 100644 --- a/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py +++ b/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py @@ -16,7 +16,7 @@ import sys -from bigdl.nn.layer import Layer, Node +from bigdl.nn.layer import Layer, Node, SharedStaticUtils, Container from bigdl.util.common import callBigDlFunc, JTensor, JavaValue if sys.version >= '3': diff --git a/python/dllib/src/bigdl/dllib/bigdlkeras/layers/topology.py b/python/dllib/src/bigdl/dllib/bigdlkeras/layers/topology.py index 4caca14278f..dccaafb9c18 100644 --- a/python/dllib/src/bigdl/dllib/bigdlkeras/layers/topology.py +++ b/python/dllib/src/bigdl/dllib/bigdlkeras/layers/topology.py @@ -19,8 +19,10 @@ from bigdl.keras.optimization import OptimConverter import multiprocessing +from bigdl.nn.layer import SharedStaticUtils, Container -class KerasModel(KerasLayer): + +class KerasModel(KerasLayer, Container, SharedStaticUtils): def compile(self, optimizer, loss, metrics=None): """ Configures the learning process. Must be called before fit. @@ -143,8 +145,19 @@ class Sequential(KerasModel): >>> sequential = Sequential(name="seq1") creating: createKerasSequential """ - def __init__(self, **kwargs): - super(Sequential, self).__init__(None, **kwargs) + def __init__(self, jvalue=None, **kwargs): + super(Sequential, self).__init__(jvalue, **kwargs) + + @staticmethod + def from_jvalue(jvalue, bigdl_type="float"): + """ + Create a Python Model base on the given java value + :param jvalue: Java object create by Py4j + :return: A Python Model + """ + model = Sequential(jvalue=jvalue) + model.value = jvalue + return model def add(self, model): self.value.add(model.value) @@ -160,8 +173,18 @@ class Model(KerasModel): output: An output node or a list of output nodes. name: String to specify the name of the graph model. Default is None. """ - def __init__(self, input, output, **kwargs): - super(Model, self).__init__(None, + def __init__(self, input, output, jvalue=None, **kwargs): + super(Model, self).__init__(jvalue, to_list(input), to_list(output), **kwargs) + @staticmethod + def from_jvalue(jvalue, bigdl_type="float"): + """ + Create a Python Model base on the given java value + :param jvalue: Java object create by Py4j + :return: A Python Model + """ + model = Model([], [], jvalue=jvalue) + model.value = jvalue + return model diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index f9f4031f835..5a20823efe1 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -16,6 +16,7 @@ import sys +import importlib import numpy as np import six @@ -60,7 +61,60 @@ def remove_next_edges(self): callJavaFunc(self.value.removeNextEdges) -class Layer(JavaValue): +class SharedStaticUtils(): + + @staticmethod + def load(path, bigdl_type="float"): + """ + Load a pre-trained Bigdl model. + + :param path: The path containing the pre-trained model. + :return: A pre-trained model. + """ + jmodel = callBigDlFunc(bigdl_type, "loadBigDL", path) + return Layer.of(jmodel) + + + @staticmethod + def of(jvalue, bigdl_type="float"): + """ + Create a Python Layer base on the given java value and the real type. + :param jvalue: Java object create by Py4j + :return: A Python Layer + """ + def get_py_name(jclass_name): + if jclass_name == "StaticGraph" or jclass_name == "DynamicGraph": + return "Model" + elif jclass_name == "Input": + return "Layer" + else: + return jclass_name + + jname = callBigDlFunc(bigdl_type, + "getRealClassNameOfJValue", + jvalue) + + jpackage_name = ".".join(jname.split(".")[:-1]) + pclass_name = get_py_name(jname.split(".")[-1]) + + if "com.intel.analytics.bigdl.nn.keras.Model" == jname or \ + "com.intel.analytics.bigdl.nn.keras.Sequential" == jname: + base_module = importlib.import_module('bigdl.nn.keras.topology') + elif "com.intel.analytics.bigdl.nn.keras" == jpackage_name: + base_module = importlib.import_module('bigdl.nn.keras.layer') + else: + base_module = importlib.import_module('bigdl.nn.layer') + + realClassName = "Layer" # The top base class + if pclass_name in dir(base_module): + realClassName = pclass_name + module = getattr(base_module, realClassName) + jvalue_creator = getattr(module, "from_jvalue") + model = jvalue_creator(jvalue, bigdl_type) + return model + + +class Layer(JavaValue, SharedStaticUtils): """ Layer is the basic component of a neural network and it's also the base class of layers. @@ -118,14 +172,15 @@ def __call__(self, x=None): self, to_list(x))) - @classmethod - def of(cls, jvalue, bigdl_type="float"): + @staticmethod + def from_jvalue(jvalue, bigdl_type="float"): """ - Create a Python Layer base on the given java value + Create a Python Model base on the given java value :param jvalue: Java object create by Py4j - :return: A Python Layer + :return: A Python Model """ - model = Layer(jvalue, bigdl_type) + model = Layer(jvalue=jvalue, bigdl_type=bigdl_type) + model.value = jvalue return model def set_name(self, name): @@ -686,16 +741,6 @@ def from_jvalue(jvalue, bigdl_type="float"): def __str__(self): return "->".join(self.layers()) - @staticmethod - def load(path, bigdl_type="float"): - """ - Load a pre-trained Bigdl model. - - :param path: The path containing the pre-trained model. - :return: A pre-trained model. - """ - jmodel = callBigDlFunc(bigdl_type, "loadBigDL", path) - return Layer.of(jmodel) @staticmethod def loadModel(modelPath, weightPath =None, bigdl_type="float"): @@ -1063,8 +1108,19 @@ class Sequential(Container): ''' - def __init__(self, bigdl_type="float"): - super(Sequential, self).__init__(None, bigdl_type) + def __init__(self, jvalue=None, bigdl_type="float"): + super(Sequential, self).__init__(jvalue, bigdl_type) + + @staticmethod + def from_jvalue(jvalue, bigdl_type="float"): + """ + Create a Python Model base on the given java value + :param jvalue: Java object create by Py4j + :return: A Python Model + """ + model = Sequential(jvalue=jvalue) + model.value = jvalue + return model class TemporalConvolution(Layer): From 334882e42952834d66332b16f4cf5d75f7628c18 Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Tue, 20 Mar 2018 11:55:07 +0800 Subject: [PATCH 608/823] Public getInputShape and getOutputShape (#2401) * open getInputShape and getOutputShape * clean * remove logger * fix --- python/dllib/test/dev/release/release.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/dllib/test/dev/release/release.sh b/python/dllib/test/dev/release/release.sh index caba1f7fc1c..306a23c41c0 100755 --- a/python/dllib/test/dev/release/release.sh +++ b/python/dllib/test/dev/release/release.sh @@ -36,7 +36,7 @@ input_version=$4 bigdl_version=$(python -c "exec(open('$BIGDL_DIR/pyspark/bigdl/version.py').read()); print(__version__)") if [ "$input_version" != "$bigdl_version" ]; then - echo "Not the proposed version" + echo "Not the proposed version: $bigdl_version" exit -1 fi From 4b6ca80ddb6daa59a92af3df1cd36931034cc360 Mon Sep 17 00:00:00 2001 From: jenniew Date: Mon, 12 Feb 2018 17:41:41 -0800 Subject: [PATCH 609/823] add inception python model --- .../bigdl/dllib/models/inception/README.md | 115 +++++++ .../bigdl/dllib/models/inception/inception.py | 313 ++++++++++++++++++ 2 files changed, 428 insertions(+) create mode 100644 python/dllib/src/bigdl/dllib/models/inception/README.md create mode 100644 python/dllib/src/bigdl/dllib/models/inception/inception.py diff --git a/python/dllib/src/bigdl/dllib/models/inception/README.md b/python/dllib/src/bigdl/dllib/models/inception/README.md new file mode 100644 index 00000000000..c334b2b33fd --- /dev/null +++ b/python/dllib/src/bigdl/dllib/models/inception/README.md @@ -0,0 +1,115 @@ +# Inception Model on Imagenet +This example demonstrates how to use BigDL to train [Inception v1](https://arxiv.org/abs/1409.4842) architecture on the [ImageNet](http://image-net.org/index) data. +## Get the JAR +You can build one by refer to the +[Build Page](https://github.com/intel-analytics/BigDL/wiki/Build-Page) from the source code. We +will release a pre-build package soon. + +## Prepare the data +You can download imagenet-2012 data from . + +After you download the files(**ILSVRC2012_img_train.tar** and **ILSVRC2012_img_val.tar**), +run the follow commands to prepare the data. + +```bash +mkdir train +mv ILSVRC2012_img_train.tar train/ +cd train +tar -xvf ILSVRC2012_img_train.tar +rm -f ILSVRC2012_img_train.tar +find . -name "*.tar" | while read CLASS_NAME ; do mkdir -p "${CLASS_NAME%.tar}"; tar -xvf "${CLASS_NAME}" -C "${CLASS_NAME%.tar}"; done +rm *.tar +cd ../ +mkdir val +mv ILSVRC2012_img_val.tar val/ +cd val +tar -xvf ILSVRC2012_img_val.tar +cat classes.lst | while read CLASS_NAME; do mkdir -p ${CLASS_NAME}; done +cat img_class.lst | while read PARAM; do mv ${PARAM/ n[0-9]*/} ${PARAM/ILSVRC*JPEG /}; done +rm ILSVRC2012_img_val.tar +``` + +Now all the images belong to the same category are moved to the same folder. + +This command will transform the images into hadoop sequence files, which are +more suitable for a distributed training. + +```bash +java -cp bigdl_folder/lib/bigdl-VERSION-jar-with-dependencies-and-spark.jar com.intel.analytics.bigdl.models.utils.ImageNetSeqFileGenerator -r -f imagenet_folder -o output_folder -p cores_number +``` + +It will generate the hadoop sequence files in the output folder. + +## Train the Model +* Spark standalone, example command +``` +BigDL_HOME=... +SPARK_HOME=... +PYTHON_API_PATH=${BigDL_HOME}/dist/lib/bigdl-VERSION-python-api.zip +BigDL_JAR_PATH=${BigDL_HOME}/dist/lib/bigdl-VERSION-jar-with-dependencies.jar +PYTHONPATH=${PYTHON_API_PATH}:$PYTHONPATH +source ${BigDL_HOME}/dist/bin/bigdl.sh + +${SPARK_HOME}/bin/spark-submit \ +--master spark://... \ +--executor-memory 150g \ +--driver-memory 100g \ +--executor-cores 28 \ +--total-executor-cores 448 \ +--jars ${BigDL_JAR_PATH} \ +--py-files ${PYTHON_API_PATH} \ +${BigDL_HOME}/pyspark/dl/models/inception/inception.py \ +-f hdfs://... \ +--batchSize 1792 \ +--learningRate 0.0898 \ +--weightDecay 0.0001 \ +--checkpointIteration 6200 \ +-i 62000 \ +--checkpoint /models/inception +``` +* Spark yarn client mode, example command +``` +BigDL_HOME=... +SPARK_HOME=... +PYTHON_API_PATH=${BigDL_HOME}/dist/lib/bigdl-VERSION-python-api.zip +BigDL_JAR_PATH=${BigDL_HOME}/dist/lib/bigdl-VERSION-jar-with-dependencies.jar +PYTHONPATH=${PYTHON_API_PATH}:$PYTHONPATH +source ${BigDL_HOME}/dist/bin/bigdl.sh + +${SPARK_HOME}/bin/spark-submit \ +--master yarn \ +--deploy-mode client \ +--executor-memory 150g \ +--driver-memory 100g \ +--executor-cores 28 \ +--num-executors 16 \ +--properties-file ${BigDL_HOME}/dist/conf/spark-bigdl.conf \ +--jars ${BigDL_JAR_PATH} \ +--conf spark.driver.extraClassPath=${BigDL_JAR_PATH} \ +--conf spark.executor.extraClassPath=bigdl-VERSION-jar-with-dependencies.jar \ +--py-files ${PYTHON_API_PATH} \ +${BigDL_HOME}/pyspark/dl/models/inception/inception.py \ +-f hdfs://... \ +--batchSize 1792 \ +--learningRate 0.0898 \ +--weightDecay 0.0001 \ +--checkpointIteration 6200 \ +-i 62000 \ +--checkpoint /models/inception + +``` + +In the above commands +* -f: where you put your ImageNet data, it should be a hdfs folder +* --checkpoint: Where you cache the model/train_state snapshot. You should input a folder and +make sure the folder is created when you run this example. The model snapshot will be named as +model.#iteration_number, and train state will be named as state.#iteration_number. Note that if +there are some files already exist in the folder, the old file will not be overwrite for the +safety of your model files. +* --batchSize: The mini-batch size. It is expected that the mini-batch size is a multiple of node_number * +core_number. In this example, node_number is 1 and the mini-batch size is suggested to be set to core_number * 4 +* --learningRate: inital learning rate. Note in this example, we use a Poly learning rate decay +policy. +* --weightDecay: weight decay. +* -i: max iteration +* --checkpointIteration: the checkpoint interval in iteration. \ No newline at end of file diff --git a/python/dllib/src/bigdl/dllib/models/inception/inception.py b/python/dllib/src/bigdl/dllib/models/inception/inception.py new file mode 100644 index 00000000000..643c268f813 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/models/inception/inception.py @@ -0,0 +1,313 @@ +from bigdl.nn.layer import * +from optparse import OptionParser +from bigdl.nn.criterion import * +from bigdl.nn.initialization_method import * +from bigdl.optim.optimizer import * +from bigdl.transform.vision.image import * + + +def scala_T(input_T): + if type(input_T) is list: + # insert into index 0 spot, such that the real data starts from index 1 + temp = [0] + temp.extend(input_T) + return dict(enumerate(temp)) + # if dictionary, return it back + return input_T + + +def Inception_Layer_v1(input_size, config, name_prefix=""): + concat = Concat(2) + conv1 = Sequential() + conv1.add( + SpatialConvolution(input_size, + config[1][1], + 1, 1, 1, 1).set_init_method(weight_init_method=Xavier()) + .set_name(name_prefix + "1x1")) + conv1.add(ReLU(True).set_name(name_prefix + "relu_1x1")) + concat.add(conv1) + conv3 = Sequential() + conv3.add(SpatialConvolution(input_size, config[2][1], 1, 1, 1, 1).set_init_method(weight_init_method=Xavier()) + .set_name(name_prefix + "3x3_reduce")) + conv3.add(ReLU(True).set_name(name_prefix + "relu_3x3_reduce")) + conv3.add(SpatialConvolution(config[2][1], config[2][2], + 3, 3, 1, 1, 1, 1).set_init_method(weight_init_method=Xavier()) + .set_name(name_prefix + "3x3")) + conv3.add(ReLU(True).set_name(name_prefix + "relu_3x3")) + concat.add(conv3) + conv5 = Sequential() + conv5.add(SpatialConvolution(input_size, + config[3][1], 1, 1, 1, 1).set_init_method(weight_init_method=Xavier()) + .set_name(name_prefix + "5x5_reduce")) + conv5.add(ReLU(True).set_name(name_prefix + "relu_5x5_reduce")) + conv5.add(SpatialConvolution(config[3][1], + config[3][2], 5, 5, 1, 1, 2, 2).set_init_method(weight_init_method=Xavier()) + .set_name(name_prefix + "5x5")) + conv5.add(ReLU(True).set_name(name_prefix + "relu_5x5")) + concat.add(conv5) + pool = Sequential() + pool.add(SpatialMaxPooling(3, 3, 1, 1, 1, 1, + to_ceil=True).set_name(name_prefix + "pool")) + pool.add(SpatialConvolution(input_size, config[4][1], 1, 1, 1, 1).set_init_method(weight_init_method=Xavier()) + .set_name(name_prefix + "pool_proj")) + pool.add(ReLU(True).set_name(name_prefix + "relu_pool_proj")) + concat.add(pool).set_name(name_prefix + "output") + return concat + + +def Inception_v1_NoAuxClassifier(class_num): + model = Sequential() + model.add(SpatialConvolution(3, 64, 7, 7, 2, 2, 3, 3, 1, False) + .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) + .set_name("conv1/7x7_s2")) + model.add(ReLU(True).set_name("conv1/relu_7x7")) + model.add(SpatialMaxPooling(3, 3, 2, 2, to_ceil=True).set_name("pool1/3x3_s2")) + model.add(SpatialCrossMapLRN(5, 0.0001, 0.75).set_name("pool1/norm1")) + model.add(SpatialConvolution(64, 64, 1, 1, 1, 1).set_init_method(weight_init_method=Xavier()) + .set_name("conv2/3x3_reduce")) + model.add(ReLU(True).set_name("conv2/relu_3x3_reduce")) + model.add(SpatialConvolution(64, 192, 3, 3, 1, 1, 1, 1).set_init_method(weight_init_method=Xavier()) + .set_name("conv2/3x3")) + model.add(ReLU(True).set_name("conv2/relu_3x3")) + model.add(SpatialCrossMapLRN(5, 0.0001, 0.75).set_name("conv2/norm2")) + model.add(SpatialMaxPooling(3, 3, 2, 2, to_ceil=True).set_name("pool2/3x3_s2")) + model.add(Inception_Layer_v1(192, scala_T([scala_T([64]), scala_T( + [96, 128]), scala_T([16, 32]), scala_T([32])]), "inception_3a/")) + model.add(Inception_Layer_v1(256, scala_T([scala_T([128]), scala_T( + [128, 192]), scala_T([32, 96]), scala_T([64])]), "inception_3b/")) + model.add(SpatialMaxPooling(3, 3, 2, 2, to_ceil=True)) + model.add(Inception_Layer_v1(480, scala_T([scala_T([192]), scala_T( + [96, 208]), scala_T([16, 48]), scala_T([64])]), "inception_4a/")) + model.add(Inception_Layer_v1(512, scala_T([scala_T([160]), scala_T( + [112, 224]), scala_T([24, 64]), scala_T([64])]), "inception_4b/")) + model.add(Inception_Layer_v1(512, scala_T([scala_T([128]), scala_T( + [128, 256]), scala_T([24, 64]), scala_T([64])]), "inception_4c/")) + model.add(Inception_Layer_v1(512, scala_T([scala_T([112]), scala_T( + [144, 288]), scala_T([32, 64]), scala_T([64])]), "inception_4d/")) + model.add(Inception_Layer_v1(528, scala_T([scala_T([256]), scala_T( + [160, 320]), scala_T([32, 128]), scala_T([128])]), "inception_4e/")) + model.add(SpatialMaxPooling(3, 3, 2, 2, to_ceil=True)) + model.add(Inception_Layer_v1(832, scala_T([scala_T([256]), scala_T( + [160, 320]), scala_T([32, 128]), scala_T([128])]), "inception_5a/")) + model.add(Inception_Layer_v1(832, scala_T([scala_T([384]), scala_T( + [192, 384]), scala_T([48, 128]), scala_T([128])]), "inception_5b/")) + model.add(SpatialAveragePooling(7, 7, 1, 1).set_name("pool5/7x7_s1")) + model.add(Dropout(0.4).set_name("pool5/drop_7x7_s1")) + model.add(View([1024], num_input_dims=3)) + model.add(Linear(1024, class_num).set_init_method(weight_init_method=Xavier()).set_name("loss3/classifier")) + model.add(LogSoftMax().set_name("loss3/loss3")) + model.reset() + return model + + +def Inception_v1(class_num): + feature1 = Sequential() + feature1.add(SpatialConvolution(3, 64, 7, 7, 2, 2, 3, 3, 1, False).set_init_method(weight_init_method=Xavier()) + .set_name("conv1/7x7_s2")) + feature1.add(ReLU(True).set_name("conv1/relu_7x7")) + feature1.add( + SpatialMaxPooling(3, 3, 2, 2, to_ceil=True) + .set_name("pool1/3x3_s2")) + feature1.add(SpatialCrossMapLRN(5, 0.0001, 0.75) + .set_name("pool1/norm1")) + feature1.add(SpatialConvolution(64, 64, 1, 1, 1, 1).set_init_method(weight_init_method=Xavier()) + .set_name("conv2/3x3_reduce")) + feature1.add(ReLU(True).set_name("conv2/relu_3x3_reduce")) + feature1.add(SpatialConvolution(64, 192, 3, 3, 1, + 1, 1, 1).set_init_method(weight_init_method=Xavier()) + .set_name("conv2/3x3")) + feature1.add(ReLU(True).set_name("conv2/relu_3x3")) + feature1.add(SpatialCrossMapLRN(5, 0.0001, 0.75).set_name("conv2/norm2")) + feature1.add( + SpatialMaxPooling(3, 3, 2, 2, to_ceil=True).set_name("pool2/3x3_s2")) + feature1.add(Inception_Layer_v1(192, + scala_T([scala_T([64]), scala_T([96, 128]), + scala_T([16, 32]), scala_T([32])]), + "inception_3a/")) + feature1.add(Inception_Layer_v1(256, scala_T([ + scala_T([128]), scala_T([128, 192]), scala_T([32, 96]), scala_T([64])]), + "inception_3b/")) + feature1.add( + SpatialMaxPooling(3, 3, 2, 2, to_ceil=True).set_name("pool3/3x3_s2")) + feature1.add(Inception_Layer_v1(480, scala_T([ + scala_T([192]), scala_T([96, 208]), scala_T([16, 48]), scala_T([64])]), + "inception_4a/")) + + output1 = Sequential() + output1.add(SpatialAveragePooling( + 5, 5, 3, 3, ceil_mode=True).set_name("loss1/ave_pool")) + output1.add( + SpatialConvolution(512, 128, 1, 1, 1, 1).set_name("loss1/conv")) + output1.add(ReLU(True).set_name("loss1/relu_conv")) + output1.add(View([128 * 4 * 4, 3])) + output1.add(Linear(128 * 4 * 4, 1024).set_name("loss1/fc")) + output1.add(ReLU(True).set_name("loss1/relu_fc")) + output1.add(Dropout(0.7).set_name("loss1/drop_fc")) + output1.add(Linear(1024, class_num).set_name("loss1/classifier")) + output1.add(LogSoftMax().set_name("loss1/loss")) + + feature2 = Sequential() + feature2.add(Inception_Layer_v1(512, + scala_T([scala_T([160]), scala_T([112, 224]), + scala_T([24, 64]), scala_T([64])]), + "inception_4b/")) + feature2.add(Inception_Layer_v1(512, + scala_T([scala_T([128]), scala_T([128, 256]), + scala_T([24, 64]), scala_T([64])]), + "inception_4c/")) + feature2.add(Inception_Layer_v1(512, + scala_T([scala_T([112]), scala_T([144, 288]), + scala_T([32, 64]), scala_T([64])]), + "inception_4d/")) + + output2 = Sequential() + output2.add(SpatialAveragePooling(5, 5, 3, 3).set_name("loss2/ave_pool")) + output2.add( + SpatialConvolution(528, 128, 1, 1, 1, 1).set_name("loss2/conv")) + output2.add(ReLU(True).set_name("loss2/relu_conv")) + output2.add(View([128 * 4 * 4, 3])) + output2.add(Linear(128 * 4 * 4, 1024).set_name("loss2/fc")) + output2.add(ReLU(True).set_name("loss2/relu_fc")) + output2.add(Dropout(0.7).set_name("loss2/drop_fc")) + output2.add(Linear(1024, class_num).set_name("loss2/classifier")) + output2.add(LogSoftMax().set_name("loss2/loss")) + + output3 = Sequential() + output3.add(Inception_Layer_v1(528, + scala_T([scala_T([256]), scala_T([160, 320]), + scala_T([32, 128]), scala_T([128])]), + "inception_4e/")) + output3.add(SpatialMaxPooling(3, 3, 2, 2, to_ceil=True).set_name("pool4/3x3_s2")) + output3.add(Inception_Layer_v1(832, + scala_T([scala_T([256]), scala_T([160, 320]), + scala_T([32, 128]), scala_T([128])]), + "inception_5a/")) + output3.add(Inception_Layer_v1(832, + scala_T([scala_T([384]), scala_T([192, 384]), + scala_T([48, 128]), scala_T([128])]), + "inception_5b/")) + output3.add(SpatialAveragePooling(7, 7, 1, 1).set_name("pool5/7x7_s1")) + output3.add(Dropout(0.4).set_name("pool5/drop_7x7_s1")) + output3.add(View([1024, 3])) + output3.add(Linear(1024, class_num).set_init_method(weight_init_method=Xavier()) + .set_name("loss3/classifier")) + output3.add(LogSoftMax().set_name("loss3/loss3")) + + split2 = Concat(2).set_name("split2") + split2.add(output3) + split2.add(output2) + + mainBranch = Sequential() + mainBranch.add(feature2) + mainBranch.add(split2) + + split1 = Concat(2).set_name("split1") + split1.add(mainBranch) + split1.add(output1) + + model = Sequential() + + model.add(feature1) + model.add(split1) + + model.reset() + return model + + +def get_inception_data(url, sc=None, data_type="train"): + path = os.path.join(url, data_type) + return SeqFileFolder.files_to_image_frame(url=path, sc=sc, class_num=1000) + + +def config_option_parser(): + parser = OptionParser() + parser.add_option("-f", "--folder", type=str, dest="folder", default="", + help="url of hdfs folder store the hadoop sequence files") + parser.add_option("--model", type=str, dest="model", default="./inception.model", help="model snapshot location") + parser.add_option("--state", type=str, dest="state", default="", help="state snapshot location") + parser.add_option("--checkpoint", type=str, dest="checkpoint", default="", help="where to cache the model") + parser.add_option("-o", "--overwrite", action="store_true", dest="overwrite", default=False, + help="overwrite checkpoint files") + parser.add_option("-e", "--maxEpoch", type=int, dest="maxEpoch", default=0, help="epoch numbers") + parser.add_option("-i", "--maxIteration", type=int, dest="maxIteration", default=62000, help="iteration numbers") + parser.add_option("-l", "--learningRate", type=float, dest="learningRate", default=0.01, help="learning rate") + parser.add_option("-b", "--batchSize", type=int, dest="batchSize", help="batch size") + parser.add_option("--classNum", type=int, dest="classNum", default=1000, help="class number") + parser.add_option("--weightDecay", type=float, dest="weightDecay", default=0.0001, help="weight decay") + parser.add_option("--checkpointIteration", type=int, dest="checkpointIteration", default=620, + help="checkpoint interval of iterations") + + return parser + + +if __name__ == "__main__": + # parse options + parser = config_option_parser() + (options, args) = parser.parse_args(sys.argv) + if not options.learningRate: + parser.error("-l --learningRate is a mandatory opt") + if not options.batchSize: + parser.error("-b --batchSize is a mandatory opt") + + # init + sparkConf = create_spark_conf().setAppName("inception v1") + sc = get_spark_context(sparkConf) + redire_spark_logs() + show_bigdl_info_logs() + init_engine() + + # build model + inception_model = Inception_v1_NoAuxClassifier(options.classNum) + + image_size = 224 # create dataset + train_transformer = Pipeline([PixelBytesToMat(), + RandomCrop(image_size, image_size), + RandomTransformer(HFlip(), 0.5), + ChannelNormalize(0.485, 0.456, 0.406, 0.229, 0.224, 0.225), + MatToTensor(to_rgb=True), + ImageFrameToSample(input_keys=["imageTensor"], target_keys=["label"]) + ]) + raw_train_data = get_inception_data(options.folder, sc, "train") + train_data = DataSet.image_frame(raw_train_data).transform(train_transformer) + + val_transformer = Pipeline([PixelBytesToMat(), + CenterCrop(image_size, image_size), + RandomTransformer(HFlip(), 0.5), + ChannelNormalize(0.485, 0.456, 0.406, 0.229, 0.224, 0.225), + MatToTensor(to_rgb=True), + ImageFrameToSample(input_keys=["imageTensor"], target_keys=["label"]) + ]) + raw_val_data = get_inception_data(options.folder, sc, "val") + val_data = DataSet.image_frame(raw_val_data).transform(val_transformer) + + if options.maxEpoch: + checkpoint_trigger = EveryEpoch() + test_trigger = EveryEpoch() + end_trigger = MaxEpoch(options.maxEpoch) + else: + checkpoint_trigger = SeveralIteration(options.checkpointIteration) + test_trigger = SeveralIteration(options.checkpointIteration) + end_trigger = MaxIteration(options.maxIteration) + + # Optimizer + optimizer = Optimizer.create( + model=inception_model, + training_set=train_data, + optim_method=SGD(learningrate=options.learningRate, learningrate_decay=options.weightDecay, + momentum=0.9, dampening=0.0), + criterion=ClassNLLCriterion(), + end_trigger=end_trigger, + batch_size=options.batchSize + ) + + if options.checkpoint: + optimizer.set_checkpoint(checkpoint_trigger, options.checkpoint, options.overwrite) + + optimizer.set_validation(trigger=test_trigger, + val_rdd=val_data, + batch_size=options.batchSize, + val_method=[Top1Accuracy(), Top5Accuracy()]) + + trained_model = optimizer.optimize() + + sc.stop() From dae332a6057842a1a31c267bdbdbf799987bc466 Mon Sep 17 00:00:00 2001 From: jenniew Date: Mon, 12 Feb 2018 17:41:41 -0800 Subject: [PATCH 610/823] add inception python model --- .../bigdl/dllib/models/inception/README.md | 115 +++++++ .../bigdl/dllib/models/inception/inception.py | 313 ++++++++++++++++++ 2 files changed, 428 insertions(+) create mode 100644 python/dllib/src/bigdl/dllib/models/inception/README.md create mode 100644 python/dllib/src/bigdl/dllib/models/inception/inception.py diff --git a/python/dllib/src/bigdl/dllib/models/inception/README.md b/python/dllib/src/bigdl/dllib/models/inception/README.md new file mode 100644 index 00000000000..c334b2b33fd --- /dev/null +++ b/python/dllib/src/bigdl/dllib/models/inception/README.md @@ -0,0 +1,115 @@ +# Inception Model on Imagenet +This example demonstrates how to use BigDL to train [Inception v1](https://arxiv.org/abs/1409.4842) architecture on the [ImageNet](http://image-net.org/index) data. +## Get the JAR +You can build one by refer to the +[Build Page](https://github.com/intel-analytics/BigDL/wiki/Build-Page) from the source code. We +will release a pre-build package soon. + +## Prepare the data +You can download imagenet-2012 data from . + +After you download the files(**ILSVRC2012_img_train.tar** and **ILSVRC2012_img_val.tar**), +run the follow commands to prepare the data. + +```bash +mkdir train +mv ILSVRC2012_img_train.tar train/ +cd train +tar -xvf ILSVRC2012_img_train.tar +rm -f ILSVRC2012_img_train.tar +find . -name "*.tar" | while read CLASS_NAME ; do mkdir -p "${CLASS_NAME%.tar}"; tar -xvf "${CLASS_NAME}" -C "${CLASS_NAME%.tar}"; done +rm *.tar +cd ../ +mkdir val +mv ILSVRC2012_img_val.tar val/ +cd val +tar -xvf ILSVRC2012_img_val.tar +cat classes.lst | while read CLASS_NAME; do mkdir -p ${CLASS_NAME}; done +cat img_class.lst | while read PARAM; do mv ${PARAM/ n[0-9]*/} ${PARAM/ILSVRC*JPEG /}; done +rm ILSVRC2012_img_val.tar +``` + +Now all the images belong to the same category are moved to the same folder. + +This command will transform the images into hadoop sequence files, which are +more suitable for a distributed training. + +```bash +java -cp bigdl_folder/lib/bigdl-VERSION-jar-with-dependencies-and-spark.jar com.intel.analytics.bigdl.models.utils.ImageNetSeqFileGenerator -r -f imagenet_folder -o output_folder -p cores_number +``` + +It will generate the hadoop sequence files in the output folder. + +## Train the Model +* Spark standalone, example command +``` +BigDL_HOME=... +SPARK_HOME=... +PYTHON_API_PATH=${BigDL_HOME}/dist/lib/bigdl-VERSION-python-api.zip +BigDL_JAR_PATH=${BigDL_HOME}/dist/lib/bigdl-VERSION-jar-with-dependencies.jar +PYTHONPATH=${PYTHON_API_PATH}:$PYTHONPATH +source ${BigDL_HOME}/dist/bin/bigdl.sh + +${SPARK_HOME}/bin/spark-submit \ +--master spark://... \ +--executor-memory 150g \ +--driver-memory 100g \ +--executor-cores 28 \ +--total-executor-cores 448 \ +--jars ${BigDL_JAR_PATH} \ +--py-files ${PYTHON_API_PATH} \ +${BigDL_HOME}/pyspark/dl/models/inception/inception.py \ +-f hdfs://... \ +--batchSize 1792 \ +--learningRate 0.0898 \ +--weightDecay 0.0001 \ +--checkpointIteration 6200 \ +-i 62000 \ +--checkpoint /models/inception +``` +* Spark yarn client mode, example command +``` +BigDL_HOME=... +SPARK_HOME=... +PYTHON_API_PATH=${BigDL_HOME}/dist/lib/bigdl-VERSION-python-api.zip +BigDL_JAR_PATH=${BigDL_HOME}/dist/lib/bigdl-VERSION-jar-with-dependencies.jar +PYTHONPATH=${PYTHON_API_PATH}:$PYTHONPATH +source ${BigDL_HOME}/dist/bin/bigdl.sh + +${SPARK_HOME}/bin/spark-submit \ +--master yarn \ +--deploy-mode client \ +--executor-memory 150g \ +--driver-memory 100g \ +--executor-cores 28 \ +--num-executors 16 \ +--properties-file ${BigDL_HOME}/dist/conf/spark-bigdl.conf \ +--jars ${BigDL_JAR_PATH} \ +--conf spark.driver.extraClassPath=${BigDL_JAR_PATH} \ +--conf spark.executor.extraClassPath=bigdl-VERSION-jar-with-dependencies.jar \ +--py-files ${PYTHON_API_PATH} \ +${BigDL_HOME}/pyspark/dl/models/inception/inception.py \ +-f hdfs://... \ +--batchSize 1792 \ +--learningRate 0.0898 \ +--weightDecay 0.0001 \ +--checkpointIteration 6200 \ +-i 62000 \ +--checkpoint /models/inception + +``` + +In the above commands +* -f: where you put your ImageNet data, it should be a hdfs folder +* --checkpoint: Where you cache the model/train_state snapshot. You should input a folder and +make sure the folder is created when you run this example. The model snapshot will be named as +model.#iteration_number, and train state will be named as state.#iteration_number. Note that if +there are some files already exist in the folder, the old file will not be overwrite for the +safety of your model files. +* --batchSize: The mini-batch size. It is expected that the mini-batch size is a multiple of node_number * +core_number. In this example, node_number is 1 and the mini-batch size is suggested to be set to core_number * 4 +* --learningRate: inital learning rate. Note in this example, we use a Poly learning rate decay +policy. +* --weightDecay: weight decay. +* -i: max iteration +* --checkpointIteration: the checkpoint interval in iteration. \ No newline at end of file diff --git a/python/dllib/src/bigdl/dllib/models/inception/inception.py b/python/dllib/src/bigdl/dllib/models/inception/inception.py new file mode 100644 index 00000000000..643c268f813 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/models/inception/inception.py @@ -0,0 +1,313 @@ +from bigdl.nn.layer import * +from optparse import OptionParser +from bigdl.nn.criterion import * +from bigdl.nn.initialization_method import * +from bigdl.optim.optimizer import * +from bigdl.transform.vision.image import * + + +def scala_T(input_T): + if type(input_T) is list: + # insert into index 0 spot, such that the real data starts from index 1 + temp = [0] + temp.extend(input_T) + return dict(enumerate(temp)) + # if dictionary, return it back + return input_T + + +def Inception_Layer_v1(input_size, config, name_prefix=""): + concat = Concat(2) + conv1 = Sequential() + conv1.add( + SpatialConvolution(input_size, + config[1][1], + 1, 1, 1, 1).set_init_method(weight_init_method=Xavier()) + .set_name(name_prefix + "1x1")) + conv1.add(ReLU(True).set_name(name_prefix + "relu_1x1")) + concat.add(conv1) + conv3 = Sequential() + conv3.add(SpatialConvolution(input_size, config[2][1], 1, 1, 1, 1).set_init_method(weight_init_method=Xavier()) + .set_name(name_prefix + "3x3_reduce")) + conv3.add(ReLU(True).set_name(name_prefix + "relu_3x3_reduce")) + conv3.add(SpatialConvolution(config[2][1], config[2][2], + 3, 3, 1, 1, 1, 1).set_init_method(weight_init_method=Xavier()) + .set_name(name_prefix + "3x3")) + conv3.add(ReLU(True).set_name(name_prefix + "relu_3x3")) + concat.add(conv3) + conv5 = Sequential() + conv5.add(SpatialConvolution(input_size, + config[3][1], 1, 1, 1, 1).set_init_method(weight_init_method=Xavier()) + .set_name(name_prefix + "5x5_reduce")) + conv5.add(ReLU(True).set_name(name_prefix + "relu_5x5_reduce")) + conv5.add(SpatialConvolution(config[3][1], + config[3][2], 5, 5, 1, 1, 2, 2).set_init_method(weight_init_method=Xavier()) + .set_name(name_prefix + "5x5")) + conv5.add(ReLU(True).set_name(name_prefix + "relu_5x5")) + concat.add(conv5) + pool = Sequential() + pool.add(SpatialMaxPooling(3, 3, 1, 1, 1, 1, + to_ceil=True).set_name(name_prefix + "pool")) + pool.add(SpatialConvolution(input_size, config[4][1], 1, 1, 1, 1).set_init_method(weight_init_method=Xavier()) + .set_name(name_prefix + "pool_proj")) + pool.add(ReLU(True).set_name(name_prefix + "relu_pool_proj")) + concat.add(pool).set_name(name_prefix + "output") + return concat + + +def Inception_v1_NoAuxClassifier(class_num): + model = Sequential() + model.add(SpatialConvolution(3, 64, 7, 7, 2, 2, 3, 3, 1, False) + .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) + .set_name("conv1/7x7_s2")) + model.add(ReLU(True).set_name("conv1/relu_7x7")) + model.add(SpatialMaxPooling(3, 3, 2, 2, to_ceil=True).set_name("pool1/3x3_s2")) + model.add(SpatialCrossMapLRN(5, 0.0001, 0.75).set_name("pool1/norm1")) + model.add(SpatialConvolution(64, 64, 1, 1, 1, 1).set_init_method(weight_init_method=Xavier()) + .set_name("conv2/3x3_reduce")) + model.add(ReLU(True).set_name("conv2/relu_3x3_reduce")) + model.add(SpatialConvolution(64, 192, 3, 3, 1, 1, 1, 1).set_init_method(weight_init_method=Xavier()) + .set_name("conv2/3x3")) + model.add(ReLU(True).set_name("conv2/relu_3x3")) + model.add(SpatialCrossMapLRN(5, 0.0001, 0.75).set_name("conv2/norm2")) + model.add(SpatialMaxPooling(3, 3, 2, 2, to_ceil=True).set_name("pool2/3x3_s2")) + model.add(Inception_Layer_v1(192, scala_T([scala_T([64]), scala_T( + [96, 128]), scala_T([16, 32]), scala_T([32])]), "inception_3a/")) + model.add(Inception_Layer_v1(256, scala_T([scala_T([128]), scala_T( + [128, 192]), scala_T([32, 96]), scala_T([64])]), "inception_3b/")) + model.add(SpatialMaxPooling(3, 3, 2, 2, to_ceil=True)) + model.add(Inception_Layer_v1(480, scala_T([scala_T([192]), scala_T( + [96, 208]), scala_T([16, 48]), scala_T([64])]), "inception_4a/")) + model.add(Inception_Layer_v1(512, scala_T([scala_T([160]), scala_T( + [112, 224]), scala_T([24, 64]), scala_T([64])]), "inception_4b/")) + model.add(Inception_Layer_v1(512, scala_T([scala_T([128]), scala_T( + [128, 256]), scala_T([24, 64]), scala_T([64])]), "inception_4c/")) + model.add(Inception_Layer_v1(512, scala_T([scala_T([112]), scala_T( + [144, 288]), scala_T([32, 64]), scala_T([64])]), "inception_4d/")) + model.add(Inception_Layer_v1(528, scala_T([scala_T([256]), scala_T( + [160, 320]), scala_T([32, 128]), scala_T([128])]), "inception_4e/")) + model.add(SpatialMaxPooling(3, 3, 2, 2, to_ceil=True)) + model.add(Inception_Layer_v1(832, scala_T([scala_T([256]), scala_T( + [160, 320]), scala_T([32, 128]), scala_T([128])]), "inception_5a/")) + model.add(Inception_Layer_v1(832, scala_T([scala_T([384]), scala_T( + [192, 384]), scala_T([48, 128]), scala_T([128])]), "inception_5b/")) + model.add(SpatialAveragePooling(7, 7, 1, 1).set_name("pool5/7x7_s1")) + model.add(Dropout(0.4).set_name("pool5/drop_7x7_s1")) + model.add(View([1024], num_input_dims=3)) + model.add(Linear(1024, class_num).set_init_method(weight_init_method=Xavier()).set_name("loss3/classifier")) + model.add(LogSoftMax().set_name("loss3/loss3")) + model.reset() + return model + + +def Inception_v1(class_num): + feature1 = Sequential() + feature1.add(SpatialConvolution(3, 64, 7, 7, 2, 2, 3, 3, 1, False).set_init_method(weight_init_method=Xavier()) + .set_name("conv1/7x7_s2")) + feature1.add(ReLU(True).set_name("conv1/relu_7x7")) + feature1.add( + SpatialMaxPooling(3, 3, 2, 2, to_ceil=True) + .set_name("pool1/3x3_s2")) + feature1.add(SpatialCrossMapLRN(5, 0.0001, 0.75) + .set_name("pool1/norm1")) + feature1.add(SpatialConvolution(64, 64, 1, 1, 1, 1).set_init_method(weight_init_method=Xavier()) + .set_name("conv2/3x3_reduce")) + feature1.add(ReLU(True).set_name("conv2/relu_3x3_reduce")) + feature1.add(SpatialConvolution(64, 192, 3, 3, 1, + 1, 1, 1).set_init_method(weight_init_method=Xavier()) + .set_name("conv2/3x3")) + feature1.add(ReLU(True).set_name("conv2/relu_3x3")) + feature1.add(SpatialCrossMapLRN(5, 0.0001, 0.75).set_name("conv2/norm2")) + feature1.add( + SpatialMaxPooling(3, 3, 2, 2, to_ceil=True).set_name("pool2/3x3_s2")) + feature1.add(Inception_Layer_v1(192, + scala_T([scala_T([64]), scala_T([96, 128]), + scala_T([16, 32]), scala_T([32])]), + "inception_3a/")) + feature1.add(Inception_Layer_v1(256, scala_T([ + scala_T([128]), scala_T([128, 192]), scala_T([32, 96]), scala_T([64])]), + "inception_3b/")) + feature1.add( + SpatialMaxPooling(3, 3, 2, 2, to_ceil=True).set_name("pool3/3x3_s2")) + feature1.add(Inception_Layer_v1(480, scala_T([ + scala_T([192]), scala_T([96, 208]), scala_T([16, 48]), scala_T([64])]), + "inception_4a/")) + + output1 = Sequential() + output1.add(SpatialAveragePooling( + 5, 5, 3, 3, ceil_mode=True).set_name("loss1/ave_pool")) + output1.add( + SpatialConvolution(512, 128, 1, 1, 1, 1).set_name("loss1/conv")) + output1.add(ReLU(True).set_name("loss1/relu_conv")) + output1.add(View([128 * 4 * 4, 3])) + output1.add(Linear(128 * 4 * 4, 1024).set_name("loss1/fc")) + output1.add(ReLU(True).set_name("loss1/relu_fc")) + output1.add(Dropout(0.7).set_name("loss1/drop_fc")) + output1.add(Linear(1024, class_num).set_name("loss1/classifier")) + output1.add(LogSoftMax().set_name("loss1/loss")) + + feature2 = Sequential() + feature2.add(Inception_Layer_v1(512, + scala_T([scala_T([160]), scala_T([112, 224]), + scala_T([24, 64]), scala_T([64])]), + "inception_4b/")) + feature2.add(Inception_Layer_v1(512, + scala_T([scala_T([128]), scala_T([128, 256]), + scala_T([24, 64]), scala_T([64])]), + "inception_4c/")) + feature2.add(Inception_Layer_v1(512, + scala_T([scala_T([112]), scala_T([144, 288]), + scala_T([32, 64]), scala_T([64])]), + "inception_4d/")) + + output2 = Sequential() + output2.add(SpatialAveragePooling(5, 5, 3, 3).set_name("loss2/ave_pool")) + output2.add( + SpatialConvolution(528, 128, 1, 1, 1, 1).set_name("loss2/conv")) + output2.add(ReLU(True).set_name("loss2/relu_conv")) + output2.add(View([128 * 4 * 4, 3])) + output2.add(Linear(128 * 4 * 4, 1024).set_name("loss2/fc")) + output2.add(ReLU(True).set_name("loss2/relu_fc")) + output2.add(Dropout(0.7).set_name("loss2/drop_fc")) + output2.add(Linear(1024, class_num).set_name("loss2/classifier")) + output2.add(LogSoftMax().set_name("loss2/loss")) + + output3 = Sequential() + output3.add(Inception_Layer_v1(528, + scala_T([scala_T([256]), scala_T([160, 320]), + scala_T([32, 128]), scala_T([128])]), + "inception_4e/")) + output3.add(SpatialMaxPooling(3, 3, 2, 2, to_ceil=True).set_name("pool4/3x3_s2")) + output3.add(Inception_Layer_v1(832, + scala_T([scala_T([256]), scala_T([160, 320]), + scala_T([32, 128]), scala_T([128])]), + "inception_5a/")) + output3.add(Inception_Layer_v1(832, + scala_T([scala_T([384]), scala_T([192, 384]), + scala_T([48, 128]), scala_T([128])]), + "inception_5b/")) + output3.add(SpatialAveragePooling(7, 7, 1, 1).set_name("pool5/7x7_s1")) + output3.add(Dropout(0.4).set_name("pool5/drop_7x7_s1")) + output3.add(View([1024, 3])) + output3.add(Linear(1024, class_num).set_init_method(weight_init_method=Xavier()) + .set_name("loss3/classifier")) + output3.add(LogSoftMax().set_name("loss3/loss3")) + + split2 = Concat(2).set_name("split2") + split2.add(output3) + split2.add(output2) + + mainBranch = Sequential() + mainBranch.add(feature2) + mainBranch.add(split2) + + split1 = Concat(2).set_name("split1") + split1.add(mainBranch) + split1.add(output1) + + model = Sequential() + + model.add(feature1) + model.add(split1) + + model.reset() + return model + + +def get_inception_data(url, sc=None, data_type="train"): + path = os.path.join(url, data_type) + return SeqFileFolder.files_to_image_frame(url=path, sc=sc, class_num=1000) + + +def config_option_parser(): + parser = OptionParser() + parser.add_option("-f", "--folder", type=str, dest="folder", default="", + help="url of hdfs folder store the hadoop sequence files") + parser.add_option("--model", type=str, dest="model", default="./inception.model", help="model snapshot location") + parser.add_option("--state", type=str, dest="state", default="", help="state snapshot location") + parser.add_option("--checkpoint", type=str, dest="checkpoint", default="", help="where to cache the model") + parser.add_option("-o", "--overwrite", action="store_true", dest="overwrite", default=False, + help="overwrite checkpoint files") + parser.add_option("-e", "--maxEpoch", type=int, dest="maxEpoch", default=0, help="epoch numbers") + parser.add_option("-i", "--maxIteration", type=int, dest="maxIteration", default=62000, help="iteration numbers") + parser.add_option("-l", "--learningRate", type=float, dest="learningRate", default=0.01, help="learning rate") + parser.add_option("-b", "--batchSize", type=int, dest="batchSize", help="batch size") + parser.add_option("--classNum", type=int, dest="classNum", default=1000, help="class number") + parser.add_option("--weightDecay", type=float, dest="weightDecay", default=0.0001, help="weight decay") + parser.add_option("--checkpointIteration", type=int, dest="checkpointIteration", default=620, + help="checkpoint interval of iterations") + + return parser + + +if __name__ == "__main__": + # parse options + parser = config_option_parser() + (options, args) = parser.parse_args(sys.argv) + if not options.learningRate: + parser.error("-l --learningRate is a mandatory opt") + if not options.batchSize: + parser.error("-b --batchSize is a mandatory opt") + + # init + sparkConf = create_spark_conf().setAppName("inception v1") + sc = get_spark_context(sparkConf) + redire_spark_logs() + show_bigdl_info_logs() + init_engine() + + # build model + inception_model = Inception_v1_NoAuxClassifier(options.classNum) + + image_size = 224 # create dataset + train_transformer = Pipeline([PixelBytesToMat(), + RandomCrop(image_size, image_size), + RandomTransformer(HFlip(), 0.5), + ChannelNormalize(0.485, 0.456, 0.406, 0.229, 0.224, 0.225), + MatToTensor(to_rgb=True), + ImageFrameToSample(input_keys=["imageTensor"], target_keys=["label"]) + ]) + raw_train_data = get_inception_data(options.folder, sc, "train") + train_data = DataSet.image_frame(raw_train_data).transform(train_transformer) + + val_transformer = Pipeline([PixelBytesToMat(), + CenterCrop(image_size, image_size), + RandomTransformer(HFlip(), 0.5), + ChannelNormalize(0.485, 0.456, 0.406, 0.229, 0.224, 0.225), + MatToTensor(to_rgb=True), + ImageFrameToSample(input_keys=["imageTensor"], target_keys=["label"]) + ]) + raw_val_data = get_inception_data(options.folder, sc, "val") + val_data = DataSet.image_frame(raw_val_data).transform(val_transformer) + + if options.maxEpoch: + checkpoint_trigger = EveryEpoch() + test_trigger = EveryEpoch() + end_trigger = MaxEpoch(options.maxEpoch) + else: + checkpoint_trigger = SeveralIteration(options.checkpointIteration) + test_trigger = SeveralIteration(options.checkpointIteration) + end_trigger = MaxIteration(options.maxIteration) + + # Optimizer + optimizer = Optimizer.create( + model=inception_model, + training_set=train_data, + optim_method=SGD(learningrate=options.learningRate, learningrate_decay=options.weightDecay, + momentum=0.9, dampening=0.0), + criterion=ClassNLLCriterion(), + end_trigger=end_trigger, + batch_size=options.batchSize + ) + + if options.checkpoint: + optimizer.set_checkpoint(checkpoint_trigger, options.checkpoint, options.overwrite) + + optimizer.set_validation(trigger=test_trigger, + val_rdd=val_data, + batch_size=options.batchSize, + val_method=[Top1Accuracy(), Top5Accuracy()]) + + trained_model = optimizer.optimize() + + sc.stop() From c5df39fffa2066d21e18fcb3cd8d82c71d75c174 Mon Sep 17 00:00:00 2001 From: jenniew Date: Tue, 13 Feb 2018 13:00:40 -0800 Subject: [PATCH 611/823] update according to comments --- .../bigdl/dllib/models/inception/README.md | 3 +- .../bigdl/dllib/models/inception/inception.py | 95 +++++++++---------- 2 files changed, 45 insertions(+), 53 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/models/inception/README.md b/python/dllib/src/bigdl/dllib/models/inception/README.md index c334b2b33fd..4f53e686b6f 100644 --- a/python/dllib/src/bigdl/dllib/models/inception/README.md +++ b/python/dllib/src/bigdl/dllib/models/inception/README.md @@ -48,7 +48,6 @@ SPARK_HOME=... PYTHON_API_PATH=${BigDL_HOME}/dist/lib/bigdl-VERSION-python-api.zip BigDL_JAR_PATH=${BigDL_HOME}/dist/lib/bigdl-VERSION-jar-with-dependencies.jar PYTHONPATH=${PYTHON_API_PATH}:$PYTHONPATH -source ${BigDL_HOME}/dist/bin/bigdl.sh ${SPARK_HOME}/bin/spark-submit \ --master spark://... \ @@ -56,6 +55,7 @@ ${SPARK_HOME}/bin/spark-submit \ --driver-memory 100g \ --executor-cores 28 \ --total-executor-cores 448 \ +--properties-file ${BigDL_HOME}/dist/conf/spark-bigdl.conf \ --jars ${BigDL_JAR_PATH} \ --py-files ${PYTHON_API_PATH} \ ${BigDL_HOME}/pyspark/dl/models/inception/inception.py \ @@ -74,7 +74,6 @@ SPARK_HOME=... PYTHON_API_PATH=${BigDL_HOME}/dist/lib/bigdl-VERSION-python-api.zip BigDL_JAR_PATH=${BigDL_HOME}/dist/lib/bigdl-VERSION-jar-with-dependencies.jar PYTHONPATH=${PYTHON_API_PATH}:$PYTHONPATH -source ${BigDL_HOME}/dist/bin/bigdl.sh ${SPARK_HOME}/bin/spark-submit \ --master yarn \ diff --git a/python/dllib/src/bigdl/dllib/models/inception/inception.py b/python/dllib/src/bigdl/dllib/models/inception/inception.py index 643c268f813..0cd656cf9af 100644 --- a/python/dllib/src/bigdl/dllib/models/inception/inception.py +++ b/python/dllib/src/bigdl/dllib/models/inception/inception.py @@ -6,17 +6,17 @@ from bigdl.transform.vision.image import * -def scala_T(input_T): - if type(input_T) is list: +def t(input_t): + if type(input_t) is list: # insert into index 0 spot, such that the real data starts from index 1 temp = [0] - temp.extend(input_T) + temp.extend(input_t) return dict(enumerate(temp)) # if dictionary, return it back - return input_T + return input_t -def Inception_Layer_v1(input_size, config, name_prefix=""): +def inception_layer_v1(input_size, config, name_prefix=""): concat = Concat(2) conv1 = Sequential() conv1.add( @@ -55,7 +55,7 @@ def Inception_Layer_v1(input_size, config, name_prefix=""): return concat -def Inception_v1_NoAuxClassifier(class_num): +def inception_v1_no_aux_classifier(class_num): model = Sequential() model.add(SpatialConvolution(3, 64, 7, 7, 2, 2, 3, 3, 1, False) .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) @@ -71,26 +71,26 @@ def Inception_v1_NoAuxClassifier(class_num): model.add(ReLU(True).set_name("conv2/relu_3x3")) model.add(SpatialCrossMapLRN(5, 0.0001, 0.75).set_name("conv2/norm2")) model.add(SpatialMaxPooling(3, 3, 2, 2, to_ceil=True).set_name("pool2/3x3_s2")) - model.add(Inception_Layer_v1(192, scala_T([scala_T([64]), scala_T( - [96, 128]), scala_T([16, 32]), scala_T([32])]), "inception_3a/")) - model.add(Inception_Layer_v1(256, scala_T([scala_T([128]), scala_T( - [128, 192]), scala_T([32, 96]), scala_T([64])]), "inception_3b/")) + model.add(inception_layer_v1(192, t([t([64]), t( + [96, 128]), t([16, 32]), t([32])]), "inception_3a/")) + model.add(inception_layer_v1(256, t([t([128]), t( + [128, 192]), t([32, 96]), t([64])]), "inception_3b/")) model.add(SpatialMaxPooling(3, 3, 2, 2, to_ceil=True)) - model.add(Inception_Layer_v1(480, scala_T([scala_T([192]), scala_T( - [96, 208]), scala_T([16, 48]), scala_T([64])]), "inception_4a/")) - model.add(Inception_Layer_v1(512, scala_T([scala_T([160]), scala_T( - [112, 224]), scala_T([24, 64]), scala_T([64])]), "inception_4b/")) - model.add(Inception_Layer_v1(512, scala_T([scala_T([128]), scala_T( - [128, 256]), scala_T([24, 64]), scala_T([64])]), "inception_4c/")) - model.add(Inception_Layer_v1(512, scala_T([scala_T([112]), scala_T( - [144, 288]), scala_T([32, 64]), scala_T([64])]), "inception_4d/")) - model.add(Inception_Layer_v1(528, scala_T([scala_T([256]), scala_T( - [160, 320]), scala_T([32, 128]), scala_T([128])]), "inception_4e/")) + model.add(inception_layer_v1(480, t([t([192]), t( + [96, 208]), t([16, 48]), t([64])]), "inception_4a/")) + model.add(inception_layer_v1(512, t([t([160]), t( + [112, 224]), t([24, 64]), t([64])]), "inception_4b/")) + model.add(inception_layer_v1(512, t([t([128]), t( + [128, 256]), t([24, 64]), t([64])]), "inception_4c/")) + model.add(inception_layer_v1(512, t([t([112]), t( + [144, 288]), t([32, 64]), t([64])]), "inception_4d/")) + model.add(inception_layer_v1(528, t([t([256]), t( + [160, 320]), t([32, 128]), t([128])]), "inception_4e/")) model.add(SpatialMaxPooling(3, 3, 2, 2, to_ceil=True)) - model.add(Inception_Layer_v1(832, scala_T([scala_T([256]), scala_T( - [160, 320]), scala_T([32, 128]), scala_T([128])]), "inception_5a/")) - model.add(Inception_Layer_v1(832, scala_T([scala_T([384]), scala_T( - [192, 384]), scala_T([48, 128]), scala_T([128])]), "inception_5b/")) + model.add(inception_layer_v1(832, t([t([256]), t( + [160, 320]), t([32, 128]), t([128])]), "inception_5a/")) + model.add(inception_layer_v1(832, t([t([384]), t( + [192, 384]), t([48, 128]), t([128])]), "inception_5b/")) model.add(SpatialAveragePooling(7, 7, 1, 1).set_name("pool5/7x7_s1")) model.add(Dropout(0.4).set_name("pool5/drop_7x7_s1")) model.add(View([1024], num_input_dims=3)) @@ -100,7 +100,7 @@ def Inception_v1_NoAuxClassifier(class_num): return model -def Inception_v1(class_num): +def inception_v1(class_num): feature1 = Sequential() feature1.add(SpatialConvolution(3, 64, 7, 7, 2, 2, 3, 3, 1, False).set_init_method(weight_init_method=Xavier()) .set_name("conv1/7x7_s2")) @@ -120,17 +120,16 @@ def Inception_v1(class_num): feature1.add(SpatialCrossMapLRN(5, 0.0001, 0.75).set_name("conv2/norm2")) feature1.add( SpatialMaxPooling(3, 3, 2, 2, to_ceil=True).set_name("pool2/3x3_s2")) - feature1.add(Inception_Layer_v1(192, - scala_T([scala_T([64]), scala_T([96, 128]), - scala_T([16, 32]), scala_T([32])]), + feature1.add(inception_layer_v1(192, t([ + t([64]), t([96, 128]), t([16, 32]), t([32])]), "inception_3a/")) - feature1.add(Inception_Layer_v1(256, scala_T([ - scala_T([128]), scala_T([128, 192]), scala_T([32, 96]), scala_T([64])]), + feature1.add(inception_layer_v1(256, t([ + t([128]), t([128, 192]), t([32, 96]), t([64])]), "inception_3b/")) feature1.add( SpatialMaxPooling(3, 3, 2, 2, to_ceil=True).set_name("pool3/3x3_s2")) - feature1.add(Inception_Layer_v1(480, scala_T([ - scala_T([192]), scala_T([96, 208]), scala_T([16, 48]), scala_T([64])]), + feature1.add(inception_layer_v1(480, t([ + t([192]), t([96, 208]), t([16, 48]), t([64])]), "inception_4a/")) output1 = Sequential() @@ -147,17 +146,14 @@ def Inception_v1(class_num): output1.add(LogSoftMax().set_name("loss1/loss")) feature2 = Sequential() - feature2.add(Inception_Layer_v1(512, - scala_T([scala_T([160]), scala_T([112, 224]), - scala_T([24, 64]), scala_T([64])]), + feature2.add(inception_layer_v1(512, + t([t([160]), t([112, 224]), t([24, 64]), t([64])]), "inception_4b/")) - feature2.add(Inception_Layer_v1(512, - scala_T([scala_T([128]), scala_T([128, 256]), - scala_T([24, 64]), scala_T([64])]), + feature2.add(inception_layer_v1(512, + t([t([128]), t([128, 256]), t([24, 64]), t([64])]), "inception_4c/")) - feature2.add(Inception_Layer_v1(512, - scala_T([scala_T([112]), scala_T([144, 288]), - scala_T([32, 64]), scala_T([64])]), + feature2.add(inception_layer_v1(512, + t([t([112]), t([144, 288]), t([32, 64]), t([64])]), "inception_4d/")) output2 = Sequential() @@ -173,18 +169,15 @@ def Inception_v1(class_num): output2.add(LogSoftMax().set_name("loss2/loss")) output3 = Sequential() - output3.add(Inception_Layer_v1(528, - scala_T([scala_T([256]), scala_T([160, 320]), - scala_T([32, 128]), scala_T([128])]), + output3.add(inception_layer_v1(528, + t([t([256]), t([160, 320]), t([32, 128]), t([128])]), "inception_4e/")) output3.add(SpatialMaxPooling(3, 3, 2, 2, to_ceil=True).set_name("pool4/3x3_s2")) - output3.add(Inception_Layer_v1(832, - scala_T([scala_T([256]), scala_T([160, 320]), - scala_T([32, 128]), scala_T([128])]), + output3.add(inception_layer_v1(832, + t([t([256]), t([160, 320]), t([32, 128]), t([128])]), "inception_5a/")) - output3.add(Inception_Layer_v1(832, - scala_T([scala_T([384]), scala_T([192, 384]), - scala_T([48, 128]), scala_T([128])]), + output3.add(inception_layer_v1(832, + t([t([384]), t([192, 384]), t([48, 128]), t([128])]), "inception_5b/")) output3.add(SpatialAveragePooling(7, 7, 1, 1).set_name("pool5/7x7_s1")) output3.add(Dropout(0.4).set_name("pool5/drop_7x7_s1")) @@ -257,7 +250,7 @@ def config_option_parser(): init_engine() # build model - inception_model = Inception_v1_NoAuxClassifier(options.classNum) + inception_model = inception_v1_no_aux_classifier(options.classNum) image_size = 224 # create dataset train_transformer = Pipeline([PixelBytesToMat(), From bfa9007db4326de809bb885fac6ba0950ec777bd Mon Sep 17 00:00:00 2001 From: jenniew Date: Tue, 13 Feb 2018 13:00:40 -0800 Subject: [PATCH 612/823] update according to comments --- .../bigdl/dllib/models/inception/README.md | 3 +- .../bigdl/dllib/models/inception/inception.py | 95 +++++++++---------- 2 files changed, 45 insertions(+), 53 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/models/inception/README.md b/python/dllib/src/bigdl/dllib/models/inception/README.md index c334b2b33fd..4f53e686b6f 100644 --- a/python/dllib/src/bigdl/dllib/models/inception/README.md +++ b/python/dllib/src/bigdl/dllib/models/inception/README.md @@ -48,7 +48,6 @@ SPARK_HOME=... PYTHON_API_PATH=${BigDL_HOME}/dist/lib/bigdl-VERSION-python-api.zip BigDL_JAR_PATH=${BigDL_HOME}/dist/lib/bigdl-VERSION-jar-with-dependencies.jar PYTHONPATH=${PYTHON_API_PATH}:$PYTHONPATH -source ${BigDL_HOME}/dist/bin/bigdl.sh ${SPARK_HOME}/bin/spark-submit \ --master spark://... \ @@ -56,6 +55,7 @@ ${SPARK_HOME}/bin/spark-submit \ --driver-memory 100g \ --executor-cores 28 \ --total-executor-cores 448 \ +--properties-file ${BigDL_HOME}/dist/conf/spark-bigdl.conf \ --jars ${BigDL_JAR_PATH} \ --py-files ${PYTHON_API_PATH} \ ${BigDL_HOME}/pyspark/dl/models/inception/inception.py \ @@ -74,7 +74,6 @@ SPARK_HOME=... PYTHON_API_PATH=${BigDL_HOME}/dist/lib/bigdl-VERSION-python-api.zip BigDL_JAR_PATH=${BigDL_HOME}/dist/lib/bigdl-VERSION-jar-with-dependencies.jar PYTHONPATH=${PYTHON_API_PATH}:$PYTHONPATH -source ${BigDL_HOME}/dist/bin/bigdl.sh ${SPARK_HOME}/bin/spark-submit \ --master yarn \ diff --git a/python/dllib/src/bigdl/dllib/models/inception/inception.py b/python/dllib/src/bigdl/dllib/models/inception/inception.py index 643c268f813..0cd656cf9af 100644 --- a/python/dllib/src/bigdl/dllib/models/inception/inception.py +++ b/python/dllib/src/bigdl/dllib/models/inception/inception.py @@ -6,17 +6,17 @@ from bigdl.transform.vision.image import * -def scala_T(input_T): - if type(input_T) is list: +def t(input_t): + if type(input_t) is list: # insert into index 0 spot, such that the real data starts from index 1 temp = [0] - temp.extend(input_T) + temp.extend(input_t) return dict(enumerate(temp)) # if dictionary, return it back - return input_T + return input_t -def Inception_Layer_v1(input_size, config, name_prefix=""): +def inception_layer_v1(input_size, config, name_prefix=""): concat = Concat(2) conv1 = Sequential() conv1.add( @@ -55,7 +55,7 @@ def Inception_Layer_v1(input_size, config, name_prefix=""): return concat -def Inception_v1_NoAuxClassifier(class_num): +def inception_v1_no_aux_classifier(class_num): model = Sequential() model.add(SpatialConvolution(3, 64, 7, 7, 2, 2, 3, 3, 1, False) .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) @@ -71,26 +71,26 @@ def Inception_v1_NoAuxClassifier(class_num): model.add(ReLU(True).set_name("conv2/relu_3x3")) model.add(SpatialCrossMapLRN(5, 0.0001, 0.75).set_name("conv2/norm2")) model.add(SpatialMaxPooling(3, 3, 2, 2, to_ceil=True).set_name("pool2/3x3_s2")) - model.add(Inception_Layer_v1(192, scala_T([scala_T([64]), scala_T( - [96, 128]), scala_T([16, 32]), scala_T([32])]), "inception_3a/")) - model.add(Inception_Layer_v1(256, scala_T([scala_T([128]), scala_T( - [128, 192]), scala_T([32, 96]), scala_T([64])]), "inception_3b/")) + model.add(inception_layer_v1(192, t([t([64]), t( + [96, 128]), t([16, 32]), t([32])]), "inception_3a/")) + model.add(inception_layer_v1(256, t([t([128]), t( + [128, 192]), t([32, 96]), t([64])]), "inception_3b/")) model.add(SpatialMaxPooling(3, 3, 2, 2, to_ceil=True)) - model.add(Inception_Layer_v1(480, scala_T([scala_T([192]), scala_T( - [96, 208]), scala_T([16, 48]), scala_T([64])]), "inception_4a/")) - model.add(Inception_Layer_v1(512, scala_T([scala_T([160]), scala_T( - [112, 224]), scala_T([24, 64]), scala_T([64])]), "inception_4b/")) - model.add(Inception_Layer_v1(512, scala_T([scala_T([128]), scala_T( - [128, 256]), scala_T([24, 64]), scala_T([64])]), "inception_4c/")) - model.add(Inception_Layer_v1(512, scala_T([scala_T([112]), scala_T( - [144, 288]), scala_T([32, 64]), scala_T([64])]), "inception_4d/")) - model.add(Inception_Layer_v1(528, scala_T([scala_T([256]), scala_T( - [160, 320]), scala_T([32, 128]), scala_T([128])]), "inception_4e/")) + model.add(inception_layer_v1(480, t([t([192]), t( + [96, 208]), t([16, 48]), t([64])]), "inception_4a/")) + model.add(inception_layer_v1(512, t([t([160]), t( + [112, 224]), t([24, 64]), t([64])]), "inception_4b/")) + model.add(inception_layer_v1(512, t([t([128]), t( + [128, 256]), t([24, 64]), t([64])]), "inception_4c/")) + model.add(inception_layer_v1(512, t([t([112]), t( + [144, 288]), t([32, 64]), t([64])]), "inception_4d/")) + model.add(inception_layer_v1(528, t([t([256]), t( + [160, 320]), t([32, 128]), t([128])]), "inception_4e/")) model.add(SpatialMaxPooling(3, 3, 2, 2, to_ceil=True)) - model.add(Inception_Layer_v1(832, scala_T([scala_T([256]), scala_T( - [160, 320]), scala_T([32, 128]), scala_T([128])]), "inception_5a/")) - model.add(Inception_Layer_v1(832, scala_T([scala_T([384]), scala_T( - [192, 384]), scala_T([48, 128]), scala_T([128])]), "inception_5b/")) + model.add(inception_layer_v1(832, t([t([256]), t( + [160, 320]), t([32, 128]), t([128])]), "inception_5a/")) + model.add(inception_layer_v1(832, t([t([384]), t( + [192, 384]), t([48, 128]), t([128])]), "inception_5b/")) model.add(SpatialAveragePooling(7, 7, 1, 1).set_name("pool5/7x7_s1")) model.add(Dropout(0.4).set_name("pool5/drop_7x7_s1")) model.add(View([1024], num_input_dims=3)) @@ -100,7 +100,7 @@ def Inception_v1_NoAuxClassifier(class_num): return model -def Inception_v1(class_num): +def inception_v1(class_num): feature1 = Sequential() feature1.add(SpatialConvolution(3, 64, 7, 7, 2, 2, 3, 3, 1, False).set_init_method(weight_init_method=Xavier()) .set_name("conv1/7x7_s2")) @@ -120,17 +120,16 @@ def Inception_v1(class_num): feature1.add(SpatialCrossMapLRN(5, 0.0001, 0.75).set_name("conv2/norm2")) feature1.add( SpatialMaxPooling(3, 3, 2, 2, to_ceil=True).set_name("pool2/3x3_s2")) - feature1.add(Inception_Layer_v1(192, - scala_T([scala_T([64]), scala_T([96, 128]), - scala_T([16, 32]), scala_T([32])]), + feature1.add(inception_layer_v1(192, t([ + t([64]), t([96, 128]), t([16, 32]), t([32])]), "inception_3a/")) - feature1.add(Inception_Layer_v1(256, scala_T([ - scala_T([128]), scala_T([128, 192]), scala_T([32, 96]), scala_T([64])]), + feature1.add(inception_layer_v1(256, t([ + t([128]), t([128, 192]), t([32, 96]), t([64])]), "inception_3b/")) feature1.add( SpatialMaxPooling(3, 3, 2, 2, to_ceil=True).set_name("pool3/3x3_s2")) - feature1.add(Inception_Layer_v1(480, scala_T([ - scala_T([192]), scala_T([96, 208]), scala_T([16, 48]), scala_T([64])]), + feature1.add(inception_layer_v1(480, t([ + t([192]), t([96, 208]), t([16, 48]), t([64])]), "inception_4a/")) output1 = Sequential() @@ -147,17 +146,14 @@ def Inception_v1(class_num): output1.add(LogSoftMax().set_name("loss1/loss")) feature2 = Sequential() - feature2.add(Inception_Layer_v1(512, - scala_T([scala_T([160]), scala_T([112, 224]), - scala_T([24, 64]), scala_T([64])]), + feature2.add(inception_layer_v1(512, + t([t([160]), t([112, 224]), t([24, 64]), t([64])]), "inception_4b/")) - feature2.add(Inception_Layer_v1(512, - scala_T([scala_T([128]), scala_T([128, 256]), - scala_T([24, 64]), scala_T([64])]), + feature2.add(inception_layer_v1(512, + t([t([128]), t([128, 256]), t([24, 64]), t([64])]), "inception_4c/")) - feature2.add(Inception_Layer_v1(512, - scala_T([scala_T([112]), scala_T([144, 288]), - scala_T([32, 64]), scala_T([64])]), + feature2.add(inception_layer_v1(512, + t([t([112]), t([144, 288]), t([32, 64]), t([64])]), "inception_4d/")) output2 = Sequential() @@ -173,18 +169,15 @@ def Inception_v1(class_num): output2.add(LogSoftMax().set_name("loss2/loss")) output3 = Sequential() - output3.add(Inception_Layer_v1(528, - scala_T([scala_T([256]), scala_T([160, 320]), - scala_T([32, 128]), scala_T([128])]), + output3.add(inception_layer_v1(528, + t([t([256]), t([160, 320]), t([32, 128]), t([128])]), "inception_4e/")) output3.add(SpatialMaxPooling(3, 3, 2, 2, to_ceil=True).set_name("pool4/3x3_s2")) - output3.add(Inception_Layer_v1(832, - scala_T([scala_T([256]), scala_T([160, 320]), - scala_T([32, 128]), scala_T([128])]), + output3.add(inception_layer_v1(832, + t([t([256]), t([160, 320]), t([32, 128]), t([128])]), "inception_5a/")) - output3.add(Inception_Layer_v1(832, - scala_T([scala_T([384]), scala_T([192, 384]), - scala_T([48, 128]), scala_T([128])]), + output3.add(inception_layer_v1(832, + t([t([384]), t([192, 384]), t([48, 128]), t([128])]), "inception_5b/")) output3.add(SpatialAveragePooling(7, 7, 1, 1).set_name("pool5/7x7_s1")) output3.add(Dropout(0.4).set_name("pool5/drop_7x7_s1")) @@ -257,7 +250,7 @@ def config_option_parser(): init_engine() # build model - inception_model = Inception_v1_NoAuxClassifier(options.classNum) + inception_model = inception_v1_no_aux_classifier(options.classNum) image_size = 224 # create dataset train_transformer = Pipeline([PixelBytesToMat(), From 92152321911c1f42bcb7c1fc536ba677056a517f Mon Sep 17 00:00:00 2001 From: jenniew Date: Tue, 20 Feb 2018 17:15:15 -0800 Subject: [PATCH 613/823] add loading snapshot in inception training --- .../bigdl/dllib/models/inception/inception.py | 17 +++++++++++++---- .../src/bigdl/dllib/models/lenet/lenet5.py | 6 ++++-- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/models/inception/inception.py b/python/dllib/src/bigdl/dllib/models/inception/inception.py index 0cd656cf9af..d00b55f1a73 100644 --- a/python/dllib/src/bigdl/dllib/models/inception/inception.py +++ b/python/dllib/src/bigdl/dllib/models/inception/inception.py @@ -216,7 +216,7 @@ def config_option_parser(): parser = OptionParser() parser.add_option("-f", "--folder", type=str, dest="folder", default="", help="url of hdfs folder store the hadoop sequence files") - parser.add_option("--model", type=str, dest="model", default="./inception.model", help="model snapshot location") + parser.add_option("--model", type=str, dest="model", default="", help="model snapshot location") parser.add_option("--state", type=str, dest="state", default="", help="state snapshot location") parser.add_option("--checkpoint", type=str, dest="checkpoint", default="", help="where to cache the model") parser.add_option("-o", "--overwrite", action="store_true", dest="overwrite", default=False, @@ -250,7 +250,17 @@ def config_option_parser(): init_engine() # build model - inception_model = inception_v1_no_aux_classifier(options.classNum) + if options.model != "": + inception_model = Model.load(options.model) + else: + inception_model = inception_v1_no_aux_classifier(options.classNum) + + # load state + if options.state != "": + optim = OptimMethod.load(options.state) + else: + optim = SGD(learningrate=options.learningRate, learningrate_decay=options.weightDecay, + momentum=0.9, dampening=0.0) image_size = 224 # create dataset train_transformer = Pipeline([PixelBytesToMat(), @@ -286,8 +296,7 @@ def config_option_parser(): optimizer = Optimizer.create( model=inception_model, training_set=train_data, - optim_method=SGD(learningrate=options.learningRate, learningrate_decay=options.weightDecay, - momentum=0.9, dampening=0.0), + optim_method=optim, criterion=ClassNLLCriterion(), end_trigger=end_trigger, batch_size=options.batchSize diff --git a/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py b/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py index e6e28b7d65e..286882920fd 100644 --- a/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py +++ b/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py @@ -72,8 +72,10 @@ def build_model(class_num): parameters = trained_model.parameters() elif options.action == "test": # Load a pre-trained model and then validate it through top1 accuracy. - test_data = get_mnist(sc, "test").map( - normalizer(mnist.TEST_MEAN, mnist.TEST_STD)) + test_data = get_mnist(sc, "test", options.dataPath) \ + .map(lambda rec_tuple: (normalizer(rec_tuple[0], mnist.TEST_MEAN, mnist.TEST_STD), + rec_tuple[1])) \ + .map(lambda t: Sample.from_ndarray(t[0], t[1])) model = Model.load(options.modelPath) results = model.evaluate(test_data, options.batchSize, [Top1Accuracy()]) for result in results: From a397a3886b9058e65cf407856176919777d793ef Mon Sep 17 00:00:00 2001 From: jenniew Date: Tue, 20 Feb 2018 17:15:15 -0800 Subject: [PATCH 614/823] add loading snapshot in inception training --- .../bigdl/dllib/models/inception/inception.py | 17 +++++++++++++---- .../src/bigdl/dllib/models/lenet/lenet5.py | 6 ++++-- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/models/inception/inception.py b/python/dllib/src/bigdl/dllib/models/inception/inception.py index 0cd656cf9af..d00b55f1a73 100644 --- a/python/dllib/src/bigdl/dllib/models/inception/inception.py +++ b/python/dllib/src/bigdl/dllib/models/inception/inception.py @@ -216,7 +216,7 @@ def config_option_parser(): parser = OptionParser() parser.add_option("-f", "--folder", type=str, dest="folder", default="", help="url of hdfs folder store the hadoop sequence files") - parser.add_option("--model", type=str, dest="model", default="./inception.model", help="model snapshot location") + parser.add_option("--model", type=str, dest="model", default="", help="model snapshot location") parser.add_option("--state", type=str, dest="state", default="", help="state snapshot location") parser.add_option("--checkpoint", type=str, dest="checkpoint", default="", help="where to cache the model") parser.add_option("-o", "--overwrite", action="store_true", dest="overwrite", default=False, @@ -250,7 +250,17 @@ def config_option_parser(): init_engine() # build model - inception_model = inception_v1_no_aux_classifier(options.classNum) + if options.model != "": + inception_model = Model.load(options.model) + else: + inception_model = inception_v1_no_aux_classifier(options.classNum) + + # load state + if options.state != "": + optim = OptimMethod.load(options.state) + else: + optim = SGD(learningrate=options.learningRate, learningrate_decay=options.weightDecay, + momentum=0.9, dampening=0.0) image_size = 224 # create dataset train_transformer = Pipeline([PixelBytesToMat(), @@ -286,8 +296,7 @@ def config_option_parser(): optimizer = Optimizer.create( model=inception_model, training_set=train_data, - optim_method=SGD(learningrate=options.learningRate, learningrate_decay=options.weightDecay, - momentum=0.9, dampening=0.0), + optim_method=optim, criterion=ClassNLLCriterion(), end_trigger=end_trigger, batch_size=options.batchSize diff --git a/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py b/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py index e6e28b7d65e..286882920fd 100644 --- a/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py +++ b/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py @@ -72,8 +72,10 @@ def build_model(class_num): parameters = trained_model.parameters() elif options.action == "test": # Load a pre-trained model and then validate it through top1 accuracy. - test_data = get_mnist(sc, "test").map( - normalizer(mnist.TEST_MEAN, mnist.TEST_STD)) + test_data = get_mnist(sc, "test", options.dataPath) \ + .map(lambda rec_tuple: (normalizer(rec_tuple[0], mnist.TEST_MEAN, mnist.TEST_STD), + rec_tuple[1])) \ + .map(lambda t: Sample.from_ndarray(t[0], t[1])) model = Model.load(options.modelPath) results = model.evaluate(test_data, options.batchSize, [Top1Accuracy()]) for result in results: From 5d1e97ef3266457e0a5d5eafc6f9127a4f29c9d7 Mon Sep 17 00:00:00 2001 From: jenniew Date: Thu, 22 Feb 2018 17:12:41 -0800 Subject: [PATCH 615/823] update inception model --- .../bigdl/dllib/models/inception/inception.py | 35 ++++++++++--------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/models/inception/inception.py b/python/dllib/src/bigdl/dllib/models/inception/inception.py index d00b55f1a73..2cf2e0f4f60 100644 --- a/python/dllib/src/bigdl/dllib/models/inception/inception.py +++ b/python/dllib/src/bigdl/dllib/models/inception/inception.py @@ -19,36 +19,35 @@ def t(input_t): def inception_layer_v1(input_size, config, name_prefix=""): concat = Concat(2) conv1 = Sequential() - conv1.add( - SpatialConvolution(input_size, - config[1][1], - 1, 1, 1, 1).set_init_method(weight_init_method=Xavier()) - .set_name(name_prefix + "1x1")) + conv1.add(SpatialConvolution(input_size, config[1][1], 1, 1, 1, 1) + .set_init_method(weight_init_method=Xavier(),bias_init_method=Zeros()) + .set_name(name_prefix + "1x1")) conv1.add(ReLU(True).set_name(name_prefix + "relu_1x1")) concat.add(conv1) conv3 = Sequential() - conv3.add(SpatialConvolution(input_size, config[2][1], 1, 1, 1, 1).set_init_method(weight_init_method=Xavier()) + conv3.add(SpatialConvolution(input_size, config[2][1], 1, 1, 1, 1) + .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) .set_name(name_prefix + "3x3_reduce")) conv3.add(ReLU(True).set_name(name_prefix + "relu_3x3_reduce")) - conv3.add(SpatialConvolution(config[2][1], config[2][2], - 3, 3, 1, 1, 1, 1).set_init_method(weight_init_method=Xavier()) + conv3.add(SpatialConvolution(config[2][1], config[2][2], 3, 3, 1, 1, 1, 1) + .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) .set_name(name_prefix + "3x3")) conv3.add(ReLU(True).set_name(name_prefix + "relu_3x3")) concat.add(conv3) conv5 = Sequential() - conv5.add(SpatialConvolution(input_size, - config[3][1], 1, 1, 1, 1).set_init_method(weight_init_method=Xavier()) + conv5.add(SpatialConvolution(input_size, config[3][1], 1, 1, 1, 1) + .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) .set_name(name_prefix + "5x5_reduce")) conv5.add(ReLU(True).set_name(name_prefix + "relu_5x5_reduce")) - conv5.add(SpatialConvolution(config[3][1], - config[3][2], 5, 5, 1, 1, 2, 2).set_init_method(weight_init_method=Xavier()) + conv5.add(SpatialConvolution(config[3][1], config[3][2], 5, 5, 1, 1, 2, 2) + .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) .set_name(name_prefix + "5x5")) conv5.add(ReLU(True).set_name(name_prefix + "relu_5x5")) concat.add(conv5) pool = Sequential() - pool.add(SpatialMaxPooling(3, 3, 1, 1, 1, 1, - to_ceil=True).set_name(name_prefix + "pool")) - pool.add(SpatialConvolution(input_size, config[4][1], 1, 1, 1, 1).set_init_method(weight_init_method=Xavier()) + pool.add(SpatialMaxPooling(3, 3, 1, 1, 1, 1, to_ceil=True).set_name(name_prefix + "pool")) + pool.add(SpatialConvolution(input_size, config[4][1], 1, 1, 1, 1) + .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) .set_name(name_prefix + "pool_proj")) pool.add(ReLU(True).set_name(name_prefix + "relu_pool_proj")) concat.add(pool).set_name(name_prefix + "output") @@ -63,10 +62,12 @@ def inception_v1_no_aux_classifier(class_num): model.add(ReLU(True).set_name("conv1/relu_7x7")) model.add(SpatialMaxPooling(3, 3, 2, 2, to_ceil=True).set_name("pool1/3x3_s2")) model.add(SpatialCrossMapLRN(5, 0.0001, 0.75).set_name("pool1/norm1")) - model.add(SpatialConvolution(64, 64, 1, 1, 1, 1).set_init_method(weight_init_method=Xavier()) + model.add(SpatialConvolution(64, 64, 1, 1, 1, 1) + .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) .set_name("conv2/3x3_reduce")) model.add(ReLU(True).set_name("conv2/relu_3x3_reduce")) - model.add(SpatialConvolution(64, 192, 3, 3, 1, 1, 1, 1).set_init_method(weight_init_method=Xavier()) + model.add(SpatialConvolution(64, 192, 3, 3, 1, 1, 1, 1) + .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) .set_name("conv2/3x3")) model.add(ReLU(True).set_name("conv2/relu_3x3")) model.add(SpatialCrossMapLRN(5, 0.0001, 0.75).set_name("conv2/norm2")) From 0cd9d0af5a8a511d49669b3566ea45bbc1d605fa Mon Sep 17 00:00:00 2001 From: jenniew Date: Thu, 22 Feb 2018 17:12:41 -0800 Subject: [PATCH 616/823] update inception model --- .../bigdl/dllib/models/inception/inception.py | 35 ++++++++++--------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/models/inception/inception.py b/python/dllib/src/bigdl/dllib/models/inception/inception.py index d00b55f1a73..2cf2e0f4f60 100644 --- a/python/dllib/src/bigdl/dllib/models/inception/inception.py +++ b/python/dllib/src/bigdl/dllib/models/inception/inception.py @@ -19,36 +19,35 @@ def t(input_t): def inception_layer_v1(input_size, config, name_prefix=""): concat = Concat(2) conv1 = Sequential() - conv1.add( - SpatialConvolution(input_size, - config[1][1], - 1, 1, 1, 1).set_init_method(weight_init_method=Xavier()) - .set_name(name_prefix + "1x1")) + conv1.add(SpatialConvolution(input_size, config[1][1], 1, 1, 1, 1) + .set_init_method(weight_init_method=Xavier(),bias_init_method=Zeros()) + .set_name(name_prefix + "1x1")) conv1.add(ReLU(True).set_name(name_prefix + "relu_1x1")) concat.add(conv1) conv3 = Sequential() - conv3.add(SpatialConvolution(input_size, config[2][1], 1, 1, 1, 1).set_init_method(weight_init_method=Xavier()) + conv3.add(SpatialConvolution(input_size, config[2][1], 1, 1, 1, 1) + .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) .set_name(name_prefix + "3x3_reduce")) conv3.add(ReLU(True).set_name(name_prefix + "relu_3x3_reduce")) - conv3.add(SpatialConvolution(config[2][1], config[2][2], - 3, 3, 1, 1, 1, 1).set_init_method(weight_init_method=Xavier()) + conv3.add(SpatialConvolution(config[2][1], config[2][2], 3, 3, 1, 1, 1, 1) + .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) .set_name(name_prefix + "3x3")) conv3.add(ReLU(True).set_name(name_prefix + "relu_3x3")) concat.add(conv3) conv5 = Sequential() - conv5.add(SpatialConvolution(input_size, - config[3][1], 1, 1, 1, 1).set_init_method(weight_init_method=Xavier()) + conv5.add(SpatialConvolution(input_size, config[3][1], 1, 1, 1, 1) + .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) .set_name(name_prefix + "5x5_reduce")) conv5.add(ReLU(True).set_name(name_prefix + "relu_5x5_reduce")) - conv5.add(SpatialConvolution(config[3][1], - config[3][2], 5, 5, 1, 1, 2, 2).set_init_method(weight_init_method=Xavier()) + conv5.add(SpatialConvolution(config[3][1], config[3][2], 5, 5, 1, 1, 2, 2) + .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) .set_name(name_prefix + "5x5")) conv5.add(ReLU(True).set_name(name_prefix + "relu_5x5")) concat.add(conv5) pool = Sequential() - pool.add(SpatialMaxPooling(3, 3, 1, 1, 1, 1, - to_ceil=True).set_name(name_prefix + "pool")) - pool.add(SpatialConvolution(input_size, config[4][1], 1, 1, 1, 1).set_init_method(weight_init_method=Xavier()) + pool.add(SpatialMaxPooling(3, 3, 1, 1, 1, 1, to_ceil=True).set_name(name_prefix + "pool")) + pool.add(SpatialConvolution(input_size, config[4][1], 1, 1, 1, 1) + .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) .set_name(name_prefix + "pool_proj")) pool.add(ReLU(True).set_name(name_prefix + "relu_pool_proj")) concat.add(pool).set_name(name_prefix + "output") @@ -63,10 +62,12 @@ def inception_v1_no_aux_classifier(class_num): model.add(ReLU(True).set_name("conv1/relu_7x7")) model.add(SpatialMaxPooling(3, 3, 2, 2, to_ceil=True).set_name("pool1/3x3_s2")) model.add(SpatialCrossMapLRN(5, 0.0001, 0.75).set_name("pool1/norm1")) - model.add(SpatialConvolution(64, 64, 1, 1, 1, 1).set_init_method(weight_init_method=Xavier()) + model.add(SpatialConvolution(64, 64, 1, 1, 1, 1) + .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) .set_name("conv2/3x3_reduce")) model.add(ReLU(True).set_name("conv2/relu_3x3_reduce")) - model.add(SpatialConvolution(64, 192, 3, 3, 1, 1, 1, 1).set_init_method(weight_init_method=Xavier()) + model.add(SpatialConvolution(64, 192, 3, 3, 1, 1, 1, 1) + .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) .set_name("conv2/3x3")) model.add(ReLU(True).set_name("conv2/relu_3x3")) model.add(SpatialCrossMapLRN(5, 0.0001, 0.75).set_name("conv2/norm2")) From 07c4fa8f8faf225aa256b962cc222e5b2d8bc662 Mon Sep 17 00:00:00 2001 From: jenniew Date: Mon, 26 Feb 2018 15:35:18 -0800 Subject: [PATCH 617/823] update sgd learningrate policy --- .../bigdl/dllib/models/inception/inception.py | 32 +++++++++++-------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/models/inception/inception.py b/python/dllib/src/bigdl/dllib/models/inception/inception.py index 2cf2e0f4f60..5987b33b143 100644 --- a/python/dllib/src/bigdl/dllib/models/inception/inception.py +++ b/python/dllib/src/bigdl/dllib/models/inception/inception.py @@ -4,6 +4,7 @@ from bigdl.nn.initialization_method import * from bigdl.optim.optimizer import * from bigdl.transform.vision.image import * +from math import ceil def t(input_t): @@ -250,19 +251,6 @@ def config_option_parser(): show_bigdl_info_logs() init_engine() - # build model - if options.model != "": - inception_model = Model.load(options.model) - else: - inception_model = inception_v1_no_aux_classifier(options.classNum) - - # load state - if options.state != "": - optim = OptimMethod.load(options.state) - else: - optim = SGD(learningrate=options.learningRate, learningrate_decay=options.weightDecay, - momentum=0.9, dampening=0.0) - image_size = 224 # create dataset train_transformer = Pipeline([PixelBytesToMat(), RandomCrop(image_size, image_size), @@ -284,6 +272,24 @@ def config_option_parser(): raw_val_data = get_inception_data(options.folder, sc, "val") val_data = DataSet.image_frame(raw_val_data).transform(val_transformer) + # build model + if options.model != "": + inception_model = Model.load(options.model) + else: + inception_model = inception_v1_no_aux_classifier(options.classNum) + + # load state + if options.state != "": + optim = OptimMethod.load(options.state) + elif options.maxEpoch: + optim = SGD(learningrate=options.learningRate, learningrate_decay=options.weightDecay, + momentum=0.9, dampening=0.0, + leaningrate_schedule=Poly(0.5, ceil(float(1281167) / options.batchSize) * options.maxEpoch)) + else: + optim = SGD(learningrate=options.learningRate, learningrate_decay=options.weightDecay, + momentum=0.9, dampening=0.0, + leaningrate_schedule=Poly(0.5, options.maxIteration)) + if options.maxEpoch: checkpoint_trigger = EveryEpoch() test_trigger = EveryEpoch() From f43cc3f5a0c81142691058cfd64d240ea8444a10 Mon Sep 17 00:00:00 2001 From: jenniew Date: Mon, 26 Feb 2018 15:35:18 -0800 Subject: [PATCH 618/823] update sgd learningrate policy --- .../bigdl/dllib/models/inception/inception.py | 32 +++++++++++-------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/models/inception/inception.py b/python/dllib/src/bigdl/dllib/models/inception/inception.py index 2cf2e0f4f60..5987b33b143 100644 --- a/python/dllib/src/bigdl/dllib/models/inception/inception.py +++ b/python/dllib/src/bigdl/dllib/models/inception/inception.py @@ -4,6 +4,7 @@ from bigdl.nn.initialization_method import * from bigdl.optim.optimizer import * from bigdl.transform.vision.image import * +from math import ceil def t(input_t): @@ -250,19 +251,6 @@ def config_option_parser(): show_bigdl_info_logs() init_engine() - # build model - if options.model != "": - inception_model = Model.load(options.model) - else: - inception_model = inception_v1_no_aux_classifier(options.classNum) - - # load state - if options.state != "": - optim = OptimMethod.load(options.state) - else: - optim = SGD(learningrate=options.learningRate, learningrate_decay=options.weightDecay, - momentum=0.9, dampening=0.0) - image_size = 224 # create dataset train_transformer = Pipeline([PixelBytesToMat(), RandomCrop(image_size, image_size), @@ -284,6 +272,24 @@ def config_option_parser(): raw_val_data = get_inception_data(options.folder, sc, "val") val_data = DataSet.image_frame(raw_val_data).transform(val_transformer) + # build model + if options.model != "": + inception_model = Model.load(options.model) + else: + inception_model = inception_v1_no_aux_classifier(options.classNum) + + # load state + if options.state != "": + optim = OptimMethod.load(options.state) + elif options.maxEpoch: + optim = SGD(learningrate=options.learningRate, learningrate_decay=options.weightDecay, + momentum=0.9, dampening=0.0, + leaningrate_schedule=Poly(0.5, ceil(float(1281167) / options.batchSize) * options.maxEpoch)) + else: + optim = SGD(learningrate=options.learningRate, learningrate_decay=options.weightDecay, + momentum=0.9, dampening=0.0, + leaningrate_schedule=Poly(0.5, options.maxIteration)) + if options.maxEpoch: checkpoint_trigger = EveryEpoch() test_trigger = EveryEpoch() From 8f4bc17bf4de3a9011171be5125ad497ea06f1c6 Mon Sep 17 00:00:00 2001 From: jenniew Date: Mon, 26 Feb 2018 15:40:57 -0800 Subject: [PATCH 619/823] update init method --- python/dllib/src/bigdl/dllib/models/inception/inception.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/python/dllib/src/bigdl/dllib/models/inception/inception.py b/python/dllib/src/bigdl/dllib/models/inception/inception.py index 5987b33b143..c87efdf9600 100644 --- a/python/dllib/src/bigdl/dllib/models/inception/inception.py +++ b/python/dllib/src/bigdl/dllib/models/inception/inception.py @@ -96,7 +96,9 @@ def inception_v1_no_aux_classifier(class_num): model.add(SpatialAveragePooling(7, 7, 1, 1).set_name("pool5/7x7_s1")) model.add(Dropout(0.4).set_name("pool5/drop_7x7_s1")) model.add(View([1024], num_input_dims=3)) - model.add(Linear(1024, class_num).set_init_method(weight_init_method=Xavier()).set_name("loss3/classifier")) + model.add(Linear(1024, class_num) + .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) + .set_name("loss3/classifier")) model.add(LogSoftMax().set_name("loss3/loss3")) model.reset() return model From cb4aec76d850b98861b1ee834e3d0b5e8f721c0e Mon Sep 17 00:00:00 2001 From: jenniew Date: Mon, 26 Feb 2018 15:40:57 -0800 Subject: [PATCH 620/823] update init method --- python/dllib/src/bigdl/dllib/models/inception/inception.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/python/dllib/src/bigdl/dllib/models/inception/inception.py b/python/dllib/src/bigdl/dllib/models/inception/inception.py index 5987b33b143..c87efdf9600 100644 --- a/python/dllib/src/bigdl/dllib/models/inception/inception.py +++ b/python/dllib/src/bigdl/dllib/models/inception/inception.py @@ -96,7 +96,9 @@ def inception_v1_no_aux_classifier(class_num): model.add(SpatialAveragePooling(7, 7, 1, 1).set_name("pool5/7x7_s1")) model.add(Dropout(0.4).set_name("pool5/drop_7x7_s1")) model.add(View([1024], num_input_dims=3)) - model.add(Linear(1024, class_num).set_init_method(weight_init_method=Xavier()).set_name("loss3/classifier")) + model.add(Linear(1024, class_num) + .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) + .set_name("loss3/classifier")) model.add(LogSoftMax().set_name("loss3/loss3")) model.reset() return model From 0f42b6ad2ecb805e3db049dc36af4021fff156a4 Mon Sep 17 00:00:00 2001 From: jenniew Date: Tue, 27 Feb 2018 16:24:47 -0800 Subject: [PATCH 621/823] update sgd --- python/dllib/src/bigdl/dllib/models/inception/inception.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/models/inception/inception.py b/python/dllib/src/bigdl/dllib/models/inception/inception.py index c87efdf9600..7481082b29e 100644 --- a/python/dllib/src/bigdl/dllib/models/inception/inception.py +++ b/python/dllib/src/bigdl/dllib/models/inception/inception.py @@ -284,11 +284,11 @@ def config_option_parser(): if options.state != "": optim = OptimMethod.load(options.state) elif options.maxEpoch: - optim = SGD(learningrate=options.learningRate, learningrate_decay=options.weightDecay, + optim = SGD(learningrate=options.learningRate, learningrate_decay=0.0, weightdecay=options.weightDecay, momentum=0.9, dampening=0.0, leaningrate_schedule=Poly(0.5, ceil(float(1281167) / options.batchSize) * options.maxEpoch)) else: - optim = SGD(learningrate=options.learningRate, learningrate_decay=options.weightDecay, + optim = SGD(learningrate=options.learningRate, learningrate_decay=0.0, weightdecay=options.weightDecay, momentum=0.9, dampening=0.0, leaningrate_schedule=Poly(0.5, options.maxIteration)) From f064dc5e00f69485a85b6d48552a98d155df4068 Mon Sep 17 00:00:00 2001 From: jenniew Date: Tue, 27 Feb 2018 16:24:47 -0800 Subject: [PATCH 622/823] update sgd --- python/dllib/src/bigdl/dllib/models/inception/inception.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/models/inception/inception.py b/python/dllib/src/bigdl/dllib/models/inception/inception.py index c87efdf9600..7481082b29e 100644 --- a/python/dllib/src/bigdl/dllib/models/inception/inception.py +++ b/python/dllib/src/bigdl/dllib/models/inception/inception.py @@ -284,11 +284,11 @@ def config_option_parser(): if options.state != "": optim = OptimMethod.load(options.state) elif options.maxEpoch: - optim = SGD(learningrate=options.learningRate, learningrate_decay=options.weightDecay, + optim = SGD(learningrate=options.learningRate, learningrate_decay=0.0, weightdecay=options.weightDecay, momentum=0.9, dampening=0.0, leaningrate_schedule=Poly(0.5, ceil(float(1281167) / options.batchSize) * options.maxEpoch)) else: - optim = SGD(learningrate=options.learningRate, learningrate_decay=options.weightDecay, + optim = SGD(learningrate=options.learningRate, learningrate_decay=0.0, weightdecay=options.weightDecay, momentum=0.9, dampening=0.0, leaningrate_schedule=Poly(0.5, options.maxIteration)) From 0bcad06f3a4c2d5c6433c5608e2ed5ab5721c7f9 Mon Sep 17 00:00:00 2001 From: jenniew Date: Wed, 28 Feb 2018 15:34:02 -0800 Subject: [PATCH 623/823] update sgd with epoch --- python/dllib/src/bigdl/dllib/models/inception/inception.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/models/inception/inception.py b/python/dllib/src/bigdl/dllib/models/inception/inception.py index 7481082b29e..91896fb9112 100644 --- a/python/dllib/src/bigdl/dllib/models/inception/inception.py +++ b/python/dllib/src/bigdl/dllib/models/inception/inception.py @@ -285,11 +285,11 @@ def config_option_parser(): optim = OptimMethod.load(options.state) elif options.maxEpoch: optim = SGD(learningrate=options.learningRate, learningrate_decay=0.0, weightdecay=options.weightDecay, - momentum=0.9, dampening=0.0, - leaningrate_schedule=Poly(0.5, ceil(float(1281167) / options.batchSize) * options.maxEpoch)) + momentum=0.9, dampening=0.0, nesterov=False, + leaningrate_schedule=Poly(0.5, int(ceil(float(1281167) / options.batchSize)) * options.maxEpoch)) else: optim = SGD(learningrate=options.learningRate, learningrate_decay=0.0, weightdecay=options.weightDecay, - momentum=0.9, dampening=0.0, + momentum=0.9, dampening=0.0, nesterov=False, leaningrate_schedule=Poly(0.5, options.maxIteration)) if options.maxEpoch: From 1695210b69f5a521e696b992a2830391a509aed7 Mon Sep 17 00:00:00 2001 From: jenniew Date: Wed, 28 Feb 2018 15:34:02 -0800 Subject: [PATCH 624/823] update sgd with epoch --- python/dllib/src/bigdl/dllib/models/inception/inception.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/models/inception/inception.py b/python/dllib/src/bigdl/dllib/models/inception/inception.py index 7481082b29e..91896fb9112 100644 --- a/python/dllib/src/bigdl/dllib/models/inception/inception.py +++ b/python/dllib/src/bigdl/dllib/models/inception/inception.py @@ -285,11 +285,11 @@ def config_option_parser(): optim = OptimMethod.load(options.state) elif options.maxEpoch: optim = SGD(learningrate=options.learningRate, learningrate_decay=0.0, weightdecay=options.weightDecay, - momentum=0.9, dampening=0.0, - leaningrate_schedule=Poly(0.5, ceil(float(1281167) / options.batchSize) * options.maxEpoch)) + momentum=0.9, dampening=0.0, nesterov=False, + leaningrate_schedule=Poly(0.5, int(ceil(float(1281167) / options.batchSize)) * options.maxEpoch)) else: optim = SGD(learningrate=options.learningRate, learningrate_decay=0.0, weightdecay=options.weightDecay, - momentum=0.9, dampening=0.0, + momentum=0.9, dampening=0.0, nesterov=False, leaningrate_schedule=Poly(0.5, options.maxIteration)) if options.maxEpoch: From 2caab16b0d152beb3d636d65f6abbda76ca53260 Mon Sep 17 00:00:00 2001 From: jenniew Date: Mon, 12 Mar 2018 23:16:08 -0700 Subject: [PATCH 625/823] update inception python and scala example --- .../bigdl/dllib/models/inception/inception.py | 59 +++++++++++++------ 1 file changed, 42 insertions(+), 17 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/models/inception/inception.py b/python/dllib/src/bigdl/dllib/models/inception/inception.py index 91896fb9112..e3606ebe8a3 100644 --- a/python/dllib/src/bigdl/dllib/models/inception/inception.py +++ b/python/dllib/src/bigdl/dllib/models/inception/inception.py @@ -55,7 +55,7 @@ def inception_layer_v1(input_size, config, name_prefix=""): return concat -def inception_v1_no_aux_classifier(class_num): +def inception_v1_no_aux_classifier(class_num, has_dropout=True): model = Sequential() model.add(SpatialConvolution(3, 64, 7, 7, 2, 2, 3, 3, 1, False) .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) @@ -94,7 +94,8 @@ def inception_v1_no_aux_classifier(class_num): model.add(inception_layer_v1(832, t([t([384]), t( [192, 384]), t([48, 128]), t([128])]), "inception_5b/")) model.add(SpatialAveragePooling(7, 7, 1, 1).set_name("pool5/7x7_s1")) - model.add(Dropout(0.4).set_name("pool5/drop_7x7_s1")) + if has_dropout: + model.add(Dropout(0.4).set_name("pool5/drop_7x7_s1")) model.add(View([1024], num_input_dims=3)) model.add(Linear(1024, class_num) .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) @@ -104,9 +105,10 @@ def inception_v1_no_aux_classifier(class_num): return model -def inception_v1(class_num): +def inception_v1(class_num, has_dropout=True): feature1 = Sequential() - feature1.add(SpatialConvolution(3, 64, 7, 7, 2, 2, 3, 3, 1, False).set_init_method(weight_init_method=Xavier()) + feature1.add(SpatialConvolution(3, 64, 7, 7, 2, 2, 3, 3, 1, False) + .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) .set_name("conv1/7x7_s2")) feature1.add(ReLU(True).set_name("conv1/relu_7x7")) feature1.add( @@ -114,11 +116,12 @@ def inception_v1(class_num): .set_name("pool1/3x3_s2")) feature1.add(SpatialCrossMapLRN(5, 0.0001, 0.75) .set_name("pool1/norm1")) - feature1.add(SpatialConvolution(64, 64, 1, 1, 1, 1).set_init_method(weight_init_method=Xavier()) + feature1.add(SpatialConvolution(64, 64, 1, 1, 1, 1) + .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) .set_name("conv2/3x3_reduce")) feature1.add(ReLU(True).set_name("conv2/relu_3x3_reduce")) - feature1.add(SpatialConvolution(64, 192, 3, 3, 1, - 1, 1, 1).set_init_method(weight_init_method=Xavier()) + feature1.add(SpatialConvolution(64, 192, 3, 3, 1, 1, 1, 1) + .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) .set_name("conv2/3x3")) feature1.add(ReLU(True).set_name("conv2/relu_3x3")) feature1.add(SpatialCrossMapLRN(5, 0.0001, 0.75).set_name("conv2/norm2")) @@ -145,7 +148,8 @@ def inception_v1(class_num): output1.add(View([128 * 4 * 4, 3])) output1.add(Linear(128 * 4 * 4, 1024).set_name("loss1/fc")) output1.add(ReLU(True).set_name("loss1/relu_fc")) - output1.add(Dropout(0.7).set_name("loss1/drop_fc")) + if has_dropout: + output1.add(Dropout(0.7).set_name("loss1/drop_fc")) output1.add(Linear(1024, class_num).set_name("loss1/classifier")) output1.add(LogSoftMax().set_name("loss1/loss")) @@ -168,7 +172,8 @@ def inception_v1(class_num): output2.add(View([128 * 4 * 4, 3])) output2.add(Linear(128 * 4 * 4, 1024).set_name("loss2/fc")) output2.add(ReLU(True).set_name("loss2/relu_fc")) - output2.add(Dropout(0.7).set_name("loss2/drop_fc")) + if has_dropout: + output2.add(Dropout(0.7).set_name("loss2/drop_fc")) output2.add(Linear(1024, class_num).set_name("loss2/classifier")) output2.add(LogSoftMax().set_name("loss2/loss")) @@ -184,9 +189,11 @@ def inception_v1(class_num): t([t([384]), t([192, 384]), t([48, 128]), t([128])]), "inception_5b/")) output3.add(SpatialAveragePooling(7, 7, 1, 1).set_name("pool5/7x7_s1")) - output3.add(Dropout(0.4).set_name("pool5/drop_7x7_s1")) + if has_dropout: + output3.add(Dropout(0.4).set_name("pool5/drop_7x7_s1")) output3.add(View([1024, 3])) - output3.add(Linear(1024, class_num).set_init_method(weight_init_method=Xavier()) + output3.add(Linear(1024, class_num) + .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) .set_name("loss3/classifier")) output3.add(LogSoftMax().set_name("loss3/loss3")) @@ -228,6 +235,8 @@ def config_option_parser(): parser.add_option("-e", "--maxEpoch", type=int, dest="maxEpoch", default=0, help="epoch numbers") parser.add_option("-i", "--maxIteration", type=int, dest="maxIteration", default=62000, help="iteration numbers") parser.add_option("-l", "--learningRate", type=float, dest="learningRate", default=0.01, help="learning rate") + parser.add_option("--warmupEpoch", type=int, dest="warmupEpoch", default=0, help="warm up epoch numbers") + parser.add_option("--maxLr", type=float, dest="maxLr", default=0.0, help="max Lr after warm up") parser.add_option("-b", "--batchSize", type=int, dest="batchSize", help="batch size") parser.add_option("--classNum", type=int, dest="classNum", default=1000, help="class number") parser.add_option("--weightDecay", type=float, dest="weightDecay", default=0.0001, help="weight decay") @@ -255,6 +264,7 @@ def config_option_parser(): image_size = 224 # create dataset train_transformer = Pipeline([PixelBytesToMat(), + Resize(256, 256), RandomCrop(image_size, image_size), RandomTransformer(HFlip(), 0.5), ChannelNormalize(0.485, 0.456, 0.406, 0.229, 0.224, 0.225), @@ -265,11 +275,12 @@ def config_option_parser(): train_data = DataSet.image_frame(raw_train_data).transform(train_transformer) val_transformer = Pipeline([PixelBytesToMat(), + Resize(256, 256), CenterCrop(image_size, image_size), RandomTransformer(HFlip(), 0.5), ChannelNormalize(0.485, 0.456, 0.406, 0.229, 0.224, 0.225), MatToTensor(to_rgb=True), - ImageFrameToSample(input_keys=["imageTensor"], target_keys=["label"]) + ImageFrameToSample(input_keys=["imageTensor"], target_keys=["label"]) ]) raw_val_data = get_inception_data(options.folder, sc, "val") val_data = DataSet.image_frame(raw_val_data).transform(val_transformer) @@ -281,16 +292,30 @@ def config_option_parser(): inception_model = inception_v1_no_aux_classifier(options.classNum) # load state + iterationPerEpoch = int(ceil(float(1281167) / options.batchSize)) + if options.maxEpoch: + maxIteration = iterationPerEpoch * options.maxEpoch + else: + maxIteration = options.maxIteration + warmup_iteration = options.warmupEpoch * iterationPerEpoch if options.state != "": optim = OptimMethod.load(options.state) - elif options.maxEpoch: - optim = SGD(learningrate=options.learningRate, learningrate_decay=0.0, weightdecay=options.weightDecay, - momentum=0.9, dampening=0.0, nesterov=False, - leaningrate_schedule=Poly(0.5, int(ceil(float(1281167) / options.batchSize)) * options.maxEpoch)) else: + if warmup_iteration == 0: + warmupDelta = 0.0 + else: + if options.maxLr: + maxlr = options.maxLr + else: + maxlr = options.learningRate + warmupDelta = (maxlr - options.learningRate)/warmup_iteration + polyIteration = maxIteration - warmup_iteration + lrSchedule = SequentialSchedule(iterationPerEpoch) + lrSchedule.add(Warmup(warmupDelta), warmup_iteration) + lrSchedule.add(Poly(0.5, polyIteration), polyIteration) optim = SGD(learningrate=options.learningRate, learningrate_decay=0.0, weightdecay=options.weightDecay, momentum=0.9, dampening=0.0, nesterov=False, - leaningrate_schedule=Poly(0.5, options.maxIteration)) + leaningrate_schedule=lrSchedule) if options.maxEpoch: checkpoint_trigger = EveryEpoch() From 96e894ca383ea543d2ec06c9824ac630a9ac0a1a Mon Sep 17 00:00:00 2001 From: jenniew Date: Mon, 12 Mar 2018 23:16:08 -0700 Subject: [PATCH 626/823] update inception python and scala example --- .../bigdl/dllib/models/inception/inception.py | 59 +++++++++++++------ 1 file changed, 42 insertions(+), 17 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/models/inception/inception.py b/python/dllib/src/bigdl/dllib/models/inception/inception.py index 91896fb9112..e3606ebe8a3 100644 --- a/python/dllib/src/bigdl/dllib/models/inception/inception.py +++ b/python/dllib/src/bigdl/dllib/models/inception/inception.py @@ -55,7 +55,7 @@ def inception_layer_v1(input_size, config, name_prefix=""): return concat -def inception_v1_no_aux_classifier(class_num): +def inception_v1_no_aux_classifier(class_num, has_dropout=True): model = Sequential() model.add(SpatialConvolution(3, 64, 7, 7, 2, 2, 3, 3, 1, False) .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) @@ -94,7 +94,8 @@ def inception_v1_no_aux_classifier(class_num): model.add(inception_layer_v1(832, t([t([384]), t( [192, 384]), t([48, 128]), t([128])]), "inception_5b/")) model.add(SpatialAveragePooling(7, 7, 1, 1).set_name("pool5/7x7_s1")) - model.add(Dropout(0.4).set_name("pool5/drop_7x7_s1")) + if has_dropout: + model.add(Dropout(0.4).set_name("pool5/drop_7x7_s1")) model.add(View([1024], num_input_dims=3)) model.add(Linear(1024, class_num) .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) @@ -104,9 +105,10 @@ def inception_v1_no_aux_classifier(class_num): return model -def inception_v1(class_num): +def inception_v1(class_num, has_dropout=True): feature1 = Sequential() - feature1.add(SpatialConvolution(3, 64, 7, 7, 2, 2, 3, 3, 1, False).set_init_method(weight_init_method=Xavier()) + feature1.add(SpatialConvolution(3, 64, 7, 7, 2, 2, 3, 3, 1, False) + .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) .set_name("conv1/7x7_s2")) feature1.add(ReLU(True).set_name("conv1/relu_7x7")) feature1.add( @@ -114,11 +116,12 @@ def inception_v1(class_num): .set_name("pool1/3x3_s2")) feature1.add(SpatialCrossMapLRN(5, 0.0001, 0.75) .set_name("pool1/norm1")) - feature1.add(SpatialConvolution(64, 64, 1, 1, 1, 1).set_init_method(weight_init_method=Xavier()) + feature1.add(SpatialConvolution(64, 64, 1, 1, 1, 1) + .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) .set_name("conv2/3x3_reduce")) feature1.add(ReLU(True).set_name("conv2/relu_3x3_reduce")) - feature1.add(SpatialConvolution(64, 192, 3, 3, 1, - 1, 1, 1).set_init_method(weight_init_method=Xavier()) + feature1.add(SpatialConvolution(64, 192, 3, 3, 1, 1, 1, 1) + .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) .set_name("conv2/3x3")) feature1.add(ReLU(True).set_name("conv2/relu_3x3")) feature1.add(SpatialCrossMapLRN(5, 0.0001, 0.75).set_name("conv2/norm2")) @@ -145,7 +148,8 @@ def inception_v1(class_num): output1.add(View([128 * 4 * 4, 3])) output1.add(Linear(128 * 4 * 4, 1024).set_name("loss1/fc")) output1.add(ReLU(True).set_name("loss1/relu_fc")) - output1.add(Dropout(0.7).set_name("loss1/drop_fc")) + if has_dropout: + output1.add(Dropout(0.7).set_name("loss1/drop_fc")) output1.add(Linear(1024, class_num).set_name("loss1/classifier")) output1.add(LogSoftMax().set_name("loss1/loss")) @@ -168,7 +172,8 @@ def inception_v1(class_num): output2.add(View([128 * 4 * 4, 3])) output2.add(Linear(128 * 4 * 4, 1024).set_name("loss2/fc")) output2.add(ReLU(True).set_name("loss2/relu_fc")) - output2.add(Dropout(0.7).set_name("loss2/drop_fc")) + if has_dropout: + output2.add(Dropout(0.7).set_name("loss2/drop_fc")) output2.add(Linear(1024, class_num).set_name("loss2/classifier")) output2.add(LogSoftMax().set_name("loss2/loss")) @@ -184,9 +189,11 @@ def inception_v1(class_num): t([t([384]), t([192, 384]), t([48, 128]), t([128])]), "inception_5b/")) output3.add(SpatialAveragePooling(7, 7, 1, 1).set_name("pool5/7x7_s1")) - output3.add(Dropout(0.4).set_name("pool5/drop_7x7_s1")) + if has_dropout: + output3.add(Dropout(0.4).set_name("pool5/drop_7x7_s1")) output3.add(View([1024, 3])) - output3.add(Linear(1024, class_num).set_init_method(weight_init_method=Xavier()) + output3.add(Linear(1024, class_num) + .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) .set_name("loss3/classifier")) output3.add(LogSoftMax().set_name("loss3/loss3")) @@ -228,6 +235,8 @@ def config_option_parser(): parser.add_option("-e", "--maxEpoch", type=int, dest="maxEpoch", default=0, help="epoch numbers") parser.add_option("-i", "--maxIteration", type=int, dest="maxIteration", default=62000, help="iteration numbers") parser.add_option("-l", "--learningRate", type=float, dest="learningRate", default=0.01, help="learning rate") + parser.add_option("--warmupEpoch", type=int, dest="warmupEpoch", default=0, help="warm up epoch numbers") + parser.add_option("--maxLr", type=float, dest="maxLr", default=0.0, help="max Lr after warm up") parser.add_option("-b", "--batchSize", type=int, dest="batchSize", help="batch size") parser.add_option("--classNum", type=int, dest="classNum", default=1000, help="class number") parser.add_option("--weightDecay", type=float, dest="weightDecay", default=0.0001, help="weight decay") @@ -255,6 +264,7 @@ def config_option_parser(): image_size = 224 # create dataset train_transformer = Pipeline([PixelBytesToMat(), + Resize(256, 256), RandomCrop(image_size, image_size), RandomTransformer(HFlip(), 0.5), ChannelNormalize(0.485, 0.456, 0.406, 0.229, 0.224, 0.225), @@ -265,11 +275,12 @@ def config_option_parser(): train_data = DataSet.image_frame(raw_train_data).transform(train_transformer) val_transformer = Pipeline([PixelBytesToMat(), + Resize(256, 256), CenterCrop(image_size, image_size), RandomTransformer(HFlip(), 0.5), ChannelNormalize(0.485, 0.456, 0.406, 0.229, 0.224, 0.225), MatToTensor(to_rgb=True), - ImageFrameToSample(input_keys=["imageTensor"], target_keys=["label"]) + ImageFrameToSample(input_keys=["imageTensor"], target_keys=["label"]) ]) raw_val_data = get_inception_data(options.folder, sc, "val") val_data = DataSet.image_frame(raw_val_data).transform(val_transformer) @@ -281,16 +292,30 @@ def config_option_parser(): inception_model = inception_v1_no_aux_classifier(options.classNum) # load state + iterationPerEpoch = int(ceil(float(1281167) / options.batchSize)) + if options.maxEpoch: + maxIteration = iterationPerEpoch * options.maxEpoch + else: + maxIteration = options.maxIteration + warmup_iteration = options.warmupEpoch * iterationPerEpoch if options.state != "": optim = OptimMethod.load(options.state) - elif options.maxEpoch: - optim = SGD(learningrate=options.learningRate, learningrate_decay=0.0, weightdecay=options.weightDecay, - momentum=0.9, dampening=0.0, nesterov=False, - leaningrate_schedule=Poly(0.5, int(ceil(float(1281167) / options.batchSize)) * options.maxEpoch)) else: + if warmup_iteration == 0: + warmupDelta = 0.0 + else: + if options.maxLr: + maxlr = options.maxLr + else: + maxlr = options.learningRate + warmupDelta = (maxlr - options.learningRate)/warmup_iteration + polyIteration = maxIteration - warmup_iteration + lrSchedule = SequentialSchedule(iterationPerEpoch) + lrSchedule.add(Warmup(warmupDelta), warmup_iteration) + lrSchedule.add(Poly(0.5, polyIteration), polyIteration) optim = SGD(learningrate=options.learningRate, learningrate_decay=0.0, weightdecay=options.weightDecay, momentum=0.9, dampening=0.0, nesterov=False, - leaningrate_schedule=Poly(0.5, options.maxIteration)) + leaningrate_schedule=lrSchedule) if options.maxEpoch: checkpoint_trigger = EveryEpoch() From 6ce7dcb8496605526bcdd5a007eb70f890b5d248 Mon Sep 17 00:00:00 2001 From: jenniew Date: Tue, 13 Mar 2018 15:36:25 -0700 Subject: [PATCH 627/823] add gradient options as scala version --- .../dllib/src/bigdl/dllib/models/inception/inception.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/models/inception/inception.py b/python/dllib/src/bigdl/dllib/models/inception/inception.py index e3606ebe8a3..908b4150d9f 100644 --- a/python/dllib/src/bigdl/dllib/models/inception/inception.py +++ b/python/dllib/src/bigdl/dllib/models/inception/inception.py @@ -242,6 +242,9 @@ def config_option_parser(): parser.add_option("--weightDecay", type=float, dest="weightDecay", default=0.0001, help="weight decay") parser.add_option("--checkpointIteration", type=int, dest="checkpointIteration", default=620, help="checkpoint interval of iterations") + parser.add_option("--gradientMin", type=float, dest="gradientMin", default=0.0, help="min gradient clipping by") + parser.add_option("--gradientMax", type=float, dest="gradientMax", default=0.0, help="max gradient clipping by") + parser.add_option("--gradientL2NormThreshold", type=float, dest="gradientL2NormThreshold", default=0.0, help="gradient L2-Norm threshold") return parser @@ -339,6 +342,12 @@ def config_option_parser(): if options.checkpoint: optimizer.set_checkpoint(checkpoint_trigger, options.checkpoint, options.overwrite) + if options.gradientMin and options.gradientMax: + optimizer.set_gradclip_const(options.gradientMin, options.gradientMax) + + if options.gradientL2NormThreshold: + optimizer.set_gradclip_l2norm(options.gradientL2NormThreshold) + optimizer.set_validation(trigger=test_trigger, val_rdd=val_data, batch_size=options.batchSize, From 61a65141ca095e349b13d516c41b0140ee8698bd Mon Sep 17 00:00:00 2001 From: jenniew Date: Tue, 13 Mar 2018 15:36:25 -0700 Subject: [PATCH 628/823] add gradient options as scala version --- .../dllib/src/bigdl/dllib/models/inception/inception.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/models/inception/inception.py b/python/dllib/src/bigdl/dllib/models/inception/inception.py index e3606ebe8a3..908b4150d9f 100644 --- a/python/dllib/src/bigdl/dllib/models/inception/inception.py +++ b/python/dllib/src/bigdl/dllib/models/inception/inception.py @@ -242,6 +242,9 @@ def config_option_parser(): parser.add_option("--weightDecay", type=float, dest="weightDecay", default=0.0001, help="weight decay") parser.add_option("--checkpointIteration", type=int, dest="checkpointIteration", default=620, help="checkpoint interval of iterations") + parser.add_option("--gradientMin", type=float, dest="gradientMin", default=0.0, help="min gradient clipping by") + parser.add_option("--gradientMax", type=float, dest="gradientMax", default=0.0, help="max gradient clipping by") + parser.add_option("--gradientL2NormThreshold", type=float, dest="gradientL2NormThreshold", default=0.0, help="gradient L2-Norm threshold") return parser @@ -339,6 +342,12 @@ def config_option_parser(): if options.checkpoint: optimizer.set_checkpoint(checkpoint_trigger, options.checkpoint, options.overwrite) + if options.gradientMin and options.gradientMax: + optimizer.set_gradclip_const(options.gradientMin, options.gradientMax) + + if options.gradientL2NormThreshold: + optimizer.set_gradclip_l2norm(options.gradientL2NormThreshold) + optimizer.set_validation(trigger=test_trigger, val_rdd=val_data, batch_size=options.batchSize, From 41e562b9d3ba742f8cbb6c3600b6449f98191bec Mon Sep 17 00:00:00 2001 From: jenniew Date: Wed, 14 Mar 2018 11:30:53 -0700 Subject: [PATCH 629/823] update transformer --- python/dllib/src/bigdl/dllib/models/inception/inception.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/models/inception/inception.py b/python/dllib/src/bigdl/dllib/models/inception/inception.py index 908b4150d9f..821f6b029e2 100644 --- a/python/dllib/src/bigdl/dllib/models/inception/inception.py +++ b/python/dllib/src/bigdl/dllib/models/inception/inception.py @@ -267,7 +267,6 @@ def config_option_parser(): image_size = 224 # create dataset train_transformer = Pipeline([PixelBytesToMat(), - Resize(256, 256), RandomCrop(image_size, image_size), RandomTransformer(HFlip(), 0.5), ChannelNormalize(0.485, 0.456, 0.406, 0.229, 0.224, 0.225), @@ -278,7 +277,6 @@ def config_option_parser(): train_data = DataSet.image_frame(raw_train_data).transform(train_transformer) val_transformer = Pipeline([PixelBytesToMat(), - Resize(256, 256), CenterCrop(image_size, image_size), RandomTransformer(HFlip(), 0.5), ChannelNormalize(0.485, 0.456, 0.406, 0.229, 0.224, 0.225), From beab12c89f41836777b851866b3cea10fae7124d Mon Sep 17 00:00:00 2001 From: jenniew Date: Wed, 14 Mar 2018 11:30:53 -0700 Subject: [PATCH 630/823] update transformer --- python/dllib/src/bigdl/dllib/models/inception/inception.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/models/inception/inception.py b/python/dllib/src/bigdl/dllib/models/inception/inception.py index 908b4150d9f..821f6b029e2 100644 --- a/python/dllib/src/bigdl/dllib/models/inception/inception.py +++ b/python/dllib/src/bigdl/dllib/models/inception/inception.py @@ -267,7 +267,6 @@ def config_option_parser(): image_size = 224 # create dataset train_transformer = Pipeline([PixelBytesToMat(), - Resize(256, 256), RandomCrop(image_size, image_size), RandomTransformer(HFlip(), 0.5), ChannelNormalize(0.485, 0.456, 0.406, 0.229, 0.224, 0.225), @@ -278,7 +277,6 @@ def config_option_parser(): train_data = DataSet.image_frame(raw_train_data).transform(train_transformer) val_transformer = Pipeline([PixelBytesToMat(), - Resize(256, 256), CenterCrop(image_size, image_size), RandomTransformer(HFlip(), 0.5), ChannelNormalize(0.485, 0.456, 0.406, 0.229, 0.224, 0.225), From fd5ea73a29c147c3fd6fbe320b7a783630681dc7 Mon Sep 17 00:00:00 2001 From: jenniew Date: Fri, 16 Mar 2018 17:04:54 -0700 Subject: [PATCH 631/823] update doc and comments --- .../src/bigdl/dllib/models/inception/README.md | 16 ++++++++-------- .../bigdl/dllib/models/inception/inception.py | 5 ++++- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/models/inception/README.md b/python/dllib/src/bigdl/dllib/models/inception/README.md index 4f53e686b6f..2a2f7d6c4fd 100644 --- a/python/dllib/src/bigdl/dllib/models/inception/README.md +++ b/python/dllib/src/bigdl/dllib/models/inception/README.md @@ -2,14 +2,14 @@ This example demonstrates how to use BigDL to train [Inception v1](https://arxiv.org/abs/1409.4842) architecture on the [ImageNet](http://image-net.org/index) data. ## Get the JAR You can build one by refer to the -[Build Page](https://github.com/intel-analytics/BigDL/wiki/Build-Page) from the source code. We +[Build Page](https://bigdl-project.github.io/master/#ScalaUserGuide/install-build-src/) from the source code. We will release a pre-build package soon. ## Prepare the data You can download imagenet-2012 data from . After you download the files(**ILSVRC2012_img_train.tar** and **ILSVRC2012_img_val.tar**), -run the follow commands to prepare the data. +run the following commands to prepare the data. ```bash mkdir train @@ -29,13 +29,13 @@ cat img_class.lst | while read PARAM; do mv ${PARAM/ n[0-9]*/} ${PARAM/ILSVRC*JP rm ILSVRC2012_img_val.tar ``` -Now all the images belong to the same category are moved to the same folder. +Now all the images belonging to the same category are moved to the same folder. This command will transform the images into hadoop sequence files, which are more suitable for a distributed training. ```bash -java -cp bigdl_folder/lib/bigdl-VERSION-jar-with-dependencies-and-spark.jar com.intel.analytics.bigdl.models.utils.ImageNetSeqFileGenerator -r -f imagenet_folder -o output_folder -p cores_number +java -cp bigdl_source_folder/spark/dl/target/bigdl-VERSION-jar-with-dependencies-and-spark.jar com.intel.analytics.bigdl.models.utils.ImageNetSeqFileGenerator -f imagenet_folder -o output_folder -p cores_number ``` It will generate the hadoop sequence files in the output folder. @@ -58,7 +58,7 @@ ${SPARK_HOME}/bin/spark-submit \ --properties-file ${BigDL_HOME}/dist/conf/spark-bigdl.conf \ --jars ${BigDL_JAR_PATH} \ --py-files ${PYTHON_API_PATH} \ -${BigDL_HOME}/pyspark/dl/models/inception/inception.py \ +${BigDL_HOME}/pyspark/bigdl/models/inception/inception.py \ -f hdfs://... \ --batchSize 1792 \ --learningRate 0.0898 \ @@ -87,7 +87,7 @@ ${SPARK_HOME}/bin/spark-submit \ --conf spark.driver.extraClassPath=${BigDL_JAR_PATH} \ --conf spark.executor.extraClassPath=bigdl-VERSION-jar-with-dependencies.jar \ --py-files ${PYTHON_API_PATH} \ -${BigDL_HOME}/pyspark/dl/models/inception/inception.py \ +${BigDL_HOME}/pyspark/bigdl/models/inception/inception.py \ -f hdfs://... \ --batchSize 1792 \ --learningRate 0.0898 \ @@ -102,8 +102,8 @@ In the above commands * -f: where you put your ImageNet data, it should be a hdfs folder * --checkpoint: Where you cache the model/train_state snapshot. You should input a folder and make sure the folder is created when you run this example. The model snapshot will be named as -model.#iteration_number, and train state will be named as state.#iteration_number. Note that if -there are some files already exist in the folder, the old file will not be overwrite for the +model.#iteration_number, and train state will be named as optimMethod.#iteration_number. Note that if +there are some files already exist in the folder, the old file will not be overwritten for the safety of your model files. * --batchSize: The mini-batch size. It is expected that the mini-batch size is a multiple of node_number * core_number. In this example, node_number is 1 and the mini-batch size is suggested to be set to core_number * 4 diff --git a/python/dllib/src/bigdl/dllib/models/inception/inception.py b/python/dllib/src/bigdl/dllib/models/inception/inception.py index 821f6b029e2..1227f0604fa 100644 --- a/python/dllib/src/bigdl/dllib/models/inception/inception.py +++ b/python/dllib/src/bigdl/dllib/models/inception/inception.py @@ -288,11 +288,12 @@ def config_option_parser(): # build model if options.model != "": + # load model snapshot inception_model = Model.load(options.model) else: inception_model = inception_v1_no_aux_classifier(options.classNum) - # load state + # set optimization method iterationPerEpoch = int(ceil(float(1281167) / options.batchSize)) if options.maxEpoch: maxIteration = iterationPerEpoch * options.maxEpoch @@ -300,6 +301,7 @@ def config_option_parser(): maxIteration = options.maxIteration warmup_iteration = options.warmupEpoch * iterationPerEpoch if options.state != "": + # load state snapshot optim = OptimMethod.load(options.state) else: if warmup_iteration == 0: @@ -318,6 +320,7 @@ def config_option_parser(): momentum=0.9, dampening=0.0, nesterov=False, leaningrate_schedule=lrSchedule) + # create triggers if options.maxEpoch: checkpoint_trigger = EveryEpoch() test_trigger = EveryEpoch() From b94a69a9e68b0a55eed778f39256ff8f64a9e50b Mon Sep 17 00:00:00 2001 From: jenniew Date: Fri, 16 Mar 2018 17:04:54 -0700 Subject: [PATCH 632/823] update doc and comments --- .../src/bigdl/dllib/models/inception/README.md | 16 ++++++++-------- .../bigdl/dllib/models/inception/inception.py | 5 ++++- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/models/inception/README.md b/python/dllib/src/bigdl/dllib/models/inception/README.md index 4f53e686b6f..2a2f7d6c4fd 100644 --- a/python/dllib/src/bigdl/dllib/models/inception/README.md +++ b/python/dllib/src/bigdl/dllib/models/inception/README.md @@ -2,14 +2,14 @@ This example demonstrates how to use BigDL to train [Inception v1](https://arxiv.org/abs/1409.4842) architecture on the [ImageNet](http://image-net.org/index) data. ## Get the JAR You can build one by refer to the -[Build Page](https://github.com/intel-analytics/BigDL/wiki/Build-Page) from the source code. We +[Build Page](https://bigdl-project.github.io/master/#ScalaUserGuide/install-build-src/) from the source code. We will release a pre-build package soon. ## Prepare the data You can download imagenet-2012 data from . After you download the files(**ILSVRC2012_img_train.tar** and **ILSVRC2012_img_val.tar**), -run the follow commands to prepare the data. +run the following commands to prepare the data. ```bash mkdir train @@ -29,13 +29,13 @@ cat img_class.lst | while read PARAM; do mv ${PARAM/ n[0-9]*/} ${PARAM/ILSVRC*JP rm ILSVRC2012_img_val.tar ``` -Now all the images belong to the same category are moved to the same folder. +Now all the images belonging to the same category are moved to the same folder. This command will transform the images into hadoop sequence files, which are more suitable for a distributed training. ```bash -java -cp bigdl_folder/lib/bigdl-VERSION-jar-with-dependencies-and-spark.jar com.intel.analytics.bigdl.models.utils.ImageNetSeqFileGenerator -r -f imagenet_folder -o output_folder -p cores_number +java -cp bigdl_source_folder/spark/dl/target/bigdl-VERSION-jar-with-dependencies-and-spark.jar com.intel.analytics.bigdl.models.utils.ImageNetSeqFileGenerator -f imagenet_folder -o output_folder -p cores_number ``` It will generate the hadoop sequence files in the output folder. @@ -58,7 +58,7 @@ ${SPARK_HOME}/bin/spark-submit \ --properties-file ${BigDL_HOME}/dist/conf/spark-bigdl.conf \ --jars ${BigDL_JAR_PATH} \ --py-files ${PYTHON_API_PATH} \ -${BigDL_HOME}/pyspark/dl/models/inception/inception.py \ +${BigDL_HOME}/pyspark/bigdl/models/inception/inception.py \ -f hdfs://... \ --batchSize 1792 \ --learningRate 0.0898 \ @@ -87,7 +87,7 @@ ${SPARK_HOME}/bin/spark-submit \ --conf spark.driver.extraClassPath=${BigDL_JAR_PATH} \ --conf spark.executor.extraClassPath=bigdl-VERSION-jar-with-dependencies.jar \ --py-files ${PYTHON_API_PATH} \ -${BigDL_HOME}/pyspark/dl/models/inception/inception.py \ +${BigDL_HOME}/pyspark/bigdl/models/inception/inception.py \ -f hdfs://... \ --batchSize 1792 \ --learningRate 0.0898 \ @@ -102,8 +102,8 @@ In the above commands * -f: where you put your ImageNet data, it should be a hdfs folder * --checkpoint: Where you cache the model/train_state snapshot. You should input a folder and make sure the folder is created when you run this example. The model snapshot will be named as -model.#iteration_number, and train state will be named as state.#iteration_number. Note that if -there are some files already exist in the folder, the old file will not be overwrite for the +model.#iteration_number, and train state will be named as optimMethod.#iteration_number. Note that if +there are some files already exist in the folder, the old file will not be overwritten for the safety of your model files. * --batchSize: The mini-batch size. It is expected that the mini-batch size is a multiple of node_number * core_number. In this example, node_number is 1 and the mini-batch size is suggested to be set to core_number * 4 diff --git a/python/dllib/src/bigdl/dllib/models/inception/inception.py b/python/dllib/src/bigdl/dllib/models/inception/inception.py index 821f6b029e2..1227f0604fa 100644 --- a/python/dllib/src/bigdl/dllib/models/inception/inception.py +++ b/python/dllib/src/bigdl/dllib/models/inception/inception.py @@ -288,11 +288,12 @@ def config_option_parser(): # build model if options.model != "": + # load model snapshot inception_model = Model.load(options.model) else: inception_model = inception_v1_no_aux_classifier(options.classNum) - # load state + # set optimization method iterationPerEpoch = int(ceil(float(1281167) / options.batchSize)) if options.maxEpoch: maxIteration = iterationPerEpoch * options.maxEpoch @@ -300,6 +301,7 @@ def config_option_parser(): maxIteration = options.maxIteration warmup_iteration = options.warmupEpoch * iterationPerEpoch if options.state != "": + # load state snapshot optim = OptimMethod.load(options.state) else: if warmup_iteration == 0: @@ -318,6 +320,7 @@ def config_option_parser(): momentum=0.9, dampening=0.0, nesterov=False, leaningrate_schedule=lrSchedule) + # create triggers if options.maxEpoch: checkpoint_trigger = EveryEpoch() test_trigger = EveryEpoch() From 52c037c96a354502ff0e098f35da4bd537f0853f Mon Sep 17 00:00:00 2001 From: jenniew Date: Mon, 19 Mar 2018 23:06:30 -0700 Subject: [PATCH 633/823] Python Yarn script update (#2366) * update script * update lenet package name * update lenet example file location * update requirement file * update requirements * remove blank lines --- python/dllib/src/bigdl/dllib/models/lenet/lenet5.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py b/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py index 286882920fd..f2302cb5776 100644 --- a/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py +++ b/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py @@ -15,7 +15,7 @@ # from optparse import OptionParser -from utils import * +from bigdl.models.lenet.utils import * from bigdl.dataset.transformer import * from bigdl.nn.layer import * from bigdl.nn.criterion import * From 0a000273e9b23b5e996964fe087bc15377d9931b Mon Sep 17 00:00:00 2001 From: jenniew Date: Mon, 19 Mar 2018 23:06:30 -0700 Subject: [PATCH 634/823] Python Yarn script update (#2366) * update script * update lenet package name * update lenet example file location * update requirement file * update requirements * remove blank lines --- python/dllib/src/bigdl/dllib/models/lenet/lenet5.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py b/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py index 286882920fd..f2302cb5776 100644 --- a/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py +++ b/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py @@ -15,7 +15,7 @@ # from optparse import OptionParser -from utils import * +from bigdl.models.lenet.utils import * from bigdl.dataset.transformer import * from bigdl.nn.layer import * from bigdl.nn.criterion import * From 2ccab9ad9c24f075601dba2c226343148d04cf06 Mon Sep 17 00:00:00 2001 From: Ian Wong Date: Tue, 20 Mar 2018 14:09:46 +0800 Subject: [PATCH 635/823] [Bug Fix] Fix duplicate check sometimes should be suspend (#2403) * allow keras.Input layer skip duplicate check * fix some bug * add some comments * meet code review --- .../keras/{test_newapi.py => test_keras_api.py} | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) rename python/dllib/test/bigdl/keras/{test_newapi.py => test_keras_api.py} (97%) diff --git a/python/dllib/test/bigdl/keras/test_newapi.py b/python/dllib/test/bigdl/keras/test_keras_api.py similarity index 97% rename from python/dllib/test/bigdl/keras/test_newapi.py rename to python/dllib/test/bigdl/keras/test_keras_api.py index 55a7a6689ed..fefdf797f59 100644 --- a/python/dllib/test/bigdl/keras/test_newapi.py +++ b/python/dllib/test/bigdl/keras/test_keras_api.py @@ -30,7 +30,7 @@ np.random.seed(1337) # for reproducibility -class TestNewAPI(BigDLTestCase): +class TestKerasAPI(BigDLTestCase): def test_embedding(self): input_data = np.random.randint(1000, size=(32, 10)) @@ -200,13 +200,12 @@ def test_merge_method_sum(self): def test_merge_method_model_concat(self): bx1 = BLayer.Input(shape=(4, )) - bx1_1 = BLayer.Input(shape=(4, )) bx2 = BLayer.Input(shape=(5, )) by1 = BLayer.Dense(6, activation="sigmoid")(bx1) - bbranch1 = BModel(bx1, by1)(bx1_1) + bbranch1 = BModel(bx1, by1)(bx1) bbranch2 = BLayer.Dense(8)(bx2) bz = BLayer.merge([bbranch1, bbranch2], mode="concat") - bmodel = BModel([bx1_1, bx2], bz) + bmodel = BModel([bx1, bx2], bz) kx1 = KLayer.Input(shape=(4, )) kx2 = KLayer.Input(shape=(5, )) @@ -221,15 +220,14 @@ def test_merge_method_model_concat(self): def test_merge_method_seq_concat(self): bx1 = BLayer.Input(shape=(10, )) - bx1_1 = BLayer.Input(shape=(10, )) bx2 = BLayer.Input(shape=(10, )) by1 = BLayer.Dense(12, activation="sigmoid")(bx1) - bbranch1_node = BModel(bx1, by1)(bx1_1) + bbranch1_node = BModel(bx1, by1)(bx1) bbranch2 = BSequential() bbranch2.add(BLayer.Dense(12, input_dim=10)) bbranch2_node = bbranch2(bx2) bz = BLayer.merge([bbranch1_node, bbranch2_node], mode="concat") - bmodel = BModel([bx1_1, bx2], bz) + bmodel = BModel([bx1, bx2], bz) kx1 = KLayer.Input(shape=(10, )) kx2 = KLayer.Input(shape=(10, )) From c0e34541ef02157926053c9da43ebf8122c7824c Mon Sep 17 00:00:00 2001 From: Quincy2014 <412363303@qq.com> Date: Wed, 21 Mar 2018 20:16:23 +0800 Subject: [PATCH 636/823] Add Keras Website Documentation for Layers I (#2414) * add core layers doc * add advancedactivations and convolutional layers doc * add dropout, embedding and normalization layers doc * add pooling and recurrent layers doc * update * change * update * add merge * add embedding * change data * modify * update * update * change * update --- python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py b/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py index 6a0eeb916cf..11bc2da5cc7 100644 --- a/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py +++ b/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py @@ -1223,7 +1223,7 @@ class MaxPooling2D(KerasLayer): creating: createKerasMaxPooling2D """ def __init__(self, pool_size=(2, 2), strides=None, - border_mode='valid', dim_ordering='th', + border_mode="valid", dim_ordering="th", input_shape=None, **kwargs): super(MaxPooling2D, self).__init__(None, pool_size, From 6c07956cc6f17a3a557dd31a7dfcffb22f6f6dd7 Mon Sep 17 00:00:00 2001 From: Quincy2014 <412363303@qq.com> Date: Wed, 21 Mar 2018 20:16:23 +0800 Subject: [PATCH 637/823] Add Keras Website Documentation for Layers I (#2414) * add core layers doc * add advancedactivations and convolutional layers doc * add dropout, embedding and normalization layers doc * add pooling and recurrent layers doc * update * change * update * add merge * add embedding * change data * modify * update * update * change * update --- python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py b/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py index 6a0eeb916cf..11bc2da5cc7 100644 --- a/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py +++ b/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py @@ -1223,7 +1223,7 @@ class MaxPooling2D(KerasLayer): creating: createKerasMaxPooling2D """ def __init__(self, pool_size=(2, 2), strides=None, - border_mode='valid', dim_ordering='th', + border_mode="valid", dim_ordering="th", input_shape=None, **kwargs): super(MaxPooling2D, self).__init__(None, pool_size, From bb542433515e8d5118fd41992f9a660f64393e61 Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Wed, 21 Mar 2018 22:36:47 +0800 Subject: [PATCH 638/823] Refine Keras-like API LeNet definition (#2407) * lenet keras seq * update * refine * update * update * fix java creator * meet review * fix lenet definition * fix lenet * test * fix lenet * update * fix * remove --- .../bigdl/dllib/bigdlkeras/layers/topology.py | 66 +++++++++++++++++-- .../src/bigdl/dllib/examples/lenet/README.md | 6 +- .../src/bigdl/dllib/examples/lenet/lenet.py | 13 ++-- .../src/bigdl/dllib/models/lenet/lenet5.py | 2 +- python/dllib/src/bigdl/dllib/utils/common.py | 2 +- 5 files changed, 72 insertions(+), 17 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/bigdlkeras/layers/topology.py b/python/dllib/src/bigdl/dllib/bigdlkeras/layers/topology.py index dccaafb9c18..76e2f8b0b55 100644 --- a/python/dllib/src/bigdl/dllib/bigdlkeras/layers/topology.py +++ b/python/dllib/src/bigdl/dllib/bigdlkeras/layers/topology.py @@ -15,14 +15,70 @@ # from bigdl.nn.keras.layer import KerasLayer -from bigdl.dataset.dataset import * -from bigdl.keras.optimization import OptimConverter +from bigdl.optim.optimizer import * +from bigdl.nn.criterion import * import multiprocessing from bigdl.nn.layer import SharedStaticUtils, Container class KerasModel(KerasLayer, Container, SharedStaticUtils): + def __convert_optim_method(self, optimizer): + optimizer = optimizer.lower() + if optimizer == "adagrad": + return Adagrad(learningrate=0.01) + elif optimizer == "sgd": + return SGD(learningrate=0.01) + elif optimizer == "adam": + return Adam() + elif optimizer == "rmsprop": + return RMSprop(learningrate=0.001, decayrate=0.9) + elif optimizer == "adadelta": + return Adadelta(decayrate=0.95, epsilon=1e-8) + elif optimizer == "adamax": + return Adamax(epsilon=1e-8) + else: + raise TypeError("Unsupported optimizer: %s" % optimizer) + + def __convert_criterion(self, criterion): + criterion = criterion.lower() + if criterion == "categorical_crossentropy": + return CategoricalCrossEntropy() + elif criterion == "mse" or criterion == "mean_squared_error": + return MSECriterion() + elif criterion == "binary_crossentropy": + return BCECriterion() + elif criterion == "mae" or criterion == "mean_absolute_error": + return AbsCriterion() + elif criterion == "hinge": + return MarginCriterion() + elif criterion == "mean_absolute_percentage_error" or criterion == "mape": + return MeanAbsolutePercentageCriterion() + elif criterion == "mean_squared_logarithmic_error" or criterion == "msle": + return MeanSquaredLogarithmicCriterion() + elif criterion == "squared_hinge": + return MarginCriterion(squared=True) + elif criterion == "sparse_categorical_crossentropy": + return ClassNLLCriterion(logProbAsInput=False) + elif criterion == "kullback_leibler_divergence" or criterion == "kld": + return KullbackLeiblerDivergenceCriterion() + elif criterion == "poisson": + return PoissonCriterion() + elif criterion == "cosine_proximity" or criterion == "cosine": + return CosineProximityCriterion() + else: + raise TypeError("Unsupported loss: %s" % criterion) + + def __convert_metrics(self, metrics): + metrics = to_list(metrics) + bmetrics = [] + for metric in metrics: + if metric.lower() == "accuracy": + bmetrics.append(Top1Accuracy()) + else: + raise TypeError("Unsupported metrics: %s" % metric) + return bmetrics + def compile(self, optimizer, loss, metrics=None): """ Configures the learning process. Must be called before fit. @@ -35,11 +91,11 @@ def compile(self, optimizer, loss, metrics=None): metrics: List of validation methods to be used. Default is None. One can alternatively use ['accuracy']. """ if isinstance(optimizer, six.string_types): - optimizer = OptimConverter.to_bigdl_optim_method(optimizer) + optimizer = self.__convert_optim_method(optimizer) if isinstance(loss, six.string_types): - loss = OptimConverter.to_bigdl_criterion(loss) + loss = self.__convert_criterion(loss) if all(isinstance(metric, six.string_types) for metric in metrics): - metrics = OptimConverter.to_bigdl_metrics(metrics) + metrics = self.__convert_metrics(metrics) callBigDlFunc(self.bigdl_type, "compile", self.value, optimizer, diff --git a/python/dllib/src/bigdl/dllib/examples/lenet/README.md b/python/dllib/src/bigdl/dllib/examples/lenet/README.md index c6661674425..9a93bd8ed07 100644 --- a/python/dllib/src/bigdl/dllib/examples/lenet/README.md +++ b/python/dllib/src/bigdl/dllib/examples/lenet/README.md @@ -1,8 +1,8 @@ -# LeNet5 Model on MNIST with new API +# LeNet5 Model on MNIST using Keras-Style API -This example defines a classical CNN model used in digital number classification with the new set of Keras-Style API in BigDL, which is more user-friendly. For detailed information with regard to LeNet, please refer to . +LeNet5 is a classical CNN model used in digital number classification. For detailed information with regard to LeNet, please refer to . -This example is the same as [../../model/lenet/lenet5.py](../../models/lenet/lenet5.py), except that here we use the new Keras-Style API for model definition and training. +This example is the same as [../../model/lenet/lenet5.py](../../models/lenet/lenet5.py), except that here we use the new set of Keras-Style API in BigDL for model definition and training, which is more user-friendly. ## Install dependencies * [Install dependencies](../../../README.md#install.dependencies) diff --git a/python/dllib/src/bigdl/dllib/examples/lenet/lenet.py b/python/dllib/src/bigdl/dllib/examples/lenet/lenet.py index 433be38325b..75669d084ef 100644 --- a/python/dllib/src/bigdl/dllib/examples/lenet/lenet.py +++ b/python/dllib/src/bigdl/dllib/examples/lenet/lenet.py @@ -23,14 +23,13 @@ def build_model(class_num): model = Sequential() model.add(Reshape((1, 28, 28), input_shape=(28, 28, 1))) - model.add(Convolution2D(32, 3, 3, activation="relu")) - model.add(Convolution2D(32, 3, 3, activation="relu")) - model.add(MaxPooling2D(pool_size=(2, 2))) - model.add(Dropout(0.25)) + model.add(Convolution2D(6, 5, 5, activation="tanh", name="conv1_5x5")) + model.add(MaxPooling2D()) + model.add(Convolution2D(12, 5, 5, activation="tanh", name="conv2_5x5")) + model.add(MaxPooling2D()) model.add(Flatten()) - model.add(Dense(128, activation="relu")) - model.add(Dropout(0.5)) - model.add(Dense(class_num, activation="softmax")) + model.add(Dense(100, activation="tanh", name="fc1")) + model.add(Dense(class_num, activation="softmax", name="fc2")) return model diff --git a/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py b/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py index f2302cb5776..9ccd7e1e5f7 100644 --- a/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py +++ b/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py @@ -29,8 +29,8 @@ def build_model(class_num): model.add(SpatialConvolution(1, 6, 5, 5)) model.add(Tanh()) model.add(SpatialMaxPooling(2, 2, 2, 2)) - model.add(Tanh()) model.add(SpatialConvolution(6, 12, 5, 5)) + model.add(Tanh()) model.add(SpatialMaxPooling(2, 2, 2, 2)) model.add(Reshape([12 * 4 * 4])) model.add(Linear(12 * 4 * 4, 100)) diff --git a/python/dllib/src/bigdl/dllib/utils/common.py b/python/dllib/src/bigdl/dllib/utils/common.py index 98ce9efd46c..6c09953d046 100644 --- a/python/dllib/src/bigdl/dllib/utils/common.py +++ b/python/dllib/src/bigdl/dllib/utils/common.py @@ -82,7 +82,7 @@ def set_creator_class(cls, cclass): if isinstance(cclass, six.string_types): cclass = [cclass] with JavaCreator._lock: - JavaCreator.__creator_class = [cclass] + JavaCreator.__creator_class = cclass JavaCreator._instance = None def __init__(self, bigdl_type, gateway): From a0ace42d43ce0d8a6901c70b77bedad205fe5fda Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Wed, 21 Mar 2018 22:36:47 +0800 Subject: [PATCH 639/823] Refine Keras-like API LeNet definition (#2407) * lenet keras seq * update * refine * update * update * fix java creator * meet review * fix lenet definition * fix lenet * test * fix lenet * update * fix * remove --- .../bigdl/dllib/bigdlkeras/layers/topology.py | 66 +++++++++++++++++-- .../src/bigdl/dllib/examples/lenet/README.md | 6 +- .../src/bigdl/dllib/examples/lenet/lenet.py | 13 ++-- .../src/bigdl/dllib/models/lenet/lenet5.py | 2 +- python/dllib/src/bigdl/utils/common.py | 2 +- 5 files changed, 72 insertions(+), 17 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/bigdlkeras/layers/topology.py b/python/dllib/src/bigdl/dllib/bigdlkeras/layers/topology.py index dccaafb9c18..76e2f8b0b55 100644 --- a/python/dllib/src/bigdl/dllib/bigdlkeras/layers/topology.py +++ b/python/dllib/src/bigdl/dllib/bigdlkeras/layers/topology.py @@ -15,14 +15,70 @@ # from bigdl.nn.keras.layer import KerasLayer -from bigdl.dataset.dataset import * -from bigdl.keras.optimization import OptimConverter +from bigdl.optim.optimizer import * +from bigdl.nn.criterion import * import multiprocessing from bigdl.nn.layer import SharedStaticUtils, Container class KerasModel(KerasLayer, Container, SharedStaticUtils): + def __convert_optim_method(self, optimizer): + optimizer = optimizer.lower() + if optimizer == "adagrad": + return Adagrad(learningrate=0.01) + elif optimizer == "sgd": + return SGD(learningrate=0.01) + elif optimizer == "adam": + return Adam() + elif optimizer == "rmsprop": + return RMSprop(learningrate=0.001, decayrate=0.9) + elif optimizer == "adadelta": + return Adadelta(decayrate=0.95, epsilon=1e-8) + elif optimizer == "adamax": + return Adamax(epsilon=1e-8) + else: + raise TypeError("Unsupported optimizer: %s" % optimizer) + + def __convert_criterion(self, criterion): + criterion = criterion.lower() + if criterion == "categorical_crossentropy": + return CategoricalCrossEntropy() + elif criterion == "mse" or criterion == "mean_squared_error": + return MSECriterion() + elif criterion == "binary_crossentropy": + return BCECriterion() + elif criterion == "mae" or criterion == "mean_absolute_error": + return AbsCriterion() + elif criterion == "hinge": + return MarginCriterion() + elif criterion == "mean_absolute_percentage_error" or criterion == "mape": + return MeanAbsolutePercentageCriterion() + elif criterion == "mean_squared_logarithmic_error" or criterion == "msle": + return MeanSquaredLogarithmicCriterion() + elif criterion == "squared_hinge": + return MarginCriterion(squared=True) + elif criterion == "sparse_categorical_crossentropy": + return ClassNLLCriterion(logProbAsInput=False) + elif criterion == "kullback_leibler_divergence" or criterion == "kld": + return KullbackLeiblerDivergenceCriterion() + elif criterion == "poisson": + return PoissonCriterion() + elif criterion == "cosine_proximity" or criterion == "cosine": + return CosineProximityCriterion() + else: + raise TypeError("Unsupported loss: %s" % criterion) + + def __convert_metrics(self, metrics): + metrics = to_list(metrics) + bmetrics = [] + for metric in metrics: + if metric.lower() == "accuracy": + bmetrics.append(Top1Accuracy()) + else: + raise TypeError("Unsupported metrics: %s" % metric) + return bmetrics + def compile(self, optimizer, loss, metrics=None): """ Configures the learning process. Must be called before fit. @@ -35,11 +91,11 @@ def compile(self, optimizer, loss, metrics=None): metrics: List of validation methods to be used. Default is None. One can alternatively use ['accuracy']. """ if isinstance(optimizer, six.string_types): - optimizer = OptimConverter.to_bigdl_optim_method(optimizer) + optimizer = self.__convert_optim_method(optimizer) if isinstance(loss, six.string_types): - loss = OptimConverter.to_bigdl_criterion(loss) + loss = self.__convert_criterion(loss) if all(isinstance(metric, six.string_types) for metric in metrics): - metrics = OptimConverter.to_bigdl_metrics(metrics) + metrics = self.__convert_metrics(metrics) callBigDlFunc(self.bigdl_type, "compile", self.value, optimizer, diff --git a/python/dllib/src/bigdl/dllib/examples/lenet/README.md b/python/dllib/src/bigdl/dllib/examples/lenet/README.md index c6661674425..9a93bd8ed07 100644 --- a/python/dllib/src/bigdl/dllib/examples/lenet/README.md +++ b/python/dllib/src/bigdl/dllib/examples/lenet/README.md @@ -1,8 +1,8 @@ -# LeNet5 Model on MNIST with new API +# LeNet5 Model on MNIST using Keras-Style API -This example defines a classical CNN model used in digital number classification with the new set of Keras-Style API in BigDL, which is more user-friendly. For detailed information with regard to LeNet, please refer to . +LeNet5 is a classical CNN model used in digital number classification. For detailed information with regard to LeNet, please refer to . -This example is the same as [../../model/lenet/lenet5.py](../../models/lenet/lenet5.py), except that here we use the new Keras-Style API for model definition and training. +This example is the same as [../../model/lenet/lenet5.py](../../models/lenet/lenet5.py), except that here we use the new set of Keras-Style API in BigDL for model definition and training, which is more user-friendly. ## Install dependencies * [Install dependencies](../../../README.md#install.dependencies) diff --git a/python/dllib/src/bigdl/dllib/examples/lenet/lenet.py b/python/dllib/src/bigdl/dllib/examples/lenet/lenet.py index 433be38325b..75669d084ef 100644 --- a/python/dllib/src/bigdl/dllib/examples/lenet/lenet.py +++ b/python/dllib/src/bigdl/dllib/examples/lenet/lenet.py @@ -23,14 +23,13 @@ def build_model(class_num): model = Sequential() model.add(Reshape((1, 28, 28), input_shape=(28, 28, 1))) - model.add(Convolution2D(32, 3, 3, activation="relu")) - model.add(Convolution2D(32, 3, 3, activation="relu")) - model.add(MaxPooling2D(pool_size=(2, 2))) - model.add(Dropout(0.25)) + model.add(Convolution2D(6, 5, 5, activation="tanh", name="conv1_5x5")) + model.add(MaxPooling2D()) + model.add(Convolution2D(12, 5, 5, activation="tanh", name="conv2_5x5")) + model.add(MaxPooling2D()) model.add(Flatten()) - model.add(Dense(128, activation="relu")) - model.add(Dropout(0.5)) - model.add(Dense(class_num, activation="softmax")) + model.add(Dense(100, activation="tanh", name="fc1")) + model.add(Dense(class_num, activation="softmax", name="fc2")) return model diff --git a/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py b/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py index f2302cb5776..9ccd7e1e5f7 100644 --- a/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py +++ b/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py @@ -29,8 +29,8 @@ def build_model(class_num): model.add(SpatialConvolution(1, 6, 5, 5)) model.add(Tanh()) model.add(SpatialMaxPooling(2, 2, 2, 2)) - model.add(Tanh()) model.add(SpatialConvolution(6, 12, 5, 5)) + model.add(Tanh()) model.add(SpatialMaxPooling(2, 2, 2, 2)) model.add(Reshape([12 * 4 * 4])) model.add(Linear(12 * 4 * 4, 100)) diff --git a/python/dllib/src/bigdl/utils/common.py b/python/dllib/src/bigdl/utils/common.py index 98ce9efd46c..6c09953d046 100644 --- a/python/dllib/src/bigdl/utils/common.py +++ b/python/dllib/src/bigdl/utils/common.py @@ -82,7 +82,7 @@ def set_creator_class(cls, cclass): if isinstance(cclass, six.string_types): cclass = [cclass] with JavaCreator._lock: - JavaCreator.__creator_class = [cclass] + JavaCreator.__creator_class = cclass JavaCreator._instance = None def __init__(self, bigdl_type, gateway): From 70b7fa353a17d26a056760c00fb5ac032bc15070 Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Thu, 22 Mar 2018 15:51:46 +0800 Subject: [PATCH 640/823] Update website doc for Scala Run and Configurations (#2422) * update scala run * update * update * update configuration * remove * fix style --- .../bigdl/dllib/bigdlkeras/optimization.py | 95 ++++++++----------- 1 file changed, 39 insertions(+), 56 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/bigdlkeras/optimization.py b/python/dllib/src/bigdl/dllib/bigdlkeras/optimization.py index 3bced61db49..6c108ea04d7 100644 --- a/python/dllib/src/bigdl/dllib/bigdlkeras/optimization.py +++ b/python/dllib/src/bigdl/dllib/bigdlkeras/optimization.py @@ -75,61 +75,44 @@ def to_bigdl_criterion(kloss): @staticmethod def to_bigdl_optim_method(koptim_method): - if isinstance(koptim_method, six.string_types): - koptim_method = koptim_method.lower() - if koptim_method == "adagrad": - return boptimizer.Adagrad(learningrate=0.01) - elif koptim_method == "sgd": - return boptimizer.SGD(learningrate=0.01) - elif koptim_method == "adam": - return boptimizer.Adam() - elif koptim_method == "rmsprop": - return boptimizer.RMSprop(learningrate=0.001, decayrate=0.9) - elif koptim_method == "adadelta": - return boptimizer.Adadelta(decayrate=0.95, epsilon=1e-8) - elif koptim_method == "adamax": - return boptimizer.Adamax(epsilon=1e-8) - else: - unsupport_exp(koptim_method) - else: - # koptim_method a Keras object - lr = float(K.eval(koptim_method.lr)) - decay = float(K.eval(koptim_method.decay)) - if isinstance(koptim_method, koptimizers.Adagrad): - warnings.warn("For Adagrad, we don't support epsilon for now") - return boptimizer.Adagrad(learningrate=lr, - learningrate_decay=decay) - elif isinstance(koptim_method, koptimizers.SGD): - momentum = float(K.eval(koptim_method.momentum)) - return boptimizer.SGD(learningrate=lr, + # koptim_method is always an object + lr = float(K.eval(koptim_method.lr)) + decay = float(K.eval(koptim_method.decay)) + if isinstance(koptim_method, koptimizers.Adagrad): + warnings.warn("For Adagrad, we don't support epsilon for now") + return boptimizer.Adagrad(learningrate=lr, + learningrate_decay=decay) + elif isinstance(koptim_method, koptimizers.SGD): + momentum = float(K.eval(koptim_method.momentum)) + return boptimizer.SGD(learningrate=lr, + learningrate_decay=decay, + momentum=momentum, + nesterov=koptim_method.nesterov) + elif isinstance(koptim_method, koptimizers.Adam): + beta1 = float(K.eval(koptim_method.beta_1)) + beta2 = float(K.eval(koptim_method.beta_2)) + return boptimizer.Adam(learningrate=lr, + learningrate_decay=decay, + beta1=beta1, + beta2=beta2, + epsilon=koptim_method.epsilon) + elif isinstance(koptim_method, koptimizers.RMSprop): + rho = float(K.eval(koptim_method.rho)) + return boptimizer.RMSprop(learningrate=lr, learningrate_decay=decay, - momentum=momentum, - nesterov=koptim_method.nesterov) - elif isinstance(koptim_method, koptimizers.Adam): - beta1 = float(K.eval(koptim_method.beta_1)) - beta2 = float(K.eval(koptim_method.beta_2)) - return boptimizer.Adam(learningrate=lr, - learningrate_decay=decay, - beta1=beta1, - beta2=beta2, + decayrate=rho, + epsilon=koptim_method.epsilon) + elif isinstance(koptim_method, koptimizers.Adadelta): + warnings.warn("For Adadelta, we don't support learning rate and learning rate decay for now") + return boptimizer.Adadelta(decayrate=koptim_method.rho, epsilon=koptim_method.epsilon) - elif isinstance(koptim_method, koptimizers.RMSprop): - rho = float(K.eval(koptim_method.rho)) - return boptimizer.RMSprop(learningrate=lr, - learningrate_decay=decay, - decayrate=rho, - epsilon=koptim_method.epsilon) - elif isinstance(koptim_method, koptimizers.Adadelta): - warnings.warn("For Adadelta, we don't support learning rate and learning rate decay for now") - return boptimizer.Adadelta(decayrate=koptim_method.rho, - epsilon=koptim_method.epsilon) - elif isinstance(koptim_method, koptimizers.Adamax): - beta1 = float(K.eval(koptim_method.beta_1)) - beta2 = float(K.eval(koptim_method.beta_2)) - warnings.warn("For Adamax, we don't support learning rate decay for now") - return boptimizer.Adamax(learningrate=lr, - beta1=beta1, - beta2=beta2, - epsilon=koptim_method.epsilon) - else: - unsupport_exp(koptim_method) + elif isinstance(koptim_method, koptimizers.Adamax): + beta1 = float(K.eval(koptim_method.beta_1)) + beta2 = float(K.eval(koptim_method.beta_2)) + warnings.warn("For Adamax, we don't support learning rate decay for now") + return boptimizer.Adamax(learningrate=lr, + beta1=beta1, + beta2=beta2, + epsilon=koptim_method.epsilon) + else: + unsupport_exp(koptim_method) From b83f1bbda4e0bacd442e2736689b733d89077a2b Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Thu, 22 Mar 2018 15:51:46 +0800 Subject: [PATCH 641/823] Update website doc for Scala Run and Configurations (#2422) * update scala run * update * update * update configuration * remove * fix style --- .../bigdl/dllib/bigdlkeras/optimization.py | 95 ++++++++----------- 1 file changed, 39 insertions(+), 56 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/bigdlkeras/optimization.py b/python/dllib/src/bigdl/dllib/bigdlkeras/optimization.py index 3bced61db49..6c108ea04d7 100644 --- a/python/dllib/src/bigdl/dllib/bigdlkeras/optimization.py +++ b/python/dllib/src/bigdl/dllib/bigdlkeras/optimization.py @@ -75,61 +75,44 @@ def to_bigdl_criterion(kloss): @staticmethod def to_bigdl_optim_method(koptim_method): - if isinstance(koptim_method, six.string_types): - koptim_method = koptim_method.lower() - if koptim_method == "adagrad": - return boptimizer.Adagrad(learningrate=0.01) - elif koptim_method == "sgd": - return boptimizer.SGD(learningrate=0.01) - elif koptim_method == "adam": - return boptimizer.Adam() - elif koptim_method == "rmsprop": - return boptimizer.RMSprop(learningrate=0.001, decayrate=0.9) - elif koptim_method == "adadelta": - return boptimizer.Adadelta(decayrate=0.95, epsilon=1e-8) - elif koptim_method == "adamax": - return boptimizer.Adamax(epsilon=1e-8) - else: - unsupport_exp(koptim_method) - else: - # koptim_method a Keras object - lr = float(K.eval(koptim_method.lr)) - decay = float(K.eval(koptim_method.decay)) - if isinstance(koptim_method, koptimizers.Adagrad): - warnings.warn("For Adagrad, we don't support epsilon for now") - return boptimizer.Adagrad(learningrate=lr, - learningrate_decay=decay) - elif isinstance(koptim_method, koptimizers.SGD): - momentum = float(K.eval(koptim_method.momentum)) - return boptimizer.SGD(learningrate=lr, + # koptim_method is always an object + lr = float(K.eval(koptim_method.lr)) + decay = float(K.eval(koptim_method.decay)) + if isinstance(koptim_method, koptimizers.Adagrad): + warnings.warn("For Adagrad, we don't support epsilon for now") + return boptimizer.Adagrad(learningrate=lr, + learningrate_decay=decay) + elif isinstance(koptim_method, koptimizers.SGD): + momentum = float(K.eval(koptim_method.momentum)) + return boptimizer.SGD(learningrate=lr, + learningrate_decay=decay, + momentum=momentum, + nesterov=koptim_method.nesterov) + elif isinstance(koptim_method, koptimizers.Adam): + beta1 = float(K.eval(koptim_method.beta_1)) + beta2 = float(K.eval(koptim_method.beta_2)) + return boptimizer.Adam(learningrate=lr, + learningrate_decay=decay, + beta1=beta1, + beta2=beta2, + epsilon=koptim_method.epsilon) + elif isinstance(koptim_method, koptimizers.RMSprop): + rho = float(K.eval(koptim_method.rho)) + return boptimizer.RMSprop(learningrate=lr, learningrate_decay=decay, - momentum=momentum, - nesterov=koptim_method.nesterov) - elif isinstance(koptim_method, koptimizers.Adam): - beta1 = float(K.eval(koptim_method.beta_1)) - beta2 = float(K.eval(koptim_method.beta_2)) - return boptimizer.Adam(learningrate=lr, - learningrate_decay=decay, - beta1=beta1, - beta2=beta2, + decayrate=rho, + epsilon=koptim_method.epsilon) + elif isinstance(koptim_method, koptimizers.Adadelta): + warnings.warn("For Adadelta, we don't support learning rate and learning rate decay for now") + return boptimizer.Adadelta(decayrate=koptim_method.rho, epsilon=koptim_method.epsilon) - elif isinstance(koptim_method, koptimizers.RMSprop): - rho = float(K.eval(koptim_method.rho)) - return boptimizer.RMSprop(learningrate=lr, - learningrate_decay=decay, - decayrate=rho, - epsilon=koptim_method.epsilon) - elif isinstance(koptim_method, koptimizers.Adadelta): - warnings.warn("For Adadelta, we don't support learning rate and learning rate decay for now") - return boptimizer.Adadelta(decayrate=koptim_method.rho, - epsilon=koptim_method.epsilon) - elif isinstance(koptim_method, koptimizers.Adamax): - beta1 = float(K.eval(koptim_method.beta_1)) - beta2 = float(K.eval(koptim_method.beta_2)) - warnings.warn("For Adamax, we don't support learning rate decay for now") - return boptimizer.Adamax(learningrate=lr, - beta1=beta1, - beta2=beta2, - epsilon=koptim_method.epsilon) - else: - unsupport_exp(koptim_method) + elif isinstance(koptim_method, koptimizers.Adamax): + beta1 = float(K.eval(koptim_method.beta_1)) + beta2 = float(K.eval(koptim_method.beta_2)) + warnings.warn("For Adamax, we don't support learning rate decay for now") + return boptimizer.Adamax(learningrate=lr, + beta1=beta1, + beta2=beta2, + epsilon=koptim_method.epsilon) + else: + unsupport_exp(koptim_method) From aafdd1b09e8e1c938a2579ed5e6e068d9cf9f01e Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Thu, 22 Mar 2018 16:14:39 +0800 Subject: [PATCH 642/823] Refactor readme of Textclassifier (#2425) * refactor doc * meet comments --- .../dllib/models/textclassifier/README.md | 21 ++++++++++++------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/models/textclassifier/README.md b/python/dllib/src/bigdl/dllib/models/textclassifier/README.md index 80d44b382c5..53e44da3a5f 100644 --- a/python/dllib/src/bigdl/dllib/models/textclassifier/README.md +++ b/python/dllib/src/bigdl/dllib/models/textclassifier/README.md @@ -8,14 +8,14 @@ * Embedding: 200-dimensional pre-trained GloVe embeddings of 400k words which trained on a 2014 dump of English Wikipedia. * Training data: "20 Newsgroup dataset" which containing 20 categories and with totally 19997 texts. -## Install dependencies - * [Install dependencies](../../../README.md#install.dependencies) ## How to run this example: -Please note that due to some permission issue, this example **cannot** be run on Windows. +- Please note that due to some permission issue, this example **cannot** be run on Windows. -If there is no [Pre-train GloVe word embeddings](http://nlp.stanford.edu/data/glove.6B.zip) +### Data preparation +- You don't need to download the data by yourself. +- If there is no [Pre-train GloVe word embeddings](http://nlp.stanford.edu/data/glove.6B.zip) or [20 Newsgroup dataset](http://www.cs.cmu.edu/afs/cs.cmu.edu/project/theo-20/www/data/news20.html) in `./data/news20` directory with the following structure looks like: @@ -26,15 +26,20 @@ $ [/tmp/news20]$ tree . -L 1 └── glove.6B.zip ``` - The example code would automatically download the data during the first run. -- If you install BigDL via pip, you can run this example locally by the following command: - - Check [Run after pip install](../../../../docs/docs/PythonUserGuide/run-from-pip.md) + +### Run via pip install +- [Install from pip](https://bigdl-project.github.io/0.6.0-SNAPSHOT/#PythonUserGuide/install-from-pip/) +- Optional: check [Run after pip install](https://bigdl-project.github.io/0.6.0-SNAPSHOT/#PythonUserGuide/run-from-pip/) +- Run the following command locally ``` python ${BigDL_HOME}/pyspark/bigdl/models/textclassifier/textclassifier.py --max_epoch 3 --model cnn ``` -- You can also use `spark-submit` to launch this example: - +### Run via spark-submit +- [Install without pip](https://bigdl-project.github.io/0.6.0-SNAPSHOT/#PythonUserGuide/install-without-pip/) +- Optional: check [Run without pip](https://bigdl-project.github.io/0.6.0-SNAPSHOT/#PythonUserGuide/run-without-pip/) +- Run the following command ```{r, engine='sh'} PYTHONHASHSEED=0 BigDL_HOME=... From 910adb79e682b29afc97aaccdb0ffd93bde6bf8b Mon Sep 17 00:00:00 2001 From: "li,zhichao" Date: Thu, 22 Mar 2018 16:14:39 +0800 Subject: [PATCH 643/823] Refactor readme of Textclassifier (#2425) * refactor doc * meet comments --- .../dllib/models/textclassifier/README.md | 21 ++++++++++++------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/models/textclassifier/README.md b/python/dllib/src/bigdl/dllib/models/textclassifier/README.md index 80d44b382c5..53e44da3a5f 100644 --- a/python/dllib/src/bigdl/dllib/models/textclassifier/README.md +++ b/python/dllib/src/bigdl/dllib/models/textclassifier/README.md @@ -8,14 +8,14 @@ * Embedding: 200-dimensional pre-trained GloVe embeddings of 400k words which trained on a 2014 dump of English Wikipedia. * Training data: "20 Newsgroup dataset" which containing 20 categories and with totally 19997 texts. -## Install dependencies - * [Install dependencies](../../../README.md#install.dependencies) ## How to run this example: -Please note that due to some permission issue, this example **cannot** be run on Windows. +- Please note that due to some permission issue, this example **cannot** be run on Windows. -If there is no [Pre-train GloVe word embeddings](http://nlp.stanford.edu/data/glove.6B.zip) +### Data preparation +- You don't need to download the data by yourself. +- If there is no [Pre-train GloVe word embeddings](http://nlp.stanford.edu/data/glove.6B.zip) or [20 Newsgroup dataset](http://www.cs.cmu.edu/afs/cs.cmu.edu/project/theo-20/www/data/news20.html) in `./data/news20` directory with the following structure looks like: @@ -26,15 +26,20 @@ $ [/tmp/news20]$ tree . -L 1 └── glove.6B.zip ``` - The example code would automatically download the data during the first run. -- If you install BigDL via pip, you can run this example locally by the following command: - - Check [Run after pip install](../../../../docs/docs/PythonUserGuide/run-from-pip.md) + +### Run via pip install +- [Install from pip](https://bigdl-project.github.io/0.6.0-SNAPSHOT/#PythonUserGuide/install-from-pip/) +- Optional: check [Run after pip install](https://bigdl-project.github.io/0.6.0-SNAPSHOT/#PythonUserGuide/run-from-pip/) +- Run the following command locally ``` python ${BigDL_HOME}/pyspark/bigdl/models/textclassifier/textclassifier.py --max_epoch 3 --model cnn ``` -- You can also use `spark-submit` to launch this example: - +### Run via spark-submit +- [Install without pip](https://bigdl-project.github.io/0.6.0-SNAPSHOT/#PythonUserGuide/install-without-pip/) +- Optional: check [Run without pip](https://bigdl-project.github.io/0.6.0-SNAPSHOT/#PythonUserGuide/run-without-pip/) +- Run the following command ```{r, engine='sh'} PYTHONHASHSEED=0 BigDL_HOME=... From 960bbc3c81579e359e50365c65362681dbcda633 Mon Sep 17 00:00:00 2001 From: Ian Wong Date: Thu, 22 Mar 2018 16:16:23 +0800 Subject: [PATCH 644/823] [Bug Fix] Fix typo in SpatialSeparableConvoluiton layer name and add related docs (#2420) * fix typo in spatialseparableconvoluiton name and add docs * meet code review * meet code review and fix tests --- python/dllib/src/bigdl/dllib/nn/layer.py | 42 ++++++++++++------------ 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 5a20823efe1..38b4f26d521 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -3663,7 +3663,7 @@ def __init__(self, upper, inplace) -class SpatialSeperableConvolution(Layer): +class SpatialSeparableConvolution(Layer): ''' Separable convolutions consist in first performing a depthwise spatial convolution (which acts @@ -3688,17 +3688,17 @@ class SpatialSeperableConvolution(Layer): :param b_regularizer: instance of [[Regularizer]]applied to the pointwise bias. :param p_regularizer: instance of [[Regularizer]]applied to the pointwise weights. - >>> conv = SpatialSeperableConvolution(6, 12, 1, 5, 5) - creating: createSpatialSeperableConvolution + >>> conv = SpatialSeparableConvolution(6, 12, 1, 5, 5) + creating: createSpatialSeparableConvolution >>> conv.setWRegularizer(L1Regularizer(0.5)) creating: createL1Regularizer >>> conv.setBRegularizer(L1Regularizer(0.5)) creating: createL1Regularizer - >>> conv = SpatialSeperableConvolution(6, 12, 1, 5, 5, 1, 1, 0, 0, True, "NCHW", L1Regularizer(0.5), L1Regularizer(0.5), L1Regularizer(0.5)) + >>> conv = SpatialSeparableConvolution(6, 12, 1, 5, 5, 1, 1, 0, 0, True, "NCHW", L1Regularizer(0.5), L1Regularizer(0.5), L1Regularizer(0.5)) creating: createL1Regularizer creating: createL1Regularizer creating: createL1Regularizer - creating: createSpatialSeperableConvolution + creating: createSpatialSeparableConvolution ''' def __init__(self, @@ -3717,22 +3717,22 @@ def __init__(self, b_regularizer=None, p_regularizer=None, bigdl_type="float"): - super(SpatialSeperableConvolution, self).__init__(None, bigdl_type, - n_input_channel, - n_output_channel, - depth_multiplier, - kernel_w, - kernel_h, - stride_w, - stride_h, - pad_w, - pad_h, - with_bias, - data_format, - w_regularizer, - b_regularizer, - p_regularizer, - ) + super(SpatialSeparableConvolution, self).__init__(None, bigdl_type, + n_input_channel, + n_output_channel, + depth_multiplier, + kernel_w, + kernel_h, + stride_w, + stride_h, + pad_w, + pad_h, + with_bias, + data_format, + w_regularizer, + b_regularizer, + p_regularizer, + ) class ReLU6(Layer): From a0d54ecd1e8c0bfb70a7d20d690b5df0e195940b Mon Sep 17 00:00:00 2001 From: Ian Wong Date: Thu, 22 Mar 2018 16:16:23 +0800 Subject: [PATCH 645/823] [Bug Fix] Fix typo in SpatialSeparableConvoluiton layer name and add related docs (#2420) * fix typo in spatialseparableconvoluiton name and add docs * meet code review * meet code review and fix tests --- python/dllib/src/bigdl/dllib/nn/layer.py | 42 ++++++++++++------------ 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 5a20823efe1..38b4f26d521 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -3663,7 +3663,7 @@ def __init__(self, upper, inplace) -class SpatialSeperableConvolution(Layer): +class SpatialSeparableConvolution(Layer): ''' Separable convolutions consist in first performing a depthwise spatial convolution (which acts @@ -3688,17 +3688,17 @@ class SpatialSeperableConvolution(Layer): :param b_regularizer: instance of [[Regularizer]]applied to the pointwise bias. :param p_regularizer: instance of [[Regularizer]]applied to the pointwise weights. - >>> conv = SpatialSeperableConvolution(6, 12, 1, 5, 5) - creating: createSpatialSeperableConvolution + >>> conv = SpatialSeparableConvolution(6, 12, 1, 5, 5) + creating: createSpatialSeparableConvolution >>> conv.setWRegularizer(L1Regularizer(0.5)) creating: createL1Regularizer >>> conv.setBRegularizer(L1Regularizer(0.5)) creating: createL1Regularizer - >>> conv = SpatialSeperableConvolution(6, 12, 1, 5, 5, 1, 1, 0, 0, True, "NCHW", L1Regularizer(0.5), L1Regularizer(0.5), L1Regularizer(0.5)) + >>> conv = SpatialSeparableConvolution(6, 12, 1, 5, 5, 1, 1, 0, 0, True, "NCHW", L1Regularizer(0.5), L1Regularizer(0.5), L1Regularizer(0.5)) creating: createL1Regularizer creating: createL1Regularizer creating: createL1Regularizer - creating: createSpatialSeperableConvolution + creating: createSpatialSeparableConvolution ''' def __init__(self, @@ -3717,22 +3717,22 @@ def __init__(self, b_regularizer=None, p_regularizer=None, bigdl_type="float"): - super(SpatialSeperableConvolution, self).__init__(None, bigdl_type, - n_input_channel, - n_output_channel, - depth_multiplier, - kernel_w, - kernel_h, - stride_w, - stride_h, - pad_w, - pad_h, - with_bias, - data_format, - w_regularizer, - b_regularizer, - p_regularizer, - ) + super(SpatialSeparableConvolution, self).__init__(None, bigdl_type, + n_input_channel, + n_output_channel, + depth_multiplier, + kernel_w, + kernel_h, + stride_w, + stride_h, + pad_w, + pad_h, + with_bias, + data_format, + w_regularizer, + b_regularizer, + p_regularizer, + ) class ReLU6(Layer): From 67a317b130e4de383a3a9d654195474a4e96b082 Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Fri, 23 Mar 2018 18:28:41 +0800 Subject: [PATCH 646/823] Keras-Style API website doc for training; refine doc format (#2441) * training api * add * finish * remove * fix * refine doc * update * remove * update --- .../bigdl/dllib/bigdlkeras/layers/layer.py | 26 +++++++++---------- .../bigdl/dllib/bigdlkeras/layers/topology.py | 4 +-- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py b/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py index 11bc2da5cc7..4d688e36a63 100644 --- a/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py +++ b/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py @@ -157,7 +157,7 @@ def __init__(self, output_dim, init="glorot_uniform", activation=None, class MaxoutDense(KerasLayer): """ - A dense maxout layer that takes the element-wise maximum of nbFeature, Dense(inputDim, outputDim) linear layers. + A dense maxout layer that takes the element-wise maximum of linear layers. This allows the layer to learn a convex, piecewise linear activation function over the inputs. The input of this layer should be 2D. @@ -478,7 +478,7 @@ class Permute(KerasLayer): input_shape (a shape tuple, does not include the batch dimension). # Arguments - dims: Tuple of int. Permutation pattern, does not include the samples dimension. Indexing starts at 1. + dims: Tuple of int. Permutation pattern, does not include the batch dimension. Indexing starts at 1. input_shape: A shape tuple, not including batch. name: String to set the name of the layer. If not specified, its name will by default to be a generated string. @@ -718,7 +718,7 @@ class AtrousConvolution1D(KerasLayer): creating: createKerasAtrousConvolution1D """ def __init__(self, nb_filter, filter_length, init="glorot_uniform", activation=None, - border_mode='valid', subsample_length=1, atrous_rate=1, W_regularizer=None, + border_mode="valid", subsample_length=1, atrous_rate=1, W_regularizer=None, b_regularizer=None, bias=True, input_shape=None, **kwargs): if border_mode != "valid": raise ValueError("For AtrousConvolution1D, only border_mode='valid' is supported for now") @@ -863,7 +863,7 @@ class SeparableConvolution2D(KerasLayer): Applies separable convolution operator for 2D inputs. Separable convolutions consist in first performing a depthwise spatial convolution (which acts on each input channel separately) followed by a pointwise convolution which mixes together the - resulting output channels. The depthMultiplier argument controls how many output channels are + resulting output channels. The depth_multiplier argument controls how many output channels are generated per input channel in the depthwise step. You can also use SeparableConv2D as an alias of this layer. The input of this layer should be 4D. @@ -1180,7 +1180,7 @@ class MaxPooling1D(KerasLayer): input_shape (a shape tuple, does not include the batch dimension). # Arguments - pool_length: Size of the region to which max pooling is applied. + pool_length: Size of the region to which max pooling is applied. Integer. Default is 2. strides: Factor by which to downscale. 2 will halve the input. Default is None, and in this case it will be equal to pool_length.. border_mode: Either 'valid' or 'same'. Default is 'valid'. @@ -1927,8 +1927,8 @@ class Masking(KerasLayer): input_shape (a shape tuple, does not include the batch dimension). # Arguments - mask_value: Float, mask value. For each timestep in the input tensor (dimension #1 in the tensor), - if all values in the input tensor at that timestep are equal to `mask_value`, + mask_value: Float, mask value. For each timestep in the input (the second dimension), + if all values in the input at that timestep are equal to 'mask_value', then the timestep will masked (skipped) in all downstream layers. input_shape: A shape tuple, not including batch. name: String to set the name of the layer. If not specified, its name will by default to be a generated string. @@ -1967,15 +1967,15 @@ class SReLU(KerasLayer): Default is None. For example, if the incoming feature maps are from a 2D convolution with output shape (batch, height, width, channels), and you wish to share parameters across space so that - each filter only has one set of parameters, set 'SharedAxes=(1,2)'. + each filter only has one set of parameters, set 'shared_axes=(1,2)'. input_shape: A shape tuple, not including batch. name: String to set the name of the layer. If not specified, its name will by default to be a generated string. >>> srelu = SReLU(input_shape=(4, 5)) creating: createKerasSReLU """ - def __init__(self, t_left_init='zero', a_left_init='glorot_uniform', - t_right_init='glorot_uniform', a_right_init='one', + def __init__(self, t_left_init="zero", a_left_init="glorot_uniform", + t_right_init="glorot_uniform", a_right_init="one", shared_axes=None, input_shape=None, **kwargs): super(SReLU, self).__init__(None, t_left_init, @@ -1998,7 +1998,7 @@ class ELU(KerasLayer): input_shape (a shape tuple, does not include the batch dimension). # Arguments - alpha: Float, scale for the negative factor. + alpha: Float, scale for the negative factor. Default is 1.0. input_shape: A shape tuple, not including batch. name: String to set the name of the layer. If not specified, its name will by default to be a generated string. @@ -2023,7 +2023,7 @@ class LeakyReLU(KerasLayer): input_shape (a shape tuple, does not include the batch dimension). # Arguments - alpha: Float >= 0. Negative slope coefficient. + alpha: Float >= 0. Negative slope coefficient. Default is 0.3. input_shape: A shape tuple, not including batch. name: String to set the name of the layer. If not specified, its name will by default to be a generated string. @@ -2048,7 +2048,7 @@ class ThresholdedReLU(KerasLayer): input_shape (a shape tuple, does not include the batch dimension). # Arguments - theta: Float >= 0. Threshold location of activation. + theta: Float >= 0. Threshold location of activation. Default is 1.0. input_shape: A shape tuple, not including batch. name: String to set the name of the layer. If not specified, its name will by default to be a generated string. diff --git a/python/dllib/src/bigdl/dllib/bigdlkeras/layers/topology.py b/python/dllib/src/bigdl/dllib/bigdlkeras/layers/topology.py index 76e2f8b0b55..64128b2e6b2 100644 --- a/python/dllib/src/bigdl/dllib/bigdlkeras/layers/topology.py +++ b/python/dllib/src/bigdl/dllib/bigdlkeras/layers/topology.py @@ -81,7 +81,7 @@ def __convert_metrics(self, metrics): def compile(self, optimizer, loss, metrics=None): """ - Configures the learning process. Must be called before fit. + Configures the learning process. Must be called before fit or evaluate. # Arguments optimizer: Optimization method to be used. One can alternatively pass in the corresponding @@ -152,7 +152,7 @@ def evaluate(self, x, y=None, batch_size=32): Evaluate a model on a given dataset in distributed mode. # Arguments - x: Input data. ANumpy array or RDD of Sample. + x: Input data. A Numpy array or RDD of Sample. y: Labels. A Numpy array. Default is None if x is already RDD of Sample. batch_size: Number of samples per gradient update. """ From 88f26b2e745afb32e2ecf02256042a158651e0ae Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Fri, 23 Mar 2018 18:28:41 +0800 Subject: [PATCH 647/823] Keras-Style API website doc for training; refine doc format (#2441) * training api * add * finish * remove * fix * refine doc * update * remove * update --- .../bigdl/dllib/bigdlkeras/layers/layer.py | 26 +++++++++---------- .../bigdl/dllib/bigdlkeras/layers/topology.py | 4 +-- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py b/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py index 11bc2da5cc7..4d688e36a63 100644 --- a/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py +++ b/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py @@ -157,7 +157,7 @@ def __init__(self, output_dim, init="glorot_uniform", activation=None, class MaxoutDense(KerasLayer): """ - A dense maxout layer that takes the element-wise maximum of nbFeature, Dense(inputDim, outputDim) linear layers. + A dense maxout layer that takes the element-wise maximum of linear layers. This allows the layer to learn a convex, piecewise linear activation function over the inputs. The input of this layer should be 2D. @@ -478,7 +478,7 @@ class Permute(KerasLayer): input_shape (a shape tuple, does not include the batch dimension). # Arguments - dims: Tuple of int. Permutation pattern, does not include the samples dimension. Indexing starts at 1. + dims: Tuple of int. Permutation pattern, does not include the batch dimension. Indexing starts at 1. input_shape: A shape tuple, not including batch. name: String to set the name of the layer. If not specified, its name will by default to be a generated string. @@ -718,7 +718,7 @@ class AtrousConvolution1D(KerasLayer): creating: createKerasAtrousConvolution1D """ def __init__(self, nb_filter, filter_length, init="glorot_uniform", activation=None, - border_mode='valid', subsample_length=1, atrous_rate=1, W_regularizer=None, + border_mode="valid", subsample_length=1, atrous_rate=1, W_regularizer=None, b_regularizer=None, bias=True, input_shape=None, **kwargs): if border_mode != "valid": raise ValueError("For AtrousConvolution1D, only border_mode='valid' is supported for now") @@ -863,7 +863,7 @@ class SeparableConvolution2D(KerasLayer): Applies separable convolution operator for 2D inputs. Separable convolutions consist in first performing a depthwise spatial convolution (which acts on each input channel separately) followed by a pointwise convolution which mixes together the - resulting output channels. The depthMultiplier argument controls how many output channels are + resulting output channels. The depth_multiplier argument controls how many output channels are generated per input channel in the depthwise step. You can also use SeparableConv2D as an alias of this layer. The input of this layer should be 4D. @@ -1180,7 +1180,7 @@ class MaxPooling1D(KerasLayer): input_shape (a shape tuple, does not include the batch dimension). # Arguments - pool_length: Size of the region to which max pooling is applied. + pool_length: Size of the region to which max pooling is applied. Integer. Default is 2. strides: Factor by which to downscale. 2 will halve the input. Default is None, and in this case it will be equal to pool_length.. border_mode: Either 'valid' or 'same'. Default is 'valid'. @@ -1927,8 +1927,8 @@ class Masking(KerasLayer): input_shape (a shape tuple, does not include the batch dimension). # Arguments - mask_value: Float, mask value. For each timestep in the input tensor (dimension #1 in the tensor), - if all values in the input tensor at that timestep are equal to `mask_value`, + mask_value: Float, mask value. For each timestep in the input (the second dimension), + if all values in the input at that timestep are equal to 'mask_value', then the timestep will masked (skipped) in all downstream layers. input_shape: A shape tuple, not including batch. name: String to set the name of the layer. If not specified, its name will by default to be a generated string. @@ -1967,15 +1967,15 @@ class SReLU(KerasLayer): Default is None. For example, if the incoming feature maps are from a 2D convolution with output shape (batch, height, width, channels), and you wish to share parameters across space so that - each filter only has one set of parameters, set 'SharedAxes=(1,2)'. + each filter only has one set of parameters, set 'shared_axes=(1,2)'. input_shape: A shape tuple, not including batch. name: String to set the name of the layer. If not specified, its name will by default to be a generated string. >>> srelu = SReLU(input_shape=(4, 5)) creating: createKerasSReLU """ - def __init__(self, t_left_init='zero', a_left_init='glorot_uniform', - t_right_init='glorot_uniform', a_right_init='one', + def __init__(self, t_left_init="zero", a_left_init="glorot_uniform", + t_right_init="glorot_uniform", a_right_init="one", shared_axes=None, input_shape=None, **kwargs): super(SReLU, self).__init__(None, t_left_init, @@ -1998,7 +1998,7 @@ class ELU(KerasLayer): input_shape (a shape tuple, does not include the batch dimension). # Arguments - alpha: Float, scale for the negative factor. + alpha: Float, scale for the negative factor. Default is 1.0. input_shape: A shape tuple, not including batch. name: String to set the name of the layer. If not specified, its name will by default to be a generated string. @@ -2023,7 +2023,7 @@ class LeakyReLU(KerasLayer): input_shape (a shape tuple, does not include the batch dimension). # Arguments - alpha: Float >= 0. Negative slope coefficient. + alpha: Float >= 0. Negative slope coefficient. Default is 0.3. input_shape: A shape tuple, not including batch. name: String to set the name of the layer. If not specified, its name will by default to be a generated string. @@ -2048,7 +2048,7 @@ class ThresholdedReLU(KerasLayer): input_shape (a shape tuple, does not include the batch dimension). # Arguments - theta: Float >= 0. Threshold location of activation. + theta: Float >= 0. Threshold location of activation. Default is 1.0. input_shape: A shape tuple, not including batch. name: String to set the name of the layer. If not specified, its name will by default to be a generated string. diff --git a/python/dllib/src/bigdl/dllib/bigdlkeras/layers/topology.py b/python/dllib/src/bigdl/dllib/bigdlkeras/layers/topology.py index 76e2f8b0b55..64128b2e6b2 100644 --- a/python/dllib/src/bigdl/dllib/bigdlkeras/layers/topology.py +++ b/python/dllib/src/bigdl/dllib/bigdlkeras/layers/topology.py @@ -81,7 +81,7 @@ def __convert_metrics(self, metrics): def compile(self, optimizer, loss, metrics=None): """ - Configures the learning process. Must be called before fit. + Configures the learning process. Must be called before fit or evaluate. # Arguments optimizer: Optimization method to be used. One can alternatively pass in the corresponding @@ -152,7 +152,7 @@ def evaluate(self, x, y=None, batch_size=32): Evaluate a model on a given dataset in distributed mode. # Arguments - x: Input data. ANumpy array or RDD of Sample. + x: Input data. A Numpy array or RDD of Sample. y: Labels. A Numpy array. Default is None if x is already RDD of Sample. batch_size: Number of samples per gradient update. """ From 7f96cfde9c27662e8c6b1f35973df60abd76d58f Mon Sep 17 00:00:00 2001 From: dding3 Date: Fri, 23 Mar 2018 19:29:30 -0700 Subject: [PATCH 648/823] add comments for options in python rnn example readme (#2454) --- python/dllib/src/bigdl/dllib/models/rnn/README.md | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/models/rnn/README.md b/python/dllib/src/bigdl/dllib/models/rnn/README.md index 2c58cc18a72..0914f91e46f 100644 --- a/python/dllib/src/bigdl/dllib/models/rnn/README.md +++ b/python/dllib/src/bigdl/dllib/models/rnn/README.md @@ -93,9 +93,15 @@ PYSPARK_DRIVER_PYTHON=./venv/bin/python PYSPARK_PYTHON=./venv.zip/venv/bin/pytho ${BigDL_HOME}/pyspark/bigdl/models/rnn/rnnexample.py --folder hdfs://xxx:9000/rnn/ --batchSize 12 ``` -* `--folder` hdfs directory where `train.txt` and `val.txt` are located. - -* `--batchSize` option can be used to set batch size, the default value is 128. +* `--folder` hdfs directory where `train.txt` and `val.txt` are located. the default value is /tmp/rnn. +* `--batchSize` option can be used to set batch size, the default value is 12. +* `--hiddenSize` hidden unit size in the rnn cell, the default value is 40. +* `--vocabSize` vocabulary size, the default value is 4000. +* `--learningRate` inital learning rate, the default value is 0.1. +* `--weightDecay` weight decay, the default value is 0. +* `--momentum` momentum, the default value is 0. +* `--dampening` dampening for momentum, the default value is 0. +* `--maxEpoch` max number of epochs to train, the default value is 30. ## Expected Training Output Users can see the Loss of the model printed by the program. The Loss, in this case, is the perplexity of the language model. The lower, the better. From 617749d070847393c9c53f3f489a68f3b3e47118 Mon Sep 17 00:00:00 2001 From: dding3 Date: Fri, 23 Mar 2018 19:29:30 -0700 Subject: [PATCH 649/823] add comments for options in python rnn example readme (#2454) --- python/dllib/src/bigdl/dllib/models/rnn/README.md | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/models/rnn/README.md b/python/dllib/src/bigdl/dllib/models/rnn/README.md index 2c58cc18a72..0914f91e46f 100644 --- a/python/dllib/src/bigdl/dllib/models/rnn/README.md +++ b/python/dllib/src/bigdl/dllib/models/rnn/README.md @@ -93,9 +93,15 @@ PYSPARK_DRIVER_PYTHON=./venv/bin/python PYSPARK_PYTHON=./venv.zip/venv/bin/pytho ${BigDL_HOME}/pyspark/bigdl/models/rnn/rnnexample.py --folder hdfs://xxx:9000/rnn/ --batchSize 12 ``` -* `--folder` hdfs directory where `train.txt` and `val.txt` are located. - -* `--batchSize` option can be used to set batch size, the default value is 128. +* `--folder` hdfs directory where `train.txt` and `val.txt` are located. the default value is /tmp/rnn. +* `--batchSize` option can be used to set batch size, the default value is 12. +* `--hiddenSize` hidden unit size in the rnn cell, the default value is 40. +* `--vocabSize` vocabulary size, the default value is 4000. +* `--learningRate` inital learning rate, the default value is 0.1. +* `--weightDecay` weight decay, the default value is 0. +* `--momentum` momentum, the default value is 0. +* `--dampening` dampening for momentum, the default value is 0. +* `--maxEpoch` max number of epochs to train, the default value is 30. ## Expected Training Output Users can see the Loss of the model printed by the program. The Loss, in this case, is the perplexity of the language model. The lower, the better. From c99578a720586d3d494935751d748b09e6e4dfc2 Mon Sep 17 00:00:00 2001 From: Ian Wong Date: Mon, 26 Mar 2018 09:35:59 +0800 Subject: [PATCH 650/823] bump bigdl version from 0.5.0-SNAPSHOT to 0.6.0-SNAPSHOT (#2409) --- version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.py b/version.py index 81df44d6a63..c080ca23d4f 100644 --- a/version.py +++ b/version.py @@ -14,4 +14,4 @@ # limitations under the License. # -__version__ = "0.5.0.dev0" +__version__ = "0.6.0.dev0" From 3fc4751616da67b16495e5740035a29b56da71c7 Mon Sep 17 00:00:00 2001 From: Ian Wong Date: Mon, 26 Mar 2018 09:35:59 +0800 Subject: [PATCH 651/823] bump bigdl version from 0.5.0-SNAPSHOT to 0.6.0-SNAPSHOT (#2409) --- version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.py b/version.py index 81df44d6a63..c080ca23d4f 100644 --- a/version.py +++ b/version.py @@ -14,4 +14,4 @@ # limitations under the License. # -__version__ = "0.5.0.dev0" +__version__ = "0.6.0.dev0" From 3435cde6882e556d7abd4b4f77b4f796e3dc3995 Mon Sep 17 00:00:00 2001 From: Jerry Wu Date: Tue, 27 Mar 2018 14:38:34 +0800 Subject: [PATCH 652/823] Add get sample method for ImageFrame (#2447) * add get sample method * add test * fix style * fix style * fix style * fix style * fix test case * fix test case --- .../bigdl/dllib/feature/transform/vision/image.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py b/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py index 2ebab444e49..0910de067f5 100644 --- a/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py +++ b/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py @@ -171,6 +171,12 @@ def get_predict(self, key="predict"): """ return self.image_frame.get_predict(key) + def get_sample(self): + """ + get sample from ImageFrame + """ + return self.image_frame.get_sample() + class LocalImageFrame(ImageFrame): """ @@ -211,7 +217,8 @@ def get_predict(self, key="predict"): predicts = callBigDlFunc(self.bigdl_type, "localImageFrameToPredict", self.value, key) return map(lambda predict: (predict[0], predict[1].to_ndarray()) if predict[1] else (predict[0], None), predicts) - + def get_sample(self, key="sample"): + return callBigDlFunc(self.bigdl_type, "localImageFrameToSample", self.value, key) class DistributedImageFrame(ImageFrame): """ @@ -252,7 +259,8 @@ def get_predict(self, key="predict"): """ predicts = callBigDlFunc(self.bigdl_type, "distributedImageFrameToPredict", self.value, key) return predicts.map(lambda predict: (predict[0], predict[1].to_ndarray()) if predict[1] else (predict[0], None)) - + def get_sample(self, key="sample"): + return callBigDlFunc(self.bigdl_type, "distributedImageFrameToSample", self.value, key) class HFlip(FeatureTransformer): """ From 72eb44f85f39241fb4d5cf5d2d37b04a7f39d4b9 Mon Sep 17 00:00:00 2001 From: Jerry Wu Date: Tue, 27 Mar 2018 14:38:34 +0800 Subject: [PATCH 653/823] Add get sample method for ImageFrame (#2447) * add get sample method * add test * fix style * fix style * fix style * fix style * fix test case * fix test case --- .../bigdl/dllib/feature/transform/vision/image.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py b/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py index 2ebab444e49..0910de067f5 100644 --- a/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py +++ b/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py @@ -171,6 +171,12 @@ def get_predict(self, key="predict"): """ return self.image_frame.get_predict(key) + def get_sample(self): + """ + get sample from ImageFrame + """ + return self.image_frame.get_sample() + class LocalImageFrame(ImageFrame): """ @@ -211,7 +217,8 @@ def get_predict(self, key="predict"): predicts = callBigDlFunc(self.bigdl_type, "localImageFrameToPredict", self.value, key) return map(lambda predict: (predict[0], predict[1].to_ndarray()) if predict[1] else (predict[0], None), predicts) - + def get_sample(self, key="sample"): + return callBigDlFunc(self.bigdl_type, "localImageFrameToSample", self.value, key) class DistributedImageFrame(ImageFrame): """ @@ -252,7 +259,8 @@ def get_predict(self, key="predict"): """ predicts = callBigDlFunc(self.bigdl_type, "distributedImageFrameToPredict", self.value, key) return predicts.map(lambda predict: (predict[0], predict[1].to_ndarray()) if predict[1] else (predict[0], None)) - + def get_sample(self, key="sample"): + return callBigDlFunc(self.bigdl_type, "distributedImageFrameToSample", self.value, key) class HFlip(FeatureTransformer): """ From 59c46c7551310504b978f2566c223cd8d3ccadf8 Mon Sep 17 00:00:00 2001 From: Jerry Wu Date: Tue, 27 Mar 2018 14:38:34 +0800 Subject: [PATCH 654/823] Add get sample method for ImageFrame (#2447) * add get sample method * add test * fix style * fix style * fix style * fix style * fix test case * fix test case --- python/dllib/test/bigdl/transform/test_image.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/python/dllib/test/bigdl/transform/test_image.py b/python/dllib/test/bigdl/transform/test_image.py index da5af4e7cf2..3b8ff71fc3d 100644 --- a/python/dllib/test/bigdl/transform/test_image.py +++ b/python/dllib/test/bigdl/transform/test_image.py @@ -38,6 +38,14 @@ def teardown_method(self, method): """ self.sc.stop() + def test_get_sample(self): + image_frame = ImageFrame.read(self.image_path) + transformer = Pipeline([PixelBytesToMat(), Resize(256, 256), CenterCrop(224, 224), + ChannelNormalize(0.485, 0.456, 0.406, 0.229, 0.224, 0.225), + MatToTensor(), ImageFrameToSample()]) + transformed = transformer(image_frame) + transformed.get_sample() + def transformer_test(self, transformer): image_frame = ImageFrame.read(self.image_path) transformed = transformer(image_frame) From c4134e5b5742afd08209608b77d6c2d92f04bd7d Mon Sep 17 00:00:00 2001 From: Jerry Wu Date: Wed, 28 Mar 2018 14:40:29 +0800 Subject: [PATCH 655/823] [Enhancement] ImageFrame adding more APIs (#2464) * add api for evaluation * rename method * fix * wrapper parquet * fix api * fix read issue --- .../bigdl/dllib/feature/dataset/dataset.py | 10 +++++++--- .../dllib/feature/transform/vision/image.py | 11 +++++++++-- python/dllib/src/bigdl/dllib/nn/layer.py | 19 +++++++++++++------ 3 files changed, 29 insertions(+), 11 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/feature/dataset/dataset.py b/python/dllib/src/bigdl/dllib/feature/dataset/dataset.py index a6e65a795b4..1f142b9c2c7 100644 --- a/python/dllib/src/bigdl/dllib/feature/dataset/dataset.py +++ b/python/dllib/src/bigdl/dllib/feature/dataset/dataset.py @@ -26,17 +26,21 @@ class DataSet(JavaValue): - def __init__(self, jvalue=None, bigdl_type="float"): + def __init__(self, jvalue=None, image_frame = None, bigdl_type="float"): self.bigdl_type = bigdl_type if jvalue: self.value = jvalue + if image_frame: + self.image_frame = image_frame @classmethod def image_frame(cls, image_frame, bigdl_type="float"): jvalue = callBigDlFunc(bigdl_type, "createDatasetFromImageFrame", image_frame) - return DataSet(jvalue=jvalue) + return DataSet(jvalue=jvalue, image_frame = image_frame) def transform(self, transformer): if isinstance(transformer, FeatureTransformer): jvalue = callBigDlFunc(self.bigdl_type, "featureTransformDataset", self.value, transformer) - return DataSet(jvalue=jvalue) \ No newline at end of file + return DataSet(jvalue=jvalue) + def get_image_frame(self): + return self.image_frame \ No newline at end of file diff --git a/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py b/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py index 0910de067f5..8a95b3864c4 100644 --- a/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py +++ b/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py @@ -127,11 +127,18 @@ def read(cls, path, sc=None, min_partitions=1, bigdl_type="float"): return ImageFrame(jvalue=callBigDlFunc(bigdl_type, "read", path, sc, min_partitions)) @classmethod - def read_parquet(cls, path, sql_context, bigdl_type="float"): + def read_parquet(cls, path, sc, bigdl_type="float"): """ Read parquet file as DistributedImageFrame """ - return DistributedImageFrame(jvalue=callBigDlFunc(bigdl_type, "readParquet", path, sql_context)) + return DistributedImageFrame(jvalue=callBigDlFunc(bigdl_type, "readParquet", path, sc)) + + @classmethod + def write_parquet(cls, path, output, sc, partition_num = 1, bigdl_type="float"): + """ + write ImageFrame as parquet file + """ + return callBigDlFunc(bigdl_type, "writeParquet", path, output, sc, partition_num) def is_local(self): """ diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 38b4f26d521..df78aba8963 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -33,6 +33,7 @@ from py4j.java_gateway import JavaObject from pyspark.rdd import RDD from bigdl.transform.vision.image import ImageFrame +from bigdl.dataset.dataset import DataSet if sys.version >= '3': long = int @@ -334,7 +335,7 @@ def evaluate(self, *args): Three arguments passed in: A method to benchmark the model quality. - :param val_rdd: the input data + :param dataset: the input data :param batch_size: batch size :param val_methods: a list of validation methods. i.e: Top1Accuracy,Top5Accuracy and Loss. :return: a list of the metrics result @@ -344,11 +345,17 @@ def evaluate(self, *args): "evaluate", self.value) return self elif len(args) == 3: - val_rdd, batch_size, val_methods = args - return callBigDlFunc(self.bigdl_type, - "modelEvaluate", - self.value, - val_rdd, batch_size, val_methods) + dataset, batch_size, val_methods = args + if (isinstance(dataset, ImageFrame)): + return callBigDlFunc(self.bigdl_type, + "modelEvaluateImageFrame", + self.value, + dataset, batch_size, val_methods) + else: + return callBigDlFunc(self.bigdl_type, + "modelEvaluate", + self.value, + dataset, batch_size, val_methods) else: raise Exception("Error when calling evaluate(): it takes no argument or exactly three arguments only") From 31216c397c7baf34d8b1b77d2cb97c71a101ba09 Mon Sep 17 00:00:00 2001 From: Jerry Wu Date: Wed, 28 Mar 2018 14:40:29 +0800 Subject: [PATCH 656/823] [Enhancement] ImageFrame adding more APIs (#2464) * add api for evaluation * rename method * fix * wrapper parquet * fix api * fix read issue --- .../bigdl/dllib/feature/dataset/dataset.py | 10 +++++++--- .../dllib/feature/transform/vision/image.py | 11 +++++++++-- python/dllib/src/bigdl/dllib/nn/layer.py | 19 +++++++++++++------ 3 files changed, 29 insertions(+), 11 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/feature/dataset/dataset.py b/python/dllib/src/bigdl/dllib/feature/dataset/dataset.py index a6e65a795b4..1f142b9c2c7 100644 --- a/python/dllib/src/bigdl/dllib/feature/dataset/dataset.py +++ b/python/dllib/src/bigdl/dllib/feature/dataset/dataset.py @@ -26,17 +26,21 @@ class DataSet(JavaValue): - def __init__(self, jvalue=None, bigdl_type="float"): + def __init__(self, jvalue=None, image_frame = None, bigdl_type="float"): self.bigdl_type = bigdl_type if jvalue: self.value = jvalue + if image_frame: + self.image_frame = image_frame @classmethod def image_frame(cls, image_frame, bigdl_type="float"): jvalue = callBigDlFunc(bigdl_type, "createDatasetFromImageFrame", image_frame) - return DataSet(jvalue=jvalue) + return DataSet(jvalue=jvalue, image_frame = image_frame) def transform(self, transformer): if isinstance(transformer, FeatureTransformer): jvalue = callBigDlFunc(self.bigdl_type, "featureTransformDataset", self.value, transformer) - return DataSet(jvalue=jvalue) \ No newline at end of file + return DataSet(jvalue=jvalue) + def get_image_frame(self): + return self.image_frame \ No newline at end of file diff --git a/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py b/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py index 0910de067f5..8a95b3864c4 100644 --- a/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py +++ b/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py @@ -127,11 +127,18 @@ def read(cls, path, sc=None, min_partitions=1, bigdl_type="float"): return ImageFrame(jvalue=callBigDlFunc(bigdl_type, "read", path, sc, min_partitions)) @classmethod - def read_parquet(cls, path, sql_context, bigdl_type="float"): + def read_parquet(cls, path, sc, bigdl_type="float"): """ Read parquet file as DistributedImageFrame """ - return DistributedImageFrame(jvalue=callBigDlFunc(bigdl_type, "readParquet", path, sql_context)) + return DistributedImageFrame(jvalue=callBigDlFunc(bigdl_type, "readParquet", path, sc)) + + @classmethod + def write_parquet(cls, path, output, sc, partition_num = 1, bigdl_type="float"): + """ + write ImageFrame as parquet file + """ + return callBigDlFunc(bigdl_type, "writeParquet", path, output, sc, partition_num) def is_local(self): """ diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 38b4f26d521..df78aba8963 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -33,6 +33,7 @@ from py4j.java_gateway import JavaObject from pyspark.rdd import RDD from bigdl.transform.vision.image import ImageFrame +from bigdl.dataset.dataset import DataSet if sys.version >= '3': long = int @@ -334,7 +335,7 @@ def evaluate(self, *args): Three arguments passed in: A method to benchmark the model quality. - :param val_rdd: the input data + :param dataset: the input data :param batch_size: batch size :param val_methods: a list of validation methods. i.e: Top1Accuracy,Top5Accuracy and Loss. :return: a list of the metrics result @@ -344,11 +345,17 @@ def evaluate(self, *args): "evaluate", self.value) return self elif len(args) == 3: - val_rdd, batch_size, val_methods = args - return callBigDlFunc(self.bigdl_type, - "modelEvaluate", - self.value, - val_rdd, batch_size, val_methods) + dataset, batch_size, val_methods = args + if (isinstance(dataset, ImageFrame)): + return callBigDlFunc(self.bigdl_type, + "modelEvaluateImageFrame", + self.value, + dataset, batch_size, val_methods) + else: + return callBigDlFunc(self.bigdl_type, + "modelEvaluate", + self.value, + dataset, batch_size, val_methods) else: raise Exception("Error when calling evaluate(): it takes no argument or exactly three arguments only") From 274d83a2e0a96d7acc46fdb63d4ac4f152c40767 Mon Sep 17 00:00:00 2001 From: Jerry Wu Date: Wed, 28 Mar 2018 14:40:29 +0800 Subject: [PATCH 657/823] [Enhancement] ImageFrame adding more APIs (#2464) * add api for evaluation * rename method * fix * wrapper parquet * fix api * fix read issue --- python/dllib/test/bigdl/transform/test_image.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/python/dllib/test/bigdl/transform/test_image.py b/python/dllib/test/bigdl/transform/test_image.py index 3b8ff71fc3d..ef105426b44 100644 --- a/python/dllib/test/bigdl/transform/test_image.py +++ b/python/dllib/test/bigdl/transform/test_image.py @@ -18,6 +18,7 @@ import os from bigdl.util.common import * from bigdl.transform.vision.image import * +import tempfile class TestLayer(): @@ -198,6 +199,12 @@ def test_empty_get_predict_distributed(self): image_frame = ImageFrame.read(self.image_path, self.sc) image_frame.get_predict() + def test_read_write_parquet(self): + temp = tempfile.mkdtemp() + "testParquet/" + resource_path = os.path.join(os.path.split(__file__)[0], "../resources/pascal") + ImageFrame.write_parquet(resource_path, temp, self.sc, 1) + read_image_frame = ImageFrame.read_parquet(temp, self.sc) + if __name__ == "__main__": pytest.main([__file__]) From b9bae9f98be03ddec735d64b9e4166f4561da888 Mon Sep 17 00:00:00 2001 From: Jerry Wu Date: Fri, 30 Mar 2018 09:43:06 +0800 Subject: [PATCH 658/823] [Enhancement ] - Add set label and get uri api (#2470) * add set label api * update api * add examples * fix doc --- .../bigdl/dllib/examples/imageframe/README.md | 47 +++++++++++++++ .../dllib/examples/imageframe/__init__.py | 15 +++++ .../imageframe/inception_validation.py | 60 +++++++++++++++++++ .../dllib/feature/transform/vision/image.py | 19 ++++++ 4 files changed, 141 insertions(+) create mode 100644 python/dllib/src/bigdl/dllib/examples/imageframe/README.md create mode 100644 python/dllib/src/bigdl/dllib/examples/imageframe/__init__.py create mode 100644 python/dllib/src/bigdl/dllib/examples/imageframe/inception_validation.py diff --git a/python/dllib/src/bigdl/dllib/examples/imageframe/README.md b/python/dllib/src/bigdl/dllib/examples/imageframe/README.md new file mode 100644 index 00000000000..06a49c2a2f2 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/examples/imageframe/README.md @@ -0,0 +1,47 @@ +## Overview +ImageFrame provides rich deep learning APIs for scalable image processing. This example illustrates how to do model validation with Python APIs using inception-v1 model + +## Run Model validation + +### Preparation + +In order to run the validation application, you should prepare the validation dataset and inception-v1 model + +1) Prepare dataset + +This excample load ImageNet validation dataset directly from hadoop sequence file, for how to prepare the sequence file, please check [here](../../../models/inception#prepare-the-data) + +2) Download pre-trained inception model + +BigDL provides a rich set of pre-trained models, please check [BigDL Models](https://github.com/intel-analytics/analytics-zoo/tree/master/models) for details + +Download inception-v1 model by running + +wget https://s3-ap-southeast-1.amazonaws.com/bigdl-models/imageclassification/imagenet/bigdl_inception-v1_imagenet_0.4.0.model + +### Run validation program + +Run the program as a spark application with below command in standalone mode + +```shell +master=spark://xxx.xxx.xxx.xxx:xxxx # please set your own spark master +imageFolder=hdfs://... +pathToModel=... #set path to your downloaded model +batchsize=... #total batch size +PYTHON_API_ZIP_PATH=dist/lib/bigdl-VERSION-python-api.zip +BigDL_JAR_PATH=dist/lib/bigdl-VERSION-jar-with-dependencies.jar +PYTHONPATH=${PYTHON_API_ZIP_PATH}:$PYTHONPATH +${SPARK_HOME}/bin/spark-submit \ + --master $master \ + --driver-cores 10 \ + --driver-memory 5g \ + --total-executor-cores 20 \ + --executor-cores 20 \ + --executor-memory 20g \ + --py-files ${PYTHON_API_ZIP_PATH},inception_validation.py \ + --properties-file dist/conf/spark-bigdl.conf \ + --jars ${BigDL_JAR_PATH} \ + --conf spark.driver.extraClassPath=${BigDL_JAR_PATH} \ + --conf spark.executor.extraClassPath=dist/lib/bigdl-VERSION-jar-with-dependencies.jar \ + inception_validation.py ${imageFolder} ${pathToModel} ${batchsize} +``` diff --git a/python/dllib/src/bigdl/dllib/examples/imageframe/__init__.py b/python/dllib/src/bigdl/dllib/examples/imageframe/__init__.py new file mode 100644 index 00000000000..1b268b4efd1 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/examples/imageframe/__init__.py @@ -0,0 +1,15 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# \ No newline at end of file diff --git a/python/dllib/src/bigdl/dllib/examples/imageframe/inception_validation.py b/python/dllib/src/bigdl/dllib/examples/imageframe/inception_validation.py new file mode 100644 index 00000000000..bcafa980988 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/examples/imageframe/inception_validation.py @@ -0,0 +1,60 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from bigdl.util.common import * + +from bigdl.transform.vision.image import * + +from bigdl.optim.optimizer import * + +from pyspark import SparkContext + +from bigdl.nn.layer import * + +from optparse import OptionParser + +import sys + +parser = OptionParser() +parser.add_option("-f", "--folder", type=str, dest="folder", default="", + help="url of hdfs folder store the hadoop sequence files") +parser.add_option("--model", type=str, dest="model", default="", help="model path") +parser.add_option("-b", "--batchSize", type=int, dest="batchSize", default=0, help="total batch size") +def get_data(url, sc=None, data_type="val"): + path = os.path.join(url, data_type) + return SeqFileFolder.files_to_image_frame(url=path, sc=sc, class_num=1000) + +def run(image_path, model_path, batch_size): + sparkConf = create_spark_conf().setAppName("test_validation") + sc = get_spark_context(sparkConf) + init_engine() + transformer = Pipeline([PixelBytesToMat(), Resize(256, 256), CenterCrop(224, 224), + ChannelNormalize(123.0, 117.0, 104.0), + MatToTensor(), ImageFrameToSample(input_keys=["imageTensor"], + target_keys=["label"])]) + raw_image_frame = get_data(image_path, sc) + transformed = transformer(raw_image_frame) + model = Model.loadModel(model_path) + result = model.evaluate(transformed, int(batch_size), [Top1Accuracy()]) + print "top1 accuray", result[0] + +if __name__ == "__main__": + if len(sys.argv) != 3: + print "parameters needed : " + image_path = sys.argv[1] + model_path = sys.argv[2] + batch_size = sys.argv[3] + run(image_path, model_path, batch_size) diff --git a/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py b/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py index 8a95b3864c4..6b8e0addf28 100644 --- a/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py +++ b/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py @@ -184,6 +184,18 @@ def get_sample(self): """ return self.image_frame.get_sample() + def get_uri(self): + """ + get uri from imageframe + """ + return self.image_frame.get_uri() + + def set_label(self, label, bigdl_type="float"): + """ + set label for imageframe + """ + return callBigDlFunc(bigdl_type, + "setLabel", label, self.value) class LocalImageFrame(ImageFrame): """ @@ -227,6 +239,9 @@ def get_predict(self, key="predict"): def get_sample(self, key="sample"): return callBigDlFunc(self.bigdl_type, "localImageFrameToSample", self.value, key) + def get_uri(self, key = "uri"): + return callBigDlFunc(self.bigdl_type, "localImageFrameToUri", self.value, key) + class DistributedImageFrame(ImageFrame): """ DistributedImageFrame wraps an RDD of ImageFeature @@ -266,9 +281,13 @@ def get_predict(self, key="predict"): """ predicts = callBigDlFunc(self.bigdl_type, "distributedImageFrameToPredict", self.value, key) return predicts.map(lambda predict: (predict[0], predict[1].to_ndarray()) if predict[1] else (predict[0], None)) + def get_sample(self, key="sample"): return callBigDlFunc(self.bigdl_type, "distributedImageFrameToSample", self.value, key) + def get_uri(self, key = "uri"): + return callBigDlFunc(self.bigdl_type, "distributedImageFrameToUri", self.value, key) + class HFlip(FeatureTransformer): """ Flip the image horizontally From f0b2040f682225d64210add57f06e5b7d0e832dd Mon Sep 17 00:00:00 2001 From: Jerry Wu Date: Fri, 30 Mar 2018 09:43:06 +0800 Subject: [PATCH 659/823] [Enhancement ] - Add set label and get uri api (#2470) * add set label api * update api * add examples * fix doc --- .../bigdl/dllib/examples/imageframe/README.md | 47 +++++++++++++++ .../dllib/examples/imageframe/__init__.py | 15 +++++ .../imageframe/inception_validation.py | 60 +++++++++++++++++++ .../dllib/feature/transform/vision/image.py | 19 ++++++ 4 files changed, 141 insertions(+) create mode 100644 python/dllib/src/bigdl/dllib/examples/imageframe/README.md create mode 100644 python/dllib/src/bigdl/dllib/examples/imageframe/__init__.py create mode 100644 python/dllib/src/bigdl/dllib/examples/imageframe/inception_validation.py diff --git a/python/dllib/src/bigdl/dllib/examples/imageframe/README.md b/python/dllib/src/bigdl/dllib/examples/imageframe/README.md new file mode 100644 index 00000000000..06a49c2a2f2 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/examples/imageframe/README.md @@ -0,0 +1,47 @@ +## Overview +ImageFrame provides rich deep learning APIs for scalable image processing. This example illustrates how to do model validation with Python APIs using inception-v1 model + +## Run Model validation + +### Preparation + +In order to run the validation application, you should prepare the validation dataset and inception-v1 model + +1) Prepare dataset + +This excample load ImageNet validation dataset directly from hadoop sequence file, for how to prepare the sequence file, please check [here](../../../models/inception#prepare-the-data) + +2) Download pre-trained inception model + +BigDL provides a rich set of pre-trained models, please check [BigDL Models](https://github.com/intel-analytics/analytics-zoo/tree/master/models) for details + +Download inception-v1 model by running + +wget https://s3-ap-southeast-1.amazonaws.com/bigdl-models/imageclassification/imagenet/bigdl_inception-v1_imagenet_0.4.0.model + +### Run validation program + +Run the program as a spark application with below command in standalone mode + +```shell +master=spark://xxx.xxx.xxx.xxx:xxxx # please set your own spark master +imageFolder=hdfs://... +pathToModel=... #set path to your downloaded model +batchsize=... #total batch size +PYTHON_API_ZIP_PATH=dist/lib/bigdl-VERSION-python-api.zip +BigDL_JAR_PATH=dist/lib/bigdl-VERSION-jar-with-dependencies.jar +PYTHONPATH=${PYTHON_API_ZIP_PATH}:$PYTHONPATH +${SPARK_HOME}/bin/spark-submit \ + --master $master \ + --driver-cores 10 \ + --driver-memory 5g \ + --total-executor-cores 20 \ + --executor-cores 20 \ + --executor-memory 20g \ + --py-files ${PYTHON_API_ZIP_PATH},inception_validation.py \ + --properties-file dist/conf/spark-bigdl.conf \ + --jars ${BigDL_JAR_PATH} \ + --conf spark.driver.extraClassPath=${BigDL_JAR_PATH} \ + --conf spark.executor.extraClassPath=dist/lib/bigdl-VERSION-jar-with-dependencies.jar \ + inception_validation.py ${imageFolder} ${pathToModel} ${batchsize} +``` diff --git a/python/dllib/src/bigdl/dllib/examples/imageframe/__init__.py b/python/dllib/src/bigdl/dllib/examples/imageframe/__init__.py new file mode 100644 index 00000000000..1b268b4efd1 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/examples/imageframe/__init__.py @@ -0,0 +1,15 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# \ No newline at end of file diff --git a/python/dllib/src/bigdl/dllib/examples/imageframe/inception_validation.py b/python/dllib/src/bigdl/dllib/examples/imageframe/inception_validation.py new file mode 100644 index 00000000000..bcafa980988 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/examples/imageframe/inception_validation.py @@ -0,0 +1,60 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from bigdl.util.common import * + +from bigdl.transform.vision.image import * + +from bigdl.optim.optimizer import * + +from pyspark import SparkContext + +from bigdl.nn.layer import * + +from optparse import OptionParser + +import sys + +parser = OptionParser() +parser.add_option("-f", "--folder", type=str, dest="folder", default="", + help="url of hdfs folder store the hadoop sequence files") +parser.add_option("--model", type=str, dest="model", default="", help="model path") +parser.add_option("-b", "--batchSize", type=int, dest="batchSize", default=0, help="total batch size") +def get_data(url, sc=None, data_type="val"): + path = os.path.join(url, data_type) + return SeqFileFolder.files_to_image_frame(url=path, sc=sc, class_num=1000) + +def run(image_path, model_path, batch_size): + sparkConf = create_spark_conf().setAppName("test_validation") + sc = get_spark_context(sparkConf) + init_engine() + transformer = Pipeline([PixelBytesToMat(), Resize(256, 256), CenterCrop(224, 224), + ChannelNormalize(123.0, 117.0, 104.0), + MatToTensor(), ImageFrameToSample(input_keys=["imageTensor"], + target_keys=["label"])]) + raw_image_frame = get_data(image_path, sc) + transformed = transformer(raw_image_frame) + model = Model.loadModel(model_path) + result = model.evaluate(transformed, int(batch_size), [Top1Accuracy()]) + print "top1 accuray", result[0] + +if __name__ == "__main__": + if len(sys.argv) != 3: + print "parameters needed : " + image_path = sys.argv[1] + model_path = sys.argv[2] + batch_size = sys.argv[3] + run(image_path, model_path, batch_size) diff --git a/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py b/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py index 8a95b3864c4..6b8e0addf28 100644 --- a/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py +++ b/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py @@ -184,6 +184,18 @@ def get_sample(self): """ return self.image_frame.get_sample() + def get_uri(self): + """ + get uri from imageframe + """ + return self.image_frame.get_uri() + + def set_label(self, label, bigdl_type="float"): + """ + set label for imageframe + """ + return callBigDlFunc(bigdl_type, + "setLabel", label, self.value) class LocalImageFrame(ImageFrame): """ @@ -227,6 +239,9 @@ def get_predict(self, key="predict"): def get_sample(self, key="sample"): return callBigDlFunc(self.bigdl_type, "localImageFrameToSample", self.value, key) + def get_uri(self, key = "uri"): + return callBigDlFunc(self.bigdl_type, "localImageFrameToUri", self.value, key) + class DistributedImageFrame(ImageFrame): """ DistributedImageFrame wraps an RDD of ImageFeature @@ -266,9 +281,13 @@ def get_predict(self, key="predict"): """ predicts = callBigDlFunc(self.bigdl_type, "distributedImageFrameToPredict", self.value, key) return predicts.map(lambda predict: (predict[0], predict[1].to_ndarray()) if predict[1] else (predict[0], None)) + def get_sample(self, key="sample"): return callBigDlFunc(self.bigdl_type, "distributedImageFrameToSample", self.value, key) + def get_uri(self, key = "uri"): + return callBigDlFunc(self.bigdl_type, "distributedImageFrameToUri", self.value, key) + class HFlip(FeatureTransformer): """ Flip the image horizontally From 9055251c11e0ed4e57030f77bfb30428c6e6b31b Mon Sep 17 00:00:00 2001 From: Jerry Wu Date: Fri, 30 Mar 2018 09:43:06 +0800 Subject: [PATCH 660/823] [Enhancement ] - Add set label and get uri api (#2470) * add set label api * update api * add examples * fix doc --- python/dllib/test/bigdl/transform/test_image.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/python/dllib/test/bigdl/transform/test_image.py b/python/dllib/test/bigdl/transform/test_image.py index ef105426b44..09b72f8e5a7 100644 --- a/python/dllib/test/bigdl/transform/test_image.py +++ b/python/dllib/test/bigdl/transform/test_image.py @@ -205,6 +205,15 @@ def test_read_write_parquet(self): ImageFrame.write_parquet(resource_path, temp, self.sc, 1) read_image_frame = ImageFrame.read_parquet(temp, self.sc) + def test_set_label(self): + resource_path = os.path.join(os.path.split(__file__)[0], "../resources/pascal") + imageFrame = ImageFrame.read(resource_path, self.sc) + uris = imageFrame.get_uri().collect() + label = {} + for uri in uris: + label[uri] = 10 + imageFrame.set_label(label) + if __name__ == "__main__": pytest.main([__file__]) From 3489c75769519f682b3e40d5135655a246b31440 Mon Sep 17 00:00:00 2001 From: Jerry Wu Date: Fri, 30 Mar 2018 14:33:58 +0800 Subject: [PATCH 661/823] add random split (#2480) --- .../src/bigdl/dllib/feature/transform/vision/image.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py b/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py index 6b8e0addf28..e444d81590f 100644 --- a/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py +++ b/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py @@ -197,6 +197,9 @@ def set_label(self, label, bigdl_type="float"): return callBigDlFunc(bigdl_type, "setLabel", label, self.value) + def random_split(self, weights): + return self.image_frame.random_split(weights) + class LocalImageFrame(ImageFrame): """ LocalImageFrame wraps a list of ImageFeature @@ -242,6 +245,9 @@ def get_sample(self, key="sample"): def get_uri(self, key = "uri"): return callBigDlFunc(self.bigdl_type, "localImageFrameToUri", self.value, key) + def random_split(self, weights): + raise "random split not supported in LocalImageFrame" + class DistributedImageFrame(ImageFrame): """ DistributedImageFrame wraps an RDD of ImageFeature @@ -288,6 +294,9 @@ def get_sample(self, key="sample"): def get_uri(self, key = "uri"): return callBigDlFunc(self.bigdl_type, "distributedImageFrameToUri", self.value, key) + def random_split(self, weights): + return callBigDlFunc(self.bigdl_type, "distributedImageFrameRandomSplit", self.value, weights) + class HFlip(FeatureTransformer): """ Flip the image horizontally From 45bd83d9beb439ad13eb0fd0eef0b702b69d6aa2 Mon Sep 17 00:00:00 2001 From: Jerry Wu Date: Fri, 30 Mar 2018 14:33:58 +0800 Subject: [PATCH 662/823] add random split (#2480) --- .../src/bigdl/dllib/feature/transform/vision/image.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py b/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py index 6b8e0addf28..e444d81590f 100644 --- a/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py +++ b/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py @@ -197,6 +197,9 @@ def set_label(self, label, bigdl_type="float"): return callBigDlFunc(bigdl_type, "setLabel", label, self.value) + def random_split(self, weights): + return self.image_frame.random_split(weights) + class LocalImageFrame(ImageFrame): """ LocalImageFrame wraps a list of ImageFeature @@ -242,6 +245,9 @@ def get_sample(self, key="sample"): def get_uri(self, key = "uri"): return callBigDlFunc(self.bigdl_type, "localImageFrameToUri", self.value, key) + def random_split(self, weights): + raise "random split not supported in LocalImageFrame" + class DistributedImageFrame(ImageFrame): """ DistributedImageFrame wraps an RDD of ImageFeature @@ -288,6 +294,9 @@ def get_sample(self, key="sample"): def get_uri(self, key = "uri"): return callBigDlFunc(self.bigdl_type, "distributedImageFrameToUri", self.value, key) + def random_split(self, weights): + return callBigDlFunc(self.bigdl_type, "distributedImageFrameRandomSplit", self.value, weights) + class HFlip(FeatureTransformer): """ Flip the image horizontally From aa58c71e01678b4cb86f1fa719f60c972f3112ed Mon Sep 17 00:00:00 2001 From: Jerry Wu Date: Fri, 30 Mar 2018 14:33:58 +0800 Subject: [PATCH 663/823] add random split (#2480) --- python/dllib/test/bigdl/transform/test_image.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/python/dllib/test/bigdl/transform/test_image.py b/python/dllib/test/bigdl/transform/test_image.py index 09b72f8e5a7..fcf8e73ffb6 100644 --- a/python/dllib/test/bigdl/transform/test_image.py +++ b/python/dllib/test/bigdl/transform/test_image.py @@ -214,6 +214,10 @@ def test_set_label(self): label[uri] = 10 imageFrame.set_label(label) + def test_random_split(self): + resource_path = os.path.join(os.path.split(__file__)[0], "../resources/pascal") + imageFrame = ImageFrame.read(resource_path, self.sc) + splits = imageFrame.random_split([1.0]) if __name__ == "__main__": pytest.main([__file__]) From 7701240afc7959617d597519ba8b7d20d5437149 Mon Sep 17 00:00:00 2001 From: Jerry Wu Date: Mon, 9 Apr 2018 14:58:43 +0800 Subject: [PATCH 664/823] fix python (#2488) --- .../src/bigdl/dllib/feature/transform/vision/image.py | 8 +++++++- python/dllib/src/bigdl/dllib/utils/common.py | 3 +++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py b/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py index e444d81590f..b4596995c6a 100644 --- a/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py +++ b/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py @@ -198,7 +198,13 @@ def set_label(self, label, bigdl_type="float"): "setLabel", label, self.value) def random_split(self, weights): - return self.image_frame.random_split(weights) + """ + Random split imageframes according to weights + :param weights: weights for each ImageFrame + :return: + """ + jvalues = self.image_frame.random_split(weights) + return [ImageFrame(jvalue) for jvalue in jvalues] class LocalImageFrame(ImageFrame): """ diff --git a/python/dllib/src/bigdl/dllib/utils/common.py b/python/dllib/src/bigdl/dllib/utils/common.py index 6c09953d046..e2265b8e2bc 100644 --- a/python/dllib/src/bigdl/dllib/utils/common.py +++ b/python/dllib/src/bigdl/dllib/utils/common.py @@ -609,6 +609,9 @@ def _java2py(gateway, r, encoding="bytes"): if clsName == 'Dataset': return DataFrame(r, get_spark_sql_context(get_spark_context())) + if clsName == "ImageFrame[]": + return r + if clsName in _picklable_classes: r = gateway.jvm.org.apache.spark.bigdl.api.python.BigDLSerDe.dumps(r) elif isinstance(r, (JavaArray, JavaList, JavaMap)): From 5fe3ac58e359addbdfe928a593d9c78abe217b20 Mon Sep 17 00:00:00 2001 From: Jerry Wu Date: Mon, 9 Apr 2018 14:58:43 +0800 Subject: [PATCH 665/823] fix python (#2488) --- .../src/bigdl/dllib/feature/transform/vision/image.py | 8 +++++++- python/dllib/src/bigdl/utils/common.py | 3 +++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py b/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py index e444d81590f..b4596995c6a 100644 --- a/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py +++ b/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py @@ -198,7 +198,13 @@ def set_label(self, label, bigdl_type="float"): "setLabel", label, self.value) def random_split(self, weights): - return self.image_frame.random_split(weights) + """ + Random split imageframes according to weights + :param weights: weights for each ImageFrame + :return: + """ + jvalues = self.image_frame.random_split(weights) + return [ImageFrame(jvalue) for jvalue in jvalues] class LocalImageFrame(ImageFrame): """ diff --git a/python/dllib/src/bigdl/utils/common.py b/python/dllib/src/bigdl/utils/common.py index 6c09953d046..e2265b8e2bc 100644 --- a/python/dllib/src/bigdl/utils/common.py +++ b/python/dllib/src/bigdl/utils/common.py @@ -609,6 +609,9 @@ def _java2py(gateway, r, encoding="bytes"): if clsName == 'Dataset': return DataFrame(r, get_spark_sql_context(get_spark_context())) + if clsName == "ImageFrame[]": + return r + if clsName in _picklable_classes: r = gateway.jvm.org.apache.spark.bigdl.api.python.BigDLSerDe.dumps(r) elif isinstance(r, (JavaArray, JavaList, JavaMap)): From e492c2c90d946ceb2c9254e0628c325a6b468738 Mon Sep 17 00:00:00 2001 From: Xin Qiu Date: Tue, 22 May 2018 11:12:08 +0800 Subject: [PATCH 666/823] [new feature]add generateBackward for loadTF (#2529) --- python/dllib/src/bigdl/dllib/nn/layer.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index df78aba8963..79473747e3e 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -830,7 +830,7 @@ def load_caffe_model(defPath, modelPath, bigdl_type="float"): @staticmethod def load_tensorflow(path, inputs, outputs, byte_order = "little_endian", - bin_file = None, bigdl_type="float"): + bin_file = None, generated_backward = True, bigdl_type = "float"): """ Load a pre-trained Tensorflow model. :param path: The path containing the pre-trained model. @@ -838,9 +838,11 @@ def load_tensorflow(path, inputs, outputs, byte_order = "little_endian", :param outputs: The output node of this graph :param byte_order: byte_order of the file, `little_endian` or `big_endian` :param bin_file: the optional bin file produced by bigdl dump_model util function to store the weights + :param generated_backward: if generate backward graph :return: A pre-trained model. """ - jmodel = callBigDlFunc(bigdl_type, "loadTF", path, inputs, outputs, byte_order, bin_file) + jmodel = callBigDlFunc(bigdl_type, "loadTF", path, inputs, outputs, + byte_order, bin_file, generated_backward) return Model.of(jmodel) @staticmethod From 154919cacfea37a14e2ab10e40d250235940925d Mon Sep 17 00:00:00 2001 From: Xin Qiu Date: Tue, 22 May 2018 11:12:08 +0800 Subject: [PATCH 667/823] [new feature]add generateBackward for loadTF (#2529) --- python/dllib/src/bigdl/dllib/nn/layer.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index df78aba8963..79473747e3e 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -830,7 +830,7 @@ def load_caffe_model(defPath, modelPath, bigdl_type="float"): @staticmethod def load_tensorflow(path, inputs, outputs, byte_order = "little_endian", - bin_file = None, bigdl_type="float"): + bin_file = None, generated_backward = True, bigdl_type = "float"): """ Load a pre-trained Tensorflow model. :param path: The path containing the pre-trained model. @@ -838,9 +838,11 @@ def load_tensorflow(path, inputs, outputs, byte_order = "little_endian", :param outputs: The output node of this graph :param byte_order: byte_order of the file, `little_endian` or `big_endian` :param bin_file: the optional bin file produced by bigdl dump_model util function to store the weights + :param generated_backward: if generate backward graph :return: A pre-trained model. """ - jmodel = callBigDlFunc(bigdl_type, "loadTF", path, inputs, outputs, byte_order, bin_file) + jmodel = callBigDlFunc(bigdl_type, "loadTF", path, inputs, outputs, + byte_order, bin_file, generated_backward) return Model.of(jmodel) @staticmethod From b31c46bf5a8952d5ee7663a69040b1d1823db31a Mon Sep 17 00:00:00 2001 From: Xin Qiu Date: Tue, 22 May 2018 11:12:08 +0800 Subject: [PATCH 668/823] [new feature]add generateBackward for loadTF (#2529) --- python/dllib/test/bigdl/test_tensorflow.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/python/dllib/test/bigdl/test_tensorflow.py b/python/dllib/test/bigdl/test_tensorflow.py index 70304617211..aedbd90cdb5 100644 --- a/python/dllib/test/bigdl/test_tensorflow.py +++ b/python/dllib/test/bigdl/test_tensorflow.py @@ -38,9 +38,14 @@ def test_load_and_save(self): model_original.save_tensorflow([("input", [4, 10])], temp + "/model.pb") model_loaded = Model.load_tensorflow(temp + "/model.pb", ["input"], ["output"]) + model_loaded_without_backwardgraph = Model.load_tensorflow(temp + "/model.pb", + ["input"], ["output"], + generated_backward=False) expected_output = model_original.forward(input) output = model_loaded.forward(input) + output_without_backwardgraph = model_loaded_without_backwardgraph.forward(input) assert_allclose(output, expected_output, atol=1e-6, rtol=0) + assert_allclose(output_without_backwardgraph, expected_output, atol=1e-6, rtol=0) if __name__ == "__main__": From 832b73a01d0f311f549c1912a479b3f500c07a16 Mon Sep 17 00:00:00 2001 From: Xin Qiu Date: Tue, 22 May 2018 16:14:37 +0800 Subject: [PATCH 669/823] [python API]add batchSize support in model.predict (#2518) * add batchSize for predict * meet code review * meet code review * fix space * fix ut --- python/dllib/src/bigdl/dllib/nn/layer.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 79473747e3e..09fb62c2b0b 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -369,17 +369,19 @@ def _to_jtensors(self, x): raise Exception("Not supported type: %s" % type(x[0])) - def predict_local(self, X): + def predict_local(self, X, batch_size = -1): """ :param X: X can be a ndarray or list of ndarray if the model has multiple inputs. The first dimension of X should be batch. + :param batch_size: total batch size of prediction. :return: a ndarray as the prediction result. """ jresults = callBigDlFunc(self.bigdl_type, "predictLocal", self.value, - self._to_jtensors(X)) + self._to_jtensors(X), + batch_size) return np.stack([j.to_ndarray()for j in jresults]) @@ -396,17 +398,18 @@ def predict_class_local(self, X): self._to_jtensors(X)) return np.stack(result) - def predict(self, features): + def predict(self, features, batch_size = -1): """ Model inference base on the given data. :param features: it can be a ndarray or list of ndarray for locally inference or RDD[Sample] for running in distributed fashion + :param batch_size: total batch size of prediction. :return: ndarray or RDD[Sample] depend on the the type of features. """ if isinstance(features, RDD): - return self.predict_distributed(features) + return self.predict_distributed(features, batch_size) else: - return self.predict_local(features) + return self.predict_local(features, batch_size) def predict_class(self, features): """ @@ -420,17 +423,18 @@ def predict_class(self, features): else: return self.predict_class_local(features) - def predict_distributed(self, data_rdd): + def predict_distributed(self, data_rdd, batch_size = -1): """ Model inference base on the given data. You need to invoke collect() to trigger those action \ as the returning result is an RDD. :param data_rdd: the data to be predict. + :param batch_size: total batch size of prediction. :return: An RDD represent the predict result. """ result = callBigDlFunc(self.bigdl_type, - "modelPredictRDD", self.value, data_rdd) + "modelPredictRDD", self.value, data_rdd, batch_size) return result.map(lambda data: data.to_ndarray()) def predict_class_distributed(self, data_rdd): From eea17f67c762b84d08b6b6199dfef4a77a07637e Mon Sep 17 00:00:00 2001 From: Xin Qiu Date: Tue, 22 May 2018 16:14:37 +0800 Subject: [PATCH 670/823] [python API]add batchSize support in model.predict (#2518) * add batchSize for predict * meet code review * meet code review * fix space * fix ut --- python/dllib/src/bigdl/dllib/nn/layer.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 79473747e3e..09fb62c2b0b 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -369,17 +369,19 @@ def _to_jtensors(self, x): raise Exception("Not supported type: %s" % type(x[0])) - def predict_local(self, X): + def predict_local(self, X, batch_size = -1): """ :param X: X can be a ndarray or list of ndarray if the model has multiple inputs. The first dimension of X should be batch. + :param batch_size: total batch size of prediction. :return: a ndarray as the prediction result. """ jresults = callBigDlFunc(self.bigdl_type, "predictLocal", self.value, - self._to_jtensors(X)) + self._to_jtensors(X), + batch_size) return np.stack([j.to_ndarray()for j in jresults]) @@ -396,17 +398,18 @@ def predict_class_local(self, X): self._to_jtensors(X)) return np.stack(result) - def predict(self, features): + def predict(self, features, batch_size = -1): """ Model inference base on the given data. :param features: it can be a ndarray or list of ndarray for locally inference or RDD[Sample] for running in distributed fashion + :param batch_size: total batch size of prediction. :return: ndarray or RDD[Sample] depend on the the type of features. """ if isinstance(features, RDD): - return self.predict_distributed(features) + return self.predict_distributed(features, batch_size) else: - return self.predict_local(features) + return self.predict_local(features, batch_size) def predict_class(self, features): """ @@ -420,17 +423,18 @@ def predict_class(self, features): else: return self.predict_class_local(features) - def predict_distributed(self, data_rdd): + def predict_distributed(self, data_rdd, batch_size = -1): """ Model inference base on the given data. You need to invoke collect() to trigger those action \ as the returning result is an RDD. :param data_rdd: the data to be predict. + :param batch_size: total batch size of prediction. :return: An RDD represent the predict result. """ result = callBigDlFunc(self.bigdl_type, - "modelPredictRDD", self.value, data_rdd) + "modelPredictRDD", self.value, data_rdd, batch_size) return result.map(lambda data: data.to_ndarray()) def predict_class_distributed(self, data_rdd): From f34594a955e493cb3433a9d829ab46fe4481308c Mon Sep 17 00:00:00 2001 From: Xin Qiu Date: Tue, 22 May 2018 16:14:37 +0800 Subject: [PATCH 671/823] [python API]add batchSize support in model.predict (#2518) * add batchSize for predict * meet code review * meet code review * fix space * fix ut --- python/dllib/test/bigdl/test_simple_integration.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/python/dllib/test/bigdl/test_simple_integration.py b/python/dllib/test/bigdl/test_simple_integration.py index f9ba1acc9c9..32de1ecc13a 100644 --- a/python/dllib/test/bigdl/test_simple_integration.py +++ b/python/dllib/test/bigdl/test_simple_integration.py @@ -523,6 +523,13 @@ def test_predict(self): [-0.5906958], [-0.12307882], [-0.77907401]], dtype="float32") for i in range(0, total_length): assert_allclose(p[i], ground_label[i], atol=1e-6, rtol=0) + + predict_result_with_batch = model.predict(features=predict_data, + batch_size=4) + p_with_batch = predict_result_with_batch.take(6) + for i in range(0, total_length): + assert_allclose(p_with_batch[i], ground_label[i], atol=1e-6, rtol=0) + predict_class = model.predict_class(predict_data) predict_labels = predict_class.take(6) for i in range(0, total_length): @@ -650,6 +657,9 @@ def test_local_predict_multiple_input(self): result4 = model.predict_class([JTensor.from_ndarray(np.ones([4, 3])), JTensor.from_ndarray(np.ones([4, 3]))]) assert result4.shape == (4,) + result5 = model.predict_local([JTensor.from_ndarray(np.ones([4, 3])), + JTensor.from_ndarray(np.ones([4, 3]))], batch_size=2) + assert result5.shape == (4, 5) def test_model_broadcast(self): From a8d94d2a032a9b3685d212f14e6ef8eb58eabb3f Mon Sep 17 00:00:00 2001 From: Xin Qiu Date: Mon, 4 Jun 2018 17:29:39 +0800 Subject: [PATCH 672/823] [new feature]new optimMethod ftrl (#2495) --- .../dllib/src/bigdl/dllib/optim/optimizer.py | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/optim/optimizer.py b/python/dllib/src/bigdl/dllib/optim/optimizer.py index 0f26db3165c..d9afd99f6a4 100644 --- a/python/dllib/src/bigdl/dllib/optim/optimizer.py +++ b/python/dllib/src/bigdl/dllib/optim/optimizer.py @@ -511,6 +511,39 @@ def __init__(self, super(Adam, self).__init__(None, bigdl_type, learningrate, learningrate_decay, beta1, beta2, epsilon) +class Ftrl(OptimMethod): + """ + An implementation of Ftrl https://www.eecs.tufts.edu/~dsculley/papers/ad-click-prediction.pdf. + Support L1 penalty, L2 penalty and shrinkage-type L2 penalty. + + :param learningrate learning rate + :param learningrate_power double, must be less or equal to zero. Default is -0.5. + :param initial_accumulator_value double, the starting value for accumulators, + require zero or positive values. + :param l1_regularization_strength double, must be greater or equal to zero. Default is zero. + :param l2_regularization_strength double, must be greater or equal to zero. Default is zero. + :param l2_shrinkage_regularization_strength double, must be greater or equal to zero. + Default is zero. This differs from l2RegularizationStrength above. L2 above is a + stabilization penalty, whereas this one is a magnitude penalty. + >>> ftrl = Ftrl() + creating: createFtrl + >>> ftrl2 = Ftrl(1e-2, -0.1, 0.2, 0.3, 0.4, 0.5) + creating: createFtrl + """ + def __init__(self, + learningrate = 1e-3, + learningrate_power = -0.5, + initial_accumulator_value = 0.1, + l1_regularization_strength = 0.0, + l2_regularization_strength = 0.0, + l2_shrinkage_regularization_strength = 0.0, + bigdl_type="float"): + super(Ftrl, self).__init__(None, bigdl_type, learningrate, learningrate_power, + initial_accumulator_value, + l1_regularization_strength, + l2_regularization_strength, + l2_shrinkage_regularization_strength) + class Adamax(OptimMethod): """ An implementation of Adamax http://arxiv.org/pdf/1412.6980.pdf From 717b0055efb8bfa09269ad989c48d2b6b9e45eae Mon Sep 17 00:00:00 2001 From: Xin Qiu Date: Mon, 4 Jun 2018 17:29:39 +0800 Subject: [PATCH 673/823] [new feature]new optimMethod ftrl (#2495) --- .../dllib/src/bigdl/dllib/optim/optimizer.py | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/optim/optimizer.py b/python/dllib/src/bigdl/dllib/optim/optimizer.py index 0f26db3165c..d9afd99f6a4 100644 --- a/python/dllib/src/bigdl/dllib/optim/optimizer.py +++ b/python/dllib/src/bigdl/dllib/optim/optimizer.py @@ -511,6 +511,39 @@ def __init__(self, super(Adam, self).__init__(None, bigdl_type, learningrate, learningrate_decay, beta1, beta2, epsilon) +class Ftrl(OptimMethod): + """ + An implementation of Ftrl https://www.eecs.tufts.edu/~dsculley/papers/ad-click-prediction.pdf. + Support L1 penalty, L2 penalty and shrinkage-type L2 penalty. + + :param learningrate learning rate + :param learningrate_power double, must be less or equal to zero. Default is -0.5. + :param initial_accumulator_value double, the starting value for accumulators, + require zero or positive values. + :param l1_regularization_strength double, must be greater or equal to zero. Default is zero. + :param l2_regularization_strength double, must be greater or equal to zero. Default is zero. + :param l2_shrinkage_regularization_strength double, must be greater or equal to zero. + Default is zero. This differs from l2RegularizationStrength above. L2 above is a + stabilization penalty, whereas this one is a magnitude penalty. + >>> ftrl = Ftrl() + creating: createFtrl + >>> ftrl2 = Ftrl(1e-2, -0.1, 0.2, 0.3, 0.4, 0.5) + creating: createFtrl + """ + def __init__(self, + learningrate = 1e-3, + learningrate_power = -0.5, + initial_accumulator_value = 0.1, + l1_regularization_strength = 0.0, + l2_regularization_strength = 0.0, + l2_shrinkage_regularization_strength = 0.0, + bigdl_type="float"): + super(Ftrl, self).__init__(None, bigdl_type, learningrate, learningrate_power, + initial_accumulator_value, + l1_regularization_strength, + l2_regularization_strength, + l2_shrinkage_regularization_strength) + class Adamax(OptimMethod): """ An implementation of Adamax http://arxiv.org/pdf/1412.6980.pdf From bdc988ba674e852542481cdd97182c869bd13d44 Mon Sep 17 00:00:00 2001 From: Jerry Wu Date: Tue, 12 Jun 2018 14:08:05 +0800 Subject: [PATCH 674/823] fix (#2549) --- python/dllib/src/bigdl/dllib/feature/transform/vision/image.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py b/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py index b4596995c6a..6fef178e9fb 100644 --- a/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py +++ b/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py @@ -384,7 +384,7 @@ class ChannelNormalize(FeatureTransformer): :param std_g std value in G channel :param std_b std value in B channel """ - def __init__(self, mean_r, mean_b, mean_g, std_r=1.0, std_g=1.0, std_b=1.0, bigdl_type="float"): + def __init__(self, mean_r, mean_g, mean_b, std_r=1.0, std_g=1.0, std_b=1.0, bigdl_type="float"): super(ChannelNormalize, self).__init__(bigdl_type, mean_r, mean_g, mean_b, std_r, std_g, std_b) class PixelNormalize(FeatureTransformer): From 20d4262197669c8ce92697563954d2403d69f3ed Mon Sep 17 00:00:00 2001 From: Jerry Wu Date: Tue, 12 Jun 2018 14:08:05 +0800 Subject: [PATCH 675/823] fix (#2549) --- python/dllib/src/bigdl/dllib/feature/transform/vision/image.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py b/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py index b4596995c6a..6fef178e9fb 100644 --- a/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py +++ b/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py @@ -384,7 +384,7 @@ class ChannelNormalize(FeatureTransformer): :param std_g std value in G channel :param std_b std value in B channel """ - def __init__(self, mean_r, mean_b, mean_g, std_r=1.0, std_g=1.0, std_b=1.0, bigdl_type="float"): + def __init__(self, mean_r, mean_g, mean_b, std_r=1.0, std_g=1.0, std_b=1.0, bigdl_type="float"): super(ChannelNormalize, self).__init__(bigdl_type, mean_r, mean_g, mean_b, std_r, std_g, std_b) class PixelNormalize(FeatureTransformer): From f82450a5526689169d9362510e5eec8b7e679875 Mon Sep 17 00:00:00 2001 From: Xin Qiu Date: Tue, 26 Jun 2018 13:53:54 +0800 Subject: [PATCH 676/823] [new feature] multi optimMethods support in Optimizer (#2560) * multiOptimMethod * some update * fix unit test * fix ut * fix unit test * fix python unit test * meet code review * update optimizer.py * update python * meet code review --- .../dllib/src/bigdl/dllib/optim/optimizer.py | 23 +++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/optim/optimizer.py b/python/dllib/src/bigdl/dllib/optim/optimizer.py index d9afd99f6a4..07ce236ba48 100644 --- a/python/dllib/src/bigdl/dllib/optim/optimizer.py +++ b/python/dllib/src/bigdl/dllib/optim/optimizer.py @@ -846,16 +846,23 @@ def __init__(self, :param end_trigger: when to end the optimization :param batch_size: training batch size """ + if not optim_method: + optim_methods = {model.name(): SGD()} + elif isinstance(optim_method, OptimMethod): + optim_methods = {model.name(): optim_method} + elif isinstance(optim_method, JavaObject): + optim_methods = {model.name(): OptimMethod(optim_method, bigdl_type)} + else: + optim_methods = optim_method if isinstance(training_rdd, RDD): JavaValue.__init__(self, None, bigdl_type, model.value, training_rdd, criterion, - optim_method if optim_method else SGD(), end_trigger, batch_size) + optim_methods, end_trigger, batch_size) elif isinstance(training_rdd, DataSet): self.bigdl_type = bigdl_type self.value = callBigDlFunc(self.bigdl_type, "createDistriOptimizerFromDataSet", model.value, training_rdd, criterion, - optim_method if optim_method else SGD(), - end_trigger, batch_size) + optim_methods, end_trigger, batch_size) class LocalOptimizer(BaseOptimizer): @@ -883,6 +890,14 @@ def __init__(self, optim_method=None, cores=None, bigdl_type="float"): + if not optim_method: + optim_methods = {model.name(): SGD()} + elif isinstance(optim_method, OptimMethod): + optim_methods = {model.name(): optim_method} + elif isinstance(optim_method, JavaObject): + optim_methods = {model.name(): OptimMethod(optim_method, bigdl_type)} + else: + optim_methods = optim_method if cores is None: cores = multiprocessing.cpu_count() JavaValue.__init__(self, None, bigdl_type, @@ -890,7 +905,7 @@ def __init__(self, JTensor.from_ndarray(Y), model.value, criterion, - optim_method if optim_method else SGD(), end_trigger, batch_size, cores) + optim_methods, end_trigger, batch_size, cores) def set_validation(self, batch_size, X_val, Y_val, trigger, val_method=None): """ From 0d60e2770fced0d9610768c7e0fd3cb19c87533e Mon Sep 17 00:00:00 2001 From: Xin Qiu Date: Tue, 26 Jun 2018 13:53:54 +0800 Subject: [PATCH 677/823] [new feature] multi optimMethods support in Optimizer (#2560) * multiOptimMethod * some update * fix unit test * fix ut * fix unit test * fix python unit test * meet code review * update optimizer.py * update python * meet code review --- .../dllib/src/bigdl/dllib/optim/optimizer.py | 23 +++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/optim/optimizer.py b/python/dllib/src/bigdl/dllib/optim/optimizer.py index d9afd99f6a4..07ce236ba48 100644 --- a/python/dllib/src/bigdl/dllib/optim/optimizer.py +++ b/python/dllib/src/bigdl/dllib/optim/optimizer.py @@ -846,16 +846,23 @@ def __init__(self, :param end_trigger: when to end the optimization :param batch_size: training batch size """ + if not optim_method: + optim_methods = {model.name(): SGD()} + elif isinstance(optim_method, OptimMethod): + optim_methods = {model.name(): optim_method} + elif isinstance(optim_method, JavaObject): + optim_methods = {model.name(): OptimMethod(optim_method, bigdl_type)} + else: + optim_methods = optim_method if isinstance(training_rdd, RDD): JavaValue.__init__(self, None, bigdl_type, model.value, training_rdd, criterion, - optim_method if optim_method else SGD(), end_trigger, batch_size) + optim_methods, end_trigger, batch_size) elif isinstance(training_rdd, DataSet): self.bigdl_type = bigdl_type self.value = callBigDlFunc(self.bigdl_type, "createDistriOptimizerFromDataSet", model.value, training_rdd, criterion, - optim_method if optim_method else SGD(), - end_trigger, batch_size) + optim_methods, end_trigger, batch_size) class LocalOptimizer(BaseOptimizer): @@ -883,6 +890,14 @@ def __init__(self, optim_method=None, cores=None, bigdl_type="float"): + if not optim_method: + optim_methods = {model.name(): SGD()} + elif isinstance(optim_method, OptimMethod): + optim_methods = {model.name(): optim_method} + elif isinstance(optim_method, JavaObject): + optim_methods = {model.name(): OptimMethod(optim_method, bigdl_type)} + else: + optim_methods = optim_method if cores is None: cores = multiprocessing.cpu_count() JavaValue.__init__(self, None, bigdl_type, @@ -890,7 +905,7 @@ def __init__(self, JTensor.from_ndarray(Y), model.value, criterion, - optim_method if optim_method else SGD(), end_trigger, batch_size, cores) + optim_methods, end_trigger, batch_size, cores) def set_validation(self, batch_size, X_val, Y_val, trigger, val_method=None): """ From 33f599bef2a22f7eccc63a4bc4edf26a94eea35f Mon Sep 17 00:00:00 2001 From: Ian Wong Date: Wed, 4 Jul 2018 18:18:13 +0800 Subject: [PATCH 678/823] bump version to 0.7.0-SNAPSHOT (#2577) --- .../dllib/src/bigdl/dllib/models/textclassifier/README.md | 8 ++++---- version.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/models/textclassifier/README.md b/python/dllib/src/bigdl/dllib/models/textclassifier/README.md index 53e44da3a5f..bc685129a13 100644 --- a/python/dllib/src/bigdl/dllib/models/textclassifier/README.md +++ b/python/dllib/src/bigdl/dllib/models/textclassifier/README.md @@ -28,8 +28,8 @@ $ [/tmp/news20]$ tree . -L 1 - The example code would automatically download the data during the first run. ### Run via pip install -- [Install from pip](https://bigdl-project.github.io/0.6.0-SNAPSHOT/#PythonUserGuide/install-from-pip/) -- Optional: check [Run after pip install](https://bigdl-project.github.io/0.6.0-SNAPSHOT/#PythonUserGuide/run-from-pip/) +- [Install from pip](https://bigdl-project.github.io/0.7.0-SNAPSHOT/#PythonUserGuide/install-from-pip/) +- Optional: check [Run after pip install](https://bigdl-project.github.io/0.7.0-SNAPSHOT/#PythonUserGuide/run-from-pip/) - Run the following command locally ``` python ${BigDL_HOME}/pyspark/bigdl/models/textclassifier/textclassifier.py --max_epoch 3 --model cnn @@ -37,8 +37,8 @@ python ${BigDL_HOME}/pyspark/bigdl/models/textclassifier/textclassifier.py --max ``` ### Run via spark-submit -- [Install without pip](https://bigdl-project.github.io/0.6.0-SNAPSHOT/#PythonUserGuide/install-without-pip/) -- Optional: check [Run without pip](https://bigdl-project.github.io/0.6.0-SNAPSHOT/#PythonUserGuide/run-without-pip/) +- [Install without pip](https://bigdl-project.github.io/0.7.0-SNAPSHOT/#PythonUserGuide/install-without-pip/) +- Optional: check [Run without pip](https://bigdl-project.github.io/0.7.0-SNAPSHOT/#PythonUserGuide/run-without-pip/) - Run the following command ```{r, engine='sh'} PYTHONHASHSEED=0 diff --git a/version.py b/version.py index c080ca23d4f..50b4b23a3fb 100644 --- a/version.py +++ b/version.py @@ -14,4 +14,4 @@ # limitations under the License. # -__version__ = "0.6.0.dev0" +__version__ = "0.7.0.dev0" From 5f12afa63434b4fa9ee678ffae2a340685778daf Mon Sep 17 00:00:00 2001 From: Ian Wong Date: Wed, 4 Jul 2018 18:18:13 +0800 Subject: [PATCH 679/823] bump version to 0.7.0-SNAPSHOT (#2577) --- .../dllib/src/bigdl/dllib/models/textclassifier/README.md | 8 ++++---- version.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/models/textclassifier/README.md b/python/dllib/src/bigdl/dllib/models/textclassifier/README.md index 53e44da3a5f..bc685129a13 100644 --- a/python/dllib/src/bigdl/dllib/models/textclassifier/README.md +++ b/python/dllib/src/bigdl/dllib/models/textclassifier/README.md @@ -28,8 +28,8 @@ $ [/tmp/news20]$ tree . -L 1 - The example code would automatically download the data during the first run. ### Run via pip install -- [Install from pip](https://bigdl-project.github.io/0.6.0-SNAPSHOT/#PythonUserGuide/install-from-pip/) -- Optional: check [Run after pip install](https://bigdl-project.github.io/0.6.0-SNAPSHOT/#PythonUserGuide/run-from-pip/) +- [Install from pip](https://bigdl-project.github.io/0.7.0-SNAPSHOT/#PythonUserGuide/install-from-pip/) +- Optional: check [Run after pip install](https://bigdl-project.github.io/0.7.0-SNAPSHOT/#PythonUserGuide/run-from-pip/) - Run the following command locally ``` python ${BigDL_HOME}/pyspark/bigdl/models/textclassifier/textclassifier.py --max_epoch 3 --model cnn @@ -37,8 +37,8 @@ python ${BigDL_HOME}/pyspark/bigdl/models/textclassifier/textclassifier.py --max ``` ### Run via spark-submit -- [Install without pip](https://bigdl-project.github.io/0.6.0-SNAPSHOT/#PythonUserGuide/install-without-pip/) -- Optional: check [Run without pip](https://bigdl-project.github.io/0.6.0-SNAPSHOT/#PythonUserGuide/run-without-pip/) +- [Install without pip](https://bigdl-project.github.io/0.7.0-SNAPSHOT/#PythonUserGuide/install-without-pip/) +- Optional: check [Run without pip](https://bigdl-project.github.io/0.7.0-SNAPSHOT/#PythonUserGuide/run-without-pip/) - Run the following command ```{r, engine='sh'} PYTHONHASHSEED=0 diff --git a/version.py b/version.py index c080ca23d4f..50b4b23a3fb 100644 --- a/version.py +++ b/version.py @@ -14,4 +14,4 @@ # limitations under the License. # -__version__ = "0.6.0.dev0" +__version__ = "0.7.0.dev0" From 80e4934fb14197ffdca9c4e81198e474e1f011f7 Mon Sep 17 00:00:00 2001 From: Ian Wong Date: Wed, 11 Jul 2018 11:27:45 +0800 Subject: [PATCH 680/823] fix dlframes not packaged in pip issue (#2588) * fix dlframes not packaged in pip issue * add missing __init__.py files & auto discover the package list & ignore egg-info folder --- python/dllib/src/bigdl/dllib/models/inception/__init__.py | 0 python/dllib/src/bigdl/dllib/models/rnn/__init__.py | 0 python/dllib/src/bigdl/dllib/models/textclassifier/__init__.py | 0 3 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 python/dllib/src/bigdl/dllib/models/inception/__init__.py create mode 100644 python/dllib/src/bigdl/dllib/models/rnn/__init__.py create mode 100644 python/dllib/src/bigdl/dllib/models/textclassifier/__init__.py diff --git a/python/dllib/src/bigdl/dllib/models/inception/__init__.py b/python/dllib/src/bigdl/dllib/models/inception/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/python/dllib/src/bigdl/dllib/models/rnn/__init__.py b/python/dllib/src/bigdl/dllib/models/rnn/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/python/dllib/src/bigdl/dllib/models/textclassifier/__init__.py b/python/dllib/src/bigdl/dllib/models/textclassifier/__init__.py new file mode 100644 index 00000000000..e69de29bb2d From 6e20f9f0484760a756844d069af30fa5d667e1bb Mon Sep 17 00:00:00 2001 From: Ian Wong Date: Wed, 11 Jul 2018 11:27:45 +0800 Subject: [PATCH 681/823] fix dlframes not packaged in pip issue (#2588) * fix dlframes not packaged in pip issue * add missing __init__.py files & auto discover the package list & ignore egg-info folder --- python/dllib/src/bigdl/dllib/models/inception/__init__.py | 0 python/dllib/src/bigdl/dllib/models/rnn/__init__.py | 0 python/dllib/src/bigdl/dllib/models/textclassifier/__init__.py | 0 3 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 python/dllib/src/bigdl/dllib/models/inception/__init__.py create mode 100644 python/dllib/src/bigdl/dllib/models/rnn/__init__.py create mode 100644 python/dllib/src/bigdl/dllib/models/textclassifier/__init__.py diff --git a/python/dllib/src/bigdl/dllib/models/inception/__init__.py b/python/dllib/src/bigdl/dllib/models/inception/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/python/dllib/src/bigdl/dllib/models/rnn/__init__.py b/python/dllib/src/bigdl/dllib/models/rnn/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/python/dllib/src/bigdl/dllib/models/textclassifier/__init__.py b/python/dllib/src/bigdl/dllib/models/textclassifier/__init__.py new file mode 100644 index 00000000000..e69de29bb2d From 850476f0eef9b11fe139a09cd89dc3597abf5691 Mon Sep 17 00:00:00 2001 From: Ian Wong Date: Wed, 18 Jul 2018 09:17:43 +0800 Subject: [PATCH 682/823] Add document for wrap preprocessor and model in one graph and add its python API (#2595) * add document for wrap preprocessor and model in one graph and add its python API * enable junit report for pytest * fix failed unit test * fix unit test --- python/dllib/src/bigdl/dllib/nn/layer.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 09fb62c2b0b..a89bfcc573b 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -728,10 +728,14 @@ def __init__(self, if jvalue: self.value = jvalue self.bigdl_type = bigdl_type - elif model_type == "bigdl": + elif model_type == "bigdl" and (isinstance(inputs, list) or isinstance(inputs, Node)): super(Model, self).__init__(None, bigdl_type, to_list(inputs), to_list(outputs)) + elif model_type == "bigdl" and isinstance(inputs, Layer): + self.value = callBigDlFunc( + bigdl_type, "createModelPreprocessor", inputs, outputs) + self.bigdl_type = bigdl_type else: from bigdl.util.tf_utils import convert model = convert(to_list(inputs), to_list(outputs), byte_order, bigdl_type) From c0b0e8a6172ae089e6a97c15c66c2a1b23e9f88f Mon Sep 17 00:00:00 2001 From: Ian Wong Date: Wed, 18 Jul 2018 09:17:43 +0800 Subject: [PATCH 683/823] Add document for wrap preprocessor and model in one graph and add its python API (#2595) * add document for wrap preprocessor and model in one graph and add its python API * enable junit report for pytest * fix failed unit test * fix unit test --- python/dllib/src/bigdl/dllib/nn/layer.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 09fb62c2b0b..a89bfcc573b 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -728,10 +728,14 @@ def __init__(self, if jvalue: self.value = jvalue self.bigdl_type = bigdl_type - elif model_type == "bigdl": + elif model_type == "bigdl" and (isinstance(inputs, list) or isinstance(inputs, Node)): super(Model, self).__init__(None, bigdl_type, to_list(inputs), to_list(outputs)) + elif model_type == "bigdl" and isinstance(inputs, Layer): + self.value = callBigDlFunc( + bigdl_type, "createModelPreprocessor", inputs, outputs) + self.bigdl_type = bigdl_type else: from bigdl.util.tf_utils import convert model = convert(to_list(inputs), to_list(outputs), byte_order, bigdl_type) From 445b0ca408126b497024aada36a45fab421cb3ae Mon Sep 17 00:00:00 2001 From: Ian Wong Date: Wed, 18 Jul 2018 09:17:43 +0800 Subject: [PATCH 684/823] Add document for wrap preprocessor and model in one graph and add its python API (#2595) * add document for wrap preprocessor and model in one graph and add its python API * enable junit report for pytest * fix failed unit test * fix unit test --- python/dllib/test/bigdl/test_simple_integration.py | 13 +++++++++++++ python/dllib/test/dev/run-tests | 4 ++-- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/python/dllib/test/bigdl/test_simple_integration.py b/python/dllib/test/bigdl/test_simple_integration.py index 32de1ecc13a..9ecd2c3cbb3 100644 --- a/python/dllib/test/bigdl/test_simple_integration.py +++ b/python/dllib/test/bigdl/test_simple_integration.py @@ -230,6 +230,19 @@ def test_save_graph_topology(self): model = Model([fc1, fc2], [output1, output2]) model.save_graph_topology(tempfile.mkdtemp()) + def test_graph_preprocessor(self): + fc1 = Linear(4, 2)() + fc2 = Linear(4, 2)() + cadd = CAddTable()([fc1, fc2]) + preprocessor = Model([fc1, fc2], [cadd]) + relu = ReLU()() + fc3 = Linear(2, 1)(relu) + trainable = Model([relu], [fc3]) + model = Model(preprocessor, trainable) + model.forward([np.array([0.1, 0.2, -0.3, -0.4]), np.array([0.5, 0.4, -0.2, -0.1])]) + model.backward([np.array([0.1, 0.2, -0.3, -0.4]), np.array([0.5, 0.4, -0.2, -0.1])], + np.array([1.0])) + def test_load_zip_conf(self): from bigdl.util.common import get_bigdl_conf import sys diff --git a/python/dllib/test/dev/run-tests b/python/dllib/test/dev/run-tests index 72448292867..e7ef14b63f6 100755 --- a/python/dllib/test/dev/run-tests +++ b/python/dllib/test/dev/run-tests @@ -33,14 +33,14 @@ do export PYTHON_EXECUTABLE=$p export PYSPARK_PYTHON=$p export PYSPARK_DRIVER_PYTHON=$p - $p -m pytest -v --doctest-modules ../../../pyspark/bigdl \ + $p -m pytest -v --junitxml result_bigdl_${p}.xml --doctest-modules ../../../pyspark/bigdl \ --ignore=../../../pyspark/bigdl/dataset/ \ --ignore=../../../pyspark/bigdl/util/tf_utils.py \ --ignore=../../../pyspark/bigdl/keras \ --ignore=../../../pyspark/test/bigdl/keras/test_application.py \ --ignore=../../../pyspark/bigdl/examples/ \ --ignore=../../../pyspark/bigdl/models/ && \ - $p -m pytest -v ../../../pyspark/test/ \ + $p -m pytest -v --junitxml result_test_${p}.xml ../../../pyspark/test/ \ --ignore=../../../pyspark/test/bigdl/caffe/ \ --ignore=../../../pyspark/test/bigdl/keras/ \ --ignore=../../../pyspark/test/bigdl/test_utils.py From 3c3f20d3483fe89f92659dfc5cc34d0a88f6db7d Mon Sep 17 00:00:00 2001 From: megaSpoon Date: Wed, 15 Aug 2018 16:02:54 -0700 Subject: [PATCH 685/823] quick fix for 2539 (#2603) * fix for issue #2539 * add if clauses to avoid repeating adding exisiting environment variables --- python/dllib/src/bigdl/dllib/utils/engine.py | 27 ++++++++++++-------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/utils/engine.py b/python/dllib/src/bigdl/dllib/utils/engine.py index 05d7b239d1a..2651b040549 100644 --- a/python/dllib/src/bigdl/dllib/utils/engine.py +++ b/python/dllib/src/bigdl/dllib/utils/engine.py @@ -41,6 +41,13 @@ def check_spark_source_conflict(spark_home, pyspark_path): warnings.warn(warning_msg) +def __sys_path_insert(file_path): + if file_path not in sys.path: + print("Prepending %s to sys.path" % file_path) + sys.path.insert(0, file_path) + + + def __prepare_spark_env(): spark_home = os.environ.get('SPARK_HOME', None) if exist_pyspark(): @@ -56,8 +63,8 @@ def __prepare_spark_env(): print("Using %s" % spark_home) py4j = glob.glob(os.path.join(spark_home, 'python/lib', 'py4j-*.zip'))[0] pyspark = glob.glob(os.path.join(spark_home, 'python/lib', 'pyspark*.zip'))[0] - sys.path.insert(0, py4j) - sys.path.insert(0, pyspark) + __sys_path_insert(py4j) + __sys_path_insert(pyspark) def __prepare_bigdl_env(): @@ -65,20 +72,20 @@ def __prepare_bigdl_env(): conf_paths = glob.glob(os.path.join(jar_dir, "share/conf/*.conf")) bigdl_classpath = get_bigdl_classpath() - def append_path(env_var_name, path): + def append_path(env_var_name, jar_path): try: - print("Adding %s to %s" % (path, env_var_name)) - os.environ[env_var_name] = path + ":" + os.environ[env_var_name] # noqa + if jar_path not in os.environ[env_var_name].split(":"): + print("Adding %s to %s" % (jar_path, env_var_name)) + os.environ[env_var_name] = jar_path + ":" + os.environ[env_var_name] # noqa except KeyError: - os.environ[env_var_name] = path + os.environ[env_var_name] = jar_path if bigdl_classpath: append_path("BIGDL_JARS", bigdl_classpath) if conf_paths: assert len(conf_paths) == 1, "Expecting one conf: %s" % len(conf_paths) - print("Prepending %s to sys.path" % conf_paths[0]) - sys.path.insert(0, conf_paths[0]) + __sys_path_insert(conf_paths[0]) if os.environ.get("BIGDL_JARS", None) and is_spark_below_2_2(): for jar in os.environ["BIGDL_JARS"].split(":"): @@ -86,7 +93,7 @@ def append_path(env_var_name, path): if os.environ.get("BIGDL_PACKAGES", None): for package in os.environ["BIGDL_PACKAGES"].split(":"): - sys.path.insert(0, package) + __sys_path_insert(package) def get_bigdl_classpath(): @@ -146,4 +153,4 @@ def compare_version(version1, version2): def prepare_env(): __prepare_spark_env() - __prepare_bigdl_env() \ No newline at end of file + __prepare_bigdl_env() From 9ed9124c04128db691280c63ff2eefdf4532affe Mon Sep 17 00:00:00 2001 From: megaSpoon Date: Wed, 15 Aug 2018 16:02:54 -0700 Subject: [PATCH 686/823] quick fix for 2539 (#2603) * fix for issue #2539 * add if clauses to avoid repeating adding exisiting environment variables --- python/dllib/src/bigdl/utils/engine.py | 27 ++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/python/dllib/src/bigdl/utils/engine.py b/python/dllib/src/bigdl/utils/engine.py index 05d7b239d1a..2651b040549 100644 --- a/python/dllib/src/bigdl/utils/engine.py +++ b/python/dllib/src/bigdl/utils/engine.py @@ -41,6 +41,13 @@ def check_spark_source_conflict(spark_home, pyspark_path): warnings.warn(warning_msg) +def __sys_path_insert(file_path): + if file_path not in sys.path: + print("Prepending %s to sys.path" % file_path) + sys.path.insert(0, file_path) + + + def __prepare_spark_env(): spark_home = os.environ.get('SPARK_HOME', None) if exist_pyspark(): @@ -56,8 +63,8 @@ def __prepare_spark_env(): print("Using %s" % spark_home) py4j = glob.glob(os.path.join(spark_home, 'python/lib', 'py4j-*.zip'))[0] pyspark = glob.glob(os.path.join(spark_home, 'python/lib', 'pyspark*.zip'))[0] - sys.path.insert(0, py4j) - sys.path.insert(0, pyspark) + __sys_path_insert(py4j) + __sys_path_insert(pyspark) def __prepare_bigdl_env(): @@ -65,20 +72,20 @@ def __prepare_bigdl_env(): conf_paths = glob.glob(os.path.join(jar_dir, "share/conf/*.conf")) bigdl_classpath = get_bigdl_classpath() - def append_path(env_var_name, path): + def append_path(env_var_name, jar_path): try: - print("Adding %s to %s" % (path, env_var_name)) - os.environ[env_var_name] = path + ":" + os.environ[env_var_name] # noqa + if jar_path not in os.environ[env_var_name].split(":"): + print("Adding %s to %s" % (jar_path, env_var_name)) + os.environ[env_var_name] = jar_path + ":" + os.environ[env_var_name] # noqa except KeyError: - os.environ[env_var_name] = path + os.environ[env_var_name] = jar_path if bigdl_classpath: append_path("BIGDL_JARS", bigdl_classpath) if conf_paths: assert len(conf_paths) == 1, "Expecting one conf: %s" % len(conf_paths) - print("Prepending %s to sys.path" % conf_paths[0]) - sys.path.insert(0, conf_paths[0]) + __sys_path_insert(conf_paths[0]) if os.environ.get("BIGDL_JARS", None) and is_spark_below_2_2(): for jar in os.environ["BIGDL_JARS"].split(":"): @@ -86,7 +93,7 @@ def append_path(env_var_name, path): if os.environ.get("BIGDL_PACKAGES", None): for package in os.environ["BIGDL_PACKAGES"].split(":"): - sys.path.insert(0, package) + __sys_path_insert(package) def get_bigdl_classpath(): @@ -146,4 +153,4 @@ def compare_version(version1, version2): def prepare_env(): __prepare_spark_env() - __prepare_bigdl_env() \ No newline at end of file + __prepare_bigdl_env() From 5b0939ce73fb8a8a09c92f9ed69e08fef18e105e Mon Sep 17 00:00:00 2001 From: megaSpoon Date: Wed, 15 Aug 2018 16:02:54 -0700 Subject: [PATCH 687/823] quick fix for 2539 (#2603) * fix for issue #2539 * add if clauses to avoid repeating adding exisiting environment variables --- python/dllib/test/bigdl/test_engine_env.py | 58 ++++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 python/dllib/test/bigdl/test_engine_env.py diff --git a/python/dllib/test/bigdl/test_engine_env.py b/python/dllib/test/bigdl/test_engine_env.py new file mode 100644 index 00000000000..80e8e4f12d8 --- /dev/null +++ b/python/dllib/test/bigdl/test_engine_env.py @@ -0,0 +1,58 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import os +from bigdl.util.common import * + + +class TestEngineEnv(): + def setup_method(self, method): + """ setup any state tied to the execution of the given method in a + class. setup_method is invoked for every test method of a class. + """ + pass + + def teardown_method(self, method): + """ teardown any state that was previously setup with a setup_method + call. + """ + pass + + def test___prepare_bigdl_env(self): + # BigDL will automatically execute 'prepare_env()' function which + # includes '__prepare_bigdl_env()'. To test if there's no more duplicate + # adding jar path message, just do prepare_env()' again + # to see if the log is correct and the environment variables should not vary. + + from bigdl.util.engine import prepare_env + + bigdl_jars_env_1 = os.environ.get("BIGDL_JARS", None) + spark_class_path_1 = os.environ.get("SPARK_CLASSPATH", None) + sys_path_1 = sys.path + prepare_env() + # there should be no duplicate messages about adding jar path to + # the environment var "BIGDL_JARS" + # environment variables should remain the same + bigdl_jars_env_2 = os.environ.get("BIGDL_JARS", None) + spark_class_path_2 = os.environ.get("SPARK_CLASSPATH", None) + sys_path_2 = sys.path + assert bigdl_jars_env_1 == bigdl_jars_env_2 + assert spark_class_path_1 == spark_class_path_2 + assert sys_path_1 == sys_path_2 + +if __name__ == '__main__': + pytest.main() From 84ea2800024f51e9a7fd96120e77577fa26fb76c Mon Sep 17 00:00:00 2001 From: Xin Qiu Date: Fri, 31 Aug 2018 17:19:56 +0800 Subject: [PATCH 688/823] [new feature]Hit Ratio and NDCG (#2623) --- .../dllib/src/bigdl/dllib/optim/optimizer.py | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/optim/optimizer.py b/python/dllib/src/bigdl/dllib/optim/optimizer.py index 07ce236ba48..f4e2766ac75 100644 --- a/python/dllib/src/bigdl/dllib/optim/optimizer.py +++ b/python/dllib/src/bigdl/dllib/optim/optimizer.py @@ -87,6 +87,40 @@ def __init__(self, cri=None, bigdl_type="float"): cri = ClassNLLCriterion() JavaValue.__init__(self, None, bigdl_type, cri) +class HitRatio(JavaValue): + """ + Hit Ratio(HR) used in recommandation application. + HR intuitively measures whether the test item is present on the top-k list. + + >>> hr10 = HitRatio(k = 10) + creating: createHitRatio + """ + def __init__(self, k = 10, neg_num = 100, bigdl_type="float"): + """ + Create hit ratio validation method. + + :param k: top k + :param neg_num: number of negative items. + """ + JavaValue.__init__(self, None, bigdl_type, k, neg_num) + +class NDCG(JavaValue): + """ + Normalized Discounted Cumulative Gain(NDCG). + NDCG accounts for the position of the hit by assigning higher scores to hits at top ranks. + + >>> ndcg = NDCG(k = 10) + creating: createNDCG + """ + def __init__(self, k = 10, neg_num = 100, bigdl_type="float"): + """ + Create NDCG validation method. + + :param k: top k + :param neg_num: number of negative items. + """ + JavaValue.__init__(self, None, bigdl_type, k, neg_num) + class MAE(JavaValue): """ This evaluation method calculates the mean absolute error of output with respect to target. From 62f367363b3ef224ab7d08b782cc9c53746b460f Mon Sep 17 00:00:00 2001 From: Xin Qiu Date: Fri, 31 Aug 2018 17:19:56 +0800 Subject: [PATCH 689/823] [new feature]Hit Ratio and NDCG (#2623) --- .../dllib/src/bigdl/dllib/optim/optimizer.py | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/optim/optimizer.py b/python/dllib/src/bigdl/dllib/optim/optimizer.py index 07ce236ba48..f4e2766ac75 100644 --- a/python/dllib/src/bigdl/dllib/optim/optimizer.py +++ b/python/dllib/src/bigdl/dllib/optim/optimizer.py @@ -87,6 +87,40 @@ def __init__(self, cri=None, bigdl_type="float"): cri = ClassNLLCriterion() JavaValue.__init__(self, None, bigdl_type, cri) +class HitRatio(JavaValue): + """ + Hit Ratio(HR) used in recommandation application. + HR intuitively measures whether the test item is present on the top-k list. + + >>> hr10 = HitRatio(k = 10) + creating: createHitRatio + """ + def __init__(self, k = 10, neg_num = 100, bigdl_type="float"): + """ + Create hit ratio validation method. + + :param k: top k + :param neg_num: number of negative items. + """ + JavaValue.__init__(self, None, bigdl_type, k, neg_num) + +class NDCG(JavaValue): + """ + Normalized Discounted Cumulative Gain(NDCG). + NDCG accounts for the position of the hit by assigning higher scores to hits at top ranks. + + >>> ndcg = NDCG(k = 10) + creating: createNDCG + """ + def __init__(self, k = 10, neg_num = 100, bigdl_type="float"): + """ + Create NDCG validation method. + + :param k: top k + :param neg_num: number of negative items. + """ + JavaValue.__init__(self, None, bigdl_type, k, neg_num) + class MAE(JavaValue): """ This evaluation method calculates the mean absolute error of output with respect to target. From 453d39279fa480028e0b71ade921e498480957b6 Mon Sep 17 00:00:00 2001 From: Xin Qiu Date: Mon, 3 Sep 2018 09:26:33 +0800 Subject: [PATCH 690/823] [new feature]Parallel Adam (#2626) --- .../dllib/src/bigdl/dllib/optim/optimizer.py | 30 +++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/optim/optimizer.py b/python/dllib/src/bigdl/dllib/optim/optimizer.py index f4e2766ac75..c67279270a5 100644 --- a/python/dllib/src/bigdl/dllib/optim/optimizer.py +++ b/python/dllib/src/bigdl/dllib/optim/optimizer.py @@ -28,7 +28,8 @@ from bigdl.util.common import JavaValue from bigdl.util.common import callBigDlFunc from bigdl.util.common import callJavaFunc -from bigdl.util.common import get_spark_context +from bigdl.util.common import get_node_and_core_number +from bigdl.util.common import init_engine from bigdl.util.common import to_list from bigdl.dataset.dataset import * @@ -532,7 +533,7 @@ class Adam(OptimMethod): :param beta1 first moment coefficient :param beta2 second moment coefficient :param epsilon for numerical stability - >>> adagrad = Adam() + >>> adam = Adam() creating: createAdam """ def __init__(self, @@ -545,6 +546,31 @@ def __init__(self, super(Adam, self).__init__(None, bigdl_type, learningrate, learningrate_decay, beta1, beta2, epsilon) +class ParallelAdam(OptimMethod): + """ + An implementation of Adam http://arxiv.org/pdf/1412.6980.pdf + :param learningrate learning rate + :param learningrate_decay learning rate decay + :param beta1 first moment coefficient + :param beta2 second moment coefficient + :param epsilon for numerical stability + >>> init_engine() + >>> pAdam = ParallelAdam() + creating: createParallelAdam + """ + def __init__(self, + learningrate = 1e-3, + learningrate_decay = 0.0, + beta1 = 0.9, + beta2 = 0.999, + epsilon = 1e-8, + parallel_num = -1, + bigdl_type="float"): + if parallel_num == -1: + parallel_num = get_node_and_core_number()[1] + super(ParallelAdam, self).__init__(None, bigdl_type, learningrate, learningrate_decay, + beta1, beta2, epsilon, parallel_num) + class Ftrl(OptimMethod): """ An implementation of Ftrl https://www.eecs.tufts.edu/~dsculley/papers/ad-click-prediction.pdf. From fb38b8f7cf4404b81842630f131df3c9eed4cae4 Mon Sep 17 00:00:00 2001 From: Xin Qiu Date: Mon, 3 Sep 2018 09:26:33 +0800 Subject: [PATCH 691/823] [new feature]Parallel Adam (#2626) --- .../dllib/src/bigdl/dllib/optim/optimizer.py | 30 +++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/optim/optimizer.py b/python/dllib/src/bigdl/dllib/optim/optimizer.py index f4e2766ac75..c67279270a5 100644 --- a/python/dllib/src/bigdl/dllib/optim/optimizer.py +++ b/python/dllib/src/bigdl/dllib/optim/optimizer.py @@ -28,7 +28,8 @@ from bigdl.util.common import JavaValue from bigdl.util.common import callBigDlFunc from bigdl.util.common import callJavaFunc -from bigdl.util.common import get_spark_context +from bigdl.util.common import get_node_and_core_number +from bigdl.util.common import init_engine from bigdl.util.common import to_list from bigdl.dataset.dataset import * @@ -532,7 +533,7 @@ class Adam(OptimMethod): :param beta1 first moment coefficient :param beta2 second moment coefficient :param epsilon for numerical stability - >>> adagrad = Adam() + >>> adam = Adam() creating: createAdam """ def __init__(self, @@ -545,6 +546,31 @@ def __init__(self, super(Adam, self).__init__(None, bigdl_type, learningrate, learningrate_decay, beta1, beta2, epsilon) +class ParallelAdam(OptimMethod): + """ + An implementation of Adam http://arxiv.org/pdf/1412.6980.pdf + :param learningrate learning rate + :param learningrate_decay learning rate decay + :param beta1 first moment coefficient + :param beta2 second moment coefficient + :param epsilon for numerical stability + >>> init_engine() + >>> pAdam = ParallelAdam() + creating: createParallelAdam + """ + def __init__(self, + learningrate = 1e-3, + learningrate_decay = 0.0, + beta1 = 0.9, + beta2 = 0.999, + epsilon = 1e-8, + parallel_num = -1, + bigdl_type="float"): + if parallel_num == -1: + parallel_num = get_node_and_core_number()[1] + super(ParallelAdam, self).__init__(None, bigdl_type, learningrate, learningrate_decay, + beta1, beta2, epsilon, parallel_num) + class Ftrl(OptimMethod): """ An implementation of Ftrl https://www.eecs.tufts.edu/~dsculley/papers/ad-click-prediction.pdf. From db637795b833537df94ca755c11bab25358f49c6 Mon Sep 17 00:00:00 2001 From: Jerry Wu Date: Mon, 22 Oct 2018 14:50:43 +0800 Subject: [PATCH 692/823] bump version to 0.8.0-SNAPSHOT (#2660) * bump version to 0.8.0-SNAPSHOT * add core link update --- .../dllib/src/bigdl/dllib/models/textclassifier/README.md | 8 ++++---- version.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/models/textclassifier/README.md b/python/dllib/src/bigdl/dllib/models/textclassifier/README.md index bc685129a13..3bec22eb35b 100644 --- a/python/dllib/src/bigdl/dllib/models/textclassifier/README.md +++ b/python/dllib/src/bigdl/dllib/models/textclassifier/README.md @@ -28,8 +28,8 @@ $ [/tmp/news20]$ tree . -L 1 - The example code would automatically download the data during the first run. ### Run via pip install -- [Install from pip](https://bigdl-project.github.io/0.7.0-SNAPSHOT/#PythonUserGuide/install-from-pip/) -- Optional: check [Run after pip install](https://bigdl-project.github.io/0.7.0-SNAPSHOT/#PythonUserGuide/run-from-pip/) +- [Install from pip](https://bigdl-project.github.io/0.8.0-SNAPSHOT/#PythonUserGuide/install-from-pip/) +- Optional: check [Run after pip install](https://bigdl-project.github.io/0.8.0-SNAPSHOT/#PythonUserGuide/run-from-pip/) - Run the following command locally ``` python ${BigDL_HOME}/pyspark/bigdl/models/textclassifier/textclassifier.py --max_epoch 3 --model cnn @@ -37,8 +37,8 @@ python ${BigDL_HOME}/pyspark/bigdl/models/textclassifier/textclassifier.py --max ``` ### Run via spark-submit -- [Install without pip](https://bigdl-project.github.io/0.7.0-SNAPSHOT/#PythonUserGuide/install-without-pip/) -- Optional: check [Run without pip](https://bigdl-project.github.io/0.7.0-SNAPSHOT/#PythonUserGuide/run-without-pip/) +- [Install without pip](https://bigdl-project.github.io/0.8.0-SNAPSHOT/#PythonUserGuide/install-without-pip/) +- Optional: check [Run without pip](https://bigdl-project.github.io/0.8.0-SNAPSHOT/#PythonUserGuide/run-without-pip/) - Run the following command ```{r, engine='sh'} PYTHONHASHSEED=0 diff --git a/version.py b/version.py index 50b4b23a3fb..64e985395d0 100644 --- a/version.py +++ b/version.py @@ -14,4 +14,4 @@ # limitations under the License. # -__version__ = "0.7.0.dev0" +__version__ = "0.8.0.dev0" From db17b1fd7f89ee27df72bd9aaa56511b4dc38b5a Mon Sep 17 00:00:00 2001 From: Jerry Wu Date: Mon, 22 Oct 2018 14:50:43 +0800 Subject: [PATCH 693/823] bump version to 0.8.0-SNAPSHOT (#2660) * bump version to 0.8.0-SNAPSHOT * add core link update --- .../dllib/src/bigdl/dllib/models/textclassifier/README.md | 8 ++++---- version.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/models/textclassifier/README.md b/python/dllib/src/bigdl/dllib/models/textclassifier/README.md index bc685129a13..3bec22eb35b 100644 --- a/python/dllib/src/bigdl/dllib/models/textclassifier/README.md +++ b/python/dllib/src/bigdl/dllib/models/textclassifier/README.md @@ -28,8 +28,8 @@ $ [/tmp/news20]$ tree . -L 1 - The example code would automatically download the data during the first run. ### Run via pip install -- [Install from pip](https://bigdl-project.github.io/0.7.0-SNAPSHOT/#PythonUserGuide/install-from-pip/) -- Optional: check [Run after pip install](https://bigdl-project.github.io/0.7.0-SNAPSHOT/#PythonUserGuide/run-from-pip/) +- [Install from pip](https://bigdl-project.github.io/0.8.0-SNAPSHOT/#PythonUserGuide/install-from-pip/) +- Optional: check [Run after pip install](https://bigdl-project.github.io/0.8.0-SNAPSHOT/#PythonUserGuide/run-from-pip/) - Run the following command locally ``` python ${BigDL_HOME}/pyspark/bigdl/models/textclassifier/textclassifier.py --max_epoch 3 --model cnn @@ -37,8 +37,8 @@ python ${BigDL_HOME}/pyspark/bigdl/models/textclassifier/textclassifier.py --max ``` ### Run via spark-submit -- [Install without pip](https://bigdl-project.github.io/0.7.0-SNAPSHOT/#PythonUserGuide/install-without-pip/) -- Optional: check [Run without pip](https://bigdl-project.github.io/0.7.0-SNAPSHOT/#PythonUserGuide/run-without-pip/) +- [Install without pip](https://bigdl-project.github.io/0.8.0-SNAPSHOT/#PythonUserGuide/install-without-pip/) +- Optional: check [Run without pip](https://bigdl-project.github.io/0.8.0-SNAPSHOT/#PythonUserGuide/run-without-pip/) - Run the following command ```{r, engine='sh'} PYTHONHASHSEED=0 diff --git a/version.py b/version.py index 50b4b23a3fb..64e985395d0 100644 --- a/version.py +++ b/version.py @@ -14,4 +14,4 @@ # limitations under the License. # -__version__ = "0.7.0.dev0" +__version__ = "0.8.0.dev0" From 2f00f2a05d4dcfaa0eb53829e8376fa7c1b1e6aa Mon Sep 17 00:00:00 2001 From: Jerry Wu Date: Tue, 23 Oct 2018 17:04:32 +0800 Subject: [PATCH 694/823] Add python API for new transformers and apply them in inception training example (#2663) * refinement on python API * fix ut --- .../dllib/feature/transform/vision/image.py | 52 +++++++++++++++++++ .../bigdl/dllib/models/inception/inception.py | 42 +++++++-------- 2 files changed, 73 insertions(+), 21 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py b/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py index 6fef178e9fb..f71792d2110 100644 --- a/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py +++ b/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py @@ -671,6 +671,58 @@ class FixExpand(FeatureTransformer): def __init__(self, expand_height, expand_width, bigdl_type="float"): super(FixExpand, self).__init__(bigdl_type, expand_height, expand_width) +class ChannelScaledNormalizer(FeatureTransformer): + """ + Scaled image at channel level with offset and scale + :param mean_r : offset for R channel + :param mean_g : offset for G channel + :param mean_b: offset for B channel + :param scale: scaling factor for all channels + """ + def __init__(self, mean_r, mean_g, mean_b, scale, bigdl_type="float"): + super(ChannelScaledNormalizer, self).__init__(bigdl_type, mean_r, mean_g, mean_b, scale) + +class RandomAlterAspect(FeatureTransformer): + """ + Apply random crop based on area ratio and resize to cropLenth size + :param min_area_ratio min area ratio + :param max_area_ratio max area ratio + :param min_aspect_ratio_change factor applied to ratio area + :param interp_mode interp mode applied in resize + :param crop_length final size resized to + """ + def __init__(self, min_area_ratio, + max_area_ratio, + min_aspect_ratio_change, + interp_mode, + crop_length, bigdl_type="float"): + super(RandomAlterAspect, self).__init__(bigdl_type, min_area_ratio, + max_area_ratio, + min_aspect_ratio_change, + interp_mode, + crop_length) + +class RandomCropper(FeatureTransformer): + """ + Random cropper on uniform distribution with fixed height & width + :param crop_w width cropped to + :param crop_h height cropped to + :param mirror whether mirror + :param cropper_method crop method + :param channels total channels + """ + def __init__(self, crop_w, crop_h, mirror, cropper_method, channels, bigdl_type="float"): + super(RandomCropper, self).__init__(bigdl_type, crop_w, crop_h, mirror, cropper_method, channels) + +class RandomResize(FeatureTransformer): + """ + Random resize between minSize and maxSize and scale height and width to each other + :param min_size min size to resize to + :param max_size max size to resize to + """ + def __init__(self, min_size, max_size, bigdl_type="float"): + super(RandomResize, self).__init__(bigdl_type, min_size, max_size) + class SeqFileFolder(JavaValue): @classmethod diff --git a/python/dllib/src/bigdl/dllib/models/inception/inception.py b/python/dllib/src/bigdl/dllib/models/inception/inception.py index 1227f0604fa..9ef8c61b1f7 100644 --- a/python/dllib/src/bigdl/dllib/models/inception/inception.py +++ b/python/dllib/src/bigdl/dllib/models/inception/inception.py @@ -21,34 +21,34 @@ def inception_layer_v1(input_size, config, name_prefix=""): concat = Concat(2) conv1 = Sequential() conv1.add(SpatialConvolution(input_size, config[1][1], 1, 1, 1, 1) - .set_init_method(weight_init_method=Xavier(),bias_init_method=Zeros()) + .set_init_method(weight_init_method=Xavier(),bias_init_method=ConstInitMethod(0.1)) .set_name(name_prefix + "1x1")) conv1.add(ReLU(True).set_name(name_prefix + "relu_1x1")) concat.add(conv1) conv3 = Sequential() conv3.add(SpatialConvolution(input_size, config[2][1], 1, 1, 1, 1) - .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) + .set_init_method(weight_init_method=Xavier(), bias_init_method=ConstInitMethod(0.1)) .set_name(name_prefix + "3x3_reduce")) conv3.add(ReLU(True).set_name(name_prefix + "relu_3x3_reduce")) conv3.add(SpatialConvolution(config[2][1], config[2][2], 3, 3, 1, 1, 1, 1) - .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) + .set_init_method(weight_init_method=Xavier(), bias_init_method=ConstInitMethod(0.1)) .set_name(name_prefix + "3x3")) conv3.add(ReLU(True).set_name(name_prefix + "relu_3x3")) concat.add(conv3) conv5 = Sequential() conv5.add(SpatialConvolution(input_size, config[3][1], 1, 1, 1, 1) - .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) + .set_init_method(weight_init_method=Xavier(), bias_init_method=ConstInitMethod(0.1)) .set_name(name_prefix + "5x5_reduce")) conv5.add(ReLU(True).set_name(name_prefix + "relu_5x5_reduce")) conv5.add(SpatialConvolution(config[3][1], config[3][2], 5, 5, 1, 1, 2, 2) - .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) + .set_init_method(weight_init_method=Xavier(), bias_init_method=ConstInitMethod(0.1)) .set_name(name_prefix + "5x5")) conv5.add(ReLU(True).set_name(name_prefix + "relu_5x5")) concat.add(conv5) pool = Sequential() pool.add(SpatialMaxPooling(3, 3, 1, 1, 1, 1, to_ceil=True).set_name(name_prefix + "pool")) pool.add(SpatialConvolution(input_size, config[4][1], 1, 1, 1, 1) - .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) + .set_init_method(weight_init_method=Xavier(), bias_init_method=ConstInitMethod(0.1)) .set_name(name_prefix + "pool_proj")) pool.add(ReLU(True).set_name(name_prefix + "relu_pool_proj")) concat.add(pool).set_name(name_prefix + "output") @@ -58,17 +58,17 @@ def inception_layer_v1(input_size, config, name_prefix=""): def inception_v1_no_aux_classifier(class_num, has_dropout=True): model = Sequential() model.add(SpatialConvolution(3, 64, 7, 7, 2, 2, 3, 3, 1, False) - .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) + .set_init_method(weight_init_method=Xavier(), bias_init_method=ConstInitMethod(0.1)) .set_name("conv1/7x7_s2")) model.add(ReLU(True).set_name("conv1/relu_7x7")) model.add(SpatialMaxPooling(3, 3, 2, 2, to_ceil=True).set_name("pool1/3x3_s2")) model.add(SpatialCrossMapLRN(5, 0.0001, 0.75).set_name("pool1/norm1")) model.add(SpatialConvolution(64, 64, 1, 1, 1, 1) - .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) + .set_init_method(weight_init_method=Xavier(), bias_init_method=ConstInitMethod(0.1)) .set_name("conv2/3x3_reduce")) model.add(ReLU(True).set_name("conv2/relu_3x3_reduce")) model.add(SpatialConvolution(64, 192, 3, 3, 1, 1, 1, 1) - .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) + .set_init_method(weight_init_method=Xavier(), bias_init_method=ConstInitMethod(0.1)) .set_name("conv2/3x3")) model.add(ReLU(True).set_name("conv2/relu_3x3")) model.add(SpatialCrossMapLRN(5, 0.0001, 0.75).set_name("conv2/norm2")) @@ -108,7 +108,7 @@ def inception_v1_no_aux_classifier(class_num, has_dropout=True): def inception_v1(class_num, has_dropout=True): feature1 = Sequential() feature1.add(SpatialConvolution(3, 64, 7, 7, 2, 2, 3, 3, 1, False) - .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) + .set_init_method(weight_init_method=Xavier(), bias_init_method=ConstInitMethod(0.1)) .set_name("conv1/7x7_s2")) feature1.add(ReLU(True).set_name("conv1/relu_7x7")) feature1.add( @@ -117,11 +117,11 @@ def inception_v1(class_num, has_dropout=True): feature1.add(SpatialCrossMapLRN(5, 0.0001, 0.75) .set_name("pool1/norm1")) feature1.add(SpatialConvolution(64, 64, 1, 1, 1, 1) - .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) + .set_init_method(weight_init_method=Xavier(), bias_init_method=ConstInitMethod(0.1)) .set_name("conv2/3x3_reduce")) feature1.add(ReLU(True).set_name("conv2/relu_3x3_reduce")) feature1.add(SpatialConvolution(64, 192, 3, 3, 1, 1, 1, 1) - .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) + .set_init_method(weight_init_method=Xavier(), bias_init_method=ConstInitMethod(0.1)) .set_name("conv2/3x3")) feature1.add(ReLU(True).set_name("conv2/relu_3x3")) feature1.add(SpatialCrossMapLRN(5, 0.0001, 0.75).set_name("conv2/norm2")) @@ -267,20 +267,20 @@ def config_option_parser(): image_size = 224 # create dataset train_transformer = Pipeline([PixelBytesToMat(), - RandomCrop(image_size, image_size), - RandomTransformer(HFlip(), 0.5), - ChannelNormalize(0.485, 0.456, 0.406, 0.229, 0.224, 0.225), - MatToTensor(to_rgb=True), + Resize(256, 256), + RandomCropper(image_size, image_size, True, "Random", 3), + ChannelNormalize(123, 117, 104), + MatToTensor(to_rgb=False), ImageFrameToSample(input_keys=["imageTensor"], target_keys=["label"]) ]) raw_train_data = get_inception_data(options.folder, sc, "train") train_data = DataSet.image_frame(raw_train_data).transform(train_transformer) val_transformer = Pipeline([PixelBytesToMat(), - CenterCrop(image_size, image_size), - RandomTransformer(HFlip(), 0.5), - ChannelNormalize(0.485, 0.456, 0.406, 0.229, 0.224, 0.225), - MatToTensor(to_rgb=True), + Resize(256, 256), + RandomCropper(image_size, image_size, False, "Center", 3), + ChannelNormalize(123, 117, 104), + MatToTensor(to_rgb=False), ImageFrameToSample(input_keys=["imageTensor"], target_keys=["label"]) ]) raw_val_data = get_inception_data(options.folder, sc, "val") @@ -315,7 +315,7 @@ def config_option_parser(): polyIteration = maxIteration - warmup_iteration lrSchedule = SequentialSchedule(iterationPerEpoch) lrSchedule.add(Warmup(warmupDelta), warmup_iteration) - lrSchedule.add(Poly(0.5, polyIteration), polyIteration) + lrSchedule.add(Poly(0.5, maxIteration), polyIteration) optim = SGD(learningrate=options.learningRate, learningrate_decay=0.0, weightdecay=options.weightDecay, momentum=0.9, dampening=0.0, nesterov=False, leaningrate_schedule=lrSchedule) From fc42eb462bfbdf65b8eff9e7d58b8b9637a2818f Mon Sep 17 00:00:00 2001 From: Jerry Wu Date: Tue, 23 Oct 2018 17:04:32 +0800 Subject: [PATCH 695/823] Add python API for new transformers and apply them in inception training example (#2663) * refinement on python API * fix ut --- .../dllib/feature/transform/vision/image.py | 52 +++++++++++++++++++ .../bigdl/dllib/models/inception/inception.py | 42 +++++++-------- 2 files changed, 73 insertions(+), 21 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py b/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py index 6fef178e9fb..f71792d2110 100644 --- a/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py +++ b/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py @@ -671,6 +671,58 @@ class FixExpand(FeatureTransformer): def __init__(self, expand_height, expand_width, bigdl_type="float"): super(FixExpand, self).__init__(bigdl_type, expand_height, expand_width) +class ChannelScaledNormalizer(FeatureTransformer): + """ + Scaled image at channel level with offset and scale + :param mean_r : offset for R channel + :param mean_g : offset for G channel + :param mean_b: offset for B channel + :param scale: scaling factor for all channels + """ + def __init__(self, mean_r, mean_g, mean_b, scale, bigdl_type="float"): + super(ChannelScaledNormalizer, self).__init__(bigdl_type, mean_r, mean_g, mean_b, scale) + +class RandomAlterAspect(FeatureTransformer): + """ + Apply random crop based on area ratio and resize to cropLenth size + :param min_area_ratio min area ratio + :param max_area_ratio max area ratio + :param min_aspect_ratio_change factor applied to ratio area + :param interp_mode interp mode applied in resize + :param crop_length final size resized to + """ + def __init__(self, min_area_ratio, + max_area_ratio, + min_aspect_ratio_change, + interp_mode, + crop_length, bigdl_type="float"): + super(RandomAlterAspect, self).__init__(bigdl_type, min_area_ratio, + max_area_ratio, + min_aspect_ratio_change, + interp_mode, + crop_length) + +class RandomCropper(FeatureTransformer): + """ + Random cropper on uniform distribution with fixed height & width + :param crop_w width cropped to + :param crop_h height cropped to + :param mirror whether mirror + :param cropper_method crop method + :param channels total channels + """ + def __init__(self, crop_w, crop_h, mirror, cropper_method, channels, bigdl_type="float"): + super(RandomCropper, self).__init__(bigdl_type, crop_w, crop_h, mirror, cropper_method, channels) + +class RandomResize(FeatureTransformer): + """ + Random resize between minSize and maxSize and scale height and width to each other + :param min_size min size to resize to + :param max_size max size to resize to + """ + def __init__(self, min_size, max_size, bigdl_type="float"): + super(RandomResize, self).__init__(bigdl_type, min_size, max_size) + class SeqFileFolder(JavaValue): @classmethod diff --git a/python/dllib/src/bigdl/dllib/models/inception/inception.py b/python/dllib/src/bigdl/dllib/models/inception/inception.py index 1227f0604fa..9ef8c61b1f7 100644 --- a/python/dllib/src/bigdl/dllib/models/inception/inception.py +++ b/python/dllib/src/bigdl/dllib/models/inception/inception.py @@ -21,34 +21,34 @@ def inception_layer_v1(input_size, config, name_prefix=""): concat = Concat(2) conv1 = Sequential() conv1.add(SpatialConvolution(input_size, config[1][1], 1, 1, 1, 1) - .set_init_method(weight_init_method=Xavier(),bias_init_method=Zeros()) + .set_init_method(weight_init_method=Xavier(),bias_init_method=ConstInitMethod(0.1)) .set_name(name_prefix + "1x1")) conv1.add(ReLU(True).set_name(name_prefix + "relu_1x1")) concat.add(conv1) conv3 = Sequential() conv3.add(SpatialConvolution(input_size, config[2][1], 1, 1, 1, 1) - .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) + .set_init_method(weight_init_method=Xavier(), bias_init_method=ConstInitMethod(0.1)) .set_name(name_prefix + "3x3_reduce")) conv3.add(ReLU(True).set_name(name_prefix + "relu_3x3_reduce")) conv3.add(SpatialConvolution(config[2][1], config[2][2], 3, 3, 1, 1, 1, 1) - .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) + .set_init_method(weight_init_method=Xavier(), bias_init_method=ConstInitMethod(0.1)) .set_name(name_prefix + "3x3")) conv3.add(ReLU(True).set_name(name_prefix + "relu_3x3")) concat.add(conv3) conv5 = Sequential() conv5.add(SpatialConvolution(input_size, config[3][1], 1, 1, 1, 1) - .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) + .set_init_method(weight_init_method=Xavier(), bias_init_method=ConstInitMethod(0.1)) .set_name(name_prefix + "5x5_reduce")) conv5.add(ReLU(True).set_name(name_prefix + "relu_5x5_reduce")) conv5.add(SpatialConvolution(config[3][1], config[3][2], 5, 5, 1, 1, 2, 2) - .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) + .set_init_method(weight_init_method=Xavier(), bias_init_method=ConstInitMethod(0.1)) .set_name(name_prefix + "5x5")) conv5.add(ReLU(True).set_name(name_prefix + "relu_5x5")) concat.add(conv5) pool = Sequential() pool.add(SpatialMaxPooling(3, 3, 1, 1, 1, 1, to_ceil=True).set_name(name_prefix + "pool")) pool.add(SpatialConvolution(input_size, config[4][1], 1, 1, 1, 1) - .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) + .set_init_method(weight_init_method=Xavier(), bias_init_method=ConstInitMethod(0.1)) .set_name(name_prefix + "pool_proj")) pool.add(ReLU(True).set_name(name_prefix + "relu_pool_proj")) concat.add(pool).set_name(name_prefix + "output") @@ -58,17 +58,17 @@ def inception_layer_v1(input_size, config, name_prefix=""): def inception_v1_no_aux_classifier(class_num, has_dropout=True): model = Sequential() model.add(SpatialConvolution(3, 64, 7, 7, 2, 2, 3, 3, 1, False) - .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) + .set_init_method(weight_init_method=Xavier(), bias_init_method=ConstInitMethod(0.1)) .set_name("conv1/7x7_s2")) model.add(ReLU(True).set_name("conv1/relu_7x7")) model.add(SpatialMaxPooling(3, 3, 2, 2, to_ceil=True).set_name("pool1/3x3_s2")) model.add(SpatialCrossMapLRN(5, 0.0001, 0.75).set_name("pool1/norm1")) model.add(SpatialConvolution(64, 64, 1, 1, 1, 1) - .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) + .set_init_method(weight_init_method=Xavier(), bias_init_method=ConstInitMethod(0.1)) .set_name("conv2/3x3_reduce")) model.add(ReLU(True).set_name("conv2/relu_3x3_reduce")) model.add(SpatialConvolution(64, 192, 3, 3, 1, 1, 1, 1) - .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) + .set_init_method(weight_init_method=Xavier(), bias_init_method=ConstInitMethod(0.1)) .set_name("conv2/3x3")) model.add(ReLU(True).set_name("conv2/relu_3x3")) model.add(SpatialCrossMapLRN(5, 0.0001, 0.75).set_name("conv2/norm2")) @@ -108,7 +108,7 @@ def inception_v1_no_aux_classifier(class_num, has_dropout=True): def inception_v1(class_num, has_dropout=True): feature1 = Sequential() feature1.add(SpatialConvolution(3, 64, 7, 7, 2, 2, 3, 3, 1, False) - .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) + .set_init_method(weight_init_method=Xavier(), bias_init_method=ConstInitMethod(0.1)) .set_name("conv1/7x7_s2")) feature1.add(ReLU(True).set_name("conv1/relu_7x7")) feature1.add( @@ -117,11 +117,11 @@ def inception_v1(class_num, has_dropout=True): feature1.add(SpatialCrossMapLRN(5, 0.0001, 0.75) .set_name("pool1/norm1")) feature1.add(SpatialConvolution(64, 64, 1, 1, 1, 1) - .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) + .set_init_method(weight_init_method=Xavier(), bias_init_method=ConstInitMethod(0.1)) .set_name("conv2/3x3_reduce")) feature1.add(ReLU(True).set_name("conv2/relu_3x3_reduce")) feature1.add(SpatialConvolution(64, 192, 3, 3, 1, 1, 1, 1) - .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) + .set_init_method(weight_init_method=Xavier(), bias_init_method=ConstInitMethod(0.1)) .set_name("conv2/3x3")) feature1.add(ReLU(True).set_name("conv2/relu_3x3")) feature1.add(SpatialCrossMapLRN(5, 0.0001, 0.75).set_name("conv2/norm2")) @@ -267,20 +267,20 @@ def config_option_parser(): image_size = 224 # create dataset train_transformer = Pipeline([PixelBytesToMat(), - RandomCrop(image_size, image_size), - RandomTransformer(HFlip(), 0.5), - ChannelNormalize(0.485, 0.456, 0.406, 0.229, 0.224, 0.225), - MatToTensor(to_rgb=True), + Resize(256, 256), + RandomCropper(image_size, image_size, True, "Random", 3), + ChannelNormalize(123, 117, 104), + MatToTensor(to_rgb=False), ImageFrameToSample(input_keys=["imageTensor"], target_keys=["label"]) ]) raw_train_data = get_inception_data(options.folder, sc, "train") train_data = DataSet.image_frame(raw_train_data).transform(train_transformer) val_transformer = Pipeline([PixelBytesToMat(), - CenterCrop(image_size, image_size), - RandomTransformer(HFlip(), 0.5), - ChannelNormalize(0.485, 0.456, 0.406, 0.229, 0.224, 0.225), - MatToTensor(to_rgb=True), + Resize(256, 256), + RandomCropper(image_size, image_size, False, "Center", 3), + ChannelNormalize(123, 117, 104), + MatToTensor(to_rgb=False), ImageFrameToSample(input_keys=["imageTensor"], target_keys=["label"]) ]) raw_val_data = get_inception_data(options.folder, sc, "val") @@ -315,7 +315,7 @@ def config_option_parser(): polyIteration = maxIteration - warmup_iteration lrSchedule = SequentialSchedule(iterationPerEpoch) lrSchedule.add(Warmup(warmupDelta), warmup_iteration) - lrSchedule.add(Poly(0.5, polyIteration), polyIteration) + lrSchedule.add(Poly(0.5, maxIteration), polyIteration) optim = SGD(learningrate=options.learningRate, learningrate_decay=0.0, weightdecay=options.weightDecay, momentum=0.9, dampening=0.0, nesterov=False, leaningrate_schedule=lrSchedule) From 3de7728f2c5eb44763f14490ae252b96be9d3c73 Mon Sep 17 00:00:00 2001 From: Jerry Wu Date: Tue, 23 Oct 2018 17:04:32 +0800 Subject: [PATCH 696/823] Add python API for new transformers and apply them in inception training example (#2663) * refinement on python API * fix ut --- python/dllib/test/bigdl/transform/test_image.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/python/dllib/test/bigdl/transform/test_image.py b/python/dllib/test/bigdl/transform/test_image.py index fcf8e73ffb6..fb2adb1f59d 100644 --- a/python/dllib/test/bigdl/transform/test_image.py +++ b/python/dllib/test/bigdl/transform/test_image.py @@ -219,5 +219,21 @@ def test_random_split(self): imageFrame = ImageFrame.read(resource_path, self.sc) splits = imageFrame.random_split([1.0]) + def test_channel_scaled_normalizer(self): + transformer = ChannelScaledNormalizer(123, 117, 104, 1.0) + self.transformer_test(transformer) + + def test_random_alter_aspect(self): + transformer = RandomAlterAspect(0.08, 1, 0.75, "CUBIC", 20) + self.transformer_test(transformer) + + def test_random_cropper(self): + transformer = RandomCropper(20, 20, True, "Random", 3) + self.transformer_test(transformer) + + def test_random_resize(self): + transformer = RandomResize(100, 100) + self.transformer_test(transformer) + if __name__ == "__main__": pytest.main([__file__]) From 3064b70ea1250d7c589b020fb72e3b5ce388e489 Mon Sep 17 00:00:00 2001 From: Jerry Wu Date: Thu, 22 Nov 2018 16:06:21 +0800 Subject: [PATCH 697/823] fix channel normalize value to float (#2688) --- python/dllib/src/bigdl/dllib/models/inception/inception.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/models/inception/inception.py b/python/dllib/src/bigdl/dllib/models/inception/inception.py index 9ef8c61b1f7..64f78c2a4ed 100644 --- a/python/dllib/src/bigdl/dllib/models/inception/inception.py +++ b/python/dllib/src/bigdl/dllib/models/inception/inception.py @@ -269,7 +269,7 @@ def config_option_parser(): train_transformer = Pipeline([PixelBytesToMat(), Resize(256, 256), RandomCropper(image_size, image_size, True, "Random", 3), - ChannelNormalize(123, 117, 104), + ChannelNormalize(123.0, 117.0, 104.0), MatToTensor(to_rgb=False), ImageFrameToSample(input_keys=["imageTensor"], target_keys=["label"]) ]) @@ -279,7 +279,7 @@ def config_option_parser(): val_transformer = Pipeline([PixelBytesToMat(), Resize(256, 256), RandomCropper(image_size, image_size, False, "Center", 3), - ChannelNormalize(123, 117, 104), + ChannelNormalize(123.0, 117.0, 104.0), MatToTensor(to_rgb=False), ImageFrameToSample(input_keys=["imageTensor"], target_keys=["label"]) ]) From 46487864783be5fbab025cc9d7ea08413e2b3a6f Mon Sep 17 00:00:00 2001 From: Jerry Wu Date: Thu, 22 Nov 2018 16:06:21 +0800 Subject: [PATCH 698/823] fix channel normalize value to float (#2688) --- python/dllib/src/bigdl/dllib/models/inception/inception.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/models/inception/inception.py b/python/dllib/src/bigdl/dllib/models/inception/inception.py index 9ef8c61b1f7..64f78c2a4ed 100644 --- a/python/dllib/src/bigdl/dllib/models/inception/inception.py +++ b/python/dllib/src/bigdl/dllib/models/inception/inception.py @@ -269,7 +269,7 @@ def config_option_parser(): train_transformer = Pipeline([PixelBytesToMat(), Resize(256, 256), RandomCropper(image_size, image_size, True, "Random", 3), - ChannelNormalize(123, 117, 104), + ChannelNormalize(123.0, 117.0, 104.0), MatToTensor(to_rgb=False), ImageFrameToSample(input_keys=["imageTensor"], target_keys=["label"]) ]) @@ -279,7 +279,7 @@ def config_option_parser(): val_transformer = Pipeline([PixelBytesToMat(), Resize(256, 256), RandomCropper(image_size, image_size, False, "Center", 3), - ChannelNormalize(123, 117, 104), + ChannelNormalize(123.0, 117.0, 104.0), MatToTensor(to_rgb=False), ImageFrameToSample(input_keys=["imageTensor"], target_keys=["label"]) ]) From 9e54f14c4ca0091c7d2a9fb1400c56220310744e Mon Sep 17 00:00:00 2001 From: Xin Qiu Date: Wed, 9 Jan 2019 09:57:01 +0800 Subject: [PATCH 699/823] add Trigger and/or python API (#2682) --- .../dllib/src/bigdl/dllib/optim/optimizer.py | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/optim/optimizer.py b/python/dllib/src/bigdl/dllib/optim/optimizer.py index c67279270a5..15e33c860e6 100644 --- a/python/dllib/src/bigdl/dllib/optim/optimizer.py +++ b/python/dllib/src/bigdl/dllib/optim/optimizer.py @@ -253,6 +253,45 @@ def __init__(self, min, bigdl_type="float"): """ JavaValue.__init__(self, None, bigdl_type, min) +class TriggerAnd(JavaValue): + """ + A trigger contains other triggers and triggers when all of them trigger (logical AND) + + + >>> a = TriggerAnd(MinLoss(0.1), MaxEpoch(2)) + creating: createMinLoss + creating: createMaxEpoch + creating: createTriggerAnd + """ + def __init__(self, first, *other): + """ + Create a And trigger. + + + :param first: first Trigger + :param other: other Trigger + """ + JavaValue.__init__(self, None, "float", first, list(other)) + +class TriggerOr(JavaValue): + """ + A trigger contains other triggers and triggers when any of them trigger (logical OR) + + + >>> o = TriggerOr(MinLoss(0.1), MaxEpoch(2)) + creating: createMinLoss + creating: createMaxEpoch + creating: createTriggerOr + """ + def __init__(self, first, *other): + """ + Create a Or trigger. + + + :param first: first Trigger + :param other: other Trigger + """ + JavaValue.__init__(self, None, "float", first, list(other)) class Poly(JavaValue): """ From 3f388e852382bd1b0ff769df19fd956e58322d84 Mon Sep 17 00:00:00 2001 From: Xin Qiu Date: Wed, 9 Jan 2019 09:57:01 +0800 Subject: [PATCH 700/823] add Trigger and/or python API (#2682) --- .../dllib/src/bigdl/dllib/optim/optimizer.py | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/optim/optimizer.py b/python/dllib/src/bigdl/dllib/optim/optimizer.py index c67279270a5..15e33c860e6 100644 --- a/python/dllib/src/bigdl/dllib/optim/optimizer.py +++ b/python/dllib/src/bigdl/dllib/optim/optimizer.py @@ -253,6 +253,45 @@ def __init__(self, min, bigdl_type="float"): """ JavaValue.__init__(self, None, bigdl_type, min) +class TriggerAnd(JavaValue): + """ + A trigger contains other triggers and triggers when all of them trigger (logical AND) + + + >>> a = TriggerAnd(MinLoss(0.1), MaxEpoch(2)) + creating: createMinLoss + creating: createMaxEpoch + creating: createTriggerAnd + """ + def __init__(self, first, *other): + """ + Create a And trigger. + + + :param first: first Trigger + :param other: other Trigger + """ + JavaValue.__init__(self, None, "float", first, list(other)) + +class TriggerOr(JavaValue): + """ + A trigger contains other triggers and triggers when any of them trigger (logical OR) + + + >>> o = TriggerOr(MinLoss(0.1), MaxEpoch(2)) + creating: createMinLoss + creating: createMaxEpoch + creating: createTriggerOr + """ + def __init__(self, first, *other): + """ + Create a Or trigger. + + + :param first: first Trigger + :param other: other Trigger + """ + JavaValue.__init__(self, None, "float", first, list(other)) class Poly(JavaValue): """ From 7094ff30c021970b9363868d6179ee269987f268 Mon Sep 17 00:00:00 2001 From: Xin Qiu Date: Fri, 11 Jan 2019 15:39:15 +0800 Subject: [PATCH 701/823] update sparse tensor's document (#2714) --- python/dllib/src/bigdl/dllib/utils/common.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/dllib/src/bigdl/dllib/utils/common.py b/python/dllib/src/bigdl/dllib/utils/common.py index e2265b8e2bc..dcef81812c8 100644 --- a/python/dllib/src/bigdl/dllib/utils/common.py +++ b/python/dllib/src/bigdl/dllib/utils/common.py @@ -228,7 +228,8 @@ def sparse(cls, a_ndarray, i_ndarray, shape, bigdl_type="float"): :param a_ndarray non-zero elements in this SparseTensor :param i_ndarray zero-based indices for non-zero element i_ndarray's shape should be (shape.size, a_ndarray.size) - And the i-th non-zero elements indices is i_ndarray[:, 1] + And the i-th non-zero elements indices is i_ndarray[:, 1], + should be zero-based and ascending; :param shape shape as a DenseTensor. >>> import numpy as np From e3503b37ee352975060d39770ad64f6eb32ea6bb Mon Sep 17 00:00:00 2001 From: Xin Qiu Date: Fri, 11 Jan 2019 15:39:15 +0800 Subject: [PATCH 702/823] update sparse tensor's document (#2714) --- python/dllib/src/bigdl/utils/common.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/dllib/src/bigdl/utils/common.py b/python/dllib/src/bigdl/utils/common.py index e2265b8e2bc..dcef81812c8 100644 --- a/python/dllib/src/bigdl/utils/common.py +++ b/python/dllib/src/bigdl/utils/common.py @@ -228,7 +228,8 @@ def sparse(cls, a_ndarray, i_ndarray, shape, bigdl_type="float"): :param a_ndarray non-zero elements in this SparseTensor :param i_ndarray zero-based indices for non-zero element i_ndarray's shape should be (shape.size, a_ndarray.size) - And the i-th non-zero elements indices is i_ndarray[:, 1] + And the i-th non-zero elements indices is i_ndarray[:, 1], + should be zero-based and ascending; :param shape shape as a DenseTensor. >>> import numpy as np From 49d81a39345b44291fb2444f9d9148f30ff5c360 Mon Sep 17 00:00:00 2001 From: Jerry Wu Date: Tue, 19 Mar 2019 11:18:51 +0800 Subject: [PATCH 703/823] update readme for v1 training (#2763) --- .../bigdl/dllib/models/inception/README.md | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/models/inception/README.md b/python/dllib/src/bigdl/dllib/models/inception/README.md index 2a2f7d6c4fd..86cafb879ac 100644 --- a/python/dllib/src/bigdl/dllib/models/inception/README.md +++ b/python/dllib/src/bigdl/dllib/models/inception/README.md @@ -53,18 +53,18 @@ ${SPARK_HOME}/bin/spark-submit \ --master spark://... \ --executor-memory 150g \ --driver-memory 100g \ ---executor-cores 28 \ ---total-executor-cores 448 \ +--executor-cores 4 \ +--total-executor-cores 64 \ --properties-file ${BigDL_HOME}/dist/conf/spark-bigdl.conf \ --jars ${BigDL_JAR_PATH} \ --py-files ${PYTHON_API_PATH} \ ${BigDL_HOME}/pyspark/bigdl/models/inception/inception.py \ -f hdfs://... \ ---batchSize 1792 \ ---learningRate 0.0898 \ ---weightDecay 0.0001 \ ---checkpointIteration 6200 \ --i 62000 \ +--batchSize 1024 \ +--learningRate 0.065 \ +--weightDecay 0.0002 \ +--checkpointIteration 1000 \ +-i 90000 \ --checkpoint /models/inception ``` * Spark yarn client mode, example command @@ -80,7 +80,7 @@ ${SPARK_HOME}/bin/spark-submit \ --deploy-mode client \ --executor-memory 150g \ --driver-memory 100g \ ---executor-cores 28 \ +--executor-cores 4 \ --num-executors 16 \ --properties-file ${BigDL_HOME}/dist/conf/spark-bigdl.conf \ --jars ${BigDL_JAR_PATH} \ @@ -89,11 +89,11 @@ ${SPARK_HOME}/bin/spark-submit \ --py-files ${PYTHON_API_PATH} \ ${BigDL_HOME}/pyspark/bigdl/models/inception/inception.py \ -f hdfs://... \ ---batchSize 1792 \ ---learningRate 0.0898 \ ---weightDecay 0.0001 \ ---checkpointIteration 6200 \ --i 62000 \ +--batchSize 1024 \ +--learningRate 0.065 \ +--weightDecay 0.0002 \ +--checkpointIteration 1000 \ +-i 90000 \ --checkpoint /models/inception ``` From 70c9fdbb20c949969552bcc12fc956e9891e4d17 Mon Sep 17 00:00:00 2001 From: Jerry Wu Date: Tue, 19 Mar 2019 11:18:51 +0800 Subject: [PATCH 704/823] update readme for v1 training (#2763) --- .../bigdl/dllib/models/inception/README.md | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/models/inception/README.md b/python/dllib/src/bigdl/dllib/models/inception/README.md index 2a2f7d6c4fd..86cafb879ac 100644 --- a/python/dllib/src/bigdl/dllib/models/inception/README.md +++ b/python/dllib/src/bigdl/dllib/models/inception/README.md @@ -53,18 +53,18 @@ ${SPARK_HOME}/bin/spark-submit \ --master spark://... \ --executor-memory 150g \ --driver-memory 100g \ ---executor-cores 28 \ ---total-executor-cores 448 \ +--executor-cores 4 \ +--total-executor-cores 64 \ --properties-file ${BigDL_HOME}/dist/conf/spark-bigdl.conf \ --jars ${BigDL_JAR_PATH} \ --py-files ${PYTHON_API_PATH} \ ${BigDL_HOME}/pyspark/bigdl/models/inception/inception.py \ -f hdfs://... \ ---batchSize 1792 \ ---learningRate 0.0898 \ ---weightDecay 0.0001 \ ---checkpointIteration 6200 \ --i 62000 \ +--batchSize 1024 \ +--learningRate 0.065 \ +--weightDecay 0.0002 \ +--checkpointIteration 1000 \ +-i 90000 \ --checkpoint /models/inception ``` * Spark yarn client mode, example command @@ -80,7 +80,7 @@ ${SPARK_HOME}/bin/spark-submit \ --deploy-mode client \ --executor-memory 150g \ --driver-memory 100g \ ---executor-cores 28 \ +--executor-cores 4 \ --num-executors 16 \ --properties-file ${BigDL_HOME}/dist/conf/spark-bigdl.conf \ --jars ${BigDL_JAR_PATH} \ @@ -89,11 +89,11 @@ ${SPARK_HOME}/bin/spark-submit \ --py-files ${PYTHON_API_PATH} \ ${BigDL_HOME}/pyspark/bigdl/models/inception/inception.py \ -f hdfs://... \ ---batchSize 1792 \ ---learningRate 0.0898 \ ---weightDecay 0.0001 \ ---checkpointIteration 6200 \ --i 62000 \ +--batchSize 1024 \ +--learningRate 0.065 \ +--weightDecay 0.0002 \ +--checkpointIteration 1000 \ +-i 90000 \ --checkpoint /models/inception ``` From 1d01d999914292e867eff46d1cd417185ff3513e Mon Sep 17 00:00:00 2001 From: Jerry Wu Date: Thu, 4 Apr 2019 13:50:14 +0800 Subject: [PATCH 705/823] flip to 0.9.0 (#2792) --- .../dllib/src/bigdl/dllib/models/textclassifier/README.md | 8 ++++---- version.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/models/textclassifier/README.md b/python/dllib/src/bigdl/dllib/models/textclassifier/README.md index 3bec22eb35b..17c619abcc0 100644 --- a/python/dllib/src/bigdl/dllib/models/textclassifier/README.md +++ b/python/dllib/src/bigdl/dllib/models/textclassifier/README.md @@ -28,8 +28,8 @@ $ [/tmp/news20]$ tree . -L 1 - The example code would automatically download the data during the first run. ### Run via pip install -- [Install from pip](https://bigdl-project.github.io/0.8.0-SNAPSHOT/#PythonUserGuide/install-from-pip/) -- Optional: check [Run after pip install](https://bigdl-project.github.io/0.8.0-SNAPSHOT/#PythonUserGuide/run-from-pip/) +- [Install from pip](https://bigdl-project.github.io/0.9.0-SNAPSHOT/#PythonUserGuide/install-from-pip/) +- Optional: check [Run after pip install](https://bigdl-project.github.io/0.9.0-SNAPSHOT/#PythonUserGuide/run-from-pip/) - Run the following command locally ``` python ${BigDL_HOME}/pyspark/bigdl/models/textclassifier/textclassifier.py --max_epoch 3 --model cnn @@ -37,8 +37,8 @@ python ${BigDL_HOME}/pyspark/bigdl/models/textclassifier/textclassifier.py --max ``` ### Run via spark-submit -- [Install without pip](https://bigdl-project.github.io/0.8.0-SNAPSHOT/#PythonUserGuide/install-without-pip/) -- Optional: check [Run without pip](https://bigdl-project.github.io/0.8.0-SNAPSHOT/#PythonUserGuide/run-without-pip/) +- [Install without pip](https://bigdl-project.github.io/0.9.0-SNAPSHOT/#PythonUserGuide/install-without-pip/) +- Optional: check [Run without pip](https://bigdl-project.github.io/0.9.0-SNAPSHOT/#PythonUserGuide/run-without-pip/) - Run the following command ```{r, engine='sh'} PYTHONHASHSEED=0 diff --git a/version.py b/version.py index 64e985395d0..86820d45817 100644 --- a/version.py +++ b/version.py @@ -14,4 +14,4 @@ # limitations under the License. # -__version__ = "0.8.0.dev0" +__version__ = "0.9.0.dev0" From c2f4b32add9a8d506f9b85bd7077005adff9921a Mon Sep 17 00:00:00 2001 From: Jerry Wu Date: Thu, 4 Apr 2019 13:50:14 +0800 Subject: [PATCH 706/823] flip to 0.9.0 (#2792) --- .../dllib/src/bigdl/dllib/models/textclassifier/README.md | 8 ++++---- version.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/models/textclassifier/README.md b/python/dllib/src/bigdl/dllib/models/textclassifier/README.md index 3bec22eb35b..17c619abcc0 100644 --- a/python/dllib/src/bigdl/dllib/models/textclassifier/README.md +++ b/python/dllib/src/bigdl/dllib/models/textclassifier/README.md @@ -28,8 +28,8 @@ $ [/tmp/news20]$ tree . -L 1 - The example code would automatically download the data during the first run. ### Run via pip install -- [Install from pip](https://bigdl-project.github.io/0.8.0-SNAPSHOT/#PythonUserGuide/install-from-pip/) -- Optional: check [Run after pip install](https://bigdl-project.github.io/0.8.0-SNAPSHOT/#PythonUserGuide/run-from-pip/) +- [Install from pip](https://bigdl-project.github.io/0.9.0-SNAPSHOT/#PythonUserGuide/install-from-pip/) +- Optional: check [Run after pip install](https://bigdl-project.github.io/0.9.0-SNAPSHOT/#PythonUserGuide/run-from-pip/) - Run the following command locally ``` python ${BigDL_HOME}/pyspark/bigdl/models/textclassifier/textclassifier.py --max_epoch 3 --model cnn @@ -37,8 +37,8 @@ python ${BigDL_HOME}/pyspark/bigdl/models/textclassifier/textclassifier.py --max ``` ### Run via spark-submit -- [Install without pip](https://bigdl-project.github.io/0.8.0-SNAPSHOT/#PythonUserGuide/install-without-pip/) -- Optional: check [Run without pip](https://bigdl-project.github.io/0.8.0-SNAPSHOT/#PythonUserGuide/run-without-pip/) +- [Install without pip](https://bigdl-project.github.io/0.9.0-SNAPSHOT/#PythonUserGuide/install-without-pip/) +- Optional: check [Run without pip](https://bigdl-project.github.io/0.9.0-SNAPSHOT/#PythonUserGuide/run-without-pip/) - Run the following command ```{r, engine='sh'} PYTHONHASHSEED=0 diff --git a/version.py b/version.py index 64e985395d0..86820d45817 100644 --- a/version.py +++ b/version.py @@ -14,4 +14,4 @@ # limitations under the License. # -__version__ = "0.8.0.dev0" +__version__ = "0.9.0.dev0" From 85dfdaef064370b89938f2eb1b1d4729d06c738d Mon Sep 17 00:00:00 2001 From: zhangxiaoli73 <380761639@qq.com> Date: Fri, 26 Apr 2019 09:49:44 +0800 Subject: [PATCH 707/823] [New feature] Add attention layer and ffn layer (#2795) * add attention layer * add ffn layer and more unit tests * refactor according to pr comments * add SerializationTest * fix unit tests * add python api --- python/dllib/src/bigdl/dllib/nn/layer.py | 28 ++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index a89bfcc573b..e1b4462db2e 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -902,6 +902,34 @@ def save_graph_topology(self, log_path, bigdl_type="float"): callBigDlFunc(bigdl_type, "saveGraphTopology", self.value, log_path) return self +class Attention(Layer): + + ''' + Implementation of multiheaded attention and self-attention layers. + + >>> attention = Attention(8, 4, 1.0) + creating: createAttention + ''' + + def __init__(self, hidden_size, num_heads, attention_dropout, bigdl_type="float"): + super(Attention, self).__init__(None, bigdl_type, + hidden_size, num_heads, attention_dropout) + +class FeedForwardNetwork(Layer): + + ''' + Implementation FeedForwardNetwork constructed with fully connected network. + Input with shape (batch_size, length, hidden_size) + Output with shape (batch_size, length, hidden_size) + + >>> ffn = FeedForwardNetwork(8, 4, 1.0) + creating: createFeedForwardNetwork + ''' + + def __init__(self, hidden_size, filter_size, relu_dropout, bigdl_type="float"): + super(FeedForwardNetwork, self).__init__(None, bigdl_type, + hidden_size, filter_size, relu_dropout) + class Linear(Layer): ''' From bbd5f9e0ccc5b8e7ccc31e0575ba8384e953dda3 Mon Sep 17 00:00:00 2001 From: zhangxiaoli73 <380761639@qq.com> Date: Fri, 26 Apr 2019 09:49:44 +0800 Subject: [PATCH 708/823] [New feature] Add attention layer and ffn layer (#2795) * add attention layer * add ffn layer and more unit tests * refactor according to pr comments * add SerializationTest * fix unit tests * add python api --- python/dllib/src/bigdl/dllib/nn/layer.py | 28 ++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index a89bfcc573b..e1b4462db2e 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -902,6 +902,34 @@ def save_graph_topology(self, log_path, bigdl_type="float"): callBigDlFunc(bigdl_type, "saveGraphTopology", self.value, log_path) return self +class Attention(Layer): + + ''' + Implementation of multiheaded attention and self-attention layers. + + >>> attention = Attention(8, 4, 1.0) + creating: createAttention + ''' + + def __init__(self, hidden_size, num_heads, attention_dropout, bigdl_type="float"): + super(Attention, self).__init__(None, bigdl_type, + hidden_size, num_heads, attention_dropout) + +class FeedForwardNetwork(Layer): + + ''' + Implementation FeedForwardNetwork constructed with fully connected network. + Input with shape (batch_size, length, hidden_size) + Output with shape (batch_size, length, hidden_size) + + >>> ffn = FeedForwardNetwork(8, 4, 1.0) + creating: createFeedForwardNetwork + ''' + + def __init__(self, hidden_size, filter_size, relu_dropout, bigdl_type="float"): + super(FeedForwardNetwork, self).__init__(None, bigdl_type, + hidden_size, filter_size, relu_dropout) + class Linear(Layer): ''' From eefbb1a8c5b68bf6af19a1acc6be1b8b9a455741 Mon Sep 17 00:00:00 2001 From: zhangxiaoli73 <380761639@qq.com> Date: Fri, 26 Apr 2019 09:49:44 +0800 Subject: [PATCH 709/823] [New feature] Add attention layer and ffn layer (#2795) * add attention layer * add ffn layer and more unit tests * refactor according to pr comments * add SerializationTest * fix unit tests * add python api --- python/dllib/test/dev/diff.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/python/dllib/test/dev/diff.py b/python/dllib/test/dev/diff.py index e0bca163fa2..72777f3341a 100755 --- a/python/dllib/test/dev/diff.py +++ b/python/dllib/test/dev/diff.py @@ -29,7 +29,9 @@ def extract_scala_class(class_path): exclude_key_words = set(["*", "abstract", "Const", "Fill", "Shape", - "SplitAndSelect", "StrideSlice", "Scheduler", "StaticGraph", "DynamicGraph", "DynamicContainer"]) # noqa + "SplitAndSelect", "StrideSlice", "Scheduler", + "StaticGraph", "DynamicGraph", "DynamicContainer", + "SplitHeads", "CombineHeads"]) # noqa include_key_words = set(["Module", "Criterion", "Container", "Cell", "TensorNumeric"]) # noqa content = "\n".join([line for line in open(class_path).readlines() if all([key not in line for key in exclude_key_words])]) # noqa match = re.search(r"class ([\w]+)[^{]+", content) From dccfdaa59bdb276f85493ec67207cba36153c7ed Mon Sep 17 00:00:00 2001 From: zhangxiaoli73 <380761639@qq.com> Date: Wed, 22 May 2019 11:05:59 +0800 Subject: [PATCH 710/823] add layer norm and expand size layers (#2819) * add layer norm and expand size * meet pr comments --- python/dllib/src/bigdl/dllib/nn/layer.py | 34 ++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index e1b4462db2e..19806837c7f 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -929,6 +929,40 @@ class FeedForwardNetwork(Layer): def __init__(self, hidden_size, filter_size, relu_dropout, bigdl_type="float"): super(FeedForwardNetwork, self).__init__(None, bigdl_type, hidden_size, filter_size, relu_dropout) +class LayerNormalization(Layer): + ''' + Applies layer normalization. + + >>> norm = LayerNormalization(8) + creating: createLayerNormalization + ''' + + def __init__(self, hidden_size, bigdl_type="float"): + super(LayerNormalization, self).__init__(None, bigdl_type, hidden_size) + +class TableOperation(Layer): + ''' + When two tensors have different size, firstly expand small size tensor to large size tensor, + and then do table operation. + + >>> norm = TableOperation(CMulTable()) + creating: createCMulTable + creating: createTableOperation + ''' + + def __init__(self, operation_layer, bigdl_type="float"): + super(TableOperation, self).__init__(None, bigdl_type, operation_layer) + +class ExpandSize(Layer): + ''' + Expand tensor to configured size + + >>> expand = ExpandSize([2, 3, 4]) + creating: createExpandSize + ''' + + def __init__(self, sizes, bigdl_type="float"): + super(ExpandSize, self).__init__(None, bigdl_type, sizes) class Linear(Layer): From 481b1058bfb59cf7907ac7aaba1912672a64946f Mon Sep 17 00:00:00 2001 From: zhangxiaoli73 <380761639@qq.com> Date: Wed, 22 May 2019 11:05:59 +0800 Subject: [PATCH 711/823] add layer norm and expand size layers (#2819) * add layer norm and expand size * meet pr comments --- python/dllib/src/bigdl/dllib/nn/layer.py | 34 ++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index e1b4462db2e..19806837c7f 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -929,6 +929,40 @@ class FeedForwardNetwork(Layer): def __init__(self, hidden_size, filter_size, relu_dropout, bigdl_type="float"): super(FeedForwardNetwork, self).__init__(None, bigdl_type, hidden_size, filter_size, relu_dropout) +class LayerNormalization(Layer): + ''' + Applies layer normalization. + + >>> norm = LayerNormalization(8) + creating: createLayerNormalization + ''' + + def __init__(self, hidden_size, bigdl_type="float"): + super(LayerNormalization, self).__init__(None, bigdl_type, hidden_size) + +class TableOperation(Layer): + ''' + When two tensors have different size, firstly expand small size tensor to large size tensor, + and then do table operation. + + >>> norm = TableOperation(CMulTable()) + creating: createCMulTable + creating: createTableOperation + ''' + + def __init__(self, operation_layer, bigdl_type="float"): + super(TableOperation, self).__init__(None, bigdl_type, operation_layer) + +class ExpandSize(Layer): + ''' + Expand tensor to configured size + + >>> expand = ExpandSize([2, 3, 4]) + creating: createExpandSize + ''' + + def __init__(self, sizes, bigdl_type="float"): + super(ExpandSize, self).__init__(None, bigdl_type, sizes) class Linear(Layer): From e2f22c4b066cd2d75c0fc6c1680d12bc46021f04 Mon Sep 17 00:00:00 2001 From: zhangxiaoli73 <380761639@qq.com> Date: Wed, 22 May 2019 11:05:59 +0800 Subject: [PATCH 712/823] add layer norm and expand size layers (#2819) * add layer norm and expand size * meet pr comments --- python/dllib/test/dev/diff.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/dllib/test/dev/diff.py b/python/dllib/test/dev/diff.py index 72777f3341a..33c3e0d8439 100755 --- a/python/dllib/test/dev/diff.py +++ b/python/dllib/test/dev/diff.py @@ -31,7 +31,7 @@ def extract_scala_class(class_path): exclude_key_words = set(["*", "abstract", "Const", "Fill", "Shape", "SplitAndSelect", "StrideSlice", "Scheduler", "StaticGraph", "DynamicGraph", "DynamicContainer", - "SplitHeads", "CombineHeads"]) # noqa + "SplitHeads", "CombineHeads", "VectorProduct"]) # noqa include_key_words = set(["Module", "Criterion", "Container", "Cell", "TensorNumeric"]) # noqa content = "\n".join([line for line in open(class_path).readlines() if all([key not in line for key in exclude_key_words])]) # noqa match = re.search(r"class ([\w]+)[^{]+", content) From d9a4af382f8eba2fbfec1b4a36dfc1e4cdc02cf1 Mon Sep 17 00:00:00 2001 From: zhangxiaoli73 <380761639@qq.com> Date: Tue, 4 Jun 2019 16:55:47 +0800 Subject: [PATCH 713/823] [New feature] add transformer layer (#2825) * add transformer * refactor class name * use same embedding for translation * fix pr comments --- python/dllib/src/bigdl/dllib/nn/layer.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 19806837c7f..6002706ef2c 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -964,6 +964,20 @@ class ExpandSize(Layer): def __init__(self, sizes, bigdl_type="float"): super(ExpandSize, self).__init__(None, bigdl_type, sizes) +class Transformer(Layer): + + ''' + Implementation for Transformer + >>> layer = Transformer(20, 4, 2, 3, 1, 0.1, 0.1, 0.1) + creating: createTransformer + ''' + def __init__(self, vocab_size, hidden_size, num_heads, filter_size, num_hidden_layers, + postprocess_dropout, attention_dropout, + relu_dropout, bigdl_type="float"): + super(Transformer, self).__init__(None, bigdl_type, vocab_size, + hidden_size, num_heads, filter_size, + num_hidden_layers, postprocess_dropout, + attention_dropout, relu_dropout) class Linear(Layer): ''' From 2584b2a742b529f4f971c4a298a9e3ec88a476fe Mon Sep 17 00:00:00 2001 From: zhangxiaoli73 <380761639@qq.com> Date: Tue, 4 Jun 2019 16:55:47 +0800 Subject: [PATCH 714/823] [New feature] add transformer layer (#2825) * add transformer * refactor class name * use same embedding for translation * fix pr comments --- python/dllib/src/bigdl/dllib/nn/layer.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 19806837c7f..6002706ef2c 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -964,6 +964,20 @@ class ExpandSize(Layer): def __init__(self, sizes, bigdl_type="float"): super(ExpandSize, self).__init__(None, bigdl_type, sizes) +class Transformer(Layer): + + ''' + Implementation for Transformer + >>> layer = Transformer(20, 4, 2, 3, 1, 0.1, 0.1, 0.1) + creating: createTransformer + ''' + def __init__(self, vocab_size, hidden_size, num_heads, filter_size, num_hidden_layers, + postprocess_dropout, attention_dropout, + relu_dropout, bigdl_type="float"): + super(Transformer, self).__init__(None, bigdl_type, vocab_size, + hidden_size, num_heads, filter_size, + num_hidden_layers, postprocess_dropout, + attention_dropout, relu_dropout) class Linear(Layer): ''' From 0019e7ee9e8759a7ef7bb24d6afe64b248b510a2 Mon Sep 17 00:00:00 2001 From: majing921201 <1834475657@qq.com> Date: Sun, 7 Jul 2019 07:54:02 +0800 Subject: [PATCH 715/823] [WIP]Add beam search feature in transformer model (#2834) * add beam search feature * Update beam search feature and unit test * add symbolToLogits function set check * update clearState and add serial test * add SequenceBeamSearch to python layers * add createSequenceBeamSearch method to python api --- python/dllib/src/bigdl/dllib/nn/layer.py | 38 ++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 6002706ef2c..7cec5401119 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -4023,6 +4023,44 @@ def __init__(self, index) +class SequenceBeamSearch(Layer): + + ''' + Find the translated sequence with the highest probability. + + + :param vocab_size: size of tokens + :param beam_size: number of beams + :param alpha: defining the strength of length normalization + :param decode_length: maximum length to decoded sequence + :param eos_id: id of eos token, used to determine when a sequence has finished + :param num_hidden_layers: number of hidden layers + :param hidden_size: size of hidden layer + + + >>> sequenceBeamSearch = SequenceBeamSearch(4, 3, 0.0, 10, 1.0, 2, 5) + creating: createSequenceBeamSearch + ''' + + def __init__(self, + vocab_size, + beam_size, + alpha, + decode_length, + eos_id, + num_hidden_layers, + hidden_size, + bigdl_type="float"): + super(SequenceBeamSearch, self).__init__(None, bigdl_type, + vocab_size, + beam_size, + alpha, + decode_length, + eos_id, + num_hidden_layers, + hidden_size) + + class SoftMax(Layer): ''' From 40216ffc820d8157845cd5c76a9533595400040f Mon Sep 17 00:00:00 2001 From: majing921201 <1834475657@qq.com> Date: Sun, 7 Jul 2019 07:54:02 +0800 Subject: [PATCH 716/823] [WIP]Add beam search feature in transformer model (#2834) * add beam search feature * Update beam search feature and unit test * add symbolToLogits function set check * update clearState and add serial test * add SequenceBeamSearch to python layers * add createSequenceBeamSearch method to python api --- python/dllib/src/bigdl/dllib/nn/layer.py | 38 ++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 6002706ef2c..7cec5401119 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -4023,6 +4023,44 @@ def __init__(self, index) +class SequenceBeamSearch(Layer): + + ''' + Find the translated sequence with the highest probability. + + + :param vocab_size: size of tokens + :param beam_size: number of beams + :param alpha: defining the strength of length normalization + :param decode_length: maximum length to decoded sequence + :param eos_id: id of eos token, used to determine when a sequence has finished + :param num_hidden_layers: number of hidden layers + :param hidden_size: size of hidden layer + + + >>> sequenceBeamSearch = SequenceBeamSearch(4, 3, 0.0, 10, 1.0, 2, 5) + creating: createSequenceBeamSearch + ''' + + def __init__(self, + vocab_size, + beam_size, + alpha, + decode_length, + eos_id, + num_hidden_layers, + hidden_size, + bigdl_type="float"): + super(SequenceBeamSearch, self).__init__(None, bigdl_type, + vocab_size, + beam_size, + alpha, + decode_length, + eos_id, + num_hidden_layers, + hidden_size) + + class SoftMax(Layer): ''' From e649a61be16efbacea430d24369895e82d16af57 Mon Sep 17 00:00:00 2001 From: majing921201 <1834475657@qq.com> Date: Wed, 10 Jul 2019 19:01:56 +0800 Subject: [PATCH 717/823] update beam search feature for interface with transformer model (#2855) * update beam search for padding value and cache structure * update python API for beam search * add comments and update python layer * modify comments format * modify comments format --- python/dllib/src/bigdl/dllib/nn/layer.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 7cec5401119..e37ca90c80d 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -4034,11 +4034,12 @@ class SequenceBeamSearch(Layer): :param alpha: defining the strength of length normalization :param decode_length: maximum length to decoded sequence :param eos_id: id of eos token, used to determine when a sequence has finished + :param padding_value :param num_hidden_layers: number of hidden layers :param hidden_size: size of hidden layer - >>> sequenceBeamSearch = SequenceBeamSearch(4, 3, 0.0, 10, 1.0, 2, 5) + >>> sequenceBeamSearch = SequenceBeamSearch(4, 3, 0.0, 10, 2.0, 1.0, 2, 5) creating: createSequenceBeamSearch ''' @@ -4048,6 +4049,7 @@ def __init__(self, alpha, decode_length, eos_id, + padding_value, num_hidden_layers, hidden_size, bigdl_type="float"): @@ -4057,6 +4059,7 @@ def __init__(self, alpha, decode_length, eos_id, + padding_value, num_hidden_layers, hidden_size) From 492b59fc929cc09212174f672d8c935c38a0d6c5 Mon Sep 17 00:00:00 2001 From: majing921201 <1834475657@qq.com> Date: Wed, 10 Jul 2019 19:01:56 +0800 Subject: [PATCH 718/823] update beam search feature for interface with transformer model (#2855) * update beam search for padding value and cache structure * update python API for beam search * add comments and update python layer * modify comments format * modify comments format --- python/dllib/src/bigdl/dllib/nn/layer.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 7cec5401119..e37ca90c80d 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -4034,11 +4034,12 @@ class SequenceBeamSearch(Layer): :param alpha: defining the strength of length normalization :param decode_length: maximum length to decoded sequence :param eos_id: id of eos token, used to determine when a sequence has finished + :param padding_value :param num_hidden_layers: number of hidden layers :param hidden_size: size of hidden layer - >>> sequenceBeamSearch = SequenceBeamSearch(4, 3, 0.0, 10, 1.0, 2, 5) + >>> sequenceBeamSearch = SequenceBeamSearch(4, 3, 0.0, 10, 2.0, 1.0, 2, 5) creating: createSequenceBeamSearch ''' @@ -4048,6 +4049,7 @@ def __init__(self, alpha, decode_length, eos_id, + padding_value, num_hidden_layers, hidden_size, bigdl_type="float"): @@ -4057,6 +4059,7 @@ def __init__(self, alpha, decode_length, eos_id, + padding_value, num_hidden_layers, hidden_size) From 32648f38685644fe5bdb8fe1b3872700c5c31c5d Mon Sep 17 00:00:00 2001 From: Jerry Wu Date: Mon, 22 Jul 2019 18:07:35 +0800 Subject: [PATCH 719/823] flip version to 0.10.0 (#2869) --- .../dllib/src/bigdl/dllib/models/textclassifier/README.md | 8 ++++---- version.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/models/textclassifier/README.md b/python/dllib/src/bigdl/dllib/models/textclassifier/README.md index 17c619abcc0..a2f48553578 100644 --- a/python/dllib/src/bigdl/dllib/models/textclassifier/README.md +++ b/python/dllib/src/bigdl/dllib/models/textclassifier/README.md @@ -28,8 +28,8 @@ $ [/tmp/news20]$ tree . -L 1 - The example code would automatically download the data during the first run. ### Run via pip install -- [Install from pip](https://bigdl-project.github.io/0.9.0-SNAPSHOT/#PythonUserGuide/install-from-pip/) -- Optional: check [Run after pip install](https://bigdl-project.github.io/0.9.0-SNAPSHOT/#PythonUserGuide/run-from-pip/) +- [Install from pip](https://bigdl-project.github.io/0.10.0-SNAPSHOT/#PythonUserGuide/install-from-pip/) +- Optional: check [Run after pip install](https://bigdl-project.github.io/0.10.0-SNAPSHOT/#PythonUserGuide/run-from-pip/) - Run the following command locally ``` python ${BigDL_HOME}/pyspark/bigdl/models/textclassifier/textclassifier.py --max_epoch 3 --model cnn @@ -37,8 +37,8 @@ python ${BigDL_HOME}/pyspark/bigdl/models/textclassifier/textclassifier.py --max ``` ### Run via spark-submit -- [Install without pip](https://bigdl-project.github.io/0.9.0-SNAPSHOT/#PythonUserGuide/install-without-pip/) -- Optional: check [Run without pip](https://bigdl-project.github.io/0.9.0-SNAPSHOT/#PythonUserGuide/run-without-pip/) +- [Install without pip](https://bigdl-project.github.io/0.10.0-SNAPSHOT/#PythonUserGuide/install-without-pip/) +- Optional: check [Run without pip](https://bigdl-project.github.io/0.10.0-SNAPSHOT/#PythonUserGuide/run-without-pip/) - Run the following command ```{r, engine='sh'} PYTHONHASHSEED=0 diff --git a/version.py b/version.py index 86820d45817..0e836e0ccd2 100644 --- a/version.py +++ b/version.py @@ -14,4 +14,4 @@ # limitations under the License. # -__version__ = "0.9.0.dev0" +__version__ = "0.10.0.dev0" From 9bb4e3292251264e146bc9e13f986e00d5bfecc0 Mon Sep 17 00:00:00 2001 From: Jerry Wu Date: Mon, 22 Jul 2019 18:07:35 +0800 Subject: [PATCH 720/823] flip version to 0.10.0 (#2869) --- .../dllib/src/bigdl/dllib/models/textclassifier/README.md | 8 ++++---- version.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/models/textclassifier/README.md b/python/dllib/src/bigdl/dllib/models/textclassifier/README.md index 17c619abcc0..a2f48553578 100644 --- a/python/dllib/src/bigdl/dllib/models/textclassifier/README.md +++ b/python/dllib/src/bigdl/dllib/models/textclassifier/README.md @@ -28,8 +28,8 @@ $ [/tmp/news20]$ tree . -L 1 - The example code would automatically download the data during the first run. ### Run via pip install -- [Install from pip](https://bigdl-project.github.io/0.9.0-SNAPSHOT/#PythonUserGuide/install-from-pip/) -- Optional: check [Run after pip install](https://bigdl-project.github.io/0.9.0-SNAPSHOT/#PythonUserGuide/run-from-pip/) +- [Install from pip](https://bigdl-project.github.io/0.10.0-SNAPSHOT/#PythonUserGuide/install-from-pip/) +- Optional: check [Run after pip install](https://bigdl-project.github.io/0.10.0-SNAPSHOT/#PythonUserGuide/run-from-pip/) - Run the following command locally ``` python ${BigDL_HOME}/pyspark/bigdl/models/textclassifier/textclassifier.py --max_epoch 3 --model cnn @@ -37,8 +37,8 @@ python ${BigDL_HOME}/pyspark/bigdl/models/textclassifier/textclassifier.py --max ``` ### Run via spark-submit -- [Install without pip](https://bigdl-project.github.io/0.9.0-SNAPSHOT/#PythonUserGuide/install-without-pip/) -- Optional: check [Run without pip](https://bigdl-project.github.io/0.9.0-SNAPSHOT/#PythonUserGuide/run-without-pip/) +- [Install without pip](https://bigdl-project.github.io/0.10.0-SNAPSHOT/#PythonUserGuide/install-without-pip/) +- Optional: check [Run without pip](https://bigdl-project.github.io/0.10.0-SNAPSHOT/#PythonUserGuide/run-without-pip/) - Run the following command ```{r, engine='sh'} PYTHONHASHSEED=0 diff --git a/version.py b/version.py index 86820d45817..0e836e0ccd2 100644 --- a/version.py +++ b/version.py @@ -14,4 +14,4 @@ # limitations under the License. # -__version__ = "0.9.0.dev0" +__version__ = "0.10.0.dev0" From 04dca1f7fd1ffdab15840539eba8cfb3c7121424 Mon Sep 17 00:00:00 2001 From: Firecrackerxox Date: Fri, 16 Aug 2019 13:00:44 +0800 Subject: [PATCH 721/823] feat: RoiAlign Forward (#2874) --- python/dllib/src/bigdl/dllib/nn/layer.py | 44 ++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index e37ca90c80d..cd8cec75325 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -5612,6 +5612,50 @@ class Cropping3D(Layer): def __init__(self, dim1Crop, dim2Crop, dim3Crop, data_format="channel_first", bigdl_type="float"): super(Cropping3D, self).__init__(None, bigdl_type, dim1Crop, dim2Crop, dim3Crop, data_format) +class RoiAlign(Layer): + """ + Region of interest aligning (RoIAlign) for Mask-RCNN + + The RoIAlign uses average pooling on bilinear-interpolated sub-windows to convert + the features inside any valid region of interest into a small feature map with a + fixed spatial extent of pooledH * pooledW (e.g., 7 * 7). + + An RoI is a rectangular window into a conv feature map. + Each RoI is defined by a four-tuple (x1, y1, x2, y2) that specifies its + top-left corner (x1, y1) and its bottom-right corner (x2, y2). + + RoIAlign works by dividing the h * w RoI window into an pooledH * pooledW grid of + sub-windows of approximate size h/H * w/W. In each sub-window, compute exact values + of input features at four regularly sampled locations, and then do average pooling on + the values in each sub-window. + + Pooling is applied independently to each feature map channel + + :param spatial_scale: spatial scale + :param sampling_ratio: sampling ratio + :param pooled_h: spatial extent in height + :param pooled_w: spatial extent in width + + >>> import numpy as np + >>> input_data = np.random.rand(1,2,6,8) + >>> input_rois = np.array([0, 0, 7, 5, 6, 2, 7, 5, 3, 1, 6, 4, 3, 3, 3, 3],dtype='float').reshape(4,4) + >>> m = RoiAlign(1.0,3,2,2) + creating: createRoiAlign + >>> out = m.forward([input_data,input_rois]) + """ + + def __init__(self, + spatial_scale, + sampling_ratio, + pooled_h, + pooled_w, + bigdl_type="float"): + super(RoiAlign, self).__init__(None, bigdl_type, + spatial_scale, + sampling_ratio, + pooled_h, + pooled_w) + def _test(): import doctest from pyspark import SparkContext From 2a8812827868bad4d59491d966e6aed6678dfd05 Mon Sep 17 00:00:00 2001 From: Firecrackerxox Date: Fri, 16 Aug 2019 13:00:44 +0800 Subject: [PATCH 722/823] feat: RoiAlign Forward (#2874) --- python/dllib/src/bigdl/dllib/nn/layer.py | 44 ++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index e37ca90c80d..cd8cec75325 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -5612,6 +5612,50 @@ class Cropping3D(Layer): def __init__(self, dim1Crop, dim2Crop, dim3Crop, data_format="channel_first", bigdl_type="float"): super(Cropping3D, self).__init__(None, bigdl_type, dim1Crop, dim2Crop, dim3Crop, data_format) +class RoiAlign(Layer): + """ + Region of interest aligning (RoIAlign) for Mask-RCNN + + The RoIAlign uses average pooling on bilinear-interpolated sub-windows to convert + the features inside any valid region of interest into a small feature map with a + fixed spatial extent of pooledH * pooledW (e.g., 7 * 7). + + An RoI is a rectangular window into a conv feature map. + Each RoI is defined by a four-tuple (x1, y1, x2, y2) that specifies its + top-left corner (x1, y1) and its bottom-right corner (x2, y2). + + RoIAlign works by dividing the h * w RoI window into an pooledH * pooledW grid of + sub-windows of approximate size h/H * w/W. In each sub-window, compute exact values + of input features at four regularly sampled locations, and then do average pooling on + the values in each sub-window. + + Pooling is applied independently to each feature map channel + + :param spatial_scale: spatial scale + :param sampling_ratio: sampling ratio + :param pooled_h: spatial extent in height + :param pooled_w: spatial extent in width + + >>> import numpy as np + >>> input_data = np.random.rand(1,2,6,8) + >>> input_rois = np.array([0, 0, 7, 5, 6, 2, 7, 5, 3, 1, 6, 4, 3, 3, 3, 3],dtype='float').reshape(4,4) + >>> m = RoiAlign(1.0,3,2,2) + creating: createRoiAlign + >>> out = m.forward([input_data,input_rois]) + """ + + def __init__(self, + spatial_scale, + sampling_ratio, + pooled_h, + pooled_w, + bigdl_type="float"): + super(RoiAlign, self).__init__(None, bigdl_type, + spatial_scale, + sampling_ratio, + pooled_h, + pooled_w) + def _test(): import doctest from pyspark import SparkContext From 4cb7cf6bb4f3ac31b62d9fc55afd8227b0ca33c0 Mon Sep 17 00:00:00 2001 From: Firecrackerxox Date: Fri, 16 Aug 2019 13:00:44 +0800 Subject: [PATCH 723/823] feat: RoiAlign Forward (#2874) --- python/dllib/test/dev/diff.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/dllib/test/dev/diff.py b/python/dllib/test/dev/diff.py index 33c3e0d8439..d72c950eb5f 100755 --- a/python/dllib/test/dev/diff.py +++ b/python/dllib/test/dev/diff.py @@ -31,7 +31,7 @@ def extract_scala_class(class_path): exclude_key_words = set(["*", "abstract", "Const", "Fill", "Shape", "SplitAndSelect", "StrideSlice", "Scheduler", "StaticGraph", "DynamicGraph", "DynamicContainer", - "SplitHeads", "CombineHeads", "VectorProduct"]) # noqa + "SplitHeads", "CombineHeads", "VectorProduct", "Pooler"]) # noqa include_key_words = set(["Module", "Criterion", "Container", "Cell", "TensorNumeric"]) # noqa content = "\n".join([line for line in open(class_path).readlines() if all([key not in line for key in exclude_key_words])]) # noqa match = re.search(r"class ([\w]+)[^{]+", content) From ed118991dc4a882d8321c33abba61a693c59cd32 Mon Sep 17 00:00:00 2001 From: jenniew Date: Thu, 15 Aug 2019 22:59:05 -0700 Subject: [PATCH 724/823] Add set input output format API in Python (#2880) * add set input output format * add static graph check --- python/dllib/src/bigdl/dllib/nn/layer.py | 28 ++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index cd8cec75325..4f9fb95cc80 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -902,6 +902,34 @@ def save_graph_topology(self, log_path, bigdl_type="float"): callBigDlFunc(bigdl_type, "saveGraphTopology", self.value, log_path) return self + def set_input_formats(self, input_formats, bigdl_type="float"): + """ + set input formats for graph. + :param input_formats: list of input format numbers + :param bigdl_type: + :return: + """ + jname = callBigDlFunc(bigdl_type, + "getRealClassNameOfJValue", + self.value) + if jname.split(".")[-1] == "StaticGraph" : + callBigDlFunc(bigdl_type, "setInputFormats", self.value, input_formats) + return self + + def set_output_formats(self, output_formats, bigdl_type="float"): + """ + set output formats for graph. + :param output_formats: list of output format numbers + :param bigdl_type: + :return: + """ + jname = callBigDlFunc(bigdl_type, + "getRealClassNameOfJValue", + self.value) + if jname.split(".")[-1] == "StaticGraph": + callBigDlFunc(bigdl_type, "setOutputFormats", self.value, output_formats) + return self + class Attention(Layer): ''' From 99e5d7110a46a2c809cca10616c95ee4d57b9aca Mon Sep 17 00:00:00 2001 From: jenniew Date: Thu, 15 Aug 2019 22:59:05 -0700 Subject: [PATCH 725/823] Add set input output format API in Python (#2880) * add set input output format * add static graph check --- python/dllib/src/bigdl/dllib/nn/layer.py | 28 ++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index cd8cec75325..4f9fb95cc80 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -902,6 +902,34 @@ def save_graph_topology(self, log_path, bigdl_type="float"): callBigDlFunc(bigdl_type, "saveGraphTopology", self.value, log_path) return self + def set_input_formats(self, input_formats, bigdl_type="float"): + """ + set input formats for graph. + :param input_formats: list of input format numbers + :param bigdl_type: + :return: + """ + jname = callBigDlFunc(bigdl_type, + "getRealClassNameOfJValue", + self.value) + if jname.split(".")[-1] == "StaticGraph" : + callBigDlFunc(bigdl_type, "setInputFormats", self.value, input_formats) + return self + + def set_output_formats(self, output_formats, bigdl_type="float"): + """ + set output formats for graph. + :param output_formats: list of output format numbers + :param bigdl_type: + :return: + """ + jname = callBigDlFunc(bigdl_type, + "getRealClassNameOfJValue", + self.value) + if jname.split(".")[-1] == "StaticGraph": + callBigDlFunc(bigdl_type, "setOutputFormats", self.value, output_formats) + return self + class Attention(Layer): ''' From 199a4dd64042f04cde26e23b28da3a59a3580493 Mon Sep 17 00:00:00 2001 From: jenniew Date: Thu, 15 Aug 2019 22:59:05 -0700 Subject: [PATCH 726/823] Add set input output format API in Python (#2880) * add set input output format * add static graph check --- python/dllib/test/bigdl/test_simple_integration.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/python/dllib/test/bigdl/test_simple_integration.py b/python/dllib/test/bigdl/test_simple_integration.py index 9ecd2c3cbb3..18be330d15b 100644 --- a/python/dllib/test/bigdl/test_simple_integration.py +++ b/python/dllib/test/bigdl/test_simple_integration.py @@ -744,5 +744,15 @@ def tes_read_image_frame(self): count = image_frame.get_image().count() assert count == 1 + def test_set_input_output_format(self): + input1 = Input() + lstm1 = Recurrent().add(LSTM(128, 128))(input1) + fc1 = Linear(128, 10) + t1 = TimeDistributed(fc1)(lstm1) + model = Model(inputs=input1, outputs=t1) + model.set_input_formats([4]) + model.set_output_formats([27]) + + if __name__ == "__main__": pytest.main([__file__]) From 32bb92628f64126e10699f101ec61a79da86cafd Mon Sep 17 00:00:00 2001 From: Firecrackerxox Date: Wed, 21 Aug 2019 16:31:29 +0800 Subject: [PATCH 727/823] feat: Feature Pyramid Networks Forward (#2870) --- python/dllib/src/bigdl/dllib/nn/layer.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 4f9fb95cc80..07531cea666 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -5684,6 +5684,30 @@ def __init__(self, pooled_h, pooled_w) +class FPN(Layer): + """ + Feature Pyramid Network (FPN) for Mask-RCNN + + :param in_channels_list: number of channels of feature maps + :param out_channels: number of channels of FPN output + + >>> import numpy as np + >>> feature1 = np.random.rand(1,1,8,8) + >>> feature2 = np.random.rand(1,2,4,4) + >>> feature3 = np.random.rand(1,4,2,2) + >>> m = FPN([1,2,4],2) + creating: createFPN + >>> out = m.forward([feature1, feature2, feature3]) + """ + + def __init__(self, + in_channels_list, + out_channels, + bigdl_type="float"): + super(FPN, self).__init__(None, bigdl_type, + in_channels_list, + out_channels) + def _test(): import doctest from pyspark import SparkContext From 5b12f52b643d486fe524856339271dd972579c5c Mon Sep 17 00:00:00 2001 From: Firecrackerxox Date: Wed, 21 Aug 2019 16:31:29 +0800 Subject: [PATCH 728/823] feat: Feature Pyramid Networks Forward (#2870) --- python/dllib/src/bigdl/dllib/nn/layer.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 4f9fb95cc80..07531cea666 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -5684,6 +5684,30 @@ def __init__(self, pooled_h, pooled_w) +class FPN(Layer): + """ + Feature Pyramid Network (FPN) for Mask-RCNN + + :param in_channels_list: number of channels of feature maps + :param out_channels: number of channels of FPN output + + >>> import numpy as np + >>> feature1 = np.random.rand(1,1,8,8) + >>> feature2 = np.random.rand(1,2,4,4) + >>> feature3 = np.random.rand(1,4,2,2) + >>> m = FPN([1,2,4],2) + creating: createFPN + >>> out = m.forward([feature1, feature2, feature3]) + """ + + def __init__(self, + in_channels_list, + out_channels, + bigdl_type="float"): + super(FPN, self).__init__(None, bigdl_type, + in_channels_list, + out_channels) + def _test(): import doctest from pyspark import SparkContext From 232635fb24f2263ab87e70b38060bc07987bf38d Mon Sep 17 00:00:00 2001 From: LeicongLi Date: Mon, 26 Aug 2019 16:13:32 +0800 Subject: [PATCH 729/823] add gemm layer (#2882) * add gemm layer * add gemm layer * add gemm layer * add gemm layer * add gemm layer * add gemm layer * add gemm layer * add gemm layer * add gemm layer * add gemm layer * add gemm layer * add gemm layer * add gemm layer * add gemm layer * add transpose in gemm layer * add transpose in gemm layer * add transpose in gemm layer * add gemm layer * add gemm layer --- python/dllib/src/bigdl/dllib/nn/layer.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 07531cea666..0acefd6624d 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -1779,6 +1779,11 @@ def __init__(self, input_size=4, hidden_size=3, p=0.0, wRegularizer=None, uRegul super(LSTMPeephole, self).__init__(None, bigdl_type, input_size, hidden_size, p, wRegularizer, uRegularizer, bRegularizer) +class Gemm(Layer): + def __init__(self, alpha=1.0, beta=1.0, trans_a=False, trans_b=False, bigdl_type="float"): + super(Gemm, self).__init__(None, bigdl_type, alpha, beta, trans_a, trans_b) + + class GRU(Layer): ''' Gated Recurrent Units architecture. From 149e483daeb6f5a67ae6a4c08f54f4e9541be7b7 Mon Sep 17 00:00:00 2001 From: LeicongLi Date: Mon, 26 Aug 2019 16:13:32 +0800 Subject: [PATCH 730/823] add gemm layer (#2882) * add gemm layer * add gemm layer * add gemm layer * add gemm layer * add gemm layer * add gemm layer * add gemm layer * add gemm layer * add gemm layer * add gemm layer * add gemm layer * add gemm layer * add gemm layer * add gemm layer * add transpose in gemm layer * add transpose in gemm layer * add transpose in gemm layer * add gemm layer * add gemm layer --- python/dllib/src/bigdl/dllib/nn/layer.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 07531cea666..0acefd6624d 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -1779,6 +1779,11 @@ def __init__(self, input_size=4, hidden_size=3, p=0.0, wRegularizer=None, uRegul super(LSTMPeephole, self).__init__(None, bigdl_type, input_size, hidden_size, p, wRegularizer, uRegularizer, bRegularizer) +class Gemm(Layer): + def __init__(self, alpha=1.0, beta=1.0, trans_a=False, trans_b=False, bigdl_type="float"): + super(Gemm, self).__init__(None, bigdl_type, alpha, beta, trans_a, trans_b) + + class GRU(Layer): ''' Gated Recurrent Units architecture. From 9c6901f694d2c86c9e4b78a7aefc573f0c89643e Mon Sep 17 00:00:00 2001 From: LeicongLi Date: Mon, 26 Aug 2019 16:17:12 +0800 Subject: [PATCH 731/823] add Shape layer (#2885) * add shape layer * add shape layer * add shape layer * add shape layer * add shape layer * add shape layer * add shape layer * add shape layer * add shape layer * add shape layer * add shape layer * add shape layer * add shape layer * add shape layer * add shape layer * add shape layer * add shape layer * add shape layer * add shape layer * add shape layer * add shape layer * add shape layer --- python/dllib/src/bigdl/dllib/nn/layer.py | 1 + .../dllib/src/bigdl/dllib/nn/onnx/__init__.py | 15 +++++++++ python/dllib/src/bigdl/dllib/nn/onnx/layer.py | 33 +++++++++++++++++++ python/dllib/src/bigdl/dllib/utils/common.py | 5 ++- 4 files changed, 53 insertions(+), 1 deletion(-) create mode 100644 python/dllib/src/bigdl/dllib/nn/onnx/__init__.py create mode 100644 python/dllib/src/bigdl/dllib/nn/onnx/layer.py diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 0acefd6624d..355fed72f9e 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -5645,6 +5645,7 @@ class Cropping3D(Layer): def __init__(self, dim1Crop, dim2Crop, dim3Crop, data_format="channel_first", bigdl_type="float"): super(Cropping3D, self).__init__(None, bigdl_type, dim1Crop, dim2Crop, dim3Crop, data_format) + class RoiAlign(Layer): """ Region of interest aligning (RoIAlign) for Mask-RCNN diff --git a/python/dllib/src/bigdl/dllib/nn/onnx/__init__.py b/python/dllib/src/bigdl/dllib/nn/onnx/__init__.py new file mode 100644 index 00000000000..2151a805423 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/nn/onnx/__init__.py @@ -0,0 +1,15 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/python/dllib/src/bigdl/dllib/nn/onnx/layer.py b/python/dllib/src/bigdl/dllib/nn/onnx/layer.py new file mode 100644 index 00000000000..8d8eeabbb49 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/nn/onnx/layer.py @@ -0,0 +1,33 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import sys +from bigdl.nn.layer import Layer + +if sys.version >= '3': + long = int + unicode = str + + +class Shape(Layer): + """ + A layer which takes a tensor as input and outputs an 1D tensor containing the shape of the input. + + >>> shape = Shape() + creating: createShape + """ + def __init__(self, bigdl_type="float"): + super(Shape, self).__init__(None, bigdl_type) diff --git a/python/dllib/src/bigdl/dllib/utils/common.py b/python/dllib/src/bigdl/dllib/utils/common.py index dcef81812c8..30ef0bbc684 100644 --- a/python/dllib/src/bigdl/dllib/utils/common.py +++ b/python/dllib/src/bigdl/dllib/utils/common.py @@ -64,7 +64,10 @@ def __init__(self, bigdl_type, port=25333): class JavaCreator(SingletonMixin): - __creator_class=["com.intel.analytics.bigdl.python.api.PythonBigDLKeras"] + __creator_class=[ + "com.intel.analytics.bigdl.python.api.PythonBigDLKeras", + "com.intel.analytics.bigdl.python.api.PythonBigDLOnnx" + ] @classmethod def add_creator_class(cls, jinvoker): From 0b0258c5436fb437d288dcf988ae8c8fd707934a Mon Sep 17 00:00:00 2001 From: LeicongLi Date: Mon, 26 Aug 2019 16:17:12 +0800 Subject: [PATCH 732/823] add Shape layer (#2885) * add shape layer * add shape layer * add shape layer * add shape layer * add shape layer * add shape layer * add shape layer * add shape layer * add shape layer * add shape layer * add shape layer * add shape layer * add shape layer * add shape layer * add shape layer * add shape layer * add shape layer * add shape layer * add shape layer * add shape layer * add shape layer * add shape layer --- python/dllib/src/bigdl/dllib/nn/layer.py | 1 + .../dllib/src/bigdl/dllib/nn/onnx/__init__.py | 15 +++++++++ python/dllib/src/bigdl/dllib/nn/onnx/layer.py | 33 +++++++++++++++++++ python/dllib/src/bigdl/utils/common.py | 5 ++- 4 files changed, 53 insertions(+), 1 deletion(-) create mode 100644 python/dllib/src/bigdl/dllib/nn/onnx/__init__.py create mode 100644 python/dllib/src/bigdl/dllib/nn/onnx/layer.py diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 0acefd6624d..355fed72f9e 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -5645,6 +5645,7 @@ class Cropping3D(Layer): def __init__(self, dim1Crop, dim2Crop, dim3Crop, data_format="channel_first", bigdl_type="float"): super(Cropping3D, self).__init__(None, bigdl_type, dim1Crop, dim2Crop, dim3Crop, data_format) + class RoiAlign(Layer): """ Region of interest aligning (RoIAlign) for Mask-RCNN diff --git a/python/dllib/src/bigdl/dllib/nn/onnx/__init__.py b/python/dllib/src/bigdl/dllib/nn/onnx/__init__.py new file mode 100644 index 00000000000..2151a805423 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/nn/onnx/__init__.py @@ -0,0 +1,15 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/python/dllib/src/bigdl/dllib/nn/onnx/layer.py b/python/dllib/src/bigdl/dllib/nn/onnx/layer.py new file mode 100644 index 00000000000..8d8eeabbb49 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/nn/onnx/layer.py @@ -0,0 +1,33 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import sys +from bigdl.nn.layer import Layer + +if sys.version >= '3': + long = int + unicode = str + + +class Shape(Layer): + """ + A layer which takes a tensor as input and outputs an 1D tensor containing the shape of the input. + + >>> shape = Shape() + creating: createShape + """ + def __init__(self, bigdl_type="float"): + super(Shape, self).__init__(None, bigdl_type) diff --git a/python/dllib/src/bigdl/utils/common.py b/python/dllib/src/bigdl/utils/common.py index dcef81812c8..30ef0bbc684 100644 --- a/python/dllib/src/bigdl/utils/common.py +++ b/python/dllib/src/bigdl/utils/common.py @@ -64,7 +64,10 @@ def __init__(self, bigdl_type, port=25333): class JavaCreator(SingletonMixin): - __creator_class=["com.intel.analytics.bigdl.python.api.PythonBigDLKeras"] + __creator_class=[ + "com.intel.analytics.bigdl.python.api.PythonBigDLKeras", + "com.intel.analytics.bigdl.python.api.PythonBigDLOnnx" + ] @classmethod def add_creator_class(cls, jinvoker): From 9ce03aeef253711e15f23dd2051a8d332b029a64 Mon Sep 17 00:00:00 2001 From: zhangxiaoli73 <380761639@qq.com> Date: Fri, 30 Aug 2019 13:32:28 +0800 Subject: [PATCH 733/823] [New feature] Add maskhead (#2892) * support for maskhead --- python/dllib/test/dev/diff.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/dllib/test/dev/diff.py b/python/dllib/test/dev/diff.py index d72c950eb5f..23ade4d725d 100755 --- a/python/dllib/test/dev/diff.py +++ b/python/dllib/test/dev/diff.py @@ -31,7 +31,8 @@ def extract_scala_class(class_path): exclude_key_words = set(["*", "abstract", "Const", "Fill", "Shape", "SplitAndSelect", "StrideSlice", "Scheduler", "StaticGraph", "DynamicGraph", "DynamicContainer", - "SplitHeads", "CombineHeads", "VectorProduct", "Pooler"]) # noqa + "SplitHeads", "CombineHeads", "VectorProduct", "Pooler", + "MaskHead", "MaskPostProcessor"]) # noqa include_key_words = set(["Module", "Criterion", "Container", "Cell", "TensorNumeric"]) # noqa content = "\n".join([line for line in open(class_path).readlines() if all([key not in line for key in exclude_key_words])]) # noqa match = re.search(r"class ([\w]+)[^{]+", content) From cecd89db139d28675d88721550e54d45f9c435ed Mon Sep 17 00:00:00 2001 From: Xiao Date: Fri, 6 Sep 2019 11:04:42 +0800 Subject: [PATCH 734/823] modify predict/predictClass function (#2868) * predictClass output modification * predict/predictClass function modification in Beta Api * predict/predictClass function modification * predict/predictClass function modification * predictClass function modification * predictClass function modification * predictClass function modification * predictClass function modification * predictClass function modification * predictClass function modification * predictClass function modification * predictClass function modification * predictClass function modification * predictClass function modification * predictClass function modification * predictClass function modification * predictClass function modification * predictClass function modification * predictClass function modification * predictClass function modification * predictClass function modification * predictClass function modification * predictClass function modification * predictClass function modification * predictClass function modification * predictClass function modification * predictClass function modification * predictClass function modification * predictClass function modification * predictClass function modification * predictClass function modification * predictClass function modification * predictClass function modification * predictClass function modification * predictClass function modification --- python/dllib/test/bigdl/test_simple_integration.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/python/dllib/test/bigdl/test_simple_integration.py b/python/dllib/test/bigdl/test_simple_integration.py index 18be330d15b..a531fa9c0b0 100644 --- a/python/dllib/test/bigdl/test_simple_integration.py +++ b/python/dllib/test/bigdl/test_simple_integration.py @@ -22,6 +22,7 @@ from bigdl.util.common import _py2java from bigdl.nn.initialization_method import * from bigdl.dataset import movielens +from pyspark.rdd import RDD import numpy as np import tempfile import pytest @@ -544,9 +545,14 @@ def test_predict(self): assert_allclose(p_with_batch[i], ground_label[i], atol=1e-6, rtol=0) predict_class = model.predict_class(predict_data) - predict_labels = predict_class.take(6) - for i in range(0, total_length): - assert predict_labels[i] == 1 + if isinstance(predict_class, RDD): + for sample in predict_class.collect(): + predict_label = sample.label.to_ndarray() + assert np.argmax(predict_label) == 0 + else: + predict_labels = predict_class.take(6) + for i in range(0, total_length): + assert predict_labels[i] == 1 def test_predict_image(self): resource_path = os.path.join(os.path.split(__file__)[0], "resources") From 113fd619e3b8de2370a513bc2e19b8eaa41ecdc2 Mon Sep 17 00:00:00 2001 From: Xiao Date: Fri, 6 Sep 2019 11:04:42 +0800 Subject: [PATCH 735/823] modify predict/predictClass function (#2868) * predictClass output modification * predict/predictClass function modification in Beta Api * predict/predictClass function modification * predict/predictClass function modification * predictClass function modification * predictClass function modification * predictClass function modification * predictClass function modification * predictClass function modification * predictClass function modification * predictClass function modification * predictClass function modification * predictClass function modification * predictClass function modification * predictClass function modification * predictClass function modification * predictClass function modification * predictClass function modification * predictClass function modification * predictClass function modification * predictClass function modification * predictClass function modification * predictClass function modification * predictClass function modification * predictClass function modification * predictClass function modification * predictClass function modification * predictClass function modification * predictClass function modification * predictClass function modification * predictClass function modification * predictClass function modification * predictClass function modification * predictClass function modification * predictClass function modification --- python/dllib/src/bigdl/dllib/nn/layer.py | 1 - 1 file changed, 1 deletion(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 355fed72f9e..251766adabe 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -114,7 +114,6 @@ def get_py_name(jclass_name): model = jvalue_creator(jvalue, bigdl_type) return model - class Layer(JavaValue, SharedStaticUtils): """ Layer is the basic component of a neural network From 8102c020cc6ef5c1baa892904e5b41e1c385e261 Mon Sep 17 00:00:00 2001 From: Xiao Date: Fri, 6 Sep 2019 11:04:42 +0800 Subject: [PATCH 736/823] modify predict/predictClass function (#2868) * predictClass output modification * predict/predictClass function modification in Beta Api * predict/predictClass function modification * predict/predictClass function modification * predictClass function modification * predictClass function modification * predictClass function modification * predictClass function modification * predictClass function modification * predictClass function modification * predictClass function modification * predictClass function modification * predictClass function modification * predictClass function modification * predictClass function modification * predictClass function modification * predictClass function modification * predictClass function modification * predictClass function modification * predictClass function modification * predictClass function modification * predictClass function modification * predictClass function modification * predictClass function modification * predictClass function modification * predictClass function modification * predictClass function modification * predictClass function modification * predictClass function modification * predictClass function modification * predictClass function modification * predictClass function modification * predictClass function modification * predictClass function modification * predictClass function modification --- python/dllib/src/bigdl/dllib/nn/layer.py | 1 - 1 file changed, 1 deletion(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 355fed72f9e..251766adabe 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -114,7 +114,6 @@ def get_py_name(jclass_name): model = jvalue_creator(jvalue, bigdl_type) return model - class Layer(JavaValue, SharedStaticUtils): """ Layer is the basic component of a neural network From 1000d73ad70918ee42858d35048636773f6ebee1 Mon Sep 17 00:00:00 2001 From: zhangxiaoli73 <380761639@qq.com> Date: Mon, 9 Sep 2019 14:11:22 +0800 Subject: [PATCH 737/823] [New feature] Add Boxhead (#2894) * add boxhead * add SerialTest * meet pr comments --- python/dllib/test/dev/diff.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/dllib/test/dev/diff.py b/python/dllib/test/dev/diff.py index 23ade4d725d..e9e442dd4d7 100755 --- a/python/dllib/test/dev/diff.py +++ b/python/dllib/test/dev/diff.py @@ -32,7 +32,7 @@ def extract_scala_class(class_path): "SplitAndSelect", "StrideSlice", "Scheduler", "StaticGraph", "DynamicGraph", "DynamicContainer", "SplitHeads", "CombineHeads", "VectorProduct", "Pooler", - "MaskHead", "MaskPostProcessor"]) # noqa + "MaskHead", "MaskPostProcessor", "BoxHead", "BoxPostProcessor"]) # noqa include_key_words = set(["Module", "Criterion", "Container", "Cell", "TensorNumeric"]) # noqa content = "\n".join([line for line in open(class_path).readlines() if all([key not in line for key in exclude_key_words])]) # noqa match = re.search(r"class ([\w]+)[^{]+", content) From 30d632c66ee63af736f274bdfa25bf2b0832a2b8 Mon Sep 17 00:00:00 2001 From: Firecrackerxox Date: Tue, 10 Sep 2019 09:43:02 +0800 Subject: [PATCH 738/823] fix: Add TopBlocks to Feature Pyramid Networks (FPN) (#2899) --- python/dllib/src/bigdl/dllib/nn/layer.py | 25 +++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 251766adabe..2c9ac554184 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -5693,14 +5693,23 @@ class FPN(Layer): """ Feature Pyramid Network (FPN) for Mask-RCNN - :param in_channels_list: number of channels of feature maps - :param out_channels: number of channels of FPN output + :param in_channels_list: number of channels of feature maps + :param out_channels: number of channels of FPN output + :param top_blocks: top blocks option + extra operation to be performed on the smallest + resolution FPN output, whose result is appended + to the result list + 0 for null, + 1 for using max pooling on the last level + 2 for extra layers P6 and P7 in RetinaNet + :param in_channels_of_p6p7 number of input channels of P6 P7 + :param out_channels_of_p6p7 number of output channels of P6 P7 >>> import numpy as np >>> feature1 = np.random.rand(1,1,8,8) >>> feature2 = np.random.rand(1,2,4,4) >>> feature3 = np.random.rand(1,4,2,2) - >>> m = FPN([1,2,4],2) + >>> m = FPN([1,2,4],2,2,4,2) creating: createFPN >>> out = m.forward([feature1, feature2, feature3]) """ @@ -5708,10 +5717,16 @@ class FPN(Layer): def __init__(self, in_channels_list, out_channels, + top_blocks=0, + in_channels_of_p6p7=0, + out_channels_of_p6p7=0, bigdl_type="float"): super(FPN, self).__init__(None, bigdl_type, - in_channels_list, - out_channels) + in_channels_list, + out_channels, + top_blocks, + in_channels_of_p6p7, + out_channels_of_p6p7) def _test(): import doctest From 74d620132759d783c3c1bb2a7b5e87312766c5d7 Mon Sep 17 00:00:00 2001 From: Firecrackerxox Date: Tue, 10 Sep 2019 09:43:02 +0800 Subject: [PATCH 739/823] fix: Add TopBlocks to Feature Pyramid Networks (FPN) (#2899) --- python/dllib/src/bigdl/dllib/nn/layer.py | 25 +++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 251766adabe..2c9ac554184 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -5693,14 +5693,23 @@ class FPN(Layer): """ Feature Pyramid Network (FPN) for Mask-RCNN - :param in_channels_list: number of channels of feature maps - :param out_channels: number of channels of FPN output + :param in_channels_list: number of channels of feature maps + :param out_channels: number of channels of FPN output + :param top_blocks: top blocks option + extra operation to be performed on the smallest + resolution FPN output, whose result is appended + to the result list + 0 for null, + 1 for using max pooling on the last level + 2 for extra layers P6 and P7 in RetinaNet + :param in_channels_of_p6p7 number of input channels of P6 P7 + :param out_channels_of_p6p7 number of output channels of P6 P7 >>> import numpy as np >>> feature1 = np.random.rand(1,1,8,8) >>> feature2 = np.random.rand(1,2,4,4) >>> feature3 = np.random.rand(1,4,2,2) - >>> m = FPN([1,2,4],2) + >>> m = FPN([1,2,4],2,2,4,2) creating: createFPN >>> out = m.forward([feature1, feature2, feature3]) """ @@ -5708,10 +5717,16 @@ class FPN(Layer): def __init__(self, in_channels_list, out_channels, + top_blocks=0, + in_channels_of_p6p7=0, + out_channels_of_p6p7=0, bigdl_type="float"): super(FPN, self).__init__(None, bigdl_type, - in_channels_list, - out_channels) + in_channels_list, + out_channels, + top_blocks, + in_channels_of_p6p7, + out_channels_of_p6p7) def _test(): import doctest From e867e2b1a83b8323e3ee118af1f5ceba02e51480 Mon Sep 17 00:00:00 2001 From: Menooker Date: Tue, 10 Sep 2019 10:36:09 +0800 Subject: [PATCH 740/823] Add Mean Average Precision validation method (#2906) * add MeanAveragePrecision validation method * Add MAP basic code for object detection * update tests * bug fixes based on results of former MAP validation method * update documents * add python binding * typo fix, style change, change calculateAP to private * update comments --- .../dllib/src/bigdl/dllib/optim/optimizer.py | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/optim/optimizer.py b/python/dllib/src/bigdl/dllib/optim/optimizer.py index 15e33c860e6..c3ea9b137bb 100644 --- a/python/dllib/src/bigdl/dllib/optim/optimizer.py +++ b/python/dllib/src/bigdl/dllib/optim/optimizer.py @@ -68,6 +68,32 @@ class Top5Accuracy(JavaValue): def __init__(self, bigdl_type="float"): JavaValue.__init__(self, None, bigdl_type) +class MeanAveragePrecision(JavaValue): + """ + Calculate the Mean Average Precision for top-k confident predictions. + The algorithm follows VOC Challenge after 2007 + + >>> MAP = MeanAveragePrecision(10, 20) + creating: createMeanAveragePrecision + """ + def __init__(self, k, classes, bigdl_type="float"): + JavaValue.__init__(self, None, bigdl_type, k, classes) + +class MeanAveragePrecisionObjectDetection(JavaValue): + """ + Calculate the Mean Average Precision for Object Detection. + + >>> MAPObj = MeanAveragePrecisionObjectDetection(20) + creating: createMeanAveragePrecisionObjectDetection + """ + def __init__(self, classes, iou=0.5, use_voc2007=False, skip_class=-1, bigdl_type="float"): + """ + :param classes: the number of classes + :param iou: the IOU threshold + :param use_voc2007: use validation method before voc2010 (i.e. voc2007) + :param skip_class: skip calculation on a specific class (e.g. background) + """ + JavaValue.__init__(self, None, bigdl_type, classes, iou, use_voc2007, skip_class) class Loss(JavaValue): From 5b02fbb32642ad4782b58f1a6bebcf4b63856204 Mon Sep 17 00:00:00 2001 From: Menooker Date: Tue, 10 Sep 2019 10:36:09 +0800 Subject: [PATCH 741/823] Add Mean Average Precision validation method (#2906) * add MeanAveragePrecision validation method * Add MAP basic code for object detection * update tests * bug fixes based on results of former MAP validation method * update documents * add python binding * typo fix, style change, change calculateAP to private * update comments --- .../dllib/src/bigdl/dllib/optim/optimizer.py | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/optim/optimizer.py b/python/dllib/src/bigdl/dllib/optim/optimizer.py index 15e33c860e6..c3ea9b137bb 100644 --- a/python/dllib/src/bigdl/dllib/optim/optimizer.py +++ b/python/dllib/src/bigdl/dllib/optim/optimizer.py @@ -68,6 +68,32 @@ class Top5Accuracy(JavaValue): def __init__(self, bigdl_type="float"): JavaValue.__init__(self, None, bigdl_type) +class MeanAveragePrecision(JavaValue): + """ + Calculate the Mean Average Precision for top-k confident predictions. + The algorithm follows VOC Challenge after 2007 + + >>> MAP = MeanAveragePrecision(10, 20) + creating: createMeanAveragePrecision + """ + def __init__(self, k, classes, bigdl_type="float"): + JavaValue.__init__(self, None, bigdl_type, k, classes) + +class MeanAveragePrecisionObjectDetection(JavaValue): + """ + Calculate the Mean Average Precision for Object Detection. + + >>> MAPObj = MeanAveragePrecisionObjectDetection(20) + creating: createMeanAveragePrecisionObjectDetection + """ + def __init__(self, classes, iou=0.5, use_voc2007=False, skip_class=-1, bigdl_type="float"): + """ + :param classes: the number of classes + :param iou: the IOU threshold + :param use_voc2007: use validation method before voc2010 (i.e. voc2007) + :param skip_class: skip calculation on a specific class (e.g. background) + """ + JavaValue.__init__(self, None, bigdl_type, classes, iou, use_voc2007, skip_class) class Loss(JavaValue): From 1c98a686bc50ccae5aa5499e4fc69f4d6872ff17 Mon Sep 17 00:00:00 2001 From: Firecrackerxox Date: Thu, 12 Sep 2019 13:26:47 +0800 Subject: [PATCH 742/823] python api nested list input and pooler python api (#2900) --- python/dllib/test/dev/diff.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/dllib/test/dev/diff.py b/python/dllib/test/dev/diff.py index e9e442dd4d7..b2a0b141fc9 100755 --- a/python/dllib/test/dev/diff.py +++ b/python/dllib/test/dev/diff.py @@ -31,7 +31,7 @@ def extract_scala_class(class_path): exclude_key_words = set(["*", "abstract", "Const", "Fill", "Shape", "SplitAndSelect", "StrideSlice", "Scheduler", "StaticGraph", "DynamicGraph", "DynamicContainer", - "SplitHeads", "CombineHeads", "VectorProduct", "Pooler", + "SplitHeads", "CombineHeads", "VectorProduct", "MaskHead", "MaskPostProcessor", "BoxHead", "BoxPostProcessor"]) # noqa include_key_words = set(["Module", "Criterion", "Container", "Cell", "TensorNumeric"]) # noqa content = "\n".join([line for line in open(class_path).readlines() if all([key not in line for key in exclude_key_words])]) # noqa From ea91eeb25ecd4d24269a63143b15a7ea6cd9c080 Mon Sep 17 00:00:00 2001 From: Firecrackerxox Date: Thu, 12 Sep 2019 13:26:47 +0800 Subject: [PATCH 743/823] python api nested list input and pooler python api (#2900) --- python/dllib/src/bigdl/dllib/nn/layer.py | 40 +++++++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 2c9ac554184..34ce2b0e6e3 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -226,10 +226,19 @@ def to_jtensor(i): return i else: raise Exception("Error unknown input type %s" % type(i)) + + def check_list(input): + if type(input) is list: + if len(input) == 0: + raise Exception('Error when checking: empty input') + return list(map(lambda i: check_list(i), input)) + else: + return to_jtensor(input) + if type(input) is list: if len(input) == 0: raise Exception('Error when checking: empty input') - return list(map(lambda i: to_jtensor(i), input)), True + return list(map(lambda i: check_list(i), input)), True else: return [to_jtensor(input)], False @@ -5689,6 +5698,35 @@ def __init__(self, pooled_h, pooled_w) +class Pooler(Layer): + """ + Pooler selects the feature map which matches the size of RoI for RoIAlign + + :param resolution: the resolution of pooled feature maps. Height equals width. + :param scales: spatial scales of each feature map + :param sampling_ratio: sampling ratio + + >>> import numpy as np + >>> feature0 = np.random.rand(1,2,2,2) + >>> feature1 = np.random.rand(1,2,4,4) + >>> feature2 = np.random.rand(1,2,8,8) + >>> features = [feature0, feature1, feature2] + >>> input_rois = np.array([0, 0, 3, 3, 2, 2, 50, 50, 50, 50, 500, 500],dtype='float').reshape(3,4) + >>> m = Pooler(2,[1.0, 0.5, 0.25],2) + creating: createPooler + >>> out = m.forward([features,input_rois]) + """ + + def __init__(self, + resolution, + scales, + sampling_ratio, + bigdl_type="float"): + super(Pooler, self).__init__(None, bigdl_type, + resolution, + scales, + sampling_ratio) + class FPN(Layer): """ Feature Pyramid Network (FPN) for Mask-RCNN From 70fe87f4e2c3336bc0e909a5e93259ee34f00a73 Mon Sep 17 00:00:00 2001 From: Firecrackerxox Date: Thu, 12 Sep 2019 13:26:47 +0800 Subject: [PATCH 744/823] python api nested list input and pooler python api (#2900) --- python/dllib/src/bigdl/dllib/nn/layer.py | 40 +++++++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 2c9ac554184..34ce2b0e6e3 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -226,10 +226,19 @@ def to_jtensor(i): return i else: raise Exception("Error unknown input type %s" % type(i)) + + def check_list(input): + if type(input) is list: + if len(input) == 0: + raise Exception('Error when checking: empty input') + return list(map(lambda i: check_list(i), input)) + else: + return to_jtensor(input) + if type(input) is list: if len(input) == 0: raise Exception('Error when checking: empty input') - return list(map(lambda i: to_jtensor(i), input)), True + return list(map(lambda i: check_list(i), input)), True else: return [to_jtensor(input)], False @@ -5689,6 +5698,35 @@ def __init__(self, pooled_h, pooled_w) +class Pooler(Layer): + """ + Pooler selects the feature map which matches the size of RoI for RoIAlign + + :param resolution: the resolution of pooled feature maps. Height equals width. + :param scales: spatial scales of each feature map + :param sampling_ratio: sampling ratio + + >>> import numpy as np + >>> feature0 = np.random.rand(1,2,2,2) + >>> feature1 = np.random.rand(1,2,4,4) + >>> feature2 = np.random.rand(1,2,8,8) + >>> features = [feature0, feature1, feature2] + >>> input_rois = np.array([0, 0, 3, 3, 2, 2, 50, 50, 50, 50, 500, 500],dtype='float').reshape(3,4) + >>> m = Pooler(2,[1.0, 0.5, 0.25],2) + creating: createPooler + >>> out = m.forward([features,input_rois]) + """ + + def __init__(self, + resolution, + scales, + sampling_ratio, + bigdl_type="float"): + super(Pooler, self).__init__(None, bigdl_type, + resolution, + scales, + sampling_ratio) + class FPN(Layer): """ Feature Pyramid Network (FPN) for Mask-RCNN From e05e315234f6ea3b6d942bd5c01a568e6a7cf277 Mon Sep 17 00:00:00 2001 From: zhangxiaoli73 <380761639@qq.com> Date: Tue, 17 Sep 2019 12:51:28 +0800 Subject: [PATCH 745/823] [New feature] Add region proposal (#2896) * add Regionproposal --- python/dllib/test/dev/diff.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/dllib/test/dev/diff.py b/python/dllib/test/dev/diff.py index b2a0b141fc9..21e811af3fb 100755 --- a/python/dllib/test/dev/diff.py +++ b/python/dllib/test/dev/diff.py @@ -32,7 +32,8 @@ def extract_scala_class(class_path): "SplitAndSelect", "StrideSlice", "Scheduler", "StaticGraph", "DynamicGraph", "DynamicContainer", "SplitHeads", "CombineHeads", "VectorProduct", - "MaskHead", "MaskPostProcessor", "BoxHead", "BoxPostProcessor"]) # noqa + "MaskHead", "MaskPostProcessor", "BoxHead", "BoxPostProcessor", + "RegionRroposal", "ProposalPostProcessor"]) # noqa include_key_words = set(["Module", "Criterion", "Container", "Cell", "TensorNumeric"]) # noqa content = "\n".join([line for line in open(class_path).readlines() if all([key not in line for key in exclude_key_words])]) # noqa match = re.search(r"class ([\w]+)[^{]+", content) From 4bc1f08ed890f160ff3854545e2f84d7e373181e Mon Sep 17 00:00:00 2001 From: LeicongLi Date: Tue, 24 Sep 2019 17:08:36 +0800 Subject: [PATCH 746/823] Add Onnx Supported Layers (#2902) * remove duplicated layers --- python/dllib/src/bigdl/dllib/nn/onnx/layer.py | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/nn/onnx/layer.py b/python/dllib/src/bigdl/dllib/nn/onnx/layer.py index 8d8eeabbb49..80cb5706e11 100644 --- a/python/dllib/src/bigdl/dllib/nn/onnx/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/onnx/layer.py @@ -15,13 +15,39 @@ # import sys +import numpy as np from bigdl.nn.layer import Layer +from bigdl.util.common import JTensor if sys.version >= '3': long = int unicode = str + +class Gemm(Layer): + """ + General Matrix multiplication: https://en.wikipedia.org/wiki/Basic_Linear_Algebra_Subprograms#Level_3 + + A' = transpose(A) if transA else A + B' = transpose(B) if transB else B + + Compute Y = alpha * A' * B' + beta * C, where input tensor A has shape (M, K) or (K, M), + input tensor B has shape (K, N) or (N, K), input tensor C is broadcastable to shape (M, N), + and output tensor Y has shape (M, N). A will be transposed before doing the computation if + attribute transA is non-zero, same for B and transB. + + >>> matrix_b = np.random.random([2, 2]) + >>> matrix_c = np.random.random([2, 2]) + >>> gemm = Gemm(matrix_b=matrix_b, matrix_c=matrix_c) + creating: createGemm + """ + def __init__(self, matrix_b, matrix_c, alpha=float(1.0), beta=float(1.0), trans_a=0, trans_b=0, + bigdl_type="float"): + super(Gemm, self).__init__(None, bigdl_type, alpha, beta, trans_a, trans_b, + JTensor.from_ndarray(matrix_b), JTensor.from_ndarray(matrix_c)) + + class Shape(Layer): """ A layer which takes a tensor as input and outputs an 1D tensor containing the shape of the input. @@ -31,3 +57,4 @@ class Shape(Layer): """ def __init__(self, bigdl_type="float"): super(Shape, self).__init__(None, bigdl_type) + From 4ccbffb6ccfd7f1b452507974fc036b540f9568b Mon Sep 17 00:00:00 2001 From: LeicongLi Date: Tue, 24 Sep 2019 17:08:36 +0800 Subject: [PATCH 747/823] Add Onnx Supported Layers (#2902) * remove duplicated layers --- python/dllib/src/bigdl/dllib/nn/onnx/layer.py | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/nn/onnx/layer.py b/python/dllib/src/bigdl/dllib/nn/onnx/layer.py index 8d8eeabbb49..80cb5706e11 100644 --- a/python/dllib/src/bigdl/dllib/nn/onnx/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/onnx/layer.py @@ -15,13 +15,39 @@ # import sys +import numpy as np from bigdl.nn.layer import Layer +from bigdl.util.common import JTensor if sys.version >= '3': long = int unicode = str + +class Gemm(Layer): + """ + General Matrix multiplication: https://en.wikipedia.org/wiki/Basic_Linear_Algebra_Subprograms#Level_3 + + A' = transpose(A) if transA else A + B' = transpose(B) if transB else B + + Compute Y = alpha * A' * B' + beta * C, where input tensor A has shape (M, K) or (K, M), + input tensor B has shape (K, N) or (N, K), input tensor C is broadcastable to shape (M, N), + and output tensor Y has shape (M, N). A will be transposed before doing the computation if + attribute transA is non-zero, same for B and transB. + + >>> matrix_b = np.random.random([2, 2]) + >>> matrix_c = np.random.random([2, 2]) + >>> gemm = Gemm(matrix_b=matrix_b, matrix_c=matrix_c) + creating: createGemm + """ + def __init__(self, matrix_b, matrix_c, alpha=float(1.0), beta=float(1.0), trans_a=0, trans_b=0, + bigdl_type="float"): + super(Gemm, self).__init__(None, bigdl_type, alpha, beta, trans_a, trans_b, + JTensor.from_ndarray(matrix_b), JTensor.from_ndarray(matrix_c)) + + class Shape(Layer): """ A layer which takes a tensor as input and outputs an 1D tensor containing the shape of the input. @@ -31,3 +57,4 @@ class Shape(Layer): """ def __init__(self, bigdl_type="float"): super(Shape, self).__init__(None, bigdl_type) + From 1c05e799a91d34e7a883658400bfe1a8567fed27 Mon Sep 17 00:00:00 2001 From: Xiao Date: Mon, 30 Sep 2019 10:03:07 +0800 Subject: [PATCH 748/823] Onnx support: modify unsqueeze function (#2910) * modeify unsqueeze function * modeify unsqueeze function * modeify unsqueeze function * modeify unsqueeze function * modeify unsqueeze function * modeify unsqueeze function * modeify unsqueeze function * modeify unsqueeze function * modeify unsqueeze function * modeify unsqueeze function * modeify unsqueeze function * modeify unsqueeze function * modeify unsqueeze function * modeify unsqueeze function * modeify unsqueeze function * modeify unsqueeze function * modeify unsqueeze function * modeify unsqueeze function * modeify unsqueeze function * modeify unsqueeze function * modeify unsqueeze function * modeify unsqueeze function * modeify unsqueeze function * modeify unsqueeze function * modeify unsqueeze function * modeify unsqueeze function * modeify unsqueeze function * modeify unsqueeze function * modeify unsqueeze function --- python/dllib/src/bigdl/dllib/nn/layer.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 34ce2b0e6e3..32ab04eb1bc 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -4899,14 +4899,14 @@ class Unsqueeze(Layer): creating: createUnsqueeze ''' - def __init__(self, - pos, - num_input_dims=INTMIN, - bigdl_type="float"): - super(Unsqueeze, self).__init__(None, bigdl_type, - pos, - num_input_dims) - + def __init__(self, pos, num_input_dims=INTMIN, bigdl_type="float"): + if isinstance(pos, int): + posList=[pos] + super(Unsqueeze, self).__init__(None, bigdl_type, to_list(posList), num_input_dims) + elif isinstance(pos, list): + super(Unsqueeze, self).__init__(None, bigdl_type, to_list(pos), num_input_dims) + else: + raise Exception("Error invalid input") class Reshape(Layer): ''' From c807f3e1e319e7a43f11322c2c986f73ab7ae06a Mon Sep 17 00:00:00 2001 From: Xiao Date: Mon, 30 Sep 2019 10:03:07 +0800 Subject: [PATCH 749/823] Onnx support: modify unsqueeze function (#2910) * modeify unsqueeze function * modeify unsqueeze function * modeify unsqueeze function * modeify unsqueeze function * modeify unsqueeze function * modeify unsqueeze function * modeify unsqueeze function * modeify unsqueeze function * modeify unsqueeze function * modeify unsqueeze function * modeify unsqueeze function * modeify unsqueeze function * modeify unsqueeze function * modeify unsqueeze function * modeify unsqueeze function * modeify unsqueeze function * modeify unsqueeze function * modeify unsqueeze function * modeify unsqueeze function * modeify unsqueeze function * modeify unsqueeze function * modeify unsqueeze function * modeify unsqueeze function * modeify unsqueeze function * modeify unsqueeze function * modeify unsqueeze function * modeify unsqueeze function * modeify unsqueeze function * modeify unsqueeze function --- python/dllib/src/bigdl/dllib/nn/layer.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 34ce2b0e6e3..32ab04eb1bc 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -4899,14 +4899,14 @@ class Unsqueeze(Layer): creating: createUnsqueeze ''' - def __init__(self, - pos, - num_input_dims=INTMIN, - bigdl_type="float"): - super(Unsqueeze, self).__init__(None, bigdl_type, - pos, - num_input_dims) - + def __init__(self, pos, num_input_dims=INTMIN, bigdl_type="float"): + if isinstance(pos, int): + posList=[pos] + super(Unsqueeze, self).__init__(None, bigdl_type, to_list(posList), num_input_dims) + elif isinstance(pos, list): + super(Unsqueeze, self).__init__(None, bigdl_type, to_list(pos), num_input_dims) + else: + raise Exception("Error invalid input") class Reshape(Layer): ''' From c6466d231a9af31ceaf04963422099d89f0d530e Mon Sep 17 00:00:00 2001 From: zhangxiaoli73 <380761639@qq.com> Date: Wed, 16 Oct 2019 13:58:32 +0800 Subject: [PATCH 750/823] support batch for regionproposal (#2928) * support batch for regionproposal --- python/dllib/test/dev/diff.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/dllib/test/dev/diff.py b/python/dllib/test/dev/diff.py index 21e811af3fb..d9ad99a1305 100755 --- a/python/dllib/test/dev/diff.py +++ b/python/dllib/test/dev/diff.py @@ -33,7 +33,7 @@ def extract_scala_class(class_path): "StaticGraph", "DynamicGraph", "DynamicContainer", "SplitHeads", "CombineHeads", "VectorProduct", "MaskHead", "MaskPostProcessor", "BoxHead", "BoxPostProcessor", - "RegionRroposal", "ProposalPostProcessor"]) # noqa + "RegionProposal", "ProposalPostProcessor"]) # noqa include_key_words = set(["Module", "Criterion", "Container", "Cell", "TensorNumeric"]) # noqa content = "\n".join([line for line in open(class_path).readlines() if all([key not in line for key in exclude_key_words])]) # noqa match = re.search(r"class ([\w]+)[^{]+", content) From 6ac7b61625208a872f429c54caf3c029a7b74955 Mon Sep 17 00:00:00 2001 From: Xiao Date: Fri, 18 Oct 2019 15:02:34 +0800 Subject: [PATCH 751/823] Onnx support: add pos parameter to softmax (#2933) * add pos parameter to softmax * add pos parameter to softmax * add pos parameter to softmax * fix review problem * fix review problem --- python/dllib/src/bigdl/dllib/nn/layer.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 32ab04eb1bc..5de838a3493 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -4119,8 +4119,9 @@ class SoftMax(Layer): ''' def __init__(self, + pos=1, bigdl_type="float"): - super(SoftMax, self).__init__(None, bigdl_type) + super(SoftMax, self).__init__(None, bigdl_type, pos) class SoftMin(Layer): From 96b73938ffb4992ff54210cc7f800f1d6365116a Mon Sep 17 00:00:00 2001 From: Xiao Date: Fri, 18 Oct 2019 15:02:34 +0800 Subject: [PATCH 752/823] Onnx support: add pos parameter to softmax (#2933) * add pos parameter to softmax * add pos parameter to softmax * add pos parameter to softmax * fix review problem * fix review problem --- python/dllib/src/bigdl/dllib/nn/layer.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 32ab04eb1bc..5de838a3493 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -4119,8 +4119,9 @@ class SoftMax(Layer): ''' def __init__(self, + pos=1, bigdl_type="float"): - super(SoftMax, self).__init__(None, bigdl_type) + super(SoftMax, self).__init__(None, bigdl_type, pos) class SoftMin(Layer): From 1ed8bd0eae80095f7719b78667ef178b258fb9ac Mon Sep 17 00:00:00 2001 From: LeicongLi Date: Tue, 22 Oct 2019 17:03:54 +0800 Subject: [PATCH 753/823] ONNX Support (#2918) * onnx dev * add onnx loader * clean up --- .../dllib/src/bigdl/dllib/contrib/__init__.py | 0 .../src/bigdl/dllib/contrib/onnx/__init__.py | 17 ++ .../dllib/contrib/onnx/converter_utils.py | 70 +++++ .../bigdl/dllib/contrib/onnx/onnx_loader.py | 111 ++++++++ .../bigdl/dllib/contrib/onnx/ops_converter.py | 256 ++++++++++++++++++ .../bigdl/dllib/contrib/onnx/ops_mapping.py | 133 +++++++++ python/dllib/src/bigdl/dllib/nn/onnx/layer.py | 29 ++ 7 files changed, 616 insertions(+) create mode 100644 python/dllib/src/bigdl/dllib/contrib/__init__.py create mode 100644 python/dllib/src/bigdl/dllib/contrib/onnx/__init__.py create mode 100644 python/dllib/src/bigdl/dllib/contrib/onnx/converter_utils.py create mode 100644 python/dllib/src/bigdl/dllib/contrib/onnx/onnx_loader.py create mode 100644 python/dllib/src/bigdl/dllib/contrib/onnx/ops_converter.py create mode 100644 python/dllib/src/bigdl/dllib/contrib/onnx/ops_mapping.py diff --git a/python/dllib/src/bigdl/dllib/contrib/__init__.py b/python/dllib/src/bigdl/dllib/contrib/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/python/dllib/src/bigdl/dllib/contrib/onnx/__init__.py b/python/dllib/src/bigdl/dllib/contrib/onnx/__init__.py new file mode 100644 index 00000000000..1c529fd7ad6 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/contrib/onnx/__init__.py @@ -0,0 +1,17 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from .onnx_loader import load diff --git a/python/dllib/src/bigdl/dllib/contrib/onnx/converter_utils.py b/python/dllib/src/bigdl/dllib/contrib/onnx/converter_utils.py new file mode 100644 index 00000000000..e7d1417200a --- /dev/null +++ b/python/dllib/src/bigdl/dllib/contrib/onnx/converter_utils.py @@ -0,0 +1,70 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import math +import numpy as np + + +def calc_output_shape(input, kernel, padding=0, stride=1, dilation=1, ceil_mode=False): + def dilated_kernel_size(kernel, dilation): + return kernel + (kernel - 1) * (dilation - 1) + rounding = math.ceil if ceil_mode else math.floor + out = (input + 2 * padding - dilated_kernel_size(kernel, dilation)) / stride + 1 + out = int(rounding(out)) + return out + + +def parse_node_attr(node_proto): + attrs = {} + attr_proto = node_proto.attribute + + for attr in attr_proto: + for field in ['f', 'i', 's']: + if attr.HasField(field): + attrs[attr.name] = getattr(attr, field) + + # Needed for supporting python version > 3.5 + if isinstance(attrs[attr.name], bytes): + attrs[attr.name] = attrs[attr.name].decode(encoding='utf-8') + + for field in ['floats', 'ints', 'strings']: + if list(getattr(attr, field)): + assert attr.name not in attrs, "Only one type of attr is allowed" + attrs[attr.name] = tuple(getattr(attr, field)) + + for field in ['t', 'g']: + if attr.HasField(field): + attrs[attr.name] = getattr(attr, field) + for field in ['tensors', 'graphs']: + if list(getattr(attr, field)): + raise NotImplementedError() + if attr.name not in attrs: + raise ValueError("Cannot parse attribute: \n{}\n.".format(attr)) + + return attrs + + +def parse_tensor_data(tensor_proto): + try: + from onnx.numpy_helper import to_array + except ImportError: + raise ImportError("Onnx and protobuf need to be installed.") + if len(tuple(tensor_proto.dims)) > 0: + np_array = to_array(tensor_proto).reshape(tuple(tensor_proto.dims)) + else: + # If it is a scalar tensor + np_array = np.array([to_array(tensor_proto)]) + return np_array diff --git a/python/dllib/src/bigdl/dllib/contrib/onnx/onnx_loader.py b/python/dllib/src/bigdl/dllib/contrib/onnx/onnx_loader.py new file mode 100644 index 00000000000..6e537c10c3e --- /dev/null +++ b/python/dllib/src/bigdl/dllib/contrib/onnx/onnx_loader.py @@ -0,0 +1,111 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import onnx +from bigdl.nn.onnx.layer import * +from bigdl.nn.layer import Identity, Model +from .ops_mapping import _convert_map as convert_map +from .converter_utils import parse_node_attr, parse_tensor_data + + +class OnnxLoader(object): + + def load_model(self, file_path): + model_proto = onnx.load_model(file_path) + # self._ir_version = model_proto.ir_version + # self._opset_import = model_proto.opset_import + # self._producer_name = model_proto.producer_name + # self._producer_version = model_proto.producer_version + # self._domain = model_proto.domain + # self._model_version = model_proto.model_version + # self._doc_string = model_proto.doc_string + graph_proto = model_proto.graph + return self.load_graph(graph_proto) + + def load_graph(self, graph_proto): + if not graph_proto: + raise ValueError("Graph proto is required") + + input_nodes = list() + output_nodes = list() + tensor_map = dict() + initialized_tensors = set() + module_map = dict() + root_nodes = list() + dummy_root = Identity()() + + for tensor in graph_proto.initializer: + if not tensor.name.strip(): + raise ValueError("Tensor's name is required") + initialized_tensors.add(tensor.name) + tensor_data = parse_tensor_data(tensor) + tensor_map[tensor.name] = (tensor_data, tensor_data.shape) + + for gin in graph_proto.input: + if gin.name not in initialized_tensors: + input_nodes.append(gin.name) + shape = tuple([dim.dim_value for dim in gin.type.tensor_type.shape.dim]) + module_map[gin.name] = Identity()(dummy_root) + tensor_map[gin.name] = (None, shape) + + for gout in graph_proto.output: + if gout.name not in initialized_tensors: + output_nodes.append(gout.name) + + for node in graph_proto.node: + name = node.name.strip() + op_type = node.op_type + inputs = [tensor_map[n] for n in node.input] + outputs = node.output + prev_modules = [module_map[n] for n in node.input if n not in initialized_tensors] + attrs = parse_node_attr(node) + + if len(prev_modules) == 0: + root_nodes.append((name, op_type)) + prev_modules = [dummy_root] + + bigdl_module, outputs_shape = self._make_module_from_onnx_node(op_type, inputs, prev_modules, attrs, outputs) + + assert len(outputs) == len(outputs_shape) + + for out, out_shape in zip(outputs, outputs_shape): + module_map[out] = bigdl_module + tensor_map[out] = (None, out_shape) + + in_modules = [module_map[m] for m in input_nodes] + out_modules = [module_map[m] for m in output_nodes] + model = Model([dummy_root], out_modules) + + return model + + def _make_module_from_onnx_node(self, op_type, inputs, prev_modules, attrs, outputs): + module = None + out_shapes = [] + if op_type in convert_map: + module, out_shapes = convert_map[op_type](inputs, prev_modules, attrs, outputs) + else: + raise NotImplemented(op_type) + return module, out_shapes + + +def load(model_path): + loader = OnnxLoader() + return loader.load_model(model_path) + + +def load_model_proto(model_proto): + loader = OnnxLoader() + return loader.load_graph(model_proto.graph) \ No newline at end of file diff --git a/python/dllib/src/bigdl/dllib/contrib/onnx/ops_converter.py b/python/dllib/src/bigdl/dllib/contrib/onnx/ops_converter.py new file mode 100644 index 00000000000..4934f64ce93 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/contrib/onnx/ops_converter.py @@ -0,0 +1,256 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from bigdl.nn.layer import SpatialAveragePooling, SpatialBatchNormalization +from bigdl.nn.layer import SpatialConvolution, SpatialMaxPooling, JoinTable +from bigdl.nn.layer import ReLU, SoftMax, CAddTable, Unsqueeze +from bigdl.nn.onnx.layer import Constant, Gather, Gemm, Shape, Reshape +from .converter_utils import * + + +def average_pool(inputs, prev_modules, attrs, outputs): + # extract attributes + auto_pad = attrs.get('auto_pad', 'NOTSET') + ceil_mode = True if attrs.get('ceil_mode', 0) == 1 else False + count_include_pad = True if attrs.get('count_include_pad', 0) == 1 else False + kernel_width, kernel_height = map(int, attrs.get('kernel_shape', (1, 1))[:2]) + stride_width, stride_height = map(int, attrs.get('strides', (1, 1))[:2]) + padding_width, padding_height = map(int, attrs.get('pads', (0, 0))[:2]) + # extract inputs + _, data_tensor_shape = inputs[0] + # calc output tensor shape + input_height, input_width = data_tensor_shape[-2:] + output_height = calc_output_shape(input_height, kernel_height, + padding = padding_height, stride = stride_height, ceil_mode = ceil_mode) + output_width = calc_output_shape(input_width, kernel_width, + padding = padding_width, stride = stride_width, ceil_mode = ceil_mode) + out_tensor_shape = list(data_tensor_shape) + out_tensor_shape[-2] = output_height + out_tensor_shape[-1] = output_width + out_tensor_shape = tuple(out_tensor_shape) + # create module node + module = SpatialAveragePooling(kw=kernel_width, kh=kernel_height, + dw=stride_width, dh=stride_height, + pad_w=padding_width, pad_h=padding_height, + ceil_mode=ceil_mode, count_include_pad=count_include_pad + )(prev_modules) + return module, [out_tensor_shape] + + +def batch_norm(inputs, prev_modules, attrs, outputs): + # extract attributes + epsilon = float(attrs.get('epsilon', 1e-05)) + momentum = float(attrs.get('momentum', 0.9)) + # extract inputs + _, data_tensor_shape = inputs[0] + scale_tensor_val, _ = inputs[1] + bias_tensor_val, _ = inputs[2] + mean_tensor_val, _ = inputs[3] + var_tensor_val, _ = inputs[4] + # calc output tensor shape + out_tensor_shape = data_tensor_shape + # create module node + n_output = int(data_tensor_shape[1]) + + temp_module = SpatialBatchNormalization(n_output=n_output, eps=epsilon, + momentum=momentum, init_weight=scale_tensor_val, init_bias=bias_tensor_val) + if mean_tensor_val is not None: + temp_module.set_running_mean(mean_tensor_val) + if var_tensor_val is not None: + temp_module.set_running_std(var_tensor_val) + module = temp_module(prev_modules[0]) + return module, [out_tensor_shape] + + +def concat(inputs, prev_modules, attrs, outputs): + # extract attributes + axis = int(attrs.get('axis')) + # extract inputs + _, data_tensor_shape = inputs[0] + # calc output tensor shape + dim_rank = 0 + for i in range(len(inputs)): + _, curr_input_shape = inputs[i] + for j in range(len(data_tensor_shape)): + if axis != j: + if curr_input_shape[i] != data_tensor_shape[i]: + raise ValueError("Input shape mismatch. Expect receive input shape " + + data_tensor_shape[i] + " but got " + curr_input_shape[i]) + else: + dim_rank += curr_input_shape[axis] + out_tensor_shape = list(data_tensor_shape) + out_tensor_shape[axis] = dim_rank + out_tensor_shape = tuple(out_tensor_shape) + # create module node + module = JoinTable(dimension=axis+1, n_input_dims=len(data_tensor_shape))(prev_modules) + return module, [out_tensor_shape] + + +def constant(inputs, prev_modules, attrs, outputs): + # extract attributes + value = parse_tensor_data(attrs.get('value')) + + # calc output tensor shape + out_tensor_shape = value.shape + # create module node + module = Constant(value)(prev_modules[0]) + return module, [out_tensor_shape] + + +def conv(inputs, prev_modules, attrs, outputs): + # extract attributes + auto_pad = attrs.get('auto_pad', 'NOTSET') + padW, padH = map(int, attrs.get('pads', (0, 0))[:2]) + kernelW, kernelH = map(int, attrs.get('kernel_shape', (0, 0))[:2]) + strideW, strideH = map(int, attrs.get('strides', (1, 1))[:2]) + dilationW, dilationH = map(int, attrs.get('dilations', (1, 1))[:2]) + group = int(attrs.get('group', 1)) + withBias = len(inputs) == 3 and inputs[2] is not None + # extract inputs + data_tensor_val, data_tensor_shape = inputs[0] + weight_tensor_val, weight_tensor_shape = inputs[1] + bias_tensor_val = None + if withBias: + bias_tensor_val, _ = inputs[2] + # calc output tensor shape + input_batch_size, n_input_plane = map(int, data_tensor_shape[:2]) + n_output_plane = weight_tensor_shape[0] + input_height, input_width = data_tensor_shape[-2:] + output_height = calc_output_shape(input_height, kernelH, padding = padH, stride=strideH) + output_width = calc_output_shape(input_width, kernelW, padding = padW, stride=strideW) + out_tensor_shape = (input_batch_size, n_output_plane, output_height, output_width) + # create module node + module = SpatialConvolution( + n_input_plane=n_input_plane, n_output_plane=n_output_plane, + kernel_w=kernelW, kernel_h=kernelH, stride_w=strideW, stride_h=strideH, + pad_w=padW, pad_h=padH, n_group=group, init_weight=weight_tensor_val, + init_bias=bias_tensor_val, with_bias=withBias + )(prev_modules[0]) + return module, [out_tensor_shape] + + +def gather(inputs, prev_modules, attrs, outputs): + # extract attributes + axis = int(attrs.get('axis', 0)) + if axis != 0: + raise ValueError("Gather layer axis value") + # extract inputs + data_tensor_val, data_tensor_shape = inputs[0] + indices_val, indices = inputs[1] + # calc output tensor shape + out_tensor_shape = tuple(data_tensor_shape[:axis] + indices + data_tensor_shape[axis + 1:]) + # create module node + module = Gather()(prev_modules) + return module, [out_tensor_shape] + + +def gemm(inputs, prev_modules, attrs, outputs): + # extract attributes + alpha = float(attrs.get("alpha", 1.0)) + beta = float(attrs.get("beta", 1.0)) + trans_a = int(attrs.get("transA", 0)) + trans_b = int(attrs.get("transB", 0)) + # extract inputs + _, tensor_a_shape = inputs[0] + tensor_b_val, tensor_b_shape = inputs[1] + tensor_c_val, tensor_c_shape = inputs[2] + # create module node + module = Gemm(alpha=alpha, beta=beta, trans_a=trans_a, trans_b=trans_b, + matrix_b=tensor_b_val, matrix_c=tensor_c_val)(prev_modules) + return module, [tensor_c_shape] + + +def max_pool(inputs, prev_modules, attrs, outputs): + # extract attributes + auto_pad = attrs.get("auto_pad", 'NOTSET') + kernelW, kernelH = map(int, attrs.get("kernel_shape")[:2]) + strideW, strideH = map(int, attrs.get("strides", (1, 1))[:2]) + dilationW, dilationH = map(int, attrs.get('dilations', (1, 1))[:2]) + padW, padH = map(int, attrs.get("pads", (0, 0))[:2]) + ceil_mode = True if (attrs.get("ceil_mode", 0) != 0) else False + storage_order = int(attrs.get("storage_order", 0)) + # extract inputs + _, data_tensor_shape = inputs[0] + input_width, input_height = data_tensor_shape[-2:] + # calc output tensor shape + output_width = calc_output_shape(input_width, kernelW, + padding=padW, stride=strideW, dilation=dilationW, ceil_mode=ceil_mode) + output_height = calc_output_shape(input_height, kernelH, + padding=padH, stride=strideH, dilation=dilationH, ceil_mode=ceil_mode) + out_tensor_shape_list = list(data_tensor_shape) + out_tensor_shape_list[2] = output_height + out_tensor_shape_list[3] = output_width + out_tensor_shape = tuple(out_tensor_shape_list) + # create module node + module = SpatialMaxPooling(kw=kernelW, kh=kernelH, dw=strideW, dh=strideH, + pad_w=padW, pad_h=padH, to_ceil=ceil_mode)(prev_modules[0]) + return module, [out_tensor_shape] + + +def relu(inputs, prev_modules, attrs, outputs): + # extract inputs + _, data_tensor_shape = inputs[0] + # calc output tensor shape + output_shape = data_tensor_shape + # create module node + module = ReLU()(prev_modules[0]) + return module, [output_shape] + + +def reshape(inputs, prev_modules, attrs, outputs): + # extract inputs + _, data_tensor_shape = inputs[0] + shape_tensor_val, _ = inputs[1] + shape_arry = None + if shape_tensor_val is not None: + shape_arry = np.squeeze(shape_tensor_val).astype(int).tolist() + # create module node + module = Reshape(shape_arry)(prev_modules) + return module, [shape_tensor_val] + + +def shape(inputs, prev_modules, attrs, outputs): + # extract inputs + _, data_tensor_shape = inputs[0] + # create module node + module = Shape()(prev_modules[0]) + return module, [(len(data_tensor_shape),)] + + +def softmax(inputs, prev_modules, attrs, outputs): + _, data_tensor_shape = inputs[0] + out_tensor_shape = data_tensor_shape + axis = int(attrs.get('axis', 1)) + module = SoftMax()(prev_modules[0]) + return module, [out_tensor_shape] + + +def _sum(inputs, prev_modules, attrs, outputs): + _, data_tensor_shape = inputs[0] + out_tensor_shape = data_tensor_shape + module = CAddTable()(prev_modules) + return module, [data_tensor_shape] + + +def unsqueeze(inputs, prev_modules, attrs, outputs): + axes = list(map(int, attrs.get('axes'))) + data_tensor_val, data_tensor_shape = inputs[0] + out_tensor_shape = list(data_tensor_shape) + for idx in axes: + out_tensor_shape.insert(idx, 1) + out_tensor_shape = tuple(out_tensor_shape) + module = Unsqueeze(axes[0], len(data_tensor_shape))(prev_modules) + return module, [out_tensor_shape] diff --git a/python/dllib/src/bigdl/dllib/contrib/onnx/ops_mapping.py b/python/dllib/src/bigdl/dllib/contrib/onnx/ops_mapping.py new file mode 100644 index 00000000000..cce517b82af --- /dev/null +++ b/python/dllib/src/bigdl/dllib/contrib/onnx/ops_mapping.py @@ -0,0 +1,133 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +"""Operator attributes conversion""" +from .ops_converter import conv, batch_norm, relu, max_pool, _sum, average_pool +from .ops_converter import reshape, gemm, softmax, constant, shape, gather +from .ops_converter import unsqueeze, concat + + +# convert_map defines maps of operator names to converter functor(callable) +# defined in the op_translations module. +_convert_map = { + # Generator Functions + 'Constant' : constant, + # 'RandomUniform' : random_uniform, + # 'RandomNormal' : random_normal, + # 'RandomUniformLike' : random_uniform, + # 'RandomNormalLike' : random_normal, + # 'Multinomial' : sample_multinomial, + # # Arithmetic Operators + # 'Add' : add, + # 'Sub' : subtract, + # 'Mul' : multiply, + # 'Div' : divide, + # 'Abs' : absolute, + # 'Neg' : negative, + 'Sum' : _sum, #elemwise sum + # #Hyperbolic functions + # 'Tanh' : tanh, + # # Rounding + # 'Ceil' : ceil, + # 'Floor' : floor, + # # Joining and spliting + 'Concat' : concat, + # # Basic neural network functions + # 'Sigmoid' : sigmoid, + 'Relu' : relu, + # 'Pad' : pad, + # 'MatMul' : matrix_multiplication, #linalg_gemm2 + 'Conv' : conv, + # 'ConvTranspose' : deconv, + 'BatchNormalization': batch_norm, + # 'SpatialBN' : batch_norm, + # 'LeakyRelu' : leaky_relu, + # 'Elu' : _elu, + # 'PRelu' : _prelu, + # 'Selu' : _selu, + 'Softmax' : softmax, + # 'FC' : fully_connected, + # 'GlobalAveragePool' : global_avgpooling, + # 'GlobalMaxPool' : global_maxpooling, + # 'GlobalLpPool' : global_lppooling, + 'Gemm' : gemm, + # 'LRN' : local_response_norm, + # 'Dropout' : dropout, + # # Changing shape and type. + 'Reshape' : reshape, + # 'Cast' : cast, + # 'Split' : split, + # 'Slice' : _slice, + # 'Transpose' : transpose, + # 'Squeeze' : squeeze, + 'Unsqueeze' : unsqueeze, + # 'Flatten' : flatten, + # 'Identity' : identity, + # #Powers + # 'Reciprocal' : reciprocal, + # 'Sqrt' : squareroot, + # 'Pow' : power, + # 'Exp' : exponent, + # 'Log' : _log, + # # Reduce Functions + # 'ReduceMax' : reduce_max, + # 'ReduceMean' : reduce_mean, + # 'ReduceMin' : reduce_min, + # 'ReduceSum' : reduce_sum, + # 'ReduceProd' : reduce_prod, + 'AveragePool' : average_pool, + 'MaxPool' : max_pool, + # # Sorting and Searching + # 'ArgMax' : argmax, + # 'ArgMin' : argmin, + # 'Max' : maximum, + # 'Min' : minimum, + # 'Clip' : clip, + # 'ReduceLogSum' : reduce_log_sum, + # 'ReduceLogSumExp' : reduce_log_sum_exp, + # 'ReduceSumSquare' : reduce_sum_square, + # 'ReduceL1' : reduce_l1, + # 'ReduceL2' : reduce_l2, + # 'MaxRoiPool' : max_roi_pooling, + # 'InstanceNormalization' : instance_norm, + # 'LogSoftmax' : log_softmax, + # 'Softsign' : softsign, + # 'Less' : lesser, + # 'Greater' : greater, + # 'Equal' : equal, + # 'And' : logical_and, + # 'Xor' : logical_xor, + # 'Not' : logical_not, + # 'Or' : logical_or, + # 'Mean' : mean, + # 'Acos' : arccos, + # 'Asin' : arcsin, + # 'Atan' : arctan, + # 'Cos' : _cos, + # 'Sin' : _sin, + # 'Softplus' : softplus, + # 'Tan' : _tan, + 'Shape' : shape, + # 'Size' : size, + 'Gather' : gather, + # 'HardSigmoid' : hardsigmoid, + # 'LpPool' : lp_pooling, + # 'DepthToSpace' : depthtospace, + # 'SpaceToDepth' : spacetodepth, + # 'Hardmax' : hardmax, + # 'LpNormalization' : lpnormalization +} diff --git a/python/dllib/src/bigdl/dllib/nn/onnx/layer.py b/python/dllib/src/bigdl/dllib/nn/onnx/layer.py index 80cb5706e11..d99b1e8e86c 100644 --- a/python/dllib/src/bigdl/dllib/nn/onnx/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/onnx/layer.py @@ -24,6 +24,24 @@ unicode = str +class Constant(Layer): + """ + >>> value = np.random.random((3, 3)) + >>> constant = Constant(value) + creating: createConstant + """ + def __init__(self, value, bigdl_type="float"): + super(Constant, self).__init__(None, bigdl_type, JTensor.from_ndarray(value)) + + +class Gather(Layer): + """ + >>> constant = Gather() + creating: createGather + """ + def __init__(self, bigdl_type="float"): + super(Gather, self).__init__(None, bigdl_type) + class Gemm(Layer): """ @@ -48,6 +66,17 @@ def __init__(self, matrix_b, matrix_c, alpha=float(1.0), beta=float(1.0), trans_ JTensor.from_ndarray(matrix_b), JTensor.from_ndarray(matrix_c)) +class Reshape(Layer): + """ + A layer which takes a tensor as input and outputs an 1D tensor containing the shape of the input. + >>> shape = (2, 2) + >>> reshape = Reshape(shape) + creating: createReshape + """ + def __init__(self, shape=None, bigdl_type="float"): + super(Reshape, self).__init__(None, bigdl_type, shape) + + class Shape(Layer): """ A layer which takes a tensor as input and outputs an 1D tensor containing the shape of the input. From 83595b5098535ade799dad197cc86e10353cc025 Mon Sep 17 00:00:00 2001 From: LeicongLi Date: Tue, 22 Oct 2019 17:03:54 +0800 Subject: [PATCH 754/823] ONNX Support (#2918) * onnx dev * add onnx loader * clean up --- .../dllib/src/bigdl/dllib/contrib/__init__.py | 0 .../src/bigdl/dllib/contrib/onnx/__init__.py | 17 ++ .../dllib/contrib/onnx/converter_utils.py | 70 +++++ .../bigdl/dllib/contrib/onnx/onnx_loader.py | 111 ++++++++ .../bigdl/dllib/contrib/onnx/ops_converter.py | 256 ++++++++++++++++++ .../bigdl/dllib/contrib/onnx/ops_mapping.py | 133 +++++++++ python/dllib/src/bigdl/dllib/nn/onnx/layer.py | 29 ++ 7 files changed, 616 insertions(+) create mode 100644 python/dllib/src/bigdl/dllib/contrib/__init__.py create mode 100644 python/dllib/src/bigdl/dllib/contrib/onnx/__init__.py create mode 100644 python/dllib/src/bigdl/dllib/contrib/onnx/converter_utils.py create mode 100644 python/dllib/src/bigdl/dllib/contrib/onnx/onnx_loader.py create mode 100644 python/dllib/src/bigdl/dllib/contrib/onnx/ops_converter.py create mode 100644 python/dllib/src/bigdl/dllib/contrib/onnx/ops_mapping.py diff --git a/python/dllib/src/bigdl/dllib/contrib/__init__.py b/python/dllib/src/bigdl/dllib/contrib/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/python/dllib/src/bigdl/dllib/contrib/onnx/__init__.py b/python/dllib/src/bigdl/dllib/contrib/onnx/__init__.py new file mode 100644 index 00000000000..1c529fd7ad6 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/contrib/onnx/__init__.py @@ -0,0 +1,17 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from .onnx_loader import load diff --git a/python/dllib/src/bigdl/dllib/contrib/onnx/converter_utils.py b/python/dllib/src/bigdl/dllib/contrib/onnx/converter_utils.py new file mode 100644 index 00000000000..e7d1417200a --- /dev/null +++ b/python/dllib/src/bigdl/dllib/contrib/onnx/converter_utils.py @@ -0,0 +1,70 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import math +import numpy as np + + +def calc_output_shape(input, kernel, padding=0, stride=1, dilation=1, ceil_mode=False): + def dilated_kernel_size(kernel, dilation): + return kernel + (kernel - 1) * (dilation - 1) + rounding = math.ceil if ceil_mode else math.floor + out = (input + 2 * padding - dilated_kernel_size(kernel, dilation)) / stride + 1 + out = int(rounding(out)) + return out + + +def parse_node_attr(node_proto): + attrs = {} + attr_proto = node_proto.attribute + + for attr in attr_proto: + for field in ['f', 'i', 's']: + if attr.HasField(field): + attrs[attr.name] = getattr(attr, field) + + # Needed for supporting python version > 3.5 + if isinstance(attrs[attr.name], bytes): + attrs[attr.name] = attrs[attr.name].decode(encoding='utf-8') + + for field in ['floats', 'ints', 'strings']: + if list(getattr(attr, field)): + assert attr.name not in attrs, "Only one type of attr is allowed" + attrs[attr.name] = tuple(getattr(attr, field)) + + for field in ['t', 'g']: + if attr.HasField(field): + attrs[attr.name] = getattr(attr, field) + for field in ['tensors', 'graphs']: + if list(getattr(attr, field)): + raise NotImplementedError() + if attr.name not in attrs: + raise ValueError("Cannot parse attribute: \n{}\n.".format(attr)) + + return attrs + + +def parse_tensor_data(tensor_proto): + try: + from onnx.numpy_helper import to_array + except ImportError: + raise ImportError("Onnx and protobuf need to be installed.") + if len(tuple(tensor_proto.dims)) > 0: + np_array = to_array(tensor_proto).reshape(tuple(tensor_proto.dims)) + else: + # If it is a scalar tensor + np_array = np.array([to_array(tensor_proto)]) + return np_array diff --git a/python/dllib/src/bigdl/dllib/contrib/onnx/onnx_loader.py b/python/dllib/src/bigdl/dllib/contrib/onnx/onnx_loader.py new file mode 100644 index 00000000000..6e537c10c3e --- /dev/null +++ b/python/dllib/src/bigdl/dllib/contrib/onnx/onnx_loader.py @@ -0,0 +1,111 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import onnx +from bigdl.nn.onnx.layer import * +from bigdl.nn.layer import Identity, Model +from .ops_mapping import _convert_map as convert_map +from .converter_utils import parse_node_attr, parse_tensor_data + + +class OnnxLoader(object): + + def load_model(self, file_path): + model_proto = onnx.load_model(file_path) + # self._ir_version = model_proto.ir_version + # self._opset_import = model_proto.opset_import + # self._producer_name = model_proto.producer_name + # self._producer_version = model_proto.producer_version + # self._domain = model_proto.domain + # self._model_version = model_proto.model_version + # self._doc_string = model_proto.doc_string + graph_proto = model_proto.graph + return self.load_graph(graph_proto) + + def load_graph(self, graph_proto): + if not graph_proto: + raise ValueError("Graph proto is required") + + input_nodes = list() + output_nodes = list() + tensor_map = dict() + initialized_tensors = set() + module_map = dict() + root_nodes = list() + dummy_root = Identity()() + + for tensor in graph_proto.initializer: + if not tensor.name.strip(): + raise ValueError("Tensor's name is required") + initialized_tensors.add(tensor.name) + tensor_data = parse_tensor_data(tensor) + tensor_map[tensor.name] = (tensor_data, tensor_data.shape) + + for gin in graph_proto.input: + if gin.name not in initialized_tensors: + input_nodes.append(gin.name) + shape = tuple([dim.dim_value for dim in gin.type.tensor_type.shape.dim]) + module_map[gin.name] = Identity()(dummy_root) + tensor_map[gin.name] = (None, shape) + + for gout in graph_proto.output: + if gout.name not in initialized_tensors: + output_nodes.append(gout.name) + + for node in graph_proto.node: + name = node.name.strip() + op_type = node.op_type + inputs = [tensor_map[n] for n in node.input] + outputs = node.output + prev_modules = [module_map[n] for n in node.input if n not in initialized_tensors] + attrs = parse_node_attr(node) + + if len(prev_modules) == 0: + root_nodes.append((name, op_type)) + prev_modules = [dummy_root] + + bigdl_module, outputs_shape = self._make_module_from_onnx_node(op_type, inputs, prev_modules, attrs, outputs) + + assert len(outputs) == len(outputs_shape) + + for out, out_shape in zip(outputs, outputs_shape): + module_map[out] = bigdl_module + tensor_map[out] = (None, out_shape) + + in_modules = [module_map[m] for m in input_nodes] + out_modules = [module_map[m] for m in output_nodes] + model = Model([dummy_root], out_modules) + + return model + + def _make_module_from_onnx_node(self, op_type, inputs, prev_modules, attrs, outputs): + module = None + out_shapes = [] + if op_type in convert_map: + module, out_shapes = convert_map[op_type](inputs, prev_modules, attrs, outputs) + else: + raise NotImplemented(op_type) + return module, out_shapes + + +def load(model_path): + loader = OnnxLoader() + return loader.load_model(model_path) + + +def load_model_proto(model_proto): + loader = OnnxLoader() + return loader.load_graph(model_proto.graph) \ No newline at end of file diff --git a/python/dllib/src/bigdl/dllib/contrib/onnx/ops_converter.py b/python/dllib/src/bigdl/dllib/contrib/onnx/ops_converter.py new file mode 100644 index 00000000000..4934f64ce93 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/contrib/onnx/ops_converter.py @@ -0,0 +1,256 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from bigdl.nn.layer import SpatialAveragePooling, SpatialBatchNormalization +from bigdl.nn.layer import SpatialConvolution, SpatialMaxPooling, JoinTable +from bigdl.nn.layer import ReLU, SoftMax, CAddTable, Unsqueeze +from bigdl.nn.onnx.layer import Constant, Gather, Gemm, Shape, Reshape +from .converter_utils import * + + +def average_pool(inputs, prev_modules, attrs, outputs): + # extract attributes + auto_pad = attrs.get('auto_pad', 'NOTSET') + ceil_mode = True if attrs.get('ceil_mode', 0) == 1 else False + count_include_pad = True if attrs.get('count_include_pad', 0) == 1 else False + kernel_width, kernel_height = map(int, attrs.get('kernel_shape', (1, 1))[:2]) + stride_width, stride_height = map(int, attrs.get('strides', (1, 1))[:2]) + padding_width, padding_height = map(int, attrs.get('pads', (0, 0))[:2]) + # extract inputs + _, data_tensor_shape = inputs[0] + # calc output tensor shape + input_height, input_width = data_tensor_shape[-2:] + output_height = calc_output_shape(input_height, kernel_height, + padding = padding_height, stride = stride_height, ceil_mode = ceil_mode) + output_width = calc_output_shape(input_width, kernel_width, + padding = padding_width, stride = stride_width, ceil_mode = ceil_mode) + out_tensor_shape = list(data_tensor_shape) + out_tensor_shape[-2] = output_height + out_tensor_shape[-1] = output_width + out_tensor_shape = tuple(out_tensor_shape) + # create module node + module = SpatialAveragePooling(kw=kernel_width, kh=kernel_height, + dw=stride_width, dh=stride_height, + pad_w=padding_width, pad_h=padding_height, + ceil_mode=ceil_mode, count_include_pad=count_include_pad + )(prev_modules) + return module, [out_tensor_shape] + + +def batch_norm(inputs, prev_modules, attrs, outputs): + # extract attributes + epsilon = float(attrs.get('epsilon', 1e-05)) + momentum = float(attrs.get('momentum', 0.9)) + # extract inputs + _, data_tensor_shape = inputs[0] + scale_tensor_val, _ = inputs[1] + bias_tensor_val, _ = inputs[2] + mean_tensor_val, _ = inputs[3] + var_tensor_val, _ = inputs[4] + # calc output tensor shape + out_tensor_shape = data_tensor_shape + # create module node + n_output = int(data_tensor_shape[1]) + + temp_module = SpatialBatchNormalization(n_output=n_output, eps=epsilon, + momentum=momentum, init_weight=scale_tensor_val, init_bias=bias_tensor_val) + if mean_tensor_val is not None: + temp_module.set_running_mean(mean_tensor_val) + if var_tensor_val is not None: + temp_module.set_running_std(var_tensor_val) + module = temp_module(prev_modules[0]) + return module, [out_tensor_shape] + + +def concat(inputs, prev_modules, attrs, outputs): + # extract attributes + axis = int(attrs.get('axis')) + # extract inputs + _, data_tensor_shape = inputs[0] + # calc output tensor shape + dim_rank = 0 + for i in range(len(inputs)): + _, curr_input_shape = inputs[i] + for j in range(len(data_tensor_shape)): + if axis != j: + if curr_input_shape[i] != data_tensor_shape[i]: + raise ValueError("Input shape mismatch. Expect receive input shape " + + data_tensor_shape[i] + " but got " + curr_input_shape[i]) + else: + dim_rank += curr_input_shape[axis] + out_tensor_shape = list(data_tensor_shape) + out_tensor_shape[axis] = dim_rank + out_tensor_shape = tuple(out_tensor_shape) + # create module node + module = JoinTable(dimension=axis+1, n_input_dims=len(data_tensor_shape))(prev_modules) + return module, [out_tensor_shape] + + +def constant(inputs, prev_modules, attrs, outputs): + # extract attributes + value = parse_tensor_data(attrs.get('value')) + + # calc output tensor shape + out_tensor_shape = value.shape + # create module node + module = Constant(value)(prev_modules[0]) + return module, [out_tensor_shape] + + +def conv(inputs, prev_modules, attrs, outputs): + # extract attributes + auto_pad = attrs.get('auto_pad', 'NOTSET') + padW, padH = map(int, attrs.get('pads', (0, 0))[:2]) + kernelW, kernelH = map(int, attrs.get('kernel_shape', (0, 0))[:2]) + strideW, strideH = map(int, attrs.get('strides', (1, 1))[:2]) + dilationW, dilationH = map(int, attrs.get('dilations', (1, 1))[:2]) + group = int(attrs.get('group', 1)) + withBias = len(inputs) == 3 and inputs[2] is not None + # extract inputs + data_tensor_val, data_tensor_shape = inputs[0] + weight_tensor_val, weight_tensor_shape = inputs[1] + bias_tensor_val = None + if withBias: + bias_tensor_val, _ = inputs[2] + # calc output tensor shape + input_batch_size, n_input_plane = map(int, data_tensor_shape[:2]) + n_output_plane = weight_tensor_shape[0] + input_height, input_width = data_tensor_shape[-2:] + output_height = calc_output_shape(input_height, kernelH, padding = padH, stride=strideH) + output_width = calc_output_shape(input_width, kernelW, padding = padW, stride=strideW) + out_tensor_shape = (input_batch_size, n_output_plane, output_height, output_width) + # create module node + module = SpatialConvolution( + n_input_plane=n_input_plane, n_output_plane=n_output_plane, + kernel_w=kernelW, kernel_h=kernelH, stride_w=strideW, stride_h=strideH, + pad_w=padW, pad_h=padH, n_group=group, init_weight=weight_tensor_val, + init_bias=bias_tensor_val, with_bias=withBias + )(prev_modules[0]) + return module, [out_tensor_shape] + + +def gather(inputs, prev_modules, attrs, outputs): + # extract attributes + axis = int(attrs.get('axis', 0)) + if axis != 0: + raise ValueError("Gather layer axis value") + # extract inputs + data_tensor_val, data_tensor_shape = inputs[0] + indices_val, indices = inputs[1] + # calc output tensor shape + out_tensor_shape = tuple(data_tensor_shape[:axis] + indices + data_tensor_shape[axis + 1:]) + # create module node + module = Gather()(prev_modules) + return module, [out_tensor_shape] + + +def gemm(inputs, prev_modules, attrs, outputs): + # extract attributes + alpha = float(attrs.get("alpha", 1.0)) + beta = float(attrs.get("beta", 1.0)) + trans_a = int(attrs.get("transA", 0)) + trans_b = int(attrs.get("transB", 0)) + # extract inputs + _, tensor_a_shape = inputs[0] + tensor_b_val, tensor_b_shape = inputs[1] + tensor_c_val, tensor_c_shape = inputs[2] + # create module node + module = Gemm(alpha=alpha, beta=beta, trans_a=trans_a, trans_b=trans_b, + matrix_b=tensor_b_val, matrix_c=tensor_c_val)(prev_modules) + return module, [tensor_c_shape] + + +def max_pool(inputs, prev_modules, attrs, outputs): + # extract attributes + auto_pad = attrs.get("auto_pad", 'NOTSET') + kernelW, kernelH = map(int, attrs.get("kernel_shape")[:2]) + strideW, strideH = map(int, attrs.get("strides", (1, 1))[:2]) + dilationW, dilationH = map(int, attrs.get('dilations', (1, 1))[:2]) + padW, padH = map(int, attrs.get("pads", (0, 0))[:2]) + ceil_mode = True if (attrs.get("ceil_mode", 0) != 0) else False + storage_order = int(attrs.get("storage_order", 0)) + # extract inputs + _, data_tensor_shape = inputs[0] + input_width, input_height = data_tensor_shape[-2:] + # calc output tensor shape + output_width = calc_output_shape(input_width, kernelW, + padding=padW, stride=strideW, dilation=dilationW, ceil_mode=ceil_mode) + output_height = calc_output_shape(input_height, kernelH, + padding=padH, stride=strideH, dilation=dilationH, ceil_mode=ceil_mode) + out_tensor_shape_list = list(data_tensor_shape) + out_tensor_shape_list[2] = output_height + out_tensor_shape_list[3] = output_width + out_tensor_shape = tuple(out_tensor_shape_list) + # create module node + module = SpatialMaxPooling(kw=kernelW, kh=kernelH, dw=strideW, dh=strideH, + pad_w=padW, pad_h=padH, to_ceil=ceil_mode)(prev_modules[0]) + return module, [out_tensor_shape] + + +def relu(inputs, prev_modules, attrs, outputs): + # extract inputs + _, data_tensor_shape = inputs[0] + # calc output tensor shape + output_shape = data_tensor_shape + # create module node + module = ReLU()(prev_modules[0]) + return module, [output_shape] + + +def reshape(inputs, prev_modules, attrs, outputs): + # extract inputs + _, data_tensor_shape = inputs[0] + shape_tensor_val, _ = inputs[1] + shape_arry = None + if shape_tensor_val is not None: + shape_arry = np.squeeze(shape_tensor_val).astype(int).tolist() + # create module node + module = Reshape(shape_arry)(prev_modules) + return module, [shape_tensor_val] + + +def shape(inputs, prev_modules, attrs, outputs): + # extract inputs + _, data_tensor_shape = inputs[0] + # create module node + module = Shape()(prev_modules[0]) + return module, [(len(data_tensor_shape),)] + + +def softmax(inputs, prev_modules, attrs, outputs): + _, data_tensor_shape = inputs[0] + out_tensor_shape = data_tensor_shape + axis = int(attrs.get('axis', 1)) + module = SoftMax()(prev_modules[0]) + return module, [out_tensor_shape] + + +def _sum(inputs, prev_modules, attrs, outputs): + _, data_tensor_shape = inputs[0] + out_tensor_shape = data_tensor_shape + module = CAddTable()(prev_modules) + return module, [data_tensor_shape] + + +def unsqueeze(inputs, prev_modules, attrs, outputs): + axes = list(map(int, attrs.get('axes'))) + data_tensor_val, data_tensor_shape = inputs[0] + out_tensor_shape = list(data_tensor_shape) + for idx in axes: + out_tensor_shape.insert(idx, 1) + out_tensor_shape = tuple(out_tensor_shape) + module = Unsqueeze(axes[0], len(data_tensor_shape))(prev_modules) + return module, [out_tensor_shape] diff --git a/python/dllib/src/bigdl/dllib/contrib/onnx/ops_mapping.py b/python/dllib/src/bigdl/dllib/contrib/onnx/ops_mapping.py new file mode 100644 index 00000000000..cce517b82af --- /dev/null +++ b/python/dllib/src/bigdl/dllib/contrib/onnx/ops_mapping.py @@ -0,0 +1,133 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +"""Operator attributes conversion""" +from .ops_converter import conv, batch_norm, relu, max_pool, _sum, average_pool +from .ops_converter import reshape, gemm, softmax, constant, shape, gather +from .ops_converter import unsqueeze, concat + + +# convert_map defines maps of operator names to converter functor(callable) +# defined in the op_translations module. +_convert_map = { + # Generator Functions + 'Constant' : constant, + # 'RandomUniform' : random_uniform, + # 'RandomNormal' : random_normal, + # 'RandomUniformLike' : random_uniform, + # 'RandomNormalLike' : random_normal, + # 'Multinomial' : sample_multinomial, + # # Arithmetic Operators + # 'Add' : add, + # 'Sub' : subtract, + # 'Mul' : multiply, + # 'Div' : divide, + # 'Abs' : absolute, + # 'Neg' : negative, + 'Sum' : _sum, #elemwise sum + # #Hyperbolic functions + # 'Tanh' : tanh, + # # Rounding + # 'Ceil' : ceil, + # 'Floor' : floor, + # # Joining and spliting + 'Concat' : concat, + # # Basic neural network functions + # 'Sigmoid' : sigmoid, + 'Relu' : relu, + # 'Pad' : pad, + # 'MatMul' : matrix_multiplication, #linalg_gemm2 + 'Conv' : conv, + # 'ConvTranspose' : deconv, + 'BatchNormalization': batch_norm, + # 'SpatialBN' : batch_norm, + # 'LeakyRelu' : leaky_relu, + # 'Elu' : _elu, + # 'PRelu' : _prelu, + # 'Selu' : _selu, + 'Softmax' : softmax, + # 'FC' : fully_connected, + # 'GlobalAveragePool' : global_avgpooling, + # 'GlobalMaxPool' : global_maxpooling, + # 'GlobalLpPool' : global_lppooling, + 'Gemm' : gemm, + # 'LRN' : local_response_norm, + # 'Dropout' : dropout, + # # Changing shape and type. + 'Reshape' : reshape, + # 'Cast' : cast, + # 'Split' : split, + # 'Slice' : _slice, + # 'Transpose' : transpose, + # 'Squeeze' : squeeze, + 'Unsqueeze' : unsqueeze, + # 'Flatten' : flatten, + # 'Identity' : identity, + # #Powers + # 'Reciprocal' : reciprocal, + # 'Sqrt' : squareroot, + # 'Pow' : power, + # 'Exp' : exponent, + # 'Log' : _log, + # # Reduce Functions + # 'ReduceMax' : reduce_max, + # 'ReduceMean' : reduce_mean, + # 'ReduceMin' : reduce_min, + # 'ReduceSum' : reduce_sum, + # 'ReduceProd' : reduce_prod, + 'AveragePool' : average_pool, + 'MaxPool' : max_pool, + # # Sorting and Searching + # 'ArgMax' : argmax, + # 'ArgMin' : argmin, + # 'Max' : maximum, + # 'Min' : minimum, + # 'Clip' : clip, + # 'ReduceLogSum' : reduce_log_sum, + # 'ReduceLogSumExp' : reduce_log_sum_exp, + # 'ReduceSumSquare' : reduce_sum_square, + # 'ReduceL1' : reduce_l1, + # 'ReduceL2' : reduce_l2, + # 'MaxRoiPool' : max_roi_pooling, + # 'InstanceNormalization' : instance_norm, + # 'LogSoftmax' : log_softmax, + # 'Softsign' : softsign, + # 'Less' : lesser, + # 'Greater' : greater, + # 'Equal' : equal, + # 'And' : logical_and, + # 'Xor' : logical_xor, + # 'Not' : logical_not, + # 'Or' : logical_or, + # 'Mean' : mean, + # 'Acos' : arccos, + # 'Asin' : arcsin, + # 'Atan' : arctan, + # 'Cos' : _cos, + # 'Sin' : _sin, + # 'Softplus' : softplus, + # 'Tan' : _tan, + 'Shape' : shape, + # 'Size' : size, + 'Gather' : gather, + # 'HardSigmoid' : hardsigmoid, + # 'LpPool' : lp_pooling, + # 'DepthToSpace' : depthtospace, + # 'SpaceToDepth' : spacetodepth, + # 'Hardmax' : hardmax, + # 'LpNormalization' : lpnormalization +} diff --git a/python/dllib/src/bigdl/dllib/nn/onnx/layer.py b/python/dllib/src/bigdl/dllib/nn/onnx/layer.py index 80cb5706e11..d99b1e8e86c 100644 --- a/python/dllib/src/bigdl/dllib/nn/onnx/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/onnx/layer.py @@ -24,6 +24,24 @@ unicode = str +class Constant(Layer): + """ + >>> value = np.random.random((3, 3)) + >>> constant = Constant(value) + creating: createConstant + """ + def __init__(self, value, bigdl_type="float"): + super(Constant, self).__init__(None, bigdl_type, JTensor.from_ndarray(value)) + + +class Gather(Layer): + """ + >>> constant = Gather() + creating: createGather + """ + def __init__(self, bigdl_type="float"): + super(Gather, self).__init__(None, bigdl_type) + class Gemm(Layer): """ @@ -48,6 +66,17 @@ def __init__(self, matrix_b, matrix_c, alpha=float(1.0), beta=float(1.0), trans_ JTensor.from_ndarray(matrix_b), JTensor.from_ndarray(matrix_c)) +class Reshape(Layer): + """ + A layer which takes a tensor as input and outputs an 1D tensor containing the shape of the input. + >>> shape = (2, 2) + >>> reshape = Reshape(shape) + creating: createReshape + """ + def __init__(self, shape=None, bigdl_type="float"): + super(Reshape, self).__init__(None, bigdl_type, shape) + + class Shape(Layer): """ A layer which takes a tensor as input and outputs an 1D tensor containing the shape of the input. From 688d15a22503045363d89786e468351f0fe17045 Mon Sep 17 00:00:00 2001 From: LeicongLi Date: Tue, 22 Oct 2019 17:03:54 +0800 Subject: [PATCH 755/823] ONNX Support (#2918) * onnx dev * add onnx loader * clean up --- python/dllib/test/bigdl/onnx/test_onnx_ops.py | 765 ++++++++++++++++++ 1 file changed, 765 insertions(+) create mode 100644 python/dllib/test/bigdl/onnx/test_onnx_ops.py diff --git a/python/dllib/test/bigdl/onnx/test_onnx_ops.py b/python/dllib/test/bigdl/onnx/test_onnx_ops.py new file mode 100644 index 00000000000..a5205a79ce3 --- /dev/null +++ b/python/dllib/test/bigdl/onnx/test_onnx_ops.py @@ -0,0 +1,765 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import onnx +import pytest +import numpy as np + +from bigdl.contrib.onnx.onnx_loader import load_model_proto +from bigdl.nn.layer import CAddTable, JoinTable, ReLU +from bigdl.nn.layer import SoftMax, SpatialAveragePooling, SpatialBatchNormalization +from bigdl.nn.layer import SpatialConvolution, SpatialMaxPooling +from bigdl.nn.layer import Unsqueeze +from bigdl.nn.onnx.layer import Gemm, Reshape, Shape + + +class TestAveragePool(object): + + def test_average_pool(self): + ceil_mode = 0 + kernel_width, kernel_height = 3, 3 + pad_width, pad_height = 0, 0 + stride_width, stride_height = 1, 1 + input_shape = [1, 3, 224, 224] + output_shape = [1, 3, 222, 222] + input_x = np.random.random(input_shape) + + # Create one input (ValueInfoProto) + X = onnx.helper.make_tensor_value_info('X', onnx.TensorProto.FLOAT, input_shape) + # Create one output (ValueInfoProto) + Y = onnx.helper.make_tensor_value_info('Y', onnx.TensorProto.FLOAT, output_shape) + + # Create a node (NodeProto) + avgpool_node = onnx.helper.make_node( + op_type='AveragePool', # node name + inputs=['X'], # inputs + outputs=['Y'], # outputs + auto_pad='NOTSET', + ceil_mode=ceil_mode, + kernel_shape=(kernel_width, kernel_height), + pads=(pad_width, pad_height), + strides=(stride_width, stride_height) + ) + + # Create the graph (GraphProto) + onnx_graph = onnx.helper.make_graph( + nodes=[avgpool_node], + name='test-averagePool', + inputs=[X], + outputs=[Y], + ) + + # Create the model (ModelProto) + onnx_model = onnx.helper.make_model(onnx_graph, producer_name='ONNX') + onnx.checker.check_model(onnx_model) + + loaded_model = load_model_proto(onnx_model) + bigdl_model = SpatialAveragePooling( + kw=kernel_width, + kh=kernel_height, + dw=stride_width, + dh=stride_height, + pad_w=pad_width, + pad_h=pad_height, + ceil_mode=False if ceil_mode == 0 else True + ) + + loaded_out = loaded_model.forward(input_x) + expected_out = bigdl_model.forward(input_x) + + assert(np.array_equal(expected_out, loaded_out)) + + +class TestBatchNormalization(object): + + def test_batch_normalization(self): + input_shape = [1, 3, 224, 224] + output_shape = [1, 3, 224, 224] + # Create inputs (ValueInfoProto) + X = onnx.helper.make_tensor_value_info('X', onnx.TensorProto.FLOAT, input_shape) + scale = onnx.helper.make_tensor_value_info('scale', onnx.TensorProto.FLOAT, input_shape[:2]) + bias = onnx.helper.make_tensor_value_info('bias', onnx.TensorProto.FLOAT, input_shape[:2]) + mean = onnx.helper.make_tensor_value_info('mean', onnx.TensorProto.FLOAT, input_shape[:2]) + var = onnx.helper.make_tensor_value_info('var', onnx.TensorProto.FLOAT, input_shape[:2]) + # Create one output (ValueInfoProto) + Y = onnx.helper.make_tensor_value_info('Y', onnx.TensorProto.FLOAT, output_shape) + + scale_vals = np.random.random(input_shape[1]) * 10 + bias_vals = np.random.random(input_shape[1]) * 10 + mean_vals = np.random.random(input_shape[1]) * 10 + var_vals = np.random.random(input_shape[1]) * 10 + input_x = np.random.random(input_shape) * 10 + epsilon = float(1e-05) + momentum = float(0.9) + + init_scale = onnx.helper.make_tensor( + name='scale', + data_type=onnx.TensorProto.FLOAT, + dims=input_shape[:2], + vals=scale_vals.tolist(), + ) + + init_bias = onnx.helper.make_tensor( + name='bias', + data_type=onnx.TensorProto.FLOAT, + dims=input_shape[:2], + vals=bias_vals.tolist(), + ) + + init_mean = onnx.helper.make_tensor( + name='mean', + data_type=onnx.TensorProto.FLOAT, + dims=input_shape[:2], + vals=mean_vals.tolist(), + ) + + init_var = onnx.helper.make_tensor( + name='var', + data_type=onnx.TensorProto.FLOAT, + dims=input_shape[:2], + vals=var_vals.tolist(), + ) + + # Create a node (NodeProto) + batch_norm_node = onnx.helper.make_node( + op_type='BatchNormalization', # node name + inputs=['X', 'scale', 'bias', 'mean', 'var'], # inputs + outputs=['Y'], # outputs + epsilon=epsilon, + momentum=momentum + ) + + # Create the graph (GraphProto) + onnx_graph = onnx.helper.make_graph( + nodes=[batch_norm_node], + name='test-batch_norm', + inputs=[X], + outputs=[Y], + initializer=[init_scale, init_bias, init_mean, init_var] + ) + + # Create the model (ModelProto) + onnx_model = onnx.helper.make_model(onnx_graph, producer_name='ONNX') + onnx.checker.check_model(onnx_model) + + loaded_model = load_model_proto(onnx_model) + bigdl_model = SpatialBatchNormalization( + n_output=input_shape[1], + eps=epsilon, + momentum=momentum, + init_weight=scale_vals, + init_bias=bias_vals, + init_grad_weight=None, + init_grad_bias=None, + ) + bigdl_model.set_running_mean(mean_vals) + bigdl_model.set_running_std(var_vals) + + loaded_out = loaded_model.forward(input_x) + expected_out = bigdl_model.forward(input_x) + + assert(np.array_equal(loaded_out, expected_out)) + + +class TestConcat(object): + + def test_concat(self): + axis = 0 + input_shape = [2, 3] + output_shape = [4, 3] + x1_val = np.random.random(input_shape) + x2_val = np.random.random(input_shape) + + # Create input (ValueInfoProto) + X = onnx.helper.make_tensor_value_info('X', onnx.TensorProto.FLOAT, input_shape) + X1 = onnx.helper.make_tensor_value_info('X1', onnx.TensorProto.FLOAT, input_shape) + X2 = onnx.helper.make_tensor_value_info('X2', onnx.TensorProto.FLOAT, input_shape) + + # Create one output (ValueInfoProto) + Y = onnx.helper.make_tensor_value_info('Y', onnx.TensorProto.FLOAT, output_shape) + + # Create a node (NodeProto) + const_X1 = onnx.helper.make_node( + op_type='Constant', + inputs=[], + outputs=['X1'], + value=onnx.helper.make_tensor( + name='X1', + data_type=onnx.TensorProto.FLOAT, + dims=input_shape, + vals=x1_val.flatten().tolist(), + ) + ) + + const_X2 = onnx.helper.make_node( + op_type='Constant', + inputs=[], + outputs=['X2'], + value=onnx.helper.make_tensor( + name='X2', + data_type=onnx.TensorProto.FLOAT, + dims=input_shape, + vals=x2_val.flatten().tolist(), + ) + ) + + concat_node = onnx.helper.make_node( + op_type='Concat', # node name + inputs=['X1', 'X2'], # inputs + outputs=['Y'], # outputs + axis=axis + ) + + # Create the graph (GraphProto) + onnx_graph = onnx.helper.make_graph( + nodes=[const_X1, const_X2, concat_node], + name='test-concat', + inputs=[X], + outputs=[Y], + ) + + # Create the model (ModelProto) + onnx_model = onnx.helper.make_model(onnx_graph, producer_name='ONNX') + onnx.checker.check_model(onnx_model) + + loaded_model = load_model_proto(onnx_model) + bigdl_model = JoinTable(dimension=axis + 1, n_input_dims=len(input_shape)) + + loaded_out = loaded_model.forward([x1_val, x2_val]) + expected_out = bigdl_model.forward([x1_val, x2_val]) + + assert(np.array_equal(loaded_out, expected_out)) + + +class TestConstant(object): + + def test_constant(self): + + shape = [5, 5] + values = np.float32(np.round(np.random.random(shape), 6)) + dummy_input = np.random.random([1]) + + # Create one output (ValueInfoProto) + Y = onnx.helper.make_tensor_value_info('Y', onnx.TensorProto.FLOAT, values.shape) + + constant_node = onnx.helper.make_node( + op_type='Constant', + inputs=[], + outputs=['Y'], + value=onnx.helper.make_tensor( + name='const_tensor', + data_type=onnx.TensorProto.FLOAT, + dims=values.shape, + vals=values.flatten().tolist(), + ), + ) + + # Create the graph (GraphProto) + onnx_graph = onnx.helper.make_graph( + nodes=[constant_node], + name='test-constant', + inputs=[], + outputs=[Y], + ) + + # Create the model (ModelProto) + onnx_model = onnx.helper.make_model(onnx_graph, producer_name='ONNX') + onnx.checker.check_model(onnx_model) + loaded_model = load_model_proto(onnx_model) + + loaded_out = loaded_model.forward(dummy_input) + expected_out = values + + assert(np.array_equal(loaded_out, expected_out)) + + +class TestConv(object): + + def test_conv(self): + kernel_width, kernel_height = (3, 3) + stride_width, stride_height = (1, 1) + pad_width, pad_height = (0, 0) + input_shape = [1, 3, 224, 224] + output_shape = [1, 8, 222, 222] + weight_shape = [8, 3, 3, 3] + input_x = np.random.random(input_shape) + weight_values = np.random.random(weight_shape) + + # Create input (ValueInfoProto) + X = onnx.helper.make_tensor_value_info('X', onnx.TensorProto.FLOAT, input_shape) + W = onnx.helper.make_tensor_value_info('W', onnx.TensorProto.FLOAT, weight_shape) + # Create one output (ValueInfoProto) + Y = onnx.helper.make_tensor_value_info('Y', onnx.TensorProto.FLOAT, output_shape) + + init_weight = onnx.helper.make_tensor( + name='W', + data_type=onnx.TensorProto.FLOAT, + dims=weight_shape, + vals=weight_values.flatten().astype(float), + ) + + conv_node = onnx.helper.make_node( + op_type='Conv', + inputs=['X', 'W'], + outputs=['Y'], + kernel_shape=(kernel_width, kernel_height), + ) + + # Create the graph (GraphProto) + onnx_graph = onnx.helper.make_graph( + nodes=[conv_node], + name='test-conv', + inputs=[X], + outputs=[Y], + initializer=[init_weight] + ) + + # Create the model (ModelProto) + onnx_model = onnx.helper.make_model(onnx_graph, producer_name='ONNX') + onnx.checker.check_model(onnx_model) + + loaded_model = load_model_proto(onnx_model) + bigdl_model = SpatialConvolution( + n_input_plane=3, + n_output_plane=8, + kernel_w=kernel_width, + kernel_h=kernel_height, + stride_w=stride_width, + stride_h=stride_height, + pad_w=pad_width, + pad_h=pad_height, + init_weight=weight_values, + with_bias=False + ) + + loaded_out = loaded_model.forward(input_x) + expected_out = bigdl_model.forward(input_x) + + assert(np.array_equal(loaded_out, expected_out)) + + +class TestGather(object): + + def test_gather(self): + axis = 0 + input_x = np.array([ + [1.0, 1.2], + [2.3, 3.4], + [4.5, 5.7], + ], dtype=float) + indices_val = np.array([[0, 1], [1, 2]], dtype=float) + expected_out = np.array([[[1, 1.2], [2.3, 3.4]], + [[2.3, 3.4], [4.5, 5.7]]], dtype=float) + input_shape = input_x.shape + indices_shape = indices_val.shape + output_shape = [2, 2, 2] + + # Create one output (ValueInfoProto) + data = onnx.helper.make_tensor_value_info('data', onnx.TensorProto.FLOAT, input_shape) + indices = onnx.helper.make_tensor_value_info('indices', + onnx.TensorProto.FLOAT, indices_shape) + Y = onnx.helper.make_tensor_value_info('Y', onnx.TensorProto.FLOAT, output_shape) + + init_indices = onnx.helper.make_tensor( + name='indices', + data_type=onnx.TensorProto.FLOAT, + dims=indices_shape, + vals=indices_val.flatten().tolist(), + ) + + gather_node = onnx.helper.make_node( + op_type='Gather', + inputs=['data', 'indices'], + outputs=['Y'], + axis=axis + ) + + # Create the graph (GraphProto) + onnx_graph = onnx.helper.make_graph( + nodes=[gather_node], + name='test-gather', + inputs=[data, indices], + outputs=[Y], + initializer=[init_indices] + ) + + # Create the model (ModelProto) + onnx_model = onnx.helper.make_model(onnx_graph, producer_name='ONNX') + onnx.checker.check_model(onnx_model) + + loaded_model = load_model_proto(onnx_model) + loaded_out = loaded_model.forward([input_x, indices_val]) + + assert(np.allclose(loaded_out, expected_out)) + + +class TestGemm(object): + + def test_gemm(self): + mata_shape = [2, 7] + matb_shape = [7, 4] + matc_shape = [2, 4] + output_shape = [2, 4] + alpha = np.round(np.random.rand(), 2) + beta = np.round(np.random.rand(), 2) + trans_a, trans_b = 0, 0 + input_x = np.random.random(mata_shape) + b_val = np.random.random(matb_shape) + c_val = np.random.random(matc_shape) + + # Create one output (ValueInfoProto) + a = onnx.helper.make_tensor_value_info('a', onnx.TensorProto.FLOAT, mata_shape) + b = onnx.helper.make_tensor_value_info('b', onnx.TensorProto.FLOAT, matb_shape) + c = onnx.helper.make_tensor_value_info('c', onnx.TensorProto.FLOAT, matc_shape) + Y = onnx.helper.make_tensor_value_info('Y', onnx.TensorProto.FLOAT, output_shape) + + init_b = onnx.helper.make_tensor( + name='b', + data_type=onnx.TensorProto.FLOAT, + dims=matb_shape, + vals=b_val.flatten().tolist(), + ) + + init_c = onnx.helper.make_tensor( + name='c', + data_type=onnx.TensorProto.FLOAT, + dims=matc_shape, + vals=c_val.flatten().tolist(), + ) + + gemm_node = onnx.helper.make_node( + op_type='Gemm', + inputs=['a', 'b', 'c'], + outputs=['Y'], + alpha=alpha, + beta=beta, + transA=trans_a, + transB=trans_b + ) + + # Create the graph (GraphProto) + onnx_graph = onnx.helper.make_graph( + nodes=[gemm_node], + name='test-gather', + inputs=[a, b, c], + outputs=[Y], + initializer=[init_b, init_c] + ) + + # Create the model (ModelProto) + onnx_model = onnx.helper.make_model(onnx_graph, producer_name='ONNX') + onnx.checker.check_model(onnx_model) + + bigdl_model = Gemm(b_val, c_val, + alpha=alpha, beta=beta, trans_a=trans_a, trans_b=trans_b) + loaded_model = load_model_proto(onnx_model) + + expected_out = bigdl_model.forward(input_x) + loaded_out = loaded_model.forward(input_x) + + assert(np.array_equal(expected_out, loaded_out)) + + +class TestMaxPool(object): + + def test_max_poll(self): + kernel_width, kernel_height = 2, 2 + stride_width, stride_height = 1, 1 + pad_width, pad_height = 0, 0 + ceil_mode = 0 + input_shape = [1, 3, 224, 224] + output_shape = [1, 3, 223, 223] + input_x = np.random.random(input_shape) + + # Create one output (ValueInfoProto) + X = onnx.helper.make_tensor_value_info('X', onnx.TensorProto.FLOAT, input_shape) + Y = onnx.helper.make_tensor_value_info('Y', onnx.TensorProto.FLOAT, output_shape) + + maxpool_node = onnx.helper.make_node( + op_type='MaxPool', + inputs=['X'], + outputs=['Y'], + kernel_shape=(kernel_width, kernel_height), + ) + + # Create the graph (GraphProto) + onnx_graph = onnx.helper.make_graph( + nodes=[maxpool_node], + name='test-maxpool', + inputs=[X], + outputs=[Y], + ) + + # Create the model (ModelProto) + onnx_model = onnx.helper.make_model(onnx_graph, producer_name='ONNX') + onnx.checker.check_model(onnx_model) + + loaded_model = load_model_proto(onnx_model) + bigdl_model = SpatialMaxPooling( + kw=kernel_width, + kh=kernel_height, + dw=stride_width, + dh=stride_height, + pad_w=pad_width, + pad_h=pad_height, + to_ceil=False if ceil_mode == 0 else True + ) + + loaded_out = loaded_model.forward(input_x) + expected_out = bigdl_model.forward(input_x) + + assert(np.array_equal(expected_out, loaded_out)) + + +class TestRelu(object): + + def test_relu(self): + input_shape = [1, 3, 224, 224] + output_shape = [1, 3, 224, 224] + input_x = np.random.random(input_shape) + + # Create one output (ValueInfoProto) + X = onnx.helper.make_tensor_value_info('X', onnx.TensorProto.FLOAT, input_shape) + Y = onnx.helper.make_tensor_value_info('Y', onnx.TensorProto.FLOAT, output_shape) + + relu_node = onnx.helper.make_node( + op_type='Relu', + inputs=['X'], + outputs=['Y'] + ) + + # Create the graph (GraphProto) + onnx_graph = onnx.helper.make_graph( + nodes=[relu_node], + name='test-relu', + inputs=[X], + outputs=[Y], + ) + + # Create the model (ModelProto) + onnx_model = onnx.helper.make_model(onnx_graph, producer_name='ONNX') + onnx.checker.check_model(onnx_model) + + bigdl_model = ReLU() + loaded_model = load_model_proto(onnx_model) + + expected_out = bigdl_model.forward(input_x) + loaded_out = loaded_model.forward(input_x) + + assert(np.array_equal(expected_out, loaded_out)) + + +class TestReshape(object): + + def test_reshape(self): + + input_x = np.random.random([1, 3, 4, 4]) + # Create one output (ValueInfoProto) + X = onnx.helper.make_tensor_value_info('X', onnx.TensorProto.FLOAT, [1, 3, 4, 4]) + Y = onnx.helper.make_tensor_value_info('Y', onnx.TensorProto.FLOAT, [2, 3, 8]) + shape = onnx.helper.make_tensor_value_info('shape', onnx.TensorProto.FLOAT, [1, 3]) + + init_shape = onnx.helper.make_tensor( + name='shape', # type: Text + data_type=onnx.TensorProto.FLOAT, # type: int + dims=[1, 3], # type: Sequence[int] + vals=[2, 3, 8], # type: Any + ) + + reshape_node = onnx.helper.make_node( + op_type='Reshape', + inputs=['X', 'shape'], + outputs=['Y'] + ) + + # Create the graph (GraphProto) + onnx_graph = onnx.helper.make_graph( + nodes=[reshape_node], + name='test-reshape', + inputs=[X, shape], + outputs=[Y], + initializer=[init_shape] + ) + + # Create the model (ModelProto) + onnx_model = onnx.helper.make_model(onnx_graph, producer_name='ONNX') + onnx.checker.check_model(onnx_model) + + loaded_model = load_model_proto(onnx_model) + bigdl_model = Reshape([2, 3, 8]) + + loaded_out = loaded_model.forward(input_x) + expected_out = bigdl_model.forward(input_x) + + assert(np.array_equal(expected_out, loaded_out)) + + +class TestShape(object): + + def test_shape(self): + input_shape = [3, 4, 5] + input_x = np.random.random(input_shape) + # Create one output (ValueInfoProto) + X = onnx.helper.make_tensor_value_info('X', onnx.TensorProto.FLOAT, input_shape) + Y = onnx.helper.make_tensor_value_info('Y', onnx.TensorProto.FLOAT, [1]) + + shape_node = onnx.helper.make_node( + op_type='Shape', + inputs=['X'], + outputs=['Y'], + ) + + # Create the graph (GraphProto) + onnx_graph = onnx.helper.make_graph( + nodes=[shape_node], + name='test-shape', + inputs=[X], + outputs=[Y], + ) + + # Create the model (ModelProto) + onnx_model = onnx.helper.make_model(onnx_graph, producer_name='ONNX') + onnx.checker.check_model(onnx_model) + + bigdl_model = Shape() + loaded_model = load_model_proto(onnx_model) + + expected_out = bigdl_model.forward(input_x) + loaded_out = loaded_model.forward(input_x) + + assert(np.array_equal(expected_out, loaded_out)) + + +class TestSoftmax(object): + + def test_softmax(self): + input_shape = [1, 3, 224, 224] + output_shape = [1, 3, 224, 224] + input_x = np.random.random(input_shape) + # Create one output (ValueInfoProto) + X = onnx.helper.make_tensor_value_info('X', onnx.TensorProto.FLOAT, input_shape) + Y = onnx.helper.make_tensor_value_info('Y', onnx.TensorProto.FLOAT, output_shape) + + softmax_node = onnx.helper.make_node( + op_type='Softmax', + inputs=['X'], + outputs=['Y'] + ) + + # Create the graph (GraphProto) + onnx_graph = onnx.helper.make_graph( + nodes=[softmax_node], + name='test-softmax', + inputs=[X], + outputs=[Y], + ) + + # Create the model (ModelProto) + onnx_model = onnx.helper.make_model(onnx_graph, producer_name='ONNX') + onnx.checker.check_model(onnx_model) + + loaded_model = load_model_proto(onnx_model) + bigdl_model = SoftMax() + + loaded_out = loaded_model.forward(input_x) + expected_out = bigdl_model.forward(input_x) + + assert(np.array_equal(expected_out, loaded_out)) + + +class TestSum(object): + + def test_sum(self): + input_shape = [2, 3] + input_x1 = np.random.random(input_shape) + input_x2 = np.random.random(input_shape) + # Create one output (ValueInfoProto) + X = onnx.helper.make_tensor_value_info('X', onnx.TensorProto.FLOAT, [4, 3]) + Y = onnx.helper.make_tensor_value_info('Y', onnx.TensorProto.FLOAT, [1, 3]) + + sum_node = onnx.helper.make_node( + op_type='Sum', + inputs=['X'], + outputs=['Y'] + ) + + # Create the graph (GraphProto) + onnx_graph = onnx.helper.make_graph( + nodes=[sum_node], + name='test-sum', + inputs=[X], + outputs=[Y], + ) + + # Create the model (ModelProto) + onnx_model = onnx.helper.make_model(onnx_graph, producer_name='ONNX') + onnx.checker.check_model(onnx_model) + + loaded_model = load_model_proto(onnx_model) + bigdl_model = CAddTable() + + expected_out = bigdl_model.forward([input_x1, input_x2]) + loaded_out = loaded_model.forward([input_x1, input_x2]) + + assert(np.array_equal(expected_out, loaded_out)) + + +class TestUnsqueeze(object): + + def test_unsqueeze(self): + axis = 0 + input_shape = [3, 4, 5] + output_shape = [1, 3, 4, 5] + input_x = np.random.random([3, 4, 5]) + + # Create one output (ValueInfoProto) + X = onnx.helper.make_tensor_value_info('X', onnx.TensorProto.FLOAT, input_shape) + Y = onnx.helper.make_tensor_value_info('Y', onnx.TensorProto.FLOAT, output_shape) + + unsqueeze_node = onnx.helper.make_node( + op_type='Unsqueeze', + inputs=['X'], + outputs=['Y'], + axes=[axis], + ) + + # Create the graph (GraphProto) + onnx_graph = onnx.helper.make_graph( + nodes=[unsqueeze_node], + name='test-unsqueeze', + inputs=[X], + outputs=[Y], + ) + + # Create the model (ModelProto) + onnx_model = onnx.helper.make_model(onnx_graph, producer_name='ONNX') + onnx.checker.check_model(onnx_model) + + bigdl_model = Unsqueeze(pos=axis, num_input_dims=len(input_shape)) + expected_out = bigdl_model.forward(input_x) + + loaded_model = load_model_proto(onnx_model) + loaded_out = loaded_model.forward(input_x) + + assert(np.array_equal(expected_out, loaded_out)) + + +def main(): + pytest.main([__file__]) + + +if __name__ == "__main__": + pytest.main([__file__]) From 3887a488dd73d1917c1939afd8ad20e39f2b9e40 Mon Sep 17 00:00:00 2001 From: Xiao Date: Thu, 24 Oct 2019 14:46:16 +0800 Subject: [PATCH 756/823] fix TimeDistributedCriterion() lack of parameter of dimension issue (#2940) --- python/dllib/src/bigdl/dllib/nn/criterion.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/nn/criterion.py b/python/dllib/src/bigdl/dllib/nn/criterion.py index f2076860bf3..038a475b8f8 100644 --- a/python/dllib/src/bigdl/dllib/nn/criterion.py +++ b/python/dllib/src/bigdl/dllib/nn/criterion.py @@ -635,9 +635,9 @@ class TimeDistributedCriterion(Criterion): creating: createTimeDistributedCriterion ''' - def __init__(self, criterion, size_average=False, bigdl_type="float"): + def __init__(self, criterion, size_average=False, dimension=2, bigdl_type="float"): super(TimeDistributedCriterion, self).__init__( - None, bigdl_type, criterion, size_average) + None, bigdl_type, criterion, size_average, dimension) class CrossEntropyCriterion(Criterion): From c51590e54fc92f2a0cd16f4ec3cdc239c4ea74ed Mon Sep 17 00:00:00 2001 From: Xiao Date: Thu, 24 Oct 2019 14:46:16 +0800 Subject: [PATCH 757/823] fix TimeDistributedCriterion() lack of parameter of dimension issue (#2940) --- python/dllib/src/bigdl/dllib/nn/criterion.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/nn/criterion.py b/python/dllib/src/bigdl/dllib/nn/criterion.py index f2076860bf3..038a475b8f8 100644 --- a/python/dllib/src/bigdl/dllib/nn/criterion.py +++ b/python/dllib/src/bigdl/dllib/nn/criterion.py @@ -635,9 +635,9 @@ class TimeDistributedCriterion(Criterion): creating: createTimeDistributedCriterion ''' - def __init__(self, criterion, size_average=False, bigdl_type="float"): + def __init__(self, criterion, size_average=False, dimension=2, bigdl_type="float"): super(TimeDistributedCriterion, self).__init__( - None, bigdl_type, criterion, size_average) + None, bigdl_type, criterion, size_average, dimension) class CrossEntropyCriterion(Criterion): From ecde8183fd9ef0af0f87312a9ebe6eefdc02b8ff Mon Sep 17 00:00:00 2001 From: Jerry Wu Date: Thu, 24 Oct 2019 16:22:08 +0800 Subject: [PATCH 758/823] revert back api (#2943) --- python/dllib/test/bigdl/test_simple_integration.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/python/dllib/test/bigdl/test_simple_integration.py b/python/dllib/test/bigdl/test_simple_integration.py index a531fa9c0b0..4b47df29f2c 100644 --- a/python/dllib/test/bigdl/test_simple_integration.py +++ b/python/dllib/test/bigdl/test_simple_integration.py @@ -22,7 +22,6 @@ from bigdl.util.common import _py2java from bigdl.nn.initialization_method import * from bigdl.dataset import movielens -from pyspark.rdd import RDD import numpy as np import tempfile import pytest @@ -545,14 +544,9 @@ def test_predict(self): assert_allclose(p_with_batch[i], ground_label[i], atol=1e-6, rtol=0) predict_class = model.predict_class(predict_data) - if isinstance(predict_class, RDD): - for sample in predict_class.collect(): - predict_label = sample.label.to_ndarray() - assert np.argmax(predict_label) == 0 - else: - predict_labels = predict_class.take(6) - for i in range(0, total_length): - assert predict_labels[i] == 1 + predict_labels = predict_class.take(6) + for i in range(0, total_length): + assert predict_labels[i] == 1 def test_predict_image(self): resource_path = os.path.join(os.path.split(__file__)[0], "resources") @@ -686,7 +680,7 @@ def test_model_broadcast(self): model = Linear(3, 2) broadcasted = broadcast_model(self.sc, model) input_data = np.random.rand(3) - output = self.sc.parallelize([input_data], 1)\ + output = self.sc.parallelize([input_data], 1) \ .map(lambda x: broadcasted.value.forward(x)).first() expected = model.forward(input_data) From fbc5ecf7dc9025209729391720513f86e28f3483 Mon Sep 17 00:00:00 2001 From: LeicongLi Date: Mon, 28 Oct 2019 13:36:21 +0800 Subject: [PATCH 759/823] ONNX ResNet example (#2939) * add onnx resnet example * add doc for onnx * add doc for onnx * clean up --- .../src/bigdl/dllib/examples/onnx/README.md | 46 +++++++++++++++++++ .../src/bigdl/dllib/examples/onnx/__init__.py | 15 ++++++ .../dllib/examples/onnx/load_onnx_resnet.py | 30 ++++++++++++ 3 files changed, 91 insertions(+) create mode 100644 python/dllib/src/bigdl/dllib/examples/onnx/README.md create mode 100644 python/dllib/src/bigdl/dllib/examples/onnx/__init__.py create mode 100644 python/dllib/src/bigdl/dllib/examples/onnx/load_onnx_resnet.py diff --git a/python/dllib/src/bigdl/dllib/examples/onnx/README.md b/python/dllib/src/bigdl/dllib/examples/onnx/README.md new file mode 100644 index 00000000000..1dc94bbef1c --- /dev/null +++ b/python/dllib/src/bigdl/dllib/examples/onnx/README.md @@ -0,0 +1,46 @@ +# ONNX ResNet-50 Model Loading in BigDL + + +## Download model file + * [Download model](https://s3.amazonaws.com/download.onnx/models/opset_9/resnet50.tar.gz) + * Uncompress the file + ``` + tar -zxvf resnet50.tar.gz + + . + ├── model.onnx + ├── test_data_set_0 + ├── test_data_set_1 + └── .... + ``` + +## How to run this example: + * Import library dependencies +``` +import numpy as np +from bigdl.contrib.onnx import load +``` + + * Set target ONNX ResNet-50 model path + ``` + restnet_path = "uncompressed/file/path/model.onnx" + ``` + + * Load ONNX ResNet-50 model into BigDL + ``` + restnet = load(restnet_path) + ``` + + * Create a sample tensor and pass it through loaded BigDL model + ``` + restnet_tensor = np.random.random([10, 3, 224, 224]) + restnet_out = restnet.forward(restnet_tensor) + ``` + + +## Known issues: + * ONNX feature only has Python API in BigDL. + * Loaded ONNX model is limited for inference. + * Most of operators defined in ONNX are not being supported by BigDL for now. + * Missing feature of exporting BigDL model into ONNX format. + \ No newline at end of file diff --git a/python/dllib/src/bigdl/dllib/examples/onnx/__init__.py b/python/dllib/src/bigdl/dllib/examples/onnx/__init__.py new file mode 100644 index 00000000000..2151a805423 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/examples/onnx/__init__.py @@ -0,0 +1,15 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/python/dllib/src/bigdl/dllib/examples/onnx/load_onnx_resnet.py b/python/dllib/src/bigdl/dllib/examples/onnx/load_onnx_resnet.py new file mode 100644 index 00000000000..d4c2bfc8371 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/examples/onnx/load_onnx_resnet.py @@ -0,0 +1,30 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import numpy as np +from bigdl.contrib.onnx import load + + +def load_onnx_resnet(): + restnet_path = "./resnet-50.onnx" + restnet_tensor = np.random.random([10, 3, 224, 224]) + restnet = load(restnet_path) + restnet_out = restnet.forward(restnet_tensor) + return restnet_out + + +if __name__ == "__main__": + load_onnx_resnet() From d6096dee9b499d6b5ac7556f473d889843a60842 Mon Sep 17 00:00:00 2001 From: LeicongLi Date: Mon, 28 Oct 2019 13:36:21 +0800 Subject: [PATCH 760/823] ONNX ResNet example (#2939) * add onnx resnet example * add doc for onnx * add doc for onnx * clean up --- .../src/bigdl/dllib/examples/onnx/README.md | 46 +++++++++++++++++++ .../src/bigdl/dllib/examples/onnx/__init__.py | 15 ++++++ .../dllib/examples/onnx/load_onnx_resnet.py | 30 ++++++++++++ 3 files changed, 91 insertions(+) create mode 100644 python/dllib/src/bigdl/dllib/examples/onnx/README.md create mode 100644 python/dllib/src/bigdl/dllib/examples/onnx/__init__.py create mode 100644 python/dllib/src/bigdl/dllib/examples/onnx/load_onnx_resnet.py diff --git a/python/dllib/src/bigdl/dllib/examples/onnx/README.md b/python/dllib/src/bigdl/dllib/examples/onnx/README.md new file mode 100644 index 00000000000..1dc94bbef1c --- /dev/null +++ b/python/dllib/src/bigdl/dllib/examples/onnx/README.md @@ -0,0 +1,46 @@ +# ONNX ResNet-50 Model Loading in BigDL + + +## Download model file + * [Download model](https://s3.amazonaws.com/download.onnx/models/opset_9/resnet50.tar.gz) + * Uncompress the file + ``` + tar -zxvf resnet50.tar.gz + + . + ├── model.onnx + ├── test_data_set_0 + ├── test_data_set_1 + └── .... + ``` + +## How to run this example: + * Import library dependencies +``` +import numpy as np +from bigdl.contrib.onnx import load +``` + + * Set target ONNX ResNet-50 model path + ``` + restnet_path = "uncompressed/file/path/model.onnx" + ``` + + * Load ONNX ResNet-50 model into BigDL + ``` + restnet = load(restnet_path) + ``` + + * Create a sample tensor and pass it through loaded BigDL model + ``` + restnet_tensor = np.random.random([10, 3, 224, 224]) + restnet_out = restnet.forward(restnet_tensor) + ``` + + +## Known issues: + * ONNX feature only has Python API in BigDL. + * Loaded ONNX model is limited for inference. + * Most of operators defined in ONNX are not being supported by BigDL for now. + * Missing feature of exporting BigDL model into ONNX format. + \ No newline at end of file diff --git a/python/dllib/src/bigdl/dllib/examples/onnx/__init__.py b/python/dllib/src/bigdl/dllib/examples/onnx/__init__.py new file mode 100644 index 00000000000..2151a805423 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/examples/onnx/__init__.py @@ -0,0 +1,15 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/python/dllib/src/bigdl/dllib/examples/onnx/load_onnx_resnet.py b/python/dllib/src/bigdl/dllib/examples/onnx/load_onnx_resnet.py new file mode 100644 index 00000000000..d4c2bfc8371 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/examples/onnx/load_onnx_resnet.py @@ -0,0 +1,30 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import numpy as np +from bigdl.contrib.onnx import load + + +def load_onnx_resnet(): + restnet_path = "./resnet-50.onnx" + restnet_tensor = np.random.random([10, 3, 224, 224]) + restnet = load(restnet_path) + restnet_out = restnet.forward(restnet_tensor) + return restnet_out + + +if __name__ == "__main__": + load_onnx_resnet() From c01ecc8338c43935c9c0cdcb9426d10e8069b5a5 Mon Sep 17 00:00:00 2001 From: Firecrackerxox Date: Mon, 28 Oct 2019 21:46:59 +0800 Subject: [PATCH 761/823] Python MKLDNN examples for CNN(LeNet) and RNN(LSTM) (#2932) --- .../src/bigdl/dllib/models/lenet/README.md | 35 ++++++++++++- .../src/bigdl/dllib/models/lenet/lenet5.py | 19 +++++++ .../src/bigdl/dllib/models/rnn/README.md | 28 ++++++++++ .../src/bigdl/dllib/models/rnn/rnnexample.py | 52 ++++++++++++++++--- python/dllib/src/bigdl/dllib/nn/layer.py | 13 +++++ python/dllib/src/bigdl/dllib/utils/common.py | 2 + 6 files changed, 140 insertions(+), 9 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/models/lenet/README.md b/python/dllib/src/bigdl/dllib/models/lenet/README.md index 271cd49a0df..c12172d41bc 100644 --- a/python/dllib/src/bigdl/dllib/models/lenet/README.md +++ b/python/dllib/src/bigdl/dllib/models/lenet/README.md @@ -48,7 +48,6 @@ We would train a LeNet model in spark local mode with the following commands and --dataPath /tmp/mnist ``` - * ```--action``` it can be train or test. * ```--dataPath``` option can be used to set the path for downloading mnist data, the default value is /tmp/mnist. Make sure that you have write permission to the specified path. @@ -63,6 +62,38 @@ We would train a LeNet model in spark local mode with the following commands and * ```--checkpointPath``` option can be used to set checkpoint path for saving model, the default value is /tmp/lenet5/. +#####In order to use MKL-DNN as the backend, you should: +1. Define a graph model with Model or convert a sequential model to a graph model using: + ``` + convertedModel = sequentialModel.to_graph() + ``` +2. Specify the input and output formats of it. + For example: + ``` + theDefinedModel.set_input_formats([theInputFormatIndex]) + theDefinedModel.set_output_formats([theOutputFormatIndex]) + ``` + BigDL needs these format information to build a graph running with MKL-DNN backend. + + The format index of input or output format can be checked + in: + ``` + ${BigDL-core}/native-dnn/src/main/java/com/intel/analytics/bigdl/mkl/Memory.java + + For instance: + public static final int nchw = 7; + means the index of format nchw is 7. + + public static final int nc = 4; + means the index of format nc is 4. + + ``` +3. Run spark-submit command with correct configurations + ``` + --conf "spark.driver.extraJavaOptions=-Dbigdl.engineType=mkldnn" + --conf "spark.executor.extraJavaOptions=-Dbigdl.engineType=mkldnn" + ``` + To verify the accuracy, search "accuracy" from log: ``` @@ -75,4 +106,4 @@ INFO DistriOptimizer$:522 - Top1Accuracy is Accuracy(correct: 9572, count: 1000 Or you can train a LeNet model directly in shell after installing BigDL from pip: ``` python ${BigDL_HOME}/pyspark/bigdl/models/lenet/lenet5.py -``` \ No newline at end of file +``` diff --git a/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py b/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py index 9ccd7e1e5f7..6ba9bb0aa1e 100644 --- a/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py +++ b/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py @@ -37,6 +37,17 @@ def build_model(class_num): model.add(Tanh()) model.add(Linear(100, class_num)) model.add(LogSoftMax()) + + # To use MKL-DNN backend, the model has to be a graph model with input and output formats set. + # Sequential model cannot be used in this case, so we convert it to a graph model. + if get_bigdl_engine_type() == "MklDnn": + model = model.to_graph() + + # The format index of input or output format can be checked + # in: ${BigDL-core}/native-dnn/src/main/java/com/intel/analytics/bigdl/mkl/Memory.java + model.set_input_formats([7]) # Set input format to nchw + model.set_output_formats([4]) # Set output format to nc + return model @@ -57,6 +68,14 @@ def build_model(class_num): show_bigdl_info_logs() init_engine() + # In order to use MklDnn as the backend, you should: + # 1. Define a graph model with Model(graph container) or convert a sequential model to a graph model + # 2. Specify the input and output formats of it. + # BigDL needs these format information to build a graph running with MKL-DNN backend + # 3. Run spark-submit command with correct configurations + # --conf "spark.driver.extraJavaOptions=-Dbigdl.engineType=mkldnn" + # --conf "spark.executor.extraJavaOptions=-Dbigdl.engineType=mkldnn" + if options.action == "train": (train_data, test_data) = preprocess_mnist(sc, options) diff --git a/python/dllib/src/bigdl/dllib/models/rnn/README.md b/python/dllib/src/bigdl/dllib/models/rnn/README.md index 0914f91e46f..1af01830625 100644 --- a/python/dllib/src/bigdl/dllib/models/rnn/README.md +++ b/python/dllib/src/bigdl/dllib/models/rnn/README.md @@ -103,6 +103,34 @@ PYSPARK_DRIVER_PYTHON=./venv/bin/python PYSPARK_PYTHON=./venv.zip/venv/bin/pytho * `--dampening` dampening for momentum, the default value is 0. * `--maxEpoch` max number of epochs to train, the default value is 30. +#####In order to use MKL-DNN as the backend, you should: +1. Define a graph model with Model or convert a sequential model to a graph model using: + ``` + convertedModel = sequentialModel.to_graph() + ``` +2. Specify the input and output formats of it. + For example: + ``` + theDefinedModel.set_input_formats([theInputFormatIndex]) + theDefinedModel.set_output_formats([theOutputFormatIndex]) + ``` + BigDL needs these format information to build IRGraph from StaticGraph for MklDnn computing. + + The format index of input or output format can be checked + in: + ``` + ${BigDL-core}/native-dnn/src/main/java/com/intel/analytics/bigdl/mkl/Memory.java + + For instance: + public static final int ntc = 27; + means the index of format ntc is 27. + ``` +3. Run spark-submit command with correct configurations + ``` + --conf "spark.driver.extraJavaOptions=-Dbigdl.engineType=mkldnn" + --conf "spark.executor.extraJavaOptions=-Dbigdl.engineType=mkldnn" + ``` + ## Expected Training Output Users can see the Loss of the model printed by the program. The Loss, in this case, is the perplexity of the language model. The lower, the better. ``` diff --git a/python/dllib/src/bigdl/dllib/models/rnn/rnnexample.py b/python/dllib/src/bigdl/dllib/models/rnn/rnnexample.py index b387928e71c..56061a29adf 100644 --- a/python/dllib/src/bigdl/dllib/models/rnn/rnnexample.py +++ b/python/dllib/src/bigdl/dllib/models/rnn/rnnexample.py @@ -116,12 +116,39 @@ def padding(features, label, length): return sample_rdd, val_sample_rdd, total_vocab_len -def build_model(input_size, hidden_size, output_size): - model = Sequential() - model.add(Recurrent() - .add(RnnCell(input_size, hidden_size, Tanh())))\ - .add(TimeDistributed(Linear(hidden_size, output_size))) - model.reset() +def build_model(input_size, hidden_size, output_size, model_type): + # Model Type is simple RNN + if model_type == "rnn": + model = Sequential() + model.add(Recurrent() + .add(RnnCell(input_size, hidden_size, Tanh())))\ + .add(TimeDistributed(Linear(hidden_size, output_size))) + model.reset() + + # Simple RNN with MKL-DNN backend is unsupported for now. + if get_bigdl_engine_type() == "MklDnn": + raise Exception("Simple RNN is unsupported with MKL-DNN backend") + + # Model Type is LSTM + elif model_type == "lstm": + model = Sequential() + model.add(Recurrent() + .add(LSTM(input_size, hidden_size)))\ + .add(TimeDistributed(Linear(hidden_size, output_size))) + model.reset() + + # LSTM with MKL-DNN backend + if get_bigdl_engine_type() == "MklDnn": + # To use MKL-DNN backend, the model has to be a graph model with + # input and output formats set. Sequential model cannot be used in + # this case, so we convert it to a graph model. + model = model.to_graph() + + # The format index of input or output format can be checked + # in: ${BigDL-core}/native-dnn/src/main/java/com/intel/analytics/bigdl/mkl/Memory.java + model.set_input_formats([27]) # Set input format to ntc + model.set_output_formats([27]) # Set output format to ntc + return model if __name__ == "__main__": @@ -136,6 +163,7 @@ def build_model(input_size, hidden_size, output_size): parser.add_option("--hiddenSize", dest="hidden_size", default="40") parser.add_option("--vocabSize", dest="vob_size", default="4000") parser.add_option("--maxEpoch", dest="max_epoch", default="30") + parser.add_option("--modelType", dest="model_type", default="rnn") (options, args) = parser.parse_args(sys.argv) @@ -149,6 +177,7 @@ def build_model(input_size, hidden_size, output_size): max_epoch = int(options.max_epoch) folder = options.folder training_split = 0.8 + model_type = str(options.model_type) sc = SparkContext(appName="simplernn_example", conf=create_spark_conf()) @@ -156,10 +185,19 @@ def build_model(input_size, hidden_size, output_size): show_bigdl_info_logs() init_engine() + # In order to use MklDnn as the backend, you should: + # 1. Define a model with Model(graph container) or convert a sequential model to a graph model + # 2. Specify the input and output formats of it. + # BigDL needs these format information to build a graph running with MKL-DNN backend + # 3. Run spark-submit command with correct configurations + # --conf "spark.driver.extraJavaOptions=-Dbigdl.engineType=mkldnn" + # --conf "spark.executor.extraJavaOptions=-Dbigdl.engineType=mkldnn" + # LSTM supports MklDnn backend. Simple RNN does not for now. + (train_rdd, val_rdd, vob_size) = prepare_data(sc, folder, vob_size, training_split) optimizer = Optimizer( - model=build_model(vob_size, hidden_size, vob_size), + model=build_model(vob_size, hidden_size, vob_size, model_type), training_rdd=train_rdd, criterion=TimeDistributedCriterion(CrossEntropyCriterion(), size_average=True), batch_size=batch_size, diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 5de838a3493..6d02d86540e 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -1251,6 +1251,19 @@ def from_jvalue(jvalue, bigdl_type="float"): model.value = jvalue return model + def to_graph(self): + """ + Convert a sequential model (Sequential) to a graph model (Model) + :return: A Python graph model + """ + jvalue = callBigDlFunc(self.bigdl_type, + "toGraph", + self.value) + model = Model.from_jvalue(jvalue) + return model + + + class TemporalConvolution(Layer): ''' diff --git a/python/dllib/src/bigdl/dllib/utils/common.py b/python/dllib/src/bigdl/dllib/utils/common.py index 30ef0bbc684..e57621b9a46 100644 --- a/python/dllib/src/bigdl/dllib/utils/common.py +++ b/python/dllib/src/bigdl/dllib/utils/common.py @@ -422,6 +422,8 @@ def init_engine(bigdl_type="float"): # Spark context is supposed to have been created when init_engine is called get_spark_context()._jvm.org.apache.spark.bigdl.api.python.BigDLSerDe.initialize() +def get_bigdl_engine_type(bigdl_type="float"): + return callBigDlFunc(bigdl_type, "getEngineType") def init_executor_gateway(sc, bigdl_type="float"): callBigDlFunc(bigdl_type, "initExecutorGateway", sc, sc._gateway._gateway_client.port) From bb5227ebf2b1703d7dbaba3dd656a1d561f4d7d7 Mon Sep 17 00:00:00 2001 From: Firecrackerxox Date: Mon, 28 Oct 2019 21:46:59 +0800 Subject: [PATCH 762/823] Python MKLDNN examples for CNN(LeNet) and RNN(LSTM) (#2932) --- .../src/bigdl/dllib/models/lenet/README.md | 35 ++++++++++++- .../src/bigdl/dllib/models/lenet/lenet5.py | 19 +++++++ .../src/bigdl/dllib/models/rnn/README.md | 28 ++++++++++ .../src/bigdl/dllib/models/rnn/rnnexample.py | 52 ++++++++++++++++--- python/dllib/src/bigdl/dllib/nn/layer.py | 13 +++++ python/dllib/src/bigdl/utils/common.py | 2 + 6 files changed, 140 insertions(+), 9 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/models/lenet/README.md b/python/dllib/src/bigdl/dllib/models/lenet/README.md index 271cd49a0df..c12172d41bc 100644 --- a/python/dllib/src/bigdl/dllib/models/lenet/README.md +++ b/python/dllib/src/bigdl/dllib/models/lenet/README.md @@ -48,7 +48,6 @@ We would train a LeNet model in spark local mode with the following commands and --dataPath /tmp/mnist ``` - * ```--action``` it can be train or test. * ```--dataPath``` option can be used to set the path for downloading mnist data, the default value is /tmp/mnist. Make sure that you have write permission to the specified path. @@ -63,6 +62,38 @@ We would train a LeNet model in spark local mode with the following commands and * ```--checkpointPath``` option can be used to set checkpoint path for saving model, the default value is /tmp/lenet5/. +#####In order to use MKL-DNN as the backend, you should: +1. Define a graph model with Model or convert a sequential model to a graph model using: + ``` + convertedModel = sequentialModel.to_graph() + ``` +2. Specify the input and output formats of it. + For example: + ``` + theDefinedModel.set_input_formats([theInputFormatIndex]) + theDefinedModel.set_output_formats([theOutputFormatIndex]) + ``` + BigDL needs these format information to build a graph running with MKL-DNN backend. + + The format index of input or output format can be checked + in: + ``` + ${BigDL-core}/native-dnn/src/main/java/com/intel/analytics/bigdl/mkl/Memory.java + + For instance: + public static final int nchw = 7; + means the index of format nchw is 7. + + public static final int nc = 4; + means the index of format nc is 4. + + ``` +3. Run spark-submit command with correct configurations + ``` + --conf "spark.driver.extraJavaOptions=-Dbigdl.engineType=mkldnn" + --conf "spark.executor.extraJavaOptions=-Dbigdl.engineType=mkldnn" + ``` + To verify the accuracy, search "accuracy" from log: ``` @@ -75,4 +106,4 @@ INFO DistriOptimizer$:522 - Top1Accuracy is Accuracy(correct: 9572, count: 1000 Or you can train a LeNet model directly in shell after installing BigDL from pip: ``` python ${BigDL_HOME}/pyspark/bigdl/models/lenet/lenet5.py -``` \ No newline at end of file +``` diff --git a/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py b/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py index 9ccd7e1e5f7..6ba9bb0aa1e 100644 --- a/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py +++ b/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py @@ -37,6 +37,17 @@ def build_model(class_num): model.add(Tanh()) model.add(Linear(100, class_num)) model.add(LogSoftMax()) + + # To use MKL-DNN backend, the model has to be a graph model with input and output formats set. + # Sequential model cannot be used in this case, so we convert it to a graph model. + if get_bigdl_engine_type() == "MklDnn": + model = model.to_graph() + + # The format index of input or output format can be checked + # in: ${BigDL-core}/native-dnn/src/main/java/com/intel/analytics/bigdl/mkl/Memory.java + model.set_input_formats([7]) # Set input format to nchw + model.set_output_formats([4]) # Set output format to nc + return model @@ -57,6 +68,14 @@ def build_model(class_num): show_bigdl_info_logs() init_engine() + # In order to use MklDnn as the backend, you should: + # 1. Define a graph model with Model(graph container) or convert a sequential model to a graph model + # 2. Specify the input and output formats of it. + # BigDL needs these format information to build a graph running with MKL-DNN backend + # 3. Run spark-submit command with correct configurations + # --conf "spark.driver.extraJavaOptions=-Dbigdl.engineType=mkldnn" + # --conf "spark.executor.extraJavaOptions=-Dbigdl.engineType=mkldnn" + if options.action == "train": (train_data, test_data) = preprocess_mnist(sc, options) diff --git a/python/dllib/src/bigdl/dllib/models/rnn/README.md b/python/dllib/src/bigdl/dllib/models/rnn/README.md index 0914f91e46f..1af01830625 100644 --- a/python/dllib/src/bigdl/dllib/models/rnn/README.md +++ b/python/dllib/src/bigdl/dllib/models/rnn/README.md @@ -103,6 +103,34 @@ PYSPARK_DRIVER_PYTHON=./venv/bin/python PYSPARK_PYTHON=./venv.zip/venv/bin/pytho * `--dampening` dampening for momentum, the default value is 0. * `--maxEpoch` max number of epochs to train, the default value is 30. +#####In order to use MKL-DNN as the backend, you should: +1. Define a graph model with Model or convert a sequential model to a graph model using: + ``` + convertedModel = sequentialModel.to_graph() + ``` +2. Specify the input and output formats of it. + For example: + ``` + theDefinedModel.set_input_formats([theInputFormatIndex]) + theDefinedModel.set_output_formats([theOutputFormatIndex]) + ``` + BigDL needs these format information to build IRGraph from StaticGraph for MklDnn computing. + + The format index of input or output format can be checked + in: + ``` + ${BigDL-core}/native-dnn/src/main/java/com/intel/analytics/bigdl/mkl/Memory.java + + For instance: + public static final int ntc = 27; + means the index of format ntc is 27. + ``` +3. Run spark-submit command with correct configurations + ``` + --conf "spark.driver.extraJavaOptions=-Dbigdl.engineType=mkldnn" + --conf "spark.executor.extraJavaOptions=-Dbigdl.engineType=mkldnn" + ``` + ## Expected Training Output Users can see the Loss of the model printed by the program. The Loss, in this case, is the perplexity of the language model. The lower, the better. ``` diff --git a/python/dllib/src/bigdl/dllib/models/rnn/rnnexample.py b/python/dllib/src/bigdl/dllib/models/rnn/rnnexample.py index b387928e71c..56061a29adf 100644 --- a/python/dllib/src/bigdl/dllib/models/rnn/rnnexample.py +++ b/python/dllib/src/bigdl/dllib/models/rnn/rnnexample.py @@ -116,12 +116,39 @@ def padding(features, label, length): return sample_rdd, val_sample_rdd, total_vocab_len -def build_model(input_size, hidden_size, output_size): - model = Sequential() - model.add(Recurrent() - .add(RnnCell(input_size, hidden_size, Tanh())))\ - .add(TimeDistributed(Linear(hidden_size, output_size))) - model.reset() +def build_model(input_size, hidden_size, output_size, model_type): + # Model Type is simple RNN + if model_type == "rnn": + model = Sequential() + model.add(Recurrent() + .add(RnnCell(input_size, hidden_size, Tanh())))\ + .add(TimeDistributed(Linear(hidden_size, output_size))) + model.reset() + + # Simple RNN with MKL-DNN backend is unsupported for now. + if get_bigdl_engine_type() == "MklDnn": + raise Exception("Simple RNN is unsupported with MKL-DNN backend") + + # Model Type is LSTM + elif model_type == "lstm": + model = Sequential() + model.add(Recurrent() + .add(LSTM(input_size, hidden_size)))\ + .add(TimeDistributed(Linear(hidden_size, output_size))) + model.reset() + + # LSTM with MKL-DNN backend + if get_bigdl_engine_type() == "MklDnn": + # To use MKL-DNN backend, the model has to be a graph model with + # input and output formats set. Sequential model cannot be used in + # this case, so we convert it to a graph model. + model = model.to_graph() + + # The format index of input or output format can be checked + # in: ${BigDL-core}/native-dnn/src/main/java/com/intel/analytics/bigdl/mkl/Memory.java + model.set_input_formats([27]) # Set input format to ntc + model.set_output_formats([27]) # Set output format to ntc + return model if __name__ == "__main__": @@ -136,6 +163,7 @@ def build_model(input_size, hidden_size, output_size): parser.add_option("--hiddenSize", dest="hidden_size", default="40") parser.add_option("--vocabSize", dest="vob_size", default="4000") parser.add_option("--maxEpoch", dest="max_epoch", default="30") + parser.add_option("--modelType", dest="model_type", default="rnn") (options, args) = parser.parse_args(sys.argv) @@ -149,6 +177,7 @@ def build_model(input_size, hidden_size, output_size): max_epoch = int(options.max_epoch) folder = options.folder training_split = 0.8 + model_type = str(options.model_type) sc = SparkContext(appName="simplernn_example", conf=create_spark_conf()) @@ -156,10 +185,19 @@ def build_model(input_size, hidden_size, output_size): show_bigdl_info_logs() init_engine() + # In order to use MklDnn as the backend, you should: + # 1. Define a model with Model(graph container) or convert a sequential model to a graph model + # 2. Specify the input and output formats of it. + # BigDL needs these format information to build a graph running with MKL-DNN backend + # 3. Run spark-submit command with correct configurations + # --conf "spark.driver.extraJavaOptions=-Dbigdl.engineType=mkldnn" + # --conf "spark.executor.extraJavaOptions=-Dbigdl.engineType=mkldnn" + # LSTM supports MklDnn backend. Simple RNN does not for now. + (train_rdd, val_rdd, vob_size) = prepare_data(sc, folder, vob_size, training_split) optimizer = Optimizer( - model=build_model(vob_size, hidden_size, vob_size), + model=build_model(vob_size, hidden_size, vob_size, model_type), training_rdd=train_rdd, criterion=TimeDistributedCriterion(CrossEntropyCriterion(), size_average=True), batch_size=batch_size, diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 5de838a3493..6d02d86540e 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -1251,6 +1251,19 @@ def from_jvalue(jvalue, bigdl_type="float"): model.value = jvalue return model + def to_graph(self): + """ + Convert a sequential model (Sequential) to a graph model (Model) + :return: A Python graph model + """ + jvalue = callBigDlFunc(self.bigdl_type, + "toGraph", + self.value) + model = Model.from_jvalue(jvalue) + return model + + + class TemporalConvolution(Layer): ''' diff --git a/python/dllib/src/bigdl/utils/common.py b/python/dllib/src/bigdl/utils/common.py index 30ef0bbc684..e57621b9a46 100644 --- a/python/dllib/src/bigdl/utils/common.py +++ b/python/dllib/src/bigdl/utils/common.py @@ -422,6 +422,8 @@ def init_engine(bigdl_type="float"): # Spark context is supposed to have been created when init_engine is called get_spark_context()._jvm.org.apache.spark.bigdl.api.python.BigDLSerDe.initialize() +def get_bigdl_engine_type(bigdl_type="float"): + return callBigDlFunc(bigdl_type, "getEngineType") def init_executor_gateway(sc, bigdl_type="float"): callBigDlFunc(bigdl_type, "initExecutorGateway", sc, sc._gateway._gateway_client.port) From 31d93e173ce230a99d4fc5afe99f2d0a7ff4531d Mon Sep 17 00:00:00 2001 From: Firecrackerxox Date: Tue, 29 Oct 2019 09:40:17 +0800 Subject: [PATCH 763/823] edit doc (#2948) --- python/dllib/src/bigdl/dllib/models/lenet/README.md | 2 +- python/dllib/src/bigdl/dllib/models/rnn/README.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/models/lenet/README.md b/python/dllib/src/bigdl/dllib/models/lenet/README.md index c12172d41bc..aaf9cb3a55b 100644 --- a/python/dllib/src/bigdl/dllib/models/lenet/README.md +++ b/python/dllib/src/bigdl/dllib/models/lenet/README.md @@ -62,7 +62,7 @@ We would train a LeNet model in spark local mode with the following commands and * ```--checkpointPath``` option can be used to set checkpoint path for saving model, the default value is /tmp/lenet5/. -#####In order to use MKL-DNN as the backend, you should: +##### In order to use MKL-DNN as the backend, you should: 1. Define a graph model with Model or convert a sequential model to a graph model using: ``` convertedModel = sequentialModel.to_graph() diff --git a/python/dllib/src/bigdl/dllib/models/rnn/README.md b/python/dllib/src/bigdl/dllib/models/rnn/README.md index 1af01830625..b62f72a571c 100644 --- a/python/dllib/src/bigdl/dllib/models/rnn/README.md +++ b/python/dllib/src/bigdl/dllib/models/rnn/README.md @@ -103,7 +103,7 @@ PYSPARK_DRIVER_PYTHON=./venv/bin/python PYSPARK_PYTHON=./venv.zip/venv/bin/pytho * `--dampening` dampening for momentum, the default value is 0. * `--maxEpoch` max number of epochs to train, the default value is 30. -#####In order to use MKL-DNN as the backend, you should: +##### In order to use MKL-DNN as the backend, you should: 1. Define a graph model with Model or convert a sequential model to a graph model using: ``` convertedModel = sequentialModel.to_graph() @@ -114,7 +114,7 @@ PYSPARK_DRIVER_PYTHON=./venv/bin/python PYSPARK_PYTHON=./venv.zip/venv/bin/pytho theDefinedModel.set_input_formats([theInputFormatIndex]) theDefinedModel.set_output_formats([theOutputFormatIndex]) ``` - BigDL needs these format information to build IRGraph from StaticGraph for MklDnn computing. + BigDL needs these format information to build a graph running with MKL-DNN backend. The format index of input or output format can be checked in: From bc6d12ef2e558b261b449d11aafc6b76a8b95f3e Mon Sep 17 00:00:00 2001 From: Firecrackerxox Date: Tue, 29 Oct 2019 09:40:17 +0800 Subject: [PATCH 764/823] edit doc (#2948) --- python/dllib/src/bigdl/dllib/models/lenet/README.md | 2 +- python/dllib/src/bigdl/dllib/models/rnn/README.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/models/lenet/README.md b/python/dllib/src/bigdl/dllib/models/lenet/README.md index c12172d41bc..aaf9cb3a55b 100644 --- a/python/dllib/src/bigdl/dllib/models/lenet/README.md +++ b/python/dllib/src/bigdl/dllib/models/lenet/README.md @@ -62,7 +62,7 @@ We would train a LeNet model in spark local mode with the following commands and * ```--checkpointPath``` option can be used to set checkpoint path for saving model, the default value is /tmp/lenet5/. -#####In order to use MKL-DNN as the backend, you should: +##### In order to use MKL-DNN as the backend, you should: 1. Define a graph model with Model or convert a sequential model to a graph model using: ``` convertedModel = sequentialModel.to_graph() diff --git a/python/dllib/src/bigdl/dllib/models/rnn/README.md b/python/dllib/src/bigdl/dllib/models/rnn/README.md index 1af01830625..b62f72a571c 100644 --- a/python/dllib/src/bigdl/dllib/models/rnn/README.md +++ b/python/dllib/src/bigdl/dllib/models/rnn/README.md @@ -103,7 +103,7 @@ PYSPARK_DRIVER_PYTHON=./venv/bin/python PYSPARK_PYTHON=./venv.zip/venv/bin/pytho * `--dampening` dampening for momentum, the default value is 0. * `--maxEpoch` max number of epochs to train, the default value is 30. -#####In order to use MKL-DNN as the backend, you should: +##### In order to use MKL-DNN as the backend, you should: 1. Define a graph model with Model or convert a sequential model to a graph model using: ``` convertedModel = sequentialModel.to_graph() @@ -114,7 +114,7 @@ PYSPARK_DRIVER_PYTHON=./venv/bin/python PYSPARK_PYTHON=./venv.zip/venv/bin/pytho theDefinedModel.set_input_formats([theInputFormatIndex]) theDefinedModel.set_output_formats([theOutputFormatIndex]) ``` - BigDL needs these format information to build IRGraph from StaticGraph for MklDnn computing. + BigDL needs these format information to build a graph running with MKL-DNN backend. The format index of input or output format can be checked in: From 66bb9d3a568eac88910064b72bc1f357199adcf3 Mon Sep 17 00:00:00 2001 From: Jerry Wu Date: Sat, 30 Nov 2019 14:21:51 +0800 Subject: [PATCH 765/823] flip version to 0.11 (#2974) --- .../dllib/src/bigdl/dllib/models/textclassifier/README.md | 8 ++++---- version.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/models/textclassifier/README.md b/python/dllib/src/bigdl/dllib/models/textclassifier/README.md index a2f48553578..9aac7e6c20f 100644 --- a/python/dllib/src/bigdl/dllib/models/textclassifier/README.md +++ b/python/dllib/src/bigdl/dllib/models/textclassifier/README.md @@ -28,8 +28,8 @@ $ [/tmp/news20]$ tree . -L 1 - The example code would automatically download the data during the first run. ### Run via pip install -- [Install from pip](https://bigdl-project.github.io/0.10.0-SNAPSHOT/#PythonUserGuide/install-from-pip/) -- Optional: check [Run after pip install](https://bigdl-project.github.io/0.10.0-SNAPSHOT/#PythonUserGuide/run-from-pip/) +- [Install from pip](https://bigdl-project.github.io/0.11.0-SNAPSHOT/#PythonUserGuide/install-from-pip/) +- Optional: check [Run after pip install](https://bigdl-project.github.io/0.11.0-SNAPSHOT/#PythonUserGuide/run-from-pip/) - Run the following command locally ``` python ${BigDL_HOME}/pyspark/bigdl/models/textclassifier/textclassifier.py --max_epoch 3 --model cnn @@ -37,8 +37,8 @@ python ${BigDL_HOME}/pyspark/bigdl/models/textclassifier/textclassifier.py --max ``` ### Run via spark-submit -- [Install without pip](https://bigdl-project.github.io/0.10.0-SNAPSHOT/#PythonUserGuide/install-without-pip/) -- Optional: check [Run without pip](https://bigdl-project.github.io/0.10.0-SNAPSHOT/#PythonUserGuide/run-without-pip/) +- [Install without pip](https://bigdl-project.github.io/0.11.0-SNAPSHOT/#PythonUserGuide/install-without-pip/) +- Optional: check [Run without pip](https://bigdl-project.github.io/0.11.0-SNAPSHOT/#PythonUserGuide/run-without-pip/) - Run the following command ```{r, engine='sh'} PYTHONHASHSEED=0 diff --git a/version.py b/version.py index 0e836e0ccd2..4bedc0e98f4 100644 --- a/version.py +++ b/version.py @@ -14,4 +14,4 @@ # limitations under the License. # -__version__ = "0.10.0.dev0" +__version__ = "0.11.0.dev0" From b78359c7073ae5c8716f31d1a76c79990004b83c Mon Sep 17 00:00:00 2001 From: Jerry Wu Date: Sat, 30 Nov 2019 14:21:51 +0800 Subject: [PATCH 766/823] flip version to 0.11 (#2974) --- .../dllib/src/bigdl/dllib/models/textclassifier/README.md | 8 ++++---- version.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/models/textclassifier/README.md b/python/dllib/src/bigdl/dllib/models/textclassifier/README.md index a2f48553578..9aac7e6c20f 100644 --- a/python/dllib/src/bigdl/dllib/models/textclassifier/README.md +++ b/python/dllib/src/bigdl/dllib/models/textclassifier/README.md @@ -28,8 +28,8 @@ $ [/tmp/news20]$ tree . -L 1 - The example code would automatically download the data during the first run. ### Run via pip install -- [Install from pip](https://bigdl-project.github.io/0.10.0-SNAPSHOT/#PythonUserGuide/install-from-pip/) -- Optional: check [Run after pip install](https://bigdl-project.github.io/0.10.0-SNAPSHOT/#PythonUserGuide/run-from-pip/) +- [Install from pip](https://bigdl-project.github.io/0.11.0-SNAPSHOT/#PythonUserGuide/install-from-pip/) +- Optional: check [Run after pip install](https://bigdl-project.github.io/0.11.0-SNAPSHOT/#PythonUserGuide/run-from-pip/) - Run the following command locally ``` python ${BigDL_HOME}/pyspark/bigdl/models/textclassifier/textclassifier.py --max_epoch 3 --model cnn @@ -37,8 +37,8 @@ python ${BigDL_HOME}/pyspark/bigdl/models/textclassifier/textclassifier.py --max ``` ### Run via spark-submit -- [Install without pip](https://bigdl-project.github.io/0.10.0-SNAPSHOT/#PythonUserGuide/install-without-pip/) -- Optional: check [Run without pip](https://bigdl-project.github.io/0.10.0-SNAPSHOT/#PythonUserGuide/run-without-pip/) +- [Install without pip](https://bigdl-project.github.io/0.11.0-SNAPSHOT/#PythonUserGuide/install-without-pip/) +- Optional: check [Run without pip](https://bigdl-project.github.io/0.11.0-SNAPSHOT/#PythonUserGuide/run-without-pip/) - Run the following command ```{r, engine='sh'} PYTHONHASHSEED=0 diff --git a/version.py b/version.py index 0e836e0ccd2..4bedc0e98f4 100644 --- a/version.py +++ b/version.py @@ -14,4 +14,4 @@ # limitations under the License. # -__version__ = "0.10.0.dev0" +__version__ = "0.11.0.dev0" From 16294f03681c421d52e306ade927d976667e4592 Mon Sep 17 00:00:00 2001 From: Le-Zheng <30695225+Le-Zheng@users.noreply.github.com> Date: Tue, 19 May 2020 06:48:18 +0100 Subject: [PATCH 767/823] fix callBigDLFunc (#3002) --- python/dllib/src/bigdl/dllib/utils/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/dllib/src/bigdl/dllib/utils/common.py b/python/dllib/src/bigdl/dllib/utils/common.py index e57621b9a46..6bb1fb6d873 100644 --- a/python/dllib/src/bigdl/dllib/utils/common.py +++ b/python/dllib/src/bigdl/dllib/utils/common.py @@ -581,6 +581,7 @@ def _get_gateway(): def callBigDlFunc(bigdl_type, name, *args): """ Call API in PythonBigDL """ gateway = _get_gateway() + args = [_py2java(gateway, a) for a in args] error = Exception("Cannot find function: %s" % name) for jinvoker in JavaCreator.instance(bigdl_type, gateway).value: # hasattr(jinvoker, name) always return true here, @@ -635,7 +636,6 @@ def _java2py(gateway, r, encoding="bytes"): def callJavaFunc(func, *args): """ Call Java Function """ gateway = _get_gateway() - args = [_py2java(gateway, a) for a in args] result = func(*args) return _java2py(gateway, result) From 34075b0fc7ef298c71d3b2e7af1b6a028328e9e6 Mon Sep 17 00:00:00 2001 From: Le-Zheng <30695225+Le-Zheng@users.noreply.github.com> Date: Tue, 19 May 2020 06:48:18 +0100 Subject: [PATCH 768/823] fix callBigDLFunc (#3002) --- python/dllib/src/bigdl/utils/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/dllib/src/bigdl/utils/common.py b/python/dllib/src/bigdl/utils/common.py index e57621b9a46..6bb1fb6d873 100644 --- a/python/dllib/src/bigdl/utils/common.py +++ b/python/dllib/src/bigdl/utils/common.py @@ -581,6 +581,7 @@ def _get_gateway(): def callBigDlFunc(bigdl_type, name, *args): """ Call API in PythonBigDL """ gateway = _get_gateway() + args = [_py2java(gateway, a) for a in args] error = Exception("Cannot find function: %s" % name) for jinvoker in JavaCreator.instance(bigdl_type, gateway).value: # hasattr(jinvoker, name) always return true here, @@ -635,7 +636,6 @@ def _java2py(gateway, r, encoding="bytes"): def callJavaFunc(func, *args): """ Call Java Function """ gateway = _get_gateway() - args = [_py2java(gateway, a) for a in args] result = func(*args) return _java2py(gateway, result) From 233e94f934c815cf7c2a1bb650e905b196ce683e Mon Sep 17 00:00:00 2001 From: Le-Zheng <30695225+Le-Zheng@users.noreply.github.com> Date: Wed, 27 May 2020 02:11:56 +0100 Subject: [PATCH 769/823] DistriOptimizerV2 argument (#3003) * call DistriOptimizerV2 --- python/dllib/src/bigdl/dllib/utils/common.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/utils/common.py b/python/dllib/src/bigdl/dllib/utils/common.py index 6bb1fb6d873..a58d61629dd 100644 --- a/python/dllib/src/bigdl/dllib/utils/common.py +++ b/python/dllib/src/bigdl/dllib/utils/common.py @@ -425,6 +425,12 @@ def init_engine(bigdl_type="float"): def get_bigdl_engine_type(bigdl_type="float"): return callBigDlFunc(bigdl_type, "getEngineType") +def set_optimizer_version(optimizerVersion, bigdl_type="float"): + return callBigDlFunc(bigdl_type, "setOptimizerVersion", optimizerVersion) + +def get_optimizer_version(bigdl_type="float"): + return callBigDlFunc(bigdl_type, "getOptimizerVersion") + def init_executor_gateway(sc, bigdl_type="float"): callBigDlFunc(bigdl_type, "initExecutorGateway", sc, sc._gateway._gateway_client.port) From 43b58f6295f07ad54778503b61d00310c6b26a07 Mon Sep 17 00:00:00 2001 From: Le-Zheng <30695225+Le-Zheng@users.noreply.github.com> Date: Wed, 27 May 2020 02:11:56 +0100 Subject: [PATCH 770/823] DistriOptimizerV2 argument (#3003) * call DistriOptimizerV2 --- python/dllib/src/bigdl/utils/common.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/python/dllib/src/bigdl/utils/common.py b/python/dllib/src/bigdl/utils/common.py index 6bb1fb6d873..a58d61629dd 100644 --- a/python/dllib/src/bigdl/utils/common.py +++ b/python/dllib/src/bigdl/utils/common.py @@ -425,6 +425,12 @@ def init_engine(bigdl_type="float"): def get_bigdl_engine_type(bigdl_type="float"): return callBigDlFunc(bigdl_type, "getEngineType") +def set_optimizer_version(optimizerVersion, bigdl_type="float"): + return callBigDlFunc(bigdl_type, "setOptimizerVersion", optimizerVersion) + +def get_optimizer_version(bigdl_type="float"): + return callBigDlFunc(bigdl_type, "getOptimizerVersion") + def init_executor_gateway(sc, bigdl_type="float"): callBigDlFunc(bigdl_type, "initExecutorGateway", sc, sc._gateway._gateway_client.port) From 48694bf468e9dedb0cb27ca19bf93ace1416ee8c Mon Sep 17 00:00:00 2001 From: Le-Zheng <30695225+Le-Zheng@users.noreply.github.com> Date: Tue, 30 Jun 2020 09:33:38 +0800 Subject: [PATCH 771/823] test python examples by distriOptimizerV2 (#3008) * Test python examples by distriOptimizerV2 --- python/dllib/src/bigdl/dllib/examples/keras/README.md | 3 ++- python/dllib/src/bigdl/dllib/examples/keras/imdb_cnn_lstm.py | 2 ++ python/dllib/src/bigdl/dllib/examples/keras/mnist_cnn.py | 2 ++ python/dllib/src/bigdl/dllib/models/lenet/README.md | 2 ++ python/dllib/src/bigdl/dllib/models/lenet/lenet5.py | 3 +++ python/dllib/src/bigdl/dllib/models/textclassifier/README.md | 2 ++ .../src/bigdl/dllib/models/textclassifier/textclassifier.py | 2 ++ 7 files changed, 15 insertions(+), 1 deletion(-) diff --git a/python/dllib/src/bigdl/dllib/examples/keras/README.md b/python/dllib/src/bigdl/dllib/examples/keras/README.md index 86fa98e5797..fa4fb56a716 100644 --- a/python/dllib/src/bigdl/dllib/examples/keras/README.md +++ b/python/dllib/src/bigdl/dllib/examples/keras/README.md @@ -49,4 +49,5 @@ ${SPARK_HOME}/bin/spark-submit \ ${BigDL_HOME}/pyspark/bigdl/examples/keras/mnist_cnn.py ``` * ```--batchSize``` an option that can be used to set batch size. -* ```--max_epoch``` an option that can be used to set how many epochs for which the model is to be trained. \ No newline at end of file +* ```--max_epoch``` an option that can be used to set how many epochs for which the model is to be trained. +* ```--optimizerVersion``` an option that can be used to set DistriOptimizer version, the default value is "optimizerV1". \ No newline at end of file diff --git a/python/dllib/src/bigdl/dllib/examples/keras/imdb_cnn_lstm.py b/python/dllib/src/bigdl/dllib/examples/keras/imdb_cnn_lstm.py index b7ce5fa4478..0b710f984e5 100644 --- a/python/dllib/src/bigdl/dllib/examples/keras/imdb_cnn_lstm.py +++ b/python/dllib/src/bigdl/dllib/examples/keras/imdb_cnn_lstm.py @@ -65,6 +65,7 @@ def build_keras_model(): parser = OptionParser() parser.add_option("-b", "--batchSize", type=int, dest="batchSize", default="32") parser.add_option("-m", "--max_epoch", type=int, dest="max_epoch", default="2") + parser.add_option("--optimizerVersion", dest="optimizerVersion", default="optimizerV1") (options, args) = parser.parse_args(sys.argv) keras_model = build_keras_model() @@ -83,6 +84,7 @@ def build_keras_model(): redire_spark_logs() show_bigdl_info_logs() init_engine() + set_optimizer_version(options.optimizerVersion) X_train, y_train, X_test, y_test = load_imdb() train_data = to_sample_rdd(X_train, y_train) diff --git a/python/dllib/src/bigdl/dllib/examples/keras/mnist_cnn.py b/python/dllib/src/bigdl/dllib/examples/keras/mnist_cnn.py index eacb408b1ac..309116c588f 100644 --- a/python/dllib/src/bigdl/dllib/examples/keras/mnist_cnn.py +++ b/python/dllib/src/bigdl/dllib/examples/keras/mnist_cnn.py @@ -78,6 +78,7 @@ def build_keras_model(): parser.add_option("-b", "--batchSize", type=int, dest="batchSize", default="128") parser.add_option("-m", "--max_epoch", type=int, dest="max_epoch", default="12") parser.add_option("-d", "--dataPath", dest="dataPath", default="/tmp/mnist") + parser.add_option("--optimizerVersion", dest="optimizerVersion", default="optimizerV1") (options, args) = parser.parse_args(sys.argv) keras_model = build_keras_model() @@ -96,6 +97,7 @@ def build_keras_model(): redire_spark_logs() show_bigdl_info_logs() init_engine() + set_optimizer_version(options.optimizerVersion) train_data = get_mnist(sc, "train", options.dataPath) test_data = get_mnist(sc, "test", options.dataPath) diff --git a/python/dllib/src/bigdl/dllib/models/lenet/README.md b/python/dllib/src/bigdl/dllib/models/lenet/README.md index aaf9cb3a55b..7a9253d3b92 100644 --- a/python/dllib/src/bigdl/dllib/models/lenet/README.md +++ b/python/dllib/src/bigdl/dllib/models/lenet/README.md @@ -62,6 +62,8 @@ We would train a LeNet model in spark local mode with the following commands and * ```--checkpointPath``` option can be used to set checkpoint path for saving model, the default value is /tmp/lenet5/. +* ```--optimizerVersion``` option can be used to set DistriOptimizer version, the value can be "optimizerV1" or "optimizerV2". + ##### In order to use MKL-DNN as the backend, you should: 1. Define a graph model with Model or convert a sequential model to a graph model using: ``` diff --git a/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py b/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py index 6ba9bb0aa1e..cb4a940d5ed 100644 --- a/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py +++ b/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py @@ -60,6 +60,7 @@ def build_model(class_num): parser.add_option("-t", "--endTriggerType", dest="endTriggerType", default="epoch") parser.add_option("-n", "--endTriggerNum", type=int, dest="endTriggerNum", default="20") parser.add_option("-d", "--dataPath", dest="dataPath", default="/tmp/mnist") + parser.add_option("--optimizerVersion", dest="optimizerVersion", default="optimizerV1") (options, args) = parser.parse_args(sys.argv) @@ -68,6 +69,8 @@ def build_model(class_num): show_bigdl_info_logs() init_engine() + set_optimizer_version(options.optimizerVersion) + # In order to use MklDnn as the backend, you should: # 1. Define a graph model with Model(graph container) or convert a sequential model to a graph model # 2. Specify the input and output formats of it. diff --git a/python/dllib/src/bigdl/dllib/models/textclassifier/README.md b/python/dllib/src/bigdl/dllib/models/textclassifier/README.md index 9aac7e6c20f..e7aea69346b 100644 --- a/python/dllib/src/bigdl/dllib/models/textclassifier/README.md +++ b/python/dllib/src/bigdl/dllib/models/textclassifier/README.md @@ -77,6 +77,8 @@ which are `cnn`, `lstm` and `gru`, default is `cnn` * `--learning_rate` option can be used to set learning rate, default is 0.05. +* `--optimizerVersion` option can be used to set DistriOptimizer version, the value can be "optimizerV1" or "optimizerV2". + To verify the accuracy, search "accuracy" from log: ```{r, engine='sh'} diff --git a/python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py b/python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py index 314e3e4146f..c4d0c5d6e12 100644 --- a/python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py +++ b/python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py @@ -164,6 +164,7 @@ def train(sc, data_path, parser.add_option("--model", dest="model_type", default="cnn") parser.add_option("-p", "--p", dest="p", default="0.0") parser.add_option("-d", "--data_path", dest="data_path", default="/tmp/news20/") + parser.add_option("--optimizerVersion", dest="optimizerVersion", default="optimizerV1") (options, args) = parser.parse_args(sys.argv) if options.action == "train": @@ -182,6 +183,7 @@ def train(sc, data_path, redire_spark_logs() show_bigdl_info_logs() init_engine() + set_optimizer_version(options.optimizerVersion) train(sc, data_path, batch_size, sequence_len, max_words, embedding_dim, training_split) From 029336785e467372d3a10c6bce48b3ff469f5111 Mon Sep 17 00:00:00 2001 From: Le-Zheng <30695225+Le-Zheng@users.noreply.github.com> Date: Tue, 30 Jun 2020 09:33:38 +0800 Subject: [PATCH 772/823] test python examples by distriOptimizerV2 (#3008) * Test python examples by distriOptimizerV2 --- python/dllib/src/bigdl/dllib/examples/keras/README.md | 3 ++- python/dllib/src/bigdl/dllib/examples/keras/imdb_cnn_lstm.py | 2 ++ python/dllib/src/bigdl/dllib/examples/keras/mnist_cnn.py | 2 ++ python/dllib/src/bigdl/dllib/models/lenet/README.md | 2 ++ python/dllib/src/bigdl/dllib/models/lenet/lenet5.py | 3 +++ python/dllib/src/bigdl/dllib/models/textclassifier/README.md | 2 ++ .../src/bigdl/dllib/models/textclassifier/textclassifier.py | 2 ++ 7 files changed, 15 insertions(+), 1 deletion(-) diff --git a/python/dllib/src/bigdl/dllib/examples/keras/README.md b/python/dllib/src/bigdl/dllib/examples/keras/README.md index 86fa98e5797..fa4fb56a716 100644 --- a/python/dllib/src/bigdl/dllib/examples/keras/README.md +++ b/python/dllib/src/bigdl/dllib/examples/keras/README.md @@ -49,4 +49,5 @@ ${SPARK_HOME}/bin/spark-submit \ ${BigDL_HOME}/pyspark/bigdl/examples/keras/mnist_cnn.py ``` * ```--batchSize``` an option that can be used to set batch size. -* ```--max_epoch``` an option that can be used to set how many epochs for which the model is to be trained. \ No newline at end of file +* ```--max_epoch``` an option that can be used to set how many epochs for which the model is to be trained. +* ```--optimizerVersion``` an option that can be used to set DistriOptimizer version, the default value is "optimizerV1". \ No newline at end of file diff --git a/python/dllib/src/bigdl/dllib/examples/keras/imdb_cnn_lstm.py b/python/dllib/src/bigdl/dllib/examples/keras/imdb_cnn_lstm.py index b7ce5fa4478..0b710f984e5 100644 --- a/python/dllib/src/bigdl/dllib/examples/keras/imdb_cnn_lstm.py +++ b/python/dllib/src/bigdl/dllib/examples/keras/imdb_cnn_lstm.py @@ -65,6 +65,7 @@ def build_keras_model(): parser = OptionParser() parser.add_option("-b", "--batchSize", type=int, dest="batchSize", default="32") parser.add_option("-m", "--max_epoch", type=int, dest="max_epoch", default="2") + parser.add_option("--optimizerVersion", dest="optimizerVersion", default="optimizerV1") (options, args) = parser.parse_args(sys.argv) keras_model = build_keras_model() @@ -83,6 +84,7 @@ def build_keras_model(): redire_spark_logs() show_bigdl_info_logs() init_engine() + set_optimizer_version(options.optimizerVersion) X_train, y_train, X_test, y_test = load_imdb() train_data = to_sample_rdd(X_train, y_train) diff --git a/python/dllib/src/bigdl/dllib/examples/keras/mnist_cnn.py b/python/dllib/src/bigdl/dllib/examples/keras/mnist_cnn.py index eacb408b1ac..309116c588f 100644 --- a/python/dllib/src/bigdl/dllib/examples/keras/mnist_cnn.py +++ b/python/dllib/src/bigdl/dllib/examples/keras/mnist_cnn.py @@ -78,6 +78,7 @@ def build_keras_model(): parser.add_option("-b", "--batchSize", type=int, dest="batchSize", default="128") parser.add_option("-m", "--max_epoch", type=int, dest="max_epoch", default="12") parser.add_option("-d", "--dataPath", dest="dataPath", default="/tmp/mnist") + parser.add_option("--optimizerVersion", dest="optimizerVersion", default="optimizerV1") (options, args) = parser.parse_args(sys.argv) keras_model = build_keras_model() @@ -96,6 +97,7 @@ def build_keras_model(): redire_spark_logs() show_bigdl_info_logs() init_engine() + set_optimizer_version(options.optimizerVersion) train_data = get_mnist(sc, "train", options.dataPath) test_data = get_mnist(sc, "test", options.dataPath) diff --git a/python/dllib/src/bigdl/dllib/models/lenet/README.md b/python/dllib/src/bigdl/dllib/models/lenet/README.md index aaf9cb3a55b..7a9253d3b92 100644 --- a/python/dllib/src/bigdl/dllib/models/lenet/README.md +++ b/python/dllib/src/bigdl/dllib/models/lenet/README.md @@ -62,6 +62,8 @@ We would train a LeNet model in spark local mode with the following commands and * ```--checkpointPath``` option can be used to set checkpoint path for saving model, the default value is /tmp/lenet5/. +* ```--optimizerVersion``` option can be used to set DistriOptimizer version, the value can be "optimizerV1" or "optimizerV2". + ##### In order to use MKL-DNN as the backend, you should: 1. Define a graph model with Model or convert a sequential model to a graph model using: ``` diff --git a/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py b/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py index 6ba9bb0aa1e..cb4a940d5ed 100644 --- a/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py +++ b/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py @@ -60,6 +60,7 @@ def build_model(class_num): parser.add_option("-t", "--endTriggerType", dest="endTriggerType", default="epoch") parser.add_option("-n", "--endTriggerNum", type=int, dest="endTriggerNum", default="20") parser.add_option("-d", "--dataPath", dest="dataPath", default="/tmp/mnist") + parser.add_option("--optimizerVersion", dest="optimizerVersion", default="optimizerV1") (options, args) = parser.parse_args(sys.argv) @@ -68,6 +69,8 @@ def build_model(class_num): show_bigdl_info_logs() init_engine() + set_optimizer_version(options.optimizerVersion) + # In order to use MklDnn as the backend, you should: # 1. Define a graph model with Model(graph container) or convert a sequential model to a graph model # 2. Specify the input and output formats of it. diff --git a/python/dllib/src/bigdl/dllib/models/textclassifier/README.md b/python/dllib/src/bigdl/dllib/models/textclassifier/README.md index 9aac7e6c20f..e7aea69346b 100644 --- a/python/dllib/src/bigdl/dllib/models/textclassifier/README.md +++ b/python/dllib/src/bigdl/dllib/models/textclassifier/README.md @@ -77,6 +77,8 @@ which are `cnn`, `lstm` and `gru`, default is `cnn` * `--learning_rate` option can be used to set learning rate, default is 0.05. +* `--optimizerVersion` option can be used to set DistriOptimizer version, the value can be "optimizerV1" or "optimizerV2". + To verify the accuracy, search "accuracy" from log: ```{r, engine='sh'} diff --git a/python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py b/python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py index 314e3e4146f..c4d0c5d6e12 100644 --- a/python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py +++ b/python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py @@ -164,6 +164,7 @@ def train(sc, data_path, parser.add_option("--model", dest="model_type", default="cnn") parser.add_option("-p", "--p", dest="p", default="0.0") parser.add_option("-d", "--data_path", dest="data_path", default="/tmp/news20/") + parser.add_option("--optimizerVersion", dest="optimizerVersion", default="optimizerV1") (options, args) = parser.parse_args(sys.argv) if options.action == "train": @@ -182,6 +183,7 @@ def train(sc, data_path, redire_spark_logs() show_bigdl_info_logs() init_engine() + set_optimizer_version(options.optimizerVersion) train(sc, data_path, batch_size, sequence_len, max_words, embedding_dim, training_split) From 44daee1672904d2638f7dbc2a982e17a8b391c99 Mon Sep 17 00:00:00 2001 From: Le-Zheng <30695225+Le-Zheng@users.noreply.github.com> Date: Tue, 30 Jun 2020 12:50:16 +0800 Subject: [PATCH 773/823] deprecate nn.keras (#3013) * deprecate nn.keras --- .../bigdl/dllib/bigdlkeras/layers/layer.py | 195 ++++++++++++++++++ .../bigdl/dllib/bigdlkeras/layers/topology.py | 15 ++ 2 files changed, 210 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py b/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py index 4d688e36a63..b2d84897dfc 100644 --- a/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py +++ b/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py @@ -25,6 +25,10 @@ class InferShape(JavaValue): + """ + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. + """ def __init__(self, bigdl_type="float"): self.bigdl_type = bigdl_type @@ -58,6 +62,10 @@ def get_output_shape(self): class KerasCreator(JavaValue): + """ + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. + """ def jvm_class_constructor(self): name = "createKeras" + self.__class__.__name__ print("creating: " + name) @@ -65,6 +73,10 @@ def jvm_class_constructor(self): class KerasLayer(Layer, InferShape, KerasCreator): + """ + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. + """ def __init__(self, jvalue, *args, **kwargs): allowed_kwargs = {"name", "bigdl_type"} for kwarg in kwargs.keys(): @@ -89,6 +101,9 @@ class Input(Node, KerasCreator): >>> input = Input(name="input1", shape=(3, 5)) creating: createKerasInput + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, shape=None, name=None, bigdl_type="float"): super(Input, self).__init__(None, bigdl_type, @@ -106,6 +121,9 @@ class InputLayer(KerasLayer): >>> inputlayer = InputLayer(input_shape=(3, 5)) creating: createKerasInputLayer + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, input_shape=None, **kwargs): super(InputLayer, self).__init__(None, @@ -138,6 +156,9 @@ class Dense(KerasLayer): >>> dense = Dense(10, input_dim=8, name="dense1") creating: createKerasDense + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, output_dim, init="glorot_uniform", activation=None, W_regularizer=None, b_regularizer=None, @@ -178,6 +199,9 @@ class MaxoutDense(KerasLayer): >>> maxoutdense = MaxoutDense(6, input_shape=(10, )) creating: createKerasMaxoutDense + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, output_dim, nb_feature=4, W_regularizer=None, b_regularizer=None, bias=True, input_dim=None, input_shape=None, **kwargs): @@ -213,6 +237,9 @@ class Embedding(KerasLayer): >>> embedding = Embedding(1000, 32, input_shape=(10, ), name="embedding1") creating: createKerasEmbedding + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, input_dim, output_dim, init="uniform", W_regularizer=None, input_shape=None, **kwargs): @@ -249,6 +276,9 @@ class BatchNormalization(KerasLayer): >>> batchnormalization = BatchNormalization(input_shape=(3, 12, 12), name="bn1") creating: createKerasBatchNormalization + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, epsilon=0.001, mode=0, axis=1, momentum=0.99, beta_init="zero", gamma_init="one", dim_ordering="th", input_shape=None, **kwargs): @@ -323,6 +353,9 @@ class Merge(KerasLayer): creating: createKerasInputLayer >>> merge = Merge(layers=[l1, l2], mode='sum', name="merge1") creating: createKerasMerge + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, layers=None, mode="sum", concat_axis=-1, input_shape=None, **kwargs): @@ -347,6 +380,9 @@ def merge(inputs, mode="sum", concat_axis=-1, name=None): concat_axis: Int, axis to use when concatenating nodes. Only specify this when merge mode is 'concat'. Default is -1, meaning the last axis of the input. name: String to set the name of the merge. If not specified, its name will by default to be a generated string. + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ return Merge(mode=mode, concat_axis=concat_axis, name=name)(list(inputs)) @@ -366,6 +402,9 @@ class Dropout(KerasLayer): >>> dropout = Dropout(0.25, input_shape=(2, 3)) creating: createKerasDropout + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, p, input_shape=None, **kwargs): super(Dropout, self).__init__(None, @@ -387,6 +426,9 @@ class Flatten(KerasLayer): >>> flatten = Flatten(input_shape=(3, 10, 2)) creating: createKerasFlatten + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, input_shape=None, **kwargs): super(Flatten, self).__init__(None, @@ -411,6 +453,9 @@ class Reshape(KerasLayer): >>> reshape = Reshape((2, 10), input_shape=(5, 4)) creating: createKerasReshape + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, target_shape, input_shape=None, **kwargs): super(Reshape, self).__init__(None, @@ -434,6 +479,9 @@ class Activation(KerasLayer): >>> activation = Activation("relu", input_shape=(3, 4)) creating: createKerasActivation + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, activation, input_shape=None, **kwargs): super(Activation, self).__init__(None, @@ -459,6 +507,9 @@ class RepeatVector(KerasLayer): >>> repeatvector = RepeatVector(5, input_shape=(3, )) creating: createKerasRepeatVector + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, n, input_dim=None, input_shape=None, **kwargs): if input_dim: @@ -484,6 +535,9 @@ class Permute(KerasLayer): >>> permute = Permute((2, 1, 3), input_shape=(3, 4, 5)) creating: createKerasPermute + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, dims, input_shape=None, **kwargs): super(Permute, self).__init__(None, @@ -515,6 +569,9 @@ class Highway(KerasLayer): >>> highway = Highway(activation='relu', input_shape=(8, )) creating: createKerasHighway + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, activation=None, W_regularizer=None, b_regularizer=None, bias=True, input_dim=None, input_shape=None, **kwargs): @@ -557,6 +614,9 @@ class Convolution1D(KerasLayer): >>> conv1d = Convolution1D(12, 4, input_shape=(3, 16)) creating: createKerasConvolution1D + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, nb_filter, filter_length, init="glorot_uniform", activation=None, border_mode="valid", subsample_length=1, @@ -608,6 +668,9 @@ class Convolution2D(KerasLayer): >>> conv2d = Convolution2D(32, 3, 3, input_shape=(3, 128, 128), name="convolution2d_1") creating: createKerasConvolution2D + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, nb_filter, nb_row, nb_col, init="glorot_uniform", activation=None, @@ -663,6 +726,9 @@ class Convolution3D(KerasLayer): >>> conv3d = Convolution3D(32, 3, 4, 5, input_shape=(3, 64, 64, 64)) creating: createKerasConvolution3D + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, nb_filter, kernel_dim1, kernel_dim2, kernel_dim3, init="glorot_uniform", activation=None, border_mode="valid", @@ -716,6 +782,9 @@ class AtrousConvolution1D(KerasLayer): >>> atrousconv1d = AtrousConvolution1D(8, 3, input_shape=(3, 12)) creating: createKerasAtrousConvolution1D + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, nb_filter, filter_length, init="glorot_uniform", activation=None, border_mode="valid", subsample_length=1, atrous_rate=1, W_regularizer=None, @@ -774,6 +843,9 @@ class AtrousConvolution2D(KerasLayer): >>> atrousconv2d = AtrousConvolution2D(12, 4, 3, input_shape=(3, 64, 64)) creating: createKerasAtrousConvolution2D + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, nb_filter, nb_row, nb_col, init="glorot_uniform", activation=None, border_mode="valid", subsample=(1, 1), @@ -837,6 +909,9 @@ class Deconvolution2D(KerasLayer): >>> deconv2d = Deconvolution2D(3, 3, 3, output_shape=(None, 3, 14, 14), input_shape=(3, 12, 12)) creating: createKerasDeconvolution2D + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, nb_filter, nb_row, nb_col, output_shape, init="glorot_uniform", activation=None, border_mode="valid", subsample=(1, 1), dim_ordering="th", @@ -898,6 +973,9 @@ class SeparableConvolution2D(KerasLayer): >>> separableconv2d = SeparableConvolution2D(12, 3, 4, input_shape=(3, 32, 32)) creating: createKerasSeparableConvolution2D + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, nb_filter, nb_row, nb_col, init="glorot_uniform", activation=None, border_mode="valid", subsample=(1, 1), depth_multiplier=1, @@ -946,6 +1024,9 @@ class Cropping1D(KerasLayer): >>> cropping1d = Cropping1D(cropping=(1, 2), input_shape=(8, 8)) creating: createKerasCropping1D + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, cropping=(1, 1), input_shape=None, **kwargs): super(Cropping1D, self).__init__(None, @@ -970,6 +1051,9 @@ class Cropping2D(KerasLayer): >>> cropping2d = Cropping2D(cropping=((1, 2), (0, 1)), input_shape=(12, 12, 12)) creating: createKerasCropping2D + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, cropping=((0, 0), (0, 0)), dim_ordering="th", input_shape=None, **kwargs): @@ -998,6 +1082,9 @@ class Cropping3D(KerasLayer): >>> cropping3d = Cropping3D(cropping=((0, 2), (1, 1), (3, 1)), input_shape=(4, 12, 12, 16)) creating: createKerasCropping3D + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, cropping=((1, 1), (1, 1), (1, 1)), dim_ordering="th", input_shape=None, **kwargs): @@ -1026,6 +1113,9 @@ class UpSampling1D(KerasLayer): >>> upsampling1d = UpSampling1D(length=3, input_shape=(3, 12)) creating: createKerasUpSampling1D + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, length=2, input_shape=None, **kwargs): super(UpSampling1D, self).__init__(None, @@ -1051,6 +1141,9 @@ class UpSampling2D(KerasLayer): >>> upsampling2d = UpSampling2D(size=(1, 3), input_shape=(3, 16, 16)) creating: createKerasUpSampling2D + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, size=(2, 2), dim_ordering="th", input_shape=None, **kwargs): super(UpSampling2D, self).__init__(None, @@ -1078,6 +1171,9 @@ class UpSampling3D(KerasLayer): >>> upsampling3d = UpSampling3D(size=(1, 2, 3), input_shape=(3, 16, 16, 16)) creating: createKerasUpSampling3D + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, size=(2, 2, 2), dim_ordering="th", input_shape=None, **kwargs): super(UpSampling3D, self).__init__(None, @@ -1105,6 +1201,9 @@ class ZeroPadding1D(KerasLayer): >>> zeropadding1d = ZeroPadding1D(padding=2, input_shape=(3, 6)) creating: createKerasZeroPadding1D + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, padding=1, input_shape=None, **kwargs): if isinstance(padding, int): @@ -1134,6 +1233,9 @@ class ZeroPadding2D(KerasLayer): >>> zeropadding2d = ZeroPadding2D(padding=(2, 1), input_shape=(2, 8, 8)) creating: createKerasZeroPadding2D + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, padding=(1, 1), dim_ordering="th", input_shape=None, **kwargs): if len(padding) == 2: @@ -1162,6 +1264,9 @@ class ZeroPadding3D(KerasLayer): >>> zeropadding3d = ZeroPadding3D(padding=(2, 1, 2), input_shape=(2, 8, 8, 10)) creating: createKerasZeroPadding3D + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, padding=(1, 1, 1), dim_ordering="th", input_shape=None, **kwargs): super(ZeroPadding3D, self).__init__(None, @@ -1189,6 +1294,9 @@ class MaxPooling1D(KerasLayer): >>> maxpooling1d = MaxPooling1D(3, input_shape=(3, 24)) creating: createKerasMaxPooling1D + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, pool_length=2, stride=None, border_mode="valid", input_shape=None, **kwargs): @@ -1221,6 +1329,9 @@ class MaxPooling2D(KerasLayer): >>> maxpooling2d = MaxPooling2D((2, 2), input_shape=(3, 32, 32), name="maxpooling2d_1") creating: createKerasMaxPooling2D + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, pool_size=(2, 2), strides=None, border_mode="valid", dim_ordering="th", @@ -1255,6 +1366,9 @@ class MaxPooling3D(KerasLayer): >>> maxpooling3d = MaxPooling3D((2, 1, 3), input_shape=(3, 32, 32, 32)) creating: createKerasMaxPooling3D + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, pool_size=(2, 2, 2), strides=None, border_mode="valid", dim_ordering="th", input_shape=None, **kwargs): @@ -1286,6 +1400,9 @@ class AveragePooling1D(KerasLayer): >>> averagepooling1d = AveragePooling1D(input_shape=(3, 24)) creating: createKerasAveragePooling1D + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, pool_length=2, stride=None, border_mode="valid", input_shape=None, **kwargs): @@ -1318,6 +1435,9 @@ class AveragePooling2D(KerasLayer): >>> averagepooling2d = AveragePooling2D((1, 2), input_shape=(2, 28, 32)) creating: createKerasAveragePooling2D + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, pool_size=(2, 2), strides=None, border_mode="valid", dim_ordering="th", input_shape=None, **kwargs): @@ -1351,6 +1471,9 @@ class AveragePooling3D(KerasLayer): >>> averagepooling3d = AveragePooling3D((1, 1, 2), input_shape=(3, 28, 32, 36)) creating: createKerasAveragePooling3D + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, pool_size=(2, 2, 2), strides=None, border_mode="valid", dim_ordering="th", input_shape=None, **kwargs): @@ -1378,6 +1501,9 @@ class GlobalMaxPooling1D(KerasLayer): >>> globalmaxpooling1d = GlobalMaxPooling1D(input_shape=(4, 8)) creating: createKerasGlobalMaxPooling1D + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, input_shape=None, **kwargs): super(GlobalMaxPooling1D, self).__init__(None, @@ -1399,6 +1525,9 @@ class GlobalAveragePooling1D(KerasLayer): >>> globalaveragepooling1d = GlobalAveragePooling1D(input_shape=(12, 12)) creating: createKerasGlobalAveragePooling1D + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, input_shape=None, **kwargs): super(GlobalAveragePooling1D, self).__init__(None, @@ -1421,6 +1550,9 @@ class GlobalMaxPooling2D(KerasLayer): >>> globalmaxpooling2d = GlobalMaxPooling2D(input_shape=(4, 32, 32)) creating: createKerasGlobalMaxPooling2D + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, dim_ordering="th", input_shape=None, **kwargs): super(GlobalMaxPooling2D, self).__init__(None, @@ -1444,6 +1576,9 @@ class GlobalAveragePooling2D(KerasLayer): >>> globalaveragepooling2d = GlobalAveragePooling2D(input_shape=(4, 32, 32)) creating: createKerasGlobalAveragePooling2D + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, dim_ordering="th", input_shape=None, **kwargs): super(GlobalAveragePooling2D, self).__init__(None, @@ -1469,6 +1604,9 @@ class GlobalMaxPooling3D(KerasLayer): >>> globalmaxpooling3d = GlobalMaxPooling3D(input_shape=(4, 32, 32, 32)) creating: createKerasGlobalMaxPooling3D + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, dim_ordering="th", input_shape=None, **kwargs): super(GlobalMaxPooling3D, self).__init__(None, @@ -1494,6 +1632,9 @@ class GlobalAveragePooling3D(KerasLayer): >>> globalaveragepooling3d = GlobalAveragePooling3D(input_shape=(4, 16, 16, 20)) creating: createKerasGlobalAveragePooling3D + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, dim_ordering="th", input_shape=None, **kwargs): super(GlobalAveragePooling3D, self).__init__(None, @@ -1526,6 +1667,9 @@ class SimpleRNN(KerasLayer): >>> simplernn = SimpleRNN(16, input_shape=(3, 32)) creating: createKerasSimpleRNN + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, output_dim, activation="tanh", return_sequences=False, go_backwards=False, W_regularizer=None, U_regularizer=None, @@ -1567,6 +1711,9 @@ class LSTM(KerasLayer): >>> lstm = LSTM(32, input_shape=(8, 16), name="lstm1") creating: createKerasLSTM + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, output_dim, activation="tanh", inner_activation="hard_sigmoid", return_sequences=False, go_backwards=False, W_regularizer=None, @@ -1609,6 +1756,9 @@ class GRU(KerasLayer): >>> gru = GRU(24, input_shape=(32, 32)) creating: createKerasGRU + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, output_dim, activation="tanh", inner_activation="hard_sigmoid", return_sequences=False, go_backwards=False, W_regularizer=None, @@ -1660,6 +1810,9 @@ class ConvLSTM2D(KerasLayer): >>> convlstm2d = ConvLSTM2D(24, 3, 3, input_shape=(4, 32, 32, 32)) creating: createKerasConvLSTM2D + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, nb_filter, nb_row, nb_col, activation="tanh", inner_activation="hard_sigmoid", dim_ordering="th", border_mode="same", @@ -1714,6 +1867,9 @@ class LocallyConnected1D(KerasLayer): >>> locallyconnected1d = LocallyConnected1D(6, 3, input_shape=(8, 12)) creating: createKerasLocallyConnected1D + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, nb_filter, filter_length, activation=None, border_mode="valid", subsample_length=1, W_regularizer=None, b_regularizer=None, @@ -1761,6 +1917,9 @@ class LocallyConnected2D(KerasLayer): >>> locallyconnected2d = LocallyConnected2D(12, 3, 4, input_shape=(3, 128, 128)) creating: createKerasLocallyConnected2D + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, nb_filter, nb_row, nb_col, activation=None, border_mode="valid", subsample=(1, 1), dim_ordering="th", @@ -1801,6 +1960,9 @@ class SpatialDropout1D(KerasLayer): >>> spatialdropout1d = SpatialDropout1D(0.4, input_shape=(10, 12)) creating: createKerasSpatialDropout1D + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, p=0.5, input_shape=None, **kwargs): super(SpatialDropout1D, self).__init__(None, @@ -1830,6 +1992,9 @@ class SpatialDropout2D(KerasLayer): >>> spatialdropout2d = SpatialDropout2D(0.25, input_shape=(5, 12, 12)) creating: createKerasSpatialDropout2D + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, p=0.5, dim_ordering="th", input_shape=None, **kwargs): super(SpatialDropout2D, self).__init__(None, @@ -1860,6 +2025,9 @@ class SpatialDropout3D(KerasLayer): >>> spatialdropout3d = SpatialDropout3D(0.6, input_shape=(4, 12, 12, 16)) creating: createKerasSpatialDropout3D + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, p=0.5, dim_ordering="th", input_shape=None, **kwargs): super(SpatialDropout3D, self).__init__(None, @@ -1885,6 +2053,9 @@ class GaussianDropout(KerasLayer): >>> gaussiandropout = GaussianDropout(0.45, input_shape=(4, 8)) creating: createKerasGaussianDropout + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, p, input_shape=None, **kwargs): super(GaussianDropout, self).__init__(None, @@ -1910,6 +2081,9 @@ class GaussianNoise(KerasLayer): >>> gaussiannoise = GaussianNoise(0.45, input_shape=(3, 4, 5), name="gaussiannoise1") creating: createKerasGaussianNoise + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, sigma, input_shape=None, **kwargs): super(GaussianNoise, self).__init__(None, @@ -1935,6 +2109,9 @@ class Masking(KerasLayer): >>> masking = Masking(0.3, input_shape=(6, 8)) creating: createKerasMasking + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, mask_value=0.0, input_shape=None, **kwargs): super(Masking, self).__init__(None, @@ -1973,6 +2150,9 @@ class SReLU(KerasLayer): >>> srelu = SReLU(input_shape=(4, 5)) creating: createKerasSReLU + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, t_left_init="zero", a_left_init="glorot_uniform", t_right_init="glorot_uniform", a_right_init="one", @@ -2004,6 +2184,9 @@ class ELU(KerasLayer): >>> elu = ELU(1.2, input_shape=(4, 5)) creating: createKerasELU + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, alpha=1.0, input_shape=None, **kwargs): super(ELU, self).__init__(None, @@ -2029,6 +2212,9 @@ class LeakyReLU(KerasLayer): >>> leakyrelu = LeakyReLU(0.02, input_shape=(4, 5)) creating: createKerasLeakyReLU + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, alpha=0.01, input_shape=None, **kwargs): super(LeakyReLU, self).__init__(None, @@ -2054,6 +2240,9 @@ class ThresholdedReLU(KerasLayer): >>> thresholdedrelu = ThresholdedReLU(input_shape=(10, 12)) creating: createKerasThresholdedReLU + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, theta=1.0, input_shape=None, **kwargs): super(ThresholdedReLU, self).__init__(None, @@ -2080,6 +2269,9 @@ class TimeDistributed(KerasLayer): >>> timedistributed = TimeDistributed(Dense(8), input_shape=(10, 12), name="timedistributeddense") creating: createKerasDense creating: createKerasTimeDistributed + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, layer, input_shape=None, **kwargs): super(TimeDistributed, self).__init__(None, @@ -2109,6 +2301,9 @@ class Bidirectional(KerasLayer): >>> bidiretional = Bidirectional(LSTM(10, return_sequences=True), input_shape=(12, 16), name="bidirectionallstm") creating: createKerasLSTM creating: createKerasBidirectional + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, layer, merge_mode="concat", input_shape=None, **kwargs): super(Bidirectional, self).__init__(None, diff --git a/python/dllib/src/bigdl/dllib/bigdlkeras/layers/topology.py b/python/dllib/src/bigdl/dllib/bigdlkeras/layers/topology.py index 64128b2e6b2..a891635c4c7 100644 --- a/python/dllib/src/bigdl/dllib/bigdlkeras/layers/topology.py +++ b/python/dllib/src/bigdl/dllib/bigdlkeras/layers/topology.py @@ -18,11 +18,16 @@ from bigdl.optim.optimizer import * from bigdl.nn.criterion import * import multiprocessing +import warnings from bigdl.nn.layer import SharedStaticUtils, Container class KerasModel(KerasLayer, Container, SharedStaticUtils): + """ + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. + """ def __convert_optim_method(self, optimizer): optimizer = optimizer.lower() if optimizer == "adagrad": @@ -200,8 +205,13 @@ class Sequential(KerasModel): >>> sequential = Sequential(name="seq1") creating: createKerasSequential + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, jvalue=None, **kwargs): + warnings.warn("bigdl.nn.keras is deprecated in 0.11. " + "Recommend to use Analytics Zoo's Keras API.") super(Sequential, self).__init__(jvalue, **kwargs) @staticmethod @@ -228,8 +238,13 @@ class Model(KerasModel): input: An input node or a list of input nodes. output: An output node or a list of output nodes. name: String to specify the name of the graph model. Default is None. + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, input, output, jvalue=None, **kwargs): + warnings.warn("bigdl.nn.keras is deprecated in BigDL 0.11." + "Recommend to use Analytics Zoo's Keras API.") super(Model, self).__init__(jvalue, to_list(input), to_list(output), From 9c006ce785b9e2d75cf98ede6fcb1f985f85804b Mon Sep 17 00:00:00 2001 From: Le-Zheng <30695225+Le-Zheng@users.noreply.github.com> Date: Tue, 30 Jun 2020 12:50:16 +0800 Subject: [PATCH 774/823] deprecate nn.keras (#3013) * deprecate nn.keras --- .../bigdl/dllib/bigdlkeras/layers/layer.py | 195 ++++++++++++++++++ .../bigdl/dllib/bigdlkeras/layers/topology.py | 15 ++ 2 files changed, 210 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py b/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py index 4d688e36a63..b2d84897dfc 100644 --- a/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py +++ b/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py @@ -25,6 +25,10 @@ class InferShape(JavaValue): + """ + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. + """ def __init__(self, bigdl_type="float"): self.bigdl_type = bigdl_type @@ -58,6 +62,10 @@ def get_output_shape(self): class KerasCreator(JavaValue): + """ + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. + """ def jvm_class_constructor(self): name = "createKeras" + self.__class__.__name__ print("creating: " + name) @@ -65,6 +73,10 @@ def jvm_class_constructor(self): class KerasLayer(Layer, InferShape, KerasCreator): + """ + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. + """ def __init__(self, jvalue, *args, **kwargs): allowed_kwargs = {"name", "bigdl_type"} for kwarg in kwargs.keys(): @@ -89,6 +101,9 @@ class Input(Node, KerasCreator): >>> input = Input(name="input1", shape=(3, 5)) creating: createKerasInput + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, shape=None, name=None, bigdl_type="float"): super(Input, self).__init__(None, bigdl_type, @@ -106,6 +121,9 @@ class InputLayer(KerasLayer): >>> inputlayer = InputLayer(input_shape=(3, 5)) creating: createKerasInputLayer + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, input_shape=None, **kwargs): super(InputLayer, self).__init__(None, @@ -138,6 +156,9 @@ class Dense(KerasLayer): >>> dense = Dense(10, input_dim=8, name="dense1") creating: createKerasDense + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, output_dim, init="glorot_uniform", activation=None, W_regularizer=None, b_regularizer=None, @@ -178,6 +199,9 @@ class MaxoutDense(KerasLayer): >>> maxoutdense = MaxoutDense(6, input_shape=(10, )) creating: createKerasMaxoutDense + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, output_dim, nb_feature=4, W_regularizer=None, b_regularizer=None, bias=True, input_dim=None, input_shape=None, **kwargs): @@ -213,6 +237,9 @@ class Embedding(KerasLayer): >>> embedding = Embedding(1000, 32, input_shape=(10, ), name="embedding1") creating: createKerasEmbedding + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, input_dim, output_dim, init="uniform", W_regularizer=None, input_shape=None, **kwargs): @@ -249,6 +276,9 @@ class BatchNormalization(KerasLayer): >>> batchnormalization = BatchNormalization(input_shape=(3, 12, 12), name="bn1") creating: createKerasBatchNormalization + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, epsilon=0.001, mode=0, axis=1, momentum=0.99, beta_init="zero", gamma_init="one", dim_ordering="th", input_shape=None, **kwargs): @@ -323,6 +353,9 @@ class Merge(KerasLayer): creating: createKerasInputLayer >>> merge = Merge(layers=[l1, l2], mode='sum', name="merge1") creating: createKerasMerge + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, layers=None, mode="sum", concat_axis=-1, input_shape=None, **kwargs): @@ -347,6 +380,9 @@ def merge(inputs, mode="sum", concat_axis=-1, name=None): concat_axis: Int, axis to use when concatenating nodes. Only specify this when merge mode is 'concat'. Default is -1, meaning the last axis of the input. name: String to set the name of the merge. If not specified, its name will by default to be a generated string. + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ return Merge(mode=mode, concat_axis=concat_axis, name=name)(list(inputs)) @@ -366,6 +402,9 @@ class Dropout(KerasLayer): >>> dropout = Dropout(0.25, input_shape=(2, 3)) creating: createKerasDropout + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, p, input_shape=None, **kwargs): super(Dropout, self).__init__(None, @@ -387,6 +426,9 @@ class Flatten(KerasLayer): >>> flatten = Flatten(input_shape=(3, 10, 2)) creating: createKerasFlatten + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, input_shape=None, **kwargs): super(Flatten, self).__init__(None, @@ -411,6 +453,9 @@ class Reshape(KerasLayer): >>> reshape = Reshape((2, 10), input_shape=(5, 4)) creating: createKerasReshape + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, target_shape, input_shape=None, **kwargs): super(Reshape, self).__init__(None, @@ -434,6 +479,9 @@ class Activation(KerasLayer): >>> activation = Activation("relu", input_shape=(3, 4)) creating: createKerasActivation + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, activation, input_shape=None, **kwargs): super(Activation, self).__init__(None, @@ -459,6 +507,9 @@ class RepeatVector(KerasLayer): >>> repeatvector = RepeatVector(5, input_shape=(3, )) creating: createKerasRepeatVector + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, n, input_dim=None, input_shape=None, **kwargs): if input_dim: @@ -484,6 +535,9 @@ class Permute(KerasLayer): >>> permute = Permute((2, 1, 3), input_shape=(3, 4, 5)) creating: createKerasPermute + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, dims, input_shape=None, **kwargs): super(Permute, self).__init__(None, @@ -515,6 +569,9 @@ class Highway(KerasLayer): >>> highway = Highway(activation='relu', input_shape=(8, )) creating: createKerasHighway + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, activation=None, W_regularizer=None, b_regularizer=None, bias=True, input_dim=None, input_shape=None, **kwargs): @@ -557,6 +614,9 @@ class Convolution1D(KerasLayer): >>> conv1d = Convolution1D(12, 4, input_shape=(3, 16)) creating: createKerasConvolution1D + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, nb_filter, filter_length, init="glorot_uniform", activation=None, border_mode="valid", subsample_length=1, @@ -608,6 +668,9 @@ class Convolution2D(KerasLayer): >>> conv2d = Convolution2D(32, 3, 3, input_shape=(3, 128, 128), name="convolution2d_1") creating: createKerasConvolution2D + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, nb_filter, nb_row, nb_col, init="glorot_uniform", activation=None, @@ -663,6 +726,9 @@ class Convolution3D(KerasLayer): >>> conv3d = Convolution3D(32, 3, 4, 5, input_shape=(3, 64, 64, 64)) creating: createKerasConvolution3D + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, nb_filter, kernel_dim1, kernel_dim2, kernel_dim3, init="glorot_uniform", activation=None, border_mode="valid", @@ -716,6 +782,9 @@ class AtrousConvolution1D(KerasLayer): >>> atrousconv1d = AtrousConvolution1D(8, 3, input_shape=(3, 12)) creating: createKerasAtrousConvolution1D + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, nb_filter, filter_length, init="glorot_uniform", activation=None, border_mode="valid", subsample_length=1, atrous_rate=1, W_regularizer=None, @@ -774,6 +843,9 @@ class AtrousConvolution2D(KerasLayer): >>> atrousconv2d = AtrousConvolution2D(12, 4, 3, input_shape=(3, 64, 64)) creating: createKerasAtrousConvolution2D + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, nb_filter, nb_row, nb_col, init="glorot_uniform", activation=None, border_mode="valid", subsample=(1, 1), @@ -837,6 +909,9 @@ class Deconvolution2D(KerasLayer): >>> deconv2d = Deconvolution2D(3, 3, 3, output_shape=(None, 3, 14, 14), input_shape=(3, 12, 12)) creating: createKerasDeconvolution2D + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, nb_filter, nb_row, nb_col, output_shape, init="glorot_uniform", activation=None, border_mode="valid", subsample=(1, 1), dim_ordering="th", @@ -898,6 +973,9 @@ class SeparableConvolution2D(KerasLayer): >>> separableconv2d = SeparableConvolution2D(12, 3, 4, input_shape=(3, 32, 32)) creating: createKerasSeparableConvolution2D + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, nb_filter, nb_row, nb_col, init="glorot_uniform", activation=None, border_mode="valid", subsample=(1, 1), depth_multiplier=1, @@ -946,6 +1024,9 @@ class Cropping1D(KerasLayer): >>> cropping1d = Cropping1D(cropping=(1, 2), input_shape=(8, 8)) creating: createKerasCropping1D + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, cropping=(1, 1), input_shape=None, **kwargs): super(Cropping1D, self).__init__(None, @@ -970,6 +1051,9 @@ class Cropping2D(KerasLayer): >>> cropping2d = Cropping2D(cropping=((1, 2), (0, 1)), input_shape=(12, 12, 12)) creating: createKerasCropping2D + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, cropping=((0, 0), (0, 0)), dim_ordering="th", input_shape=None, **kwargs): @@ -998,6 +1082,9 @@ class Cropping3D(KerasLayer): >>> cropping3d = Cropping3D(cropping=((0, 2), (1, 1), (3, 1)), input_shape=(4, 12, 12, 16)) creating: createKerasCropping3D + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, cropping=((1, 1), (1, 1), (1, 1)), dim_ordering="th", input_shape=None, **kwargs): @@ -1026,6 +1113,9 @@ class UpSampling1D(KerasLayer): >>> upsampling1d = UpSampling1D(length=3, input_shape=(3, 12)) creating: createKerasUpSampling1D + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, length=2, input_shape=None, **kwargs): super(UpSampling1D, self).__init__(None, @@ -1051,6 +1141,9 @@ class UpSampling2D(KerasLayer): >>> upsampling2d = UpSampling2D(size=(1, 3), input_shape=(3, 16, 16)) creating: createKerasUpSampling2D + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, size=(2, 2), dim_ordering="th", input_shape=None, **kwargs): super(UpSampling2D, self).__init__(None, @@ -1078,6 +1171,9 @@ class UpSampling3D(KerasLayer): >>> upsampling3d = UpSampling3D(size=(1, 2, 3), input_shape=(3, 16, 16, 16)) creating: createKerasUpSampling3D + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, size=(2, 2, 2), dim_ordering="th", input_shape=None, **kwargs): super(UpSampling3D, self).__init__(None, @@ -1105,6 +1201,9 @@ class ZeroPadding1D(KerasLayer): >>> zeropadding1d = ZeroPadding1D(padding=2, input_shape=(3, 6)) creating: createKerasZeroPadding1D + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, padding=1, input_shape=None, **kwargs): if isinstance(padding, int): @@ -1134,6 +1233,9 @@ class ZeroPadding2D(KerasLayer): >>> zeropadding2d = ZeroPadding2D(padding=(2, 1), input_shape=(2, 8, 8)) creating: createKerasZeroPadding2D + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, padding=(1, 1), dim_ordering="th", input_shape=None, **kwargs): if len(padding) == 2: @@ -1162,6 +1264,9 @@ class ZeroPadding3D(KerasLayer): >>> zeropadding3d = ZeroPadding3D(padding=(2, 1, 2), input_shape=(2, 8, 8, 10)) creating: createKerasZeroPadding3D + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, padding=(1, 1, 1), dim_ordering="th", input_shape=None, **kwargs): super(ZeroPadding3D, self).__init__(None, @@ -1189,6 +1294,9 @@ class MaxPooling1D(KerasLayer): >>> maxpooling1d = MaxPooling1D(3, input_shape=(3, 24)) creating: createKerasMaxPooling1D + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, pool_length=2, stride=None, border_mode="valid", input_shape=None, **kwargs): @@ -1221,6 +1329,9 @@ class MaxPooling2D(KerasLayer): >>> maxpooling2d = MaxPooling2D((2, 2), input_shape=(3, 32, 32), name="maxpooling2d_1") creating: createKerasMaxPooling2D + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, pool_size=(2, 2), strides=None, border_mode="valid", dim_ordering="th", @@ -1255,6 +1366,9 @@ class MaxPooling3D(KerasLayer): >>> maxpooling3d = MaxPooling3D((2, 1, 3), input_shape=(3, 32, 32, 32)) creating: createKerasMaxPooling3D + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, pool_size=(2, 2, 2), strides=None, border_mode="valid", dim_ordering="th", input_shape=None, **kwargs): @@ -1286,6 +1400,9 @@ class AveragePooling1D(KerasLayer): >>> averagepooling1d = AveragePooling1D(input_shape=(3, 24)) creating: createKerasAveragePooling1D + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, pool_length=2, stride=None, border_mode="valid", input_shape=None, **kwargs): @@ -1318,6 +1435,9 @@ class AveragePooling2D(KerasLayer): >>> averagepooling2d = AveragePooling2D((1, 2), input_shape=(2, 28, 32)) creating: createKerasAveragePooling2D + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, pool_size=(2, 2), strides=None, border_mode="valid", dim_ordering="th", input_shape=None, **kwargs): @@ -1351,6 +1471,9 @@ class AveragePooling3D(KerasLayer): >>> averagepooling3d = AveragePooling3D((1, 1, 2), input_shape=(3, 28, 32, 36)) creating: createKerasAveragePooling3D + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, pool_size=(2, 2, 2), strides=None, border_mode="valid", dim_ordering="th", input_shape=None, **kwargs): @@ -1378,6 +1501,9 @@ class GlobalMaxPooling1D(KerasLayer): >>> globalmaxpooling1d = GlobalMaxPooling1D(input_shape=(4, 8)) creating: createKerasGlobalMaxPooling1D + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, input_shape=None, **kwargs): super(GlobalMaxPooling1D, self).__init__(None, @@ -1399,6 +1525,9 @@ class GlobalAveragePooling1D(KerasLayer): >>> globalaveragepooling1d = GlobalAveragePooling1D(input_shape=(12, 12)) creating: createKerasGlobalAveragePooling1D + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, input_shape=None, **kwargs): super(GlobalAveragePooling1D, self).__init__(None, @@ -1421,6 +1550,9 @@ class GlobalMaxPooling2D(KerasLayer): >>> globalmaxpooling2d = GlobalMaxPooling2D(input_shape=(4, 32, 32)) creating: createKerasGlobalMaxPooling2D + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, dim_ordering="th", input_shape=None, **kwargs): super(GlobalMaxPooling2D, self).__init__(None, @@ -1444,6 +1576,9 @@ class GlobalAveragePooling2D(KerasLayer): >>> globalaveragepooling2d = GlobalAveragePooling2D(input_shape=(4, 32, 32)) creating: createKerasGlobalAveragePooling2D + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, dim_ordering="th", input_shape=None, **kwargs): super(GlobalAveragePooling2D, self).__init__(None, @@ -1469,6 +1604,9 @@ class GlobalMaxPooling3D(KerasLayer): >>> globalmaxpooling3d = GlobalMaxPooling3D(input_shape=(4, 32, 32, 32)) creating: createKerasGlobalMaxPooling3D + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, dim_ordering="th", input_shape=None, **kwargs): super(GlobalMaxPooling3D, self).__init__(None, @@ -1494,6 +1632,9 @@ class GlobalAveragePooling3D(KerasLayer): >>> globalaveragepooling3d = GlobalAveragePooling3D(input_shape=(4, 16, 16, 20)) creating: createKerasGlobalAveragePooling3D + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, dim_ordering="th", input_shape=None, **kwargs): super(GlobalAveragePooling3D, self).__init__(None, @@ -1526,6 +1667,9 @@ class SimpleRNN(KerasLayer): >>> simplernn = SimpleRNN(16, input_shape=(3, 32)) creating: createKerasSimpleRNN + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, output_dim, activation="tanh", return_sequences=False, go_backwards=False, W_regularizer=None, U_regularizer=None, @@ -1567,6 +1711,9 @@ class LSTM(KerasLayer): >>> lstm = LSTM(32, input_shape=(8, 16), name="lstm1") creating: createKerasLSTM + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, output_dim, activation="tanh", inner_activation="hard_sigmoid", return_sequences=False, go_backwards=False, W_regularizer=None, @@ -1609,6 +1756,9 @@ class GRU(KerasLayer): >>> gru = GRU(24, input_shape=(32, 32)) creating: createKerasGRU + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, output_dim, activation="tanh", inner_activation="hard_sigmoid", return_sequences=False, go_backwards=False, W_regularizer=None, @@ -1660,6 +1810,9 @@ class ConvLSTM2D(KerasLayer): >>> convlstm2d = ConvLSTM2D(24, 3, 3, input_shape=(4, 32, 32, 32)) creating: createKerasConvLSTM2D + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, nb_filter, nb_row, nb_col, activation="tanh", inner_activation="hard_sigmoid", dim_ordering="th", border_mode="same", @@ -1714,6 +1867,9 @@ class LocallyConnected1D(KerasLayer): >>> locallyconnected1d = LocallyConnected1D(6, 3, input_shape=(8, 12)) creating: createKerasLocallyConnected1D + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, nb_filter, filter_length, activation=None, border_mode="valid", subsample_length=1, W_regularizer=None, b_regularizer=None, @@ -1761,6 +1917,9 @@ class LocallyConnected2D(KerasLayer): >>> locallyconnected2d = LocallyConnected2D(12, 3, 4, input_shape=(3, 128, 128)) creating: createKerasLocallyConnected2D + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, nb_filter, nb_row, nb_col, activation=None, border_mode="valid", subsample=(1, 1), dim_ordering="th", @@ -1801,6 +1960,9 @@ class SpatialDropout1D(KerasLayer): >>> spatialdropout1d = SpatialDropout1D(0.4, input_shape=(10, 12)) creating: createKerasSpatialDropout1D + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, p=0.5, input_shape=None, **kwargs): super(SpatialDropout1D, self).__init__(None, @@ -1830,6 +1992,9 @@ class SpatialDropout2D(KerasLayer): >>> spatialdropout2d = SpatialDropout2D(0.25, input_shape=(5, 12, 12)) creating: createKerasSpatialDropout2D + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, p=0.5, dim_ordering="th", input_shape=None, **kwargs): super(SpatialDropout2D, self).__init__(None, @@ -1860,6 +2025,9 @@ class SpatialDropout3D(KerasLayer): >>> spatialdropout3d = SpatialDropout3D(0.6, input_shape=(4, 12, 12, 16)) creating: createKerasSpatialDropout3D + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, p=0.5, dim_ordering="th", input_shape=None, **kwargs): super(SpatialDropout3D, self).__init__(None, @@ -1885,6 +2053,9 @@ class GaussianDropout(KerasLayer): >>> gaussiandropout = GaussianDropout(0.45, input_shape=(4, 8)) creating: createKerasGaussianDropout + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, p, input_shape=None, **kwargs): super(GaussianDropout, self).__init__(None, @@ -1910,6 +2081,9 @@ class GaussianNoise(KerasLayer): >>> gaussiannoise = GaussianNoise(0.45, input_shape=(3, 4, 5), name="gaussiannoise1") creating: createKerasGaussianNoise + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, sigma, input_shape=None, **kwargs): super(GaussianNoise, self).__init__(None, @@ -1935,6 +2109,9 @@ class Masking(KerasLayer): >>> masking = Masking(0.3, input_shape=(6, 8)) creating: createKerasMasking + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, mask_value=0.0, input_shape=None, **kwargs): super(Masking, self).__init__(None, @@ -1973,6 +2150,9 @@ class SReLU(KerasLayer): >>> srelu = SReLU(input_shape=(4, 5)) creating: createKerasSReLU + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, t_left_init="zero", a_left_init="glorot_uniform", t_right_init="glorot_uniform", a_right_init="one", @@ -2004,6 +2184,9 @@ class ELU(KerasLayer): >>> elu = ELU(1.2, input_shape=(4, 5)) creating: createKerasELU + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, alpha=1.0, input_shape=None, **kwargs): super(ELU, self).__init__(None, @@ -2029,6 +2212,9 @@ class LeakyReLU(KerasLayer): >>> leakyrelu = LeakyReLU(0.02, input_shape=(4, 5)) creating: createKerasLeakyReLU + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, alpha=0.01, input_shape=None, **kwargs): super(LeakyReLU, self).__init__(None, @@ -2054,6 +2240,9 @@ class ThresholdedReLU(KerasLayer): >>> thresholdedrelu = ThresholdedReLU(input_shape=(10, 12)) creating: createKerasThresholdedReLU + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, theta=1.0, input_shape=None, **kwargs): super(ThresholdedReLU, self).__init__(None, @@ -2080,6 +2269,9 @@ class TimeDistributed(KerasLayer): >>> timedistributed = TimeDistributed(Dense(8), input_shape=(10, 12), name="timedistributeddense") creating: createKerasDense creating: createKerasTimeDistributed + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, layer, input_shape=None, **kwargs): super(TimeDistributed, self).__init__(None, @@ -2109,6 +2301,9 @@ class Bidirectional(KerasLayer): >>> bidiretional = Bidirectional(LSTM(10, return_sequences=True), input_shape=(12, 16), name="bidirectionallstm") creating: createKerasLSTM creating: createKerasBidirectional + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, layer, merge_mode="concat", input_shape=None, **kwargs): super(Bidirectional, self).__init__(None, diff --git a/python/dllib/src/bigdl/dllib/bigdlkeras/layers/topology.py b/python/dllib/src/bigdl/dllib/bigdlkeras/layers/topology.py index 64128b2e6b2..a891635c4c7 100644 --- a/python/dllib/src/bigdl/dllib/bigdlkeras/layers/topology.py +++ b/python/dllib/src/bigdl/dllib/bigdlkeras/layers/topology.py @@ -18,11 +18,16 @@ from bigdl.optim.optimizer import * from bigdl.nn.criterion import * import multiprocessing +import warnings from bigdl.nn.layer import SharedStaticUtils, Container class KerasModel(KerasLayer, Container, SharedStaticUtils): + """ + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. + """ def __convert_optim_method(self, optimizer): optimizer = optimizer.lower() if optimizer == "adagrad": @@ -200,8 +205,13 @@ class Sequential(KerasModel): >>> sequential = Sequential(name="seq1") creating: createKerasSequential + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, jvalue=None, **kwargs): + warnings.warn("bigdl.nn.keras is deprecated in 0.11. " + "Recommend to use Analytics Zoo's Keras API.") super(Sequential, self).__init__(jvalue, **kwargs) @staticmethod @@ -228,8 +238,13 @@ class Model(KerasModel): input: An input node or a list of input nodes. output: An output node or a list of output nodes. name: String to specify the name of the graph model. Default is None. + + .. note:: `bigdl.nn.keras` is deprecated in 0.11. + This will be removed in future releases. """ def __init__(self, input, output, jvalue=None, **kwargs): + warnings.warn("bigdl.nn.keras is deprecated in BigDL 0.11." + "Recommend to use Analytics Zoo's Keras API.") super(Model, self).__init__(jvalue, to_list(input), to_list(output), From 50a07baaecbe9e5fc12f254153794327035bef4b Mon Sep 17 00:00:00 2001 From: Le-Zheng <30695225+Le-Zheng@users.noreply.github.com> Date: Tue, 14 Jul 2020 09:23:10 +0800 Subject: [PATCH 775/823] flip version to 0.12 (#3029) * update --- .../dllib/src/bigdl/dllib/models/textclassifier/README.md | 8 ++++---- version.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/models/textclassifier/README.md b/python/dllib/src/bigdl/dllib/models/textclassifier/README.md index e7aea69346b..ca184a81e0b 100644 --- a/python/dllib/src/bigdl/dllib/models/textclassifier/README.md +++ b/python/dllib/src/bigdl/dllib/models/textclassifier/README.md @@ -28,8 +28,8 @@ $ [/tmp/news20]$ tree . -L 1 - The example code would automatically download the data during the first run. ### Run via pip install -- [Install from pip](https://bigdl-project.github.io/0.11.0-SNAPSHOT/#PythonUserGuide/install-from-pip/) -- Optional: check [Run after pip install](https://bigdl-project.github.io/0.11.0-SNAPSHOT/#PythonUserGuide/run-from-pip/) +- [Install from pip](https://bigdl-project.github.io/0.12.0-SNAPSHOT/#PythonUserGuide/install-from-pip/) +- Optional: check [Run after pip install](https://bigdl-project.github.io/0.12.0-SNAPSHOT/#PythonUserGuide/run-from-pip/) - Run the following command locally ``` python ${BigDL_HOME}/pyspark/bigdl/models/textclassifier/textclassifier.py --max_epoch 3 --model cnn @@ -37,8 +37,8 @@ python ${BigDL_HOME}/pyspark/bigdl/models/textclassifier/textclassifier.py --max ``` ### Run via spark-submit -- [Install without pip](https://bigdl-project.github.io/0.11.0-SNAPSHOT/#PythonUserGuide/install-without-pip/) -- Optional: check [Run without pip](https://bigdl-project.github.io/0.11.0-SNAPSHOT/#PythonUserGuide/run-without-pip/) +- [Install without pip](https://bigdl-project.github.io/0.12.0-SNAPSHOT/#PythonUserGuide/install-without-pip/) +- Optional: check [Run without pip](https://bigdl-project.github.io/0.12.0-SNAPSHOT/#PythonUserGuide/run-without-pip/) - Run the following command ```{r, engine='sh'} PYTHONHASHSEED=0 diff --git a/version.py b/version.py index 4bedc0e98f4..b02a11aac5c 100644 --- a/version.py +++ b/version.py @@ -14,4 +14,4 @@ # limitations under the License. # -__version__ = "0.11.0.dev0" +__version__ = "0.12.0.dev0" From 2eb8795c0872060f9fa77062a78a78245b602595 Mon Sep 17 00:00:00 2001 From: Le-Zheng <30695225+Le-Zheng@users.noreply.github.com> Date: Tue, 14 Jul 2020 09:23:10 +0800 Subject: [PATCH 776/823] flip version to 0.12 (#3029) * update --- .../dllib/src/bigdl/dllib/models/textclassifier/README.md | 8 ++++---- version.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/models/textclassifier/README.md b/python/dllib/src/bigdl/dllib/models/textclassifier/README.md index e7aea69346b..ca184a81e0b 100644 --- a/python/dllib/src/bigdl/dllib/models/textclassifier/README.md +++ b/python/dllib/src/bigdl/dllib/models/textclassifier/README.md @@ -28,8 +28,8 @@ $ [/tmp/news20]$ tree . -L 1 - The example code would automatically download the data during the first run. ### Run via pip install -- [Install from pip](https://bigdl-project.github.io/0.11.0-SNAPSHOT/#PythonUserGuide/install-from-pip/) -- Optional: check [Run after pip install](https://bigdl-project.github.io/0.11.0-SNAPSHOT/#PythonUserGuide/run-from-pip/) +- [Install from pip](https://bigdl-project.github.io/0.12.0-SNAPSHOT/#PythonUserGuide/install-from-pip/) +- Optional: check [Run after pip install](https://bigdl-project.github.io/0.12.0-SNAPSHOT/#PythonUserGuide/run-from-pip/) - Run the following command locally ``` python ${BigDL_HOME}/pyspark/bigdl/models/textclassifier/textclassifier.py --max_epoch 3 --model cnn @@ -37,8 +37,8 @@ python ${BigDL_HOME}/pyspark/bigdl/models/textclassifier/textclassifier.py --max ``` ### Run via spark-submit -- [Install without pip](https://bigdl-project.github.io/0.11.0-SNAPSHOT/#PythonUserGuide/install-without-pip/) -- Optional: check [Run without pip](https://bigdl-project.github.io/0.11.0-SNAPSHOT/#PythonUserGuide/run-without-pip/) +- [Install without pip](https://bigdl-project.github.io/0.12.0-SNAPSHOT/#PythonUserGuide/install-without-pip/) +- Optional: check [Run without pip](https://bigdl-project.github.io/0.12.0-SNAPSHOT/#PythonUserGuide/run-without-pip/) - Run the following command ```{r, engine='sh'} PYTHONHASHSEED=0 diff --git a/version.py b/version.py index 4bedc0e98f4..b02a11aac5c 100644 --- a/version.py +++ b/version.py @@ -14,4 +14,4 @@ # limitations under the License. # -__version__ = "0.11.0.dev0" +__version__ = "0.12.0.dev0" From 0de0e167223be427364bb94c5b368892484c8044 Mon Sep 17 00:00:00 2001 From: Le-Zheng <30695225+Le-Zheng@users.noreply.github.com> Date: Sun, 27 Sep 2020 02:10:54 +0100 Subject: [PATCH 777/823] [WIP] spark 3.0 (#3054) * spark 3.0 --- python/dllib/src/bigdl/dllib/models/utils/model_broadcast.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/python/dllib/src/bigdl/dllib/models/utils/model_broadcast.py b/python/dllib/src/bigdl/dllib/models/utils/model_broadcast.py index d333cb1772f..8e42ee311b8 100644 --- a/python/dllib/src/bigdl/dllib/models/utils/model_broadcast.py +++ b/python/dllib/src/bigdl/dllib/models/utils/model_broadcast.py @@ -19,7 +19,6 @@ import gc from tempfile import NamedTemporaryFile -from pyspark.cloudpickle import print_exec from pyspark.broadcast import Broadcast from pyspark.broadcast import _from_id from bigdl.nn.layer import Model @@ -49,6 +48,10 @@ def dump(self, value, f): value.saveModel(f.name, over_write=True) except Exception as e: msg = "Could not serialize broadcast: %s" % e.__class__.__name__ + if not self.sc.version.startswith("2.1"): + from pyspark.cloudpickle import print_exec + else: + from pyspark.util import print_exec print_exec(sys.stderr) raise ValueError(msg) f.close() From ea6c4a6e8a180c358f69efb61a1f0530329b7181 Mon Sep 17 00:00:00 2001 From: Le-Zheng <30695225+Le-Zheng@users.noreply.github.com> Date: Sun, 27 Sep 2020 02:10:54 +0100 Subject: [PATCH 778/823] [WIP] spark 3.0 (#3054) * spark 3.0 --- python/dllib/src/bigdl/dllib/models/utils/model_broadcast.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/python/dllib/src/bigdl/dllib/models/utils/model_broadcast.py b/python/dllib/src/bigdl/dllib/models/utils/model_broadcast.py index d333cb1772f..8e42ee311b8 100644 --- a/python/dllib/src/bigdl/dllib/models/utils/model_broadcast.py +++ b/python/dllib/src/bigdl/dllib/models/utils/model_broadcast.py @@ -19,7 +19,6 @@ import gc from tempfile import NamedTemporaryFile -from pyspark.cloudpickle import print_exec from pyspark.broadcast import Broadcast from pyspark.broadcast import _from_id from bigdl.nn.layer import Model @@ -49,6 +48,10 @@ def dump(self, value, f): value.saveModel(f.name, over_write=True) except Exception as e: msg = "Could not serialize broadcast: %s" % e.__class__.__name__ + if not self.sc.version.startswith("2.1"): + from pyspark.cloudpickle import print_exec + else: + from pyspark.util import print_exec print_exec(sys.stderr) raise ValueError(msg) f.close() From b39fbb1a25969ccc0160d09085dd6625422b4a95 Mon Sep 17 00:00:00 2001 From: Le-Zheng <30695225+Le-Zheng@users.noreply.github.com> Date: Sun, 27 Sep 2020 02:10:54 +0100 Subject: [PATCH 779/823] [WIP] spark 3.0 (#3054) * spark 3.0 --- python/dllib/test/bigdl/dlframes/test_dl_image_transformer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/dllib/test/bigdl/dlframes/test_dl_image_transformer.py b/python/dllib/test/bigdl/dlframes/test_dl_image_transformer.py index 23e1d7c61a6..abdf3ce6203 100644 --- a/python/dllib/test/bigdl/dlframes/test_dl_image_transformer.py +++ b/python/dllib/test/bigdl/dlframes/test_dl_image_transformer.py @@ -49,7 +49,7 @@ def test_transform_image(self): # test, and withhold the support for Spark 1.5, until the unit test failure reason # is clarified. - if not self.sc.version.startswith("1.5"): + if not self.sc.version.startswith("1.5" and "3.0"): image_frame = DLImageReader.readImages(self.image_path, self.sc) transformer = DLImageTransformer( Pipeline([Resize(256, 256), CenterCrop(224, 224), From 1a8f3708cd91637056e8ad27c250b014bb306bac Mon Sep 17 00:00:00 2001 From: Le-Zheng <30695225+Le-Zheng@users.noreply.github.com> Date: Fri, 23 Oct 2020 08:25:13 +0100 Subject: [PATCH 780/823] add warning to remind Optimizer() deprecates (#3062) * add warning to remind deprecates --- python/dllib/src/bigdl/dllib/optim/optimizer.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/optim/optimizer.py b/python/dllib/src/bigdl/dllib/optim/optimizer.py index c3ea9b137bb..793b7cda31e 100644 --- a/python/dllib/src/bigdl/dllib/optim/optimizer.py +++ b/python/dllib/src/bigdl/dllib/optim/optimizer.py @@ -32,6 +32,7 @@ from bigdl.util.common import init_engine from bigdl.util.common import to_list from bigdl.dataset.dataset import * +import warnings if sys.version >= '3': long = int @@ -860,6 +861,8 @@ def __init__(self, :param end_trigger: when to end the optimization :param batch_size: training batch size """ + warnings.warn("You are recommended to use `create` method to create an optimizer.") + assert isinstance(training_rdd, RDD), "Only type of RDD is allowed" self.pvalue = DistriOptimizer(model, training_rdd, criterion, From 12ba3e794bf825fbaca3d69e4e11d9d2d73b807c Mon Sep 17 00:00:00 2001 From: Le-Zheng <30695225+Le-Zheng@users.noreply.github.com> Date: Fri, 23 Oct 2020 08:25:13 +0100 Subject: [PATCH 781/823] add warning to remind Optimizer() deprecates (#3062) * add warning to remind deprecates --- python/dllib/src/bigdl/dllib/optim/optimizer.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/optim/optimizer.py b/python/dllib/src/bigdl/dllib/optim/optimizer.py index c3ea9b137bb..793b7cda31e 100644 --- a/python/dllib/src/bigdl/dllib/optim/optimizer.py +++ b/python/dllib/src/bigdl/dllib/optim/optimizer.py @@ -32,6 +32,7 @@ from bigdl.util.common import init_engine from bigdl.util.common import to_list from bigdl.dataset.dataset import * +import warnings if sys.version >= '3': long = int @@ -860,6 +861,8 @@ def __init__(self, :param end_trigger: when to end the optimization :param batch_size: training batch size """ + warnings.warn("You are recommended to use `create` method to create an optimizer.") + assert isinstance(training_rdd, RDD), "Only type of RDD is allowed" self.pvalue = DistriOptimizer(model, training_rdd, criterion, From 45efa7b68019a635a19399a3624bd8cbc6b8bfe7 Mon Sep 17 00:00:00 2001 From: Le-Zheng <30695225+Le-Zheng@users.noreply.github.com> Date: Tue, 17 Nov 2020 08:14:32 +0000 Subject: [PATCH 782/823] flip version to 0.13-snapshot (#3074) * flip version to 0.13-snapshot --- .../dllib/src/bigdl/dllib/models/textclassifier/README.md | 8 ++++---- version.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/models/textclassifier/README.md b/python/dllib/src/bigdl/dllib/models/textclassifier/README.md index ca184a81e0b..5150e737b75 100644 --- a/python/dllib/src/bigdl/dllib/models/textclassifier/README.md +++ b/python/dllib/src/bigdl/dllib/models/textclassifier/README.md @@ -28,8 +28,8 @@ $ [/tmp/news20]$ tree . -L 1 - The example code would automatically download the data during the first run. ### Run via pip install -- [Install from pip](https://bigdl-project.github.io/0.12.0-SNAPSHOT/#PythonUserGuide/install-from-pip/) -- Optional: check [Run after pip install](https://bigdl-project.github.io/0.12.0-SNAPSHOT/#PythonUserGuide/run-from-pip/) +- [Install from pip](https://bigdl-project.github.io/0.13.0-SNAPSHOT/#PythonUserGuide/install-from-pip/) +- Optional: check [Run after pip install](https://bigdl-project.github.io/0.13.0-SNAPSHOT/#PythonUserGuide/run-from-pip/) - Run the following command locally ``` python ${BigDL_HOME}/pyspark/bigdl/models/textclassifier/textclassifier.py --max_epoch 3 --model cnn @@ -37,8 +37,8 @@ python ${BigDL_HOME}/pyspark/bigdl/models/textclassifier/textclassifier.py --max ``` ### Run via spark-submit -- [Install without pip](https://bigdl-project.github.io/0.12.0-SNAPSHOT/#PythonUserGuide/install-without-pip/) -- Optional: check [Run without pip](https://bigdl-project.github.io/0.12.0-SNAPSHOT/#PythonUserGuide/run-without-pip/) +- [Install without pip](https://bigdl-project.github.io/0.13.0-SNAPSHOT/#PythonUserGuide/install-without-pip/) +- Optional: check [Run without pip](https://bigdl-project.github.io/0.13.0-SNAPSHOT/#PythonUserGuide/run-without-pip/) - Run the following command ```{r, engine='sh'} PYTHONHASHSEED=0 diff --git a/version.py b/version.py index b02a11aac5c..4f81c266b13 100644 --- a/version.py +++ b/version.py @@ -14,4 +14,4 @@ # limitations under the License. # -__version__ = "0.12.0.dev0" +__version__ = "0.13.0.dev0" From 2d20c85f707a232ef0915cc1cf5735cc0d431dc7 Mon Sep 17 00:00:00 2001 From: Le-Zheng <30695225+Le-Zheng@users.noreply.github.com> Date: Tue, 17 Nov 2020 08:14:32 +0000 Subject: [PATCH 783/823] flip version to 0.13-snapshot (#3074) * flip version to 0.13-snapshot --- .../dllib/src/bigdl/dllib/models/textclassifier/README.md | 8 ++++---- version.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/models/textclassifier/README.md b/python/dllib/src/bigdl/dllib/models/textclassifier/README.md index ca184a81e0b..5150e737b75 100644 --- a/python/dllib/src/bigdl/dllib/models/textclassifier/README.md +++ b/python/dllib/src/bigdl/dllib/models/textclassifier/README.md @@ -28,8 +28,8 @@ $ [/tmp/news20]$ tree . -L 1 - The example code would automatically download the data during the first run. ### Run via pip install -- [Install from pip](https://bigdl-project.github.io/0.12.0-SNAPSHOT/#PythonUserGuide/install-from-pip/) -- Optional: check [Run after pip install](https://bigdl-project.github.io/0.12.0-SNAPSHOT/#PythonUserGuide/run-from-pip/) +- [Install from pip](https://bigdl-project.github.io/0.13.0-SNAPSHOT/#PythonUserGuide/install-from-pip/) +- Optional: check [Run after pip install](https://bigdl-project.github.io/0.13.0-SNAPSHOT/#PythonUserGuide/run-from-pip/) - Run the following command locally ``` python ${BigDL_HOME}/pyspark/bigdl/models/textclassifier/textclassifier.py --max_epoch 3 --model cnn @@ -37,8 +37,8 @@ python ${BigDL_HOME}/pyspark/bigdl/models/textclassifier/textclassifier.py --max ``` ### Run via spark-submit -- [Install without pip](https://bigdl-project.github.io/0.12.0-SNAPSHOT/#PythonUserGuide/install-without-pip/) -- Optional: check [Run without pip](https://bigdl-project.github.io/0.12.0-SNAPSHOT/#PythonUserGuide/run-without-pip/) +- [Install without pip](https://bigdl-project.github.io/0.13.0-SNAPSHOT/#PythonUserGuide/install-without-pip/) +- Optional: check [Run without pip](https://bigdl-project.github.io/0.13.0-SNAPSHOT/#PythonUserGuide/run-without-pip/) - Run the following command ```{r, engine='sh'} PYTHONHASHSEED=0 diff --git a/version.py b/version.py index b02a11aac5c..4f81c266b13 100644 --- a/version.py +++ b/version.py @@ -14,4 +14,4 @@ # limitations under the License. # -__version__ = "0.12.0.dev0" +__version__ = "0.13.0.dev0" From 196db7dac8fa10bdc6408043c0c094a5cfd88c6f Mon Sep 17 00:00:00 2001 From: Xin Qiu Date: Tue, 23 Mar 2021 16:18:55 +0800 Subject: [PATCH 784/823] fix callBigDLFunc return a Int while the true return value from java is a byte array. (#3111) --- python/dllib/src/bigdl/dllib/utils/common.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/utils/common.py b/python/dllib/src/bigdl/dllib/utils/common.py index a58d61629dd..f15d7a554af 100644 --- a/python/dllib/src/bigdl/dllib/utils/common.py +++ b/python/dllib/src/bigdl/dllib/utils/common.py @@ -634,8 +634,8 @@ def _java2py(gateway, r, encoding="bytes"): except Py4JJavaError: pass # not pickable - if isinstance(r, (bytearray, bytes)): - r = PickleSerializer().loads(bytes(r), encoding=encoding) + if isinstance(r, (bytearray, bytes)): + r = PickleSerializer().loads(bytes(r), encoding=encoding) return r From 390aae478a3ab2a0478aed5c45a767ff044cad58 Mon Sep 17 00:00:00 2001 From: Xin Qiu Date: Tue, 23 Mar 2021 16:18:55 +0800 Subject: [PATCH 785/823] fix callBigDLFunc return a Int while the true return value from java is a byte array. (#3111) --- python/dllib/src/bigdl/utils/common.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/dllib/src/bigdl/utils/common.py b/python/dllib/src/bigdl/utils/common.py index a58d61629dd..f15d7a554af 100644 --- a/python/dllib/src/bigdl/utils/common.py +++ b/python/dllib/src/bigdl/utils/common.py @@ -634,8 +634,8 @@ def _java2py(gateway, r, encoding="bytes"): except Py4JJavaError: pass # not pickable - if isinstance(r, (bytearray, bytes)): - r = PickleSerializer().loads(bytes(r), encoding=encoding) + if isinstance(r, (bytearray, bytes)): + r = PickleSerializer().loads(bytes(r), encoding=encoding) return r From 1c3931948da18de4e1256d6c5b54289627ce27fa Mon Sep 17 00:00:00 2001 From: Yina Chen <33650826+cyita@users.noreply.github.com> Date: Thu, 25 Mar 2021 14:24:08 +0800 Subject: [PATCH 786/823] add list of df support (#3113) --- python/dllib/src/bigdl/dllib/utils/common.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/python/dllib/src/bigdl/dllib/utils/common.py b/python/dllib/src/bigdl/dllib/utils/common.py index f15d7a554af..f4f51834af3 100644 --- a/python/dllib/src/bigdl/dllib/utils/common.py +++ b/python/dllib/src/bigdl/dllib/utils/common.py @@ -627,6 +627,11 @@ def _java2py(gateway, r, encoding="bytes"): if clsName in _picklable_classes: r = gateway.jvm.org.apache.spark.bigdl.api.python.BigDLSerDe.dumps(r) + elif isinstance(r, (JavaArray, JavaList)) and len(r) != 0 \ + and isinstance(r[0], JavaObject) \ + and r[0].getClass().getSimpleName() in ['DataFrame', 'Dataset']: + spark = get_spark_sql_context(get_spark_context()) + r = list(map(lambda x: DataFrame(x, spark), r)) elif isinstance(r, (JavaArray, JavaList, JavaMap)): try: r = gateway.jvm.org.apache.spark.bigdl.api.python.BigDLSerDe.dumps( From ec860ae4db752d0f7d28cdf0ad7bd2945477d236 Mon Sep 17 00:00:00 2001 From: Yina Chen <33650826+cyita@users.noreply.github.com> Date: Thu, 25 Mar 2021 14:24:08 +0800 Subject: [PATCH 787/823] add list of df support (#3113) --- python/dllib/src/bigdl/utils/common.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/python/dllib/src/bigdl/utils/common.py b/python/dllib/src/bigdl/utils/common.py index f15d7a554af..f4f51834af3 100644 --- a/python/dllib/src/bigdl/utils/common.py +++ b/python/dllib/src/bigdl/utils/common.py @@ -627,6 +627,11 @@ def _java2py(gateway, r, encoding="bytes"): if clsName in _picklable_classes: r = gateway.jvm.org.apache.spark.bigdl.api.python.BigDLSerDe.dumps(r) + elif isinstance(r, (JavaArray, JavaList)) and len(r) != 0 \ + and isinstance(r[0], JavaObject) \ + and r[0].getClass().getSimpleName() in ['DataFrame', 'Dataset']: + spark = get_spark_sql_context(get_spark_context()) + r = list(map(lambda x: DataFrame(x, spark), r)) elif isinstance(r, (JavaArray, JavaList, JavaMap)): try: r = gateway.jvm.org.apache.spark.bigdl.api.python.BigDLSerDe.dumps( From 7653b9a5bb4e50bcab470df70dd830893f3a3ecd Mon Sep 17 00:00:00 2001 From: Le-Zheng <30695225+Le-Zheng@users.noreply.github.com> Date: Tue, 25 May 2021 06:33:08 +0100 Subject: [PATCH 788/823] remove DLFrames (#3124) * remove DLFrames * update * update * update * rm dlframe example from test script --- .../test/bigdl/dlframes/test_dl_classifier.py | 173 ------------------ .../bigdl/dlframes/test_dl_image_reader.py | 58 ------ .../dlframes/test_dl_image_transformer.py | 71 ------- python/dllib/test/bigdl/test_dl_classifier.py | 173 ------------------ 4 files changed, 475 deletions(-) delete mode 100644 python/dllib/test/bigdl/dlframes/test_dl_classifier.py delete mode 100644 python/dllib/test/bigdl/dlframes/test_dl_image_reader.py delete mode 100644 python/dllib/test/bigdl/dlframes/test_dl_image_transformer.py delete mode 100644 python/dllib/test/bigdl/test_dl_classifier.py diff --git a/python/dllib/test/bigdl/dlframes/test_dl_classifier.py b/python/dllib/test/bigdl/dlframes/test_dl_classifier.py deleted file mode 100644 index 4325abe63e8..00000000000 --- a/python/dllib/test/bigdl/dlframes/test_dl_classifier.py +++ /dev/null @@ -1,173 +0,0 @@ -# -# Copyright 2016 The BigDL Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from bigdl.nn.layer import * -from bigdl.nn.criterion import * -from bigdl.util.common import * -from bigdl.dlframes.dl_classifier import * -from pyspark.sql.types import * -from pyspark.context import SparkContext -import numpy as np -import pytest -from numpy.testing import assert_allclose - - -class TestDLClassifer(): - def setup_method(self, method): - """ setup any state tied to the execution of the given method in a - class. setup_method is invoked for every test method of a class. - """ - sparkConf = create_spark_conf().setMaster("local[1]").setAppName("test model") - self.sc = get_spark_context(sparkConf) - self.sqlContext = SQLContext(self.sc) - init_engine() - - def teardown_method(self, method): - """ teardown any state that was previously setup with a setup_method - call. - """ - self.sc.stop() - - def test_all_set_get_methods(self): - """ run tests for all the set and get methods involved in DLEstimator, DLModel, - DLClassifier, DLClassifierModel - """ - - ''' - use linear model and MSE criterion to test the DLEstimator and DLModel - ''' - linear_model = Sequential().add(Linear(2, 2)) - mse_criterion = MSECriterion() - - ''' - initialize a DLEstimator to test setter, getter for - batchSize, maxEpoch, learningRate in DLEstimator - ''' - estimator = DLEstimator(model=linear_model, criterion=mse_criterion, - feature_size=[2], label_size=[2]) - - # check the set and get methods for batchSize - assert estimator.setBatchSize(30).getBatchSize() == 30 - # check the set and get methods for maxEpoch - assert estimator.setMaxEpoch(40).getMaxEpoch() == 40 - # check the set and get methods for learningRate - assert estimator.setLearningRate(1e-4).getLearningRate() == 1e-4 - - ''' - initialize a DLModel to test setter, getter for featureSize, batchSize in DLModel - ''' - dl_model = DLClassifierModel(model=linear_model, featureSize=[1]) - - # check the set and get methods for featureSize in DLModel - assert dl_model.setFeatureSize([2]).getFeatureSize() == [2] - # check the set and get methods for batchSize in DLModel - assert dl_model.setBatchSize((20)).getBatchSize() == 20 - - ''' - use linear model and ClassNLL criterion to test the DLClassifier and DLClassifierModel - ''' - linear_model = Sequential().add(Linear(2, 2)) - classNLL_criterion = ClassNLLCriterion() - - ''' - initialize a DLClassifier to test setter, getter for - batchSize, maxEpoch, learningRate in DLClassifier - ''' - classifier = DLClassifier(model=linear_model, criterion=classNLL_criterion, - feature_size=[2]) - - # check the set and get methods for batchSize - assert classifier.setBatchSize(20).getBatchSize() == 20 - # check the set and get methods for maxEpoch - assert classifier.setMaxEpoch(50).getMaxEpoch() == 50 - # check the set and get methods for learningRate - assert classifier.setLearningRate(1e-5).getLearningRate() == 1e-5 - ''' - initialize a DLClassifierModel to test setter, getter - for featureSize, batchSize in DLClassifierModel - ''' - dl_classifier_model = DLClassifierModel(model=linear_model, featureSize=[1]) - - # check the set and get methods for featureSize - assert dl_classifier_model.setFeatureSize([2]).getFeatureSize() == [2] - - # check the set and get methods for batchSize - assert dl_classifier_model.setBatchSize((20)).getBatchSize() == 20 - - def test_dlestimator_fit_dlmodel_transform(self): - model = Sequential().add(Linear(2, 2)) - criterion = MSECriterion() - estimator = DLEstimator(model, criterion, [2], [2]).setBatchSize(4)\ - .setLearningRate(0.01).setMaxEpoch(1000) - - data = self.sc.parallelize([ - ((2.0, 1.0), (1.0, 2.0)), - ((1.0, 2.0), (2.0, 1.0)), - ((2.0, 1.0), (1.0, 2.0)), - ((1.0, 2.0), (2.0, 1.0))]) - - schema = StructType([ - StructField("features", ArrayType(DoubleType(), False), False), - StructField("label", ArrayType(DoubleType(), False), False)]) - df = self.sqlContext.createDataFrame(data, schema) - dlModel = estimator.fit(df) - - res = dlModel.transform(df) - assert type(res).__name__ == 'DataFrame' - res.registerTempTable("dlModelDF") # Compatible with spark 1.6 - results = self.sqlContext.table("dlModelDF") - - count = results.rdd.count() - data = results.rdd.collect() - - for i in range(count): - row_label = data[i][1] - row_prediction = data[i][2] - assert_allclose(row_label[0], row_prediction[0], atol=0, rtol=1e-1) - assert_allclose(row_label[1], row_prediction[1], atol=0, rtol=1e-1) - - def test_dlclassifier_fit_dlclassifiermodel_transform(self): - model = Sequential().add(Linear(2, 2)) - criterion = ClassNLLCriterion() - classifier = DLClassifier(model, criterion, [2]).setBatchSize(4)\ - .setLearningRate(0.01).setMaxEpoch(1000) - data = self.sc.parallelize([ - ((2.0, 1.0), 1.0), - ((1.0, 2.0), 2.0), - ((2.0, 1.0), 1.0), - ((1.0, 2.0), 2.0)]) - - schema = StructType([ - StructField("features", ArrayType(DoubleType(), False), False), - StructField("label", DoubleType(), False)]) - df = self.sqlContext.createDataFrame(data, schema) - dlClassifierModel = classifier.fit(df) - - res = dlClassifierModel.transform(df) - assert type(res).__name__ == 'DataFrame' - res.registerTempTable("dlClassifierModelDF") - results = self.sqlContext.table("dlClassifierModelDF") - - count = results.rdd.count() - data = results.rdd.collect() - - for i in range(count): - row_label = data[i][1] - row_prediction = data[i][2] - assert row_label == row_prediction - -if __name__ == "__main__": - pytest.main() diff --git a/python/dllib/test/bigdl/dlframes/test_dl_image_reader.py b/python/dllib/test/bigdl/dlframes/test_dl_image_reader.py deleted file mode 100644 index 2a40e7cf267..00000000000 --- a/python/dllib/test/bigdl/dlframes/test_dl_image_reader.py +++ /dev/null @@ -1,58 +0,0 @@ -# -# Copyright 2016 The BigDL Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import pytest -import os -from bigdl.dlframes.dl_image_reader import * -from bigdl.transform.vision.image import * -from bigdl.util.common import * -from pyspark.sql.types import * - - -class TestDLImageReader(): - - def setup_method(self, method): - """ - setup any state tied to the execution of the given method in a - class. setup_method is invoked for every test method of a class. - """ - sparkConf = create_spark_conf().setMaster("local[1]").setAppName("test model") - self.sc = get_spark_context(sparkConf) - init_engine() - self.resource_path = os.path.join(os.path.split(__file__)[0], "../resources") - - def teardown_method(self, method): - """ - teardown any state that was previously setup with a setup_method - call. - """ - self.sc.stop() - - def test_get_pascal_image(self): - image_path = os.path.join(self.resource_path, "pascal/000025.jpg") - image_frame = DLImageReader.readImages(image_path, self.sc) - assert image_frame.count() == 1 - assert type(image_frame).__name__ == 'DataFrame' - first_row = image_frame.take(1)[0][0] - assert first_row[0].endswith("pascal/000025.jpg") - assert first_row[1] == 375 - assert first_row[2] == 500 - assert first_row[3] == 3 - assert first_row[4] == 16 - assert len(first_row[5]) == 95959 - -if __name__ == "__main__": - pytest.main([__file__]) diff --git a/python/dllib/test/bigdl/dlframes/test_dl_image_transformer.py b/python/dllib/test/bigdl/dlframes/test_dl_image_transformer.py deleted file mode 100644 index abdf3ce6203..00000000000 --- a/python/dllib/test/bigdl/dlframes/test_dl_image_transformer.py +++ /dev/null @@ -1,71 +0,0 @@ -# -# Copyright 2016 The BigDL Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import pytest -import os -from bigdl.dlframes.dl_image_reader import * -from bigdl.dlframes.dl_image_transformer import * -from bigdl.util.common import * -from bigdl.transform.vision.image import * - - -class TestDLImageTransformer(): - - def setup_method(self, method): - """ - setup any state tied to the execution of the given method in a - class. setup_method is invoked for every test method of a class. - """ - sparkConf = create_spark_conf().setMaster("local[1]").setAppName("test model") - self.sc = get_spark_context(sparkConf) - init_engine() - resource_path = os.path.join(os.path.split(__file__)[0], "../resources") - self.image_path = os.path.join(resource_path, "pascal/000025.jpg") - - def teardown_method(self, method): - """ - teardown any state that was previously setup with a setup_method - call. - """ - self.sc.stop() - - def test_transform_image(self): - # Current the unit test does not work under Spark 1.5, The error message: - # "java.lang.IllegalStateException: Cannot call methods on a stopped SparkContext" - # Yet the transformer works during manual test in 1.5, thus we bypass the 1.5 unit - # test, and withhold the support for Spark 1.5, until the unit test failure reason - # is clarified. - - if not self.sc.version.startswith("1.5" and "3.0"): - image_frame = DLImageReader.readImages(self.image_path, self.sc) - transformer = DLImageTransformer( - Pipeline([Resize(256, 256), CenterCrop(224, 224), - ChannelNormalize(0.485, 0.456, 0.406, 0.229, 0.224, 0.225), - MatToTensor()]) - ).setInputCol("image").setOutputCol("output") - - result = transformer.transform(image_frame) - assert(result.count() == 1) - first_row = result.select("output").take(1)[0][0] - assert first_row[0].endswith("pascal/000025.jpg") - assert first_row[1] == 224 - assert first_row[2] == 224 - assert first_row[3] == 3 - assert first_row[4] == 21 - assert len(first_row[5]) == 224 * 224 * 3 - -if __name__ == "__main__": - pytest.main([__file__]) diff --git a/python/dllib/test/bigdl/test_dl_classifier.py b/python/dllib/test/bigdl/test_dl_classifier.py deleted file mode 100644 index 9bedbce686f..00000000000 --- a/python/dllib/test/bigdl/test_dl_classifier.py +++ /dev/null @@ -1,173 +0,0 @@ -# -# Copyright 2016 The BigDL Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from bigdl.nn.layer import * -from bigdl.nn.criterion import * -from bigdl.util.common import * -from bigdl.models.ml_pipeline.dl_classifier import * -from pyspark.sql.types import * -from pyspark.context import SparkContext -import numpy as np -import pytest -from numpy.testing import assert_allclose - - -class TestDLClassifer(): - def setup_method(self, method): - """ setup any state tied to the execution of the given method in a - class. setup_method is invoked for every test method of a class. - """ - sparkConf = create_spark_conf().setMaster("local[1]").setAppName("test model") - self.sc = get_spark_context(sparkConf) - self.sqlContext = SQLContext(self.sc) - init_engine() - - def teardown_method(self, method): - """ teardown any state that was previously setup with a setup_method - call. - """ - self.sc.stop() - - def test_all_set_get_methods(self): - """ run tests for all the set and get methods involved in DLEstimator, DLModel, - DLClassifier, DLClassifierModel - """ - - ''' - use linear model and MSE criterion to test the DLEstimator and DLModel - ''' - linear_model = Sequential().add(Linear(2, 2)) - mse_criterion = MSECriterion() - - ''' - initialize a DLEstimator to test setter, getter for - batchSize, maxEpoch, learningRate in DLEstimator - ''' - estimator = DLEstimator(model=linear_model, criterion=mse_criterion, - feature_size=[2], label_size=[2]) - - # check the set and get methods for batchSize - assert estimator.setBatchSize(30).getBatchSize() == 30 - # check the set and get methods for maxEpoch - assert estimator.setMaxEpoch(40).getMaxEpoch() == 40 - # check the set and get methods for learningRate - assert estimator.setLearningRate(1e-4).getLearningRate() == 1e-4 - - ''' - initialize a DLModel to test setter, getter for featureSize, batchSize in DLModel - ''' - dl_model = DLClassifierModel(model=linear_model, featureSize=[1]) - - # check the set and get methods for featureSize in DLModel - assert dl_model.setFeatureSize([2]).getFeatureSize() == [2] - # check the set and get methods for batchSize in DLModel - assert dl_model.setBatchSize((20)).getBatchSize() == 20 - - ''' - use linear model and ClassNLL criterion to test the DLClassifier and DLClassifierModel - ''' - linear_model = Sequential().add(Linear(2, 2)) - classNLL_criterion = ClassNLLCriterion() - - ''' - initialize a DLClassifier to test setter, getter for - batchSize, maxEpoch, learningRate in DLClassifier - ''' - classifier = DLClassifier(model=linear_model, criterion=classNLL_criterion, - feature_size=[2]) - - # check the set and get methods for batchSize - assert classifier.setBatchSize(20).getBatchSize() == 20 - # check the set and get methods for maxEpoch - assert classifier.setMaxEpoch(50).getMaxEpoch() == 50 - # check the set and get methods for learningRate - assert classifier.setLearningRate(1e-5).getLearningRate() == 1e-5 - ''' - initialize a DLClassifierModel to test setter, getter - for featureSize, batchSize in DLClassifierModel - ''' - dl_classifier_model = DLClassifierModel(model=linear_model, featureSize=[1]) - - # check the set and get methods for featureSize - assert dl_classifier_model.setFeatureSize([2]).getFeatureSize() == [2] - - # check the set and get methods for batchSize - assert dl_classifier_model.setBatchSize((20)).getBatchSize() == 20 - - def test_dlestimator_fit_dlmodel_transform(self): - model = Sequential().add(Linear(2, 2)) - criterion = MSECriterion() - estimator = DLEstimator(model, criterion, [2], [2]).setBatchSize(4)\ - .setLearningRate(0.01).setMaxEpoch(1000) - - data = self.sc.parallelize([ - ((2.0, 1.0), (1.0, 2.0)), - ((1.0, 2.0), (2.0, 1.0)), - ((2.0, 1.0), (1.0, 2.0)), - ((1.0, 2.0), (2.0, 1.0))]) - - schema = StructType([ - StructField("features", ArrayType(DoubleType(), False), False), - StructField("label", ArrayType(DoubleType(), False), False)]) - df = self.sqlContext.createDataFrame(data, schema) - dlModel = estimator.fit(df) - - res = dlModel.transform(df) - assert type(res).__name__ == 'DataFrame' - res.registerTempTable("dlModelDF") # Compatible with spark 1.6 - results = self.sqlContext.table("dlModelDF") - - count = results.rdd.count() - data = results.rdd.collect() - - for i in range(count): - row_label = data[i][1] - row_prediction = data[i][2] - assert_allclose(row_label[0], row_prediction[0], atol=0, rtol=1e-1) - assert_allclose(row_label[1], row_prediction[1], atol=0, rtol=1e-1) - - def test_dlclassifier_fit_dlclassifiermodel_transform(self): - model = Sequential().add(Linear(2, 2)) - criterion = ClassNLLCriterion() - classifier = DLClassifier(model, criterion, [2]).setBatchSize(4)\ - .setLearningRate(0.01).setMaxEpoch(1000) - data = self.sc.parallelize([ - ((2.0, 1.0), 1.0), - ((1.0, 2.0), 2.0), - ((2.0, 1.0), 1.0), - ((1.0, 2.0), 2.0)]) - - schema = StructType([ - StructField("features", ArrayType(DoubleType(), False), False), - StructField("label", DoubleType(), False)]) - df = self.sqlContext.createDataFrame(data, schema) - dlClassifierModel = classifier.fit(df) - - res = dlClassifierModel.transform(df) - assert type(res).__name__ == 'DataFrame' - res.registerTempTable("dlClassifierModelDF") - results = self.sqlContext.table("dlClassifierModelDF") - - count = results.rdd.count() - data = results.rdd.collect() - - for i in range(count): - row_label = data[i][1] - row_prediction = data[i][2] - assert row_label == row_prediction - -if __name__ == "__main__": - pytest.main() From 080ee308961f3f15e2b79ed8d67c97e6478e0f63 Mon Sep 17 00:00:00 2001 From: Le-Zheng <30695225+Le-Zheng@users.noreply.github.com> Date: Tue, 22 Jun 2021 02:49:35 +0100 Subject: [PATCH 789/823] make default DistriOptimizer as V2 (#3129) * make default DistriOptimizer as V2 * update --- python/dllib/src/bigdl/dllib/examples/keras/README.md | 2 +- python/dllib/src/bigdl/dllib/examples/keras/imdb_cnn_lstm.py | 2 +- python/dllib/src/bigdl/dllib/examples/keras/mnist_cnn.py | 2 +- python/dllib/src/bigdl/dllib/models/lenet/lenet5.py | 2 +- .../src/bigdl/dllib/models/textclassifier/textclassifier.py | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/examples/keras/README.md b/python/dllib/src/bigdl/dllib/examples/keras/README.md index fa4fb56a716..af4ece12bc1 100644 --- a/python/dllib/src/bigdl/dllib/examples/keras/README.md +++ b/python/dllib/src/bigdl/dllib/examples/keras/README.md @@ -50,4 +50,4 @@ ${SPARK_HOME}/bin/spark-submit \ ``` * ```--batchSize``` an option that can be used to set batch size. * ```--max_epoch``` an option that can be used to set how many epochs for which the model is to be trained. -* ```--optimizerVersion``` an option that can be used to set DistriOptimizer version, the default value is "optimizerV1". \ No newline at end of file +* ```--optimizerVersion``` an option that can be used to set DistriOptimizer version, the default value is "optimizerV2". \ No newline at end of file diff --git a/python/dllib/src/bigdl/dllib/examples/keras/imdb_cnn_lstm.py b/python/dllib/src/bigdl/dllib/examples/keras/imdb_cnn_lstm.py index 0b710f984e5..9cba23b1805 100644 --- a/python/dllib/src/bigdl/dllib/examples/keras/imdb_cnn_lstm.py +++ b/python/dllib/src/bigdl/dllib/examples/keras/imdb_cnn_lstm.py @@ -65,7 +65,7 @@ def build_keras_model(): parser = OptionParser() parser.add_option("-b", "--batchSize", type=int, dest="batchSize", default="32") parser.add_option("-m", "--max_epoch", type=int, dest="max_epoch", default="2") - parser.add_option("--optimizerVersion", dest="optimizerVersion", default="optimizerV1") + parser.add_option("--optimizerVersion", dest="optimizerVersion", default="optimizerV2") (options, args) = parser.parse_args(sys.argv) keras_model = build_keras_model() diff --git a/python/dllib/src/bigdl/dllib/examples/keras/mnist_cnn.py b/python/dllib/src/bigdl/dllib/examples/keras/mnist_cnn.py index 309116c588f..3bf787c6fe4 100644 --- a/python/dllib/src/bigdl/dllib/examples/keras/mnist_cnn.py +++ b/python/dllib/src/bigdl/dllib/examples/keras/mnist_cnn.py @@ -78,7 +78,7 @@ def build_keras_model(): parser.add_option("-b", "--batchSize", type=int, dest="batchSize", default="128") parser.add_option("-m", "--max_epoch", type=int, dest="max_epoch", default="12") parser.add_option("-d", "--dataPath", dest="dataPath", default="/tmp/mnist") - parser.add_option("--optimizerVersion", dest="optimizerVersion", default="optimizerV1") + parser.add_option("--optimizerVersion", dest="optimizerVersion", default="optimizerV2") (options, args) = parser.parse_args(sys.argv) keras_model = build_keras_model() diff --git a/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py b/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py index cb4a940d5ed..6a1e0fec765 100644 --- a/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py +++ b/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py @@ -60,7 +60,7 @@ def build_model(class_num): parser.add_option("-t", "--endTriggerType", dest="endTriggerType", default="epoch") parser.add_option("-n", "--endTriggerNum", type=int, dest="endTriggerNum", default="20") parser.add_option("-d", "--dataPath", dest="dataPath", default="/tmp/mnist") - parser.add_option("--optimizerVersion", dest="optimizerVersion", default="optimizerV1") + parser.add_option("--optimizerVersion", dest="optimizerVersion", default="optimizerV2") (options, args) = parser.parse_args(sys.argv) diff --git a/python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py b/python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py index c4d0c5d6e12..dfa731f6680 100644 --- a/python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py +++ b/python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py @@ -164,7 +164,7 @@ def train(sc, data_path, parser.add_option("--model", dest="model_type", default="cnn") parser.add_option("-p", "--p", dest="p", default="0.0") parser.add_option("-d", "--data_path", dest="data_path", default="/tmp/news20/") - parser.add_option("--optimizerVersion", dest="optimizerVersion", default="optimizerV1") + parser.add_option("--optimizerVersion", dest="optimizerVersion", default="optimizerV2") (options, args) = parser.parse_args(sys.argv) if options.action == "train": From 2ebcba76409f917ea262cd7b35ec1e2bdf143980 Mon Sep 17 00:00:00 2001 From: Le-Zheng <30695225+Le-Zheng@users.noreply.github.com> Date: Tue, 22 Jun 2021 02:49:35 +0100 Subject: [PATCH 790/823] make default DistriOptimizer as V2 (#3129) * make default DistriOptimizer as V2 * update --- python/dllib/src/bigdl/dllib/examples/keras/README.md | 2 +- python/dllib/src/bigdl/dllib/examples/keras/imdb_cnn_lstm.py | 2 +- python/dllib/src/bigdl/dllib/examples/keras/mnist_cnn.py | 2 +- python/dllib/src/bigdl/dllib/models/lenet/lenet5.py | 2 +- .../src/bigdl/dllib/models/textclassifier/textclassifier.py | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/examples/keras/README.md b/python/dllib/src/bigdl/dllib/examples/keras/README.md index fa4fb56a716..af4ece12bc1 100644 --- a/python/dllib/src/bigdl/dllib/examples/keras/README.md +++ b/python/dllib/src/bigdl/dllib/examples/keras/README.md @@ -50,4 +50,4 @@ ${SPARK_HOME}/bin/spark-submit \ ``` * ```--batchSize``` an option that can be used to set batch size. * ```--max_epoch``` an option that can be used to set how many epochs for which the model is to be trained. -* ```--optimizerVersion``` an option that can be used to set DistriOptimizer version, the default value is "optimizerV1". \ No newline at end of file +* ```--optimizerVersion``` an option that can be used to set DistriOptimizer version, the default value is "optimizerV2". \ No newline at end of file diff --git a/python/dllib/src/bigdl/dllib/examples/keras/imdb_cnn_lstm.py b/python/dllib/src/bigdl/dllib/examples/keras/imdb_cnn_lstm.py index 0b710f984e5..9cba23b1805 100644 --- a/python/dllib/src/bigdl/dllib/examples/keras/imdb_cnn_lstm.py +++ b/python/dllib/src/bigdl/dllib/examples/keras/imdb_cnn_lstm.py @@ -65,7 +65,7 @@ def build_keras_model(): parser = OptionParser() parser.add_option("-b", "--batchSize", type=int, dest="batchSize", default="32") parser.add_option("-m", "--max_epoch", type=int, dest="max_epoch", default="2") - parser.add_option("--optimizerVersion", dest="optimizerVersion", default="optimizerV1") + parser.add_option("--optimizerVersion", dest="optimizerVersion", default="optimizerV2") (options, args) = parser.parse_args(sys.argv) keras_model = build_keras_model() diff --git a/python/dllib/src/bigdl/dllib/examples/keras/mnist_cnn.py b/python/dllib/src/bigdl/dllib/examples/keras/mnist_cnn.py index 309116c588f..3bf787c6fe4 100644 --- a/python/dllib/src/bigdl/dllib/examples/keras/mnist_cnn.py +++ b/python/dllib/src/bigdl/dllib/examples/keras/mnist_cnn.py @@ -78,7 +78,7 @@ def build_keras_model(): parser.add_option("-b", "--batchSize", type=int, dest="batchSize", default="128") parser.add_option("-m", "--max_epoch", type=int, dest="max_epoch", default="12") parser.add_option("-d", "--dataPath", dest="dataPath", default="/tmp/mnist") - parser.add_option("--optimizerVersion", dest="optimizerVersion", default="optimizerV1") + parser.add_option("--optimizerVersion", dest="optimizerVersion", default="optimizerV2") (options, args) = parser.parse_args(sys.argv) keras_model = build_keras_model() diff --git a/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py b/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py index cb4a940d5ed..6a1e0fec765 100644 --- a/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py +++ b/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py @@ -60,7 +60,7 @@ def build_model(class_num): parser.add_option("-t", "--endTriggerType", dest="endTriggerType", default="epoch") parser.add_option("-n", "--endTriggerNum", type=int, dest="endTriggerNum", default="20") parser.add_option("-d", "--dataPath", dest="dataPath", default="/tmp/mnist") - parser.add_option("--optimizerVersion", dest="optimizerVersion", default="optimizerV1") + parser.add_option("--optimizerVersion", dest="optimizerVersion", default="optimizerV2") (options, args) = parser.parse_args(sys.argv) diff --git a/python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py b/python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py index c4d0c5d6e12..dfa731f6680 100644 --- a/python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py +++ b/python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py @@ -164,7 +164,7 @@ def train(sc, data_path, parser.add_option("--model", dest="model_type", default="cnn") parser.add_option("-p", "--p", dest="p", default="0.0") parser.add_option("-d", "--data_path", dest="data_path", default="/tmp/news20/") - parser.add_option("--optimizerVersion", dest="optimizerVersion", default="optimizerV1") + parser.add_option("--optimizerVersion", dest="optimizerVersion", default="optimizerV2") (options, args) = parser.parse_args(sys.argv) if options.action == "train": From a2df23cd51c387932623ace296d7da4759bbe2a4 Mon Sep 17 00:00:00 2001 From: Le-Zheng <30695225+Le-Zheng@users.noreply.github.com> Date: Fri, 2 Jul 2021 16:23:52 +0800 Subject: [PATCH 791/823] move dlframe SharedParamsApater to AZ and roll back to OptimizerV1 (#3137) --- python/dllib/src/bigdl/dllib/examples/keras/README.md | 2 +- python/dllib/src/bigdl/dllib/examples/keras/imdb_cnn_lstm.py | 4 ++-- python/dllib/src/bigdl/dllib/examples/keras/mnist_cnn.py | 2 +- python/dllib/src/bigdl/dllib/models/lenet/lenet5.py | 2 +- .../src/bigdl/dllib/models/textclassifier/textclassifier.py | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/examples/keras/README.md b/python/dllib/src/bigdl/dllib/examples/keras/README.md index af4ece12bc1..a27886f3409 100644 --- a/python/dllib/src/bigdl/dllib/examples/keras/README.md +++ b/python/dllib/src/bigdl/dllib/examples/keras/README.md @@ -50,4 +50,4 @@ ${SPARK_HOME}/bin/spark-submit \ ``` * ```--batchSize``` an option that can be used to set batch size. * ```--max_epoch``` an option that can be used to set how many epochs for which the model is to be trained. -* ```--optimizerVersion``` an option that can be used to set DistriOptimizer version, the default value is "optimizerV2". \ No newline at end of file +* ```--optimizerVersion``` an option that can be used to set DistriOptimizer version, the default value is "optimizerV1". diff --git a/python/dllib/src/bigdl/dllib/examples/keras/imdb_cnn_lstm.py b/python/dllib/src/bigdl/dllib/examples/keras/imdb_cnn_lstm.py index 9cba23b1805..9a7b91f21f6 100644 --- a/python/dllib/src/bigdl/dllib/examples/keras/imdb_cnn_lstm.py +++ b/python/dllib/src/bigdl/dllib/examples/keras/imdb_cnn_lstm.py @@ -65,7 +65,7 @@ def build_keras_model(): parser = OptionParser() parser.add_option("-b", "--batchSize", type=int, dest="batchSize", default="32") parser.add_option("-m", "--max_epoch", type=int, dest="max_epoch", default="2") - parser.add_option("--optimizerVersion", dest="optimizerVersion", default="optimizerV2") + parser.add_option("--optimizerVersion", dest="optimizerVersion", default="optimizerV1") (options, args) = parser.parse_args(sys.argv) keras_model = build_keras_model() @@ -103,4 +103,4 @@ def build_keras_model(): trigger=EveryEpoch(), val_method=[Top1Accuracy()] ) - optimizer.optimize() \ No newline at end of file + optimizer.optimize() diff --git a/python/dllib/src/bigdl/dllib/examples/keras/mnist_cnn.py b/python/dllib/src/bigdl/dllib/examples/keras/mnist_cnn.py index 3bf787c6fe4..309116c588f 100644 --- a/python/dllib/src/bigdl/dllib/examples/keras/mnist_cnn.py +++ b/python/dllib/src/bigdl/dllib/examples/keras/mnist_cnn.py @@ -78,7 +78,7 @@ def build_keras_model(): parser.add_option("-b", "--batchSize", type=int, dest="batchSize", default="128") parser.add_option("-m", "--max_epoch", type=int, dest="max_epoch", default="12") parser.add_option("-d", "--dataPath", dest="dataPath", default="/tmp/mnist") - parser.add_option("--optimizerVersion", dest="optimizerVersion", default="optimizerV2") + parser.add_option("--optimizerVersion", dest="optimizerVersion", default="optimizerV1") (options, args) = parser.parse_args(sys.argv) keras_model = build_keras_model() diff --git a/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py b/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py index 6a1e0fec765..cb4a940d5ed 100644 --- a/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py +++ b/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py @@ -60,7 +60,7 @@ def build_model(class_num): parser.add_option("-t", "--endTriggerType", dest="endTriggerType", default="epoch") parser.add_option("-n", "--endTriggerNum", type=int, dest="endTriggerNum", default="20") parser.add_option("-d", "--dataPath", dest="dataPath", default="/tmp/mnist") - parser.add_option("--optimizerVersion", dest="optimizerVersion", default="optimizerV2") + parser.add_option("--optimizerVersion", dest="optimizerVersion", default="optimizerV1") (options, args) = parser.parse_args(sys.argv) diff --git a/python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py b/python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py index dfa731f6680..c4d0c5d6e12 100644 --- a/python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py +++ b/python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py @@ -164,7 +164,7 @@ def train(sc, data_path, parser.add_option("--model", dest="model_type", default="cnn") parser.add_option("-p", "--p", dest="p", default="0.0") parser.add_option("-d", "--data_path", dest="data_path", default="/tmp/news20/") - parser.add_option("--optimizerVersion", dest="optimizerVersion", default="optimizerV2") + parser.add_option("--optimizerVersion", dest="optimizerVersion", default="optimizerV1") (options, args) = parser.parse_args(sys.argv) if options.action == "train": From 2da5ba275ef4063e078e6b98a2e4ce9424eb2862 Mon Sep 17 00:00:00 2001 From: Le-Zheng <30695225+Le-Zheng@users.noreply.github.com> Date: Fri, 2 Jul 2021 16:23:52 +0800 Subject: [PATCH 792/823] move dlframe SharedParamsApater to AZ and roll back to OptimizerV1 (#3137) --- python/dllib/src/bigdl/dllib/examples/keras/README.md | 2 +- python/dllib/src/bigdl/dllib/examples/keras/imdb_cnn_lstm.py | 4 ++-- python/dllib/src/bigdl/dllib/examples/keras/mnist_cnn.py | 2 +- python/dllib/src/bigdl/dllib/models/lenet/lenet5.py | 2 +- .../src/bigdl/dllib/models/textclassifier/textclassifier.py | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/examples/keras/README.md b/python/dllib/src/bigdl/dllib/examples/keras/README.md index af4ece12bc1..a27886f3409 100644 --- a/python/dllib/src/bigdl/dllib/examples/keras/README.md +++ b/python/dllib/src/bigdl/dllib/examples/keras/README.md @@ -50,4 +50,4 @@ ${SPARK_HOME}/bin/spark-submit \ ``` * ```--batchSize``` an option that can be used to set batch size. * ```--max_epoch``` an option that can be used to set how many epochs for which the model is to be trained. -* ```--optimizerVersion``` an option that can be used to set DistriOptimizer version, the default value is "optimizerV2". \ No newline at end of file +* ```--optimizerVersion``` an option that can be used to set DistriOptimizer version, the default value is "optimizerV1". diff --git a/python/dllib/src/bigdl/dllib/examples/keras/imdb_cnn_lstm.py b/python/dllib/src/bigdl/dllib/examples/keras/imdb_cnn_lstm.py index 9cba23b1805..9a7b91f21f6 100644 --- a/python/dllib/src/bigdl/dllib/examples/keras/imdb_cnn_lstm.py +++ b/python/dllib/src/bigdl/dllib/examples/keras/imdb_cnn_lstm.py @@ -65,7 +65,7 @@ def build_keras_model(): parser = OptionParser() parser.add_option("-b", "--batchSize", type=int, dest="batchSize", default="32") parser.add_option("-m", "--max_epoch", type=int, dest="max_epoch", default="2") - parser.add_option("--optimizerVersion", dest="optimizerVersion", default="optimizerV2") + parser.add_option("--optimizerVersion", dest="optimizerVersion", default="optimizerV1") (options, args) = parser.parse_args(sys.argv) keras_model = build_keras_model() @@ -103,4 +103,4 @@ def build_keras_model(): trigger=EveryEpoch(), val_method=[Top1Accuracy()] ) - optimizer.optimize() \ No newline at end of file + optimizer.optimize() diff --git a/python/dllib/src/bigdl/dllib/examples/keras/mnist_cnn.py b/python/dllib/src/bigdl/dllib/examples/keras/mnist_cnn.py index 3bf787c6fe4..309116c588f 100644 --- a/python/dllib/src/bigdl/dllib/examples/keras/mnist_cnn.py +++ b/python/dllib/src/bigdl/dllib/examples/keras/mnist_cnn.py @@ -78,7 +78,7 @@ def build_keras_model(): parser.add_option("-b", "--batchSize", type=int, dest="batchSize", default="128") parser.add_option("-m", "--max_epoch", type=int, dest="max_epoch", default="12") parser.add_option("-d", "--dataPath", dest="dataPath", default="/tmp/mnist") - parser.add_option("--optimizerVersion", dest="optimizerVersion", default="optimizerV2") + parser.add_option("--optimizerVersion", dest="optimizerVersion", default="optimizerV1") (options, args) = parser.parse_args(sys.argv) keras_model = build_keras_model() diff --git a/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py b/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py index 6a1e0fec765..cb4a940d5ed 100644 --- a/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py +++ b/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py @@ -60,7 +60,7 @@ def build_model(class_num): parser.add_option("-t", "--endTriggerType", dest="endTriggerType", default="epoch") parser.add_option("-n", "--endTriggerNum", type=int, dest="endTriggerNum", default="20") parser.add_option("-d", "--dataPath", dest="dataPath", default="/tmp/mnist") - parser.add_option("--optimizerVersion", dest="optimizerVersion", default="optimizerV2") + parser.add_option("--optimizerVersion", dest="optimizerVersion", default="optimizerV1") (options, args) = parser.parse_args(sys.argv) diff --git a/python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py b/python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py index dfa731f6680..c4d0c5d6e12 100644 --- a/python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py +++ b/python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py @@ -164,7 +164,7 @@ def train(sc, data_path, parser.add_option("--model", dest="model_type", default="cnn") parser.add_option("-p", "--p", dest="p", default="0.0") parser.add_option("-d", "--data_path", dest="data_path", default="/tmp/news20/") - parser.add_option("--optimizerVersion", dest="optimizerVersion", default="optimizerV2") + parser.add_option("--optimizerVersion", dest="optimizerVersion", default="optimizerV1") (options, args) = parser.parse_args(sys.argv) if options.action == "train": From 52d4520c9fb67de265776bb74a7c5cf3427617c3 Mon Sep 17 00:00:00 2001 From: Le-Zheng <30695225+Le-Zheng@users.noreply.github.com> Date: Mon, 12 Jul 2021 15:49:55 +0800 Subject: [PATCH 793/823] flip0.14 (#3142) * flip0.14 * update --- .../dllib/src/bigdl/dllib/models/textclassifier/README.md | 8 ++++---- version.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/models/textclassifier/README.md b/python/dllib/src/bigdl/dllib/models/textclassifier/README.md index 5150e737b75..3d7e518370c 100644 --- a/python/dllib/src/bigdl/dllib/models/textclassifier/README.md +++ b/python/dllib/src/bigdl/dllib/models/textclassifier/README.md @@ -28,8 +28,8 @@ $ [/tmp/news20]$ tree . -L 1 - The example code would automatically download the data during the first run. ### Run via pip install -- [Install from pip](https://bigdl-project.github.io/0.13.0-SNAPSHOT/#PythonUserGuide/install-from-pip/) -- Optional: check [Run after pip install](https://bigdl-project.github.io/0.13.0-SNAPSHOT/#PythonUserGuide/run-from-pip/) +- [Install from pip](https://bigdl-project.github.io/0.14.0-SNAPSHOT/#PythonUserGuide/install-from-pip/) +- Optional: check [Run after pip install](https://bigdl-project.github.io/0.14.0-SNAPSHOT/#PythonUserGuide/run-from-pip/) - Run the following command locally ``` python ${BigDL_HOME}/pyspark/bigdl/models/textclassifier/textclassifier.py --max_epoch 3 --model cnn @@ -37,8 +37,8 @@ python ${BigDL_HOME}/pyspark/bigdl/models/textclassifier/textclassifier.py --max ``` ### Run via spark-submit -- [Install without pip](https://bigdl-project.github.io/0.13.0-SNAPSHOT/#PythonUserGuide/install-without-pip/) -- Optional: check [Run without pip](https://bigdl-project.github.io/0.13.0-SNAPSHOT/#PythonUserGuide/run-without-pip/) +- [Install without pip](https://bigdl-project.github.io/0.14.0-SNAPSHOT/#PythonUserGuide/install-without-pip/) +- Optional: check [Run without pip](https://bigdl-project.github.io/0.14.0-SNAPSHOT/#PythonUserGuide/run-without-pip/) - Run the following command ```{r, engine='sh'} PYTHONHASHSEED=0 diff --git a/version.py b/version.py index 4f81c266b13..302e0a86caa 100644 --- a/version.py +++ b/version.py @@ -14,4 +14,4 @@ # limitations under the License. # -__version__ = "0.13.0.dev0" +__version__ = "0.14.0.dev0" From f90583bf49181a9e2ca9b30760c4a4f070626924 Mon Sep 17 00:00:00 2001 From: Le-Zheng <30695225+Le-Zheng@users.noreply.github.com> Date: Mon, 12 Jul 2021 15:49:55 +0800 Subject: [PATCH 794/823] flip0.14 (#3142) * flip0.14 * update --- .../dllib/src/bigdl/dllib/models/textclassifier/README.md | 8 ++++---- version.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/models/textclassifier/README.md b/python/dllib/src/bigdl/dllib/models/textclassifier/README.md index 5150e737b75..3d7e518370c 100644 --- a/python/dllib/src/bigdl/dllib/models/textclassifier/README.md +++ b/python/dllib/src/bigdl/dllib/models/textclassifier/README.md @@ -28,8 +28,8 @@ $ [/tmp/news20]$ tree . -L 1 - The example code would automatically download the data during the first run. ### Run via pip install -- [Install from pip](https://bigdl-project.github.io/0.13.0-SNAPSHOT/#PythonUserGuide/install-from-pip/) -- Optional: check [Run after pip install](https://bigdl-project.github.io/0.13.0-SNAPSHOT/#PythonUserGuide/run-from-pip/) +- [Install from pip](https://bigdl-project.github.io/0.14.0-SNAPSHOT/#PythonUserGuide/install-from-pip/) +- Optional: check [Run after pip install](https://bigdl-project.github.io/0.14.0-SNAPSHOT/#PythonUserGuide/run-from-pip/) - Run the following command locally ``` python ${BigDL_HOME}/pyspark/bigdl/models/textclassifier/textclassifier.py --max_epoch 3 --model cnn @@ -37,8 +37,8 @@ python ${BigDL_HOME}/pyspark/bigdl/models/textclassifier/textclassifier.py --max ``` ### Run via spark-submit -- [Install without pip](https://bigdl-project.github.io/0.13.0-SNAPSHOT/#PythonUserGuide/install-without-pip/) -- Optional: check [Run without pip](https://bigdl-project.github.io/0.13.0-SNAPSHOT/#PythonUserGuide/run-without-pip/) +- [Install without pip](https://bigdl-project.github.io/0.14.0-SNAPSHOT/#PythonUserGuide/install-without-pip/) +- Optional: check [Run without pip](https://bigdl-project.github.io/0.14.0-SNAPSHOT/#PythonUserGuide/run-without-pip/) - Run the following command ```{r, engine='sh'} PYTHONHASHSEED=0 diff --git a/version.py b/version.py index 4f81c266b13..302e0a86caa 100644 --- a/version.py +++ b/version.py @@ -14,4 +14,4 @@ # limitations under the License. # -__version__ = "0.13.0.dev0" +__version__ = "0.14.0.dev0" From 42256a4f02fa2683e5fc7cab4cdba22ee5691098 Mon Sep 17 00:00:00 2001 From: dding3 Date: Fri, 13 Aug 2021 14:39:30 -0700 Subject: [PATCH 795/823] rename python package --- dlframes/__init__.py | 0 dlframes/dl_classifier.py | 150 --- dlframes/dl_image_reader.py | 42 - dlframes/dl_image_transformer.py | 52 - python/.idea/misc.xml | 4 + python/.idea/modules.xml | 8 + python/.idea/python.iml | 12 + python/.idea/workspace.xml | 899 ++++++++++++++++++ .../src/bigdl/dllib/bigdlkeras/backend.py | 4 +- .../bigdl/dllib/bigdlkeras/layers/layer.py | 132 +-- .../bigdl/dllib/bigdlkeras/layers/topology.py | 18 +- .../bigdl/dllib/bigdlkeras/optimization.py | 8 +- .../bigdl/dllib/contrib/onnx/onnx_loader.py | 4 +- .../bigdl/dllib/contrib/onnx/ops_converter.py | 8 +- .../imageframe/inception_validation.py | 9 +- .../examples/keras/imdb_bigdl_backend.py | 4 +- .../dllib/examples/keras/imdb_cnn_lstm.py | 8 +- .../bigdl/dllib/examples/keras/keras_utils.py | 2 +- .../bigdl/dllib/examples/keras/mnist_cnn.py | 14 +- .../src/bigdl/dllib/examples/lenet/lenet.py | 6 +- .../src/bigdl/dllib/examples/onnx/README.md | 2 +- .../dllib/examples/onnx/load_onnx_resnet.py | 2 +- .../bigdl/dllib/feature/dataset/dataset.py | 8 +- .../src/bigdl/dllib/feature/dataset/mnist.py | 4 +- .../bigdl/dllib/feature/dataset/movielens.py | 2 +- .../src/bigdl/dllib/feature/dataset/news20.py | 2 +- .../dllib/feature/dataset/transformer.py | 2 +- .../dllib/feature/transform/vision/image.py | 6 +- .../src/bigdl/dllib/keras/ToBigDLHelper.py | 4 +- .../dllib/src/bigdl/dllib/keras/converter.py | 8 +- .../bigdl/dllib/models/inception/README.md | 2 +- .../bigdl/dllib/models/inception/inception.py | 27 +- .../src/bigdl/dllib/models/lenet/lenet5.py | 12 +- .../src/bigdl/dllib/models/lenet/utils.py | 6 +- .../dllib/models/local_lenet/local_lenet.py | 10 +- .../dllib/models/ml_pipeline/dl_classifier.py | 4 +- .../src/bigdl/dllib/models/rnn/rnnexample.py | 14 +- .../models/textclassifier/textclassifier.py | 12 +- .../dllib/models/utils/model_broadcast.py | 2 +- python/dllib/src/bigdl/dllib/nn/criterion.py | 14 +- .../bigdl/dllib/nn/initialization_method.py | 2 +- .../src/bigdl/dllib/nn/keras/__init__.py | 2 +- python/dllib/src/bigdl/dllib/nn/layer.py | 46 +- python/dllib/src/bigdl/dllib/nn/onnx/layer.py | 4 +- .../dllib/src/bigdl/dllib/optim/optimizer.py | 30 +- python/dllib/src/bigdl/dllib/utils/common.py | 18 +- .../dllib/src/bigdl/dllib/utils/tf_utils.py | 6 +- python/dllib/src/python-zip.xml | 28 + python/dllib/test/__init__.py | 18 - python/release.sh | 83 ++ python/run_lenet.sh | 18 + scala/dllib/pom.xml | 26 +- version.py | 17 - 53 files changed, 1315 insertions(+), 510 deletions(-) delete mode 100644 dlframes/__init__.py delete mode 100644 dlframes/dl_classifier.py delete mode 100644 dlframes/dl_image_reader.py delete mode 100644 dlframes/dl_image_transformer.py create mode 100644 python/.idea/misc.xml create mode 100644 python/.idea/modules.xml create mode 100644 python/.idea/python.iml create mode 100644 python/.idea/workspace.xml create mode 100644 python/dllib/src/python-zip.xml delete mode 100644 python/dllib/test/__init__.py create mode 100755 python/release.sh create mode 100755 python/run_lenet.sh delete mode 100644 version.py diff --git a/dlframes/__init__.py b/dlframes/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/dlframes/dl_classifier.py b/dlframes/dl_classifier.py deleted file mode 100644 index 34cc298b641..00000000000 --- a/dlframes/dl_classifier.py +++ /dev/null @@ -1,150 +0,0 @@ -from pyspark.ml.pipeline import Estimator, Model -from pyspark.ml.param.shared import * -from bigdl.util.common import * - - -if sys.version >= '3': - long = int - unicode = str - - -class HasBatchSize(Params): - """ - Mixin for param batchSize: batch size. - """ - - # a placeholder to make it appear in the generated doc - batchSize = Param(Params._dummy(), "batchSize", "batchSize") - - def __init__(self): - super(HasBatchSize, self).__init__() - #: param for batch size. - self.batchSize = Param(self, "batchSize", "batchSize") - self._setDefault(batchSize=1) - - def setBatchSize(self, val): - """ - Sets the value of :py:attr:`batchSize`. - """ - self._paramMap[self.batchSize] = val - pythonBigDL_method_name = "setBatchSize" + self.__class__.__name__ - callBigDlFunc(self.bigdl_type, pythonBigDL_method_name, self.value, val) - return self - - def getBatchSize(self): - """ - Gets the value of batchSize or its default value. - """ - return self.getOrDefault(self.batchSize) - -class HasMaxEpoch(Params): - maxEpoch = Param(Params._dummy(), "maxEpoch", "number of max Epoch") - - def __init__(self): - super(HasMaxEpoch, self).__init__() - self.maxEpoch = Param(self, "maxEpoch", "maxEpoch") - self._setDefault(maxEpoch=100) - - def setMaxEpoch(self, val): - self._paramMap[self.maxEpoch] = val - pythonBigDL_method_name = "setMaxEpoch" + self.__class__.__name__ - callBigDlFunc(self.bigdl_type, pythonBigDL_method_name, self.value, val) - return self - - def getMaxEpoch(self): - """ - Gets the value of maxEpoch or its default value. - """ - return self.getOrDefault(self.maxEpoch) - -class HasFeatureSize(Params): - featureSize = Param(Params._dummy(), "featureSize", "size of the feature") - - def __init__(self): - super(HasFeatureSize, self).__init__() - self.featureSize = Param(self, "featureSize", "featureSize") - self._setDefault(featureSize=None) - - def setFeatureSize(self, val): - self._paramMap[self.featureSize] = val - pythonBigDL_mehtod_name = "setFeatureSize" + self.__class__.__name__ - callBigDlFunc(self.bigdl_type, pythonBigDL_mehtod_name, self.value, val) - return self - - def getFeatureSize(self): - return self.getOrDefault(self.featureSize) - -class HasLearningRate(Params): - learningRate = Param(Params._dummy(), "learningRate", "learning rate") - - def __init__(self): - super(HasLearningRate, self).__init__() - self.learningRate = Param(self, "learningRate", "learning rate") - self._setDefault(learningRate=100) - - def setLearningRate(self, val): - self._paramMap[self.learningRate] = val - pythonBigDL_method_name = "setLearningRate" + self.__class__.__name__ - callBigDlFunc(self.bigdl_type, pythonBigDL_method_name, self.value, val) - return self - - def getLearningRate(self): - """ - Gets the value of maxEpoch or its default value. - """ - return self.getOrDefault(self.learningRate) - -class DLEstimator(Estimator, HasFeaturesCol, HasLabelCol, HasPredictionCol, HasBatchSize, HasMaxEpoch, HasLearningRate, JavaValue): - - def __init__(self, model, criterion, feature_size, label_size, jvalue=None, bigdl_type="float"): - super(DLEstimator, self).__init__() - self.value = jvalue if jvalue else callBigDlFunc( - bigdl_type, self.jvm_class_constructor(), model, criterion, feature_size, label_size) - self.bigdl_type = bigdl_type - self.featureSize = feature_size - - def _fit(self, dataset): - #self._transfer_params_to_java() - jmodel = callBigDlFunc(self.bigdl_type, "fitEstimator", self.value, dataset) - model = DLModel.of(jmodel, self.featureSize, self.bigdl_type) - return model - - -class DLModel(Model, HasFeaturesCol, HasPredictionCol, HasBatchSize, HasFeatureSize, JavaValue): - def __init__(self, model, featureSize, jvalue=None, bigdl_type="float"): - super(DLModel, self).__init__() - self.value = jvalue if jvalue else callBigDlFunc( - bigdl_type, self.jvm_class_constructor(), model, featureSize) - self.bigdl_type = bigdl_type - self.setFeatureSize(featureSize) - - def _transform(self, dataset): - return callBigDlFunc(self.bigdl_type, "dlModelTransform", self.value, dataset) - - @classmethod - def of(self, jvalue, feature_size=None, bigdl_type="float"): - model = DLModel(model=None, featureSize=feature_size, jvalue=jvalue, bigdl_type=bigdl_type) - return model - - -class DLClassifier(DLEstimator): - def __init__(self, model, criterion, feature_size, bigdl_type="float"): - super(DLClassifier, self).__init__(model, criterion, feature_size, [1], None, bigdl_type) - - def _fit(self, dataset): - jmodel = callBigDlFunc(self.bigdl_type, "fitClassifier", self.value, dataset) - model = DLClassifierModel.of(jmodel, self.featureSize, self.bigdl_type) - return model - - -class DLClassifierModel(DLModel): - def __init__(self, model, featureSize, jvalue=None, bigdl_type="float"): - super(DLClassifierModel, self).__init__(model, featureSize, jvalue, bigdl_type) - - def _transform(self, dataset): - return callBigDlFunc(self.bigdl_type, "dlClassifierModelTransform", self.value, dataset) - - @classmethod - def of(self, jvalue, feature_size=None, bigdl_type="float"): - model = DLClassifierModel(model=None, featureSize=feature_size, jvalue=jvalue, bigdl_type=bigdl_type) - return model diff --git a/dlframes/dl_image_reader.py b/dlframes/dl_image_reader.py deleted file mode 100644 index 96c8c9bd71b..00000000000 --- a/dlframes/dl_image_reader.py +++ /dev/null @@ -1,42 +0,0 @@ -# -# Copyright 2016 The BigDL Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import sys -from bigdl.util.common import callBigDlFunc - -if sys.version >= '3': - long = int - unicode = str - -class DLImageReader: - """ - Primary DataFrame-based image loading interface, defining API to read images from files - to DataFrame. - """ - - @staticmethod - def readImages(path, sc=None, minParitions = 1, bigdl_type="float"): - """ - Read the directory of images into DataFrame from the local or remote source. - :param path Directory to the input data files, the path can be comma separated paths as the - list of inputs. Wildcards path are supported similarly to sc.binaryFiles(path). - :param min_partitions A suggestion value of the minimal splitting number for input data. - :return DataFrame with a single column "image"; Each record in the column represents one image - record: Row (uri, height, width, channels, CvType, bytes) - """ - df = callBigDlFunc(bigdl_type, "dlReadImage", path, sc, minParitions) - df._sc._jsc = sc._jsc - return df diff --git a/dlframes/dl_image_transformer.py b/dlframes/dl_image_transformer.py deleted file mode 100644 index e4fe70a0e7a..00000000000 --- a/dlframes/dl_image_transformer.py +++ /dev/null @@ -1,52 +0,0 @@ -# -# Copyright 2016 The BigDL Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import sys -from bigdl.util.common import JavaValue, callBigDlFunc -from pyspark.ml.param.shared import * -from pyspark.ml.wrapper import JavaTransformer - -if sys.version >= '3': - long = int - unicode = str - -class DLImageTransformer(JavaTransformer, HasInputCol, HasOutputCol, JavaValue): - """ - Provides DataFrame-based API for image pre-processing and feature transformation. - DLImageTransformer follows the Spark Transformer API pattern and can be used as one stage - in Spark ML pipeline. - - The input column can be either DLImageSchema.byteSchema or DLImageSchema.floatSchema. If - using DLImageReader, the default format is DLImageSchema.byteSchema - The output column is always DLImageSchema.floatSchema. - """ - - def __init__(self, transformer, jvalue=None, bigdl_type="float"): - super(DLImageTransformer, self).__init__() - self.value = jvalue if jvalue else callBigDlFunc( - bigdl_type, self.jvm_class_constructor(), transformer) - self._java_obj = self.value - self.bigdl_type = bigdl_type - - def transform(self, dataset): - """ - Apply the transformer to the images in "inputCol" and store the transformed result - into "outputCols" - """ - self._transfer_params_to_java() - return callBigDlFunc(self.bigdl_type, "dlImageTransform", self.value, dataset) - - diff --git a/python/.idea/misc.xml b/python/.idea/misc.xml new file mode 100644 index 00000000000..448b4d3663f --- /dev/null +++ b/python/.idea/misc.xml @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/python/.idea/modules.xml b/python/.idea/modules.xml new file mode 100644 index 00000000000..614b3c13ff5 --- /dev/null +++ b/python/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/python/.idea/python.iml b/python/.idea/python.iml new file mode 100644 index 00000000000..e98082abea8 --- /dev/null +++ b/python/.idea/python.iml @@ -0,0 +1,12 @@ + + + + + + + + + + \ No newline at end of file diff --git a/python/.idea/workspace.xml b/python/.idea/workspace.xml new file mode 100644 index 00000000000..4f6b228c869 --- /dev/null +++ b/python/.idea/workspace.xml @@ -0,0 +1,899 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + lenet.py + lenet.p + b + bigd + bigdl + bigdl.d + bigdl.da + bigdl.data + bigdl.datas + bigdl.dataset + bigdl.nn.layer + bigdl.nn.onnx.layer + bigdl.transform + bigdl.optim + bigdl.util.common + bigdl.keras.backend + bigdl.keras.backen + bigdl.keras.backe + bigdl.keras.back + bigdl.keras.bac + bigdl.keras.ba + bigdl.keras.b + bigdl.keras. + bigdl.keras + bigdl.examples + bigdl.contrib + import + bigdl.nn + bigdl.models + bigdl. + + + BIGDL_HOME + bigdl.dllib.feature.dataset + bigdl.dllib.nn.layer + bigdl.dllib.nn.onnx.layer + bigdl.dllib.feature.transform + bigdl.dllib.optim + bigdl.utils.common + bigdl.dllib.keras + bigdl.dllib.examples + bigdl.dllib.contrib + bigdl.dllib.nn + bigdl.dllib.models + BIGDL_PYTHON_DIR + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 1628804636938 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/python/dllib/src/bigdl/dllib/bigdlkeras/backend.py b/python/dllib/src/bigdl/dllib/bigdlkeras/backend.py index 4c08a792c7b..bcbb75b704b 100644 --- a/python/dllib/src/bigdl/dllib/bigdlkeras/backend.py +++ b/python/dllib/src/bigdl/dllib/bigdlkeras/backend.py @@ -14,8 +14,8 @@ # limitations under the License. # -from bigdl.keras.optimization import * -from bigdl.util.common import * +from bigdl.dllib.keras.optimization import * +from bigdl.utils.common import * class KerasModelWrapper: diff --git a/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py b/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py index b2d84897dfc..cc63b531d15 100644 --- a/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py +++ b/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py @@ -16,8 +16,8 @@ import sys -from bigdl.nn.layer import Layer, Node, SharedStaticUtils, Container -from bigdl.util.common import callBigDlFunc, JTensor, JavaValue +from bigdl.dllib.nn.layer import Layer, Node, SharedStaticUtils, Container +from bigdl.utils.common import callBigDlFunc, JTensor, JavaValue if sys.version >= '3': long = int @@ -26,7 +26,7 @@ class InferShape(JavaValue): """ - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, bigdl_type="float"): @@ -63,7 +63,7 @@ def get_output_shape(self): class KerasCreator(JavaValue): """ - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def jvm_class_constructor(self): @@ -74,7 +74,7 @@ def jvm_class_constructor(self): class KerasLayer(Layer, InferShape, KerasCreator): """ - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, jvalue, *args, **kwargs): @@ -102,7 +102,7 @@ class Input(Node, KerasCreator): >>> input = Input(name="input1", shape=(3, 5)) creating: createKerasInput - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, shape=None, name=None, bigdl_type="float"): @@ -122,7 +122,7 @@ class InputLayer(KerasLayer): >>> inputlayer = InputLayer(input_shape=(3, 5)) creating: createKerasInputLayer - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, input_shape=None, **kwargs): @@ -157,7 +157,7 @@ class Dense(KerasLayer): >>> dense = Dense(10, input_dim=8, name="dense1") creating: createKerasDense - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, output_dim, init="glorot_uniform", activation=None, @@ -200,7 +200,7 @@ class MaxoutDense(KerasLayer): >>> maxoutdense = MaxoutDense(6, input_shape=(10, )) creating: createKerasMaxoutDense - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, output_dim, nb_feature=4, W_regularizer=None, b_regularizer=None, @@ -238,7 +238,7 @@ class Embedding(KerasLayer): >>> embedding = Embedding(1000, 32, input_shape=(10, ), name="embedding1") creating: createKerasEmbedding - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, input_dim, output_dim, init="uniform", W_regularizer=None, @@ -277,7 +277,7 @@ class BatchNormalization(KerasLayer): >>> batchnormalization = BatchNormalization(input_shape=(3, 12, 12), name="bn1") creating: createKerasBatchNormalization - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, epsilon=0.001, mode=0, axis=1, momentum=0.99, beta_init="zero", gamma_init="one", @@ -354,7 +354,7 @@ class Merge(KerasLayer): >>> merge = Merge(layers=[l1, l2], mode='sum', name="merge1") creating: createKerasMerge - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, layers=None, mode="sum", concat_axis=-1, @@ -381,7 +381,7 @@ def merge(inputs, mode="sum", concat_axis=-1, name=None): Default is -1, meaning the last axis of the input. name: String to set the name of the merge. If not specified, its name will by default to be a generated string. - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ return Merge(mode=mode, concat_axis=concat_axis, name=name)(list(inputs)) @@ -403,7 +403,7 @@ class Dropout(KerasLayer): >>> dropout = Dropout(0.25, input_shape=(2, 3)) creating: createKerasDropout - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, p, input_shape=None, **kwargs): @@ -427,7 +427,7 @@ class Flatten(KerasLayer): >>> flatten = Flatten(input_shape=(3, 10, 2)) creating: createKerasFlatten - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, input_shape=None, **kwargs): @@ -454,7 +454,7 @@ class Reshape(KerasLayer): >>> reshape = Reshape((2, 10), input_shape=(5, 4)) creating: createKerasReshape - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, target_shape, input_shape=None, **kwargs): @@ -480,7 +480,7 @@ class Activation(KerasLayer): >>> activation = Activation("relu", input_shape=(3, 4)) creating: createKerasActivation - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, activation, input_shape=None, **kwargs): @@ -508,7 +508,7 @@ class RepeatVector(KerasLayer): >>> repeatvector = RepeatVector(5, input_shape=(3, )) creating: createKerasRepeatVector - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, n, input_dim=None, input_shape=None, **kwargs): @@ -536,7 +536,7 @@ class Permute(KerasLayer): >>> permute = Permute((2, 1, 3), input_shape=(3, 4, 5)) creating: createKerasPermute - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, dims, input_shape=None, **kwargs): @@ -570,7 +570,7 @@ class Highway(KerasLayer): >>> highway = Highway(activation='relu', input_shape=(8, )) creating: createKerasHighway - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, activation=None, W_regularizer=None, b_regularizer=None, @@ -615,7 +615,7 @@ class Convolution1D(KerasLayer): >>> conv1d = Convolution1D(12, 4, input_shape=(3, 16)) creating: createKerasConvolution1D - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, nb_filter, filter_length, init="glorot_uniform", @@ -669,7 +669,7 @@ class Convolution2D(KerasLayer): >>> conv2d = Convolution2D(32, 3, 3, input_shape=(3, 128, 128), name="convolution2d_1") creating: createKerasConvolution2D - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, nb_filter, nb_row, nb_col, @@ -727,7 +727,7 @@ class Convolution3D(KerasLayer): >>> conv3d = Convolution3D(32, 3, 4, 5, input_shape=(3, 64, 64, 64)) creating: createKerasConvolution3D - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, nb_filter, kernel_dim1, kernel_dim2, kernel_dim3, @@ -783,7 +783,7 @@ class AtrousConvolution1D(KerasLayer): >>> atrousconv1d = AtrousConvolution1D(8, 3, input_shape=(3, 12)) creating: createKerasAtrousConvolution1D - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, nb_filter, filter_length, init="glorot_uniform", activation=None, @@ -844,7 +844,7 @@ class AtrousConvolution2D(KerasLayer): >>> atrousconv2d = AtrousConvolution2D(12, 4, 3, input_shape=(3, 64, 64)) creating: createKerasAtrousConvolution2D - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, nb_filter, nb_row, nb_col, init="glorot_uniform", @@ -910,7 +910,7 @@ class Deconvolution2D(KerasLayer): >>> deconv2d = Deconvolution2D(3, 3, 3, output_shape=(None, 3, 14, 14), input_shape=(3, 12, 12)) creating: createKerasDeconvolution2D - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, nb_filter, nb_row, nb_col, output_shape, init="glorot_uniform", @@ -974,7 +974,7 @@ class SeparableConvolution2D(KerasLayer): >>> separableconv2d = SeparableConvolution2D(12, 3, 4, input_shape=(3, 32, 32)) creating: createKerasSeparableConvolution2D - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, nb_filter, nb_row, nb_col, init="glorot_uniform", @@ -1025,7 +1025,7 @@ class Cropping1D(KerasLayer): >>> cropping1d = Cropping1D(cropping=(1, 2), input_shape=(8, 8)) creating: createKerasCropping1D - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, cropping=(1, 1), input_shape=None, **kwargs): @@ -1052,7 +1052,7 @@ class Cropping2D(KerasLayer): >>> cropping2d = Cropping2D(cropping=((1, 2), (0, 1)), input_shape=(12, 12, 12)) creating: createKerasCropping2D - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, cropping=((0, 0), (0, 0)), dim_ordering="th", @@ -1083,7 +1083,7 @@ class Cropping3D(KerasLayer): >>> cropping3d = Cropping3D(cropping=((0, 2), (1, 1), (3, 1)), input_shape=(4, 12, 12, 16)) creating: createKerasCropping3D - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, cropping=((1, 1), (1, 1), (1, 1)), dim_ordering="th", @@ -1114,7 +1114,7 @@ class UpSampling1D(KerasLayer): >>> upsampling1d = UpSampling1D(length=3, input_shape=(3, 12)) creating: createKerasUpSampling1D - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, length=2, input_shape=None, **kwargs): @@ -1142,7 +1142,7 @@ class UpSampling2D(KerasLayer): >>> upsampling2d = UpSampling2D(size=(1, 3), input_shape=(3, 16, 16)) creating: createKerasUpSampling2D - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, size=(2, 2), dim_ordering="th", input_shape=None, **kwargs): @@ -1172,7 +1172,7 @@ class UpSampling3D(KerasLayer): >>> upsampling3d = UpSampling3D(size=(1, 2, 3), input_shape=(3, 16, 16, 16)) creating: createKerasUpSampling3D - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, size=(2, 2, 2), dim_ordering="th", input_shape=None, **kwargs): @@ -1202,7 +1202,7 @@ class ZeroPadding1D(KerasLayer): >>> zeropadding1d = ZeroPadding1D(padding=2, input_shape=(3, 6)) creating: createKerasZeroPadding1D - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, padding=1, input_shape=None, **kwargs): @@ -1234,7 +1234,7 @@ class ZeroPadding2D(KerasLayer): >>> zeropadding2d = ZeroPadding2D(padding=(2, 1), input_shape=(2, 8, 8)) creating: createKerasZeroPadding2D - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, padding=(1, 1), dim_ordering="th", input_shape=None, **kwargs): @@ -1265,7 +1265,7 @@ class ZeroPadding3D(KerasLayer): >>> zeropadding3d = ZeroPadding3D(padding=(2, 1, 2), input_shape=(2, 8, 8, 10)) creating: createKerasZeroPadding3D - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, padding=(1, 1, 1), dim_ordering="th", input_shape=None, **kwargs): @@ -1295,7 +1295,7 @@ class MaxPooling1D(KerasLayer): >>> maxpooling1d = MaxPooling1D(3, input_shape=(3, 24)) creating: createKerasMaxPooling1D - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, pool_length=2, stride=None, border_mode="valid", @@ -1330,7 +1330,7 @@ class MaxPooling2D(KerasLayer): >>> maxpooling2d = MaxPooling2D((2, 2), input_shape=(3, 32, 32), name="maxpooling2d_1") creating: createKerasMaxPooling2D - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, pool_size=(2, 2), strides=None, @@ -1367,7 +1367,7 @@ class MaxPooling3D(KerasLayer): >>> maxpooling3d = MaxPooling3D((2, 1, 3), input_shape=(3, 32, 32, 32)) creating: createKerasMaxPooling3D - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, pool_size=(2, 2, 2), strides=None, border_mode="valid", @@ -1401,7 +1401,7 @@ class AveragePooling1D(KerasLayer): >>> averagepooling1d = AveragePooling1D(input_shape=(3, 24)) creating: createKerasAveragePooling1D - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, pool_length=2, stride=None, border_mode="valid", @@ -1436,7 +1436,7 @@ class AveragePooling2D(KerasLayer): >>> averagepooling2d = AveragePooling2D((1, 2), input_shape=(2, 28, 32)) creating: createKerasAveragePooling2D - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, pool_size=(2, 2), strides=None, border_mode="valid", @@ -1472,7 +1472,7 @@ class AveragePooling3D(KerasLayer): >>> averagepooling3d = AveragePooling3D((1, 1, 2), input_shape=(3, 28, 32, 36)) creating: createKerasAveragePooling3D - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, pool_size=(2, 2, 2), strides=None, border_mode="valid", @@ -1502,7 +1502,7 @@ class GlobalMaxPooling1D(KerasLayer): >>> globalmaxpooling1d = GlobalMaxPooling1D(input_shape=(4, 8)) creating: createKerasGlobalMaxPooling1D - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, input_shape=None, **kwargs): @@ -1526,7 +1526,7 @@ class GlobalAveragePooling1D(KerasLayer): >>> globalaveragepooling1d = GlobalAveragePooling1D(input_shape=(12, 12)) creating: createKerasGlobalAveragePooling1D - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, input_shape=None, **kwargs): @@ -1551,7 +1551,7 @@ class GlobalMaxPooling2D(KerasLayer): >>> globalmaxpooling2d = GlobalMaxPooling2D(input_shape=(4, 32, 32)) creating: createKerasGlobalMaxPooling2D - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, dim_ordering="th", input_shape=None, **kwargs): @@ -1577,7 +1577,7 @@ class GlobalAveragePooling2D(KerasLayer): >>> globalaveragepooling2d = GlobalAveragePooling2D(input_shape=(4, 32, 32)) creating: createKerasGlobalAveragePooling2D - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, dim_ordering="th", input_shape=None, **kwargs): @@ -1605,7 +1605,7 @@ class GlobalMaxPooling3D(KerasLayer): >>> globalmaxpooling3d = GlobalMaxPooling3D(input_shape=(4, 32, 32, 32)) creating: createKerasGlobalMaxPooling3D - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, dim_ordering="th", input_shape=None, **kwargs): @@ -1633,7 +1633,7 @@ class GlobalAveragePooling3D(KerasLayer): >>> globalaveragepooling3d = GlobalAveragePooling3D(input_shape=(4, 16, 16, 20)) creating: createKerasGlobalAveragePooling3D - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, dim_ordering="th", input_shape=None, **kwargs): @@ -1668,7 +1668,7 @@ class SimpleRNN(KerasLayer): >>> simplernn = SimpleRNN(16, input_shape=(3, 32)) creating: createKerasSimpleRNN - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, output_dim, activation="tanh", return_sequences=False, @@ -1712,7 +1712,7 @@ class LSTM(KerasLayer): >>> lstm = LSTM(32, input_shape=(8, 16), name="lstm1") creating: createKerasLSTM - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, output_dim, activation="tanh", inner_activation="hard_sigmoid", @@ -1757,7 +1757,7 @@ class GRU(KerasLayer): >>> gru = GRU(24, input_shape=(32, 32)) creating: createKerasGRU - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, output_dim, activation="tanh", inner_activation="hard_sigmoid", @@ -1811,7 +1811,7 @@ class ConvLSTM2D(KerasLayer): >>> convlstm2d = ConvLSTM2D(24, 3, 3, input_shape=(4, 32, 32, 32)) creating: createKerasConvLSTM2D - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, nb_filter, nb_row, nb_col, activation="tanh", @@ -1868,7 +1868,7 @@ class LocallyConnected1D(KerasLayer): >>> locallyconnected1d = LocallyConnected1D(6, 3, input_shape=(8, 12)) creating: createKerasLocallyConnected1D - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, nb_filter, filter_length, activation=None, border_mode="valid", @@ -1918,7 +1918,7 @@ class LocallyConnected2D(KerasLayer): >>> locallyconnected2d = LocallyConnected2D(12, 3, 4, input_shape=(3, 128, 128)) creating: createKerasLocallyConnected2D - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, nb_filter, nb_row, nb_col, activation=None, @@ -1961,7 +1961,7 @@ class SpatialDropout1D(KerasLayer): >>> spatialdropout1d = SpatialDropout1D(0.4, input_shape=(10, 12)) creating: createKerasSpatialDropout1D - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, p=0.5, input_shape=None, **kwargs): @@ -1993,7 +1993,7 @@ class SpatialDropout2D(KerasLayer): >>> spatialdropout2d = SpatialDropout2D(0.25, input_shape=(5, 12, 12)) creating: createKerasSpatialDropout2D - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, p=0.5, dim_ordering="th", input_shape=None, **kwargs): @@ -2026,7 +2026,7 @@ class SpatialDropout3D(KerasLayer): >>> spatialdropout3d = SpatialDropout3D(0.6, input_shape=(4, 12, 12, 16)) creating: createKerasSpatialDropout3D - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, p=0.5, dim_ordering="th", input_shape=None, **kwargs): @@ -2054,7 +2054,7 @@ class GaussianDropout(KerasLayer): >>> gaussiandropout = GaussianDropout(0.45, input_shape=(4, 8)) creating: createKerasGaussianDropout - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, p, input_shape=None, **kwargs): @@ -2082,7 +2082,7 @@ class GaussianNoise(KerasLayer): >>> gaussiannoise = GaussianNoise(0.45, input_shape=(3, 4, 5), name="gaussiannoise1") creating: createKerasGaussianNoise - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, sigma, input_shape=None, **kwargs): @@ -2110,7 +2110,7 @@ class Masking(KerasLayer): >>> masking = Masking(0.3, input_shape=(6, 8)) creating: createKerasMasking - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, mask_value=0.0, input_shape=None, **kwargs): @@ -2151,7 +2151,7 @@ class SReLU(KerasLayer): >>> srelu = SReLU(input_shape=(4, 5)) creating: createKerasSReLU - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, t_left_init="zero", a_left_init="glorot_uniform", @@ -2185,7 +2185,7 @@ class ELU(KerasLayer): >>> elu = ELU(1.2, input_shape=(4, 5)) creating: createKerasELU - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, alpha=1.0, input_shape=None, **kwargs): @@ -2213,7 +2213,7 @@ class LeakyReLU(KerasLayer): >>> leakyrelu = LeakyReLU(0.02, input_shape=(4, 5)) creating: createKerasLeakyReLU - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, alpha=0.01, input_shape=None, **kwargs): @@ -2241,7 +2241,7 @@ class ThresholdedReLU(KerasLayer): >>> thresholdedrelu = ThresholdedReLU(input_shape=(10, 12)) creating: createKerasThresholdedReLU - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, theta=1.0, input_shape=None, **kwargs): @@ -2270,7 +2270,7 @@ class TimeDistributed(KerasLayer): creating: createKerasDense creating: createKerasTimeDistributed - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, layer, input_shape=None, **kwargs): @@ -2302,7 +2302,7 @@ class Bidirectional(KerasLayer): creating: createKerasLSTM creating: createKerasBidirectional - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, layer, merge_mode="concat", input_shape=None, **kwargs): diff --git a/python/dllib/src/bigdl/dllib/bigdlkeras/layers/topology.py b/python/dllib/src/bigdl/dllib/bigdlkeras/layers/topology.py index a891635c4c7..51010298f39 100644 --- a/python/dllib/src/bigdl/dllib/bigdlkeras/layers/topology.py +++ b/python/dllib/src/bigdl/dllib/bigdlkeras/layers/topology.py @@ -14,18 +14,18 @@ # limitations under the License. # -from bigdl.nn.keras.layer import KerasLayer -from bigdl.optim.optimizer import * -from bigdl.nn.criterion import * +from bigdl.dllib.keras.layer import KerasLayer +from bigdl.dllib.optim.optimizer import * +from bigdl.dllib.nn.criterion import * import multiprocessing import warnings -from bigdl.nn.layer import SharedStaticUtils, Container +from bigdl.dllib.nn.layer import SharedStaticUtils, Container class KerasModel(KerasLayer, Container, SharedStaticUtils): """ - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __convert_optim_method(self, optimizer): @@ -206,11 +206,11 @@ class Sequential(KerasModel): >>> sequential = Sequential(name="seq1") creating: createKerasSequential - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, jvalue=None, **kwargs): - warnings.warn("bigdl.nn.keras is deprecated in 0.11. " + warnings.warn("bigdl.dllib.keras is deprecated in 0.11. " "Recommend to use Analytics Zoo's Keras API.") super(Sequential, self).__init__(jvalue, **kwargs) @@ -239,11 +239,11 @@ class Model(KerasModel): output: An output node or a list of output nodes. name: String to specify the name of the graph model. Default is None. - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, input, output, jvalue=None, **kwargs): - warnings.warn("bigdl.nn.keras is deprecated in BigDL 0.11." + warnings.warn("bigdl.dllib.keras is deprecated in BigDL 0.11." "Recommend to use Analytics Zoo's Keras API.") super(Model, self).__init__(jvalue, to_list(input), diff --git a/python/dllib/src/bigdl/dllib/bigdlkeras/optimization.py b/python/dllib/src/bigdl/dllib/bigdlkeras/optimization.py index 6c108ea04d7..50aafef7e45 100644 --- a/python/dllib/src/bigdl/dllib/bigdlkeras/optimization.py +++ b/python/dllib/src/bigdl/dllib/bigdlkeras/optimization.py @@ -16,10 +16,10 @@ import keras.optimizers as koptimizers -import bigdl.nn.criterion as bcriterion -import bigdl.optim.optimizer as boptimizer -from bigdl.util.common import to_list -from bigdl.keras.converter import * +import bigdl.dllib.nn.criterion as bcriterion +import bigdl.dllib.optim.optimizer as boptimizer +from bigdl.utils.common import to_list +from bigdl.dllib.keras.converter import * from keras.objectives import * import six diff --git a/python/dllib/src/bigdl/dllib/contrib/onnx/onnx_loader.py b/python/dllib/src/bigdl/dllib/contrib/onnx/onnx_loader.py index 6e537c10c3e..b4b29d6e172 100644 --- a/python/dllib/src/bigdl/dllib/contrib/onnx/onnx_loader.py +++ b/python/dllib/src/bigdl/dllib/contrib/onnx/onnx_loader.py @@ -15,8 +15,8 @@ # import onnx -from bigdl.nn.onnx.layer import * -from bigdl.nn.layer import Identity, Model +from bigdl.dllib.nn.onnx.layer import * +from bigdl.dllib.nn.layer import Identity, Model from .ops_mapping import _convert_map as convert_map from .converter_utils import parse_node_attr, parse_tensor_data diff --git a/python/dllib/src/bigdl/dllib/contrib/onnx/ops_converter.py b/python/dllib/src/bigdl/dllib/contrib/onnx/ops_converter.py index 4934f64ce93..a2acce3b832 100644 --- a/python/dllib/src/bigdl/dllib/contrib/onnx/ops_converter.py +++ b/python/dllib/src/bigdl/dllib/contrib/onnx/ops_converter.py @@ -14,10 +14,10 @@ # limitations under the License. # -from bigdl.nn.layer import SpatialAveragePooling, SpatialBatchNormalization -from bigdl.nn.layer import SpatialConvolution, SpatialMaxPooling, JoinTable -from bigdl.nn.layer import ReLU, SoftMax, CAddTable, Unsqueeze -from bigdl.nn.onnx.layer import Constant, Gather, Gemm, Shape, Reshape +from bigdl.dllib.nn.layer import SpatialAveragePooling, SpatialBatchNormalization +from bigdl.dllib.nn.layer import SpatialConvolution, SpatialMaxPooling, JoinTable +from bigdl.dllib.nn.layer import ReLU, SoftMax, CAddTable, Unsqueeze +from bigdl.dllib.nn.onnx.layer import Constant, Gather, Gemm, Shape, Reshape from .converter_utils import * diff --git a/python/dllib/src/bigdl/dllib/examples/imageframe/inception_validation.py b/python/dllib/src/bigdl/dllib/examples/imageframe/inception_validation.py index bcafa980988..148dea16f2b 100644 --- a/python/dllib/src/bigdl/dllib/examples/imageframe/inception_validation.py +++ b/python/dllib/src/bigdl/dllib/examples/imageframe/inception_validation.py @@ -14,18 +14,19 @@ # limitations under the License. # -from bigdl.util.common import * +from bigdl.utils.common import * -from bigdl.transform.vision.image import * +from bigdl.dllib.feature.transform.vision.image import * -from bigdl.optim.optimizer import * +from bigdl.dllib.optim.optimizer import * from pyspark import SparkContext -from bigdl.nn.layer import * +from bigdl.dllib.nn.layer import * from optparse import OptionParser +import os import sys parser = OptionParser() diff --git a/python/dllib/src/bigdl/dllib/examples/keras/imdb_bigdl_backend.py b/python/dllib/src/bigdl/dllib/examples/keras/imdb_bigdl_backend.py index ee80c97a04c..86c8ce06dbf 100644 --- a/python/dllib/src/bigdl/dllib/examples/keras/imdb_bigdl_backend.py +++ b/python/dllib/src/bigdl/dllib/examples/keras/imdb_bigdl_backend.py @@ -20,8 +20,8 @@ # after building, compiling the model and before the training. # model = with_bigdl_backend(model) -from bigdl.keras.backend import * -from bigdl.examples.keras.imdb_cnn_lstm import * +from bigdl.dllib.keras.backend import * +from bigdl.dllib.examples.keras.imdb_cnn_lstm import * X_train, y_train, X_test, y_test = load_imdb() diff --git a/python/dllib/src/bigdl/dllib/examples/keras/imdb_cnn_lstm.py b/python/dllib/src/bigdl/dllib/examples/keras/imdb_cnn_lstm.py index 9a7b91f21f6..ccc17fdfb53 100644 --- a/python/dllib/src/bigdl/dllib/examples/keras/imdb_cnn_lstm.py +++ b/python/dllib/src/bigdl/dllib/examples/keras/imdb_cnn_lstm.py @@ -72,10 +72,10 @@ def build_keras_model(): hdf5_path = "/tmp/imdb.h5" keras_model.save(hdf5_path) - from bigdl.util.common import * - from bigdl.nn.layer import * - from bigdl.optim.optimizer import * - from bigdl.nn.criterion import * + from bigdl.utils.common import * + from bigdl.dllib.nn.layer import * + from bigdl.dllib.optim.optimizer import * + from bigdl.dllib.nn.criterion import * # Load the HDF5 file with weights to a BigDL model bigdl_model = Model.load_keras(hdf5_path=hdf5_path) diff --git a/python/dllib/src/bigdl/dllib/examples/keras/keras_utils.py b/python/dllib/src/bigdl/dllib/examples/keras/keras_utils.py index 89d8a8b46cd..e95f394e7c5 100644 --- a/python/dllib/src/bigdl/dllib/examples/keras/keras_utils.py +++ b/python/dllib/src/bigdl/dllib/examples/keras/keras_utils.py @@ -14,7 +14,7 @@ # limitations under the License. # -from bigdl.util.common import * +from bigdl.utils.common import * def save_keras_definition(keras_model, path): diff --git a/python/dllib/src/bigdl/dllib/examples/keras/mnist_cnn.py b/python/dllib/src/bigdl/dllib/examples/keras/mnist_cnn.py index 309116c588f..05d9b41e0e4 100644 --- a/python/dllib/src/bigdl/dllib/examples/keras/mnist_cnn.py +++ b/python/dllib/src/bigdl/dllib/examples/keras/mnist_cnn.py @@ -22,7 +22,7 @@ # See README.md for how to run this example. from optparse import OptionParser -from bigdl.examples.keras.keras_utils import * +from bigdl.dllib.examples.keras.keras_utils import * import keras.backend if keras.backend.image_dim_ordering() == "th": @@ -36,8 +36,8 @@ def get_mnist(sc, data_type="train", location="/tmp/mnist"): Download or load MNIST dataset to/from the specified path. Normalize and transform input data into an RDD of Sample """ - from bigdl.dataset import mnist - from bigdl.dataset.transformer import normalizer + from bigdl.dllib.feature.dataset import mnist + from bigdl.dllib.feature.dataset.transformer import normalizer (images, labels) = mnist.read_data_sets(location, data_type) images = images.reshape((images.shape[0], ) + input_shape) images = sc.parallelize(images) @@ -85,10 +85,10 @@ def build_keras_model(): json_path = "/tmp/lenet.json" save_keras_definition(keras_model, json_path) - from bigdl.util.common import * - from bigdl.nn.layer import * - from bigdl.optim.optimizer import * - from bigdl.nn.criterion import * + from bigdl.utils.common import * + from bigdl.dllib.nn.layer import * + from bigdl.dllib.optim.optimizer import * + from bigdl.dllib.nn.criterion import * # Load the JSON file to a BigDL model bigdl_model = Model.load_keras(json_path=json_path) diff --git a/python/dllib/src/bigdl/dllib/examples/lenet/lenet.py b/python/dllib/src/bigdl/dllib/examples/lenet/lenet.py index 75669d084ef..f1fa6f6a44c 100644 --- a/python/dllib/src/bigdl/dllib/examples/lenet/lenet.py +++ b/python/dllib/src/bigdl/dllib/examples/lenet/lenet.py @@ -15,9 +15,9 @@ # from optparse import OptionParser -from bigdl.nn.keras.topology import Sequential -from bigdl.nn.keras.layer import * -from bigdl.dataset import mnist +from bigdl.dllib.keras.topology import Sequential +from bigdl.dllib.keras.layer import * +from bigdl.dllib.feature.dataset import mnist def build_model(class_num): diff --git a/python/dllib/src/bigdl/dllib/examples/onnx/README.md b/python/dllib/src/bigdl/dllib/examples/onnx/README.md index 1dc94bbef1c..5e34c73d1bf 100644 --- a/python/dllib/src/bigdl/dllib/examples/onnx/README.md +++ b/python/dllib/src/bigdl/dllib/examples/onnx/README.md @@ -18,7 +18,7 @@ * Import library dependencies ``` import numpy as np -from bigdl.contrib.onnx import load +from bigdl.dllib.contrib.onnx import load ``` * Set target ONNX ResNet-50 model path diff --git a/python/dllib/src/bigdl/dllib/examples/onnx/load_onnx_resnet.py b/python/dllib/src/bigdl/dllib/examples/onnx/load_onnx_resnet.py index d4c2bfc8371..e6428fe57e6 100644 --- a/python/dllib/src/bigdl/dllib/examples/onnx/load_onnx_resnet.py +++ b/python/dllib/src/bigdl/dllib/examples/onnx/load_onnx_resnet.py @@ -15,7 +15,7 @@ # import numpy as np -from bigdl.contrib.onnx import load +from bigdl.dllib.contrib.onnx import load def load_onnx_resnet(): diff --git a/python/dllib/src/bigdl/dllib/feature/dataset/dataset.py b/python/dllib/src/bigdl/dllib/feature/dataset/dataset.py index 1f142b9c2c7..e27247110eb 100644 --- a/python/dllib/src/bigdl/dllib/feature/dataset/dataset.py +++ b/python/dllib/src/bigdl/dllib/feature/dataset/dataset.py @@ -15,10 +15,10 @@ # import sys -from bigdl.util.common import JavaValue -from bigdl.util.common import callBigDlFunc -from bigdl.util.common import * -from bigdl.transform.vision.image import * +from bigdl.utils.common import JavaValue +from bigdl.utils.common import callBigDlFunc +from bigdl.utils.common import * +from bigdl.dllib.feature.transform.vision.image import * if sys.version >= '3': long = int diff --git a/python/dllib/src/bigdl/dllib/feature/dataset/mnist.py b/python/dllib/src/bigdl/dllib/feature/dataset/mnist.py index 881d8db4a95..319ca236e1d 100644 --- a/python/dllib/src/bigdl/dllib/feature/dataset/mnist.py +++ b/python/dllib/src/bigdl/dllib/feature/dataset/mnist.py @@ -20,8 +20,8 @@ import numpy -from bigdl.dataset import base -from bigdl.dataset.transformer import * +from bigdl.dllib.feature.dataset import base +from bigdl.dllib.feature.dataset.transformer import * SOURCE_URL = 'http://yann.lecun.com/exdb/mnist/' diff --git a/python/dllib/src/bigdl/dllib/feature/dataset/movielens.py b/python/dllib/src/bigdl/dllib/feature/dataset/movielens.py index 93bcefc831f..bf8da2c92bc 100644 --- a/python/dllib/src/bigdl/dllib/feature/dataset/movielens.py +++ b/python/dllib/src/bigdl/dllib/feature/dataset/movielens.py @@ -19,7 +19,7 @@ import zipfile import numpy as np -from bigdl.dataset import base +from bigdl.dllib.feature.dataset import base SOURCE_URL = 'http://files.grouplens.org/datasets/movielens/' def read_data_sets(data_dir): diff --git a/python/dllib/src/bigdl/dllib/feature/dataset/news20.py b/python/dllib/src/bigdl/dllib/feature/dataset/news20.py index 7bdd1c977d5..179ae921e7b 100644 --- a/python/dllib/src/bigdl/dllib/feature/dataset/news20.py +++ b/python/dllib/src/bigdl/dllib/feature/dataset/news20.py @@ -15,7 +15,7 @@ # import tarfile -from bigdl.dataset import base +from bigdl.dllib.feature.dataset import base import os import sys diff --git a/python/dllib/src/bigdl/dllib/feature/dataset/transformer.py b/python/dllib/src/bigdl/dllib/feature/dataset/transformer.py index 8af69d8c220..856aa02c5e6 100644 --- a/python/dllib/src/bigdl/dllib/feature/dataset/transformer.py +++ b/python/dllib/src/bigdl/dllib/feature/dataset/transformer.py @@ -15,7 +15,7 @@ # -from bigdl.util.common import Sample +from bigdl.utils.common import Sample def normalizer(data, mean, std): diff --git a/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py b/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py index f71792d2110..db166a15a01 100644 --- a/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py +++ b/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py @@ -15,9 +15,9 @@ # import sys -from bigdl.util.common import JavaValue -from bigdl.util.common import callBigDlFunc -from bigdl.util.common import * +from bigdl.utils.common import JavaValue +from bigdl.utils.common import callBigDlFunc +from bigdl.utils.common import * if sys.version >= '3': long = int diff --git a/python/dllib/src/bigdl/dllib/keras/ToBigDLHelper.py b/python/dllib/src/bigdl/dllib/keras/ToBigDLHelper.py index 6e0484eec4c..bfbe7d21dce 100644 --- a/python/dllib/src/bigdl/dllib/keras/ToBigDLHelper.py +++ b/python/dllib/src/bigdl/dllib/keras/ToBigDLHelper.py @@ -16,8 +16,8 @@ from math import ceil -import bigdl.nn.initialization_method as BInit -from bigdl.optim.optimizer import L1L2Regularizer as BRegularizer +import bigdl.dllib.nn.initialization_method as BInit +from bigdl.dllib.optim.optimizer import L1L2Regularizer as BRegularizer def to_bigdl_2d_ordering(order): diff --git a/python/dllib/src/bigdl/dllib/keras/converter.py b/python/dllib/src/bigdl/dllib/keras/converter.py index b6df571fab2..2cdf6f86fc9 100644 --- a/python/dllib/src/bigdl/dllib/keras/converter.py +++ b/python/dllib/src/bigdl/dllib/keras/converter.py @@ -15,14 +15,14 @@ # import numpy as np -import bigdl.nn.layer as BLayer -import bigdl.util.common as BCommon -from bigdl.util.common import get_activation_by_name +import bigdl.dllib.nn.layer as BLayer +import bigdl.utils.common as BCommon +from bigdl.utils.common import get_activation_by_name from keras.models import model_from_json from keras.models import Sequential, Model, Layer import keras import warnings -from bigdl.keras.ToBigDLHelper import * +from bigdl.dllib.keras.ToBigDLHelper import * def unsupport_exp(name): diff --git a/python/dllib/src/bigdl/dllib/models/inception/README.md b/python/dllib/src/bigdl/dllib/models/inception/README.md index 86cafb879ac..e995a492f1f 100644 --- a/python/dllib/src/bigdl/dllib/models/inception/README.md +++ b/python/dllib/src/bigdl/dllib/models/inception/README.md @@ -35,7 +35,7 @@ This command will transform the images into hadoop sequence files, which are more suitable for a distributed training. ```bash -java -cp bigdl_source_folder/spark/dl/target/bigdl-VERSION-jar-with-dependencies-and-spark.jar com.intel.analytics.bigdl.models.utils.ImageNetSeqFileGenerator -f imagenet_folder -o output_folder -p cores_number +java -cp bigdl_source_folder/spark/dl/target/bigdl-VERSION-jar-with-dependencies-and-spark.jar com.intel.analytics.bigdl.dllib.models.utils.ImageNetSeqFileGenerator -f imagenet_folder -o output_folder -p cores_number ``` It will generate the hadoop sequence files in the output folder. diff --git a/python/dllib/src/bigdl/dllib/models/inception/inception.py b/python/dllib/src/bigdl/dllib/models/inception/inception.py index 64f78c2a4ed..fa413cacef8 100644 --- a/python/dllib/src/bigdl/dllib/models/inception/inception.py +++ b/python/dllib/src/bigdl/dllib/models/inception/inception.py @@ -1,9 +1,26 @@ -from bigdl.nn.layer import * +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +from bigdl.dllib.nn.layer import * from optparse import OptionParser -from bigdl.nn.criterion import * -from bigdl.nn.initialization_method import * -from bigdl.optim.optimizer import * -from bigdl.transform.vision.image import * +from bigdl.dllib.nn.criterion import * +from bigdl.dllib.nn.initialization_method import * +from bigdl.dllib.optim.optimizer import * +from bigdl.dllib.feature.transform.vision.image import * from math import ceil diff --git a/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py b/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py index cb4a940d5ed..4125edbc46e 100644 --- a/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py +++ b/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py @@ -15,12 +15,12 @@ # from optparse import OptionParser -from bigdl.models.lenet.utils import * -from bigdl.dataset.transformer import * -from bigdl.nn.layer import * -from bigdl.nn.criterion import * -from bigdl.optim.optimizer import * -from bigdl.util.common import * +from bigdl.dllib.models.lenet.utils import * +from bigdl.dllib.feature.dataset.transformer import * +from bigdl.dllib.nn.layer import * +from bigdl.dllib.nn.criterion import * +from bigdl.dllib.optim.optimizer import * +from bigdl.utils.common import * def build_model(class_num): diff --git a/python/dllib/src/bigdl/dllib/models/lenet/utils.py b/python/dllib/src/bigdl/dllib/models/lenet/utils.py index e90444fda33..85a3329d8b1 100644 --- a/python/dllib/src/bigdl/dllib/models/lenet/utils.py +++ b/python/dllib/src/bigdl/dllib/models/lenet/utils.py @@ -14,9 +14,9 @@ # limitations under the License. # -from bigdl.dataset import mnist -from bigdl.dataset.transformer import * -from bigdl.optim.optimizer import * +from bigdl.dllib.feature.dataset import mnist +from bigdl.dllib.feature.dataset.transformer import * +from bigdl.dllib.optim.optimizer import * def get_mnist(sc, data_type="train", location="/tmp/mnist"): diff --git a/python/dllib/src/bigdl/dllib/models/local_lenet/local_lenet.py b/python/dllib/src/bigdl/dllib/models/local_lenet/local_lenet.py index 125f060ff32..fee0dec22c1 100644 --- a/python/dllib/src/bigdl/dllib/models/local_lenet/local_lenet.py +++ b/python/dllib/src/bigdl/dllib/models/local_lenet/local_lenet.py @@ -15,11 +15,11 @@ # from optparse import OptionParser -from bigdl.dataset import mnist -from bigdl.models.lenet.lenet5 import build_model -from bigdl.nn.criterion import * -from bigdl.optim.optimizer import * -from bigdl.util.common import * +from bigdl.dllib.feature.dataset import mnist +from bigdl.dllib.models.lenet.lenet5 import build_model +from bigdl.dllib.nn.criterion import * +from bigdl.dllib.optim.optimizer import * +from bigdl.utils.common import * def get_mnist(data_type="train", location="/tmp/mnist"): diff --git a/python/dllib/src/bigdl/dllib/models/ml_pipeline/dl_classifier.py b/python/dllib/src/bigdl/dllib/models/ml_pipeline/dl_classifier.py index dd0833fa6f6..89c6cea013a 100644 --- a/python/dllib/src/bigdl/dllib/models/ml_pipeline/dl_classifier.py +++ b/python/dllib/src/bigdl/dllib/models/ml_pipeline/dl_classifier.py @@ -1,7 +1,7 @@ from pyspark.ml.pipeline import Estimator, Model from pyspark.ml.param.shared import * -from bigdl.util.common import * - +from bigdl.utils.common import * +import sys if sys.version >= '3': long = int diff --git a/python/dllib/src/bigdl/dllib/models/rnn/rnnexample.py b/python/dllib/src/bigdl/dllib/models/rnn/rnnexample.py index 56061a29adf..ebeef9eb2b6 100644 --- a/python/dllib/src/bigdl/dllib/models/rnn/rnnexample.py +++ b/python/dllib/src/bigdl/dllib/models/rnn/rnnexample.py @@ -19,13 +19,13 @@ import re from optparse import OptionParser -from bigdl.dataset import base -from bigdl.dataset import sentence -from bigdl.nn.layer import * -from bigdl.nn.criterion import * -from bigdl.optim.optimizer import * -from bigdl.util.common import * -from bigdl.util.common import Sample +from bigdl.dllib.feature.dataset import base +from bigdl.dllib.feature.dataset import sentence +from bigdl.dllib.nn.layer import * +from bigdl.dllib.nn.criterion import * +from bigdl.dllib.optim.optimizer import * +from bigdl.utils.common import * +from bigdl.utils.common import Sample def download_data(dest_dir): TINYSHAKESPEARE_URL = 'https://raw.githubusercontent.com/karpathy/char-rnn/master/data/tinyshakespeare/input.txt' # noqa diff --git a/python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py b/python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py index c4d0c5d6e12..8dcca08c312 100644 --- a/python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py +++ b/python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py @@ -19,12 +19,12 @@ import re from optparse import OptionParser -from bigdl.dataset import news20 -from bigdl.nn.layer import * -from bigdl.nn.criterion import * -from bigdl.optim.optimizer import * -from bigdl.util.common import * -from bigdl.util.common import Sample +from bigdl.dllib.feature.dataset import news20 +from bigdl.dllib.nn.layer import * +from bigdl.dllib.nn.criterion import * +from bigdl.dllib.optim.optimizer import * +from bigdl.utils.common import * +from bigdl.utils.common import Sample import datetime as dt diff --git a/python/dllib/src/bigdl/dllib/models/utils/model_broadcast.py b/python/dllib/src/bigdl/dllib/models/utils/model_broadcast.py index 8e42ee311b8..7aa16b8be58 100644 --- a/python/dllib/src/bigdl/dllib/models/utils/model_broadcast.py +++ b/python/dllib/src/bigdl/dllib/models/utils/model_broadcast.py @@ -21,7 +21,7 @@ from pyspark.broadcast import Broadcast from pyspark.broadcast import _from_id -from bigdl.nn.layer import Model +from bigdl.dllib.nn.layer import Model def _from_id_and_type(bid, bigdl_type): result = _from_id(bid) diff --git a/python/dllib/src/bigdl/dllib/nn/criterion.py b/python/dllib/src/bigdl/dllib/nn/criterion.py index 038a475b8f8..2555a060a8b 100644 --- a/python/dllib/src/bigdl/dllib/nn/criterion.py +++ b/python/dllib/src/bigdl/dllib/nn/criterion.py @@ -17,10 +17,10 @@ import sys -from bigdl.util.common import JavaValue -from bigdl.util.common import callBigDlFunc -from bigdl.util.common import JTensor -from bigdl.nn.layer import Layer +from bigdl.utils.common import JavaValue +from bigdl.utils.common import callBigDlFunc +from bigdl.utils.common import JTensor +from bigdl.dllib.nn.layer import Layer import numpy as np if sys.version >= '3': @@ -978,9 +978,9 @@ def __init__(self, def _test(): import doctest from pyspark import SparkContext - from bigdl.nn import criterion - from bigdl.util.common import init_engine - from bigdl.util.common import create_spark_conf + from bigdl.dllib.nn import criterion + from bigdl.utils.common import init_engine + from bigdl.utils.common import create_spark_conf globs = criterion.__dict__.copy() sc = SparkContext(master="local[4]", appName="test criterion", conf=create_spark_conf()) diff --git a/python/dllib/src/bigdl/dllib/nn/initialization_method.py b/python/dllib/src/bigdl/dllib/nn/initialization_method.py index aaddee5597f..4f32027fbe2 100644 --- a/python/dllib/src/bigdl/dllib/nn/initialization_method.py +++ b/python/dllib/src/bigdl/dllib/nn/initialization_method.py @@ -16,7 +16,7 @@ import sys -from bigdl.util.common import JavaValue +from bigdl.utils.common import JavaValue if sys.version >= '3': long = int diff --git a/python/dllib/src/bigdl/dllib/nn/keras/__init__.py b/python/dllib/src/bigdl/dllib/nn/keras/__init__.py index 086e4e1f057..9eb670a1780 100644 --- a/python/dllib/src/bigdl/dllib/nn/keras/__init__.py +++ b/python/dllib/src/bigdl/dllib/nn/keras/__init__.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # -from bigdl.util.common import * +from bigdl.utils.common import * init_engine() redire_spark_logs() diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 6d02d86540e..bfa012e690f 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -21,19 +21,19 @@ import numpy as np import six -from bigdl.util.common import JTensor -from bigdl.util.common import JavaValue -from bigdl.util.common import callBigDlFunc -from bigdl.util.common import callJavaFunc -from bigdl.util.common import get_spark_context -from bigdl.util.common import to_list -from bigdl.util.common import INTMAX, INTMIN, DOUBLEMAX -from bigdl.util.common import get_activation_by_name -from bigdl.optim.optimizer import L1Regularizer, L2Regularizer, L1L2Regularizer +from bigdl.utils.common import JTensor +from bigdl.utils.common import JavaValue +from bigdl.utils.common import callBigDlFunc +from bigdl.utils.common import callJavaFunc +from bigdl.utils.common import get_spark_context +from bigdl.utils.common import to_list +from bigdl.utils.common import INTMAX, INTMIN, DOUBLEMAX +from bigdl.utils.common import get_activation_by_name +from bigdl.dllib.optim.optimizer import L1Regularizer, L2Regularizer, L1L2Regularizer from py4j.java_gateway import JavaObject from pyspark.rdd import RDD -from bigdl.transform.vision.image import ImageFrame -from bigdl.dataset.dataset import DataSet +from bigdl.dllib.feature.transform.vision.image import ImageFrame +from bigdl.dllib.feature.dataset.dataset import DataSet if sys.version >= '3': long = int @@ -98,13 +98,13 @@ def get_py_name(jclass_name): jpackage_name = ".".join(jname.split(".")[:-1]) pclass_name = get_py_name(jname.split(".")[-1]) - if "com.intel.analytics.bigdl.nn.keras.Model" == jname or \ - "com.intel.analytics.bigdl.nn.keras.Sequential" == jname: - base_module = importlib.import_module('bigdl.nn.keras.topology') - elif "com.intel.analytics.bigdl.nn.keras" == jpackage_name: - base_module = importlib.import_module('bigdl.nn.keras.layer') + if "com.intel.analytics.bigdl.dllib.keras.Model" == jname or \ + "com.intel.analytics.bigdl.dllib.keras.Sequential" == jname: + base_module = importlib.import_module('bigdl.dllib.keras.topology') + elif "com.intel.analytics.bigdl.dllib.keras" == jpackage_name: + base_module = importlib.import_module('bigdl.dllib.keras.layer') else: - base_module = importlib.import_module('bigdl.nn.layer') + base_module = importlib.import_module('bigdl.dllib.nn.layer') realClassName = "Layer" # The top base class if pclass_name in dir(base_module): @@ -807,7 +807,7 @@ def load_keras(json_path=None, hdf5_path=None, by_name=False): except ImportError: raise Exception("No backend is found for Keras. " "Please install either tensorflow or theano.") - from bigdl.keras.converter import DefinitionLoader, WeightLoader + from bigdl.dllib.keras.converter import DefinitionLoader, WeightLoader if json_path and not hdf5_path: return DefinitionLoader.from_json_path(json_path) elif json_path and hdf5_path: @@ -864,7 +864,7 @@ def load_tensorflow(path, inputs, outputs, byte_order = "little_endian", @staticmethod def train(output, data, label, opt_method, criterion, batch_size, end_when, session=None, bigdl_type="float"): from bigdl.util.tf_utils import get_path - from bigdl.util.common import Sample + from bigdl.utils.common import Sample output_name = output.name.split(":")[0] path = get_path(output_name, session) sc = get_spark_context() @@ -3927,7 +3927,7 @@ class SReLU(Layer): creating: createSReLU >>> srelu = SReLU((2, 2), (1, 2)) creating: createSReLU - >>> from bigdl.nn.initialization_method import Xavier + >>> from bigdl.dllib.nn.initialization_method import Xavier >>> init = Xavier() creating: createXavier >>> srelu = srelu.set_init_method(tLeftInit=init, aLeftInit=init, tRightInit=init, aRightInit=init) @@ -5783,9 +5783,9 @@ def __init__(self, def _test(): import doctest from pyspark import SparkContext - from bigdl.nn import layer - from bigdl.util.common import init_engine - from bigdl.util.common import create_spark_conf + from bigdl.dllib.nn import layer + from bigdl.utils.common import init_engine + from bigdl.utils.common import create_spark_conf globs = layer.__dict__.copy() sc = SparkContext(master="local[4]", appName="test layer", conf=create_spark_conf()) diff --git a/python/dllib/src/bigdl/dllib/nn/onnx/layer.py b/python/dllib/src/bigdl/dllib/nn/onnx/layer.py index d99b1e8e86c..f2e255f5e31 100644 --- a/python/dllib/src/bigdl/dllib/nn/onnx/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/onnx/layer.py @@ -16,8 +16,8 @@ import sys import numpy as np -from bigdl.nn.layer import Layer -from bigdl.util.common import JTensor +from bigdl.dllib.nn.layer import Layer +from bigdl.utils.common import JTensor if sys.version >= '3': long = int diff --git a/python/dllib/src/bigdl/dllib/optim/optimizer.py b/python/dllib/src/bigdl/dllib/optim/optimizer.py index 793b7cda31e..2786272c993 100644 --- a/python/dllib/src/bigdl/dllib/optim/optimizer.py +++ b/python/dllib/src/bigdl/dllib/optim/optimizer.py @@ -23,15 +23,15 @@ from py4j.java_gateway import JavaObject from pyspark.rdd import RDD -from bigdl.util.common import DOUBLEMAX -from bigdl.util.common import JTensor -from bigdl.util.common import JavaValue -from bigdl.util.common import callBigDlFunc -from bigdl.util.common import callJavaFunc -from bigdl.util.common import get_node_and_core_number -from bigdl.util.common import init_engine -from bigdl.util.common import to_list -from bigdl.dataset.dataset import * +from bigdl.utils.common import DOUBLEMAX +from bigdl.utils.common import JTensor +from bigdl.utils.common import JavaValue +from bigdl.utils.common import callBigDlFunc +from bigdl.utils.common import callJavaFunc +from bigdl.utils.common import get_node_and_core_number +from bigdl.utils.common import init_engine +from bigdl.utils.common import to_list +from bigdl.dllib.feature.dataset.dataset import * import warnings if sys.version >= '3': @@ -100,7 +100,7 @@ class Loss(JavaValue): """ This evaluation method is calculate loss of output with respect to target - >>> from bigdl.nn.criterion import ClassNLLCriterion + >>> from bigdl.dllib.nn.criterion import ClassNLLCriterion >>> loss = Loss() creating: createClassNLLCriterion creating: createLoss @@ -110,7 +110,7 @@ class Loss(JavaValue): creating: createLoss """ def __init__(self, cri=None, bigdl_type="float"): - from bigdl.nn.criterion import ClassNLLCriterion + from bigdl.dllib.nn.criterion import ClassNLLCriterion if cri is None: cri = ClassNLLCriterion() JavaValue.__init__(self, None, bigdl_type, cri) @@ -789,7 +789,7 @@ def optimize(self): Do an optimization. """ jmodel = callJavaFunc(self.value.optimize) - from bigdl.nn.layer import Layer + from bigdl.dllib.nn.layer import Layer return Layer.of(jmodel) def set_train_summary(self, summary): @@ -1189,9 +1189,9 @@ def __init__(self, l2, bigdl_type="float"): def _test(): import doctest from pyspark import SparkContext - from bigdl.optim import optimizer - from bigdl.util.common import init_engine - from bigdl.util.common import create_spark_conf + from bigdl.dllib.optim import optimizer + from bigdl.utils.common import init_engine + from bigdl.utils.common import create_spark_conf globs = optimizer.__dict__.copy() sc = SparkContext(master="local[4]", appName="test optimizer", conf=create_spark_conf()) diff --git a/python/dllib/src/bigdl/dllib/utils/common.py b/python/dllib/src/bigdl/dllib/utils/common.py index f4f51834af3..684fd22ad60 100644 --- a/python/dllib/src/bigdl/dllib/utils/common.py +++ b/python/dllib/src/bigdl/dllib/utils/common.py @@ -154,7 +154,7 @@ class JTensor(object): A wrapper to easy our work when need to pass or return Tensor to/from Scala. >>> import numpy as np - >>> from bigdl.util.common import JTensor + >>> from bigdl.utils.common import JTensor >>> np.random.seed(123) >>> """ @@ -189,8 +189,8 @@ def from_ndarray(cls, a_ndarray, bigdl_type="float"): Convert a ndarray to a DenseTensor which would be used in Java side. >>> import numpy as np - >>> from bigdl.util.common import JTensor - >>> from bigdl.util.common import callBigDlFunc + >>> from bigdl.utils.common import JTensor + >>> from bigdl.utils.common import callBigDlFunc >>> np.random.seed(123) >>> data = np.random.uniform(0, 1, (2, 3)).astype("float32") >>> result = JTensor.from_ndarray(data) @@ -236,8 +236,8 @@ def sparse(cls, a_ndarray, i_ndarray, shape, bigdl_type="float"): :param shape shape as a DenseTensor. >>> import numpy as np - >>> from bigdl.util.common import JTensor - >>> from bigdl.util.common import callBigDlFunc + >>> from bigdl.utils.common import JTensor + >>> from bigdl.utils.common import callBigDlFunc >>> np.random.seed(123) >>> data = np.arange(1, 7).astype("float32") >>> indices = np.arange(1, 7) @@ -314,7 +314,7 @@ def from_ndarray(cls, features, labels, bigdl_type="float"): :param bigdl_type: "double" or "float" >>> import numpy as np - >>> from bigdl.util.common import callBigDlFunc + >>> from bigdl.utils.common import callBigDlFunc >>> from numpy.testing import assert_allclose >>> np.random.seed(123) >>> sample = Sample.from_ndarray(np.random.random((2,3)), np.random.random((2,3))) @@ -495,7 +495,7 @@ def to_sample_rdd(x, y, numSlices=None): :return: """ sc = get_spark_context() - from bigdl.util.common import Sample + from bigdl.utils.common import Sample x_rdd = sc.parallelize(x, numSlices) y_rdd = sc.parallelize(y, numSlices) return x_rdd.zip(y_rdd).map(lambda item: Sample.from_ndarray(item[0], item[1])) @@ -720,7 +720,7 @@ def is_distributed(path): def get_activation_by_name(activation_name, activation_id=None): """ Convert to a bigdl activation layer given the name of the activation as a string """ - import bigdl.nn.layer as BLayer + import bigdl.dllib.nn.layer as BLayer activation = None activation_name = activation_name.lower() if activation_name == "tanh": @@ -749,7 +749,7 @@ def get_activation_by_name(activation_name, activation_id=None): def _test(): import doctest from pyspark import SparkContext - from bigdl.nn import layer + from bigdl.dllib.nn import layer globs = layer.__dict__.copy() sc = SparkContext(master="local[2]", appName="test common utility") globs['sc'] = sc diff --git a/python/dllib/src/bigdl/dllib/utils/tf_utils.py b/python/dllib/src/bigdl/dllib/utils/tf_utils.py index c19a508d79e..58abc2695bd 100644 --- a/python/dllib/src/bigdl/dllib/utils/tf_utils.py +++ b/python/dllib/src/bigdl/dllib/utils/tf_utils.py @@ -26,9 +26,9 @@ from tensorflow.python.framework import graph_util from tensorflow.python.framework import importer from tensorflow.python.platform import gfile -from bigdl.nn.layer import Model -from bigdl.util.common import JTensor -from bigdl.util.common import callBigDlFunc +from bigdl.dllib.nn.layer import Model +from bigdl.utils.common import JTensor +from bigdl.utils.common import callBigDlFunc import os def get_path(output_name, sess=None): diff --git a/python/dllib/src/python-zip.xml b/python/dllib/src/python-zip.xml new file mode 100644 index 00000000000..1ece79d1ab0 --- /dev/null +++ b/python/dllib/src/python-zip.xml @@ -0,0 +1,28 @@ + + python-api + + zip + + false + + + + **/*.py + + + test/** + + /.. + ${project.build.directory}/../../../python + + + + + + + + + + diff --git a/python/dllib/test/__init__.py b/python/dllib/test/__init__.py deleted file mode 100644 index 63e9a94acc5..00000000000 --- a/python/dllib/test/__init__.py +++ /dev/null @@ -1,18 +0,0 @@ -# -# Copyright 2016 The BigDL Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from bigdl.util.engine import prepare_env -prepare_env() diff --git a/python/release.sh b/python/release.sh new file mode 100755 index 00000000000..306a23c41c0 --- /dev/null +++ b/python/release.sh @@ -0,0 +1,83 @@ +#!/usr/bin/env bash + +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -e +RUN_SCRIPT_DIR=$(cd $(dirname $0) ; pwd) +echo $RUN_SCRIPT_DIR +BIGDL_DIR="$(cd ${RUN_SCRIPT_DIR}/../../../../; pwd)" +echo $BIGDL_DIR +BIGDL_PYTHON_DIR="$(cd ${RUN_SCRIPT_DIR}/../../../../pyspark; pwd)" +echo $BIGDL_PYTHON_DIR + +if (( $# < 2)); then + echo "Bad parameters. Usage: release.sh mac spark_2.x" + exit -1 +fi + +platform=$1 +spark_profile=$2 +quick=$3 +input_version=$4 +bigdl_version=$(python -c "exec(open('$BIGDL_DIR/pyspark/bigdl/version.py').read()); print(__version__)") + +if [ "$input_version" != "$bigdl_version" ]; then + echo "Not the proposed version: $bigdl_version" + exit -1 +fi + +cd ${BIGDL_DIR} +if [ "$platform" == "mac" ]; then + echo "Building bigdl for mac system" + dist_profile="-P mac -P $spark_profile" + verbose_pname="macosx_10_11_x86_64" +elif [ "$platform" == "linux" ]; then + echo "Building bigdl for linux system" + dist_profile="-P $spark_profile" + verbose_pname="manylinux1_x86_64" +else + echo "unsupport platform" +fi + +bigdl_build_command="${BIGDL_DIR}/make-dist.sh ${dist_profile}" +if [ "$quick" == "true" ]; then + echo "Skip disting BigDL" +else + echo "Dist BigDL: $bigdl_build_command" + $bigdl_build_command +fi + +cd $BIGDL_PYTHON_DIR +sdist_command="python setup.py sdist" +echo "packing source code: ${sdist_command}" +$sdist_command + +if [ -d "${BIGDL_DIR}/pyspark/build" ]; then + rm -r ${BIGDL_DIR}/pyspark/build +fi + +if [ -d "${BIGDL_DIR}/pyspark/dist" ]; then + rm -r ${BIGDL_DIR}/pyspark/dist +fi +wheel_command="python setup.py bdist_wheel --plat-name ${verbose_pname}" +echo "Packing python distribution: $wheel_command" +${wheel_command} + +upload_command="twine upload dist/BigDL-${bigdl_version}-py2.py3-none-${verbose_pname}.whl" +echo "Please manually upload with this command: $upload_command" + +$upload_command diff --git a/python/run_lenet.sh b/python/run_lenet.sh new file mode 100755 index 00000000000..4b5764cba94 --- /dev/null +++ b/python/run_lenet.sh @@ -0,0 +1,18 @@ +SPARK_HOME=/home/ding/Downloads/spark-2.4.3-bin-hadoop2.7 +MASTER=local[2] +PYTHON_API_ZIP_PATH=/home/ding/proj/clone-ding-zoo/analytics-zoo/scala/dllib/target/bigdl-dllib-2.0.0-SNAPSHOT-python-api.zip +BigDL_JAR_PATH=/home/ding/proj/clone-ding-zoo/analytics-zoo/scala/dllib/bigdl-dllib-2.0.0-SNAPSHOT-jar-with-dependencies.jar +PYTHONPATH=${PYTHON_API_ZIP_PATH}:$PYTHONPATH +${SPARK_HOME}/bin/spark-submit \ + --master ${MASTER} \ + --driver-cores 2 \ + --driver-memory 2g \ + --total-executor-cores 2 \ + --executor-cores 2 \ + --executor-memory 4g \ + --py-files ${PYTHON_API_ZIP_PATH},/home/ding/proj/clone-ding-zoo/analytics-zoo/python/dllib/examples/lenet/lenet.py \ + --properties-file /home/ding/proj/clone-ding-zoo/analytics-zoo/scala/dllib/src/main/resources/spark-bigdl.conf \ + --jars ${BigDL_JAR_PATH} \ + --conf spark.driver.extraClassPath=${BigDL_JAR_PATH} \ + --conf spark.executor.extraClassPath=/home/ding/proj/clone-ding-zoo/analytics-zoo/scala/dllib/bigdl-dllib-2.0.0-SNAPSHOT-jar-with-dependencies.jar \ + /home/ding/proj/clone-ding-zoo/analytics-zoo/python/dllib/examples/lenet/lenet.py diff --git a/scala/dllib/pom.xml b/scala/dllib/pom.xml index 2c0c0ffb77f..c34eccf3e57 100644 --- a/scala/dllib/pom.xml +++ b/scala/dllib/pom.xml @@ -245,10 +245,10 @@ - - - - + + org.apache.maven.plugins + maven-assembly-plugin + @@ -261,8 +261,22 @@ - - + + python + false + + + ${project.basedir}/../../python/python-zip.xml + + + + package + + single + + + + org.scoverage diff --git a/version.py b/version.py deleted file mode 100644 index 302e0a86caa..00000000000 --- a/version.py +++ /dev/null @@ -1,17 +0,0 @@ -# -# Copyright 2016 The BigDL Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -__version__ = "0.14.0.dev0" From de01afc55db518c3cdd53554d4988eaaf0105d8b Mon Sep 17 00:00:00 2001 From: dding3 Date: Fri, 13 Aug 2021 14:39:30 -0700 Subject: [PATCH 796/823] rename python package --- dlframes/__init__.py | 0 dlframes/dl_classifier.py | 150 --- dlframes/dl_image_reader.py | 42 - dlframes/dl_image_transformer.py | 52 - python/.idea/misc.xml | 4 + python/.idea/modules.xml | 8 + python/.idea/python.iml | 12 + python/.idea/workspace.xml | 899 ++++++++++++++++++ .../src/bigdl/dllib/bigdlkeras/backend.py | 4 +- .../bigdl/dllib/bigdlkeras/layers/layer.py | 132 +-- .../bigdl/dllib/bigdlkeras/layers/topology.py | 18 +- .../bigdl/dllib/bigdlkeras/optimization.py | 8 +- .../bigdl/dllib/contrib/onnx/onnx_loader.py | 4 +- .../bigdl/dllib/contrib/onnx/ops_converter.py | 8 +- .../imageframe/inception_validation.py | 9 +- .../examples/keras/imdb_bigdl_backend.py | 4 +- .../dllib/examples/keras/imdb_cnn_lstm.py | 8 +- .../bigdl/dllib/examples/keras/keras_utils.py | 2 +- .../bigdl/dllib/examples/keras/mnist_cnn.py | 14 +- .../src/bigdl/dllib/examples/lenet/lenet.py | 6 +- .../src/bigdl/dllib/examples/onnx/README.md | 2 +- .../dllib/examples/onnx/load_onnx_resnet.py | 2 +- .../bigdl/dllib/feature/dataset/dataset.py | 8 +- .../src/bigdl/dllib/feature/dataset/mnist.py | 4 +- .../bigdl/dllib/feature/dataset/movielens.py | 2 +- .../src/bigdl/dllib/feature/dataset/news20.py | 2 +- .../dllib/feature/dataset/transformer.py | 2 +- .../dllib/feature/transform/vision/image.py | 6 +- .../src/bigdl/dllib/keras/ToBigDLHelper.py | 4 +- .../dllib/src/bigdl/dllib/keras/converter.py | 8 +- .../bigdl/dllib/models/inception/README.md | 2 +- .../bigdl/dllib/models/inception/inception.py | 27 +- .../src/bigdl/dllib/models/lenet/lenet5.py | 12 +- .../src/bigdl/dllib/models/lenet/utils.py | 6 +- .../dllib/models/local_lenet/local_lenet.py | 10 +- .../dllib/models/ml_pipeline/dl_classifier.py | 4 +- .../src/bigdl/dllib/models/rnn/rnnexample.py | 14 +- .../models/textclassifier/textclassifier.py | 12 +- .../dllib/models/utils/model_broadcast.py | 2 +- python/dllib/src/bigdl/dllib/nn/criterion.py | 14 +- .../bigdl/dllib/nn/initialization_method.py | 2 +- .../src/bigdl/dllib/nn/keras/__init__.py | 2 +- python/dllib/src/bigdl/dllib/nn/layer.py | 46 +- python/dllib/src/bigdl/dllib/nn/onnx/layer.py | 4 +- .../dllib/src/bigdl/dllib/optim/optimizer.py | 30 +- .../dllib/src/bigdl/dllib/utils/tf_utils.py | 6 +- python/dllib/src/bigdl/utils/common.py | 18 +- python/dllib/src/python-zip.xml | 28 + python/dllib/test/__init__.py | 18 - python/release.sh | 83 ++ python/run_lenet.sh | 18 + scala/dllib/pom.xml | 26 +- version.py | 17 - 53 files changed, 1315 insertions(+), 510 deletions(-) delete mode 100644 dlframes/__init__.py delete mode 100644 dlframes/dl_classifier.py delete mode 100644 dlframes/dl_image_reader.py delete mode 100644 dlframes/dl_image_transformer.py create mode 100644 python/.idea/misc.xml create mode 100644 python/.idea/modules.xml create mode 100644 python/.idea/python.iml create mode 100644 python/.idea/workspace.xml create mode 100644 python/dllib/src/python-zip.xml delete mode 100644 python/dllib/test/__init__.py create mode 100755 python/release.sh create mode 100755 python/run_lenet.sh delete mode 100644 version.py diff --git a/dlframes/__init__.py b/dlframes/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/dlframes/dl_classifier.py b/dlframes/dl_classifier.py deleted file mode 100644 index 34cc298b641..00000000000 --- a/dlframes/dl_classifier.py +++ /dev/null @@ -1,150 +0,0 @@ -from pyspark.ml.pipeline import Estimator, Model -from pyspark.ml.param.shared import * -from bigdl.util.common import * - - -if sys.version >= '3': - long = int - unicode = str - - -class HasBatchSize(Params): - """ - Mixin for param batchSize: batch size. - """ - - # a placeholder to make it appear in the generated doc - batchSize = Param(Params._dummy(), "batchSize", "batchSize") - - def __init__(self): - super(HasBatchSize, self).__init__() - #: param for batch size. - self.batchSize = Param(self, "batchSize", "batchSize") - self._setDefault(batchSize=1) - - def setBatchSize(self, val): - """ - Sets the value of :py:attr:`batchSize`. - """ - self._paramMap[self.batchSize] = val - pythonBigDL_method_name = "setBatchSize" + self.__class__.__name__ - callBigDlFunc(self.bigdl_type, pythonBigDL_method_name, self.value, val) - return self - - def getBatchSize(self): - """ - Gets the value of batchSize or its default value. - """ - return self.getOrDefault(self.batchSize) - -class HasMaxEpoch(Params): - maxEpoch = Param(Params._dummy(), "maxEpoch", "number of max Epoch") - - def __init__(self): - super(HasMaxEpoch, self).__init__() - self.maxEpoch = Param(self, "maxEpoch", "maxEpoch") - self._setDefault(maxEpoch=100) - - def setMaxEpoch(self, val): - self._paramMap[self.maxEpoch] = val - pythonBigDL_method_name = "setMaxEpoch" + self.__class__.__name__ - callBigDlFunc(self.bigdl_type, pythonBigDL_method_name, self.value, val) - return self - - def getMaxEpoch(self): - """ - Gets the value of maxEpoch or its default value. - """ - return self.getOrDefault(self.maxEpoch) - -class HasFeatureSize(Params): - featureSize = Param(Params._dummy(), "featureSize", "size of the feature") - - def __init__(self): - super(HasFeatureSize, self).__init__() - self.featureSize = Param(self, "featureSize", "featureSize") - self._setDefault(featureSize=None) - - def setFeatureSize(self, val): - self._paramMap[self.featureSize] = val - pythonBigDL_mehtod_name = "setFeatureSize" + self.__class__.__name__ - callBigDlFunc(self.bigdl_type, pythonBigDL_mehtod_name, self.value, val) - return self - - def getFeatureSize(self): - return self.getOrDefault(self.featureSize) - -class HasLearningRate(Params): - learningRate = Param(Params._dummy(), "learningRate", "learning rate") - - def __init__(self): - super(HasLearningRate, self).__init__() - self.learningRate = Param(self, "learningRate", "learning rate") - self._setDefault(learningRate=100) - - def setLearningRate(self, val): - self._paramMap[self.learningRate] = val - pythonBigDL_method_name = "setLearningRate" + self.__class__.__name__ - callBigDlFunc(self.bigdl_type, pythonBigDL_method_name, self.value, val) - return self - - def getLearningRate(self): - """ - Gets the value of maxEpoch or its default value. - """ - return self.getOrDefault(self.learningRate) - -class DLEstimator(Estimator, HasFeaturesCol, HasLabelCol, HasPredictionCol, HasBatchSize, HasMaxEpoch, HasLearningRate, JavaValue): - - def __init__(self, model, criterion, feature_size, label_size, jvalue=None, bigdl_type="float"): - super(DLEstimator, self).__init__() - self.value = jvalue if jvalue else callBigDlFunc( - bigdl_type, self.jvm_class_constructor(), model, criterion, feature_size, label_size) - self.bigdl_type = bigdl_type - self.featureSize = feature_size - - def _fit(self, dataset): - #self._transfer_params_to_java() - jmodel = callBigDlFunc(self.bigdl_type, "fitEstimator", self.value, dataset) - model = DLModel.of(jmodel, self.featureSize, self.bigdl_type) - return model - - -class DLModel(Model, HasFeaturesCol, HasPredictionCol, HasBatchSize, HasFeatureSize, JavaValue): - def __init__(self, model, featureSize, jvalue=None, bigdl_type="float"): - super(DLModel, self).__init__() - self.value = jvalue if jvalue else callBigDlFunc( - bigdl_type, self.jvm_class_constructor(), model, featureSize) - self.bigdl_type = bigdl_type - self.setFeatureSize(featureSize) - - def _transform(self, dataset): - return callBigDlFunc(self.bigdl_type, "dlModelTransform", self.value, dataset) - - @classmethod - def of(self, jvalue, feature_size=None, bigdl_type="float"): - model = DLModel(model=None, featureSize=feature_size, jvalue=jvalue, bigdl_type=bigdl_type) - return model - - -class DLClassifier(DLEstimator): - def __init__(self, model, criterion, feature_size, bigdl_type="float"): - super(DLClassifier, self).__init__(model, criterion, feature_size, [1], None, bigdl_type) - - def _fit(self, dataset): - jmodel = callBigDlFunc(self.bigdl_type, "fitClassifier", self.value, dataset) - model = DLClassifierModel.of(jmodel, self.featureSize, self.bigdl_type) - return model - - -class DLClassifierModel(DLModel): - def __init__(self, model, featureSize, jvalue=None, bigdl_type="float"): - super(DLClassifierModel, self).__init__(model, featureSize, jvalue, bigdl_type) - - def _transform(self, dataset): - return callBigDlFunc(self.bigdl_type, "dlClassifierModelTransform", self.value, dataset) - - @classmethod - def of(self, jvalue, feature_size=None, bigdl_type="float"): - model = DLClassifierModel(model=None, featureSize=feature_size, jvalue=jvalue, bigdl_type=bigdl_type) - return model diff --git a/dlframes/dl_image_reader.py b/dlframes/dl_image_reader.py deleted file mode 100644 index 96c8c9bd71b..00000000000 --- a/dlframes/dl_image_reader.py +++ /dev/null @@ -1,42 +0,0 @@ -# -# Copyright 2016 The BigDL Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import sys -from bigdl.util.common import callBigDlFunc - -if sys.version >= '3': - long = int - unicode = str - -class DLImageReader: - """ - Primary DataFrame-based image loading interface, defining API to read images from files - to DataFrame. - """ - - @staticmethod - def readImages(path, sc=None, minParitions = 1, bigdl_type="float"): - """ - Read the directory of images into DataFrame from the local or remote source. - :param path Directory to the input data files, the path can be comma separated paths as the - list of inputs. Wildcards path are supported similarly to sc.binaryFiles(path). - :param min_partitions A suggestion value of the minimal splitting number for input data. - :return DataFrame with a single column "image"; Each record in the column represents one image - record: Row (uri, height, width, channels, CvType, bytes) - """ - df = callBigDlFunc(bigdl_type, "dlReadImage", path, sc, minParitions) - df._sc._jsc = sc._jsc - return df diff --git a/dlframes/dl_image_transformer.py b/dlframes/dl_image_transformer.py deleted file mode 100644 index e4fe70a0e7a..00000000000 --- a/dlframes/dl_image_transformer.py +++ /dev/null @@ -1,52 +0,0 @@ -# -# Copyright 2016 The BigDL Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import sys -from bigdl.util.common import JavaValue, callBigDlFunc -from pyspark.ml.param.shared import * -from pyspark.ml.wrapper import JavaTransformer - -if sys.version >= '3': - long = int - unicode = str - -class DLImageTransformer(JavaTransformer, HasInputCol, HasOutputCol, JavaValue): - """ - Provides DataFrame-based API for image pre-processing and feature transformation. - DLImageTransformer follows the Spark Transformer API pattern and can be used as one stage - in Spark ML pipeline. - - The input column can be either DLImageSchema.byteSchema or DLImageSchema.floatSchema. If - using DLImageReader, the default format is DLImageSchema.byteSchema - The output column is always DLImageSchema.floatSchema. - """ - - def __init__(self, transformer, jvalue=None, bigdl_type="float"): - super(DLImageTransformer, self).__init__() - self.value = jvalue if jvalue else callBigDlFunc( - bigdl_type, self.jvm_class_constructor(), transformer) - self._java_obj = self.value - self.bigdl_type = bigdl_type - - def transform(self, dataset): - """ - Apply the transformer to the images in "inputCol" and store the transformed result - into "outputCols" - """ - self._transfer_params_to_java() - return callBigDlFunc(self.bigdl_type, "dlImageTransform", self.value, dataset) - - diff --git a/python/.idea/misc.xml b/python/.idea/misc.xml new file mode 100644 index 00000000000..448b4d3663f --- /dev/null +++ b/python/.idea/misc.xml @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/python/.idea/modules.xml b/python/.idea/modules.xml new file mode 100644 index 00000000000..614b3c13ff5 --- /dev/null +++ b/python/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/python/.idea/python.iml b/python/.idea/python.iml new file mode 100644 index 00000000000..e98082abea8 --- /dev/null +++ b/python/.idea/python.iml @@ -0,0 +1,12 @@ + + + + + + + + + + \ No newline at end of file diff --git a/python/.idea/workspace.xml b/python/.idea/workspace.xml new file mode 100644 index 00000000000..4f6b228c869 --- /dev/null +++ b/python/.idea/workspace.xml @@ -0,0 +1,899 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + lenet.py + lenet.p + b + bigd + bigdl + bigdl.d + bigdl.da + bigdl.data + bigdl.datas + bigdl.dataset + bigdl.nn.layer + bigdl.nn.onnx.layer + bigdl.transform + bigdl.optim + bigdl.util.common + bigdl.keras.backend + bigdl.keras.backen + bigdl.keras.backe + bigdl.keras.back + bigdl.keras.bac + bigdl.keras.ba + bigdl.keras.b + bigdl.keras. + bigdl.keras + bigdl.examples + bigdl.contrib + import + bigdl.nn + bigdl.models + bigdl. + + + BIGDL_HOME + bigdl.dllib.feature.dataset + bigdl.dllib.nn.layer + bigdl.dllib.nn.onnx.layer + bigdl.dllib.feature.transform + bigdl.dllib.optim + bigdl.utils.common + bigdl.dllib.keras + bigdl.dllib.examples + bigdl.dllib.contrib + bigdl.dllib.nn + bigdl.dllib.models + BIGDL_PYTHON_DIR + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 1628804636938 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/python/dllib/src/bigdl/dllib/bigdlkeras/backend.py b/python/dllib/src/bigdl/dllib/bigdlkeras/backend.py index 4c08a792c7b..bcbb75b704b 100644 --- a/python/dllib/src/bigdl/dllib/bigdlkeras/backend.py +++ b/python/dllib/src/bigdl/dllib/bigdlkeras/backend.py @@ -14,8 +14,8 @@ # limitations under the License. # -from bigdl.keras.optimization import * -from bigdl.util.common import * +from bigdl.dllib.keras.optimization import * +from bigdl.utils.common import * class KerasModelWrapper: diff --git a/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py b/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py index b2d84897dfc..cc63b531d15 100644 --- a/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py +++ b/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py @@ -16,8 +16,8 @@ import sys -from bigdl.nn.layer import Layer, Node, SharedStaticUtils, Container -from bigdl.util.common import callBigDlFunc, JTensor, JavaValue +from bigdl.dllib.nn.layer import Layer, Node, SharedStaticUtils, Container +from bigdl.utils.common import callBigDlFunc, JTensor, JavaValue if sys.version >= '3': long = int @@ -26,7 +26,7 @@ class InferShape(JavaValue): """ - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, bigdl_type="float"): @@ -63,7 +63,7 @@ def get_output_shape(self): class KerasCreator(JavaValue): """ - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def jvm_class_constructor(self): @@ -74,7 +74,7 @@ def jvm_class_constructor(self): class KerasLayer(Layer, InferShape, KerasCreator): """ - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, jvalue, *args, **kwargs): @@ -102,7 +102,7 @@ class Input(Node, KerasCreator): >>> input = Input(name="input1", shape=(3, 5)) creating: createKerasInput - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, shape=None, name=None, bigdl_type="float"): @@ -122,7 +122,7 @@ class InputLayer(KerasLayer): >>> inputlayer = InputLayer(input_shape=(3, 5)) creating: createKerasInputLayer - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, input_shape=None, **kwargs): @@ -157,7 +157,7 @@ class Dense(KerasLayer): >>> dense = Dense(10, input_dim=8, name="dense1") creating: createKerasDense - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, output_dim, init="glorot_uniform", activation=None, @@ -200,7 +200,7 @@ class MaxoutDense(KerasLayer): >>> maxoutdense = MaxoutDense(6, input_shape=(10, )) creating: createKerasMaxoutDense - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, output_dim, nb_feature=4, W_regularizer=None, b_regularizer=None, @@ -238,7 +238,7 @@ class Embedding(KerasLayer): >>> embedding = Embedding(1000, 32, input_shape=(10, ), name="embedding1") creating: createKerasEmbedding - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, input_dim, output_dim, init="uniform", W_regularizer=None, @@ -277,7 +277,7 @@ class BatchNormalization(KerasLayer): >>> batchnormalization = BatchNormalization(input_shape=(3, 12, 12), name="bn1") creating: createKerasBatchNormalization - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, epsilon=0.001, mode=0, axis=1, momentum=0.99, beta_init="zero", gamma_init="one", @@ -354,7 +354,7 @@ class Merge(KerasLayer): >>> merge = Merge(layers=[l1, l2], mode='sum', name="merge1") creating: createKerasMerge - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, layers=None, mode="sum", concat_axis=-1, @@ -381,7 +381,7 @@ def merge(inputs, mode="sum", concat_axis=-1, name=None): Default is -1, meaning the last axis of the input. name: String to set the name of the merge. If not specified, its name will by default to be a generated string. - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ return Merge(mode=mode, concat_axis=concat_axis, name=name)(list(inputs)) @@ -403,7 +403,7 @@ class Dropout(KerasLayer): >>> dropout = Dropout(0.25, input_shape=(2, 3)) creating: createKerasDropout - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, p, input_shape=None, **kwargs): @@ -427,7 +427,7 @@ class Flatten(KerasLayer): >>> flatten = Flatten(input_shape=(3, 10, 2)) creating: createKerasFlatten - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, input_shape=None, **kwargs): @@ -454,7 +454,7 @@ class Reshape(KerasLayer): >>> reshape = Reshape((2, 10), input_shape=(5, 4)) creating: createKerasReshape - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, target_shape, input_shape=None, **kwargs): @@ -480,7 +480,7 @@ class Activation(KerasLayer): >>> activation = Activation("relu", input_shape=(3, 4)) creating: createKerasActivation - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, activation, input_shape=None, **kwargs): @@ -508,7 +508,7 @@ class RepeatVector(KerasLayer): >>> repeatvector = RepeatVector(5, input_shape=(3, )) creating: createKerasRepeatVector - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, n, input_dim=None, input_shape=None, **kwargs): @@ -536,7 +536,7 @@ class Permute(KerasLayer): >>> permute = Permute((2, 1, 3), input_shape=(3, 4, 5)) creating: createKerasPermute - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, dims, input_shape=None, **kwargs): @@ -570,7 +570,7 @@ class Highway(KerasLayer): >>> highway = Highway(activation='relu', input_shape=(8, )) creating: createKerasHighway - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, activation=None, W_regularizer=None, b_regularizer=None, @@ -615,7 +615,7 @@ class Convolution1D(KerasLayer): >>> conv1d = Convolution1D(12, 4, input_shape=(3, 16)) creating: createKerasConvolution1D - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, nb_filter, filter_length, init="glorot_uniform", @@ -669,7 +669,7 @@ class Convolution2D(KerasLayer): >>> conv2d = Convolution2D(32, 3, 3, input_shape=(3, 128, 128), name="convolution2d_1") creating: createKerasConvolution2D - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, nb_filter, nb_row, nb_col, @@ -727,7 +727,7 @@ class Convolution3D(KerasLayer): >>> conv3d = Convolution3D(32, 3, 4, 5, input_shape=(3, 64, 64, 64)) creating: createKerasConvolution3D - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, nb_filter, kernel_dim1, kernel_dim2, kernel_dim3, @@ -783,7 +783,7 @@ class AtrousConvolution1D(KerasLayer): >>> atrousconv1d = AtrousConvolution1D(8, 3, input_shape=(3, 12)) creating: createKerasAtrousConvolution1D - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, nb_filter, filter_length, init="glorot_uniform", activation=None, @@ -844,7 +844,7 @@ class AtrousConvolution2D(KerasLayer): >>> atrousconv2d = AtrousConvolution2D(12, 4, 3, input_shape=(3, 64, 64)) creating: createKerasAtrousConvolution2D - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, nb_filter, nb_row, nb_col, init="glorot_uniform", @@ -910,7 +910,7 @@ class Deconvolution2D(KerasLayer): >>> deconv2d = Deconvolution2D(3, 3, 3, output_shape=(None, 3, 14, 14), input_shape=(3, 12, 12)) creating: createKerasDeconvolution2D - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, nb_filter, nb_row, nb_col, output_shape, init="glorot_uniform", @@ -974,7 +974,7 @@ class SeparableConvolution2D(KerasLayer): >>> separableconv2d = SeparableConvolution2D(12, 3, 4, input_shape=(3, 32, 32)) creating: createKerasSeparableConvolution2D - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, nb_filter, nb_row, nb_col, init="glorot_uniform", @@ -1025,7 +1025,7 @@ class Cropping1D(KerasLayer): >>> cropping1d = Cropping1D(cropping=(1, 2), input_shape=(8, 8)) creating: createKerasCropping1D - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, cropping=(1, 1), input_shape=None, **kwargs): @@ -1052,7 +1052,7 @@ class Cropping2D(KerasLayer): >>> cropping2d = Cropping2D(cropping=((1, 2), (0, 1)), input_shape=(12, 12, 12)) creating: createKerasCropping2D - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, cropping=((0, 0), (0, 0)), dim_ordering="th", @@ -1083,7 +1083,7 @@ class Cropping3D(KerasLayer): >>> cropping3d = Cropping3D(cropping=((0, 2), (1, 1), (3, 1)), input_shape=(4, 12, 12, 16)) creating: createKerasCropping3D - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, cropping=((1, 1), (1, 1), (1, 1)), dim_ordering="th", @@ -1114,7 +1114,7 @@ class UpSampling1D(KerasLayer): >>> upsampling1d = UpSampling1D(length=3, input_shape=(3, 12)) creating: createKerasUpSampling1D - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, length=2, input_shape=None, **kwargs): @@ -1142,7 +1142,7 @@ class UpSampling2D(KerasLayer): >>> upsampling2d = UpSampling2D(size=(1, 3), input_shape=(3, 16, 16)) creating: createKerasUpSampling2D - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, size=(2, 2), dim_ordering="th", input_shape=None, **kwargs): @@ -1172,7 +1172,7 @@ class UpSampling3D(KerasLayer): >>> upsampling3d = UpSampling3D(size=(1, 2, 3), input_shape=(3, 16, 16, 16)) creating: createKerasUpSampling3D - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, size=(2, 2, 2), dim_ordering="th", input_shape=None, **kwargs): @@ -1202,7 +1202,7 @@ class ZeroPadding1D(KerasLayer): >>> zeropadding1d = ZeroPadding1D(padding=2, input_shape=(3, 6)) creating: createKerasZeroPadding1D - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, padding=1, input_shape=None, **kwargs): @@ -1234,7 +1234,7 @@ class ZeroPadding2D(KerasLayer): >>> zeropadding2d = ZeroPadding2D(padding=(2, 1), input_shape=(2, 8, 8)) creating: createKerasZeroPadding2D - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, padding=(1, 1), dim_ordering="th", input_shape=None, **kwargs): @@ -1265,7 +1265,7 @@ class ZeroPadding3D(KerasLayer): >>> zeropadding3d = ZeroPadding3D(padding=(2, 1, 2), input_shape=(2, 8, 8, 10)) creating: createKerasZeroPadding3D - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, padding=(1, 1, 1), dim_ordering="th", input_shape=None, **kwargs): @@ -1295,7 +1295,7 @@ class MaxPooling1D(KerasLayer): >>> maxpooling1d = MaxPooling1D(3, input_shape=(3, 24)) creating: createKerasMaxPooling1D - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, pool_length=2, stride=None, border_mode="valid", @@ -1330,7 +1330,7 @@ class MaxPooling2D(KerasLayer): >>> maxpooling2d = MaxPooling2D((2, 2), input_shape=(3, 32, 32), name="maxpooling2d_1") creating: createKerasMaxPooling2D - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, pool_size=(2, 2), strides=None, @@ -1367,7 +1367,7 @@ class MaxPooling3D(KerasLayer): >>> maxpooling3d = MaxPooling3D((2, 1, 3), input_shape=(3, 32, 32, 32)) creating: createKerasMaxPooling3D - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, pool_size=(2, 2, 2), strides=None, border_mode="valid", @@ -1401,7 +1401,7 @@ class AveragePooling1D(KerasLayer): >>> averagepooling1d = AveragePooling1D(input_shape=(3, 24)) creating: createKerasAveragePooling1D - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, pool_length=2, stride=None, border_mode="valid", @@ -1436,7 +1436,7 @@ class AveragePooling2D(KerasLayer): >>> averagepooling2d = AveragePooling2D((1, 2), input_shape=(2, 28, 32)) creating: createKerasAveragePooling2D - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, pool_size=(2, 2), strides=None, border_mode="valid", @@ -1472,7 +1472,7 @@ class AveragePooling3D(KerasLayer): >>> averagepooling3d = AveragePooling3D((1, 1, 2), input_shape=(3, 28, 32, 36)) creating: createKerasAveragePooling3D - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, pool_size=(2, 2, 2), strides=None, border_mode="valid", @@ -1502,7 +1502,7 @@ class GlobalMaxPooling1D(KerasLayer): >>> globalmaxpooling1d = GlobalMaxPooling1D(input_shape=(4, 8)) creating: createKerasGlobalMaxPooling1D - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, input_shape=None, **kwargs): @@ -1526,7 +1526,7 @@ class GlobalAveragePooling1D(KerasLayer): >>> globalaveragepooling1d = GlobalAveragePooling1D(input_shape=(12, 12)) creating: createKerasGlobalAveragePooling1D - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, input_shape=None, **kwargs): @@ -1551,7 +1551,7 @@ class GlobalMaxPooling2D(KerasLayer): >>> globalmaxpooling2d = GlobalMaxPooling2D(input_shape=(4, 32, 32)) creating: createKerasGlobalMaxPooling2D - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, dim_ordering="th", input_shape=None, **kwargs): @@ -1577,7 +1577,7 @@ class GlobalAveragePooling2D(KerasLayer): >>> globalaveragepooling2d = GlobalAveragePooling2D(input_shape=(4, 32, 32)) creating: createKerasGlobalAveragePooling2D - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, dim_ordering="th", input_shape=None, **kwargs): @@ -1605,7 +1605,7 @@ class GlobalMaxPooling3D(KerasLayer): >>> globalmaxpooling3d = GlobalMaxPooling3D(input_shape=(4, 32, 32, 32)) creating: createKerasGlobalMaxPooling3D - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, dim_ordering="th", input_shape=None, **kwargs): @@ -1633,7 +1633,7 @@ class GlobalAveragePooling3D(KerasLayer): >>> globalaveragepooling3d = GlobalAveragePooling3D(input_shape=(4, 16, 16, 20)) creating: createKerasGlobalAveragePooling3D - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, dim_ordering="th", input_shape=None, **kwargs): @@ -1668,7 +1668,7 @@ class SimpleRNN(KerasLayer): >>> simplernn = SimpleRNN(16, input_shape=(3, 32)) creating: createKerasSimpleRNN - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, output_dim, activation="tanh", return_sequences=False, @@ -1712,7 +1712,7 @@ class LSTM(KerasLayer): >>> lstm = LSTM(32, input_shape=(8, 16), name="lstm1") creating: createKerasLSTM - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, output_dim, activation="tanh", inner_activation="hard_sigmoid", @@ -1757,7 +1757,7 @@ class GRU(KerasLayer): >>> gru = GRU(24, input_shape=(32, 32)) creating: createKerasGRU - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, output_dim, activation="tanh", inner_activation="hard_sigmoid", @@ -1811,7 +1811,7 @@ class ConvLSTM2D(KerasLayer): >>> convlstm2d = ConvLSTM2D(24, 3, 3, input_shape=(4, 32, 32, 32)) creating: createKerasConvLSTM2D - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, nb_filter, nb_row, nb_col, activation="tanh", @@ -1868,7 +1868,7 @@ class LocallyConnected1D(KerasLayer): >>> locallyconnected1d = LocallyConnected1D(6, 3, input_shape=(8, 12)) creating: createKerasLocallyConnected1D - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, nb_filter, filter_length, activation=None, border_mode="valid", @@ -1918,7 +1918,7 @@ class LocallyConnected2D(KerasLayer): >>> locallyconnected2d = LocallyConnected2D(12, 3, 4, input_shape=(3, 128, 128)) creating: createKerasLocallyConnected2D - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, nb_filter, nb_row, nb_col, activation=None, @@ -1961,7 +1961,7 @@ class SpatialDropout1D(KerasLayer): >>> spatialdropout1d = SpatialDropout1D(0.4, input_shape=(10, 12)) creating: createKerasSpatialDropout1D - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, p=0.5, input_shape=None, **kwargs): @@ -1993,7 +1993,7 @@ class SpatialDropout2D(KerasLayer): >>> spatialdropout2d = SpatialDropout2D(0.25, input_shape=(5, 12, 12)) creating: createKerasSpatialDropout2D - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, p=0.5, dim_ordering="th", input_shape=None, **kwargs): @@ -2026,7 +2026,7 @@ class SpatialDropout3D(KerasLayer): >>> spatialdropout3d = SpatialDropout3D(0.6, input_shape=(4, 12, 12, 16)) creating: createKerasSpatialDropout3D - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, p=0.5, dim_ordering="th", input_shape=None, **kwargs): @@ -2054,7 +2054,7 @@ class GaussianDropout(KerasLayer): >>> gaussiandropout = GaussianDropout(0.45, input_shape=(4, 8)) creating: createKerasGaussianDropout - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, p, input_shape=None, **kwargs): @@ -2082,7 +2082,7 @@ class GaussianNoise(KerasLayer): >>> gaussiannoise = GaussianNoise(0.45, input_shape=(3, 4, 5), name="gaussiannoise1") creating: createKerasGaussianNoise - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, sigma, input_shape=None, **kwargs): @@ -2110,7 +2110,7 @@ class Masking(KerasLayer): >>> masking = Masking(0.3, input_shape=(6, 8)) creating: createKerasMasking - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, mask_value=0.0, input_shape=None, **kwargs): @@ -2151,7 +2151,7 @@ class SReLU(KerasLayer): >>> srelu = SReLU(input_shape=(4, 5)) creating: createKerasSReLU - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, t_left_init="zero", a_left_init="glorot_uniform", @@ -2185,7 +2185,7 @@ class ELU(KerasLayer): >>> elu = ELU(1.2, input_shape=(4, 5)) creating: createKerasELU - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, alpha=1.0, input_shape=None, **kwargs): @@ -2213,7 +2213,7 @@ class LeakyReLU(KerasLayer): >>> leakyrelu = LeakyReLU(0.02, input_shape=(4, 5)) creating: createKerasLeakyReLU - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, alpha=0.01, input_shape=None, **kwargs): @@ -2241,7 +2241,7 @@ class ThresholdedReLU(KerasLayer): >>> thresholdedrelu = ThresholdedReLU(input_shape=(10, 12)) creating: createKerasThresholdedReLU - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, theta=1.0, input_shape=None, **kwargs): @@ -2270,7 +2270,7 @@ class TimeDistributed(KerasLayer): creating: createKerasDense creating: createKerasTimeDistributed - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, layer, input_shape=None, **kwargs): @@ -2302,7 +2302,7 @@ class Bidirectional(KerasLayer): creating: createKerasLSTM creating: createKerasBidirectional - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, layer, merge_mode="concat", input_shape=None, **kwargs): diff --git a/python/dllib/src/bigdl/dllib/bigdlkeras/layers/topology.py b/python/dllib/src/bigdl/dllib/bigdlkeras/layers/topology.py index a891635c4c7..51010298f39 100644 --- a/python/dllib/src/bigdl/dllib/bigdlkeras/layers/topology.py +++ b/python/dllib/src/bigdl/dllib/bigdlkeras/layers/topology.py @@ -14,18 +14,18 @@ # limitations under the License. # -from bigdl.nn.keras.layer import KerasLayer -from bigdl.optim.optimizer import * -from bigdl.nn.criterion import * +from bigdl.dllib.keras.layer import KerasLayer +from bigdl.dllib.optim.optimizer import * +from bigdl.dllib.nn.criterion import * import multiprocessing import warnings -from bigdl.nn.layer import SharedStaticUtils, Container +from bigdl.dllib.nn.layer import SharedStaticUtils, Container class KerasModel(KerasLayer, Container, SharedStaticUtils): """ - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __convert_optim_method(self, optimizer): @@ -206,11 +206,11 @@ class Sequential(KerasModel): >>> sequential = Sequential(name="seq1") creating: createKerasSequential - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, jvalue=None, **kwargs): - warnings.warn("bigdl.nn.keras is deprecated in 0.11. " + warnings.warn("bigdl.dllib.keras is deprecated in 0.11. " "Recommend to use Analytics Zoo's Keras API.") super(Sequential, self).__init__(jvalue, **kwargs) @@ -239,11 +239,11 @@ class Model(KerasModel): output: An output node or a list of output nodes. name: String to specify the name of the graph model. Default is None. - .. note:: `bigdl.nn.keras` is deprecated in 0.11. + .. note:: `bigdl.dllib.keras` is deprecated in 0.11. This will be removed in future releases. """ def __init__(self, input, output, jvalue=None, **kwargs): - warnings.warn("bigdl.nn.keras is deprecated in BigDL 0.11." + warnings.warn("bigdl.dllib.keras is deprecated in BigDL 0.11." "Recommend to use Analytics Zoo's Keras API.") super(Model, self).__init__(jvalue, to_list(input), diff --git a/python/dllib/src/bigdl/dllib/bigdlkeras/optimization.py b/python/dllib/src/bigdl/dllib/bigdlkeras/optimization.py index 6c108ea04d7..50aafef7e45 100644 --- a/python/dllib/src/bigdl/dllib/bigdlkeras/optimization.py +++ b/python/dllib/src/bigdl/dllib/bigdlkeras/optimization.py @@ -16,10 +16,10 @@ import keras.optimizers as koptimizers -import bigdl.nn.criterion as bcriterion -import bigdl.optim.optimizer as boptimizer -from bigdl.util.common import to_list -from bigdl.keras.converter import * +import bigdl.dllib.nn.criterion as bcriterion +import bigdl.dllib.optim.optimizer as boptimizer +from bigdl.utils.common import to_list +from bigdl.dllib.keras.converter import * from keras.objectives import * import six diff --git a/python/dllib/src/bigdl/dllib/contrib/onnx/onnx_loader.py b/python/dllib/src/bigdl/dllib/contrib/onnx/onnx_loader.py index 6e537c10c3e..b4b29d6e172 100644 --- a/python/dllib/src/bigdl/dllib/contrib/onnx/onnx_loader.py +++ b/python/dllib/src/bigdl/dllib/contrib/onnx/onnx_loader.py @@ -15,8 +15,8 @@ # import onnx -from bigdl.nn.onnx.layer import * -from bigdl.nn.layer import Identity, Model +from bigdl.dllib.nn.onnx.layer import * +from bigdl.dllib.nn.layer import Identity, Model from .ops_mapping import _convert_map as convert_map from .converter_utils import parse_node_attr, parse_tensor_data diff --git a/python/dllib/src/bigdl/dllib/contrib/onnx/ops_converter.py b/python/dllib/src/bigdl/dllib/contrib/onnx/ops_converter.py index 4934f64ce93..a2acce3b832 100644 --- a/python/dllib/src/bigdl/dllib/contrib/onnx/ops_converter.py +++ b/python/dllib/src/bigdl/dllib/contrib/onnx/ops_converter.py @@ -14,10 +14,10 @@ # limitations under the License. # -from bigdl.nn.layer import SpatialAveragePooling, SpatialBatchNormalization -from bigdl.nn.layer import SpatialConvolution, SpatialMaxPooling, JoinTable -from bigdl.nn.layer import ReLU, SoftMax, CAddTable, Unsqueeze -from bigdl.nn.onnx.layer import Constant, Gather, Gemm, Shape, Reshape +from bigdl.dllib.nn.layer import SpatialAveragePooling, SpatialBatchNormalization +from bigdl.dllib.nn.layer import SpatialConvolution, SpatialMaxPooling, JoinTable +from bigdl.dllib.nn.layer import ReLU, SoftMax, CAddTable, Unsqueeze +from bigdl.dllib.nn.onnx.layer import Constant, Gather, Gemm, Shape, Reshape from .converter_utils import * diff --git a/python/dllib/src/bigdl/dllib/examples/imageframe/inception_validation.py b/python/dllib/src/bigdl/dllib/examples/imageframe/inception_validation.py index bcafa980988..148dea16f2b 100644 --- a/python/dllib/src/bigdl/dllib/examples/imageframe/inception_validation.py +++ b/python/dllib/src/bigdl/dllib/examples/imageframe/inception_validation.py @@ -14,18 +14,19 @@ # limitations under the License. # -from bigdl.util.common import * +from bigdl.utils.common import * -from bigdl.transform.vision.image import * +from bigdl.dllib.feature.transform.vision.image import * -from bigdl.optim.optimizer import * +from bigdl.dllib.optim.optimizer import * from pyspark import SparkContext -from bigdl.nn.layer import * +from bigdl.dllib.nn.layer import * from optparse import OptionParser +import os import sys parser = OptionParser() diff --git a/python/dllib/src/bigdl/dllib/examples/keras/imdb_bigdl_backend.py b/python/dllib/src/bigdl/dllib/examples/keras/imdb_bigdl_backend.py index ee80c97a04c..86c8ce06dbf 100644 --- a/python/dllib/src/bigdl/dllib/examples/keras/imdb_bigdl_backend.py +++ b/python/dllib/src/bigdl/dllib/examples/keras/imdb_bigdl_backend.py @@ -20,8 +20,8 @@ # after building, compiling the model and before the training. # model = with_bigdl_backend(model) -from bigdl.keras.backend import * -from bigdl.examples.keras.imdb_cnn_lstm import * +from bigdl.dllib.keras.backend import * +from bigdl.dllib.examples.keras.imdb_cnn_lstm import * X_train, y_train, X_test, y_test = load_imdb() diff --git a/python/dllib/src/bigdl/dllib/examples/keras/imdb_cnn_lstm.py b/python/dllib/src/bigdl/dllib/examples/keras/imdb_cnn_lstm.py index 9a7b91f21f6..ccc17fdfb53 100644 --- a/python/dllib/src/bigdl/dllib/examples/keras/imdb_cnn_lstm.py +++ b/python/dllib/src/bigdl/dllib/examples/keras/imdb_cnn_lstm.py @@ -72,10 +72,10 @@ def build_keras_model(): hdf5_path = "/tmp/imdb.h5" keras_model.save(hdf5_path) - from bigdl.util.common import * - from bigdl.nn.layer import * - from bigdl.optim.optimizer import * - from bigdl.nn.criterion import * + from bigdl.utils.common import * + from bigdl.dllib.nn.layer import * + from bigdl.dllib.optim.optimizer import * + from bigdl.dllib.nn.criterion import * # Load the HDF5 file with weights to a BigDL model bigdl_model = Model.load_keras(hdf5_path=hdf5_path) diff --git a/python/dllib/src/bigdl/dllib/examples/keras/keras_utils.py b/python/dllib/src/bigdl/dllib/examples/keras/keras_utils.py index 89d8a8b46cd..e95f394e7c5 100644 --- a/python/dllib/src/bigdl/dllib/examples/keras/keras_utils.py +++ b/python/dllib/src/bigdl/dllib/examples/keras/keras_utils.py @@ -14,7 +14,7 @@ # limitations under the License. # -from bigdl.util.common import * +from bigdl.utils.common import * def save_keras_definition(keras_model, path): diff --git a/python/dllib/src/bigdl/dllib/examples/keras/mnist_cnn.py b/python/dllib/src/bigdl/dllib/examples/keras/mnist_cnn.py index 309116c588f..05d9b41e0e4 100644 --- a/python/dllib/src/bigdl/dllib/examples/keras/mnist_cnn.py +++ b/python/dllib/src/bigdl/dllib/examples/keras/mnist_cnn.py @@ -22,7 +22,7 @@ # See README.md for how to run this example. from optparse import OptionParser -from bigdl.examples.keras.keras_utils import * +from bigdl.dllib.examples.keras.keras_utils import * import keras.backend if keras.backend.image_dim_ordering() == "th": @@ -36,8 +36,8 @@ def get_mnist(sc, data_type="train", location="/tmp/mnist"): Download or load MNIST dataset to/from the specified path. Normalize and transform input data into an RDD of Sample """ - from bigdl.dataset import mnist - from bigdl.dataset.transformer import normalizer + from bigdl.dllib.feature.dataset import mnist + from bigdl.dllib.feature.dataset.transformer import normalizer (images, labels) = mnist.read_data_sets(location, data_type) images = images.reshape((images.shape[0], ) + input_shape) images = sc.parallelize(images) @@ -85,10 +85,10 @@ def build_keras_model(): json_path = "/tmp/lenet.json" save_keras_definition(keras_model, json_path) - from bigdl.util.common import * - from bigdl.nn.layer import * - from bigdl.optim.optimizer import * - from bigdl.nn.criterion import * + from bigdl.utils.common import * + from bigdl.dllib.nn.layer import * + from bigdl.dllib.optim.optimizer import * + from bigdl.dllib.nn.criterion import * # Load the JSON file to a BigDL model bigdl_model = Model.load_keras(json_path=json_path) diff --git a/python/dllib/src/bigdl/dllib/examples/lenet/lenet.py b/python/dllib/src/bigdl/dllib/examples/lenet/lenet.py index 75669d084ef..f1fa6f6a44c 100644 --- a/python/dllib/src/bigdl/dllib/examples/lenet/lenet.py +++ b/python/dllib/src/bigdl/dllib/examples/lenet/lenet.py @@ -15,9 +15,9 @@ # from optparse import OptionParser -from bigdl.nn.keras.topology import Sequential -from bigdl.nn.keras.layer import * -from bigdl.dataset import mnist +from bigdl.dllib.keras.topology import Sequential +from bigdl.dllib.keras.layer import * +from bigdl.dllib.feature.dataset import mnist def build_model(class_num): diff --git a/python/dllib/src/bigdl/dllib/examples/onnx/README.md b/python/dllib/src/bigdl/dllib/examples/onnx/README.md index 1dc94bbef1c..5e34c73d1bf 100644 --- a/python/dllib/src/bigdl/dllib/examples/onnx/README.md +++ b/python/dllib/src/bigdl/dllib/examples/onnx/README.md @@ -18,7 +18,7 @@ * Import library dependencies ``` import numpy as np -from bigdl.contrib.onnx import load +from bigdl.dllib.contrib.onnx import load ``` * Set target ONNX ResNet-50 model path diff --git a/python/dllib/src/bigdl/dllib/examples/onnx/load_onnx_resnet.py b/python/dllib/src/bigdl/dllib/examples/onnx/load_onnx_resnet.py index d4c2bfc8371..e6428fe57e6 100644 --- a/python/dllib/src/bigdl/dllib/examples/onnx/load_onnx_resnet.py +++ b/python/dllib/src/bigdl/dllib/examples/onnx/load_onnx_resnet.py @@ -15,7 +15,7 @@ # import numpy as np -from bigdl.contrib.onnx import load +from bigdl.dllib.contrib.onnx import load def load_onnx_resnet(): diff --git a/python/dllib/src/bigdl/dllib/feature/dataset/dataset.py b/python/dllib/src/bigdl/dllib/feature/dataset/dataset.py index 1f142b9c2c7..e27247110eb 100644 --- a/python/dllib/src/bigdl/dllib/feature/dataset/dataset.py +++ b/python/dllib/src/bigdl/dllib/feature/dataset/dataset.py @@ -15,10 +15,10 @@ # import sys -from bigdl.util.common import JavaValue -from bigdl.util.common import callBigDlFunc -from bigdl.util.common import * -from bigdl.transform.vision.image import * +from bigdl.utils.common import JavaValue +from bigdl.utils.common import callBigDlFunc +from bigdl.utils.common import * +from bigdl.dllib.feature.transform.vision.image import * if sys.version >= '3': long = int diff --git a/python/dllib/src/bigdl/dllib/feature/dataset/mnist.py b/python/dllib/src/bigdl/dllib/feature/dataset/mnist.py index 881d8db4a95..319ca236e1d 100644 --- a/python/dllib/src/bigdl/dllib/feature/dataset/mnist.py +++ b/python/dllib/src/bigdl/dllib/feature/dataset/mnist.py @@ -20,8 +20,8 @@ import numpy -from bigdl.dataset import base -from bigdl.dataset.transformer import * +from bigdl.dllib.feature.dataset import base +from bigdl.dllib.feature.dataset.transformer import * SOURCE_URL = 'http://yann.lecun.com/exdb/mnist/' diff --git a/python/dllib/src/bigdl/dllib/feature/dataset/movielens.py b/python/dllib/src/bigdl/dllib/feature/dataset/movielens.py index 93bcefc831f..bf8da2c92bc 100644 --- a/python/dllib/src/bigdl/dllib/feature/dataset/movielens.py +++ b/python/dllib/src/bigdl/dllib/feature/dataset/movielens.py @@ -19,7 +19,7 @@ import zipfile import numpy as np -from bigdl.dataset import base +from bigdl.dllib.feature.dataset import base SOURCE_URL = 'http://files.grouplens.org/datasets/movielens/' def read_data_sets(data_dir): diff --git a/python/dllib/src/bigdl/dllib/feature/dataset/news20.py b/python/dllib/src/bigdl/dllib/feature/dataset/news20.py index 7bdd1c977d5..179ae921e7b 100644 --- a/python/dllib/src/bigdl/dllib/feature/dataset/news20.py +++ b/python/dllib/src/bigdl/dllib/feature/dataset/news20.py @@ -15,7 +15,7 @@ # import tarfile -from bigdl.dataset import base +from bigdl.dllib.feature.dataset import base import os import sys diff --git a/python/dllib/src/bigdl/dllib/feature/dataset/transformer.py b/python/dllib/src/bigdl/dllib/feature/dataset/transformer.py index 8af69d8c220..856aa02c5e6 100644 --- a/python/dllib/src/bigdl/dllib/feature/dataset/transformer.py +++ b/python/dllib/src/bigdl/dllib/feature/dataset/transformer.py @@ -15,7 +15,7 @@ # -from bigdl.util.common import Sample +from bigdl.utils.common import Sample def normalizer(data, mean, std): diff --git a/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py b/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py index f71792d2110..db166a15a01 100644 --- a/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py +++ b/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py @@ -15,9 +15,9 @@ # import sys -from bigdl.util.common import JavaValue -from bigdl.util.common import callBigDlFunc -from bigdl.util.common import * +from bigdl.utils.common import JavaValue +from bigdl.utils.common import callBigDlFunc +from bigdl.utils.common import * if sys.version >= '3': long = int diff --git a/python/dllib/src/bigdl/dllib/keras/ToBigDLHelper.py b/python/dllib/src/bigdl/dllib/keras/ToBigDLHelper.py index 6e0484eec4c..bfbe7d21dce 100644 --- a/python/dllib/src/bigdl/dllib/keras/ToBigDLHelper.py +++ b/python/dllib/src/bigdl/dllib/keras/ToBigDLHelper.py @@ -16,8 +16,8 @@ from math import ceil -import bigdl.nn.initialization_method as BInit -from bigdl.optim.optimizer import L1L2Regularizer as BRegularizer +import bigdl.dllib.nn.initialization_method as BInit +from bigdl.dllib.optim.optimizer import L1L2Regularizer as BRegularizer def to_bigdl_2d_ordering(order): diff --git a/python/dllib/src/bigdl/dllib/keras/converter.py b/python/dllib/src/bigdl/dllib/keras/converter.py index b6df571fab2..2cdf6f86fc9 100644 --- a/python/dllib/src/bigdl/dllib/keras/converter.py +++ b/python/dllib/src/bigdl/dllib/keras/converter.py @@ -15,14 +15,14 @@ # import numpy as np -import bigdl.nn.layer as BLayer -import bigdl.util.common as BCommon -from bigdl.util.common import get_activation_by_name +import bigdl.dllib.nn.layer as BLayer +import bigdl.utils.common as BCommon +from bigdl.utils.common import get_activation_by_name from keras.models import model_from_json from keras.models import Sequential, Model, Layer import keras import warnings -from bigdl.keras.ToBigDLHelper import * +from bigdl.dllib.keras.ToBigDLHelper import * def unsupport_exp(name): diff --git a/python/dllib/src/bigdl/dllib/models/inception/README.md b/python/dllib/src/bigdl/dllib/models/inception/README.md index 86cafb879ac..e995a492f1f 100644 --- a/python/dllib/src/bigdl/dllib/models/inception/README.md +++ b/python/dllib/src/bigdl/dllib/models/inception/README.md @@ -35,7 +35,7 @@ This command will transform the images into hadoop sequence files, which are more suitable for a distributed training. ```bash -java -cp bigdl_source_folder/spark/dl/target/bigdl-VERSION-jar-with-dependencies-and-spark.jar com.intel.analytics.bigdl.models.utils.ImageNetSeqFileGenerator -f imagenet_folder -o output_folder -p cores_number +java -cp bigdl_source_folder/spark/dl/target/bigdl-VERSION-jar-with-dependencies-and-spark.jar com.intel.analytics.bigdl.dllib.models.utils.ImageNetSeqFileGenerator -f imagenet_folder -o output_folder -p cores_number ``` It will generate the hadoop sequence files in the output folder. diff --git a/python/dllib/src/bigdl/dllib/models/inception/inception.py b/python/dllib/src/bigdl/dllib/models/inception/inception.py index 64f78c2a4ed..fa413cacef8 100644 --- a/python/dllib/src/bigdl/dllib/models/inception/inception.py +++ b/python/dllib/src/bigdl/dllib/models/inception/inception.py @@ -1,9 +1,26 @@ -from bigdl.nn.layer import * +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +from bigdl.dllib.nn.layer import * from optparse import OptionParser -from bigdl.nn.criterion import * -from bigdl.nn.initialization_method import * -from bigdl.optim.optimizer import * -from bigdl.transform.vision.image import * +from bigdl.dllib.nn.criterion import * +from bigdl.dllib.nn.initialization_method import * +from bigdl.dllib.optim.optimizer import * +from bigdl.dllib.feature.transform.vision.image import * from math import ceil diff --git a/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py b/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py index cb4a940d5ed..4125edbc46e 100644 --- a/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py +++ b/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py @@ -15,12 +15,12 @@ # from optparse import OptionParser -from bigdl.models.lenet.utils import * -from bigdl.dataset.transformer import * -from bigdl.nn.layer import * -from bigdl.nn.criterion import * -from bigdl.optim.optimizer import * -from bigdl.util.common import * +from bigdl.dllib.models.lenet.utils import * +from bigdl.dllib.feature.dataset.transformer import * +from bigdl.dllib.nn.layer import * +from bigdl.dllib.nn.criterion import * +from bigdl.dllib.optim.optimizer import * +from bigdl.utils.common import * def build_model(class_num): diff --git a/python/dllib/src/bigdl/dllib/models/lenet/utils.py b/python/dllib/src/bigdl/dllib/models/lenet/utils.py index e90444fda33..85a3329d8b1 100644 --- a/python/dllib/src/bigdl/dllib/models/lenet/utils.py +++ b/python/dllib/src/bigdl/dllib/models/lenet/utils.py @@ -14,9 +14,9 @@ # limitations under the License. # -from bigdl.dataset import mnist -from bigdl.dataset.transformer import * -from bigdl.optim.optimizer import * +from bigdl.dllib.feature.dataset import mnist +from bigdl.dllib.feature.dataset.transformer import * +from bigdl.dllib.optim.optimizer import * def get_mnist(sc, data_type="train", location="/tmp/mnist"): diff --git a/python/dllib/src/bigdl/dllib/models/local_lenet/local_lenet.py b/python/dllib/src/bigdl/dllib/models/local_lenet/local_lenet.py index 125f060ff32..fee0dec22c1 100644 --- a/python/dllib/src/bigdl/dllib/models/local_lenet/local_lenet.py +++ b/python/dllib/src/bigdl/dllib/models/local_lenet/local_lenet.py @@ -15,11 +15,11 @@ # from optparse import OptionParser -from bigdl.dataset import mnist -from bigdl.models.lenet.lenet5 import build_model -from bigdl.nn.criterion import * -from bigdl.optim.optimizer import * -from bigdl.util.common import * +from bigdl.dllib.feature.dataset import mnist +from bigdl.dllib.models.lenet.lenet5 import build_model +from bigdl.dllib.nn.criterion import * +from bigdl.dllib.optim.optimizer import * +from bigdl.utils.common import * def get_mnist(data_type="train", location="/tmp/mnist"): diff --git a/python/dllib/src/bigdl/dllib/models/ml_pipeline/dl_classifier.py b/python/dllib/src/bigdl/dllib/models/ml_pipeline/dl_classifier.py index dd0833fa6f6..89c6cea013a 100644 --- a/python/dllib/src/bigdl/dllib/models/ml_pipeline/dl_classifier.py +++ b/python/dllib/src/bigdl/dllib/models/ml_pipeline/dl_classifier.py @@ -1,7 +1,7 @@ from pyspark.ml.pipeline import Estimator, Model from pyspark.ml.param.shared import * -from bigdl.util.common import * - +from bigdl.utils.common import * +import sys if sys.version >= '3': long = int diff --git a/python/dllib/src/bigdl/dllib/models/rnn/rnnexample.py b/python/dllib/src/bigdl/dllib/models/rnn/rnnexample.py index 56061a29adf..ebeef9eb2b6 100644 --- a/python/dllib/src/bigdl/dllib/models/rnn/rnnexample.py +++ b/python/dllib/src/bigdl/dllib/models/rnn/rnnexample.py @@ -19,13 +19,13 @@ import re from optparse import OptionParser -from bigdl.dataset import base -from bigdl.dataset import sentence -from bigdl.nn.layer import * -from bigdl.nn.criterion import * -from bigdl.optim.optimizer import * -from bigdl.util.common import * -from bigdl.util.common import Sample +from bigdl.dllib.feature.dataset import base +from bigdl.dllib.feature.dataset import sentence +from bigdl.dllib.nn.layer import * +from bigdl.dllib.nn.criterion import * +from bigdl.dllib.optim.optimizer import * +from bigdl.utils.common import * +from bigdl.utils.common import Sample def download_data(dest_dir): TINYSHAKESPEARE_URL = 'https://raw.githubusercontent.com/karpathy/char-rnn/master/data/tinyshakespeare/input.txt' # noqa diff --git a/python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py b/python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py index c4d0c5d6e12..8dcca08c312 100644 --- a/python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py +++ b/python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py @@ -19,12 +19,12 @@ import re from optparse import OptionParser -from bigdl.dataset import news20 -from bigdl.nn.layer import * -from bigdl.nn.criterion import * -from bigdl.optim.optimizer import * -from bigdl.util.common import * -from bigdl.util.common import Sample +from bigdl.dllib.feature.dataset import news20 +from bigdl.dllib.nn.layer import * +from bigdl.dllib.nn.criterion import * +from bigdl.dllib.optim.optimizer import * +from bigdl.utils.common import * +from bigdl.utils.common import Sample import datetime as dt diff --git a/python/dllib/src/bigdl/dllib/models/utils/model_broadcast.py b/python/dllib/src/bigdl/dllib/models/utils/model_broadcast.py index 8e42ee311b8..7aa16b8be58 100644 --- a/python/dllib/src/bigdl/dllib/models/utils/model_broadcast.py +++ b/python/dllib/src/bigdl/dllib/models/utils/model_broadcast.py @@ -21,7 +21,7 @@ from pyspark.broadcast import Broadcast from pyspark.broadcast import _from_id -from bigdl.nn.layer import Model +from bigdl.dllib.nn.layer import Model def _from_id_and_type(bid, bigdl_type): result = _from_id(bid) diff --git a/python/dllib/src/bigdl/dllib/nn/criterion.py b/python/dllib/src/bigdl/dllib/nn/criterion.py index 038a475b8f8..2555a060a8b 100644 --- a/python/dllib/src/bigdl/dllib/nn/criterion.py +++ b/python/dllib/src/bigdl/dllib/nn/criterion.py @@ -17,10 +17,10 @@ import sys -from bigdl.util.common import JavaValue -from bigdl.util.common import callBigDlFunc -from bigdl.util.common import JTensor -from bigdl.nn.layer import Layer +from bigdl.utils.common import JavaValue +from bigdl.utils.common import callBigDlFunc +from bigdl.utils.common import JTensor +from bigdl.dllib.nn.layer import Layer import numpy as np if sys.version >= '3': @@ -978,9 +978,9 @@ def __init__(self, def _test(): import doctest from pyspark import SparkContext - from bigdl.nn import criterion - from bigdl.util.common import init_engine - from bigdl.util.common import create_spark_conf + from bigdl.dllib.nn import criterion + from bigdl.utils.common import init_engine + from bigdl.utils.common import create_spark_conf globs = criterion.__dict__.copy() sc = SparkContext(master="local[4]", appName="test criterion", conf=create_spark_conf()) diff --git a/python/dllib/src/bigdl/dllib/nn/initialization_method.py b/python/dllib/src/bigdl/dllib/nn/initialization_method.py index aaddee5597f..4f32027fbe2 100644 --- a/python/dllib/src/bigdl/dllib/nn/initialization_method.py +++ b/python/dllib/src/bigdl/dllib/nn/initialization_method.py @@ -16,7 +16,7 @@ import sys -from bigdl.util.common import JavaValue +from bigdl.utils.common import JavaValue if sys.version >= '3': long = int diff --git a/python/dllib/src/bigdl/dllib/nn/keras/__init__.py b/python/dllib/src/bigdl/dllib/nn/keras/__init__.py index 086e4e1f057..9eb670a1780 100644 --- a/python/dllib/src/bigdl/dllib/nn/keras/__init__.py +++ b/python/dllib/src/bigdl/dllib/nn/keras/__init__.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # -from bigdl.util.common import * +from bigdl.utils.common import * init_engine() redire_spark_logs() diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 6d02d86540e..bfa012e690f 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -21,19 +21,19 @@ import numpy as np import six -from bigdl.util.common import JTensor -from bigdl.util.common import JavaValue -from bigdl.util.common import callBigDlFunc -from bigdl.util.common import callJavaFunc -from bigdl.util.common import get_spark_context -from bigdl.util.common import to_list -from bigdl.util.common import INTMAX, INTMIN, DOUBLEMAX -from bigdl.util.common import get_activation_by_name -from bigdl.optim.optimizer import L1Regularizer, L2Regularizer, L1L2Regularizer +from bigdl.utils.common import JTensor +from bigdl.utils.common import JavaValue +from bigdl.utils.common import callBigDlFunc +from bigdl.utils.common import callJavaFunc +from bigdl.utils.common import get_spark_context +from bigdl.utils.common import to_list +from bigdl.utils.common import INTMAX, INTMIN, DOUBLEMAX +from bigdl.utils.common import get_activation_by_name +from bigdl.dllib.optim.optimizer import L1Regularizer, L2Regularizer, L1L2Regularizer from py4j.java_gateway import JavaObject from pyspark.rdd import RDD -from bigdl.transform.vision.image import ImageFrame -from bigdl.dataset.dataset import DataSet +from bigdl.dllib.feature.transform.vision.image import ImageFrame +from bigdl.dllib.feature.dataset.dataset import DataSet if sys.version >= '3': long = int @@ -98,13 +98,13 @@ def get_py_name(jclass_name): jpackage_name = ".".join(jname.split(".")[:-1]) pclass_name = get_py_name(jname.split(".")[-1]) - if "com.intel.analytics.bigdl.nn.keras.Model" == jname or \ - "com.intel.analytics.bigdl.nn.keras.Sequential" == jname: - base_module = importlib.import_module('bigdl.nn.keras.topology') - elif "com.intel.analytics.bigdl.nn.keras" == jpackage_name: - base_module = importlib.import_module('bigdl.nn.keras.layer') + if "com.intel.analytics.bigdl.dllib.keras.Model" == jname or \ + "com.intel.analytics.bigdl.dllib.keras.Sequential" == jname: + base_module = importlib.import_module('bigdl.dllib.keras.topology') + elif "com.intel.analytics.bigdl.dllib.keras" == jpackage_name: + base_module = importlib.import_module('bigdl.dllib.keras.layer') else: - base_module = importlib.import_module('bigdl.nn.layer') + base_module = importlib.import_module('bigdl.dllib.nn.layer') realClassName = "Layer" # The top base class if pclass_name in dir(base_module): @@ -807,7 +807,7 @@ def load_keras(json_path=None, hdf5_path=None, by_name=False): except ImportError: raise Exception("No backend is found for Keras. " "Please install either tensorflow or theano.") - from bigdl.keras.converter import DefinitionLoader, WeightLoader + from bigdl.dllib.keras.converter import DefinitionLoader, WeightLoader if json_path and not hdf5_path: return DefinitionLoader.from_json_path(json_path) elif json_path and hdf5_path: @@ -864,7 +864,7 @@ def load_tensorflow(path, inputs, outputs, byte_order = "little_endian", @staticmethod def train(output, data, label, opt_method, criterion, batch_size, end_when, session=None, bigdl_type="float"): from bigdl.util.tf_utils import get_path - from bigdl.util.common import Sample + from bigdl.utils.common import Sample output_name = output.name.split(":")[0] path = get_path(output_name, session) sc = get_spark_context() @@ -3927,7 +3927,7 @@ class SReLU(Layer): creating: createSReLU >>> srelu = SReLU((2, 2), (1, 2)) creating: createSReLU - >>> from bigdl.nn.initialization_method import Xavier + >>> from bigdl.dllib.nn.initialization_method import Xavier >>> init = Xavier() creating: createXavier >>> srelu = srelu.set_init_method(tLeftInit=init, aLeftInit=init, tRightInit=init, aRightInit=init) @@ -5783,9 +5783,9 @@ def __init__(self, def _test(): import doctest from pyspark import SparkContext - from bigdl.nn import layer - from bigdl.util.common import init_engine - from bigdl.util.common import create_spark_conf + from bigdl.dllib.nn import layer + from bigdl.utils.common import init_engine + from bigdl.utils.common import create_spark_conf globs = layer.__dict__.copy() sc = SparkContext(master="local[4]", appName="test layer", conf=create_spark_conf()) diff --git a/python/dllib/src/bigdl/dllib/nn/onnx/layer.py b/python/dllib/src/bigdl/dllib/nn/onnx/layer.py index d99b1e8e86c..f2e255f5e31 100644 --- a/python/dllib/src/bigdl/dllib/nn/onnx/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/onnx/layer.py @@ -16,8 +16,8 @@ import sys import numpy as np -from bigdl.nn.layer import Layer -from bigdl.util.common import JTensor +from bigdl.dllib.nn.layer import Layer +from bigdl.utils.common import JTensor if sys.version >= '3': long = int diff --git a/python/dllib/src/bigdl/dllib/optim/optimizer.py b/python/dllib/src/bigdl/dllib/optim/optimizer.py index 793b7cda31e..2786272c993 100644 --- a/python/dllib/src/bigdl/dllib/optim/optimizer.py +++ b/python/dllib/src/bigdl/dllib/optim/optimizer.py @@ -23,15 +23,15 @@ from py4j.java_gateway import JavaObject from pyspark.rdd import RDD -from bigdl.util.common import DOUBLEMAX -from bigdl.util.common import JTensor -from bigdl.util.common import JavaValue -from bigdl.util.common import callBigDlFunc -from bigdl.util.common import callJavaFunc -from bigdl.util.common import get_node_and_core_number -from bigdl.util.common import init_engine -from bigdl.util.common import to_list -from bigdl.dataset.dataset import * +from bigdl.utils.common import DOUBLEMAX +from bigdl.utils.common import JTensor +from bigdl.utils.common import JavaValue +from bigdl.utils.common import callBigDlFunc +from bigdl.utils.common import callJavaFunc +from bigdl.utils.common import get_node_and_core_number +from bigdl.utils.common import init_engine +from bigdl.utils.common import to_list +from bigdl.dllib.feature.dataset.dataset import * import warnings if sys.version >= '3': @@ -100,7 +100,7 @@ class Loss(JavaValue): """ This evaluation method is calculate loss of output with respect to target - >>> from bigdl.nn.criterion import ClassNLLCriterion + >>> from bigdl.dllib.nn.criterion import ClassNLLCriterion >>> loss = Loss() creating: createClassNLLCriterion creating: createLoss @@ -110,7 +110,7 @@ class Loss(JavaValue): creating: createLoss """ def __init__(self, cri=None, bigdl_type="float"): - from bigdl.nn.criterion import ClassNLLCriterion + from bigdl.dllib.nn.criterion import ClassNLLCriterion if cri is None: cri = ClassNLLCriterion() JavaValue.__init__(self, None, bigdl_type, cri) @@ -789,7 +789,7 @@ def optimize(self): Do an optimization. """ jmodel = callJavaFunc(self.value.optimize) - from bigdl.nn.layer import Layer + from bigdl.dllib.nn.layer import Layer return Layer.of(jmodel) def set_train_summary(self, summary): @@ -1189,9 +1189,9 @@ def __init__(self, l2, bigdl_type="float"): def _test(): import doctest from pyspark import SparkContext - from bigdl.optim import optimizer - from bigdl.util.common import init_engine - from bigdl.util.common import create_spark_conf + from bigdl.dllib.optim import optimizer + from bigdl.utils.common import init_engine + from bigdl.utils.common import create_spark_conf globs = optimizer.__dict__.copy() sc = SparkContext(master="local[4]", appName="test optimizer", conf=create_spark_conf()) diff --git a/python/dllib/src/bigdl/dllib/utils/tf_utils.py b/python/dllib/src/bigdl/dllib/utils/tf_utils.py index c19a508d79e..58abc2695bd 100644 --- a/python/dllib/src/bigdl/dllib/utils/tf_utils.py +++ b/python/dllib/src/bigdl/dllib/utils/tf_utils.py @@ -26,9 +26,9 @@ from tensorflow.python.framework import graph_util from tensorflow.python.framework import importer from tensorflow.python.platform import gfile -from bigdl.nn.layer import Model -from bigdl.util.common import JTensor -from bigdl.util.common import callBigDlFunc +from bigdl.dllib.nn.layer import Model +from bigdl.utils.common import JTensor +from bigdl.utils.common import callBigDlFunc import os def get_path(output_name, sess=None): diff --git a/python/dllib/src/bigdl/utils/common.py b/python/dllib/src/bigdl/utils/common.py index f4f51834af3..684fd22ad60 100644 --- a/python/dllib/src/bigdl/utils/common.py +++ b/python/dllib/src/bigdl/utils/common.py @@ -154,7 +154,7 @@ class JTensor(object): A wrapper to easy our work when need to pass or return Tensor to/from Scala. >>> import numpy as np - >>> from bigdl.util.common import JTensor + >>> from bigdl.utils.common import JTensor >>> np.random.seed(123) >>> """ @@ -189,8 +189,8 @@ def from_ndarray(cls, a_ndarray, bigdl_type="float"): Convert a ndarray to a DenseTensor which would be used in Java side. >>> import numpy as np - >>> from bigdl.util.common import JTensor - >>> from bigdl.util.common import callBigDlFunc + >>> from bigdl.utils.common import JTensor + >>> from bigdl.utils.common import callBigDlFunc >>> np.random.seed(123) >>> data = np.random.uniform(0, 1, (2, 3)).astype("float32") >>> result = JTensor.from_ndarray(data) @@ -236,8 +236,8 @@ def sparse(cls, a_ndarray, i_ndarray, shape, bigdl_type="float"): :param shape shape as a DenseTensor. >>> import numpy as np - >>> from bigdl.util.common import JTensor - >>> from bigdl.util.common import callBigDlFunc + >>> from bigdl.utils.common import JTensor + >>> from bigdl.utils.common import callBigDlFunc >>> np.random.seed(123) >>> data = np.arange(1, 7).astype("float32") >>> indices = np.arange(1, 7) @@ -314,7 +314,7 @@ def from_ndarray(cls, features, labels, bigdl_type="float"): :param bigdl_type: "double" or "float" >>> import numpy as np - >>> from bigdl.util.common import callBigDlFunc + >>> from bigdl.utils.common import callBigDlFunc >>> from numpy.testing import assert_allclose >>> np.random.seed(123) >>> sample = Sample.from_ndarray(np.random.random((2,3)), np.random.random((2,3))) @@ -495,7 +495,7 @@ def to_sample_rdd(x, y, numSlices=None): :return: """ sc = get_spark_context() - from bigdl.util.common import Sample + from bigdl.utils.common import Sample x_rdd = sc.parallelize(x, numSlices) y_rdd = sc.parallelize(y, numSlices) return x_rdd.zip(y_rdd).map(lambda item: Sample.from_ndarray(item[0], item[1])) @@ -720,7 +720,7 @@ def is_distributed(path): def get_activation_by_name(activation_name, activation_id=None): """ Convert to a bigdl activation layer given the name of the activation as a string """ - import bigdl.nn.layer as BLayer + import bigdl.dllib.nn.layer as BLayer activation = None activation_name = activation_name.lower() if activation_name == "tanh": @@ -749,7 +749,7 @@ def get_activation_by_name(activation_name, activation_id=None): def _test(): import doctest from pyspark import SparkContext - from bigdl.nn import layer + from bigdl.dllib.nn import layer globs = layer.__dict__.copy() sc = SparkContext(master="local[2]", appName="test common utility") globs['sc'] = sc diff --git a/python/dllib/src/python-zip.xml b/python/dllib/src/python-zip.xml new file mode 100644 index 00000000000..1ece79d1ab0 --- /dev/null +++ b/python/dllib/src/python-zip.xml @@ -0,0 +1,28 @@ + + python-api + + zip + + false + + + + **/*.py + + + test/** + + /.. + ${project.build.directory}/../../../python + + + + + + + + + + diff --git a/python/dllib/test/__init__.py b/python/dllib/test/__init__.py deleted file mode 100644 index 63e9a94acc5..00000000000 --- a/python/dllib/test/__init__.py +++ /dev/null @@ -1,18 +0,0 @@ -# -# Copyright 2016 The BigDL Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from bigdl.util.engine import prepare_env -prepare_env() diff --git a/python/release.sh b/python/release.sh new file mode 100755 index 00000000000..306a23c41c0 --- /dev/null +++ b/python/release.sh @@ -0,0 +1,83 @@ +#!/usr/bin/env bash + +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -e +RUN_SCRIPT_DIR=$(cd $(dirname $0) ; pwd) +echo $RUN_SCRIPT_DIR +BIGDL_DIR="$(cd ${RUN_SCRIPT_DIR}/../../../../; pwd)" +echo $BIGDL_DIR +BIGDL_PYTHON_DIR="$(cd ${RUN_SCRIPT_DIR}/../../../../pyspark; pwd)" +echo $BIGDL_PYTHON_DIR + +if (( $# < 2)); then + echo "Bad parameters. Usage: release.sh mac spark_2.x" + exit -1 +fi + +platform=$1 +spark_profile=$2 +quick=$3 +input_version=$4 +bigdl_version=$(python -c "exec(open('$BIGDL_DIR/pyspark/bigdl/version.py').read()); print(__version__)") + +if [ "$input_version" != "$bigdl_version" ]; then + echo "Not the proposed version: $bigdl_version" + exit -1 +fi + +cd ${BIGDL_DIR} +if [ "$platform" == "mac" ]; then + echo "Building bigdl for mac system" + dist_profile="-P mac -P $spark_profile" + verbose_pname="macosx_10_11_x86_64" +elif [ "$platform" == "linux" ]; then + echo "Building bigdl for linux system" + dist_profile="-P $spark_profile" + verbose_pname="manylinux1_x86_64" +else + echo "unsupport platform" +fi + +bigdl_build_command="${BIGDL_DIR}/make-dist.sh ${dist_profile}" +if [ "$quick" == "true" ]; then + echo "Skip disting BigDL" +else + echo "Dist BigDL: $bigdl_build_command" + $bigdl_build_command +fi + +cd $BIGDL_PYTHON_DIR +sdist_command="python setup.py sdist" +echo "packing source code: ${sdist_command}" +$sdist_command + +if [ -d "${BIGDL_DIR}/pyspark/build" ]; then + rm -r ${BIGDL_DIR}/pyspark/build +fi + +if [ -d "${BIGDL_DIR}/pyspark/dist" ]; then + rm -r ${BIGDL_DIR}/pyspark/dist +fi +wheel_command="python setup.py bdist_wheel --plat-name ${verbose_pname}" +echo "Packing python distribution: $wheel_command" +${wheel_command} + +upload_command="twine upload dist/BigDL-${bigdl_version}-py2.py3-none-${verbose_pname}.whl" +echo "Please manually upload with this command: $upload_command" + +$upload_command diff --git a/python/run_lenet.sh b/python/run_lenet.sh new file mode 100755 index 00000000000..4b5764cba94 --- /dev/null +++ b/python/run_lenet.sh @@ -0,0 +1,18 @@ +SPARK_HOME=/home/ding/Downloads/spark-2.4.3-bin-hadoop2.7 +MASTER=local[2] +PYTHON_API_ZIP_PATH=/home/ding/proj/clone-ding-zoo/analytics-zoo/scala/dllib/target/bigdl-dllib-2.0.0-SNAPSHOT-python-api.zip +BigDL_JAR_PATH=/home/ding/proj/clone-ding-zoo/analytics-zoo/scala/dllib/bigdl-dllib-2.0.0-SNAPSHOT-jar-with-dependencies.jar +PYTHONPATH=${PYTHON_API_ZIP_PATH}:$PYTHONPATH +${SPARK_HOME}/bin/spark-submit \ + --master ${MASTER} \ + --driver-cores 2 \ + --driver-memory 2g \ + --total-executor-cores 2 \ + --executor-cores 2 \ + --executor-memory 4g \ + --py-files ${PYTHON_API_ZIP_PATH},/home/ding/proj/clone-ding-zoo/analytics-zoo/python/dllib/examples/lenet/lenet.py \ + --properties-file /home/ding/proj/clone-ding-zoo/analytics-zoo/scala/dllib/src/main/resources/spark-bigdl.conf \ + --jars ${BigDL_JAR_PATH} \ + --conf spark.driver.extraClassPath=${BigDL_JAR_PATH} \ + --conf spark.executor.extraClassPath=/home/ding/proj/clone-ding-zoo/analytics-zoo/scala/dllib/bigdl-dllib-2.0.0-SNAPSHOT-jar-with-dependencies.jar \ + /home/ding/proj/clone-ding-zoo/analytics-zoo/python/dllib/examples/lenet/lenet.py diff --git a/scala/dllib/pom.xml b/scala/dllib/pom.xml index 2c0c0ffb77f..c34eccf3e57 100644 --- a/scala/dllib/pom.xml +++ b/scala/dllib/pom.xml @@ -245,10 +245,10 @@ - - - - + + org.apache.maven.plugins + maven-assembly-plugin + @@ -261,8 +261,22 @@ - - + + python + false + + + ${project.basedir}/../../python/python-zip.xml + + + + package + + single + + + + org.scoverage diff --git a/version.py b/version.py deleted file mode 100644 index 302e0a86caa..00000000000 --- a/version.py +++ /dev/null @@ -1,17 +0,0 @@ -# -# Copyright 2016 The BigDL Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -__version__ = "0.14.0.dev0" From edf6b53287950d47f2cfea9b1e1f4b81da43c46e Mon Sep 17 00:00:00 2001 From: dding3 Date: Mon, 16 Aug 2021 14:43:41 -0700 Subject: [PATCH 797/823] fix run example failure --- python/.idea/workspace.xml | 780 ++++----- .../bigdl/dllib/bigdlkeras/layers/__init__.py | 5 + python/dllib/src/bigdl/dllib/utils/common.py | 6 +- python/release.sh | 27 +- python/run_lenet.sh | 10 +- .../dllib/utils/python/api/BigDLSerde.scala | 6 +- .../dllib/utils/python/api/PythonBigDL.scala | 2 +- .../utils/python/api/PythonBigDLKeras.scala | 1494 ++++++++--------- .../utils/python/api/PythonBigDLOnnx.scala | 134 +- .../python/api/PythonBigDLValidator.scala | 162 +- .../dllib/utils/tf/TensorflowLoader.scala | 2 +- .../bigdl/dllib/python/api/PythonSpec.scala | 2 +- 12 files changed, 1287 insertions(+), 1343 deletions(-) diff --git a/python/.idea/workspace.xml b/python/.idea/workspace.xml index 4f6b228c869..af60d1ae9da 100644 --- a/python/.idea/workspace.xml +++ b/python/.idea/workspace.xml @@ -12,35 +12,31 @@ - - + + - - - - - + + + - - + + - - - - - + + + - - + + - - + + @@ -48,8 +44,8 @@ - - + + @@ -58,69 +54,72 @@ - - + + - - + + - + + + + + + + - - + + - - - - - + + + - - + + - - + + - + + - - + + - - - - - + + + - - + + - - + + - - + + - - + + @@ -130,10 +129,6 @@ - lenet.py - lenet.p - b - bigd bigdl bigdl.d bigdl.da @@ -160,6 +155,10 @@ bigdl.nn bigdl.models bigdl. + jar_dir + get_bigdl_classpath + spark_profile + make BIGDL_HOME @@ -180,7 +179,6 @@ @@ -243,10 +247,68 @@ - + + + + + + + + + + + + + + + - - + + - + @@ -426,105 +608,129 @@ - + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - - + + - + - - + + - + - + - - + + - + - - + + - + - - - + - + @@ -532,65 +738,31 @@ - + - - - - - - - - - - - - - - - - - - + + - + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - + - - + + - + @@ -598,299 +770,79 @@ - - - - - - - - + - - - - - - - - - - - - - - - - - - - - - - - - - - + + - - - - - - - - - - - - - - - - - - - - - + - - + + - - - - - - - - - - - - - - - - - - - - - + - - - - - - - - - - - - - - - + + + - + - - + + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - - + + - - - - - - - - - - - + - - + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - - + + - - - - - - - - - - - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/python/dllib/src/bigdl/dllib/bigdlkeras/layers/__init__.py b/python/dllib/src/bigdl/dllib/bigdlkeras/layers/__init__.py index 2151a805423..9eb670a1780 100644 --- a/python/dllib/src/bigdl/dllib/bigdlkeras/layers/__init__.py +++ b/python/dllib/src/bigdl/dllib/bigdlkeras/layers/__init__.py @@ -13,3 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. # +from bigdl.utils.common import * + +init_engine() +redire_spark_logs() +show_bigdl_info_logs() diff --git a/python/dllib/src/bigdl/dllib/utils/common.py b/python/dllib/src/bigdl/dllib/utils/common.py index 684fd22ad60..af7a03cb89f 100644 --- a/python/dllib/src/bigdl/dllib/utils/common.py +++ b/python/dllib/src/bigdl/dllib/utils/common.py @@ -32,7 +32,7 @@ import threading import tempfile import traceback -from bigdl.util.engine import get_bigdl_classpath, is_spark_below_2_2 +from bigdl.utils.engine import get_bigdl_classpath, is_spark_below_2_2 INTMAX = 2147483647 INTMIN = -2147483648 @@ -65,8 +65,8 @@ def __init__(self, bigdl_type, port=25333): class JavaCreator(SingletonMixin): __creator_class=[ - "com.intel.analytics.bigdl.python.api.PythonBigDLKeras", - "com.intel.analytics.bigdl.python.api.PythonBigDLOnnx" + "com.intel.analytics.bigdl.dllib.utils.python.api.PythonBigDLKeras", + "com.intel.analytics.bigdl.dllib.utils.python.api.PythonBigDLOnnx" ] @classmethod diff --git a/python/release.sh b/python/release.sh index 306a23c41c0..e070f510b47 100755 --- a/python/release.sh +++ b/python/release.sh @@ -19,9 +19,9 @@ set -e RUN_SCRIPT_DIR=$(cd $(dirname $0) ; pwd) echo $RUN_SCRIPT_DIR -BIGDL_DIR="$(cd ${RUN_SCRIPT_DIR}/../../../../; pwd)" +BIGDL_DIR="$(cd ${RUN_SCRIPT_DIR}/../; pwd)" echo $BIGDL_DIR -BIGDL_PYTHON_DIR="$(cd ${RUN_SCRIPT_DIR}/../../../../pyspark; pwd)" +BIGDL_PYTHON_DIR="$(cd ${RUN_SCRIPT_DIR}/../python; pwd)" echo $BIGDL_PYTHON_DIR if (( $# < 2)); then @@ -32,13 +32,7 @@ fi platform=$1 spark_profile=$2 quick=$3 -input_version=$4 -bigdl_version=$(python -c "exec(open('$BIGDL_DIR/pyspark/bigdl/version.py').read()); print(__version__)") - -if [ "$input_version" != "$bigdl_version" ]; then - echo "Not the proposed version: $bigdl_version" - exit -1 -fi +bigdl_version=$(python -c "exec(open('$BIGDL_DIR/python/bigdl/version.py').read()); print(__version__)") cd ${BIGDL_DIR} if [ "$platform" == "mac" ]; then @@ -53,7 +47,7 @@ else echo "unsupport platform" fi -bigdl_build_command="${BIGDL_DIR}/make-dist.sh ${dist_profile}" +bigdl_build_command="${BIGDL_DIR}/scala/make-dist.sh ${dist_profile}" if [ "$quick" == "true" ]; then echo "Skip disting BigDL" else @@ -66,18 +60,11 @@ sdist_command="python setup.py sdist" echo "packing source code: ${sdist_command}" $sdist_command -if [ -d "${BIGDL_DIR}/pyspark/build" ]; then - rm -r ${BIGDL_DIR}/pyspark/build -fi - -if [ -d "${BIGDL_DIR}/pyspark/dist" ]; then - rm -r ${BIGDL_DIR}/pyspark/dist -fi wheel_command="python setup.py bdist_wheel --plat-name ${verbose_pname}" echo "Packing python distribution: $wheel_command" ${wheel_command} -upload_command="twine upload dist/BigDL-${bigdl_version}-py2.py3-none-${verbose_pname}.whl" -echo "Please manually upload with this command: $upload_command" +#upload_command="twine upload dist/BigDL-${bigdl_version}-py2.py3-none-${verbose_pname}.whl" +#echo "Please manually upload with this command: $upload_command" -$upload_command +#$upload_command diff --git a/python/run_lenet.sh b/python/run_lenet.sh index 4b5764cba94..fd45228b04a 100755 --- a/python/run_lenet.sh +++ b/python/run_lenet.sh @@ -1,7 +1,7 @@ SPARK_HOME=/home/ding/Downloads/spark-2.4.3-bin-hadoop2.7 MASTER=local[2] -PYTHON_API_ZIP_PATH=/home/ding/proj/clone-ding-zoo/analytics-zoo/scala/dllib/target/bigdl-dllib-2.0.0-SNAPSHOT-python-api.zip -BigDL_JAR_PATH=/home/ding/proj/clone-ding-zoo/analytics-zoo/scala/dllib/bigdl-dllib-2.0.0-SNAPSHOT-jar-with-dependencies.jar +PYTHON_API_ZIP_PATH=/home/ding/proj/clone-ding-zoo/analytics-zoo/dist/lib/bigdl-dllib-2.0.0-SNAPSHOT-python-api.zip +BigDL_JAR_PATH=/home/ding/proj/clone-ding-zoo/analytics-zoo/dist/lib/bigdl-dllib-2.0.0-SNAPSHOT-jar-with-dependencies.jar PYTHONPATH=${PYTHON_API_ZIP_PATH}:$PYTHONPATH ${SPARK_HOME}/bin/spark-submit \ --master ${MASTER} \ @@ -10,9 +10,9 @@ ${SPARK_HOME}/bin/spark-submit \ --total-executor-cores 2 \ --executor-cores 2 \ --executor-memory 4g \ - --py-files ${PYTHON_API_ZIP_PATH},/home/ding/proj/clone-ding-zoo/analytics-zoo/python/dllib/examples/lenet/lenet.py \ + --py-files ${PYTHON_API_ZIP_PATH},/home/ding/proj/clone-ding-zoo/analytics-zoo/python/bigdl/dllib/examples/lenet/lenet.py \ --properties-file /home/ding/proj/clone-ding-zoo/analytics-zoo/scala/dllib/src/main/resources/spark-bigdl.conf \ --jars ${BigDL_JAR_PATH} \ --conf spark.driver.extraClassPath=${BigDL_JAR_PATH} \ - --conf spark.executor.extraClassPath=/home/ding/proj/clone-ding-zoo/analytics-zoo/scala/dllib/bigdl-dllib-2.0.0-SNAPSHOT-jar-with-dependencies.jar \ - /home/ding/proj/clone-ding-zoo/analytics-zoo/python/dllib/examples/lenet/lenet.py + --conf spark.executor.extraClassPath=/home/ding/proj/clone-ding-zoo/analytics-zoo/dist/lib/bigdl-dllib-2.0.0-SNAPSHOT-jar-with-dependencies.jar \ + /home/ding/proj/clone-ding-zoo/analytics-zoo/python/bigdl/dllib/examples/lenet/lenet.py diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/python/api/BigDLSerde.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/python/api/BigDLSerde.scala index 0ddbc005081..62f4a6f49ba 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/python/api/BigDLSerde.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/python/api/BigDLSerde.scala @@ -24,7 +24,7 @@ import java.nio.{ByteBuffer, ByteOrder} import java.util.{ArrayList => JArrayList, HashMap => JHashMap, List => JList, Map => JMap} import com.intel.analytics.bigdl.dllib.nn.abstractnn.Activity -import com.intel.analytics.bigdl.dllib.python.api._ +import com.intel.analytics.bigdl.dllib.utils.python.api._ import com.intel.analytics.bigdl.dllib.utils.Table import net.razorvine.pickle._ import org.apache.spark.api.java.JavaRDD @@ -99,7 +99,7 @@ private[spark] abstract class BigDLSerDeBase { */ object BigDLSerDe extends BigDLSerDeBase with Serializable { - val PYSPARK_PACKAGE = "bigdl.util.common" + val PYSPARK_PACKAGE = "bigdl.utils.common" val LATIN1 = "ISO-8859-1" /** @@ -108,7 +108,7 @@ object BigDLSerDe extends BigDLSerDeBase with Serializable { private[python] abstract class BigDLBasePickler[T: ClassTag] extends IObjectPickler with IObjectConstructor { - val PYSPARK_PACKAGE = "bigdl.util.common" + val PYSPARK_PACKAGE = "bigdl.utils.common" val LATIN1 = "ISO-8859-1" private val cls = implicitly[ClassTag[T]].runtimeClass diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/python/api/PythonBigDL.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/python/api/PythonBigDL.scala index 9b51ae5276d..a0871de9970 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/python/api/PythonBigDL.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/python/api/PythonBigDL.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.dllib.python.api +package com.intel.analytics.bigdl.dllib.utils.python.api import java.util.{ArrayList => JArrayList, HashMap => JHashMap, List => JList, Map => JMap} diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/python/api/PythonBigDLKeras.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/python/api/PythonBigDLKeras.scala index 967bdabe413..167499f9ffa 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/python/api/PythonBigDLKeras.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/python/api/PythonBigDLKeras.scala @@ -1,747 +1,747 @@ -///* -// * Copyright 2016 The BigDL Authors. -// * -// * Licensed under the Apache License, Version 2.0 (the "License"); -// * you may not use this file except in compliance with the License. -// * You may obtain a copy of the License at -// * -// * http://www.apache.org/licenses/LICENSE-2.0 -// * -// * Unless required by applicable law or agreed to in writing, software -// * distributed under the License is distributed on an "AS IS" BASIS, -// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// * See the License for the specific language governing permissions and -// * limitations under the License. -// */ -// -//package com.intel.analytics.bigdl.dllib.python.api -// -//import java.util.{List => JList} -// -//import com.intel.analytics.bigdl.{Criterion, DataSet} -//import com.intel.analytics.bigdl.dllib.feature.dataset.{DataSet, LocalDataSet, MiniBatch} -//import com.intel.analytics.bigdl.dllib.nn.Graph.ModuleNode -//import com.intel.analytics.bigdl.dllib.nn.{Container, SpatialBatchNormalization} -//import com.intel.analytics.bigdl.dllib.nn.abstractnn.{AbstractModule, Activity} -//import com.intel.analytics.bigdl.dllib.keras -//import com.intel.analytics.bigdl.dllib.keras._ -//import com.intel.analytics.bigdl.numeric._ -//import com.intel.analytics.bigdl.dllib.optim.{OptimMethod, Regularizer, ValidationMethod} -//import com.intel.analytics.bigdl.dllib.tensor.Tensor -//import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -//import com.intel.analytics.bigdl.dllib.feature.transform.vision.image.{ImageFeature, ImageFeatureToMiniBatch} -//import com.intel.analytics.bigdl.dllib.utils.{Engine, MultiShape, Shape, SingleShape} -//import org.apache.spark.api.java.JavaRDD -// -//import scala.collection.JavaConverters._ -//import scala.language.existentials -//import scala.reflect.ClassTag -// -// -//object PythonBigDLKeras { -// -// def ofFloat(): PythonBigDLKeras[Float] = new PythonBigDLKeras[Float]() -// -// def ofDouble(): PythonBigDLKeras[Double] = new PythonBigDLKeras[Double]() -//} -// -//class PythonBigDLKeras[T: ClassTag](implicit ev: TensorNumeric[T]) extends PythonBigDL[T] { -// -// def toScalaShape(inputShape: JList[Int]): Shape = { -// if (inputShape == null) { -// null -// } else { -// Shape(inputShape.asScala.toArray) -// } -// } -// -// def toScalaMultiShape(inputShape: JList[JList[Int]]): Shape = { -// if (inputShape == null) { -// null -// } else { -// Shape(inputShape.asScala.toArray.map(shape => Shape(shape.asScala.toArray)).toList) -// } -// } -// -// def toScalaArray(list: JList[Int]): Array[Int] = { -// if (list == null) { -// null -// } else { -// list.asScala.toArray -// } -// } -// -// def createKerasModel(input: JList[ModuleNode[T]], -// output: JList[ModuleNode[T]]): Model[T] = { -// keras.Model(input.asScala.toArray, output.asScala.toArray) -// } -// -// def createKerasSequential(): keras.Sequential[T] = { -// keras.Sequential[T]() -// } -// -// def createKerasInput( -// name : String = null, -// inputShape: JList[Int] = null): ModuleNode[T] = { -// Input(name = name, inputShape = toScalaShape(inputShape)) -// } -// -// def createKerasInputLayer( -// inputShape: JList[Int] = null): KerasLayer[Activity, Activity, T] = { -// InputLayer(inputShape = toScalaShape(inputShape)) -// } -// -// def shapeToJList(shape: Shape): JList[JList[Int]] = { -// val shapes = if (shape.isInstanceOf[SingleShape]) { -// MultiShape(List(shape)) -// } -// else { -// shape -// } -// shapes.toMulti().map(single => single.toSingle().toList.asJava).toList.asJava -// } -// -// def getOutputShape(module: Container[Activity, Activity, T]): JList[JList[Int]] = { -// val output = module.getOutputShape() -// shapeToJList(output) -// } -// -// def getInputShape(module: Container[Activity, Activity, T]): JList[JList[Int]] = { -// val input = module.getInputShape() -// // TODO: inputShape can be nested MultiShape -// shapeToJList(input) -// } -// -// def createKerasDense( -// outputDim: Int, -// init: String = "glorot_uniform", -// activation: String = null, -// wRegularizer: Regularizer[T] = null, -// bRegularizer: Regularizer[T] = null, -// bias: Boolean = true, -// inputShape: JList[Int] = null): Dense[T] = { -// Dense(outputDim, init, activation, wRegularizer, -// bRegularizer, bias, toScalaShape(inputShape)) -// } -// -// def createKerasEmbedding( -// inputDim: Int, -// outputDim: Int, -// init: String = "uniform", -// wRegularizer: Regularizer[T] = null, -// inputShape: JList[Int] = null): Embedding[T] = { -// Embedding[T](inputDim, outputDim, init, wRegularizer, toScalaShape(inputShape)) -// } -// -// def createKerasBatchNormalization( -// epsilon: Double = 0.001, -// momentum: Double = 0.99, -// betaInit: String = "zero", -// gammaInit: String = "one", -// dimOrdering: String = "th", -// inputShape: JList[Int] = null): BatchNormalization[T] = { -// BatchNormalization[T](epsilon, momentum, betaInit, -// gammaInit, dimOrdering, toScalaShape(inputShape)) -// } -// -// def setRunningMean(module: BatchNormalization[T], runningMean: JTensor): Unit = { -// module.labor.asInstanceOf[SpatialBatchNormalization[T]] -// .runningMean.set(toTensor(runningMean)) -// } -// -// def setRunningStd(module: BatchNormalization[T], runningStd: JTensor): Unit = { -// module.labor.asInstanceOf[SpatialBatchNormalization[T]] -// .runningVar.set(toTensor(runningStd)) -// } -// -// def getRunningMean(module: BatchNormalization[T]): JTensor = { -// toJTensor(module.labor.asInstanceOf[SpatialBatchNormalization[T]] -// .runningMean) -// } -// -// def getRunningStd(module: BatchNormalization[T]): JTensor = { -// toJTensor(module.labor.asInstanceOf[SpatialBatchNormalization[T]] -// .runningVar) -// } -// -// def createKerasMerge( -// layers: JList[AbstractModule[Activity, Activity, T]] = null, -// mode: String = "sum", -// concatAxis: Int = -1, -// inputShape: JList[JList[Int]]): Merge[T] = { -// val layersList = if (layers != null) layers.asScala.toList -// else null -// Merge[T](layersList, mode, concatAxis, toScalaMultiShape(inputShape)) -// } -// -// def createKerasConvolution2D( -// nbFilter: Int, -// nbRow: Int, -// nbCol: Int, -// init: String = "glorot_uniform", -// activation: String = null, -// borderMode: String = "valid", -// subsample: JList[Int], -// dimOrdering: String = "th", -// wRegularizer: Regularizer[T] = null, -// bRegularizer: Regularizer[T] = null, -// bias: Boolean = true, -// inputShape: JList[Int] = null): Convolution2D[T] = { -// new Convolution2D(nbFilter, nbRow, nbCol, KerasUtils.getInitMethod(init), -// KerasUtils.getKerasActivation(activation), borderMode, -// toScalaArray(subsample), KerasUtils.toBigDLFormat(dimOrdering), -// wRegularizer, bRegularizer, bias, toScalaShape(inputShape)) -// } -// -// def createKerasMaxPooling2D( -// poolSize: JList[Int], -// strides: JList[Int], -// borderMode: String = "valid", -// dimOrdering: String = "th", -// inputShape: JList[Int] = null): MaxPooling2D[T] = { -// new MaxPooling2D[T](toScalaArray(poolSize), toScalaArray(strides), -// borderMode, KerasUtils.toBigDLFormat(dimOrdering), toScalaShape(inputShape)) -// } -// -// def createKerasActivation( -// activation: String, -// inputShape: JList[Int] = null): Activation[T] = { -// Activation(activation, toScalaShape(inputShape)) -// } -// -// def createKerasReshape( -// targetShape: JList[Int], -// inputShape: JList[Int] = null): Reshape[T] = { -// Reshape(toScalaArray(targetShape), toScalaShape(inputShape)) -// } -// -// def createKerasDropout( -// p: Double, -// inputShape: JList[Int] = null): Dropout[T] = { -// Dropout(p, toScalaShape(inputShape)) -// } -// -// def createKerasFlatten( -// inputShape: JList[Int] = null): Flatten[T] = { -// Flatten(toScalaShape(inputShape)) -// } -// -// def createKerasSimpleRNN( -// outputDim: Int, -// activation: String = "tanh", -// returnSequences: Boolean = false, -// goBackwards: Boolean = false, -// wRegularizer: Regularizer[T] = null, -// uRegularizer: Regularizer[T] = null, -// bRegularizer: Regularizer[T] = null, -// inputShape: JList[Int] = null): SimpleRNN[T] = { -// SimpleRNN(outputDim, activation, returnSequences, goBackwards, -// wRegularizer, uRegularizer, bRegularizer, toScalaShape(inputShape)) -// } -// -// def createKerasLSTM( -// outputDim: Int, -// activation: String = "tanh", -// innerActivation: String = "hard_sigmoid", -// returnSequences: Boolean = false, -// goBackwards: Boolean = false, -// wRegularizer: Regularizer[T] = null, -// uRegularizer: Regularizer[T] = null, -// bRegularizer: Regularizer[T] = null, -// inputShape: JList[Int] = null): LSTM[T] = { -// LSTM(outputDim, activation, innerActivation, returnSequences, -// goBackwards, wRegularizer, uRegularizer, bRegularizer, toScalaShape(inputShape)) -// } -// -// def createKerasGRU( -// outputDim: Int, -// activation: String = "tanh", -// innerActivation: String = "hard_sigmoid", -// returnSequences: Boolean = false, -// goBackwards: Boolean = false, -// wRegularizer: Regularizer[T] = null, -// uRegularizer: Regularizer[T] = null, -// bRegularizer: Regularizer[T] = null, -// inputShape: JList[Int] = null): GRU[T] = { -// GRU(outputDim, activation, innerActivation, returnSequences, -// goBackwards, wRegularizer, uRegularizer, bRegularizer, toScalaShape(inputShape)) -// } -// -// def createKerasHighway( -// activation: String = null, -// wRegularizer: Regularizer[T] = null, -// bRegularizer: Regularizer[T] = null, -// bias: Boolean = true, -// inputShape: JList[Int] = null): Highway[T] = { -// Highway(activation, wRegularizer, bRegularizer, bias, toScalaShape(inputShape)) -// } -// -// def createKerasZeroPadding1D( -// padding: JList[Int], -// inputShape: JList[Int] = null): ZeroPadding1D[T] = { -// new ZeroPadding1D(toScalaArray(padding), toScalaShape(inputShape)) -// } -// -// def createKerasZeroPadding2D( -// padding: JList[Int], -// dimOrdering: String = "th", -// inputShape: JList[Int] = null): ZeroPadding2D[T] = { -// new ZeroPadding2D(toScalaArray(padding), -// KerasUtils.toBigDLFormat(dimOrdering), toScalaShape(inputShape)) -// } -// -// def createKerasUpSampling1D( -// length: Int = 2, -// inputShape: JList[Int] = null): UpSampling1D[T] = { -// UpSampling1D(length, toScalaShape(inputShape)) -// } -// -// def createKerasUpSampling2D( -// size: JList[Int], -// dimOrdering: String = "th", -// inputShape: JList[Int] = null): UpSampling2D[T] = { -// new UpSampling2D(toScalaArray(size), KerasUtils.toBigDLFormat(dimOrdering), -// toScalaShape(inputShape)) -// } -// -// def createKerasUpSampling3D( -// size: JList[Int], -// dimOrdering: String = "th", -// inputShape: JList[Int] = null): UpSampling3D[T] = { -// new UpSampling3D(toScalaArray(size), KerasUtils.toBigDLFormat5D(dimOrdering), -// toScalaShape(inputShape)) -// } -// -// def createKerasMaxoutDense( -// outputDim: Int, -// nbFeature: Int = 4, -// wRegularizer: Regularizer[T] = null, -// bRegularizer: Regularizer[T] = null, -// bias: Boolean = true, -// inputShape: JList[Int] = null): MaxoutDense[T] = { -// MaxoutDense(outputDim, nbFeature, wRegularizer, -// bRegularizer, bias, toScalaShape(inputShape)) -// } -// -// def createKerasConvolution1D( -// nbFilter: Int, -// filterLength: Int, -// init: String = "glorot_uniform", -// activation: String = null, -// borderMode: String = "valid", -// subsampleLength: Int = 1, -// wRegularizer: Regularizer[T] = null, -// bRegularizer: Regularizer[T] = null, -// bias: Boolean = true, -// inputShape: JList[Int] = null): Convolution1D[T] = { -// Convolution1D(nbFilter, filterLength, init, activation, borderMode, -// subsampleLength, wRegularizer, bRegularizer, bias, toScalaShape(inputShape)) -// } -// -// def createKerasConvolution3D( -// nbFilter: Int, -// kernelDim1: Int, -// kernelDim2: Int, -// kernelDim3: Int, -// init: String = "glorot_uniform", -// activation: String = null, -// borderMode: String = "valid", -// subsample: JList[Int], -// dimOrdering: String = "th", -// wRegularizer: Regularizer[T] = null, -// bRegularizer: Regularizer[T] = null, -// bias: Boolean = true, -// inputShape: JList[Int] = null): Convolution3D[T] = { -// new Convolution3D(nbFilter, kernelDim1, kernelDim2, kernelDim3, -// KerasUtils.getInitMethod(init), KerasUtils.getKerasActivation(activation), -// borderMode, toScalaArray(subsample), KerasUtils.toBigDLFormat5D(dimOrdering), -// wRegularizer, bRegularizer, bias, toScalaShape(inputShape)) -// } -// -// def createKerasMaxPooling1D( -// poolLength: Int = 2, -// stride: Int = -1, -// borderMode: String = "valid", -// inputShape: JList[Int] = null): MaxPooling1D[T] = { -// MaxPooling1D(poolLength, stride, borderMode, toScalaShape(inputShape)) -// } -// -// def createKerasMaxPooling3D( -// poolSize: JList[Int], -// strides: JList[Int], -// dimOrdering: String = "th", -// inputShape: JList[Int] = null): MaxPooling3D[T] = { -// new MaxPooling3D(toScalaArray(poolSize), toScalaArray(strides), -// KerasUtils.toBigDLFormat5D(dimOrdering), toScalaShape(inputShape)) -// } -// -// def createKerasAveragePooling1D( -// poolLength: Int = 2, -// stride: Int = -1, -// borderMode: String = "valid", -// inputShape: JList[Int] = null): AveragePooling1D[T] = { -// AveragePooling1D(poolLength, stride, borderMode, toScalaShape(inputShape)) -// } -// -// def createKerasAveragePooling2D( -// poolSize: JList[Int], -// strides: JList[Int], -// borderMode: String = "valid", -// dimOrdering: String = "th", -// inputShape: JList[Int] = null): AveragePooling2D[T] = { -// new AveragePooling2D(toScalaArray(poolSize), toScalaArray(strides), -// borderMode, KerasUtils.toBigDLFormat(dimOrdering), toScalaShape(inputShape)) -// } -// -// def createKerasAveragePooling3D( -// poolSize: JList[Int], -// strides: JList[Int], -// dimOrdering: String = "th", -// inputShape: JList[Int] = null): AveragePooling3D[T] = { -// new AveragePooling3D(toScalaArray(poolSize), toScalaArray(strides), -// KerasUtils.toBigDLFormat5D(dimOrdering), toScalaShape(inputShape)) -// } -// -// def createKerasGlobalAveragePooling2D( -// dimOrdering: String = "th", -// inputShape: JList[Int] = null): GlobalAveragePooling2D[T] = { -// GlobalAveragePooling2D(dimOrdering, toScalaShape(inputShape)) -// } -// -// def createKerasGlobalMaxPooling2D( -// dimOrdering: String = "th", -// inputShape: JList[Int] = null): GlobalMaxPooling2D[T] = { -// GlobalMaxPooling2D(dimOrdering, toScalaShape(inputShape)) -// } -// -// def createKerasRepeatVector( -// n: Int, -// inputShape: JList[Int] = null): RepeatVector[T] = { -// RepeatVector(n, toScalaShape(inputShape)) -// } -// -// def createKerasPermute( -// dims: JList[Int], -// inputShape: JList[Int] = null): Permute[T] = { -// Permute(toScalaArray(dims), toScalaShape(inputShape)) -// } -// -// def createKerasCropping1D( -// cropping: JList[Int], -// inputShape: JList[Int] = null): Cropping1D[T] = { -// new Cropping1D(toScalaArray(cropping), toScalaShape(inputShape)) -// } -// -// def createKerasCropping2D( -// heightCrop: JList[Int], -// widthCrop: JList[Int], -// dimOrdering: String = "th", -// inputShape: JList[Int] = null): Cropping2D[T] = { -// new Cropping2D(toScalaArray(heightCrop), toScalaArray(widthCrop), -// KerasUtils.toBigDLFormat(dimOrdering), toScalaShape(inputShape)) -// } -// -// def createKerasCropping3D( -// dim1Crop: JList[Int], -// dim2Crop: JList[Int], -// dim3Crop: JList[Int], -// dimOrdering: String = "th", -// inputShape: JList[Int] = null): Cropping3D[T] = { -// new Cropping3D(toScalaArray(dim1Crop), toScalaArray(dim2Crop), toScalaArray(dim3Crop), -// KerasUtils.toBigDLFormat5D(dimOrdering), toScalaShape(inputShape)) -// } -// -// def createKerasAtrousConvolution1D( -// nbFilter: Int, -// filterLength: Int, -// init: String = "glorot_uniform", -// activation: String = null, -// subsampleLength: Int = 1, -// atrousRate: Int = 1, -// wRegularizer: Regularizer[T] = null, -// bRegularizer: Regularizer[T] = null, -// inputShape: JList[Int] = null): AtrousConvolution1D[T] = { -// AtrousConvolution1D(nbFilter, filterLength, init, activation, -// subsampleLength, atrousRate, wRegularizer, bRegularizer, toScalaShape(inputShape)) -// } -// -// def createKerasAtrousConvolution2D( -// nbFilter: Int, -// nbRow: Int, -// nbCol: Int, -// init: String = "glorot_uniform", -// activation: String = null, -// subsample: JList[Int], -// atrousRate: JList[Int], -// dimOrdering: String = "th", -// wRegularizer: Regularizer[T] = null, -// bRegularizer: Regularizer[T] = null, -// inputShape: JList[Int] = null): AtrousConvolution2D[T] = { -// new AtrousConvolution2D(nbFilter, nbRow, nbCol, KerasUtils.getInitMethod(init), -// KerasUtils.getKerasActivation(activation), toScalaArray(subsample), -// toScalaArray(atrousRate), KerasUtils.toBigDLFormat(dimOrdering), -// wRegularizer, bRegularizer, toScalaShape(inputShape)) -// } -// -// def createKerasDeconvolution2D( -// nbFilter: Int, -// nbRow: Int, -// nbCol: Int, -// init: String = "glorot_uniform", -// activation: String = null, -// subsample: JList[Int], -// dimOrdering: String = "th", -// wRegularizer: Regularizer[T] = null, -// bRegularizer: Regularizer[T] = null, -// bias: Boolean = true, -// inputShape: JList[Int] = null): Deconvolution2D[T] = { -// new Deconvolution2D(nbFilter, nbRow, nbCol, KerasUtils.getInitMethod(init), -// KerasUtils.getKerasActivation(activation), toScalaArray(subsample), -// KerasUtils.toBigDLFormat(dimOrdering), wRegularizer, bRegularizer, -// bias, toScalaShape(inputShape)) -// } -// -// def createKerasConvLSTM2D( -// nbFilter: Int, -// nbKernel: Int, -// activation: String = "tanh", -// innerActivation: String = "hard_sigmoid", -// dimOrdering: String = "th", -// subsample: Int = 1, -// wRegularizer: Regularizer[T] = null, -// uRegularizer: Regularizer[T] = null, -// bRegularizer: Regularizer[T] = null, -// returnSequences: Boolean = false, -// goBackwards: Boolean = false, -// inputShape: JList[Int] = null): ConvLSTM2D[T] = { -// ConvLSTM2D(nbFilter, nbKernel, activation, innerActivation, -// dimOrdering, subsample, wRegularizer, uRegularizer, bRegularizer, -// returnSequences, goBackwards, toScalaShape(inputShape)) -// } -// -// def createKerasLocallyConnected1D( -// nbFilter: Int, -// filterLength: Int, -// activation: String = null, -// subsampleLength: Int = 1, -// wRegularizer: Regularizer[T] = null, -// bRegularizer: Regularizer[T] = null, -// bias: Boolean = true, -// inputShape: JList[Int] = null): LocallyConnected1D[T] = { -// LocallyConnected1D(nbFilter, filterLength, activation, subsampleLength, -// wRegularizer, bRegularizer, bias, toScalaShape(inputShape)) -// } -// -// def createKerasLocallyConnected2D( -// nbFilter: Int, -// nbRow: Int, -// nbCol: Int, -// activation: String = null, -// borderMode: String = "valid", -// subsample: JList[Int], -// dimOrdering: String = "th", -// wRegularizer: Regularizer[T] = null, -// bRegularizer: Regularizer[T] = null, -// bias: Boolean = true, -// inputShape: JList[Int] = null): LocallyConnected2D[T] = { -// new LocallyConnected2D(nbFilter, nbRow, nbCol, KerasUtils.getKerasActivation(activation), -// borderMode, toScalaArray(subsample), KerasUtils.toBigDLFormat(dimOrdering), -// wRegularizer, bRegularizer, bias, toScalaShape(inputShape)) -// } -// -// def createKerasSeparableConvolution2D( -// nbFilter: Int, -// nbRow: Int, -// nbCol: Int, -// init: String = "glorot_uniform", -// activation: String = null, -// borderMode: String = "valid", -// subsample: JList[Int], -// depthMultiplier: Int = 1, -// dimOrdering: String = "th", -// depthwiseRegularizer: Regularizer[T] = null, -// pointwiseRegularizer: Regularizer[T] = null, -// bRegularizer: Regularizer[T] = null, -// bias: Boolean = true, -// inputShape: JList[Int] = null): SeparableConvolution2D[T] = { -// new SeparableConvolution2D(nbFilter, nbRow, nbCol, KerasUtils.getInitMethod(init), -// KerasUtils.getKerasActivation(activation), borderMode, toScalaArray(subsample), -// depthMultiplier, KerasUtils.toBigDLFormat(dimOrdering), -// depthwiseRegularizer, pointwiseRegularizer, bRegularizer, bias, toScalaShape(inputShape)) -// } -// -// def createKerasZeroPadding3D( -// padding: JList[Int], -// dimOrdering: String = "th", -// inputShape: JList[Int] = null): ZeroPadding3D[T] = { -// new ZeroPadding3D(toScalaArray(padding), KerasUtils.toBigDLFormat5D(dimOrdering), -// toScalaShape(inputShape)) -// } -// -// def createKerasGlobalAveragePooling1D( -// inputShape: JList[Int] = null): GlobalAveragePooling1D[T] = { -// GlobalAveragePooling1D(toScalaShape(inputShape)) -// } -// -// def createKerasGlobalMaxPooling1D( -// inputShape: JList[Int] = null): GlobalMaxPooling1D[T] = { -// GlobalMaxPooling1D(toScalaShape(inputShape)) -// } -// -// def createKerasGlobalMaxPooling3D( -// dimOrdering: String = "th", -// inputShape: JList[Int] = null): GlobalMaxPooling3D[T] = { -// GlobalMaxPooling3D(dimOrdering, toScalaShape(inputShape)) -// } -// -// def createKerasGlobalAveragePooling3D( -// dimOrdering: String = "th", -// inputShape: JList[Int] = null): GlobalAveragePooling3D[T] = { -// GlobalAveragePooling3D(dimOrdering, toScalaShape(inputShape)) -// } -// -// def createKerasSpatialDropout1D( -// p: Double = 0.5, -// inputShape: JList[Int] = null): SpatialDropout1D[T] = { -// SpatialDropout1D(p, toScalaShape(inputShape)) -// } -// -// def createKerasSpatialDropout2D( -// p: Double = 0.5, -// dimOrdering: String = "th", -// inputShape: JList[Int] = null): SpatialDropout2D[T] = { -// SpatialDropout2D(p, dimOrdering, toScalaShape(inputShape)) -// } -// -// def createKerasSpatialDropout3D( -// p: Double = 0.5, -// dimOrdering: String = "th", -// inputShape: JList[Int] = null): SpatialDropout3D[T] = { -// SpatialDropout3D(p, dimOrdering, toScalaShape(inputShape)) -// } -// -// def createKerasGaussianDropout( -// p: Double, -// inputShape: JList[Int] = null): GaussianDropout[T] = { -// GaussianDropout(p, toScalaShape(inputShape)) -// } -// -// def createKerasGaussianNoise( -// sigma: Double, -// inputShape: JList[Int] = null): GaussianNoise[T] = { -// GaussianNoise(sigma, toScalaShape(inputShape)) -// } -// -// def createKerasMasking( -// maskValue: Double = 0.0, -// inputShape: JList[Int] = null): Masking[T] = { -// Masking(maskValue, toScalaShape(inputShape)) -// } -// -// def createKerasSReLU( -// tLeftInit: String = "zero", -// aLeftInit: String = "glorot_uniform", -// tRightInit: String = "glorot_uniform", -// aRightInit: String = "one", -// sharedAxes: JList[Int] = null, -// inputShape: JList[Int] = null): SReLU[T] = { -// SReLU(tLeftInit, aLeftInit, tRightInit, aRightInit, -// toScalaArray(sharedAxes), toScalaShape(inputShape)) -// } -// -// def createKerasELU( -// alpha: Double = 1.0, -// inputShape: JList[Int] = null): ELU[T] = { -// ELU(alpha, toScalaShape(inputShape)) -// } -// -// def createKerasLeakyReLU( -// alpha: Double = 0.01, -// inputShape: JList[Int] = null): LeakyReLU[T] = { -// LeakyReLU(alpha, toScalaShape(inputShape)) -// } -// -// def createKerasThresholdedReLU( -// theta: Double = 1.0, -// inputShape: JList[Int] = null): ThresholdedReLU[T] = { -// ThresholdedReLU(theta, toScalaShape(inputShape)) -// } -// -// def createKerasTimeDistributed( -// layer: KerasLayer[Tensor[T], Tensor[T], T], -// inputShape: JList[Int] = null): TimeDistributed[T] = { -// TimeDistributed(layer, toScalaShape(inputShape)) -// } -// -// def createKerasBidirectional( -// layer: Recurrent[T], -// mergeMode: String = "concat", -// inputShape: JList[Int] = null): Bidirectional[T] = { -// Bidirectional(layer, mergeMode, toScalaShape(inputShape)) -// } -// -// def compile( -// module: KerasModel[T], -// optimizer: OptimMethod[T], -// loss: Criterion[T], -// metrics: JList[ValidationMethod[T]] = null): Unit = { -// module.compile(optimizer, loss, -// if (metrics == null) null else metrics.asScala.toArray) -// } -// -// def fit( -// module: KerasModel[T], -// x: JavaRDD[Sample], -// batchSize: Int = 32, -// epochs: Int = 10, -// validationData: JavaRDD[Sample] = null): Unit = { -// module.fit(toJSample(x), batchSize, epochs, -// if (validationData == null) null else toJSample(validationData)) -// } -// -// def fit( -// module: KerasModel[T], -// x: DataSet[ImageFeature], -// batchSize: Int, -// epochs: Int, -// validationData: DataSet[ImageFeature]): Unit = { -// val trainData = x -> ImageFeatureToMiniBatch[T](batchSize) -// val valData = -// if (validationData != null) validationData -> ImageFeatureToMiniBatch[T](batchSize) -// else null -// module.fit(trainData, epochs, valData) -// } -// -// def fit( -// module: KerasModel[T], -// xTrain: JList[JTensor], -// yTrain: JTensor, -// batchSize: Int, -// epochs: Int, -// xVal: JList[JTensor], -// yVal: JTensor, -// localCores: Int): Unit = { -// val trainArray = toSampleArray(xTrain.asScala.toList.map{f => toTensor(f)}, toTensor(yTrain)) -// val trainData = batching(DataSet.array(trainArray), batchSize) -// .asInstanceOf[LocalDataSet[MiniBatch[T]]] -// val valData = if (xVal != null && yVal != null) { -// val valArray = toSampleArray(xVal.asScala.toList.map{f => toTensor(f)}, toTensor(yVal)) -// batching(DataSet.array(valArray), batchSize) -// } else null -// Engine.setNodeAndCore(1, localCores) -// module.fit(trainData, epochs, valData) -// } -// -// def evaluate( -// module: KerasModel[T], -// x: JavaRDD[Sample], -// batchSize: Int = 32): JList[EvaluatedResult] = { -// val resultArray = module.evaluate(toJSample(x), batchSize) -// val testResultArray = resultArray.map { result => -// EvaluatedResult(result._1.result()._1, result._1.result()._2, -// result._2.toString()) -// } -// testResultArray.toList.asJava -// } -// -//} +/* + * Copyright 2016 The BigDL Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.intel.analytics.bigdl.dllib.utils.python.api + +import java.util.{List => JList} + +import com.intel.analytics.bigdl.{Criterion, DataSet} +import com.intel.analytics.bigdl.dllib.feature.dataset.{DataSet, LocalDataSet, MiniBatch} +import com.intel.analytics.bigdl.dllib.nn.Graph.ModuleNode +import com.intel.analytics.bigdl.dllib.nn.{Container, SpatialBatchNormalization} +import com.intel.analytics.bigdl.dllib.nn.abstractnn.{AbstractModule, Activity} +import com.intel.analytics.bigdl.dllib.keras +import com.intel.analytics.bigdl.dllib.keras._ +import com.intel.analytics.bigdl.numeric._ +import com.intel.analytics.bigdl.dllib.optim.{OptimMethod, Regularizer, ValidationMethod} +import com.intel.analytics.bigdl.dllib.tensor.Tensor +import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric +import com.intel.analytics.bigdl.dllib.feature.transform.vision.image.{ImageFeature, ImageFeatureToMiniBatch} +import com.intel.analytics.bigdl.utils.{Engine, MultiShape, Shape, SingleShape} +import org.apache.spark.api.java.JavaRDD + +import scala.collection.JavaConverters._ +import scala.language.existentials +import scala.reflect.ClassTag + + +object PythonBigDLKeras { + + def ofFloat(): PythonBigDLKeras[Float] = new PythonBigDLKeras[Float]() + + def ofDouble(): PythonBigDLKeras[Double] = new PythonBigDLKeras[Double]() +} + +class PythonBigDLKeras[T: ClassTag](implicit ev: TensorNumeric[T]) extends PythonBigDL[T] { + + def toScalaShape(inputShape: JList[Int]): Shape = { + if (inputShape == null) { + null + } else { + Shape(inputShape.asScala.toArray) + } + } + + def toScalaMultiShape(inputShape: JList[JList[Int]]): Shape = { + if (inputShape == null) { + null + } else { + Shape(inputShape.asScala.toArray.map(shape => Shape(shape.asScala.toArray)).toList) + } + } + + def toScalaArray(list: JList[Int]): Array[Int] = { + if (list == null) { + null + } else { + list.asScala.toArray + } + } + + def createKerasModel(input: JList[ModuleNode[T]], + output: JList[ModuleNode[T]]): Model[T] = { + keras.Model(input.asScala.toArray, output.asScala.toArray) + } + + def createKerasSequential(): keras.Sequential[T] = { + keras.Sequential[T]() + } + + def createKerasInput( + name : String = null, + inputShape: JList[Int] = null): ModuleNode[T] = { + Input(name = name, inputShape = toScalaShape(inputShape)) + } + + def createKerasInputLayer( + inputShape: JList[Int] = null): KerasLayer[Activity, Activity, T] = { + InputLayer(inputShape = toScalaShape(inputShape)) + } + + def shapeToJList(shape: Shape): JList[JList[Int]] = { + val shapes = if (shape.isInstanceOf[SingleShape]) { + MultiShape(List(shape)) + } + else { + shape + } + shapes.toMulti().map(single => single.toSingle().toList.asJava).toList.asJava + } + + def getOutputShape(module: Container[Activity, Activity, T]): JList[JList[Int]] = { + val output = module.getOutputShape() + shapeToJList(output) + } + + def getInputShape(module: Container[Activity, Activity, T]): JList[JList[Int]] = { + val input = module.getInputShape() + // TODO: inputShape can be nested MultiShape + shapeToJList(input) + } + + def createKerasDense( + outputDim: Int, + init: String = "glorot_uniform", + activation: String = null, + wRegularizer: Regularizer[T] = null, + bRegularizer: Regularizer[T] = null, + bias: Boolean = true, + inputShape: JList[Int] = null): Dense[T] = { + Dense(outputDim, init, activation, wRegularizer, + bRegularizer, bias, toScalaShape(inputShape)) + } + + def createKerasEmbedding( + inputDim: Int, + outputDim: Int, + init: String = "uniform", + wRegularizer: Regularizer[T] = null, + inputShape: JList[Int] = null): Embedding[T] = { + Embedding[T](inputDim, outputDim, init, wRegularizer, toScalaShape(inputShape)) + } + + def createKerasBatchNormalization( + epsilon: Double = 0.001, + momentum: Double = 0.99, + betaInit: String = "zero", + gammaInit: String = "one", + dimOrdering: String = "th", + inputShape: JList[Int] = null): BatchNormalization[T] = { + BatchNormalization[T](epsilon, momentum, betaInit, + gammaInit, dimOrdering, toScalaShape(inputShape)) + } + + def setRunningMean(module: BatchNormalization[T], runningMean: JTensor): Unit = { + module.labor.asInstanceOf[SpatialBatchNormalization[T]] + .runningMean.set(toTensor(runningMean)) + } + + def setRunningStd(module: BatchNormalization[T], runningStd: JTensor): Unit = { + module.labor.asInstanceOf[SpatialBatchNormalization[T]] + .runningVar.set(toTensor(runningStd)) + } + + def getRunningMean(module: BatchNormalization[T]): JTensor = { + toJTensor(module.labor.asInstanceOf[SpatialBatchNormalization[T]] + .runningMean) + } + + def getRunningStd(module: BatchNormalization[T]): JTensor = { + toJTensor(module.labor.asInstanceOf[SpatialBatchNormalization[T]] + .runningVar) + } + + def createKerasMerge( + layers: JList[AbstractModule[Activity, Activity, T]] = null, + mode: String = "sum", + concatAxis: Int = -1, + inputShape: JList[JList[Int]]): Merge[T] = { + val layersList = if (layers != null) layers.asScala.toList + else null + Merge[T](layersList, mode, concatAxis, toScalaMultiShape(inputShape)) + } + + def createKerasConvolution2D( + nbFilter: Int, + nbRow: Int, + nbCol: Int, + init: String = "glorot_uniform", + activation: String = null, + borderMode: String = "valid", + subsample: JList[Int], + dimOrdering: String = "th", + wRegularizer: Regularizer[T] = null, + bRegularizer: Regularizer[T] = null, + bias: Boolean = true, + inputShape: JList[Int] = null): Convolution2D[T] = { + new Convolution2D(nbFilter, nbRow, nbCol, KerasUtils.getInitMethod(init), + KerasUtils.getKerasActivation(activation), borderMode, + toScalaArray(subsample), KerasUtils.toBigDLFormat(dimOrdering), + wRegularizer, bRegularizer, bias, toScalaShape(inputShape)) + } + + def createKerasMaxPooling2D( + poolSize: JList[Int], + strides: JList[Int], + borderMode: String = "valid", + dimOrdering: String = "th", + inputShape: JList[Int] = null): MaxPooling2D[T] = { + new MaxPooling2D[T](toScalaArray(poolSize), toScalaArray(strides), + borderMode, KerasUtils.toBigDLFormat(dimOrdering), toScalaShape(inputShape)) + } + + def createKerasActivation( + activation: String, + inputShape: JList[Int] = null): Activation[T] = { + Activation(activation, toScalaShape(inputShape)) + } + + def createKerasReshape( + targetShape: JList[Int], + inputShape: JList[Int] = null): Reshape[T] = { + Reshape(toScalaArray(targetShape), toScalaShape(inputShape)) + } + + def createKerasDropout( + p: Double, + inputShape: JList[Int] = null): Dropout[T] = { + Dropout(p, toScalaShape(inputShape)) + } + + def createKerasFlatten( + inputShape: JList[Int] = null): Flatten[T] = { + Flatten(toScalaShape(inputShape)) + } + + def createKerasSimpleRNN( + outputDim: Int, + activation: String = "tanh", + returnSequences: Boolean = false, + goBackwards: Boolean = false, + wRegularizer: Regularizer[T] = null, + uRegularizer: Regularizer[T] = null, + bRegularizer: Regularizer[T] = null, + inputShape: JList[Int] = null): SimpleRNN[T] = { + SimpleRNN(outputDim, activation, returnSequences, goBackwards, + wRegularizer, uRegularizer, bRegularizer, toScalaShape(inputShape)) + } + + def createKerasLSTM( + outputDim: Int, + activation: String = "tanh", + innerActivation: String = "hard_sigmoid", + returnSequences: Boolean = false, + goBackwards: Boolean = false, + wRegularizer: Regularizer[T] = null, + uRegularizer: Regularizer[T] = null, + bRegularizer: Regularizer[T] = null, + inputShape: JList[Int] = null): LSTM[T] = { + LSTM(outputDim, activation, innerActivation, returnSequences, + goBackwards, wRegularizer, uRegularizer, bRegularizer, toScalaShape(inputShape)) + } + + def createKerasGRU( + outputDim: Int, + activation: String = "tanh", + innerActivation: String = "hard_sigmoid", + returnSequences: Boolean = false, + goBackwards: Boolean = false, + wRegularizer: Regularizer[T] = null, + uRegularizer: Regularizer[T] = null, + bRegularizer: Regularizer[T] = null, + inputShape: JList[Int] = null): GRU[T] = { + GRU(outputDim, activation, innerActivation, returnSequences, + goBackwards, wRegularizer, uRegularizer, bRegularizer, toScalaShape(inputShape)) + } + + def createKerasHighway( + activation: String = null, + wRegularizer: Regularizer[T] = null, + bRegularizer: Regularizer[T] = null, + bias: Boolean = true, + inputShape: JList[Int] = null): Highway[T] = { + Highway(activation, wRegularizer, bRegularizer, bias, toScalaShape(inputShape)) + } + + def createKerasZeroPadding1D( + padding: JList[Int], + inputShape: JList[Int] = null): ZeroPadding1D[T] = { + new ZeroPadding1D(toScalaArray(padding), toScalaShape(inputShape)) + } + + def createKerasZeroPadding2D( + padding: JList[Int], + dimOrdering: String = "th", + inputShape: JList[Int] = null): ZeroPadding2D[T] = { + new ZeroPadding2D(toScalaArray(padding), + KerasUtils.toBigDLFormat(dimOrdering), toScalaShape(inputShape)) + } + + def createKerasUpSampling1D( + length: Int = 2, + inputShape: JList[Int] = null): UpSampling1D[T] = { + UpSampling1D(length, toScalaShape(inputShape)) + } + + def createKerasUpSampling2D( + size: JList[Int], + dimOrdering: String = "th", + inputShape: JList[Int] = null): UpSampling2D[T] = { + new UpSampling2D(toScalaArray(size), KerasUtils.toBigDLFormat(dimOrdering), + toScalaShape(inputShape)) + } + + def createKerasUpSampling3D( + size: JList[Int], + dimOrdering: String = "th", + inputShape: JList[Int] = null): UpSampling3D[T] = { + new UpSampling3D(toScalaArray(size), KerasUtils.toBigDLFormat5D(dimOrdering), + toScalaShape(inputShape)) + } + + def createKerasMaxoutDense( + outputDim: Int, + nbFeature: Int = 4, + wRegularizer: Regularizer[T] = null, + bRegularizer: Regularizer[T] = null, + bias: Boolean = true, + inputShape: JList[Int] = null): MaxoutDense[T] = { + MaxoutDense(outputDim, nbFeature, wRegularizer, + bRegularizer, bias, toScalaShape(inputShape)) + } + + def createKerasConvolution1D( + nbFilter: Int, + filterLength: Int, + init: String = "glorot_uniform", + activation: String = null, + borderMode: String = "valid", + subsampleLength: Int = 1, + wRegularizer: Regularizer[T] = null, + bRegularizer: Regularizer[T] = null, + bias: Boolean = true, + inputShape: JList[Int] = null): Convolution1D[T] = { + Convolution1D(nbFilter, filterLength, init, activation, borderMode, + subsampleLength, wRegularizer, bRegularizer, bias, toScalaShape(inputShape)) + } + + def createKerasConvolution3D( + nbFilter: Int, + kernelDim1: Int, + kernelDim2: Int, + kernelDim3: Int, + init: String = "glorot_uniform", + activation: String = null, + borderMode: String = "valid", + subsample: JList[Int], + dimOrdering: String = "th", + wRegularizer: Regularizer[T] = null, + bRegularizer: Regularizer[T] = null, + bias: Boolean = true, + inputShape: JList[Int] = null): Convolution3D[T] = { + new Convolution3D(nbFilter, kernelDim1, kernelDim2, kernelDim3, + KerasUtils.getInitMethod(init), KerasUtils.getKerasActivation(activation), + borderMode, toScalaArray(subsample), KerasUtils.toBigDLFormat5D(dimOrdering), + wRegularizer, bRegularizer, bias, toScalaShape(inputShape)) + } + + def createKerasMaxPooling1D( + poolLength: Int = 2, + stride: Int = -1, + borderMode: String = "valid", + inputShape: JList[Int] = null): MaxPooling1D[T] = { + MaxPooling1D(poolLength, stride, borderMode, toScalaShape(inputShape)) + } + + def createKerasMaxPooling3D( + poolSize: JList[Int], + strides: JList[Int], + dimOrdering: String = "th", + inputShape: JList[Int] = null): MaxPooling3D[T] = { + new MaxPooling3D(toScalaArray(poolSize), toScalaArray(strides), + KerasUtils.toBigDLFormat5D(dimOrdering), toScalaShape(inputShape)) + } + + def createKerasAveragePooling1D( + poolLength: Int = 2, + stride: Int = -1, + borderMode: String = "valid", + inputShape: JList[Int] = null): AveragePooling1D[T] = { + AveragePooling1D(poolLength, stride, borderMode, toScalaShape(inputShape)) + } + + def createKerasAveragePooling2D( + poolSize: JList[Int], + strides: JList[Int], + borderMode: String = "valid", + dimOrdering: String = "th", + inputShape: JList[Int] = null): AveragePooling2D[T] = { + new AveragePooling2D(toScalaArray(poolSize), toScalaArray(strides), + borderMode, KerasUtils.toBigDLFormat(dimOrdering), toScalaShape(inputShape)) + } + + def createKerasAveragePooling3D( + poolSize: JList[Int], + strides: JList[Int], + dimOrdering: String = "th", + inputShape: JList[Int] = null): AveragePooling3D[T] = { + new AveragePooling3D(toScalaArray(poolSize), toScalaArray(strides), + KerasUtils.toBigDLFormat5D(dimOrdering), toScalaShape(inputShape)) + } + + def createKerasGlobalAveragePooling2D( + dimOrdering: String = "th", + inputShape: JList[Int] = null): GlobalAveragePooling2D[T] = { + GlobalAveragePooling2D(dimOrdering, toScalaShape(inputShape)) + } + + def createKerasGlobalMaxPooling2D( + dimOrdering: String = "th", + inputShape: JList[Int] = null): GlobalMaxPooling2D[T] = { + GlobalMaxPooling2D(dimOrdering, toScalaShape(inputShape)) + } + + def createKerasRepeatVector( + n: Int, + inputShape: JList[Int] = null): RepeatVector[T] = { + RepeatVector(n, toScalaShape(inputShape)) + } + + def createKerasPermute( + dims: JList[Int], + inputShape: JList[Int] = null): Permute[T] = { + Permute(toScalaArray(dims), toScalaShape(inputShape)) + } + + def createKerasCropping1D( + cropping: JList[Int], + inputShape: JList[Int] = null): Cropping1D[T] = { + new Cropping1D(toScalaArray(cropping), toScalaShape(inputShape)) + } + + def createKerasCropping2D( + heightCrop: JList[Int], + widthCrop: JList[Int], + dimOrdering: String = "th", + inputShape: JList[Int] = null): Cropping2D[T] = { + new Cropping2D(toScalaArray(heightCrop), toScalaArray(widthCrop), + KerasUtils.toBigDLFormat(dimOrdering), toScalaShape(inputShape)) + } + + def createKerasCropping3D( + dim1Crop: JList[Int], + dim2Crop: JList[Int], + dim3Crop: JList[Int], + dimOrdering: String = "th", + inputShape: JList[Int] = null): Cropping3D[T] = { + new Cropping3D(toScalaArray(dim1Crop), toScalaArray(dim2Crop), toScalaArray(dim3Crop), + KerasUtils.toBigDLFormat5D(dimOrdering), toScalaShape(inputShape)) + } + + def createKerasAtrousConvolution1D( + nbFilter: Int, + filterLength: Int, + init: String = "glorot_uniform", + activation: String = null, + subsampleLength: Int = 1, + atrousRate: Int = 1, + wRegularizer: Regularizer[T] = null, + bRegularizer: Regularizer[T] = null, + inputShape: JList[Int] = null): AtrousConvolution1D[T] = { + AtrousConvolution1D(nbFilter, filterLength, init, activation, + subsampleLength, atrousRate, wRegularizer, bRegularizer, toScalaShape(inputShape)) + } + + def createKerasAtrousConvolution2D( + nbFilter: Int, + nbRow: Int, + nbCol: Int, + init: String = "glorot_uniform", + activation: String = null, + subsample: JList[Int], + atrousRate: JList[Int], + dimOrdering: String = "th", + wRegularizer: Regularizer[T] = null, + bRegularizer: Regularizer[T] = null, + inputShape: JList[Int] = null): AtrousConvolution2D[T] = { + new AtrousConvolution2D(nbFilter, nbRow, nbCol, KerasUtils.getInitMethod(init), + KerasUtils.getKerasActivation(activation), toScalaArray(subsample), + toScalaArray(atrousRate), KerasUtils.toBigDLFormat(dimOrdering), + wRegularizer, bRegularizer, toScalaShape(inputShape)) + } + + def createKerasDeconvolution2D( + nbFilter: Int, + nbRow: Int, + nbCol: Int, + init: String = "glorot_uniform", + activation: String = null, + subsample: JList[Int], + dimOrdering: String = "th", + wRegularizer: Regularizer[T] = null, + bRegularizer: Regularizer[T] = null, + bias: Boolean = true, + inputShape: JList[Int] = null): Deconvolution2D[T] = { + new Deconvolution2D(nbFilter, nbRow, nbCol, KerasUtils.getInitMethod(init), + KerasUtils.getKerasActivation(activation), toScalaArray(subsample), + KerasUtils.toBigDLFormat(dimOrdering), wRegularizer, bRegularizer, + bias, toScalaShape(inputShape)) + } + + def createKerasConvLSTM2D( + nbFilter: Int, + nbKernel: Int, + activation: String = "tanh", + innerActivation: String = "hard_sigmoid", + dimOrdering: String = "th", + subsample: Int = 1, + wRegularizer: Regularizer[T] = null, + uRegularizer: Regularizer[T] = null, + bRegularizer: Regularizer[T] = null, + returnSequences: Boolean = false, + goBackwards: Boolean = false, + inputShape: JList[Int] = null): ConvLSTM2D[T] = { + ConvLSTM2D(nbFilter, nbKernel, activation, innerActivation, + dimOrdering, subsample, wRegularizer, uRegularizer, bRegularizer, + returnSequences, goBackwards, toScalaShape(inputShape)) + } + + def createKerasLocallyConnected1D( + nbFilter: Int, + filterLength: Int, + activation: String = null, + subsampleLength: Int = 1, + wRegularizer: Regularizer[T] = null, + bRegularizer: Regularizer[T] = null, + bias: Boolean = true, + inputShape: JList[Int] = null): LocallyConnected1D[T] = { + LocallyConnected1D(nbFilter, filterLength, activation, subsampleLength, + wRegularizer, bRegularizer, bias, toScalaShape(inputShape)) + } + + def createKerasLocallyConnected2D( + nbFilter: Int, + nbRow: Int, + nbCol: Int, + activation: String = null, + borderMode: String = "valid", + subsample: JList[Int], + dimOrdering: String = "th", + wRegularizer: Regularizer[T] = null, + bRegularizer: Regularizer[T] = null, + bias: Boolean = true, + inputShape: JList[Int] = null): LocallyConnected2D[T] = { + new LocallyConnected2D(nbFilter, nbRow, nbCol, KerasUtils.getKerasActivation(activation), + borderMode, toScalaArray(subsample), KerasUtils.toBigDLFormat(dimOrdering), + wRegularizer, bRegularizer, bias, toScalaShape(inputShape)) + } + + def createKerasSeparableConvolution2D( + nbFilter: Int, + nbRow: Int, + nbCol: Int, + init: String = "glorot_uniform", + activation: String = null, + borderMode: String = "valid", + subsample: JList[Int], + depthMultiplier: Int = 1, + dimOrdering: String = "th", + depthwiseRegularizer: Regularizer[T] = null, + pointwiseRegularizer: Regularizer[T] = null, + bRegularizer: Regularizer[T] = null, + bias: Boolean = true, + inputShape: JList[Int] = null): SeparableConvolution2D[T] = { + new SeparableConvolution2D(nbFilter, nbRow, nbCol, KerasUtils.getInitMethod(init), + KerasUtils.getKerasActivation(activation), borderMode, toScalaArray(subsample), + depthMultiplier, KerasUtils.toBigDLFormat(dimOrdering), + depthwiseRegularizer, pointwiseRegularizer, bRegularizer, bias, toScalaShape(inputShape)) + } + + def createKerasZeroPadding3D( + padding: JList[Int], + dimOrdering: String = "th", + inputShape: JList[Int] = null): ZeroPadding3D[T] = { + new ZeroPadding3D(toScalaArray(padding), KerasUtils.toBigDLFormat5D(dimOrdering), + toScalaShape(inputShape)) + } + + def createKerasGlobalAveragePooling1D( + inputShape: JList[Int] = null): GlobalAveragePooling1D[T] = { + GlobalAveragePooling1D(toScalaShape(inputShape)) + } + + def createKerasGlobalMaxPooling1D( + inputShape: JList[Int] = null): GlobalMaxPooling1D[T] = { + GlobalMaxPooling1D(toScalaShape(inputShape)) + } + + def createKerasGlobalMaxPooling3D( + dimOrdering: String = "th", + inputShape: JList[Int] = null): GlobalMaxPooling3D[T] = { + GlobalMaxPooling3D(dimOrdering, toScalaShape(inputShape)) + } + + def createKerasGlobalAveragePooling3D( + dimOrdering: String = "th", + inputShape: JList[Int] = null): GlobalAveragePooling3D[T] = { + GlobalAveragePooling3D(dimOrdering, toScalaShape(inputShape)) + } + + def createKerasSpatialDropout1D( + p: Double = 0.5, + inputShape: JList[Int] = null): SpatialDropout1D[T] = { + SpatialDropout1D(p, toScalaShape(inputShape)) + } + + def createKerasSpatialDropout2D( + p: Double = 0.5, + dimOrdering: String = "th", + inputShape: JList[Int] = null): SpatialDropout2D[T] = { + SpatialDropout2D(p, dimOrdering, toScalaShape(inputShape)) + } + + def createKerasSpatialDropout3D( + p: Double = 0.5, + dimOrdering: String = "th", + inputShape: JList[Int] = null): SpatialDropout3D[T] = { + SpatialDropout3D(p, dimOrdering, toScalaShape(inputShape)) + } + + def createKerasGaussianDropout( + p: Double, + inputShape: JList[Int] = null): GaussianDropout[T] = { + GaussianDropout(p, toScalaShape(inputShape)) + } + + def createKerasGaussianNoise( + sigma: Double, + inputShape: JList[Int] = null): GaussianNoise[T] = { + GaussianNoise(sigma, toScalaShape(inputShape)) + } + + def createKerasMasking( + maskValue: Double = 0.0, + inputShape: JList[Int] = null): Masking[T] = { + Masking(maskValue, toScalaShape(inputShape)) + } + + def createKerasSReLU( + tLeftInit: String = "zero", + aLeftInit: String = "glorot_uniform", + tRightInit: String = "glorot_uniform", + aRightInit: String = "one", + sharedAxes: JList[Int] = null, + inputShape: JList[Int] = null): SReLU[T] = { + SReLU(tLeftInit, aLeftInit, tRightInit, aRightInit, + toScalaArray(sharedAxes), toScalaShape(inputShape)) + } + + def createKerasELU( + alpha: Double = 1.0, + inputShape: JList[Int] = null): ELU[T] = { + ELU(alpha, toScalaShape(inputShape)) + } + + def createKerasLeakyReLU( + alpha: Double = 0.01, + inputShape: JList[Int] = null): LeakyReLU[T] = { + LeakyReLU(alpha, toScalaShape(inputShape)) + } + + def createKerasThresholdedReLU( + theta: Double = 1.0, + inputShape: JList[Int] = null): ThresholdedReLU[T] = { + ThresholdedReLU(theta, toScalaShape(inputShape)) + } + + def createKerasTimeDistributed( + layer: KerasLayer[Tensor[T], Tensor[T], T], + inputShape: JList[Int] = null): TimeDistributed[T] = { + TimeDistributed(layer, toScalaShape(inputShape)) + } + + def createKerasBidirectional( + layer: Recurrent[T], + mergeMode: String = "concat", + inputShape: JList[Int] = null): Bidirectional[T] = { + Bidirectional(layer, mergeMode, toScalaShape(inputShape)) + } + + def compile( + module: KerasModel[T], + optimizer: OptimMethod[T], + loss: Criterion[T], + metrics: JList[ValidationMethod[T]] = null): Unit = { + module.compile(optimizer, loss, + if (metrics == null) null else metrics.asScala.toArray) + } + + def fit( + module: KerasModel[T], + x: JavaRDD[Sample], + batchSize: Int = 32, + epochs: Int = 10, + validationData: JavaRDD[Sample] = null): Unit = { + module.fit(toJSample(x), batchSize, epochs, + if (validationData == null) null else toJSample(validationData)) + } + + def fit( + module: KerasModel[T], + x: DataSet[ImageFeature], + batchSize: Int, + epochs: Int, + validationData: DataSet[ImageFeature]): Unit = { + val trainData = x -> ImageFeatureToMiniBatch[T](batchSize) + val valData = + if (validationData != null) validationData -> ImageFeatureToMiniBatch[T](batchSize) + else null + module.fit(trainData, epochs, valData) + } + + def fit( + module: KerasModel[T], + xTrain: JList[JTensor], + yTrain: JTensor, + batchSize: Int, + epochs: Int, + xVal: JList[JTensor], + yVal: JTensor, + localCores: Int): Unit = { + val trainArray = toSampleArray(xTrain.asScala.toList.map{f => toTensor(f)}, toTensor(yTrain)) + val trainData = batching(DataSet.array(trainArray), batchSize) + .asInstanceOf[LocalDataSet[MiniBatch[T]]] + val valData = if (xVal != null && yVal != null) { + val valArray = toSampleArray(xVal.asScala.toList.map{f => toTensor(f)}, toTensor(yVal)) + batching(DataSet.array(valArray), batchSize) + } else null + Engine.setNodeAndCore(1, localCores) + module.fit(trainData, epochs, valData) + } + + def evaluate( + module: KerasModel[T], + x: JavaRDD[Sample], + batchSize: Int = 32): JList[EvaluatedResult] = { + val resultArray = module.evaluate(toJSample(x), batchSize) + val testResultArray = resultArray.map { result => + EvaluatedResult(result._1.result()._1, result._1.result()._2, + result._2.toString()) + } + testResultArray.toList.asJava + } + +} diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/python/api/PythonBigDLOnnx.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/python/api/PythonBigDLOnnx.scala index e755b7972f0..444fa2cb0b7 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/python/api/PythonBigDLOnnx.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/python/api/PythonBigDLOnnx.scala @@ -1,67 +1,67 @@ -///* -// * Copyright 2016 The BigDL Authors. -// * -// * Licensed under the Apache License, Version 2.0 (the "License"); -// * you may not use this file except in compliance with the License. -// * You may obtain a copy of the License at -// * -// * http://www.apache.org/licenses/LICENSE-2.0 -// * -// * Unless required by applicable law or agreed to in writing, software -// * distributed under the License is distributed on an "AS IS" BASIS, -// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// * See the License for the specific language governing permissions and -// * limitations under the License. -// */ -// -//package com.intel.analytics.bigdl.dllib.python.api -// -//import scala.reflect.ClassTag -//import scala.collection.JavaConverters._ -//import java.util.{ArrayList => JArrayList, HashMap => JHashMap, List => JList, Map => JMap} -// -//import com.intel.analytics.bigdl.dllib.nn -//import com.intel.analytics.bigdl.dllib.nn.onnx._ -//import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -// -// -//private[bigdl] object PythonBigDLOnnx { -// -// def ofFloat(): PythonBigDLOnnx[Float] = new PythonBigDLOnnx[Float]() -// -// def ofDouble(): PythonBigDLOnnx[Double] = new PythonBigDLOnnx[Double]() -// -//} -// -// -//class PythonBigDLOnnx[T: ClassTag](implicit ev: TensorNumeric[T]) extends PythonBigDL[T] { -// -// def createConstant(value: JTensor): nn.tf.Const[T, T] = { -// nn.tf.Const[T, T](toTensor(value)) -// } -// -// -// def createGather(): nn.ops.Gather[T, T] = { -// nn.ops.Gather() -// } -// -// -// def createGemm(alpha: Float, beta: Float, transA: Int, transB: Int, -// matrixB: JTensor, matrixC: JTensor): Gemm[T] = { -// Gemm(alpha, beta, -// (if (transA == 0) false else true), -// (if (transB == 0) false else true), -// toTensor(matrixB), toTensor(matrixC)) -// } -// -// -// def createReshape(shape: JArrayList[Int]): nn.onnx.Reshape[T] = { -// nn.onnx.Reshape(if (shape == null) null else shape.asScala.toArray) -// } -// -// -// def createShape(): Shape[T] = { -// Shape[T]() -// } -// -//} +/* + * Copyright 2016 The BigDL Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.intel.analytics.bigdl.dllib.utils.python.api + +import scala.reflect.ClassTag +import scala.collection.JavaConverters._ +import java.util.{ArrayList => JArrayList, HashMap => JHashMap, List => JList, Map => JMap} + +import com.intel.analytics.bigdl.dllib.nn +import com.intel.analytics.bigdl.dllib.nn.onnx._ +import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric + + +private[bigdl] object PythonBigDLOnnx { + + def ofFloat(): PythonBigDLOnnx[Float] = new PythonBigDLOnnx[Float]() + + def ofDouble(): PythonBigDLOnnx[Double] = new PythonBigDLOnnx[Double]() + +} + + +class PythonBigDLOnnx[T: ClassTag](implicit ev: TensorNumeric[T]) extends PythonBigDL[T] { + + def createConstant(value: JTensor): nn.tf.Const[T, T] = { + nn.tf.Const[T, T](toTensor(value)) + } + + + def createGather(): nn.ops.Gather[T, T] = { + nn.ops.Gather() + } + + + def createGemm(alpha: Float, beta: Float, transA: Int, transB: Int, + matrixB: JTensor, matrixC: JTensor): Gemm[T] = { + Gemm(alpha, beta, + (if (transA == 0) false else true), + (if (transB == 0) false else true), + toTensor(matrixB), toTensor(matrixC)) + } + + + def createReshape(shape: JArrayList[Int]): nn.onnx.Reshape[T] = { + nn.onnx.Reshape(if (shape == null) null else shape.asScala.toArray) + } + + + def createShape(): Shape[T] = { + Shape[T]() + } + +} diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/python/api/PythonBigDLValidator.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/python/api/PythonBigDLValidator.scala index e2c4a236fe9..58a654cbb15 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/python/api/PythonBigDLValidator.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/python/api/PythonBigDLValidator.scala @@ -1,81 +1,81 @@ -///* -// * Copyright 2016 The BigDL Authors. -// * -// * Licensed under the Apache License, Version 2.0 (the "License"); -// * you may not use this file except in compliance with the License. -// * You may obtain a copy of the License at -// * -// * http://www.apache.org/licenses/LICENSE-2.0 -// * -// * Unless required by applicable law or agreed to in writing, software -// * distributed under the License is distributed on an "AS IS" BASIS, -// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// * See the License for the specific language governing permissions and -// * limitations under the License. -// */ -// -//package com.intel.analytics.bigdl.dllib.python.api -// -//import java.lang.{Boolean => JBoolean} -//import java.util.{ArrayList => JArrayList, HashMap => JHashMap, List => JList, Map => JMap} -// -//import com.intel.analytics.bigdl.dllib.tensor.Tensor -//import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -//import com.intel.analytics.bigdl.dllib.utils.Table -// -//import scala.collection.JavaConverters._ -//import scala.collection.mutable.Map -//import scala.language.existentials -//import scala.reflect.ClassTag -// -//object PythonBigDLValidator { -// -// def ofFloat(): PythonBigDLValidator[Float] = new PythonBigDLValidator[Float]() -// -// def ofDouble(): PythonBigDLValidator[Double] = new PythonBigDLValidator[Double]() -//} -// -//class PythonBigDLValidator[T: ClassTag](implicit ev: TensorNumeric[T]) extends PythonBigDL[T]{ -// -// def testDict(): JMap[String, String] = { -// return Map("jack" -> "40", "lucy" -> "50").asJava -// } -// -// def testDictJTensor(): JMap[String, JTensor] = { -// return Map("jack" -> JTensor(Array(1.0f, 2.0f, 3.0f, 4.0f), Array(4, 1), "float")).asJava -// } -// -// def testDictJMapJTensor(): JMap[String, JMap[String, JTensor]] = { -// val table = new Table() -// val tensor = JTensor(Array(1.0f, 2.0f, 3.0f, 4.0f), Array(4, 1), "float") -// val result = Map("jack" -> tensor).asJava -// table.insert(tensor) -// return Map("nested" -> result).asJava -// } -// -// def testActivityWithTensor(): JActivity = { -// val tensor = Tensor(Array(1.0f, 2.0f, 3.0f, 4.0f), Array(4, 1)) -// return JActivity(tensor) -// } -// -// def testActivityWithTableOfTensor(): JActivity = { -// val tensor1 = Tensor(Array(1.0f, 1.0f), Array(2)) -// val tensor2 = Tensor(Array(2.0f, 2.0f), Array(2)) -// val tensor3 = Tensor(Array(3.0f, 3.0f), Array(2)) -// val table = new Table() -// table.insert(tensor1) -// table.insert(tensor2) -// table.insert(tensor3) -// return JActivity(table) -// } -// -// def testActivityWithTableOfTable(): JActivity = { -// val tensor = Tensor(Array(1.0f, 2.0f, 3.0f, 4.0f), Array(4, 1)) -// val table = new Table() -// table.insert(tensor) -// val nestedTable = new Table() -// nestedTable.insert(table) -// nestedTable.insert(table) -// return JActivity(nestedTable) -// } -//} +/* + * Copyright 2016 The BigDL Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.intel.analytics.bigdl.dllib.utils.python.api + +import java.lang.{Boolean => JBoolean} +import java.util.{ArrayList => JArrayList, HashMap => JHashMap, List => JList, Map => JMap} + +import com.intel.analytics.bigdl.dllib.tensor.Tensor +import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric +import com.intel.analytics.bigdl.dllib.utils.Table + +import scala.collection.JavaConverters._ +import scala.collection.mutable.Map +import scala.language.existentials +import scala.reflect.ClassTag + +object PythonBigDLValidator { + + def ofFloat(): PythonBigDLValidator[Float] = new PythonBigDLValidator[Float]() + + def ofDouble(): PythonBigDLValidator[Double] = new PythonBigDLValidator[Double]() +} + +class PythonBigDLValidator[T: ClassTag](implicit ev: TensorNumeric[T]) extends PythonBigDL[T]{ + + def testDict(): JMap[String, String] = { + return Map("jack" -> "40", "lucy" -> "50").asJava + } + + def testDictJTensor(): JMap[String, JTensor] = { + return Map("jack" -> JTensor(Array(1.0f, 2.0f, 3.0f, 4.0f), Array(4, 1), "float")).asJava + } + + def testDictJMapJTensor(): JMap[String, JMap[String, JTensor]] = { + val table = new Table() + val tensor = JTensor(Array(1.0f, 2.0f, 3.0f, 4.0f), Array(4, 1), "float") + val result = Map("jack" -> tensor).asJava + table.insert(tensor) + return Map("nested" -> result).asJava + } + + def testActivityWithTensor(): JActivity = { + val tensor = Tensor(Array(1.0f, 2.0f, 3.0f, 4.0f), Array(4, 1)) + return JActivity(tensor) + } + + def testActivityWithTableOfTensor(): JActivity = { + val tensor1 = Tensor(Array(1.0f, 1.0f), Array(2)) + val tensor2 = Tensor(Array(2.0f, 2.0f), Array(2)) + val tensor3 = Tensor(Array(3.0f, 3.0f), Array(2)) + val table = new Table() + table.insert(tensor1) + table.insert(tensor2) + table.insert(tensor3) + return JActivity(table) + } + + def testActivityWithTableOfTable(): JActivity = { + val tensor = Tensor(Array(1.0f, 2.0f, 3.0f, 4.0f), Array(4, 1)) + val table = new Table() + table.insert(tensor) + val nestedTable = new Table() + nestedTable.insert(table) + nestedTable.insert(table) + return JActivity(nestedTable) + } +} diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/tf/TensorflowLoader.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/tf/TensorflowLoader.scala index 54c910b9358..aa312c5e734 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/tf/TensorflowLoader.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/tf/TensorflowLoader.scala @@ -26,7 +26,7 @@ import com.intel.analytics.bigdl.Module import com.intel.analytics.bigdl.dllib.nn.Graph import com.intel.analytics.bigdl.dllib.nn.abstractnn.{AbstractModule, Activity} import com.intel.analytics.bigdl.dllib.nn.tf.AssignGrad -import com.intel.analytics.bigdl.dllib.python.api.{JTensor, PythonBigDL, PythonBigDLUtils} +import com.intel.analytics.bigdl.dllib.utils.python.api.{JTensor, PythonBigDL, PythonBigDLUtils} import com.intel.analytics.bigdl.dllib.tensor.{DoubleType, FloatType, Tensor} import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric import com.intel.analytics.bigdl.dllib.nn.tf.{SwitchControlNode, SwitchOps} diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/python/api/PythonSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/python/api/PythonSpec.scala index b34cbec6b01..afa4469e53f 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/python/api/PythonSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/python/api/PythonSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.dllib.python.api +package com.intel.analytics.bigdl.dllib.utils.python.api import java.util import java.util.{ArrayList => JArrayList, List => JList, Map => JMap} From a1e4ce23985420d032e48135d0dc9f6bfd897e31 Mon Sep 17 00:00:00 2001 From: dding3 Date: Mon, 16 Aug 2021 14:43:41 -0700 Subject: [PATCH 798/823] fix run example failure --- python/.idea/workspace.xml | 780 ++++----- .../bigdl/dllib/bigdlkeras/layers/__init__.py | 5 + python/dllib/src/bigdl/utils/common.py | 6 +- python/release.sh | 27 +- python/run_lenet.sh | 10 +- .../dllib/utils/python/api/BigDLSerde.scala | 6 +- .../dllib/utils/python/api/PythonBigDL.scala | 2 +- .../utils/python/api/PythonBigDLKeras.scala | 1494 ++++++++--------- .../utils/python/api/PythonBigDLOnnx.scala | 134 +- .../python/api/PythonBigDLValidator.scala | 162 +- .../dllib/utils/tf/TensorflowLoader.scala | 2 +- .../bigdl/dllib/python/api/PythonSpec.scala | 2 +- 12 files changed, 1287 insertions(+), 1343 deletions(-) diff --git a/python/.idea/workspace.xml b/python/.idea/workspace.xml index 4f6b228c869..af60d1ae9da 100644 --- a/python/.idea/workspace.xml +++ b/python/.idea/workspace.xml @@ -12,35 +12,31 @@ - - + + - - - - - + + + - - + + - - - - - + + + - - + + - - + + @@ -48,8 +44,8 @@ - - + + @@ -58,69 +54,72 @@ - - + + - - + + - + + + + + + + - - + + - - - - - + + + - - + + - - + + - + + - - + + - - - - - + + + - - + + - - + + - - + + - - + + @@ -130,10 +129,6 @@ - lenet.py - lenet.p - b - bigd bigdl bigdl.d bigdl.da @@ -160,6 +155,10 @@ bigdl.nn bigdl.models bigdl. + jar_dir + get_bigdl_classpath + spark_profile + make BIGDL_HOME @@ -180,7 +179,6 @@ @@ -243,10 +247,68 @@ - + + + + + + + + + + + + + + + - - + + - + @@ -426,105 +608,129 @@ - + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - - + + - + - - + + - + - + - - + + - + - - + + - + - - - + - + @@ -532,65 +738,31 @@ - + - - - - - - - - - - - - - - - - - - + + - + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - + - - + + - + @@ -598,299 +770,79 @@ - - - - - - - - + - - - - - - - - - - - - - - - - - - - - - - - - - - + + - - - - - - - - - - - - - - - - - - - - - + - - + + - - - - - - - - - - - - - - - - - - - - - + - - - - - - - - - - - - - - - + + + - + - - + + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - - + + - - - - - - - - - - - + - - + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - - + + - - - - - - - - - - - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/python/dllib/src/bigdl/dllib/bigdlkeras/layers/__init__.py b/python/dllib/src/bigdl/dllib/bigdlkeras/layers/__init__.py index 2151a805423..9eb670a1780 100644 --- a/python/dllib/src/bigdl/dllib/bigdlkeras/layers/__init__.py +++ b/python/dllib/src/bigdl/dllib/bigdlkeras/layers/__init__.py @@ -13,3 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. # +from bigdl.utils.common import * + +init_engine() +redire_spark_logs() +show_bigdl_info_logs() diff --git a/python/dllib/src/bigdl/utils/common.py b/python/dllib/src/bigdl/utils/common.py index 684fd22ad60..af7a03cb89f 100644 --- a/python/dllib/src/bigdl/utils/common.py +++ b/python/dllib/src/bigdl/utils/common.py @@ -32,7 +32,7 @@ import threading import tempfile import traceback -from bigdl.util.engine import get_bigdl_classpath, is_spark_below_2_2 +from bigdl.utils.engine import get_bigdl_classpath, is_spark_below_2_2 INTMAX = 2147483647 INTMIN = -2147483648 @@ -65,8 +65,8 @@ def __init__(self, bigdl_type, port=25333): class JavaCreator(SingletonMixin): __creator_class=[ - "com.intel.analytics.bigdl.python.api.PythonBigDLKeras", - "com.intel.analytics.bigdl.python.api.PythonBigDLOnnx" + "com.intel.analytics.bigdl.dllib.utils.python.api.PythonBigDLKeras", + "com.intel.analytics.bigdl.dllib.utils.python.api.PythonBigDLOnnx" ] @classmethod diff --git a/python/release.sh b/python/release.sh index 306a23c41c0..e070f510b47 100755 --- a/python/release.sh +++ b/python/release.sh @@ -19,9 +19,9 @@ set -e RUN_SCRIPT_DIR=$(cd $(dirname $0) ; pwd) echo $RUN_SCRIPT_DIR -BIGDL_DIR="$(cd ${RUN_SCRIPT_DIR}/../../../../; pwd)" +BIGDL_DIR="$(cd ${RUN_SCRIPT_DIR}/../; pwd)" echo $BIGDL_DIR -BIGDL_PYTHON_DIR="$(cd ${RUN_SCRIPT_DIR}/../../../../pyspark; pwd)" +BIGDL_PYTHON_DIR="$(cd ${RUN_SCRIPT_DIR}/../python; pwd)" echo $BIGDL_PYTHON_DIR if (( $# < 2)); then @@ -32,13 +32,7 @@ fi platform=$1 spark_profile=$2 quick=$3 -input_version=$4 -bigdl_version=$(python -c "exec(open('$BIGDL_DIR/pyspark/bigdl/version.py').read()); print(__version__)") - -if [ "$input_version" != "$bigdl_version" ]; then - echo "Not the proposed version: $bigdl_version" - exit -1 -fi +bigdl_version=$(python -c "exec(open('$BIGDL_DIR/python/bigdl/version.py').read()); print(__version__)") cd ${BIGDL_DIR} if [ "$platform" == "mac" ]; then @@ -53,7 +47,7 @@ else echo "unsupport platform" fi -bigdl_build_command="${BIGDL_DIR}/make-dist.sh ${dist_profile}" +bigdl_build_command="${BIGDL_DIR}/scala/make-dist.sh ${dist_profile}" if [ "$quick" == "true" ]; then echo "Skip disting BigDL" else @@ -66,18 +60,11 @@ sdist_command="python setup.py sdist" echo "packing source code: ${sdist_command}" $sdist_command -if [ -d "${BIGDL_DIR}/pyspark/build" ]; then - rm -r ${BIGDL_DIR}/pyspark/build -fi - -if [ -d "${BIGDL_DIR}/pyspark/dist" ]; then - rm -r ${BIGDL_DIR}/pyspark/dist -fi wheel_command="python setup.py bdist_wheel --plat-name ${verbose_pname}" echo "Packing python distribution: $wheel_command" ${wheel_command} -upload_command="twine upload dist/BigDL-${bigdl_version}-py2.py3-none-${verbose_pname}.whl" -echo "Please manually upload with this command: $upload_command" +#upload_command="twine upload dist/BigDL-${bigdl_version}-py2.py3-none-${verbose_pname}.whl" +#echo "Please manually upload with this command: $upload_command" -$upload_command +#$upload_command diff --git a/python/run_lenet.sh b/python/run_lenet.sh index 4b5764cba94..fd45228b04a 100755 --- a/python/run_lenet.sh +++ b/python/run_lenet.sh @@ -1,7 +1,7 @@ SPARK_HOME=/home/ding/Downloads/spark-2.4.3-bin-hadoop2.7 MASTER=local[2] -PYTHON_API_ZIP_PATH=/home/ding/proj/clone-ding-zoo/analytics-zoo/scala/dllib/target/bigdl-dllib-2.0.0-SNAPSHOT-python-api.zip -BigDL_JAR_PATH=/home/ding/proj/clone-ding-zoo/analytics-zoo/scala/dllib/bigdl-dllib-2.0.0-SNAPSHOT-jar-with-dependencies.jar +PYTHON_API_ZIP_PATH=/home/ding/proj/clone-ding-zoo/analytics-zoo/dist/lib/bigdl-dllib-2.0.0-SNAPSHOT-python-api.zip +BigDL_JAR_PATH=/home/ding/proj/clone-ding-zoo/analytics-zoo/dist/lib/bigdl-dllib-2.0.0-SNAPSHOT-jar-with-dependencies.jar PYTHONPATH=${PYTHON_API_ZIP_PATH}:$PYTHONPATH ${SPARK_HOME}/bin/spark-submit \ --master ${MASTER} \ @@ -10,9 +10,9 @@ ${SPARK_HOME}/bin/spark-submit \ --total-executor-cores 2 \ --executor-cores 2 \ --executor-memory 4g \ - --py-files ${PYTHON_API_ZIP_PATH},/home/ding/proj/clone-ding-zoo/analytics-zoo/python/dllib/examples/lenet/lenet.py \ + --py-files ${PYTHON_API_ZIP_PATH},/home/ding/proj/clone-ding-zoo/analytics-zoo/python/bigdl/dllib/examples/lenet/lenet.py \ --properties-file /home/ding/proj/clone-ding-zoo/analytics-zoo/scala/dllib/src/main/resources/spark-bigdl.conf \ --jars ${BigDL_JAR_PATH} \ --conf spark.driver.extraClassPath=${BigDL_JAR_PATH} \ - --conf spark.executor.extraClassPath=/home/ding/proj/clone-ding-zoo/analytics-zoo/scala/dllib/bigdl-dllib-2.0.0-SNAPSHOT-jar-with-dependencies.jar \ - /home/ding/proj/clone-ding-zoo/analytics-zoo/python/dllib/examples/lenet/lenet.py + --conf spark.executor.extraClassPath=/home/ding/proj/clone-ding-zoo/analytics-zoo/dist/lib/bigdl-dllib-2.0.0-SNAPSHOT-jar-with-dependencies.jar \ + /home/ding/proj/clone-ding-zoo/analytics-zoo/python/bigdl/dllib/examples/lenet/lenet.py diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/python/api/BigDLSerde.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/python/api/BigDLSerde.scala index 0ddbc005081..62f4a6f49ba 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/python/api/BigDLSerde.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/python/api/BigDLSerde.scala @@ -24,7 +24,7 @@ import java.nio.{ByteBuffer, ByteOrder} import java.util.{ArrayList => JArrayList, HashMap => JHashMap, List => JList, Map => JMap} import com.intel.analytics.bigdl.dllib.nn.abstractnn.Activity -import com.intel.analytics.bigdl.dllib.python.api._ +import com.intel.analytics.bigdl.dllib.utils.python.api._ import com.intel.analytics.bigdl.dllib.utils.Table import net.razorvine.pickle._ import org.apache.spark.api.java.JavaRDD @@ -99,7 +99,7 @@ private[spark] abstract class BigDLSerDeBase { */ object BigDLSerDe extends BigDLSerDeBase with Serializable { - val PYSPARK_PACKAGE = "bigdl.util.common" + val PYSPARK_PACKAGE = "bigdl.utils.common" val LATIN1 = "ISO-8859-1" /** @@ -108,7 +108,7 @@ object BigDLSerDe extends BigDLSerDeBase with Serializable { private[python] abstract class BigDLBasePickler[T: ClassTag] extends IObjectPickler with IObjectConstructor { - val PYSPARK_PACKAGE = "bigdl.util.common" + val PYSPARK_PACKAGE = "bigdl.utils.common" val LATIN1 = "ISO-8859-1" private val cls = implicitly[ClassTag[T]].runtimeClass diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/python/api/PythonBigDL.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/python/api/PythonBigDL.scala index 9b51ae5276d..a0871de9970 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/python/api/PythonBigDL.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/python/api/PythonBigDL.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.dllib.python.api +package com.intel.analytics.bigdl.dllib.utils.python.api import java.util.{ArrayList => JArrayList, HashMap => JHashMap, List => JList, Map => JMap} diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/python/api/PythonBigDLKeras.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/python/api/PythonBigDLKeras.scala index 967bdabe413..167499f9ffa 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/python/api/PythonBigDLKeras.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/python/api/PythonBigDLKeras.scala @@ -1,747 +1,747 @@ -///* -// * Copyright 2016 The BigDL Authors. -// * -// * Licensed under the Apache License, Version 2.0 (the "License"); -// * you may not use this file except in compliance with the License. -// * You may obtain a copy of the License at -// * -// * http://www.apache.org/licenses/LICENSE-2.0 -// * -// * Unless required by applicable law or agreed to in writing, software -// * distributed under the License is distributed on an "AS IS" BASIS, -// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// * See the License for the specific language governing permissions and -// * limitations under the License. -// */ -// -//package com.intel.analytics.bigdl.dllib.python.api -// -//import java.util.{List => JList} -// -//import com.intel.analytics.bigdl.{Criterion, DataSet} -//import com.intel.analytics.bigdl.dllib.feature.dataset.{DataSet, LocalDataSet, MiniBatch} -//import com.intel.analytics.bigdl.dllib.nn.Graph.ModuleNode -//import com.intel.analytics.bigdl.dllib.nn.{Container, SpatialBatchNormalization} -//import com.intel.analytics.bigdl.dllib.nn.abstractnn.{AbstractModule, Activity} -//import com.intel.analytics.bigdl.dllib.keras -//import com.intel.analytics.bigdl.dllib.keras._ -//import com.intel.analytics.bigdl.numeric._ -//import com.intel.analytics.bigdl.dllib.optim.{OptimMethod, Regularizer, ValidationMethod} -//import com.intel.analytics.bigdl.dllib.tensor.Tensor -//import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -//import com.intel.analytics.bigdl.dllib.feature.transform.vision.image.{ImageFeature, ImageFeatureToMiniBatch} -//import com.intel.analytics.bigdl.dllib.utils.{Engine, MultiShape, Shape, SingleShape} -//import org.apache.spark.api.java.JavaRDD -// -//import scala.collection.JavaConverters._ -//import scala.language.existentials -//import scala.reflect.ClassTag -// -// -//object PythonBigDLKeras { -// -// def ofFloat(): PythonBigDLKeras[Float] = new PythonBigDLKeras[Float]() -// -// def ofDouble(): PythonBigDLKeras[Double] = new PythonBigDLKeras[Double]() -//} -// -//class PythonBigDLKeras[T: ClassTag](implicit ev: TensorNumeric[T]) extends PythonBigDL[T] { -// -// def toScalaShape(inputShape: JList[Int]): Shape = { -// if (inputShape == null) { -// null -// } else { -// Shape(inputShape.asScala.toArray) -// } -// } -// -// def toScalaMultiShape(inputShape: JList[JList[Int]]): Shape = { -// if (inputShape == null) { -// null -// } else { -// Shape(inputShape.asScala.toArray.map(shape => Shape(shape.asScala.toArray)).toList) -// } -// } -// -// def toScalaArray(list: JList[Int]): Array[Int] = { -// if (list == null) { -// null -// } else { -// list.asScala.toArray -// } -// } -// -// def createKerasModel(input: JList[ModuleNode[T]], -// output: JList[ModuleNode[T]]): Model[T] = { -// keras.Model(input.asScala.toArray, output.asScala.toArray) -// } -// -// def createKerasSequential(): keras.Sequential[T] = { -// keras.Sequential[T]() -// } -// -// def createKerasInput( -// name : String = null, -// inputShape: JList[Int] = null): ModuleNode[T] = { -// Input(name = name, inputShape = toScalaShape(inputShape)) -// } -// -// def createKerasInputLayer( -// inputShape: JList[Int] = null): KerasLayer[Activity, Activity, T] = { -// InputLayer(inputShape = toScalaShape(inputShape)) -// } -// -// def shapeToJList(shape: Shape): JList[JList[Int]] = { -// val shapes = if (shape.isInstanceOf[SingleShape]) { -// MultiShape(List(shape)) -// } -// else { -// shape -// } -// shapes.toMulti().map(single => single.toSingle().toList.asJava).toList.asJava -// } -// -// def getOutputShape(module: Container[Activity, Activity, T]): JList[JList[Int]] = { -// val output = module.getOutputShape() -// shapeToJList(output) -// } -// -// def getInputShape(module: Container[Activity, Activity, T]): JList[JList[Int]] = { -// val input = module.getInputShape() -// // TODO: inputShape can be nested MultiShape -// shapeToJList(input) -// } -// -// def createKerasDense( -// outputDim: Int, -// init: String = "glorot_uniform", -// activation: String = null, -// wRegularizer: Regularizer[T] = null, -// bRegularizer: Regularizer[T] = null, -// bias: Boolean = true, -// inputShape: JList[Int] = null): Dense[T] = { -// Dense(outputDim, init, activation, wRegularizer, -// bRegularizer, bias, toScalaShape(inputShape)) -// } -// -// def createKerasEmbedding( -// inputDim: Int, -// outputDim: Int, -// init: String = "uniform", -// wRegularizer: Regularizer[T] = null, -// inputShape: JList[Int] = null): Embedding[T] = { -// Embedding[T](inputDim, outputDim, init, wRegularizer, toScalaShape(inputShape)) -// } -// -// def createKerasBatchNormalization( -// epsilon: Double = 0.001, -// momentum: Double = 0.99, -// betaInit: String = "zero", -// gammaInit: String = "one", -// dimOrdering: String = "th", -// inputShape: JList[Int] = null): BatchNormalization[T] = { -// BatchNormalization[T](epsilon, momentum, betaInit, -// gammaInit, dimOrdering, toScalaShape(inputShape)) -// } -// -// def setRunningMean(module: BatchNormalization[T], runningMean: JTensor): Unit = { -// module.labor.asInstanceOf[SpatialBatchNormalization[T]] -// .runningMean.set(toTensor(runningMean)) -// } -// -// def setRunningStd(module: BatchNormalization[T], runningStd: JTensor): Unit = { -// module.labor.asInstanceOf[SpatialBatchNormalization[T]] -// .runningVar.set(toTensor(runningStd)) -// } -// -// def getRunningMean(module: BatchNormalization[T]): JTensor = { -// toJTensor(module.labor.asInstanceOf[SpatialBatchNormalization[T]] -// .runningMean) -// } -// -// def getRunningStd(module: BatchNormalization[T]): JTensor = { -// toJTensor(module.labor.asInstanceOf[SpatialBatchNormalization[T]] -// .runningVar) -// } -// -// def createKerasMerge( -// layers: JList[AbstractModule[Activity, Activity, T]] = null, -// mode: String = "sum", -// concatAxis: Int = -1, -// inputShape: JList[JList[Int]]): Merge[T] = { -// val layersList = if (layers != null) layers.asScala.toList -// else null -// Merge[T](layersList, mode, concatAxis, toScalaMultiShape(inputShape)) -// } -// -// def createKerasConvolution2D( -// nbFilter: Int, -// nbRow: Int, -// nbCol: Int, -// init: String = "glorot_uniform", -// activation: String = null, -// borderMode: String = "valid", -// subsample: JList[Int], -// dimOrdering: String = "th", -// wRegularizer: Regularizer[T] = null, -// bRegularizer: Regularizer[T] = null, -// bias: Boolean = true, -// inputShape: JList[Int] = null): Convolution2D[T] = { -// new Convolution2D(nbFilter, nbRow, nbCol, KerasUtils.getInitMethod(init), -// KerasUtils.getKerasActivation(activation), borderMode, -// toScalaArray(subsample), KerasUtils.toBigDLFormat(dimOrdering), -// wRegularizer, bRegularizer, bias, toScalaShape(inputShape)) -// } -// -// def createKerasMaxPooling2D( -// poolSize: JList[Int], -// strides: JList[Int], -// borderMode: String = "valid", -// dimOrdering: String = "th", -// inputShape: JList[Int] = null): MaxPooling2D[T] = { -// new MaxPooling2D[T](toScalaArray(poolSize), toScalaArray(strides), -// borderMode, KerasUtils.toBigDLFormat(dimOrdering), toScalaShape(inputShape)) -// } -// -// def createKerasActivation( -// activation: String, -// inputShape: JList[Int] = null): Activation[T] = { -// Activation(activation, toScalaShape(inputShape)) -// } -// -// def createKerasReshape( -// targetShape: JList[Int], -// inputShape: JList[Int] = null): Reshape[T] = { -// Reshape(toScalaArray(targetShape), toScalaShape(inputShape)) -// } -// -// def createKerasDropout( -// p: Double, -// inputShape: JList[Int] = null): Dropout[T] = { -// Dropout(p, toScalaShape(inputShape)) -// } -// -// def createKerasFlatten( -// inputShape: JList[Int] = null): Flatten[T] = { -// Flatten(toScalaShape(inputShape)) -// } -// -// def createKerasSimpleRNN( -// outputDim: Int, -// activation: String = "tanh", -// returnSequences: Boolean = false, -// goBackwards: Boolean = false, -// wRegularizer: Regularizer[T] = null, -// uRegularizer: Regularizer[T] = null, -// bRegularizer: Regularizer[T] = null, -// inputShape: JList[Int] = null): SimpleRNN[T] = { -// SimpleRNN(outputDim, activation, returnSequences, goBackwards, -// wRegularizer, uRegularizer, bRegularizer, toScalaShape(inputShape)) -// } -// -// def createKerasLSTM( -// outputDim: Int, -// activation: String = "tanh", -// innerActivation: String = "hard_sigmoid", -// returnSequences: Boolean = false, -// goBackwards: Boolean = false, -// wRegularizer: Regularizer[T] = null, -// uRegularizer: Regularizer[T] = null, -// bRegularizer: Regularizer[T] = null, -// inputShape: JList[Int] = null): LSTM[T] = { -// LSTM(outputDim, activation, innerActivation, returnSequences, -// goBackwards, wRegularizer, uRegularizer, bRegularizer, toScalaShape(inputShape)) -// } -// -// def createKerasGRU( -// outputDim: Int, -// activation: String = "tanh", -// innerActivation: String = "hard_sigmoid", -// returnSequences: Boolean = false, -// goBackwards: Boolean = false, -// wRegularizer: Regularizer[T] = null, -// uRegularizer: Regularizer[T] = null, -// bRegularizer: Regularizer[T] = null, -// inputShape: JList[Int] = null): GRU[T] = { -// GRU(outputDim, activation, innerActivation, returnSequences, -// goBackwards, wRegularizer, uRegularizer, bRegularizer, toScalaShape(inputShape)) -// } -// -// def createKerasHighway( -// activation: String = null, -// wRegularizer: Regularizer[T] = null, -// bRegularizer: Regularizer[T] = null, -// bias: Boolean = true, -// inputShape: JList[Int] = null): Highway[T] = { -// Highway(activation, wRegularizer, bRegularizer, bias, toScalaShape(inputShape)) -// } -// -// def createKerasZeroPadding1D( -// padding: JList[Int], -// inputShape: JList[Int] = null): ZeroPadding1D[T] = { -// new ZeroPadding1D(toScalaArray(padding), toScalaShape(inputShape)) -// } -// -// def createKerasZeroPadding2D( -// padding: JList[Int], -// dimOrdering: String = "th", -// inputShape: JList[Int] = null): ZeroPadding2D[T] = { -// new ZeroPadding2D(toScalaArray(padding), -// KerasUtils.toBigDLFormat(dimOrdering), toScalaShape(inputShape)) -// } -// -// def createKerasUpSampling1D( -// length: Int = 2, -// inputShape: JList[Int] = null): UpSampling1D[T] = { -// UpSampling1D(length, toScalaShape(inputShape)) -// } -// -// def createKerasUpSampling2D( -// size: JList[Int], -// dimOrdering: String = "th", -// inputShape: JList[Int] = null): UpSampling2D[T] = { -// new UpSampling2D(toScalaArray(size), KerasUtils.toBigDLFormat(dimOrdering), -// toScalaShape(inputShape)) -// } -// -// def createKerasUpSampling3D( -// size: JList[Int], -// dimOrdering: String = "th", -// inputShape: JList[Int] = null): UpSampling3D[T] = { -// new UpSampling3D(toScalaArray(size), KerasUtils.toBigDLFormat5D(dimOrdering), -// toScalaShape(inputShape)) -// } -// -// def createKerasMaxoutDense( -// outputDim: Int, -// nbFeature: Int = 4, -// wRegularizer: Regularizer[T] = null, -// bRegularizer: Regularizer[T] = null, -// bias: Boolean = true, -// inputShape: JList[Int] = null): MaxoutDense[T] = { -// MaxoutDense(outputDim, nbFeature, wRegularizer, -// bRegularizer, bias, toScalaShape(inputShape)) -// } -// -// def createKerasConvolution1D( -// nbFilter: Int, -// filterLength: Int, -// init: String = "glorot_uniform", -// activation: String = null, -// borderMode: String = "valid", -// subsampleLength: Int = 1, -// wRegularizer: Regularizer[T] = null, -// bRegularizer: Regularizer[T] = null, -// bias: Boolean = true, -// inputShape: JList[Int] = null): Convolution1D[T] = { -// Convolution1D(nbFilter, filterLength, init, activation, borderMode, -// subsampleLength, wRegularizer, bRegularizer, bias, toScalaShape(inputShape)) -// } -// -// def createKerasConvolution3D( -// nbFilter: Int, -// kernelDim1: Int, -// kernelDim2: Int, -// kernelDim3: Int, -// init: String = "glorot_uniform", -// activation: String = null, -// borderMode: String = "valid", -// subsample: JList[Int], -// dimOrdering: String = "th", -// wRegularizer: Regularizer[T] = null, -// bRegularizer: Regularizer[T] = null, -// bias: Boolean = true, -// inputShape: JList[Int] = null): Convolution3D[T] = { -// new Convolution3D(nbFilter, kernelDim1, kernelDim2, kernelDim3, -// KerasUtils.getInitMethod(init), KerasUtils.getKerasActivation(activation), -// borderMode, toScalaArray(subsample), KerasUtils.toBigDLFormat5D(dimOrdering), -// wRegularizer, bRegularizer, bias, toScalaShape(inputShape)) -// } -// -// def createKerasMaxPooling1D( -// poolLength: Int = 2, -// stride: Int = -1, -// borderMode: String = "valid", -// inputShape: JList[Int] = null): MaxPooling1D[T] = { -// MaxPooling1D(poolLength, stride, borderMode, toScalaShape(inputShape)) -// } -// -// def createKerasMaxPooling3D( -// poolSize: JList[Int], -// strides: JList[Int], -// dimOrdering: String = "th", -// inputShape: JList[Int] = null): MaxPooling3D[T] = { -// new MaxPooling3D(toScalaArray(poolSize), toScalaArray(strides), -// KerasUtils.toBigDLFormat5D(dimOrdering), toScalaShape(inputShape)) -// } -// -// def createKerasAveragePooling1D( -// poolLength: Int = 2, -// stride: Int = -1, -// borderMode: String = "valid", -// inputShape: JList[Int] = null): AveragePooling1D[T] = { -// AveragePooling1D(poolLength, stride, borderMode, toScalaShape(inputShape)) -// } -// -// def createKerasAveragePooling2D( -// poolSize: JList[Int], -// strides: JList[Int], -// borderMode: String = "valid", -// dimOrdering: String = "th", -// inputShape: JList[Int] = null): AveragePooling2D[T] = { -// new AveragePooling2D(toScalaArray(poolSize), toScalaArray(strides), -// borderMode, KerasUtils.toBigDLFormat(dimOrdering), toScalaShape(inputShape)) -// } -// -// def createKerasAveragePooling3D( -// poolSize: JList[Int], -// strides: JList[Int], -// dimOrdering: String = "th", -// inputShape: JList[Int] = null): AveragePooling3D[T] = { -// new AveragePooling3D(toScalaArray(poolSize), toScalaArray(strides), -// KerasUtils.toBigDLFormat5D(dimOrdering), toScalaShape(inputShape)) -// } -// -// def createKerasGlobalAveragePooling2D( -// dimOrdering: String = "th", -// inputShape: JList[Int] = null): GlobalAveragePooling2D[T] = { -// GlobalAveragePooling2D(dimOrdering, toScalaShape(inputShape)) -// } -// -// def createKerasGlobalMaxPooling2D( -// dimOrdering: String = "th", -// inputShape: JList[Int] = null): GlobalMaxPooling2D[T] = { -// GlobalMaxPooling2D(dimOrdering, toScalaShape(inputShape)) -// } -// -// def createKerasRepeatVector( -// n: Int, -// inputShape: JList[Int] = null): RepeatVector[T] = { -// RepeatVector(n, toScalaShape(inputShape)) -// } -// -// def createKerasPermute( -// dims: JList[Int], -// inputShape: JList[Int] = null): Permute[T] = { -// Permute(toScalaArray(dims), toScalaShape(inputShape)) -// } -// -// def createKerasCropping1D( -// cropping: JList[Int], -// inputShape: JList[Int] = null): Cropping1D[T] = { -// new Cropping1D(toScalaArray(cropping), toScalaShape(inputShape)) -// } -// -// def createKerasCropping2D( -// heightCrop: JList[Int], -// widthCrop: JList[Int], -// dimOrdering: String = "th", -// inputShape: JList[Int] = null): Cropping2D[T] = { -// new Cropping2D(toScalaArray(heightCrop), toScalaArray(widthCrop), -// KerasUtils.toBigDLFormat(dimOrdering), toScalaShape(inputShape)) -// } -// -// def createKerasCropping3D( -// dim1Crop: JList[Int], -// dim2Crop: JList[Int], -// dim3Crop: JList[Int], -// dimOrdering: String = "th", -// inputShape: JList[Int] = null): Cropping3D[T] = { -// new Cropping3D(toScalaArray(dim1Crop), toScalaArray(dim2Crop), toScalaArray(dim3Crop), -// KerasUtils.toBigDLFormat5D(dimOrdering), toScalaShape(inputShape)) -// } -// -// def createKerasAtrousConvolution1D( -// nbFilter: Int, -// filterLength: Int, -// init: String = "glorot_uniform", -// activation: String = null, -// subsampleLength: Int = 1, -// atrousRate: Int = 1, -// wRegularizer: Regularizer[T] = null, -// bRegularizer: Regularizer[T] = null, -// inputShape: JList[Int] = null): AtrousConvolution1D[T] = { -// AtrousConvolution1D(nbFilter, filterLength, init, activation, -// subsampleLength, atrousRate, wRegularizer, bRegularizer, toScalaShape(inputShape)) -// } -// -// def createKerasAtrousConvolution2D( -// nbFilter: Int, -// nbRow: Int, -// nbCol: Int, -// init: String = "glorot_uniform", -// activation: String = null, -// subsample: JList[Int], -// atrousRate: JList[Int], -// dimOrdering: String = "th", -// wRegularizer: Regularizer[T] = null, -// bRegularizer: Regularizer[T] = null, -// inputShape: JList[Int] = null): AtrousConvolution2D[T] = { -// new AtrousConvolution2D(nbFilter, nbRow, nbCol, KerasUtils.getInitMethod(init), -// KerasUtils.getKerasActivation(activation), toScalaArray(subsample), -// toScalaArray(atrousRate), KerasUtils.toBigDLFormat(dimOrdering), -// wRegularizer, bRegularizer, toScalaShape(inputShape)) -// } -// -// def createKerasDeconvolution2D( -// nbFilter: Int, -// nbRow: Int, -// nbCol: Int, -// init: String = "glorot_uniform", -// activation: String = null, -// subsample: JList[Int], -// dimOrdering: String = "th", -// wRegularizer: Regularizer[T] = null, -// bRegularizer: Regularizer[T] = null, -// bias: Boolean = true, -// inputShape: JList[Int] = null): Deconvolution2D[T] = { -// new Deconvolution2D(nbFilter, nbRow, nbCol, KerasUtils.getInitMethod(init), -// KerasUtils.getKerasActivation(activation), toScalaArray(subsample), -// KerasUtils.toBigDLFormat(dimOrdering), wRegularizer, bRegularizer, -// bias, toScalaShape(inputShape)) -// } -// -// def createKerasConvLSTM2D( -// nbFilter: Int, -// nbKernel: Int, -// activation: String = "tanh", -// innerActivation: String = "hard_sigmoid", -// dimOrdering: String = "th", -// subsample: Int = 1, -// wRegularizer: Regularizer[T] = null, -// uRegularizer: Regularizer[T] = null, -// bRegularizer: Regularizer[T] = null, -// returnSequences: Boolean = false, -// goBackwards: Boolean = false, -// inputShape: JList[Int] = null): ConvLSTM2D[T] = { -// ConvLSTM2D(nbFilter, nbKernel, activation, innerActivation, -// dimOrdering, subsample, wRegularizer, uRegularizer, bRegularizer, -// returnSequences, goBackwards, toScalaShape(inputShape)) -// } -// -// def createKerasLocallyConnected1D( -// nbFilter: Int, -// filterLength: Int, -// activation: String = null, -// subsampleLength: Int = 1, -// wRegularizer: Regularizer[T] = null, -// bRegularizer: Regularizer[T] = null, -// bias: Boolean = true, -// inputShape: JList[Int] = null): LocallyConnected1D[T] = { -// LocallyConnected1D(nbFilter, filterLength, activation, subsampleLength, -// wRegularizer, bRegularizer, bias, toScalaShape(inputShape)) -// } -// -// def createKerasLocallyConnected2D( -// nbFilter: Int, -// nbRow: Int, -// nbCol: Int, -// activation: String = null, -// borderMode: String = "valid", -// subsample: JList[Int], -// dimOrdering: String = "th", -// wRegularizer: Regularizer[T] = null, -// bRegularizer: Regularizer[T] = null, -// bias: Boolean = true, -// inputShape: JList[Int] = null): LocallyConnected2D[T] = { -// new LocallyConnected2D(nbFilter, nbRow, nbCol, KerasUtils.getKerasActivation(activation), -// borderMode, toScalaArray(subsample), KerasUtils.toBigDLFormat(dimOrdering), -// wRegularizer, bRegularizer, bias, toScalaShape(inputShape)) -// } -// -// def createKerasSeparableConvolution2D( -// nbFilter: Int, -// nbRow: Int, -// nbCol: Int, -// init: String = "glorot_uniform", -// activation: String = null, -// borderMode: String = "valid", -// subsample: JList[Int], -// depthMultiplier: Int = 1, -// dimOrdering: String = "th", -// depthwiseRegularizer: Regularizer[T] = null, -// pointwiseRegularizer: Regularizer[T] = null, -// bRegularizer: Regularizer[T] = null, -// bias: Boolean = true, -// inputShape: JList[Int] = null): SeparableConvolution2D[T] = { -// new SeparableConvolution2D(nbFilter, nbRow, nbCol, KerasUtils.getInitMethod(init), -// KerasUtils.getKerasActivation(activation), borderMode, toScalaArray(subsample), -// depthMultiplier, KerasUtils.toBigDLFormat(dimOrdering), -// depthwiseRegularizer, pointwiseRegularizer, bRegularizer, bias, toScalaShape(inputShape)) -// } -// -// def createKerasZeroPadding3D( -// padding: JList[Int], -// dimOrdering: String = "th", -// inputShape: JList[Int] = null): ZeroPadding3D[T] = { -// new ZeroPadding3D(toScalaArray(padding), KerasUtils.toBigDLFormat5D(dimOrdering), -// toScalaShape(inputShape)) -// } -// -// def createKerasGlobalAveragePooling1D( -// inputShape: JList[Int] = null): GlobalAveragePooling1D[T] = { -// GlobalAveragePooling1D(toScalaShape(inputShape)) -// } -// -// def createKerasGlobalMaxPooling1D( -// inputShape: JList[Int] = null): GlobalMaxPooling1D[T] = { -// GlobalMaxPooling1D(toScalaShape(inputShape)) -// } -// -// def createKerasGlobalMaxPooling3D( -// dimOrdering: String = "th", -// inputShape: JList[Int] = null): GlobalMaxPooling3D[T] = { -// GlobalMaxPooling3D(dimOrdering, toScalaShape(inputShape)) -// } -// -// def createKerasGlobalAveragePooling3D( -// dimOrdering: String = "th", -// inputShape: JList[Int] = null): GlobalAveragePooling3D[T] = { -// GlobalAveragePooling3D(dimOrdering, toScalaShape(inputShape)) -// } -// -// def createKerasSpatialDropout1D( -// p: Double = 0.5, -// inputShape: JList[Int] = null): SpatialDropout1D[T] = { -// SpatialDropout1D(p, toScalaShape(inputShape)) -// } -// -// def createKerasSpatialDropout2D( -// p: Double = 0.5, -// dimOrdering: String = "th", -// inputShape: JList[Int] = null): SpatialDropout2D[T] = { -// SpatialDropout2D(p, dimOrdering, toScalaShape(inputShape)) -// } -// -// def createKerasSpatialDropout3D( -// p: Double = 0.5, -// dimOrdering: String = "th", -// inputShape: JList[Int] = null): SpatialDropout3D[T] = { -// SpatialDropout3D(p, dimOrdering, toScalaShape(inputShape)) -// } -// -// def createKerasGaussianDropout( -// p: Double, -// inputShape: JList[Int] = null): GaussianDropout[T] = { -// GaussianDropout(p, toScalaShape(inputShape)) -// } -// -// def createKerasGaussianNoise( -// sigma: Double, -// inputShape: JList[Int] = null): GaussianNoise[T] = { -// GaussianNoise(sigma, toScalaShape(inputShape)) -// } -// -// def createKerasMasking( -// maskValue: Double = 0.0, -// inputShape: JList[Int] = null): Masking[T] = { -// Masking(maskValue, toScalaShape(inputShape)) -// } -// -// def createKerasSReLU( -// tLeftInit: String = "zero", -// aLeftInit: String = "glorot_uniform", -// tRightInit: String = "glorot_uniform", -// aRightInit: String = "one", -// sharedAxes: JList[Int] = null, -// inputShape: JList[Int] = null): SReLU[T] = { -// SReLU(tLeftInit, aLeftInit, tRightInit, aRightInit, -// toScalaArray(sharedAxes), toScalaShape(inputShape)) -// } -// -// def createKerasELU( -// alpha: Double = 1.0, -// inputShape: JList[Int] = null): ELU[T] = { -// ELU(alpha, toScalaShape(inputShape)) -// } -// -// def createKerasLeakyReLU( -// alpha: Double = 0.01, -// inputShape: JList[Int] = null): LeakyReLU[T] = { -// LeakyReLU(alpha, toScalaShape(inputShape)) -// } -// -// def createKerasThresholdedReLU( -// theta: Double = 1.0, -// inputShape: JList[Int] = null): ThresholdedReLU[T] = { -// ThresholdedReLU(theta, toScalaShape(inputShape)) -// } -// -// def createKerasTimeDistributed( -// layer: KerasLayer[Tensor[T], Tensor[T], T], -// inputShape: JList[Int] = null): TimeDistributed[T] = { -// TimeDistributed(layer, toScalaShape(inputShape)) -// } -// -// def createKerasBidirectional( -// layer: Recurrent[T], -// mergeMode: String = "concat", -// inputShape: JList[Int] = null): Bidirectional[T] = { -// Bidirectional(layer, mergeMode, toScalaShape(inputShape)) -// } -// -// def compile( -// module: KerasModel[T], -// optimizer: OptimMethod[T], -// loss: Criterion[T], -// metrics: JList[ValidationMethod[T]] = null): Unit = { -// module.compile(optimizer, loss, -// if (metrics == null) null else metrics.asScala.toArray) -// } -// -// def fit( -// module: KerasModel[T], -// x: JavaRDD[Sample], -// batchSize: Int = 32, -// epochs: Int = 10, -// validationData: JavaRDD[Sample] = null): Unit = { -// module.fit(toJSample(x), batchSize, epochs, -// if (validationData == null) null else toJSample(validationData)) -// } -// -// def fit( -// module: KerasModel[T], -// x: DataSet[ImageFeature], -// batchSize: Int, -// epochs: Int, -// validationData: DataSet[ImageFeature]): Unit = { -// val trainData = x -> ImageFeatureToMiniBatch[T](batchSize) -// val valData = -// if (validationData != null) validationData -> ImageFeatureToMiniBatch[T](batchSize) -// else null -// module.fit(trainData, epochs, valData) -// } -// -// def fit( -// module: KerasModel[T], -// xTrain: JList[JTensor], -// yTrain: JTensor, -// batchSize: Int, -// epochs: Int, -// xVal: JList[JTensor], -// yVal: JTensor, -// localCores: Int): Unit = { -// val trainArray = toSampleArray(xTrain.asScala.toList.map{f => toTensor(f)}, toTensor(yTrain)) -// val trainData = batching(DataSet.array(trainArray), batchSize) -// .asInstanceOf[LocalDataSet[MiniBatch[T]]] -// val valData = if (xVal != null && yVal != null) { -// val valArray = toSampleArray(xVal.asScala.toList.map{f => toTensor(f)}, toTensor(yVal)) -// batching(DataSet.array(valArray), batchSize) -// } else null -// Engine.setNodeAndCore(1, localCores) -// module.fit(trainData, epochs, valData) -// } -// -// def evaluate( -// module: KerasModel[T], -// x: JavaRDD[Sample], -// batchSize: Int = 32): JList[EvaluatedResult] = { -// val resultArray = module.evaluate(toJSample(x), batchSize) -// val testResultArray = resultArray.map { result => -// EvaluatedResult(result._1.result()._1, result._1.result()._2, -// result._2.toString()) -// } -// testResultArray.toList.asJava -// } -// -//} +/* + * Copyright 2016 The BigDL Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.intel.analytics.bigdl.dllib.utils.python.api + +import java.util.{List => JList} + +import com.intel.analytics.bigdl.{Criterion, DataSet} +import com.intel.analytics.bigdl.dllib.feature.dataset.{DataSet, LocalDataSet, MiniBatch} +import com.intel.analytics.bigdl.dllib.nn.Graph.ModuleNode +import com.intel.analytics.bigdl.dllib.nn.{Container, SpatialBatchNormalization} +import com.intel.analytics.bigdl.dllib.nn.abstractnn.{AbstractModule, Activity} +import com.intel.analytics.bigdl.dllib.keras +import com.intel.analytics.bigdl.dllib.keras._ +import com.intel.analytics.bigdl.numeric._ +import com.intel.analytics.bigdl.dllib.optim.{OptimMethod, Regularizer, ValidationMethod} +import com.intel.analytics.bigdl.dllib.tensor.Tensor +import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric +import com.intel.analytics.bigdl.dllib.feature.transform.vision.image.{ImageFeature, ImageFeatureToMiniBatch} +import com.intel.analytics.bigdl.utils.{Engine, MultiShape, Shape, SingleShape} +import org.apache.spark.api.java.JavaRDD + +import scala.collection.JavaConverters._ +import scala.language.existentials +import scala.reflect.ClassTag + + +object PythonBigDLKeras { + + def ofFloat(): PythonBigDLKeras[Float] = new PythonBigDLKeras[Float]() + + def ofDouble(): PythonBigDLKeras[Double] = new PythonBigDLKeras[Double]() +} + +class PythonBigDLKeras[T: ClassTag](implicit ev: TensorNumeric[T]) extends PythonBigDL[T] { + + def toScalaShape(inputShape: JList[Int]): Shape = { + if (inputShape == null) { + null + } else { + Shape(inputShape.asScala.toArray) + } + } + + def toScalaMultiShape(inputShape: JList[JList[Int]]): Shape = { + if (inputShape == null) { + null + } else { + Shape(inputShape.asScala.toArray.map(shape => Shape(shape.asScala.toArray)).toList) + } + } + + def toScalaArray(list: JList[Int]): Array[Int] = { + if (list == null) { + null + } else { + list.asScala.toArray + } + } + + def createKerasModel(input: JList[ModuleNode[T]], + output: JList[ModuleNode[T]]): Model[T] = { + keras.Model(input.asScala.toArray, output.asScala.toArray) + } + + def createKerasSequential(): keras.Sequential[T] = { + keras.Sequential[T]() + } + + def createKerasInput( + name : String = null, + inputShape: JList[Int] = null): ModuleNode[T] = { + Input(name = name, inputShape = toScalaShape(inputShape)) + } + + def createKerasInputLayer( + inputShape: JList[Int] = null): KerasLayer[Activity, Activity, T] = { + InputLayer(inputShape = toScalaShape(inputShape)) + } + + def shapeToJList(shape: Shape): JList[JList[Int]] = { + val shapes = if (shape.isInstanceOf[SingleShape]) { + MultiShape(List(shape)) + } + else { + shape + } + shapes.toMulti().map(single => single.toSingle().toList.asJava).toList.asJava + } + + def getOutputShape(module: Container[Activity, Activity, T]): JList[JList[Int]] = { + val output = module.getOutputShape() + shapeToJList(output) + } + + def getInputShape(module: Container[Activity, Activity, T]): JList[JList[Int]] = { + val input = module.getInputShape() + // TODO: inputShape can be nested MultiShape + shapeToJList(input) + } + + def createKerasDense( + outputDim: Int, + init: String = "glorot_uniform", + activation: String = null, + wRegularizer: Regularizer[T] = null, + bRegularizer: Regularizer[T] = null, + bias: Boolean = true, + inputShape: JList[Int] = null): Dense[T] = { + Dense(outputDim, init, activation, wRegularizer, + bRegularizer, bias, toScalaShape(inputShape)) + } + + def createKerasEmbedding( + inputDim: Int, + outputDim: Int, + init: String = "uniform", + wRegularizer: Regularizer[T] = null, + inputShape: JList[Int] = null): Embedding[T] = { + Embedding[T](inputDim, outputDim, init, wRegularizer, toScalaShape(inputShape)) + } + + def createKerasBatchNormalization( + epsilon: Double = 0.001, + momentum: Double = 0.99, + betaInit: String = "zero", + gammaInit: String = "one", + dimOrdering: String = "th", + inputShape: JList[Int] = null): BatchNormalization[T] = { + BatchNormalization[T](epsilon, momentum, betaInit, + gammaInit, dimOrdering, toScalaShape(inputShape)) + } + + def setRunningMean(module: BatchNormalization[T], runningMean: JTensor): Unit = { + module.labor.asInstanceOf[SpatialBatchNormalization[T]] + .runningMean.set(toTensor(runningMean)) + } + + def setRunningStd(module: BatchNormalization[T], runningStd: JTensor): Unit = { + module.labor.asInstanceOf[SpatialBatchNormalization[T]] + .runningVar.set(toTensor(runningStd)) + } + + def getRunningMean(module: BatchNormalization[T]): JTensor = { + toJTensor(module.labor.asInstanceOf[SpatialBatchNormalization[T]] + .runningMean) + } + + def getRunningStd(module: BatchNormalization[T]): JTensor = { + toJTensor(module.labor.asInstanceOf[SpatialBatchNormalization[T]] + .runningVar) + } + + def createKerasMerge( + layers: JList[AbstractModule[Activity, Activity, T]] = null, + mode: String = "sum", + concatAxis: Int = -1, + inputShape: JList[JList[Int]]): Merge[T] = { + val layersList = if (layers != null) layers.asScala.toList + else null + Merge[T](layersList, mode, concatAxis, toScalaMultiShape(inputShape)) + } + + def createKerasConvolution2D( + nbFilter: Int, + nbRow: Int, + nbCol: Int, + init: String = "glorot_uniform", + activation: String = null, + borderMode: String = "valid", + subsample: JList[Int], + dimOrdering: String = "th", + wRegularizer: Regularizer[T] = null, + bRegularizer: Regularizer[T] = null, + bias: Boolean = true, + inputShape: JList[Int] = null): Convolution2D[T] = { + new Convolution2D(nbFilter, nbRow, nbCol, KerasUtils.getInitMethod(init), + KerasUtils.getKerasActivation(activation), borderMode, + toScalaArray(subsample), KerasUtils.toBigDLFormat(dimOrdering), + wRegularizer, bRegularizer, bias, toScalaShape(inputShape)) + } + + def createKerasMaxPooling2D( + poolSize: JList[Int], + strides: JList[Int], + borderMode: String = "valid", + dimOrdering: String = "th", + inputShape: JList[Int] = null): MaxPooling2D[T] = { + new MaxPooling2D[T](toScalaArray(poolSize), toScalaArray(strides), + borderMode, KerasUtils.toBigDLFormat(dimOrdering), toScalaShape(inputShape)) + } + + def createKerasActivation( + activation: String, + inputShape: JList[Int] = null): Activation[T] = { + Activation(activation, toScalaShape(inputShape)) + } + + def createKerasReshape( + targetShape: JList[Int], + inputShape: JList[Int] = null): Reshape[T] = { + Reshape(toScalaArray(targetShape), toScalaShape(inputShape)) + } + + def createKerasDropout( + p: Double, + inputShape: JList[Int] = null): Dropout[T] = { + Dropout(p, toScalaShape(inputShape)) + } + + def createKerasFlatten( + inputShape: JList[Int] = null): Flatten[T] = { + Flatten(toScalaShape(inputShape)) + } + + def createKerasSimpleRNN( + outputDim: Int, + activation: String = "tanh", + returnSequences: Boolean = false, + goBackwards: Boolean = false, + wRegularizer: Regularizer[T] = null, + uRegularizer: Regularizer[T] = null, + bRegularizer: Regularizer[T] = null, + inputShape: JList[Int] = null): SimpleRNN[T] = { + SimpleRNN(outputDim, activation, returnSequences, goBackwards, + wRegularizer, uRegularizer, bRegularizer, toScalaShape(inputShape)) + } + + def createKerasLSTM( + outputDim: Int, + activation: String = "tanh", + innerActivation: String = "hard_sigmoid", + returnSequences: Boolean = false, + goBackwards: Boolean = false, + wRegularizer: Regularizer[T] = null, + uRegularizer: Regularizer[T] = null, + bRegularizer: Regularizer[T] = null, + inputShape: JList[Int] = null): LSTM[T] = { + LSTM(outputDim, activation, innerActivation, returnSequences, + goBackwards, wRegularizer, uRegularizer, bRegularizer, toScalaShape(inputShape)) + } + + def createKerasGRU( + outputDim: Int, + activation: String = "tanh", + innerActivation: String = "hard_sigmoid", + returnSequences: Boolean = false, + goBackwards: Boolean = false, + wRegularizer: Regularizer[T] = null, + uRegularizer: Regularizer[T] = null, + bRegularizer: Regularizer[T] = null, + inputShape: JList[Int] = null): GRU[T] = { + GRU(outputDim, activation, innerActivation, returnSequences, + goBackwards, wRegularizer, uRegularizer, bRegularizer, toScalaShape(inputShape)) + } + + def createKerasHighway( + activation: String = null, + wRegularizer: Regularizer[T] = null, + bRegularizer: Regularizer[T] = null, + bias: Boolean = true, + inputShape: JList[Int] = null): Highway[T] = { + Highway(activation, wRegularizer, bRegularizer, bias, toScalaShape(inputShape)) + } + + def createKerasZeroPadding1D( + padding: JList[Int], + inputShape: JList[Int] = null): ZeroPadding1D[T] = { + new ZeroPadding1D(toScalaArray(padding), toScalaShape(inputShape)) + } + + def createKerasZeroPadding2D( + padding: JList[Int], + dimOrdering: String = "th", + inputShape: JList[Int] = null): ZeroPadding2D[T] = { + new ZeroPadding2D(toScalaArray(padding), + KerasUtils.toBigDLFormat(dimOrdering), toScalaShape(inputShape)) + } + + def createKerasUpSampling1D( + length: Int = 2, + inputShape: JList[Int] = null): UpSampling1D[T] = { + UpSampling1D(length, toScalaShape(inputShape)) + } + + def createKerasUpSampling2D( + size: JList[Int], + dimOrdering: String = "th", + inputShape: JList[Int] = null): UpSampling2D[T] = { + new UpSampling2D(toScalaArray(size), KerasUtils.toBigDLFormat(dimOrdering), + toScalaShape(inputShape)) + } + + def createKerasUpSampling3D( + size: JList[Int], + dimOrdering: String = "th", + inputShape: JList[Int] = null): UpSampling3D[T] = { + new UpSampling3D(toScalaArray(size), KerasUtils.toBigDLFormat5D(dimOrdering), + toScalaShape(inputShape)) + } + + def createKerasMaxoutDense( + outputDim: Int, + nbFeature: Int = 4, + wRegularizer: Regularizer[T] = null, + bRegularizer: Regularizer[T] = null, + bias: Boolean = true, + inputShape: JList[Int] = null): MaxoutDense[T] = { + MaxoutDense(outputDim, nbFeature, wRegularizer, + bRegularizer, bias, toScalaShape(inputShape)) + } + + def createKerasConvolution1D( + nbFilter: Int, + filterLength: Int, + init: String = "glorot_uniform", + activation: String = null, + borderMode: String = "valid", + subsampleLength: Int = 1, + wRegularizer: Regularizer[T] = null, + bRegularizer: Regularizer[T] = null, + bias: Boolean = true, + inputShape: JList[Int] = null): Convolution1D[T] = { + Convolution1D(nbFilter, filterLength, init, activation, borderMode, + subsampleLength, wRegularizer, bRegularizer, bias, toScalaShape(inputShape)) + } + + def createKerasConvolution3D( + nbFilter: Int, + kernelDim1: Int, + kernelDim2: Int, + kernelDim3: Int, + init: String = "glorot_uniform", + activation: String = null, + borderMode: String = "valid", + subsample: JList[Int], + dimOrdering: String = "th", + wRegularizer: Regularizer[T] = null, + bRegularizer: Regularizer[T] = null, + bias: Boolean = true, + inputShape: JList[Int] = null): Convolution3D[T] = { + new Convolution3D(nbFilter, kernelDim1, kernelDim2, kernelDim3, + KerasUtils.getInitMethod(init), KerasUtils.getKerasActivation(activation), + borderMode, toScalaArray(subsample), KerasUtils.toBigDLFormat5D(dimOrdering), + wRegularizer, bRegularizer, bias, toScalaShape(inputShape)) + } + + def createKerasMaxPooling1D( + poolLength: Int = 2, + stride: Int = -1, + borderMode: String = "valid", + inputShape: JList[Int] = null): MaxPooling1D[T] = { + MaxPooling1D(poolLength, stride, borderMode, toScalaShape(inputShape)) + } + + def createKerasMaxPooling3D( + poolSize: JList[Int], + strides: JList[Int], + dimOrdering: String = "th", + inputShape: JList[Int] = null): MaxPooling3D[T] = { + new MaxPooling3D(toScalaArray(poolSize), toScalaArray(strides), + KerasUtils.toBigDLFormat5D(dimOrdering), toScalaShape(inputShape)) + } + + def createKerasAveragePooling1D( + poolLength: Int = 2, + stride: Int = -1, + borderMode: String = "valid", + inputShape: JList[Int] = null): AveragePooling1D[T] = { + AveragePooling1D(poolLength, stride, borderMode, toScalaShape(inputShape)) + } + + def createKerasAveragePooling2D( + poolSize: JList[Int], + strides: JList[Int], + borderMode: String = "valid", + dimOrdering: String = "th", + inputShape: JList[Int] = null): AveragePooling2D[T] = { + new AveragePooling2D(toScalaArray(poolSize), toScalaArray(strides), + borderMode, KerasUtils.toBigDLFormat(dimOrdering), toScalaShape(inputShape)) + } + + def createKerasAveragePooling3D( + poolSize: JList[Int], + strides: JList[Int], + dimOrdering: String = "th", + inputShape: JList[Int] = null): AveragePooling3D[T] = { + new AveragePooling3D(toScalaArray(poolSize), toScalaArray(strides), + KerasUtils.toBigDLFormat5D(dimOrdering), toScalaShape(inputShape)) + } + + def createKerasGlobalAveragePooling2D( + dimOrdering: String = "th", + inputShape: JList[Int] = null): GlobalAveragePooling2D[T] = { + GlobalAveragePooling2D(dimOrdering, toScalaShape(inputShape)) + } + + def createKerasGlobalMaxPooling2D( + dimOrdering: String = "th", + inputShape: JList[Int] = null): GlobalMaxPooling2D[T] = { + GlobalMaxPooling2D(dimOrdering, toScalaShape(inputShape)) + } + + def createKerasRepeatVector( + n: Int, + inputShape: JList[Int] = null): RepeatVector[T] = { + RepeatVector(n, toScalaShape(inputShape)) + } + + def createKerasPermute( + dims: JList[Int], + inputShape: JList[Int] = null): Permute[T] = { + Permute(toScalaArray(dims), toScalaShape(inputShape)) + } + + def createKerasCropping1D( + cropping: JList[Int], + inputShape: JList[Int] = null): Cropping1D[T] = { + new Cropping1D(toScalaArray(cropping), toScalaShape(inputShape)) + } + + def createKerasCropping2D( + heightCrop: JList[Int], + widthCrop: JList[Int], + dimOrdering: String = "th", + inputShape: JList[Int] = null): Cropping2D[T] = { + new Cropping2D(toScalaArray(heightCrop), toScalaArray(widthCrop), + KerasUtils.toBigDLFormat(dimOrdering), toScalaShape(inputShape)) + } + + def createKerasCropping3D( + dim1Crop: JList[Int], + dim2Crop: JList[Int], + dim3Crop: JList[Int], + dimOrdering: String = "th", + inputShape: JList[Int] = null): Cropping3D[T] = { + new Cropping3D(toScalaArray(dim1Crop), toScalaArray(dim2Crop), toScalaArray(dim3Crop), + KerasUtils.toBigDLFormat5D(dimOrdering), toScalaShape(inputShape)) + } + + def createKerasAtrousConvolution1D( + nbFilter: Int, + filterLength: Int, + init: String = "glorot_uniform", + activation: String = null, + subsampleLength: Int = 1, + atrousRate: Int = 1, + wRegularizer: Regularizer[T] = null, + bRegularizer: Regularizer[T] = null, + inputShape: JList[Int] = null): AtrousConvolution1D[T] = { + AtrousConvolution1D(nbFilter, filterLength, init, activation, + subsampleLength, atrousRate, wRegularizer, bRegularizer, toScalaShape(inputShape)) + } + + def createKerasAtrousConvolution2D( + nbFilter: Int, + nbRow: Int, + nbCol: Int, + init: String = "glorot_uniform", + activation: String = null, + subsample: JList[Int], + atrousRate: JList[Int], + dimOrdering: String = "th", + wRegularizer: Regularizer[T] = null, + bRegularizer: Regularizer[T] = null, + inputShape: JList[Int] = null): AtrousConvolution2D[T] = { + new AtrousConvolution2D(nbFilter, nbRow, nbCol, KerasUtils.getInitMethod(init), + KerasUtils.getKerasActivation(activation), toScalaArray(subsample), + toScalaArray(atrousRate), KerasUtils.toBigDLFormat(dimOrdering), + wRegularizer, bRegularizer, toScalaShape(inputShape)) + } + + def createKerasDeconvolution2D( + nbFilter: Int, + nbRow: Int, + nbCol: Int, + init: String = "glorot_uniform", + activation: String = null, + subsample: JList[Int], + dimOrdering: String = "th", + wRegularizer: Regularizer[T] = null, + bRegularizer: Regularizer[T] = null, + bias: Boolean = true, + inputShape: JList[Int] = null): Deconvolution2D[T] = { + new Deconvolution2D(nbFilter, nbRow, nbCol, KerasUtils.getInitMethod(init), + KerasUtils.getKerasActivation(activation), toScalaArray(subsample), + KerasUtils.toBigDLFormat(dimOrdering), wRegularizer, bRegularizer, + bias, toScalaShape(inputShape)) + } + + def createKerasConvLSTM2D( + nbFilter: Int, + nbKernel: Int, + activation: String = "tanh", + innerActivation: String = "hard_sigmoid", + dimOrdering: String = "th", + subsample: Int = 1, + wRegularizer: Regularizer[T] = null, + uRegularizer: Regularizer[T] = null, + bRegularizer: Regularizer[T] = null, + returnSequences: Boolean = false, + goBackwards: Boolean = false, + inputShape: JList[Int] = null): ConvLSTM2D[T] = { + ConvLSTM2D(nbFilter, nbKernel, activation, innerActivation, + dimOrdering, subsample, wRegularizer, uRegularizer, bRegularizer, + returnSequences, goBackwards, toScalaShape(inputShape)) + } + + def createKerasLocallyConnected1D( + nbFilter: Int, + filterLength: Int, + activation: String = null, + subsampleLength: Int = 1, + wRegularizer: Regularizer[T] = null, + bRegularizer: Regularizer[T] = null, + bias: Boolean = true, + inputShape: JList[Int] = null): LocallyConnected1D[T] = { + LocallyConnected1D(nbFilter, filterLength, activation, subsampleLength, + wRegularizer, bRegularizer, bias, toScalaShape(inputShape)) + } + + def createKerasLocallyConnected2D( + nbFilter: Int, + nbRow: Int, + nbCol: Int, + activation: String = null, + borderMode: String = "valid", + subsample: JList[Int], + dimOrdering: String = "th", + wRegularizer: Regularizer[T] = null, + bRegularizer: Regularizer[T] = null, + bias: Boolean = true, + inputShape: JList[Int] = null): LocallyConnected2D[T] = { + new LocallyConnected2D(nbFilter, nbRow, nbCol, KerasUtils.getKerasActivation(activation), + borderMode, toScalaArray(subsample), KerasUtils.toBigDLFormat(dimOrdering), + wRegularizer, bRegularizer, bias, toScalaShape(inputShape)) + } + + def createKerasSeparableConvolution2D( + nbFilter: Int, + nbRow: Int, + nbCol: Int, + init: String = "glorot_uniform", + activation: String = null, + borderMode: String = "valid", + subsample: JList[Int], + depthMultiplier: Int = 1, + dimOrdering: String = "th", + depthwiseRegularizer: Regularizer[T] = null, + pointwiseRegularizer: Regularizer[T] = null, + bRegularizer: Regularizer[T] = null, + bias: Boolean = true, + inputShape: JList[Int] = null): SeparableConvolution2D[T] = { + new SeparableConvolution2D(nbFilter, nbRow, nbCol, KerasUtils.getInitMethod(init), + KerasUtils.getKerasActivation(activation), borderMode, toScalaArray(subsample), + depthMultiplier, KerasUtils.toBigDLFormat(dimOrdering), + depthwiseRegularizer, pointwiseRegularizer, bRegularizer, bias, toScalaShape(inputShape)) + } + + def createKerasZeroPadding3D( + padding: JList[Int], + dimOrdering: String = "th", + inputShape: JList[Int] = null): ZeroPadding3D[T] = { + new ZeroPadding3D(toScalaArray(padding), KerasUtils.toBigDLFormat5D(dimOrdering), + toScalaShape(inputShape)) + } + + def createKerasGlobalAveragePooling1D( + inputShape: JList[Int] = null): GlobalAveragePooling1D[T] = { + GlobalAveragePooling1D(toScalaShape(inputShape)) + } + + def createKerasGlobalMaxPooling1D( + inputShape: JList[Int] = null): GlobalMaxPooling1D[T] = { + GlobalMaxPooling1D(toScalaShape(inputShape)) + } + + def createKerasGlobalMaxPooling3D( + dimOrdering: String = "th", + inputShape: JList[Int] = null): GlobalMaxPooling3D[T] = { + GlobalMaxPooling3D(dimOrdering, toScalaShape(inputShape)) + } + + def createKerasGlobalAveragePooling3D( + dimOrdering: String = "th", + inputShape: JList[Int] = null): GlobalAveragePooling3D[T] = { + GlobalAveragePooling3D(dimOrdering, toScalaShape(inputShape)) + } + + def createKerasSpatialDropout1D( + p: Double = 0.5, + inputShape: JList[Int] = null): SpatialDropout1D[T] = { + SpatialDropout1D(p, toScalaShape(inputShape)) + } + + def createKerasSpatialDropout2D( + p: Double = 0.5, + dimOrdering: String = "th", + inputShape: JList[Int] = null): SpatialDropout2D[T] = { + SpatialDropout2D(p, dimOrdering, toScalaShape(inputShape)) + } + + def createKerasSpatialDropout3D( + p: Double = 0.5, + dimOrdering: String = "th", + inputShape: JList[Int] = null): SpatialDropout3D[T] = { + SpatialDropout3D(p, dimOrdering, toScalaShape(inputShape)) + } + + def createKerasGaussianDropout( + p: Double, + inputShape: JList[Int] = null): GaussianDropout[T] = { + GaussianDropout(p, toScalaShape(inputShape)) + } + + def createKerasGaussianNoise( + sigma: Double, + inputShape: JList[Int] = null): GaussianNoise[T] = { + GaussianNoise(sigma, toScalaShape(inputShape)) + } + + def createKerasMasking( + maskValue: Double = 0.0, + inputShape: JList[Int] = null): Masking[T] = { + Masking(maskValue, toScalaShape(inputShape)) + } + + def createKerasSReLU( + tLeftInit: String = "zero", + aLeftInit: String = "glorot_uniform", + tRightInit: String = "glorot_uniform", + aRightInit: String = "one", + sharedAxes: JList[Int] = null, + inputShape: JList[Int] = null): SReLU[T] = { + SReLU(tLeftInit, aLeftInit, tRightInit, aRightInit, + toScalaArray(sharedAxes), toScalaShape(inputShape)) + } + + def createKerasELU( + alpha: Double = 1.0, + inputShape: JList[Int] = null): ELU[T] = { + ELU(alpha, toScalaShape(inputShape)) + } + + def createKerasLeakyReLU( + alpha: Double = 0.01, + inputShape: JList[Int] = null): LeakyReLU[T] = { + LeakyReLU(alpha, toScalaShape(inputShape)) + } + + def createKerasThresholdedReLU( + theta: Double = 1.0, + inputShape: JList[Int] = null): ThresholdedReLU[T] = { + ThresholdedReLU(theta, toScalaShape(inputShape)) + } + + def createKerasTimeDistributed( + layer: KerasLayer[Tensor[T], Tensor[T], T], + inputShape: JList[Int] = null): TimeDistributed[T] = { + TimeDistributed(layer, toScalaShape(inputShape)) + } + + def createKerasBidirectional( + layer: Recurrent[T], + mergeMode: String = "concat", + inputShape: JList[Int] = null): Bidirectional[T] = { + Bidirectional(layer, mergeMode, toScalaShape(inputShape)) + } + + def compile( + module: KerasModel[T], + optimizer: OptimMethod[T], + loss: Criterion[T], + metrics: JList[ValidationMethod[T]] = null): Unit = { + module.compile(optimizer, loss, + if (metrics == null) null else metrics.asScala.toArray) + } + + def fit( + module: KerasModel[T], + x: JavaRDD[Sample], + batchSize: Int = 32, + epochs: Int = 10, + validationData: JavaRDD[Sample] = null): Unit = { + module.fit(toJSample(x), batchSize, epochs, + if (validationData == null) null else toJSample(validationData)) + } + + def fit( + module: KerasModel[T], + x: DataSet[ImageFeature], + batchSize: Int, + epochs: Int, + validationData: DataSet[ImageFeature]): Unit = { + val trainData = x -> ImageFeatureToMiniBatch[T](batchSize) + val valData = + if (validationData != null) validationData -> ImageFeatureToMiniBatch[T](batchSize) + else null + module.fit(trainData, epochs, valData) + } + + def fit( + module: KerasModel[T], + xTrain: JList[JTensor], + yTrain: JTensor, + batchSize: Int, + epochs: Int, + xVal: JList[JTensor], + yVal: JTensor, + localCores: Int): Unit = { + val trainArray = toSampleArray(xTrain.asScala.toList.map{f => toTensor(f)}, toTensor(yTrain)) + val trainData = batching(DataSet.array(trainArray), batchSize) + .asInstanceOf[LocalDataSet[MiniBatch[T]]] + val valData = if (xVal != null && yVal != null) { + val valArray = toSampleArray(xVal.asScala.toList.map{f => toTensor(f)}, toTensor(yVal)) + batching(DataSet.array(valArray), batchSize) + } else null + Engine.setNodeAndCore(1, localCores) + module.fit(trainData, epochs, valData) + } + + def evaluate( + module: KerasModel[T], + x: JavaRDD[Sample], + batchSize: Int = 32): JList[EvaluatedResult] = { + val resultArray = module.evaluate(toJSample(x), batchSize) + val testResultArray = resultArray.map { result => + EvaluatedResult(result._1.result()._1, result._1.result()._2, + result._2.toString()) + } + testResultArray.toList.asJava + } + +} diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/python/api/PythonBigDLOnnx.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/python/api/PythonBigDLOnnx.scala index e755b7972f0..444fa2cb0b7 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/python/api/PythonBigDLOnnx.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/python/api/PythonBigDLOnnx.scala @@ -1,67 +1,67 @@ -///* -// * Copyright 2016 The BigDL Authors. -// * -// * Licensed under the Apache License, Version 2.0 (the "License"); -// * you may not use this file except in compliance with the License. -// * You may obtain a copy of the License at -// * -// * http://www.apache.org/licenses/LICENSE-2.0 -// * -// * Unless required by applicable law or agreed to in writing, software -// * distributed under the License is distributed on an "AS IS" BASIS, -// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// * See the License for the specific language governing permissions and -// * limitations under the License. -// */ -// -//package com.intel.analytics.bigdl.dllib.python.api -// -//import scala.reflect.ClassTag -//import scala.collection.JavaConverters._ -//import java.util.{ArrayList => JArrayList, HashMap => JHashMap, List => JList, Map => JMap} -// -//import com.intel.analytics.bigdl.dllib.nn -//import com.intel.analytics.bigdl.dllib.nn.onnx._ -//import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -// -// -//private[bigdl] object PythonBigDLOnnx { -// -// def ofFloat(): PythonBigDLOnnx[Float] = new PythonBigDLOnnx[Float]() -// -// def ofDouble(): PythonBigDLOnnx[Double] = new PythonBigDLOnnx[Double]() -// -//} -// -// -//class PythonBigDLOnnx[T: ClassTag](implicit ev: TensorNumeric[T]) extends PythonBigDL[T] { -// -// def createConstant(value: JTensor): nn.tf.Const[T, T] = { -// nn.tf.Const[T, T](toTensor(value)) -// } -// -// -// def createGather(): nn.ops.Gather[T, T] = { -// nn.ops.Gather() -// } -// -// -// def createGemm(alpha: Float, beta: Float, transA: Int, transB: Int, -// matrixB: JTensor, matrixC: JTensor): Gemm[T] = { -// Gemm(alpha, beta, -// (if (transA == 0) false else true), -// (if (transB == 0) false else true), -// toTensor(matrixB), toTensor(matrixC)) -// } -// -// -// def createReshape(shape: JArrayList[Int]): nn.onnx.Reshape[T] = { -// nn.onnx.Reshape(if (shape == null) null else shape.asScala.toArray) -// } -// -// -// def createShape(): Shape[T] = { -// Shape[T]() -// } -// -//} +/* + * Copyright 2016 The BigDL Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.intel.analytics.bigdl.dllib.utils.python.api + +import scala.reflect.ClassTag +import scala.collection.JavaConverters._ +import java.util.{ArrayList => JArrayList, HashMap => JHashMap, List => JList, Map => JMap} + +import com.intel.analytics.bigdl.dllib.nn +import com.intel.analytics.bigdl.dllib.nn.onnx._ +import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric + + +private[bigdl] object PythonBigDLOnnx { + + def ofFloat(): PythonBigDLOnnx[Float] = new PythonBigDLOnnx[Float]() + + def ofDouble(): PythonBigDLOnnx[Double] = new PythonBigDLOnnx[Double]() + +} + + +class PythonBigDLOnnx[T: ClassTag](implicit ev: TensorNumeric[T]) extends PythonBigDL[T] { + + def createConstant(value: JTensor): nn.tf.Const[T, T] = { + nn.tf.Const[T, T](toTensor(value)) + } + + + def createGather(): nn.ops.Gather[T, T] = { + nn.ops.Gather() + } + + + def createGemm(alpha: Float, beta: Float, transA: Int, transB: Int, + matrixB: JTensor, matrixC: JTensor): Gemm[T] = { + Gemm(alpha, beta, + (if (transA == 0) false else true), + (if (transB == 0) false else true), + toTensor(matrixB), toTensor(matrixC)) + } + + + def createReshape(shape: JArrayList[Int]): nn.onnx.Reshape[T] = { + nn.onnx.Reshape(if (shape == null) null else shape.asScala.toArray) + } + + + def createShape(): Shape[T] = { + Shape[T]() + } + +} diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/python/api/PythonBigDLValidator.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/python/api/PythonBigDLValidator.scala index e2c4a236fe9..58a654cbb15 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/python/api/PythonBigDLValidator.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/python/api/PythonBigDLValidator.scala @@ -1,81 +1,81 @@ -///* -// * Copyright 2016 The BigDL Authors. -// * -// * Licensed under the Apache License, Version 2.0 (the "License"); -// * you may not use this file except in compliance with the License. -// * You may obtain a copy of the License at -// * -// * http://www.apache.org/licenses/LICENSE-2.0 -// * -// * Unless required by applicable law or agreed to in writing, software -// * distributed under the License is distributed on an "AS IS" BASIS, -// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// * See the License for the specific language governing permissions and -// * limitations under the License. -// */ -// -//package com.intel.analytics.bigdl.dllib.python.api -// -//import java.lang.{Boolean => JBoolean} -//import java.util.{ArrayList => JArrayList, HashMap => JHashMap, List => JList, Map => JMap} -// -//import com.intel.analytics.bigdl.dllib.tensor.Tensor -//import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -//import com.intel.analytics.bigdl.dllib.utils.Table -// -//import scala.collection.JavaConverters._ -//import scala.collection.mutable.Map -//import scala.language.existentials -//import scala.reflect.ClassTag -// -//object PythonBigDLValidator { -// -// def ofFloat(): PythonBigDLValidator[Float] = new PythonBigDLValidator[Float]() -// -// def ofDouble(): PythonBigDLValidator[Double] = new PythonBigDLValidator[Double]() -//} -// -//class PythonBigDLValidator[T: ClassTag](implicit ev: TensorNumeric[T]) extends PythonBigDL[T]{ -// -// def testDict(): JMap[String, String] = { -// return Map("jack" -> "40", "lucy" -> "50").asJava -// } -// -// def testDictJTensor(): JMap[String, JTensor] = { -// return Map("jack" -> JTensor(Array(1.0f, 2.0f, 3.0f, 4.0f), Array(4, 1), "float")).asJava -// } -// -// def testDictJMapJTensor(): JMap[String, JMap[String, JTensor]] = { -// val table = new Table() -// val tensor = JTensor(Array(1.0f, 2.0f, 3.0f, 4.0f), Array(4, 1), "float") -// val result = Map("jack" -> tensor).asJava -// table.insert(tensor) -// return Map("nested" -> result).asJava -// } -// -// def testActivityWithTensor(): JActivity = { -// val tensor = Tensor(Array(1.0f, 2.0f, 3.0f, 4.0f), Array(4, 1)) -// return JActivity(tensor) -// } -// -// def testActivityWithTableOfTensor(): JActivity = { -// val tensor1 = Tensor(Array(1.0f, 1.0f), Array(2)) -// val tensor2 = Tensor(Array(2.0f, 2.0f), Array(2)) -// val tensor3 = Tensor(Array(3.0f, 3.0f), Array(2)) -// val table = new Table() -// table.insert(tensor1) -// table.insert(tensor2) -// table.insert(tensor3) -// return JActivity(table) -// } -// -// def testActivityWithTableOfTable(): JActivity = { -// val tensor = Tensor(Array(1.0f, 2.0f, 3.0f, 4.0f), Array(4, 1)) -// val table = new Table() -// table.insert(tensor) -// val nestedTable = new Table() -// nestedTable.insert(table) -// nestedTable.insert(table) -// return JActivity(nestedTable) -// } -//} +/* + * Copyright 2016 The BigDL Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.intel.analytics.bigdl.dllib.utils.python.api + +import java.lang.{Boolean => JBoolean} +import java.util.{ArrayList => JArrayList, HashMap => JHashMap, List => JList, Map => JMap} + +import com.intel.analytics.bigdl.dllib.tensor.Tensor +import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric +import com.intel.analytics.bigdl.dllib.utils.Table + +import scala.collection.JavaConverters._ +import scala.collection.mutable.Map +import scala.language.existentials +import scala.reflect.ClassTag + +object PythonBigDLValidator { + + def ofFloat(): PythonBigDLValidator[Float] = new PythonBigDLValidator[Float]() + + def ofDouble(): PythonBigDLValidator[Double] = new PythonBigDLValidator[Double]() +} + +class PythonBigDLValidator[T: ClassTag](implicit ev: TensorNumeric[T]) extends PythonBigDL[T]{ + + def testDict(): JMap[String, String] = { + return Map("jack" -> "40", "lucy" -> "50").asJava + } + + def testDictJTensor(): JMap[String, JTensor] = { + return Map("jack" -> JTensor(Array(1.0f, 2.0f, 3.0f, 4.0f), Array(4, 1), "float")).asJava + } + + def testDictJMapJTensor(): JMap[String, JMap[String, JTensor]] = { + val table = new Table() + val tensor = JTensor(Array(1.0f, 2.0f, 3.0f, 4.0f), Array(4, 1), "float") + val result = Map("jack" -> tensor).asJava + table.insert(tensor) + return Map("nested" -> result).asJava + } + + def testActivityWithTensor(): JActivity = { + val tensor = Tensor(Array(1.0f, 2.0f, 3.0f, 4.0f), Array(4, 1)) + return JActivity(tensor) + } + + def testActivityWithTableOfTensor(): JActivity = { + val tensor1 = Tensor(Array(1.0f, 1.0f), Array(2)) + val tensor2 = Tensor(Array(2.0f, 2.0f), Array(2)) + val tensor3 = Tensor(Array(3.0f, 3.0f), Array(2)) + val table = new Table() + table.insert(tensor1) + table.insert(tensor2) + table.insert(tensor3) + return JActivity(table) + } + + def testActivityWithTableOfTable(): JActivity = { + val tensor = Tensor(Array(1.0f, 2.0f, 3.0f, 4.0f), Array(4, 1)) + val table = new Table() + table.insert(tensor) + val nestedTable = new Table() + nestedTable.insert(table) + nestedTable.insert(table) + return JActivity(nestedTable) + } +} diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/tf/TensorflowLoader.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/tf/TensorflowLoader.scala index 54c910b9358..aa312c5e734 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/tf/TensorflowLoader.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/tf/TensorflowLoader.scala @@ -26,7 +26,7 @@ import com.intel.analytics.bigdl.Module import com.intel.analytics.bigdl.dllib.nn.Graph import com.intel.analytics.bigdl.dllib.nn.abstractnn.{AbstractModule, Activity} import com.intel.analytics.bigdl.dllib.nn.tf.AssignGrad -import com.intel.analytics.bigdl.dllib.python.api.{JTensor, PythonBigDL, PythonBigDLUtils} +import com.intel.analytics.bigdl.dllib.utils.python.api.{JTensor, PythonBigDL, PythonBigDLUtils} import com.intel.analytics.bigdl.dllib.tensor.{DoubleType, FloatType, Tensor} import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric import com.intel.analytics.bigdl.dllib.nn.tf.{SwitchControlNode, SwitchOps} diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/python/api/PythonSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/python/api/PythonSpec.scala index b34cbec6b01..afa4469e53f 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/python/api/PythonSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/python/api/PythonSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.dllib.python.api +package com.intel.analytics.bigdl.dllib.utils.python.api import java.util import java.util.{ArrayList => JArrayList, List => JList, Map => JMap} From d849c2550f71a381a954b578e4055512a920944d Mon Sep 17 00:00:00 2001 From: dding3 Date: Mon, 16 Aug 2021 15:36:14 -0700 Subject: [PATCH 799/823] remove uncessary file --- python/.idea/misc.xml | 4 - python/.idea/modules.xml | 8 - python/.idea/python.iml | 12 - python/.idea/workspace.xml | 851 ------------------------------------- python/run_lenet.sh | 18 - 5 files changed, 893 deletions(-) delete mode 100644 python/.idea/misc.xml delete mode 100644 python/.idea/modules.xml delete mode 100644 python/.idea/python.iml delete mode 100644 python/.idea/workspace.xml delete mode 100755 python/run_lenet.sh diff --git a/python/.idea/misc.xml b/python/.idea/misc.xml deleted file mode 100644 index 448b4d3663f..00000000000 --- a/python/.idea/misc.xml +++ /dev/null @@ -1,4 +0,0 @@ - - - - \ No newline at end of file diff --git a/python/.idea/modules.xml b/python/.idea/modules.xml deleted file mode 100644 index 614b3c13ff5..00000000000 --- a/python/.idea/modules.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - - - - - - \ No newline at end of file diff --git a/python/.idea/python.iml b/python/.idea/python.iml deleted file mode 100644 index e98082abea8..00000000000 --- a/python/.idea/python.iml +++ /dev/null @@ -1,12 +0,0 @@ - - - - - - - - - - \ No newline at end of file diff --git a/python/.idea/workspace.xml b/python/.idea/workspace.xml deleted file mode 100644 index af60d1ae9da..00000000000 --- a/python/.idea/workspace.xml +++ /dev/null @@ -1,851 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - bigdl - bigdl.d - bigdl.da - bigdl.data - bigdl.datas - bigdl.dataset - bigdl.nn.layer - bigdl.nn.onnx.layer - bigdl.transform - bigdl.optim - bigdl.util.common - bigdl.keras.backend - bigdl.keras.backen - bigdl.keras.backe - bigdl.keras.back - bigdl.keras.bac - bigdl.keras.ba - bigdl.keras.b - bigdl.keras. - bigdl.keras - bigdl.examples - bigdl.contrib - import - bigdl.nn - bigdl.models - bigdl. - jar_dir - get_bigdl_classpath - spark_profile - make - - - BIGDL_HOME - bigdl.dllib.feature.dataset - bigdl.dllib.nn.layer - bigdl.dllib.nn.onnx.layer - bigdl.dllib.feature.transform - bigdl.dllib.optim - bigdl.utils.common - bigdl.dllib.keras - bigdl.dllib.examples - bigdl.dllib.contrib - bigdl.dllib.nn - bigdl.dllib.models - BIGDL_PYTHON_DIR - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 1628804636938 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/python/run_lenet.sh b/python/run_lenet.sh deleted file mode 100755 index fd45228b04a..00000000000 --- a/python/run_lenet.sh +++ /dev/null @@ -1,18 +0,0 @@ -SPARK_HOME=/home/ding/Downloads/spark-2.4.3-bin-hadoop2.7 -MASTER=local[2] -PYTHON_API_ZIP_PATH=/home/ding/proj/clone-ding-zoo/analytics-zoo/dist/lib/bigdl-dllib-2.0.0-SNAPSHOT-python-api.zip -BigDL_JAR_PATH=/home/ding/proj/clone-ding-zoo/analytics-zoo/dist/lib/bigdl-dllib-2.0.0-SNAPSHOT-jar-with-dependencies.jar -PYTHONPATH=${PYTHON_API_ZIP_PATH}:$PYTHONPATH -${SPARK_HOME}/bin/spark-submit \ - --master ${MASTER} \ - --driver-cores 2 \ - --driver-memory 2g \ - --total-executor-cores 2 \ - --executor-cores 2 \ - --executor-memory 4g \ - --py-files ${PYTHON_API_ZIP_PATH},/home/ding/proj/clone-ding-zoo/analytics-zoo/python/bigdl/dllib/examples/lenet/lenet.py \ - --properties-file /home/ding/proj/clone-ding-zoo/analytics-zoo/scala/dllib/src/main/resources/spark-bigdl.conf \ - --jars ${BigDL_JAR_PATH} \ - --conf spark.driver.extraClassPath=${BigDL_JAR_PATH} \ - --conf spark.executor.extraClassPath=/home/ding/proj/clone-ding-zoo/analytics-zoo/dist/lib/bigdl-dllib-2.0.0-SNAPSHOT-jar-with-dependencies.jar \ - /home/ding/proj/clone-ding-zoo/analytics-zoo/python/bigdl/dllib/examples/lenet/lenet.py From 1a41b3f34b4cf8ec199abe12fb158d45c0e834c1 Mon Sep 17 00:00:00 2001 From: dding3 Date: Mon, 16 Aug 2021 15:36:14 -0700 Subject: [PATCH 800/823] remove uncessary file --- python/.idea/misc.xml | 4 - python/.idea/modules.xml | 8 - python/.idea/python.iml | 12 - python/.idea/workspace.xml | 851 ------------------------------------- python/run_lenet.sh | 18 - 5 files changed, 893 deletions(-) delete mode 100644 python/.idea/misc.xml delete mode 100644 python/.idea/modules.xml delete mode 100644 python/.idea/python.iml delete mode 100644 python/.idea/workspace.xml delete mode 100755 python/run_lenet.sh diff --git a/python/.idea/misc.xml b/python/.idea/misc.xml deleted file mode 100644 index 448b4d3663f..00000000000 --- a/python/.idea/misc.xml +++ /dev/null @@ -1,4 +0,0 @@ - - - - \ No newline at end of file diff --git a/python/.idea/modules.xml b/python/.idea/modules.xml deleted file mode 100644 index 614b3c13ff5..00000000000 --- a/python/.idea/modules.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - - - - - - \ No newline at end of file diff --git a/python/.idea/python.iml b/python/.idea/python.iml deleted file mode 100644 index e98082abea8..00000000000 --- a/python/.idea/python.iml +++ /dev/null @@ -1,12 +0,0 @@ - - - - - - - - - - \ No newline at end of file diff --git a/python/.idea/workspace.xml b/python/.idea/workspace.xml deleted file mode 100644 index af60d1ae9da..00000000000 --- a/python/.idea/workspace.xml +++ /dev/null @@ -1,851 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - bigdl - bigdl.d - bigdl.da - bigdl.data - bigdl.datas - bigdl.dataset - bigdl.nn.layer - bigdl.nn.onnx.layer - bigdl.transform - bigdl.optim - bigdl.util.common - bigdl.keras.backend - bigdl.keras.backen - bigdl.keras.backe - bigdl.keras.back - bigdl.keras.bac - bigdl.keras.ba - bigdl.keras.b - bigdl.keras. - bigdl.keras - bigdl.examples - bigdl.contrib - import - bigdl.nn - bigdl.models - bigdl. - jar_dir - get_bigdl_classpath - spark_profile - make - - - BIGDL_HOME - bigdl.dllib.feature.dataset - bigdl.dllib.nn.layer - bigdl.dllib.nn.onnx.layer - bigdl.dllib.feature.transform - bigdl.dllib.optim - bigdl.utils.common - bigdl.dllib.keras - bigdl.dllib.examples - bigdl.dllib.contrib - bigdl.dllib.nn - bigdl.dllib.models - BIGDL_PYTHON_DIR - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 1628804636938 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/python/run_lenet.sh b/python/run_lenet.sh deleted file mode 100755 index fd45228b04a..00000000000 --- a/python/run_lenet.sh +++ /dev/null @@ -1,18 +0,0 @@ -SPARK_HOME=/home/ding/Downloads/spark-2.4.3-bin-hadoop2.7 -MASTER=local[2] -PYTHON_API_ZIP_PATH=/home/ding/proj/clone-ding-zoo/analytics-zoo/dist/lib/bigdl-dllib-2.0.0-SNAPSHOT-python-api.zip -BigDL_JAR_PATH=/home/ding/proj/clone-ding-zoo/analytics-zoo/dist/lib/bigdl-dllib-2.0.0-SNAPSHOT-jar-with-dependencies.jar -PYTHONPATH=${PYTHON_API_ZIP_PATH}:$PYTHONPATH -${SPARK_HOME}/bin/spark-submit \ - --master ${MASTER} \ - --driver-cores 2 \ - --driver-memory 2g \ - --total-executor-cores 2 \ - --executor-cores 2 \ - --executor-memory 4g \ - --py-files ${PYTHON_API_ZIP_PATH},/home/ding/proj/clone-ding-zoo/analytics-zoo/python/bigdl/dllib/examples/lenet/lenet.py \ - --properties-file /home/ding/proj/clone-ding-zoo/analytics-zoo/scala/dllib/src/main/resources/spark-bigdl.conf \ - --jars ${BigDL_JAR_PATH} \ - --conf spark.driver.extraClassPath=${BigDL_JAR_PATH} \ - --conf spark.executor.extraClassPath=/home/ding/proj/clone-ding-zoo/analytics-zoo/dist/lib/bigdl-dllib-2.0.0-SNAPSHOT-jar-with-dependencies.jar \ - /home/ding/proj/clone-ding-zoo/analytics-zoo/python/bigdl/dllib/examples/lenet/lenet.py From 48bb8e64863f7683cd7e82427ad36743f7336444 Mon Sep 17 00:00:00 2001 From: dding3 Date: Mon, 16 Aug 2021 21:01:18 -0700 Subject: [PATCH 801/823] update test scripts --- python/dllib/src/bigdl/__init__.py | 18 +++++ python/dllib/src/bigdl/dllib/__init__.py | 15 ++++ python/dllib/src/bigdl/dllib/nn/layer.py | 4 +- python/dllib/src/bigdl/version.py | 17 +++++ .../test/bigdl/caffe/test_caffe_layers.py | 10 +-- .../dllib/test/bigdl/caffe/test_load_caffe.py | 6 +- .../test/bigdl/keras/test_application.py | 4 +- python/dllib/test/bigdl/keras/test_backend.py | 8 +-- .../dllib/test/bigdl/keras/test_keras_api.py | 10 +-- python/dllib/test/bigdl/keras/test_layer.py | 4 +- .../dllib/test/bigdl/keras/test_load_model.py | 8 +-- python/dllib/test/bigdl/keras/test_loss.py | 2 +- python/dllib/test/bigdl/onnx/test_onnx_ops.py | 12 ++-- python/dllib/test/bigdl/test_engine_env.py | 4 +- python/dllib/test/bigdl/test_pickler.py | 20 +++--- .../test/bigdl/test_simple_integration.py | 44 ++++++------ python/dllib/test/bigdl/test_tensorflow.py | 6 +- python/dllib/test/bigdl/test_utils.py | 12 ++-- .../dllib/test/bigdl/transform/test_image.py | 4 +- python/dllib/test/dev/prepare_env.sh | 4 +- python/dllib/test/dev/release/release.sh | 27 ++----- python/dllib/test/dev/run-caffe.sh | 4 +- python/dllib/test/dev/run-keras.sh | 2 +- python/dllib/test/dev/run-tests | 22 +++--- python/release.sh | 70 ------------------- 25 files changed, 152 insertions(+), 185 deletions(-) create mode 100644 python/dllib/src/bigdl/__init__.py create mode 100644 python/dllib/src/bigdl/dllib/__init__.py create mode 100644 python/dllib/src/bigdl/version.py delete mode 100755 python/release.sh diff --git a/python/dllib/src/bigdl/__init__.py b/python/dllib/src/bigdl/__init__.py new file mode 100644 index 00000000000..7f835104b9e --- /dev/null +++ b/python/dllib/src/bigdl/__init__.py @@ -0,0 +1,18 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from bigdl.utils.engine import prepare_env +prepare_env() diff --git a/python/dllib/src/bigdl/dllib/__init__.py b/python/dllib/src/bigdl/dllib/__init__.py new file mode 100644 index 00000000000..2151a805423 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/__init__.py @@ -0,0 +1,15 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index bfa012e690f..5f36b75bc60 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -745,7 +745,7 @@ def __init__(self, bigdl_type, "createModelPreprocessor", inputs, outputs) self.bigdl_type = bigdl_type else: - from bigdl.util.tf_utils import convert + from bigdl.utils.tf_utils import convert model = convert(to_list(inputs), to_list(outputs), byte_order, bigdl_type) super(Model, self).__init__(model.value, bigdl_type) @@ -863,7 +863,7 @@ def load_tensorflow(path, inputs, outputs, byte_order = "little_endian", @staticmethod def train(output, data, label, opt_method, criterion, batch_size, end_when, session=None, bigdl_type="float"): - from bigdl.util.tf_utils import get_path + from bigdl.utils.tf_utils import get_path from bigdl.utils.common import Sample output_name = output.name.split(":")[0] path = get_path(output_name, session) diff --git a/python/dllib/src/bigdl/version.py b/python/dllib/src/bigdl/version.py new file mode 100644 index 00000000000..3d92def3019 --- /dev/null +++ b/python/dllib/src/bigdl/version.py @@ -0,0 +1,17 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +__version__ = "2.0.0.dev0" diff --git a/python/dllib/test/bigdl/caffe/test_caffe_layers.py b/python/dllib/test/bigdl/caffe/test_caffe_layers.py index 6623addf0d6..640a6f8efc3 100644 --- a/python/dllib/test/bigdl/caffe/test_caffe_layers.py +++ b/python/dllib/test/bigdl/caffe/test_caffe_layers.py @@ -14,9 +14,9 @@ # limitations under the License. # -from bigdl.nn.layer import * -from bigdl.optim.optimizer import * -from bigdl.util.common import * +from bigdl.dllib.nn.layer import * +from bigdl.dllib.optim.optimizer import * +from bigdl.utils.common import * import numpy as np import pytest import tempfile @@ -56,8 +56,8 @@ def test_caffe_layers(self): if len(inputs) == 1: inputs = inputs[0] bigdlResult = model.forward(inputs) - print cafferesult - print bigdlResult + print(cafferesult) + print(bigdlResult) assert_allclose(cafferesult, bigdlResult, atol=1e-4, rtol=0) if __name__ == "__main__": diff --git a/python/dllib/test/bigdl/caffe/test_load_caffe.py b/python/dllib/test/bigdl/caffe/test_load_caffe.py index 6067082c695..d71039fb948 100644 --- a/python/dllib/test/bigdl/caffe/test_load_caffe.py +++ b/python/dllib/test/bigdl/caffe/test_load_caffe.py @@ -14,9 +14,9 @@ # limitations under the License. # -from bigdl.nn.layer import * -from bigdl.optim.optimizer import * -from bigdl.util.common import * +from bigdl.dllib.nn.layer import * +from bigdl.dllib.optim.optimizer import * +from bigdl.utils.common import * import numpy as np import pytest from numpy.testing import assert_allclose diff --git a/python/dllib/test/bigdl/keras/test_application.py b/python/dllib/test/bigdl/keras/test_application.py index 0d9a4587608..10f5172b864 100644 --- a/python/dllib/test/bigdl/keras/test_application.py +++ b/python/dllib/test/bigdl/keras/test_application.py @@ -20,11 +20,11 @@ np.random.seed(1337) # for reproducibility import pytest from keras.applications import * -from bigdl.keras.converter import * +from bigdl.dllib.keras.converter import * from keras.applications.music_tagger_crnn import MusicTaggerCRNN from test.bigdl.test_utils import BigDLTestCase, TestModels -from bigdl.keras.backend import * +from bigdl.dllib.keras.backend import * class TestApplication(BigDLTestCase): diff --git a/python/dllib/test/bigdl/keras/test_backend.py b/python/dllib/test/bigdl/keras/test_backend.py index 96fddbb927a..97d5c53bf63 100644 --- a/python/dllib/test/bigdl/keras/test_backend.py +++ b/python/dllib/test/bigdl/keras/test_backend.py @@ -20,8 +20,8 @@ np.random.seed(1337) # for reproducibility import pytest from keras.applications import * -from bigdl.keras.converter import * -from bigdl.keras.backend import * +from bigdl.dllib.keras.converter import * +from bigdl.dllib.keras.backend import * from test.bigdl.test_utils import BigDLTestCase, TestModels @@ -73,8 +73,8 @@ def test_lenet_distributed_ndarray(self): def test_lenet_distributed_rdd(self): kmodel, X_train, y_train = TestModels.kmodel_seq_lenet_mnist() sc = get_spark_context() - from bigdl.util.common import Sample - from bigdl.util.common import to_sample_rdd + from bigdl.utils.common import Sample + from bigdl.utils.common import to_sample_rdd training_rdd = to_sample_rdd(X_train, y_train) self.modelTest(X_train, kmodel, dump_weights=True) diff --git a/python/dllib/test/bigdl/keras/test_keras_api.py b/python/dllib/test/bigdl/keras/test_keras_api.py index fefdf797f59..37a79b4fc86 100644 --- a/python/dllib/test/bigdl/keras/test_keras_api.py +++ b/python/dllib/test/bigdl/keras/test_keras_api.py @@ -17,13 +17,13 @@ import pytest from test.bigdl.test_utils import BigDLTestCase -import bigdl.nn.keras.layer as BLayer +import bigdl.dllib.keras.layer as BLayer import keras.layers as KLayer import keras.backend as K -from bigdl.keras.converter import WeightsConverter +from bigdl.dllib.keras.converter import WeightsConverter from bigdl.dataset.dataset import * -from bigdl.nn.keras.topology import Model as BModel -from bigdl.nn.keras.topology import Sequential as BSequential +from bigdl.dllib.keras.topology import Model as BModel +from bigdl.dllib.keras.topology import Sequential as BSequential from keras.engine import merge as kmerge, Model as KModel from keras.models import Sequential as KSequential @@ -124,7 +124,7 @@ def test_merge_cos(self): self.compare_newapi(klayer, blayer, input_data) def test_lenet_shape(self): - from bigdl.examples.lenet.lenet import build_model + from bigdl.dllib.examples.lenet.lenet import build_model model = build_model(10) input_shape = model.get_input_shape() np.testing.assert_allclose((28, 28, 1), input_shape[1:]) diff --git a/python/dllib/test/bigdl/keras/test_layer.py b/python/dllib/test/bigdl/keras/test_layer.py index 18a38849a60..8f3dd2a3f70 100644 --- a/python/dllib/test/bigdl/keras/test_layer.py +++ b/python/dllib/test/bigdl/keras/test_layer.py @@ -23,9 +23,9 @@ from keras.layers.convolutional import * from keras.layers import Dense, Input from keras.regularizers import l1, l2, l1l2 -from bigdl.keras.converter import * +from bigdl.dllib.keras.converter import * from test.bigdl.test_utils import BigDLTestCase -from bigdl.examples.keras.keras_utils import * +from bigdl.dllib.examples.keras.keras_utils import * class TestLayer(BigDLTestCase): diff --git a/python/dllib/test/bigdl/keras/test_load_model.py b/python/dllib/test/bigdl/keras/test_load_model.py index 0be15f22b02..52b7f7cd8ff 100644 --- a/python/dllib/test/bigdl/keras/test_load_model.py +++ b/python/dllib/test/bigdl/keras/test_load_model.py @@ -19,13 +19,13 @@ import pytest from numpy.testing import assert_allclose -import bigdl.nn.layer as BLayer -from bigdl.keras.converter import WeightLoader -from bigdl.keras.converter import DefinitionLoader +import bigdl.dllib.nn.layer as BLayer +from bigdl.dllib.keras.converter import WeightLoader +from bigdl.dllib.keras.converter import DefinitionLoader np.random.seed(1337) # for reproducibility from test.bigdl.test_utils import BigDLTestCase, TestModels -from bigdl.examples.keras.keras_utils import * +from bigdl.dllib.examples.keras.keras_utils import * import keras.backend as K diff --git a/python/dllib/test/bigdl/keras/test_loss.py b/python/dllib/test/bigdl/keras/test_loss.py index c06ad5ef9a0..42990c20751 100644 --- a/python/dllib/test/bigdl/keras/test_loss.py +++ b/python/dllib/test/bigdl/keras/test_loss.py @@ -19,7 +19,7 @@ import pytest from keras import objectives -from bigdl.keras.optimization import OptimConverter +from bigdl.dllib.keras.optimization import OptimConverter from test.bigdl.test_utils import BigDLTestCase np.random.seed(1337) # for reproducibility diff --git a/python/dllib/test/bigdl/onnx/test_onnx_ops.py b/python/dllib/test/bigdl/onnx/test_onnx_ops.py index a5205a79ce3..4ace83b5bf1 100644 --- a/python/dllib/test/bigdl/onnx/test_onnx_ops.py +++ b/python/dllib/test/bigdl/onnx/test_onnx_ops.py @@ -18,12 +18,12 @@ import pytest import numpy as np -from bigdl.contrib.onnx.onnx_loader import load_model_proto -from bigdl.nn.layer import CAddTable, JoinTable, ReLU -from bigdl.nn.layer import SoftMax, SpatialAveragePooling, SpatialBatchNormalization -from bigdl.nn.layer import SpatialConvolution, SpatialMaxPooling -from bigdl.nn.layer import Unsqueeze -from bigdl.nn.onnx.layer import Gemm, Reshape, Shape +from bigdl.dllib.contrib.onnx.onnx_loader import load_model_proto +from bigdl.dllib.nn.layer import CAddTable, JoinTable, ReLU +from bigdl.dllib.nn.layer import SoftMax, SpatialAveragePooling, SpatialBatchNormalization +from bigdl.dllib.nn.layer import SpatialConvolution, SpatialMaxPooling +from bigdl.dllib.nn.layer import Unsqueeze +from bigdl.dllib.nn.onnx.layer import Gemm, Reshape, Shape class TestAveragePool(object): diff --git a/python/dllib/test/bigdl/test_engine_env.py b/python/dllib/test/bigdl/test_engine_env.py index 80e8e4f12d8..dc824267356 100644 --- a/python/dllib/test/bigdl/test_engine_env.py +++ b/python/dllib/test/bigdl/test_engine_env.py @@ -16,7 +16,7 @@ import pytest import os -from bigdl.util.common import * +from bigdl.utils.common import * class TestEngineEnv(): @@ -38,7 +38,7 @@ def test___prepare_bigdl_env(self): # adding jar path message, just do prepare_env()' again # to see if the log is correct and the environment variables should not vary. - from bigdl.util.engine import prepare_env + from bigdl.utils.engine import prepare_env bigdl_jars_env_1 = os.environ.get("BIGDL_JARS", None) spark_class_path_1 = os.environ.get("SPARK_CLASSPATH", None) diff --git a/python/dllib/test/bigdl/test_pickler.py b/python/dllib/test/bigdl/test_pickler.py index a3e41eec34f..69e5f92cafa 100644 --- a/python/dllib/test/bigdl/test_pickler.py +++ b/python/dllib/test/bigdl/test_pickler.py @@ -14,19 +14,19 @@ # limitations under the License. # -from bigdl.nn.layer import * -from bigdl.nn.initialization_method import * -from bigdl.nn.criterion import * -from bigdl.optim.optimizer import * -from bigdl.util.common import * -from bigdl.util.common import _py2java -from bigdl.nn.initialization_method import * -from bigdl.dataset import movielens +from bigdl.dllib.nn.layer import * +from bigdl.dllib.nn.initialization_method import * +from bigdl.dllib.nn.criterion import * +from bigdl.dllib.optim.optimizer import * +from bigdl.utils.common import * +from bigdl.utils.common import _py2java +from bigdl.dllib.nn.initialization_method import * +from bigdl.dllib.feature.dataset import movielens import numpy as np import tempfile import pytest from numpy.testing import assert_allclose, assert_array_equal -from bigdl.util.engine import compare_version +from bigdl.utils.engine import compare_version np.random.seed(1337) # for reproducibility @@ -35,7 +35,7 @@ def setup_method(self, method): """ setup any state tied to the execution of the given method in a class. setup_method is invoked for every test method of a class. """ - JavaCreator.add_creator_class("com.intel.analytics.bigdl.python.api.PythonBigDLValidator") + JavaCreator.add_creator_class("com.intel.analytics.bigdl.dllib.utils.python.api.PythonBigDLValidator") sparkConf = create_spark_conf().setMaster("local[4]").setAppName("test model") self.sc = get_spark_context(sparkConf) init_engine() diff --git a/python/dllib/test/bigdl/test_simple_integration.py b/python/dllib/test/bigdl/test_simple_integration.py index 4b47df29f2c..cb49fd6213d 100644 --- a/python/dllib/test/bigdl/test_simple_integration.py +++ b/python/dllib/test/bigdl/test_simple_integration.py @@ -14,22 +14,22 @@ # limitations under the License. # -from bigdl.nn.layer import * -from bigdl.nn.initialization_method import * -from bigdl.nn.criterion import * -from bigdl.optim.optimizer import * -from bigdl.util.common import * -from bigdl.util.common import _py2java -from bigdl.nn.initialization_method import * -from bigdl.dataset import movielens +from bigdl.dllib.nn.layer import * +from bigdl.dllib.nn.initialization_method import * +from bigdl.dllib.nn.criterion import * +from bigdl.dllib.optim.optimizer import * +from bigdl.utils.common import * +from bigdl.utils.common import _py2java +from bigdl.dllib.nn.initialization_method import * +from bigdl.dllib.feature.dataset import movielens import numpy as np import tempfile import pytest from numpy.testing import assert_allclose, assert_array_equal -from bigdl.util.engine import compare_version -from bigdl.transform.vision.image import * -from bigdl.models.utils.model_broadcast import broadcast_model -from bigdl.dataset.dataset import * +from bigdl.utils.engine import compare_version +from bigdl.dllib.feature.transform.vision.image import * +from bigdl.dllib.models.utils.model_broadcast import broadcast_model +from bigdl.dllib.feature.dataset.dataset import * np.random.seed(1337) # for reproducibility @@ -77,9 +77,9 @@ def grad_update(mlp, x, y, criterion, learning_rate): rtol=1.e-1) def test_load_keras_model_of(self): - from bigdl.nn.keras.topology import Model as KModel - from bigdl.nn.keras.layer import Input as KInput - from bigdl.nn.keras.layer import Dense + from bigdl.dllib.keras.topology import Model as KModel + from bigdl.dllib.keras.layer import Input as KInput + from bigdl.dllib.keras.layer import Dense input = KInput(shape=[2, 3]) fc1 = Dense(2)(input) @@ -87,12 +87,12 @@ def test_load_keras_model_of(self): tmp_path = tempfile.mktemp() model.save(tmp_path, True) model_loaded = KModel.load(tmp_path) - assert "bigdl.nn.keras.topology.Model" in str(type(model_loaded)) + assert "bigdl.dllib.keras.topology.Model" in str(type(model_loaded)) assert len(model_loaded.layers) == 2 def test_load_keras_seq_of(self): - from bigdl.nn.keras.topology import Sequential as KSequential - from bigdl.nn.keras.layer import Dense + from bigdl.dllib.keras.topology import Sequential as KSequential + from bigdl.dllib.keras.layer import Dense model = KSequential() fc1 = Dense(2, input_shape=[2, 3]) @@ -100,7 +100,7 @@ def test_load_keras_seq_of(self): tmp_path = tempfile.mktemp() model.save(tmp_path, True) model_loaded = KSequential.load(tmp_path) - assert "bigdl.nn.keras.topology.Sequential" in str(type(model_loaded)) + assert "bigdl.dllib.keras.topology.Sequential" in str(type(model_loaded)) assert len(model_loaded.layers) == 1 def test_load_model_of(self): @@ -244,7 +244,7 @@ def test_graph_preprocessor(self): np.array([1.0])) def test_load_zip_conf(self): - from bigdl.util.common import get_bigdl_conf + from bigdl.utils.common import get_bigdl_conf import sys sys_path_back = sys.path sys.path = [path for path in sys.path if "spark-bigdl.conf" not in path] @@ -420,7 +420,7 @@ def gen_rand_sample(): optimizer.optimize() def test_forward_backward(self): - from bigdl.nn.layer import Linear + from bigdl.dllib.nn.layer import Linear rng = RNG() rng.set_seed(100) @@ -442,7 +442,7 @@ def test_forward_backward(self): l_grad_output = linear.backward(input, grad_output) def test_forward_multiple(self): - from bigdl.nn.layer import Linear + from bigdl.dllib.nn.layer import Linear rng = RNG() rng.set_seed(100) diff --git a/python/dllib/test/bigdl/test_tensorflow.py b/python/dllib/test/bigdl/test_tensorflow.py index aedbd90cdb5..2d8e3f1918f 100644 --- a/python/dllib/test/bigdl/test_tensorflow.py +++ b/python/dllib/test/bigdl/test_tensorflow.py @@ -14,9 +14,9 @@ # limitations under the License. # -from bigdl.nn.layer import * -from bigdl.optim.optimizer import * -from bigdl.util.common import * +from bigdl.dllib.nn.layer import * +from bigdl.dllib.optim.optimizer import * +from bigdl.utils.common import * import numpy as np import unittest import shutil diff --git a/python/dllib/test/bigdl/test_utils.py b/python/dllib/test/bigdl/test_utils.py index 4eadd7bec69..a35061c8eda 100644 --- a/python/dllib/test/bigdl/test_utils.py +++ b/python/dllib/test/bigdl/test_utils.py @@ -21,13 +21,13 @@ from keras.layers.convolutional import * from keras.layers import Dense, Dropout, Input from keras.optimizers import RMSprop -from bigdl.util.common import * -from bigdl.keras.converter import * -from bigdl.keras.converter import WeightLoader, WeightsConverter +from bigdl.utils.common import * +from bigdl.dllib.keras.converter import * +from bigdl.dllib.keras.converter import WeightLoader, WeightsConverter import numpy as np from unittest import TestCase import keras -from bigdl.examples.keras.keras_utils import * +from bigdl.dllib.examples.keras.keras_utils import * class TestModels: @@ -323,13 +323,13 @@ def compare_loss(self, y_a, y_b, kloss, bloss, rtol=1e-6, atol=1e-6): def compare_newapi(self, klayer, blayer, input_data, weight_converter=None, is_training=False, rtol=1e-6, atol=1e-6): from keras.models import Sequential as KSequential - from bigdl.nn.keras.topology import Sequential as BSequential + from bigdl.dllib.keras.topology import Sequential as BSequential bmodel = BSequential() bmodel.add(blayer) kmodel = KSequential() kmodel.add(klayer) koutput = kmodel.predict(input_data) - from bigdl.nn.keras.layer import BatchNormalization + from bigdl.dllib.keras.layer import BatchNormalization if isinstance(blayer, BatchNormalization): k_running_mean = K.eval(klayer.running_mean) k_running_std = K.eval(klayer.running_std) diff --git a/python/dllib/test/bigdl/transform/test_image.py b/python/dllib/test/bigdl/transform/test_image.py index fb2adb1f59d..bf25db39d29 100644 --- a/python/dllib/test/bigdl/transform/test_image.py +++ b/python/dllib/test/bigdl/transform/test_image.py @@ -16,8 +16,8 @@ import pytest import os -from bigdl.util.common import * -from bigdl.transform.vision.image import * +from bigdl.utils.common import * +from bigdl.dllib.feature.transform.vision.image import * import tempfile diff --git a/python/dllib/test/dev/prepare_env.sh b/python/dllib/test/dev/prepare_env.sh index 8e61a0f0025..4e3194c7d04 100755 --- a/python/dllib/test/dev/prepare_env.sh +++ b/python/dllib/test/dev/prepare_env.sh @@ -28,9 +28,9 @@ if [ -z ${SPARK_HOME+x} ]; then echo "SPARK_HOME is unset"; exit 1; else echo "S export PYSPARK_ZIP=`find $SPARK_HOME/python/lib -type f -iname '*.zip' | tr "\n" ":"` -export PYTHONPATH=$PYTHONPATH:$PYSPARK_ZIP:$DL_PYTHON_HOME:$DL_PYTHON_HOME/:$DL_PYTHON_HOME/test/dev:$BIGDL_HOME/spark/dl/src/main/resources/spark-bigdl.conf +export PYTHONPATH=$PYTHONPATH:$PYSPARK_ZIP:$DL_PYTHON_HOME:$DL_PYTHON_HOME/:$DL_PYTHON_HOME/test/dev:$BIGDL_HOME/scala/dllib/src/main/resources/spark-bigdl.conf -export BIGDL_CLASSPATH=$(find $BIGDL_HOME/spark/dl/target/ -name "*with-dependencies.jar" | head -n 1) +export BIGDL_CLASSPATH=$(find $BIGDL_HOME/dist/lib/ -name "*with-dependencies.jar" | head -n 1) echo "BIGDL_CLASSPATH": $BIGDL_CLASSPATH if [[ ($SPARK_HOME == *"2.2.0"*) || ($SPARK_HOME == *"2.1.1"*) || ($SPARK_HOME == *"1.6.4"*) ]]; then diff --git a/python/dllib/test/dev/release/release.sh b/python/dllib/test/dev/release/release.sh index 306a23c41c0..e070f510b47 100755 --- a/python/dllib/test/dev/release/release.sh +++ b/python/dllib/test/dev/release/release.sh @@ -19,9 +19,9 @@ set -e RUN_SCRIPT_DIR=$(cd $(dirname $0) ; pwd) echo $RUN_SCRIPT_DIR -BIGDL_DIR="$(cd ${RUN_SCRIPT_DIR}/../../../../; pwd)" +BIGDL_DIR="$(cd ${RUN_SCRIPT_DIR}/../; pwd)" echo $BIGDL_DIR -BIGDL_PYTHON_DIR="$(cd ${RUN_SCRIPT_DIR}/../../../../pyspark; pwd)" +BIGDL_PYTHON_DIR="$(cd ${RUN_SCRIPT_DIR}/../python; pwd)" echo $BIGDL_PYTHON_DIR if (( $# < 2)); then @@ -32,13 +32,7 @@ fi platform=$1 spark_profile=$2 quick=$3 -input_version=$4 -bigdl_version=$(python -c "exec(open('$BIGDL_DIR/pyspark/bigdl/version.py').read()); print(__version__)") - -if [ "$input_version" != "$bigdl_version" ]; then - echo "Not the proposed version: $bigdl_version" - exit -1 -fi +bigdl_version=$(python -c "exec(open('$BIGDL_DIR/python/bigdl/version.py').read()); print(__version__)") cd ${BIGDL_DIR} if [ "$platform" == "mac" ]; then @@ -53,7 +47,7 @@ else echo "unsupport platform" fi -bigdl_build_command="${BIGDL_DIR}/make-dist.sh ${dist_profile}" +bigdl_build_command="${BIGDL_DIR}/scala/make-dist.sh ${dist_profile}" if [ "$quick" == "true" ]; then echo "Skip disting BigDL" else @@ -66,18 +60,11 @@ sdist_command="python setup.py sdist" echo "packing source code: ${sdist_command}" $sdist_command -if [ -d "${BIGDL_DIR}/pyspark/build" ]; then - rm -r ${BIGDL_DIR}/pyspark/build -fi - -if [ -d "${BIGDL_DIR}/pyspark/dist" ]; then - rm -r ${BIGDL_DIR}/pyspark/dist -fi wheel_command="python setup.py bdist_wheel --plat-name ${verbose_pname}" echo "Packing python distribution: $wheel_command" ${wheel_command} -upload_command="twine upload dist/BigDL-${bigdl_version}-py2.py3-none-${verbose_pname}.whl" -echo "Please manually upload with this command: $upload_command" +#upload_command="twine upload dist/BigDL-${bigdl_version}-py2.py3-none-${verbose_pname}.whl" +#echo "Please manually upload with this command: $upload_command" -$upload_command +#$upload_command diff --git a/python/dllib/test/dev/run-caffe.sh b/python/dllib/test/dev/run-caffe.sh index 8234eb90d9f..6ad394782d1 100755 --- a/python/dllib/test/dev/run-caffe.sh +++ b/python/dllib/test/dev/run-caffe.sh @@ -28,8 +28,8 @@ echo "${cyan}Using python version: $p${reset}" export PYTHON_EXECUTABLE=$p export PYSPARK_PYTHON=$p export PYSPARK_DRIVER_PYTHON=$p -$p -m pytest -v ../../../pyspark/test/bigdl/caffe/ \ ---ignore=../../../pyspark/test/bigdl/caffe/caffe_layers.py +$p -m pytest -v ../../../python/test/bigdl/caffe/ \ +--ignore=../../../python/test/bigdl/caffe/caffe_layers.py exit_status=$? echo "running caffe layer unit tests" if [ $exit_status -ne 0 ]; diff --git a/python/dllib/test/dev/run-keras.sh b/python/dllib/test/dev/run-keras.sh index 87219f64127..300e2af3f4d 100755 --- a/python/dllib/test/dev/run-keras.sh +++ b/python/dllib/test/dev/run-keras.sh @@ -32,7 +32,7 @@ echo "${cyan}Using python version: $p${reset}" export PYTHON_EXECUTABLE=$p export PYSPARK_PYTHON=$p export PYSPARK_DRIVER_PYTHON=$p -$p -m pytest -v ../../../pyspark/test/bigdl/keras +$p -m pytest -v ../../../python/test/bigdl/keras exit_status=$? if [ $exit_status -ne 0 ]; diff --git a/python/dllib/test/dev/run-tests b/python/dllib/test/dev/run-tests index e7ef14b63f6..0e77a96fcae 100755 --- a/python/dllib/test/dev/run-tests +++ b/python/dllib/test/dev/run-tests @@ -33,17 +33,17 @@ do export PYTHON_EXECUTABLE=$p export PYSPARK_PYTHON=$p export PYSPARK_DRIVER_PYTHON=$p - $p -m pytest -v --junitxml result_bigdl_${p}.xml --doctest-modules ../../../pyspark/bigdl \ - --ignore=../../../pyspark/bigdl/dataset/ \ - --ignore=../../../pyspark/bigdl/util/tf_utils.py \ - --ignore=../../../pyspark/bigdl/keras \ - --ignore=../../../pyspark/test/bigdl/keras/test_application.py \ - --ignore=../../../pyspark/bigdl/examples/ \ - --ignore=../../../pyspark/bigdl/models/ && \ - $p -m pytest -v --junitxml result_test_${p}.xml ../../../pyspark/test/ \ - --ignore=../../../pyspark/test/bigdl/caffe/ \ - --ignore=../../../pyspark/test/bigdl/keras/ \ - --ignore=../../../pyspark/test/bigdl/test_utils.py + $p -m pytest -v --junitxml result_bigdl_${p}.xml --doctest-modules ../../../python/bigdl \ + --ignore=../../../python/bigdl/dllib/feature/dataset/ \ + --ignore=../../../python/bigdl/dllib/utils/tf_utils.py \ + --ignore=../../../python/bigdl/dllib/keras \ + --ignore=../../../python/test/bigdl/keras/test_application.py \ + --ignore=../../../python/bigdl/dllib/examples/ \ + --ignore=../../../python/bigdl/dllib/models/ && \ + $p -m pytest -v --junitxml result_test_${p}.xml ../../../python/test/ \ + --ignore=../../../python/test/bigdl/caffe/ \ + --ignore=../../../python/test/bigdl/keras/ \ + --ignore=../../../python/test/bigdl/test_utils.py exit_status=$? if [ $exit_status -ne 0 ]; diff --git a/python/release.sh b/python/release.sh deleted file mode 100755 index e070f510b47..00000000000 --- a/python/release.sh +++ /dev/null @@ -1,70 +0,0 @@ -#!/usr/bin/env bash - -# -# Copyright 2016 The BigDL Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -set -e -RUN_SCRIPT_DIR=$(cd $(dirname $0) ; pwd) -echo $RUN_SCRIPT_DIR -BIGDL_DIR="$(cd ${RUN_SCRIPT_DIR}/../; pwd)" -echo $BIGDL_DIR -BIGDL_PYTHON_DIR="$(cd ${RUN_SCRIPT_DIR}/../python; pwd)" -echo $BIGDL_PYTHON_DIR - -if (( $# < 2)); then - echo "Bad parameters. Usage: release.sh mac spark_2.x" - exit -1 -fi - -platform=$1 -spark_profile=$2 -quick=$3 -bigdl_version=$(python -c "exec(open('$BIGDL_DIR/python/bigdl/version.py').read()); print(__version__)") - -cd ${BIGDL_DIR} -if [ "$platform" == "mac" ]; then - echo "Building bigdl for mac system" - dist_profile="-P mac -P $spark_profile" - verbose_pname="macosx_10_11_x86_64" -elif [ "$platform" == "linux" ]; then - echo "Building bigdl for linux system" - dist_profile="-P $spark_profile" - verbose_pname="manylinux1_x86_64" -else - echo "unsupport platform" -fi - -bigdl_build_command="${BIGDL_DIR}/scala/make-dist.sh ${dist_profile}" -if [ "$quick" == "true" ]; then - echo "Skip disting BigDL" -else - echo "Dist BigDL: $bigdl_build_command" - $bigdl_build_command -fi - -cd $BIGDL_PYTHON_DIR -sdist_command="python setup.py sdist" -echo "packing source code: ${sdist_command}" -$sdist_command - -wheel_command="python setup.py bdist_wheel --plat-name ${verbose_pname}" -echo "Packing python distribution: $wheel_command" -${wheel_command} - -#upload_command="twine upload dist/BigDL-${bigdl_version}-py2.py3-none-${verbose_pname}.whl" -#echo "Please manually upload with this command: $upload_command" - -#$upload_command From 96e641957bc93d1e0acc9bafc28b63b9d09fc3f1 Mon Sep 17 00:00:00 2001 From: dding3 Date: Mon, 16 Aug 2021 21:01:18 -0700 Subject: [PATCH 802/823] update test scripts --- python/dllib/src/bigdl/__init__.py | 18 +++++ python/dllib/src/bigdl/dllib/__init__.py | 15 ++++ python/dllib/src/bigdl/dllib/nn/layer.py | 4 +- python/dllib/src/bigdl/version.py | 17 +++++ .../test/bigdl/caffe/test_caffe_layers.py | 10 +-- .../dllib/test/bigdl/caffe/test_load_caffe.py | 6 +- .../test/bigdl/keras/test_application.py | 4 +- python/dllib/test/bigdl/keras/test_backend.py | 8 +-- .../dllib/test/bigdl/keras/test_keras_api.py | 10 +-- python/dllib/test/bigdl/keras/test_layer.py | 4 +- .../dllib/test/bigdl/keras/test_load_model.py | 8 +-- python/dllib/test/bigdl/keras/test_loss.py | 2 +- python/dllib/test/bigdl/onnx/test_onnx_ops.py | 12 ++-- python/dllib/test/bigdl/test_engine_env.py | 4 +- python/dllib/test/bigdl/test_pickler.py | 20 +++--- .../test/bigdl/test_simple_integration.py | 44 ++++++------ python/dllib/test/bigdl/test_tensorflow.py | 6 +- python/dllib/test/bigdl/test_utils.py | 12 ++-- .../dllib/test/bigdl/transform/test_image.py | 4 +- python/dllib/test/dev/prepare_env.sh | 4 +- python/dllib/test/dev/release/release.sh | 27 ++----- python/dllib/test/dev/run-caffe.sh | 4 +- python/dllib/test/dev/run-keras.sh | 2 +- python/dllib/test/dev/run-tests | 22 +++--- python/release.sh | 70 ------------------- 25 files changed, 152 insertions(+), 185 deletions(-) create mode 100644 python/dllib/src/bigdl/__init__.py create mode 100644 python/dllib/src/bigdl/dllib/__init__.py create mode 100644 python/dllib/src/bigdl/version.py delete mode 100755 python/release.sh diff --git a/python/dllib/src/bigdl/__init__.py b/python/dllib/src/bigdl/__init__.py new file mode 100644 index 00000000000..7f835104b9e --- /dev/null +++ b/python/dllib/src/bigdl/__init__.py @@ -0,0 +1,18 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from bigdl.utils.engine import prepare_env +prepare_env() diff --git a/python/dllib/src/bigdl/dllib/__init__.py b/python/dllib/src/bigdl/dllib/__init__.py new file mode 100644 index 00000000000..2151a805423 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/__init__.py @@ -0,0 +1,15 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index bfa012e690f..5f36b75bc60 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -745,7 +745,7 @@ def __init__(self, bigdl_type, "createModelPreprocessor", inputs, outputs) self.bigdl_type = bigdl_type else: - from bigdl.util.tf_utils import convert + from bigdl.utils.tf_utils import convert model = convert(to_list(inputs), to_list(outputs), byte_order, bigdl_type) super(Model, self).__init__(model.value, bigdl_type) @@ -863,7 +863,7 @@ def load_tensorflow(path, inputs, outputs, byte_order = "little_endian", @staticmethod def train(output, data, label, opt_method, criterion, batch_size, end_when, session=None, bigdl_type="float"): - from bigdl.util.tf_utils import get_path + from bigdl.utils.tf_utils import get_path from bigdl.utils.common import Sample output_name = output.name.split(":")[0] path = get_path(output_name, session) diff --git a/python/dllib/src/bigdl/version.py b/python/dllib/src/bigdl/version.py new file mode 100644 index 00000000000..3d92def3019 --- /dev/null +++ b/python/dllib/src/bigdl/version.py @@ -0,0 +1,17 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +__version__ = "2.0.0.dev0" diff --git a/python/dllib/test/bigdl/caffe/test_caffe_layers.py b/python/dllib/test/bigdl/caffe/test_caffe_layers.py index 6623addf0d6..640a6f8efc3 100644 --- a/python/dllib/test/bigdl/caffe/test_caffe_layers.py +++ b/python/dllib/test/bigdl/caffe/test_caffe_layers.py @@ -14,9 +14,9 @@ # limitations under the License. # -from bigdl.nn.layer import * -from bigdl.optim.optimizer import * -from bigdl.util.common import * +from bigdl.dllib.nn.layer import * +from bigdl.dllib.optim.optimizer import * +from bigdl.utils.common import * import numpy as np import pytest import tempfile @@ -56,8 +56,8 @@ def test_caffe_layers(self): if len(inputs) == 1: inputs = inputs[0] bigdlResult = model.forward(inputs) - print cafferesult - print bigdlResult + print(cafferesult) + print(bigdlResult) assert_allclose(cafferesult, bigdlResult, atol=1e-4, rtol=0) if __name__ == "__main__": diff --git a/python/dllib/test/bigdl/caffe/test_load_caffe.py b/python/dllib/test/bigdl/caffe/test_load_caffe.py index 6067082c695..d71039fb948 100644 --- a/python/dllib/test/bigdl/caffe/test_load_caffe.py +++ b/python/dllib/test/bigdl/caffe/test_load_caffe.py @@ -14,9 +14,9 @@ # limitations under the License. # -from bigdl.nn.layer import * -from bigdl.optim.optimizer import * -from bigdl.util.common import * +from bigdl.dllib.nn.layer import * +from bigdl.dllib.optim.optimizer import * +from bigdl.utils.common import * import numpy as np import pytest from numpy.testing import assert_allclose diff --git a/python/dllib/test/bigdl/keras/test_application.py b/python/dllib/test/bigdl/keras/test_application.py index 0d9a4587608..10f5172b864 100644 --- a/python/dllib/test/bigdl/keras/test_application.py +++ b/python/dllib/test/bigdl/keras/test_application.py @@ -20,11 +20,11 @@ np.random.seed(1337) # for reproducibility import pytest from keras.applications import * -from bigdl.keras.converter import * +from bigdl.dllib.keras.converter import * from keras.applications.music_tagger_crnn import MusicTaggerCRNN from test.bigdl.test_utils import BigDLTestCase, TestModels -from bigdl.keras.backend import * +from bigdl.dllib.keras.backend import * class TestApplication(BigDLTestCase): diff --git a/python/dllib/test/bigdl/keras/test_backend.py b/python/dllib/test/bigdl/keras/test_backend.py index 96fddbb927a..97d5c53bf63 100644 --- a/python/dllib/test/bigdl/keras/test_backend.py +++ b/python/dllib/test/bigdl/keras/test_backend.py @@ -20,8 +20,8 @@ np.random.seed(1337) # for reproducibility import pytest from keras.applications import * -from bigdl.keras.converter import * -from bigdl.keras.backend import * +from bigdl.dllib.keras.converter import * +from bigdl.dllib.keras.backend import * from test.bigdl.test_utils import BigDLTestCase, TestModels @@ -73,8 +73,8 @@ def test_lenet_distributed_ndarray(self): def test_lenet_distributed_rdd(self): kmodel, X_train, y_train = TestModels.kmodel_seq_lenet_mnist() sc = get_spark_context() - from bigdl.util.common import Sample - from bigdl.util.common import to_sample_rdd + from bigdl.utils.common import Sample + from bigdl.utils.common import to_sample_rdd training_rdd = to_sample_rdd(X_train, y_train) self.modelTest(X_train, kmodel, dump_weights=True) diff --git a/python/dllib/test/bigdl/keras/test_keras_api.py b/python/dllib/test/bigdl/keras/test_keras_api.py index fefdf797f59..37a79b4fc86 100644 --- a/python/dllib/test/bigdl/keras/test_keras_api.py +++ b/python/dllib/test/bigdl/keras/test_keras_api.py @@ -17,13 +17,13 @@ import pytest from test.bigdl.test_utils import BigDLTestCase -import bigdl.nn.keras.layer as BLayer +import bigdl.dllib.keras.layer as BLayer import keras.layers as KLayer import keras.backend as K -from bigdl.keras.converter import WeightsConverter +from bigdl.dllib.keras.converter import WeightsConverter from bigdl.dataset.dataset import * -from bigdl.nn.keras.topology import Model as BModel -from bigdl.nn.keras.topology import Sequential as BSequential +from bigdl.dllib.keras.topology import Model as BModel +from bigdl.dllib.keras.topology import Sequential as BSequential from keras.engine import merge as kmerge, Model as KModel from keras.models import Sequential as KSequential @@ -124,7 +124,7 @@ def test_merge_cos(self): self.compare_newapi(klayer, blayer, input_data) def test_lenet_shape(self): - from bigdl.examples.lenet.lenet import build_model + from bigdl.dllib.examples.lenet.lenet import build_model model = build_model(10) input_shape = model.get_input_shape() np.testing.assert_allclose((28, 28, 1), input_shape[1:]) diff --git a/python/dllib/test/bigdl/keras/test_layer.py b/python/dllib/test/bigdl/keras/test_layer.py index 18a38849a60..8f3dd2a3f70 100644 --- a/python/dllib/test/bigdl/keras/test_layer.py +++ b/python/dllib/test/bigdl/keras/test_layer.py @@ -23,9 +23,9 @@ from keras.layers.convolutional import * from keras.layers import Dense, Input from keras.regularizers import l1, l2, l1l2 -from bigdl.keras.converter import * +from bigdl.dllib.keras.converter import * from test.bigdl.test_utils import BigDLTestCase -from bigdl.examples.keras.keras_utils import * +from bigdl.dllib.examples.keras.keras_utils import * class TestLayer(BigDLTestCase): diff --git a/python/dllib/test/bigdl/keras/test_load_model.py b/python/dllib/test/bigdl/keras/test_load_model.py index 0be15f22b02..52b7f7cd8ff 100644 --- a/python/dllib/test/bigdl/keras/test_load_model.py +++ b/python/dllib/test/bigdl/keras/test_load_model.py @@ -19,13 +19,13 @@ import pytest from numpy.testing import assert_allclose -import bigdl.nn.layer as BLayer -from bigdl.keras.converter import WeightLoader -from bigdl.keras.converter import DefinitionLoader +import bigdl.dllib.nn.layer as BLayer +from bigdl.dllib.keras.converter import WeightLoader +from bigdl.dllib.keras.converter import DefinitionLoader np.random.seed(1337) # for reproducibility from test.bigdl.test_utils import BigDLTestCase, TestModels -from bigdl.examples.keras.keras_utils import * +from bigdl.dllib.examples.keras.keras_utils import * import keras.backend as K diff --git a/python/dllib/test/bigdl/keras/test_loss.py b/python/dllib/test/bigdl/keras/test_loss.py index c06ad5ef9a0..42990c20751 100644 --- a/python/dllib/test/bigdl/keras/test_loss.py +++ b/python/dllib/test/bigdl/keras/test_loss.py @@ -19,7 +19,7 @@ import pytest from keras import objectives -from bigdl.keras.optimization import OptimConverter +from bigdl.dllib.keras.optimization import OptimConverter from test.bigdl.test_utils import BigDLTestCase np.random.seed(1337) # for reproducibility diff --git a/python/dllib/test/bigdl/onnx/test_onnx_ops.py b/python/dllib/test/bigdl/onnx/test_onnx_ops.py index a5205a79ce3..4ace83b5bf1 100644 --- a/python/dllib/test/bigdl/onnx/test_onnx_ops.py +++ b/python/dllib/test/bigdl/onnx/test_onnx_ops.py @@ -18,12 +18,12 @@ import pytest import numpy as np -from bigdl.contrib.onnx.onnx_loader import load_model_proto -from bigdl.nn.layer import CAddTable, JoinTable, ReLU -from bigdl.nn.layer import SoftMax, SpatialAveragePooling, SpatialBatchNormalization -from bigdl.nn.layer import SpatialConvolution, SpatialMaxPooling -from bigdl.nn.layer import Unsqueeze -from bigdl.nn.onnx.layer import Gemm, Reshape, Shape +from bigdl.dllib.contrib.onnx.onnx_loader import load_model_proto +from bigdl.dllib.nn.layer import CAddTable, JoinTable, ReLU +from bigdl.dllib.nn.layer import SoftMax, SpatialAveragePooling, SpatialBatchNormalization +from bigdl.dllib.nn.layer import SpatialConvolution, SpatialMaxPooling +from bigdl.dllib.nn.layer import Unsqueeze +from bigdl.dllib.nn.onnx.layer import Gemm, Reshape, Shape class TestAveragePool(object): diff --git a/python/dllib/test/bigdl/test_engine_env.py b/python/dllib/test/bigdl/test_engine_env.py index 80e8e4f12d8..dc824267356 100644 --- a/python/dllib/test/bigdl/test_engine_env.py +++ b/python/dllib/test/bigdl/test_engine_env.py @@ -16,7 +16,7 @@ import pytest import os -from bigdl.util.common import * +from bigdl.utils.common import * class TestEngineEnv(): @@ -38,7 +38,7 @@ def test___prepare_bigdl_env(self): # adding jar path message, just do prepare_env()' again # to see if the log is correct and the environment variables should not vary. - from bigdl.util.engine import prepare_env + from bigdl.utils.engine import prepare_env bigdl_jars_env_1 = os.environ.get("BIGDL_JARS", None) spark_class_path_1 = os.environ.get("SPARK_CLASSPATH", None) diff --git a/python/dllib/test/bigdl/test_pickler.py b/python/dllib/test/bigdl/test_pickler.py index a3e41eec34f..69e5f92cafa 100644 --- a/python/dllib/test/bigdl/test_pickler.py +++ b/python/dllib/test/bigdl/test_pickler.py @@ -14,19 +14,19 @@ # limitations under the License. # -from bigdl.nn.layer import * -from bigdl.nn.initialization_method import * -from bigdl.nn.criterion import * -from bigdl.optim.optimizer import * -from bigdl.util.common import * -from bigdl.util.common import _py2java -from bigdl.nn.initialization_method import * -from bigdl.dataset import movielens +from bigdl.dllib.nn.layer import * +from bigdl.dllib.nn.initialization_method import * +from bigdl.dllib.nn.criterion import * +from bigdl.dllib.optim.optimizer import * +from bigdl.utils.common import * +from bigdl.utils.common import _py2java +from bigdl.dllib.nn.initialization_method import * +from bigdl.dllib.feature.dataset import movielens import numpy as np import tempfile import pytest from numpy.testing import assert_allclose, assert_array_equal -from bigdl.util.engine import compare_version +from bigdl.utils.engine import compare_version np.random.seed(1337) # for reproducibility @@ -35,7 +35,7 @@ def setup_method(self, method): """ setup any state tied to the execution of the given method in a class. setup_method is invoked for every test method of a class. """ - JavaCreator.add_creator_class("com.intel.analytics.bigdl.python.api.PythonBigDLValidator") + JavaCreator.add_creator_class("com.intel.analytics.bigdl.dllib.utils.python.api.PythonBigDLValidator") sparkConf = create_spark_conf().setMaster("local[4]").setAppName("test model") self.sc = get_spark_context(sparkConf) init_engine() diff --git a/python/dllib/test/bigdl/test_simple_integration.py b/python/dllib/test/bigdl/test_simple_integration.py index 4b47df29f2c..cb49fd6213d 100644 --- a/python/dllib/test/bigdl/test_simple_integration.py +++ b/python/dllib/test/bigdl/test_simple_integration.py @@ -14,22 +14,22 @@ # limitations under the License. # -from bigdl.nn.layer import * -from bigdl.nn.initialization_method import * -from bigdl.nn.criterion import * -from bigdl.optim.optimizer import * -from bigdl.util.common import * -from bigdl.util.common import _py2java -from bigdl.nn.initialization_method import * -from bigdl.dataset import movielens +from bigdl.dllib.nn.layer import * +from bigdl.dllib.nn.initialization_method import * +from bigdl.dllib.nn.criterion import * +from bigdl.dllib.optim.optimizer import * +from bigdl.utils.common import * +from bigdl.utils.common import _py2java +from bigdl.dllib.nn.initialization_method import * +from bigdl.dllib.feature.dataset import movielens import numpy as np import tempfile import pytest from numpy.testing import assert_allclose, assert_array_equal -from bigdl.util.engine import compare_version -from bigdl.transform.vision.image import * -from bigdl.models.utils.model_broadcast import broadcast_model -from bigdl.dataset.dataset import * +from bigdl.utils.engine import compare_version +from bigdl.dllib.feature.transform.vision.image import * +from bigdl.dllib.models.utils.model_broadcast import broadcast_model +from bigdl.dllib.feature.dataset.dataset import * np.random.seed(1337) # for reproducibility @@ -77,9 +77,9 @@ def grad_update(mlp, x, y, criterion, learning_rate): rtol=1.e-1) def test_load_keras_model_of(self): - from bigdl.nn.keras.topology import Model as KModel - from bigdl.nn.keras.layer import Input as KInput - from bigdl.nn.keras.layer import Dense + from bigdl.dllib.keras.topology import Model as KModel + from bigdl.dllib.keras.layer import Input as KInput + from bigdl.dllib.keras.layer import Dense input = KInput(shape=[2, 3]) fc1 = Dense(2)(input) @@ -87,12 +87,12 @@ def test_load_keras_model_of(self): tmp_path = tempfile.mktemp() model.save(tmp_path, True) model_loaded = KModel.load(tmp_path) - assert "bigdl.nn.keras.topology.Model" in str(type(model_loaded)) + assert "bigdl.dllib.keras.topology.Model" in str(type(model_loaded)) assert len(model_loaded.layers) == 2 def test_load_keras_seq_of(self): - from bigdl.nn.keras.topology import Sequential as KSequential - from bigdl.nn.keras.layer import Dense + from bigdl.dllib.keras.topology import Sequential as KSequential + from bigdl.dllib.keras.layer import Dense model = KSequential() fc1 = Dense(2, input_shape=[2, 3]) @@ -100,7 +100,7 @@ def test_load_keras_seq_of(self): tmp_path = tempfile.mktemp() model.save(tmp_path, True) model_loaded = KSequential.load(tmp_path) - assert "bigdl.nn.keras.topology.Sequential" in str(type(model_loaded)) + assert "bigdl.dllib.keras.topology.Sequential" in str(type(model_loaded)) assert len(model_loaded.layers) == 1 def test_load_model_of(self): @@ -244,7 +244,7 @@ def test_graph_preprocessor(self): np.array([1.0])) def test_load_zip_conf(self): - from bigdl.util.common import get_bigdl_conf + from bigdl.utils.common import get_bigdl_conf import sys sys_path_back = sys.path sys.path = [path for path in sys.path if "spark-bigdl.conf" not in path] @@ -420,7 +420,7 @@ def gen_rand_sample(): optimizer.optimize() def test_forward_backward(self): - from bigdl.nn.layer import Linear + from bigdl.dllib.nn.layer import Linear rng = RNG() rng.set_seed(100) @@ -442,7 +442,7 @@ def test_forward_backward(self): l_grad_output = linear.backward(input, grad_output) def test_forward_multiple(self): - from bigdl.nn.layer import Linear + from bigdl.dllib.nn.layer import Linear rng = RNG() rng.set_seed(100) diff --git a/python/dllib/test/bigdl/test_tensorflow.py b/python/dllib/test/bigdl/test_tensorflow.py index aedbd90cdb5..2d8e3f1918f 100644 --- a/python/dllib/test/bigdl/test_tensorflow.py +++ b/python/dllib/test/bigdl/test_tensorflow.py @@ -14,9 +14,9 @@ # limitations under the License. # -from bigdl.nn.layer import * -from bigdl.optim.optimizer import * -from bigdl.util.common import * +from bigdl.dllib.nn.layer import * +from bigdl.dllib.optim.optimizer import * +from bigdl.utils.common import * import numpy as np import unittest import shutil diff --git a/python/dllib/test/bigdl/test_utils.py b/python/dllib/test/bigdl/test_utils.py index 4eadd7bec69..a35061c8eda 100644 --- a/python/dllib/test/bigdl/test_utils.py +++ b/python/dllib/test/bigdl/test_utils.py @@ -21,13 +21,13 @@ from keras.layers.convolutional import * from keras.layers import Dense, Dropout, Input from keras.optimizers import RMSprop -from bigdl.util.common import * -from bigdl.keras.converter import * -from bigdl.keras.converter import WeightLoader, WeightsConverter +from bigdl.utils.common import * +from bigdl.dllib.keras.converter import * +from bigdl.dllib.keras.converter import WeightLoader, WeightsConverter import numpy as np from unittest import TestCase import keras -from bigdl.examples.keras.keras_utils import * +from bigdl.dllib.examples.keras.keras_utils import * class TestModels: @@ -323,13 +323,13 @@ def compare_loss(self, y_a, y_b, kloss, bloss, rtol=1e-6, atol=1e-6): def compare_newapi(self, klayer, blayer, input_data, weight_converter=None, is_training=False, rtol=1e-6, atol=1e-6): from keras.models import Sequential as KSequential - from bigdl.nn.keras.topology import Sequential as BSequential + from bigdl.dllib.keras.topology import Sequential as BSequential bmodel = BSequential() bmodel.add(blayer) kmodel = KSequential() kmodel.add(klayer) koutput = kmodel.predict(input_data) - from bigdl.nn.keras.layer import BatchNormalization + from bigdl.dllib.keras.layer import BatchNormalization if isinstance(blayer, BatchNormalization): k_running_mean = K.eval(klayer.running_mean) k_running_std = K.eval(klayer.running_std) diff --git a/python/dllib/test/bigdl/transform/test_image.py b/python/dllib/test/bigdl/transform/test_image.py index fb2adb1f59d..bf25db39d29 100644 --- a/python/dllib/test/bigdl/transform/test_image.py +++ b/python/dllib/test/bigdl/transform/test_image.py @@ -16,8 +16,8 @@ import pytest import os -from bigdl.util.common import * -from bigdl.transform.vision.image import * +from bigdl.utils.common import * +from bigdl.dllib.feature.transform.vision.image import * import tempfile diff --git a/python/dllib/test/dev/prepare_env.sh b/python/dllib/test/dev/prepare_env.sh index 8e61a0f0025..4e3194c7d04 100755 --- a/python/dllib/test/dev/prepare_env.sh +++ b/python/dllib/test/dev/prepare_env.sh @@ -28,9 +28,9 @@ if [ -z ${SPARK_HOME+x} ]; then echo "SPARK_HOME is unset"; exit 1; else echo "S export PYSPARK_ZIP=`find $SPARK_HOME/python/lib -type f -iname '*.zip' | tr "\n" ":"` -export PYTHONPATH=$PYTHONPATH:$PYSPARK_ZIP:$DL_PYTHON_HOME:$DL_PYTHON_HOME/:$DL_PYTHON_HOME/test/dev:$BIGDL_HOME/spark/dl/src/main/resources/spark-bigdl.conf +export PYTHONPATH=$PYTHONPATH:$PYSPARK_ZIP:$DL_PYTHON_HOME:$DL_PYTHON_HOME/:$DL_PYTHON_HOME/test/dev:$BIGDL_HOME/scala/dllib/src/main/resources/spark-bigdl.conf -export BIGDL_CLASSPATH=$(find $BIGDL_HOME/spark/dl/target/ -name "*with-dependencies.jar" | head -n 1) +export BIGDL_CLASSPATH=$(find $BIGDL_HOME/dist/lib/ -name "*with-dependencies.jar" | head -n 1) echo "BIGDL_CLASSPATH": $BIGDL_CLASSPATH if [[ ($SPARK_HOME == *"2.2.0"*) || ($SPARK_HOME == *"2.1.1"*) || ($SPARK_HOME == *"1.6.4"*) ]]; then diff --git a/python/dllib/test/dev/release/release.sh b/python/dllib/test/dev/release/release.sh index 306a23c41c0..e070f510b47 100755 --- a/python/dllib/test/dev/release/release.sh +++ b/python/dllib/test/dev/release/release.sh @@ -19,9 +19,9 @@ set -e RUN_SCRIPT_DIR=$(cd $(dirname $0) ; pwd) echo $RUN_SCRIPT_DIR -BIGDL_DIR="$(cd ${RUN_SCRIPT_DIR}/../../../../; pwd)" +BIGDL_DIR="$(cd ${RUN_SCRIPT_DIR}/../; pwd)" echo $BIGDL_DIR -BIGDL_PYTHON_DIR="$(cd ${RUN_SCRIPT_DIR}/../../../../pyspark; pwd)" +BIGDL_PYTHON_DIR="$(cd ${RUN_SCRIPT_DIR}/../python; pwd)" echo $BIGDL_PYTHON_DIR if (( $# < 2)); then @@ -32,13 +32,7 @@ fi platform=$1 spark_profile=$2 quick=$3 -input_version=$4 -bigdl_version=$(python -c "exec(open('$BIGDL_DIR/pyspark/bigdl/version.py').read()); print(__version__)") - -if [ "$input_version" != "$bigdl_version" ]; then - echo "Not the proposed version: $bigdl_version" - exit -1 -fi +bigdl_version=$(python -c "exec(open('$BIGDL_DIR/python/bigdl/version.py').read()); print(__version__)") cd ${BIGDL_DIR} if [ "$platform" == "mac" ]; then @@ -53,7 +47,7 @@ else echo "unsupport platform" fi -bigdl_build_command="${BIGDL_DIR}/make-dist.sh ${dist_profile}" +bigdl_build_command="${BIGDL_DIR}/scala/make-dist.sh ${dist_profile}" if [ "$quick" == "true" ]; then echo "Skip disting BigDL" else @@ -66,18 +60,11 @@ sdist_command="python setup.py sdist" echo "packing source code: ${sdist_command}" $sdist_command -if [ -d "${BIGDL_DIR}/pyspark/build" ]; then - rm -r ${BIGDL_DIR}/pyspark/build -fi - -if [ -d "${BIGDL_DIR}/pyspark/dist" ]; then - rm -r ${BIGDL_DIR}/pyspark/dist -fi wheel_command="python setup.py bdist_wheel --plat-name ${verbose_pname}" echo "Packing python distribution: $wheel_command" ${wheel_command} -upload_command="twine upload dist/BigDL-${bigdl_version}-py2.py3-none-${verbose_pname}.whl" -echo "Please manually upload with this command: $upload_command" +#upload_command="twine upload dist/BigDL-${bigdl_version}-py2.py3-none-${verbose_pname}.whl" +#echo "Please manually upload with this command: $upload_command" -$upload_command +#$upload_command diff --git a/python/dllib/test/dev/run-caffe.sh b/python/dllib/test/dev/run-caffe.sh index 8234eb90d9f..6ad394782d1 100755 --- a/python/dllib/test/dev/run-caffe.sh +++ b/python/dllib/test/dev/run-caffe.sh @@ -28,8 +28,8 @@ echo "${cyan}Using python version: $p${reset}" export PYTHON_EXECUTABLE=$p export PYSPARK_PYTHON=$p export PYSPARK_DRIVER_PYTHON=$p -$p -m pytest -v ../../../pyspark/test/bigdl/caffe/ \ ---ignore=../../../pyspark/test/bigdl/caffe/caffe_layers.py +$p -m pytest -v ../../../python/test/bigdl/caffe/ \ +--ignore=../../../python/test/bigdl/caffe/caffe_layers.py exit_status=$? echo "running caffe layer unit tests" if [ $exit_status -ne 0 ]; diff --git a/python/dllib/test/dev/run-keras.sh b/python/dllib/test/dev/run-keras.sh index 87219f64127..300e2af3f4d 100755 --- a/python/dllib/test/dev/run-keras.sh +++ b/python/dllib/test/dev/run-keras.sh @@ -32,7 +32,7 @@ echo "${cyan}Using python version: $p${reset}" export PYTHON_EXECUTABLE=$p export PYSPARK_PYTHON=$p export PYSPARK_DRIVER_PYTHON=$p -$p -m pytest -v ../../../pyspark/test/bigdl/keras +$p -m pytest -v ../../../python/test/bigdl/keras exit_status=$? if [ $exit_status -ne 0 ]; diff --git a/python/dllib/test/dev/run-tests b/python/dllib/test/dev/run-tests index e7ef14b63f6..0e77a96fcae 100755 --- a/python/dllib/test/dev/run-tests +++ b/python/dllib/test/dev/run-tests @@ -33,17 +33,17 @@ do export PYTHON_EXECUTABLE=$p export PYSPARK_PYTHON=$p export PYSPARK_DRIVER_PYTHON=$p - $p -m pytest -v --junitxml result_bigdl_${p}.xml --doctest-modules ../../../pyspark/bigdl \ - --ignore=../../../pyspark/bigdl/dataset/ \ - --ignore=../../../pyspark/bigdl/util/tf_utils.py \ - --ignore=../../../pyspark/bigdl/keras \ - --ignore=../../../pyspark/test/bigdl/keras/test_application.py \ - --ignore=../../../pyspark/bigdl/examples/ \ - --ignore=../../../pyspark/bigdl/models/ && \ - $p -m pytest -v --junitxml result_test_${p}.xml ../../../pyspark/test/ \ - --ignore=../../../pyspark/test/bigdl/caffe/ \ - --ignore=../../../pyspark/test/bigdl/keras/ \ - --ignore=../../../pyspark/test/bigdl/test_utils.py + $p -m pytest -v --junitxml result_bigdl_${p}.xml --doctest-modules ../../../python/bigdl \ + --ignore=../../../python/bigdl/dllib/feature/dataset/ \ + --ignore=../../../python/bigdl/dllib/utils/tf_utils.py \ + --ignore=../../../python/bigdl/dllib/keras \ + --ignore=../../../python/test/bigdl/keras/test_application.py \ + --ignore=../../../python/bigdl/dllib/examples/ \ + --ignore=../../../python/bigdl/dllib/models/ && \ + $p -m pytest -v --junitxml result_test_${p}.xml ../../../python/test/ \ + --ignore=../../../python/test/bigdl/caffe/ \ + --ignore=../../../python/test/bigdl/keras/ \ + --ignore=../../../python/test/bigdl/test_utils.py exit_status=$? if [ $exit_status -ne 0 ]; diff --git a/python/release.sh b/python/release.sh deleted file mode 100755 index e070f510b47..00000000000 --- a/python/release.sh +++ /dev/null @@ -1,70 +0,0 @@ -#!/usr/bin/env bash - -# -# Copyright 2016 The BigDL Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -set -e -RUN_SCRIPT_DIR=$(cd $(dirname $0) ; pwd) -echo $RUN_SCRIPT_DIR -BIGDL_DIR="$(cd ${RUN_SCRIPT_DIR}/../; pwd)" -echo $BIGDL_DIR -BIGDL_PYTHON_DIR="$(cd ${RUN_SCRIPT_DIR}/../python; pwd)" -echo $BIGDL_PYTHON_DIR - -if (( $# < 2)); then - echo "Bad parameters. Usage: release.sh mac spark_2.x" - exit -1 -fi - -platform=$1 -spark_profile=$2 -quick=$3 -bigdl_version=$(python -c "exec(open('$BIGDL_DIR/python/bigdl/version.py').read()); print(__version__)") - -cd ${BIGDL_DIR} -if [ "$platform" == "mac" ]; then - echo "Building bigdl for mac system" - dist_profile="-P mac -P $spark_profile" - verbose_pname="macosx_10_11_x86_64" -elif [ "$platform" == "linux" ]; then - echo "Building bigdl for linux system" - dist_profile="-P $spark_profile" - verbose_pname="manylinux1_x86_64" -else - echo "unsupport platform" -fi - -bigdl_build_command="${BIGDL_DIR}/scala/make-dist.sh ${dist_profile}" -if [ "$quick" == "true" ]; then - echo "Skip disting BigDL" -else - echo "Dist BigDL: $bigdl_build_command" - $bigdl_build_command -fi - -cd $BIGDL_PYTHON_DIR -sdist_command="python setup.py sdist" -echo "packing source code: ${sdist_command}" -$sdist_command - -wheel_command="python setup.py bdist_wheel --plat-name ${verbose_pname}" -echo "Packing python distribution: $wheel_command" -${wheel_command} - -#upload_command="twine upload dist/BigDL-${bigdl_version}-py2.py3-none-${verbose_pname}.whl" -#echo "Please manually upload with this command: $upload_command" - -#$upload_command From 718e9488497541e38679ea1b65f44092a1567264 Mon Sep 17 00:00:00 2001 From: dding3 Date: Tue, 17 Aug 2021 19:51:27 -0700 Subject: [PATCH 803/823] fix create whl issue --- python/dllib/src/MANIFEST.in | 1 + python/dllib/src/setup.cfg | 5 + python/dllib/src/setup.py | 119 +++++++++++++++++++++++ python/dllib/test/dev/release/release.sh | 17 +++- scala/make-dist.sh | 70 +++++++++++++ 5 files changed, 208 insertions(+), 4 deletions(-) create mode 100644 python/dllib/src/MANIFEST.in create mode 100644 python/dllib/src/setup.cfg create mode 100755 python/dllib/src/setup.py create mode 100755 scala/make-dist.sh diff --git a/python/dllib/src/MANIFEST.in b/python/dllib/src/MANIFEST.in new file mode 100644 index 00000000000..fe4bb943569 --- /dev/null +++ b/python/dllib/src/MANIFEST.in @@ -0,0 +1 @@ +recursive-include bigdl/share * diff --git a/python/dllib/src/setup.cfg b/python/dllib/src/setup.cfg new file mode 100644 index 00000000000..0499089e987 --- /dev/null +++ b/python/dllib/src/setup.cfg @@ -0,0 +1,5 @@ +[bdist_wheel] +universal = 1 + +[metadata] +description-file = README.md \ No newline at end of file diff --git a/python/dllib/src/setup.py b/python/dllib/src/setup.py new file mode 100755 index 00000000000..d343be18dc4 --- /dev/null +++ b/python/dllib/src/setup.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python + +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os +import sys +from shutil import copyfile, copytree, rmtree + +from setuptools import setup + +TEMP_PATH = "bigdl/share" +bigdl_home = os.path.abspath(__file__ + "/../../") + +try: + exec(open('bigdl/version.py').read()) +except IOError: + print("Failed to load Bigdl version file for packaging. You must be in Bigdl's python dir.") + sys.exit(-1) + +VERSION = __version__ + +building_error_msg = """ +If you are packing python API from BigDL source, you must build BigDL first +and run sdist. + To build BigDL with maven you can run: + cd $BigDL_HOME + ./make-dist.sh + Building the source dist is done in the Python directory: + cd python + python setup.py sdist + pip install dist/*.tar.gz""" + +def build_from_source(): + code_path = bigdl_home + "/python/bigdl/utils/common.py" + print("Checking: %s to see if build from source" % code_path) + if os.path.exists(code_path): + return True + return False + + +def init_env(): + if build_from_source(): + print("Start to build distributed package") + print("HOME OF BIGDL: " + bigdl_home) + dist_source = bigdl_home + "/dist" + if not os.path.exists(dist_source): + print(building_error_msg) + sys.exit(-1) + if os.path.exists(TEMP_PATH): + rmtree(TEMP_PATH) + copytree(dist_source, TEMP_PATH) + copyfile(bigdl_home + "/python/bigdl/dllib/nn/__init__.py", TEMP_PATH + "/__init__.py") + else: + print("Do nothing for release installation") + +def get_bigdl_packages(): + bigdl_python_home = os.path.abspath(__file__)[:-8] + bigdl_packages = ['bigdl.share'] + for dirpath, dirs, files in os.walk(bigdl_python_home + "bigdl"): + package = dirpath.split(bigdl_python_home)[1].replace('/', '.') + if "__pycache__" not in package: + bigdl_packages.append(package) + print("=========================bigdl packages=========================") + print("\n".join(bigdl_packages)) + print("================================================================") + return bigdl_packages + +def setup_package(): + metadata = dict( + name='BigDL', + version=VERSION, + description='Distributed Deep Learning Library for Apache Spark', + author='BigDL Authors', + author_email='bigdl-user-group@googlegroups.com', + license='Apache License, Version 2.0', + url='https://github.com/intel-analytics/Bigdl', + packages=get_bigdl_packages(), + install_requires=['numpy>=1.7', 'pyspark==2.4.6', 'six>=1.10.0'], + dependency_links=['https://d3kbcqa49mib13.cloudfront.net/spark-2.0.0-bin-hadoop2.7.tgz'], + include_package_data=True, + # package_data={"bigdl.share": ['bigdl/share/lib', 'bigdl/share/conf', 'bigdl/share/bin']}, + package_data={"bigdl.share": ['bigdl/share/lib']}, + classifiers=[ + 'License :: OSI Approved :: Apache Software License', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: Implementation :: CPython'], + platforms=['mac', 'linux'] + ) + + setup(**metadata) + + +if __name__ == '__main__': + try: + init_env() + setup_package() + except Exception as e: + raise e + finally: + if build_from_source() and os.path.exists(TEMP_PATH): + rmtree(TEMP_PATH) + diff --git a/python/dllib/test/dev/release/release.sh b/python/dllib/test/dev/release/release.sh index e070f510b47..7b7ec4f62d1 100755 --- a/python/dllib/test/dev/release/release.sh +++ b/python/dllib/test/dev/release/release.sh @@ -19,9 +19,9 @@ set -e RUN_SCRIPT_DIR=$(cd $(dirname $0) ; pwd) echo $RUN_SCRIPT_DIR -BIGDL_DIR="$(cd ${RUN_SCRIPT_DIR}/../; pwd)" +BIGDL_DIR="$(cd ${RUN_SCRIPT_DIR}/../../../..; pwd)" echo $BIGDL_DIR -BIGDL_PYTHON_DIR="$(cd ${RUN_SCRIPT_DIR}/../python; pwd)" +BIGDL_PYTHON_DIR="$(cd ${BIGDL_DIR}/python; pwd)" echo $BIGDL_PYTHON_DIR if (( $# < 2)); then @@ -34,7 +34,7 @@ spark_profile=$2 quick=$3 bigdl_version=$(python -c "exec(open('$BIGDL_DIR/python/bigdl/version.py').read()); print(__version__)") -cd ${BIGDL_DIR} +cd ${BIGDL_DIR}/scala if [ "$platform" == "mac" ]; then echo "Building bigdl for mac system" dist_profile="-P mac -P $spark_profile" @@ -47,11 +47,12 @@ else echo "unsupport platform" fi -bigdl_build_command="${BIGDL_DIR}/scala/make-dist.sh ${dist_profile}" +bigdl_build_command="bash make-dist.sh ${dist_profile}" if [ "$quick" == "true" ]; then echo "Skip disting BigDL" else echo "Dist BigDL: $bigdl_build_command" + cd ${BIGDL_DIR}/scala $bigdl_build_command fi @@ -60,6 +61,14 @@ sdist_command="python setup.py sdist" echo "packing source code: ${sdist_command}" $sdist_command +if [ -d "${BIGDL_DIR}/python/build" ]; then + rm -r ${BIGDL_DIR}/python/build +fi + +if [ -d "${BIGDL_DIR}/python/dist" ]; then + rm -r ${BIGDL_DIR}/python/dist +fi + wheel_command="python setup.py bdist_wheel --plat-name ${verbose_pname}" echo "Packing python distribution: $wheel_command" ${wheel_command} diff --git a/scala/make-dist.sh b/scala/make-dist.sh new file mode 100755 index 00000000000..4fde83e9e89 --- /dev/null +++ b/scala/make-dist.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash + +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# +# Create a folder contain all files for dist +# + +set -e + +# Check java +if type -p java>/dev/null; then + _java=java +else + echo "Java is not installed" + exit 1 +fi + +MVN_OPTS_LIST="-Xmx2g -XX:ReservedCodeCacheSize=512m" + +if [[ "$_java" ]]; then + version=$("$_java" -version 2>&1 | awk -F '"' '/version/ {print $2}') + if [[ "expr $version" < "1.7" ]]; then + echo Require a java version not lower than 1.7 + exit 1 + fi + # For jdk7 + if [[ "expr $version" < "1.8" ]]; then + MVN_OPTS_LIST="$MVN_OPTS_LIST -XX:MaxPermSize=1G" + fi +fi + +export MAVEN_OPTS=${MAVEN_OPTS:-"$MVN_OPTS_LIST"} + +# Check if mvn installed +MVN_INSTALL=$(which mvn 2>/dev/null | grep mvn | wc -l) +if [ $MVN_INSTALL -eq 0 ]; then + echo "MVN is not installed. Exit" + exit 1 +fi + +mvn clean package -DskipTests $* + +BASEDIR=$(dirname "$0") +DIST_DIR=$BASEDIR/../dist/lib + +if [ ! -d "$DIST_DIR" ] +then + mkdir -p $DIST_DIR +else + rm -r $DIST_DIR + mkdir -p $DIST_DIR +fi + +cp -r $BASEDIR/dllib/target/bigdl-dllib-2.0.0-SNAPSHOT-jar-with-dependencies.jar $DIST_DIR +cp -r $BASEDIR/dllib/target/bigdl-dllib-2.0.0-SNAPSHOT-python-api.zip $DIST_DIR From 3de9966d0af8b58545c2913392e386db8073130c Mon Sep 17 00:00:00 2001 From: dding3 Date: Tue, 17 Aug 2021 19:51:27 -0700 Subject: [PATCH 804/823] fix create whl issue --- python/dllib/src/MANIFEST.in | 1 + python/dllib/src/setup.cfg | 5 + python/dllib/src/setup.py | 119 +++++++++++++++++++++++ python/dllib/test/dev/release/release.sh | 17 +++- scala/make-dist.sh | 70 +++++++++++++ 5 files changed, 208 insertions(+), 4 deletions(-) create mode 100644 python/dllib/src/MANIFEST.in create mode 100644 python/dllib/src/setup.cfg create mode 100755 python/dllib/src/setup.py create mode 100755 scala/make-dist.sh diff --git a/python/dllib/src/MANIFEST.in b/python/dllib/src/MANIFEST.in new file mode 100644 index 00000000000..fe4bb943569 --- /dev/null +++ b/python/dllib/src/MANIFEST.in @@ -0,0 +1 @@ +recursive-include bigdl/share * diff --git a/python/dllib/src/setup.cfg b/python/dllib/src/setup.cfg new file mode 100644 index 00000000000..0499089e987 --- /dev/null +++ b/python/dllib/src/setup.cfg @@ -0,0 +1,5 @@ +[bdist_wheel] +universal = 1 + +[metadata] +description-file = README.md \ No newline at end of file diff --git a/python/dllib/src/setup.py b/python/dllib/src/setup.py new file mode 100755 index 00000000000..d343be18dc4 --- /dev/null +++ b/python/dllib/src/setup.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python + +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os +import sys +from shutil import copyfile, copytree, rmtree + +from setuptools import setup + +TEMP_PATH = "bigdl/share" +bigdl_home = os.path.abspath(__file__ + "/../../") + +try: + exec(open('bigdl/version.py').read()) +except IOError: + print("Failed to load Bigdl version file for packaging. You must be in Bigdl's python dir.") + sys.exit(-1) + +VERSION = __version__ + +building_error_msg = """ +If you are packing python API from BigDL source, you must build BigDL first +and run sdist. + To build BigDL with maven you can run: + cd $BigDL_HOME + ./make-dist.sh + Building the source dist is done in the Python directory: + cd python + python setup.py sdist + pip install dist/*.tar.gz""" + +def build_from_source(): + code_path = bigdl_home + "/python/bigdl/utils/common.py" + print("Checking: %s to see if build from source" % code_path) + if os.path.exists(code_path): + return True + return False + + +def init_env(): + if build_from_source(): + print("Start to build distributed package") + print("HOME OF BIGDL: " + bigdl_home) + dist_source = bigdl_home + "/dist" + if not os.path.exists(dist_source): + print(building_error_msg) + sys.exit(-1) + if os.path.exists(TEMP_PATH): + rmtree(TEMP_PATH) + copytree(dist_source, TEMP_PATH) + copyfile(bigdl_home + "/python/bigdl/dllib/nn/__init__.py", TEMP_PATH + "/__init__.py") + else: + print("Do nothing for release installation") + +def get_bigdl_packages(): + bigdl_python_home = os.path.abspath(__file__)[:-8] + bigdl_packages = ['bigdl.share'] + for dirpath, dirs, files in os.walk(bigdl_python_home + "bigdl"): + package = dirpath.split(bigdl_python_home)[1].replace('/', '.') + if "__pycache__" not in package: + bigdl_packages.append(package) + print("=========================bigdl packages=========================") + print("\n".join(bigdl_packages)) + print("================================================================") + return bigdl_packages + +def setup_package(): + metadata = dict( + name='BigDL', + version=VERSION, + description='Distributed Deep Learning Library for Apache Spark', + author='BigDL Authors', + author_email='bigdl-user-group@googlegroups.com', + license='Apache License, Version 2.0', + url='https://github.com/intel-analytics/Bigdl', + packages=get_bigdl_packages(), + install_requires=['numpy>=1.7', 'pyspark==2.4.6', 'six>=1.10.0'], + dependency_links=['https://d3kbcqa49mib13.cloudfront.net/spark-2.0.0-bin-hadoop2.7.tgz'], + include_package_data=True, + # package_data={"bigdl.share": ['bigdl/share/lib', 'bigdl/share/conf', 'bigdl/share/bin']}, + package_data={"bigdl.share": ['bigdl/share/lib']}, + classifiers=[ + 'License :: OSI Approved :: Apache Software License', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: Implementation :: CPython'], + platforms=['mac', 'linux'] + ) + + setup(**metadata) + + +if __name__ == '__main__': + try: + init_env() + setup_package() + except Exception as e: + raise e + finally: + if build_from_source() and os.path.exists(TEMP_PATH): + rmtree(TEMP_PATH) + diff --git a/python/dllib/test/dev/release/release.sh b/python/dllib/test/dev/release/release.sh index e070f510b47..7b7ec4f62d1 100755 --- a/python/dllib/test/dev/release/release.sh +++ b/python/dllib/test/dev/release/release.sh @@ -19,9 +19,9 @@ set -e RUN_SCRIPT_DIR=$(cd $(dirname $0) ; pwd) echo $RUN_SCRIPT_DIR -BIGDL_DIR="$(cd ${RUN_SCRIPT_DIR}/../; pwd)" +BIGDL_DIR="$(cd ${RUN_SCRIPT_DIR}/../../../..; pwd)" echo $BIGDL_DIR -BIGDL_PYTHON_DIR="$(cd ${RUN_SCRIPT_DIR}/../python; pwd)" +BIGDL_PYTHON_DIR="$(cd ${BIGDL_DIR}/python; pwd)" echo $BIGDL_PYTHON_DIR if (( $# < 2)); then @@ -34,7 +34,7 @@ spark_profile=$2 quick=$3 bigdl_version=$(python -c "exec(open('$BIGDL_DIR/python/bigdl/version.py').read()); print(__version__)") -cd ${BIGDL_DIR} +cd ${BIGDL_DIR}/scala if [ "$platform" == "mac" ]; then echo "Building bigdl for mac system" dist_profile="-P mac -P $spark_profile" @@ -47,11 +47,12 @@ else echo "unsupport platform" fi -bigdl_build_command="${BIGDL_DIR}/scala/make-dist.sh ${dist_profile}" +bigdl_build_command="bash make-dist.sh ${dist_profile}" if [ "$quick" == "true" ]; then echo "Skip disting BigDL" else echo "Dist BigDL: $bigdl_build_command" + cd ${BIGDL_DIR}/scala $bigdl_build_command fi @@ -60,6 +61,14 @@ sdist_command="python setup.py sdist" echo "packing source code: ${sdist_command}" $sdist_command +if [ -d "${BIGDL_DIR}/python/build" ]; then + rm -r ${BIGDL_DIR}/python/build +fi + +if [ -d "${BIGDL_DIR}/python/dist" ]; then + rm -r ${BIGDL_DIR}/python/dist +fi + wheel_command="python setup.py bdist_wheel --plat-name ${verbose_pname}" echo "Packing python distribution: $wheel_command" ${wheel_command} diff --git a/scala/make-dist.sh b/scala/make-dist.sh new file mode 100755 index 00000000000..4fde83e9e89 --- /dev/null +++ b/scala/make-dist.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash + +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# +# Create a folder contain all files for dist +# + +set -e + +# Check java +if type -p java>/dev/null; then + _java=java +else + echo "Java is not installed" + exit 1 +fi + +MVN_OPTS_LIST="-Xmx2g -XX:ReservedCodeCacheSize=512m" + +if [[ "$_java" ]]; then + version=$("$_java" -version 2>&1 | awk -F '"' '/version/ {print $2}') + if [[ "expr $version" < "1.7" ]]; then + echo Require a java version not lower than 1.7 + exit 1 + fi + # For jdk7 + if [[ "expr $version" < "1.8" ]]; then + MVN_OPTS_LIST="$MVN_OPTS_LIST -XX:MaxPermSize=1G" + fi +fi + +export MAVEN_OPTS=${MAVEN_OPTS:-"$MVN_OPTS_LIST"} + +# Check if mvn installed +MVN_INSTALL=$(which mvn 2>/dev/null | grep mvn | wc -l) +if [ $MVN_INSTALL -eq 0 ]; then + echo "MVN is not installed. Exit" + exit 1 +fi + +mvn clean package -DskipTests $* + +BASEDIR=$(dirname "$0") +DIST_DIR=$BASEDIR/../dist/lib + +if [ ! -d "$DIST_DIR" ] +then + mkdir -p $DIST_DIR +else + rm -r $DIST_DIR + mkdir -p $DIST_DIR +fi + +cp -r $BASEDIR/dllib/target/bigdl-dllib-2.0.0-SNAPSHOT-jar-with-dependencies.jar $DIST_DIR +cp -r $BASEDIR/dllib/target/bigdl-dllib-2.0.0-SNAPSHOT-python-api.zip $DIST_DIR From c0e28c310e189a793c37a91b991d6ba2dc3d5a28 Mon Sep 17 00:00:00 2001 From: dding3 Date: Wed, 18 Aug 2021 10:42:06 -0700 Subject: [PATCH 805/823] fix pip test failure --- .../dllib/src/bigdl/dllib/bigdlkeras/__init__.py | 15 +++++++++++++++ .../src/bigdl/dllib/bigdlkeras/layers/topology.py | 2 +- .../dllib/src/bigdl/dllib/examples/lenet/lenet.py | 4 ++-- python/dllib/src/bigdl/dllib/nn/layer.py | 4 ++-- python/dllib/src/python-zip.xml | 4 ++-- python/dllib/test/bigdl/keras/test_keras_api.py | 6 +++--- .../dllib/test/bigdl/test_simple_integration.py | 14 +++++++------- python/dllib/test/bigdl/test_utils.py | 4 ++-- scala/make-dist.sh | 13 ++++++++----- 9 files changed, 42 insertions(+), 24 deletions(-) create mode 100644 python/dllib/src/bigdl/dllib/bigdlkeras/__init__.py diff --git a/python/dllib/src/bigdl/dllib/bigdlkeras/__init__.py b/python/dllib/src/bigdl/dllib/bigdlkeras/__init__.py new file mode 100644 index 00000000000..2151a805423 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/bigdlkeras/__init__.py @@ -0,0 +1,15 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/python/dllib/src/bigdl/dllib/bigdlkeras/layers/topology.py b/python/dllib/src/bigdl/dllib/bigdlkeras/layers/topology.py index 51010298f39..891649342f0 100644 --- a/python/dllib/src/bigdl/dllib/bigdlkeras/layers/topology.py +++ b/python/dllib/src/bigdl/dllib/bigdlkeras/layers/topology.py @@ -14,7 +14,7 @@ # limitations under the License. # -from bigdl.dllib.keras.layer import KerasLayer +from bigdl.dllib.keras.layers.layer import KerasLayer from bigdl.dllib.optim.optimizer import * from bigdl.dllib.nn.criterion import * import multiprocessing diff --git a/python/dllib/src/bigdl/dllib/examples/lenet/lenet.py b/python/dllib/src/bigdl/dllib/examples/lenet/lenet.py index f1fa6f6a44c..8dd314a1c0a 100644 --- a/python/dllib/src/bigdl/dllib/examples/lenet/lenet.py +++ b/python/dllib/src/bigdl/dllib/examples/lenet/lenet.py @@ -15,8 +15,8 @@ # from optparse import OptionParser -from bigdl.dllib.keras.topology import Sequential -from bigdl.dllib.keras.layer import * +from bigdl.dllib.keras.layers.topology import Sequential +from bigdl.dllib.keras.layers.layer import * from bigdl.dllib.feature.dataset import mnist diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 5f36b75bc60..1a712e51f39 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -100,9 +100,9 @@ def get_py_name(jclass_name): if "com.intel.analytics.bigdl.dllib.keras.Model" == jname or \ "com.intel.analytics.bigdl.dllib.keras.Sequential" == jname: - base_module = importlib.import_module('bigdl.dllib.keras.topology') + base_module = importlib.import_module('bigdl.dllib.keras.layers.topology') elif "com.intel.analytics.bigdl.dllib.keras" == jpackage_name: - base_module = importlib.import_module('bigdl.dllib.keras.layer') + base_module = importlib.import_module('bigdl.dllib.keras.layers.layer') else: base_module = importlib.import_module('bigdl.dllib.nn.layer') diff --git a/python/dllib/src/python-zip.xml b/python/dllib/src/python-zip.xml index 1ece79d1ab0..b7ae153f09b 100644 --- a/python/dllib/src/python-zip.xml +++ b/python/dllib/src/python-zip.xml @@ -21,8 +21,8 @@ - - + + diff --git a/python/dllib/test/bigdl/keras/test_keras_api.py b/python/dllib/test/bigdl/keras/test_keras_api.py index 37a79b4fc86..97968b56bd4 100644 --- a/python/dllib/test/bigdl/keras/test_keras_api.py +++ b/python/dllib/test/bigdl/keras/test_keras_api.py @@ -17,13 +17,13 @@ import pytest from test.bigdl.test_utils import BigDLTestCase -import bigdl.dllib.keras.layer as BLayer +import bigdl.dllib.keras.layers.layer as BLayer import keras.layers as KLayer import keras.backend as K from bigdl.dllib.keras.converter import WeightsConverter from bigdl.dataset.dataset import * -from bigdl.dllib.keras.topology import Model as BModel -from bigdl.dllib.keras.topology import Sequential as BSequential +from bigdl.dllib.keras.layers.topology import Model as BModel +from bigdl.dllib.keras.layers.topology import Sequential as BSequential from keras.engine import merge as kmerge, Model as KModel from keras.models import Sequential as KSequential diff --git a/python/dllib/test/bigdl/test_simple_integration.py b/python/dllib/test/bigdl/test_simple_integration.py index cb49fd6213d..981a28e8b72 100644 --- a/python/dllib/test/bigdl/test_simple_integration.py +++ b/python/dllib/test/bigdl/test_simple_integration.py @@ -77,9 +77,9 @@ def grad_update(mlp, x, y, criterion, learning_rate): rtol=1.e-1) def test_load_keras_model_of(self): - from bigdl.dllib.keras.topology import Model as KModel - from bigdl.dllib.keras.layer import Input as KInput - from bigdl.dllib.keras.layer import Dense + from bigdl.dllib.keras.layers.topology import Model as KModel + from bigdl.dllib.keras.layers.layer import Input as KInput + from bigdl.dllib.keras.layers.layer import Dense input = KInput(shape=[2, 3]) fc1 = Dense(2)(input) @@ -87,12 +87,12 @@ def test_load_keras_model_of(self): tmp_path = tempfile.mktemp() model.save(tmp_path, True) model_loaded = KModel.load(tmp_path) - assert "bigdl.dllib.keras.topology.Model" in str(type(model_loaded)) + assert "bigdl.dllib.keras.layers.topology.Model" in str(type(model_loaded)) assert len(model_loaded.layers) == 2 def test_load_keras_seq_of(self): - from bigdl.dllib.keras.topology import Sequential as KSequential - from bigdl.dllib.keras.layer import Dense + from bigdl.dllib.keras.layers.topology import Sequential as KSequential + from bigdl.dllib.keras.layers.layer import Dense model = KSequential() fc1 = Dense(2, input_shape=[2, 3]) @@ -100,7 +100,7 @@ def test_load_keras_seq_of(self): tmp_path = tempfile.mktemp() model.save(tmp_path, True) model_loaded = KSequential.load(tmp_path) - assert "bigdl.dllib.keras.topology.Sequential" in str(type(model_loaded)) + assert "bigdl.dllib.keras.layers.topology.Sequential" in str(type(model_loaded)) assert len(model_loaded.layers) == 1 def test_load_model_of(self): diff --git a/python/dllib/test/bigdl/test_utils.py b/python/dllib/test/bigdl/test_utils.py index a35061c8eda..a96d6dee25a 100644 --- a/python/dllib/test/bigdl/test_utils.py +++ b/python/dllib/test/bigdl/test_utils.py @@ -323,13 +323,13 @@ def compare_loss(self, y_a, y_b, kloss, bloss, rtol=1e-6, atol=1e-6): def compare_newapi(self, klayer, blayer, input_data, weight_converter=None, is_training=False, rtol=1e-6, atol=1e-6): from keras.models import Sequential as KSequential - from bigdl.dllib.keras.topology import Sequential as BSequential + from bigdl.dllib.keras.layers.topology import Sequential as BSequential bmodel = BSequential() bmodel.add(blayer) kmodel = KSequential() kmodel.add(klayer) koutput = kmodel.predict(input_data) - from bigdl.dllib.keras.layer import BatchNormalization + from bigdl.dllib.keras.layers.layer import BatchNormalization if isinstance(blayer, BatchNormalization): k_running_mean = K.eval(klayer.running_mean) k_running_std = K.eval(klayer.running_std) diff --git a/scala/make-dist.sh b/scala/make-dist.sh index 4fde83e9e89..e2f5c5c7426 100755 --- a/scala/make-dist.sh +++ b/scala/make-dist.sh @@ -56,15 +56,18 @@ fi mvn clean package -DskipTests $* BASEDIR=$(dirname "$0") -DIST_DIR=$BASEDIR/../dist/lib +DIST_DIR=$BASEDIR/../dist/ if [ ! -d "$DIST_DIR" ] then - mkdir -p $DIST_DIR + mkdir -p $DIST_DIR/lib + mkdir -p $DIST_DIR/conf else rm -r $DIST_DIR - mkdir -p $DIST_DIR + mkdir -p $DIST_DIR/lib + mkdir -p $DIST_DIR/conf fi -cp -r $BASEDIR/dllib/target/bigdl-dllib-2.0.0-SNAPSHOT-jar-with-dependencies.jar $DIST_DIR -cp -r $BASEDIR/dllib/target/bigdl-dllib-2.0.0-SNAPSHOT-python-api.zip $DIST_DIR +cp -r $BASEDIR/dllib/target/bigdl-dllib-2.0.0-SNAPSHOT-jar-with-dependencies.jar $DIST_DIR/lib +cp -r $BASEDIR/dllib/target/bigdl-dllib-2.0.0-SNAPSHOT-python-api.zip $DIST_DIR/lib +cp -r $BASEDIR/dllib/src/main/resources/spark-bigdl.conf $DIST_DIR/conf From d83c8ae4a4f0f2b695a645fba07d2bac6d893052 Mon Sep 17 00:00:00 2001 From: dding3 Date: Wed, 18 Aug 2021 10:42:06 -0700 Subject: [PATCH 806/823] fix pip test failure --- .../dllib/src/bigdl/dllib/bigdlkeras/__init__.py | 15 +++++++++++++++ .../src/bigdl/dllib/bigdlkeras/layers/topology.py | 2 +- .../dllib/src/bigdl/dllib/examples/lenet/lenet.py | 4 ++-- python/dllib/src/bigdl/dllib/nn/layer.py | 4 ++-- python/dllib/src/python-zip.xml | 4 ++-- python/dllib/test/bigdl/keras/test_keras_api.py | 6 +++--- .../dllib/test/bigdl/test_simple_integration.py | 14 +++++++------- python/dllib/test/bigdl/test_utils.py | 4 ++-- scala/make-dist.sh | 13 ++++++++----- 9 files changed, 42 insertions(+), 24 deletions(-) create mode 100644 python/dllib/src/bigdl/dllib/bigdlkeras/__init__.py diff --git a/python/dllib/src/bigdl/dllib/bigdlkeras/__init__.py b/python/dllib/src/bigdl/dllib/bigdlkeras/__init__.py new file mode 100644 index 00000000000..2151a805423 --- /dev/null +++ b/python/dllib/src/bigdl/dllib/bigdlkeras/__init__.py @@ -0,0 +1,15 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/python/dllib/src/bigdl/dllib/bigdlkeras/layers/topology.py b/python/dllib/src/bigdl/dllib/bigdlkeras/layers/topology.py index 51010298f39..891649342f0 100644 --- a/python/dllib/src/bigdl/dllib/bigdlkeras/layers/topology.py +++ b/python/dllib/src/bigdl/dllib/bigdlkeras/layers/topology.py @@ -14,7 +14,7 @@ # limitations under the License. # -from bigdl.dllib.keras.layer import KerasLayer +from bigdl.dllib.keras.layers.layer import KerasLayer from bigdl.dllib.optim.optimizer import * from bigdl.dllib.nn.criterion import * import multiprocessing diff --git a/python/dllib/src/bigdl/dllib/examples/lenet/lenet.py b/python/dllib/src/bigdl/dllib/examples/lenet/lenet.py index f1fa6f6a44c..8dd314a1c0a 100644 --- a/python/dllib/src/bigdl/dllib/examples/lenet/lenet.py +++ b/python/dllib/src/bigdl/dllib/examples/lenet/lenet.py @@ -15,8 +15,8 @@ # from optparse import OptionParser -from bigdl.dllib.keras.topology import Sequential -from bigdl.dllib.keras.layer import * +from bigdl.dllib.keras.layers.topology import Sequential +from bigdl.dllib.keras.layers.layer import * from bigdl.dllib.feature.dataset import mnist diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 5f36b75bc60..1a712e51f39 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -100,9 +100,9 @@ def get_py_name(jclass_name): if "com.intel.analytics.bigdl.dllib.keras.Model" == jname or \ "com.intel.analytics.bigdl.dllib.keras.Sequential" == jname: - base_module = importlib.import_module('bigdl.dllib.keras.topology') + base_module = importlib.import_module('bigdl.dllib.keras.layers.topology') elif "com.intel.analytics.bigdl.dllib.keras" == jpackage_name: - base_module = importlib.import_module('bigdl.dllib.keras.layer') + base_module = importlib.import_module('bigdl.dllib.keras.layers.layer') else: base_module = importlib.import_module('bigdl.dllib.nn.layer') diff --git a/python/dllib/src/python-zip.xml b/python/dllib/src/python-zip.xml index 1ece79d1ab0..b7ae153f09b 100644 --- a/python/dllib/src/python-zip.xml +++ b/python/dllib/src/python-zip.xml @@ -21,8 +21,8 @@ - - + + diff --git a/python/dllib/test/bigdl/keras/test_keras_api.py b/python/dllib/test/bigdl/keras/test_keras_api.py index 37a79b4fc86..97968b56bd4 100644 --- a/python/dllib/test/bigdl/keras/test_keras_api.py +++ b/python/dllib/test/bigdl/keras/test_keras_api.py @@ -17,13 +17,13 @@ import pytest from test.bigdl.test_utils import BigDLTestCase -import bigdl.dllib.keras.layer as BLayer +import bigdl.dllib.keras.layers.layer as BLayer import keras.layers as KLayer import keras.backend as K from bigdl.dllib.keras.converter import WeightsConverter from bigdl.dataset.dataset import * -from bigdl.dllib.keras.topology import Model as BModel -from bigdl.dllib.keras.topology import Sequential as BSequential +from bigdl.dllib.keras.layers.topology import Model as BModel +from bigdl.dllib.keras.layers.topology import Sequential as BSequential from keras.engine import merge as kmerge, Model as KModel from keras.models import Sequential as KSequential diff --git a/python/dllib/test/bigdl/test_simple_integration.py b/python/dllib/test/bigdl/test_simple_integration.py index cb49fd6213d..981a28e8b72 100644 --- a/python/dllib/test/bigdl/test_simple_integration.py +++ b/python/dllib/test/bigdl/test_simple_integration.py @@ -77,9 +77,9 @@ def grad_update(mlp, x, y, criterion, learning_rate): rtol=1.e-1) def test_load_keras_model_of(self): - from bigdl.dllib.keras.topology import Model as KModel - from bigdl.dllib.keras.layer import Input as KInput - from bigdl.dllib.keras.layer import Dense + from bigdl.dllib.keras.layers.topology import Model as KModel + from bigdl.dllib.keras.layers.layer import Input as KInput + from bigdl.dllib.keras.layers.layer import Dense input = KInput(shape=[2, 3]) fc1 = Dense(2)(input) @@ -87,12 +87,12 @@ def test_load_keras_model_of(self): tmp_path = tempfile.mktemp() model.save(tmp_path, True) model_loaded = KModel.load(tmp_path) - assert "bigdl.dllib.keras.topology.Model" in str(type(model_loaded)) + assert "bigdl.dllib.keras.layers.topology.Model" in str(type(model_loaded)) assert len(model_loaded.layers) == 2 def test_load_keras_seq_of(self): - from bigdl.dllib.keras.topology import Sequential as KSequential - from bigdl.dllib.keras.layer import Dense + from bigdl.dllib.keras.layers.topology import Sequential as KSequential + from bigdl.dllib.keras.layers.layer import Dense model = KSequential() fc1 = Dense(2, input_shape=[2, 3]) @@ -100,7 +100,7 @@ def test_load_keras_seq_of(self): tmp_path = tempfile.mktemp() model.save(tmp_path, True) model_loaded = KSequential.load(tmp_path) - assert "bigdl.dllib.keras.topology.Sequential" in str(type(model_loaded)) + assert "bigdl.dllib.keras.layers.topology.Sequential" in str(type(model_loaded)) assert len(model_loaded.layers) == 1 def test_load_model_of(self): diff --git a/python/dllib/test/bigdl/test_utils.py b/python/dllib/test/bigdl/test_utils.py index a35061c8eda..a96d6dee25a 100644 --- a/python/dllib/test/bigdl/test_utils.py +++ b/python/dllib/test/bigdl/test_utils.py @@ -323,13 +323,13 @@ def compare_loss(self, y_a, y_b, kloss, bloss, rtol=1e-6, atol=1e-6): def compare_newapi(self, klayer, blayer, input_data, weight_converter=None, is_training=False, rtol=1e-6, atol=1e-6): from keras.models import Sequential as KSequential - from bigdl.dllib.keras.topology import Sequential as BSequential + from bigdl.dllib.keras.layers.topology import Sequential as BSequential bmodel = BSequential() bmodel.add(blayer) kmodel = KSequential() kmodel.add(klayer) koutput = kmodel.predict(input_data) - from bigdl.dllib.keras.layer import BatchNormalization + from bigdl.dllib.keras.layers.layer import BatchNormalization if isinstance(blayer, BatchNormalization): k_running_mean = K.eval(klayer.running_mean) k_running_std = K.eval(klayer.running_std) diff --git a/scala/make-dist.sh b/scala/make-dist.sh index 4fde83e9e89..e2f5c5c7426 100755 --- a/scala/make-dist.sh +++ b/scala/make-dist.sh @@ -56,15 +56,18 @@ fi mvn clean package -DskipTests $* BASEDIR=$(dirname "$0") -DIST_DIR=$BASEDIR/../dist/lib +DIST_DIR=$BASEDIR/../dist/ if [ ! -d "$DIST_DIR" ] then - mkdir -p $DIST_DIR + mkdir -p $DIST_DIR/lib + mkdir -p $DIST_DIR/conf else rm -r $DIST_DIR - mkdir -p $DIST_DIR + mkdir -p $DIST_DIR/lib + mkdir -p $DIST_DIR/conf fi -cp -r $BASEDIR/dllib/target/bigdl-dllib-2.0.0-SNAPSHOT-jar-with-dependencies.jar $DIST_DIR -cp -r $BASEDIR/dllib/target/bigdl-dllib-2.0.0-SNAPSHOT-python-api.zip $DIST_DIR +cp -r $BASEDIR/dllib/target/bigdl-dllib-2.0.0-SNAPSHOT-jar-with-dependencies.jar $DIST_DIR/lib +cp -r $BASEDIR/dllib/target/bigdl-dllib-2.0.0-SNAPSHOT-python-api.zip $DIST_DIR/lib +cp -r $BASEDIR/dllib/src/main/resources/spark-bigdl.conf $DIST_DIR/conf From b4012044a4c9168d765a4495c5f27ed28eae63ac Mon Sep 17 00:00:00 2001 From: dding3 Date: Wed, 18 Aug 2021 12:37:32 -0700 Subject: [PATCH 807/823] remove dllib.nn.keras (#4510) --- .../src/bigdl/dllib/nn/keras/__init__.py | 20 ------------------- 1 file changed, 20 deletions(-) delete mode 100644 python/dllib/src/bigdl/dllib/nn/keras/__init__.py diff --git a/python/dllib/src/bigdl/dllib/nn/keras/__init__.py b/python/dllib/src/bigdl/dllib/nn/keras/__init__.py deleted file mode 100644 index 9eb670a1780..00000000000 --- a/python/dllib/src/bigdl/dllib/nn/keras/__init__.py +++ /dev/null @@ -1,20 +0,0 @@ -# -# Copyright 2016 The BigDL Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -from bigdl.utils.common import * - -init_engine() -redire_spark_logs() -show_bigdl_info_logs() From ee953ea325a19a905ced4ae4b2afc8640d381943 Mon Sep 17 00:00:00 2001 From: dding3 Date: Wed, 18 Aug 2021 12:37:32 -0700 Subject: [PATCH 808/823] remove dllib.nn.keras (#4510) --- .../src/bigdl/dllib/nn/keras/__init__.py | 20 ------------------- 1 file changed, 20 deletions(-) delete mode 100644 python/dllib/src/bigdl/dllib/nn/keras/__init__.py diff --git a/python/dllib/src/bigdl/dllib/nn/keras/__init__.py b/python/dllib/src/bigdl/dllib/nn/keras/__init__.py deleted file mode 100644 index 9eb670a1780..00000000000 --- a/python/dllib/src/bigdl/dllib/nn/keras/__init__.py +++ /dev/null @@ -1,20 +0,0 @@ -# -# Copyright 2016 The BigDL Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -from bigdl.utils.common import * - -init_engine() -redire_spark_logs() -show_bigdl_info_logs() From f0cdb0799425b7f25c70dbf0e8f4c6df8d12bb6e Mon Sep 17 00:00:00 2001 From: dding3 Date: Wed, 18 Aug 2021 16:00:25 -0700 Subject: [PATCH 809/823] uncomment EngineSpec (#4511) --- .../utils/src/main/resources/spark-bigdl.conf | 33 ++ .../bigdl/dllib/utils/EngineSpec.scala | 320 +++++++++--------- 2 files changed, 193 insertions(+), 160 deletions(-) create mode 100644 scala/common/utils/src/main/resources/spark-bigdl.conf diff --git a/scala/common/utils/src/main/resources/spark-bigdl.conf b/scala/common/utils/src/main/resources/spark-bigdl.conf new file mode 100644 index 00000000000..0e7737ba6d9 --- /dev/null +++ b/scala/common/utils/src/main/resources/spark-bigdl.conf @@ -0,0 +1,33 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# +# Spark confs for BigDL. +# +# If you can init SparkContext in your application, you should call Engine.createSparkConf to get a +# SparkConf. The properties will be set. You neednot pass these properties. +# +# If you use spark-shell or pyspark, you need to pass these properties in. You can put them +# in your spark conf file. +# +# For more details, please refer +# https://bigdl-project.github.io/master/#APIGuide/Engine/ +# + +spark.shuffle.reduceLocality.enabled false +spark.shuffle.blockTransferService nio +spark.scheduler.minRegisteredResourcesRatio 1.0 +spark.speculation false diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/EngineSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/EngineSpec.scala index 408ecc34e8a..e7388812e57 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/EngineSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/EngineSpec.scala @@ -14,163 +14,163 @@ * limitations under the License. */ -// package com.intel.analytics.bigdl.utils -// -// import org.apache.spark.SparkContext -// import org.scalatest.{BeforeAndAfter, FlatSpec, Matchers} -// -// class EngineSpec extends FlatSpec with Matchers with BeforeAndAfter { -// -// private var sc: SparkContext = _ -// -// before { -// sc = null -// Engine.reset -// } -// -// after { -// if (sc != null) { -// sc.stop() -// sc = null -// } -// Engine.reset -// } -// -// "Engine" should "be inited correct under no spark environment" in { -// System.setProperty("bigdl.localMode", "true") -// Engine.init -// Engine.nodeNumber should be(1) -// System.clearProperty("bigdl.localMode") -// } -// -// it should "be inited correct under spark local environment" in { -// val conf = Engine.createSparkConf().setAppName("EngineSpecTest").setMaster("local[4]") -// sc = SparkContext.getOrCreate(conf) -// Engine.init -// Engine.nodeNumber should be(1) -// Engine.coreNumber should be(4) -// } -// -// it should "be inited correct localmode under no spark environment with java property" in { -// val property = "bigdl.localMode" -// System.setProperty(property, "true") -// Engine.init -// Engine.nodeNumber should be(1) -// System.clearProperty(property) -// } -// -// it should "be inited correct coreNumber under no spark environment with java property" in { -// val localMode = "bigdl.localMode" -// val coreNumber = "bigdl.coreNumber" -// System.setProperty("bigdl.localMode", "true") -// System.setProperty(coreNumber, "3") -// -// val tmp = System.getProperty("bigdl.localMode") -// -// Engine.init -// Engine.localMode should be(true) -// Engine.nodeNumber should be(1) -// Engine.coreNumber should be(3) -// -// System.clearProperty("bigdl.localMode") -// System.clearProperty(coreNumber) -// } -// -// it should "be inited with correct value under spark local environment" in { -// Engine.init(1, 4, true) -// Engine.nodeNumber should be(1) -// Engine.coreNumber should be(4) -// } -// -// it should "parse nodes, executors correctly for Spark standalone" in { -// val conf = Engine.createSparkConf().setAppName("EngineSpecTest"). -// setMaster("spark://localhost:1234"). -// set("spark.cores.max", "24").set("spark.executor.cores", "4") -// Engine.parseExecutorAndCore(conf) should be(Some(6, 4)) -// } -// -// it should "parse nodes, executors correctly for Spark standalone with single executor" in { -// val conf = Engine.createSparkConf().setAppName("EngineSpecTest"). -// setMaster("spark://localhost:1234"). -// set("spark.cores.max", "4").set("spark.executor.cores", "4") -// Engine.parseExecutorAndCore(conf) should be(Some(1, 4)) -// } -// -// it should "parse nodes, executors correctly for Spark YARN" in { -// val conf = Engine.createSparkConf().setAppName("EngineSpecTest").setMaster("yarn"). -// set("spark.executor.instances", "6").set("spark.executor.cores", "4") -// Engine.parseExecutorAndCore(conf) should be(Some(6, 4)) -// } -// -// it should "parse nodes, executors correctly for Spark YARN with single executor" in { -// val conf = Engine.createSparkConf().setAppName("EngineSpecTest").setMaster("yarn"). -// set("spark.executor.instances", "1").set("spark.executor.cores", "4") -// Engine.parseExecutorAndCore(conf) should be(Some(1, 4)) -// } -// -// it should "parse nodes, executors correctly for Spark Mesos" in { -// val conf = Engine.createSparkConf().setAppName("EngineSpecTest"). -// setMaster("mesos://localhost:1234"). -// set("spark.cores.max", "24").set("spark.executor.cores", "4") -// Engine.parseExecutorAndCore(conf) should be(Some(6, 4)) -// } -// -// it should "parse nodes, executors correctly for Spark Mesos with single executor" in { -// val conf = Engine.createSparkConf().setAppName("EngineSpecTest"). -// setMaster("mesos://localhost:1234"). -// set("spark.cores.max", "4").set("spark.executor.cores", "4") -// Engine.parseExecutorAndCore(conf) should be(Some(1, 4)) -// } -// -// "sparkExecutorAndCore" should "parse local[*]" in { -// val conf = Engine.createSparkConf().setAppName("EngineSpecTest").setMaster("local[*]") -// val (nExecutor, _) = Engine.parseExecutorAndCore(conf).get -// nExecutor should be(1) -// } -// -// "readConf" should "be right" in { -// val conf = Engine.readConf -// val target = Map( -// "spark.shuffle.reduceLocality.enabled" -> "false", -// "spark.shuffle.blockTransferService" -> "nio", -// "spark.scheduler.minRegisteredResourcesRatio" -> "1.0", -// "spark.speculation" -> "false" -// ) -// conf.length should be(target.keys.size) -// conf.foreach(s => { -// s._2 should be(target(s._1)) -// }) -// } -// -// "readConf" should "skip blockTransferService if bigdl.network.nio is set to false" in { -// System.setProperty("bigdl.network.nio", "false") -// val conf = Engine.readConf -// val target = Map( -// "spark.shuffle.reduceLocality.enabled" -> "false", -// "spark.scheduler.minRegisteredResourcesRatio" -> "1.0", -// "spark.speculation" -> "false" -// ) -// conf.length should be(target.keys.size) -// conf.foreach(s => { -// s._2 should be(target(s._1)) -// }) -// System.clearProperty("bigdl.network.nio") -// } -// -// "LocalMode" should "false if onSpark" in { -// intercept[IllegalArgumentException] { -// System.setProperty("bigdl.localMode", "true") -// Engine.init(1, 1, true) -// System.clearProperty("bigdl.localMode") -// } -// } -// -// "LocalMode" should "true if not onSpark" in { -// intercept[IllegalArgumentException] { -// System.setProperty("bigdl.localMode", "false") -// Engine.init(1, 1, false) -// System.clearProperty("bigdl.localMode") -// } -// } -// -// } + package com.intel.analytics.bigdl.utils + + import org.apache.spark.SparkContext + import org.scalatest.{BeforeAndAfter, FlatSpec, Matchers} + + class EngineSpec extends FlatSpec with Matchers with BeforeAndAfter { + + private var sc: SparkContext = _ + + before { + sc = null + Engine.reset + } + + after { + if (sc != null) { + sc.stop() + sc = null + } + Engine.reset + } + + "Engine" should "be inited correct under no spark environment" in { + System.setProperty("bigdl.localMode", "true") + Engine.init + Engine.nodeNumber should be(1) + System.clearProperty("bigdl.localMode") + } + + it should "be inited correct under spark local environment" in { + val conf = Engine.createSparkConf().setAppName("EngineSpecTest").setMaster("local[4]") + sc = SparkContext.getOrCreate(conf) + Engine.init + Engine.nodeNumber should be(1) + Engine.coreNumber should be(4) + } + + it should "be inited correct localmode under no spark environment with java property" in { + val property = "bigdl.localMode" + System.setProperty(property, "true") + Engine.init + Engine.nodeNumber should be(1) + System.clearProperty(property) + } + + it should "be inited correct coreNumber under no spark environment with java property" in { + val localMode = "bigdl.localMode" + val coreNumber = "bigdl.coreNumber" + System.setProperty("bigdl.localMode", "true") + System.setProperty(coreNumber, "3") + + val tmp = System.getProperty("bigdl.localMode") + + Engine.init + Engine.localMode should be(true) + Engine.nodeNumber should be(1) + Engine.coreNumber should be(3) + + System.clearProperty("bigdl.localMode") + System.clearProperty(coreNumber) + } + + it should "be inited with correct value under spark local environment" in { + Engine.init(1, 4, true) + Engine.nodeNumber should be(1) + Engine.coreNumber should be(4) + } + + it should "parse nodes, executors correctly for Spark standalone" in { + val conf = Engine.createSparkConf().setAppName("EngineSpecTest"). + setMaster("spark://localhost:1234"). + set("spark.cores.max", "24").set("spark.executor.cores", "4") + Engine.parseExecutorAndCore(conf) should be(Some(6, 4)) + } + + it should "parse nodes, executors correctly for Spark standalone with single executor" in { + val conf = Engine.createSparkConf().setAppName("EngineSpecTest"). + setMaster("spark://localhost:1234"). + set("spark.cores.max", "4").set("spark.executor.cores", "4") + Engine.parseExecutorAndCore(conf) should be(Some(1, 4)) + } + + it should "parse nodes, executors correctly for Spark YARN" in { + val conf = Engine.createSparkConf().setAppName("EngineSpecTest").setMaster("yarn"). + set("spark.executor.instances", "6").set("spark.executor.cores", "4") + Engine.parseExecutorAndCore(conf) should be(Some(6, 4)) + } + + it should "parse nodes, executors correctly for Spark YARN with single executor" in { + val conf = Engine.createSparkConf().setAppName("EngineSpecTest").setMaster("yarn"). + set("spark.executor.instances", "1").set("spark.executor.cores", "4") + Engine.parseExecutorAndCore(conf) should be(Some(1, 4)) + } + + it should "parse nodes, executors correctly for Spark Mesos" in { + val conf = Engine.createSparkConf().setAppName("EngineSpecTest"). + setMaster("mesos://localhost:1234"). + set("spark.cores.max", "24").set("spark.executor.cores", "4") + Engine.parseExecutorAndCore(conf) should be(Some(6, 4)) + } + + it should "parse nodes, executors correctly for Spark Mesos with single executor" in { + val conf = Engine.createSparkConf().setAppName("EngineSpecTest"). + setMaster("mesos://localhost:1234"). + set("spark.cores.max", "4").set("spark.executor.cores", "4") + Engine.parseExecutorAndCore(conf) should be(Some(1, 4)) + } + + "sparkExecutorAndCore" should "parse local[*]" in { + val conf = Engine.createSparkConf().setAppName("EngineSpecTest").setMaster("local[*]") + val (nExecutor, _) = Engine.parseExecutorAndCore(conf).get + nExecutor should be(1) + } + + "readConf" should "be right" in { + val conf = Engine.readConf + val target = Map( + "spark.shuffle.reduceLocality.enabled" -> "false", + "spark.shuffle.blockTransferService" -> "nio", + "spark.scheduler.minRegisteredResourcesRatio" -> "1.0", + "spark.speculation" -> "false" + ) + conf.length should be(target.keys.size) + conf.foreach(s => { + s._2 should be(target(s._1)) + }) + } + + "readConf" should "skip blockTransferService if bigdl.network.nio is set to false" in { + System.setProperty("bigdl.network.nio", "false") + val conf = Engine.readConf + val target = Map( + "spark.shuffle.reduceLocality.enabled" -> "false", + "spark.scheduler.minRegisteredResourcesRatio" -> "1.0", + "spark.speculation" -> "false" + ) + conf.length should be(target.keys.size) + conf.foreach(s => { + s._2 should be(target(s._1)) + }) + System.clearProperty("bigdl.network.nio") + } + + "LocalMode" should "false if onSpark" in { + intercept[IllegalArgumentException] { + System.setProperty("bigdl.localMode", "true") + Engine.init(1, 1, true) + System.clearProperty("bigdl.localMode") + } + } + + "LocalMode" should "true if not onSpark" in { + intercept[IllegalArgumentException] { + System.setProperty("bigdl.localMode", "false") + Engine.init(1, 1, false) + System.clearProperty("bigdl.localMode") + } + } + + } From 26eede0145c920061a091246689b833d6b7c8d9e Mon Sep 17 00:00:00 2001 From: dding3 Date: Wed, 18 Aug 2021 16:00:25 -0700 Subject: [PATCH 810/823] uncomment EngineSpec (#4511) --- .../utils/src/main/resources/spark-bigdl.conf | 33 ++ .../analytics/bigdl/utils/EngineSpec.scala | 320 +++++++++--------- 2 files changed, 193 insertions(+), 160 deletions(-) create mode 100644 scala/common/utils/src/main/resources/spark-bigdl.conf diff --git a/scala/common/utils/src/main/resources/spark-bigdl.conf b/scala/common/utils/src/main/resources/spark-bigdl.conf new file mode 100644 index 00000000000..0e7737ba6d9 --- /dev/null +++ b/scala/common/utils/src/main/resources/spark-bigdl.conf @@ -0,0 +1,33 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# +# Spark confs for BigDL. +# +# If you can init SparkContext in your application, you should call Engine.createSparkConf to get a +# SparkConf. The properties will be set. You neednot pass these properties. +# +# If you use spark-shell or pyspark, you need to pass these properties in. You can put them +# in your spark conf file. +# +# For more details, please refer +# https://bigdl-project.github.io/master/#APIGuide/Engine/ +# + +spark.shuffle.reduceLocality.enabled false +spark.shuffle.blockTransferService nio +spark.scheduler.minRegisteredResourcesRatio 1.0 +spark.speculation false diff --git a/scala/common/utils/src/test/scala/com/intel/analytics/bigdl/utils/EngineSpec.scala b/scala/common/utils/src/test/scala/com/intel/analytics/bigdl/utils/EngineSpec.scala index 408ecc34e8a..e7388812e57 100644 --- a/scala/common/utils/src/test/scala/com/intel/analytics/bigdl/utils/EngineSpec.scala +++ b/scala/common/utils/src/test/scala/com/intel/analytics/bigdl/utils/EngineSpec.scala @@ -14,163 +14,163 @@ * limitations under the License. */ -// package com.intel.analytics.bigdl.utils -// -// import org.apache.spark.SparkContext -// import org.scalatest.{BeforeAndAfter, FlatSpec, Matchers} -// -// class EngineSpec extends FlatSpec with Matchers with BeforeAndAfter { -// -// private var sc: SparkContext = _ -// -// before { -// sc = null -// Engine.reset -// } -// -// after { -// if (sc != null) { -// sc.stop() -// sc = null -// } -// Engine.reset -// } -// -// "Engine" should "be inited correct under no spark environment" in { -// System.setProperty("bigdl.localMode", "true") -// Engine.init -// Engine.nodeNumber should be(1) -// System.clearProperty("bigdl.localMode") -// } -// -// it should "be inited correct under spark local environment" in { -// val conf = Engine.createSparkConf().setAppName("EngineSpecTest").setMaster("local[4]") -// sc = SparkContext.getOrCreate(conf) -// Engine.init -// Engine.nodeNumber should be(1) -// Engine.coreNumber should be(4) -// } -// -// it should "be inited correct localmode under no spark environment with java property" in { -// val property = "bigdl.localMode" -// System.setProperty(property, "true") -// Engine.init -// Engine.nodeNumber should be(1) -// System.clearProperty(property) -// } -// -// it should "be inited correct coreNumber under no spark environment with java property" in { -// val localMode = "bigdl.localMode" -// val coreNumber = "bigdl.coreNumber" -// System.setProperty("bigdl.localMode", "true") -// System.setProperty(coreNumber, "3") -// -// val tmp = System.getProperty("bigdl.localMode") -// -// Engine.init -// Engine.localMode should be(true) -// Engine.nodeNumber should be(1) -// Engine.coreNumber should be(3) -// -// System.clearProperty("bigdl.localMode") -// System.clearProperty(coreNumber) -// } -// -// it should "be inited with correct value under spark local environment" in { -// Engine.init(1, 4, true) -// Engine.nodeNumber should be(1) -// Engine.coreNumber should be(4) -// } -// -// it should "parse nodes, executors correctly for Spark standalone" in { -// val conf = Engine.createSparkConf().setAppName("EngineSpecTest"). -// setMaster("spark://localhost:1234"). -// set("spark.cores.max", "24").set("spark.executor.cores", "4") -// Engine.parseExecutorAndCore(conf) should be(Some(6, 4)) -// } -// -// it should "parse nodes, executors correctly for Spark standalone with single executor" in { -// val conf = Engine.createSparkConf().setAppName("EngineSpecTest"). -// setMaster("spark://localhost:1234"). -// set("spark.cores.max", "4").set("spark.executor.cores", "4") -// Engine.parseExecutorAndCore(conf) should be(Some(1, 4)) -// } -// -// it should "parse nodes, executors correctly for Spark YARN" in { -// val conf = Engine.createSparkConf().setAppName("EngineSpecTest").setMaster("yarn"). -// set("spark.executor.instances", "6").set("spark.executor.cores", "4") -// Engine.parseExecutorAndCore(conf) should be(Some(6, 4)) -// } -// -// it should "parse nodes, executors correctly for Spark YARN with single executor" in { -// val conf = Engine.createSparkConf().setAppName("EngineSpecTest").setMaster("yarn"). -// set("spark.executor.instances", "1").set("spark.executor.cores", "4") -// Engine.parseExecutorAndCore(conf) should be(Some(1, 4)) -// } -// -// it should "parse nodes, executors correctly for Spark Mesos" in { -// val conf = Engine.createSparkConf().setAppName("EngineSpecTest"). -// setMaster("mesos://localhost:1234"). -// set("spark.cores.max", "24").set("spark.executor.cores", "4") -// Engine.parseExecutorAndCore(conf) should be(Some(6, 4)) -// } -// -// it should "parse nodes, executors correctly for Spark Mesos with single executor" in { -// val conf = Engine.createSparkConf().setAppName("EngineSpecTest"). -// setMaster("mesos://localhost:1234"). -// set("spark.cores.max", "4").set("spark.executor.cores", "4") -// Engine.parseExecutorAndCore(conf) should be(Some(1, 4)) -// } -// -// "sparkExecutorAndCore" should "parse local[*]" in { -// val conf = Engine.createSparkConf().setAppName("EngineSpecTest").setMaster("local[*]") -// val (nExecutor, _) = Engine.parseExecutorAndCore(conf).get -// nExecutor should be(1) -// } -// -// "readConf" should "be right" in { -// val conf = Engine.readConf -// val target = Map( -// "spark.shuffle.reduceLocality.enabled" -> "false", -// "spark.shuffle.blockTransferService" -> "nio", -// "spark.scheduler.minRegisteredResourcesRatio" -> "1.0", -// "spark.speculation" -> "false" -// ) -// conf.length should be(target.keys.size) -// conf.foreach(s => { -// s._2 should be(target(s._1)) -// }) -// } -// -// "readConf" should "skip blockTransferService if bigdl.network.nio is set to false" in { -// System.setProperty("bigdl.network.nio", "false") -// val conf = Engine.readConf -// val target = Map( -// "spark.shuffle.reduceLocality.enabled" -> "false", -// "spark.scheduler.minRegisteredResourcesRatio" -> "1.0", -// "spark.speculation" -> "false" -// ) -// conf.length should be(target.keys.size) -// conf.foreach(s => { -// s._2 should be(target(s._1)) -// }) -// System.clearProperty("bigdl.network.nio") -// } -// -// "LocalMode" should "false if onSpark" in { -// intercept[IllegalArgumentException] { -// System.setProperty("bigdl.localMode", "true") -// Engine.init(1, 1, true) -// System.clearProperty("bigdl.localMode") -// } -// } -// -// "LocalMode" should "true if not onSpark" in { -// intercept[IllegalArgumentException] { -// System.setProperty("bigdl.localMode", "false") -// Engine.init(1, 1, false) -// System.clearProperty("bigdl.localMode") -// } -// } -// -// } + package com.intel.analytics.bigdl.utils + + import org.apache.spark.SparkContext + import org.scalatest.{BeforeAndAfter, FlatSpec, Matchers} + + class EngineSpec extends FlatSpec with Matchers with BeforeAndAfter { + + private var sc: SparkContext = _ + + before { + sc = null + Engine.reset + } + + after { + if (sc != null) { + sc.stop() + sc = null + } + Engine.reset + } + + "Engine" should "be inited correct under no spark environment" in { + System.setProperty("bigdl.localMode", "true") + Engine.init + Engine.nodeNumber should be(1) + System.clearProperty("bigdl.localMode") + } + + it should "be inited correct under spark local environment" in { + val conf = Engine.createSparkConf().setAppName("EngineSpecTest").setMaster("local[4]") + sc = SparkContext.getOrCreate(conf) + Engine.init + Engine.nodeNumber should be(1) + Engine.coreNumber should be(4) + } + + it should "be inited correct localmode under no spark environment with java property" in { + val property = "bigdl.localMode" + System.setProperty(property, "true") + Engine.init + Engine.nodeNumber should be(1) + System.clearProperty(property) + } + + it should "be inited correct coreNumber under no spark environment with java property" in { + val localMode = "bigdl.localMode" + val coreNumber = "bigdl.coreNumber" + System.setProperty("bigdl.localMode", "true") + System.setProperty(coreNumber, "3") + + val tmp = System.getProperty("bigdl.localMode") + + Engine.init + Engine.localMode should be(true) + Engine.nodeNumber should be(1) + Engine.coreNumber should be(3) + + System.clearProperty("bigdl.localMode") + System.clearProperty(coreNumber) + } + + it should "be inited with correct value under spark local environment" in { + Engine.init(1, 4, true) + Engine.nodeNumber should be(1) + Engine.coreNumber should be(4) + } + + it should "parse nodes, executors correctly for Spark standalone" in { + val conf = Engine.createSparkConf().setAppName("EngineSpecTest"). + setMaster("spark://localhost:1234"). + set("spark.cores.max", "24").set("spark.executor.cores", "4") + Engine.parseExecutorAndCore(conf) should be(Some(6, 4)) + } + + it should "parse nodes, executors correctly for Spark standalone with single executor" in { + val conf = Engine.createSparkConf().setAppName("EngineSpecTest"). + setMaster("spark://localhost:1234"). + set("spark.cores.max", "4").set("spark.executor.cores", "4") + Engine.parseExecutorAndCore(conf) should be(Some(1, 4)) + } + + it should "parse nodes, executors correctly for Spark YARN" in { + val conf = Engine.createSparkConf().setAppName("EngineSpecTest").setMaster("yarn"). + set("spark.executor.instances", "6").set("spark.executor.cores", "4") + Engine.parseExecutorAndCore(conf) should be(Some(6, 4)) + } + + it should "parse nodes, executors correctly for Spark YARN with single executor" in { + val conf = Engine.createSparkConf().setAppName("EngineSpecTest").setMaster("yarn"). + set("spark.executor.instances", "1").set("spark.executor.cores", "4") + Engine.parseExecutorAndCore(conf) should be(Some(1, 4)) + } + + it should "parse nodes, executors correctly for Spark Mesos" in { + val conf = Engine.createSparkConf().setAppName("EngineSpecTest"). + setMaster("mesos://localhost:1234"). + set("spark.cores.max", "24").set("spark.executor.cores", "4") + Engine.parseExecutorAndCore(conf) should be(Some(6, 4)) + } + + it should "parse nodes, executors correctly for Spark Mesos with single executor" in { + val conf = Engine.createSparkConf().setAppName("EngineSpecTest"). + setMaster("mesos://localhost:1234"). + set("spark.cores.max", "4").set("spark.executor.cores", "4") + Engine.parseExecutorAndCore(conf) should be(Some(1, 4)) + } + + "sparkExecutorAndCore" should "parse local[*]" in { + val conf = Engine.createSparkConf().setAppName("EngineSpecTest").setMaster("local[*]") + val (nExecutor, _) = Engine.parseExecutorAndCore(conf).get + nExecutor should be(1) + } + + "readConf" should "be right" in { + val conf = Engine.readConf + val target = Map( + "spark.shuffle.reduceLocality.enabled" -> "false", + "spark.shuffle.blockTransferService" -> "nio", + "spark.scheduler.minRegisteredResourcesRatio" -> "1.0", + "spark.speculation" -> "false" + ) + conf.length should be(target.keys.size) + conf.foreach(s => { + s._2 should be(target(s._1)) + }) + } + + "readConf" should "skip blockTransferService if bigdl.network.nio is set to false" in { + System.setProperty("bigdl.network.nio", "false") + val conf = Engine.readConf + val target = Map( + "spark.shuffle.reduceLocality.enabled" -> "false", + "spark.scheduler.minRegisteredResourcesRatio" -> "1.0", + "spark.speculation" -> "false" + ) + conf.length should be(target.keys.size) + conf.foreach(s => { + s._2 should be(target(s._1)) + }) + System.clearProperty("bigdl.network.nio") + } + + "LocalMode" should "false if onSpark" in { + intercept[IllegalArgumentException] { + System.setProperty("bigdl.localMode", "true") + Engine.init(1, 1, true) + System.clearProperty("bigdl.localMode") + } + } + + "LocalMode" should "true if not onSpark" in { + intercept[IllegalArgumentException] { + System.setProperty("bigdl.localMode", "false") + Engine.init(1, 1, false) + System.clearProperty("bigdl.localMode") + } + } + + } From 8bd20db7888dc9b41d66c0b015672760b461dcdd Mon Sep 17 00:00:00 2001 From: Le-Zheng <30695225+Le-Zheng@users.noreply.github.com> Date: Fri, 20 Aug 2021 10:26:41 +0800 Subject: [PATCH 811/823] change integration.torch ut path for testing (#4521) * change ut path for testing --- .../intel/analytics/bigdl/dllib/integration/HdfsSpec.scala | 2 +- .../analytics/bigdl/dllib/integration/Quantization.scala | 2 +- .../com/intel/analytics/bigdl/dllib/integration/S3Spec.scala | 2 +- .../analytics/bigdl/dllib/integration/SparkModeSpec.scala | 2 +- .../bigdl/dllib/integration/torch/AbsCriterionSpec.scala | 2 +- .../analytics/bigdl/dllib/integration/torch/AbsSpec.scala | 2 +- .../analytics/bigdl/dllib/integration/torch/AdagradSpec.scala | 2 +- .../bigdl/dllib/integration/torch/AddConstantSpec.scala | 2 +- .../analytics/bigdl/dllib/integration/torch/AddSpec.scala | 2 +- .../bigdl/dllib/integration/torch/BCECriterionSpec.scala | 2 +- .../dllib/integration/torch/BatchNormalizationSpec.scala | 2 +- .../bigdl/dllib/integration/torch/BiRecurrentSpec.scala | 2 +- .../bigdl/dllib/integration/torch/BilinearSpec.scala | 2 +- .../analytics/bigdl/dllib/integration/torch/BottleSpec.scala | 2 +- .../analytics/bigdl/dllib/integration/torch/CAddSpec.scala | 2 +- .../bigdl/dllib/integration/torch/CAddTableSpec.scala | 2 +- .../bigdl/dllib/integration/torch/CDivTableSpec.scala | 2 +- .../bigdl/dllib/integration/torch/CMaxTableSpec.scala | 2 +- .../bigdl/dllib/integration/torch/CMinTableSpec.scala | 2 +- .../analytics/bigdl/dllib/integration/torch/CMulSpec.scala | 2 +- .../bigdl/dllib/integration/torch/CMulTableSpec.scala | 2 +- .../bigdl/dllib/integration/torch/CSubTableSpec.scala | 2 +- .../analytics/bigdl/dllib/integration/torch/ClampSpec.scala | 2 +- .../bigdl/dllib/integration/torch/ClassNLLCriterionSpec.scala | 2 +- .../dllib/integration/torch/ClassSimplexCriterionSpec.scala | 2 +- .../bigdl/dllib/integration/torch/ColorJitterSpec.scala | 2 +- .../analytics/bigdl/dllib/integration/torch/ConcatSpec.scala | 2 +- .../bigdl/dllib/integration/torch/ConcatTableSpec.scala | 2 +- .../bigdl/dllib/integration/torch/ContiguousSpec.scala | 2 +- .../dllib/integration/torch/ConvLSTMPeephole3DSpec.scala | 2 +- .../dllib/integration/torch/CosineDistanceCriterionSpec.scala | 2 +- .../bigdl/dllib/integration/torch/CosineDistanceSpec.scala | 2 +- .../integration/torch/CosineEmbeddingCriterionSpec.scala | 2 +- .../analytics/bigdl/dllib/integration/torch/CosineSpec.scala | 2 +- .../dllib/integration/torch/CrossEntropyCriterionSpec.scala | 2 +- .../bigdl/dllib/integration/torch/DenseTensorMathSpec.scala | 2 +- .../dllib/integration/torch/DistKLDivCriterionSpec.scala | 2 +- .../analytics/bigdl/dllib/integration/torch/DropoutSpec.scala | 2 +- .../analytics/bigdl/dllib/integration/torch/ELUSpec.scala | 2 +- .../bigdl/dllib/integration/torch/EuclideanSpec.scala | 2 +- .../analytics/bigdl/dllib/integration/torch/ExpSpec.scala | 2 +- .../analytics/bigdl/dllib/integration/torch/GRUSpec.scala | 2 +- .../bigdl/dllib/integration/torch/GaussianCriterionSpec.scala | 2 +- .../bigdl/dllib/integration/torch/GradientReversalSpec.scala | 2 +- .../bigdl/dllib/integration/torch/HardShrinkSpec.scala | 2 +- .../bigdl/dllib/integration/torch/HardTanhSpec.scala | 2 +- .../dllib/integration/torch/HingeEmbeddingCriterionSpec.scala | 2 +- .../analytics/bigdl/dllib/integration/torch/IndexSpec.scala | 2 +- .../bigdl/dllib/integration/torch/JoinTableSpec.scala | 2 +- .../analytics/bigdl/dllib/integration/torch/L1CostSpec.scala | 2 +- .../integration/torch/L1HingeEmbeddingCriterionSpec.scala | 2 +- .../bigdl/dllib/integration/torch/L1PenaltySpec.scala | 2 +- .../bigdl/dllib/integration/torch/LSTMPeepholeSpec.scala | 2 +- .../analytics/bigdl/dllib/integration/torch/LSTMSpec.scala | 2 +- .../bigdl/dllib/integration/torch/LeakyReLUSpec.scala | 2 +- .../analytics/bigdl/dllib/integration/torch/LinearSpec.scala | 2 +- .../bigdl/dllib/integration/torch/LogSigmoidSpec.scala | 2 +- .../bigdl/dllib/integration/torch/LogSoftMaxSpec.scala | 2 +- .../analytics/bigdl/dllib/integration/torch/LogSpec.scala | 2 +- .../bigdl/dllib/integration/torch/LookupTableSpec.scala | 2 +- .../analytics/bigdl/dllib/integration/torch/MMSpec.scala | 2 +- .../bigdl/dllib/integration/torch/MSECriterionSpec.scala | 2 +- .../analytics/bigdl/dllib/integration/torch/MVSpec.scala | 2 +- .../bigdl/dllib/integration/torch/MarginCriterionSpec.scala | 2 +- .../dllib/integration/torch/MarginRankingCriterionSpec.scala | 2 +- .../bigdl/dllib/integration/torch/MaskedSelectSpec.scala | 2 +- .../analytics/bigdl/dllib/integration/torch/MaxSpec.scala | 2 +- .../analytics/bigdl/dllib/integration/torch/MeanSpec.scala | 2 +- .../analytics/bigdl/dllib/integration/torch/MinSpec.scala | 2 +- .../bigdl/dllib/integration/torch/MixtureTableSpec.scala | 2 +- .../analytics/bigdl/dllib/integration/torch/ModuleSpec.scala | 2 +- .../bigdl/dllib/integration/torch/MulConstantSpec.scala | 2 +- .../analytics/bigdl/dllib/integration/torch/MulSpec.scala | 2 +- .../bigdl/dllib/integration/torch/MultiCriterionSpec.scala | 2 +- .../integration/torch/MultiLabelMarginCriterionSpec.scala | 2 +- .../integration/torch/MultiLabelSoftMarginCriterionSpec.scala | 2 +- .../dllib/integration/torch/MultiMarginCriterionSpec.scala | 2 +- .../analytics/bigdl/dllib/integration/torch/NarrowSpec.scala | 2 +- .../bigdl/dllib/integration/torch/NarrowTableSpec.scala | 2 +- .../bigdl/dllib/integration/torch/NormalizeSpec.scala | 2 +- .../analytics/bigdl/dllib/integration/torch/PReLUSpec.scala | 2 +- .../analytics/bigdl/dllib/integration/torch/PaddingSpec.scala | 2 +- .../bigdl/dllib/integration/torch/PairwiseDistanceSpec.scala | 2 +- .../bigdl/dllib/integration/torch/ParallelCriterionSpec.scala | 2 +- .../analytics/bigdl/dllib/integration/torch/PowerSpec.scala | 2 +- .../analytics/bigdl/dllib/integration/torch/RReLUSpec.scala | 2 +- .../analytics/bigdl/dllib/integration/torch/ReLU6Spec.scala | 2 +- .../analytics/bigdl/dllib/integration/torch/ReLUSpec.scala | 2 +- .../bigdl/dllib/integration/torch/ReplicateSpec.scala | 2 +- .../analytics/bigdl/dllib/integration/torch/ReshapeSpec.scala | 2 +- .../analytics/bigdl/dllib/integration/torch/SamplerSpec.scala | 2 +- .../analytics/bigdl/dllib/integration/torch/SelectSpec.scala | 2 +- .../bigdl/dllib/integration/torch/SelectTableSpec.scala | 2 +- .../bigdl/dllib/integration/torch/SequentialSpec.scala | 2 +- .../analytics/bigdl/dllib/integration/torch/SigmoidSpec.scala | 2 +- .../bigdl/dllib/integration/torch/SmoothL1CriterionSpec.scala | 2 +- .../dllib/integration/torch/SoftMarginCriterionSpec.scala | 2 +- .../analytics/bigdl/dllib/integration/torch/SoftMaxSpec.scala | 2 +- .../analytics/bigdl/dllib/integration/torch/SoftMinSpec.scala | 2 +- .../bigdl/dllib/integration/torch/SoftPlusSpec.scala | 2 +- .../bigdl/dllib/integration/torch/SoftShrinkSpec.scala | 2 +- .../bigdl/dllib/integration/torch/SoftSignSpec.scala | 2 +- .../dllib/integration/torch/SpatialAveragePoolingSpec.scala | 2 +- .../integration/torch/SpatialBatchNormalizationSpec.scala | 2 +- .../torch/SpatialContrastiveNormalizationSpec.scala | 2 +- .../dllib/integration/torch/SpatialConvolutionMapSpec.scala | 2 +- .../dllib/integration/torch/SpatialConvolutionSpec.scala | 2 +- .../dllib/integration/torch/SpatialCrossMapLRNSpec.scala | 2 +- .../integration/torch/SpatialDilatedConvolutionSpec.scala | 2 +- .../integration/torch/SpatialDivisiveNormalizationSpec.scala | 2 +- .../bigdl/dllib/integration/torch/SpatialDropout1DSpec.scala | 2 +- .../bigdl/dllib/integration/torch/SpatialDropout2DSpec.scala | 2 +- .../bigdl/dllib/integration/torch/SpatialDropout3DSpec.scala | 2 +- .../dllib/integration/torch/SpatialFullConvolutionSpec.scala | 2 +- .../bigdl/dllib/integration/torch/SpatialMaxPoolingSpec.scala | 2 +- .../torch/SpatialSubtractiveNormalizationSpec.scala | 2 +- .../bigdl/dllib/integration/torch/SplitTableSpec.scala | 2 +- .../analytics/bigdl/dllib/integration/torch/SqrtSpec.scala | 2 +- .../analytics/bigdl/dllib/integration/torch/SquareSpec.scala | 2 +- .../analytics/bigdl/dllib/integration/torch/SqueezeSpec.scala | 2 +- .../analytics/bigdl/dllib/integration/torch/SumSpec.scala | 2 +- .../intel/analytics/bigdl/dllib/integration/torch/TH.scala | 2 +- .../bigdl/dllib/integration/torch/TanhShrinkSpec.scala | 2 +- .../analytics/bigdl/dllib/integration/torch/TanhSpec.scala | 2 +- .../dllib/integration/torch/TemporalConvolutionSpec.scala | 2 +- .../dllib/integration/torch/TemporalMaxPoolingSpec.scala | 2 +- .../analytics/bigdl/dllib/integration/torch/TensorSpec.scala | 2 +- .../bigdl/dllib/integration/torch/ThresholdSpec.scala | 2 +- .../analytics/bigdl/dllib/integration/torch/TorchSpec.scala | 2 +- .../bigdl/dllib/integration/torch/TransposeSpec.scala | 2 +- .../bigdl/dllib/integration/torch/UnsqueezeSpec.scala | 2 +- .../analytics/bigdl/dllib/integration/torch/ViewSpec.scala | 2 +- .../integration/torch/VolumetricAveragePoolingSpec.scala | 2 +- .../dllib/integration/torch/VolumetricConvolutionSpec.scala | 2 +- .../integration/torch/VolumetricFullConvolutionSpec.scala | 2 +- .../dllib/integration/torch/VolumetricMaxPoolingSpec.scala | 2 +- .../bigdl/dllib/integration/torch/models/AlexNetSpec.scala | 4 ++-- .../bigdl/dllib/integration/torch/models/InceptionSpec.scala | 4 ++-- .../bigdl/dllib/utils/serializer/SerializerSpec.scala | 3 ++- 139 files changed, 142 insertions(+), 141 deletions(-) diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/HdfsSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/HdfsSpec.scala index a2b88ebd5c4..785954cc83e 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/HdfsSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/HdfsSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration +package com.intel.analytics.bigdl.dllib.integration import java.nio.ByteOrder import java.nio.file.{Files, Paths} diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/Quantization.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/Quantization.scala index 855bd48feb3..561b9b7cfd6 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/Quantization.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/Quantization.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration +package com.intel.analytics.bigdl.dllib.integration import java.nio.file.Paths diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/S3Spec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/S3Spec.scala index add100c796f..e64dd931ce4 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/S3Spec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/S3Spec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration +package com.intel.analytics.bigdl.dllib.integration import java.nio.ByteOrder diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/SparkModeSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/SparkModeSpec.scala index 515b3f784c8..15bb7811b17 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/SparkModeSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/SparkModeSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration +package com.intel.analytics.bigdl.dllib.integration import com.intel.analytics.bigdl.dllib.models.lenet import com.intel.analytics.bigdl.dllib.models.vgg diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/AbsCriterionSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/AbsCriterionSpec.scala index b5823aa0c0d..cf7da4a5d0e 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/AbsCriterionSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/AbsCriterionSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.AbsCriterion import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/AbsSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/AbsSpec.scala index 0ed2ae8b89b..48b8ace4061 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/AbsSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/AbsSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.Abs import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/AdagradSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/AdagradSpec.scala index eb700dc9688..5244dbb0624 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/AdagradSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/AdagradSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.optim.Adagrad import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/AddConstantSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/AddConstantSpec.scala index ab2445796f0..f9c8c3fa7cf 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/AddConstantSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/AddConstantSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.AddConstant import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/AddSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/AddSpec.scala index 11d80b3de9e..9f92de43c93 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/AddSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/AddSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.Add import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/BCECriterionSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/BCECriterionSpec.scala index 3d79d4b0c43..0e98739a866 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/BCECriterionSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/BCECriterionSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.BCECriterion import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/BatchNormalizationSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/BatchNormalizationSpec.scala index 9948086c7f3..cbcbaac0b45 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/BatchNormalizationSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/BatchNormalizationSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import breeze.numerics.abs import com.intel.analytics.bigdl.dllib.nn.{BatchNormalization, GradientChecker} diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/BiRecurrentSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/BiRecurrentSpec.scala index dc21eac314e..e078b2f8e58 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/BiRecurrentSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/BiRecurrentSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import java.io.PrintWriter diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/BilinearSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/BilinearSpec.scala index 97430483965..b6c4039ef55 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/BilinearSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/BilinearSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn._ import com.intel.analytics.bigdl.dllib.optim.{L2Regularizer, SGD} diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/BottleSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/BottleSpec.scala index 924f44d867c..2c510956588 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/BottleSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/BottleSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.{Bottle, Linear} import com.intel.analytics.bigdl._ diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CAddSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CAddSpec.scala index 19c20eb14aa..2bf24d5697d 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CAddSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CAddSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn._ import com.intel.analytics.bigdl.dllib.optim.{L2Regularizer, SGD} diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CAddTableSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CAddTableSpec.scala index a941cde57c8..02d18ee385b 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CAddTableSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CAddTableSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.{CAddTable, ConcatTable, Linear, Sequential} import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CDivTableSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CDivTableSpec.scala index b2b1b93b0dd..c242d576b48 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CDivTableSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CDivTableSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.CDivTable import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CMaxTableSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CMaxTableSpec.scala index eaa1b6971d2..435f737fc94 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CMaxTableSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CMaxTableSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.CMaxTable import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CMinTableSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CMinTableSpec.scala index 677b06e2291..c9421db3fa4 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CMinTableSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CMinTableSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.CMinTable import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CMulSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CMulSpec.scala index fe7912e5b04..c583cfe0a15 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CMulSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CMulSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn._ import com.intel.analytics.bigdl.dllib.optim.{L2Regularizer, SGD} diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CMulTableSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CMulTableSpec.scala index 14e3d5291c7..d22eeb72f58 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CMulTableSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CMulTableSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.utils.RandomGenerator._ diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CSubTableSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CSubTableSpec.scala index 43b5ed7e36b..22f45b8299b 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CSubTableSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CSubTableSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.utils.RandomGenerator._ diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ClampSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ClampSpec.scala index f2ab73552a5..a27ea3992ef 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ClampSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ClampSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.Clamp import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ClassNLLCriterionSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ClassNLLCriterionSpec.scala index 60ac838c462..6f2e96c2d38 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ClassNLLCriterionSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ClassNLLCriterionSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.ClassNLLCriterion import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ClassSimplexCriterionSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ClassSimplexCriterionSpec.scala index 437a013f3fe..3beb989a977 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ClassSimplexCriterionSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ClassSimplexCriterionSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.ClassSimplexCriterion import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ColorJitterSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ColorJitterSpec.scala index b69e8fc91d1..08b683d54f8 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ColorJitterSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ColorJitterSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn._ import com.intel.analytics.bigdl._ diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ConcatSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ConcatSpec.scala index 960cec170cc..68d7ceaad05 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ConcatSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ConcatSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn._ import com.intel.analytics.bigdl._ diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ConcatTableSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ConcatTableSpec.scala index 858e7af3256..bb7e63c2d73 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ConcatTableSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ConcatTableSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.{ConcatTable, Linear} import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ContiguousSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ContiguousSpec.scala index b1e30685231..88376141ffd 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ContiguousSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ContiguousSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.Contiguous import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ConvLSTMPeephole3DSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ConvLSTMPeephole3DSpec.scala index 7f2e1f977c8..ca18987f798 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ConvLSTMPeephole3DSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ConvLSTMPeephole3DSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn._ import com.intel.analytics.bigdl.dllib.optim.{L2Regularizer, SGD} diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CosineDistanceCriterionSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CosineDistanceCriterionSpec.scala index 91c148f44c5..be0d8eb44ef 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CosineDistanceCriterionSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CosineDistanceCriterionSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.CosineDistanceCriterion import com.intel.analytics.bigdl.dllib.tensor.{Storage, Tensor} diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CosineDistanceSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CosineDistanceSpec.scala index 8c2c1d8fef0..1f327b63e63 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CosineDistanceSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CosineDistanceSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.CosineDistance import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CosineEmbeddingCriterionSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CosineEmbeddingCriterionSpec.scala index 57997bdc634..fd23144f217 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CosineEmbeddingCriterionSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CosineEmbeddingCriterionSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.CosineEmbeddingCriterion import com.intel.analytics.bigdl.dllib.tensor.{Storage, Tensor} diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CosineSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CosineSpec.scala index 3e21877478d..f8e00002919 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CosineSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CosineSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.Cosine import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CrossEntropyCriterionSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CrossEntropyCriterionSpec.scala index 5331fd53bb2..90623e94a47 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CrossEntropyCriterionSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CrossEntropyCriterionSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.CrossEntropyCriterion import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/DenseTensorMathSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/DenseTensorMathSpec.scala index b9daf8faf6e..48fde1f12e0 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/DenseTensorMathSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/DenseTensorMathSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/DistKLDivCriterionSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/DistKLDivCriterionSpec.scala index 211a75bbb2b..dbb1a51226e 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/DistKLDivCriterionSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/DistKLDivCriterionSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.DistKLDivCriterion import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/DropoutSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/DropoutSpec.scala index 3a4767babcf..e609d7f3411 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/DropoutSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/DropoutSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.Dropout import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ELUSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ELUSpec.scala index 74a43dfaaa7..e0886fff5ae 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ELUSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ELUSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.ELU import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/EuclideanSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/EuclideanSpec.scala index 3dbe6edaf1a..1da401de1a8 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/EuclideanSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/EuclideanSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.Euclidean import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ExpSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ExpSpec.scala index ca8f7c2551b..a3275639682 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ExpSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ExpSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.{Exp, Power} import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/GRUSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/GRUSpec.scala index 985262107b0..ee45f345958 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/GRUSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/GRUSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import java.io.PrintWriter diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/GaussianCriterionSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/GaussianCriterionSpec.scala index eb57ba71d6a..4efaaf5bc3a 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/GaussianCriterionSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/GaussianCriterionSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.GaussianCriterion import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/GradientReversalSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/GradientReversalSpec.scala index 1054ecd47b2..fbee64b2206 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/GradientReversalSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/GradientReversalSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.GradientReversal import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/HardShrinkSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/HardShrinkSpec.scala index 664c07d6fbe..f5efbdf4ed8 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/HardShrinkSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/HardShrinkSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.HardShrink import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/HardTanhSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/HardTanhSpec.scala index 8406ae55407..6125c0a3bb8 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/HardTanhSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/HardTanhSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.HardTanh import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/HingeEmbeddingCriterionSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/HingeEmbeddingCriterionSpec.scala index 4b12db8d426..a0e28b17633 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/HingeEmbeddingCriterionSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/HingeEmbeddingCriterionSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.HingeEmbeddingCriterion import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/IndexSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/IndexSpec.scala index 77c59439db3..6474704add1 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/IndexSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/IndexSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.Index import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/JoinTableSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/JoinTableSpec.scala index 6cef3f66c05..0dc8f0e48db 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/JoinTableSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/JoinTableSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn._ import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/L1CostSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/L1CostSpec.scala index 91a96fe476d..e357ad82fac 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/L1CostSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/L1CostSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.L1Cost import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/L1HingeEmbeddingCriterionSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/L1HingeEmbeddingCriterionSpec.scala index 3ea0df4e747..2f18920b802 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/L1HingeEmbeddingCriterionSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/L1HingeEmbeddingCriterionSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.L1HingeEmbeddingCriterion import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/L1PenaltySpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/L1PenaltySpec.scala index 30c2f66e201..83a17537d93 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/L1PenaltySpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/L1PenaltySpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.L1Penalty import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/LSTMPeepholeSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/LSTMPeepholeSpec.scala index b627bfc8596..b8684854958 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/LSTMPeepholeSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/LSTMPeepholeSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import java.io.PrintWriter diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/LSTMSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/LSTMSpec.scala index 824e5272189..5d4c6b1c10a 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/LSTMSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/LSTMSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import java.io.PrintWriter diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/LeakyReLUSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/LeakyReLUSpec.scala index ff24dd6b54d..7babfff4def 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/LeakyReLUSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/LeakyReLUSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.{LeakyReLU, RReLU} import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/LinearSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/LinearSpec.scala index 2f8de518eda..3ce6ef6923d 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/LinearSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/LinearSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.{GradientChecker, Linear, MSECriterion} import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/LogSigmoidSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/LogSigmoidSpec.scala index caa4cb5e48a..549fc43be75 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/LogSigmoidSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/LogSigmoidSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.LogSigmoid import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/LogSoftMaxSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/LogSoftMaxSpec.scala index 81a198c785f..4451b282989 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/LogSoftMaxSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/LogSoftMaxSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl._ import com.intel.analytics.bigdl.dllib.nn.{GradientChecker, LogSoftMax} diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/LogSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/LogSpec.scala index f80609c3c04..973b7b160b9 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/LogSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/LogSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.{Log, Power} import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/LookupTableSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/LookupTableSpec.scala index 67ac0afa53d..ed402844a7c 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/LookupTableSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/LookupTableSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn._ import com.intel.analytics.bigdl.dllib.optim.{L2Regularizer, SGD} diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MMSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MMSpec.scala index 3853af2b5ce..81697982955 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MMSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MMSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.MM import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MSECriterionSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MSECriterionSpec.scala index 9552cc46516..85690cab4aa 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MSECriterionSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MSECriterionSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.MSECriterion import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MVSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MVSpec.scala index 17d42c889e1..b5ade16e705 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MVSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MVSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.MV import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MarginCriterionSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MarginCriterionSpec.scala index bb020314e79..87a3cd272b2 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MarginCriterionSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MarginCriterionSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.MarginCriterion import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MarginRankingCriterionSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MarginRankingCriterionSpec.scala index 180707f4565..93c2ddc065f 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MarginRankingCriterionSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MarginRankingCriterionSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.MarginRankingCriterion import com.intel.analytics.bigdl.dllib.tensor.{Storage, Tensor} import com.intel.analytics.bigdl.dllib.utils.Table diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MaskedSelectSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MaskedSelectSpec.scala index 203c3e2456f..08b32381afd 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MaskedSelectSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MaskedSelectSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.MaskedSelect import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MaxSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MaxSpec.scala index 6e099d39a85..bb9df7dd69e 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MaxSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MaxSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.Max import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MeanSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MeanSpec.scala index a4730a828cb..76c73b6acc5 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MeanSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MeanSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.Mean import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MinSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MinSpec.scala index 4fbcc427358..0bf05085ad8 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MinSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MinSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.Min import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MixtureTableSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MixtureTableSpec.scala index 4acafe63160..a653841dafe 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MixtureTableSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MixtureTableSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.MixtureTable import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ModuleSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ModuleSpec.scala index ddc706ae887..93dc44e1ed7 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ModuleSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ModuleSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.{Linear, Sequential} import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MulConstantSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MulConstantSpec.scala index 0bb07bfd4b5..21609bc3a37 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MulConstantSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MulConstantSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.MulConstant import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MulSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MulSpec.scala index 221614e8057..22194725a78 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MulSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MulSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.Mul import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MultiCriterionSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MultiCriterionSpec.scala index 0afb17ac48b..5e0653bc299 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MultiCriterionSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MultiCriterionSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn._ import com.intel.analytics.bigdl._ diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MultiLabelMarginCriterionSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MultiLabelMarginCriterionSpec.scala index 2a61ec7a75f..0532cc5bca5 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MultiLabelMarginCriterionSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MultiLabelMarginCriterionSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.MultiLabelMarginCriterion import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MultiLabelSoftMarginCriterionSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MultiLabelSoftMarginCriterionSpec.scala index 9a34639a34f..d73b545be2e 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MultiLabelSoftMarginCriterionSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MultiLabelSoftMarginCriterionSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.MultiLabelSoftMarginCriterion import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MultiMarginCriterionSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MultiMarginCriterionSpec.scala index 578c1aa8f4e..3277a552e19 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MultiMarginCriterionSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MultiMarginCriterionSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.MultiMarginCriterion import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/NarrowSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/NarrowSpec.scala index a0334f5c97b..bf273d992a5 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/NarrowSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/NarrowSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.Narrow import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/NarrowTableSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/NarrowTableSpec.scala index 9f305dfb2f6..9aea6ddb84c 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/NarrowTableSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/NarrowTableSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.NarrowTable import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/NormalizeSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/NormalizeSpec.scala index 075485f0f38..532098e15a8 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/NormalizeSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/NormalizeSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.Normalize import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/PReLUSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/PReLUSpec.scala index b05d48e7f5e..263df53089f 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/PReLUSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/PReLUSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.PReLU import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/PaddingSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/PaddingSpec.scala index 5e7c4eace18..0ef9bc195cc 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/PaddingSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/PaddingSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.Padding import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/PairwiseDistanceSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/PairwiseDistanceSpec.scala index 7b7a84c315d..8517ca31a16 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/PairwiseDistanceSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/PairwiseDistanceSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.PairwiseDistance import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ParallelCriterionSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ParallelCriterionSpec.scala index e2d7d790b1f..9c453ba847b 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ParallelCriterionSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ParallelCriterionSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.{ClassNLLCriterion, MSECriterion, ParallelCriterion} import com.intel.analytics.bigdl.dllib.tensor.{Storage, Tensor} diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/PowerSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/PowerSpec.scala index ed95f79c21b..4fd42b1a3e2 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/PowerSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/PowerSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.{Power} import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/RReLUSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/RReLUSpec.scala index 8b67f2df662..403e4459097 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/RReLUSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/RReLUSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.{RReLU, ReLU} import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ReLU6Spec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ReLU6Spec.scala index 7bfda36e9fb..10b6a821516 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ReLU6Spec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ReLU6Spec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import java.io.File diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ReLUSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ReLUSpec.scala index aa640deae1e..1ed77093c5b 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ReLUSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ReLUSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.{GradientChecker, ReLU} import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ReplicateSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ReplicateSpec.scala index aacb73736a0..b8e9f90802e 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ReplicateSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ReplicateSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.Replicate import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ReshapeSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ReshapeSpec.scala index 7ab8b669f54..48980b0053b 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ReshapeSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ReshapeSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.Reshape import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SamplerSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SamplerSpec.scala index 8ca4249dd16..ed7ed733323 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SamplerSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SamplerSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.{KLDCriterion, GaussianSampler} import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SelectSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SelectSpec.scala index 219cbf6ad82..e5ec0cd74fc 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SelectSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SelectSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.Select import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SelectTableSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SelectTableSpec.scala index 42d4f8e15bc..7e736383c7b 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SelectTableSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SelectTableSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.SelectTable import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SequentialSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SequentialSpec.scala index 678caa6063d..90585799583 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SequentialSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SequentialSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.{Linear, Sequential} import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SigmoidSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SigmoidSpec.scala index e74c319bb80..ee27b72cb77 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SigmoidSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SigmoidSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.Sigmoid import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SmoothL1CriterionSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SmoothL1CriterionSpec.scala index b2609113d62..7b7fe591b29 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SmoothL1CriterionSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SmoothL1CriterionSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.SmoothL1Criterion import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SoftMarginCriterionSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SoftMarginCriterionSpec.scala index ee5723e72fd..5aa275cdda8 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SoftMarginCriterionSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SoftMarginCriterionSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.SoftMarginCriterion import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SoftMaxSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SoftMaxSpec.scala index f198b17b6c1..8c6a409b8f4 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SoftMaxSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SoftMaxSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.SoftMax import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SoftMinSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SoftMinSpec.scala index 239273f9158..facff942f6c 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SoftMinSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SoftMinSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.SoftMin import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SoftPlusSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SoftPlusSpec.scala index 50a421581be..828d2174f2b 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SoftPlusSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SoftPlusSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.SoftPlus import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SoftShrinkSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SoftShrinkSpec.scala index ddb3ec71a85..27f93c792b2 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SoftShrinkSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SoftShrinkSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.SoftShrink import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SoftSignSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SoftSignSpec.scala index 2fa849924d0..e417eb0d0b1 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SoftSignSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SoftSignSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.SoftSign import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialAveragePoolingSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialAveragePoolingSpec.scala index 433edb5c44b..3ce8d30107d 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialAveragePoolingSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialAveragePoolingSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.{GradientChecker, SpatialAveragePooling} import com.intel.analytics.bigdl.dllib.tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialBatchNormalizationSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialBatchNormalizationSpec.scala index 5a0e6a2bd8f..fb8280e88f2 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialBatchNormalizationSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialBatchNormalizationSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import breeze.numerics.abs import com.intel.analytics.bigdl.dllib.nn.{BatchNormalization, GradientChecker, SpatialBatchNormalization} diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialContrastiveNormalizationSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialContrastiveNormalizationSpec.scala index 0a65d141b93..30ec7886a5d 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialContrastiveNormalizationSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialContrastiveNormalizationSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.{Sequential, SpatialContrastiveNormalization} import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialConvolutionMapSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialConvolutionMapSpec.scala index 9000c205979..d2827decca9 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialConvolutionMapSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialConvolutionMapSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn._ import com.intel.analytics.bigdl.dllib.optim.{L2Regularizer, SGD} diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialConvolutionSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialConvolutionSpec.scala index e57b301d6ff..2393c5fbdc8 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialConvolutionSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialConvolutionSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl._ import com.intel.analytics.bigdl.dllib.nn.{Sequential, SpatialConvolution} diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialCrossMapLRNSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialCrossMapLRNSpec.scala index a4dd82243f3..a9af9f2356c 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialCrossMapLRNSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialCrossMapLRNSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn._ import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialDilatedConvolutionSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialDilatedConvolutionSpec.scala index cc6e4586c36..35049fbe2bc 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialDilatedConvolutionSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialDilatedConvolutionSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn._ import com.intel.analytics.bigdl.dllib.optim.{L2Regularizer, SGD} diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialDivisiveNormalizationSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialDivisiveNormalizationSpec.scala index 5f6fec70f87..e32db890466 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialDivisiveNormalizationSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialDivisiveNormalizationSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.{Sequential, SpatialDivisiveNormalization} import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialDropout1DSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialDropout1DSpec.scala index f55d6bd373f..4e894088508 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialDropout1DSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialDropout1DSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.{SpatialDropout1D, SpatialDropout2D} import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialDropout2DSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialDropout2DSpec.scala index f4fd68379fb..8ced1761c33 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialDropout2DSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialDropout2DSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.SpatialDropout2D import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialDropout3DSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialDropout3DSpec.scala index efc748ccc86..14d2fd0968a 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialDropout3DSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialDropout3DSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.SpatialDropout3D import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialFullConvolutionSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialFullConvolutionSpec.scala index 1e80f474b4c..c71bc2938a2 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialFullConvolutionSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialFullConvolutionSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.{Sequential, SpatialFullConvolution} import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialMaxPoolingSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialMaxPoolingSpec.scala index 75665bad1fa..276c0acee56 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialMaxPoolingSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialMaxPoolingSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.{GradientChecker, SpatialMaxPooling} import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialSubtractiveNormalizationSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialSubtractiveNormalizationSpec.scala index e66fba701e8..4d1156dc7ff 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialSubtractiveNormalizationSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialSubtractiveNormalizationSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.{Sequential, SpatialSubtractiveNormalization} import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SplitTableSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SplitTableSpec.scala index b20c5635789..e29ca971451 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SplitTableSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SplitTableSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.SplitTable import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SqrtSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SqrtSpec.scala index a99edc70b29..b394121b14b 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SqrtSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SqrtSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.Sqrt import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SquareSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SquareSpec.scala index 8295641d260..0a097b298bd 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SquareSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SquareSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.Square import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SqueezeSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SqueezeSpec.scala index 5581689b4b2..7e402edb527 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SqueezeSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SqueezeSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.Squeeze import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SumSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SumSpec.scala index 0dfafc646ac..6200bacc0d0 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SumSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SumSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.Sum import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/TH.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/TH.scala index ddca269f197..15a7e1c506a 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/TH.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/TH.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import java.io._ import java.nio.file.{Files, Path, Paths} diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/TanhShrinkSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/TanhShrinkSpec.scala index e693d1528af..5a298f696eb 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/TanhShrinkSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/TanhShrinkSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.TanhShrink import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/TanhSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/TanhSpec.scala index 76567cd88e7..4e5eee18bc7 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/TanhSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/TanhSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.Tanh import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/TemporalConvolutionSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/TemporalConvolutionSpec.scala index 0f3c8e6d5cc..3c561cc0eb3 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/TemporalConvolutionSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/TemporalConvolutionSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl._ import com.intel.analytics.bigdl.dllib.nn.TemporalConvolution diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/TemporalMaxPoolingSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/TemporalMaxPoolingSpec.scala index f9a5ca22cf1..a4010e33c36 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/TemporalMaxPoolingSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/TemporalMaxPoolingSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl._ import com.intel.analytics.bigdl.dllib.nn.{GradientChecker, TemporalMaxPooling} diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/TensorSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/TensorSpec.scala index 844cab34a17..64f0e889a1b 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/TensorSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/TensorSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ThresholdSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ThresholdSpec.scala index bbb8c8646bb..f0a3a7120ad 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ThresholdSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ThresholdSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.Threshold import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/TorchSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/TorchSpec.scala index ad8d6d1b386..83342ea57d8 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/TorchSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/TorchSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.utils.RandomGenerator._ import org.scalatest._ diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/TransposeSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/TransposeSpec.scala index 02f16dbdf3d..2fa51c776e7 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/TransposeSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/TransposeSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.Transpose import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/UnsqueezeSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/UnsqueezeSpec.scala index 6578afc42d1..b02fd4abb97 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/UnsqueezeSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/UnsqueezeSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.Unsqueeze import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ViewSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ViewSpec.scala index a961350fc97..2c1e64dd45c 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ViewSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ViewSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl._ import com.intel.analytics.bigdl.dllib.nn.{GradientChecker, View} diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/VolumetricAveragePoolingSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/VolumetricAveragePoolingSpec.scala index 179567c3603..eb849aae533 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/VolumetricAveragePoolingSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/VolumetricAveragePoolingSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl._ import com.intel.analytics.bigdl.dllib.nn.{GradientChecker, VolumetricAveragePooling} diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/VolumetricConvolutionSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/VolumetricConvolutionSpec.scala index dd6a06473fd..beaec1809f8 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/VolumetricConvolutionSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/VolumetricConvolutionSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl._ import com.intel.analytics.bigdl.dllib.nn._ diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/VolumetricFullConvolutionSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/VolumetricFullConvolutionSpec.scala index e455375de45..1b00ba767da 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/VolumetricFullConvolutionSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/VolumetricFullConvolutionSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.{BilinearFiller, Sequential, VolumetricFullConvolution, Zeros} import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/VolumetricMaxPoolingSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/VolumetricMaxPoolingSpec.scala index a9267a0874b..ee336088423 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/VolumetricMaxPoolingSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/VolumetricMaxPoolingSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl._ import com.intel.analytics.bigdl.dllib.nn.{GradientChecker, VolumetricMaxPooling} diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/models/AlexNetSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/models/AlexNetSpec.scala index 3512ba2c681..ebf1f6dd17a 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/models/AlexNetSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/models/AlexNetSpec.scala @@ -14,11 +14,11 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch.models +package com.intel.analytics.bigdl.dllib.integration.torch.models import com.intel.analytics.bigdl.Module import com.intel.analytics.bigdl.dllib.example.loadmodel.AlexNet_OWT -import com.intel.analytics.bigdl.integration.torch.{TH, TorchSpec} +import com.intel.analytics.bigdl.dllib.integration.torch.{TH, TorchSpec} import com.intel.analytics.bigdl.dllib.nn.ClassNLLCriterion import com.intel.analytics.bigdl.numeric.NumericFloat import com.intel.analytics.bigdl.dllib.optim.SGD diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/models/InceptionSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/models/InceptionSpec.scala index 2f7ceca9c6c..1c088905b17 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/models/InceptionSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/models/InceptionSpec.scala @@ -14,14 +14,14 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch.models +package com.intel.analytics.bigdl.dllib.integration.torch.models import com.intel.analytics.bigdl.dllib.models.inception._ import com.intel.analytics.bigdl.dllib.nn.{ClassNLLCriterion, Graph, Input} import com.intel.analytics.bigdl.dllib.nn.abstractnn.AbstractModule import com.intel.analytics.bigdl.dllib.optim.SGD import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.integration.torch.{TH, TorchSpec} +import com.intel.analytics.bigdl.dllib.integration.torch.{TH, TorchSpec} import com.intel.analytics.bigdl.dllib.models.Inception import com.intel.analytics.bigdl.utils.RandomGenerator._ import com.intel.analytics.bigdl.dllib.utils.{T, Table} diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/serializer/SerializerSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/serializer/SerializerSpec.scala index 92bd511296c..3832def8c50 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/serializer/SerializerSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/serializer/SerializerSpec.scala @@ -69,7 +69,8 @@ class SerializerSpec extends BigDLSpecHelper { "com.intel.analytics.bigdl.dllib.nn.mkldnn.Output", "com.intel.analytics.bigdl.dllib.nn.mkldnn.InputWrapper", "com.intel.analytics.bigdl.dllib.utils.intermediate.IRGraph", - "com.intel.analytics.bigdl.dllib.nn.mkldnn.RNN" + "com.intel.analytics.bigdl.dllib.nn.mkldnn.RNN", + "com.intel.analytics.bigdl.dllib.nn.ops.TruncatedNormal" ) // Maybe one serial test class contains multiple module test From 0b63c3ed8ce4c08383c4fde84371bf17f771965b Mon Sep 17 00:00:00 2001 From: Le-Zheng <30695225+Le-Zheng@users.noreply.github.com> Date: Fri, 20 Aug 2021 10:26:41 +0800 Subject: [PATCH 812/823] change integration.torch ut path for testing (#4521) * change ut path for testing --- .../intel/analytics/bigdl/dllib/integration/HdfsSpec.scala | 2 +- .../analytics/bigdl/dllib/integration/Quantization.scala | 2 +- .../com/intel/analytics/bigdl/dllib/integration/S3Spec.scala | 2 +- .../analytics/bigdl/dllib/integration/SparkModeSpec.scala | 2 +- .../bigdl/dllib/integration/torch/AbsCriterionSpec.scala | 2 +- .../analytics/bigdl/dllib/integration/torch/AbsSpec.scala | 2 +- .../analytics/bigdl/dllib/integration/torch/AdagradSpec.scala | 2 +- .../bigdl/dllib/integration/torch/AddConstantSpec.scala | 2 +- .../analytics/bigdl/dllib/integration/torch/AddSpec.scala | 2 +- .../bigdl/dllib/integration/torch/BCECriterionSpec.scala | 2 +- .../dllib/integration/torch/BatchNormalizationSpec.scala | 2 +- .../bigdl/dllib/integration/torch/BiRecurrentSpec.scala | 2 +- .../bigdl/dllib/integration/torch/BilinearSpec.scala | 2 +- .../analytics/bigdl/dllib/integration/torch/BottleSpec.scala | 2 +- .../analytics/bigdl/dllib/integration/torch/CAddSpec.scala | 2 +- .../bigdl/dllib/integration/torch/CAddTableSpec.scala | 2 +- .../bigdl/dllib/integration/torch/CDivTableSpec.scala | 2 +- .../bigdl/dllib/integration/torch/CMaxTableSpec.scala | 2 +- .../bigdl/dllib/integration/torch/CMinTableSpec.scala | 2 +- .../analytics/bigdl/dllib/integration/torch/CMulSpec.scala | 2 +- .../bigdl/dllib/integration/torch/CMulTableSpec.scala | 2 +- .../bigdl/dllib/integration/torch/CSubTableSpec.scala | 2 +- .../analytics/bigdl/dllib/integration/torch/ClampSpec.scala | 2 +- .../bigdl/dllib/integration/torch/ClassNLLCriterionSpec.scala | 2 +- .../dllib/integration/torch/ClassSimplexCriterionSpec.scala | 2 +- .../bigdl/dllib/integration/torch/ColorJitterSpec.scala | 2 +- .../analytics/bigdl/dllib/integration/torch/ConcatSpec.scala | 2 +- .../bigdl/dllib/integration/torch/ConcatTableSpec.scala | 2 +- .../bigdl/dllib/integration/torch/ContiguousSpec.scala | 2 +- .../dllib/integration/torch/ConvLSTMPeephole3DSpec.scala | 2 +- .../dllib/integration/torch/CosineDistanceCriterionSpec.scala | 2 +- .../bigdl/dllib/integration/torch/CosineDistanceSpec.scala | 2 +- .../integration/torch/CosineEmbeddingCriterionSpec.scala | 2 +- .../analytics/bigdl/dllib/integration/torch/CosineSpec.scala | 2 +- .../dllib/integration/torch/CrossEntropyCriterionSpec.scala | 2 +- .../bigdl/dllib/integration/torch/DenseTensorMathSpec.scala | 2 +- .../dllib/integration/torch/DistKLDivCriterionSpec.scala | 2 +- .../analytics/bigdl/dllib/integration/torch/DropoutSpec.scala | 2 +- .../analytics/bigdl/dllib/integration/torch/ELUSpec.scala | 2 +- .../bigdl/dllib/integration/torch/EuclideanSpec.scala | 2 +- .../analytics/bigdl/dllib/integration/torch/ExpSpec.scala | 2 +- .../analytics/bigdl/dllib/integration/torch/GRUSpec.scala | 2 +- .../bigdl/dllib/integration/torch/GaussianCriterionSpec.scala | 2 +- .../bigdl/dllib/integration/torch/GradientReversalSpec.scala | 2 +- .../bigdl/dllib/integration/torch/HardShrinkSpec.scala | 2 +- .../bigdl/dllib/integration/torch/HardTanhSpec.scala | 2 +- .../dllib/integration/torch/HingeEmbeddingCriterionSpec.scala | 2 +- .../analytics/bigdl/dllib/integration/torch/IndexSpec.scala | 2 +- .../bigdl/dllib/integration/torch/JoinTableSpec.scala | 2 +- .../analytics/bigdl/dllib/integration/torch/L1CostSpec.scala | 2 +- .../integration/torch/L1HingeEmbeddingCriterionSpec.scala | 2 +- .../bigdl/dllib/integration/torch/L1PenaltySpec.scala | 2 +- .../bigdl/dllib/integration/torch/LSTMPeepholeSpec.scala | 2 +- .../analytics/bigdl/dllib/integration/torch/LSTMSpec.scala | 2 +- .../bigdl/dllib/integration/torch/LeakyReLUSpec.scala | 2 +- .../analytics/bigdl/dllib/integration/torch/LinearSpec.scala | 2 +- .../bigdl/dllib/integration/torch/LogSigmoidSpec.scala | 2 +- .../bigdl/dllib/integration/torch/LogSoftMaxSpec.scala | 2 +- .../analytics/bigdl/dllib/integration/torch/LogSpec.scala | 2 +- .../bigdl/dllib/integration/torch/LookupTableSpec.scala | 2 +- .../analytics/bigdl/dllib/integration/torch/MMSpec.scala | 2 +- .../bigdl/dllib/integration/torch/MSECriterionSpec.scala | 2 +- .../analytics/bigdl/dllib/integration/torch/MVSpec.scala | 2 +- .../bigdl/dllib/integration/torch/MarginCriterionSpec.scala | 2 +- .../dllib/integration/torch/MarginRankingCriterionSpec.scala | 2 +- .../bigdl/dllib/integration/torch/MaskedSelectSpec.scala | 2 +- .../analytics/bigdl/dllib/integration/torch/MaxSpec.scala | 2 +- .../analytics/bigdl/dllib/integration/torch/MeanSpec.scala | 2 +- .../analytics/bigdl/dllib/integration/torch/MinSpec.scala | 2 +- .../bigdl/dllib/integration/torch/MixtureTableSpec.scala | 2 +- .../analytics/bigdl/dllib/integration/torch/ModuleSpec.scala | 2 +- .../bigdl/dllib/integration/torch/MulConstantSpec.scala | 2 +- .../analytics/bigdl/dllib/integration/torch/MulSpec.scala | 2 +- .../bigdl/dllib/integration/torch/MultiCriterionSpec.scala | 2 +- .../integration/torch/MultiLabelMarginCriterionSpec.scala | 2 +- .../integration/torch/MultiLabelSoftMarginCriterionSpec.scala | 2 +- .../dllib/integration/torch/MultiMarginCriterionSpec.scala | 2 +- .../analytics/bigdl/dllib/integration/torch/NarrowSpec.scala | 2 +- .../bigdl/dllib/integration/torch/NarrowTableSpec.scala | 2 +- .../bigdl/dllib/integration/torch/NormalizeSpec.scala | 2 +- .../analytics/bigdl/dllib/integration/torch/PReLUSpec.scala | 2 +- .../analytics/bigdl/dllib/integration/torch/PaddingSpec.scala | 2 +- .../bigdl/dllib/integration/torch/PairwiseDistanceSpec.scala | 2 +- .../bigdl/dllib/integration/torch/ParallelCriterionSpec.scala | 2 +- .../analytics/bigdl/dllib/integration/torch/PowerSpec.scala | 2 +- .../analytics/bigdl/dllib/integration/torch/RReLUSpec.scala | 2 +- .../analytics/bigdl/dllib/integration/torch/ReLU6Spec.scala | 2 +- .../analytics/bigdl/dllib/integration/torch/ReLUSpec.scala | 2 +- .../bigdl/dllib/integration/torch/ReplicateSpec.scala | 2 +- .../analytics/bigdl/dllib/integration/torch/ReshapeSpec.scala | 2 +- .../analytics/bigdl/dllib/integration/torch/SamplerSpec.scala | 2 +- .../analytics/bigdl/dllib/integration/torch/SelectSpec.scala | 2 +- .../bigdl/dllib/integration/torch/SelectTableSpec.scala | 2 +- .../bigdl/dllib/integration/torch/SequentialSpec.scala | 2 +- .../analytics/bigdl/dllib/integration/torch/SigmoidSpec.scala | 2 +- .../bigdl/dllib/integration/torch/SmoothL1CriterionSpec.scala | 2 +- .../dllib/integration/torch/SoftMarginCriterionSpec.scala | 2 +- .../analytics/bigdl/dllib/integration/torch/SoftMaxSpec.scala | 2 +- .../analytics/bigdl/dllib/integration/torch/SoftMinSpec.scala | 2 +- .../bigdl/dllib/integration/torch/SoftPlusSpec.scala | 2 +- .../bigdl/dllib/integration/torch/SoftShrinkSpec.scala | 2 +- .../bigdl/dllib/integration/torch/SoftSignSpec.scala | 2 +- .../dllib/integration/torch/SpatialAveragePoolingSpec.scala | 2 +- .../integration/torch/SpatialBatchNormalizationSpec.scala | 2 +- .../torch/SpatialContrastiveNormalizationSpec.scala | 2 +- .../dllib/integration/torch/SpatialConvolutionMapSpec.scala | 2 +- .../dllib/integration/torch/SpatialConvolutionSpec.scala | 2 +- .../dllib/integration/torch/SpatialCrossMapLRNSpec.scala | 2 +- .../integration/torch/SpatialDilatedConvolutionSpec.scala | 2 +- .../integration/torch/SpatialDivisiveNormalizationSpec.scala | 2 +- .../bigdl/dllib/integration/torch/SpatialDropout1DSpec.scala | 2 +- .../bigdl/dllib/integration/torch/SpatialDropout2DSpec.scala | 2 +- .../bigdl/dllib/integration/torch/SpatialDropout3DSpec.scala | 2 +- .../dllib/integration/torch/SpatialFullConvolutionSpec.scala | 2 +- .../bigdl/dllib/integration/torch/SpatialMaxPoolingSpec.scala | 2 +- .../torch/SpatialSubtractiveNormalizationSpec.scala | 2 +- .../bigdl/dllib/integration/torch/SplitTableSpec.scala | 2 +- .../analytics/bigdl/dllib/integration/torch/SqrtSpec.scala | 2 +- .../analytics/bigdl/dllib/integration/torch/SquareSpec.scala | 2 +- .../analytics/bigdl/dllib/integration/torch/SqueezeSpec.scala | 2 +- .../analytics/bigdl/dllib/integration/torch/SumSpec.scala | 2 +- .../intel/analytics/bigdl/dllib/integration/torch/TH.scala | 2 +- .../bigdl/dllib/integration/torch/TanhShrinkSpec.scala | 2 +- .../analytics/bigdl/dllib/integration/torch/TanhSpec.scala | 2 +- .../dllib/integration/torch/TemporalConvolutionSpec.scala | 2 +- .../dllib/integration/torch/TemporalMaxPoolingSpec.scala | 2 +- .../analytics/bigdl/dllib/integration/torch/TensorSpec.scala | 2 +- .../bigdl/dllib/integration/torch/ThresholdSpec.scala | 2 +- .../analytics/bigdl/dllib/integration/torch/TorchSpec.scala | 2 +- .../bigdl/dllib/integration/torch/TransposeSpec.scala | 2 +- .../bigdl/dllib/integration/torch/UnsqueezeSpec.scala | 2 +- .../analytics/bigdl/dllib/integration/torch/ViewSpec.scala | 2 +- .../integration/torch/VolumetricAveragePoolingSpec.scala | 2 +- .../dllib/integration/torch/VolumetricConvolutionSpec.scala | 2 +- .../integration/torch/VolumetricFullConvolutionSpec.scala | 2 +- .../dllib/integration/torch/VolumetricMaxPoolingSpec.scala | 2 +- .../bigdl/dllib/integration/torch/models/AlexNetSpec.scala | 4 ++-- .../bigdl/dllib/integration/torch/models/InceptionSpec.scala | 4 ++-- .../bigdl/dllib/utils/serializer/SerializerSpec.scala | 3 ++- 139 files changed, 142 insertions(+), 141 deletions(-) diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/HdfsSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/HdfsSpec.scala index a2b88ebd5c4..785954cc83e 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/HdfsSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/HdfsSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration +package com.intel.analytics.bigdl.dllib.integration import java.nio.ByteOrder import java.nio.file.{Files, Paths} diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/Quantization.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/Quantization.scala index 855bd48feb3..561b9b7cfd6 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/Quantization.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/Quantization.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration +package com.intel.analytics.bigdl.dllib.integration import java.nio.file.Paths diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/S3Spec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/S3Spec.scala index add100c796f..e64dd931ce4 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/S3Spec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/S3Spec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration +package com.intel.analytics.bigdl.dllib.integration import java.nio.ByteOrder diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/SparkModeSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/SparkModeSpec.scala index 515b3f784c8..15bb7811b17 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/SparkModeSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/SparkModeSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration +package com.intel.analytics.bigdl.dllib.integration import com.intel.analytics.bigdl.dllib.models.lenet import com.intel.analytics.bigdl.dllib.models.vgg diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/AbsCriterionSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/AbsCriterionSpec.scala index b5823aa0c0d..cf7da4a5d0e 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/AbsCriterionSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/AbsCriterionSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.AbsCriterion import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/AbsSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/AbsSpec.scala index 0ed2ae8b89b..48b8ace4061 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/AbsSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/AbsSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.Abs import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/AdagradSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/AdagradSpec.scala index eb700dc9688..5244dbb0624 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/AdagradSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/AdagradSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.optim.Adagrad import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/AddConstantSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/AddConstantSpec.scala index ab2445796f0..f9c8c3fa7cf 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/AddConstantSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/AddConstantSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.AddConstant import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/AddSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/AddSpec.scala index 11d80b3de9e..9f92de43c93 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/AddSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/AddSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.Add import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/BCECriterionSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/BCECriterionSpec.scala index 3d79d4b0c43..0e98739a866 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/BCECriterionSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/BCECriterionSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.BCECriterion import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/BatchNormalizationSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/BatchNormalizationSpec.scala index 9948086c7f3..cbcbaac0b45 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/BatchNormalizationSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/BatchNormalizationSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import breeze.numerics.abs import com.intel.analytics.bigdl.dllib.nn.{BatchNormalization, GradientChecker} diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/BiRecurrentSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/BiRecurrentSpec.scala index dc21eac314e..e078b2f8e58 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/BiRecurrentSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/BiRecurrentSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import java.io.PrintWriter diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/BilinearSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/BilinearSpec.scala index 97430483965..b6c4039ef55 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/BilinearSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/BilinearSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn._ import com.intel.analytics.bigdl.dllib.optim.{L2Regularizer, SGD} diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/BottleSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/BottleSpec.scala index 924f44d867c..2c510956588 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/BottleSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/BottleSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.{Bottle, Linear} import com.intel.analytics.bigdl._ diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CAddSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CAddSpec.scala index 19c20eb14aa..2bf24d5697d 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CAddSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CAddSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn._ import com.intel.analytics.bigdl.dllib.optim.{L2Regularizer, SGD} diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CAddTableSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CAddTableSpec.scala index a941cde57c8..02d18ee385b 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CAddTableSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CAddTableSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.{CAddTable, ConcatTable, Linear, Sequential} import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CDivTableSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CDivTableSpec.scala index b2b1b93b0dd..c242d576b48 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CDivTableSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CDivTableSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.CDivTable import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CMaxTableSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CMaxTableSpec.scala index eaa1b6971d2..435f737fc94 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CMaxTableSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CMaxTableSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.CMaxTable import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CMinTableSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CMinTableSpec.scala index 677b06e2291..c9421db3fa4 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CMinTableSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CMinTableSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.CMinTable import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CMulSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CMulSpec.scala index fe7912e5b04..c583cfe0a15 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CMulSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CMulSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn._ import com.intel.analytics.bigdl.dllib.optim.{L2Regularizer, SGD} diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CMulTableSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CMulTableSpec.scala index 14e3d5291c7..d22eeb72f58 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CMulTableSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CMulTableSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.utils.RandomGenerator._ diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CSubTableSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CSubTableSpec.scala index 43b5ed7e36b..22f45b8299b 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CSubTableSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CSubTableSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.utils.RandomGenerator._ diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ClampSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ClampSpec.scala index f2ab73552a5..a27ea3992ef 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ClampSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ClampSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.Clamp import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ClassNLLCriterionSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ClassNLLCriterionSpec.scala index 60ac838c462..6f2e96c2d38 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ClassNLLCriterionSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ClassNLLCriterionSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.ClassNLLCriterion import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ClassSimplexCriterionSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ClassSimplexCriterionSpec.scala index 437a013f3fe..3beb989a977 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ClassSimplexCriterionSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ClassSimplexCriterionSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.ClassSimplexCriterion import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ColorJitterSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ColorJitterSpec.scala index b69e8fc91d1..08b683d54f8 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ColorJitterSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ColorJitterSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn._ import com.intel.analytics.bigdl._ diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ConcatSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ConcatSpec.scala index 960cec170cc..68d7ceaad05 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ConcatSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ConcatSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn._ import com.intel.analytics.bigdl._ diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ConcatTableSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ConcatTableSpec.scala index 858e7af3256..bb7e63c2d73 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ConcatTableSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ConcatTableSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.{ConcatTable, Linear} import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ContiguousSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ContiguousSpec.scala index b1e30685231..88376141ffd 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ContiguousSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ContiguousSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.Contiguous import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ConvLSTMPeephole3DSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ConvLSTMPeephole3DSpec.scala index 7f2e1f977c8..ca18987f798 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ConvLSTMPeephole3DSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ConvLSTMPeephole3DSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn._ import com.intel.analytics.bigdl.dllib.optim.{L2Regularizer, SGD} diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CosineDistanceCriterionSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CosineDistanceCriterionSpec.scala index 91c148f44c5..be0d8eb44ef 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CosineDistanceCriterionSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CosineDistanceCriterionSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.CosineDistanceCriterion import com.intel.analytics.bigdl.dllib.tensor.{Storage, Tensor} diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CosineDistanceSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CosineDistanceSpec.scala index 8c2c1d8fef0..1f327b63e63 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CosineDistanceSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CosineDistanceSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.CosineDistance import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CosineEmbeddingCriterionSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CosineEmbeddingCriterionSpec.scala index 57997bdc634..fd23144f217 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CosineEmbeddingCriterionSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CosineEmbeddingCriterionSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.CosineEmbeddingCriterion import com.intel.analytics.bigdl.dllib.tensor.{Storage, Tensor} diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CosineSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CosineSpec.scala index 3e21877478d..f8e00002919 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CosineSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CosineSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.Cosine import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CrossEntropyCriterionSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CrossEntropyCriterionSpec.scala index 5331fd53bb2..90623e94a47 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CrossEntropyCriterionSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CrossEntropyCriterionSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.CrossEntropyCriterion import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/DenseTensorMathSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/DenseTensorMathSpec.scala index b9daf8faf6e..48fde1f12e0 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/DenseTensorMathSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/DenseTensorMathSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/DistKLDivCriterionSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/DistKLDivCriterionSpec.scala index 211a75bbb2b..dbb1a51226e 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/DistKLDivCriterionSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/DistKLDivCriterionSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.DistKLDivCriterion import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/DropoutSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/DropoutSpec.scala index 3a4767babcf..e609d7f3411 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/DropoutSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/DropoutSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.Dropout import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ELUSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ELUSpec.scala index 74a43dfaaa7..e0886fff5ae 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ELUSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ELUSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.ELU import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/EuclideanSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/EuclideanSpec.scala index 3dbe6edaf1a..1da401de1a8 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/EuclideanSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/EuclideanSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.Euclidean import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ExpSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ExpSpec.scala index ca8f7c2551b..a3275639682 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ExpSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ExpSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.{Exp, Power} import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/GRUSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/GRUSpec.scala index 985262107b0..ee45f345958 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/GRUSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/GRUSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import java.io.PrintWriter diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/GaussianCriterionSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/GaussianCriterionSpec.scala index eb57ba71d6a..4efaaf5bc3a 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/GaussianCriterionSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/GaussianCriterionSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.GaussianCriterion import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/GradientReversalSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/GradientReversalSpec.scala index 1054ecd47b2..fbee64b2206 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/GradientReversalSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/GradientReversalSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.GradientReversal import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/HardShrinkSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/HardShrinkSpec.scala index 664c07d6fbe..f5efbdf4ed8 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/HardShrinkSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/HardShrinkSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.HardShrink import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/HardTanhSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/HardTanhSpec.scala index 8406ae55407..6125c0a3bb8 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/HardTanhSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/HardTanhSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.HardTanh import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/HingeEmbeddingCriterionSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/HingeEmbeddingCriterionSpec.scala index 4b12db8d426..a0e28b17633 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/HingeEmbeddingCriterionSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/HingeEmbeddingCriterionSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.HingeEmbeddingCriterion import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/IndexSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/IndexSpec.scala index 77c59439db3..6474704add1 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/IndexSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/IndexSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.Index import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/JoinTableSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/JoinTableSpec.scala index 6cef3f66c05..0dc8f0e48db 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/JoinTableSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/JoinTableSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn._ import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/L1CostSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/L1CostSpec.scala index 91a96fe476d..e357ad82fac 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/L1CostSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/L1CostSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.L1Cost import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/L1HingeEmbeddingCriterionSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/L1HingeEmbeddingCriterionSpec.scala index 3ea0df4e747..2f18920b802 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/L1HingeEmbeddingCriterionSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/L1HingeEmbeddingCriterionSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.L1HingeEmbeddingCriterion import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/L1PenaltySpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/L1PenaltySpec.scala index 30c2f66e201..83a17537d93 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/L1PenaltySpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/L1PenaltySpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.L1Penalty import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/LSTMPeepholeSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/LSTMPeepholeSpec.scala index b627bfc8596..b8684854958 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/LSTMPeepholeSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/LSTMPeepholeSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import java.io.PrintWriter diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/LSTMSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/LSTMSpec.scala index 824e5272189..5d4c6b1c10a 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/LSTMSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/LSTMSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import java.io.PrintWriter diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/LeakyReLUSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/LeakyReLUSpec.scala index ff24dd6b54d..7babfff4def 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/LeakyReLUSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/LeakyReLUSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.{LeakyReLU, RReLU} import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/LinearSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/LinearSpec.scala index 2f8de518eda..3ce6ef6923d 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/LinearSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/LinearSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.{GradientChecker, Linear, MSECriterion} import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/LogSigmoidSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/LogSigmoidSpec.scala index caa4cb5e48a..549fc43be75 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/LogSigmoidSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/LogSigmoidSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.LogSigmoid import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/LogSoftMaxSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/LogSoftMaxSpec.scala index 81a198c785f..4451b282989 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/LogSoftMaxSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/LogSoftMaxSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl._ import com.intel.analytics.bigdl.dllib.nn.{GradientChecker, LogSoftMax} diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/LogSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/LogSpec.scala index f80609c3c04..973b7b160b9 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/LogSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/LogSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.{Log, Power} import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/LookupTableSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/LookupTableSpec.scala index 67ac0afa53d..ed402844a7c 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/LookupTableSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/LookupTableSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn._ import com.intel.analytics.bigdl.dllib.optim.{L2Regularizer, SGD} diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MMSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MMSpec.scala index 3853af2b5ce..81697982955 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MMSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MMSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.MM import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MSECriterionSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MSECriterionSpec.scala index 9552cc46516..85690cab4aa 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MSECriterionSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MSECriterionSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.MSECriterion import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MVSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MVSpec.scala index 17d42c889e1..b5ade16e705 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MVSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MVSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.MV import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MarginCriterionSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MarginCriterionSpec.scala index bb020314e79..87a3cd272b2 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MarginCriterionSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MarginCriterionSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.MarginCriterion import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MarginRankingCriterionSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MarginRankingCriterionSpec.scala index 180707f4565..93c2ddc065f 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MarginRankingCriterionSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MarginRankingCriterionSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.MarginRankingCriterion import com.intel.analytics.bigdl.dllib.tensor.{Storage, Tensor} import com.intel.analytics.bigdl.dllib.utils.Table diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MaskedSelectSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MaskedSelectSpec.scala index 203c3e2456f..08b32381afd 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MaskedSelectSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MaskedSelectSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.MaskedSelect import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MaxSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MaxSpec.scala index 6e099d39a85..bb9df7dd69e 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MaxSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MaxSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.Max import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MeanSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MeanSpec.scala index a4730a828cb..76c73b6acc5 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MeanSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MeanSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.Mean import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MinSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MinSpec.scala index 4fbcc427358..0bf05085ad8 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MinSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MinSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.Min import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MixtureTableSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MixtureTableSpec.scala index 4acafe63160..a653841dafe 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MixtureTableSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MixtureTableSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.MixtureTable import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ModuleSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ModuleSpec.scala index ddc706ae887..93dc44e1ed7 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ModuleSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ModuleSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.{Linear, Sequential} import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MulConstantSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MulConstantSpec.scala index 0bb07bfd4b5..21609bc3a37 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MulConstantSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MulConstantSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.MulConstant import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MulSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MulSpec.scala index 221614e8057..22194725a78 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MulSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MulSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.Mul import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MultiCriterionSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MultiCriterionSpec.scala index 0afb17ac48b..5e0653bc299 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MultiCriterionSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MultiCriterionSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn._ import com.intel.analytics.bigdl._ diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MultiLabelMarginCriterionSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MultiLabelMarginCriterionSpec.scala index 2a61ec7a75f..0532cc5bca5 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MultiLabelMarginCriterionSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MultiLabelMarginCriterionSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.MultiLabelMarginCriterion import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MultiLabelSoftMarginCriterionSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MultiLabelSoftMarginCriterionSpec.scala index 9a34639a34f..d73b545be2e 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MultiLabelSoftMarginCriterionSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MultiLabelSoftMarginCriterionSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.MultiLabelSoftMarginCriterion import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MultiMarginCriterionSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MultiMarginCriterionSpec.scala index 578c1aa8f4e..3277a552e19 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MultiMarginCriterionSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MultiMarginCriterionSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.MultiMarginCriterion import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/NarrowSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/NarrowSpec.scala index a0334f5c97b..bf273d992a5 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/NarrowSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/NarrowSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.Narrow import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/NarrowTableSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/NarrowTableSpec.scala index 9f305dfb2f6..9aea6ddb84c 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/NarrowTableSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/NarrowTableSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.NarrowTable import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/NormalizeSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/NormalizeSpec.scala index 075485f0f38..532098e15a8 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/NormalizeSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/NormalizeSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.Normalize import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/PReLUSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/PReLUSpec.scala index b05d48e7f5e..263df53089f 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/PReLUSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/PReLUSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.PReLU import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/PaddingSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/PaddingSpec.scala index 5e7c4eace18..0ef9bc195cc 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/PaddingSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/PaddingSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.Padding import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/PairwiseDistanceSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/PairwiseDistanceSpec.scala index 7b7a84c315d..8517ca31a16 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/PairwiseDistanceSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/PairwiseDistanceSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.PairwiseDistance import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ParallelCriterionSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ParallelCriterionSpec.scala index e2d7d790b1f..9c453ba847b 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ParallelCriterionSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ParallelCriterionSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.{ClassNLLCriterion, MSECriterion, ParallelCriterion} import com.intel.analytics.bigdl.dllib.tensor.{Storage, Tensor} diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/PowerSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/PowerSpec.scala index ed95f79c21b..4fd42b1a3e2 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/PowerSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/PowerSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.{Power} import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/RReLUSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/RReLUSpec.scala index 8b67f2df662..403e4459097 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/RReLUSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/RReLUSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.{RReLU, ReLU} import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ReLU6Spec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ReLU6Spec.scala index 7bfda36e9fb..10b6a821516 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ReLU6Spec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ReLU6Spec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import java.io.File diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ReLUSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ReLUSpec.scala index aa640deae1e..1ed77093c5b 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ReLUSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ReLUSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.{GradientChecker, ReLU} import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ReplicateSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ReplicateSpec.scala index aacb73736a0..b8e9f90802e 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ReplicateSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ReplicateSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.Replicate import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ReshapeSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ReshapeSpec.scala index 7ab8b669f54..48980b0053b 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ReshapeSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ReshapeSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.Reshape import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SamplerSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SamplerSpec.scala index 8ca4249dd16..ed7ed733323 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SamplerSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SamplerSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.{KLDCriterion, GaussianSampler} import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SelectSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SelectSpec.scala index 219cbf6ad82..e5ec0cd74fc 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SelectSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SelectSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.Select import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SelectTableSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SelectTableSpec.scala index 42d4f8e15bc..7e736383c7b 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SelectTableSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SelectTableSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.SelectTable import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SequentialSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SequentialSpec.scala index 678caa6063d..90585799583 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SequentialSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SequentialSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.{Linear, Sequential} import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SigmoidSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SigmoidSpec.scala index e74c319bb80..ee27b72cb77 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SigmoidSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SigmoidSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.Sigmoid import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SmoothL1CriterionSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SmoothL1CriterionSpec.scala index b2609113d62..7b7fe591b29 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SmoothL1CriterionSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SmoothL1CriterionSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.SmoothL1Criterion import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SoftMarginCriterionSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SoftMarginCriterionSpec.scala index ee5723e72fd..5aa275cdda8 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SoftMarginCriterionSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SoftMarginCriterionSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.SoftMarginCriterion import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SoftMaxSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SoftMaxSpec.scala index f198b17b6c1..8c6a409b8f4 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SoftMaxSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SoftMaxSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.SoftMax import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SoftMinSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SoftMinSpec.scala index 239273f9158..facff942f6c 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SoftMinSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SoftMinSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.SoftMin import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SoftPlusSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SoftPlusSpec.scala index 50a421581be..828d2174f2b 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SoftPlusSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SoftPlusSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.SoftPlus import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SoftShrinkSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SoftShrinkSpec.scala index ddb3ec71a85..27f93c792b2 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SoftShrinkSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SoftShrinkSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.SoftShrink import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SoftSignSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SoftSignSpec.scala index 2fa849924d0..e417eb0d0b1 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SoftSignSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SoftSignSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.SoftSign import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialAveragePoolingSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialAveragePoolingSpec.scala index 433edb5c44b..3ce8d30107d 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialAveragePoolingSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialAveragePoolingSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.{GradientChecker, SpatialAveragePooling} import com.intel.analytics.bigdl.dllib.tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialBatchNormalizationSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialBatchNormalizationSpec.scala index 5a0e6a2bd8f..fb8280e88f2 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialBatchNormalizationSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialBatchNormalizationSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import breeze.numerics.abs import com.intel.analytics.bigdl.dllib.nn.{BatchNormalization, GradientChecker, SpatialBatchNormalization} diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialContrastiveNormalizationSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialContrastiveNormalizationSpec.scala index 0a65d141b93..30ec7886a5d 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialContrastiveNormalizationSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialContrastiveNormalizationSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.{Sequential, SpatialContrastiveNormalization} import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialConvolutionMapSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialConvolutionMapSpec.scala index 9000c205979..d2827decca9 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialConvolutionMapSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialConvolutionMapSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn._ import com.intel.analytics.bigdl.dllib.optim.{L2Regularizer, SGD} diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialConvolutionSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialConvolutionSpec.scala index e57b301d6ff..2393c5fbdc8 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialConvolutionSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialConvolutionSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl._ import com.intel.analytics.bigdl.dllib.nn.{Sequential, SpatialConvolution} diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialCrossMapLRNSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialCrossMapLRNSpec.scala index a4dd82243f3..a9af9f2356c 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialCrossMapLRNSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialCrossMapLRNSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn._ import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialDilatedConvolutionSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialDilatedConvolutionSpec.scala index cc6e4586c36..35049fbe2bc 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialDilatedConvolutionSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialDilatedConvolutionSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn._ import com.intel.analytics.bigdl.dllib.optim.{L2Regularizer, SGD} diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialDivisiveNormalizationSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialDivisiveNormalizationSpec.scala index 5f6fec70f87..e32db890466 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialDivisiveNormalizationSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialDivisiveNormalizationSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.{Sequential, SpatialDivisiveNormalization} import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialDropout1DSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialDropout1DSpec.scala index f55d6bd373f..4e894088508 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialDropout1DSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialDropout1DSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.{SpatialDropout1D, SpatialDropout2D} import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialDropout2DSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialDropout2DSpec.scala index f4fd68379fb..8ced1761c33 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialDropout2DSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialDropout2DSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.SpatialDropout2D import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialDropout3DSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialDropout3DSpec.scala index efc748ccc86..14d2fd0968a 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialDropout3DSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialDropout3DSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.SpatialDropout3D import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialFullConvolutionSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialFullConvolutionSpec.scala index 1e80f474b4c..c71bc2938a2 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialFullConvolutionSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialFullConvolutionSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.{Sequential, SpatialFullConvolution} import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialMaxPoolingSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialMaxPoolingSpec.scala index 75665bad1fa..276c0acee56 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialMaxPoolingSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialMaxPoolingSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.{GradientChecker, SpatialMaxPooling} import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialSubtractiveNormalizationSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialSubtractiveNormalizationSpec.scala index e66fba701e8..4d1156dc7ff 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialSubtractiveNormalizationSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialSubtractiveNormalizationSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.{Sequential, SpatialSubtractiveNormalization} import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SplitTableSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SplitTableSpec.scala index b20c5635789..e29ca971451 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SplitTableSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SplitTableSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.SplitTable import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SqrtSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SqrtSpec.scala index a99edc70b29..b394121b14b 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SqrtSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SqrtSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.Sqrt import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SquareSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SquareSpec.scala index 8295641d260..0a097b298bd 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SquareSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SquareSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.Square import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SqueezeSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SqueezeSpec.scala index 5581689b4b2..7e402edb527 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SqueezeSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SqueezeSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.Squeeze import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SumSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SumSpec.scala index 0dfafc646ac..6200bacc0d0 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SumSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SumSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.Sum import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/TH.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/TH.scala index ddca269f197..15a7e1c506a 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/TH.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/TH.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import java.io._ import java.nio.file.{Files, Path, Paths} diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/TanhShrinkSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/TanhShrinkSpec.scala index e693d1528af..5a298f696eb 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/TanhShrinkSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/TanhShrinkSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.TanhShrink import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/TanhSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/TanhSpec.scala index 76567cd88e7..4e5eee18bc7 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/TanhSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/TanhSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.Tanh import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/TemporalConvolutionSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/TemporalConvolutionSpec.scala index 0f3c8e6d5cc..3c561cc0eb3 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/TemporalConvolutionSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/TemporalConvolutionSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl._ import com.intel.analytics.bigdl.dllib.nn.TemporalConvolution diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/TemporalMaxPoolingSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/TemporalMaxPoolingSpec.scala index f9a5ca22cf1..a4010e33c36 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/TemporalMaxPoolingSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/TemporalMaxPoolingSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl._ import com.intel.analytics.bigdl.dllib.nn.{GradientChecker, TemporalMaxPooling} diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/TensorSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/TensorSpec.scala index 844cab34a17..64f0e889a1b 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/TensorSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/TensorSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ThresholdSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ThresholdSpec.scala index bbb8c8646bb..f0a3a7120ad 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ThresholdSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ThresholdSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.Threshold import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/TorchSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/TorchSpec.scala index ad8d6d1b386..83342ea57d8 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/TorchSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/TorchSpec.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.utils.RandomGenerator._ import org.scalatest._ diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/TransposeSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/TransposeSpec.scala index 02f16dbdf3d..2fa51c776e7 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/TransposeSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/TransposeSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.Transpose import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/UnsqueezeSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/UnsqueezeSpec.scala index 6578afc42d1..b02fd4abb97 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/UnsqueezeSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/UnsqueezeSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.Unsqueeze import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ViewSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ViewSpec.scala index a961350fc97..2c1e64dd45c 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ViewSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ViewSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl._ import com.intel.analytics.bigdl.dllib.nn.{GradientChecker, View} diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/VolumetricAveragePoolingSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/VolumetricAveragePoolingSpec.scala index 179567c3603..eb849aae533 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/VolumetricAveragePoolingSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/VolumetricAveragePoolingSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl._ import com.intel.analytics.bigdl.dllib.nn.{GradientChecker, VolumetricAveragePooling} diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/VolumetricConvolutionSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/VolumetricConvolutionSpec.scala index dd6a06473fd..beaec1809f8 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/VolumetricConvolutionSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/VolumetricConvolutionSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl._ import com.intel.analytics.bigdl.dllib.nn._ diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/VolumetricFullConvolutionSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/VolumetricFullConvolutionSpec.scala index e455375de45..1b00ba767da 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/VolumetricFullConvolutionSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/VolumetricFullConvolutionSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.{BilinearFiller, Sequential, VolumetricFullConvolution, Zeros} import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/VolumetricMaxPoolingSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/VolumetricMaxPoolingSpec.scala index a9267a0874b..ee336088423 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/VolumetricMaxPoolingSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/VolumetricMaxPoolingSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch +package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl._ import com.intel.analytics.bigdl.dllib.nn.{GradientChecker, VolumetricMaxPooling} diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/models/AlexNetSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/models/AlexNetSpec.scala index 3512ba2c681..ebf1f6dd17a 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/models/AlexNetSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/models/AlexNetSpec.scala @@ -14,11 +14,11 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch.models +package com.intel.analytics.bigdl.dllib.integration.torch.models import com.intel.analytics.bigdl.Module import com.intel.analytics.bigdl.dllib.example.loadmodel.AlexNet_OWT -import com.intel.analytics.bigdl.integration.torch.{TH, TorchSpec} +import com.intel.analytics.bigdl.dllib.integration.torch.{TH, TorchSpec} import com.intel.analytics.bigdl.dllib.nn.ClassNLLCriterion import com.intel.analytics.bigdl.numeric.NumericFloat import com.intel.analytics.bigdl.dllib.optim.SGD diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/models/InceptionSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/models/InceptionSpec.scala index 2f7ceca9c6c..1c088905b17 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/models/InceptionSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/models/InceptionSpec.scala @@ -14,14 +14,14 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.integration.torch.models +package com.intel.analytics.bigdl.dllib.integration.torch.models import com.intel.analytics.bigdl.dllib.models.inception._ import com.intel.analytics.bigdl.dllib.nn.{ClassNLLCriterion, Graph, Input} import com.intel.analytics.bigdl.dllib.nn.abstractnn.AbstractModule import com.intel.analytics.bigdl.dllib.optim.SGD import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.integration.torch.{TH, TorchSpec} +import com.intel.analytics.bigdl.dllib.integration.torch.{TH, TorchSpec} import com.intel.analytics.bigdl.dllib.models.Inception import com.intel.analytics.bigdl.utils.RandomGenerator._ import com.intel.analytics.bigdl.dllib.utils.{T, Table} diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/serializer/SerializerSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/serializer/SerializerSpec.scala index 92bd511296c..3832def8c50 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/serializer/SerializerSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/serializer/SerializerSpec.scala @@ -69,7 +69,8 @@ class SerializerSpec extends BigDLSpecHelper { "com.intel.analytics.bigdl.dllib.nn.mkldnn.Output", "com.intel.analytics.bigdl.dllib.nn.mkldnn.InputWrapper", "com.intel.analytics.bigdl.dllib.utils.intermediate.IRGraph", - "com.intel.analytics.bigdl.dllib.nn.mkldnn.RNN" + "com.intel.analytics.bigdl.dllib.nn.mkldnn.RNN", + "com.intel.analytics.bigdl.dllib.nn.ops.TruncatedNormal" ) // Maybe one serial test class contains multiple module test From 39409d5514e872ff26d2337ead53aea32fe49a6c Mon Sep 17 00:00:00 2001 From: dding3 Date: Fri, 20 Aug 2021 14:22:17 -0700 Subject: [PATCH 813/823] move common.utils to dllib.utils --- python/dllib/src/bigdl/__init__.py | 2 +- .../src/bigdl/dllib/bigdlkeras/backend.py | 2 +- .../bigdl/dllib/bigdlkeras/layers/__init__.py | 2 +- .../bigdl/dllib/bigdlkeras/layers/layer.py | 2 +- .../bigdl/dllib/bigdlkeras/optimization.py | 2 +- .../imageframe/inception_validation.py | 6 +- .../dllib/examples/keras/imdb_cnn_lstm.py | 2 +- .../bigdl/dllib/examples/keras/keras_utils.py | 2 +- .../bigdl/dllib/examples/keras/mnist_cnn.py | 2 +- .../bigdl/dllib/feature/dataset/dataset.py | 6 +- .../dllib/feature/dataset/transformer.py | 2 +- .../dllib/feature/transform/vision/image.py | 6 +- .../dllib/src/bigdl/dllib/keras/converter.py | 4 +- .../src/bigdl/dllib/models/lenet/lenet5.py | 2 +- .../dllib/models/local_lenet/local_lenet.py | 2 +- .../dllib/models/ml_pipeline/dl_classifier.py | 2 +- .../src/bigdl/dllib/models/rnn/rnnexample.py | 4 +- .../models/textclassifier/textclassifier.py | 4 +- python/dllib/src/bigdl/dllib/nn/criterion.py | 10 +- .../bigdl/dllib/nn/initialization_method.py | 2 +- python/dllib/src/bigdl/dllib/nn/layer.py | 26 +- python/dllib/src/bigdl/dllib/nn/onnx/layer.py | 2 +- .../dllib/src/bigdl/dllib/optim/optimizer.py | 20 +- python/dllib/src/bigdl/dllib/utils/common.py | 16 +- .../dllib/src/bigdl/dllib/utils/tf_utils.py | 4 +- python/dllib/src/bigdl/utils/__init__.py | 15 -- .../test/bigdl/caffe/test_caffe_layers.py | 2 +- .../dllib/test/bigdl/caffe/test_load_caffe.py | 2 +- python/dllib/test/bigdl/keras/test_backend.py | 4 +- python/dllib/test/bigdl/test_engine_env.py | 4 +- python/dllib/test/bigdl/test_pickler.py | 6 +- .../test/bigdl/test_simple_integration.py | 8 +- python/dllib/test/bigdl/test_tensorflow.py | 2 +- python/dllib/test/bigdl/test_utils.py | 2 +- .../dllib/test/bigdl/transform/test_image.py | 2 +- scala/common/utils/pom.xml | 247 ------------------ .../utils/src/main/resources/spark-bigdl.conf | 33 --- .../analytics/bigdl/tags/Integration.java | 14 - .../intel/analytics/bigdl/tags/Parallel.java | 14 - .../intel/analytics/bigdl/tags/Serial.java | 14 - scala/dllib/pom.xml | 5 - .../imageFrame/InceptionValidation.scala | 2 +- .../bigdl/dllib/example/keras/Train.scala | 2 +- .../example/languagemodel/PTBModel.scala | 2 +- .../example/languagemodel/PTBWordLM.scala | 2 +- .../dllib/example/lenetLocal/Predict.scala | 2 +- .../bigdl/dllib/example/lenetLocal/Test.scala | 2 +- .../dllib/example/lenetLocal/Train.scala | 4 +- .../example/loadmodel/ModelValidator.scala | 2 +- .../mkldnn/int8/GenerateInt8Scales.scala | 2 +- .../mkldnn/int8/ImageNetInference.scala | 2 +- .../transferlearning/TransferLearning.scala | 4 +- .../textclassification/TextClassifier.scala | 4 +- .../example/treeLSTMSentiment/Train.scala | 4 +- .../example/treeLSTMSentiment/Utils.scala | 2 +- .../udfpredictor/DataframePredictor.scala | 4 +- .../dllib/example/utils/TextClassifier.scala | 4 +- .../bigdl/dllib/feature/dataset/DataSet.scala | 4 +- .../bigdl/dllib/feature/dataset/Utils.scala | 2 +- .../feature/dataset/image/BGRImgCropper.scala | 2 +- .../dataset/image/BGRImgRdmCropper.scala | 2 +- .../feature/dataset/image/ColorJitter.scala | 4 +- .../dataset/image/GreyImgCropper.scala | 2 +- .../dllib/feature/dataset/image/HFlip.scala | 2 +- .../feature/dataset/image/Lighting.scala | 2 +- .../image/MTLabeledBGRImgToBatch.scala | 2 +- .../vision/image/MTImageFeatureToBatch.scala | 2 +- .../image/augmentation/Brightness.scala | 2 +- .../image/augmentation/ColorJitter.scala | 2 +- .../vision/image/augmentation/Contrast.scala | 2 +- .../vision/image/augmentation/Crop.scala | 2 +- .../vision/image/augmentation/Expand.scala | 2 +- .../vision/image/augmentation/Hue.scala | 2 +- .../augmentation/RandomAlterAspect.scala | 2 +- .../image/augmentation/RandomCropper.scala | 4 +- .../image/augmentation/RandomResize.scala | 2 +- .../augmentation/RandomTransformer.scala | 2 +- .../image/augmentation/Saturation.scala | 2 +- .../vision/image/label/roi/BatchSampler.scala | 2 +- .../image/label/roi/RandomSampler.scala | 2 +- .../bigdl/dllib/keras/Activation.scala | 2 +- .../dllib/keras/AtrousConvolution1D.scala | 2 +- .../dllib/keras/AtrousConvolution2D.scala | 2 +- .../bigdl/dllib/keras/AveragePooling1D.scala | 2 +- .../bigdl/dllib/keras/AveragePooling2D.scala | 2 +- .../bigdl/dllib/keras/AveragePooling3D.scala | 2 +- .../dllib/keras/BatchNormalization.scala | 2 +- .../bigdl/dllib/keras/Bidirectional.scala | 2 +- .../bigdl/dllib/keras/ConvLSTM2D.scala | 2 +- .../bigdl/dllib/keras/Convolution1D.scala | 2 +- .../bigdl/dllib/keras/Convolution2D.scala | 2 +- .../bigdl/dllib/keras/Convolution3D.scala | 2 +- .../bigdl/dllib/keras/Cropping1D.scala | 2 +- .../bigdl/dllib/keras/Cropping2D.scala | 2 +- .../bigdl/dllib/keras/Cropping3D.scala | 2 +- .../bigdl/dllib/keras/Deconvolution2D.scala | 2 +- .../analytics/bigdl/dllib/keras/Dense.scala | 2 +- .../analytics/bigdl/dllib/keras/Dropout.scala | 2 +- .../analytics/bigdl/dllib/keras/ELU.scala | 2 +- .../bigdl/dllib/keras/Embedding.scala | 2 +- .../analytics/bigdl/dllib/keras/Flatten.scala | 2 +- .../analytics/bigdl/dllib/keras/GRU.scala | 2 +- .../bigdl/dllib/keras/GaussianDropout.scala | 2 +- .../bigdl/dllib/keras/GaussianNoise.scala | 2 +- .../dllib/keras/GlobalAveragePooling1D.scala | 2 +- .../dllib/keras/GlobalAveragePooling2D.scala | 2 +- .../dllib/keras/GlobalAveragePooling3D.scala | 2 +- .../dllib/keras/GlobalMaxPooling1D.scala | 2 +- .../dllib/keras/GlobalMaxPooling2D.scala | 2 +- .../dllib/keras/GlobalMaxPooling3D.scala | 2 +- .../bigdl/dllib/keras/GlobalPooling1D.scala | 2 +- .../bigdl/dllib/keras/GlobalPooling2D.scala | 2 +- .../bigdl/dllib/keras/GlobalPooling3D.scala | 2 +- .../analytics/bigdl/dllib/keras/Highway.scala | 2 +- .../analytics/bigdl/dllib/keras/Input.scala | 2 +- .../bigdl/dllib/keras/KerasLayer.scala | 2 +- .../analytics/bigdl/dllib/keras/LSTM.scala | 2 +- .../bigdl/dllib/keras/LeakyReLU.scala | 2 +- .../dllib/keras/LocallyConnected1D.scala | 2 +- .../dllib/keras/LocallyConnected2D.scala | 2 +- .../analytics/bigdl/dllib/keras/Masking.scala | 2 +- .../bigdl/dllib/keras/MaxPooling1D.scala | 2 +- .../bigdl/dllib/keras/MaxPooling2D.scala | 2 +- .../bigdl/dllib/keras/MaxPooling3D.scala | 2 +- .../bigdl/dllib/keras/MaxoutDense.scala | 2 +- .../analytics/bigdl/dllib/keras/Merge.scala | 2 +- .../analytics/bigdl/dllib/keras/Permute.scala | 2 +- .../bigdl/dllib/keras/Pooling1D.scala | 2 +- .../bigdl/dllib/keras/Pooling2D.scala | 2 +- .../bigdl/dllib/keras/Pooling3D.scala | 2 +- .../bigdl/dllib/keras/Recurrent.scala | 2 +- .../bigdl/dllib/keras/RepeatVector.scala | 2 +- .../analytics/bigdl/dllib/keras/Reshape.scala | 2 +- .../analytics/bigdl/dllib/keras/SReLU.scala | 2 +- .../dllib/keras/SeparableConvolution2D.scala | 2 +- .../bigdl/dllib/keras/SimpleRNN.scala | 2 +- .../analytics/bigdl/dllib/keras/SoftMax.scala | 2 +- .../bigdl/dllib/keras/SpatialDropout1D.scala | 2 +- .../bigdl/dllib/keras/SpatialDropout2D.scala | 2 +- .../bigdl/dllib/keras/SpatialDropout3D.scala | 2 +- .../bigdl/dllib/keras/ThresholdedReLU.scala | 2 +- .../bigdl/dllib/keras/TimeDistributed.scala | 2 +- .../bigdl/dllib/keras/Topology.scala | 2 +- .../bigdl/dllib/keras/UpSampling1D.scala | 2 +- .../bigdl/dllib/keras/UpSampling2D.scala | 2 +- .../bigdl/dllib/keras/UpSampling3D.scala | 2 +- .../bigdl/dllib/keras/ZeroPadding1D.scala | 2 +- .../bigdl/dllib/keras/ZeroPadding2D.scala | 2 +- .../bigdl/dllib/keras/ZeroPadding3D.scala | 2 +- .../dllib/models/autoencoder/Train.scala | 2 +- .../bigdl/dllib/models/inception/Test.scala | 2 +- .../bigdl/dllib/models/inception/Train.scala | 4 +- .../bigdl/dllib/models/lenet/LeNet5.scala | 4 +- .../bigdl/dllib/models/lenet/Test.scala | 2 +- .../bigdl/dllib/models/lenet/Train.scala | 2 +- .../bigdl/dllib/models/lenet/Utils.scala | 2 +- .../bigdl/dllib/models/maskrcnn/Test.scala | 2 +- .../bigdl/dllib/models/resnet/ResNet.scala | 2 +- .../bigdl/dllib/models/resnet/Test.scala | 2 +- .../dllib/models/resnet/TestImageNet.scala | 2 +- .../dllib/models/resnet/TrainCIFAR10.scala | 4 +- .../dllib/models/resnet/TrainImageNet.scala | 2 +- .../bigdl/dllib/models/rnn/Test.scala | 2 +- .../bigdl/dllib/models/rnn/Train.scala | 2 +- .../models/utils/DistriOptimizerPerf.scala | 2 +- .../models/utils/LocalOptimizerPerf.scala | 2 +- .../dllib/models/utils/ModelBroadcast.scala | 2 +- .../bigdl/dllib/models/vgg/Test.scala | 2 +- .../bigdl/dllib/models/vgg/Train.scala | 4 +- .../dllib/models/vgg/TrainImageNet.scala | 4 +- .../intel/analytics/bigdl/dllib/nn/Add.scala | 2 +- .../bigdl/dllib/nn/BatchNormalization.scala | 2 +- .../analytics/bigdl/dllib/nn/Bilinear.scala | 2 +- .../bigdl/dllib/nn/BinaryThreshold.scala | 2 +- .../analytics/bigdl/dllib/nn/BoxHead.scala | 2 +- .../intel/analytics/bigdl/dllib/nn/CAdd.scala | 4 +- .../intel/analytics/bigdl/dllib/nn/CMul.scala | 4 +- .../bigdl/dllib/nn/ClassNLLCriterion.scala | 2 +- .../analytics/bigdl/dllib/nn/Concat.scala | 2 +- .../analytics/bigdl/dllib/nn/Cosine.scala | 2 +- .../analytics/bigdl/dllib/nn/Cropping2D.scala | 2 +- .../analytics/bigdl/dllib/nn/Cropping3D.scala | 2 +- .../bigdl/dllib/nn/DetectionOutputSSD.scala | 2 +- .../analytics/bigdl/dllib/nn/Dropout.scala | 6 +- .../analytics/bigdl/dllib/nn/Euclidean.scala | 2 +- .../analytics/bigdl/dllib/nn/HardTanh.scala | 2 +- .../bigdl/dllib/nn/InferReshape.scala | 2 +- .../bigdl/dllib/nn/InitializationMethod.scala | 2 +- .../analytics/bigdl/dllib/nn/JoinTable.scala | 2 +- .../dllib/nn/L1HingeEmbeddingCriterion.scala | 2 +- .../analytics/bigdl/dllib/nn/Linear.scala | 2 +- .../bigdl/dllib/nn/LocallyConnected1D.scala | 2 +- .../bigdl/dllib/nn/LocallyConnected2D.scala | 4 +- .../analytics/bigdl/dllib/nn/LogSoftMax.scala | 4 +- .../bigdl/dllib/nn/LookupTable.scala | 4 +- .../analytics/bigdl/dllib/nn/Maxout.scala | 2 +- .../intel/analytics/bigdl/dllib/nn/Mul.scala | 2 +- .../analytics/bigdl/dllib/nn/Normalize.scala | 2 +- .../bigdl/dllib/nn/NormalizeScale.scala | 2 +- .../analytics/bigdl/dllib/nn/PReLU.scala | 2 +- .../bigdl/dllib/nn/PairwiseDistance.scala | 2 +- .../analytics/bigdl/dllib/nn/Power.scala | 2 +- .../analytics/bigdl/dllib/nn/PriorBox.scala | 2 +- .../intel/analytics/bigdl/dllib/nn/RNN.scala | 2 +- .../analytics/bigdl/dllib/nn/RReLU.scala | 2 +- .../bigdl/dllib/nn/RegionProposal.scala | 2 +- .../analytics/bigdl/dllib/nn/Reshape.scala | 2 +- .../analytics/bigdl/dllib/nn/Scale.scala | 2 +- .../analytics/bigdl/dllib/nn/SoftMax.scala | 4 +- .../bigdl/dllib/nn/SparseJoinTable.scala | 2 +- .../dllib/nn/SpatialAveragePooling.scala | 2 +- .../dllib/nn/SpatialBatchNormalization.scala | 2 +- .../bigdl/dllib/nn/SpatialConvolution.scala | 2 +- .../bigdl/dllib/nn/SpatialCrossMapLRN.scala | 2 +- .../dllib/nn/SpatialDilatedConvolution.scala | 4 +- .../dllib/nn/SpatialFullConvolution.scala | 2 +- .../bigdl/dllib/nn/SpatialMaxPooling.scala | 2 +- .../nn/SpatialSeparableConvolution.scala | 2 +- .../dllib/nn/SpatialShareConvolution.scala | 2 +- .../analytics/bigdl/dllib/nn/Squeeze.scala | 2 +- .../intel/analytics/bigdl/dllib/nn/Tanh.scala | 2 +- .../bigdl/dllib/nn/TemporalConvolution.scala | 2 +- .../analytics/bigdl/dllib/nn/Threshold.scala | 4 +- .../bigdl/dllib/nn/TimeDistributed.scala | 2 +- .../dllib/nn/TimeDistributedCriterion.scala | 2 +- .../nn/TimeDistributedMaskCriterion.scala | 2 +- .../bigdl/dllib/nn/TransformerOperation.scala | 2 +- .../analytics/bigdl/dllib/nn/Transpose.scala | 2 +- .../bigdl/dllib/nn/UpSampling1D.scala | 2 +- .../bigdl/dllib/nn/UpSampling2D.scala | 2 +- .../bigdl/dllib/nn/UpSampling3D.scala | 2 +- .../intel/analytics/bigdl/dllib/nn/View.scala | 2 +- .../dllib/nn/VolumetricConvolution.scala | 4 +- .../dllib/nn/abstractnn/AbstractModule.scala | 2 +- .../nn/abstractnn/IdentityOutputShape.scala | 2 +- .../dllib/nn/abstractnn/InferShape.scala | 2 +- .../bigdl/dllib/nn/mkldnn/BlasWrapper.scala | 4 +- .../bigdl/dllib/nn/mkldnn/Perf.scala | 6 +- .../bigdl/dllib/nn/mkldnn/SoftMax.scala | 2 +- .../dllib/nn/ops/CategoricalColVocaList.scala | 2 +- .../bigdl/dllib/nn/ops/RandomUniform.scala | 2 +- .../analytics/bigdl/dllib/nn/ops/Tile.scala | 2 +- .../bigdl/dllib/optim/AbstractOptimizer.scala | 2 +- .../bigdl/dllib/optim/DistriOptimizer.scala | 2 +- .../bigdl/dllib/optim/DistriOptimizerV2.scala | 2 +- .../bigdl/dllib/optim/DistriValidator.scala | 2 +- .../bigdl/dllib/optim/Evaluator.scala | 2 +- .../bigdl/dllib/optim/LocalOptimizer.scala | 2 +- .../bigdl/dllib/optim/LocalPredictor.scala | 2 +- .../bigdl/dllib/optim/LocalValidator.scala | 2 +- .../bigdl/dllib/optim/Optimizer.scala | 2 +- .../bigdl/dllib/optim/ParallelAdam.scala | 2 +- .../bigdl/dllib/optim/ParallelOptimizer.scala | 2 +- .../bigdl/dllib/optim/PredictionService.scala | 2 +- .../bigdl/dllib/optim/Predictor.scala | 2 +- .../bigdl/dllib/optim/Validator.scala | 2 +- .../optim/parameters/AllReduceParameter.scala | 2 +- .../parameters/FP16CompressedTensor.scala | 2 +- .../optim/parameters/UncompressedTensor.scala | 2 +- .../bigdl/dllib/optim/parameters/Util.scala | 2 +- .../bigdl/dllib/tensor/DenseTensor.scala | 2 +- .../bigdl/dllib/tensor/TensorNumeric.scala | 2 +- .../utils/DistriParameterSynchronizer.scala | 2 +- .../analytics/bigdl/dllib/utils/Engine.scala | 4 +- .../bigdl/dllib/utils/HashFunc.scala | 2 +- .../bigdl/dllib/utils/LayerException.scala | 2 +- .../bigdl/dllib/utils/LoggerFilter.scala | 2 +- .../bigdl/dllib/utils/RandomGenerator.scala | 2 +- .../analytics/bigdl/dllib/utils/Shape.scala | 2 +- .../bigdl/dllib/utils/ThreadPool.scala | 2 +- .../utils/intermediate/ConversionUtils.scala | 2 +- .../utils/intermediate/IRConverter.scala | 2 +- .../dllib/utils/intermediate/IRGraph.scala | 2 +- .../dllib/utils/python/api/PythonBigDL.scala | 2 +- .../utils/python/api/PythonBigDLKeras.scala | 2 +- .../utils/serializer/ModuleSerializable.scala | 2 +- .../serializer/converters/DataConverter.scala | 2 +- .../converters/ShapeConverter.scala | 2 +- .../bigdl/dllib/utils/tf/Session.scala | 2 +- .../bigdl/dllib/utils/tf/Tensorflow.scala | 2 +- .../tensorboard/FileWriter.scala | 2 +- .../dllib/dataset/BatchPaddingSpec.scala | 2 +- .../bigdl/dllib/dataset/DataSetSpec.scala | 4 +- .../dllib/dataset/TransformersSpec.scala | 6 +- .../dllib/dataset/text/DictionarySpec.scala | 2 +- .../dataset/text/SentenceBiPaddingSpec.scala | 2 +- .../dataset/text/SentenceTokenizerSpec.scala | 2 +- .../text/TextToLabeledSentenceSpec.scala | 2 +- .../bigdl/dllib/integration/HdfsSpec.scala | 2 +- .../dllib/integration/Quantization.scala | 4 +- .../dllib/integration/torch/AdagradSpec.scala | 2 +- .../integration/torch/AddConstantSpec.scala | 2 +- .../dllib/integration/torch/AddSpec.scala | 2 +- .../torch/BatchNormalizationSpec.scala | 2 +- .../integration/torch/BiRecurrentSpec.scala | 2 +- .../integration/torch/BilinearSpec.scala | 2 +- .../dllib/integration/torch/BottleSpec.scala | 2 +- .../dllib/integration/torch/CAddSpec.scala | 2 +- .../integration/torch/CAddTableSpec.scala | 2 +- .../integration/torch/CDivTableSpec.scala | 2 +- .../integration/torch/CMaxTableSpec.scala | 2 +- .../integration/torch/CMinTableSpec.scala | 2 +- .../dllib/integration/torch/CMulSpec.scala | 2 +- .../integration/torch/CMulTableSpec.scala | 2 +- .../integration/torch/CSubTableSpec.scala | 2 +- .../integration/torch/ColorJitterSpec.scala | 4 +- .../dllib/integration/torch/ConcatSpec.scala | 2 +- .../integration/torch/ConcatTableSpec.scala | 2 +- .../integration/torch/ContiguousSpec.scala | 2 +- .../torch/CosineDistanceCriterionSpec.scala | 4 +- .../torch/CosineDistanceSpec.scala | 2 +- .../torch/CosineEmbeddingCriterionSpec.scala | 4 +- .../dllib/integration/torch/CosineSpec.scala | 2 +- .../torch/CrossEntropyCriterionSpec.scala | 2 +- .../torch/DistKLDivCriterionSpec.scala | 2 +- .../dllib/integration/torch/DropoutSpec.scala | 4 +- .../dllib/integration/torch/ELUSpec.scala | 4 +- .../integration/torch/EuclideanSpec.scala | 2 +- .../dllib/integration/torch/GRUSpec.scala | 4 +- .../torch/GaussianCriterionSpec.scala | 2 +- .../torch/GradientReversalSpec.scala | 2 +- .../integration/torch/HardShrinkSpec.scala | 2 +- .../torch/HingeEmbeddingCriterionSpec.scala | 2 +- .../dllib/integration/torch/IndexSpec.scala | 2 +- .../integration/torch/JoinTableSpec.scala | 2 +- .../torch/L1HingeEmbeddingCriterionSpec.scala | 2 +- .../integration/torch/LSTMPeepholeSpec.scala | 2 +- .../dllib/integration/torch/LSTMSpec.scala | 2 +- .../integration/torch/LeakyReLUSpec.scala | 4 +- .../dllib/integration/torch/LinearSpec.scala | 2 +- .../integration/torch/LogSoftMaxSpec.scala | 2 +- .../dllib/integration/torch/LogSpec.scala | 2 +- .../integration/torch/LookupTableSpec.scala | 2 +- .../dllib/integration/torch/MMSpec.scala | 2 +- .../dllib/integration/torch/MVSpec.scala | 2 +- .../dllib/integration/torch/MeanSpec.scala | 2 +- .../dllib/integration/torch/MulSpec.scala | 2 +- .../torch/MultiCriterionSpec.scala | 2 +- .../torch/ParallelCriterionSpec.scala | 2 +- .../dllib/integration/torch/RReLUSpec.scala | 2 +- .../dllib/integration/torch/ReLU6Spec.scala | 2 +- .../dllib/integration/torch/ReLUSpec.scala | 2 +- .../dllib/integration/torch/SamplerSpec.scala | 2 +- .../dllib/integration/torch/SelectSpec.scala | 2 +- .../integration/torch/SequentialSpec.scala | 2 +- .../dllib/integration/torch/SoftMinSpec.scala | 2 +- .../torch/SpatialAveragePoolingSpec.scala | 2 +- .../torch/SpatialBatchNormalizationSpec.scala | 2 +- .../SpatialContrastiveNormalizationSpec.scala | 2 +- .../torch/SpatialConvolutionMapSpec.scala | 2 +- .../torch/SpatialConvolutionSpec.scala | 4 +- .../torch/SpatialCrossMapLRNSpec.scala | 2 +- .../torch/SpatialDilatedConvolutionSpec.scala | 2 +- .../SpatialDivisiveNormalizationSpec.scala | 2 +- .../torch/SpatialDropout1DSpec.scala | 2 +- .../torch/SpatialDropout2DSpec.scala | 2 +- .../torch/SpatialDropout3DSpec.scala | 2 +- .../torch/SpatialFullConvolutionSpec.scala | 2 +- .../torch/SpatialMaxPoolingSpec.scala | 2 +- .../SpatialSubtractiveNormalizationSpec.scala | 4 +- .../dllib/integration/torch/SumSpec.scala | 2 +- .../integration/torch/TanhShrinkSpec.scala | 2 +- .../torch/TemporalConvolutionSpec.scala | 2 +- .../torch/TemporalMaxPoolingSpec.scala | 2 +- .../integration/torch/ThresholdSpec.scala | 2 +- .../dllib/integration/torch/TorchSpec.scala | 2 +- .../dllib/integration/torch/ViewSpec.scala | 2 +- .../torch/VolumetricAveragePoolingSpec.scala | 2 +- .../torch/VolumetricConvolutionSpec.scala | 4 +- .../torch/VolumetricFullConvolutionSpec.scala | 2 +- .../torch/VolumetricMaxPoolingSpec.scala | 2 +- .../torch/models/AlexNetSpec.scala | 2 +- .../torch/models/InceptionSpec.scala | 2 +- .../bigdl/dllib/keras/Cropping2DSpec.scala | 2 +- .../bigdl/dllib/keras/Cropping3DSpec.scala | 2 +- .../dllib/keras/LocallyConnected2DSpec.scala | 2 +- .../bigdl/dllib/keras/UpSampling1DSpec.scala | 2 +- .../bigdl/dllib/keras/UpSampling2DSpec.scala | 2 +- .../bigdl/dllib/keras/UpSampling3DSpec.scala | 2 +- .../bigdl/dllib/keras/nn/ActivationSpec.scala | 2 +- .../keras/nn/AtrousConvolution1DSpec.scala | 2 +- .../keras/nn/AtrousConvolution2DSpec.scala | 2 +- .../dllib/keras/nn/AveragePooling1DSpec.scala | 2 +- .../dllib/keras/nn/AveragePooling2DSpec.scala | 2 +- .../dllib/keras/nn/AveragePooling3DSpec.scala | 2 +- .../keras/nn/BatchNormalizationSpec.scala | 2 +- .../dllib/keras/nn/BidirectionalSpec.scala | 2 +- .../bigdl/dllib/keras/nn/ConvLSTM2DSpec.scala | 2 +- .../dllib/keras/nn/Convolution1DSpec.scala | 2 +- .../dllib/keras/nn/Convolution2DSpec.scala | 2 +- .../dllib/keras/nn/Convolution3DSpec.scala | 2 +- .../bigdl/dllib/keras/nn/Cropping1DSpec.scala | 2 +- .../bigdl/dllib/keras/nn/Cropping2DSpec.scala | 2 +- .../bigdl/dllib/keras/nn/Cropping3DSpec.scala | 2 +- .../dllib/keras/nn/Deconvolution2DSpec.scala | 2 +- .../bigdl/dllib/keras/nn/DenseSpec.scala | 2 +- .../bigdl/dllib/keras/nn/DropoutSpec.scala | 2 +- .../bigdl/dllib/keras/nn/ELUSpec.scala | 2 +- .../bigdl/dllib/keras/nn/EmbeddingSpec.scala | 2 +- .../bigdl/dllib/keras/nn/FlattenSpec.scala | 2 +- .../bigdl/dllib/keras/nn/GRUSpec.scala | 2 +- .../dllib/keras/nn/GaussianDropoutSpec.scala | 2 +- .../dllib/keras/nn/GaussianNoiseSpec.scala | 2 +- .../keras/nn/GlobalAveragePooling1DSpec.scala | 2 +- .../keras/nn/GlobalAveragePooling2DSpec.scala | 2 +- .../keras/nn/GlobalAveragePooling3DSpec.scala | 2 +- .../keras/nn/GlobalMaxPooling1DSpec.scala | 2 +- .../keras/nn/GlobalMaxPooling2DSpec.scala | 2 +- .../keras/nn/GlobalMaxPooling3DSpec.scala | 2 +- .../bigdl/dllib/keras/nn/HighwaySpec.scala | 2 +- .../bigdl/dllib/keras/nn/InputSpec.scala | 2 +- .../keras/nn/KerasIdentityWrapperSpec.scala | 2 +- .../nn/KerasLayerWrapperSerialTest.scala | 2 +- .../keras/nn/KerasLayerWrapperSpec.scala | 2 +- .../bigdl/dllib/keras/nn/KerasStyleSpec.scala | 2 +- .../bigdl/dllib/keras/nn/LSTMSpec.scala | 2 +- .../bigdl/dllib/keras/nn/LeakyReLUSpec.scala | 2 +- .../keras/nn/LocallyConnected1DSpec.scala | 2 +- .../keras/nn/LocallyConnected2DSpec.scala | 2 +- .../bigdl/dllib/keras/nn/MaskingSpec.scala | 2 +- .../dllib/keras/nn/MaxPooling1DSpec.scala | 2 +- .../dllib/keras/nn/MaxPooling2DSpec.scala | 2 +- .../dllib/keras/nn/MaxPooling3DSpec.scala | 2 +- .../dllib/keras/nn/MaxoutDenseSpec.scala | 2 +- .../bigdl/dllib/keras/nn/MergeSpec.scala | 2 +- .../bigdl/dllib/keras/nn/ModelSpec.scala | 2 +- .../bigdl/dllib/keras/nn/PermuteSpec.scala | 2 +- .../dllib/keras/nn/RepeatVectorSpec.scala | 2 +- .../bigdl/dllib/keras/nn/ReshapeSpec.scala | 2 +- .../bigdl/dllib/keras/nn/SReLUSpec.scala | 2 +- .../keras/nn/SeparableConvolution2DSpec.scala | 2 +- .../bigdl/dllib/keras/nn/SequentialSpec.scala | 2 +- .../bigdl/dllib/keras/nn/SimpleRNNSpec.scala | 2 +- .../dllib/keras/nn/SpatialDropout1DSpec.scala | 2 +- .../dllib/keras/nn/SpatialDropout2DSpec.scala | 2 +- .../dllib/keras/nn/SpatialDropout3DSpec.scala | 2 +- .../dllib/keras/nn/ThresholdedReLUSpec.scala | 2 +- .../dllib/keras/nn/TimeDistributedSpec.scala | 2 +- .../bigdl/dllib/keras/nn/TrainingSpec.scala | 4 +- .../dllib/keras/nn/UpSampling1DSpec.scala | 2 +- .../dllib/keras/nn/UpSampling2DSpec.scala | 2 +- .../dllib/keras/nn/UpSampling3DSpec.scala | 2 +- .../dllib/keras/nn/ZeroPadding1DSpec.scala | 2 +- .../dllib/keras/nn/ZeroPadding2DSpec.scala | 2 +- .../dllib/keras/nn/ZeroPadding3DSpec.scala | 2 +- .../bigdl/dllib/models/AlexNetSpec.scala | 2 +- .../bigdl/dllib/models/InceptionSpec.scala | 2 +- .../dllib/models/ModelGraientCheckSpec.scala | 2 +- .../bigdl/dllib/models/ResNetSpec.scala | 4 +- .../dllib/models/maskrcnn/MaskRCNNSpec.scala | 2 +- .../analytics/bigdl/dllib/nn/AddSpec.scala | 2 +- .../dllib/nn/BatchNormalizationSpec.scala | 2 +- .../bigdl/dllib/nn/BiRecurrentSpec.scala | 2 +- .../bigdl/dllib/nn/BilinearSpec.scala | 2 +- .../bigdl/dllib/nn/BinaryTreeLSTMSpec.scala | 4 +- .../analytics/bigdl/dllib/nn/CAddSpec.scala | 2 +- .../analytics/bigdl/dllib/nn/CMulSpec.scala | 2 +- .../analytics/bigdl/dllib/nn/ConcatSpec.scala | 2 +- .../bigdl/dllib/nn/ConvLSTMPeepholeSpec.scala | 2 +- .../analytics/bigdl/dllib/nn/CosineSpec.scala | 2 +- .../dllib/nn/DetectionOutputFrcnnSpec.scala | 2 +- .../dllib/nn/DetectionOutputSSDSpec.scala | 2 +- .../bigdl/dllib/nn/DropoutSpec.scala | 2 +- .../bigdl/dllib/nn/DynamicGraphSpec.scala | 4 +- .../bigdl/dllib/nn/EuclideanSpec.scala | 2 +- .../analytics/bigdl/dllib/nn/GRUSpec.scala | 2 +- .../bigdl/dllib/nn/GaussianDropoutSpec.scala | 2 +- .../bigdl/dllib/nn/GaussianNoiseSpec.scala | 2 +- .../bigdl/dllib/nn/GaussianSamplerSpec.scala | 2 +- .../analytics/bigdl/dllib/nn/GraphSpec.scala | 4 +- .../bigdl/dllib/nn/InferReshapeSpec.scala | 2 +- .../bigdl/dllib/nn/KLDCriterionSpec.scala | 2 +- .../analytics/bigdl/dllib/nn/LinearSpec.scala | 2 +- .../bigdl/dllib/nn/LogSoftMaxSpec.scala | 2 +- .../bigdl/dllib/nn/LookupTableSpec.scala | 2 +- .../bigdl/dllib/nn/MaskHeadSpec.scala | 2 +- .../analytics/bigdl/dllib/nn/MaxoutSpec.scala | 2 +- .../analytics/bigdl/dllib/nn/ModuleSpec.scala | 2 +- .../analytics/bigdl/dllib/nn/MulSpec.scala | 2 +- .../bigdl/dllib/nn/MultiRNNCellSpec.scala | 2 +- .../bigdl/dllib/nn/NormalizeSpec.scala | 2 +- .../analytics/bigdl/dllib/nn/PReLUSpec.scala | 2 +- .../analytics/bigdl/dllib/nn/PoolerSpec.scala | 2 +- .../bigdl/dllib/nn/RecurrentDecoderSpec.scala | 2 +- .../bigdl/dllib/nn/RecurrentSpec.scala | 4 +- .../bigdl/dllib/nn/ReshapeSpec.scala | 2 +- .../bigdl/dllib/nn/ReverseSpec.scala | 2 +- .../bigdl/dllib/nn/RoiAlignSpec.scala | 2 +- .../bigdl/dllib/nn/SparseJoinTableSpec.scala | 2 +- .../bigdl/dllib/nn/SparseLinearSpec.scala | 2 +- .../nn/SpatialBatchNormalizationSpec.scala | 4 +- .../SpatialContrastiveNormalizationSpec.scala | 2 +- .../dllib/nn/SpatialConvolutionMapSpec.scala | 2 +- .../dllib/nn/SpatialConvolutionSpec.scala | 4 +- .../nn/SpatialDilatedConvolutionSpec.scala | 4 +- .../dllib/nn/SpatialFullConvolutionSpec.scala | 4 +- .../dllib/nn/SpatialMaxPoolingSpec.scala | 2 +- .../nn/SpatialSeparableConvolutionSpec.scala | 2 +- .../nn/SpatialWithinChannelLRNSpec.scala | 2 +- .../bigdl/dllib/nn/TimeDistributedSpec.scala | 2 +- .../dllib/nn/mkldnn/AvgPoolingSpec.scala | 4 +- .../dllib/nn/mkldnn/BlasWrapperSpec.scala | 6 +- .../bigdl/dllib/nn/mkldnn/DnnGraphSpec.scala | 4 +- .../bigdl/dllib/nn/mkldnn/FusionSpec.scala | 4 +- .../bigdl/dllib/nn/mkldnn/LRNSpec.scala | 2 +- .../bigdl/dllib/nn/mkldnn/LinearSpec.scala | 6 +- .../dllib/nn/mkldnn/MaxPoolingSpec.scala | 4 +- .../dllib/nn/mkldnn/MultiModelsSpec.scala | 6 +- .../bigdl/dllib/nn/mkldnn/OutputSpec.scala | 2 +- .../bigdl/dllib/nn/mkldnn/RNNSpec.scala | 4 +- .../dllib/nn/mkldnn/ReorderMemorySpec.scala | 2 +- .../SpatialBatchNormalizationSpec.scala | 2 +- .../nn/mkldnn/SpatialConvolutionSpec.scala | 2 +- .../bigdl/dllib/nn/mkldnn/TopologySpec.scala | 2 +- .../bigdl/dllib/nn/ops/MaxSpec.scala | 2 +- .../dllib/nn/quantized/QuantizableSpec.scala | 4 +- .../bigdl/dllib/optim/AdamSpec.scala | 4 +- .../dllib/optim/DistriOptimizerSpec.scala | 2 +- .../dllib/optim/DistriOptimizerV2Spec.scala | 2 +- .../bigdl/dllib/optim/EvaluatorSpec.scala | 4 +- .../bigdl/dllib/optim/FtrlSpec.scala | 2 +- .../bigdl/dllib/optim/LarsSGDSpec.scala | 2 +- .../dllib/optim/LocalOptimizerSpec.scala | 2 +- .../dllib/optim/LocalPredictorSpec.scala | 4 +- .../bigdl/dllib/optim/LoggerFilterSpec.scala | 2 +- .../bigdl/dllib/optim/MetricsSpec.scala | 2 +- .../optim/OptimPredictorShutdownSpec.scala | 4 +- .../bigdl/dllib/optim/OptimizerSpec.scala | 2 +- .../dllib/optim/ParallelOptimizerSpec.scala | 2 +- .../dllib/optim/PredictionServiceSpec.scala | 2 +- .../bigdl/dllib/optim/PredictorSpec.scala | 4 +- .../bigdl/dllib/optim/ValidatorSpec.scala | 4 +- .../dllib/parameters/FP16ParameterSpec.scala | 2 +- .../bigdl/dllib/python/api/PythonSpec.scala | 4 +- .../bigdl/dllib/tensor/SparseTensorSpec.scala | 2 +- .../vision/image/ImageFrameSpec.scala | 2 +- .../image/MTImageFeatureToBatchSpec.scala | 2 +- .../vision/image/opencv/OpenCVMatSpec.scala | 2 +- .../bigdl/dllib/utils/CaffeLoaderSpec.scala | 2 +- .../utils/DistributedSynchronizerSpec.scala | 2 +- .../bigdl/dllib/utils/EngineSpec.scala | 2 +- .../bigdl/dllib/utils/GraphNodeSpec.scala | 2 +- .../dllib/utils/RandomGeneratorSpec.scala | 2 +- .../bigdl/dllib/utils/ShapeSpec.scala | 2 +- .../dllib/utils/SparkContextLifeCycle.scala | 2 +- .../utils/SpatialShareConvolutionSpec.scala | 2 +- .../bigdl/dllib/utils/TestUtils.scala | 2 +- .../bigdl/dllib/utils/ThreadPoolSpec.scala | 2 +- .../bigdl/dllib/utils/UtilSpec.scala | 2 +- .../utils/intermediate/BlasToDnnSpec.scala | 4 +- .../utils/intermediate/DnnPredictorSpec.scala | 4 +- .../utils/intermediate/IRGraphSpec.scala | 2 +- .../utils/intermediate/IRconvertSpec.scala | 4 +- .../utils/serializer/DataConverterSpec.scala | 4 +- .../serializer/SerializerSpecHelper.scala | 4 +- .../bigdl/dllib/utils/tf/SessionSpec.scala | 2 +- .../dllib/utils/tf/TensorflowLoaderSpec.scala | 2 +- .../dllib/utils/tf/TensorflowSpecHelper.scala | 2 +- .../loaders/DepthwiseConv2DNativeSpec.scala | 2 +- ...thwiseConv2dNativeBackpropFilterSpec.scala | 2 +- ...pthwiseConv2dNativeBackpropInputSpec.scala | 2 +- .../dllib/utils/tf/loaders/DigammaSpec.scala | 2 +- .../tf/loaders/FusedBatchNormGradSpec.scala | 2 +- .../dllib/utils/tf/loaders/LRNGradSpec.scala | 2 +- .../dllib/utils/tf/loaders/TopKV2Spec.scala | 2 +- .../dllib/visualization/SummarySpec.scala | 2 +- scala/pom.xml | 1 - 567 files changed, 685 insertions(+), 1028 deletions(-) delete mode 100644 python/dllib/src/bigdl/utils/__init__.py delete mode 100644 scala/common/utils/pom.xml delete mode 100644 scala/common/utils/src/main/resources/spark-bigdl.conf delete mode 100644 scala/common/utils/src/test/java/com/intel/analytics/bigdl/tags/Integration.java delete mode 100644 scala/common/utils/src/test/java/com/intel/analytics/bigdl/tags/Parallel.java delete mode 100644 scala/common/utils/src/test/java/com/intel/analytics/bigdl/tags/Serial.java diff --git a/python/dllib/src/bigdl/__init__.py b/python/dllib/src/bigdl/__init__.py index 7f835104b9e..349eb33267b 100644 --- a/python/dllib/src/bigdl/__init__.py +++ b/python/dllib/src/bigdl/__init__.py @@ -14,5 +14,5 @@ # limitations under the License. # -from bigdl.utils.engine import prepare_env +from bigdl.dllib.utils.engine import prepare_env prepare_env() diff --git a/python/dllib/src/bigdl/dllib/bigdlkeras/backend.py b/python/dllib/src/bigdl/dllib/bigdlkeras/backend.py index bcbb75b704b..4e76273da12 100644 --- a/python/dllib/src/bigdl/dllib/bigdlkeras/backend.py +++ b/python/dllib/src/bigdl/dllib/bigdlkeras/backend.py @@ -15,7 +15,7 @@ # from bigdl.dllib.keras.optimization import * -from bigdl.utils.common import * +from bigdl.dllib.utils.common import * class KerasModelWrapper: diff --git a/python/dllib/src/bigdl/dllib/bigdlkeras/layers/__init__.py b/python/dllib/src/bigdl/dllib/bigdlkeras/layers/__init__.py index 9eb670a1780..2ad55ee4823 100644 --- a/python/dllib/src/bigdl/dllib/bigdlkeras/layers/__init__.py +++ b/python/dllib/src/bigdl/dllib/bigdlkeras/layers/__init__.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # -from bigdl.utils.common import * +from bigdl.dllib.utils.common import * init_engine() redire_spark_logs() diff --git a/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py b/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py index cc63b531d15..8eb90be9261 100644 --- a/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py +++ b/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py @@ -17,7 +17,7 @@ import sys from bigdl.dllib.nn.layer import Layer, Node, SharedStaticUtils, Container -from bigdl.utils.common import callBigDlFunc, JTensor, JavaValue +from bigdl.dllib.utils.common import callBigDlFunc, JTensor, JavaValue if sys.version >= '3': long = int diff --git a/python/dllib/src/bigdl/dllib/bigdlkeras/optimization.py b/python/dllib/src/bigdl/dllib/bigdlkeras/optimization.py index 50aafef7e45..89add68c51b 100644 --- a/python/dllib/src/bigdl/dllib/bigdlkeras/optimization.py +++ b/python/dllib/src/bigdl/dllib/bigdlkeras/optimization.py @@ -18,7 +18,7 @@ import bigdl.dllib.nn.criterion as bcriterion import bigdl.dllib.optim.optimizer as boptimizer -from bigdl.utils.common import to_list +from bigdl.dllib.utils.common import to_list from bigdl.dllib.keras.converter import * from keras.objectives import * import six diff --git a/python/dllib/src/bigdl/dllib/examples/imageframe/inception_validation.py b/python/dllib/src/bigdl/dllib/examples/imageframe/inception_validation.py index 148dea16f2b..a39a5c39074 100644 --- a/python/dllib/src/bigdl/dllib/examples/imageframe/inception_validation.py +++ b/python/dllib/src/bigdl/dllib/examples/imageframe/inception_validation.py @@ -14,7 +14,7 @@ # limitations under the License. # -from bigdl.utils.common import * +from bigdl.dllib.utils.common import * from bigdl.dllib.feature.transform.vision.image import * @@ -50,11 +50,11 @@ def run(image_path, model_path, batch_size): transformed = transformer(raw_image_frame) model = Model.loadModel(model_path) result = model.evaluate(transformed, int(batch_size), [Top1Accuracy()]) - print "top1 accuray", result[0] + print("top1 accuray", result[0]) if __name__ == "__main__": if len(sys.argv) != 3: - print "parameters needed : " + print("parameters needed : ") image_path = sys.argv[1] model_path = sys.argv[2] batch_size = sys.argv[3] diff --git a/python/dllib/src/bigdl/dllib/examples/keras/imdb_cnn_lstm.py b/python/dllib/src/bigdl/dllib/examples/keras/imdb_cnn_lstm.py index ccc17fdfb53..70ae25c8c94 100644 --- a/python/dllib/src/bigdl/dllib/examples/keras/imdb_cnn_lstm.py +++ b/python/dllib/src/bigdl/dllib/examples/keras/imdb_cnn_lstm.py @@ -72,7 +72,7 @@ def build_keras_model(): hdf5_path = "/tmp/imdb.h5" keras_model.save(hdf5_path) - from bigdl.utils.common import * + from bigdl.dllib.utils.common import * from bigdl.dllib.nn.layer import * from bigdl.dllib.optim.optimizer import * from bigdl.dllib.nn.criterion import * diff --git a/python/dllib/src/bigdl/dllib/examples/keras/keras_utils.py b/python/dllib/src/bigdl/dllib/examples/keras/keras_utils.py index e95f394e7c5..209d640482d 100644 --- a/python/dllib/src/bigdl/dllib/examples/keras/keras_utils.py +++ b/python/dllib/src/bigdl/dllib/examples/keras/keras_utils.py @@ -14,7 +14,7 @@ # limitations under the License. # -from bigdl.utils.common import * +from bigdl.dllib.utils.common import * def save_keras_definition(keras_model, path): diff --git a/python/dllib/src/bigdl/dllib/examples/keras/mnist_cnn.py b/python/dllib/src/bigdl/dllib/examples/keras/mnist_cnn.py index 05d9b41e0e4..74eabd524ee 100644 --- a/python/dllib/src/bigdl/dllib/examples/keras/mnist_cnn.py +++ b/python/dllib/src/bigdl/dllib/examples/keras/mnist_cnn.py @@ -85,7 +85,7 @@ def build_keras_model(): json_path = "/tmp/lenet.json" save_keras_definition(keras_model, json_path) - from bigdl.utils.common import * + from bigdl.dllib.utils.common import * from bigdl.dllib.nn.layer import * from bigdl.dllib.optim.optimizer import * from bigdl.dllib.nn.criterion import * diff --git a/python/dllib/src/bigdl/dllib/feature/dataset/dataset.py b/python/dllib/src/bigdl/dllib/feature/dataset/dataset.py index e27247110eb..86b59f45307 100644 --- a/python/dllib/src/bigdl/dllib/feature/dataset/dataset.py +++ b/python/dllib/src/bigdl/dllib/feature/dataset/dataset.py @@ -15,9 +15,9 @@ # import sys -from bigdl.utils.common import JavaValue -from bigdl.utils.common import callBigDlFunc -from bigdl.utils.common import * +from bigdl.dllib.utils.common import JavaValue +from bigdl.dllib.utils.common import callBigDlFunc +from bigdl.dllib.utils.common import * from bigdl.dllib.feature.transform.vision.image import * if sys.version >= '3': diff --git a/python/dllib/src/bigdl/dllib/feature/dataset/transformer.py b/python/dllib/src/bigdl/dllib/feature/dataset/transformer.py index 856aa02c5e6..0f6d1f1a561 100644 --- a/python/dllib/src/bigdl/dllib/feature/dataset/transformer.py +++ b/python/dllib/src/bigdl/dllib/feature/dataset/transformer.py @@ -15,7 +15,7 @@ # -from bigdl.utils.common import Sample +from bigdl.dllib.utils.common import Sample def normalizer(data, mean, std): diff --git a/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py b/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py index db166a15a01..19cd0fbaedb 100644 --- a/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py +++ b/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py @@ -15,9 +15,9 @@ # import sys -from bigdl.utils.common import JavaValue -from bigdl.utils.common import callBigDlFunc -from bigdl.utils.common import * +from bigdl.dllib.utils.common import JavaValue +from bigdl.dllib.utils.common import callBigDlFunc +from bigdl.dllib.utils.common import * if sys.version >= '3': long = int diff --git a/python/dllib/src/bigdl/dllib/keras/converter.py b/python/dllib/src/bigdl/dllib/keras/converter.py index 2cdf6f86fc9..b7659a2f1f7 100644 --- a/python/dllib/src/bigdl/dllib/keras/converter.py +++ b/python/dllib/src/bigdl/dllib/keras/converter.py @@ -16,8 +16,8 @@ import numpy as np import bigdl.dllib.nn.layer as BLayer -import bigdl.utils.common as BCommon -from bigdl.utils.common import get_activation_by_name +import bigdl.dllib.utils.common as BCommon +from bigdl.dllib.utils.common import get_activation_by_name from keras.models import model_from_json from keras.models import Sequential, Model, Layer import keras diff --git a/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py b/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py index 4125edbc46e..dc0c25c73f3 100644 --- a/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py +++ b/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py @@ -20,7 +20,7 @@ from bigdl.dllib.nn.layer import * from bigdl.dllib.nn.criterion import * from bigdl.dllib.optim.optimizer import * -from bigdl.utils.common import * +from bigdl.dllib.utils.common import * def build_model(class_num): diff --git a/python/dllib/src/bigdl/dllib/models/local_lenet/local_lenet.py b/python/dllib/src/bigdl/dllib/models/local_lenet/local_lenet.py index fee0dec22c1..250c18ea6d4 100644 --- a/python/dllib/src/bigdl/dllib/models/local_lenet/local_lenet.py +++ b/python/dllib/src/bigdl/dllib/models/local_lenet/local_lenet.py @@ -19,7 +19,7 @@ from bigdl.dllib.models.lenet.lenet5 import build_model from bigdl.dllib.nn.criterion import * from bigdl.dllib.optim.optimizer import * -from bigdl.utils.common import * +from bigdl.dllib.utils.common import * def get_mnist(data_type="train", location="/tmp/mnist"): diff --git a/python/dllib/src/bigdl/dllib/models/ml_pipeline/dl_classifier.py b/python/dllib/src/bigdl/dllib/models/ml_pipeline/dl_classifier.py index 89c6cea013a..f998e5a8cd7 100644 --- a/python/dllib/src/bigdl/dllib/models/ml_pipeline/dl_classifier.py +++ b/python/dllib/src/bigdl/dllib/models/ml_pipeline/dl_classifier.py @@ -1,6 +1,6 @@ from pyspark.ml.pipeline import Estimator, Model from pyspark.ml.param.shared import * -from bigdl.utils.common import * +from bigdl.dllib.utils.common import * import sys if sys.version >= '3': diff --git a/python/dllib/src/bigdl/dllib/models/rnn/rnnexample.py b/python/dllib/src/bigdl/dllib/models/rnn/rnnexample.py index ebeef9eb2b6..f2f19a94c20 100644 --- a/python/dllib/src/bigdl/dllib/models/rnn/rnnexample.py +++ b/python/dllib/src/bigdl/dllib/models/rnn/rnnexample.py @@ -24,8 +24,8 @@ from bigdl.dllib.nn.layer import * from bigdl.dllib.nn.criterion import * from bigdl.dllib.optim.optimizer import * -from bigdl.utils.common import * -from bigdl.utils.common import Sample +from bigdl.dllib.utils.common import * +from bigdl.dllib.utils.common import Sample def download_data(dest_dir): TINYSHAKESPEARE_URL = 'https://raw.githubusercontent.com/karpathy/char-rnn/master/data/tinyshakespeare/input.txt' # noqa diff --git a/python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py b/python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py index 8dcca08c312..578b6bafa58 100644 --- a/python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py +++ b/python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py @@ -23,8 +23,8 @@ from bigdl.dllib.nn.layer import * from bigdl.dllib.nn.criterion import * from bigdl.dllib.optim.optimizer import * -from bigdl.utils.common import * -from bigdl.utils.common import Sample +from bigdl.dllib.utils.common import * +from bigdl.dllib.utils.common import Sample import datetime as dt diff --git a/python/dllib/src/bigdl/dllib/nn/criterion.py b/python/dllib/src/bigdl/dllib/nn/criterion.py index 2555a060a8b..642637f1689 100644 --- a/python/dllib/src/bigdl/dllib/nn/criterion.py +++ b/python/dllib/src/bigdl/dllib/nn/criterion.py @@ -17,9 +17,9 @@ import sys -from bigdl.utils.common import JavaValue -from bigdl.utils.common import callBigDlFunc -from bigdl.utils.common import JTensor +from bigdl.dllib.utils.common import JavaValue +from bigdl.dllib.utils.common import callBigDlFunc +from bigdl.dllib.utils.common import JTensor from bigdl.dllib.nn.layer import Layer import numpy as np @@ -979,8 +979,8 @@ def _test(): import doctest from pyspark import SparkContext from bigdl.dllib.nn import criterion - from bigdl.utils.common import init_engine - from bigdl.utils.common import create_spark_conf + from bigdl.dllib.utils.common import init_engine + from bigdl.dllib.utils.common import create_spark_conf globs = criterion.__dict__.copy() sc = SparkContext(master="local[4]", appName="test criterion", conf=create_spark_conf()) diff --git a/python/dllib/src/bigdl/dllib/nn/initialization_method.py b/python/dllib/src/bigdl/dllib/nn/initialization_method.py index 4f32027fbe2..79762f18444 100644 --- a/python/dllib/src/bigdl/dllib/nn/initialization_method.py +++ b/python/dllib/src/bigdl/dllib/nn/initialization_method.py @@ -16,7 +16,7 @@ import sys -from bigdl.utils.common import JavaValue +from bigdl.dllib.utils.common import JavaValue if sys.version >= '3': long = int diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 1a712e51f39..6e5d200d43f 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -21,14 +21,14 @@ import numpy as np import six -from bigdl.utils.common import JTensor -from bigdl.utils.common import JavaValue -from bigdl.utils.common import callBigDlFunc -from bigdl.utils.common import callJavaFunc -from bigdl.utils.common import get_spark_context -from bigdl.utils.common import to_list -from bigdl.utils.common import INTMAX, INTMIN, DOUBLEMAX -from bigdl.utils.common import get_activation_by_name +from bigdl.dllib.utils.common import JTensor +from bigdl.dllib.utils.common import JavaValue +from bigdl.dllib.utils.common import callBigDlFunc +from bigdl.dllib.utils.common import callJavaFunc +from bigdl.dllib.utils.common import get_spark_context +from bigdl.dllib.utils.common import to_list +from bigdl.dllib.utils.common import INTMAX, INTMIN, DOUBLEMAX +from bigdl.dllib.utils.common import get_activation_by_name from bigdl.dllib.optim.optimizer import L1Regularizer, L2Regularizer, L1L2Regularizer from py4j.java_gateway import JavaObject from pyspark.rdd import RDD @@ -745,7 +745,7 @@ def __init__(self, bigdl_type, "createModelPreprocessor", inputs, outputs) self.bigdl_type = bigdl_type else: - from bigdl.utils.tf_utils import convert + from bigdl.dllib.utils.tf_utils import convert model = convert(to_list(inputs), to_list(outputs), byte_order, bigdl_type) super(Model, self).__init__(model.value, bigdl_type) @@ -863,8 +863,8 @@ def load_tensorflow(path, inputs, outputs, byte_order = "little_endian", @staticmethod def train(output, data, label, opt_method, criterion, batch_size, end_when, session=None, bigdl_type="float"): - from bigdl.utils.tf_utils import get_path - from bigdl.utils.common import Sample + from bigdl.dllib.utils.tf_utils import get_path + from bigdl.dllib.utils.common import Sample output_name = output.name.split(":")[0] path = get_path(output_name, session) sc = get_spark_context() @@ -5784,8 +5784,8 @@ def _test(): import doctest from pyspark import SparkContext from bigdl.dllib.nn import layer - from bigdl.utils.common import init_engine - from bigdl.utils.common import create_spark_conf + from bigdl.dllib.utils.common import init_engine + from bigdl.dllib.utils.common import create_spark_conf globs = layer.__dict__.copy() sc = SparkContext(master="local[4]", appName="test layer", conf=create_spark_conf()) diff --git a/python/dllib/src/bigdl/dllib/nn/onnx/layer.py b/python/dllib/src/bigdl/dllib/nn/onnx/layer.py index f2e255f5e31..a907f5501c9 100644 --- a/python/dllib/src/bigdl/dllib/nn/onnx/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/onnx/layer.py @@ -17,7 +17,7 @@ import sys import numpy as np from bigdl.dllib.nn.layer import Layer -from bigdl.utils.common import JTensor +from bigdl.dllib.utils.common import JTensor if sys.version >= '3': long = int diff --git a/python/dllib/src/bigdl/dllib/optim/optimizer.py b/python/dllib/src/bigdl/dllib/optim/optimizer.py index 2786272c993..af0ddfa3bb6 100644 --- a/python/dllib/src/bigdl/dllib/optim/optimizer.py +++ b/python/dllib/src/bigdl/dllib/optim/optimizer.py @@ -23,14 +23,14 @@ from py4j.java_gateway import JavaObject from pyspark.rdd import RDD -from bigdl.utils.common import DOUBLEMAX -from bigdl.utils.common import JTensor -from bigdl.utils.common import JavaValue -from bigdl.utils.common import callBigDlFunc -from bigdl.utils.common import callJavaFunc -from bigdl.utils.common import get_node_and_core_number -from bigdl.utils.common import init_engine -from bigdl.utils.common import to_list +from bigdl.dllib.utils.common import DOUBLEMAX +from bigdl.dllib.utils.common import JTensor +from bigdl.dllib.utils.common import JavaValue +from bigdl.dllib.utils.common import callBigDlFunc +from bigdl.dllib.utils.common import callJavaFunc +from bigdl.dllib.utils.common import get_node_and_core_number +from bigdl.dllib.utils.common import init_engine +from bigdl.dllib.utils.common import to_list from bigdl.dllib.feature.dataset.dataset import * import warnings @@ -1190,8 +1190,8 @@ def _test(): import doctest from pyspark import SparkContext from bigdl.dllib.optim import optimizer - from bigdl.utils.common import init_engine - from bigdl.utils.common import create_spark_conf + from bigdl.dllib.utils.common import init_engine + from bigdl.dllib.utils.common import create_spark_conf globs = optimizer.__dict__.copy() sc = SparkContext(master="local[4]", appName="test optimizer", conf=create_spark_conf()) diff --git a/python/dllib/src/bigdl/dllib/utils/common.py b/python/dllib/src/bigdl/dllib/utils/common.py index af7a03cb89f..ac2d1844984 100644 --- a/python/dllib/src/bigdl/dllib/utils/common.py +++ b/python/dllib/src/bigdl/dllib/utils/common.py @@ -32,7 +32,7 @@ import threading import tempfile import traceback -from bigdl.utils.engine import get_bigdl_classpath, is_spark_below_2_2 +from bigdl.dllib.utils.engine import get_bigdl_classpath, is_spark_below_2_2 INTMAX = 2147483647 INTMIN = -2147483648 @@ -154,7 +154,7 @@ class JTensor(object): A wrapper to easy our work when need to pass or return Tensor to/from Scala. >>> import numpy as np - >>> from bigdl.utils.common import JTensor + >>> from bigdl.dllib.utils.common import JTensor >>> np.random.seed(123) >>> """ @@ -189,8 +189,8 @@ def from_ndarray(cls, a_ndarray, bigdl_type="float"): Convert a ndarray to a DenseTensor which would be used in Java side. >>> import numpy as np - >>> from bigdl.utils.common import JTensor - >>> from bigdl.utils.common import callBigDlFunc + >>> from bigdl.dllib.utils.common import JTensor + >>> from bigdl.dllib.utils.common import callBigDlFunc >>> np.random.seed(123) >>> data = np.random.uniform(0, 1, (2, 3)).astype("float32") >>> result = JTensor.from_ndarray(data) @@ -236,8 +236,8 @@ def sparse(cls, a_ndarray, i_ndarray, shape, bigdl_type="float"): :param shape shape as a DenseTensor. >>> import numpy as np - >>> from bigdl.utils.common import JTensor - >>> from bigdl.utils.common import callBigDlFunc + >>> from bigdl.dllib.utils.common import JTensor + >>> from bigdl.dllib.utils.common import callBigDlFunc >>> np.random.seed(123) >>> data = np.arange(1, 7).astype("float32") >>> indices = np.arange(1, 7) @@ -314,7 +314,7 @@ def from_ndarray(cls, features, labels, bigdl_type="float"): :param bigdl_type: "double" or "float" >>> import numpy as np - >>> from bigdl.utils.common import callBigDlFunc + >>> from bigdl.dllib.utils.common import callBigDlFunc >>> from numpy.testing import assert_allclose >>> np.random.seed(123) >>> sample = Sample.from_ndarray(np.random.random((2,3)), np.random.random((2,3))) @@ -495,7 +495,7 @@ def to_sample_rdd(x, y, numSlices=None): :return: """ sc = get_spark_context() - from bigdl.utils.common import Sample + from bigdl.dllib.utils.common import Sample x_rdd = sc.parallelize(x, numSlices) y_rdd = sc.parallelize(y, numSlices) return x_rdd.zip(y_rdd).map(lambda item: Sample.from_ndarray(item[0], item[1])) diff --git a/python/dllib/src/bigdl/dllib/utils/tf_utils.py b/python/dllib/src/bigdl/dllib/utils/tf_utils.py index 58abc2695bd..72b348c2338 100644 --- a/python/dllib/src/bigdl/dllib/utils/tf_utils.py +++ b/python/dllib/src/bigdl/dllib/utils/tf_utils.py @@ -27,8 +27,8 @@ from tensorflow.python.framework import importer from tensorflow.python.platform import gfile from bigdl.dllib.nn.layer import Model -from bigdl.utils.common import JTensor -from bigdl.utils.common import callBigDlFunc +from bigdl.dllib.utils.common import JTensor +from bigdl.dllib.utils.common import callBigDlFunc import os def get_path(output_name, sess=None): diff --git a/python/dllib/src/bigdl/utils/__init__.py b/python/dllib/src/bigdl/utils/__init__.py deleted file mode 100644 index 2151a805423..00000000000 --- a/python/dllib/src/bigdl/utils/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -# -# Copyright 2016 The BigDL Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# diff --git a/python/dllib/test/bigdl/caffe/test_caffe_layers.py b/python/dllib/test/bigdl/caffe/test_caffe_layers.py index 640a6f8efc3..537dbc03e41 100644 --- a/python/dllib/test/bigdl/caffe/test_caffe_layers.py +++ b/python/dllib/test/bigdl/caffe/test_caffe_layers.py @@ -16,7 +16,7 @@ from bigdl.dllib.nn.layer import * from bigdl.dllib.optim.optimizer import * -from bigdl.utils.common import * +from bigdl.dllib.utils.common import * import numpy as np import pytest import tempfile diff --git a/python/dllib/test/bigdl/caffe/test_load_caffe.py b/python/dllib/test/bigdl/caffe/test_load_caffe.py index d71039fb948..c5c3760b88e 100644 --- a/python/dllib/test/bigdl/caffe/test_load_caffe.py +++ b/python/dllib/test/bigdl/caffe/test_load_caffe.py @@ -16,7 +16,7 @@ from bigdl.dllib.nn.layer import * from bigdl.dllib.optim.optimizer import * -from bigdl.utils.common import * +from bigdl.dllib.utils.common import * import numpy as np import pytest from numpy.testing import assert_allclose diff --git a/python/dllib/test/bigdl/keras/test_backend.py b/python/dllib/test/bigdl/keras/test_backend.py index 97d5c53bf63..7df21569750 100644 --- a/python/dllib/test/bigdl/keras/test_backend.py +++ b/python/dllib/test/bigdl/keras/test_backend.py @@ -73,8 +73,8 @@ def test_lenet_distributed_ndarray(self): def test_lenet_distributed_rdd(self): kmodel, X_train, y_train = TestModels.kmodel_seq_lenet_mnist() sc = get_spark_context() - from bigdl.utils.common import Sample - from bigdl.utils.common import to_sample_rdd + from bigdl.dllib.utils.common import Sample + from bigdl.dllib.utils.common import to_sample_rdd training_rdd = to_sample_rdd(X_train, y_train) self.modelTest(X_train, kmodel, dump_weights=True) diff --git a/python/dllib/test/bigdl/test_engine_env.py b/python/dllib/test/bigdl/test_engine_env.py index dc824267356..ee0a889d524 100644 --- a/python/dllib/test/bigdl/test_engine_env.py +++ b/python/dllib/test/bigdl/test_engine_env.py @@ -16,7 +16,7 @@ import pytest import os -from bigdl.utils.common import * +from bigdl.dllib.utils.common import * class TestEngineEnv(): @@ -38,7 +38,7 @@ def test___prepare_bigdl_env(self): # adding jar path message, just do prepare_env()' again # to see if the log is correct and the environment variables should not vary. - from bigdl.utils.engine import prepare_env + from bigdl.dllib.utils.engine import prepare_env bigdl_jars_env_1 = os.environ.get("BIGDL_JARS", None) spark_class_path_1 = os.environ.get("SPARK_CLASSPATH", None) diff --git a/python/dllib/test/bigdl/test_pickler.py b/python/dllib/test/bigdl/test_pickler.py index 69e5f92cafa..242fbdfd072 100644 --- a/python/dllib/test/bigdl/test_pickler.py +++ b/python/dllib/test/bigdl/test_pickler.py @@ -18,15 +18,15 @@ from bigdl.dllib.nn.initialization_method import * from bigdl.dllib.nn.criterion import * from bigdl.dllib.optim.optimizer import * -from bigdl.utils.common import * -from bigdl.utils.common import _py2java +from bigdl.dllib.utils.common import * +from bigdl.dllib.utils.common import _py2java from bigdl.dllib.nn.initialization_method import * from bigdl.dllib.feature.dataset import movielens import numpy as np import tempfile import pytest from numpy.testing import assert_allclose, assert_array_equal -from bigdl.utils.engine import compare_version +from bigdl.dllib.utils.engine import compare_version np.random.seed(1337) # for reproducibility diff --git a/python/dllib/test/bigdl/test_simple_integration.py b/python/dllib/test/bigdl/test_simple_integration.py index 981a28e8b72..ff0ce17abb1 100644 --- a/python/dllib/test/bigdl/test_simple_integration.py +++ b/python/dllib/test/bigdl/test_simple_integration.py @@ -18,15 +18,15 @@ from bigdl.dllib.nn.initialization_method import * from bigdl.dllib.nn.criterion import * from bigdl.dllib.optim.optimizer import * -from bigdl.utils.common import * -from bigdl.utils.common import _py2java +from bigdl.dllib.utils.common import * +from bigdl.dllib.utils.common import _py2java from bigdl.dllib.nn.initialization_method import * from bigdl.dllib.feature.dataset import movielens import numpy as np import tempfile import pytest from numpy.testing import assert_allclose, assert_array_equal -from bigdl.utils.engine import compare_version +from bigdl.dllib.utils.engine import compare_version from bigdl.dllib.feature.transform.vision.image import * from bigdl.dllib.models.utils.model_broadcast import broadcast_model from bigdl.dllib.feature.dataset.dataset import * @@ -244,7 +244,7 @@ def test_graph_preprocessor(self): np.array([1.0])) def test_load_zip_conf(self): - from bigdl.utils.common import get_bigdl_conf + from bigdl.dllib.utils.common import get_bigdl_conf import sys sys_path_back = sys.path sys.path = [path for path in sys.path if "spark-bigdl.conf" not in path] diff --git a/python/dllib/test/bigdl/test_tensorflow.py b/python/dllib/test/bigdl/test_tensorflow.py index 2d8e3f1918f..ba77e399b00 100644 --- a/python/dllib/test/bigdl/test_tensorflow.py +++ b/python/dllib/test/bigdl/test_tensorflow.py @@ -16,7 +16,7 @@ from bigdl.dllib.nn.layer import * from bigdl.dllib.optim.optimizer import * -from bigdl.utils.common import * +from bigdl.dllib.utils.common import * import numpy as np import unittest import shutil diff --git a/python/dllib/test/bigdl/test_utils.py b/python/dllib/test/bigdl/test_utils.py index a96d6dee25a..687277649eb 100644 --- a/python/dllib/test/bigdl/test_utils.py +++ b/python/dllib/test/bigdl/test_utils.py @@ -21,7 +21,7 @@ from keras.layers.convolutional import * from keras.layers import Dense, Dropout, Input from keras.optimizers import RMSprop -from bigdl.utils.common import * +from bigdl.dllib.utils.common import * from bigdl.dllib.keras.converter import * from bigdl.dllib.keras.converter import WeightLoader, WeightsConverter import numpy as np diff --git a/python/dllib/test/bigdl/transform/test_image.py b/python/dllib/test/bigdl/transform/test_image.py index bf25db39d29..9c178cff991 100644 --- a/python/dllib/test/bigdl/transform/test_image.py +++ b/python/dllib/test/bigdl/transform/test_image.py @@ -16,7 +16,7 @@ import pytest import os -from bigdl.utils.common import * +from bigdl.dllib.utils.common import * from bigdl.dllib.feature.transform.vision.image import * import tempfile diff --git a/scala/common/utils/pom.xml b/scala/common/utils/pom.xml deleted file mode 100644 index 317974e99d9..00000000000 --- a/scala/common/utils/pom.xml +++ /dev/null @@ -1,247 +0,0 @@ - - - - bigdl-parent - com.intel.analytics.bigdl - 2.0.0-SNAPSHOT - ../../pom.xml - - 4.0.0 - - bigdl-utils - jar - - - 1.1.1 - 1.1.1 - true - false - com.intel.analytics.bigdl.tags.Integration - - - - - org.apache.hadoop - hadoop-client - ${spark-scope} - - - org.apache.spark - spark-core_${scala.major.version} - ${spark.version} - ${spark-scope} - - - com.intel.analytics.bigdl.core.dist - all - 0.14.0-SNAPSHOT - ${bigdl-core-all-scope} - - - org.apache.logging.log4j - log4j-api - ${log4j.version} - - - org.apache.logging.log4j - log4j-core - ${log4j.version} - - - org.scalatest - scalatest_${scala.major.version} - test - - - - - - - org.scalatest - scalatest-maven-plugin - 1.0 - - ${project.build.directory}/surefire-reports - . - bigdl-test-report.txt - ${runSpecsInParallel} - - true - - -Xmx6g -XX:MaxPermSize=1g - - - - test - - test - - - - - - - - org.apache.maven.plugins - maven-shade-plugin - 3.2.1 - - - - com.google.protobuf - - META-INF/maven/com.google.protobuf/protobuf-java/* - - - - - - com.google.protobuf - com.intel.analytics.shaded.protobuf_v_3_5_1 - - - - - - shade - package - - shade - - - - - com.google.protobuf - - - - - - without-spark - package - - shade - - - true - jar-with-dependencies - - - - - - - - - - - - - - - - - - - - - - - - - org.scoverage - scoverage-maven-plugin - ${scoverage.plugin.version} - - true - true - - - - - - org.scalastyle - scalastyle-maven-plugin - 0.8.0 - - false - true - true - false - ${basedir}/src/main/scala - ${basedir}/src/test/scala - ${project.parent.basedir}/scalastyle_config.xml - ${project.build.directory}/stylecheck/scalastyle-output.xml - UTF-8 - UTF-8 - - - - - check - - - - - - - - - - - org.scoverage - scoverage-maven-plugin - ${scoverage.plugin.version} - - - - report - - - - - - - - - - per_platform - - provided - - - - com.intel.analytics.bigdl.core.dist - ${os-flag} - 0.14.0-SNAPSHOT - pom - - - - - - parallel-tests - - true - com.intel.analytics.bigdl.tags.Parallel - com.intel.analytics.bigdl.tags.Integration - - - - serial-tests - - false - com.intel.analytics.bigdl.tags.Parallel,com.intel.analytics.bigdl.tags.Integration - - - - integration-test - - com.intel.analytics.bigdl.tags.Serial,com.intel.analytics.bigdl.tags.Parallel - - - - diff --git a/scala/common/utils/src/main/resources/spark-bigdl.conf b/scala/common/utils/src/main/resources/spark-bigdl.conf deleted file mode 100644 index 0e7737ba6d9..00000000000 --- a/scala/common/utils/src/main/resources/spark-bigdl.conf +++ /dev/null @@ -1,33 +0,0 @@ -# -# Copyright 2016 The BigDL Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# -# Spark confs for BigDL. -# -# If you can init SparkContext in your application, you should call Engine.createSparkConf to get a -# SparkConf. The properties will be set. You neednot pass these properties. -# -# If you use spark-shell or pyspark, you need to pass these properties in. You can put them -# in your spark conf file. -# -# For more details, please refer -# https://bigdl-project.github.io/master/#APIGuide/Engine/ -# - -spark.shuffle.reduceLocality.enabled false -spark.shuffle.blockTransferService nio -spark.scheduler.minRegisteredResourcesRatio 1.0 -spark.speculation false diff --git a/scala/common/utils/src/test/java/com/intel/analytics/bigdl/tags/Integration.java b/scala/common/utils/src/test/java/com/intel/analytics/bigdl/tags/Integration.java deleted file mode 100644 index 62e7a0bc3f9..00000000000 --- a/scala/common/utils/src/test/java/com/intel/analytics/bigdl/tags/Integration.java +++ /dev/null @@ -1,14 +0,0 @@ -package com.intel.analytics.bigdl.tags; - -import java.lang.annotation.Retention; -import java.lang.annotation.Target; - -import static java.lang.annotation.ElementType.METHOD; -import static java.lang.annotation.ElementType.TYPE; -import static java.lang.annotation.RetentionPolicy.RUNTIME; - -@org.scalatest.TagAnnotation -@Retention(RUNTIME) -@Target({METHOD, TYPE}) -public @interface Integration { -} diff --git a/scala/common/utils/src/test/java/com/intel/analytics/bigdl/tags/Parallel.java b/scala/common/utils/src/test/java/com/intel/analytics/bigdl/tags/Parallel.java deleted file mode 100644 index 055a35e125e..00000000000 --- a/scala/common/utils/src/test/java/com/intel/analytics/bigdl/tags/Parallel.java +++ /dev/null @@ -1,14 +0,0 @@ -package com.intel.analytics.bigdl.tags; - -import java.lang.annotation.Retention; -import java.lang.annotation.Target; - -import static java.lang.annotation.ElementType.METHOD; -import static java.lang.annotation.ElementType.TYPE; -import static java.lang.annotation.RetentionPolicy.RUNTIME; - -@org.scalatest.TagAnnotation -@Retention(RUNTIME) -@Target({METHOD, TYPE}) -public @interface Parallel { -} diff --git a/scala/common/utils/src/test/java/com/intel/analytics/bigdl/tags/Serial.java b/scala/common/utils/src/test/java/com/intel/analytics/bigdl/tags/Serial.java deleted file mode 100644 index acca2825be8..00000000000 --- a/scala/common/utils/src/test/java/com/intel/analytics/bigdl/tags/Serial.java +++ /dev/null @@ -1,14 +0,0 @@ -package com.intel.analytics.bigdl.tags; - -import java.lang.annotation.Retention; -import java.lang.annotation.Target; - -import static java.lang.annotation.ElementType.METHOD; -import static java.lang.annotation.ElementType.TYPE; -import static java.lang.annotation.RetentionPolicy.RUNTIME; - -@org.scalatest.TagAnnotation -@Retention(RUNTIME) -@Target({METHOD, TYPE}) -public @interface Serial { -} diff --git a/scala/dllib/pom.xml b/scala/dllib/pom.xml index 9c16fb73586..ff726e4477a 100644 --- a/scala/dllib/pom.xml +++ b/scala/dllib/pom.xml @@ -21,11 +21,6 @@ - - com.intel.analytics.bigdl - bigdl-utils - ${project.version} - org.apache.opennlp opennlp-tools diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/example/imageclassification/imageFrame/InceptionValidation.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/example/imageclassification/imageFrame/InceptionValidation.scala index db95575b07b..4802f161f84 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/example/imageclassification/imageFrame/InceptionValidation.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/example/imageclassification/imageFrame/InceptionValidation.scala @@ -22,7 +22,7 @@ import com.intel.analytics.bigdl.dllib.nn.Module import com.intel.analytics.bigdl.dllib.optim.{Top1Accuracy, Top5Accuracy} import com.intel.analytics.bigdl.dllib.feature.transform.vision.image.{ImageFrame, ImageFrameToSample, MatToTensor, PixelBytesToMat} import com.intel.analytics.bigdl.dllib.feature.transform.vision.image.augmentation.{CenterCrop, ChannelNormalize, Resize} -import com.intel.analytics.bigdl.utils.Engine +import com.intel.analytics.bigdl.dllib.utils.Engine import org.apache.spark.SparkContext import scopt.OptionParser diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/example/keras/Train.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/example/keras/Train.scala index 558facbb4d8..7e05b4cf67a 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/example/keras/Train.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/example/keras/Train.scala @@ -22,7 +22,7 @@ import com.intel.analytics.bigdl.dllib.feature.dataset.image.{BytesToGreyImg, Gr import com.intel.analytics.bigdl.dllib.nn.ClassNLLCriterion import com.intel.analytics.bigdl.numeric.NumericFloat import com.intel.analytics.bigdl.dllib.optim._ -import com.intel.analytics.bigdl.utils.Engine +import com.intel.analytics.bigdl.dllib.utils.Engine import com.intel.analytics.bigdl.dllib.models.lenet.LeNet5 import org.apache.spark.SparkContext diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/example/languagemodel/PTBModel.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/example/languagemodel/PTBModel.scala index 4c767a60e62..ac3ebfaa67a 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/example/languagemodel/PTBModel.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/example/languagemodel/PTBModel.scala @@ -20,7 +20,7 @@ import com.intel.analytics.bigdl.Module import com.intel.analytics.bigdl.mkl.Memory import com.intel.analytics.bigdl.dllib.nn.Graph._ import com.intel.analytics.bigdl.dllib.nn.{TimeDistributed, _} -import com.intel.analytics.bigdl.utils.{Engine, MklDnn} +import com.intel.analytics.bigdl.dllib.utils.{Engine, MklDnn} object PTBModel { def transformer( diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/example/languagemodel/PTBWordLM.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/example/languagemodel/PTBWordLM.scala index d99f6070f31..2fbd7c19d0b 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/example/languagemodel/PTBWordLM.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/example/languagemodel/PTBWordLM.scala @@ -22,7 +22,7 @@ import com.intel.analytics.bigdl.dllib.feature.dataset.{DataSet, SampleToMiniBat import com.intel.analytics.bigdl.dllib.nn.{CrossEntropyCriterion, Module, TimeDistributedCriterion} import com.intel.analytics.bigdl.dllib.optim._ import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric._ -import com.intel.analytics.bigdl.utils.{Engine, OptimizerV1, OptimizerV2} +import com.intel.analytics.bigdl.dllib.utils.{Engine, OptimizerV1, OptimizerV2} import org.apache.log4j.{Level, Logger} import org.apache.spark.SparkContext import com.intel.analytics.bigdl.dllib.example.languagemodel.Utils._ diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/example/lenetLocal/Predict.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/example/lenetLocal/Predict.scala index d35a1d463e3..d041a78e9be 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/example/lenetLocal/Predict.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/example/lenetLocal/Predict.scala @@ -17,7 +17,7 @@ package com.intel.analytics.bigdl.dllib.example.lenetLocal import com.intel.analytics.bigdl.dllib.feature.dataset.image.{BytesToGreyImg, GreyImgNormalizer, GreyImgToSample} import com.intel.analytics.bigdl.dllib.nn.Module -import com.intel.analytics.bigdl.utils.Engine +import com.intel.analytics.bigdl.dllib.utils.Engine import com.intel.analytics.bigdl.dllib.feature.dataset.Sample import com.intel.analytics.bigdl.dllib.optim.LocalPredictor import org.apache.log4j.{Level, Logger} diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/example/lenetLocal/Test.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/example/lenetLocal/Test.scala index 41eb6a19f44..ad5fcf20a24 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/example/lenetLocal/Test.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/example/lenetLocal/Test.scala @@ -20,7 +20,7 @@ import com.intel.analytics.bigdl.dllib.feature.dataset.{DataSet, SampleToBatch} import com.intel.analytics.bigdl.dllib.feature.dataset.image.{BytesToGreyImg, GreyImgNormalizer, GreyImgToSample} import com.intel.analytics.bigdl.dllib.nn.Module import com.intel.analytics.bigdl.dllib.optim.{Top1Accuracy, ValidationMethod} -import com.intel.analytics.bigdl.utils.Engine +import com.intel.analytics.bigdl.dllib.utils.Engine import org.apache.log4j.{Level, Logger} object Test { diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/example/lenetLocal/Train.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/example/lenetLocal/Train.scala index f8eb3c8eb76..a87973d1bf3 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/example/lenetLocal/Train.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/example/lenetLocal/Train.scala @@ -22,8 +22,8 @@ import com.intel.analytics.bigdl.dllib.feature.dataset.image.{BytesToGreyImg, Gr import com.intel.analytics.bigdl.dllib.nn.{ClassNLLCriterion, Module} import com.intel.analytics.bigdl.numeric.NumericFloat import com.intel.analytics.bigdl.dllib.optim._ -import com.intel.analytics.bigdl.utils.Engine -import com.intel.analytics.bigdl.utils.LoggerFilter +import com.intel.analytics.bigdl.dllib.utils.Engine +import com.intel.analytics.bigdl.dllib.utils.LoggerFilter import com.intel.analytics.bigdl.dllib.models.lenet.LeNet5 import org.apache.log4j.{Level, Logger} diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/example/loadmodel/ModelValidator.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/example/loadmodel/ModelValidator.scala index cebd8f07d72..f3c5a911c55 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/example/loadmodel/ModelValidator.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/example/loadmodel/ModelValidator.scala @@ -21,7 +21,7 @@ import java.nio.file.Paths import com.intel.analytics.bigdl.dllib.models.inception.Inception_v1_NoAuxClassifier import com.intel.analytics.bigdl.dllib.nn.Module import com.intel.analytics.bigdl.dllib.optim.{Top1Accuracy, Top5Accuracy, Validator} -import com.intel.analytics.bigdl.utils.Engine +import com.intel.analytics.bigdl.dllib.utils.Engine import org.apache.log4j.Logger import org.apache.spark.SparkContext import scopt.OptionParser diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/example/mkldnn/int8/GenerateInt8Scales.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/example/mkldnn/int8/GenerateInt8Scales.scala index 198575a8428..4de7e269e3d 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/example/mkldnn/int8/GenerateInt8Scales.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/example/mkldnn/int8/GenerateInt8Scales.scala @@ -19,7 +19,7 @@ package com.intel.analytics.bigdl.dllib.example.mkldnn.int8 import com.intel.analytics.bigdl.dllib.feature.dataset.{DataSet, MiniBatch} import com.intel.analytics.bigdl.dllib.models.resnet.ImageNetDataSet import com.intel.analytics.bigdl.dllib.nn.{Graph, Module} -import com.intel.analytics.bigdl.utils.Engine +import com.intel.analytics.bigdl.dllib.utils.Engine import org.apache.log4j.{Level, Logger} import org.apache.spark.SparkContext import org.apache.spark.rdd.RDD diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/example/mkldnn/int8/ImageNetInference.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/example/mkldnn/int8/ImageNetInference.scala index efc0675a67a..e56e76a8c11 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/example/mkldnn/int8/ImageNetInference.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/example/mkldnn/int8/ImageNetInference.scala @@ -21,7 +21,7 @@ import com.intel.analytics.bigdl.dllib.nn.Module import com.intel.analytics.bigdl.dllib.optim._ import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric._ import com.intel.analytics.bigdl.dllib.utils._ -import com.intel.analytics.bigdl.utils._ +import com.intel.analytics.bigdl.dllib.utils._ import org.apache.log4j.{Level, Logger} import org.apache.spark.SparkContext diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/example/tensorflow/transferlearning/TransferLearning.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/example/tensorflow/transferlearning/TransferLearning.scala index 6c345dcc6fe..6e80e7d632b 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/example/tensorflow/transferlearning/TransferLearning.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/example/tensorflow/transferlearning/TransferLearning.scala @@ -22,8 +22,8 @@ import com.intel.analytics.bigdl.dllib.feature.dataset.Sample import com.intel.analytics.bigdl.dllib.nn._ import com.intel.analytics.bigdl.dllib.optim._ import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.Engine -import com.intel.analytics.bigdl.utils.LoggerFilter +import com.intel.analytics.bigdl.dllib.utils.Engine +import com.intel.analytics.bigdl.dllib.utils.LoggerFilter import com.intel.analytics.bigdl.dllib.utils.tf.{BigDLSessionImpl, TensorflowLoader} import org.apache.spark.SparkContext import scopt.OptionParser diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/example/textclassification/TextClassifier.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/example/textclassification/TextClassifier.scala index 3cc1559ef79..4ef6a36b433 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/example/textclassification/TextClassifier.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/example/textclassification/TextClassifier.scala @@ -19,8 +19,8 @@ package com.intel.analytics.bigdl.dllib.example.textclassification import com.intel.analytics.bigdl.dllib.example.utils._ import com.intel.analytics.bigdl.dllib.nn.{ClassNLLCriterion, _} import com.intel.analytics.bigdl.dllib.utils.{T} -import com.intel.analytics.bigdl.utils.LoggerFilter -import com.intel.analytics.bigdl.utils.Engine +import com.intel.analytics.bigdl.dllib.utils.LoggerFilter +import com.intel.analytics.bigdl.dllib.utils.Engine import org.apache.log4j.{Level => Levle4j, Logger => Logger4j} import org.slf4j.{Logger, LoggerFactory} import scopt.OptionParser diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/example/treeLSTMSentiment/Train.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/example/treeLSTMSentiment/Train.scala index 9573324f51a..8cbf429f30d 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/example/treeLSTMSentiment/Train.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/example/treeLSTMSentiment/Train.scala @@ -24,8 +24,8 @@ import com.intel.analytics.bigdl.numeric.NumericFloat import com.intel.analytics.bigdl.dllib.optim._ import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.utils.{T} -import com.intel.analytics.bigdl.utils.{Engine, OptimizerV1, OptimizerV2} -import com.intel.analytics.bigdl.utils.LoggerFilter +import com.intel.analytics.bigdl.dllib.utils.{Engine, OptimizerV1, OptimizerV2} +import com.intel.analytics.bigdl.dllib.utils.LoggerFilter import org.apache.log4j.{Level => Levle4j, Logger => Logger4j} import org.apache.spark.SparkContext import org.slf4j.{Logger, LoggerFactory} diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/example/treeLSTMSentiment/Utils.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/example/treeLSTMSentiment/Utils.scala index 50a7d320fe6..334766be172 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/example/treeLSTMSentiment/Utils.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/example/treeLSTMSentiment/Utils.scala @@ -21,7 +21,7 @@ import com.intel.analytics.bigdl.dllib.example.utils.AbstractTextClassificationP import com.intel.analytics.bigdl.dllib.nn.TensorTree import com.intel.analytics.bigdl.numeric.NumericFloat import com.intel.analytics.bigdl.dllib.tensor.{Storage, Tensor} -import com.intel.analytics.bigdl.utils.RandomGenerator.RNG +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator.RNG import org.apache.spark.SparkContext import org.apache.spark.broadcast.Broadcast import org.apache.spark.rdd.RDD diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/example/udfpredictor/DataframePredictor.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/example/udfpredictor/DataframePredictor.scala index 9cdfc77f6f4..3339cc8c509 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/example/udfpredictor/DataframePredictor.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/example/udfpredictor/DataframePredictor.scala @@ -16,8 +16,8 @@ package com.intel.analytics.bigdl.dllib.example.udfpredictor import com.intel.analytics.bigdl.dllib.example.utils.WordMeta -import com.intel.analytics.bigdl.utils.Engine -import com.intel.analytics.bigdl.utils.LoggerFilter +import com.intel.analytics.bigdl.dllib.utils.Engine +import com.intel.analytics.bigdl.dllib.utils.LoggerFilter import org.apache.spark.SparkContext import org.apache.spark.sql.functions._ import org.apache.log4j.{Level, Logger} diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/example/utils/TextClassifier.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/example/utils/TextClassifier.scala index cf4992913ee..a8c56f7a25e 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/example/utils/TextClassifier.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/example/utils/TextClassifier.scala @@ -25,8 +25,8 @@ import com.intel.analytics.bigdl.dllib.nn.{ClassNLLCriterion, _} import com.intel.analytics.bigdl.dllib.optim._ import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.utils.{T} -import com.intel.analytics.bigdl.utils.{Engine} -import com.intel.analytics.bigdl.utils.LoggerFilter +import com.intel.analytics.bigdl.dllib.utils.{Engine} +import com.intel.analytics.bigdl.dllib.utils.LoggerFilter import com.intel.analytics.bigdl.dllib.example.utils.SimpleTokenizer._ import org.apache.spark.SparkContext import org.apache.spark.broadcast.Broadcast diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/feature/dataset/DataSet.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/feature/dataset/DataSet.scala index ec244e9ec03..f768e831099 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/feature/dataset/DataSet.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/feature/dataset/DataSet.scala @@ -26,8 +26,8 @@ import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.feature.transform.vision.image.label.roi.RoiLabel import com.intel.analytics.bigdl.dllib.feature.transform.vision.image.{DistributedImageFrame, ImageFeature, ImageFrame, LocalImageFrame, RoiImageInfo} import com.intel.analytics.bigdl.dllib.utils.{T} -import com.intel.analytics.bigdl.utils.Engine -import com.intel.analytics.bigdl.utils.RandomGenerator +import com.intel.analytics.bigdl.dllib.utils.Engine +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator import java.awt.Color import java.awt.image.{BufferedImage, DataBufferByte} import java.io.ByteArrayInputStream diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/feature/dataset/Utils.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/feature/dataset/Utils.scala index 7f40dd4da15..3404895833c 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/feature/dataset/Utils.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/feature/dataset/Utils.scala @@ -16,7 +16,7 @@ package com.intel.analytics.bigdl.dllib.feature.dataset -import com.intel.analytics.bigdl.utils.Engine +import com.intel.analytics.bigdl.dllib.utils.Engine import org.apache.log4j.Logger object Utils { diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/feature/dataset/image/BGRImgCropper.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/feature/dataset/image/BGRImgCropper.scala index d54db60f17b..b6a9a1be9ef 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/feature/dataset/image/BGRImgCropper.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/feature/dataset/image/BGRImgCropper.scala @@ -53,7 +53,7 @@ object BGRImgCropper { class BGRImgCropper(cropWidth: Int, cropHeight: Int, cropperMethod: CropperMethod = CropRandom) extends Transformer[LabeledBGRImage, LabeledBGRImage] { - import com.intel.analytics.bigdl.utils.RandomGenerator.RNG + import com.intel.analytics.bigdl.dllib.utils.RandomGenerator.RNG private val buffer = new LabeledBGRImage(cropWidth, cropHeight) diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/feature/dataset/image/BGRImgRdmCropper.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/feature/dataset/image/BGRImgRdmCropper.scala index 1b651a52779..7d21e18f00d 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/feature/dataset/image/BGRImgRdmCropper.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/feature/dataset/image/BGRImgRdmCropper.scala @@ -33,7 +33,7 @@ object BGRImgRdmCropper { */ class BGRImgRdmCropper(cropHeight: Int, cropWidth: Int, padding: Int) extends Transformer[LabeledBGRImage, LabeledBGRImage] { - import com.intel.analytics.bigdl.utils.RandomGenerator.RNG + import com.intel.analytics.bigdl.dllib.utils.RandomGenerator.RNG private val buffer = new LabeledBGRImage(cropWidth, cropHeight) diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/feature/dataset/image/ColorJitter.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/feature/dataset/image/ColorJitter.scala index 859d1084f5c..c5844337878 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/feature/dataset/image/ColorJitter.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/feature/dataset/image/ColorJitter.scala @@ -18,8 +18,8 @@ package com.intel.analytics.bigdl.dllib.feature.dataset.image import com.intel.analytics.bigdl.dllib.feature.dataset.Transformer import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.RandomGenerator -import com.intel.analytics.bigdl.utils.RandomGenerator.RNG +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator.RNG import scala.collection.Iterator import scala.util.Random diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/feature/dataset/image/GreyImgCropper.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/feature/dataset/image/GreyImgCropper.scala index 8966a924c97..1dd920a9781 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/feature/dataset/image/GreyImgCropper.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/feature/dataset/image/GreyImgCropper.scala @@ -35,7 +35,7 @@ object GreyImgCropper { class GreyImgCropper(cropWidth: Int, cropHeight: Int) extends Transformer[LabeledGreyImage, LabeledGreyImage] { - import com.intel.analytics.bigdl.utils.RandomGenerator.RNG + import com.intel.analytics.bigdl.dllib.utils.RandomGenerator.RNG private val buffer = new LabeledGreyImage(cropWidth, cropHeight) diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/feature/dataset/image/HFlip.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/feature/dataset/image/HFlip.scala index 962220af593..a91cb320d7d 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/feature/dataset/image/HFlip.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/feature/dataset/image/HFlip.scala @@ -17,7 +17,7 @@ package com.intel.analytics.bigdl.dllib.feature.dataset.image import com.intel.analytics.bigdl.dllib.feature.dataset.Transformer -import com.intel.analytics.bigdl.utils.RandomGenerator +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator import scala.collection.Iterator diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/feature/dataset/image/Lighting.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/feature/dataset/image/Lighting.scala index 0b718967568..a505265cf40 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/feature/dataset/image/Lighting.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/feature/dataset/image/Lighting.scala @@ -18,7 +18,7 @@ package com.intel.analytics.bigdl.dllib.feature.dataset.image import com.intel.analytics.bigdl.dllib.feature.dataset.Transformer import com.intel.analytics.bigdl.dllib.tensor.{Storage, Tensor} -import com.intel.analytics.bigdl.utils.RandomGenerator.RNG +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator.RNG import scala.collection.Iterator diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/feature/dataset/image/MTLabeledBGRImgToBatch.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/feature/dataset/image/MTLabeledBGRImgToBatch.scala index d570d201490..52e4719647a 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/feature/dataset/image/MTLabeledBGRImgToBatch.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/feature/dataset/image/MTLabeledBGRImgToBatch.scala @@ -20,7 +20,7 @@ import java.util.concurrent.atomic.AtomicInteger import com.intel.analytics.bigdl.dllib.feature.dataset.{Utils, MiniBatch, Transformer} import com.intel.analytics.bigdl.dllib.tensor.{Storage, Tensor} -import com.intel.analytics.bigdl.utils.Engine +import com.intel.analytics.bigdl.dllib.utils.Engine import scala.reflect.ClassTag object MTLabeledBGRImgToBatch { diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/feature/transform/vision/image/MTImageFeatureToBatch.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/feature/transform/vision/image/MTImageFeatureToBatch.scala index 3786daa8e4f..45486c2e6b5 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/feature/transform/vision/image/MTImageFeatureToBatch.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/feature/transform/vision/image/MTImageFeatureToBatch.scala @@ -23,7 +23,7 @@ import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric import com.intel.analytics.bigdl.dllib.tensor.{Storage, Tensor} import com.intel.analytics.bigdl.dllib.feature.transform.vision.image.label.roi.RoiLabel import com.intel.analytics.bigdl.dllib.utils.{T, Table} -import com.intel.analytics.bigdl.utils.Engine +import com.intel.analytics.bigdl.dllib.utils.Engine object MTImageFeatureToBatch { /** diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/feature/transform/vision/image/augmentation/Brightness.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/feature/transform/vision/image/augmentation/Brightness.scala index 046887197e1..ddb450fc8cb 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/feature/transform/vision/image/augmentation/Brightness.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/feature/transform/vision/image/augmentation/Brightness.scala @@ -18,7 +18,7 @@ package com.intel.analytics.bigdl.dllib.feature.transform.vision.image.augmentat import com.intel.analytics.bigdl.dllib.feature.transform.vision.image.opencv.OpenCVMat import com.intel.analytics.bigdl.dllib.feature.transform.vision.image.{FeatureTransformer, ImageFeature} -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ /** * adjust the image brightness diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/feature/transform/vision/image/augmentation/ColorJitter.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/feature/transform/vision/image/augmentation/ColorJitter.scala index 5a67cf77b2e..b6443af1174 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/feature/transform/vision/image/augmentation/ColorJitter.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/feature/transform/vision/image/augmentation/ColorJitter.scala @@ -16,7 +16,7 @@ package com.intel.analytics.bigdl.dllib.feature.transform.vision.image.augmentation -import com.intel.analytics.bigdl.utils.RandomGenerator.RNG +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator.RNG import com.intel.analytics.bigdl.dllib.feature.transform.vision.image.{FeatureTransformer, ImageFeature} import scala.util.Random diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/feature/transform/vision/image/augmentation/Contrast.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/feature/transform/vision/image/augmentation/Contrast.scala index 3a2aed4ef3a..91baf78f22f 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/feature/transform/vision/image/augmentation/Contrast.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/feature/transform/vision/image/augmentation/Contrast.scala @@ -18,7 +18,7 @@ package com.intel.analytics.bigdl.dllib.feature.transform.vision.image.augmentat import com.intel.analytics.bigdl.dllib.feature.transform.vision.image.opencv.OpenCVMat import com.intel.analytics.bigdl.dllib.feature.transform.vision.image.{FeatureTransformer, ImageFeature} -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ /** * Adjust the image contrast diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/feature/transform/vision/image/augmentation/Crop.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/feature/transform/vision/image/augmentation/Crop.scala index 295fa777249..67c745d2104 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/feature/transform/vision/image/augmentation/Crop.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/feature/transform/vision/image/augmentation/Crop.scala @@ -20,7 +20,7 @@ import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.feature.transform.vision.image.{FeatureTransformer, ImageFeature} import com.intel.analytics.bigdl.dllib.feature.transform.vision.image.opencv.OpenCVMat import com.intel.analytics.bigdl.dllib.feature.transform.vision.image.util.{BboxUtil, BoundingBox} -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import org.opencv.core.Rect /** diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/feature/transform/vision/image/augmentation/Expand.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/feature/transform/vision/image/augmentation/Expand.scala index dcc01ed203d..3f3b0bd1df2 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/feature/transform/vision/image/augmentation/Expand.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/feature/transform/vision/image/augmentation/Expand.scala @@ -21,7 +21,7 @@ import java.util import com.intel.analytics.bigdl.dllib.feature.transform.vision.image.{FeatureTransformer, ImageFeature} import com.intel.analytics.bigdl.dllib.feature.transform.vision.image.opencv.OpenCVMat import com.intel.analytics.bigdl.dllib.feature.transform.vision.image.util.BoundingBox -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import org.opencv.core.{Core, Mat, Rect, Scalar} /** diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/feature/transform/vision/image/augmentation/Hue.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/feature/transform/vision/image/augmentation/Hue.scala index 6b7c7a990a1..7c6914ec58d 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/feature/transform/vision/image/augmentation/Hue.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/feature/transform/vision/image/augmentation/Hue.scala @@ -20,7 +20,7 @@ import java.util import com.intel.analytics.bigdl.dllib.feature.transform.vision.image.opencv.OpenCVMat import com.intel.analytics.bigdl.dllib.feature.transform.vision.image.{FeatureTransformer, ImageFeature} -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import org.opencv.core.{Core, Mat} import org.opencv.imgproc.Imgproc diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/feature/transform/vision/image/augmentation/RandomAlterAspect.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/feature/transform/vision/image/augmentation/RandomAlterAspect.scala index 4b613caa96a..08205f79eb8 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/feature/transform/vision/image/augmentation/RandomAlterAspect.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/feature/transform/vision/image/augmentation/RandomAlterAspect.scala @@ -58,7 +58,7 @@ class RandomAlterAspect(min_area_ratio: Float = 0.08f, cropLength: Int = 224) extends FeatureTransformer { - import com.intel.analytics.bigdl.utils.RandomGenerator.RNG + import com.intel.analytics.bigdl.dllib.utils.RandomGenerator.RNG @inline private def randRatio(min: Float, max: Float): Float = { diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/feature/transform/vision/image/augmentation/RandomCropper.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/feature/transform/vision/image/augmentation/RandomCropper.scala index 6ff92be2cb0..123c6900f9a 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/feature/transform/vision/image/augmentation/RandomCropper.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/feature/transform/vision/image/augmentation/RandomCropper.scala @@ -18,7 +18,7 @@ package com.intel.analytics.bigdl.dllib.feature.transform.vision.image.augmentat import com.intel.analytics.bigdl.dllib.feature.dataset.image.{CropCenter, CropRandom, CropperMethod} import com.intel.analytics.bigdl.dllib.feature.transform.vision.image.{FeatureTransformer, ImageFeature} import com.intel.analytics.bigdl.dllib.feature.transform.vision.image.opencv.OpenCVMat -import com.intel.analytics.bigdl.utils.RandomGenerator +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator import org.opencv.core.CvType object RandomCropper { @@ -40,7 +40,7 @@ class RandomCropper(cropWidth: Int, cropHeight: Int, channels: Int = 3) extends FeatureTransformer { - import com.intel.analytics.bigdl.utils.RandomGenerator.RNG + import com.intel.analytics.bigdl.dllib.utils.RandomGenerator.RNG val buffer = new Array[Float](cropWidth * cropHeight * channels) diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/feature/transform/vision/image/augmentation/RandomResize.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/feature/transform/vision/image/augmentation/RandomResize.scala index 262b62890f2..772e2f913b8 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/feature/transform/vision/image/augmentation/RandomResize.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/feature/transform/vision/image/augmentation/RandomResize.scala @@ -17,7 +17,7 @@ package com.intel.analytics.bigdl.dllib.feature.transform.vision.image.augmentation import com.intel.analytics.bigdl.dllib.feature.transform.vision.image.{FeatureTransformer, ImageFeature} -import com.intel.analytics.bigdl.utils.RandomGenerator +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator import org.opencv.imgproc.Imgproc object RandomResize { diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/feature/transform/vision/image/augmentation/RandomTransformer.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/feature/transform/vision/image/augmentation/RandomTransformer.scala index 8381e2c35b1..8cafe983e29 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/feature/transform/vision/image/augmentation/RandomTransformer.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/feature/transform/vision/image/augmentation/RandomTransformer.scala @@ -17,7 +17,7 @@ package com.intel.analytics.bigdl.dllib.feature.transform.vision.image.augmentation import com.intel.analytics.bigdl.dllib.feature.transform.vision.image.{FeatureTransformer, ImageFeature} -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ /** * It is a wrapper for transformers to control the transform probability diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/feature/transform/vision/image/augmentation/Saturation.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/feature/transform/vision/image/augmentation/Saturation.scala index ea7573e62c2..795432802cf 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/feature/transform/vision/image/augmentation/Saturation.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/feature/transform/vision/image/augmentation/Saturation.scala @@ -20,7 +20,7 @@ import java.util import com.intel.analytics.bigdl.dllib.feature.transform.vision.image.opencv.OpenCVMat import com.intel.analytics.bigdl.dllib.feature.transform.vision.image.{FeatureTransformer, ImageFeature} -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import org.opencv.core.{Core, Mat} import org.opencv.imgproc.Imgproc diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/feature/transform/vision/image/label/roi/BatchSampler.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/feature/transform/vision/image/label/roi/BatchSampler.scala index e6bff8c56cf..f6a30690b7d 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/feature/transform/vision/image/label/roi/BatchSampler.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/feature/transform/vision/image/label/roi/BatchSampler.scala @@ -18,7 +18,7 @@ package com.intel.analytics.bigdl.dllib.feature.transform.vision.image.label.roi import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.feature.transform.vision.image.util.{BboxUtil, BoundingBox} -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import scala.collection.mutable.ArrayBuffer diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/feature/transform/vision/image/label/roi/RandomSampler.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/feature/transform/vision/image/label/roi/RandomSampler.scala index a7e113d6900..efcfce83120 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/feature/transform/vision/image/label/roi/RandomSampler.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/feature/transform/vision/image/label/roi/RandomSampler.scala @@ -19,7 +19,7 @@ package com.intel.analytics.bigdl.dllib.feature.transform.vision.image.label.roi import com.intel.analytics.bigdl.dllib.feature.transform.vision.image.{FeatureTransformer, ImageFeature} import com.intel.analytics.bigdl.dllib.feature.transform.vision.image.augmentation.Crop import com.intel.analytics.bigdl.dllib.feature.transform.vision.image.util.{BoundingBox} -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import org.opencv.core.Mat import scala.collection.mutable.ArrayBuffer diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/Activation.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/Activation.scala index 3312fcf3643..2de73e0e447 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/Activation.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/Activation.scala @@ -19,7 +19,7 @@ package com.intel.analytics.bigdl.dllib.keras import com.intel.analytics.bigdl.dllib.nn.abstractnn.{AbstractModule, IdentityOutputShape} import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/AtrousConvolution1D.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/AtrousConvolution1D.scala index 2db217714ac..d5ad253ecd3 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/AtrousConvolution1D.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/AtrousConvolution1D.scala @@ -22,7 +22,7 @@ import com.intel.analytics.bigdl.dllib.nn.{SpatialDilatedConvolution, Squeeze, T import com.intel.analytics.bigdl.dllib.optim.Regularizer import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/AtrousConvolution2D.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/AtrousConvolution2D.scala index dcbdce30655..37a2c562750 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/AtrousConvolution2D.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/AtrousConvolution2D.scala @@ -21,7 +21,7 @@ import com.intel.analytics.bigdl.dllib.nn.{InitializationMethod, SpatialDilatedC import com.intel.analytics.bigdl.dllib.optim.Regularizer import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/AveragePooling1D.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/AveragePooling1D.scala index e5b65fcce9c..51c160bbd1c 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/AveragePooling1D.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/AveragePooling1D.scala @@ -21,7 +21,7 @@ import com.intel.analytics.bigdl.dllib.nn.{Sequential => TSequential} import com.intel.analytics.bigdl.dllib.nn.abstractnn.{AbstractModule, DataFormat} import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/AveragePooling2D.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/AveragePooling2D.scala index 8d68d448af2..633cb5c3a6f 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/AveragePooling2D.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/AveragePooling2D.scala @@ -20,7 +20,7 @@ import com.intel.analytics.bigdl.dllib.nn.SpatialAveragePooling import com.intel.analytics.bigdl.dllib.nn.abstractnn.{AbstractModule, DataFormat} import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/AveragePooling3D.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/AveragePooling3D.scala index e395f7eef69..41f8c1cdda1 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/AveragePooling3D.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/AveragePooling3D.scala @@ -20,7 +20,7 @@ import com.intel.analytics.bigdl.dllib.nn.VolumetricAveragePooling import com.intel.analytics.bigdl.dllib.nn.abstractnn.AbstractModule import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/BatchNormalization.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/BatchNormalization.scala index fc84faf5b02..0ce81380dbb 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/BatchNormalization.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/BatchNormalization.scala @@ -20,7 +20,7 @@ import com.intel.analytics.bigdl.dllib.nn._ import com.intel.analytics.bigdl.dllib.nn.abstractnn.{AbstractModule, DataFormat} import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/Bidirectional.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/Bidirectional.scala index ef043320cf7..2f172090395 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/Bidirectional.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/Bidirectional.scala @@ -21,7 +21,7 @@ import com.intel.analytics.bigdl.dllib.nn.abstractnn.AbstractModule import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric import com.intel.analytics.bigdl.dllib.utils.Table -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/ConvLSTM2D.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/ConvLSTM2D.scala index 45fe03eb076..53b0406a059 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/ConvLSTM2D.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/ConvLSTM2D.scala @@ -21,7 +21,7 @@ import com.intel.analytics.bigdl.dllib.nn.abstractnn.{AbstractModule, Activity, import com.intel.analytics.bigdl.dllib.optim.Regularizer import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/Convolution1D.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/Convolution1D.scala index 1e72866b076..00364d17629 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/Convolution1D.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/Convolution1D.scala @@ -22,7 +22,7 @@ import com.intel.analytics.bigdl.dllib.nn.abstractnn._ import com.intel.analytics.bigdl.dllib.optim.Regularizer import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/Convolution2D.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/Convolution2D.scala index 3310374efba..5b4cfe857b5 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/Convolution2D.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/Convolution2D.scala @@ -21,7 +21,7 @@ import com.intel.analytics.bigdl.dllib.nn._ import com.intel.analytics.bigdl.dllib.optim.Regularizer import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/Convolution3D.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/Convolution3D.scala index eb6a7880358..57de22cf6db 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/Convolution3D.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/Convolution3D.scala @@ -21,7 +21,7 @@ import com.intel.analytics.bigdl.dllib.nn.{InitializationMethod, VolumetricConvo import com.intel.analytics.bigdl.dllib.optim.Regularizer import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/Cropping1D.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/Cropping1D.scala index 6f20e06bd2e..c863cd10dc3 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/Cropping1D.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/Cropping1D.scala @@ -20,7 +20,7 @@ import com.intel.analytics.bigdl.dllib.nn.SpatialZeroPadding import com.intel.analytics.bigdl.dllib.nn.abstractnn.AbstractModule import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/Cropping2D.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/Cropping2D.scala index 1fd33426338..876d2edaa4b 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/Cropping2D.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/Cropping2D.scala @@ -19,7 +19,7 @@ package com.intel.analytics.bigdl.dllib.keras import com.intel.analytics.bigdl.dllib.nn.abstractnn.{AbstractModule, DataFormat} import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/Cropping3D.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/Cropping3D.scala index 7dbaef87417..565be19d536 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/Cropping3D.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/Cropping3D.scala @@ -19,7 +19,7 @@ package com.intel.analytics.bigdl.dllib.keras import com.intel.analytics.bigdl.dllib.nn.abstractnn.AbstractModule import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/Deconvolution2D.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/Deconvolution2D.scala index 210383bf57b..8597c59cc8a 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/Deconvolution2D.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/Deconvolution2D.scala @@ -21,7 +21,7 @@ import com.intel.analytics.bigdl.dllib.nn.{InitializationMethod, SpatialFullConv import com.intel.analytics.bigdl.dllib.optim.Regularizer import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/Dense.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/Dense.scala index a0665bb7cdb..b974be97810 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/Dense.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/Dense.scala @@ -21,7 +21,7 @@ import com.intel.analytics.bigdl.dllib.nn.abstractnn._ import com.intel.analytics.bigdl.dllib.optim.Regularizer import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/Dropout.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/Dropout.scala index eb66efa8bcc..d9f1a428ae2 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/Dropout.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/Dropout.scala @@ -19,7 +19,7 @@ package com.intel.analytics.bigdl.dllib.keras import com.intel.analytics.bigdl.dllib.nn.abstractnn.{AbstractModule, Activity, IdentityOutputShape} import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/ELU.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/ELU.scala index b5238415191..fbd9b759688 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/ELU.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/ELU.scala @@ -19,7 +19,7 @@ package com.intel.analytics.bigdl.dllib.keras import com.intel.analytics.bigdl.dllib.nn.abstractnn._ import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/Embedding.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/Embedding.scala index 7f686065837..e55ce93c79a 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/Embedding.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/Embedding.scala @@ -20,7 +20,7 @@ import com.intel.analytics.bigdl.dllib.nn.abstractnn.AbstractModule import com.intel.analytics.bigdl.dllib.optim.Regularizer import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import com.intel.analytics.bigdl.dllib.nn.{AddConstant, InitializationMethod, LookupTable, RandomUniform, Zeros, Sequential => TSequential} import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/Flatten.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/Flatten.scala index 66a6960f290..0b4f6ad2421 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/Flatten.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/Flatten.scala @@ -19,7 +19,7 @@ package com.intel.analytics.bigdl.dllib.keras import com.intel.analytics.bigdl.dllib.nn.abstractnn.AbstractModule import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/GRU.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/GRU.scala index 33afc6b49cf..c0c8f773dc3 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/GRU.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/GRU.scala @@ -21,7 +21,7 @@ import com.intel.analytics.bigdl.dllib.nn.abstractnn.{AbstractModule, TensorModu import com.intel.analytics.bigdl.dllib.optim.Regularizer import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/GaussianDropout.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/GaussianDropout.scala index dfb36a11795..81fc24aeeef 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/GaussianDropout.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/GaussianDropout.scala @@ -19,7 +19,7 @@ package com.intel.analytics.bigdl.dllib.keras import com.intel.analytics.bigdl.dllib.nn.abstractnn._ import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/GaussianNoise.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/GaussianNoise.scala index b7688983a21..108f42cca81 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/GaussianNoise.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/GaussianNoise.scala @@ -20,7 +20,7 @@ package com.intel.analytics.bigdl.dllib.keras import com.intel.analytics.bigdl.dllib.nn.abstractnn._ import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/GlobalAveragePooling1D.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/GlobalAveragePooling1D.scala index 763fdf4d66f..a0e77924b13 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/GlobalAveragePooling1D.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/GlobalAveragePooling1D.scala @@ -22,7 +22,7 @@ import com.intel.analytics.bigdl.dllib.nn.SpatialAveragePooling import com.intel.analytics.bigdl.dllib.nn.{Sequential => TSequential} import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/GlobalAveragePooling2D.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/GlobalAveragePooling2D.scala index 195cf8a4de9..989062c6e19 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/GlobalAveragePooling2D.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/GlobalAveragePooling2D.scala @@ -20,7 +20,7 @@ import com.intel.analytics.bigdl.dllib.nn.{SpatialAveragePooling, Squeeze, Seque import com.intel.analytics.bigdl.dllib.nn.abstractnn.{AbstractModule, DataFormat} import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/GlobalAveragePooling3D.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/GlobalAveragePooling3D.scala index bd79e2f3b04..7333d1c02f8 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/GlobalAveragePooling3D.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/GlobalAveragePooling3D.scala @@ -22,7 +22,7 @@ import com.intel.analytics.bigdl.dllib.nn.VolumetricAveragePooling import com.intel.analytics.bigdl.dllib.nn.{Sequential => TSequential} import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/GlobalMaxPooling1D.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/GlobalMaxPooling1D.scala index aed15a0642d..04725a548f7 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/GlobalMaxPooling1D.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/GlobalMaxPooling1D.scala @@ -22,7 +22,7 @@ import com.intel.analytics.bigdl.dllib.nn.SpatialMaxPooling import com.intel.analytics.bigdl.dllib.nn.{Sequential => TSequential} import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/GlobalMaxPooling2D.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/GlobalMaxPooling2D.scala index 79e5df665de..ed2a3cf01a9 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/GlobalMaxPooling2D.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/GlobalMaxPooling2D.scala @@ -20,7 +20,7 @@ import com.intel.analytics.bigdl.dllib.nn.abstractnn.{AbstractModule, DataFormat import com.intel.analytics.bigdl.dllib.nn.{SpatialMaxPooling, Squeeze, Sequential => TSequential} import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/GlobalMaxPooling3D.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/GlobalMaxPooling3D.scala index 4f174356108..b7c9dcd060d 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/GlobalMaxPooling3D.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/GlobalMaxPooling3D.scala @@ -22,7 +22,7 @@ import com.intel.analytics.bigdl.dllib.nn.VolumetricMaxPooling import com.intel.analytics.bigdl.dllib.nn.{Sequential => TSequential} import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/GlobalPooling1D.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/GlobalPooling1D.scala index 65c37ed68f1..930c6fca8e6 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/GlobalPooling1D.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/GlobalPooling1D.scala @@ -18,7 +18,7 @@ package com.intel.analytics.bigdl.dllib.keras import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/GlobalPooling2D.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/GlobalPooling2D.scala index f341f836cd8..2abd02741cb 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/GlobalPooling2D.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/GlobalPooling2D.scala @@ -19,7 +19,7 @@ package com.intel.analytics.bigdl.dllib.keras import com.intel.analytics.bigdl.dllib.nn.abstractnn.DataFormat import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/GlobalPooling3D.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/GlobalPooling3D.scala index b4a3f9ad28b..0f8e6e0507b 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/GlobalPooling3D.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/GlobalPooling3D.scala @@ -18,7 +18,7 @@ package com.intel.analytics.bigdl.dllib.keras import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/Highway.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/Highway.scala index 7405efd7c6c..5bc5571c94a 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/Highway.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/Highway.scala @@ -20,7 +20,7 @@ import com.intel.analytics.bigdl.dllib.nn.abstractnn.{AbstractModule, IdentityOu import com.intel.analytics.bigdl.dllib.optim.Regularizer import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/Input.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/Input.scala index e122d120c42..9004529cc56 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/Input.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/Input.scala @@ -20,7 +20,7 @@ import com.intel.analytics.bigdl.dllib.nn.{Input => TInput} import com.intel.analytics.bigdl.dllib.nn.Graph.ModuleNode import com.intel.analytics.bigdl.dllib.nn.abstractnn.{AbstractModule, Activity} import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import com.intel.analytics.bigdl.dllib.utils.Node import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/KerasLayer.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/KerasLayer.scala index fd2975fee66..bc0ed2eb5f6 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/KerasLayer.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/KerasLayer.scala @@ -26,7 +26,7 @@ import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric import com.intel.analytics.bigdl.dllib.utils.serializer._ import com.intel.analytics.bigdl.dllib.utils.serializer.converters.DataConverter -import com.intel.analytics.bigdl.utils.{MultiShape, Shape, SingleShape} +import com.intel.analytics.bigdl.dllib.utils.{MultiShape, Shape, SingleShape} import scala.collection.JavaConverters._ import scala.collection.mutable.ArrayBuffer diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/LSTM.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/LSTM.scala index 9e6a1985492..48e0c8ef2f3 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/LSTM.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/LSTM.scala @@ -21,7 +21,7 @@ import com.intel.analytics.bigdl.dllib.nn.abstractnn.{AbstractModule, TensorModu import com.intel.analytics.bigdl.dllib.optim.Regularizer import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/LeakyReLU.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/LeakyReLU.scala index 121e85e0d20..1577c61dbe2 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/LeakyReLU.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/LeakyReLU.scala @@ -19,7 +19,7 @@ package com.intel.analytics.bigdl.dllib.keras import com.intel.analytics.bigdl.dllib.nn.abstractnn._ import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/LocallyConnected1D.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/LocallyConnected1D.scala index f9c87878384..2ed08567c90 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/LocallyConnected1D.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/LocallyConnected1D.scala @@ -21,7 +21,7 @@ import com.intel.analytics.bigdl.dllib.nn.{Squeeze, Sequential => TSequential} import com.intel.analytics.bigdl.dllib.optim.Regularizer import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/LocallyConnected2D.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/LocallyConnected2D.scala index 7556275ab37..f02d02d1e0a 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/LocallyConnected2D.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/LocallyConnected2D.scala @@ -21,7 +21,7 @@ import com.intel.analytics.bigdl.dllib.nn.{Container => TContainer, LocallyConne import com.intel.analytics.bigdl.dllib.optim.Regularizer import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/Masking.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/Masking.scala index beb66bbc9be..989d7bf0d9a 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/Masking.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/Masking.scala @@ -19,7 +19,7 @@ package com.intel.analytics.bigdl.dllib.keras import com.intel.analytics.bigdl.dllib.nn.abstractnn._ import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/MaxPooling1D.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/MaxPooling1D.scala index 000450b85bb..ee53543dfd6 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/MaxPooling1D.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/MaxPooling1D.scala @@ -21,7 +21,7 @@ import com.intel.analytics.bigdl.dllib.nn.{Sequential => TSequential} import com.intel.analytics.bigdl.dllib.nn.abstractnn.{AbstractModule, DataFormat} import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/MaxPooling2D.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/MaxPooling2D.scala index 35be1504ce7..d0b10e417f5 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/MaxPooling2D.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/MaxPooling2D.scala @@ -20,7 +20,7 @@ import com.intel.analytics.bigdl.dllib.nn.SpatialMaxPooling import com.intel.analytics.bigdl.dllib.nn.abstractnn.{AbstractModule, Activity, DataFormat} import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/MaxPooling3D.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/MaxPooling3D.scala index 8f2f076c1f5..733f35afb90 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/MaxPooling3D.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/MaxPooling3D.scala @@ -20,7 +20,7 @@ import com.intel.analytics.bigdl.dllib.nn.VolumetricMaxPooling import com.intel.analytics.bigdl.dllib.nn.abstractnn.AbstractModule import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/MaxoutDense.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/MaxoutDense.scala index 294852ca488..68f91a86f90 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/MaxoutDense.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/MaxoutDense.scala @@ -21,7 +21,7 @@ import com.intel.analytics.bigdl.dllib.nn.Maxout import com.intel.analytics.bigdl.dllib.optim.Regularizer import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/Merge.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/Merge.scala index 19529e49cd4..edc16ba84fa 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/Merge.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/Merge.scala @@ -21,7 +21,7 @@ import com.intel.analytics.bigdl.dllib.nn.abstractnn.{AbstractModule, Activity} import com.intel.analytics.bigdl.dllib.nn.{CAddTable, CAveTable, CMaxTable, CMulTable, CosineDistance, DotProduct, JoinTable, ParallelTable, Sequential => TSequential} import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.{MultiShape, Shape} +import com.intel.analytics.bigdl.dllib.utils.{MultiShape, Shape} import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/Permute.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/Permute.scala index 4d136dd843e..b62eca8299b 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/Permute.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/Permute.scala @@ -20,7 +20,7 @@ import com.intel.analytics.bigdl.dllib.nn.Transpose import com.intel.analytics.bigdl.dllib.nn.abstractnn.AbstractModule import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import scala.collection.mutable.ArrayBuffer import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/Pooling1D.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/Pooling1D.scala index a4c7d7cd9ab..3b923ba68e5 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/Pooling1D.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/Pooling1D.scala @@ -18,7 +18,7 @@ package com.intel.analytics.bigdl.dllib.keras import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/Pooling2D.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/Pooling2D.scala index 9c33da8792d..ad127b58d49 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/Pooling2D.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/Pooling2D.scala @@ -19,7 +19,7 @@ package com.intel.analytics.bigdl.dllib.keras import com.intel.analytics.bigdl.dllib.nn.abstractnn.DataFormat import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/Pooling3D.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/Pooling3D.scala index 50aefe9ffdd..886ff31d06f 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/Pooling3D.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/Pooling3D.scala @@ -18,7 +18,7 @@ package com.intel.analytics.bigdl.dllib.keras import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/Recurrent.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/Recurrent.scala index b7bb183825c..00dceb5c943 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/Recurrent.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/Recurrent.scala @@ -20,7 +20,7 @@ import com.intel.analytics.bigdl.dllib.nn.{Cell, Reverse, Select, Sequential => import com.intel.analytics.bigdl.dllib.nn.abstractnn.AbstractModule import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/RepeatVector.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/RepeatVector.scala index dcc6e637ebb..db29b5a3002 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/RepeatVector.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/RepeatVector.scala @@ -20,7 +20,7 @@ import com.intel.analytics.bigdl.dllib.nn.{ErrorInfo, Replicate} import com.intel.analytics.bigdl.dllib.nn.abstractnn.AbstractModule import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/Reshape.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/Reshape.scala index a912d9665eb..b9df82aea16 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/Reshape.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/Reshape.scala @@ -20,7 +20,7 @@ import com.intel.analytics.bigdl.dllib.nn.InferReshape import com.intel.analytics.bigdl.dllib.nn.abstractnn.{AbstractModule, TensorModule} import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/SReLU.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/SReLU.scala index 6479ac50e7b..846143cba5b 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/SReLU.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/SReLU.scala @@ -20,7 +20,7 @@ import com.intel.analytics.bigdl.dllib.nn.{InitializationMethod, Ones, Xavier, Z import com.intel.analytics.bigdl.dllib.nn.abstractnn._ import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/SeparableConvolution2D.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/SeparableConvolution2D.scala index 26bacf3a149..5b9857312df 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/SeparableConvolution2D.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/SeparableConvolution2D.scala @@ -20,7 +20,7 @@ import com.intel.analytics.bigdl.dllib.nn.{InitializationMethod, SpatialSeparabl import com.intel.analytics.bigdl.dllib.optim.Regularizer import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import com.intel.analytics.bigdl.dllib.nn.abstractnn.{AbstractModule, DataFormat} import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/SimpleRNN.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/SimpleRNN.scala index 1d316c62b58..3263ea661ac 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/SimpleRNN.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/SimpleRNN.scala @@ -21,7 +21,7 @@ import com.intel.analytics.bigdl.dllib.nn.{Cell, RnnCell} import com.intel.analytics.bigdl.dllib.optim.Regularizer import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/SoftMax.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/SoftMax.scala index 8f339d95f68..ef155f0e3e2 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/SoftMax.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/SoftMax.scala @@ -20,7 +20,7 @@ import com.intel.analytics.bigdl.dllib.nn.abstractnn.AbstractModule import com.intel.analytics.bigdl.dllib.nn.{Transpose, Sequential => TSequential} import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/SpatialDropout1D.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/SpatialDropout1D.scala index bde3d05db07..0ce5f9b12a6 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/SpatialDropout1D.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/SpatialDropout1D.scala @@ -19,7 +19,7 @@ package com.intel.analytics.bigdl.dllib.keras import com.intel.analytics.bigdl.dllib.nn.abstractnn._ import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/SpatialDropout2D.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/SpatialDropout2D.scala index 2bb9cbdf686..e13a3d55621 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/SpatialDropout2D.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/SpatialDropout2D.scala @@ -19,7 +19,7 @@ package com.intel.analytics.bigdl.dllib.keras import com.intel.analytics.bigdl.dllib.nn.abstractnn._ import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/SpatialDropout3D.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/SpatialDropout3D.scala index 9029d4941d0..83131243b15 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/SpatialDropout3D.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/SpatialDropout3D.scala @@ -19,7 +19,7 @@ package com.intel.analytics.bigdl.dllib.keras import com.intel.analytics.bigdl.dllib.nn.abstractnn._ import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/ThresholdedReLU.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/ThresholdedReLU.scala index 78f4a7f1c0e..608723ba133 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/ThresholdedReLU.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/ThresholdedReLU.scala @@ -20,7 +20,7 @@ import com.intel.analytics.bigdl.dllib.nn.abstractnn._ import com.intel.analytics.bigdl.dllib.nn.Threshold import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/TimeDistributed.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/TimeDistributed.scala index 184200514d8..018cd76b3f9 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/TimeDistributed.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/TimeDistributed.scala @@ -19,7 +19,7 @@ package com.intel.analytics.bigdl.dllib.keras import com.intel.analytics.bigdl.dllib.nn.abstractnn.{AbstractModule, Activity, TensorModule} import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/Topology.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/Topology.scala index 3de24f7684a..97c22391559 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/Topology.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/Topology.scala @@ -24,7 +24,7 @@ import com.intel.analytics.bigdl.dllib.nn.{Container, StaticGraph, Sequential => import com.intel.analytics.bigdl.dllib.optim._ import com.intel.analytics.bigdl.serialization.Bigdl.BigDLModule import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.{LoggerFilter, Shape} +import com.intel.analytics.bigdl.dllib.utils.{LoggerFilter, Shape} import com.intel.analytics.bigdl.dllib.utils.serializer._ import org.apache.spark.rdd.RDD diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/UpSampling1D.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/UpSampling1D.scala index 248933f8f32..1964000a589 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/UpSampling1D.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/UpSampling1D.scala @@ -19,7 +19,7 @@ package com.intel.analytics.bigdl.dllib.keras import com.intel.analytics.bigdl.dllib.nn.abstractnn.AbstractModule import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/UpSampling2D.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/UpSampling2D.scala index 92250983723..c1ec0fc858e 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/UpSampling2D.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/UpSampling2D.scala @@ -19,7 +19,7 @@ package com.intel.analytics.bigdl.dllib.keras import com.intel.analytics.bigdl.dllib.nn.abstractnn.{AbstractModule, DataFormat} import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/UpSampling3D.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/UpSampling3D.scala index 62876d22f0f..8a4818854c2 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/UpSampling3D.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/UpSampling3D.scala @@ -19,7 +19,7 @@ package com.intel.analytics.bigdl.dllib.keras import com.intel.analytics.bigdl.dllib.nn.abstractnn.AbstractModule import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/ZeroPadding1D.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/ZeroPadding1D.scala index af43bd4173a..ee5fc3fa62b 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/ZeroPadding1D.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/ZeroPadding1D.scala @@ -20,7 +20,7 @@ import com.intel.analytics.bigdl.dllib.nn.SpatialZeroPadding import com.intel.analytics.bigdl.dllib.nn.abstractnn.AbstractModule import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/ZeroPadding2D.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/ZeroPadding2D.scala index 968c2ce2c12..fe3087ab011 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/ZeroPadding2D.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/ZeroPadding2D.scala @@ -21,7 +21,7 @@ import com.intel.analytics.bigdl.dllib.nn.{Sequential => TSequential} import com.intel.analytics.bigdl.dllib.nn.abstractnn.{AbstractModule, DataFormat} import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/ZeroPadding3D.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/ZeroPadding3D.scala index 46313d30ea2..f789d530cde 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/ZeroPadding3D.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/keras/ZeroPadding3D.scala @@ -21,7 +21,7 @@ import com.intel.analytics.bigdl.dllib.nn.abstractnn._ import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.nn.{Sequential => TSequential} import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/models/autoencoder/Train.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/models/autoencoder/Train.scala index 06ae4da76e4..ce455d62fe3 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/models/autoencoder/Train.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/models/autoencoder/Train.scala @@ -27,7 +27,7 @@ import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric._ import com.intel.analytics.bigdl.dllib.utils.{T, Table} -import com.intel.analytics.bigdl.utils.{Engine, OptimizerV1, OptimizerV2} +import com.intel.analytics.bigdl.dllib.utils.{Engine, OptimizerV1, OptimizerV2} import org.apache.log4j.{Level, Logger} import org.apache.spark.SparkContext diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/models/inception/Test.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/models/inception/Test.scala index 11420ff3368..c54c14508d3 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/models/inception/Test.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/models/inception/Test.scala @@ -20,7 +20,7 @@ import com.intel.analytics.bigdl.dllib.feature.dataset.{ByteRecord, DataSet} import com.intel.analytics.bigdl.dllib.feature.dataset.image._ import com.intel.analytics.bigdl.dllib.nn.Module import com.intel.analytics.bigdl.dllib.optim.{Top1Accuracy, Top5Accuracy, Validator} -import com.intel.analytics.bigdl.utils.Engine +import com.intel.analytics.bigdl.dllib.utils.Engine import org.apache.hadoop.io.Text import org.apache.log4j.{Level, Logger} import org.apache.spark.SparkContext diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/models/inception/Train.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/models/inception/Train.scala index 5c98e337d64..93a6fcb8903 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/models/inception/Train.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/models/inception/Train.scala @@ -20,8 +20,8 @@ import com.intel.analytics.bigdl.dllib.nn.{ClassNLLCriterion, Module} import com.intel.analytics.bigdl.dllib.optim.SGD.{MultiStep, Poly, SequentialSchedule, Warmup} import com.intel.analytics.bigdl.dllib.optim._ import com.intel.analytics.bigdl.dllib.utils.{T, Table} -import com.intel.analytics.bigdl.utils.{Engine, OptimizerV1, OptimizerV2} -import com.intel.analytics.bigdl.utils.LoggerFilter +import com.intel.analytics.bigdl.dllib.utils.{Engine, OptimizerV1, OptimizerV2} +import com.intel.analytics.bigdl.dllib.utils.LoggerFilter import org.apache.log4j.{Level, Logger} import org.apache.spark.SparkContext diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/models/lenet/LeNet5.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/models/lenet/LeNet5.scala index 6be04bfefa1..fe1a8cd2041 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/models/lenet/LeNet5.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/models/lenet/LeNet5.scala @@ -59,7 +59,7 @@ object LeNet5 { def kerasLayer(classNum: Int): keras.Sequential[Float] = { import com.intel.analytics.bigdl.dllib.keras._ - import com.intel.analytics.bigdl.utils.Shape + import com.intel.analytics.bigdl.dllib.utils.Shape val model = Sequential() model.add(Reshape(Array(1, 28, 28), inputShape = Shape(28, 28, 1))) @@ -74,7 +74,7 @@ object LeNet5 { def kerasGraph(classNum: Int): keras.Model[Float] = { import com.intel.analytics.bigdl.dllib.keras._ - import com.intel.analytics.bigdl.utils.Shape + import com.intel.analytics.bigdl.dllib.utils.Shape val input = Input(inputShape = Shape(28, 28, 1)) val reshape = Reshape(Array(1, 28, 28)).inputs(input) diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/models/lenet/Test.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/models/lenet/Test.scala index 10de98fde2b..3b91b81b8f5 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/models/lenet/Test.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/models/lenet/Test.scala @@ -22,7 +22,7 @@ import com.intel.analytics.bigdl.dllib.feature.dataset.DataSet import com.intel.analytics.bigdl.dllib.feature.dataset.image.{BytesToGreyImg, GreyImgNormalizer, GreyImgToSample} import com.intel.analytics.bigdl.dllib.nn.Module import com.intel.analytics.bigdl.dllib.optim.Top1Accuracy -import com.intel.analytics.bigdl.utils.Engine +import com.intel.analytics.bigdl.dllib.utils.Engine import org.apache.log4j.{Level, Logger} import org.apache.spark.SparkContext diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/models/lenet/Train.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/models/lenet/Train.scala index 56b31ea091b..c4a5040eb9e 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/models/lenet/Train.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/models/lenet/Train.scala @@ -22,7 +22,7 @@ import com.intel.analytics.bigdl.dllib.feature.dataset.image.{BytesToGreyImg, Gr import com.intel.analytics.bigdl.dllib.nn.{ClassNLLCriterion, CrossEntropyCriterion, Module} import com.intel.analytics.bigdl.numeric.NumericFloat import com.intel.analytics.bigdl.dllib.optim._ -import com.intel.analytics.bigdl.utils._ +import com.intel.analytics.bigdl.dllib.utils._ import com.intel.analytics.bigdl.dllib.utils._ import org.apache.log4j.{Level, Logger} import org.apache.spark.SparkContext diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/models/lenet/Utils.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/models/lenet/Utils.scala index 7f2bedc9c01..4bfa5a02a43 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/models/lenet/Utils.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/models/lenet/Utils.scala @@ -21,7 +21,7 @@ import java.nio.file.{Files, Path, Paths} import com.intel.analytics.bigdl.dllib.feature.dataset.ByteRecord import com.intel.analytics.bigdl.dllib.utils.{File} -import com.intel.analytics.bigdl.utils.{OptimizerVersion} +import com.intel.analytics.bigdl.dllib.utils.{OptimizerVersion} import scopt.OptionParser object Utils { diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/models/maskrcnn/Test.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/models/maskrcnn/Test.scala index 0bc69742416..e69ed6de47a 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/models/maskrcnn/Test.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/models/maskrcnn/Test.scala @@ -20,7 +20,7 @@ import com.intel.analytics.bigdl.dllib.models.resnet.Utils.{TestParams, _} import com.intel.analytics.bigdl.dllib.feature.transform.vision.image._ import com.intel.analytics.bigdl.dllib.feature.transform.vision.image.augmentation._ import com.intel.analytics.bigdl.dllib.utils.{T, Table} -import com.intel.analytics.bigdl.utils.{Engine, OptimizerV1, OptimizerV2} +import com.intel.analytics.bigdl.dllib.utils.{Engine, OptimizerV1, OptimizerV2} import scopt.OptionParser import com.intel.analytics.bigdl.dllib.feature.dataset.{DataSet, MiniBatch, segmentation} import com.intel.analytics.bigdl.dllib.nn.Module diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/models/resnet/ResNet.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/models/resnet/ResNet.scala index b1ea35f09f6..151d53bc6b7 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/models/resnet/ResNet.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/models/resnet/ResNet.scala @@ -25,7 +25,7 @@ import com.intel.analytics.bigdl.dllib.optim.L2Regularizer import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric import com.intel.analytics.bigdl.dllib.tensor.{Storage, Tensor} import com.intel.analytics.bigdl.dllib.utils.Table -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import org.apache.log4j.Logger import scala.collection.mutable diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/models/resnet/Test.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/models/resnet/Test.scala index de5b1bff964..53fa1ebf2b1 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/models/resnet/Test.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/models/resnet/Test.scala @@ -18,7 +18,7 @@ package com.intel.analytics.bigdl.dllib.models.resnet import com.intel.analytics.bigdl.Module import com.intel.analytics.bigdl.dllib.nn.Module -import com.intel.analytics.bigdl.utils.Engine +import com.intel.analytics.bigdl.dllib.utils.Engine import com.intel.analytics.bigdl.dllib.models.resnet.Utils._ import com.intel.analytics.bigdl.dllib.optim.{Top1Accuracy, ValidationMethod, ValidationResult} import com.intel.analytics.bigdl.dllib.feature.dataset.image.{BGRImgNormalizer, BGRImgToSample, BytesToBGRImg} diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/models/resnet/TestImageNet.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/models/resnet/TestImageNet.scala index 6adff50778a..4f1c23d0e75 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/models/resnet/TestImageNet.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/models/resnet/TestImageNet.scala @@ -27,7 +27,7 @@ import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric._ import com.intel.analytics.bigdl.dllib.feature.transform.vision.image.{ImageFeature, MTImageFeatureToBatch, MatToTensor, PixelBytesToMat} import com.intel.analytics.bigdl.dllib.feature.transform.vision.image.augmentation.{ChannelScaledNormalizer, RandomCropper, RandomResize} import com.intel.analytics.bigdl.dllib.utils._ -import com.intel.analytics.bigdl.utils._ +import com.intel.analytics.bigdl.dllib.utils._ import org.apache.log4j.{Level, Logger} import org.apache.spark.SparkContext diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/models/resnet/TrainCIFAR10.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/models/resnet/TrainCIFAR10.scala index 008da1898ee..36238a4e382 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/models/resnet/TrainCIFAR10.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/models/resnet/TrainCIFAR10.scala @@ -21,8 +21,8 @@ import com.intel.analytics.bigdl._ import com.intel.analytics.bigdl.dllib.models.resnet.ResNet.{DatasetType, ShortcutType} import com.intel.analytics.bigdl.dllib.optim._ import com.intel.analytics.bigdl.dllib.utils.{T, Table} -import com.intel.analytics.bigdl.utils.{Engine, OptimizerV1, OptimizerV2} -import com.intel.analytics.bigdl.utils.LoggerFilter +import com.intel.analytics.bigdl.dllib.utils.{Engine, OptimizerV1, OptimizerV2} +import com.intel.analytics.bigdl.dllib.utils.LoggerFilter import org.apache.log4j.{Level, Logger} import org.apache.spark.SparkContext import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric._ diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/models/resnet/TrainImageNet.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/models/resnet/TrainImageNet.scala index 53ac3765505..8ca4fcb109d 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/models/resnet/TrainImageNet.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/models/resnet/TrainImageNet.scala @@ -25,7 +25,7 @@ import com.intel.analytics.bigdl.dllib.nn.{BatchNormalization, Container, CrossE import com.intel.analytics.bigdl.dllib.optim._ import com.intel.analytics.bigdl.dllib.utils._ import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric._ -import com.intel.analytics.bigdl.utils._ +import com.intel.analytics.bigdl.dllib.utils._ import com.intel.analytics.bigdl.dllib.visualization.{TrainSummary, ValidationSummary} import org.apache.log4j.{Level, Logger} import org.apache.spark.SparkContext diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/models/rnn/Test.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/models/rnn/Test.scala index be45d1af8b0..13b17baa316 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/models/rnn/Test.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/models/rnn/Test.scala @@ -24,7 +24,7 @@ import com.intel.analytics.bigdl.dllib.nn._ import com.intel.analytics.bigdl.dllib.optim.{Loss, ValidationMethod, ValidationResult} import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.utils.{T, Table} -import com.intel.analytics.bigdl.utils.{Engine, OptimizerV1, OptimizerV2} +import com.intel.analytics.bigdl.dllib.utils.{Engine, OptimizerV1, OptimizerV2} import org.apache.log4j.{Level, Logger} import org.apache.spark.SparkContext import org.apache.spark.rdd.RDD diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/models/rnn/Train.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/models/rnn/Train.scala index e2edee20076..cc111b9b8ce 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/models/rnn/Train.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/models/rnn/Train.scala @@ -25,7 +25,7 @@ import com.intel.analytics.bigdl.dllib.nn.{CrossEntropyCriterion, Module, TimeDi import com.intel.analytics.bigdl.dllib.optim._ import com.intel.analytics.bigdl.dllib.tensor.{Storage, Tensor} import com.intel.analytics.bigdl.dllib.utils.{T, Table} -import com.intel.analytics.bigdl.utils.{Engine, OptimizerV1, OptimizerV2} +import com.intel.analytics.bigdl.dllib.utils.{Engine, OptimizerV1, OptimizerV2} import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric._ import org.apache.log4j.{Level, Logger} import org.apache.spark.SparkContext diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/models/utils/DistriOptimizerPerf.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/models/utils/DistriOptimizerPerf.scala index 139b2244f83..7a53ad609fd 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/models/utils/DistriOptimizerPerf.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/models/utils/DistriOptimizerPerf.scala @@ -23,7 +23,7 @@ import com.intel.analytics.bigdl.dllib.nn.ClassNLLCriterion import com.intel.analytics.bigdl.numeric.NumericFloat import com.intel.analytics.bigdl.dllib.optim.{Optimizer, Trigger} import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.Engine +import com.intel.analytics.bigdl.dllib.utils.Engine import org.apache.log4j.{Level, Logger} import org.apache.spark.SparkContext import org.apache.spark.rdd.RDD diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/models/utils/LocalOptimizerPerf.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/models/utils/LocalOptimizerPerf.scala index e40544544a3..f2d225e2201 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/models/utils/LocalOptimizerPerf.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/models/utils/LocalOptimizerPerf.scala @@ -23,7 +23,7 @@ import com.intel.analytics.bigdl.dllib.nn.ClassNLLCriterion import com.intel.analytics.bigdl.numeric.NumericFloat import com.intel.analytics.bigdl.dllib.optim.{Optimizer, Trigger} import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.Engine +import com.intel.analytics.bigdl.dllib.utils.Engine import scopt.OptionParser object LocalOptimizerPerf { diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/models/utils/ModelBroadcast.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/models/utils/ModelBroadcast.scala index 583210289b7..12884bb424c 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/models/utils/ModelBroadcast.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/models/utils/ModelBroadcast.scala @@ -25,7 +25,7 @@ import com.intel.analytics.bigdl.dllib.nn.abstractnn.Activity import com.intel.analytics.bigdl.dllib.nn.mkldnn.{MklDnnLayer, TensorMMap} import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric import com.intel.analytics.bigdl.dllib.tensor._ -import com.intel.analytics.bigdl.utils.{Engine, MklDnn} +import com.intel.analytics.bigdl.dllib.utils.{Engine, MklDnn} import com.intel.analytics.bigdl.dllib.utils.Util._ import com.intel.analytics.bigdl.dllib.utils.intermediate.IRGraph import org.apache.commons.lang3.SerializationUtils diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/models/vgg/Test.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/models/vgg/Test.scala index e63fac8056c..7b5730c7488 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/models/vgg/Test.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/models/vgg/Test.scala @@ -21,7 +21,7 @@ import com.intel.analytics.bigdl.dllib.feature.dataset.image._ import com.intel.analytics.bigdl.dllib.models.lenet.Utils._ import com.intel.analytics.bigdl.dllib.nn.Module import com.intel.analytics.bigdl.dllib.optim.{Top1Accuracy, Validator} -import com.intel.analytics.bigdl.utils.Engine +import com.intel.analytics.bigdl.dllib.utils.Engine import org.apache.log4j.{Level, Logger} import org.apache.spark.SparkContext diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/models/vgg/Train.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/models/vgg/Train.scala index cfddf5879bd..cd6286c1050 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/models/vgg/Train.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/models/vgg/Train.scala @@ -25,8 +25,8 @@ import com.intel.analytics.bigdl.dllib.nn.{ClassNLLCriterion, Module} import com.intel.analytics.bigdl.dllib.optim._ import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric._ import com.intel.analytics.bigdl.dllib.utils.{T, Table} -import com.intel.analytics.bigdl.utils.{Engine, OptimizerV1, OptimizerV2} -import com.intel.analytics.bigdl.utils.LoggerFilter +import com.intel.analytics.bigdl.dllib.utils.{Engine, OptimizerV1, OptimizerV2} +import com.intel.analytics.bigdl.dllib.utils.LoggerFilter import com.intel.analytics.bigdl.dllib.visualization.{TrainSummary, ValidationSummary} import org.apache.log4j.{Level, Logger} import org.apache.spark.SparkContext diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/models/vgg/TrainImageNet.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/models/vgg/TrainImageNet.scala index ec96f72e56a..616cd27a25e 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/models/vgg/TrainImageNet.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/models/vgg/TrainImageNet.scala @@ -20,9 +20,9 @@ import com.intel.analytics.bigdl.dllib.nn import com.intel.analytics.bigdl.dllib.nn.{CrossEntropyCriterion, Module, SoftmaxWithCriterion} import com.intel.analytics.bigdl.dllib.optim.SGD.{Poly, SequentialSchedule, Warmup} import com.intel.analytics.bigdl.dllib.optim._ -import com.intel.analytics.bigdl.utils.{Engine, MklBlas, MklDnn, OptimizerV1, OptimizerV2} +import com.intel.analytics.bigdl.dllib.utils.{Engine, MklBlas, MklDnn, OptimizerV1, OptimizerV2} import com.intel.analytics.bigdl.dllib.utils._ -import com.intel.analytics.bigdl.utils.LoggerFilter +import com.intel.analytics.bigdl.dllib.utils.LoggerFilter import com.intel.analytics.bigdl.dllib.visualization.TrainSummary import org.apache.log4j.{Level, Logger} import org.apache.spark.SparkContext diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/Add.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/Add.scala index 06ee858d4b5..f5825435240 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/Add.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/Add.scala @@ -18,7 +18,7 @@ package com.intel.analytics.bigdl.dllib.nn import com.intel.analytics.bigdl.dllib.nn.abstractnn.{Initializable, TensorModule} import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import com.intel.analytics.bigdl.dllib.utils.{T, Table} import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/BatchNormalization.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/BatchNormalization.scala index 7711a22ea2b..a57c547cad2 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/BatchNormalization.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/BatchNormalization.scala @@ -24,7 +24,7 @@ import com.intel.analytics.bigdl.dllib.utils.serializer._ import com.intel.analytics.bigdl.dllib.utils.serializer.converters.DataConverter import com.intel.analytics.bigdl.dllib.utils.{ParameterSynchronizer, T, Table} import com.intel.analytics.bigdl.serialization.Bigdl.{AttrValue, BigDLModule} -import com.intel.analytics.bigdl.utils._ +import com.intel.analytics.bigdl.dllib.utils._ import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/Bilinear.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/Bilinear.scala index 5d0173eea68..d0c04896edd 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/Bilinear.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/Bilinear.scala @@ -19,7 +19,7 @@ import com.intel.analytics.bigdl.dllib.nn.abstractnn.{AbstractModule, Initializa import com.intel.analytics.bigdl.dllib.optim.Regularizer import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import com.intel.analytics.bigdl.dllib.utils.{T, Table} import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/BinaryThreshold.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/BinaryThreshold.scala index 3e0f5b38a53..210d2b68d8a 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/BinaryThreshold.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/BinaryThreshold.scala @@ -19,7 +19,7 @@ package com.intel.analytics.bigdl.dllib.nn import com.intel.analytics.bigdl.dllib.nn.abstractnn.TensorModule import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric import com.intel.analytics.bigdl.dllib.tensor._ -import com.intel.analytics.bigdl.utils.Engine +import com.intel.analytics.bigdl.dllib.utils.Engine import scala.collection.mutable.ArrayBuffer import scala.concurrent.Future diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/BoxHead.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/BoxHead.scala index e010bbd9814..b397e8495c7 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/BoxHead.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/BoxHead.scala @@ -22,7 +22,7 @@ import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric import com.intel.analytics.bigdl.dllib.feature.transform.vision.image.label.roi.RoiLabel import com.intel.analytics.bigdl.dllib.feature.transform.vision.image.util.BboxUtil -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import com.intel.analytics.bigdl.dllib.utils.{T, Table} import scala.collection.mutable.ArrayBuffer diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/CAdd.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/CAdd.scala index f2948dfdf43..fc4434fb597 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/CAdd.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/CAdd.scala @@ -20,9 +20,9 @@ import com.intel.analytics.bigdl.dllib.nn.abstractnn.{Initializable, TensorModul import com.intel.analytics.bigdl.dllib.optim.Regularizer import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import com.intel.analytics.bigdl.dllib.utils.{T, Table} -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/CMul.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/CMul.scala index 38a96d893f1..71f3daf255e 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/CMul.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/CMul.scala @@ -20,9 +20,9 @@ import com.intel.analytics.bigdl.dllib.nn.abstractnn.{Initializable, TensorModul import com.intel.analytics.bigdl.dllib.optim.Regularizer import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import com.intel.analytics.bigdl.dllib.utils.{T, Table} -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/ClassNLLCriterion.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/ClassNLLCriterion.scala index cda284bbe24..dcf5c1317a5 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/ClassNLLCriterion.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/ClassNLLCriterion.scala @@ -24,7 +24,7 @@ import com.intel.analytics.bigdl.dllib.tensor.Tensor import scala.concurrent.duration.Duration import scala.concurrent.{Await, Future} import scala.reflect.ClassTag -import com.intel.analytics.bigdl.utils.Engine +import com.intel.analytics.bigdl.dllib.utils.Engine import org.apache.hadoop.mapreduce.v2.app.speculate.TaskRuntimeEstimator /** diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/Concat.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/Concat.scala index c777ee2e6de..dc3f15da5a1 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/Concat.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/Concat.scala @@ -20,7 +20,7 @@ import com.intel.analytics.bigdl.dllib.nn.Graph.ModuleNode import com.intel.analytics.bigdl.dllib.nn.abstractnn.{AbstractModule, Activity} import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.Engine +import com.intel.analytics.bigdl.dllib.utils.Engine import scala.collection.mutable.ArrayBuffer import scala.concurrent.Future diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/Cosine.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/Cosine.scala index da4e9f9dda8..fefc97945e9 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/Cosine.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/Cosine.scala @@ -18,7 +18,7 @@ package com.intel.analytics.bigdl.dllib.nn import com.intel.analytics.bigdl.dllib.nn.abstractnn.{Initializable, TensorModule} import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import com.intel.analytics.bigdl.dllib.utils.{T, Table} import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/Cropping2D.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/Cropping2D.scala index 4c657e5a1ed..4862c78ba96 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/Cropping2D.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/Cropping2D.scala @@ -18,7 +18,7 @@ package com.intel.analytics.bigdl.dllib.nn import com.intel.analytics.bigdl.dllib.nn.abstractnn.{DataFormat, TensorModule} import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/Cropping3D.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/Cropping3D.scala index 4b75011e334..585bed8694a 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/Cropping3D.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/Cropping3D.scala @@ -18,7 +18,7 @@ package com.intel.analytics.bigdl.dllib.nn import com.intel.analytics.bigdl.dllib.nn.abstractnn.TensorModule import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/DetectionOutputSSD.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/DetectionOutputSSD.scala index 95e932e375d..9fe8a4ea1fa 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/DetectionOutputSSD.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/DetectionOutputSSD.scala @@ -24,7 +24,7 @@ import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric import com.intel.analytics.bigdl.dllib.feature.transform.vision.image.util.BboxUtil import com.intel.analytics.bigdl.dllib.utils.Table -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import org.apache.log4j.Logger import DetectionOutputSSD.logger diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/Dropout.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/Dropout.scala index f6278050fe5..378818bb4c7 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/Dropout.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/Dropout.scala @@ -18,9 +18,9 @@ package com.intel.analytics.bigdl.dllib.nn import com.intel.analytics.bigdl.dllib.nn.abstractnn.{IdentityOutputShape, TensorModule} import com.intel.analytics.bigdl.dllib.tensor.{Storage, Tensor} import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.Engine -import com.intel.analytics.bigdl.utils.RandomGenerator._ -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Engine +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.Shape import scala.collection.mutable.ArrayBuffer import scala.concurrent.Future diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/Euclidean.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/Euclidean.scala index 28e6a43956b..bf394d753a9 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/Euclidean.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/Euclidean.scala @@ -18,7 +18,7 @@ package com.intel.analytics.bigdl.dllib.nn import com.intel.analytics.bigdl.dllib.nn.abstractnn.{Initializable, TensorModule} import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import com.intel.analytics.bigdl.dllib.utils.{T, Table} import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/HardTanh.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/HardTanh.scala index 5d66447ac02..8b94b43a6aa 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/HardTanh.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/HardTanh.scala @@ -18,7 +18,7 @@ package com.intel.analytics.bigdl.dllib.nn import com.intel.analytics.bigdl.dllib.nn.abstractnn.{AbstractModule, TensorModule} import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric import com.intel.analytics.bigdl.dllib.tensor._ -import com.intel.analytics.bigdl.utils.Engine +import com.intel.analytics.bigdl.dllib.utils.Engine import scala.concurrent.Future import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/InferReshape.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/InferReshape.scala index d2ecf1f0ca4..eb87dcae313 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/InferReshape.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/InferReshape.scala @@ -19,7 +19,7 @@ package com.intel.analytics.bigdl.dllib.nn import com.intel.analytics.bigdl.dllib.nn.abstractnn.TensorModule import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import scala.collection.mutable.ArrayBuffer import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/InitializationMethod.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/InitializationMethod.scala index beeafd5256d..143bf276805 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/InitializationMethod.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/InitializationMethod.scala @@ -19,7 +19,7 @@ package com.intel.analytics.bigdl.dllib.nn import com.intel.analytics.bigdl.dllib.nn.VariableFormat.Default import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.RandomGenerator +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator /** * VariableFormat describe the meaning of each dimension of the variable diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/JoinTable.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/JoinTable.scala index 1d2be0cd4f9..2dbe7ce6d66 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/JoinTable.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/JoinTable.scala @@ -21,7 +21,7 @@ import com.intel.analytics.bigdl.dllib.tensor._ import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.{NumericWildcard, TensorNumeric} import com.intel.analytics.bigdl.dllib.utils.tf.TFTensorNumeric.NumericByteString import com.intel.analytics.bigdl.dllib.utils.{T, Table} -import com.intel.analytics.bigdl.utils.{Engine, OptimizerV1, OptimizerV2} +import com.intel.analytics.bigdl.dllib.utils.{Engine, OptimizerV1, OptimizerV2} import scala.concurrent.Future import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/L1HingeEmbeddingCriterion.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/L1HingeEmbeddingCriterion.scala index f5b68f422e1..d5921d3c5de 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/L1HingeEmbeddingCriterion.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/L1HingeEmbeddingCriterion.scala @@ -18,7 +18,7 @@ package com.intel.analytics.bigdl.dllib.nn import com.intel.analytics.bigdl.dllib.nn.abstractnn.AbstractCriterion import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import com.intel.analytics.bigdl.dllib.utils.Table import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/Linear.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/Linear.scala index 648dadcf5b6..c0ff9c3abab 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/Linear.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/Linear.scala @@ -22,7 +22,7 @@ import com.intel.analytics.bigdl.dllib.optim.Regularizer import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric import com.intel.analytics.bigdl.dllib.utils.{T, Table} -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/LocallyConnected1D.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/LocallyConnected1D.scala index a0bbd0faabd..1013ba36006 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/LocallyConnected1D.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/LocallyConnected1D.scala @@ -21,7 +21,7 @@ import com.intel.analytics.bigdl.dllib.optim.Regularizer import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric import com.intel.analytics.bigdl.dllib.utils.{T, Table} -import com.intel.analytics.bigdl.utils.{Engine, OptimizerV1, OptimizerV2} +import com.intel.analytics.bigdl.dllib.utils.{Engine, OptimizerV1, OptimizerV2} import scala.concurrent.Future import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/LocallyConnected2D.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/LocallyConnected2D.scala index b9fcb40ed30..1918e743c84 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/LocallyConnected2D.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/LocallyConnected2D.scala @@ -20,8 +20,8 @@ import com.intel.analytics.bigdl.dllib.optim.Regularizer import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric import com.intel.analytics.bigdl.dllib.tensor.{DoubleType, FloatType, Tensor} import com.intel.analytics.bigdl.dllib.utils.{T, Table} -import com.intel.analytics.bigdl.utils.{Engine, OptimizerV1, OptimizerV2} -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.{Engine, OptimizerV1, OptimizerV2} +import com.intel.analytics.bigdl.dllib.utils.Shape import scala.concurrent.Future import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/LogSoftMax.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/LogSoftMax.scala index aa080ed245d..183ceda2de7 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/LogSoftMax.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/LogSoftMax.scala @@ -22,8 +22,8 @@ import com.intel.analytics.bigdl.mkl.MKL import com.intel.analytics.bigdl.dllib.nn.abstractnn.TensorModule import com.intel.analytics.bigdl.dllib.tensor.{DoubleType, FloatType, Storage, Tensor} import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.Engine -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Engine +import com.intel.analytics.bigdl.dllib.utils.Shape import scala.concurrent.Future import scala.math.exp diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/LookupTable.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/LookupTable.scala index def98481166..34df896e066 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/LookupTable.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/LookupTable.scala @@ -20,9 +20,9 @@ import com.intel.analytics.bigdl.dllib.nn.abstractnn.{Initializable, TensorModul import com.intel.analytics.bigdl.dllib.optim.Regularizer import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import com.intel.analytics.bigdl.dllib.utils.{T, Table} -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/Maxout.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/Maxout.scala index 3cafc41bad6..02b3dcf495d 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/Maxout.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/Maxout.scala @@ -24,7 +24,7 @@ import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric import com.intel.analytics.bigdl.dllib.utils.serializer.converters.DataConverter import com.intel.analytics.bigdl.dllib.utils.serializer.{DeserializeContext, ModuleSerializable, ModuleSerializer, SerializeContext} import com.intel.analytics.bigdl.dllib.utils.Table -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/Mul.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/Mul.scala index cdf631e8280..54f69b913c1 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/Mul.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/Mul.scala @@ -18,7 +18,7 @@ package com.intel.analytics.bigdl.dllib.nn import com.intel.analytics.bigdl.dllib.nn.abstractnn.{Initializable, TensorModule} import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import com.intel.analytics.bigdl.dllib.utils.{T, Table} import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/Normalize.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/Normalize.scala index bcf90c9ac65..18696f83c29 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/Normalize.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/Normalize.scala @@ -18,7 +18,7 @@ package com.intel.analytics.bigdl.dllib.nn import com.intel.analytics.bigdl.dllib.nn.abstractnn.TensorModule import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/NormalizeScale.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/NormalizeScale.scala index 9a9bf8461fd..22f3b58da64 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/NormalizeScale.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/NormalizeScale.scala @@ -21,7 +21,7 @@ import com.intel.analytics.bigdl.dllib.optim.Regularizer import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric import com.intel.analytics.bigdl.dllib.utils.{T, Table} -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/PReLU.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/PReLU.scala index 397bd3442d5..23831caf17c 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/PReLU.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/PReLU.scala @@ -19,7 +19,7 @@ import com.intel.analytics.bigdl.dllib.nn.abstractnn.{Initializable, TensorModul import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric import com.intel.analytics.bigdl.dllib.tensor.{DenseTensorApply, Tensor, TensorFunc4, TensorFunc6} import com.intel.analytics.bigdl.dllib.utils.{T, Table} -import com.intel.analytics.bigdl.utils.{Engine, OptimizerV1, OptimizerV2} +import com.intel.analytics.bigdl.dllib.utils.{Engine, OptimizerV1, OptimizerV2} import scala.concurrent.Future import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/PairwiseDistance.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/PairwiseDistance.scala index 50973823fb5..9eed20c87b3 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/PairwiseDistance.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/PairwiseDistance.scala @@ -18,7 +18,7 @@ package com.intel.analytics.bigdl.dllib.nn import com.intel.analytics.bigdl.dllib.nn.abstractnn.AbstractModule import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import com.intel.analytics.bigdl.dllib.utils.Table import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/Power.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/Power.scala index 0251a0f1a5e..93a6f2d776c 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/Power.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/Power.scala @@ -19,7 +19,7 @@ package com.intel.analytics.bigdl.dllib.nn import com.intel.analytics.bigdl.dllib.nn.abstractnn.{AbstractModule, TensorModule} import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/PriorBox.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/PriorBox.scala index aed0bd68537..641918aae1d 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/PriorBox.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/PriorBox.scala @@ -20,7 +20,7 @@ import com.intel.analytics.bigdl.dllib.nn.abstractnn.{AbstractModule, Activity} import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric.{NumericDouble, NumericFloat} -import com.intel.analytics.bigdl.utils.{Shape, SingleShape} +import com.intel.analytics.bigdl.dllib.utils.{Shape, SingleShape} import scala.collection.mutable.ArrayBuffer import scala.reflect._ diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/RNN.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/RNN.scala index 8b09a25e57a..f88eb1c37fa 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/RNN.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/RNN.scala @@ -22,7 +22,7 @@ import com.intel.analytics.bigdl.dllib.nn.abstractnn.TensorModule import com.intel.analytics.bigdl.dllib.optim.Regularizer import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import com.intel.analytics.bigdl.dllib.utils.Table import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/RReLU.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/RReLU.scala index eda26786ef4..04526d4610a 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/RReLU.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/RReLU.scala @@ -18,7 +18,7 @@ package com.intel.analytics.bigdl.dllib.nn import com.intel.analytics.bigdl.dllib.nn.abstractnn.TensorModule import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric import com.intel.analytics.bigdl.dllib.tensor._ -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/RegionProposal.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/RegionProposal.scala index fd99f49d4d0..7050fcc571c 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/RegionProposal.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/RegionProposal.scala @@ -23,7 +23,7 @@ import com.intel.analytics.bigdl.dllib.tensor.{Storage, Tensor} import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric import com.intel.analytics.bigdl.dllib.feature.transform.vision.image.util.BboxUtil import com.intel.analytics.bigdl.dllib.utils.{T, Table} -import com.intel.analytics.bigdl.utils.LayerException +import com.intel.analytics.bigdl.dllib.utils.LayerException import scala.collection.mutable.ArrayBuffer /** diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/Reshape.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/Reshape.scala index 2290dfe7933..bf70f9e867e 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/Reshape.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/Reshape.scala @@ -22,7 +22,7 @@ import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric import com.intel.analytics.bigdl.dllib.utils.serializer._ import com.intel.analytics.bigdl.dllib.utils.serializer.converters.DataConverter import com.intel.analytics.bigdl.serialization.Bigdl.{AttrValue, BigDLModule} -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import scala.reflect.ClassTag import scala.reflect.runtime.universe diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/Scale.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/Scale.scala index 954ae8a7a12..5668800b8bf 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/Scale.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/Scale.scala @@ -22,7 +22,7 @@ import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric import com.intel.analytics.bigdl.dllib.utils.serializer._ import com.intel.analytics.bigdl.dllib.utils.serializer.converters.DataConverter import com.intel.analytics.bigdl.dllib.utils.{T, Table} -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import com.intel.analytics.bigdl.serialization.Bigdl.{AttrValue, BigDLModule} import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/SoftMax.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/SoftMax.scala index ffdf3bbd004..9383d65db93 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/SoftMax.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/SoftMax.scala @@ -19,8 +19,8 @@ package com.intel.analytics.bigdl.dllib.nn import com.intel.analytics.bigdl.dllib.nn.abstractnn.TensorModule import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.{Engine} -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.{Engine} +import com.intel.analytics.bigdl.dllib.utils.Shape import scala.concurrent.Future import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/SparseJoinTable.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/SparseJoinTable.scala index de394e02d92..232c03447fc 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/SparseJoinTable.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/SparseJoinTable.scala @@ -20,7 +20,7 @@ import com.intel.analytics.bigdl.dllib.nn.abstractnn.AbstractModule import com.intel.analytics.bigdl.dllib.tensor.{DenseTensor, SparseTensor, Tensor} import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric import com.intel.analytics.bigdl.dllib.utils.{Table} -import com.intel.analytics.bigdl.utils._ +import com.intel.analytics.bigdl.dllib.utils._ import scala.concurrent.Future import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/SpatialAveragePooling.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/SpatialAveragePooling.scala index fa6fb8bc210..0cc48c1889f 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/SpatialAveragePooling.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/SpatialAveragePooling.scala @@ -25,7 +25,7 @@ import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric import scala.concurrent.duration.Duration import scala.concurrent.{Await, Future} import scala.reflect._ -import com.intel.analytics.bigdl.utils.Engine +import com.intel.analytics.bigdl.dllib.utils.Engine /** * Applies 2D average-pooling operation in kWxkH regions by step size dWxdH steps. diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/SpatialBatchNormalization.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/SpatialBatchNormalization.scala index c8c998c50c9..f4272cddab6 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/SpatialBatchNormalization.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/SpatialBatchNormalization.scala @@ -20,7 +20,7 @@ import com.intel.analytics.bigdl.dllib.nn.abstractnn.DataFormat import com.intel.analytics.bigdl.dllib.tensor.{FloatType, Tensor} import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric import com.intel.analytics.bigdl.dllib.utils.{ParameterSynchronizer} -import com.intel.analytics.bigdl.utils._ +import com.intel.analytics.bigdl.dllib.utils._ import scala.collection.JavaConverters._ import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/SpatialConvolution.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/SpatialConvolution.scala index ae4d1fcc5f0..2d8eedf33a7 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/SpatialConvolution.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/SpatialConvolution.scala @@ -24,7 +24,7 @@ import com.intel.analytics.bigdl.dllib.tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric import com.intel.analytics.bigdl.dllib.tensor._ import com.intel.analytics.bigdl.dllib.utils._ -import com.intel.analytics.bigdl.utils._ +import com.intel.analytics.bigdl.dllib.utils._ import scala.concurrent.Future import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/SpatialCrossMapLRN.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/SpatialCrossMapLRN.scala index 3f88eb6c196..0fc074e21c1 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/SpatialCrossMapLRN.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/SpatialCrossMapLRN.scala @@ -19,7 +19,7 @@ package com.intel.analytics.bigdl.dllib.nn import com.intel.analytics.bigdl.dllib.nn.abstractnn.{DataFormat, TensorModule} import com.intel.analytics.bigdl.dllib.tensor.{FloatType, Tensor} import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.Engine +import com.intel.analytics.bigdl.dllib.utils.Engine import scala.concurrent.Future import scala.reflect._ diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/SpatialDilatedConvolution.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/SpatialDilatedConvolution.scala index fda1c3480df..e80fa9345ed 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/SpatialDilatedConvolution.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/SpatialDilatedConvolution.scala @@ -21,9 +21,9 @@ import com.intel.analytics.bigdl.dllib.nn.abstractnn.{Initializable, TensorModul import com.intel.analytics.bigdl.dllib.optim.Regularizer import com.intel.analytics.bigdl.dllib.tensor.{DenseTensorBLAS, DoubleType, FloatType, Tensor} import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import com.intel.analytics.bigdl.dllib.utils.{T, Table} -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/SpatialFullConvolution.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/SpatialFullConvolution.scala index 3e8a3cb94b3..f06be2d8440 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/SpatialFullConvolution.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/SpatialFullConvolution.scala @@ -24,7 +24,7 @@ import com.intel.analytics.bigdl.dllib.tensor._ import com.intel.analytics.bigdl.dllib.utils.serializer._ import com.intel.analytics.bigdl.dllib.utils.serializer.converters.DataConverter import com.intel.analytics.bigdl.dllib.utils.{T, Table} -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import scala.reflect.ClassTag import scala.reflect.runtime.universe diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/SpatialMaxPooling.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/SpatialMaxPooling.scala index bf185552818..40f21b45f7a 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/SpatialMaxPooling.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/SpatialMaxPooling.scala @@ -20,7 +20,7 @@ import com.intel.analytics.bigdl.dllib.nn.abstractnn.{AbstractModule, Activity, import com.intel.analytics.bigdl.dllib.tensor import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.Engine +import com.intel.analytics.bigdl.dllib.utils.Engine import com.intel.analytics.bigdl.dllib.utils.serializer._ import com.intel.analytics.bigdl.dllib.utils.serializer.converters.DataConverter import com.intel.analytics.bigdl.serialization.Bigdl.{AttrValue, BigDLModule} diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/SpatialSeparableConvolution.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/SpatialSeparableConvolution.scala index 444d7a9a18d..b820d9e9c4b 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/SpatialSeparableConvolution.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/SpatialSeparableConvolution.scala @@ -20,7 +20,7 @@ import com.intel.analytics.bigdl.dllib.optim.Regularizer import com.intel.analytics.bigdl.serialization.Bigdl.{AttrValue, BigDLModule} import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import com.intel.analytics.bigdl.dllib.utils.serializer.converters.DataConverter import com.intel.analytics.bigdl.dllib.utils.serializer.{DeserializeContext, ModuleSerializable, ModuleSerializer, SerializeContext} diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/SpatialShareConvolution.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/SpatialShareConvolution.scala index c2e4cfc3f13..b9dc7bfc2ec 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/SpatialShareConvolution.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/SpatialShareConvolution.scala @@ -21,7 +21,7 @@ import com.intel.analytics.bigdl.dllib.nn.abstractnn.Activity import com.intel.analytics.bigdl.dllib.optim.Regularizer import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric import com.intel.analytics.bigdl.dllib.tensor._ -import com.intel.analytics.bigdl.utils.Engine +import com.intel.analytics.bigdl.dllib.utils.Engine import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/Squeeze.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/Squeeze.scala index c7faf3ff587..7a15224236d 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/Squeeze.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/Squeeze.scala @@ -18,7 +18,7 @@ package com.intel.analytics.bigdl.dllib.nn import com.intel.analytics.bigdl.dllib.nn.abstractnn.{AbstractModule, TensorModule} import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.{NumericWildcard, TensorNumeric} -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import scala.collection.mutable.ArrayBuffer import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/Tanh.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/Tanh.scala index 5196a6dc9f8..da873521a2e 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/Tanh.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/Tanh.scala @@ -20,7 +20,7 @@ import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric import scala.math.tanh import com.intel.analytics.bigdl.dllib.tensor._ -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/TemporalConvolution.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/TemporalConvolution.scala index 437dbb14d12..de9d54f4daf 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/TemporalConvolution.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/TemporalConvolution.scala @@ -21,7 +21,7 @@ import com.intel.analytics.bigdl.dllib.optim.Regularizer import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric import com.intel.analytics.bigdl.dllib.utils.{T, Table} -import com.intel.analytics.bigdl.utils.Engine +import com.intel.analytics.bigdl.dllib.utils.Engine import scala.concurrent.Future import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/Threshold.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/Threshold.scala index 5b32165486b..d0b503f586d 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/Threshold.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/Threshold.scala @@ -19,8 +19,8 @@ package com.intel.analytics.bigdl.dllib.nn import com.intel.analytics.bigdl.dllib.nn.abstractnn.{IdentityOutputShape, TensorModule} import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric import com.intel.analytics.bigdl.dllib.tensor._ -import com.intel.analytics.bigdl.utils.Engine -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Engine +import com.intel.analytics.bigdl.dllib.utils.Shape import scala.collection.mutable.ArrayBuffer import scala.concurrent.Future diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/TimeDistributed.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/TimeDistributed.scala index 968d01edced..0d4472b5ebd 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/TimeDistributed.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/TimeDistributed.scala @@ -20,7 +20,7 @@ import com.intel.analytics.bigdl.Module import com.intel.analytics.bigdl.dllib.nn.abstractnn.{AbstractModule, Activity, TensorModule} import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import com.intel.analytics.bigdl.dllib.utils.Table import com.intel.analytics.bigdl.dllib.utils.serializer.{DeserializeContext, ModuleSerializable} import com.intel.analytics.bigdl.dllib.utils.serializer.converters.DataConverter diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/TimeDistributedCriterion.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/TimeDistributedCriterion.scala index d60fb866019..9895c34e767 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/TimeDistributedCriterion.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/TimeDistributedCriterion.scala @@ -19,7 +19,7 @@ package com.intel.analytics.bigdl.dllib.nn import com.intel.analytics.bigdl.dllib.nn.abstractnn.TensorCriterion import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.Engine +import com.intel.analytics.bigdl.dllib.utils.Engine import scala.collection.mutable.ArrayBuffer import scala.concurrent.Future diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/TimeDistributedMaskCriterion.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/TimeDistributedMaskCriterion.scala index e203b0a28af..88dc05c9d36 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/TimeDistributedMaskCriterion.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/TimeDistributedMaskCriterion.scala @@ -19,7 +19,7 @@ package com.intel.analytics.bigdl.dllib.nn import com.intel.analytics.bigdl.dllib.nn.abstractnn.{SizeAverageStatus, TensorCriterion} import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.Engine +import com.intel.analytics.bigdl.dllib.utils.Engine import scala.collection.mutable.ArrayBuffer import scala.concurrent.Future diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/TransformerOperation.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/TransformerOperation.scala index e3e0eb23980..71b74825dcb 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/TransformerOperation.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/TransformerOperation.scala @@ -24,7 +24,7 @@ import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric import com.intel.analytics.bigdl.dllib.tensor.{Storage, Tensor} import com.intel.analytics.bigdl.dllib.utils.{T} -import com.intel.analytics.bigdl.utils.{EngineType} +import com.intel.analytics.bigdl.dllib.utils.{EngineType} import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/Transpose.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/Transpose.scala index e6ff556380f..1ae2b0f7af1 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/Transpose.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/Transpose.scala @@ -22,7 +22,7 @@ import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.{NumericWildcard import com.intel.analytics.bigdl.dllib.utils.serializer._ import com.intel.analytics.bigdl.dllib.utils.serializer.converters.DataConverter import com.intel.analytics.bigdl.serialization.Bigdl.{AttrValue, BigDLModule} -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import scala.reflect.ClassTag import scala.reflect.runtime.universe diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/UpSampling1D.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/UpSampling1D.scala index 537a66eadf6..0219399959f 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/UpSampling1D.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/UpSampling1D.scala @@ -19,7 +19,7 @@ package com.intel.analytics.bigdl.dllib.nn import com.intel.analytics.bigdl.dllib.nn.abstractnn.TensorModule import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/UpSampling2D.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/UpSampling2D.scala index 5c50860922b..0502ab95463 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/UpSampling2D.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/UpSampling2D.scala @@ -19,7 +19,7 @@ package com.intel.analytics.bigdl.dllib.nn import com.intel.analytics.bigdl.dllib.nn.abstractnn.{DataFormat, TensorModule} import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/UpSampling3D.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/UpSampling3D.scala index 87eb218fef6..a5160d7b319 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/UpSampling3D.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/UpSampling3D.scala @@ -19,7 +19,7 @@ package com.intel.analytics.bigdl.dllib.nn import com.intel.analytics.bigdl.dllib.nn.abstractnn.TensorModule import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/View.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/View.scala index bfbabfad6b2..ea402643f9f 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/View.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/View.scala @@ -19,7 +19,7 @@ package com.intel.analytics.bigdl.dllib.nn import com.intel.analytics.bigdl.dllib.nn.abstractnn.TensorModule import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/VolumetricConvolution.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/VolumetricConvolution.scala index a413bf4e6e6..84d269565f8 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/VolumetricConvolution.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/VolumetricConvolution.scala @@ -20,10 +20,10 @@ import com.intel.analytics.bigdl.dllib.nn.abstractnn.{Initializable, TensorModul import com.intel.analytics.bigdl.dllib.optim.Regularizer import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric import com.intel.analytics.bigdl.dllib.tensor.{DoubleType, FloatType, Tensor} -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import com.intel.analytics.bigdl.dllib.utils.{T, Table} import org.apache.spark.sql.catalyst.optimizer.OptimizeIn -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/abstractnn/AbstractModule.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/abstractnn/AbstractModule.scala index f5e5363efe6..f2c8fcd1896 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/abstractnn/AbstractModule.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/abstractnn/AbstractModule.scala @@ -28,7 +28,7 @@ import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric import com.intel.analytics.bigdl.dllib.tensor.{QuantizedTensor, Tensor, TensorDataType} import com.intel.analytics.bigdl.dllib.feature.transform.vision.image.{DistributedImageFrame, ImageFeature, ImageFrame, LocalImageFrame} import com.intel.analytics.bigdl.dllib.utils.TorchObject.TYPE_MODULE -import com.intel.analytics.bigdl.utils._ +import com.intel.analytics.bigdl.dllib.utils._ import com.intel.analytics.bigdl.dllib.utils._ import com.intel.analytics.bigdl.dllib.utils.caffe.CaffePersister import com.intel.analytics.bigdl.dllib.utils.intermediate.ConversionUtils diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/abstractnn/IdentityOutputShape.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/abstractnn/IdentityOutputShape.scala index 7d4b865f0e5..e90d361f510 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/abstractnn/IdentityOutputShape.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/abstractnn/IdentityOutputShape.scala @@ -17,7 +17,7 @@ package com.intel.analytics.bigdl.dllib.nn.abstractnn -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape trait IdentityOutputShape extends InferShape{ override def computeOutputShape(inputShape: Shape): Shape = inputShape diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/abstractnn/InferShape.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/abstractnn/InferShape.scala index ee080b0ce91..7cb21e5b67a 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/abstractnn/InferShape.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/abstractnn/InferShape.scala @@ -18,7 +18,7 @@ package com.intel.analytics.bigdl.dllib.nn.abstractnn import com.intel.analytics.bigdl.dllib.keras.{Input => KInput, Sequential => KSequential} import com.intel.analytics.bigdl.dllib.nn.{Input => TInput} -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import scala.language.existentials import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/mkldnn/BlasWrapper.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/mkldnn/BlasWrapper.scala index bd28781af60..9c91552bd8b 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/mkldnn/BlasWrapper.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/mkldnn/BlasWrapper.scala @@ -26,9 +26,9 @@ import com.intel.analytics.bigdl.dllib.nn.abstractnn.{AbstractModule, Activity, import com.intel.analytics.bigdl.dllib.nn.mkldnn.Phase.{InferencePhase, TrainingPhase} import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.Engine +import com.intel.analytics.bigdl.dllib.utils.Engine import com.intel.analytics.bigdl.dllib.utils.{Util => NNUtils, _} -import com.intel.analytics.bigdl.utils._ +import com.intel.analytics.bigdl.dllib.utils._ import org.apache.log4j.Logger /** diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/mkldnn/Perf.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/mkldnn/Perf.scala index 19b63f3f252..220cf3e23a3 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/mkldnn/Perf.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/mkldnn/Perf.scala @@ -27,10 +27,10 @@ import com.intel.analytics.bigdl.dllib.nn.mkldnn.models.Vgg_16 import com.intel.analytics.bigdl.numeric.NumericFloat import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.RandomGenerator._ -import com.intel.analytics.bigdl.utils.ThreadPool +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.ThreadPool import com.intel.analytics.bigdl.dllib.utils.{T, Table} -import com.intel.analytics.bigdl.utils.{Engine, OptimizerV1, OptimizerV2} +import com.intel.analytics.bigdl.dllib.utils.{Engine, OptimizerV1, OptimizerV2} import org.apache.log4j.Logger import scopt.OptionParser diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/mkldnn/SoftMax.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/mkldnn/SoftMax.scala index 8cb2bfd7284..ad8d70bd7a0 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/mkldnn/SoftMax.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/mkldnn/SoftMax.scala @@ -22,7 +22,7 @@ import com.intel.analytics.bigdl.dllib.nn.abstractnn.Activity import com.intel.analytics.bigdl.dllib.nn.mkldnn.Phase.{InferencePhase, TrainingPhase} import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric import com.intel.analytics.bigdl.dllib.tensor.{DenseType, Tensor} -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import scala.collection.mutable.ArrayBuffer diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/ops/CategoricalColVocaList.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/ops/CategoricalColVocaList.scala index 276ebe9b34a..dc036719be5 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/ops/CategoricalColVocaList.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/ops/CategoricalColVocaList.scala @@ -17,7 +17,7 @@ package com.intel.analytics.bigdl.dllib.nn.ops import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.HashFunc +import com.intel.analytics.bigdl.dllib.utils.HashFunc import scala.collection.mutable.ArrayBuffer import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/ops/RandomUniform.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/ops/RandomUniform.scala index bf4182189e9..969b7d9b130 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/ops/RandomUniform.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/ops/RandomUniform.scala @@ -18,7 +18,7 @@ package com.intel.analytics.bigdl.dllib.nn.ops import com.intel.analytics.bigdl.dllib.nn.abstractnn.{AbstractModule, Activity} import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric import com.intel.analytics.bigdl.dllib.tensor._ -import com.intel.analytics.bigdl.utils.RandomGenerator +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator import com.intel.analytics.bigdl.dllib.utils.serializer.converters.DataConverter import com.intel.analytics.bigdl.dllib.utils.serializer.{DeserializeContext, ModuleSerializable, SerializeContext} import com.intel.analytics.bigdl.serialization.Bigdl.{AttrValue, BigDLModule} diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/ops/Tile.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/ops/Tile.scala index be72b01cbcc..7b4f9c971eb 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/ops/Tile.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/ops/Tile.scala @@ -19,7 +19,7 @@ import com.intel.analytics.bigdl.dllib.nn.abstractnn.Activity import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath._ import com.intel.analytics.bigdl.dllib.utils.{T, Table} -import com.intel.analytics.bigdl.utils.{Engine, OptimizerV1, OptimizerV2} +import com.intel.analytics.bigdl.dllib.utils.{Engine, OptimizerV1, OptimizerV2} import scala.concurrent.Future import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/optim/AbstractOptimizer.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/optim/AbstractOptimizer.scala index 5a48d5dbd92..96bd70b5693 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/optim/AbstractOptimizer.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/optim/AbstractOptimizer.scala @@ -24,7 +24,7 @@ import com.intel.analytics.bigdl.dllib.optim.parameters.AllReduceParameter import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric import com.intel.analytics.bigdl.dllib.utils.intermediate.IRGraph -import com.intel.analytics.bigdl.utils.{Engine, MklBlas, MklDnn} +import com.intel.analytics.bigdl.dllib.utils.{Engine, MklBlas, MklDnn} import com.intel.analytics.bigdl.dllib.utils.{Table} import com.intel.analytics.bigdl.dllib.visualization.{TrainSummary, ValidationSummary} import org.apache.spark.rdd.{RDD, ZippedPartitionsWithLocalityRDD} diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/optim/DistriOptimizer.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/optim/DistriOptimizer.scala index e18ffde0838..4eed1b7a693 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/optim/DistriOptimizer.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/optim/DistriOptimizer.scala @@ -27,7 +27,7 @@ import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric import com.intel.analytics.bigdl.dllib.utils._ import com.intel.analytics.bigdl.dllib.utils.intermediate.{ConversionUtils, IRGraph} import com.intel.analytics.bigdl.dllib.visualization.{TrainSummary, ValidationSummary} -import com.intel.analytics.bigdl.utils._ +import com.intel.analytics.bigdl.dllib.utils._ import com.intel.analytics.bigdl.{Module, _} import java.io.{File, FilenameFilter} import java.text.SimpleDateFormat diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/optim/DistriOptimizerV2.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/optim/DistriOptimizerV2.scala index 428a323f71b..172aa4ba28e 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/optim/DistriOptimizerV2.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/optim/DistriOptimizerV2.scala @@ -26,7 +26,7 @@ import com.intel.analytics.bigdl.dllib.nn.{Container, Module} import com.intel.analytics.bigdl.dllib.optim.parameters.{AllReduceParameter, ParameterProcessor} import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils._ +import com.intel.analytics.bigdl.dllib.utils._ import com.intel.analytics.bigdl.dllib.utils._ import com.intel.analytics.bigdl.dllib.utils.intermediate.ConversionUtils import com.intel.analytics.bigdl.dllib.visualization.{TrainSummary, ValidationSummary} diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/optim/DistriValidator.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/optim/DistriValidator.scala index 3c77066e4c8..42ac3f4300d 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/optim/DistriValidator.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/optim/DistriValidator.scala @@ -19,7 +19,7 @@ package com.intel.analytics.bigdl.dllib.optim import com.intel.analytics.bigdl._ import com.intel.analytics.bigdl.dllib.feature.dataset.{DistributedDataSet, MiniBatch} import com.intel.analytics.bigdl.dllib.optim.DistriValidator._ -import com.intel.analytics.bigdl.utils.{Engine, MklBlas} +import com.intel.analytics.bigdl.dllib.utils.{Engine, MklBlas} import org.apache.log4j.Logger object DistriValidator { diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/optim/Evaluator.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/optim/Evaluator.scala index e2d2f14064d..5bb590ce070 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/optim/Evaluator.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/optim/Evaluator.scala @@ -20,7 +20,7 @@ import com.intel.analytics.bigdl._ import com.intel.analytics.bigdl.dllib.feature.dataset.{MiniBatch, Sample, SampleToMiniBatch} import com.intel.analytics.bigdl.dllib.models.utils.ModelBroadcast import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.{Engine, MklDnn} +import com.intel.analytics.bigdl.dllib.utils.{Engine, MklDnn} import com.intel.analytics.bigdl.dllib.utils.intermediate.ConversionUtils import org.apache.spark.rdd import org.apache.spark.rdd.RDD diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/optim/LocalOptimizer.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/optim/LocalOptimizer.scala index eaa379f74b4..595963af886 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/optim/LocalOptimizer.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/optim/LocalOptimizer.scala @@ -27,7 +27,7 @@ import com.intel.analytics.bigdl.dllib.nn.mkldnn.Phase.{InferencePhase, Training import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric import com.intel.analytics.bigdl.dllib.utils._ -import com.intel.analytics.bigdl.utils._ +import com.intel.analytics.bigdl.dllib.utils._ import org.apache.log4j.Logger import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/optim/LocalPredictor.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/optim/LocalPredictor.scala index 23400f2f6a3..162bebcdf96 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/optim/LocalPredictor.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/optim/LocalPredictor.scala @@ -26,7 +26,7 @@ import com.intel.analytics.bigdl.dllib.feature.transform.vision.image.{ImageFeat import com.intel.analytics.bigdl.dllib.utils.Util._ import com.intel.analytics.bigdl.dllib.utils.intermediate.ConversionUtils import com.intel.analytics.bigdl.dllib.utils.{Util} -import com.intel.analytics.bigdl.utils.{Engine, MklBlas, MklDnn} +import com.intel.analytics.bigdl.dllib.utils.{Engine, MklBlas, MklDnn} import org.apache.log4j.Logger import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/optim/LocalValidator.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/optim/LocalValidator.scala index 4d83f7feb66..5169fed5b48 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/optim/LocalValidator.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/optim/LocalValidator.scala @@ -19,7 +19,7 @@ package com.intel.analytics.bigdl.dllib.optim import com.intel.analytics.bigdl.dllib.feature.dataset.{LocalDataSet, MiniBatch} import com.intel.analytics.bigdl._ import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.{Engine, MklBlas} +import com.intel.analytics.bigdl.dllib.utils.{Engine, MklBlas} import org.apache.log4j.Logger object LocalValidator { diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/optim/Optimizer.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/optim/Optimizer.scala index f8ccae9aa40..81956035967 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/optim/Optimizer.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/optim/Optimizer.scala @@ -27,7 +27,7 @@ import com.intel.analytics.bigdl.dllib.optim.parameters.{ConstantClippingProcess import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric import com.intel.analytics.bigdl.dllib.utils._ import com.intel.analytics.bigdl.dllib.visualization.{TrainSummary, ValidationSummary} -import com.intel.analytics.bigdl.utils._ +import com.intel.analytics.bigdl.dllib.utils._ import org.apache.log4j.Logger import org.apache.spark.rdd.RDD diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/optim/ParallelAdam.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/optim/ParallelAdam.scala index e6cd9b648d3..57989e71ab4 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/optim/ParallelAdam.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/optim/ParallelAdam.scala @@ -18,7 +18,7 @@ package com.intel.analytics.bigdl.dllib.optim import com.intel.analytics.bigdl.dllib.tensor.{Storage, Tensor} import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.{Engine} +import com.intel.analytics.bigdl.dllib.utils.{Engine} import com.intel.analytics.bigdl.dllib.utils.{T, Table} import org.apache.log4j.Logger diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/optim/ParallelOptimizer.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/optim/ParallelOptimizer.scala index 4bae6b6b9ed..eb05d72b2a2 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/optim/ParallelOptimizer.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/optim/ParallelOptimizer.scala @@ -26,7 +26,7 @@ import com.intel.analytics.bigdl.dllib.optim.parameters.AllReduceParameter import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric import com.intel.analytics.bigdl.dllib.utils._ -import com.intel.analytics.bigdl.utils._ +import com.intel.analytics.bigdl.dllib.utils._ import com.intel.analytics.bigdl.dllib.visualization.{TrainSummary, ValidationSummary} import com.intel.analytics.bigdl.{Module, _} import java.io.{File, FilenameFilter} diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/optim/PredictionService.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/optim/PredictionService.scala index cc8fe44dc8a..cd1eebaec6e 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/optim/PredictionService.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/optim/PredictionService.scala @@ -25,7 +25,7 @@ import com.intel.analytics.bigdl.serialization.Bigdl.{AttrValue, BigDLTensor, Da import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric.{NumericBoolean, NumericChar, NumericDouble, NumericFloat, NumericInt, NumericLong, NumericString} -import com.intel.analytics.bigdl.utils._ +import com.intel.analytics.bigdl.dllib.utils._ import com.intel.analytics.bigdl.dllib.utils._ import com.intel.analytics.bigdl.dllib.utils.serializer.converters.DataConverter import com.intel.analytics.bigdl.dllib.utils.serializer.{DeserializeContext, ModuleSerializer, ProtoStorageType, SerializeContext} diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/optim/Predictor.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/optim/Predictor.scala index 031d6bf4037..b7e5509547d 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/optim/Predictor.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/optim/Predictor.scala @@ -28,7 +28,7 @@ import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric import com.intel.analytics.bigdl.dllib.feature.transform.vision.image.{DistributedImageFrame, ImageFeature, ImageFrame} import com.intel.analytics.bigdl.dllib.utils._ import com.intel.analytics.bigdl.dllib.utils.intermediate.{ConversionUtils, IRGraph} -import com.intel.analytics.bigdl.utils._ +import com.intel.analytics.bigdl.dllib.utils._ import org.apache.spark.rdd.RDD import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/optim/Validator.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/optim/Validator.scala index 98072e832b7..cadd5e6e108 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/optim/Validator.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/optim/Validator.scala @@ -17,7 +17,7 @@ package com.intel.analytics.bigdl.dllib.optim import com.intel.analytics.bigdl._ -import com.intel.analytics.bigdl.utils.Engine +import com.intel.analytics.bigdl.dllib.utils.Engine import com.intel.analytics.bigdl.dllib.feature.dataset.{DistributedDataSet, LocalDataSet, MiniBatch} import org.apache.log4j.Logger diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/optim/parameters/AllReduceParameter.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/optim/parameters/AllReduceParameter.scala index 8696be3845e..6038a5d6d50 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/optim/parameters/AllReduceParameter.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/optim/parameters/AllReduceParameter.scala @@ -17,7 +17,7 @@ package com.intel.analytics.bigdl.dllib.optim.parameters import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.Engine +import com.intel.analytics.bigdl.dllib.utils.Engine import java.util.concurrent._ import java.util.concurrent.atomic.AtomicLong import org.apache.commons.lang.exception.ExceptionUtils diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/optim/parameters/FP16CompressedTensor.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/optim/parameters/FP16CompressedTensor.scala index d0c1e1eaa7c..c5a01ee2e7e 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/optim/parameters/FP16CompressedTensor.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/optim/parameters/FP16CompressedTensor.scala @@ -19,7 +19,7 @@ package com.intel.analytics.bigdl.dllib.optim.parameters import java.nio.ByteBuffer import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.Engine +import com.intel.analytics.bigdl.dllib.utils.Engine import scala.reflect._ diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/optim/parameters/UncompressedTensor.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/optim/parameters/UncompressedTensor.scala index 4574d8a1656..c2b3fd617c0 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/optim/parameters/UncompressedTensor.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/optim/parameters/UncompressedTensor.scala @@ -19,7 +19,7 @@ package com.intel.analytics.bigdl.dllib.optim.parameters import java.nio.ByteBuffer import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.Engine +import com.intel.analytics.bigdl.dllib.utils.Engine import scala.reflect._ diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/optim/parameters/Util.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/optim/parameters/Util.scala index 72634245a87..b59a529a960 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/optim/parameters/Util.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/optim/parameters/Util.scala @@ -16,7 +16,7 @@ package com.intel.analytics.bigdl.dllib.optim.parameters -import com.intel.analytics.bigdl.utils.Engine +import com.intel.analytics.bigdl.dllib.utils.Engine import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/tensor/DenseTensor.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/tensor/DenseTensor.scala index d9d4f33a4c8..f8a39b966f0 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/tensor/DenseTensor.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/tensor/DenseTensor.scala @@ -21,7 +21,7 @@ import java.util.Comparator import breeze.linalg.{DenseMatrix => BrzDenseMatrix, DenseVector => BrzDenseVector} import com.intel.analytics.bigdl.mkl.MKL import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import com.intel.analytics.bigdl.dllib.utils.{File, Table} import org.apache.spark.mllib.linalg.{DenseMatrix, DenseVector, Matrix, Vector} diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/tensor/TensorNumeric.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/tensor/TensorNumeric.scala index ba688e50f3a..ec2b01c9bb0 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/tensor/TensorNumeric.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/tensor/TensorNumeric.scala @@ -19,7 +19,7 @@ package com.intel.analytics.bigdl.dllib.tensor import java.util import com.intel.analytics.bigdl.mkl.MKL -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ /** * class of math operation diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/DistriParameterSynchronizer.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/DistriParameterSynchronizer.scala index df68288982b..e0b0933b833 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/DistriParameterSynchronizer.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/DistriParameterSynchronizer.scala @@ -24,7 +24,7 @@ import com.intel.analytics.bigdl.mkl.hardware.{Affinity, CpuInfo} import com.intel.analytics.bigdl.dllib.optim.parameters.{CompressedTensor, FP16CompressedTensor, SerializerInstance} import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils._ +import com.intel.analytics.bigdl.dllib.utils._ import org.apache.commons.lang.exception.ExceptionUtils import org.apache.log4j.Logger import org.apache.spark.sparkExtension.SparkExtension diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/Engine.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/Engine.scala index 737d1edfac8..1b75560367f 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/Engine.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/Engine.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.utils +package com.intel.analytics.bigdl.dllib.utils import java.io.{FileOutputStream, InputStream, PrintWriter} import java.util.Locale @@ -22,7 +22,7 @@ import java.util.concurrent.atomic.AtomicBoolean import org.apache.log4j.Logger import org.apache.spark._ -import com.intel.analytics.bigdl.utils._ +import com.intel.analytics.bigdl.dllib.utils._ import com.intel.analytics.bigdl.mkl.MKL import com.intel.analytics.bigdl.mkl.hardware.{Affinity, CpuInfo} import org.apache.spark.utils.SparkUtils diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/HashFunc.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/HashFunc.scala index a452ae9ed2e..31ada82a05e 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/HashFunc.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/HashFunc.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.utils +package com.intel.analytics.bigdl.dllib.utils import scala.util.hashing.MurmurHash3 diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/LayerException.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/LayerException.scala index fb8d4977438..3912868638f 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/LayerException.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/LayerException.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intel.analytics.bigdl.utils +package com.intel.analytics.bigdl.dllib.utils import org.apache.commons.lang.exception.ExceptionUtils diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/LoggerFilter.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/LoggerFilter.scala index 5d038caf52a..1411db1bc46 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/LoggerFilter.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/LoggerFilter.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.utils +package com.intel.analytics.bigdl.dllib.utils import org.apache.log4j._ import java.nio.file.{Paths, Files} diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/RandomGenerator.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/RandomGenerator.scala index 5447453f743..ea63a675b89 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/RandomGenerator.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/RandomGenerator.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.utils +package com.intel.analytics.bigdl.dllib.utils import java.io.{BufferedInputStream, FileInputStream} import java.nio.ByteBuffer diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/Shape.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/Shape.scala index 097a625314e..3777a3bd2ed 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/Shape.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/Shape.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.utils +package com.intel.analytics.bigdl.dllib.utils import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/ThreadPool.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/ThreadPool.scala index 75b5e30f48b..32ce1f7e54f 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/ThreadPool.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/ThreadPool.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.utils +package com.intel.analytics.bigdl.dllib.utils import java.util.concurrent._ diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/intermediate/ConversionUtils.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/intermediate/ConversionUtils.scala index 7535ef076af..306dd7f876a 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/intermediate/ConversionUtils.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/intermediate/ConversionUtils.scala @@ -20,7 +20,7 @@ import com.intel.analytics.bigdl._ import com.intel.analytics.bigdl.dllib.nn.mkldnn.{DnnGraph, MklDnnContainer} import com.intel.analytics.bigdl.dllib.nn.mkldnn.{DnnGraph, MklDnnLayer, MklDnnModule} import com.intel.analytics.bigdl.dllib.utils.{T} -import com.intel.analytics.bigdl.utils._ +import com.intel.analytics.bigdl.dllib.utils._ import org.apache.spark.rdd.RDD import com.intel.analytics.bigdl.dllib.nn.Graph import com.intel.analytics.bigdl.dllib.nn.StaticGraph diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/intermediate/IRConverter.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/intermediate/IRConverter.scala index 2a2e533cb1a..b92db245685 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/intermediate/IRConverter.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/intermediate/IRConverter.scala @@ -24,7 +24,7 @@ import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric import com.intel.analytics.bigdl.Module import com.intel.analytics.bigdl.dllib.utils import com.intel.analytics.bigdl.dllib.utils.{Node} -import com.intel.analytics.bigdl.utils._ +import com.intel.analytics.bigdl.dllib.utils._ import scala.collection.mutable.ArrayBuffer import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/intermediate/IRGraph.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/intermediate/IRGraph.scala index 2c36dc31229..1ce4f371f97 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/intermediate/IRGraph.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/intermediate/IRGraph.scala @@ -24,7 +24,7 @@ import com.intel.analytics.bigdl.dllib.nn.mkldnn._ import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric import com.intel.analytics.bigdl.dllib.utils._ -import com.intel.analytics.bigdl.utils._ +import com.intel.analytics.bigdl.dllib.utils._ import scala.reflect.ClassTag diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/python/api/PythonBigDL.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/python/api/PythonBigDL.scala index 06b0a1b4155..6eace3f7e23 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/python/api/PythonBigDL.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/python/api/PythonBigDL.scala @@ -29,7 +29,7 @@ import com.intel.analytics.bigdl.dllib.tensor.{DenseType, SparseType, Storage, T import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric import com.intel.analytics.bigdl.dllib.utils.{Table, _} import com.intel.analytics.bigdl.dllib.visualization.{Summary, TrainSummary, ValidationSummary} -import com.intel.analytics.bigdl.utils._ +import com.intel.analytics.bigdl.dllib.utils._ import org.apache.spark.api.java.{JavaRDD, JavaSparkContext} import org.apache.spark.rdd.RDD import java.lang.{Boolean => JBoolean} diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/python/api/PythonBigDLKeras.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/python/api/PythonBigDLKeras.scala index 538dc019408..c28277b84c7 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/python/api/PythonBigDLKeras.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/python/api/PythonBigDLKeras.scala @@ -30,7 +30,7 @@ import com.intel.analytics.bigdl.dllib.optim.{OptimMethod, Regularizer, Validati import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric import com.intel.analytics.bigdl.dllib.feature.transform.vision.image.{ImageFeature, ImageFeatureToMiniBatch} -import com.intel.analytics.bigdl.utils.{Engine, MultiShape, Shape, SingleShape} +import com.intel.analytics.bigdl.dllib.utils.{Engine, MultiShape, Shape, SingleShape} import org.apache.spark.api.java.JavaRDD import scala.collection.JavaConverters._ diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/serializer/ModuleSerializable.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/serializer/ModuleSerializable.scala index 0331ea3febb..c2e29d252cb 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/serializer/ModuleSerializable.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/serializer/ModuleSerializable.scala @@ -25,7 +25,7 @@ import com.intel.analytics.bigdl.serialization.Bigdl.AttrValue.ArrayValue import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric import com.intel.analytics.bigdl.dllib.utils.{ReflectionUtils, Table} -import com.intel.analytics.bigdl.utils.{Shape => BigDLShape} +import com.intel.analytics.bigdl.dllib.utils.{Shape => BigDLShape} import com.intel.analytics.bigdl.dllib.utils.serializer.converters.{DataConverter, ShapeConverter, TensorConverter} import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializer._ import com.intel.analytics.bigdl.serialization.Bigdl._ diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/serializer/converters/DataConverter.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/serializer/converters/DataConverter.scala index 270552d2107..69120d2cc8d 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/serializer/converters/DataConverter.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/serializer/converters/DataConverter.scala @@ -25,7 +25,7 @@ import com.intel.analytics.bigdl.dllib.optim.Regularizer import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric import com.intel.analytics.bigdl.dllib.utils.serializer._ -import com.intel.analytics.bigdl.utils.{MultiShape, SingleShape, Shape => BigDLShape} +import com.intel.analytics.bigdl.dllib.utils.{MultiShape, SingleShape, Shape => BigDLShape} import com.intel.analytics.bigdl.serialization.Bigdl._ import com.intel.analytics.bigdl.serialization.Bigdl.AttrValue.ArrayValue diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/serializer/converters/ShapeConverter.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/serializer/converters/ShapeConverter.scala index 9cec9e7cfa5..71c3e6e8ee4 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/serializer/converters/ShapeConverter.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/serializer/converters/ShapeConverter.scala @@ -18,7 +18,7 @@ package com.intel.analytics.bigdl.dllib.utils.serializer.converters import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath import com.intel.analytics.bigdl.dllib.utils.serializer.{DeserializeContext, SerializeContext} -import com.intel.analytics.bigdl.utils.{MultiShape, SingleShape, Shape => BigDLShape} +import com.intel.analytics.bigdl.dllib.utils.{MultiShape, SingleShape, Shape => BigDLShape} import com.intel.analytics.bigdl.serialization.Bigdl import com.intel.analytics.bigdl.serialization.Bigdl.Shape.ShapeType import com.intel.analytics.bigdl.serialization.Bigdl.{AttrValue, BigDLModule, DataType, Shape} diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/tf/Session.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/tf/Session.scala index 857f42f4423..aa980910649 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/tf/Session.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/tf/Session.scala @@ -25,7 +25,7 @@ import com.intel.analytics.bigdl.dllib.optim._ import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.{NumericWildcard, TensorNumeric} import com.intel.analytics.bigdl.dllib.tensor.{Storage, Tensor} import com.intel.analytics.bigdl.dllib.utils._ -import com.intel.analytics.bigdl.utils._ +import com.intel.analytics.bigdl.dllib.utils._ import org.apache.spark.SparkContext import org.apache.spark.api.java.JavaRDD import org.apache.spark.rdd.RDD diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/tf/Tensorflow.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/tf/Tensorflow.scala index 7f7554b5615..428ea59957d 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/tf/Tensorflow.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/tf/Tensorflow.scala @@ -22,7 +22,7 @@ import com.google.protobuf.ByteString import com.intel.analytics.bigdl.dllib.nn.{Graph, Module} import com.intel.analytics.bigdl.dllib.nn.abstractnn.AbstractModule import com.intel.analytics.bigdl.dllib.tensor._ -import com.intel.analytics.bigdl.utils.Engine +import com.intel.analytics.bigdl.dllib.utils.Engine import org.tensorflow.framework.AttrValue.ListValue import org.tensorflow.framework._ import org.tensorflow.framework.TensorShapeProto.Dim diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/visualization/tensorboard/FileWriter.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/visualization/tensorboard/FileWriter.scala index 80ef4eba4ca..68125ecb34f 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/visualization/tensorboard/FileWriter.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/visualization/tensorboard/FileWriter.scala @@ -16,7 +16,7 @@ package com.intel.analytics.bigdl.dllib.visualization.tensorboard -import com.intel.analytics.bigdl.utils.Engine +import com.intel.analytics.bigdl.dllib.utils.Engine import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.Path import org.tensorflow diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/dataset/BatchPaddingSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/dataset/BatchPaddingSpec.scala index bc756e8d05b..e245114dcc8 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/dataset/BatchPaddingSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/dataset/BatchPaddingSpec.scala @@ -21,7 +21,7 @@ import java.util import com.intel.analytics.bigdl.dllib.feature.dataset.text._ import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric import com.intel.analytics.bigdl.dllib.tensor.{DoubleType, FloatType, Storage, Tensor} -import com.intel.analytics.bigdl.utils.Engine +import com.intel.analytics.bigdl.dllib.utils.Engine import org.scalatest.{BeforeAndAfter, FlatSpec, Matchers} import scala.collection.Iterator diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/dataset/DataSetSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/dataset/DataSetSpec.scala index 16b1c76179e..69281f4dd97 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/dataset/DataSetSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/dataset/DataSetSpec.scala @@ -26,8 +26,8 @@ import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.feature.transform.vision.image.{ImageFeature, RoiImageInfo} import com.intel.analytics.bigdl.dllib.feature.transform.vision.image.label.roi.RoiLabel import com.intel.analytics.bigdl.dllib.utils.{SparkContextLifeCycle, TestUtils} -import com.intel.analytics.bigdl.utils.RandomGenerator -import com.intel.analytics.bigdl.utils._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator +import com.intel.analytics.bigdl.dllib.utils._ import java.awt.image.DataBufferByte import javax.imageio.ImageIO import org.apache.hadoop.io.Text diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/dataset/TransformersSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/dataset/TransformersSpec.scala index b39d08998fe..d4b765cd608 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/dataset/TransformersSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/dataset/TransformersSpec.scala @@ -23,10 +23,10 @@ import com.intel.analytics.bigdl.dllib.feature.dataset.DataSet.SeqFileFolder import com.intel.analytics.bigdl.dllib.feature.dataset.image._ import com.intel.analytics.bigdl.dllib.feature.dataset.text.{LabeledSentence, LabeledSentenceToSample} import com.intel.analytics.bigdl.dllib.tensor.{Storage, Tensor} -import com.intel.analytics.bigdl.utils.RandomGenerator.RNG +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator.RNG import com.intel.analytics.bigdl.dllib.utils.{TestUtils} -import com.intel.analytics.bigdl.utils._ -import com.intel.analytics.bigdl.utils.RandomGenerator +import com.intel.analytics.bigdl.dllib.utils._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.Path import org.apache.hadoop.io.{SequenceFile, Text} diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/dataset/text/DictionarySpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/dataset/text/DictionarySpec.scala index 65c332b985f..39824add17b 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/dataset/text/DictionarySpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/dataset/text/DictionarySpec.scala @@ -19,7 +19,7 @@ package com.intel.analytics.bigdl.dllib.feature.dataset.text import java.io.PrintWriter import com.intel.analytics.bigdl.dllib.feature.dataset.DataSet -import com.intel.analytics.bigdl.utils.Engine +import com.intel.analytics.bigdl.dllib.utils.Engine import com.intel.analytics.bigdl.dllib.utils.SparkContextLifeCycle import org.apache.spark.{SparkConf, SparkContext} import org.scalatest.{BeforeAndAfter, FlatSpec, Matchers} diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/dataset/text/SentenceBiPaddingSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/dataset/text/SentenceBiPaddingSpec.scala index 42b75f77c42..60c961d1f33 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/dataset/text/SentenceBiPaddingSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/dataset/text/SentenceBiPaddingSpec.scala @@ -21,7 +21,7 @@ import java.io.PrintWriter import com.intel.analytics.bigdl.dllib.feature.dataset.DataSet import com.intel.analytics.bigdl.dllib.feature.dataset.text.utils.SentenceToken import com.intel.analytics.bigdl.dllib.utils.{SparkContextLifeCycle} -import com.intel.analytics.bigdl.utils.{Engine} +import com.intel.analytics.bigdl.dllib.utils.{Engine} import org.apache.spark.SparkContext import org.scalatest.{BeforeAndAfter, FlatSpec, Matchers} diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/dataset/text/SentenceTokenizerSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/dataset/text/SentenceTokenizerSpec.scala index f46835942a5..7d74342acaf 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/dataset/text/SentenceTokenizerSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/dataset/text/SentenceTokenizerSpec.scala @@ -20,7 +20,7 @@ import java.io.PrintWriter import com.intel.analytics.bigdl.dllib.feature.dataset.DataSet import com.intel.analytics.bigdl.dllib.utils.{SparkContextLifeCycle} -import com.intel.analytics.bigdl.utils.{Engine} +import com.intel.analytics.bigdl.dllib.utils.{Engine} import org.apache.spark.{SparkConf, SparkContext} import org.scalatest.{FlatSpec, Matchers} diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/dataset/text/TextToLabeledSentenceSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/dataset/text/TextToLabeledSentenceSpec.scala index 0059932ed87..3f88986a352 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/dataset/text/TextToLabeledSentenceSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/dataset/text/TextToLabeledSentenceSpec.scala @@ -20,7 +20,7 @@ import java.io.PrintWriter import com.intel.analytics.bigdl.dllib.feature.dataset.DataSet import com.intel.analytics.bigdl.dllib.utils.{SparkContextLifeCycle} -import com.intel.analytics.bigdl.utils.{Engine} +import com.intel.analytics.bigdl.dllib.utils.{Engine} import org.apache.spark.SparkContext import org.scalatest.{BeforeAndAfter, FlatSpec, Matchers} diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/HdfsSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/HdfsSpec.scala index 785954cc83e..6b8a5db87ba 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/HdfsSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/HdfsSpec.scala @@ -33,7 +33,7 @@ import com.intel.analytics.bigdl.dllib.utils.tf._ import com.intel.analytics.bigdl.dllib.utils.{File} import com.intel.analytics.bigdl.dllib.visualization.Summary import com.intel.analytics.bigdl.dllib.visualization.tensorboard.{FileReader, FileWriter} -import com.intel.analytics.bigdl.utils._ +import com.intel.analytics.bigdl.dllib.utils._ import org.apache.commons.compress.utils.IOUtils import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.Path diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/Quantization.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/Quantization.scala index 561b9b7cfd6..6dbbeacdf58 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/Quantization.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/Quantization.scala @@ -25,8 +25,8 @@ import com.intel.analytics.bigdl.dllib.models.resnet.{Utils => ResNetUtils} import com.intel.analytics.bigdl.dllib.nn.Module import com.intel.analytics.bigdl.numeric.NumericFloat import com.intel.analytics.bigdl.dllib.optim.{Top1Accuracy, Top5Accuracy, ValidationMethod, ValidationResult} -import com.intel.analytics.bigdl.utils.{Engine} -import com.intel.analytics.bigdl.utils.LoggerFilter +import com.intel.analytics.bigdl.dllib.utils.{Engine} +import com.intel.analytics.bigdl.dllib.utils.LoggerFilter import org.apache.log4j.{Level, Logger} import org.apache.spark.SparkContext import org.apache.spark.rdd.RDD diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/AdagradSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/AdagradSpec.scala index 5244dbb0624..18b70c6bfef 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/AdagradSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/AdagradSpec.scala @@ -19,7 +19,7 @@ package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.optim.Adagrad import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.utils.{T} -import com.intel.analytics.bigdl.utils.RandomGenerator +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator @com.intel.analytics.bigdl.tags.Serial class AdagradSpec extends TorchSpec { diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/AddConstantSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/AddConstantSpec.scala index f9c8c3fa7cf..f7420c5407d 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/AddConstantSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/AddConstantSpec.scala @@ -17,7 +17,7 @@ package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.AddConstant import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ @com.intel.analytics.bigdl.tags.Serial class AddConstantSpec extends TorchSpec { diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/AddSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/AddSpec.scala index 9f92de43c93..f1406b99b31 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/AddSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/AddSpec.scala @@ -17,7 +17,7 @@ package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.Add import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import spire.syntax.module @com.intel.analytics.bigdl.tags.Serial diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/BatchNormalizationSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/BatchNormalizationSpec.scala index cbcbaac0b45..8f2ac29656a 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/BatchNormalizationSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/BatchNormalizationSpec.scala @@ -19,7 +19,7 @@ package com.intel.analytics.bigdl.dllib.integration.torch import breeze.numerics.abs import com.intel.analytics.bigdl.dllib.nn.{BatchNormalization, GradientChecker} import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import com.intel.analytics.bigdl._ import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/BiRecurrentSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/BiRecurrentSpec.scala index e078b2f8e58..0f65dbadf41 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/BiRecurrentSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/BiRecurrentSpec.scala @@ -24,7 +24,7 @@ import com.intel.analytics.bigdl.dllib.nn.abstractnn.AbstractModule import com.intel.analytics.bigdl.numeric.NumericDouble import com.intel.analytics.bigdl.dllib.optim.SGD import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import com.intel.analytics.bigdl.dllib.utils.{T, Table} import scala.sys.process._ diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/BilinearSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/BilinearSpec.scala index b6c4039ef55..356d0d5a9ce 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/BilinearSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/BilinearSpec.scala @@ -18,7 +18,7 @@ package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn._ import com.intel.analytics.bigdl.dllib.optim.{L2Regularizer, SGD} import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import com.intel.analytics.bigdl.dllib.utils.{T, Table} import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/BottleSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/BottleSpec.scala index 2c510956588..373c779425a 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/BottleSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/BottleSpec.scala @@ -18,7 +18,7 @@ package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.{Bottle, Linear} import com.intel.analytics.bigdl._ import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CAddSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CAddSpec.scala index 2bf24d5697d..2dbc27ded97 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CAddSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CAddSpec.scala @@ -19,7 +19,7 @@ package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn._ import com.intel.analytics.bigdl.dllib.optim.{L2Regularizer, SGD} import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import com.intel.analytics.bigdl.dllib.utils.T @com.intel.analytics.bigdl.tags.Serial diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CAddTableSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CAddTableSpec.scala index 02d18ee385b..e667a17de94 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CAddTableSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CAddTableSpec.scala @@ -18,7 +18,7 @@ package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.{CAddTable, ConcatTable, Linear, Sequential} import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CDivTableSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CDivTableSpec.scala index c242d576b48..9f40ead2f9b 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CDivTableSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CDivTableSpec.scala @@ -17,7 +17,7 @@ package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.CDivTable import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import com.intel.analytics.bigdl.dllib.utils.Table import scala.collection.mutable.HashMap diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CMaxTableSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CMaxTableSpec.scala index 435f737fc94..cce6472ae3b 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CMaxTableSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CMaxTableSpec.scala @@ -17,7 +17,7 @@ package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.CMaxTable import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import com.intel.analytics.bigdl.dllib.utils.Table import scala.collection.mutable.HashMap diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CMinTableSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CMinTableSpec.scala index c9421db3fa4..a8983af1ceb 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CMinTableSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CMinTableSpec.scala @@ -17,7 +17,7 @@ package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.CMinTable import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import com.intel.analytics.bigdl.dllib.utils.Table import scala.collection.mutable.HashMap diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CMulSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CMulSpec.scala index c583cfe0a15..deca23b2d7e 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CMulSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CMulSpec.scala @@ -19,7 +19,7 @@ package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn._ import com.intel.analytics.bigdl.dllib.optim.{L2Regularizer, SGD} import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import com.intel.analytics.bigdl.dllib.utils.T @com.intel.analytics.bigdl.tags.Serial diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CMulTableSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CMulTableSpec.scala index d22eeb72f58..22a32b1b71c 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CMulTableSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CMulTableSpec.scala @@ -16,7 +16,7 @@ package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import com.intel.analytics.bigdl.dllib.utils.Table import com.intel.analytics.bigdl.dllib.nn.CMulTable diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CSubTableSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CSubTableSpec.scala index 22f45b8299b..24a8128352c 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CSubTableSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CSubTableSpec.scala @@ -16,7 +16,7 @@ package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import com.intel.analytics.bigdl.dllib.utils.Table import com.intel.analytics.bigdl.dllib.nn.CSubTable diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ColorJitterSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ColorJitterSpec.scala index 08b683d54f8..5a9f95798e5 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ColorJitterSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ColorJitterSpec.scala @@ -19,14 +19,14 @@ package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn._ import com.intel.analytics.bigdl._ import com.intel.analytics.bigdl.dllib.tensor.{Storage, Tensor} -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import org.scalatest.{BeforeAndAfter, FlatSpec, Matchers} import scala.math._ import scala.util.Random import com.intel.analytics.bigdl.dllib.feature.dataset.LocalArrayDataSet import com.intel.analytics.bigdl.dllib.feature.dataset.image.{ColorJitter, LabeledBGRImage} -import com.intel.analytics.bigdl.utils.RandomGenerator +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator @com.intel.analytics.bigdl.tags.Serial class ColorJitterSpec extends TorchSpec { diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ConcatSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ConcatSpec.scala index 68d7ceaad05..d18ce1cf700 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ConcatSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ConcatSpec.scala @@ -19,7 +19,7 @@ package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn._ import com.intel.analytics.bigdl._ import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import scala.math._ diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ConcatTableSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ConcatTableSpec.scala index bb7e63c2d73..d5ad378db2c 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ConcatTableSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ConcatTableSpec.scala @@ -19,7 +19,7 @@ package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.{ConcatTable, Linear} import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.utils.T -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ContiguousSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ContiguousSpec.scala index 88376141ffd..33c13abcac2 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ContiguousSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ContiguousSpec.scala @@ -17,7 +17,7 @@ package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.Contiguous import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ @com.intel.analytics.bigdl.tags.Serial class ContiguousSpec extends TorchSpec { diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CosineDistanceCriterionSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CosineDistanceCriterionSpec.scala index be0d8eb44ef..8184f929f14 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CosineDistanceCriterionSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CosineDistanceCriterionSpec.scala @@ -17,9 +17,9 @@ package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.CosineDistanceCriterion import com.intel.analytics.bigdl.dllib.tensor.{Storage, Tensor} -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import com.intel.analytics.bigdl.dllib.utils.{Table} -import com.intel.analytics.bigdl.utils.RandomGenerator +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator import scala.collection.mutable.HashMap diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CosineDistanceSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CosineDistanceSpec.scala index 1f327b63e63..fe9c2d75c9d 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CosineDistanceSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CosineDistanceSpec.scala @@ -17,7 +17,7 @@ package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.CosineDistance import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import com.intel.analytics.bigdl.dllib.utils.Table import scala.collection.mutable.HashMap diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CosineEmbeddingCriterionSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CosineEmbeddingCriterionSpec.scala index fd23144f217..dccb8b3ee12 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CosineEmbeddingCriterionSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CosineEmbeddingCriterionSpec.scala @@ -17,9 +17,9 @@ package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.CosineEmbeddingCriterion import com.intel.analytics.bigdl.dllib.tensor.{Storage, Tensor} -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import com.intel.analytics.bigdl.dllib.utils.{Table} -import com.intel.analytics.bigdl.utils.RandomGenerator +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator import scala.collection.mutable.HashMap diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CosineSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CosineSpec.scala index f8e00002919..2ef8d4be419 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CosineSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CosineSpec.scala @@ -17,7 +17,7 @@ package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.Cosine import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CrossEntropyCriterionSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CrossEntropyCriterionSpec.scala index 90623e94a47..b4407937ad9 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CrossEntropyCriterionSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/CrossEntropyCriterionSpec.scala @@ -17,7 +17,7 @@ package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.CrossEntropyCriterion import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/DistKLDivCriterionSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/DistKLDivCriterionSpec.scala index dbb1a51226e..d8f8e86c1ee 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/DistKLDivCriterionSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/DistKLDivCriterionSpec.scala @@ -17,7 +17,7 @@ package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.DistKLDivCriterion import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import scala.util.Random @com.intel.analytics.bigdl.tags.Serial diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/DropoutSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/DropoutSpec.scala index e609d7f3411..8b27e3fc23f 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/DropoutSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/DropoutSpec.scala @@ -18,8 +18,8 @@ package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.Dropout import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.Engine -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.Engine +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ @com.intel.analytics.bigdl.tags.Serial class DropoutSpec extends TorchSpec { diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ELUSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ELUSpec.scala index e0886fff5ae..eb369dc9a66 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ELUSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ELUSpec.scala @@ -17,8 +17,8 @@ package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.ELU import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.RandomGenerator -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ @com.intel.analytics.bigdl.tags.Serial class ELUSpec extends TorchSpec { diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/EuclideanSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/EuclideanSpec.scala index 1da401de1a8..f33bd9c322a 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/EuclideanSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/EuclideanSpec.scala @@ -17,7 +17,7 @@ package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.Euclidean import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/GRUSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/GRUSpec.scala index ee45f345958..c68ea42ae24 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/GRUSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/GRUSpec.scala @@ -22,9 +22,9 @@ import com.intel.analytics.bigdl.dllib.nn._ import com.intel.analytics.bigdl.numeric.NumericDouble import com.intel.analytics.bigdl.dllib.optim.{L2Regularizer, SGD} import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import com.intel.analytics.bigdl.dllib.utils.{T} -import com.intel.analytics.bigdl.utils._ +import com.intel.analytics.bigdl.dllib.utils._ import scala.collection.mutable.ArrayBuffer import scala.sys.process._ diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/GaussianCriterionSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/GaussianCriterionSpec.scala index 4efaaf5bc3a..d53cbdda58e 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/GaussianCriterionSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/GaussianCriterionSpec.scala @@ -18,7 +18,7 @@ package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.GaussianCriterion import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import com.intel.analytics.bigdl.dllib.utils.T import org.scalatest.{FlatSpec, Matchers} diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/GradientReversalSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/GradientReversalSpec.scala index fbee64b2206..8dc0a894b7c 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/GradientReversalSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/GradientReversalSpec.scala @@ -17,7 +17,7 @@ package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.GradientReversal import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.RandomGenerator +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator @com.intel.analytics.bigdl.tags.Serial class GradientReversalSpec extends TorchSpec { diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/HardShrinkSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/HardShrinkSpec.scala index f5efbdf4ed8..abe65d38cd9 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/HardShrinkSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/HardShrinkSpec.scala @@ -17,7 +17,7 @@ package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.HardShrink import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.RandomGenerator +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator @com.intel.analytics.bigdl.tags.Serial class HardShrinkSpec extends TorchSpec { diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/HingeEmbeddingCriterionSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/HingeEmbeddingCriterionSpec.scala index a0e28b17633..11ddd5a9efa 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/HingeEmbeddingCriterionSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/HingeEmbeddingCriterionSpec.scala @@ -17,7 +17,7 @@ package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.HingeEmbeddingCriterion import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.RandomGenerator +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator @com.intel.analytics.bigdl.tags.Serial class HingeEmbeddingCriterionSpec extends TorchSpec { diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/IndexSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/IndexSpec.scala index 6474704add1..bb37c4bba38 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/IndexSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/IndexSpec.scala @@ -17,7 +17,7 @@ package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.Index import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import com.intel.analytics.bigdl.dllib.utils.Table import scala.collection.mutable.HashMap diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/JoinTableSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/JoinTableSpec.scala index 0dc8f0e48db..c034a257708 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/JoinTableSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/JoinTableSpec.scala @@ -18,7 +18,7 @@ package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn._ import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.utils.{T, Table} -import com.intel.analytics.bigdl.utils.RandomGenerator +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator @com.intel.analytics.bigdl.tags.Serial class JoinTableSpec extends TorchSpec { diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/L1HingeEmbeddingCriterionSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/L1HingeEmbeddingCriterionSpec.scala index 2f18920b802..8a005e26a8e 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/L1HingeEmbeddingCriterionSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/L1HingeEmbeddingCriterionSpec.scala @@ -17,7 +17,7 @@ package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.L1HingeEmbeddingCriterion import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import com.intel.analytics.bigdl.dllib.utils.Table import scala.collection.mutable.HashMap diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/LSTMPeepholeSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/LSTMPeepholeSpec.scala index b8684854958..4747a35a08e 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/LSTMPeepholeSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/LSTMPeepholeSpec.scala @@ -22,7 +22,7 @@ import com.intel.analytics.bigdl.dllib.nn._ import com.intel.analytics.bigdl.dllib.nn.abstractnn.Activity import com.intel.analytics.bigdl.dllib.optim.SGD import com.intel.analytics.bigdl.dllib.tensor.{Storage, Tensor} -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import com.intel.analytics.bigdl.dllib.utils.TorchObject.TYPE_DOUBLE_TENSOR import com.intel.analytics.bigdl.dllib.utils.{T, Table, TorchFile} diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/LSTMSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/LSTMSpec.scala index 5d4c6b1c10a..d70f7ea58f1 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/LSTMSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/LSTMSpec.scala @@ -22,7 +22,7 @@ import com.intel.analytics.bigdl.dllib.nn._ import com.intel.analytics.bigdl.numeric.NumericDouble import com.intel.analytics.bigdl.dllib.optim.{L2Regularizer, SGD} import com.intel.analytics.bigdl.dllib.tensor.{Storage, Tensor} -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import com.intel.analytics.bigdl.dllib.utils.T import scala.sys.process._ diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/LeakyReLUSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/LeakyReLUSpec.scala index 7babfff4def..92907048916 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/LeakyReLUSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/LeakyReLUSpec.scala @@ -17,8 +17,8 @@ package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.{LeakyReLU, RReLU} import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.RandomGenerator -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ @com.intel.analytics.bigdl.tags.Serial class LeakyReLUSpec extends TorchSpec { diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/LinearSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/LinearSpec.scala index 3ce6ef6923d..df975429339 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/LinearSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/LinearSpec.scala @@ -18,7 +18,7 @@ package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.{GradientChecker, Linear, MSECriterion} import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import scala.util.Random import com.intel.analytics.bigdl._ diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/LogSoftMaxSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/LogSoftMaxSpec.scala index 4451b282989..f6ddeee01f4 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/LogSoftMaxSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/LogSoftMaxSpec.scala @@ -19,7 +19,7 @@ package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl._ import com.intel.analytics.bigdl.dllib.nn.{GradientChecker, LogSoftMax} import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/LogSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/LogSpec.scala index 973b7b160b9..c21112de35c 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/LogSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/LogSpec.scala @@ -17,7 +17,7 @@ package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.{Log, Power} import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.RandomGenerator +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator @com.intel.analytics.bigdl.tags.Serial class LogSpec extends TorchSpec { diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/LookupTableSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/LookupTableSpec.scala index ed402844a7c..e8da22c058d 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/LookupTableSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/LookupTableSpec.scala @@ -18,7 +18,7 @@ package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn._ import com.intel.analytics.bigdl.dllib.optim.{L2Regularizer, SGD} import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import com.intel.analytics.bigdl.dllib.utils.T @com.intel.analytics.bigdl.tags.Serial diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MMSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MMSpec.scala index 81697982955..7b650656452 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MMSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MMSpec.scala @@ -18,7 +18,7 @@ package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.MM import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.utils.{T, Table} -import com.intel.analytics.bigdl.utils.RandomGenerator +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator import scala.collection.mutable diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MVSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MVSpec.scala index b5ade16e705..c1936455872 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MVSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MVSpec.scala @@ -18,7 +18,7 @@ package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.MV import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.utils.{T, Table} -import com.intel.analytics.bigdl.utils.RandomGenerator +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator import scala.collection.mutable diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MeanSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MeanSpec.scala index 76c73b6acc5..32a5ac63170 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MeanSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MeanSpec.scala @@ -17,7 +17,7 @@ package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.Mean import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.RandomGenerator +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator @com.intel.analytics.bigdl.tags.Serial class MeanSpec extends TorchSpec { diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MulSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MulSpec.scala index 22194725a78..85f07c12cb0 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MulSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MulSpec.scala @@ -17,7 +17,7 @@ package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.Mul import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MultiCriterionSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MultiCriterionSpec.scala index 5e0653bc299..9095c09d288 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MultiCriterionSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/MultiCriterionSpec.scala @@ -18,7 +18,7 @@ package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn._ import com.intel.analytics.bigdl._ import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.RandomGenerator.RNG +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator.RNG import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ParallelCriterionSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ParallelCriterionSpec.scala index 9c453ba847b..e1e6303e099 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ParallelCriterionSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ParallelCriterionSpec.scala @@ -19,7 +19,7 @@ package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.{ClassNLLCriterion, MSECriterion, ParallelCriterion} import com.intel.analytics.bigdl.dllib.tensor.{Storage, Tensor} import com.intel.analytics.bigdl.dllib.utils.{T, Table} -import com.intel.analytics.bigdl.utils.Engine +import com.intel.analytics.bigdl.dllib.utils.Engine import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/RReLUSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/RReLUSpec.scala index 403e4459097..80952df0402 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/RReLUSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/RReLUSpec.scala @@ -17,7 +17,7 @@ package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.{RReLU, ReLU} import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import org.scalatest.{BeforeAndAfter, FlatSpec, Matchers, fixture} diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ReLU6Spec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ReLU6Spec.scala index 10b6a821516..cb841b4a775 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ReLU6Spec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ReLU6Spec.scala @@ -19,7 +19,7 @@ import java.io.File import com.intel.analytics.bigdl.dllib.nn.ReLU6 import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.Engine +import com.intel.analytics.bigdl.dllib.utils.Engine import org.scalatest._ import scala.collection.mutable.ListBuffer diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ReLUSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ReLUSpec.scala index 1ed77093c5b..65cb948a58f 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ReLUSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ReLUSpec.scala @@ -18,7 +18,7 @@ package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.{GradientChecker, ReLU} import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import scala.math._ import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SamplerSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SamplerSpec.scala index ed7ed733323..89949a6415c 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SamplerSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SamplerSpec.scala @@ -18,7 +18,7 @@ package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.{KLDCriterion, GaussianSampler} import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import com.intel.analytics.bigdl.dllib.utils.T import org.scalatest.{FlatSpec, Matchers} diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SelectSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SelectSpec.scala index e5ec0cd74fc..268caae921b 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SelectSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SelectSpec.scala @@ -17,7 +17,7 @@ package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.Select import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.RandomGenerator +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator @com.intel.analytics.bigdl.tags.Serial class SelectSpec extends TorchSpec { diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SequentialSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SequentialSpec.scala index 90585799583..5a1e13b8873 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SequentialSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SequentialSpec.scala @@ -18,7 +18,7 @@ package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.{Linear, Sequential} import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import scala.math._ diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SoftMinSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SoftMinSpec.scala index facff942f6c..fcc7501e687 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SoftMinSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SoftMinSpec.scala @@ -18,7 +18,7 @@ package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.SoftMin import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.Engine +import com.intel.analytics.bigdl.dllib.utils.Engine import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialAveragePoolingSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialAveragePoolingSpec.scala index 3ce8d30107d..f4807a8fb74 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialAveragePoolingSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialAveragePoolingSpec.scala @@ -19,7 +19,7 @@ package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.{GradientChecker, SpatialAveragePooling} import com.intel.analytics.bigdl.dllib.tensor import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import scala.math._ import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialBatchNormalizationSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialBatchNormalizationSpec.scala index fb8280e88f2..40a49fe8227 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialBatchNormalizationSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialBatchNormalizationSpec.scala @@ -19,7 +19,7 @@ package com.intel.analytics.bigdl.dllib.integration.torch import breeze.numerics.abs import com.intel.analytics.bigdl.dllib.nn.{BatchNormalization, GradientChecker, SpatialBatchNormalization} import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import scala.util.Random import com.intel.analytics.bigdl._ diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialContrastiveNormalizationSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialContrastiveNormalizationSpec.scala index 30ec7886a5d..1d1e90c8db6 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialContrastiveNormalizationSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialContrastiveNormalizationSpec.scala @@ -18,7 +18,7 @@ package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.{Sequential, SpatialContrastiveNormalization} import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialConvolutionMapSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialConvolutionMapSpec.scala index d2827decca9..715ea36dee5 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialConvolutionMapSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialConvolutionMapSpec.scala @@ -20,7 +20,7 @@ import com.intel.analytics.bigdl.dllib.nn._ import com.intel.analytics.bigdl.dllib.optim.{L2Regularizer, SGD} import com.intel.analytics.bigdl.dllib.tensor.{Storage, Tensor} import org.scalatest.{BeforeAndAfter, FlatSpec, Matchers} -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import com.intel.analytics.bigdl.dllib.utils.T import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialConvolutionSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialConvolutionSpec.scala index 2393c5fbdc8..3c5f75e8fbf 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialConvolutionSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialConvolutionSpec.scala @@ -20,8 +20,8 @@ import com.intel.analytics.bigdl._ import com.intel.analytics.bigdl.dllib.nn.{Sequential, SpatialConvolution} import com.intel.analytics.bigdl.dllib.nn._ import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.Engine -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.Engine +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialCrossMapLRNSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialCrossMapLRNSpec.scala index a9af9f2356c..32f06c12b06 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialCrossMapLRNSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialCrossMapLRNSpec.scala @@ -18,7 +18,7 @@ package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn._ import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import com.intel.analytics.bigdl._ import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialDilatedConvolutionSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialDilatedConvolutionSpec.scala index 35049fbe2bc..46b85d22b80 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialDilatedConvolutionSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialDilatedConvolutionSpec.scala @@ -19,7 +19,7 @@ package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn._ import com.intel.analytics.bigdl.dllib.optim.{L2Regularizer, SGD} import com.intel.analytics.bigdl.dllib.tensor.{Storage, Tensor} -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import com.intel.analytics.bigdl.dllib.utils.T import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialDivisiveNormalizationSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialDivisiveNormalizationSpec.scala index e32db890466..8d3926af71d 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialDivisiveNormalizationSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialDivisiveNormalizationSpec.scala @@ -18,7 +18,7 @@ package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.{Sequential, SpatialDivisiveNormalization} import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialDropout1DSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialDropout1DSpec.scala index 4e894088508..f6b88e500f1 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialDropout1DSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialDropout1DSpec.scala @@ -18,7 +18,7 @@ package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.{SpatialDropout1D, SpatialDropout2D} import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ @com.intel.analytics.bigdl.tags.Serial class SpatialDropout1DSpec extends TorchSpec { diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialDropout2DSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialDropout2DSpec.scala index 8ced1761c33..c77c269a2b8 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialDropout2DSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialDropout2DSpec.scala @@ -18,7 +18,7 @@ package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.SpatialDropout2D import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ @com.intel.analytics.bigdl.tags.Serial class SpatialDropout2DSpec extends TorchSpec { diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialDropout3DSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialDropout3DSpec.scala index 14d2fd0968a..9779c5d58ad 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialDropout3DSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialDropout3DSpec.scala @@ -18,7 +18,7 @@ package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.SpatialDropout3D import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ @com.intel.analytics.bigdl.tags.Serial class SpatialDropout3DSpec extends TorchSpec { diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialFullConvolutionSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialFullConvolutionSpec.scala index c71bc2938a2..1c962999569 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialFullConvolutionSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialFullConvolutionSpec.scala @@ -18,7 +18,7 @@ package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.{Sequential, SpatialFullConvolution} import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import com.intel.analytics.bigdl.dllib.utils.{T, Table} import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialMaxPoolingSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialMaxPoolingSpec.scala index 276c0acee56..af9ca4b9503 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialMaxPoolingSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialMaxPoolingSpec.scala @@ -18,7 +18,7 @@ package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.{GradientChecker, SpatialMaxPooling} import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import scala.math._ import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialSubtractiveNormalizationSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialSubtractiveNormalizationSpec.scala index 4d1156dc7ff..2daa72a4c66 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialSubtractiveNormalizationSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SpatialSubtractiveNormalizationSpec.scala @@ -18,8 +18,8 @@ package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.{Sequential, SpatialSubtractiveNormalization} import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.Engine -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.Engine +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SumSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SumSpec.scala index 6200bacc0d0..58a5f510c77 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SumSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/SumSpec.scala @@ -17,7 +17,7 @@ package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.Sum import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.RandomGenerator +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator @com.intel.analytics.bigdl.tags.Serial class SumSpec extends TorchSpec { diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/TanhShrinkSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/TanhShrinkSpec.scala index 5a298f696eb..494ac4a7beb 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/TanhShrinkSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/TanhShrinkSpec.scala @@ -17,7 +17,7 @@ package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.TanhShrink import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.RandomGenerator +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator @com.intel.analytics.bigdl.tags.Serial class TanhShrinkSpec extends TorchSpec { diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/TemporalConvolutionSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/TemporalConvolutionSpec.scala index 3c561cc0eb3..aab0465dea9 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/TemporalConvolutionSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/TemporalConvolutionSpec.scala @@ -20,7 +20,7 @@ import com.intel.analytics.bigdl._ import com.intel.analytics.bigdl.dllib.nn.TemporalConvolution import com.intel.analytics.bigdl.dllib.nn._ import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/TemporalMaxPoolingSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/TemporalMaxPoolingSpec.scala index a4010e33c36..723660fb111 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/TemporalMaxPoolingSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/TemporalMaxPoolingSpec.scala @@ -19,7 +19,7 @@ package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl._ import com.intel.analytics.bigdl.dllib.nn.{GradientChecker, TemporalMaxPooling} import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import scala.math._ import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ThresholdSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ThresholdSpec.scala index f0a3a7120ad..819f8b4b138 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ThresholdSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ThresholdSpec.scala @@ -18,7 +18,7 @@ package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.Threshold import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.Engine +import com.intel.analytics.bigdl.dllib.utils.Engine import scala.math._ diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/TorchSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/TorchSpec.scala index 83342ea57d8..88f14b79a91 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/TorchSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/TorchSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ package com.intel.analytics.bigdl.dllib.integration.torch -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import org.scalatest._ import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ViewSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ViewSpec.scala index 2c1e64dd45c..dfa5289a3d2 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ViewSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/ViewSpec.scala @@ -19,7 +19,7 @@ package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl._ import com.intel.analytics.bigdl.dllib.nn.{GradientChecker, View} import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import scala.math._ import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/VolumetricAveragePoolingSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/VolumetricAveragePoolingSpec.scala index eb849aae533..566b5879fab 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/VolumetricAveragePoolingSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/VolumetricAveragePoolingSpec.scala @@ -19,7 +19,7 @@ package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl._ import com.intel.analytics.bigdl.dllib.nn.{GradientChecker, VolumetricAveragePooling} import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/VolumetricConvolutionSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/VolumetricConvolutionSpec.scala index beaec1809f8..4e8a4ba2ce3 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/VolumetricConvolutionSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/VolumetricConvolutionSpec.scala @@ -21,9 +21,9 @@ import com.intel.analytics.bigdl.dllib.nn._ import com.intel.analytics.bigdl.dllib.optim.{L2Regularizer, SGD} import com.intel.analytics.bigdl.dllib.tensor import com.intel.analytics.bigdl.dllib.tensor.{Storage, Tensor} -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import com.intel.analytics.bigdl.dllib.utils.{T, TestUtils} -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/VolumetricFullConvolutionSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/VolumetricFullConvolutionSpec.scala index 1b00ba767da..1cf066fcb64 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/VolumetricFullConvolutionSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/VolumetricFullConvolutionSpec.scala @@ -18,7 +18,7 @@ package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl.dllib.nn.{BilinearFiller, Sequential, VolumetricFullConvolution, Zeros} import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import com.intel.analytics.bigdl.dllib.utils.{T, Table} import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/VolumetricMaxPoolingSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/VolumetricMaxPoolingSpec.scala index ee336088423..22b6ad61511 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/VolumetricMaxPoolingSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/VolumetricMaxPoolingSpec.scala @@ -19,7 +19,7 @@ package com.intel.analytics.bigdl.dllib.integration.torch import com.intel.analytics.bigdl._ import com.intel.analytics.bigdl.dllib.nn.{GradientChecker, VolumetricMaxPooling} import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/models/AlexNetSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/models/AlexNetSpec.scala index ebf1f6dd17a..d81b74dbe60 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/models/AlexNetSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/models/AlexNetSpec.scala @@ -23,7 +23,7 @@ import com.intel.analytics.bigdl.dllib.nn.ClassNLLCriterion import com.intel.analytics.bigdl.numeric.NumericFloat import com.intel.analytics.bigdl.dllib.optim.SGD import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import com.intel.analytics.bigdl.dllib.utils.T import scala.math._ diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/models/InceptionSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/models/InceptionSpec.scala index 1c088905b17..f5cd49d9c99 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/models/InceptionSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/integration/torch/models/InceptionSpec.scala @@ -23,7 +23,7 @@ import com.intel.analytics.bigdl.dllib.optim.SGD import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.integration.torch.{TH, TorchSpec} import com.intel.analytics.bigdl.dllib.models.Inception -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import com.intel.analytics.bigdl.dllib.utils.{T, Table} import com.intel.analytics.bigdl.numeric.NumericFloat diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/Cropping2DSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/Cropping2DSpec.scala index 93719cc6870..09fdef5df34 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/Cropping2DSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/Cropping2DSpec.scala @@ -19,7 +19,7 @@ package com.intel.analytics.bigdl.keras import com.intel.analytics.bigdl.dllib.nn.{Cropping2D, _} import com.intel.analytics.bigdl.dllib.nn.abstractnn.DataFormat import com.intel.analytics.bigdl.dllib.utils.{TestUtils} -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape class Cropping2DSpec extends KerasBaseSpec { "Cropping2D" should "with NCHW work properly" in { diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/Cropping3DSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/Cropping3DSpec.scala index 8a26928d46e..ca75840d2b7 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/Cropping3DSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/Cropping3DSpec.scala @@ -19,7 +19,7 @@ package com.intel.analytics.bigdl.keras import com.intel.analytics.bigdl.dllib.nn.{Cropping2D, _} import com.intel.analytics.bigdl.dllib.nn.abstractnn.DataFormat import com.intel.analytics.bigdl.dllib.utils.{TestUtils} -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape class Cropping3DSpec extends KerasBaseSpec { "Cropping3D" should "with CHANNEL_FIRST work properly" in { diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/LocallyConnected2DSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/LocallyConnected2DSpec.scala index 89cf01a877c..33246d2b2e1 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/LocallyConnected2DSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/LocallyConnected2DSpec.scala @@ -20,7 +20,7 @@ import com.intel.analytics.bigdl.dllib.nn.LocallyConnected2D import com.intel.analytics.bigdl.dllib.nn.abstractnn.DataFormat import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.utils.{TestUtils} -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape class LocallyConnected2DSpec extends KerasBaseSpec { diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/UpSampling1DSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/UpSampling1DSpec.scala index 4f0b213eff3..b2360fc1db3 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/UpSampling1DSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/UpSampling1DSpec.scala @@ -18,7 +18,7 @@ package com.intel.analytics.bigdl.keras import com.intel.analytics.bigdl.dllib.nn._ import com.intel.analytics.bigdl.dllib.utils.{TestUtils} -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape class UpSampling1DSpec extends KerasBaseSpec { "UpSampling1D forward with size 1" should "work properly" in { diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/UpSampling2DSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/UpSampling2DSpec.scala index 77b05994e54..b5a14a2080d 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/UpSampling2DSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/UpSampling2DSpec.scala @@ -18,7 +18,7 @@ package com.intel.analytics.bigdl.keras import com.intel.analytics.bigdl.dllib.nn._ import com.intel.analytics.bigdl.dllib.nn.abstractnn.DataFormat -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import com.intel.analytics.bigdl.dllib.utils.TestUtils class UpSampling2DSpec extends KerasBaseSpec { diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/UpSampling3DSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/UpSampling3DSpec.scala index 3199b3880af..f75339e2149 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/UpSampling3DSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/UpSampling3DSpec.scala @@ -19,7 +19,7 @@ package com.intel.analytics.bigdl.keras import com.intel.analytics.bigdl.dllib.nn._ import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.utils.{TestUtils} -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape class UpSampling3DSpec extends KerasBaseSpec { "UpSampling3D forward with size 1" should "work properly" in { diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/ActivationSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/ActivationSpec.scala index 50a9b33a1a0..7f0c31c5719 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/ActivationSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/ActivationSpec.scala @@ -20,7 +20,7 @@ import com.intel.analytics.bigdl.keras.KerasBaseSpec import com.intel.analytics.bigdl.dllib.nn.abstractnn.AbstractModule import com.intel.analytics.bigdl.dllib.keras.{Activation, SoftMax, Sequential => KSequential} import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/AtrousConvolution1DSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/AtrousConvolution1DSpec.scala index ae61b9aec75..583648546c8 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/AtrousConvolution1DSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/AtrousConvolution1DSpec.scala @@ -20,7 +20,7 @@ import com.intel.analytics.bigdl.keras.KerasBaseSpec import com.intel.analytics.bigdl.dllib.nn.abstractnn.AbstractModule import com.intel.analytics.bigdl.dllib.keras.{AtrousConvolution1D, Sequential => KSequential} import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/AtrousConvolution2DSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/AtrousConvolution2DSpec.scala index 138b8f6abf9..9676c8e5947 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/AtrousConvolution2DSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/AtrousConvolution2DSpec.scala @@ -20,7 +20,7 @@ import com.intel.analytics.bigdl.keras.KerasBaseSpec import com.intel.analytics.bigdl.dllib.nn.abstractnn.AbstractModule import com.intel.analytics.bigdl.dllib.keras.{AtrousConvolution2D, Sequential => KSequential} import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/AveragePooling1DSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/AveragePooling1DSpec.scala index 03e07b41f78..8d0a64f347e 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/AveragePooling1DSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/AveragePooling1DSpec.scala @@ -20,7 +20,7 @@ import com.intel.analytics.bigdl.keras.KerasBaseSpec import com.intel.analytics.bigdl.dllib.nn.abstractnn.AbstractModule import com.intel.analytics.bigdl.dllib.keras.{AveragePooling1D, Sequential => KSequential} import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/AveragePooling2DSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/AveragePooling2DSpec.scala index b924f046538..8ad5f846828 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/AveragePooling2DSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/AveragePooling2DSpec.scala @@ -20,7 +20,7 @@ import com.intel.analytics.bigdl.keras.KerasBaseSpec import com.intel.analytics.bigdl.dllib.nn.abstractnn.{AbstractModule, DataFormat} import com.intel.analytics.bigdl.dllib.keras.{AveragePooling2D, Sequential => KSequential} import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/AveragePooling3DSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/AveragePooling3DSpec.scala index 3d2c87433ac..a60ba788760 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/AveragePooling3DSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/AveragePooling3DSpec.scala @@ -20,7 +20,7 @@ import com.intel.analytics.bigdl.keras.KerasBaseSpec import com.intel.analytics.bigdl.dllib.nn.abstractnn.AbstractModule import com.intel.analytics.bigdl.dllib.keras.{AveragePooling3D, Sequential => KSequential} import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/BatchNormalizationSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/BatchNormalizationSpec.scala index 89bef90a7f7..f79c237aab5 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/BatchNormalizationSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/BatchNormalizationSpec.scala @@ -19,7 +19,7 @@ package com.intel.analytics.bigdl.keras.nn import com.intel.analytics.bigdl.keras.KerasBaseSpec import com.intel.analytics.bigdl.dllib.keras.{BatchNormalization, Sequential => KSequential} import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/BidirectionalSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/BidirectionalSpec.scala index 9dcece3be74..c11a9395b9a 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/BidirectionalSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/BidirectionalSpec.scala @@ -21,7 +21,7 @@ import com.intel.analytics.bigdl.keras.KerasBaseSpec import com.intel.analytics.bigdl.dllib.nn.abstractnn.AbstractModule import com.intel.analytics.bigdl.dllib.keras.{Bidirectional, LSTM, SimpleRNN, Sequential => KSequential} import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/ConvLSTM2DSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/ConvLSTM2DSpec.scala index 1bfd2344195..45912dd42c0 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/ConvLSTM2DSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/ConvLSTM2DSpec.scala @@ -20,7 +20,7 @@ import com.intel.analytics.bigdl.keras.KerasBaseSpec import com.intel.analytics.bigdl.dllib.nn.abstractnn.AbstractModule import com.intel.analytics.bigdl.dllib.keras.{ConvLSTM2D, Sequential => KSequential} import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/Convolution1DSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/Convolution1DSpec.scala index c5a0e3e59a3..358a5b2712b 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/Convolution1DSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/Convolution1DSpec.scala @@ -20,7 +20,7 @@ import com.intel.analytics.bigdl.keras.KerasBaseSpec import com.intel.analytics.bigdl.dllib.nn.abstractnn.AbstractModule import com.intel.analytics.bigdl.dllib.keras.{Convolution1D, Conv1D, Sequential => KSequential} import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/Convolution2DSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/Convolution2DSpec.scala index 97dba4e779d..bd28e40f116 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/Convolution2DSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/Convolution2DSpec.scala @@ -20,7 +20,7 @@ import com.intel.analytics.bigdl.keras.KerasBaseSpec import com.intel.analytics.bigdl.dllib.nn.abstractnn.{AbstractModule, DataFormat} import com.intel.analytics.bigdl.dllib.keras.{Conv2D, Convolution2D, Sequential => KSequential} import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/Convolution3DSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/Convolution3DSpec.scala index 36a4717661b..1029f7235df 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/Convolution3DSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/Convolution3DSpec.scala @@ -20,7 +20,7 @@ import com.intel.analytics.bigdl.keras.KerasBaseSpec import com.intel.analytics.bigdl.dllib.nn.abstractnn.AbstractModule import com.intel.analytics.bigdl.dllib.keras.{Conv3D, Convolution3D, Sequential => KSequential} import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/Cropping1DSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/Cropping1DSpec.scala index c0607e9646b..fd7e7b174c4 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/Cropping1DSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/Cropping1DSpec.scala @@ -20,7 +20,7 @@ import com.intel.analytics.bigdl.keras.KerasBaseSpec import com.intel.analytics.bigdl.dllib.nn.abstractnn.AbstractModule import com.intel.analytics.bigdl.dllib.keras.{Cropping1D, Sequential => KSequential} import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/Cropping2DSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/Cropping2DSpec.scala index e98890aac67..cb08e26cf33 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/Cropping2DSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/Cropping2DSpec.scala @@ -20,7 +20,7 @@ import com.intel.analytics.bigdl.keras.KerasBaseSpec import com.intel.analytics.bigdl.dllib.nn.abstractnn.{AbstractModule, DataFormat} import com.intel.analytics.bigdl.dllib.keras.{Cropping2D, Sequential => KSequential} import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/Cropping3DSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/Cropping3DSpec.scala index aad6966ef88..014f85a8777 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/Cropping3DSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/Cropping3DSpec.scala @@ -20,7 +20,7 @@ import com.intel.analytics.bigdl.keras.KerasBaseSpec import com.intel.analytics.bigdl.dllib.nn.abstractnn.AbstractModule import com.intel.analytics.bigdl.dllib.keras.{Cropping3D, Sequential => KSequential} import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/Deconvolution2DSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/Deconvolution2DSpec.scala index e5f3832fbf5..7e05a7eb7e7 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/Deconvolution2DSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/Deconvolution2DSpec.scala @@ -20,7 +20,7 @@ import com.intel.analytics.bigdl.keras.KerasBaseSpec import com.intel.analytics.bigdl.dllib.nn.abstractnn.AbstractModule import com.intel.analytics.bigdl.dllib.keras.{Deconvolution2D, Deconv2D, Sequential => KSequential} import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/DenseSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/DenseSpec.scala index 37d21a06e32..2bcc9bade3c 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/DenseSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/DenseSpec.scala @@ -21,7 +21,7 @@ import com.intel.analytics.bigdl.dllib.keras.{Sequential => KSequential} import com.intel.analytics.bigdl.dllib.nn.abstractnn.AbstractModule import com.intel.analytics.bigdl.dllib.keras.Dense import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/DropoutSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/DropoutSpec.scala index c6ea70c4682..a53053246e5 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/DropoutSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/DropoutSpec.scala @@ -19,7 +19,7 @@ package com.intel.analytics.bigdl.keras.nn import org.scalatest.{FlatSpec, Matchers} import com.intel.analytics.bigdl.dllib.keras.{Dropout, Sequential => KSequential} import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/ELUSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/ELUSpec.scala index 5bd9d159bbd..baa594c781d 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/ELUSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/ELUSpec.scala @@ -21,7 +21,7 @@ import com.intel.analytics.bigdl.dllib.keras.{Sequential => KSequential} import com.intel.analytics.bigdl.dllib.nn.abstractnn.AbstractModule import com.intel.analytics.bigdl.dllib.keras.ELU import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/EmbeddingSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/EmbeddingSpec.scala index fae3d030d82..2b054ad85fb 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/EmbeddingSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/EmbeddingSpec.scala @@ -19,7 +19,7 @@ package com.intel.analytics.bigdl.keras.nn import com.intel.analytics.bigdl.keras.KerasBaseSpec import com.intel.analytics.bigdl.dllib.keras.{Embedding, Sequential => KSequential} import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/FlattenSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/FlattenSpec.scala index c6118365576..c2a27aa21af 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/FlattenSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/FlattenSpec.scala @@ -21,7 +21,7 @@ import com.intel.analytics.bigdl.dllib.nn.abstractnn.AbstractModule import com.intel.analytics.bigdl.dllib.keras.Flatten import com.intel.analytics.bigdl.dllib.keras.{Sequential => KSequential} import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/GRUSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/GRUSpec.scala index 9e743c0e515..a351628602d 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/GRUSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/GRUSpec.scala @@ -20,7 +20,7 @@ import com.intel.analytics.bigdl.keras.KerasBaseSpec import com.intel.analytics.bigdl.dllib.nn.abstractnn.AbstractModule import com.intel.analytics.bigdl.dllib.keras.{GRU, Sequential => KSequential} import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/GaussianDropoutSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/GaussianDropoutSpec.scala index a266e99ba5e..2ea0b43f3c8 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/GaussianDropoutSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/GaussianDropoutSpec.scala @@ -20,7 +20,7 @@ import com.intel.analytics.bigdl.keras.KerasBaseSpec import com.intel.analytics.bigdl.dllib.keras.GaussianDropout import com.intel.analytics.bigdl.dllib.keras.{Sequential => KSequential} import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/GaussianNoiseSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/GaussianNoiseSpec.scala index 986271ad141..cd5bd224a75 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/GaussianNoiseSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/GaussianNoiseSpec.scala @@ -20,7 +20,7 @@ import com.intel.analytics.bigdl.keras.KerasBaseSpec import com.intel.analytics.bigdl.dllib.keras.GaussianNoise import com.intel.analytics.bigdl.dllib.keras.{Sequential => KSequential} import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/GlobalAveragePooling1DSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/GlobalAveragePooling1DSpec.scala index 6e74b60690f..74beb31785e 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/GlobalAveragePooling1DSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/GlobalAveragePooling1DSpec.scala @@ -21,7 +21,7 @@ import com.intel.analytics.bigdl.dllib.keras.{Sequential => KSequential} import com.intel.analytics.bigdl.dllib.nn.abstractnn.AbstractModule import com.intel.analytics.bigdl.dllib.keras.GlobalAveragePooling1D import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/GlobalAveragePooling2DSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/GlobalAveragePooling2DSpec.scala index 8519d979139..ab373a1491c 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/GlobalAveragePooling2DSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/GlobalAveragePooling2DSpec.scala @@ -20,7 +20,7 @@ import com.intel.analytics.bigdl.keras.KerasBaseSpec import com.intel.analytics.bigdl.dllib.nn.abstractnn.{AbstractModule, DataFormat} import com.intel.analytics.bigdl.dllib.keras.{GlobalAveragePooling2D, Sequential => KSequential} import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/GlobalAveragePooling3DSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/GlobalAveragePooling3DSpec.scala index f2ccabceb68..fb913399889 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/GlobalAveragePooling3DSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/GlobalAveragePooling3DSpec.scala @@ -21,7 +21,7 @@ import com.intel.analytics.bigdl.dllib.keras.{Sequential => KSequential} import com.intel.analytics.bigdl.dllib.nn.abstractnn.AbstractModule import com.intel.analytics.bigdl.dllib.keras.GlobalAveragePooling3D import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/GlobalMaxPooling1DSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/GlobalMaxPooling1DSpec.scala index 9f39397b1dc..87acee024d0 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/GlobalMaxPooling1DSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/GlobalMaxPooling1DSpec.scala @@ -21,7 +21,7 @@ import com.intel.analytics.bigdl.dllib.keras.{Sequential => KSequential} import com.intel.analytics.bigdl.dllib.nn.abstractnn.AbstractModule import com.intel.analytics.bigdl.dllib.keras.GlobalMaxPooling1D import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/GlobalMaxPooling2DSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/GlobalMaxPooling2DSpec.scala index 0630d748cca..06b63337961 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/GlobalMaxPooling2DSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/GlobalMaxPooling2DSpec.scala @@ -20,7 +20,7 @@ import com.intel.analytics.bigdl.keras.KerasBaseSpec import com.intel.analytics.bigdl.dllib.nn.abstractnn.{AbstractModule, DataFormat} import com.intel.analytics.bigdl.dllib.keras.{GlobalMaxPooling2D, Sequential => KSequential} import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/GlobalMaxPooling3DSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/GlobalMaxPooling3DSpec.scala index 5548429377a..7100318e703 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/GlobalMaxPooling3DSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/GlobalMaxPooling3DSpec.scala @@ -21,7 +21,7 @@ import com.intel.analytics.bigdl.dllib.keras.{Sequential => KSequential} import com.intel.analytics.bigdl.dllib.nn.abstractnn.AbstractModule import com.intel.analytics.bigdl.dllib.keras.GlobalMaxPooling3D import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/HighwaySpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/HighwaySpec.scala index e900675e250..80d8548db08 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/HighwaySpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/HighwaySpec.scala @@ -20,7 +20,7 @@ import com.intel.analytics.bigdl.keras.KerasBaseSpec import com.intel.analytics.bigdl.dllib.nn.abstractnn.AbstractModule import com.intel.analytics.bigdl.dllib.keras.{Dense, Highway, Sequential => KSequential} import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/InputSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/InputSpec.scala index ff1de2a8c0c..e5acf7ae495 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/InputSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/InputSpec.scala @@ -20,7 +20,7 @@ import com.intel.analytics.bigdl.dllib.keras.Merge.merge import com.intel.analytics.bigdl.dllib.keras._ import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.utils.{BigDLSpecHelper} -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest import com.intel.analytics.bigdl.numeric.NumericFloat diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/KerasIdentityWrapperSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/KerasIdentityWrapperSpec.scala index 08e253d1c53..30ffeba9b93 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/KerasIdentityWrapperSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/KerasIdentityWrapperSpec.scala @@ -19,7 +19,7 @@ import com.intel.analytics.bigdl.dllib.nn.ReLU import com.intel.analytics.bigdl.dllib.nn.abstractnn.AbstractModule import com.intel.analytics.bigdl.dllib.keras.KerasIdentityWrapper import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/KerasLayerWrapperSerialTest.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/KerasLayerWrapperSerialTest.scala index 896ed542924..4d8b28f367b 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/KerasLayerWrapperSerialTest.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/KerasLayerWrapperSerialTest.scala @@ -18,7 +18,7 @@ package com.intel.analytics.bigdl.keras.nn import com.intel.analytics.bigdl.dllib.nn.ReLU import com.intel.analytics.bigdl.dllib.keras.KerasLayerWrapper import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/KerasLayerWrapperSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/KerasLayerWrapperSpec.scala index 37ccb3dfc94..5b63b7ee26a 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/KerasLayerWrapperSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/KerasLayerWrapperSpec.scala @@ -21,7 +21,7 @@ import com.intel.analytics.bigdl.dllib.nn.Linear import com.intel.analytics.bigdl.dllib.nn.abstractnn.AbstractModule import com.intel.analytics.bigdl.dllib.keras.{Dense, KerasLayerWrapper, Sequential => KSequential} import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/KerasStyleSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/KerasStyleSpec.scala index 47f2e1f4d18..63b7bb0c327 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/KerasStyleSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/KerasStyleSpec.scala @@ -24,7 +24,7 @@ import com.intel.analytics.bigdl.dllib.nn.{Input => TInput, Sequential => TSeque import com.intel.analytics.bigdl.numeric.NumericFloat import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.utils.{BigDLSpecHelper, T, TestUtils} -import com.intel.analytics.bigdl.utils.{Shape, RandomGenerator} +import com.intel.analytics.bigdl.dllib.utils.{Shape, RandomGenerator} class KerasStyleSpec extends BigDLSpecHelper { diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/LSTMSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/LSTMSpec.scala index f24c1ea614c..9ef892b18cb 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/LSTMSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/LSTMSpec.scala @@ -20,7 +20,7 @@ import com.intel.analytics.bigdl.keras.KerasBaseSpec import com.intel.analytics.bigdl.dllib.nn.abstractnn.AbstractModule import com.intel.analytics.bigdl.dllib.keras.{LSTM, Sequential => KSequential} import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/LeakyReLUSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/LeakyReLUSpec.scala index f9445fe597d..f4b38d537d0 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/LeakyReLUSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/LeakyReLUSpec.scala @@ -21,7 +21,7 @@ import com.intel.analytics.bigdl.dllib.keras.{Sequential => KSequential} import com.intel.analytics.bigdl.dllib.nn.abstractnn.AbstractModule import com.intel.analytics.bigdl.dllib.keras.LeakyReLU import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/LocallyConnected1DSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/LocallyConnected1DSpec.scala index 07cd91af6f5..a024c1593dd 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/LocallyConnected1DSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/LocallyConnected1DSpec.scala @@ -20,7 +20,7 @@ import com.intel.analytics.bigdl.keras.KerasBaseSpec import com.intel.analytics.bigdl.dllib.nn.abstractnn.AbstractModule import com.intel.analytics.bigdl.dllib.keras.{LocallyConnected1D, Sequential => KSequential} import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/LocallyConnected2DSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/LocallyConnected2DSpec.scala index 6b510084e13..478beb35a1d 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/LocallyConnected2DSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/LocallyConnected2DSpec.scala @@ -20,7 +20,7 @@ import com.intel.analytics.bigdl.keras.KerasBaseSpec import com.intel.analytics.bigdl.dllib.nn.abstractnn.{AbstractModule, DataFormat} import com.intel.analytics.bigdl.dllib.keras.{LocallyConnected2D, Sequential => KSequential} import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/MaskingSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/MaskingSpec.scala index 161d4217f80..2488757fc1f 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/MaskingSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/MaskingSpec.scala @@ -21,7 +21,7 @@ import com.intel.analytics.bigdl.dllib.keras.{Sequential => KSequential} import com.intel.analytics.bigdl.dllib.nn.abstractnn.AbstractModule import com.intel.analytics.bigdl.dllib.keras.Masking import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/MaxPooling1DSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/MaxPooling1DSpec.scala index 5e051795d62..b89d1b2ceaf 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/MaxPooling1DSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/MaxPooling1DSpec.scala @@ -20,7 +20,7 @@ import com.intel.analytics.bigdl.keras.KerasBaseSpec import com.intel.analytics.bigdl.dllib.nn.abstractnn.AbstractModule import com.intel.analytics.bigdl.dllib.keras.{MaxPooling1D, Sequential => KSequential} import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/MaxPooling2DSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/MaxPooling2DSpec.scala index 98f6bc26045..378cb15505d 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/MaxPooling2DSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/MaxPooling2DSpec.scala @@ -20,7 +20,7 @@ import com.intel.analytics.bigdl.keras.KerasBaseSpec import com.intel.analytics.bigdl.dllib.nn.abstractnn.{AbstractModule, DataFormat} import com.intel.analytics.bigdl.dllib.keras.{MaxPooling2D, Sequential => KSequential} import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/MaxPooling3DSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/MaxPooling3DSpec.scala index 648ebd9b346..d19b36a5094 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/MaxPooling3DSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/MaxPooling3DSpec.scala @@ -20,7 +20,7 @@ import com.intel.analytics.bigdl.keras.KerasBaseSpec import com.intel.analytics.bigdl.dllib.nn.abstractnn.AbstractModule import com.intel.analytics.bigdl.dllib.keras.{MaxPooling3D, Sequential => KSequential} import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/MaxoutDenseSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/MaxoutDenseSpec.scala index 838320fec22..082076a7603 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/MaxoutDenseSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/MaxoutDenseSpec.scala @@ -20,7 +20,7 @@ import com.intel.analytics.bigdl.keras.KerasBaseSpec import com.intel.analytics.bigdl.dllib.nn.abstractnn.AbstractModule import com.intel.analytics.bigdl.dllib.keras.{MaxoutDense, Sequential => KSequential} import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/MergeSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/MergeSpec.scala index 65bcc49610c..b3aba1c1691 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/MergeSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/MergeSpec.scala @@ -22,7 +22,7 @@ import com.intel.analytics.bigdl.dllib.keras.Merge.merge import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest import com.intel.analytics.bigdl.dllib.utils.{T, Table} -import com.intel.analytics.bigdl.utils.{MultiShape, Shape} +import com.intel.analytics.bigdl.dllib.utils.{MultiShape, Shape} import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/ModelSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/ModelSpec.scala index 642681354bc..09653c2f53a 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/ModelSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/ModelSpec.scala @@ -18,7 +18,7 @@ package com.intel.analytics.bigdl.keras.nn import com.intel.analytics.bigdl.dllib.keras.{Dense, Input, Model} import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest class ModelSerialTest extends ModuleSerializationTest { diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/PermuteSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/PermuteSpec.scala index 34bea52f133..8e3eb83e8f5 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/PermuteSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/PermuteSpec.scala @@ -20,7 +20,7 @@ import com.intel.analytics.bigdl.keras.KerasBaseSpec import com.intel.analytics.bigdl.dllib.nn.abstractnn.AbstractModule import com.intel.analytics.bigdl.dllib.keras.{Permute, Sequential => KSequential} import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/RepeatVectorSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/RepeatVectorSpec.scala index 82a35a6962a..e1c79c4cff4 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/RepeatVectorSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/RepeatVectorSpec.scala @@ -20,7 +20,7 @@ import com.intel.analytics.bigdl.keras.KerasBaseSpec import com.intel.analytics.bigdl.dllib.nn.abstractnn.AbstractModule import com.intel.analytics.bigdl.dllib.keras.{RepeatVector, Sequential => KSequential} import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/ReshapeSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/ReshapeSpec.scala index 30c1fc0679b..09938be0a20 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/ReshapeSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/ReshapeSpec.scala @@ -21,7 +21,7 @@ import com.intel.analytics.bigdl.dllib.nn.abstractnn.AbstractModule import com.intel.analytics.bigdl.dllib.keras.Reshape import com.intel.analytics.bigdl.dllib.keras.{Sequential => KSequential} import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/SReLUSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/SReLUSpec.scala index aa7496d6bb4..c9348fec4f7 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/SReLUSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/SReLUSpec.scala @@ -21,7 +21,7 @@ import com.intel.analytics.bigdl.dllib.keras.{Sequential => KSequential} import com.intel.analytics.bigdl.dllib.nn.abstractnn.AbstractModule import com.intel.analytics.bigdl.dllib.keras.SReLU import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/SeparableConvolution2DSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/SeparableConvolution2DSpec.scala index 534310e91ae..6d57de42d35 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/SeparableConvolution2DSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/SeparableConvolution2DSpec.scala @@ -20,7 +20,7 @@ import com.intel.analytics.bigdl.keras.KerasBaseSpec import com.intel.analytics.bigdl.dllib.nn.abstractnn.{AbstractModule, DataFormat} import com.intel.analytics.bigdl.dllib.keras.{SeparableConvolution2D, Sequential => KSequential} import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/SequentialSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/SequentialSpec.scala index af18516decc..a044e49ceb3 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/SequentialSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/SequentialSpec.scala @@ -18,7 +18,7 @@ package com.intel.analytics.bigdl.keras.nn import com.intel.analytics.bigdl.dllib.keras.{Dense, Sequential => KSequential} import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/SimpleRNNSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/SimpleRNNSpec.scala index 001b94e6e90..c266c55ddc3 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/SimpleRNNSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/SimpleRNNSpec.scala @@ -20,7 +20,7 @@ import com.intel.analytics.bigdl.keras.KerasBaseSpec import com.intel.analytics.bigdl.dllib.nn.abstractnn.AbstractModule import com.intel.analytics.bigdl.dllib.keras.{Dense, SimpleRNN, Sequential => KSequential} import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/SpatialDropout1DSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/SpatialDropout1DSpec.scala index b7e7bc9b3ac..d92283c1ff8 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/SpatialDropout1DSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/SpatialDropout1DSpec.scala @@ -20,7 +20,7 @@ import com.intel.analytics.bigdl.keras.KerasBaseSpec import com.intel.analytics.bigdl.dllib.keras.SpatialDropout1D import com.intel.analytics.bigdl.dllib.keras.{Sequential => KSequential} import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/SpatialDropout2DSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/SpatialDropout2DSpec.scala index 13e78abeef4..7e51e5f6686 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/SpatialDropout2DSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/SpatialDropout2DSpec.scala @@ -21,7 +21,7 @@ import com.intel.analytics.bigdl.dllib.nn.abstractnn.DataFormat import com.intel.analytics.bigdl.dllib.keras.SpatialDropout2D import com.intel.analytics.bigdl.dllib.keras.{Sequential => KSequential} import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/SpatialDropout3DSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/SpatialDropout3DSpec.scala index 29e29435fab..275ea13d952 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/SpatialDropout3DSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/SpatialDropout3DSpec.scala @@ -21,7 +21,7 @@ import com.intel.analytics.bigdl.dllib.nn.abstractnn.DataFormat import com.intel.analytics.bigdl.dllib.keras.SpatialDropout3D import com.intel.analytics.bigdl.dllib.keras.{Sequential => KSequential} import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/ThresholdedReLUSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/ThresholdedReLUSpec.scala index 9e5d01a6df2..f8631a48920 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/ThresholdedReLUSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/ThresholdedReLUSpec.scala @@ -21,7 +21,7 @@ import com.intel.analytics.bigdl.dllib.keras.{Sequential => KSequential} import com.intel.analytics.bigdl.dllib.nn.abstractnn.AbstractModule import com.intel.analytics.bigdl.dllib.keras.ThresholdedReLU import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/TimeDistributedSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/TimeDistributedSpec.scala index c09da12f573..02176a810ba 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/TimeDistributedSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/TimeDistributedSpec.scala @@ -20,7 +20,7 @@ import com.intel.analytics.bigdl.keras.KerasBaseSpec import com.intel.analytics.bigdl.dllib.nn.abstractnn.AbstractModule import com.intel.analytics.bigdl.dllib.keras.{Convolution2D, Dense, TimeDistributed, Sequential => KSequential} import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/TrainingSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/TrainingSpec.scala index e29c4ac8fdd..0781eef63c7 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/TrainingSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/TrainingSpec.scala @@ -19,8 +19,8 @@ import com.intel.analytics.bigdl.dllib.feature.dataset.Sample import com.intel.analytics.bigdl.dllib.nn.MSECriterion import com.intel.analytics.bigdl.dllib.keras._ import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.{Engine} -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.{Engine} +import com.intel.analytics.bigdl.dllib.utils.Shape import com.intel.analytics.bigdl.dllib.optim.{DummyDataSet, SGD, Top1Accuracy} import org.apache.spark.SparkContext import org.apache.spark.rdd.RDD diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/UpSampling1DSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/UpSampling1DSpec.scala index 345292312a1..002684688c7 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/UpSampling1DSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/UpSampling1DSpec.scala @@ -20,7 +20,7 @@ import com.intel.analytics.bigdl.keras.KerasBaseSpec import com.intel.analytics.bigdl.dllib.nn.abstractnn.AbstractModule import com.intel.analytics.bigdl.dllib.keras.{UpSampling1D, Sequential => KSequential} import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/UpSampling2DSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/UpSampling2DSpec.scala index cf87ace86f2..57235153a22 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/UpSampling2DSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/UpSampling2DSpec.scala @@ -20,7 +20,7 @@ import com.intel.analytics.bigdl.keras.KerasBaseSpec import com.intel.analytics.bigdl.dllib.nn.abstractnn.{AbstractModule, DataFormat} import com.intel.analytics.bigdl.dllib.keras.{UpSampling2D, Sequential => KSequential} import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/UpSampling3DSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/UpSampling3DSpec.scala index b181be95a15..eae949a1be8 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/UpSampling3DSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/UpSampling3DSpec.scala @@ -20,7 +20,7 @@ import com.intel.analytics.bigdl.keras.KerasBaseSpec import com.intel.analytics.bigdl.dllib.nn.abstractnn.AbstractModule import com.intel.analytics.bigdl.dllib.keras.{UpSampling3D, Sequential => KSequential} import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/ZeroPadding1DSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/ZeroPadding1DSpec.scala index d46d7c3a28d..281a818170b 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/ZeroPadding1DSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/ZeroPadding1DSpec.scala @@ -20,7 +20,7 @@ import com.intel.analytics.bigdl.keras.KerasBaseSpec import com.intel.analytics.bigdl.dllib.nn.abstractnn.AbstractModule import com.intel.analytics.bigdl.dllib.keras.{ZeroPadding1D, Sequential => KSequential} import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/ZeroPadding2DSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/ZeroPadding2DSpec.scala index 27350c6c30e..7c902b1e808 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/ZeroPadding2DSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/ZeroPadding2DSpec.scala @@ -20,7 +20,7 @@ import com.intel.analytics.bigdl.keras.KerasBaseSpec import com.intel.analytics.bigdl.dllib.nn.abstractnn.{AbstractModule, DataFormat} import com.intel.analytics.bigdl.dllib.keras.{ZeroPadding2D, Sequential => KSequential} import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/ZeroPadding3DSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/ZeroPadding3DSpec.scala index 442f5bce26c..9deda06941b 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/ZeroPadding3DSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/keras/nn/ZeroPadding3DSpec.scala @@ -21,7 +21,7 @@ import com.intel.analytics.bigdl.dllib.nn.abstractnn.AbstractModule import com.intel.analytics.bigdl.dllib.keras.ZeroPadding3D import com.intel.analytics.bigdl.dllib.keras.{Sequential => KSequential} import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/models/AlexNetSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/models/AlexNetSpec.scala index 51f785691a8..cde7ae69314 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/models/AlexNetSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/models/AlexNetSpec.scala @@ -19,7 +19,7 @@ package com.intel.analytics.bigdl.dllib.models import com.intel.analytics.bigdl.dllib.example.loadmodel.{AlexNet, AlexNet_OWT} import com.intel.analytics.bigdl.numeric.NumericFloat import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import org.scalatest.{FlatSpec, Matchers} import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/models/InceptionSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/models/InceptionSpec.scala index 55e424e2d3e..5adc62b2c55 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/models/InceptionSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/models/InceptionSpec.scala @@ -20,7 +20,7 @@ import com.intel.analytics.bigdl.dllib.models.inception._ import com.intel.analytics.bigdl.dllib.nn.{Graph, Input} import com.intel.analytics.bigdl.numeric.NumericFloat import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import com.intel.analytics.bigdl.dllib.utils.{T, Table} import org.scalatest.{FlatSpec, Matchers} diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/models/ModelGraientCheckSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/models/ModelGraientCheckSpec.scala index 1064ed9ad8c..473377ac547 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/models/ModelGraientCheckSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/models/ModelGraientCheckSpec.scala @@ -18,7 +18,7 @@ package com.intel.analytics.bigdl.dllib.models import com.intel.analytics.bigdl.dllib.nn.{ClassNLLCriterion, GradientChecker} import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import org.scalatest.{BeforeAndAfter, FlatSpec, Matchers} import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/models/ResNetSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/models/ResNetSpec.scala index f93082717db..ac04c10abf1 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/models/ResNetSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/models/ResNetSpec.scala @@ -23,9 +23,9 @@ import com.intel.analytics.bigdl.dllib.nn.Graph.{apply => _, _} import com.intel.analytics.bigdl.dllib.nn.{Graph, _} import com.intel.analytics.bigdl.numeric.NumericFloat import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.RandomGenerator.RNG +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator.RNG import com.intel.analytics.bigdl.dllib.utils.{T} -import com.intel.analytics.bigdl.utils.RandomGenerator +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator import org.apache.log4j.Logger import org.scalatest.{FlatSpec, Matchers} diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/models/maskrcnn/MaskRCNNSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/models/maskrcnn/MaskRCNNSpec.scala index d27ccee8c18..1c7bd2d72a8 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/models/maskrcnn/MaskRCNNSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/models/maskrcnn/MaskRCNNSpec.scala @@ -24,7 +24,7 @@ import com.intel.analytics.bigdl.dllib.feature.transform.vision.image.RoiImageIn import com.intel.analytics.bigdl.dllib.feature.transform.vision.image.label.roi.RoiLabel import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest import com.intel.analytics.bigdl.dllib.utils.{T, Table} -import com.intel.analytics.bigdl.utils.RandomGenerator +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator import org.scalatest.{FlatSpec, Matchers} import scala.reflect.io.File diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/AddSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/AddSpec.scala index 7682876dc16..ccdb76d9816 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/AddSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/AddSpec.scala @@ -19,7 +19,7 @@ package com.intel.analytics.bigdl.dllib.nn import com.intel.analytics.bigdl.dllib.nn.abstractnn.{TensorCriterion, TensorModule} import com.intel.analytics.bigdl.dllib.tensor.Tensor import org.scalatest.{FlatSpec, Matchers} -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/BatchNormalizationSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/BatchNormalizationSpec.scala index f2d889c005e..21725d2bd6f 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/BatchNormalizationSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/BatchNormalizationSpec.scala @@ -19,7 +19,7 @@ package com.intel.analytics.bigdl.dllib.nn import com.intel.analytics.bigdl.dllib.tensor.{Storage, Tensor} import com.intel.analytics.bigdl.dllib.utils.{T} import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest -import com.intel.analytics.bigdl.utils._ +import com.intel.analytics.bigdl.dllib.utils._ import org.apache.spark.SparkContext import org.scalatest.{BeforeAndAfter, FlatSpec, Matchers} diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/BiRecurrentSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/BiRecurrentSpec.scala index 6b494970576..e22183f5964 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/BiRecurrentSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/BiRecurrentSpec.scala @@ -16,7 +16,7 @@ package com.intel.analytics.bigdl.dllib.nn import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/BilinearSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/BilinearSpec.scala index 24a8c120d67..f7997b0bef0 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/BilinearSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/BilinearSpec.scala @@ -18,7 +18,7 @@ package com.intel.analytics.bigdl.dllib.nn import com.intel.analytics.bigdl.dllib.tensor.Tensor import org.scalatest.{FlatSpec, Matchers} -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import com.intel.analytics.bigdl.dllib.utils.Table import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/BinaryTreeLSTMSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/BinaryTreeLSTMSpec.scala index 71841edcb7c..7d8c65db902 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/BinaryTreeLSTMSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/BinaryTreeLSTMSpec.scala @@ -16,7 +16,7 @@ package com.intel.analytics.bigdl.dllib.nn import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import com.intel.analytics.bigdl.dllib.utils.T import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest import org.scalatest.{BeforeAndAfter, FlatSpec, Matchers} @@ -24,7 +24,7 @@ import org.scalatest.{BeforeAndAfter, FlatSpec, Matchers} class BinaryTreeLSTMSpec extends FlatSpec with Matchers with BeforeAndAfter { "BinaryTreeLSTM" should "works correctly" in { import com.intel.analytics.bigdl.numeric.NumericFloat - import com.intel.analytics.bigdl.utils.RandomGenerator.RNG + import com.intel.analytics.bigdl.dllib.utils.RandomGenerator.RNG RNG.setSeed(100) diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/CAddSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/CAddSpec.scala index d52d597653b..c9aad0afbe0 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/CAddSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/CAddSpec.scala @@ -19,7 +19,7 @@ package com.intel.analytics.bigdl.dllib.nn import com.intel.analytics.bigdl.dllib.nn.abstractnn.{TensorCriterion, TensorModule} import com.intel.analytics.bigdl.dllib.tensor.Tensor import org.scalatest.{FlatSpec, Matchers} -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/CMulSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/CMulSpec.scala index 16845a88cc2..f671072cfe9 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/CMulSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/CMulSpec.scala @@ -19,7 +19,7 @@ package com.intel.analytics.bigdl.dllib.nn import com.intel.analytics.bigdl.dllib.nn import com.intel.analytics.bigdl.dllib.tensor.Tensor import org.scalatest.{FlatSpec, Matchers} -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import com.intel.analytics.bigdl.dllib.utils.Table import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/ConcatSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/ConcatSpec.scala index 6a6e82b62b0..d6f353a60ca 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/ConcatSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/ConcatSpec.scala @@ -17,7 +17,7 @@ package com.intel.analytics.bigdl.dllib.nn import com.intel.analytics.bigdl.dllib.tensor.{Storage, Tensor} -import com.intel.analytics.bigdl.utils.LayerException +import com.intel.analytics.bigdl.dllib.utils.LayerException import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest import org.scalatest.{FlatSpec, Matchers} diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/ConvLSTMPeepholeSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/ConvLSTMPeepholeSpec.scala index af5ce60fd4f..9490e61681d 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/ConvLSTMPeepholeSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/ConvLSTMPeepholeSpec.scala @@ -20,7 +20,7 @@ import com.intel.analytics.bigdl.dllib.nn._ import com.intel.analytics.bigdl.dllib.optim.{L2Regularizer, SGD} import com.intel.analytics.bigdl.dllib.utils._ import com.intel.analytics.bigdl.dllib.tensor.{Storage, Tensor} -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import com.intel.analytics.bigdl.dllib.utils.TorchObject.TYPE_DOUBLE_TENSOR import org.scalatest.{BeforeAndAfter, FlatSpec, Matchers} diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/CosineSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/CosineSpec.scala index c46dbb805a0..ea20cbcc9b4 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/CosineSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/CosineSpec.scala @@ -18,7 +18,7 @@ package com.intel.analytics.bigdl.dllib.nn import com.intel.analytics.bigdl.dllib.tensor.Tensor import org.scalatest.{FlatSpec, Matchers} -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import com.intel.analytics.bigdl.dllib.utils.Table import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/DetectionOutputFrcnnSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/DetectionOutputFrcnnSpec.scala index a72094eeb82..cc46b9ced6f 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/DetectionOutputFrcnnSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/DetectionOutputFrcnnSpec.scala @@ -18,7 +18,7 @@ package com.intel.analytics.bigdl.dllib.nn import java.io.File import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import com.intel.analytics.bigdl.dllib.utils.serializer.{ModuleLoader, ModulePersister, ModuleSerializationTest} import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/DetectionOutputSSDSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/DetectionOutputSSDSpec.scala index 997b7028743..29610ea2cf7 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/DetectionOutputSSDSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/DetectionOutputSSDSpec.scala @@ -18,7 +18,7 @@ package com.intel.analytics.bigdl.dllib.nn import java.io.File import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import com.intel.analytics.bigdl.dllib.utils.serializer.{ModuleLoader, ModulePersister, ModuleSerializationTest} import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/DropoutSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/DropoutSpec.scala index a86f0357cf6..af149624c89 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/DropoutSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/DropoutSpec.scala @@ -16,7 +16,7 @@ package com.intel.analytics.bigdl.dllib.nn import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/DynamicGraphSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/DynamicGraphSpec.scala index 86a97c6ef40..656a3241cf1 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/DynamicGraphSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/DynamicGraphSpec.scala @@ -27,11 +27,11 @@ import com.intel.analytics.bigdl.dllib.nn.abstractnn.EmptyGradInput import com.intel.analytics.bigdl.dllib.nn.ops.Less import com.intel.analytics.bigdl.dllib.nn.tf.{ControlNodes, Enter, Const} import com.intel.analytics.bigdl.numeric.NumericFloat -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.utils._ import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest -import com.intel.analytics.bigdl.utils._ +import com.intel.analytics.bigdl.dllib.utils._ import scala.reflect.ClassTag import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/EuclideanSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/EuclideanSpec.scala index 94b2d0c9ca6..15f0c8e7e18 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/EuclideanSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/EuclideanSpec.scala @@ -18,7 +18,7 @@ package com.intel.analytics.bigdl.dllib.nn import com.intel.analytics.bigdl.dllib.tensor.Tensor import org.scalatest.{FlatSpec, Matchers} -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import com.intel.analytics.bigdl.dllib.utils.Table import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/GRUSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/GRUSpec.scala index cb85c5436a0..37df3ebc349 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/GRUSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/GRUSpec.scala @@ -17,7 +17,7 @@ package com.intel.analytics.bigdl.dllib.nn import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.RandomGenerator.RNG +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator.RNG import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/GaussianDropoutSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/GaussianDropoutSpec.scala index 26bb899ac62..7e176368ae1 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/GaussianDropoutSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/GaussianDropoutSpec.scala @@ -17,7 +17,7 @@ package com.intel.analytics.bigdl.dllib.nn import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest import org.scalatest.{FlatSpec, Matchers} diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/GaussianNoiseSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/GaussianNoiseSpec.scala index 482a9aef889..2bf41a928f3 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/GaussianNoiseSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/GaussianNoiseSpec.scala @@ -17,7 +17,7 @@ package com.intel.analytics.bigdl.dllib.nn import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest import org.scalatest.{FlatSpec, Matchers} diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/GaussianSamplerSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/GaussianSamplerSpec.scala index 027ae6bb505..f137589f507 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/GaussianSamplerSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/GaussianSamplerSpec.scala @@ -16,7 +16,7 @@ package com.intel.analytics.bigdl.dllib.nn import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import com.intel.analytics.bigdl.dllib.utils.T import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/GraphSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/GraphSpec.scala index bb95f01405e..720df8c2e6d 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/GraphSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/GraphSpec.scala @@ -26,10 +26,10 @@ import com.intel.analytics.bigdl.dllib.nn.abstractnn.EmptyGradInput import com.intel.analytics.bigdl.dllib.nn.ops.{Ceil, Less} import com.intel.analytics.bigdl.dllib.nn.tf.{Const, ControlNodes} import com.intel.analytics.bigdl.numeric.NumericFloat -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.utils._ -import com.intel.analytics.bigdl.utils._ +import com.intel.analytics.bigdl.dllib.utils._ import scala.reflect.ClassTag import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/InferReshapeSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/InferReshapeSpec.scala index e7cdc36df20..76caac7cb7d 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/InferReshapeSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/InferReshapeSpec.scala @@ -17,7 +17,7 @@ package com.intel.analytics.bigdl.dllib.nn import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.LayerException +import com.intel.analytics.bigdl.dllib.utils.LayerException import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest import org.scalatest.FlatSpec diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/KLDCriterionSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/KLDCriterionSpec.scala index f903b1bcc19..406525afba0 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/KLDCriterionSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/KLDCriterionSpec.scala @@ -17,7 +17,7 @@ package com.intel.analytics.bigdl.dllib.nn import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import com.intel.analytics.bigdl.dllib.utils.T import org.scalatest.{FlatSpec, Matchers} diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/LinearSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/LinearSpec.scala index 69ed63e0918..c6576e5384c 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/LinearSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/LinearSpec.scala @@ -25,7 +25,7 @@ import com.intel.analytics.bigdl._ import com.intel.analytics.bigdl.dllib.optim.{L1Regularizer, L2Regularizer, SGD} import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest import com.intel.analytics.bigdl.dllib.utils.{T, TestUtils} -import com.intel.analytics.bigdl.utils.{RandomGenerator, Shape} +import com.intel.analytics.bigdl.dllib.utils.{RandomGenerator, Shape} import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/LogSoftMaxSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/LogSoftMaxSpec.scala index 570f98b268b..cfecdadd7b7 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/LogSoftMaxSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/LogSoftMaxSpec.scala @@ -17,7 +17,7 @@ package com.intel.analytics.bigdl.dllib.nn import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.Engine +import com.intel.analytics.bigdl.dllib.utils.Engine import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest import org.scalatest.{BeforeAndAfter, FlatSpec, Matchers} diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/LookupTableSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/LookupTableSpec.scala index 500589d29d6..5e9c08182a1 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/LookupTableSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/LookupTableSpec.scala @@ -18,7 +18,7 @@ package com.intel.analytics.bigdl.dllib.nn import com.intel.analytics.bigdl.dllib.tensor.Tensor import org.scalatest.{FlatSpec, Matchers} -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import com.intel.analytics.bigdl.dllib.utils.Table import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/MaskHeadSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/MaskHeadSpec.scala index 9bb4c627d89..4850f925840 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/MaskHeadSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/MaskHeadSpec.scala @@ -18,7 +18,7 @@ package com.intel.analytics.bigdl.dllib.nn import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest import com.intel.analytics.bigdl.dllib.utils.{T, Table} -import com.intel.analytics.bigdl.utils.RandomGenerator +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator import org.scalatest.{FlatSpec, Matchers} class MaskHeadSpec extends FlatSpec with Matchers { diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/MaxoutSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/MaxoutSpec.scala index 737b52b8dfc..ef97110ed26 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/MaxoutSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/MaxoutSpec.scala @@ -19,7 +19,7 @@ package com.intel.analytics.bigdl.dllib.nn import com.intel.analytics.bigdl.keras.KerasBaseSpec import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.utils.{TestUtils} -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/ModuleSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/ModuleSpec.scala index 42274952a3d..6adc3702537 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/ModuleSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/ModuleSpec.scala @@ -19,7 +19,7 @@ package com.intel.analytics.bigdl.dllib.nn import com.intel.analytics.bigdl.dllib.models.lenet.LeNet5 import com.intel.analytics.bigdl.dllib.tensor.{Storage, Tensor} import com.intel.analytics.bigdl.dllib.utils.{Table} -import com.intel.analytics.bigdl.utils.RandomGenerator +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator import org.scalatest.{FlatSpec, Matchers} import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/MulSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/MulSpec.scala index 75dec4f0d7d..c3bb3dc1383 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/MulSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/MulSpec.scala @@ -18,7 +18,7 @@ package com.intel.analytics.bigdl.dllib.nn import com.intel.analytics.bigdl.dllib.tensor.Tensor import org.scalatest.{FlatSpec, Matchers} -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import com.intel.analytics.bigdl.dllib.utils.Table import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/MultiRNNCellSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/MultiRNNCellSpec.scala index 4a55a4a7eb8..b57a51b87cb 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/MultiRNNCellSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/MultiRNNCellSpec.scala @@ -20,7 +20,7 @@ import com.intel.analytics.bigdl.dllib.nn._ import com.intel.analytics.bigdl.dllib.optim.{L2Regularizer, SGD} import com.intel.analytics.bigdl.dllib.utils._ import com.intel.analytics.bigdl.dllib.tensor.{Storage, Tensor} -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import com.intel.analytics.bigdl.dllib.utils.Table import com.intel.analytics.bigdl.dllib.utils.TorchObject.TYPE_DOUBLE_TENSOR import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/NormalizeSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/NormalizeSpec.scala index 64d4ab336b6..3ef3249b0c1 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/NormalizeSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/NormalizeSpec.scala @@ -18,7 +18,7 @@ package com.intel.analytics.bigdl.dllib.nn import com.intel.analytics.bigdl._ import com.intel.analytics.bigdl.dllib.tensor.{Storage, Tensor} -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest import org.scalatest.{FlatSpec, Matchers} diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/PReLUSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/PReLUSpec.scala index 9f9c256e916..e5004f5707a 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/PReLUSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/PReLUSpec.scala @@ -18,7 +18,7 @@ package com.intel.analytics.bigdl.dllib.nn import com.intel.analytics.bigdl.dllib.tensor.Tensor import org.scalatest.{FlatSpec, Matchers} -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/PoolerSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/PoolerSpec.scala index f05f7429017..5a8c24c38b6 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/PoolerSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/PoolerSpec.scala @@ -18,7 +18,7 @@ package com.intel.analytics.bigdl.dllib.nn import org.scalatest.{FlatSpec, Matchers} import com.intel.analytics.bigdl.dllib.tensor.{Storage, Tensor} -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest import com.intel.analytics.bigdl.dllib.utils.{T, Table} diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/RecurrentDecoderSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/RecurrentDecoderSpec.scala index 9e88a65dfe0..5dfdc830831 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/RecurrentDecoderSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/RecurrentDecoderSpec.scala @@ -21,7 +21,7 @@ import com.intel.analytics.bigdl.dllib.nn.abstractnn.Activity import com.intel.analytics.bigdl.dllib.optim.{L2Regularizer, SGD} import com.intel.analytics.bigdl.dllib.utils._ import com.intel.analytics.bigdl.dllib.tensor.{Storage, Tensor} -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest import org.scalatest.{BeforeAndAfter, FlatSpec, Matchers} diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/RecurrentSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/RecurrentSpec.scala index 617896608ab..0e378f7fe7f 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/RecurrentSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/RecurrentSpec.scala @@ -18,10 +18,10 @@ package com.intel.analytics.bigdl.dllib.nn import com.intel.analytics.bigdl.dllib.optim.SGD import com.intel.analytics.bigdl.dllib.tensor.{Storage, Tensor} -import com.intel.analytics.bigdl.utils.RandomGenerator.RNG +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator.RNG import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest import com.intel.analytics.bigdl.dllib.utils.{T, Table} -import com.intel.analytics.bigdl.utils.Engine +import com.intel.analytics.bigdl.dllib.utils.Engine import org.scalatest.{FlatSpec, Matchers} import scala.collection.mutable.ArrayBuffer diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/ReshapeSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/ReshapeSpec.scala index 0012c466ada..28b2667b080 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/ReshapeSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/ReshapeSpec.scala @@ -18,7 +18,7 @@ package com.intel.analytics.bigdl.dllib.nn import org.scalatest.FlatSpec import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.LayerException +import com.intel.analytics.bigdl.dllib.utils.LayerException import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/ReverseSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/ReverseSpec.scala index 6bb846c670e..d11b65e2715 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/ReverseSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/ReverseSpec.scala @@ -16,7 +16,7 @@ package com.intel.analytics.bigdl.dllib.nn import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.RandomGenerator +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest import org.scalatest.{FlatSpec, Matchers} diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/RoiAlignSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/RoiAlignSpec.scala index f4d4bd0b0f7..ecaab939f31 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/RoiAlignSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/RoiAlignSpec.scala @@ -18,7 +18,7 @@ package com.intel.analytics.bigdl.dllib.nn import com.intel.analytics.bigdl.dllib.nn.mkldnn.Equivalent import com.intel.analytics.bigdl.dllib.tensor.{Storage, Tensor} -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest import com.intel.analytics.bigdl.dllib.utils.{T, Table} import org.scalatest.{FlatSpec, Matchers} diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/SparseJoinTableSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/SparseJoinTableSpec.scala index 3139feed5b6..5cc16d61245 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/SparseJoinTableSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/SparseJoinTableSpec.scala @@ -18,7 +18,7 @@ package com.intel.analytics.bigdl.dllib.nn import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.utils.{T} -import com.intel.analytics.bigdl.utils.RandomGenerator +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator import com.intel.analytics.bigdl.numeric.NumericFloat import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest import org.scalatest.{FlatSpec, Matchers} diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/SparseLinearSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/SparseLinearSpec.scala index 6924d86999a..4f790038363 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/SparseLinearSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/SparseLinearSpec.scala @@ -20,7 +20,7 @@ import org.scalatest.{BeforeAndAfter, FlatSpec, Matchers} import com.intel.analytics.bigdl.numeric.NumericFloat import com.intel.analytics.bigdl.dllib.tensor.{SparseTensor, Tensor} import com.intel.analytics.bigdl.dllib.utils.{T} -import com.intel.analytics.bigdl.utils.RandomGenerator +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/SpatialBatchNormalizationSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/SpatialBatchNormalizationSpec.scala index 523db022c4e..42cbdc85769 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/SpatialBatchNormalizationSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/SpatialBatchNormalizationSpec.scala @@ -17,8 +17,8 @@ package com.intel.analytics.bigdl.dllib.nn import com.intel.analytics.bigdl.dllib.nn.abstractnn.DataFormat import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.Engine -import com.intel.analytics.bigdl.utils.RandomGenerator.RNG +import com.intel.analytics.bigdl.dllib.utils.Engine +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator.RNG import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest import org.apache.spark.SparkContext import org.scalatest.{BeforeAndAfter, FlatSpec, Matchers} diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/SpatialContrastiveNormalizationSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/SpatialContrastiveNormalizationSpec.scala index 15250c987dc..c7eb1d9babb 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/SpatialContrastiveNormalizationSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/SpatialContrastiveNormalizationSpec.scala @@ -16,7 +16,7 @@ package com.intel.analytics.bigdl.dllib.nn import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/SpatialConvolutionMapSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/SpatialConvolutionMapSpec.scala index 188da9097f9..30567e48324 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/SpatialConvolutionMapSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/SpatialConvolutionMapSpec.scala @@ -18,7 +18,7 @@ package com.intel.analytics.bigdl.dllib.nn import com.intel.analytics.bigdl.dllib.tensor.Tensor import org.scalatest.{FlatSpec, Matchers} -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import com.intel.analytics.bigdl.dllib.utils.Table import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/SpatialConvolutionSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/SpatialConvolutionSpec.scala index d3365ea2187..3e7cef5fdb4 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/SpatialConvolutionSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/SpatialConvolutionSpec.scala @@ -24,12 +24,12 @@ import com.intel.analytics.bigdl._ import com.intel.analytics.bigdl.dllib.tensor import com.intel.analytics.bigdl.dllib.nn.abstractnn.DataFormat import com.intel.analytics.bigdl.dllib.optim.{L2Regularizer, SGD} -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest import scala.util.Random import com.intel.analytics.bigdl.dllib.utils.{T, TestUtils} -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape @com.intel.analytics.bigdl.tags.Parallel class SpatialConvolutionSpec extends FlatSpec with Matchers { diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/SpatialDilatedConvolutionSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/SpatialDilatedConvolutionSpec.scala index 671c41d50d3..fb2c9795468 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/SpatialDilatedConvolutionSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/SpatialDilatedConvolutionSpec.scala @@ -18,10 +18,10 @@ package com.intel.analytics.bigdl.dllib.nn import com.intel.analytics.bigdl.dllib.tensor.Tensor import org.scalatest.{FlatSpec, Matchers} -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest import com.intel.analytics.bigdl.dllib.utils.{TestUtils} -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/SpatialFullConvolutionSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/SpatialFullConvolutionSpec.scala index 0696fe5a300..f657aab4c17 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/SpatialFullConvolutionSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/SpatialFullConvolutionSpec.scala @@ -19,8 +19,8 @@ package com.intel.analytics.bigdl.dllib.nn import com.intel.analytics.bigdl.dllib.optim.{L2Regularizer, SGD} import com.intel.analytics.bigdl.dllib.tensor.{Storage, Tensor} import com.intel.analytics.bigdl.dllib.utils.{T, TestUtils} -import com.intel.analytics.bigdl.utils.RandomGenerator._ -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.Shape import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest import org.scalatest.{FlatSpec, Matchers} diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/SpatialMaxPoolingSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/SpatialMaxPoolingSpec.scala index 4335300ebe2..de8bdbaf0e8 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/SpatialMaxPoolingSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/SpatialMaxPoolingSpec.scala @@ -16,7 +16,7 @@ package com.intel.analytics.bigdl.dllib.nn -import com.intel.analytics.bigdl.utils.RandomGenerator +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator import com.intel.analytics.bigdl._ import com.intel.analytics.bigdl.dllib.tensor import com.intel.analytics.bigdl.dllib.nn.abstractnn.DataFormat diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/SpatialSeparableConvolutionSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/SpatialSeparableConvolutionSpec.scala index 8f53d8ec62b..4cdbcf2b1ec 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/SpatialSeparableConvolutionSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/SpatialSeparableConvolutionSpec.scala @@ -19,7 +19,7 @@ import com.intel.analytics.bigdl.dllib.nn.abstractnn.DataFormat import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest import com.intel.analytics.bigdl.dllib.utils.{BigDLSpecHelper, TestUtils} -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/SpatialWithinChannelLRNSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/SpatialWithinChannelLRNSpec.scala index aabcebc02ff..15f1f18d554 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/SpatialWithinChannelLRNSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/SpatialWithinChannelLRNSpec.scala @@ -18,7 +18,7 @@ package com.intel.analytics.bigdl.dllib.nn import com.intel.analytics.bigdl._ import com.intel.analytics.bigdl.dllib.tensor.{Storage, Tensor} -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest import org.scalatest.{FlatSpec, Matchers} diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/TimeDistributedSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/TimeDistributedSpec.scala index 89435431562..48ff769208e 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/TimeDistributedSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/TimeDistributedSpec.scala @@ -18,7 +18,7 @@ package com.intel.analytics.bigdl.dllib.nn import com.intel.analytics.bigdl.dllib.nn.abstractnn.{AbstractModule, Activity} import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest import org.scalatest.{FlatSpec, Matchers} diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/mkldnn/AvgPoolingSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/mkldnn/AvgPoolingSpec.scala index 2fac230f384..132b8cbb4a6 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/mkldnn/AvgPoolingSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/mkldnn/AvgPoolingSpec.scala @@ -22,8 +22,8 @@ import com.intel.analytics.bigdl.dllib.nn.abstractnn.DataFormat import com.intel.analytics.bigdl.dllib.nn.mkldnn.Phase.{InferencePhase, TrainingPhase} import com.intel.analytics.bigdl.dllib.tensor.{DnnTensor, Tensor} import com.intel.analytics.bigdl.dllib.utils.BigDLSpecHelper -import com.intel.analytics.bigdl.utils.{Engine, MklDnn} -import com.intel.analytics.bigdl.utils.RandomGenerator.RNG +import com.intel.analytics.bigdl.dllib.utils.{Engine, MklDnn} +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator.RNG import com.intel.analytics.bigdl.dllib.utils.intermediate.{BlasToIR, IRToDnn} import org.apache.commons.lang3.SerializationUtils diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/mkldnn/BlasWrapperSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/mkldnn/BlasWrapperSpec.scala index b744ae1d88b..716d69b53d7 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/mkldnn/BlasWrapperSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/mkldnn/BlasWrapperSpec.scala @@ -26,9 +26,9 @@ import com.intel.analytics.bigdl.dllib.nn.{Graph, Squeeze, mkldnn} import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric import com.intel.analytics.bigdl.dllib.utils.{BigDLSpecHelper, T} -import com.intel.analytics.bigdl.utils._ -import com.intel.analytics.bigdl.utils.RandomGenerator -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import com.intel.analytics.bigdl.numeric.NumericFloat class BlasWrapperSpec extends BigDLSpecHelper { diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/mkldnn/DnnGraphSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/mkldnn/DnnGraphSpec.scala index 30176d573e1..d1c27141740 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/mkldnn/DnnGraphSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/mkldnn/DnnGraphSpec.scala @@ -26,9 +26,9 @@ import com.intel.analytics.bigdl.dllib.nn.mkldnn.Phase.{InferencePhase, Training import com.intel.analytics.bigdl.dllib.nn import com.intel.analytics.bigdl.dllib.nn.{Graph, Module => _, _} import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils._ import com.intel.analytics.bigdl.dllib.utils._ -import com.intel.analytics.bigdl.utils._ import org.scalatest.{BeforeAndAfter, FlatSpec, Matchers} import com.intel.analytics.bigdl.dllib.models.resnet import com.intel.analytics.bigdl.dllib.models.utils.ModelBroadcast diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/mkldnn/FusionSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/mkldnn/FusionSpec.scala index 8d937940a71..1e018b173f3 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/mkldnn/FusionSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/mkldnn/FusionSpec.scala @@ -22,8 +22,8 @@ import com.intel.analytics.bigdl.dllib.nn.mkldnn.Phase.InferencePhase import org.scalatest.{FlatSpec, Matchers} import com.intel.analytics.bigdl.numeric.NumericFloat import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.{Engine} -import com.intel.analytics.bigdl.utils.RandomGenerator +import com.intel.analytics.bigdl.dllib.utils.{Engine} +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator import com.intel.analytics.bigdl.dllib.utils.intermediate.ConversionUtils class FusionSpec extends FlatSpec with Matchers { diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/mkldnn/LRNSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/mkldnn/LRNSpec.scala index 3b006ede0ca..5ea3725ab9d 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/mkldnn/LRNSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/mkldnn/LRNSpec.scala @@ -20,7 +20,7 @@ import com.intel.analytics.bigdl.dllib.nn.SpatialCrossMapLRN import com.intel.analytics.bigdl.dllib.nn.mkldnn.Phase.{InferencePhase, TrainingPhase} import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.utils.BigDLSpecHelper -import com.intel.analytics.bigdl.utils.RandomGenerator.RNG +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator.RNG import org.apache.commons.lang3.SerializationUtils import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/mkldnn/LinearSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/mkldnn/LinearSpec.scala index 532773f450f..c5bc606c73f 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/mkldnn/LinearSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/mkldnn/LinearSpec.scala @@ -23,9 +23,9 @@ import com.intel.analytics.bigdl.dllib.nn.mkldnn.Phase.TrainingPhase import com.intel.analytics.bigdl.numeric.NumericFloat import com.intel.analytics.bigdl.dllib.optim.L2Regularizer import com.intel.analytics.bigdl.dllib.tensor.{DnnStorage, Tensor} -import com.intel.analytics.bigdl.utils.{Engine, MklDnn} -import com.intel.analytics.bigdl.utils.RandomGenerator -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.{Engine, MklDnn} +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import org.apache.commons.lang3.SerializationUtils import org.scalatest.{FlatSpec, Matchers} diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/mkldnn/MaxPoolingSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/mkldnn/MaxPoolingSpec.scala index 97eb350a295..0d8730d1466 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/mkldnn/MaxPoolingSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/mkldnn/MaxPoolingSpec.scala @@ -21,8 +21,8 @@ import com.intel.analytics.bigdl.dllib.nn.mkldnn.Phase.{InferencePhase, Training import com.intel.analytics.bigdl.dllib.nn.{Graph, SpatialAveragePooling, SpatialMaxPooling, StaticGraph} import com.intel.analytics.bigdl.dllib.tensor.{DnnTensor, Tensor} import com.intel.analytics.bigdl.dllib.utils.{BigDLSpecHelper} -import com.intel.analytics.bigdl.utils.RandomGenerator.RNG -import com.intel.analytics.bigdl.utils._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator.RNG +import com.intel.analytics.bigdl.dllib.utils._ import com.intel.analytics.bigdl.dllib.utils.intermediate.{BlasToIR, IRToDnn} import org.apache.commons.lang3.SerializationUtils diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/mkldnn/MultiModelsSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/mkldnn/MultiModelsSpec.scala index c68890d6405..2127a5fd802 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/mkldnn/MultiModelsSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/mkldnn/MultiModelsSpec.scala @@ -22,9 +22,9 @@ import com.intel.analytics.bigdl.dllib.models.lenet.LeNet5 import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.feature.transform.vision.image.{ImageFeature, ImageFrame, ImageFrameToSample, MatToTensor} import com.intel.analytics.bigdl.dllib.feature.transform.vision.image.augmentation.{CenterCrop, ChannelNormalize, Resize} -import com.intel.analytics.bigdl.utils.{Engine} -import com.intel.analytics.bigdl.utils.LoggerFilter -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.{Engine} +import com.intel.analytics.bigdl.dllib.utils.LoggerFilter +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import org.apache.spark.SparkContext import org.scalatest.{BeforeAndAfter, FlatSpec, Matchers} diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/mkldnn/OutputSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/mkldnn/OutputSpec.scala index 0621df716f7..6aedb930f8b 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/mkldnn/OutputSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/mkldnn/OutputSpec.scala @@ -26,7 +26,7 @@ import com.intel.analytics.bigdl.dllib.nn.mkldnn.Phase.TrainingPhase import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.numeric.NumericFloat import com.intel.analytics.bigdl.dllib.utils.{BigDLSpecHelper} -import com.intel.analytics.bigdl.utils.RandomGenerator +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator class OutputSpec extends BigDLSpecHelper { def model(shape: Array[Int], layout: Int) : DnnGraph = { diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/mkldnn/RNNSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/mkldnn/RNNSpec.scala index a1fe4faa5ea..4aaf7af8654 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/mkldnn/RNNSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/mkldnn/RNNSpec.scala @@ -25,9 +25,9 @@ import com.intel.analytics.bigdl.dllib.nn import com.intel.analytics.bigdl.dllib.nn._ import com.intel.analytics.bigdl.dllib.nn.abstractnn.AbstractModule import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric.NumericFloat -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils._ import com.intel.analytics.bigdl.dllib.utils._ -import com.intel.analytics.bigdl.utils._ import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/mkldnn/ReorderMemorySpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/mkldnn/ReorderMemorySpec.scala index 1cc0449a322..5f20914a261 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/mkldnn/ReorderMemorySpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/mkldnn/ReorderMemorySpec.scala @@ -23,7 +23,7 @@ import com.intel.analytics.bigdl.mkl.Memory import com.intel.analytics.bigdl.dllib.nn.mkldnn.Phase.{InferencePhase, TrainingPhase} import com.intel.analytics.bigdl.dllib.tensor.{DnnTensor, Tensor} import com.intel.analytics.bigdl.numeric.NumericFloat -import com.intel.analytics.bigdl.utils.Engine +import com.intel.analytics.bigdl.dllib.utils.Engine import org.scalatest.{BeforeAndAfter, FlatSpec, Matchers} class ReorderMemorySpec extends FlatSpec with Matchers with BeforeAndAfter { diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/mkldnn/SpatialBatchNormalizationSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/mkldnn/SpatialBatchNormalizationSpec.scala index 0b30926a132..70ed19ff9c1 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/mkldnn/SpatialBatchNormalizationSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/mkldnn/SpatialBatchNormalizationSpec.scala @@ -22,7 +22,7 @@ import com.intel.analytics.bigdl.dllib.nn.abstractnn.Activity import com.intel.analytics.bigdl.dllib.nn.mkldnn.Phase.TrainingPhase import com.intel.analytics.bigdl.numeric.NumericFloat import com.intel.analytics.bigdl.dllib.tensor.{DnnStorage, Tensor} -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import org.apache.commons.lang3.SerializationUtils import org.scalatest.{FlatSpec, Ignore, Matchers} diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/mkldnn/SpatialConvolutionSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/mkldnn/SpatialConvolutionSpec.scala index 55afa35992f..12353693a93 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/mkldnn/SpatialConvolutionSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/mkldnn/SpatialConvolutionSpec.scala @@ -23,7 +23,7 @@ import com.intel.analytics.bigdl.dllib.nn.mkldnn.Phase.{InferencePhase, Training import com.intel.analytics.bigdl.dllib.nn.{Xavier, Zeros} import com.intel.analytics.bigdl.numeric.NumericFloat import com.intel.analytics.bigdl.dllib.tensor.{DnnStorage, Tensor} -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import org.apache.commons.lang3.SerializationUtils import org.scalatest.{FlatSpec, Matchers} diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/mkldnn/TopologySpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/mkldnn/TopologySpec.scala index 0f13cf49ebb..8d7849d4036 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/mkldnn/TopologySpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/mkldnn/TopologySpec.scala @@ -26,7 +26,7 @@ import com.intel.analytics.bigdl.dllib.nn.mkldnn.ResNet.DatasetType.ImageNet import com.intel.analytics.bigdl.numeric.NumericFloat import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.utils.{T} -import com.intel.analytics.bigdl.utils.RandomGenerator +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator import org.scalatest.{FlatSpec, Matchers} class TopologySpec extends FlatSpec with Matchers { diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/ops/MaxSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/ops/MaxSpec.scala index 9ebf8c32db3..74b8e609512 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/ops/MaxSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/ops/MaxSpec.scala @@ -18,7 +18,7 @@ package com.intel.analytics.bigdl.dllib.nn.ops import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.utils.serializer.ModuleSerializationTest import com.intel.analytics.bigdl.dllib.utils.{T} -import com.intel.analytics.bigdl.utils.RandomGenerator +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator import org.scalatest.{FlatSpec, Matchers} class MaxSpec extends FlatSpec with Matchers { diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/quantized/QuantizableSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/quantized/QuantizableSpec.scala index eaa8f1a19a3..864272018c0 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/quantized/QuantizableSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/nn/quantized/QuantizableSpec.scala @@ -24,9 +24,9 @@ import com.intel.analytics.bigdl.dllib.nn.quantized.Utils.ANode import com.intel.analytics.bigdl.dllib.nn.{Linear => NNLinear, SpatialConvolution => NNConv, SpatialDilatedConvolution => NNDilatedConv, _} import com.intel.analytics.bigdl.numeric.NumericFloat import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.RandomGenerator.RNG +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator.RNG import com.intel.analytics.bigdl.dllib.utils.{T, Table} -import com.intel.analytics.bigdl.utils._ +import com.intel.analytics.bigdl.dllib.utils._ import org.apache.log4j.Logger import org.scalatest.{BeforeAndAfter, FlatSpec, Matchers} diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/optim/AdamSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/optim/AdamSpec.scala index 08e2b040f36..cf6c446aa6b 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/optim/AdamSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/optim/AdamSpec.scala @@ -19,8 +19,8 @@ package com.intel.analytics.bigdl.dllib.optim import com.intel.analytics.bigdl.dllib.nn.{CrossEntropyCriterion, Linear, Sequential} import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.utils.{T, TestUtils} -import com.intel.analytics.bigdl.utils.RandomGenerator -import com.intel.analytics.bigdl.utils._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator +import com.intel.analytics.bigdl.dllib.utils._ import org.scalatest.{BeforeAndAfter, FlatSpec, Matchers} import scala.collection.mutable.ArrayBuffer diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/optim/DistriOptimizerSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/optim/DistriOptimizerSpec.scala index 496b61458d3..ef08ba17dd8 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/optim/DistriOptimizerSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/optim/DistriOptimizerSpec.scala @@ -31,7 +31,7 @@ import com.intel.analytics.bigdl.dllib.optim.parameters.AllReduceParameter import com.intel.analytics.bigdl.dllib.tensor.{DenseTensor, DnnStorage, Storage, Tensor} import com.intel.analytics.bigdl.dllib.utils._ import com.intel.analytics.bigdl.dllib.visualization.TrainSummary -import com.intel.analytics.bigdl.utils._ +import com.intel.analytics.bigdl.dllib.utils._ import org.apache.log4j.{Level, Logger} import org.apache.spark.SparkContext import org.apache.spark.rdd.RDD diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/optim/DistriOptimizerV2Spec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/optim/DistriOptimizerV2Spec.scala index 2789fc00b7b..bf8ab86f6bf 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/optim/DistriOptimizerV2Spec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/optim/DistriOptimizerV2Spec.scala @@ -31,7 +31,7 @@ import com.intel.analytics.bigdl.dllib.optim.parameters.AllReduceParameter import com.intel.analytics.bigdl.dllib.tensor.{DenseTensor, DnnStorage, Storage, Tensor} import com.intel.analytics.bigdl.dllib.utils._ import com.intel.analytics.bigdl.dllib.visualization.TrainSummary -import com.intel.analytics.bigdl.utils._ +import com.intel.analytics.bigdl.dllib.utils._ import org.apache.log4j.{Level, Logger} import org.apache.spark.SparkContext import org.apache.spark.rdd.RDD diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/optim/EvaluatorSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/optim/EvaluatorSpec.scala index 60a8949365d..138c6ff7c85 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/optim/EvaluatorSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/optim/EvaluatorSpec.scala @@ -21,8 +21,8 @@ import com.intel.analytics.bigdl.dllib.models.lenet.LeNet5 import com.intel.analytics.bigdl.dllib.nn.CrossEntropyCriterion import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.utils.{SparkContextLifeCycle} -import com.intel.analytics.bigdl.utils.{Engine, MklBlas, MklDnn} -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.{Engine, MklBlas, MklDnn} +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import org.apache.spark.{SparkConf, SparkContext} import org.scalatest.{BeforeAndAfter, FlatSpec, Matchers} import com.intel.analytics.bigdl._ diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/optim/FtrlSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/optim/FtrlSpec.scala index 5642a0d5491..25fc4db519d 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/optim/FtrlSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/optim/FtrlSpec.scala @@ -18,7 +18,7 @@ package com.intel.analytics.bigdl.dllib.optim import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.utils.{T, TestUtils} -import com.intel.analytics.bigdl.utils._ +import com.intel.analytics.bigdl.dllib.utils._ import org.scalatest.{FlatSpec, Matchers} import scala.collection.mutable.ArrayBuffer diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/optim/LarsSGDSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/optim/LarsSGDSpec.scala index ab9f7155248..6d41f92abb6 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/optim/LarsSGDSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/optim/LarsSGDSpec.scala @@ -22,7 +22,7 @@ import com.intel.analytics.bigdl.dllib.feature.dataset.{DataSet, MiniBatch} import com.intel.analytics.bigdl.dllib.nn.{Linear, MSECriterion, ReLU, Sequential} import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.utils.{T, TestUtils} -import com.intel.analytics.bigdl.utils._ +import com.intel.analytics.bigdl.dllib.utils._ import org.scalatest.{BeforeAndAfter, FlatSpec, Matchers} import com.intel.analytics.bigdl.dllib.optim._ diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/optim/LocalOptimizerSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/optim/LocalOptimizerSpec.scala index 743cdbab199..a29adf6702c 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/optim/LocalOptimizerSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/optim/LocalOptimizerSpec.scala @@ -25,7 +25,7 @@ import com.intel.analytics.bigdl.mkl.Memory import com.intel.analytics.bigdl.dllib.nn.mkldnn.HeapData import com.intel.analytics.bigdl.dllib.tensor.{DnnStorage, Storage, Tensor} import com.intel.analytics.bigdl.dllib.utils.{T} -import com.intel.analytics.bigdl.utils._ +import com.intel.analytics.bigdl.dllib.utils._ import com.intel.analytics.bigdl.dllib.visualization.TrainSummary import org.scalatest.{BeforeAndAfter, FlatSpec, Matchers} diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/optim/LocalPredictorSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/optim/LocalPredictorSpec.scala index 8861bef21d1..92c7cd58d2d 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/optim/LocalPredictorSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/optim/LocalPredictorSpec.scala @@ -28,8 +28,8 @@ import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.feature.transform.vision.image._ import com.intel.analytics.bigdl.dllib.feature.transform.vision.image.augmentation.{CenterCrop, ChannelNormalize, Resize} import com.intel.analytics.bigdl.dllib.utils._ -import com.intel.analytics.bigdl.utils.RandomGenerator._ -import com.intel.analytics.bigdl.utils._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils._ import org.apache.commons.io.FileUtils import org.scalatest.{BeforeAndAfter, FlatSpec, Matchers} diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/optim/LoggerFilterSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/optim/LoggerFilterSpec.scala index 4b8e6df1bf3..6041c82df9c 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/optim/LoggerFilterSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/optim/LoggerFilterSpec.scala @@ -26,7 +26,7 @@ import com.intel.analytics.bigdl.dllib.nn.{Linear, MSECriterion, Sequential} import com.intel.analytics.bigdl.numeric.NumericDouble import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.utils.{T, TestUtils} -import com.intel.analytics.bigdl.utils._ +import com.intel.analytics.bigdl.dllib.utils._ import org.apache.log4j.{Level, Logger, PatternLayout, WriterAppender} import org.apache.spark.SparkContext import org.scalatest.{BeforeAndAfter, FlatSpec, Matchers} diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/optim/MetricsSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/optim/MetricsSpec.scala index 73f01d83019..400015b60eb 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/optim/MetricsSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/optim/MetricsSpec.scala @@ -16,7 +16,7 @@ package com.intel.analytics.bigdl.dllib.optim -import com.intel.analytics.bigdl.utils.Engine +import com.intel.analytics.bigdl.dllib.utils.Engine import org.apache.spark.{SparkConf, SparkContext, SparkException} import org.scalatest.{BeforeAndAfter, FlatSpec, Matchers} diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/optim/OptimPredictorShutdownSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/optim/OptimPredictorShutdownSpec.scala index 5b30cd282a3..d8943f498b0 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/optim/OptimPredictorShutdownSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/optim/OptimPredictorShutdownSpec.scala @@ -29,9 +29,9 @@ import com.intel.analytics.bigdl.numeric.NumericFloat import com.intel.analytics.bigdl.dllib.tensor.{DnnStorage, Storage, Tensor} import com.intel.analytics.bigdl.dllib.feature.transform.vision.image.augmentation.{CenterCrop, ChannelNormalize, Resize} import com.intel.analytics.bigdl.dllib.feature.transform.vision.image.{ImageFeature, ImageFrame, ImageFrameToSample, MatToTensor} -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import com.intel.analytics.bigdl.dllib.utils.{SparkContextLifeCycle} -import com.intel.analytics.bigdl.utils._ +import com.intel.analytics.bigdl.dllib.utils._ import org.apache.log4j.{Level, Logger} import org.apache.spark.rdd.RDD import org.apache.spark.{SparkConf, SparkContext} diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/optim/OptimizerSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/optim/OptimizerSpec.scala index eee485a74c1..b8a832e239b 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/optim/OptimizerSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/optim/OptimizerSpec.scala @@ -24,7 +24,7 @@ import com.intel.analytics.bigdl._ import com.intel.analytics.bigdl.dllib.example.loadmodel.AlexNet import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.utils.{File, T, Table} -import com.intel.analytics.bigdl.utils._ +import com.intel.analytics.bigdl.dllib.utils._ import org.apache.spark.SparkContext import org.apache.spark.rdd.RDD import org.scalatest.{BeforeAndAfter, FlatSpec, Matchers} diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/optim/ParallelOptimizerSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/optim/ParallelOptimizerSpec.scala index 88355f8ada1..ca9dd6ef2e6 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/optim/ParallelOptimizerSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/optim/ParallelOptimizerSpec.scala @@ -20,7 +20,7 @@ import com.intel.analytics.bigdl.dllib.nn.{ClassNLLCriterion, Linear, MSECriteri import com.intel.analytics.bigdl.dllib.optim.DistriOptimizerSpecModel.mse import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.utils.{T} -import com.intel.analytics.bigdl.utils._ +import com.intel.analytics.bigdl.dllib.utils._ import org.apache.log4j.{Level, Logger} import org.apache.spark.{SparkConf, SparkContext} import org.scalatest.{BeforeAndAfter, FlatSpec, Matchers} diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/optim/PredictionServiceSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/optim/PredictionServiceSpec.scala index e9e6e27582c..f2bb8e9fa74 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/optim/PredictionServiceSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/optim/PredictionServiceSpec.scala @@ -18,7 +18,7 @@ package com.intel.analytics.bigdl.dllib.optim import com.intel.analytics.bigdl.dllib.nn._ import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.RandomGenerator.RNG +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator.RNG import com.intel.analytics.bigdl.dllib.utils.T import org.scalatest.{FlatSpec, Matchers} diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/optim/PredictorSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/optim/PredictorSpec.scala index 7fcd8150f48..0782b9f2ddf 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/optim/PredictorSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/optim/PredictorSpec.scala @@ -26,8 +26,8 @@ import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.feature.transform.vision.image._ import com.intel.analytics.bigdl.dllib.feature.transform.vision.image.augmentation.{CenterCrop, ChannelNormalize, Resize} import com.intel.analytics.bigdl.dllib.utils.{Table, T} -import com.intel.analytics.bigdl.utils._ -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import com.intel.analytics.bigdl.dllib.utils.SparkContextLifeCycle import org.apache.commons.lang3.SerializationUtils import org.apache.log4j.{Level, Logger} diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/optim/ValidatorSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/optim/ValidatorSpec.scala index a517e62dca8..431adc18cd0 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/optim/ValidatorSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/optim/ValidatorSpec.scala @@ -20,8 +20,8 @@ import com.intel.analytics.bigdl.dllib.feature.dataset.{DataSet, Sample, SampleT import com.intel.analytics.bigdl.dllib.models.lenet.LeNet5 import com.intel.analytics.bigdl.dllib.nn.CrossEntropyCriterion import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.Engine -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.Engine +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import com.intel.analytics.bigdl._ import com.intel.analytics.bigdl.dllib.utils.SparkContextLifeCycle import org.apache.spark.{SparkConf, SparkContext} diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/parameters/FP16ParameterSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/parameters/FP16ParameterSpec.scala index ab07850f63c..b88c8568de6 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/parameters/FP16ParameterSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/parameters/FP16ParameterSpec.scala @@ -17,7 +17,7 @@ package com.intel.analytics.bigdl.dllib.optim.parameters import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric.NumericDouble -import com.intel.analytics.bigdl.utils.Engine +import com.intel.analytics.bigdl.dllib.utils.Engine import org.scalatest.{BeforeAndAfter, FlatSpec, Matchers} import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/python/api/PythonSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/python/api/PythonSpec.scala index 076f9de4606..6a6dcdbf065 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/python/api/PythonSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/python/api/PythonSpec.scala @@ -25,7 +25,7 @@ import com.intel.analytics.bigdl.dllib.nn._ import com.intel.analytics.bigdl.dllib.optim._ import com.intel.analytics.bigdl.dllib.utils.{T, Table, TestUtils} import com.intel.analytics.bigdl.dllib.visualization.{TrainSummary, ValidationSummary} -import com.intel.analytics.bigdl.utils._ +import com.intel.analytics.bigdl.dllib.utils._ import org.apache.log4j.{Level, Logger} import org.apache.spark.{SparkConf, SparkContext} import org.apache.spark.api.java.JavaRDD @@ -33,7 +33,7 @@ import org.apache.spark.bigdl.api.python.BigDLSerDe import org.scalatest.{BeforeAndAfter, FlatSpec, Matchers} import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.feature.transform.vision.image.{ImageFeature, ImageFrame, ImageFrameToSample} -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ import scala.util.Random import scala.collection.JavaConverters._ diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/tensor/SparseTensorSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/tensor/SparseTensorSpec.scala index d824ad5a7f1..0647907b1a9 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/tensor/SparseTensorSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/tensor/SparseTensorSpec.scala @@ -17,7 +17,7 @@ package com.intel.analytics.bigdl.dllib.tensor import org.scalatest.{FlatSpec, Matchers} import com.intel.analytics.bigdl.numeric.NumericFloat -import com.intel.analytics.bigdl.utils.RandomGenerator +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/transform/vision/image/ImageFrameSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/transform/vision/image/ImageFrameSpec.scala index 782094df832..f5847093a03 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/transform/vision/image/ImageFrameSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/transform/vision/image/ImageFrameSpec.scala @@ -25,7 +25,7 @@ import com.intel.analytics.bigdl.dllib.feature.dataset.{DataSet, Sample} import com.intel.analytics.bigdl.numeric.NumericFloat import com.intel.analytics.bigdl.dllib.feature.transform.vision.image.augmentation._ import com.intel.analytics.bigdl.dllib.utils.{TestUtils} -import com.intel.analytics.bigdl.utils._ +import com.intel.analytics.bigdl.dllib.utils._ import org.apache.commons.io.FileUtils import org.apache.spark.SparkContext import org.apache.spark.sql.SQLContext diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/transform/vision/image/MTImageFeatureToBatchSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/transform/vision/image/MTImageFeatureToBatchSpec.scala index 0c9e7512c4a..3592e197363 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/transform/vision/image/MTImageFeatureToBatchSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/transform/vision/image/MTImageFeatureToBatchSpec.scala @@ -21,7 +21,7 @@ import com.intel.analytics.bigdl.dllib.feature.dataset.segmentation.RLEMasks import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.feature.transform.vision.image.label.roi.RoiLabel import com.intel.analytics.bigdl.dllib.utils.{T, Table} -import com.intel.analytics.bigdl.utils._ +import com.intel.analytics.bigdl.dllib.utils._ import org.apache.spark.SparkContext import org.scalatest.{BeforeAndAfter, FlatSpec, Matchers} diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/transform/vision/image/opencv/OpenCVMatSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/transform/vision/image/opencv/OpenCVMatSpec.scala index 8923fbf6ae1..d995c8568e3 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/transform/vision/image/opencv/OpenCVMatSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/transform/vision/image/opencv/OpenCVMatSpec.scala @@ -22,7 +22,7 @@ import com.intel.analytics.bigdl.opencv.OpenCV import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.feature.transform.vision.image.util.BoundingBox import com.intel.analytics.bigdl.dllib.utils.{T} -import com.intel.analytics.bigdl.utils._ +import com.intel.analytics.bigdl.dllib.utils._ import org.apache.commons.io.FileUtils import org.apache.log4j.{Level, Logger} import org.apache.spark.SparkContext diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/CaffeLoaderSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/CaffeLoaderSpec.scala index 471940944c5..bbd3e00bc04 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/CaffeLoaderSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/CaffeLoaderSpec.scala @@ -26,7 +26,7 @@ import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric.NumericDouble import com.intel.analytics.bigdl.dllib.tensor.{Storage, Tensor} import com.intel.analytics.bigdl.dllib.utils.caffe.{CaffeLoader, Customizable} -import com.intel.analytics.bigdl.utils._ +import com.intel.analytics.bigdl.dllib.utils._ import org.scalatest.{FlatSpec, Matchers} import scala.collection.mutable diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/DistributedSynchronizerSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/DistributedSynchronizerSpec.scala index fb7ec98adf0..a2a1c3a9138 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/DistributedSynchronizerSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/DistributedSynchronizerSpec.scala @@ -17,7 +17,7 @@ package com.intel.analytics.bigdl.dllib.utils import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils._ +import com.intel.analytics.bigdl.dllib.utils._ import org.apache.spark.{SparkContext, TaskContext} import org.scalatest.{BeforeAndAfter, FlatSpec, Matchers} diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/EngineSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/EngineSpec.scala index e7388812e57..582d2fc79ae 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/EngineSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/EngineSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ - package com.intel.analytics.bigdl.utils + package com.intel.analytics.bigdl.dllib.utils import org.apache.spark.SparkContext import org.scalatest.{BeforeAndAfter, FlatSpec, Matchers} diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/GraphNodeSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/GraphNodeSpec.scala index a24c088197c..f3de3e7abdc 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/GraphNodeSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/GraphNodeSpec.scala @@ -22,7 +22,7 @@ import com.intel.analytics.bigdl.dllib.models.Inception import com.intel.analytics.bigdl.dllib.models.resnet.ResNet import com.intel.analytics.bigdl.dllib.models.resnet.ResNet.{DatasetType, ShortcutType} import com.intel.analytics.bigdl.dllib.nn._ -import com.intel.analytics.bigdl.utils.RandomGenerator.RNG +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator.RNG import org.scalatest.{FlatSpec, Matchers} import com.intel.analytics.bigdl.numeric.NumericFloat import com.intel.analytics.bigdl.dllib.tensor.Tensor diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/RandomGeneratorSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/RandomGeneratorSpec.scala index dc391695e65..4b8bfadbb4f 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/RandomGeneratorSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/RandomGeneratorSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.utils +package com.intel.analytics.bigdl.dllib.utils import org.scalatest.{BeforeAndAfter, FlatSpec, Matchers} diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/ShapeSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/ShapeSpec.scala index 74761b93d5f..75ff40d4d4e 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/ShapeSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/ShapeSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.utils +package com.intel.analytics.bigdl.dllib.utils import org.scalatest.{BeforeAndAfter, FlatSpec, Matchers} diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/SparkContextLifeCycle.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/SparkContextLifeCycle.scala index 00e7fcb9673..88a4e2f8168 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/SparkContextLifeCycle.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/SparkContextLifeCycle.scala @@ -15,7 +15,7 @@ */ package com.intel.analytics.bigdl.dllib.utils -import com.intel.analytics.bigdl.utils._ +import com.intel.analytics.bigdl.dllib.utils._ import org.apache.spark.{SparkConf, SparkContext} import org.scalatest.{BeforeAndAfter, FlatSpec, Matchers} diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/SpatialShareConvolutionSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/SpatialShareConvolutionSpec.scala index e5b84fa9c80..f4d9eb8f663 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/SpatialShareConvolutionSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/SpatialShareConvolutionSpec.scala @@ -23,7 +23,7 @@ import com.intel.analytics.bigdl.dllib.models.resnet.ResNet import com.intel.analytics.bigdl.dllib.models.resnet.ResNet.{DatasetType, ShortcutType} import com.intel.analytics.bigdl.dllib.models.vgg.{Vgg_16} import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric.NumericFloat -import com.intel.analytics.bigdl.utils._ +import com.intel.analytics.bigdl.dllib.utils._ import org.scalatest.{FlatSpec, Matchers} import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/TestUtils.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/TestUtils.scala index 7d015eea9c5..b8bf39c641c 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/TestUtils.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/TestUtils.scala @@ -23,7 +23,7 @@ import com.intel.analytics.bigdl.dllib.keras.{InputLayer, KerasLayer, Sequential import com.intel.analytics.bigdl.dllib.nn.abstractnn.{AbstractModule, Activity, TensorModule} import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric -import com.intel.analytics.bigdl.utils.Shape +import com.intel.analytics.bigdl.dllib.utils.Shape import org.scalatest.exceptions.TestCanceledException import scala.reflect.ClassTag diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/ThreadPoolSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/ThreadPoolSpec.scala index a5a89cd2e3b..8d01045d6f3 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/ThreadPoolSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/ThreadPoolSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.analytics.bigdl.utils +package com.intel.analytics.bigdl.dllib.utils import com.intel.analytics.bigdl.mkl.hardware.Affinity import org.scalatest.{FlatSpec, Matchers} diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/UtilSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/UtilSpec.scala index 03e36e1af8a..3d1d9ddabec 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/UtilSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/UtilSpec.scala @@ -22,7 +22,7 @@ import com.intel.analytics.bigdl.dllib.optim.DistriOptimizer.{Cache, CacheV1} import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric import com.intel.analytics.bigdl.dllib.utils.Util.{setExtraParametersFromModelRDD, cloneParameters} -import com.intel.analytics.bigdl.utils._ +import com.intel.analytics.bigdl.dllib.utils._ import org.apache.spark.SparkContext import org.scalatest.{BeforeAndAfter, FlatSpec, Matchers} diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/intermediate/BlasToDnnSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/intermediate/BlasToDnnSpec.scala index 51dbd02fdee..a5c121483b2 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/intermediate/BlasToDnnSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/intermediate/BlasToDnnSpec.scala @@ -29,9 +29,9 @@ import com.intel.analytics.bigdl.dllib.nn import com.intel.analytics.bigdl.dllib.nn.mkldnn.{DnnGraph, Equivalent} import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.numeric.NumericFloat -import com.intel.analytics.bigdl.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils._ import com.intel.analytics.bigdl.dllib.utils._ -import com.intel.analytics.bigdl.utils._ import scala.util.Random diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/intermediate/DnnPredictorSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/intermediate/DnnPredictorSpec.scala index c85e55bab6c..32b038a61c1 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/intermediate/DnnPredictorSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/intermediate/DnnPredictorSpec.scala @@ -25,8 +25,8 @@ import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.feature.transform.vision.image.{ImageFeature, ImageFrame, ImageFrameToSample, MatToTensor} import com.intel.analytics.bigdl.dllib.feature.transform.vision.image.augmentation.{CenterCrop, ChannelNormalize, Resize} import com.intel.analytics.bigdl.dllib.utils._ -import com.intel.analytics.bigdl.utils.RandomGenerator._ -import com.intel.analytics.bigdl.utils._ +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator._ +import com.intel.analytics.bigdl.dllib.utils._ import org.apache.spark.SparkContext import org.scalatest.{BeforeAndAfter, FlatSpec, Matchers} diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/intermediate/IRGraphSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/intermediate/IRGraphSpec.scala index 46949af8781..137f4e995a5 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/intermediate/IRGraphSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/intermediate/IRGraphSpec.scala @@ -27,7 +27,7 @@ import com.intel.analytics.bigdl.dllib.utils import com.intel.analytics.bigdl.dllib.nn.{Graph, Reshape, StaticGraph, TimeDistributed} import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.utils._ -import com.intel.analytics.bigdl.utils._ +import com.intel.analytics.bigdl.dllib.utils._ class IRGraphSpec extends BigDLSpecHelper { def modelIR(inputFormats: Int = Memory.Format.nchw, diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/intermediate/IRconvertSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/intermediate/IRconvertSpec.scala index 3dc98ae588d..e5a22146014 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/intermediate/IRconvertSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/intermediate/IRconvertSpec.scala @@ -26,7 +26,7 @@ import com.intel.analytics.bigdl.dllib.nn.mkldnn.{DnnGraph, Equivalent, Input, O import com.intel.analytics.bigdl.numeric.NumericFloat import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.utils._ -import com.intel.analytics.bigdl.utils._ +import com.intel.analytics.bigdl.dllib.utils._ import com.intel.analytics.bigdl.Module class IRconvertSpec extends BigDLSpecHelper { @@ -96,7 +96,7 @@ class IRconvertSpec extends BigDLSpecHelper { def kerasLayer(classNum: Int, shape: Shape = Shape(28, 28, 3)): keras.Sequential[Float] = { import com.intel.analytics.bigdl.dllib.keras._ - import com.intel.analytics.bigdl.utils.Shape + import com.intel.analytics.bigdl.dllib.utils.Shape val model = Sequential() model.add(Convolution2D(6, 5, 5, activation = "tanh", diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/serializer/DataConverterSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/serializer/DataConverterSpec.scala index ae98d597b7f..cbc9de78f22 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/serializer/DataConverterSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/serializer/DataConverterSpec.scala @@ -23,13 +23,13 @@ import com.intel.analytics.bigdl.dllib.nn.abstractnn.{AbstractModule, Activity, import com.intel.analytics.bigdl.dllib.nn.quantized.{LinearWeight, LinearWeightParams} import com.intel.analytics.bigdl.dllib.optim.{L1L2Regularizer, L1Regularizer, L2Regularizer, Regularizer} import com.intel.analytics.bigdl.dllib.tensor.{QuantizedTensor, Storage, Tensor} -import com.intel.analytics.bigdl.utils.{MultiShape, SingleShape, Shape => BigDLShape} +import com.intel.analytics.bigdl.dllib.utils.{MultiShape, SingleShape, Shape => BigDLShape} import org.scalatest.{FlatSpec, Matchers} import com.intel.analytics.bigdl.serialization.Bigdl.{AttrValue, BigDLTensor, DataType, TensorStorage} import scala.reflect.runtime.universe import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.TensorNumeric.NumericFloat -import com.intel.analytics.bigdl.utils.SingleShape +import com.intel.analytics.bigdl.dllib.utils.SingleShape import com.intel.analytics.bigdl.dllib.utils.serializer.converters.DataConverter import com.intel.analytics.bigdl.serialization.Bigdl.AttrValue.ArrayValue diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/serializer/SerializerSpecHelper.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/serializer/SerializerSpecHelper.scala index db175592ed5..009ae7d733b 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/serializer/SerializerSpecHelper.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/serializer/SerializerSpecHelper.scala @@ -22,9 +22,9 @@ import java.lang.reflect.Modifier import com.intel.analytics.bigdl.dllib.nn.abstractnn.{AbstractModule, Activity} import com.intel.analytics.bigdl.dllib.nn.ops.{Exp => ExpOps, Pow => PowOps, Select => SelectOps, Sum => SumOps, Tile => TileOps} import com.intel.analytics.bigdl.dllib.nn.tf.{DecodeGif => DecodeGifOps, DecodeJpeg => DecodeJpegOps, DecodePng => DecodePngOps, DecodeRaw => DecodeRawOps} -import com.intel.analytics.bigdl.utils.RandomGenerator.RNG +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator.RNG import com.intel.analytics.bigdl.dllib.utils.tf.loaders.{Pack => _} -import com.intel.analytics.bigdl.utils.{Shape => KShape} +import com.intel.analytics.bigdl.dllib.utils.{Shape => KShape} import org.reflections.Reflections import org.reflections.scanners.SubTypesScanner import org.reflections.util.{ClasspathHelper, ConfigurationBuilder, FilterBuilder} diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/tf/SessionSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/tf/SessionSpec.scala index d2846ad08a7..c20abe7dd71 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/tf/SessionSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/tf/SessionSpec.scala @@ -19,7 +19,7 @@ import com.intel.analytics.bigdl.dllib.feature.dataset._ import com.intel.analytics.bigdl.dllib.nn.MSECriterion import com.intel.analytics.bigdl.dllib.optim.{SGD, Trigger} import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.Engine +import com.intel.analytics.bigdl.dllib.utils.Engine import org.apache.log4j.{Level, Logger} import org.apache.spark.SparkContext import org.scalatest.{BeforeAndAfter, FlatSpec, Matchers} diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/tf/TensorflowLoaderSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/tf/TensorflowLoaderSpec.scala index 957e4cf3a36..250aa135387 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/tf/TensorflowLoaderSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/tf/TensorflowLoaderSpec.scala @@ -25,7 +25,7 @@ import com.intel.analytics.bigdl.dllib.nn._ import com.intel.analytics.bigdl.dllib.optim.{DistriOptimizer, Trigger} import com.intel.analytics.bigdl.dllib.tensor.{Storage, Tensor} import com.intel.analytics.bigdl.dllib.utils._ -import com.intel.analytics.bigdl.utils._ +import com.intel.analytics.bigdl.dllib.utils._ import org.apache.log4j.{Level, Logger} import org.apache.spark.SparkContext import org.apache.spark.rdd.RDD diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/tf/TensorflowSpecHelper.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/tf/TensorflowSpecHelper.scala index dbd2ff4c7a5..003df61c377 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/tf/TensorflowSpecHelper.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/tf/TensorflowSpecHelper.scala @@ -26,7 +26,7 @@ import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.tensor.TensorNumericMath.{NumericWildCard, TensorNumeric} import com.intel.analytics.bigdl.dllib.utils.{BigDLSpecHelper, FileWriter, T} import com.intel.analytics.bigdl.dllib.utils.tf.Tensorflow.const -import com.intel.analytics.bigdl.utils._ +import com.intel.analytics.bigdl.dllib.utils._ import org.tensorflow.framework.{GraphDef, NodeDef} import scala.language.postfixOps diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/tf/loaders/DepthwiseConv2DNativeSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/tf/loaders/DepthwiseConv2DNativeSpec.scala index 21b8acb7620..04590ddddd0 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/tf/loaders/DepthwiseConv2DNativeSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/tf/loaders/DepthwiseConv2DNativeSpec.scala @@ -16,7 +16,7 @@ package com.intel.analytics.bigdl.dllib.utils.tf.loaders import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.RandomGenerator.RNG +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator.RNG import com.intel.analytics.bigdl.dllib.utils.T import com.intel.analytics.bigdl.dllib.utils.tf.Tensorflow.{listIntAttr, typeAttr} import com.intel.analytics.bigdl.dllib.utils.tf.{PaddingType, TensorflowDataFormat, TensorflowSpecHelper} diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/tf/loaders/DepthwiseConv2dNativeBackpropFilterSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/tf/loaders/DepthwiseConv2dNativeBackpropFilterSpec.scala index c191982ef91..c9c85d00662 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/tf/loaders/DepthwiseConv2dNativeBackpropFilterSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/tf/loaders/DepthwiseConv2dNativeBackpropFilterSpec.scala @@ -16,7 +16,7 @@ package com.intel.analytics.bigdl.dllib.utils.tf.loaders import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.RandomGenerator.RNG +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator.RNG import com.intel.analytics.bigdl.dllib.utils.T import com.intel.analytics.bigdl.dllib.utils.tf.Tensorflow.{listIntAttr, typeAttr} import com.intel.analytics.bigdl.dllib.utils.tf.{PaddingType, TensorflowDataFormat, TensorflowSpecHelper} diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/tf/loaders/DepthwiseConv2dNativeBackpropInputSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/tf/loaders/DepthwiseConv2dNativeBackpropInputSpec.scala index 354a84cc541..751aaf391bc 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/tf/loaders/DepthwiseConv2dNativeBackpropInputSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/tf/loaders/DepthwiseConv2dNativeBackpropInputSpec.scala @@ -16,7 +16,7 @@ package com.intel.analytics.bigdl.dllib.utils.tf.loaders import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.RandomGenerator.RNG +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator.RNG import com.intel.analytics.bigdl.dllib.utils.T import com.intel.analytics.bigdl.dllib.utils.tf.Tensorflow.{listIntAttr, typeAttr} import com.intel.analytics.bigdl.dllib.utils.tf.{PaddingType, TensorflowDataFormat, TensorflowSpecHelper} diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/tf/loaders/DigammaSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/tf/loaders/DigammaSpec.scala index bf29c74de13..98ca446be2d 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/tf/loaders/DigammaSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/tf/loaders/DigammaSpec.scala @@ -15,7 +15,7 @@ */ package com.intel.analytics.bigdl.dllib.utils.tf.loaders import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.RandomGenerator +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator class DigammaSpec extends UnaryOpBaseSpec { diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/tf/loaders/FusedBatchNormGradSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/tf/loaders/FusedBatchNormGradSpec.scala index 128b2dcb239..7c9a669a839 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/tf/loaders/FusedBatchNormGradSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/tf/loaders/FusedBatchNormGradSpec.scala @@ -16,7 +16,7 @@ package com.intel.analytics.bigdl.dllib.utils.tf.loaders import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.RandomGenerator +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator import com.intel.analytics.bigdl.dllib.utils.tf.Tensorflow.{booleanAttr, floatAttr, typeAttr} import com.intel.analytics.bigdl.dllib.utils.tf.{TensorflowDataFormat, TensorflowSpecHelper} import org.tensorflow.framework.{DataType, NodeDef} diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/tf/loaders/LRNGradSpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/tf/loaders/LRNGradSpec.scala index ebdb452c6aa..94cdf71d25c 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/tf/loaders/LRNGradSpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/tf/loaders/LRNGradSpec.scala @@ -18,7 +18,7 @@ package com.intel.analytics.bigdl.dllib.utils.tf.loaders import com.intel.analytics.bigdl.dllib.nn.SpatialCrossMapLRN import com.intel.analytics.bigdl.dllib.nn.abstractnn.DataFormat import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.RandomGenerator +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator import com.intel.analytics.bigdl.dllib.utils.tf.Tensorflow.{floatAttr, intAttr, typeAttr} import com.intel.analytics.bigdl.dllib.utils.tf.TensorflowSpecHelper import org.tensorflow.framework.{DataType, NodeDef} diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/tf/loaders/TopKV2Spec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/tf/loaders/TopKV2Spec.scala index 41f2cdd0791..0af496ff9ca 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/tf/loaders/TopKV2Spec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/utils/tf/loaders/TopKV2Spec.scala @@ -16,7 +16,7 @@ package com.intel.analytics.bigdl.dllib.utils.tf.loaders import com.intel.analytics.bigdl.dllib.tensor.Tensor -import com.intel.analytics.bigdl.utils.RandomGenerator +import com.intel.analytics.bigdl.dllib.utils.RandomGenerator import com.intel.analytics.bigdl.dllib.utils.tf.Tensorflow.{booleanAttr, intAttr, typeAttr} import com.intel.analytics.bigdl.dllib.utils.tf.TensorflowSpecHelper import org.tensorflow.framework.{DataType, NodeDef} diff --git a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/visualization/SummarySpec.scala b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/visualization/SummarySpec.scala index 61ea08b0629..295429cfbbe 100644 --- a/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/visualization/SummarySpec.scala +++ b/scala/dllib/src/test/scala/com/intel/analytics/bigdl/dllib/visualization/SummarySpec.scala @@ -21,7 +21,7 @@ import com.intel.analytics.bigdl.dllib.tensor.Tensor import com.intel.analytics.bigdl.dllib.utils.{TestUtils} import Summary._ import com.intel.analytics.bigdl.dllib.visualization.tensorboard.{FileReader, FileWriter} -import com.intel.analytics.bigdl.utils._ +import com.intel.analytics.bigdl.dllib.utils._ import org.scalatest.{BeforeAndAfter, FlatSpec, Matchers} import org.tensorflow.framework diff --git a/scala/pom.xml b/scala/pom.xml index 29b6902eaae..27b575afb30 100644 --- a/scala/pom.xml +++ b/scala/pom.xml @@ -181,7 +181,6 @@ common/spark-version - common/utils dllib From 18b1e55e36c99736a9c136c8115ec5d9a48ebc4c Mon Sep 17 00:00:00 2001 From: dding3 Date: Fri, 20 Aug 2021 19:52:29 -0700 Subject: [PATCH 814/823] resolve python conflicts --- python/dllib/src/bigdl/__init__.py | 4 ---- python/dllib/src/bigdl/dllib/bigdlkeras/backend.py | 4 ---- .../src/bigdl/dllib/bigdlkeras/layers/__init__.py | 4 ---- .../examples/imageframe/inception_validation.py | 13 ------------- .../src/bigdl/dllib/examples/keras/imdb_cnn_lstm.py | 4 ---- .../src/bigdl/dllib/examples/keras/keras_utils.py | 4 ---- .../src/bigdl/dllib/examples/keras/mnist_cnn.py | 4 ---- .../src/bigdl/dllib/feature/dataset/dataset.py | 6 ------ .../src/bigdl/dllib/feature/dataset/transformer.py | 4 ---- .../bigdl/dllib/feature/transform/vision/image.py | 6 ------ python/dllib/src/bigdl/dllib/keras/converter.py | 5 ----- python/dllib/src/bigdl/dllib/utils/tf_utils.py | 5 ----- 12 files changed, 63 deletions(-) diff --git a/python/dllib/src/bigdl/__init__.py b/python/dllib/src/bigdl/__init__.py index 66f0235c748..349eb33267b 100644 --- a/python/dllib/src/bigdl/__init__.py +++ b/python/dllib/src/bigdl/__init__.py @@ -14,9 +14,5 @@ # limitations under the License. # -<<<<<<< HEAD from bigdl.dllib.utils.engine import prepare_env -======= -from bigdl.utils.engine import prepare_env ->>>>>>> upstream_bigdl-2.0 prepare_env() diff --git a/python/dllib/src/bigdl/dllib/bigdlkeras/backend.py b/python/dllib/src/bigdl/dllib/bigdlkeras/backend.py index 827930ec607..4e76273da12 100644 --- a/python/dllib/src/bigdl/dllib/bigdlkeras/backend.py +++ b/python/dllib/src/bigdl/dllib/bigdlkeras/backend.py @@ -15,11 +15,7 @@ # from bigdl.dllib.keras.optimization import * -<<<<<<< HEAD from bigdl.dllib.utils.common import * -======= -from bigdl.utils.common import * ->>>>>>> upstream_bigdl-2.0 class KerasModelWrapper: diff --git a/python/dllib/src/bigdl/dllib/bigdlkeras/layers/__init__.py b/python/dllib/src/bigdl/dllib/bigdlkeras/layers/__init__.py index fce9b8eff2d..2ad55ee4823 100644 --- a/python/dllib/src/bigdl/dllib/bigdlkeras/layers/__init__.py +++ b/python/dllib/src/bigdl/dllib/bigdlkeras/layers/__init__.py @@ -13,11 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # -<<<<<<< HEAD from bigdl.dllib.utils.common import * -======= -from bigdl.utils.common import * ->>>>>>> upstream_bigdl-2.0 init_engine() redire_spark_logs() diff --git a/python/dllib/src/bigdl/dllib/examples/imageframe/inception_validation.py b/python/dllib/src/bigdl/dllib/examples/imageframe/inception_validation.py index c18955d511b..a65a054ec35 100644 --- a/python/dllib/src/bigdl/dllib/examples/imageframe/inception_validation.py +++ b/python/dllib/src/bigdl/dllib/examples/imageframe/inception_validation.py @@ -14,12 +14,7 @@ # limitations under the License. # -<<<<<<< HEAD from bigdl.dllib.utils.common import * -======= -from bigdl.utils.common import * ->>>>>>> upstream_bigdl-2.0 - from bigdl.dllib.feature.transform.vision.image import * from bigdl.dllib.optim.optimizer import * @@ -54,19 +49,11 @@ def run(image_path, model_path, batch_size): transformed = transformer(raw_image_frame) model = Model.loadModel(model_path) result = model.evaluate(transformed, int(batch_size), [Top1Accuracy()]) -<<<<<<< HEAD print("top1 accuray", result[0]) if __name__ == "__main__": if len(sys.argv) != 3: print("parameters needed : ") -======= - print "top1 accuray", result[0] - -if __name__ == "__main__": - if len(sys.argv) != 3: - print "parameters needed : " ->>>>>>> upstream_bigdl-2.0 image_path = sys.argv[1] model_path = sys.argv[2] batch_size = sys.argv[3] diff --git a/python/dllib/src/bigdl/dllib/examples/keras/imdb_cnn_lstm.py b/python/dllib/src/bigdl/dllib/examples/keras/imdb_cnn_lstm.py index a5b53070b85..70ae25c8c94 100644 --- a/python/dllib/src/bigdl/dllib/examples/keras/imdb_cnn_lstm.py +++ b/python/dllib/src/bigdl/dllib/examples/keras/imdb_cnn_lstm.py @@ -72,11 +72,7 @@ def build_keras_model(): hdf5_path = "/tmp/imdb.h5" keras_model.save(hdf5_path) -<<<<<<< HEAD from bigdl.dllib.utils.common import * -======= - from bigdl.utils.common import * ->>>>>>> upstream_bigdl-2.0 from bigdl.dllib.nn.layer import * from bigdl.dllib.optim.optimizer import * from bigdl.dllib.nn.criterion import * diff --git a/python/dllib/src/bigdl/dllib/examples/keras/keras_utils.py b/python/dllib/src/bigdl/dllib/examples/keras/keras_utils.py index 93ea5eb87ef..209d640482d 100644 --- a/python/dllib/src/bigdl/dllib/examples/keras/keras_utils.py +++ b/python/dllib/src/bigdl/dllib/examples/keras/keras_utils.py @@ -14,11 +14,7 @@ # limitations under the License. # -<<<<<<< HEAD from bigdl.dllib.utils.common import * -======= -from bigdl.utils.common import * ->>>>>>> upstream_bigdl-2.0 def save_keras_definition(keras_model, path): diff --git a/python/dllib/src/bigdl/dllib/examples/keras/mnist_cnn.py b/python/dllib/src/bigdl/dllib/examples/keras/mnist_cnn.py index 7462134bf7c..74eabd524ee 100644 --- a/python/dllib/src/bigdl/dllib/examples/keras/mnist_cnn.py +++ b/python/dllib/src/bigdl/dllib/examples/keras/mnist_cnn.py @@ -85,11 +85,7 @@ def build_keras_model(): json_path = "/tmp/lenet.json" save_keras_definition(keras_model, json_path) -<<<<<<< HEAD from bigdl.dllib.utils.common import * -======= - from bigdl.utils.common import * ->>>>>>> upstream_bigdl-2.0 from bigdl.dllib.nn.layer import * from bigdl.dllib.optim.optimizer import * from bigdl.dllib.nn.criterion import * diff --git a/python/dllib/src/bigdl/dllib/feature/dataset/dataset.py b/python/dllib/src/bigdl/dllib/feature/dataset/dataset.py index 56ad9e9dd7a..86b59f45307 100644 --- a/python/dllib/src/bigdl/dllib/feature/dataset/dataset.py +++ b/python/dllib/src/bigdl/dllib/feature/dataset/dataset.py @@ -15,15 +15,9 @@ # import sys -<<<<<<< HEAD from bigdl.dllib.utils.common import JavaValue from bigdl.dllib.utils.common import callBigDlFunc from bigdl.dllib.utils.common import * -======= -from bigdl.utils.common import JavaValue -from bigdl.utils.common import callBigDlFunc -from bigdl.utils.common import * ->>>>>>> upstream_bigdl-2.0 from bigdl.dllib.feature.transform.vision.image import * if sys.version >= '3': diff --git a/python/dllib/src/bigdl/dllib/feature/dataset/transformer.py b/python/dllib/src/bigdl/dllib/feature/dataset/transformer.py index 39ed3bed717..0f6d1f1a561 100644 --- a/python/dllib/src/bigdl/dllib/feature/dataset/transformer.py +++ b/python/dllib/src/bigdl/dllib/feature/dataset/transformer.py @@ -15,11 +15,7 @@ # -<<<<<<< HEAD from bigdl.dllib.utils.common import Sample -======= -from bigdl.utils.common import Sample ->>>>>>> upstream_bigdl-2.0 def normalizer(data, mean, std): diff --git a/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py b/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py index 0b2a9184a86..19cd0fbaedb 100644 --- a/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py +++ b/python/dllib/src/bigdl/dllib/feature/transform/vision/image.py @@ -15,15 +15,9 @@ # import sys -<<<<<<< HEAD from bigdl.dllib.utils.common import JavaValue from bigdl.dllib.utils.common import callBigDlFunc from bigdl.dllib.utils.common import * -======= -from bigdl.utils.common import JavaValue -from bigdl.utils.common import callBigDlFunc -from bigdl.utils.common import * ->>>>>>> upstream_bigdl-2.0 if sys.version >= '3': long = int diff --git a/python/dllib/src/bigdl/dllib/keras/converter.py b/python/dllib/src/bigdl/dllib/keras/converter.py index 4e9389c220e..b7659a2f1f7 100644 --- a/python/dllib/src/bigdl/dllib/keras/converter.py +++ b/python/dllib/src/bigdl/dllib/keras/converter.py @@ -16,13 +16,8 @@ import numpy as np import bigdl.dllib.nn.layer as BLayer -<<<<<<< HEAD import bigdl.dllib.utils.common as BCommon from bigdl.dllib.utils.common import get_activation_by_name -======= -import bigdl.utils.common as BCommon -from bigdl.utils.common import get_activation_by_name ->>>>>>> upstream_bigdl-2.0 from keras.models import model_from_json from keras.models import Sequential, Model, Layer import keras diff --git a/python/dllib/src/bigdl/dllib/utils/tf_utils.py b/python/dllib/src/bigdl/dllib/utils/tf_utils.py index 85176ab7e2b..72b348c2338 100644 --- a/python/dllib/src/bigdl/dllib/utils/tf_utils.py +++ b/python/dllib/src/bigdl/dllib/utils/tf_utils.py @@ -27,13 +27,8 @@ from tensorflow.python.framework import importer from tensorflow.python.platform import gfile from bigdl.dllib.nn.layer import Model -<<<<<<< HEAD from bigdl.dllib.utils.common import JTensor from bigdl.dllib.utils.common import callBigDlFunc -======= -from bigdl.utils.common import JTensor -from bigdl.utils.common import callBigDlFunc ->>>>>>> upstream_bigdl-2.0 import os def get_path(output_name, sess=None): From 8d804d191decbb098c391efdb4f9f6ac87386d98 Mon Sep 17 00:00:00 2001 From: dding3 Date: Fri, 20 Aug 2021 19:54:43 -0700 Subject: [PATCH 815/823] resolve python conflicts II --- .../bigdl/dllib/bigdlkeras/layers/layer.py | 4 --- .../bigdl/dllib/bigdlkeras/optimization.py | 4 --- .../src/bigdl/dllib/models/lenet/lenet5.py | 4 --- .../dllib/models/local_lenet/local_lenet.py | 4 --- .../dllib/models/ml_pipeline/dl_classifier.py | 4 --- .../src/bigdl/dllib/models/rnn/rnnexample.py | 5 ---- .../models/textclassifier/textclassifier.py | 5 ---- python/dllib/src/bigdl/dllib/nn/criterion.py | 11 -------- .../bigdl/dllib/nn/initialization_method.py | 5 +--- python/dllib/src/bigdl/dllib/nn/layer.py | 25 ------------------- python/dllib/src/bigdl/dllib/nn/onnx/layer.py | 4 --- .../dllib/src/bigdl/dllib/optim/optimizer.py | 16 ------------ .../test/bigdl/caffe/test_caffe_layers.py | 4 --- .../dllib/test/bigdl/caffe/test_load_caffe.py | 4 --- python/dllib/test/bigdl/keras/test_backend.py | 5 ---- python/dllib/test/bigdl/test_engine_env.py | 8 ------ python/dllib/test/bigdl/test_pickler.py | 9 ------- .../test/bigdl/test_simple_integration.py | 13 ---------- python/dllib/test/bigdl/test_tensorflow.py | 4 --- python/dllib/test/bigdl/test_utils.py | 4 --- .../dllib/test/bigdl/transform/test_image.py | 4 --- 21 files changed, 1 insertion(+), 145 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py b/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py index 25963287e12..8eb90be9261 100644 --- a/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py +++ b/python/dllib/src/bigdl/dllib/bigdlkeras/layers/layer.py @@ -17,11 +17,7 @@ import sys from bigdl.dllib.nn.layer import Layer, Node, SharedStaticUtils, Container -<<<<<<< HEAD from bigdl.dllib.utils.common import callBigDlFunc, JTensor, JavaValue -======= -from bigdl.utils.common import callBigDlFunc, JTensor, JavaValue ->>>>>>> upstream_bigdl-2.0 if sys.version >= '3': long = int diff --git a/python/dllib/src/bigdl/dllib/bigdlkeras/optimization.py b/python/dllib/src/bigdl/dllib/bigdlkeras/optimization.py index 4e4eb657997..89add68c51b 100644 --- a/python/dllib/src/bigdl/dllib/bigdlkeras/optimization.py +++ b/python/dllib/src/bigdl/dllib/bigdlkeras/optimization.py @@ -18,11 +18,7 @@ import bigdl.dllib.nn.criterion as bcriterion import bigdl.dllib.optim.optimizer as boptimizer -<<<<<<< HEAD from bigdl.dllib.utils.common import to_list -======= -from bigdl.utils.common import to_list ->>>>>>> upstream_bigdl-2.0 from bigdl.dllib.keras.converter import * from keras.objectives import * import six diff --git a/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py b/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py index 66d7f678b8c..dc0c25c73f3 100644 --- a/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py +++ b/python/dllib/src/bigdl/dllib/models/lenet/lenet5.py @@ -20,11 +20,7 @@ from bigdl.dllib.nn.layer import * from bigdl.dllib.nn.criterion import * from bigdl.dllib.optim.optimizer import * -<<<<<<< HEAD from bigdl.dllib.utils.common import * -======= -from bigdl.utils.common import * ->>>>>>> upstream_bigdl-2.0 def build_model(class_num): diff --git a/python/dllib/src/bigdl/dllib/models/local_lenet/local_lenet.py b/python/dllib/src/bigdl/dllib/models/local_lenet/local_lenet.py index 988e991d7cb..250c18ea6d4 100644 --- a/python/dllib/src/bigdl/dllib/models/local_lenet/local_lenet.py +++ b/python/dllib/src/bigdl/dllib/models/local_lenet/local_lenet.py @@ -19,11 +19,7 @@ from bigdl.dllib.models.lenet.lenet5 import build_model from bigdl.dllib.nn.criterion import * from bigdl.dllib.optim.optimizer import * -<<<<<<< HEAD from bigdl.dllib.utils.common import * -======= -from bigdl.utils.common import * ->>>>>>> upstream_bigdl-2.0 def get_mnist(data_type="train", location="/tmp/mnist"): diff --git a/python/dllib/src/bigdl/dllib/models/ml_pipeline/dl_classifier.py b/python/dllib/src/bigdl/dllib/models/ml_pipeline/dl_classifier.py index 83e3b0fb70c..f998e5a8cd7 100644 --- a/python/dllib/src/bigdl/dllib/models/ml_pipeline/dl_classifier.py +++ b/python/dllib/src/bigdl/dllib/models/ml_pipeline/dl_classifier.py @@ -1,10 +1,6 @@ from pyspark.ml.pipeline import Estimator, Model from pyspark.ml.param.shared import * -<<<<<<< HEAD from bigdl.dllib.utils.common import * -======= -from bigdl.utils.common import * ->>>>>>> upstream_bigdl-2.0 import sys if sys.version >= '3': diff --git a/python/dllib/src/bigdl/dllib/models/rnn/rnnexample.py b/python/dllib/src/bigdl/dllib/models/rnn/rnnexample.py index 92b575096fd..f2f19a94c20 100644 --- a/python/dllib/src/bigdl/dllib/models/rnn/rnnexample.py +++ b/python/dllib/src/bigdl/dllib/models/rnn/rnnexample.py @@ -24,13 +24,8 @@ from bigdl.dllib.nn.layer import * from bigdl.dllib.nn.criterion import * from bigdl.dllib.optim.optimizer import * -<<<<<<< HEAD from bigdl.dllib.utils.common import * from bigdl.dllib.utils.common import Sample -======= -from bigdl.utils.common import * -from bigdl.utils.common import Sample ->>>>>>> upstream_bigdl-2.0 def download_data(dest_dir): TINYSHAKESPEARE_URL = 'https://raw.githubusercontent.com/karpathy/char-rnn/master/data/tinyshakespeare/input.txt' # noqa diff --git a/python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py b/python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py index 32c6f665816..578b6bafa58 100644 --- a/python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py +++ b/python/dllib/src/bigdl/dllib/models/textclassifier/textclassifier.py @@ -23,13 +23,8 @@ from bigdl.dllib.nn.layer import * from bigdl.dllib.nn.criterion import * from bigdl.dllib.optim.optimizer import * -<<<<<<< HEAD from bigdl.dllib.utils.common import * from bigdl.dllib.utils.common import Sample -======= -from bigdl.utils.common import * -from bigdl.utils.common import Sample ->>>>>>> upstream_bigdl-2.0 import datetime as dt diff --git a/python/dllib/src/bigdl/dllib/nn/criterion.py b/python/dllib/src/bigdl/dllib/nn/criterion.py index 0c58f6e1d5c..642637f1689 100644 --- a/python/dllib/src/bigdl/dllib/nn/criterion.py +++ b/python/dllib/src/bigdl/dllib/nn/criterion.py @@ -17,15 +17,9 @@ import sys -<<<<<<< HEAD from bigdl.dllib.utils.common import JavaValue from bigdl.dllib.utils.common import callBigDlFunc from bigdl.dllib.utils.common import JTensor -======= -from bigdl.utils.common import JavaValue -from bigdl.utils.common import callBigDlFunc -from bigdl.utils.common import JTensor ->>>>>>> upstream_bigdl-2.0 from bigdl.dllib.nn.layer import Layer import numpy as np @@ -985,13 +979,8 @@ def _test(): import doctest from pyspark import SparkContext from bigdl.dllib.nn import criterion -<<<<<<< HEAD from bigdl.dllib.utils.common import init_engine from bigdl.dllib.utils.common import create_spark_conf -======= - from bigdl.utils.common import init_engine - from bigdl.utils.common import create_spark_conf ->>>>>>> upstream_bigdl-2.0 globs = criterion.__dict__.copy() sc = SparkContext(master="local[4]", appName="test criterion", conf=create_spark_conf()) diff --git a/python/dllib/src/bigdl/dllib/nn/initialization_method.py b/python/dllib/src/bigdl/dllib/nn/initialization_method.py index bf4156527e7..fe4154ec619 100644 --- a/python/dllib/src/bigdl/dllib/nn/initialization_method.py +++ b/python/dllib/src/bigdl/dllib/nn/initialization_method.py @@ -16,11 +16,8 @@ import sys -<<<<<<< HEAD from bigdl.dllib.utils.common import JavaValue -======= -from bigdl.utils.common import JavaValue ->>>>>>> upstream_bigdl-2.0 + if sys.version >= '3': long = int diff --git a/python/dllib/src/bigdl/dllib/nn/layer.py b/python/dllib/src/bigdl/dllib/nn/layer.py index 93a04f0ae68..6e5d200d43f 100644 --- a/python/dllib/src/bigdl/dllib/nn/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/layer.py @@ -21,7 +21,6 @@ import numpy as np import six -<<<<<<< HEAD from bigdl.dllib.utils.common import JTensor from bigdl.dllib.utils.common import JavaValue from bigdl.dllib.utils.common import callBigDlFunc @@ -30,16 +29,6 @@ from bigdl.dllib.utils.common import to_list from bigdl.dllib.utils.common import INTMAX, INTMIN, DOUBLEMAX from bigdl.dllib.utils.common import get_activation_by_name -======= -from bigdl.utils.common import JTensor -from bigdl.utils.common import JavaValue -from bigdl.utils.common import callBigDlFunc -from bigdl.utils.common import callJavaFunc -from bigdl.utils.common import get_spark_context -from bigdl.utils.common import to_list -from bigdl.utils.common import INTMAX, INTMIN, DOUBLEMAX -from bigdl.utils.common import get_activation_by_name ->>>>>>> upstream_bigdl-2.0 from bigdl.dllib.optim.optimizer import L1Regularizer, L2Regularizer, L1L2Regularizer from py4j.java_gateway import JavaObject from pyspark.rdd import RDD @@ -756,11 +745,7 @@ def __init__(self, bigdl_type, "createModelPreprocessor", inputs, outputs) self.bigdl_type = bigdl_type else: -<<<<<<< HEAD from bigdl.dllib.utils.tf_utils import convert -======= - from bigdl.utils.tf_utils import convert ->>>>>>> upstream_bigdl-2.0 model = convert(to_list(inputs), to_list(outputs), byte_order, bigdl_type) super(Model, self).__init__(model.value, bigdl_type) @@ -878,13 +863,8 @@ def load_tensorflow(path, inputs, outputs, byte_order = "little_endian", @staticmethod def train(output, data, label, opt_method, criterion, batch_size, end_when, session=None, bigdl_type="float"): -<<<<<<< HEAD from bigdl.dllib.utils.tf_utils import get_path from bigdl.dllib.utils.common import Sample -======= - from bigdl.utils.tf_utils import get_path - from bigdl.utils.common import Sample ->>>>>>> upstream_bigdl-2.0 output_name = output.name.split(":")[0] path = get_path(output_name, session) sc = get_spark_context() @@ -5804,13 +5784,8 @@ def _test(): import doctest from pyspark import SparkContext from bigdl.dllib.nn import layer -<<<<<<< HEAD from bigdl.dllib.utils.common import init_engine from bigdl.dllib.utils.common import create_spark_conf -======= - from bigdl.utils.common import init_engine - from bigdl.utils.common import create_spark_conf ->>>>>>> upstream_bigdl-2.0 globs = layer.__dict__.copy() sc = SparkContext(master="local[4]", appName="test layer", conf=create_spark_conf()) diff --git a/python/dllib/src/bigdl/dllib/nn/onnx/layer.py b/python/dllib/src/bigdl/dllib/nn/onnx/layer.py index 8ec8f83a516..a907f5501c9 100644 --- a/python/dllib/src/bigdl/dllib/nn/onnx/layer.py +++ b/python/dllib/src/bigdl/dllib/nn/onnx/layer.py @@ -17,11 +17,7 @@ import sys import numpy as np from bigdl.dllib.nn.layer import Layer -<<<<<<< HEAD from bigdl.dllib.utils.common import JTensor -======= -from bigdl.utils.common import JTensor ->>>>>>> upstream_bigdl-2.0 if sys.version >= '3': long = int diff --git a/python/dllib/src/bigdl/dllib/optim/optimizer.py b/python/dllib/src/bigdl/dllib/optim/optimizer.py index fc26da7e268..af0ddfa3bb6 100644 --- a/python/dllib/src/bigdl/dllib/optim/optimizer.py +++ b/python/dllib/src/bigdl/dllib/optim/optimizer.py @@ -23,7 +23,6 @@ from py4j.java_gateway import JavaObject from pyspark.rdd import RDD -<<<<<<< HEAD from bigdl.dllib.utils.common import DOUBLEMAX from bigdl.dllib.utils.common import JTensor from bigdl.dllib.utils.common import JavaValue @@ -32,16 +31,6 @@ from bigdl.dllib.utils.common import get_node_and_core_number from bigdl.dllib.utils.common import init_engine from bigdl.dllib.utils.common import to_list -======= -from bigdl.utils.common import DOUBLEMAX -from bigdl.utils.common import JTensor -from bigdl.utils.common import JavaValue -from bigdl.utils.common import callBigDlFunc -from bigdl.utils.common import callJavaFunc -from bigdl.utils.common import get_node_and_core_number -from bigdl.utils.common import init_engine -from bigdl.utils.common import to_list ->>>>>>> upstream_bigdl-2.0 from bigdl.dllib.feature.dataset.dataset import * import warnings @@ -1201,13 +1190,8 @@ def _test(): import doctest from pyspark import SparkContext from bigdl.dllib.optim import optimizer -<<<<<<< HEAD from bigdl.dllib.utils.common import init_engine from bigdl.dllib.utils.common import create_spark_conf -======= - from bigdl.utils.common import init_engine - from bigdl.utils.common import create_spark_conf ->>>>>>> upstream_bigdl-2.0 globs = optimizer.__dict__.copy() sc = SparkContext(master="local[4]", appName="test optimizer", conf=create_spark_conf()) diff --git a/python/dllib/test/bigdl/caffe/test_caffe_layers.py b/python/dllib/test/bigdl/caffe/test_caffe_layers.py index b5f1604a873..537dbc03e41 100644 --- a/python/dllib/test/bigdl/caffe/test_caffe_layers.py +++ b/python/dllib/test/bigdl/caffe/test_caffe_layers.py @@ -16,11 +16,7 @@ from bigdl.dllib.nn.layer import * from bigdl.dllib.optim.optimizer import * -<<<<<<< HEAD from bigdl.dllib.utils.common import * -======= -from bigdl.utils.common import * ->>>>>>> upstream_bigdl-2.0 import numpy as np import pytest import tempfile diff --git a/python/dllib/test/bigdl/caffe/test_load_caffe.py b/python/dllib/test/bigdl/caffe/test_load_caffe.py index 67555927c5a..c5c3760b88e 100644 --- a/python/dllib/test/bigdl/caffe/test_load_caffe.py +++ b/python/dllib/test/bigdl/caffe/test_load_caffe.py @@ -16,11 +16,7 @@ from bigdl.dllib.nn.layer import * from bigdl.dllib.optim.optimizer import * -<<<<<<< HEAD from bigdl.dllib.utils.common import * -======= -from bigdl.utils.common import * ->>>>>>> upstream_bigdl-2.0 import numpy as np import pytest from numpy.testing import assert_allclose diff --git a/python/dllib/test/bigdl/keras/test_backend.py b/python/dllib/test/bigdl/keras/test_backend.py index 55cba2a0f96..7df21569750 100644 --- a/python/dllib/test/bigdl/keras/test_backend.py +++ b/python/dllib/test/bigdl/keras/test_backend.py @@ -73,13 +73,8 @@ def test_lenet_distributed_ndarray(self): def test_lenet_distributed_rdd(self): kmodel, X_train, y_train = TestModels.kmodel_seq_lenet_mnist() sc = get_spark_context() -<<<<<<< HEAD from bigdl.dllib.utils.common import Sample from bigdl.dllib.utils.common import to_sample_rdd -======= - from bigdl.utils.common import Sample - from bigdl.utils.common import to_sample_rdd ->>>>>>> upstream_bigdl-2.0 training_rdd = to_sample_rdd(X_train, y_train) self.modelTest(X_train, kmodel, dump_weights=True) diff --git a/python/dllib/test/bigdl/test_engine_env.py b/python/dllib/test/bigdl/test_engine_env.py index 43edb2f7b24..ee0a889d524 100644 --- a/python/dllib/test/bigdl/test_engine_env.py +++ b/python/dllib/test/bigdl/test_engine_env.py @@ -16,11 +16,7 @@ import pytest import os -<<<<<<< HEAD from bigdl.dllib.utils.common import * -======= -from bigdl.utils.common import * ->>>>>>> upstream_bigdl-2.0 class TestEngineEnv(): @@ -42,11 +38,7 @@ def test___prepare_bigdl_env(self): # adding jar path message, just do prepare_env()' again # to see if the log is correct and the environment variables should not vary. -<<<<<<< HEAD from bigdl.dllib.utils.engine import prepare_env -======= - from bigdl.utils.engine import prepare_env ->>>>>>> upstream_bigdl-2.0 bigdl_jars_env_1 = os.environ.get("BIGDL_JARS", None) spark_class_path_1 = os.environ.get("SPARK_CLASSPATH", None) diff --git a/python/dllib/test/bigdl/test_pickler.py b/python/dllib/test/bigdl/test_pickler.py index acd4818a535..242fbdfd072 100644 --- a/python/dllib/test/bigdl/test_pickler.py +++ b/python/dllib/test/bigdl/test_pickler.py @@ -18,24 +18,15 @@ from bigdl.dllib.nn.initialization_method import * from bigdl.dllib.nn.criterion import * from bigdl.dllib.optim.optimizer import * -<<<<<<< HEAD from bigdl.dllib.utils.common import * from bigdl.dllib.utils.common import _py2java -======= -from bigdl.utils.common import * -from bigdl.utils.common import _py2java ->>>>>>> upstream_bigdl-2.0 from bigdl.dllib.nn.initialization_method import * from bigdl.dllib.feature.dataset import movielens import numpy as np import tempfile import pytest from numpy.testing import assert_allclose, assert_array_equal -<<<<<<< HEAD from bigdl.dllib.utils.engine import compare_version -======= -from bigdl.utils.engine import compare_version ->>>>>>> upstream_bigdl-2.0 np.random.seed(1337) # for reproducibility diff --git a/python/dllib/test/bigdl/test_simple_integration.py b/python/dllib/test/bigdl/test_simple_integration.py index 77e1d7642f3..ff0ce17abb1 100644 --- a/python/dllib/test/bigdl/test_simple_integration.py +++ b/python/dllib/test/bigdl/test_simple_integration.py @@ -18,24 +18,15 @@ from bigdl.dllib.nn.initialization_method import * from bigdl.dllib.nn.criterion import * from bigdl.dllib.optim.optimizer import * -<<<<<<< HEAD from bigdl.dllib.utils.common import * from bigdl.dllib.utils.common import _py2java -======= -from bigdl.utils.common import * -from bigdl.utils.common import _py2java ->>>>>>> upstream_bigdl-2.0 from bigdl.dllib.nn.initialization_method import * from bigdl.dllib.feature.dataset import movielens import numpy as np import tempfile import pytest from numpy.testing import assert_allclose, assert_array_equal -<<<<<<< HEAD from bigdl.dllib.utils.engine import compare_version -======= -from bigdl.utils.engine import compare_version ->>>>>>> upstream_bigdl-2.0 from bigdl.dllib.feature.transform.vision.image import * from bigdl.dllib.models.utils.model_broadcast import broadcast_model from bigdl.dllib.feature.dataset.dataset import * @@ -253,11 +244,7 @@ def test_graph_preprocessor(self): np.array([1.0])) def test_load_zip_conf(self): -<<<<<<< HEAD from bigdl.dllib.utils.common import get_bigdl_conf -======= - from bigdl.utils.common import get_bigdl_conf ->>>>>>> upstream_bigdl-2.0 import sys sys_path_back = sys.path sys.path = [path for path in sys.path if "spark-bigdl.conf" not in path] diff --git a/python/dllib/test/bigdl/test_tensorflow.py b/python/dllib/test/bigdl/test_tensorflow.py index 082ce48b57a..ba77e399b00 100644 --- a/python/dllib/test/bigdl/test_tensorflow.py +++ b/python/dllib/test/bigdl/test_tensorflow.py @@ -16,11 +16,7 @@ from bigdl.dllib.nn.layer import * from bigdl.dllib.optim.optimizer import * -<<<<<<< HEAD from bigdl.dllib.utils.common import * -======= -from bigdl.utils.common import * ->>>>>>> upstream_bigdl-2.0 import numpy as np import unittest import shutil diff --git a/python/dllib/test/bigdl/test_utils.py b/python/dllib/test/bigdl/test_utils.py index 943b58b46c0..687277649eb 100644 --- a/python/dllib/test/bigdl/test_utils.py +++ b/python/dllib/test/bigdl/test_utils.py @@ -21,11 +21,7 @@ from keras.layers.convolutional import * from keras.layers import Dense, Dropout, Input from keras.optimizers import RMSprop -<<<<<<< HEAD from bigdl.dllib.utils.common import * -======= -from bigdl.utils.common import * ->>>>>>> upstream_bigdl-2.0 from bigdl.dllib.keras.converter import * from bigdl.dllib.keras.converter import WeightLoader, WeightsConverter import numpy as np diff --git a/python/dllib/test/bigdl/transform/test_image.py b/python/dllib/test/bigdl/transform/test_image.py index cdcd9f7efe7..9c178cff991 100644 --- a/python/dllib/test/bigdl/transform/test_image.py +++ b/python/dllib/test/bigdl/transform/test_image.py @@ -16,11 +16,7 @@ import pytest import os -<<<<<<< HEAD from bigdl.dllib.utils.common import * -======= -from bigdl.utils.common import * ->>>>>>> upstream_bigdl-2.0 from bigdl.dllib.feature.transform.vision.image import * import tempfile From 9e2f2c966e4bd5994d5bc86a246766d5d3f7b568 Mon Sep 17 00:00:00 2001 From: Chirs_Leo <597714572@qq.com> Date: Mon, 23 Aug 2021 09:50:30 +0800 Subject: [PATCH 816/823] Nano unit test (#4503) * nano datasets * lisence * restructure * fix import * rm transform seg * ut * jenkins * rm debug info * lincense * rm unused codes * shell print * license * codestyle check * diff * type check * torch vision whl --- python/nano/setup.py | 11 +- python/nano/src/bigdl/nano/common/common.py | 8 +- .../nano/src/bigdl/nano/pytorch/__init__.py | 2 +- .../nano/pytorch/accelerators/__init__.py | 5 +- .../src/bigdl/nano/pytorch/trainer/Trainer.py | 7 +- .../nano/pytorch/vision/datasets/__init__.py | 2 +- python/nano/test/__init__.py | 15 +++ python/nano/test/_train_imagefolder.py | 68 ++++++++++++ python/nano/test/_train_torch_lightning.py | 102 ++++++++++++++++++ python/nano/test/mypy.ini | 11 ++ .../train_image_folder_jpg/cats/cat.0.jpg | Bin 0 -> 12414 bytes .../train_image_folder_jpg/cats/cat.1.jpg | Bin 0 -> 16880 bytes .../train_image_folder_jpg/cats/cat.10.jpg | Bin 0 -> 34315 bytes .../train_image_folder_jpg/cats/cat.100.jpg | Bin 0 -> 28377 bytes .../train_image_folder_jpg/cats/cat.1000.jpg | Bin 0 -> 5944 bytes .../train_image_folder_jpg/dogs/dog.0.jpg | Bin 0 -> 32053 bytes .../train_image_folder_jpg/dogs/dog.1.jpg | Bin 0 -> 25034 bytes .../train_image_folder_jpg/dogs/dog.10.jpg | Bin 0 -> 12166 bytes .../train_image_folder_jpg/dogs/dog.100.jpg | Bin 0 -> 24171 bytes .../train_image_folder_jpg/dogs/dog.1000.jpg | Bin 0 -> 23241 bytes .../train_image_folder_png/cats/cat.0.png | Bin 0 -> 124141 bytes .../train_image_folder_png/cats/cat.1.png | Bin 0 -> 144593 bytes .../train_image_folder_png/cats/cat.10.png | Bin 0 -> 329867 bytes .../train_image_folder_png/cats/cat.100.png | Bin 0 -> 279929 bytes .../train_image_folder_png/cats/cat.1000.png | Bin 0 -> 42144 bytes .../train_image_folder_png/dogs/dog.0.png | Bin 0 -> 280890 bytes .../train_image_folder_png/dogs/dog.1.png | Bin 0 -> 217431 bytes .../train_image_folder_png/dogs/dog.10.png | Bin 0 -> 112219 bytes .../train_image_folder_png/dogs/dog.100.png | Bin 0 -> 260259 bytes .../train_image_folder_png/dogs/dog.1000.png | Bin 0 -> 203464 bytes python/nano/test/run-nano-codestyle-test.sh | 8 ++ python/nano/test/run-nano-test-ipex.sh | 26 +++++ python/nano/test/run-nano-test.sh | 26 +++++ python/nano/test/run-nano-type-test.sh | 8 ++ python/nano/test/test_imagefolder.py | 65 +++++++++++ python/nano/test/test_models_vision.py | 76 +++++++++++++ python/nano/test/test_models_vision_ipex.py | 84 +++++++++++++++ python/nano/test/test_trainer_ipex.py | 83 ++++++++++++++ python/nano/test/tox.ini | 3 + 39 files changed, 599 insertions(+), 11 deletions(-) create mode 100644 python/nano/test/__init__.py create mode 100644 python/nano/test/_train_imagefolder.py create mode 100644 python/nano/test/_train_torch_lightning.py create mode 100644 python/nano/test/mypy.ini create mode 100644 python/nano/test/resources/train_image_folder_jpg/cats/cat.0.jpg create mode 100644 python/nano/test/resources/train_image_folder_jpg/cats/cat.1.jpg create mode 100644 python/nano/test/resources/train_image_folder_jpg/cats/cat.10.jpg create mode 100644 python/nano/test/resources/train_image_folder_jpg/cats/cat.100.jpg create mode 100644 python/nano/test/resources/train_image_folder_jpg/cats/cat.1000.jpg create mode 100644 python/nano/test/resources/train_image_folder_jpg/dogs/dog.0.jpg create mode 100644 python/nano/test/resources/train_image_folder_jpg/dogs/dog.1.jpg create mode 100644 python/nano/test/resources/train_image_folder_jpg/dogs/dog.10.jpg create mode 100644 python/nano/test/resources/train_image_folder_jpg/dogs/dog.100.jpg create mode 100644 python/nano/test/resources/train_image_folder_jpg/dogs/dog.1000.jpg create mode 100644 python/nano/test/resources/train_image_folder_png/cats/cat.0.png create mode 100644 python/nano/test/resources/train_image_folder_png/cats/cat.1.png create mode 100644 python/nano/test/resources/train_image_folder_png/cats/cat.10.png create mode 100644 python/nano/test/resources/train_image_folder_png/cats/cat.100.png create mode 100644 python/nano/test/resources/train_image_folder_png/cats/cat.1000.png create mode 100644 python/nano/test/resources/train_image_folder_png/dogs/dog.0.png create mode 100644 python/nano/test/resources/train_image_folder_png/dogs/dog.1.png create mode 100644 python/nano/test/resources/train_image_folder_png/dogs/dog.10.png create mode 100644 python/nano/test/resources/train_image_folder_png/dogs/dog.100.png create mode 100644 python/nano/test/resources/train_image_folder_png/dogs/dog.1000.png create mode 100644 python/nano/test/run-nano-codestyle-test.sh create mode 100644 python/nano/test/run-nano-test-ipex.sh create mode 100644 python/nano/test/run-nano-test.sh create mode 100644 python/nano/test/run-nano-type-test.sh create mode 100644 python/nano/test/test_imagefolder.py create mode 100644 python/nano/test/test_models_vision.py create mode 100644 python/nano/test/test_models_vision_ipex.py create mode 100644 python/nano/test/test_trainer_ipex.py create mode 100644 python/nano/test/tox.ini diff --git a/python/nano/setup.py b/python/nano/setup.py index 236494c0de2..a78642e6711 100644 --- a/python/nano/setup.py +++ b/python/nano/setup.py @@ -97,12 +97,21 @@ def setup_package(): ipex_whl_name = f"torch_ipex-{ipex_version}-cp{py_version.major}{py_version.minor}" \ f"-cp{py_version.major}{py_version.minor}m-linux_x86_64.whl" ipex_link = ipex_links[ipex_whl_name] + + torchvision_version = "0.9.0" + torchvision_links = parse_find_index_page("https://download.pytorch.org/whl/torch_stable.html") + torchvision_whl_name = f"cpu/torchvision-{torchvision_version}%2Bcpu-cp{py_version.major}{py_version.minor}" \ + f"-cp{py_version.major}{py_version.minor}m-linux_x86_64.whl" + torchvision_link = "https://download.pytorch.org/whl/" + torchvision_links[torchvision_whl_name] + install_requires_list = ["pytorch_lightning", "opencv-python-headless", "PyTurboJPEG", "opencv-transforms", "intel-openmp", - f"torch_ipex @ {ipex_link}"] + "torchvision", + f"torch_ipex @ {ipex_link}", + f"torchvision @ {torchvision_link}"] download_libs() diff --git a/python/nano/src/bigdl/nano/common/common.py b/python/nano/src/bigdl/nano/common/common.py index 9bc13c99b29..96a317034ec 100644 --- a/python/nano/src/bigdl/nano/common/common.py +++ b/python/nano/src/bigdl/nano/common/common.py @@ -66,7 +66,8 @@ def _find_library(library_name: str, priority_dir: Union[str, None] = None) -> U res = subprocess.check_output("find " + priority_dir + " -name " + library_name, shell=True, stderr=subprocess.DEVNULL).splitlines() except Exception: - warnings.warn("Some errors occurred while trying to find " + library_name) + warnings.warn( + "Some errors occurred while trying to find " + library_name) if len(res) > 0: return res[0].decode("utf-8") @@ -74,12 +75,13 @@ def _find_library(library_name: str, priority_dir: Union[str, None] = None) -> U res = subprocess.check_output("find / -name " + library_name, shell=True, stderr=subprocess.DEVNULL).splitlines() except Exception: - warnings.warn("Some errors occurred while trying to find " + library_name) + warnings.warn( + "Some errors occurred while trying to find " + library_name) return res[0].decode("utf-8") if len(res) > 0 else None def init_nano(use_jemalloc: bool = True, use_openmp: bool = True, - print_environment: bool = False) -> None: + print_environment: bool = False) -> None: """ Configure necessary environment variables for jemalloc and openmp libraries. :param use_jemalloc: If this is set to True, then use jemalloc library. Otherwise disable diff --git a/python/nano/src/bigdl/nano/pytorch/__init__.py b/python/nano/src/bigdl/nano/pytorch/__init__.py index 1b268b4efd1..2151a805423 100644 --- a/python/nano/src/bigdl/nano/pytorch/__init__.py +++ b/python/nano/src/bigdl/nano/pytorch/__init__.py @@ -12,4 +12,4 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# \ No newline at end of file +# diff --git a/python/nano/src/bigdl/nano/pytorch/accelerators/__init__.py b/python/nano/src/bigdl/nano/pytorch/accelerators/__init__.py index 409e3746840..c560e371d49 100644 --- a/python/nano/src/bigdl/nano/pytorch/accelerators/__init__.py +++ b/python/nano/src/bigdl/nano/pytorch/accelerators/__init__.py @@ -20,7 +20,6 @@ # To replace torch.save in ipex, you need to import and exec their __init__.py first. from intel_pytorch_extension.ops.save import * - # And then you can replace torch.save with your customized function. # Note that you need to temporarily store original torch.save, # because it will be modified in ipex.ops.save. @@ -33,8 +32,9 @@ torch_save = torch.save + def nano_save(obj, f, pickle_module=pickle, pickle_protocol=DEFAULT_PROTOCOL, - _use_new_zipfile_serialization=False): + _use_new_zipfile_serialization=False): def to_cpu(obj, iter_type): # Extend original `save` defined in ipex.ops.save # to support converting a list of xpu tensor to cpu in torch.save @@ -60,4 +60,5 @@ def to_cpu(obj, iter_type): obj_copy = obj return torch_save(obj_copy, f, pickle_module, pickle_protocol, _use_new_zipfile_serialization) + torch.save = nano_save diff --git a/python/nano/src/bigdl/nano/pytorch/trainer/Trainer.py b/python/nano/src/bigdl/nano/pytorch/trainer/Trainer.py index 89dcc953f17..f9fb03b07a9 100644 --- a/python/nano/src/bigdl/nano/pytorch/trainer/Trainer.py +++ b/python/nano/src/bigdl/nano/pytorch/trainer/Trainer.py @@ -36,7 +36,6 @@ def __init__(self, num_processes: int = 1, *args: Any, **kwargs: Any) -> None: """ A pytorch lightning trainer that uses bigdl-nano optimization. - :param num_processes: number of processes in distributed training. default: 4. :param use_ipex: whether we use ipex as accelerator for trainer. default: True. :param cpu_for_each_process: A list of length `num_processes`, each containing a list of @@ -95,7 +94,8 @@ def __init__(self, num_processes: int = 1, # which leads to an unacceptably low performance. # So we import when we need. from bigdl.nano.pytorch.plugins.ray_distributed import RayPlugin - plugin = RayPlugin(num_workers=num_processes, use_ipex=use_ipex) # type: ignore + plugin = RayPlugin(num_workers=num_processes, + use_ipex=use_ipex) # type: ignore if use_ipex: from bigdl.nano.pytorch.accelerators.ipex_accelerator import IPEXAccelerator @@ -104,4 +104,5 @@ def __init__(self, num_processes: int = 1, else: accelerator = None - super().__init__(accelerator=accelerator, plugins=[plugin], *args, **kwargs) + super().__init__(accelerator=accelerator, + plugins=[plugin], *args, **kwargs) diff --git a/python/nano/src/bigdl/nano/pytorch/vision/datasets/__init__.py b/python/nano/src/bigdl/nano/pytorch/vision/datasets/__init__.py index 35bb244f837..6bd70376270 100644 --- a/python/nano/src/bigdl/nano/pytorch/vision/datasets/__init__.py +++ b/python/nano/src/bigdl/nano/pytorch/vision/datasets/__init__.py @@ -14,4 +14,4 @@ # limitations under the License. # -from .datasets import ImageFolder, SegmentationImageFolder \ No newline at end of file +from .datasets import ImageFolder, SegmentationImageFolder diff --git a/python/nano/test/__init__.py b/python/nano/test/__init__.py new file mode 100644 index 00000000000..2151a805423 --- /dev/null +++ b/python/nano/test/__init__.py @@ -0,0 +1,15 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/python/nano/test/_train_imagefolder.py b/python/nano/test/_train_imagefolder.py new file mode 100644 index 00000000000..c5e65d374e6 --- /dev/null +++ b/python/nano/test/_train_imagefolder.py @@ -0,0 +1,68 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +import os +import torch +from bigdl.nano.pytorch.vision.datasets import ImageFolder +from bigdl.nano.pytorch.vision.transforms import transforms +import pytorch_lightning as pl +from torch.utils.data import DataLoader +from bigdl.nano.pytorch.vision.models import ImageClassifier + + +class Net(ImageClassifier): + # Common case: fully-connected top layer + + def __init__(self, backbone): + super().__init__(backbone=backbone, num_classes=2) + + def configure_optimizers(self): + optimizer = torch.optim.Adam(self.parameters(), lr=0.002, amsgrad=True) + return optimizer + + +def create_data_loader(root_dir, batch_size): + dir_path = os.path.realpath(root_dir) + data_transform = transforms.Compose([ + transforms.Resize(256), + transforms.ColorJitter(), + transforms.RandomCrop(224), + transforms.RandomHorizontalFlip(), + transforms.Resize(128), + transforms.ToTensor() + ]) + + catdogs = ImageFolder(dir_path, data_transform) + val_num = len(catdogs) // 10 + train_num = len(catdogs) - val_num + train_set, val_set = torch.utils.data.random_split( + catdogs, [train_num, val_num]) + + train_loader = DataLoader( + train_set, batch_size=batch_size, shuffle=True, num_workers=0) + val_loader = DataLoader(val_set, batch_size=batch_size, + shuffle=False, num_workers=0) + return train_loader, val_loader + + +def train_torch_lightning(model, root_dir, batch_size): + train_loader, val_loader = create_data_loader(root_dir, batch_size) + net = Net(model) + trainer = pl.Trainer(max_epochs=10) + trainer.fit(net, train_loader) + trainer.test(net, val_loader) + print('pass') diff --git a/python/nano/test/_train_torch_lightning.py b/python/nano/test/_train_torch_lightning.py new file mode 100644 index 00000000000..44e483da14c --- /dev/null +++ b/python/nano/test/_train_torch_lightning.py @@ -0,0 +1,102 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +import torch +from copy import deepcopy +import pytorch_lightning as pl + +import torchvision.transforms as transforms +from torchvision.datasets import CIFAR10 +from torch.utils.data import DataLoader + +from bigdl.nano.pytorch.vision.models import ImageClassifier + +num_classes = 10 + +data_transform = transforms.Compose([ + transforms.Resize(256), + transforms.ColorJitter(), + transforms.RandomCrop(224), + transforms.RandomHorizontalFlip(), + transforms.Resize(128), + transforms.ToTensor() +]) + + +class Net(ImageClassifier): + # Common case: fully-connected top layer + + def __init__(self, backbone): + super().__init__(backbone=backbone, num_classes=num_classes) + + def configure_optimizers(self): + optimizer = torch.optim.Adam(self.parameters(), lr=0.002, amsgrad=True) + return optimizer + + +def create_data_loader(dir, batch_size, num_workers, transform, subset=50): + train_set = CIFAR10(root=dir, train=True, + download=True, transform=transform) + # `subset` is the number of subsets. The larger the number, the smaller the training set. + mask = list(range(0, len(train_set), subset)) + train_subset = torch.utils.data.Subset(train_set, mask) + data_loader = DataLoader(train_subset, batch_size=batch_size, shuffle=True, + num_workers=num_workers) + return data_loader + + +def train_with_linear_top_layer(model_without_top, batch_size, num_workers, data_dir, + accelerator=None, use_orca_lite_trainer=False): + model = Net(model_without_top) + train_torch_lightning(model, batch_size, num_workers, + data_dir, accelerator=accelerator, + use_orca_lite_trainer=use_orca_lite_trainer) + + +def train_torch_lightning(model, batch_size, num_workers, data_dir, accelerator=None, + use_orca_lite_trainer=False): + orig_parameters = deepcopy(list(model.named_parameters())) + + train_loader = create_data_loader( + data_dir, batch_size, num_workers, data_transform) + + if use_orca_lite_trainer: + from bigdl.nano.pytorch.trainer import Trainer + + trainer = Trainer(max_epochs=1) + else: + trainer = pl.Trainer(max_epochs=1, accelerator=accelerator) + + trainer.fit(model, train_loader) + + trained_parameters = list(model.named_parameters()) + + # Check if the training and the freeze operation is successful + for i in range(len(orig_parameters)): + name1, para1 = orig_parameters[i] + name2, para2 = trained_parameters[i] + if name1 == "model.1.bias" or name1 == "model.1.weight" or \ + name1 == "new_classifier.1.bias" or name1 == "new_classifier.1.weight": + # Top layer is trained + if torch.all(torch.eq(para1, para2)): + raise Exception("Parameter " + name1 + + " remains the same after training.") + else: + # Frozen parameters should not change + if not torch.all(torch.eq(para1, para2)): + raise Exception(name1 + " freeze failed.") + print("pass") diff --git a/python/nano/test/mypy.ini b/python/nano/test/mypy.ini new file mode 100644 index 00000000000..62854974ec3 --- /dev/null +++ b/python/nano/test/mypy.ini @@ -0,0 +1,11 @@ +[mypy] +ignore_missing_imports = True + +[mypy-torchvision.*] +ignore_missing_imports = True + +[mypy-intel_pytorch_extension.*] +ignore_missing_imports = True + +[mypy-ray.*] +ignore_missing_imports = True diff --git a/python/nano/test/resources/train_image_folder_jpg/cats/cat.0.jpg b/python/nano/test/resources/train_image_folder_jpg/cats/cat.0.jpg new file mode 100644 index 0000000000000000000000000000000000000000..7c99c277bab71024046f5c8c640a624e79dcb9d0 GIT binary patch literal 12414 zcmbWdd00|=)CP*;oSFlenwoQhLk{3<;Q%QL3J4}?$DvSgNF`0}*l{jN%>hku$PiFe zET_qq3Z?qB!b&9gVK*&A5Dwbr}dHT+!oxdKo_ zqx?_+DJcLzO7Z~wd;;(WD9Ot!$jd1yC@AbwQre}et){A?qN)$pQr9-wXJ%@=&)CS^ z(&>OX1O_oOw)M1wxwsuXc+kwo$Isi{*U96c+pjN?QrfjkRYg@#O-;|u!q~#?|MT{9 z1fZ@Ul_vEGC}j+gR+j>*OZ^-L?34UYS*ico0RQJEB@L93m6KOcRN5uEpid1TEd>Nh z%K&9%Wn?5*XGuN>$f(O|7}aedLnh;DRg&YYbM;|{yiHW7s=%-TCPN!#N=H(X@o;zQ} zsHkLCRbQ&9WwDz%TwY5nzpcBcx9@uYz~IoGyQ5>{6O&W-M6=?#`Ns>3OV3wdti4>{ zc(u9p{=>&lpa1^y&)09iAJ0G;(&h#j!KzP(lUiTH+%m}wEvOp|D9mD|Gy;rzXbcg zdM_N|Y^AMFiE_F`~zsh=c{PXE^eEU4B zFm=zLj!_}KF z=DS~tQ>x0*t1;!{ez7hdPOJ#LLCX8iWgwuBc`p%s@DVtQjas`cWp%bJJHr z=-4Q}7tq9JSZL#rItIPr02~qo=*1!JWI^~@09RsXhLQtl^e(7ZDU4{O(!@xED6mIr zbwJ}_%isk3eV+iRBiN2-FNTqi%g~YTrLi1bZXZ6K5#*3*Fq9=Qn4N`3Vth4#Xcd@lP^{F-Y) z2rYOJo-U9Is8%<=Ol~Ufcu3hX9*hkA;1{kU{4)p{@!L^XyAzw6#69$GO-qLznanwA z1cIBr&IO1IJ?>vG?1*c&duIKnI0w^TNJ;2=iOMix%=>HK-7!gS-=*-|5P%CYEzk`) zoe7TUQj+`I`mf3mFaiMB1B&><9qH5wZ%Af%*&ZAG(eCd*taQh5zemZ53Ok>KzPrF& z6?7Wd7h=-iUt`;jSig4aNtdx|hM@JhOh}kn)@}pAK@tXO2Ign?G9~N&YwdsSMqExJ zGdq$u%c}EZ=9bsd)~nl>(OsisrMA%HuRHM}X?A@Hkxg||-np(bk!?2%h*QGIXF^ba z@!CdTcu$3zr#v^Kr!r-D8iQvPbfjLJS~sL+5xSEmZPVMUUg0mn5;JBmT6KVWapW>< z5VLm_)XQYoBVX6dlHZ<$jG2ha46{Lt{_%gs5I|L$Mcy(VjtR{iPk)>3bjRa7Ou~_$i^A03(JdsyC;A>oP1EAV9t2~2AHL{ zoLuGM9E;Cs46YZe*0z38{)kqPyT-&c8;!ev;$>hK=9ngWy85R>V7-(Jc+1dd^AS&S z!ai8?yN5!fN%*9=+7I*)*miTm{!8X*2IoX=_FckbRdX}*&558bzRchRJXVkKy14VA zIj%>GVbNx2!Db*pI6uTL?;wZ@z>eR$&J&$5QaQ0DtO=#h_nyeQ5H{Z4;ZvbKZAt<+ zRF$&S5@qef@c)VW<+T*@puNu+!@dCL_;6tmJbp5SG}c8Ft8UG zc%8M%m^FxkK;BTnjJ`}f^<$%0_d)fN z??rb(T2Ck^dxai)I?XiZ6RNz>}FHVAn#=!BZ~ZC$4z8 zx2?yg5i}ggYF^yNpy{Dn+XNxtPxJrXu-TvT27Ut>-LaQJ9z!uZ0NKWnG3+)pz(%O%&GqtrueA$PfCO%PtBgbU8Mw%h!?m!)k~ z$fEl3<^y^`?1~&OsdRQRj$~3sE~mlyq#j38aynR7e8D$*Dpon61yhQ-)_C z-0^uB&~wCYP}8hbOAs)p1&GVT{?eqcASf@8fR`I3ND1W&SE;vU3vs>O&jqL7K$w#gh=KMUy;%T+I?o z>(ctfY?$d>VaFS!7`6DRgZBx}>wI|lm>Ae2dr}bTo#g+yJ+n7r*^V>`C++&9Cgji! z%HkI4o+IqdOXs!u^;!r;yTdh5=j|v}4{QIvzt*fZ zaZK4yPbq-zAM9)4yk0K{k#*%8DOs>{NJU!J71ex&BrYhI!7-v<3~YTmW1XX}>BquT zEgO1pv7jJmS&bxg;`SR?=LJPQwIf)N$8s#ZR6}$K_oX!*ubUOk&gRCLm!VkB)PVFUJt24oh4Hs4wYCXOp4!fBV$HR1wIYl zkL{h<49$Hy*fLf1#$&)oP^$lq+nfWNVu-J4I0Wfo}I7h{ro?x9?=5ym1m?&SjDer97e9*LW+ts2BMYe_C6r*!<<&& zu)^jn`Patiq9XKU(nW_Z+0kV`_+^vaW$G7=yhd7`FEF*!FW@tu&}|I??No)HAnNRA z$ch|(7lhu>TD|Y{9@mcsf3S!i!{k@hjGnYPb*LZ_@l}%4tDltbfi&t<%n}W!_lkH#cDT6$T9J=$kFVxU|ccxT0?cmE8(cG{X@VP zIC}d?Pd}ba6h_{-+1|&Q{vlUm^*r*U@@0JU4`_pK@kOe%hyS=e*luyr%?a^9{&Oek zElzB+FnrJTqL*cRg-GGWa$m}!g?-P%o(T+(0e9a{#zAeMub<^#a(t;Un;{G@0!&D^ z9lefdkEj$4Y-C-;X|0Yw4_9T@N)NWmKX3>~F(+s%B)enp!LXt~)$TB|>~`OSEOx$n zogsQk#$hBD?qvW;fv}nM;QR?7mRAa)OMHuo#|0bRsjibS6!FQxH@M+|>v||%G{HAc z6Tt;7;V_D&YzB14qzs3*wv%`i&4apH16eKG4SF5`dM;6I66`h>l$|6AUbzh{KpOQ< zvs?p9&JCs;FvCI^afA?FrLUZS_4%B~V5VrY7?r8}SQzu{Ztq(lqq;MUe2YWT|m(X+HtmbJX=CU#&l8 z2GgI}$`S{6;+*df7g@8C@s)#D;TKe_D)lrFZx}4K$keru#EPIi(@&hz7s6UV;^$_J zMqT_Xv%iMptL+_81NoU4=p_m4ijw;QD+{B8EPFJ}UyL)%{H2t&J>rZ*(C`(4RmiARs+o29!e~rJr*SoW?GL(?=ui}rmZia39HOqZ} z{{%p*j(i*TYWfK{^%L+{D)i#tNB{fQ*~4VKnauz2d1BjM1M6S@pNR3=MqG38-cLY7 z%95Zf0=pP^W8Z(n^!bJDyTAKB=#5GyCl}8jw+MPqw(Zw@Tp4nvc?mfQ_Rk4FSlDgJnCTf&&^MbP^I(0kt~ zhUDg?CV8cFo>CG=nt}5}TWCM`{}m)92@Z(O0nF-h#^42j!=mO|_sR1It017+$uUEh zCuKq9p(0N4qIBQe=CRvB&^pWdYy~j$(_tU|d#jELPM=$fNN~_cVAVG9`1z}ZuqVUV zn(p*Yzuz?jE8f%3gznin<{^H_kOK$~VfD#AA5|HTTo8&0#}e`RG%25SGBm?3;My zxon8(r4ev3mN`*4!Yz&J=l91C`b^^*0Fm`9QPgUHl)%`)%4y z%x5%;zp=I#Zum_F{?q9jXdW{>-%scvix2&P<*&(P2Zgphr}-<=`o2 zjye2jyIgVasq(++S1^9MDGT#G5!$Ca8PjG+x@)>yVVKfp$H;=q{%xcgC1oD==zX9Z z8wDWT?z~%8CLyRJRL>?w(CqEj`9>9(za9i>2vh3y{pR2&0hAd8o3EgPT9*V5=5<|d zss)T>Rv^ugH3hbE*d2$Vkoy6k&rRK)fnaUEtPuku0hT*us-Ue{E(ow?0SdAcv&#qt z(h`hPN&=fGmLG&56)7uyS6WFD0IT3l%+o+fYsUa9kcb_`PEM@|4wQsxmNDv zB#1U)y65`+LYuSG2KMP~Oug%}aPhMb#_~!IxSNWvEaeWU>zIC+(XJ$`W86!s@$$nl zQ2e=3xar^fzMC8CS7WnKRMwE?*k_P;fLmkS}O2 z^1qWcMgcszhOt~o>(UY@Uig?ry<6H05`4e)V^+%MKXIZVRY69#LJXYJZe5g1q4}RC z>TJbcy4w_X(^bD$^gIwFZg;w$EojLIJ>{MJd5KI2&K? z3yGB~hC1oUwHP-?BDDKsqsy*4xAn+W*&-9d5T|~JDz+#7b#r2q^f)<4 ztAcO(n=$0brdC?Qy06sPfp)^vPEl8@bcN57{RF(>w55fc%eT2>gv@e|7x6_#B5j~X z8tXR}GWJ0DG2*~{d${mgMsnHcm(KH|TlpWWZ!Jwb2VotK;=(UlKxpf)BhQ8p!YPN# zcgy4uoh=vj)v8%_=9@nGc_Q{Dg3WVzdT>;{kT*bJubwJZVN&bU_M`aB zxl?dg1E!Sy?5`@Hg;(^LvqZN7^VvD0NEK{hyBU@_isdyz;js5!a=wr| zlIS5&n^00s{=f(jX(S<>zs;(9?L4pYwaR*bq-Wb$ZpCTiXh@PJB;m&J(x4Ef_T(1M zk48;U{0Vpt2?i=>e&O#K^D`+>*1jwn+Wzd%kyVa8K7?lCZA)+qNn3 zZ#`CR>ZHEYK4#|NRcABiM)LGy*q^ip1-Jg_?{fzyU1m1RX*gf|mb|~X*nV8h`iXrd z8RzHT2CexKTF$Q_|6T2|{m)AL-8`?sfMxDwuZKI9(r9}k$kAy7dC*VZ-qkY@G}gk% zJ*RXvIcqIfEykeF#hDO#I&7UKKCw%9ENe6d?7LOj_!A&7Tlw0?rLj9GPi%eysMG&$ z9qR4e{hkO7n>oJc#Pl1TpMYDj|2?|={WG@ug5eG8pMcoue?R*D{XOporA@>jd8UPa z|NST6<0H|(*Z=tXigq2*4C^>2IeC2IKeFz@$4);!!H+TN$g;3&{hD^)H}{WVUGej^~lP^>LbVCqx=f>f|&7+Rx+N;ly z0ZR-h>43B>lP9@nfdO9a|waGBMzLzdZ~sRTQ2;+oZT2JHLRuaCYVH`}TR( zO*OmzqJB5kzR@B|bOGhZb~O6h@$wTXbk^1>{Bq5`=DyOBx4SMo9I{Mp>%ndXrotmo z+4e6=*-d9dsQS(wYje3yC;T)so~ph)#5GN`*f5NNTi^x7NoR;P%_m$|7%!Pi3$GX& z9iKVR?*4G z?Kgq))ajF%o&LsOg|EUC3Kifooq=S&zUZsRVFRYwmhTOJdWbGYP;7=X*V)@cFdaG2 z-v>9b-}jQyqt&pUB&E|4vnw%9XIXu3ZaTTbV!12(2y3qA-D;PLX?-T?<*jmMn)KCG zWX%06wPl}vm$Giu!RJDx_0#f{yV67L?Y>dRatFgHK0zZJP(FSYG~KOowfKM+ZsWq8*MGq-OA0P+Ku$v%q7Lux{8d}$&DOx+2lTeChLP@tIBtVkM$uw z3--?3?if#uJJD6C&ywO%n7T5DV1qoMKT8sT95BnMSgEhguq7NqRp_tbFc7R0l_W3l zvFv`=S*|uFeqp_!?#tyL#k^zA8dh&Z58WUidUyWlEx8A~$=gnb2sv5UJ%*(Y#-@q8 zMxT1#zALcyQicJqCp8tKslc=V_MPhQSATuTw$mYOF8W-GLunifpOWSZdDWTH|Isdu zEm|I#j-{q-8cQAbw}Hv$NLngGqcA6vvrr4E~RJ-=#sV2<6Xo{$GZgh=C0zE zc^xrRw=lt%o)?a5j;>^cvL29$=UbW>!Oe}*Q5Ry>bA>I#_#Gm)%oP`yQAG5mpaCV z-pCF9d9EzS>XM4MOvjw5WF2w`muMi%OsT0lkHzZkk}e~Fc$ik>os zz9zAd1Y(NoSm331iZ*ovK8S|jGqB}(0Q%xcRXbNdCbG!10bu^I^e_Bz4rNL~-K|IOv5IN&uy)?( zHWBS~GC#Ox^-|9O{fnC=zoWk4L9NYAs3;EqP(Q)pdS&Dm;b1{Dqf&clcEt1=_c63YQLWC2@vWu$**5ZwZ8zw`ETe#$|mz?WiSvn1erKs6z zukz4aV=a`lFS_p)hmYnwfy+KVf5ek{mOV(YeDPM373T(ea)YsPswAU#VURAEHoo>z zcQA{5deHEOfwRC7s}i&9rF)$V)(I0%_1L(elUaTGGb@z#Db-mYCFQM-q`~d*66992 z;NhQl=P_xks8^PctYEQ3sR+d{;Ctz`RZx%|m~_J}9Bp1)71?|Gs$={71Um4N*V2*E z=7N1yD#r)hci(G}6;0mAOUA|(*mapEzdf>KbwTJLAJl`z+Fhir!H-t;O~Z#fw#)8> zo^Co=d-c#rQ+-)w#AUyB?9E`ST7mHt6FuhO_j0a;Dbv1gQa1OfFZ9&=t`N)m`KPY0 z@5-BMKKd5o6f6C?{k@tm_%7Lw*?DR<5jt_K(W?B4Z&`QcT_>*QKc#QPrU&wtd&0IR z+iphH_e87@W*C=r3KhAmZS#!Uk6W}Jr0xdrGPyV)dA^)8of(HI%FF@w0fJ9_ZIKg{ zSNi1E%i3YwiHf^%SkMUdPVTvNCLV~E)N|>6hhGH93Q?av2K*i*8=4`+*V04Q^l9y@ zETh`#JswbbMz0RiYm+=y8I^Y!$22L!{1*FfoB4TCo6ka<{^Ng&IN!YsA41bEF0o^W z`Mjdj1_w%Aakz}LVtg+$!{8QVWLDNWq&`iFCYTE~Zx8#>J2G}dhj66(-Fm;xIk{^) zat58|DSKH6YLke;K1{g;%L5I908OI2fMx$*Rmeder2(NT*C@!t0kJB+CrX~;_q6u2 zb_Fk-))S5qtLOx+*1uR(g)3GWLQ?v>EQR)+L19QLxHr_QD~2zmHIns?rwUKrUEK|R zgv#$R_}Uqo%(|8@6)MgxPp&*nO<27M6$4ThHiOsRThF2cYmyG5H$(vk$S5XVNdInH z+u{~&^O9<6^}uhx`~2_c_A{!fiu+leN6m+x?=^AUfqg`|G=rWq#3V4%T?&JTd{!yF zs~2nqy&F|SVnubSE+yeM@$sN5)Ir&EK6a$&czTeH2O^uk1~M?a4JHY@vS*nhq6NJgs3QghFqq@RFH&YtAV$J#@A z-zo{_k^<#3n5(r)@Zm5U_UGm|cHbUb^$RZ!W8xNl$5)|Q7k0;%RwtmB4h$RaU+QU@ z_nSO_7G6Ah{mA9{{FM*1Iat5^!LbSPe)^Po-QVXt{93cwZTGvJjz7!L7aC;wDCimV zx%(;VB=)j=uiT3Mmpan5)$*Yo*9)25R}vZ=qo@9rgk>X9sz-Is88GXl1f)W}rlC8pcQs!zWtiKjWKTG%7bqEv%2Le#iC!)-iBJs6&hWXE1snzV1Rej5pf1r zE~f(M$=l&9`^szLx|;gnC0f!>?z-(pW7s#u`uTD%U4anY>D%hS2wD*6%7kh}5Olpu z`wYHC!@Y_^)c>l)t>>mtodp`;38i1Xk!Fuq+MioiR#~=?r@{95uo~DmNSIs#tce~t zrp;af5X1yTOKG1~)J3kdt|yA7Jt1i=Hp^9Aj#|H2RElO*ZpcT)scEs;swoAs!Ai87zoB)(yJ5?Lzusu_Wz5Osm4%Jx z%AAH@FC%EV4PNufxIWAvm$y9e3cdHyY;j4DG%}!6;kGm9MI_H(FK2*%4jl6Gm}g7T zK*!Q+@`=89Mpm$&Cp;31q|99iNCvtznRVd`c^vf9_9Tn9b8x*z9LY;xs= zp(sCCcg2Iz%7EJsQ>cNUy0(g0ABbOn+QhkdT);9?%X~wiM2>|GUUlEns`WMpPd*B^G>W#g1C31E3q}+j347?@?OD()0 zJJM$!YkUcGTS>*4Ql?ZcVS4`7N=>XXCE8_jdh3Z@2)RpF(UO{^CTO5p?h+{5xk*nq z49DrlxG>FrY2K2&d3*T1^MmG{O0TOwIdtxpXEjZ%^EZZ+eR0H6KVc?p zRV02<-}%Xmbmg+7e3rb49eo)c7@nVSoaRKq5b(kQOF`g|D=+N;%kw_o6) zSK9YmIa>RhDE74VNz-Up+duBCY!BF=ydcnG=a}-~B%I>nv2E|t4HXB5Vr9os*);r| z^mU}oxAAd)#SUjb1&U=*rMN(J$dWX|D4gkG+W+rcvl*qh=S-le}|RFbN6D6YJUH2ckUm$e~mlnk|)% z9`PG0o^?R0SOWePQ)(Myg<4WyIo24V5$s4*b?aiQHLizfD(x0*MaNQFYuk@pk@}mr zo}0DFnFG9@4J0czg89^)07v2Y?M*PpE=cAhH$)eO9ud0S>6o zSV>kgQuPwioWWaVSjo_P3RZh429(psS+jNJdBNuKG1C1lywSjD8{f%jT;);+X77d} zkz$$({!1S>`N^OeZk!X;PMB+r^{tWp)FSl;*B+exKD4_`l>IkPn-(@x?iHV4Ja}QL z`8cpIEMhk~(sMJI<6q=c$SWJasxAJT2RobFOYr&7eKSLN^n7_x+w-EOWtL9{H z54%8+T}J|+`$_*6(~)L1^Cu{!VCqXUil(*!z0@f!mP#lZ_UnXmpg&-jQP5n4OJ zDA~~EPaScMB}Y$tTe2nwk70uSD6)@?dT@|-rwh3>R^O?^!IXelp;M|3d*l#|Qp<9X zEdH-hcL~-3?2(YLSfM50jW}OKVPV{qwZ-w&2Q)_-Cp%g}fws6- z@Kn~EYQY}S;p`Wcu^g}?CV<0m<#G#oAgijx=gsZ*lF`o=0j_h3IdWIA8E|wWJ)HS4 zvh?0lk0VS+@gzAKkRdhfezQrKz zYF?1y3T<`69#@=J73%QTnL&1y#oXv+RZP79J>97xeHz=}_UbvZP+o=nxcMc#46FCp z$I<>wC9oVtWu$Y3#OyRG9m6i0pj=9Y;aGqqSPbat42vX1$C#I~yr>IOC3Yn;v;IU= z;1N*}6_OnT%3o*50zfxKQPtUd7)eo?1Uo$RGA$q$SBaIAbk&fmIxVV_k~Z{6SUQX% zIZ>unJj1Y;+>$?2FVgf<8{}!CCGZ5xj_dP!i%n|{Z}(;IHLRhUui9*2ZBl0I^E%iQc-f$WJ2WAvs+FC zy1&FX&10ASud_UxWUR5f8Ic?zF?a3Wr#GL&;1-!!qm=P2C-L5*79ioCn*<)7QENHY z$oxYY?B-?U!5|Oj{|+a`-Stpkkq|4P1q>j#$GSgstgkhblqLE=t6rt(Fa6+knLQVt z3_l=YWvIu3mCp`k=ac8_FWq;c+|k1b@O%8 zY3~AkeyK@~mdWzo$7(s{VlK!db{4?Qqryu23ex|y6MW~c&6|DrEVwerWT2Dr8;l%J zVL-ES|M4cFPP1`YIbuxWDLc}YsnP?lXMRD5RPys|_n8~!wX*`*=}6?c@y4(WVa%AGI_~nNY#8X5kwzTNM4BAX#+9xDU| z8A_&Rf&dtcZON2y?PN0yhfp9_!f*jmlCfb)gA|7}x^k!9LIPNLh6M-ae2OcXo3z;U zzf7}ek`?N(8GytsuUK}r+U%Jbq@A)0&xOt4_!}qE9C5`nNUEkIsdAGjDRHrf3}cM# zA;T8zJ|88$tC>m68Kf4%IpFRf_u~`s4ARnYK&R#l>@Ea`^YV?-5;~-U2x=04#s$dy T&#y?h7CT#c1__e-x%htoM-w@V literal 0 HcmV?d00001 diff --git a/python/nano/test/resources/train_image_folder_jpg/cats/cat.1.jpg b/python/nano/test/resources/train_image_folder_jpg/cats/cat.1.jpg new file mode 100644 index 0000000000000000000000000000000000000000..2106d4e1f116b5063cffa5ac9a3b3941025edf82 GIT binary patch literal 16880 zcmbTdWl$Vn_%%4_;1(bd1|M90SjeD*Gq?^;f`&k_!6kUm;10pvU6T-o1Q^_dLm;@j z+wyea>^9bDw6PRsbZ*3Q7t9G&BGJ?fC#aEdXQzct9Wy z5E~B%2L~Sy51)vNgou!kh?atql!}?21rAr3|+K52d-5iv;1}5--aG{}lKA#w*m{^Sb*ksaLKy$%gO&h`(McZ-vJB$|Ap-T0``C7ng1}OjvI7{pXSrn&l z{N@lz@Gdh3qx;!8%*DXdQnqc+!EVA0!(`v`%?$EHb_4nzY#k`*w)&Gl#l{4S9s&~N zJGv`D@gN7NK=c|R3YuTmt?m8x1M|A0_ht|Qf0O;TZkSFm&brpUY>uL`TlB=x$iCrM z5w_{9vL#=)5sv)${waF9@Ad(nS+Bxi12A~h$&$LfVN*;F+d@+cO#~Y#An8^oc15MV zef^xg;`>QRZZ#~J`BmWXrqz~&VDu9}JnE}P^PTQbZQ;4Nq0i?#lV?32Qo84igm{Z1 zSjZ0`w8YSr0M5+*{11ouUSGuIzX_0Ai6-1ydf0iZ5_?$6L~a%ad?O|R+`B+uS>f@# zkJ?e;GbGZlUrWdxqPNWapl_9AL^=6I8tn<73rKa{X$rWdhajln=3A~g6LO3y4i5;V%s#B`-k)^!cR;Q_Q zbCRgDmk_Pdt?7c)UWe2E)A(pvfgHiHwfW+<_iZiBshY}issbSH>|?WGf=4|wm<}dUzUMN! z>78%Ii#F)%d3%Xk(=9d^f`J#&`O)G5MR7mcFD^$Uw!g)PexZKHRghD8ee3zana?b4 zMK@P;_J+^04+Rl&jZ57~f~E(V1f3D*(n@d&`OPWZn7ElQ7pPV=hN*G@%8$;Ah*YrY z8SZyn?ZjRXR8p`3Zs9c3(ZflMK&rcDhe(Ov7!zfG)!`9qIqiGKT0ZG|8p$;_?6dtS z3wFb#A?~@gR-?~-R2RUX9KkYP0IL&Io~*J|8O%A8eYTJ$;te$d#Q{;XSPipH9_lBVKu4GJ~H0 zq5PyJq_e`xZKzc}%bS&1*s_3RvmTr$1#Im@x?Iwl;D4-KzMthb>J!iJzLf)f zYg^+HuOX|)17C~(l&&3Kwyz-g@X?it6=@!oV)$8rwX3;rYwS5ZV&rzU>O?2|u@^~D zj13>8UIYAzjp}JpX_zg$6bZ5Zh=Cjd#ZvTwpeWH+%bcYToU-8yr#U$nQYzd8^(f9- zf|jgR~OV>HJsM!d#fZD+)hM$5M^`>m7?R2KeTasEWG$Ieh; z$}hTJY1Z^bOCC-tAPacj;}v???|fYM4zyaf7Yl=Fy_mL{L0Wd@_!sUM_)0(=;lzbn%r?$mNca(o|5= zL1xdCq5pI@!b=#n!xn8sR}VgBLc9b-Co0EHQi(sx z@Fm0jibMZ-Nex9C^Z@NZ@89Iozq>%goRMrgQPUO`xP14Jn)qu|! zPXIr>{rR1ffx1WM%Y_dkoq{en6iwS?HQ`BMF&ZXBCJb!CJ zQ=yTcd0nGE;8Si$4s!E~5VjLL`^KH^9KlKfT!Fgl(R8`%QREr%14D1lB|4SguQgfw zYmG(alX3Mdevv(m=_SdQyav-2K6mm$hl%pYPDC=d~7;3qojIDL22o; z)v6|l61u7=5L<=#k{aQ-J-r4Xr53O8^(+}sff)dxOH|a=C6foA3KHgi1tPAzD?Ejd ziS%B^*{W^S20*akzQ%?Z0oD{Mag~@BZ zU1b&7mz1p(3w)%(>Vlbnn}GX>C%`wZTygBTKMeI9R@PM&Q35h+93@uXzAM4uNf(~KBSlDfJsC+oYA1tsPM&@f$Qr-TAwI6vFNE;B z!q?rL<|%;g2}W56Nuw@?Wx>}0yyRzK6aWN}P^uvPJM3S)szdnm&>F=y=l8I4LioE6 z7NI8|+|IU`YecinRj`hzI=Y*Cr6C^^q4C(crtwu{ay>Fcs%*O`OxtU1rrNZj371lN zCAPgnv`|QRf}bIkiagPCF>E}zx|hh0!x^>}%n{YS?s_wk3R&^{IM&{+(&@u_77r;& zNx-uAgwVe6tdGuA z{cV2N=>t2ZnPe+w2k;*X>Pe1#y%qtSoL9^~FA7|JrH%>+jYz|S_kGBf+v%8|CKp*M zQmnPJiUVsfA}uai!mnIGDUDMdOgTi7J=jK~C=_h~y%P$FK2wVi4yUz?9{XugJD9cB z;UP>?9>yO{E**3oAiBDy=RJmzXiTx)Lh>Gc!5h9lrL^s!WV9aj(g`|HHWD!;soB1T zfM7RS(x|T>ec`^{lCC0j=I|A?z9#_juZCnb9sXk37xjuJiLNL@I9s4V#y<%D;D4Wd zrt=ha=YJZlkm(@60l8;Ul?;ivn1N;fD{i+I5v?!tgo>I4-w||r5r3Kvns#7{gy+VK z;T8J=6LUYv5n|*7?0h9&y?r&*F^SX?w-<(r`NV(J=e2m*USAE3v;fKBB6sZ?uM_?L zas~<-6j0(Agv1p?c+W9U72g%}>Jr&^K6HBEa?bXTRxAHxX=8p5q}>yqr;Ja5(``-U zFm!+Zk4914y<~#uz1{C>Qw>I~Sb3v6t3tO7HW1p*Mb#YxbI@OaS=NgMy3X7k^^7R4 zm-D4{5}m!pMN!)JVY@f$Y?wJ6JGcJnyeATKbJ$)wU9OMhnWn*}1CxtS092ENg176O z;+nD4P23qfI3}tCfN_nOB>!6H)V9glmmdL}36PRJ;O@bp@ z(NfvKfqDmK5l#1FB8+pL?q9m$u{0Y-#pDz^`PEg`uDz|wKbE@zLYg~*wGND+I9WA3 zeYTtFmttgeTbRBeKMoh%9zn)rdYw!(#+#y9@LT@R4KbtoOCje~8O$vGOU7YKo8uN; z3oSO9el9O>Lv;On*N!Y-+ZMULF3)wcZf_Nt2VR953M(iq(3DXa@=i6BIU3IttFv0M zC|@3E@!E_{WLbPsAx)JdYw3~h^_|{mJ`Az3tS|Ujo}paFjMu4Qv|-&Y5crZI6lL^f zh*ZWPk;->Es^9Ph>w2GKw&d7;2!2uzJU*8@p4do-!$U;EyUa+&@)A2X43{YAawIk& zD5zF#yjX{n4a?|9m9U1me!yxutTmP_AUxQ2!l%&4Ppv$Efp>f?E3Q~jvsw3xx0p;N zWy0uI?lA>h#@$6sh~p@Os~)eWsd4c11UccA^kTQij$x~Hd7&a3_dls}x^umK14rgX zqTFI|z5S51M*({u13_-y&P<02*RMuc=-5Vt^&<5R)R`1TcR`)r7aPe$2`0FJJOQA6 zQ+0AA%DLh%%(~d|t`8*?J8aY(UjO)Mk}H6b>`=HxVRz&)0i!;M3Rc-SD$-t$7}34b z2zackIK!pbY^G-AQ&;~~2e>LRv>o-ZjuKGlvB3fU4+|A3TIlc6`tu$Br#%z1#n>wg zc1k-mOOpnKKs9D0n{DGK1pNmrc$yNLNV@%epJ~z`m*rk>(@$Z56bt-g7;XLDbaq+S zrGxSg%aUVYqXHjQj?b&Y_Y(^Je94LsXG>m>lY}t38~Rk+B>@i>>l6Sle?ps?+Y{iG zRm^4B(C5~%CqS-pr#Dl+KALV{l_7=H=}G)cV9xJOpW*lf=PC@v94Y2%l|RWJhsdV! zy%H#-qD92cu5-_J)IaB&qzycb!Gq)dx>n-shE&48ia4|Z%)dtK4Htc6OM8_Zzh!4F z1-67NicN3`_zJ+zlD{he2_oj`Q1?Fi@+EUUWRCOXAaV>0;9vSx12b6dckVl>8yEr3 zMPJwDE|(pTtnjKnTDU6c@?!L&Xu}Lv`47K~(x7$;BM2q(JSo|T!@Ie^xfzkwwWP~J zC%wu+SvF@+fIU4%tXs<+;|U{^k)m$^W~x$jQGfED{>|)!F*l0x*ne)~6O$%enLt`K zfCQ1o%yhaH`+_r6R1RXl78TP^2KpCP+r|O;UJ%e!Z(6Kh%+PLuvoI{kNr%^9C^P>vA$JG%h4wqGUclDiy01fWH2u_R%?rH{0|sgdQmpm z=-T-Npsl35;t%-68&5e{+{Iv7KMZG;9$V29fQ9p@+fYeKue)#}zuBXEW5PX=^r1Kc z>7&Z^J-xLb{278fkNTwIHS({E^TZ0m2W3%8~i9^i%F}hd+pf$(}{hp4J=f+oGaCSzfloj`n z{1t#?I@G9jqe-=di9Cgx+yC-VFku@yYF8sv*;q+!0!)~TfZQmEh-WriP#0Ity`HMe zZ!;ki?@e}Rd#K3{1N4*n|%DvyKm(1OjV0 zMRP=sbT`NrMhNS0hH`({B&BWv!B?Yp$td%*G%}OzmW{&qUbBe{-VB__e&4d50FA1r zYTa6kM#TKtFn9W2J|}wT`4Uqv`8Wkpi8SowA^n(^vn2BHu(4j#m4J)ybyBA7Cj|)1{ncHg>Bl&e#p`)xbZ_RYdMb>vwfQn|)rz<>K4YJ1 zB8=X17Y5AC0=JnJ+NLq6<>an-9Vj)#hH-QSYl*+L4D0=eb9Ljok9*I_qrJp)P?#ll z6OGHcL`-`!N%cC0WvcqavPl&^%^ccI;oFcF{zSpC%VG5j-hIo+L#Wrw_(E?K1SR3u zA$H06yFH`K^vf?iPzC6H`8vk$S9R#%|L~n}MucOVO?Y9ndlg^x>PK~a6A2*9`6Vk+ zs-&YBC&l7{u6j}vh^+A(z!402C}b1unI{`6w{?|Kj8dI>h@#zDW9fU&M~tIFoAFhFAyc~jX;>a^x9VRI-X(%^vW~yevV20c=N*l(s{*WS5%_-#tv4JW9A8*tTzb1+E&7 zc7c^q$5DyTNDpv=L@*~rO18Mw2-GlMc-RSci3qJY+eWL~lH=jiVLA22*7Y8@Oe4k8 zs;vyASd$%B6?HP_1)2)2>~JwQB(PL?85+gZ%PNMt34;FaTvY40we-_gX=6Dh_wC{x z$v{D**KisdPm4a?Z*R~HmDnWIfly_Gf^_A>M)7*NyX*uPz=-IvhD3~PqtMit#j+*$ zY16gI(ZXrVKP84!$So0}Ebkh>)B;%Mr8kYQB?Nd1qurYj~M3BHe!Kn3!NKAxE~_v z)c+*PGb2{(d>+L?+C>Pdh7~cv5)GCJ3=i1o4D$(vinKs9BbmFdwR{TSrY<)HHUzbH zOKhZ)u6C|1l|(l1J=!5IKKgJW7JB=~;~$4bt4B=W9Q6%K4>PuHFJ5t?UT7hNwJwqs zln_cdt{^QQXMQ``JR9T3`X_bndl*lbjEI; zkn}fh&s3rECGf|C>oq(h zWHfK2{MG#_gK0)TF#6FDxTF6=p1f3rhuVC76-L+#q<`=+(@iueHn^Y5l&S5I;q@gT z?mpj%&P<*aI#K@WY#GM@5-4BEq6^7Cn9n=jFIhFE4Lg+vI;nczpP=$8jbipGrFmA* z-e*s(h{ku*{kS-|#dBnIA;^v-UvvKYfnbTx88Z6GHCH(m=chLdZsR6?Iuf(I55iol z?yB7rAT-X{9tw)?!whj$7u~AE_&ZS&+AUaU%_2iYO40eHw6^?UM_Zt{H;3S#wN`L( z1ZF%?9Np(ZRF8Y5DjY*ZoZe{1I+Bdi4J$f)9j@`cZQUv!7y4k=KXnkyyKe2whYeJo zAZx4_)K@UKp9q8WFzn2Sn6mMme*I~FhM~H}#!Oz(qw$twC13oiNPxj_R~RbtT0|A2 zX`Sf8AUAzdt{$1)VapFA;h!Q%8 zIe;Z{MBC7EbWJXt)$nL--<7~e$?7CrwWw7T>Phl=y-}>W-aUg)fFE|9pEg^K>Ab2_ z)?ZPb9KqBTIEYop&|jzl-Kh^?MZt*W1cjUo%7eRDwCX)$kJANv`xQE8vD-vwitFv$ zHAy1$x?bYflgyT+#)(7s&LRuDiX{Qc8LvY)&k|-)w7x7;Tj47URUoikl{Zyms zVU6|j2G6C1X#p84-y3Uv#Q6QK zyrJ!EAv2&ysvUiyJvSRYZSup>p}A+OQ_RP1I=9P>hLubrlAxjKlN;dwL-!4do!&1F zvlQ`O+0&m2in0gMq_;B~h%XS!L|aMz_ejGtZa3({vdqa<0e} z9@aaT-XW^K^5z12v?;f>y+`jVm_yD?xAYtUGf{#+YBz8EO0L$4SC;Ip5KH~!OkC-7 zar^O^sRnt}oss0{3-QU0bfdK|3b&>c$~~kBk!vnliGI&S`9fc8%h+_g-q^{OC0IN= zgqp!7>(1uv^~4f(*mKiBeI*@TKtp6ra(2%!_>V=nb-ihzg*Rs4dsVbRzq_@`<_A%iIb$886{|vw_~4$0vW4|I)AU%MU~h7AzogyvqvRC+1+Dd zY^NLDMR5suLJ?EBO(%0#iXM-{&jq(@0h#c{wS%vuRAYcM&#laVbH2v?ad^{Am@j?@O0##d`|aSfA;oh z71f0&jO`{U+U%`roR|Fk-(u)FR@zk7d{%xtdk67^KNKaLqPRU`@~+E}v4p%{P`aA}}+0)d%~v zhPWP=dF0+OGq%>)_}HXj<4l$&--vBsbdIg=dM2q7TlAQ~aM5=*lSIMZ{qvfNxhWYU zDPGr9nS0A%;xpju0r?k=HXnU0S{%$WSH+uHh)vt?mZv1Xd=YYbZIBnQ1bQ@|YaMWu zB|;_{gdLuTI*jD4bBs!+G@BRoFeh;068YfbSV#muhlSCUG2d(y9~J#K1Ax>OcB6;< z?ug@zYWoj_^9T&ubLDzo^Ii7%o`{-u-OLM+D4g>ns{d!IxlO|xUH*CirrF;uiqh2w zIVt8$%^q_0Us)l!0q+JXoz;6r@ps0An**@sas5Y-y^U4}O9$78ixRJP2HS%JbZ-71 zK?a{!^{0Gt?K&z(A5to0Yvic+OW*PCbDjd<-x8dcU-${|*mc=7w&^PYyfSDQ^!`)g z5ELCxF1S(m{5{=LsMQt$n`BaH5lJTJ7PXmjs|h(Eo41>=Z6$NEsN{ALAYLxLd%H$; z>w2I%-XvI3sNv33rqjOkeTKQ#EAQ=f3(Xge>&w;~@+W{V+ve*6&rF%3VyZioO{H-z z1JyB+qy-N3=<4-jN)-12&yUWfaG9T)dOES~;H1AHAqS?kcPt~B4Q0GADvZuOlZ)P! zjCcGg5jT>Dt08-jCznOjB_#o=R+@ni*@pez((@>coa|c{zmEZGuxYNrX=>em;-pDx z@c_}j7w@&d_H+OKU~&@db+6d!p*Z&moo*Xna!TDO(mwRPT=c8gzP9P9HMeh>8S9gwPm~fwb-3~T+v6zGu`TWB znY$}>0zCNq%XInT7weOpd{F*Ti>2D_j&N(lTkbPjZZhK9{ee)hM`In6FL7l5h0S|A zd&Z{Y+n-f;M=aE^nIju2Jis~aH-NfN*2?md$!=P;^M4v`Jj>k3=$H%lOHk^(~HB3#9cM4IlIaD=f6(3sP;sD$IfHp_3zkq{DXGan;st4 zBjSBK<=t1TqY7M!El6v|Te&E8z$B9p)kX`?6uBr8xhO=BH;B{2f5=DVP__4RhJ93c z6mH9|(Rn|!brcI#j`fkgyp|mn4)kli0y|8Lbc#jS{Nic zgB*eu@VT5{Y%4;mU4`Go;Ux#5S^kL}-7psRS;w^>TkP&Dl{AO<(FbB+r+k0ua*5KA~>Jps;bV)@-j>X^dj-0YybDhEeHUCq2u* zWkro2i+wvFy)fVq+`MV4))9&qquGD%M8*(lf21iS5L`N>?iXfvyX}g~xR)p&bZE0e z4yTOPZDYBf`sIz#8G{1{+O!Sq!Oq^ju9=zB7yFCi0*9FY=|S`)k*9v;huEPr;o@p# zV)DM+ljYQRUD7o+ddCmF2`&jrL@4uwzMZ|dL8aHlbKTaSXCX<$)?O__o%n7e@IM*@ zl9$A9?Ht(ax0UkuWX=?#0H{wYVO?xgM@n9@kjO#8B7n45__GG^w!NC@Wvh~BB1Te3 z7ICqI{7~^1zeC*fViB(7l>YZV4seJhcKeoQ2-rPUkEY z_o5MQL#lkYFUwTA$n;Mvb*wX2{#VUOp|;N+2x&U8 zGE5@x8qw+D^K-nYUtGab3)97Ili(;po*&WcHdl^;StotKT7RyOB}2Iue=Zj6^npV^ zEnKfWou!XrL!v6h>$r+!G#1g?ix~JVV@oH>Y2rl5nF6vGc!(A$CD-Z4COaFKZG$`w z)}LcQPh*K*dh(=VHlcOKp-lK$TN;!OiWQJrnMyDs3^KRuq?X`+0!)|DK^PN0d$Z?# zLItXep4{L5$%UxBU`&u=7O@^#ojf9vSJJE3wz84}318S-T|lmeAqxem-7(aFhWwi} z&(ha6gL-Y+78aqdtq9t0Q?6||Aj#$of>d@{hMl^rnSOIAZn7uG6 zXJ(_6^D6z%2qM8w-UkxZNscFen>>b4>7A6qZ+oTeh%Gh|(gS#(n{2~I{%da3$TbN$ zf>7%civY$pM_SMk@*Uf5(KW^NJvN~;ixdH?If`i2Fz_1feg4>k&MPT@Q|EmcYe`5n z)3gdP)>NW4J7dbT?1O7UNp;T+3R5lU9dXvUGV3qp8rp2Fs7yBC{$~M!^Wa=!eVboCFoAwj^0@@O*t=evZ z6mn8cO>qR*<4_X$jV(R838}^@jUDKBxNrC>>5%7lD6({Pq=u%wJR;_CS_ltEfE;$c zsUe`tXN56&PRW<4X>KluAK7*55JNaBog~20dcdz2}{to7J*LP$0LoA_F>7D9>j)? zXC8Wk^7bDw!O|Q>N0&DQR0*iJ58aCYK0TDD886^c?<~+P&mk=)p)$_{B`` z&#CJ!?BQW#EHBoVkOybw zEiBb|_XIH2GH~~e?{YX;3pL40)Rq-J33ko8oi67YqG9~QZ60T^+PtEmD84y~U z+6@^r46OJ#cI>5SEI^!{eUuFzIL`4kh2)brUp+u-}wR1T5_|&X4qw!i9;_Ni+ zockrtl$rnQW2~PKWMcZdI8{wI8H^(tR)~=#1!S!{fTH;->8XQ*F~_48WFjE51q7=GKDoGyS( zbC+?$*RW_QS{!$Bdy7;C7jzB4P9kU{#L!Sv&T)a>@z1s*H;;OwdD4UMa-mX(S`h<% zD#npt0V@oSP#JzDllPi`##Dkl{=h1xf_19C1ApAP$=~EQ^G{k94zVb z8LF!!kwX6%^lS_HPSwx0tFa0xuI+kV(AJ-06bsP!4#FGmXxDYxTsQj}skm~0wR@^{Gp|;oPq~TRxy>!k zVuvfY3n5)smzA2fB>tDP^PJrcHw8L;oQEP3oyC;4U&a<;pME#UFGd3xTm+XJE}(?M zxirhOpSi9wg-xFOxkfYlV?CRnW8S)k&)#!9NO1OSmer%0@ty#lz}$((6yc?DL`tl( zgf?cFaH5a`GdDI4eT>^T$rN&aLWKiC=QZ}{#x6W3%V$Q++R}-#%<1h?n(Y8KHQtg7 zmeXJrF}ME{z~a~-ivUkKrdcS(-`WcP^F42U^>?K$n8MZ$YRGtRVY-o*_bo*C?c)7! z5qP}Y2w()~d=21UYHD}cN}BFqcDCiYj0O%f697U3*Q#A-UQ{hueK?_BsUZGF%uGMf zq&!dcrX1X;K|2>aToh@%>7~R%BO$3Xvn^dYzGVE3YV&jc8X*S7b01M%7j%oYbEJ#D zXc+OikFCCqI!X=%@JWIGQgrVnb{3twddY(uX;L-bif)1Pek&gXOqW4@gd)#GM>ZI@ zPq=@Jd#^}F{Or`oAILgi`GX&^-trA0VqJ@chqS70Ffw3J9ynC z;GNyj%PR${LbwhI~QK zPV8E1v2=+N-q4yq4koHCA-8YJvu2DxTD5}Dd9clK>+n=md$vdk{V<(mFTEa_srC$8^OVqG z7+7nY$UT`CJSKTrlsJcLn8MPmRkm*!|D{+h=6hZA2zMD7yTxqGRwuTLTO@k<{~Bg$ zi&6PBUe4U{avQ(h!q<%YM++GTTUl8*f?{53^eyLyW}KoJffwfX^tn_QE3QiJ^huC- zh8S?~c;@R@_0-ML(f__)!X7vWVoE>WS8IF2za*#WxKni`j8K@I|BLSpr#p|HgOu_v z4QQxej3H(fVkjaM)=F%1w3C)|9$bLrAwoDlV!guN8T3o$ZT<@3nRAh6JTGIbX;Ppy ziaTm90>U9#=IuvPv7`yyZBGE}mxu?iJd(fHY5ZFU{Ql38u&U2(&wVH>u45YiBaM%# z;=!;8S@G{5d%)Cu9`tY87Bo#`BJF@|vE^pdo7(?yExT}G^0-F=XsK2>;Slw?t?hfV zM~qC_6CeomDCPfpY}W2Gz`n}jbHz%%hbR3pgW|^Id^^tPj|{xHTg`5I1pQ(%QFxGn zwc3dUQ_eBe#Ui{wQs2AE&KD%|Bz$>;nUMHbx!P$H)1AfAU;UUr z;o+YTp6_zH<^td^lzS@-@t3*o3BZ~roknNfC`;Qw40SpJFZ#a~qw5g6I7jZOn+m^{ ziT%<(I%1rKh==lOv;ch?9at^T-jqqT!VV zksupx!w!S2Mn6Ok7h%KK?dcg8p*^-VnnO*)2KY;?S({zJl|9_xaHG+}Avx4ry&H3t zLb0>lX`<%K?Ag7CN1Y6YApZx@I8m#n2q5L&Q#zK9pjLkO#S!k23lN2<@{|yHkU@uIa zXv{O7HkNe$sag$6WB^+(Ow{r6g4^BBW^wK)Kx?;CS{1cJ2`v+%hxc~b!@qf_1d`dq zPe#3GUmo&5@K z*97=+j6WoS^g0@3KVDgpa)){%f}^ozPnL|KK|+(p{1P72HeU0pPA@RLix$bo8~Gq3 zE`FY_BgDtK7GPhAG#+0&x#JX;JYLWA>!sGa-w*l0{7FsIq1jEUiAMd7bTp*`=_vwC z>CRoQlMt{^7EwmkV5CCMFV+NA8!~W0{lM1sWQpJo>Y5nA)UZ$ROY1FG=Z30&mnETaGP%LC1jT{K!vf(nIt&SPIkI-fJyhw+P)T^8}jV?1+_6 zs)2;Q`%_MLg22Xl?F#?veu*3CbgtMWjwT;bF zylpUtf8yG{#w|Z@T!=dsR+d~ zA!Bq&gORa*x31R{MB$=Uy)OD9qm$<~vC9iXExPXqqe&;37hY=#%_5f%Lt%YeNk)jc zMq`dvM935w+_n_~)*}^4O4c=#+RM@*IQ`{80gk=E>h1@%MLt^o@jL0O+#MCSWd^rf z(_#_w7Hazu`AEqeErd!C3JS$!S+a#>y_g6)MVd(d^S`3y9FkvB`WC;EMR205Irfq~ zE=}ZLB7>1^FG3|Z$q`#{M-{Wu>cuufYOF>gN)Jy$tofx+xhL zrmwngj|1VkIH&l6-;muE$Xucjz-UZ)ax)#i^*)(IH^3`EnG`mg`P!>1$)$nB1N*LY zZw6`xhpZFMRaanql3k)be2Pw*r?jr-w@ zBl-Jb4PM|k-Ts24oA9%Zc&zW`Tcu&{D=ju*tpn{G?c05f(y2thwo=54Va^|iT~ROf z?D^KoMAsHdI9XEq`KW%~FGQkBoz9L_gAEP=l3w_fd~K`Jdr7(*f2^Y3B$;s&V@wkE zJ2}~n^t>SX4sYc%ou#J#{pflN=h|C!BD&BU`dWAs@MXDt_f=COgp}DqTV3(0ak_%k z(`$?iBepa)EHmOOk6NXgGjkNd;RpbuRMWr1_A5)ODop97uKVOM;KAL0QvSnpiEw2M z#=ffDE3J^Z`?|#m8(q5{cOTYgcsqtJzytWXImA8E#pQ5_8F=AkY zky}>Dv(X*Lpu2!vF$;tUq&FBX&m7{odX}4;_iuKCRJJ7x~p*x(CT%4%U)vhVFQ2Tdm{OHUqAhucSgw& zKZ(hd1Zrsy3E+~(M(g%RwLg(u$gl5x$vVGD;rD9zZ_S28d8Op1zHW_QC+n`n>O; zr;l4SXcU3kP|2gXkU!&990n{FZYnG!O1Lgpfz`P$*M&lZh$JtGTpu`_ce{BDSQ5fB z+CoOzYF8&pA_=rW2eT74UVvn(i(bW2a)KcE<0RO=JW$>5`hk;H;X_ZOV$()C)l|T9 z5!(998=Sv1ewILtI@Rvzx*KAsMpd|GIgT#VkBzmym~FMGGcfaHW~OJSU(% zMjpzdF?^%PEr_*B6{sP_Z!|WtM{VVhA1(swbd;!%e&7%i>bF#_>2-gvEv+x6NZy+f zcZ8cL9rNRuPb}bK*PzHMRFCz{N{@;#7({Hjr^}n%KmZ>Y+rTzuG?MzK@P!_Vp{sG;2RCmyp1jXkQji`^$Kp*^lro~BxqMgz31iIr= z6g!LU6xmVE!Fd}(h5HlP?zY$;g-Q}bxJCH}v zsT&4ieW82a%t4o1ZG2~w=Gghe{d5-H8VGSc!#L-(^3;m6)^8JqmA5?jm#z#hPZ9q)RRcfQ91sNo5*S%Yav-@U0TU$ zJHNfEa5YgCK1kukh?j{of3P*YKwoj=Ya;>zX?~_ZhFnx6EkYAR$%6(Cft<)S>hrwf zrQ3r96-Lw1pb?S1+_DmuHUW-BblQcMoVG_lLNNwL?xFUSKPg`cmNe`d>5pt8i=#WK zI8kE!u?mv814W5Asd~|;q*3>k(JqkN=QT?f!MP~dfMr8aHan`GUJ^J4kZxJo`2A_5 zowNUM>stvwyU$j_Whbz5kTy(D?Z+yg>*zb#eS~Vl;UP7X#Ry)QzbHmd=iq5&Imn|5 zPVb`dHIlaLe63m{wC(LYr^)_e%&S%!_3=YwTq(U#(9eN8DtiS$>f1Jvok!ovpJRl{ zAE@kg5^}!MB;@96LpjSitA{$j@z+@6Vj59s;qpWB0CPP_~w z=n56etL(0Sp&ZPLIvtmvOWcd7;J|RxGnCL2>?~9z|GhEFmDu zG|$7|Wtf+H+L#|UnJioL=8-S1wOLoo(8mQn%Vw>d%*O4heLL;eyT%Yi`|kY6xYNT= z(402%(^GGuk&{^U;=!(Y;~OO#ca_1XlH%siims7w%+uI_(7}!m=#c^kvkQU1L<5YO z6V#qUS9@xFb?3-bHrNzS2v{BXxzD^X*jyce7UTWR?n4(*n7Dt#7+s}{wb9B<;uaDEeYaI^$jT4j|H zi5+XK5HeUz3^C}&I?8!r={i8Q<+U@<>_4*#Ke(zxe|SVB+#z8A5((SOlY=iGtH2@C z1|z%Zo5n8uS|am;>dVAJvARUQ&r!~dG5p`BNdYnKFq9Ay*J5$3vMno9?N zL3llD^@0>L=17mm6>u54dLcf=WF7aQiPpK5zy3(RZNU1E$l!`iDD2CTYaM6eOWOg4 z?al)~zunEOy-il7)3Z@fuKDBZC=<$y*ZJ07F9GO)eMnxDoA70Y6B?q0ZCmtA4=Y(= zi}T@B<`j0p)@;+8I81VebBcsueSZ|%-prYQt%>4E>@I%-K+3o~y)D@c7I2X0H1TcX zaFt49q-o{~VYefESDjG&-QS8xWk!=N+WuP~U9|x5QuF9~dI7Y7oaDqaihiqxRJGru zjDHlAJ$FWTRKIBGM)!d&;BPQyvI1^*a{T(=FNX=oTjn4GZntAt12-8=gCJl;$g{8N z;lpIxbNTSTDv_Es%hCL^$O=*CK+3*lGyZz6pz{FjEH|;!Of|RWP}|v>RtMig9I%r< z{PgnmiwbbQM|NrGO=GJB6z?4$?PHcc-aqk}m2lKg$i?W6mUyj8FIW`~$<_N0F>b#y~90UJAXLXA3Pc$g;kIDR_$qNZCeM+5KWoC!4Tpz53!Biw1p)p4^Bg%8|sIimG*9Tol`DaO<2WT>n#c9;DYbCxFOKRhA)2UR^+z z-se>v1&`&zdzKU`uAsBSomfQNZ)l37EGY7>*yK>}#wXgX+>ycBiB~XCk?Sx3sjING zME07IW>740uMK2*3N7Y}uKJu|oz5;dBI=rHefP|&k>3CsEQsbfr;)0gmU<(U6BsEL zY{1W=dlq+gq|3XNxm_UnRzMJWhjbU}9;6&c(Nxw6{Ab~HPh4AVqh2+iB!4ErtDJXW zM0ur^?#duX8t+_mN`yALZii3RJuHD5WkQSQ?{SVJy^IZDeh8Evm)9wSsDgbZ( z)mI%-3d@w1Y7)OIKQPi!0Y#ONhB0$XfpTkguC)`nqZGYOcou(}mE7Cfe*ROd3oMLGQ_Y^uaVa>X%c>!Ph5OVI>vSN!V;v{r!{#gCA zt*bIT>(^GF+*fqGpZez(Cih;d%bWiVW}SHSgWza2KapU8w=YwljFPpc_vm1*eSl~p zSIme(urg`m$CU?(2Pocv(gwE{^T5xSoJEN2zt3BU5{#w?g)GE>iYH1#c1Ef0&u+%e zIF=U`KeG28FNB~YLV;QS?mRyd`~GPUMhXUYn+0pg&Q~nWJrw6?(mXqIR#pgHAD~7` zw2Xx4YuuHSS#|x8qL<{xxLe-uU36K|@KLl-`s1q#9T@Ez45mpQOBMabRX^36NuB^# zAM?J@Nj+n91{$hutNovV67%8&2Ujn@6!{iu2l8`n3UFkt{GLmB1Cz3!${KI*VQqA} z;h$N1#;kShpGraUHXC>3d{OVP#H4_y6^Fs|dKNLv{bO7pn_1m7LRSV{ z2kL<*0LdY=TK>}3LaYWukN_LRuFp2+qdlh2B&_clEBF%#l0jQGpsg^_&C21=WG@j?>FbwZ$ILm`f)(_9C+t!I(FMA{9$&!Z&m(M# zMJ5KkkSE`Vq)t$^v<^)uWF-?wu{FcLnOX40Yuebh>E;Bl~u+4|Og>Per< zxBkd;bfCZX5mGouv-rK91(5}E6Df!SwP26?W2xa@Y;QHCShxb#jRE$EtQN#+|&Us2)uBabrB*SjB`=(1mHfU zc6?J{cJcUPK8o|V`B1-Ylbmd!iva<^$E~F6(Ro4D;Xf{{ZXNW_O-Pk)`sAZ9*PO5^bQI_2)%buIwTZH=wP8YLm+e_MM{8BrFW(G-n(?9 z1QjVF>hJo0p1bZ{>%O}8?pf!&IkV21&zjkLX3ux#zvch7i0`;q#6(2I{|eE6 zYeY|ps3|BYDaffQDJkz#Q{SayyGKV$OUDghVPFID3JD7E3h)by$~_VWNrU(W#MLCE z6_iv}RfQgEXg^hk%BiR-{m)N`sqfyUqow1zcaKZyzQBE@|IhYcHxUCR@ekr75@G?O zI}F4m48;HS6Y>6QCmHeoGNS*}i0_b)l95wTQc>UiSE1=1(H&wEk~^d%WMrhI|7ySb z_nnB8fsBz~LXDis*pWg2!7LepDWn9d*YyBQX1)nZIUysds99Lq*g1rRMec*Z(lWAg z@(PMiH8dewP;DJkGjj_|D{C8P7gslT4;b9fKOitDI3zSGIwm$QJ|QtREj=SMD?0~U zR9u29Ei1=Y)HgIXHMg|3wfFY*4-5_skBrXF%`Yr2Ew8L@@9ggF9~^!;I{tqC>bDJcml#s6>--wFEnkT8&v@k@|1su@!_BA5gu zBPf~GF@<$KR6r?{ZvZFc3^j|O^tRCV|3Uj-WdHBLBL9Dp{a;}JH`gi=4GHnT&Ld$U zdQ9}gtwCEHs=$pfEzx8mk*_31U{EHtz!Mu`cO<>2AY&2E%74E zwlmk|0P9^07dkFckIM!1%dA737WC{1vmGUevaHrq{qqrS@U-#JIc}L~l1%9Xm@R@U z5jB9OtlVb);~`N)!DZ|#HF|G78Ap*ZNWWOithSiu;!bpsAAyYDm}gW?$Cd2LAC>|9 zhV46FcOAvjn`#ar>qplvtEU z#J|9#L7LD@QvVE+Vsg{zDC1#q&$!v9v1(91l1Y$vW6BU0u(+D^v5gA zPE@b!-Y{NvK5%y*-(5(B5tlJfzbP=alk5-M*NpChR{cSh`+MD0?m>P1VCKSS)pZ-&e84a4bD8}1UN$5Kg3`$f$MNrOI zd}`tACc#&r0MSAr8T@?GY<9fiCf=gRhZCD2t|@xVO~JKO{6Z%M4N7_<@OTG5jWFWj zC^Lw}axNnPoQdgZ5DJ_is(l@!&$=-oF=$hYdfB9+^M%yRO9ku%mg6!KAZPP|n;jXq z-c(o(jm>5mUyyT|%3^ckm`5K1C)pwu4lwGw5s7+H!|y*>j3}D}ag015636zyQt9E7 z$OWd6#()yahTI>kk%1qEcjbSXt07B#=l>&Wq~L}}_E+_(Uc41IjSMbti8E2-GKYPR zYMh=X;9o+?p`|$u(~C}avV^v+`Hzm=U)1Y@TOC5kvClmwLn^dYIoa`YUrT9OKF(mK zj5o6;EVx4|9(Gf|9S3qvE7{G{NojX7naw&`vX@b?i%65b&7nYFgrk1UR*G#kYnOHS zkWesF6 zGQrzWlzDvuA76xdNwdpg#jLZSb^0~##+CxL;7GpRM!&7^nxS~`)}DItA;8jvqbN2k zN%f7Si0IT7oVm@_OHQB+#-fH+;4&gOwD{G}D672)O-vO>KRXUv{FMUHlG#Js$k9aO zPc{9qtcfKNnRge(@<^_EZQe>d-(My?uYos49pYy-GG&67B|i)cyw8foO{ZHYpHYV} z4}aVmfNDzX3cQYeuf^jA4&ve{FMs(Ju2=H>SvrRq?|qy=B$Ovp3M+V|p#lp4dFevu zf4gP(X2%O|C8qLd%07|D3lOx7FL=?QY7-}HZv;SuMj|ouvrtG$3Uw;O`3}1Hi*lkP z;2{WgpMm7nagL&yfvTga=4?b)9LBgw2QFLD8lU@eLHdoq*`wN9Oq?*VzTsZo1k-*f+* zxA7+Fq=uHNw19oq<^OGNwKTH0gM4MPhyW;*nl?9%}`J=}_&*Jhk zkfhm;16U{o4~QjW&gY!!B-B#BW6o4yYP3DP5f!?~FVG{4ofPdTOxkQl&pKMyRlyZN ztgeW#?4xnGDAm5_W67%q*+%BiJ$kkQQZ9YAAj(g)yASukq&qA8nyJ%L+xCVLYFI|M zM-mP06>O_!EKOaSK zhxe%+`EaWaGZS=%*yBJnP9*_azKT-;ONLGhbapbB_FZe5x21rL6z4N1&*?Rp?Vk|4oZop+QLvbnjI;3u<{P1K zf%?j&JNksBSzK)J4zIua=M;DiOhFnKsA|^y5SV})g9QGfTd2qioB==?GUD)6%$VtK z%fHOmH0bZUr~DOu+m%qaq84be^fI1mNN)i$Z=yw7M7ZF>4z}h$ImUckr?Zmvv#2H< z2KQs`^z#HR>Tqkd>ifMA6&EM-Sz*IJxHw2%?_`v$JA{4J$o%XPz<9RtSv1O)Wz{Re zixzdm=Tb~vdQoSTjvxJUSmF)2V}89AI!}w=5Mg^9KN0Z3u4JZ>)na>iD<7qr-!yq* zBf84lxH>$mvA%ozajy)`|AWzNSq97a20;r9GTG zF_Ed?m5^5*%&&BVWAl$5kNd%1TvwNAtF&`?J`M09(PnbEJ_gJFa8`~~%@BZFGK7=H zkK!M2EXX%|Y{f^ARp~1(Bkye2p z%@D=|NY;HBZ#GHPIeqn=`x_d6)jS5Fhv+7<#&a)AF>_2Su)Nx#gI>nqU>&kz9OIl1 zvf1sL-0f}uHV?d2?x#w5^ZDswyUn?`;(@;El~VcCx`O>kEsQ!kgUs%WjSo_x z*^Z;mrHW>OYb8V>U^c(@rrns)$SsCsog6B<@L@B;(BuBGztRH3Eu@c3zxaYzMZpjI zug7?r+(>{tz+|2hGmk;VN%@@cbWyn`f5Z_sfKW3OPlIAg?A0(9__%Mu-7^7Od#>ne z@kC#Eb=g9`mgT36_e$y9U*CV>zpo{S1mHBPMQYi4H=f6&g);;&@U!JfbHU5ScoMqc zua@kkRimb!esQ*V#K@+~(SgUNq&1PF`7@EEQE{*)L|eEveQ8am&{;y_tkK)`=-;$< zC%-qp9DjgW6b@dXy>MQ>^16TxYGbpZ=(d!T_n|&8Z z>LnG~CLN5LsE)WxZ+OsA^^etu+`_(dC4kQc^*_NN`@ zx+>Rmokk3wXrB~55U!dsZu%5^zT%b?VZmOi4UXMgA}KnawbBy^Qq|?r!Uc{n+jVj$oDDBf%SsOK^)7IpkfpI8jU4tw#0;oM9wB%^_2l7x-O3v z)AZnde^;dvA)Vbv7Gxz>llNt3ewnhw`e(&Nut{fL)6oTD+rMcwZfo8X3~M*}FlQkk zC&{}l&0eCsweSZM(aqe;bX!!>tIuBF857Fk_3VwK<^EdB`*M?4C4#WFgx_ROVXVx1 zRq-WKT6dv`)WyQB_AkuaJ6k2fd-;4Rssop<@ig9C=&G=Y%{J|A-FIn^GFkJ<^jydN z)v0`3ZI4X!-j!^HT0|=%HB=oWEN0kV76w;>?#gl60^DLfR|W8##8aVV2)Bypil_@f z=XZf`&v|vK1^J>-yyZ9~W^{}_;jJ$f!R9af`M(4r%@cFr+kI8>E9v8AG0)|5s51|a zDH2&DUvec3Kgus@*8SiIT|^|Nvbx1+U*OzjinPd9bUZak&EJNZtr<`DV@%iTR&>kH zPUY816-{2{sDMrfi`QSl5>LNPm*kTvjdd$XzmV@{d!KA-6q;Mn>#Eho`G<@&V%o%q zZ%z{4zrAU;hSmf(rix^02^4Didqo*YN~&I}z~EGcCE|iSAi6NDr4ah`P$zjj}I@d*Jk`fdsK$mO+%J11-6bA)FTL0UX9R7yQP$drSwk zU%>Y7b0V6nhToSxd83oyWegQyyzBEg$zx5O+d_U<@D;rCUO%C2>^?8rQ=f*RMlJ#~ zN8ZrRM0q+3j0sl?JxqKKmKK>v2VJmrSIn*hT|j9(>*j{48@5hs+Wb>O2|cA}Wbcye zIvn)0Pc_YDPtTxh0^)t%UIn8#?IOrH2K)q8){8pNOu- zHetot|L%o5WA~JwZj1h54BU+F53%Hyu;2KI;~E#tqMT`=wUzrd`R-L|b`3^w_fEB0 z^LE|#OZ+d>oW0ct#juLc6;m|Pq2zLAZ&jcKX z_jqj@v%Euf?7YYP9mM_gN2D3vA{*G@za8e>k!TL=@Zu7=>}jxS;|Xq*XSa#72*=89 zjm5MEXZ%Oh3GyYLvWYVEzVe(C9k6!LNrtnu=X9}K>^vw#UZ2?0$I5&L#pLz*zTSR3 zS}&;oJvX4jheOVqb)RpKS4XfWdST{-0m`mM(WKlq1c=z)aD7Br;JstU=9vHSvSU)w z44799bwetY7ISaS;YLg`#)I_C5=C2ni4bRKVcl$GvYMd55UhM#@UabM<^%k`@ms83 ziH+NCHRK%3ALPJHS6Y;5ZhhwkXEJH&mX%LSTc#eI3n=uy(=XtpNGD%OUv96Pl4Gu#D9RgZXj-SKY0bE9WpgM+*jU5j zWa?>d+=PtRFUuS^S_4E{edvFhy`Zk^b75LH7mcq`?_cJ0{nsvTmt90soRhrt*Sd(F z2d4=AV3#LSsVbW0qZ)p~ySenC)%W=kD2_Vdmc!ownmFGbTz$1O<8K9JRFw6%Cxe$X zFYonrdE1w}`Yc~oRhs`tl#uTDob&^)4O1(w?kICDbA_0Tz2+N4$+e5CbtSPFHvcMeK~kEmnYp(d+V?N#t;yjMZAQjELlM$ zIj}Fs8T`EbNQYxjr~V-=lQov6C0)s+YHr}1N<0Fc!#F&)ks;nhdI_A4bXolJe>Rzd=;<+U_DZ zWKm2|a~(oVg8gCd*J~N-Jy$ww#+&jVQLx1f!lg~tp(t728K)&K44Hz@HO`gF5wo-L z;^1pPA+UY-OilAikb>y3C|5bM>lixhZamMiy!cz;A-1O&;{SLcaVUZPIcBy%pMurB zT=r=$A`z*;O~qW5JZYBPTM{dRRIl-sgvl-*ROGr`hz&ei6*%AH6~C=L(%DLvXys3L zF0-*UKN&i*C`l%OD-wfhpPm?1tgID&nIfCg#r}Ffw=hd*$$n&^rDuUoE_}Y;KP%=^ z$ks>QBoh0=`1=R6)Pz3-vHacrt`s{`UZGqtS!`5`1Uuh4=ff|Uu1wdYpAo^CmFtHw z*t5>4I&=NdVZAeHb}ls^%5m`r{EJ&>LhZ(odp54itpou%+ ziCZkywapo2UEqljpe@A@K6x3oP?D|B=D){)ttt^cQ}#3VcPpAHil>eQu|FwNU~MP+-FscxFu5q%&>q{}eKFx@6SKlJJS!WN zOCVS)*spMl;})`$L*DrLFG)0jfFwNWb^88j<)O`?+x4BY5bl)d(WtM9E{;XYL{$H# zL1gvBlGC1?E9c>0*cs_g&4WF|$YHC{;{S-AnwgtVXL@gDmkG4r2j#lsRE@FPLhpNi zSdkn{C1$Iq;XSYlk2mCH-W>vSp*g#pF_Id6FW(dXp6Bl1*2O$XR64jQRzoY_Jk?oZ z9DJW{X*%cI^LyZ_ckpo`57P4egQ-Jo-dJ*H4o^9fmQu{ZkGb!RZEgA-EZm5c`apbw z{lKU-43YnPv^6KxoIxxS_}Cay=*RQxK}DZJRUU}pa+o#k2a4FH<+R6E1ZVnfqqU%) zK|=IAbpxK+o7FpIU>rlTie2E_hA5iyG~4nZ3=ZGp32zqWSzq3-WO;+kGXAXY#=NP5 zKg9aLmeYEYgh635cGs#ANkHk`^O#&u||gFX4nMXV^WtN9d)rcw1V4V~bdJMG6qtO`4(7B?|Y*e#pkP zTkC9{1{}1|QXk7DNn`>ans2YIID406{JO45J?nkUU?b)1*{}LE@yA=+o2oNF@d)I^ zJ|in$|5N%3Nsy7!tamlk(=<1KG>mhXscc+P0#NLDcVm*Fp(%vVVz71}9&QgIa!?o$ zMzaXI0r5jR@2Y!uC(>EJ2n5g98o(pI3Fl_L<+l?JFtyG6@TP=dh_5`iZEST2HASXc z=rK&J|CH0taM6qTjN0-aFyf(}%cOFQ8TC;R3*DXVATDd5-RwHf$9ZMO%erKqnBX4N zsJb=Fe^CZ_(p1=dCE`KY`oa{ws}eEl_2PepnOFQ6a{^t1e}4%PN^|Zl0CCN}oWLs1 z!6$8)9{gH{d*0<^cuZQWX5%;YLMpsK0*;f#K9aslM=3iC)bV$E!erY7YvHf}h(7ju zai(83yo#eiWM(_-z@>+TojLN)nLBAd@o=?=t7THekW8%-^c{QVCU=v+!zfaoym9;6 zp9&3jXFED=V3%HrdIiz2jMFRu>XEQ_L0-6!#tX_qK3maMr!wD)u=@IXpF?)U2`RDK zo<`m1RgG@SZMV~vQk{O?Sd(Aq;$q9KbGGjkp6lG@^Y-YXoMT~k{G7g%R^QW)C0mV> z6PPbbZ#MsN;mxNMqTLg-0@;Ys%;DHuBMfBe)dc!=)7wuId8mS5I*HZNt{ki$fsOkQ ztogXE3cc*3mlkjfL@mj)0N$ z8^8QztgC0_55Mi146do3>;;*C1K9npO;Gn3y23cpt`i0gJ{z0wCXbV4)DcV|f_mj5 zFAnw8(}3bPlkib?fV>~p#Q?pnlksAD;V?S8)y86;Q-NTi;p!;(Bt;QFpt*Gtytk+q zUmf$bhhTCdp+v)@g0RV_YqVO0rN3;YruyTl>8DtIKMx}QPn0n=Zqb?Ey1)5q#&qRd z24ML{Q{Vk>Ei<#h;JXQh7%6GSU=<;XY7(47*!IsE$o&B#=~;$>$6Ryz*0i(~bw_G! zT>dyGi9`n{P-XTAu9#K+$VB5@loqveCBp+7)%BG$d|&3Qw30U!mv2ggRDSotchpel z6wuqN(0yV0%zgVy&EHiZamMqSu%(jKHPVP6#jj^l)+$3`X+nLx83Mm;ic@rk4pvoU zt3|hX?sC0mW~bYHVtTY#`e(F`0loFRNWZAE(?_z`7kZKphoN_WYy5$~3rA_YG21Cv zrRos+TD^iqcySuDir=^c^LLvBXGDL0Pt`)`p}evj>HA_9IiLTVz%n{Nu%qk;6hm_q7K5V(qDl&@^${`Uf zmfzzDm47uWY(AtE#$ibUs_JE=8dLzh(btpL)*35F#|n{auD?fJ$%siXd71xo-7{LY zT1QKMXkCj~O5~#f(!_CH6n#E8UqT3I9G;j`atIS~x$G25>OYVZyZtf3wHWWB*0Nb& zD9j#MgRIn+*$p{v+ry5UcxsqHI$J=DE-5Oa^j57j%#X&c&|q$ZFGY^;^4AwVH{DvjZFgiUkV@51S36P(KoP z&#XrL&kRMH{97ce0?&%m?DF~}*bCZZjGhb)A@)bfLl@I6Z@Hf+^Axr9NQ6XO$l$33 zhO3ZI>GCs7(Bb}>S)w9zp{9BLK|D@$8Bs69FzXB>rZ%;r?$n`)2Mq>;v2m-mG~-T|bVi1E;9iS7O~`}{H!0i@$$NfT)n>ex0#1N!MCPh|Hu6xo zSugdjZa^xZRo#?+Q4_P4r!>p;ET=D;Yib~OlM5b&uO zQi;H)Q*=X=5{?q@^cBZKrzYuCEV1>;EQBKc>MtI#i`%ZG&@qAq&q(uG3YB3wn>1OD z+lEs>5A_>rFSGzD$nCk_Ma5?~ht{NF;Clf~ar&dlA)W)Qnlr1FAM%9p@L?#OXxX>R z4qNN0*O7H%)%pV`le?m8)9UCZV_X}7IZd&`>C#UwFWf>#G)#W7MEhQOvYA~c1cFu!ee-^3a+ zOSxeUWwhBwUxnvl4Svtg-eS8@)rj||OZ*WOygGLd)4EH~h%ROdoXB@Om8THAR5_Wb??TucL_bU&@ll?3rjLQhuexNZ)Lit@JQ{*f%>e4 zf3B4R@0qwV^mvu0)UgTr<3%_lF)0;(rsBO1-$cOg zac{fIC`B0T5N1tVloKme<{0ZqCTv-RM$=9v40-Z6eS3>6^gITT*Mcm9sc*ndJh=sr zvi&At@7OT9#3)nxVIt#vPpdIqfoFRoFy56br^5UEME6fDifB$^Cc#H@q#y36O-dP8 z20E{By{>@LHmE34AGjuJ)2(wI%B&nWzat^zrc)(3j4l@&E#Gg+iQya<$`+bZPA zG`YO&*ew$}VLelJ$c0LKBDtzpjR0mkRsK(=&&8i$a#91qi}s;Tw~RM$%#|vtH}{ zkLu(ro8tkoIz);H{be1OGm*B-N{esuocq068}TO>?_Ivsb^>maR8}y}9p`VN14h||+L0)w z^1l0Ck&MaR$XE$o9Xf4?;|F&nmoF<3m#o|!`0{pwoo}|?Teaw&sq-xI!=JrGADGmW5rSrw zs@pZqcO7EffHFw_^gal3iyBlkjoi)rjYn3_RZG|TG$ziF$Oe-ndMk!YdMv+{l_vS; z*o57-Z#NXFHCnT+Y2l*HKSKmYmjY9f>`(bG1d8C_L82VkfYU3({pd*pax#C5>u7H} zLfSk>MKkP{#Sh>Ds^@fO?BEJDaT>3wgRcYOaYK`4V4m#UJx++cbMu* zdSzPjcb8UKX!+<4qK;{g?zM~0Zj)JXB$Uxarv)vJf#QXF_2Y=WB|FelEJ0d z>2q=l6G}3XJn3_n5i<~qNJ0JLFlJH~;^d4Gjc_Bos|fW4gjdq2Poc#Pz(seyMSrox z*=UNf`YP37w8Mvt8e&v9BTV1Dc=)R0H3Y!UE0Cx7+CpL$%9vZtz~X4FQYsqa6cVyx~tfNFk1a?NFpO0P#!k@-x_I+#U={n2MqZ%7VbhjnHa?icm?<9#A z+%PF`Va1)!)dN{rwtFi4=!!Xs(;7jn)HYzMSe3*dM)zUJY)^D`kegpQ2a(A5qXpSU^YkoYxaNsl@2bzGI|G6nAe1 zI|;thpyv+Z%f$$RCiNlJO|njhOiDbYI&ZY8XxR{1l#$}Kn^=qNRp$4h&}nW)0^L(5 zt~J$s@k|5^a%icEpV|3^q$6hFML#f3|L#KklxkDj9oo}wmq+r|lz_vSAUXDt_g?w4 zQu{ZxOCQRVye+(59!8j?OAOva8tUZBLx^7@Mua2S8T*H0t!)KSG*`dOn!m+HsPhxI z*B!%uFZTN0C2mosem$=NWAnBeq6Q^Na!0N}zRB^ntAIt2lR=D#Ar;Wok>KePW9rA8 zQ*cAdIV&xupxXo#_{jEUmF!_+_9&BI7AbmxwjsO|3C|;RCyc+Nl97`(cFAfvy0UOl z5J`L!TeQ}$-YRC`$!#}w-#eQIf%=Z!sQY{;{{oi{X&9Ea%Qw*-(hIx zjX5GX6{8^Bv5DmjEw$B>{gwhFt@i^R*eUrYC)&oYV&pS|NYVIR5iYl^EYJdSbo z1TLIDDD!{?M^|2YKN+4$t@Oh@%Q*JW`7&gRO(ymoyYS#K z%dwdxZE0n9ngFBZacE5lpwPE&jBZGbjZVUxfSGT;bbD^Bo?u|Zj7tu&(BhBS+AS!o zygK3@J#`ZmXJ@~xs2}{U@FpisSyu5Cyw5w$bEprkHX$WzoBCLGU9jN9$f3X{V%yn^ z!uacAZMSCc%j2gJv3M*uhTyXPu?QED<=UBtlHDz=_DJ#W`~KD3W+gK2ZK?vmi6!GI zvJ6-a^|6*6Kl-R@yINNN^#1bbQ+k^g7|S%Byh_E+8lQK!!<^2J+sn*+yD>iQU`G?O z<=Z#Wi%}46N+5PBSAs0`-)X)P>pSyhxatK_o@qEYp2-^UXREt{7NU!HVQ(Q!3 zEU!)%wZFdU%YRaNnr~B4%VpkLY|67EVLTcYEPcq*q{d>K;CGMvy8)zI02}9d{}Rux zUN;rWF}tgIcV4o0j9~ZRPkf@q@S#Gjn*Sz>4B1}Y@Nz0&t&GY_5Pi4bR0$m3Is{;i zJ4KH0pN4A}Eq3@YTy+CDaQe3; zidp>N-4vL)VZO}euTX=X)OaNhWplYKPpxO1BlwyjPsAk zU}xl3X;9F^#Q2fACZ~(mXKFWfF+>)$3ymK-#vTh^0P48MJwXp4mTgrhw^L0GWP5?t z#zIO?)`m{&ZcSp%C1ez0p%y`_4v|FF*Kvo&xZ9|7hxfB2@1H!SJrph@&dx8n0_2fR zFIUmoehgiV>1}_~px?^J3)AsWwbW z>c(?E|FZC5u3LQOMJY^CBL`N}`^aI#(YZ-OJOIdC-W&1n`nqyH+etLBX z>=J&^gUk7Sc`nMI`2b<8bZaEgwEOxLVY0%|fV1d8)Z^;kq-GgsvCW$5<Y9n4Oq zm?S5Acm*W=u79GN597XaFhgtdhM_OW`R0~I>F8*M$50PzhF5ehOAi4LcRq{0_D`ZB zQNc$gn~5{6=D+QzwJ{(gOA=)@8z@xiEePca;c=FTmNq9?b52b2Ba_;S6q4=!^XxAgjHCK|D2=%lmscF1b!?hrts;yS!^&T(qHR)sF7ci z?NV=L&KJfE#nk)bjpOB|E0CalHLgU@VX}2LbER~O18z;*wd9To z&HH4A)pkwsnFnxhcfKHk`I(5v+8Bwvj0d=?jHH^s`LjP8D$4F+H_$G-&VWE&BMzIi z-qU+6<9@o0G$0f7xXV(FV-y%Fe;{yc9%p0#!#tE+#)nqxSww|pB$L*NKK#2Z??7R* zwDyC^tTtO974Bs|Dc5gd+qj>km}V#2gu9_Egix^ycZpbHrF|ff^`%GUC6>)G8Lu6? zYs#S`xqKN#e>RU#e?RuUwr~>rU5plu>w?5?VFnuQcLf4HH5$9g;6ujbG~w8h;W<8`{`)!g8odVXNk!^_QU-^NE38{k9puv&Jf*)BK2Ut1`XbVufuL zf(0o@_ctTKF>s*O`0`FOWANyo@w&@{q8*TGzI&^bo^|@G1r;*A#an^(2xCk9vw{SC zuARXxaFrs^dG1^JJDndvoKa$%xxb6QfNLak+e^eB{hHVd1SG%NVJFoqr%t~171cQ( z0U|MjUl8*t6i0T1vm9V%(7SzZ^&h|Hpg~VBv;{l5)_MKvzNSUkpIWRrnV*s&UNbd3 zZ>gKQJI(Pm_D+P2rqw~UXa}>81COqQS>U~c>h=GK%HeJWi&fpZYsorEgdrKrNj~{fviXL5)s6)9y9*jo0x7DM z?(MbHe$^q@C!=K>Al+EU0JKDEo1isMi4$Er4W%RQg&S7;ta^AwO}sn1HB+H27ILcc zJ$oC-(R49<5Eo6$^CsRXf$;-%(g4gFI9@iKjNt7ZEbO9uVd|-$$;6-={vVOuw>RLq zmimUk9}GRu`xjn;=%(v!a4&??J$jbbxhE3gT^0*_mG5D8)~D?!X~ax#H&^3V#kmD+6;Egm1Ohj*E)q~e$Cs*X9s zckeWK5q#O7zgPIR7~85kqZp%HY~j7o(-Q4?qiiwa;fC0}A5E)4`i$PuN@o_9*eIT*XE{YegLya^`yg)(FMa;-+`Z}A^d z_`=C!#;1x=2OyQ6oyOgdEjdU8R!2Lb6#Y;);mlmBr)ct4R@?}ATgO|N@^Jc(+2+K$DQoL>-4x}_0V%^t z*tN*DzVXozw$#akBdOgi+LRMVn*QkH*01?yUS#1%d9_RYsp>wrj+l&O3Bq`58hH3a z_-`Hz6@3nC6V>5W@l>+9Tu)mkUJZHQ3>yzeCHnLo8T9{=8 z&KqbddpSle7K?0#8w8iN-Bq{mr>!CfPn5os8IKxE$n%>7Eph~{aX+0br>`}(Kg%f6 zd_jGKoB(y-_T4Em*^aLIHy1i5zJu7lP!4!;gAB)FhTty)IVm?pwoS)H z>FegM%iZOHwK5eoDd8P`W(;0t%d8{|_@Ob)5m9U$|LT7B9>mKTQ3(P=B;hDOKHve# zYApn?1mw-oYOu9ygg_@Ug|hg#cX*GnGlmmf9yJ#(&I*TGhy zxijYXhF|@a3;e!zb6IwXuI%{$0c~%QE;it5gXq zvNI9NmAhnI{$&xfHiU3FofJ15Lzx@Ai>>2w!4+#&2;zOiBWaZ%$^#DbIlqfdLz|c0 z1icTvxC={lwJ=T5yoXJ}u`((+1?E`=RuVb*nR-!RlOmY++7l$C42&c6jIn8a)X_tZ zCBE>aHCB`28t}EvRBlF2#6@Wfx$kPtUWcIn#)RLm$}yT_>aQk2mb&;gZK2N-*SZ$= zgxs{k8^3)7cHWJ*7myGS^H8+Lx{cPt8aeFSn)#8xM2K&z~X z6e?$yr3-g$}GeV<)kT?KMe$lp~RzEG?n!SXredRuz0% zQ|F9vH02-{3|*9qcUZ$~7=P&Lu+%TI9h0eVxC1kVRpv3SNH(&vGPF#pGkmqVT8+J2 z-Qyj6n2pz%l*fvzl5M%g&UvlPnDYr1i7((V zc0KsL!ETFpAN&UwDuT$a2?8}5KEKv7wvhhX z3Gp;LL+U&f&N=!gk%dSPm8z-Vp3zt9JN~VC|LHdB#B`Z{V`knpvvkWe+Z2}i@_PZ? zKKh5=h5-$nUSB(a2Yv5{p}Xa6F@Q=SP0*yyZjo!l!?cGvd~d!E#rT8*y_46UmNuQ_ zjC^hG{L|uZ98PE~xe-#C|89hIQA2agwbYelvpt!96M_cpyFWg2KUct~V{HH<-q(t6 z>1mEI0;vG)Lv&HQEv@XR3^_>qF@bAQ&yS$#P8+q-K3`dG)30V?ob2S8igc=*6R7f> zmVul@h7Q!}n_NmuCqH%Ap!xdR#>7r_S^Le1zAy{N4|1UdqFd^4QH3etLT%FOq>fjWIy20UpG=KDtM8T; z3$&2Me%oBPsP6Q_qXTDUW{V;kxvmFPMTb`+Qxq<$i`jBU^)pRY1nroxWp5mt=!j+T zluwgd@rf_kTqk^rstZ}?&pX!Knirj&qJ88K+HZq1w*Fn+!Yk_+y{h;;oEM~gz*05wD0n>;YF22}wEs-5In3Lr&QA0~<}`_@N6@M51#>VgAUn&G zX!H%Es3O2j2+y*3NW3)xs=eeFdwbishrNI`?u$0jk1Cn1lE--Ctg9qS4~EA37Mo0_ z9;AwcENk{31tj?(S+UQCSM|Jsxr<*TWeS=G7bjPO(W*t2!8JRgp2A7?K@7uyiQkHC zezogllXHuStUd=L&lK~Ai$$?$&4ZHoeKJ{_ubD%N>*0Bg;-?xqL{<4)0k?y&n?2 zgi01>Miowxr`g`eTG7V}`fB$c)jywgkyN%?55CGR$Y@knL>HTb#13wxR1f8k>(@Qyr_#q{Ak?VnRWn?8my zIuo0Zkxx2+W}=f|cal-JbygBQesN8NeX82&XXKUqj_ijzn4uB9&VcsB3aPVmerKa6 z!3Zij%GnguF0D;g^l-YBYe}@kdc8B^y1wvaAV5MoaEA7bh@>mjC>abzW?VcRllG9p zSYOM1fjrq=0+^hbMl6Iod`5%A+k07;p@cL`XP^<8^`pSR%+X4ij{(JJM?2cm_YV3W zY-&!CgvEAlUh|H;Xk8i~G$$A7n@uWk*otFN|ol0S$_= zx3ad)2f`VIW%@kCY3EKIEC;|sc(4xku?rND3FS7XBaJY-a-b7cJeb}ErMwyhmw0{G zA#D>ik}79)NioWtQZG$Cw(#%MZv5re;9cV!&@2Co+P|h87rO&n&XlCG7Xm-OYjc&w zZwJUNaPrLU$yYy4vdN^a6CazxmkqbvZFm;;fzT3c!CS2l(x<`40$uoDs2%=GK2oc~&ejGKcA@ zbm-yR1L2cb9%6#P&Hq-m^B2vD53ck^UlH%LfgNpw-g3```5V_dh${X|nj#lvDUWR$ zTt6%bQCEvHKMo_o>HkMWIq+%4dh&(~wVkBlJ9lidf9$IYS~%=uX=`X`4~k=y+NqCGW|66T&j9h^pf-{(qNNS5t}3`kypS;b=cgi^aD5NGh^S_({Dn~FqT@vxc*6(*RtThJFJ$LLOU5r_^Tidu9&oK$PQ-7H!&qmhEn{gvEpCfj7fT6*q zTq@J+d=ZPbiHT@=lfKi|kq({rdMtiotICrO)sY}_}ZH>)_!v`b4MSA z^XDUct0;F>JRcrxZwr!STvdz#$7Rb&ud2YasO$A074o%#ze6#^2MU|Qz+&oyV9kZZFY|2+GIv{tV`cvj0I5J$zljeS2b!q` z&9$uK%#GxexiwCoDZGWp{Pl(gB{;nk6bhAoJIXo#bO5?)n;PA`9e< zqWtThV^xyjU`^Xtu=F(ak9+Q9!5#Y2NwtYmtC8FadR$_`sRw>g1MAYYV7Lwj0sKXC zlcdPmAcY^rik8~(k%nX=w^~wNhK`h~I3ZAH*wrZ`k~b%4+xpX?z6~A-Imd4GQuW=? zvJy)22YPWOtD6uwbPNu7=h~IJR$h^M9ZJ96_v$NAijkbw>6}tlWee^QeQJ9TxmEAT z*f4w4_6!7lXzC)&e z`(v5`{_|7SBb`7uE4#ipt~S}NtOJ0!YVyx3r|%tDdkl)oQDr7oiZ~(%AcpKfq;T(( zyN=auZQ_{YaZL5aD@P5e$sS{W2q&6J_XlFVwTzoL$mLY@1F5TDXB%>#Asx*;Tu4Uq z9uHyZNV2nrEwrCQS|&>@SX{cWSA2o(S7f`8wj*QtRy#;xT!0re(|*B}p(OXk4^bLZ zUfrh~S(n^XZP?2wX%wpW!KhODXHavM^c5w&)wh|o!zupo6eKfY5gB%*gsJRmZ=yYM z_!^}v1+H0*-Oe2VUqMp3CeOMmymv9SHjzcu%EbvK)p+VnQkH^f;(TIUNL-d6bWnJ% zEhUy&R1k1aLtPJu^+bDyjUUZy@<%~f2{46&V4ibJp=q5ZyoS(4v2rn!n%lA0QfF9J zJ-F&ka+f-&dsHem9Dr+H)vTRijU9J33pRV!R<@w^JBvvfmnQ%on5%)SKFJr$Kp!|2 zd!VX|5XqFf5Dztzb7~qk$oYpkt#UZ3*NM6Y+tiAEZU+f_C9KyaBqg@?0=a#1X%&2^ z#@Mr-Yowag!+h>~f^rRGYjRu2tnw8K1~HoGiy`RFXj(xp&*lP1{c4;yPh~FD;|)`0 zHJq$28A+tSiV#%Ber~3;-*85U+y2l1JQ~u{;1bF5$+3rH-nnacWp(de4wb1}M8Hg> z9PmXDbNKHx+e6ClT2jEc8uYXis{@F8L7xTMo>VY-J1>=*sxigfnV`HIn8Azx0l zmD>#ssAkinK#-z44vJ05z?ANu^h{A`(z+(9e}G5JF*1ApI|CA^N^70 z#C>YYGKB@oQkw3k&rUynvZ_$a2OBa zH9RF#?qH$*A4<|L<~0#U?pbyZT9VR9!eg46_ZtTTtxsdO%jhZFQx?dYN&O8o&A9Y6 zpxDPjOykn1M}&o^0g7-KpyxbSHST>nibHTS%_FYi#VO{R&x)mc=x07#YCyvVko2V| zCV&azw@*XNn%6-WcWQb&qvRYPT69Z;(xh8?C2x^3PrfK3<%k_xRWdPPys;;f_}1>8 z@=Uv-vIwX6i!o-Zx+aYz^X+LOP!3pwyC0Qy_Q_>s2qm6I1BPL>f6wPw&BYd*Gwm#{ zaIG?=3F!X-GatgVl1U>_klU6Y-Ybfd9ahkvk|mM*o?G>+&|XCA#j>dNB-cAAqaDQM ziFC;PL9Gj7k`3E(-|cp%Jkc=9Ku@?FR3B<&Bo!(>D>jNO$c4|$Mmw5Ri3dHZD6Ub+ zXKp(#D#A#u^8u0SdsQJaIiZR06#jJ@Tie7khb`)-BCK4D#&PoT+ml$bJ0tFjT(CXG zK4yhuN!UcY_zmBTW~GAm0GpZ;828;&t*&HpJnT5>%GRWI&^m^}2h(n7Y*w)twT|t> zs`2hPtB~5oZZjjU-uS09Y>?qNA45>a$clHwUp?8dDK;``HC;L+DtZ3^b$P0``lK^C z7QvK`o3T|c&94CljmupEOF`rRfPOB10Zp`zh>||i_YtcMi6pd)gxd8Pw42GxVk{i~2UFYi!t)dISB_ z$K_oo_Jngn%M6IXjyV~wMKtDQ%8Z{tXHr(;??MydtI^-r7)K+`kp<*4-pZ1oLNo0{R7*Ypd zK>l^Ab|%D(=_KSv*aDpLM!>F6H@K|x46--N8>T9CSwwg%laJ!-LU#s)c^~fMX!Tm7 zacv}n@LAcEAt+;VKk{10CK0jXj#|*aKI8e z*F8*JDWYW#Z45i3BLFsM6*iK6_TFSmB)9n81fT)QW_RuS(Xlu#Wiv&d2P1YIo2sQ-wg|xE<;o)$9o* zjN5xT9IgQ>P6-{Wa@#|UC}}rHJrv_U)h?B#$pnRfKIq1}Xd;gTE490TIIQJS-pI~T zMisPIOE8HL!5>hFI{i&=T4=Wy>^DZl{{VRPtBE2+!F;eVKK3(L;lGn0ESo^|8LZrp zi#8q{wYE@X;k)CKYf)^iQGa&fmpt7p=MXpxK zE*;g~+irLnuA0%sx?rK+-80T9p)yODh=is<fF`1wQ1Mol|z04ur2Od+a6;E!R{+T?_+fH6;+Nq3Vf(iG}`V-$k_<` z<25JSBu7}tUEYc-4&MD#pE6cO{^+e+EBgjUeafx<;(4pJh`5*c_eiH~gfDeH>ZBK| zB9@k4D?QZU)>gZGOR=BY$SRXpq%tTHiE)rsb)U0jHWD=NRL zkyWjxf=h57LapnXqkDVSIKVjmDmg%(S-!P4TacAR}g zFfo`9Y!-hEar82g}guRd0DF~J}=TIn<^sS-Ab zV1w({wQF0F%`54xQbJ(L`j!=((w1%N**?`J>?M@A$UXB_A$9{9Qfb(eO4j~JXI7CH zZ_^c_dvy)Xxnw8_9SIo~jObYbVnN3hMcw0lqn!7qmte6w%NjmPFkpI$*we0VIB4)x z`gN{yQsAp@4fL&j66MjrV};d1)j zm5?ijW9wE$$`kiW=hCs1v|lM>pZ1O^QtB}HmPSGjGn~@pur}GDdhpJiWH9bHH4U=M z9vK`+zJziryAod_MP)Sb>l+grVZWUw?i&{xc;#Y}Cc=HGu_MDE+aTI|gG4jKI{9*_ z5)S9RJ)kH719IoLQAd!CYio%xRiAqI1m>oSXrs7S&cb~SXG0SOS5`S5hOP(%z!e}L zdWR)zF{Inj=^%aT58em8SVbGM2Q`(ZnPW|j$Q`pc2~h+5*|qjS!9t-rRnttlp*hMH&3 zjO2=RV3lw8X{Z6~Q^vS%dJ&5CHYF!5TNsg+>DH`5jIZ-A=O66YOPB<&^Y2$JVvtn;ew{dsj5I)Q56) zwnNDPAMaG2PI(xnr13w>q^o~)&^*L!0g!4e#d6AoDl$JBav%hh0UNObphD=wvHt0) zi@5uP2hjAXcR)osnG}K<{-=t~mMJFXoUVAm=qmNZ@*vs>CAui8200zh<~$yLzLinC z+!;`yjhQ1qG2p*?v2CX#GRXe`J0DuNZ(tN}a0?Uifm>EKDx^G}K;U{+y%wU}tz}?f zUot=$yAHynDI>Wg8q#uzr*;$yjyYpsNKQLeT+QlVDOI6upDb(#u4^vt%OtZYBhxjv zG|eLg2Z7j9NfMO#s&CY!bJ6ZY_X4ldpE^qFHVo#ponS`5k8^v_^2V^_fT!tJq!uxT zlxp3ud#l9G^^sNYHNwJ>hU{#o@>61%&rX(?d8Cm;*NY@v!y2ZGK z2hyOO?w(>J$z#^3UgVLplt7yR9<^jaZzaCuVjOh_tH%t|OR@3|Ow0B%V>o>J8h6wT z*&M{%T=7?YKO-ip8aV>4dFK@b_EOz9?&Bn%N`dtI-8`$vj0Fv7Qr!1VHtibG3lKQ< zs||Y?Y;La+hsPI|jIpsL7mQZ^mE)*yrdVHcvFV!Ulw`UZ#YMeY>89ZruBw_mLt z1Xq&8!x?sbaniBmQ@r5hdWzGzONT7G$o~Lj)E3%vvX&(7Jt?gKg4)fdlneu!-?h@7 z(RYE=XQek!wieI6MZpzj>sXlv$kka-T-97RAe#P2*sCYY(>*IL@<)~TLnv1C%}Xu( zj^sRfQ`OmaRdJe^@Ue@L^Ha6^W&q~RX1TM(I_jWB>ma;`L8(;hl~-P7*ul& zBMIV~0hM++>7GSfw3=B6-NSVH)t8IQ&e9!EZsMO6up@e@A5N55S1crQ3^Sm|wP-_^ zRn9=^xKk~VJYYyT&tF=$mdzUwK6GxFtmQ6I4vaJ`dBAQn*!^nFq=W4RdsK5Hc|~)a zbUo_5jmb9w7=lG{)U2;&ZwU1|h>j)O0m%WC%GmpA(IM^sL=iRW`~%rxf1B%F9-0 zYC57qgaR{Mjr@|{Ib)rtwMva8>+g`Ay@gnHR7_(n+P8{PGH;vErKH)ST<>lhHPd-( z>MM=Ap615wbI$^jL<= zV%kl^G4DR8d8yIlgzR-@Ud4t8r@?yaySAqt0}i!w?dc-?@da+YaNMY=p!uK++g$ob5>-zSXA->9feSu`Wtm5=XXFe z{VKe2dGZ6a0(m5pO|D=?_mS=!8N!Z89qQ~5Vm!BArm4XbJnF0WbDjrU(su{~AIh?t zwurcp63doF01x6KnI!m9bGYNJB+l}_7=?{}JJb_8;bTP{@z&aUR;y} zGp|1Ndhh*GqQ3{zy=6U%O5otKj@7hcB+*%JVFL=SD--w%w2VxAs9rjrwG+qX9spJP zQ%GhY@&V-5i+914d{wO1Zvu;;$WE{8mM@p}28Mk$2c4j>nG{s`Y_L#Cr z3`#*h^&9DC6m7^ zqbiGF7N<)z7XgrwgVgk{lEUQ*gcv8*mo>qo%k}>N0Lp{Riavc5S0g5ed8gbYg&9h> zrF|A-NBimxbN(RE?QWoF2m=9;fm%jQt1+god3L9&yICx3aNu)T@n1~wG-1vN^{ZO) zs$9H%c5%|OVa_w9{4PT%0a?;;cmjN~##(dRD4^o(p}Y zK5oQQa=yguZfSZ=o7#~ain6HYr;X&a9$O9H?ROQ-rPH7<4A?!!cr`NWS0P3V1|31< z8kC7OFD{+uRgs3%(Uz=8>kJC*A$9c~tC_f(<=AdfhUi6E`#txOgx*FvcBaQ;eBJ5p zitfsN`_Q_EoWO;^$6Dk>b8agpQN2jwuGrfgptM% zagaynS|4SS+YT7CZPC={6)swqI9eUsM2B~wp`d(~bO4T-6=5HE0oY{y);;~G`#?|K zTp=s`EINu>u+U;!#@rSF^~P!za)qO3ZOhI`=9_XDny(^8>?Mql!`F;cwmLX|p!1R1s@e$Y?NF-~ zINIF}UPRaz)OV^{dztN*ZZ|Jl(gVzcaN(1JE6%SzQKUn1L0t4V|TQex*kyMT3meYWF6{wd2e3kzIBC}1R&OXxFqQOR2Q`@1f1ahi!qOsBBKmjaIZYu0^u>h9( z=7)ACV)$s?axx7DHGaG=@~WO_>JHLr%RkIF5;4KWG>q9D$DMA(kC{K&9MUT@g3-F4 zP(P@vit5T0@KpDz_ckky@i7F|Bu&TL$^qLV6v_4CuO-Mt*|}YN1{~IWQcONsP%@6B zjN-1PyiB{m+C7gIGO`*>dtXFO(GgbYFmYAnN0E+QaC#BWDdO5wmx*I_2iKZ~?xjZ9 z&@5-vHbq6pZsLnHU9w8ba6UxNbM9(sZzGC7l!XIQZg?uN;cIbobWv= zi{3Jv7yaX$0qIY=NdoR(Hy&8_tf!q3%SL?1=nrhwMkxWiY~pziIxowgT9R3j35^aj z)QZJ|>PUZfScY-AkD%-;S{pzeS;=M~W89h=1TWYc>uWmgQh2AmtWmDSOp(CHTGnXD zRYt^WtI&0*BvpzwegHi))}bDo$obZ*d{RG0!!Vr&@q~%!UB; ztD453*DwT8f!`HE!fz}UY>#@;>@;6@P1p;JR5ByVFr0x+b{okjuS%L1P&a~i4!-oZ z3lYJm+)fyvU`I7#0?-hb4pn**MrzHp(w81yGEEn{e6xmTBoWRkkeK(mw2f5&Sr@KH zTHDg~Z8;EmlBmGrBDl!xWb*=?j2?!roj9g5oK&OX24n?$)`!~c?Jd}a3`cC>iV!tziu03h;x~3>K9vNv`_A}eBaRtQe|pZk z*5JFDl0Cu4KuuD8M&)G&GE|PJ)ZO|5G!pUAnmd@o5&T1gT6$)w8^&S`3ZLPA)sr3U zSBv||_DVXB@Tl4|18+meP-&Qfvq54)NTCCx+;2r9~*^cJ&+)T>YNq zlWRr`FCdOHT?7{iEJGlx4#vEkV`@brjmtLRJu5ye3#L6XYfkAwWA9l~x0i4;)7HGm z$$Lc3n5_n1s97^oTRs$E;-l0)X!ohfsH}vJ20^b@PWEWhEiDUD!X#jEnsZx`f_e(O z1+gofQe7cLz!g!G*^?xPTVYca{{S_Ay-+4rCSrNTR`X42jP77tD{RPRbsh26rFirA zRB}63EsVD|s-cy(=NQdz#=>L{pN=|;rn-cq%2NziVJq&P4O3{{9C>ZI4?CBUQo(r0+g(Gi>rIHc z_H||t1gcxm{{V$6gml{JPTjt_!x@Ec zRUNa*qTNj6dm|((&4;%+;P}em9i~_ zB~`v-LtQ}VwBs0?%TRt*8T!$y+>PQi1Ew=o^!6(VZiSnzM|5}1xbz(6vXi2L^LC^M7NwPoUEd|^P%4QLnCy7vpY%N*mmtjnv0mCBYXpk|zoH^|Q2 zum`15xpsDJ1GfUOdNV{Nx!);R_yem|g_1O4N8SBRXW7V6B;0zQzSOArPce_mKT5cv z%Ek7*EK0cwJq1BDW@Qos!00PAZv?Tt-3LHTShBv}0=o{qD}9WbGwbk;|e2M|#J56ym2_r59Slc^6B=oCOIxaSlJ!*`;WNn^v=~bp_ z$mck#O5o9_HmD$7v61ajJ=;%jxUt7`SkkkTj$%JEZ6r$-`{E5%1J z+DzjXMT%8yl_2%4SZ~7&IR&x7sC6wF(n41_G+OB_S8|eZ*wv+S9f=Bgu4Gup)~rH< zZ|{3lCUI&Cu;dPgv2L}pW-2%U^ry-u&3mm$V2PS65WE~#6q>)9$1C*)qrRD& z^|fyhblgDsxNQ2;Y@(W4lmqzH#||n%y*^h3$3kjIouO7j2kzS-dsXS7Ng!sA1mkxU z(26id!a@*`r;sY<(Gv%6(yC`*1x^~YW0E=Tz>z^X?NlaPwODQ!YXTU3(@OTJ(iCxq z>ZH|6Nm3~SxjS=!-qo>pZ1Y)pu)gdLp!TUGtcvRx18XuKm2I(c7yxXLdX5-XD9DjH z1Tf*fs7uC$#Bu^b>6(|Y8g{UsY#ye1(^%LC_?6D63`I98tDY-T+Sy`2d)7voINydf zvLwy`HS1y9an!jx8g|i!UrJkxV9U~@x0oV|oZNyid9CepJCwzXIL0Zz*+ycMPEfR(R5NRS2;1Jjz#;hWS1T2{LVa02yGL^>0LlAPqof{bPE8zaMA-lOuZnsm8%5hTMkkyGI(NT^BthpYQWi%6?lI5{bCa$X3XeN*KAE>KQ z>M_Z}$%E*_6&%tFAZ;+GS&8<=Wop*x3?-y&w_#fIUt6l=29dwnHP7m@wZ;P?u1)|0 zpT?SQ`h|{i=skZLHw1PT zmx4p`aHzQaX}9+)6eXC0fJqsw`zK_G2Vff;pB+7bs#o`9M$Pw|I39wdmr>j;l1tf) zvbNMG2Y)S8DyQp;$F{jCw|UDRfL7&%ZphF5;f$K6pb|;88%59FcH!ZYCmF271=&G<2kBPgiPBIN{#?=ID;?#Os?ddQxa(H2u}^VW`h1~bQqDSa zS0vP-Xqq%}xdw6VLXE+@4wg8}<$%HLYclTU8$!#F19AxUtc$&N;y02BJ8~FSCDx-P z>9SH!p{rVMD-!CupPMmL+oAQW8Rh-NNIo${hdY*!VOHXVo9g92=KV$YY*pAS;(t9 zMt<%{G+oS!@y0&$gN*xCWII{k?v2^wtqBjHw?u6&I*KiC6(mJ-m8h-cNphs*5_3>q zp|!YBgM-SCTChf>*73WYjfVaoLsaeVe$6qCQc=5PaB3vHW?iaJ(DRRKmQ9yY{_um2 zj;EFS&@#)*f?5^>yCLL#MQG`|ros$t6}mia(W{##co{()uiatXRj8$TIT%(wzrfOZ z>Pgalp$qwlFf)cz)Kr%-q>(H9tc3jtKU%|*?npw&7j{R?I{p=*X?qmOv=g<5D0>=) zBWFvE;K3ZJ?^|fW$4bi5nO!!F^+(c0uW0y#;99^(sp0>AN}gjjMsZzyrd-C6lN8@O-m9xH>S^2C)YRB_FcR6FGq(hO6vs)R zAgSZ~rni&JE47mqPUaN;6NBh^tGSzZBDTI{0F>jtJ5}*>YjUH02*>erSgh|HxhaqK zlvU`YhEN2XbJ*hsh6UZ0s0$-tdvi`jiLiX)gV2hqEXFsWU_aS3BrnJ=ap=OQnT3rJ zZ4-=n!|>voY@=L|>VvuBtN9EUaSD58rH&ZT;g06_=qg(shCCrbWq*`)V_ElxHGzaG z8@YDxTJnhevyvOz6_0w3s#^f(p!A|Ufr)n5SK0R)hq%9(2R|qma0(_9#rs?yI}8>x!dmY|Qxs3`axStXxbN{T?>K z1Lu0TU@u%>1XC0zEXS7Yzx{fqrKFV*?kgW$p8lqwR&0fg9-I^W>N~5?Dl-ghNk4u4 zO$h;@w=oNaRNBqZ5nXId64oW-cMR3Q>8Y||BOH^$=hnMxAku&aPfU|p$pFj8XO>Aw z$O=jKsdVdwxK-M4BLDy@wxK#A$fGaaAR4hMAYIMxi2bKu|fZ2L59;%N5{fitl_v!7Q73;1WKyz{|9`Dl2I0O@>2m zBy6*OCZ%UYU9qV>xbIP;9out&c;d7W#~9eE58+gz*j*xy$8JCaJ%Fp0&=p{;LXM}_ ztOirrayuyRPDI+~Ne&b*L+eA_Iu)f;B*$sOkII;1Bd63yAuV}&`+YQdG#K-;jjR1nuhnwoEvkOnC`6kHmdzFAP@0D4u;8uYfRt04ts zQl~!k&%6A%?^iUZKiV)HobMDNJ&W4JIGB)5Cw*;TN>MCnlN@ZAQ9<&XzQp^_;CgdluTg}^_hXAgQN@nEi~yBEDkKuAkqRXcQMo1T?w<$zt4>C%4f*{q(H zMm5dJS;*`v{hh+Ph0j{4ErF4X4Aj=PrY~$)W0WMcIZ~35pvlH*nD!ML-71ZQ^U|5; zdy1PEa=P3|>&_eNQM9VuHdK#FX*l_LPpF}jI0P$t8Z1E4&DqE0NV7U;5vz({EUCuM z52>lqB5oKkKA_QHxJ9TUE?Od|zcpbs2waSudl6BKD9_z~ea$M^M&ll0;-$rIJC?=7 z4xzTU`~-?!@xKU%C)TSjpBQhxaqUyAai9tuf$dV_JBc9JG5~L1YNq*+oyX-piRo9j zuGV69qI3uNjZiaSmI$mL`@hzlcM~zSIATC}1oo^UY$TLm5_8hJd&_2(3EIT>WBn^8 zx)7jxY;jEZ+4>7#epj!;Ykyww8}O>;#k5x#FR|o;Ziu)r$f5 zwsD`vv|51$#M^g9i~|1v5eKbQSc9-&;QeX%U8L;_Jxw)OB9|k9`qNuz1*>pY<|yO& zvZ)W2%|^KCq58K#whZ-{OHb%#6=ky>6Dn7xz6PYS~ogQ+WA z+z09ch5Tz%Lv^-+j&`c#pL(}pY>50(ICxq!*@+dy%HTE^n&~`Eb-qEg^&P!yk#hUa ze;TEzbSxv`u)LpNN}e}jLZ(X|^t9Ixl+{~n$XPU1k-8^yF3Z82Y!{9N-g^+JxKQl=Ct^sWJ&MG*8Cvf0; zRoiQEj1$izG;>^5MoefoFvsIl5$34eIRhNjj}Mh5NEj!zDnvJ9*QFO3N^5ot<}V(& zsWgT2H8>O>n|P})8>0`Hb5^vQWlc$LL|}PJS+UUZMa1+O;ug-LR3De6RJUe^PDje! zL8;lnDd|lBLN{BL2ab0C0EIT( zji(^>&P6^QzFUwwa?4U%LAbJ>jxw!HFeV59V5|;##Yt>cjtgLeP1wj;Ncoqc6y%KW z+^2@BxQopWTju+ucBl+uFk7#^X{1|VN6&09HA?0zWU2Xj;+%CMkjUlJAcIv)5O%QR zH7f$6+M;v2>O0lg%vj_xuq)H~)RF|2GU@o$eB@%C8#-jD$nHI8YpF=-JSBF*ZLWT` z-J@^_uQt*oSngIx$N=y~cQ#kb$`Ouh=Ce9&PnjCSV(j@m)fv}vpI+6V9UN06wpSb) z;HOnbMre$yclpg1E;nMOYbTCU-HEE#Q9kYg;=L%&E|%u0-J--eU@FqWti3Bb*`0VG z)|4;PBC?H3Y|Sj0Qe7ZCfx*blN&S`XYf%UypRjt@L~A>4LxgX!$~wp!Ag(a^EcB5#xYX$14G zNL9h@wyNSLM3GO-27LusNZ9mIpGtx$af-JTjORXC1a~wr9~Hcf)o{OBk>EvA z9e`0oPIl~4ea~vnzn#X|!xm@kJt;N=FK%OS6#-b+q3&v+U$lUUI)(MyR@T_6@3`(U zf&3z>q#Gt1A8$Ny+wh_&3oj6k09}=pMic|dtQ*8el(}QL^Z;iS(oCVC*tuu!c8pdn z)r&->$YY-5RmItg#zgY1yez0mWA~U=TdB!lr*ZYIOOyyggd~cn#Z^gW3yw`}n5iPqaDXvp8;*Tz0>K(h zL1PDl$u-&O)dXPyc-j;vd!R4+&Jet;c6h1)vgC?k5#5b86 z7(6e%B()B04p&h_Ad=wy+~8JKwYyI^R>)JyHPPx3291$dT0a0c68`!Z>dE@jPKl|jB{3Cd@BHY^Utj`<}uE4qw}PEum(PRjPOUbJNG16jB+Ck zVTsQ^wO@Y63fvGf4K2)zB6*B9xZ$x>bg#+J0Ct}lOPks$O)s7gBPf`99##D7DKQQU;KgO0h z58PsQ^gh%M)z)^X0O;PGpQ-n7I^ENq&gjq);}^H$@w*r4|8X&yhcTtg!qI()6~ zTV}wRIufk8{A%xVUW8WXeWND@l=1Ibw>!5I0T}A0r-E}4Wqz5(Lvs0IsQTMsJbU!cu(_R@w`W9eRIHky-HHm3A8;+TaT^HZhE1B}*8(iAGo-jQ_; zhYTym#KFP)t3yPtZixee4KBoR2>_bPi&2e-Bz|=|UM}JTb+0+j>djcLR<<-&+B3*e zPg`UMfmLI?7{=jKrN}4m3go=v^kcg$EI?DXR3hYO85F;0U+)Tj)~9wmvni(DH~`go z+#t>ael$BgU~S1A^H7;tkIaOM`vi=JG^%^ly4=aqOE8Nl7%p)h$8ZYWD0q#XYMT9Iz0 zWy3B3@BaYSsUn3WZO@ln@&O=5s}ZWg zH%8Va}k9u6@Ff>s<}>QU+&yQ+IKkA+Gko zqFc>y_}Lm@R~4Xv5t>7d!45dB+wDl&=z=6+x==Xp?^SYG{VwK9o5^Eh3b`bl^rz}6 z9C9!K5gO;8UbR=kN*;Nx4{gSz)+JK2m4BH~V8;IdX-Nw)89h(8T9E>U z-WLF1WYewS5L?3mOuM0CPeliwY11mILkJ+Vo!O{m502j;wlkAbt-A*V4wZh&)U*tC z^38#oyA_FMQU-IyX4umxn^t&+GlSZzOKeqwuOJbC0;DT~6d#oORF^S5oSRr<(T74Q%2r6pLzCPZU?2-{fnyky9!IaeG30p8$Ir{q z_Z1U{M&7?E9feBQNjiS9`!W3Sr9M_ovpb~d>+&-P@6?GPcfE78(VM$4)xjE+ZnXv$1*2Re`@BS zwmw5TFU=Xr2h>*MU%iVKUf?la#8p|6%spD)$hc_)WmWUXp{%BD#H*f1-L9`zSg-6> z<{tgM>ydMbPaW%c>WriW=N+mx7|9hUQfUhF&}ONUL~?V_YEv##imJqtMK(5Vjp|y=R+MrcrATrr zbhqina!P2HR%tFVoz)lGE4EPDFM9InGU1!m+0_l?t2!$`?52r!+KZ z+P$P>oyZP7D!u%_@iG|UV@5dj;;(pl8vg)MONR#tp?T~FYL+y0FD;-`r>TW1){a6P+K+h|n)j~FMD?O3^LK7|{rKiw|gqpf9M$K^-6IAhS7>ZgrD zdkW^Yuo(h^K5n01O1o$z$}vbne))5cy6eF(!F%43Egk7G@>1 z5}+Ma{(Y;RSy^tGU2vy{_Xqqck`>9GHH!Kf2Ma2J?V8($*&Gaxy=xa~mL=}dHy+}* zwCjN)sA2PBgbc2lc#d`eVPqefs(R{}p^3oztAoXKj+yC?l;l<)h(h^WeQM*mEW%5( zZlwwNTz*Ee+4H}>PNjfeduA3c;bN& zu;iawL)g>U0x{m3leYv?waO5Iw>iyPu`Ry~(yCy9Pg>B^8Bm<{tZRHMsBURHY=kI0 zp0x@-7#e%IW1iIuH#`dRB(*wGqZ0-JhvQe`86vXm&JQ)I26qoi@o`bKnWEJf;$8(= zjVioHrCUQ@MB5mX0H|uN2c=JPaJ5r+wQogUA}o2X1_Wf9uIn?BcCyupra{R#9=WQf z>O7FIpRQ}{6EYZMVscQLXhzQjDE6g$lj=U1sLja(gk04o1k3V$=HuRx?<0s3Mt!QY zOb9%JYO|*PWNo9~pDn~qKCe6e?lcHJYKp9rP7JKg&p>Kqa-^h0Dv!L!lUjDRv7~#- z`DMBgc|V;BDkvKp6bg&I7Y7*6aas{X&VfE<$m)3pp$LfIEyAfiiR)LbWRfwM(8=$! zB9`D+Yi}(}mB+gGrlQLy`nB4=y=wE`ToJNMPCZxU{{R{i<~(D{+ClHl39v*_#1w%h zN&f(Lr+Z~=ZQMAdlH{pTh6MTy)T?p<8{}+}(Bh@5m2!-hn;D(>RT;_ZYnjwow2aGv z>0LeRJiECqpHp1b{8A0r;GO`gi?B6wIyTzM%-Jf{7?fK@aPGq_n^)>cr`*8M+l~oi zp0z^eKx0*No;MoRYRItZWM7&vAw8H;@dVIw;`HV^MO=dcyf z+U^Mgs^C5|j>E1h?7*>f7FTx%04c)$99KUY5b96jYoR1M%7d0KilS{>%wnZM#sak< zPROZp8bnynL|vk>^CVY_%hw=ROL1<*GkIqV*N(M`cM};DFC@2IW2H2}OZRDRl@Oe` z13uN-XfZ9c5X1&dolm!OT%VD>ZR7(i>_!3i>G{^}hN9Ndw1t$su2Bc~R0<>qZEtfM zI0b-?F}c5>s~Sw^<>#2>Zj)-~&{qv}tF(5TBV)_+j;!6QrL@0YJ57(wBYxe_&q8}v zY9P&xLiRO(DO)Uh=8p@ zP0I?A*yr=CZC(LM1h)I#WRc?o0I}`Tx;2EMWharsnx%VY(%cyUzyl}iib`4z+MG4T zp;Eb1&tI)o5wqn_PwX;BGc zXN>&A=3Y9A>$Iuk!jQlYKO<9XicHnEwng0-9X?@FPJs%sU%W7CKRib|w;w1f-r}y@ zD3;XY?=LhYP0n)mQ55UDAU;&%*R^x*jV;6A4357_==E4L638%0umo2*2IG4qc_F>( zx&ZrjlVd43kw_r=in|TUqX{D<5y?*A_o$(bBpdUGLxuy>)}y^q9m4bs#~JOMQyV&a zdzG0L$@i>}5P;Eeea0%SrTawd8*-#%dwPmpYELrVb`$CiIBrWhtEcPFYO5FomK|zY z+a%|?rbafOZ+hul8E#6Vl%G!YgFU)ZkxSG}jmI4^Nw;@C)guhhqXhSiC)%pB65VK` zwL)N6jzPvM3yEYQa>tQH6>DHMwCIzMly$4QCEwU!dr?JWEryDumbd`&D@xilR#FrL zMHD@T%{1mRPc>XY6U=uYJkdo;5p9$v0*nlfl&uU%2~<#ilu=ZJsY2;MQ^jLCE6hs_ z;)*I3<@6|DsPh8jj270-EsUC+O6MR*2p>UOnzXVmmw?1>W1o5`p>7>b%?>m+@MT8`E2_1P zat_|qQCZ7yE6f17U$6D2&Wd1=k?Tbj3wBfJQPdGkFy(8W)uKgmLV$88qOLoLr>i2i zirE?YMQ|4*1mcP-qB|08_)=*wgL(O6+CGOh*Xg8!%iGiOqKcwqUottJ!D4v+m9=W@ z)?;zc$wd_`lPdL&#~I^_=iF@6gD(xc8KR09(2;z)x68)hbMMxjsDb4kJJCfQL{ma@ zp5vOce8j*qeJG-|j_4(^&PYz~zO*6jMHRnO5=Hye@%<>GkRiD_Jkk%oXrh9HKR!EV zuG%7*RFR%&qNaN=iM?seiVbWhLq!$G9nG0S_DBBl>s3+Mzgj4+9Dl%RPjg1x^sNZ` XQAK>6eUY;nbN6dMG*MGeVITk5zm#%8 literal 0 HcmV?d00001 diff --git a/python/nano/test/resources/train_image_folder_jpg/cats/cat.100.jpg b/python/nano/test/resources/train_image_folder_jpg/cats/cat.100.jpg new file mode 100644 index 0000000000000000000000000000000000000000..0046d3ac934d4114c0a0cd01621e1959612b95d6 GIT binary patch literal 28377 zcmbTdXH*k!`1hI6dj}~2g7kn$mq?KsdJDZNO^{GS=m=5-A%xI70s^6f^p4Vd?_HE8 zU8z4%bp1cip51fy)$U$%X5P$w=G>n%bI(=2*Zp_-?-qbY3#JJJ0D%Ai@ZSOWy9Q7J zkP#D;5EGG+kdTm*k&#m|(@;@TQn52K(K2&$^71_7e8|No^yD$0prjzzLs2C$NvN!X zf&#CIs-}vZ#uIr3+5dS6kc^z1ijoRMLj#f(cqkzI|JnX_187Nr7r+#J;6nf&EfAj; z__rUx`EQhY_ymMR#3ZC-q0gH?$*uh-45CPx?T@tz6f`2%q`EBVq3yK}yE-fSHArmycgS5F#lh{X_=(R7F({ zrmmr>Wn^q(YG!U>>EQU%$@!HF0_EfD=N}Ll6!|tP`dv&cIyogZEj=SME4#3$7*|qS zR$fux(Ad=6(%RPE+t)uZI5a#mIy*PNu(-6mvikMg_Rj9!{`Z5!pXV2sSJyZH-QN8V z7Z8B|zp?&Z|2NqGgNybb7ajotJ^}InZ~^iB|DE`>1cY2-M084q#CAyfhvMNR49ZC# z>Uv1ICE!08?Y(Enn0O?=^8WlEwEso+{|+qT{}BkRXV=j=Xr`9`(a2@V1?PBzX6t7uGk^$c09Y?ToQH(7M}-<{5P*+M zz!NeK?vfx@0JH6O7dPlzwA@}&JeVWQ=)WH35`Go!JV9#}vVzc8!o*~P9qc(c`LG0R zNxssDC`FbtRVZ)WS~YP}?rg7y1NM@lWa}NYcu-v{{kcs-7mP}PK$m9RS-!k=WI$Lw zLf}l1`o&{T@i{$lFEKR+z@qQY@z_{dXx$gLLoij^RuWWDM>_?k|Z;^0x_dB3k?gwte2WK z4S_X%cI0TEa}y952Os=z>q5zZ$fdw`8m~dn0qtO)mP%A?r$`1o|Y+gk}>u+*DIofIyK-SLz+}5O{{6 z?46z5d_w#eOIwLa!&S7{p9uyc94!ne>A=vG*}qC1DyXz}SzJnCpGg`&gcdjW-J41@ z#Zsyvftl_mnvE_JmOw{nQV^#Y+&7D**#qxpTg^LK4aUWcS5Fw?#~(=UwyAV`1f#8o z{e<_Y=ZP#Br*P1hlrya*Vw5S0an0v=xp^*wG{<^~Tt_VASaDlQS?CjRh?^nl{xHvg zIT;3NeI_kPwc~xituq*8R6Kczcw*_m`GBP){WhNDi#+UDVgM{Wgz-2OTu$&G_k!el z?Y`iQseOUDgPRuh9ssizS9s$~_kdDgND?cN5Dbj%p3p&SD%L>lsprsOsUH zANd|+%D7j=m=l-`B{Rdvl9p9Bjrj>)Y|+8w1+HUdsaHlWpOQKQei}|Z)q0!P(OM<@ z-h+9!_mf|Q9wARqSKC|YYrM9D)da*xyI?3=`nlb$TChxx56a_$J^`$p*^ z0%Txv^SUo_-X*FZ%DcR3mAst?(HQp3O5kXUr20rP=SDg0cV=p2q*-5=tgb^p8zoaG z)6`aLqq!AnsG(T##`U(Abf`;mgUJ2~-e#I^Ov4`Dykj$@eI!{|uXvlGJHT!>BNWIm z2%+9AS0>V#4s)!nRQX(F!bdC*=|9|+@pxZR%R|A+{B~q*fF$f{%K*riNAn3#OfX9G zctj6p-q||vM3R9hNl{or*BP%!Jc>n*3@{hXJiQ~sS4!0d{awi+Tl)@-CDds7bJJ>AuSwl-0te9qI~-RNPI;y94DlIu!q4bZo|OA9xxuCbE$D}v zm0W)T;crq-g{HoFwE<2%T0sjjH6TFd82<1O!bPgWwuq@o9=2o@u@&|)a!tzj41{sS zKY4EFT&_(|NR8nD9$EefhzO{UPbA&C8n8ylS|mZ(vsT~vKYKmb=O)9?vm{6y`?^hf zqW6gky5mU<7ww)z>w;j(%j2E+woDOC`HHMhu`hd&r@$(Am43sf{L^BCea zz4{`=Ma$!k6_$_s{3OXugvXi+bq{&Fx&sMMB{EvdbH%b2utZ-MWma#plGTL|BQx?2o0z@TYKoI za&FMIG z+CpNkzLh+EnCH2{<}kN{!h#cru^UrjmT%iRrO#`B%4cnXTMkzBT?Q zY}WXZdAE}PmGl_qI`icE7y0OmANrQ|XONE!jjwFarn84?6s;uc8>URXs75`s-LkC% zJA@C&x5(B}4tIZ$m1UZ39x6JpL~r1YuYV+PZbIDG+4$AS+`EpQG-*~ZBv}vTcdGw1 zP5ad5l|ss)tc8Ij4>-1t%>D*8>4A4AEri+xEn{{k%L zR?BriuJnDTZY`TyEPDSi4XO}*Wb@Lij}g9afZib2*a~*e-g(@qi4sc8e;A(o=%sMR zT_es=!y2oOn3(@Gn1owD@aQ!Cu&hGs^{mUJdVk(?$H1H00~Kj1EGJV|F_)X`O%{;xVOY@}6xtB7Iv8Os_xH0~;z!SfOV@Un0FBl#t<`j5DQ%&md)zJ63Ji(& zhuVS4j=nh8dYgE&JT&uOu#B#0C=D%q;6ck>}_akM>h| zcBrmMNorfW$wUR`%W{aQ5rKG#!=;4|{Ow~-pMhtUx^K-}pl>^NINy*h94x5Ts4_JLl6t!Wp5pU97iaK%AX;kMX zWOsS*E%3vW!C)-Px}@%F)Ge+#9{8-Mxz;uH@NLl}JsIoaw^@PAGW^vRSSK!%7sSky z!E+vV*~^)`I^tLG&-Xz!Z24V-0RzhO9p7tg=)~X8(*5bnHGUMxWfYkkyg=%~ht_dS z9o{PnE-(tX!yt7BB7>|O+ryHb46GJDrCz>vEEv5!8m?O2D9#jx7d`WzHyH-<3=)xk zs7(F>KW}D;{tg|MvdQF{gy~y)IYh8Ob!L`-Hu#}!z4#^Gj_rl+ZG4w%TH5H&PGr)b z5W!?Rdne*OmTfs>INd1chaLKSt=At`H=^~quRaJ-T3|NuYEfz2&>@)tAzhkCdkPSB z4yU!TSV9hkD9N+0_kt!GPNTc(M+YW6)uBDOQ(adsvOtG~92V%*(vR}j(?g)JfV*Im zSB#i8*ed#&NMkkE7=?q&I%=f6>562}8_Nh=hMXg7xdlRy4`wb10HuT{>x1V1*te}x&E#Dfd+CqO?cJJROz7fs@m%fs!dDPAGQQ{1`wX2n^Pb&_Hb6DQ{2RAxw`-Lz1V(LDN@K?nm3Ye)*(JCGwNE z&bwXcU@+%jfQ<=3o5|`Ehh&!#P$t}qgn%=&H(N96_|=flutclJV9<4?!4bXk+jq8_ zR|gg_#e#Gejx3(NO14;u4m-AYV32WfUT;I)iqf}OTcux18-tpAg>651H~K&$eQ*5Q zNb;1&``N?7n6q2RCGCt~CX~w=?fT89m8!UW+=UQl1|CWbm`Zfc-Xd=?%R5*X$&IJ` zaL>g0ibQh1nkmg6zonB-?b@bzV~S^Cxr>%r?dtDXo#>uBBknjfYwA}A$sNfVloa=mB4{6TThTywsbD=Q$jdx{aifSWkGzE6t$W0#P% z%bEke*^EP{vhw?W4mB_TyGIkb@tt3vCg6hF0O+aDb?9BH9m>I{i}VR1B|po3odO+e z8>ej($u5QF7v9u;PBq^Sg-QHjoI4EVOkUMp+!k!T`CTb?vP0Gn?)|ZTbEaw&FFcds z7Qw%2jCl;LWj8J6D=#?@A->J^HllBkgEIY5gQwb75{6^G11&gqe=Kk}tCVsKNI_|3wyTsdk8WF!2j~2bi9{$64?R9Uq+d!{kL*=QcjH89u=av9=%@%22jpcj)|Y})Q*M0s1d;tNxrw3 zKwEXC?5dqC#X;g1eTf;}+B^RL4QiX!BGcLwNbl2nRluW^TN9{!l0o0dsfSGfmWS8<$4Oo(&M@I5D0e%!LGWmak+9R$H12}BQ33T` zp{WH|G#qNnNUX;Gz2Q@-HX27XE8Vz;q$?XzvXcH0(UDzlITXcGZN7v6TJbstCj0xp z3?5E}F;k%oR0ky{Ue!`SS*mUmv#zlVuab82<$Mamx$p_d1jwn4zQ`zCK|XzrfL@n0 z+TH!sJG|_N%|HocJRGV5sZMjGG{E1Z4w0?uq=iWkr8Y zeS`maDLc`!E@Y*fD0!PJ>`0Mju;Wg2s>B7GDTl5D34+J?zsZ#G!F0Z2iJgiEl@Ce3 z6$VU-L+^|>4U`BX5PYb}NVK9VLYlB05Zbs$kd0Ppbc}>w&lh8U4XjS|<~>8Vb*wpN zX?aD3j|>GJocd`qldTZ7{TjxI<8|Jl2ZriaC>@;aRp_leH94a>O3_#~a_R0N3^w2| zKwe0B+k^DwetFek5^Km1*x~bk(+dlOvuL5=Jt?lSXEE*yT#cyFGC2*i_g2`T%jBshP=YZV-2{9>P<%TGL_ z@9I5}O&Z{CeA`R~Q)ed>!);PAiw??^vn$#6G)G}Tp}vN+MW!!7t#Wx{dPguRzacKR z8UK)3jnd%~_cCv)W4yrOVv|+!gdBx0)e4Vxssj7UM$tSc6>P^32clo4We9aw{Nlx$ zczc^)GCJ`lOhJm1C6dlesnaKwUSxtO{I-9)NXznmFy(IF-j}1!;8*~Tw63?V$_zq& zCsnDADoxIcb(~inL~M;R+eOP<9q5M;thw3@4J-M`=Ae*6xBotwQ9Ms5<*?y@CP-0n z!c7%h&~_=qNhJ@Oy#`uom%Roo;BnjhFtz9Z|Z-wuSV)d|i zln`r1P;v>4{|+R3ux-9rU4;9nhcfARg%eht?d_vhiV(pAJK@sTSPl7@3agd4BsIoQ z^H%tKM%X?^&NK<}R&T{Evb9JZX0{7VkZF8VRDB!sChFESw|$-0QdG36ddCSMa*CXj zS~Rx`$pe$_T;b50%&0oN+S-$S|-Kr=3BzwF|>M~ z8Ae|fC=*IC>?Vmrj5_fr%)abDOMk_zB+CfCFsv1C<3e4QXFnDIy#KFIX7dUA#bP2= zT+kqKI_k_|in6~5@DPJCLn<+Jcc)8JHiqY08&lz^J$svK9yoT^AaZ}=Ma_p#;WTDc z^PfWm*)YD}DxY`8C=t+Nv#K)M6T|$sE^p%WJM{u;G-ysGe_^37{n9XxIjLj%&g6#T zB>BtZo5POoOo#oC6H6@6%K7LTvvkw=)aV^=Meb^dHP4bZLkuG`MjG-&x%<|ePU$u& z<8J;5Nq^DFvay%aW#Pm48nd+B#7^<8b}_fR?PqhhaK-UWIN_u>aO%ilgIgk!H9hh5 zI#p4ZHo||3^&$c_q==fktW2Jdy(jQ{B6W~&adOl23o9ONuKtR+Jnq|u$v`QFky;L=AvP5DvzDHz8t*OX z)REz+rOl!KW9H$@gMWV7LbO50$B6J%uXg|eaCvSgWkmh_j~jd2l>S9F`YWrn!46h& zYPL9)nrm6mrXP9q2dUZ-FBPI1jbf0EWpTP~7-6`~cN|wTY~AL*ThQbb(GCL`4Sl>F zfHIHV7}Er9uVwokIcS&6ifoRT@YxK3?dvmJ!}T2qGu`|X6` z?1xn~D)o+H;a%i3R8U4Y@$!w|%4INW3S$0n?1Jb<+DbC=OPAHU=9-G4_HKd3q(%Gw z?_YECB2R|JpE`G`6Bdv6>UIK$WDblTy89)Y-lp1u<&UJ3gRytZEzBB4x3S%oUOE~E zGS9!ZFEFg?9q2tMV>vRlOChkudw_Gix}|!uyc}*F)qOU#Z%NGAyDXLO%9F)xjeSSV zh+vA}HH;92?!_#BZ;CV2TGO8lJVY7|j#Itff39$POnYmbw5A$p+cu>fP&sYYT|G$+ z3WHoC_E6c{hUq^#BL&~Lo+7>rG41JDfts`uYxDrCGS4hiVI6*3DC3XQ+i9x9cCC*D zp2uSEkF@0684Vo_ycv+D4hkhTkPRKjxkvq>wyqWm->U*Li9q|=EBft1s!mUn(SjH| zqil$l;JPVG$7+VJp?1p=KUzYq->$coG^=>p@#zaA8<1T>nmH z?W&@PtHDTqOneKdp?WUSL8P|Dn*D@I0>fI85un@ghSkR(2prv)o5MR{G=DDmhhBYk z=~wAR0kO3$|G2o2!b=C*(H)c6Q@Q@t2cU&O5B-c5hG-0>k_3xvcQHe$2G$#bo5fi(Df;j%WM*ymp zXpgt|iG`+b241bL4wv=U3QP>K6KgrnKz^5r;^UG01t?suH&Nb@w^}N$c@1d%Zri_N z1ZYxb4eC*cHZ|G3;`5^%9Rb=D`xE~KR9nofO|<)h-HmR%(M%1N%FUaaZ5i;Ee zmaD$G^`A$>Dyc#~Pcag&u<)swY+-2hty64Z@y>OL&n7UZ2Ejm8oTH`2Xh--X;tbX> zHmKba@>^TS{JAqYYydfl%oA#wcVFCm;BS4Al(*rE?jq-%Bh+o>Pr<$_^&3f6H5i<& z^^+y?CcRtuU`Rq5@+RID9Gh#OIr^daR{uvO z^^ZzBIu-CFKn~>h7x1v;*OQfcCl($b;&8oafpiIOssg0>$R%W=9$c3^ObVdds4!mu z$D{b=MTz-Y_9Yh$*r0CLeFeRYpTX;M%auA~b*pnXz6D@_}MEdQ+7%-POYYxGR zb^F9TZvo&5%L%;rHOf$(iyfgaIsJAhW`I8f?fG6;!YP~p4UpMu@L(fY7|O%~cNSsg zL63??WcW}2NeI>^8(1TgRtdesaWjzbjn;!K)1UE*VqdI z^5GNn=fh{FhRQV6H|NwXM1}74;_;<l$TZCsM3f@{?Xgr~S{G7vy(Fj=1v2QWH2vkY>2FDyM?aK@%##+_ek>2qCW3rK2p;K6pzxE_)|zaM9lgqqCB2izVj_)wIOwLU(o6hc;2CS zQ{laGYX%4Y1w10=ct7yUvcE6N&6{Mq$*QxCNc2$QaQmm+Ux13%4d&R2hJ*=f5L=(>Fn3s>sp7GI$<#2(8gY)J8fq@N$)xEU+`>FT{ra~)&x6;sC0rs zWe&nzV;kR1v{FV4akD&6TMH!^b5#4b1sMMM2!h`tCT~Q34V`MFv_*9wjd7KP-JZ+?4Ms*X0`#{-s2TzT&H1btCU%EZW^DB{o= zum19<5aq;;tBFLo4ejz)c4L)RYb6b54#C3910&_N7uiL8FVrZ4<}u8V8J1HWq(r_> zFFe%B$=^h0mW>p?51TsD+{AnfLufH!=AOX!* zXckX(36`z@c!ejp3uI1~IdqL%k|IcUGD(VqAx~_G>UV|LFZr=uLQ|(VEMXgir(QvS z7%}fRu-T@T>G8|aOL7ugNQ(>ojE#)vZNmc!_gOV&W8E69O6#)~U44mLwA0eE#^*Ou z?saj`GFXQd?E|bY{+ss1PFEFp$bse`C^o*s$Dj5qOJ6>bg()3v#+gGmhdU@|Pndha zJIRP+%VMO+Em@d45Go(y%KhuYp`wuJ>B!`q};|H6(JJI9KZ)hHAWqLa|h;*8LBElpC*HWqeMQs-F4( zs%_&g<(8^9)jBd_nRi*R~-REVE7hbF0q1+HTb$K45i!+;VJHtFRNUxoxEDt1372wrGTH58Vaxe)h?_^ zxjfqovIpAkPuUQdB#i%FU21dP4EdF?#p?zExBK3wp(f?tBRh;4*mvCLf1>M8Xfqt% za^tg1yy#$~cXi&A1Kqv3JG41APA_{ToR|kQmLME>T`@id0v1|S=Yb6rs zgE(S4Tw`XgWINb~nzGo$60)qA4{T%>2=7FO$hMOUyoI15W==lJE|rCY_XTG!ZS|;Y zD^kR)E9$oVJr)>R>eGpg0o#DQ?RjH~b&>Hp;u(FM)9H%!Vu$?$?=MgaiNjZZkYezy zWsy~mFNVNm=a02dvBk8sGJsf_QnEulu!9LMsy@DNyUlRU@N87$c_#NKNL$7S>n(3J zt$HHzjxZ>6FDfJwH@3K;CvjxOBvvWPFFbK%K0~Nyj9mH`0of2GJ=E%cx^+F^(j1Y~ zs5|y50#=DZnwsInd3%9*2t%vt1hI||EKy$mYh%R=IDRxk|JZ>Dks!tACqe%lG;Hz} zxM3d+X^}Jm^syAv=Cal`XSkb0?20CI;IwyljDLlct!edm(`2#>`Wd$tG4u~N!x;y( z%w{v^DPpQsxVkGTApE`iU0@hrIr$W=2`Ah&K6}pn$DAV6ijhjmEJ~$zNqF|QqD-S z;acGwRoB^?tKNPfn^xD+q__m*+}Y&99kj}qayC>ffYEv8-B*_^*$7CtZw_IL5RYM zi{rw_^3)<1*+QjKh;>7>aA}RcUYh;xTa60lA5b@Jnla+LS=4P#_2aYsWa(rk^4yfX z!i1p19p35~ZcY1k%|Uj=CoY|xoeTW*n485PQ9au^IY)j4#?a7cr6-F?H&d;V^7aLDyoOE@yDXE7hG z2=&-n;ttWvRI46w$E(2S0*lEx;H#7z&=gDI@R&jkmY2=9-Dr)^gpdx-ps1y zo&KwW`;+m{M?7xG%Ws9^oiz%^i*L1qL3g<~B6z15j33WW8^W~M^aU1tA%~RvIqRk9 z%iwse6B@3y4piNcp3rPmwuRDeaiG1Jk)@WpF5}DckwfVWGBjhct5Wl_^df&>g&}bDch`U<@cmotY0#ESK|ni&;yGyMG_`@<*jy9T^QsVsfzG ziZ4O-jUJa1v#s&kbpDRWv!gTP=ac8s_84;!P3yh#N?KQhb+j5k6r7J}picz++GzgP z03Lw~d>=|`-D~7lW=PyY!<%N1^X(Yb9CB=++~TLCf9*lHG0!af)L*(q5nOxzi?o*J z*yVONBPq{xdEJM2*7X)Qb$^y$mmlKF@TBC4Ak#Hbb$eoJvT|tm$6bPk#m=TrrcISxW2Mh9g`aCSkTt+lM?uSBk)pMvu^OGDjOS zLp}hZu)yb6prk4@$|k*Ov8mZcDTc>lyk6>`EsM_w=VZK;oS;N?T{A0esk^V*v#9kJ z;DIz3UCW}|#JCp$__7e0^NTsqqI~ZJd79ej{v1yI1LC@kA%_#9-SQEyK#^6aL^)3) z0r4sm9}^}qJYX6BK9*LX`x@sNY^YYE?<_SCD^5g+$|dVOKUgjQ3(!&{yz>BEm@sV= zD|LPPp)kLEnLJao=*yi3{tLLP@U-8_4r1c$$$lOG>mAvU+z;e|X;mlnOV@Ylewo`< z@4^uAT}|VZ=UU2FlY&x|mOkO>!aMsoGB=)VGeK8D7s54iy6`1@@@dn>G|BW;%LjnE zLy4=Jpw;QXg0^){V*cFAYUmkG5vW)pgO}40r|%WFc%1!pVlMPYT^};x>t&_aL>GyD zhw&wGk_Da$Pxz7yC~@IYCx!)0A^2;flC9~(9J{Y%NfM*dk#*F>okq6pixV!@K0P>ycr1(;X;k)F(!V0bV1DOQx663Y{clJRKR(US zdi@QTz6{}Z26Y+FS=7v&1E;4+34hQN@dH9ycVm`pX|-s&T}=>{z2K-n*GPb zn0X6STT}9eUbR*AT`*D4$Pj0>Nyox|n|=EJn1NX2+yVU1%5IQscbPCc&V8rXAX)xd zH9W_Nnom<`8K7p+Zk9%6lGJruQOP{K{_A?5o4L&6@OZ|CwbIWzj1951&%aq{Z_`3q z^ij^31_Bk{i8{d|E`#y%VC(yP3~49cC%(u|=%fk<5%8Lk+gzyJq}$C`ZvH1MXYv7l zDG2jN%4ekrMhb3uE?A9cdgxuh{PgqGg@eq181Wg2X^+a6vg)-Cm@$MK91*x?@yJzN z%U?V(q+%nxEk{!bU8DIJyUfFJ6*_8AMWrX3U|G#Z2=WdN>$I<4OZ)mdeO=Gn`%N`Z z2jwHGA+0tS_f%<_?JtZa&?@;y%wV_e1ihxlEFr6&gYq6jOI_-M`&B}z_r@qwCxNMI z7Acli40Weq^f?%rq&8h;zK1TqUcMK=jdw(0^T#9u)fbD4H!Q7_@?LatVdkT{OIQS2 zt0X@ZujxJ3{qo99H7_*|FIyRT<08LyEVLdBcb@+WagqM~i*1 z+V-(+gM7KOZj48fk*#~u_%6tVJkyCAsZ0Xh`Kir*nWN#FY!YyrAZ6!)$udd#x4we; z-cPW$i}JLzi_@ShHlDFg*VUyuOi{=T5jAtguB6`Y;A3h6t6eqT+A?A|pNqj4TNu-i zah{$aFM(tbcHv11y}+w^eC)<&vi6zY6>%%&k`MnRzK>)`VKj?v{9IW3al$Y@UXEc~N}2b>HM3YrdZ{dxl zA~5N@KM%8$spS=AOo_oaHzg^}5q?2r*K|Te@Eo(|AhgRIf1oC56mg`DZl~D#TC6?G z8a5JIEM6$h3<%h*@K}sMc6DNcHFlfXNcyM>t9xitDr!)ZL31_5it1@C@}kt1!cNJi z20xiceKs^d2lX|$)!nDKa*5FIJ=6bD@+qaxyupE84D%NNS!_0H$>oqZGiqwdd`IAO zSsMI(T)q%fHR)1lD}BNfZtU1%KY+0nfe!eXS^D(<=zI0A+M8kli|-g_0b}+FW~ysk z$8mMh+-kq-nfUA2A?q~j9=f|P=H%Z%TGPLD4aeAGyIDUoI5NHAp`f5JsgyLDj@aPw zdP*TCXZjXS2oSF!*0(J1r@cWg$s7upTmLRAmG7wt)yN7=FXpXhBwF9upG}gS&*}?= z^x+p?msh*(Iz%fI|I*JjXcE3L%=td3c6z|km}h@C<5gGaNncwcbym25519=tjEjmZIZ03y zj7M>ss656I!A`Wc>uksW0yayssJ#)N_W4p%j?{YlarcNPP=>h)t^-B$p@kF1rSX-n zcT$6g;HHqpw+Qv$~ab+OM3w!2G)X z;sD(}bM~<1R_Bk)6fi`nP?PSP@KA!D8nru>ODPjP!1g`$NRvK3x{@fh1PS&7KBf5K zYV%>UHB{-y@bg1~~hJBi4N+!zZj>11ERXy+oo^hXrom^A63ENH1rd|13V zg5>DTx+UiIGtgQ|3x{=a3fEC~ahuJAi{xq6sA__sQ?#<;A?cLlHt2+_FBf|s`zH+q zX?Ku=(^*U%{88hv{3nLR?|LD!Kb2kDF$=7mYJ*<^=-O+7PmlClVm8(ZqvEzsoVsfQ zrmiQHvRz0qVm&-Q_QHw$j(Z+U&)?Rbw0Cfi>i$ z8rIh}v0D!xm6alQfF_O+%AcWv*t{#u$46A=Q=4w1@6@5|lWhD&91)H5Ix#f_$wb`j zYPythSwQh*F?4e}nvJd9xM_jRSNPHqX+9XGnADJ9S?d4>v~?dM4Cx|!x0__5)JAM8 z&n$bX{k%v09kp2?r2HKBhvYF}lktf)>;lt=3g(6MC5mp}DXsmTw)U(a=N=00U9_~W zpT3&Tz27gCrbzz~!z{Be@6yq^r3`3-&v}T3!sL-=U&G!}K@H~^w$~R84L7plU(ZPI z47-IXNl8`jyxU801#KspNb!hmY5}^5bAztRE^h+KW%t|rII^fMa?=pfS_euwD)TCc@DJG6O z_>>x1;Rs3BA~lnBIs0_iiSbG->89Hxjr>_5(B9siyIItcK8cGTy%mr{LOY>avv!($ z0sv$Br<*&;x)1GDy4Z2zm}t|PGVBzZs@%3;e8tg#|FGg{tw)zPQ$NU8nvLOOs7EuW ziM}J%1*}I`Q5vI&@t=(zY;qhBK49wv6Z@!ydIkaTN-8aIw@fJh*PHhqn#`9q!SIk2 zJED-OAB&9iO-VDFEgw?E|5eN>D`a}U1}!+(v*}<&iFxzGI)9m_jrD;-9nX5tAC#cX z{qJ%!lB;OI@~O5z@%7S!8$PR*S|AH?!cvnI8*N3?gyUkNIK;fyJdZpGW#bWB zRfXj7CwL$)w*DsXs<*gMigvXCExJtIf`ZD6WzWRxViZODYrKz#6kgQHYhIVke!{$A z`o&R86dR=H^qiTeF>3bZwO?8<^^irmPOn9VIg$}i*=Bh1XDsp)z(LV&zTjC5|R!hy(M3+zd7GT4jcdOkY{GAoiuGS@J6)E z+1{07Q}Y~XeNQA5G`pceVZ?HbU|`2WD#Yer8%FHLNZA=s(1>9hD?Y#kZ5AchC%UU0 z2+~%1*V6m_C4_0|G=@*CuO@NASZ@~OTy{gnII9wmHapyC(ci?7!y7qef9W)dv`2y}iQa>lKJU{(?sNmcUpdbMAR*lT%IB z0)I))16vG8Jn>N0A$zvjpok6m-=YtheIFN3&>O$TYV#JO31eU9e3NE7#P@1{!L64u z{PGZtSRf$UA?*NrRM8_BspWogGo=YZj!iOfvC1{nRN<^m3n3ndvCWb+aT}84Z zbwqeq{Ig|4c$Kyk*VUEgs;*dt)=G@XsrOQ{Mr{$mC_(EfRpUFH4~REY&)87lOt%GU z`MOJhj;q)swec-dNk!IJxL;oX$S{z1=UYQrjwNYq$MTF(NZS4P@|{d3Wlk-dceABy zupS#zaOBu^(O6IU^_+!^IVm?jXdxAf&uF1=Vy@qzLx>u95>Tq)l~r+!ns>^^bNiX? z0zEJp6a5}=QzF&FuE`k%=6P?b3Q?EJ5R&A%!K>Ex>i2>vG-bVh8IZ%me`Lh}(z~GM zSPfOK{s6$F5Q*;T5 zQYerTVJ#IJyT=zt?>j^;@e}vOr=MR!wX1%d*CK=-@QYmT@^1yy3?Z4qCRKrhIjX#9$?#ssx4-lgkowI7_)Xz8T_S9 z&f1bF>%v+PhAQB)d}16aZARXTNt&Pk+>cml+gP)?oi02R))=EbL}TR9r+ye&kz|UVK3*Jza{UQ!F<}acmh{tKq390{JZY0&Yw=&n!c$T#nHIx1jFT=abrDfFX}4j&D=tY2yc9a&?#DJd zDy~Y2%l#NOcWV=Jj9PvQa zXM4)z1{-^6K~*WA-`Cocjz)c>fo|Hi5F zED8lT%zLd4l$|QN*E_RIn8Kwv_iDLIdFr;W z&#MeE+V8=^i)mR? z%JIx?!aRN=VqpED=K7`07qK!&l-Ri$&!GI%grAj&-n~w^@M;yzzzi%lyq(b7WQVB( z66-Z-^{%`(X5h!?$gGJVcdz@Qc+J=-4MoUyFtGmswO)Y9sVRj{%Ll9$NXN3$4vYX%Y@tEqX#Q%<6FKprdyrx zEFhdM>Y5Rg)3%`L;un{G-14~YiMyUA`-J#U>lW4XAzsZRGoM}Gx(jI$>qO|oiWeHh zr!z{hDyPU$+=0ipl5^FyTUh4E`cPMR*sFD)BiCsA!e&uiT`s?%cuw13C+?|9tERYr z5dDCjhePx;{nDTjwSXmP&(OY31`0TNOkrH==yagg@nC-;O|u(+r4sVXgsPaScUY9< z*!6+MJjXy!A`dkJ=Ip(fSbfJgS>eMKLhdeCPgk7xl+%(v5tRr1meWG86C-@g=KHWd;6ea@+;Llz?KuoHWdcX29tzd90<5&oM3GgCxTzgL3&3|Cp|Mr(D z+H3(-Eqf?^8wo@gQz7Q}A6EGAmCGh8a7S0FpDw6=tyuomq(MS$q;AP}#!C1wx@EKc zA*W1PPBj#tK1nk+8z0ALEty#umyEnaZSwZUDB#4)LHljh>xpf_Nd@t$$k?8b;UJ zjh27tZ9zIlrZ%T$!)g~3=`T#QD~lr@t~CseLi!Q|rY!fbn8$f@E0a3FzE%4|+#0ac z0onJ;7BA@D`k1UQOXc#ob`0R}Im?g{YiTD)cv=&k9Yb!SXZAkPanQnG#4Xj1?jKP3av^Lhh`cKy&H8ZiJP{^#n_HU__-o*KGo zbBW90P~q^^)jj@%VP(H$?I#}?g~q2?gp1@u4I|MJvcnE3`zn}3?_k!_t|UQ=I61G7 zB!{q?t(`=%InH|o;$YWk+(*^msjV}^kON}2FzjEq&7{l}qGmR($^NC2n3n*uGvq+RP9%@c0AU5gQyZA;wR^3Vt$2 zTSr6r<*pgwvtNr6VLLlsn#?8ZHsO$uJV8`~3gZipJY3htXCu>@+2eHujo~o)BaT)6 zi0`AIM$bukAw%!cWiQy5b~WCYbRbKAv8!Oh)))nR;JgW2up+Gm=O+zfoM&d1nH>w| zRJBpMWNhqWpKqbB}Oz^N({#Hu5E)`c0?-qYv3kYA5;!!G!&mf3`;^9n~+4aA|2TA$) z%S%1VGqr*cCQewxR%sG>$Ykp7OY)Rx=oC)f*O^wBVGHX`DXSlC2bT4ZKwcL4mKD!M zlCLOsW-*Jxhx`>Tm)AyK4Squ|ZRX2pmS(HpT3AQO2UcV}p}oy{2j)Dm(*CO23(@59 zShKE(p#JzT-fZk>#@rw7oRn;k(+oiX)xlPU%p$0J!E*0h-|wzI9-~=03(vk)imnI# zOp_D6M|4m?tG7;TyyO8O(aN|RM8;3DHkMmgCE913lB=&HiLx|`#oFh8;OC{$Q$d2C zeU8+!!sq{&xtVkoqKD&Xlt`k%1wNPMPtm@7Le+;tl#+uJa*>9g%=pXxj{vnTO46h< zG1h{kB;z8YX+UgJ3C4P7@t|TpeFiF&*}*D<=~tfN8AvGEs?NxBnoJSRU22nC%1{=^ ze2zalqj9UxDwQXU;1#Zh^G}Lx@4?Spo(*x=_FEoCMj_Dl85NA%>~+D$*Fwv5tOAzZ zk-!zwcy`b1_D?Ao^DqMXDE|QU{*}sD*xXBUnQZgN9OUpg`sTaH;TE=0L%WF74Ep-i zUq;HQE}hC3W6qF_75W@;S@WEb00eq}bXCiz>yw-gr@d5>m2y}!dk&wKa<=cG)lGY; z72OpP!36P-dd-$5$=Ws&ITf(tC4{oYxsDHC^XpmnNh9<-Mlp#lVNV4uH1a#VeokL|j>liYYSpdc{-n8SoS3pXRMh`skSgqPLR%UD~G;@P8 zi1zS(>!GxAB2t5oN{B~qI&R<*)b*)ifGp}!cOd`)RdsC(im*umc^SGB(xC`#;4@qx zkYgYU(kQ4GBX>DI!l~-)zEM0Xh2-X<`f6z=(lM?)TMeiP5t*}r?N@Czb2=H7wvm&L z)x_!r1jSV_Gn|q4su=EMVzHy1qk*1lPiJ`CP?g@Nq5lAD%g>o6?hipiyprU(>&;~8 zaJh{Pfx2=lY9%rvE_RCJiojnbvG zY)Mg`ltkND8ck>n*WE={7Yl(!WnxH0pc3Zth|QpP~(0MIPF{cKQ^fiiD z5VFfTLft?eYSqAU#BtNFHA+u0kwW(4pW@=8QZLl{-T>N)ROQ{CK1_}WV3=LFW!lHJg*(7~m7W%$b=KW0Ul%pSrh`_fmdejdXWbcMOCxASaGJE1tR0?jChi zMgflptvxMbWV;t2o#u^|^4w)#npm!7R`S!3JGZ53=^h??OKp<4+`#_tCaKY&USnilKE3Yq?R_1{egM^;X`{MJq~VEOU?n z;)P+Rk$ z10HZVscz;;+!aV-t%K=SqZj~^Na@y~RNc-w%~^~C_v+)RGIvDd(*tfg;--jX9y`+` z2p1hV^{G%`0uQ0)o~Y!L(4HVm4l#~CwNqcXSZx*(N6(TDeJe&N#s)sM#`w2U^EESU z7;T9_Tz5U|XzODbwaw^JxYQ99BN71InEYo#zflbK8o?5ry5BVe{m7t)u~A zgd882Ao|fZwmm#O-nJ%$I9>)uIW>CNxg3G?s*$N-o^kqB_yAl6`H!_vLrSe!(Y1C^ zM-`zJ#t6ydHI*I#-M~G)>dr}J8KFMMn^Fo)*$fy04;5kx&93n!=|k#q9FPXKvv1?^DGgz*#=&&OK{>>q?4cI{*hAF<3XYa@vBg5!gSA2a3tX zqoy?zE4)f$WmHu;?cS45(*D(`A~t-ma@A5x0K^~5x21WM5Aj+xm@)NQKHqF zIWAVwqj4g~9x;J{1!Y{xxUSF!G8mtz{{TOQNqIXmV6ev==CfXX&E+r5bH_E!DK{(h zE{!+4B$7k81#IB{6;@eP7Tiv92TGPyKf=Q+)MB8PWK1zTaC%ij=Itb|Qk#CmwnHB| zIQ}ZnkY2e~j#mJC)!DTex00!r^fg}>R}!db@V!?9?$-S@Ii&8*%_7oklm_xqCPyGv zlIhT+5E+3djE;t*(XAvF+hrt>L01FcrE0noR5I?zKAEQM>}4st*rP4Tg@WgS)}g*^ zKu$BbWkCAXeQMT52_zZH{{RoIOJvV?ZdPO}!#|Ov(tDL2<|AH99mdWH10j8@KJwyX z3Ejfvk)HLs+B-GM=aMs$M{1EYe>k$S<2$(iRW3Hz(aP$_AVl+x<2X~xW}PdvN~@Ax zj~T9w^j{_iP(UTOdxKP0MQKa0&fqbeXCCymYi$Ls?qIsjF(4i4M(;zyOUZ|=qj-Yp-=$l7^a?p+4+YsGT-)B~}1|p5LV?0Hkt1TCjux z$*3{%z zK?S4C!~{hpkKtZxdw(30w8{*Q>{t>z*IVOT_Oj4Cw2%od93ThNoY$Pk31WeI_ae7~ zy7Vg#2Pd}Xji_sf-f(hqLiGMsyA*juCmb$5m7is8u>cv!Cy`p{q!0lF55t$l_oeVAQ}Gv^;3p-4D|M?7BEIP z9y$|E+6F*V*A&u7<-Ojd)~rL4a(TsUTeg1+#)U|H5sdON#cElxB=qT9Q+k}W*HZJ0 ztyilDl&B9{>SrPfZ+dk%_^A{;4mwd!SlFmEP({%4N6qx8q~scV9Chw085rPm+O%Y$ ztDI2IdLAjNx%a2}X^aetMHOb9P~&h1-m_=aSsaMlZ#X^cT3cvbKvV{bn->)I?BlET7+s4ga><^_RVz{ax8IqXul$^Hi6Rv6;nsH`z?W${{Ys@$r#1} z=~eHpI3sWbf(O>Q>dwl=!k;wKTNN%PW6t~x)DS-D0S617qM$5atoa9QH_bEW_E3=a69Ix%MaQgYjy5$Fe_ddn5!>BM;ue5(=&p8QV(BR)(PKJmUR-) z&#}^?X?Lmvj+p2LZ`)Ye#6zlYZj5RqQy(#?UfA`iUf{sJ0s0E*duVaV^J58W_Lky3C0qS_G7gG6g7Tvc!E0wy{H{AW5 z12ehCD;C?8j*3ve?AMw|=TVj!#ttY)fJfzzm4W%OQI%csp#gJ_4;35QMrSNoeCIgL zPNzyr)-)}yP_a}Sfa_P#hZ*QAHrg2N&=J!(tI0A28%Xx29UiAGMXrRhAlPy%ow~R3 zn7JgMrF46hy$C+_1=Xs{85>tVxTe+hL!(oX>T)hra(KsFQWq{gani2dSVOfWgeQ&2_#BOPkmFQ?b)C|hWAP4*=NZ#l?4s(4p2XM>UHPz~FZaqCv%AyK4m z6-HXFI-HW&;XHS#dD^Ttu(o`K`IL5JSz3L&V|ysvM^8$M;!n2f*B1-}d6G6hndA9Z zwX|ihq+kxXttBffBh|yxP=aNsP&aZ^s3IsGImaN>YOGr<4_@Y{8RYlrN~pt- zImKE`%bMGfmcnCaBZF711~H1NVJ(sX8@koVS71?F!Zv1AWUO84wFw-WnSSj={{VQ_ z?B+vh4_aD7){#Zni1OFzML?2a0L>K0u?ucJYBHzRtTtDIIvS1finbyQmFj;gX}DAy znYa}p0;70?0yB)_rQX^UtVVC*+FO2c=}a26w$r*$G1%s)UFq)_CN{=76_0R}4jAX! zvvX^7ZwhIfH`n>Y@MSV0sjEjtSgsdy8&a4IIf4nmS1MG zlJZRMnBfWTxc>ke&CNYc61D8h*RmY4DEUS)`By)6Gd!GqYg+bSmAwi4dR8o~0}^>- z+PErS_GhDqjg`jXH*Nr64#TZjv|?FvkUI}rrc0vXa87FNmeq43wn!X$3Q=}vO;3c` zPq;8f;CZ09$Qk1u&MLZHa8w_ihCdpJ&UcI)$BgIRy5QB1GEk9d$>pdy&U^NuJjt0= zWBGcQJ*upG?(RYO-R%Fi3 zIQFJSf+h1D{n6H&Po~Bx=qs{A9smkGs;ZDrocId3`Kvni@@vbn4XG%n1i9=gp0Rvv z)inEjw36@v{&g?R)k4besRxLBLlR2RM&4tmCnJni4Jlcq`%|c1zFtQ;#bs(%0&5XanC zHE{C0tnsq4dXtRSg^j`Eb0FL9XpV@Ql+#y6jra|2`R+KbUS^YfYz08>YpV{9F_3dq zZZyV~vY0uklw6*LVXCTnNatcM04&+7F)HBWkbg?MHkvJfE;ID1@3VzY-D@QW9nSv% zXs59}V0%>RP%vJ4RaikBk%Ln%LuYR4Nm$BAutrWtLB?ryW-$1E?c)Tumw*SU{{Z#U zDcrye3X=9OJ4==>31tJXxHQzYvCsC>ea}79w9hXL{{SnnKY*?4dwdQvf!3@?V6l;v z{{Z#XwTLSNgU4EG&sKU;s<~{4qKss4N%X6cA_oADooQoTrvMI4bL~?p?%WS*euot0 z%t)Xv2;^p^Qaj_HdWD#6#t$C!=$T5AIU}tO=;xBs<&;mGc5po@=uY4X=A()7aokj@ z2PBd^RN2@IraUbKx! zVuM8`6z(EMJ%6n+vB||qa!27&u&ZPd{JE!ygDyw&sN0HeSadivh9tJ_gU>aSs6h}+ zylSjS`GD47^?VUIn1t1RaS zEx4Msa^;C{Zg}Zc)k`o4zfx)Frc)a)ImFM%TvMR(C2WplG{HV)g@${;b zzUMhTbBs}RwCW`%DR~V>Eh`Y@XMxR4xGKpB8@7S-f1j;X4yxS%C$1{Iml7`JQS(1Q zIX?BgF0^FSk7J|KmP`|siZP7!$*kRT#8$SdEn)%LS7`&GBd_zV9d}ZXO0$ubOGxAHj@9SM z47Zo@nL}n{yXXxyX7$jf`x?{BG_3e6%blQSrx_ouIbwJ(Sw|*S1P11`MW~9+8Pk)2 z)30oPlr`1A+n}CB1fbx9)DUWPO|dw?nPFz!um+g?gk=b;*78LYW?X#P#s+I6Te*(j zV;l%Z_3QQNP+43_b8g3Sw2|?%)O-5U)$3x@Xz1L$T=YAG?KQa6OU56!a*tt&S|9x1Yl;a+#uYpcmz}ug#&Q|ZYXgG&h{G^+shNj z6yV=BdV%`Yc6KfIhh8aCOnM*2i=P8E!>Dj81qRYSd(a3i4>x zs~Jl~O0n_?>^ZBpR;J=ypSquVpn@imf+)#uIIY+%*4`kC*Qq|Wv=h?jB}p0AV{F(1 z0~}N}YW;C0YPLA8xV;Pu5gsWf>qIAvQpzg20UaZBpi-WciW@~G6u_B5C=azYiS+GD z86q+DG>m(CR*cMqoD-UjrcwA*%sI|!q7{A#`qXMtv6Xo6OuKWtpGw2Iw?O;U93Sgl z74@n`8Faua*0VIIe%pLya9JYc=b<%}mbx8O-j_9Eh*?;KT#yD@v9Dz%_$MLB$n^gJ zCbhL0S!3D&Ap2Gn%o`{BzTH2@xhk~ebm60~GbuUq3Fo-y)~wq?+kpe8CYZ5;az{Du zSJhD>$m&LSe@e+l=WOMrkqyMSU8;Bjvt(>%2Y_nD!{A^Xj9^xL%v-1BS+>(Oqa6s| zO|PC)9FbI-i5~H)clp zaoW4T4Do2}?0m@_mdWp4X`)&YbLX+$)Gr>@T=Fd9?98i+VABF~^Dz8B&-AD@+2OOXJO1ba zK*wG{U)HsxMcx>W6lXQZ>T^D=t|hc;#yI|VIpf>;R&MsOyk&cN9PXnH&Z&IzN^lG0 zW2QOj_*AwQajndo5R#k$(*nAGup(PIz|PV~9P(=#(c?>r5kLx`I}&*~9FOHt*`~yL znMiCJ+#SrUMsPdwYZdo25$*v{f-*Yo!TzF_{`Omo=;Dd^VYz|z&+_C^Dg!D7Acl>% zj(8acp@#1;HgX67PJL**cl$6Y z)sUPYZlqA1levG9qiZX)#59?G#~}XzTGxu}I|?y;@c!P_&cw*(JB30sfWUn^*3Flk zW!@Jk?VMxlSaC1PqXJQ`pv>tfhuQWZ$w zR9bsw$#p#TIjMZI@DSZ`RUEuTGO=arS0K0&5124M^wq9oB-NRoVZ@kZ!0DV+GDH>0 zz|UITbxpg7&*52jE_}ijX-ypz>bC4a01cCXYE*5w8?%v$fs1Yg3{&@#L1U5aSfr7( zmC!NpP?D@RZ}lCy_O(JK;74L(uD zMG5AnlkU(RNNDOPqLD*rp!K4F1thNry$r`WrlSQ1G}n`_YPQI*k0##4=BmBZVQj_R!2waZxceryOn8csq9nEW5TP2;UMg|y+vGp~Vq@!G3 zmt{L683c~3dVf0Ekp6X^sa&~f?xQ;T#^M* zDY=0s3Jy8`b;&JTJqj`9i8Z)9Fidy<0D6*Ma#;MMpcDXC2S4jU3J0gZrA2a1(da32 zU0AzbMM;}1+#LEl?t$*oAHUC6k=jPC1H;8M8@fK^Wi z{QFY*p=ZwHou?j#t}?r^l%m=?JwEa6qeXMnb^6xLz3Rzt=45fe;<(8!5X_tm4sr)> zD{DuYS_dro0EW-7&2_?8d!BV_D%UseHyddR^4ktSD$2L8jsY{WoXo%vek(=`l(<<` zFyNKPryu=#iseenZ5sAD{Do}wI91k$Zm&PttPuv(k=t$n_BG4PYxa9d*bT+<0rd5* zpHsJcxY&W6gza3{J*lJF*$DHNkysu(;d=gG&a;;K8>OMlv}5g(OvfRj1vA?LNAsz# zBQV-70o+2KKZhg#0IX`&rHiy^?~%4Wu1fL=!4&JOc9Pr{1T=~=bJH2B(XB?dv6DOm zEU24)^2zuC%}$Hv+eUs;*gvmoRRxw)^D`AFIPIFTWo05ETwsMDcBQTE8n>YPIEtts z?M^U#PFj*yvI$~lPn4Y2bdb6>PZhEU6<+Eh z;1PfU=BjXeqG`^~#!HfMkVh35by5Hw4P0bgmSNET6;4F}k733uo!y1ezlj7?9&tsrbk@h{b^!CSaD7YI<`WS*Es(GCal8_ za6l*9HF2^;%2q{-J64)Nz1aSh(nAtkL?Vm;qra_Iu(kU{MxQox@|$-aavWOO|;PkPY0)S`%CTO{-~n{6<= zy;wq=u{)Uh3d&oX!s}C4PPR6-yI+?lB-FDu7xC-qQjzJ>s@*xp!JmFACiDdFE14Ht zK_GGHb5(K}0K@K%O1HD$1lDG@g-luKE5^w^Z)m}mr z=YOVZWUZmq60D1Z@{z#lQQ@)<0NsK)sT_RSExMRNAeJK|iq6cvN5;S_Zu{8lO<%gs zKsn9_twX%4eK{EKNEwUn2PdUz5?auZu`(+a;ElMe_A@fuOfug#0NQcVveEhHJZHT= z-d`^fe9SS)&0Jed#&K4%I}HwO%_NL+r+^RTTSXQ^u0Y(mBDqZ-e8-Z&W&ZAcfFG@N zRxZx&Pfoo0V!Gh(qdcrcEbfZ;M8|6E6EBq?*0C=wq@DzlL4uz$jORYJ(@hz80)g{0 zf_SK8)4+oS7ic-lfb*7R}nS?uj@IKsD#lDCDi{t~2a(dT9ma)gk z0~iDAib)i+gi(xj_N4D}J+DNJtC$3KvBMd`+%tj5;-t5*3ga0+E;*|nQ%O6L2^<0S z6>T9qvU`ecCWk7PS1UtjwoTk%j%w0dDzhB&PDM%p9S7%50;xeW2T`^nC|uxmr-4Q> zN-IRPGZawky-4<;Q-?|#y0J?`I!kv^jBXW1NSRo%Qa>ImW>wF8XY{Q3fFomcIT@~L z#>Y)5vYra5-A=$NSGQVLP+7MgpOq|0<^l-8tdw-I(%zR6F}UH$BOrn+cS^PsOS!({ zV2^WDA%kh-81$}^&fGz6tBe!Lt!cvY)ogNBpF~giQc~iKS6Rp>6k?YYQvxZMD1McY zMjVfN+HsXNjVQ#dJ?H@!hO1pxs*a+qC*7b3h~|=<{{VWMQczJv6ahwQysif(vaYU4 zE?5#V!2D}Uc#MoVz#|mKvx+#*a!|yfJkd3O`z}xwhfCVk--)XnL`ZniB`| z(MTkP89wz?%IsTr2RwVxIbu~5_sd}R=}c$bQK|h%B z76%8mIi}vFOR^P_NDN0mT4$I#CzZkp+D zAd$xH=x`4_*3@^+1Yo;k7&z&UE3T7CU7R)PIjs(zRQZJQj>F!cjMZB!lW?sk1#`t* z7%P!oF|?88NjSSPg>o@UMi&(7L(Mx_k(ka8Z%PUSk?Y=_ib_om{e}bqy}HzZVvr6- zX-y^EY%xkH2Os@<1v`>Et|$}_N>D0*QPP|#qMRrh9I8mF5(AR2{{VefjGuJYT6r>Ft8k~_EILax^e-0mD=uy$)htvxJkX|esP zSi#4su7KmPto;t%RyLO$=V<&o){_ovu9|8(o_yWgu*xZMNm?0&B`p-dYkk_sxEP$) z%XF-p!HKg#nFn!Jqx-d2-mJyS(gCs$nww9Mb4;YlLeWJO41!RhhqXg2sEe0iIH$+} zJ$U@u4)$&-$J2D8-d9mf&Nv~ zHyrk)n&mZ`O2iVsRmW~mrDEMoC1Z!G)6R%PrL6EKqJu(eA6v!Dqgc5W0>r1)GE6FE0;}i|f zFh@~>1y!uX?A422c#JHq^AK==)RNqpA|$Z+m>lP|XFr+3uin8wQT-~?0p+6S8*m5W zKl=52+E~iXGr7|)6o?=XjHcuIe~nm|SB~BGV<&N54?ruM(=Ln`+k%avEu%fh`4q^m z4aTI9$lL}49^L-6x~Xd%Qo~+O=XQ)86}jYR=}tA5q+flZq(>)sIRo0YNUo&QS3Js* zaFv+Ow3!sRq~kP*1*Gpwo(L4xUsFl9U=E)A)3Zt5j@2=2WBbyIU^MV5&7SB!6V+Ws;+HEa=DNP?Sb3ctV9%z!>H+09PWI9 ziq+DHdx+KWS9k`pw?nF;<&Dm#PlUr`5+1vG`kJssH~HQnMrNTXW(vQ<$@j%Bal5DMdLy#m7qFlRZi*%Ef8PBjp{vY7)Ls0oxpX+;dNwMOVVH$lw7`I8z{2 zT%HLzted`vYL0;KlaxC_$owj?EV&&(?N2Up%>6n6j8sL{K~PWSQ_bAfTa;IJ^3FlP z{t$htGDb^qp!DXK%*i-Cx>QUDwlI70KT1jHFG0Jo;3?cWH1<2bTy4&H^r%)W@*4yW zPkL8kuByO|qdXq8@6>784{oF79B?UxK|G8D(u`zhVdv1(U72#&1orgenvIZN`VCZ4 z=jJRq8R<)mutD3X6bH#6S$75QF`5oa2ZLUD1mR+70ruqATcGMGb#5Wu z`^dojYqpG)ta(`4PTc6$G}1B6KAL&02;E$ikg{Qyz7BcKYgsDBq112)HJuV9+gXNt)|6l@SRR#( zQe8({T3G8NZSkvmfN9kQoA0!Ry;M~7*q(<7gi{%6N^1%LfYMZsDk5oswMz75GgeCh z-m2ZYlr+T3Bc)3ZT8Vp#d?)6lB6l2SqN$ti)II29LeWJOiYv*C6;fKWq_mnwZ{kzX zaazowbH!GgM1@N?;flVZh0__QL*!ZD^2$p{~J(U$0zL21vo@AC*1Z zmM5HgR1z}*ouK6YSgCqyTDA2J$^qGd?tLmHEw}FmImp_3Qc1L9XzkO~Q^o?4GCc)0 zm60oZ3z3W{z{ea4VU(Fmv+fw<=}MW&7{}MOBeEb0K^gCgX%B4?BC@LUpI&*UI~LCv z8OKl7kSH&m%|n3i+<-XurkMHqsmdf%Pg@a=`80U(XBjuly=I^w1sFpFj zmB?)V6)bo!(1sM-Td~D0*(_iv2nok}*wSYve30dBu#@&Tt!fMr$&ByZ$>nzerlnX&nr;( zsYfeO9<(q)iYTIyLuKfGm5NkNdKOQacXTvSPFs$a=A49qfmynjNd5ZJMRL|Rqw$!m z2b;IH!f{lmINj|<70E5nW(6r&?*l%S4Z$IlXT20wM^P34UOj1w0hA8Joc^>?N$d=X zMq>_s_HmzTmLZu4Ju)bwt(mQc$IPQW$2AOWLons1aliaZ04_{15V?ikI+~$3_NE(wQg7#IGFDMMQ>L zkMD|=d(lNw6t^Ud{Kx7)rD{yEKG$fzM%l>l+4hbnK1vv$nk`gS-&BiVJf6iSefQkr^2uK3rumNzXaDY@ecRc{+ zzkTB4{40R}3=S?351)XLh?s=*Z$Sek02c=c#Ki;RkE=e{+ahd%Pzmp?yF{&%nv`h?|F3Ok6@zN?Jzcsj8Z~hNhOj z!3#qpV-r&wTRVH$YX?V9FK-`TKmUNx4`Clag-1lj#U~^tC8wmO<>uuV6c!bil-AbO zH#9aiw|xEH-P7CGj~W=9n4FrPnVtKIURhmR$82nFZSNc%pPXXP&i`Co{^P;{0RPSU zyZ#&YKU`FQUATC7Ks>^KTsXMCe*15yDL0DXu|2$`F=^Wem*lRYQh z@7s9|&5QnSDJy=FqHoLTP*#`04e?xwH2R%SNg!EY3}6Ro?Oy{bGlEt}X@5HRYDD?c z-I`R1&+Sbe)~8QY_vK!8hXD`o0N7niCGW?>LA#cj8-^~Ql4Mb1>xA1cee)eo<`&du zW!&RLXU`d{17HQ-^0f65x3)ueV1!s4&Q(oogvQ~%(X)Iko8#8pl-QML@sNyvV1~=5 zVG*g6v~{d_%%oxKmz$LSo7^z%V!n}9*ZxMhg>GH3I_ZcTe$qvfl-BJQWdSmF`=x+j z1+SxQWp-3{vL_BDN{X!kssBgwvy*4x4DP}_YTUxK^tXF9#cf~3)@$d66nn|WaylBm zN$Krhv!HZO^5k$a2SL?R&WTs*V(>7{2jex@mdTD4E_o_$-bAhSRN)4xB7iQutS!Ez z0fqz5Xg8|pV+8X;?8?KB8K8=GTLPjO@Y*`)zCdl?&*-<)QC6DR5*O?GG<8Nr{7xWZ z&agu4LmKZ=uNzHdA6)|8C~nL%Lya`G6DVDBvn4-6jlW~9(>*6#${JxqymK(~RB*$j zse=Ppi(XyNbhkivXYaemRBv6NS;T<@x8SkRa++<1H4gTCkm`7ERkXrN<@`5tK;Q zJZdI|QkaTdt8!s$m>LkKjw-1Bwj6ug;Fj;ht{$gv<1s<|D$FXFMv^6PnPIc!0lg`v zcM;_5I`_v9xAO?;9M5F`9vxU#|At zsvA}7OkU@p=yRic@8)D1;&`m;S+a;q2Jk9IQ}18h0b0R0v13QM=&&JDUvI9mvzA9D z-i%=%)M}!*$%t87L$b*br-x~FrTIQW3=des3o^p-T8z3bDLUUgajb`4NUA0+i)s_l z6*B^1*=u2UfZfT58yi1fs?ao+N5yBj^do=9<~fyPMpHNBKvQu zN^+!0KgX`_jxn3gj|Z_onNsa&XyELr6}o>ZOLlrzULmu3z__*G5A?+O@!KD8+t5)JlZDPt+sl3ymN5y)c$c#2$y0fOkrwPdM(|`sceYl;(BXFC4M2O zW8!Ox>e61HUJgc*R6dnm8J5Tzf9v1Qdps}RBfv7{W=zBBDfv>ytMuz|nFq$;l~9%A zmK^=cPjBwb?Lsb6kGEcW_#vokUW|6?mR3fSv}VOi&ev%>uhj8%GH(-l5zZ=F$&y-w zos}GYx>EZ_;$}QK9nI8@7IcUGDr0wmm$e70Y#@G3CU|C1)Ascak{QAy_;K^&SUP+| z`WMHCI^p&8Szp>rtp_%vGCm*-THyMu&Q6!W6X{j;qB?zKeC>;dNj^B zlm)>Cbhrm3^@QAF@|0X{mG-gIgWZNKhX)1l6RJt815tgy%sEXrH{PJEZ)oF(A(634 zH7yzLX$j*=Q3}6;xCPg)mZtkH8U)C#YP6pwmY+^(4qhm@xD1;o+s~~rNslfBZi0{} zmX6fg;p`7>g>&p&m3b=_az*SVW`2UF?zgF5MK!OV96ix=SopzeH6 zg+Y?c%+Q#uz1#P-J44NLIuV@3ujTawlrBCRgPTrETpPIg4=S0Kqq{F6k+={1@PgUJ zO%JdYNlaN{ATvVE4Cu>E6et zBjHEKk8gM(Y+Rr98Pk(z>n`L92~L&FCafC|hN znnx-Y_1`9($@42xQ;&tCs+ecKav%B(7y1v=R{3+=L3`)vu$u*eqhYzG+X1hdA|C_E znp&!;Cih*P5B51?H~mQlSs?>V>c>_76Rz^(c8n({-0bpe;!Bl?;d7}hCO<97`YYAM z8o{xv&h`w=D$!{Hwnzb_*ypKxmRjd-AWKpK>L9Qt_wm|grsy9dM1K+g$s$^h0wkhS z?yg3-bonsw66%2$!@)J7!Kkrs;r3DdcT!!qw|fY0Na;uv+1VZ7Ng0<5?va^aL}L># z(f+QPHOMyS}Y z=qrd)bumdRB-;7~Z(!Jyh&ioXnnYNo@Lr5C8%?e(b=bnbRN-ih;kVuNW6T>CM|ifB zsGBaK16JUtVtB|Bz9I0v;wNo<3}zi1)qn^5z7#{}9X&77Z!8d-AL)LY!l}rFJBbJg z|4cI)3-i^V9ZTy(rySNJvib>6O0>%a*97vkqjjY)u-EA4`NbU7k0vh-_p5$Z2<5WS zSsO8aE&;qb(8Cy0ne&^*4K%cChPv<@V7D`;i>!Q|laKWLMa@M}`3L4<%<}3l{aGgC zIO3KplnroyG}Lns=h}>}_0z3^0dGn8UJt=M^M~X-M$-?$l>#Z16BAS?6xXL`xpl33 zICgS}zC<5jt_Pt~sLCO=sne-RA?x;(^awWY5)3ki98P8Ob;nvPbA@ey7qw8+U#T$a z2&+v(^B~R5A3*g|^avj@^_w$vr}5ZiF5 zS@B`HjTO%&XPv_ypu$M`<6JoGV!=v)U5Rl~=Eur}tYo|3??xd!4WIosIhlGBCEU zbqJRVgmaY*;H`*WTbf(0{7x~!n`S=5VCCySIIndF$ef;AKa;j$ObclI>{jV-yjTi-k;5R zGN(yJ0d`0sKn)li5V;l1p*23QRE-JhN}SKU&Dc#Fe_v-#>{aHkm(I#%AqDnjuVM=D zsnj}ash5z>uzy`JQ;-Ua?o?&T!J~dG+*LG$(f0IU@Eh0p99@~;G_;HKEyj8=&kIu1 z*)W=6jL+VkASNozpRS&{G|pIx=HZiuNWTR1cbpWA(^!?UnPVhEsu1yWMmkJZ?KSC+ zO`J7!i{p2IuH6J=R=xHyZ$^LEmV9E3x3@1DLRur4o`fnk2<&x6qlr`D&pZ-PXdff? zl@3UF$hEgd5INX!JLNR6W;oZPJnu26@tlyuyBCxtl)@O0Fbm0-**2S`Qw(|MIh`@TElunLa^QrzVEN08y}`N}zG0M&;sjko4F{WQQH&Vs2X=Of}yz zZpQ^vHK&t*a+WA9F0lCP=iW=sSDEG$gru3lp5{+Q#qPBbQ`{q*+_nIH_-$7oq>iay8 zBAPm#b7=XGOoN+4h-zY}U>1YIsPakk`h<#`c-eK$fu7b@MD*SWx8Q*f`Ji4q@5BH{ zx%BsG<8PZv$NSfgQxKGN=d~ls6;p`|e^C$*q~Am%4b_&BBriho;uT~0{zQPz(QBzF ziRGA(bS_+!OycN`*vLf+K@<+2^%fU z5!?tJl@n=_tA z=q$2K&EZOi`;l;z4Xo5*RAU}x;* ztVP7r$fw30BK-%kyUb$q<&0HTv*7@oBBF;erI_Iu? zZ@;%Pak`or)Lv=tv%4-45+mRBd-zwHbstsCw7HQ+@qsCb`!Nz&M&Y-i z$HE$Z`ukeX)Y;*R;N%!1pU?8T1n8qVW_3xvvj&TIqrrCozT>r`Ytrl0w)Ecd)cO(w zvfsha17;&Ny=L%jA#F{P18pB=z`x&=@kVZZ!R3dw+17swQh}pe=)0V~<gu!Swu&RraFlJ}=04sWqKv-rmFI{-i3y-HQv!SM*cHnyo@IYVN~Yy7elz6t_xz;o^k?DsCtov%sigBI-fJG8 z+1WQV$+u=7kOm@e`Jr5r7gm|cZ#ew3z6W1%D&I< z7qGU6#wKLLs6*a9$6b zY$4n2EiVrw&xGOLWPx;CUlNh-@cM2X+9;GY>c&#?kexF$tZKvVDt|p#7 zD%mt_^e`VRz`?mE7hpIV3Whsw4Yz6y#rUm>yO|_77zy{g=78Qk!YzO8(Ocr^Zq0F` z0t;TCGH#Jg%P$XTOC{iZBEoBe!%ly_)7C*m;0cvpJ2X{X0Kso1pY7- zXCEF}-us+<{*13U$mO|4*Clj*2YLXsBFXuZ`{I};~+{xnrdv;l=|fh;*?nklzjHdj0H$_5^!{0XTgb0Y#n45j!szv z8=7t7YJ1R7Q`NS&YHnVPGmZfK?Jq< z>kOFSaWRS(n@|h;R+&5sQN)i~;ouA;0VpJWZoTCYDT1aeQc&U8 zfCA3h45@JMF@g$5E}b)k9N#A3V%-)373sjyu!>256i0nZEI?$$a!h}af~bP>-_g4C zof(XZE;R}!ia81Mb3ZhaT3Rg5E#!FNqOQVwv!qzLXPn54f)p(9Lwnja?1j$jP#cjv zDEGP3E83fhBXpK5MAbPe#!a7n)Vf=0+qh}x;pn42(KPN#hijcB1>%~OO$BSn&u3nP zo)6)wph>2vmzzSX{)-a4aV-aYSuq>06lNL$QuM2le3^mi>Q-?J&-j*60rrLH5jsoZ yYH}I_F$nX|8I6=AxqvGJcZVa=57wHC=tyoIq5@Tw&Le9KpZ@{5`B9tz literal 0 HcmV?d00001 diff --git a/python/nano/test/resources/train_image_folder_jpg/dogs/dog.0.jpg b/python/nano/test/resources/train_image_folder_jpg/dogs/dog.0.jpg new file mode 100644 index 0000000000000000000000000000000000000000..553efa6b4c8ff9ffb55398c42a9b2c628c07749d GIT binary patch literal 32053 zcmbSybx<79*X6(fg9QnY;O>^-It&urHE3{`0E4?E1O|79A;{qF5+JzyK!6Z}1`i&5 z`F-EkZq@#^yRWOes=Hrx_v!9;@44sR_q6o14j@pHQgqSGaDCa;FdH^aR3Xl-xsUJY| z{G8}0|J4Bhv!S2@(ao$A`b#<h#M0VacVT`!5&%wI-63->TA>{qWz$;g?QSyy z>*(se*EcY@-|00O9B4_uy~Q_zo- zbNDnC+f-Z}G9&0uK68lP8S}6VJ9KF>j|&&DV=WnlS(p#e&PR^uGCmTIeY#1RPs|Ci zYa^3f2*i*l9a74b7r?bGHXz1TZA6l76b}eQu%dEm!Ta#5#JT_h8zC%%vSL(hZpd^> z7@+##Tx+mZ1TkEa#53`h%qO2wepdaDeR-!&OGviKub-ZfDmgqm6bqWpikGZEG-!J# zRM~Fa6fnMb3n;f_IZ-*Nn1!og-?_PeInY*v%oN>fV?LbG1n)iXxIO`t0C6Ow_vkVn zYub3c3LBO0gekpBcVob#MECR|46#P`oWF#(P?q6&RHJ+M%99SW5~S7Y>Knugqc#@s zGi9@Dsy`?RJb7pV=hIIBQ)F~+WTE*zOIF-`q>CZFdy+e1)iqd!JnLptAJ0S{JOPx+ z6(2@9g6%c3i9gVzs!x#e>pHVYxZ7?bdXq=j3+>$L4P<>j_Vdz7pqgYaw_qvfB;-5^#oWfw$bXFxDfje zDBa2;eF7krVBkrP$vowi@VkSyUywC-`EAu#;o=QM$|kE6bBo7#7)1;cI|HFbHE1Ekh6@oWUY=T>(A7mki z`+nHwO{T~3qtWD?obRPEIu{txnhS)d3N3GR@UNOh(st}6>d9M?t$T^8OJnm^+8j#* z9;|`6lv`VQcd=rJZnu_y9OmTWdnoVbee9;hVW81;X`1?48n)pJ`pj)-@tW=TI$^vo zaN%)w9Eo6<8OrmUDPjef>G=I+9~(HkjrX(o`x9LHnyZEVh-{PC*SrI$Cpb716Rz1P zi(d-&qUCYkrPOQs7LaN^HHU2YVmQ~J=g5trl*fZ?$ef6RZMO+4tTN=_Qe*yc(2Bas znE$)F0oS8Y6OrGCv^#8v7!<9kDRJY&p8%Jal^fMl;4Yj{*AUN*4DBxcq2jky9%QMR zZknl6hP4Np_1;D321G0*z#6~ib)cp&OA|ov$C0dRS-v3i65Ma%4%K z{ySGb)kDpXV*_Gi8_91DS#S2=S)(tFXWSLR3?j_$?ayLcc-h2vlB6=Ss+@SsJ#KZ2 zcFLLEWsTyGr+I$+7F&IN_(K`3h=L?_rJZQ;K^%6?Oz>d1uAF*7_0oCz!2`R& zPrcWxJ#MRu;ZmI;w+x6;YBJid!AE9OrdUdF{+3{b)96FcV8#7_dE@G!T zX((Ep!beh#u4AQp0u<@nny>#d&Z~k~5(c$+=71!^(Rt|N@<&odb^{*l#Hly-q>20X zvZdEk0Si4L{&Qje8g^yH_3SvVY#Rh(y6}_&5Hf5Y6HR|_Ew>Rkq1tF!i~^N1{$zv# z=98Op$o|OhOch~A!$z>$RrWi+n2o&JfM4AYof~jR8D=F!4(fv>jmvq?Xi6nU{76(V!3V1|L&(IOp4t| zwpJ+PH8f{)G49f3jZ5jL>l=pFlPDszM}}gKI1-&^xykWaewl)jmC0kcN~qC~dPqVA zDcYRiOv+Y^-WL8~#RwWq)P*E4R69|17~-SN)XT@ilefN$uB!H$z9?^Ap*T7GKzAnB z9dj#i-EUg9?L%QmZ||OpUEApGTz}~%)x!%V)LJx}Dqy`$Kt)p<=`|v7d(go@+kGsx z9JWp2=(e25We_jn*13p}5fx41X!J8+5 zL)r5{aDM_MDv|n;3&zZTCZXp&@#&aiU}qynEkVrdV9|fH)Sx3BFeGQqPn|IQbpN$t zB$RGYbpCJPD2Bln%$}627zDG-0$5UJX)4%F5%<)yG8ZK6u)gP2Ca%fex}ecWO3TW`QigDcKGhfTICYs?&d0aBA3L z^>JeCq$B&aPAbh9xx}n@Ak2>pybVVVge>MRh91d#zhpm6gMBo!Y1T3#n&)t zfOi7;mb>t%$Ommxcg5EDlDu8Vh9T2PZ9Arbk%-0jWO9@-)L+gf&{nsqtF*5zh}bO2 z?`JMRG?O(3@{BUs*_Sv|<3FbE3dI*y%Z!!>9Ermir4!kfK_){C7aM7$NXOu{eo|lm zKwh8wC&2AsXqiA=tbH|5a0pMsrKeLJW0;&yV;+$cvRXa z?~F)kHs3p6v`Slu1f+=*;NhPkpe1?CZvN@?^IY5%4wIj5i5>+wDEKRBT!%U7q3_IGBVD|0pmW<*#vWrp%ROLX#$s1DmNCiGo*&}rm z`ZyNtYA3l~WO)R2ZW%&&CuHHzsM0nm(D2rNq9Jk7`+_8uerwBf!f}tV!C<(r`sELk%04giyOhRk9YQ-v%T2WHwr~$p#&2)u8 zFRQ=;8HQ$f7ho8-^AkX%7&W`Atf!?|0oI=46J<){S*J4%&6??2Lne9bWsScGPu|mu zfL%)#A*Sc21);hhkB=6Hxv_?%M#|;iv+@vI?hj^`BiNi`*L}*bg)|1Pw=@Fz^m%{% zRKG5!Q@7suimku(XcJ9781A1ndq5aRAJcZpf?5uK81JKK#+Ti^EF}atXrWILPex9| z$ZG4xUaGfEDAcXP{Ax(cH1UJbF;2sbUg!)tWWG$*D#jVWfH6qVg%PRgl##Hqy>pO1 zQpeIl-^i(ua(F)e9alTmO&w=yT;^bj79K3g5{DE8(A^tss0hbsCemo;;`M>-=UgbWkGqPpiLTb>$xagWNgA z3F3>58ZjM+CfDPYAaeODhK&zgrd`@?QjF;6*RLv9a`UXqX{CT6P%ixZ2-$^tV>pWB zAgyga1=J@<+*t|_mB>qUYPyY9DRKxEDEAvC$yiJ;hY|x^H1LG*AG>!Se3};WjNs=A zyW&J6%#D}6h8s6U&XbUKk$axK$Dl^;Miaf?gljL#C4aP{_uPvdDU8plx~%JGZ00D$ z{6NbI)yyYQC)E@$iU}xOM9_S;F<2TyA5pZ9ohBHvpCa9@QY~MiO)$qUwQNK=3*E~f z4XM3*5wt@Qof@0c7##x<0o}(>Hg}3~9w+$@ED4)svcz}HUEUv{uBck42CPd?jr!+* zJl``q2quXbHdpz?*mfK|495~+CHc{OxzhNcI7}15a8V&D63i51C%8pl?JIm>fP9bR zJ}YO;76e_af#mV%(P(7L>fX`nne-e(&n)jM@)67#v89JDG3Y9+uwOi!PXPaeA+SS% z4}HSQ+9%L)McmqB&z+9rrb2aU(hkuZ)uE4LWa5?=5*HEoX%tm0a-fO2(Df(4D^|6Q zhIiL#g~Q8C>+Z#I8xZP+Ec&#;-&@fs(DOljz%6VZrf)e$%PTF1xzF?O>XNEL;&iLN zy@%R~b07krz3T};?|TX2pUeTj2iey7vGv2SPUflEIZEFKES+Wnw1C{02YL+FiAonN zO`6BOEfmM^BQy>!4jm{W``-;0hR7SU=}KPsu-Mc@~dC>h+0`3MukeR;_e_*6t(y4bd?@_dR0 z(H#^9DRtgi6kBRG0eB)Dn2qO)!d0w?1jJBY;kWF{3Ww*-Yr67qW1W3LHt}zuw`>}` z7hdW%M9wJ)uH6(}u{M+X>AV)YLfU`xMfLZbusjH&yW@{_whwy(^zSx7c~u$D$e0py zr?k}hnV=3H)?6QkWPLYse-}1?au)v>{I}Ae#)W0r7Y{6oU9+6xxlxW3V^HJy^X5Zi z`M0o;Cid{y#tAN0g@418^SwShiLZ)rr0n%&)l%sZR1kJunPl~UR^oGi!w`*=t4OL zf>?Nsvc-5uW4sz%daWPB+8*eIRm!0JpJEFA<~uYLGvyVCNlo_MjnLRx^ua0Jv4(yRU(`UTX_>TEzhD-A z{&}4l!5813wb$tX!+b6!VmUYx7?Z$rvyR6%>8EAQ@jAc~O6<_=)#K~N%B0~;=^rE> z_`*#Hs(R(~n_II;!jV;Y%53QVy2`g2eavKvhr6&uaVtzc*i6+zAY#(I*5Ph{nUBoPb_%-?2=U` zb1w@56ncwYw?TpM`HM+$)hrq~&2w8@LjNs_Tr99zhjS|lWIY9hj8>TL-NHmuj{H-*V=14VVFpjWD-^zvOx?7)+loFLNbAZO7tZS;lC2$4`S5!x7co||dr z>P9gW$38>7#6smV1BR%tx@aRDbqPE};+MO^S6ddfDz3`Ks{=u(MNSfSOT@|Dh6+}v zc-81l_s3VGSUP86;X_(($!m?C6t>ONk39(yOhUMEuW5s0xtvrnM+pOtclqooU039g z^dzY$%ZGTICqt&yBIQD_TL{J{JfzMsP(BP%U#cBa*fB=@RN|~UA9l-kWmoX1xNgu` z%X%;!g`ta~^#-?w8{hu~y+NZ|c4h`!_BuO}XD@h|t94+%XxJfRi9fH)#y} zA0IHId+*;0uYGdYE?_8u3}2TuS<8j#$gP+B%W!K99lnR8nk}0QCgUbB%Jc9M#jao9 z|8;aqN<2)tFV8b$xvNOEycpu|XNt*oW?(oz+Va-EtT3A3GS(&~h$bQe09Wby!u67+~5$nNAQX@f2{g`pgm; z9&c$tH2Amf#avwl@|@T?_8)m2SG)00V_cwJTh3R@im5iOxsNG zEz@ylPa7&$|1I-{1?C<@Z3m=L@|lFHAp25%C)sre1W+Uvc$v@{G5NH=`kg977Co?y zcPsf>QvG57Ub@-BzYe%8q7edEuI0U^ZiLmAqkzkL0QPY0(+PCCK8)NrRxQAQxY-5V z&IUtlQDBq9D_haM))vT^gzib}U3%<1yTL4n67Dz6#%*mm;Ot`jCPAOx^2BoILoS4U z=@+k2RO)Vr^uQ|vvgUj<$duvy;yq>acjeX{#yZA#M2wHRiH_c$z{sMQQJtVYuQRYj z&aT8|Zl%wqop3#sizJ#7^TS3M75GdIg1h;*G7{mMRAw2L0e)tRdeudPyZ%E>?!4=i zAwgZY$ag%A4WBNB;T!gas$4~Gl&38i1nbmQ;o-MsRSG9>I5%Z^8U_;Ji9KjrIq`-y zC7lum4PM{56~4}H%noj|rTz1Yx>{5TAr z|3NMok}+#3Z(jbEM=ouuQQUr*rl0J~8S#;pS6c)v@kgE->Z%M$2>h{0hG6Iy6nfPwbx(bJjy8^{2Zcz+!63ne}HvgazxkZgXd)9*Fl` zZqO**_uKLI%jzzjd2F{^WsDZ~6y9lr!W)$9F=y%y2gmA?Ipw`eZ~znZxn{JP(0 ztHBaLmQ}3YXz_Ob&>DTabTRe!^3lWiB}3PA(ZWYqX9$V#vQYxm!0`0dB7@Rgl6o@h zEN(J`_pRsA2Za6aJFtWht#!|O%(mjAEq&+Qhq|<>e%`SiR7TGAuV%LYzSNN3|Dn-l zU;Li_UP)<;`uFwXI7P?>gUD4GV?#$Xjg3K))N72lKIYGc!au^uv4G!YzqIH@4~Bc7 zoWn#(>-3VkBh@#i+{r!fV|4YuM+~Kpf8l!bSxDcDt@Gax8woTo@s>#cK>zJDb2bfo zMq7S#M{_S2Mza#Sxf%*Cme^JD_h%*BBvfG_QC*E&6j^tiw#J)ge?`)O1vYoktN3#$(N{L4^=%e zsMnuE!dS})r@Gk#%Lg-s($U&RMi=yiUu4myk2bVAZ8}*kfMeB6E93e)Xm+(rL8ujy zP*Sk}Lh7X8?!%@|Gnm~^7N16FZnj8vq=a<}81Gj{?&k`&&n^-bcn)&kQ;E%@Ok?Nq zv*gXcN~o(|^Phv=x0RvpqNYZufm!)Asi@jm5johkiJy`iGCQX9spwfRg%!Fz6!!;5=bD5-mr2y8$NKePV08A{&e z$bQgEA=V8HYa0>H%_^^m6tg@Yt#@CG+csY>Gb5~_^*ZiEpPkI`&tX+01$S~k6!oly zB@nY+{sy(m*>XL7cwfK=<0qLL$QdSVKa`ecb<-;5O5e*rgFMaz%l3zhQ)rdi*b@o< zp^18xw1U5n$N+&~o%`;3!sAN-!j~IO5YMZnokSrt3(WP1kJ+w6H9#5}?Ez=>@g3bv zN_A!&+YrHKe6U!MM?6=mrv~zR44VJ*r(u2gQrgt{ut41GAWIVqYh*bNo~T*ymt7M` z)H+D1IWZ$$Uuw$J8E$({GV+T}M+d?KIw87lSiGdE5@aZy$5H=XoFsRXq!Htv#0$wp z>|-dGan+#QWU1Cu#^Iqnnmu|(Q>{oud75*0pNCGdFxp;D?Hb=ZVQIqJ^@6><}wZ0m;>R0AO-+H*51$%%jw(r(hc6v z@Dj|a?b!ww$+>Go+zDr8$rf|hfpE}w>rv+k(BkV= zVEHLiIk*x}nJ@T_Qcyl(|Mkq28>>~6+I=}H-_aB}m>saN_(snN4J$+g7(U#4QLQ6O zdHs?vl{R2}ZPhB~ktWx~f;R45p4L$etA(QY)lNavr=KUE;6(3=n?n=cqtWcJn~cg< zV8ju%fVM5-C|boY?;Qp!?3tQEQNcaJKjVr#b}t3w*5uwW#uU_J9O8G4?EZaBtwUdL zYj8^%FMo?DqZ;~#A2dG=%eQr0iI!hakq)Zl%X@H#8o4TeLaFz+T2L0VVZ(|>PyMy? zA`AmWy}HxH5Sj(9mv6k?B5;^FG+S(Q=f)h~Ni_(^w(LKpNlVqIHi~19JKM$m+E88C zpNU%C|Inf1UDp1#M*~;cpf}upzogh|is3+qj`1b?suoRqaIV6aGIXWZjWUxxsg?22 z<|b2GYI}`*!>`uF$YKhkBk4K9k*I06b87ib!|9*JqxbMKD&1BAg9PsZf_QG(;j0qY z_ur)kMn!qod`j}z4*N)lC-^r~r-BQGQ4$<($#Vv>U7`Xd2TY{m2g?mub=gHR7%nSHv0PYPko?h0o!=ZHCZKw|MY_xsbNR6KhMA)5dXM5t7~9?5I4 zecv~K?euAE569$xv+M&MK4Wmpo(c&ykfD`Xr3{IM;A-26RHdezF3&37O8NKrbe7qQ z)pKEy6FmkI6>IRQWZ?;8dB4b1+!wIBzyP(Ljp7UD+B`<<{)tUS7_#7?lJ zjWZmppM0N9rv%bjA{%$INA0tChr$`13eN`=a*|dT<0R!LhPuE}XJ_~Vs@59iy(wj6 zXq2!5&y85#ADzvuOi@FlMVn^`HxZ_)i6Xc7T{U~u0;hEpWSz4>G)GDzJk!M2#T}35 zPcZq;RMeYgHkG6>m>TrgH`cb7WyIB9bq-$8adYKQ_B!qb5f15BP4&NgT~}Rb*9%B@ zmsnsX%NrHHWZz03Lg`rfx%MrVQiD2+4iC@)9L zMi^FSr}K@DnmHvwU)(GwH6ZVq(tqW+IXb0vuj<3)=9%<;zT<-4vVeKlHQ_2=NmW&W zx7nRH%3cDx;*R3a?C)C+%P(sgNh7dNr(YULbXBnEd^x-g8Rjh4c~+|kj&6fR(^a{g zUCm?SjIF3M7%^_wagh;ssbR%3A2J)Z#NRt)TROW~MeMUmUa_}^kD@w>S5 zuE}-JGHy}He0ylG?nQ$A{oh)iBvI=M4BplqwldFacihAxbFJF@8@x$Fu-a{ z$DNhi>T+%Da363G>~ry~v`l`>n5a-?2r-7EORZD$vM*G}IRBXF=uB*IK~dAk*NpGF zf9zuYHurM-aS<*1QMLLszf&Stia4Yg>Tu=}GmHgRBMF37X>T9EfT8X1=*%s`y zyHSieb&D)_zcuN4Q15yr&4$xIj6<`*Xxq~D3TLUOfyO!~-7>M~#k6-dj4IgWmni1e z+2Csi;dNW;Tz0VuR`z2Gf)p9+Z)x?FG&qT$JFrap5m>RUL2Mm_+=LTz5e$WBGOO zsLJ$n{@mbJ{0%WwmRp}OBwob~dOx6lw#>qjBGia@&Ewdr)BZS)t>+xzATb2nwV^!%J zopbIHS=D`3MV4`mn62?^MF7u>@;Zp{p7sf?2b}aCuMw%SUPej%f?e1=6qzGm#cSBM z=^ZVQdq2Zh%epOfg;AsLQ}KSTw$7w65)x*bp_~7@$Jz7dJhwh%6D$h7MFPUhFvwy&IP}w7YD;U$*@Z*`Y^R}dk$4sx#c3PikqAb!=Jn^=P2=EMr35< z!MBR^HKS$L_?$*7D`dBzs&y{65K%>iuLWy=S0|r7>CQeF{K-`!Wns&i>Wfm4sD`KP zOV-U22fMlWpvK|3m}ETyKq1v$S_0vOGWnTekT*1 zLCcCMq~XOiyDlU0dNW))HffMtNWG9*HbnflQCc?h^cnyHB1^KaSc@{m%EJmIAxb;P z0!vPZFa}oVRv;;7pxc8z*`&RTb*{yxiehD&3WpWz)T+c-sZDRam@JNP*(dqdIiGK-;OxwaD}gZ&L+4JOZWTt*&)M#X)D?h($a z=-|ytTu}Fx&Ily0=SWL_UDdQUX-x#z5=grVZhJ2O(6AJpIMk0ZS~lVsl}{xbt??Fr z0>na>(<)t#lc&yG&<*dR&u zlwiE0Jr4w)Quiys#^O%r6q$sux7ZU6Ptb`Bf!4B_x)TEiTG&6*iS+I%XeIlnSvK;F z>hgt#VYs@o?;8sd4gwPILRT+nc4r&5+Ezr5%szug74{y%<#x74(R1`~ap6ZRbF@T> zIx;NXgaiIpK8g}rhc;8%)AOhGsmbHfTV7NY32rfuW;By^_cwW7=?o=Umj{Mk4N>%V zu+PhqlxKj)luvfN;7)or8b4kRd1hg2$oHV!_+n+sD6?|vSIdeckheo>zaE0C-YYk+ zyd0IZOQC9|;k|lbzxtZz$1vFOHNnjJXiylEiu}7c(kMKU{b<#w9MYrISbWXQ8`I%TP&DiIhVQ)7?EU8^|tk9sY0k3 zNSI$*1;#f{Lt5y8ulk9GOdq?jOtNQVC>lqvy}e((#X9blH4kxBglPHc)af_f|JEQz zBfu&#;_>B$TX$VTs35=_(a&ItUjQAPp zG1%(0ER+B~1N{qX6tJ}t9nDQ=-ib>{L#7o}QxdkN0C@dFAOaO^FKv*a(zb(J0X^ zf69ey-(TE`)$~pTh=1-Vo2SE-zi~_ycl#2CPJlTyqt#0{E2B!nL%2n?5B9mlZ0BHA zWnH==3(>?Fj~)Pd3%iSEoT}hY9=utAh{z^c**_p@g?y8mcpaPb7VLAkC#}m+)d!#R zX)|w4A9&Ypwe-Q%va>W!OEw#b-DO1 z0dii##t=VR>`7Xh+BTp#IP~mRWQkGsdVSG?1e8w6856=2VePF4S;gfl49zL|N;8-} zGxeIo`^M|H?`3g8cEXjW__B{M$~Y|Mc19NvrG`JL&t0iQ$(pJHf%7m4Fa5BqEL&)y z$OpafGJ1XnSRJW$n42ymhTtj&%7Jzt3&V1Vp^GtqaOkY8Q1hbntJr4)*-gZ*)K}`l zXRS=P7=tbRilxa2V4263EXO3xt|Y^Z(%iaBpXwGG*EnXgT7{FJ;V~g>Z3-#|vPi)y z9twdb6V7x$7=yq2Fc)X80th}o0ql_66)uac4V5sK*eH1*dT_tSyKLE&=>8_@l~B!* zF*EF~H=S|^LEILf*zwcBLTtx@Vs0mD^W4ij{gs7Ih1-FvF{sc7bghJ*S{$FVKd-|t zpHuu_cd3*XXFyjItMy~9y?BxUm%TrZFF)Y6KUV>}1Q;x7&^s+-B72iodRv;Cwm z%Y`RbgPJ&h=Nf=1IOKWhli)WYo+KVB5mV!7^wnKgf91PdlwcxNRY4X~!dq+<@mr$0 zzL&9^4uhz-K&i=}Bd^{ir>quMOHTe8OH`O_WvYH32Mt5q;38G#GD(vFrO{MfsU(JKiAi-bDo9WnOiRTx^2|hQ==onX)%N4AYdiEf*r!?N=~59>`&%M*m3R6+Lb=By zdxwm-cAp)9F4xEH=19dI9z4udb%y)PBFD&>#7|@BP8U6(MgF%!%VKV5W@TuC+hDxs zl5fD9*}}R7@ibA-NqyWKClU0(=W5hvJ3oK_o!I3@k|AHIh$+d|R>K>W49`^#WGLwr zWhG=DUZ>&uZ!ahR^nv|C^uC#stGLBrlQwaQ2I1<^mHOROu8K{enkVexrK!f+7Xd$2 zP6xsfN+0DEihif1HwWRXJ>DPZ)Y)ZuF$OQRn)2OlNJej&Syz^r z6nggwHh`>)Z{?y?z|nTDk%KuPI%+q`6)-qYC!M^PTXn1HUIXRR%uHH;t>xMw-}>yD zq3;?L(x#0Fh!0nZ@jG(-?ko_>8QkBD5*eLauz#!J&*SguG`%?T-EAYa0RViLJ1v2G zYoR5A(j<&AL+53ym|&>`!)3u#Y6Ow|N8ay$*35;s5(g|?ehDsEsMjEh71op#Or*Fd zoJUVPYMzzIumvH8ruqeLr6VjJn(1oO*G&BsaP#@TIBTYE0qFdq1Q^YKa4e z!pC(TuEsJ&ULd!-2q0(C$R%ACw2bTEc5ONGp#D#n(T|_az_=D~f$ZQ&$tY${+T2wY zfSRw=(1vLg$3`Z@{eopS=IBK7P$%yt#3y_iev@)hg<sHF3|lzJ$_+sNS>q*Dnkb|Z%U(mN-m}^;7 ztRy!UJQ5A4;mHZZBX||&0Zye?V2=5zVz(fXdQ@fqd8#1!+pEXM ztajl#*BwOA5K~MNzTc3F^pn^L?I>%2Uo=2px(Q@`dqWbdDz zY2;D#QhJ!I>n`<|tbOjI6gW$0%tf`j{&IT}0Wd_-0$ukz`RG74j#X1*tT5os^EUU7weIk4|BzZRT*vwSd8M-{q|ePJA5@lv=T)w+e54OwOWemK*&G*e3WR(6HytR zc=rSAia2(o(l~2cREVlVP`rkE!N<5ZGEObfAvpnaN?V#ZX|>6j=Y4M#>o7;+TAiiy zMBKN9I-4RnAvej5kuP%>h|7&EzrSuO&vmS;uosxSkr6MMy|J~vd*Xe;6&5Me?b9`y zyRysMCgomWjO)ig(BfFy6_{|?!SY%$25e_weh*U<-pc9?gle4Qj2^}wVxTKB((h#* z#wNLN=N{Y8XPwXdk&nF3T}V2LW6W!p4F; zidXYm6Ak0Z4MIOnN8yeqouytfMv~$vFv|eRLmD?F2I6eF7$U-an-wBtL7GXxD zbxH6m=He`LX~8foakfMD@mFJ89U=@LvGH%Nikm{KSx}@njmW5ogJV(Isu}=fp+i-? zl7gF5;1`ao?-Eup3jv!_Qk!d_=Qn3~?qE-f5Hs03dvt@a5J)wwptTO20kE z%8*mlAvBG_5c@QljwDtyh2kKJzX&{8fSxr+l6_AyOpbAM>_DL?CT3bjTA7qaKyt6q z1+_9_c+ftk`wd3M9FF{$vjFGciVMmGsk;_5Zvh`lIM|*zGpD;Fqil1n(h9f;^X!)z zyrg+s8oF-f4z^<|n3SZvsKh6hcJVM6^o=>D3EjOxj-NDQ7r2M{+gn;*PuJ|QzOATq z@~saT72VQmKh+}rE=G8*_yjo6HmP^Zitynw{1a{4MO+j!qafd^62)j&*#;D;rg`54 z`T+9^V5b|V8OS=Uwt00`Lq`i}8aQ!`T6-^I--?n& zu-&V1i9LRj3i{_s5fhGb07`N=SML+Rx=Wr!9KG+xKitd%-dJgn=<_q{(?odV+-fT& zbs+nvb2N*kGK%T6jbjQO-i9Mnm!ezj-6j+D$tE|b;Lr8X5fb6k$!p~f*Y1i96UmIj zg8c|c)MaJ#(nKY-%}GvTQjmojM2SL}V}q-JWTEd}d0bCICwsHx#Ry5f8)Q64m+N3D z5M94ji$U&gRzgv8$!p{`T;m}o@|6wE0cqqA^tpPuO#FwAyI+071Y-tS52dS{p{R}q zr~4G$O(7YD+}q&WpIs9+A}zV`d9fe*5c%_#qt5p|IMVDZpHbctmV?&G3#bG-hT-OwrFdk&Z7wKa|g>Jy#? zAu*IG#Mz?<&Un>}@|Bg5#IqzLjtTQ0Mx8BM%I7aKZOuffn`YOPep#h6Bz#$fn4TYK z8vV$2#G=G=$HI9y=DB$ZVUQ$)e$D3=?lr#uwu$%`L^|MTpQWB|RsU@oCuuEnh~|q! z<@Al$)mdn6f1U6?Nd1`}<)phSBIu7skLkMNF+%u0a%x)Oy>~bw1)eiYv!58Blf}; z6+DSA7uA)bO|*{JJ!BHPgF?ZOK9UI*ucbq$Jp$=d->b8B!Kpk6-`nS8(&fG){4OvQi{M3U@yI&i z6mT(rws1CkICi>@69L~ca}Acd)r-jI|LBS|^}dFef#>5No9HMzR-z3u0grfm^9#LR zRYg_x%uHxW>ar}5YEkTDBb3b+GsJl8nuz*0pZg&JZKbAj?kMOdMHKV7wj$o*9)A%p z7sjI=uDvX6Plno`xdqajU=ZT6LUfe2ofbE*Ul()c%3O_aP$I7pDA4#IR!#O+=4Y{^r{B~d z`Rk6n2cvD0_4KhtfpgQgix=g!70$tHPE>+n%8=hG-;4?i8y2dHb2tQXT%|Ta%lRcH zybgmSapT*PmqRkQ?|D5&ieZAE2X|P=)}Q~HOWi7R|1S$>2E9bmEBeVfw^{`+&lCKNe)(b~ zQIFQ*)TY$8U;EYlP!I6Ds8^Li-ithAoo!?fS@%&~-qH54+2^@X+MB-iwVFXOwLM$o-+R!#!O5RLoLi=b8mlw#b`K_0qMdQ68Rmk4fP;95#QldMubQLmvMzjPRnrs@#K z;xd91te3l85hu&DRGyC0;S>%+R?@@8(v$3ymMYxw*a68aMDc5H;Bc$+vF+-OXBy+M zdl{>=CfNJ2>z>_L01@o{ddbRG_`{bR^51KFnLOlmlCF&I?hXbcQ8kAPX`2(@nYnF9 zy>2YxXX`$`$=g4TI2!HJr>5Au|I#F}17%DyX?UfB&Y<1HLP;||cXX)7`l;reH_U67 z^6m+cZkIJj}iHQ%eYpRiz08%QRK`o543+6%Ro_wXt4vn zzMccd1V6|3EobX_+mXVMx7VbQ-?!7~kS4)&e#P7Z?3O?B{}!S?cN~21+8DMNdwWeA ze}1Cho^ocwJDSTViAmi?gKXYua&!iFt}&k_s*}+kD*?I@`jJghE@UX!lu7vQj2V&6>}9J zY36o$aD!d-b=f6*XZPbo_#gErfHCF`a?0ruX)18_yDB;@^MS5@qIY0WhvQnS}<5tQnqF^ zS`xg`SoFuc|+vjz_r z^!8^_?LlqZYo)%qgv)sHGIje-)38mEiHAN4DCsWeAp>m|) z#_R1i7Mdjt!>G0&2fG{F$Y)KMdg%VDy52vDR@%bAQYIr_EQR{V#Z|QuYfA)~m<#JC0P`m?lH#lvH=lAY&qf}b2om_sfq;J7;|Kv4XRab?jaTV0 zPW$$0e#RAad`TVZfVk%z0>6ncRE)N{DZ)-6v$mgv3}ANaHd~f>6&2$eB3?4W5U(`X z(O_jDxanxNDUm8YgP9`sq8zZ=jm(Y9x0(^Q9-)>A}hpc&%Dxpn> zldtv*{vfMoG@K*{rT=2L39^}FZL zu`OIJlD6Md3CYDBh%De=g0KWxOOK6v9-qgJe_>~4a zuWr}Pj?G4k^UH60E{|Mnb z9>j`I6~Ts8w{|{-fUbNZ6NY9{?4;G(eG2CCAtklD6+guMW{a}cCUVggH9cbX{{ZFQ zVT>Gs6CD0^nq`@R4RtZe{wWP>+-NtppuCVVUhFvo@~ra`?mLR{*wVXZ6O^xZ%I*u1 zxF+vVfl2(j{xsa}7$8-jv!fm)Q~30zya7iqf$Ax0V%^9IU6^j9@@Nt$=no#0r^*>$ zGBSP59#5HqfO=pY&>;+f@k0C?d3UFuj@%5myMJ{4)Re{A!vLg-%tg;eZ zdA8jD0C<1(=>!T6Vz_wEqa2Ud`cONwU^wh);m5UO{{W_-`Al+;sbBG>7t+Q*eQzrJ zh1@?+>r6S#o#P#;8P~05!>L=!a3xZGGsyfZb8r}Nb4tTgM?2z~3td}79zcASBe1}% zOQlt}XOq%B>n&MnVfXcWh=IiU*p*+-eJbYBS zrb&2_#{_Zi%PtrNHEfJ47w)^|B=Dmg53N&XXh+&g7Gg@Plfd^CPUiksjT0;vd2)NR zd(t?@T}rYrBRM0}wK-$mZdrg>V>u^3e9~GH?$Sy?ROEm_IiaPJFRl^&ncGW)yPMJ+j^uz|%1%c-rah`&Ju_%%_Lcze3a*XGmynciAtvi`T z%o9le0Cz3(1L#ISP`{mX{&aF`fD`!lT?GCnuOrCVWNC@h@7C)!!PAcxe zbkAMD7N@wVtL7i6QcVbK0nnJS?G z!+??!J$C2+0M+_c`G~ZU?A>=;$_Zba13kyKPebY}7E}?t74r&Yqp27wbI0T9_=-bB zf(v-(0{T7HgzcGggBd*-eLB%5gwZZhmUzQuPB6+leMi&OQ?H|%2;-7L+@zdv-i$Hy z{XMEn3Dyt?47ej{JTm)GBB-}*I$*95Ll2Z>^ZtJho#`&Din>huS~t$-_5T1oR=v_e zcVf2l7Aw{{ar3KZ^gIA+v}+_mp`Ap^e(b70Qa{hV11i|tfnsMTh#Z1PU~5)O6}Dm& z<|aGF5AvkbQ_J%sXw-=^RYL>3dwc%?ELN_bkWFFatiEh9|sI#L59O%^}a zMxy3YJTLY65k8*~rn}VA{lZC(>pGrpwDBX;DhGRvDa$*~`h=5~tBxDQ` zkxu(aJcMt?tXo-2sC=R$I}W>782m*IXC1u0Ot7>v5ASA+4_jP}NO7H@4|=<2tX^0G zEm}D8e({Y7`o`i2*vSiHuJXKMszW#j1adukQ?Rcj?rGm@m%5G`=H=WkspInL zRpWM66EZ$JVb~>i5=>gpP6$4g-9{^E4aMl7{DdGs4}* zed~eo9-jnSk$ZBukY%F>?w?vF*6;2xdXrHNFa9q08~lj@+#3V#-PUz{z)ayPvj|$G>?&kf!J|K=XMa=rot|WW7q+c zQQ;!p_GI$MsX1f#)ws0VtBBf2?c{a#T83+BAv<0+Q|L!}1+A_`gK-)4>P=Xf2(8pN z-8|uY^#1@FjORJYC({_KB}2eNt|%dtgu@&;1Je{!e4jzZ6a~)y-rgTG%-EP-gaS@} zrk^#+Mx?O-4_f1-)^#aH(Jz#^;2_0Qkjpb}mQv@`oYh?QUNtX;9dqjzalT%4ByoL#Ha_w9ekdf&A-~^3Lu#tt}ftn^u&7-dsud+|hF%t<-PbHAji`@^-vak8#iS zr8?f7Ai$bc&-arxVs8N_m6$HufBQ->Yn#8*?sWTxx1KVrKO;rQEJawW$mn67*|vx# zc0cSE{y^1%JjkavET^ywy#7OxTy$ZVFUSj0rOnKE#Epa7G+cjNcG%R^b=lPUW)+(s z^V$MGt!B;Q3$;RG0u}?0EU_PvIQ=P2&zbkjRiUBHcV;C|EQekWf6p{sn7Y*@^k#qW zWYWC(Sea)fnZNyC=Za6Uq_T$F*Y4nBA5-)ntvhS95nIUN7%|WLqyGS}PPe!VDDt;$ z-@*0m_A^va5A)4pUAFl*MliU-is&_^`&7|IxEW+6n2+I5 zIX~ypx$C(j^Dz(tG~sZ1p7ee#6_04BnNwQ7Y&H)41 zkz3YkVuj@B-e1f3jz3dC6rysMvFzj{55u4Sy>42j_n93-62p(KYZ}cnD@Z^iImL8V z=@^;?;K?TEByoX*kLCK%0doqy!p7hj;Tb)#&PX3!!m?G_^%>o{11G0marp8-O1{#` zHlSh)mGcW7y-Cj>$WScX?PZ1tzzvS4l0W+N2!?pbiEd|;=Enr{-1huyCTYZxO#~-w zdrP|<7IHm(D`!-HHr5+q_no#A(2Sg)>sb$M$l?YCK`fm*9G~kz%;gSaXF2&mIQrFw zCQL8)hB*4u8X^;80vG@fy-8%d=h-2_&tFeUKo+Uz!Ek_|IdI4ae_H7L8V%m0(z?3I zYZfxSH=a5C0j!uTWEVm}SZ@2pI*jpMUxgzrZFg|M{`O)D?H~}K0o&_a;p2IMxW{B= zjt4w(jCx}}`4z@^j`3Go2wQiQn@>Ie0M0A2k&5l@;mf8pJ2vQyZ`0w2$zwI0?5c#5Wn2%vl*#a52H;WAv*jqck^y)XFX(eY?pY>ev(i z0N4DhFHD*^?UFchN-<{0&pcwa&HJIi;Z?Fd$3Nph8*|y)URuX)(Snh%`MMCKbNbgi zf1@mHvB4k*aof++{A*SV;8B3&3@I7tIR5|&r#w$QQf+3=OD{OaGC!p-DB0dgZeG(- zhhaSnF9Xw`>EF{eqcyrj7HC+i5uScv4}YaP&g9DaUEYnD|dcLsZjy{>iC`Xo6QS&np26|L}VMi!? zbmu>nO#=szFzPx`Bhj=?8XYrFRBSYiH#2(sR)34M^_I;ii9vW<2i;Hq0ACeZ{A5L< zMwad8s)b_B^5lP$Sr)2EIGBP9vr{{XF7BJ{90jR^18AI^~K4lo)tZon|j1}>v<<>*zG39K#*Hz2EL zgV!V1=~9MGxKWos#A(lQ9L}vIK$YVeBy-ybiXnzM?p0w&%!CC{*%_eBje_8|c^!S} zpkYbd)2(l}g)h_-xigHAPc@x!rryHdOIx^M&jTa$ptl(<(MK5=0FXOXkFYdxtk*|i zZ@C_RXa4{SmKkBO!@PcM`1x1(Q|s?l=9+7%npxblH&fDqY_$!*f+Jm-{Qm$tjm8&l zPckkM4ui z{cA6dpkjkZWb_mqHc7^7q43_UwsN(+#45^02>$n_bGxzE9@Hd-h!hGkv$^Y5_X^-- zf=^n;@tw3!rpG=3ba1))kN&k_=-wpqQ^~txcJQ(uaqMbO5$loOBsS1K^#r@H_3iy= zXGE$-P}IXX^1|czw-f9Bf2p7~?{+A&m2qIQ~Mi9^VEt$j%0R zD^eBRYKY?*9$RzJj(@E%B(Z^BSt8C#jEr{&KmBUfg-YCsWX4x)d%psInXLUKU+oQ) zK4fY^`hY*Ettm<@awjY|i3WS|Ohawr33Re#5y{EzjQ;?ISOsF`hE>pjN2x#lx~h|b z6S!PqhCYXinozA`!#N`$?ey$^qJS{xk>b?i3=S4hNhj(nLi*T9?IH~0_lw7&=ky}8 zuT#&D?Hm_RmB9XWuWcco-DJ*LN6b$-IR5}9kjgifG2KG4D{{kfu=Vag&$VV49ytc^ zIp8q=06lA=Mp#EGe51)wIb7iWBc)3AP|qIaQ0sw_(tt9zymp1pB@~~Ro_)Wg)4j8x@9^h73iStj;5sj+p5?GfY00V8#YQ>1#ZyYEw%YxbHIv>yaQ#A|U zIZR}+aDHM>;ZoTwvJjDQstdMw_5T1G<^xXFGaLuY!Hkekxc>kqtljR41UpYH*Zln} zOG&Y^iZPKb&iKJp3xKEjo}==mm@Vz3tcXyj%A;u%4(3(0(#G6KRY@#+Vz>0ou9o|U z%5shE-}=`v9kknt+DegumFw?YnmkgnC?t$wLC!hPGzps0Y3VU|@HVl|LF68B{XZ(i zOG!N05=I26C0LRU26OFQOt2GZ#yo%k$@j-!^Xpiq-dU$u$tvJBNfa@on$;HO#(cqs z<0l7^f=BeHX^}niH`*;w5WB(d{{UQ6TGNRxa70*y|KU$XQkCcvw3d0~{imW6)!jaE@<4u*yoL1M}lNo++#uNebtWVNyCek~?<5sL>fgW1h4D2^~o% zI2_Ou0|N|CVb--RblZ(WU9iM<@s~X-sM7p2mUwcZ34q_)C>#4dXpdfUJQZ`{CoPz4}J1de|mw1l2d zO4NvH(eWfRNWRQC{VPRg@bcjK5|;F3C;6HNmCkVk;E)Le&{LWzT&W-o^OIW|mDZmy zL1z?-JCn!hR8^6<$r$w%!K)cl*tEb&w=9Y`5PRp7`qT@jO0s28;5Z}cj!kVFcHg-l zk?3*%0PCo4=D3CxRflqqm9TS>_rSK_0@osN{-EFEv{uZVo%I`TkUfbC>qm(_x%rIOsjeG`AsDQh3{R zGN8!MLyUioUDO$5fnrcL1;-rn2N?W^=T$tjC9JRz3op*6kC{(j*ZI%{w%4x=>u{uo z{w$s|-2VVtzD$2=jUBedCm0>QkNNbhC6ERA%N8AZ>)x*=*kKtN$QWhEYAyx1<8QS! zqn2Ig0G@D1^y~Rl?CooDwrr5+0jfx3OPJ$!%S?xOJ#PDs#DZr$gSJ zB)7J=e?DfJ5lH!>UOt@D4Fu3yO-EUh>6a;Ln*hdsX5&AX^RA{xNMhe1#?g{V{w3h{ z?m7Pe>(+ON?0(0jp^#w8e8~wM{Mh#Gz~|7^I>pO6EJ2w|5M(3J4nID%=*HCPU5T9i z;>|tVTr!1clOT2j^86@@-tgPJ8*>vCZ$sbwYWm7ojewrSno)4%5~BmqSDxM7*sf4r zB(j1rp2XF)wJT{921G?aG^YpW1AsW|jAx%(drI%zoO{q+@D~{a*P1MX?7bXs1a0K7 zjIbr0z%JSL>_5(HXH7BL!eg3JWpltiM>za3$gDd{=7_Q~BSwDsZ|Pb`RJ56eyl5i8 z%f34MXWOUeMTU#X3Vgr>Cds{KYI% zGQ0uX4f2DN-!&sQzSfM(2qy?K#CLE0w@=Qg&Q{-JU<|7Kb9+`{>y_) z)7D9L2OO*@#GL!psIEm1GTu;!9WG12k+qsllmkHCyYb%yV5x>4EZ-Q0&@rM*!m# z#!#n#b3k3Kgk{PSKsV4((C)TD_&&mi< z>^Z3|?B%(bMHWeKPwPMx1RIITP-3<_ zp(Na;ah!eZ`cWGyHe415xIcEBM)88clU6i+IkjCyZ56hzGR@y_f7H-1L}7pj1~bpg z)O{)O=$G>YBvMB#dEkZ|{Y7lv_(kAC)^`P%e+%QE#;92T0BKuX`C=In_}!6^c<(`O z4Q_^5>Q{ON+eb7(gMH*YejwKT9x}1oe8{Fo^x6olO>!xi#c_0=Su3?b9ee)(TE|(| zA@WAg&+(j(%9#1wbm(( zc;cJ=w|N>Wq<~|f+A7t*im&2a1&PnP{{V$M8ZKU-z8JWSZ@HBrjCEpbJ*5%I%N7cQ zg?w^p*V>ilqQR~uEWMBn0r*ueLgaQB%^^vygoal?Us7l-k&?hu7w-c~Mo-XZgPI8* zTWb<9Xd?q8eAOIevH6BZdgi25^c+wEsK*|(<;OW4DFI~zZyxmM;b~kV0?c{wN<9Qn^nlILU9PrE5pwyKgV|uIxwmPADltOy#bvp`RH5 zGMs`qBl7J~M+COP{?1O;=%=7HyQ}FY%3|u_qmc2Qxvaw;ags--C>E*NnaO8yAbeZP z<^KQ?Rmf9{YPa22`DD+^a6#g>B6fuCNeUk1b*WnUk1aVk?#K`IqQg(QgLPzb^FpK) zKQj6Q)B02oZtQ&3^8~%4kAwBk_*X|R)(I?o3yv~_0fFn4s&^LRQMwY*p2H^~{YPKM zm4|d=4O;dB1n@Gj=))@CjyOM+U5{9nXm+d&hq+<^IOne%(_TG6;~V9X*Pv3Nhe7Ho zu~^;QNxDEIQi`F-?%NR0_4;zkn`c%bZ4##06{o|gssibIuPn~q4I8bDV z<1}i|V>GdwD2^0i*bD~x3Snn!nB|i=WjO7QqMvDG*Y}|$Bg%*8etMC}U+YpdQOMYh zTO@Iwm2M^bCBKyvQb@(OvL{zQr~9I|r4-*Y*$Ecz?-D4OkGMYQN@fUHx-1_;Ljo}{Q!MRYLU={kVf zdu%P`Z{x=V{Y5fg3rx`&ET>73eU5pF54*p5mfvZRXM%fmq!UBWcA^t#RkM2>VC`-OV;PP0X)4$_VTF`1+H1{YXZ77Zt5)8%#$WJN$L52 zKjtZx5KR}`G`ZCIZiZO%j1S=>@FW_NJKF?yR&oGs@sr0@1P}A?S5BafZE>j1fg%xz z)qwlX#1b*>%>y}WNuI`Kx{hG1j53FUN7lOwUliHAB+jg)Ddh4G(zxlC-Q*%OyPN^m zku98)g8OmYaB7nX%1CtnQad>T?8ZRA-^(eNdl~0Fvo$&{c4&` z5u3{Mm7Dvbnr3eIESra50p}jIwV`R&TAH_;C=#z!&N4eWmh=Tzpo zdE_!k##s;dxTm44rxvV{TrtTI#220pIsE7)afukSdU{fqW5#g4!joq9z$1?2QZX5p z2X6p^WBi?}!4tT8{0{B?#F-Y8rK1L_ppdy6W29V=*hp6aeJ#KBA~!YZuy(K6LzcRp1J(4wqUsFZh>Lg7QR) z4pJgWe$7}7!2~zGIH{60AYiZSObznmADv5QG^Nb$BSvwaw9XZN_am_tImuiT+t#Dn zN`sBw!jU9@jTGD;r?nLI4GWKGBno{)N$r}A;gwgEAbxbdVPJFCpB%CR6>?ng05lAYGk|_kkM~7IATP+|=b#v_ ziairkc=yR@7&7t!8NvN)E?XB#%xE&}!A2Ox1959~REc=x5`Ai15zu1-uAM6IL64NAQ`xts;-RtF z_Mk3mf5J~I#zc1Lt^MqhD>hr}>EACbxhJy_w9|qB9lHu?I}Q{Q0{YNbGh&93@wtl; z(36@{0aa4!4?O0eEX$naA7e;2e+lEg1d~pMq-I#rRP^%BN9UjO=~ecgcp%qL;ndS7 zwb?qS+7$8__4-z=)`_Rw#zI0PamXNlg*!7&M%B&{5UKmbRi#MIa!c57Jp!!V~SAf+<|}@ez>G37M0AeBzeFwJAaFc+p!u&lKIU)nIIiX=O2bE z56zIC0Li9_8JGgt9q1)HYG~>@i`~A&C1K7&cRyOHfrk;VKF1iMD2om3z$hHCALlgZ za?!9swmz97ll3*Cj+d5z-S-bOYXyyiKlg`0{_nrN1R~^x+|9`ZH$-O!Vw{NT7+OE`)ZS18%#b`m|yOaP=ZySvN013hVwT&VI?SM+L7|Ep(hU5-% zYHl?qwmK#8bUO$(PRFhR70qfogqN33HWo6+gI0vpeQYp=~ zW>)j$&o1ZMr-x9xxA~#Ck|KKTJX0iu$B<9U)G+T+=jIsa@t}fPr;_e(HgFx!-hP?< zsqyZC%O>T|U~&y2DxZ~d7rqTDg2QU5U_}607s7d>H*?A%i2SJIAH534_hm0P6U(T~`)-5osbKB-Y{9}b8p{9)@(=y|C zx0GedxaW^r>@ArZ%uYc0kbLjSKEH-*ku!Pj3!qgkot*w1YkNwPLu(nhxK#4a;T&u8 zA57A*P3SjKwGkDGV{Od4vfYQ!clJKDo|X`@XBc7pUB?x%KCNkHr_F`;H$%uL@jOyR zr`)x?my?kkILh<$HhN<5>FlbSA8FcED?T7Wj(UolT1Duv1Z(D&|vf#AJ(3xzR8?KQIOz~f$NG{ zCE7Nq&OK{q#1dHBpR!vdW>x#n4^MBcbMJ$UbL&NnnzOKs?!cj9bJ~y+SxXG@lSa;j z~Akndh=d03-q_g~rkickiL;lM}wyG4#!0_=`cdw6-@^pC-}+ zAvykZ1gSKxb0mf&kDKtwrr?&M&89<<2PS)|&bq<1`HxvTeK=D?Hp zvO1H{4iQfPR{ojcX)YJdx_>EKor9_TYWA1m^tM4g&(9&o)hC6m$GL*i*`%ITUA;-@ zYc|_=X7Q)iM6hN#9PTj+o_pD`5 z4%~d+l_qs9eE`195J5iFd4-cB9S;f4n3r!)Zw1BNUxc=<>9=A9&QPYJe{ z1#|L~{3p~@Zj0v-GI{AjWD&Q4*mUB6xy@>^OJ{H*xCcE|w;o6JsO;_@JCOTXiRyP| zy1C$r>fAwco7bp4$K(AgIxi2!YjNgFD@zd>UO>;WscOZkUviG2adD_ltBl0BZHJ70 zCc6C+KejY)e8`s8s0dQs`*q@|Z{EIKFru%ZPw}Xjl_UTO2a*2()~1%af|^Hj7PF>A zJh+`zy;%PMj}=o^*KMJ)ZzgEoLc0}IgO7UTkqP9;6F!wGw`YjqmB*;2SG8pjvN{&P z=aw}Je9{Igc8akl%D!02fO=+_$<+1dsG!M+z?SD9O4!kL*V4g>;u1#OvV+ZJAqw1% zf1OApJ9m--=dqv@v(W2b#*w0a>ecg);v*H4sCc6O<{}WR3PvQJox?n1y-A(PqJzxKNZVsyynQ`| zB%WJ13-XS`AE~5iUR(lz1^qwz^&1io89AT^U?cR(n(Q>|8(l*~h^`A`lkZ%B!sj1) zt=Vc5SU37S#dG(RjE117w6`s4dJMLJx+9f64^T7BWjs4nh8>1YZ@-EzL}8j3*bbl* z{Hd_r>Ds9RczmXB;zr2LFllr-V8Z7d z4rnE-6+?v^arq8t$ad&|8dJ1q|?%o z??(^NbrfKdK{)NwfsV_C85k7h+(f4*Py(yt1&(k&T4-!BQhDpmC}2rt#xamFOee3WsMNf z1(bzFJR*z%{xwm^+yMLC_@pZDl>I1;4c^g3(9YCp(L|{n7axu*X%A=5*3MMv@?V60C;E;QC z`c#Foef_D8mi#?vGYscBJ$hhL$7ykKpC%`A0Q;jfk|W0;TpqdnE3)vV<)E@Qq^xs> zC+0tRP*hTG%;Fw4jmpUzAqTj@&*MuPmLuP>uTHtMx4cc47U?kcWx*efVP5zpT1J+_ zV7`&KWDqW5}BK`8<@+!!TMn*#M&M9u}Zgjc4tBA_UjDyq~iHRslDo>^;Gij|D z$ConhIQFUS&=d1u58=*gy7K=3Fyp5cJZ|c{J@_VqNm&%H8~{2{m@Eoy1$yOz(4*uS z&QCb(X*Q@`#~JJ~ngDPys+_YBa!%ggjasrWwBRxQ&?9g@o$8TXk~WYr>A|N)06gbC zFn`ZV0UdhyS5KUYgSd?H0^+U5;ax5=50uT0FwOoILqXOys=+gjy#o`_``1$&2gf6? z1XHoq4pz1?uRJ$lG%O>C#^dg?lf`iAVv6U>xE^F^aLPZJuSdVTODjpIb^~G(JKEFD!XK4bQFeS5Uc#ao3S9rI5~Sx!`uimu#k1Lfz^o`V1qKOE3ObQG0$70DgI z=Bvu8?edIveiv=pM=$2b>^7U@V=BRhu&)}HYgNk$l9!31+i zp!6V-*XcnoLNbgP=eNBX10J-b22-4qLhVt(IiM~Falznw3SJ4x^rvks*j{+gJ*lK` zI0Ot2&VexbNg$8`=W(Q92hEUw15P|90OWNX)NaIZG7bd;6B~yukUJB_H|9bNWQ-HW zX-O-Afs;++fyPfq>(;^&Yii9XjSkQX^m-ouu*kRmk*>r!p~Nk&qORoBO%-tz$}Z*%VRO#{>=!Ui9SL zPEIlWLlvyH5UJoP$oaQt3*QH(4NV=ouQ4bL&6QFYxA(Ae4;?FL;biv%Iiius0ea)7 z08*`#q;tl3OdJFB{{R|)_Ii+b$myJpPv_E!b!;EqAyJG1bHVTQsp{v%hI111EvW6Rhy|N)K?EJTjp}k-|gkUolgav zw^F20MqQ5p0sQI#zzTDoxvtm4-(j`WB5={HvVHP#pQS@>l@-kOC|Y-p5it4E(R%B=~Uu#cODMfu5GSKZp!P) zu2$kHAd#VwFv@U7YtSw{N2b8Lh03}9?0l14HmQB~%|cs%o>+7DN2L}ssG_zjuP7HZ$Kej0qJ~k}-jfq|+P{+p~~8>E%W+Ny+U^EUgeb zNI%8T`S+j%QS+$XfKOi3Kg#&Xd>n9TN;yonNF7*qrlS1W!73;MQS-1k0|JlrO`MVK zLb=)Hf^nRXIO$Qj`MAkG^Z~V`c#`8!K3&r^(3ALz?JTd7@*yNL2?O*s<+DEG3h)O( zQ`zblb}hQvc@uY0ns+sgM;*@J#p!C=STFvwf9}0~>xE;C1yP?|b*XN33!AsyJnbuE zs~mw*lk)NbKGcbvxtQNCE^vNsy=rArS2;L7^&gzlHh2f8N{~6o2LNNO1P>r^IN;}{ zU9!_Iui2xwh_LEdi~&y3bg4B9)lcCAEZ(BM0?yhiSQa=`DyZN9GH5i6E_0FiG~r$D zlyu60`c*AQ!naTnlWa8yp^@lN(1I z4DfMKZe-xMAoGu2^a!)%MKfkRs~+8ns7!Ahg;--fjdVT|g4*)&s`T;pwmp@+?3P z;xrQC(T=Al0N|d+w5)YKR@f+pR#Y92D^i_0D}1WW3%95|turX6A@`q5aX_xn>&$A0 z&6!d;yS#b9`ijb)+E^qi{Q=>{T`#nq%vT2^6=fME201=-VUh^xKwkTpt-;;6axq9$ z^$a?8>sK%I?#6-KBWx{?$Uxk;r5&2?J{7M^AtDxa(Q4>VicgzjjK#~A=M7CvuHJ{#S2U~6T}0nv!J1Y(T5a58gOBA*PzWa- z`03Lk}0dipc6~Xy^4K%68B<((;mDl_B*+R>V<)(vw@syXac?4`hT45SjY85L**NE*rIVQm3J4wjY9<9n z2>j?X6r3EMGuzUX1tV}Ik@cktgynz6l^BUJ(0b4WtCBOfuX;}85yc3+~G0Bm;{=~8EL8$rP9ngHuOAv8+_;E*INKP*>6F0Ey1 zq{%YmE=q%w`B$4n?5uX=6UJ(DKE@0VPk!_oDcM-)@4R08t(-wdcz5SNg<;=n_jhZu zY1sB#XE`4Ohn*eMnUQ8PL6$&?P73ojP?~AYSEm4I_HWU z{EGPpC!p<~X%JtERJl@;0CGN3N8YLvAx!@BDtiIaq=mNb90EE6#wx|YRz-C|xSm1i zXahY6BMp!bAbQd;2PKATd15)<00DvHH6I0Xdi3IfTw6QYbo<1f)Ro!Ux;=D<#{)wEEz1d-|430VZvD?ye$zLM@Qn3NgSZ zBR;g~n+GgGqo-i0^r(-~kNTfy{LygGeIP~|di|a#aF#r}r$ZuMi zWb$KHR|g{`dR3^dVY*KyQmW0@KjTbB84NhU$>@D6Ps5hWajT+B$^*FP&~z0A)tkb> z7+`0prrKOJoXaXU84m>a_MoKCN)A+fnBZf9+Lz7sIUbeBMdJotqHA2>bF||%B%U&e zCQ{MK^xi6trBXIJhL9ZPii+lZPpcx@HJoG_Ujc{rMPy!hrt;_%i9{-TU&^Q3v=o8>ritDt%7(MJXBJsAeoOYp7c>bF+m1CfR^u4q%7bN zx%_CNfaq9DIoN@-{{S8pBsx{q#M=-IpK(PLGNZ2Jr_yfV7?LnB4Pv zr1MIo2i!(+MHC2f<19~3xTk;{ox~H@6j4COi+9_VBi5{2*iUgP#0r*Uv}Dmm3~22% zpAFksqPwciFveNO3Vo|@A}|2!(-@+P4#cdC%~w{BPl&vLpG5<`aN4G&aj8sXm9VGo zAn2lsC@`qmo*M@k>&9t_LCS?0UbIm`kuhP;6+q4p6fgsD?lDCa2?|Q&whQ@x>HSP$o+51n_!OfZHJ8 zAMWH)MF0%18wT|12c|z!c6` zI2rV+%(%`NXX!;002L~@7$X@K4#XMb1TIAsQ$RwYrQE}(y*e@zE0aYOfJEW2*z3m? zHaH}bO8fCe6ab1hh8Y9j6)2Sax%rBdpL!^uV;$JzcSF+^3-Ql!MHC3s)9(buZ7&Ao zZ@9y%)qPJ+eLj7qw<8~vpL!^u5uW7bmjD4#yBv@>6j4ACv$pN4^3ykGAanJiiUtgD u3g(avqt}d+MHC2WVX$soj@17EAnX!5e7%6{MHIkI$T}eWXrhWhfB)H-meH30 literal 0 HcmV?d00001 diff --git a/python/nano/test/resources/train_image_folder_jpg/dogs/dog.1.jpg b/python/nano/test/resources/train_image_folder_jpg/dogs/dog.1.jpg new file mode 100644 index 0000000000000000000000000000000000000000..a6a878c1897723e9210bf0d4c69b8f551341f4b8 GIT binary patch literal 25034 zcmbT7bx<6^+vgXT;1(c@F98+_?(R--C&1zkixUVQ+}+*XJ^12o!9s8+xRcB8eXs7S z?ytM+shOIo>FViEP0c*^?Y}F3w*WZu(sI%OI5+?R?%xLZy9ST|prfFmq9CKAqN2V- zM}LP!jDv-ViA6?8giB0AK?|a$pr-oB$n)tV11AF&HLDmKC!c_@urMu)q@0AHERT?| zz<&mTLx1-U3lj^7g98+xr=}P9e{O$!0l27e*KjfjaMS>JTsQ<=xW9t{ihuJ&g8Q!q z_@5099sv;v83h#${oTI~%{TyfI0OWELl9rK`lULW!)Y8_` z)q|LsTUc6I+dy62+&w(Kyn}*6LcfQFM(y$A`%rFGM<RI8V0h>s)SAyTQwh(=-n*e6EMO#;NhBT_F zL=*Y0W@i(Qen-Agm`DhXxaMrEr$+2paob>vLQY|jveGp=noD$q_{Xru_~+!2X+In5 z^UU#HNc^x^MS{OsP!4g#hbux&UZl8;(^QABxRYWhy zQ!6s}(W0%uOD^hq67L7A_R=`<(o)}`t-!?5`S!T8>)Gkt6z2m={2cF4aX z$U9zFkG>Sr-8}BfBFj!c>^mgEIsLJa5!ngj3J1FKO}@V5<&Da}R*Xlb#HJ}N*6eIa zE0rQdb0f;W>^Z%dXeqKQN)nCl7_HRMh~!0}o5xfkm5(k5Nuqo&XNGPZs7m<_s^I$L zQbiXWl}@&XaExTV5ZB_REuP&fyq~VlcUPXEUn#(lPF&B6nRt&~EO|9q zdss@aa!nNRbGNIvPUs;&aQ&;Poa?i>mtj;QM}kUSWqP~=FEluk%x618+;jp)T99mk z4%Wqb*TOd}A|K8JwJIKy>Cgy4bAK+HdzTWrN8#Lm<>GlHIU$cUI<+anANWMLewfR8 zlsi@~gqpA{cK6Ymn4_I4t<`AJ*Dwl$2Y6A&d!J178R$Es zSA(t2zz&NpWiAK#tk^5Yp%IQ&j?$cMdhwp<=dk!b4Ys`jv6^E|-pTpEQ*6g!FLhx2 zT=C4G&|3BFxhmleIJOR4hrYC|J%0Wn1oRgWoT|DuGqqARV0A_1#SS<#5-TB`d*{YE^9@{oi}Vc2nCw_)tgr5&V89?Z8X<=(+VA(Q1x; z(Tba!EOVr$m`^UAXI%E?ru4k6`npQ*n@L|YSdK7PF+rqX%}yl3I2!cOR(*$dXTJgn z*>CUMHqPmXCTlznZ@wfT@q4dN<5VSp=JTW7DEL9f1=j3mQ_3F*Lwu-mHhCe>bqTuN zqfES2d0)uT3#lso* z)!x>;o&BI_Lm-jFFS=@p>PmFUm^R4Rp?g5l1aH z500AxJX`c}-v;aOhmVv|X3+Jc$IZ<`oW8a|9_Lv;ywTo`zaG%t7WFyU7*rJM)&Qv2 zDJJHLv*&{5MV5-%lvcA){{q~oqJTcvitU4J1|u+3ia@{8h}8CmIr>1yjL#M9Yub+s zE^XI=Si0*w*X1t4%N*yAJz^1|i%1(cQYz~X-}t)=q_3rVVH0r;2T^K6ALJp1ZTr<# z#$qn8no}%>p`D1|d`Isc_4aP~yXD97nav%whNQ2&67jOYB@8O*v_13vjkyPkPW4P} zygLX&HrpNP@`gC0UrvvR7DtC<5Rok8C<=GC6Noph#^nd&&?@DNh=;wUl~fbWB;eE$ z5_adXD&u41N-Ka;Hx@;xqG~==AN|?aQ%rp)-Hp(dTJ5Rch$o%LSiR0|JN{&eQD?sr zpwGQiqphDhSxMDs|9S&mN|X$-SY{@axkOMXFL9uSyeNLlHvCg1>o(K7D;-klLZUw8 zg1lI>M;Flhg5S^)1`E{X`RofFfyLis4931G(fo|ByFLk~wH6 zSMQ{sk{CC}c6u?Q&K$(Lt8HFlQ)i@Kz!8!LKu?TX;-aaZRrP(RZrnlroG8Wi7qD|c z9x3J0J-K-5Z8$4d^{bZTdS%J4oQdde;)<8#kijkUd+R0Ui=VEs`3T=F;p8u_p#h>q zx*U{Z#X_$}HvuuC;0SUwHvda_5e~)n_GEqcnaXu_UHR&C$X@`{xANljt|1zrX}FAj zBFS?LVX-QK49SGos}j3&3t_y*F7{L+#BU2D8SPa`g?wpjbiNr+FBnwyY4whiqH{~~ zhpT#VkNStN(R0NC_!FX^fz%O=oUB5P75==%^qW%R#f7FJNh=-n*qfTLJq3_SQD%bl zd*StYlQ5X40XPmmAUCgE*X#=?T;p4jlO=l+52nxz1-84R~bShB=_#1T?D*<*#2s3 z2qTWIGHAh)mzPQOMTEAL%%5K)((spJZjiz{~d#-XyXs8CTZFeqCn#INqgM%%rSg&5t3E2o~JzfKgo zmY48k-z@=-?US-l;q=h@s9Z*eS}5Wfu#|j3e4N-{l$dHk(?V1jD!4L4VqhOv_cYhi z%DF!a0@YN~uu;H*9?%ULKnl4@SJJ0%gqSPq7@~XO7Lc!x*_7ETx^^GdvGZbJbm^ z$ovbaHu#tpst^Kw&~}MhdLuUFWh&Zl4#(qDINCLaIOMl;)o4mIH;A5Kh zOry|SP|q2f&_&%cfDK9x$iRC*PKptdC*B@Uu<rX?ZWd@ zQB(_mxs{0^UBI6SL1sH;Xzd8;(#9ZggtWeUoYz~B7~?g482IDd`*Pef7{~|o$J+8B zm&>o@{%PYON7+erW|na*ey(vEEqv^3@YX6A3To+l)S#Hrs-Ea^H3q&6)Ic?IbWDid z*?}=~XKKY}c(EKx1WYndw9l-g^p#Hw-k$NbXD(gHqm?~h@qk`m=vs-(``0>xux19@ zzpC%+y^Bx7uQ`*GaG)wteOio`=5AX$7qv_1leSs7Q%BSM<6jAn-7QU(eW6$-A;3yU zp;zN(>YLft5p4Ao6+d!98mhT2ExOZIx~U)gAj?#!48baoA9jxLD+I7V&T*4UqY$i& zJ} zfP{G92I_3*Y}-bTyB=^It+kQqk*0s=Hb@P|BHDg(4H?{^82o z`~5uBea9Tr4dKtW+-D^z(iz-?fGrC3hzWx>us(rG%OKxt3<4) zI|bxG$zc~|>|zH4&(&cS?Bcvo%`^b9v1z-nb9f5vwA7@f+`VW`nv_c-^W=wW7r77j z@&p=no`q`P6;g)CB-Ma9d{y$_WZz0^Mx;}6Xco>VJR0rC;=Z?7bCF@~kEEa*iCmyh zV2F+I3Kz6C{lFz&(Eo+aE*vneS#;hkp~X63p9N5v%g_i4L=_J0hUaC+Cm^&woR3UKSC!{W6@2um3#FH^-nET(CPLjeHG~#3nA3~TL9NVIObu?S^V#fNu zMoNtw0tyl=ud3O81xR^SbDz7H>*`!|$^8Y;Ft7P6nZ@t3t9UP5vgba?m7?Sy#Fkg! zeHu|QW-L(Hv;?nIL`;|(t`BPlhE!f?Qa1!V z)uC~ZbSVskCQ;d!<&)q6dg|W?jFf`7TKS*JxquuOH0p+N|-8a7Z% zF(K5{OhCJKT&rezOnfq8+|^5Y=c=~WJGjPXBRJ&*V)72@6q|FXD_dE$@E~VeKGX<`-h3CNr0&YQxunfNQuT zJ*`MbR_qUfYyNA>`Xt5W+`0CL;GMZ@f(^2cd&@~#O)cBR*FpH?2x)tTIQVi? z=x0@o(!CC$1|)7v*-h~&ZmIuwMb*P{ij*8 zmJNQ)GM{HH->FIm`vm$LLNj*!T=_loPss>0v3lRJ^pSe*xYg>T3eq>Ki;Oaxx0uY@)YTGisThH59DS{<=tF+1{mvQ?qbeo*y*b+QLzz^M31YzlIGrAHA^DL~Ubb4y-3NNEN!T zAsBydb2|QO4*hkatK_r9&v53#lkV{~cx2~ct@O8A zU-7tCipNUfbye_Uf{Y`xQ}}#Lv$b2gDx5@2FZoS7DT9(ww0Z0NX`im2sX#pJ@daG; zl+$k-0s1*(28}J1FWjGY{R{94{pYEtL-gH0yg`0(VNVoM8J~JtAYM+|>r{;qz>*;s zIyu$}#xi2_)E#8(?^CV8sKkht&44emAo1A;cS-s23J`a0dmE@z$xy|G%1nD^Ect;$b7q-WB$q!;6Xy(nYDj-{quBzx2 zof5T=$%d#P=LjC6kCK1~5OHj7QY`d~HP4-6_12WYOY~R0ZGx7>@U^)TJY^rUpz%jEL>2+>#i0n<@A&-1QnR2j zhJf;uaC2!D_CxOa0Sbq(2-|1&z*2-}jnpoSoDiKE#O^8kIoYU(f>O5_@q=tb&b?|lWmLU1mggZ{^$+6J(Yu9I614~s`R@ZABeS!2k~%p$PfzHF z-G%vVddH47w64FICoN1f-Q+cF_zlzZQbq#B36%0b+Z%Bbi>M{k~O_n z=8G2`p8n@&$O$PYHQ~Be^8;m&>L_oRyx+X0cU04UhZ;zkiGq;(1W7kK>KASfmwu@? zX>hV$H1l3}@H*|M~vx>2AR186Jf)?~J5L|hs=S6)R+!N&*8oyv44 zWzbzkTG027dsfD_TP!CiofzwKYCoVm(nkDpHIJF)MyRE}-^nN-2CU7PZyc7D7v8SxhrE{L!ptcGnl$G>&qel+?LSf#ng* zk9{5dG@miIOug+DOZ_sa;ez2j#qdappCHAqNs9m>v`QJ~-?Bebgqd>42PcLeS~*`^ z3W4K4=H&hs*pMd7rVWQU1Si<4VIvH8O}nc!M0^XX8P&BXiFvx$qFlC1V0vNVVt|CVx5x z?X7_f$~7Da4`dMeH#7dMq0SNI`z>=Ir8^JV3=8(X7s=z)!aTOijz82J%RO65AuDyB z{{6X*m#>^F@YmYC!rC^@tFMvbq+{24WsG>O1V)q~_=mDySpT~XbTV79-y2Lbhp%QQC z;z@ZA)FvZf#wvEiFVQUe?74Mq6!zyaMH>a^&nkkxueGT6_7NTR`u!ii3bn>MSH5f$ z8vc!6R5BTEi}*(O1X63zbpi8&%+vsFj^4d)b(1HsOOlgpz9fAYn|^vRpF}KtixV>BYc$ev=;1;Eb=XlutG;NXq@P<2zgVc z7v)aOK}M4TxVh1oWgx+Y-CIo#c>$3U;5Mnt+}!GWF0)+BzP-84)A-6B8zZ>TwQ(xRR zM=%M(dkmY*38uK7l=Aqr#VoH>P=)Ah%?BRbG%6=f|H zhMeA=p$cO^dJ+z#L>LHT)s@N6VZrN`?`V+e%Qhr^#lHsgM*d!ziU6 zA>)oCk~wV(B?U4LKeMkT^7^F8A{=UXlS*LRcE_FBB!3%FaoLrAV4Nz9Bl-M8U- zL3so@l3SysbIvf=^)i|14W(bX1ZqDK#vpZHPNbkb<+FtO8#X^3aW_hfcHfLYL#ULG z4Gr%ea!yG80`$HmGezk)+cQt#ri5Wmv2`B2CFeO+NN+CYl5U?y?fkne(%$2{De5CQ zjq}Y*T&FnDcA%l&Xt^8`v&YK}=G!WNbMy0*@9l*SkxArdm%{d?z;!3EBb`L+2HoS? zf0^Vgilq)_HZ&Q=X_`m%W@5rQ9_M=?dDG_hp>v~jtsA$P)+S6OT5GYtIk55Dzrxpv z9*(~IoMv=Pz7_*Z<& zp;)z{Fg^%?mq(u2p`p+|i?1!MyKMEr{W>Ud(#~*G&`QA=?rQg*OZk_1l+ARXQGoZG z+xu+V(@j%~WWV#BEYe9GyGBR4#OJwRU-x2bTbr%*NAB?sg!s{63UZD6^1_-2)iyts z`|(Pcc6bjc}@f$BZrg+2WKq z`v=UrLT4jY0GZx#I9bruhB-b*!#Q?Ux!w;Rc)mz--UtTgkY@KQjsztE#YGC4#qFG9 zJZfn;1m79x$TZ|arc8HDytly+>}+2y0d=^-D4c>*6VA&bom!;9TQlgRjvI*cHeQz6 zCesqZUx(SEGkJ6eDqU$- z-YIL0{_`#F^$SJFazuBDbP0=$m}E{0Qui>^mmUN3RUMV23ak$#oGW&b^2E`FT;8-F zvPb)C5&jg=t63;Fsf`b4W6~k@Xp$WJEazZndvKVx3R+fwIB9AO|KWi<+P9Tx@bH{w zx2!>9wSJIP0~@6T7`v5?jF4O?vsA}0T+(H{@)CM9d|ak&^-tOXJY_&B;R1}61vHbw z&UP2nO6J=zVs2BpewH%F=Qanok)723(a22L{>2NH&}QoO^7n`8Z(?T+RS6mvsf-gA zE{qmd{peIcZ5{D?!#y#W?aFyOmEARV3SZI8c85Ye$8!@trCwGA(Ir!Y-Tk4T#}p)L zT7rjg+*eP%>#J&UF()0b>_=fsHP+gH0k)a|CuBg`ixe@VgD-G)Cb>P=Hf~}k3jgij zM_HpzJazHbUeKPoaI7HFvA9F=Cw`+vWUu~BIw{W~Ft;-Hp^PQ`LvVjV(5fVy>C{Yw)!O@7p}Z#W(qRnmtaog6O)}r>HxTscM6Cq)sf=#^ z!|tm!i{oyaMd*^yVb>_tQLK%;e9SCRdC0)3yXi9e!^~;Ro z2W|J22Tf$`Vi=+k`&FUey2?mY&dcu$o_?aQlH6l*Emn$AKaR%)#fgCX95)QRyK;w! zA9bE$7>g8AeY_xbQJAN4c1}pp)xi(x`+e)IXeVQ>qvPF#PlIpagxcFq5mNdq11hUAH0k|TUC5k<`7r4gljI?8P&=(`{Q2~M2KOQud zSQZ8&rm$3Kz&3~C)otc!IlR4PB<5okzcqqpqW1H$S&B2LbD69-uIh+6pVNE#P&0JK zv$n8)R#Z)wv51f6Az^L0%Vyod{-YgzkcV;`ng`;w6N#Qp+!tG{IM$pIiONLuZb z3f*3q3xQU3HNqdLKu2tdXTuz8guc}UiQm$R-TZV2fv+|cLRvEky%@e*Hpx>Bapq!2 zFcd!IfRgc#+w%&)ajD2!YB&srt7{cpubB9Brq$X>N(}^Ce_C3|hL4)MU|OtuDTM8) zh6v&-(~}q3726Ns53~9;C%OsmQ<9??WVHJ5nJc*3d3-E0`l*~|4k{7;v)@`u<}hI+ zcN!gy8&!#)<0Pwh*d#j`5EoMg)gdQ}k-#~2C*pl!%%960X-v_g#9NcH2g8DIB0^Nxw%^iNG1-6q_#58VS+Uc!MV^R{cV3X8{Z7M_tagASC2<|TZ zDP2sL_ig<^zCE&O*}YcXN85qbx9A=@12VYCT(Rn=^r0l}~krdF2av5>9WHc(_!J$9*aH zc_DqnZYUvCGIeB`d5w2U=18+x6`1yxrMB8LDUKr2Kfu*s!A#0d!t%*%Bg+x=eEs=X zuXg@+^G=gC^T#cMtV9hrIyIFP_V-awOV-KuXk9{1YKGad>PPzg`{c9O`E)l`m>7xf zSpCvIS=vg~pf5kAej)BT$5yMaOk0>Tda=Rt(L{hrKK{bhB%T(0wW?}YYLAs(QhCwv zp!_y%@@iw44kMVf3b`bM2|oHSY^V@6werm-X<#anuYJF*Y~_$~2-*0tp}Mf-dd0Jr zCKgW2{?D9#4adx}>`t9b?w~)CYU;#ANlE=E!YzS^WI9mlZ9ch>C_Zy^89*AN-})C| z6vjNSnL#&Bjby4`>b;z?$!ljwBbxU~MNj#{q#FInj!pO|0`mZnAcQ1U_93^Wk2sc% zNFdfB8KR(i2W^hwQ1BKt7ugV*izxtGFHPE(_M8$KD~r}>4`TG=Utx9KpBc7IulQ*q zFc{?2`LDJi9=J0IMaJw4HJmoA^Y_RND>Ev{&b$0%4tx(~BQ%=BA^W_SN<31%!Hpzq zxNd@De14MA*R7nSWIt9$8<+G;p;7#KrQ->`FK~o_EX$qHWvxmxih93QNOv5&El>JTGT2!<&!H0%*-kOrf;Gi zLj`cV@bOkXH&l${fDTHObd3wheviB`l5U@oe{FQ?asfv@r8t;b*{?{uV2Og3I34tk zHchS$k|zywPTx5v!VJip(?O?G+!(1R;v_cbTRUUX*dl}2gsChvrusBo?2JMFatJlx z;`=mR-L+W7jIwIUJL(7#?t5^K#q1QA@!-m$H!1&KnNeLSc*v0{#EkX{cL?%Jaheao zZ{USRq}m?E3s;ERjT|Z5GU2cVRr6pzUiBfxn)8)J6>FZU3YmYq!jn!jNMR8aae5l_ zo3&~qjr3de1!d0hCh8Zkf3xtHuQ`scyAV+{x)eG-E}D<(&cnXtr3iGKjF!k2ZJCoiEuI_ zOJ9Qa$>-#2AUMXilB-<>A|u**%_6PQMxNZhow#?TnzzGFU;O7bsE^pC6RNPi_ZJYZ z$4Ukwv>G~x*443P1Y?j|*SMr+fCY|y17#aoZ)ITk4{z2xxEB9Wpm0;=p3n=N>`5?CCYjeBx7a*1R zI-bDTvFxBJ`>6?6o`sg!5U6YX32zuJj?=YR79z&LWsTfL#r3_m+V~(*A zk``INzsT}s=!VV2F}zKaY}x<&%O~$8fvl4^rUXno8O!n3hL^ z*qH@)H6A{h)$V-H(ZV|_0T#|AqTTp(SCw}j8B+Zq{NB&)2$MVSE}}Gj4L_Zk&=Yet zx~_U#t~M~umZatNL;UwM=GA4&^WE&I+D&8I#fGh0?lBtW5JHW$gO;r;IgV0O*kGW% zQG%e0kZDexwKU2EYkw_15uQT^Yv5Iq6A0hzFt1Q3;m4AgIot1 zVk2mjmY~+@h?6kHU+AzsLjsu2D+H5=Lwi4lVv6Ohm8It$J$8|b5HweqsuEo=8n6A_ zTStH~l=3Z?fy%%etLK-BcU9j3u|xYIlJ$mf=&vA3NK8Nj<44;@rU3k1aE08Kkyjr zi9YkuAilX-jhDj;O?2v}X(*Bj&N>6j$V5eo36Z}S*hVrRggNe{Fo_e>)3mKKN~6*Q zaz%fpzbmz!fGjz%4PRUmQkiRD_=10CaQrgeaRy%ZFS^ zuLV=l!=nJ~wmA!#W;CepokrJgtIVd-m{}~tHaGVisHE0qEev+typiJt^^!A$vIfV5 zZzQpkp0%+{rOgHQM0|>ESH*<7hA&i{(PA%ZJiRkZ3mHn9W%SuO>(k5*oIV6xyb;Xi zS)Vz!#|@9p#&>9$w}#D3?%Alak9l3An4B+i7j83C&xx>qDA0axAEBE|WezQ}=5+2u zo~7|;Ti$2ZNxfQ2<+GpzLIWNa>K;=1&jr1D46@+a;a*Uy6i!yi)yjs}>*1hT&l>fc zyYCawihuTF_1=`t@nMmF2B;=|W)xpjG@)2!;Sr#Cw9M^Pogy6l5D~}9fJwZJH*u;7 z0<+BoXzI{Ur*p{nx=yjF{ZceF?q#^!c>x#nc|bqXM78)V4xzs5U*QHZi$(shMV+$imnqUlLNHz4{_IihJ z$V0EY$yrh-Z^ik+XPL^&Zor+DZ##=DP0oK1*}kQql$8Uj$##@yC6K+Ylc$lQO^BDIEv{?qsuxo%@Bs{BG?|Y*Cx9 zrsqH^rxz`sHWe6=|L1&=LxF=ujwa)Vgn7peJOJNl%~LjSwL?wreGVkwt}yBykqov3 zlCCb=Y4vqIG&Twb%vdyjD%O(v&_)x{2TRTz)|Mul@EjfH7^w;$WfGJ~XhJYrRq5`t&~F$WF$QtTU|_HN8hmB()#i$)ksnjtAp~|5@{5&OXU7(6 z)>?mB_JFZAEvhKQL$Zgmj{*y7lAHeWh0$^{Vgb2RP4WjqlNKA7l`@B- z7Z-Y2W{|nW72I^Qv;}h^X_cGGm~1+>{3OW7y*&nf-)w%{emKld7-GOn*-Yrl$tPYRXw>wGkz8)X7t| z_qN=~+=~Z$$MmPjSSGd5U_jW`{rz76`o5!j4Wl@ju8 z;y-9L_eT8GB8DI}CJemzXco1E`tU?(V4#j{tVZ4=n+@+(Z>;g!{jTIuXLPs{V`L=c z#S+N#FSu&KJ!z5om;(&KaC{Czn531k@V7J%g)2? zt-5GR2o!!r#ETPm*seCdOK@Bzti*PfHdyWhl8_i-zrm+u z7l`w{PSk>ibW=`=>-gSUqg)>Xe0yXGvZC95hrO2AZhRo$ZH=kZJN%f^h_rKC=I?s) zCf%)R*zMH0OWpC(5G{m24kPCG9MU4I6jR+b8-SK5v!J?f;Dv*fZD|jL4BJ}ZLZ|H1 zKN$;z@!3xUPlGB(41v5^?-2@Ny_L}3AijaG+>oBOr>A_Kp9|%5Zt;(__WYd&_mlzV z+kF;_*rB|jdr~!ks-&HTw5v#5It`bVzGeI?#oc|45ZfUG*G7gqF28k9>}hNGo%scJ znmHAc9<#e~T96S?lu|b%&>O@LS^RqGDB?wl<9Li%TqSWDAn3p((Q2Su%L-WE*{s=& zvc3XBsLNKxzpJ^Ci8+XX=N6qFU%djW}sMv$?7{BbvaASI4?{zcf|V}b@INb$RC~+;7SI} zqyv~Hv*2Lz2RFT>_F0G{d>2EmSGlY*==L9AC_Uf$>ym4s<#uY_!3O;a_5+TX+LXU= z``v_II~Qf7F=i^^+lgNol_zKSR}Tt8j*gZZ&6y)-C3>h;YR9!dn%Eux1q^C)sTwqu z0M#Geo?!>+0`BPSRJ@#v**emkPf2WiU4+fCzLKAPx&1pu0^Pbz)WickD zrN<>B1agGHNh$kFn(3bXp)_PzeAR9CYP%e`*9bv3Rg=0eExUVqOn!X%EQLSBrEo_r z%s}*HuTs#)3DP@h3jZ7UYPs?T{U(}+Ho`x;6I^_w6_aOG-7RSSc~-b>Ewn<`?iD?PIn zt3$6x1lldq`4FjTO1QbM*naL`)|a#`-c;+~;EZ}S$fNavSL@L&L8E2k5y>-}UNoyN z-I$nR#DVPAi2zeOSFp7>5t*|D)vETwXOtY9cVmO!rq!k7e1P9S$IIj3hP{;4*m6`H z6nOT@cB0U{T6k-YC5(Afn5TzJO~g%bM;cBn#+y z0G%BpHk}*w9h-Lz0%K?syDI?Ca_|CtmM1$piXB_yl zu9>Dx23hB^A4jkD)&KzDNr2XV8z$T? z<^aM1srHPo+;-A%Kx)8C{mV((C2~>4Ge=w-FJU#!m?^I*Ax^5uMvYuS!rfyLcrr1& z=2&g7gQt#zawjOkd?5)+@lN_RSHhOkbbjx6c z9?S&EH%a|xtK(}V!Duuq5i#dqnAkq|CeF|QEBP*6p;55RaGdi^a@H^=-I8n&&O7rR zkSC{{*X-r;U1-s8}ot2O4y5p4a-bBVAbsi+QVFNM~hQXOD(KS|{mNGu$i;wITUPInltf(}as{SYd^oc)Hsdw+V9>p77|2yXZTwj#cqM$FiX!No3Mc%N#>uUmU!)6T?I8d4Zc7SvJUazipX6vemUmDNn8ye z+y(BT!n`F|zjUi{nnB}Zi^;{wWhVhH8PQeWwMFR8_ht7}-IO8stoHdrlm0b(+?0a2U92Yh?+0aZdc7$W)3Q zCQxq$g6_()ki2L}&>kZ4nU>i%5aApgbMt^jgvh_WTPK;&qt}Cv$ z3^=31Bx?!#sY(dt>3RC&nCr9aqx%!>v?I4eHnzm18e8AoE39{0I^6+6BK_4T2ye8| z%SG!IRJ*`|{O^Zf3#4sf*=ZZH6aVx~;jgyKn_uYqr>Q91*K_B(oG>RQn{bywH)Zj~ zLa_Ik-~@Kxhw5jNz^zYwTd#~nnAlRwY~RX#v=5c z=~I%W9l%9q`&)mp?{U@2xZ4*+O)Ch|H zZTEt9gon!!B$>$p&bh_zM#%G!4yZHbh*1#Na2x@^Ny*#N|E%9ycF+aZZA%D6>!c|~ z*K`gDP}VAav~9a7Vk?{0n;WK-_A1#*oJ*S+pZ&q_#2iz2RRR}!Q{y=75#MK#5`mJ* zHvNLAt#ti_zSU*=<=IOy>r)c^zie~(VPt64!S0vTtnJzd8Os-1oKTu`Zn)cuiFCy$ zO-3}cNZnS?wC}p1Y>%X40+|XrIlTik)N!A>ab-X74atfM2J+TK0+)$RfeQ}s*qv&f zXU8JoKp9Ii`We)pi{U%S`Q}0uYRFy${cu2yfT?3ILk&Eycd>I3?sc79NWzs;7E2}N zhptAHISkDt)!Tf@KyKMbv1tiiE$-T~Afy6Lt)H(qunSpVlxCgfot{+nwiHgHyLyAK z>rl4oFU1ZkA^ODbWuG<3g_P`GqN04B^?lw&5^qHzX~D?6W18DZU)pc?>u($UewlGB z{soj(CkvvBnpwm6UoV6JM-Q6S&MJ3`k8gV#^X-?IcLH#);QcnaSqD+Uvl65a&C|go z+T|@YH?@?9gOF02J3FtQ53>FzfP+7^lnHE@yYX^A!>UTGRiHA;CLxGhcfHkR+i-q= z#+?(uF==JJ#(@5&2x+W+laQ0nh`wx48}Dxy6*X^lJmzJ%fWE#VuB@(a)b)280b0!X z| zdKT-GfpUjP>d_?M(__$J%P)+DZ2{)dSFfITgx}fO!a_#KkcAlP#E18#K1iQs+e|cO zch+C+1bprd!NJ%HfHliPgw-iJcIq!~@-Te5uP`wDc@Mf(#afKA$h)+vf1$sV_El!& zT1V(x%Xv$pib*CA*C!r5W@KBgRi8A-uy&6d6TLXnBx%J$tq?a8wL}2s@GsXqXs!1Z zN}?j{ofxUWpSMLh-}{SX`$%V#4nfd&v6Hm^@ExwWnyA7%Z^HKm_fqi_b$sd1>OaL3V%n>ey(RG!Cd3xuy*uK4;#0)WK9mx$ z4)K%cbl9g(Cy`>E<$t+pGiAVz<84KTr^ui9Ul`v|-;zBHcBI=z-Y8Ba;Fc)464u2fjX>`^zOD=zFUB;w{F3uxmZ-pSUNBy;wg%Pl zO9%JJEI>aGc>h^Flf&>NepF_J_m#eyx|j_8Jz!ZTNUCR`m(BL)?1l$0l7>0^T}+$V z^`uf}1B_RgLVTk5=imC61o4*D79RXq4o8Y;_l72XeKtir?ZAI4L1<>C)DmX1ataaT zToJ9$OIz4fhgzuW;**Ev$k+_Y!|jpQuLfKsSOo#z7#gXY=q+}mjsp2_C%_0Yo(?3g zM!tug>wMZ6XabViB$^BjLeL1;;_6|e?-DMqx*+e)^Q=(5Od(zd;#f?GN5Z;ds&WvqwV z7v}tr0IncW-`M820m~5IC{xZk&$Up!AxZggM;H_oj4PQNg$K3;RRA8?_r+Vf#_hvD zO0MHLAmDmc#F-LWsB6do9Qle5KK1Q((J$HK$lxfi9MWU+bvPFv#0`5C3dSV^upR4Y zob@*yAlgO`;YiZ5k9ur`qLTfniZaK0V?CjM@k&b#*?*+u8#JrAt zmcjSpqLFhM`FeU)p9i))OuTi8{{W?B9nGciv(UDjan32Hj)JBM-0?}YbnWk3oa8vq zYHr+AXvb z&xrPo0_SnZ%h&OsIucn-v1xES{Hn1Gl6dJ}dGQZbTj#Y{Gq97}t}9J$bsrM>Ru+n3 zwux0;^N=t)lid0WTuYk)yFv*i!BO z$f&Gd-r{NJyUd9Sg0bh9Z$ax$F}oC{%C2TjcP+^)X%QuH%8b{e_IN_{Q?){x^MwFmPgF!T$gXSnRhq5ArIR1vCl2XvIkNM^`;=$HmT|BN`cQD)S?lb;;4cy zSoAsesbf+zns7~~q~Ho^17uB#;Cw*l@5FMDeppl|(ARfx>l{wxj4G4Tyq4o!A8ngE ziFaeW@u-{7(wf%hp;6Fr#V3cD+xV&;N6bq8Vyj!~P`1{}1c&)TS#zCyjZB-i4it=R`tNMlg zzMX2*+{uL<2U5q{mq0ONy|&Z!-9~xj+ZaXsPaF!t)O8I@#Ln|ut4C!41cvF?^)!~z zKa2Helo1R8iF|b{#szivc97XnG`B# zN-t!GQEU6i_gqUK`XnTD$>O(kn;9&kGD_ZM)c*i9&FXzVmC%{Y&QDkHXtyz~kZh5+ zKh~nQ_dYEjxkeZaJ$&z?Ys#9kSl_C$<1AV1K7MurZ5k3 zE2=;Xvmd1{2;gTF(m|=j$?)>siSmgAk6OgP@V=p=W!1dcv&aLbdY04bML~OPv%=B0 zMjhyJO$|w-$5zZV-egV`AG|^2Rn&l$+an27_9CtLpFp|N3tk}t0{Fl^&1YI%2@ntt zeGPL;Ei-?$z03aF7 zBer@{D9=+=NT7p^8gX2TVMI~DsQka5aU^~;TnakmjRuc(ARQwhemqx+YB9+Sgp8f} z&$WAAw35D&bBvHzq59X6Ur2*`$iF`F27N0jKY1F#d#vZgvqa!-Z2fC%NuFt2C#6FT zyfa)6Fnqq%t7jZ>%wUI{bTs89Z3dTidewxR>|kt@x-~sLs?OD6)0%?H7KY9k_L3C< z{VII%^JIP%W+D{s2Q=I=^{GUQkG)KMFgPNZ5t2F@Lck1lsNC`f4+ol6P}#vhjVpzP zCw2uCc^Ns)6vuLXeRo>lANCF^4G4e4p4%Mp{c7&IGi`VP{{WV4{Hq$zQio0Or5Ah_46firJ@-c8T=rdgWHkY0))IQZYXzXL)GVl+w{{W3KuP^m!h1BRi z$t#HR-NEd>z<2Lmyl^BCq)?)wAo)NT#Y@-`jilPgYvLQYAdE*ELc3IcwWF=%Xx9XA zO5}cZPROwFRp8J3vRb*XZW<-YJmD#UZp4FO22cF6QhzG*Ux{(5Yqs(M_oErFMzK@< zpKp_o-LLYmKKPuQZtdIH4E}X8S3`@lFm1M-yK2Oof;!bZ79=mz6?vESu2yys%mUjU z+3Q;V53`BlxxKx0TS!Q9r}&8NShs(3Z#>sk;VmKWBaT@emc?*HZPG^d`c{;dw==5r zvA?ETJ^k1A#oBFV1Y5&fNB;5s+}F9+>o}6<~0A6}ilYf=MK)$JU$z zkaL=Hu6U(kg=|m*Vk2o>@=kM4!ya}V}nl)3BeQr z%ltr6{v8-!`D(T2){C{a=Djn-tIeSsX3jV~)-^Xbr?C5l8-ZcRTF$;I z$AORQTC?=a6M{L%YRI}E@C{;(PM_hNSksna>?3PB2r5nj@0mW@jyemoXgOTwMdpTanC)dBRI+Ysja-^ zahex*8K4D`43RPbIR^mruPN7jH*}XW$uG)76X-EswYMWZD!uibH;@SDiCsy@J!)kZ z^+iTpr;6HX^9zFbAv42z*ID784ZZEl&cn-9$IM4s>p$Tc(^O!_NME`$(yv;<9jty- zE44>&T7?IFOO*7`xU1!;T=W>Fz%;<6$6jhca6W8S#w1RJbGD;6D~3Uuv$=NqR1#pG zd7xxX8j--q(x7+;P|UxLPnI2N2*}1M0bN#L+)a`1MNEn_iYU|z9woej&eDH1#_2&t zPV7joBT3P2^?Rf@W>I4iz$E?Q-AACSn!cHL;vIVaYq_LLJ62>*_OaM|d)D`d3Y``Z z4?KhU)jil1{6P|1*jdL41yr2?1GQ|O#BeJw#M@TF2FLuoC-be4=V$Oi$N~Pf>Q=VVT3aMA#uZcm0InCpcE4%Y<7OFHRJJS7 z-ZD8IE25J3ZgbUs^@zb4%`SH~F->JCaslV9J0K0-m1bg+bH@~fk_WlPG|HrmDdv_l z<~Rp}IiO_QMk$gEUqZX|37Yd% z2jn0UNUv1Yl^;mDPI-;G{VU8Z*LMRyD9vM2b819a8C-&ECgbn>ir2bw6qAhm)-;)E zIpmXyh?>2RPw*M?JUJFe-)1B8uCXNCIOjEs;fn;e&}|WyEwewJYxyUVhZWHsj#+BS zO81~Eb?Z#?2mxJx8hFDF4r+ulMPA)%JAc}+xyAhCD|W#I2FS9)>%Hr>2m09NJ8qo z;FG|v!%MPDD^aQHlJ1a32xj}St~kYTelE9fv&(xNHwkGxo1-%S01-WZI#Mb-n7Ts# z>k3Xu>5AEPD<3?X%h6fm8BBBB_N_S^sNI_6mgi#F>U;%lmb|zuYl! z?6yuhHDh5t4Mvh+kloK(RvpOzVSz&c!jw9Y10HDz%W=|~BqdZgQfLe6wJ|~08Keu( z(v^l+x`fGTD(CMZ9Q`ZHEytFUIU~^5t;>dq{{Y=T%DkljVIl$5jb~MCYf0{8>M{%h z2U?|P0lU;~;2-kRKtD59wKdq_d)1E$#3u12pd%xGe=5!h(r(E0yL?3gMT`Uh56Y}A z4>>ug_Vdz|0xO{$)+8#-I~r(AasdXMtl8t5kb83=BkzIYUb4{K@|zWkw?E;mOGLG#xu6CmVR%&S(wzY5 zJCD|>k}ru9(7Bo@IwrRIoT(6{EYFe%y>O{&R=@8FBP14dyyhSzsun}Eg5+FSy zQh2QuS$sz!k+(x)hY^en$bY?$N&t?=DfOr!mf(4IQ6Sq9>mdWFu4Ce*%Ex76XgJFk z+Bg7oAI`lQB3NNqm!g0(>0Vp$MFg7FydaWN9R6maS|a1CFf`+`EVFPrRmpzh9y5+{ zS$26JUsKw(_bG*QgWkDb=T*-4!OSG^`j7}IBLmHJ2s6oM{{X(Lo$yMZ>}d;q(=o^8 zT@sC_f^dCnsyic+Td~^+`>s#clVi67Y5IzlMi-h7PfDu@g^Mbjh2o$_BZ8#jtjs`& z^VXt;8>keR3Ab>`MjwSV60Qes`4wS)UI#Sd7wb;JaiT;c?+?zD`w>hitWP+mv?@ku zxLA(v7Yt9vdMx(LKVO5(_T}nEc z+~)%z{*|lXOMf%OF&urxW~__W%I(iRjdVW`2JLNt86qP}1=QH|j-6@8BAu~^@E^{D zz~nZ5mATA5Fbaxs%~wX+*(U?utOi5;L-VRa6KMv6)2%z7tuJnrz=-swI*rPHwKm^c zia>V`dS;Lz+koQ~*6awT?U)|ag(k*${ApYT>(Y-X9`u4%QdQSz>~q$XVU9sKpkc5f z>-kVXJeqz7Q%QmKpa@+@b5WGR=~I>W{ON=d#Q-%&BO;1;9(@nIMsDr zBC3AwB7Rg}#nr$di2QOBBb&TBnFL z)|BsCEr9TNmk0Fc^scHUNR%Qd01N;J0-}N>g%u+JC>Q_=oD6bkKp(u;Btwo|}v)nT&k(8`S{pRgnGi@A`OkDEL#GLw*Sxwnk+M2t$ zHLM3jlP1I6Ze>m4hK_HEMY$PCYk}NyK;JpiDYJSy9P5>8yyQ$1q=5_Gyxvd_lUsF zBYJU9+q<9Ef}Dyh2QTBNKiM$E2>{45`5N=-TqxtUdKZre{7Rv2n3t!o2E6Ui@J|5O zJx_?}r|_BgTbY2av*FAIwV7NJHbisyn&w_IWF+(*E4KJ+8k?Kf>O!de$)^p_RQOKy zK^{|O=ysxx+&_|U8aHm0DVv*0|3r2e>!kZIA#?&2iBX8 zXc-t;0sH)YDWo}-cv5Oo2&QNE%1@;L2XOuneuor=c*e#bOjNCo4H!741RGS0Fyq`* zMm!uVF5h~+)OMsOJq0Tc#N^G@U$oQ8fPmyp=jCdVOk`i~5h9tw}DO zW5f}U-akN3Wv!b_NUd#>7|k}5H6nI0|IrSS%? z7Cpkw79u4-#p=iFRc)c#%Z9-xjMG`{_wRX-@qh(R(dYM(F>pHZT=LZER!YR~ReF%v z6p-4y`$#z4!vnoU12`jsanibP2iZwzZ4q%pb6Dg`G! z@G4##fVB}qWXhg9RhZ(!lfe9Ir5wy?UYn2MQY?5q1vo^#c&DAJnaKRAtVsfAbDRJH z#XPX--iU&g1-T@XPxA4SXaRr>XBZTuam6^}r}CgF#RDN`^ckgIDo-pE=|KPtP$P)= zi#`(00Qe-c6#XlU$L_%#`r^A^AHf#*CYOwyP1sUR=Hu~~ufQY@#3@LfX0+_^u&S(J&fZ07L z<1_*}=AFhmQUW7@O+s)oYGR`Wsv@~O4k!XQ8E|@0Przr#2NX~;wy}E6c-m!#Jox}F z%lremKJ@)JOovMnD!VgELoqdFY@)Nfo>W}C&AFWQK7;92c5eWY+pRaahy-lRPU51X zc;sCE^EGp0U;x}ydj}-Vo(=~+2yZy4xvZxqPbVpRHOwk&l^nC$ir z++_EwnvR~51$E(nh~up;krZtHREP{`prq!SDMh9lE8IzZxXWbpHSHQ5ir8t&ZFoiGT8%)vWhS&B|;n8r^y4YYK)j3 zJ5+Lz5vlpl<511mdYWyBRq22_Q^tB8DT&j)Jf7l^3Kx?>Q;w7a(w=dQ^%TI51a+o5 ztFHutX?O1@8RC$T4Inws7}&)&n6t_Y$q@>Jvgf^d(-sU-4_+A8sCcD(%>v|(s5gF9 z;!SsP9OaeIQ^jL7snJVgrYJDGi91K7j<)W!XJPF?5?~q#Ij5WsJDO<+jL-psz3B+Z&S)o^UDzEc0SN$e`O&#WBXT$g z9`zVp5_5`I7_Ecrfk;LMJJ6%3#%aK0b4)~Nzcm#n${UkK08empFzFcfZtV-+kB;&ADgpV|)!kX0LVllMW- zO20mjw0VH22h>-lL7-VjA(BYLfPO{nqqco%^JuzN!-kgCPCa^7E)3aJJeuy&u3#3i z#&{U>F~_}J)BHY-0u|2Scmt`gPM1ovkx>@i0!B{I2OQPdE#Zw9Y)YgIACyq3+#O!W zJ>lIBTRqdxy9*2iJ%1YLz#+L6HUJ$c37XYL%EnFI-I1d11jA1zoDS>5jDean_t;U@&@jr5jfmJP%p`$?+5pGz*jib28$*n&VBkwhTh= zZuzfmyo6m^OK*@2iaG&) zPfF-+5n;66i}SM)`BrLqfyAkhRQlAZGWk)`_)_Jpb!|Q5!4a{E$nFmruV5WI=Dc4* zv}?_3)*!`qva!cbxvyo2-~~I66{H@9H8&ncMo6S9`qQ#D=Xc>roQl-UW^tU-EeI_DkULhW7mOGaz{#~EZ$VF$0r!1*doE?!(0d+l zs(5Bdoy5p??j>1k0_#umR!yo{AO*59$9nd+fm~sbFz=C6Z!{ZAw%pOKc9q6BBhs_> zZ=s^8-19#STU<}B+U!WSFq%xXoS3 zBZ6wKB;MzC06fxTLZHHq4Nd?6VD#@yOp~;Vn`Stn0E7JIf^a%eK?_Y>oMxDX!0k~= z3ozJmj7iF3{=!>4METw?~5G3WuMa3rR&;L(z5 z8H^PmI`%^mrmI+`*EdSVVUjCJ;?$}gws zPX~8HN=F?i83~O?I3I;LM1*8yeGNOy3Jw4nIW*IhBe(-__v(qA+m>oq4Fd64#s<8Yi49Iyc`c%aOcBUjt z3hh67S^HXg?IZ=vXfiOz(^x}XSvDos>P7#&cu_Y}C9EDWDkNT7k(@ zPVOqnT8crt0){dl;s-dQo<}2$(MT>|We1We0*rK`imsvxPz4lI0KlUF(M25qFg++L zGt!DEKp{{^N&<{@qKZHm7^TNFQA`BD&S^*(qKW`&sPv!!dr?IIIaCgUokbMT2tcF& zdQn9TLI5d18KR0vV&HmGV0uwSFab(M6i^|!80kg_N+_TO!1SfSqKZH?asf1(7YBjX ziYQ}2wS6g7K@?F;7!*=I=%$#Ee+?8;0YDh<`O~^|qKW_tk4i#|=qRF?4A{6F3MitG GAOG2(7rpra literal 0 HcmV?d00001 diff --git a/python/nano/test/resources/train_image_folder_jpg/dogs/dog.10.jpg b/python/nano/test/resources/train_image_folder_jpg/dogs/dog.10.jpg new file mode 100644 index 0000000000000000000000000000000000000000..33824b0a93df9a3a767939ef6ffa921aad1ef750 GIT binary patch literal 12166 zcmbW7XHXNs*X{!Wg7hv(s6jePC!tA^Dn&Zdr1zGDjv!q`TIe8zl7t?V4k8LlM~V=7 z5$PT2f{NGw``)>C=6<^Oo}Jw<`^?V%X3xx?IrIFT{ksC7)78|`1P~Dc07U-=;O_!J z9Y6sBk%P!6$jQkmDJUpuSm|h}scATvS?F0ITzC0+xp;Z_Mda@D3rP#{@QSNSNW&DA zm6h+_d#IzXs4b_Ytni;hh$tv2X{c$ybaY?^L0&4|{! zM1Ok#T>tJ#O7vd=_@6;U3?v~X1CigNp!_FjrUMWY0fEFMKvGf?l7H%mfBOIudQt`+ z2~{#iBL@&KoJleoRZ0#~tM6tu{`!MY$`KKBi-P4gE8CsB`~reP!qPIba!{DO`a=y( zEo~iL6H_yD3(F@~PR=f_ZtfnQ$bi70;E>R;*tqzF7l|*EGBUHWb8_?Y3(7E9TzN%h zRdqvSQ*%peTl>3DpL=@y`UeJwCMF3}(=)Sk^WRt3);Bh{ws&@qPfpLyFD`#xUH^xR z2mt(Vtbgl&gZ&>|^#8bsNl1Vsp#N|Y5eNU9Kzb5V9tkoARU?oCoRL>Dnw&`uRa)PD z3nFFwgV_=Bm4by&`upAE|DgRB+5a6_%>NhJ{{s8pT=M`bAkn|a1JVN?00tRx(}=t+ zyj2WGGMH-Xtm^xuu?PY}`cg?>TMVm4lyZzlNQ%T1DJT?46-}Yo0O?57q;l3#N8h;r zgTV^cauBD11q^K5#DI#A@4_de*sek;$S-8LSzRBi0K)t=cqpL4W{uyE!;7rn>cIAq2~clCPHJ&80M2Iwd^XD&9RjPIPRJ0p|wY3^xiB0lQ+%zaV6 z`@tAG#`L}CbMZ=fH^WUEZXcuWTzikFhF7AY-=J)UCq_b zU|faomvQXOL;^GuusmyY{mv_D_pkHZ5laokRd$Q}K$ zXaBTHFs^v<+oF;RT%Jbh@|)>Yr%8`P*{~87nK#mO5gLMfg^>XORq1XH5~FHx^jNvj zD2tlTN>Zl#pptUdr}vpVy#c)tP+%3k8U~GC+NSPkA=SfYb=xGFw^GRoK=(b`xHlun zKk=;89pMFLxy3#X+9ZQbW2019m9t73&<2K`>bTiv<#mu=tJot&- zr)F5O?(;0==2ym=xvwNgUsiDHDV|ks{rH0|?0qLS1H^fUDw@RU!LA~+a>c_dB(L?R z=XxAoIF?ms!TQ9*khZB^A$^l(b)r1kkpU!5m8Go&(RW~<@lS7+-iDdHBz7K!>E_yb zi6~Ytf)J4_dp6CZHHup~vnBSVXVsm9Iz`~toas3RU81oiaN(5sjJ*M~?F&5e(FY46 zRnhQZ84MXpxW}cPOa=&5+OhQPTl!FznX(^$JL8rNoDH0UFSZG+la0_`dHAYDdWGhk z%A!?>|13Rr-+jX&AQ;S+V4x|)bk5N@-9CJuPM>~c=qd`1BAEBq;rtLKka(xxV(R9^ zYX~C62>@ky(1;-4`WX|Vgh5iD2e5Z^qz%2m?Mh85Rd#cP@JA;_zMs^_DT2>fSB$|C zf<#c(?%!3kunKe-rw5U+KXl9rpgzKH0A*?)7(2n9F%c7u60HP}u#NpVRaHFeBSbj#eUqu<7fD0Ub9d=HUM^8jcGdxA zUlHxq?UG0)K|UHSYFr2}hwKA+e@cL%UctaIr6@~nOJ*?RaWJjlmQ6}Jz?IgWkFwN_Zq&ZCsqUfPD#XD;TrU=;ebW72* zafhe;QVVq=attG@n(|H+4|D;8hagYSp}OeLWMeQgH0W?}QGv>>_!Bk)Six638l9o# zX*9--APdC;GIXvdD5D40&amBMrQ1Tf>uxx%fiosc5`l3z8r}4oBF2z9IZDe)Y83vh z82}C_xyMh<5AO@XDJEZF0TB|*1(YugVhc2VI;;92$43SFHH(EogC+NZs=xYdDHiJ@ z0at*tZ6VZ4=>Eqdnq$pK53-1nKwpQqF97!ECES!b&`V~fQ<%%iY?KzXqFSQcQRiHG z;*iERow1({cr}?FR(NUvpr1&O0ja=t%n+bwJu5UqwOBAs)^~2zC)YW6=0ZKojSl0^ z7p^^o^Kc)2-cz|RVS-0C%8jePQZ(*BUk@c-jLB$O5|%FxHXMW80A>o-?sg7aa$Gn! zu}8X}eO5v@*|Xkyo*$7~Ho^TCElDX)X+r9BSw%CtP<;xkowq+F70*e?Gaz6%5PV=B zNQ^J0ALSnD7X)eawPdX-N-M0nY$ zLqEZmjz&}~aUB@%$3Z{DJ@1Mr@J_rF=P6kCw^r_%is!kp_>_5N?p$V;-|~SeYyz7R zxXk+N4wh51`ps~2gUX2OQlEqW%8d+SJ}~Qm8<;BOnl@^T-I8W-qdZ9^Nvu?wuo9=sRHeaUu3 z{$RP+E=GW4qdxlaY6_#9U4}@B{oKIH#INI|fyp;XtGuR%JDC&B1L-R_`)PV;5rLA= zADaQ$Fff;qeoyR8=-RGFBtrY@mhx88JtDLwGR9p1727|Zc^UO*{hY&{V(aI4WPm%- z-X9OCexqnri$29(?lg7Xp|C@sM}J;*@rhx%1_b%r>4z0y552LO zSC&zKw$^D#NAZ4z=)iPJI}4tCDf5gr>$&h@2F!6_*)q<7;G5FESS((dit_>^6m*&% zN-lDuKkDX7Cz_kTCtVYTnK=TvN~&iB^m8P5>%I?x}XNbViNuLX5b|A9=!?pyIiN!+E|`o8qkG z!r0h;yxHK}v8JV$b?TVJZenp0V1>W~n;DL}5z&U$3f-N+M|)VbT!NhJ3NtEdS9|Y_ zc|c#v=G$8cl%!zDX9=P`KO^pYiLH)Wk1l9;tX=8nG2;t6HQLp=@QTZOvkYQmTF7IX zUs&YVrBfH*y;CRO5N4a2bF%@O3udP9nvLYLZMxx#V-eLvXdl^0`LK`nE@T!0_w1Hd zd=zK5=`iIp4WyvEpy)HfEkZnkt^Kk3M#sihs7>Y>H}?Q`GI&yJwt z@$yKYL6&9EXofkKCw&vFk$!uGih~mqZl;C->$Xj9S|k-*4g@ZjQ6IQiYjY?xhKCr;PBkcTy{O03H+l zDywiruos#Kw$Cq5!Do}5l$-NQ6c_GNK{!;C@RMJ*Mz?S`DYr3rET%IZ_gB@;c_X@N zo>w(lxY&o33w>7Y@Qz&!`^sF@IeR;!yB@m{HwQm1)~(bjk}-)LEtzu|_`Rq8v_nqn!zP#_TEGn)El_tONYb?{y%x5@mlrh1Mzr@x&x= zDMomfM%t=bZwD%KO-y_J45f;SndR7J9Hj7e0MfU@ox$rI?m&!JMqQCJwdsd0!N8dKM zMjPDgVo=fDqoL(*93N1av*&v7ao!N3h`xd0CiZPb2FRaZqdRhN!3VZ*vYSIN%laW-R#yXd&iu35c^QMXH540jhDroYNNs22 zp@OyR;CgDdhbUij0G&LgPla#^6}qqeK9jB7ydq|*&Bk0R%yL4+B~%Z(qudf9&4UTMAB~KjzwqI54_IX$Y z$bJ^34LUHrFIIFmOR)w(EA4%Kf2;hts|OLTE5op;vs2h)Td7&zK_a=#omG3mNX-+OvE&{fc_3@mZ|@y3=&)R4mmG)kC4W=;$*UotK!w zd;5o?8|jZIsf!FGsoiX~(bhqJzEgXxVWM(Thjti}&9*sa#{~V;$&IJ;&aPJ})1? z!_q*{?Qog>f|<~#xvDZYApgfJtvT1{Q}ShZ-8<^iw+2K{^)73%#&j%@*nT!9tP^xw zNP#HH4qp738N!bKe2z(l?&+LKuZBMc8ty&n-fqh8H;_^VwVsx^MOygr=L#q<6ZTs) zVO+XKX{(CAENn-^srzW#4h%_7(H>a%NwUe;pf^<|Mbv9=Gg{L;Miwa-thZR~R0e)2u|L!$@ z#pu4%w4@1JrxblH2TwW^yE8O(kUdGo3WzLwA8sh7h&oX3h?KLy|3v=boKlwnfn!xT z?o_L@QWjE#z}5q*Z+uq`B8??-dJ7S_nuK zIw_y^j7hgmEm9tDjWd{i-fL> z+ZwWasI55v5UFtD<0lKJFgAR{*+eIs7UrR*wcK^a)FbGt1qF86hoB=`k}Kak;r-&b z7b8R%d&x&CBskI8k`}MP0q&=^+KtSx{&92k>+*go3H^{Hy<$bre#Nl0-yzj-wZ^vv zk&cP$Z-OqEOGR{9mAYxA(nr`i$>eDiXbm1;q{i2kKcUg zebN%UT(_{JFm~kYXgTw9szG}UXBb*yYpwq_-2h$)*OND$u2cLSJHt^axD)Ufz~h{x zU|G*(lV)s)w2hCO$U`-@f9%*6dG+;YIyD9@KLf{UsyROxPF^{~veaWwX){9w1XAHi zecc z!ctEHPYT#W$xi$0gybBs3k_2D?_te&Bvq9FBX^75PsFwdRl)kpNSr=+S^6OFbZmQS{(-GA ziyN!-tL{BsC~+7wIJb9zEN*!MyN;3wpR0JYQpHaELAo8zFGcNw`sNKuz!Y2OlpE>s zkJTs+fAe0sY@`_vd>I6y!KPqu3?L`2~Yg&^k+>)d+bN%Ie*>Lx!2)2rp6YP zGA1PSH_Syzbk$B-*xBMmuI&5VTWNM4QU#&_;?tA+khENBhv)A;W^);2#dm_ZtjNMt z77@DfPaURXqsWpiL_j$^xAs8xo_4n)EmCJ+x_SK6@w@gzHgcVR^)KC#c|j<+9tadD zy#5Q2vdRpJJdyUwPHYN+0y8|Mmu?#ouyn6nUrOZ!ZHvhJsq6IHvRN3Uo^<2uM2A<% z+~%Q!N_%CTLcg5wxbYKY`cn=GTi0Cdg2%LwX9*agbLjQp>5uc~gLe1tPvu~1$*VQ^ zuVVQ8huePv?yXM_)@eHql|}fEbO|qA0vS?^#Yg0Hj>SgPU~dU!{E|T_7fK_vOPGmo;@+0zgfdV&m~jkX2D_WHkIF#Zl*cW6Ws~uFB6^u7 zB(acku!jb%**r*=Uh)iZee^;CU0vwfk*AU*UI`$sv3qwV;6G+BqTMa_W$~MOFaEtm zu#i`KR+-AT{PNKc)(Ie`kJnRsjuS+_J{Qz=B5{OgQo&>;$67!g#-bsW>driTfa}hH z8=J9?z=qXL{JIg^LRA^0z`wir=YTJEBr)By4GLn0)N1{r5i-({Ei{3s`uy=V*Va<6Vu|6ye%5 zU%wUWN@@rg5Tecs8>um;H~}El;RNK20>%~P#HG5w=K)9cX( z2127l)z+oG03KfHyJ^5q6L&`A0;yJsJ_ONWgh%aIFx33Y5j1&Zj%(7b^e-WL(u$Za zNVPPgBmP%kLqk2qs{xgF3tw=XnERGto$F?mp5IA%g_@D>%cb5c$dP1qQ#1j}9zAUS zm@b_zIQbJ4)(@2*Bt#~*Lw;gIou~B{t1*VSLbb`MSuq(Nt!q+RQBR%v zSR6TlNyn-b^-x&28lA*=GU=bL`eI5BbndyT2NB%k%h{6-?7A+iNG$t2H%%#G3=Byu zbdz-Z6|x7cZ!I?-x#+kg-Sy5unr{J)3k!w{|DbP{`qgxc`|@(iV;+)G(x}utVjv-z z6sJgF+VIO|xThp(*6ej~RBC45;ygKMDDn!@}D=&$pwSE6~5HPyI!Rj_GFJBv<~a82TG{p=vV#D za)Y8Fl8;3Q%$Iw+h#oP^qxw4H6Ro-8Mj_D= zJdKg8X9dvqp&SQyO!$-uS6De6W>%eHm*?_hA6pAHB^oJA6F}Qu$U$R-%5RTNXpc*p z3Jg=AXM@s!f;*dhc8R3d#}u6@QXp_&Op82`u?QW8VE%_D5*G*M5FUc|x?QEDg*D!C z1xnYkP`+g?k9e*NFnyXEZK9i(c&Xf+Fk``B$R}(pUYP3}W$ZrvdVD+QfMMYas*93P zu)&`pb@5VVzP2~snc92Tjyq$BGE=OUGCD@KF(Qq449>a`df4T0JM}P4 zJeMZ4+f`v4Z%N_TQ@Gt7i(F#~wUdbD!XW%#0HtkHkc!^(D8?)EdzUii8{^x)lw@Xh z6;~!-TLec#s*epC9WJTRDi>8~`y`!hQgNVg=9m@R+ZW7ffmIQ?9?xP92oxcNucdq+ zG@-a~YW~5y!D!o$pX3SAvTI;d^OD5RoR1Km10=~7d<)+DNeyqS&`lLop6x zMvKFfO~6dyY3z~lfOU$or0^$kW|I-sMPqg>5Lk>8$vPLvFfioUE&9Emg9WCl>>Ub* z=lB}2qIRLRc%vhi7B^E0M(RDZCkDQtjfg8ja$ z;z;#JT$muNU_yZ^u#CpaJ`&=gwXw*3rv9#@wIfx~)qOSJ;M`6s>0Byy2Is%5Fpedl z37X<}9@j2MU6#fyZa$-RgDmO?#+XvSIi53ts`jhO5iu@*;V{Td>Xms_Rp>W|+MQNnEjCp-4|B1+nWaBssaX zLx>w>A>B^XxLbYlTvi%=;)0)P-p8uU(y0L7R5AS~?%ZW@UFsU)s~u=F8P&2X%|WRu zA&n16;HD4P3rQh+28;vH1$}7|S!-j9k4$gs`@qCX)Mlz8#(C>bN%&qzvzR5=$#?{* zGK}}1A!S9l5yF`A%}R@%k-cjB#c(9%u$<`mr^aUX>`a3^p)H3kWrD#^pI14`{RP8Q zV)s*vrLcFIgaqJwhIt<>aa9D%kH2-x?i0%X0)!g@q|wY{JM(3>R@T*WW_IWg*&W_& zmR08?o<(Ewx?h@x zELudtx+%Cd_$gkX-Y7`qW5JKZPzowHKtSzl9iwweMge!e`dxkNoGtmV!v_+o1$~os zU+%=pr*u(CUMH=l^#pdD6sakXBGWw=eJR!hA~EArWWu#~l#TjSFRHjReOLbd!`|Uw zgu3r-x(`rc15hWtVEFK(V}z9LCt3-#1NPg|A2EbX~px)S?HU3Ug1>Qu%K}osk3q_6smtZ z7Y-?m-P22tgmH9)O%UIntk^O@XrJ|}om2;595Q_NqUQ^~RKt8Frc0z05Lus@(iW8a zvip74``qEf0QfTIm$qWyWv{fgaLcM>7}A6WI@htI&s@fyAePNtr(n7*u5c&j$@j!( zS0-nb1{EMm_T)(wPoa5OC0b*YIZ-*U`Vs6I#}}P#;gF#U4nu_JYv$;(w)Ed|f=aVU zoj6Qi42|-?X_d+wQ@|dvSpaWh?N{UpifAd zK;EaJz_zTnnb8~Y$Jnxu*`Gtp!J6)A`4v!nDC#-Imt!!htUmw@dYKTqi#-uxSv|Af zdu{yK+>A(NJEBHgIU$gHxx$b7+D7&^C^fVkH=+4@@Cx4RVhN;Mu1TKOKg3CE^SpfI z^BUx5AIgjyMLHQzvEDO^w5Oz+qQ@&YV~52$;>F+UFrBki&s0fStLJ=apc7XTE@YA1 z$(0i>xk99Lzc9qPz-E_=l$|9H&qVd|p{hxx4nkt0SbIj(LuXfG%Qbk7d5AW1c);~E zi@ZVE&{N6YnaqvCn?OE9)R5{rI@?0``I15V+Rz9ejdSjd``Vsu*17g-6~=X{CYJpI zaBP$NQkVU_4{Eh@SjRBb{mP!n>U*7=SN^=#Zz%)J_$Z~x*P+vPWa`tnUS9PwdHatD4KH2pb{$01w8PC#o;J9N6j^(%B^%T;0ch6Y-qpD@&&sZm>7th`l zx7D=WWE=Q6nRtgzANa8UpiJ8_ts2NR7MkDug81j6=-)v=}od> zcbPxzFb7QzBwo3autJ4R;aXxTOfM5UyiKdJ=$4B-YN{7?RbtrKIgX<|e`&cOSq<;f ze~izvPa11Gt;O~fR=9Bpjx#m$-IlLSCL6$j+9S0#bj8kZu^zsscrf?RYH=A*ub$`G=jv37@e6nZGI6LMPOL-BaAQpKfgF z8O?H2#vJ*%dz~7Uw@3$v(5V8Ouy*RivZj;b7OQ}7gi`f~8U^wJJJ1Pp6~D^jNL#e! zo%L>>prt5Mff||QP@~~NK}qjjQoE?bIo7===D!;ywu+~l@W9*b2pcq@^x?>G``3A$ zbxWH^HI$CIr5uDhF|N)-(ud zc>0gpjH!L-sCxB6yzk23<~-5*K>$$S%@JXsb6(Ck##Z~fLprHJ=_#geO}&MTMx2(Z z*5IG}sl+V<#z7qj{v0f6zs60-JH2Hev4e}=l|R<$NIrvZMuGb)*v`u{-@drvj5B%m zMTm85BBr~cqV&K-aLKarQTC#a8--Ni6x*AaR#4KLpd;`mivS@!=@otaa@cjj3Ou-e zr%$JEeIy7Bb`B@d21L2MWWL2{o6(X;Cf^vq%ssvq(m(Iw@p?Af=qJUrv`w=_QAocU zc7CKuC01|@XPRqC6KPrDcgZX80Ow~tXZ2)ls3pa)(#V4KC0!NUh<9aOIW40=11t~k zclhJ$9Qj|se4BS*;$9 zpIuH~mWwaFSNk9n8|#jzi%MTkMYRo&?jpM`m?k=;3a`vD_^c1JYa}i3ex$8=k!z08 zRRe14Usr>1aYl$Mv1v)EWv&EuR!n$Ug-Gc#@=Jcs_zB0`B;J;rzctEs1~##uCv3aXMaHRgq0 zL7;z!AFh(?qS|e8f_i7WU8dA*=&FR-q=LLqN;DL$1`@!tU6`#SdOz z;z(NUKsat5Cvbb$RVS@gTxCicUtO3Z{t+PkcLhl>v+Z9%fUZu_ho&xEuxF+SeKIQM zL-X(;<-FeXL~?n^z=7?XoSM@*iSs?vBX7)mJ@3!7?;?a`;Jve}^9p#$b=%pPq36@w z+Zuj5cDxB2&)PO5#+Glhj#xOK+!g;qrgc)C`wn3v`JN%vL|_SSb5en0gjij{^N}NxPozeDM;0kDT6s@_E1y zFMe#w5u5i2z0{{qBJs|?QF705RxiJH^~98fZ~K}!M_bQM@xL@`WX}e*cq!Z|ANfoK zsjrsXPS@=10wAAcdj18>9p~meBieg%``!ENsSe>o?HA@sRt2wodxks(ODmWD^+xUD zUcy-w8J8QHUm5cA!T30P&hUe4+O^*cWLExlL2ejB{gNG9T1Cb;vI?7F0Bvu^Gkg&} zfeqp`ux|DwU=T7&R~T+8JJF0E8X#IZ5k}XjE2Y*ze=Ai|cNj3~Ewj%$FEO~o{wx4h0pozGw2=t_g)18vp)6GtT5U?) W(O{C`sW>bUr!~@GN_y~j?tcJ$%<)zL literal 0 HcmV?d00001 diff --git a/python/nano/test/resources/train_image_folder_jpg/dogs/dog.100.jpg b/python/nano/test/resources/train_image_folder_jpg/dogs/dog.100.jpg new file mode 100644 index 0000000000000000000000000000000000000000..470702ae9749170c620c05e716a02d5ce971c0cb GIT binary patch literal 24171 zcmbTdXH*k!`1ctI5PC;i=tv2obP{?I2uSY&Do78V2uKj5D=~yng$SXA76d`5A_yoD zKzflb5_(f<3ZlRD|2%tk&)HYId(WA9GxyB7KQrgde6Q>JUjNPgw+di4Gd48_P*DK@ zRR1=>e~SP^03!%Q529nFr>AFPWMpFHV`pVyVHM)J!oepYa#d1XL|jZtPW^_Itct9d zxPrc-3QSW+N9XGGo2G_ZaCL1R&HuRx6(bW9D+?=_ogJ(xBQB%)|9Sk^4d9@sN~byj zQi%hoIjDdfRR4Vhi2Pe8E!F=Tfd6w)Q3Gjc=|J=hj7*$NvrX|KQ^I$3;y;1Ec}{4;K}6)W03bK|?F1NXMyf1M&#t z5?4x~=QhYJZR}x?P`3Tfb3c57@rtC%*Q-DN2kn26{l5cC{QpJve}VnqTnhkZAl1K@ z2jl?g0`}evz{;hiaM^^Wq|d4a)yahylL_BO_$d|FS8|vEKPyCoiLidnaZuxkcCy%@ zsBlhU?x?>d$PG{Z)j~nw*Bm!auJFejl0SQ22W|0HH~}d5`;9b7 z#tSTRS+riKKrOr0Ck9pV534^Ezcm$4-qjyilfB#9{Oqkz5AA`Mu98<(XHYro`k#Ot z*0ay1NTh<7*UY4Wpa;u@SR?J zS3xJNl}CmT?E8b%MtxaHbnF?0DD)E8-hZ{bZyC_H5Z)m_$x8oW^?~Y8ey%;T59NJf za=mbSC&yd!Pr!(GcTt{BLGN!HZnfXI_pf6azPVh<+qgEr^!GGpUw@?*hXzm{A652$ z!bmBp{TkIja51s?wEh-Xw73{8!B2y-BM)&Z#g5$WP#mq*Lc~4p0&F7~zIZXd2qW}n zJGs~0Xa9gh4xl`5co1U7*}ok^jL3I${B-UxX~h0sXiqhK%K(kgrmyH~`K>Z<*u^V| zyFgdYTcgq}Y1%IChbGTW@9%B^;+9k{k#Am@{jr8gB#upg9AzJ< zpgessQ6^_o&p;!58e;htp4Qw3OSMTo`$RHJegRjivDlV0pku4Pu*n_lx%Bt4rK44J!afvkY87}RG^p+WUO}t4 zpTJ{fzSYSZFC)6OXQ4^2^$DDSXP|u|LD$1<7ruzrBo~~|9)(D$` z&}ix3LPunVd1)2{gSlPn!XM~x+{%nC|A&N?m!)F{{_{ES2JI=}W7`k_XLR*b$zwAR zR9jYMcpN88XC?I#WC22 zCzY~Ql$D~F)EUA4+id)6`FyN1;)Gw^-I9U*cEPL20mN~q#g&z#puxsxU;y-$UWV9w z{?D~`T`rfVY?Yn7K<`uq_-UU^2j^%=)X0&i_OY3yZbVDwk{!U|zM60}xrC#mYRho} z_3T#Oz7WsHaL)1fD~rF#bff1A7j1xM6O zHTW0NC)D1L=*i+C7G!x{xXCJwGIneAn`h{3y4ZpTL-ep*GXWnyWsTN%Qwzz(&{2C=f1vUD+cTfa6& zR!c=RTctmp-k!1k8mX6;e{VALw=Jg7C-b>Pom|2#ySA)Fi}qYrg(sl#&jq}`_cvOV zIZd_qrd3nZRc;lnvyk;b?XMIR;I;h^4{Yb*27DNb#0s8j`KN!BMrik1iu+n=R|$}* zP&k%_ubY{-*etTUn_VMj+JinkJnXcxUeW54#$|tI=4BDTLg+|^4jV$DIU(v~t_Ml$9 zu8!0P=_C>+5~*B5@`jLvlW*ZI0`^;kSXqV8*jUJ&k1bQ!oXG+S|9*d|kNCNp5`19c zljDTh(j}e?aFP{Er?KgXy8DUnOA||oX3nuFSy}=gxvEI8pe?{9LWLl~sb%v~X6^ps zj$uRJ13dUx3qLFKdHqgaJw2)6*%)K>mz8w7hOJgF0sW`~m~DyVhIe_j|K8VrV_Px3 zJ!i@vwv8w(pN4s13ptHJ!A<+vCHz*lZHNA+8lo+l~{`k9Dk~011wXb6Z5;qi=ZksYV$kQ*0c^4AW zJ)QPA40xH{rkxrtW5Li5-+S8vxygT8ANl2@C8h?+nf>Qv@7!?&TN)B`)c-5EL$HR& zKwjkel={{9R8$$*ZFah64p5)_ zB@1g7+gZ2y(_b6V?v$kocBKiInbb(hj9pe0^i-nd<@j?7{Kn=o8gXBVm=#NV3pf}6 z&DD(E|H-3$`j)-*O5eM*n8y*FGMzo9`!?rqM&LxMDLf{_HPazNJ%{K3|HJcc{w>U8XjP0*5aL3U#|$3-f#@wTENFX4a%gi3;R8L{`-fK zWJ{Dyd+!yNW9m)?zp{yao!i3mUuLn2J9-<97~4Wt76EScS0Z(M18r6Ag%@`A1bw0H zaS&w^|BegW6tS~srdjxt#Ev$7#4(xb&Q<~Vij}3loL70}4j=eo$hJQ>rS%*XQWqY= z=d;!MAWpvckNAox0KkRc>LCNQA^b7yg4ULykasPGS^6So2=P1AZA9#elAdWQYe zf1ug6vx`uZ2paDd=*}>MUN&M4~-yibG(R2eSdv?ZLjdZ1Nyt$uFvKxnuXqm)^Dj!(M9|LlJGSlzs zebx+gu2LLJz*RX)knrS!9{R%f>@Mck? zJ;0bwcFBT$KT^1fxbGf`oU`8>psu{-{%Z>~nT4A+pRX^t=d&c@@)X{8uIn~aXV8-r zVt&Yzf7qSNzholb^vpaDztk2sBdF55(M_qiS-p$ahaJie(0mUyLENahzR2c1bQEf0 zme>DdwK%+ywR}UobctbyT-_t9)WFA2a%#hC(~WKf@6SH?HXTLYvwIDqh3{(4Iz17I zcy`(yEN6qAaKxPXQq^>GS>wgcQjlnUF~TBZWAoX60AZJjALe)OTzmp?UVjW3czvuh z$!|51RP5oz8}4-~nJ$zIxxbof&aw@X^fE z4OiY3)@`q~o2NXCAx<<{HW+t3r^8jO0>o}#tx582g8ZB*y*zVaxvy@D&(N<&>`14V z*FCBBHv)= zs1=1aD!Y#SCEbPn<=V4;ZuqmH{p7&#X@e!-1=m8M_e2^o@lct01Y9NgK%$Mba{IoH zt$y9r%V4F%!xeKVmjUeBv@P2d@Eke!-Z9c0BcMrLSJ%07)D6N!o z(vP6uMacUy-sx(qXR3M65g)Hmrs#;D=ij#m_eZh+l6o^ZVlgLS0!(2-x^ zjak$CCq<1v%MlPWe=a>9wRH!t&pJkdi$_Uk0_lU8KJtKwS^8o~ee6z2wFF&e8mwn6 z`OpPzczf)X&a@m4%tt)=_dK*un{wk=7=*B8isbQfgoeVtnGxlv-&Z&V%51>|N>vsUKH>fxX{he`_)NGL!$24BY*qd5LIXV$91MN$s= zWg&mgZHpVXUtn{VG=_-EbCv~Rn|7$zBP&aLBrc}(1K76PKzO_kuhw5xfn1O7bCUCs zE3=-Etb#eRZ7~(Ni%37n;bk1-TND*CL3L;VMaC)EQ=5Y9b!l!&n`!VyLQduyeBY!1jb-;V;P_}f3c=c&zXtum{sk}$05&EN8qba+WvtKoP1_(O&9Y*-`7t`)$L!dCOo?%lC=yE7T3ibTmnZhE-1+QQIRKt^{Yy9NhQsh?mx4 zL_L7WIN|_6u$S%Po#wjUp?vRA{#YaZcgo=Uv+JMuLu^`ahN-|5hSbHh zJM~E(Xaee_c2?=-_@K9#2z04nf2Dv_k9p~klM2H5#UvXaDYZvY zt_9;na#BNVq1<($*+x(0Qz{Cwc6ysWAXtgar@9q}$e^?PJmU(%4jmlU7K#PWV1uV4 zcN*Jn)%r|Tc87E_U8q-4oPw^IyF6=mxT~rwUahivB%B;aU8||ZYV(|#m?&%ebwD&& zxb5YVMNXz~;1hxjX)hc;x*#*{WYX%Mw-yiALR-@z@#dJBas>%#{t|btL_eB!Jav{S ziOQ0G0AX%z#SKRNZZ(;{cI~PP?+K^gtmPNnm-kX<{oozK>sDx>P*g zgs`$C*l7@u*X<9^Qnn@?>U5QBLz3izHL-EIT<{5uT#JbfzKC(dr_|hwR`lnEZTi3~ z=*yY*Os8+zzows6E6FaY+l2`gJUAG2K7wxM-lC3Fym0V(pY~%@>PwiVy7eoWU&3p- zDK#j~0*UT8lV%s!wz<&A)^bOYGjwVzHwzR}_B=9e{)`#7>>Bzol%%gJ#8?hhvU=TEOniD}y`f_q)LHS-T&l4ZGFw;}5 z7MGNM-WZb>s`*3DSLG!meOj~VX@E1ZT!WQVQjWusj(pO=&4S1KvJ7R!npDaAwKdjO zZjT|azvYQW{3O*UcYSA-MTk3ibky{@V(8o$%h=c59f#ThjGjw6^ zH$%E-lSuF<(vl9aAj87a!L#yPS*LSY`bpRA$4`UToy--GsbgVo%lLOL7_m{h!{+fs zw$%XVT0Yacp?`Wnjq6&#?-^J~nA;K8du9a_tGXyRt}I>P2=eVpp*)c{?8f2T<>dp_ zATjBR^gl{pg9AOiN9}{$V+7tl!El_THy>!0h9EcI#O>MwMGWsi;)zM-c`xNyR!jLE z`T9cDarCde&4H;B=x;wgadKIo4-ygPy9$zm4KkaD@}JU^v>sp7(0d#uDy~$>7(RfO zd&qZS9HcVkIYPV)=)!EuGL(@IPDXBf@vqvM-xg&(ssCWZpIOKp4+@+{y0zbVHvRxu zQl)h>o0{Uy?Ip|ddxfZ{bg@S9v_t~ex8Yjp>Rdp zdq#p9BYgbX&-E{*p49g`x#sfHzn1||hqU47tU4_LlJ*G(UepYi+wC_)OPQ5%%lZR6 z!5fqZw_oskJQ=9|=1?H{IG1;8jJkd-u=n%+Jd0BLDAgF>j)lCc*r9E-sir-9S-B(z z8$V@3>|yye$w}2O6TDFJnmrs^daITW%t2Nuhn+vxU;_%dYFBqA%#FX(f9%)Vm|p+> z+JAsc2m5a4@8xJh@2V@@;q}CjTAS`@)*;&)OHbV>((eox!LAQhrEWKbShzQAu#^R~ z3wo=Sc{7pDKML7R>={MkBW2iz=D3k_6Ehjj5)rG`VpbA#2~*r6Z`n`=aUPc4a;&iW z$Nd56+{lEEO6{6Q8tM%OooCzF-6a?wW|IpP4Yx3_%`9pY4cFe5;jPyd zNN0&Q8%h@b4^RZHq_4^u^UOdPz6<@Q+<#KT*n_$$vnAWz$deK0<-(J(PUG6=!_0pm z+53#zgYHiW2A5r@avplUt5}3HBkgDywm*M$^s5O#;8}Cq7k48iwGI93g)HZ*#<$gI zy|>WsaDkR=&=JRX#faD0S;n$P$C|(iuKvC7gz1ytUtG{ey)CF2&FH)AuY}G10|?Bt zhVBYQ)4g!aeZm|5#y9HMw5R5qi>c+;&OdGmfO@fYOOrvROGc3n*VvjB-gHnUB3G(n z8;3VUI`(WK#tNkQv1W$m2ES8v`Lx7Q=rl7`;a1%(;MX_ttcdh?DLL0!dC?88n*V|O zRUXq)SGDxT({|P&{nUT4qozRa?>A^k<9m(Pkyk7e-cDuWOUjHr~Q`7EWBb z{X$#wezEvQcpGS;gfs9FYqQ*)gu!Pj_1=Q{2X{kRZUrf1%i-Qd2wWoJ(RQ|W6jZ*TZy4u%&V1-xp>^+~_Yit^pt zaeQ^fF!q+;{9Xb&a9sWq8yBWLk)Fsu)>cCW-KycSvJLSph*dgV_%@V&dn$@Nx9a5B zPG555peMb8=M{~(+&O0~uai%mYac2^_x)g`ZWzJ1AB+TUZLE=YTVCJse;s`*&s+fz zK|(z%9a5rJwt&!SSFjcI}R|IfMoKQMdIC%j`TSy8>T3TyrkZ+7S?`tV% zs;CzdzI=>+E_t$vx6{hf`InZy0s>X2c1uhAsCn1LAcbf5-6+p@67`$XS($zTKYz|& zGq0SP@XOuLOaLT<2!hA z6ZqttD3cZN&zgfYYXnDyIB)31xG={IT*L)dZ)Zwnjmjj~+Hy&9yg88P>!&Q)+w_39 z_!c^s2()fF3z`*$!F|*}#3)GXSMBm2N4@WV{KjopWzw3{KNPNAT@k4%v5>~Uhgbe) ztY`slR%9h#`W;w+Z|F3=oYS%v^V9Yl1gbodM#sb}vxd3dz`PQO**+MebxzgS9*Kf3 z5+0t^1N&Y}>|Gno$c+3Ko+?44IK&hNv)vzN4!4joc7y>5Ij=(f>OCQ96wxUMwHC#TnSP_2nxC`B9U_gRG}X|4 z2DpzYr*(n>?yQH3=vwdK+iBqscU)L^pQVD?%q*=u%4EL`x%s>fHjSpU5;5zLJFm!z zHL`!;<1J6{o2RY6oj6b6(IcjwR1cX3NSH_uPsaz?_81xi#Ps;37a9*kELO_M^rE4a z$=|A9Y)oXn8k3gBPd2yEGI5Z%R9QOf@3H-SQ5T>Uw%BX%XR;F^INeewm>({o^}DSb zT0_AkWkD;_?LH0Pu$!$5VkD}momDP!n)P;XMa~3Ux#U@pVOj(ulR)ZQSyVN_;fK`N zKae}axAY~R0}esxfZYyOrN>m071(PxMim{t1se9($9sVb?Hps!ri=Qo5#Mt*&R_m~ zZ+C;pI8u&4;+@09<{GJ%HDp_f>)Ma`5g~lYfQF0 z=I$8?yCXHTWX0bPp=q&F{xr}c=QozKmE&-zF~mjN0%jTQj+I08-1Nw_8nO%x5aN5c z$HxyrqL*6uK$S?eL@jr*!8i>v`@j}hjQeiYY1HPsLa6$oT1sEQ0QNksqD%{B=hJ>V?lHUcYb6QS)!N<>8 z2?TrhYv6a9p({N{YQx|s(7Uw<_Q{#=0>gejkuxy2kSOkFyW6=lKwzLu+Z}o`ZuuA# zcNFi~0KTLMaFl`R=qzNzcFYIIq_J(~ajsQ5O=3$pkB)J6wzvMgz&g!%MSC(9x&H)>+ zJW1sN7COi>$YmRnf-u*=)p?G=%%zZ8o_)h`Ikq{Xj`QP&d?9Bw-!XQ zTWU^<(yRhvYwA0oqspd0LAh_bYL6M=U&-AxTeF0xx0qQGyAbNyMAI|kGH!BLLz00` z4MV|u)h1KwT!KjNA;)_TSFe$cqboMAc=MiP^|p2_+~MofyMmyW7<&);U`xL77lOv# z%S&G`i9U=_v6vZ^sI((>;AcYfr0WO1=V8{CsLTQ*(1Q`!seRz~_kz zFq=>6RcD?JuTjnkRtKNdemCBc>W0%j+qs_=DLj!Q8ANG;z`Fvo7$jH}!icQ~PpYtMFQS`ZYXcX)YRLcH?e1tDKsJ zOzO}21d)fUSlQt~@N>Rpd*VcPV&KPTR`jqvtfG}dj(4BGWl1QV;%6RL!50YwvlT7F z8wTaCsTIkU%A6>>Wdr8bY1a=T+cMJ@rN6qzco-FyKD6&xed(bQcqIyPIF4r#4+)R) zeD?A&R~UZ9D%ajU7?_RTIyir84D1RhHQ5_*nU&=yKV0{1ui-ta?$;ZAOl4)R{=*UT zDc6!tA*v8yU8c1(Oc0BpzXe!?T6 zJB*2w%KsO$nbUu|g@?GsOjud2qI&veb_PORoX%ZvS(ebgK)I)`bU9%oQqiKvw>oBV zFF!=O;0LmxC4`4N@GydfDF1^Fm2;weD+2ua#T7>%zSTxi{Nd?hVji17%kqb8Hy{@Q zWpCL({#MNI==NLWWTri}UJtpdsLz?2*0^eOEUeGVKQc8364sG4j#i$+vcRSr0m8`H zWK6J>ej&<1Dqh0&==*>WJx>{_ripYIVrYLr@y*`gQ;p@Np%YQlU@QoxcRI(d0xaxt zMeo(k&uXVg@leqO{Md7dB^B|x$6`BEIaddi(##}4*@PjG;h`Th%(8v-y!p;bJA$u> zhT6ni{@F8ISk-=89~Yc^r(94kVzbotdzQV2-~Hxi$1;*!gc!$1)%S4?{QRok*ROb_ zu4bTeDNY7}Z}{Apbh!ew;gX#2pH4X=TqB4^0Wfp5)h{^reR9K|OM;Wwo5Be>0V7Ss zwIJc~C}C8BIsS`i<}qvgN!3ScWm*2^Nusoe{4~neBBkED{H;avYv%_%C0@644Z&gL z%H@|KfK2-9+7Oz1*^fhB!K&J_qpm^dvDWMAo|c15Syq!TxTxu(wRgO2S-;Hi`1Oue9_0(1m;`7$USe)^duYRWqv2%V10SDD5#24_>gX`sTW+~2f1qPP`pXbI2 zWifnA*HW;qT0g582EZJL8usP`d^^Pm>xWxENbT}c&(d^t<>+)>aBg1zf+1gA+qtD| zt!B+$aUF{sgQh^Q5i(U;rPO}ix7BR)R%T(FoTGTA^C3$y>X}Z8TjQG+DDhk$2>M?6 zF4?n^txiQ}|9GldlR1Z3-7do=Zh7gXQf%qnbUtX$bUXQk76={nhl3I_Pl!xO(^P0` z@b_@YO)ctZ=!Gr2Y}={I6?a*pCg?NXpk$T6uw#bNoaoetX-&1Ue6Z2S`Ji3Sn1mY! zwzx6TBXX?KyRz6JdKzrNJ$<%Oe}*xX{+B84&Q#5Ds*IuRuzj7Jau`gN=IELy3`V-r ze=n_`L**#FDG$b9f3;Pl;|^Qe^{AU%{?`OWFBrQW?FO}DK5H${zvF5ly0YmK{5itu ze~$WpT34LUsGzSSbRezo67{2CgFAL!X7-yndG`l>JHVOL-9;3ORkP zcC^K>llq1t0Ts!Uvz%a6vJwIu7z&u*%6PGPE$oXBDe3YO5Q#l44bLYtE1(i4N!*OY z%t_ozikiccdPJ?F%zPreIzQDj<&xyL$ViRm&CkB-C1$Fv(|o6Vh!74L76$Z;)rpHP zL`|WBb?sDFWt&q`$4>J1S`MvvByryiOgyKi?I9Il&qOwtybt4qJvQpr(D@oWzhsY8 zMQe|=g=o4Q6Y&kZ!s(osmVythL+OQL5ni#oan@!ts=mflpZ&kt)xJ&~EaVvLwDm)Y zH-8L@w{Fp__m{u(dYr6rq>Wg-Z!hgnb2?%d;p}yrUQ4*iEsd*;I~WzI(=3_M91Zy6 z-;wzg^0jfH9B0`|b zK=eBZ)leZ)=N7x}^K%@rp5cNjWDUbd2HuI5BD`-3u zjN%J)en}p?ZMG-pX3wOF{{DQEilNQ@7`=IE)gF@jJ@PHJZk-&0dBD#zjs>YKEP& zA_5bww9YD+=htW4Ov{o!l8qfYcalA7?Y$BeSYr{=t?6z3EGmP3W6SaSS}|zjK#}JX zJ5G{K(DP6Wr-yee89i5UFF)Q&V;7+arxI`FibTV;$<-~SLSMql$)v(V(O6nd<_>Uo zjpiQ(T8k|X>|=ppduwE(=4N3wyi+9fXGK7|4rYe6UXiTRUXx{XDx90w3jK=v<5;}O z{ODD-sbordq;WPkjGsH6b;*G{^P&NEstvI(^ei1koe?jekgjoWL6A1ej~2r$xg@B_ zG!r<>P#M!xy8f^OBQpu|s3W?had(UM?Ihm^@Tvgy<;TrJQz6+>dREX5oRwA6S`bFu zMViw}|Fiy;hWDho9XtJqx3Fu|`vm_Qzh@M`emc*4z+r@iOGDQh&**e#n<8F+2^@4E z@CS;xRJ67A!?bCU!^RQDv!W3Y3J_W@h;`-~F(>yR3q{Q0gB=B5)GH{OQ5w$7IhrVY zmSAStu~$6TwXK4eWrr$EK*0Ch$xZ;-^Zx+XPpf>>Q{MFa6<(`;y8#DEOwSbQ7#n*p zFi2pg8U_p0q03V|U{Z?ZME5r*0;i3sx8tFreUc?_aQk6rw5sT3S+1QI_o__LZANLQK=#(#%_yT7%Ef6v6#IZr>}>5h_v#HD>40&DA!fr2u(kSwFhk3`ktlHZr!st%p}N zss?@}mA)CPFt*lw&@c0Lo7X6^C~C{!ZPK#L`IitX#(vVZoY_;=EloTm`w;D4s^T-e zxmXylul)z{iPu6LW6VT_ej{W0d`m>%UKrGZ({jhfFm`!fFF#~&s?8_s^b_IT>%J?A zJK|546J_l-dzejsswmaVi9E+bQie$qBA+|maM`1*NQ^*Lw4j!J%Ge{uFqhq%i;cNr z@>y`;_;idh73*V0tSMuoFZrICp~it=K)tLpWjY#H4})FI9;QLa6PQ)6#DF5>V?Jhr zrCh~C&_~2atbN$8*$9{I5L3+;f-^qeEInHGh?sxihFi7pEm02YlrrVmXcBFYl{7{ zJsEeEUEl1?9=5~t!DrRd6U>p{piuXBvfr!>7-LX-N%4!ot`OZFRkVu}f3ocbS2@JCzb?CZVURsMU^!;rRqENJW-sk9W zHQb74M1GXKFV(w!3oMW@p2|14^&Wv&J8vA7@i+#zs^qh!8E(W-E@o2nj3?WLPnjPH zQ;i{-`Fy?(+AK{xpnnzGZcvRJGpcmXdM=3UK19Tln1X(nm0D%u4Z^a!h5W|vSeS?_ z=kg2v(&)>n<4QF8t(xyOP*R7;!ISE4xgLrT#6GAB z1|zDBx6L0|5Anan=b3=~$HWUdkbf5Zgr1iB81wC{G|8`DF9&VV)f`9?)+$>yb@W0} zYn86)0-TTr7JTr2VI+@Ip!dK3nXne;JU35ULK3^CJ*Kn)L=#obwGs3rX{Cx)_erav ztw5l<&wlV8KD`Wtp@s|{geoT@%_GDT2}KVpJhsuV4PMUm11bmEmeZ+vz}b#Mts;5l z{!j<)SpS7y{|KeSpg3|b{I9q4rrknd=OAx=fqkGLZ6G{%!NJ7&a`%F0Pib{o zE~3(6Ik+~?fMn%oBA4~Wtc|a9EoX^#Bfd22RPjx8kRnHe_rG%JI%6}-WlD>wh$JN( z-zxMPj`_7~y0uc<1i={Kj4TFCuY4rwIEr#Y2dm_s8)pO?Ys?1Lv}3IR4naTZ{{u*3 z%7u`|l)2@s&Dmbns}7ZNIO1llRro=bMYIAx#K8&7_$eSJu}(S98<^+)w`q99=x=|^ z#Hy@_TFjIqSBK)@I-!YEtuWO8Q~T8n+ZgNGJU-nbP3&hwD(Jd>Ld(g&d?h0ab736~ z7LLmawn&jb7`grz+GlKdWoJ}&o9*#r(LgNig6RlMe~daen!-)j12I;#i8VEiU=B-^ zQB1XKm$}YO;STZ9V%5pDvRO*mr-T~R&z_PFs>n%@s5n(3Hz6Ep2}pr~r>2eNdVj>8 zxNG|tfU@Ib143y-f$=7tj)Y$I$C+luHkN8(eK%cRdouhNufJ@4vYN%hM9y|4SP(riKTyI z%981*U+~xaaE2T4(I>T{o{W~fH3O*nHd}>vGs(jobL#?hax%+DH2|db(2K77N~Q;n zZbc=mU+M&JiE4(LREROR-Zv{1(YL-+&*ZzIO$yA(uhuy)+Xm?;B*yv^rCun_w3_|T zRrX$3C<&_u>)uAJGgF^{fZ2w$CG(kfhpTua%{C|S5NR;dws!mYO?WrtM73G?AjJ+`hoqy5X>_Ce$5OGT6rc zu-pTPqtm}cl(#$W6{8+x1!_il*>T)@Z)3UU)uGw=-+!xrleOo|`C7~YF)y6D7a7fQD1+bdXwGV_#+P?oV##7;QWo@Y=qd?HVPITW> za-t=YqeB93@#J5I_Q3gGNL!0wXyjJ1suF_4E8p&X#?!SKrAih=yup1;3@s}S*ovTX z;;&3U)c1kzBu!?$7D|rxT5%9MJK3{@e*q#RzS)*#ACWjs!l6|i9|F6))^Qrx8MUBo zsZ92i3`SzYO0mrA2Dz*2BxlTi@Rtr~O2l z`$yq_07$=>AvO2bY=vt<`Ff-Gi8jKUTgTHi^jFzXXCcNw3W?7qJm?0Iv$>u(=aDZH8XGwu{0?F z)`%>r;2#!@Z1S_u1yTQsfGsQxO2{_bCqg4m$K*hE|F5ltxw3B9=Qx8FS+bDR^AYt( zkcYW{t4PjxJ#B_MILydl!3wHkEkNj)#5I624I4!ixR=(wrl^+x_(_8*r0@Q!7!#h+ zqkZFWqwx$@j7d@TkMHvd;1ypq$)`#U)l*d(4>OU9$5|NfS*3iB=TtIcCg~Q1cPt$^ zH4FgxAx1`xIldBfFAuINx$;~Y|Cq|pEtj@U|J9g&tkd$vJpfm>gX_8$6}llk#X9Ea zH;Xy%JhEt@Bx`=nj9TSU*nrtnoxW}Oft|z^0|kh~;*}J;Vl7-z8!^-w2htxg+i)1d zI!9G6Lg1iQ)tfC|rg_DPBV|YHKsVq(n2nTxPh9s}1u~VqXIhI1`(Uis4LkHFrT_Zb zC(deV9E<1;`X}DA+a?60tbkQ@ZUzPsJr`3pHqm?+0yuyPA84g=uf{LHH)MtU4ezM{ zOc<-FBFEin+Z#19n)c*BA?~X{%!WQlLr-hPjiK6zXB60|;y-iaIaP;s;`;vW1d&qG zx1Lqxjw8>^AYkfF)Md$$=G3$WkAVD$oe?`T6DE%5IuD%xe;x%j!Gt?sjy`Ua7hNiM zjtmOc-+a|n{gDE?aJ+LI6cMgZ_11ym`NRhqZP}jQ%+^cP*}4bn5OTlRN4u1(6?9VFb0TqOI`A;;|%&B`}ClNQl4&CU3)JpFTP ze}8B{W^DP?Zon-QNkZ<-zN)C7ISzhm)UPLGp|a9bRejTXQe>?^fXjN?t3A>ua{Xjm z7lM#KG{?JMc@^Vl%DLjt-^!YNa;fPML7T+&%Y7>GQMuM1$dt_l zN9~d^2DS~6YCbHdLCxeEpHLI%H4`dg^>@8*F+UxEp#DCXYO0^5iJ9#}WiPf>ueV@5 zvf6Z}-Te6_WdN+isTrPx!JgWX`#ni=*QvGs{x)kxh3T>!6{C_%gp6e?6`Q+k#!%PZCI*gMzf?-^ z*!J&v*faRDw3A+D|Gin!jrD)tIX-*mdQZaurL3y&71l$n$P5H}v*n`J8{o-Y zQ~VFW>F)}N54imnH7lL=&wk@`ZjwHNx1HeuW+cojww=^U$fs(dE5pIzfLg!xdMkam zF~^93Ak%QT)(>==CoNVGANnZNsZQm4csaLtlW-$Tl@=P2>bR9KXE*WY=NNZ=H!Ez+ z%%JTA*SMZx*38fNuVTrw;LNF4GZW7bLf;QDFpYD(1{-s!QKoUc1}E1D-+6DjS?flA z^{->5>b_k&Gu+JUM|Ppzu+S>kK%F9kv~I3N`T!VwN*v)7%o=H8dp=ubG}no=A*BfC zO6^|=RZ|Jnezx|D5rz3(b#+Z--YY5yXg3V&+j3#3#SM^awt6z#r$(fnlO zm))ZSBF4kw`rQwKx8b{WzXYCq8nD#|M-a2V8ppZMm<8ZgiXhW5xb@7k>)qYUld01H z5h)ALCqAnp&nJl-jnuWdVn{W=*v6vcEM0-u*>60xQ+z&R-l^U;=j9)jiIm!J2&Udh zjV3uMDN88AXnr_K>0JW*l^`|Ah18ZHgdqnrwnTUtTANv&u4FZ0p0sXY<=H1ezSD~} zL7H}?iwXGp^chSPecY0y8z9@7%zF7FiIVD4r=#3s1+!7)=uIJ>2>-TSzaOrF`h$ zp@8g{)fe>=sPN(yrstO=_zPH6p?$uN(piQ3D?jfco-!?EKMV#*e>3d`A{?FnW(f%ifclFA*u6Ecp zSi<&9|Vt+S&IIEyE<-*8b16Xda9j~^ybRYPiUgWFf`!SVwVMB|##8fcW-+3!^Ij&dNg^n?y}bH`2(qpI z5CFbkbKSKIu|B8RxEsS*nV$O}AfQF&ej23xt7(n(>0K5BD#>u2T!nMQy@J;_Ur_a< z>|U2pC7f0k=vfxdL`w~>7xz|vmThv*y4lmS<3QNpF!7vo@UM?te+;QyZ>OLUGEc;p zTG1$9o%U*$#o*C!ZHVf#^!e435!n{64_@psjJ2f*=1EO2EQ^+_Y>%`w&$Y1nZVmgC zToaJW-3!iUG3S=)?F1)D)>wazTaW(GX$UZ#7hiuU6lVNk4n7!3{@sjFP^0jG<*)!s zh4Zy)Vj)|dyxHre^zZTP(I=Hu9eL8+{W$V(Puh%T3l<`F4u@m=PN`sUBY2M|Y(-v` zuvmUg3*(pxAIAKiO>eF(54<^wm`hxk*BVZx&8S|DrFFV-AAHcBU%mTcncqnWzGrCw zWrlLUB{XEY=Hom}9HEkbU z(+^_;d{V}qJ}OHct22pwFA)Uh7_k>7nn6DMIMYPv=IXYZ0RV}bKlh9yTlGxp9xR{2 z2<5L8U?tS^09y%T*KYuXyFGGoAuq9AxEH2nTZj^=uHvOlg>hp9CC@7HObqcFEmjo@ zEkk^iMCfUyx7B}uyZEcOIeB*oIoRF3CN)}PTF@au5iMK-NiTd-;Uu|Q95@9U>n7LO zDnYfZ`x%)O$>1<6MxI;1sbLC$Hvob3MSs^p%f`3oBXs_9L_q1r0I zl%jIaAgiGtkc58vsP0r%>fwNRxerC8PG*-)|Dp63t*JXlbCgTj5&=hBDq3sPx;ip*3?b>Qe>?%uU4jU^bXog^MnL9zZ+LohZI&3Tuu13 zSQw`|^`k@e)5A@@mOK(k!MSiUGwc|~@~KV8|LerlWj>wce>LG^GcT+L>#`h|*K>o( zKCV+Ap^LS`iGI3`9N(qcu=Wq&!VI&N!`p43qmsN4!{&#a|3?5u9l7E!*EEJhj0MQ` zti_a$4o}vlN!WL%pw@--xKoke>r=+-jO1ps1d*JU#XdQ3!~>Cx)V0`VgpL*#Ir)#R zMHe7*(xG`lRp&m`_}x?wdK%Divn(Q(KBwBJZh!(hbf^&EWKygA#I-|6*Fo&05mn`E zXBA{VdelyUo|QVFYUX=qJu~&BRA3J7^>v_J_r)lKBi^f+IY?41<#!LIOR#Mt8Rn!2 zRCnT>iMS3aCw`+V2+to{d5Ht><*~(1!wrBx3Tq4wm?yO>*^0@L$orFM7!;7istI-< zN?tH(CMPv%qFhPjbwTpyC!iFuTS-1V!?b#Ls?JYPagOx#o>lv zlk;TSNGk9w9{iLO8mGY`kD4}HZX(V=sy+uIG6;4uU55%P+` zyVQ*GZ)VGM^vBYw-N!mNC*>Uo=xVy5!6&V66s@6zlj=!1M}9YCdewJ`9N=!LjsYC; zif|i%3r3bj?T;c0&c}%|g)ZZPE27no`$s#o3)sn(ZVC zSmg8utk1|5S4l!G5a%A1dDpKM&oj0Lb;9L=6;kCxLd)FoSLa606O|n`62*5ol zvbzAIKaB<1xIH}uHO>eC)ZLY93AAH#a%t$`E-F3!vMI~=xg1p8>;!=J?N3$C3H+)Z z;4#l8rB9hb2Na#q)3E65p3D{C_9z?LatQP6w@Z{rRp-#)NACZyBkX{!V-QR6z?idsU`PiwG7# za0URXjUh(c+cjZVX#gDOsHmrbgX9o1$jw!*W6bVfv(rTC9oXX>)yXu(jW@>4z|BLmlR=-PH9Fr)UIYwc%@e?2J_Hk{3r#Lx)6JUDp#?)T*zMxe@bFA zqmd}<{9SWVHDg#yxinH|>r(-iJk?z6a@Zsum0=tl9A>d<=J#F82s6RKsS2s-%~Ue& zY=Ut|o&t@e{b*=>)-DLf20;2#(FRC0QD@IXpL&%Yjxu?q(ge{n#-xA-C@^DbO!8@> zKne9dDr9W)xB>4{>^4^_a(T}*(hCB_1K1k2${jvXKMJnPwqfN-=#~#Fbcf|OBgCtGK0k?#5H2z3~`EZ zhy#(F3Y}x*;+|vU=KU!ZYZau8WKX+*D>~xX+rBtpMsh1%(1jGAa$#Z ztYxo-=TLg$t4yCL1Ps&5hGEG1 zQaj;*Y;#ak)WX%{Wc}idjsfWQGI@6LeN!`==Qn?BDc{H0U!$S8s$>W+(PIw~&iZH|y zb4rVa%M4Jg!uAmc=J%@+>_N|3jTPHmhkTAlEfms4> ze2S|z$$W)kK9y++vdzuXry2RbO3aY6w&L4~>4Q$&zbwS$Qw&9Vaw?KU?i)SmrWqwu zfV_PuqTqXgAa$U9-hF9!$0CeVuni-cLOG>0hd+%pAsxbwPAdak&)TEzHg&D;azQz) z4RkQOjGVX0z#X}$m5YvsQ%0<#a2)!IrE{m-$OsFA>MHps8Og`JG@&iKJa?#ZOxEn@ z=h9|CN)Ir8SgM{&rtI)XQC)f>fLIKgg676Mh53O8)CyLRbUB!z-q|!rpzy>Dbj@tG zk&J|pf%L4qy(V;U_%b0qM{3iVSjj}qX~>j>IPJx1SowS%+~*{o^wiUorb^`fDoE{B zK37r~@}+w_gNdm3aJdTXxIM#FGV#62s-noFj<}>tSW_W+CZ~@~G7ZeaHS7mka*}-q z7`v4j6n$;RoGT~i?HMC>iMPcqMo?b@DSj{Y4<%R(0J!zg<&)zkn z`b>$!I=A79h4kB;@S~4INnXQpD;6j@H1)aQU;=%pYdKlMXOY&gTG%8*E~K&k_iAw` znUO?Y$Y7^|o@&BDbpHUtptnFxXsHe39YZUEGgB&dG*uHiB$hTiRQ+mB6?pGhkj=D$8;Mu zYk44c80lRQlGx>(YyjtwML%kl$s>xc`#t?3vrxcpt+09je#8x6>(-$S4Lc9Zr)EArO!Nc zrmA4=!Q+Z?V5&Iwu7s|xa<$x6a2Nb)w33o=qm$IoP26_}Vu}%RN$XLB zY>oYWs#n6kKpazD$;VMZ)vikIwFZ0kp$6sayV{`yl*_Ttr9DOmCkx({w6NMIRmW^p z6I`Tu&n?(gR~JMnSo4ospi47nc7C;Si(8UU=Ddt^oF3GUy9Rrj64l7ITWh@d^I`Nsnnc+0H9;V3EyXYvD!gY7Tx(nwE=+ zl$*)<@rqEP*!qutDR$uF`qS4638-aXxX^)n{#5XD$6R%zaK=`3#;Nw6+77f026X*ocB*N<8}eoqdn@Zh&JFU299pyX(UGp`ACp%KD84g zM0dvAXFE+lZ##SaX?e-r$DpUVI**3<9;Al!t@|mQDwRBYjw_dR9J27$v!z`LAXNS# z*0P*ZS{)FVMs%R=P2GKS+N6vrAP@^%pBzy#x(o+LP826+6cEuSdK&pE3=c~M|@ z=AOO5d6k&}Y?HyD;P9g+m4RXScdGM9$bu68W~o&(xox-)q!gvfJoR; z+db(VWjMx1TvV|zWD4wZIs;Rvkql{uB%iHHz^Ldk*wE?&3=eW>SKf9Ekw>W2`UoR; z8?o2kigOSSGJh&4&@Rtd&ePMaH$7-<5;(x4aUCmDH@KO{6pfr`tsUrNk-(s2YDsQ- zS3j*eeMzz~p$02^1h4~vSQ^%UvY8IHH$1&jiq~*R>0rk9u|i{5n>8m0%cS zz^7vzW`!y`aY{!V)m&mL3=!I+V#R5N4{rD!XDrx#UmteBZ({cWllc;M_T*$+F zB!AruIl;}nQ(y^FVcw)QCx$~&5+<`yNGlbVdCS8|$iX9ZiWRr?cejXMhDX3PlJ z*1092+Ma~!_~Sg(j{B5ga5~jccmUJY%CTH!XG^la?NTfY;YGDriAQHJ*mWPVV;?w515r4Q)0EZ0Dv5vel+EE#{&X}9ORl802Uy7 z=7dTowuz;YRFlc#r_>0FU4g#n1CM&G70W{6L^*1LX`V-EBs;T_&(fNYRwpijytv~y zrZ^AsTVBdy0CZvj$U>>x!29 z4#I)J+`OLEBr%0$G7b-KN_leXyMPM5f|cGSiA-dd&&`@i_a4O091+INxR%*eKJ#Q} z9jbX8I^eKA)aWAxONh#FzhRHMf~HLrv@_4j!yUU*(BqtAG=*|$>J2LoP+WJVBn9b_ zNE0VHz^Lvfjs)E8*PsKFNn8^gt(=a6qmuGCvb#$#9k{A;O*G^zf4g4gdeD$DbyW=` z6TlpkM#wF3UU_F9H_E@=2A>7B#oL6pjJP9~#{=tHcDhMb65ZhxbLE0F_|xt+s4Y;# zXBgbPVR!G)Pmx6(2dh-?kqb*oWH=ubJJdwzE}&cvR)=CdB*VZZ z11UJFuqSfLr$0D2^c`ueqjM5S7~oUXuq&V9sj#4JU`ahHTPnqHfB}QW48X9!YKQJTM)SDO(F@aHsE5;Ajq%ZP=MuGxBINil2(`ot zVn*z7@{mRaI(W)vL7l^;Ndozi5OLF~s!qbQdK~4;`BO3uIqy|ni?(t|9cuQc7Rjv=BTD;ZfY<>FT zrsL2UI3R7o9cfn~NIVZ}HYYRA`yT+>1y{K>_QOp^LO>4rB;lnK5G6{Q?m|c zdZ`~SMI#K3O;(EaALV1Uvy3+tIFKCHsUigfZha{V(&Qfh0EH(c(z3W&4LBgy4ytje zF+Fex;ajJJ?_Bn`pSKnqWdSsqYeYew$~saz1o@qUimdk_e5iLZ{VFpQl8nmRm{f-2 zrN4I!unnWTW0O^uES=6_>VKOhl`c;tJUX;KIvao(B%gt{o&`5kb1rzOZzcd5wf zic_}-r7=(va%y=-n=VHra&$eZ=a2#CilAd&qi}A-8hTG8IP(Z`+ccbSu=$GyGoB3$ zg?m(yO%%#7;5L5g;-qjoew10qaoXiz;0ziAWbx9R>5;;LQK}XH02-vx4G-Kw;*G3N z_|ox#>)w>)4hcTSi-vEA9 z!4wPvYBpwe=cOlSxW-83pMk*rYeP$y<->IyDu_+MW06x^I7tX*`c%nXXln;%bP&t9 zk6}?wF^$PNz^HAc_vW2_JR%d9`G~9(UG_Shm76WglDWtr;-E*^#PkH>t5{xnu#o&R z^!F7c#Im!BPBsY3nOvo-pz@3L`c_nbFb5T}E2YE$Hv^Ds9OW3s56ZpkSREc%@^5v7m^rwQrXBnq(B$6PRDtY&( zE?IVN!j5T(S@=AyNb1}G00WLY(Q?wzk7z75oPpMzv4ZXctsyw$7;2p(k-^74)i+&4 z)$|b?V8uUX&zl`Z5k_!#oG?8p#R$&K;*%AuM#rv49<;- zcB125$HYbfboQkGnRlyx< zNTqCmd8U~!LVRG5ag0|p;w5u>s;2-9oO)MESeoZ`z)koW#^nPvo%)v*b1flMVaY6h z==H5AtQao)h7phM_cfrFFxkS4qj6w4J*y8=zLIwf_f(9K4_aH7YiwJWQHIr8J8&Jb zf_pFHSySCfFfBBS26`T!g;t(lA|l~IC3AyWvdtv9c5bJo7VJwc&3W?};TZn_YN~Hy zd17FFl`c&|sF_%94RU}H(YM|Lo`^UGqbd*_QZ1s4usnPP zC3@skWjCLjiqIsgfDc-Q&@joO=4}=ERq8Q8JBJ+7v}2}e0NQvxD()eSHl}$Zn4l5R z;-}+wG1i*d>6$b`00wGQ6V3h^ry!k9R9T)Hm_cTG}$C& z`@B(Wl#*nw>VT@_p5m%yw`8|}&Xi@cNv5}@EuDnZCW`JY4 z6&&KLyz1&e`evIe^4J*Sq38xU%~C+__7&L_fu3nLmf-iOTRYD*zH0-DWM0n9nFq{p zGm4wac1|N8dkT#*gRXKjR?Xz1As8SGRJ+`?Rb1mJtT9g;nMh)P3UcHEGt|@OPcIlf zM>(eSb_2TIKhGw zJXr8#^M9VL-Ku@HyM624H|Nx;`>9ji)&1@MyYzPhK&q~+rVPNq001!lJ%GO}00jU6 z5QqoFCBVbOBP1XoB&H%Ie)f!*9z;n-1!jE7%EHLP%*HMDij7m4lbMBIPC!^(Qd(O2 zC7+_2f|RNlR9f0z9yMT@Ofq`WLH^t!Fqs0p$xSYMPg9>>Qk2!Xlz#5OE0wMI~hw zRW)@3LnD~6iK&^L{c8tDCug{qw~w!%e?VZw`^c#1nAo_K)U@=B%&hF3PlZLrC8cHM z74;2`P0cN>ZC`r(`UeJwhDSzcX6NP?7MH#+Z*Fbx?C$OVJUBeRxV*aleRF$v|36$9 z0IdIw_3!(?!TujyWdFD@v9YnRf&asWf$8_}#UjJTVHUt8m(v5-(1T8A}ox5mxo0LkO5pc7dXW# z52_UNh>AUD@}FgTSk#kFeZ#rN1y~>az<{Q3u7BwP&6@Laav84i`gL$4Fd13E>@`yT zlgAz-)}C%B#aSz6RcMbm@U+NHeSb9Pn=%nji-a9Es$2X~%fkN4C;OZw>(FG)lw-SZ zzB1`L<$bIv0jTh8Yr94U=%HBahvT{xmgi)5zgt}Y0`4p22|vV}LtZ_W6~>^s1V>t> zflQpgSw5#omgopKI~8NwoQRQZCfmuVPdT%?nXn9ZOYtIuHWpGWmz!3jC-5mHyat20 zT(qQTT(aEPa$ARKWqarz<{GQL2R{TACLu3=lH!c#M|_`nv%Y4h! zk{sB?h;MA{hTLCQ|K*yw~~^EuA!$@hSlntL~<! z9>iL%a_|ZyQ0h11wtfJH-`lQ_E>pVoPzJj^ykG7fxTKHYK?^@Q0ZN&pvg6|5oXMt! z{~FcOXwA#2p?p-Cf%JV2{F2_FY0A7k!LPC(4JTEthY&s2rEV%s9)>9$H9$N4tVOAW zXG7TM`rq;TAO1@o3sZxO6_6Q;_6xtoSe<)k@lOBGSz?p>F3QIGIe?+9V#TmGpF8-+ z8SDcK?VL}I)DEo&!gyy_AZfHraWsW}^oJk4m$B(LfT~PbSqM2zAmuizaz55gFkEru z^`w$4UT^Msqfhark@_j?{x^pvR^Rdq=N9tkH(-bTW*q;1R8iPs9xYSEkyXRzwE@al z|9^VD(5AI6O(mosqE~FCa-oEPvf0Hl$fy((9PA$8v`~r#4?flu& z9LMcrY>$CB29#*9Ug%I60}$v`@%)VIR~R(A@p<^$K8o^GD=t3nw^tww$qEQpRo}dh z)SPS2v}rB>P$WAA!1AD+lbtE!i3>H|ZLK01e0ayPB#9#}qA$KGLxZ^|aS$asd2D^3 zGqh0fo&TrydfS9A+JT68(dO1za?9ln_gL^l3)Vu7WGXMx(t||29DtR3uGODBQB9s-ICLnzonwj~9`BbfpWg+U6UN8qV*Pd}45JD4uPQx3A;9!M)tcaL!?EWg^ zY@YDD+>UQh2$^eGL^dG`EtUR+o=_cZxT2ccw0sd>fFA(e=H!a4BWc_Dll6G?;a%H= zUmy{KRhWCwC?UH4pYNB?`NotVfwhBvs_vRs%e(VV;;+gORP$)AVyN-(#GhG|x{d6% z)xBP_D1UTGZ=B(2TvI;$J*$@?8M`XFrNStxdQY?9Z;39}&zZ~Mt$nT~X!+g6WR5j_ zQY{@+WHjDog5^h-7hE|df~H(P%#d*q(C9nu(Zv9R>uzcaO*;91E{1g*?21CJw>ruK z!%nrFpNlu6rVoTvKu-#YKznaPt%D z!zhPp@}}S0P8kUu;8Oe>qCO_eBoB?+(_k4{?pK5Q!Z2<-y88CD$@tbCBBDH@)A@xe z!Z*%z%wOj}eP*v^e1b2exeu4~qR14`ol3^eX#%v7plLhq-0B?+6HsJO`^Mn zBDEZs&ZZu3gVS&BsjW(!9~>4Nj@2$yN~K<6 z?B5p1Mg&{GQ#+gr99c4CVu!?4uQ;UVtFUME|S_O11hUY zF}n4V*FVBL`fbUL#UENvCSI46An(a^niQ*sRZgE&q!xnFFTS;IyrFYkZ+46y|MAeV zOLA`8uo=g(r~Dw&-AC+)DUj$K4|YH;&o)dij6A!1EE^_C0|kwr6XxK zzS@gp?|#YSiuKQ6rLwuWUue28813JUY405qqKSg z?0qdt!4W|zKI(CLWVYtRwu#S2PQCM;*ohMga%L+4Bf!8WQ>u!{Uk@L&G>!LVYEG{H| z_f0`|Eg{{L7XK=sXeX&R>$1}B8M-$so;jMH%ZD~Iu9(^&p?=}LD?un;g@T=`^?vFs z(goOm^uf@p8wQryKd>Y_%8eU17ySHNZ&Lc#pT>oqPNanBb1W(_xEM^0YOdd2&(B#< z!gTRvide2PUoYNz%GMpQCiT;x}{w*7R>GVnTZN$Kp6O)NE6 z!)$3~7d`ZNB`e7$rMnY&)VK?FP0Cvyfa|?ZK}m$0%^SB#y9DN`CQmEMdW~vr(_-ux zfKA@}`#%9SooT6yag&wvV0D4D^c|gSl?R1R>DR+p2nx{4#pMr@fYlv@$wyX#EEIB6 zcQnF5ZZ9uEUYy6--gTe_2}iN9mAcT+7lH?Ch#@v83x?|AmEuh#t+y{8LVzC?Q0As{AP`M!V%V3DwZrk7wf=JEjy|>6ngjThUkp{+ce6VRh&JM zns(do|8M-EZ%>?2n>&7iK_mFp@ur=Qs4Nx62)&0+9q9Xk-%DZda3$jlu+h#=0*S%3 zu+?(ha`y7mmQ1Yh2S#ywla~&!5?;q6emtSf>wk=~CJ?^ON1a$Gb-!Q`36n1)lOVzc zye&j8wEIR757F!B{OR=iY9L4v@QB}2I=$Rym}7S@q< z8mPY{nzz>`^P=qsIq;!y&+KqyzZ%iS6XpAL9aX1ng*V}?@0*>dr8kvERAgWm8{m!$ zy`L*tJ_OxuBy;<5IopN_YWA1XAr`)@7CLxwv>yt%QxBRv*tkr0aMSW7eAp7_QDyyI zL*8wMndh05d)}H{d1svO@9B1L@{(GYj!#Ye-X3PeQ&3Pu>p3?c8zr!t*ERpKtSVsV zMCOF#S2f3)rwZs}d0DETRd}1$J*iovdZb2DA6rdEQebWHnm++Nu>$bSa&SMvu~fB#?jvl_$ z>okyh_zRHm`k_M8uW5hEJ;t#A7qDo&+wzLvvj~}}uk@+6EG)OyVgmrd-fOOhkM~)< zdp$R*G0;Y?7@0Qs!gm^3DB??Nz_%6Ff=bAH3Ys7KGch}OU?t3bH&2*Ev7(aR6q$RU zheN|s^Pk4El?c=&u`Hw2)qz88y~X4-?hxyRmeyR zR`2)%dVn;cx>1Xn4hzYZ&=UCZ{4c=l+h7nemtr`T^dW{0Z(eAew!$ux=nfMJcgFPu zSNBqP^7eWa*D2jz@p3yN&X|*hBi~xuxLm8_xC3%XjKf4F1Al_kx{Nb6HeyirJ$S$9)YkkGtr-7*ZHr@ z=CSGsLxXZA&sjCt;yHDt^{f1E23)v^5JpWwA&t&rdblQI${rKnX#8+JC}#o$UA-IRS0L~K$21;icc(qCl%1$0qX z_}lic`rl~AlykIT8YKOedvjnc4QBq2ZY|duqRe4=${TT{hD=86(8#fWIOAJ@90P_9wczA(XQqFi*SAH>o4T7~v`X84LPnIzq{ADH zCdAI@4QF@32H7$MpAqci{u36eQQB%1GyV870%gkYiTaM8BfPtpfW-QV)>En4qg!v0 zd_JW*rZbF{q^zq+&Kxrp#MYuOs@bg9L@02B_%Fb~OyVS0&y-qm)0q`vW%G-3IR=BSDL%NS~U8TGT8B9c@~L%k_cS-hb@ z4kCe@#D8fgZ6VZwYKk?;@Y}GC=AzK%nzqJmp18zNDB#Ev8xS7i|1N#kbRaJ;(ZYL? zuilzyF2p%RmLnJNon3@@?Fob7+E~cG%m3%nURsrbwQ5Y?nwQac5WfbQfwCb#m@NnN z{k3Vr^M6?#i0?P|@_a-*lCWOabWfhYjrH)8h-(+6WT1KI}cM6 zEz8E0`q*TKGZY+KC_ipgH;FNHRo)jMr`!)99zyS$cpYZCudp;HmrKRL72?I0cHomG`0sP>Su_BZ#3l z^ddvrSSC@2n!6`>eTkw*lAP_lhbbvA)p=!BCQ(^p2X!}-t)7(j&O1t4RI<*%LkXsa z4-F`PpX;f_jKs>x69`+V1deQ#B4vTX{7S8DbOx`f;HpKe*7k8~yGHu`!PBW~?gmSj zAunoE`(I$-v56Q#q>vbYnnHTWY-vaEC++_`(uL$x}21(o>bXo z;sEJ+Hf?^xy&P@Y$dMA}QMqf-yYc&Z$laj~kRjO=oZD! zB{fo}Op=TeFf(N_p;Q2ppTHxF>C7_`iRy<=V)@lrV8%9X(@Bv4+en~Y+;C%ioHK}+ zY>MvOtWP^bm)b%Vie=PkntaE{JGY~b6=)RWzR_x3qq|`T9WZp`RAuVLxTd)W$*KSsuR=HK?Q_a3pMq?>+u~ik&2}#o7=FW zByFzB)^-52b?U0deMwRkk43-ht2b;h&JoAveTo`uI{h7e2-qab2#IewcoL`G)0;s2 zr!mH6v)!?+b*OKSjh~~E znSU^g9<6({;c8GJf*VFV*bLPk2PYcqT8>GR$N6K9SLw^}G$cH21(Ga$0ehc?2D=FL z$s0Fhu(TyP$U)_%AddoO@^o~ImapFLl(&gxpC}nafSsT^n%o!fOKd9A4(X-gr*=#X z0#?pN>(KN=^-!{0y2T<{sR2_AYzBYv&!R4dlebVFY1=15f*Kf1MktwUyJEZ8*fwHO zyquQmcdyjjl|<^-78nuQ>s{|>V}TVY?Jbt*DUHv$7O- zYY8*%Tnxk52D*j<$dU{Ja%+2xI#Eq}u!NFH21t>za&nSm%h9yKMHI72cU4gjPo!qS zquMVAV>a%+2Qjhl8!fEWC1Bjz7B^$@o|U9M8?L;Yuan%~9|U$9K5&k(ZZo8vyZ0kf zHL{9gyH*Go)c1;LdRbEUJfL_LT5dGCUFOTEg2<>JuOd!Vgb2ZKjh_y)FdhfX&Gfjx z0AeWoa|Yg+ZKYx*oeb9r)@+=I$Ey9@s1mDIn$=%GI@@kpt)J(rK@>4SL?SH~>tQZj zA&>U5qF(@lP2l-TD)-xaQ*yFNr)Q}dECqZRz1PKe!$)yEIB7CsAhz9W>WW|G`+g!U zZO=kw)PbHxJHMW3mi60PzY`&lCKi~`+OrPS_rtN>!E2YcRl5=X4puQvj2`Wdi2F=2 zaf298r$aW*(2r_in@L1RDMY6Tn-pWrVVa;z*BC#N4!UmP&5UqT-=R~F78lB0w26yL zl`)=n9*BcqxP%y(3Gq`;h>je}I}&@!<-Z-0L>RfbQ)*;ySvZKU++D)nc263K4;g66 z_8~(QG?#%8soAM#XD9Lr`>C4vOi9qMX7i1jWa%QxKd|P|fiunfO&Xjp46v=SO!D|; zm*u%Rn8xv^zAY>h^qs$d@BHoaP0`ej%>7m2Id-$uy(QDCx_y|`C&`F71<-EU-C5z1 zOq4eKXV&;2+lMjS{Je!V?SEk?)e>b4*%?hLAEMcir++v;xSuDCA4>O}&WB0*i%(ej zql$2)xQVkX{6363(NoZQQ*AZ*)#Wrz`TT0oUVky0IET3|=bk?Z>Bhpk0{ns0yxE#a zn7#F-9gbnNNAphE4P6fS`VvepnOE8i_UAvzFAAND(04F$!g`CQ8-fK@E?HWQ2YsJX zhpMd>PTu>e+a+K5{B+SYnQ*vkaVa`2BJExftdD56xyVD}xH=z|v0yp-(B}WF3Mr_G z$o*#NqGvL1wWJ!-v4oq=Q0(D2=6u-yTOn0H{_Ydi*V(>TH?y7U`U}5edHJYyyy>3b zEyPT!lsNH9{`5^mGz28jvCB%5jDh?^U0(Gb8u$GkwWy=LQ2PsTTKFtpga0|;q6hl0B%%r(KO&T~ZIIXRu3&8|VI)w$8R6bOA37ZkO!0e~M$BKXOIK z@fNxe2EDds`0jVCi6p^3KZwJtD=5$+n{oRKm@bv0vmySb%Db9Cv*(l8LbGfh!4hg% ze=Ke#82GT4*?04qHaNY;G1f93i!RjWE18~hs-;WS!w%{uSqvj}5(uU*lkbYw*n}x* z6b}#UP4c_ggr`q|VoNgXYg1sll+OHJMHoq!SCz4E6S7yi>oa>j=iZ%jE90{Z8*^5; z%qo;?(P>?9Qawh3S_DBz5n5;9dMCJd_U>LWC}xztgV<$4MbGkUxVz<69^GSw05w&2JCKr*2c2v zi(_4ixYO^p83!1Z4_jzP@Tg=M0aaA(RsZkAq85Q({&y(@W)(~g6-*CJbJ*wc8 z29JtKy;Ti%phh{o)W3JG zUEx>JJBn>W$1}&}qEB@W2ccYwvALHz7nwDyIe|>{YWvL`WVAMr71h&G+!voawO{1r zk{GDl(yL7fV==MXU!sfEi&76%3yeDqM=RMrpNH&^WD%7nb9ve)F&p=`~Z`l5gGA6hDz{A_JV%5)$ib93DpasA$292Ju(dr zR}cT1;apEn^(yM-NLxvo&%eERY6rSkM*%=6e7s=Acet+wk!zI5X(ZAPOnzJLMP`Fm zrh_Hf2?wE-x~H_SC;gukZiRG#bl5Vt_lxp-O5(Croq^k+$=f}*F5)>q3?ZsR*F% znB=K*T}|#1XEyS&c7_AkNrXXxrA?8-w=2E{m8}YvUz|vsuE+8~SkjVYG@KUdKZB!) z_8bz`Nrw{l(RQfE%8a1l)pd&~p9~m=v_%3(H}%cF^!0bEetv(d8Wr8yv}pfsTy0_q z;J0wLC}aY*WVRN|1RPEG=K@5njeF%jMyCNelO0Q2jsCuD-T0+Qu>3M6#aq%5pX6-|`3F#R@YMsM|1u312+t;=9ldhq@+91p&cRTJ&w;ISlm-Wqx!+ znQ}Jr`n0PF@z#Y2;(|zW+-q}8Wk;JOt_DvVjb^oF|D=@WY|OQlolafNja!W}Ovgsm zzJ?A2Xu8`09OHicI;~&G9HFSrV%tbkIMUzx0;2YlTw6WYYgRjzPHT_VUcDSb=qybZ zr2AJ*;8s>#?;q)`h;(L)#XqVI&dL`$elv&xc*%nJ z_C@umEz{Fg6c$=7{Y+f0yqv!n|*__+b*9>@j#TKf960@60L2g5O_up9RQ>gOh55JPs z*tbo;ckj=dTz851HP_tyj-@&1%D9EKDlCnEhb%WZXtC}fhXHL%V<`k55SwTsHvV;0 zG!{Ph(J%sLC#j+?mGNxhV_k#(bcKC9d#6{IVAfO4ceT?WGtIwN!z_jpUkfO>PI?Ib zn2D)Y;zhctPAUuKpdutZR6y$7p##mA>qRe^&5;;Kmbxkw*|cCzTheB2k|yPgdWkP2 zAI5

!seN1gdm3iwCI_f_Q&COgtoE&EH1%zYO`NDyCrd@ol2txn$g;#4mY9~NjSz1rd=#Ms4QU>!|2b|3#5@?F) zUYkR@udBIzo?ZH4Ham)p>6QGG&dVresh+2X#hFY3HtZIv%=n(<*QNHo;{=W-=&=Ls z1@0)P<;5?)u}O!gK_;_yAn6d7?vh&tg4_ERGj5a37mh^lv zx9C=`MwR$|N_&3Ta?6zJ-VhUO#V@qjzRn9>*`b+Ofxm|6A}h|!dW{{ zs&^ue`ZFiHY7D}(_bDo7(K_j{@7Ui-T&2n(57S~Yw+B;u9Eu`y2i}zixbbeeMCEq3 zJO3oqv2)Ae=9%JX&sg?e9QM>Hk?8``WjQ=?LYhz&O>1M&Hg15sb*Sik4^!B|U%*jK zF|G1G^jf@RZXX6FyYQM_%;6w2pOji~t6Ed$sHz#QXKtAO?1S6S|FQHo3A0@b*jHIp zsm7XTUnLHWbW)BdU7>l8_08}@)At)YI34;!naKC_q$2*%l!|CyXu07~MB^8i8&T{mtxBaxoE9F~@o6=Zw`b2|- z40^8D`w26g&+lqf{u|Aj9BL;SM9F~&#|vVnr024R>R!wHZ67+b*yWByLF*=z_5`j# z!f(PR-!+Es?A*}7UhbTT^riS)ahsMH>FY2N?7w0s%ebCYy{V9QEw8koVRbfj)_&6OLr-;=1Bz0~RVwZ%2uPaRSmLipO8I3XymO!cNff z)V{@IiP5*`Q(N5Zl<*?tRFwFds%N3((57kgm%U5~;6Igix+qt}q$=lwS1F4EQGWqJ z5$b(kc*DDQnpLV*lfKGKmNi$}vR#n-@C8~Yz$n|d3sEnW|Ar{DQi2-5j0NKba5ID&k9TM zcF=wIv(f#r=-w&8#2j+tWEFTFpD9*ZPdVSls?-&uo~F@y;d;A%UGLg8Thr5Fp_^;e zPSjS_RdCY>x1RtR4N7%#FHD5B#?L-4vH9eLMkh3}GynN%yR__l1HlAb2}a zyaO|{0AnKP5pOJ!4pn(|2}f8MbITu}YNYo>pS%38`H;Rt$8%7X2@z?gOlZ|e4M_^W z$GGIeRM_+Jg_5+-_ho^|z-`8bSZ<|6$q{hZj&FB=4fQ|>2khu(gL>V)UD@6ynWXUvziLfEk2Jh;49N|5#^fWozN#@c zvtDG~AaTE%eJ*s$XC-f@u-JBBa!JlQ4B&inkqhPU3Wsv#9*X%66?3P&p4{KFAQ(of zvCP}qC)78%dCLq|_jErd?VJ0NzIfA)Ru5m)u_;sFly4{g6+%F{l`+^@@1PJ7iVLhp zNVB$077GAbAiX60sZTh*FUVu+X*w=VTAy23kbdXo!}|@uxR0=`pQc02rx!%LRqwx0 zqWHu+N*rVxH#qz``v6DunqnQwb&Pr)FhXC?)fQ8}V;!^zMSe(5J<#On_Zd+4ENxQ$ zKEW&?%0WL;lFI;ORM#gKh;xTJkZXgxfo`goROIs?}@Xt&ul#+Sy zX`c5!Bc))1NVJrGRCkG*ePCYKT-#8!BJ$nOZK#&2w!mW<#^&(Z{mj{vf#$QNS)_n0 zF%iZv-oC3AeG6_AY>7}&KGWvbFmq7!3R zmCKf2vwdjNx52^~W(#{JSpnANS6J17j1l?sBGi4B3cE`zxea_&Ev`X6HGLX0Cp)uS z0ZcI?36vZ{vkQ%wu3CjuCxZg2%VOq&kU|qCd`9MAXR#XbqZ0+8h{db2h+l%7#5b%X z>|sNl9_x~GG(U7(BBSgs6iA^Xj5F=7NeGLf1Xt^}lyzfjx5$9|zkut*f^!zGx9lL7 zDp@(TR_cwM=6chKKQ%_!e?aOXB;3tE_m-ek4mn@&V!ieGpsNpdxCets}VD$4b3iNx4gR448~Ya`}8CCw&k7pM|KO3r0qwlFkL{Y@Z_RFC zt!_>FPraAW{G(82yL<6#UZfX{DM`}!hX_Tg3hAJ>PLssUAO6QYooaWam&Qmyu|Zp? z+>%et?B;c=hQ^3cim5UC0s6=yxcHc7qao}h*Vu1kgsjS4s>9Cz;Hs?hSEk0c1hkO8 z+(=2;9{jIAs{=c*D0Tif^YIBow!6YPmt11*S8%~BALfIjaFp~87PZ4ca~r)f{fvOY zkG2UPLB(~omvuw%{WDT`8}wujt@uHF!Ahggkx>yVIxS%#4aJH0$}q3*pRO~ia#?Pl zeR0<&Bcfsd7jPikg~xrQjnbjUx+qb(Am0i8CR>rzACuWT+?2f;E}Ekd`QLcn{GJ$j zgSt5*H@V7Zv*QjO2`z`O{H&t@>Vt(g@l3ZbwMyfT!1J28q=NV4t1&tpRDi9xkYy8+ zYj_9&*Q-6FBwO82(qFSHQu6}$GACFOdhHFw;q?yhpYMEg15Ou_mus58Ct{YbVV z3-{GN$cZea0svBEhu$jdi#WL|>8Oy<#TvPx__b>EM%b^Er%;%vy3)}W`61C}_*Rgi z1GZ4HIW2nO71!U&8nAJN1yWU3{0q342-Xrj_K-c%N&E9h2CgCwhK^J+XP2&L3fE-W zs?g0MG*gTq1DaF}T(uApF0nvBvd@t=52m4cxdtW&hJuZ@#LPa{ktS}ae?E}2C;#bq zR;~D14uJJQZ=b4IDo&X$4v+)BAUo41ra<==#7AP^&$S(M5~ZxErMgQje)8w;EB7Zl zZM*mZ{OJPj>7uiQ8MO}XCWbO03qeOYlyNz5jXK00-S&Z8k=TzCYlT-jUq>Fl8cIUy4tw?0sbEE zUBewt#`U4G`+2oxmhg7TSLATis*9j`eCu~GhWsBa$WW(O%K~Lf0_#gZBr=+a8`DKg zdsJCau$?Cov+gby_y}LB&*V6BqtIgaPTpb6FORRUGx#dt`hAVX&f5)UEj~(YHN=_qA49Yf`%{mr; z(9!Cgs}`k2oX`WERjIf{4~aY{R`iz;iDYJv?sL`3LEqfD4ZDoJG@>htDjYl1a%g+I zcli(a_0YC%J9&{Z{!J0KYatY&s~$j@I;G8d{Bm+;>E6%*8qHKEoGtIhNj<3zB@srQ z_DHHSM8^fjdi7AF?|19yp7~uKKFO= zqzOr29PBQ-iR~{Sr%{MGv;2Nmb;BU{lr8T(9Sa<1%M7O2btRzk^e_2_dAgSuo#P+~ zSE7>oh8Z@%j7@{zN2i}U01%#u8k$LndqbU%v$hcNO}B92BND9+(B+S~EZ;~hPR6mg zTrR+3Yk!6F6aA)ry^jD_WlrBt{rHL``in_@gJ&z(W2Lvu{v&s%4k}=1p{~(I&IOm$ z+M(G)l}hTtaP_D9uzf+QVI};>Cvi@dh#EaW{)6joLQ`vfu+9RE=7rne%J>MTdYf1_kM6CKe9-4x) zyT|ZG4v?z%23u`8tawKD;F4D$$L`|wWWQ(Y0N@twCTi~u z4`w7NsJq8P9`)B8x$I{R0zK>n(h*GaS>ENlJF8v5b&dk_*;?wFRO0}aL|IAs`pxDI ztV6_-5$yShlnY_Nh}dKmt)$%4C{*RDQmfOHnA0gfkD>uNXL0<~cPH#e&N|ee*U)=- zW}@TcN}jCc(>)A{#lTH@IvG>RTmjh^J`Y=@x}Ql_?cSIRvLN<=Yw#*||Kd_+R`ygf z)bI8gU`-`$6=&EAr4C{kuGtNQNRaJk7ufWe1SQYV9>~<*K#@eXbO{SI7Zbo*nD`jE9hTp zy_5kn#p}VXTxZT%s~5js(HBCK_a&{Te(eP5TkuqsV0r}Z#nmw&OBd1VMANT~sjntv zdE+lxYu49r)KMxlZh#PpaHUHP#a9lr4p zI)f%Vch!tzfeuA{^rYusA4ab2EY0Y~L;_ytYp*BWlC8tph9c4ETI8!*F~acIj5kp1*~3_zTc>aW-f4 z6UkG7)U+GbJ8y4x(|Rg-y&+Fg_V7xWjeoA>^T+TU*|=btVkb`VOqoPT%bo114Wxsiqyh#(IRhp?t4-yopAS!!B2b|^URFBW!B$&Px9uqTvaCMh!hb<7Lt%Np;O=lkoc#bP zBav7HWF=Z8*+hdh!11OI{fWH-u2{!j1HMPNO%irDPE_`FJ~Bzv75dG4LB}F*o_!fk zykVu5x~Wm<&q-^>72l2+fAujMVpKEHc%%^&BBR@({D;T;zr|FGH!)`{U-KX*+XEc| z1geAEN60k}PZ%NHTUcOJQ%GXf#i-ivo6maF-osa&5*!p!9Q>;LY;9UJHa0w&Zx~O% z0VbaQm&#B&EAqsxn{I2`?ddPS;&nA#R9t2@Sj&V|VYp%<9e1FoJ&X9UY82@L?oN3* zWV?ah)zuLeglO11n}5w{`(xw>imk~VqI~*e?<#qrWHVB};iw=JtbrKr86i$&(IM_fOa zgUK0HSWocA zQonIx0Ru4$L-J;s=r};3s2UT>u*d*y*5;v`7hVBbg|MF&4Z<>{?v3miCl*0&VOaEV z;LbLPNp@6V&Y@DX=vCQOd5N|^X2s|vX#$Nchef#f_w+KsYHDn^A(1zGD3gB@G&Vds z^=hk#NZeJg*x|?Ay`H(CDYrNi9Sv8M6E!ZBv-onRa2b)OC4Bnl1kAkKcBIjcvjCTQ>v#=8$GzDy~V_((R7eQ7xy2ozHwdF|0>6Tf=|jr;9}_*U$M!Eh*N% zBT1SBy*bN1slI+*=1dv?W(-P0)kri+O3j|vig@K_Wr#Cj26Me+6mgz%*b?F<;*>IC zOL8+yVJ?YjxO^^`e?7o~S(aj=N?&n>ctuSgXyE2LIWb8BPLvDwKNeKEUwH zml)^2uu#Y#ut`@9q}|Mpvyx zr9Y+b439i0Bol8|!$D?9|lANsEqo$OiSsAQk4 zSPOr#IvF8??hsj9SYspc9sVM zNfB`;j|E_g9Y(lohK@V2d0^hvtP_pqKnP)orOiM9B_<;U~LB+aJS zkYyf~oo9;8!+nx+jOy98vHSoQ9$bkgaGCeSykpXJXi@&(7HXa7^LWXtwSkj99Ue+6Y-v zb%DQtg^KFMN4O(Nu_rXB}a@~fT_R8dle@%DtGTZvuD014FA61I-B0;((G zsj=6VT!~+m{(5(2#vim53SODA*voyvn8;yq8K+0h9!Eg)LdrIml)yv3Q)1=>oZ<|q zpztjozA8|XVxmjE)fLE^vP$c~7=jxNY0jz+v3|pRm`QEPyXs!rznc>rmd^pOr4gqq z%l{{x&h3z;#QdoU#e?Pq5t{S^R~ao}QGg>&*XL8}fRPkn!eiydgMl34QKX_a0$eO5h`8|seZmtA09Mqyfz`fjwjE;m zvhu+pPg`Oh{}`k*p+{|+v6UmP2Id+ps!M<#bNmH(w@&SwdpduJb+xT@Zsy@zhUrr> z{CTQ;Ej<2FG*QfHr^}jxYX<;z`84g;?}_>GpYDthc0ug%TLCBv%sS|O|m)eBKU;E^HbD$@%6;Mx6Q`&Ye)$HNB8 zLl*0pOCe9$-Wq?9(5lXEk|0RB6+ZE$%_rnq8@TyL z_yZwD%{OX7{-K%j3*v9M&)W!|AnF|+S!=Hq5h;0(4x)K+!n^mm z3_-Bs+LshpkfQf0uB-w4-9FX!E&cvXq`~kikH(ZTgQVU@b%`=8Al5=-++4=%wf;JE znw@lb`M*@p>_vVu@a+OZSB*l=!dv-NpayP*P)i4pL6o{$Fz7m zL|=y7me6q8evOmn_+z1lRUHT*=`HV`!6mFi9LzKtdXVTGm?hZga*0t%W>%YQrtvdH zEapUxdSWm}V%Qc{BP*w>^|S1E2wDWhkHY@tv*d^zZ2c^<$!D-PSqNiR>1(WSIQY+> z=}L%t4RY;7L>O7B0dw%}`3m1)vEI`Z^T>JXNDo#^3Zl&_Ie3SU^aqWX)tC)I&<4HX zP)_Wfhu`JdI<#ufbfxUk*iqOh*)!(OG#^u##|za*y-XD zW^((DBs#U+<`R`_x}5Xd-_JkK>v^8f^Sqz;`~7)iTPqJCGBPc`GK#AmtbEtF2Z%Q{ zW#+^(@=IiW-;Y|;*5jgs!WI;y@3x}Dn}z5D|!An;x3}a;)*N259AHOtx65`K@ zo&3OX4YorGqRDWX%dK&%Usq=CeAXA|`scv0i<zFUh z`DrOWMyc*G7u^XyB)&8BTgZ~rAEzHa*1-xdA0IV4q=* zWykr0izdeUK-|cRAbSl;zW@(eU-4&V6x#A*Cli*vARkdu{B~7#iKiuG{!^LrK#HH~ zNoF#vgW*4y{=&4n`L?K^#Sq|Vo3lZ-c~l6=uixcN=V_sQkb5=+ekdmONj#Rd62!LX zyC;(r3@!E%M0dFJ4U;Cxx1Hxw(UF1vYtlopzUS4M8mWss(~wNB->!EpFSy$BcWRdl z9^7@#dH|@B^=b<5S>55DUk*JnAdN*Zuc0LRrM16IF)_mWQ|-q?Y@=(r=cYnDu~_5q=&b(y{+HpU|;aV-o zC_+2#poGAEO3#tNC$4x1VAsDMJZVj;vyc&obgzorPvcz^O&aY#1hse7;+$b3?S?szZG+qZ0dX#y$HJUC+INW1g7G$PeMtj8jXye5hJPJ*VQR_8{8F~D zVZ}AVkxad7Ylr;ro#+kDcyE7qoUVDO+so<~(uQ;=(8^Z>`P_OX;GxtmGl*3z#a(q> zcgfScayVl^RV7R9tGLn69XSRSoYgicF;d!rR34joMhpbeqHS2D8t?)I1TxpHCrEZ~ zb$}O-N4}u)0?$_02n62(p~2}>M5J_B0LiLFRZ~r zOG2i5GV?$4T+D#&Iu-5#C4*eB@d-Mztl0E_LY$VX94&R>VBQPIF(8-q+Tf1s;HI{b zw2Tb69qFf;?8CB2zk)~W9Y&d8{}u4^3K9H(EZfCAhzI#@Y(Guc$IlJdH6yJjJxDX+xvx|&dv1CCy`p7e`pB0}Y&_JzMtg*nJim{AqbkENo}f!FV&m{uB8(5K(`j3m$Bo?~wRsM)Xh z#(tMhXl}#NGxK^6Nv;#M3mR$-^eqwVISj z_@^F+KYo}ulIgzkWCu*<%jgcyMrDT0p1LZCA$zR)pKm0&@Fd}S_4_F4InxgeMhLb2 zKmtLPl|?=LJt&xMk+8vk#6BZy5B6EDC_?O3j6Z?@QGM=7VhjRS+3`zCnqIWcgzh|C zu|Uf_IU`{==T%Ct!eFF3M!nen((!qgU~JF7f#b7}&-TRZu_V-P@49LSg`_;)eL7v> z_{#a)x!S-7hicH$Peb#DNQ2Od>Zxp$dff5VyluN$dH;EGp#?+`^-8?|1+l%Rp+R2v zCSs1i?eQd{o+aUAe@S%!{sPXiutsgwLep<`nFw-P?1CKgQT}7oH_Y&|C z@vUQn#@ZZ2PyX6Pfg((FG zBq4Pe#P}z<-tF;Bjj`PITtSZ^+H%(qTm7+2x}DF(Z-L^hVK^Y>=8i?mxrGvfyS9uO zEn7+Xw!#?~J7|xBa4l@(5L7-#WOWkWC$w&o5T4*%5wvDdngSecK#kwh%emxA|g}vtgOu#cqF@} zT~!Icnqwuz(F*R=u~RYfXOoZ5jwo&oe`Z|#$TpvOSE?3OY!J3tCE61VK+T<}QWt{v z_o~kjRdS&{Lt^&|Y+knc5@$?gELnles(qj2qCqI-vg|PHLzaE=Y+|@0dzIcKs440?KB=ONtOnh_N#3I9UE^a+=Pwm*hR5qzPg%--@UyHRP;XV;2`PSy_rkW>*a^R=`11y0vEf+$a z*9MIq8H7)u)t;sy z1N%E~>SkOTKbvhspK@^G%Y|Q?!voTK?AsP<+V2w({TA_m|G}Ggwb+EPQB(}GyAdL3o+RIE^eyqI9iC%@qq{D%}sNa z7O?O5MXX+hU=s+t%!^>->QVip#?pY+tz#K^u5G?*Ez5?Uv4m00^wV;KEPGD?_%O}I z0Ko(vITkT{Bv~elskP1OV60LX_;G8+KC83SYzVAVw7HG@u?5S~>&oN(3zJ!qQi-@q zK?fJIJ)dVy_pe<&d*erqOudp4pj)uJxpviqW*+j^&knpli8kC^Ku-yy0Y5^yV9^_~ zigui)$yqx5+k#k-Ie?RTEG3+c(%v(U?>6T2r*B5-p?eHr29>H#J8`eYd2dOqOEX0T zB}GLHVGzI3S+(I<44ms+^w_J$DZW>Uacl;}itn%<0-^K#Uv4#pS6T^BFfOlj}OHI_6U@z{5ZJ?jMGU<0pEfKJDKnirkn{^t;WxWOgJ9f68y1=g@VetVlhVD^Rbb+)Vz zL|SM)y2(#2&jJm~DaPL9%A|}ygAb^{h6%lBt2e$LM+G3lQ-vPF|DL-a?+Kv*NR4PM zUEI~+(?Mx}{IgLVdI3@pycFHtLthD+P*yZh zS&^p4T99c{LR**svHWe+2bzonuz%{ct3=GC`_{{Yd-RfcU|$C;BMf=TKV$i;j-PK@ zahB2YiVt#g(W%$A2bA!d?xg@e@+&`@<)&A25AajB_mf+hFJ0ugPxEy1BV4hsD8Sp4 z0I~`A3xD>Y6GQgnpQdSi>6>QWnQ6cb@FJWbxF>N!5<$@|g&boPY-5+@Oi z-ipN|N${1Al>Ve%EN4UYxMWr}uyJ$ykaFrEbEC#j66d8%H)KunNIkA?5u7geDS*_N zb5zQR`8a!;<-SZDTpXuC6IQ%|!V~Ld%H*|lhsC#R?r<(S@f^fp=vEu6Ee|v2YR� zc-8@m*2rDIL~=ghUey?$LVi2iF?*V}XI$&isu_x7{o69?p|c7NzZ|lsFL6`5V&Ed$ zcBUopSpVeJoqUMx=Fnev+l-p}%d_k4MDK#ldFS+!?k^gNej(^L^Lh&kfC)pI@7eL( z=P>5R{fUXyxp;*rS64t1ki9o(?4Y*Q6RTBKs9li4Wi~W5xkh1VZ!{wjks7(LEym?w zag*A>iBz?~5GA0Ad0{Zb9TZkGT?6Fm*6b*oycMU@)6UkVhy_N#p!4t7AS?oXP9%&C zrLgjWv9ot=R=P>CN~eVMu2xl`l&xQH@W4$n znh)Boy1i0%rMSVG;pTf&{tGdZV)XAmkk4eEt#PS#23c3w>hBxq&rWNO##SkqeQK4u zcs5_T5XIg1$0yWhMFl$(J#*u3Rn<{euCe<~1YeIASH`roU-tlzGMrfqBxLw}_>IGF zabExUU0^nu{rmM8!!#Tyd{w`cr}puPj`xci6IkYdK$|r|&20ZffGjAWXrIgl34e%( zT}9Y#RlYrZ{@edrcQL2$C9N+-kx>dViFK=T@jWhE5JrYc+IVsY+a1R~znClq*>6^M zE+uglg3q!az}lV0^OR=AnbImqJpf*2TqGjP&N3f1Oea^paGXOi!aWOoQ^+NHJ;&BF zx;W9lT>NK^mA64_1;?T`jM)R=%f02t&GGo*77Gn3arhbFUiJUT#j)V6K;6+3tAzz@R(TSLV3Oq=WAyU%Wx)H8Ra7Omzk~ z@@7^H9!l_14xU@0D2u+4e%1ep>30)n$0A>Kvry;0>LnC~svHiF9E+wsCd}Wo z^EE9)nA$WM6TkD7X{JJc|mh%D34cjgLgZ`u?O05l50Dk`rW6EQFAZ(bHG&$J9Iub_n zQ^LNjdpSK!xxhd0Dl{Tyrvv8dDfdR&$W?wcqEKmOf+D#Ygu0QLK%yV`wtODO0N?dF zl|tB|rj%v5ViQqfxp11<{-Uw@V!+iGzvwR%WJ#ANt0c_xRdSIj)0rZ?blIunBn0VguG+enEe$hc*~Zu_r(ZM`WY zO|AoiVf{^wfzq%v+}EO)my2x|%Lu+tv63&u;OCSkn=tZ5<@=$-BoP1N$9WE|Ab|id z$+$Vmd9FMvhqzU zw|DHsbO6Fr542?e6ys zf^V}z@0Cz;-|j-eiqnNtCAkManiA$qd*vQv68W~>5yUfpH&M+aJ<`K~Fc_i9XP$ZcrOKd5$z}~G?tMLgLFu=cw|cz_2|ZSn z;n5Vk%)aEOz&B5kh*t=LsqvZ`rh7qsS!1%FJAQ);Ztqw6#vb)3k&YL%hhOnN@_-r0Xz%m}FH&E5QpXhIZOps6r4yx#5?e!0Rfzsc-2~#SFKhk@<(N3;I)02Ji zp6$jDG-wH!;coLH-s;4B_$t0JY1QqAmBln5nz*~&1t(HJp4qJKUzO3H1eLtwKJLc6 z{X_1ejY^fhaQUAYt7m{M7eX!NH_ZS2O>4M#T|4FoYUyjTOH3lUpYjjpyrT)FFq z3~+}W_C^2fwhrUFtuzHo{riAK zj1}TUa3k~}L@x>?L7o$y3Qd$*O$R$TR^CQtxQ?_{Xk1x1ssIYYVXLa8IH1W0 zrPr-qiLC&nY0<7`cw1aA+RYY5WBFyy$X(Sw0ry9`A0?7_kJ6;4#3KIFvxLOXf22_ zYG>3v`uon+yY(mXqq&CaDNzs~UTve{chbGn*&x`S<34S*ALo*WVM-k5N+;mVvX?yv zq(5JpzaZGGQ9bh~M8*=C^!nP1HxLdOW%Pt$YIH!*Bl@6{Qd4D(Mr}I~$T!)dLkn9M%JfffxsKbhj5Z z<@*S>Qshw0PyWjVbh^6ZKR|`Da~o1==CJ7WZNzgg_2i!N)V0Spl9aLB+>X=5mc4i` zxx{tE!J(%AsTRKK3@{pSZZH!eM>7=;DDw-k;xb*6a@dncEW)ln*mTE1d{ug!41L(k zl5f;dMFrSZ3PK$bU8*`$Y0|iWZq(f8bLq2rkl|&)AR=ald4UhKiD1C|S$aBFTv6z( zt0KUCvP?P8Z5(pL0R?+)&mgw=sr+z&`EUVLB|6ZJSFCltzlPAob9YVUB`;kf6{N$@ z9~r94dZ~J*>{f7IG7l^|*b$mPueM+}O8@2A-?AR4_$W}VRZ0L`0bdReXMQ0bp0QVt z4tFb5KiNFjH@m^NS|44n>qKbR7@q5PXr9S-)-Y>G{27Iahbugeht91jrf}f>e;GkhLI#B0hG~ zigMjBp~S(##ZuXlJbUIjI;R)-L)waa!$V09%DR=C325oNg05thYKxx3T8Q|RvA>GG zKSuQ6r!75XxuS{=>9>AOQ3+eoJwc7GeX>VBGN@#rR3!6)h(w4Uclf$4g^GC;?&tz1t^rf&k7=T>iQGYebK zy>r+P+wfLuT-r@%b(VL^YbD{@ZQPvEa*ZWVU;JIU^FC`Qcx<$ixSlb7dmf|fv`EnC zC)x$hrc$`r9onL=@=X*6Q{Op*=Yogyw*6h*w$WS_iL}+Yc(x&!fiBOEF7I{cNr=i7 z41$O2Dz+1b4g$~>3`Ki3^K|hHyn+$sOEp81((^X_J*%NYWSo1mMPy#HzZaaldc-3x zAe4aybz*xVICCJXf~M0)RIwYqD38=Zm(%X(;%ofY%g{WmI^t)v@}IGPqbo|SqFgPz zi+m#6V&lVJV?mEXdJp)R7CN>~7>ABq`J?tziu z>^w>-YPJl`B_z$?w5Jcs#-ItJMW2(dQ5B`Cr#(}n!jF2tf^jsG0k-+KSaeS`2qT4O`+aDFcr@lE6x#W_ z5S5UzJ2#Ky+Rm)rqr1SmQKXL|TifcUU0V2g28JF!R7tQHedeWvDe(0`SJC*?HP^Sh z^PW*&p5K9k`8a=K__~LZnY=-0LDj4(VF+D8SE^|a*H^>#4XMHWz?Kf}=Cb)2{P*Vn D9N2Qg literal 0 HcmV?d00001 diff --git a/python/nano/test/resources/train_image_folder_png/cats/cat.0.png b/python/nano/test/resources/train_image_folder_png/cats/cat.0.png new file mode 100644 index 0000000000000000000000000000000000000000..0a2920d9e89040b4e2312c4b561277c5585b032c GIT binary patch literal 124141 zcmV)AK*Ya^P)00DLg0ssI27tH+*00961Nklei7yL7$`_)Zge;kJ1=N8tT^RK6_nAR%J%G8!+_%a}SS* z%&ZHmNUp^KIkF-=%+1YjFxUhz!T5AnEeeMw{fiW!GNBHhwe9^=oD_l|A~hQ z{^LI$-@Q8=9;S!m7t=E;h>^6GK^;Sw94nT_9MG}}4 zBvBgLNul+#<+6=R2&Bf7AOXo#{aQZ(=xW=u)LywFUrk=r#lM+sQ_HWHtsgTZ1e3ed zlb#eHFePc4CX#hCoOIe`z8nsUba*qLyd4h5X*x{P!CP~0B%L5>NKq6_Y zt|Uo7ZYC?qOf!So!J8)%G_!;vGlWQFA~Wac=S3oVsF~MdTTX4 z+*ku7A-%X=uWmPTX9RG}E1OmQ&6WFpG&g4cHg!{IPZ?mn55q>`in7a%erH6m;Dh7L3`GSjRSr%5uB zCbb-==_YzhED>|>OLRnk&QnTrDJlF;ll*u2F5gI28$VaW zep9j#SVlK5#`~tGFWP+V=|+R^Tkh5qvEQECLZK9W9en-uw@~gcLMXq1-0Sx*=w$1` zM6KwJyD8Gl&>VtD(^y(t&r~1-DR;tsnp%QC{N+ccIq7Vr-&}Ka3PNxfng}xsDM2ZF<`Oygj0jkd%yW;Yc|P~(Aia6B!{fv8csv}M zgN^;cL1fa;CQYMuqfO$L~%^Z$|mbr^3x=H%shaUj=VY23pG?|n_L@q{S4={nG z1!$kW2LCQ!UG%vy$e(MmwQ8rmj`|XPRk$v5)jY;){l0xr8H_h-9jn%=tG}7#B9S6Z zDm=-OVW{?YGtl1-midh>_mUg^{slcy1D-2`xlSPg7%4L|HCQm37-42Mg%O?uKx3mN zC42Y750oM4a0AU6%?I8FP>=|v2!j!V8zfVcLTL3;N;HF{6iBC1-duuPS7ye%^z*Wu zBbSVj5(=TG&Y24!bD!Gr;o#FDl-?SxrJ`p@mnHUsETi+&6#Ag5RyZM*3-1jS*U!&g(CKHEp&&<;AW+ow9>s^PG9^aTe$a0ET&qTo#XxzEI&(_GMX?JT#Jx*4i|A zo0=cInOVd_7-VL0mS2IfUGoao2h=K-!E4AfMtNohrL`h)%e8#4I^C+OJu=4^|Q%x82P)N zerwD9TOaO5ZUFZ$(C@W--I&Zv1W{{^54N22wy$NJucc8Qo0iy+6Hw?__xpDq?}8K|URU zMP~Gd3@cLu_>Rncmp`(6J}k4EE8OY#3W#7~s*n{vQ|%IWZqjkkEwB(f4Ef4mxEkr3 zE0so+%gqV{fG!imZ|6{5hdOlqmm%O^EME@G)PPf2;I~9G184>iAWS4Z74FtDwIm{~ zA}EO_14wUe(t!p6AlVSr_r=THvqCMZzL0G5I`jeF+mq=NdemN@iEOh ztS>#5$cTCAbEH4~K+=t_d|tYi=*lIOnUjbNM#6cV+SJ^p)*c^O=#!FCEIpN(*;>m8 z8O`0?2qLA-dG6;n#k*6Sk2$%V;WhgmkU@oIS}ruh@E(0v?%(BKQ!e_Q>r=j3<(GiV zYqj@=Lv3Jn z(aCoYk2x1`o@>IycF|QUo>%SxKlA>+Ca?AL zx$*Dc+9)x1pyHU1w)^fM3IUBUcC0Zo z008%jbBCm%oIg!S2a~mGN2-TlVqo>zm5pz|CFXvk<+m-{5`MoCz;^rYr-5%IMc4*BE2A)nQvCGV+cAU(oMt3vQvOu93^M?;lKOLG8fBC^$)3VTO3DDJ-0&8F) z9dLOg)+_uyLh`qhmnXAVmCW}$^AWl7BCXH}xZ3ZfzXkD&=A~S{uqFT|#3e{ZDE}%1 z(uVH(EirdoYw%6W@0I($ft35bKKj~!v1v0BYi=S>_i>XSJv5Mn0|4Pxd2Egc&*ZX% zwPebKq{$K@XpNa^s?hj=kipeU%7W#xBa~XhV@Of9-rHu@0H_O;6qVQ3M~6Mi8h|B{ ziXJ*c=cSj@r?-Ro!3gMlK0h9gQv>EX`#HMYXymYDg{ie4j)!;02Xj#Pj~{=S`+1({ zu24kJh|Gw1XpQ$ z@mb@!fn{!gx{HgyX|&DB0NQXg7AnTV*Crez!eK-Px_T;rqyZvK@`1tiy6d?>GDDWz zJ;a1E6J}Bx#$5y5lpr!Q6Dp%#3L(T4+ueQFUYK!C6H2zB2!vI^Ag<8j6rkMZ5W#wuth zIlQU_5E!H>rt*vf18;c|I-kvP3HBZc}TKrkKphp3wu!z^pDUP&8KHY`E;urgslN{BS%_F?oe| zf`CIgpGBJTr@#NbD}kJ*HXSCPEOf}sLmu7TCZihw&Hd@a`yPuKnQG=~W+`d{47Mc- z=>Ea@F29MqhPl7?mi^WNb^p9_oy0^G)V{_wsvMr5-75hUv%^qS(fID?K3|QbN>SE@Iv^_wL|sFejhgZ?S#8sL`fCx;R9=PqorkQmI#5# z01>?dGGvl2%9)7-XqK4)DUk>-E7r(*K8S=UlX@M?tpQzXnWgl5>H2w|m%hvqX>QgG zWQ*3PAlQ7O6PZ9}E*Tj;mB-fJ9Ul&EkB7tLZf2Ao4$aIcb|9P+GwC#}1ZS)L{#|~1d0Dwfyu96h6NZw1 zUeRuCjeQam0E`5s8?W9M2`~D6*E>q_Y{vVw{C3LyM)taU53(1Aeq*-3aZ!Gyj=0h0 zSHr$>3EVkEHv6-_W_4xGl$l8hw}ctYsEjCMd^ioKH{hj{ff5yGQ4-rj$q$iX_d!9?NoiK0lvm zn!F+92RHudFfQ&Wc#-W?VJ8+0Cpxo(v}!B@ooson83oMHO*a6`A;V|n>-icalsPF9^MsM8BLTj z4JoD+LJCQxB)mWV)T4KWdGmvZrqemj*-z%3EvSr$)7i}4xxf42-Fuw_io_!HNI}iq zC!=YeCk*5zSM+c*FEy(*bE@F@+)%LM2^Ky+<8$8&3@* zUeR-}Ubk;Nd%SknFBfxdQ3DK40M=}3aHXf$rK2mzVamhpahO2CwTS_mYu$34!BA!y zE`07ldAxSlkiP&Jcs{oOJ8E=@TT>2bpO!}9#|^862-e?gp(%i(aahX+m5Y%TrB zV@FJ8Khi{NMO+I4yT|go{1)=bKs~=vocoRQb5oSpnEOS!n^_rCyhdOy>THE&Uwe}F z>i9);WrK8H9NYaaueGnAD%+P8=X9;wsf+*ovZOGwVjqjXw@AV~m4E>Ep`1IB^kykm z&RQxJIk_{Fg}G#+s1{Qd2U)XAW>&n7lRO++=A=ndK$(b$h&Y6Mrl%u9f{L>Y>OGMu zCYb~?1aX?rQY>&Hh=g%E9v>eb{|neAKO7E^ zhiPgyISX3IPstb2Co#AR74`!JbuyP<=V%fsmBZgC>BY84@ks zG7>Tx9jf@FO6R(FV}F;oWJFub_PU-xN>CQf!?e6-M4W!tt{d&3e z-^y~wPT}Hm2`-v3BEYS59QJ$RW$gZ~$z!kC_bY(C?%pf$8~y%9`+fhE22qy=@^+)& zH=u4i7c(n<{W6owk`f6cj7XZn8eJwB&@xkognFipz|fuj4$CtAh2ks-iGo)OC@V3D zW$AZ_$`+(3QD{z^8pA!E&8$e=rOXH;Y-;{+`1khMTAQY+5k@t#0gRr@+yihpMWGqV zMuWjNO^(Y1{MJ^|KcigP z>?6wsFmVe|PePfI5t1IyPs{w=fBDDh@BefD`~TkG|J~!+5E*mybJv2t*!+&~I+@gC zOi#o^8k+%_h)mcv>E8JgZ=yy1b>w%m+;>B`+hVI`cS+j-x1s(Pmis+aGB7~q`s>ad zHrtw2bx^zlmAt60Ke~Fj!CvsvByn|Cxi+hUqB#nlY*wY+p){)hcB={+lVqv6iU-ex zTPiI<&|nQy$Kq+d8=RTSQBhS=LOJ}bqn--^l#dsJ!waV=)Mtmuk#Q)`DqvmYNHe|Y@iQOogo^hTtdYR%nX$__QqQZzJU+36&a<9ON?BzF~);o;%o z@UZ;!aIn@|EDOM+n;R7AQzOz;^mBvQ24s~51OQQ4xDq5)$+z3_vRpnL7{S{c|D<1jk+YW-?pqLd? zs2MlEKIZ-!Nay%yCJpSe3_~XOz(0menP9qxNQe+}S%pkMR$7!MlJ+Gt=Pzj(fD)fo;LW>*( zu|#^Kk#i=Z=X}=rti!>2SHfwuZr$mQ?1nU0%18%Be(b-ps`z?RihTXJ85!Itdx&Br z3g&|7pkaymw49&jJTKKcg~LOos(A(w^qR*L%!M&~bf|+_n`$r7%GH#5*^>V*UsJ9; ztMz%nGLY>f4>#lQvz7aOZsBI+mfWYFYR$4)E5nh5yQ}hWvZfkTBzr_`{fxzaL}pIY zv_Hz;qQ`Dty&2mLF0`L5Zq)j|<#qcNJ6g}Le!xmgzZ3a)n!&P?i4y8a zbE3EyT@umg$5wN4EDu7{2_cZ#-K+|kO3FfJh<@~Wj`O+4vH+O; z@Xq}}9uCKLJU-Bl4UgtWvN@gRkM95|`oT>`M2Kl%%u6A<#9%3|`E0%2<7RPi7;7?H zvMNa-gyjRyhxZT7V|v%7gn{npi;PB+la^WLY#7&=hN>Rs=8j8%c7iFi4w~>{`S}B=jZ4;pL@>P4n|WC^>DU>PX})k+{M<@ zdeGv;PW1y92V4%(MXB!24s)*&e~OK@m)FkjbNk8MFy8CGuhirEP9M9H=Kf7!nd_%kyK~zHtz}t8 zw134H?xBX=vwnljym6pUwA??jp-O{rGkF3g zMFh;?5W!?s%<{o2pq355vMfvQ>a`Xqi+_EHtG|^}h;;C3iB8JQW>ZW?_|)iZkL~co zbbRN2ASU5}JOW3<1hj-RPL+mUMFEO9C#?W9qRIt0m_TKPdJ5jz>n+TfLfc}AAeMjCuA{4|uDPcWzR#r&ShC*T4GiJrPFBy@U zpf&oDt(k+q*mOXf;0O97H`)Pgf=}p^7UqJOVUsY)Q;a)A_xo-74a9CgT+DgiwfTCO z`)g=SX6%&v#^tSLS!g5gbg#aB1l4j~?B8nk8+?2_^ozLAYa6%SKiUZk{lA2ft|p3B z|0>soOU$qCcOW&D!xCiDr&wbiz6UT0l3GBLzRt%-!GJ8Qs_=9|L_(pyMDMwDN$&24<21E4kvaMCV6AzZ&|L1;CR!ui(M=t(pm$gY zgA{$W*RN*ZFJvtLE}te>nw{h*(+cdNgZ&-&CK;-G(|a?(w?mf+9}xzSbYJ5x006JS zFMFMSf4CdfjF&X{+vfip?Ijs^5nn0T>R%Th|{eLep*fP@pq$< z*);5nO8hnBef#M~zwcY#Sm=8te)HkA_G8NmagCp}k@64gTqDtGXUP!QDJU~mL9F7d zXqf;~ZtCe}XPHd!ln`z4g6PnVM1`z>|WCBt13YfJr7z+fN00S|in_)6tCqpLYx_Vi`m@5jIkRnC6TLVQJ zMP)yo-7F6+XSYU}-~|XrVj&EiI2?Dr%N_VGugkb2S58e;a)SnA36#h{b}nZvPu}Or zf)!%hA)`T5K&(s&1E8)hOoSHX0wgHo8m&5zLil%U{kM~gW?$KFc>8N!2;ja|DCB0d zm(twA)hl$JRBLU`ce_&7^E_{u%-Z+@(d-rR{&0I+`^NNs1FpFl_G|4gXMs^Vl>=tq zc$%TNTymhL^-@8B^xY49^TsF;lbG47UQh)RkDp>95X^3 z>}wT1KZ0qDDnceA|BFqvJeNd)N-7DowmO|?qKkv3%g_4)B;^wpV56wW?wZwsM}gkW zRvdwl0yC>humqOMAf-5;x?!H&TJuITu^EhtMJaJ&UrbI(+ zX+bP~KAlfb)AKL=`Imlv&wge`_UuYE_K9XlHOuChh#r{Wo~;>)_5fwMd2Tp=uHp&4 z*YNvIs;Ga< zUIT7af+mW}$_p>;UMmb6)1f`i2^0pq-Xl6qG$m6;5pd>E&v=Zf&cZa@-F5!;*6v-l zWg$eOQebn_?~o8i1CW^!0k2>Nl3JFTq_;+=yCx%>(Oi8=H^f3VFv$dyd?j7xySySR zCX;413cK{qSaLai`0&g7pMQRM|F`F#f0{phP;`T`!w3aVNIHCJewaKDEhbw$+?qX3 zfK88x4hf2JPwhL;`)@4wfXv-b*M{$JQA2g7L$z7%W|rDm$hQ#5D=21fyx(;D8j}7| z3;jiT4b{QLPJj78x@2>_**{<$G>zIqZfZGmZ~z{WUd%6oNltgLQ6V^@X__V-g0$yY zDz>Kd^l)&p77axMx?OjX^kv6YYDP16!Z2$A5Qt&Wn|jmKm7W&JWF}3@G-~w}ewWV{GgH;t zjli2YcFksdKhxcX5UQE;09LD89csurh2JPHPn7G$4 z!gmbq4<)ZE_x&rnmwhw~c6RI5p4Xw`(!v#F2awr&pCv_vySH}Oo6jVB%xfpitORxJ zIvfyUHNY;u^=7m;!?-z-{jYCp|6t=9Fndv{E6|=J%M$Bw5IHbtCP{N6L8z)-ThY5G zjAZ2)c>YKCbGC+@joq<80p^l(ilHS58q(Ok^_j=>2Pp=|LOMA7(GeyXj7Arkv?R&* zj~fP;j+F<2plEE)$&~1W!msE>%O_<4?fWt4+7QK-aONe4VY5TXjW!cr9Aa|YZT zjiAY!8Dv-hKb=k=7QKJeUmo#q6T$+%z#iBpv*R*|2kZynqvaNta$z~P%N?sq7(*ZJ54;|1 z#L2M;sn%Sdeg7b-twzRnjss6TnZu=K^e65okvz@PK7U^a4a{xYWCU4nnKq``;MedU_ zJzWV;xnpvvu^kShhH8u|BCwiV)=Xqr%|oj44E1!UNQZJXE{oqV%9Vi7M&jmxq+Z); z&hjDmMa5*ZX36Fj#y-xTn?b(;abhUz%~9zxGju7Iettf|TAH_q$Af$H=$zf%%4Og} z8qA;oeDD$gCW~uLVIRry`BO;IC=XBeYV;@*>Yj;FbXSsK=I*T-vLPd5l-qCv9cF+N z=A?@Tpk7#C5(s7bTHW#G#J%72q`%9{vgy^&R_-^r_O|h5zQt4sEZ#80s1Kn0~47fV`~6zR>N zT>HQX&jH%QLqx=~bO1S!?!sVH#rwd79AITQjfGBGDm4|T^fqB+R@L@hu;=Q8a9XlR zEs2asQpqaXmt5%>+$X8HDFaZF>x40;fg(Pu`wUrHN6)4ANG(0*lpi0Ck3Y(a^dN!@ zC@9S*1C&V$)<$

y$sUACDKaqQX=;z!S^N%!q{<^@_=OIJVSemXMgymqiiH;7*b& zck@(k_JsA`=N`5!@GvN=6j|m=4!rO3_2mkUxCC#!s>=4d_e(7IZD(8uFYdQ*huNZE zO5B|*`n)g4v1!J8v;9RJe195mjPriJ`RJ+L>h}aI#lKDpzzBszrKA?ry^UwYiQBsJ zT7k&TF?*YHX|(1lhbWCe_K-_RQ_=7df+i@t3GJA@_ZbLD2&585(W|O;IW}N7g)amH z)956sP@sXUAzzX5RZL_A+!WQEnP`$bx8`P+FcpW9u;z8V?BjmhY=8Cb`)=)jW%(5j z)sI^4;|QyYEZLl*W>zc2G9@+@z)O@Q_kHeV-}x7e_OmSa8(60J35Susn&DfUvfpnk zdai6=Ox*Zm_x64=WiUA=63S?5rf!tU(wwWC($$iHA0OyFL5T~N;@1q0dYcpyqh1|B z3gI=v5vmikCeo0S(J0B3G9ktsG>9()KtP^MlL2PURI7g*#4?uHlA2U8kE^l1qi&W6 zhmsX))JP`|K#;Q{`*ME&{(YNlI(Wj|QL&&jle5T%xzp|}Ps51S*)W+ut?^!2sR;wh z8V6H0(!)TGtYb-ZYpoG8n+S6nQm$6b4u=B(7~y7(Hnrh?w`OfJYcOMTy2A{%Ru;Vs zJ0ST{}vCPXF&5StBVK+-6!#AmdGSe98aK~qSf$~++vrQf}>&J(HN zpqME-ger_a5?L?jdNucRi{Em#t^_@ZDRWNgyqrFKfZOrMzogTV@>bJ_Gp!v^K80eW z_d@pVWvIOf)Ba3S{>+*~*DT?LvJ#XCWQMXM5`dZYWr7IK^D;Xp#m7$sF~kDOwe^ zBbk%{rm$MyAt9Lp6p)8(@~rIF1dv9GV0BY0fMxMN?V7i%xwbz85D|4Wpe{8 z6n&Zd2mBH}C4V?rb4f`c|!G z!dPemG&41tnNJf?%H6Fwn_F|5HTQNnnosZsI$+4`2yb>ZiXq>f+rPhTD(+iF%WTSY zrMSGS23o&6RUpO_rTR8DM%-wD+7&*Y8*YMnq0_u5Yyd$t zB2?N}$TH|DAwdbsvbQqICYi5$cLBu)wyo8e0 zWJCtQOk^S+&s-MEsAYyBS*%rRP*z9?YOdT*s+)e-pHa)#vaa-d-7FwoQg%j1%<9P= z*+UXj?sO;8+`ZyxA#-;#hf$=piOtR3+|A5egBjdKFB3;%WEQp1?O49+_TO6wFMJfE zkk)D%Us}A0>1?jfdns+c_n@B7=RJ~cW`*wc-Z#a)_nr2>Y_8XP&d5)Z8#R6Z_2#Dm zq1>JRJ;a34peUkSF+B2|m?VI4HD`0o4pea* z$;g&O9v+Xi0CH=`O7}f8++*q4l@Um#(Xi!A$;_PY?rvad^AN$HlrSR3_-mC?DjpJH z%mmG;JT0>cU{Z3k;L5}<(FCKpH7k0tFlw5dsS1&*5_{3UIZGo5?p_aebI{!7;q2w~_Z;qTbifoT+et|?6 zi&%&zq$F12Saiz}gzV90#bPPYdXh3yCJ4$dGm*?p_vYpHxlF+QvPA5F$MuSUDuG74i}1-XAo|D)d*F!xP6*5WvO-?o3l zIQKp3y~*}#O)A@yU3>#uh~$e|srUWA@x;7siQoVFT6^0fw{4&_Bj{x-*0I}NYfj}w zf9K$Nx0QBgkIYnqa)OB%+^vQbZ8FGMIYcaS0W&&CEDi%YXh`~mR#$AXCC(X{P>$lfzO4^Hg9vqb^e6{>UK9A3L<| z0BnZCc085@_MXEfUZ zkiO=@BvPi3k{Pyi#8NAvRzXIDna$mpycE&~O!T$Z%H=(kik^;s5FO@~6(<`z*$c8PyagEXk+&zF&>9VXJ0I*lt3`)y8r%VNkaV+VDnQ184$Ke( z9W*x!biiFfnwhh;h$V5ZY?WSglY`h11~5gWx(z@UkVZhd1X@zSTCJ6;%1>5qR3lZ4 z@c>1bsxriYql`|n;^n0*sxV@tr?ZmjYIdr4nz0uR2&uR-I|0-Aw4C*`H69)h?eU0Y z&&a-vq(Et{Ia5`+*O-m%j&r2({HcVpssv-@Rn0_Ss2341R zd-b*tk;KL#Ikfpb@=jHjkXG_VRC#C^At#!f(rHXHPY)`kB#M|pPJo_fF~%48&pImeqprCjpe>aSGHy)TeJ*Nu~nl% zsq)Qi$Ld-A#4+5Q%*{dp*Gm891Mc+uTkUbxtn@o@!CP*Um_|zKR{aix=iOFs(`LVy zI7C%NCZtr=@&Je?b9oaJ44Q*RbMF`-Fr&;~jZ=~xFadD)T>7%aJQx3a!-!^(iByX9 z0jU(_$-;)ZhGJK_e*sH|sllWakpN{vM)4-(R#z4zSQ$tou7y(WaTuDZM(S0?$Es%F zTCm-m2$v$`G=KQ}-_7vv{4f0B2QX-c43NkGokeT2Uf%K3pH)_RltdCK^i*FEiHPlv`TbA%{2ZsJ`ShH+ZNgym$}LvFB;HFKM6ysfpG zx{f-%r22iA-%)-YEb}7zvA5i}Ety%$9V`6e?zR2#@e#Xuru?e_rh{)A2a3H*bh@{) zKZ?w}ZRy{%`IPpXet+#)x(<=#Fx_7khZQ)Yq|p)*n1rB8{xlk_p{uFE;5}6-pWQMb z%SMZ0g)}p>-Xa#~+{?758jwX1HR@(70fo{)15=}DDzr{#U?5Bg$byT7Lq=0ZLRKOE zU`G}iJi(Tfze1s7D%hv;T4x{CoGs0{#^xE%(I;41dy0Pk<$Y4-{7WYOttnWEGnok! zvtlGo0ozToCj-p04FKvi-A)+97$_#K7qy2|8>Uy-9rC%@|EM<_gjW%2EZDPQ`Y*% zBy-bvBsV!O`1>`~^OiEbgIQh#&~KFZ{keR4`vTKP1mv3Bq+NtNa+EjUEdsC^?-xaA zm8Ha$XpvzfHeIX|9NxgR1SdBgV=y>$OAja*X6{N;nij@3Y?3y${sf7#Kq%Kz>RD+> zlz_}!rkeCkEXi~<4-a8#^#v zD}*yQ9!P*iPLqJGr&G>O4lD`l-P5QBSg1-~ekoUTM5G{RLs5gLOeRg&7jDg6EoL(e z?lQ>e#RJ}2Gb1Rt0oD!uJb%bMW<*d7FaQI?ERkt<%H_`~+&uazq>y5YspUM+&rhch zPfzbZ^yhAYDtp=$nP@~ag+->A=t?>mV3I+TPl$@3nRkZxVkM2_#VxhVanU?I50W`PC zr-qceV(Agl{_8``%RJ9L7!k;b?1~h#6{Rj1MH?6kB9H!;T67CjGMN$sNZJw1O1K3O z92M8vd@?i2z_i~0}f%lZ8Y9q*3s=8STo z&vQ@#hCAVEleGhRARSncos&149^NfJ9j0S@aGN?us-AJOXq2tlQyI)|p3IgnyHxoK z54+H-i`_Q6{qREVp-&fH^}w}TZT)ktf?VI;o^0;mXy?sNn?kADCX{O49qrM#4#thu z8fY|zHMm8pQCyvR?ym{!MT8ZHew6~~JBMETL`!oIR=nNjeq;TOh-F!NA1vfkr90Q< zxE26g_EO>1ZyCs+ForjN{R|WP)B0Z%Sh?R;bK^b&^6jq1b>5*jTg&6|v9$>E@KlfN zN-n`55yJ3}1`=U%5fi-FqpTxhi3LIoLior`t~6t6Y;s|(1Ws}VJP5d1MIYu$>SFac zN@JO|5S{`P>cCo)l`B>8dJ4JXUF)*Vw;NrpZZ5??{`U7DQ;&WG4=@#@f}3@%1e^-P zYg0T*v;S*L869J3ge-4<)lQh_IZkJtd!LhmGoJgX!e06#eAp<+%q8JL5uTTd$FP;r z&y8k5C%stc?&NF6EY;$0`*L-%-{p0=g=JpEvTwBc4f>3ijE?(BV*fcaOS!vyf$!U1 zdst>QkZ+xw1+cDK@73V+o71_Uy54SoFJI6wz}L7FkmNQ_L<%Aj+`(Cn!HS(&bpG3bLU zQFeW^pMQCO@acH`!PtNXpFxMwo25c&mX=I28(cS+u_bQfIsO$RCfDk?R`qQKDIhsS z@3-B_9AIQ1tuZ~fdQ$JbQ(}rRcm(|#0*L_ z2h1ii3`j=64B6$~%s^%~Dr{*+qGp0zddD2mr3&mMXj(1zf@Ug3)r7tzMqXp?$M4oS z6bzwEnlr%`vA_@phm!@{5zOX??ZMcx9{`xOaopV#1>itOkL8yq>+pAOJOCaFe`htX z$c$93q{{Xx)%(Da{?&wuNM^9OL9?redL@ydna!0{I-VXRfhxFRFu)w{@K%X!V|qxU z0^7_dD;4h!pJ2uCKK$f6*%ODoxfs3QzNG~H(kI0pB-(U0>)!+Wtdhy<{ z+;4W=cW7n>#Z1$5I2^VLO|KmKb}qTG+_&-I6DFkB&gH)Sy?n}qch!Fing7Xd^V>QRl*C( zow_0#`&SZ0P*GW@=)EuJIG^MEyv!#P+}S3Oa3{Mo6gSN)6I$Nbnz`Geb5UP#!Zb}j z9osaOif3~aEfH4IwgK?&YOfbw_66g=eyz5@B)$_6NuQ{cKdRzCv42&~PXWbYUizIXKSC-1 zUtf^icBcRtY}N0psh;&h=|UWDcF>?uuh50oe8LpbXZ4^X7G@gyVh!l#pd})d$>L>) z+cZtF^jNZco9Cs^rQZc0GcISRVuq=hdy{bmWVVP(Bo+-PkVH&0HUj~=naqsd%&LkZk2KTlDx4v*0twsr{!O;-2e5KUSC)C*SWo@ zuwHEaiI)4m@ush_rl*oD@O`8F-M7buv_M-r>&<{}t@YjZ8zsIU>b(;GM9Y1D+)wEr zfD|OlNCzPVH8M$!l{N2!fRWsla7yIR?|{(^82VkxONH~plxdQZuww6Wg7E19y~Ba# z7Cpk`)?F#2a=gQxMeTO2ej4>hnVMiIrn8*5ZSeiJ5tp@R?NKoblW|z)rf%e zl9@qCislq3Bb%{Jta?9YX53-`rs-Idecg+(SMH*8)`!Nnqx=&~2C^f%mO0MnemeEj^ZemyK0O)r_;_rh zmSCF6GE)cn7MdoRqqXMFB`l$!Y_h}Si~Z+xl-_FO#u7 z9vAam5(%bdrdl62ru?a1@H*9%lmk52;c)PFpu0$TlX-$w(kUS$Hn;zsj(pW}e_L)~ z8SD!609WQZ{a@t9eIvP5?k~y>iQ{(ZZH?~O`YM^GX%E8O9BI?(+m@%Nr+xoxSUp$k zUK^Oh5JzQg05CHS!ozZ)*^P$%sQUiK?Td%jY&?^f#Qcg@lPXb&Bkp?(8IsjT14WBi zfB|$mJy)9eEtYiXSV@D&c`?YQ!_?f>BEtJ2GxNh)B;-QMEZMUYiS{t5E4z$#baR{N zzAVeXHShC$KA(HhKwQ7k&Dk6i=|%c99UdOqbhPPU)1-FL)I{c0jl&#{ zk72*R3cuXjSoNX&aLR!SWR9XRtu>DBfa2J}_z$^Lv_Q0`x5t#6wB=0jG%+w0mlO{hr0 z8{g6jnIU>7NmI?gMuRye^fYs$P0iIa-1{79y`g0nmaZ;!nB;J{gb>{2IbN1NXP4w` z38C~mDXFS(*hGmm24z$;r;%nMG}vIqtRM^+1exz1gB))kTLtl9WW#BDu{jT#FgKZj zOZLO0+VRX=!Rx&?ycF>e5zs{mE2fV5u?VJq!%63 zZ&19oy)!AezE<&*^ActC-t8zEMovl zty8M7nSg9%k<7g!D{4ppm5KylWsM*4Gh8iw-q71_F=^%?L8Y-YV_Ed}AWbp#rcm~< zWx=WEytqxabeO`DCNoXmxR{xIqBUzv>+@20y?SzDB6G_#sHK%8xL`q4BdVH9a*(fXDXQK#i|db($FjDeMCPR{ouBh zD8P92oNmowhRJw77o`oNrbwm~uz7){+f#DT} zIhIK}v}5Z&9ekc`ncMk%26L&mql#;%NFO0TLoe>;ib##y@8YtkUf=Jqb&;$y2XAJtC#w{`!LzGk_9BYk>1 zrkm5eK5<@u_$so$P%k7g-8cFa$sMiC-c-N8h;Mt4DKj_6&UiOS<8Qr((LhD-$OUe#wF8uv z*-w2wO!JxUz~o+z>Qn~JBr-dE8Y?8-yZ-(_vVUJbtGy&6q^uzK=yPAreVKD9%|0(n zA2Xe`3ec6MUIC0B7S?EuRyFo8$UrVd8FM6?Fv_xd?J^hFH||XA zH_vfJaz6R8)Q7zh`H7eGtIJ+@W2ex!_=(GwZzflvkiX_y+?d`sqrW)^y)+fqmLjw* z#j$2Ti{$Zm+}Q|u-_<87aa`;18>M`+<&Cee^-rll%=&$`-(^x_3xN-kML|=HV`J6G z%Iz2H16jnu>mh13*sd3f;lq?k>&ucg1`=@h!Q~KnmI+lzi6G1YcSm2G4Ls19ZoT*H z%AVOnmhJ?IgBLVpW_V5@7YV&yye%%)6SSfbO!myy8B-AoOK@p80RaFOH`r)EwMDtavE`SfA(W^{N3<8>)AQD!L=NUsX7cMt6E9CY!T z>GSD3j`<)1*&~)^IrZ~temwOjl}xo zl-*%wGLl}-B9+q0YuC@KS-~$1{zmAw87{bkW$-ylY2R`Sy#B;}zlH24?3>T~4>d`C z3*+NeZkf;RXe+3M>#KY-OTRXwZVfi&UYgzOA-u8E@AkdXzqo!%Zj|yH?O)XRU(o+b z*gG79WNK7mx>RlgqJmf|XMWS~HRW@S`z%BJGC6H8QiT`pDtHSkGXZJIvbK^&Ny-E! z1L%_-)Dqp?E#2bax%57tmswbF2+Pa}weZYz?`Ag5OKYvq%lX`vWkE+~MpP`LRTw2C zK%N%L2((&6Xy8s2A%qbJLm(-dK}1E@jm0=@5NxhDIw5b~+<+9a5(8*TWS`IZ)FbkQvP2F1YyREfwS+TaD4WO*JT6eE)9XhLAm$U&Z+L@^LQogkP z%v9UQ3jW%miW^vF)$(uXrTtIb5Z&#q+4Sl5=QosVZGHvIe8KGgw(PIN3mK~S_UWfc zB_(b4`>i@&Z|Bxv+y3JC%&=MRRlcKQq(+0g*YkeBZ=b?PPw=A?-fKN<7jV0w1T{!~ zu?1+1W+RmR615>+wU=~vkW%h8K>Af%wKhrWKnRi|l$kPmD@GV2QU!+PFqvsIL+d7= z=P^6v!^5Fs95aDP_hvSSx8pJAxx1gw=kqd$^>TiOcDGWf3f-Q}b^G<`wUuHUtP2<; zTW2qBzYR@CGdoTINJVxUEXD#WV&-f%KTJKmGF6qTiiU40nm?v|?t4DkM@B^NeOb;W zr_(8(Pf+wZ=Z;9Ag==;tyuoHCrG^pQl4iihV_Me4YP?@`eI1b)W!zsHswrOuWwsE{ zomB6I#VfBA!i`b&DRQIShyJ-N8Gmchn@IXs*Xb|HE$05B+8rij=`29HTt2Imh70IX zJq%3_3`>)P<%#!`@Qu39jMY#O7%QRdbz|7gjyDZ(ce7vIaPnM#R%7hsqR0O5h+5rL(z+$Uwny;jKeD3S_+B37)ciYDc*NspM z3<(?yoy4X_>ysq_rz2zl+d79XKF-#8;5@TCEx%F|14hE1- zYZBjfLAJl!hjpb=Yx-sWu)Yhn=?7><=*IuP)s={JT;AyI^RP#`4;GhWUP&biSg29v2TNNenVN|xNSjHb@i5IQ3Nt$>HU0Op6By<=?kI* zLLy^$vr!I?u5MMEchoIZWm<4c8_aAgVP{EQ$#LA(B7S4;yL!~iG^xM5=+Hf^w3+ZL zgkl>e`Ry*}?<%)0if?4de`V!ApezE<{M%=A$ z?p5|1{k(GAApKHUH3j45O067M+|gui{O?le7DrCQEyv1jbp@^J&8&2&xg!Ps`{Vh{ zHl2Ouvc%H4WHJtI%AWKk+}*nwPzd-B554#KtiFVVWaX?C+yo6h&h0f$iMcug1VBT- zZx?WH@s(!`Ba;Rsz~ZJj91e@Dr(&6PKA|0bI-)g7gZSaXo?ixhy8;b9D;@B65$^dC zq+w%@uVZdZ_0!zvC6*p@oSsgn56|;+UuI~55(2`C>I)=Hh+JggWE~?%!hj?*(=ruV zIFCof(%k}UWHZ+Zm|R`o*Rp@WuHLxuv+aM~B$Xu1nsz4t{xjiwBk!MVO1_g}>2K2; z;@i0ZzM67>8yLBfy*7MNx!=5Uzy1yUm4^I{65r2EubJfc{p_FkeFFf+|1Q;EN&x#-pjn-UNnYsj{s3K~rt5puy=}ecn8SDlFR24HH z3s?eUf$q(bRM(Q3z56mFJ0xV#92sOo2I!YdGUT6!DQr#n;u?E+i8GJP9=-SFJZtId z(dT73Ez4Jaz2UP#OM@JF3|e~oAZWX9QT&*bjm9QMbT&t~7ma^HX0 zKNfr4^eJ%7jdlN7vNxY^K73aDdo%nsrN2(aZ}oG3YWwTM3|Y|+)SDAUKTu=_EY+{! zg_H#B_}{C<-nD+my}`a6Ykk?q_cg!RXz&S-c6c}*dYqq6czUwuGg5G)!HjaVX07?u z`=Y1TMp31hqtDU%yqsquCv#11a;GdEKJeCY1$aG}z2&|_06+ofF)*2N?hXLcHaY zEBB56fgI@wt7g}>Wjn~PxT=2?xxDzR&8|xb&8pc)U>$I6?R>6$P38WIG5)gS`!&n` z{^7XMhMy(B(qO;Yeq(y?QJNQ({!I`2uRBmO_W97vD5gis20i2;s8MC-k8Ts>;E!CR zA=6}c`h9oA>+8p&Tvi2{2sTiU%%mAPfy5z>)?nUSn@?xVT1sI;1f0g!JPy5^H*3w7 zb6-x&`FW0*d++85gWLc(41;R{b+Y8T+IC)XlI%1)fD0W}c_AuHh?NdYcb|X`6{k7- znaOezAdO~_L&cgPVCAE~T#LLV>)G9)7q@?F**L8B+v*<^lmS&S^WM+%e17gv&!?xS z)B6wSr{|a>7e+^}uuQ_x0HX!bCCUh^q~H`Kgh;BY>cY$eEm7sxIWE^~2LPxGHZJBi zzHPF+lpVM;U-pnoeJGHm7k|3h zzAqNBVQpU^_ZBzad}BY_{$7diQJNPm_j~*O#p9AGVdKX(WWu(Drj4NHZk%P;5BnX! zjeZ}O@r_4itPL@AWp%*@z$mMPrz0oGTcdk47ck=)o#Nn;YC76tbobV#sn4yMS$IRO z0?f>sSsUDYB&;}oIXt6IP=g5h+Gbw~3D$LwsvOAbfbAXTq`4_m$WT9XacD9al*}mC z10^b;#)6IiX-e(s@|Ve#>@YJkmROeMJooc_djItF{+IVZ{e1f6DNl1m=7M6V#^tjgvz(z((`juA-`BGc07H}KZ}y6#u2+Y`&N6kC&_ zBmgmrb!~clo76W3sedia&MH;|teSn*40WUT^{i@D_2txd-wKx5tT0%zWKjFfp0YUlqu>(}u{a~<4vg?5)p5;JmD?!#szvv;-{wHX8jV@7X=jtV+04q*kd>T@Sx`>Kye!X8^V5gt_rJ`~C(d&$>K*e^X}OXdkQtGgBMEHq zZ)Soi7W85eWtpR3B8stXty#Y_-c_0@1z(|<4S8SXUbAn{|2}JUGlq6%+`2}W6{p57 zSo~i^#qU(mjzMqe@$0X@T3%J|n|j?_&?L1>M$j()ecyiH!=7Gw|KVn_w*7rETILO< zeDn0&kN@WH+wEUG+(#joH^z3~@@7rI!Sy)o6t3j7dmrh_M-jV8j<#0gBLJ(~r1iQD z{C%AD1><{5VyI3m>J%_)YfwlxVmbh_clg11JRaxs^XYWz=Tk(u!`Pb9jz=ttL-@4B zA@ZT42U4LPux94X4MZBu;2dmyWkyEwus(|1{Ub30fR%MW$b+-MBL`^K5=kL@_QjWz z`a3qQNnVNQf(c@`EY6(q#71d9+9%!0kIL&SvlljA-eT?K5)6Yx2rN}gq*4tbDf`mr zvzG28g}%&ADl+Ef{B+U+1`=qFzQE{iPDk39$c=~q2{X9Ej6kIUP*oLc%^V!nQ_a?~ zB_rW7775XnE2n&_rG5KqlJB4MO{Z@L`)=pWgc<<^#1>MqOX=7?-4*Gq4u?^QyZ8OY zyYee!2R9|~*__etn-;PalOf=#SRehovTlBbe5*M3e-`;%Q2w>OJ_U2<9W;O0j-Uv+ z3~P*ITEXrdxeKckb(16pK@Bnll6(Apyd)PY!Mn43%RJaE4>gK=C+IvJ8X#j-Y+diI zS*Fyb00LTaI)s+~^z##1GP={;h{*~ZIb0IBD&kWlb2ryr4`aabR|s^a*XAH3lgbed z=7Jz{Moch68W`}w`HPfUF}J-Hqy4+fN;ieovb(t223k`5a-floHfKPY>M1C}got!7 zK^hG*nS&NJwH3$yxcuzv^*_9U8P@lx59`=a8+JW#l>+FxcU)R50 zJg6^d_J1PLn;ELBtVNAph?e;Z#`~X6x!+*x_%!|gBH;dFKd*hgKakca#)7ATGMBxG z!z2R50uK}lEwK!Ga!?PT0%T4hruVIZf+2PhcfN-B{(mWZKYi*`zxf{PNMnwvOarrjOkiqBM5DxpN0EC|1Z;`~k_F`pob3t@Tz4<3g1+T>%*ZPj_q)0 z(}XPar4YWFL=7}?N^7tKYytr`gPBYs;YK%W?o+eY*c@&&8?^!M-OqYKsoVC`Dlb0v zZ;5Qz@4tv)v>Lqo4a}fHz!L*&HdA5f zOy`9ab$xS~tVE@OPH2sbFZ-Rhuf$8QhH69w^CiPgBCFf9t z=8W@_z0;&{^qC1r=yjq|kFF}%xngIysrcelxp-W$6(?D@zlLiuqjF=H`DA8xSdb01 z;K&J{ki@7Eq@qOl!%G4h_!!q5(UdZwKxAej@^EO!<8*j9w4+62cGR7x+z6{0N>f^c zAK()#oKCvYMI&9M+_eTW2?!`r#1k()427`2o-}ZCn?CY>-74;X9nH@5171_~+_t=! z#Pn;m{GU#_m)tAsH!SzpV3rsAxe0~b_i)qg_^>m0xJF%5+Q9eH??5q+VF1mxx5EYI zs9!g%_>#36$ae6;OjU_KrmF?ki)CS0W0fTH()i@aecp5M;slJ85By*^QpBq#Stx#1iZ)! zt1(9K*)C`5k0w{F1HkGfS49m~?oUKUmJV6TJCqj?s@%mxaJkecx5~XfMlACTB{iGie5FUdJ$kIjDpZb`MW+xB#WBn zYLK61vv1$lfnmg)RJrL)z+{6O0p?lANJ2FF1ZJbp0kj1XJ~c2yE`}w{P2QMjvm}9I zp<*UF%n{MI^SqB_LIpvt`hEXn)dqzklg!B8BNh|`ut^F-mDmoGRY)bdRv!Nvuh7?( zmEHf&8i-V;A|paek7Zs;KDPDr8p7>NYIS5G?yCS@CI*S<&zn0gbt|+Ths$H z40cI?h_WF*cbW5<4%NN9VWj_2C*xP-#&W;;2`CG$RR1nHvE(Z~mVY|s{)sruNB8EYEUzpUZ6+uZN8ekcpDc&7&QvYFE;u6387i2zAD zov=Kl(1VdZx@0gh2{#Ai@|@hG9%`DCG7}B7OL6R%g>Q0t} zY?J)Ms*Be`;UuZz)iWbvaLYVDpUzL`e$MRMYD~~H0{u^Du_e zHv{tv&F!x(|D1r#i*j$p{|3?`svhP=-riwbXYJdQtbVjczHjZ7yEwN`JH%I!{YAXN z%HP0A?t#MlxZ0Zr+rIVJFBr%g`@i1{*o@HO7W2_YBgjlv%y;TJs?$d)FAd1z;mXz* z#fphsBV2eH9?8^oAnpo^a2wCHz4XcpmF55{+Hq7o9kG^K`XER7to%wzp@MPn9=dy_Yt=+TPYPT-_&F0z=JW(@Cs>{BQ3#&5Dq?3ldGAI#DVrGJHj8>{vvbO z5pW5K1Urb_uPrpvH4K@$vN{JE_WFCYI`AthcI12{GgE+7GV>q-9zK|RpazgxaTu>x z5v#)rIKFt?*NmN&wUGccV$T4CRuZ0-cYbhLSX7~r;T~J-zh^KKoe(KNPAC%@TY~QD zN75Q9ztsr4!R5|aU$wy>g>3MSguwLN(zsOWgjdr+)m~XcK zUb2U?FE9MB0A4;Fi~qzS-CV2t{49 zj77;l3jUGS+Tjp-e1x5YAZK)13YmhYX;OwFiH<%_QzM~i4?AMJICx39YcqbX!A+(z z0ue}|E>83`8V=j8>xVyFJ=J@uv%4wrtsSxAGDq(jp-g0ArCz5tGM@D;m4IK&+p4|@ zT&}^U(3{ys#aD8v*?u0lHzvNzHsSvkFBxxeTnd}I6XC@(rNzpNA7O<=!% zU~lz*_0nGLOYOoRFFv+vbm4!$gp=$n(ygrauRn#4$gX6Mn-y(0-5(y0a*Hw;tz!wu z6G=J)Kw4T5E0y;e^EXD1`%j-LiijoqYPdoRWcG-8u?S=!sqEw-)gePl9$rj7zqdv5hHR!W z*EIi#Q17wyWmz;Y^E}Vll}Zc1h$iKjSN?#Km*=D~nVPgRlaZOJ z9=-QidhdN%7A&#MTITEv0E?c0iV;Q$sVE1y&8C^Tf$R|?)%>B!;OXh+VZae5WjR+i&{n_cx418{7C0WKmwn{=_XT}MpHbDCYoml%pr|dw zc()2Lh%3t9m63jU9e8ciZ&>_J{ncMLFqM7LGBX~<&S|R4g7k`fFLsqbGUi^Ey9rqj zKrvhH&RSAc*jVOVmYmP%9m*aZQWj$FP7is5d2uGDvUgKUSOg47v;dnd0d{NAd$$Pd zQJUTQk`W{=6HFL|f0q2-e;v`@tF&3}uPHfx0_OgjuKX%G^ef6Y8p+q~_O;7+^LcZe z8kpAw;EP|3#VEFh%@qGTif^Ra7HRp8|2;oDA>#ASIwFiUq^$C8%{P@qE*j1DuPYM%&5q)c<9&K>=#Y<3!VFAfvs=pM(j0uN#!SpA-smsSh#sUJw3S z9R1glUH`iQzitYZPkLKWVtxHHa=$w+m+}$_c6D5CFy*>eUFlD_R)4GvRs9nNApaWK zX1;=D3VAOeUd8ENb!zUPAtu3%ANZdCo#H-P=G(2bY%ukTWf}BDPh>}57+u*J9lfIm z`oajR3Nx2x*TsHcXSFe4ViX`+L;f$P{&qmrD3c9j9d|)Yij>J9Dg}YbL7-BO+&`!c zmgb@C%bKo_qaI#8KADI>bYvn6bAm-wQhRQElH;v3GQ$K?mC*XF z`58X($^1ms|6TTT!|$xz|7al3y?m}^_tD;N&ew1?r|m-+3!k|xhRcF4UdELl_@dw8 zK$?6|uoq~PQ01Eg8VRo3$X^@(=P5fci>B1g5{f1J;yut8^p3&{dSCw!DupPAACVXG zsiL<=QR@sov|{`k!JfMXFZpFuO!^o=zEB=A{Og}BwK6lZWeo;E{&obB#WJG{R*-wETIujxMOnA*i#z!2j%`{`RCFS zzpwm(EcdUhbpDWXf4&z>?OMOr-z|P-)Aein9sfI{itVizE}6z_I`?n*dK6uICL*2AjwW2YXUdnx1OAC^las7gC=B@EiMQ{_yj4&lH$hMd%N~l#r z@Fnngbn}_m`d9Q(t^SqDD;hY=Wax zp6|Z@?M37WdfDo$VSw@@4rU9H9Q^OKqTcx5$#QaTg@(Uc{QgTPEEFKqLlNj*nHC)i z^h9*@L?ng@pQ@s55-X>O8kLbr8by#r1Z)O?KEnPtEdHanBLy=DdS=FAnT}L+W+Va< z6-L|C{a3pVUr+Woh=5X(qIYJf$2`xc)9HLVold9aJWG{BfY!j(JL&FH*4?IwBUcP4 z8c1-hMt!*qt+mz;t+i1jd$Znu^HaXx{w<8ZKjy9fy7GHk?k~axpBswzPb4p&?~Z;Q zBf`7jmkoyj_E9cZyXT6}1YtYqz77|puVuZi0QWZ^?9YUuBe9auDiW!JiL(b|!T8$; z@*V-Nysb*4sR0kdz0&WX!HhH!yujjx{$SI{ufF8jVH1$*nM==D*dr}XVoF*wEeR>5 z&rSvS)|UGwj#1{MO!xEo%#40IJwHD`fB5kH{{8#+?>~I_;4^yf65MT7?&fZ0lRMi) zZ!(v=+5~rVgFBnEnTT#SlAES!YOPJ~?!FPre8I4PjpdGwbm2PS?YrgvW%7rBWq#i= z{Fd_a`KIx(R#OKHSXwTc4l%z+G(Nmv+}R-u;Gr z?r~o!LApP|Dfvp3ImW?2;O-;fMo% z20pMmumEZe`zY#10bcgpSPBSKn%i^$q(JZKr+#=qn_TCf zGQiPH4l{tAWQ&;>*kW9Y1P~oq!!gJ8F#s(_hCv#F5G@E1Yz`TV<~(Jp+$jJL1)-A! zQwk}AK~u!iHJ27!=?3&T!6rk*ho_&;=VwXE=+DoOKTPLe&gavI`RC{JKTfB=pZm}0 zr|Cm~P9i5{q$xSU-N^uD4mQC=Qw2l{I6s;h8srdx$#Iwn_oYQzfZ2?bP4GrBb91il zUeCAV+&}hC-M){7x)qZYx?YX6p;vl*oWW>Qtm=DX1+lb;WPDD5m@3u`eC2P8KTbB+5(? z7LonkzA#I@5^*%XGNrvO_EKm{_5TK|`b|q# zxjT?V@7a6yP#YFW;}2wWjW1F2EN!N$x=EBG6kdjH}5gB}it z!@Ga`%RBr)vY$@#@!@~{fBip~pFh0+>(5Vr`^WjG_xW^==*dkrjLW&IVjT|CLS7F5$g+1=*I>~ql&M^Hls5c*NAbRW(nfqzSEkC*A+rpdLXN6q zro=UMaDVgOhIeaav-`a)%T^BX%IXpfQA?*oyZ8)BxmN^``a;QB82<^wa%`PN9#LpE*Y){dTi{yVrkPdXiU{J9h>UzbmCT<+CV%{}e9f z-$rhX^e^oM*cteHM#&3%aEfFy07Aaj)5b>sWV%`ST~~VIj`N%eB~=F&qcc*mMDOas z=&Xu&3Pl!um?10hM`fNWi=+6WEGjXVSq6OWz$=ZIA3&-~&BzTCczunrr=yUHLviXc zDBY`YHkCGpYy{>&o~5*#Dtt^ZFNWj zuu(^eZgsIzAQVDo%8Z$C*wE4{;^U3mV$<)o2`*7He;~538>iGq9PEE$`3zX*`y{!S zSC#Uc%6k7Su07mOqMsVVu{W@}D5ot;z_<7uAu!D3*ueLDZ|_iFOeO}i zV*(2FMZKeU%ZTC2XW=1J{^m*}Pv{bHh16HHYw34OZLhtjjz>Fxe1qtG2`XhR$1f*#g?M|E{T~U#Q7q6sWwvgJbDc@Y9 zesg&{=-|7Qe7D?xQ+X{G;?rd$h)Wy)`&;!rFajcXa7_@7Ot=%!s=Eh1vx@Ed61Vk0 zj>5wk8A~qbSe6!n2+L%oWgvU@1s&KLUSU*dRGK0dnu(<@ypk=hwFM(AH5y9Ghr!+IbfIQG#Tin3JqgY zT%N}o#a39UG6VXYnZ3^uF)!!S>GbsUexB$5$AA0({QakY{PjQo{>v}FynlcGaQlJ4byVfp$ZsRa$v~n{mvCGUhQ`QgM}_HbmwEPD7~LRA_LjkmzZZQXGT{9BN#n< za+!0Uwaf%6=|Rsk2je6<(dl+%n&KAsC}m9 zt|gFS2n{A=N;Y);zLv>$wcpSemEC%EOeTe?4aP~JboXfA=kr>O6s663@0op`=N|Js z&*$^$bUHmfJ$?A_%lUl%fB*OY_H=rl&)wPnBr*_|MUO*->1Y;_8>b7 z-c#qLM{0x{ReHesKS>KS(zyC>caEGa&$g<>i^S5Cg?+6cjhQv0U%Vr~tz_mdT4pD9 z1<3q*Z1Z1OzB^QZPbu8@20+{{@@o#&?eFIQG_8%-$DFyU`z<)700B5tvZP$K&IU&L zpKDV%i4h6NtRTaoxfIAipRJ$!GBY~+$}!MG%aSo;SwujQv5W}!QiCRB16E(2jd)B^ zRvYK4*`JkYvrP=ggevN-RAs{Y-LMDq{V^)6>(34<9~!_;5a-|Cj&ze>$Ddy@y$g zh`!|HhaVpv|Ih!I|HUHg-0eK$w8ZJTfB3MpSdLey%~7Z|GsB)+wZ;&=KknINAwu*Xazr% z{cYT+2u483X4I6bQZf)`Q|%wU8x;N1!bT$Y6-D_Lfx3xEd=d0BCXVtp^NrBisb7zv$MByU+(|^tX(-d zx!Jv$eXpA=AS2w(^gftJAdvuy#cnpq?k<-SR3Z@>8R6k>Zmy=LMuTNy2%vk8xW-t& zdpt2^Vs`sk)!b+r+F+qS2tr|0MAyxZevV}?k-Ju0=UzT=htbujh5@82M&) z0*<#~@7rm9=39}6F8`}pa}#U+iVM)mI=cDwXYEyH)y$czPW6o>!%OggMtdZs{DaH6 zvyeC4^6HL`Yd!d3${Tb=0B&m0e9?2~G2#Dy?OhD|3pV6^Oy?b}(m(C{j2QNTLy7WSy9-kZNfXgfgeLJt-5IWy?WMfWDaUIx0z?gB>;U zRZ+^CN=P9xGn7ytShK+TqOutHYNp-p5$&Ii=%er3G0CZ< zW^pbYxrcdk=Ht66Ae+v;|IQeQ2*$t&L`Q@a$|yz+70Rqqpk?E1R+1uv zf~v?+(MXs=9WM!)Mv_apL^l;rb@1+I7ajZe`WaPBJEh2oP%2V#<<@N-T$3gfAP_Kn z**4vcocaD6&)vXKMg#(V>|^A1-Fpwf_H8_G`*q)YM2eY}p*SY35Rf#6dS`_H`OklL z_jO&DrJ0#jlK5$RxDWG%%nU^cDHpWu?3=FJ_z-vCmn^|^r*Vo0I@Wn}i*RTw43#!~Kz_+LR!k7Nb zwy(w9Uo8Bu*Nt!OJKpS~?u__*$N#Y$`TJeoJNvBrPrSVa+>{f$1&+vK6^f+H3O$&J zHIItY3EX+4(xk+sWQTpH0Ha>)I{@fp(NsH4f@tChjmWY47>wa#D6)inEdv=4prj;~ z=x|C&rH~{EL9nwoN+pb#yh0EvzD>@m(5Ku52@!7P`@eXS`G%2qbX{gD#Z+b@8L7VS z?4yl2onJ9~Dl+?mV-<&M5!Uru-kQ zeFZFY_>zC$?0~n+_^lg$_u{WO{`YEcP5jRu$x~kBaMI6h)1+IFB**CURVI%2rw0s7 zAU=|2FZrv0kcCSY8xr!;0U`Nf-)DbL!$0@?2M%o%fQ+uuIlARw26K{U6vUX`(> zvF%GAmD%$WWqqTn|FbRj9uZ~PW26Q(lu7dkdblr!@T|OBg}8oN&qsMYaY_mx$w*`8 z%Sa~No4bcCX)U3_eM{xYfBRqmpWL6u=RfbRt9Fd72NPPZP`bqG3{X{zhnDqpB=y}P(??1%vo>#Hd*l4JLj13y887czi?^gz2~_w zYw{;BcjnV#GpAVxrtFLmJcWm*DJ5{5G?XMGBccHJZjp%07&ba0kVB(v{ZJ-yfPhk& ziXlmeB27cR)+>-xE_lKzQ-vmvoJljIL8_R(DU)WmzWN*z=OsdvXiU7Uj!ue5Df-Ze zq6JB+@FK~~JTa5o5&jV@lbI2bBeM#@YDN*n3z_l(pd9r=NxI#uS5^sWvj!lJM_Cy| zB%RM@25+cz1ayU^Y$z~%H*UlgQ7omQ^r^Va%Go;M@ndxZ$wQvaXas{Wl76V-oIAxi z>hzK(!8ai1ujsjd*$LB;Xoefal6=Qj{}G(IKW6&{0oA`p_r14Gdj~*&yHMY4>%MZt z{08lpS@zc-DZ??wIF0*h#82nl_P~oDPcxh0a+|*bDr+_O+NWxmBxGfo3qdpK#@J(Q+jH$S%q-nwsiHt;>*day##?~y z^`)NuW;B6r3Mf*y59KuUuQ!4}|Co*~RGtY6kclQK$&@k`;t&%|K+AGo?Xgz38=TE< z?=GeF@~JLLFw3OC<$+^hU74G_rL$83^&67DD%Oj9yGosw-OLUo?w!eTyzS>U(cP`R z8VrBMsJ{K|H|P2{Zp}3?S!OT)Drx1;xEwL-5A=~=F9t8;;Ps69Mb7o@fVg|{&KJD_ zYVN+Y{xlQoUp}SV|K5Hb_(2uPvNN~FWu7x{PkZy?{rg>Zx_dGsk`RcYzFmJ3GIJQz z7BeBEH`=naXX)6&JCkOuwbpFSZFhHXi_?)t(%p@%Z|Ux4t|VXyv(})-<}r&QrHGRJ zrb+#YGs;PaipFj!Y*${NegCX}wI1FBk%*3Kw+v{{xJEu_?5PbF*$e?GV+iR)Gg}GY z$4|brOxyQ->f3eOMh1ZEb+7wcyj_;{!PeFu7GGL?h8EBfjbSo_H*3p?1Lt8@^}JLh z^3(QU0na7`5DBA&5i(xKXYT9e(|`4K4JmrImCnybBbv~FMdsiFFmozBP@oDl8dPqO zXrRkeupxA5j|ev=*L8^qhz5d|Fyi5M(4LmH;;48vzX)KrFrWpR()QRjE*D$@tqkE{Q<_0L0}EE1-qgpB4Wg1EPWP9$V7jST{6p2H7hz9JgkM5A8}Yr{gi zCCp)&;K25%>rIhEAH&-*<~oYx|BE{Bt_TN}<}gL@uXvxX#Q z_rIRbujb%AuJC&P{+h*DK3dpn$pk860MJD83gCiNFQx?k%~|et_8%^sytTKp?izqY zVsPQxw}7J{7iyHif%MA%bPYJZH_{sN+wkBQw}Oa8HiAf*q^mM$G)NO z*?W#*Jrs%SGni5bdS-Iw7$Gy2F%l6{!bGW?DWU9rDEHo<_wBm%9wUlnm~A;0e%r6j zAcQyZ2n6#*2--I|io|kHknts!BDL&wTD?Uc3M2Yn?ozmuAv9|9T#qnHe4t znTU+2k_s^8jyD#ZP8hziQcL2+>ET29-q;kFH({pX$ zGSi3?dAje1iO6LIJIpxU9EZE3{PdF(dG1Jb^v^%QO#0UD+r9hndtc9o?ezCw@1sBb z_3cM)Hn8uszxTa=v@%PIslHy?+$vZm#^`-FK~7iZ2&dFX8qG}ZB#nf7_s9v4x}Dii z1?9@Mh1}5=qS7jxD@t7}DHvT7cqJ*qBP&b`DRXG-(RW1bpFeZk;=1AaioM%N#!#k> z9Y|pao#;ZR1~3R9^RNyGAXApPWsit$-}j=N#ON6jSS|p*KtaC?DVd~*z&_yJMLwEU zLG3e!ccjbD@mkNg?2q4Me$@l5N~9!bI#WcBaLsBS1`Qo?hy$gW-eKP0Kl6;r*AEV7 zQX&`;5s1h>vS;=*5ecjCFy6E8cs=jxXDA3TK&2wa7)6o%kN^4q>-*>N(_iwZzwqft z|NMlfXYBE*7V4lGWh$))8I&{EvMPblj1X#VH>0C^+uYopZ8rL1=CD>*mBj1$RX2Xl z6#Hl&|E>KI+CAp}$2akO6Ht9?-|@y+|6v{Zp|NO5X$qZS3UOyhLm4Biz?u#LE;#6( zJ~*i|mTl0zt*eEj#f0FIFq7FBkT2HU=%N{0w8eaxP>iuAIV%YU;+W;CGlxcuzQuOU z81ek9zH95+HtyX+x)egBHszk#k-HGBHNxO*)-pH-V)P#UsXuMK_udmUcV8Fd zGL&Yvn7eyv5`bB#b%dOLD$`CcJ4Lsb)$+(Vtt09^%8ChMpaZN`vyX_d7)8Tp^)`wp zGtVpw!9mSfbQOoikxKsFZd#K2lAwz&no53=1X^3(HSQIwd~5N|0#XZjh9RKH*oFSz z{@Z`MKChc$&ylfX+tHdLav($Lla*@zp66}}1PO3z-4y4Z#sgb#X9T-6+#v((&?vZ3 z$T;HbM-S-bUx{1%y3^xZ`y1LT&;9qlm@l=rZusp5-`V%O<8NR5()jsD9``Tpu8+=> zs(Oeo;XaaPZSe#cDJG3V5DE#FLL|%t5E&#}qjTS$!4R| zThUw+jAnpUdSk`n5?ke?MrOv?_kEAPtM?qkBQU!9jvg__RG;!iN+G147~n{z0h!^> zW=6QW7^BDNy+8N8_ufMymW95!xh9#C2r!k|{EI|7rIy_j=CC@Y9Ykltv<@FKPg zKvHQIE!-EjmR6!AO|7XdV+@+oSxSkSIW&a0Y&I=>L_{BB#OTUlMN`aBByvz`pt+40 zRS|-ey8v6*0E{sJB|?cF+G1>D-}}C&l1PILL#YfxB)z!dW~mW~jPPOIZIat3(XAMf zP?c5Egt3%Dg(l9YIsYRe7n5mbCLl7SBL{LpByG?SL_ULXc&4TX@y&?J1GI_P;NB!4 zO1fUJJ$N11BDSF}h$b5JjQ2p8JJ0=mz2A85+{Si&zFx1_?fHNFU;nvpKaHRM5p*3 z5+R?qEHf2d3K`SDhH!&(mis{(DYI>Sw&qK_xGykw_gq$Y9~nT>31nzQ_K1<0P0L}~ zXRZ!22pC10&*aBWgVWr714&V!1+7ep1E4ZO+qU(tean6CeUD5eP0bt-Az(mGxD!q` zv!eg0ST??-Czn&&8)Zu*LCDjxXO%#@k%&ZPT>(Uu3myk?l;C=Yzm6Lnxtnl#G>%Tnmyf}%J4NI-o zCQC&@D4Z;}!a;bJ9N~A~{VH1{_(TK0a+`mEaf2{Tb%lzI~ z@-yu#LGv#kb_T)zIIZTpZi0mFuC~S)#zv9}1EpyUO5Rb5N_j%|q+r;#5l(j*C8Z(F zjMTON)LP57n!9^TU$Ql|21y4XlQ}Ybj6EW_evb%^kz*(arGf|=tvQ_zg(xH3;ZFH- z(5p>|X`duB+k0I1{`|CW8)B3d#%tlD}LP@PzL zf$vfk^TS!7Fh7+~u{CP{gQDj7M-CNYZGTJxNrhmg&k9sLfZkv zp7_xr?_ZC*zm38HpjW*)x-|e&g1lTV+w=8uxioM*Z@ur9HMi(($qb~?(72?K(Q|B+ zB2$TG#g&NBlNsKIdXCXa4>PxgN1h5Cm&>v&xNgr+pZ`MB5#Bt?Wp7`iM$qc1RJ@3RjxB^k@UGe(~sTIEA2xHZ@qd#TvxwvEeW zT`we%fh;9WvE(YPID=It>R2pFc^CNoW2v0F0*^VUD85XK=D>HP2IPxI53pliYYP)>i+HFh;$h_H1{NE`*nh$&|#Z5wH!Q zoGOxTwwtxdmTbgWV&T3$$F-9PJe43WNk(Aq%xMWHsZd4d4rz6zo}Pc238-SVW|WCd;3G`ssZS{l13t1kyyC0+J?FMIg)5SG(=hRz%ttm}O!z2RZT-`kx~5IFOmw zsReAH`H~IR!j_CsXdr8wYH3I$ z1c!+hq-X=48Wbz7u&j&8jhLSN%Orn^ws}uU5io;02y{Wr5`%R(CuPD?lqdmYikZk% zRK}4E(oo;GalMY~mNAftQYa8$bZ|M+9KFDP$O;}{RV*46EEXaWg#jC7i@BA87)_<$ zTVQd{FyE|?%7d|ZgJt8;xFpcbI;y;AD7vB}7o;f$tpN$k%oGF3WWsp!DQS!mV>~RZ z=rJupW{QbC;Ku~aPP{}1xuwR3WIQoC(qu047lsTi#F8PYTjtDZIt6DwjG*34u3vW& zeQRIY{%MW-mp$>@!FQVb-{yMWc<#v=se1tjlb{jJ&>V{^JR>tBGgo6)KBg%I&;t%+ znoCd={oK4VFe%R_RpMf1&Wb!$U(KBn%oGTlj2sNhBo;#hL!cA_1ZGOA(iT-@J2FRv z+X7&Z=zHvY?md;5&R}(_KrT8|XB!DYX=YQ)Fd`Ys4k!g#0to{e{bGx|o4cFCO55ZH zrJb?X2`X0^ic`Sk#Et+!K{F>rr7|L{kG2lB#+kTZIl8qGC|Si>l=ijE{);B_Y2Rzr z93w~M2=&l5G-iDh_Nmo#Z^YO2gFx%W%D7t$+hgDNecyZU|Nig)zHe<9MvfdC`iRJE zEua4blOcmc;2DL+R1uU?otfqS3WBZ-CccP^o~i?+!wA>`RTtECluy8Mbd<)DPpJULTcSY@%j3Oo#REQ^V>%-+ym$ETEPwE{_+ zkSi0|2O_Y#jWM=;?0b%`Oe^SCS>^Q%Fr}x}6>2dDz*Gi(!U%!Ly?0a95J?0Ka9TSA z?F)Cn31u9dNNBPlrNl@RvNcqR!E>9UP!%$TAr%q1Z)omgU6!F}G!uu?X_0C9vT#H_ z3aQq$)4PM`@(Ir}!bd^~+omWM7GXW+DQSz{nF zaxfFXz+z@JLkSWT(GwzDvV_kAtU+4plp;u$z)sb!d$*mIeJ(L(W}-VA z>j@I(BG+}QLf%?%k`b@??=P>>AvK3*7{e%N(8Bp|87n_M*!7b=fAY^y`uvpF>v;PA z)^j0YG;@;jS%eFi1a=dVApm~Nc-f2Vep?}+Ec+|^+BEB?JYoA&Aa>SPKT?@qf>eKS z&;4Ja{nHxvFSkF=nB#P=Zp$9czsTrS-fl_T2d%?%j9E3ORw)QY#fn7bH)RN^CZYUy z`ZShI5Y&tr>fo(qf`m|rwxaSpO-QgL6fqL?iv91}S5&;<1z zC|!gMrMU$Z$N?*5%C)q<0OVwayfp-G$H|?E2T+zSA~G#glh8<`yXd`lyl33s%}@pK z@bEyOJa_kWLt*Y(UAfq0v9-aW$o;w<;$}c0Ggj0|9u>M!hIsBOZbZC~kG%TocWCVT z#d%4;`^@^*&ep6v@>F@#IQBoaZU$zY<0L4##Pl>>s<(t}I`4dE42g)ekui~Z`( z@4j0{;`!alwx zSfcTy8I>^DtnNnAD{H+5&EDgv50K`VoM+XhUV#)XhQ=5Xion(<(^0e$Q9k>=kAx*u zV(>|UlQm~+mRv$cBBem~7^7TN8POc;igsC-Woe6L<`J1aZx9eNqe7#V^Q1gPfDhrc z33+4Au;{&L^VWMe3=O;b^YioLzY9h*pe33WngSDY5XdNdri4w^$?MfdTJMaCLsan! zE+HkqQ+8UKMJg0A^n*_WC!*4eMJAN$s@mRFiholg#Gv{dX&R$~(Y_ zG1Mck7~9egvZLm(spnjKB~+iAkh9#LjtW#bJhmEInR!<5njNS&o5?rr-JQ)Byq}MJ zyRYO*{^MdS=Oq0t+c)F>JGb}8Ix12)_t9n6TWgg9N>a*%c<8@p=0W2a2b&{G$b0YC zCL^171tXQK`VL-5NQ#geNI9h`{(!oJVp{nw#9;gk(h~kS|%C_e5am?%f96zc(iNU1whLmH+S>KS{7vk|Mq9lq2mR#oe!aW}JoC}j%Ch!^5Y(J@PX z@j!-WL{+NlbG0wAs;UYl1x2cl?Jxh6rds3ZkfoZ5*~jR}9vT^ukQw20YoAAt=)I3r zh>;wrE{tS?k~MnLOyo#L9EIw#aBNwvI&A9&^n`(pbaAPxKIIq@hSa|Au|09yEIa$I z{+Z*N@jP;Oj>U%4B~1WLDw3>8gXJMlN6w9;6PD2r*VazUZf0i`p>VL<7dUJa-;?LN zd;c83ya}k@4)wRkza9U#r~2~2f8Jy(sgvqEFbkx?TLlDYurQ9yF>-Y5fy}wD z?ShB^M&F*V`E-pjkg~0ZDzs=YpNOy-u3KgdClH7}awK{vz!V)h$sWqgJ$CO}7bA6u zH_ROP1BzGT#UUz*!phtkkT7XKs<6c2*&_udGcjUZx2Hew*#)72C+0 z%DXQ%{jS{^>$mnDZ=994miq4a*XOK|?C~dnya~+yD)Z{MYVU4`g+9yHG>Q&n-}w9ze;OpHCoo|b5a zQWRenv!$}8lsWdm$Z(&QheQ;Wy||4xy>rGUlk>s|49PKZnfEDHT{01vVWEo zF(Udg5l#!6L^9v{E&17k!F6?DDGB#qBM0;tUa3^a*! z<5p`UJqr+{mrz%~c%jCbIeLyUV(f~%JU&=<#AwLIuna{gwbKOBB=}h>qn6Hgz*&?* zB&Knn>$;#!W{#0&QMtNzL$2c1d`V*Z>+2}I>+ipe^Y_k)8`FQgsQfDJqqDuyp8K1= z;zZrQ!7|_4t9DMWdQr$swx;B;6#JA%1gdBp%_M7!%NAi)g}X49rM1u1{!$1j+_7@bK)Nkpa=UH`%on4FgCyv zYU?6tD3Jz@vh5NYFC9HIlq&I=wxG4e7 z467U`lx%NViwa2-@{|!YXX5kqr_78P6`8GaPGqf_0>+v4)sm8|6*vSk8Iptm2A)6n z2xI^%WGU){4+Tka#xTnK($E%QvWZGRp_U_K4tFOr$V7^&0))sdvn!)ERn-Y1a%q`~ zn5hXW#A;Sont)V{BssK0nn+%QRm^G&M)C+D!_8^d(x&fhxz?_xFY9{-Wa z(~&myY619_Y~#nC`>)*eqDu75bN|)=+|bDGY(30r4jM}3_W0S@m{o7pA=uRYXv;Fp z%!)=4nTJMsCg`dnogon2@01jZ!-iLZXgtf)sMNe;L zNi;E&qJdc)IhRL-hK7a&B}lOO5gg@^8k9iNlAfG;D2=T2_UfO&QOg3tA$tP6n0VL} z{VmgZ?YoQ%kx)j+3gOPIz(r2jhlD%KtoNL9l~J`W+{|dc>p9PYc2EX#Oo_DWSTi&O zi3p_d{r4*{;|(&ICkYszpNEKeq=+Nw0YELuuh`FL%MaqHckQG1Kl0 zYp3*R{r>jF*?!%X^Ov?e-}bb6w*~Lt9{JIEzOZe)`}xt+xph)*U)+8B?O$)<%iA;E zotdwZO?R)@TB%b#XY`Do8c{`lis}HfvOrvXaZ6wRBkvEXTAr-(1S_6Q2l+HmAaD#C!Fk^sL3lCCIF^2@Q12n=dkuvv!(|X@; zHatKoN9-{cJAyGVV{5)HZS`eot(kkJRtY&`4?&|Z56fL_|8AV$O{I5c(2bAs_V|w) z=Ps@I^>fcwBkJ{c&FhJyPJ@nItq-QxtLuE*X1xd8-^E8(WsdCE_gs5e{nJ~gg8QlSnRG{R{1(gsi0i)^IA7o+pD)*cq)i4>?z|6m` z)pO?{9TsNr24Whl-eW`toDN@FYsqGJIBQn9y8^t;G*u)gFa-eR+q1xxiPcoU2sdJw z)syv0yC7gu2(}O!8GXz+FsH5q6t#MxKO7&&Boa0n$+7nyfdq+maVy<}$hOw1Q%gZg zQRKwO@76kxk`yyBN|i>e*hVo;?I5lC0uttMC#tRs0oWpBW@ii}6Jrb{q*#1{Mg`*l zkfH)Hsv1@SCL+=z5PEug^5CMju$e91s-)jb8tpkU=VkvApZXlkUwQoP2fyCs|JLr> zH{<@hwOhk|hf$vGb|}{zTP|xR*gEzhYG!O2o41U}$jnZS%#qPEvp9gpDfguUWM&D{ zGJ$7w6kyTkMbg)W&ZI&kGFc^dm_{!ZE{Q^BB_olOE;zS#G@II!xB&+w6|$SIr6Cu{ zusFOy)035%nX^RELNjgmwDM6Z?X}a8P!>O5?6yKIiAzolCsd_dnN#%zS;)PcnYk@3 za*UxwDpHDIsPx$C2AU|jXQse@)I@bqGJ!L8x_WN=P;brK<>KZVL%nA!cqAsUJ`=|@ zj;yiLF^D7K=t)`erBis zuiU;F_g}evJ@PnzeH}7-*S_Db0nOL$VP>Ui^+MlvD<)iK+DvgbK!TG*l?3%*$P5yS z7fDoReAyTR=pz6&x8~-JiAaq^F3n1;OqF5@FZdyI=;kB*8y@`s9~S?Q-#DVQZ(2ei0XY4PAaTw%=OtFCYJfruPG?-yHd`ZQqRh zZ`bY&_eY-lTW9AyhO9ExCZMJ|00(4@2t`B$GfA1z-Q1XRlI8%CYV^oVMT$Z~LOJI; zstXmyLN{NHM8pUM4Xyc5T45fhR97N1jKw&dW|LERb_P08%CjdXBG8iMi3?zvUJz)_ zWE!Jy#oKmw_vWvF$eb(Pq;Y2E$jH#N0%k@wPT|{I6h=n4)jvt)ky>+WzJ$RYi&5T@ z1~1fhyT%k^%TSsW5+PpjAcQ2qP;5+yU|ctMU)(aYU!PSvkv3~uvd|TW7q^sVX0?hz z)JO&xva}Gm;Bn>S!+L4HG#UjYF-S=1M6ybi%4JL4U?P(4jKN{B#PX+y7};a=o?~F| zGv@K>2e_fN*4o;Z%X+z79zOZyVObVyt&n%!IO!)Yd<&Z6wGqF2f%lx1kB|S-l(}6q zc=PSIAOGgLf7HGi_g}gF6`uP?+ts^KJ3s4{t!`P|Z>G_V9NS*9h(yWNiZm#^U5c+v zQ&FCg1F5tjNdW|cQXFYXm8ohnBRGmaNkS&3G>cCQRRqH1paeUrf1AM#BzmuF28tS~ z!r-Bj8F$FwiR=?5>M@EE2%;^`Y+06NS)0Zyz070DnEv=_ba-Uauhu5^g88T7^O?K3 zmtUWmG_$4ox_lZo=nEFpFtJPKNE$DQ(CDc|B1!|o%<=SGI|G>tyD&4FuKV6rb2N=T zc0{1Hv}H=?yQ5i^z(`66IZPu*&wY>4Gm{wru>63_+8!_K<3oE~+!$0SG5S++SV3E8 zZE@O?#OefWBr@2$q+|%7V#|_XHnaviYN)P@`O@0DE^BM+x-JjPvb1&G_ubjx!#!BS z(!e3c4jp0`ci{UMF!7I^m9KQP>x+40sNQ|{o5TJ!?VEA`?b=5;8&B zN~5Kj38f*<=&*ECBbvz=s_Hwwk}QMEsnkWJqD03Irz~OskYH6~8zL(ONfOu)F$$5T zC24SLrc9s$n>nh4=QMVdA60=Sfm>y6=dDOYBE!-&MrdFRz+vUclO=V00<8yCcy`4r ze86rn7Wc(kN>S;gCl7&icexb5y4{N(Ln!_b2 zh`m{&_>#;tFP#{Y)&$9nff*@P&IGX%Z*I*Sm~98cAbh=Js7{cfBEj zbJN4JIEI+ViRM3PF~blNNydhC9tW|o&qnen2Gv2{aG0|_xP5Lxy&20*peCH=1t z4|TyqBNdwBoLC-Mlp_-B<%}f5%WZFL1&v7~5rN#H0Ss7Taa`KN>M1e2MG2n`CTQxD2hpT7Uo zRBJoUsH99CX-xIT7X~WcnDw8X0y-Lti_3|Ha4D^;_ma$Vn%hI>fZBzhXBxUm%7|W+n(tO-q9gZ|IaqbK zQ%_H8#0Z$oY1T6`Ln0SN0ETq|*dsd}%hEi{P|`SaCQ3ngX#}d=$EcDZWhRnKv2^O4 z1}p|HjG@KxaKYoshee+}*QVBth>Y075aw{lQo!|fFYck%hs$!g6tPhad+u^cvXCN+ zHV)unUCl&J70FW#F_wkvnwLxZ8rh!qpRW7U7(vm}*VrO8^#L)vD!EhRFm zP_=~6`q+*UB4u)o7Rbc)Sve51wj!a%JjQh(k(oP{MRqi^w4CzvGcV3-zrlpLc~y9l zRMl_E%t5ju^M{cjfTrouPbq6M1W3G48n~7g0P&CQ~_vMUY%zK zgjv;nc{7XaXXFR-@k6uG>dR13AyPPX;op&fcL?s%pJsP^Gk##+Y{C z`1$2kDaZ1`mA!lFwGAyl&*&`rt?&;f?rhb?kNPlgU$cHqH||NV@AVUqE^2J`0)RwZBoBr948- z<^BR_tpQ1s*Ya@mo!LFap;OiQ@PsNLTgNM&Pq*N#kqgir60#H#X7Mg%VkqhGO1M3S z9Ds~j1O3ZBG7hi&#;4KUU(aK{c$(wjU&G=4@|^nPx3BE3U)sJI_g}ev;URqlJKjFp zpB<*7_Z%{CaW}MvAdF$8Xl6_Y+~h1Gr2Etik&rCdR#QH!wZ-N6JQ8fnnus1fr4%wp zB5H%-4q9>f&>Akym&MIWh1jtio*Gcte*GE|hjE|T{UrqR&^W8PRtby-BOP3q)(E$3 z4sUV?EhjH!dN~JeeRDH{<@>wf7CW-ls^u1*z(F^udu1ny(Hy zi3p&IZHTOZ8mUo)$i*kmU9yNSd1gu^thI+2y$Y4NwIIeor~*K)Q>3z4&Ea zy-nl3T-pt!bj&r837H*siju0f$TE?UNx!ITU~JXI1cD3UOz8O*cnwr=XQoNjw-ZE7s*g(gH& zIscV3Lwzsjf<)!<@D3;2Tj6b1$BYnr54Sc}TCk;-$WXIHmIB1Rl|RPkt9a?lU-O<8 zdG40*+UWmv_WiAg^!M<2|K9Dl*=l?^0s9xf|MMD7uz>?Xp@uG^El zd(3R5h%v^9fSJf5KI*)Xc^Qd$S1CZ%QN0guIN>gHnYq0m-vNNuTB0J)2NOsFaA~V% z`fnV#wt+VK9+8=pSjo7>Kuw=1W|5&?#w|z#AJ^|(N|7Np)h9EDPbU4`Mu4~Wy{`V1 z2kNjdSgtpp{VUt8=YE{$AH&4odG5He48Qt!eKYRAQM=og;ypuu9{KM3(+mhcD-5%V zMjDaCx~#5YB-D{cr8ycsxgQ_5gv1n0bkMJRY0OuN_VmU8;p3gi;Ntg&}hmi8iwX zjliJ>T(LDufO*cPWI`mF8JR;+!hisZ<4&dk%XVJ*xn;2Tq(3FA?7Rp&0@^#DFS`d48_gTHI1-k#yF_S}D6 zgy;8de^(d&n{oe5+gmL13`gCK8PVo zC6GUfiE@XxDGCOl7A6Tt`YTRiB1w^BjD&75rMf$8TTJcwc^3jwRV!*{t?^LeDadGU zS7*ewByC0VCzFXHvE|<1kY`s`(EP?&v`rLNc;kh5H8Sacb{e2c0-m}a2tNiU( z=lMtT-0>U2?%$02@6+DGGGF4Bb{893z>XyLvrGjtgdV~n107-E^BwJ?KwTh~>GXDun`{lXhbFG`YThfq>m=uO^G z;{2Lx9KgI^{sA+)Uay+I`&0+0KV6>#FZWdLd@4`ddV8;s05ATJVe#eR^6As3AO7v% zFW-N^!P;_RTUu*e8jnbZ0`In);rbe9aNvz)f9tsXjqM-TVgI|j>UXW3tH)P%)?GCG zeE*mB_3cf*9nBwoFW!us^Lp-ne$5)%=2y`u6J zA{22136kWehXtsb*QKx{1M0JFVO!3$OnIZX1^w2-mJM_>8)HZ>5Q*mcRGO4X)Yl3y zlGfZ81I7rcBZ2i&xlYJHJ6h@W)Hxp>9!5sV`d8;gDB?v2{GellARG}gb8i*#U}gYL zsx@QtWqnv59@kIbUB3Iy+7h95Sz*nrL8fF{W(>yOiM3u~ZtgV3T z1DkoX6Z!?$f44EJ|IT;5dG3Ex`}!sA>wWWc0^HjD^NU9GKKkfeL>LB&;X5NdjJMAD48kwxMyx^R415^cyfSOl&(F-d+_xWo?AS%Od7H?w3U90L| ztt5wqt)T1SvL0^ju}n_rd^D0s&N>=%J{G}0&qpSkCLqg|B@T!~b*eJxIg^F}+**g~ zfmvvR1ZfdLs*o8!GNH&>aq!rbyrdWO-t)Y*8JwQobq<`P?vJMDZeo&5PF-jLu+`0) znbkfZM2ZPxPH3zCr^KSo-e(L<+h=xCV08;GnEb8#axRBI>qLH~m-&zDx&M~!n{of^ z+np~zJ^G(J{(h9d*%`dMB|2wI9FFH0J8}T2swEMEMrvf=`+gl`l$+3}U*$?hVBbdX zG5Rd`eA;v!WRtB=tt%Lk5D_#23UMU&(M1Pw5YNL}U9dU4gH)CjZ;s573TCOYcw5Zf zBVb*ds7jip0yUI37QCPk%gF#?p1{b9tk_FG>mZy&l+P7= zpTwS_3PWZlmC^UIp`0p=^&oh5)KpJ9{4Fy(1F!%{xSN@Y0f>Y*Dv7g@_a5Fz${nqF z8QZ&IP*B@>JI5`*j4=`?|7TVdJn&~DGG#_NC7EL;ewm?!nujwIeXxm6wvvV;6R>hN z^UNoBI}~rP6E}{h%`MlL*k8Br-+J^n_wu3PzB}7DEB}S|&A9)S?On_Mi$4DDB>D13 zIL%%bD=w9lC^XH2ERuW2wxd7{0N0s9nt;VX@3C(qLJ^vke*|*+k021BPlh{;U9zeO z0v~A4gCP{MUxe-Ohq5fDqz!|>Fr-^%hH?nmh_-ZL$sC`2nK3$*j~rEz?u~b7q;`g z!nlwCOXp_NV+@pNV#AmfOSlj7C0#ASOe8_b&E8n{8~Eac5nlE1 z*KAK;K{UrZhowG#{o==8?E61pK=n6k-;Dd;)ZU_p9}Vq0=jE;EjyEgByRb|$1CcpK zMkEIy~(b>DMTkg9u27qnUXH&@tjviwaNp60N`3q;`j5v}=Vef+_Ck`yjB?6wh zaypW|_lVfzq3C(YAq=-%8k>#BhlT2_l6*@gKn>8WeFr9xlmjUxRppbYsw`EUm&{_w z7wrFlWq~mwG9#6$S|V$x4o5M~1%?%nVPiTxfa@N)ehD2o)elbSwdmjUT z`TV3D5j_J2TvmK~#N!20H)3_H3rIA>BvPxP3}_02sWMXYyOc^*W64xik`NK``8ViH3kAz4eHnOSQB z4&~k(s?EVsoIZ8QsFCC>Rs!1O&zw)WB$Xp5B{=bx_ApV7>D|{daO_a#UZwfXfbrA* zS8b*$AbW6crl$|01bM3P7R~63l}0if=_A7;kg`msCDY~!vlg3Ujhw@>Lab%n&+OBW z#_-z-avt{9bFVJ_XgV2?oCc;9 zHqgwRo;jpr?Es3YaexC%S@t+erb@Ec zsW>E!23W)^f|>}MmsDj94{pp&MB0pYy4pZJ;lf7BYp;rZV`M1;NNCfW28oDy9+gH= zE%3F3R}jLChiaIleG<|0*cKei3}*pV97C*rX2NWuJ=Gh@U1`gAs}AJqw3v6s{S=Y} z(Avppsz1(Q>}+=f_{%Q#-{^OLbKAer+P!1=bK5K2xT|OTE57)4{lESBh3|d06_Uv} zKW`WXukaK6!t?7K&nK#gm-D~=-qNW3;5;|+y_b#o%ZnEiR`0jgikmSjXlKa=XgAx< z#$qXl!jQD8CY@9;MGHtwG-BT~ANEZ21=n)k33wX0`a;FB=y8?X(C9^y3BssIXrKX? zi>*sD<7M^j+QVcIuH;qwwjdqt!Eo z`&63QW8X*59Nr(Gp$LV<9An!Px$h4T4@E_9Fl!GS*pmRlFS~4y1QG@`rm&Ch&6H-| zDurJ_$T6_T6)*`j^Ja=vLejDTDk1=RrF4y)nY*)OxJt7V zz2e(mIur1tLf*)V1UP93i-FSFAJ__HD|KJVlnz+IbdnD71GE+H_%ByPV(dM}UxtP} z)(g3sd`Y8WLCOXitYL9^Hk%`y!m-MbJV-$#UrKzCRJ;>|{gR1Dy;OPUl*FG>mdD^H zM6yaeV$MH1{+6hyLv4)a(p#I7I?&$v(hnx)apw0f^8+4w9Gk-8?=q_rWFFZEtO~d$ z69gFEnfqU|NFMM1_1Ffz?1ZX~8w2#sbN@Zt-3Io2=XOE5+sV2w?sDvbe!R&k6DAm| z*mSJxTAStT^HWwrY6S)luEo*}snu;+d@-aAN^S^ql4Fc=MK70SU01>)A{GPE%w%c{ z1Ss)?@v@4-r|C@YdNqthzXMMqt4NKi)3 zGD@>euXk3kJzSH7cw+PfbnHAt7DJI0r8v%Vdp4bTPW`DKrONshGE+kKX00N+RmU8l z(q0>rkKFAO7`Yp|U+M z!lOQ&NSI)4!J5RnG}CB3HM+LI2t-%Bt{p@gp;VLXdO> z3{UeroyU66$41a3Rn_b~eP2xreTe?%O!q&|GZ(f)e!ze@sw*NB7;w~XiPliq+b1Y*YeG{|3>Y0KD=w+ zZ;yOy)^Y5-oE~(t5sL|m(oIY?FHez_Io8FAN^<-zBKCoYp^+AoGc(QLW#QwBwK>up zOIxg`OWF}GbN9!~vaU@+Uu4P3=$EisRhjlRm$p1SK3@Lx!{ZO%KR$k19xv8BGBR@O z`yTr?df)ebYt&A-}Z>nkw?W*H$a@FKzo`62_Ya7 ziGv$nQ>=o?36cgZDXf;ZpfuDaMWdN5%W{D~JY2s2 z{?qrL*2}}PG(h|Hxhs0~eQf)_?-?=1hyVaV|Gpj(qB~LXK+iCa5JE#{Hj_J{6jXBJ ziR`xR96QyaKm(SBQ8EDu4W(rzi-JkB6e42uOie|D5<-yr+&u((AAJvZhuWtLdDPg$ zEa4RUzDq1)9mslJ&T)wR%_0HH z-*-ViGVWh+S>FBZKY^tu60q~J%{S*g*8VOy3kSsC`+jfx_M374joLZZf2rqw`#wpt z)-0PpT$)C1gV0cn9t#s`45yHp8c>4aE-4$}ZjfbSL+na;Dq}INW{ZIew`?vcWT3;9 z&D+8OWsF!o8^Fa{bBPh*?(4cdnmsz^ zd05~va*T*b9$BCf7=ga$zNue>0ZRmNSuui{h>^o)&JmF1j1KPm-h1Cu!`&?@#7UAF zJtNR}MBsA4vfMiClr&;&qi4?( zGP{o@$FLm8rs!$Tv6xxH$f{W_vSGsXB` zunvcb}7=dWU#Y<{Nav05B%`wIvV_RB7v*vAC7D*9iW|zydHZBVs z+y1;sVa^<}?^lT$5s}C#Tb8h-0!=i5fFjjXyA;yi`xqs2h}O)^Mvgvk-O#s;fF)YO z43S82>rur3eGr%>Ld-0PeZThJrD#id$sk1Z-XkX2zjr9nJfSKhO;srqA_JpmAESqo zP%43*7=a=_+C;|7v9~;;rKVX5=`?eNz?@_acTOCp%33){t8!U01*!6w^5qP*T#3re zqlgC%%RaMMQ?m{#tOOEfeef}aJoYrlRiSEw&U@9qR%+?tTDw&QSyE2Yk(B_f zN?Ru%?|+lvgcYX9@~%M2@Z)O_4!) zQw;bJ4Plx!9N#h1UkIpv;rQG3!eO2pU8@zBojg&=Kg&?<=?5b+tgva z_ZaTljmE|Mj^|glJN)i^(r_^)>wL{pJVy@*nYJ@%mvi1sBU73Bmj(x`%5j|oIm%dx!OLO;q47L*L3TGAkV?+YkZS=lJq&W-?Vu~Rq zG=!0eoX|(ICEJ7_ULLxUZq~-w6Bt>-3di!(!HjyR=GQy+;n#$E_`!U?o7d-IJ`GnOC?t|L_3%0?K+Qp^g4oSG z5PGj20r2}a^Qhl6yW#J@GM+bbKp%SUcSi7A`z_nMPR)GNL^C!gjG|t27j8X3c#eJ@=7PWL&RTjez7nkDwbJBLXqH`70`ghGvaaRT_mE zoG{7^ZDDisMtAS~em=puU1vp$L!y{UX5t9ga%s162}OS@Ntvnd=3d_V!JNqwq|B3| zVJ-=_7T0LbVog;8Om%>o)|nb3#%{Z@UEmEd=pAaN1k(AAXZ^0%b^2%T8pn5!|7h?_ zUy}U&BeWBh0*c*Aq(C8A(wx<8W>q1q2_thLV?-_w%Xx;|7atk%kJ`D|+;#hHSN}XL zb1uTiC;tg{Qh&Ji1#sqkTJsSatPy65t!WEjwI$d!D@RK~HDRQIl-s40XaFRm{J*ue ztq)+Pyql5msdF`}K4A<0Q4l4k> zPu;V!v*-bLf{pMTwsk&ldA;KKX)~Z1*x=2qYL7rNDwRVKP(YeY^a^;I0S!(AnBV|u z7PMdvAOtt2EwWYdc_ut*W^<-gjP-G>1R?}WPHMXWnx*P!?gnydr302`bci;){CswW z%T@tCXi9r2sq~m`yGv?CshK)ol0>T-+%iXP3%m+$%@}LtXKD6;)LLaxShd$_ z==t$FgHh+nh$?apFdcLwN{dHnfgTaj$MbmiGHyNb+h!lEbGJyqYy14tzGU3*4*llE z|2^96Ot>BNx3lH;_;kris1Rl}Ef!&rjU7Uwn2bt-NY1QFLkig`xKrlgE{&TL8-UU> zIYr)&Z!c&5UT$hbzM=!2 zMwVhR9XXF@$?ze&Vnp`LMCxgK!t0x@*O$xX{qOzYdQNM8RyDYB!e{$=Smxey|7VDS z{NdWIWq&&#Zl73k2o!1NKvm;hCHf^YHZx^FDKo-SMb99U64Jm#Pf+%RxISUo+sZFbfIE6AGgK0U$jIw4lRQ+Pb3IlzW8r zZ46=HFz%V#h;2k3^7JNF5!bV;1warvOOcZz=;a_2%E(!^h!7>SgCSWmXAv`Y@$&4Z zCD)OiTNA9*9_0=ovzHu61v5))ATzuj#h?(gRwpUR+#HqyY2|V2$WsNF71o1vtkZZg zrZbb)NiR5$s%;1%v#DZ#y6&%l-J3CRH-`1bU%|~gw`X{3@g*I-{&xpvK3Z?S@V$3t zeZEFORy1}yRw|qRx?=Q*4H?3ql4ffa`~B+0?4!eOZSsqWCSMdOzIpDyd%K+gck|(0 zLobJ-xFZBzZniE<9Tq;i<=plSH<=^*E*KavS7ZjpsMz8=qTT$ zP=XVwMWwHs8%ecK(>_XwK#t5D%E;VDj8K$0wE^L1GkD5V@j zy+;PjaJldiYL17CKVH^lF>f+ZwTe_We`y#@4BFho8t7hKR0XFF1^gI^`iD^Kqg}VK zI4Tzcn83`)boWS&lB1G=%t|JoPb~$jU_w0|H^SvccX=rg0F^FFQVM|cSV9Ri&gvZ! zz?_GB=CheuDd(E1wyHUfswAQT9DwWSNhykC&*D|i-4!+UB7jqMG6Ac&Ko-J7$_#FB zN`xt)f~bTPU$!(a;LJzB%iVDOde8l3k@y2xn2#>C$^*QbiSu;=Y4W23<}GAIQE+n% zX&7a0G(7n7<_s_A|N3?ZELJ%QbREF2B~)S2lCJwo7q{7xu&%wqeVuBVkq@0f+q>^!$n{P{;sNeLl82l zl#k2^_82|Y6Ds0Pqo;{uS-z*KcoXjKPWOf6kAHa{L)YuL?t5{}eYv!@Jm~7gve?@E zviQ={0NDZs=>#gfMuCx#xHQmBw2?6~G^GgGM~*-w5*Wo;->&A%SjQOAdlwc$k*XSm zT--xeB+f#7@WMq9U?!6l`OuK493Lx)oIFF3nhGj;5K0ai07<2*!%%@@t>&9qmC1&A zdP8gQMl+eunjz)xYX;&ZMM3=Tb&(MR?l^^RB&fzUrpSMTe{`wnO`Csa}?>B*Q+<3ZvZ1C!B%!}6;LotNP zZu44E%^5cvpxdta&_n+q_HcXsxA=R04O`~^J6>R!ul&Tf_6ypb9slTnn)%^iFj?S< zu^CKe=FV=xo=O<$WeZY^n^Wn=-7|Bv=Drx?Fz(IT^dVD;W>OC2UdTz&%H2Z(JYi-i z3uLXCnYNe7tem{;L}sa*PR|T7Ck1onq0lwg3`B&)KmYm9nH(Mat_Uber@OawcP>V6 z6-)FA7S>{;boSbHh)4W0fp}F;ujI*475bunCiaou_exxC(P&2FIEpka>xB#!Wf45= zdsd{cJf6~RH^}K@-*xCVXQga?5yCuz8D-xAhzC`_HD4BQjb+^3RbbSiM=l1puurG(d2tA$3?8O)P0BQxRZiEL9u#&YI&r-~9GZU+@LL z{8!v834b(yviDPj^%WJ;>{YbvoXEX*W0=v+Ig3@EWmSNiZm;7%3aEa;-rg<}FH6Pp zV$$0Y&_Da~Zb2qy#rN_SX0d{=Z)Y#OEDx{Ff8Qz{oUlxMiT#zj4OL;+fwiawy>EYN2+fr zZ|2TM8(~cPr-$Bq?_;C}Y!Ewg4}AB-e;G;c%Immp`=Gc!tVY}ar?tzne)8pUjP~PC z`+xlB^{1bnug@*}U}AMHE_bcIT|6$?8~Ly-7k6VwERWLNBVv!f_fg7RG+)dv`$UaV z%#d^>QnqhhG=nFk*hj!)0Uma(t9^n$W&@e>eO(^Dd+cHseMDgC+(Z=tR8{zsnOd9A zCx$MUif#dBX)v=SX}FAiK5;2zp>!cIA#xB&(8F4^X70n6A&vn^Y-pr8sI zoxVRpbGk4>F{YuY{1vxlS&|%N6v8gS@gGiPqUxb|Lt9sRs~RCHV21;A=M}bC>)-$z zZ87>GURI+z$tvI_a>o9Qk9Z9P-?npXpTAbAnVF3%R*b~Sxo811Lkx3OXmyg(R(n0o zYU@c-yiwLYsx98n-!bM2ebP?nwWY>B#xT%nImYwnr|b1;N>URt(ambV>xS_iq}F^{ z7DJ3aoZz~C_rnEMZ+_n^!^0eAzV)BZ{xP%C;vi|Q;i%hwRsfb1{!=m2%-XuHtu?@u zEakgfq?D8G_ITQyf7A@spLGHL3EHhYe|xiU;hL%6U6HsS{7TqRozW1uBuj1PpjF)Hu8@ex3LCmZEae&637%jgFPRxKDUM>JrrT#Plg>ibb4P)S z_a0P=s@den=b{Ke2Q41cy$Ek_eZ;AxR_xfB?KPP^?_s`r-jLipt)_h%82UDY&M}!*?nHorupqo;C z#I|ql-r7q#L)r1?AAh_}Qme}Y;LXa`lP+oBN96-vF12`QM%QuP$YkY8`iC*@Z+C+K z-1hy}*SPiEPYmwm-X)T5E(kKntC)2rMub>V^MG?`^#A2VYSw6@T*HIrm&ic~&rWqd)lb-BQ7gvJ=&9{Y9w{L}Tv z&->HUxNh1vjjq_Ggn`Z53Yu$y9Lutfw$PhcDveoG8I^u-c< z1Zy@6X#+AQm6;>sd4KlZm*LCBm*&f|(3Q;Cp7IdPWr{*2lh%#blPJ)~k9+cA7>j4`RHW`x;G zI#N`5)RML@SSB*;7+9Lg^tIcVrNB80WNTL07Ea{k3lPfPRDBoBx&2f1>TlCd>xtQ` zmEoXmduGLHBFEmxzVF*M##ol64oJ-sSgZ`T<7~^anGo3xT-Vmz$Jh~Rj?1!|S^4GW z%#l5M-#Tufm1S8B=Jdk)bjVg;`!kBdU}^5foUfGunj{^PXFaT*FM-*A7~@{;7TdxX zu#|tC_GS18Zv$2@N6g|qJOUvkX$@4%6-X&aI`H^|LtEm3j<&it@7Tc{IZ~M>q|8i!5*>Zz* zn55pjj>09G_tT_&L&0j{o0^m05iu^s;bPc-I7H7K)2*S$4bT1Mm=Bl&?|Ny!jnj5s z!`vPnNJ!r{Gow%gNRm6rrTSK>7u*B_&{nT3kbU>;ecyK?Yid9Nz-xUaiInv4=;sMa zG<#W=)>>;TfKt>rY7azadaIpTxO;?7zkb^`&W)~@j)QyUzP@mI|6z>#-6;9zcH6(y zm?zo1nk2&Z-m9dl(dcMcGgFlgoe=u~gJ#Rp0ADYNkuioc1Bj8CF(M-Rb|ht(abk>s zv@VZwcCq&a+FgYvsX}K{sDAlR!q6u4E0B9;J|s;y9UA zVF1fYyMbxu%@|6}^_g%`RwbpF$W@eE+_MVx`vaDJJUsb=EKBm0Lj?$vHB z`!QC2?`!(o+Lzs;TUY&jM@VL6J-@`Z#u!8<(M-V1p6?;V9uYm2BE8X?u_^OE{^$Rl zeT?3j0;uG^qi+~JNVI0HSzEj{w?;FdtpnLdkZ6m0YiO&yMH=Xk0~S)sWul`>ERCLf zj1lO2T=%W_;P^r5W^41FF3ah9yGR9NXmpiMBAfw;8C9AY<*=8xK3#qspazgM5}d1r|659_(VP2Zo9)n!p+?^W7mR0Xk! z`0>XdBjS1Au6w`kRgFDE`IjGiU9h_&$G-0*7jJFxWm(*P?EQLulJx1*r%#_gEjC(f z?hRn?<9gkC?-B9SPd~Z)x~`YY<#M@H?9|Nu$N%`hd6S0f*2>dA^@WZ$Sxu>N9B=<% z#vN~vnSa*yV#<&CT|ZN4lX9nmV*xO=t`AbM%1gu;nR}0YADk7QjZR6Uj~E(#JY6G3 zC=7su8FAft-AYQV*xHNvf~6HbCXf*sna#}Gn(}SW>$b@*%i>K+&~Ka2aHfv205z2 zMl#b9Axv=?Isw^YZDm8ls2#V1F@&e>DJ(kr{cN5F&G^S0dWbRC|QM%}LiaXWV!Ck_ds!)+*H?HMwhf*lry$ zPn5<_t!WW;c?CGk4g_XVawU+0WkVPo%TdURmNcjVP2?@+Uf*(J?zgbvN1po^9G4f< z|FR8w2_c-T#ow>#h*bbnZgVbL@!TU~>|^v3A=&r+(7DUrdzFLfF(m!?(_R~_#l6Y~ z09cj<$l|Y53EyQ|A{oj5^8G{KwOyYhV%xUs^?JQtd+(nn>Z81Yu+XWAtt8*N#u$Kad7@nWcHq z(;5guJz@;Y#hA!VeQf)_KYrgL+mc8if{`W=#!y!}0dW-ny|$v)&pnagpk-UQ8r7T) zkIRx>jIsJ~0!pB&drlR-%#5KIyiT-KJ%)h!h6 zLZS<~<-jvc;)9Sddh=w`WaR;el#l}gEomZ7TpeIx>M!hz^Mwp_Z>?E#k|T0JOkOT) z5~878^isQ|nY(0|Dqtx2FtU94NtD)$O1q*2H4{}0Tw-Ko=03)hcpER#!n&TLSO*w_ zrCB0pOrYd_Du3S9lBO57j1HU`n%kFOFp}@Q^|yceU1s+U+%(%Rm1HR*onW(YFKHl2 zh$=gPqv*L}Pf39o9Ak`a+a$Ht#?k*^F3a-i(?9u_A9WK4K5fN!-#r#e(^_k9@AGay zb^9hsft6%T7Md#~ZDt&o3V$K)0(eT^%-zE#x89r2H`zX2K=Ciji z{5brj#{l3(d+V;>&VIPVVLlq`?iP_nb#QY48Aus`t&i*VNjco%%li18wRQQo@BYia z@6VsFKmPdRk3atS{L}M(-7@n}Km5BQ`ab%$NAHooKK!s;6p*!cS=+KKfcn^Kw%GboNjWmmZ~;oD zw7bl&?$;9{vZ0Bj)1Y?-beLpFk$JAuRHV=ZHW{fvq*ZLKI!%+*veC_Cx;u-qfRGL_ z*deZlA~1j)bNc{f$mf|75yx*uU=`us+$j)<6f?6@=gzR5A|6T843eGOmI;{X7}yGh zu=lPqI(zez=I*Up%B`!rCpT2m(M45Ypi>>W@{s_pOA{Fi&t?WSu|lY6by13n7ORt# zS!@zB$}NU|NXjLvhx=uJ#-oON$u2K?OrKXXZ?|r%3{1^Tl7b--(VSG7rU@vb zAU(Md>(aK(*R4mO*y--h=K0Jl2EkykqCrz}cL#ZQx(5*8b?e@HYn!*N_l6WOsiriB z1xrwo#u~uM)O1e$ED1`)`Tjk$iq`A8BpLmDx-Zjdy)4RZB`iy^5o#(WVuJz}BNa}a z8v%T&dx9e2I205_Bn!(&C`rKC${uaU6HyAZ;&iq$*)*G074_JOf2FNvR%$7=TCGVo z7p0jB8=%1^p*g9!S4|U*xjoz|dto4D(Il0J4F@idQbdi&F02v&py?GA!~ThN;N}6U zB)4a8M$Oh=03s?we_u12q$75sw}7bnfv7|vSI3+1nirQTFeiYBczRv6>+B4T-b=*@*Z&Gl5w{mxs@1~E{3aOkv)CMVVR&asTt zBI+bWT6!YnE>qaOw@y|4v|cj|=IyIh3d2I4{{1h5V*_OFU$jb@|MQ2(BY#!ZQU~PH zS}Ud0TK942Yj;~(G1-HA`3J!=uMh@&R&P0e`+0}z?eQvnzuG{&-ei2~Prkmx`1!|M z8U5E^=T&3}aNJJ3%x+$d_~)I$IEuo?q=jVSrcz|KBtd!z7>cPqEv>mXf;LSmWpbyx zKRrCAdMj#}sI*RWcb9q^9@krM>vh{!?^{F&F%^htEhSJvV;G@ZhZYS?p;a?`>K!0d z=F^>8o!0ZREZf>CeWLYkA&_E~Mrx#LiZ;3=cRtXVY(R)$SyrZudJL+Dh=5W`2?XJ( zxSnzbW?BmXr{Kt}t->apXDcV0PF9MdC}2+*Ok1|q34k{h*lKA%5) z_;7cBE~Sw4%_Ca1QY-2Pv)?{*=gUcOd_CZZSHbS5%`5om_VtgCd>jBuO8I$g?!8A0ZwgaYD3uyP5AVH=#5GiCs-+Ym z8LhjwZR?TNJepPzaFsN@czC?Jd*9l&t!>+SZ}iB{JUL;WRip~4NKq}STBoy>2~wO2 zA}FdA!l$RJs*07O;t^laOhBM0Wfq8i4a7JOJ+nPydO{iMO4P-onnw= zY17r9Mk{6ptLfU|B#QusX@*tQ307mOR%cZkS~nu4PL}ktqGo2P<|vxZ*+Kw4ARQ=R zmw>tgw~6p*UN6JXMBP1|N;CR~g#fBL&GS5;p(^2Mo~ri{N?CDrszQ}-V#ss&1=gW=Etz_XYgpLE3ih={OyhDd@f?i;z<<6>qJ{dhih zOGpvS^SpHg%GNQ>g}ci_5l!^wA5l;}z_qtB>sQCu;f`0Z#_f*>!1vY@UM(uGj=v4- z|HAoezx^P8b+|9fg3ls8fWtr)H}M@n&?CZo_J2}fT1?gEd1mte`qq8R3Ye;Lagv4| zmW{q`+j?DYKM3g&-M#hRAD?1mF>UMKqen{r7J)!b=VJ3*=2~j4wW!7YdozPbIGmBU zTh**g?jD;1fayHXLn^=^Siu91``ogD3XHM)5sbs!crsyt;<$! zZpz611Q|weYXA_aiUj}_waIkG@R1Tq+On!-X_6!qViIZa8pquR@+;H5DR&Hqq6mP2 z%uoc`Yn7tSIob~<%D~5w9auRtpi-*UqE^Ms>SQ)YDPk3>)IzhpMRy0>Z!-6n+4|p} zxqqJg{bP>X$t8Qczlf9yNC&rUMUDL+C4Br<(pN>yU}h2#X>}m!j}s9@Tee&~fy4Qn zB%j)VRaI5gL`_M#31S}XgnO_%fN47A49&{m-h07Lxo-X8vXoLvwPPc~PTR{c{|H#- z_Ws@G+PA+wp7U1lz0LITA`^cO%5On2w?FPAN$6slProf;#wx9(f(Ywvy86doSk zTey=!I0Pss)H==eG|%U$PE(yubDgGXJ)M#E8NK)J5djfZ-re0@uh;AKy7rdg_*wm7*(N4)wIQMNQk%^t-vg-ga~FCG?qfZ zP$x_$D|5L!Tb=8CE@duOZZPY1H!gc5!G>VwmMK!rM+!uM*WD!h`20;W_uKLRYc(;wHBXbN zf_>AUh0TcQ>kr=Py@N=v1cX$r%d&N|qQxx$EsRnkB3$53m6kU}C4Fn1lz<38fxvRs zahH+69`Skf59efu(29s!+1DPe#EyQ9v2q)L>=&(_H{yG6f=c&!Q-^p3k+;Wty$d zI!~oe6{(%86hYWb5PsmF!83*i^Pi+GgPpT?i(zUoLMX4=4#Qm#Cww;?+lMU0d^rn5Pgi zG{0P~m&*k}b0=A+GS3x2(xmP&DH03r*>_i}P1DpGqC3aCd#Z8e_~_xS_1>E)td#S4 zo~l*VTIX8lTBfW$PW5!Ir)~MTZ+bM<%5vHG^yyQXr)<(&*Y$F_WB@9hl}0ThfI>|K z2*tKuu}2e%iQ0gRWyQ$cAxcQ!n}zZbSV5`dF%nZj45AFhFcquSrm2c>xRfGHgu)z~ z#Y#9*0D$Q^1)AvYZ8B1mV9I>4h>?+6jM+=|Ja`96GtMMGi)txlDmDS=-6LRSa1cQP z!a;UQBHfD-k}owZm-iCxku7DPYVXl|Z>C>_?)iR#ZnRHP;Nv z3pp*kREi|OTu^=SC%-&8^PfR-pHd3j{TTLn)XaIcDg?t=EV4Bz4}nP{>>`QhWTw2M zAWH2DrdC@}LIO7uJ=g1nH&t8_$w-wj?D05v487V6A%+XDix}BK^~go2h)_ggq#7@3|Y z2O^S9kB|T7|M|`DzW?@{-%;fM`yW4jdU$e1DU*oMLxpe@st99qZoXJc-XLR}tj|W_ zHLjm3;LhkQ?uAt&xN%$YP9wB?B}58k7Clo-NS!7LZr9~9S-ty!-+l8S{P*>8y8Gk9 z!^7j_B_gJJZ)V-MWnEP7+tQZHs)}09@9xX8UeBCO)Cr}aXiDaqnavY2B!vXTt-79?RLtkeipiu}C#ag5Sk++? z>Pim{m6QW)n>Qgymq6tyF!i=9%d%~om&M(CCj%k{q7Y>uv5~xq5Sp5UVDz$~1PjY_ zDsswp)g~CLDO$iqTPd4KkBG(RaCh&i%3Bo0P!+l16oHsT0pLMr@U+2H%_h|Xnujoi zkn83>)I+F6N~C6T7ym|2A(8@BT?*AOon1?a2C~7afdTaNQeEa+ik4DRw8W6EDYmX_ z{+Q-aLr8&WWzD*RiK!~Jj2cNfR_?K`CQIIQ%=dx*@L0ogdcb6NM}YwFR9t`q33-Sl zfEEOlDtY+G@E#G5Zm&@gPy)MhLuG?r&ZP8lW+7xn^IAGACLa`$K zDEvzLk4x$>8`MZT=w_7wak6MCQ80U)i3MPdu-d*Ht&sF_tw0(Xc@dl ze9snnCotPzDQvz3_5v8k3am0E46%ZmaK0og6Gn|FGL2N4K%_q$tVpkFnqSgHXR|TMWl$TkV4OWI31W(Vqf{3&ah<7EX(4g zC`CeR`t863%!;?9>#ULt_XuN2)v}(3+yK)#y=`jD5wafR2|>0eFce@?P5JF(@9$8C=lN$$A9`qNM0=?=zeno{qj_r6@W zd~)utvTsOLi&CuY)2oSFPOef)(icyVDf6fHmFZP(TDAOOALg9u(+7TH~SpLa+v zLgcHyk-Rj&084)8p2BUDs`EZDVV`Ht*ecg=h@6GMRDb!Y^ms$MG|R z``f|v%ZtY?CVu;k;+HKR>0$W!>%4VjG)(L-8<&m*FcpGim-EjQ6(uWz$MZ8oR3%K*VFDD9 z%0ygHrA)=D1$k}TbzMGP9`E#iZCi8iTU&X!Ltty}&9`kOJpDg&8zLf%*tQmd$)uDb zN^cZpDWFpYW>!bnKknvXKBoc^s-hOP0-#Jqhm{~GGEzui8bGKhIGns~tX`!QD<$1A zD2e=!9Z?bmM?cXZN%U~{wnan+@&Ht|HFyjc)=V?QJp$9DqbLieiz>ya!YBx36h;9G zArys3Z$g5hbc-OOCP$u<Ty3jw7s-sd@S$$}4dznYvq&=MQmfatGjm`U%fW zr1y^2Fu=5u3qbGQdq15{>(VZl%et=H)^i56)~4w$BC^V*kOuF-uIx5eF%dbIIlF;j z4~*;8lD7Xzp7k?wd%W6hcsY(Sbp83kzbClAh)xBYQ*0N6l1w0T?uW&fgZo=S{4JKto6ol|2mM!t)|=q|%P@E7Ur8ul9misO5Z?1X z-R3?S+;g<8tzXxkIXl#b6O!P3e>YDjEd~Zt$*9)T=`>rEDwZ2=P=t*h?8~yav-Rj< zL@R8ck_~-Y(f>PKuN`D@r};rV5e_^a0{jgxrx}iWAr)-vS_7LHOP`J3#HiJ z|79X#P}!+|&s$G%RYR`9l3b3x0D$MqB|k&Zdu-wKCcf_+%{@bYk+AR4;dNw(M|SDv z9FrhC;Bj|%myr=bs<*ecwbt{4=-y1Ds!4??wM1Z36={2f6{jb^F(SAxFe8HF`7`26 zJNwMaZ+3w%HiFzy-`ES8!58Lyw^EURLvTNW=Ig8P?%RH(;Mulq3wvvAUD~$w%oDw{ zg=a?}0mAb&Gs>UK-Iw3>4K12CdVs(;wqygX`|9i8zEOFjLZok`7`JZae|6^wptbdQ zYuOvhNuEw8-Cei<3?-mae*EcUL_lh()khzss@ds$M{r$NdKbj0SUULBDy66(=_grZ zr@MFGdc)S(JjGxts1!j}HDL%nG@uwe?dkrWMVIjH(^G$ZlBoo^_3nJiQc_5EmWFtU z2cb|UGy*M}IFg-tI!)6wsp__@iDuTKRnSU!t`u98LF{%B4Y@l;aQpzMy9VaBp1NZcDuV>c@~cY zLO7|^C`1w{6QGd+$I@Q!%?I z`LMaWR|;KSv5BN?#@511ntZ3sF+NCD8S0Z>@6DyAwb zT2%Go^yW|7^6B!lfz#djba&Qb%W|EjX|f4oQ1_0lhjT6IUZkp4ojoEom72G^c9G!q9{xVa2LtaT4+Yg#9s*@pGHCyrbtzQz1<&Wf`~@Wmek)dQIl*kQPrYV zbW$;~+03Zyy+?O40a-BP1>Bqutg;V%8@7>2nMq@ zt72Lq5@G|p=g_+xX(a%lGrA{1eru=+dFA^TwN_gxMG#H3YIt=oZs`~ys!KcAI0FjH zte=sKL+&ZWERwkSmUzu!X87VCk{X(^PoLvEGKW8AobFNlB`$gc(TqKD$H>R2ic-}3 za&&=}QdGmt_RWxzo4b%qjWCib%1pJrM|7b}43ZPYk2rCZWJb3{0x5g{!D0V>B=X1Z zLC4nO`3VQ&y+7&28gR>J`Q^d=C`gWlbe#Q>u%~SIK75s;0ftCVPK+Z0Jt#UQFAV@C zR7#G`w}Lz5Ik;n#BOou;C2x z=`DPX-rRlLmRx{a^UJa=E1N6PAQjMOx*GK@u52(u9u%4moE75Ex**Xj^aCI88-|!(JgudOeY{T&{H*f=BbI)(#>-_YN(9A#=oHQVPt( z0;Z@Xje^7yc{eCT6&bbTxt|{4HzI;65X*{=jU>`Ui)m3U$(G4&M3Y9zxyc?8;k^eV zpdxCb;q1+^ZID0&_F0+zB=8~2lDIAjp)k-TWNQlr(uzPJ8leFR5vnjGkU9hijLo=n z6tM&_B71O!5NGPhD>Njl0*nCckg0%WoXQs4;Tf`%+YQrHrfHh+j?hkYKnFY23bdgb zv_%nTbZ@<{z4zW1-wsS5sWQ_vRV$^ah%^)HR*Ki*n9eF$9Fmm1J(z;Mj|^r9$3~-# zO*3}5;{ZLrvgMV5lJix3&-r*;M9si-GwWVNb~EG9Rc1s#PQc8}P^+alPh#u61%TdN zRmtm-*GGhCMR@e)(Y^KPPB?&4jahV{WG8b_5?tl;adbEnqpH8h{b#P^GbQ@fZ!s#a zBc|tX|GwZpuF;X425<`&-(vg{0WC$;Ax80}#dxs0>`E6|VBWuWgre4o_j z082?{-fCvo$ER9ku4V<*vLA(}Kx`1n!57}V_1=4Htv9YqTb6#gVCx=iL=RYff0E3& zA*wJCaDYGjbaCS0x_o?s7^KGC2RolnYx_yGlv;-ffgpEmnO}>hS7!k|>y*o~c5e}J zp6WDBsOsC=Jv>m7TW&O=6`5-(MYuwBmrEPzyrhb6(SsoC94Z%F5`V~hA&@d40x2d| zh8C_ErK+f)$rxgj*4mQo_+|uTd4! z6s45XquIdO?@PAAOL`)fLe(xJ?)A{UDy3M_LBHX^L$c?jyhn5dm14Cr z5D*NkVt?Nk4N60DK;%Wu??rsqeF1phgz*(deJqNvGRWul$j{^2TL;Ne3{eB%?%5_s zN`|2bBq2vtdyj+~v_PYU0Ft|G+x-qKBp_ih;4P-&YnbJB9sA<*-~1GBVDY?Jv$$(z z}ycw8cUhU6Vb{nB}U<$UJp+kbrKQp zzFeQOac-W@DafUg)Q_T?eabE}$b2SNi_S%gLEs{c-n%z=clUK&(>i(EdQY;SqjhWz z&AWS#%d-}c?whj!fBbaa8d^XL%&@L@xys%BG@s85Z>`l@t!Qr|#ZE=t{q8i^V%xTW z&E~m?tlL$m0s!7UU}2f=3UPmiC?JRqDJa3#R}XxcCsj?V^<>3NGHoZrBzrQ)E99~I zQ$%ESQp~EECI~>0h}MIOVo9!4&s_t<8JT$mB^*K7I@p<7-a-+bPP5hG%i2ll-M0(A|*iWV$i6J#iKaUp%hcABBEt%le}-y8J^@uEvnHSy}3s*t!k}Z zJm4F?(m5^bzfU2Af-m@y*JWT)w&NSIW;wy*;K|W50Prc9>J~FT1r?ERZZ@o44`Zn zfr^5ccA)dJn~P@3a%46KytGfyJy>z@7nJDXBkvR$)Jzo-=`SeG$E=OsH%N#MO(7sg z=`skg_Yu>3?|{}?hqQ$EqZ2dP>k67+u0{2Hp66xj%ermd8QO*68T}H~+M9~qlVv|P zZ4r^Y`y0SLFDB9@?FNK7LXfH6CivJT2zxtfKYxS#KyHHu2u|`vgyrDsdKv}nT6E*) z&Gxj6i`2Aad*BLNXQ&Ek51Vcqt10bV$-+J@#jJ8SO zgTNrqYy=M#n9wweYLPHim2{|!pl4+7TT;yXw)Oy1L9D)YZR^6;uyt%JwuY^vbv*U9 z?>e6A5CQa#=Ab|g%F3wW#k-x(Q|9SvFtu7u82}u~dI}X51u0?@rWGY32#?;nL=l*Y zrd#PK52FawP)bhqNwPKy3y4y~NfA>G#3*ZztSuov`0glUst}_Z6Ap)lq>KPFxfY>Q zoI}byLRy5A?xA8EJ>6g3T_PtEpi`>;QF;LGiOKkJmlNhNg_^3t=CH}EDoq&CDUG9F zCG%BHNsrX(Wp-C;w4n!h23hZWY*Zw87(~3E=!LjXBJact!i7@j>4%z zVww(Erj%alh|aZE(Zsd$$+AcD0VDX{NsuqKZQCt|W5W^_WiMP)MMQ-H6_6zj0&gZ; zzO0Q0lZa_Rx}an`LngdA%7>ZBA>?;pFYa3evQsA+fbO1MU=dLi!qFFEW-7(3)aL8f z+3Cd2J0EnOmvA{Ss4rn=M|0{yg%pun;s!|vGxzu1q69Z-?yKnZcNA154&94mn=yj* z9>hiDkQ36wt>(y*cYrL525*JM<1PO7Q+;vTUws{U{msFCc@^c~Zl3xXRn(g!Vu-yQ zFZ+xLkpn{MojZ^CaF)rbRZK|t-bz&!RI^E0i&TvQF)}q@dhb0k`=xEmwywUd>>aJc zJ3O#8tPO2L=Rwi}IZ_=5f}p@AOtsGQsm!xk5z)K5^R{hp0#S;ZOs*2{TPGL{Dk9Yn zMS-HltatVvy_4u#t%z8$)|~(yfoxrdLKHH3se{m9mrwwqK$093OvfS}pe4KLt`#XN z7HT1CViXHis0GwVs|`uXJghmIqo<>H5AP6~yR41dnzhz3Y`SMH&M3j$b`y+jTJgZx zj*3A*fjOfQ@Gy-IYvdLZ2e6g{6$_xJbbySdKu>2$igdv`jWN~w^cdN|h^00V{MPHU773tcxa;npLiC7)Gg z1d;wL6bWq70Z1?sbC;$~Ra=on0>$f{$mn7%xj>kR$oXy-8BOf9)(rY0QdJK8JbP4%<2(&!i%@7HBD>g|SZYWk7yEshzv^p|QN^_307dLRv_U6%xm@B#DCHU9 z6Dd_i=2}NfdGFml&{Ij@&DjGYiL_ao@ref+!2nX9TK24eKrY95nVB4qf2m@4B=!5Q zS`Xy?SSd&zAUXb4D&&6qW%Sg(fJc}zp*zrz?4(OhYbf(~0V)N;k!@iQ@$v@XA-^za zzWMxL7TaNZ`~h$Oe~bVAR&c*%_>AX`t)}EIR(6~npsH?L;}8bn9@3u$Vv|xxO)0YrF~rxLUcj} zU4q2IEhoHEii}iCm`yYuBFTeG8AXy^l>&+igjxxs6c<%16ib6N3i`lkqjmI-)}w9S zdi0Lg*jm7&wWN3fE~Mr=Jgv$6R?;%J8C&3KPF8e;2N4|_)aY?*9GV>2#V- zv(=9Y`ig1sE?FOC2sLz}l>s;UZFsmTI;3jQgrO?fx@I*t!kwrIg-SN--PBVze&LmCup_mx83% zTW0i{tpN#o+=f3nqM=XIbck-cd-s;9`noN>ckclpE$wpen6NW2C?tn#c6)m2J>RK2 zJ$jFpsLKtrg-1XJb|HZrI(x{-toNA#d3^1BZYd|}vGdtq5=oz<{7usNcTpkpihaI- z93wt&eXgzoCf9i?|Nw0n4$ zaPtmE35KT*Kq?SnPHYij=`@vD>jW#2#5NbFBce-DlW>NL)4lho zN+7_l(Ixr0q^gRzF!k&l#9UM6gvs2S!YnA>WSJ$zuZ zstR){0s+O;U?ogtnAZszA=sMt)?4$|*;;J9=fEE3!vxugT&g6-9im0E~R&F!KJBCJFM`)9D+iG2PJzB#Htomb{Tq&P!m(rBC5y$ zXg1Ya=egEX+JQ``yR%KTOedslrc@HjB32;gwKGY2Er{-H;R2=xG zbJ!XXR!UaLc!bCt_iYrmsS0K2D0HL6HHS(X#qDul@5GP9{0QjBBkt@y%>1YsF5Fag zBt)f}87!NcO?F0<*y!pMk-64HeUimSV0_;lyB=!q*=^b~l+{|#=Tb^3c9^qg&2ML6 z#HN$X|E8|7DfBxIg>W73$Qvjog+TV=D)g<`{tFAIZ*!@>7TljfnEjjE zEz+|V#NVQ0M?SY%W7K`XTY-V&>u8zWOW`?EvskZsw1b15SwYmIv)N=iRhz07OU2sG z6^oyMiaVsMz;AydTtD9n;0e6HB*jhnX=}18YRVh+MFqK}E z!mz>i6G~BwKxld|6`DyYRwpZ^oKNqjX}Y^RPv@!5lTC(NP%51?fht9aqBcpHQ6{k` z2BN!*yOSEO;oXmF!~qKD*JW#wohb@2gQyS`X9y%H35AS?L;xwSki02UQZGT2+p-M~ z>#Au~oAJ6^*zO$k%%#~2s;_vHA~KgD^`&OLC%fJ#77<$Z(n9eA=c#+Lwa3;0Tm*YI zp!V#T-vyIZ%lUl1zrQcj#2)EMux;JUhK4gadeLI64aqR32ZA;jL9%XZc3P<_2m9vb&st7IDT<(A7H}EeD-$fc=gNyo9--o2l#>fL=govl;>LUbdvcZf(e%~1|5Ac2^Qp^Hb75HhnB74FrA zpqqPE2vJIw6LjQnsr*NfEVW6-C?m4QEJ9$tE;nfV=qe)7lZpp{YU)xcvo4+8xuMYp z%1R;Gdhb0Kyl|&G(<)lT2=9*YgcT9NAd<+=5cg4%RW&j3fDGIjqJs!Xpu0d|0Sic# z9f^K1Eg(wIpfm7N$ftZfDcpR+Mx_Z_)EHWI%zAoI5!FP5(3tG%Ur@K(WA2-#cFM#po8 zAdMTnUyN;FoX_VJWIqPqeiuU(mOc;e0M%f{CM#3P%-yO;k1gL~)J2C%Mr<4MUk)T6 z&(PE?EF!)wBbQKUfmN{zEqcD2t>Sd9^T|{}h#J-C?o*kl8rm5Qb|Ex@RMFH%H)X=P z3ahoIMpt6IA~4g*O7GnSCIx_uqL2e%7B?itDI~^T7$MBC?s(>&lW+={DFuzf+|^Qg zI0>bO8j(n*ko3O)OEikiGLmT_5V5t*BSHpuK-xAk)EITSlC78#Ju}m^Eg7Awf$Me^ zD^^&~MV+D2T)T5gA2$g=OF`|JCNaYaUJI+5z*NIbrfO4#vem*;r5aUPs@79Yk9!-! zJWgj8!a+>)33Wy}!A^jI6;OzD1|1&Wy*KZz`PN(7K!1L;*33?^J7ub9y2yzbhxk1Q z%nbo{%hpkgAcwSqK%~}r?`X*#!T{;lbv3hHGxE8hnz6X<$1^U!8xL7F@a@5Uo@X;b z&%g)n$&ff6P$T?c(`0HPft(byuZJ{zrSzI_c1w2)2eDE zPIq^W0w_%evg7WM4J6>m3mwEYmZwV96ZJf_*@{d@KcLD;_ zS`&?VyoPGgT9M_`?tp2L>%~>eT&G>NKdn$j&*sv}PEZTysk}eW_jCFEckk~eJDC_e zd@F9)(4N+9>&vn|E$gzx(rj~Cx-VUO*VdxDgwl#Ly0vH>wVZ>!N6-Ado_PPR&gXhQ z^;*kRys%lQs@5pB(p7j85oJes>)=#NseZjYii8wz+`K0gV-%?&5#k;OR#Q=!hRA9_ zHI&j+f=M3{SOKokfk&8XA#_(_iH295-5(T_i1C5Zq;Xji4{S~&9x7e z5@8rz*u{Hm%{)LUZoZnV*0&<=yGb;)miJp*y=^p9gHGM0w~ji43JRRoVGY`dF7&$( zxoHWP@GWFW`M62%ok3Gvx2L$Y(^TKTe}Db)2YYuveRF?*Hl`0?l&ROVmm+dfn^0zy8l^xbR9r!!!vh--@A`+<7rzEps7rx2-`ciZ*Xz3V zt*vX@uIsvXZ&b=e-vlu1O_UCUKp=BAlD;kCFk+4Xtc@0IR+~Uqr5rE5c zhpLMRrEN__baS(I)K^91UfUsTP-{(^N}-nYJP>&t0ZDxfN6y-Np@v7@SlUoIC_ z{r1}r^XYWGK7P7h-Q9m$-@kufrYg<9nclzq=B|1F@jw3Z$3OpZUDvfO4yvYO{MKUY z{j_ynJD28{tzFmcvh=NWEw%5YK~tdzqXoIyb>u0?y-3p4N6&L=yTOl4PQ*6sd~*OA zA!q_G(t|;;h^VDw10!O)iJyNZbN@x)@@>rhVj6zj(O(W&zkPbY$_xEN4%v0VB7(i= z0F7-6@7_lNu9k9|=FC(wT_KsJ8Xqkyy_7N)t+mcmd3T!M-JkBK>AMg2v${ra*T=SO zt@mZQT(8%~W6h0rr#n2DTgcwSV<-NM^MeR_v_Ob*nlQgB?>?OF-_6rmPp7F=DW#Zb z=H>tp40N*YTU`%I1ni{{63IZMbvwy)NB~42s@Zg!Vlo(*jF1Wg)R0o?wnLUV$VUMR z!bm5hv7L&n6+lR)Jzhwt0u*-_HR&PZfpHNcSS8D(z4_?kO>>?D5_2*zssIqULogJ} z0_YA^K+t?7sX=LEcj*ClJgy6XfNH7G5+a?5h|6_#Z<+)Wm1=sL>)m<&cK+t>?(Y76 zo=-)kt3V2+$ocfXR?Bc!=F?P9r_-s{S?*oTtjy2?6iA^>qQym^Y48TmI++j<@0)L} zt=qP)v9-3QdaDmTsFr!e4s-pZ=Edg^9vNutde`|hlP&Y>BOlLsbf`HV7)wGueVClc z;ZV2(pP`0%o*y4CKmPdR)6>(st|Whd{!UC)wS`|EmLC58{r#W*^rxRbe#{`ORn3Ya zFhNxx9|z4Q-*=v8v3d5eI<1Y%x~*$lT3@$h*Tj5h0f`7n_zV(bua(5`WY7)sCV=s5 zWBZ~Wt3zBqUR}N0AKwh_wgKr&p(P+?%)V<9;t3>Dhjrt$)P# z)Mhoa@~loL(=@%izx(jv!`t|9BPtt;1_IJ>kF-_za!7*y1VRAo%ew&>}uzWet6{kNz2T&=Ja zDZ79OfVdNA6Xvp(k`)tFLA5pU3xn`7a)y$L=r2_g8`b=O4%N z^@|ppzyJ8fi^m%$rF*-u-e|={g!HYHayp;x?oS`?-+lAVH{X2o&E4q?M6zT^x;Of+ z3h?c{na#CMrA*bPqE%QV9)7$;^w!s{t*x)U_ZHs5uWPiiZC8i%lNFYSa z2*5}+Q@sKL_mpB(0VRq=F~>* zE*{bYjoQ7p)&iSI%0ST_ry1|=Prv{E+duu``|p4E?RVe3|NZZN_wL<&>7SAeBkC#x zUbl*&usBsw1=CE*po(hHGPVyXo}KyMVBZDBg&t&~Gh7|$&ek)3Z)ZF0(&?2IG0=hqP7&^jKWT-D8NS=%1OGo5d>Ze~^|^WFMpnhMEvUDx&6 zd*7Oi@_N0B$hY5q`|jPnsy>8AgH8^ej#%|MkDs* z{fmgm&xK`vPB6U<>o+{k*CGD%llc5LfBk%3EH=N8BKcb%Z=HWJ+qRA5>2%5pQU>=s zjdyp`?|=8*_uqg2?YG~)dw+L-I@emgw|q5vCudcirm2(?-<&1AD}{nU-=b}6wC*eW zvTaN6o3m316)3>s(YnI}JrLPzsi49{U37?{WWsK^duLRw)lPTQ>CWmTP(}`J6}oaPqW5=EC{q6$+H8EzYc(g4XS{UDX+g>@40 zyiG^u@Yl+soM~@VAVP{;Mx6N0ldlrNu znL4v-4X7e$9>u*&KwY;MPH|BWG-hO1hODT~x%RDZBsdwSimBjtAMU>Y{kQ+ifBVzF z{`n8z|L(*4_xE?_Qz?Zpi-?)PsudY&dQl3@OlpBxI)nfK6*?saBqqw)`0BB(eO=me-PYcgwrp)%-Me^4ceEbek+9Nlyeocv+|HfPv-kl-?B>*wo+%i? zWnI@nrW6VH79raPpw{Z{B1l9_MAr41Pu~0~Empq$_QUD!9ke`MuRlJ0e0q8k3{yZb$L{i$}bed{8k(^H?#VIYXZtrS!S=;I9dRf-3yLU!J-g8F|6w{l+y8}fvT5C*$AClocTxo)#g#xCV!r(+I5@$Do4d)FJVJQU? z7_-$NjF6-&0zfbrndFclY;T#=*hLlr2;hgLGqdN00#XU81hb`%qc%WNwjuoy;n;fg z?rU>*ZY&LK)HZerl8P`Zm~16Tz}3?ngIGHVX%QZ!gC5d@gskBnXdByx0?hC5?KkJ| zzB&J||F=K?;rHME%b&mh{<{zN@8)SLrij?|-AQuwE>?=xd9ta9*}AQ+X6Y+PB}W5< z_o38=fHHj^SZ2;C9Z4Rw%B@v12m59z;Afm z&f?F5`>UuTBJQr5KH8)7L+J4jNtmTDm=^Qv*7K!G@wzz>z4z{|*6N!_^z->_<#b*8 z)6h%rgU0dbA#`HFaNmOGZ4?HEHal9WVh%PayLEq(bAkEME+pd#1BxpzqbNLKs%_wWDkyYK(?&)>a& z|NeL1fBWsX-_G-Fpt}dVMX0JM4KiepQsBI*cmslTXqZG`1Dh!+G!w0*!ktr{C#%K! zNj4I#b#wR_hni3&50-V~)Edp(FTR{=`aJT2)2i0~nCleu|B5E2-~ zav&X_5iEHZj($Rj2w`fT_7u?^;Bc7Zy0gmM**vz@x9+_~C!5RKv9=_VwydzG=76Qo zgb1m|?wj+s-`ver zOT{!5RrP54;Tr%@fvJ@uQ-P%<+d26|X?qSN)UH?qsxU#2y}`R=a3{e&fOBu~PH#&0 zm9edKha);$>&q6)+8-a6%NkGDZCQJB>m;<8X#B>({fpD&7L$4PBc!HC)vnnBP=%&8 zBYS*D)oZAz?@E#E&h8rk!bw5*)|!ZP=ck9KAAbCpc0=>KyIFMY{^>72^sT8XBi8l$ z&Aa!eX4URar_sm{JX-P9l!7yG+%eEj3(iTqY>i9fwnn-_y0bebAxV`;IuoGV#;paZL2aQuq*Hqcp$=YppNbZ#lWy~D-~H~J z|MB1c^?(0={|sHsB~Z%T?B-_(Jbrpw*Y)YLT(9eOZBLi=gVf!9;)r_rjZFmu9~^gi5KK&YbdDW)q-u&GCp|yGcR*Ba>%O(Yx_rD|mt`rX zynFZV$DckuJYKeKYwPyz-Mc@0|2;*oORuF*rC6<2&$XO--yR+w9v>gQOzXN`E|+Cl zdrw(84YD;CQD}A{Sa+t{_Wg(VNvFWhJ14Vg?zwvW;?Ng)@bNHi`j~EGYzm^~QC|J! zu};1fL*0GvF*#mcfI$K|{%YUrzLZS3d0bWF22t9-`Nj0BS7&>B0$+UoEy@4(xYcC% zMaQeR9wKPZs*;1#lr<1`-+VirPJjB-pZ>>x|L=eN;~zi#{*Ti%)oDUeh)8suCY6Zp z6p9WO@$)jQixwy}#E9lyLY(4Sbn66ZW(?`EZltSjU3w^V(y19dT)H!}cqyn;nM<*` z)LQ3xwyIW@<{k|qVwlR5S1cPqYN&?T34!Q)1fQXAs z6AsZRkzDXpwL&5lxPg?K3qoLJTbd_JOjWc5q`5a2wTYBKNgIs_0h9s{1CqmnjO-9a z=(>3edM62ra&I4K91`0}5?wo%&bIm1y}7imOGw+KIo&~G4Z=ZUDp4x{tP9o^9w`bh zEhK0zmS#t*2+3>PKAiEd|N5{0`~TyA{qvuG|NZ;(-ASvsAr#$IpaC|J1J~06QAiOn zSP_-h1Q`8cI?UP-4p@oay1Rr6XhLy@x8551DiO69>PgVsWw|^&n%qU8HNURyy0pio zJw7cD*LGdKhXzzMcSRX;;;s33Ox|-Ea{VHE!-P+3;CqaZq zDMbXsdG+RS#*PaDqCk+^odnA?1IX^SQfB~0%orUZ5uFs4RQ{vMU?}gZJQ#|$ zNfpsBFa+$KKFkDqgm>wIKw`IZow#;m+nlW@XdoyCRJEy;$x1b=&}n!a5YXLfJp>9p zpU1TvFRVTD6)MRP2uwnfAL))Bj3D6YyW}3eYx(qmJGbUri?z#| zI_4eQCR@*39ubHJcOXy->()ecTcfq?LPQHV#X(QpOX2KITs!Iq{pQ2FZ$8|gim-zAtJU%T?YrkB#%jRoib7|o{WawZFR_A}YUGi-h`8=ij(vdp=?7Hpw zOx_qs=&tof0M}(57a&D>$Ar_V6jiNOrfEv8pVpdh?F#nZ+tz(2FOm$$<#OqKZm*M_ za8>I3j7=+_Sy(0K(QS$;fSA?eXAXHt0yL|M@^i?Uq&m&>jt9SPaDRope165hl3(u$ z_+N+h$uCz{U;X&cI!0HO47GaMOW?its#Z?ZHy=Ly;SYcK^B@28hwp#??!!0hc`l`B zF^Gsq-}`m77$x+!Ld$?a!U2E|IuJrfS~|2f%~99Q*XGx4+gfZLJv2}#mFUfV-OwB) z$^@J0G|%&?CLg2LB1Pas&nO^(IH4h~GFeq&b|0q&M>RE5=?N((%$r)oDAe@1PFilN zlNM1?)H*c(M8w^DjDqGN=P-fNP-+YchyWI1P7A_Q*b5-h+8Eqf}YhI<1g!s$SmSA;=_v=5R;A)}nB6dgv`L;&7L zaX#FNXWxV(RA?l_B^>L9t+nRSqlfg(uTTAYji+V1T-Jx{_IPbi>-Mzxb>re#TI-<^ z45YFE2gv(hEWE4ShLQX#0KhkP-f=huG4Y!Nw;^+-9RG3vGm>qUfs`$a6VV?0k`feY zsuaN?Bsra)1)yLek`e>!y5+k((=EzjfvwvKyAmi_AT_ zj2}LH_`@Im@GpP*^Y`C>e?FfFzk^7{V`ob*Q%W;c2nB@15MD}QrU;}payX*XgWln> zyDYV3>DRSiuI;*Ht;~D(b@S`eu1l;9P?U=4e5%tlpQmXqQpB=(Nhx7a4H^jFg$V&| z%}t{URMkXHtfnuiCVL0jbf>~1FolE*R1gNLu&9WLLP}Z8kla3qpj(e&!GnzK!#HiY zkhy_CFoFV@i@1mjNb8|abx4o2A!R^)$V28Z@IAx}RC){d5ywYxcW2Km-kk_X^VnJ} z-Pg@Muyr(#tw##QkT5Y14+k7BR1H!B7Nk=`Bq+&Qfl8!3R>7&@{{DP-u0@F0&|#cF zBG7t^E(JD8fjRsNHK-#BB2Z>Vfw)Xna?|a*Y)y0oi!?^i`!MCEvwL{k9v%@?no4!= zm+Sh&U#_1%eR_C!T9)PE@#*2ZUADe8URyMe)&WEu2`(g6_kYvL@;4m6Jaf;m@Qev( zPA<=rGmfW`ESY3bL&D(<>4*S&1hdaAkT_mz%{zOE13^+%a`Y$_n`CI%VzMF@TM^+0dNN}!q)>O6jBSmyJv{_0o9X(g_A1os1$`I(vf zTd>Tlxc{qsqF?$aAD{|`fB60H|K(r)^w>mu1;9vd>eQ z&ZjcfGL>43n2Ism?8V7Q?4KcEAsn%>At}Mghq;)36RdZ3;`IU;dBam^sEkQ7wJT&Zowdubl5yR8D;D}|7=%A z-3NhaEu+zQ*KQg{-Y)LG4Tb}^VC!f-dV>?Kb8~nX5A`6)p1w>S5in51MKTtQej!S( z$*FaVAgkf-jQhK}80o8q&eK$oZMD5`+Zv%zw`z5o$`xWL6);Ez2pZJjF{-0&*Yk|b z-#a5x(P222^yW?zCE1#o}pLAp&XQqI+0^M)U_ff3Or(sXlLm*70=+W~`&_T8~y64Ct{7E8$ zR;G?lYA&80+G!47Ix|-oZ#Mk-SS(bvh^jW#a)iEO5)mG4&q2e?l+n7A90H_hI?Q9W zryqtMWvcNy>$1H{+W}Fr&h7ZtC^{{^XBRt;pHd1J+B5JBdphA;4O;r$z5YmK-q4GM%3{w${ z2-RZ&R;tYmF?}@sMkp<0xZ?z;Q z(^ETU&)iF>yKgFMK|sM5A! zu9&KxPo)|(n!-+HZrg>F9)7(p*B%n;I?XJTT%ZNMK`J~TAzBcIbwv=K2+<1P4H7YH z^kZ}*)57k(;qh_#&;R)GKmPs4fB&~1|MNe8{NXP@T`tSDds3@i02QsG)rRzx$S9Iz z*8&5=_%9aRk-|jJ>_Fs?Z%=p~@qG_DfbWk42=1)-1ZND!tVyP2E+mOVNvkL5qM;h* zng4^87-@JmmnGpinD3J9qh0qH=zYoaQdKi8rG!wS%z|w6)J!^}`NSqK-Kped!THGC zkFx4<{N@E!**Wj8c2avx$E)D}7G37{58uk%e@@!|Tf;JZqX>06ozCZTwx_!Lx~|1t zA#%TDVFVKJ?(6_fNM94*C7jG$(HU`N32*2t+KO$7<+5F`zAkNPm&^6>@$uuwhaY~r z{PcKvx?pYi_|SXzZ9@>H09LXVL_rI&Ae5ml!qLY>cHN~8a>7$Y>~uO;i&{{`g`t>C zX<#qNLQ}0)OiU;Y1y8j|xFCd)DQpTi6%&%3(R(y$a1k+xY`ctEI66~W-AQgydiSk) z3kJ2*oM>T<5|k~%L$+XZM$jYJo{c7wrvQKgYu%r9^QAMAEi;Gp-nZ_N>qnpknv0Vj z>JE=cJ?2RH1W_?5qO|Pl&lJ)~XKESa5P(XDYZy?9p64>x0`cBj-^$0wr&6WSA1>>Y zvo>^@Ta-I=s6q>-39(5`;uXn<3yOE~hStzmq_eAih~WX$TWi;CU6yTI+rRzWUp{`i z{QJNC_@Dpr@yDMo*K2qO37x^w&|0+=*es>07*#-68l;da@m~;=!A&f?edMdjTs$X& z0G|0jqn!10f=%il-_vBZ4ug zj(J?KeF)?!jp0lgQ?VBreHC^u4$9La-_suuoynFXC;a?Z9hKX;eyuiboJg50X3*1BG$H{h1CKRu@GUXtD;Os4R)~)`G)iX#tW$fic}g(Gz*{ya={L#y*rT}jeZdGBu&KLS zGRK>DCh{R3bQcealtuHTwZRDx%{u_iI;kTf-GE(HnWPrU0w|yepiC$grD&aOu7>b+ z-5wufy?!c1o$=|qUIJ4H>Ro8rbfFqbmDr@7;9aexld?vLckLa0)7}vqg3F_{wqDoe za=BbCmrsum51*bcOZz|mkN?;88jlZnx*{C4*!lj>>a^h##HbdcPIVdRM3G695NKje zF(~u@oZ0e~BL*iNz~0X+`0A{``9+L0l0%?hp8pX;n{gCVb7R*d|3-Tck6no=m}!qX14$`NgvR^ z!)$#14pQJsx50Yw@pv>r6EvWeLU#r_ zzyg)59#Zhecsh0f7)Z=BE%5e)KRFLA&`v?Bd|M~IplV2CCYmZ6IL?@IFIKXO|6sQ=_F>g z)XA)h2zY-p+7q)15e+UN(r}3)4uKQ^mW6IUPo-hVk?k!nBMzAj)qiD9y*2NS7s1={ z2R0w`{C2{tya6A{?bF8D3965^Uzs3^9K4xfWvnXHQnEJVeX3&A%r7?YP6ug1Zyzn< zq>`xxsD$K@gD6e2c9khSK(U&)kpl-JX#iRm#$onkIwXxXJ()$4b2gH9LeW#`%_C-t z8Et81VR!cNNbrAsSmtj&KHqTt7H{!uz5ZK!4(?>gS5HpSqHWtiN^qF~O7G1D%0{~P z-nYh{T7a5fV-$4u06k(^mWRtz{AbE;Z|-Z4zO?1>a=k3=a&@3TJo@EwS(mnK(HeRO zN&w29%MFkWaD|TX6-7LG^F#^`cBrMQNReGHdzz-X&Xdk}bvl>nRO+ctXDd@ZsU@Bx zqZe3u73wMU^)@27T_%^a9`v%Bv{M){|aokq-2hzIVxWq&95q<``9XZ@vha{oV zeOZ?4wVr@YWS#K%biJZ0C}LJ74cUJ9_|wOS-~GW-zpRv^y173-Jv}`JT6#O!Q`eiUE;>`AZ++mrj-& zH2lRY{MXz}fARR@yuMju`0BwSMlF$~EDl3Dcn|KJOWa^7H<#iVOaydLbVm&IY_ZPc)W zTCCQ3I-Tab`FuXllbz1x{O)vqH_dlbJ=c1wrD(x1>{9f6CmXoaJBBodARHF3GHHpn z5!fAKK~e7%?UJ2FL=SG=*Dbc*RxPdhvi5E3TgTR7YwV3Jm^zoE(44ekM2Uk+G{QOn zGEyXD#Oe{?Gh1^E3sxptPf9RIPq{M;7TwV>d5r(PsQ1-zn|>qfx_$ig_?JrF0q;~e z{rJ-ZeN)CXou(7Dc-tP=wuOFZ9TBzG+FH9@t{*>se0;oo_uY4=`EH(PQl6fkK7M?7 zczD>h?eX!`)}!}M5z|VwX`0j~SRr(hVpbs*F|@04#~GvC^$bQQ@fO68l@$59>x$_!tqUaIdfx(5(5Z@zI zM0&?Sam-Vf3F;zNvKHq9cr>jZT~!M*{`^xj_uI|a=Z}AW{L&%vYFqbLbM{{v^xu5G zym}5vdc^v&+q5yRHdp`6+_XJ2*s2l6dCQ$!hj=fJyJa;^?>08 z$X!HrG9;joChDcLBN)VH}b?Nv3%dQb|LydzFMR5JT80gLx{khCng8F*3j3Nda$$coPHsvh*K* z_*nX~-eG$x%cDJfyl8Y4OjFDo#md%W+uHdDB__x$VvmoHkB^Vr)&czZ({-Msl(n_? z>C>l=A0MvQtGkz?Bvh-YmNJRe0D|H}t3aRz74PcoMnX#hQcQ3+m8gCmVf+h&{f)ZX ztGeGm;P|zYwpVfK_&B$(e_Pkguc~PutvT?rM73XLmC-($5eWmadEPdrya)E@GvS&% zjTAY}r8Y)Go;ag#J+Z;?RN078ab^F%9xU_b&HL&-y^ZX@ZqEMJtL+ni+)BbD1QI{~ z_~Usl@86wJ*q7ze*Lkw}RNM8EpFDc^*0yDB%eD9ZyWjn;P6pMrwaa?FuFLguxm+*T zc5Z#`eG52*-D7oZfvur=YpsTeT0x=0lqRhY$6%+3h9KRd#s-?HD4?MNh^cCa*f0#c ze|LWW{{8*C^WFXFyYJrDbGdtWKHW_^8>)zb%7QawmwXgtOE>8@r^%h~BJh;%=8gqL zEhrEZt5&DAY}?kCCWsy)&RAN1YW=co>$d&0#K5b1^bqgTI$NNHlj1QrXgk!P#^<|p z>1GI zC#AEuj}LE z^mg)o+oxzjX+> zyF}!cUjrSXkB^U^KF#-c)9Iw``qa1UROP(Pt5Pw9C)l~I?Rr_4Wl=4s(@BbKyBJTF$dhBe=s9ts+N#dV^$nQ9?u|Vvm&q z0_NzB<j|RH@e3R>!6L)3QBWw{=}VUg@TwuG8;EQ_yZ87kTSwN^t0eahb z0WE;eb`1%fX3P};+yUvQ6HX`lkH0LBPyXXiOGG?AKK9PN+crk5iMFq6hm<hNi%&(Za ze;(ZBMi%83mH|*inH5vfQO=?QO5>0$Q$`4t3?PFnCN;t%9Mm1|%U`9UYB|m- z#@5pVn%PE-6(TA43gqIQ|Kk6jGxyKq`>$}8f9-ro*)?&;6bb;*UCYDess12$b46^u zEhc=j>xc8Ks_9P3=xdA1>eqGaKYco%n@!fkmvvpY<+7~H(l5QVwq={78i>(F8B72= zT?-_lcZv`pMkq zh#*yEB=?P^U;rL!YETVCfe8dbiiC<-6FQYwZ~=i$E819*JZ(F zi$w1u2oa=kSi0yXs~qN_?CS00y5E|N>^J!4mz4H(jG$z!hf+dBGggblXkW<*Jf_x{ z2gDoEEC7=-nW#1uLeL25(AeGafZjo-PEbV0y0*usk7k-O6Hk}TetJ|b*?1M9R!(MC zYc(@AP*hY0T|~wL-rXa7XvF1`L}ts|7^wS6D}-tPr~9&Z6QI8$)B5Fr&gb7AA@D1+ z@7GG^e%)ilqR$j$Z$YO#L9^kkB$9oq%H)=Yq(feWs0uj>96D?Q_oVxo0l0TOrRH5w z6)E)tB<3ixD`@8~JJn6&C5NNO5#0YP!7{(a@mCY}-}=am(jl|yx~?BTdOM$r5$@re z63)oXy<`{iwq4ily0lA+m8?EHM+$V+C4-pXl&dXrMYg7rNwoN>x#>U?&`Uev4{w$jv-8$E~&l? zFr~|qjw>=sptpJY?Pou}5kepVD(FLLj3FY(j#@fIL5d0o3V+<(+oAPl@B*_oUQ9}% zRfu`ewWQWp?#F{c4_P-qO|q`+j`LZk`4kb|@$h&(-onhxDq*d&^=>=)GQi~Jr*Aoc zU?6;e!dYDiC#Pf7$nNL92jH6@|Jw2Mr`11q0{+tDi}&&NBZwT>3;?L+Hc^4mxCEqC zRt%ql8&FC_6q0C)3~K$TqS}baIwUz%BZ6914zpGf&7~uEr}UxOVgM=rTwY2d1>}{U zJ8q)kEsy(lW&V2ld%LFX$kFq0ypF17ui!7b?q59KdXcwu+us^$Z}HjRdj4CluTlUE zVv``5sQ}jA2wj_qEX71pGB7ATA|e@(MHImfjXJHZt=DDpuu{6nM!ixt>UHT6okU0o zg$VC-O6xQ&-K2TAyP4K1?##i*k;d*7UIKtKRMpe~@7;@-S&`uReE;1y-+uV;VXh{M zd9Ls8Pa+hBq`hY?D7k(NVvwXtu#N{IMnF_E=U|W?y+^pq<5O#`Z!IM_+Sb?B*Tzy$ zagEL4FtszS(|W1?aP6dkMxux870{ApPLa@&paftT>VD(a0XQTkDn7etB6937e&+ErLDyH`d>*52O+w=Nta$7} zR8^TlOaQFw3Ls5kIM6>KqKqvcRLIak4EO~B3_;3#go9w48{*?+GL7Ia1w0-c=e|Qa z`<5smLugOSbKL%#hw87m^4IdKlf=H;={H0x?`eIr3#H)X}LbjI<<$Y9iQ#(0T-j2+tbokibm` zGCj^v2{OR=?z`W;fB*h;I+bMFOXd_zO>rB_@&ZCULV(^PGe8wYNq@Lq#u5xM8Sya* zi0A1;Xbbf$|*)5ubk4S7#0??6>r1rCrFp!L!&-*o4 z=G9{RHz(S!Zu3o)-&2ttSHds9um7y2;O+C&jPLu?GX0T&OK59Bw@@eqLWQ*JM!FKF zTC$T9L1#pCK%uCQ($~d5eSG@#@$$n@kDngbhwJuuS=Wt09Gt}r>lozpcvfo7sLfrL zPhO6t| zcKGKz=0-o|qMn6Bu+NCh*N^C&XskO)ZQf*;0y;kIXcq@azm-wTgyJ}!eb@REu9=MO+9zo;hMEPIzsvpf7MbH90xVBh)ekvh4vN=gi! zq5Z}8*H;B#&v$@G;UMF%QsU8}0eQI?AULQtQA8q}*N53)a8#J|hU4^Gd8od;G++4s z`0)ttziym<>t_GfJA>nW$L!hXSWQnhApxhzK6@Di5>1w1Nc#7XKw*e8=ycE-fT$W` zh&W>pA|jUh4^XJ06k)g&QF;VAUCEw^UNF+J2vEzWxSr6o6}dZ2@7|wEDI`;c%A3Mt zh<}B%MT<@PWW7qME?G*-tX-%gP%Smn-3SbScQcrQB6RV7eY`Hq^7M4QUa!k_y3 zr|WfT4-ZctKRtc=c=`0WUbcSS-~km)#?j6j1Lya6h3i(Ga|2#1CpY5pzE_afJoN=QUoPSsic!8j1UT0Iy^!&&>|4fUE-(|W%LL-pg>Wa=5l{` z-g@Y^U9UJ#;^zW{1nE5*f}H?rtNmm%)Cz!zMNCx{#b5=Io~728>o`rDO05XSvbJ@5 zdU#rv<$77x>vg$qm#6E;Pmh=D_Vl@KB0=$NeRqe;nD2+U%B_p$&1lb|a+|jA4O-;T zhZ|ts$l~|szqqPz4*FMD<7IsRna}?@i@{sRTky*-J-&FhUpcE+unaK7zOyxtwOfUSJ@fLp($_9%4ypqnK+6BVP)^5Jzfy#&JvY_$!(F-?LHr>Vg1>!E^-$ zySOY|n(kiYB9g3f?!br$O7hKB)IhRQA}F4DgLps)PZ@HI;(#essNyQVt{q{;*ji0^ z$osafjhzWG**q61M59C}qJ6SjY^p_Oh@fP`*UIo7Q_~I>fEq5|;Dpm#+t$7;+p=uy zx@}8)e7r1A>$avctq3%AmQuU#Imn2BA_u8)q~U?r;wT4idk^dKjO2{p000c5geci+ zk~p;phVBYOq(IY@3U3Qk1)p{MZu7pPF1rh=FoX>uzNgL!^^E*k6cPv*)$EUjq_|Xo zjP3(?5lJ3Yx=h8eVLOBaHN~q)j^bWsl6#B8ZG6w#w1{MOB|`ymT-4zo`trIS^YpEG z^ePj_Zv*4}{l|+P_H(klLv%NH@MRi$`+Sgu#LHfQ*x{OIoFpQS%P2?E&z-B34PzoQ zloJ6Zk`+msjPzUv^6mjKKvhO?(EL?$XK?Gn`m-|kS6i50blfKFuYP=nWyWPXP?|Si z=kGuM=KaSndyD6jjjU}q-vBjH5%pBTbPo}8sEz&;xJdRQi^>*IF_5YBG<=eHdjWUk zARmk2I4u-`+E~@AWfdJHD3PTX2oPO`C8fOy2DaqWU#@$l7( z&hvbC|IVaDXX~*x2Y;y3G~eADD03hRz@Q|^52^?t4W#!iwsqUK%hU4l$4?(Wetdeo zK3$i~(|TR~@nN}M*Vb|%SbpHRQCEcyQo!INj4ym)W;lXu{)voaYM%w;8J~@#oJv_n z9q4yP!wi-bDht)|426Auh`fZ__pfE|^vuB>Bpf+FR1pCcgdoBtOd@)8HOT0J!cbOZoawWB?;y3hud|j4%8Nj+eW6K|LzfB=#Sl+}x3jf6Ti5+a0n?Lg$aiq7@M$ zWB)q>blRB(6rv?ZfkKK2RCQ~Mgk)ZqFDXP2GL1R2^H#Hkt?%lg$wbr@!xLnuEvQG8M z-A%O@t*T&nAiX%c2T3pUG&Oi!FU$4m^5b8A`uOqVzx~_4{pCM@e0X@cUSnCYHvIqE z`?n?8aa~&w9027JnQQH{>e#8WYP0_T|LTW+X*;`WUsh($2zL<}{Q!`nZXWJ2W6sQ7 zc4%Q9a=AoNBnS?GxJ(yZuc$Q@y(G-s5t+lm#>3f==7z_1IPGYQ!NS=Gn%WUp+Hi(c zlK!+y1Sd3B-&TwBahHCuQrE`+utq1tJULd0pc6Pz1QZYkQJbq*sG?c0Bm*f0A`k^K z7kTn}^9cPw5tB5n%b?R?vDK=`_AHzr7x~Z&BvFMjsPX02=j#pCr^9>fLA}nPQJh+7USZ@cG)CSu6`hKw<&Zi#9ho8l zY1_Mzd;!?W^`|pb7Sy@Mhl4Q%^m;1I`uFqt29?$Tuo$*fE+A}A9@K%o>O0)T^*0tiIaD6D?Dyd>rHoX#)^ zuI3W5OY=XBx5!qtvvDt*dK~~!A&cVEtN^i=gdL{j)QzobNZc;1w!fvGfetKQa8OLe z03c5LDGgQP~JsiFJxGQk^rHutNI*<`Tp#X_n0ccxYW?ZqGyyFDf6u-Q9 z8v18IAx)_l5tdXK!BCYHcYnv^;xp$cnMHkV4nHJugmJXf*S0KEHU*w4S#wR5n)9W%^J(OM0DeAOHvI8ycsAF3 zEwtZM_$3J9R5WR?yjuF@pbduyyYuaqwr-ClCV$jC%mnF7pcG*SGs;ZV;obht>NH*7 zZ`ZePm)|abxZUVCyZ%AD{>$HACgo)Ud~&bvZ@*oy*H_i)`ZB#=4ZOViRA(2=wSN2d zew*!&KfZl?ud37KxZV09uRnfPCkbmgcSK}y7f5g0MLytSKO^lO zv3WT(g?moh;h~SrwN~0<##a)0P~!p;ku1YMg(FTwv$iM3jp!Z&`?8~7Y#2H35i~c&{VcE)8YXguVghQCT9~CdtX*5{W*| z7)fkBEh9a(D3ZQQd+~4xs0ad>CJ8>+){9oZTc5hSC7f+7%U5>yuoLPRhn5~qy(79{ z^jK`No<%{og|r*0J1}j$tLC;bwZ{;xL_-B6xzW&hN^HY2O_-LBQ`e!Meax1Ul?K(UymSDhQo>~^bfZ*RBT?VmH)u=yypM=0q{s{zR3NYPv6 z5%RZP=HvCZ`GyrIEP#nfT|9o%12*rOULy35P~njqL>alFvRXpmQUstvtah{ECpJ{S zpV6iOuQ7I) zHi7bB)5EcA->jZRRZGf+5o}8`kCH^(cS6J*@7LL^zQ4a;Ccv-r?c4wR|MfqNI#>Vw zeb(#k`u5xJzyJRB_V%}Fsw~u(m)XlC0N!u6>-GAdzx{Kb?YBSPzWs3p^!1n9G`*0V zjZi6?M%Rc$v9+UGA!|az0Pk46{y1vm*lqY)``QIfZ&7DfxEO#ekt3sW01ySOv(7%^ z!B0oAsU0sX%C7z~5Xp`Va$#!P3K2>BcO92@gw8!4NH!6Fg~fYg*7YBiJ&V6b%%d6# zi{v8V5Z74$BB;o`V!lml!Pf4jle43;{?bJ&h!FJNIm?#7X(P`|oo5j#It93Co}O~O zz18ZsYV$nLvt6&(_qW^ac6;|lt+B4D3~gRMOPTYS&jV-aGIbQ@V> zOP=HfP2B7IZ-NqwN_4-3<(mKzrKpuBkb8UEPqO&d^i?^0;)m_o96Z;EHFDn&&A$FS zExGf6aqT(8N6Xap=Dn8BMSd=C*Tm}J=*p!ADswEeT9G2nVNqownqEdd)R*qz+!Kkg zvnxs|VP5HuYBkc6f^!l1b zWF|nkHO0N`h#oD1NUJS7Ty&)diB`#iEr+=v-fj=KBU^szu~T%rQ%5pa3jmRyRVYH$ zo*g8riR34t4oA|301{D(;oi>oKM{tJ*1E?=!fP)(Hs+Yxec`!L^{`{nmOAb76W~H} zz;KGpCDGI~B9eS4m$XLD0&P3tu?cKVsbZ;fR{%DQT^IXBq|@`JT!O8mM79`nn0a;n z{>SY$&+qTI_iL!u>pa_R=JHEJ?kn1h7%wZ{730*LhK7spaikJT8z+EtC}T)|lStTiG~x(ktBL-i7j$pjt2{<*7y?2*@y?z_Vh$VR5k3bRHX<| zAtKc?F_@N^vm)YBE+|@^UL#|%>wJ4Ra}sV>FYi^R%eR@dYPWCi zZ*OmJ?hc*Yal08Q)hpp<_Vzxz`)p$7Rzaw!fW?jG$X%~+>lje&ogvsD_tMcNJF|;E zad-pNF*ou?ZG5@+vB$~a@5AFTkY{J>ahyi_hA{f-PT zhwx^KGfZ2xBo(G9^oS6b&I+d4DjBG8$T2dCEN7i;q$r!BAi^2NOHmmP*U65u1%W65 zK|$L;Kt?$SkXHL>F|+DzQFn-lGF^-N4rd+yH)scD zStuy_3WUT)jYz@(LlO;jO*&1LSZzYT6>a|@w#=T*eVW6^>{~5;ClCFwNuP)9BNd(_ z_ebh~1i8x&U5u>x5y3M zIR>;`%A}0IGHw(lCWrsWZ*Q+J`pZiZc(r%0u8K0r?{DsMeXm`{^y+n<&CE)HJLa2L zVzR0_nfdGl3+XkuISCYpn%z8ePC7EkI7Dc55O;@>I@P&1R@jhZACBhk)JLa!G>bulhZ`b;EyS~3)ueW-usOAotjdKkr0u`A8 z%LGXgiHVo`!z2-@C+zdtZRyYG)EL?^#>?nES6be@pbCb`^0M@Xv*9AaDBJsz5>BcB z07Zd>623ST@XWh{2nwS2bndP3Uu>Q7;o}}$JwDyA2XFk^7r$GZ0O2d&my?YUy;94p zwPKAWcPT!ic!`8kFaAUXv;e7Sp~^3pi=mo^k4!jGImv6CsZ&HLpo6oMvkKqd>aVYO z|3#;g5pkyp;?r%O{k^_dCruR76xkyuqtMkISF7Q7Q~=0D3jkFb=W3b$4QCBbK*zk%MeI70tzJ9ITAjMWbil&mQ_mx^;qPs%7CBxclsXBE zq-=vtPK;uxU%)jVm*6c;Qc2byjb0ejh@}nmUChJmPT=+R zwQtz?0M6`six40pO)iuOGf*N3Af+g#Fu$FotgK~vT;}g&XT2WH07ziOBh9X98iY0~ zquqLwfV9`sFT?q;p>^ls#b#0xgS3 zs^EiGA|-azpatDp4oG)Rl5cNsF3=(qrik$+B4v`Im!QJS}7g2if}5^iz+Crp>&o~jqW~8FM4PweDOjb9_OvjUhHpn;>!lWdPuPqikc&rUQ5%7=3$U7mgND z7!g_xRE0ESYxj6AJ86id?clK3NNezNvT1B&!2#Ktrm4?{I?#;0C3&W#K|e-?LmXV| zAF6sOlSZU!4aZ2_Za0NQ>JTj#n1|~6+Z;L#npv$r->g=5N1csc%rf>cs3Ju~%B6K@ zd%6-3zfP`g*QB(N9U}5xuW`@<*$4+Jc^ zTr`CY+P?h&=Ue2jJ%{M>Dl?#Q+{=_RS_GJ3IT-=XJ#qKL*%UV)l54Qqzy0kOBu<#k z)LnvkQN^f)IK|wZJ`=O^pKsMHqKekr?bew)D1e4e1`yTUYU0h%0xo6T1-gjnEu585 zkmsY!ZFgu0b`Jo7BYWe+`N-YnUFO+tm1b}v!Yo?^>WXye0##Zlk3>X9u}p@~dp#V+ zEuzdQ8Qn@P0D_{2ah7RT)oGe8WxBkSX_{V&s%nHx5(tpd+({5#e1_1~4W9Y;&I5taq?a#4zOq&eP}SPZTt@D8al4*o2x)N(vt{Fg~l4Xwns!`)1&-J#v&DWcfrx@$IIqW(Gh8 zCWwVa#JnkBbJrIEBn1gOyK)8q+RnkKtO+FE1DPc-Z!KTXsWox-3kG&pB3vz5od8uM zAD2pbamVang;~8;yI!yJJpXq4#msKE+x0r%ZZ^-MhY3muDGI8hI;pype?5@GID z6`IngNsb_=*p3zv5mRl{(Y3K zubX|Rsoz=l{Qb_w^9gq^rF6YsZi2F~&AkU`(lw>Q)`*VK|Z520K@03(z5Q zuiy4?Ar7={Gc@k+&E2n~Y;G#b@ls+HrH>0N3r{90x--j^!iq6nZ|OZZ))YGg?dCTzGtTpTyTJ@SKyr||WVT(3m?|ZC7DTiNH8N9a5_CyDzs%w- zi~(!o9u2u?zt_5e9+GBo7m_!iLJb89Rc)bl88--AL`5x^EFFAe*PrLJ*welnre;C*4}BS2%bK6b`dehVZT2aUBUz1aguV-H;LBasoMo6(L)(q79S!#2N2 zN;6|-X2hU5i&K0Nc(eV!q+BA&=F);Qm*wUdxqPDWNLieZ`o`1=XGg}>ZkP05T~oP| z;!!s()!h~uAQ-j=7R^k2ZIRN4M|SVlg*S!mG6_jVN+BcANd%+x+eQ`u1Mm-*11s`8?z5H+PsN zH&@M&XC<QiZJBva*9jz-8`%7*hXX9`4;tfOsC=`X@ z+w2XwZP_!VTzUF{vW5z(EQ|zy z0&0%*-`?k2#lQaBKW`Q9*Ls~fSKemas@Ce)zTYfRg(#>+R9yrz&a!4s9>zG$oC)yQ z&AGf#AtD}G8iO|jkTQvgIJBjnk3PGlHl+)!d0sU=U_}G%yBL63H2N^o+Y%W{Q-sYE zb>QqG;wKrn(s*Rx?6I835SY)kGWNh?>}-A)5d|qS0fi!<3Mbekcu~0sCTNMtD*ihE z9g*ZxUDZ`wAqv!+k)kdz=M2uy3OahS2Yg~f^$us>TzQ{eZg~Bs{!U^2>m%*i*K_Ck z2hV@5^u{j>``LSOuY?8A?b%lfcce;=<2B=)T$lrO0I<=4lAr2Itq}@RxGXMdVnv{* zo)93-C_C26mU}>1Yi3&}gsBUiaB;6nn!ATLT`1k@4vT0~VdhLF>7BC&*caC(esn{z z!yZ0HtbJ)4tOnWL_mApnwW0q3p;qdG{|J6Jc95fS*Xm#|n6}!s4d}}f0P8VE+T*eY z4ZV4ZaDpPG4XtOmp*CFIDiJ`&XYZ2SKy74`S#~=^BXX}dn0r_getWz9_I7)Fv-isX z{Np;$@?KdD#zI`eCr&2=Alh~48q4ZjYr}K#lw5(Vg$@&8?5CQzf~-)wC{$rsa*5NA zH&lHL*ERI~=HsIzm zo6e@QtGSY}zrS=#q3>18+%GZ|tkp7D)BT5!yEDmqZ6n|OT4w#}a>#&I$mni*u%i`p zJeKuO+b+7_!yT)l*--#H{a`-H{s1#@s=tfd&!<4!|AmaB*{3 zP&7RzxsxMPZlCwe-0vXwr%Hd^$k_+jyQ}}C9CG$?K*oh(HSTQQ2l-Qi>xG^p?_$xR zo@HeB4LY1GBgx5{oA(QF)ZM@TcB^LB>+O%X`S-W$A8+;B6?27ad7p8uoU0P*66pb_ zQZjp%Q|MBS&9K#i(f9C>xjYj9iYP>4yS9L$q*as1OaPZ=#7b0iiQUUFfF$tpk}+X6 zo~5&;L4l))80hnSt5*N^Iw24TLsb+R?jTd^=S8{#a47WwfKH%Ef37N!cDHnAd}@JM zEGMwYuK*Zrb|^7BCda@0cZWcslSuw4MC66P)_U_g&)50w{rBs2j=j_C4FoD^G#7IL zrHGXIWo9$MoZ9xiY(95mKM=uC`ejShy=G^66ahKeeJQ1QSvK_BRZNldU;*e*w>uZa ze>WT3GulBM4n{fQrT3%D?w6%{l1o!th}13)H{pHtI(;G;>tX%24jk4GR7WmolILyV z&cUOaF9wSQy8s9&q_Mh3*s^$SiZU~+lAPtWwGfJsEYcJW>EGDw3FrWdy~S0Q#*^Co z6`~S_WaAu8h|WT@V*R~@LRZ$9gyV=1w6zE0@JZcLNc_KKbgE@fwL5&uVjR-ZIDu>SsZH(3*NaCw@KwltEii6Y*tH6ZCBM4;!qTVf|PH~rRL ztYzGB{*L`-mtRRbjBcVPT1rtVI?q+)rRa;4U%_&_`rF&>dcFSZd!1){f4{w7ZNB;S zhW8uh3Mp{Hwb%s|QzcYY@LE99-5LWJMl&qF$B|K;BSKDCsf&SLB(p&(Fo^_TzZRxUG;ZW2oI`ek*xA*yWyZz^Tt(Ef))nJZbU!?%5XgOL_!jfR%KKAVn%xsBGFHK<~;Xe#A z;@$P@+qouSwe!gM?5SI~9_B@W?x!FEdN+6tUH4)b8$3Avja}F7J<;XGWs!BajaM%i zR{Bor{djf)V(}`83|t+5cy$AcnQ|7p3EuR2nWo?0`P=)q-`?sU6<0+Sz7bc#)hyG! zK!AxbrKKP=J|j{RWMDNj|r;Y3f|c)~31PePsuvv=AiYc&xDwz9A4 ziTW2M$vP|na~~&YX;HQ;@4JLuY|U$}rG!T-)*Z|KUnHDF2$AOh9ijVzz)8AsbO154 z#TO}KiNy1lNc7FbdkkDWmU{}TXA3)%7+&r~74gZ^$9gl>`>z1kXO1fwKKsdT3YF|{ z89tXrS2?Er5rcAR9PY@(OA&4AZ@8g4Z`a!&Z+3mJ*PG8b%(K@SGhh|QVCyPSh-!gR zmt?LP76;5^&=^{~#dcff-jRp9N{CKOnNny09ss1D82fuwpfDZ_Aykd*l}k?$I4OXz z_myrHD?|L=e7K|&Q!_c0bTsL61O>Ynza%RR}fi$}OqXtSD9 zalYT|T5wbQ2D}ij#EYWH{M&WDRlLo(0dql}aCO`qaw!%j#Evj`u<*DwHd*kyJyahl zqo}wU?8D{mei>Qk7dG;KZ6l|CN(vdTAF=APwqtpnN9y0NYG>=*icOOPXVD=6j&>Xu z+;)tK5tAx!CIE}*qN3g|fj44mK{+z-j5|_dpoW$%o3Ti!$OJ>=uJ6B+t&bhD<}x2# z{8Z)l$8Lwp6FWBL;$T_O;pbgzV6L$30bQK|d1n>XqD56*;O^I3b+2#l^L6I+>bD!L zVh%+^WCAMYbf`FkF_A#h|`OrtK9-IB;ko5T#ST1?7c`C z-ntVwV$YGx6e2;yP5{LfB2Ed1jUp-%k?4`phY}|tLqCH8l>QbXa6wBrg&lwI1J~R~ zmKA-oj76C#pt_)UCICfoznb#~z7msTA|^)xu7D9UP!*M|6zdh%*Ja>72$_9y_K6imj+#Fyx5AW3fVS5e_ zI24S%98UJ$F4!UEppyZw0|`tde2~mcgd)&P-kh+*CFetXl9L40$#CvSpMFtiDE?_+ z4RJCe3Oamj3lCBJ7nE`0H6pxs@h9rC3^UXUW1*Dr2?cGgG-ItWC*2KFP2Bi0UBY92A}SCW z*CA&{B`aVz_7D86w-$o}F`)(lE#SgNN_&7LxF-NgDkwaC_m>%3t%yi?)F~CpFgmBE zvYz7U?h)H81`$BT9g}Lv#+%*Lk-v|#dp)>I^;6-k9_v-Kot^^$7 z;3TdvQIx5cVvrks2B8xGp@Ai&uMpR1Guw9k(`uPdlr?=X2eK~rr}Q3)e6BH{xN!rE z9+~$0%SK**SmERK?-}~xs}1MJzU{ERDcoCpzK)}p8hXqFHjB*0{bl8u8M@bw)6xCx znb`*ds0)ZlB*(RB%Ay|G?rX0%48EfTiS7E9BR?5FPt`T&qZ3RyHD+ig=i^&1q#SQ5y(UobDyivH&%zcn3FCj zQ=y7VSnwfDCRxM@S80AbM^mgv3lJH6@kOv~&A@V9JhRt&^2Xr z33^jtTl3hC%ghK+RmCxtXkfyBZ%QGGw-XCZ0-@-DXiF|KaQV*PM~P&dj{)Sdo>2Mv zQjAO&@%FxYQJR?)(4kZ}#|(jMp$eq97K6-$1yl+*2&jq!E!*}I_!{^gJNHdFBl;7v zUcGH+?vJz1=Ni3wyKF53$Sn$9Ze z!H5M|qC*e9Jn`Jvs>+37c-1k=N>~;n{7>5}^hjF`9p~b)@j+cGU5?qhZ+Xx?YsjaH zY7d6I2q5{QVwV^K5>jSPV?hMuOS#

Buk|ODx^?UAZ8No_#zq393mz%i2_bgLhYU`8MQ)0;G76mw6p;j zb?A+jQ%KGAfheMYxI|2juzXMO6RQZYY-o?K9PblIGil*UOHSpVD8La<0V$Z{&QdAmKe4!dEMbb zr4(=Z$47$jVS0=Q-@{usZ+F)IN6zncAY0MonxFtyyI1Y44+rahKbYB00~3$#=5F6bV4Vw(Ey5A73ve?{6R`v` zTvu{HjSbPCzT;W9tc7FLT5ENy+2N$rS#EA~g>$N|P8u=Ope4MEr}9D#aGTg5ICbP& z0x}y`5kPRgdPKM_rO4o{WP?A744Robzl#`(#%@l?jxLh{o>jFbR_&5fQQm@NPkeG81)? zsrUT2qPH;at0fIUv6};v2vjl*O~|@bNbDESw5X0#QE5&IP*qn2+!eq@33`j6>}Z&% z4idvLa{o?rd{;8f5@FubLD}1S;EwNm^q*Be2kk%6&u40x{(Ju$E{M|g*Sb5Yq(`9b z!KtkQ6J79od)%Nv@y*ja56U2SxiWO!J;-LD!!A*Xw8)XYRri=}%+hg<>_26IGg+7# z7BKr&V!-FiS7DT1oHEq8f6W+E$?gV>VpVEIwfK{{dUc&msQ`l%3z5GJ*vh!SF~*(&A4r zc>R5_|3k~O&pkupAKta;=dnFh2>yPXIal1==ZXr>Cg!Y03;j!~Q>=jialxYUk8wey7fnwfnWlK$ zCJX@AWq`UdSm{zN4a2HTJ*b)m%|b{Ti-f4LT>ib3yby5d7IgQ%BE^*1hXV(y_0-Z}9T2sgE1AKN`bx zhvxMMcZNQ4_N9OJ{kHiWKts^OD)30V;D}Uq**5N%XX#DatL^FkN0j~ijSdV}>+a^4 zP|H|_8O-EXF<0IkGimPTsB}U@kfT^?gC#jC9Y>S8E&Q??Vl-Rl-k8Sv7rK%3ldwqD zAwofs(gPSV?Hn1w7$y=XUMLJ73xya=JpB4?Vu5PQn-Swq5@iaY5GulS%Bzu6m?=Dx zC<8sHz+F{6Z4YRI5=zddeF5)zu^tpDEtC)e#?C@?&te%CdVWgmE|5#Efta=uPZnkD zXKf{_ECQTwis^pZykOqV*6I>NWSD~u<5|rRG zM!HolBIsf3!@)ErUXy|)G&7z7*er)J7C0!W;RXe!PYA^UZ`^+&uw^&kDJFHl?B-i% z>s#c$U()Y0)5B+dXKU}550|ResR4a>GAE{93t4lH+1G!$&1|4R9$)Uz2zvz}eM;cq;Ay>UevL`Rlf26L|iHv!BWkpDI1PTvpy6+F3i`WOc7bGekDgYH(A_kenggxkKD(4p-12g}P`T zDDZA`1waEF=LMdic)fAjnzoSQVf*{?QH)xkaBb)ym+GeD6F??u?-v%50Nf+ArV|RW z68W1fBj!~MyYk+EhPxM+z~G(0usD%XU9=F8h;yJCYgtT!&ojv?FCwvzRRx4rxFwbs z1a1PADH-b$9X^6Qb4r&$AWVWNXkLIMFtPS$O&D+uzXQ4VGOp5lI#9`LrQi!AfhPSakho^mzmfmut9~V;wQ~v+g@7&yr|QtrrdW4&;7v z9?Of%TC5rFWtx8&=Goo7nmFk$bY+B_aUoG@?l1zHNyR&RXE=2Z{#lwXZQjBbE%ess zGRayix`E9CJX-(g1d=oY5p*RCX3b0&+sftB77rweH7%?mL4xY;7FD^bMl_^bv`4#g zviZ;nQo+*W2?Ky+I0P3j351IXRMp9Rw%|!!%u?p^i_k?_qyV+aK;HIH1<;6y52}Nj zVJ?NDgpdU&28ncL0a%>5_5vuJ`?FSRna?(OTlQ>y{3$+F#tqd7>K`lVbIkY8Dxd4I z{aw#4Hsf6nGWrc$XUf;BR*w-rv|SUT@W=y5`=!hqY}wJH>$()8lUA?;-au z1krcOPe3hW8Of~k(@g}K!D~Zo#o5+ zr@OluNL(%%^n1nG1tIqYs>z_FUtR+GH%OHkR`0UE{w#RPDS?(1 z{q&dfac$J8YnB}Ia9MYt)q|aS0zi?1=4!0jdoEJE8bzUnRP!>s-mF%0=&uH=ZgLq-pZi=GbR%j{X)o!HO1-DcoO9MMTu+ zSPJPZj)`81P-UVB+?^Wc!Vy+eX*m;<6%2*}lEc>4jM(q}oBro+Bn6n(?mpgT4PC;& zDct^;xqYE-Re(%u&^Fir*bv42i#=5LjOh_&%N?ZOtNmK+^I(M>w}Mjr#~Odk~V6<#@0SMp8YL0ZF`msA0@M| zGy>lEf~vRFIqq;)SZtpw3HPvBY+V`{Ft%B+cs=Z!dM1E2UIs3FEw5SD)1lZJ%&!;Y z<;=1ecM#^i4KC?)=}nT$8%df`RXQ2C!z^=}tP_WW@!<&l zpt9k`kIeHu``+BOzF=d_zEYJu^4!yP!Z$d*LCqEcb*$iQCHU|)$3bahwD6809LuOJ z>Nj@L{+=wgd+yTO@#$xGou9w=UgOvAA-Q3BXp=|<1YIiK-OXrWb>dE#JDs4H2<7h% zGLy1~M!HCe1}3wFX9&ON zp1e@Cq??<$342VTC4Vp5bAv+C1_+p>01+9o$E$}sXR{JJO^OMiD$PVh%a%x*+#H?R z;#RhkXBIX-rSd zr5?McrATb!dv7RvBZt*;)$CsO$bC;gU&-8i2aPRQA@rq}M;Ena8Mre5U(RYr{=PZC zn#^A&EPv3R&B5bW9x~8A{T7Maw~kATLsJXyet$(g`622beWTEy+q;n565m<(9zJK_ zupcf1%NAS`)}RI$$*l8>v#L5>9No3Q_-Ur!J7k>F&r-PyLRBCTCsiVn;$o%a9)L~( zBT!Q74H2jyLJU(u3zKz_N?`;Rf=IA-!J@@((a)E{A)Z7tB#CVv6;K3p@nziOj&yLd zf$&E5*U6w6*ys0Y?B_*A*CXPMC?xh;Gn__85wY~8Nx4YoBcOsx7l#rQm`D)_g^}|; zHVvoT*8#~X>Qo{SdSmfiq^E3$fkTw}<8fRAh}HSwp!ZVuYgsPKZ$ARLZ^|CFV}smJ z;QKht8@7A=q-T-N+H?$ShQ9y$RpqSx|W2{`WeT(Um4bo09=$}qD5&Lg)5A^L+ui|wFGolzeWpP@@z zAFhXei+5$&amrtR1at3Dan;@dm9N?~*e#5gZLU1S*I&}w4MX2Q?mlw=s`h+>v){jE z4cpsKZQ|$R?`XEk2=`R900W;Z`_*xX-rlD=LUI_|AGXTC@a#qX>DYkF3v-?gsU4CVN~4;LrlBH6FeM--HBm{-ChSw#fTt#$Th z5$J7bj80Lubv=|33rIiW?_7CwEjnn!rR^qcF(jp100oKuc&tPg;i-6cHTR)+8 zf23@djW1tC9&g4Ey0XQ3WDfwjo*r>LsAW#x{;XRDtw&!VfwocB5&Xd47tY>4@4m$; zzZ=vhOAAM*p%J+s_FKmpCqI2JY{y-RZM^35l0kC343yt+Q($k+Sm7dq436uV!%Zn7 zAXma1mFWoI{MwngT_VUL8mR1lX@nlHLa?i4+zbot!IO!HDW@_3YCx60UEkQq5>gIE zXISdCqY&zs1}3yAgiw0u?DVYe3}jtf{~I#fG&5)bG)>LBf?6PC5s*~PAYZxMkkb+} z2Oc9B%Y#5kYGV9uef0RuJ?x54T=r$meU#Dq!w$iClnjSFT+i?cE~VCLW+Wpjbo?68 zJ7ZAOqEt&xT1_XD#x~s`OEh;+&_PYpzk#7)8!6qAA7QQY+Do;rDO=)$q5-2 zyYYjH;1-A*Ei72$!QLj-=9BF*V)fh($GXz=zF@F$>h*bT(SJK|lhs|pvX*mGnXn)@ z1XUdl;^!Fm+4X0`eMVO}L^i-#H=WVBUSi+S=g=0XJ_?bSE}j!1+N&d?-MX}8ouYt3 zrz&(Yis#G(2F+DN?&J|TYpfU7zc%0$VtN5%MDQ6M!yl`OMN8Cin5&e*|>BU*T?!X(v5oXtX{m z62XF^rR~Zm!uR|jhWj&1Uqt)*bT|#xWwC&4*-&H$+qV%S56eg%UqUKto#8{*N2}c- zKnoV#Ao|h+1Gad8{Pt_)*?PcdotVC;oYilUBw8A%R#B)(p}?7GmWQR5EQO%M-NjIx zaEb%9S-z8w;BTJN0I;aGhmDTqlf&P7ucq%?Hbj>SK!kgfh|m{GUL*A5Q5a~xi=+U> zlMnb%i35<*7rB+;jWr2lgrR}i0WlGGP70;ExIje+cPkRPTb;2fBbI9nN)vozWic)x zQb+T4TTj)*`*#-a-Ro!9u0w9@$r$^Pzxbj_`&3!Of#au-jI5t|kQx2_H~P_K|578O zALXGz{}`t3O{^xHBAyGQ=R=F-NHYzoH4rr7qNC49ronM*bdmaj9dhq%x_u6hZs$9H z2jFwvP*EbXGgHo!HFA^lb6nES>iMwWzNef^0Hlg|C>ul}ss%%w;kLjP&?w?~q0-D| zayFTXn@hFuZ_1!@eVNyRt#h5Gb;aF>#snXiuD(EXN`NsK9j(C&ARt@QB&ld0yWZc! zdv<^Vv`BVrINhytJ@!vTT$r8y4lNh!z}%zpqLYeH$)c!05k`z15=8`riKt6BPU~P~ zY=Xf0-g?jmck&@b&oK@=W$wDX`#8r#TaDIT9^BH!AEwAxFhNyQzC+B{;9+&2&Xu$*4`B+r+V8_w#F)495Z0i%eB zl0`DxEf9eNMQOZpx>>!M+-BSyv&pUcY;+bfY-f7DU4TSE-}cy0-N_q72y!+?8d?dp zjQ!Z@C~>roCYBM=CAe0rZsBMkQV_vPWM9h*vhKuC7w%`xsF~?2ga{(e`$PmPa$)lK zB4OU?L?sFYMhes*;oT}?xdA0T_MHv(k-f1Tq6Ki-%s5`W+58NKTeb(|ayy#Kd|9%N zvvSI;*TAaJ!$`gys`c2x4*4rqgeHsSi+yMoJigS|vhP2AG1xEW7}m};4n1^Y>O8Rn z@Bl)OKDvL~ow-*w(`>-+A0ziXAOK8L3A2Q*4$fw*NBl#Q-nV?SB?@??~kR@R^9)GV>H`5#)>9c{Vqw;jJ zjJ##;6 z!~NQVF@MN2*X(t>r`H=Fk1oT>Z1de$+x&D6kn{YV4buBJK6?F=1Sds93mjlk03%5c zVagq5R;%A`^?IFe=6}pDwfg())!ZotF&B4lO8eNz@-#jqY8S0K9vin+=1BPM8fVq1&n18UARFz0Aj1Rf!t+t9#G8oT3J7sPTP|PK?j;qS;O;{({AIkT+nLKFjmlV#rhJ|{ zNN+TJT{cvID01J+&f2y{_dw|k?2m05q2F2A^LOCOnDw8C+z4UBT(XGSEf?P{10zH$-&|#SR)NW9KXYt0DJI z5Y-hpSTZsF=+dyosUUo(9CND&TK?1C?x&RvE#ka$#?SRN+J2I4u=NM5kH)Ib7S}uS z-{(Zb{UDkVEuJ5DjJUrPQ}RA7?#nBThBtTG|7_#Gq}oNRQQu#B@wc-Eh1Q+6-wv>)Q( zBLPKDyFN)|?C>R=fV9wg5zD9tymJuJ-3~Ad8Xl9}>H!id?5Oq>tu__@dguXoU7`SX z?56~)iKL5OTp4&gbf&)Wlx|%I>7CPBy9V#AXrIq3Nihz}^wLi@a}Km9gOcWSxl5P* zIS$ZfnD*sts&5!UKbXx#R!ODv`suhHfTD_P}(AW1He}A`c@AI{mw;Qf* z2G_|IR7MsK6Cup#`9zJp7BG)ck-%asoko6S`1M%o)+J)ps6H4RGubUe7)dp|bUqAN$Cm{Bx%U?~v`tuOFj0Jb)GX#(G*hF3Cp#9gUIQ$ZwR3R_9fM0^eu zU^WA^MCMj@jW(q$Xp98oYcqHB@h1Tko?$1`iH}U<-2f{jH(;nOx@d^rqyAhNb{^N; zu3HI6} z8$|r4m!UoPy5g%Eyz$9G6Wh=1p!2lDjb^BGtuxI1{pyusRz!4~#JxD6YVSzROH#+0 z*?0uFJuyP-Q&`Y=Iw36+a!^kx+M6C*sfq|#$(AKbM&w>XEh8o9+JV9$G7KQpzSxZU zE*i8as3jvcO39RJ%N)7Q*+E?kk^u~wWkH(`cGT9rokeUEOSF6 zM&?*82dO;z4h)Is)O4XgddHs52=`2#@`ML0zK$ENscmHL<4=;TIRo;nbTVnoo)jGa zI$Q5RW()RvBIB_o+zDhqsBGxuCp__fIcm?3UF}APtSHg>;ZeCSf!q;!Pz|+Wo?(%X zBJKTEBZyc*ts>PTUvU8JFp}Ybx*W1f0+<;#`aCTnlvF=5+nRw(f{ubCO1B^|Y=JVHQapCR*pJI98d-SML#>SEx(^=FstAO2N1`v4}s3%Nf{SDCD4ncF$L?YJJso&`a= zTV~7%I3If!A{M4DkSfAS3$(IC5ERWoLrZ3rIT4=Nx3J$Q9*qXF+uQ2?|Mk-xq=}g$}j{}&?y1=-OyX&trOz<{6v4f z!H;*K{}yRdn7xZ^M$o`Aj{vw(8Akjbj{pnX_Ag*ySGW^G`p&$i_0%r6O!T}7f6;@(+f_9qXQpZ?S1^?c&`vvPL%$hh~_ zE|?v?yBf9_c8A8VdEjkZX~Fv|U$DLSALDXAK-<22jyrES`@%l{q;k}rAKRcDs=@2H zkLGa;amk3YpsJ~uDwHq-38y>U)oNq`B}$pd;-T1v&~slg`LO4%a`3g?7;4kTr2b=& zFO*9z(*vD5*8<{F+yNA%FA&0DONh3iNzxM*D*%|XJ=g`B8jsL^I3XI_4^5RKr5Vfv zi@GU7MP@pY1UIQla^|jYq`+P_6@Zo`U^QN0G@6>3uJ6i0V~fXO0G$-vnMQ3z7im_f zWVCtk+zQxE89bxq)YYbL3HzUp^ah>#I{z(NRg^H87m(24n2BVvX$~q2Nvafx znMGzhce+q5`MGVwOooJzl2F6!OOc@g;Ret|mdFnyO@`Ap@5F_L+BUaf;+hPdaE*Wt zWNr`m;R?4reOXP~tYmcTT4Xm_RP8I7as**NDV{s-3brTHM$?iC$Rt=TCjn%&nw^y{ zNvyu|L`{Eo{&XzP7Qd6LPZpuGvc%78V&M410e8pO6~XmQ*T6VS_yG=4AzNtH*2Aks zqP6)?W3PX@T4qx|RLgu-aqlRkDSQd?+=;10QH^+4r6^emDQ*U81Y}o4P%BdE42W72 zhJ_L~%Mph?bnSGAM(ErA%Gwc zMh`^vwUyRojY+#48qwSRIs;ltafi9Pd&|m*#?BH8#NLkCy>{;3%j`I-`nd#icp95w z*a~Z{H~L$jsx_YF80%yvqhmfen9lzE zegiOm9hubd4sAq6LW<9Vr%J0|mIjl^IM)Nq=xx~1iHJ1QH>{;b2Y@rbhbPrFdxAMU zJ`ioU^%MZ2pvwUl(L8oQT zx(R8q_K<{&18o6l3(~Un+7<%{M4CtOuyflKG8lBmf%GZHR5mA7w8^8FmqHhhLZ96l zg$L|I5m9A?j&TIDkM}_fS4Fa3E|;iqJ#ZqTT(B>&E01afZgq*yr?wKAtsn| zIz$Qt5?N#+D3T_fr~FF$aL0rT%}kxEsz!X_=rJu&RkPY|T~%179dqB5XDRi;@)#Uz zYMIbcfg1o0^n_w;p~UVzbNGyDu7~H(+jeZF&~7|1BiZ!c;WZDI{Xnca@5tF57&A0> zu`$jsCcPLkVgndLyeb*4>syV`E~(o`a|5a}6HJbxE$Eu+tTyH@QUGp(l!>H6FI`MI zp=%T}_$n5Zr2$7h5`~{x>jm;aR@*=0ZJCjjf!JOqiLRDec2lwTF+C3X`-yUAK96-S z5dI^s(eY;EJ0cRvfZ64vM_P6GUi?LW7pZ=~F*e~|z?xNhgq-}k&5hDy3-W^C$W z{)i?`w};EF1D95}+5|&Z;$erDn+I_%ql_o_2j9t#`?HpVt=7A7kRQxuTLaJ^9oWi# zID0?!Sw1`)VZ*t0_AU@;ai?aSnPB3<_hKWl?JXJ|9y@fN#wTF1Dql;qZU4rF>A~C6 zdo_gw$w9r48V`(>M4Pw7GcryJQfJsP{?IK7Cs_mr7N{0FpspffxfLu97KYL$?7mB5 z>jS(S@(@I3wdQn+) zCO-u{kA%?yiHMlr>!0s0&r&1Fmfk0HCQwy25iTS)r#oQ=Gl)~WGrxm`xiQjfi@K!J zI&x$q#pFKI8jNVJK!e$a@Wd37yMxha@_?-qjNH?WtuH`07c&4-M1qzTg+d1>#_B62 z)*%5hN+{#&nwdBdY+WK%cx3AHo)IsVFaRLRQEjpvf))vbK&`gRwBiU3!R4ieU^zoQe-ISQ&d|r6K#PMeATwVvmONb!Bih%b$cN_ zIe*sX4!pu_+^&yMTq_chbzq~- z!P)s|L*q8)z7q1#5cVT2j#+^t#i z4*;4?OK9LJjucShN(BHNyFKJP$-zlsx|zJ6F!zn3raw11Q!~e;5D#D!@EJjWnSy#T92~OvV*+?~q^GdT_5N&sz2P z`Nyl>K=no2`g4(cqu3?6_K$!&htEAce?GrrIGxSW6A(bd7QsCo-rea25=De60A3<; zfC&uFVl@({3c=V=4ef0SVD90E*}zbXwbD5{us_5j$h`yFFpxo4!mzP*2AWQ+aFO&Z z1&~1iAnDmk08VHc(4T1ZTLq2pW3cKuqqd=D z+&OZMP~x)07S9&7m3AB4Y8>Pk(9Nri$bH1(^u{rXfvtB! zP7WTy>O+o)!$V7x_X=gZ&jgH0kz5WV$5Zl>E)dTO65;5(=Il#{cls0&qbooSS%+=k zbJ=Qk(2@s>M-K9nvc_#~fsvudnujX=kS*n+)xWRQGG|Zy!ZOU#M@xTOkpn{c@ZQ5` zefj%tD%>w5U-gvXV86w>y>fHe?%u8WJWeR=&RcvT2q5`j9-^Os-nkAdMr^Hw!e9<1kD@K`*9+sunF3#!}583Iiza28kqD+tsw=p)D|C-%v%{F3BZ8 zAY8Niv(h*_nE6YQf$b8QDgY_cAh#zk1pun5AWhE0B_?}d=$S5qWqXlvY0!iQK)yMb zEAKY@*?HX=^!cyS4wq@$_O1t6XH|5)ZL#_~Z6@%C%*ajxX`S6w0|QJ7C~sCddY0PoMcHsyI$c4OO;&FW%t?k5It?jby zjrYq4_i*F~kk$Hpc;4H*+fKAUUeN>#H=4uD40Wc@PB#irR7%lTk`qm4t2Fwe!WKC3 z%LUb8Pzm#M3HU3lm;SbGvr$}JMCsPF_fVF2!Ku5&r^3h$0EJEii$FC^dK98c$4XVu zKC4K}fx7PF+1h5Xgx&-|m_};Qwnu7$uDuwi#!%0qqDW78#Ai_@;Z6b?Qjb!FDq)cb zgl#^P^*A*?S&yazp`(PVIt)-DE`SJYWXezhU}UT)C$j8PGnc)Yd$RA? z5^ouzI}~UKOZ_n#edfAAbDQy4UVN}LE1G+$q7Tc6&jt-l5E8NCv9XETP|msjU?b3c zVs?K=*`WBQbH}hZ_~RJCBjrekkOKRlWM_m7y0b@d_7=}3Abz@>*|DD3%inL!UOmzd zRu(Mnr+ZtYaru|?`mbf~8?Bv4Vlxp*DQ(5^BL#$r9^M#tHT&ouCtX*3zoF*)T?tYOY4=DVYM00C#P#5(ci5cb_TzJy)OMb4<1u(g}3~tOa^iojT zFcxi#$e3!VxVcg6(%hVPtygvRf}fs=i(Ilcg&cMITb;mP01L<713THpKou=R+cZu~ z?pk1wxcb4Fj`+FghG%m3SM|~<5ucL#>V3Nr4}M`681gKqy+8r9v${~>BOd@eQ&?(VcNqq7b-%ebePP84deqA`S|UL z*};waWsE1o-m2k-wOR%%F={zDe%2Wx{>ml66KAQz7HSqm*oSzl{`JZ7fSu{8oXcNl z4|uNryO*D__?+9FvngES~xOuMk_imyfP zUq~l=rONICwx7b1O(1NM`?%&=O$XDGzD-lfI>o!YImxrYWT)@-WY4V;?#J`_PIs=> z^=ONT4|y}#!=#8vSjV_WglBOpAO&+SnSm|au^{`(p(@&p+}kAw{6Tj5+xXV=gopqt z1qs=ObOaj=SQ5a!Ntegs+CfhX|17X=~GM)qv#0fg#NMVfX>u?cyKJ^i`EKRcH}4?Qo+#egW?ycWg#~cscKf zv#vW80UmkKfX6p>^!uogOCYE5bKnVIS-wh=JK*fx{4`?r#=cVTvY5t;EX+Mn!h-s< zwG8GSpJ!G~U(4M0pXXLv$LHPqlDsI*0f1FPqb*L-Afz~1E)2u(>NU2FRW)qe4@X8- z-JvRyI`n*fU-5?yy0+7(kk3xeuPUNcs;YOHTGn>8Z7X$Q_tE0s^p76!EQkBCu|)2d{lKpYLN_da z(B$@mJ!}tg{>Sbae2#t&(fVq2?<~GqNDfIW531^nzSfFGWsm;TITJaflV`&!tSs%Y zQ0QmEGxV}7SBSTFx)?~pX-*M&0T{;N9xPQwpd&eNR5XFy8<4-p1Y%KS0{>pa@^+5M zupuJqLXB|0a6p_uL|;!{oUTwQd$dq-k3gqR>=g7+H({aJ#pm*oU3ufh{dg(T2P-2B z#APIxXS;^T$lh-&6yN`t!EHW>c5+%2i$7}c_jNG$qrJuzb03HE*-3k558e%=#>u8C zN-YImU`DgVP?yW)Rx9CV=8++$KwW(kt%(bNKO_3H_V7-gtK;@RiNv4O(d&s`pvU>^ z_ilu$LA|aBi6?WJtmUxiuPjcN=|0AU1|Ru~}7 z3Nv;oeNlt-mrXu9a@&2;_H=JzL@;z!@uuRmaS`QW{W;Rop&twxdO8&m)rQ6;EjI&e zs+KZLE{8GwwpN3+kw4m_mtdxc3}Hp>M{nQ66D6#5(J4dwsUE8L2^34GcW^UfT231%OAeMrH&X64MHO=OK*Rac$?sWe00w%? z`JPe@$9V2TIQ~N?rcU%<%jHKN(MywJwqLQ*nf4ZUbK1g%&?zE!w^Yv>^sG4itXk&D z$^6mfdp12Br%5QN&4Z)UPfyr%nnV1RvON%(9gqkQ>qnL<7VwF*}iQKreEGZA}J#uI>+2gBs5ZlfyF|-;w9ApBB$hk#y zERz{>yhk(`^z-@j7PK<7{fj~=zN$8olicEFZ|gHb8aV8X z#Xn8@!(uk_b2%@n5dsY*8tU zXSp(kd=$KI7;*pg$=BT8DAJ7`?>VEc7l1am(jd|Ys!8ykJ7A$=2=2d51lZ*c+-+}NJ&wC4rOH*4!(j($PgbqnH z_v&%|;Cw&?DWEjdn*{}AUyY{jeHZiL_V*d&ZJ`L(*_~;XLv5m~mJjX!N{VWndExJE zsTlb?0Ms$I%y{$W;H#QNE;wB^(?Oha_5!iK0C=;00w3pS7bu zee4QIuF*O=>(AJDKY)Hw=ePmUwcyWdzL`gqdA(xo;4?2mok`21MC6_Q)Ux@aEU+aK}CJ7 zABT0bH~3)jnCwszq%(q{gqD>6=%OXH*+kQ!W5*oN_RRX(9OBN|2h8N)}v?)x`^M>YICE${jh2!NDLbe($fG zCmV9l^V>9jA0XwQQ}*M2j@;t{Aafzbyij_Icj+rcN#wo=c?sJOk^6TPCDEcX3Lpm8 z9_Bo>(2JfydfYl0KCxCWtZHAkuygigw*zNS70uCFcCY!%{2{smByVHvVRT%9fm%9I z#Yp*(i+CKzayNwdX`_6_-?vUoL#aGL?)+-l-fLc%dXul?*KxV)Y`_1*;!xLEl8iNr zh>+VTj31RJ$@j@9m|s=)xO)%4J8U1Z>WZ^>kg>Y)iDqnGb0pT!5y^^^uJe+2F24go zIHR0yKhqi$xO}9yRsY00Y~ECHV>ooSDm+*={B8_EJksmusAYfGQ}xKO`!+O*B;3PWr!*=W zEoTawA_tP1(`jmT@(5hus}gAGdlboYJ_WlsH>57zSRlUhjCg{OyM z!)P;$0dh`0(QApUxIz=3wcf=25?VYHLLieCRh@Y zH|&;i5gDVcj-U0+9)IU$npKS;9q1{zGRmDxNNd!NP+rFv7t(Afd(@-$8co-E+zXk! zVkFl4%>B#*Gh>e|MsEyZ`sL*Xk~p)4dkll9DQ_SNc=Z9*q?%;PQY9u`9zD8 zMWY}{4kb5oLVRxbay?$v_JQr=r{h_A)(^hjVDC8!7LVf9Vp&oW6#(o`7TUyo9St6 zx@goQ2Xy5_!luEVZ6#+H7>A&eP4gy03pkEQSwaN=5Vi1GIeWx|6S6mWc%9@?LqNm+ z2pLe*v(|T~4fyc|KVwe~q4RW#8=cmp^Idm#z&wlKnSrlO6L~ zU6Y7@V+Xkig?oF)z<@z-n`gE2-^-&Pui@-UO~lcWI{+x#x|Ih#Qpk#YHl2A$w|%~~ zhs0PM2pRi@_In`0Y0rG91K0^??T5V!Wn+Bn9HAr`8-2}WB&u`-houd;du&f=w-R-> zg|M@45vZUzNIEmyX+m{HhHcgK9cxwkMJAGAirV&sq&2C(?6TRY?OEd>-r+)g1Qo8z zCz+|REiR1aRr$5qHpXg79 zQmJ!o01=hEHSEqSkJs6&ozd87SDcXKW>}Z6zS6xe zVVsl0V1VawN*do5(k{b>?ii$vpzK2j^nmrg~YeG2t@ z1_xfJ(TLlhlq62}czoxdv1|TMi08f$0|%}>aQ4Mq@(j7l+AM2f?w%bh+FyJz$Bp3*^8Z=LJ=?JQy<_&SjMITfM6=t5Dq%S#!rVK~gl|_Q>=L$* zeM@8npV@orq=FDgQ@`}?ZJ9j)HYiSlte)-3p!vC0ZFCgNFqtnLVvH?wPIX^1I;Z?$ zFv;Wai9POUrzN#por$r%N;lSDuD_fx>$BJW9-q0V5algl)Y zInZ$EVt|qH|9lG)qvjS}U`w|nc`X>CrSWX9vr0R8I9|_^&?(aY==w7VKWN?>R9`T= zEZ65XY7A$tfOJY`1Yi=LD1OD(gQ%7c3s@gBs2QFJAfT$0(ydEN!wwE4?@kRn>F#p} zUhQ!r(jpy=vE;QMr$o^#4!?HtJ`3`iD7<{4EnX*27>5Z!Q>(X>VH*RqGoAZc?aw0p zv8JCI&wZ@(!{ZG@AKSKX;ZFu!_ew^3=`S+kl0JKvAs{EQ3*FLzW3kJNzAP>1ieYT>4q_^szJ!xNwEl_&U0Y zY9vrfVa^bt2x9M2*P==!FlMJ6Goj)ySEbi+WIuh^V&oct`!S|Lm3+A|oV^-8|o4f4}Sp{cIy^yk*rM zS=?cm+LCC$_8TyWo9l7!JHx6fE5P2p;D@|xEeTNu?m$rxp+JEqyD_sax465p^HY)x z;xe{6MM_yx6?Q&O(kjWyF^~~OAa!0vAqn@09NV z;C}p2(M6rh!G4;jKI4aP!Lib(&z~{;#9-+JBA+4Q7W^F$fC2%T`x(?| za3jKKt+H{eeV1im_v1eNBK~t{Piw!-(9+gVXW@*R0Px zps5{3lb8iq_OHW1=GNLE^wtwV4luuJ1n-xx0s*Ow=?5nQ+@D?^Y1N*4-o5f*y7d$s zXrmbqA6YpD&Te4$RPqkV^Na*%9|)#TG5G;P01QOvVBMVr| zokt(Ge8>I6cnjU2tIlvXpJ)B{ELGnbuYu!%%f4>LL)d(tJ+?7Zm&H~)q$ohV@o4X_ zc^1kzpT0^FP5^RdUW289qwu%VxsNY@{zq zKL_Cqz#@vR0l2k19UZfCd8(2$ZkLc%^SB%kxkPvidE1%06YdGy^GaiCnTbIGsEv~V zaG)fo+Ghya$NyS0hdUj8c={|*-C0e$M|_gZT4%<})>(XJ7Z}ohHU8e-d!xR~pmx}6 zW{x+6Png51f4(f9`h=}TbL-ows;jApoaBSD%$>S3rl(Y98Ln31wosqW3cAibn?a4K-?{NmQ zXwUK6hklsmGz1<`Nbcxde%t=I&z}9E5rNLV@Zr%8xJhkXc2FGzBVgf*zc1juLwIInX6JTSKDd{Ns7w&e zpQ)WG`yQQ1>@j}4NRMFd*;#x}Drahtlk1;l9@MoIveG*-bv|3rykYC>>%WZVr;VrI z{s9}FyJ4oA^ZSOX{OasUp~7;RS`2O?4l%&T9`*Vveo?CzH-SM-ltoQT6*|nT2vHmb zP@_QqXlfiO%|kj&AbJ!k<6a5{9@)j2-sa*_tDlwL;iznnd2xA2iN^q3gQ+xP+M8j7;*TzU76KgxNCoZty`jR*Ro{R0%s_QW-?0 zHL(XMCtlZ=WlI-hI7zUQR|IF4@nS+U*J}%-8^F2(c%hJLk!6qqr{z+hZE;r$3H5kZ zePx638MeJ^a&nGlVsQW}I?Z&z9W`R3L!fYA_8AHZOTCsvPYAb3L=;q^f}o3M?|X~@ z)FFh1iM#Mu0Hj48z#gxnFD$ZaP1c2GDK@uf|z_!@fiCsKW<-73T z`1;D4{Y(-xNG_jAS88jaO4P;*2d>cP^(vu-Ywg`T1}k02PcYJ(__00y;H=#2Et^E71hwcAZtMl1%!l^&L{u95B!;`EiBo$=t0R9ugDF2jB0QTf!>_WwV7os#Ud*=M zbxw5W6@t;b!##6`lOSAC&=R#uFj7KPASS_B1qBcdl?qTMniEcUf-0Cad2vB*{7S5P zbP7Z}@^g0+p-3ctW$Yt~2=vU6JI@vfJEPo<-dT694;D5uG6z|bMy(&b#Be{UeIVw8 z7T+z8)7u`LjlUl2+_9T!`)l^MoSj^~+nSHJuN;5M3nF8B7*6PVGRv}fu3l_mD60q| z<277sxO!7JZ?W&G2g})A<7ge$fA-M*`KQ*2Qzp2pFi)IGt7lI+XTBY4k3HOG(qa`5 z*L$~m8cYRn9(m^ufepVNW0a9T-F<(}SpPWFzL3R#IEd_u7ocfZQUlX2zURl#nY%mF zYzACEboJK3(>b^!wTh;|^N6XFwlOFufRh&LQB_O@BA{0m!VCgcyk7KDw3Je71z?1c z?&cH|CkZJc`C=)#rNB(UMX40HbRAtv<%B|V2Y$WYHKvjx;8`O0J3Y4<n?a7R8Khq5WkCCFN8 zEn8Z?AzVY)@1AvT6X=`E=0Mr|_K%Z!Z1WitIV)##f2{KkbKjglSfajE0vPJKf7R1v z8Q(Q>&v+4!(cAYtdTaxvhff$Ek=OLn?>ZbB$v_&<_0;_kwfCrZ{M=66CFS9re(IOR z!5GyY3!Q_zs92*6Ocdl)VsQ4jKm_hpnW_s!sCN-#Er7n@qIi9oUSBS+Q<asT+X`5LD{&2kU%flyd504#9 zs@1waJ~uK$$&gN{^~Uyn`?BJ6i1>5lKl1wgkJeg$$spXa@d%I^ma@ z!EE7^l+&WK4Q(Cg`_aiwYd#1+u%8T@nX|uf|MhV8?wNa@A2_?Tj(r9Pf;E@ULmZc}kBP?=k*jYP!+cwvaeCm8NZ0KzGMNLwUe$diG__ zuUBWrz8E)@yG4yiNTv=%Txpb+@hhqd6{Kw>@5X zj+n@ulaKJ4vvzLS*=Bm5HT$rpv+G;U%b6I|Lptfec{>@NY!6q~UyW@oS(t~G9LPxr z)!rGJu{~wP`csB)I^g?z;EQTZpn=3ZqJ{eKy+=IV=(Emr0 zXzN=!`P?w{m_7#*e^>ouyT7=M^Sw+=kI?HO)8Aas7{_rJtVSaHBG=pgb?74l2oz=v zHXm17<{&``l?~+C&xDOyX7M!`7oJn@a;g}(cl|JOoxAC#k?rfM0mC&%kG#8s{nYsn zM309{XP_J8KK^>xp3QvNIwf_Uf5v3j9CG`APoFLfoYf6zlU#MVXP9s2lj7w3X*b{P zXDYZFclJ0!`O^o()R}uwIQz+y;*RuGB3$}}C17Dg>B0uMB0UqQf?qD>m)G*ktNeP= z*Go}_NEOZ{0Tw1fQHW?Mg(RVdi&FqGlA^!VH#5#Rn`gVts1;UWj@vcXX*E(Eq|*g3 zEggS&=P3#jeKU4>)BfUTT{Mou2yw_jlTissh>lsMxa?-1>hz`c;mGPx8^TP}J^SxPbY zjA|-WE$+47+nc8lGaI{fDT72Zw*1VF`&p!RXZ{>Ovm^DwC^L?&{eEzG+3t;cZKHf37$k-s zD1ZOH<0C$#_6N)8_pmD4D2c1_>Is#iF986zAo?b-wq?NK?&cr`5s;lUb8^`XjFVAH z0WEbSp~Z2hyH!FJrR3Yk@pLKg-|9d9QU1^W@!$XPw^y*v*FWm~N4?d*{c@4QB6=w@ zsa_^6MJ81iu7zy7EwL8Hx|?IUYt)l2J9{FZ$FuirB5@JN?y};J6t4gz_v? z*bv_#(L7TB{`xbr`*ay0?IDqk$2|_x2Gj)a81YU@*=OpPo#|xWdmj)VNtXTKY$E-L z^1vfTiL*O`_OC1OzOPNrHEiPq8Rcd)lsXnIfF_oQKon(?N$r3CALSqa^6TGzEz?vv zzq{PZRHfkcRSHxUMKKj#lto3=UoJ(YK*U8L#VLRq_-mPFE4NFzUFX})-()WIci^I^ zGp+(NI&W+NDKD2xlcESX=y0L3R*trFE{Hh8s~wn!hSl4ZEtOL0`g262A^J10J9O+( zc{VBPa^uGme7`Kp2Z6&jFbW@a&{;ia7q`!GoJIvm11F; zS^erW)o9~W2VE^?^=2ZN)TgQZ<6r*%_g~7(g@tCAUy4stxlDR_c@#o11xXiDH(ch3>8u9pNjlL+pM&n@9uwXp z_YD)@kjsX@-zjJH?8};KZ@wNddR3L!EKM6*26_w2U1M;BOsfx)-c;NorKs9m=i+`P3ns`^E-x4P zRli$X+LR3?RjCA^JP1cfS6)S(b1g$4F)R&oN0>dR|+QGF}7UtccnxX%9iBGbDt)dbw0lA$qk>Id67z(unP+Mk!9(Pu6quN zpQY*=(&lNrnN>XZ*Jc#g9_$jwjN|8Qft2jNHU***98;(->|ilBZd!{-K1U=dY5^iY+L zbncsm4J*M1SKfc_cb3N%*9}PTUl(ZKc+l$hL_oVLdtZ&cEzPkj;1t%6BrZS%6R1MC ziCb|i*Z2CO7oS7~MM@F5=w*`0Rf<5hOhqR}$xt-uOh7466)7Ic=R_|8hE=%=i2_x* zyi5+rt@^L8db3ii&(&@>s};Y0gFB4L=&R>eaHzf$-g#(j;?=srg0zkb0!aGXBKNQk z>e-1U`MdOlO84-|!T27H=~HF3gnuaUkI%j5#54k;PayZUY@9UtK65`Rsv=rtttAXT zB0J=M*22D#)jy-0Z5V!SIYI}=Wy8FNgXfSY*Jaa^eUqLo-JW-RTpVFdLLats{MzZ~ zln1Z1^~=|j_vqAkhV=Ix{u9cc9@@hPw*HZp_K>r)F>WA_yAAs>a&I$nQyc9io&grR z8!!us^YS{e7Wzbk(;YC-=sp3WVWnP*7C{*{u^kF6o_6a3!QU^h=I)LNW~`vXi$crv zN}sK`PYzn;&1$XnZ*RYt+3hx8uebLrZa3WOGABZf=Y$(YDuAM+UTH)bz3W!iYYWf! ztPohC`;DV-AEpPV_Olahn|ARjyBYkj-T@9*#LZ@2f` zZNB+zxXz%%S}NDPGtxJ%SWO=HJ+?!39G%J``zcAlNa^t!+U2&~GO&wte)6m);7QIt zBKP}T_z}$Y?3KOjuiv9}tft&1#*>Ga_JZN5bDN|~lo)F+jqsS#0%h>K1YW}?%p*)&ahxs;cemtSV9Gb{0L|NftJHNq-|!Q6!c zpU0(3I*-|B`@4g*7L>k(=*h3cHLEk9m4r_l@QJ%7-(f>+#+a$QieGij8uka<+GHLU zM1?@4v1QNU z50K<&{W0!b;sRZKZs?qtx-|Oky~Cx7yI}2(62TfjCRscBW@M+bF9i36TdcnBf!fjs!!czFZaX zg+v37@gn6{074=hZ`vYmRv}u1PB)`Fge*V}&u#~4-L9P~W=5z8D8bNnYk_{ljFgxJ=(PvcAe{7?f>{yZ?k`U=ld16 z8Fj{Wh8eEE6f>)4AXFv|pFye=$*7Sz`Z?Xa=AIneRy#@0^nub?yddJ3wl!i8t54U_ zCl_l!v*J4`93F6LE%Pj*{~l=8*3zBoaDA}*^2v&-P#^|?j+u>MER zV-3KE&l9!DlY{O~5h_HQ*^;OzWA>zBTbQ)ogHpp={8?E0=a;945{p`4;9~hT=3LAo zk-qqa$kl)a#9#?|m!hONF*~)IMpg~wttzE{(|W0>#>?eWrdbu2sgx3nxsy&>c&@|~ z{@dUFUP=i}7E!*ZX_~YQV5J+LLrAaXqGg^YcQ^XQ?Pa!V1j=>h->&uj=JPD!5>;#c zx7$=J<~ibvRJekq3#QVLDnKCO=g7MGKVT9*{@fsku?-{l;mZEH86D+@QZo9Kbed8^Ar!<4D?~=nEyyI1W#0j!ipN$lqE3Y%>%3H;J^Xqk#mBN3W(ia^`fE0NtC6qB;yC>^Cv7>snRMmu- zWl4CX2li^jO|`r7-0G(S5d>6c2;D$uOB8sUJQ=6^+=%Ir4*x4jZQoIbRewA0NK$&x zbCO-lN)doAp*|BA1)N$!xh=uL3lUD4p$^c7Zz!qbiU(JQaSW^bghz*`^Cr zRY5I&DN+ii^8PaE>+AH(>qW~nU0z-PAtsTja26Ecrt-f}zg(~Kt=QX@^X#+148=jh=}H(a5)?-MM7Eez(Q%It z$M=orm+_^so@qTXfZb;Lh{wk&_R#q7`uqJeDk8>{{ipTU4RTKfZ*S6~P6bKI#_sK} z$JsHs1#P%3V4RhUB+m#c}I0-|`CFe!@S*H`>%B*iHrMTJ^K zJ;L)AMcb``C^F_^q+5y>We;C2?rt`dVsMZX0CB=apqdTV$zi6bf1Ap6DldBZWwPsy zbEO$(!>yt^s-@E0jw}h9oRrd`NLR}Ukm#hL`R~Q-U&or3_c%rFXLDxc-TQ0Ku0Q&c zPdVf4o3E0}*oCTl@4) zC4E3P1F%g^7zy1`|3-g{ZrO3jQ)_b(@97 zIjNe;Byy`o#3x{ayz6v((Yf+wUQOPw^K7_U#x}6P@&d2l#U;Mu#^dwPQuGSBW8ffX zD1NVd=_f~)dguB(s*?5`hN)scH^+vgY6XLat+K0p?Y-qMFlZR-YaMrZZh zBgG53P|fP(ouQFsNmwnkr@Hm0ovM~ctzx`TB=x`J)9=^6QMivv=O0x5Eabj_^Z`^9 z#{%J-T0;b6)2JY9M7$-nC{$BX=;SnkQ;GmpC-RRriH zXV&6ShX6%T6fU?B*IJ5}si+9t-OU|FH{$(rxtQbSa#2MQ)dGrME|W0T{7|NC71WJz zkm7C;f?naEsTKv*jr7QuJr%0Z=BtRI@bb$vo6j!S+2`t)OR0wUTYa1TR^1}!AMsXu z_oB}p=01lrdt1?D(<3LN{PyJFK6Uv+<%3L2yB2Jl@Zt4$X#VWRL$uhLc{^we(qRi_ z`8uj){(>cZubXE?uy{(_ifC^l{V&w12lb3`h2jjc-^*Y4z}?P2``ni^HPPAi8FQwM zqN?VL3qli2ZP}3HbAD*?_?2vq*#H@m&st@^K*X)3t9xN|bA zhWg?{7BeYDif}4Y)m5j68V?AZn)KrHY}MUq07-Yi9pW^Y)mrNuh6oNp{8pjlMQBA- zRSl5}wY*Gobrgk~NS)Q_;^5oczUTXA>R{4^KF>a~FU|PQL9Q;{&VATW zz0U%6$hR3ht+xTjYc}O+XQ;srdaQqwndq>QhiHAMZSz&eljnDxKJMtzC?BYIw(B&z z9xWB?T}P}44SplY_~9}{{yAJ`HYax;q+58q(Kwy8^K2YX?cYzzo?zl&9hGo(fK_kz zmu(X&9Z_?hPE%nkvDf6{M8r=gBlJq-u_H)aill8jBU2uPG?gL(qnk4{-33k;_go|l z4tIkyEUD#odvErua6|P-L8lIPXLXc_9Yz6At6v0fH-K(tuTvrPO})CC%&!&mEH5v( zO!_(%f0-_re*bnW2n|mu5j`*%PPwQMEM{}{TI+nX>K-8s=<#@pHLd0}O=jkF^<3qO zq7kSJB9rv5hHVmKk zbKNOPnuMam?@J?XF#BEM^pon(e5%ZRkc5`rUYwo|Yi;sDf9M&jJ8REo16CksB2}st zhU4*bb6G$6xd=gjI`BW6^TV&QI|H;uD8LXwcaY4S6~ibDQ-%d_C_qK1zZp`; z;4T((peu_TNEo(JVzt)-T9}?H)Ci~nfNN&LRw%`weQ_0tph%>{2=%)NRY5|G6p<3s zuY{RSQM#L(0Rw$P5wl7~Omzx0 z7fR73m>|W>L=>@z9#Q_`c>THMG(q+u3yyN1@1K1=JbB2+PYPsZ3YiCEPv`d9N&6a$ zKb^W8@IDFzn+xuD!dd-yp1bdpj>w`604>z1YznSVmW_6y54z}zv$x#UV^{B z%~yA=)idul&Xge{h zDV&1Q*S$vOu3m=QST0>0yw|Ed{r{YDhsC#pm2(ywAYZQpt2#vhK~c>;b}YLhNl46m z&t0uYQdx`ypoB-_T~*RV1PlvF7ePr^SBCv+OPS-5><&b6?VvzFkDJBe;wlxIE>S4r z3Ms$ca4}w-@6{%iBK9(A5xiXVA_zu*DWyy@i30d$RYY2X2hl+f8ZYZccQvCK-0Y*=UsB1_%SZ6o0n|YsfmQyeEQ4zmqyYO$uYhV(8K7>C_vv=6II}BZR zBJKF~Y|adveb2ZbVeTXTg9!SD>i$8=i%tVMWjnDRFp6+D@o+wP&kicPx&y2K&IP#x zi!n^(AdwaY3?G5qGad9Unq$%>mpsu6rHa6pT=RJlzn+CP+K5U0AgZ*$fD$gbOLf87 zHN9niMtO?#YbbB2$q|5!G~Py2&q+Av&l9fNLUU5h;Lz>Q1jTgS)@01g}rl4_b9I8etX(v2*OT zx*1%O{ymbtweB~wh%LjANsteXp5JlyzaDbmz;-!A;wMBu^6}O619F$OW<&07sBYMI zr>&89%VIJZYO^#V6mZk>e$$86ESs6;elVlH5Z|?!D(p4ymro4ctGa4lKR&bKI79BE zlN6d#l|;4e_{RArcdgEUqEyK}*xYvuEshkI0NKrkDB3LKhPBi(aarixTl(~hw~7nhV8l*(4+_c?HjLv6DpV-KHvcJ z3M$xKTNgDZNLX+~7jYO_oFpc|#EWSWs^C&>RulIrEL#+%NgoRMYfNXxH1GMWRVRgr zC^(VgMl-tm{GwrWM-p(lR1Z@cSPiWU$7#2jfKBKZGXgfU`-VxDUL6n4DClQ0{b}?4 z4&1)a(KgfLej(YJa+J=yF)*^tuOm)hGyQ&MVTGSY)_3Hr6=M22<~}SWi%8gWy>1>9 zk$~?nw2zd{BG(^u8Jdhm1+ac4`h zSwx`ffX_AeFo3~{2{c24>EpcE2DkbPC}Wl~RklntEqcRQ4j?28n9;s0p9oz+p6Gu7 zlx3{5tHrC$2!KGUQp6!M#GUG#0gnw6QHEkq6%q+l;i^)KTrNclt~a>3nMGdQiz-Aa zqd6M%5x67YB@zzr)Gvb$y3dQ%Tx$DC?Zn9_KD(&QydHwM%|g;kY@mrq*S)rO9@npO zIjMiAu;KD@`O1yYD^TLqxhMgG2iUYM2CMDR+^nMrbl@qa5kUWh#0G{KV0%fWOs#1k z(&MW?9bK!_ZZX&>@bx^%-i%sa6-af5CRh?B~n?qLPIY`kO|&N00xRGCY4Klqre^XG#@li zz)ZjdHB>l_uML&N;$Z~npujMP@066#{?zJK1UAE~nIcrGH&sA==id+qFTXF6eFh3X z00#rQuHk@H2+}0iGnp(_m2mX&2yh=l&ujGgrnJ4VYUi>)f;aaV^AHvPSy7jA*OYg6b7cqAK3P;R+Q^WUw;-|{z9`saK zj|V^Pz5c70pNZdl;q#Kw5j`*FUQZ3Th^UJD2w|W~hp1*80uiVplG2CDdC?1gWSHNS8STBqJ$U{@~3H2P--_%UHDMeLtBSKKQt? z|DUQE$Bf{AmE-bN%>5L>ebJ)Z$X|OUdwZ}Ih&c+s6F8WpNci~&l5HYDirvc|xIk3J z32hF+PD%#L4Yf(CfHrLj&6P!YBB^lLPlN=iL%G?648z61qqC;~-(jaT<2OoqDjIMT0cG;qM$TIAQJ( zS?s~JOEu?{_hkrPhCgnKOd^xDHR~Rs#qzE+4bQHrzedF&d^gT8^S%)vWdp4!+c>PAK z*QMspukZF@sI~^?9=xQ^c{El+Ld!89IDN=pcs4B|e-D|($dNgSEr-ndZ~@rVgFe{0 za?3np>x0KP7Bx2SC1^A_6=%e&;Q3j5ER;!0A$^-1$jZk5M3K zoB?Cy*iA0%oaxZGWs5Hdv;?`*xqiMw2=vPn4?983dY$9JlA4q&zUOkx2M#O$i literal 0 HcmV?d00001 diff --git a/python/nano/test/resources/train_image_folder_png/cats/cat.1.png b/python/nano/test/resources/train_image_folder_png/cats/cat.1.png new file mode 100644 index 0000000000000000000000000000000000000000..e9c91cbcfb522fb4e94ea5370ad484508626fa8d GIT binary patch literal 144593 zcmW(+1yoaS8y|==5Gj$8oJffvosy#)934tGO4kTMK^kc(1qn%Egv1DGMoJ7EB@F{c zcYOE%-E-goXM@Xq-{*OL^?uUPQhq>8PYeQq9;m87^*|sz9^iu(j1T<8@yIL_ga-nt zLgilhWv5(kCbJoPFoR3C0!Mh;mf~BUA>wzpDxHKz9|*}MGJH!E%s&4KQMf5Sd}C{C zZ~KB!h5SCg!`F-5%F2NOK>-0$nVo>htzaEr^${lz=+t+%0m0w1bK5iH*kT!2H!k>w z3&cVTy(7!Q2>~4~c7!)sEX>Y&!6oTa5i@M;ZDzdY;-sv!)}XJB^y}Gm^|(tORLkjb zVNwlGS*cTV96x*E#Hnj+ zQdBHI@Ii|0%qblFtwasFKY#y?8W9BnYhj8WaW)7=kQaO#=({`XzrVHW9&(A1j*#@-&Cbg^MK>VA`Xz#0^_auTTngv7MIkV*iO~|D@)V&9l|Wq z0mUVOzH|V`@=uvSp>LpG!{Q3&=yz(YNBl~kh;+;AdNQ)QrFu9aQzj7k&pfCDsWv21 zmFs1|QAs5lzQD}Pyf8ms?<$$2)m&MY<-HBhT~F@sqFu?DH?v^SvhVHf?OD7!@u@S> zWKRa}xP0o^s(NqU-Z)LK6iZLhyZp{LZE!m~hTl!(@wyod##K0Em%83~YSD&0I;wU2 zVn32Q*X&D1fvO=0ZN=qyLIm^+xtPwh=G5MVG;PK6)t*xz<6}AWvh+hdAl|l)61+%R%uR=q9 z5BAW`T|h+&SCGmqcK1LOW6tCh=DY5e#O($j!!nJ12t%Wyw8vwdH5&LkalhC+`7 z9~CfP?<`!^37#a#++NY#)|Qky`3EKwu~>}@&G-pr4K-Dw4zIShN;8-YV5x7N#$r6mJRLfxT-lyF+H}9&L1sghu{L2XPxT)6iUj-aOBHe zB4qBfiml6IHZPw18!y&Il!I9#$YWY+rYL`Rbl|a=I%hVPMvE@Alwyx_Bz=p<_vbEs zu}#6(7ppTf#!Gugn}&ph<5IpH9Kd!8I^4$A40r--5XV2hS<87?U!ky(RYb&UlY{^3ZUIJ`>N+ch}2jZ{{VW@i@5dnIcV%j*2#KE=~S0(tz>GZ5&5Vpm|o@4-Q? z?hSo4YA%$EE|qZ_V@|G+=D4>xaDR&m=u@Pd$fHlk0gUpraZ1n>no3vGuoTYtyqL5!dwxvJed^k&(%%)>rb;Xi3Th+8g!Zloltc+Y)VR87GaGS% zIOKfI?sN?gEH=jzD&%7A{9HYj-{j7eiL*Dy9SA$K)lv$j`u8{KsLRMxCJLCxNt+S# zJxILL6ANPO0H}9H)esiOn=Dn{=x4 z7~YibJ2b2NF@NmQxP56chqrEq_U5#TZER_g5*PQsxjH8c-U~lOAPPD=rA9K1neFU0 zS5fI)PpL*xvR(CMh(Xn*r?jP5CIgoOvuercMQ-$ z-L_zA{U%wqyr>!;l$ZXd2+JL4ikR4XQNg~K#PewU(Bs42Wqwet>tlt!8jS~Y9*r%n zt?~SJ#&)r+`TKrBK0ZF{!;GN4d9*j9^@MAAbf-z#V~qxVFdjwm_>V2FmvW~qO~FLp zallRfyk0#oEDv z9&(iYw|ns6W9J>9ZWm6y&1n?BX_vWT%F4=CJu)MJ7mtl4d6I&n^EWWVfwz|nHu`Q*z?>{Or%n45&p$Vm*0B$dfXW3tL{0nlvw&4My>Gp`I~8+-#$#m)H0B z$*)P?7%_YDYEI&GI_4&Fd2=+(A{M1~t?K*<6vxktP)3Be;xP40aY_YaC5~U3EY=&# zO4ZW&d&HO$a6rdB-JRR?3V_3r7FC#fO)l|uFHaNanEcV5VkPYlI+MGvOO7li2EhfV z)wuAl_YB%u`4OkOpJ~6(KKkxkxKs?O^eCaWG=vr^et0o*IO+5u>hL^BAaiit6FNay z@G&D3+vN5|>8_DxqNkRke4)Dg>%=U%O0QHx*4cE0dB%l5^cO!P1wW97lE+nN^7qi* z36xidzV3VEkDtlJ#73TFFWp_vIl;6B>j$T!4a)+SqZvr*T$_z}CdAuXTPv{vh?%G; z36){-NlB;|P?7x{dE{fs9NN<7E6FuHZC3{<_tx`!pw`Pd+_0S>5(Kl~vZS@nMnGok zhy0|PLT=}0W;Q!IV!^s+dmD>c#g_J&mkYP2Z48gPQT8JPJXu=Bu9&U$Y$nDkGzx%G z?~oR~E1u4jk@F47{b%zzlH>blLs{sMgHD-?wXCYDO5WS6kq{3LzqkJWx~b;@h^x(o z+jN<7xa%c*OQZ0q&vk6==IDuIi5@ZfxZ#lz1Wycqr2()Fcj4;-rfL4h zYf+pLugSq5XjS@5*3HCAOG~+xR&7LHiiu@t6(hno|Lup>eFRSyZe2=8STsyo`8UtF zw48!mDkd56LCVU?goK32nSN%G|VAa`+iaeh5A(8HUg?#<`x$7v$Gq4-ZVe(I3U8b7ngE-uN(ys ze>2`RJfsM{sOE}jSB*$|*wx4XHX2_x%3;A*IdW0wqLnXN<5W$JD72pz>SbK?f`lR` zJDVpR#e?cuei!@h#fG;zANBp?8m|i2UMxSCDp(GBP);k~(Sc!uq#Zr?eKkVF8v2!= zt1v=^M3+FZOOHx*P!~!G7Kl-?qK+e~kQ0JJsT-W$7w$+x*_;PTo&Fj?tv+`<8N&*i zDiyzTqHeKOH(N1$t+M$T3qX#olRu7CE*tIi>I5#6*J=K#s*Bo0qwtHQw!U8N? zxI(I?qMgg!C;{j8jG`F^CI<4cpx|Wr@={`tX%`WCnvEE#5sG(pCcwrJIP{WWOlfwC zqkZz4?aovZICA<9NS$D1?3c?I_ID}_vHU%d&(F)Ggy-|$xLxKV!&wg~?N}4ph|Y3| z{x+yP*%OwPG&%XT1w7Htl@gbh&ebX|7~2RZlP0H_(*{~qX=%6ByTHI_5PN%jMO{DK zWh>e<%7MIvRc*q^K<1z6?uPWM!^k)P_L7s1KY&>BPv?s>KEJs0m4BtYKYPeL*L-0< zQCJflytndhJus`W$hmC41RH4moK8PZ*)sU}vfo8QQi?NLer~w56ljfkx952>L1#Kh zcDvxKv;F=3cd`3NM@QW;G+c#QSy@ANf_Wp8_n${wRT~BEchdl9{X*N@4=CK?aW0Bg zTmD=v>KH{#ec9v|@RmV_fU4ZBImBT=opkoF1po<;HvEJwIm>CRqogOi&C{NGe91@w_5{2fxAy#P2J^j6kSHO@v(K# z6sVpWYY#9Qs?b;f24z>R9iCtL1j7RZ1DnlANVBSc#gIuV3dhcT3xyZlTshU@hw`9` z#y7nrh7L9AI*Tk=^XtBc4Aqe6W(!XJZFRk<-#4$K_|oC(k}2LhsEGi+!_~wS{J}K1 zqp@|KTA~OO!MT;>el}E1C01x<%1`Zyzi@rH<&<|$Q+a?Erv47qO17t>TOXH?12Er| zl@x~QtwFjrZ^s=H+u@f5y^H)N&drtjkIoi5?g%doHv)8M!;~tyc52J}73>&^gX4yC zBp&uM(>bphHQx}4xej;I&mB85M+kYL4TH%0>d7* zYJIgKnKvyZDeC$A4f(;7L%`YyVuZmy@N{UyWMBL-9I_0=Zv0lDM~1H`~vivg`1Oxm~JbP zEEw!AXnEi0$;0$ar&qP+bH_sgh?b^SD5W(;7z*N?HT18;tMQk28cEpO?2&c)u&6z| zI%%=&wGWanC9e9lZ-n^FNRXZ0`qW@ z;@YrwJdG%N$L_i#{IdTPcpZ5Fgj@ysf+?ZAKoe=gM7c3~o{Pf-(weAq7SD>At0J?4j z+Nqxu6N4y7ClcXEpUN4xucyWxac90#Nf9mguBgb7Y77|p_6VLQ&R)sbU0Cl6L4a)Z zyL4;I7(V#g6hohD>R3Z3@D*hSOJE4^cZ&E#<+;3&dNBTEP%r)?KqEKfksgm1~Z0f zj@uZgrKWlgPix?9oCnGEe9RzCh*yYS_r+z;iQ|H9L#`}u*Ta>`_^Rf8;C!If5@Ieb zyYc2z95U$INr+A+;Htwu4=%!#5OSEcpuwI@!v_@mkm%OLrsqXBU+}lHy)wfpU>}75 zSTf{jOxe~r_+qmd=x9w;Q5oy!hL&(|0K%dyZzw@pT3QBWlN@nR0CllM22&*G4fpU_NI%vtoo_Hir=vyPF+(O+QE;@VFcCBJ;NSX+EMZ6wEltc_ z3PwgoPLYg1)YllR{d@PEU-uiSXI4Z0_zo3WXcv#y3>@-2PM_TGAIp)nkBp3L^oGGU z5uU@Vjrv`rp$gzBU0pZPhPobF%73Mt7xk{#kJi!Bp=|a%Kgn5szEYf@(mZ1IHO5RV z>)P^EvoQydQc!$iQCtuj>h3NpLp4A=JOYHc2Y?DX52`#_-q=t_u9GcLMc?u(LzH#mz6k-lrQVi;khOluEwe)lckyAkF>b+ z^W!fFSH#WUTM;_L<^5lS_pM1DkQTS(eSIvYq!>X(rI+wdBIhIde2c&5yLaD+Dp4r) zRNj}Chk{#{FX0zQYimkxVp$(oyPF^95?1-i=1(xsf1}a5vyjjEAHK=2bcUOlqfg*@ z5toZJC=3s(X|bY?_nB3!_Z;Ytm(O(l%lysDz2rQAC9K#WEev6Wbax^2aY|@38hCs$ z-T9965bo$D_uCV9++6dia>$YLV34OL!iYytE#y2sA+S_P4wd_k&qY4qMn-UNBCQFmgP_Dwee;QI8FTpO~Tt z9=oK%f4fARk5gDETnroU@Y)6|poH*;_$+oWK1?B^ z3zTSV`m6k{G1B-cb1c7Y(j(*DSVcU7d2wT|!mk)ElxyyK&sT2SM{OOFk6#~>Uv|Br z3Ef!V<5i>h?)v)Aq~=Dzr%#!Rl&li#0|E=>Q!rRgE~B>cG7n0rKmg@}7G{kAikvB7 z{qUSG1r$0NDJfOOm{&Ww5f{5M)akJo2i9WJ9{4I#D04z6&a=nJD|5)x{&`_a zL;>PI+%qqf<&8A?lbA+3n#4*8Llj5?6k4u!q6q1?zjTa66I&d^Wz4!j9^A7RSLdFZ zGuUtUU$Vu#8Wh@M91VfhUI@KHZ#kNye=NPqGEQ4{5OQj&XGk?p1JD5lm(n8;SK{4J z)*dN7fLAfg%VT+~+~Y|}6UY+8iU?14tEf@4wwx@3_Ql2K<66DR8=Yk|e7L#B%b2|D ztIl5S385dp`+x*otev;@)bVER;zA-~?`+???vZ~kpdh#SA1b*+zttBMJV|BfUa)8j zt}wndtg@i9mRL{%D61-45|kiQxYSMD?^@eRR*O0eq|+tt`}XaNtRWAelPebh$Yk9z z&G(%1sV-x!HW2@#jWmN)s7ETS8OkvwxR$>Ug z8W?U~C3?&q@%njOFAN5oq5uUh2)hD|F&)+ToeGI-?m0!iWdnuVB<7Q~yU4P=+eYFf z;+c#xo_I2(@UW2WJrRuA1MT*%80XhfCt4pUwCDR1r^?z`wdoEJ`;I->x(+U$$F>6Ez&-%^98^kj5)c9OsUW1F&8sOs z`1r;CPr1j!sj=N%x=?ewxWX=0f)NHrRyu;$|9(CC4gYQvMPwaoj)vdYar~l0@egP+ zSy_*f^mnn7k-CvSqq|>_qu@tAbe`rza>Im*p;1{wN{6|>(q&$8DMIhs3G-H9P4Jcg z>R^`{2tdPs`_-Oq5;<;L|4H{FG5G_B2S-OOB|V7Ee;(C0L+-%^l{jSJsh(Pu1@g;~ zowxb3kD2S|zHOK0Lm4swM;k!(#&pYj&jV;_WO!IhTN_hfH&q%f*9F{Hcz8JQA4jXe zV2ZjeLFWfoS3%NwWhn+Q;T)}FB3t9B)9dbTMWE!7I@dLMw$(N@Ja<4U4AxY3Z?yQ| zM~%(iRSy5WD#DlTogywspB~`>zn;1z)ibL~N2A>>+k$7jHFa%_?F5~LPG6HiifNHJ z3~fM~N;`pzj(~cB6lTIXEyZ@q1S2#A+?~67)uZfJ;DB`Q=uc17ryAJKxzb2zY`=SEggKe7oMDqh%u}pwHL$(!!3Vp9~^g|wp}`AuDV+wZ{~el zF_S}`fv30I$}+8f=UWB6+BYL@R|^Zk3<7kR8XlmtS*92=Im{kf+S?N(F$l3;FZw!gR~My&q74tVvw)IAUx- zg4W)23V^H+Bm6P-40W(_c6NK&aC^R=tX$k1d5gP6-y%w*E1mU-8NzqTKq8zr?c{z7 zPZvD*_>cIHI(%E*o9pXs*!YQG*#`T)$cg19ciS2oI-ax&#<)yz1vvlY9@z8~$kGsW zCU*9~GEn}M)u4RX+r_VHrIIbbC{2J%CTkft>sbW#zi@}=<0Wdhc*op(S{T@qQQ^8b zA5-F_=kw~7tctGA>2PGU+K)RnF;vzBi=E-XBLtKi{b=|CK!*Vee)=DGKi~+-C-y~J zFf%a$db4$GQCIJB^+!d-jK=vp_=59I0Vnb5xR7Rn-=H1BxC-s0-c%__jmK8lzElwo z@r90_y5c06n=%6aG5Wq@z76f&uDQ!q*!4Lo6bx$c!leB1ld7N6VoyE}sGupH`oV(= z55Hp@2k0@hPwd3}z`vHeN;F6_6X5-ZzW@R#G^BLb)c1|roR?muK^9<_L?tl2gw)Jm zO7NgH;s0-7Sw)U(sBUFr1K6ZTok$PM7fknnxfugrDE$-fbJ7V%oFbYr$=Ba-AT)UZN2Mdez{34XG^8cUc0!9Fa_oIS3(EzF z75HCxuQvJq#YGdg3D{wOPq9^PyeOc#Ujb7g2j5mrBLwh4$HvA0m`fA$fq!jdqc1Q7 z*hm2ZU-bSoKSSw-4FS3kuqTnBKtYsQUT@7K17)uMk(HHNtpVP^CSq)^p!KMuLoQZ# z4n=-Ji)<3&zZh)@E&rFidrSr2Gmm=Nk#_R6jGwoR2eBmyNO6Fba+>vLs0^7D-F;ucnp3X` zsVCqHHE-W+81$I=DCJa>hFaa(dBA2cJ8vSHH}aoH%-43M*RWfloZ#l>w$)_7r(rHX zR|qgZrav4P;HApn3UA&6CU%{j0N|3FSHH&F!$Y>1gIT0oaL4X!hJp!(=85h0A<&y0JLzMC{;;LvRRBew|j=xB2ssEWaZiJJtnQz974U1K-hw1S1DJO6 z4>;ys)ECt5rA22po**;Nbm*k>{QLUP*TZr;peI~!4dRg4f4NfhSAWqdJV}E>1p*+64zs(8uR~ir)0iMqiuq%xu%j|B~ zEU&Xj5EIHV#%~UMkd7kWsq2RByzxa(l=nMwmV3#DL#|^&jOHz7Z};9jysmM5nOIh| zsKntb9pI@yB3gaohue>|ln$iUyujT=h8&|qZVi*iAr`*1Ndv=LTFiC`MBOiJ{M^Q` z@^~#fj^kqiSCdGo9Gk;qjFnT!F*ZbPHaCaydT8Owd)(zeLmb)Y7w8d`X1`kUCAN2& z{35@vXBpBJ;G^?}mzwAhJbm`d?dSc>*B^^K{4-a_$EL;6^{!STQz!F$qGe@X%Q6Zs-nOEdRrMCei&mdZqR zOmE48#YmksV%1}aC?=?@e`RL+i&;iSYG;=XE!UXXVH$Q~a+dI!-Y@QU0ZZONdJ-_DuKwUir}fMH69B{yCvH z)0LNR>1vIT88C6lWPbUd50x$4uDOW(o|eAd5>3nRkfkLhE3GuA7O(fruq#GfmFIX$ zt}l-vBkh^B4wpZ(|LB-t3Fv&T^K)b9qDh)dt7h@4PT^{ZdC<(89uWQyp>sEWp?T zj<;^EFJ;a;$Xd^qVyfD1{(hkZ0c~r+H>%{D9r)kkqU?87Re^MO6itvCm$|t){A9+y z4G6s)*H&VIf;l~!i3y`Q0W2-4kP|)Q@&!O!a;oD-QHE|c-3y&@F@SxDDo0~^5Uv;i zRsHGgDIGEzlGC1|*9|ATJaZ@j7Z-nHlbdaLUmh zUxw(+-+z(G!EL>gz=BC$rDMY1fb3S`!|LLcsD@piyQ$PUz$E{ zz7>w0u{Tz-p$M>#NzpS$141bo>k<52V4!4{CezES*Nxf#=qY zr#L)NIx=D}clxKC-0YG%V4fdiQK)m1n%~K$BIfdv^xDihIV7f9jEcv|H-_k{W$z+tymxldi^pQLk zRg>RhUeP?o%)oS~-Kvf%sfHn}`cPEVyMZg3r-}#S^Ud+96x$S(a_RlhiVaVJ>M2P5 zee5XtXMTg}uy0i2Zz&&)8-;$*uQ8z$hdwEl6Q%@+v~$@AOx!zJm8f}Z5sq9%qL+;0 zo>E27rkqOe3A%vq231v6@$vCt>H&>!v%g;zxo^T_3ZV2xhJ)rnN`ey(<7*Tjy8E zu|QZs?D*ER0ppyT``g>w|E;L#LXbWzHOfI=evV-F%@{-=bKsCd!`gl*YSfjvEE^@s&hJSo>dae|@odBsG4O^w9kOTSJ+77-4Q9g`v zXeCVoY?kojNsw$GFDX?Lph&PT zGI$Z}!jN>y`9yI1sRXwzlSy zvr^F|QBq)=Vxizypb5e$s4yf?{VGcN)2vtI&ghCE9nel!>My)N!yiZedM+$K%{u_- z6vDq6ri#vnDgIj83d7Y>-VM9@oR8PL!am33`yH+g*$DihG%EaaXVj??@CK&r_zwb* zH)s8)RJmLzJy2u}O;X!`NaZr`=8j+ppw4UONxzhk2sl4jwK0hpEna!~A@G!G7EpA` zs#<}cJ_WISiZU}dzXoKf&UKkHK&Nletn!r;x&M_X^WbD>6^q_nZuU*p+KeJ$akJI? ze~RrOz3o-$C}0%;S;cFB{9lYMT+b~y-cOeE_((2$UQK;uFF*rQ*EhRZQ zJAuMR`Aj1>&asKMbFc5D%r=&FwFXbcIL&0O*n|6EUK>dQ>j=M~jUbF;lxBYC%f|z( zTHrUS!djzQ-Yw}F5?2HCKR>CzlDBi;YPlcy>#6Y*fD&zD@O~PUx@kWZFA|AP;#hqA-EpL5lCkaB3+dK}jhg|0$sd;o#pKfNto&A22sD zn4NL=t0Sfii&b6KQ&YuHypu$b%b(auFmD6uNxUm$mHp-u+~uwSl@gnc)}ZwF`a{>e zHB>6w2v-;FPUWD*$1DO+om6=0VWzlVecbi&M8y*E4o(=UU)rUjWF%0VVrvIq2uWxlUGUYdU-2jum$Fw8PIoq7Sfz3LWvI}|IaVa%Y zr{VaPxt=v(D^7QoVqE0dH4&WoiHH0n)m=^$O8TQn*YbFBk)wzv{PthCqa97zD(47k ze8QL3`>eID>OHYafT)#PW2MnE_(?$nQ*N5;JpR*w71JfhLg$>Q3GR4J9?x#1jvp~d z{^T+-QBJJp1zXX-;mD=09e3$1$iBFL4IrHgF_PRl3ylH@vYKA!OCD37?#po0_ve)p zuYJrL%y{h)#$Y-4*z&S9^a==H=&1o(PP(k*uT6G+g=-)-4G=JH{p?I2D~kn^7Jztl z27C^=w!B5l+$_sj@TRw&54qo5x80sm#%p{kHTFRvA|O6+2`DGzm4!0~=Ba|nR7{F8EWnY7s-3hOtsa1^E7TvXi#h-BsD z<%Jyda{%R(dU|tyxPE(;cY8=CeI`4zPT#m6ABn`VkX(Jcz9EhFEk5E6E86MNQ~Ls+ zd$x3MIu*VY|JcE&ci!8L?7xui#o{ZP@kQ%4OMMZ}*?(hXG}%k_kz^FEaX-N7*~2~1 zh(Tk!IwOOKiYM3k|9n0^p0y-I_-?R=<%GY4YIOEIo-e7aI|Z0oXwvI+_1RzwdQd2@ zA8R?gwdTLSgN;&U7^P2^-6axY(gHnFfhQV7S&;n3cNF3;UrwU#us)CDfINS<8|WTe zH|1SXC0g-<-Xj~N`q@Q%dU(}FQ0|%h3Ndzor>e&4NbBTe=J7>nWHRp#$Z?^G?NcBXV`LRo23Q&Y{9Q(j4tK>VarA)`=v z9nd>r%-Rnv`7Bz*#Krp$ug0~3L8cuN09=8AmjLa&-DAGF1F57gtK75$1Wkd=g?Z`A zH6WJY{UYZZsFy0J7=4ii4BewMFSEE$8$bl(=*X+i#LN7&llf+G)|e^Z0Lnah)ICq^f!{n_Aa{u@kx&k=4{VutDpv3{a#raqQWk)S=09?M z2(LvZAS#Vd|-s z>=#cdM6&x%`R}pRK$QHq3*>@tfYQsAhL`?J>(e@?vySa#j8%og7h*#1`2pz#S^y3;2PjrFC-C#8 zt5b{_=yG?8ii*;6@}?bPFs4H-K59?J8i5^ej2G4ji_W zA51y{nn~$9#LH{a?*qSYB;JB4*0n z(~x9b{?`>(&P!@HUg;`ekH^xTNYx)pkY9L52{o`Zg%z9+ zq%bhAjEwSVEW`z_ij#y!dk`ISIWV?QX%teOHmU4p0Yk(oKv7xXP4^kCZnAag|+k~p|xoN6t^Rrn{q zoxeJcQQ`aQ)s#fI^dssAfH2&4iyY~Ntzkrp;;m%^+Ik1SrXIh`&CPY1$^ByO+ScH? z^e*I_yc*j?UO&e&9?!FI`m((ql{`cJu?TOugK;O+*H&k^ajZjbeClu2&J4odL}CMo z{rWM-Cuj?%KVGJedGyVD_|b>4=^`M8`Uo^yUyv=KQJ+I1^H8*&s!O z7Uu6eKE?nsybv}mpxyf)bl4#pTl^&}4*F|jtCgPsVMm6A>Z_9kU{@AelJXCRK5}+A zjl}&OU%@ekYm14|Tn=xwa(s&7>T>^-*V<=(55$n1SUqg)40*lC!gArI7~!K5BQg?j zI^o+!uyfr>3F6WFnaTe#4D=SrONnADE07611U7hdp;>kaj6Cjo@X%HCi5<1>wH9d(_RdBpb$f?_HgpwxIre2cxKAsYD1|20qq82SE2>ouOd@L>eH-FLSIjO$d=ODW= z{D-}akaz}hM6{$wz!b$YHN3Fj&F!roJcdGveswl~T+df~CA=Z>QRBr5 zckRx1j}tG+0p44FAULqOdHg^~^I?IZBs=5_&)AE#voo;>p*+6kO2Hl{p^`@l-iW8& z|Cy9aOH2L9WHOiA2-<>+29sdew3USTucoF8?b}V6+nK!Uvo*k=zPQ-Ff?iv6o0Jb_ z`14zuTh2-QAJ~jfxMK9&%In;DEpNwZezdlphg@Mpay+B6R;evm<)C;>Ek$tEJE4H2 z&5G@8uJjuu31Nx2B)tVWJkaA{5Ld;grqEJ_9_3}{zjy+6XyBzFV4)IC=otGI)^?ok)Yz;y_2}#V|^>e`c**i;$Z6lfI}fya|vb22?h=6t{CDdRv@Xw-K-8O#hWR-JB4KzjCjp< zkE%a=vhIVc9Gu?gJh##65#5@rKzOFDw$&*s@jN!JJ&>rgqbkW<$>!|kj(6%uMrs`@ zt$iAhEEVKB$UsARzU<1tsLsaj6A{I8I)RUUY~MFsw((5fY_>$uK2eTm_|76|w%6)M z_G+XfL<>l>^1m)H-OSf6~KV3cd=rJprd+`H-gK~ zVH{?^?Y%uoU-uoBzB~#9LQ``SSxrzUPm~!%dfBz?=zk^uv z-!4KDGBUTq$#*bD+R(lKE~nX=rHD zPjNCWjk$oTC)LPQaYpkB}*3!jCE}qkU zEyra$^yZ(&U+XZg_YzU7H~i4gm+vp0$28!G$kuULax;&LHZHKZy7~P1+Dm3>Gs>MU znNzaVr#4{~_sczoKigCr@c=pDCO3H!*}hP)4Nr%Twis1XDSvVp5q>DK+^47$2|TJt zYj_HPHMGL$Z6g*zl$gZuod7h5Pc(}Elbp?U1&{4s7??*+H?ICqR%`$rU6`Olg&>#2 zYiKQ}$%<_bW25VRm#2w|VJTiMYQ-4Ne3C0%e{Re#ER)iq$3+Q5!(Mmsld?d^H{DT>_a7gdoit{!uKt zDj`9eK<@faik@*AD}iM%P(Q%2LGbl8;i*X0B%sC}UXMt>6o2WIb+fp*_^*I4J>Ag6 zHAAl{1jid*zEF;tPK7x>IEJ?w^R!+rhIx_(>}V9baM{?c_?peY*R7f@DeCSfHeEzJ zMQ`)52RmEC)u%~1a%Y{IAK{S`uuXpa{aY6*3+|w`CUm!zVTZnICQ$pOK#=tP-Pzlh z;W_71!lS&W&mi#{pM4#Dtz(6U@c>$I)Dg&RMJ%gJ-WR3HNsUKA^!8K{|F=s8hTAvI z^?gMB%<#Y9^XpWG3F^LPwUQ`Vm3!p%G_;D)&o(`u!C+GsY^-2yMNtK@@G^)Lo31ulP8ks0{J;!Pmkf1{gipnAcD3PHy_QtXLs1UBE8i zw2{YmrR*_Ud!(+x3bm2sdA2il4rLoTOpv)TLe)+prNqR5h$5$Y?;yzWr+e+==od!f z>+^8>P#~Cl%~e|>KZ0v=6Qb8-e>h+GR5*?`b&@(1(9oYpFRiVqT?QGzyy5V&mp@CL zt^mU#{-Hy-LR}(i^^a}$*qCnY{?dVy6q8Kg$u@8>3)qQfF7HU%ThxAXrfRK%xBP<+O~KPUJ!-uiyOb zP_Fuc>r~zL?``+rkV(DJk=>+efz*3O+=>RJ0mp!frzE*?U^Za*p6D zDOOf^DZD+`BMe6=S`Por_4#J&j%JGv!mMN2zDgxjbvw4`lKvNEm5&mQ39K(a_#K)6Wp{-RwPh|;? z)^&&*!fe%k%=3ifIF=8_3LKI1DjMeFmD}7L_XitM=C*+g^{&zUCQD3KRQ!D|Y$hrs zB&zI+x_kyOT@BP&fO$h1dp)bddX`88`9&WW9mF|KFh0t~f;(O_ly(OEikB}IYxO5+ z890YmUI(!7cj&u*btaC_4<{!lf#VcFdBx;^Rc&<4x|l>j5b;0XPE@#kt74uFnRrgE zi3vWt9%=5=Zu1wH6r);CJYQS06UYQaSJ8VuJ$6&6yjccMB+|}TVY%ycW1WZ1qaS=) zx0+4Wbq0x@e>#4~B_r5$R=PW4(3W@o)fGh~nMW`Uhr8V?LG_!MX_s1gnQkMaUi0*T z5S>?ZO_l6VSxuZD4%EziEgZy$JcLpy$>RGck-R^lcoNlm$kj8;N&#K4nc9|Rz>5Mq zkl0N{cbVT88S^2FW~h00`M!vcJPa(XBun7@puL_Jzkj1~Pzs6zk3K7=Y9LUbxw+Hj z5uIdwM;=le%PY4nS+xuCO!z%amzDH%zeBKw>jg@{y4o?_-4Zwh6;p{VM`7yM=d0%D zi#WA&gKVZ-zI8%IU`^!w4QZK|t(NEq6s&a1I#j-L_yQ?uq$&^W&}mIiN`RJ?t%;E$tRT@Am>z> z74Y!A!Hx#05`T|~2oP0vWxg7XFx7?QuKlF!Mj8$qdNTTL^b8P1j=Kg0WtBL+0TPyj z8q*39l=QuqgqkMh20l7=;kM%(KtSPBxsRv|#-f7~>e6h&sD*gez}&~s^Q%Jf15(S0 zrzhh)=q8-`O4fW72&p{C1;+Lf_@suNBJAjTgzL~Kxp+0Wv^kb<2Ds>{70Ji zzkEfrydtL!wP!cio(-Pw5JdOLELeJ_}& z7I#g`ackHnKR@y`7(4Bq6`cNFkor@N4~&^X;}D2K#?)h?zSFXHNqaCP^g^zN#`)R; zE-nFMcJ^i4?Z&^X7*u%YKJwDHryx8l3xlRfpZ8sm3N)5F7aU42LN1ZbKRx>REZyAZ z8PP|dj2Z<^{lD^*KRqQPMOzy}L0_dCe&Q~FI}JZ6YW+aV=(rUld|Zevuc;X4s-4<5 zWn`7TU?dy@k{{2ob&s#5@kRq>UN!;AKgvFb{BTZ0U+VogbtXrqrldZ8(>R5mzQMoD zCN}@0=)51H{^K})_F1`;bVN9ZBYUsJIqQtL?2x_5-em9X$j;s?J0g2zXJr(!S40Sn z@8^4efFInw*Yov!KAw6l5&9WcD?AATk-0~BpoQf!JBaV=KbwJK@+%+y-yr`VJluZ~ z;OgTePNO*IS^LK>S~)5o-#x>C{b)tf=Ow5{0KeU+K7>2P=*ueYb_w!PbKOdFIn@-+ z$b4%Y%jLpyr}vbn8r6BmE1Oz2{E1zdMf}(_>?btdDzf_|r*#T2+Ze1F8Sm=oW|S^a zqd++laYHZ*Xzwx_=hd~QfEIFFsb&6m?;G29Sk5iJ(66Ze##gGM9u6PxG|(CGo%sI> zi-ult1^+C*{1$gb=>Meow*H~~`{3y80QfTg+{djsx7Zn5ONv&;hYY#h?|uH9ZJ*;l z0htMxfPlR-;GP`z@5hI`Ll+&2jS6;2iPj6&SKspkTrZBZb-ti(4~4+x(Gb{#4zg*{ zF~%Wp;57{NkDC$s1t{Nf7ijrs5Z&<;jA-F<6ZwFayo% zrhg1*APUaa+%Fz8y~os#rEy@*rVt8h~-wH_JjIF#4q0a{P=&JbQar_N<%5b zSUc-xyMK&r2ME$jg)ux^yRyniQY@}w;GwPlVY$=jC5 z9eHOQc%uu`1o@C?yanUqla4TLZM*O}J_w*(W*G{j{U=}fS?~0w^p$OI?SPAmMP>dx8i0wgx!La~ zjUSx+?f4?XU4mSr>OHrj85R+F_<)}KYs0>0^`0mU-nC)xEeHY07S}^=a4yw9zrJ`p z;61O+70wLe2IorHfag!x;mqbckkF!f0`yb=fG=_kkrMpqaQ}o)SLv0}wi_vzB~f|H z>LF|b`YD;>k>fjALdvaNR5@jdqxTK|I-h6=%7ucu!9J(r&HEX%IzRwM#sCYJDS zhmY_vM>%3NA_O7`=dt}kT z7h!WJ@Kb@5w>1{>M1-8K2LUe@FMPeSci(BSn`s5-Fs8_v?@W7wLP5!2$6O0R@KjI{ z^ZS*@Ys;&`&7!OF_dj@^{Ywb(yYf6yy*Tx^I-vOXJ0tw$&F@!yQHX*sXL2g|eMqwG z;jP>8PnLqz8+v?zar(XeikC+w03Lah|M0%fL)3~9UxI+>81VY!Xfnu8*D(v>rY_)* zR`Pg1JV8$zzQ*CW|1vY{nVQ}DRPXc^sY{LTMfk`zVf0p}U=Lc+Vt@vA$WgXQS@hA6Fz7!2gR`|xe%Y?(Zwk5LlI554W$b{|az3+y% zdXPx&te1#nQb_c2>-_ltX|iQHdpI6-RNb-J;hd;?)3Wevn7(@1BA-d@x`&sMcCq>w z^2=8@ZFkb}G(S5MkYhRG4a@DhZ2KoE&i z+}Zo-plR+9+4odgcdd7N0?+W1p@K@vj#^qsPuZ4;9a`Fd3nq(i6We>7G$6 z1qkzL(Xt=7;Q4iRzxvl*c7fZdTsS#9`(#*M00xPxI?W;&S-(4W+1J(A*G@QQ5CnT( z(i_xZ3Hx zdwP0$F-E=vj_~91jM?4pnw2rAhX<_ zpvdZ^>kk;nLnsP#A{0{U7jqzew!-vDWyXh6Ou9Mrg^UU81{ua{<1x09S;jAbIe6y_ z8>I1G+h5r(w@o|6LHnlhaT@HK=QSS%pd#vbEs`?L2XKyL$WTrqoYPYZi-yAvc$;!E z_M@>bjqh69XdD~912;d^(tv&3yjHL^B^(9~fsn5KSiX8JADCP80^o?A|X*Z$BUGZmfMsc%1cN zUKb%Xq}F+nz0LR(0<|l*4;J<&0HfeJ^Hm>>GNfb^#}SS(MjCPR*dK{zL^PRM6V|sV z0m5p@BxX`s^8&{Cho)beee+pW&sb6mq52l?hx%Y!sU}7u6gcpwx?ZPcjfg@@1ad{g z1Y}yP9+OihH_!;PGH6{AZhoYy%DG=<*G6SMb6*l6Qd*LR5az6}xgWtc);L%4;nNyU zSCqfDrJ&ns#hXAgL0g9oRdXmg^>K4^v%?v1DkUW*q6zgAt!KwmP8JCF+dlkN@V`C; z(g5lU4DaS%I%iw0Tn^wtn`+jl5zy$-XjpI9C|V!8mx%|Xa2~BQUa^1V5Qeq(1$UoF zSunuTu=s>=J)rKFEIwf+VL{^TJDGHl)jNlR?5dqZcW>WP9)08^DWp%GX_ktFn2H-X zbo&ek{}ZjNxYa>3t=-XHDO)E^i6>pDJ2{c)=>OVh{G%?5VV=w z>AoBAcYEUHtG|F{K#9-9_yvIRip;VZRvLmq$#b8l8SH{<`M0guC^l1Sb2P9!{sz4F zyYHh%1<65Z$$QVd^^R_9E7vNiO*IU2q>OT?@;ZzqXT?fK-ls!WTdIVJEmpk(nk_-l zOp90`hZi9f9k@oEsFTmxRl!9L+FMpUx6#p&Z~zxpiaj4<2o@onj8%@6JMuEDTe=9{ zV0;IhOZUi#Fs&cV2zmq}$}U9P2HRy-jSs{Q4Uu;4uGO6{6lP#zc^c0!-Bda{_XcTe z@33&rL!kE}H-2ikKWNIME7)M_dTPG#-GeJFY(sR&*3adN)O!76$Z3NT5E$~2KE4f- zd4aDwD-7-2_xTpsroXDlq^NUY^Y7amN)$5Gu)TRncwxEju%q?lKr&p`QmJ^x!>~2b z+|1O)a@5d_qBlyE-Xd86YrgEPIp0Xg%}FM>Nt<^UzEo69VcNQfL}Ed=s75}alQ5E% zyTncJUP_PZ zpj1d99!h0=MOP~f6Z-hR;-#R=ZI!zZ^0vC_N8hv!I%n24RN%7mTn58^&Lx-2=gg`c zie3P7wGh=tW;Ps0aZ0kber?W~U=J^$lOFJP%GPgwdEFVwc5tr(QYF+i6%}yvv$C!R zF4i0tNkUxt^+QLey`6~a{_U|cCA?}er_MS0t}>WXN`5#kKvYITQvCP#;bFrkY?9yK zpJ#2vZvn&!FkrpZuU`Zv+ra1IpzO=|@kSm1MYAG4e-G-7Y*&ZMiVuQnX1Pdtm^nz> zS@Q^rJT2pi`hM}QOlIH97=SMypX9+Puu+{V;tj)6X;+U(uh=xbuK%zwGaY^Ay?gu6 zl)&HQd$%&xN$ORS9AAK{LG*~Dmxt7c%ddJmGlQo`-X8!S)t>n)4?hpL-DcQpv9>m+ zl_W)0c#aP1;m7yLG2nvWZdIG2V4?P5pP>A!JU*O1W>jgP;MExE;-lZR;u1cD(cqEb z+dIE6pDu85znxetFG(9Ew1au1Yf?b&Qp?d(V2oefY7+Wr04fk}^~+5pDTQ?6K18=Q$F@gv972(Str? z;fgPhgd-rE3vfHT_n6mi-4U)3juakWn`!%7=;h$xppW!|!EyV8iVUyqh_C*uKR|75 z^r4X?PP?xU(_6AG`x=;=v{+cey@UYT!Qb*Xxep)uoG^xezH#nitiNYL#sV20oR0v8 zXi90b{b5=`!kjr}cm=y-2`R?&boj{=(Q5n4lR}5h2Gx_VLI-l)Y*?Eb?t`GJ%oHG% zLb|z28)o@$e^;)Fzs{eOJ(jh4;`pHXCHTO=KCu)P%qY7XEFLS2|(mX!=l_zZ!Y3q{64b*UNeAC@X2CG3!lkj8)ECd zoi<%vKThL0U41aCK`-uAGjSRqXqFLWF&WAQOx%r6uo~C44uMQ)AmaHShqD}(7O*Z?VC;;CVhcxmVv71X90S^Vs-oVJ20(6N?!{0 zJtuGDSbjSj4)>#b-R6L<$+_8&tIsY2wD#|5985HuZf}iDzP?va0XT6@HTt{@aNBck zezFi49=T~skF7iMkik@qZ~(z^iJyxuyDkJjl z%=9CX@b!-$w|R-yJQBp|8P!-fCRWmajxYnyvTiCP@rbfq_vP1soeY>Cy(Sx}!sxz! ziUzMGsIdwjOn*1N>ii?156bH-0f>c(y-nD-4xVb8r_Jf3IP*o%jD``0*qYrofXaC# zbNwsK-)2?paK3Kfv2ZUQy+4yE(gb6 z1AvLi9l*wzoh{}Wi_It=K+*P=Ahh_GxjEpV#6E*PaJqhXiQ zJ#vW5EbbAL*SBWBu+fiWTIU0j+Br@X?N2fth_#|AFu?zU?It!_@bx8|Q%?nPmNxvy zDPfepA3sZMI`W{}yz=dQt7~{~BRA2z;|C*-oOk(Y^aD=l_)BT%9G5*8v7QxEHFnuW zNaE3qE{j|AhH*cUg3lb==dbbapW{4dDQr`T*W~rc=jqzT#Ga$|3+%F`r}D;{;sX{r zLZsb~q-?fK1!ZJ%CAY2*G6G+Dh(8}3;e^JDeB%ZYun?%M)C8{4hwhgk=m9F3fmq<0 z{9X(afF0;WjAeMfXzNwdNCiWu_dK8w0Pje*!xKolxu6RhU{NJ+5D7Vnzu&Y}-MF-- z{|XruW<=@b_|J?g=~)E&PwtcE(uQ5jzWi+l6N)$d9$aSDPOs{ocs|*8H&8O-y>FHN z>SpQciO+9l6+d^+r3-ak8U-!Tw0ReqOoC#XO8%1cBaVAe`Y6j@LsT4RNH`&GI#d9G zLoMD;D_+|69JOdYD?F1FEHMU~~1omLr^Bm=}_;2;qV17maYI z!tlm>E?AeyV@L{MYxS;emyDfv^_%Gs!!&{X%C^0!Pzwb#b(|vc_%a6;p%8;cPu|ez z<#-YTkVEjXGEXz9Rw72oD7*rh{CKj z|M!kbZmnNl>N|e^6UkeC`WB@7TgseDdzQ=n@E<0zNeKe6|Vj+jdvJSTh>3 zmZ)=!aud%3;20465~MPcvk8sco`kxrIi*!4f(^KPw{u?WNaRl@R{wAmtP5^LsDJk8 z38ke(QGcd)XN`u*58y@Lm4j}lOt&JGMLvlFuAPA}kK+fWavH8DpV#DX;ar_w^>+2U zXD*XWf9KQ|yWg!?i_C>a6|vzcKYG(C zdMZf|rgg}YRx!v@@2?CJw}D3)o$b!64}l8m>*%~`uXFHl7JgWNpa{`VCgi5KG=;J_ z#2U9REr^iGs^Oc0&!vmOklDHP#bk((Y&>WWITp`<#Sm-0BF{F2v(n!o5)xCNy%wQ& zS-3RYv+s^9h?#|a{D^hl}u*K_%g;C3W_Hfwc&TKw<9YdF38HsSEJq;_qCDHHL z^Xu^h4>3+2$NMKOjnIS>lFD)yD}T%6N1UbT%VNp2Qh9)^qWF_1{($BFIOpXz57DfL zxrx>!rJa+0d@h|Pb)DgL!mZ$V0B5ygPMivgt|>%*^5$e)hxxvD)YVTPlKk!m;TEuD~oDZzfBQdv?p5@G6jGW1KCNY|D zgon&_yj*GhGl>iOtwbgBub7J3`^6Lgpg?!48gl5OCxPsYd;V-it|9-QNJ zIkH;V#2zPeVLmhO`La zb*x$u)Xyx#rDh%sa#TwW$fmI%s=2F7vojFrD4JTP6GCUcE;@&_K4NH~a+1IP{EneD zvq;|!1d68~41POgZOR!6&Iy<3$QS1qw>h)Nf$qj7u}rxu*Q^uB!Iwo)aRSyOnKhol zL-Z`NL&&#zTa&k+hcm;EnLmP}!{HS==e%Q?Az2m}8hm_c3XlY%vZvMmRHtOT;{EPX zjVHh`qy<~32`KVCpD6J&E*~0#$=6dB>#GEZU6QeYLj};Cig((_R30)g#Mjg=#*F}h zH&el^X5b}%e}!sIRC-a#6wQriV?bL>^~dX1hQQFylC7j`_h-SYkTZM19Cc^f94(#D z?FH4{E{JM)>sJ^#LKX=IGcOZ?il^Li0E|SgN{oanw(-80Q;v{Bc(N=Z8Z}L?Y^@<_ zj2=XbB zt0y)ABwRJ7zW3BA1YgNKs-Ce8aC2+K3+yi7+2wKGJ&bIgXc}e?1g|Lm{XHr5`3h{S zWC&w$YDrcMwO<5X^D9QMGp7JYVy;N0hFY1=ak_BD=)q4<9<(0~(c8 zVqO86GW|lWvumm!zCVi&idFhDts(qkNJ zTJlaj)54QG7H(w(Kdn2T$gA7$eM?+wI8osOgB@!j?E>!@2U9K1ffwe$$mAe(u2g&BoFiGno=M#rtK z3|d%FYAONXyhzZ&cSES096nsO*PCo;I8!!U6HFHpf=4IkcG#NSVNwmb8wX}_!=I8| z=hSy#T$1J+7}2z?Q9;46BT9PwT}evYS!~-%2y4RS)NO?abwoS+mZ|sZob_138@at1 zr9`axmt9C_{9Zg)eJ2u~X~j8|iBjXdkQqX}Ye^hgt~u`TYoTcD_;;$pBmv8!;q_Vj zNeNBl+Lgef;9ATRv0%DZ&hYhTYs8xn;MIhrMLg%%JgsyBCZYkc%bjeGvXAEo%_w_> zt=IQ(E#+Q4Q5vruBpItGu|HpeHOQ6~898IkjXT0l9tSiS@ak2kw_KjOY{jB>x3&zH z%8kQRG~j*j8Cy*f6l@-URv^b9 zSX;wH<29eKo!_>G_pY}uR?8<{^HZVF8O(hR!uZML(H;gpD(^&h|S~iPuOKa!60tEgrknk6yL=dhvL#APR{vCHI*fl`pPwA zRR8rh{^w>V<3-O|!jO)aI*~bbR@TgpE23k98l+h1%=i-kkv*C=+_Ya}mp!KUGEpOO z^nWH|QH{*)U#iqoAfvT#xO^*%CS-Rf0!1&GYgvAl^vK|@RBF6OV0MZh$P_R|v0(O> zWb3{6@H)xHoyqz8w~61nD*_H-lPqxNaAPwfp)fZJ=mw)7H)rJwzHz7)g#p?!Popgm zzq7+kgIf!i+I~1^JIWk$b=1{n89#<5M8tr}z#m1s<(t_pcDNr2>J7AOgaE+7<^^h5 zLd7ztug}OQNCHhEY-C}gvBJcyf%9}=33SWiN@5*;A6x|n0yT{TKz?|Ab1X`z>G=tr z7TQB<2sY6QvXv6|kiEdY+p{y=h$1ufMHMr z!nXWE1fF@K@or9;2vU*=pyS8dDX%W5(=Zn`a>wODT_3->&8(U^GBr|Dx~KBKJ1c5J z-6RjcDy=8l@I8XK_9}>V)zc|nCNe)uVgT0lG#LvZ^quL`n-L7ChJ{N3$1JAKxkQVD zumk4U2B&pWP%=P}9%@j3zT{ms(SZFIOecrrh^pg9i^Y|;v}UW-w$xH31a*!XuC)b8 zeNzoIrJ3Vi0MMXnifjMAdNVz_VIC7)6D8g{{e>KJqaAJPd-wi4a51X&cpcG4(v}i8RA#;t7K{C z!B>^_AX;M6YQhKXvW$zP@hciJ4XhIzqKj0Z9+m?ur#~`I9Q~nI zq}-?)PEaBz9r1`}sx5OAey7PO@pJ*vH+X$x&1vANKzj`^10n|ZRw<_iuEC*21tN6I zh;v;WZ1Kn2y@lfb#wQhW#?= zb!^-?VW+nE3E9^`3M+%55v>6jE3+I7FPtbrM5@nnHGN!QSQ-F=YwyNV3V=}N8nVQA z$>GVP$)3DG#__AS7BjoRN}m$}(zfC7`J2`|`LIT1G_G{Li|_pC`*j}G_s6i;^8v2a z`WiY33poky*FS#jmpOd6Ij98Xetq|xhKOTd5+GBZ+U_{sd5)B;fgX-P%=>y)wxXg3 zdYle(>Zs&;EWx4*_1zzn-GaMnK;NM+qV!Y50o~dkG1cr~sUcdshp~{7ig0~5*}als@s7;>E2xwqskbhA8-Z!lxd!*7-<>@ zX0EY5l8oN9)atk&%t&M?TxDfvcM~AE0p$=2)j0|n&Ly{n3mNxGc&pQc+r+%Ifye^x zM%?d-Z8t9$AFnHu+;~2gn+a+!!beb*8nv)>3+6r?c2T5bSKnNjv=|RWgo+pFVZa{v zzl?*&2FqAagRsHxPP&ML;P{rpku0vp;Ft(J&lNUwgbq&%`)iOUBnw=_=D@MN0rXSU z={?H@C}TAvi(@*o!U%OU!%0*ny|EF zCO~|aVGqSckI;KeiUSDQCgOAHC%Nk8@c`TSc*P;b1Be{@`20FwlbLXPo>Bax9pI&m zUiE2V-*;K7_Ha*PDZ1+yl}I*UU2Z=4`}EaSPF2;}xLwtXL{zx(Vyei?HItTZzOh_6 z5Aiu$;ZX1Xq*`VLSz=*c0^!Zk=q$Yp#luFt!hJvLa^?ixf$sr4i2{ac0-OR|d}B`a zlQb#uc)>;E8i|Sb#$9~B3bzOlH=@$c#%m_rnw^`t$84T{5FAod+yr>B42%pJRzLXu zZGZ5v{5$*N&x@hVlc2LvbJtrycK&vBl<4(PMJ2oSTtJPAYEonB!>?=LI_&>@Zxl$A zdj+_(8}WLs+0Z^C`^wYoQQ5B72l>l~<^*IHk>q@=Ld70SV2Ab;!hg!rF&lqq} zJP#}F%c4fIvZ0}Jv_npyr@L ziA`&Q7`yScp3X9i+siL1P!_(bO z9iwxxsSq~uXgXyK18hwTqJ81019A3zPqqR2ida*kjdW7-7^;BO!R>rEz%>)25(I=j z$yO16MFiy0aE%z~1b|He+!K5zUG^(Cqft!2>YDHan12t_-ff%f+$&Xnr_YVC87@A% zIo{qJ{@+IL{rKcoH;OGUFRQ=5_$g391RxpeO=P4bB(AqMIqKJp0F=w>tDmReCLT0L z%cVBBw$dk|Fy0|dHLrbl*g1p^~~4%K1m9KQK4J9zP37F*S?ZYBnS_sv#d z4Y0KVbtJvLy}(66noeRDe{oVDg^)AbL5w`tz3~J1-wsRzp(VHg<=T_A2d?cL#suz$ zmm|5&L?6xgjQo;b{^2=qub(JAK~Gpbk?=E(-t~wv%@-SxQg;zJ;)*qOX_H|7Nj9ZoM4I3r6()1PeIv(am%uGPmd| zLqXik(&d&EYepw2?0l%+><&G#w@3Zd(odcnOU}%smRL};(4qmLQMN*_qeC8RmC?{z z)0z#0!_dE~>+^MWTm$?%a#yz9fEFrS?Z-a98b1FlPLi=-$Wi|SV7&?dhYa@jZ>rBW zt0yfmFfee&0w^{ea|tP_{SL)lj-;w0ZowUWo%#L!%F)$}WA!hhEP^mpcsJEj-Tb_0 z>r(GwAcdCrBe&N!Fcm4ZwN>H1E<(WT`KMA@ULbrP%^>6*n3jnikkE@ClFSwxs@U|a zX&Jll6U=g47~8IT3)ay|P_kNSJ-?ptYV?=|q!Pyk)sxOAH^0xH%G}t#GwSYoZ!21z z^x=H;$3`zuZq!*R{}Yg!zUMyr=q8*8)LJp+F+6x3MYB6*m0^{2XO!z>pVi^S&gITd z+KDu=sRo1v^R3&iYcZQbzit5lo2n6kw8Y7Vaq-vn+8TF0iS*o+FW5Wkyixf~V0sy| zAEwWhQJhB~7+YgtWh1zYWx`92rhrj9VEc@33@*1yXg>7&3z0+z* zU7wq00EJi&QjA?u(VP8Y0OGE_;EHj0`s!G=aK2ZQc8Ns^LJcJpHU+QSN_Hl&xKue> zQ(VF$D}*O8!ZQ}kV**Oi#H@I%p`Me;0u5xuueLEjIB&}g44|E{N&a{ZCq z!;liVl%Ke&Gas5yG|v!cD*>1zPaSn2;ND6-r}iRsnZ)^+v&RDt@c zAA5Vh)(-&2!xpSor7!QKbK&{Ncmb76%aviT9PQ7fW@^tr#MAKkc zZWyHo=;tQ6L25{1Z>9lhsa#KEu{TpK435$d{ZC%j*PnIn8ZU|*>ZEj5jms4{ii_&W*^9>tX;ixx8W!U3eSMp|Klf5UT;~%$D|d6L$cLph4Zdb44Su7 z9x*TrHVNzrrhSuQVL&^dqV}n*$M*ly4V*p4{~d*bS!j1oe{!K8Vb_T5edpVJ3JlfPGZrIDP^1bW*}$* zpngEhU&Wl(BBG-gtnI`UY{`-R{zVm!nm;ryX<9lqS1Zk#Q7s3Fm4<_d=-_bWXqW{G zy($kt2@&Hk{JHsN~>)OYxL`JSI{^iNpNwp`@hUVXlO&G%yR zP9kmF`41mQCtb0?MNjrI&SKc+#qYDfT~Pp+rHj`C;0C;8lJWEK3;H`hTW(vykN_~* z3C+5y=;Zj=TUUq;vVVmd=RYPp6neOISC+isTVg!}7T>}Xy_BgO_68kZtOZoEHPiMP z0+m$Xzx>bh1njS#de2qw*S~n3B_$z)K&^S_cyb6ZYH;XPcO%J89!H5YCOrA~y9vPd zy`Fj#H)1ImGs^cmuESwcBQaj&eA~_S+)5d!_DQqpdQq#JbKYsQ21IhJs~<6baKHNf zd+SJe%H3id;7~7A6OQMi`LE|ZyMaEpfm;}g+2LF|i2)7}v<>1njGY6MOXajLc^)3= zAD#TXV0y3mcOA@9z+w?EWZ5?T$p3`r{cXk<8>cse3zh=2Um-phG9;s`8nU!1oq`Tf zJh~1&XZN4>*f!6y%uKd1a+SivoNlMYJ7u!po*_af+d3$@Q|_(OKEz|eH>bwiKti5r zWEfc!CdkS~f*h$a4@034>Jhy7rpUntP>4INaajbk?wEiBjj~)8kyyJg`c((V=Cd0M zNnxYF(P-(~el~obUh19*c}6y*Zy_)yxE>777F+h@R-!RW90x#}en8;o`gC*fx3$ch z4?%yH{i!UO9{X)|{rlBV#j$yC-Bez@xw&t7_Ven+Lk32GQebxa9C!Cf0kO_F|ALd-Uy<2T|!aQvLIZ6*_#$U*<7oaEP zCma#*gD#1PlNqvLN>0VC4MgeFtqlBtVv_Ujr<_*rcs{D3siWrYt>RsI|Gzw(d4I@C zR3bk8H@y6cNF}3mKjibjH|}*FJo1OiJdad?h|FRBe!g74^;?*!lMQ97k3ps@6BriH zhhX~Ab3Zs><18ec&JpBB0^_!}w(+wsGt_;3R8v(mmy!jglk9tJRQ)Jvz2EifXMBw_ zKB7J%LcymUKs;~$-qlK6782(eI<{VF4E?bUg{PkJ|7<+ z(D(j&y6HhYbUy2dtYlPjrHP_dAj2@HFz>sR3WRB*G{KaQ(&&IZIYcfv$r$|7@I+&7K|f?tmA9a`T1ax1ijXC&1#gCS|~%DpdyL|S|T^&Mh)~v$$e(}Wd^&W zqMuI*PsFjk;hTw1fXLc5AYlbBL+rBa931?RjC$gUo8pPFLx~>ZzXjrDN;EMY8gL5v zsRzIE%OfEEU5UHCqw!LYA2Sllm48t~P*bSX>K2!68@3V>qE^vl3F2g%$XEn0Ql8L| zS`z1cn*PFfduV6DtMf~lV`dOUcEyd6WAEaIIe$Vx;A5r)4aLm*wrd(|BK%{fl>t44 z6TLR35+O#>?`gc)(vlWa3<0_-9U|fQ$aD}VOvo3uL`MSU8KX`G;AbDiq>L|b{_vk7 zs8o7K%vz6!(scbDupD=RO*F(UzQ)VNWr3N8%=2~+0 zyva|JWo04BuMb9~subS1G5}>)G_ix^_j!E)BKU?p9x{CI!VeUlMVw@gigsE_+zbyZ z%q=#9DUU|mhNHF6N;otQtA0Wk$-af0b1 zjK?RW8+UHLIEt!rCKR0l&lR3LlfUGS-!2`Gp^C5`es7@iFYzIl!+s_5!AD3duEsR*QL+V zbk#-!LA~y85#0IrMer7J$c88~9D*OQUXY9oGng`t#Q!qUrsm!*Ge7;~&MxJE`(^|> zB!8A8JQ$9(r`c_KgkY9PB8$lVDlD<9MI$K6u=r3>-Za<@2FAuBw*(X#DDC+%(*$VD zv>HBF%*VUZ85qgL&Cj+5ijYUV3H;X4ycU&mTUMX0J!Cj!!Id9Nv0PcC{Yrz}cEX0S z?rHRXb$$HfNN@@>xKb9Y2>vLZ7`_e?OpRX})t5sglW*U;_9c%1AZLtgsZEjWohGq#zer(PNh3wm`g#5wwV0l=|QhU^TvpG+u} zROMX*lm_W+UaQMP8%LHdr`|7kFcF!9iIlDrTPdwP8$ZIGM9I-W%|QZerO+g{WZ1k} zt(T{D*{nTbJ_(4hPfl@efUuKeZ_<&oaLVYjrAL*qMwpYrk@t z%B3DfR^sr<1)W=scz4|-LVu-Q1!cJ8I*J1AsV6(fH?Fh76Yt(h^#AM|QRPUJl9Ec? zCga7ca473{0rhQ#yoAk_DefuY`+jQVC-dlvMb`QiwWne*nwV#U!@0$}lg|WW;c*)L z5Ac#>Xr+U<%x#NsELIet_cCO5f+#i&h>ZrRIZyzP4UgYc7C@^C5+D;0TGLSDSz*dq z9Cj%7k#lGj{6vG)(fFMFd%fTXW zq(I}cNA;#>Czl&$sg6uM0fE7Li|WOJ5M=kLE#tMqQo^kyh*_{OvI^Qx9CL(h3{oIg zfg7Fx{>X23<;NMH!8Bdv?!#O#~$SLjxI_2p;tV%~i| z@m{ITu;|=3Vh4xznik+b`TI6~6|dB(d+{M(1L3$L0|XLU4D)vdh8mX~=k7+J%=q`_ z4=)^fA9wOU77}Xe>Z-*F!c-B@h~0_h#DjyetSM^PSy)wcgpr$EvtY=k;Lv-yf^cDV zA#jmgy%MJga{l}-aHfCpQmXWMW%F8)MnEY%+~HA4c(QX_j!LNrTfDaIN~@iqtPJdD zD02fhhV$4%64bbM@(i=7521E=_3i3UWo6~X#f5X5gJ7|4b?K}f=XPITsBV%fpuSk2 z1Mh9F*Fa&`i|&>tw$eGgS2PMhL1CVk6j1wnd@M$3zNe_gFq}JFW(TU!=Gqod6azc1 zIJZf~zm;o>nENEL0U-x`_K=G3)FGDvrNz!DaR)~7{VIC8OK_wDH`@m|OF&)2)ou>xfpJ-bzWyoE999<{{QtP*7-9*7iUkE1S_ zKGL!j-wb6|7>}V9$HN(3=oDgMk%eZ@qkp!$%^i4E$Yol2(0?(K39ImkN`yurWo;Ov zZ9zi8^roL_<$6P&(M5&24@RV7NGpwK2P-rQS*RjF!YNwT+2dZBcA2fl*<8H!+_kkl zu244WNyWhOwJS7@Qp(QT2ZAra>i5f61jQ$7Bg)3pFqi`6NNELnaNTqJ3LrZif1slWZY^2ZUg`{Mxn{mH*G+CuJ>Gb4_D zCD;0uHmBEgkqR-?ZtK_CW;6Wi_&s1xwMYByplRn;;E!=NPzZf{+gl@^2yheHzPxp6 zTRdb-8x@cB=$-W3{yFDZJ{z<7YpbLVhY^qIHJx@QiHymxLZpo%&wwASWZpoEaGst0 zFJ3`w+D{Px>b}0V_Ob67n(Ed5q*<-5DuP;+K5_8A4J2==oDd* zWp?cDc42Rkv584ZvoR%B23gv`X5=6;&J4znv)dsD!6=BY140&72<;#WSWa|RWRT;p z7m>Shxg8i%kfphz9k)8sWkFYj-aA!A?y;;PFO;AFI-w1 z?Rvq@K@$rufIn=(Q{sjh1w*)W!`=CYZHvJVa$VhPYMXhK-y`NToG7&W)X@w z=1l>rFl3@{+pHeHrM;!fEU>6qNRg4ITJ}Y9_ugie(vVep^J~Wzpir!<6V?X9QeXwht!ZZT&cBib!tSUij1hg&~e@ zD^K4z|NUQFkdUxzGsIMyzugnpSik-g^adg*CRT37_Qm-*xKQXRRUaVP<6Mjvul^(? zB%nC|-5l04z0uC8_4=?zq7ZFnKk*D?sw_CF-4h>v41yT*G^tn59`3l)l{Oeule0|b zN~Eck0i@~N$=!}TjXfpFgl4QAh*SJcR+jzTK{wOn)Oed=p2-?{KYt#Ea;xY3#%wPq zmrn7Mml_LdDk+Rv_HlgKv?s8yGjhmrsxQT$3N`B&)n{vTtAR<$lH&o-xaSw=`o-lh z#s&tye=Zh_ksY3g8PAZ8j*b+>VcXl==5IeAwyMwOW@mT!eE)6A4{+(~=Bt(*{p_ns z{_%ZrR(B7*ASOW5kfqEpldvYLv$)|WTCPy zMDkYXiAx?DR`4ftTf^~KP6Y_&136px_~fLh;e<)OQu0#d{0Wu#>+|zCK#^WKJ}`T- z#^f@Y=3J!DZ7oPQEDV$&0%43|eXjJM!n$1Pz;jWH1~IDw+VhZXby#Y3-~KqCpp^DYgU}RZ^JaopL5+KCsM^2Jg(Yxdxv7VKNzxm(Ld7W zR7PNJ{BXKw4*4xI1XJn!K~JKRIG;&Dt9WMG7Q5)V`Ilc&)h1quzelrC$EG!V5 z08I&hfIyTL+G!f4VWjRG0G_MLZODTQe0yy0Aw>nG%WX(H~7tA`~9uM0hCNLm4H}DtjSaz8M## z{q~_42_X@-2_L^#JT=#SJ=8@b!n`1p*o({x86nOtwRlPF7eEzNT-8xj77qIO8S;X% zC#ED+xiShF31MI>(PjW;TH=a|AVWr6mSq;1-Kj`%F6O#42En*TgXncAio#l)SasR@ z<6EbGOLQd?WzH4)(Uc}xOE^o8^RXRecq#)@&|Z>LD$A^u2d@@=)@}%RYF-ScGCe%@ zdalm0$xqQ?<}JP=n#+zi80Ef=1)~+f>kVHYl~PlPIAl(HvNN_H2qDd}7&@UAHUcF= zd;SJ)dCWs4vhWfbx?)`NpFe*r1tp$5S#wl*M!zL?bG5QDrPxXucx>k4 zp(hKPfiS!Im9v+hwRM?c?d9zsoCCp#?jr`^d8#IuGeK7P@X6O^939)XQ@#oX`Dm)pIpM0(gq(K z%vS@7U6RZ8)fL{yVHs>s4+%F)OlY!0cY|<9Z>H#Ppyzz4)SF2tKX= zt#FS8ClDEr;Y5^@y(5I1i3f6!;^AY%*n|!42mp}%*`s!lY)KLJxlDR+XeK=b4#7h% z#vCRmJ#8DoX;VO33h;x+OWa2-CBa4cyAR6bDXGyNVX@J%aQ@eLMU9zmF#QV60TdcV zN&qD!VWZHF*BmRz01!k2z+P9zuiF!HW2Q{wq8TP$BU9y9Xk&;IFsDQQ$I*4iL;e5p zBXmYN`$#t3*)t>C5f|c;?2*0q%w7qZC)x9igk)tWE4$2)5h434d;i}3`lrXE9*^?4 z`+VN-*Xub6c0Hw7SKQ#l!%6frJ6`lX@b&iZ6!AHj7_`OB13tQFD1va!pu3MqtFk6jvPnA#s%@^q@<;@GSZQ0 zB^m>E_cKl4c-&*c&_c4(zBq|EEt2q$I&A5Pdw^|+M;Q>Qy0iLqB^HhKbvK?W@Y-L7 zT3?Ny%=O_A~OsC)lLKo|2j(WS^TTOrx z{w>UWE`&F-ar|EkQ)$*Xj^bW+3K&BufQGUX0=a1t5{-qhelCuo#}O!34DwTxUn1UU|ACWXi_mC}&rre?v$)S> z_gpfH@khzOu*8I!Fh`fb{KMOrPvb496wfZ5P2Fef8GF|W?@CI^s}d4m@o}CHqbWqm zqDtjSbkIuhbp=95mnHrMC^pREsDay=JLz|pX`?%C)*h61f{Nu?VNbcd z+3ouqt&aB{GPo$&B5dzxlCWlLif3+@rLQ~k+hnZnbeK~szp;{x-{u|@s_<@>G2+K? zrKoiZW(;{Z>QO6t8ag3Y>Uf@|y?F!gzC>@Exuk`VgqgXyQ9xi5psoqDEjZL=3}+PT zN&B~?575Qp%33StJh|*L;$Rc&^`7AXTD?VdDG^UEvMe!qN^4P!ajW>Q7hg~~Gz8{j zTTyD!fZcjvUKP($DsB7MyZd zKbI7Xx_VZZ1>q77KB|q_`=w*ptO>Mi)m96#llg{<^IkJ7$HTCA;^058fTse1q z4*C}T%6eCEG5D;Us#@FLKCqK0yIh%I{v_4ImVYP?rfmG|OZvv-FSBnJ83a)35Gm;& zDI_Ju#oj*0QzJQ`+1$j?I2ho7U?Ik0zUpx^smNONTwMimrF{Re1>(<#8#O10!b8_U zXYDQs8V|l2$Zct`Cr11Xz#cW1KKX#X$92?RI2j zBqHXmovq4hdgHXGuooyM**xrze4elM!Rf{NMLMR;)72H`I}ds$+ZQs^Gc<8jeb~$q zYb>VTDOQ545kb2jk*LsKvcB&-wI3e+C@!)HV*55;frg%hgM^ihf*7eWmR>8)_^Uo} zM1-uf_C4!Y?dEI=CZ0vjHhaeyLG3Aee{Omu>PIZ{FwwMRoh*;C#77-CjA{oYOdSG+ z%+|hKD}g{5p`2J@kyRs#p2(hJihgx|D|l3dkeRwIQUQfxP@}mWP{bQ!5!NrNL9l3^ zCy%ivP>9B6#LI}u@X2$G>C<(w!2eW>pE&2*!>OEv=b;UAeAWrU6{iTfV z!(o%6@jHeP1YtqcC+6}TY3RdN*)>Ku9Ocj(68DrEpYS(Fa@K%325Rv+bY~RGin!fK z>`8w|n|v;T(hY{DKu)|BS9@(Mhk&8=1^H>?0LL=O*vhLGcQTdSM zuzkJXA5X!o6uMjS{yi^0zkkk@<9%}S<6DdgKcIvp+B~_*Xq@}%;LPM$2?%LQ07c(f z>Rk2p_4aicRt!X(l@<@;$2JeczOhIACY?s(6p>oV69P{3_|m$@F2j1%D;2}A@&rNs zCM|7kb@f2wk9YHHT^=ryqQT;^tbNp%Z2q*z_B12=1)rZim(6@sl2VD)ta+q`W zvG!-6C^cuv$BwEZl1$1$$#DaLI$6_OXHjYVr$co$B; zMBovU5W_eXY#}Up-?}n{$cb60zgrTCK$Y>6W8e0M-Gq~!*O$UWe|df_)mD&qXJNQS zfU#pEhX}uq&mX;^3y~P*OLFT%TDb9k&WmBe&yT>z!wD`4ttpa+fm}L4l2Qu}$9pm7 zeb|&?-syT7cyvWY1%m8f@cbR3!@z}2a*4Gl^Mmt1R2(>8iZ>4Zo9AwKe(RT1bY)ff z+0h@}xR|xOwKd2$*z8=OTke;?)^uOKe)kp%u8mF-Cd@0s#ITDIKZg+X5Yva+ov|&hAsI2L1nXLIx(cF>RiTC^}L;4+lD^QAnov^ z{Tj&Uxl{W;Zyuf-E3%}Guoam9$QTeLD`0^Z5fFq^c4~eQjK6S24AG+~@B}q#ChTSW z4*(R2!yr`*$EPLuWdt0Vz&JN1z%gB{d zX=f>2^*Eil$&wFCr-BKU{{l^=ltl1hzF0;>~Wqnyy{KW-#Nd8}F9<|XQ zTE;l864ALN)XGDJhAGnJXdxo%gP#ej!;s3N-@I`V=5Q_%&qhqAa5TP(AXfxJK9OtU zscC;|GD}1pO@SgYHI!RkQ%eg!PIxS@-dq^1Oifgss7NFYQ{-ml#82xq^8E3d2D8hA zj!*4RO;qIOPyExzciBgr=kS0)pIW(XmtA=kjzX|X+_?4jekz8GaH!GNaB|wi#@nai z@S7*xKX1vCdmHFEGX2-GebhSvpqaqzg23sfJu`W+T?d!}@4_Ti4(?R6@UN}y?e$2p z|NOH&m>#iU^Js|_SaoDgbx0Bt5|$;d?-UvI^ZosK-kcG1##@~6ONEBa3;!c5H zN$$es>epIsaLxz~$a;R>@$b>gm1zjt92m(wq+&&}rE;gz-V9f$nP4RdN5IKi4n@jw z>7yeH*DIh;p?u;bMea10C$+zE-v(0_u}cmrKaHjW|NidV*x&{&A}VF8@;cFQWucm= zoL;q0^>%~`i-e)q{uWp83}o>GCQf2~ILv58f!f9=_lAlL?% zz`g0Agev1^-N6;Il+}HG{6D92y3M*(Q>3TJ&RUv}&h2Y26{?scss$q{ja-QpLy6+o zzsqu#y>QHUk$8Iq3CRZvI&ig}3AVd0{d=#Xli-b zX;L&8&Dm0 zi5D*kLLq8gDG{)RV#S_d8&|}*hxds|m!kC@gM;PL(c5>oW-1Rs*=1^K z>SgP;a{~>*jp6Qbo%Ju>BB9JqvU9@f1{K)|PO5bdT0<~Lp?j(1o_b~ z0J}Z48+5$n6S%-kMkJ-j67i|>(JKYyjS!W>d>)c|rv!!0YFJz~paU$08$W(5=refU z!R#XYy+d-%O*Z!?c4E1|(PJ$J7a#~nz`{dqw4NIb3dUKEdNFSPmuLO9eud-wGN>z( znTS#gLDP#5C4_u|bB}B2h?Uhzyf>uVDThcw*ucMlgK0A4H4lTIPUm{C>f@6K(Tr9G8za4JOn)e*;JoX6SHzn#5oDk z$E@S2<|mmL*ml<55;tdOThsoc#z33G0)^pWLK6|8h#tal=yJ5jurjp>dbF=7MHLkk z0%41>pn!#9C<;P6px;xtb)%~nA(Zr-rDRcxs`zMkequTT(usCl7A4XqhCsp!2N>GN0>8Eo&dK%~KCLp1Ksx_6C3Jx@o(e?QNE?&snD&8)mk3aqf)0EhciW3O-G3 zf^S%}giyZix_O~J`1%TbulDDuo}f`t2D0ck4|cTho5vrgrCCuVLxp|OlIpJ&nW*^a zUcXi4_-XmYjd98=cV}yGXb^z|Gmg{jx@VIPx1Fj6JPM>d>7s{*H8Qz2Jq-;XGp+15 zDhjRX6{AU6bsA=A-GCk&!p9nFpb1k3x651Yyc_h2(0qa0=(l zs;?rC2txT~RGHA7k$jb2(dhbjpiysJxj9~VWY zLVYV7pSu96V3tE3ho{XQLy!>J{aN(8H(95!f+{ClU>f#0zC0_tB?~8A9`?7Y7ndqR zOe+J8CP75P@Hy9{DTP<@U;1j7*+AF7t=wAPVtp)*k@ZHhW|~^@q^(MOHBJbfnkT%$h~!|x31B^ZuGnM0mJ5&_uKXIU~ve_~QHrj6#~=eN)Dxt}Sf zQ8>HJ#LT3IeA___;F>pd&`@hYJ5GoSn3d`Duk9N?Cig7mipYy1hJ;{w)F^NmN@{Kb z9D)@Khb+3u=47&~n(2hyVLN&F`Xs+qO)}X*W-gmqu0Z99``NQH(QSGK6iF$eVG+SV zo;=4r2tE!Y7*)iLhZb=`hS(4ZM6%3uY7L1N`c*b6BxynTthl0>(nE^k3c}v@Dw1#| zh2+Z@BnQEXS*7MCcXp0g5I)UdwJ%lEco6EJfrr>9%69_d4+u6^kHqeP9Y5TNi_{=E zYzBoVqK=P<)@tEdf&CZ5f>6f2{7N4K$0b%r-3ow|v~+jWEET0xEVFqhK!pbh z!Ndr^gB39?jp9#`c;fV8r6}ZwPt62fF>3bO){PLsn69_WdO$0cGNM!du&kCH2Q5To zYVcL@4i9VK#`+Rue_ny=yI6@B@HtYX=J4fV?=#$LUwF)wk_)mg4;`Kz9Up^IB?ZzQ z^(`nwsn0#!Y_Jj2|B<)MvU%<(&{PQnWi7V5M@jtr0}C{xRkwU>`1jvpr{L|gL+|Dn zpj%9bYtFfWn3Xivqc>G)WOq|X_P>^Bj~@7s5A|RH76<@6Tg2jrAo6}Kepx!;Yuom?YiIZ%dBe8>=~McDy;!jR0@cycbiz z^Q`jZ2GFw=nt&3SDi>#GF3MM)Qcue#KBkY)%=CZX!z1qmO@V8|b{yO(YC_gs=HE6C zMWvPFjp#@7OO~o5D847j7om_btF}_KJ6b21rdAT^Dsv;0o7hOt!?239J5+o2ykJZ9)&@ocbv-&gU z*crZ7Am&GY2TebBUcsUZN+;~;<0?i~9qp8JYo^;+Kqcn`_<7E)(2p7aChG2vY zREeLOsp{mh#56`P|3E#0DirAulog<~u%!^u^hh=q+|~dK)?8J|`H>j!v=*lpl$GWl zBw6)2K6hBOkSf2H&@bpxyQ~h!eVJJWBrzM!=dtlehjh%1%Z5r*zp%g{(eH^%ms;-=vr!pL^_Q(tf$3}jfI<({nkGJ8jmugPzvjqF`k#$bn;iSX)`FZ2d>Engq z1ux1M&F76qw?i4XK)bU3$mWgzrW@^$IH#2UP*YcTL~7=+iuU-XH!Uq~1kC?i<9A!> z`5zJz5_(2P(jL~AAeqyp#a)t2sYg#}6Ab8HhKHJkLLwqhzOHGE{ajes^;*{HXY&2F zd5p(m)f{CP*`_MBK8F@b9(8pzi;@=$b0vX%n~*1bdq3M8#&HmU7fl@LZ>huD!=jCp zmk*Es`=2%+HcycaN9HU2+g5p;^YOOPT_y;Gn?T4iP>-@Ssp;Brv6+&&nY@YV!HY2r**ivU1_C*mV+$ z;~Tu1R`9yJ_Jf1J;}gg=@Cj&CIXP`muY00h;g$|Fk%>|A^iNqD%}7FpP>`?yJ&11U zuWJA1(Rh4pGB*>?K-lOFi;{l*ly$KdNaB<-K9ck6uWvH=l&IPI2idokh2U*vlyVfOgt4eh!K+u||(6ycG5`=Jbcir7ywwvB4N?ixte+e^CTG|EYK!aaI* z26em^pxS=%kZ<6d&Tan;Vu}K0R+*aEZiqe71;F<^Y#m_~)sXYZ>p@GeY0iY;k)70}E9?wA)%buv$L_*2oR0ehtwr-tU}4bTub63%im?9)aS z_3QsA8z?ByhHr0eNyOiPq^qku24DbaJZt*!A#r5w5W5KSu#*N>L~X6#cROShTPtmO zN}UFnyd<8f6;6*^{r)yLcMhB&mZRqIw*BD;P1Y7~YXK`duMULAzIz4B60cAN6ZSv` z5!5aaAxO3is>%F+0By|dxQ@KV>HFh6>JUs=f`J2&3`DX}KoR!z%SjB3x4v(Z1-IPi zau6I@H{YOkcmHsi9#$n)S7~yPe)jT-%}M_=4X#|%m4Mvx-uR_}C^WZF+-n>Rq0HFZ zB}6P#v=rXKlI=jC8 zrZj5UsA12|HWaM6DCzg4?l@`~!JZ>8gK4S2-V{$hfB}LE3WPa%FBQFIwV!QZ-VPV!mnO*>w#4vj2JcpLjTI5g`@N&v4zy2u*n z7IU6!!o|Kh^q0vTs`oa;jw-U2tjKyjXr6NhS3pzci>r%MfRJ9FFkicSc>%;$6=^(0 z7!uT8rIPmz0lZpH4YOd-2w65@sL4^4aAW*e?`&>v;zJqg)L7X~-zHk{S2+-@!G>c3Zjk2k_Yti}eC1arzOvU5jh;mKIyS$7Ifhq?JVzH^2B@)KA8s?#qRd}b?{V$h24@0N< zV|rG|A3Y;Q2*#V9W< zfS;Mvgge!G)tdWo&i)hvZT5P7&`BrrS=(;Yzius~sqDzlei{f=0XWb{E0!@{e0> zwlCYe*#lR?m;c6Pe2|mmuWf^c1+MhS>97C}4$(kDo@7kO)G0M@OIoBv`ni8dpzH*epl~_5}@)uQKah+lmd#Ve< zKsJn4xqKmQHiH65?TNse^czKTqjhr8`_k}8X47@z-i2<}-`YsRkO&yi+Kk0~(Yo&2 zZxKxQG=An$DkQN}J0nCsruIzFyJ3vKk*Is;B> zvb;A|K*}fPrGiH zdHOwPgsbr}Xu(QXU$v1tkHorrdryiEIr2L(6x(HB{-VAhX=9^(H&^e0#~^-LxBPJ0 zCbtz-Xn_`40L8g6ZVEm{>sRP{b)Ph=LaD-HBmw?T>e)BiUsdK}fy_2PtM~D8G-wPiOc{|j% z{5e-}fe}zl)dix8#`7ZV*%{eT3i-+MQ49^s(K^ps`c%t~v3I#~TyxX0hDWhU<64A{ zU+`agic&ed4&E=$c^s+9*IgxDOO6^ZY%T41mj@*nS9Mi~z-OvCdEco0m^RAT7qq*h z@P*zJOOT^k5)oa9NKO_BWdHF2TV4W3mT=H!arZz|2|i(~ zZ?;BctDrauEjY4=Wm~R4d*9`AO&3_5_uq74C4ddm#DL1>YIAs>y1Ro-CcS97`VRA5 z(51sh$?7|@Po;X+_ADoz2pw&;UhOdaUFbcE4G)sTtlAfu?8YlRO}(!VLn(H>im(tg zgibgEC~4RGJZDY6IAsuyO*W|bb+hB>C>|pZwHkzW{yF*2($3Dx&#iiD9Z*}OYs2q( ztB|;N6I9*Gg~qQt6DdBne9}WuBi>2GA0Ho!9Y+woXJTecVbm?$g!BuLee`kqjYV^9jdGkiLhKz#NanU#jPi)lqmQigd;FP z|LUY9{K03HK`7thwK&B#x1iD|XtA8o42e+2Oj>)h!5W2kLI_Wzi!9MEd)!Y2B03lz zS2%z#jfF&kjg_mgvN5Go_Jk(~0}TFD8&EofL{K9_ciFeX03up->0`g0G~m&v{R zmp7N`DWbS!n38^3)l3`U7zjeSD6p9OnHs1qF`oQgBxhutHf5(^X*WQFx9i0FpF9b4 zadizy>@O-=E57_EQ=YIt984GC;o(W$aBGN~RGDa9YiqVD$ILc3tJOI$ z8OkpN;Xn>aH6=(kH96_+HmOiP4ItqQxFF05lbf-|I$6{IIGq zC1vAuf35A);q=Zwpwulu)&)Mf zwY3F`iE!>8tM`1tWwLO5wp}{Q+(ojD>iIzS1U(#yH6meK)Ne(F6opWJ7K}ps3X)uW zwM~3JO7Z&GR-@`ZJ@fiqQ=3NjuzerjU5gl0ywO#UwB#8djd6P0pONSZ>vqH(91xcoB^>a(TLLu9=z$){hu2qe!2aZ>y z9xSbs7r_{;lTAvG5r#x6aWL3QoFaCn$j0nBqGU-ny%gV#6I2~pLeEm;uewXke1lh1 zw=~hH@jD*v3scZ#>zdP(J&TuO8ul~ZF_2d`%GLt{7OwveJa{xPAphS7cgEPSO}kxe ze*=S^{eE4AzaX)!nz}p5X~IYEP0Z{uZEOd zGyg-1BkP2Q<&CX#P!f;aIq<(!=emIq3-#nqeW!anQY#&3d?d41U1aYTWDA4 z-tA4#&24Zp93il$Is-HZRjc^=!HDRPr&Q~Q4_f?bF8718)w#ky53JQZdGaJ8QIHIX zO71hsfL7#2BWGvl)vf8nG}EAUaJmC812_?y+_w#CrfYf2Ma2%&2v$i`1^Z~tYw_#5SWem#%JG@Sw}-TYK;QwY1-U%q}UdwR)3_xrzFh})zZXau1u zH3X+>IY>iBN{`#CWrAd)gI4OOAr+CslKXN_ zj>YDG=!AQy>6%R0cppIVd-UK@kwzv+AYJlYpw~}C6j6G!9Viih5hUv{2b!G}wSkYT zw5ub=E#KDbcn%F3ro5?MDI$ZvfBNPqUXWN-SwW!~CJ2lMnFB(dUsv6;_61|UL^I#L zi^PG=9h{`156retBvR3KZiuYY4+PtW7--o3lxF4^n$Z@v#LE4Bme-UBaQ z-UQquc*d$1)q3Y1NFyvg8Uxy9w}X})TvUMYfB}GnA!`pS zD`d8&i;IiH;&4xoQm>my@a3U%mD4OZaDZfSxAl1b;GotibAP|4xw$z~QQ?LbD8#I3 zY66KPU*YE~0>OUEJ+Fs`G`3$Jek!!fU|pu9X6Hy(XNU(Tp@Pp3JMjy=s9$dJLgb-f zuwjZ9f8N@>PBJ8s!&q{^kmhe(nZGR|JHJODWoy=WX`Y!T`tHa45B1;PGI3gTL^|(l zhY`@oGxlaoC=?2N(re|~3#;GRg+dsbPt*?#k*pjN#mcF_XI|Oca);m}?xl1aeJ`K< z*nN;^u=_NDMms{WWcFxV_qZr%X@j;KWdWfuXL~w!{Je!2Po6i5IY-lxJP%UKNGdC$ zSRsw@6_vbKj9n7=Dw`wKLmr=9f(6TMCRPndG`{T~-;#2?mr;L1{sOV(?SI+!@N-^- z8gZGY0V|_!JZH4d{hCjr!A9ofOio)twGJ7pV0K=XxKTx?tj(V*8N9RfVz;#f$dW&Q zd>I}G30Z8~`cGe=!ODcOFNHyHo2LZ*&ojttb4vwKAOW}Bjiue3?nV2*+G@M*q zfRGdPh6_*tr66A_?_7q6hDiJ}NN@yU5P#4;7Z=j3zk$a-)oE7l;uraKXYdJm&e~jC z;Q4mLze$pg-w0G^G&s+lwJw~uo}QipHpRcSn0;RGFEOh&b*%xP;@1Pu(@`8LU@`oT z=X~1tD|UOUwPLQf7p{H-7A-b*AUEg?xh5S`V0yXH=`dAUXK)v}a>u3Ru;V5(JSwS= zOJhJtohMaOkf(!2Yk(@o`z%mPm6!%1QIvf5ov>zQ(Y3R{NH9 z2Prb*5uASv{Z8n%O zFRCaz;0UtLph#$*3o2izH_qfqAF?U2CIt=_G+3d_#pi$|SZy-}d>Gws(jdh^My95{ zT>v*|U85Kcvj4_JHMVahGF0d~`1mwT+m8$lb%*s<%NH?Zn87CxJU34MXqJg8BgHf( zH+>@FfX%2S*z>ET4biPm7U&n9{^)|d*%aNexBQyd&B4dJ|8_gspo>pB{GuBbK)(Y8 zim$4cza*?%0K1Xu1LB=fX0Y1r?Jj>5&(J9|y{?$;q*a37WvV z1ST0NSy_Nqr!7k?NrUT4n_4pdN%^x|7C{!v3i(=}JlB(#^|Cc>Uc6|Yc4Ub9MR+w< z^7#Dn>bUK=6)Y9OrvleFth<#JXYPo{a3gj7CU#&2RAzqCn=v^XsmVwD!9lamYc;TA*A zkHex=9B4!Krn*9-rKsg`2(~c>%?Ox^mvr7Hr$OAjAel-TlS=@X9(!%jv4%B&3HS3Z4xs|6?3u9!HweKgxzneri$a`xtf zA@2%l!&<7!Y}b9y zy|F!T*BIAT_>g>dn~P4H?U%K%-Iyq}13|+4vz1h_<*L8D{kU1qck@vEKC=rTBYF9F z-}CbQ7SXSoIy}FDwg0y5*D8zmls_#M1BQi0qa8rDP6H^w$fyV#wAW1s&Zx}v^tOOg z_vX1_OJQItSlF$4&-<@j7TgSgVugmO)cg}oO)LXW+@xyCc3fi3zf>+R%#s_B{6Hg7 z|KY=j*49ff-Yk4_Th6_#2|oG*Mvnw0J0xMJV9-%bu7-xjx~oopt<$Up^qyt<{Kp#i`({ApwBo+~FsQ=8z!xXa9Zk5a8Cjs^Ny z(;kQ<%wb^BD=WpoiG&L8uO-8*XPp;cW_uqO=^C5p2YuQL5q7^kt3%`Z{iq|DLIzia z8x;ZWa09aT^FtiscF?d!tD=>jQE2$4Hsr+W{#wH0U^L3U*hp0 zB_##iPHCor6%+Pe&BnK$ee&9z+}N<^PCYxDYR?(62CJi9g|2Df;e>O;IjETZSF!@s z7^&zLU>At?^{tfyW-GgkfAp3?1yFQ!3aS@V0(sl~we7N6oxx@H!YQ~2K=>4p|2){9 zf1PH62Q=@UW|QI|lV&&Y*ypOd{e&*~;9+o=%tu)gyPG z32}YYdg~cpGXE@0aollY#2AhajiBTR3(@T4(9-!_ruceG2OX`)9be6M8LzJ1xMP&Mb2PHbUr_lDv9 zoHL(*0Q+;C`?zf_P$;R7fiKD&zO%C<6>nIfE6yYf4qfn?)fqVX`5g|zf;V zw74}R&9g@6!dYCiEo*tR1QJ!02VzX-HnU`EzuegDhTEw8Gq1lPt%UrGhg`#aumeq%iQ zX#u;qt1srka}Y(wAn7!`Tm-N;jK!0K{~+_xtRd z&89wVEDGqrQi1e;Z+%Q-NH{~Ci~RQOgFuPks{pp7lf}!-fq{XxT@W$g5_~=jei}f= zHcs9E*(A@ZOfJFy0Ki(g(|gyyJIUq9!sL+@1Sasa2>Pg9ZFH;PZKYfaod~tWj8{y= zoYYBd>4NoYrsC7J@M~hORd~;&d;grqRZK>WXM`Tn9P9W9eR_5P)9`&e-{l6TG;hrc z6&#iz9|?Q;th_Nf>T$9Z#aI7`i#B6hA8v~jeDmd2W59zY<3>Kuip1`kdrP(=#9Xt`$t!d+>z15q9E?5!f()|d z$3cN2DSfWP;2GH3dXKSpcHY@r?rrMSM^MiPop0ZJoMY7F+2=I}2x}d3aw@g8#l_Ep zalpE0P_ZhZdq1>ft9;{&s#1eImmwa z;~E%vxw@9y*oRsvGT@~l3Upa~0VC$;WJTc{)KhdT=FKO(vVP1$fUp=8?WL74ao>pi zJNTOcKKuJO=}TI%B)yF(drdAee$lKWixHPIQyOgi(77`EO$gyKWlzmJGKlLi{m1Vi z!fC3a>rZMH#L4Ud$&1#)Qqm5mRm*S7X6p?fffP ziCllT;qY(WW$YVzt-Qtsrl`5aXHq?V?cP$^LG8>@z<1GVQuW=|7LQmw71cHWxqNDK z!hXBRuL;7FN6s~|^ZW+r1N4T!rd-@gZzR3N%KX=erK(jK;J<%df?S)5%2AVbWwua`2P35lI{M>t32X zj9`CS?bUDiJiJ$%8gO_73NV7dr%EylZ+!RQ&)j-UNP|S^*PjWz)9&%*zVhFD*=K4h z`p9Xf(;JAaDlz!##qeJt`7njSOTj-cgBJEbQCCwZEmkKE789)*xmM-iA9|AE40)j)zeQ znzZWpEo7dKHru*ZCXxGMFL#HLj#)47tNJMl1s3(OxK!I0PE&PMuTau|RJiZ+E`#Wj z^px?_eH&o4VwWT9OrK7*E^KaVGbz*g!lIIAGn~~3uHf-;tV-$cziOX-t?!%i(t0!+ z9iDuU&i{MgKkL&_Yd5k^BXqo1VMXA1&6oYRMr+Blnbfj(x zn0{?J+8CI9VCb_G&}aihO%I-^+B{iJ?_NKie^K>R5@^%AK7o zlgD&}*3#J@pB_j1|F_5lspdPLZ+$5;=5c6X;q3ftCUpQ5$#z_bw`t3jS&x-vml)bi zO;N@hBd9m-O@z130q(5UphB}8a25-G=bPZSboJqpVGb(i=IOKl!jBigiE-K5iYc=d z%GtyE(jJej3X2B^l7?!G@W1I7d3@iexVU&WuxjmQv@L^< zD}5}>ZKWIQ9z4ugwo=as<-~Bd(tDP#k};}wp#+=}(W%&`zWkvdUdBWBl%iy{LMdZB zcl~xt>yUQgVv9 z5s2v(>6B8xjknYl6%K9uRDqSGmc{6x3pSM%)L+KyP#i5W-55EhEW%3RQ7J~4fBg#c z7WEEEG3<76cM}N81=u-JaqUq9sHYX|H+os4TWmK|92-sjY8L=Fm)^}zg=GrKZE{~% z+@ov1+X{wL$13;eNo8wt{4{Yf%Xq)AyU^ybc~dT{&0{UzE(72Mx55^O#23c(1Um%* z!w8bh%O8TRK#FZ=Yb)Z41zMn(FDTcbS24A5*zA~-M~s2IQ5;Jo!OSo(nY{I!=D>?@JG8E z3zW2*HgDuk7Q;=d02k@ec(Ji@mOXn6G4pZB)c%*n6lHD%((XZ;&(?GeAg91hldUP| ze}Kpw+IU>u(qdGmGn^~8u9aIu)o52d~@3y{GpBhWJB8_ScQSW_-^m%+`{D?umC*y0}h<2;LEA3{q^U+ zmJ{UeYrOFCnlL*Gf33m4xKTIo;Ji0)fDcQ8%I65-=z?$FYWox1_7@(;mO2Wum|Mb2 z7Oj^>zV@+6%PLjAz9|Ny;9>&@Hdluhl`FlZb|33TBj#OH=Hn%=1F^R~HT!aDpSWcn zevz+HRwG1nM`G#Lrh){9tt>jo1C{e3s^Blh=nv|53a@GXCg4iqkFvdJMGBRmD(bydc4tj$(d$Fch zjusXcK|yU=8E0d1mt&eGg?95U3vF|ohX4x(viXMDe*^`bmV~hNpOp)IG(UeT;xNB} z#W29h+#CR7@7lQYN5a-dau*gBTy*oxmLgJI_)jMZ=0Ijw*Pi%T=r~;G)2zNmk;=i9l zCY2ll;4M`g=C#UZ-?Gze%y5q3$Z8CjKNTUiWi&C+uZ@)G{6!K(i<$Ju)-S(U4AmT| zZxCc-JULU(g^Gtno=#=LWjMq$*ia+dQ2kBxfr6=TenxakZ$iMzA({TXXPqkE#&>q7 z#KHlKl(p2tK7Pihj^<=c%!fob-2(cvG}Et6a^hv?W9h{GS`zu=*KK+lXw15B@5f$M z<5!x)TZ81{3M{#)NM-*E7d#-w6+4}pd!7%srWkCWJ(?ovwi}ySQOuTnQLMd-!=&e4 z{I)@t1;`3SG`TIOYL@JS+w)nu*aA5pPagK@e13BBs88*3V`bnKYH5*{K8PS1vSGeNq=y^JAX^Ac~r z+;(wsbo7*}K*SUmThr35<;n&cfMj1_6w@!$S>2d;-XH=Av99skTT;ehhsog zm(2yD&LQiX@3sKJqK@qbkF;%VQHbOHa_#0aQ32ZhLu>Ijitels>U>Cs<&%sp$@fcT zRwBcmJ&(WX45yH)$(i%sARBo$_p3ba7tglkC|+E!?@^vgW{cT6!>-s)VnX#ziwh7@_h+!9$(ab+*qpWPEm@hGAWcY9<*cnEY314F6&Uq%@M&k4Y3Q7l(3D(xuJ9F(GhZ_^%3k7*p zAZl-M(ahdH1_O8jGROp)FIRvzme82D(PwwQ!mt6H4iAxYfN*qQiD7yUgv)cz$Kcrj z%in~39j453EFB0L=5_}x7@7B+5y9926nJ_`_^5DtqD7oOP7L`t~=DE&Jr7_oo zxY7CfoHx;ul9KsY05?g*53GCy@m`(=H~wAlI}6PMI3r{Ik+s-EgTgaQkyLbQvGvWM zKfOu6^C1O@8}ghIX)zQQqeN})+3Hu)^OwC^L9I-+6!9{YaTbWTMsEsD_sY|jB)Vi) zh`ENn&l0KV;-U?{d5=Dm+ttRZA=t-R*!NjkaQH>+YXwuMD5#YF6F0_(Ew0~5Tn^RVC_GeIphUO)K2 zw(B3^M*I5IxKls!ZUV&SG@yE6xBYylIa^bJOleB3?egfsv?DDXe0WqYrypY#@6r2 z!ZVuSZJwEFf_*e`aUBLA*M8&kn&}I`9M#o(_Xx(R6xKEPxB@%xwzo~O&6@QZ9W5=) z4dghwxa^*seA3~*%S>ML0OEV_Cn@S?)&n1K8v)E1Xc*e|&I|!L5AKj0xlcit8z9Aa za+OMvlRa-5j|Q5^2!SgcKJE4W-u_9mM^>IHKs`+PIyRne`$BoZ?>oa@WL32_k>=;d zB?(z8aOrnOnSP~akcWqa+OkLx-3b<&OUS6@aiU!ITdPiCY}@M^8r}O<_5dp2=$`WI zzIT4gmB&wbpe`>V3We_bK&J?-ERqF1^7s)j9YrLKn{zXW@7V|4!nf=hzWx~>{JGui zLyE)rSsyKa`g?VgS*Rzuk3;Nzy8D9H94I3hvIbGUcC+=nKptXMqkroQpb}I1psea+ zAgQ6)Zm#vkV|3$(5Aun`G{DtAH#e8MBF5hh9Bg#jRa0OP1D7YTnUoKKT`@hqY^b8Q ziSYT`1F%*soX_tDfPg8GaAQ!WoZsSqIJ2?AkT4$os+4#m?o68V^!&lOTO)b?p{;iwRr0bL~v*0FPqF z&JhS31yHdT0wP~C4S0yt0NEI}PC;Nc248$F8fy-JV}+|ZT~6VKuAhe+RVJ!)I?fKR zJPCraP*gf;Bg6IF`{=Er%nA0MJql<$$qPb>N7RaZ{?0)ihf+4>-I8fIf?$&+Krn^G zHtE3A?0DtJ%$5PVe}7)=A+JxD*x5JlzW$VO(mwqG9t#oNTGf!U3M@dx6d*=X#>*rV zMRtAPzxEaFN_k}o4?J0T>?e20zdRs8_;^A~EKm)No_m8We?DTrAaakdwYH2;6a5b;nb^|No;)DT+{tj1odN9eX5Y@4aPY@4d=OvdZ2eWM^gzA;iHkLRMCs>^;x# z<@@u`eLT8#FMZDE{eHcka|^<;GJ!IrPnXV7DdSTr+{@P!+HrYz5!m3q{`&}C;Oc5S zicq_{v9hV>wFRy$KAxVQ&d%HYLK{hW@crqCIAzM_?iQbcqNGmW(9m$S@BmoI2f;89 z$UKC&i?OjW^okjEFjutgzXj?;FRQGU2@=Q(2Tbc-H6P?51n z$=f)w=3N$}N$XJA=Ubv1M!({$Ag1u*w-Z3U5=K4Q2=?T!<*i|-RxpUjPn#W&s6va0JcSVAj^U}p`0R!bm?P!PG^4>a;e2TkZcu3G zR+T;ItE_{_)LhjS$K~$PQOyE%$?tG=orzt3MI?IoQJXEHcd@BSQ8HD_j5`?tv?(Vjnk72=|7GiR^2AC{d4|!Iw89MfZgzQQX z&a!Z7Uuxdk$sBm#h&8g5#Xp8f;@ng~7izM?fq&lDax_nS_V)m8rJcGzdJc^RS~Z;q zAaWS5N$*`u>4Ok!Em@d24a=A=0Ihn(6HqS#Yfuc_fWT2jkQ?GZPDH=}0s+&ixpsR^;vW9oF0o zHGD{+Rov+TqEy091YxG&`2ylLOHWl##t>CB*B`>&nofy9Y`0N1p-*SJp;JNYwUNi;u0tTP&u!SLjaog}oH@5k~^lsV~k zacovx#JuNh2vc>>#6-9`WxTAp^~G29h_#)q@PHr+1ROwz%twosvtdTGjQT9L*@65WGnuOh;q8H ziM3KycH7eVGMFQoSzyvZmGc8>OZ4kqhrL?Dfw$Y59~rCh{Rz<(!0h zc!I)&^sXHFo=?u4yXQK`q4FqSxbUd$S@0w(HPi^;-*5kD^(Q4^GcZY2daf?Xrjf$G zSCEoJj9ajCYl325K-7}@^86ajO$ik4J2f`$mPbYEvQ}D85D!V3W=)X2AHO>!i6hNL zQKEOVo>QtyhYc}cKYzMYqZImIg6+29^M`w3G5x!YTQ8lZV&xxUSRKZZ%A)3xv03{+ zJTuPRDk^vzEub+VG=V8TkZWlo>P6}umTxLn{@~v5B*T?JaLYbm3mBN3xeK9j>qDpi zcwZ;NAq+dp@$}580>H2EQ~EZmvxFm84^E1TiXet8QG@q}i@!gp{8M2GhB&4-Z{E~j zr_1#{+%)0Dcjfq`&eFGE4@Qd_2x@_kMr}onZqW!lh0{~tCd15>l)?<(V$+IRvNiIo?;f`AP=u-9doefKPAzGv=h%T6C0JH{}Pd3y_H zqJZ5|R8sQs$F>s%@QH}rl7>#8V>IXP9&Zz|sgsjRS2uikLAha-r3@+k3FcPp$;c?5 zCYl#);n^i6`QC5s#ik7OMDBk|N!9!H+gFf>hqR09-pKa1`lnuVrybylfbUIiCwhdlk99g~Ih|O1hPKAc4ur!beKlPXx#F@9<8VDMv#FrQ z_TMPF!8dvXlvMJOUG7DPxHn4mUnQC&8;5BKTpbg82s0xq10}Nxm2A+PEJmO zn3{%W-@D1Z0kn9eZ-Xs5@Lqvyxuh648?bK-$zW9TttNPQT$-lg5n(1q5_Q))t6j#f zzJ@U`H#el^9MhthO9J#e{ZQDvw$=@}E#=W5#g`yvAbE%)xGQuM_q~ZW)5yg1bZc9i zYJR|P+nVO1J_o}_Pp;U05t#4(es}|t}gGO#FoF8ZK6?IElO_Lf%?V+-yHAi9a@coL|+sJaEh#5?k3FiLnwGhTx z+NI=IKW=d{w2$6pE=RB1%!09y4~mZJm;L zkzM}s9x-a?_^g?I;HcRsqeqWF^3N-7`8@re7)mpDd%=iP`?7QnGY*61FE!2Clty3< zDqGxnO0S+Xc&+_cMH*e0b+vTMM$`TT)ru>B)YEcp#r(>&Ue1X8d`mFc8P*W&U)nQpdBDx9sihUL^)A!RXHBzV z?nIPc>g#-NT*|?81zYE@{{9;;M9j7-#j-!Wnwlvro9-I?fVvqe zIywe@i|=x`c~wca4hLiV-bB-Iu(2&T*EKaY!6JbO%f8TZ0FzUS$$-%W=7uuJ=z%E# zpZ+4T<0AI15mwBg*G4SyOUuh3awr>{zn7B2zOXZH%4G!qII?YXW5x-WJrFp57o|r3 z_xox!IycU)iJoXE@UDV6bLrla6E!RQZj7KXZIq&>pCRw< zT=img26G1UV$RE9IHq{i1JU=zmT<`ZsfkD>&kP7&&b<`1+wO5xR~Fbi>Ou)rmp6Hy z=7~j$q5l5&Ql0gs=z}gNS1;Xw2~NUxPP1Uas94!t>Mkw{ull{^o)$r(NddZkkT9;LI;N! zp8}{YAH>Uoreg97_-j^I3*#!LFpRO?7uF&rMR-UXQ!Qc?gphxI4xKKNS>g}RfNL)` z^py$XbSoC(<9m?%(~KL!51f8N_PP7X<;wUrSQk*JN7+_orH_n|!q{=kmKwVQsL)E2 z6Oqe((LyBV8!+oOIywsJIK%1!gcmwISSJ&?+(C5$Mqv-ZbjT$v$L#nSX(_nIc7G|h zs1(r1*lKcwm)OK4%g~{?q-0g0_23G~4>FcD3ZLm>OalTSQS|n^r(o(m+L6s2a>{k~^-Yt_1?40+ zA0NnQLc%xL_bjU|5W@hzei7+j8-N8#x*t%rpW^O>+|Px}DOuja$48Hqk{@$k zjnN_u(u83p(?B5~oiPUDjv#v_`n`7T=bZR_%qF;3pBzRm?T@@kOR9XWg6G`m`~!cO z&u~{aTmQ|Y@5UcvT>86m2d@fI>us#y6_K%gB&&Y;p;P%!rQm=IA5Fo+yLSQGX_7dJ z;e;j_RhHq8t`-AxNur-E1?CP1tmY0`m?mE#?ru~}(K)TM6Iu;DM4#(C zyoLLkqi*A*bnWE_f^f4&g}=APS6nkhr-mLRxeB_#YV9%O=i#zbZf=ka8JVyThY*n0 zeEoVIypTIQJUo!)-Q^ZL3aa#q@qh5qSM71Tz$j%4`lugOczJpv8#u9mfrC*`Hvh~x zMxGI{EzzTeSpD*t?oSG6VV9+!0A-eX$tx+*QvbXE2h_ZkK&V}RHZ6Y;U%o18gXYB}-1f*W*E zg8X2A|3b|Tm=L7Zr!2uL4>~;9;c~0SFjkr1B@oRN6Bc$&iy$ty=QOJv@7d~x`UTZh zaEM6a&m7>)9v*7nfBm7`ET`UO>raPaS>kgB(5dr@iP3g#HEcBNF_UR(Y66Yq*8HWu ze!{xi5ENm0nGOK|i+OFo1PaX!Mjo{F$i=Up9BjxLtFR3^K6GR2t+1ndNE(E#NZTBy zxlT%_O06=X=qV7EqfC`=Kl8e6i@4K4_p8xX)HQs1*&pw3UGnQxKBc}n$&-Ifb*+>u zF;0#_b!UM1VVtVucavfSuie8K<#;YHiM-q%*_04sV6PFlR^}Lr9cx#FiHOu+E z9k#kWQrX=DX1U`l1X>(9-6Z2x zi1}>nxL`O&iVBg{pTh1BPKy<4*sS&CH|Q6RkL%3PQZVW=rltM6OJncWa4SGr>48KN z1<$T0FTQVJ_`~iK8UnB1EB;4=Z>~x!-Z+pw+Wlv{^{*~odU+w>aON~x^PT{HQ0WZc zU1!JBIGye2^M)4eGwc+E2vA))X~>lnF!@9?H8s%~Fo3+HZsUVp#`a=YRz^EViV_31 z&fokOO<>h>P#H~icfBh#nMLRAskpchzS8BU{AbS?pFg?!r1%D{a2)R`y13KvNa3m0fJt`|#@Ni__=UjU*wtM@MI)f)gs1x;l$Tnu=A^fCH zFB?)$r%%FOpe(PF=h`V7Ct2b_cK=53rZ`?i@a5}l{G89DB@>wo8Qur7SkWcnLLBUS zI@y9Qs@HN0Bq8@mK1=d{iB;fL#2t+vlq$;l$B4X}QY!bAed>OCM@3r0J zxb0{KTdq_9BF$y$8!~>r=c>sowS>zpS@&}cUmAQP$Z)Ym&5P+I#4D07k&}NSgVWlC z8sHBojleMQ6ShNpLLW;>FZ~7flaJla9L4N`h`$S3D*$QdzL%kMA6Qb50a{!4IHj4E zgATKDyjXP=Fb1Gkd;o^`he_)K9D+d}{=_zi|M`xC|4wRR#l>0nI9}EO1QkQMavmE~ z85t=MX8~t?fqI^|GrZ@`r`zvr`PVp8dKatogV_z(IXFCgL=eMLUsZ8FN=l)5W4k{o z(#<1*{uB1Gzl(so16(0K*z+?ODxCUJX#zW&t(Pb z2KHzKqjhr&1W1k!4$I&+NVc{1GI`Gpr(|=$g*n9*)M}WLuR{gCm81GPI^&Z<7jo5* zUUL791OJV`#@N4q|AO8D_Z2uFL3w?RK9G+1nyHN7f1tB*<&H<)5}Mi+L*GB!e=&47 zk~aBwpPNg-Pb@dPHQ_Uj%(B1o&8MuYg>T(_Slm+y|2A)jyq^twKD;t9#85|1H9y!X z{n@7APDYN^TQ`9PR*sjy(PHtA(ROZWxM|!1M;gz+s__?h?wo`=xw`W581O5&I5m)3mK;BhzoG{^kQp1DX_be*MAx>CCq^YH)EKZh@6dVBX z<zN49=XZ{?{)|XWP1MFQN*)nd4rf zN$?-o^dZ(WdetX2HC0u%Q}8$tRB)B!q5*Rx#4I=sL7d@&YrsbgRTwbzJLvww8c}nw zpE*VFS0$)W1w;e}FINtA)6aI}2^+rQ@J>|Q*h`Jj%nZNNd3edV4J}`a-Qpb{!0$@@ zjIs4rEk#?&bb#%yYN`9vO~a2o)Y!Kl&_bi{XfGRbigDN5%N9sHG?C_BxxGD(yaDQp zdqA7Mi&HP^(=zg_)@O&T1_;rPkH5861d{-~nVA9eR5Qu|hWSDlt32aQ4uGd;u*$7f zkJV4DgEqc?3Y0d$n#$%39$x4-U{-f;c$5omsQGcL|JVuJeMK=|hLt)*m&(v~N3FA; zAhZ+%lZgohI~!Xg2)aU4%XLevbo(oNi!R=-{IPQg3;_sv4H<<>)8Ua93cq)=17Vy@ z81q{7{2N9u{hPugHZ}&5x1ivunuxUJ_w)1wD`5yRJk=xLEY$+I-Sbr`p#-5ms1i1S}*quRLyY-_DcoER7ZOHD5)ZSG~~xS1SXW2tGO+KR*L4JOD=! zbWwn*97#O{{RQ^840{Z?3P-#w(C@V8V6DOcje8N;g31u-rGT*wbS^W;qhCQ};YMGs z&o1{6dk8V6K-fW^&rLA(j$S?K;P4;ZIDoQ8UNJEx=D7Uu?oX+`i){l&1_l{MNqvjl zA(HHgsDyl3G>;kx>SATQ3?U~KtmE))FvS1-YpTSae)Kmd`UcTH;uV(#-}}hIL!cU|DgT8v=#@+Sln*-2&bb>wRP*IR_lshEp>_jRDwZa~;@QLQR+-EG=7d z7}V*Pr2p(=F9t2T@8RdEb%3+H92`*9V5xkJ?<(XmfO%}Q2eqR!);d_cu6J_l!zToG?sL-@{JiwMFG5tU zlWs??wbt07O)vbhT^hwQ%#Rv5iT2~EZpkCX1-RQj1m*c zbC|6k>mi2RNB{_{OEMk2e)e&49l!f(E3`f+<`J7szSN420yUMJMY$RG(Ly)rq&s%4 zQPIN=i!06)O;^lOg1jHDymET!!8wk$8T6lDAK_U;(LEa`%*+>bKV2`8}Sx zs$r=hd+Nj}%_Z%-|Bh~Iil-?*OB3#t^By8qyr(sXhw@q#m&$!_IYFtYzw9#i*?y(X zk&0L~Q1v-}R4AV-k9d?7q+C(Aw0`QP2_j*PbgadQ%`l{$ii(aLUXf(J`*!|sL%V?1 zHBeEP$B^Dz-}4SOW{H=S^_>x_^yg3P1z6S7$7hV!u7C0Td=B8trDbvtkArOHl+4V` z)YK>6!S=w>(zxSl!}0z(>{fL6`1J7U2C+g|ZB>;NzSrSq)@Te1*|nfn@z^`L=oQnt zVf(Mi5&#e{{%SI0O6>#Ndp??*aDFQ`68aU>M80OAmg_ixSc97bRMT<2=F7{L(S_$j zL#oi1%aYhm_mW>%IHAVmPTX@EP=$-c=QPmm9lxFb6%8(imrRP z_D}M6WaVR4sR?|QoDLMy+{veZM(uhw|Ez|aP{=;)t+2>jCJd6Hr|#lYu(Y?xGc5Bs zwYc|QTSrHt0KVnJh3*96Y-9^DK?q%e31mxkbaaGtm=^$$;Nd+L_TQa6+F-*n+VAfz z8^lSmEx)D@V@QL?=btG)Dk;AUtX?RtIWp@me^lqJml3y>Kc~qXg!o^VK+1f!{SJ?3 zrfGsqFWHzBDds3n3pq4o#XllG`n}JXCJKCY6$*N41CU4lv07p)$9q932AGcd;p(V> zfvqj1WQ?lw)Vvf_FNI2IjcP70FUP4)`Ex{pdGs4Z>8>K$#6kcqeUhBNq|yfyOS7W!@Hco36|JrVHcaF{x%pg%W)G; zDD)WBJ`j0l+5Ipr|ER*TB445WL&UFsWX&BW@@21T-AFDMClfEuO-KWZQ+1xk>`dmd1it^eBqWA_;-IS7w`m$``^Lt-OWez*=1j~cE%-B?NZp$ZXGDMb03mRa zzCPa-t(LrN036mB&cc9oM#`~|uf?v5JJd*>58=Q)z(%B;Cd6U=oyT$Ap4z;BdGxTSlhp9#vbZ4^&7T#) z#NfNRB$&_|vRN-83FP~=o3zx_!6slv{5aTMpe{>*w0UqfcNk188CBXwMgf}_szoB# zlBe^jMgX4xz71a>LL7-YpD8}m`of@-LKAj(aiU$Ms`hwEAwBo0R&T*=v)XY{a9Zl2Vr`%2?0Uql~1iT1#RI3c+Zg**|bx30RDn}&ymR_^DX#$>JD zfzrjBzQ_Y<>FHDr^!0%fHdg!KH*b8tADvy)8^iIN0`n5nvq7#-5;vWq5zsR}qxB2s z>b#~xR-#()$!g*y|5cS6H*PR!oSSjKA08WWBY&Dz$U^^l+!_p;KGO>z)e#eibV|0~ z#Sp_g9i+IpkMeWo_Arktg+JUPGtjyznUYrutYAJG;*DBbjEY7sM4A~Hg!NM)kkyYF zEqS9vGg&kKau_~KGhWN-lOFaZWo1GMXXBPcu-rvr#^#Z1y>P`3d>|L_+uc+%7)y^7Z3h|(S+kKh*NZ(DEvcwgi?M+UaD8d zrI&rTp@B7XZ0GREnC6zHn`kn#0ef$+-RHDF7H3pGY4?41SLff>)rwJa?O#}KeI3&q z=6wWlpB#QOGrxaR-{^8yY@9T>801}0JJY^93UqoiTj%P5nmz&f32~9i6xb_svIw^RV*6$leX;T@BP=#4wNOB z(Blgvn2HLsrl>}v-};?bVXS;gHiS#DSNNLWtKd>hn#;@qRo?_?^kbQ#;e4kE1#wSj zwEymWHgl26#QOj`=bNG&Mi`P!5-6RA?7-&KW2(}zkUt=QMPql)zo2>Jn(lKiJ zgH#NdWs)Ibdi#?EFObwB#eDo%=Cs$|h*tCN1N!f9gfuZ!B&Ak3fB9NhT^*n2RXN^R zQLkAtdNCsrjdSqx^ZRn(w{zlOy7an{BUX(!!&`;S1izw6{F0}C()TQ4QzSLX(^$ll zpCP3f{CT$Ck0z+wtij6qG`Bo8NjBcS=S}?j?!jfQcY_(B={rPAa#W@_XfLCv94ggB z2>RF&=1ZcOz)9f~GW>6B#3JpZ;$GBVnAZql9wpDYd&_6Vo|->S+V=F3Sw|dyt(b)m zItgZm3EjU7JXRom^88%|b1#F;A7m%7DWZZcR*fubo>Q7BR)*|pL_EMU6Eib#M!Zfl zgKDVe4Wc*g(mCZ-RSZ|d&R4|F@@wo`&h{sP$Fx-cVRZmy3F+zSfWU(94fGrSo2^%X z4^ZXlaAH&Ao2>a-L-WHOKZjaEhq#mY4wPWEUb@bV+|%Y&zP!p>8&W%EnzI!25B^|Dln)Qh))1^ z1A8|3@{f8Ut=}$Jup{Vy*7HFlCWJ&jN4uJvKNSoIr!Op&V9Y}v8%#YJS%f;U=-wA>@vjs#BcG#(xw=+=HsG|uoN&K{v zPPX1ZNKMXB&T_UFR!)oEv3-jpw>0?R)P!_Y8qrf%CzDpr2;Gv~xUUyXzu29CKm%ZV zP6{Bun)XKZGL~h=zhpDheUtg{yCuLV?;VIKbP`~eKLDb0eoOlvCm8UEK|u)82SbO~ zE~Pf1Ljw&4Pc_`6HeVtOewAtz$Q*??odw0ll1k!BBJl%5riZ#$-UmmshO5Sn%W*V` zW0{k8!K|yJXm-VU+Y;WExSpnGeiufDiPN*80`q>l!fqYtCY(H1H2_< z`YLe@GHtxCQ$ioL%VkG~giNL{F?np9fn*;Xrd0sX7nfcI!AED08MW``v`*FbWLLa0Vx=DT4sq{Qe8Nw=8TFj-Nr~o@un0!OjO7R}+z#Jj z&;8k<NofEM1FPowDM1Id1|Y1uJ28UWkY(4&2(>&rNN)Z zyT{E+#F7XWir%Z<3TWFYZ9P4;TWmR_!YC1xjfc>MRH`+0CfI&=mIiNoqvyhIC?ue_ayy-nn+mEX8aDO9z!m~?Cfl{meC0B+VPOL=0aDH ze~}Ic;&a>)GvS+QhKJkOE{J~-Jzkg}^t^O+-MwL~%)Sx)GGEz@G+a>x*_;|zbqx)t zyelR@SKIYcMoV`wg{k)7t7*WI@%dWq;ExI^4YCw+uLK4eX?$lKe7J)fK73LaPqKWV z8PD6d94WL5 zHR~EnqdzE$xe%L}{9W7c_^p-3nFLYx=bet@CWgNP{NEfYB_=B1g%B5p7qUuOll0Vq zKEb0VlkL%?m3q>r{l;bSvA7G9MgN)} zKE2%7$YqtcrYY{!UDpOpJMTo+|5Jr;8X|ZCPPaH`bcXs_}?iBSg zvlD*O-N7Z)hJRXO?o{FZ;6}DH+hrLabEi5ra9B=Dx(IJuyPXW?plpkD%d~{jGwgJT z+W>J?L%+KhtMVl4tEAX$8l#N5)Pw7CgdwHYeVPV=rQ5vmriP(bx1ZiL(*B*O`NMYZ zqrmz~@8t4gI5~H^j^d%dB@eA#zXOR0K}NNVm-pd{m!baLjwa{ivoM011J^~dv)cUS z1KVt$W=w}AJGGO0=}zClk>7y+XOV{OUPIUYizxk078Ugfx?olDBK2O-HZkv#->yfm z@DjFb7V+(g1P{ES^=DRy<6^wUbG`l7uU|1u=X-l@24gdjoizATy+o6%6L=3!3g7*4 z%QJr?G_L@qaJu4-aH3OOuv5<;g`B~#`tpE>(+_*UKQ6a{9gE z3Q*fk$@t z&wXJI_ij4xq;hyZ==F5)d{FH8nU`TkS{i5P>oyUGik21|9nJiy_s7%+H*UBYIPl(4 zZS~+zhQTT|n%2&JV(QTKm*HQ5ghy{J)yE2uH_+;0ZxIrCi}<+cz)OfWUP?u#C)B(t z>urDTNPeEQ!@dzass!Dw&z|>)6ZJRxU}Y7uTcibbkzCWMW-;@Xyw~;ZO?%cn z5(1B6yQMCDG`F%02$-pWCT$OQcLwUyo_N-6cZ#PfDr6t#^w3qDgzvJojr0ft#aGYc zAKZvWkw*bshR+ut*mK3MZHD_U4>rJ4)uz;8;=*~ zs-cohtI*k%pD*x;^_Gx~8cr6bUOp`6yr5iPNK}-Cpp&A}3sbLGqFyutq}|9!rc{G{ceux`q3k`Dl zQkqx3my7rMvDf;}&ZdP`@G|Ru_F3yr_UzA)Z1F+w`1_8Z0h-^_I+i`nBkcTZGiJJS0LB{x5c^?fkqrfl-Y zq>xG0!>l-AdeReL6g0R=zJVS7!iErF{4!;?lDE`xetaxV_B4LUnd~VZZ+n5!(6@nI z-pN93cM8u&w$kDf9+Ce-uPXRqJ=HWu)*5^wT>3_{qAthgSPhg_SCJb{e&k(spA0uq zdf~osi|F9ln=A7J17iC(XmaP<)6XzWe#!%j`}rKxjipqIIn(yTX<~=i^8o%*{g$&f zo90Yfn(NXghECPlo77!+;tW>bC~~UUteTGwv#eIu+;@VArr*t*fjVL5nG}vg)Al0p zq6cNz`59l3-RSA-n?-qUZdrz!`0h}(9D$CWE$XkN1CVch72zv`( z4TUN2-5#SVxOAq^s^GRgCl@E}ZJbVlW;K`We5YOm=! zpW_?eewx5Qk;T~KH#2r!ZgBW5Idgbnazf^J!A`<9RRkk4D2&;f-cyB4!*e|`SU*5K z{4*znbe7{?N+Y@^f&cnrni#1|dLjjC!k9b4yx5G;ID+dQPc$y-)##8mLsiIqW&|#N z`naX)&%H;eSP94J)tHTVj_%l33;H3ezw>*rCK2DpB8_tRa<#E(CsD*sV3-|o*H zJ{<~PKN`es{&fE_c2)nc?>?OZy22QN_=bR@VV2~JV!u(%1RMkej#xRF*kIsI1;lsM zN9?$Ww~s!U@Y|MWVJd{%t@V9v-p&s=&UYuiZC+rNLfhe$Ay<qfdtSDx0l$ z%Tt&#@{wA;7e56dE~_0H>^7G!!T+uJaFL1jrh(t09LE)j&G<6k#l^(3ho6jv)%Zwq z@lqJ+@vBU11#fkz$UBQ(37=jQ zf8!+N%PTCD%{XO_Fh}WX);zEh3|2P%LiY4M9_P%#BWb(8k_@=UmpC;v{s=GQ6EH4( zXCV)CF^r3wcd5_Gmp)8elVnEvkfbd?yJEC%$;iR-^-(=4qdl98gwztK!%r3RSYdZ!e#2j&qs6+~EtRRyVU zv>)24lU&OqeyX9NK#yzEJN!Cpvc~RZPg`5tM31!!SzP=%L$#gF-$;427N9RZns?GQ zyZX7yn4g5H)CTZw=liQNC*W28q`Jbs^su)skl^{-(vj3i_YwZ9z6VSJduEtQ4!;$- zghguFNd3p}+J@X#Cr?tF3>`c?#yDN2I{;D}Ua5j`qGoU%LyvcIMa`t0Y=OB(sX@d3 zAZjB@ItLjA16f5yg|oL{OmSocS3#L}+2ad8xs1V6{Bzp_C!IL0(Xrp&j~=2cp1(oM zRGXZ)(w^CXB9~DyJQ!cOM6WP2C zGZJNru5xpo9e%pOj`^)5E*$zeGEsbb9bfJ*_i)Du+iMg)#UME-Sg&>#O5#d}A->_Y zUDLdEBUqI?qkrxHVEaC84XDx+tC*g@K}o6EcLT1IWx3PyFoEJXPt)Bv6h;n9;;U{*=A_`934I{ZY1F=%prm zrBTHGB6nHoHhE%r_7~eHW5As@U!nH}8O?U%>1+^D^KLGOMv;9&E(Ks@L%gu>Lw4|4 z(}thY$dBputILfe)<34Ts&zqoBP2}8&h84aRVL=ECu*-%Vxn%fhkc;G7s2e!n?4Z* zuG?qrXtrJowO9t5YQexK_>QyBBBk|IpUZp(;2iZXSTVd@OaVRo#X3CulNC$BF#-{u z{UhBxhlD=P(ww>ry`qj^K49%E@$Me#oxJsJ<^C}HmkE!EZ^y+)404+{nn^0lzD;7) z3)9(CF12DGiz2?1R%DF2y>xp0Zbs#0)V*dMJtF$;-x`88$v^BY+R5eTD+EqUCe3sF z3Y9!X?_f`RHf&Sp3N3a1d8uG1+fzF}Z>mGf#|wSH_2k=2y+y3C?%RVQh*O zLf2G7M`xv{=E`guS6&+F0$W+Jaq!CwtMDG&hU z%lWk8=q-g~XxOstNhbDt+P>5P+k{=EE#{94IKm#&ZeahAauAf4s9fr7#SQYEUyaAM zLqg1`oq`6 z1~1KhZ-P*hmnrWVy60NC1ex+-&MCFi!lwt+CBAL;qU!Xl+YF!-JE5p~eERSYHuc@e zq(e0v=dZdWmuMJci^j-nl%!=+E@jY{cD~OHm2i2{UgM@Q=qL7hY(j;%$?t?eH(Ryf zZ#}z~UaqQ4<*KXT{+XlT-2jYG*`k*yjIZk>0$#iv;ej!gD{4=1obRQndJ7`69xw$% z;9cQ;Tw+8@p8DO?`)TBO&xfj*DLDG^^}cX_b+zEBtt?jhaf@{=hTwH8`oq;_a+&0e z)Z2agH&GR(zJ{k>CSN&f+6msz{;OQlV^hML>5shjB4XKZ(d8?F|368jU@uPyQYn}t zq`y*+pi%uLiKHVwlH6S98jf5;F!8a8=gv|eTjW=FkBPoeD?r5D1WUSkc^@SbFPxk^ zuA1vWq)$chEL{}3c-;0Fp_Y0O;=p~I&Oh7E$G zL^om4n6u6=D9*gQH!z6poe27sfqOkb0oaRe|YmwJO;1JR338a_u zKSm$e770?zyBd6DW9{XhIe1VDacR8W3h0FY(9IjEdE@p}70sE;xvtzFfb8iHW4S4r zpvbF0ig=k~qG|hPB)o#x&gb71#Qjd(1-#o4#=&9Ix1uuKnJtQLUqaw6i2um#rS}bg ziQ_VW_7ZwuiSt46PP)Uk|JH9~JBvr;`hOPB7aH9}JQ{pY2~^mV(}{)A&p+fJ)+r1f z{qjD~;JU9*x-6)$nfCsMFvV!nDWa`5G+)Q=-8`1Hc4{}7+7tSLIp1o`=zbW8YC z`@Ea%DH(&6zc@QiXpwdBMp#!D+IS~CzMC!lvI}10*iyP`n1wZPRbgrrd z#pgR#e0ngvgReR~`|s@RE0^Z#YH(DmNAjjB7>q&l*`1lDlX9^eI<2N0yy_)vLn>J1 z>ORG$+ZO8tRIF+13QgM2R%J^DKaU9|6n%PYr=g)SGd_*|hkk~^Kw95QeH1a2FF4D? zTR{^|AFTDbbnoO0=GSyZaE+@uEtcsgz+1v7!%ux9hxuB&_IVE!rdL{*ykGXxyd5u# zC!V%IIk@Xx6v7#(OpGgVoA^=~9!^a8(Npa3jpSM4^tbHkSFeaPaC2}BE_=5v-}(7d ztxtIMr}RQC%H{gp5mi9c1U&*p{PDdd@qdw1 zPw!c_9Fxbh)F%%6l1(hmyeRWQ9EbO*h_HbDh)ue?i(tB=?l5MAbNe7#T1Vv&5>x3r zkzqM(y@eWG?sq>Fj9BxbT`ib_93qnb1&p? z!Xr9;1HIzfeWpwiWV#~qzwm58l&2eA+<5PPGFLC+2V=UL@VL*$`Te}muZH%P(?39p z<8&>>MPe=7DkX(Df_V7laO0e(on0TTC6o-GjUKH3+qmnd>5AHcIJMXqm%M^bE37tt zUg2syD;2!D4Gkwo$3uyi&s0*DBpg9n@hoR>m}-W%R4E?~y(YNl)6?fYJv~q$XyD;g z@c0gq`qM=2w{DfX!Q-u?)1BUxi}gCffmOy%+GFvLH-1Mo9+F?lj}P5x>*c7jc|;>s z?cRIvM=^gJ6nxqxqZ%3wkgkX8BDC5&1mtF}s!42xt1X!5awv)HvMT@7>OqPCXd^isbc8NHG~pT#YIufiQ6F}@FV6mHG=Z-_I#1{!sI z;h^1b>$+J$d#@x=n>#JqX#__w7B*K>v%(+ihp5HLS&vIsW zVD1-aT%@wBcr)AB%F^!>O~T8NL`?j-u(0Tk$}RaH@HjcswQE$$*|G~@wNZ0OQ%PGA zkbKSq%~?X}@WY!Y%a!W24`}Pu+~Grr17q^RRS%kyn8vA}wVM#*g1yix#UG+5K!-GL zA8+pE>T8=)V~_ixrg-9WGo(s(1)Q~SdHSj~R95&d4(3O{Z+m@-Ps6UhI{xV>(U55UGpDQdMjMfofM=R zCe#cvAl20;3(2;A(;(zTZKHYO+eYQoUjS|?I^s6qWUMmdEtljtDr=oA7y7%gz<_4VQP1|!n;~C!8|HUbyg+gfzxxV zEanf9*Q2}IvzhNiUWSVJPv|$DS04Cp zooxUBaNdHYy$E9WKWLbNKy*l0I=q54c5zX|F9*sMw3UU{DW9f!A;G5J{zCMPUfBgP zzgzn1$wmup7c@yhXr2khNwZPHgFq?C5x<#ywp61(RSS*6FBCz=cdNJb?^5@$QjT_e z?dX~!nsTt|74gr@=?H}6=FXR>|LB$iM~r+Ls0;jBW66I?h{hB%4~_SW^HaQ#81Hy| z4F~bf1?S=Wa8{mg*AOVj7exe(lWLM}5> z=7X^y=)=YhKHNGLmScXTu92>o&;7B2t=B!R)!1=pgH)racBGn>%Fxm5S3Ksiz38~m zF{b5saICr2u+fES^I4OoC2vUc1{HD5T90!MSNib_hcxk#f!z!}J0bh0D!1s=@p&Wg ziTTFHCq;k#{!JMBEBa4*(0{#FH~+m%$-}Tnila2fQ|1C;DH$1YlUk%9e*irE#wvjgOjXDzw1*sj2j-kfy(*9m4M@o^Qo4T{;w?_*y;Sor8Cymf)`E2 zClCqqos&?8k+`co&k8}Op&_sg{5-_?T$ne&T9{&{TH<7GzIJ|gQg7cgNe#_$_a4$K zoj_!w$Ig;ngQ`h{%vm%ip$G?s#giL2kK`g^3MgabMs*|}<}4<^>uEXbEnM^KbS+Nv(mDxF z6d-iVvl6Bxx(NJ+zM}4zNzz%0j=8I!uXTpyKMZRJBN=kHiH$XoZZ_@U#82|a@$s=S zuduMF^AynMS|9~r!+*>9(#^-c$!@&m*QAew)gs-lX_qT8-fc}xy#&-KF(M$WGbBFr z%Wv6SOAUjkkLkJThHt7Ol&|)GSb&5opWTJ>S9J2RYg1<(@iy~Zml4b?S==w@g?lU5 zlnC2nG`hf~tiUtGyPS4GohzIIQ4je!$S6p(1G^#+i)HF86(Yuy2~tP|LYPP|p|#eX zTNte7)}R0MuSJIEswS^6aG(m~LbtjO46FqGZ2tm%;m_Uxs2#fU$EVgh^QMEh_w$Lb7i88!h;Hr1YG;cvIU7ScemxhavqXVMx-LoLR>ZPTJiT)CT;{Ww z0lsf81FzKdnQDsKzEUmKN1SkR{mvFIH1e`e%c>q~e%gV1y!Sah)HOSYEZZQRe`kB) z1MN@5#S9a(oYByC>FZKC#G4!tZ5=o8_9Pp!kkGCwA|ev2%E!LOss&4}vy~R^1n;kt zzuz(AfPJ~5<34amQm;P#{fFn6`3-A!6gCc!f^_weV9#x^Bn2YwGVs=%=D&a94?iog zKE1HFxY&txA1m?yPzu4XZj3zn%kn~;q1J3l2Cqy0&HVc-IOW#Sb8-SbEj*7nPtlQ1zog;8r}+K|KDS6fM`o1peF`K{JQVaF{BDy zBM^%ES=M{ms2RH{ESy}v=x6QHKB}}Z<(FuhKRTt4D1MO2oir(TK=O3`ih8A-_n)EI zCjeF#(D|Bq={BDgPk!>Uw4{~p*A8yi=J8|Vlx z;U9Aq993#z3wapv$P|L6$Qr+L+-L)LC?73mbJSd+{^p#^y=GP-XlYw9`~_Zu!otDl zQ&|(Am9AtSIPW^D8q2>hDYBDt8jX+YD=7-y3^{oEMmGm;;(N}iTW6lhIVy2*g~HfF zUTYeJ`4Yx5`edECAe>QBe7qtNUdsC~Ino~Eql`Q0aXa|NzH@IpPi{zmyYsTs+TCHn5=A!zZNk?ADkV` zzX!OWf3`;*(II(m>>=d^`QLVC3kHzY?g!B*&cpF~;VrK=W$EPrso3@UA{`vHpMFT* zt@~;7a!Wuy2@q=H0pv#V&-y)})UE#51yiX{z@PQ?rxUAP-2yC)&(BT0dEd>ng;$;O zpEM{N_NWo@>$vl`?Y90QR|?XQJNhgZVOK`Z0*0;;z$pArPVRhL%Oa3H{|o{n-M{>H zod=*{8od4hz$RZBz5+`DRM3yB&L|B%M$Rg zVB5{iW0T`E)ExEu3E8JLs}boh`VMU?9m%|b?++tsxU#FAOFOHYUd#K+TyB?;B?{fG zoH?{m5la<0rS^M)RapI};Pna&tucE6XznnTNEkW~WvDzFl}Cxe!y~otaxt7M%FAH} z03>+KPb^+fNUtHcF0*eqDgWE8`Zpe@G6jmSiZ+xAerc%mmI29=YByEwo>3Yt-t~ujTU~n|k2ys^rjX z^OxJd-Ukkh7_tna(5(8$$`x*GkTRZ8YNg0Xc;$vYbNb`$g~^-`jlpMoR40gOQ}Apk)Mp+VHy`=wBN8iXJ5 zFx1rN)}fEzW&msf&`iP1-rPL7_CuJk@qSyn}rJ)cNqUwF~1XZt0yR_KVoN@cHz|-1RWo>Gk=Ym74wY zweJFYo@UTO)t2k$OH#$87>r+OdO$WgfIIT+x0K}R>iM1r3lZcP>dRjrR?>g^=?9*; zbCko7h&3V#O2M?_gl&l@Ny2!06fBang{f2=`edmups{)D)>zGVOiX04Tt-o?O+4`I zp9uch(>P(G97$y);3YYTpnM+J75g1PKHH{2w-{I?or{f#;F|D}IQ#Ma_$M{L>WNjm z#(smO^?XG@ppdA&nDayc}WnK5_UMo z2+o4Z!xZ3z7tek;?G6u|dH&;@+-?W4u*lM4g3o{y+E5C*b^o!P)s4)Ff`wcQAAsw% zV&_FL_?=7ZhoLrb2OQngUBo+x*ju2e#@`lHoq*WY3qVNP{MTK^NMlRXH*+*7QPLoj zCQOBopnKC562rW^?CDilLxk;7UE#dP8Y3wYBbkSLVw+u_{AA3IybRo02FMkVZ(RHH zaqYKV2J6Kd@I?YrQpWb+;DgJ-z~OkrSsy7!y8xbe^DcSZ!(YzijH2;t0SE$5+nXPE zdMIG?Y4l8q51+=;7jmW24-ts-5z7tbhI6PqkBNuIa`}FBLrBMDUrqS!=ZAi^6YdR6 z%=@;!^dIPn4Sq3rtY4pl4H}XW!r%I~*V?CvM^j^hwVm^}$#TmlxL3ngX{ zJZ>zSV~rRpHzVE=Q6SGaq>TdLl6^wVp| zV`aiy&+cy8IO}ANykF9}LaG7yp&L_$SZWC$mJ;80ruC7P zzZcj2jysn>10HVmu z-bN~d;V;>YK=f?#bEZ(1zHJAH6Oc2}r}}zYe-s~(_%KWwQLm6rlZZS+GnJdI6}iD% z;Fh5-7LU8}s8k_eM(@$YDj>)Bps@!}RM-i$nfAOjLu3?9K5ifo*DdXTy! zdPlo>AvheS%*~|>DRMf-T}<}r*ZgnAo{xr{iBoBbU}w`y^wz&{3`K+CLk*yidYXHv zNkrVEy|wbG`vV{O26Qf-o(veH?i( zc_@r2I{ABUx5c;OXX_DXmmAFJv%9`7G_|*_T<=2jO7Ky*fe3gAA6l1);E5_I8teTt zH#t`4QC2B&44@KIXyj%#y>VIxx`2*k}m{=LWE-w01wZl6k3)M zU{}i#VW83C&zIjf|1SLfd7zB+13W`3D`lW{11PLip~&0?s>AH=i?d?!k>TD*mgfli z(n0n^nlt<*r6q7P#!nDu1}J%(3T+G|NXd}ZKh!Zjx;=CKRHYyuH1P~!{f~5PHUY#o zXf5gok++2ArSIS7`!_a9!@9bMR9UYxjfDw=q+|Q@&*z*Lzklze-}1YZ=9#LP=oO8g zi?PC+4IZKi9@-eg;aRxSlP60>XyvNQ_jt4|BIf-7)8(t&5AN`mTewkd{>U{!JOIK- zML%_)I0T}GM_{?)yh$IyN&*wps$7sPAt`yYaS~@V_rY<*TV~9cE^QL9-8c#EUUUy9 zoWw<(0?kY7VRzWmrt{B0+grZ(S}HghRou%)ui;a~4@B;hEB^h5Qu{ISA~z2|Ck^s4 z)?9$M{k;8K3I5;iLEK2={CeEg&(Bx(#wJ|~^M4i)Ew8`2EHmd5vrl>rgA+lwL1x{~ zIzKn#yaGH@6|o|JlF!e}G_Dr}DX$pX?PNH$Z@9z6vI?7$z0)5JxUzgMVE2b(4A>xi zh*FQS2{(NIW>XVb-M8I4@DJ-uA_0(@NMP)V=I^Z)HmAZCzn?$*EA!>g%$NUS1YmCd z1Q@|*Kc0P;@-36HTxyV~As&B^J`t(HaK3u`_F)JG8BpLP?mT9+`=(9UJQSe{tg7FV;!MzgP!EBmy`@} zaV+vh(;SfSR)az&Fx9+#37QdP_#*cv_l1YfSfClU4-sr9{K)27<7#|53$_;B{EKr91T=JoO&CIXs2(%`cC zqf34kBwJhZ{k#jm6lvOpW}$GOYyobLPe0HiNbNEI#?}3b@`ya3gNqsUKVP5{;oYC! zVi73xU3wsI8Xw1r{}k4nW|MkLRjW@s zr8^y8M7Uac2hgv&NN*OYww2q$PUIxq?i=xSmi8SNV23u8Qo3n)n1LvyO(I2L%u?d* z^y$f0#gz(^7ePU~KpRTgdq;#=mdSl}% z$dm2eS^?y<=780VQo**j8#pTxUS8nbi2=us5)f*3qV*S9k$XYI)?k zna6Dnagg~h7<`I3Hk(*hDnw|>8paKj)fymV;!S<$hw%CJYI67jEVl@%3RTGt5= z{(-0_o8M$+i1D^P?Tvi}l4d{3_E-1CcwEbPFS-9aoj0z?+(a8GBY%voDjB3?2Hk)k zEDpXD(1@J58@$-?wY3$GkOlAi?1hUdYlVgFU})9}Ru$LJHzTE(gGQ_L1pt7|DC)Y> z=RXUYmuen@0>`xT zx2J<{nEFJuc%4OgUdH9O1R^q0nA;n!J6t02i_Mj*C8mh|vNv5CNb)140w`CY{jh7k zMabw2u@j$6hQbp%hLZ~;i-frpW2E>&`Hf=V&f+4-H-@N`V%{m&Fng1*{meM{=LXMy zE_{^vA;N=G0&w<>A_xq(7dOLv)Fq7Cq>sy+v=yFn{pE$F~J?N^9YhaTzY@In#M{3?Z8HP==URN<{b)KCWEXYONj(alm+VZ{k?&Y6m)qpZ z_S&YJ#Cz;Ns&$1gz!jeDAG4heO%>WY5pW<-LBMS;W)7D=OsIZ(xmR58ur( zVtw4$VA@oq6&X>`F9kVy(66!n=o;y0y7c>riTq^Npc6o{!m6G>=Rr3GtXek{CqKRy z4&!MNi_h?wc(GPidX+p6Y5@I&hz95L1@VtDG1AdYCGzFVATbRRMRz#|Tzf$UkmrZ` z2b{%F2-YYcV~Y6ThEv2p8)~d`RGYDiSmupCRma+Rp_Ad$OLY;o7Hc`C8yVXr$t27KH#;@m09=&$5 z!y6z*8XrakGy)vT%TGV<+=usSO5RV{!0pRYM6Q&AT`XLcpCd-?Dh92ay%3XUQARQv z&eKSxL{+Y+;vak*U@E1H@}NN?5MNojV%&@lE3kiC7Z=fS}Q5Yk&3}|f1 zJIg2{9r?#0h(9-NxJ7RPyA`~$o%4l%T0G-?rLXr#BIiKA4Ngvc&bR!qGg(#kX=!KX z50RXe2uILQKK1iLUX90t)T@=zCw4c6?%S^Ctp^DiFpo*m5V+Wj19fEMPR(LM`T}** zW0tBZW<-P25}PilL^7O#vPG^NT=6d*S$4yMB?w?7oz^`4qOJ;*a=1-w1Cq1Gf=|@+ zulO>Z#fMp0{2%|TJq8fd-vdu~RIyD(f+&s5AMnO$u?ID+pmRlfTNeg`ifoAuE(Q88 z-q@(}pMwrXo^kAD@)O#V&3c$w1}ShKz!g%uFPtZ@U`W@{=k3}lTG1u`x5_GY7WEc_ z<1HD7?%$sVw_?3y&oUnLq2xIVOVs1-3P^i$vf5h6=PxA_vb!@Fl`Fy%qbH5L5?N?G zrtdR`H>rTGsUEwBf=QhI2#S(x6+Ow!Xc0KRRT(U2wd;Tl^V!WrpxTVk_1}Yua`*RL zB0`_8%PY*FwbM2hd%-4{xyG!o1!EWwz)kj^Mu(ZXMF zx#P1Fsp2xR?S^HCI{QI@I=$qiNKNnN`_s(^$7hsyIG9lunKN*+>jxfk{UJQQoc3E; zt=QVEpYsQMj?a7p01jWx&?3IE$F7^OL%B9(m6caYPA>nx8>}JD9{Q`eE&TK2;U&Oz z0NUysFo4j5{HDQ(;;T?^RJf`@*$@W7oJ z&cvx|CY;le25TV@`Av2S*}namwbq57v8A=j{JHiaibz!&u?3^(b8aMX)mW@OUs=9K z2u-Z=Bb|dY!x$hW;vh~SIxQX~jZ~LMwIYT1p}^G|0MeLUyI%$RES@cM+sR~XLPqlB70;|so2@ppzp^CB$!tF(y!_@5 z7ar+bKB|jPcdo#T)il`$CRo51+&zZeU9hC;{GNjkC7l~g3!Zy9T*<0v)lADwv;7$- zb_9BkaM{-to2&x7upSct818S&>efhjxsmdOJM;>p@RR`Oy-FKJ-atP;x-_|X_&Pu? zl(r!=jrWv#An4Q2?VEylzz}B|T9u8UXvt=zqfeTC(XsYz$_jOg4(aq*t(d$)H>pe+ zAaq+*hDhxfXnU&ER^vA=!1z&_-^n_U+rP{S`m094D2r5Fmap zyhWa9pDjyBAC@Yt2?=wAq3pB5R0kR_3Y182K=3d}L&M<^xD)xMl5f9eNEDoODE|Gb zIsrRi-u4#GPIy2KO)XmT!f`@|d-C6Zr?d;B(`@`R;zx&uy1%yUXYGJA5O~x89arS# z4%fmStM7WBi@5?Vp<{-rq*+`bq5{CRKnLZM6JfB2M!%HY7iEz7@vVfLo&O%UPu zr|r&wcy)L`INx4gCvrb_UVB$Jq zz&@3-<^Smi*)pwiKP@fpJeBCDb;Rm}oGmdW4aZ{=ONSqt-9$Cdt!Ped2oP4qZ5j8(CemZUYEJw3Tw;(&9+S0Bif2~|5w zeJ+4GbLG>_^VE(U+4%L|orQT_ClO+6k)ioBCGZJ$!)KR*m)FM3O%)+oi!J!`W^l%-hzAl=4^%T!VZ#s}2$F(uEr+ za%7^Zr3aSNl`P?%3+JL_u%kHO$RfJ}eWFvCc^sst4dd3}YJxVhF|?`V!7xWQHglLM zOd%%BY(JYq{ggO#{6h?f@RxgNVj^70SSm!*nP2FFJlKl;9&Y(FIPIhXwDI=IhYt^Y z3=a06;dL}*6vgBv(x*0J#rV0wLx+EA_hi^z9lTGBi%)i(*2Kkth$Mx)?6uhN2P7^{ zip{AxbQM>IGdM)9f7;Q4zxEc7t8$9?|K1z3tr4tI`o6Ba15T=4o8s3?i^TAPoQ{^t z3K+zzw*oBA0@MlfIF&v=c0qOi9<1YuVuGHl@V-i~ehOKk$Ou{p;fn(4;0pn^{e$L# zu|0&CE8Z3Tjj#FMto}@OkyVpK@}i@XZ;tvutV)Knp+B>Cj=!ni+n`T|KR?V0(z`jW z?lt=K#q5u}1ak!bLXip>scyHT>fG}uR>31X2d=D^Wx(r;Z*Va#I;#Jtxv^1^HkAdA z@oKvpV182G2uC@^p>=6+YW6(`U@wwW+lka^q)MtRARxz z-rsw4HJu2TC4zerh+M+ZAVaV9(3;VOyf}1$VFtL?3NIN+Nk@6Cm!JFV-#$nXgoCCd z=~rD{(e8pe0^Mh5lH3mB<8_nney-or_N*A60771v07mnrtv~?D0UzFvJK)aHO4g^? zp&;Rc$GRSzTKy%XEiVD%Mhk35v zr5LP-m8DdbWo2d@P<2iHMHNwB({#=D3UIn>ENYynv<|FN20;I0ILGmflWv192O`48 z=D(hvvPg$Z3_y3jUMq&70~SP!Jq>P6CWGc*BH~~8qhE%2s_-p1^4|esM#qe|_}{Bk z>d78vV6#M|1Dr-6-Wr|)1s&l1S46p%p(EcM{PzrNr+-zXDCrW6?E#7t>f{Oa90BoS z>H}*=t^%vImA;}5YIuQ6iJ2=kSQh9|uAv`tSUfCd2D>oxrq0MPu@wGFW1(EhXGDtO zEXAhGN|vI{g~UMGC{AH6hw?H7lO!{Si3JJ7ln%yFO0+E4geVX0RhjVTWwKHfufuY` z;$597DX3%-dCon+)y&eZDqWuR+)c5vi_UYqefv64AhY(NX)4mRde1_$DyOs0ZEpN% zXvW5X|0V>1f)#^WyS!!_JmH+WBvAVYdgZ$o0g}WG%d9AcOX&+yd+)wHvBFL_QAuzd zuebb{)whrPVf-r#?Bxt6O@;f0;&J-1*fHUFcx$d{T5ISOH0w+(pj1zS_~bSPY|vm( zXp9p#iwh1!?Rt0^8#kmF!JEA$bH)AMCeO#cQoYnjpPb!FzDO?@8hBw;nge8FhD&l> zF(un|PIPAr_Xn3VbH_#FT7bgl_R&~dugYMI{D=96Ya&^S^?o-)MLvA{c}Dd^1l&^m zzk$sztR8%&!IP}TiUAKZKs7rfSxa4yFfjnU?>d*%F(Fm}7}}MWMdN|n&l8UUB{KM7TX<8ee;WpJ3zS{t@!6$)=muiwNzYd9j*fW!Tz>SNGqyr$aoG` zS{}abO*A|7$Pi-b(DhaWdH|ECNhf#ETs zYNd*@T2Muttz+a93-D57Ib`5># ze2^E|2*``c-}?bxWsg^YHVQfLS5>0ND%7tT84>SrGo=H*#4J$R!>;8gz_JaCMGpd- z^i%O@Go38Q3m5fb*`>lrDG*B#!LE(b(EiL{^E>O&7pu#UQ$_wp5YB9pHg%+KjaqfZ zCp!aSjq*N=Bk$S^$!(K+;@SpFy)Z5slwJFu%dvRse{8;pNcxqn)-n;}uU1WUWO}rV zPzOBB*YyIL3yeJ7LfRN*os7zi$Y}UyaenTjJh)D#5-d+S?q4Do@{UaONjWJN|0jvVh&UN@boeZJXacm-p>7o~gRKupLjtM+qp*nLfIJ+D zzW#%G+HIn)Jf508cCFT_a++%QWXv4QAbBMULfYm6EZr_&Oh`$(J{`eTSYs++tL8_3 zi_%R`=z!yOqYU6}Pp2#(h)u^7(iCLl#E1B{u$>PXILPcR_ImkMUF{gzAw75=FL84` z^a1F_1%UHnvgNyki}DA;q8`8h!1J(2PyeXXUf6?P)mPOF)^or#TiYPr2|aSgcYV`D z8{Ypbv06LzXfHA;+h7=rPc-BPq7HxAm`%|4^>1DpU;F}rVK^&x$xp1KYJo~mIBv9w ziDAuDK}{H57#Gwu>8TjginJn;RG)$U44^SydHHeW7-$Ql4ugf|NXiydDHy;+&JzKn z5i~n_{T=>xYY$XnoQ}Cbe9G)aI;xKdmkS4O(y0;zUPJ({mXC5HH zl1vceN|N3tu^*XG=*i*%Zaf45vSd8B7v~`DFD+4EOY9nCL#H944ur#IE)T<522)^q zbgHscGn(`kPl(V>w_Nco9s|ZS9XRFEK6d_lzr&lu18cv5AlurBPt?Eb>t3#|9p2*7 zg#nvQ6RRKVsmB2S4)hKzF22I{DIQ)aqJa4mOxb$*xhqIeD3ZC42~R@cUF0VRXQA3lIjpg)z8Oh$(Qh|w7jbYXV9_9C^Hq@ z>cMl>io&!vpxl1*@~DOdseR#0+Z&QB6c&~vfQ}Ybe2cSfe{%h9YU;Npwd4u+n9pH? z{X6pzB(2!_XYc|m$AO-6q~QXZmZdWO5obOzC+kU|pxHpb&3% z^lMt5?Nf;TRdp~MoV`;>PGk3nd-l#=AT*_oLL+BnAWla%h>QIkrZ_~CGfS<29QO^o zP*Whoi4T`m_j@RQ!=nv1qAkxWX%?k2v{gR%Dx(FhRL1ZP%rKI@OOC-Q9_e_X=~ZD+ zG&7sO^gD2pQnwuGaQFM592i;PkHLX{Y4I39|3>OGybQkO=U3CvP>JaVa9_H{Y2d*DfkKl3+sBl+XtGwY@R&<X?OVdF{W)}gyD@pfEzWV_BGU1Tm(DMks*`y9DI+{#TzNlqa% z41W*4KAgI5*=R);q(?_R-X0TnjC(8f-yNT@C@Jj9`jo|~USRK7VUWM!Bp*kRWr)dz z?QgMt0Ry!ju^T(Heq%%NcEw+{6e2r^QgHjFK8lR=puOduC+?n}+1vMs2(SLR^(uI|k}sqUR395)3jTCfFl?fZj3C2r zWywQqwj^J@HjPAXMhCV)6ndl^=3uzVEI6}}mD>B*0;r2|q$B&_W*+J?Fgqb_GlMx? zDgJ4xHd42flU^)272$;I0&KG5}_y#4D@-y{uF!qmyZr{&VzQAS6rr}gEBS# znmo|DSS>MAZXfPyXOPvLKA>`KK>Pv!i599lbz3el;lsC2%j3s6^QsHB`4q4Nr{6|A zZ2wu?Z)~TfZ?JUg#kY86{@?auGDPw`JjCcc>4wpCezxp!<>pq+iXFoB2? zs;~@)t7h{T4NrG15s9;4DLt`z{5vBE^k4sTEve7_yXn97Ue-t`ExotzeA~2XgHgyJ zpk2fR;kU=&T;rrm9Kl-~76{xc&u(Tz(8u7vzuVg-Snn!8@27&R{b!?lZQerQa-^BU z%R=%RK1Eodkvv=nNivjdeUszCr_h{IX^#zv2$@0(5r_=)$v$jo>5M?|js{ei9wr@^ zD`SNa5WK|rXFJclrGsPv6{6B2HV@--AHK0Yabus37ihA;t@5FjdWKHMn{{6rFCRhE zPI&_Al*U`^Kcg}@Z1iyGd-O0=;`^CXH$I#>b7mX2_PypxiG>B}4lq_D9Zxjv_%NK+ ztfhHMbYQg;9->U2eozk{uV*%Al@QlC$ZI=hG@QKCBAQ=!S(pinR*@~WMnv>dG#AWJ zLqb)bN2r_XJpQ>I+P?$Bn@3VsgPiMqn>ieg=~#@4*v|8 z&Hqi@Znj$kR^}{UBej2#;nSecZQ<-+IP~UU&AKtjm<2S#(9i6)no3HqB@cPtoSyQ! znwC+y60f)LvDxIz?=N=9UwtmW3sdcYhaMB7^f3Fv)ejm8fFLg93Q9(5vi#+6e#-WO@*P8)el8ZgUTkHXaWs>_V1Ky zIF*(K!QHRf!0Ir|Am*1_&@7YhR3H59)7$Z9w^j z!;wTp#P@h1X>o!e?MSkQ6C$m~vXsM=)U$XE*ZK3C zJ^n5tn_lXw!7BF0`wsd#p5L55ekLc|dz&rwO2ZeP6B8dlSwCg-wLCW8B&l)pE<I zWe6Z3e?pM$SXSwT=cxyH6gLfSS=2V|rJBB+G45UmAj+R_v#|`C?Fwt)A;py5O8|EN zk2A#$lEVKiedt%gTUNpCNEWz&P?&?o$8|$j7s&3bt zgI?-|wLjlo9~v)=Rvd1q{JXbsc3=|$v7vzn!V(FLmD2j)79=XgxJIY_%~aYp~``Di)fdVyF(Z}G?sd7GVz(hz~*NRV7C5a*JS~~ zVMAp>9-YGy4>KAr*gMF&6!0!KFY&_QKTHCsAA93J1DX@S(=CK?ObZ}0e0=bKiZY7Q zi+)bE{JV-^7|v`^K^X#Xp&)EfLJI+NUahBeThn)!A!-&|N(83IPz`iEZR;fH@oqpQwm{aS+Q)WsiM zN7I8hgVMBuZ+Uec>`#FH))g%{o9uE8F3nZvoIdI@X~LFfpQr}ARlD3hVL+azCXhG< z`mlz_1U7A+dDeS++`jF4^X3Fof%Z6Irt^_X!nsd5hny`D&x_=N$86*p;Iz$=$5#WK zSM4@dLBlOTcHjQ-Re(Eut}sR9N)VBlJni)uv)&T^&#k@DeYJV|&qlrpSgwHIiI10; zy_Mmx*^!RuBvg-3lx{U?-?-DE`m1pQ;Sdy#@>1M?=4FM7EDD1ANW%*YTP(RqJGadudc_3_E7PO{|cN=rFC??6r@hWaPR@_kh__*6wwh*JJ~Q6o%^FWXHQ( zU4CZ$cW37Q96x{xi5)l(G&A7KJq~7yGwxV1Z$vX7PX05pf)c+R_cM&Fmu8|iFc2k$ zb3%{-<>jYO&HTh;41D<78N&&r9ni?7s_d9Mk?gJtz5SYQS z`hI+TH}>TYAAz{M=)#K!>JF|5(? zMXz@z^u-#0h}kw`b-R>()kt{1l+aufX97dR?ZMnxauhD8bbbbW%D_E`fUB}WB9IPg zFan;;5$BDJgegGmvbrr~dZCUuLBCu3X;YJR)#SI=2Sb(`?7UaN|CG(?VIM9)3V}yb z%SQgir!HSU(|lV>aJiurjDDt;LQz1=0hVbvt*vDW_$W@ln@FwRW;pA|j`-XD5vTi4 zD95dgd3Kj>eEljxTW7@|UNHRVD5_;}J9Fd+Q^{9+AB56$0qg2KjJNo?M+&@k9_BC| ziXZ{p%px6EfEuy28SSl zgA>YV8m9x73;WUIP8bE~AU~{Su3%Nf;mr&CK_Dso{N2@*9=hq4a<)fH<*#U}qcP<2 zt7JfwKb`B@=D_9cn(F*J-`X!9@9mU);Cmc}M+(xgDQ=~C*B+ZyEpKJyl~OY9=_>_O z&Izvu08TkkR=te~T;6$)nxdNBL6H4?(%fLy8}-fc4kA_W`ZceuTKN1l7;%+rPUNc*oK4Gz_eQs6IZk zfaSK}G3+G|2mo8xju{L|6;p_>!V-$J4G`x@egr{IoyTxCw$~ZC=lH8}3teAi#xPXpQ40M0q-IrihuewP{!os1%(*;mMk zf0N>;V2};_tihOki-IY$%=T)OE^2ycWD-HItkAObdC55@SjJ}rvgXj?+j*GTdRBjb zf_V9ZG*=Qy=Dj`~Q>MynlOC}sOiB*tQ=Lk!JPbXO>g5cF6d^PUW~mTaDOhMc4cITru@`rDz1*-Z$4D@zvhPGc&%({qj_v6J2m7)lGV*t_2}3U#)DTU+y{jfW zW$vjzawbvzY<}YC^2e8k0Z&@soMMj;PU31FuoBTDiP-d8#bYNG4z8N>$rCv5k>Uxds10_QGms5aKn&s?hz04~iA(5T^{mSjtTF@eNp6FEX zPNA?l2Hm>#<>xSH5m*2R6+R05jkA8OwGrLP+uM6Neq}3gUI2y$8GsSqs)hT-X9k#A zsovtgMsjd@8Z1hX_%u(Uv{W>*piObpuoa1Mya4McDI)X_89F5rb+}$dJ;BOa`O1S+@*iEPu2DmiBwTvn}-R*bjwY*wz8vVK@D@fV~YKlzkpFN z5PmBx9c4V!Gh_0UwgYwGuN^L3KM5=YY?;pFTI4JftCoz7r1V8}Qiy+Ar|FqHo~xh2B>G6}FPNiA?Zr-nT{B*p1USBl~2 zY$vbbA*lb+sYE`%Gx6TJgl}c#u2OGjmDc1qr|1I0x5L7Aq}ku1kMbXsQ7A22)4QYARS)x^x%+0l^$@(K*h za87qXW^JOP@-WxbGvS{>M(-skw5z0CGRem2(}Bg91gJ>$os?SZ2)HW+5Hr_y`&`sw zq-Y^U6mRhi&^d4tO@jl;^Eq5DJ6PSeKV3w(@UD71OEYygouCt4SV-9qS5?YXV+^yi z=|+jUQrZJH!q8^iMDZPm>dACN*%xb9ngeTrlT@a!`l`6m{)B{Z)MAau#qK0&un2$L zU~?U;-%taaiia=aVLeRs(Xj|P(E57$#ucWL+r7&vF&-IUTR-k@_uPjIo>c^3-Q^Yv z*$XK$Nf7`pk=a+Il)l_>7f{ue`47jlK*ThA!Kc!y^3cF003EJ8d6|&tUS<4wCocQk za7~QkL>=&}h=}$9nP* z4^K4tSWZ4TS+F2>IM{O0p{^;1=e&&29IkSvI(1=5VZQn`RcXt-Q7FVr!8Ki(1&<*g zLp~Z1IFZw6yZ*R(etsdM1nC&3jMlcg&{w;1a!l0%w&SEa{^?V3je8vkKyhXs{ zfBEh3r>48V1G5)yH|~Ne_52oO*;pSY#)9xfou}zgGXBVF& zXv6#?kdbIIx2P~Rd%TW=r^wKWs~uUU0KtwLG7U$$(zW@$hztvZJYD@(QNz!1YL7}w z^DqP=Let~sWIw*R0EnDNB*YyEcbcB3IHybQ{rh&tSB_QMK5wXQJekdJH3rCb2^Adu zzn5x0t(hAPYaH(x`tf`4>5Yt{8I8yy1VKf7|BQw)UH7EG_>YTpn(uMq3=JNa40xtM z9bG~?YF?2|e=bw}z^Q|7e84ft$49CoU%D~QV*(!q5UHLw^?lZ*^`*{d&E5y!bB$S7 zeFWS`yhAyTqAU=m%{oHgZv3>S#sUm$@eaTqrc(?o-Ow~~SjZz>!i-jAkuw>5&K;7$ z1ksSusr?-OyiXBp+f%2U(uEYv=pdN~QfLGH}~IK&AQ!$p23zn?)9=L)sSt zUA{C34`YGDGe+u!!80IwduUe!@Oh-e4#>tW5FUU=x*oHUf=N$sUC^{@RW|s{kAGZx zjUiV=5h~Xs`mt8`RXHF5BmEG1Rv7@eAqXoL-_{0d0j`v(V+w(KUMvUR+!Jm!sMj#T zejQ-kwzMy>1RlHh%4Y<#gb6xibfZ&TSeUA;Mh|bsu<8lu8BZ(kG|XE}Hd#24kDCh~ zmx|Et*WAZnK*?Sx40-p)h-9=;@F>~Sy&SN8y)?L?`r6Xu59{oI+eNO}UB0Y@@xR=t z)H5@jZ&0X;1zf*H(7bJfX%;b3|Sug(b7Gg+NHDC>;9vRfKFCMD91UohUr^zl z@|EB_j-27b7-X1Rs_tiVg~wR;352<6J-k{_cbl&{0u=rn{`&dKpSc6W*gwlwFTYqS z-dwv2JiN8n#|kc$;tEO^J1x=4*zzWTz;L%R9*3d*Cc!m^spK6P2zU>D;K&EwGyez& zq-xOBI5xJGEuc533Eb8|f&)UVjS6&6QS+J2Fy*4QB{=)u46G_J}4yIqJUCark77X%Da5rubT-P zYWjVDZGGC|cKouwzVqqNv?SPKOj4&u-BjOVD=xpGI5^@5 z7Ym5sB5hZ|=>Y1Y>$AV9WxI7PAMTLHAfPLZS>>i+Te}!6uXb$ zzG-heSoh^If2B;b_nuhAl$c~D5w6ExKw)X#i@}FJa?TEXF3H(ocp^HEh$ae0si4!C zs<1Xyg%Yy|dMA(iviSxb7h!x;iykRV%@)hu<6a1VOZ9D-IxXsWiRk*@#bYE;52pFC zH`Rxd2SZzIT#24rzHt&yop>&u%8Q26fQF^T2y;C-CWX9}eQNPQK89WU`ys8^9>wHP zu-U$+|KN2AG}V;^48g!y)~k6i3`}^Uk#h|rr8YMw#n1|_VP=&}rgd0lR)u8`woge> zx{8oLXfXU8qyo`El>qQx&>nu%jw~X6h)W>s0tpnIaB;qW|6bilt;Yu>Bg;EW;L0&f zu!f0zk>{4-xybMsC+XHmAKOz_HL6yUSCoI9#z9m?ccBbN5}%%HG2n&k#57sWQzd8* z*0?APb2j{0j6ogqMWtJexj&Y4KaXZ6#tJQ>OnRn1{?&MPmuy$i_6@KKOowao^MB8! zv`aaM55K^n9OJT6d1TujNNW>?rB=9z2o+Y5cPH>*6$>p%xxVgU))?B>eF5s_Te%@O zP~dK%fOOQKIO^~@dx6X-EG<=9mewCy`|&N^5S}jVQc#^Df!{BAkqm+(4Gj$nG>tQ7 zEJxJXb)x%I9o;~WlIYC1bd?6U=gGK)#4n2otDgU8bD49e?^^aI_j`?&C8n3@DmFnzLR^<A-`*{je-%6bLJ~u^yNf38d_N#;RSj+_&`;x?QgL*=XUQR<;Zk`ek&x; zTYBwR#mXs5E0Hpi>-Fo!lgsgX2jfN#eEh3@vRAK}K(4B<`TmdE`fIrdw6O6ti`X?a z-w9UKr-StzP~p;@$gGPBRxb?@ny{QAO8T5L!`H_Wm->qf1UV0;5T^1lPGy$YkP0#E z+Op`kbOi(Us~QXWgKofhIE)?G9KxoI?|5kdYUG&HF~GLN8VTb2T$12Jpp;LV4eG>! zGLvqWUIr`KY#8fYadCfy3t6_^5Li}UeoC!9&htGwmLGsTlY`J%o62Q(=CGYCm`)*|za6Gtqi1HtEEw}~V|M1=B>FGT+-QENIDvXVmtlf5d;7o8A*tI zs-C`c@BD9#E)pSly8q(kH_CC(o&{ifVx_LvEh{SafSa)r1tB|JBDv#DR*e&ma0C2A z5~AJzTIEXjK*fj{U})=IDZ_L-;o|c;{cWLFEiDgLMp3|}m06UpphbH;W>X+@zjbzf z>XtI{Qk^%*=GNApB#o4kgb4bgdK+gjjdoO)Y_`=KGv3HKj0hMAcm=W+H0`r|CC(Gm zUBS^)N;|CsoSto#Tgg1yQOorsphM8Cnbfrzq;+J%lFQ*TWA5wMudffh*sA#Fb&Y+z z^J@!iVu%9%SP2X-qfd*1Brp~T*$bN7>+U6Nok*|^@|OBpTdyZgOcQ^a9O&Y+RZP)J zH}snffp;&*zU5ERf9U>MpLv$7e>E<`Af>4L|^d>D7GNPB4B=gDS|iMrFlLbPm;iH6Fts4|QJR(y_XHxI4_ zy#2^n`lXh{L(gY4z$Xmuu~*YN-{C|`ZM`pdGZ!P1`TT9zO$dY$UP-CCUC`m%DdFCz2;HgkDlU~MMVxkNDR z4gY0SfeZkWAoVg3`|h337AY;QT(ZngDR-Lr#(D7FN^MkFl61OZ$c%;q5-?d8Eo-f< zqU;b^_N1{bORFDua)|w!|L7>pROfLPOHaI*31Mh%wbl%0fqQM>n>1J&9d+z~ZS^uR z-_irzpD6EuBQ>K4ycN=m(&u;zUERgPhpR#^kVrk=T&P>vBPLCv?=KnCF7V=pYf48H z46YJd$_tQ=k?_zcdttQGgcvKv);mo47)Gt>k>!mK0L(bpWs7le{{~Z07nn1geAn!6ygaJ`?#Lgaiyqg4N#mw`;Zg8D zghQxrS*q$u$FD~&C);6+%;l98S{|2Ox0($Eh8gQdCHC>UJ3&D>B>${5SgOHWfrUkU zzw;poO04(j8aTEqpemk1u=-gELczeqv*f3s=Ao}P;;t@NNSpq5N|2Ye7UHpVB?;0xarL@sNi-dsNG`1Oj{{w78}hxhUS zBk8;YsowuTt|*kO#8IcL6GxpmqR2>yIx?~nnc3N!gzQcBCR{|>A$uGg`y^##m6?#e z*YD-~yMN#7I?g$t_v`(9J)e)~V4GiK)a28o3R=nTz(!(WF$oD4^>uh7IOu%pyWm2N z9H40{P@R_d?nYBh<&mGVzNer)I4Q(q=8X{w4G!iYp_QhasY0)bzJD3;&$&h$sJ)lf z5^KW<2E=uLyppxP1e?GLAz0a=#!87HmjHdI@$vCc+x$8704k9_VJ6tcQlzJ_mm)vklgK(n*R4V=Yr3X)Uy796t?Kt(Dy?WTE ze7{q`Pky_L3lCEntsigwM%2%bQ{tcnvZvwYEyT3Y|CjNGzW%q^iUihN1PlY9iuMjEN9$>8| zoXW$))7gj{1k2E=uZh2U9fX0O%mZh6xl;}IvPjfO|8}l2d;YM+?792GLMLyB%gV2Q zrsYwXlKJ5-P`$ZhTQlr>SY8QyNbPA=POMeI4Kh`Rg7$UH1^KHV^EHd^T#!DK@+w$8 z$+gw)@{c$p;iQ-Ouwi|C{jrLw6dOXKksGwEZnn@TNn|jo_c$$V-GmjL>mwWB5VAl9 z*U4mc%56Ky*mLuvuH(lDPLJX*Uv{RTa2=X;Zd$vW3HK@}L`-60!qShk_<-7(n^U;pU@$qCfBd~lCb};T1FoMYJPn73170lzP_x^8S*|g;k57U z{|WVWc6I`92(rL{p_IM1ktIZYf_e~V^wdV?yz%9HgN3Bi(@|5%l)-|M1UY_-5hrBLuqkW_ah7@r8EooP*+?%2c?peY zjH)CGfl39{6l}!xkj$qHn8aB_rknCAZL;$G(@#^35`CK9DgPCso_iyldLv=ubsbcd zWo2T;gC}K4&tZb{6uL%)k4v z2KJ+P&vyUb=XR!#h%vw(YsPDjS)E zNi!{?q^kHI@;Tn5oY}V}g-&Pu>%HuwTKUeD3#m5^h;Gg2!`fX9k}rw!8m7fpcFjE7 zeZXij+pq4L(&zBavK*T1^bxLUG;JJgWLB)&gN|LQ4B34j^r`yMaomj9gciQ`MKuZq zGbM(^DIwbC%gxf3NaalLv_7{G8fQ}a8b2Kd&$u1 z0s#_nB$B^vkBcz%`$NB#cEgeEA9*f(A3UMto_PMw7BM$hZ$^**bameQr^q?J6IQ8{ zH@M)7e!!af_c`0!i}iTuF>=IPLLjjG*T z5`X;6+{{*g2#zmeis;7zOHs7rP%D&OWK3!ix$hvEV4~)tQ63*RFcVf#t8W!lO&K;($3H?NEu7=mQ7vcn}G3) zYBM{{I9KZFTJc*#&4311Q=^rkkZk0Wd8~qvkfcbBB^J(||DiSpl`P-zcPdxd%RLj-4T4_3iWqoOfX0ruOl^&qXfzQzS-d zd~GsW5*l$DaVVVKT<=xlx+pO#1{%LL>*_siR;JPq{zI1~3tIDRy{@z#f85Vt*LKfy zy$8VX4S!v|UN}c6L~K!SGwY>7Cym+Tq*uMCTFBT=lbsGUt%hfI!GP(&3jtoma0NlZ zbR#^VJWSvLCn)dp!Qu3G%mYLCL zV`ueQ?#LBI{I&=h~4^&EfA~02SqF6MB;A>~RoXP;d+Xy_9`ycsNq7fVXyK z((rbif&~@<1C67-y}hI3rtaZKPY@Z+l7N18xO`M|?4C9!zT2VbH2abHfM=KcAXL5j_Vfv>&g2(2!DY!oo5*^AriR z#~$vdd+ZC2cXpuyC!;94@?m^EZKM%iOZlt&tAJEL+JsiJ^(#dB&3XK>Prj96=-$RC zQ?ZiMQodz<_%+Pn)P&ZX1e7!3ZlNLUL5)4Jz*)n${x;* zxq^R^6nk={EA|z;>b#cw(~t%-u$m*4ePg6R$4 zr)5+@M$rV6uMKs#R(uJ}K+zonUAKKn42&4ga1q>X!gT{^?FlPvL(=)}H1C!>?}L+6B8kF0M^KrtP=qF3oXM z@4(i;z_U6H^+2aLZy-6u;adz)(UR*qa?PeRTMvI4tq%N zPwWHp&&_;$yw2_a@!C9yH@vmSw6n{Xe={Xl3sJ-!qV0*-!824TzFw#w_Gz0JJbig14~Z&mZk{* z>Owr*g6lOlkR2HL|Mxjl`ynvsc%@)AlH<{@!^$=T$;bb_O8WFvR!yyEZe5ANsb*}| zMUDcO?SNDI)qel=#)pL+xG9SKc@YPcQ?<>V)hzrf1zbn1zKD;2pG0fWfbU`}8+*{4 zNa2DVW)=}j+j-?iNF=L`Xb2AD&t^<(%nbi?q8$Yr{iDEtiRSNw7)?b_J?55SP&(a; z_MwwxU}^WA;1&I2UYT@CNBH+DO=$?Z=X4lb*!a`Pqp3$iTVM~r zrT$Te_IaAN=o{6+&3BW1?GDI@DWPkvWCWQ))epTdm7MTTmGAJcK$1YRuXq4^yK0I{ z@t)4tS568kLbh9r!(*?P(%k+HkKB-&{2@|FD{21GIVU$)_w_t9(q0E>XiGmoyuZ*4 zdzln2;o=3k=J2pH}9~Omz$={5KH@Q z-a$qRIuySWOp_S7I&1{IrM`d0C`Hk>QL-V4k8CXv{tK=Y2I3Se^-q3&{~q%Rec1Q) zGlVH;_wK(#QsG%8-RjaB*UdOUta*qBR45#XBNb@b2Ty)hnAe2o|EVr4JFWQm&Xf9>Y#(qh{P061I?%_I2p6rUDg!-J3ED zPs@gSng&`g{aG!yd(SWUQshvmNK)n?evAD%uKDw4q4}&Y9r=sfe-1enL{>z;l9KIh zf3i3x&-&;1TX0vy;r&(VeYZDWBYyFM*EAC4Wcmft=B7}=2JGiT^KWEgY^kn1%9whN zL+~wLBOPYh_?F0!4YrwPJ+iKI57tNsXMSIBiO573j6J&>dsY)j0CshaTid&tnAFADy^)}W z%^&s(=g@rk>#X0rZx-Y2X7YO5!yhYuFs3I~9=?By4pf6-_EN}?`6`WmyNZQPEcH1* z$pQ^?0nL2W;Hx{~Q?KhJ3i>{4FaN7qzDq@#(wm8eY*{Ay{>E24^m&X~EiZn5UqM!0 zdDp0zN|#%)hiI#hu(GzU)J(Oiberu?bZ*!eEAE^5)~fF8?5zKvSaiqReKghc>o^#@ znf<+}k=?wiYL3W93_z4c;W|x9!(^mlu8(1F@1_|40Z+f_XHA5Gbd*b)vzUn}A_^50 zLf7WvWP7%jK`1+H9h`}xlYA4b=cK_Poi?@*Bot?ay%r|Z_=!xS@#R@30liR4BWyF(%dPD;!n-WA z6c>I)D4kf(NAlHc?;@MS(~b*;pDxaKqoll7`P*D>#|u&Zhr?XO=q4V{$nU&mj8pD( zPgmo+xD<}U>E)0p#`J}`w{YttIHPaR&Y76xn*M*x{(ORhCL>!t$$}9q!bvDx+3;sfK>Wjg0r0j< z2P!9Z8eYJ+&z%bhzs4uuZ=1pb>olE)*N16ty_*3(glqe4H{*48`)H0HJ64z0m)IT- z^c1--mE=Be(ja-~%}Ew#Z!>`7d$0dMt3ZE?Hlr&Td8G}D|@`EvNWCg1R6 zbtogwpcH5&2Zo>#lIXeKy1BUtzi$gHm)&%VKbIaviyx|ZXLjZ%>0Rcy@xd+yRWV2Z zFw08>L46WS_}e|t8dRuNQ}LoG_G3S^0{nQT78Rr9iT!f{GK6@caDK!F6@KRzx%gwV zwN_papmFkIFj_SpGf?+-9d+uXY1d_wG7arPa-XJy15q|0uz_4tvs}O3!IeJkdOx`b zz%tSU)K`T!NX1u3q0}lu{Ajv0gbC~*g&!-eC(@u9JTVa-X(A#b#a%(l=Pq%pqKkz` zY?0bZDtR%H3YM1E9C7zN)_mt&E1#*8`<(Kj?JDj6k@tpkOg49JmDB z+;WGlID7?qzN(y+v7jZBK>F!ZogXh9#^C$pIQhjz$R8>Ga@GHnoMj70((QdSu~gz5 zu*mew&vum-nzY(#p&6*;12W_1gt|)o@oWLiHk*P1SHsP4QK;MoCM_zz^Qkk4&R^Kk z^{UJSo8!ka!U*x`9kTM?rsqk9N)%5Tt+%LyNW0E*(sV50HVO|A~n`zg@FA-0hJtRRNje_r5;J+mzLz$&)`` zZ!hCj0{8TjfM?++0eA0%D)fy2QaBaN`Nl|-S#HNm(mAEJ5nfe`w_z6Bj0UHb5l*{0l|Nszr3*F`}n2~ z6~cKx$+BTXjm1Z7UG}>c@O+E(t>5jj9v>Y&ILZBN*Kb3ymG0pJfQ6Iec~8s-wT=q| zdRUkKI|EpGvVZ(Jh~YsGevW}5}h_T z1}PCJlt6cteq8tn`-AZQzfNbT=-Lu4p7BD3PlfJ_`98gyP8wA_Y_atSFI)2cRe2_= zcg8MmL95EHwZc*<4i?&c3Q=O3S`5+?oBnuxwW0?zRBvw>suFC!4Tqv`c(dB*HrY?>?9Ay;aSsCRtS#nrn0$-QCkI^> zdtGqW#Ex9jp*f#a&%W3#W%`Ft#JborORd-2clrrII*bR8*@L5f{gob4u%Vo^1t~d3 z?yASdOMEfZWRyV%X2GKY%3qDGu?RNb&Y89E-{q=Oz7(dTI{q$|zqvoL{Nt`*C7FZ^ zuz^p77{|xOA^cszbcrOm6GX<9&@no1c13!O%b+-%d#awC#bu-Kk@&iLdTP?iKf^zN zGUE=7%fPT?Vmx;o?|F1y38_`J_3C4vZ1eQ@y#83ba;*Y%Fyog~0hR26p;OQMVuy!^ zd(9ULr>>@p4|bQ&z}x zfBl36RDv`ngnTM;>Uk!nN3kbTbBRNIE1EKlJnP8DA};@uzkca%cfpTjGY(SO$Kg}Y z+7ETm7Z5xn-q_x6~|$7rX1B{EO_^B=geRD zPBcXnKl=#o)Y_Hi^S*DTXrw56=hh1sM8X9TR6N?%54ppqzNp?3Jy^lRz^%j}cC<4E z&$&YnmfmMj@ujxue&V4KkYAtfh=xjqMijeQRA|E*CM)p`fv0c)H4gGxmvz9h`qFrR=vU_R|)sc_be)NQ-Yg zk5gl~fHZTTu&%6b^v~{Rjx&<)r}ie%j6vZB3hOj)#9n*75~*Bnx4Q=ZC~foYy`E=t zfBL4Z-SzE^G?y{`$x+C?Zq7wTcPg8%t&@mg^U=9HHh` zh-xWKv8uHGlNTF{_h!|IQ$sU6XVmh3`f#$Tn2Df&KV5Nkexb4uP&`o=zw{Q5{1w2X zYF9dM#>yJm+m}5>yM|9#eZIhv&Hai;9kKBfhI0vuf=?K+6N#@d3hVLc5-|4ugh*E+P)O2Cf%Y;N=4 zI8^ffc;yJGmea;M)edH04rNp)lOVadtUk*KDKr)Tx&82ETriSMA_i4qX&`$>;_(%3 zsUKKLS}M}A^bW(esZ~>fNuhdiBw$n}-Z9Ac%mw43HvB;}{q>N;_fJ*Ka&=ye4mYDv z=oDIyjuZFa!s{Zz_c1@inWXu+9#eJ2cNvM>fPhN0V=ZTiAks(-A2)$F=P z5Yt{=xLLTXmEvN0CxV}!KkzmvH`*^=sHi!bnYj{$Lf@<2$Trig><6%Q6v_-FXp{SE z964qlu+VUypHYXUgGIol7IK!?hWh&RMdJH{@kKf?dwIB+nazkF>>clCkh2UA4JB|B z<>cil*?EHxtStHCyM*HAg`Sl>H`Ew?@>Xj+WG zbYbwm`<0FBZ+B!U3MyYA3@&4G)lUrPuKVjFf>9a|P(hQd8>4+L?WJF|cnWnV zxE0mhe5`19{bY{@{`Rm^r@5?k{8g2XR13fOO~%-{^8TN#h^t~5?WOnr5^QS$7I!&p z{P3SMENsBbI0+vv_2oBnBFF2VN5sq)=<^`;ZCPKpM-o!SJ&%5tn-5zkFth@*E;sG< z-X8 z|LmakiBo4bOZB<) zI)_g^_!@&^&_}J-=I1Bnw3i0+QR5-cLCfHuu_RmwVU%+ru(rN`9btf!UhVE){Qg~) zNBsc>%wb%7fb!{SdtLJU=xcz-r{|cn5-UvUM`fw0KlKC*(E|cYJ>n~WzAtJOi62aJ zi3c%nReCHq>r}a{(%fJ5Y_%OVy@MC}yCnm|P+UCIdB1~_DQ6}6s5eJHW7Pqv>^xEL zZtqq$TWwV{T*%Nlr!Zlu@yuitK5y`Gudv)6m8ct#%~pn6Ze%%YAh!e+2Nn|+{gs9S zk0m8g{<<==Tw7XI z=CskGq!Fn#(Zv*;`s|AF^ovd7hKVA#akueJr_alh$KCwIFakaRp3QAxFW{Ev@+TvJ zN4I{_b(|wYw6wn73Xvz7@qVFgaS;fEpceJde|Nf49jE#Hxy$B{kG`yFb6nEo4$pRF zsl$IM=Lzuhs}v1dkJnb!*HiP7vKfLK6utrKunUXoop7uG2+*zQDObUjCf<+qWLQxp zG{fVRuR?GqsLuBaqYafkrN!_NSV97qTTnq9lH87Bf;hA($|cQ&K`9DVymBrYWq_{I zlKHiJt?QikJ6RJ-BW-a+V%!1*ty{^ib6jMlk>%!k4729*{7*#%9i^Xg`9*n8%gA!; z-}5pu@TF}<;U0(5hYE!^VSKP&rvD+7v~FCYmKke$ZKoU}d9Zk4^IAe>BY8~9#bO3; zvij4B_P1mCJ4z5!rbQw9EYgp-x=0)ET;(j7geRCEZL@7?;fE;H3X;nP!)xQU5EI<` zDzE+gCxhXvKJ1FSFD-9i6bho^7I)1NX2f+UX1zO5>7bpSo-S@bJKEU^peq9dkg3Z@ zmaHbKg~Y^CpBf_C679FK80VS2Fda`9K>v0Gzqa4(?PB85s&QRd$;dLlvJi#Z)hHS!#|<=v-J&~!6#YpB9D#bjkM?^j9swe@ z1bUBJRT9(0+*T)z`USKHdE>iS_3;SaTRbMR+@?d{<~I`gP5kjEvEAyrx-eRA3$u~C z-HYCXDtBH0GKh+xSz+l`$Qopes-8B=C^vvvbq|;D!>mOC?x>yHdoUDnIMkQ*NWLFF5k&FhH${Wa+Um5vFH719&=GY-Ch7MOxVRGI zw0YDgR<|ma%9fxliC_^Y-F^srJGX~;7t{i)?G_d$jxeEjqbo@zSQV- zF!eUl?>G%wlnwa-6AMCRiv!&2eE%CyMWn+N-Fj0Q?AcMP30c-y9E%?(YvG31iSo zri*!7t|OdUZ}Yo3FWQ+4=dH2zExQZ&uET4pj#z|}uY;K5-$H&|R7`G;C{e|+2OHYr zz@7bWd`jr7S9T^E|C=_(CqMu6>KUWJsK`Wj5$Ch|;x?B3cG`na?8f+4AU`wu!l$0eE%l%fQ`;XoD}rUL(1OgR^ry>aqM4 zV2dXAaNgYfaslFx|WqUWd6){qA)pr&iPA;OrKfuaE zK;6X5EbqtSjz_yKlph4%E?n>__dxh#3#ð9X8;s4WMFPfGs2fKZI&rD*;gD-~{} zAB^nkH7@gR2f(*9PV~;5>*v;7M6bQbS$GYNDycoMlGwU#<4a(g(J}YeZx8-cc#-Rj zMEjhb*MjAzg1J%+`F_|_I%pGT?UFLw^uI1_1CovQQ6U$!P)MKZDitNJAvq{&(W?nj z%2kbE!wPesppjy+xN6(NPZix_%dkVCBF%t`G@%1qa#_M0!k=DL@P$~y9<|OMxsSdb z%VYaMxW6_fa#5UicyA_(5Y<8Gg`Pg~|hhsUOw{RLG&^A;5#CBClP zKKQl&s{ChaSkG`(6>A{2Ybb8Nce@spe!=*znRQodtUm{h=zkgGaYj`W%cQl&h_mSL z8UI+{oW4`bO2N+CVr!AeF<5$)&|$2ciKcctTuLp__ar&wf63vme_DLfRf+z`k*OTL zfS}-ZxO3fli0;8b`GUH~Z-e8>iGyVpT@8(+f{DEz-MyJ(!q1<9-=@jO4+8T$k4KKT zczLxwu4kL4?@cM51&WF5Ob}NZHq;^ahsSy{%i&6Z_(4d+N=ez=aw2%qM@MUIr~J`t znC9xjbTQ}GX8Nl?%WsZZsZYCV#DPt<{b5+l^N&BU#^EgDIy~{mP~At*G2o~js?iu6 z;A4vj=?4-xrQW0S3kQ|;Wo9*JS#$Z$}-1<^(617395vO7X zfP#@#EseOTz)-?ezbzKKS;D%ZSO>8A@N&o zby5M;UJPXV81CD%&k2qIT$*x|Ku5^<^qDgFvq?UrcU4nUl@C8H9#3Q_9d>AIL{w^q z{kB0*@B$u6SPaq^%|zkCg_`qva@+ z5`&Ss$;dinLeY|h2p01FS7~$niITKr6IC`|Q(rthZI<2Vcy+5wM?SzkaWw>Zj72(i zlfp;PGSbJD=5Zss!!+r>`lq+eh|Y5>ZP_uf#;15p(TrPO;=J~`KdLb#^Kq1+(Bg|M zThFiZnXuDA1#1mDVce{15a;Nc>sf9ms0fVNnr%Nd zmSI9Ps275j^Puj|yfS&}H#otiI!dUd%;^2uu2TjuySPPTY0u~|UYD2_a+;X9cCNW2 zKn(Y=D2qJfeY(rrtV243$3fAb?@8RhIzifTN0iPbLQ22t;&R`=>YUiF8IZ{Y;=49w zq=xuRNA8+lB}olf3;p+k4Mex+q>~q9Kr^Gl-@RP$_xtw<_noXNt&DmV5!W@tnz4Gj z%^w~OBlDk+_UDdeekI6psw8y+;F{QT>p5g`9pwAF|Jyye?^S1_k;H`A5B+Az0~8N^ zFkL$JZ+7924oA7?&ZG2{)Ko}ojui|l=fz;U;w;m}+?>F0V{gCSS6R0?Y;qzK-&DIX zQd0vKzj3!^zcUY54t@dd$)j@QU&V_jHv`V%(=5XcB{oDOQ3*L} z==fh@euE_Gl9Ke1@z1;I==rHG`ejTB&DscDrh(L_qp*QJx;OZO_!#!g6 zF?JsE0ic0&7zzh{tqh4XRYx|a_wGst#y=$3lBM!fks=J9 zS~P1tDH@cb;)#_-NldK@apE|mVGuz1{-?}c%;qmRur4}6{i$!KiJV2$X>sW1lG@1U z_g!!Ayx5iWV)8pOg$|+j=EWA`IC_5xZ&o`DC+ZYCYy8b~NIX^uNB_`NR`8+DQ?)y* z+-*WQY*#52!Fx9GVFojSu2l}ng_T|v>*~eHwveOk{PhLoeQdjZ*b{JM^@ULnDddGN z{$ab0n+#>>Dup>L<@=-%Tr>)vEqu#R%L zrJVY$2cwGP*5cS~`yS*aJ!U%RP!b0WdMHJ4qQ3tA<*)-Rmdd~)3EJ+E1noDQ zYos*y0qtK`TZ_Ej;dh>?al^yPYHN0B#P8+5qy6QR8n)4e3TFR%wu^s$sPn?%Z z!P(wkBvv-|$&a3&@c7DANcGk~j};-i_}}^*airp~^JWoTjojgkIH@acm}q#j8w{ub zx$3d{xibOOSARqQ&cjeMS{LYPDSghvL>Y8JiUU>g2r3Ek3p?`ZuLsQx!Lmz~#;avr zl$~Lq50S*vWyc^5?gYLs6U+NL2>^6$1o`!>)x}P<*W8WR;2dGFNR^eT)K6)~sbLtr zSxxU1@aHP>Q}w?pZ$!x`#bARhZ5SU3#H-xuvb#!f#Q4&-J!TNnnZDJPlv;S%zt=t# z(-nnE$YCb)YVZ6-MtP#PA_>Zp&y_M_gl!*6rb|&a#WdS7f4I+=8G{N`ewv^ZOpy^o z>u2%sv9{3$(JIhL=5s!o^)UkU z&UMkEJbz%3WRWG$T*pnH87OUwGZNeR<&&*E-|xB01YhVEFJ8bX^qi;Zi~QYWSarLv z&Mc=0JD$UKL(K1Yt!KK?*pT*9WVF za=A`{9+n+178m#zbX|T|dE|DLF2W_4n0gPSo7TL%Jbve~GtXw+TJ#DW0dpHS1K0!9 zu6_D`O={{=Ypb;0%99z%+!19}T#QRMO5zWzZ!J*XJ#PPKOLECPTH5|^3?R|q0<(QOZS6k%EP{3675KI+qN`SI9U>I&}-5&}u?+JDIF`%i+rEE2lq8e6?|yEdmt8ID~< zjS&g^6F056hekJXdl4(j~xQP@R#KeqyIpDI+24wBW^Znzor7{yfK zJ8-qw;8z^4t*b86zCwLgluOc`p0L{dt(HBA;d-3GjfZH)_1Z>AR@Qh#p)gl$KItLy zI#e0D`Q$t8NTglT=C1Fy#`AXf_JRoUC~n5D)yQAp$@$fjn_cicd{HCf`(7ni!E^qJ zuCVj=SgMt$iCeOMpNj>H*z1jx#|Z$I6w*XjH~JOtB=_{`eL%}*DLu(7R?b6ITl`rw z{X?(?r}Pd}fd0E4<2#TvKL6M9y%cwLalZVtOP~zV`OToMm23|w%5XSfAxIBPRMc5#7>y!F7I)A`q4(N z=e*~kE@5gFOjqL@If9lJpTuBTPPF*j?5tC5KSxbmUv?$BrFrZ@{nY*x%kGCM7m%?I zMn_Ya0^nN*3ZmAF7jAH`p}ThkFz$N}f4%}*q)My8@`QgSY(MM384VV8pQZc>r9X#a8t4g$(v+lm>&|@c9{pv6IuYLDgzS zcAEs;f>}x9pwq~^Y?6%u73LMjE7SpGT!-MvusUOhH4vp>`tg2U}p(F%WS|=T2C>&gl z_p-$IIMkwX-p}KDX2IEsZTr-i+ix~f;Z{dY%&@Cc&1X=P&Wy21(Z3!C4`QyT-d^++ z2X*qRndr;h7ubNapZ!@m57=x4Kf-2aW=?v=cO@kRsUWm4bGx-2OZDSx!+PrS!Jmku zKTAK<QcFHRD4tE;a0=doFJu{Vh6rg+-9)XPYJ6xuz!D`tm;cVQ~&D z*`Fr)W{4xVt`fZU+24A;A362tHI_EvhME`=?($Ub-A_Zg+X78iKTOE|z1<&Z48w%bH=_>~6wHV+qxM~eaXHw*yG z(y*#4zB(~bA!uT3Ts?MD#~2^a#NzVnl$u(@`dSDf9BEmZT+DjmpiF{zPpsSq7!T;+ z@zrqWGhW->Wr{x*9O1sIw)H0WDU80oH-b-4UQ<8;5ePNZ@Q*x0o8?v9W{Z@L(aT3~ zxD!fy`i8)n4~Y>wxhS2Y06MonUo%+Gxz*hl7z)npD7C0j0F5giVr5QxPLHzs>bA#6 z7M#?r$8XG@k5RUnH%9o+2zjF)zDMCQ**P#n6-zZbziM#Z6yEPUOEDn=Z13M*x_9k7 z1L95G9dHlOk|h}7u?!EvCE0!3S5G3wprwt0&c8K)X-MX#$-|9M`6DWj)^E#-QM4i zM6vBrJ%NNm1!7N6xp0qAcr;?`+oHrlKIS%6R(>XXNHDue>1$5Qr?M6!s%PBo?fG$k zR2^joFN9xY(6}fasa#!Nemzi&Gdjx1%tIfZJ=UK4Bah;!FG{|dDEjeg*@Yld;Z$)8 zQub3~%Wli?re%0Sb8+gDkRtSO>X_|Kh5+&if(T{lhl>qIgU5T|%dJ}Yo(E5-2_d`b z$FyX-Z_>Yv-!`^*@AOBb1m8!##ABwA+S?z!}4LD_8t- zNYvz)BhN!T=rV9m`PsI2!hSj`I+|30ONp+C^VWvv4$UzkRpg`RYJB>&3wyygU@mpD zNQ1p^VNLIw{Sxbf0?ScAby$1!XSv&N`0>6u_0m9Sk*T!mBA0?`HQe`Z;^?pAzP%El zX1O2JV+;`)hxu@|~NO$U2n zjZ$RI$ntt7fPz^3z<8(cM9ucO^s|NO&w?w0Y8r`&iOgZZqw1$21&b>yHJ+9a-cBr) z51O5<1QVhcN%xXY(ql1Z%}>fc@UxKwKZ#%aM}POg@u81c;3kdvTxF5bX9acfAyI}S zTQ)+I)SM}Ut(VG*MsoHm29N}{J?*7U308--vYql9Pto+hUV4w8RU!LfLEq}8;F1Xk zo^a}{kd=UgM0t1VCw&CmYT>2B7YHbNC-(n=0l4+UB?4yXNjQniHJ1Q|I&8tn zW(=WuTiaDIMkS9xTmtB*h1<2;#|L%dJ1YVAZ;SD7Kl0powB)p%w*T+AO|Tv)N@}=+ z%HypGkHZCB>&g)@As_mP@AyCAzm8nF(kD`o5>=$0#;mH87xot{j%xRt2o3qX6 zzn0cSCdOv$XDz~o19T89-HqsdO^iQgzrsarHtB8n5#9gQyDeK)j_B^x5bFRtpsih3 zURAZ*a#8W*x&ix_hP}jwoplSI)DFWm*R?z#ik3{UjN22%l8j+vaQN>@gWE9&&EYrR zlLO#j1|nb|tzY-!x#e}?NA0J&Nfbek`=X@ z_7!|;z~-^w5ky8@UWzjsv6%t0;3a+3O?G1OeldkUw%72sC|*Vh%!K|v*H(t8@GF@u ztw9UqKzL)QiNo8#C|EJ|)YLiaT{0OFHAH=u&7+{)7ac4R&N;DLZD(sYr>FRs4$_N8 zumL1_s?U7ZhU*#awor%A*`ib8fhw=_pJRH7M7WHGPo4E?N)@TxKZLbfq|v=(c@E#7 zOU5v_Wh)0z-+Hn2t|>;Jj!=r6MfgV=Nylbe2=gPhP7{G>5>6>kPtSGwrdItMSCIRy ztWGJl3EZ04)+SirP5zZ?e^l@G>{%Qnj?3%HfYyv8z?gsfuyb;aek(y;*L_pnV`=aR zPCq~v9sX_DEkC&$oW$%Rj=w9OxNZRH?y_Ao;lAItxx2fo=(*SbKiBG56iOeA=D@X_ z9)5GTr_9C5qgv@B&imcN$o6$)!01qV;tf8ktWKbYLxaIPVVg{z+svHo;)1&y*E+I{W))O9fhJ%A}I9`1Y2N5yO7gK+l5v&aL`)yonVW?YX*D zE0c%YWBYM};s-5LigppB+bZIlDh-D(8mcNPwsHi;)@F*{9GZB{njB5?io3(9TxLQ9 zHi7Tg+1XiG`DLLepZ^A-|KyZtINm-!0;j#lS_c7A9ub7JIrkkri^tKf37F8J=_3cU}l@hiRtZDxO&e{?xdA*cxAcJMU1iFvc05n8_CfUqV0#O|2Tcy^M&kkxkQ+ndd5RNU;eG?kzrA~1-v_u(VZx_CSc@%lnT%N|*PeP-C^OD*d0|+)pRYUB_MCmM7S@#W6>>(BPltiXo^dWMJRwK5M}s z9r$&3cv#_M-Rp(B#C4kc4im9A)!X7U&5Ih_+Z~s3F$~FViMONOoiO5#bpA4QgrI)Z&b&?F$z!`=2@+ zRvdLNoL$zey`xI2a5%rF5EKFz(Kf_+WD#09bg7(~aqc{bZ)pFc32uY0_Hi4X8%dWn(xG1DsOX(-3 zofTtYYOY{ANlH96=A95Hm5+;#?>Jkyfww`|dL3E@t~oj}einBj*3i@U<$3$%>dd@@ zU3-(Gf6Mh>N5@|O8t3=ik1c|NlI@1$L(s#SC~>gHJ6a;tyAIArmtqDNHh(LNeibcRa~^BpDqGGC~Q`<8KfbV59B+e$Rsk{O)em(vs!L zav1t=o~d2pp;G(t@dq3m;M43eye$EpvhgT^;3yBzp@j>g#G7+Z>X%0fvzYfBO4`@J zdSo!tIk*14iK7rL}r30{%o3J6d7Y5Re%&O4lH&T$io+I1V?qufi%6qkE7R!(Hne8FSGk*;H#tj z9SSymTA_*yinka)s%d=Nh+%n1VK8*fe1sF&nc3$*wdk#BmmWDLDOQUM3vu&uy;;3t z5K-@hoz%N+s4=i!i=`l;LivJ&FceOxhm>cZ^GW8ISy@?W#cJn_0O8kk5X-aWm6WMjjFz^2(>Ad5q?*BJ# zJN7zpl&qvfDk~i#oQ#gWZ-k8OJ+re{R!FuGLS)bEtPmw3*(=$EtoprtfBkXaf83Am zI>+bpe!s?bJ+G9x`?v%xZBm>-z*N2l$kV`H?IK=PUq5bBgJN8AVIC1)DSW9>^c6hp zr>CbmdGh{eV^7bX{Qmy#0c~wj96kLpk(u^l__CUMHNTTUcgQeTgl85M648lnVmA=jrm2b>(D;UFfLKcp z$3dxpj!fj@#3`BSF;ay7C`});-7oe1_6pZq@%zJ0=Ox>^nZ2yT3|J6hSa!u9@xEZ` z`3v6tjX2*w_s^yHClzsi>xz&5m^K?hox3uW-RgIuf|SiZ8nB!`nm+V}3pFb}y~+1z z_sbV-jn?0LVCAckc-FCPG66!ny?egw9xZ#HU6H( z^uamEioVSVj~kb;Q=a|hF*018WgTBAaZ;w7WMj4cRZtuI&)eL#q_Xy;tzIsW3&M`* zkg$=BT1b&jS-^DiV}~4T^jo$@&MK1^Fw|zl^>fY4`LrrXZS9-~S3>H`C>D(pxFDDj zeP19~;3_(nRv8^Uj-f0Ur0FZgOZkn+DX}Y3!5t3-e3;PG6oLl{^AL zXD-@*7Svm+ttSs9_k<+(2Oa%EMW58)1uD}r7EtOS3OBYcrLtfINWXuuNe_`z*= zp5mRn`9uz^5!f2~xVQefxF3z{LyPmjvlW4E)q8(EOGe+XWcgrXVVba_uq3QIv(|VqOuBHS z&tjLdv%XA`2mCZ?HsH$vjBgi+;JDSrGB*!=uDHH$Xjt_uvrfz9>>M8c`V~3k^379; zfxs}@qHkd*#Wdb#&P{BPOdRtzw#Q68DW}#_k8$xf-PGRG2l*OH8qAtV5_~&0A<8Q+ zRIZTtm`EY!yjWeq6_=Ka9ZBKBv23-+?CVpBw325ajtFm>Nk^xZg6QTk|AF0I@-sWJ z{0Bi>UE;ySS+Z`~ud=`9zPqnLLxKt+n)=l@gmc~UtP*Nu#C&`)= zO5qIg%wi(uSbsc$jy*R*4f5}yf+v^nEDTYxYE3Ba5FpvO*?2(V0jYt z2DYZ~eOlVvQ<hDkS@zw+%w9^QPJhyFf2{TE!7An*0t|1}C9VWZp( zoDUIob57%1hjR?4$78LZ_D&zp9Dh?O+yh9sb(+WSBE&91>wLIgH3T9GUoWo-;lY@g z7*N9SL{^4e){PJOn5hWGp6veVZ?KejRxTmt*UfR*`g-;&?-PC3+# ztcIBU#84wUqjK9Ql`B%-CG{(H2$1!t1?CaRdnRx5`G#)W4b0Py;62BCPCF7IE%1yP z>x{KRCi3U1M>BMA@bU90(%na&wv)A!NjnG4e_%})@BCHVZOR>Kn#>-S+qQ2i`6|ST z?|LN*SfWg_Oo)>yj9p?kia3)O zb*5*KZ1>N}xb5YRI1QQ_t4Z(S^?Z7^A%IB5Azq;B)x)0yrAl~&2q$m~BwVCH30VU--UslM|~ggMBv zM5rnKuA~3#^N}8v%)4IQ@zvp*q7B~r7lziCD2v8^Xv=gT@+ov+ImR|T?lwD!C60Va zW#`c6mKV*D6uIN!QGsp&E^wR6Nr9PcUvC_V(NB%;`={O|Pny2XM+fPrv@}nu3&~!? zuI?sX`ixhx)>MGsKaYmgB;au96O$u)inc2nI)Hz(xmU#Dg z`8q}MD>i+x99z9fvdU2MTi2h6+#%OAXQA1Y@Ov{;+>qz2|LI4)r{#jhNyWSOf1XL6 zT>rOEb5EIba|LkL{0woWBo;Tiwh4Qp{R$`F7I7Z-5+1g$J`b-I8GBOq1cEo%AHa^f zR3(n#asN)CCDW#uYaM&h`!~%?_FMOSfB%k2PmfvV&rH?V7yY~%bNQA{wOC50it%y9 z3*&6MT!unbVw{_$pS?KiT5)j{y$XWi>PTopuS-p-y(0RsxA;0&)mgT;0s<7n2^p{QI!`eokC?6;boIvF(5VCW$_A z%=FuD7#n4#JX3L5KlE)IyJ}QD^sFraAkUyK$R6rEgjaW*&NdBnvSVe2c}X4r2p~`N zAm-8?h-KP^LjV+Ypnt+7_Py39YBw%#Y4P9u(E#?^LU5>x?|cBULsXY3gt7g?tKHaWw@OP^DO|vS*fgLIZ|2a1-vz@;hvggj z?0$JzD9j`m{Y99t`%BLiTYgl4Bc$mCp`9~myCbdBrv8?X{*qDuy5+VI@%zN@{s{1H zxSYOO4REe|vGRG{p(JFTai0Pn*`13q&=#njVhe#NcmAukR)_b6E~8???~h$C&)3L4 zEfleQ(5L(SJzAj%{qpte02m;~9kCbExX!h;@Jm};!sh_)EdX5fuk2k@%|a5*fzuZz zb>IUYkUZ^_%#eKAqK1B3S{iDCI0^XgzyF|M!OMlR=VJWdzc-?C;g|_r6SS9Prc_ zOXU&C^sGyXJ6Hg8l`PcZxoDk|5z5xqIM(mt9#07;BFP=`VfS};>3XTo%7)|9mAf-X4tmL;+wXzzS@LW|Qo>^$g<%(HxwC#~)(Itr!Z_QT25*3z1;^ z`lKAVpa7p##V0;=(sBzykU>5BUIxKT9o}qo3zy(fm)^g0)1cl7s%5y^SF)e_ySv}c zRFVAqp)MOPdO&9H#0n=}}hqvVp8z}^OPkF7%}6Cp$b#=$>+{D3Mw z%kMyM%-wHh%T27fxa04ks_#4lv`1j`27z~>YIZm3-Me?!qX?J;-h|Go;dvx!@Z#uy zT3>sqfC>ZO?oBrEmq%LV?KuH3W4q;Uw^1aIEXsy(+Z>?cLZPgAZ2G(j!=XR z??MDEmcQp?B|+y;m}=o7!fRc=prHmcD&6E8jL#qyxE_0 ziQ2iA0mcABEyP>@s)m!#g()mxdlvbW4LkL{ zp;pjQ!fRKOyi`=gBTWm#=EavWxrh&mvtdbnkH2q-#H;gM=@j&>ns^WP!EYjWswNET zCT}47@pc6GXpD?uRTLul-dOZrzAz$a&WVplEVq905{Wb@32niz)_pOqDr|b|u<9t6 zuTL`+RxW%(P+v%W2xO-=P`Wm(pQ8p*P(P-oLMw_!N43LDVp&9YVb6N$-$v`%)3trr z$jH5WG7pnNXZq8>TmSy5{`)HFwRu!{DJBhVYfQ)1Q&C(DpE(q3Vq?RbsnWS9bMs}B z-^qcMFhSV%&(C?a_D>J0vTsNK?qew2gbk?{FCm{ucgl)htQjWDTyNdWsCDLv65BX| zT6HH$NaPsnn@mtZ^PLS?RL#00L-D!4R(5kmQg#MxWz&8K9ozkhtV`U^{H*4hs|^jr z%9^#cwPC^1H(&DncU>Ly6D%Rp^gYtFcEd<}0H^)8f;U6KjxQ9>}(Up=|L-umPXhfM1b8+P^ZG?=E^ zWBbkzZo9>Hf(!m7TV6h9L>>0?jWi-RoP0S$K#9440kXRWxlj{i@PP^sKy6$5-nnRO z4uLBK(@Q11mdi7l6iKRt8|a0CAuK=(pB@*&%=!C9ZP{AcTJ5C0(?XT~35>uYGD@6v zXkY+@Lcjny54_{SBt1wVVz1vifmQCE-U6Ve97@D2m+NH^sdDm3&1` zk-tgv9O|A4^lxJwF%WALyYTgw{n)d|U9-gblb807%_d!3C$QWvbj)REJsE|BIdI54 zj42QPkYGv-XGmKZk6!Y%2{}NZX#;r|W({~QTp>V&=dT6<%XaqPRkqd*d&kv%@AFCh z{QNxJBea~&ScoPh<@g}1 ztl-3*$a8@_kPYTkB+>vMZ7>K)Y#LdWN=E_|3d8rZ|IUAoupQGVf=FI{J|Gt|0r-Kx z0ML2x*I+Q**Ho2&cJ{Ep(sQnZH~%-hZ-AfbSYAFRq!BM~YMNTJ%fdK!>=CFZQC%_D zqvnXvP~y=++Kbl`Nf(b^1aYA$D%ND!33HJ;+*?fh-58aG;k&U5%I++2OGIPtWYU!d zK3V_0yFDBAS)=RKh-!G`28r}f%_uTy%5H=JnED%=O)D_0R3DF6uHqE5!#Z==5e*aK zO7)>87?j&==__Ps5J$WmSOww11j3O~lgEw3KJAjx(b2n(-g}*`XW9O+aoXv9e!tdi zy2Z~24E4glrW#x=eb)Y5iF?1=@uZAUo>bVlLN`Mgd*sXN!j46d1MkS#^CF{d-Jj$I zPfblB6BNM*F+2ZDf|7hpMj{j^c#?~VH4###YdziCRO$9dJR zQJ0T53U)=(?^B1gn4LHSd6^%d#6NFwg2Dfok-uD=o5^b8E54=*=y zoU&)%U!$;!DO*#k6Qayp6Dz;y^aFl-Jp5_u^gU$Gmua}(CAA|2%GS~fE{Ab+^&Vn9 z^S;%IJe;ELuN>BsdL5Yy7`wKE97yat@zeDp2=;*@#oxbWW(!^bb8%MqQOGl1) ze#t0#PyoD{KktsyQ@>(SdWV1MZ}+96djz+4&jd}d+^Mu?N{3^%u6&9#YOxG0&)&DE zUNVb5aCiAbb&Qb(k2TKpB9Sx1@u-hl9B*xz-3`SQg|7{2mraYJ?bH>8eWkp26&Ft(RJ8V;%2+T{KvgE4U6 zfb8pqf;8QWAtqV%Wuni{5Vf7lk3;W>htGL}lwBo)zGFA2403Dk8X@wAP|qF4M1)Iq z`EK0cBgz_iL%QT5`a8Htp<&bp^itHMZN=cz!`NT$(KEV7WHs%CIq;Lr9(q`?aB}ax zo!%JS7jc*G?m$^|>qZ$Ri9|%8qvYT59wNEaX~P!9OFL@MFZDmaLHx5oN&q_N3FL&DAqQ;pihkpAv2~~$;TNYK9opAXZKb3 z2ITe55koG59wQ4UF5&fvwUsY|BNxpXB9^Igvh1#o1e&h>FJ+A-#ENsvK?^fLczugkv)B#AW)(b9(-YYbN5v)8u= zUm@j5hrf=QcLfhoJR;I(u}hnM@wjRN^0z~1hxWELgiF$D5P7)-F^Ik2we4rfj!X{ujBqb$RPNt#P?cn!3-Eq9dH*o95jh5eE zo;8+%$+8Hn7Z5aV(BuuNDhgh|#y>M083=4&NuCqcak~5{SxfdCx}9=4neo4Yib28C z=Vm?O3(q&i(rk=Bh&;@+Xv!R1M#Uy_bveKP>l8qzNLG7;NuA_la)l^}kJW)6^(8Fu z0p(U4=Y|&rw9;R0KWBpKupi_k8VaJZcU_k$5$7z&I93VRu}I%QTsM!sd>0w|)1?buEZWI@{)Yev1S9b_@tlT?nU_1NF1_5@V*f}tA0OfK1 z*RNtwo7;Y~5{5QjBz@!X1gi7&^z>j40Ynh=O<=wOdD_PJ8i=Fluar@TQRnQtz1q6w zn1%Yza2iWNW~+>-F3#7t{wPDH_Penr_ccbMImKJeL}R1yPhk`#tTAY@a4Vhp&s4q* z4J95IB=99HM>dv`O4Igys5R`3$ut@}Bmm8|vMj>fwIRJuM|v95T3`O&rGAVJbT@FI z!!l5#2z0j`n4Sx=IY~kDzWk`KYwoj#{N3eGz+vpU#IVTrF;OnhdF7KBDH-?)$```1 zbU}fv0ov3FvI&qYy2heb~a72*!A8(|Sr2ei2}cZaY#GBakR zB-IU>Dx_w7{VrlJ1btrtHzC1d= z*1qoH)X8SkknmvkpU7+gZN3ad_>@Lamp4)n~1qVEhij){Ts3#zRI$TCb+7_|~4wvjROP{;D=|5yrQdC=og8Nc*jC z9Jp>W;?n>@zHW>XB3ElxV5mQVoi?F27jvt!%qH;fh|ao)Ydk6`RCxEwk+S6q780TN%4ae`^hDcz}+BxcbhfNaEy*X*8SQ6=Q5 z6n63tz$S98GaRsgpmcrrsP*36yA(~8f#bR<*|#ap#98mgUW?!a(Yt1Wb=nqbsgA9= z!^P=eTsb};?qm6brQVdH72ZjLJ4xo<7)ZI(3Z~vWlQQ;dC@ALB_c()+8yvP!fMQE^ znn3m4;~Y;v%M_;zGDD_7@!*{-CQOfW68+4fU(b@%H>y>{M{I-usyj1eh6{y??Lu9N z=E*@`84yIY>pVL!tndIPt43A&l3*BjA!tDqF669K@J6Z zFkTrf@=ybK7vWR}(dgJ3-%thLq(dD?;KO8y1^DtM0We*zu1R0Og-Q5H^o|e(icd>A z{tF-k0b&6HaN}N>;19n9J8;7)!8yf)FpDmVIi)j!i;Ux|m9UmJ@wSJhWskFgo(?9~ zu@)xq50kZmQY@-)2r$?z5JA$&GA4>MYo(K&-b?O3nUsk8h@ymEH9?URP8tPSqbM%f zvB!ayddckMb|Gtx5SOQ|J^Or)Jh&hEnwZ~j#G&py|9a`;3qWAc96s^zpuE!TV#SmB zaT;#n<9%Dpp)6qj#>knOn9TKE%lf7TNWz{h&-0;(d@Rcm7}kD+{CVkUCHoOT_n=XZ zWVi-bH28Hk8`sMGTc7UC%Xacpvo**gmoTwcaj^2muqY;&P9Dt9o4y;njwF|!jT!vB zylipvrLD-NUqR3N9@Anw=bSoJm1z@;<>!y z+E5mKbw=DA5FA)eLUfaXc(r{!#7L1tN|wCLkyfa0`tjP2T&1Sl=#UyIMZywIi9=(k}ot8 z+TPx8b+sNwe*2?F@8o$hY2%X3wRvBdR)EgIcrjE1EOt+cU5;*?7cifEkmx$KAaXMcFgl zr^-bhl`ErO4ZxwnWV%oK0||vzbu7RmUHLHfAufO z{ie^tQ_3qijie6@k7+JDJx&WDl_S`Pf}W;!&jHMk@1!g5fm8R|}MyR#-Jr4^e=MxX~ecZQo7oGRf!p-6-}5 zXRxV4_qlNc_!;MdqJjeW!bH-Wz4w0&3=9}PnF7Hs(7M?jf1jLz7!lBKWYmzAio{U< ziLgnl1w{;N+@%V_BG z;CJ!~%CWndMku`ogHMNN@b}Uw86{=&E&pr<3Z0N)vp5No#J8#v`&tbKS=re$Q&SrU zVa3HD>Y1$7mJPNBL$=naEZm=7xC1}7UspngcNTi)il{mRQr}^7_)(Nn6x4>mhMutd<__D-{-Dyt zwvuAbetK>s2ayaOGK+IaC?HhDisTQ?1LgRlYg~U9?~N1T84In_U*kJn^YrqS()8H= zn{{EwfA%VlU}E=#cvAZ2BKz%*J#nonlvyXgU@9P^XCEONE~0o}MYlWS{kF|i316s( zs|8l8GlxQ{(dZ}`S%a!kEj@5_3syI?)rscJoZHYM)$pyQT4v#|)BwX70+bsX8X)lr z5;H#1K&+gr3WWxm+m;E6p&qe0%EKh{d2z&=3-1(uz+u_Kn^WO5c^Ip zOptTYN6{8<63P@(lafWW^cJ;cdsj4^S(f)E>~1oJqD>sMxG|EN znHsm?2|Ih?>mUsg{EbB|SX9AZo{U}ywoe7^{LxqBV;A?l#+@lrZd8LlMPj>+u*;OL zECVW0Y1)7!(t;L^BJemzS8SaalX$IV zZcU8eSM8}XMhQ)7+=qNat(%NL>#Biu8Ga+h88lBDWJ6%xp@}thX9>UH8Sx-OxAD7H+!m5FIX*7#_Oo~HzI--(riC8Wj$2?>z!N$>S8esTp7x)M`H7{` zNOJfY;$2fwRxW+_E?e@ce_buf{!>(~ur}#8KpWUQIJmIDuIIM$9AC1zz}G&hrw@~f z1BteH*Yjpv73b<*8X#D`{29gdei$|tZ){SAcV3aFR#(`96)9NI_Y1iCyG#XC)T*Xp zcm$)9a*Fc&nU0mDX>|wG(zeCp1SsB;oc@>cp>r;tRt5p}d_W%CqI_DxjyRw+rL-u8 ze!>h8sO6l=z>5QMp$A<;pb081cN6zvJ|#Hkp%fjvB0Ux_E(n`6F%yBOiw-CDl@h1Q zQKIoIQ|_?X%03A~y-^g%*&cj6^x`g^j6+9l72l=j+iYYVTdDM3Su48uDz_QU^fZ7j zI(ver(kLyZO*9W#-#8@fQU5yOP0Yg10&2J}6gXP1qz|^hdZuhGOVo84YJ4pCO5DV9 z+pv?OcklYZjR13@=W)J^fQ-KQ9+W816XsiAM9Wr9tY@k)c+(8O0-U2ZdM*s1$0Vk;S8{kb>s) z57bP)9|Q2(+g|6QfyfcC7SO>+&KPDwIi>q(CL=8kUl`xbi0PJ*xa;!3xhn)-Q%tOF z%|*MLdWMF?N)+W~JL4arY{s_ZeG|41flNet)Grua3_XEdGaX>KCd154dV zjm5O1`fFT&)~l@v_vDq!5A=6B=-Fd1J|zB z{p@Iib;5X`v1`3T$xLx4flv09l?~P5Z6gT|6#WGx)K|;+EP=|A=Of@>NUnQQ9GoH1 z)e61S-2xBnj5kSKTp<)-asdxL=k=NYv1h+r-t(>S;^IoaA^!3x+D4rZ{02YelQc-G zA~@67u{=s6Vg5v`7dHO;7XONS|^cN*j($g zmi2JvkefM3%`j;`bFYt;WU^cwxDd2-Qen!RXqG%>Ec3S*EAB@v4K3<7brgW!p~Xr? z5^7Kr-~ZI|ba%ceFLps3gjM~DiYqgxYK0?Sa}z7ON2dob6c^^eZgYEWf7NB8W@B~L z`kIBD2bXojyVK4E6WX7REwy@ke)IL;e*doCk}X?^3tKLq0r*gZ%PRR5L?9FL*{DTT zj)>3H^)WD^bX2}9dqhL0H#zYb{D#}J-}%9Ime&z(KM4wlFm74(*AWp5DCHh-`7g8Q zgRH(&$pDS2`F=yQOeZo zdWy%IF4mhoEiWy$c_&FtkZ|kgh|r$2lxERa*mlt^f6ttgIOKb|+Z`rqb;pCw97Y@} znV2jPRUOSAY^oGa9_C>zevY;7(*XX1X6kIaVQN}MM~e)n`@t+I?k$Vn!-wXgZ%eQu zS4dkrjO(-#%taq0+~a5-9c9A3mGeiEI-|mzLEqe~IpEMl!bPM!-?r~2%Ov`I<+c%o zTkahmIK3J4@NsZZNm@#Y*Wv~Th9ke!iDwgg6sH20Qu&9&%R-TQV-(5MmzTZ#3n%u? zgjptuOr+^G8|OUBBv?yW)*2yANKMRzC4Bxv#kvxABFvI|L%=l}hcqEsde8muoU&d6hQgrF-cLnDNEA9TUq6M!#YCp-tNqw626QG)U zS-Lx5n`N4y_1RW0s&C9(FDt?m6_$hQa%LfM234bZRhY@wwSBIr|B{oTFS=Xmf$T!$ zcr@*`5z>58QU$;pVe(i|IR*O<;9CP2!=a9jlci^4a}%0wixO616JGZ}cW1l=$RTRw zZsx(jS2v_AfrQRTXI=-BQKT?lg&lTnFOKD9+q+OW?e#(hhk!7rMpq2f2SRS zjGnMZlJ{2<+>$;b@?+{D<4?p_#H4|o4HME34(uCiIixJa!f7nJi6nk-GqZ)6eWt#j zzdv0(L8{rymqX;aFrVx4*=#w{e7f5c=fB+ve6&*TS2pim>Catj zT+%o{Lu+0p>(F9L>VCc2B`#+j*NWGG>OY_gzZ*sX{K2Eq5?K%$=_Xc+CV!*4GyPG4 zhJ#3XnKcO2SfrF@0z(64Ih+bDP&I1wt)5~bnH5bf)lVEqdV$T&q3x{g{TCT+d7e-n znaSlGmVhKeZFmD?S31}^+at{NP!rHY^GMW`m&f-n8FP@b=khPB9@t-RDVZc;zp(XA zgVWI|cH%mQ2K|23HgFi{sq|%b@t*i@$WrzxpH>xVFsvORCxzqaaNk?{Blm@;F!Pb*dX`i97Qhq@c(FyeyLbaz8N z)svRG=4LWNpZ9ICazMDht(=$i!8w{ajf=D2R2@igr{zLl+?e&fqOB9k!5zj^3KL1O z(!NV?to4zyUD#9&@&wt*+8l2pCzLUs)W$iRwyKIy^P*b%#{SSrvjobr9gj}w?9%}7 zW|(>^)~G=gVK-R@#KD3BmCQz4KUY9c>uG-}TCdCYH5IQ_$r5uv+Fi<%hq!@dHWc*< zz>w4JdzkOWx}JVeSBNy%R_s!s^)9oeAfQQ-!1@&4h$rW##3wE;E`|ypWfo8)#nR~v z)YFa^wqw$0LOTwu)lk(foKyjgn5DM1^qr33=0Ktl)T|Jm19;eJxf&giS1DhG@der2 z1PphmN;BYhXq6{Y2I0{%jgMY&qI_Tk(e$iXWEW_JQ_XbS4bd;r`7kx4>~s^GvJ(EX zDopfx>txolbj`bUhP0PAij&&ZG3X_Wk~Hbww6qQNW!<%t?zgx5{I@lC{$A`1-k4w8 zU0n0`*yW@2_v>w^m|*xuOOB@zgkF3=KuC=0d^LRVjqzDSXaDsCc3_B&9q{J6@Mglj z?fvh>A07#iK;KO4LskMe)eB!<8YZd=tuvCp= zOQy9tC9B=h=DQ2dl2K<|*c6T^-YEKO#em8ui+q5xP3v}6v*Mx*7R`zSk_Sv0xp3R! zj^eDvX+)Itd>K&-x}!3k!x}t(n03VjurGD;bA|*76!F;tlD|Y~(83a;Rc&ocG>UYS zuj7iX#VX&z=vs-JP4@X3Nn*u+sQ=igdz?)08Nt$)kOCdgOrv{=E+3RAK?6z8A(6vq z78JBGF;ruF5E7tbUuR_7A!{Am6PD9@y8Fa`n{>u!e~D>jW#w)r$YoQ~%2Hqsx&TW< z-EW@4tSq7~k8N%HoqxDklXHjR_u+9 zRi19xK8nnHJU0wxK=Lb(G#?j7tVXkXiu9`|E9x@tkTKGBW4sn~Tn8;&Eoyi7-}&y3 zSLBV_@Vr!JZiM@#(t4^|E`+PfReNafG$27YtvCrXwm7okT-$T4isFQA$daPDEIl9; za!L?6_{wOGD9(?SZ;Q*Hz1$VggPouvbk`tq&L=VD#-jU8ljjy9B6z%z>EhSO-+yq; z<>D2;Ci1J)BCol!vZErX_~*jTc!h+nn9nToegc=dU@GJlzyPX(0Ph&~C6#ZJ`^{bL zm=wCe{%6?aVQ*%(*T7`dYf)AC_Q1RO9KjB?dqOd&C1sQM8Z!7475gbd;tlTIx2i5L zcUt{&H9qItT2oYRIi)HzK>sp zS&9JPS!tKHd~g7YCb6F;_}X}Ow+}8OuLh(D5jEzCuiAcIizDBM6 zEtSF9GkjTg|KqcJR-*VRJAT>eR`;$65C-l@XwJ8P+&c+HYrRMekvshFZSzfo&wnvp zAscOuy*@LU1Fw7n2j?Y_wqKZ3#(P8Jxj2w{WM-)<`H$6~6Mw##ESCRgI=8jQ8Vwz)ZEIn2y4>eQ1R!aLYZda45r# zW#wyM-oj43?#C}&!|oDMQ~Zd>ku<)A20V3>_ET!7QmL7ljIu2nJOY&T@UNl|xN!$S zPPG3~6-k%nisv$-We%aWAcA06?Rl^1y#xI#xCB&~nI3EzL6BwV#?zJeJXOUqywkZz z3rC+Y#I%4>DXfa$Y@E7hFtg-x;hqu7G2d;dksk8YMMM~xlA!_a4TDAp6Dz;0t24SY z{#E!4*ul`|1hL!O+Yf#kx-9A)VUprd0%vmC;PKgEc40_{sO#~0r`9=Za1(M{bI|~1 zu|Rb@zh7OQP$|3?RayM2!1|@)NR2*K?p8MKod^mIa+L6J@<1Ort%fyRB>?MoTK|U2uq}SF2sij$VKo|&ZFKIj|NSDv@#xVxoERp{5oZ-`m#^ z&Ma|CBYWb~3A9+Ud?Jo00n+l1GGE9f2;WEC=^i-jQl#d>xxHGORJ1yI!Yzrg-j0cT zS}vUFD;ct26eTw!L7XB{Lz=F3QI<{?VJ1fl10GM6067E)C=2aIF2=|;+5k*jJK3^z zzqH3(@BLQRNtXXIZmeed-~Fu(kI4x;{TH&fKU6J$51+%v*v9$f^+YerZCd@f>->8h zUJ20i`7MnqrU3{h!ZJl6Kbo00(w8H93G<1++C`Kt>}Geh7BBLdm-dURxZi$~iOLl$ z9?i6$>|RO0|8C7qG}eq?k;hz*(xxdy2F3A!5z!0$47&EH*M9hmHHWAlB_4sR%+gCK#5gHn^4UxDBjnHcjJ4VSF7yI8<; zAo^AUdi#2^BJ5r8D|1r%@5Fq4BU$gVI`6v=5&;G*U;i(UNv9ov3i{sL@@iSA=j$ht zMiFsvd}X=2q2l8b7|2DT@oj#d6Z=M&4`AyNx#JFX(u9fy{JxQSV)Dyt-B^1t>#V6D}O|-pKX7UQ3 zBe<{f$Ljv8vk^)^OH28YYPXx$5d`re@VBiXxiYNS_Un^ zosoRms+kL?4iFfyy+f`dxN@kN<@9>a?=S7fVfYVQ+p%U;!8=ic1HsGReQ-X-Vvr1T zft`=U73?IsXmkSk-H!M>bDTn*P^ya%a8r+G@H1c6NkeWn<>^2*@ zz9@)V(BhR%jIea3C+0C6WVx%tbNS%7F<0fkLd*a}MBkCnbR^lP`#?7#nPoboy4%3n z9ld}#^1VknKj(xBn{&cg@UPVC6lbiulNJ|$vsKPo3GXy192wQR5|!(F@9E+;oM2E) zPHaK+Ti^iRU7ESq&lXS?ccJ%(q(L29n5U0W&KzprA6$6`?EaMWbi(_>ddbpWAX&Fa zTZWx#=zlG)f3{6zUwAxu(($ujSY2efopDMwHj=BR;Zoc{x5AK0UmWBbVD2x1l% za|=&&e)xpW*c#TQ5(@eIFln+}o0;NhS1Rz_!M7A{bS&}acR2hwhB@ApyylW2Va?Wo z%<;n`6XNUSFIz)50=yd81KO-|LUyG9klcHjnZcgp{fgn1*n5}7F<5k~&OG^4$ro)6 zs_VSAT6^UY@yQDNCM`5|m>>VY+dZ8&`B=n+{X_n&A%03ZG}L+;JoH_F;EDRM=Q~w< zTlrasS~8bs9()$~FGzBR3xd7oLF{mW2h>u%DuuI1kCKE)n^iCx<#Np%A- zdzeWcT0l9G3#cSD`k7p~&DJdrr!)c<0+K~X;IJM4wvNZpkOUim`VYdkkd)-$e}F8y zi!nAe-LO6!u!n%I;E?o2)JCG z{W%i*GQ^w{U{$)rxV-$x>*xq-N@!)F^O&x8wuQwmAkY>upaLt2`cy&CA0|EoIaL=^ z8?)v;KY`MM2kZ%XpMOFW74#h}J-`jx?)d4^1Oroo0kgVkJZpcXY_fUa9cSnYEZ)y- z0dO?$%j{s+t^sc*SrEH|>DVDsnJPD)9fU6S`o(Xz`5*)T7Wu+A>9Sb68-%#O86W^x5KG+pcFCfNTsOc4C^wQqkYY3j8WDB_eNRx^+A{jUIah*m=ium?~o+`~BkCCy2C{m%t*BX9i zHnNWO?ap15d_68|B1%S&bJVu;hMhAoI4+!yFrbg+oA4U#_KkU!Jx^i)bFZY~N{2?@D*f3NHY>-y3tFsxeEx=i3w ziVEHR@!PKZ!J*F-f3&Rn&Bfguhk#~iCxh7AMV?VG-Aw8??{Nl=JnTo^Bt}==w6w1g zr4}gj)Yy+zW>koNdhAlR=9&>8z!qlmu~(`qWpJ4^IMBXf$tBhNi2pZyfw06&1gX$Gm8~0pu*=G`Q~)x9yN6NxZU>jR50bA8MkuxxBo* zp`mEayJVE$2o=n)8|3nMXO1U+Bde!AT*PM;A5+l%w7_!&vQAUfwR3w8S*2?w0E-kF;}PC)}5Z{DCbKqy%aOP%-cDao(JzmVSaTYMf4Bj!2I za1BPrTz*PIF_c=O5T#ku&7=bFk10+Z4F=Tfr+Ak`1!SF$(*_;{*xUhLg^Q>Iz5j5b z-w+DpnI5mACWYngIboUQ!l|@mPaSHezPj!9s5WdGZ&YcM&6lWp%e;5phiL0UV}xn6 z#LzQH^$fE_y=l^#yZVj6Cdivk0i{^zxw) zx#Gpe$$8CNdVK3ZSXMb-L!CD2J-f~BriZ^F(TwHw124m(Z?%l}SF6LLSZhv&* zFViUz4<|WuhK)35WpneOGd6F!1J(Jt$&y*@qKrC`i*kIJxgg}Exf@!dEP_f2=SeRh zDN9vRkM)wNu7GXl-58;Xm=nrb3BFW4+0}iqRNu1C4(8%Ru7sXqV;<;ylb`bHNaA)+ zj8bmI2lXN?KznJlO2UyByI}fZM?8+MU(lELD#)di6s|@)BZ(8ZjFn)a-^pKK4T_(i z6R-(DdV;{h!vn12+in16fb8t-lT)U7zpY#7toB2)xA)t=HB4?`V?gR`Jy~4L872+e zObBd|`h87ZT);u|YU+yKP_4p|LCFYdJ02xt@Ppfleds8HA&N<}_H1en;h*l@i}&78 z2aL9zolKnu+RWrH|*%lVs7(N&{hXm22^(y9lF4y9FTy=zwI&OYm@Brn!^d(6i zh0_v^tx$S646&I1^F~nx;Y@{X5vWd*oZ8 z?zmJ5gT-({p~Sw&f)lCg50rSw=-*k$T) z7)a7z$XdZCP}iPvH)J%E{ajn?p_{x#MQUu{Z7~kwWUfdq9KF|crP9nHGYOlhTfNTK zW1%iZrgwk0X+XvMac|yJoJc=(<>>FL^uc_g37|L9U!AV(i8Xuo_U(@rzoySt8RA}* z5LRSo7ht&vhr`fbRBrX8ef*!^N(0^(#ww4rN6(A2;^MNmZ$aV^Li zI+6@3TQw;;N%e>MB|7~?lSK%PwgxYtjK_^F6 zPJxRaji)`TZ8$Qbc+)=JR%r3dX#7N8}o61F;XK};oGTR|MvG-Hf%L&pwZuN9F<1opIXPh-eVTvclzDtMB{Nbjm)F&x6~)TRO2H8u zt3ug{7y5B8qom}{zBh%upc&;a4Rp?2_Kcl~fD;t2xvZVWHTw2d6S~Qeuu4U=o}}@$ zuSSYQI=+AhFV4HU<=p*ljT$!@u)rGyn zRtS0RURd4Pf$H?ak3C^!#VV8vTX1{(WZi!de#^~Go0B+2;PECfHZBhAd5K*3LL|b` zN50cmHEgV`{LkKTZOrn|#_{&kuQR;4jE z-5+uyyWG=ILEhoVRC_ja{Ho;jUG9+Y&fg9y26@*dWA7jTRjNE@`j54;ti`ooS{Lg= z-{p*wi(LVgg+tKJ$+WT zY*PVR4n9h=em4?QdGo$JbSyp=@SS|6JqXr1{8qvys{igu>s!w*r$Y z@7+ZK4@wEIT^R&^g1ol3XDXxr zNBbI5CR5NMc@1j$pjdbuo9?8CG35#sa`(sScCY~|f+T7YY z^LD%k)ij0FE!}^VhKOS+;gACBO$|C(f)F>Ajkg2hL!A{?Lrr8Sl7_-T=Hb&cJ@mf}r5bJGmcxwmnpm2$RZ4`L9({8Nwr4uB z89IHL(CpkrU7Sn4Vs0TOGEI%Hc}09Q*?Kmqdhh5lNTQ3?#T}`a=nskxU_t6<&WqnKFya2p(e#ioFO%nhLhD__%;tu%sFlYRsLB*^4?K|%(oAT1!>-Q6IP3In8jbdGLBKsqK)x}{sX-rJXdIcIRrp69;5ab2HF zZ2a=_Jv`Mk*}ZwEn)5+#Q(K$Y)!BaxSYY^vv^6kZvWg6r0a)iZx&gCEH_)`>xOREj}LAe%ai939J+1n6ghP>}CXcOpsi^;`eO-P@%N63V^9+L%<)*}q=Fg`Op5wFl3H%3%q?8_CT^xox zb#Wn)>2wB1ZArwZN9uR)Gx-E6%S|XxPpAKt8d8}1`2I8)wRyMX`}=FV?$s$)j6uKK=J~Ru)JXZY%+@o0-HP_x zn@d#uyWXNeE|afKdzWheah4Lg2Cw1YgIJ~9lmGAFdUwI$PI*L*1H%aL*d1<|=-*pf zIeQCFSi|4!&=`0W_s3H>dC+Gb5ZF93f`2%sH8niYw_$~Iv|%wmQDLZDdtJ-crXwvd$1`Bb{- zngbI^@4@x8ZS!nPyZ0U0|F%~h890*^VK{fBM}J`3N9I^G>Uv2<^M<9v!KcjHF)=Z< zdmR_YS;L{=)X^wOuuK6trH5CB{0QAFHUiIleab`5>yv4F6zs|2j3XzSAH^yzEbQqk z{vEF8@m`(Jh?6^T9_MSl!>fKYWLr3o$sslm(Y7ZX^AZ}GcV7Ut=Gl=sF)Us~k__}} z)Z3neJ@c*>PH%*>cS=9}(D#2uG?twGGf}1PnW($Ns@ph+VNVZ&Sz2L;uMk0R=t7uA zQqG!?A~#{D)%Qa~gd?A(yzbrT9a)aqCB8fr`>YSA!6N7=9ymvSwAs^|)qioD zC(0+@`yt9{BSS4y29aAXrrvdn_cfFJeZT;M;mBeqX8@wal&du6zLdb+!~)adkE z6vOnO^KH#6h(>q?b*DG0SPdEYI9;zn6B@IuE9U6};xYAYL7d><6#!tY;Hv22ar4c$ zI6ORN(y`rpGkAk#CI$>z*-(b6|w8N!_^KSzk&ENi`ACE5%s< zK2{xAiHl)BDy7NKne9~3Ph6et5*cc5_W{~}`CMYEy{;w#wG26U!4_wX=WDurq}k>PJ4L+MY8umn2LKcna4DGn+LG(Y^;^i2 zg590m!Tv2KE=mdt6V<8jqb_l4&90Lxy?^kS`2Cvs5_abeZ!;t>(j)R!zF>1qHem`M zc)#CSRR~Z~=QDYqOXdnB$??6h&jy+ry%eFLt=u$QANs6M2#m^OaI@a_->=cCN4h8 zh!KqQQ|XCyH<;*cT$j^9v&dLmYbV9HZ~SGUjW*{tCgUf6xT+K-HT`a z?mwu5i(sm$wi!OV4E3*Jb*ekz!0}ESQ1i*~Q~4GTUx*2DD2HF`ToGzk!?G#Y6At|c zAPCfrDF{x|%^hpF*d_l)XmcMz--!N~xwFkwvc zyPo)*D~>q(^P?S})XhMIs&3+CyS{qEnz=ccB1#t>EywGW5$yd1&E)HXNBL<=><*qq zPdv29IvSIC!)mE|dwY>R&jx_m+ra)?1AaTcE=A9Cco8_1K{=L^%t}1LI(B9xzF6F; zUPWS_{{xz@{T~np1{G9BLnDm%pKQ6#^JuB$$$l}@PFqu8uvY?EZ!P@FdKl^;A^Jcn z{HQ3iBNmqLH5RS2^ncCPc803n@-s{KLIgn%wnTS zf-&|%Z(!?MF3Oo^2N+F2V0!`Dul)l}@|v)8;n5N5@f*hZxw$$W@?oIAlP7Z2o37@R z&NHQJ+hbQ)@zfjqRoA9Z|0ra|bN7N#>7L$vFH522lv87kEAH}JElZ#x%*}nIrBR?m} z|Cdp@P>qva+5=>e?O_FX2&6r10hA-pJ47sGneiu6{!bghuIKb&*?aN}y^al($;^Wg z)62zEM=bySZo$|{_jpS6gxpVy$Yq==4UkMjT&G96(W9cwYa$0*VkuN3m_xP z8KzWuwt}gg3WhcPM>tH6$t3Bcd%$mfoJfC?Qbx+*z~f;t*$yuqL0d-{jE%liba*4B z_~jwlGKYSmm;$vv4LC>H-k@!aCriig{PCoJY+Xcx9v7LFL)~>4>(N?0TLJ!fkBlT% zg$m3N7_=uwtjN-V&;pFLTl1_&$$yU(dmyl<2$E^TDBfv9X=KdSGYw(0&W`#0L@@I5 zYK=MG9c?eiSVIA`1)2ob@90XK0c|dgHrhet(U8a$0Ddkq#Iq`-n8>I^Z>T_MLj#2g z^St8wR?d&sq{Anil%XJU9@uY+B^SJNXkL6H2nJpT0Bg}V>ku5pA6j!B7Vdzw0P|nR zarT9vb%N>S-l-!Wj(4#4_~eE*jjRVhQOakegVjPcUV=^YUeTYRw6B{7t3^nTrA?^{@aq{pCd2eJv*IBGkJ`qYz0 zX`aufPW5H2a%G1BXo2Iw6$i{#Z{bz@Wcx0%t19s984)p@OR^{B4BkFvM1HE1>(T15K_hWYIj~$WWp| z`Y(Jd6GxHWp5d%Jr ze_$zm@p6O{YSN?bLSDrZ55y?j zLz;tk({+nB6$qUWMX~>!vP`vqzlWv2mU`bM6?~xq?QY!nZ8sY1I(iTH^zqUCFR3KZ6bJph-PEZ954g=yP5B89SPdwe<2mwWvo|*+O<3A?h;v&EHSV!$v z{a%Os3mIiJjdVaZaKwNB8&FRN-g!6(xI)o#S{#)66DEAv*W->1c(?@fuuI2!;F7Sz z81TF>$6mQczmd#G=p=?dXLJhGD5=%u`eLur5^8vc1{pknmDRPB1TIAdtWw8)^CcYo zha>^5zzSm}P;C8!05!%CFND`fg+oD`Iz)g*Oxh5vjRW!3xhfoxwB3aUuR$@sc`U+$ zw~$w~QMdrcOj*WnrL!_*_Boo^$O4|ffOd}sR9m1XUl@e+P*zZ7S(pX}vJ3fre-|K@oejXzRF9RRDJxN56gBIYCBzC6@GLa0FY#!5}PO6kfjJhMHA@cdI;MwRFFL7x60# z%2DMCn2G)MuyCZL zq-+E-pCe$T+Q(^cYs(l}Tki+WZJ9CEh zg&<-9HACATY5pifo9|`agq3B)dXoeLg-YWBtm|vc&JKPThcxC`I$^ZQPc&AugL|`GA_=gW;IJ85!s~_56aOeb;%shhoi!Yt@(`1Yj+sbf)(;Ac^r@W~ zCJB7{0?iK_{|2}hR^onFTA;J^nAwKOB>bD8xKABdb9Np3QYV!m@Aq2L+)Hzgo>b5! zkQ%i)JDTiq)ys)uojYSJ!r;7g2RLk-rn<60M}{1-d24osC6V$qLAfr=l1b&Inb94w zWprPTzBQ;(0SU;(WApbE;Ia65YnDv#n^_HpERyKP>6HC4s_Iw`KkkxM0oM%6u;VX8 z{?{R(66tl^_U!ELe%E{O-R;2x<+v!`KQR3ieN6({L2#7%|>LqOKEkcx?qk!iQ__Ob`h%4t+{2kxn{8rsLby?2&Dl~ z22OY@xZdv&EezYKGI4pCq?%)|0iU?en%@J;Eh^G@*3QwOoe(wE{&Sz<0cuVVOT-3r zOB$OdBd(pACQ%8dF##b})CRlu?bjzOD|oC1Hs>9o4@0PQpE_ljH-ZM2JyP0b3K!(H zp#ab5}0L6}k$)#2gC5@sSZlie)EuVUVt#7HObpX zs-;-}t4CxgS6k3sK*vOO^cy}c!z3izzm61?X^@J(5Kwj&mxtnu3F_K9(6sqo(-ShS zG2DW2?ez5f8r!cZumBKyO5Wvh@fW%VG$cNM2Z&XLe0ck-S5&2?{5^hyxO>-;*yXQO zOE8d5VhOB5ZkIPVH=COzDK8tkvp;!wcof!d{U|B4mQfL;kA}ry2JV)7|8*Kzr9N&k z)drp+kWu%wIzYkM-JMVm0H$9>Vc8&5J4n}GyN1kB^Zl5#`Ex0uL;QWttCd|Ur~^oi4;UE34yO5&6sYqX)08kFJb;Wn zb^TBwz9Bp}jYH3&IB&38jO2M|(Sl1O5jc7yb*BjRCa^4YnQj)<4c3sMb6)jO}^dwm^gI)yjqFfo!=o zcP#?zYqVUvUH$#uBQQtmuiII50$OqKtLyE_NLGa#K(lkt^u6e|s{O^E&bxDSqmLPU zU7+{h1Lb!B+}4+%T8JKWdm8a@#5tel^whoZZ^XkS)v(1s3t3FPj%D{sF(t5=C0PIS zVPX04JJM3x06upr+oYnz&3CmcNz1+)%62 zRW}QZ(JUFVyXj;w7oFrL$yhU85Vl#XjKDVpHp)=(!ZkS5x4rm^xnn)!UdHaPQ>x>+ z>Tt>k)!G8$#9KYrhodToD#8)5cp7X-%S7!MHdQ3>sTNtLnA^MJcF0P;T2OC{93-=9 zsT){v`0?#cr^d@uEpii;e5@Dphe}-bZ%3auErLn1ZlV)kazf}*3z<+64X3j6be%A* zf-2SUm_VQtwlg^mOPaG7jnlRXAFuNBk%42cYrr-D{h*}hVV~Cnpi^i0Irhcb(-RuP zqx>GDOd|{45WQ|e>OrJ@t_ldzu)PG&Jut|p*!{Fw-k^@AB9V-ma@ETmGD|cCRz`CG zq61z7z~`8-dcef=(#i(V&uqjp$Ov#T2Xw{=ASN0*o#h0v2`B9^9kepj(f|e# z5WmyQ_+gDo~8)6$FtC}r~wbukafGv zsE?HkH=6{y(F#0?u{vAuw}XzuVzr>^eHN3$VB1KX83RTIfqbwng}8Ml%E{BGgV*#< z1OKUe<8AprTrLtHUCktshO$lGSwKBV{^9nZ;~g`3cPbT%T`3S^3__8VJ1bqS@7??t zhFEJfVZd++L>2px3n?ne?3-mv^;X`k9C8r}btL-SBRy({>ig>E>-#JOOG7wWiEQ>; zZ$NUMWeOx+0GIZ~mvU>B8f-`*!a+|ye7u?ruAZwx+9e?Ir`t&cLiUZv2l#9LB}8Uk ztvm*vVi6IMYTIVa)}j*#>l(Aj0_Ya1$f-rfSQ091tW8RvkWu-u*D#vn*9fcL~e3qcHyhq(a=dP5_ICM+9i`E_Q{?0MFZtjbv} zL8*1I1o^Kpo&cP;rEV!sEr6&Fq_@LFPjO%2iW+}!pDM!>P>xsJNI@!xn?XW&xmJy8 zy_{X~Cs!=^6wqq4-eK*R0pyY)teLeU}L8r22T`L%Ny8Syt}t+&Qp*H+96n7mSqU(SseQoW};$G z-raY3+WKaIV`%;kUSzkK&#??MQ+gD_>L@3WW0_(oK$1}|WSB9?iw7#Mt)VC?X}V({ zK(l+503t5sBIVGaQtR&gfaqh_tr0!$c3N!cv5^>eNytl81{oKT)v%c5?l=LpF2`sq zxsPwizW`M;p!Bj8{wGS?CpH2d%VSoh8sNum*C(YLW$GPF~f^RgcKg&o2WBCHM|JHUHobRifDa7od)4vI98T7N1npzS-H} za+KHgI}7vX+1D9sX|#wZx{;>Ixj)y^j1Qir2zXu?A|dJY{=*1)nV7|j2-?kBn#5HLHx66tPphXR0&nJ zIC}p;KqbVzf<*s0UDDY|vUd-U#;C$S$=x)U^Gw{@z2j(WqzyTvZ%w^eCouU$6|`83 zpw8)YL)GFN{A_kKIwo0dix)Cwb|Y@mtN(Fw)8VvcPYCIlDVxRpirqrG$33iR_G)@` zdgqS;50&(1ZxyG(rJ^kn*^M>NDji!m<21$5)r)r`b8kkTn;(6vPPSB$jeu!u3+6^y zH?h{()cMTU9|Qp({muW?CrP)5A8tUl`PJb7w`NI#sdWh72xW(8kpi7;f&WsDuP)JRt^Jb)+O(>MOY%z`di|?|;?nUs~=RV3@l|R;$ZUu)YMY zvNcZyFq8rs-I9rYF>W&6s0E(=u$xG#xHaKKrl4*De7yp+)U$pFYDX7(3={;do=0KM zZJ+8xiPLaWhs@$kfBS$j*4j}c@>Lw`dTs!|jLKg?Q2`{Z$&fOQNGhCsu*-!+%fT`# zpSri|2iNro+Y6*1#z~^P>=R5IDVD*Z`<$LkAd2@m-NA?H@J$2{6IyRw|I6&wZ}-|gyts(-EpT7u-aef_d1SB8Ds(D z^~sRXe1PVP=iK$Rc(}xmOfZ3llY%Q4+`kK^&9f3P*w^7T)qc5LY|_Su3cL%zTMel# zN>9yN{#|Inb3uqtHWL8;GwBf{k(aJdLOS?w0Wxg=Bve~N-$9z_Z|2r<)0hPqvBRMc z3yPaM2L;I$vHjpMQHbi3Ppa@IaD*J4G2B?y{QFM< zPT~ZlA;hvcetRm zn%1TH?e@Rg_xIgJ7(irG1}%KF_lLyxW`bC5fKm04kQ5VZX{7@yBG!?-t)G+^y@08~ z@e;!@Z#}%`JmBips_)qS_(;2XmcR&XocgQDFsZ9PWA^UV@n~QiN1bO)H+wd?TX#Mo zv7Pt6J|U5JPC*iOcJ`e=8gPDqnXuhD{Np9Y$St_l**ONvEbZ{u&1>GZr3}P-+Ff{f+#Gcbe z;{V?HH~7Ea^9Mh%^c7x0jFNsc7W_TS2LQMJRXd^imGTR|MRgD`n?Ye06jcQV=U_y| zPyxrdmj3be@Bq)x`gg(9==S#`ub^d9cm`t&p~Nf;>bQq%Ot)vf;9A#CJRbOPQ_T^f zQNosgb+J|PZ*grPY+zxcEeIhCpj^BhW&384t6vcrXRMycS&yJZsEuU$ll(wXbCpGHZcNn2hs2@~J<-E|gZZ+OnNpV8hL!5F|| zEGMr=iy@!#G#4njwez%JLtVw^1-8}ZHeAcxk?+flx3JMA! zK@Y=Ig;PyJ^IurMnCZeJeP3I0K{sfqBqNwg{)4x&=}TI!Ma@td6GKAr(vsuwWBah* znSsH$kNHT3y;}WTSRuw>HOPkxb)*kFFsfQlXIRZwh11hgyxTWMfB+e_pzf24REW75&pj-`zLdvhZJe?ASmkb81j0X`3Ur6d&4G+NfckyZ=d`$$!= z0ea9&7u!3qdI(5pmMH*OzPt?LGj+wXo(1_tS%`qcDHwZSDeWiNWXuakV!QH_0lDdB zlOLF6&US!5IocUuL+(Paq=*&RRQ}uyK=!ri#%aP1Cl0+T^E=7IbAUq%TjyhkzeFY} zOGl4J^Cw}CiMMx2S+Yi$Lw?tlsq_oS=DgBk`0y62T5yYksg|1jKoSX-EYTNM?UX+% zh`}|*`BTE+M>E<`9hw%7hH{jCMZuN}ADvZ-n}^E_=ukDDCbFEad~53xi`b6nISrXQ zeNlCPn@w!X2e;i`^Scmr>fAl=ZB6%bM#D(v9p(K~Cgt5M+TwSEZu=P#v()316!_l> zGVrCsi*SYbeoBT==?Og>poz0ik^1pVzJ(*An$uwe(DFA}ah~RnSqP*Kq0!F=oCH(X zR|hcQNZxm#>OC;wThzU1`v7M6F!EehW%Q&?V9g!8K&s7;O!{-PGvA7`r|nYf-qCc6 zL~W~@|18AZZ({;m6`e9-76U~VPjl^JOHSqZICI%#qk}^O9&jj17cMU^1G-{vuJQ2s z{O$F;|3Q4FAK>wWKY4a`cH`&a;&OOhl0-;&wtzbKiIIB9Vgj4VSg$j9kw2p{N+;00Jn-(K5V9fy0kEOU+wYWEs8n*5xcaJ*)Z zk8AfXK_86X_zQ;A=is$(zt}7R>Uj!^dy)8X-+VOyVA-HSeU@K~HFzPyKoQSsD|jCa zsit>=HUp{%7pPYHz;cjO>1X0-JNLavyL!?hRuaFfzPGP|GSOi^lqy<>+~P5Je+~zT z8iIDI(x>UBJix0>p?VnmkcHKh9{o3GMQR~6@!8!^f{R9tILA*0_=^7?n|(2>vg7Hk z`nO6>g?YX8dJA{89hr^VP6&0-$U_9R>U2{cF5cXIbE+gWDL-ueUU>1+E=PSv;|&@8M(y@CGFstooyby25_df3*#QpgcP)%bG(6Ci*UmJN1#{KbF(`&^o6 zE~U>T2wgR~`{F35s3d3m*4T+nFnm8bNlY5A1`z;}qvGM}99<+rRtD%xO^HX~ z9`!2OILlyrz+n5exKj{kgFC1;!nGtq_!75(lA05K*T=k7-&o5Y61G7hnZo+X^sBqO5YqC*>wz*lgyY%M4)mf3G-4?|%i~r@zUL)L z&zq9UA_5onE%p4)P4_;#%4`JAo6BX`Osx9FP4td!cb2HuU8X42X+vIufQi}SAiBlg z6C&@4*o6;NuwxC>yOPW12V@b>c0A$(fFf-y^9pE!)~3G<0;ZKfc}OZV*DoG2;4=UP zdCj>YWNvPbU3%4%QI=Ta2)~Wg9yVWbOz^SJRw&hWwvwx&{6Vk6xR2ob?~e3%$E$q2 zWF?$O9Tkd=HiGw4hvYEYiM;6XNWQ22Kp3NfaNq(fDesneA)$J3GJ?C7(+QCIB~phn z#J#h(E->Ir{=0z&229Dz^Gw_afe+~_@lra#hJD9Pv*hjzH#H2nnCF=|ldLhv%IFLw zcE_}-y7wg$i37E^&1s5JBw%t~GBM^FCdJePcqUkyYVc0~m75#JXdohkkP+POQ{G~= z;5}gPVAQg}weCEN;U#!H-OoxLNdEo?$bdq6CP-(PKEjO2N?-m=gSpd36Iki}$Ss)^ zIeD)FAG-(AY$I_H!y37tzELA6u}CaarV_Wj1>}y8pKv1d?5lk>nYi5P%C@mr zPWhc&`rqLr#hpG$?D6e_rR$ncc%h=!&6YB*x2*(DqH1N~6hzY1&bsDN_K0&W z{@McEevgk|kYQac@b5=Q&PuIgx`FQ_3x8B^gOJf5)BCfX`LUEVQMv|?6W25<>C6Hy z>`;#rZ;T#1d5EV#$tEOM`_)4#&K?-yp9w}vjbbeuG*pVjbTJU7lWJ*$pt7dGA_1OF zz!NA#u8P<#B$--@2$GSHKP*6`#5{xcVM?U;-737jyi!6UA32>}b)&#Kpuf#FwQCU; zk;6Y!{~hb(4Tnofqt$T|^M3I-P5tIlxsmGbFVkC#lDZEU8bojbAI)Z8$XzM(_O>#D z11oc(w`?Xddg>5OxU{jn*2_ub6n0~;?h!$vA~Q0a173gE zeX4Y&OZxeh<@p4L;*K_KK(o_#o;skYi=Be36-`HQ7?8EP5qb-|>U|`7t1!9a_H2Nj z9K2fcumR66pub5=W6YHZr1@<5-O;AO-SVRDT{pij#y0;a2#L?xl8a*<$q;|;0&?tt z2HniLnZl1h<*Y3>3V7}Wl);?qPa0T@WkPnH=7hsJ_L_YfJCUKj|l(rW0owe?nTf6T2=*VB zt1hMA%`#c$@bVz4=&9E#zw*v(v1X1|9BUeFShkcJ7_a-zYr;X9H0e{V#rbuyK)VIs z>~2y7iMEr}S2>0*Rd~K7&I^zLH{u?rT=<{qWFm4b=LZ52L@PtwOCT;uBdbr73=9m$ z09BdM|8&(mgB^PZ^Y@AF=zlqqYjt(M ze0vW^)wlPg?fha|t`+2ZFJPc!yLL=ig8!O;>D5Ud8aw2TfYDBiKGAM zpS0qrMQkLP`8T)dVqC=bV|N!ZXLAm&e4vbTFvLwD%=3h@k)6E-GGYh+1umPBbTV6f zlBZ(Q8EfFeAjX!Vua50DQaKFE4Bf}eZltzhZ3X+PAFFHmfWOv8-8~HPvguAED^QMADPM1*;eb<6< zZIto5k`&W@NyQJroqIyfY^-Gd{;3RqpcSP?cdzmKUK#rLEsm?$aPeHs+2I&9NTq+$khjqsHYO&NtOwJ~&%B`^g7_)Z-rZShqDa1W*7K z&%aLG=6X|crNnDTCs{pq9Ph>~j!+uN}dqx~-C(wn56K7o~C z4`hGS6c;+z`If3{BhF&%xX3kh+%m2yGnHGw)gl9W_dEP@1 z{0Oej(3?9}n5V-6H^36+Rx5JSN0@G0`laM+Co2}Wq)^@+bV7;1wv9NFdWd~GH(lqQ z8Q7dg4zcz<>d-s<{L0I+-ymcBI*3moD<&~tJ4%MUh4Bdt&A$xGR~x7f;(ZkHu=ixM zmYt>wy)nZNz>bPViO3BwhHk%Yzv%*e0kAA}VrIenjzx10`i84Npy1hX(Qo%lM{a0> z0~RRpGq1PJf4_7Ciy8R1TewhRxG@TuJ{>!h3$F z&94i9Xc+oEUPqazGheqHBdbD`4>1P9l}Zwk`mU~*4UnK#g_9H?9`Dgl2eZ@-BzYa_ z*sWQSa=-8&f&CO7r*=M#Ow@!f3Dtw)znfpnVPs8)FSORDC?Xag4)LVPr~__U9IGS4 zX49}Ii%c@Iipr>V>hI+&3Kf6srA%a559IzND%9U{IL7$99L2gSZKjXKxMPeRvIU|y zVU#BWW5o|A+#(52(J@nx-1g>w+PO~pmF`kciTYj>I)*o^zkDTzKTTSEeOBthsPstv zkD6%COm8e_)w!)X+jn8YSIi+=WxrB-#71zfKn$nQd=zlJO-?G5%;q7u?xA+%bG954 ziLomwR+mn6bvVu1_JD6Mha+!AJMk;h&tcU0L6@sim%$g#L;fe#4L_EI^R*LeH_7!d zbpI*6^Sqb)g~J0HvhT|Tcdk^5V{L404OQ){=FAUYZqU}z*PA0^>958!%;ErnGk1F( z>!-mJww}2_P0fhVqEZ63(fQk(Q`LR|B?egylGht=_Kifv#ao-2;#7e;>EE!){~F`3 zS0VI`hb#!_>Pdp829he2&|u4s-T)UjKmAa*krY%E265n;+8MUE>3{?miB{k7Np!%q zQnk}Wo>46uwa3{`17jcaTXE-6G5Mg z$Zn%n{oDrG#Fzg1hQ&l;E#J!S3+0C^iOv2>%vAlIxuoP$8xW~V^o15oo^?2B;ZVta z@^`mO1DTL8UV2JBO!_c8=$MFt&TWpj)nYjATX{L%t=vCvIQKnvxu-dUT?JyUt?MpG>d-wfJ}x*~b@KK1|4<=dXTQ4y zy8&ji_l?C0P+o^OXqvgQ&v2LK*?|GYuQ8xMVq{=w0T~7B`@nzz4o;~UE!WvuJE4di zz*hdb2qeYbQ^X4q4h2jUA3?Ar6f{$>Qnj>0zflG^G_3)IwSL9p{E(-FJzE@d0ZHGO zA%A(hs(6t{Vw6AUt2w0}fBP>u5xDy9NbhGD=tA>rI7!9S$#74lB&y=d6+y)9*cdZU6QUY6 z4C6LK3Dj!mZ+j4qOaqe+6Rn8Nq%!RX$jC3G$&TmQ;yG)aBQnj#lEcF zx>(uOI8MIIZi}{Jll(DmwHJ1H92}vxvqF3M0VRGy=@rbW2;${fs=*jS=b4_lmXncA z4>0zMkG$u+cX~=Wq%m9L!qdKRtwV!vN|G^Z2$^yN{XEe3`-?OD<)JUah!xB`KD$3s zVx6+)X{v37$c_pr+i|Jth+#qh6i_}A)2R?b^NWj$8mn<1ezBZV{T2goL1h!gfe_ni zx3XW%Aq#`A<5qTex$Iagr2)Sky%AtfF3HFI9<1hoD-TGV7pO>Cz}iPZ`4Y%om_T5> zm)FI`#00Q~SnzMUHOKeiHMyqoQE>ptAK1@v7@{T@oCwz?(_aVfa^54wD#LOGiuMaO z=^x;=dtdg;s=r$y|4jlbkuF0zq$URx!%#ocKMTk1WB5ba7 zqA~y=ENL@+pM)?+>5KV}ZymG86QgT`l4hlDHQO)L8S;F+@9T1sY^xv1YmJ?QeI_{wCp{0F52?;zfQy~w>V zyi*(}7^?}#I>cD(-D`JNs2=9oz}KN)T169fg4!rcK$J!Kwf7Q%>nTO;>c-YtPh9wfm)7?A|Wpf)ca-pL%AF&%4?>bnk95(r2h(fOvwZY6;&<1D~rwA zbJWsjveYC|-1fR4c$`LU1B~kbO#oKk_#MWBaUg)pOjw2Ph!AO7?0jhP=!9GzN#6eZ zzrT&7QsL|?5IH(I@iE3OIuiJi^(%6!1pe{hOMWqHJ`K)g#@OE9GA|kc_sOFIjFy>y zX28M+E9*) z3VT#3M?{dy_5OZ1RWh>3uF|glJ;F1iAdOb!(O$>h9JSiGI9@p#7S*EJ#F?Ftfd6{J zv!5#)BjVC=tH0B+hZ-Pw$bS^j=<*;DrQ_Sb;yLU3mhzQl8YO;Zu9ORFV3B_*LcZQYfqr(%@&%c&$6)CXM>J+QUhO4H# zMOx^@nG>LWD(vV9Vg{!de>mE_Om4f&PhXpwS|8c>{V2^rSV%dZ8X5^MGj>GobgjlS zrJqKPpB`3B+iq!CS1Adl0{N{43x~QTz5BE+_X4;_$hsmUkB&>Jk}ap7JtSt4DR!#W zuS>c1+go2v#VBMb(>g0-gx8>N{EvF)8BQ4i?@6CtAx%qzE)0y%nua;@wKG)V!0IcD z+2YyX62ee^8zg2CivcwNJ6RR1n}2B193+05nyxuli_B?-&}FV7;J zUBQQE{bv7g8h9_y{%-hG;Rsyq|J~l)w5lThWVu=Xl+5?b_ej_hvQz0f94(s&9{%PD zIKFmlLDUbk`XSFMt&yOPBivZtR!&8>ei)hXTE_CeYB?qw0J%o{Y+efh7OvfNYIhK8 zk)amB^J+0YC3DL(9=PpLcG`y8qo5VX9tb%46ji0eZBv1O{r&N=nW?84BJ{NZlhA8r zG;kUw=WvpKNt9*0j}&Zcs)C_Hdobrd$ z4w{=6y;=orw#4sWQL2((6A7rpEt^iH`XBBsmoUw&cyCH*-;HLPkKe^os?H2NZR0)- zcWST{H5NW^lf?4%SD(JIe(GUJPe4Ll^GD`M@ehz}17Kfc3!}9*pD!k3gcD4)ZHI_a~`#X=UAX922|g|EB5|W3J3BcIh;qFffE&+sCt-Ru5t-+ss>Ne zSydPZ^eX^`2N*hrfX?oY95D-OGD?-?FB|Dg+qELpq)h`@UWZcUBUQNIBhIj-m}QTQ z+$|){lK~QGRy5I?PkJdmi@w*XYUF)mEV=tec-&uY?AvK z?RpJ3Sa--I7P>*MeZ(dY8L%rHt&NP)f@H8U3pE;`HrXy2GRxl*VI$GvU?j&UUGou1 zC&~G>B@iy7LbDMX7*1AvL@9PybL!bSvpX zSPY7@1e0Q8URQz0X^sS^)kLrkYK}@)m@XDbeTwc!dx`gxN)u`H8;(R6La^8ZyVM^P z&5TJOihZa^^87;5>6$)SXRC7o7W`kFdmt>OoWgJ(mhV7>680ivO;2AxrYHB{Jsn6>)k&01ln1%d{-Z~} z?h=_@x)!KUIXpnY=1kY?h?*1=7mr>(ySRX3jw6py%=^nQ3Lo0gLQ_b{XW)z607Pc1 zi5aEq7{kem^1NIOKdEE&IQ*qZ`%tq!ptIl3uuUtax^Eqj1+mn{ZdZF z-V@G&CyLmAi7)_U_Xh34@3nq1#wJ^vPc$dO zkf)6pCYeX)h0Y~>i?MQh6e~Xd$o5N|wFrdj{E_^3l~ZgpRA%AnqdP*q2Z+z6EJXh$ z+Wk)4@y;-wjP_C_5zgfP^f`*R_e2SuC)0($(^N;Lt7y?Y`@J9A!(*APGXXZD$Rbmw z4T!>{d0-R1vSRTW+1J(8wb3_bhO!deiq?6ie5dkoZ_i%#HQ2_mlK>MukkX4GEdz!@ zK8xr(xwPFBGaflBS4xSNL{pchvbeRdY|NHa^TeWV3eqm6ZVz8pQsp%qQuv!EqUkYc?zU%PBiV0WdKCH(9rH z9%D&ws{O~a*b<~)P1iyDAGAO}snSfg&DxD=V``|j*dJx zP)g0`O9Z*=`le|bh6ZvEI1d_8u+^$m{P;`Xc|EB;vqQP3C)&TD)!rOJ?P+H}p49K> zr4JsV(l_CY9mfKN=s&6bts=I1(jN`8AAJGUzq+oMqQECqwdq2L+uK{qvT8w4_XEdq zG`bxM2aQIa=cZ{0!GyF4VYBJ{{^1UXMV^cvjYbef9%y&XR;?&X&I?U5bVK8UMq%h$wzWfd*Q*r>H=w69 zYIV~zf#3rH?)DEMiNNM(I275o{rdH5ENuAx`|knpSe7-P&x0WN>Z`Br?(T}BSgls+ zWXu?YdKmS*<9T_Wm1P;n@p8G;*w&@DEK7?~I7H-SflLVUCo4?~h~Iii=T_=%#hh>v!61&33T zjiza<6||rkA!M~$8Kw@{CeQPt5RflvIxWj0%M#AHB0}uhBATXIuQgk(j%8W*_qV2L zo}ZtC_Q}ZEDcg=K zz$hYwNXdURo8dl`QbZ2fF$x5K#|hkLwC%sV>h(}c^E{`DfQgN>E#_PaS@(k=2)f;_ zXFFwCmU)5BC-CdK9-|@+g21-TB#FwhR7&5n;5EbZys{KZ(XuRuliAa!PcJSmPEJl> zzXkGCS>{m`-QAUG98V_WJkMd_Tv01ANvmKw+qUV3Qc8W_H*E`llwlZ+z&8wI6Rt|0 zX&NyM%`mlE-DiwZrj!!8PAyv}gy=f4ZM{?u_&0aAw?Pn8@pg4}g>p-yj4>7`$$YWM zi=rOX6(P*f1yMOK6{CVE!?p&;N0#eIO2ka9qi_>PzVF9bdVDxIKR-vJj-t)Y-7WCg zR;LY~2E6L>JjdRMR?SCJP1BTUtTG5>jEPc$Q6zV^H)UB;Y6v0nTq-3!$8z1eZJWBT z<$0>nZMINU;Uoo-Fp471x#PK(Wl5qS@8F!zW;2a4ow6*;dhJfT-CnO&Ny)*UWsC(u zfZBVv&7#~&vF5S=ZAmE)4-Z?d*6r4)pETicQ-ea$=Gq6!^1-~3ASx1 zN(jLYlZ?x^=*Tdmf0l8iOzQzhX>C2ViPH);B^hN7BG?-V|RCVj^luo zhx)VGY~J17nWni|EFj@)Hk+>NUR+$vmkXr?rPQ)4+qN)X$4kQ0A)desRCVTly4$-c`q6Da zus*0H8Qb?|szR_@e80_LRYCi%*~6i-dq4r(r5ymHD(9Qs3t)TPartArXm+oSQK}Kb zvMevsOxJbK@msAnAuNv5BuZ`5bY0i6&;!UgPf!;bhGtu?R9pyUTTYV3oO8jY&_$MI zQIzC)t})~M{QTzT=KA`&-EQi-K64iRLlq=1uym8 z;`M@;VHldG+3j|1+s5Wo4A!KSjL}A;E@dfX+3B<>B_+=dQ#(2y#Bpqyj%C?OsWizM zqq?rei_O8oLA_ot(#&z3EKV?pJ32bTIRZ&e7>1+KD9f@|M}rNyQVOJSr4+PUhGDc? zEeK)3Sx?jS=;#Q(Qc}u#Jt&IeyYIfM)qKLJ<5-|vM^T7!8iBzWgXs%&_j#Vz8ujWk z(liZ|C4BBloH~wU*+!DYVHj!}^*l#OQ51R24_w#JvP_mj*G-MY#=t4< zcYDoRJ=|=dK`G0UX*#+=&bi}SQpz}vrQB*V=+4GL>~`sB)lx*A8io!7PUsk`5Rg(T z$tk6t=Vf_>&8>tGtSXRF4h93$h6h5vUa$W@n3H0a?DY$100000NkvXXu0mjfs^TXZ literal 0 HcmV?d00001 diff --git a/python/nano/test/resources/train_image_folder_png/cats/cat.10.png b/python/nano/test/resources/train_image_folder_png/cats/cat.10.png new file mode 100644 index 0000000000000000000000000000000000000000..5cae17a710e48f293901f5a774cb4919105e563d GIT binary patch literal 329867 zcmV)ZK&!urP)MC5Z(RS^N8s-+Y&V`dRC&8k{! zji=%sGcz-+O1#3|B^iMDY^xHlixNM@-(APd9Ph2Qx_e3~r4;wCE04c|Er6&;M52!( zGLB#ryP}-*IKK) ziAYKeG5}`P-9e}OulcIhI!x0vP1Tf$CHKvcaMAWF&2R2oSE6(ODUz4 zuIrXDSq6lNy3Y2;0}=ICJ9QhaeD>uRzyA;a_|O06pRQis&1aD+@$uDkE88Q#WOEm@)vEN=HQV&l zmT@BT=ob?)bLy^AO3_&W3Xi&~YMb9yd7B-v>~8K3s_dp#r)khSnOU8RyQkE3l0fowvXcM+mDRRbUcs$GqgDNJ z(cB$Y;p%Qoo(QpYiO5+%!Ym+h5+Wj3JXMwkUwoL5$j!`LE5isTL;zAs0Kg0q1A~Q# z43y-w{>c$*j2(dhJdcu@xtf`A;`j=4_O>(#0w!l7P9*F^U@~F=$?3G$9(`DFV;QAb z2C%ASsye|eF*8F%h{TCFC8k7@z=?=l$H!w5@N)6SDC%Tpun%w3LJL?qf5HJDl8+VYMDF#u*Xyy#&(?kn_X zW&*n#Gr4=r8i2N8R~W}k?r=vN>vpjmP5^!v0x#MNL zYOT{WMH{ZJuG%hP4zO!;qcwof-HU0oxsC8SZHSUryE-VrdY;xBJ4(zkv6D+YN<_@C zxy!(53lq^)Yf!ST>%lpZla%>olS3QcCV>09l}b}_Wp4tANqSF{E|%rTm-Xa6cjbfa1RUoLIBWb#?N+c^3#?_llW|pLp1Ha~ttoLt8W`F~IJmYK>`P z20)FMjK>kcy~b`@G4<|VN?G$e%kVKnRn6R;MDnsF+zsI1_2-3d%Ts<%oTJyv=QAFP ztnd4jlDoS$+$7W7*g|U>hrk8#0TgB%io#xE?R>*NlAEKBo?PD zNNap>C@c`~HxivC0>}YpAqP2(7;X+Y;0_1K9c3t#1mtlDAfoYj{O;S~-TmFe;V_oz z^Dn>L+-!5|#NnzeZV1BZlEeTrb$7T1Ap#Sub@>8jPFw|OUJPqmbvDrp>b0*p8*_KZ zYI!`le;VOmz4{DZSBH zBKE1Ab=*%~zSy>(ROgzcEt5hk9`1?zAJOLLYR37(!1=T;Biv}`Oe~j;?0(jrSUc}| z@5N8P8krQ}*mYgkbvfr+E!aj?3#4F@xy1k)f4a10L74!0-~^`eVkQpszd$<7>}Nbk zv+$wsI|rGQ2c8lTeRIRyi0B1TfZ*BwS>6&%{o~C^@OU$T5OM@miG;?S)|Q?*6Id>g8(PE2UWUc z5*xn154$f%&)sQ-m_&HI6i@SVK1W0sx11e{^}Jtf?Z>n{rtv-fdfd&WwgiE*LzsyYfwM6u5g{}9+{)Gasw&4GZp>GlBP>vn z-O21E$1qP_HZuo-XhzJLeGVRu++pTkwboiUn~ey$b4b{UoXDA}R*h{l_cJ=+G}ki! z5oY!{Gpc za|$tE(TZ6tOCcRNZRnHKVp$^Y4sbO7V#QR|SuSMJB6od!5Jr0+aZcVe0SbUYi=?Ox z%X~YtB;u4h)apcwM2lu7Vma)@0`fRZnY%lwX{~~kNkpVb5|YEialhY}adb5s^|Q}k zZ#ElH0hvS1pblns0$@ZSW;b_Yu)BN6OqLNo4Z?@8d%eDCxyhH4;%ThtysV#k1i4sM zc*5?}D<0jwAkO)-PxIZ&^#$)von4{!@~%^b&+y%AE^mc??Wv_%fs`MQ9iLdN?M(_O zk0otbiNfAfD9+cbufLkc&?9=e{(z@sVi)CNMJ*SJFDb!t!v%=9&-3D0=ICP@|F{7} zl)6wNSzTUfR?SRAh}(*vw{XZb63=4efQ;p9@Yf5*`ASqW%RSDz9RilP7{XzyAmKhIg{wD;5}@P`XSfABKu%EeMb2{q$M<6Qa#mf&12bU{ zl@mZG#q!+Q(<^ZKx+k|YwN?1UO^@OD;-1IwadCaApP4~|XV}*d0`3Kuw|>h7e80b* zPcP@31lj6?oZoiF53L5jwew;N6Oq4XG}g3j`8-R~J_fblKNRl1@L7*fMHkOr%&V#>uA9jG+Z__cSuUH=$B!QJC}3x z40-L!!V>bC+QfY&A78&twJ@_th%cfNC-Paca<;z!xSdgjWyrd&TbrFMgD8}Rvq>`! zfj8PyN&$$8&sMv)r&eO|(>acrQ%WIW2&qN%T12|8BO(snF(4wP6hFztPt32Eda=FZ z6-_U{n*Moq7lp8fx;NUt!fsy%2Ox8EI0=hz$dTv80K};kP5$0q!A|ay!oY&uy;gHT zq^+$soMds3>ia&X*ODoGnkF(4VRGsvfvKoIyuF?FM-(lG;ng2sU0+|{ynNC3+iEB< zCoqYbH7Vz!*;vTX6ASuT?2etX+Brl-WT$1hthcsG&#C_Dne%t8DS8YG5#b~=zeMmS zQQ!hK*7f6rFLMK!iQ2K&o6hO1_|V;Uqyvwi z!|tU#**iKHUtG-T$55|FEFifx^ur@z_tNI&sfDGOZOmFD-QC=Xo>}PU?O%Uj=ERw5 zRWlRLsmm!PGlY1WIW6!Vtd^OPI5EIZfLm~(?lkiU4w5jQ2{sO@WwqYmZq39BAmTve zTFu>4Vj?oD%!!D?ly2_4g_%b1^y^;Pis|-QisSAniy5f`dZrzYR|Q_XPwoy%er>=<6&CE z%2IZ_Gk%;1Zm}}z4Axj|H0Lon5p$5mG*7iMa}FELK=-PCE@O#RB}7bA(byhd$SO0; zcwteA9AJ@9UJEsH)yq~@t5*S~)N!Jz7Au;VoU={({kMNRKJ34&pMUYi7w*4*@d^Oc z)WHxZ5_|-vJVL9dZaBZN<|l%_Enk^=(ZD}h)dB?TEqbCAkDglH_F?#5qsjAqX$xbq zIs3@+oSn&Zw5l+m*x<KYex1xwLShyM9F5u>czq@q_$!9*BDGe;zUHMy4cLEHFZ(K5I7iwr76pDiLH`S zTJ!bGnrea!=2|D4t+T`aByI?u;bNMyl#q=q%r(uL!(kZ0Za3$=5FV^{*V>{Hz;?S` zUKj6*JGN*!r8d(ogNCH1PMn3+4dnU(5Jog~U#wF)%@Y}ALh_dou>{`MdK`JZ;1 ztLyDfx-1~DBuTYeOu4GpTA`LiQcBY>F!OAKyx@+{|M_b36M!1jY3nyU$|SmU7m!I#;0E}sTV6Vgh`lr5h8+sg@a&6{plIe^cr)0rM@O2nuR2D zFDen48Ic1L8VPrIuSRS*or$qna#^#9q?J=h<2-OidzPZAL=yJkAt8VeGn<}qwQD-& zE8b|iVa7dT=0pi*Veg%DW==GpzllgB&M_yNx~}u2^IQ7VjN8J#`%D-R(AxGc`O+MB z=d>J3nqsi2@W_+QXLDi(yMxT#U`*l!hdH>Nv>9+rb?p0o+wTHTV^sx!a-#4|0bISt zcr$1|qT7sPVa(e;V73yAp2vz`dK;s;7O5{|7p2xSnZ~kj!^Uw<&1RTm0JMnk=Td8R zg2!rSB5>eB9gnd&m&pivun2rFR&ztNHjBQ@fru>gxPF9IGOsmuNBwJSnKlAAljEBc zJCn*HlAftuxI`c)SuGv3831z5yif&@h*-!FWL8M>TG0!iPj&>r!A?z8TcQ_iV0C9R zIJrA1IlF*1sZX4$7FG4C?EJ%Dzn_Nk!}mY^?hn6z_2x6l7^fzHW@cjHuCJx2>NK1j zAD%4hUALa)c70`A@*CP>zo#()h>u(KNtHZ_P5-+TcZEAB#InnlU!7FQK>#_N7yk9{ z8L?koK7OcaRX(z+g{r;(Ekv}ISkiLzC-$bvd-lAQa@Vkc>10NYn1g6DpJ9!vrIdEN zorp}+6uv4VQfpnT{lZVOwZ>1+UeLF-2;$VBl(3@YQAzMM>p@vQ5H)(W)DRW+eO`D{ z;KaJtdZvzAz1f0B03oYs_a9Vp&SBvfa_wkOJszVD0Pv;~Awc-#ofHmcs%pCrD#x2!U0=u#<5fgA>}7P;pP^ue_&!_qDC#& z6ykzpo;9^=tFu!kqM}m}iN;5WhxEVJJPgA)4Pt5k?r?v5*xwz;rUHmy-t2Z$XLCD7f#~uuhUl@{NJL?obfXF zVS^g%8iR}|q45t5->;lx87 zoa`olQuX3YRjPnJPkz zlTu1Kr>=t;%ca&UpV{R~1%D%m3{-@k`6@u;vC}O_aA9E7+x` zj_AxZ@Urgn(TI8cKyw~#_NAWp&~loW+68v6f4#udb)C;m*$LuXPfr)0T&VLh_eDB2 zTb_7YORhth)}4*sg@q;S_-N*MEnf3|kAsaAkJdeB$xn77&?vUQ6?__g9tm zrU1@Zv;Qp$u|olPAh>&T4q28Q+`)n76dH9b9ehdv{lqI8F!`xYIN|9>p4(q|L}jRXNUQZ8d$DZ9d&Rv!M&2y5KgJh5TIo0FsdzSvU<5fOfRuT1Vkb9HJ>; zX1txfYMpdSmcscvNuoqF>l+oy#zqE#k+DQ8KomhpoZxUHo^gQ;WGrBY2$hLucMuM{ z>6lZIybK&p(W!jl4%ak;PUi+%kD8f<(~kMcGtg)GGW$s}0zh0u1loA#7FB8{QBN)u zcVCC#2!=gkFcFc{9OY`gs_N5KEjD4c3XB!aXw9A#LPQJ>n-iLi(C2^a+LXq{{wM!q( zcYkD)F6)1u?|xkQlQT)@^1Ai2?{DGy*Qn&VDZZcq>lej`6lym!gY&wUQ(d!12Q{nh zvQ!=F^3YYTwd9L(kBFGhV08WAqDujD%B_2g8$GRf>P3XTYU;^~e7S*$%tD3}nrFio)v#u_F}V1a@4lO z#!6{6@*$fio&mTKtpsEN-0q%OV6LTFRfL;4wRSpTaeax+rrM-5i^R##ZHm?-+=!^c z8mR+|Bw->_k({fD=+vgN)=DWc)3&=3nusK&PT3S%rl}l{|N7tl`~B^^;kf_u%P(GB zT}vXES2d(`v3MR2*ZSg_PJ2Ol0FS%}e_KEd>?R_BnSQ;z`y2v(!v_8jD$EQwX13Xz z&w;Q>WnwfF9ma|@;St~c{&E4h4gk-20zAPQuYYsa?{-Jk^& zn3zDOCssNo5l*$%VE4kP$qlA%=8{e?jmlT{&QDnCtkHHsC3jD=AI;-lH^II8qN`jh zqb_Q`80a8KU>b(u$y429yUukl*5SbBx342)wVRolB{kW1Sj8#A{J%@2swD#U#u7}@ zBu8PlPR%#9s3Ls*=jp)8&V8CDo*fZow6r#m>D@nl8I;{vNGXxPU3o;D9N_p z4rMCSajXT5oVvTWcf)WP#>4U9;p?xze*Nl2*LCC;Dw*}jyr(S9_IY$p*!}M;bs$=f zC1*SQx2pWNr8NJ3g_+?9ciAZ3By1kR92bb0JN3jsy{Eq?l>C%deav(`iLKAx`AoGJ zEx!Qn1p^;fzHF#Rb|%l3l+z}OciC0dYvb>r5KCN`jF>rdYKkm{Ha|KLAqUrHuu-9& zSOVY@QP*|CVrnBvND*v797>dMoJ!yKIcKj`rzwQ*?$WVWH#4oK=I-ppD%a+3nVB;Q zGbM(Ik@-F>lNHP?%p^%k)PdA1sMes$$z5WRq!vH7*>AwD);iUiBx@5*Fp*Fah&c1B z)Hz^kr4&=AI8<;_H+45Rb#t&%GY#g38Zk&t?p2M9NO)l@WHs?ApKw1^JRv9I9!_qg zG3(xej1zcU`fv(e;-|Yi4DM>-JVlgJ;>^KBS284sIC~SnxVkH}3Xu@PEGZFDA_1UO zQ%Z>v3}!~=iI|xuRWfHYcCdrFk?^T6O?&88U;=_LF;rcc`b4F`bgXm$b4qC=FU>}G z-}Kz2ZmM(~r*SCP-DccRKM!y9P{!kU{PK&>zIc6eb7O4grd3H;DA63AH|HJ>4}Fm} z=E&jTp@q?KCvzqVv)@`PE1NsHhyyT!5lC%f$si&IInCCyGVeh4)9H!;WYHF#11u1B z4!2)-Ymt?j4A&@JgCUE8fzDLTz$5K65mlH&$vn(I-OR}t$U)-?7+-$ej`i~QzbVeiRutG$bjz&ZY$${!Hb#fyT zawaC?#K`Odh&c<8*no&n83TglaNJO4#hg;xI7|d)EC#*t)nVq-{30@%%t5w|im^;# zScUm~dH(0_1SDp2YZGj6w1}Ayaw17W%sR^qiO6-z1jdKYfXnPI1Z25>!SKdUp}7ic zLYXr&tZQRtxEr0NR#0^^k6_rCMeu7bry+6^fXworm1~B~@k3~qX#hBdUcyXZW{3X4 z<|Gm#g2t+sQoyvBivVEr@S+4g>mRl5-FiMwl`KSirnw7nV76RNF1B{*o;9CwAXuZo zJfu@dwTZwiK$Fs_2FnwxtpLqB(D-DFxt_HP+lggHETU}_VQtkOhckewBXUy$JaQ~( zY#@^OBc5|^vA~Fm)_Qw;uUdxTI89UeeA>Qw6%j6Z4iu9lZ#I3cmB0q2ND)A1&HIS) zZ%Fl?d;XVVb^R&f?60WjJx}=~I|rHtvKJRWW@LQvn42{-2y-#B0N;!K{{A|D^>qB5 z^GLH?``@+Ds(j=l%$&kRWP#mgC&xhun#n}?NwuT#dI8PUa;?XIvIV}f*N(p_rLZgn zuufZJrW^{mFvo-YN)oLKpgOYyxgaN?#WR=)1oGWI1mSx2~-b|%t=G&Ws zf!rgq6;X1AnG!n@BD!zrtb_$@AR)=6j6z9W%P_thhvV@$4#WN3-Pi0dU%u@7p5e7< zb*Bw?U4WMfiML5W{9BvSpO(a>gzDv%i~@hMPG4AFYQufO6`bbue|@2aYGH{Wyry36>` zig~V)?`iyVYom?Ha?c|r%?Uyuw}2MO{48*pv)Hu9pM0gSip@xEW=t@1GA9exzKKFP zqI8=RQHT}7lB{|K&|H%Ng1A+O0}NO5IMgaJAdGgVjVB_mt1Vo1Llt+}EN}x9Cl#o> zH`|Z-RLht>uid?>nU&h6Sww=fjv_z=&PDTHYL37Qk-(!xvy}DBU0`?go0rixUqW;y zVkV9w2>}r@e5sRX1!Eiwd=4%~ma$rn0o(v023Z)`#3$N9p3PyJZIH?18w7Ix$*dhi%qp8)i~Z&W`>08VdQ(c_Ql z?|QR8zTxt^&p$o0CS&9uH+9|qwC1WB@|wXK-#vp$lcf-m(FvQDr{G~n(cDRVHct+7 zh$jJSiK7G{be?_RhXFM(PY`*-S2t#MBB;5!DOoma85)Dl44&N(mJcn4_-UyuZ4hE} zv*pM(qu{->UGSSWzdEO03#CZoHnh`+`X?7yj@HKc~~sYrr`6W z5)dYRi#JXDwMTYu%UfvH)bYv}326f$n%4M!H_W!`?99qyqH|bqDor&SWMDRjr-&&Q z*nLt&6D>hKoe0ex9_%m$Gr1wi4Tu3FuoBz?5jX@L;(*h(+m3!LRhdDO)QXon>hQ1s z^4|{k4^Xq;*^3vi`o0&TdGj+OxB^Vk*ae;w2F2K%jrKBVE1Ai2`*rKC>WC@-9n-{yco+d~L7c`_ns%xt_}p z(fh|I;)T3D9;^5C`l9_GyuKc7dQ_IR>W{{J`Fi5J7dq+gFfubn+yA_J)5TCOf6vOQ zHk*CfFNg$YDJnAub~ov5;IO;88{81KMnMSxGnfNp1;q-3YCG|rLYmBxUKVezv!xzG z!=bN;VNwqmj_?Y8YI;M-ATt0c&5oCFGh<_CHkm!|-N|Qb=<~^=NAb8%vlP@d%myPW zHxL>*T0Ai49IWn6x*~yS``Fz{7(@nl0w*D9pd0+P&%JVYU_N9pn7bffq5L$na0Xhb z)K25kb|A=C;%{$MCv1e^1Oc^h3QdwG0GwQyiKrNdVvC%@MI$t{SG#Lht5Y#kAut$} znE3mje$-k+6W~=}zI?s8+7VSpOLgHyn7vEgeF@<}jb#iC(WPJVM;7{s^YXH!XU4@8 zCwJdZdfof$$Bab3qIu_!TvmJfy8;ORReX2r@_aJmNCdo!Gjl-(E-)JBFtU+tClJAz8kSjb+U`UVT(-72TaDjDNKIP0fF%^% zNq;y^Q?#LNEpE(jARs(qXt}9+RD@ZA%p`Nx?Eo@8t;SQ|{ojB)4CxHF59F{Kuj*I|$fBMPAvB3_Dt9nQ{Xu4L>Irs`cv zUDw0hwWyY=BMg$kj`48(@w@NKFy6ks`|`^#Kl}X6&CN~T^vukXIfbGd$w|-GEzu0z z=NCVf@18f_`}Y4LOfy%Ysgch%*WXusk^cPsOgFIhLc9RSi#zcMso#%-^G0BgNRuWs zpK(`wAYF_+opXfG!S}E3@23)grFrjz?dKkRs`88L7f|^aM)1@iTzp7N{JArvUmbkn z;Sct9Ju7P{ceR%5g@__XhZ)@at}D}|sswa7mz-6#meNr+i{O`TP%uH59a`O>Fpo@l z6?GObWOt{v5s^FGwbqrW=QLOmwUvk{N!WCU%M3>rw|I8Nch``hBLX5eX-^CIsi$#Y z)}=kep3Dc5N8{#AAE0g&Q3=93NiBvff*;Skv&+opU+k!px^*_Lw-&1)>}r%Ho7Xv2 zj#iyW7);U$Gn2x}P(7zUIx-s|wsoIUi&9{&zAB7pH90wvHxqJdgoL%_Dl9**zc-R(66{=47#?^vE|-Wh5?;{M*>sRgEe4&%JY zC?2!kPd@$_zW=T=Ge4`pAi`|@@~q6{e_~lv4u901WrWirm@_k;T`MC8EEYD?7tIw> zI>oiHQcEe*G-;`Q-|L(bkY_oAyAx;3Db3b2@OV5fagEI^U{qDL!U#+R5i^*p)*2IO z1~(_SNHJMey{c*fz}yl50jSmH1i_2k`IMgHM61;5oX3Eg|GnF61z2Y_V41xPnhtQa zC&TdB=C6&B7hVFIN3)l{>+D{?{0g9-Z9bh?Kt$6tO=TpaIkJ_9gXWTTsMcu~g*Pgd&#$QK}HnO%p9oR33E!s%tB2(P7E^Dl-S+tREX$icU85UriqN0nB9x&!@CEa z?06Wz`|gMT_<#QC)vH(6FJHL3sC1h?#^iWB0w5BmcKBeHoYOdtzC~MrYMXJN)elk6;)3t9!ob}r$PvnD2grR+x_xr8 zT^?yhM{`W`_xxQiS~Vpio~1hGO)AnXsd5JrpGE3C=cc%T(I?R@dJ$GwunKQ+eM6{8 zv!7Z!f>YCjJtnCabYk6$b>G9S!`*R8<24%%Ge;t05$@Z_0N@nITp+F%(doebzv_ zJN%T@Ww9xAKMVF~X0$Q|JuMe+`A{xk#rSjv59tIf?1EF{G!Y5ug9Y^(rw;3^CX=NYiu@Eba!@Fad#)9MP|}~cqYUbbL{SsWm#2q**4aUD4WRJ z+C2y{x0#! zZBUq&XkU#{B_gn8&dUmR_ga~mQ%a?b?M16K%FNty5x~?`Yd8gfFpkqa?6$b80AMOc z)hRIvGbc(6kdUxsCWpmE7ofOm=~5T%Q8!gR?vG~v@bK{P@bC|R{Kr53(?7j<@xl@e zLz=Vk0cdVVa|dTyepVJhS;OKyZqG8L^yu)rd{_be+5L|zS%1EO#a~?>1Jje0H)uF% z6KQ?C{yX1(>hFB^UgWut$@!QaZv9F(=q-Yz6Ni(b z!`+x!8|b+!tkzPBmI?`=(NfABfN)k2`LeH8NqlAo)AERQvyWX zzSWSEd88k9X*LFna&M7k+lnt+{?<88cTi$BcaCwJWs-IZW&$7yyTfKVtcUv&Gtahq z3t+TnV~N=oLZ!e^66wsV8PN2k0Em!+g;_|oNDI!sNRpNX5Jv|Fw0d<@GB<=NAhXm` z(1N8nk%KJ0%5a2pB@)Sr6J?{6B#s1dvuaRg5GC`9${qC%FQ^&rK%Mk{dhk2@`R8Ab z$6*|%FTeTf^_$OLym&z*X6Afy5ehz9bvAQcz-rzU3!p65PuP9Jg(rlqZ9UKOgxCUM zQe|d@Gay@kSi|eGqL%IUc&~nHc|?THep<*C0}(GL*3En1?>#@Q!RV4~F#BrE-&6B* zzg|>$@w)$Y%5(F00oISFV};ak4?f+((1Q4cLTk#mt{F@G?)5kZe#NQexrQz`2UF&> zh$-6R3l_`ynP_S7I_=ZCZ!Nc3aHli&nR!ZVb552x!n;D%(N6RljS-QQx^^b@Wy73V zm}1U08!cgXpA+f&?2Jnbf#xuZIJJKvl#Sq-yIDI~o7>EI1p+R#>S7p|+9LN*yN73s z^_VSd?IFixm_qOT*u=h&cw!hpre%tQhf*==n&D7#W~yXfi&_z35pptv zvTxKXISX|iC#G7o7Sr5!KmYXhVSoJL=b!%RPyhVKKc*KcOIp^at5%t%f=kY`^&$kG zHa>};X4U$7Pd>2=)^oaSAD^!3xj&g24`!zIUO{Zo6{+GciMpt0xI?97R%w0v=^rRh$9l~bIArFKF{9o%3IiILk^ zFgMZZ`BF0@_i(sTC?yZuGN7ezz&uh*S=vT3XfkkW{Ty14A88uhovcN-Y(q3hJMb85 zZq|HpLRb(HbMM-S%^h%eU1zqO=Z}HvEomoanl#IpXg!5Aledd`#eA}Hl*V^2leo?u z4k0YGO@PzkV40afvCU=^)+VJ+3%Qp(6_oE&D94?&3XT^7~VR!ZGX_RdK5USIle7<_=8Ah+JTvYq3%VVT|5<8Z!Yxw(^ zIxotTv|&ylwWQWO@z)nsE!Ur`{5@@0FZKVqz#ZlBJU*UlpAX;5*JbCSA_@gEFnilC-bJfquWkkr0+_jaMaytT6k%=v zfL$v92Z05|ZmkzNT?9pG?Po@aq>@U6!D=3ussOrfBb|gi@FMkce2|)vPLv%!!z>-ybB)uJ2&<*T4Mj?*9Ja zu>a#9|M>ME-b9`UQ$zx`wPNsMFP+WrY9B5{alHxGd^fFRCd){~X`wDEynq|` zOYR%Lwya-==dr_AUw`$xmKQyy4Ub3lf~A=E?*}k<-Tuc-y&ydAB|@Jn@BXxMfvs2# z%ah~N8nG@|z~ju=+Rx{74c~EdTBp1@>#^Aa6KhNyWm;M+%Q4)S*pCsZPM5u1rn=zl z`V|XyFJ=<$`iPLPXX;#x`953uQ|RZYOmuyC6YcTCgq=bzsa$?-Uj`nzww`R2`=FV=WXkNEBIIKi-q`ProsIU)WN`JaD9vN}KH_D(;^F&Jmu4OStxPZ+d_@|it*I8p)vSj77PplTeUG=(C* zn>)Fa&KWZ2cDB_Pf&dXov-~D*(=f9mC8HOh5S3$CL|L{Fqm)d;RO) z_!wSK)vi_}o_SFlw~uSWntcARTrS}IxoN$ai1jJt(NsRWFjAMK)~ZFp$-LBBp(K)b z{U$v2*j&{_i9LZ(6RT>~rW<59fgBFk2B~ZoGF4%6>Z`k()-o0CbME>c&eJr(X{xol zrQCPhO`@c3wbqWlD5X>@2er;>?oG*5*x4ub>Jfv3;n|s$J(Yu*);gJiNQknTm&qcV zCWPUtRxNUL5|VBSMyTu+sN@cF_ZD1~(*`C!iGqbY7TFxW15*`JA!Fz!6>=jC(+CHH z#EFeqJ!~9b91pRFISD5b&JYIEaVjiKECiTUgNHeHNg7Jo2(w4j7HHvtk;%ke+}KTs zm?=&5F6S=i1aK{C1|rVf1qPYdu+t#|T;U!O*x7>;SQz0kvyj7!TQyI!;X#Gz#63uy zLJeySW}Y3d;A94axmJSbL`*N!H~am5e;CpMc^F&!c>Qe$eGv(?$*pl;Nf!1 zEgBfRx3J7CB+S!P2mop0JY9()iP=e<1Pjq^do{vOwoM)%VkcK3Fx=uY!5rpZgL?%< zW;-n6bX{?0i{ugjCGpudd(njvK=te(W$JTqw#IZwN+Jm)9+pwhxHQo$rnnF~G(?Vd z$?UB!&8%yIrC@o8n(y;twx%I9#cAm@bmH0F;g!fZvAYs#yq#Ex2-J)kICo48l;_%i z)|dH>n_*8x3a{#;n>!|PI9XN6B+MdCRKYEUH|qAu`q$jRx`WCVs|-#I28duFQYa!y zGs7f8JD3v0nU-kg&GMuzp)}Q6YmK06?rs{f2*M78!&5*+oXN~w4aJ}aHFtAnXLbs< zF5B$i;ZTdnB<`EdhDpq97zV3#7zQ|p5@}v8Zl#v!o0`?pwVD-nbID4{xw=pg_J-HKQxDc{C9Kv080?m<5 zwA2u~3qWqW5n5K*pa0FB{`jZz^*{dMs_Ts$c9x>Rfe35)e(&bNxY3~$tzr1*VbvF&ep)Kcf?BH&Rx7l+K&Q!_g74wXVn$|8Ob~Jn z5j3@MbZP{f1Hr;5Tx5_8AabxbV~c{x!!QMAu5L=M0L)A@t7PUxNX$u)9fAx18vDsy z{FKRoLFCR#;V>8!W$J>B@}MvrJeHFaNB(3G6G0m%VcjgKakCj5BR(QTJcyGU!z&Rx z1(|_aC3q~Uu-FKa>281#hYg_;V zQSN&Z8OL#&rlNJT+4Ox6X@P^+iVYGLXDKWa*<@nmtC@v)U&NDIBtQVN(BjU@%sJ<^ zcw;g1n_Fn)>BsMX+3!bl+H9}7Zo^$aZ!-sAWFRv#I++wbvuzr`^SmH0^mjpCqhtf0^BQk3Me3);$18!Pg8ifA~l?H z7mwkB3lUndLFP#2N#r7kqf)I^QafQHBrrrE1(D4bfo=LvubxGd)GR${b~UDDZ3V%L zvkNU@z?{gr`8Kn&ha^Z$*|eCBwHTq_Y}!#h0%pP8jns3_)RJ8kGchyOb`o561(CQn z^C6p^P6;w+wT=M7wAw`$ktPB3+#C7$Fn#xL-(B6j+H9|1f1cA~XH5>ckrTPLBtasXJ7 zc1CnIBrA~(vVsNf?8c0UE51@eSi^94YnC33_cSF7OAuEgadGj?&0h1ABEmg5Zb5)# z1P*X_bs{(Jmiay{^m!ed8EByvOhl_Bv27v7XsfDuU9fwpkR_>^Go`-coKmf&lwlmP zc&l>TsG)Ne76=H$oX8zoj0AA?$T>$~fS5S^C@qMS8AQTJv3Vpcs>j(}J-#yvy}Eim z)q4B${lET~|Ni2|tE;Q4?e!I0+tG$cHnZwgm;MwC zoH9GR5L_Gpj9Mnyko!l<2qq^uBncA<$t>D!Z;3NR<8zrwB@RLsW@o0(M5Zhu=pbVe zg#O9c0CT95IZKi`zpxS@0Z&MydW0fsfua(Js07ShnbK_d+h$pFQzjwSM8GLXYb7D~ zpoA?Z6^SbMK09oO92{Zk=&f;Zb5vM-HR{qb{qYWKjy&yv?^p#nnD_H~0j)8p!9AMu zH#7G_5s5~!^r`Dq4+Dll595KEQ%X!CEc4PQ(;DkfMBFvP9J>3G_txFlk{g?vKLInh z_)A1Yq``Nhaw{B1=2)RIq; zX)Nr@i9oK1=#!0E@vZAOK3O8plDnI_fsGXKfP;S;-)rie1Q6T7y1(8Fm zdP+&88CExAzmQ*;m7*NU7y++dYeg4|^{_N_a$^RO6rIENa=$ncTtE^F!YnKTKxBz3 z;6&=!>|qu{pcow!7j`mpfr+Tt$zjvo_X%l|ZU){AB-YiARmgChcPndRn83@71~ zQtCQVZIU-tg)c^;RvLCQbC{^k8|-9*fA$x|EJP=7mu4p;X9~b&KW}YUk4jy-+FT!| zuCA^=`~0;*Eg}fRl#JluRTl2^;{I$}$j5NdzAk8T)~Mzm%X9jqWjvp+ zda=9em}#?i`rHk!=+5)gT?**n&zilb)vh^6eXs4 zOU%OA1}UL7zQ*>RyRC<=)+^c`;=Nq+_CDwU(mIDwhqc z?qp#lLuTHE@Lk!Q4AK(6l;Om0L3W{TVZX{;l0f;(J^=F^s+ zW*tBTIY$$o>*8<(z#U`+IH=BM{8-MUvm*~XU|z_?tRaY==Pjbu={Q8mGr4M4!rmjy zVb=Cv9Oo8dA~e|!+})3Lh{_|*U!<>0DWxt){lyfuEhi$1Fy+K8%LI&?eV)sLdd{zx!7^ z%AdIImsID}uz7e-I2Cbzp26-@wZFNz@S?)a*zqZfr$ zau)oWD%DR&iRGG%9@N?~J{ll7ZAO{x`e zbC{MsbKh-ss>h1KHFxBoVn!gz1UOd90R#IbE zp%G;=lTEkX?dsjZIRrcIfDuQn;q zYON%i`<@d4rdq;wFizjKQrD$WxP}o3b5cWCb=n-}JEcTmGaJV-w8SZ;IYqNg(*yvd z5o0+*s#25kVVb7Jh{)$toBJH}5fSK_S%T(~HuDm34f!J08_JTjJzd1iZKdgkNfbEJfvv$e#{8fmq|{kGp+EzVhMAFfhL zgnH*Wc~jgV(z(O6Qsgs{$hEzAdr=EAx0F@0*R#{u>PPONre{Y4ix|b;LT!*4Q~@Gy zR_9Eo8t@U^W$R;$$>s|^j^%`z41bU|L^SqwkCrwtGm#i9;$#q-RLadj7J3olWGw*q z)9n`#oAY)>dt&@+F<}tEZsB>xM4d>jm6K#9$zo30qGMnYIWFFXQ5K%*qFfNp;2f7z z=~7ATyE6gZddl31DD~2dORd!nbyC1mN-5{A>mrCOUF#E zrmA%sV(4@3a?WKspy*Ut?~Z$ShlrZ1Rw#s+neT=I^_2Qecby4ZD^&yY zReyaPrm5;Q7LXXRq~s)Ht?%}?BDq@G9}kJ8?>BwlU0q!fj1#@Seo=MP39~@bm$x<5 zT6HQB8Z>N+QT2X5rrf=_xhC>y9LH&7&evBj?jMHC6ux-SjW>y>K5v673F&!S}<seEF?f_4k)2dDzxFJYhWUpCaXWuKwkxXZh7%RhAbq5w~1}WH2@(IG~wG z+)S&FZb%jewFDOW)W*;w1Il$XKE9&VNhWg^m|5F4#JNwA^&D=C33?FA;?-I>zeYDQ zCxm&Jt4*V6VP~OioRFYXQMJsH`>yZ%-+lF_l#-k_eb)zuDx@QB#(i z{)R>`rA(tlUu!X|RY645b=`KmCH9Ahhy8FI56As5ZjhZstxVGdiVR^=;pVDdQKie= zOJeb2ilRr?NHr@0;-u;at5Ye}m7xh8=FBX_mC3$zY2-CTB8D&pCy-F6Un8oof=FvI z9v#A_a~!RPy|pSD_v&c)uUaF0BK#y`UO;e(Gjrk^Xf{4!Zoy!QLe1_G*BdM1A0qFN zmH=Q?tI91IgVWXQtHzwe;mu*Ca$_LOta;f4)6w^Rg#T65rVa|}l@WJgfF8td~oBfw%WWE z&u>@=;bZh!;Cn@8{Pf|X9yG8%vDF`J(%&I}@RQQ~>6+hPmiMk%%XQ^MYZ>6P>Vyy0 zzrOP1yFRHb)HK?b+=Wnd^jA7TT0!l&DFd6clXD`?)nPCIVBdpA0EIi;{CSUT;(pM+)0;YDnlNN zSvG~4oAOk39Hvrqv)PTM0Osf0`r&^6@Y&60(==6;wCOAPR7Wc{b;3!? zIAJX2nsb+7Fc;9?ELdI7piDyCNwREE0+X>3v68zNuWm!PehaQrdBiXdOGbNEZVIGN9#hjoE5cd&)nRQ(k z?Kji#^9pd6{z|=u4UtIN>gHjn(@?(sx4(pX^XkQwllRhvj)YdZMn9F6i$@kL{$LFk zH)A#U>kaSilyee&oW!Club(huJEzaS`6_ier7laly1IIG^CC;f+yBUY$>lZh_`{t{w z>rI#N^5qSa|Mb(3%%nu_4~gLWxRfIv)lB?dyMycz8{kh$B!C`VN)15=&Wj7$}!{==2fja!;#oE#0|o|yZYkd<;*O^?oPsHOk~=) zTs0$&6gvQNnr-94d1ltCfym8W$lPb?WGH_P9--2WB5ajAx$7Jzj3|N^wp7Tq6mDV1 zDYD*0gb=_T3c#(gx*`cPbrH_o1#R$+>yO**&w}$Tpj6Str3*UF*6F&~VFTTcM2rJZnvVMH1^5^PbKA%Y7Z^3sj zP@WxPMy6gWl@WDQ$)=?P3Yrcr>FRC3e$u0WwSJ3H8}Ks&O2GD zPK3@=M+`!>T8j?Sb=j;|H`7RWY3^4P7EB{*nH)cT z_vf4Io6o=e;^mvqcQ0PP`uwx)7q8yD-2C)Y`KLeq^UcjooyvdvZ~uMRAG&R~+kJ6= z_wM!US1(>&OCl!ix z_2H>;q_OEX8tLuDy&D;J^C+Rn} z*86ww9v%*581HUxfBEH?JXO-E6C^N<6bcx#-@Lpg!f^j~I6fG`CH;T?-~Z2>moIK7 z{qoB%U%!4m9LDc{`0oDp?r-057QWi{!}xFQ>8iJnrpn8SOxCw^OYw`Tn@4oNivdWTuW_8j zvPVKy;;tWE2QA}NMysv}a~Feu+co12HFYO9GA_(qs;0z_CRUad zPEX-=S%cdo0>HxAth$Rk0qV}P5uph8EF6-SC8=?&4CXog?i@aMS;s(2ak18V$2n}h zs)EDnbFQ_X_!ez3p2{#{gOrj%)hn3#zRx+E*T^)ntlFSDIR(QW(hmwfq;N{es!c3o z*3dyjRLiQ_i!RHJ|&{pO^c8u#0JvJDtin zjznHc(P}Aim-AGLd96BWDW%p3zU)BDzpZAqx+<$PAe1uGi{18%m#?=u*;s5U3`!~O zC11a|x%vFF&C3_or@hwOpME-cRUzp&!!Z2t!w>iO`@|w3NonhtOll>wTBo55M{(-~ z>Z)EvQ{L`4CxssF>hW+_b}#*>AOH1V|MjPzeu7b#a+kQzT^Wzlc({Fc_2$jXF44pN zyX)&6Gk^)s?&gq`a_)1#OVy-1Z1#siHtFW|)ff1De>~jZ-~RH;Pvyscv)kGHq>3ZikBv+{~?qD{v{WLu6kCQ8RId6CS;kciM`{VI2 zjRj^->Sku&)15n+IUHu{1ZOrPRrO57&{9E84hAt%v0^4f22T`bm{dc)xMZte@)3p9 zVv#&x33m=GS7=@+>(#s%o;Pf1Ii_dcflr4-4kv(cC0#KQlTFNQ(o9Qxj-Y*?t#75C7}>ddnQ6-@44>og35nT6Pd7@?A! z%JlB--Ti()Ok+xUyWOSS!sj;dqGhK&yQ6R}GkJ2$eG;-P{MU2v?aQmr`R;X-&hGyo zVk`Z)87Io|NFeasBfp~l^_Ayl;a8Mc0PKdAhp$*wsFBPf8fF$rDdkR_SzW7n&)p|= zgdT~o*BtjJm1M7WmR(DkSSq>o9f3-r!@dYJ(WbMGSNf>yyMwTGbReVN^fs>B){`+oJx5ZiNv^$%vV=e z+is`7_#~i>mx|rhCh=!om$Qh&s7`ECv1(S?}Mub283`hljnZrJJjKedWM@8o1W_<#lH1a_+A9`h{bvs3X<8*S$~=deH%0 zoYSkDyZ=A#uWyILPsjbwI^9b>^m3o*DD=zWIGK=VmM)w4U{ukUU0tP=OsPZH!81Hz zHZYBNb~dLyR()YG$^|s3jp{{Roh)}^LM{xkA}~jVyVF+>BR2^(LE)7YsP0KRWtfu^ z+Jh`0x`HDqB_W$!t9vz5s3qa5!*H}NZ~L_Eq?4NTf!|zx5r%*fb-B zA}k`^iW90$5YUE5$+Ts-7m@q>dke=~7b2qBNcP5fJnXaTXPa%cI^92zfJ#5<)NS&s z&8FmfcX#)<|N38ZDY=w4fBF+PB-}AO3DA+TO;$NK$2qu%6J7XVsvS@JtyxJDF}b0b z8k~~=Fn1=Bobg^J(L-Xh^6F-(VF$iMCaVTDI59iilt`0x4xUR1>lt_mtd?xe&*AmQ#3dfp3F;cQFoSjKJ<{XBxz#Q(?<_N)cQDbHs+ zfNVa3IILD$7>jk%^EyWO=xSO^up-$>Qr8oNNZ5o)7>S6$G;D4Fm{nM{TFDE$30bYj zOrFJIQ=KNA#;TL6jpHF+Ct5cNN?J-ejwX4-EW({q9xI%ba@Xa=D}P|XX(p{p?)B)n z!&~@H6CjdsGn_2qZ5SLL&M|IiPCw!VS!9N%6|#h12F$ElohC;a#&SFqb5ZR?LK9U= z8OyZzcMPsqsoPy`UvAS*utn)$8@QR8c`e1KI#sVqQ&n}meRF;Pa6gzOAurQ|+1^o; z9BNg=cDohkjqh#=Ufmvs4yJf8%)BRuWvq+b6N0N!z!RnE2x5)ug;+#m> zg^6hQ<@J_cy&)e6BXF-`@Y-sDWzerWahO>1Zy-;1Sd!_RPs44qB4@f-CWHbO5`94e_(Tf6}rSpXNMU% z+{w*DD$!TvX*ilr0EtMg3RI(1Yhg}CiO7Ij3d}5A2NW%AA~nmHa7RW2+;(p$=6Nb&ZdzB?Rj!rKwJ(l;iQRAC4katq_rPsq4D^vHtMG532ef`pxhE;deK$ zUNUp7H5Sarb3rtVWy4q{oJJR@Tp0O}spr|BKGd4$`0fiv^@E4{C(B2*el4EJbB8LW%P{h)|#a6`#$HMz+ZgvMZf7bn+~v2hgv7KGLAlu7LtUNBuP$P zB68L$Qq@o4cBRk%$s; z$0EduPl*eih=jppOpUb!w%e;s*Y7slUAMhv`3$VcO3<%gK1}1oG!2B2=wTXuzP-Ek zx<3xVrm2&0(5Ylhl${7*aj&j4P42EGiPS8n31;pF3cpGtqGKUpuL+5)cx2xq}qGC;k1s^i4W9<)cp zM5M`N0N1r0MQn1ObNGhm{S4ocxBfy9v^v@f36qi&YOP}}IZLo0S1)ctAMo&Sn5J5H z|0Sh#eSMubor_D=h%a!lWg=c#1T8r6xhpgb?6x!hH)0Gtz4rnaF90&6K=bRKdGMLq zmJe&-UW0F=k<0z7ycNs1(QGRjJQ6d%NBV!k??2n(XIe`{Yekq#^X3VxK?swBDYFvM-Kb=c>BXE-gj@2Ov;g9D^;Q8jK2R|JS55sWu zQsG3^B)n;=)oBuuE+@(fU6M`LZ8s@r8z!se-8lSw_kdwf(;zlwGZ-WenUp|ckT_Ex z4iCi~DY+J_M4-$rL}otKajX?Zhr7Efl8%Egvx%wtG>L1^1b70GDS;B@lq3ncb1Bm> z9^75FuZLkMb#w>FL};_=a?YQB@jB<65?7rLhoa`S7S(clJ5*IC-*PWPHp0A)(`07D zR1W*$;V`OtPTQ-i>y+|-zaPdjRt2S$*}&CGvFYmN_15W%uysr0(H{ro;$B< zW`nEDhQkh|zN_$Bt+{9(&pyn3*%&0+qy|(mbc^K z`<~zJ_?P=yN;!_hp^g*W04MRBq2^i83yCO9>o}Fk2^;UTNM_=W$&)Lz#tBK8o!FfP z7^o!OKGy|7IgFB0&4Rmmc%8E%(AISRg)5jyXSa%4I4J ztYQW(O!sqWzD8}V4%g4K%oJOHd8*kbt(;3~#_#v@(VxWEKhf-l-Dl*QuojxnY`FQN zfT`R#U3EBnxcCF=Wt?ezB62u0F*{vha939|(>j$p9FB(v9fvZGbsDDeDDX|+bt#7v z-0d($ydJFuY5;}BmO0=WnMg$l5)LoT2gYaZ>oNfAk!U*F_#+B6NH+HHP0kEA zEg@C`nh+-<-KLa?;gXCzqBIZ_rl}r}$Nio`WDUl%QIMkthiQAWyC97IkGP%14l@CsbW+_}~;WI8Fi0zc@gan_5JKf);b9HT0fvGY!F7N!;< zAfg+x8TY8j{Fuz$FF zxV;+=`{8)dshF0`vfK83?f|Hjy{G1Gi^eX9Gr{_tb51F97Rf?ja)uLmorO7m`Qf?3 z-q`7xV5?b>a1Bj6+(RGbOD(6UM=YBe-W~ z5(qN^hR8^un@v}>4CA5HW7nmeGswX>91aoG!fnoR4lQ+1o8|*`HLJue_7i8Olqm^V z0IszjhkDpg!!V?jC3UG6Gw4+Brg!g%@7}%}YpLo;UDpqsHfAPSZVz{~-EOzJb|Txp zyV-aq-J~==Xry=-22o{Fw`w-jQsDvI74C$P4{R{0QvqehXg;9aLpvQhNS==A_EsK* zlXTn7)y?&8x7#AqCV#oxOs`)Yr}E2w|89SHn8qW#GLPo_G961Pj?`u8@>`!=GZ!`L z5<|6AE7hwLm?PtjfmqxA2q}LPLxYf9zyzYHD62!6YXlXd$YqfFjhVSCoIA-#RaI*# zp3JPm%%SF!DY>zmdCo?zums^mW=>8H4UWRK@fE?QGx^#=1~{>|I%m6HC#@$UH4BD1 z^cwaBW_z_GYT*c$%?1Fl zX3+Z--~ALky;q98CSFhF0zYxw^T~OE@^z-fPvyJk{4XZ~#YxzFVjlhC%5z=#RKEMH zwZV6L3vaoo`%X8^N&}*1!4#~ow`5XeLZ)yi0Xee>$)y?lB9)igNP(zA!j#-nkgZ~?CasV zkeS%|PmKbx3t&w#YpsLabhJ$ZK2M3UFO%;pWkCBm{o(gtP5WcH-}A9-fL-DZr(_t0 z!Hl%{gnB5$UdO>|g$rLZ5phbw!eTmAg-taffL^u;yOi~JKWsJRda6@%!QB#b`T__8Kk{7jFk6MReU?Bi0wXn{Th$%^n%T%Xn zFl&L0h&|`L-EN7nySd(Ow>kIB$qfzxgeaMjDKo*yYYD?hbN68wYPC_#P^O}}*OU@V zDqd>ZRIveNZ|q&lpS`%*%GO^F!;lV#X_^dfW@9Z>Ytr6LUA1TlXGERz(bSNnGE>)r zR5SF&b|b1(#$3Hu>~yGg8Xq2ta?ZKm?$V~eO1XP+{pR}mXf~*s@M!)pjPD*E?)S%1 ztJQM#S!MvFMGsnGW`^oQ)gzjLLdZZBk>=iZVeCB12{Z!p5NV_e^X8)fnw`jan9;JJ z5iN4?V9e1{>k)(nPSRa5O|^t6%k0iWMB+}hn%U%zKJ|6dKmW_0bI$)MrGC4qQz7Bt zk2wC2AqHg)L+4EH1(1D=rC?)qSD{4P)_n@NasJ_=lJ(b5(pA#R|7$Jq2_m}6?S594 zs+PB3P=e>~KlQqe;EC^E1?6=jo(IC?gctuTWF~}J3v3{B3uy~bty(=Ej}LeE54Z1* z$7B8Z&$ZT4N?_$Q)nR`ahoOvpHu>>Z7@1H;n_1 zhX|C+?wm5vU2xnVB4D$;n;bB-bW!SX_XJmu7|{qj+V{%4|L_0Tf4YDB?#F-o_U^kM zhlf4%Fc_#>wd%|&G_V)3#Mo>)WTB~tE5q2q5Roi73=WJUU4o%jrDBlKbPWb7>cwG1 zB1z2E^>};dus-#f39t;>Nb0Re*dx1|8^KAK6EVr{yW5mG=WHoWwbTl#&Vp^%he=gt z*={#|-_KG}l;YKDokqktB_(GjP7W^B%nJfKi<$ENVI-O&oj7w6kxkd<++Xi*hH2a% z$KyDRby8K%iAA=%&Gu@Ow%tUw*MWSq-F4lr`)s#891iK>VHk!|N|`1b1{+7OmCdBq zG)Dr78Q!TMr}2I|99)&jbMgd`eLLOdsvE-%<=3eLL5HL44*-}F-M`CS-)*-2_Nwc* zU6;32yWLf{*;TXKyN93O-rhajk7Gff&1Zd0dH=cL$#M9|^8S%|^4AkY&g^cZ6y|U}J{%M1 z=vv0%;r8z5pMH4z)6WkN58ZU{bI`Od^<-YFnNs}I%H!y+ni95smaZ#9@uYPsLL|bQ z%_bru8;vd^3MJciodLC0BO^!HGP_|Mr)F0`FjLjaoKq4r zV-{0Tg*J?p#MTC%qyILy|a_&P2v)k=1b-iv!g?>Jg=&nyU#BLmi{xR(;SRS5Jmr5^y0SuA>YGxxZshOlh;( z?v7)+)=jtRyX~9LJG}Xdng8;d-@bkOl_w*f)T)|o zu4A1!%*+)el*y}^87TlLGb;&;3p0xeX3IPVIkOtP+B8*How+x+suiQvqDi{0+wA(j z?-Qjuh0&l%m?ScXN@r0*QrAKo${f?gS2@FoC;~kbQAD%qy4YJJE%1lNNw$wz>c!Abupc16pirAUMRgSs8-6ZJ zsFsU++EWfDN-3E(TR64_H$2>r!(o5_aDRKdzrP!A-`Sjypiey$)mn8@ zt6HWS#w3|D=M7{>sZY{%Ic;+1P%U*lj_>Nb!{OLpUGI#Q*d=$@*IxB_JZ`U&;ncR- zmN7=`)cv)#21MqL2+(W>HG?rIGQD_JNaRL0RY3LPFtmIIkbe2`Csg&J=s21g5|c86 z$dd~jqr$`KfS3h@X>3u--6kR-FpMD~u?&k~hIo_y)M{Z_<=m2pMOb^!BIIBg8&fc4 zfWcx#Sc4B><~7{lu_~ObT>yY2IkM%}s%Aqeg$2w!j-#1{6&ew{Bc+t2`|;;e^4uZ&fC84 zbNY6=f4P16>a&+`zJBASKHS~^^23i4%0H+R_6_kAT1vh0=u9b`hG7dvI4rPDL(g}zmA@+SOyKC#de({yP z_!0so9JLHy|Mt_*cRJnIhHwIvin|h}q3Kn}HmXSf@hHZp6;+p;L=cC}j-+%f=qZF_`fH z%|HY*D|Ki)pUJ63zl77v7q8$nj-!aUlUKKD=9W`ZR$;EtX(+$^_|wln{P@KeZ?@N0 z5zG~NseCuf23vU5fV%f{mybDCA`)*>W{-^Di>Txofcw-kOKGsq^|MCy z_cZTuz}&tPUKYE_}6E!?s>i=v-AqY71@w3eYzWXvHdV?7+l zYXjxZC8yn}wX!4!n&mC9nyA&;YnJoC0*>MCMlHU6BzABp%$>|V((p*sqpni0hrHmqyRO%XXG!-j3Pmoy@x z;PkuHcd6%u?RLBAw-0`}z1{C0j#iC`UcY|B%w1w8skK%u?kd1r+(WZZ8%>Ei^09qQmRg>MO`P;f~w?|y=DTp0I~q!1|zXbfEcGz$}k-s z_Q{1DnRDNDUDu`iT~6t0)8%bWls!Q+3j61t?xyLN!~XU#Jd`p{UZ-N1O3Ij%KP$1B z7ihJH-4-~-)m%t3n7P{m2+3tQpCcsz)1YAjYX)~ya<3rv5vFB=xoPcEC!E=dh)a0; zM5Ozb<9D#ws=Bz71^UF5nP3PJ<(vUbbwHB|oMaaux_)tE29X28u%-DK3Easkc*4W9 z|M|zC|Mu5!Kl|ddujK2^t_P?T6S1z_5NWBS;3RopOAOYVYG%ToWlr%7aIb$2j=M3r z)CBFkg7US-^5aXe(qw+(YF8QKXax{7bK*;XFPe7&%By}|#+>r`sMOd#voOLWqBD{1 zYV)VWw1k@GrpX4JhB3}xQ?2FUaQDmG+h2a34##i*@>db9G!?Pk~aTXYWQVmg$m zOj@fFgpr2h)OQ`_O$zO&hc=rMC^0iAff8ZY#t4_kqNfkN?l3`Yk~3txQ{|G)0`XrJurH~jbd$l;wPWIwTZW%L)@<@aXJaZ4a}&z z7+F4Mm=_C>3z^CK${M-ad1EnMypZ8AQy31B5((gI-=z~}azqZfTC1tHA%?pYqe5M zD_}yTZ5B98$CVTu9;f~!b`w#e8Y={57~P9E*^HS{5@A8k$rAHEn#&H@+B!6VR1F9xP!}Ns0m49CTUHX}h>SMnQ54#3`pvd~_4?JzmoKlbcBPd2`}^D5+q-w~?(gr1hyBZ!FT0!%5Br~g z{3)}1^Tn6neDT%hYWE-iZ~xb4pMCak|MkDU{q}noD)+W8_d4Dm2@@sd4$_PCo0K;> zyL(TmgZ18bHsmS~bsV);70hZ42APu(RX8z4q-?ipjeV``)FBH*Cg#Zi>mJhXuciUHe|1!V$;_7;MS!%8S`NyBD>Zn!8)kqB!lmkKu1_l!|gdi?VbkL(a3B*Yu znMi_L6v>d#KT6m2KDwDPsLz?)_V*8e{kLzs?S>P5@zs|prAUuw zR>OSxG2eY58+pH|WZB_MCA9bx5;4urcENIb!@xDi{dX3kMIL$Cp2uJVJk58nMd6Q# z&=c7GOSzyJQ{AAi~3-IsAnARDU= zlbaEQi4wY$lIIP8PEyx(aWWgnu{gAAO(JuJ7FE}(s={5CB#GU0e?J`eL+-o0$tiV- zl2;#wX*`V6O%-Bi2}5bDR>gW0AAg>LhMCqfY8|Z>MzcnZK=w{xH{#@>UV5e~B1+@_ zNJL=N-PqwxrxZ3J`y!f2ASQp6jgY~9XBC~F*QSBGNwpF5mqoE zXJdBv>PAHB5QsjVxC6)EEEs2;LGIF#Y!#^$8vwKyk3^+a9H z<}EC$5sVN?aZd~&k*Y4(O+;aTvVcN-4%MZzhntVW#i58isXcFnAYKnz&D4w6ZNC9f zt$>IGARou^+i$;(Q+2=HynOi*9WQx*Sf1v)`A3!%Suxirnev zUw--d?O*@$m+!v&?*8`9ie=_aza0-FE0GEmL`ae)31u=S0K$@qGqJlX$lzI;ZKAQd zsjz{J$bif&38H}3YV0Y2RCd?bUAHmA;o)$9f4{lDN$OO5+n+GIv!IM{*LT#6mKv zSE~*tFjGzpH}&F1L~PC{Ejb_{lVGsTDYhj+z;H8%fuQb0u5NH+JQCqGg+(y$%hA%Y zh#esj7(EP)-9nUNB#}0hD5S1cYbgxbzRX0l-D;h@OeT`uNX=*yx+UuSzVEMZu6L!3 zX14$7tydj2T5yS>`o^nL%$=9_UNL&_8E==FY@e%{}`8xN!T|DUx#?Q$eZt_49^RMpG?hkHb1 zWLB}r$JgwhGynhB@bqauzOG?aJwrgT>$wdf^VLnz$s&Zm3nXtMz%UH6^Bm)f*9wd04HT z&*z{2_9HTIPXGMR|NQXqfb0OoUoX$wyt`+|H>z)cFP#1EW6$08;O6f1N)G(?@&9ww z?h*Os@7~58_`Ou|ck}K&HhmM#RTaR{)lGNerkd_9E1f$5gy4j)fEetV8UT97cH*C{ zwd?iz^z?K-pY6|o{_^F^hhKhuc{!tb;==9(00Qh(C_ARy)3wA%TA&k_)PaN$Ga)dM zL)f398=^C#Q*!5d0e28Fa7ZaMCij%XQJqT#onUa2gfzpy<1s;8$1p6^8mc zb~iJODo#aPkKOx#f&l_C8XzS^2?N5v8@n0M5j6(S-4xsc{csmoYu3!!p*WSCi6Iw~ zW@^oWIbq`DOsKjnHPUW&BqB#+j5(9gL5ji_0XWL3?KPoeyk>J(^xv{2JLI=toY*6< zEq4hI`#2X~bA=_=x_T1xAu^$QWHV6Dv*D@aBJgLQ3*aVkSj!<#qvC9|2h zJAeNV^EzKHm-BK~my~HNC6A@H1}4|*wbshar_*T|Mk2EF6@U2QZX5;u(=Wd~j^%VZ z{o_CV>G}Ej!>11~FV7FhyMOxM{`+to?f-<;8kvwnYr2S6&_|eAm+bl72$SKR(8*YZ zPR7MBJ09*PGpm|}g_*@#15wbX%|O(e0$8W|yCIzKl1p=`Zs1 zQd_TWJ}+}AMdu47F2`xSzdPPNyh(UpPtUc!)Ol4~TD`8zQrA|snVPC-R8Z`M00KyT zzke&jMCjz8E)6LGytdmNlq{*LdC*ZH)I-9Ht#{|mHlk-=$F!Lvli!4Izv)$e{qlbQd2ADO{3d*xA%I~_ zLq~O}8t@k)!ZroM253y-#{T-}wwL5ks|E#ARoCnM>C>m5e){S2=g;(i{$-x$d7cR{ zmw}PiWvz3~d7#LwkP?^7DLUAEpw?Pz&0QIgkT`Lnaape5fbQWcoEXD8HJ8yuYF$i~ z0l~4>y0+@Ptfq@eW1b!!PTze;Wk3%2{dZ#bH<$7Cx@l-Fcz1kN?Px*7dQwtFAnIf``#BFd_j( z$hO#n0NNpJ=vomNI>f;U16Zx?D-~dy90g}0a-i2xjxown&;Y@Jgusv$b6~v!*wTG#6CZ~y5e z!voy^j30jd{OPehy<{e$(yGg~Ibb~_lw3(Z(>#ncP1!+~<@(1l#{r1ygJ`art3Xp^ za?k2irB!uxb3iBLFW2qf17b&*7a(J(y~hYdz+|XbT;_Uy$)$`o=H)Yg;$aw0$Kwyv zFw-xCYuLc4!&f#xUNm#J-kT<%tT7q97V;|RIlC= zLLx{gB1-~GV9d^@h>Tn~F)AwVZSBr>Zt}Y^LTcH9)4pc>r~7-prCGz(S~|l~ouL1ArPR08ye$#K!0VP6&jpwMaqK9Gi8rv(xI; z+ydoiBtUg{P?OMTF(U#e#30wHr9e}DQ?uVv;BOBSrA@)i9n954w5qg7doZMrNq}?m zLcksdPaBaPH|k4CIbKf?iFQI{#i!^vvw3R5RYBAw7{u&MkR8?B)RYvNptu@`RV6he*XFC@sS@JA`%S%u&gSww$@a2U`#2MQgSIdXWpos zm@*P1rU3c?Kx%8PVLG50lBe9GN=Ta>ZC%>hv>6ePBac%i3Y8m~tF|Q{2CQfFC(60{ znV0)6?A&F@dh0@T)Ae@R+RqLE$vJv3tXYD3eN#{}7si39ATuHw5W#Co`pv<5#%}#vpBP zy;_iqg$Z#u9*?KvVH(EA=XIXvRu@E!d0!$VqWk;%oVnIoYele9O0CPfuFFypG3R_d z-knazaU20nRbF0RuGjPVtk>(cwYDrvDTSGb`*$Jc3;6Bgp`;=A3Oo(oRQNGRwf^^ zhNhOe`yBc07Q(vq40P|Cji~`HYVH8=1uWN=wZ=D6O4DVU$}mmS;c&<~-#G z^e_MT{)hJ@ayQ@(=Srcuds*z7QB0CErFo_7etF_lZ zv2E^6zSK8nLzmM-#D)zWNWsie)J43Bs{s&@20fwYiROSt+hO3UU}|PyhN>pgRF>=Y z(}!Pw{q@&RA3t8sXKRu+hUoqVbIzf75t}V~lEL>!L^B8_=P0(^y)Nszu0#aeiwE1B z9YqF1Mab({a#66sL?WUUmt|SkRYaKcK64byY)49R4#E@NJ_7(^@cuMV2y+EQCqQPV z#BmI8O33Vl49r`+`|Yhv1YO%aSAHj>HfMF|`R5QvD-5IBf{7RWWT^MH{m_z?bVIBNa|#$^++;(m;uc06c^POU%Ja^VURUrSdPy+;aH6~RhGz3+3Ch#tW6VN9TlD!5q{!N@xRcsEhiBb`v z+H0y1f!q3&V>g}$AZE^#Rkby07#?ZjW`vlKIycWG5gXX~%hR=8=XxH-{O11NoTF`S z5Ch17h{&YP+zddyi;(wnGhQfdQ5Os#6!EurK|!MeUV<3Y$ zfTli4FWda&*L%b_Ru+6+Zi|bn1doTxR-Oyepm03sTRAO^Pv0QPjd2kvdPn1wIS z>oj*GK?gQKHE51vDj{g2h-kziNoGlr$PidKI5gFwz;cTAa+CS;^6}RXzx?#`$6tRw zKfjn+j_PU)UR%#R5z#bFdpL?me0^CK0ALiIg+tR_AH_$9BgR0wMkCVQF+v>8;GvO{ zkS=Kugn^_^V3dF~Qqm0ae4L*cTa_+I!w-UEh z%8B}D$E!O>AFWUD%iv*c5I-XXhV67>*O5N0KiccImukYLnm51z0J&mpdt=EP?wAeW z&`39OD)pywjtG}Bem*1Q*7O1dnhF;P#rtz>HR`XroNS3pq^9p+t9?McLueH})-%fN zeA<`1_eQMj@{4_{4BVW%Ajg7tE8bV#7!rwQ{Lt`1B9#(>8qR5=j&8{Dop1q zNRTN*E7*zXvvkh!5n!tqW&V}jJb1PG_x$)~y)R6WiMr)#a6jm=se^_cCydZGu_&@T zhTAE*vaG@s3C%%(slXz>)QP$d{RzJo*~L(5oQNL~S-@OqEA+nDHBpzH5wDiXH_|kJ zE8$>bj*+kW(C*OX8?~;z;FLJ?ARoo%A)5K=^W5#Gu?l{|%JwoZRU=w>BjFN&nwU}* z{ON}D=xdGD9%rVvt%7|q8(XI?Dnk9$ToP>#8*gNqV2sR}+sXV**|%Xbk9aCa?Ej3k zx?*LIP@5a#&1ddrxsw|ACu1}8?%#UcC(MJ-P#@O&E$&68;hVV|1imh2r#ceU;;vRg zHYsA=%IHxPm6uuO5#_(#Km<1Fq5i!%9l!}4m|i3h&;{rNQrwL^6*9W!x0@HPxC1;} ztMKId9X>rO`}aeuQ(yMzi@c;%F^r`%kvHdO|T zh~d3**9aN-nnm#N0~t@CL)>k53NaYN+g2^78A)o5dURQIZvFs>doBMAV=)Z^1{g`fB%d6|u+=9AiT9iheLH>(In6@FWA$f0&u zc!KRtziqO#O}b?!_^5d=R8dJOtkv>G-bK ztfoM0%>WKpySH(5uksIRJHA(ZO!jJns8NG3GTI#6Nkyxh6JLTIag6$*e(ko$tzuSW z-bF_}oJ zV!C*G5)Mysu)li=gIX(88LBrC+%3o%z1z8_X#7LNY z!(T_PAJdf+ojbKdcr$UzqLdGSnqIz?re3%Mblw4)UcFWeO=23aK0E*V8$bzn*}#{1 z$!7T1X~|q{+TZL1y^8UpIwv6bO?ABMhl`lAk{l_D<8WG1y>eR#xXAqICt>$$qnomp z?QNTyF49+<9+5_p16y$S(%1`Ahr&c~u_2ZIw{^ccwxk{!8UO3UTN3lVSP78^h{Ron zg-ThHAMbzYw&q!q+Ujmn|^ z(WRi^3GB?7)ej&`eMzT17Q|bQ#0ON5!mzKF?Uvy+DMDZzjRw$Rb0k<(T&!k?H-A0$ zpT&ynbc&JK2H(nyNJc{s#N0je5y4ceGev#9j`I2G5-R;G)%3$r#!*HiNhBaE;+5Nn zMoA+8`DJ@Khq+*ynzZnHP0g;OP~)mbc;Or8mFY15PeQ)kmGQiwmv-EKMczz0#wJuY zqtrd*2#HUaU+Epn^79OBK$8D#mBjR z!V{cgQI)Z;UeA~5Y*p9k`fkBqIaO=N)9LGgn3$jcEOd8&HKmvS?UMB zZGB4lfguagj)y?SR=NKrnZP>yvrn%B+Ro0*_d~v#%4{hlsq2GW+&XN?FTY!5XI8$0 zv}NJ?M!qNXrx?YSZggz-kKq?$ztELWxm}$Olqm&lee2tvn_NbsOOGM4trsKEJtH1l}gA}vd zJT+M&e)Tj-*xEl93u$7H2h*UlQrk^6VTVoBYhYh~#k@nDFNYfI;G|q~*$!p>Yo~M{ zCsjwgZl~Ng&-cd1$ANib>zS#GBNGKhuf&CYf3^-{4*Q`S-EAStDUth$Bt!PK65KB& zVVvHPfbI<#4l@JQ}F}`J7a`*~=)u)h=mE_nG*YhkH z&D-#^RCJ+aC`Y1lHYfM_YD}ln6@0I=GjHr?Kz$~CfCz^sZ@gG(&aeM|-re2ZTpX$M z<9_j1fyT%uuM1N<38H+R0yXt2R;~6SRD)N`_vg*h+t@v(FtE}ZoYUGSu*2L zHG3NRO^`NC@-KCzS$`3uVr$52f&U-?RVQU_H2hkMC<3{AtPWB!s;A!OvePoVO2>&x zsBEhM$!>WZ*p)az4;*d1sXX$iaNf5CJ6w=Vzy{PQMN~wM+J8=uKiqphO!dd*l&;77 zzB$+u^yey+zGx)TQoESQGHG!50ZaWsDc)ulqeEbS4-i4nv$WuUKsdW~zAkn@uFR}J z`Q;Q}u+xMtuAi{9ck#)s`Me@-%tAdI(zqCE*K8nH>?e8;<_;a;^R{PI&Zmq4)&%nO zqBxTS@gHN-oXWO(M?^rls3;OJBvaX3WA;*Kc=ugk=briJ|aVFT3Viihr20fjMKKX!wd<#N5rq#z)%m*_+ zR2FwWz0&uk2-r$AHZ8$7DEca+Ewq4cfQ`W@`6Y)`!{GjQU8xDA=)SWwX zl>9KM;IJ@y?sxDdmuzQg?M!E{^8u`8G6foCrpcj{Xt`VA_lMK3$2hz}2K&Yb>_=9| zt^)L_Hkr%{GR?eFmbxNrwCM*$Qn5#(KBu0Mb4=gPtRC_7c~#T6k=f6foxV&;aAAn} z*9in#C5soLI6W+1BQ)E39W2c}c-%eW!pnCj;YV65iL0;YEt5?G;^y4IpWySYyD)~r z2L$$x`R9L2)28SKk95R})f7Koy|D#>_@@Nh2irfQs_eEzeB?h#nj{!RagM!y!m zM}-HeR-+_bCq|70SXExuV!b)ob^BU2XQ?A7uG@u|vLtt(>DyV!Xc3Ck+4CuqnNHl$ zYahRJY_;CGR!wus=q?CNs!@atwsXH`@uF=6@*ms5MjXbwL9m)KPDE!!!r-V6&l*`5L z?8Qm>v*yu39wn=@25#f7O0Q1;B3&iuVtJv!RoH%hE_l=9sjH>TD`nAPW- z(o!*WTqvcndiX0ejRyAq^y&Z;IOxK3RN}($9MU_J<$@Pki*AjOjz_do(K_Kq=9~%x z8%+SjE)Th!Q0QP?rc0H7o)A_~j@a>&XYqBi(~4*MjrPUlI#)j862L3x*5_KehS}o--esBZ55BOp_$b)-()Ia1&y%&bYJ4a#o-S>Iz6W3l z3rX{E;sZfSdjachWq9wV|E>`Bj->v#h%`V~yL;n5B}P1IC@Nb?`VUwn_>RO z+O(~N;PQ&hHrlufXd8()p~&`J9e}W`iUd$PVsa>)oOLQIWcV^rCqC61W=|cXlE=PM z>wB^larJPE6;ID%f9HKlw``H4Sy@d@Yfb!cZ2aw-D6oh+DDp{ci^Yi3oJPN0QMU#J zQS-b$=>&(epRe1>j#};@8UceL83P=v6*|7#W7jVc>mQ~q0s=39QB^IHB9n%suX5;} zbh(Ulp$)tc$i=HXTfCi7VDn3jkg)qYuYM0UFdcTb`rr8t2Glo}Os+Tgm{oKL?SAb7U%wt1ojv-J^tJAaZ(nR~mh;cS*vP2x^ zOCGS^R-KQI{2|55q-u(egc)f+gX36HkDxn{R3bgq*oJ=opV#&OQ{x*+w%@>gL-7NBR9niudpGe}?beoxeLi z`tjLpN_8b5lXr-_BQ+bOOQq}!CJD0>-|`aSK z2=rK27+8Ufw`v%B_Xr&QL_`+v4fmS!G6%=Th%f+`hd&^)Z;r_ii@pDfvq@%E4AsB+ zQZFmb$7stdJkPyE0dV0B(+f*iOz%E>i1>SXKD1vOA2syP8mt}5=!bk1mhN$;G`7I(!k#nPe3b)uubqMF%3qU=S zG|FjA`6(+e1t;5wVR>Q(;<)gh%*@Q53E*6$FeGF8`Q5+OQ)#BfyVnu0w^1qtI5|vh z*SUzO3fDra(kGO-M*SX|$|cE9S4^x|O3suK2_jwyG+X@>#<*|*1Z+sW1azDWTx4lk z@SY6ioKK-xtM$c1J&3G4&zehC;sxg}Q z>py74^LoN!WDXs+#8RpuVH2TjY5HR^N^+DE7$?v5(kx5L-qw_HuDR)XnWk9L73l$rQZ&9)**ZiE~{Gvt2&j*3=Lz7KciJiPH342?8q(lW_hxH#U87 zH#2}jdxd7HYd>6b3>S40|Ce{ZKHEIuTe?UP2roAW33k2I_`Xr#9 zNIl|KAu$RCu>Q2a724lrU*5(zu2avw~hVndTlRJ1YTfE$h*RbLIIU~MvZYssUsI7fukLg`tEKPg8MEl zU5ww4(W@TX^fL7M(!BXIO$}M>U@rKygEKLdJR!0eIalWDZe?oLGZOGctExPKzMY4N zt8nF05%lg>RGa|b%+y;-n!h80NW|oo^NRC4pA>dugA}f9sbv}J7TKD>f!*;`cn!s2 zSAU)N8fUN<|IJpr)ZkPPn(`+72hvt6tdvd$!iHm@Nln6>KO4~!6-d4iocz7=dl+Xk z3mk>coo1rG5RvDUh211C^wGaEezP2VWGsDMSU^kUI^PdHpCgM3-bNbs_A6gimZBkA z#w3_;ftG8Z-t_$N6&CkPVHfU$?hxAM0pe_*Yx%ltmk##wZNJhrFBVLipctI2pWc@T zM#FlMKlrwD;-a*>Y(Za+_XpiK;-4%sL*K+K6&p9b1Iul00hOV!9Hyi)HpylO4;^Yr zHt`a!X-l}d#Wuk#zdSL&#uOPSE&t+5HH%12V1gr?Z)fyN&T2rQb)8VXqWun~>#OFd z_Cb$H=u(~2BJPe<{HgnjWaPiV52s|}&5C?r8}4O2_9LMWyzF=%S_<49N)CS=DuP;` zCU>fnNR>(XaS|H_b~@t1hM;`ik0Ewz(8R6bv#$bPW`d{k?^wQj<5Lh4zaoAn4if-) zf7TMG1FzXW3vHP?jK*Wv-|oU_z__6)2i`tJKwzNKA9+}Kc47m z=QpC1Kz(Qp8nTE3=HUS%(#4*LacLdIo6{|DD6i?CLxg?a6^MJOL3aG=-Cv;Ere4VB z%kYXf`y zma~yjn{~#%sHsagvLq@hen{r+a=E9=$&x4Ipm&N}g94?2c!xBg=~Q%Xr_p2%R{1&a zU90vMU@J<1)LWM?4KE>0MVmzQIQ8QqbpPuFKUebv*6_YCnfRZo5TVlzNAPO~-Q*pN ztLvCc!|c zM12xUGaRP6VFtQ<95bRt5HpA_7~a1!ylGiWp`*;o9vJv=d+Uca-|-4mLg!+ZzjcPR z&|NxOzcE^Zsn7tV3mIP^p?qAv=z31EFe}e|r!szn3K_pgE(sbdm8?Q4tQiu@MiSFd ze0)OJoi?kdYEXMx)2M3W4tlFpoao>Z5iXI_{oymCLhM%P`2MxLZaiINwuBV0*iww~ z`)8w1uVee^2pbuHRCuNyf&pGD#9S47GjW7vM~J1uojSSHF z#Bx5GCVr~(yktdM#2Z@4zrpRqu)0@0IEy3JVwmhdOgz)wylnWGeybiGN5@`#hhb5n z(-R*NcfR7ADeOf^rabvWeJjXp`+D>AbTi0$Pa7$gQ7sskd-nCz_1yMO$2mI5$4I&_ z_VEfC-v=;(rZ8pw-->}br4n1_WO7jv_L{>41gxKLTlCAbFF!(C$t$W!owQ<-f2;gJ zDt0opV89{jnizT2o_1+0ukT8+oxoIP1hosC;w;rV+p zz;lD@svp3Cuu%q}LPcmY!qwN9A^419^Uq310ph8HM{9)d*(f2DhP2@GJf3sAK~V2U z9>mH{{!Xps6CS&6q@k0EDU>rge{nEBUU)k+lMq4|`-16tKZfoXJ1*R6pNS5^CwwZ+ z0#Kfo9%$o+CSx3d|7HCkdp4XYQ|#(1z8InE_zwnzl>Jzj7p{x*fqT#m0a6k)DhzsRuxypt$+ZOr8p2UM5Mn! zu8kw^q^&nT50z%Wtxzj{*&Ej#4a`RaC0x4qJ;9ykCs2DIO$Yy)Mp9{z?^xgOuM}$Z z$$^0so?Y47{;gw)FtBS)*at6hzZz{@+=Q+*n{$$06%>FTfs~)zH_;_`q z`xfC$S*`4RcHAEnskd@NlSt?cEn)Et0-K34hD(w}pAyIx%x5!7Gg7+3wfXKYuv~Kn z){xrWr6NxT#O2-_brnz%}OvEBBjWt>1gLIUAhqxBFY{9Xx-XD z?>$&p|1n8wz1o?CT?#11)tWA4teZ+aJwyw zedE0hEnv?DcZkK^ivvn!@LD|L^r@H}ljf1jd-q4}f%CTSZWxF?<*295B@lACr;C{r-QLVm%e-3!^v^2w2gE z&eXFo#?~RJbb4|KA)?VidGRQ0m7&(gS= z3Bh;bBrM8Q9(AOgcC^~($L5kVQ3PW1YIzQ z5 zsSbZ?NiAMTM>oU&Nl780aVpzr?CP~VjbKF{FGsCcbglZ!!hS6WCO$!1(Z@aDZ}O9qh_be_xao7$FJ?f;w<62zdBC378$u+p=L-3D*w_yQ?` zYJA1Q9}uyT8^Y880L6tK6pEMFlQT2lCON5{ey;tLmgOfUc?PJaEA;9s*Vmf2jF1`g z8%o+WVW5*F5KoK)kyUl-)B3Gprg}CfZf$5NreEhb4ES{0FSrHlc3LWtTi-&j>hBLJ zuF>xO7SL3)Xp$D?fyYk8HRj}`!ES`HNLd}=hv4EE*;s?hCfUzcm`F7G1&!gvU^33q zs&w3!q+=8%NLwa%UX%Yo6*yiM@aox@!Vx!HqwR50N6THTKGnm)K{84ToiXn|woAeP zGlly%8Bi!7}lm*i%?>0MlExPTc0BZ~vzv+-(GSM=LwQZx|P0!DwBjN2%?eb zdgvdQ8OPMGJBNtVc;?q|qzo?kWw)Bq&R zP{&*rxZV6rIfhU7kE}ciu)S?7WY5H9<1PE{Kwa^(?b_VT!GV9!r7%<;YYN#VuYl!* z2TM!0OHaQxm{|4yX4{UU1W-{9kfaKmJJyJ@v>fAliy5@#3Ma5i+k9IbgIAn;*XH;N ztu^~egew+rBopLkzI18S7BfoAsei@fY`UH}Ibr93n@B;VyrOu}@(C4E2*xYBgv)C{ zPpb4UWkbKG7|cfQE!3)+0BrNDTMPu4dT#wm*Yn=Hvjm7VxYYo#p`S~Cw7s0 zCp=-GKn;tRt_%5D^yF;@C`u*VwVXqQc4lx*DZ5H{6TmKcfIBt~^xYiYV-7y0<($0C# zS5AW1dI58Uva~R{w4I8EU7ZwHzc*w^ryl_HRKeMX|D^sNu1Kfg$g->+Z?sOM?6&wI zZ)#hT_#ZzNo7P{x$Z0EJr5SIFFuNE{K}@nMKL&ZCuW@Q#oNoADrmVGXmZ0PD82H3r zU&`Ir{}^lCLR1Bry(Y2Nz1j_b%|X%2VJ6~WQTKSTmtbgM6knLuE+DW`H&@kRdBP!3 zQy-thm^<28yM0PyLqg401x}6s#-w28i)f?+jU|~$oh$zy zKc7LKhQO}t;V!nS%utE(|IwCy=2Hzvlog2sIe?&hS?2r;-N=hJ(x>NH;BymMa7n*N z1B-1yUUE})Nq$ib2};MfxK%FIw?=0G?;hvtisHP%~I{ zZR?ZiV(RarvVksrhbgLVRWk|kZd3kN+r?3({a9RLIqU+xs(-|aai}5|fQ;*5*}m|O z4ofU3_f=ru60I=yiLo_fr{`bND<_t(6p?KOuYl}S=;I764TSTjSBur5U5jjgSloxb%K0*yC=+4Hb>V0vRxIXcuI3-AU=;FMZeIJ7zk5Cb$869!|S5~w%R|67>m`n=&+y-=`^kN?d&2DujG-Sl6$%9 zhObjoqq!<(A07x%v$gsuny>hqB33hq2<3 zokC^jYR3D&*NCybbe}HW3HMK#sAUH?sq9`)&p(Iv^QZv>x~1E4U10JVxDEtr_j>vw z2Jci?n$i=)^3&!ml`tT+u)+V?%1F1z5_CQbX4G5G}L7Xuj>40jMpc%HM?w^_p*B^7&8wj!LI&5Wv@OsL9|x8IF{g z|8MtdiO0bkvpUnwEgKbfKAqO%+o#88!QaRO&R@3(l_ud*O#(eUHujUR&#ezsNo>%| z3+B|oSY#15Zfr#tuPwbDvZY-R&LNPO9_&TF6Wpr-wigHHfe;##SzKC23>ePmpaVS zi;=!2&HhzYW=1v{I8i9PXx(_`P~Hzs{H_2kx&u#6+NL5*BIU~bVWAWre*M;-txhc` z1b+)Nf;M0U%;^hps@mBBR0rt>JgNv{8I+9O?lwzpazi^9FA|%V}n_xKh7VeKGm|#r)d=Ji5#)*9!gxR_VCz4L@{M;=A0h4 zbOt63Ev;k2LRDZ@@d=)0x_pi!Xs4+7bjLq5Aly#^#bx{~gb;1igi$>qBR31x3<89_ z7D%Z`P+?my)Di#B0cn_3@?(;kx{*FI3tHo2Dildpm`FqSu=Iv%aVcy0yT^jlWYsu= zKlp+pI5+nCdWRqq%6N!>4PxnTEzR$R;75|jP}41rLcyVTe|h&kOh^D_l3+q3C-Ord zF`VWZh89R>NeseC+XFKLEKH29B4CQIU10gzb-fQ``da5en!L3IWzV#5Q}~2!)y5ms zGls9e3e=hSFlh4?1DoSdrOCa8)lY^q*T+R@7KPn35q?zm0bJY0qJ%INblSK0kA@}K z%pnok*9gQ?hDbw&qt}p0y+hVzEK0JoWmuZ=0Nw zLVGj{Fu&^&eEJuIEqIr@LT+jvuDdk--{_8XC=bAKdS_*dV71w#bS)t%++{mM4j4hg z2~Cr0UJ^W?@%bLSI^3!1a_}Oz=e*(1ksqB#e?zAT6NW1b)6DQ9%NL}9eJkNwdPoDD&x_9TS6K;p9~;g0n`7ZePl=+H z-!#C1q{VK6Nah?we~9$Xa~l|OkA?gGkZwU@9sBzNaZECGcX&4=2A~TymQ5LI5c`pe z18}p#Ru2ExZ64A!!!Pa+BiGv`zs~;omu=)_M3zYZ_`=IOCOdOqYL$L5UO0ceEc0-y z?hv^gk6y-io161L#tV?6IAJjRrVWU4iT>KGoA6e{tP9sQ0Yg{hJ&nU3k0qz&g;$V{) z*fI=%9=~StHJg<%VRqEh+L#z;OmmJb`(a{s9;X^J#x`5IB>e-REci;7kBdVXkhUZo z+8H^yN+e#dbs{x7t^jP7lK2cfIYA2N@S9zPvS`6S{^Pn0%al(g>0aMU*(E3-j<-5LM$yFe&u z2ZcAF1vJT>oHD6wD*iXdLqu>PBb+56WCW*BtJBgPe~Q};HcHOLN{)4(GzwP`Bh+r3 z@y~_ROh_AP^HqLvGdgxO%lkCms4a(eg^64Lw|sx4cPg)>sHmNrv6-{fb+gVI@%r^^ zEj~2sk>1at}dZX3}!Vv ztKwxABYM5~m#<9cmvOVR0i`ujDkM9eImpGF+_ZOJK2M z>sBJYD1@-%6=j2}PoNL3n!f&^Rw`fIEinq!@K{0-lF7?sopCjqc( z5eRpSo7fqVNA>ZSBaBL~t_tc;cu1Ui1KetHS^D>)KgAW zBT8dJHt8ylG^sYUs7drd9l(Bm!hvuJsgB9+Iid3MLVifOYw6-YWkw8mgF43J z$M^w(%hLSITZ1=x5Ifa@qZ$+=;7`DOJ8nWp$A*+eozss#Mb58qxSI z`#DI9L@{O!N)#>qd}&)fBT*&W7(<+6H|aS~@;6}v_g`y|u13Iu29UDOBq%g!tG7=a zS8XgvmMK;qh!r0IY^!LRkL~K6#~50udNe%eV+}vY{>@tl7r=at5QVp z!yRLD)|{!84R&dgF&L+W?uvN0$DX*0qazIF{&4yJZq`PLYP4ctxrdf;3dojjBaJYi>_gzm^TFr~sm2I(xFi?BK+n2sm65h~w=Q?6~4?@@0|; za3mgmxY5LKXlUS;V)L!2YvTAmQ=26g$48eByYSGrdCvYD>_{F7eg(i;EK&?<3}p@H>;iAs+(~P`sq-#;5(d@-qA7mnP;nIkB(?@A691!0`G2F> z#;8enMZ+YrRqZqsVNXW3r<*qR#N|i;oFyiU*)bLrxRQ0~;9B zYA$hZ#OuR(sc9L8yYzgCVfj--mq$!)zz4qeb1mI4Et0VP((-`L8T3OTf4mkg9nRd% z2X?hCX3Lrg0|UJCT?U1Q9)=v;EGbP8))Yu%gtGz1)ygZ3E;grLuCl!L!UTe;T&vN;2Y@;D47SW%$8PGG|jkL zt(^&rabsPgUS^f7_w`xOgtX?2Z)252r=c`vz9lZOlKtUfBlpu|OkCfKTg>PJ_QmcD zI^Bx6A6JUF{HmD9lu)CpTC zFbaP7Yfc#MBrp{_&gb?oXPcW?J+jQ*E!Kwd_#b*9ZuLI=r<#hPeo;`~Xny=tor;S8 zq{3vFpKmnpGe~QSiAfnBb}k^Li>dj)HCDy29IHr0XXl7XkN@D%>QOkb-15SdB7h-3i=bbOwh!gV_{+6hem6`2H*K~xRqRPaRR8gW3=3wWp}apK8*EU3mXkKr z|ITUqpWa#^NF+QA`_*C1p_y2e?ii$OvtqTrt7WeHw2qu&iy4GgVo=p_%el3)7~CE4 zpZ1S*8*6LhzUfuU$Lva>4}!)2u1;&#ySbKU)jBP=92IP;^{S?GhAAHY-rpaFF`RhO z-Ji2Y+?;-TxIH-;h{_#058{RjHw}4GPUZrn{o;qBm}rLNkwz+$J+^M!8UWz!TPa2Z zi88pkQQ`NIZnS7TmFC^=dGmeOcKyE{z`lvtkEn&EEfvK?wR}T%j%4KbxsHhrEjD zC!ecnYUbOrQKj0?7R6OB(5rsNC~iPzq}fYWWo;hLGgy7cO8dN-Fs;QcqLc1(!>gA@ z2MVLRfa#{r^CU8hihl|Aw_(@khfDuzA`Y5mB$lXGEMl%0yq4{C^rj2xi!mPaqQ%e{ z4EI;d@o`qgFl?uHeS1o*1Uy=1`Y!HhhoDB=i#k<(6?Y-+jyKZfn>M!>vJ}j~9>?9m zTm2FpqSyBAW2avQVy2RzgTMXycluUWq7SLl_Rr?cAE6&LyE|nA5XRG%-(QH=&&+s` zB_&$~u{MfQkM+93Q~|ZxkLY^^`}V`mUrLQEeo?jCkRU(77QHvQi+xf{-S2N^5q6XA z(E$s&h3KdcysaVsF>fg<0X!ZD1RWFiTrMG_@*%z9&j0i%MGFr_PqWL!0WCrz4`d5j z$|cY*6%*a)fkf2BTN-~}K<1mcteDU3%`?^o??>^-+bBg>1kAl#%X%NZY3Z|dNf!&lA0JQY zy1(}QLztX-W9;#PA0i|qG`rv0h(+m8pF_;tBtE-f<*J+)=6^;IhT3msh|+x&Kh?~h zqWy?%@|r1l#6CEUXJ9{LvwiG3Ouj4h9)rHn3TXR;H3Pr57z=IdiryGIAMPSjwv^Vm zogKi570-IqB5q_cps<;fITX*$|6N3{q}Ju@1xkdQ+YT1XR#NDS4d%-OHOzE7OC`Kx z;%BL9%qPieFhT32_Ezz8LQpb;2a;g2DWoCf-|6sFC%xqUv4|(I~%Q@=a|jg z1hr*ny%^DLQ#Xybddh7DD{eepUmuUS+1JC4DwIy?mlQe0P}`L*u!foW8GZ~RUT;cd z^BE9J^KV^WO%1L*sB6%R_xyK-E%>d}{UY8X{@edKQtTiWU?ApR)#f@T<2c5J_KFy6-r)Vs=m^mVHwd2_TVPJ8k~Rp119-!B)3hLCoMfPFs)-7$>fTAK<0< z=|YB9>}dD)`@_2(CZzk((~RnqwrYkaUS{dsXFjcEu8|1^Z~Ua0uw?az6Y84Gc&crz zB#GkwEzOV#p^Jk2 z$6Yh*%D-~|@89L=a(G90PzPgdAXu-}*==7Q+%2vnlY#W1FCY>3^`)w7{2@E*ST3T^ zEJg;PE;?|LA3McT7Mr%USncvJa}I$Tl*^t^nD<6x;Q8}R{(i06f@P-uOBo*WI${$wBujH_`a?rZ(u7-H^k3_)|G`o z9;f8>r(A+>nPnU&2H;<>?kSkkS{%<>YpVL6#TgAQn5GxmQ)40inpRAUVr%A-$9w@M)<{IFq!F z7Pyq-wC@ z%k#g?brziBgEsaT?v_iwBA3@e;f#ZxQeB?qi}%aeSs^8Xl$5u-^PXTNIUBTSU)@4% zvXp}L+wJwE%!mgJdWDwOd^>oPAPSWEu6=!5NbmWFD|0tr1Gmxb#%c3x#kj5h-G2Z; z)t;znn%h`d?%?xy!c2>r-cG6gIkb8|)WYpst4Z%@$mERXDrK-Q&1et4wC3Nr%#gi& za)lBwR+Nk5-Y+Ni%tec@?kl3<&-b8YksF1?A|&L9oag)KuduH34a~?v_3B0SPZ82@ z%R6^b=*lF|a_-~mbzSyvK1Tq6<o4{zso zy6>(=)kjgzbg{1oDK(~bpkyGC3@XuRYJBcyI zuHN4(78`4;LZ{D$GxQn-eLG{U6jC>EO6~7g^^RzBZPKwX+2`+VFGYRabr=g7i-GtR zI$0I}vHVx?(h7^4yzfA}d<(@PBE!CCx`BXX!GioQ0!s_FjUl%B{RITmhiQ3H_5kuLL&t?f#UuM}yG_cmkHJI}&Xoz$=@ z|GyV2NYBDs&rhe44(qesl@pal^>$}OfzFP`OUk{Vec!48NTS@l+rX^dQCEZsBX?Kcrv@?Oq~TY$Myz+CmkgNodl$z;Ki`!tjA)*GB!2C|KsQ^ z9O-=fFn%2`+cw(0Kfj?Fm6bT<>zY`TqM+T?V%qr2Yc_xcZTKX+W$ z_5EBo{%;ZhC$WhKd)rf~Bx+rN8ZhDUELdS!*B+z>GbuN&FAQuA_PDF|6}|OmlH9aU z?!txlzHRTi1`rucduA2ENF6Hz9#3y>HV0Q7{qG(!_I(ROQkwYQD3W5&crZcTUrEEz zoAYbJ3ha0G7@;01sS%z(V;`eQ%=9WTS;l^&EwoMeH`ul8&JAY68Z$~{-t|TWn`0vs z6A8Nv3P14j4GBfC<+(TW<);f0siWrCI7>~|vujy}dO(o!Y53h}g#V@vPDuLW}EqnRL`UMgt&pIxg55s zHx^i7uR6reYHZ%JxVpF|LbYr7m-vE6K__CUwr_gah#E}YovEw!hFMs>Xj1OYS2-m! z)(iz25)(*YY(iwX7jK9bbp&7j*^6&{GZzms0d8ZSMu-dz;s$i=z$%?xR#NCF<@+qPYcsIA>>8gjp+u8 zJ3HBUV=Z{$4}67jsiha~o;&BL+ZI9jqRnvo#><+Ab*O%jO~&SH%$Xw}gfOQh3(_MR z@w#JqisS3;F+jl(v4`8Yyl}v2HR2#oOB| z7m=l(N+Q?>J=)tRo9S#UCEO*XskZjzTQSYWqXY#+@~i7DN@fPt2?MgX|N9vfyaUAX zUEq$Uiobr-JZZa~vcKu-sGJjg7xKY^YLd`j z+GGrJ@a1LE+I?41BhBFqA3kG`eILgy^WfY7?fy6-;`9vQx9pj=QtOA9@H|ONY}vC^ zmxQ6}MhRh(kyB55SSY6w$vaGw6Eh9LA{$0?#DV!#uOQZmRBq3*$mM>vr;(IE;>ZZ= zhViDa)m3?(&rjUf4g50nyOmxh@ViRNwNUg3Hn&tmky)HhO22DE-blT|IqF-lN#x_0 zoDO$k6CB;%oCqgKX}X=tG149xg>P(FKzdz5@nyiFuQ_cetzynn0EOxJ_;}H4x#jK$ zTgq8$rQOJ1Y8mI^5+EDsCQH6Nj9Zt{B&w&buui;&t#VyjASV65poEXcR zN^20FpB5hw(spxc6eWH-hLv`$FD&hUv}b0dy_slqkRxF*gd5nN+%g^7SRT|~KLINB z_SucAAl757`T)g81}4*1HYDA!A|x35cym|5+23`9X;YttX)QEI9L@u>Nb<=}xv zi;EyiiWf*oGuS>G+*G$~TaP2kS%UOVjE87FoUPyNo~3Nxg{SoO)t56gO|hQM@f^@q zI*@zz&2&yBqWFr2iSlw1+1H2oS=NLT%}OK$wE3d9s{WGl>SX_{Lh|OzCA}!d{w^pD zgEctZ?QP5-s@E@~P{1@ULkYiz4 zW|LZ=uKGF5VQen8qdE2M{mq(TE2tp8&xT0BS`l_&IrKkQ_n;j|B=)SZ?okxr8Ku#2<&n{bq{M0=-l%)tBCaB;o!2mU(N` zC{20EqY>6$HW~TeqBrDY^%vyW#l)!R<8XLP%fWT?^UFq?j4lcCsovb`(lC3qm_z(os5_(G z$zaSf)7&ZqQ)7Lpb6wBa>f(qxru<`{FM#4{S2(>#gIb>p3N?5I3fw+<=GkWv!SD0m zES+y#du!|WJ%A+uWHAA=%Fj1>`labr2umb8)=e4R5Fb93pKDI`9KTG=9dexMCDa@> zSiN`227d4gH{VDH08s-r%^D*pn0!9(Q$x{y^S~Se!Gpg0cpNScO4Tbn)J*}DG$l(O zX+Ok+wdH>Ebp6o&lp+g{6=m;Gn&!n1{S*r>p4U#rC!C8}G{~dqaKBvF+tc%X-hihE zf*4vcgX|p1IoCj&{qhQA@pb#J%a)SLMy2VC+zG5|Lk=-0(rT|B{UGD*JGkHDQpUXF zMhgwUjF~83TCvRp3Tm|q{me0?*{?=ezBxgcY(^4!5{JtBZ(+_ zcHo5A=4?tZ%Fi#6at;c1cGI6wQc+@$q=XQl+22N&9*fJG$M>n@T!j!z%4nFh=Y@wu zowhVX{@Km_OvfWBw@P@6NMc>x-CcN<>D6Vy>I&JY3O@2~Y3>ux!XrcS<{?A&t``8! z$r+38BLaU;(~5m%SOW)F)B)_d>>Qf?TW-#cyFc8SMx|^#ycW=^%j=sHz!07}!`M}I z6{9I$<$P07H?4WfsYnv`&W5Fg@CBUN7_OgN(wLEIS20Ds?X##H1c&pVBiZB-07WqMwh=B_#lUAi5{oK@atsbDD4>KJFlEiC`Ho3cH9Xr7ovj zy8m^_MilUVM+?QV9E}(YBm)g~ejkyllBRS@^r)PGwcU?(GX>bYl!+FNyv%q7UiT0w)<>n*_@Q1XsA6MhBgi6%}*lNxvsM zNy&dbBS7g|P+Syr`4yTj{9ve+;^9%3>GlUhq;3xx>rXd;besaMq>F84QaA%F7K>@- zLV))#5!gz5gzr^I;<^)`tR(j28a=egpZLz>MVtw_X5D3YMC{%UQcLWsnIHsTELinZ z)WN|T@Q&zCbsO0i5^om~3-T4p$(9 z7Bd1~oaG&gg-WmVw+2OOpPaRvA*qWI2EVBzv?b~I6Ze8ELrGrNtfGJyr_^E-(^%-O zIE_%K;tL5$mbCj*K?XH2XXwCA8cRsR>$-CoC~}W-5$E2=-wDJn`47XK+%;cu@CTuZ zbVuywdMwRftbCP0G5{JNF6#euc^#`v@JeKu*)#b5m@U|ed02pB)cCM6SRt!QWt8~q zde1szaOArI0Q(^~Gl;5)1GCqn=G}6^+MhGu2sMtoP<55BpTq#Uj-!yWUG8kKT!s74i*bmoNoGg{L)28mMZkrskNd&jD z!brk2rtd_SmaDcbu?4SiVly@G!-~mFTNZz?3o_6(rZZ5ul6#?Xx_kukrOZX}M4-Bh zD}btwe<_l=uS(ghZwv=1$Zh}r**y)x?$aN$m&VAT+shUgtp?uie>D*YycZ9u%o2-2 z-~5`%2i~jOWuF3&mB+>ZY)0T=?cbz7vfam#?ncwc{a)TLi=YR)_jo>VNkDJGfLf+20H4#?{ zGW(F@uk!(oL{=M}dtZ^1C$j6M7eh}^zg)!FVx1QmH3?pZa9$b}c~iPj$rp4&-!Rg9 zRB2WV-?Pitq&ExaH@D_}TsqW1!%Fy}zR5JNXK`MZFlHxjL?*=a_a?_NcfKl2OqN!k5@I*1>h;@^Yz-p`$Ri zGhpPg!*F~527m*Ps&H5zRz(gvXqGxS1OeLcC`XD! z4)7kB6`0w-@NWl>WyJ{+I`4w88aJ>@idZt}4=&G-t_HiqOYDKd4M)*KsZ20O?F(z*o9DhBvvplPO(T3B(pSWN3IO~+) z7z}9TRXnrv%aqbo!{u%ksj*Tl-{xVjpmCCPZ&6rxw;+3=QHW6HFR?&=h}26z*%xqg z!~)|is`KgJQ=esmTf$8^LzTnae>te`@$sf4w$#Ep2M2*sHPJy%H7^yfcdg(R-l)D4 z0VNGn&+Z%FfWSZ?b7_%n#XKO$z}ZaY;h&H;3@?B-vj*Irn~0ARa#`UE<@zjzjg-n^ zfJ`NNIv)Cr;63_oBZZ7wZSDGi2kJlk;~7YC9oUuzPpX}j=Xke83U~{;V~g^DqzV+8 zzh7KXZCf>kGxk=RlZ`?^KrWS!0ganzbI6iquAa(~p5*(SrWBOdmk;s^J`9m=nO^{_ z5oR@b*%s|cI4Wim6Tqsl^XXp~A}x0uu|pL@cdTtexnc8Bk7PzEm_;`aC(gcZYm${l z?fzmpRQ#n8;c?@iTlrq1E2X84;dH}j(t;g+DlH~+H=_yq%(#&)lpb_OuP0&2EKq|{ z>V+s{)-TMoyQ+c3}`#()=NCX#v)gpM- zuOFzv!zwCycx1bYh){w<&J9rrMl8`4(u00Elmp6IXdTRngap~7H;YIgWIlBi)-AV# zI##h4dqWwozaTuyf16}+74sx~U=8 zEr20xoSo8K%V^WiwxZI5rq-M}!<)#Q4q$_wjRsbcj*5wHBbc^yIfj^}!|*r?4HWRF zx{^Zfe7)y?G#D5?<$^Q0%RUPFqisawrg=D6DI(fd*}k_?EPZ=M2x?@OC*L&>wNVY@ zs8uvKo`R9{gqX`xW%)jV=2V{O-#$kNq#8ypZrLUM6N^uOL6~>+_rLAM6(7 z(VHNx9C7idyQdSUbZA6SNUBk&c2?3(QmqohzBB)l3FD1~p--0yFoHfz_3n52rLeTukglSvq)2$^T6O9Rym? z)zX^3m4OwG<$!_-cLRNV0M{gY`v&MBaiUoYe0sP7_D1aMga zF-Q6hkgO=*7gCy_@gEvm{MhyDy~JCId=9d$&or1$tKM&6F=(^|c}BGpe(gAoPakma ztKJSi^c`RE{vKlcH**-N^Lym*>T;QD?+!ipXD|3s+p8H8$VC0mLa6qDR#Z)Hu8G^>L1QSl@ENT>ZzpuifGFl9(9pa`xsh3nRa zmL&iSE>Tc`QJ+e5qkyHKK~qW7nl`xcvtBjpIIFFZDko{I@)kg9^rp9v8~qaXev%)A zaj}#Hxx2{A*?GtMC&P62FE-a6?eymR2#ij7>9!{xGQ5uV9x{%4{ROh+DLUK;zxAGp zpBAg!-nckSvFT3`tb_96al`h)!d61tl(|q}I!k$B{iS7ATHuzjnLq$xP|J%D+4iR%T0j&*2NtD+TuTq#ERozb<3^|D* zFw@=sP<5f+w6W?O&dc6E)tSw4@F%Hrtf1pLFQcj3&c>SR7c4i4e9RSo(b@O8h#jh* zGSVi5AWfy@G|;SZmJVAsr$IKy(`CmKZSh2jsvr&0Z3J3`?C)n%u%CDE5-=!dU5Ee_O`uz5-q#kEhq0F4@40 ziZ?+0JwI=)eB;Jm7z`_=%b(-}{tYjGF*;Vrk4UrL__xDFLlAjo?*S<`ukP~x9;mbU z)>51ZY+}U^D~Q*KjaJI3JKu38?B)zL%i%{-sS?649(XG<%e!-Q$Q` zxi&plSeaI1M*Os?$hWp9)A@7xc>zEAf<)C~L+|@NOF_x;vuMAA98fFj@&o5TYm7%P zYg)oTBp*?7JK7JO0>ccGK3wWVkVEu%7&&!tOY5Hrz4^ek@e_qeWNl2quQ&hnN4YBx^DpsdgDr zmk0jh3p~+z8#a)&TM^Ohyun{HKG{{C4t5Ep6Pk#Bs_=pcPZf|H1HrSRwZWz@{=R`t zxZx#=3~H*PC zDqaUK;G`}F6N1M%%?iFcwAUtwxul1!^QO+OH`jhJ5kjR){1#E-eHq^r?hu@tBq!`Eg`kJ}{ctztiVXwgXM;;5m8U`}66l*WfW0IWTNX*!GQY_MeE0?sqCdO#g(TA;u5E%e6`UdS}HcwfK%Q_*)XO zHK}Q~`x732TeTAQ>X?r2y3ZFgh)ecw7J#OO>+^g6I?ISYb#>*amGK)kieFP4<`@fH zh1_@v13$hoh1i9|{|H2f=#8F2KOV25f!m%Q??ANqYwp!Mw~6s#^u#JCKj8zd zY&sfvzq^8^kC4q4vfo>i{(i-E8(Fuuyga>Da2WhyxIR^sDBv>iNq7Ftap(LcBUD}I z*xV6+kFba-*$TGaR? zdD#Re2;F)o3LRZ86gz^N95*($JLgxYbV>JHC0_ck>n7>FJ@1BUTFhOPBO)X!*vlAN z)0Wa=$V#{2iPuJigVjemXlQ72*k{UwDT(ONxT@)5TUsPFB-|A9 zYLRk!)c9MKeri!%6`u^iuZ!%Vf>9BEZZAQ4R0WHj?CQAEr(p0H3F&a!ABO309w(B? zFoe9CGYGA4BNpYEwe~v)iuccJgIF|4e0Mz&-tF!sBzn(3Fk#@# zL+O^52=yU5`C?~}Z%zHc>E^NX< ze)XFSGE#>#Q#&-9>GDLEekLYXm_-o_%?QD1k;BJdx_AaUD4^`8@O zkwwu=Wb?~{(rhEcz2=db$QCSYWpr3!%LF~J_rQBaqM2xW^Xev=cW!`d`6%nN4W@1P zm6|An{tQ<;hyVH>P?~^NXIWd)byx6o#E!GTN^f+yL0eb3LCXC6+$SQn=Xsb@MM`p9 z#88=eH|g(>P8)MU1ZCGMfh&ApE?e&|$2xP(`X14|ynLx&@5dN^>#|EBa(bjv6Yh1k zzZ<~QJ-r2+%Ah!2y8YNH~LLX5gz{pr; z8bNphJOaG0IzJCbv1j~F^o)d!Kz8}yY5=i1Ou{nSInX~U|NBNcyyA^rzNN>9^^d1~ z1Cvz+x;iOj`sY+tw3)`&_M;^DKAiIh@C34}Q$ct*W=Yfkv2)|39$EHpASw_}x zPTV283J;AAR$+`^mJFRl%|+^O3)Ej1vd>U}w|%w}SpXW%awdMUU+Q&uNSJ=~#DkYp z5I__EuXf?)M*nvvAtW{x($j@9hMNgN2N62(2z$8FVL^>0FySCfW(hB-&dK4{5tL?i zVdpEJYwgMypP;yTFIz4MTh0;0Vribv@>}Zp*o&+&l<`(68S;oDqHGXSzt3XT;}P(@ zR!2O%jO3}wNFrx zZ(v|MTZ+hE-e?(Z4+rM$zLXg8pzCAg3_rn9xQSG6k?od&RKCLB7ALB&5KjZkSu2kA z>DW2LQM{&=`lL<#E!MSM>o3?iDC8nm5*qy}`yK&;L6}#}DaEp}d>h~Z(I^d2%jXGR zC194T;0@U3uVxkt*IBG9b9>MopGe(eo5}zJT>;*sJF#;q%WI{4=Bkx6?fJ$MTx0P; zEC8@!ga8|57qFgfWdkBrNQdv~B)gvOFpsyof_AmfB*bS*I5P?Ilj}j!(Y!r(U$nxTOc6u&1p) z5Qyu%W?c+Po68#E=jGVfFVVG_)$lR^qa2K4+#rUlr2lN#d#B`GZHMSN#+~o3!G#wp}H3=0;=o0enp(aWOQ7u9KV;n8#OL|#}aIa4*igME(qu<$c5&-`m4 z47&egTKD%yVSJ#wBJ=WFHlgcC6XdcHl8NjW12Bg&tq$CQZ5ADx?$=MJm&JNN%>E>C zT6R-_8vWo4ey?~U%N_Z2f^rH9uYxdNw}m7Z6l!|~Eh=eiSuWB0gPf!BZcRXr8=B^E zBx+4RprRrRz)>%3IbVM`EC)NmuT6_^O!gQl)x(g12dj>*{}Ay#!gNf`433jneIcqf z`07{8a<@C{V?zgR*y`rP;NYFt=u!5m=3iByODpJpo%U&!_KCOZxf{1cT;(2&3H=4B z3d)E8l+c7-KU`nlrmO%C8)s$x>)IQTE~>zo=z5}h6icI#OqPXsQcjE%R%hepjI@77(ZA`bE97+Cx8tj9dj;8_cF+u-?$~6W zx+{0TSLfTc<<1^GTFRoWc)H#_Wl?Qy+KG{lwMT zehODo--XFqhhHFnwm3*)6>>@qs%DEUGjzI|PLD`dH)L0t;05QUMZB*tt5~mzU*vk9 zI+W=qnx$;=9-Okj>l3xR1~|r7OLoIU3%XQc^xi1IHShlM4(LBg(UiZ53A!Hb5;os5 zNnHTk@%;-vpwfKt$P4=AL*H~koh6-(cr+(<{Li*4RZB7W-Zy%RhI&D{eObZ|efH(2 zT@|FE-9zJoeLuXG?fE&ra-lQt=;AnaT1RPgXwzbd&q3?NS4J{F=kp|xM3;u9z9LQn z1Nuk5gPuNouf|2BAR=-(qMD3y?I=r#S2AoyFx;kDKY0U@8)Onb+_ zQNA3bkMMa%v$g0jY|BbaVyY9Eo`c2iJgDFZ})zDGdBC`5~CDRl)UTU zcutR_i87-c>z!(=e65aEP`rPLq*@@H(;ELwN{SQDLi{#kaTxwaE<~`fQ3<~ZLqCo! zDSZe*1l8pSo%f|sxf@lTJ)Iam`VDs=Do!*z{_suz5Y! zE`Zh3DK3n?%X81SV|k_zV*GxiBJQ8LC3NqOvH3``t5J4|{b=?cmDC*Fv>awsq%p)9 z$#+OSWS@}5CF6c}`F`f^Gze(iZTPRB^Qz-1pdyKLN>`=(+CT6XSl(!MNlSV=KTrvp z7G}c3RzC~mLn`t6JgG2((}oA<{KN~PMfsk&iI%HSkCo=mp^$9a{>K+<_~S(#-uH?R ze}LT3{msN>bnQofjSIhI5i=NqT1P@8h&uY@7N;kYZ)K(-!j)y-s zH+h1=AZ%WSq+glq>yea^#&%;Z9$Ng*LvS?+ziP2>VSdMd;iv|}rz>rXL~OuIi=fb@ zx)bGj31+aGm$|88e`uzhmL{b85c}uPpFCg;&DS!2z;Te;$XX25Y5%^BrY$ep>>@W& zBTP4!nZEVFmc^7NE^dkJ?ca-OKg&R-{q^XpF=IH3ot+qxX>3*y))a2Sam3V6`fRVO zrWTGye1$dx9RtYROuGQnINCP1x^6<+B)c}0i{XlR^5Eo5ry zA`vgGhHm0+q0j0sx6~zc)SQQEm()1k;MKi&|C~{lOZ6}qT^Sn}xg%2@Ck}}qRDA1~ zQje*D=rl8u+2Z80Qo$$-pDm=qZoFF$tavle`t$&}4_&QA`cJXGPMN*mQ5e?0JxmW; zD`VEG{R81rcVS!4GQkM1N+MO6pv%|q$Iq1yB3_~eQ(vx1rzXse8p=RKOl z3o>*RU_4WasU=M^92ApW+(g~Sp7KmQ3@E9!VS*{}3MY*7FgB|%j(E^gR)Nd<`Hxnz zbTbY=TBl9OW7B*&Ry1w*+QLud&>xmkJhDDL-Fk~8TX12(ai&G`peOc2$@+B5#JWGF3^@%M`EQH*K z;7@MG!mFmc`^e6RL5Gc(+k1{&Y54PWQn)7$#bJu@PV{W3SY@TH<4$fe&cB8u**TzB zvZ(FiV%(x4Iku!s)fBP(k| zgZP-1V?9xWQz680ZSN?G>~6-LVIajETU5 zd@0CV`=OhY#Pk}d-F~|63%dVFHJwf@d6Z?`LMw$Q%v32oRKC^sjrhZWYPd;yk&TGT zL`kB0pNS2?;1CCh{Z&`ttSfx$7J0ge9T4PR{N)?|ZqAm=ZXMzUh(ZA&bcO3>54JWg zA)BOJCQ}S%&T4-Xi|W3#@~Gege*EQyBhR#ztR|V2tbN-eB%|3dyA*g|vTciFcYfB6`nNff=x-AGg8Nwen_k`*6+Rh)XCwHS9qt}*0oX_0MZfCGBAie)gi zV6kqW^-}NL^kPwGdl+0}7ol+cLyD5Ik`siEPS_ymmd*TM${q@(AHH_?p$O|5nyv#Z zBiJ_NMsnq#Yh5Udy^DCijXvy1>lwR*h@)i)3Vyfd0Nu=d*O;i|p|uAxiu+{|r&`2o z+k`>Sn1f8jqtr3}T()fAnze2)4-!+eakqLk8U5+r8?9(pk_H_#wqgOK2TztDET_&O z&;eC6nxtr6R`8FI$&)=$d-zgA#+Xs`+`ME)?urb*n;TSCbze`MHFQVa8~=MQv2m(S zNN=(fx&(;H<=x~AP_Nigj52-MBX}c%_ZMrvw$OFtU6Wn`Ce;!ixlE4BTef`j>Q?^T zpFr`TCllRF)r!m{Q=B}xgxw+V_)Shi2o&muYXUC2E2SF+iu{(u^5scr70|O{cCCVw z3(+hPiz^rRXRT9SZv*Z1XoI3Hw>;9PP}7foiS7|xyjrOF*b<$&6Mq5 zP;AmRcU(L(d(+*6291<+Z`HG_FLvcBUx7m$lS{c;2nBoWKUOv_YU`oCRW24HWR@2M zU)|k{KMe-`!G793dJ?ms>WkChCRN4r?t0tGPbi6wk0TRPLa%lF$;xxVf}R`_j3YR< z74vy#=M6CO|0RO2DX6A>LHzNU;d|VzDysylV#S@=ZriI<+h+z!lfT;{BT~Q2=vH`f zDPhCh<>3n(yZ2(YV_#$Y{OLLx3G^I^qe|PZ=zsD$DDeMGl&h!k^%tQjoy-4Y{vu#- zmG}EzdHwJ*G&wWPhq6fg@g!!yl}=15nsqx0d&@BFiv+f&>(@Jl-@XaO#k#Ic8%hq# zbaWgHzaG?S76q1vb=GOD0D~#qL?HV(VF$uL>jWV;IV`^Jufpt=R^B^wi#|gMPQ(S( zdlt5=XM4T;Lh+i{ZSj>C$lAoBBND$?DkQ6xd_WsJH{&A{A@M!Wx68m@a$?gis#$Xx z{h>}g0nPNZG?VAE_V~}xw`vklSn0NUfVzXmi_P0x7(v09phnkGPlcZMqa~d!>#U#Z zLBlMyIfa*3iqdYXF;CG|s5uP&`s#6Y^=y|$lvvB4u&lN+-?2wuo-0(#*Nb>c*N(zY z-lpGTwj=31w2SJixwxm_65b0$|DD}YfPtqE@qg36mYA*{yR1s-dm|DO3idLgNJM83 z$U5JVxuS206D3GUa>nsXI6DRQIF^h9|)vTewtqvrvZoxRXJ6lS~dmd=H(+2 z%wTp<2E)4nefeAD744RfOb-yUEiNvO4_`TyD$cXi$U2IdKjf>594WO7Il?NTO^uBw z)}5Wkn#9ZIbHjTZE}>(94tjRRnTR`?LC@=j_7aUOH+m^fC_VMVK3+iso9bzV3YzqA z&U|L^>aIxp-Q~kXI8arX^{z9WqVR9?GHn?ZK{q`0E#K0%$Y`rg`Zq*wQzQ)q|L1tt z-c0Ad0C%}XFbln)m6zvKyhg=tjfmLe@w)1sX1M;>MIVUG%tsdx=W~}zrGp9n59%1f zBCG;kMX!!uSW0q{d3ew<_rVo|M?3D0y%#E<^8z6!KfitvJ-u*!kS@bT zx|j?sa+#=pK`Z&4Xk5Q_l05SPZ0vVh(i|B4c~l1mdI{4XqeyCm(PB$wseW!yGT^jb z$t<-@1GB$2LZAC%+#U8I{~9Q8r3JHo?TsE?-5q)g=Zpeh|I>NSgp{=hsYs>mm!Ew(dFAb7b6ze-yDu8=o3KIE8O zX3s8BXm}7$_s(SRt}f4<%=bI|>%JQh39lgQX?XvBl`(ys()6B}B?Kk!yt|co?B+k{ z^v#vJUx!j^-VwMjV22;q#k{PmntQkq_>Ojz>siEp5&R-6QGm7R?! ze>#aw1fU%?i+Q-o)4pZ1%xlNbG zpUX;n;UA2JYbglg(%u+_KyWlY-?=QqGWY&A3-a06E)LaRD4Pe;6wl2yRXtae`e@Q$ zrwp`d3_jVgT?9Q;1zlgS120`eJBPz}GwgG^dT`wVfv*?5_}gdaheu*;`$I$YZTpyr z_uT1g7Lc8cWfTyhYe`OrehNE23zRu;L@|{pmr9~i!<~+<^Ql@5^xW(XoGIU>6z~>t z?oa6a?{^5#*2cy$s?qq2)s=#=aXKNqV1p44N@XK^0R#5shexlVGp``;tHV40Qu*AK z;YNGwTVsr-gr5AC6g#o%`A9aXlqsyA~ zlwky+=p_kX{BN$;&wC;`ttUg@r{XS^pnIB{9+8Xu6H# zY6@fxaBjq>4obAtvr$_PUVvk}BPie3lS#CVn+Qr+pAKYX zvBzXL|FRO%lnfwF3n-ggXnRn@0?JnpXYrl3mbCmoYlU*8SivD-FutfA$VklQ z#f2Mis`X{MDKg8-h$v~~CXQ>6g2B=~%merm1OGJMNmK6$P+O7a3C@I^F?;{2UY)7N zFuPhD6!4w=q73^StDS3w4T6_itv0UWnMphN=bd6pR{O;p>f56Rtp>y>N)YIri(|+0(EBMisrFg$S!vTEbuo9gm zqUnK}E}&X}w>~KOB*D%1{7t*Q9cxo^hyrFXbK9HUwq@FwZq(PeGhNP7a1WiU%G+dq z)I-^2`~6fzX9NXUqgEQsQpg@p5G5A&1R9s@lrD}O0*x<1hCY#v5z8OVN$=QxNbYME zNzSyR2s2^zajg-*Uv(-v(Uq*Kd7hL&+$}Ahp?RwZ6BMBigYmeTeCwgBa?8u)k^C_dOH??K%HF-4)zJb-J&&tb zL3hBCz1gDHaoqWW57S^h`zs#MTB&3w8ZseLm1K8}NVbUJ*z{}(xv^O|w<3S1#eezv zV%Fo^%TQ#OkBVF@pms3JznA9|`FwP}?rFFE^X>NQLKe;>#%qU-cBRdOFjvuv`2i8r z1~htRSxtp#*vb37iCzEnqWIJ8`6tb|A%X1C=eO$@8b(%}iiu`yf<(8|YMR3<+<2*} z0)jt;5OOm}WH%?{*2dDb*eK7At{aVQKtL-^7zTCmq8FK%{0(O3P8YGQJm%L3D)O4fmXNW zIzNFUcjrYSsCsUtZGRV?2D&V_7ii5gn%)N|mYWdPEIIbRQ){-agiP_X2J7a$M?%#B zyQIT`ghVQj*KvCI_F&=k6*s|PO6D2*ErZ5Lo#sbqiL0ri+Q4RJ+Bk~3vLIG@6Z+S+ zM%2XMG>xLWi~8})^0eoezhao>dSEl@2x~7hdMHnv&UTy6kcvded|boW;aJWgAW zN=WE_D43MU4bA6yx9VARfnJreiAiu__$PdFJdR3hs{}}1?mpa9HXH%VSICH~`@PI* zOzDzP)vJsKm&t9iAYZBco84b^xJ+h7(>G!*C@*YBJ%y4oF<*6i8M+1rQi=!Nub%OA zh#{}$54GxInDER71nF-kj{I4QfeYX$`(w`2V^B`j{?j%v#Dy)}r}jHGtaxyDk+!|1 z)$b928|DNy8ttG_L@BEb*bhZ@+hi`ORH@ADr`8rVt*GNn{zPo(1EHF$?IY-VI$gYo9nV8bI1ucAie46z4aHwgjVnwu+GePCI zf2Ygpd*SQOL=zVmTH5bQ7Vh>1qE&5)`AwA=ug`tfSTm1b^&~;sGaNW_tz(VcMl_(z>4O?jS)L9M3a*ztnCAH@h~6R?^PPF zVLvOk9h_qkjg%x~@JyM$jmIvYKv=(VgN1r~&A;TSTf-8vE#NYdb9J4Eh)I4Z zQ5`^q*JK`V-V=yV{`}A^ZYUnqWJEhQ6KeCid#Xx+gvAE=Rm0)Nd^c@?r|_rb5c#WK zV_~6So7{V>B?<6k&8+fI7dkq3%+Zx1DeOdYgRI#Esx-R%t*VxO7r>+`2=a}&E{>+l z$gp;Azk&o(eepuCCQl7@kir9fbXG*q8~>a;&k+S9c}s`jrJqF@ll+~GR!d1uOUU@Z zLP-pUQS=N|hvW%2eb{-~+TM=Z`4FfAqdx~aMzcn(h!eb(k717v9Yyt8swfn^?EuXSbju4_;+w{OLGa~9bkf}2@UYRaxz@*&Zqba zZW36vMtM@6*gy=k_crqj%WU@*=<*zrxF>UIWLZvr67K8%rQ60=*ad;F=Zce+mRb|X+^&b1-ro(FAjX6XuuQwg`;N&`>&nWzK%pHcDjJy z25xT)tr)$+0)-H{$$YCn|B#aLUF^kHfq`PH^Shp?)U>dkbsv!=lNEaj#1} zS~4Eue_Oav@xwL-`?nVG@N3Gd|+?5+3%`M!sxwg#7$hNWsl9l!U&b5L>K z#pdAtwkC|=U%>2G@1~=?+3TRYh7PNOpzCn)z}wY6t$LI%i5l{9Jr0KzbpRDv({h3x z6`MDD{p{?_UA|6LULukEL;!zuw79IvWJ-Iinr-mi|X3jn9X)sqO`MaNUalhD$c*96veL6jjVLDJ0KJg=p9^A zvoBA?sCTDx?@BFBbRV9GF;Bix~o5b(LQGf~ZfCau^_C}aC>J-qM{`nIy8}|&Z2v?Q= z_}Yseu7h_AYTR!QQ~8KWPQdhjYMg5+fj~q+x0wLzH^Wss9W*z>V~R#dyt8vC(bbd3 z)7Wj_as~Ox0xA#R26AL6Cgn_c-JBxGD~mt_9eVO&U$HFcqRsWibwz9dT>0MNUqVis zTu&3jLdS69at|`-w&K~3?kTVK3ZT`Afl!%CL5f>L<9g(wf_;;3=~zkR>`bTfdR1Pl3gDFwu7Hy7_x zyl4$i1qw9c618W5lh4Fv7PZw87;D-8k~pB+=YlE^drK9*@SBh@ix}TbYibEdXe2Eu z@{0yqdBR)ul_yH*!mMs^{3Xs~wVL&nVH5w+{6+;uJM_cyYU^vK)t3rZ6(|wq@HnK~S@a_Nx|ig_>I`*n=+o9>?N6XY0) z2R0cV!ICn))8?BiVTDVZfk7fPG1KM-RM0KWaG|s`*if639jSV8+ai9Xb3AoECtb(+i;Ob{gC+Qgi#v-Z_+khC!NQ7^v;e%h zD3)t!m~!F9gbgZ4v7gCJ5A(42%qBf7^M-TuW|02Q2x81|^!SfKmiK9RHlX#2;}mIt z?ujMGB13DY*(rJ(+Q$rDO4@l$wixlF?!?avK&@AJO0_@6p^xoLd;^D(I6W^nCiyxj z8Yv51BIg1jP(cPu2Z&4!yo;-uJJKm{hZu|Dn9trXiZg*HH;E#Plss(nAQtkc4!|D6 zr^}%g73{30TSTJL%Jh^-WcEHe&Rh7bhk+T>!}WTMEPJlsotl>44g<60NFS)?i&C-1 z@qPZ+pG;zkuWa{9uCx}O0AB( z42;1GQKKa~O5Az5ujl_YwYI^=uj#C{21ES#t%2USA3t=yVB9^s+dy)BT?q1L zL1V21@o<@hbY?ZGSw?s4X6l!1e-=j}K0_h%2SHLf`h2NShzqtzsyeBk@>mR@|FyNf zx54*741tO%03vJ*WOmFAxCsgG@fk+*;Q399j_Fl#|NYtJ{5jcu^CIq-_za5TD^TH( zl}Zs-*ux3`O^k8?%qo z4bq!~WvVecI@cbzG~B%<<%hB@4Xg=PR<2AV2EtYBO*b$zx@-dyfR=^~VQVlocH=hv z*Xpyo)6toPQsTju?A{m@1E^4Ky=ifR5IvR_p=|rtdlFlVxOobJy_Vo5dG@do1tpgYXD<@9WxN`bZCmqLV@GRW&7$;Y-#=?{vLA>+h@90 zGrt?bY+dQ*fm`7`69%F2`UmPSXsHUrlSh;zW<=?(81dMb{Dp9OjrtCeAIGW66?EZ` zvn+y^z&YgA+0JSF2}n{L(-*2K>A>3aJPSXD|KMN_f6&Sw=pL=0v(AwbADQEkJ}l{WfsY^VF2-(CS*0P z+?P84{gs+JXFG1&uP7YxOgcy4ReI0g)rd=i*jtGCNjn>$n|e*X*jGhM5mfEq9iZ2B zs?ihY{dv=|*ZV7%Hk5KZE=BELXN11|BY=V=sJx;=V6D=U(fQ$6SUdAkTBihHDS(+D ztf%255VFxm9A;0eD}8kKeGE|Iol0-eHFd_GuE}rU&gJnTX)GR`3Tdc%s*yC6O~xX} z^@Uv;B5#qNq;~x}16n5vg=Q%@uh?s$(FS6p?IHS~xwmADB?$w*Qf_b0VJ`_sw^(!G)36(U)&liyNcl`^Rgz_2yx|D68qhh&|a=G-}5{jW7lMI@{}AZ zQOV?khvkJV(!H@)x#ljx^9_r1qn-vQ>#9?Icgf6)Axxf=&6T+@2r=O@f)9aA{>}vY z!Rkt)@t)||DAnMxerF}vN5gsSn5KNOKL&2c=ut2GA=mt%XQnwzRx%`@dTsWbyX1eC z#I4>C;vuoclY7yN%98px-{&iH7BSn8@(eCaa&nBarki(}`TAu!0{I#H1pnf!jK zE3`ei1Sx-%BUM`SF7*QGJMVa0pJIAIdXMf2znh8dfY~pZ;<&q;jk{a_+n?v>G7i%{ zmk;r!hDekcPS|fq!bsbh+0Og&rn|v`*o6{J?ZpDFM5Qvjh=j`%Srcsp@wBA>cyN6S zDE!Hqh3qkz|tzj)@V%dYDYxAi4N2=20C2vSP$I#!GR|b95A6+ zHk2Tpa<2xDft!5OhY<8XDT8hZuzWlIHi_=H)_S$LPx@t@NHkA1M8k1k*x zb+ImP`hd{P&W@HlpA%d;xO@7ni_+Qro8|ZJjgw3znoM^aJ0cfcd8Un(okeuFnD2pb z33}Q3$c1BkX0PS3{1d6E>2DIL0?oL{o{bX^Ipe*We6^Xgt_%D~u>yuIN^S3XXUNIY zS-JY}J#?+{(yI02#Z+Mu6^Hp?F2$_Qnl&IkSM@LZ4~?oCe_oLxRy^ij1^-cEEylCR=N=r9&vXxd3GkvrLEoj z!qo@=2Y*Ny3E0dv<24|ivIn4gLraIT$EZ&|zZ;rLfe_QlyD_A9yHd8Y%1fh)D5c77dLh>0j#mnALN)(h5ccd8;yMHIS!0_B1K&Sm88;SPK z=wbkXdBGoHS3#X%(?^tRuDparu0LEE6sxFt*QERaWLjKFU*B_ihAm07W$>>(^(~#5 z?+N2ENF51O!P^O*`q%w!;RbH(-M>R@TWi9?2hJxqe2;b!U_9ILcL*{vN>%|36RQzF;A%zYNcaMqA9O0lnk0q zv#TSf&w(R*!Am!4?a!i5tGhEDb&puidtCtonEWRxBXZYFy6m_AZpf64GC0`XeH~;QX_wa~H?XSx4fibBSzjoi z#TXY+n;((!k=<@@^eD0}1RrdScOoaGA%5naP(bcpa`W%{W3Cj9t4j#=!4szRleb^L zS1CJ#*eDZDaUtWc4LA&h2I8n=p8!7lhW#KH<@K0ucw%9)chYTRR`;TeZS%hI`#@Ok72)pkpeZn_nN(jDk1UJ=2~?*2{V&sjW@^o#HnxyowdEDx7b)JoZ_8w8|T^hQLZ5bmYf}* zH`<*#OdxZBT$G`XX2mH~=43y{T&!N;MBM)>+g7U^mS^U&gvKj>X#5kK*}FiKbnQ4c zk5P@$F^uJ0j>COEJUU4{NduI{>fB5Yv+;B|2homf9_Lap=dk*(k7tdwhswQ{k&M6B z!7b%Oxx4x(<#A}VBsmsKPB(#JeC{7@^iz+2#M5GR$x* zFk}@v=udp|z^zPhD#{U3;sS@1J1OlAmg{Gvav#Eg zNpf&$pZXAO;>lEc9^m_2RkpLOV=$gX-)zh2=7f#7r6gM_D>AA1qkA2cnTH8iOE+ z`~CH^omnxdmPG!WXWXx%{zhpt6L4ngoo8)TwMkl*7ra5$y@h@<6i8OtDbSlvg~Fuh zPz3&B=!{_@#VVS{(D#dONA!O$?cYiQ_s(|BN1ssCx+TW_cWUZ^(}P~h8&9^6_Jzvr zL79jpGtMAuFZ{uMO*n!84M@z5l=Dx5__UYM!ukX|kXalmn{GH6m`tm&NeOVyA_1Ea z06)}jnG~d{qwSMz#cL!m`YsxFC5TP6jEb&j4{#jJE?Q$4tDZ2XCzleWMx>x>kFSxX z5AhGp@;g6|%~&VpTZ1e2>|W%qKpez%S;G;iNe_BBY*SbX$27c zQpwW1pvu8kdp9KA{&aIc&8{xKN0noA|cb2Grs66mF?2vuqBXQG0)cOcavJ}(6hJn0s{$N z36&p&V|qSgKITZblrm*0R?T}7U-v7<+@9mVt?vOZy5I4{zk zR4;2|=MH!N;Zr8&9BJ2Sp$G#!r0P`yFy24Z&#!vX$l8Q&*L8?}#7m+g@Z&=FW>TBh4H!RiyoB zwp)gOnR-Byo>XiB0&OLYRr`@**i{YvozHzT5bKqN6%0!si%K^z$iVUZu#;a3pXx29 ze~Qzr`zv5>=NquxyNck?D zsGJkerg&1G6tz&DUQSN;M2`J{V}Maf=p*Ppm{!^yXtT%1T(L(h@KJV`P~!uCRhv5r zM1sj8%n&}21y@v~{?`1Kir0e};3){g<2T4`R=cK!Uoj`XHd17WeSJh&scZ|?i(#ZQ zaAIuYA^{O1qvuG~L36N@Vw~q>^w%D+Z1S@6Y zG3(r^yMf6?gZ|z+M#HO&bqK$#tyr6kknvndgC<53Y#PDv*Bz=T)s{Ooa+a#V^Q-GZ zd@0J#uNGQD+!#Q*wH!<~-7vS9C+g{JZHsc;^V_Og&LaJBPa3dWws?w%9&Boe9x_j9 zasY)A5T6a@NgNJi`(_~fo^*s2oo(y+OkexQmW{c}rFqQ5n~!$f$(1;y{oa^-_;H#E zMe>_q7U#P(xK1}GN+dwdU3rzUNBGg@Rr#g-puVD*bEkdpcM zxL!|`jiWk5Xz0pi_-v)R=x&AU>c0NfH?p|<)o~Xx;o6#42ll(SMWwwzO*cm4rlr4Q zQhwPKT@gCdK^!+u-B{N#$<;D%v7NnpXpk0yKf@X<$U?UeN; zWP8W~KI8FSU_$QNL8Q89q#qHv_K$N=qQpEX3+228fa2eLp}hNyfN%~{KpoSJ!s5_4g&gz*r;nPdxmXRu6X*9lu3Z_Ww; z0K3Qwg39d``1msj;+b1kEkytt@nI>UWNRX-bf>{ePbL6hiH!!@2jA8)0R$l8GTt$t zj!!Wq53)TBwxIxpx0r8Ms%&C@5A1Rz8z(94W4q?*L zCv57S@G9mg0m$W9FzaXn^UCDv8wuZB#3vxPA zz}icPd|Six+%H72N4PHZnJWEi5kweHt~1=9$`u*s#htVR3wN4EVsr4n>CnEesA(JH ze0Q1z##LAOWm4ah<7F}j5({B6LtesQUZp?;!mmdU!5GGV7(N-;@7K=}wG-`N74jZQ z6Zp=9DCCmMm>9TK54*rx5bY$5!Oh3xlb(%aE4YezGEbUu*k}~b_bHr=$dB5w|J_eI z>WhZ>-TmmCkaw6m5@{RV76Hf!<&bZrR5K~FgH8j(wCU6bfX+)=1TvYS0n9JIh;3r~ z)mODM!Z1c2OaKbzEEt`H6uHM5KM5E_f=?J;)*~S&$PE6$`0SrF+Rnqn;Cuns~{?Y|xv4=xX=El)Q^l$auDWBvoQw24k?Y;6EqSB2J zT_SI9)}p4i;cxCp)x)x3$ z_1ZGU&JF{+R4zOIk~NOT&M*)5WP7p+k||8?c#7{>>3h+3DT*Z7zWqZ61OXXnC`%NCrwFxbpe zM|@l!9|{?;wECF+3U2eg_B&61S#eK`!X?m3-TmT&&ZN^JVZHv*6|GdC& zStK9^d<=^80P|2c0S!W3ZR7}?(z0JPl&A~Nmk3U(?h?aommy5{lh4*xye>_x3)uwm z1qGfrS4ouX^^XMkhhP4FdwaaVK`@G%dL1Q%*h|(+4Xu4eq-ZXlKGW zi#e%iC#bd7w&xn0{yXmXJTS(RNz=Hpm~Knt7C~UT^w~U3Eo@Og0K)>7R?nVpO)EmC zoi*maxWR>XC(o1{hyo^z##p)r9U*^ryKnMErySMJ{#-dq7oN4;4fCY0dpQHhxc(?X zS%O~gvweFD1$%Hyj@W%8wIbQo8QCz-OZ(K1I}401e}uH(~q2V_Dt=0EnaA zaDn?z7|_sCU#{gQ{EGNcvqQf>2{I+nKsgU~;=sp7o0R1mF%b&WDl(0gq_1 zED*>MI&lE{th=S8-^fzZj~Fx|#69=q*9jd+?*fC;jbsom2d$v!kwlz&>C!ADI zTFryxXa1%hA8oxS3D|WrNQKXDtL5hy=+5FD z#s-B`USi$RR|`?rO=m3Cr>Tk(S-BAYE{6R`VcT>yYAzgpy}v-CP(Re^d6tK6)K}q7(q!;n-wF4`LwnUuk%xl zz|X&Ie^fiicStHj8ny$x9R6WJx}@yJ_ZH0l)3?`hj@xkkbZK(Gmcw8A@jf6=D3M!9 zg8p?qKmSPXcc#|*nDEPy@SS=cq_ASYcSX=W898O?XeePv4!Ym#}7ds z`X<=9eZ3+w$4&N!rcTa83w6S|vq5bFT>;-VVGhUaOSTRVFlGROm z;}uc726SsSD@{FfJ&IS1Y;+pELRpn-J~2JI5v+<+03YISIN<#;@DINrv=%-a*wiq` zL9B@$Jw5;RD+dy)&0~-KoiiHe-uib><>&=I@@<~k+3VOnwgmfg_FyIUkIl!%p6gRi zNnIl5&wYLJ{_nbJ7Gt*W`)xIFqPsIU154c|cAkG*M)!9i^vDETfD~O_^?rQ1T;#>I z5hssaw853)WBTw;`_3Je#s6f|v_D?noc}Fyd|s`z13-SuBlcU5m!ut%phnvmEj2*q z4I@SxU8#hK<(!D(N4HBl0D7*CsHl$-MWpZ8C6xvT@-h=mYF2D-ensMXZk%n5J(;Wb z7kojOv$e1n-OQ`5u3#e;laEFvLpXuF3b=!M90I9UDvoo#b#Ywj{^WbqdP7e)5C9Ux zu65NiRi^RM)Kh62q?(89oCaV0+qg}^|C87J*1IC>$SN_QBhf87l@0&!T{HW7>enY_l)@Tvt#44JaMd_K$!c;JS;_~ zyE=QWA6z}t370jO;zBobbyZXt;mUBSBrhQ4#&suRUiCGrO-_t1hmssO5l0xYH{7H*0Gc8Rf?}N+Pw-SR@r?J-u3)* zlg#sz(vwSg=&lKnTfodvKY6Dd=Q^#c%>tPo?l|a&f3$UJs0x^dpSc&6l$0)o{QGmd zeXWGgGLZmN|Ejpd0KkWtDTL>3y1QM}1kCGhYVrIVWDg8j%#tjdF4u3$dweU{Jo{l7 zJ=`2x)VC*Q-H-v~!dq<|#Bvr>^F(!s#z`&j8z4og#6*?>RiRDGVEOLb^-%P#8!R|) zm3HZ&<3mW;&CtdjCgr3ABK0zC`k4eRyz6_m=A8=j?TRy{ew)A6)>pJOglG+01`S08 zg9C!T66q^uJd=nES^xLI1ZPdFLhlb)t|Z&|dm&GnzJ{^?N+BPRESPn46vm1q=BOGq z_1#*FabP3hFK3Y;Shwii&4H+?tBX!b&1B+3F8EH=fs7FoL+L0)m<6yHB5Q|w6D->z zR4T_MwU=9wU3)BNToDo6|H9)qMz*Dc%{RlEl$y zi!1|sa)*_*b5y!lE-rEtyw2&rV&GEJ3_)D9r&hY5_wgI}hkxc5zdmskpVZ35R(*N) zsasJnh+>7Z0&|!|-u@PQ{Q4cLaO65Wm)poc@+7;2fj^$7+w7y{TNur!MKo9$9Hx^b zu`<07MHQ-P!0h93@6UHM75JL%KjJ_=rU$d01#3@8g4zPvt=>Z;PuMIq?cU+dkl>Os zl=O9=>)b^X1BcquK%9(Zi1ecp?`iktAh87T#xW?Y=B0GF ze3&eb-a^1kDNbi zDcjlM_$pmX#|B~cD2BErm(bA<>#tx$>|8NQ@}z3Ool(%zIS>yp*}Q{lN3OY-5~&*@JXtkFLCgv4yZl}>L#_fb7<+eo_^leje|>#5nRk88 z35nM4*B`Zg9aApafR>>8$^ZAuHuPD7c6w!l6u3p6hb3-plQ)pd@HQ&Q@yMW zpTSPi*_iz(_b&Y-O^#ua?;a*sasbZLz0ax{mWMT0$UA!Az+K!=)ZHNPd&nV=-j9DR z0hxALVdqXY%mnoG+}tjjln>Y-JbE=#i75&()Ne0-4+3BCQ(g!~{OqHEFRk9N&k!d$>78A9B+vu@oI=1EEZ%5QQb^s`vTiVo3Y>Y)tGyN`$O zwwkJ=PF5HJturqRD7g3KtJ_{YYK}5#!(^2V(2~|6bVnHNF)zRCwZ73r(OE-oCPETa zo#k69yjjC$LwN|&=0hV-SM|HBWFKUVE`=0Vr(UG)Y-1U-i$n<%*xu;xD<89b$Lf_C z4kdVrf>VKnjk=pfvuHS#2En&7Nu`AE$pVJ0x#%iGZwCx8Rf~N9y()}bA(%7sjz%*J zqFqQ=!j~^dzY=g&nT38AbT4FqmZ5%&^~#zw$y4*a z6I;1iQvxXgBfYctHK|E;uu0r^8if$c!}I2b7WjuLQ1upC43r#8E_kRR0lw1)L*+wc zFnr%>X*8X_l!rMP1$}w! z)svxJ|6XBO993AAd;O0Sr#{DuQLF&bpDF02DX-j+`SU(CtQ1+o$#SYF9i7Tp$!6Ab zoKioK$S3sQS_xHLi?VRl$Wk5W0g=Vy_dm#nCg;AGw>LFQZ4LZUU=Pmfvn=R{?Fa$G zqgP#e73=`ISRIj{j-`<&;?3^o>O^iXL^pM=m#00qzZ~wy`(g^?x8<(sclSK965|0s zHVZR0<=R}G#rjDzN~lOU)>!%hW3w0Hhf^;|!vw(6L}b)6ThuRt$(T*&cdy1BzR2g= zh%kxw=L-GjamsP#7K*wpsI5OhEIkhQii71mYr>5=3Zg z*<0nrNZH6jOKMsKgH)$inGg@_CIPmbuv&;sOh4T-Q71|g9x@dKOBhujA;A21$kADM zfy?-|%1q{{XafRkH0<;a)^rSLtgE?`yjRc#ha2ab~+^XhGD zyVxQVyZOsVD6C*wn#biSWFr!v#k*v8{o>)Vj&>X3_OY-B`5%|{ZdTJUvbzOA4w63z zrF{N)u?~x9dM#hS!c6RHkv0&z;oFSi$R2cUU%*pDu)C7tw4lhyNPOZJ>dZzu>ySxf z3GP$Es0+4X1B$NU4ju*A#idXgWYyt9c+EH2X%y?Clt7 zwdYFr1G-G!WGH^i6>UsLD1G=&jcvIn+q~^;gM}|}$M-i9aY&VaE}%KW zu%)t)Zm(u?UcuO2Iw*)($BCG>YPi&Z;#Mm6?ono`Gk-@*@NFc=z(&7rEYAL*W+7}1 zn%H>NO;W$(R1?pTyg8n?AqZgYVgtQ?LouIMe>HMc4Wo(oeEzsIY?mmPs(#YQ(qc=g|M}f@L-=b{pY4fid^a?3wb&!HJLpg& zP@TQzL74Tc_kM_Ww7IdWQ`};a{K?AT-_-*vekf(vWhKHM$Nk7F} zU}>46-%*_Lg6`jIucocd8g|V;OZXGcE9na6ImKyOt`4WR;)l%LqVilG@`=e?$Ans> zP;L%=aD2wTUovU+@xGJt07)R(oFcCYAeAmF)oVhiYJQ#q#yJTB?bR^6*h%V*n%Yz| zW=KKgu*FL=07>o|^H{rkI$k-yZ@d@n$};hUcr$}-t57j~&4Z>r7CPk!@}sFa4e!1r zZ2(fkKe#<5neP?DXNl#nP=p$-ssvmg(jrP+-StB8a&LN^Pzl2&Y4>;a3@=IIRin?- z;7y#D(=opj4YnxY6Or^@4y?AlO>EN48~%$Rj8kypLg&NL(m=-Q%==k0S@yxA#23UE z0<`tg`gl2(KR9WpF-BZit-TlR>!7m$Z1(c2n?CCjDR&Wi@2P}}0_X+WylO3hTzpP) zQR#IDrp^hH<^p&DZ`$irh0{7LpkrM`;ppbeY6BA(>^0mDSKhFqBi~IP$dOo3M(E#W z3Vt!|aDA~w1^5uOmY1r?lZB{DK0ndbe&dg)j*LO+PgPyclhKFmbGOpioX`$+xS=Bo zE&&M^sh)>HGLl1md>eUuyd3J)yYQXv#)uP4sA`s(+JQGNM%9v+<)?lF&%3PRfGOqU zu+}(G;k2S&&O<(0LgFswT`N25<)+O>x}6-~%NWdy)F^LJkFAh=&wI-fRY@M;%5f1? zT~)3{5IC3w2~&IpJ`}1*grd(rZ92F;w>>GQr^S08`l8f}O!3Pc?*a5!pz8SAxMyFm z<-}w)DQb{$!Ao2hz z>WAmXFP3ICfoPr+?X_s0&>!tkArMiy_ExSrjl+~PtL;f?FFs=vG%gG53_-1*M2y)) zW>4p0PW4gKvK}4TP;&g^=zPDJ-(>9ZxKj2(^62l@Yu)fSo!nN^;OL&%3>ba4Kgb-1 z*N2R;tP^nz4!fKbi+IwI|Wsr;M&g3cjRkdMd#{mN=#dC|bh|>19Lv2dE231da^@eS|HTV$t&bzOzEU-~xk<8n(8`#;^w@va=Wa^pE#Fne%abW1~D$8N^c|rF2hs2~ zJ*%S_lG>gy7hcvBXN~h!-k`FQKJa}GzXYV@18T{Y~ z`rIPuanH7n z2(s8OXr$$^Mvw!aCBw@}CAyon&r7(Y-Xxm=d`nP*3b*3lZYMU5(OLXysxNfdD)NFR z^6_*;kuc!U#6!vWomb}Vglq4^S1Z~q8Qsfh1EIY$#l=1REVMir8qaxeHOw`O$C;M9 z-@o_D!@(Lre*dY;3kPdDyw#^t^NBdA{C;GQE(u)?0zvz&R*xXS=Bc#ngyw|Sj_o^C zU8HxR(|tM+J)I(VbuPs(g`L0&?ccRY{TPR?1b=iE53HBEvXw36ze2wvIG@Yf(bEj- zgFizRTeK4thr2hc-M6&|cdFt19Snr+SL_;5LpQO7nM44x%acELZUWTJm$#n9E{IXj zs4I#BFC0lmqX^9?Gvh75(!M%58-} zuntBAN*m3y0GuWZCI03BHm`Rln?h;L$qtz!hYi5Fs@53nRp?FVp9#czZ#|E5D)X$o z!J2hQF1ch=8qLc=^zkJM#h?txHJ4!5rcdZfEH)94oh=l>b+_?V1uGRBOc7V}H#q13 z`i;l9x#5or5?MQ$OkTf(cI^L-#7vMEPld2ok6}Q+jw9k*dmH`Az3)$O@qg_xw6k9D z6M~%7oqscmI^my!vjkWX0K&ZLJ{kJg`L=BedesRbt)|*OS$(^BwyF2Ax3K2A&>Qvd zFMq~ew~MSafY+#TG&g+MQv1SRyuK~YT)0}e3t{jvS7FZ{@S)x?K&)aQ9u?U!w(>WoukbHRg99{{Lf-og{K@?ac6jyf8Yvv6O+=UV)yMKT)C@h~SH6C;lILDL zCFAwUfgh#k!WLy`AQUd%_(;jwdE#ZQDlbdx!=tQ@WWSZA`>sRXkMi)1HE4G>8Ql|= zAUEF8R}X_yyY)2N%NQY~M*4I_=h#gw8*zfe4mku|C}!|UgIK||risS!qdg*3n)T(o zEC~SD3#)k3&zJSE>T|i51s`G4xX8jiEBENg$jkr2#QSaVZCE@t>sG0z&LnFhqPCI(A(91EguN zU6GY(nhi-OJ&-A$U#~Jk<=UFryKv)Ey=Lw@J5UVWzbXmX1I^*hM6w))shP3PXuY|q zww}@dHW6X*+Gc&mBC6NZ!enMh_a{pQwG=~bN z`AX`H5PIUX&uw{cTI<6P-y=&%s|LhCh+D%hKNh75&`O$H4C!%zGbHOa1S@M$Y%DSo z0_GNDX-V-tca@F3i2FnRuM8g}2<$)o=XSpf2xg2tx1$FH2OFSirZ+-;hh!B5b*n1N zd~h$e>*z=joR1`)pS_$JwvIVgIC}Tki+SL?^x(=%oy!65uo=bCIXu^KbdNe1h5R3v{nGRy zOz{P&KEC7sWr_*j>=SFr5-8IWQL?Sek&K_Gb!Zz*9b2Rk9?Ncz%xMpMvZvS*EI{ds z39PJwWxq7W5hFuc2Z#cm_ZJEowtpd^TYEAL$bPx{(ckd^}kfFHw=p9@xsoRDo(udb7 zz3prup8hY)Tn!~iz>N-LY){PRinywVb9D|+!z(R8)t3#F)i%e!_ zW?YybNeBuaa~eLbfg`*%%&&OF#e9FH|I}|M_7mz zae6IQZ5z-$`yZ)+0JxMJrn1KK-4VOEa{cSqFDr7U_E;W3aHEyG>_AS5<9g5C1snR{ z>?ngz5nMw(mXOXBJQ&N<#Ys0hV}7!c_Rhc7?5=2;`i?Uqj7q%Yp+0#}0vHg>**hx~ zE&zP5s-jOvHPx2+fLGf|L#H;O=ufZVpX8#BE8R-&H!9X>bgmRsxC5B5Xe=5leU9ME zdHNMFiN=(ctix?i98)S?x$%IETyqYC`u7&-vv|18#!kyo4dlJ#-m75^$I~n#)_a_f zd>F1)_oH_P@uy3?b;B7MdtxUU-bhQJ0d}Dm$STEqrMRBmZ%646a&{Q(kB&T&#-AJT zRgB1Uyu}K?kqV1U*T_?N*6a0#b0a6^Uke?LcD>~5nDcW8R)cwao9^`t_c6Tp%j%c@ zE@<5a*iQ^0Snuj#6%q*aor-+Wx|2%vq$?E`{BjFdjw_!OdCruInOczaO;Bb9c;wBQ z9^Se@C~K1j-zjN`(L8UtYP0iC;1|1TTLEI^e2Jug?gxzb<`GnQ{4gH`)C#j9)03BZ>zu;q%n)B$xLYrJa_rG4V)*HeF=u>QEgkU8 z)ZmuuCz#}ZsgmW}{9WK%I%ZdTsjT$HJP!Mng%1GL4?m!lL!SEVuUsC3JL)|dSckIy z)7@w-H^KK1gZ>ICAkU5>irxdBMaz-;N!nCR{!($Zf z5mG7>uP?<r=mpMth*;MQEez&REj@nP3l$1U% z5v`h85_f$(N_{W6A^9Z~a3Fp_&S^Gojo#VV{}VQjiu=)iK{`sX^09t(fe!%4s_5eB zdc^vQoAm=op^3wqg*+4*Z6+W6F6`vaC(PkPDGh>^k}kFF%GM2mTntnx#^W@vFq8Ol z!~8)oYYLDWGMykmdMaJL?mMG3mBvst<;P8)MZM=J^ge2)_YZHFidTuOawGzsM zU6bEt66TX^xgq^WlxVZ^PL25W&kWDHm+eO6{YES~g>O*j9=^W;rYl~30 zM73ciS790*T!MS#(Ix~7*q{&l zRb>oHjYO;BDkQP2mzy;amyT+Gmxya(L-=nMZXF`u>us*zoh@~WH{OfaHYzSqWPO$M zC<~;Vf(J4=lTDp0*U-l_T zsTky(Z`1qKXH$7LAaOHsPzWV%bpA=}tH5#9GE=hi2M6^WK@NGhHBVylo!Lr z;nUrbmr8COW~7rNUzJw3JcSoHC%TRXXQx5?0I zJg&D+nNm=={d;hLzW@`{h#ipu(!1=A>4m8>;XEeIB9k~AF1qhToT6T~;-}+!+g`sL z>?&C4wfe6nsS18(3&pJ=0*EA}RjDE)ZG%)s{(U5Pbqoc;p90%_FaoQWAC(G{~zbe-$4vwxAv@(Ni zX72cs@OZsBTDO^j&_doSz^Di^IQ%$7so2CcKY;%KC_2lyCci!oZ!|~^5RqoIB8}1v z7%d%2O39z@?h-~wiIj9nI6!1{gMf5*cZUe0dG2R#`S50NpZlEO`F^h}N@6?hUk4pb zYByL+`In8)gj(L`34}_rr&;l=-1?vI6MKpVT}a{9bj9ho3B9F)1ssmgHD7!ru%6g_ zDfVkJ1HpzxGVJT$ymOQP<)9`ZcnarAzyj^7hHW=<0lji3fVv~_pEj2HHW#$~UwuAJ zFLO~z!G@|0Imw*Wx=W5zEIU_x?{fK-?{`4gRn>b;boSU-_1fW6C59g-t24n3BZO}v zaYi7L`V7I+;60p1R{jFglqcp33wIi)CSrBV zjJ{fb>i%Y7plQGy6oDw&SJ0we?UC+m5|?sf_z*UONCjj8d#-IOLxq)grgJs7cLDzQ zZi|km+SBK^=o1TloVCp>+qi)HS#6Slje^$L0kJp$Th%am_WtM3m&wU0+zZblv52G6 z0F1=9xyBu=TDNGZXAIu@OtH1)0r&*ezO}d4bOIpX#HHXV9z~gP57PF82t`;_-1DzN zM`r`G_v&ni_#V#ASTTw6;g}3w3Ig+t+ZF(vuuU!cS_laP1!t$^NLDSpqbF2|ulabu zA4uN^8Wi8eT7tBH(a!s^TI{&$Xnk9KmAP#Z(2Q-lNQAF#I$#;vf{A%$UN3o*aDKAf zcGnA73oh@QLIyTQ7^EbRfvv?0*@tgZcSn1Bp*qSLym_`e0KnIZ3;E4(h|{fQ$)saZ z8c5zkb2Cczb)|ZS;zb+}!T&uBip=2-PI6C?DrR=~fh)+azvAIN*2rPVk#aE><1fnb~dSShb(bv^|0gmTC z98lcH@3llkaxv4HcXa$62QsOOU8t9{dR;Qrc5Hs&S5*$U2Gu;6lDocSO_eG2U2j^L zTGGw8&`()@J2co`^M@d}oVzT=P5#~4@Aggw4ALE+<8?*Wt!S3~`XB2W<+@NP7am+g zAa%{W#qVS|nQV*qG&6#0Tt^OB{)St4@)JU*h|9w1px}*%##_78gAEtIK{$)4wc}9J zYj|wb=)wl|foV7!$UKQB?^ zgoY@0-T&5^NK$PYGjq5nfSzNTX^BT`Oqu9=Ml$s6Vz(=u-Sf#^K6IuuY^S&LJbSDu zZUbnv11W)X*S4i?AL0`lEGoCS>Jg@`Oc6dZgs1W5d8+zYRMcNYdeoSaboY?aRQ}^( zoCrjiue8km6<7x2jlj&fq~Vn}BUqqFCQ2Ni&66?rF!=sw>+sCNxZFR6w&t?ACbRXm zVjBfqp}YlRhOf7zq;KlhGF}l~ z{}C8E^!6hBHBkSy?W{F&#NEE^Om_IXS3L+&5laS~t8#wK-g!Keay;Ky-chJ(W|s4ZlB8gYoQ&c~W%4z0dFgUfmbmU%JlX<8O9vCz9C@vs z$`BRD^M3JMd$|>1es)BO1~a`D4I{`W*e9~xa#&Gd#_dYiVXO@PnT&G{pcPWArHrA# zQxTer)$>PtJsFBz)_aH4os!#L_A_h(E;slA$_Er$Ous+-LhP$*TYp|(1%4O;EGpMR z;s;LF;y3ivgb>DC!w-$Yi|s{*&vZY9Q}nX~SBdlUO;jApGY%KEl+j14gO1j@6*Ohc z=1{xUC4}%K{tNKnS}Eo8o8_GvAXfNqx$DKr(ff_YisW(4OuDCrRIeQt(nbqC_WeAf2Xt!2D6%*zP;}zqeZ*#e$%&@qmq+nXG-{b80?O&JV zg^R{y={V{?{0AYNw|z^BDNK#gOS|n-q3J}gyYpZe3F4COSe)vy;Swp|j|Y4@EeNhB zWxSpDk`KL)SGdrO9n}r|v5^hl?a#eO*b)nYLv5I)6FASad`z7Mp z=s)SuFhz29Frm6Uv~*W^R>em$+d}Dob`%=p|3b$JV845d17Kqor075QZ3B4IqoP4~>YM>Y0?S$Q{yW498(M5zb*Y7a$b&qKD%FSt* zy`O1#Cep(k72`!h_)~{JtgQwx z?3^6F>Z*d}0*)jqsO;hYu&}0uCSCa4(l$}kugjn+!xxVVi){`{ahnFAjE+zi-Kjw* z7yiFGUNeJs_m4S)4DPDxS;K>PUT*|+SvjGgZ7@{OBfjrPS>*9{^j90l;&)nj;McD& zTbCR`C1t21q>#YpY`0e^z_?wYIafJT>mXi&9=rFWc#3cV%QqWwPY9y4Ziln9YLtM| zJ*j$@@2%+c?4CP;roGj49G6{q@GJmJ51}t+wE$FO$3t_N&-Sg-mNFI(6~Ee|h+}E- zm>?o~KB*1)w}#HEuF1PFZY1Z!p=y}IsYw{XZgu0Q}@vXUHC&z z%+fKp*}hm68O;V)G-5YcViHH>=0gb2BbO(Gy05?M-4EvRMqJGU; zr8JP=Mq+#)xKlssXG;YDl3RewzcTHIwc=si7H~@`P70sobNZ8<^3iO&?A{9D4Y@}v zeu@8)NBmz3;`h1u!s*kuoIVrGtpaYt-|Mr?$BlI@Bn7m-B^IxV|EJ3aEj%6u7XOyK zXoSc}7J(H7V`Z74!?5La2IWi|Z%?9%t=;esD-??NP&Mw)BKSQoc9hTXgdn@TY@H3z zaJ98{Q$FwZfnu;jFS!n<%`c~(P7VnctMDaZvX~dc3*m_Op{LJpjrYV6sq{CA*yAsb|$|#!ZtUr zDylCzog6~tJ7GRAbfCK=n3>TvgJJiXzIfVTh~|l^n?Xv^vc%*qG3{Mn=mukc!p`UQ zs3H~Gy6NIffXWJtEer!bp9ID}E1dkyp>ohLuSr)s5A2N2&ws@Yloj$Kc>BOZ0}#o-^JJ36EMB34L~#ICZAf){<2~CnagtvITkoeKM-l zz=y(~xdloyLDzSCzJ`#N+It|Vf9{jLh;~ppCuF+oYUwJt*o;6Ofp(2saS33X9edJ` z)L#ilL5bmr$-!oUSw#JUdfj>jJdKQ*?H7LfHfy;q#5b^Y&}|!Dl>Gks;C>4CYZ>tz zc^7&Nhl+UaHAkbcfwRl^Pf#+#gyUdgQ_`0ZKF_R=`mWi4dKEba?us;F5a{KyW=QFx zZ}2C5Jvc~V&D#oT5YonW7=1CR_qBb=pgddj)gV#JH}&45%J8J`>DKe#%3u84^hu&x z7)Wx1TCZlfWmuWLhWy7@D6hB~jGtI9J@Zjez{74xkYXOTBr*IUe{%G#v>o~fBm2Z` z4~sDVn)jzLegDBUxRvisM8wq6gcp7V3W*?IFWf!TU-SSH)zW#L3Q8ly(vL+q(NpdXdN;DXrGmOze`Fi?lk2__@B8WQk-B z$)E?WE;D{!askY#g`;T$6>UkNoWGj1KiB>OJNGrndCGCTrQ&Rx?z=}#>y^{I$StLAY|O(W{>(3^?OZ13 zrsh)6ZNiAJm47~g-~Nk5iL{C;34@ zUtlO9rYH%WQ97MG%HC~@dkby*eFRooCwDH8u=jX51t41OJgo56n-Po}bB1$d_Tz`PHvjux0C|3M6N>lv&~{&$6L?*nsXkN_ax$HZ0p<4` z%kMqCuS7ULK4U;|hzVktn#!!>#Tjp$fRa8hi={v6_-^I=cBGLBAzoFBD8@T3#3J6s zi!(i@CgxTcI&_2?Q`R0d<$Hact#81*iOBiuD7@~??tEbe^HOjI4Y)Z1g&>`8U(`6+ zm9&*X&MTLI4(4O}|5dCP?xIr;c1E1cY*@j4ztemSwrxJZp*%kVPZM-d4e51O)3{={ z_JS}ptX-NsmeH@tc*wqTOM_Dys(XLB)&-~6b03$ z-nPXgIQ*kRgW)%ZL2!|@At~Z0MK^Kz0Aq^3-I)dX=0BsP{4V)bbUs#S-bM2XGnpF+ zR#CD6g?Q6T@AutcMkdC@8S1i?@a^d_IF>Wi1y>h0)__XeTq#+rrm9om$3j}V5dILB zj+Zk_x5fs+wUPoOqadR*jtn$T+DqFRl@^-LqB=nqs;`UG{RmQSIA?2sYakGA4Mx6R z!;Sc^_L>1X0t2PnnO$$_^c!snJckl6aBP!+7AtJ7b}4a6DXYWKnIuij77Uti7D)pO zlbt22CpyO9X_10u-?}&%0342OpX}6+&`>R9yDWCEswEN(6cIT;ZGea_k%OZEpo=-C ze}N}Th}!Y_|3T~3cf@w0bltR(!nx6u0RxI??KVM*up>e^GLraDm5G~IP(Z-vk2s@q z$79FCeZmBAdc_z73xsOVU+~?Z5Qr)ZjyKDY$d2>7Hv%$7Ds$)nV<#RSfE;u)W&HRz z^~1x_OxxzfAs(z-sj8%j9GC|sG2%avc0A@_s4f~42xF!hVh|>G;!D%z`(1C4KTHZ; zc_XDB16FYRWOKSZf@Sd9&GoS7+MkO5brA$qr%_z(zF*oE1peNWKL}n~l)=J;*Mfs% z;fga0Pa)(_coQt_4D2xLMi{b}TqLt{a<~@)Tb3L@uG9*BNI{#}EpFls#(p30nJnbg zi!Bv$V_{6>+;74)TW2kTksYIN(RDMj=3=bU-3}|$9z!Bh5)y#qlhet`$?(;=_~Xq% z<%s%-E&H%IQ67~YaZ<+xc@iIKmW@OP_@>aWv~sI$*~F<{f2&h0QkNx9`t<=` zHBJa{mmo=>&H)!%k^piJpU$36&tDTtMnJ5u)Igxb+G49ui3tgcX=H96WuUwT9&Mfk z%|6a8V4_T|eu(^YM%I*M7IsMIdksyl(8h9+k|}% zB$kSE$6pdN@rR#~05{SUZ|NW9S)9Z;_IwkUihYPMMVH$!pO(|oQzAAcCB&2O05UX$ zTdIchFiYg2ey4rzRZ)OeSzquT#hJbArkfU-$L}I4tz}dceGH~hXXP- zerHE5-7vjxtFRzYeRX?;0|KrYflseaYRk)0_dk)nB%>Y#=`JS3Z~1GZMbtH^9seIy zY72U7J@K<@FyaQ*-lv)G-~ZisLr}=~Vn7mO{&LIx<{;?i>-WXM~|~vX+StbxLpu-h!Bi8;UP;Cqo#HzCHJGdVUG-jGGdTDoPyRwI+3Aa>EJTM z_hXc%XrHa^;pSjZJaV8%@IUw>Iv}IiphnkC+u=9@Zx9Qf z6$`m`;)twSqobATK6SZjCBNvqTI-5@zjDwxZ4h*ORM#?N!x?zFfgiAL&=d8YtOii>ygXh=& zEq`Y!KSXg@>c&kzSo3uRhvg&xUCq=BFvE|xOq4b4j-&ff#|Unjf$Hwp&mc^!F^Au5 z z@PagjpD!HMhDG$`vd)~@}*Amou#(5>OAx*W?dT1;h|^y1_lA2qJsWrMIaMZheHr1lniRh*N< z18Ader7CkISE+`pZ*t}5sBK9k35Ic7u|OvgK|$uK0-XZwOU zlyDupc%Dxrm|_6nXsn$gSez;(ZmFOLqx{msIp;K|b6(rGc& zN?L(*Numba=q}{PxH_&#_5uO1SVJA~$S=^Z&Ja$^C)2l_%p2Sf7 z*F%yiTB0e9JfL~ArWxJy?E3Ed+`+Iw`TpkO0IYm>B=z7anKN{FbcC#>wC`-YKXTG6 zPjY7bmO4;m`%9YfY)vsKqOz8FYN$_tu|?tBBLg9P^{Y44H1&;o zFZE*Er(hOV{G=+cNnd{@lwZXe%$ntnrKk3YVjmdH(9%-?6^!>d6ew~>v8fN(7^qF` zi8qO<`DpbPJ61SDz^yLHk7$FEF4<-vPBK6;v-S^-6s}3w`g-f1(71XvVg^do|YJDx*wY z+=Ud<*hj7C4yH}$2uPnUq|2Y%d{bQDV0d1W=UI-vO!BQ$m zktLwVhEACCH+w7GD1M#2gS#tqQ!70OF*=`SB(Vf4WDCC#$6=u-p_|>Cw&on zQO{fwsX@pI|Iji3mf;YUGwps0Dpn&VB5a2(EotxPez!>4OiDpo3$U&m97H^YEgDlm z=`i;cSCdneVnPZvKoi>l+p&3l~j^1?)Ha(_% z6@cM2zU)Rz?znB%;LA(i>p0UE>?Pmk67-~(iY?R&HIfqwA%v4KkOV!}gT{s0$F^Zc;ebkg$ZVN9^Xcl(FW7CU-)7N14oQxx9+pHhl|RUwjg!R zrD?xKkRZ<8;yu;3$>Xti%+*+;Oa*^1Pt~?#FrMSed}3OAO)5(@*xAQTCwMnpYKO3R zdk#+UV})Yj#Ankmzi=UiKCj}ewb`Dq8PjXb5@B)^wY3bVLV7CjW4YN=$_khgXqIK@ zAJmvjydnqj1{=!4Llg79i;tWO=482PrzN|UgsTI>0!ZalMy0L6RDZY<8F{t-n*wJq zZMbFkpFyp4-nF+%q*>3k*KL%~GttHXWI_&)^}>O}nhU8beO$wtH2jm<%c&?u>wN<6 zmj+CfR@l2#S5FQJeMi>dKgX3_@l3ffrrr2TQC|j?_?c4m3_Wze@&-;+{J{kESz2)x zSyB4E9wmw9ru$)y{PYbMvO_3q4#@d{4{DkkzyGL}VW4YCt{YzP)^fMA$`jEAf~#dI z1X;5T0I&wKh%ZRg!cos4a5Z@Ybn4G}a=^LMH<$NY3uvd0OEUQ>gP)py%0|x~nKgv& z;^+~j`NuLsXAE@jecOPOMfolR55x8jdlAQP%uWe{MuvRbcxV`%!bYkTcGsgLt(44G z8)E)(U2f6%1yooVlJUi&y^omq_;445p?PEu{~9zbc)jMaVo~SS7$1V^vEwYu60G(D z-0QvL+~ZR>3dU>45@n^2oa7E+z%c9`q<<(Orqz!~*ALn;iF(Na4Gv>AXpRYMdZ&YB zGQ87vx5bkqmDwNoa^ceW8(DIr$S%*rK_m%7S|TestL%U9JkH*Jv3wF|rKHDmEC59(XEmyB$-EVsMNg3reQM$5(fE{+3Yce&>nqLBQIK z6jE~y-ve2$EJsKM+lH?y$ioX45^EJ6=mjKk!yPOfhj7=rBMq3)KTq2MQJ$cZ@|m+mg%hLUwofJ$;H>6nQ}tHj+^g;iCtFl>XJm(2;i-HYKRG~Ys}C;Z4xUJ8!ynv zi=Ll}NP=sip)TA7Gp{Cha5`ZgZI$pP6ZTm)vn{VGN%JB(OAqwxE44(?<`qsvA}L%d zq7lnKXC!tOiMLQ|{9(n!#6;%F$Bdnc>I6&xbqzAlr^=UM{G7DXjKM1!VlinxNIPrt=zD0U;_d1afd=;YAEagyQFtq=iRuY57s1UcL>o?$l}maok0p`(DmOfDL^oH zcYXc$^8RF|?j$PVIkTC^%e=KbN2(k7QYMNQ zrTya*=#NrNfJe>-)p)9Kreg z$`g=tjsU7_yH`Iyk5ReB^WNOwqE{QS0-)2Xu$12G8Z*%%SqSeA?exi+0HkQMgWWyI zsjZtX(3Bd1=!34tP0%*jdSiyLzD?J%!U79IEemIb(?alc=@ua&!g;(IZ_0DmBB!mG|fOQqxNZ-vPjlpxbWgx+R*Xk-h_h!~sogp)Ye zj2$4?36?9y!QN6BvW22rNXMMj3+w-tWxEaAsPRLh%cjhWr7LLiFPUK_=oZfv^c8Q? zMBbdUiP~8DBD&>J?DO`ypsH!s{ZMe~0ILUij@p*y)b0Jl&5c2=1XPUJTR`$Z_@=N5 zlLoDy(k20eq(5FMfP4l#2V_}y9>$Lk@z0Q`k2o7j{>|vZp~)- zUe@%E{frXw(f51nT}9Yi;T*w4y_>JDHnJxSvsh-z4-ac7qsS zthbx#3JEtzbBl9-VA1FGUX)H1e!v2$K8YLax5=QbW$s72j<=vvw4!S_OTtBi1yn(% zP!@oRRH3#xh)6HkG{<0xw|veA52b}9)V#B!pZK~->=;4%M^6UN#hC?{$qEOOzyc4( zOJ;#B6qh8hsIsiaX{3u7jbed{zp79Q)K``_^&k@b{1q`^C{d;U%%NCpuqtv|5AxH- zV>DTRoF7%=V95EY>m930b|H#Z92eTB&+93+cOUk?{UZ-Zj>tCF1cE*2AS+A*aa|uI zvm-MmO-!#xhc_vFRMZiODf=oB+e+Y;!TzvkrRe}kgz#O>+w{9Y%o0ACCLl0Y+AR_= z+L!cM>XQx{>E#TayXqWdBimQpcT=~L%{SufgkM-uhgFKRRLET1?(N}&Og?-7T1YBO z2#%OdAF_4Mty!l&u`@zu1$t(yUF!2xQ?)px8PXsFm<^|<0(&}wzKLtsM)5O2%r6_( zI}c3RfBg!~`~x~G4#6kvoAGC->hXYCfT8k~=dUA()C0C}!3DA$!keOf$^|TKvW#v} zg-=1XM0ZS{3T#-UBi@hY6AW!r1v#A~6yIF1et5kf#?1J3Tfz}EU$cVwhFHfx8H2<; ziiBZdWKk_E85!xM=Lt{LWtkt^ zU&UWuWPS%8+JKdV+TP+oyuaO2AFpUTW?!r`2i*MI1FrR?p?AQM9Qd@JxM#Qyy9u*j zh5S}pP|3+`t|YfVfhAaV6wGdAz*TtCP_B&dH~YNCdI8QJnG*kd=jw~ zpH`KY8eoo!YbqnglDMKn9UW09>e`0-O!YE?ZsO@&;QaRof;KqsoimiR8-_XVM;i=g z;p2(g{r=V@aF(nccs@5cN(-L-6LU6gi9Bd65-hsr`qd|nxm6X2ns{k?p6vV=r^2Ar zci2WIbe;v3j8y_PdMUiscvLwuG6Xw8o6A|)-g)v&xSRiJ?=DH0U3#;kE2PdY#Dxtf zo3MYSk7*st5vsOo8eJCUpNP}2{Ncm!4tx4`@i6bJ@8>4>o`%1w(jo7QS-KTH@~gXE z^JR1_8*r-9arkBbwaLEJ`wWJ#uZgfjHF$AaoDzP)%@+*Z_|wzK%3fzv6jH2sH9cPe zs_^I*flHZKLI|e&u*mIfw%n#?)6uQu$S*_0nWqtQi-pFc0RTFnMvNESo_SuwHE{iR@;9E!ZC34oe=8S0?SA90u8$(v8qCm{)S0?l zhXpTLjh=p92eI@5FK$!H%rvVtoKlqDKy4}1+zNk@1|(d4ZGdfKj$u4g_s;Ly;qme) zX}tG?q`yzbGaWuVWMCSl12u1DgP`)zCCYYqE}afrqc80%IiHA4e%SpgiBfs|P?~As z$KzGWu<}r`G%!slqy&^NLS3zLCAC*_YYF2EBVE|UzG{-7<<0#m)t1> zWf$vRc|t4GSz9Xwh2uMs&lfcX{}WBX6q|ka~C!&gq?aoI44^>pHgXy<5%sClbFSi8Hkk;fp5o7q{OH@80;=;fj1; z$v`(mM=`o2`*a@{)fN(1lWpkr_wwxF?9rMyd{u3<4)3`|x@L8fY@{e2CJX*+9&&|0 zdbjW6%Vpi^vPeS~Z#-R_OAF+P)huN9zvF<~^=Ai-UKIi6i6qlxN7MR~rxouQWhgOs z9ORP@E?!Jzx?LL3XePy;XS(4CQN;*5A~UhfTEAiU8!6j-WDZXAoI5e>E*MNIY3eT9 zKl0SpeQp39TpTj26QbPZD}Vm~QwFKU6Xcj|1xpdd>mOspqUK(~Z5`O*xRKK%Ia2iI z&v+^CS2LRjoN-l{%>O~Co#9H1z(^FOZ`%5^2P1W{xm%owJV8v9Pf4wiXjB&YWbj96 zhJ~G;1Q-U5`^fOd1o^beZNx$)!5^xIC^@zw0*5aecvuwEyWgbB(8~Nr`pg>79Ht@! z0FfI~O)~`q7+=3a8X(I|NE1s*e9%XhE6kqc-u-VkRR>y^(bZ~6otvpjaA9@w@&>;- zng~y2GL|a6sGC7a2tKqt@=PX}#3q}V6sEiMRE&Q4sZspr*A)i5qU)P=Od9r94+WKM z@k-?hpJO4Th^4U{adUS@nlb!H+)IZaPzb(z(~nyKLcTx*k`ZAq?#d@WKO5LNZ^kmi z0kM2v9kF)cJ*(z&>mRN8O6^`cZu>KpUF+Y_07dUewg*YR|22|^UMNxWt~Dlx&>ObntGge9c)f(VuAF@ou_zI42j z4tU%~^hux6Mg^huoO!kP#R%wA04a93l@&3HM*$?W8{INyAQWuo9u;v$3S-EYIF6G| zm2f_qu#vYdYcgoMZ`~{YSh;Tsx@{fERH{a>)aWNXmwxJ?Am{m(Hrl{wX>@36&={3& z^Y)-te9EBf<-N-nk|gg!UOrR0YN{eVB~Gi2q1Y9Zji~rENr=OOrzNREGz8B+cD1dg zg$|7FOM{-oVI7^me%m#2YMszyHxG0nTvoQtp7?OlpuN;+rW`qG#Pkng6&f8K!Ec$R z@JkWNWNBsQ>b4Cu3CrTgoo5Ov6?m&>pIs0e4q? z!{Zl$zltXW0|4o}t7J^y@_uW@Sgd|ZQCeOLx(A$Vy-+J8cxe6%3}>I5&)Boch})JH zA0WMBGK4}c`FC$AtMX(=)oN-{lT;la29Ak2cVzu2jYF_{t%;%gZ9uniW6{K4&$%Y0vg#wb0!=tKfS((=OJ+d4|G zQ7P3)onnFUaxb!v#9lwP{sZr_FF8aC3X@3Ihk^1`4?rxFejf<=ic3GY6B^i=p|OJ< z?{6am=9x_~BEzSyLd53B#Zm@2iv}}EMT++P8+VBc?(F>r&|fmrWDQ; zGD{mvwc{fx{=?9Yi1D-1nQ*$QVS;XmaO{dbX*vBuwzxbi116Z*qsz)S8069<_sIKR zUP0-dodXKz1(|m8pntpj{O{q$Mkh|W$Z`ET*h?YDxAb3(62t1It*f%59XX3Qw?zK) zTB$`IwiJon)93kze#uDq7`}D`gOjO z-@9Zfj-pVZ@8=uSyPe}OzrS6N@-8>M%}u}l;q=bM`8PWe&~rV7DeU{loo3=V%P=15 z*cL{%vWml4u0T&P^5aMsec2>lKgX!SRA3911#2)HMRZEgt1etgZVT#QVq``k znKC&|Gi8qhNu+fTyJz7oFJcGuMUrG=e!>qgr8ex>A*kLe^+zg%_vt9(E&(oSh(x8z zL`^0o*7tE+ce9(ObpB3%3I z-30f^yQB9v%*^sKJ64b2p6w^O@CSj(qTZ+R*pq0Yc#kd!cmQ7E5hk0`tg zX8nK=cHxX~iy%|uqWMiZ!)LS^pDId_i?^uB%zwV;9w#MNR#9Y9In$C-@cUAC~%F~>{6_r`9 zk8ED(_mUJyJgA**sZ%sX#)C!e*{pHg0jol(2Q4PsG3jRs%YNd~DuK_-zwWEfYH;^T z5D)aqXwYiMGe34?3{pnz6$Xxf6~1n+?h<}IONNPqgTaC?rY&CS34xqpP;r^dv#Q9F zB`Nwa<_H$+RB=m7m@EJoM=vxwJNok`lX?epMj85#1|68b!yGblor>A^uSUH*hit_7mhvndolL=KfeM2`yc^HSW zKU`Ir2>B_94-TRYl<^g_5NW=-9Y|Ui7Im53sUQ0lWo1SD02xY{?FCjlc9l@rKD-_x zL-GnE`s3&;_Kmn>4dCOf!!bu9Tl zUV$SDSAj7*Wp3#?XI1J@BVpL|YgO&+;)&|^5M-2i&Q7MA`SS5Wt;0eizk6fM=UM`K zhJ{7zaNhm2KVav*j<2|LKsCY2yz^gS!P{p0%N<8j5{Cc(!n&L@7IQhwFIPwc|6=6o1dSPexn#V6cDR`gJ^%(`bR1)cKUtN0!ZI+#mq3 zmJYN9n55t@!VK-2rjyc$mPWOF%?1BiEoso|8XIa}kk+NUQf?O;bN%yei73(r=uDE@ zN!Fw-&CUJ~TLXKp54jEiCvAN3QtuJ{xbb)kyb~X522;*||T8nW{Gx)!_InqE& z>UOHFU85SNcacE((r1Tm$-vkcl~T=ww=pr%@T@GH?>7sjD{LuQV%dEKtyO36RP02C zPOwn`0SgZ$ds>}P@a^4ShFg^Y4`#CbbO^p1#YMuRwle1*jkplTBjJRyZ~D7(^mUG` zu05}RNQ)7xUIRC;a0&tD#Pg8TRvxg%o{o#)q&g2jZJptHTkw}WbA6yocG8+DgKspXBLmMP`f!&_2>sx&(>gN0@YCy!om@1;RhhR~EU{k&y`ylNCRV;Y@ms#KD$Ow4{j zP|up-{`M+#?<(LC3v%YmELP0SM5`g*CR?qVK*q0N^byoA^mq)GfF3}jHkrXiF7%M59a^H6=UdDm)Xc3 zQ-~GrSh77hn5HFTGwRHopNUyYB0RgXf&Cpj>VPug+ni7Wnm@o&T7)i=5RE z?+s&)i6KLr3K-mj{OcH)WUJL&6}3w<2lvFsKPA-%#wq_#V_49Osm^5@`u+=oF_9^N z|E5la6>5zneM0t5h%RoSxL+{}SW702KFeZL@lvV0<`2u!+9(hg9Ul^oEXyr(ct5|R zd2FgN_VS>fVIyfS9v8E^5XRolU{a7&aA018f3hoCQ z1}2K#A*GX_y5@`uLsP!$d@Q-WhkGHiLUr6{&h~bzS+8~r9PzY=da(wG7JZVb?uBrh z;}ByP!L1*>ky9rQ#_wbr{KZdhZ3+0^$ctnRpI`3@s`_MT^#idgJ^?#|>A^pV9Qcr_ z#*mlw$}fWtabnY0)0v~FWM28MT+T40>3YF$FoPN8ql#rGN83AmR_v6YCM=QS5#<{;!>K+`YDH#JLYN{u zmk+jsVl zr=tG;GCC|Q6CD_ZCg5}XkMf%|;urA8r+itJTz16gPy5E$XyPj4tYv*ql45Av=aNtJ zc~Dl0$iu&_fzk?} zZ;o=vj?TA#Cgo3TdIG^rxaH5One*V8IiMR;de5fhy8nv!El=rF7HOoO${&xP{h_ z{S)j~ESC)XrS`4`t>$f_C>hI<;?JK2FN_7t4kVeN!Q@)jI>eQ0hz&d#-Y#ReAjjsk z!#X5VdDF$F;5k;;0T(^P%}C)^O7KCtbV>i4KBd6h%ZvX}bl!nb|9>1m9FD9*Lgv{; znc4e{WRJ=rGS1$6WyaYf*+kjCXP>>v$Pw9_?5%9V@AK<_f8@^R^M1cxkLTm*)lxXY zvC$rQ?{pY+d(pwWMfKsHQ(R*Rn7WlU+(;J~Z&~S}L4C1@t6Mp*k`SdSd>@!eK6MH0 zP&nEj(2i?QRxOkI*Mk(V%~HH2FLEKM-ilcA0_x2~x}5{H_Yhwm=Q%aLy@>6(l5$-a zT{(UZPh{>URsP{kv#iUaUsBfpAKg)^1E&7iTj)z(BHZ&8Iq_N0?o|ZCbQPp{f)PyV~G#h_wz62qF}*X-R`h*=BI%!L5Ba#ceCT}?%S@jxa{v)kms6rV(Z;Ew_nTpvjcnAm>N}HL z|L!`ZVEv8ZKBR>zy5$YJvT`~tEiL75v6gAQW7w=Nnk>_uGZZoO&dYvs7$;F1w_cRZ zF&G+z`~d^LbJfauAo(CQoP?xppYQsGYt||SUlbV^`36gXn1{{W*s&LPVQHBvb0q7B z;*H)X1u`$9lD4}N7LM0)MTKA_)1UDo@%0`K`sNh9{IwoFHJVqD9YwpGp%5H4=BfX? z4XI2BnGg%cFsE?GJN{K|_Fp0uz?V=E6?z=W;KeV!&?5a2FGA%QM85Rr0;ILeFTFjm zbJL2;iUfQ0)QI--X?==w$}e*^GMo-vwwo;%HJxs=u90JpHg0>JDy8u%vzR|-b)~&o zFt4d-&S+V{AWJt|Sc)ZyZH1}HCH!(bnDI<6&2n&7nH%*$;-^r;U!Hb-I9-wUw$%fzH3Ol5Fg$e-+^~m^ETA9+z4w(v8w33nH)yS zD^setTH?9j1=E(iCzN+H7aA>7n#3+SNeWR$7wns12N&#Td9+jmnY$74-MB+tpzTPRJWHFu>yv{E{uFxfOOv1i?r5aMnbLdcAje9w_jXEB=o>pUiadOKMXYcF zYqvdk#D0x>-kTk+lb(UMV>DE;@}(odFm(gaR~)n}g)lsO(Q$h|57<)y zTn_ORifTBJfuI;Jtb5IwWH^Ip%>8Da&f}33IzM<#j&7uz;ywPRFZ*(t^fCnS_PXLR z)5b9OzzL2=L%kIByeqAo=ii;^>*(q|ZGIsF9h2bJON#T_mRn?dn`O8xJ4y{j<9x-_ zvDxap#3t;J$Ns_fR$2EZ{M}<16~Z0yeKBP?ijGnMSH+?>avy=g$3e>e2wm$HbXWV` z38y69FOtq1XIWPYbDBR#WQsm-*u@Q%N z{CWGTG-jmEMyk8hFy5Ko){~d~appdpof}R~@<~x$WmySkRJFZOn zpFybY`X$A#CmKNdAE;|{oWK15tRjsTyW$W%o z5JQ5cbI&{H>@dITCbn&-11m9i=XT@50v1Ag+KVs#T895jd$2VqE+XU*H_T9lp&9ny zcAZ-^v9~W&?#5Y<9)6pG=yeM7ZXf;0ONKAVm5mE zg7fw~up>LIu>jgm&LZd`!pyn@5k>d-6_x#_zIp^@kU0gAbje?DL1Ma^>zW5O0<9mg zzD%AChHnVfc|R4z&7yHmdWlENUf};GxR#NzLsk%*VTqJIA4cUN&Ik?` zb@%(;J!U8L-Axj)v^LtN(Jwxl-3!2|yL++XbWqzpyNJLiAX7r7w(Uxv&E6N}B-5&=&xS2rEbN+A;$5z#sZ9OHp z&fs^-IBCuaoaJ|>O`@P0)QbGHe6I{Xa*jTIBSmvO)=LrWJh^x#w`G3aQU6-)Y2knK z?T)Lx%E#3yG_YQ>clKFD)X4tCTB=G5(}W7*E=d)NNKH#d!R+(A=YhQ&vZM<~W9O^y z{~WC?`39(EtJ5b(!$M^#BVE*08T?OLI!T38QXQok5Bi~vq$-_E1Ai6u?iI7}DV%hHXH*07W(O(JYY0iAKn+p}Pzey4p%GqI9FDA? zdNHB6xalpWp*TWX?p`8i?bK>KB)LTa@L#Ri8Mr}5N88t90EN^_s}1?&g2vSCN;=%! zU7stHh;IpR3|iBEH{;>pw?it&9@(QKL+_+kYOjc(vV7ZNGl@$5)qy&JH7@e;{ZXR_uUQ*OR~< z)*0}(Uw7hM>5QcYzlO8B?bS!RK5c*c3e>67Lq|sYXOW!ypHdIdL1M^W2s|Y~=jgt( zC;s#G=)>|1@w(m=!7L&nt#KfL{yhZ3pJW+NMcZOR`yi!F=wD}s(&km&(LAWtUx5nV zOwKF=_6HofuKwTo3s4iQ4UfIz|I|P+-xBCeQwhu-y0R9oL;A0x`UfaHq?wdYD?;6{ z6vt-7%Kzz1j%Fr+a&&8^95GhgEy@*h3wd>(CV&uI+v0gz@nQZaUE8w{Efv7x$`upK ztI9$v8PXW)3#VsrP*S#2OkkD{p4I+bx6$f=;UC|r(I21xz_vNQ5sOXMRVwgej~^Hg zpWQc$XR}c!PcIm0(#$wv_oc5Qp!&x;6p^p)(tFJYy5=`W$`gvL&BSwX!M+MR%zqv4 zl;vJycBYnXbq6QVGKY5u6@817)okk^wEBu#tX7oTv`S!Y=@=Neye60z?>ka6K~51| z70!Q#{>BS-Z@u)f@>}ornEQG`V5@#P+Ove8*W*)YeY)$U&9&`O-g9B;+>-I~2=66( zEMn;FjF0LkJMIWUhLxuHtGOsUM<5_kIEg3~f6Q{@Wg0T01A?h*pI$WF?1q)tBl+ui z)Oh0gH9i$k+g0-vUe*0``yitmJU}tL4piNYl{iQy_i40sW=B6+3Xfk&CHItPDZ7$>$D zK4nf~2~p4VMNcAz4aw4FV$lH$9PmzdnIGx_rjrR#I;I`{jv!f172&=|LCr>0TG7!h z_Iqi(#P>CeI3R7-;witQXZ(bJY$WK0S}HE;9WQQZK0q)p9GRAou#zY6*xvU)J$es8 zsDr>7f}WqE2|Av^?;hv;{68}q!J_X__vhj7NTQ&Xk!TcKL5lg4t;?cQe&m(U6En-} z{!|8)315JgazijZMYEm4KlQcNMkvVsTm<+EpC;o?tNaO^(|Yu3h%K7vPe3#8xPwN6 zXJAVuGfjAYu_oN4Bbd7yY(b8dDx5-CUSP5k2X;JW=RYh~j2S2kJ)ZREMNq%!7rw+* z`Aj$TSB_<^J}zfdz^f+#DmAL`xZi(?&t#3&i6Y_E%E(LeRaeafqi@&b)i*wsi1L8v zR+a*|BxSp5PUm@2_li}|>+Trpam=b7pu~91em~AK#FFrvDNurN)K$0Z)yI$Fn4+>J z>V-waM{skHvP-x&@N;oE>5`$OI|{++GG1KnE)V5!w1U9{3drIA=z=Bc5=(G>#7#SM z%JcI}V!FYdW?+KgjFs~<=8d2$4y%B+wtW7B>D%@r0uMilFS%4iS6{_UWdcuM57Q@W zW$68$`+->iKj%cg8-B6;%2v}LBZi0twk4I5+NwTKcX-qONPO5W^VX{+5O^6%7uU7` z7kHq-9KC0^8(Sc^@u6dW=%FuA<^14FYBT!N%IT`1!Sq29$H7YOM~Xqm`Ke}fp21OW z8i#d3nSUFVPHr)+flAR;9e8(~kO(D{lLq|4?Yf$*Cz~rL}kp(jZv_@d37F1O}Q~F$x0U3~5NFZG}wbX925EvttGZ zW{XB>tB$nA86J*Yd|KK7O*c)drwS+V;7(-OmSZYMZP-)rJ0%i3;FKBts-9WJN}8>vR|wLnQ19=P7mh}$beUU9 zGS8S(rEJF#)XxVO>A0ZsgBr!JX?mI#qTe*Np@A4W=@YIcN4-i@Z(xI7!~4MLaZ}*H zIMDqr8*o|0{-kjG*3L>R{Dj&ozEoZ#Bg1!*mvcJJsuZ*6bTK^*Ft&RK^%=gP?@ot8 z85yq~=cW(^`yew=p~%b&I`fe2l}#|j2D*>1KkVc^)Q<*EjygeE2tKs=AWp`dFPJhhRPG7b1AK%oqryKfN|&S%~!z$E-w@M8}*aD4%f$V-&|9UYpb+#$)@WRe71D{xlm_zsJ$(K4|CY= z_%c`55lFl*AKF`p2VIK=UT*X7{POtlH+7W6IAmb~G3$jk`o=wwtS<8<;GNX-bZcZq zGo#KK3vHLnSsWavB_N@xMSJ%|XD%3@76b4Q;V|BDUf-AFuw)b@o3dkzE)WkAt0dPq z-zMbFEV|Ucrur@3=T;GU%RlPbiu-$%N1lYZ0WV#Wb-&BrOT^nVMn^8J>sbh7L3Zm> zv64dK7p35a8E7OJ9D&!uz#JArvUnrJI3N{pU{f`g{|O>VDzLTV)+8k>`{k!Ken(q) zW;hHo=#hKU%8O=_K z^&8oWyGIQ`9c*iBimuG{uR*i(u5Z=Qw-{^8@0SUw*>22BHx5 zm{f~}XEntoQhskV*i+fa>rLza#S=ej$u0Y=oA=6|U`qa9dWT+#u8lo0s4SB!`s%JR zc8boSO%R{0)~f&sTsW293oIzhNK{yw^GlY$E5r;zP9L}+xqdwcug z4gECPOE@hy%Wsr0!8T1?D{aWv5D9&T|Ig+s-oZ9wOKcv?>ozcIKVcV=aVq#Rxuv1L zOL20Koa*3qgkD6WAbT%RwtYY33;fKCpp5><4GPtHnCnhP2dvQS7V)lZ(HXB{KMd9L z>(Nv6Ya+$u%m4=1%v=0=Z+?Dne$PG@|GTJx_wh4rk{tO7T>t7KPMr{yG$qBo1XOw# zzTQZw?a8$wYVDo;{K_w|%N7rhP%=oRL)uM2@iZ@5c{i}i-e)wMvany(r*+KYr8oP^ zAsJ()Uvar|L7}W<=U|WrrKA%vJSBZW$Kdf_e^*9{qaS=r7nf8drt6V}9rq`HKE(VK z&ZVeCZ8B+l+PZ35B)EI?SJCFk(ie~?`dK?(xQQiZeQ>U4vHOIFk3&HrFvCq^2@VGn z%q=jm^nOsy!Ooz7=7O$_naHAz|KGYee><9%XG)|;KB1ul5pxZSTTf0=W{I$CEKKI< zmuGCW@kiS-8%|CAMwZNPLb!Lr#}c~tK9G0iq_f}MUM#=*|JVGTJc#W?s`3WV8DHEu z8m0nM_o0oi2mo6rwQ_l^@9KZ-nowwJk6)ycfl?3GBy`z7oB>EXnJF96W>teTP8b71 zHe36$Q5a{GwMsJOm!16sxfOKTq}1gQP1q0*hyC`*irQQcd_=%t`InRmkDaPU9EdR~ zk&!V(Nsx7&cL)&url$N7=78#YzTqpgu|fVmd7BER7zj#yqu8FmthAUt9QKN$u>RVH zF%th+aJXnCeSx*_cLrU?qT*Jk{w*zbr&>9`>Ajc>&~0~d?}01l5-qh-aSjB6izNveE4CHPf)8B<6mP|7w2DNr4a9>|~MK zN9m7nSt6)H#aVpt_z9&MPC8p_M~N87h<{u3EfPIB!YXcQYJu^NOdjmsd9DyhKUD!M zlcDe#Gc%H2c*rRVnt=!`<#03g-o>!GxrEuFy&8_3pU=M=7SI@P&F0V`Xms+=_fvN9 zUS$_=dUHlVFaQUVEShC9pMRZCmC;S@c>JDHLlgU$zG2JOdNp#DW~vD!%_R@3O+Q?5 za0VOMLu>HNsglwfrXA;Z>m!L2+1D6{!P0U+6BVPxZ?bFA{yc@5&sI?BMOan(X&d#e zwe82&-e~C#78pIA$oqVESw6uUUbdmZ_{ z9UF#7EaKTx=N)Dhe*$CUm)POnk=uL;=qFB6#8hqbIYt?<6gTmqW1iF?MM8KC$AKnO zPX~>}VUjgBNCiTU8_TZAqrvlh$RYl)Bp#$)hAYR17(Qat2?@BEX$ zS0H){aX?{`4g`fM%?Fq%v5&5JP5xExbX^!*QZ?o1gU6k9`~Q0Yhd86g zuh7xR$*7OxQYP6HQ6D+#?bLC*-_i?0%}8o=Uk#L&zH5lvj8Z9zjd$9vzE@c~fZj#i zEhqFt@yO-wi)k;lP7a-VR%CDQ1ootJKgz;(j@MXcs6MS#h7Zh$STT+<@2cew$OHu`>0e~7`MC??@0T|g+Z=uPiCw-e1|c39@_VQUk>J*TpG>8-!S#|ukX7|qf);Xw7+fg_({KMZf^51?SuQj zL(hSV5bM(2*D`^kPq>E~NArYI0-M!YiZX^8Ji-dXd z+5XN2hJ?~lC<4PQb$f&UKI06knyqZ}7;j98!k1um+huP9=iwrrCN2X5n`+gr?2o~N zb?zl}3_N}7*|q8~b!jYd@a3#6VG1yV_nVxdqea-{T2=GzbUy+IO}mU~NB{j6z;0tOuYc?*-L>tNzd-gNw8_Shoev0Q}z(mlS2=6f5D z;!6VPC-uV*6nB7S#Bhk^(V-(bhac^KL5$tH#p6T~M0lWHU<IX6GP3cMe=F8yf7c83%1drE9 zNy}2L)I0jm8V^s9Qn&F)!anhH08SPnLlER8MTVMN{$IB*1gtevi}thi!=}|x4KepG z^l?{87T~tX}nO^z|S_8Z68>`<#8+b^= zp${#~7jx(He#>`1fmsI7CbW2T;koRgPN~whr>M$J@fpC(f4*Fh7EJLS(OE?fU))I7 zNmx9d0R&V>T~&wnxKe&oz?Y|MTaGsLtDPzXy@SerpT#O{;mG?bBk?!+0W0LwIPLUb zQm@d92d7$&;S8|54jBi4ETZ zgfIKjn^VEb!84$&j53qE?aRiZY}I?3L2F%R_jT|B>Oqjr04 zutCHyOGle4WG3c- zf`Ah#u5L}zXCEWV<>-@VRc1Udv`e?D_sNXsv7^$AOo8y3$4n8Zj8T#W{Qd=#k){m6 z7_xJlScAe8C9u^CAv}1WtxSge3ad=|;$Q*3i0ZH*X{SOtV!$o4NPQo|fE$;r6bBz3 zRj?aN6yKy9Gn9j5kPMX$NASFGfkZ*RxoD7H#8=24^f%_ zuc8d&^;q|Dt0?n-VocoxIwg$vk9|zQ%VCL3#aIUvxRFw4x2KrC?5lHwa7t*|_S4x9UT0nRBb4C}v&`%z=@q_vg_ji6q-kh%F zS+?%la<4*RzPZ-x>^!eVn1oKkm9b{bzxuh~@3rf_N~S8IeV;6~!LdccE7I6xQoNRP zc?n6D>T70;@8}`BUq5DsCa{6;q0M4UC(qyQ7EeP1yyYb`KmDdNlQx=ntd`B(xWAxnb`?*pG?Mvzp24uph!zaXLA zJF1+;v;VcFPt{|2c^p&-iCnV=G{Oa+P%m{9t3cY0sb4&t-jn{Xw0H=B_Wqy$KiLA> ziqWK?^ef+XhxyxsDKh*w$dMEZVculz*+hoH5QcvNP1KnA=7%Ta8#{HzMH|M5oT3e& z-2gC`B*?Eec7^^jQfsQTKkb5thkqNUR8*8HxPd*3EPAS}B|D=EgRh5G9-!F29vVzW z8g4W`T&z1OHMw>S0OeIbjV{ntbm8Lq7=s6BQtBl8zsWNILZI_%W3&D#lxrSr4!gi=0Xm3#sBsQc#2i0r<7;ety*Tuy2O zqPCjpaTM$%j?@Mbf|1m}vpLEOdews=y@@0Sd|QPyR=S#>3SPp~slH~thI5m0Dy{JzMB2RTO9hpI69t3PBIHEdOFHKWL=7WL$tXxx4e zxl?^;F*f2pB5tv%c+&#p+bPEY@-K}SDWl;oB?lkbI%!Mk3@Ebw?PRp;$+LHW)^!aA z7I%UE{(*tR;z9o|tJ_K??cBX5kdqTo21E8&BkZ`na=77^YN5SUnHb$NJ6lZY)9^{E zZ~V#WEoEWNVdak%PoMM?OX!X=e><%X{u+E~5iY&?IKI}BQMA~iVpl7H+5Gwo{>d7e zBf!o)wu=$66By$){+ft&(4)X%a%VJqUBKHO8JXIhKQhL4>EjP}6C`AL1e!;G^e1JZ zge2qP;VfbJ^KejXl3$Is*9BQ{D8LU&e$Im4*Ga8elKx<(^C`y%F;IknFK09d?St1; zDNYUW!0w(^Q@wz|W!~BJ{^vd5uDd(BKdeYxue{$`25N`0ecsM5ya6>j+KZJe0fL|d z8gni9M3u4TR?4THvf)>?ZIe^#MHv)05W~ze4Dt|ds|E~k;4dG{S-+Fg_etj{)b9QV zxJBQuz>-t1uCBxjJ=)%}^$6DPId0?&J<1c#t# zS@g#TG%ywR;yId!Q+0K7!)g>*!xLdP0>Z@%&>(~^-}Q%`QVSWgI1j8BNri*r!D)Vh z=LO6CYpf%K_gsNJ*`1H=Xocpq7&Oo z1iUtrlMzaA7sOGP#Rq~4jgFuok+7|;;YDcEYtIX;2UZS{)n#+S7aRQTj|0Sgi$Sm^ z$D20H>#WbsAv_?5n>IV2u|SF>AQzV5Fh5oHT~0gC8uQwl!Pc3~pawNC#v3*N!s!6d znN(}lMQ5g-uQ1^FHj4Y5xU!wmo~s&;=69UPd@{iAQ@*{x%1=tb%%(NR_1@ zxoLW~q>Jo7)hiAQCx7jJXqiOpwL8swb(mZPS|{yz__Zu=d#viY{s=h@L$zp0m6jPk zwi7>s^l8o7g-;*9@&7m881RplfZS#Xv3xK`F?`P~b*v)sC1dc9_>$xRieKWP%;}N? z=ooI2cyakwm|ExV5`W>F6$Hdj;UENd!xeOA;ZhL=eIR_^LrVwZO=fj7bHP#I_Y?iH zZ#&zCA}^T_{`a0U1%#6w(m$dan=kMRB@nbC|KNMR$tVKvWN&ERuLZDlK2$6M`a?B= z(gZyoB_lqoSBC!yJiym_0Q#}CP`_1Mf>S0+i1#K^cdp)%cXly{8~N+Ide_6_gAIHF zJDQ4gc#6vNBm#U)3gtEVzpHUpDPfmB&3Z}mi?`MhD1)qiHRj%LEDq2yVca%p)ccQO zeb8}MkT<7-ZBbGoPLwJv`5lCn1xH-0Id;riPfk!y3(NUi89mXQ=8@d%0N$2!Ql3n#djoXS(@@qYp{3;w{3pp$Kiwy-GBk^uKF?ief?eDQ8s8D%5Vik3U+Rj^j^N3}t+5yfEWo`=C zVk7%DF($kJxKrw99!<8#An;yWUar`y&zJ?uX;6;Or*HT$PR)J3h>^67f5LpV1-4?5 z^chK#=4qWCj>C1rMnmmR{^-v*Ra1#>ZFnLI)SY0GTHiMiF9{(FrhhZ$DyyN!vA`O_o`1$o^7xTK4)wU6J!Sh5wj!68@ zqLkGM6KV=>M5v`)cF_ij++6F6r-e_%D0O}W$0k#LYV7#uWR()P8eoUqYtH>?4RlUF za&1|h0K|z0!W(&MfVOQ)P4|lpdt#Kb)5#> zbp)TK)wge3PfZow&d z`%=CVXVZ4?>LuiggD3=XSt%Nqot!a%Wj$AO3tivP&=5HPyVj6Dh>un&k8JoK5JZ8F zfQ8IW`w%ai!fe`=ko{GZh)%T+wmdMi7?+5)fP$kuyhk%HKCS!HZe1} zYweOq=;~4bmx#Y>T0vli_{qC|HnU|m%btWWQ=6ID-k|rF(J1r{WpGxDtyoWQzr>UA z#?<~-_qO{oo9tte#cpy7NlG8*zI>9@{)g2JrhMD-9!9O=^{GAmzZ;P-BZs=K*Ip8J z1d3N0+7#IVD>b)kYa?I~P3&ijg=HLuOgdTtE+PM`^IK+s+v>PKtIQ~xngho2enDA9 z6A1s7_PRjeOoH|o3QN&V?H&51O91*v!$gw-{9lU4IRIF03(&*t3eXUMUKR;kSzez_ zo8I4DbOha;^A(=fNo{4@qvhpdLE^udTb%8AIN;#OTYFP2o|D@Tvbv9%UccojH1C1+ z`xCYv7N!72G|^6vlgs1I=x4bj4jH9gqTzxTAcumL{?r8tHX=%5C{bces6ATAz$A;G z0?<}Zr&*(DlxcIAP4bbM(;kaa1z{t?Jkf{4AR2! z%!Gk+3+qd%nI9bT?GY&m5~#_AG-mT6d_GE$!OAR8ep+>-8#wHVNqM0mxzh50GA6{F$GktJZSh+`s2MFMX}dQ|P%2 zU*FlWB@H>6OTW5WE7t3tKX^So*_o3rHkk796We@PIe@4OJWE%9O8g%i%g)ddL*rmWi}gS#%mydyulCmAz2Qi zo= zg;`7~Jwfr;%Faer9u9f#`SW=obZ~bwy@3~Sb8F8h8C)2PI8U2C@&xeS=V%Oe2Vfw+ z$O3aK&aeKH$j)IMet|*Zh9b1BqcY~T<8Z*!xF1enhktpFqd!5@%|3%%a>vw<>Nqd_RRPs8BhB19~Tl_A9g0|hQ2?oD8qs~ z;S7fFjU_=dBAoe4{=C@1;q*eo^(f9?U0-~2vzF+(#JucO+|1326vrPXg}+T-)3BRg z6cXZsTe8Xti#pSeMQ3X>cI8VvQ6RB6r=Dw8hJ~_u>Wd0(bi{{`>HCmIU}>vztfGX0sPNEPoJb7JZ5L-{1z;dkA0vmuHkNlMqm7Ejfr)x-{ol_R!Q+4>?C zFd=ks5Jd4G;f_-Z!Rq;AJ>i)($4maxB3a*+652Ja8NoKvX**up<>}?>=G5Xo6eI?h z9R+oL4_SmXPlXfYaB_-A&LUKQ7Nzi~EV@6txjVS1yuWk~xgQJ&i`Y;nxWGxW5cd|Um`%nULDG24dE$}KV~RiThabE^D2B^9{ebzJGjd>T?9Pv5R)r$r3sGlwyv|_Qy-e}-nQ$CdbE`$-F3jA7_0-zzdKgYBN}JBp!$vxB3c z`FFXPbGzn_)_`)rd?X~%I7`|jk_Dt;>2iZJaCEXTZH>;c1@zSkiq^^D z;uBovAsJ!5Z&WW&s4t!f^`jVJ%0{|y^U-JK-u4EO79+w% zGdxz%3|+F)jj zh0u-MP*h3SMUtKM>vOnj6n?NSWhu-!uD7bNSDeEU{gu_wxl#LQZNqoh@CiGpy!qfY zr5Ma&u}kNz0p>l+cXlumsFnmi`Egq{I=spf3{X`JV{!EsdBX;RTrV$EX3yt_L^a zIG&zoQ}i1ijmKcZ|5PMWE^#K#+sbx@HBJKjK4{{kgDEOIyNsXCa{L6pdPxn2(tvtWNwC@A z(Fo5f@^iIg5?ueYyG+*Yh2=1YHQyQ6r1W#5H{e&k6$+BS2 zS2#+RMErlD$jNh4;6)3>&(9CeIG%BtZF`D=AfYWmcc%j3Q+X|R93brz#bQZRY+(uz z$3S1q8;#7md&?5hCGrx58I&r+U*(2o2cy`1PRn~HPi6*sTy7g{WQv?&Q8CMUBx0}! z0u)&gpBNghRz1nYh1O|VoLUMYyvz)xMw)^mCmK7Rqm`DB^2~DL?+y;U%x;PWa`I%s ze{YrlrZdo*9Io}z9_ri{p~A-(2Un$u0+-u@AqsXi#QN0V)9p6(;FC~ z;TAaM`{|=VEan%&h4Xwsc68h}GuDF1ZTRf~bs1KIgRndhj)-e0q zP3an!$|JU(>sxlkt8(^s=+4kQoa!q!^eCW*2jX z6ewYIaE#Gy2ltM-*2)X@!<^kP&(T44Z0PDUj`Gf259`QAJX z|40;W;EAiMmu%H+=0;FD_ zl{Pt4c6eyZ^>t1SEje>|*h@j?hXuMp5&kzw%v9jAkq|$wim&BfN&R(JO*5t$L01f$k(Q^sQuB4Z5ed?FqGlL8vj5pV^A_3)GSzxT^MY`EH+u1p)kzrVb=m_L~QY1+}= zuKkQz=2l-V!`_HgjjlE2mA(CdE*0M#C3l|Vnh>~ig~%BC^=T4JpAydu?_W$hfmGho zF-SMc(!i=l^P-9_SeJNOf|DBdo~DSJFBdsPW+v6=N_IEt?Got-ru1`BnA=%e<-|y8 zHMeAaW#iBHwyg2ra=08h6uw7V;yJ_VyuIPaPC<)b$QT&EVs;KutUdI-52L^4fT}-1 z-e-}0_xUR-wfToZNZ%X6u{kw(-{4T{RW4IUi}CV zV^{)Obnb3jtPElA`mebQ*I|@4UA?%u)w*9@ z?#TU_Hbe>m1PiAuOik`;>0-;z9ERsgRsA%c2`=0t=XB<3F=VI zEWX+G=k>0cs7|HPO%E<8IK-dJ`$B76l;8o3GQC{D%hw8LA#P`;2m1>`1a(OG^C#kk z?0+i@kArOkg)n%a&b)A{avO=)aBlmX~o5#rK#fP_

XbFLMEB4Rd)){Bl=Ng*thEauOW zb?qFN(r}%V>2DCTwV;K~+%do0hxl#*^n1-R($Y|!=}&$XcbxmpEFc_A`D6ONbyQ1@ zB&B%xIaMqA?}5naM|9S7ABsd`2!_;8u)0r2d%qkzU!BsmW{YH>&q)OyPB0KCRigweNR{Eb6>xxSZxc~-W zU0VGwo-y%%4bYxQvmy%jGC5-Wmw&iJ0+Ivj7HTT^GP6B}dPoNAtxAVaU3_HJxsVm; zq+J)WK0Jci5L9q_$Vmjz(UAmf=WdC)GKc8gwNjbJIWGU=chPqY^?8Nha^ER4Ac-DEAM1cpNn?_+WhXdaZd(w9zfQ)RpCsQ zrk>Wn6Vvyo*vR4slASNE&H{ji{_P)+=AX5h8~N->*|nCetY+?IonTnUzw2U_2cemn zSx*F01oqH;LxT5zw}uYf0zLp7YMQI6;uBp36~00sLqVZ~{{LU;xG?@~7|PDb>od`U zs{fy2hh#SYzBaD0z<(mfXCRc#k$8U?Nso&dL-$9wq3;a*n_~43kv2Buay8C@Ru4Gh zw87s;Mi#^jq%j}W57fFC_*2gcw%nFJM$?34g3dz-mBFYUg zA-)`i;higFI+TTp^UKly`S|z@$of@KS5~=KUC%Aq&jUgt8?LtK+Pp- z88^5pLooOL}7U48$Fh`u=<4HFhr04pdZVChH~pn^55jV%#?Gpkn9 z#ZPkmW**-Sa{0BRV1zS3uX?eDt6(4ri@)b2?J(c<8$z(Y5&;mBb72qI1;LyojSLW; z++M8TGpFr|H`S8wo;P4_cl=DLSAbzpvG)=8AU{d}8nx6^3T5?E5sNO~p&=8c8g@|h z?pt`Du===*>D0VbWPNA~HtM6l{%81DB^%6J=vCPhg`|MlIl?8B4Q=lt+vm;UeNX03 z^Ip~B6QWNmWAPT*AG^;m$t zA*<~CFKGV9(RoJ0_5EFZM(<`2y$%v1qW2bM^zsuWQKB=^JEQk9M2`eff(SvB=)Hs} z(WCd?89|h&&;37dEH5l;x#paE&i?MbKQj)~O4(tNUH;?yHoGw0EI=9^cC7RMj_e%b zW<(IzXj+s4Rg0O*XFW?RAD-ZQYY(Fl({5b9r;Xt3_p;9#-|0yqkw@ORj+En^O;f#NDvKIE(A5FlDfyJ7a4!T^Hp6 zqZFz^{b5iToEP<(D8kbkgG)l3RAGe3S~QPF&W_0GwL@3vuWtUxe&5@4&tDQr8_6}S z%Cc|26dnLG2OxDLhu9GBUDR+Hr^$TxCJzV0_qYKwg<$}#(L&I_x?==MN^Gn!r(rNIr`63IM|^VAuD;o}7}veHe=A(! zhu_m!W=~eAcp+xJaSL5l*0~)37ouInJ+Rvh-E%38iU?viyI|g77KPvAc}@brF5vUck{iTyv0$+5WL9WxEF;$V+Sm| z5>%_Spm_>%b8F5qiM2I-I85V^4Cq%OCX0z!5jSBs6wVG>)=}@%Bo0lky~9d!m=gHb2Ivo(?_RtKo)waqs+LM(zGh-OBcp&XW6JHvC_L3 zvNIq2O>qF3NPBAYtjE%VW_{sN%*rXqa+_=Lz?gJ}SqDJHq{QNO>Ce;|n>rsU$Q&Lb zr-w>J&~rW(&|2*vrHJ5d!Y7aYGEU@S3djQAB$IA@S{+5gN3F!4T-K&86hX@tEA6II z=F^dMMmmo&1a<<&h`QDg4gW!qUOk`g%bXaypQ|^=A>{#{o8|AfTWvG)cfA=X-?Czsg0lwH6eze8ww%@fZ=S{gR-_9a({P?-odeIo=) z43>mGP8F1P9f*vURiQgw6W_brUr-&do;)}Q)V3e`o<<*Z9O5N;oa7)Dv_+O5#or8+ zFeV;95B7c7i_sV4T;tegRqa`yR5`zBInMI%U^Nby3?6D#YBfH0LQn5yN|h0af6Of4U?~?kJUR^rFq`G?SoM5f#NvTp3HLV>sT-YU4*d9W}I?k_(Huu%NOS~xq9oIT44rdXkgXQP)>{F~Bjd*(_1wW((_WE!nJ(Pb zn^j)gzD&W=PVtcZ+9Vcb+hrlL4SSb$B|R8V12CGZ^G9 zOyz*YC-Iw1KH_GVhxuFz?ELwPAodmq!sj+F#I)i^exEYsJuTF>Uv`CQa}uOwB?-fCyW~* zL=}N9Nl~*I1Cd(5Dh$*=+!jB-TXVg?s9ZP0$rpF%T{yb94U7{-+Faf&8GYcS$ZSM? z&w-qXC#E!A;f1SAk`=9zy~c=r`~w!+b8nO)j``ajpr6^MBjox54`#1gah&lgp3JNJ zb3a?v&TCU{E-FokM663H8V?;hga+)i8f$6(;TsNe@_*bENl6t!tWf1`1P3}RF%LW6 z>5XovMu1vAXmc6T{9V$ydh(_{0n~6j)h66^e&mEDjO0BGtH_IA|jt6~yh;f*rqMC&2h)1R@nUoOIYm4}TT%LsnD!G$Uu6UekwEqDj-fE;=E*^~M0=nI~fX#hgp@e#1i zb;pnc;vsP9JpP|~ZzJS3iJz}4D#?1ZZKb5!tCmgt86?X#cW2k&u}`H#f!?ptM=2efL@XvGSUg7U5Jg)#v7>im9si-CzOZlS#B{ zU5VgC0ja_iDR`p?8?9hP&dIQa0}6eY%n(4^mep)O^BSe!mM=QJ!)oF5{T=%Kr|aIZ{!sInizoJd~e zKJR}ljif9}D&GrQ77Io)ES&3@IAM5Gnr`4|WI<^iuPSHa3k&`UThk#;T`TRn=`*0WVYuNyw~X?G zw|&FG`Bq@&PSadMrUo3=`WJh7dvMFna<)sMO9k@ot8H#nvcTbykJUW5x|}X3^XEA%$^E zv3ce`YxTW@nbC)vhx>=?4VDMrHnU+Ki8um*_rjaP2zEfQacna?zO-#ya;ckcF2(*L zJsIGsnA%g4v2k!DY04|CU2N~*LRDzC()xwKx}>{7qAc@hS5#)`olwo4SylA2^W~c3 z@@!a@c!8W_LJgQ98Z^!>vD1>#9sdTy*b)CR_f3Ts9-g z2*JrX<6ko^-^LT)w)5UFAUU3>7zqETN@js4$8?EiQ^IGMjRyoptfi zIJ=+UosegJ9k)4y}URHIzCbRsMtPA(ViFqA3Z0-)#@g_M@+hq>tfPWIUZJqc!VOog$dDL1QRBR$CeYO@soN$Hg23g+yt zo%gbytbF;YH}}8?{a!5rp@q%v(AZ9ycIV+={8%9IpYUv5_?P%@FKM9Hz`tqH?VTgp zY>r0X7$I&-Nexg~g~tAYO1SWF~4;lX=*)MoQ}$z5N)9APR2`)HThdR zMJrFp(QADX7r;)8E%(X!*qI(A*1P=1>H6aL5S_vV4#5zZOVX9`4*b@Iv=p13D;%(t zN=7=TUM_RKP;sJ#HbErH2d#`}3>-{nyn%Y9vyL0oQE|up+RmrvTE!)4*>S?A$CoY3 zwbQ^O-#OdbI!j;J`juT|-)wf=6z|8_+wJ8WjPf_rgJbSbu6WU+VJl#{U)jGiC|sjF z7esDYyrV42F1;z=&3Q)lIoMZzx|oFXzq{+Z4O2-%qfhJ_-Sa};2tZFvLPqmZl~N&p zhSS)02?*kIA&YfbUa#`(UU^U`$Xl^j%a#uG5XS1k6(z{JL2&l0GO!_mUxbQE1UOTb zFg7;B#}^0N0ekz8H^KrxJTdKOnMgyP#JaZbzoY1X7{2f6zfF*c#9jDh)1M;xzB;+IWYH&l`l;X{%v#o;zz-g|u!2uI>9nTCkshKvG%iFM2;D=f4{eDkI1oER*{8TZ` z^+xi}kA_cJ2;J_+sm*f?&&>@)~GcCAlXD_ioPRHgJX1(b*mOGp4*z{X`)H@9Pa zFj)3q-^oGc8-+|vgFB-hbgCW@NA)TT#qZ@-@P2cl zU(@+*Q%0TPLD!V+wsBcRNz@P{t_$s}yoK>FuLorreCWhW&}pwu;w;ltXK>5iQFVFG zJocQ4di6=WB_Cc>y7wZRn~o69tiF{RB)K#EC3=>Nu<45y>u@K$IlNzvb$xLxA=3&6 zX~+aoe4?aa%&eL7`LT6TguA_73ri&o$&^siBDSp76$+7C1bz5qG*8KOuzcMo@N3AI=-i(PQm z@n!wCAlbzCc=or#auf9{hvS-5*2)t~8#+W)kjQKFrUQDPR0e zMzMp$--SsW|0s8%qN}bqQ>O6xLBp0Q!tVLCoDIazX=O#dFS)v}tWD&yO06$Kji z&9=L?&HVT*AL6)lOg| znO!-Yig+9&UAUTu!ryJJSGH+>X6=D~Mjvct>+R{22S`8?Y7)$w{LGoZGq9^ZxHD!& zVDrB%9XV5ggMZhVvFCo!1tol7RzAKAfCR^)qZk`s>KXjWiTi`pzypm}C$+6*O_j6& zg3f~LcSSVUUl=nBTf34j4oBKZ<1Ws~YP@GHhgBvB(x`EfyoJjybm*x=)grT zJN(@+PyY1ytYwwTN|El}M|uiEHte4XN>;3R$`n{A4-v;ltW*>knO4bNrdrM$tlkSm zC?WJ4wDMCdom*lo1`3Eu#3MzQQZd-l%{P8J!;E@;krUZ*orCJ>xlo6xg`?n>(bb-8 z@vYrcwSsIkjN?mlUM0GT^-`s@!oB2Zg=K1Zs^6ZV)GLy-VFvvhQ(`=Ze<>?Erdz&2M+DDW=gNG z|6!5XCwJTX`@dRfDYm3d-km7^+|5p#qaLNiP4@ACRArE`D|>9R|; zM&)i$DUpixCbMs+ z;$U+TVY;+FDUNaP@+9iNRFF*$sMaO=zh=#UHal`YQMKHpVht*h$LTP)i>mu5gNnF zb8JhnMHjYA+nS{>c;BCH4FXWe8NPJmVV>A;yG<`EpQ$FqgVa0*Hhi>isw02I#zJ0P ztUgw$c}58GJpjCCI_qh={+F1{PBbfVI$LoM4*}>hAgE^%ma*QQd`Xt%mcD8Ss&D|g zv$UM+rO_bEr(JR6s5f@kftP`JZMb1UDF9XjgxwXv<&UNJ*B<3{xuZJ}Qk2HL@%{QuvEBALExGY7*GOx0JS%e1UL}b6Oa+%nkN`X%x&*fGjnAtWcivAM`VdSm zs*0<<2&euXU%oW|+snB*`swO&-=`v%BlTPAl3}NI6fdjmzG{2Yg6!hCjPnU*^VyB- zPv*C=n=V|CT4zLXK{yImy9U>1PMPL>zD-9Ql}olWUshxlGQO2>tfSVKd(0%ZxpDg5Z|-jHDU(C2--0 zem1MjuIQg7``S*VRkr4?!^VX?#S`|gCVujIjl^_^HI#g<0$+mXZl1zqOYm9{T_#};lhyOGfE^8f zY-fLqxaQ~S5<27?WEL(ks|6vvUr>H@UuuF8gCe#Lj?I7aib*TWrvg20g!t&9NJ=Wi(HRzPufxhMa$#zkx&D8L|5o&wzsC zyM@s9z9KnmyLS|k&U{TP*G-Iv!x&o}1WwLV1;}q)IZ)6(e*VoZg#au2@6thrPz*rJ z6?ighetmz5={ZthzW<%-2>Jn`rL4;7ii`l<~u`V2kji(3F4&pS_2A z%A0i2|03s?l?-3)Z(dW6{d$>Oxk?engOi9r`-M)}X96;MrGeHBLzhePY@mx*g`9>n zoCa3in)3F8t)B$I7d_jOUfIUYN4&4;XeBstq2+r<)vZ{BGp1eK%}+8R42cO}r9UX< z6~Q1=D8l%pL-^8(!`PY$EXml}9~an5;@(n#g+D$mrO)limtYQ|@#A%TwTruO?4&`_Cx}@B7R;7(PF2eQkS)UpyrO;q;R^v4*a8n)*Egj}ruH zsFN}>s6745`I^5DqU6(|ao( zHg*xJ4H<~kwL2(g*@eXHin;BgCHf9!D+ua}qsekE9x;lV|0;8g4^@DTa7Cr^e->aD zIKvyM!UB74m$sWxD^aNANpmnxwoU&+td@+ppqP?Q^%JetKDmB!-FZveJ#q5)toYO7 zoqwBhP=ot9y$FM@I@bF6-Cj&%2qH4EIjuls0w=&t2Z4xxW!t6DMf;5tClpuk zZ#xSV6zza-MJ>EOoyz?U%vBJvsXKTVtP;icM#ZIkZk_9KrW7e7UB=hXFZ(q<1>r;@ z>$tzwHBC3aHbPUf1B+m++w4tmSaKvKD}D_Dn6jlbRng+3@ukAMvCx|<8YkIsGE6GM zabo@(9E?E_v09Rtxjc>c z#CBb#1R*IVxTED|V@}Y=-~2WlWzAzsJAQ51JGLj@w&m||cQGz?+nPV<>>G1imKFj^t@D~qAM99eAfW6`C-+bnwTc!*@N)piEVQKbJR z2Pa2xU}+t@Q)^FbR!vVc0>1-JdM0hoOXC`cj1{hV*1TNl={U@^%tSt>Lak;{=5OgI z%NTPxXpGB|z@k0+Jk64S`(r9St2K;60Z&;_7eg46 zX!6S0A;@)x?Hk6P%zCBJe)n!ct$eWw*4|HIT7)11Gk&2MT}Kj~jp)WM?C1@KQir5U z>}BK4_WVqybWQ|56zrj+_~c>);SVDvOdWAa;*>s;T6$9w`Z$upw54RbJgD1an1*FY|b0B$$A}03>nqL89&UG@ zn^&SxPvZPI|7|>f5PB;el(7u#i6=2Ey_d%m!tUyWrSuzhL=~Y9&=wSxJ0G;&g!aZ` zzWs_Nr}Q{olN6f5I2Ok8A=uON5St_{*@$cFf`$wYu^tfM!*-DWMixomuMi<_-27to zXQh}JzllkL$8~w~aa;&5Lu^&0H$Dh&HqGg?P<@Dv3Hii2qKkPL%F-sFc!iSp-|lh8oGRB?Ln43}nQ$dw8!$)|kUat-S0&L?^MHco)qM&E|O zJRjH=SZk*E+*YXDi6bB4N9dJajA8o_fki1m@%^vZp7?2F<*`{p=Y9T9Nj+40b{D(- z>Ml8@+T?s_*bC*svYOXM*N<0dcI)+h{0Elf;-`L?TvA|3N0!Sburt6(g!RkY3Clwx z9G-Zy+Wkofp~3O3dD8v@zsG3Ld9g(O)VKf<#`uQObMgp~kUM6Hu7Dq{8_iu%)A z$`_8)4tC1K;!{XaXQcMGI84#-P`tgt5(1LbB3BmaDjBsdk*Bs_Em6_#c@@Wjay722 zpEibRcL6<^9D=Tcr3sS82!y+;7t`05-Eo#2Rrsc_j0#miovoMYyDAfD+k z+r^V`(4_d$zWeOjKapt3IQlF83DXVP>rbF>Z?xE+GK=84lnx{kOeG|96mS}m4b6+TWbAvt*kkA))&|zi{l|JwuOS6Q+bqA2Aac0 z9)=J#oY|M*2#+av1cDW%W@LI=LT&6pj3yryDrHD0GcpjWHt1v~^&S9peaeJ-o}(?# zZD(@EZ`D=&-*dx``R4j($`n)lVm3 z0oIQOHStdIBjF3B9;k#jBp*Jzly0#_HHV1Te%jdWcz-pAeR&}i6z1aYcgYzW*9Gba zywfV|AVp$)x=>9^^2M&bMAaP6Sl!=Ljcy}h&~$Gf{|yTZDi{#WMLCwIFhrC|*?FQk zM^iYJi1|6Gwb8FU6Na;)EkboH|hiwPh$R8-dZ%Z>kIlGqw~bp$}rP=9S}}GuzxZ zLg?;oUukM}QYWK*z`gB1A==)eeJq<+d?EY3kp|t56&XH#$>8LmeXqlQAG~`1s*QvD zQ#(0$sf#hYnMkY&|EwF_KMv_975zyLVkJ*PK$P8xe3-y$I21Aa>KJ6u7~>(SWG89Q zcUJZYDOzmeWE}!+g@kl9obrhmsI}w7r{KuqkFF1Y?WmcDsFiW#6glLU)!)Ei9FfDW zFNb|j8O32lp|I)}HeyLZHX&R#84|^|@-j987^9wf2)drrrk|ELvY=XDGfq z8GO2Y3r(4rSaXLTdf!#^d>^I^9&Lx8yyRB@sY_=6STB<%{!Ru-DI)XI$O)-e|9u%jvCNrg#dIEYYcHFM#u-j592^BvMEk z8w-Yhbo`}WF)PA6ZaO{?PO?e1jT~kpX$9YPLFnA2dfggQu6;V%wkDTLFf06O78^$Y z6idEW{tkNf(^@2Gu+N}P1iLuxR9K7cY6}B+aTwW06GPjhTtT3kqPR&FwT2`fv#bJ% zmi)|T8lION9J*j`s05aH%jeRV_Pmy|dOMQs?3nJqw?KeOmUIKXTL;uj%+#?7sW@Ae z*I+kG#e>zf;R1;pyAGR67A3ekd!71IkrTH?freS?Z#mnuUK6K%45pgnMC`CUG(IT zT)D^;AS*FKy*xKPaj3J?fnfqbQn|S^b0nQjsdwDDILhDI+R|8E>gHG0kP%gj*Nb!G zYJTx*33qJ_<@3d&CcG4Ew;Kp^d$X8Pavl8~o6U06&i_G(GtT{eOj*mPR<#_<%kFO3 zV=|2Epri{HofAZzd}VAy*7;1HEwg}ZVUpx+SxH10Q30i>x4~90nV;*uViYnC%fiBC z*@8IyMCe5`!mW2WGI)k;+015$$ou;qn%I;_JecRh?}ozJJE;>p-_o0sT^53 zJATw766y9IA>SRf)iMh9*wvmxLaXQmfhGOG?tv3-AtU%$&uO?WrG>(u>MMXLRVUb^_YCE>QZX%J{c($3;>cRRjp3nM{F= z&0olYh$NC26ee9zX*R0nKiX)4Hns>h+LfONUw$z&DMOn@r!}~9Zl`gi>o(#uE1K&c zZ2h+7iwFUFOM1JpAe|fa*q!{^%TG>^Uw|aIU`1)YCxD~36PS%YoFs*`zvcx>=c8)t zk?~fPoB7Y4p@F^6<-Mwo0x^W`bP}J2>`FA^-_iLBbI5}UuaQ*^u4dB|uslq()wl6@ z+pKYLmik!tjT0b@0K%s!3gg0DA}ns{DKoR5~` z566y=iN&RSN5I)7%mH%?WeK@`Ar{E5nFTYKWlkC=CVI^vwz4~TuHl=c^ad_&SYC%Z zUWTTWeywNXJ|8K!S#0)vv5^I4bB=g|L;i-phMyQ3BdyVM8QQ@j`*gn3k%Xc183Sn~ zPJQFz^ZmZ3Fl}jqbrT!0w}CYp>@9kF-ZMOzi&LF5{NHw20`im-qc^D((h@<++-(d? zQlHQYNCZi|tI^s0^(D<8Q}Nz%r8)QS36!x_&#|O~Pp&e%MRWBWzv!YZXr*_YkZ01w zS*qN&IIL66Wjsfk^X(tdhS!_dI_i$k3SV6Bu*hRmiD~(R2u6uuSp3@-RUacHgz=Aa zXd|<2X+6{hbjt=kI)*R3O&urrmGd4~m8_tncvsDW@d_i#MbxM~*oS&(bx1hWTVC3- zx^^Mwc*rDu^~;nGdkO#NTsz*;0MM1?1|+{qBI=XB;vyGhElcYdFrCN;s^R#Ymv&jp zsL08X`|jsa?3|z;;nOv@uPg=#z_!B>lwdn$LJN<8NvnDeyzyO;AR9nGQybf9=AfP&` zNc`0JQ^y9~i+eu37{;C!Q!B>Ls>t=tfkcu5s^Tg5`gEFBieLBb{){3bc^Q3H|IU#4 z>uyy(XHo{?zi52<4(xK#*`9a(thubloOklCQ^4IA@OKjUmLQ2l0=qkfScPt#Rt(!4 zy1vwR5HKZ&0_^(wR6?9X-cbn$AGh;=e`Mmu)$Q%A=4k3;D`f=_|I2$g8Qxj_bXBF8 z+zs%v7n)S>7weW!L`U0#y#$lsuapbiUKWuVoFrzFbX&5-EOcca8&acPg9Q+NH;jSg zCV!l{0$UHYc7j5L7w_90N3qna@(tcMHcBO9SGwRaxlC_=sDHEA<_cQ;2KM1(1ULdXj`UAQfM@8H(@uOX+3S{BCWGY|HHMX+ePpml9Q?Z(m$aNR4)_U&Llu+L{{l1KkPO1=Th8`)|XBm3Dx{g}X%>PZS zxzVN`))V5v{j>V1jx`J(-QL~otY+b@@keXVyu2%tucq$}l>M~x&t;mwlwHo5S^nW@ z@l?y)EkGF&uY8$UdTd*k83Yu+T}qP-0SB8S(H1`Qj=;8+orRl`BLP0I(U#Yx^zoMkoIWYf1IeI24$O@cXE%Xg=u4J#Cu)HS{AOZAWFkdxULC@eiuZvoFya zZEc=+Be~3eit-I_X~u@NG~_X0w)Ah`c@I!5&es?^Dsikkna$Kept5 zF@9u`;7JVylY(Qx$%n(T^w-~`4-*;u8zd;aV!7lH&dotT>>q5}rQX?Pg%L^V%=bTn zf3g3#xTI@5Q_uS-Pl~j6Z_1eXlTZz{N}NO1j-Qd2(KK*+@wP15IMGpuuDdg8E>t(0 za3vnptI~tmnjpOvtk~&cpfYSDba6m(H$qoZ^T`}f5GkvZDpP))1RiR`?B`oexPXS$ z<4)tUx~ct-1n+dz-~xi;5gG;AhO`9bn;B3&FKZ;lrgIwV-N1L%L5?Q_aPr!UG7hUQ zORh6%zQMNopFawrU8Fg$MR>xDwM|>N3q+kDdac`reIUr`3c6o5CVQ>d{k2QO;7n!Z z;A_5gU|91n_*h}>pn1AE+vc_LWn2x?EaJq5ga;O@G&~t@Ht=Gm*Tnxd?G^uZhih2h z^N^b(6kNjjba6o*lXPQe?wus%zJOSniMpBbctpMiY|bDb2)++Ljy$WJJCz(3$v%q0 z2ce(~;sCLG9`hz)#vW~0^zpBX!5Lh-W<*QixGJDCR%W!s-)8FWgg(W2DK{+~j__^e zahT^FO=8gsb9~r)6LBLpJ2xUGRq*C>hJv|~aLz4` zp={!Ka4Qp8ix_EC{7CPAoz8Z&P)AetVgiPmyH6KiT4~(f`N|~OS97i20mRuY!Cvu& z;A6iTyJf0#AA{PMxW|{%m6ZvXRfO-&t_LUBW?I1o=7!F~`JI z8pCLzBl_Cl{UR+ykHn@OZup`hNDL%wx3EEHF=Kwjql1`@Q@R>9o3HG%**_7UXDwp- zw`YF5=x6Xo8KtSF=TNh;!36H4Jz`H&--qkV>eo3oRFuWAzv{HJRh?A#Vu9HkG5805r4=*2u=a zA-IBbQDlNNCN2NZ2QHi)nQCCw%+gfsgP5cRga(MAu{I!}B+I|4t=()IumPQJ z1)F@Z1y{*Ej32@dU$C#+;+Uj*vA zS@R;VM0XdYd<(!G(RxYY`^Mn8@-z|`v}SlyTDK)5nchf~COq(nhl+quBL0Uhh#yv{ zb;XsJlnwvW=YH6)Hd)iR;qJQUivOh?x^0OE0-7fu(eGB)Nq_8lw56sfxKVd(> zCbc?DDXgD+>5L>%jpvYYLs8gx70tE4lbrbmEEsU2NYe@-;)er6Vz|fc*A2cmT4|`2 zpQG(7&#EOtP9LK)JYP=T{C1k?B>t~X&VxY|i3h<4T6&s!&ZI{h$^dQ>Ko)OVBH;Pb zEljG&3uKes->Ps;zGv%{hZ5R9fWCWfe(nhi*3(u00>t3;8A>rO`^=KBL%$A(Rz1x) zdpzL$l{N?urGwPeJX12Gecl=}Vif(|<}sA6@AU6qVJ5uD^fZc{o*~t2-Cwpt_J+os zHXJrK5TfN`1|p08uI2#Y4+=XM!5w^qfo*0xTlE?g%>i~DAjD*18hfm2`HR}~)W5_3 z6)##(E;Ahi-X+l%M>qh+d@s#AXtkXqm9f)Gu*ev8?2N1ET3R;kSK?VW-zs6K`CVK| z>p#4aq_=%6701&Ls4(xO`3H}yduP^rzWDQi%*GCGwf4Gs>^>hk{KYA3_F>Z7`NhtX z(Q)wt>(H9%tvWD?Ky)p84rk9ZyzT{t2qx=8DcJ(&uWvvC+@UTWlw=kJ!%T~c@BWsv zQJ+VhkC6E&6Q%}iKHFTW<(0A`ve_hCB^p+k17)&O!UTZJ2wBX@m6s%Ql3)K?ql46S z)e73!6g=aXJ-#}x5T>ZskE~V%Gd%Sht!rQ)y7uj4cY%ApQpevLD8k#-l+%2{GWqOz zI5XRDrxYBYm5R{EL01zbk-`e1l!}lF7jv_DT*xz!^3u~*4)j)+u`Uv52%Yn4_hoyMP4r*+ z!2}y1R=~OUpt-eu1i0($ocZOta&jV8C=CLGKxznieHvy&_8s$US^P%|)Ce3enUKKe zWPTsj>3%1luJPCZGy6nVq*Ju4ipn$69l3ms=ZR2*{YVYFUJWg9eXUCniv_a4GvN3k z*f0Lx_`kYo@0`y``=wH&kFqu8vs&Q!zR8*mol6FKc7%r5P(5{DIR)y0VBA0sL#M{G9;FYv;c#LW}`g>o}@2DHpSvz=U?h} zYe~LsmYZyuaH#Po9VL_)4!J@dws;lTLKF3_MS#@{4pZrY$Pj_wsn)javqj7E3gall z#H=}3we3{}36dsQBb)m96DKTzuz3^7IUCE zCE{%SE$jm9C57DfRNe|ZW8K(fL(otNEn&t_HK+hOt2AyCq>#2Kiu#5P2NX8^vm;ur3H7rpW8(ZfmPKlbjRiYF|8mxR7jn5o}d}I(HASw{DRuA$inRamgyA)^SZ8U#W-+04M(!7WH zNC%2bOLNOeC9tO9q5;Vfw5tzG+@O*574QEsNkg*^bTO z`Z2*iY`G%ekx-Si5}b~DNVk5Rl=Q&&uxVX-e^nRg#KDmyw=_ZK6_TpEQSmYlP5(~1en zIUN^F5KK%_?oFNIJ|qEDkIfcYn$Fm724RpfBIsRT>0e0t>Sq(2FL!#eBbE@qmX=-gu0>yoYy5W;2cR^Igea^(U!|jSjddNN?VkqiV^=TD2?pD9MOv&L` z>PIlimF@!MOSZ_is5Qee=;tHUWyIw?$tBH0dUbB!(v_jZVa=60a}3%A{<-S;qKk)X zX>-Lh?sDfTUH4=(9fqTbEIA5i-3^S>udFK)q9;W-ya(Y2 z>lMRdQe@`b>xYT?zDhs9d*9}bP(#^eQ^r&Dkt**&jRkXC=ii*=EAz_s2%MeI0;w~L z*8DR7KU>jWe`+$JME=3~Z$pnkk#qpnsTP>H_>^3+n-ndEfUJ3w%0xT@>8|TG6Z>UH zkvL(E*Q4HTJApR2)z69=xWJf@$2$ba3FXZF1C6A<ovjl1!jbTvXX41Vwse zV=}w`;&3P$TaH=QmloiF56?L^RShILfAuScRkt|fwb;SEM+nl3C963Q1!X*dz{>c9IBGnYkNSa|i6 z&wCe!#)DJbcDfqtruRincsSEji|y&7od|+szEmV8>ncPe=SAAy>7)W7=4lj8UE1#x z@z}AzzMQk#%B4`mpD;6uGQGm|2dUpo$6Tq-^IrUD!@?SKQ{~68ELE}@G+S={vR9({ z?LoPvQjyj8nDRTwIid_m?06|J`LDbi4zg^z!Td+d=M>`TLkn_xSN!8@Q!3Ju8lyM6 zwko${Eh7%~)Ta9e&w#@s@Jksep2{)gZ7}q)`kZ3-Z6tWy0H2U6>Wi&=-=mKEMnC2W zMqdl+rcne7NEPwz_?$HKG# zyR(p9Y_^xFlM%h8<|k|+Jqi#Jz9mXd2r)T5S9>DZ_^K?HHX|hycm^by!wjJU;0$}` zw-o@fxtizl=G;g2Q{T*lFq0my>gnHm?2}(jMYbu~Q9|lHm`R2v#M)+OJc0xiz^H1LJ}HemuIZE z-EH|RDpPAU+9iH7dB$6Wxc5i30k4C`kpv;FxVep=Alm4+xgo+w+ zyF7}hM1gNXJf%R?3x8&J6$*W^(Nq|}?QE83Bp+Q`&x2FHeC$9hU(>wIS6TNA(RogR zw2reI*6%@^-9wEZwO8lGn3c=yY zK}sntFHZxDOJj{y?ZjP%u&$}#qgtULKgCxuc=gdviAMpa`}kBC7xrM!ZoeOr^m~4^AJ&m&l11g+bUq?DC18rTTg)?|UcSJ0L}!fKdTX z`G;Dg!B={*)gvXl?B?B_{OSqoqEnuxPpdc&1mZZ|Yc-KA+^v}3<3;9~|2dr-%ypG` zd#kGcM@4t&(?$g)vV(G#r*f+PuEYkLWjVr0{dO{_Nz_Dt1kTWE%?Fw93MXW@FCLP5 z@LtX6NiY@8L09_F1~VV*vt>wd4|{?8`xCOI&iG00XnK{u*--$hU@x_9=6Qi^H}Gxv4E^r`m#=fasQvdG@{iTBah`N?|o z6XdEs=4^%fU+f711VYSay|^^epYu6lB^nkss2V@WD?uT!DZCU(vy<80aa|^wcA6dL z8}oA8&>fP~Gch5@4=zknMXG+`>udLb|n3;K9TbT~)|AESjyo8TEENh*4er%y~S#vSZ)j?f-Ab0ZHE~Q*{&T`eskLOY$`v$;6Rvo}3dUTmRGk3lCh%v@c?Nf5e7PFOttIGg?o&`>E zbxbTEY>YB&_cV^**23aL(Q)<+Lo)~MjIm^(Oi{v8q+CRJftbMeV@q)w6Z#-{Ve zPF@LNz`f?WTCCq3a{^JxrxxDHFG`x4V$nX8mELQlyBwwwBkwhK_N|@kELp-HBxhUY zladA=GWTKX@nn7GxqB!EDb!o+PBFITWX1x(qV02mU%~twcZz&IRR(d_c?hldn z-3FHie+rHglNJAy7?1n2Wv17gWUVJ*WxJSSY_UhFPIOJs-5d9CLX2YQ37a=*K01EF zL`lT=*q9Om;a4f_e(IPEv9{TrImp2uZj_cONUXBVjhUJ8lGb`wZ;pfST>Du-$}y8V zi@zd0naE~0)s}a(7CCRA-#UcW0REsUCyfGPdIn4-z@<7$p=6(sQ6P25Kq^AR9mA=J zhbE1}bWr)oP;y(iDa+%{bsRe%Mr1^R_ANg<#jI~b(`e9Yra=YHwdb-+x=X@qQ(4#N zH~-GZcWykogXX-6DxW}OV~g7YMcz`#a!FCKQ*41j%Z;t&t>@EiZ0y;>R%*Y40*qz( zwY8nba7TOn<>_C4h@~z`ZiRoZ4g*kr5y_>xc;@{H)R88jx!M-asN_dT0W=K>b+5G< z;#%_N_+{9>{Qhjab}+K<(J!1pg2=qT|81o-7R|I(XhGK_nrMSkZH=~`>5Hmb?z*F^ z=Z#Tj^{eR;K<41@3<)LlQ<;29VmLxlx2KeblC8PYc)`0C!!Q!2q9ze(>10sb;s}@L z)wn!a@nlvjj5&C-di(2cpy~oK1A|pTzA_JV=r0{=LTl2|csX2l(4qOopstlYr`z*7 zWGxm)Q$ayI2h2vim##bZQs*Z`z8~#oL|e)VkxXs!b4G7g4yWtRfA~*Vt)GZ4+_EvI z7y_RisYUK;O1)kgMb*~|a)_Y9JvjwjiFr;EPri6N{{$$rlXf;*#k50T6^|k;Nm9pI zQ?o(n68--8cy#%(<_IOIW0er+yDR(fnUEGF$LUu=R`O%-MBY>qiX-X)_mG2iH@!f}9 z`daU3`yatlZ*q;^V`hgHcLJx?Whs&pVG&FPVFlK5oS7}L};PuJ)_!4@O%+(v?_a&K>M z%j@!>{w^Rt5ix^0bK91-G*f+;-q|EbG?LdJyu7Gm59~7)?-^Ty`d0!&7r!&*nvg52 z@svm9D)CnqE6OJND!Z?Q()PMNGXHmrM9`JOrkyWW@dP&UXW%?64WjRL!RVtC%hM)) z^-%$K^79~B0P_fZ0zRf-{1`rZ5aZsK6fXwY*S1-bwzLh~2z6jzpq67rZJA?q)f{l= zssY4cr51TFr>ZUk#Y~c1=)XVMqycvg+zjiSL~Fjdg+MPDOdSkEG(^}t;eUw|$njJ> z+ucmws7md=yIL^MIuB`Yt%%!B3rxk4k5bi<)=HEd&|NZi?G_aII6Rq(_c$5y5bwcX z-B0>xd_Cf`jlsj>!|F=M4C$d)umAaX$dD7h{$ckqpabz%L`2?KTcgqS{Uf_1<{!7` z42Z5AXt54w&Fa$a?sT;fmq83{B*u9m@)hjjWZ%>&K`2H?aDHO)(j21*`>Y8i%La{r zNSDI`WH)C%LJNiCbsl84AnDmTQk%b};rS zHpqOT1K$=b2+ghB@RD4AQL3xv;mdAul8zjib#&em#{ogh2l*-w5sY2xRv;MGzNd}8 zn0=;C%wia=qa@A*!_!0^#CIrEZ9a8;l&(oJO23`nflHA38EAW>Rx)iJLjTESE!}#^ znwtTavoq-G9LQst?=B&cnd)N|>1<$5yBYnw&lPs~JnWNvJN>G&-+Rr?CKoV37QXJ{ z90)NAJgiNSc)-1DMi1FC9K#+6Y7X50?#VAIDw4aoJXpaZ&IETXlcmek7iV}VSzmoP z=xdV#so==`$41gddCg8{U@vcW!l?*3io%OhSy~6Z?_qxX{>w@^POBAuYk~GP^Uu)0 zdj{wP&Gc8_$iTAroPkD9o-dd4y`*&)O|x_*q`f8c0Vn{nFY!MZW$fUef`nJk}bKn&zGN5+YpeY8+qm>*tI`oD;AActzB8 z)rTGGpx&j2>KV3MqjREMt-g4hEj952Re3F)1hY>@MO_0bJwc}P!S-qM`WVAmPM)&X z9KU6q_k-(&5nvhamHQ9=V(eoK14In){`V?E|M2Bk_vZQ;pY*drx)Vb72@*L_f&^t#HL=cAAp2m)I|E7B9b7oRRunI5cv&$r z_P+KhP-y`g^}x6D{Y|J@{Q__v>k;CA^}hy^`QEim`t@3{AB)Q-UFNrPgD1Osa4jVX z#-xFR-qc)^If9$^#b@SISlhaV@qnJdOiMcQ92inyyc6x~q)rP^poblTZUaI9ozv?GiR+W#f)Si}8U*@If&v@4Dw5W#Zy1%q)z5w zrbT*v21ADlr)7d%tJ~Y~_gl|ygGJs47f+Bla=26n)F9HSUN0@^+t@S;_FFX?1pE8P zg{^i147W-T)Wo>1iIYKIwa6o|aI^OZ|5(TKg@&@#*Waj8ATgnIB7^o>f*Gla!_r#I z{J6csJn7koQ4{xC03KZKc1F%L6MHt`B~B*XfNE}w-|+$>g25U=vudpiz`=?fGk2}8 zvoBT(ixc}_KJmVj4C7GM#p!#8w?GZC%dGw&$p2zysv`b#Jeru`@JJ`&B`25xnhvu~ zPL)uf4$D{`1ubpbuYb@@`4sIMStm709G#~ zPv*6*b<2NfU9IZF505%Pj^wq6y80RHK8RoM^(IFvE#E4{;(XH-Z5lx&Gx-Evu6O*A zyY%oiLX|4* zGM(J3fv?yq4<=10)XGxARV=#X6w_1RFV}Qa*CY)-WS^2?AxAh!63hiK4p;SQRp|*V zFCLn4v4ugEoHAmE8sn{(8-5mTer78*G1)Z%f^I#h3=h$02@-42rm1H&Cv=pn0@yab z36N}Gbp=_hW*Ym3&A@ecm+^8lHi&Uf{8cOLC#^;ua93kv`$uBWrJGrLuHOi-(8&v)un)P7b$U)fYXRs8{yS9|D>ScYkKgA7swY?}T)9cE5Y} zY!yyZ6BBy5vw%H6-`N?LyT{6P-~T#sy+7Ow#k$^2Bo)b;nebI)GY8QbMvODE=M%tC z4TiBW4t8rqCx{wK)dBHVeCtMb{qrq_&!y%N8CxPjS3TnA%1TRU#HmDc()XaL&t>LR zshu$Oqs09W2uV%Wa?4raMC(C!w*Z@VXUV7XFs>crFBrH#V_%E55fP}^#NsZsVLM&8 z(N8tKxj~$xoPPjR(QEN-1hT6(-7#pRMPHw%%u_m{3@3Hi6Ng4pvw+tS4V zlV};TD|%x!htZ3dkUr%L)Z({4r7nk@LrwBhiQ3x30o_ws64euEBVJoRE{aS+_n|%& z=(46Fr4`roVAOhLOtA)e9f7?)d5NTcijn3@!3s!5r#}qz^)onEOy1o+3jP6}ISu~a zkkpKSq6vbNb=XK}491qlesVOcK+^t*+#pm8JJE^+QOANvSW`LjKKwWge~7oyMiF}i zZ+Y}4Lg;SuGT7yx-0#$tGUjD^9RHV`9;bDW*JB^~b4mNHtSB<`{yw3LwOsT(Hl?D6 z`bU~aER*FKyy>)E9!mY^8@L7OHN_SGj~+mP=jBu462ke&g&ZSfTwIH{_XnRR>>!c= z!rhIZWC1y1Mm+IoA^)4{Dc0M_#IYpco&FbnA z8e7%`9~bCji*B2Vk(Vf@^IHV1%-QPaf&W$}9+kbe;snm0cO}=pN5nC0zN$m!(5DXm zD$TX3wG<(Ne;n{Aa=lP5$ZDAf4lNfjK|L(ek#D^n9%Sk7z0AQcmkz!>I~Yhp__CxI zPd;Hy{Ux6s<2zva@7KUEOX%If?XYj^k6=?5>@DUhG5tip{-UEMBzMU7RPJ`p{N9wF z(97Pr%goHI`{wH3Z(Ad?l|%P2i*wBJZw#hB^k#>pr~BrhqccR}{^suf@8lt<4)Sy_ zoJ%S&KCwdzWGwcE-9{0X^qV9HnPt*Xr}H2}A@<-%@lC-9@tOsu3C2Lo6j*+!s+l|Vxd^E`=xtqXx;(_ zU{c+!nNRhkdrHb^51bhp(s&__DSsFi{ysAVy6n&=n-*$L&EBSY`XX6M!nLjZ8~f<6 zE%C0v02lQy>KT?#$aU2sAr`jAd*4ivhfVx=6Z(Bls<2Or2GLHNxfKxV*M-DvRF>pO z8+IiS&KEKeB_4S6QP@jr6i|BR9#VwCg?wCNv*|R&JIKGWCTbWNIXyh3q0vWWT|1lQ zNu+|(p!7IuR#3G~x>i2{6=o<2eto`qP$ueQ*?#~COo;Q})CQ&wA@v8d@^GUC(GiuZ znS%gerG5*0TtTVoxtQHh|81$DC}3vn$@9Oo%Xp@yBL2P@ZGSQLjl zhYpPEFC%-zOW4mtNL_1`QL*jkC%HKj;~D!&p*L8HqL9l=EUM=g)6*DQ4I~Qh&wB3G zyGS}Z(rx((NK}_*7-T z_B}p6tBu&QPeF7Z=Z0gRc;>mX!cOOAl=Alt^NHsQVPOs z5sMmzC-#sUO4~b@knlzbX^-UNOsnX|u+?Kaws!YH_MpV5{7tvv^hRKV#Jt)gRiMRT z!DSzo@zbL;pX(vCM3QI5t7qgX^@`O6J~*EC(|8RXz6!R;(3MM;P+z<~mpM4mLZ)={OqJ-6df)51~}-C&brU_KDw_VK$tW zv@pgWalHxb3>JIjC4{P-zS^KT-o|!MK6<6|k?*<6CCkR@d~dyv)m4EWG|--0($rRZ z7&oqpX6k!WTer@vrVe-A$kqvYt904(FCgmoMhWrFivHtyyGNEH}Qhh{!wqRVTjc5CQ3&? zPKZjsRzC-&_E<7ow3*;1rzO1&BX%Pj!KR7$bAa&O1?I~jb)y)ztW;yCM~r;y`-$Jn z6-V#Lm~%ON7K|g~Y=U5*iB^GkT`-#}O78_)80&prsqS5W7!|u7OoZG|bW`>Z@*##fs#ia`p zS!AW}FJFI98J%PwFeE1&eaYm>g}vJOJ&s|@YUwPTDl#{l%EADI;Nzw`&;0)|ivoM~ zMWHf96@al|Cx1M7Rfc6Kf0!*}MvqpG)CTi;0uj(7ZrQbFCU9YU=j$vHDbTU35z78c zSWB5NgQyed2b?g)ij+i0X(Fa|@dMbhcoc&|LRtmR40P#WGa*B~m4GH4=Pukqp_941!mLLI3{}7^)@s-I zAymi25xB{|ICDkdCa@2PKe&kRWlzP*bJhZ_fGMnoYN=#Qa{NzDj~AI{!$P)L>);UC zmouX)^WN^F$it_LM#_CzgRTO&jYr}!6rga=J z%gEvOez_+eS-{#gIAWv$+RAOUxBvZFl`zwHk%-rye8t7)`=m8FrbI5H?6`q8NkwPn-~1zJeBsh&R?h{H@*n3*f^2%q94>;C2po<5bz-(Q<@h=a`x8eC5c&3q zDKE}#h`Ci(Do1uJd@=Wn5{DAZ?d^;)vJUN+AERzsFb$bg0JpV);HpT2C4Spwrts15 z9?|lU8*GmtrJVBiaQ&L&388cUZTqIlS;7Pp+mC=Mx&6fgt~ROn-_~(`x-*)~Ok#rx z=Y)oQ!a85M;ke<&fC!UQw)+yEZhCSW1D$dMg6l7T5}y8=sA>q6hcG^`7Rt(x&e7eS zakx2~LHM2~-DB@>TtjdEl+gItquXjOKE7z!EzEJxkAZrqNizXr8*6dS_cmc8@%Fj> zJiebbkg2YZa~mE^CP^-SJy4cVfY7!}g}qKXDnC?Y{4fgow0}Ze*zQvkc26{XCq345V_2;izw;IUwTz}EMHP53{ay{rOOdaa}Y{TU9Lf@#2 zQD#UE>XqjYna+uSgVuapf@$9FigEL?b)p3P7)jTr*|VWl@d(>>-_yjQU1R_ zt$FlKW!GKC%_{qJDiJ!Ls+$GMRXDfhX{}UT$l>~&A|DqI>jr;$NvmoN9B1S(eL@<$ zS}^?c;T~j=q>W|t^bF43Ci@n1-_m*QK6kvLW44Q;{pe1g&ZDg$BEcp6 zD4mzhA&`n&xS@9CV~HTXJl*1d(+vuw-@t;B3d7TV*T2dGH5x@lj*HNFdzLC=_zzka zY6~bDM(G#+t7-%y3zooIKXdl{=GPYLf+ewb?)tW)rrKCfS)S)_esr`N-g2=DiclSAG3D3o+-1%(TS<>`HtR>Tm}qu9NDe z6XM;(Pq8|ukbZWBxakHf(f1Q7REBVu5;|xr5wg9m&T-!VHSCRtvB6e4=Yzc?Ubm&A)M@Nm=X4kmj zR#x^{m7hkv1NJ|)xao(XP$FBS%=JuGE2oLz1QrOPI((ki-~3|vlYW-1k)4G?b4)Bh zM9pL&igRlQyMbU?3%%L!&n2MEZmi8PN)H1|8+vsKgQ#0drVk7YwP(it@}`_?$#=&) z*@ctqF~sj3dgF!5l}c@pb4#S4{RAR47p}Og=uuUMAwL=hM>u%Alw>L{n-5403~Kae zL-f@6JjeKRR8_LchXJ>8=C4(fv^DXcxs%Ac-6vrBtDSU!xLsOTN%J`UU$^umw z=WFe3#t3FG90Dh1`-oth>7iD*WjnOIZ(s}q7bHUy=ucbgbblbM=X9irg2iP}*FO%A zyK9cMYUy8lv#MIIHC0tbsr)>z=pk8NvmLwletm5|iIP#K+~OmXh|=)cn3$YgNT;IQf-^jpGlFDXEj@%*LjsJuMXad`V@1`amqer0>|>0uu5`ti(9KZ}&m z^Yf2@bIQsj^7{%v5(Hjs&MSSU@Z*ltD^q6#GaVTAoJ6gC!7A{1V^KX$Ip)F^-S%pq zMOu*vT%js-U{%}ro8PiV)`3$WlF05b_<*h!`yVwi~xLEARNHylB~=bB(w<-Yp-fLg z@9(d!ecBm^kIoY($hQtwHf}p6bwk;H=|kea&eiwa{noav*)6&on+3;8kjc{*tPEU^ zQI3M}-0KGdzN-QKT4kHvrR|fYoxpRVVEC&G|9k;zf?uZkt>$R-G7mhkhLFNb>XMUP zQ5;vE@$RcChsF%eYI*-dc-@m43^8b3$a)aY20j3xat*bkb=5xiZU`1?A>8>>1qr;= zRevcH=y8+00YhK%HYZnIA%i7Wr_7N8ySr5f@_hBqZ0o7xZs>~m%X_Y4?JQ2!#ODgh z&!93#Zb=*zApN%~$Hpj>Ya(Uo~wZh&a0Lg__!jUjxen z8d1?ZTc?8ZRR)e5`gn-JxC!U)iF8-1lS8d0dzaYbj}INe!ID1vN>n1JUDRze z%3}m!}Q;@jFO$-kBVtKz6&1NU`4tR9O8y@!uC337mp zFqIRi&&5JHxXpBzmB;SBzRwBo^t3}*dGYc;CbZ=^`Ranuu@o`;$IHHfZ^!Zf zt>xYJW)DT8%AGmi{3gRg0D_=?UDnM{ z{CVC|xvq(Fe5J&#(a%Rl2;+!GulWr_!YZFKUS|*X!tf~LxR@v$vY9+g4~+vb-F4?7 zCVHmGjMXQ8gY5X6)p#HFHQ3VK00SfXb2#cti3F%!Jz~I}A*N9*ztUahy9RxGw5RtI zd}!Abk?%3_aXMBsPG%*FUWa_xPSu(@i75Kz zG7~!d#gD#gZf-m%ENU`vdC*7h<|Ef#2*r`i-&UiG7F@Bkw4P-}n1)S~wv3~%&LzT< zmHl|tIFKSYT6)h*^0T9BWwk`>D=f3cM|uYP{ok>%;*#Hvn-SKM1pvKBS}P~VP`V<= zaE>?}#|OqbOUSOl2_qsSM;r)hMJ}tkrK})Z76TZrfcmi-k92`p$1DS=!9pz`!S=@mD?N!7CzT))YVL=CQ0oKVxG#d|~YFJH}Q2Tx3EQ@oLhK%pZlai8mQ>+830VVq`mA}T4N2&hCy#*aA# zrh>sn6iV`wBckpT>HDP=3iq%C>}=pgGhY%j@l?ABu@Th0LbW`PtCVtsRYE zUv9rYXsa^R7Zcb!$>V=aYyZf7X*Y2rtBqfO&BOKPl-Gr4l@Dr3Z+|y#{_&F^cZ#2* zGJNO3{iiSqiWxu=B4JBcxp8piC7D#4;f)0UxSiki&!+%LuR~bIyr>*&pycYCT^Z?p z+*Q`jTJe>wwm{6B22(uqM!q=O!axfp#FiX7HXy6o$ zY&rmi?Z5SgE2p}2&sUQRm!}iX2vMZ1D{S)$`(4bVs#h(Q+2re3O_wR)6CV{*OL820 zzW8oy1=|`T-kkoMYyZv4O3g+#6NmNu{$3adw;@$ZqN7aA!1MPU^yN=I-KaX!pK~%y zj1LtJ3cmQuNK>#(?FfbxBv}M)`D^jF`j}oX60+te7Tqn~ z2JnqPeLtmgT5+=h@LXB$it?uK{bf6_gqwQ~+2R$8|Im8k$HjWt21CLL=)7syb~bue zNN-_E3v%fzIK<>Ev^XvA`XPu8Am>7CN{?CbA}dYR=a9KZ3`5ddyY^8*PYgcJTajyG zmOQ|UBdMO5{;=bPYc3!|DP2%`cK0_ojhYvg*zd%nPrbb%xR#}k_~9k$6!oFpKiz@h z?);!Abi_^T0t*@HvMwVXj&mU?<+>oP+fMjILSOpt9lmPiMp_Xd?|AmC^*Z4A3*Vvf z$#VIFCn25wgsuNpXnqp6iLojXunWG$KK9?=U7;jWN#>(YE9PnYv+86z z2j$(7t`rjD(PJEvX74DeK6m-ioTI6Y0AfxC4*c@H+x!t;wW6eQDote51imL)0gF>1 z%Z?ZGT+#v4ZuJ#UPe(6!iIxVfLr#7d3Lm71=S3@K$;m1QVQEz;PNtD&V|jCV z*2B6fJq3tJ-5Eg%^q!enSy}Gr6TLBh#R0k;@v<3Ovozf?q4vZL+>}P)W_Bb53&DE- zCpj@KpOhb|?a^Z%N%*+w3{~diOTaE%Y@Ejso`%ZRv>g8a(Gzh` z0Im?ng{$McKjG+13>twmKYYm!s}u~?qU`>3(g(xiS~TtTwPyzP8~bF9v5VyS`i9pI zbIRkCx@5b#!EA(~iaK92r_r7<>6^V&@QaZw9en<<68gS$w_NPm^Maz?EeE5Yt1x!b4T(z5)*b z**D;Wy}J3O%ng^fyySiw?i*}RcQ=?D#GB{So#_@~snSK(_286jnNEM{XMp!HUP{pd z`n@RzKBE?Yq!K@}>q1$HjZY*fUWbZ|^EDN=idO@JPz{=)GzI*VlhqZI3fZ`Tf?q$YaIWL){5y- zxECNFOyad%3{Ow^^~kmAf~rnwiJriIWkSP=L+>qFzIRbsxI_$bR4f+}+;UXDKQ+vd zOx;uAl@BSdI;n!5B5g0HqU5@Tp_+)$fy*L|QilylP{aDzIeW-*tj*=hu8rj_IzE0o zN}N?qY>bZ6=FJef{r1G@3L?DU<6{mKB}cUor2i>a`Dg&BhYe4yG4&qG0{4md;59^6&YcdG_AHA4_rRosFLh(f%sKeX~89&{pt(SjVlw z%iSE#(gVw+ihFsjyMf=DftT-2nH;&al1_yqhcAui*eN+&EGjAAoa}_uBkR=3 zj10b#uooKsPUf`a+9F^$dfa45SLHBi6!s4J*@Jr{pRwc%Q8T@PR>kt?(aICLr ztvr%BaB+)O4&2rFLwOF~pBo zxrL-FE%U1)^JRml&|B$1g9-3CmGiR$aI#DopAbp(BpnD;CO>RD@DA0{9)a`eaX@TR zax9xO*`U>vqqr_Oti+7?veq`|`z^q4VqRUO%$q=UVvdfvgo zR_Zop%^)0Y`+|fL=VK%t#ClUOMM;Tj7nYhrFZdGTF=?t_>+RfiwrG*}?cD#Y8@pi@Y{{btg9TiuNon@YQd+xto=QujdA5C#^$?C#!kwmeUYId$pJ#nh=P<9Np1$C_3+D1{5An;l+yX2?nVXL zpNG;n64|YmSs|A~ZUbsq3&XqEN#PWU2kDMM0n6;~Xs4&8?vl^r$xPn)D+VhR8|wq;Qwl05H!PP+iO!W?;y>Rggx*Yi=?!i6lNK6N5hg4 z>CNSXsDv&yEB6ZLNuu4uS}_q{*9!JhR++y)%RVZBEXiZ_kq(&JSu>@r<`y$bA(d`b z8@>x2>yHpUoG@5Txv{#=lhJ~B&`KX!EMj0DqMpg7&a5g-nTwF<0YT*n(l4Lca@(ZC z^4U#fc1u;2=PH*UKR`UyWugqH98nb{J>2Yc4=!pZS0KXoMX3A|_7yu-IOFkB(1@3) z;Exj`Fd`Q~TRD6C_GQedbX36J!=ay^U!Z>u;dDJ+A@IWwu zFRQBpa<_M(A@?h%vcWg*BZ-T(w0j2y%)d>e36>p}197{bo*XBgx}G$YBg4cUhR7P1A8*04%ZNL3a7ncoLEGOKe;( zU8W9irMJYpzFJcTYBV;Y+QA-Xj}Ry)n@Ij1%RtF(^IXUcCdCNEe?;asVL$Hf@5P#o zO`)QDLD_x`dR7ndJaCU46=u0uK^R^f5cvoD{*iRX)B(SqdP?6IAqmR<^s3h$9xz-? zWz87x<=cb}+`kD7nlMYeKfKF_q*(niFj)Lo1;e2nKY|teom>=kSXqr5m z&kHEzmVfSrLn-WN%wzledR(z10=^-aYCdgSmb2lR8Aj^f@>}gEyxSRp>|NXLMaSy`l97YQ-JB* z>|BRISP$52R8V<_#^()GaxRbpVGMt<_v-D=Nfgzu$GNCh!8DOMR;a z`FC@(;>)7Fr%^Lf(8yt=cJiUFc;f3tDuLjzz;s?%80Xt&pe>cTlO=b1mDF=TK(Tg@ zl`arU()p6KW&OQpSc5tEB68|u(zC#Yi{oSPX5?Y(jCGmUU4hKskmj7g>8HSSSPhR? zOW}ffIk?a+wc-JH67PBfcw9??w zyOdGVW4Ly9qPmrXt2wijHSK9-LR7*LW!_wkTMl;{!$~o}&W}kdT;PH(i4O#Pu2QnV@;gPoPzA^Cpt zOVWT-7<3%m#LYD^l*h5QZh2qzFwFTrw;(mpA z@dq7D>5C{^Z9G;%k?9ak9C*I)Q5` zvV;4_`n69KroT(;D)>=6_MW71$viPt=%ZsJvQr!6bV%-1j+H5eNlAc}Y?!E;dR<~3 zT)cXY=vxj*r$g}A9FW%@+ynm* zcP;qt9}pC*!W`FrG638}=`$P2a-kQ0mezvr2a068W8gDc@kmM1Pk-poXbt6!B2g<*o^{$(3dfQabZ1} z?|%62QW=jG-0i3WsNV&mNxJ5cUpvyPU0}~i_0)*a!|u#6CPuXSrVDcg+4KkDNvniq|&-Sh@Qdik|B;lr~G!0G>oM*M3&Mos1z!D{17(Yr+x z(dP7@=uXUrI1jh#V3@#Rad>6cKV>+>i`QBcG^m_IA7bi=0N%VOHRZG5h*uzac!lT# zD7R5&G;i#cVm#3cX}7M?im%pSo%h#CJ!=62Ymb*>d0#8K z4T}V!WI_iX#_`6U1 z%vCoVSC=;%;w;o3&ll(NV>!PE(G?XI{?9V^$4gDMLtu2I2IatyS4w%Il*tL3UdjFx z>3}hJNKMAO5xk zp_$1;k_sq06^M>Qh4_i*J}efenEcZroK;1M%1+1|&-S}?zt=V(g*p{{@J+q>cOHw} z?f9we?cZVTDVE7EUA8HjUo4dEHBw^(Fc6wR)Ws^QJ2?3Nn-{xILLxas{&a;=RR?}R z^9{kLN5%R^>??DQCXzuh9M}ejC(j!Ddv<6mxPMG4nwqL!H2&=L8UssD)3G{^k#|=* zkCiBQi?)R+`u>tRot+?EG;nrzs?YX4VWJ}PRE&RHg9{Se%^%>Uao@_1h{&l*fAawJ zCeUex6m1ITqu7>rp}E#33-DB2|8w#1hk5;z8`072@7#B{6_=N*!^2!3yxm@|=2c5X zw2SG9V5I198rWjaXqL7eduoQ@x^8$nOMnKIRR<79c^$y);g5!x%tV!}*!5)!6BYBr z2U)$f6FIQF(s25yW=is{{Nat~(xG>50RU|M1_}kg&*yY;Yygy;&Q6%|rfKmV|H4;H zx@xgpM`mSD$zBJ(CF9xgTYB-;(u6E;C4{dcohsZdJo!DpG=XnDQd#k=i6304=aNW*S>0?`+yNA4O{r*(Sdqp#12cP$sD%@;#IB=!e%8{r;h{`%>a>Jc4 z7iOU`EWt!^wT0Na4G6s53FxWktFRnd3%)q$xor)-J$4O^kJ{M^dVl+&4{!jz(}$@{ zdvFull&62%(5*+#1824bn67uvY|-ZP*Act>#kJRMaV1vA|4n{a@luXev1zHmeP#V9 z-B91z*%}yqZ0QL#tLz3((gx0bmHUyu>7V(7{cl$68?hxmsK1G8E%Jw>LolxL9LQyO z>pM-%(?HAas=!MUb}qgBJfdHJ$y*Ezg*^#C8`4w5_2%ZMtpSn+nYs%<7Alrqq?z_d zxvgjvakn^$bTbNni!;<}X6`6IhP%vDHwQdBpAaf2{4s-Ym+nPqUxoCLmyr_DJr2xk z{^ZjvOZ;1q(te?%=2mdx`=FrOX+Ut-9S|!UA2%_gfbi|`?OgfYxk(4p#enGqFiav8 zvY%SI1MJutr-jHmS7c3@W);*;=e_YkqBDy@RmIy1Lwpx=KmcY&S_@P>k1;-J6_qIp zXk867%(alQu#o7TrS<(H3SYEagabKUX2|Ab9OvuldQ7UidYTBi6ps$@Kx(Ey# zK8gU9M`gn2iqg#F$gi!Kigac9i=CLR#=oTE5E3wW^W@20U8f5}gu%5`!_^Y*U#Vca zIhS|!OoP`Shs{H<^>Vxo7mI%d7GnBq?wH7@y4je|NRB1?e|;LgtDEw_4<(Eybh;K= z_2t21$5CWu(33@bvcG69>h6r?kVx)$PTd;ZN27$MI`qWnJWJXQdR` zdvj$|L?N4NW$!)XimYU>WP~olwXeNb3YWO|+N&EktDF5hzdztU?&Chrxu4H_yk5^2 z%EHn;WHkSijc9yUCcoTd6-Z+}o7|SD2!d`frw9$5JFvh7=Waa2)p&ARCG~QKTII=o z)y~W^W)pYn8_aUFk*DTAm}xE`)@ht7i6PutJ5Z)p@8=Df{Q&;R50#dn>v&9ckK#P& z$Dr$6jgIc4^Q#3*D^yE27Qf|dUP;1f=rF!qpbyE+1C`wkB)~>hApu@j(~^aTXBT&Y z2)$V#a`iCZVmwg(-O9~&=gm?54Nyf~{aG70-Ya}rvytydrX{29wX3SFuOB_Kp-+p& z*0F269%!p4j#nelKRa#4G7cw+9U@ z{Z(pz_AhL9tJ~UhANfwri7$AmOBqKw=!;%~wKfYXCHP>gFyx5xR--E3A!1 z=Slz2J5<>ey=t#{=Lf{8F{6)el8(K;I>a!E!J8NfcPs5@&=kM#4Vmly zxyA}=eCn&NOI1TP2+4>|gi_Ib!Fdb5R55!GS3IA4XL`HKFt2og6Xu}(I=N`BW- zUL8m@gcETGOgSfjwx85D;A{OBCZiBx+~U1+ix7EOd?nq3-i z%>EI2dtITrzXLU~ynhtF{c~#ylSTS2sCM?e%(XTN$FV9@m%=j$1SiEtYhhzOgZSvZXp@%FktnY+(X zLaW7qJ!%fWkCUsb7<|A0QeW6U;05}>Q$TK}O9`6%f|6=80y=s3E+6CzlDl%wlRlc( z0j>1(G!Xu-aC+A20;1|ndGM_`KXMH~cn;lexor#raf@rpL^~}!?oeo9V^@rflCj;c zM9segp__whE2^3+5anwxoIIksG(JZmWcRIvQukSDeSY2C0{tVzFa^(f2EXVgis3-N zl8NV|#<$*^zR*^QR3RDB#!^GJmYeo-L!T`m1dL8`<9v z90@Bo4QN$eu}3#w$-_TjL8aN&C^DK6pTZqEA)(3hj=F<7)L-=mCuLDT{8dMtEv6&? zVGKhoy?D%~5UB~>m<9O#=b$)?abrsp8PI*1*YhX@R?juUK`4Bg;ZLdmMvshV$`t*(V%EXz#SwOP zHs-XE_dWHPv?@5F&K+QCwl2e~iF97j9eDxh-C9wlN2Y=dVN7$XI z!kQ@Msx+M+%qS%m?CtlE67&6FgNcOXMDfgfAr4>471-z5IviwFu$F>3^)Bn(?=e_%aGou zNF+g3Haq<5SHG){LO8dJK`$;4!^2z~=bxk%!Ief7gfCP57I0 zy7_}3TwqgitcM&4$?o-wNJ1E71RnMgyD8mPogNfaYx8|(!<@ofUx(w_)SX*`>}6px zMo_hI0cKWTuEZ>fdu*w*iAFp4o}0_dHAzF?TwqrbtT^3zp;W;yC}bSwPQ#2V0`h*x zGmjHqwVFMj$82v9fr~P<--k^;LjMkJ&o__nFKvLeIpC$oqhS$IUvNs&e}JCiyA0Up z`6Un~0Jm9rOg&1fU(Ax%AOC~Vq^+v$(K;D3PZK``;x}(?xesOjxY==5=zW>M%NkMo zdfDQgH$oNG@_8VayMQfJ_7_hnAH726C-FncyDH)T4$coNa#}CW0m}vuWy(6ek@Y5* zlPJSr0Lzx7Ls540IElpKmuu$gqp^Q}qXTOG<9eY*dt@90w4tYtwn)YGP@K zMxX#Vb5d?19WtkvO-=FcA)5sO+x4P;K9qp{81x9yS`)LaC0t1(eU|3)o)E1=|LU#? z?e8bhwxYq)yXzeOE>Njgp0(3dn(;m5eF|1{x0%AtBfcyed31noYYRB6o~DecYJi$*T>nZ5G^f$jj8hT1zIW|3kCE}2QfnLr#MbsSJMt%&U&9nwt`|a=* zWGwdPPhyxL5aVTiEM~I3=3hFf3abPc@a~3jD8v@H-Z9I+S7E1|GB9cr86sG5wVh~G zwG(5qcZKy3{J_E5`;v)s#{6l#ab=;b*tn~_xVL#t5FT;*1L0PS6bi6%tM;IfAxf;X zo)7M^y}k;#3|_pB=G;z2eh-V1&7!}0cR##YgYHVyGtbdZ#3~2CAc)ThK3+ngmiGfa z_hnP7W*NZ-vstnqE*k4_fRaAj*uCD=^k-O;d(1) zLNwTLK+#0d!%EjM8CG7%BbLdR5tmu|*z#IDQv%%fRU;^IG7j7)+CVKxC)5{8MJ$rb zAUi8yV9P)x{RZc8b+*&0aopT$tXW*n2RsMcn!6_G=d%;F`cOOmd(-32vz_LV`jAYii6eys70B`peMaQ+JwA zA1KjiN2p0~;5GJ3--2)(lY>Bw{T?v;>!<_qNC|pv4k0D3|9^~iu+*<~0 z@2%yRqCJ3|zCOKKs~3HLW%6&-`UWR;dDm<|__8xnpzi`};kr~THfPgt$J5`Bgp@q4 zw2!3sz`G}C#gc2@W~QAZ)SVg{Z6Ce*y6^J<^_R7QWV;t|3?f!3BHSYTV&AuYrQSoh z!z@hMXh1enTU{sj``De(eK*eDX7e{c+x-jHM~ZF{IY93cbEE`*TE zP(Wax|DGPkR%UGvu>3r(@d)GO)#iE4GY=7OCu*yCnE!^7%nq*$iA{ou%D$;YS-=a_ zjPOfg9tKGexCCW}>{F&P7e@Lo0*-fdpS)I8)lVVgA4~zQ#@pIae~Sj8%;QNZI!#-j zV16HaF-S#UEMF14>T^z<$44@%)EE?er+?RhhF!D|5-{^$7A%tD!aocBJ5j8HR5Z_; zM;{=omX@` z@B7RsZaj!Py&h2szuLgt^j!Pmr>YKZM=y zTiD3<8|6Qq=vxW}G8*`pC}rP=>m`>3cYD5Vh^9L9jx6Wta;flk4}&)p>yQg(#ILIn zt%xoa{?`MuGGx#2oMpmuIl4GvURkac+E`xZScv&Y{c5) z-uER5bNNNnX%3M#dMn%aEb}72P5ruT_-A*qK5`VUuB z__s{V7C1poNiDd_nq-k0jrHmp+%IhO$5ewswc!Z0%31iIx~8 z4y0=?QnPHyPW1o-qd3Q-++ffXUZw(9xQDqXVap8_mwF^w$b;64PqfQ*F%&eIoC{`2 zkaPq+Kn|5>Xcm{0Y=o!YVRd8x%ax=Z*558vdK||aTd3OU>W8m^4%@;Sk@Y?i5)e}l z3`c>9uB+|#;9}lXzdFdX)sljo_&Qs16Fuc`h3Cz~yrgDjN!R9(Pmbo1-S02)5ck>t zV4_r)pS=;|sZv(`2Tt$}r8Z#lriyN-4V=3l7%0=UU=JS@v)`3=zyp*K| zvXtw+7HlHVDo&MC{YsrBXQ%ULMqV?5&J#+a6+YLv83}6c><;geHaOv;CmLUc*;KYG zm8|&>9>$U?RWKP6vKXOUui}kWX)@G*Q>d2Kt-3&YV-J{^+ben6le)(hKfmYBWE2A{ z%pkKwY`QPgZg%LRE(aFO965@&eGJ7}={DB>otzwAUb@P12;OJV66ZS}4HdF_;ZeJS z9pCq8HxHKcIIBDSg+P`4M@u(7lH$41MKXNy0~1zHG&|Dk-O13KU=JXkEnfL81XwMY z;{_e%+pa*Wr1M5$T@KOKx zz~%e{xw1(-Xp5#sjbQ@X6FXuw>0alhKCYoBeOc#V$!Sj6Qd1d>) z@!v-p8voSM25nIpYiCyc22(uzp)GqYpkUu!?W`p)vBnON1~h2Zji$E-L6Zhz(A zcx&eyuuP@{%uKFMI*v~&tcZl2G7CF$1f7HK-^zTZa6*QBA*sMod0%d*xV+hFW6oo8 zY;uybg*f-J8`dajHd}_G01ch-NU9@pA+(8S%QZiqT*A?CBMaVo7F~6}qdr;anDqXUPeh!`rID%Cloq#a z_@C!5dZao}vKWd=v=r{KKBV&*WkXl2}!{RZUFsi>$JCHm|GaZ~ao-x3Gx{_FnqC}w&_)%3#nagLOH zw1xZZ!pPnmZG9aGLzUPRNgAcr-*`_-#?kbrQpmtVDo+6z1#vv@AOeLLn)G}0<*<;@ zs@``XY<&=5m6N`_S(uauL=710hRAXlZpe0HdWMw-(4(+uhN{WmA8V+GGtP%cxx0mA zj_nUpC?IRiSyleiKc2henY&(lqiE~qgeyoPR>EM z4)#n1(5kQ2pld!;i}{&x8Rbh$_qrOYni|w|lci9te`h;D4ntvlV&O=LJSVMmDpe-v z8GmgyBHN;ahdcAb6Uc+4SP~%y?9WQ2m$Dd|LfI`vZW1=`bU$MMOj#=oeW1;BL!&;W z)_OpmYZ%L}xuL_?lqxab%BdMOI|q~B1nOjV z(e^OcI$BSZuE^GYV942?(Spj&nVFdhqYT@&YkuX%nZGROPRRA)S&+||Ez}f`48r$O z9UvFjW-k!?X}GHn8)F*q8MaasjWBl*rI0 zJ>c;Fy;$C0C(q&F`n;Vu#fD!c$KI*-jghQU8{j!P!|>3en))}1Vp~G&nNX}LJF1Q%-TaB_-#S5> zwpqpngue{0Ck7IKoNi1EnP-{}S$AArGBBml(I*lEyVYj#|6DC7wUK(MyS~cyvQG3^ zlv-$!e9CoF(b4`;<;Dkz7Ws2r!Cu6`NjodK!#{UN-DHSbox%~8!m@1(*3@46dIt$E zBbtfymc6x0O9pvA(h#zXJin%;c;5{GXBrUlZv7xvFrg{dOOAwC_ZTXR}^?~K~UzO9iQul)|Tt`43c}^ ze6yL(d&>2Gd(=I3R}E5*VH8=HbGV^xf3lYPk++F@AlH$?R~VdkzU8}oTkRd;%-|nhJ{j0h;>cmGLUj^sVR)r`VPGb@E@Rwgb zuUY|x$j4h0C~{Dn9;A!geu0?_`ju2ejAhY0T#*8#a&rkh{=dtj)FAqZ!NXwxAd(gH zdpvFp0r2iI+B`wZ0!nW0H+8AP9WvxugGpduQU4~HLNTeB+6uXj zR{8866n1uG()4%29eHzIRaG@RVpMg;?J!O|Z}Hu1%JM-BC=b@h*TI$kkG*}4QuSwC z0DX&p9o7^8fC6!70J9KxH?qSD-`a{YtI@OcbZY-_YCZ0T{3}MM!>`96a3YMIwJ?RM z4yj^B=V+9C`I)S&5MCTpz{hic%1*bvC(kwSjjQ`lKMsCMn`!1J831Fnusa^{@88cX z4l!BoxR6bi{U)~FjqzL^oIKt#*aU(^(cr*xPrG$d#p%8-J^Xx49NWW@~LP$1*)#`38xp6Qll*%^~VeWT|8>Cz~>t43jXMRF7@Odl+1;FiUic#Z+QAW zNQLXBy^d~x=3uT+woG#*gurnMiH@qiN?h2%H<@8{r!oZM?|}L zdmDrUv>YVwLu$656tj{M*Vo5DsY<}4m8eAA?42C)=gB=g_U*yyiA;_r(H{mTuPm=` zv#4|N8qa6Ffqk~~zR=r)f7l94yt+QM4H`aic)!{5_vtz$y`Uy=mVE!U!)hBj&E;2# z4W-?}?;On|)MSESXB}2SZyhyTkUDR`15@+TH$QKQO3)oc;H{{5#*gJ$vOC^l)E`$G z(e&Uv&?xwe!zcUFko-RIYm=X{!#rEGaxaHya-&&f4V^r!ULN^h|5NhT1sbIDBcNOK zsBg7!_rM(|4jf9B|BfbusWp;Hk_Ox^i+Zqk)GbZXD~b*;Vwsr_!!P**h!bDr)lq1t zwMaeO?DYq)oYR~V5rBPwTgD+x8NsAL0hKQR_Fcfb<3@LBp(xH2dR2we+w^VikM=<& zdDWH>OYry9zkIw51BNHtMVKoC#B0t z25Rc&%~)-Rmdw-YgOu(_iflRBK+g*)zcn~{_3Xc%3T3uZ72@LSraNIG6R^gn zCc9dFGF8%DAH(pAA68;X@ypgjUJ-4~)%Xi=NT~eHA3!A)9?4eO)ok4TrEqkt1A$mc@>Vt+Ied_Zviw-olh+jQ z#C;os4_X||E_hOkSU(8x=vGM7oq0nsz->w*Ofr}_ z8u@{?&XyIe+URGNMH~Le3Sz8orv`BpeP1q?yy@`> zrSJ?*k@PfLCllt#OK$v!47$ui$Hm79f#`$2s4|!7kwXQb*@~1VTrT+!rv7d@3%9(Q z@Vz?v>nm4uy!IgReTs>#?e?m6DpQ0L$M&5V4I}1K$PaNm7fOHVcia+_^ns)aI9fWtZSRA#B8e z;w{2+BMBKlQw`(A$Mqj52GDn+IavVLb%SDW^z(?c@07wWt2_>w?|dM5N!+JNUSeBB zoiHNHUcntcPwfUr7O>9&s^u(W<6UWVlAZqgtZG)w9&VvtPG_1wiNR+PLc(zSqL}IT z!AAqXql4m?hKU=Jp)BQ+vt~7wW$+Ft+U-Qzve)ac;R=W`vFCT^-8Z{{qT-FV2q`xt zQCnCY+z9|!Uk4ssry}jsxVM=u>=CY|!JRD$qh|b% zH+Hb~6ct%#I8B`Cqqr$4Fa}c(**r@h(b4|1F`L$$lqTaZv0VCpJP^bjoO zt}E~ZxN)+Nb2H=IB&;m=tz4}>Wu%FhU$9XP2G+`8F>?hEc$&nW)bB~})9Cc(o0sJ> zGcte_z*HZ_z`X4r6-w>U|L8HGdmCrw=~a>{djZ|RJxW;EY7rvnvHiZ3R8%sW6&hr2 zRORJ+>&L~xNpK-aBl3`V>Q<2xW?47Un04A+*Ln)H53+>bqE9 zahJ7ski^6YmUccWU7tqVXZVc6^q(UfYN=FPaY}}SD=uaOWqlxog3Aj~^9qB~ebYK# z>6+t{^FEUBHlxDuDeMKZ=`qXu@|#&rDH{~48~lP5A@Xzsb2WgI+v;%+)*T-poKyLt z-wXu+OnSPl{kC_iJVwv78EJ9`xs^wnp#>v69S*v8UDI9VIgGyPYX~nX&{v|GQNY&` zE|Bhbz)fF%!E<&?Ki%w4sjGB>Elu9uVZXDWrujqp2^HO9zdlI7mpfsS-<5+b0qJuf zfa@3z7@SO!Egxk%4k(q4>=wJl+i@E%MPs1aMpYq6S|E*98Njf|rj7e{rh?T zS@+dK4dLcZ=({L(JQ<`f*kXxsx-2d5&q&Fq4eW*Ib?sfZJ`uI0SB^nm??pfTpr8qy zX%JMS`N-t_^Bmn$2SP|>*BJcu-(1Rr+ z8V>GSxcCJS>{$BOFq5%w7;@03*63El!9ezxQ0we_;d=C9WNB*vRTG;A4Dk0jHrTc$ zQ#NE%VT*_yK!HW@)hq2{F+)e&;9IQyDR&^zWiE1_iG3W*w$c zVeRvl?N**RMD~wgOW974Q4 za{Po+sHw#NH*v@;vC?15SMsT|>G%5`rb9TMYhdc$XF9 zl=r89g#9BrG1fIN7?n?T&^3~)vN9t9J)IB;`d#E8O0oQ)O(MQKW^&Z-uDW_TuWEX1 zZvSdnr%ZF@g<8X%?uF%V5K@+Bqpiko2e!8yq1{8vjWBABNeQ8c+`PGJf(CGjet^UC zJ;+%g8-4@C+Wjin@|xyW`=|~z{69t{NCWib?PX7=xz&rHyR;^-f0lT68=WJYEWFZ{ z&#`u}jrxrrycMEL%TCV||GhG<)0z2CQDhl&Q|u!DGAS8fie20U2B~#Y5waQaF)ps2 z%L(fKj0sINMa{EB7sFi~4YznY!A%;yWT_q{uM$Dn7&$)cr9?+qJrXaU1bqtVSqOuh zS|-=YC19Y^z-nRJh1tHB(`2MHOr%vR|7T&7S5}J6soJ4eNVsTS$>Cvod<47i-(ZWx zA_r1Xg>n5=H>%-_Jv3u*(k?Iq2JvM}$ntyHhw#)EHza3Bt*(v~lmVC+h{pZxN z;*<#ohF!_0{X^p)_Pf(Y&(w0uJH>;%p-^{fUam9MY1)+0(N_zW4r-79Tn#E8Mm_5z z%5j`Hdw3iX(us|1!}NUFf;I508tK{vB^}_aNzm7Z_s&r2{r1&yENNAV=#r`pr)Lm7i3U)$3nJ^ z(p6O*lr@NBKdq^Qnp%qBmXSfpCempd(;F5CU$fnWon-v#5%!L+_zE;t@2yYU#|};w zic2)Pb*>-?%9A=%)*=hbNHK#-@VEXPoTf3?eQ_fMQL?;f^xYi^Fqk5jyPQ1R4H_F? z2Fu-*9W^NfGr8%>Imh|H(+3B}nW6!EfG3LpB*iAd>9*L*iG|gr{sS2hD1`n`TY|E` zC2v2ZJi??o+t@+`*HPmJf3?h(=%T9+$slrudU@q67Ju!5h=WeBNs#$IshvU-=@u66 z*olrh_x2V10*XrFTOv@*?-n$BJq(usRb~AMFWlI4fm_t~&1FGw7GRF(1!MBHjYAma z6o^8?>rhCJj$bazT)+Aa#pjcHFJQ0A%R90mwiQD^gZL1Nkg;Bnq)i<7NT}oTeTw;f z|A3;j=HqzMc_&}+vXXDy9D$s8t;YmqYPvW8b!68xZs_5(FQ%YN+2-Lmw+!#wjS^LT ze|5Zu!bF&`o5=2{@poySCD4CMToEMZ_4c~*S5{O}DJeDrsP1Q?#OAJq1raZCZ@63+ z7#&bufjSMoJY3VHZ#UVHrPcT`WjNnIC%nS}KUDnIaX(a%qmoOrknr>kvNkhBSyY0j zAhcHL+RBIPIp&}6>Mm^`ugtQeL*FNvf#_*nS2>62%ws^|IPI!o`w<@y<*#w?9xwB6 z#L!_=^&0qjnBb!y+f9t;&Hr8iiZG<89|m?O$&W38$k=7b*xtRvQ15IW1nP_dPAh4r@okx2zx6 z=U8(xB`8jB*iDM3nz;){0t<+d4RhI~lzutu$<|hxQ)^v`KS0(a^BZWXM5lRc9jKQg zpnYXQ)07Ap%6}Va+T0vGV1hwp%Vl@5OAw|ip3u3{QRfKN#~j$x-R;#akoJBDlz( zMTj?&;Zt74An@>x6>HoUmT;A}DZ3>NeB{ypwsOMOl|yd;^?Q+e#UN=HUTmrGTk4>qocyrQ*f2b@-tx&Y0FX)MPgN<;l8&yh#GARRdZPRp zcf}-FhU8_25NOLJGSjA~aH&P7*$|0AP_)G(Gn$8gI_6-MxL=#m+X3bxR(Lr--B~;3 z)Pe9^M;3+P+DDnaaYtW&QeRQw{|2QIqOMM>gwAajA0EnxDM*a(HVgXwz1tkLT=3}k z!1Sq-{_*FP6<>A^t}S-1RoUpV1Fl`rfqH7MI8wo&K*NN17^>iVT=z|=K9s9z4=9?*eiIiUU~m*~VWBUeDtER0m3twP3R z3rl7kGDynQ`4H6*u%|>`k|HP3#CD1e^P49#1aZJ(EarceKiQ0qPUm&`a_7GoAIA)} zO@CPq5_Za&_N>Obc}#}!3*-4z7{+ZMX6dF6{-DJexPM&u4* zEP>$Gl1{>uCHOheJ6P2(@M}CT;r^hqq)fO_PF)&*pV$8A2j66&+df1fWwMYg)h@TD z<*)F7owf!y6zW*_@zB)(tkk@g**!G#vdMTN(rACl4L@(y=fT3DW>?YRM^@z>me~9! z+lvy)tf7E@?PgzjYe_zr0NAB)i?Wow)5k)CyqlP}5BV6Q-!netxTY(^eo8-|4jjN- z+zPzpSAN6JvnlfaX1zX>ryt%D`7`i{YF#GrJ-PgbLhI(?A8!gO=gzD9+`oOFMsu-g zVpI-#cf-*2eDnvDiiaOV6h#4`0Txftj(CimYl-~#Crc}D+wQgMut|1ITGbhZL(xFe z&kGv{N3>W=Yp}+VA3gd~jGQ03+w=uvqY{lLh*aPO4iRR920I;hsVP7MqNAS6__ljv zHBV5{_o3;esw^zG%JL|wSk<2P^u@VQ_jjpD{uBzVX}9D63F<(>88lIj)yUGb(9Q)X zOk%T8CMiIo07X}#++;M#NnLGKYXiNYm5!EQT!^`)yvef*gU7ShImp$yz4>|}Df)8% zUr-F}UGDi`A57!`o06T9$g{1SfCxXW41NgWB*3cAwm2zoe%7f{L88YJAHZwJS;!~CuqFf*6joB)|ZKtxa?Wgq`D z260|Vb$cc*EryH$tE2}^nJ|~Oonv#l8>yh6hKA}DcCLGg2Yp8^b)`$cmb_X!&^TYO zg$ZCk*ZHar*AeKE#=hKu9tMK8Hx5~v-d9GLXXn;)SK6^OdgBEjM&f9C`8|{kYJAo` zW`zRWigbmEM%F5Iz9Q;qsptpDBxjeJ&75+_7-xE2xhjv>PQn$r675(+ zHB*9fkoB+O8SMErwDd#I`TR&I|42@Y##T~-`CbWdlmZ5Sx!R7JI(JtO%S%Y}7nyJ` z9JuN6yDO&#h1!GLA80~fJ;opjslDm)V6XXgt4)J{w*=k_iIJqRY4rUA@`*qVT2CAE z$hQ=E*DP`HJ1fCj-8x&;qo5?2%0@9UeBPVy+26C#a~^DDPw^EkOI8Ljd3z5UMe_yg zg{?rN4bcuYdmgeA58Bb_(LW8^d{j(w+J!69zdKcWqPlKkwPd*PUmR~cv9`67m#I;x z9WVEcVOskht=3PGMV-P9*dwQiL`Fdezg?z3wrP;dP#(m~{pNlbq7@gNX=4Lqy_vb+ z@`aNAX|4ACr>}BNh@uVW3w3Pj{! zu5akTd0b4 z#lo`hmf2gUTk8jRr-9A#zdt7@06d0sM?;qcBs5_kWEDXqOqf7s^=HLGR^U=_cdNM% zq{q3F4VHKdcdyoEkNiX*ZOEc~yNykl)%3`KWThD85X&SbSTpx2fOTKC`x!s%m6^KmW7N1xRRLD3X~{p_-(h%94=r`4zpCpwDh12gxCurNQM3sE$b`|;Av3;sbB zn_;Z0^AcUu^OMWt-Td@Flq@`Hbk&xvYi|kBp*07u)=EI+-v>N=CZu|T{%ISNA>QlI z?v^EtqrTjhYu2IcaQXeW_=zpHp*qx(9K=DlS2+BUpiR!}b-invCnTWet5Gu6F*5Eu3-D zu8^@gnDS4Tq;@bY2W(#staz!SzHmf^5bjs@Fi*uCFpR_}q*sZNDvN0nfgD9i=e)F* z08(p*qt~0c*5obdgKbZ7Q1z|G#(A4XMvSY-9PA&>y%4NMNQ9Q% zZ+Ii#M)f|1(M5Hi>ry>>q4hr^wZUypPy)|AL(p)F+1e>^y;_;*0*(CMYU23AP>F+6BM@}WsT=3{d7@N86j-(vvdB<&g`m20<}U{ zx>I?U7!*}wsP3Ad`pszShqt^yaixy2MYiTYL{I@;qMh<1{#BClPhx!Zbb?b#Q>W*r zQ7CN8uD7zHk`htxY3OMI3Asy3W)s_aEW9JM^Oeo}@CKbQE6eIi;rq&yA(OG2OA23x zeyYx}_6qKfL8&g7if-xH!w6HiXM+N7h9!t792kKxX_6OGrZ}W0{Hz>*EQzl9AX`l5c7WBGxGIH_bg8skDEtG_OHt<#Z&MTd-Z*xtJ-F1& zkMkdh&n zcxJM4Z^pvU#`1X8QS7~-#b;#skS?uf@M110mckI`Lt_9*#m0WJM`6AbVV=$JggmVH zV%D4}1WopGR{M3heSS@xAV^#w_)y^^526w_{+fzSqFfhk@vWMG&8dKG!k&#~f*uUE zS7fzzq{>$nB5M&#?!*Gb_jRIoH(>>G45TJY-X)1w`#hV0uROGS$(H_glK=iRf^=Pe z{D(dG9TOIfH~q(+ZV+`$4hS2qkv1{(Lb0UW+Mv|QAOF|MqPSa6*NxTuqUCa1Q(y z?S27);NbrV)?gE_QYErXeJbs3WVNtek#@l$-P=*oNu7jPQEEUFWTy{Xw6IzT^T2KW z&Gg&UVU;kPWpd@&IlJ)TpqtIz5p)i08U#{$g@Cml!^h~nVaT19FL}F4^$S46BtqI0 z`RXz9C-%_0;T#ok_2-a9;;$VGz*%L&%&uC@S~O>)I>BJ#x1wdr-V=$~5o&s%RGzlH zp;d*DfyC|F9`l9vR@kvCP{*0S*th!7V+D&bnKq|S+mKCY2e9-gaq76CidkDfsUNAO z@}6%;?YD2ctQ6NE1PVOz_|&_;2M7~XL0?Q^$e+VGL~Loe_|YwnkxSa~+OSXk0sT0@2Bd-CUaFx+rK*I}g4#c*O zL#V#R3vb?vo5-kNa9=qnrnjMgs-ZhFVs#4s1;$653_(uP_uDQO>TUA!-luG^5U=gdDkOL2Wywj%c({j+AQAZZ%#^>VAj z@c~J@0$g19*3RQ%G+QgFhtj#vQV8y@iox=0Oozu>%)v0?f!-5G2^<`acdz|L8!%Wy ze&0wuept9?OGWU(0!h89K#@)!1B_W1f zG6}lb6L02aXEi#TOOFU~^2fKJ2~2TjFHE7n1G$o$*)fGVlZcvOC}^G8pGCI0vCcinPk@b~GKFGB!4s?H^A@ru z{d5kFOHTXw$p>wijtsvELtK0%Xl~Mi)<1h>%yjMxe(a%IM@%T4Q^ydFzF@ zri?eaGuB8!!WZ^iBIri@XG^xTWiu@`snY9CBJfx`8r?y>c;Cfgd;@S0Y3gdMzs`1Dk8Oy=&)$rb_0P>gcoip7%7-( z%_HS}BQvMf9-Ql?7sPpZBwpr7RAu!R? z^B3|_8IL4!e`zE20Jn6CGPNl5KiC8(B@yMslkF8+$`_#1Cmw!7#2`}>2|rTDb#$1G z-d8P(+j1G%3@SKpSs=6pHGX;?*p4>Q#VVB5vVNr-fc||#XvJn*nga~C5J)!JBJNNt#UaP9ugq)4sn0C^!?BW&hJe@v(G_sfW+6n-ewXl$!P5ZkvIy#{SdHkzXyXJA*D^C)==_cM$#gt>xL^OlF8Q9?~6j7(kPmLb` zdlBD6qd@_okh~PTo6hB_tmZF@JD5j;=@}og*0wAiw1!!p7dGmH{%}Z^V8sBrtRnas z@W=G}uUlw>6YQL~6T+o^|4=8~*87G3eOofJ=VOZ0g1Rq(^VAgKG;WGB&cXSLAfK%d zctc8^%mw6M%V-+Xm9|*^{JS7qDG7jd>)mVJmcBY{yHfM)vb17(oH9~+E1GeqDBZL^ z1~LAzK0<&C0%4qub?WY7)u}ADxvqkN__ujLiW~=tz)TpgAMk#97h>@a3ZDPkLe&_qcU#<(e zaL#$&_jBK`J4I~@F?iyI47J>S(2Wd(I@a**@=*snlqRqztv!Jga1Idm&Cr9wUVR_~ z{kYFxVi^!qK6~zTdHL<-qv(X40nk z-`%QYSPNo|#iD1?E870-pRy$oChMFa)77#SD`)`<@Ac+_x z(;`8k_!}7+?f}N0Hq^0kL<^@m=qx8r=szryP;oZK1AJ+XYe!L-p#16o!n_4l*TYh_ zpb;CRV(KqO_P;at371i1pveUn2U}Xgw=qP61^ZA|9?r5p+DG+E&84ZU{t+&;9nas? zmocTge{j>H^Bm(LAB>QW>HClg=dP$%`Hn9d8GLw{?h%pI`9X?3 z{V_{#GG}-P$cBwy6>Y~1Rb~)uGjh2cV{Xj#Rzz&vXHy0p_JX9G7vA~NlqiE5T zFHXjYZNBNQ>F(nDophXl+!};%O+CJm|HbEQzQMySy1pVb3am!9`I*aw%7u_0Ez88Y zG`^t&P^bhBjJ&0#sn2Uq-1V8W61gq4_O5+rC4%SD|(4~i~>&M zJ4W7*!E+4sW&Wdg1&ZjWujNJsmIiyk=#{i*2{g6%=OtUO{qqe*3X-NZla;2O5ILUNN^dfJ-;FOU1vmV`F+K#WSWmwdD%fu2w{ND{k z78fC+9$)00FEA!wZ}dKzx{tsE6_q*E)KpPCSE@&U#1?!jglF*LOnxADsk*lsp4H%- zli*I7q^S06f5p>Y5t!!~?W5E-#m9D4y%T|BmWTeulIJc;#vG4b1%#A&UJ z{Nm8dCnA-+VG{J@%iGNEw?C7k6$J_TTyTbBgtH~WzM4F_ANaY`-p&FnwOwtXtNt76 zEzmz3^`*sL(*QZ({eqZz0Qz&dflT=H9%wFWDL5h280#bAGQ(PRzIMg1DEFnxa@s%z zn$o*XD__;z@>~_0ORKH2@EWSD+)#4a3u*stpG9>JjL+}n-ZV;Hf%MK6lg5lAocBBN zPI*dMlNLsu$+suO$Lrhi3SN3T<|ZI%96%H+3RDbVUYSXV8$<20>vuOWuT?rs4sh2y z6l;G*=0rs*4Ax>mI}`Yd2~V09wR7-eWlwWTn_Fs|YK^X`de!!%*Q2Nk#Gyn_+3mJ} z9Y9#WhoskZHM~I*>;)5rCP%Mg2B3lYVc&Lbo-ZzD`TU$mGRlySpst1UQx_u6rUUEL z0bcMYx6p?$Z!g>Z?vX@75XZ9ghps-g59}atSj^(HB7rzd0(KJlmn6{|gFVG@z9Q|L z6KIgRoy0Gg25o>^^x;b6@oVxnG3qJFHXYAKV&c5hwRcJRKxfT`7ouN5r`|7|Hx8aLG(LJsnV+`wR4{1`P(aK5$Na5q z3Y-VG+e+ybUcYf9e|mX1G9*V;X!qLkJ2h4FtD^+__-KHnc>`lpko(fH_PCJDA6UGV zPmvPoH9P9{~1iR8dN5Rnm=3_8QF5^8CT?39UTFYhJon9TIypSw|%|_JX%!? zqj*Xpl)VV}`&U?E5QR!uIP1VQQ$;N@M}vDmPsI3n>!F9%OebR)`LFj%lcz^+d6e(m z8D*NnwIw z#ej1jSr5b38h3osZvNoF=eR>YEv#Cpc$tXGYC-EWopu0VioEgiitjb)t^fb#jcRSU z?fl)_zaAvXym7J(L_7xo{pKAF(R9yrJ3PR5nc*T}S-`FhO>bE8qokQQ5dgvb789-zeOj?{0oa zmxI-COA@$gOLoRafe)DeF4C|UGeNZ=^|%8ICH$Qoo2o@}ke73ZgHv|vC+&6~!N(vE z*BW?%Uw_5G(yh(DzTQvzB4!8+rxlK(z;1m4`V$q&mK^w!1yTJ+qwVv?O92t$UlJ`n z2lmaMuqdA{?qIQqh#syJMZA#@{pLzM!MCF;mzS4Qxcp6j-PQx+{wPx9;4c+EvoJCn z9!`a_v(6?7rNU+x6afeiV8i(X*_$9Uis++A2^gL2``i7^!9=Jo82Aa z3AyfxvEkdhP=O|vk!KSL3dRO>&;y&YDi8h>$uAZ1R^Hi6QwN=5q%sLykDU7g)p7O_=m0Pq#}^VvShJHLL%~+M_*um zR700RmN~?|OC6YyRBv%@ccrG8#J4 z)FPc#;PhLHH3}AIx;6oI&$EE{oG)5F1uu*BGf$A&5bJ>&f#}xn=K)`Fx5gdNFduiE z_5DmXVjF&sSL#{QDB!(Ei!RW+F0xFC&_?QV-pvm^xWcIy6LX&*!8k984`s&VXLA4mQnGPGAX>TYpKLKZCL$Y zzyk8>dFdD8cyLst&T3C18z|nX^YgLkEPJ|~UI(~74vf!U72GO!p*Qs(Tev)uf;g6* z04lBMcBUyOA5CEqhBpc#pQnHouDlY+2*)FLrYOqVJ!Y7{{ghz~At$#`vjY_|3dMMJ znuMHOmIG?QwqV5$+E+KHEEV=i|Mob&iJi{}LKx(IXrlnQ?75OVUv#IB7@}gC&nPQsGxT+@$)~2r7_o3bd4_Q5WZU-Kt~lJlkw21G>}tCvYKlJ4m2LHn zBpcS~Z0yH)a&|@ctJF*$3%A$uPfH=u5y56&gxyglrN~L4Xo##*vC&I@onwjPCk_LZ z(qhlvJ&?(5vs++%Pu_E~NT34_2TeoQA#2iukkJ^FM8QT54yA4O;-t*7t~B-7PaOG# z$l~1a1^;N-B>9{!Vg{0;TbOp+cIW(Rj=#h{B)ERq1OLIjz7cqN&I$Dy@nK?iB04({ zfwtX3GN#7R&UCoyZ5ivO4$2~VO7-Pu2RiF^_dC-TK`F{)dMhDDxL`-#m~Pm54(k(8 zTYL;$O^2=g@naxT8RN~{CR*gn-`w3&>j-95AZ7I<6DP6SSnA;Wr};lU#CPZgL{?|} z+D8$=KMLyuf###{^F-5RPBf`5zCUk4*L4mjz_NOfHIH~om6_vcZJB2l4{e{)>aBLC#fO&(#HVL*4bHx+i(JlKjEh+o%8DiNBTcu zCKCDcHuN}e=X7(H`be%%+N{DPnv?4OlZwm8s8pR?&S&79rEUXyhH zp>!nCOF1xW$mQ{CsVi82##}uX!mnA}+dLmI4B)oJvQC!}hM$}DchN_0vXR0%FJ{mM z2S1uYAezaj!KEhrxN+P+diYM{Hcveq`jH1Hy`sO>hA z67(RP68@eFl40PAUv|%Is3Z#V9UnA^mqMC3kvrNSoj_ow1LAxoSP&`uNY<~te8Zo9 zi69*RBQL~rgh%~S#DFjg#$`ppfbd>^pP}+HZiQ@mycr#gkrh?nzBqO#1Z5Pd#z2ju zE*YO@kNwTcc01PDFB>A4YH}A&1m#3Knnx_Ni4|o?8Hx|S1u5Fwvz(!o?-n{Kpnxw^I%hw;Lw@C3#4S!^XF3!TMt z-k&DVTrieZv1-8jGd(JGX7xL=40a_TRSL3}79_0X;e+Y|{cmGOMt&{#rcx16*cbpd9U5xIZfc^?uN=UHR1bpcye}Do}NSvBvL&rxA`XXBywBS zl+W(Z>PzP^t2L(MR9>4hC7FK!Myfxf=0&)$6D`7VUcZUQP_37sFJAX?fGk7U-97w_ z*nxx$^62_1d|s&EnF39y2mVKUTc%nt2<=#N-6?@?aLw@;ndhc#R?rD!-)KsmZ2r`+ zi0SQUu{NMp>dWv~IhlJd61hA=lknSPl6%#bE1#J*k4rNJt=)?AB|(!N1zjEPc%@JI z1D)Q3Ye#M zak0U_r+bJ_DV|*pW$ktF?kjT8Q~v7u=$1p!J2E-?G4kKfg%*zv3A8sbe%yZ9WY9^3 zD;BLUqw{lZ2gm{6>p%Z#_QFzJCuE9dGZ{09FoHN6%ZRcxoETYftSM|XW2*7O-Y9ZX zS=juD;_}bSumFea+{+1K>1R*g#SY;beUdUBdyh}{P~&L{f{2k}E_p>}1!>@SE=;*R zA|l)rvtMg%lSwFV&+WR!KmYUV&^-A%)2?b>pp%zwPr)EMw}v7Ab+X6``PP{?KDFI@ zxn`Xid6X^{Y%qJ>>S`L`2vbnGn7tVv*`xBYxxF}ga5Razpuq)Gr{tUfPKL9yGp{VV zFD+cY@^5Zd&;E%b-jBc(!gxxDf^dppX$O1bX!uooS&NCvdzjd`{W4CQ9-5m5(6=^c z3rUWS;^@k)GdJgHR9!nO1s_1SC)~uxZ+~!5APP^x0hLW=SJm!b-QM2Ly$mgqjo&<>1GPBI&HNRb$_d8g zc;=PC`=5bwj(5N9U3a=X){^EIV!a0>T>p9)x&o;s)6lE!a_f7;4YcC^_$g12!r%W8 z(s7V9oPnP_0sRpJ&9JYM3Y99i8`$M>*YV~_NtqI$ji=mctYC|SjyihwS(=|S>M=uTI?zlpSlGx~tqz^jpzd2A7fyIx#z z%IC`L?Zhs31_y?AVXr%DewgMFc-`VZd*6zJZ~9AK!sH52*cxR=Ir+ce8KA@VZ5FX$ za^??8gs=vwyi*~o4W&UCneR9{3ugtKRd3GsvsK@0lVH}bZ&}S;}fKCV6d|bHrT5ASa@TN^?nPNT~G7M|``dNSSsmq0@_Yp?Ar@8*z#g>uoZW zT&0XBH8+L_8fM@U5HzWt4i5yN(ZLybA~Hr!Z}bx1F?T*LfE4jAR4niVha62AGEg!B z$vr%9cC7cJFed@qU}o*+@VoSM9m$WYd{Uvo*MCP@f7xfH`Bye~-g$gh_4)PsV~8Xl zPwCRk$u7TJJcA`oq_;5;cYZ%CWH&uBqQ=vo8lk||gpM+M_FSZ7_k~H_q2Kj)R-pg3-R=Iju+52CZ>QzO0I*bF9*?-ZxWAE_}tm-(fwiA!v$_gn=}n;kRy)lt zyAsNKsFk9BuCM(*b~E*+;2Lcb6MnJPX3l@o-ws?m>6CRLj{=7G$}nTPq^v9g)OT|R z839kL#;%E_qwj%$>g*;cV3PNvm$rBv^&!BGr=V`HRR2BRqA>PjLZ^5WQHidj< z1}w3(UGR-W_ccpWk)+W*kgFOv9vKhM@TP6tc=J4*KD;QL`ouDxCW1BmmjY3D7*D^* zf0x>PxU1Z+dwLI2GH8(=V_*)82UR%t+HodklHk_FNLHOb$hJ~hywXIegiqxpy5Vo# zJJb3gZl$e)Rq|fP^+7y1$bFr^8J!dB3U+P9<=7yEqn=Ne(8-R(t|6Vy0I4g@8lJd? z<6P?O*tI3P&nAhEXYGw-z%6>;-E41f5Dax5+6PpSaz3^c zg2H|bUvjz`Zk+tjW>_!6n9sP`7e3W~;0jC@nBteP-lcbyY3UV_&V%E9ZRn0EJgbV7 zf%%Nh6#v!P4;b3L>pX4k-R6h+&pgL;kg2PuZ-}%ILoI=|--^{(s?ehmVP8&ZKZi$5 z{TED}prbUO*#ROXk=eyMv}Q(MNsvtCJ$Tzg{v#}Zkh4(da~gDi^RgPP7AxJYMg%1% zS=tt!1t~~h_34uI_LaYtBfJ;S`li69$~XYaLkyLP?1uOK6bicZxjP;XjC;mbCP5XL zda)J~8Wjbxc&|hh1z!j&+-xn|*6g*E(M7 zdn}+*nh^%#aPE-usv5@C_e-igOA95bP7g&|m`&~n%~-}D^27z~9)#TP9I$gF@NfLN zb$ILO9nyJp;m6qOyTE-luTW0P3i=XhzkL6Xh;ZDZP9JUcXHY>buh*SsvU}9H1D`df zWgi20|5|YaE$wgQD4y^4kc85AC0Vyu(1Wx<9~S-|%`Gi2#_s`!D$TmO*EY5esywN( zB7S%#$DQn0g3dedM}j*d3{pTd)j&`@_<6}|J$K0*;qT^7_|GO3F>(m|tU&C@AaGvG zYoNF(I!OKtuBR5St+(W~1Mj#jidLobi}2)h-xg6)*f*in$=!210DzsnO6 z(SMZ0lK20&YOnEKZ1Lz?#sw={z4R8jYr1&B{C&EL@9Ik`x@s zi#{`v&)%x9-B$yhIMpr9RNsc#Lp;B|H8~GvUN_zyu~c@ksrlJnJ2~^oZtTUeH$Dkj zcuWE$xr&LpS6rXw9e5B|B1i{Y zYnAxpNk>^(MQ&wZl*skRj6eRaz-S91a(w>C)fk~MMfk)9{Y+eC8Kds-)H)E=sW#PT zXWEK6uM0iWAS9hs{dc;uCUr9CjI^fQLfR$_MUCLO`0kJ#ZDAog+(`A^?fDlM;@$M)1x5kX-@netd3@%Fl- zD30~X*PRD*jwSZk=1K!R5W<5n&2MjcpvOb+V-MHrnUfpc;$L!iEF%l8FuyPnk&dpm z&waeDxn=;-!e?!Y+Xp62Lr28AD4=*SUvHSKvnt54v&rcQLU~HRb#+k_F_$~o?^Qn1+6MtUYlXW&;xl9VjAu=9_kxg^ zJkNRyZf|aea`zY|HraUifV~QQbHOMc2Ih(#n>Fl4+(f_bibg zG?qCHij8rm`M7JwU!LMIxQEnqla$|_Dr=C!2qoaY{zdGr%V{fcG~7(^b1CB{U7EFd zB5VAmT>GvZlC^S}bO7=4UPHPDn@5R!`;QE5H9}%gUjHB8YFrm=1Kn4hI+-UJUUjA1 znNtS}8>~_O?}pU2@uMU}eNl^0o250)yMrEK2|QUHa5b(q=&bj9LWKCbx^>SN^bmT< zSzBJ6Q4ls&z0b|7^d7`zJZqiqKz}lVMsR?9nEfAppoD)(qe(KSWFr2K*QR_OX zL+Twd*lb6-!QyLegj#7>X7kdG`2;)v@aIUK#8eW>?STbn65pAmjTOOJ`6x36MFe57 z%VN^8kUHx9Alfkd*h4g$XCTKDLOr|D^t`6^-nZge-+`&X$DP*UI$)VM_)PEnAPN;< z$v=@piB`EkheL^Glu~-X5D;q%S?P9NO9x`rv+uu$xaZk!>#J0r-Jb1Z()moqje_H1 zk)RvaEtj*mfd;!ZKAGjz)H9PR_NdQvG0FIfdT(aE7hS+D&!+x1JsVziW(30VW{*TA z+zYWu;o$+#NdHL#fR887RS%J>5-kHGBQB{`W2>TW4iD@35a6r3Bqn<1Xhnz!Y|sfQ@}nqh7FtsQM|%) zef(R*NUt=KdJ9g-Wi_d9I_YGX5aYnFVoJLVT6?j8HLz zSWE};J9Cjlzia2;9Pp#Tuw)*e`UVMfuxHWWwwFqgUd<;58?B<0><^K2aZG<{U!C!@ z>p_S(x_VcD5#a{uOF2S#R9RgJb0`WwhDwU_ibxAnH}YFJ7P5h)Nw&bUCfNP?#>nu^ zvR=ICHLX#`cK4- z-ph;W;};H$ig^_%SH?pOs+hzDcR9CSuSu=}Zo~}gkk-3LkNf8#zwHGFic<^D&)?c& zlAMO+gn|YWvzSv)1n|P{yE(aiYBvyUT>24?qygeqL0{jN*=If6L(R4a{(eAmI$TWl z2<)a`N5`*d6BC=^FRBp_Woh@uqmS^hpN6LF)Gw(aXfwpj7(^m1732H1YG~Q71^K9q zviIr%M`hRTSzYMuiRn@&s#t66s|q(k$y!-T%FRRG%oXuZIoyKP&0-5!Ox3_zy(Cj} zT83=AyTQXpHH?30gKwghY!dSie+5XrZ>zjUt`UwK^4u|(q?r07nNnB1Xx*E zPE2cTowIm%pV+GbWfh)b4>*5Pv&2rJOx*>3hDMO^g23Nt=z3_&hd z^+JbMKdhoOHD}YhCubukV#=*_E$1UeL%K-T3QXYob`H0Y)mHNqbZJ^z4eF*zGzF~F zoBoNgtjwwkZ$bA#vHrH!jm}t#a*TT)sT!sJ9(3R~!vG_b+TLW{Wggonnvpz>3e#63)t? z@em~QsYf(Jte^u*$i~7-?Zq$8PuRS+!d$@?6e{j^=(K;h>Sjj;O+lWhjSN-`qPpE$+}Uu~Eg;+En2w=v18b~Rr3aYKm!DF<2N}v1V6Fxy8AEBJ z_2sR(tCSQ7Q!*mCeZgVj5%IwhnFh$|zP_lwCj}8QJ+*9HYEFbT&ECeX$Dql})NwBj zo*%vVG%SD2>?p_vk|J8iU<##2ZC3<`;-^)4`-F-F2f8{gHLPuPTP&(@Id`1mmlxYD zT1TJERCqX<4aEu=#nbAEyc1#uzGb~hM;GFGfP17|1U(DMs)#I1U#}I~9Hq~`7m@5` zNMX|gJJGO4fdUSf4Hu12%gfA31Ff#hw-*CacPDrM(oJu#hT9XJylI8_@EglNr!T9~ zSn~X!(IZd6X`csEvVKD?TKt0(KRXM5Qs^G<_cezl3L?xGO2I-8$C8GIsdwg&6pIp& zZFKTkD#_b%JZ;C!zR?EY6Th*z$?UL8#V!1}r9gWo3sP+iAdUftg}^{LRG^mNW8oOT zzKZLJO`ncW4R{%@RXwXokA70mFv3fVFlE=BRKFV3fUqD2U~+hUeQRQ*Bllv@fi=}y zQc#e}BfjTp0vj6@5-Gt1^xnGZIxPEAzICl(HH>Fqr^XZPr!{<>o#L~D z5TE|TJ{$G+Bs7Bg8%r(p-*7)@>R?70LjEETFx$gFd7 z!sG;Yw-;;b{BI(Zo+nLpm4C-_l6SPVg?Lk2rK3;GEkRW&Jtb8623zm!BU zK{r1+Ewbseogmv@ne$V?xO#oFAutc!ayKu7%KXor zp6nXyHCJ;+6#)D#*~7G%?YubTIv;WHN*wZyS_V9ZK~~m*)rvff7o}*UpA0AqOUuky zfB=4Z4xHM6f9TWlu78)~p|wWjtfd+9Bf$Z&c3}knruj0uv@CBqw@~RQqT0}ZYs}0^ zMsZObJka`2$1E|+?FC*pJnLzNh(1tli7MvmyRiLcbjOLw_@T&`KUus-_9Z?v` zEKM%hs(*yc-A7gWB+(h{E|OGlF1RiG>iqlb3SfRM)DY`?_iZdTkUt$Ad93*`;6R5T zf$g;L&EJNOs0vPtH))NGA}F2R!tCK83i5*B|B50iw(9e$B> zJC_?!7CL1jE6Su7n&Db_J1}R|wTvBsK5If7IVRx>-%_%U0GEaOFPcm?Sv|Y~P|{0k zd@(7S%LUA|d+#1>;*Br+juaX72(aY_;F=p#N84UmX+#!RQg@g;9Pnqad6A-kY;=!Xp-H=L z#qr(kZlvuC?4g-)|mmtHC^S^)-ShK_X))OA!G~Cli9kTD_?_w zqq?K|C~ZoBCaxnbAbYd(nQFX~nMDI39%R}DRaHM>#B;YC^5f?fru*`H<|FiWvz72#14bCJaxJ?HDXd!y7DikuXqKo@uObOPnma-lv(j@#pN) zv~C2%9iNMAr02a&H$w`9HtedfoG(d=xh;8_e@rH0C(&4oQ1^!ewhZ+eO3I`R#h3O= zy+|3QN3C1)-lJiiFJ2RtmXn9g98DFhy%;Ytra(;Mn^ZT>A6=RTCUok7PB{iTNm zMM1z+mG<>OB0B(8hORzy;8Lq1G!De-$w~K3c^Pi#U=5dWLec4z>7044fnQru#eZYn z%~>l@sHS*i*g^oQ_M@zv;uU(k;bx}< z>j_dEITCk0Hw1WnclUh?VN`yu6n&sYJqxq-Y27;;p{P`=IOD`o~qgf9C^QSQ(#Q8pcY;4A9IC350en`Dy^-`M$Yr!(>Wi zDVu}T)?$RwMcHirZI5v~@JNyZ^tH>A&Dq)6a?`lb#M+6YkC*4?6mn|PlS7@@DhXoQ zKAv*C`y%f?8Iy?gC(6vSiZGB9AF2=Lg(R}Dc_>fSbL$EQq7VEn4tI*SWCxZyz-{CC zxUcxeGzh}Xy4lY%(`%ZpJIjqlWO2xZ)7K=nr^>&NX|E*(f*bI~4T44daQ7igE-&xJ zhvUV}R!R|B2eMdz6(I2m6Cw~$YQsFT(y>Drx*$9n5COFu=sqEmJ(DVzUIlE2EHDuQ zC+D?aZmtk>LBtEZ3bmq)E2;5%+A~u~`oxMS!yv}rcu3y=E}r=2&_T$YBy;w-^%sQZqEI>BwSjvP5FcPv-I>nX>E+a8`YnV;n(hqR3iP^Y_uQ$T_@DmRlLMa2 zXk!tIs-pe*Ry+_LQpxcHi5Ynnx&ugR+ne+STdvRiBefk6cT#Rw;^_#5zyPb*qXP5O zorCII%+LZosjGIis1<`AJ)B=TMj?_XB&t732qR(KfCmY3EN-5F=a*X=AA_55m3O_^STQ83C z@Ae6ZLCjKjsUCul}?@L;bYdsOP4qPZH0{QQkLHj2noE@MNiarQ3z8$`R8o;AhZI3+_} zPUlk|3yoXwjA8K2Qwxw%6pzM?0T%SP6jA?Z2x~M9#L~-x%KSUkJ{sel1H*`+wFvy;amD`4W!PC+ zbAb0p|GaPuvk~|}vPN@EhOAoXzmu0qzpeWrviEX4lz0Gd@{1#B6T#L&k^hFNV6hMN z?y<%_p%*cc;uwr6AHEEd$mR1l`QE_6Cm%8!up%fJw3HQ!vmU@)kWIlgW*beeUA@S> zmT@?o@A7x(FQHQFOb0Fb->uIrii$7t#y7bbyGMWgJK!eCvWO42TlSSLh_SJm11a6&xhkVV)v6!%gqQm>A6V%P{*ZU+?`t%U~sPU@mZAks^DK|95kIX5i~S$tbOw zDtu2ho-31&{VW=+gv*prel~hC)qs>3LWHi$Qqiq=^V=6!z4j0Ua+xwLsBfJbPKIE^k~^^$h; zwTX5M=)@)sSYnuO=3L)u(oweL=L;_P=8V7h-6csr zUSwxBL!?~p0{=Xzy53e&1b*Pi{4;5A*u9D<9)9|YfQPXxUXNZ8V649V&|rTqU$^Kd<#OL{W| z?MQGSr;eaG4F~?K?Ph&g@1>CXaF9RhtyGTYX4~8E;+s4M@!J!k3WS5%D>SC_jvRn;ry&~GK`9(5E&y%h%&s1N}w>W>_uB;z{$0+rs zxE*MhUSB!v*#|e}n>Fp+a+g73k%@3=nUQLH63qhF|1{d1<<}j7N;BMezEtSd>9&_> z?3~t@_D2ochK{SmtJfxhU9G4l6PyHe(|U*7RmMU`?qDR zW<)aYTne_+8591Dl}(_vJNikgc22VBMFVZjm3)WtBYP%sDp|ATB{2hwD4`Ucs|^1@ zU5Vc>2ZY2S-$ys6SAKtHZy)J#VvCy+&mAk+$X~;>X1DSkeOa=yB2>HNWow=_@p0A@ z!ZRe~6o@8%7-liTO*NX)J8AWLRt2L$yrBMkYK*&K@sU>i!7 zOh%evtwlVtS)Sr6uJwxF6Q!>NWrP(fu{_rP6Yl=FA^Tp|V?W7kchNaEAu?v^p6z-E zJ7k&CpolKuk|mDmf7m_E?(NiO4O$6P-vcoiui3azitT+4iOyt639M&g5%Jqel(A#{ zHlWNpkkio(ENNSDkHf}>w*f!U2@t(I$%Gg>0^LZvmqEANFT1X{%k%#~aGm?s`QQ0D zx1_d(pudv9ydm`3;hO(XqMwCAX~6x9A^Mb3zOq*BoVO{fqGlU{lV;@vl+SY{ zGJidF8ZyryT+y-kTr;zj-8fIt{;Aq}xus@rf4H?S(X3w_Fnt2XQd!V_2B3WH2_Qym z!pi(p3>`mvn2nyk!2z*~{JZnNKw6~vph|+U?u_Tp#9B2_jBGqR;A3jUhsyWJ)5#h` zlra|G_kxBOA_+jl@OL?5vr|D?$MBKOrkvwgo~Xk*0AQitv2;AIi zE#moxf9C(w=iP#_;$1}GWh(vrxz_I5+)B+ECbQc4zfx5|ajpaOW)HeI=(Z}#!N0w` z<=)2FB%I-Or*kt}yW*3mx(-X8Z8#}peuTm1TTU%fspNo3ElMpM3_v-zzlb~4wDQ)D z?-As*784vsae7L$XTD1&_$5*lzR=>_>@?Y-v&Tka1w=x!Sj3kzArVzs)fDlk<|fOD z{zX#RzUAX$F@TVDyVpnSNz3J4GYzy?{VA;UPIKenDx&?Mbn;xk+Rzn+9i+z_eItWk zmdy3=Aq40H*$SxO@qQp}Vt1C|GvTpL?M>6cL(aklt4Mg<7b4mehc zPWwE8Gh^JyZlRfuw!)I@pxJMayC)2U_lYbtc=AYOb*a{`uG-9u9mzx-7tKPvbI+Oo z@#j~|5m^Pdb|F~ji*VmiQ;pljSsXXE;}~D}#fSkyWt;6rXw=_^fNDR~PQa+xu?Wdw;$&Ph+R7n99het-p z&B`G_WvEEr@Um2kAe$dqhOgyxzkSNSdj=nXSb$mkWMDyAu*#6vAXHb^=R{{s$A*LJ zmQRMCjWi$)_~U{sG4`TAXj|JQq!PU`=8xxJD^)wx;2Ho)06=3cGaESzoWEtw86Or> z=Q^-mR^bti`nA!bLqUx)vB7gjy>H^38QDlB7YtFdF*yCY>g7~c)Jk?Z#Hy9^S#nja`MsAa)0yhaC79dc03*!$S1|dO!`@weg`YbSosvk;x51@uhg4&)cTZz zUrJ_$WK^Sg*i5hV=N`yGLi_$ihY(2}6h^34zY!MxgjglTV!DwtzqMWV<)0S7_4vBenPo5r%ta*(0hKmODDQk@p>gCOO^*_h z_8(%D_tcUaA!AM3HgL9zMX*Z;_W#RT*aVUhSj7mtuw_vc|Qx+ql_;E|a3} zL!O%l@-rVbrl^LNWS^M5siG=26<_XL(kOq=8gz3EtSRJ6nsy%Q?zXo-Wm*H=uIYeS zvD#WkEqL-cz4>%W?xM4Y7$l@KM8rR7b^J%W4a-47In+UG$46v*uwfPy#~m>vEb1m`Pihthfq-Xi@Aw5P91)LMEL#Pw;hdZ1jE+vf?yVd&EF*wxJdMyWc#| zmgPLZsEHV4l1>`qv#b0ZFhyc~L~|W#k_K*k?l!mqda-HUYXZ-cH~X?-o*F!EH7_=i z$S~frdkW#P$(eOiyC=WM-Pb*@g-uEVqwlfOfVdN z!UkM*1Z|_tW#vrmsgqf!oy^8oYW|W>o^0l-AA@WtJ>*e`&)A4~t9&0t z&oJbMsbZ4S#AzV<%F^=?k?fS=@rM0KWygYV!xR|Ox^4g!`;n*`L#y-C;LR{Pu5w&o zA5V0~|8aEQfmHwh7r*vR#=WS>zKX8MxO_q~uRXKUwP$4Sac$y~jJmk?$kw&8qKir> zD_i!Samn7l_xE@I_s6~7@7L=&&UqX*?0`t8ceP;S7qw$kI&G2ia_&ue-Mtsp$2;}t zcv8&K<$c=7YQC@~rjI^m!GZpvp|VNJ2AHl;ah+s+YI_qG!YGHozb2_Jd>d#kfF+Ba z{T-dAAX6W0wbg+CzU!&uBjGp0?8QAiH6XGyI-EIMhnqSi&vqFn(6=`12n)%=)h>xH zQwm`2AMk->vfM>xDLC#0F*tboOFm^7gtR8~_ob>4Hw=yinFT#j)0U4-H?9r!wec%v8%k4D!Uo8kfX0HcaVZ za4@*bz10@6(LnMVG-Iqdg2Q}>lIzI-+1CTJ6(%zraLtmx(t5xRc}HUWOKjkombw|d zdqpbLKlqe*eR)mI-Kzm0j3R|D341}Ip?=BDoS2u9p&`M^JZWZz3+*4#$6sUwiw<^b zEuW1_YWe#!zM)2xU=`yDekmfsEs=@W4n7u3h}?~d*wflN_S935yBt`)#c`{**Rlj2 z^EVIJ#&cBq{W}z4ca1Pg!q60I81fwdw?Sw$y5yryPU1~_qBVT5_gyC;ONL{rb#$?+ z;NffBaMr%22scb0w+0wy-BxWnMW5u^sUw_~@k?eAS1XBj1udY_nQu5e#zO_YaE&Pc& z;6?FpEeJH6j`7UlASMP^l!vKqU`-zp_q+F3^PorZ)LgP(w5etEBU$K_c|oA8ff5L* zXLU``e|LV24=IuVJPIDykj)07Z}op+D$V`sNFWtes)l!AW2jeJ1DQGLb8|#)a*%Z# zyaqEe33=8MhFXT$cdoP;8z4Kw#cU#&!Q3cn0vFvQIVZf2&kd(9gDyF#0Wv?9q0;&F zM!gO{uV~Z%49eckAqO;sfqPD!%ByrcQ#pEZ!(s!UvH@K9k>1nCz_I{{kFa~OGrYaW z9B{Y2VSKxWpykzMnkGiI`k;7iKURWNcia9hP84+bs2QlC#qP=J-P7N4NO|DC`;At_ z`WM@Kk@xDCbfhhpijNK77TU)bN2N0BW0|?heP}$&uv6 zJdarE-^YCS-A?-aq^qMWST*{0j9sf&cNbrYt8+L=$ATYZfB6HHQ zk|X;%Jgvv>gr@eDE6&X}w6i^k8D3^?h3hr-+XPa_7{ zUos{4Y%B`AnN;5(j5fPX1#+ev&MqZxHn=8E5mF9<_TX!bVSHi<-(1EHTXG}Q2_gRZ zM3#78a>p>D??0yCD)E6&k6jYcHx`~e6Zj7*a%=0=zIR6P4-ERfA}mKX+k7Tl8`@6u{OxWE%%ek%K;5y&) ziw*jMiHz2^8KrTzQ2*0yHw(QioqhOPfH@!t(96R85nR9EQ$Tvzr6*XNw*)@zUz0Dp zs_`ARl%oB|B8O%ezo>akaXmP_^=F`*^i4O(=Z7GOV!Wxk8vM(bFTyIK3KPML%k2qu zs#_poJq5H1QpO?s>O)hpwAD$Y5C)$SbM~7J{MwOig_cQ29O?pSZ!j=4@XCm|onb8i*~N z#POcx^AsoyhG#vh#4t+~j~GupkR}ekTtFnD&`v2V>3`%vEgHO#NT)lP2dzYWy-P6B zMT1Q*&kj|L1?J&muYqS&x{rJA+hvGQk>=WK*3r1Qe-G;A`A89B?7X1Z4r6dddOEd= z&3_5mq#$va;wFZ)&ZEPQ4sN5KTCm(JBJzOkrVvQckX&Au(neRAo1BiPnUyEp10*W9 z`Ri+SfHNmajg-BpyAJ3{lRnL^*;}6Z<)^Gg%n3s`K_F_dB^}DIoz6jSCUmCMW?xR6 z1OmCd!ngP8(t^N*;=FV??V1}8-wnx_>GG3jzy)lhT@4cEaB)nm*A`hNxBpsq854R& ze|;)@4ZweaQs*)pQ{uIAtc6s6R>cHcocLBNfa&_#PSnE$v3-U7X5G}wK)9#qp$+f7TSXXu({v}E*zj#V zjd+o0CNDg{7EqZOr&5qH=5I|Ij516jkU~+r&$J{>s}(xf4(^W@f#kddzG_zY}irRc1tIxFbAQ7#OfC+5~EH z&v1FoXGOq0oUMs&Tjws~?q6rFZW9(SCC2<*uAhORezT@T`bLx0#YK-!32Y8{Ae9spzkqqX{9F?sbz{?m(( zKb~tN^)sa3L+H9hzWr__?EDZ^OqYaASzRr8nsOl(N~T~1KOy92YUqZ|gp|ZfTIreZ z=hC?tQtM~i0P%SSqdFIbF-|O{_Bzlj&{Jg!nEi+TG#f~~JWv%Db%09?C zAcQU_=oP|dIBSVXM{==NAXldCqATeeVS`rGUl58@%&yOzfmmf4C{`DWoA=r0VJc!B86CBX1dt%m035VJ3WNVMie*J+r;8%4`zNKfK^Y zPiHAR087=d5lcLo&wvmGg_r_^@F?AoejJR+23LIgJ%&DQWtrq===s*bfPfe@mixH4 z_(X;(o`m$x=c;Gq%SPKeve#9^|440%n0QUyj?WuS?oCy-AVdwH*8fo}Ir;FuXe0dW z4fsEfhjJ&Z+m33J4~u0Bambf6o^QJGATpHOb+fq$U2yn#zfh+@lluT-{EU@o1AxzN;&{ z`MC*j48l)O#cqPf%AKbk;m2>UN()N7jwO9ZpT^G{^9K&)u0>Z>Z024a+{txLsK$ib zn5GUNYef!^_tB=EF-ix#TR{_hq-|9~dZOERhFxMx+TuIh-?8F?Cmc_0`=Y0ha~m6v-P$g<-PqM0*E#@F>VdUfnd_aB>nqvo zE?jrg@E2WuQu=Z2cq|#Lo>_pGv~G4!T8MNAT}ANTViwclw);yhc_=NfeK*o>te1_{ zL|x+F-y|BX$Qw9eNAlPQU&Zq7*UB+L!Ip)Tcaumc6f6}CY-r|K>AWZBFjm1=wXmS$ zc}g}{nXb6uBoevbe`CJQgiAG%JJ_N~Gh}>rWx{3D?zFz+1%olBiRtT4_?Ly2O6Lb^ z2BC?GYiu@IYa8A<7FB8g|2Xf-UM;zW?(YEsjTUSB41YpSPY14#L(fV1dmPWYLQl5O zcrhpb($Xr1hPg-1IjY+YsoFC8!DopICS)uUK7t{)XPQ58%8xj3;J*#DYL1S4Jnr{k z{@$FKafj4V=j;`!F0{-CEp0@%JR!oiOv1LOh@m9kq(GtEOmQ{GkMd^IL_VZ{cL`s9+ik{PE;yD1*8gZG*5`L}`I`n-wD*89l z#0l-lwpg3>E4_j0;1S`m9GL0%1J9mA@z&sto} zNp0OtaTuC#G+)8~p*oVO^YbfyZ8l(FKJbYegs)%oJc-rfS*(fLB_z<_pN}wyKC`|B z3WI#Zrmm5z`Vdgf0Qts2b2}z8=Kue>+6cYY0i=lM{~D|CPTF2^b+a765rrhbFZQNJ zsiB?8SgdaUK50iW-nW!m?w$QFMpFBNjHa#!`oFPUDmH&zGNMs5V1Y2w0}#V+m^!%{ zZ>&Ipauiq&GYEmi{^WBMQO=VwQX3FDAC-LrWBIK5GU?Hmkp#vIMGvZFS9d1YkCg1j zB%{$5k)#=A!EXQPGa5)hqIbDJ8;WtZtr2zIB#e)*4iC&vgsxAfCXWCNo_VE)ZD8p6 z%HGKUT%Vd1+)xR%&IzlX8;f%e+lo|%#kZh)p zPY^&CyAxXjI1_-JJ10X!@*ToMDFqeP!t&`KrzR@pH%Cwd0g}a&lP&67_ybg&YctV= zIO=D?9|7u})_4qQXEq$=36uv}qIi8^*k)_hr+2VJzyAjK&9&^8{RnNwl3Evjzre7x z1)VNd2twbxqMgo-oVNEpxvBT1y?{v)kZ9er^6#Bj<=s%@<#!(l_5AMwB!uYpOgI<` zx805|OKxw^Vzh?6WpdHa6I;FdO~F<|PR8Q|Y;YCfp_R6yR4(a-*yY8((3DN~YLd-A zTnGH+wLk*rAGNPM0jDogNkUTQ&X&T1t}5EhLXY2k;HY^GVRJ>LUiwbqoK(0A0C{G` zu)-{{(Z@{V&!|*bjT#>hF=Ox*KABNp!TUKJ;0iFV!;<%67&@8xq8xK#!Rfsp%c7R# z9-RhF1GU_0fPvaIn+aoALSNW}4lo|ehh2D+$;^Y4;1GLBQmp8uv9&bgLvP+ss>aR!||{;?WsCI6K?ak0`&|1J0rkuNRqZ5=6L@-FU5Y zYyhV$+r^~ys^U*(;isOL-qg!0h3T9fu4eR+mT7X8Fa;LM2EC(9|d{DQAM0yb+#VQ z_*Q>WQHtiAzFJ2MeI$^rlT?R=tQCiGvxG4-zsrzxCY@)Ao*fP4d&?-z=rqV)1>v4Q zDO{M_ZF$Nv;FQG4?)DM5HV&@^if}(;=!MpXuN$(89e+*soco4ex}RR2@(up;pPrj- zo1Fs|rAJq{KwP}Ozh+iE)!>U(=6_`>x#`kG4Y$#uXW3^FgBvQY^m^)LKA}cOe4E>L zb!8wqbxF|*1P9U-Mi1)$fcm!F)L=n|w0e;W*#HEXM=ujqjsF;GAh9L3_S`=lGr~$W zeDvQj+w6YSlVP?I5#A@?xN%kf%IQ1P^_4$8ci}-@G7$dhN1VJEc*=kBz!s3!?QeLu zYL)=+R&|T$8}6n|KDuS{-*PQL4!TP{6LPIjLA`QWbF2BRu-1jObjKtBpg2H*NYW(0 z!PW zj9bjcEmnE#a+<`1nk*{P6yrYqiYWS3>kOa4toxp?THPG3W5c&Yyut^;G7_MN#ZX@v zAx3%atn|&+I~zt@OQ*KnFvhhk$>g){PqYV5|PZtpTB18JRmJ=IItnt zv48qk&8l^D_p5?o=yA)OBsn#cXK8)FEgzHabaehzb=1FA z*dEm}FNg~#{xj9u*L;f1;P%~nDO@ba{{3aJ~8`UA>eDHWELo*(Fgkrvhp;<}1tdf?&-aXCL7kGo?C z5XgbozRb!!BODqg!T^0ZVWbgvlL6bx5{23IwIGJJUb^y^Fv|jHIGyVQ&XBW>*VQBE ziumiRl7aKq>n+*qikr&4_tvt8fo$E=b%6p%u{1%TB+V{ep)jBy*4G|c@Ik-9P<-Iv zQsnkFFemYGz&~~KXMdt2+L7=zX;Z`rp~CZD;n>1T;MKu~e!rLF;hw+PGp?)gs_UN5 zYj8%;7;3iBt+DaE61|OoD2)81TY)7UnhU1@%NsQWjE6ZI+{=zv@51OuImWSN35QIX zAT&IeW5sU4sNtSqZbm5BQL*0gKam&~@qvqiat?yAx($?1tP`iNF4@;R*xTM*M|h<% z-hKmK3_!y4;C2;+jaMD*KUI^vh0iRgn<{~wBC!Yxee=g1p*q#D=a~L77n5RF1|aiP z&K^+4rNI8y%5o6xu4~=SR??2@u z5l7~Ga&FNqtX$mky5}B?(!YE@(Sg;e*J*Y=C^u)BJrbawCLL4LhK~e^#Dnq6M|K*> zaQ~{e3grr+XbVU2luD!Uua#mJ#p6I5APu~ky(RZc^eX}%7Af8E0M6c1J2N2ZQoiFc z=LcQbIBLA>k|lOSlb)6pLQ(@hPQ|A61ZAfoo*6&d-w14;8+Q@p7zS8QhEvCS>U&L9 zkOyUO4;5o{)$y^GM!4_lK7&X(caBz|3mQ&8sQyuZ5xZV8kc+;4Vbgm*+c5$mi@|{+xN+$R8bq~w zZi&s6=(M}BdbTr)zdQ>5zQf8%Xz81K+U1YEdS-|#(J+f$ zw=Vuy8V5o;7rY8{E_ArZ!Gl=)I0=P|h&vMJd=sp4_KBg#d!Cu7e2_}u9Lra2kOG3 z0HHmDzN7;AVs3)+e0d6Z)@U#asv=Rldz>;cSr8B4zdskWpomQCxnI-~uBkO0Tb)_{ zXrE;zAH|-7CdF`M1{}nLEX%cMV|D9NY9)7{Pqh-!#J5IZctW}qrTyxpWb{TINN!ia zyg>RbePHPLhs4C;Lf9f*9bAh;`JqEn=BI9oDn>%NNMJDI17AN()zKkrG?&V zNZXcHZ!cNK)12+k+`Dt`QrQ?)l#1xa^0{ELXIoX{aV;K?h1GLUo=r50k53x@4QX4; zIu1@0aV`4H-O_lqdp&J;-4cA~<8slwZ9!v!>CJTrW6BKXE)0yZ^egptb(!l)kF+{Q1+YxS7`v zwfN`%n_qjsbAGkW+BjAyi^xP`fzZU2keG+Lip&q*4?KlsV{S{F6XpyJI^;4zKK^Yp zI7v@Y#o%Vg2c#K_2}Rgo3xY4tV)HDCK&M3CPYX$CAvVwrNzT^JzQL?DYEZn=3yki= z(qC+I@D}ZFC}NlxRp^krM~7){#JRwS!`MQq=EyD%+S~5L{)r?=E5KGZr7xpglrLVE5nU3Z+RFf zmcdw>mVHHrFlOleRm}Q{mpl7C|AJL++Wno$k#>-S|Eom(HB^_)5duPPkka1uIj8^~ z^N}Is!L$mbhNTk(GSh9B$}77;#+5otu0{e;Guv70b$tf%d`(apW8fmAq?=Ds`8sWo z3ITmFhcPoiSR!CnmbMMNf`vf@^DJk-kk41Ek8`j0-m%$SZdqKP9d3=!o&$xEZX zD=?)kH!iSxfA3cIWRUa(GTa8tEzqM^1Z3g(%xLV3x#~z_dNJP>Y#sg(+$D&HE#Wmc zA1i>d6O6nj+Xa`#f}5Thpx);+U3c4J8>aPo0k1A30+%@z7tyz-59TgrD#`MqbCfW) zlcu$tj4o4kv$ijJFXCe#GUm#+yVG#?;R@)V-NK_U#}-;57EV0A9MELc@;gs;k9xnC z6uOj|j}fku9TtN|z9K~;1X>P&5yN=!aKrIzCTcCON3t?;GDFG*v>Tn>N(+OSw>Jws`ckW#dSqkTv-$3XGbB6_mEXVo~~zn2Hagi?w%M0 z)Pd?wk5$EN%iHaUn*fsRR${N6Vd>QOstjhHCQcA-@oiB>|i^A79g8Q@@&4_Rr4Pi*(bY<)nS7`6NyJX z-k0ubzys+IKxB3o6fggc1G0pR6`{}<WLBGIOe+~!;~s!XJaAIv-|)ZFCMltUSHz>=rpFxV`Xi+r_4 zEKgO$Qk7W|6D8lgjR$i0QV54vlcPVn9CJJ?uXZ8ND16Ssm)_a=@!tc=a+_}WeW@sh zRz!nXT^`U!b7U@!^bUHMyAje>fTbGn4uU|IZurmDeefgDyvLH_2ZODBURC(zf?wO) zj$Y}3Pvtu%J*Qv49JQ8*!H;>*0kul31trJ{f$&oLQ2e^22qEjpHyuT z8ugq!Ms?s*z*Jw)_3u4Wq^*K-Ar9x@ivJ9eM~;w-baZQqti^YS3BabVAdza(6*PIU z7#y!~98|IkhcIRrbXZF0oc-A0ynan~@4$R&Krf8uHi*<%?5zVy)U)&lI8ebwwY@aT z?L%bV+V*D%$gErfGPI;nSx7ep^ z$-}uWx3Ar|9}9aNjBxXF^SDGrKHxBj4LGl zVjo46>%!VG%a5YMSz@7o4t=A-FDwj+jpDrhlG@m09PW~YNME0v77gYud97$9tiRbe zQs`8sr0~in_Ya3~^nq2}09#}XCi7ic42+B>*Pr(Q9VgkpPa}8UnpP{S#JK>KED9Mp zlOwOd^KZ3|b_F947PA~`kE&NwgaOmHkx2#M5dN)fC#}^7%)ef9mDo1ftRz*Ia)9zYRLwb+yr=HA_syuP z!-2lSd3Ct}rWf1mCJMUAJr8+P#xGU7fOl{+R*w~di83SO*%ag1$mr+5i+bw%5%&1! z(1+-$aXqK0qO&KfntVzSCBwa^k!2aUm%$EHOz5A`B#k?GEfSnfZ>-Jpfn0@u%xcs- zN)+Ml@mzY@?`*k1XaD|1o*P)jsi(Yve4=O-+7oTBBKeEDB)4 zZ7x6_e#5k%_bd3gt6!6?ca)@*S={^POJ!0;uH?#F27;PQ!l2q@{U#R7SLV0v-Inr} z!jzc8KVo?`EQq8$ZsFumNT9(h=?rlA_wQeG*LUkE$(c_+Wwdo!+KbkusH&=xK>Y?` zdPt_4NJG%G@AKL%4vIGspUZ@FkjaaHK-}q5VKu8R*?Xeu+K*AYq;*-I@Q5)YT z7`L{1zL;twbte-Fxw~75ZWe9o<906|1(C3gP#mwUn`Q4m)*Aj|OHYyx*tUaxZY?58 zo#)28mS{mJA-th9Myo5%^S2oW?C-$jp*k4?+!#kZi>p06`SwXQfyAEL4nQ3DZm7`G z^TA&o;DRM9^@Tc`y(-j=RRr^f@$z$N9TDP~p)9log*}YZz2>(dEb+b7euyojU?$y< zs@DANy~@lAzeKnOr!I?MX=RK>W4xU!)`HGRe)wpDAba1p_}J9)7iDK~&dxY@k*)xH zfZV=g+6T(gaN}_lT>)oZ*=H1(V}e{ zzwwt;KqL^zQ98DzD%hOK&agpql9yO~90v9TagS`sX0C9%yxfT7WBt)p$aA1i|MwWS$PDCuph%%9dShMYPAg%R z`7;gMUHS)p-RGYosMMpn$u|>PNnPBY>;SzLJ*Ui=D91TJ1x3m$JMi?hGbckMxGi%@lER>(%pWLqDaWTX8CWL$=O`~b zmn9mgoajL}AOg~kVPb)<#HRer$?BI%@8J-Iu`SagQ)5rWA$_RWN%Byu*v?8 zx!*Z~LZh!Y>k0#Ms>Z#56tE@%z`rD8k=nDexJW$xBK=vje&_0^Us>CN4=+E(MiR|SR>jT(OPm;? zjW{rOZ-A#x8!zFondsyc(XI2W=(rljM|0#M*zYWe->&~BF#ReVhfr)11^W)0O-s>bwrVinhMv;J-D!78oFQR76 zHesJ_-k4^ZSixar3Uo(RA-80X@O7y?X?nB3r*(GDH8I#}EP_RtGLG#AO9Yb%r_Wp& zb`h0g_d>g5*lXt4hC5`;y8b2UQB|wylmB$z6Hmu&1_Pxxf2k+~+SyhY!#bQ(nF6HV zF~c@Oufg09>GJr~)RG5cXzMUgDtB!|0eN-1o}b{Sa-HkT*_ARc^L?p@rX3^9l^!R; zh(LrB6LP+>bv-i0ZUur7Q?BR7=akVBZ!iiIHu2mY0p<$xd{|kbo@XrxDEs*fYWUcY ziBSCks?flNy_p8oP`~=lhHd(5Sa;494djxQ;hB?G-WwJ1yZ?zR+aUDx#XZANbq_(9 z@f}~*7Y@CeG`yP!`z});+8aR{l8-32VxvuM=G4EU)IAtVIODOTi@hl>a<7xpN~x_W zOOdMNq(kR^WShLGV+AhQ5YpnFrha^H_$LYya#2c`C~}3c)f7Nfa$1;ix%$ilW z@R5Y^>e|ZtlvgJUk)yBE(Zql_kh_c!$Hu?YeHGcN9>+kv&uam#_&JdGMDrVn zqNY{sozVGxZwJh|1Z;pJtvv8<{y+u6@5#jsVen{@kdlqDR-gTc=#sJAS!lgyU1V4l z++Qy&E@aHx_tcP7zcmY&Uh`MiYR?L+94}68 zp}Adu6uAo_ss4*k^$Vta{4nV3!qb_w)5XfZyn*DJ~*>UvoajN0ZirupB4cF1IH1ko8Nvu=ZjY^ zn7vt!!4h1KXK39~DYGayQNhp0vl;4-j}z}JSi25|FEYQBjKapM()5OrM23Tslws5X zi_BIuU+y_LEN#LH0@SZQ)|cxLsgO6hPx|=YmB5SO=RQwZM^KGEbIvDs>Wv`VZlUOt z{u^|N^n9=vBe)uy7ArxeY-ufszQN6`3NLbgSUQ42r^Fy5>~ctTVp(WuG2P5zx0qG< zUs%SIFmn6!sT8!t{NiC@$@4|~GEqK&)vHz93a5=GfZvIbX29-dyat&+!>c|2Z6&>! zsbr0S=-YHMw>%Rf6j)()B#S$<_^Oye?gzfAl+c)j)hu2^L&u>JWRj`&T~ZyG^=}eV zKHg8YJ3wDSdPh#MWp)<63RqauB*F_`$0OU5M^NgHR|kXVT)yI3dGws-Ag;Bqw?3O! z4QHj=0Zg9pdKV|6mYj55?@eT3w~D%vW-EdTVPjua67^?paJFT{N}1+W{TM1PZOJH( z)#HT4eiirb+KDJ7_`h>$4YW zrju6mU0~{bWlgEHeBX0Y1C4IkK1VeYj&_i5u(fcg|9DmM{$b!HfJl|p?>}AY`X3wZ zedw8e!+3+hP4=9>(toW+LtMr_$e#GV=K1?A!MtND!HXPC#2?KHM!A1Bl?i@#+}Ks3 zgC7Y&{{`sW>`=G=l*p4~Nv+liu6EBUUbBKI`g3*G>?8yBig-|dbH{vz`yYm4a#1v0 z2TY3b7$fvXeK@xWS9`0gc_Jj{1&(tG{h15|O=&*e@F9kwD%##h0sE|+)GW_=%QE!a zo7`n`ZbRx$T6HE)1*&ju{%Va|hSa88k?DyB*~HXT7s|E|uN)^Ft`@l~btP5rn4AYv|@g@=uQ6_5e zdh^TI0mNH2@fwrIDf*fs6-UR%ZXR<3LRD+~O3Qs89q))~h0EU{2`sA+gW=L7wwS=Q zd{F&o&ndZJkbDY|(M?LW4O>8eYsmVQR=s__ecC{IZpK!K_E|9I29%83at$O&sYtu3 z>;#JbyEZ~Y@pJgrk314s0STFciZkV_Re-J82TD!{P3C_!GW<7f!z$t^NwHeWfRe|z zpth_A! zM~5sN3skWfoTyQW6A1~pHUTe`o%wvXrn+sSv`{iw@zy7R(*gwP?khBC5mrg5#s7~y zyXQ`kgu$GzCj1$LoQ|-9gq13QX}mfxgY;yj2Kk?#71i4l-Uh-Z(c6Uj<_3QJtt|!e zVYXsPk}#AE6Rxn_p>UOjb6_uv2zgAt7hK;k;dk=V8EJJm1~d_*-n zbN895qi!O4Tn|MgjLke*WHt+Xd_>+;8{hiwt#GNFAVK_3v#Jxito~@Sj|qy2)k;nx zeHNbuTglSy2_j^rQe1BPV0!sjmv{YIx?aBRt&?pX&&Zt+_yH14wHu}b?k=#Fh5fz!Xhm%fjq7~UXX)h&@dOk3{e9R4NDQz!d zy6zDIt-6Cuf1?jyYzMezU*6-nn1uY4$JbnsBwz}*)w2JMKB7hWp{4)xi?mK~ttu~{ zGHOQ=hrXPOU@Obx+PN3^)TLm)PGreJuBI+|v{%$EaXtoyxi#_gl3&?~k#`mFZb8hF zAcvdND4SIZ-M};%w&r9E8B|4f0ZATZwo-#Pd!7EZIc5~di^exB4%BL7fwn(CWvB_R z5~;%!8vXbfi(M^l%^Z(|B$3b>*CU|NhOtc}P9lA$jCi3f8>t>0Q@erLxV7_L;Ma@b zB8_ChnUoZSCtxK+whFY9`Ge;+8|usPqt?6ui&=`c#~ygFRJ=&As}WL3?;eSBF=ztP zF=1JlZ7BDg=B&X266Uyj)@~qgsN2^qN1J8n%3vcch*%uaFn+$}gSbW43r$zh$K&~b zi}h*b6m`sII@nN-y8c!STfr*kb*n)OO`h}E7R8}sezw<14^qzccR*=vnr5o6@d(od} z$5jHTh|x|2Y_rRGE3v)_zusK=1iJFk+zC|fZ78A-8P0-t{+){|#pRdt(Swrp*?&7- zHn?nEC*nh$JWVu)kn(LGZ-U-I-yQXrZz5*adV4Oe=ft`?o6BpDQkmo#+EG11`}3fp z9TpTSL!IRoOujwsd~Ht2S=G&D9l}Va0?fu%qq;RQmhySL5#Ioy4j_U9E(G(c6>o>Q z$+uO#(kE{ZHrkp__s*_otj>NSVm&E9RLh{Q9qx|xo?3t@=CgNowt^{m3hsxMZ%HZ3 z#Gifb`W+0|@TFwOO7TMFdcFUuUb}o*ypo>W-Y+8M1MhUveoX#e!KaxBi}SzmWVt(s z_P%%3!vpF1?K9KYPJ_|Id-d&#UZYaJpEXF zWg=(KL1EZlT9xrxq)45DJdL)86s&-9pCuNNnO=tRC3(=RMfde`uzKx4VJ)0a`x*Fy zu*i#G^oOlOlsW_ZR4PoIV(^1wb)hDFO}!8lHL--}YRZqyT%YyJoSjtw?Witpf-I?v z!FeB!vm2Mg9F^6}JS%tZ7VCh)+6gqbH$-W=EI1kqRM`Ef5Qyg~BApHOT{dRphX6r% zitWG`-q;vWj-^pb(tXc9VhFBVt|p}d0`VAaZ!`d&L@_^=e-pbAPG72cguZ@sT<*}hO2m?ihWTmcW=*Ybbu6kyClIZwp?toC>Y zwDk4Z#+B1`gC*}%?#@~;!bmgH_wvE9YRmJbWyPVVM{@~L+3$Pr9Dm(7ruXcGX9rKn ztF9AN-iAnXxJ-p!oN!;}kwkqkF=!0_yH`iGeD({SUHlVb4l!}i%Nck{s%0w5nD^Mx zMvSFTa}z82-BJwEiS);5EW?A6Ml~b8Gt`Vt`DQ(?5`Uic7_?StKlcNFJlo~)!Rx2w z%X@c=NqpGV+0<+>ag6$KpB%u)5c`kmAo8SsuN14d8apKeG5&!jPL1UZYy)6Dg93NH z{%}x73-wvR?}Et7r&{`7o+55~{&+JtgPO`JY?M*Ppm;dQ;H(i3^UGQ?GMo^9lo|uB zZaZ(C#95F%fpTY;vG2CEnU9_tvK<~&2HyI`e3Ib2uGs2s`u5KKgHcOmgo%^Ov0&b- zPj}X8l|}g>K*q(mMQALLa4qf{f2Wk)yS<5ZDLP zJa0Y3{#)@3e=cqH*D)qAEvnijAj7xd#tkofzuOxuuQ?-@hCWnvabrt=!ShTF6U(#$x-8UxezDBm}0XFkfv60@couTbD>l-CObO-Yl$d ztpw0jK&a&Pg>j>c&-kGW!${_UVX!bnz80h9F6QZ@>rR#krXXW;C8LRlo!S6@-y#|Q z2SZ6oGsDj4RX8gl%n(zPnfc_Prr3U%+^tbV@Xh9r!eANBod>U82+(sbZdM<H&mtQnphcJ?ZF1aG$o(+s)-pneQelqA>WZF2k>(~*2OMpE|>`9Sy>rYR4eA9 zBlL|X;uBMBs5dW3#+g+p7&V?!2R@y*jQ8i4fNE1w^Tg%oE|@@$y(!8p!rd=C~8>Wp;FVZaZSYu-knOHaI~MiKFx1MO z&{ae1DS&~sisMv9ezKAEov6${ysD4ogN>58ztL!{JO|L0CntcB6Nq$H+2R>7_O(~~ zMsia4?ja*$_Ul@Z$da(@AYxvxDO=B*iyL0b25 z)%f#i_L^=Dl4lj2uD8wz7j=*i>bB(5u$lp`q$@rI;yz7_&T@xcza}y+8%(Dlh~-g= ztWD4Ex}P3hN3MvF4rHV6$kOL8YWgN0tLWueAa&r4y3NuX^chfPj%Te&^?6;64C=!Q zB+TaLi3e~}vp^PG+*TGWk*O{>!5M3>8)S8|KMYYh;Sj>)D^a*4l#s$4)mXyWPgRN?G z{A~NkvXrt}00G@5#FAR_{=ybxAH06J>TuplphIYT&^vv!=+zuWKOWtiULGC4_edz% zCeKIy{dzF2o+~Nn`idRe#hja)D|>YYsM9i87y*PqBs<4x&3BsKdr~`&Ip0-QxF|0h zuugdV`R$j=>Tm>Ua|@~QFYJDAe7Z-xeP<=r?c&X$W)NP{;D>Y64bSfG*ax7C?uEBw z@B}JWwf@^fIu?%4in1`>K>i<0?Ov*nIE+8+0U7#z3%5+Krnm5t^nmub=|fSfCnsHr zhasUm=tPGj5i#QhS9bx8WScn%;pH#p6gnu`+khG^QVJllx?By*Jm!6SEE&v_9{v-D zqrifyZiZeqT*zJ}3SBR?T>~iMsa#;M08G&XF19F`H82l8TsviSd)S%d{l5II!}$9& zz5G$~sg;vD`;POwjEgJy(bx~#PXFNZwplt$r)-9lZ-YBUh$rVi5U~qFyJw#sT?=tn zu07%GvY{dgU~kbvCY=2(=BEq8CS%gJE>JP8?x87&+bc?zkM9hQopt<&^qVo9X1|$M z_DDA_e(;=5MqTZ5QUH}9D6fYi(f-|t*|#@1xQ>sfw=Gl=Mhm1{J6$$9y^$P8k&vLK znkO8xL0Z`KBH#BK>K&${`a+KJ`3cCjKN*Q6SkHcU<7?7#KV2BnMPMYu8h)$o5E5chBH24TkEWR#iaK=>{l}w7^j@v5)6zg&h`SXFMyCPAg{f-vI3=lN89p&_DwmG zyh6w?FMW`gfiQ(GeMH{P=CBVYg*WJ_gd5vEAd<{D|0J>-ko2Um!5c{l)*?8>OVR4F z2BT{2;V+tSGi2eLH;e8D9|rLi%dT=sqMG_SLS8R$f7odmsSqDuKeg_3-s~I}On;%S zNmD-A^=NVi&{@x1;z*S86K#YnesM0Cso`i6-6Pj^L3zi3{2}6q>3xWQsS za{_hOfy*mdCgR&@BY?E{ivL3V(3c#+z~CHZntrt{>~qN_eI8A!EBDsf~;#xy;B z11)Qgd;4grxkVnpb?D#W`q?V3uaejOT&&Tw*v4U(;rP~ryjm07KY#o%r^OsVBECM^ z=L8C`7Nv!RyTAN<$!oqmHI0F8$4{f5$^-^l658kHo?*vznp+sl$}2LYtm#(Iw^s>e z)`8NEJu5O=`u$T4jUvkOm{D;A56COw`(Np#5J`^Qu67|1U!8`Kcsr53eXAMj(4U3K z{Y7@6wWp&gS+w+C^D`uhJgJd2Bk?vsg%S|=Hf{pq$+WC z9Q)ZmeqI1JFTp;u;{M+a&p_iwn||^8vYJI?Z3WKutKnbtqw2lBW&J_=-Q2WV-d&I$ zuy9?o3M~xbwCHOkDk>pqyyKxv#xQlhGPYTeJy*M*u&c>L$r*-)hz5`j&@D&rbuFEq?g6)yzs$JHIN@i~>LWOr-P>L3!mJBW%gjK0rtZjAYynx)SSgy_0TX`{Ud4|20!bJ`KduVKu=a1O%7eI5I3jp9GpAVAkY)^ViCE}K?RwsC|5S4gFcUct0)DN;Yd8ca*nR@eN*-S~XpI>QH|q)n>u z^l(^IYS!sf-O>x z9G`ohe_K0_`|(Tt(s}?S@_qfLAHN2Rjq_B4+d#v11sxLkZ))Z2;m2R~_{^3>f?A}1 z2v794nQY;Ea|v9o{IBejUnG0=V@ak>I~|VEcQzC^r3cb_llpe>2^c9EQ-koHQ&Az&MNYMb&RPX0XIkZb(VfJiq=44lCC zd^`99I2cd-?)$pV(-isOfq#oWEc+85blK<5j78$2U2#gydIs|Rg!DtnfzFy5+oYmz zDc>XcEt$EE-$A+UxhZ58$?t=oF6TJy&=NOv9)TpEJ;R-cEjKqYsMjwn0R{Ia>+->h zFTd`|M%TqG0%o}h?;GKY&y(#Ms7Uw7^NHvT7o()p;G44RE_X3(FjYXB)p1CH#GG5F z8lble-Df8dngwQb8D4pIUI4P#~a%-#VbaK(PU60+fdDfq*VpO+=u7B}PDs^fI zods45&allI1ERN^fQQI?I#4|%u_r?w?@4c&R(-@y=cE2R2J@L4e)RIp@kE&`waWKV zwI>ceek03g*q8yveJJ)kUv55VJ=kOAJJ-j}0tJV-1DYrE0kHFU3M;?oTRy%F=@f4M zx2qF0RwhUU?j;|oCDQY<$kQRgQqo}3Bm2KIXGdqggnxg}MNc-u!&2_rY>oD|_E#=Y z^72aF)1dnZ8niq6X)FB(v*2@ZNIV3hX0yWDt75YhV^+6oOXZA9$&n>b2j>!DRl4?^ZwACE#Q{~cp zs)g0{8ke(259DmJE$)7Q-W#D=yg8S`_9Joqh(zJPWq+N>K(ev{Y{GpI zwrDM~^h(F{_DiA>xhJD$;!e+J&leK%0;J$1&vg$kFXxF51Myo;iO>_>ny$wglefYy z|HLIet3H2d+Gdag(nwfI_kB9GzJEv|a|go;#-6aln+?tJ7JD*dhY^#J?>swH&sA`d zgrge(MlZ2lE^j zTv44>%aZ>EEM!Niw7$0)s1W}y{`F+$;Lh&P`D6m{H*UPKju`{|Zg4GIWsTMQQRRu- z*v+ZMf#I#zFC9R9-aBXS2Z4)`cCt^MwY!_r8f=v64F4{^$Oq;NDO&!D0H^#&132!i z*Cq{lshyxngQq_`7bIw`zCENXV1rfY&Y2%C--Et%E^o~wG61xQgzKPdqF>{;RmQ(A zv4Pl+{?l3?TuHU{JR*oIy$Bk|PMguOaV$L~)ymXG>Q!(N>xe%SbRcPnLd2##Bk|Lj zUL$bUdZ{67X(GBgrBq^YuAxU)05JaMT|nWj+#BCZZ@%+D8aoT_6KIhCe#(0?5C}}~ zcU;z&KhOK?vYVr)pj+xXlGk8*$hZx56o!`_nO*6eR>Iji8}Y%mfiBu z^q~Gj@h!fCqqv`M^{ZvTJk>g$OS1u2*H?t!`Gr*Vt|mZnWVPxk%&UB!8K_du7Y+6^ ze-=QzJApI>S~1WX0cDd6Qz+S|4-fZim4rkqW?)IX_@foeP$N|9l2>>n3BLEKm+Mfh z>#e*3kYBy_Gqi5~6qpUxGIfh0YJF&ZgkzN=MUSk0(_RUIpPcgyeK#@a5Z+XGM2&^h zLzs>J+sha2%+7ViDApB&-w02E*tdz7euT8UE<&dcF$cR5{6U3PcEzk>h4gw_sn2-W zx}{M?C2)0*DjU9D{vMzEpl0LzIe^*sY&F2n6C#cuj6vwSdkGsM9rR!+p0>Bi5T}MY zi^&H}hQq@?_Pr!(W*&^+rSjaBw3l?7RLUH*9P?-6CuH*>eBoKV^A$98!>r#zxg0K@ zQEl6Xh@{i>Zak5dMk*_(9RFG_5cp;W%t}(71+g4&|2Z(cdCT4?7s3E}K~=?YyJH+g zj>0oYRfA;W&jq2ohqYdGAPQbOKhct7rupCPWzUYnc{~)z=^Pjp1<4;h3LIonph#FZ z_`zwy`JHx4|GA%ksv5BtN}t)6Z$Gv=hU%XWJNHE0|BpR|2PbPlQ|kYprUU3pHANP4 zT-zNY;sM3I!;+*O2l5ZSH8ZE@=NqZ2*B5^Wi{@}&WxvU#**OT?#{zxyY2UE})|p>y zmH)bU|6H&$pDI-txxXZhdam4SmyEm(XXh|Rs3l!zOcf9GxH(h8q{=i`NrdvsYjt|_ zML^1)n*Euk#Yv6hC5l~3Ue0pbF~C_gtfIw%wu`Y50He}RUad{X5apsyJrnveH%^!X z@PhWZSr8A2!Zahzlg;HZo{HG6->{x^T4Xe^;_thvW(U8)v|@zxU=7i)np7L^VFmag zQ6+C{+C}mz9>=$F-7Czb&w2+ThP{OSXfmdg`qv6J8CtTIQdY+E?k%4S;_H_1S0@e2 zNBBlsS>q^F9w^SQ1Tm*~q~9q@{v-u6J;LnzHtJ3rh)Q15%)k8DnbUK7n9G!oZZTrXZ!v0^v4yUaUn+( zf6peo5#X+Kam{!|jeXt|vr0F;I3zH<-!RB5#1){kga7_LJKs2v`!3qyIc1RWCdx9c z9D_2k+id;S^MHjs&=~ZU)nQSdxRNS_st|PSx@6QS1>LkdNl(cIXLg&9#k+dbFOM}` zxg;~BD;T78%s4t5xH5BmTvxw!&jtuoX+B_t8bH`YBt-As8o?1ftrx7o-wSdTSyDQz z>VgX;eh&Tk(R@NFmA>jV&nHycm7Wq-@hQ0Lc&Sw)xY1DzlQts|0-?e8l5n!nzd(2<_*#@02{*%>9T~w zn!1Vm71#DzbP$fo?;O}KJg$X@MBr(Z-h;1Kj7fz_9MDq4@RIqs#d8&tp4TNU_x+>`P4TayJ3gI*LAPu*t+W-8x`!u? z(hlRvXj#&(Z$BJsgOf`%e;u^mi-{RA^Y%pzkTa%lowObSx`HJ9<)Tdg)wjbd*_Pw? z;RYst@xCRa%HZx$3hL_$gnUZB)fGMtmA`Nfa%&^XY(fTUS1>qJ)`X_TKwd#V{^akQ$JFEZ&Xq*YXjkM~d|FRjltu(~~^e-dYgGw=<&;<;5LuCMSbs)t4R zMp^6TG%JR05XLCoty20Z6YCyoHophx;-BHk0V^8q{@Z&-vpOcUgRCOzS}!jtO-oHp zKWJcTGVfey?o*zJ2YR0<%X2ulbD?B;6u~jLyGr>KQV#Q-HuZC?;Tk^d9Mj@_6<^LL zXY25X&Wxs}_8Q3weIV_9fA(e)iHSso&%coUo@-S;HNfPF4q}b>_z^&*P~FPy`n2x* z_x=PlKR+Tlvii-hg<&u#1yTsp{N7}^q)E*(nLL>>P(cz6r;i2{;$9^?@=ml*lkd@Q%{!J-NbyNA5c#wSODeQ-^iYbE{p9@m z3@tn9c~pdpSZNwuRy~^Nu#^|liyJ=3WujtZtHh$}P<4%d!)L`$_ifRv>0jenliX=n zZ-YuTKj=+D?n4}%Ncj0^Bt6(uAbAB#ePDipNG8NjlK$b#A@LoujgU2zisUwORj*#tn&z8R-hFv;9xlL%p2?o66wA948UkR$6(x z*x*`RQydu+h&6hcZozucM^!ZBm60S%bRy_AL;w!|7_n%(;9cmBWtHMPd>Irp-&U3X z2@|Jk)%x_;Ls2I^0ph5av`pp&wVKuOGsP2hN$A zj@*tS2UlkP8RU*cJT;!rRkT#ZP>w{oE9OQ%{%L zfU<{K}fje<2aph5^qeWJ$=kmnX0rN*g4jiMP0@`dww8nwHI-rEY!BzVlk0 z>VEo^LVvbXVkVKl+A(il78W_&V&yJgT#H+C5bChXQSI!&hX*jQfaMuzj{M-=u_+*- z#Jj|{zhV;vMBJ)Ll=?F`x^?{GmOVWWCJEuV$V1PgHNwfxxO1T?)W^g})+frsxEm3G=2NevJ49RpiIvaEZmM3+Hb+|GKHA3YwXX zr7jK0z#nSMI2+{kcBqfdeSQNkj4}!GYh7b4YxGP$RLfs@HIaRcXAwHh(1Y#ZLLb03 z^3mO*e`@F^&0mWYG98}=z=aX&41&;PPN!IEOA4+YtmqTH`}-;e+xSRM(D(R7;b4vO zOq!zX54O%mc`0vN7@h3{kmm9`;mMN$pX(ZZJYTW%{Qhbb$U;sb)4KO?M01>RY;wo* z9w3x-f@*+I+eUdHO{KeWGbVcwd$=u9Z!i5N5XGs1rg8Tok`6=Ds>>3aA+$gEYoKC- zGAO{HeRMdftq!Gc8Jd4Z8yJqZorQZ|wZ2*o;1DEn&^?mk<~)E0OS2$l`Dt%F;`5Vg zO)B%Y%v$i*z}C8EVMVnwGW3SUoZ5NkqiCa1|%Y2BLn{~QEm2NaTso|-Rj58cc}FIg{^Y$uKxnw4$RT#ym*;EPm-``sk5s7E+{J&B_?2e1mP_8SBZeMhFei+L1D!|C z9tKQU#^+hlo$>f5dKI&+qB~z{|1+N?w!DoW%XG~%(hpy zpR#U$)1vA6%gky)fYa#dBtK2m=^u(0NE+rI<(|f+O1ZzE&L+a!o6k`zhlEaAw?oQk zlszAmgp`r^%<7Y#i;TYyDT6uh!$*D7UF@KL2Jb1ai!`4*ss&=>WlO#Lj3jHn5Z^0b zi{;~aWcwL?)HB!Qr%U|V)Ko-KO={e%rC$duYr>OVo}e{uvU>dBUSO6qbRip(4*9{W z+{np6XLd$P@wetv)Q2&L{^{_0w!Y4{Uf!#6%M(<5T)oux^!=O)qwCl`^PM^{peZ*t6nB3 zSr}YPdz3oJywyww~wiFpUMX9r92Kj^2$hTstb>Z$j;Q+yLohldWW4!WZ;&kIT9X#OYv>8)FD10)X|lbMe@#FIPb0HB)CJjMB^r)I9wKQoZbGL z=$gD*EQ$fe!9kyAxd0JkhzFH0eFHZU2s6z_&z>D2V>rFj^R{6^=F{=;osQIH{Ot3 zL=Q@T&7cTOI!6P@j>A{R`mH&-4d_p#b28pfw4a_|5RL`|3V$dqfXGjWHqf8@9K=%7 zrCgnBFDxc83AWc~?(O@Y%I8;|p4=Vx7tdWi*?Vhve0%dbpQOh_c$LbbFAhc4w@Wo- zNP2nYe($_`@pHSIW*wc9{MlCfgg5Wo$2?72=twsr*HnXyh{QbF2f=S-03+u5^acFG ziv_WQ&w^)3=a`zltPxXGl!}!lJS?YM;}N)3M#SIyp`M2Fs0V&G4*g?|GEsS-A_WW` zsgXWutJi6I`d7BBX16nPKu@S50|t&_%T2o1dpx_?bg;Ynof35XI5HQ!HDc$TWaiQF z)Z3YvtEcXu&NFK-r+E*DW5?k$c^cqs-RZm8$VGg?1fYAG)}~Us6j*ojNOCy>Wf;*K z+{=8MO)rq@ijIkojs;rKlK>xWOcSAee{iV)((D<&i+Yl6wY5w*0mSglKUcLR+%&Ob zuDoTqw{BC@Bg)r;TYv%?V9rNl!giZHrCM!(GH4+p#6%_J=;!I>C9qWDu-3=;)mMr` z)fH}=#h~>bfGE3X&!UGWHKx3uw4pC{gB?zZgqWTLj2W~VpfAEytOq;2R<>Fl_R$mv(CagP-zg*G3x z-y1*4KxNwjbVS`G38nvz>SUeCjNcb_V?%QKG>&|z_w3B*TlDlAlZZ;&z&rwl*_VFp zPraMpAKBQuc39WEz1FO3_~UO5e`orzL?f?ue((+9Z`Rlzz^Bd@r0Zm3e${rQ^ETooylXujM2U zt{v>}Fw*v4EQWG1uYOE*svDEky&- zADFo#V|U4VX8J9uZ}lB-{lWBl8l6W=hE%-Kyu4hN56REF6{A2&n74|LnGJW*t_|~) zoFc*siu2y?cESQ178)Cj9RsOFk>S9kQoq%(V&GPKf@Bfz2iJ$4>3kle5C&+x-?nVn zY6FWek`%$Hm?Dcn$i+}RERRXCFa<1^3DZVLeqjXy-%W?MV#tLya}82kzRnI@HKcke zK9a&UZIYcROPt_VNe~!C_?D7kG&`u9b;)`!(;Gn4xI@1fWTA5zKrJO9RHM0@8*u?9 z95ocK1`7Zjt~e&UCb}uj0LTF?ZP}`YKC;`~$sl zsHM=Og7p3Zj7uH{`70vVWAu=fHh8zA8SZlbt-ojX3PoTvWC`-Qg|FO zOBCUXFwrIa#2ZWXv$4E&y+4=YxgJt)p};_Kb4>kla_%G<^lcf&^20v=fB6Oo-lt6s zZ-%byYMah|BljnCEQ<(Duj=+b4L9K_1dhKQ_Su=bJ7b9|vAOg)usN&E;QABaXInMp z8Pp{&o$Lcs!>LXz_81_1G}I7%SnZ@-N~SP^(qX$sRr&M&@0Eu$3QbT~cg;7O77=zm zvV&iA7*&2!&{>zYdg#eaoW1Xg7!XBoM*9N8)XM!sLfh9a`htj@Rf*ccU)LV)jxxC?+R+Ia5>;~;cq}}G*C{_Pm8$LSu^Sz$4msjJa>0Rjga$V#5 zmFuv{SMQQE_t}G12(4!Y*jSm1e$wHw_&lnH@-%v>(2=07NCRZ6=(L%Fo#0YafJgXy zMNPe-L{MUGt|C~7PYFCh?4DE+AGc^l^}`x_E*`q)knGYYzSiJfgga@F4{*`26-*$B zLJKf3@DkVDYuS_B@x-xPtoTmIh@WgIO^PI@ZYT-b5*ch$NF~*Vp zwHVKJ$9`+Q57aOl{G?uyyg}-gNAjbfR#BVI&M}lgX$><-L4k68UFV`(+4#4pmOlmR zGw9ZKsb;(vZgy5Ko*rU4bo)xV4K$gURGV?_SIPs8?VIzb;(gIiUeEICfR(tXw?@~j zZ$}&KR~t7Jja6o&V=E<%Pu!RIkNKG;PgYP+KW81{pJGZjdIVsE=`M%vbqwAS+ft54 zM7-FQyf0c?W_VkSklvMmUe8aV7K8lku^C73dz8Z}O$G7J5!EhQ{d8Z)x*4N6=w(SR z_KG6CC$74#Pu_o+dz^haiVMK(Wg_kJP=)l5jN;PLDe|q6dIoH2-@B=(DbmcJt2R2M zNNTU(JMh=2uajpDp4|9@EeFEoe31vP?S4~7fF~in*FfkBm;$#83Y2C5uCHdCQ{D&; zcjnhB+ANTj;UPAiY3fHpzt^G4qI7)rf z-zkk66^Q7PDyx0V$<>OOS>(Hx{LT>r*aY9raaqumuSPxVSb&H3HEqzX-mFwf2_l=S_Zgw%A`>DeB4ZVEqvrw-TuKMU2$>8TEf$MIH!a^>j zsY-~)Gc$%-)>?72GYWs+qK_#YxtrA49k3^{Xjs;~dctKB+Nw%%F_!h!QbY4(+nIQ@ z$(t9gz-cDJ&N9%Pf22Y@XRnTg^t=6>abiuTe`x!1JrXvI{pl*g+y9!xZs{ipzlb1n zhPS^Ti;p)@4VC)bC=LY$ExYIr^+)Nt>L%CPYLkvCrT}XWUixhvy#-WDUTlp%3tM+W zxUIhn^hK^D9F<+-Fw}WA+0T?Kjch{3l2x(U>A}|7yDliJzStLTd4`{=V9#weYu)Pv zqG*ul{%mIIn&|eqywo#d_0tkN=$(LsG>0X!6c;ia#VY~0ka#u(6cbj!O37i}16;9_ zXGcd9sT1+r_AzMj(SO$w?N4``C63~JMs`d`0%`#o5KIcigoJCDo3PoY48^hWaSTu< z{(x9`-~*PLwkDeP3D;|#*TCx2*GBbd&w)L3ZmVfjQc6GpWTHV5Wd{N^&H|gq%LBjQ zNXyOE&W@Ezx7w3fZ0oGs6TbrVP-?LqyLG6l>*B)p+-P|n9x+OzI0FJ&O^F{U%yXZ>wrMG z#oWH0J7zI^Zd$397UuXS(iST{@Q3h~U;fd3O1rMMhBA2}43s^N9JJm}l#=|c85St{e(e~|-G2sD{g3y|%|lkg9}8;buwlS?dC6ZP+HP=Lvu7m$UTY@ zUTdx7ul4XMA90mxzfI?HtfmeYQ9}}FXeNX1gCN+&3_uiv^S_!jxyJ*7CkFi3!<7|rHJCC_nvmeAx;#wg@ z`xDH?VnT?7OxcmWV&*VD1%=?j**G*^N;ylff)4fYq++G?_j$d7Hl)FN&of&!V-f*~ z5@|vQ-ivG-yh&G0&14=(iZG(2Or?Gt_=;`8c-Ko;?q_ko#Xah_Qaq-zCUCc-{Voyw z;dn)+`z;WSS|u0vH|o`Sg|CKz6!v?WHZl@yg7$lDTs6=Xju*O-t^aWn4=PV2Berd= zwF8@_$dpd0$J!Q6IO7XXw$6@HgX_JUxw9@!r_ZD#Tc36_Y1+M zT5i)hBa@1fK)aqk*stdvqodM`4GgA0a`V%64xwq~2%nmo+H^G8(e^J0>@@kIDkCy9 zq29$_^PeIzF35iAYZ%kyq~3$MJy`*0yvauHh-5QwT8g527OQO<6xyk{n*M%b#OF6`e1eV!x)_Z((0XlIpld;Gd2KA^oDMLw2MmB` z3&7U?AHa+VCj0?_UhgaZ>(&kD@tNdRqWbpS?V)r8DGLuDABawxk3B7s74Z!al!JX* z;0m9(g+;3K+{>rfAY!9TpbYN%f6@l$hWywh#j0LxEtgQ(DPLU=?tmR?H0^IGzNJf% zVx?!XdFh97U(E!68zZrXK!&@GYWY6oy_~~LGK2P9=Zq#3@5P#lD%vAYB^b>n>Ni=0Q z+Z#i~B}a2i9UZp~!k%|QTmDmq5b}{UAKyo{1_}o0m*`MY*4A~#=K{X2r21Enh#!wM zd#_k{?4cr6G%3EDQ-QK&R@HIkne>GoQO(Wr+6c)VLg)Q~&adYN&%8YQr;GzwU5zG8 z>7%m@axBD0PY<3Gnj^1rZq{sW4pO}hU<@T}OY(#ZAWqv}O`KIihH$DK{kWdjc200m zE+j;TM}|c#-~1&EuBfVVO?Y~IY?hkYcC2L;GB-<)jY1bxWzwTmeh5_6%v$B#T32V3 z2J0}Um)`v^KaHnqvmPRUkhf2EU2Lj#^D{YnmxX*Q z)11yafysXELRzI99rOifV)4z0(LL|vZ{t|x=c%1AEz5#urr*0YwEnVMf5rnr*Z&82 zo;^ozo;SUu*dYEc1uxtrCQB5%qGR9k(vvYtDdgYqf3sXQ3TUDn5ZfYidrU-{>;(tM zN;PC@(Q48OxYv=SJ+O|amEuMkR%)7mPk<>oc2oLP zi%k0%8YLvqu-=KPJ}U{G12!@8myZVr7n?h%Src0}^F72W_=(6|;yzT6`b^I*g>3(~ zz*az^KfHTlve5o4bR|CQCWyc6o8gNW3@v#KQy~4=WY%OC6x+{VdFl4cIx^!{l^niP z2-m_Trb*g(-=OdDI2!v03ommr&9U*x9UJ)*ACq;%K5#FA{;(b%F#XfRp@vu|g2&d%ZkfE0EPb=Tw+8r434;^i zp?S%56YaaRUpf1$Q&T8lqplr}wQP5%BSnk2-_i(E>pwujn9zY0Itg8e;Y)4veNnl{ zf_12Zn0Y{0{o3{R%}J3e@cIA$_~6xKe6B-~bsHb=@6rDZ$U)H7hXMHcseISP_gMiK zf_!iR=aKwrs}~7tmB%A?!!_DyIe7a`sTus_><{zPn$-?2Spd}iGyOBm%zfxkD_wfd zF8Qy(54mkGFXR#ds z0Rjj`m4OU!O;P~U45oQhu7u_lFf~V%$5O&uiPY^+la+e)!frPPu`Pu8Zb#24_D9Eh zt@F?n@7i}Jc>%ctRa_9D(INUS>{2h_qpPi#f{Ehn%dyLSzO(e5Rbhh-7WT_eEc2z^ zz#uaIU_yrnXB(Abdw46wi;C;^yKjXhuX9V;qp~d0)S2|f0tz=rO+5skS1yDrJZ@p& zsAQ&C>Fj)avGzO74uJAZ6ZNmozcC%bRQLb4>Qf**+E~!}9Ra8>_EM5HujASV>i>Cm zzLK(7Y4Sv$&X@Dp>rv@CGvbwswO-UwkmPt+;A`jlvY9csbP>ob@hEC^32MP1c2}fODAQTVmSLV_ltj zZ+1Sb1T0U{;txi$K@eJkrKHdKuGik23Xh`-45G$c8g*xVX!Th~YF*NZT}@a73B0P` zNw|85^{%OgBS_b(*i{+iJWG2b%BPzBa5aoGRR`ltn%~>(ogIwsk7_uH#D8rS{wDX1 zmRMVQ+VKacG}YObc&Z_=I3_rRzq9n{rop5N*C*A8h$#wpO+5;7MzD1ULi|IB%&rbC$)V+0;dhf$i0?H2SU_kRtGG?A ziQ`L?l%UMb#{n<%wx!h2ZK2#(bJuI1jVY6%!gnP?0){#_O;bwhj#=3t2!1E~=cvT3 zJKf@QeRbb*P@aeHQ1g+xdStdXS#XP1}ww2sClY;?FT(~|i~ z;so~tfA>l<8uxgK&yt8^O;7a3-KzXiGktL?U34N3l88ab|K1v_iKOp{+c;_9VXztT z_-kU`s*a?Vs!XL|tG~q*f`jWnt^GxY5Qj^R0(+PiwfY@PBFLO2s1Z%6#}qGMb>)9GmvTAMN=K{2wH#yUr~y zS1PU37X0jAfppgGO8k%Z)Y<2)@7rgrh(u~&@G|r}h;yeR5q!&Pm;2_A_wIaq*v&lm z&2=E`1lNo=KR6krMdZI{soyYNMvc4w30pQFbgFl%tbZ%7%ZA#_=%#C?;1s-`=xnZg z6D-Ce6SN(q$8xN?ynNmS2uQ!yWUgy2$vQ#%k`WyKoG=07*Euf|UtYMrf(b$`=>9_mw>8;nT-O-PXmn+>&cncVYBJnaoG>ahJ#VWe!X#Nq8 z9sPXr<>V6#Gg1w!!NOeT#YnQv5PZfakpwtDj zi-SBd`KS&rw74gpiV{p9s2ox6~p*h=-h!X@^fuBmT437`BI% z=IyX+Qibh*$Z4)vOL!s|1BmE>1iE7wg{yRH6P=^ED=9qB3UWQm79rWt*03OO(NZs(3~wL?YW=NCE7b)CdF@rG|Vh=otMWogR}MPoxjM| zj2fyJ*Fg((8hRE#Ry}ic-n7l*v}G+OhFvTI7*_jLD<7Ea_c5uJzk}GYX{83JfaaZi zm@$3Wrg$lhQIThskR>Zk=(Wvxi7i%f;H8eheSx-gxsvbAjkZ4j+D9p z^D%RMOl2D)gaCY#0)_z?79tIwMICPk$*pdZ#3`8u-hLQ#N(*a-wE|P7vr!%cuJD$f zQtrqSusZ$mhu@+bPMzQ}DzOLlPQDN(7V@aQvpks1g+{{j9T2lw-TFt-Q38R`>}}(H z+#X6D&XkRRel{^Y7o~$@D&b+M_RhvFntY<+nETC8>fn1Rm?vc>mvf z>ecLKLG(w^p!Vtw4fS+I^x4A|B@SW!lU3cAXFjsj5t`W}uaJ z=V`lLl%)IVidzst1vB}(= zJCz)sBMqsN8d!2=Zt%9_RZ(On$OcRTMGfQ@12z-_4hGatv&Zs;3Zr2vC(s|LciN9=m0|ZOM1fgq!*_Vl*@}yd#-(VODS&o7VSf*bvX5Qalj(FI6 zWdPglcuUg=*;4|V&c3~gLe6ArunbG*aj2js2K8r}AnaGnE1kXs^37RxnuyI&oX{U` zrHgq-ukscx4z=_)_bZrE9z#U=hfzT&RfpGz>EftLGCww2BgAv`3*E;RoriQsxg4kr zTVrDk6r+iJ@q{afmAUYu_|JGe-t?BQXWeBy^S*|xfU(X?Vza`w=fo%mKfPq+1|q;y zEcwIWw`DUN!2Fe>6VKLEQsj6JbtJB1hk(%lh&==4JeUIGpVq6W*NsHehxf z$wWPXjlI@c>BlzC@{|^cVr_6k!j`a#4e7JxDz^}2uGM9>x8+jW%GJ^I{oazFG8L2# zBmb>&uXKi6*2Ce%cX;w{Ll5h)C;!gRUM`-cQ?U0OEudXt>L1KPtuZ zNS~74CU|;Ayl#@`x#otP$v5MKu+d_h!KkCi-v#^Q} z?Z^;i|Rx9dJ?4qq8q)@Y{xGL`9B&mqToqI}Z= zJaro7pUhd4M>DAFl~2hdw+R=7En}DG_EG?r3iw%`6L&7RcjR{fc!nyBoS&Y!Ml)2| zO4EV(wVnN6U!V+1=l0I7&YMJ$h`UzrA31Yb1NT~pRMq2;;bCE)ncs#wZUULE&T&-~ zwX%F(JfIxWc}x$AJ#&B=JT~ZoR;IUP<{@Q?OA$kdtj~&s?>i?5Ee;Et1ezT?Va%+P zI6ez}LVsFtN)31{JRBl#&fmLtb|-reJ!7$Fy~)OqvvtjOW3rDCrFx6C>Vt0`uaWt1 zmf>51FBv+~9wZ5ju=!3Z)|JGb#k7k=Nw0o7s^k1JpMMvbwVFyOTv_U-?qyK< zLjs~SeefV^!L%|HM#-YzKW+kH6>D8Bk&orMO$xLn>(OkA!M`+{2FNED{1RQKiPR`_ zdA7w$zE-XMrBuqYVl`Nlw9#41&Yx}`t;o!{orG7BEB)C41NoGw!P8nB<#d1bTA(l8`P7&$=dfPag zmoqA}fk-}7T1;7A4_e2trqp-P_UN*jLWF~d6?4k!qJ~bKwe1ncQOu)l!tD+SGdV$V zgQt3wq%UieIM~h|sB+R|I5)xDtfsI>3R6a3ord(e4K^}N^%`sY@Dqb;zvmYZhn#Am zevN_;q2!-46NSfGXJHJFSMbI%PZd=@qgfw+ zU}4Q4&tJi@zq~9WT(lekdBf{7{2=~_8<=#?D2I^AYV08oaAoV6u=lwCW~on*uL}3t znkUT?Lb&xtKq+X=l1w5!=bG}DS@!X|B(=YalTADtV_f1fK6#ZAHY=TBmoHc>vYnn} z19N1HO|=BB)UClmota{^)|ld;hIAFb!w@U3Eziv)YHR1hA2z*Eh^O0r!K~&_ZQ2M< zBUbEfI^>lA`ZpY<8_9O6|Cl2Be$hcNiN@^4pqGo`Z^REvVmUUKtRdiUO;^K|Rjrx+ z#df@Ioj;S&tF(Iw@|xvT@hTj2`IHz2Ug%@hQ?|qjs;=q_Z<`f|3p9N?YBd8&YB>YVF96V!`Ome5N|r;a|wr5)zX(*9?O+F?2XZ z!knzqz!$Rk*u<#G;Q}HZB29iZVwuuDZ%$AKp9AY5D-T#nl0m?mAZP5I@^ZCx*zUxK zubhOTWSP_ye3sGX`R-z}L2uO&) zCDZxpo>T{(#et7}i2N6+E!ZdI23S+KnmpMB>!+p5zq6qV^i}*Sx*KB9&p@{{=fWdZ zW=jB`6h@-iqb8tmOWAy!oD`ZRy(%>YbnAiJ=l|cLb3MWK6Oau;MP$;Fw(B#~@#zJZ ztIMsThdKaI&EaBaK-N$|K>D?3=y|98EskRfd5Ma01PFK-wuzo7NAk*_bi5kuxH)NC z&qG!cww^z!3}0!amDvl8AfjU>3e~gTxVFN58=P$_(R@f9e=(Ynk%(ne^sS`a%hWev z2fxEtPNfb|r%T2u+7v1pw`Q7&qpG3iU#v!DNTol=#+a*taf^#8Kjh3imO8)CeDK^O zbS4xn9z4JOz$i*+VhdT$z!|AJkz*p1yj-*hvR$#f!0YJH+nF9NZ(NqT@NX{HqN6O1 z`j#elY$M_JB*ou{irJu!rJpitYNQE8xED$w3v zggjWNJT?t;D<|)u4N!5}GPQKhiR7_>wjxvn17*!qIj7L1^2Tqla?KT-4@b;CX4!oLkXISuU z6*uu`&Qo`axAHM3th0m1$63q^@(Y$*^bKkp!4z1klotwy6`2N9LbN$VybE|f?yAg; zY4F*(3SAtuTqJ0(+slq?f<#T%PMy>!imaTHm(lH~c%P9&$Q6f-e)7vCb{+tg*b9b0yEX%K#m(f(-=eP<{_np~=iiczZMZGbBw z=*PJ<&~jS#0@iv>b#?f`PB}K9Fs^6M$>&nINtsq;845(RMrSo3zd~ftvAWunFgRR; zNDB)*M?K`=^0J;ZBBf}xL_HcL5CO8Z`6RZ7%K6bi_w>Zp&j3QfJi^8m2JGxK=`C&udGzYy3XnpMSu8 z+|TE}->-4b^C;R6Vp+0Z$iGiOE3Kzt8Zx?UxTqe1qS9#6nT$uieYd~8#OS?aW@WW; z%}AcVc=OIreRE4_XtND7hx(1`5pl4WW?~e-!`(Yh|CS+~33!+HLJ>uo!PhSM<-EbIsq=d$%Jaw{(qpl5CJ!O*Kzys5QuhvAA z+a3g5+K#5hQ+qTm01YW{I~RyDZbv?~=pz&ZbnG8Tl}BnOn)19zG~Ip)mW5Jp zmMPM-V2-m0j$Zcy@g>nHQWs;Gd6}D*lY5O<82Sm}Zn{Q5z?q^XRM7C7aFw+Tc~ZHXj)TP;iHYSumXj9K_!V3M0?3f$ z0!4zrMfJN6zjG4Hv5&_O=b za=v}f-9l=M#WjvA8`TmfB{PtuN!h)(I}__XeNj59gXjwJmLW) zutd~s9cd`Fe5qxJU@gp15#~)2vf|LRE4wqIsw_zB$dr`2gwAIyiNVq`tFLCpOpT%p zp`&tdf~uZ>$f)f{Lj@3?kCtWkT8@HpfuypU5*!LnUA8`ZdG;}qSH@Q;g%`YbzDG1# z0@&l?@ z$H_m7do4Ls4t9eC-rfB2>}=Zns9ObZzo{RqM<{dSj~rWOydK{4sC+pR;th(ViF)(j z6<6uK(bUrY`T0~+`AgJ*ZcXy1v%`Cr%RE3;7~*X;8J`%{^Q`llX$*5o&=bI0IMhY2 zdhZ^RS$1y*Zc$neXcqRcr5G6yb`@-P-%^x}j)lCTF(#q8Nu$y=B)n=v(KK73*?+{q z(R(t64VVqvI(cw3&$k4|4hDfhG%(9M{x<+yG!vyRC5FwN(SYje(|PD4!76+1B{_0q zX)R+)t zue04HhPP@*S4G;V#`0MsXsOp3~E_ieI5%5#X zJP|E@+c^?J!FMI3GJmwLcBp6Xq)|1`y(AfJKXRPK&g+*`>Mz6-hf?Hi(i}Xgt}1k| z(XtVQ-{9flfn&@x{J_4EZIs#c4(4wvw+t-J^_9#0qHwv6s}qNpdwYA~t$WzAkVb3s z_}+I1BFipZ_B@W=^E2Ui31F0y5Qh&7UuVY^{ZKkpo+t)tad$iG#*954L7!z@8Uqa( zJ8Ym|o_6F^MelGQOMyNfzm`B#c%qQUli8_t8AvOA>kLlJSh z?epCzG(e3bg<#qBr9p?h&!LLjM3=;YRxjN}18(7+#I z?$<v_pqf2HWIU6oA^f3=y4-k zpSVd32Ls?9t0l^s3$Yg~f(K#r6Z&o6dt3^XxEMHPoekq7Bz^pp^)o21(ORi$lrWQ( z(KSBQc0S^xO;AuLGrxsIuNeoR8_?-&3Mh@#zFdVDSGA_}Bu_mMW$UheYlo@#$UMTP zfC1Zk?{#*WwR26q1xt6sp2E=8hAcC3xW#J;@$#F4*2jRlKYpEQPr6%^r#u5ABmMGS z_!3ytf69gm{3dv*myP_8`}qyjqgzETDhtE^r&;+16!kcxmDs>MJeTUO*y6w6GHb#- zocJB>1we)L*I~;G+h_?Z`^2TqC2h3?tOc;z&Hh&B>25{Z4ToD?p6wY0MyT?P-29r8 z{sAb%A5B<{v5QCv2lD^eAiV+tvJhfFy4lQ+J%qm#%)g0!4r762hEjHIhgeN`+B~$- zS^~saOmvW3gU4UWnAT{X{i#0>Z}lF+CuJ>4$-ltAJTo3fGybd9d_+T& z&62g@Qek47#XdGoi$yHh$#X&<%fB%_v-1s7Wr-^F%bMi}@q_ad4)HH^L~>7=H2G-Ztp0we{-xED&q+P} z(@ZLYEi)OeqJX;C(rKZi$HcwvsMM}U=yLp{k0@oCi<$wsAAM&?M~TV(BK zbBjT9r|P&*ZhWHO{lLC4Kd}yvcLg(Nxf;q#AenI3>TzWoy}|ndAKo|L0s~U7Oexbt z7`1rE-q*NL}AprD0kh!IY*|KPJVyEHkch;?%6rjB>z!8y!9PnlYfS9zf0 zZu@l_J<8z<%gxj6qmU2wZsL3XK0K)n^Pt-atTa&A0^8}Y@t(SVHb4}LZOBS2=PR-N z&-nkEP-^m)v;Q_6NSYIt6Y@8E;{?&wPj39H4(o+>K^eT9K z6(HY{gQQU!1L_j>ENB+qpK-J(^dC*v;EMo%lWz z2rPN(g-pXHFjCGXLyP+zb4%9znz53LqXFq2*vZ zF~$|0A~UYin_m&U#Q##ds2F2i`ECOmdX1)`elcAe&TAU@1*zlL_*^`JRzv<3i-9~a zAPzSu45Z7`P`Y~?|1z>@ejj(dn0gOH4@MY3)7km4cOPf0)oKZA!&4vg?O9l4I-Z;! zxcAi_Tmxg7-vjHvpXbktf$d#?XE_i9_cUylR9CA|r-yAPU7So!`DT}F9K3|3O1G;? zGYUK95*wGGY}9zYN+W>)2PT{$y{4Tw7*-Bh!?v z4mA_ks~Y3dsdrX(o9uq!T|dbaapv17MmT#O4lz;P97s&yd~g(URT3<>W0P`VIKIZL z`k>=!8I5dr1(mJps;{{MhzYb!y5YY2Nae-+lzHays*{e|Pc|gIx ziImA6cL>$O@)}NnGZay2xKL7JqkT85*ilGJ&;h|)Fc$-oK@OZLR*8UR6J2MIDj6IS zA>8i!GdipdTP8Dr!qQ zp}X!MOe`WhM()PN45PGum1Jw@FCVpD^iPw0{psIGsIOlhy4U)~)N(Sqf=x_bEtQp9 z)3`0H7dfd1#mHpsg!xpFT0WAH?urE9Ni84N4%QDvTt;|XaUcOw!Nx%!ZX)0ksK6=% zY>wKD{HoHEpKNSy$3lH+KVK}4AkcTLr;PQ#5hMi1ELR#cvPF0x=-wuw0&^AOLVC(PRb=sldta29z%-f$ zMWvcw_#4jqh#9Q&+7E`xzit6sN8mxoWdZ?UjhvJ8cLnr4==bG`h&v>$#)eIsMyq+y60ZvU7rwX7@- zhm)mWyKZtI0f_+5u)~U98Bi$}2-4B_vU7T}!Ls=aCbx|h#xSN`i3zB|wajCK@w#gP zvhK|(9mku!j~U$Hn;Q^9)6tU+)>#kW2Y>B$wRE@ME1ggW!;DvFHKR!nB3Vqc9_RZa z8h^S)H0oA#xrGJ>EWk0U3lKRWIPW01exjOGRw09=PldP;)C<~i#M z2u@%{S=E8lM`HvaVntcpQ_e8If+z0?=;S;_O>~20|eI7VetC&H?pyAob8h0WvA}}oDUk|!TMz>zdOvtKrlvQ-S=}Jc@BnnZ48?e)p zy`&tF-h2_oBwwAG+~@>}wxcb!Mv&WE*k(?LvF!2MPnZ_mPIh+476RD~s#BgGE7YN@ zo8W&fQ-y+OkH!z3-{KY?F{|^2N2SW5cF4@*Mgp!ylmT2eKG}qVJ44_v*<#9x0s9iG zOh%3+323;vNLy8hnYzX0X{IffefTiUW+J|Mlhl_SL8@;BsZI6(@VuU$%G#}8MaAX_ z-}JEA$>FQTmhTlM`n4OE=6;#)2X z+|c!tl?lJ670fpLXqo=sSve*i{~bO z$MZ}1VDaam@o!aWrqy77MnQ-+(1~*W$ent2#dyz4DAwAH3zSS7cGH4sqXgjjSCf%T zISwY`EIOMS3f84Os}@Ap-P%^sK>G2s!rNtvAw~bV@Xx*rI-ZJ#ZKr*xuM-^%xuaEwfGwq~(gennWeko1-f8I9CZor{whI~lI4MfJ^8 zNTwuSERx6a1bLTP&!U7f1C5ecQ=u zA=5}{ZH{RMmM`THezH;xG(ks$!8436HWnQ##0sh?XHkdfVbap}o<6BiL&ZVgHKpgRZ6G7kK8cFvA>8o< zD7aA5KV}4!h(0m42wcvmD3wy@AkEOmSh#c{j*(gHC5fwXk)U~R`V`QN<4^dCs<}BO z$h5%xDlA1}a@^9L87ovF`rJ8jGa& zmonjswu{ZugNOp$i@pRs!##6X)vhvONR`oL;DV-hT59IIb#Tye>&3)npP+y&RT=CH zHsP5?4iT2UU`YhbTGIZHGvtQ&zGba^skk^k#t|Dcu%o<&I8iU4CL)m6!+rSPN-D z$`-n;$wHB72&w=g@7T}2y-?9&URV~tQCKp*(yYf?&k2zLmpYBZyjqjG-eJsDT6a>- z@J|;4Z_@<}Yxi|(d_a3u0d)P-M{S%0R1QH*5Ji#3*Iw0OgEaR;$i23|o-QAZY4Oyjmbc`6oEUAy!1Mq83pTM}(VD~EMp5a=W(^kIArNR8 zFTU3Bjm!jprCHeA#_R0>=(dQDMmtF#haQf(S884xw;ROfLQ(9|ml^gkVh~-X_qmTg z7ne@PmA`i}F(->u(>B!-y0v}LumiFay7h|h3bl57SOl*RVtEel|GUo9iUb@D%b`UR z@o3)tVd5Sv`GKW-|L3iKMvz&~wb*nS7vA3ZPfxV?+sAOU0&@Hl{c?XKAoh!CGYT)n zkL%;-8++7raD`XdcwbJdRW97teskw-M!Z#2;6C@jg=GjSywax#Q9+-?Kq*Uyr* zjhUOJ)_7TPFqZ)io8js9pR>&(*Aq9m%n%9g1EqVe*@nGu>Zj}Qch_tV!>EfogK~+W z<^mRa23<^r$m)O2q-bn>Q$u8! zg?SPBxu&8COb~wv@(LR21Y|kY^sOwBYc|^PRJrJagES1T-Fx=QACow$yz- zEa*6Une|ZrFU?hnr{YpW@yc(|aW&(A%#A8*^RFkO8|hsu-(qtNQTr=@^+);vP+f{P zHK@qlU$20ff+bj?S0^cz5Q^9>W@%MwFdCFj{w z(Y1rMB|}3)5f^)Td;S+2wwIP0tQ<)$+-*8#*KBQVffnJvw;u?S0n#HCa|nPG?CgTH zv!8rH^W_VM8T9q**Unwzv0m3($eJ+yQC9au%#V{$sv7CjI?r&C+)uRymS7ezS-W*b za*sm$3^O?snXvy+Cq1m+<;QiktbB|HJ#?5s61F_IB~=YmtvpG6c@-9UGBd953yq6< z`D?&>U?2q^Mf~_kxN|l@Q^#NPu9`-{5_hf_^Z^t+F~6Olce*t@3rJvAM%UzWh{`u5 zJYomPXndRolM}e)>B%|LUf<4c)?8x7pei#xCmUCAUpTf^#D+N)C{ip={hEDsJ3vmZ}ot?Hp!PRWTrZTs(t3R5`uy*0DlC(@|}R=v@2V036#|n{1~EF>@=7)VlG{3S;Af|e)qLB zfkm1K&3oe8qMOaVG1K&MfXCf7^UkvzI%gvF=}9QSv%8<;Ub$3{+=-P-4sw&Jwv+3

WMDS`~Djnt5Qu#)pRp&y>shsSKi!oe%4!fR9PFg2~tDPXiYO? za4{XpT(bD=xZx5|fb^tZ`*p|y zR!k17@tnv+58W#OOH@woy8|wtRrQx@?~spS%0EIsVp?T+aN9flc4m%dL-> z&lGclcP>xzJoi<+T6&rQwG-YZ0)SuS-SbNQ4;^|g(~4VP4sYEdF&ci%Jo*(y9shhb zcE8~Q%QWLaCNt>jz1!;#y*!@!Gnge3^t8_0G@B>N0w&i_C#(K|#|0m`b}W5( z-Q%-w!8z{OdUS2Pm8A{7F!Q~8_>s{mjL+FV<7s7wgxSsR+{3GbSiz@f+Co%!?+O>l zEn@)QJ-5S5sdi`uiy9l%m7Kh(F?v4C??_wLpKP?mpa(v0@?mBo{ubxDWLJ2VGho%d zDF?nE8ZR%7T6HD>xOlx|D3!WswwJ05Ug5Z1Ixusz+;9~A6P3SuFdnvn-znT^31BbA z-04m)e}o7Ea&yA+sl9TthxFh<-h}M28 zc|iEYRpqc>nr|{q$u;sjVLhu~YTjv=>mzl=aHv74A!1N?P$MRjMV2Lt-*6OeRtX%z z+^%1cxS=mKxu(HEK|rpsq-1=$8E5LFpce@)y!6y4{^B6j2}`4!Xxj2x?()M*c`$0W zz0d^UuTZH$GEljOs7I`zCNhOGjtVPpw?T*(qW3klOO<~(?l_Oxq|7DtWiMGLaTxK$Kbr`={>Em-?HkfNe5l7K!wQ5qXzvB%G7I)r{bVGjYe z$VP+ zp;&R&%elYPDI;lQ2jvEu0++9^+oLHKw%#ULnNd(`g2XAKcKi^5 z#NpA!$oakaMKFU~!TJxoG{I<0>8ld!U;K0o8vRkR>Qu0w_SD!uH4B?D0x@VtXUBk$ zj6Cmt40rH=6fD^0Uk5b;_+-6uUWgg^L2YU7V{NOxu!t{rKaH_JJ zg$2jFQp3(h|5C8w}A&ZZ6)#7o#8WJ zFzUgPGytapr;A`scWi3;{(i2hJHj9kqpP=SkI$Z&9AMJWykA`+QG>AS`WI(UmVX<) z^lx03mAFGx9%4f+E9Z~{#2{+|344G9PJW2rwG!mY|1;n2_C%SThogE37UQ>kE-z9oZp$D}OR`>wu z->m9?B7~>SPpP2ALF6R@9+Tr5Sr6Z;iU^+xzc{v@ z7@ID26?J3=`}`}NuF~0!UsR+Op%9B&G?@~&2Oocnqbk?ZA4(@%AA1Y4NwYqDp7T*Y* zN1l@dguDmzky1^NTamnnu{}k$2w(3~nBB`8LB;rwsXaYIJW>Xu{r~+1sE*lD*Jg0( z2LM8Pd%Km50}TuUDk(Yo)XqJ_G`pl%rzF4zybFw-zMX})_EcL(gzvXZSn?yE+3-M3 zD%snn0+R1&7nWXC28WUk#g(5xl(yUjlO7dRXvNU@uql-@$Wzyw|6t;v=vM3TRYa|d zr|~)(VzoMz+Z{w?V$2sY(TUE9KHs8qk%erFVl0C89T;N_+x_opk>Hm!Wc1b+)NZoc zraeosgWtbIQwZ|wlwdku-boar;1;2kUQGY-m)Ool4YiRhee@ehgS;;A9{b4D8O6)H;#|nuh)&QnxfqKt)H+E(_U|p zfF^k6BvT}`QNI|&|0n(Ue9LMl49K0?N&Vb8mWa+=+=UfU`goL`J!^i6e~IrJVCUDe zzgRvn@DxkskUgJly*!xoj<^z;n#yyTdU?EqOm=MCDa0oA2&XhuZ8U8olV7>-zbkZy zs)vy}=(_d0^FKvPb8Po$6|W;}>b!yWk%AIV_3&?uLDVZIW>Dj6y2F_xe7}_Jx@QAF zimG({v~G(u^JLUYB+Mcw*F3Z8Hnkk?j&ATBm32il^r z9XEZm=@dUqF7SHqOa$fb9Fgu_?50lkZ0$au+NmdL3!pypSL_<1z2)*@<0$O1 zK7!=EhFxMeOoTZ*P1P|-{ctg`Xt@&tD0sAbT2jO6gN`?=1GOwnCeg+e+d{}^u2s}m zCI{&0891lXn9>vIbJ&;|-Ffu0KqI}$>}i||gB>irRDoK?I_eyeZ3>*jw0`8N;MqMP^y9h~g#gJ(YeEw$ZET?2Zkv1zStHcgM!)&BGG9!mrjh+my|-zQQ^cH<`& z`!1GC0W2*H+QKUtBfitOLvF|21cJ$a4uH+4+a_1-!|_i-c06}9yTE9Xi3HP&i$&Q( zOMBIcUk)-R*^)Q6mApBrEvIYVW6L)#^Qt5fZPiQ5OWVojrm922!G&SJxl`jS07B`O zWW)&v(XV;hzT~QRfUxdKs`cXK-sQ#6xaD{Gl7A!eRT-;7Es->V(?H7HmcL=OXa8?= zWwvH;oSF}&NS8~)&S!mKDyzK0(j6I8GakpDr3jf&G&*otnH&$!M5iYuNfrC6Z+g;b?oY3&OIXG^@%3|7-~&JJ>)+m@OI&mR<*yP^mp?C^Cg*f-nv)?v)IW>%)@O zv-~nWchDk8AVq$0adB@CH5*vqubAVe(ui^|yxSk7KWd&>`I9X26-x1w{9H$W;Pjum z+Wt2si6qtVi?3nfV*Q`%#(TNI#2_I=wf073R}EtVYJSu;x3oR9_vyZIMDNp{cI+3ux79pQ6KZWfj6ZQ2{8Vqi0bBa{;_DZTMh-Zd7ArMC;&3Se+8O2 zR!CoP^~X22HjBqBKLSpe;-SfyVOXa5eA}7_dOf%m7^e2p{+v<)1Jk4C=6*6F*ks-u zQO(zS23X0hZEa`4#{l>PKZNr1G_x@5d@&t1S@%i!M*1uFVLyJD>~rtQx||_Y`I>*W z+dyUQO!hi&MP&eJxS~R+1Ef;?BZT_XCvVyk&15B#DqR11@RxwK)E-g$1X+6_f_@ia zBA|8l1{??#s^I29E`5?(ZlKR{Ejv`L^Bh$L@jm|N?<3LLkcw6|7*#f^NQs{n%6{y= z(Q+4TVj8F`IALyzl{Sjp%!i@cM%hMw>H~zPY(Kg@BS2R6%Je&DRx>m zBk*qNVoBr${#rg1`WNX z;Hr)PO1klH2`(DFd?dyJY1#S!#}&@5ziSksQsT@;s7C9-J#&yi%g1**aYbCX-dHw@ zPQJctqEF?foyI5R4V^--f(=bctN5h#QL2BvXL?YR8&29nI!x8lfJ|}qV$xe?;{GXH zWpf$BcUad|w^YXyK*x6h$f3-JFfLsM_wVI;MfNPNRKZX_`dYy|a@`HqM%5N{ zuO59eBRJ+dFE~KoCb{s$Ox;FR$v0n*kGx^!o|VTAT`?r5^U3-RlKDH#NVV#}syn6( z0|}@FCU3^{E}(SoO4YGRk6c%$y#nQjo~psA1^k6{96(nJUh$iDjlp@MZIq-L??WJ z&!xa7)GP<`Yv;G`MTFvp77q>rvJLle(O1eUyg#VS50@d!@#2W?n9_NzNO{Kpd_-lrzdq18K6^A_*k2IW4PyH zcXw~MA)$A2dBT*=#LY9d-j6Lg)>P=ZBr3T6TJ^z^W524)<;4NJ`sjy4mX?iJiz_04a|9o;l*vw-jV3SBjk;pRZWogBP^l~9Y_w~(eTf_x{J0JX;{06cNiXXYyLS*x z@3$KGjrQdtUs!e?h${(keRTJEq-$j@ZU_b8Kr%fBLW4>{(eS0DFZsd8EkKH4rHw?X zRHeVp3*=3Pzl3=+E@ZMqR6M2kPCvtLi^RJ(vPtloM<)BymbfU*@PO5Js7S(4(l_$} zn};Les(A^FSHJKYbJsUNMkKhH%C;HQBrzT(cP6mdQ4vZgjMLBwU%$ZVf2G6jFVTL> zD|+%0QE|sQt|Xo57ukG%hfcy30Ffh4;Y<5XKofO4mx`IsSUVq1mlDawL`SZm^r}h6 z_fsj<7*|Z>Uy;tCWRs~2TZ`dRIgW-+9J^Dc(5%|cZAbj6=$R(J;e8F1^*Q>z8{!45 z%V-BbVJ7c$rKi(T(MDw1vL`KUHeyEg^5nc{&HwaZ!uDh4Lqu_HTx}e@YdWB2hV(YE{sXgawCvOlqQ8pOXIV!3 zd*66q`?V?$Pz7wM%cXezAv}5V-_`mr7+PGT5;yL{uO#C+fB!-y$B?*pf{I9%sJv3{ zpraePnZ*XS36P`E#iC$uvWa>Ij7A!Ym~7_7)oFjUE-))(QD1*!q8_$AVd~ElHrs^emr*#7 zkQx7U7^6A&rS(abxQew$_XmPc zy;IcM(gLx69)FTf$(&|)o&9KXQLk^dc?oAUVO!9eG8jnz()CRm#noHLmsF6yz{`S# zDcDKxcg}1aGj$(Z4s?~;Ai(L;mX*}9J=SQK$?=9C2!rAhX~xQoiWM9J>ISKMtnm9c z6Q4xJRSpL(?ip^v2P>qHmh)fqEltS&@k}}s+3;Aheog5JRFBn*Jmyolzd!XABaqHt zMY`KfWMBovK7|@)m_6E|9e|%k(csvOr#KlWh{v9DG>qJb=fjQyXte%aFo)=N%MR|) zuhE(iEqMjaw6u!*qDlK_3Uy@BVO4XK>qeIEIT;!cFwXSfkX&|ef`9p;zkcItQHZWM zT_p9K-96A!*cR@I`O;VJEFc^X5tb$N{FY#KlTyg9epjUoMf@U0;>>c zo^gF{dsbpr!+rFvwh3dISD42x`QgY!0FP!umdLExspVjobgjK^We6!U2P*;tx9OccfV!ZwPdb-?=(c!<`26!5 ztXOn6r>+@YqXboYK0Us!olCvLZQ>6C30I_H)qj3tH`7LhO`FFh_!#a;PSkt17Qr`8 zK}5BJQrCZb0+r25K1^L=!Shl(s-N}KkMQxSKbkiCflJ`?j>;4gPMEfMTduRU%{y|n z=h5MdC>f$8jx4u`kmZ!&Xm(IOXzeywo3W%z8V}|$>b}coIULLOi@$LjN_VdD5Ian$ z^VCZo#pFV6o)xfn`m=o``ZZEYoYjWG^mj^N5_M&8!0Rw7XRvqQVod+wzU7WeSz_ul z+lhca!QvuFYWPs6S-IUzbAv*>*o7+@hUur{^N)LS6VS0}4E;Co4>`a@Sv3X*Or#KK zaq{wp3e|!bqyz}Sp|m&$dU5gXIKs$$x6jEVuhbrX(M{!{ZHZSPvC^r@W|FKOtwz4Ip8CQNVaU?_V(=FH%E{T z!jgA^=V|>ic?3&#R=A>din?MTVtl9_*+gs@p~XiRd8eI@QRC4ssDQ(m8c~n1v7I zJ{vM?)?IE+zM)ic^$%dC|3*U>t;z^1YGx^T-N*p)*9;(+D$rO%+dK=@aZ)qv)a^A~ z%_n08#lOqugHVKXd58T;>{LTKLG8uhlqbfrN{x4pde>ErH#$Pn(CmvJlt5k4x5-%& zL=*!U_$gtpu185%W$-d;`|Bd0cl;`DjfvK=Z0Funjp=6>tJzpys+w!sd3&Fyu-q+E z9MeKb_DcA)Vu?yop@6J(qQtk2`ZZKSU1)drp0P-`%Nf9mqpiWcT#}iY4SaRu02c>l zH*|U4dTI2xqvibH)~ z<=#B+2mDNH$TIe#PS_alW1d_%|5vLQdD^~FsB>UN3Hf(;SW*GJ;ehdJZoXX_-TRY_ zaW)3?qrcxDpKDYK%>eW*Xh5t*YM4Ybd7kc>e$oSrK@6&@I77xn6wTQn2zhG-AB7-8vv5*PCZ7r!t`(L&zmo># zi17SwKK8BceNvN%2op0)Ow8Nuemao6Bj^*KrxncyOtkX+YXPP-X)u{D=W&~}eYUa> z=!)!yqFe$tD(>J;GkdmD6MJ_yFCX6mhTpw38l&s}2EJ-6xj-pk-zpw+Eaw`-8Sb>PRqq`c!q`{_TD{=;AzY2goXipgJop0*;|IiL*_UKuslqGmpS{fTjfsgCyA?Fhs!q+NyrZV)q zN2X(z8vwa~@lG0{d|&81hAOaZdD z9sLyDbd+6plLOdkAy;5?@#eHBEcyr|Tl&xXHcu$Yb($)W%f3+G?E4FFPFqA!$jAuk zSBQ81jqU+BZ?B>N)%{TI2C2RN1Hn(FY(4wf!|~LEyX0!#G9JA?vLLv{UD;T>u;kR# zkR80Zl393I&}F~v#X<^je4Iw>MxkZ1Wn>qdfb9`zY40RhHP6Ch@-B#G))3N;N$|2sdJG<2?aU5erz`Evy+g z2m=UAUxrjzzV-LRM6&QZ)X7QIUm(5A)s%_GHs^g+j`GQpi&qr-B<%s zYNpz4Kb=$|?`?k_uP-xbEZI8=Rh%!+aLsnlS-{R7h0&x)2WOT&j>7KGnDtex(^Sh+ z?nyLxY@F}S`^Y6g1y^p_ShTTLEH)Mk2@qp-S-YR?a-{2jg~>%O~8Hd8v|^x?s7Ln^*lAE+Jyd3;GAioqz_MIpEBb}Wl9 z=*q~K+IOeVPXuSZ>ZQ96f&~j`5-t%4n9!hbGRN#%o-kYZ#VurKJ`2lJu?{oDJrZ%| zL9K~916UIZ#LsOJdya@h%Arh)pRZkxyh0%aN7-psUcX3fH<8Q&vtlUp;Bp!U4t;a$ z>A3H-7OVp_oW`KBrwa7cCaxZvSQmXa%5HfNQXBVz3OCZ`R2sWVpqB-kbSy~lQ-)1FZDUuPHBPT zpLjqv>kX6?z^d>o`q$PoU@x9q`e_j;M{d6%`;k^x)xWsgkb8M6I@7T+b=;^F&_(Lc zRbiA}enPuGUxCv{p(^yLKqi2(jaAJI`bZ#a^Wyn`W;NAT9Dz4u6S&kT>kQ0ZuJaIB z3by17qywmoZqEQ)v<>{wUQo7Blj---rmAsj}T)SGyLP z%{&AX{9Emfj|tZ8eoqfHy`=N1I|{^6fPKt=BtcB_vaWfe-ng=ypAbPD+&+9ySL^DOhk$xZAW8D(*WQQqChvZtYDZ; zd($&WLoc!4>>k(oRPOQylG5A$S|>?EWOk+d8zUU?D)pDWgktWG@BmPj?!%GeI|GEd zzd{3Hlql$+aAsQ&iH(Aps^fa<8Fdn2XB5~G!m_^SdC6#VjUd74jklj0SNwy$)#;`> zW|G&D-j{Ujndf(qATipT01k|3?$ZG0!ve9@!$JG1%QpkG;x0I zYH~`p%mMeCyWGy;m(FK?E#1d{KXW>M)Y0?U%Caa&aPefGfA7o_$>?sqIMi!%->n}H z_zoL`5=;4m5GReBU{FJt1U~TkgWIJ^6Q(phs|KQvwc&^aPBPyzMp-1Qy)#uRGzkZd zl@Q>@W{4&xcfAIopX%5f>R);My(z)!p)XT@%;)G1u)k&k+V7YRItg z=cgB9`OS26X%?K)UBoL>&jHJ$&d=?Ae}hiVKm@YJ@%JB9kir|+3L&g&^*+rvoo0FoI#MK+Aeiy4d?0`XnH>+<_P6k zco1xi)irXrd;6;*v6c%14+^BSW zYFn`m(}Z4$vQU;A-DpaAW-T7+%c8_nhbgNFv2p~1F5A%v6=~<|ltgm*-` zx0(spZh*)5GHzcG<`}6eSrT=>f*Ltk$hZ>CK)o+YElW>WakU>S_C1Wxb;A|J()sSF zIp(&zO>6D&0O>1$;l9q(>*-Q`xm$b8akVGu^9R`6Ls8g(D5=GD68I7$B>M|=MCN#e z5N0Y|GB2^WFC&u+pO+VbZ*rU#<<8k5xR9|&mL zLyAJ>W^k)^k7M|xzK})3%G{sdYG6BlIF>^7BKN|&BD-K(u1ED}^Xl$zFz=$VGtwy^ zjwZ0Dg#f}fPL7F3wfxklVnzx(yrNVO-lp;&*B;7$S#3bx&|SkW%wrTXKKVZff%2OX znd*YV5MHyJC^aVy4SN|!Ra-y>{mJYwEW+bG`fSTPVgv6@nhD-P>W|9h7!_kaExT9N zmc-7L&EW&jzpD8CkE62=YqDXZ_~=GP4kSkor9*}^BSbn31nKVXu7R{jcY^|=($bSo z>F#cjuJ3t2{)X#fySDqe2c;{2v&Rwi^cOP85o1zPcLfMT=ww`NR4<1$uBjlrDumtN~ zI0hRO1Hc#j0N^kc{Sf^>*nF7ZSZ1f(^pn>y5kaJcjbVLk=Zq3*Tx*JWCj zHKJ%BWV!D1U*O1Mu@p_~H0L4u9+5z#ZB9;+vRk^QlBlWAbB3h=z;QTyONs(;!LNyW z&=Uggt~CCxU?z@*YDL8vM{fY1)!UZ;$~$=HWmo-DGSdC!p^pwCN^02bJ$DL$SSLlu zuezh#tu2drA+)PmEm;rmv^6ml_tu^_M-gUT&m$u46Naxu&WY8Tv;y&ulBBLS&<`j* z$L`hn&5xtsb^zhDO&na^1d#Be<<0f&L{-utPgsd@1DQEBk86oWNItiWP2PS323CjiS;Et1PDPEMt2q|wsg3W@#X+b;MC|x90E5sR&^8PNUpJ!$-%C? zpP-4%Jv_Mu4H2~ugqJpvxhUD-_g9t4y zV8TZlC|@q3f+r7Q{|b;OI7K0v8YG;Qt?Q>KS$7hOa%zJHEkSQRLqFJC^T3(v*6(e-Bw&vp63n0Dd)lLvwi;3;f zKFlN9<2&aF+E$k`KG8(ytsMW+3lt5m5xx+XIeD5BQva(3i1h@BK8q9}J@Hy=7^{}} z1sQHoG%8~Aa5}H>4n=4ceTeh(Q!pVV5C6vsiBGpwwXYQid_}W{QV5`;QX@Gq932Lo z^MBstLZjgf#(-i>Z*%99P6c*(BUitYk8K0k@)vz$_VCghKwb)>W3EFTGT{wlHace%isjq%N-w7QI(5bK8U}Yc`c{rO zp2)=BDM;#Vj^xZ^=RS+H2z9$9W^BH66PijSm0O22m%sU<}TKn=9lhmbw~ZE6eZV ziFcJ*20V&Y)%%o}z_~m5uw7nU>e|4@Bt%jDQ`6Jl>(hf9yg*XRBYsccz?RF2Y}BrD zxBKD(Gtc3S2d&~DMAybmwxlHB#6^XV>2Q-+w{J&GEz?k+VQ=li>lUuEvEdEuyg$bR z5QQVZeaqzttlM*3<=jjOlgRGOlHE}0C3=(nI(3STvuo#cgT~a|Eg-}aO!Gi?~_c3@S?Gl0;c>gY-fpUT96b_G1% zoIc!OYxG_EJ$Z(@vVK4&0c*(GZWAIwvia`#(Wb9vz<%hL>J6(dTBI+C{fNJEYKpg% zkARN~Tm++X`gDL_+*7{|O>|Lh;0-Vf@I3O<2L{%ThJgY4xbw3!0rM2iD4Rzb(lsZd zUqJi}o%)X^W;)LfG__m9Ln$mhz>Kp9$U?(8&H2OkO$_S(b2;7nMv;KF+*5zG`?n&MK)dL<$>NMA9Od?`>4w4FkId`O9EP} ziod6{Jp3-#4vz<|;`Xm?)y23ajiKMj)-ywL{}cEu-frCS54-SQSZB#R=BxFeS1CMf zY_N`yLZhTguYy-G%7zW*-nD+;ui}0gds7|YEuSuD@Yl!3%d4!W>&=XG>%+DP-Y`Ga z?Qo(Ctju&e9o^Pxr6|*&|CtenE&n%Zt)e7qFxl=;Nyke@3F(n78DxgQw$xK@DE^1l ztP!M%mKWg&;O#Fn<1M(+cLkh zdAQQ$EnHx8Rf{|o%c&v@dWoD%e%0Gfs0R`+(f%Pz)nqrf;auk8L?)w1G5+mR-whe^ z5{av6#EF@k|0Ny=4oT5Yt(Ld{7V9>%@;&UEklMMrSH(8(OSp#(f9<-JXo^9=^`7&s z_7nr)(7{UOBP&tX{#l3*%D8QuUw#nC{v7JsB(A-HT0kyqXX-5w>fU%Y=vk)?F5ey) z#C6ZQ|A|s<>_45R^r)&C*ACLtG0Vs*50RKvqF4;@6K^eEkbM3@>ASDQ8tv{gRK@akqd~!XKVBbawCmB5dj5;w|S!EkFmHXz-Qr zC%xr4swyW~1_n)^qgcgIYdVm1c;>uaDlyqHlSC*sOHo{RKw0@yet`jdf%E;4Gw2>M zbVg)Dk1R0ZEVEb-WeGO@Mkr+$?e?o!1tmf?Bv;3#gn6ddkn~slt30{m8@j#CTki73 z%L!>Y@3|MBiTrfHbP{XlhWkwPd;COa!{3$keq!8?XA~!qB{o@7ZF7CqiJMAcp=29s~GJFc%%QV4wQwWl3w?J}5? zG_ApGu5+47sQp7~Mi+D>x67n?Oms+ae1p#FvSCy1|l1F8VZB5y^p}- z`kZ8(U>5F7^^PQ2{l;L1HD$p*4l!aSqtcI4?;6+pdk207-6Eswu`8 zYPw6nzRJqRxom1&?w`h8t+`aY99mJ(#B{!>r35^V>n)l++yp+uRjQqc=-zs7-eL+j z+5swfriNc7i%f)EYn#K@xe0V#=8Elhi99?LJ7ecq$e?zHP)MF} zT0>H#mMSbLFR#Ie!+w8x;KAta;PCflS*;MtrzO)lw~P~cezyH4?}ssz(8Hh7s!m%IOSYcOnU4EB*zEB_{9 zUm}Gx$6^T1FRY~`MSftd$c^Nn!&f8RmDJGG@z`(vw6k(L`9vbFDS`(}qa%fe{pY1U z_pnuXu&1cXxvxY?(nW17AAnVBNEX49Ug~Q=4c}c|;g%_)B>(J`*N8HhYxe&j9Ao!2 zEh2X%h)3keM|Wp<`#XV1Xi>R~^ttL4d{3^W`^50y&u6SpCHiH!W`8@pvHC@V8 zRaLh?FnH0?yaDBLChV=?UT6<06mPyG_zUu85)CviL~AGxQ=BiJaLK`rVaes3t2zhY4{sirW9)8$QE20 zV$?&GjCEpvRKavAm;;Bd1p=(2rAxrdQIVwUS$T#)<_bWV6c*yiq-~UyjHQ*5L3dF8qdAC?HfjH3s;#ZuUzj+?7Ipk4uTNXB4 zm8|ANW!~+8$?H0^qdW<}{#5#QOjzF0_WJu5Oo21@0-L4TTC7f8?E!Km5_YztvC$cx zA)=WzT>V)X@j`3L@EFy(`>9InkYWSL^&pFwBp3u;?_Yz8O-qZu!3v;B!(f3a5UPwH;)9wmyVt;qiuG8C1BonE%lhZ9 z)!(DIhQ4E_NF|N*L<#3%^qI?GOd*NgxATte?BImX^=8^Hpj`4v%gR~2Hbge8=3G=UmEteCz_ja>z zyQH)z(Lk$75KV49XJS1hKbZh8{Ddk;26E}>$$t4`GWQG^1En{WEha>QN88k!1K;-K-nlp_wga0+xRf={Hl}3lQ3X0 z4cPAO3efmwTljB+t0aN_Fhqa&w&ifn-RE1mEMjQudc&K$Xq{ib(!nfU`+QA8TWJ65 zd>oK6rTIN#kxEM0>L0hP$FyAGLBXZ^%todFfX{K=UP(=G5jEfeDytV3h((PWxb!gn z?L;rF0ZhpA@>^qoFz#d3R{HL>ulrPU1)6C}B3=vM>!YGT-D^pU(T_BfS@ z&&ik392`q~8Uf^Jj6dI^eNR$E&JFxMu!SKS9kp}CV#lE~p7*KjeO^}E#)_t>y{|}l z8)d1!nZ%JnJ^K%fDk@!);_Qg0%IDkrTTrf^4X2(9z~YNQBHYP%S&4;38(z|S{#!Y) zT0V?2v}b;p+B13205F{KKmyEAG;{7qS`fd0fJe3Du`!Ay0VlPsniBJk(*bs9TX*D> z)IW3uf9&l@G`L(Kp4eF{S3#r>lk?FX@u3{d8;R```a!%T2igPzH;PEe4?4G}d~K;j z6o#vghk{b*Z)MLq@UlhyK_pW0&_KT#9Wh39OvUUFt!X@jE3!;N>n*XytW<^cQ?~la|Vt8*q{A)F*O#gdk*7C6m5Zw;= zF1_(@d6`Jv~n$lI5zEv=F=IA;jwKl)|$OnEg8EO!*S^x@V`Ij zIi%-y?foUXsYzn1&w!09HCBkkMU`dpuRfmf0Mc6E<(M_p`O$>{@jPBLIw%!l41712 z3N5S>Y2uN=HkGb-gzivp(MHSTc&HhtED=~8VH9&nyj!WEGZ+n@;y2(n3V+|pz^pr( zxI-`@g-8ImH${LHViS1DZ6F0i8c7|w|Kw`*d_VJlt6P|NsSFQP22W;wM>SCG_$}Q) zOkCM|%(1gI6PVFh-b@!3b2{^)%i=ttz2JdjpIm;JyOlzT=4|p$jb)}*6CD`mXQqMm ze_H|iV*@+=c2Z?M^m!kFpUMMd7=cK+C-$?-x7!5=K6A+}RctaSRu$Hw@4sqFoChWz z_0Vx0k;p*ghIUr0Vfxmnd{-xzs3z?yQMjLyqdy6ssGjGId@uNS{=b2SlI_9OXE^Lq zQ2cAbZK68lQ?)`E@t|unl44@@7zTM)_*Bw~-mh2^ z1GIdCpQ8#HI}0NsZjV5gW8=EqFKNqye`u#Ispln zR!S1sfM!B+GhWLOE!>Sh_@FyNIb06r+flS)^L4h!eO9l+Frz}ghBPP#T*#&5Dnxq} zmXfGtG*Gq)wANT0oF(vF$;+5-@ithW2MAl^<;+j82=l6D{WQkF zeZ=;*@p*sC(H@WufM>XO_dyBvg&$Bvt+PP$F_a@xnZNn$l$wPsoH;*w)wz{nZT_P)pCkL&i>Cx>1ek zZl?E(sniU_%{99@)pW3?M77N|UMy0-j(&r*#fG#n#8wC?=_$R+Br=Ky{~8~qUFgAd ztH;Y5hEIsL>Q$w^-hLzh3A^=^k_7Hk+Rr!$bL$>23N+d$nam1b|N4;s06TCkHkD&% zEDoHF74ggOh!7Sl*&|I6<-5ox!a{Wz^YIu%v18cjw>*#|N5_EQ-pqiYDo$`Yu>gsa zA(4QZXyuA$LnJP(US7YW?1*9QGVW3m`_aqU;VMt9+*}eZ!FVUpo3W$z0UzLCLy9Wl z$y}eQ32IskWLIdJOlS6#U?s*;Fjl0c9m5Hn(B+w;g5nE*+}F)F0N!_}lcvXZ0VfZt zCzy=Iw;VsVKib5!%nfK0^H70OH_25zl1d;tT1xjL!CKLlC&4a5Mfv5}==8Y!Z3(*o zU2*YE!o#UER}paZjN_A^BZ=urNW#^`Hb-YMc6%!%r%>GaWlq|KNnjFTzK&a5+NmAZ z1|Da&B_*~%7a-9m8HCfCYC$=}UUSDbtfSz6nkWUHq>!qE3{BZF*$g{6Gp2$F@9;+J z!NJccl>;gl43>(UO5%2xNQBdn zV9=yz%$#1w<=#ML_Md5gjj7y4;N)Y(n&);7{4-RjF^WoZR>B&k>m^izon4b}<|7!3 zD`GI6m0ytWb$|D@DX;-Kol#-Rcdqk?09s<`dwZUg@4eMDx`y0@a$jlPIYc{=QswZTE@FVroX#LR4(0O4oK^ z5~so%D`$D99&I!(D1kSq;ANzs3>E>N+5qjgeEu^q%j?uV}5l8 zerm1!TSa_RpBl9Mgdi(g5gB?NH+!#x%>Q1<@OXC*crf0U`*NldW-eqoA7Lc$txUPT zf;GH+29;(YCwK#P61*gALuQj|3}q6af1QOq4M$#Hxg9Gb>uZvoz5V>$E~VhBkAEZV z`hQ5!BkAV=JhE&0aLSq6d*Eviz_4gVyEn$8*U^x`ICg#O7dfQzSE1sE7(P z+9fyrIf0OgM^v!oa446@a6{LrFk2+YJc1?!nO4N2s??Lz7DO1CW|B#3LQ89b=|XF4 z63y1hDw9hKo_8XgnXKiCL;6mvQ3Qwd0kl`^ic8|dmBsf;(j|+H<>-7g?gzU@|2o5- z$c|3qG~P0}YRd@R@)xonBNQXO%F~XKO4ylE1>Ht7puY53LRQ`Vy_A z!1(lJy>Odkj3Fx0QcbBBn%HVxhf5YKJxP&9uD-%+7hlOguu$V|yE48&t`m}J0xLo_ zB*VK7>S-F*Mc{$t>$TtUuOFO%lnG%LJ$Ez#^rjVUHJ=YfaVLdhWPxSIeJ9iM@YG^v zvT{{;{ya|#*?q6-y5!@@6DWQ+7Bp(G{VkT!rmh84^dAD58=yyFR7pB6uT#W6iK*70 zqG!FcqkHnL2Mg4KV9Jxk4f7jHo0hR)oM|x@bsnlX0Hzc(-6;j_aHNyxqKdFtW;=&GMW0orUSAdTdcQGV9}=Au$XY#N)#yLdKkWv03iw);7h z%c78$*%pqN1BW7wU~2Tb$w1yBK}3ofHMGI8#4r6L$q8_S_}HeswNdT=c$cQxs0Vb? ztlOndZh&;m&mawOM6n`>KxLP*883)kC$zsj>7Bp3J!%c`_YeQHvUUPt4%Xj3TciK= zXKZKnP-BD6a88&}6{Z7-Hbk?y3D%fh%ai9GumOE*)~HwDPwNw#kGef+uMG>LR61wr*QGKC+5+3!6DbR2Z#2f z52casQZ=G*+5BU{qeSbae#?A`4B2I4S} z9>^STA79gzsq=H#Yr#sWTz}PNarEPU4+rDn^x&Yb^?~b~p;F&Q`Ue})Sh!8+A)9e}>|b#yRBk?R&EUr>rW^6?yK>2ZhfMQW?}ZB>^iBze z9QTpT+FX3;1kV{Z|ExPs^f+~yK$N>aJL4B_AKpG)P6onBpmSTH9N-gV}R6orz@N{Z5myx*51A<66S!6oc% zBhp(S32~G`MrbjTJSB9M67&*%F42E;b_!B7S$N^?OWM!vmc1*uQC6I|S4Qf*KHJn( zzi+2st1y@XYBJ$b(oWdt79gbHob# z=xnYz;kwRr4cse++dk}Si)_J*M2E}!y1$#_7@zB*1AP=aNoq#BkvJNS!^?7YTwRBG?0R}$qMJs*(M_Eh2UAl^&~>!>g@>)p=g9@z_z;>@oB=8>;qX^KqmcU* z6O+4E+@lsGd^lEUT2Mx1$M^~3ZM48bOiSEkvWh`s0ep`qVZrItgcL{H*hz82Tu!it z&`R}jIx#Zmp*C&!6AB06u6T?S%fq1odH9#pcT?vlhMRiC(Ash+1f-|>`8}To!f>-^ z1F3w;XS)PsjXN7nv=5w#b$|HeovyhX4k27--&wXSI%{8w3(c)(9E?tAWxoB*w-*v!5OfegJ`}q}sWBp^G$Ds*!uldQ~;4W7*mbhM7^=i^5hOY1~fzLR?bCx@o=hjFBNAN`GB z)~z_p#0LYv1qLn+gmGJYu#S^ia8P<=>4Ay<#pwCQd7?{piB^0I#>qi5m8f{l&om_B z2C_;sKDu@)0#Xrsr*7b>VWS{Z`*e~(p(Py&I0o%Md^#5=Dc05;YKs(^8WU4tFHBN$ zm#Z;aUQLci>v%=};b~kqban{nGX4COKp^YxK8~)&ZGu`q6LV9U;^xc6KSYEkLI*vl zhRr0B-hmp1j}xGt%n>8xKHx)+{~gUNP1{TG1qC;1zXI1tPzy>)x$jputt^LPeGW-) zit5^dFKb{3SkyMh;8sz0=gg5sC-$fa-QK()2H7h}@7(7~GVqgVd9dS%($W~yVTMYK zyAk4We8?VR52zePB1v6TcV3OlqN@BQ>96WNx*^Ba2zL8r6Al;?0a~(VV5*3KKuJeg zVIrIg)Gqnwl6I7(-HB$c`nkcLJ^KJ0XY>LP97#$_LK*JF2FC=!vUBdP73T<0g-}g` z=5ChpiH|K-S4$-xc&kBvInvK+M1Z-y%qG4dKYi|Vjq|>!OYCu(2u8C&VSV{;|H8UL(x3hnG2n}rx?3nni7R?NFB=s2XBM6gD&N@|J7|9$+1)$|OvS2c z#JZm_r@9P5G`S=g)o?;gCU+3j^==i5gl{8Je*{{BCSziosCjBE{LO1UL_m`U19zdF z^frgyQP?Ah-8LK%fn6$> z$7ijvOU5?5y|{z`t2@YL_+G6*Z&j&Naz1Aw?VS4Za-xHmP9YPSswD^pNu~0x{P*t| zbT+bx-Hb7faRLc5qFSn`H0PcFK=H#VGYK8+Jp;USB-zmz%4)tOT{>To=zzJtCa(_^ zxQ@9Gl0^kiSu+ZUa~3_P$wmjo;yRI-?5sC`az&Fs{`GUH5SkMpC8!@f)$sPKys4GyN)3%)|869OJz@ahJTYQ=WveES;aH^>b#Otu~@=S+3x z>Io&N^Fu;DApxVqGujgxY7V-7M^qxG3+@lYR39nnGK!wcc-dH|J;cmcq$G_8=JiEB*Zh5`*u8v)G_}H-=pz1Jz~pe+4#E8AH@a zofvMSznY8EY(c1Hxo9o%=-s6(y&Z!G@7LC37mjh+lr_jmP9{E@e2b2NhDRC_u8oP1 zfY6(J;oaD7>1@T%jjMUGw!D_&;D7ZO2K8kA>OISa{R9UWfP=0{z`4miSBC!t#P-6U z^07b1WHj#d4UOlEeaVOTQM6wG+~A!Z(J9}$kBwX_kdmP#)~AI~m%fCG1~bj(brF63 zpzg>y#Qn?UEd^p--vep1nR}m$JhfDiY8M-B zphP@pWm3JYDkXm*weGv!cTvP7$Nrl38>x`T%Xl-%%-Bc{aK_BA0)S2Z;Gwgw?F@J- z?LLTsFE{eaf%qVwfPgp`+7iy8)$Ib0NTS`nJ)a>U4bGW3u3}k&W{!_Hy)3vqoRrC5 z)m6=)SytJ1RG`bkGo6PQfC9X3qnyhFLSC-^S`EQ0p!`@;%z&g%KAPCjD&zEnlcd=4#unKq7{>~u7c z3|xX4$`SDvQ_Ko!!Az_qGL|?eE%qXHn0-Cdg-GZ#GhIsMQ&3q!T0%yjWkGk6qXB*Y z4aG5UH4w3!I3xNIb1_`(H^al5~y zwym$e{xGNJ3^V9xY*lFyp~R?k@^)fb3c1}efFpLbJa zrUx(juPZQYklKM{h)=TLs6_-`GQQA@G zed4&OxOY#3ABM#JtdV)f*sAGigL6vVum9pncRNpd6tYM+s)j-0!#ylQkRcImodEXD&XWk2{gv3$ z`FH8%@*kzmp!pKR%cAYdh#ojVuNf|nE*z?a%?2e|CJ?4eFyHhS63p#kr0!oW3!HzR_Rq!DQDLm$^#JN zA|A2X1@Mz&QQf?ejQ}^(0CMI#W1E`g-egVZ@_|g3<^_9mN@9wlr2Y-(bROh8cGqsd ze~X36LTW237ADcYj-oSZd}baFCaIkCA31^`|Kg49<|tVr>V-QXn4ox$l-0xhmN?=b z;t?~~FNypmGVrv<#xg>Z`>7*IgpVP@($dgeHSlnEsPYu^CI(xiME^FZ`mQ&-qt}H? z2T|zrDZ9aAAwnqsl$IaRpjesjW#i>FIUC!nPS!?(<)fO_rYEO1McQt+KL zVFsy1W&J~P>BGYxf1kfB=+jt|!J%CcH&JD3jI`)S+w3hM9ym$!P8pEw`%xs^-nSfx zBMEiIF9exZzgR3=91Sk$mhfsCyzR$oPY@tMW0m1=lcj!C4%ZgbX58)~1{0*>#_+-~ zs@N}IwJ&GF6ghGYL&FPKp~B(LnzGVLitCoA(VyG4V4~jYG5=mi3o1h0sT`fBO7Ks6 z-MoxBt^}PUB~=3M&+ClRwbE3#D!2ErLs`F!-ZWMwS*@nK6o=}kmy@8mD22jZNa;-w z2Zt&su6(S)Z#5Ty1`x=Aa0HuUMHFCly%BfE>P8&jSG-&h4p{pg=%&#?~#Ck9V_xfN8YX`-!>5+KPi)dvhRG=lo{Qp;e!Ii@kM& zvA0M4r@2y%>VJl}v)4m#ln$r7yB|4d}AE`J8L^zjupI01&y zN#)%$fQZN%ymsYMd*O{c<|5O_nt~4cFgH=5*_G(Ep8GbvujMjr52B(2vLr#4E7pa8 z1H_#9l6Ofw_QCryL1`Rhk?#%W(5~ur@=@z1&jHEX?fn^c=vF_inzIe?m@o-VNV`~c z((kr(!X~Ni&0AB10bglWW;Zw9g3vNkmmU4~OaY1kSk^dKx(dh;`ORoq8-xoQ-E{_wLsf6(6+=tL<}c3uTbt-HY%+_a{__+z zN@%+&N*bsJeJ-_zl}*y%%Kr{0nU_0X?^M5);_6sr`2|0rh%0HHnqu+(hwtnAU|#$D zr5K$-x7I6$oW;F8064p&*2@HnCqNXFiU2$SF(Q=7y&8VG$=oan!N;RU`}ep^PQ}`c zW=dtypaKDFy?@mrK^&0S!)j~zP^74Jn+wDnAOYa0M54-WYP6MPa2k`5L+%?xB!@b?8Kmfhwm?fB=4d5|1K_syK4~<9KMOUsJa}1}Uovu_oFW=Sw zkelW-B$7w0KShW8Ck7GaGX&e+Svx_7X})v%Q}O%xYa5?5XndNzWs{;`%4MK@;=W$$w37oGO2ltF+w5}sKFY3JEYYD-bzduW^LHi$ zX&&Wong;^F8=TwxHy}lGLdvk0hEdBN-4Zjz^!I#%AZSxuw@LPv!%~af03X^#S8n* z|0`C34RW^W#ch1WR@OGR@i&F_B1XUFR=0E+m^ zOE>TPlQqD^Rb#14SiK^drHYhbDU0=6TJOC)`VWuCd@1;TecOD9yBrg;e`h#}{o>QS z$`e*M?EvvvB_=2%X$lj6xKqa%Y>A$kd?c4Xa1qk!$QUIMhLOx{XsTJzEp{_O0;2uq zB=BV5-ms<(yqb4IL2{*_$p|V$Sh*CGF}g5^;ctne<>gN%%GOxr_uDN@l7~H>`cs=I z7KRlo`xZ`ECBlBQb_2p(8zbV4g<(kshP`Rv__N^*pYk%6n5jw9iuW(XgOXBRbPVOx zI%b2bVUyas4UM-2ai%h+^IS!;U!JNdV#}e@ceTr#(ixKP8osBUv+V&SO8whKta0kH zPr^bfY40X}Z*)x6*#IoOM!k=mtU_d}jSh1+Cte2Y_xFD3+h=Db(~4mt{geKTQ07$P z&&0B1OvdgWrw3va9kq-8r7=KKEooeLW95LA#9%)&Ncn@X3vE80_OKHBMYxfs z^W~(_?>>Z!Qq8lcU=};K{)^+~?0r8yAg>v}Bb%bKv6r1~UR=wj> z!~!uklGw13k)Sbk55UBAf{Xh=X)v%LWWh(6>Q}u&7I;2_SVzUB9j}3(IgXFMk6L)WGx^x?@Hq{bx?1aw=o^`03RyrPdF_M^_oF%bu+)w;aOB3^R<{N0Va{NYhv)eE zx;c(Z2Wt6vRrNUB`fwevxYT;z9dI}NpLg-xnYz{Y*6Y^O=l)#K1N=Vu7C>y2d{5O& zZfXJ?*^C{)`li-x-E>l}iByVv5hoKkmj!ZSNXfXOyKED$O3?mWuL|Zg>k9>P=} z>U_p5L)tdjT3++G#CrSnHktU<0tCojaK=Z6vvfQ%=sz~Fz!PopLiz?kB9V%kgXZR~ zUyg__?itL@EH*C=Dl5yQaoIu9rMnO6paegQ7lnl^tgk|wI+44hvZ)h;xCx!s@p0Q( zahE<*QB7qqa1pAL7a=+8qn2huPZ!csWC3DCpD?d*H$nv^I42{~F?%2xj#DBH z=ZnELbg{3x_5FOAR#VOhKp^{>w~EBe3FwvO`gLDfOGYbDBQOhvaojpUXa`fVf@skg zLrTlf?f{cqkq#)p&kw**_q(tK8~9(dKKU@Aj6nPQYx0-dDulN{!;8O&eB+s&ag1y|#ZVY!N0r?Of;gp~ zuSsE=f*A^-)t3|?vqk^SNQlWv^BC%3Ft&!nl>ZDG_1VW6+nrzy7oAay7r&LxP)g9q z5hF~o0zLfQFa(2Xb%8TDEn*m1Ps8e^WJNPP5a}_g;HwUL3}5fMV)~f94?n@TUS9vM z7EA|@qy~`!@3`hjP-ik2vT#<3#8Gm)8l{_Lup6=M8WVh>;N~uk6un(sSP1aQAt3{%`}d7jY$L%Ok6N zXl-jJR{;7qZgO^t2T;IGwJv5~2L6#_nf%X@X2VrP?(dk=pN3pq1dMHSdvfn<7p=O} zr6)ey^1l7%hvQsQaUo%7HXpkGxM6cvg^BTN+Mt*MqSqFgK;v9h!|YnPO*}{hsLuK1 ze|2ttiAIjHu02*U{0pF;QJ~sB{Hv?-N*zgNqj;I7j!PI1hO=C38V2Hq!Y$s!7U$bB zfp!eh&;uQZ*dwdEemuv>s!O1gnVNhmz{H9%$}O-Zq{KXeZf%_x&B7Sj;ly4 zyH58RURQAp*3AtN{c!+A9#TJi_HELcQs-+83E2LF9~r7!H$V&s zBmFN4b5%XH*a{NvL<=N*lvqVgys^QHibamUF~lRJfB`~wCPQ4V8Y;lelpFU#;`X>Ku$U#{(|L-9nnOM zyIlsv%*tlMKbN&piuiw+u@Q&?Qe;$N1w9L1lZd&XiBsf?bmmKYNOI^Qw;~P^wULQU z+oXZXz}HACWxRIfr|+igOGLg+)dU;a5DirLtm@^7XFu`fHm{x3+?%a&uV{XO@#3ha zvj6nkY}yOw*H^#4?a}5NKwI?G6d!ySeWo;L@BK4Wy&&)YPFc}t_X}oY>>*EH^`scP zov2lFR@4X*mW-8s6F6kf#HHD8{;I33T^)_J^L>q>+ewmAtSwE1LkT8nFyNGQxj39c zwdw%6SvxSO;i}1w&;WE#dtS5(34p`yzvG)%_W9#V!2Oej=uI`NB&u(^&uBR|mp3&xA0E=a?f@1R4}7Q_(>Eo|cukM9mGs7;#%#nXZOw z8)EVy%7~ddcvt(or~@a_tc&+!=4g;xVHfA6RMH@|KaVyQgfI;mLyvbwx<6}c{}Ltm zjCp8>S=sYy-R>*Z)ckjGdj;-z&k)(ts4deydDj577~% zzeB~~8O_JI!^PRnxagi`rgK4^4=Fbf{+~SMq4*MjagiuvFlR*)qy|gd;WHsT7@l-I z2(um{G*q0WVbE21gkd;Bus2HsZzNt=K$*~GBByb5&P$nzIxYB;a)1L2;#4%{x?%FK z94YCaTw{HkpIlL*(2xZ%FY>@sG>d$GaI3px;MKdPsUMTNa)mEG$%<_uAPS^GAy4_u zrQ8Tjm~AF?t(q^d@?fuMzctIs$sPB0r^^1zN<^4Ti78{)#xu=Q>h%w33)){=c?7%G z4Np*9U1>yhBopsvSOmWR<_4UF40iAq`*xNJ0ctljOajeD6i&ZFUUFRny@;&Y@R(H|X~y;D|8hCKd8b zdOkoq_O@p1+V`Q~;E@H-|Mory7<+B{SRz5sx5Q#gtkQMI;x2C=PagqDu0g9`Ob6?O zSE0P#*?R;DkSI5IU|!sf(R}^%wZ-8IsYcXmsIkUER|MG7pL|v^c5`KfNR`+!ir5kI z=nZLH6>cnC9~mHArIk082d~?&cv%rg9pZfjnOOkYlywbzn^dp^J`A!x}B8#p5 z9%j!x|C9C!go}TOJ2*IR8!x!oE^b?V&fmO0&-|&I*36c7#Wh9J*Zp71C65Ana#@;W z&M>bnp^THBcSQ}5l+<(f+uofcGxc+0k>J1?zhuhfQ}9;i`yRqB4PY*?$|TXKU{x)@ zIpG72LqlrN^x~rxYRTcVMoMR&d5fAiO1mz~d9_LYDHhJhSZZbWY#FS; zoCxMSy)cd|J314^-H|LtnS)e~5R}Ha&@wlx1FPL~>)nNae3Agh;6%?H#2Yx6IPwBZ z>gazIop&JAe;mfo$O<`BXYWXYak!*(2E@s}ho&b=g835>6RsyX+8W z`+a}^;V(D7pU-K6N|^z9k{kF&=oK!*x$@r_;4<47c<$ zr0#fkH^U@%FqjOnAg$g>hc*aqi{)u$w%H2fMv@XnJqk&JuUb2hm}<9n zTj7q}xm`^7epR|@rGV=G?wPHQ6>}nTIwtr~qQ+^rgGq~#2CP*~k6>~1JQ`eO;P?kL z$=fsk(^yrKYblM+yw;4xbNP;OwKv3g*cyBAvug?Ow0vzO=W~S!h zGoWY$e^o34yspD)@o-b|hJ0sUA7+q^;5T^AN%1t6k<$+^v{Q~@{cxK*2{A8^rY@+& zD+26qImY}7sW*9k(P)C^sGBkBriUF;qXzI@D0x;vKN;HnYrUo2+hNv3XN>uqAw+5{ zSr6Gsw!2{n^vNCD*`xx>?kQ$%_AjXdd>ifco`KL zmqhQlTj{sA$+dnvIPne)u3ZaJXnrM-p%d}AQ^?&%Cg#=ZP4*yV>O$D0273b5t`0A1 z$e-iy2jT;md`oiL9=n}5?&I_x+O)si!G_x9ApyX1-0?2Zh-w~mE9o1ZlTQDN!!%1l zvAEgCHD#5_pe7Q7MCgW4AvAfSHaXLxEN-cg*CiOeLHPYl<-kakNj5oU?U*V0g z`_ts+S*j)9(p8ou=d4s^FNZCltZU2(H08kVvpE#40asTVA1B4f3rOc{qymz2-{i=^mg@3MN|AMSE%ihRO`+piwBW3HYtO2{G!jCZ<^&ey8f4uzV|(2K(Kw;8 z!oa(lj-2*-7oFcxjd$U73<<9YUh-VPev)i#kRVtmq|5G!j)0WYu_!l9&yB5ZTg)++ zUO?cL<@QSbLU?ga>}*9tTa9Ee525H^ti`t8>w+ViC#ygL2f%ndI+J+h1kWdJF=HLjy7gtoi@z54^vvXA4*71r&1neE8ldR1^=5`hS3Pb7nScxl<5dn^dhaD zRFq%aqwHpnlPyh*V0G4!fPJSf&utHt^ph~!`IG7q^D>f0R;GqvuVwXMQR>awLeVl1 zC|Gv5J06|fl9ed0$CdORKoCOJb@+Mp9|w~DI$$IM`my{gwGmdigR2(d+a2~#P4=%O039iy*Zbdy z!?E6Vx&CNXhIi%YOt3gNv~!US7|g!2&I|k?hzLot(r2duvUX85!bfuB4>`xjj}Jmp zS7j+I!JBsR$!zb0_DapStKdL}^FsKzY|Kn^Qju?pN4f$7nFjiw*=v$9z2ZtFCJY$$ z820&aa#m_UX;ni9F&$=s?B@RdFja0CL4c}-c@}X@U7)$HTgsjBcdu#+2mUcTmA-SV zoHep!USOwUAe%vUb^%1R6KkaZ$EuvR+A8nmrx`BIcCsw!^KR0qm%hyYK}ZYu%$^`> zlDE93`6(vwX>L`{G}?X96Dh3-O3@kv@E6-i{1|t4oU_B$tPM}UsswHxURQU2`ftbt z`)=DE7^aYrK5HE?NjngWd%LF>;lq^6iI*AQr*F76;zVoQ1(3nFC9mW^c`+MEA~C=P zRw(BGaJ*V!$2DxfE|{~No6x~dE#m?D-pTVSg~)>H?~z;96@GW1D-nu$sz{-_*ZI1? zrOfhVo;wYFc4S{rXJ28Axa|>#z3{c{7gs$ESgc2$du2G)!0wfwuqj1BU6%Gx(Q?UJ z^s}-!;P;4#2x7vga65ODt*~H7my(WEXXHh$$uk{+FEz(lK!XAo(og~JNg!;z>*hK< z;3I%@7QP#7GlQg80x_7U3AW7WvJK>zFlv@f(L7GnlWArZ^h+vph=N6|RQ6J|dyffcj^tr*Ci(lH`NKzBYH6gLG zimRcqd5Fi1Cqo@hd_e+J~w`;>mvgv%8f6__g9Wt$8=r1kI3$V ze)z@L_@A)fQAPqZS($lNIp+I#fQ!9s3yisCmGY0aw;vK}2DLu;M>c3Xm-lGVAn%Kn zcU@BEnikP!&*C}2FzqwB1mKaKonuGn7(k)~#Npt(0J!xd`%a){?CS>JO%ew?7Ath* zQ;aX;-grS~_&(h8$b|Fv8i1XQj>$&Q47b-C+YW;7DFZ(o9fm|kULCIM=8y~AsTod1 z5P|?=#B&|M=-HQFGQJ)l$FTwcDHr!`j<=kvp759njEk1XMQ&!Fx?JD){VN2(_g z2#oZ2EenPC?xz>u|0( zjAxz8(|h79gSwy4}awHdBYrzr%}PBqJwEPO}*d(w_P3l#3j>6ofB z8}^W6c{NR2HDL&Le(ig6e!LwZ>+LqQv@zBGIKkBEhn%UvQM>RG1m7Z1yWN;8+n8$& zS92$zvB}A5e=$CgST6KcTYuWDM$Do zmb3wL0q!zHKJ1fyR}@b|u)HfTZ4Xc}kd6-fQdE#Vfxv zR!0e{w)j``I|A$YroJjKsJOS^+WomD@xJ#s9k{~XLgs1YFTNQ{^|TS)Il z9UJCNohHRPs$XNHW1kU}<$20)x;xZ2-=$8P+#ERrvIeBE05m3AVc$@rwRo(nLy6|& z*a=_ms{XsSle`3|ewDk@@Vct{@lZS6 zCu*j@<|#!XSIqOFoCl|-4q1N*aHns5t3gJKwzLyqp>o)fGIZP8J7dsapabX&K9Mx# zG-CjK;yjhRcUGF%BX#qlc7kay&g?@6c;8R!47Y^GqM1`(?h6Z`+U91=Q&gKxes5-i=qB?*IJr149kj2TX&!c{E#tMhdGH5gwie=!2>-V>}(1Gld@kBn{K!^ z*Luba5(fL+;)=^B5@>43cmz+0sbm+pwci_Q;;C1vh$da?b0qka-2Ba{I)}H?m)TSY z?lwM1JMhQ&-CQ-}%d++UD==FeW^_|}FaQ5pp2|H+&a?vk$$zVZ8B1MJXNT(MYD{hQ za7I%&&vS-E`qv?puL$GKk@9P9g*1**M^&2GJi_6iD_wNRXY5wmkN>gX$kq4Pw`go$5U@ zV(yrJfi!J{3R_b~I;y?OG7vQklt8WLTx`AI@>%rO6nogqvngCbR7{{FE?$WZi&$Ek zb@^KC)?)wT>VYyZ)T9B^-sV5YZ-oH%7U?WZ_XCS3CMAGjgOoaD4MWUp!jYeQKq$eh zYsqu~UMg2Q0JwB3Th-><<;r|={BYa%QcQKM6unMPSU;Ww;&`uTT2u-U+&G~FIIcWL z(j+)gbD8P~!UvLFQ$#>Wk+PZi^o;=>VGCB*%DryGrpaM=Y47ciTQR=OhN}V(Qi_2z zv&CqDM0tI1bGm;+1om!W%0eFjoIT;0qfH>dy|LbZxR@4W&6W`wRlf&m{BAuf|EXik z0Q@_LeejBpQ_#0};KIE|pG@sJu!YWuSBsR(aE$Zq(sdBnn~Sl<>-{4#NK%un;?4$@ zVJuQG7PJRefb#VcUR0ut@)O!hB{NM+hzuT^z8WS;S*^E7i6bs_gx2vlo5~@8? z?7q9*u2`yDXGbg*e~fu10#X*GL=dmlcVgO*m~gIgH%9dL;*ycsaLH}0AJ#S)Y#~v% zywazcZ5O%HM|r>*^P?S!Z7~)W1IvD<)8PF0!F(HB*zWltz@MtW8t--QJ7^_Q&&?{g zm~`76-NE3e=FFYzC~<7Uq$`E9o~e=&zu28&6aAOc>tY3zKGzY7O4xlUQ~mobYK3c? zJRT|4TG6z&_g1MDq4#q%)JD-DrXv4$c6~DStF4I&9O818I?fxSE=hE`1*VLIKM)tI z0dhcn)`u7{H(w}^mlY}3av6~orv*)!#8uso=G%A$v;{$^Z#IfjW6QT8(&vx5F2yMO z*rG21DW0`G+c=X+2)!vnztn8O`Y%u z3gSFiW?lEXxf#b+knu+Cgecyu`G#gItC6GSW8 zrJzMiY2JqO_6X@oHAP3?_0AnDBaoyo!Hq3+B>zsrELBsucXLrogBI)I5G_4dFy2>& ztDL5-SAHC`fC3`bYbjU#3Bt|)g6zsRG>YY zB=>2*aikX*u}ps_4{TBN^||Hapd(8a!SeDb^OI9Mw;abaJV_Rb8X`TF;l;1ey&~R0 zB|C$K1vkSQF%7N2%zqMA)uGv?%#USz#*NXW=&N5ylLUv_OO>pp7* zbLNn#_LowAZx0gJyc=vJUe|%N)&X`8@-r6~b@dMk#XrA--5YB~t50)5YAx(?3Dez{ zFgp7fB$1_Sfz*ZDd(KeC!6j~TTw{HY_1WT0>CGE|VB>Io5}^nnDZyFiM#=O53 zTwph%vv?fmVpU$6Ce@Z3gi-+7C@Ur$&2Q2w7HcDgZqsMVT**O$3vVEUZe$*~Ak?56 zMBZwyqeYdX>60-iW>4XqyFATZARAyZ9b+@BJs`aOf35V4>j*%E;ylXt39{zz=EoM$ zr2vsJ_VsinU0lsZ~Zm7i?B zJ#c}SW5^jaVvSShA|qj^FcG19rkuSgmX7l;;pN(@sz-kNP~Yyd;xSvGZ%;%9PE3Jy zY%{*G>JC8Nv=-53tx2p&?0m{7p!F;jlhyFaj%_#P;#8K3H(?BQ-#*$M)HgSLHyOZa zhRl6dvmDdqB<4CxHrmrJ^~r`F*~URY7wD;#naNLXNjPc!v7r53+9Dw&uH%P^=;N}_ z3B?pn5iHY&`*wao__Mb5-}#M?6Uf6uX1>LGeMaiGvyiiWDpgCnU2Uaxl?EdKxd|{?IOb-;S6& z3+X`kLenh9LpA4hPB+=zA#(8dB#C^mfFWPo+A_~~f$y>^VqbnG6ECnZ6C{!U2K%uF zPRj1ra&c~?DKX|pNT?xvA50QRb$3N4ae!|p_xrLYGA%!8SRIq2R2yCuDfz!17rL%0 zK&BIT{p(bJuKsfY&~TNGujZ&bd-@XmfcXh4mI!mp-X@TV^Xzd(He>3x^Std*TVdgy z9g9iLXbmtquLk>F--GI6P3Fw7ho@#J z1{yJC_BL(Wy6stOM7wm_9kI3IM>FI7B)K3jD)?ADLbo*is@Zm(y1I=exhuCB^ zK#2rq0gLe;;u3%}Mc?iJ91nGFj{!`e-Ntbc3Rxpi#gWn1ko2Rxozes6CkrM5Ge-qx zLqGsn=zlCT`iQx_b4hW)n3n(feky(D@7(ZbVBSn_4qu(2B!3g-Kgq>fl!Za@KmSSa zmJgjZImxV`G9@W~_S16S4h~*}sC)APm?k(8Y|pEzDYQiGpeF~^-3=xvkRg{OnUq*_ z1YEkoVA*``?JpmSzGSj%)ogU+VytD-!SUi^Tr>pPrgCYrcU}?IRSAs)*@4*6*tiCc zzjP*^Fqq>MS1%0259Mgf`Qr%w&7w!%(o)c9L?6IhVBJ)pIm5c?v3HdHG6})*p-EAH zCbWJJAfMt5hsL-HrMw+HXlX#qN14Wd3&t-Xfz!&YJYL z)&pFv!eIf|TWtjKTT)6+=-a2@jiVMo%n)d|-~&nA3}8%Dp~9Gyr=@H*Ax^0yG!ake z-M8Cwn;9DNfA0o8o+94^ST}JgCOW_hWK~Avszp#YwWl*8RHgq;d(o`$PYrX0((!7a zuEo*nNjK(!JdFDXUDFJgXyv|<#(6fjk;eSv{lzZ2tOAXny6gplks_~qti{bU69j23 z?RAnSIxHlqzHb|R^d*t@@#8>`rFRgL8N-5}JNr34*&vo$KXMBxSQ8u0aaDGji8BDj zcn#5L9~jr^8eqGT`zn2q>R1(Ol7zYitc~HC5S8TL75&mXUS2=l?>watCyI;x=+;lt zf8nrEEjJ6v!~@CV)KPBmqIgZXCo`8_Z4)NWD+y8fZ+$L$(T92eec@q1XQ%UW;G2nQ zy-}1pYlo!x#lfnkU|x)czZdc(=jZMTLjSJ+gvjlCS#x31;?r1UZsXKXu6>B~!WR#l zj~L5nV5I=uc_Sv=BmB#npYL98EJ&qkE6)MYX0bOv=jP@b1v%T0L0t+X?PA(>$NzuX zflPzN{bqPYSH#)j@irjkml)3oE$NE7AXHYli5p!pNs{{VcJ(d=>E8TIpw#wi&Y&-X ziUFg(ns3!i_!6?R!Cz%NHiHcpNGkyUn;IZ0OtlOE%f4AV2!S@oBNA?-MLS2_0{Y%} zI`Q=j(Z;s=&K*4$vH9CWs*!wVu0!rh_gf>T7WGObh^V_Wls1rEPmK87tzopP;>j0% ziqZ@feP+E0kGAk#)Vm|ax1aRkAECxfV0NO$aKr?I)s$ylRgRPr&iZ4ZB$1>GDTrR; zHfv>Jfr~*64Zo97Cv%dyV#9Dxvvdscxa`Z~25lfgU+rH*$Qu{lwfp zS1gM%6z#^IjjhYssIL-+sKTMj+X^z=bE%HF*XIY{2VX|(-FI&JB=$v(pEe*5+Z!$$ zeZkxD#Qb@z$ggA*gTVL(_$KhitBrL^gs%qJ@4J>gvVS^o@UL~;Wfzf}Ie)djj>P;7 z13sUc-3S0cKk3G=PxL9Jh)cOBgdZq8I{=%D$?PG{Z>&nNk1TY4t*j$Fz4mbmRESc} z;)#(^9dhScI}gq{A`JZ(^>QWXDEhzg&3BeL@ zpfYj)yx@IdCi&%u(Klb3Z{Exd$Kcz+lh7Bl)ISD%TNk{>apKJ?OTDrqfe^Ngms=jN z(9AK5VQcG~HlCzA2N8;qz-k@IgjK&ZM&09hA<`yykp#ak*jC^*;`5u5ha%~VeAE( z%J(0e({_Tv>f8i$1%F!u$B`q%TIij>Rwh@!b&}}G_xAd#OL^a)zj8(Erxr%oz^1dz zWW5vgCrK_5f||yuMX!Ov9(UlH*c@uD(@{_`NSC^rNHgkD^4wq(;`8kmD29^+@~$UIU8_XU1l*p?v^vJt z35Zvxic(A>t8(J%=~$m>M)|_6XBtI5%B5uMJh{_$m;L6nwWDM4sePA^4yY+dQh>-v zN8p``(3(fqSqwB^%;~v!GGfh%k-;zl@zX0 z;S=KD+G(we8j$_4lIOcu?2#7xqfMBtM3N%4F-LT!zd;jDpy=b4Tj9!edL$zi-b9y1 z0f@41mFX{Dy}DA+`VFj^fl)3XAqNU`z?Gc$>GJSUO#iu$SR|*|6R#~%SSjSMKTPFab&^N0c}Z_g_ZuLFkl|FGjYx*ej8WJ?Lr9SMI^%_=I?uNqRQhLI|VUv)pFwgV)Q!;qY`M~^@8e3Bf+b57FCT6Tt$aP96YM9xH4T z!j+kXk#F&0j+OJtbgs#8pNIhi$oC9pL}%^F1A1~i zzKChV?ey^2C3LJKk7KeCJt=Xu>%G_&sl@d&WAVBN4SWRS`Cj`iPxUpd6s_%qaH6aK zELrhv!QvO|dG_tXg4Gr?hks@Pos10cy)N6q9NL)AM#G2KQ4gzt3t?TN^?cNE@oqo8|3CwwyKb;mRQ@ANGG$kxrJ6UK*$(YI)&xmqbU?V{(SNcq^Vgu zs>2oUTwNthbIrGvPN`Q92~Bjs>WEH@u1lo$%21R-t>NRL8jv{dm>NN0=Ts394%f}J z`JW=Z2AN{M!?I4^ig;=k^cOx(vnPCq^fG1Q@o`T)i%$dO@?6r&X|=3h&gwxEm;V(L zWPzq4#c%xFvF=qPjZ^oB@00v|irw?&8OdD4VnGaB=YyZ3Z$R$B^XeFn+x%Z-c8Hg6FGfjjN3!dgZhVy)+sF-? z>-75Zke{EY@Mq{5)eHZP9j#g9pWaMb@hz>ATZRv+F1F5N-tr}4Z|u#!oON@VhQ*n; z2u~p;6A7+}i$T9Kzw1xHK=ay>cWtHTNd8a~*^EVLONok2i40UeEO6_U z3(?ku-CkvK6{5&=v2w7B)%HuCbtiKg*N^FP8?c@PBA%}Gp+I)hqr8;gEm$Do_0?y7 z)VC$ywD;FcG2P+NfQTCwQtz5w;KHeKztz(Yd1*3$94bq$cyKFYhl>|1hQ zntpS;gSxM5%j<81;z-Rn(DYVeyil4!*m(UUS9!a*vSusxtTjOOB)}j4IUU%xuloT{ z$F_{Y+iat0;6a?Mbc{X)vJdHVxmf0* z@u+E_SK?JgGt+Q;f8N$NH_hf^e9(ZvFoKQZtsiDy&iC~y;*HFIg?yp9fYzkpYwu;#SF?fs?uI43wfa#??XU$nI<35>O6oqK#G^Gl zUofV~d}Y%Myj>*4Wa8Ju*y2!kmt~t)pq5+m(MTM3T}#6pYvM96Q0cn5&Tv&eV%%t+(M&zbSdwro-;(BPoZlbrYNIPT7&~Y33KD7Xft_k}&``bG4O81&A$mJWGrFBMZan4!D<%EmP6D=1jQS@CojRrD%W zmyt3lUm&T<8L~q6BAI}0#>=j6fjqyPN-jap7na$+b^TiLvFiL+B|KYzV(F*6N0R|w)3{Js%tL?L18jtqjz~&I zAhY`pkWMSF^)K6DD!glv?FdLGa(K1ly$u5SF}`z?dt1Ztrq1Gr>?-L2S{vu4pmM>4*EVDGxuKcCtIZ!EV}-OEkad2L_rZxdXceGR{( zZj~{oV<%KTx-U+JhO@n#%bIW>)MHHo71=vUi4zmx{>G2G(SxPA|8e)m0^m-mP!-M< z`P9%q3kkobZH4B^z~4v1;2n~tfmF}b_fdD4#lLF^a6p;4^(RHvYHK>C;$wX2Svrt= z$v+5+K?tLugbrl2mlo|E(e!s}gp5xRJ5OOJZqCbt?!Jo+9{A2IBP#Q{X3tO&f2{#= z9&@|WFNiXyArnMGcI~!KF$T-*Sn^yv99SmfKHY$o6n+K`AAuu0>E8+|E4%W23b?(^ zL>*LC;8`*e{=DP0N=*Bp3n)rw8o{5aaM1Q<{5iKhJZ{@pJa2adt$-p-c&~7MV;&Ly^4JzekT&QxyyRHGk$|Wex<^zrW(2 zw63s*FY!5c-?!rlO?iv#1+ehMw^dArwPeCAj$TJY|2STZ`4bSD_6Cb3AV z)zS_Ezi=A)mQcxJ{vB8$aAv67chpy%~{7&l`~)+N3Hh zm`2va?Eu|fPiQ%NFy0V#*ZBA^syq47r#v9Qy4b#}qwqHnnH_z3GRFoxnF0J;IA*g& zUuK{SrKMx4)F;?4YYh$eJl3U`G5(rhE!J!_H8D7xzNpe4`g8$_59%yV57kJ%5rs}fhXu`!* zv#8&*fD-!-MJfeSeU}YaH6GmMzN4MQKoB+W1%u7@=5kK(UP|33}zrXXdAoRwq+t$9U}9Ia>>&8=#Zd%Ql!viJZ3;ip%Vl`>?&oG!uSj;Q^X?Y32GFOOkJj)ya5C!C_IZTj2mjO*>MvP;zox$fru*fx zie8=8lC92fpJmuFALYAJDLkuCFQ*URYb&#fY=3Pb{|V0C_VpEJRlX+hi(^tA{Q)nq zrQc~=HC$!$8YMvN^SKsRi`r93W+IlNh7!J=ADwY~u{{!r9-Nu!ub@dPB7Ez;*?VJQ zO8Tiw+Q`om=|4t#g35(l-o&b`@HpnvwjBLg{5GFe?t;VKNHM)xm? zy#XxBimx2VhLuZ;rT-GK0fJVU!X>(d8-;3q4InMRtb=+=6OjDhP1hHs?Z+}3MdH-r zto03?E7fn=l`<1f6(2paci%nok5tUVRYWAVz})MOS$Vya3Pz2>7v(7m2>8c_aS?#= z`FyI7&`3wEA*rBEx?^Ea7UziWc^^2arB8$})_^^o0o2L6z}6lRzIJwQ$OHUaljI<- z^1y{!~fE!5=MuYk`#!@ZIFG_O$J*TtmYwavL0WcS*Nd2sTSPGq<4&?Qx@ zQX5QIr_^w9dWnCI6=*Xjk^o}AbrM!c7J6Sjhc_T-#8UR(&pf?VIhun+afnAp%x9o& ze^pEpI54mAGatqpiHd*0Sit>*8%AZS@Yui1i^ZZ~NJvWi<9a%%84)jI;&OnT{$!JU z$!HTwmID`1%+(ELPjo1npix)<#(Hnsm1NI-W68@e(=!RN5Q_jqikms)*+FFLS8r){ z6_gX4Ag4!Xl>c^stNrNev*1Ib^mp1eZdD|o}R1r`K_4h>BM$G9N*ffRvUV>EjR02WHl_BoMW+ zgPa(d`O(5cg68_FW{#IuPsM#i)qgiIYC6C)|8h5 z`s3l2l&J}Avb#w=XaaN(=aU{nN{N`E85Ley5v@w%F$0*Fdl}_0h+M%kawmagYN0)B z8mkwE`&lOOhk7XM!82~Hgx6W^TwF0(WQ}FfPenQ(lMURs+%`=4$eqNVTva`5?F3t< zmoisr$at_Mb8Yi+?))P>CZjKZRjd4s(Jkkq)TtgSn?;}OvNDns&VF#e!A|elyDxwi z>PIe5>y=Hz(|`h~nXZy`+mP*`Lch1w@D&j0?XCliI6sa~((K$dm58^l&EctD^GydS z*F{=w;GC>xkb*2=^9B0MA1vY0)Au%|R`UzuN!Os+KmUe&C4G-q0v3;#S9mr=NXJuP zKO7Y}w{wIG1xD6~>rA?W=|96idH%nv_sa!@eykb2+*^!us$jIDzfZNv@TKj`9 zItvU82AJ0EW=efeLVSpA`ud5rU73H1tmT;4HD{JBxu}oU*F&C{oznBnvH&@ zL!KEMBCqe8#-bbq>Qs5)vf*O&tY?RcPTez>aV2UJku0K1o?k#tlP^m^lSM+LJB?{R z!SZxG5&RgsLHEAi>2D^;Z(}vD-!zIKY@lzz7#I7 zD)WDQrh1}pEm9-G{>g3)xfE-298(-|bpp*Q@$tBgy%~E|4#{z|;D|iVS`$xfaKqD$ z=Wi$IspXO*Z;Y*pwfcJE|hbb=rgPD{m z2o&cif4>K$7h*D_TH!e#V==&zUGP>+*wLSO8tsdF!mp0BhF}f#$6mL)NlucSH;5ZL zhW_AjZM#%Z0+4hLbno>1T#{L9exVkn;%$xDWdjb-j0_Z@xS=0g;4EKg4+a8-{LLQj z{jYkzsYIH<)ej-$cS1SK8$PK9@FuxzvXU(3yi8zs{Lb`vRMEv_xE4mOO8PPkB0nrQ z*}BEO6h~$>!eGjXAhreFS|~`8J;shQgTkdNH?;a8oz=-x;Vt#Ea$p(<18oGZQ}C!= z{<$X$OYcQ5GmUb-!rwgwGe(fIe#bW%4(dhFV1ema#cmdcjJ2nMvlsDhl3$ItygnW6 zdoLk|$KW|uQjydGl8+_`)x;U(FLXDl*SDd>y$;UEJ;C-V->AJLUCj60K`{jz#!88 zfH~i*miSLMy87=yRM&zR&}~))pcYo162x?G17;Z2Y8qj=p4}AB)>`$TS09KvEpvHf zJewF+hX~+W21tyTjYaYA6|#xHfzrWd@qY`?IwY}cj{>|sVh)=XXpy;w9&Q2&UcJdioobS$TAA7*`QSRdVF&fMPxx4esiH^*q z6kP3%^Ik0&>un z09t!sJD6Cwl)-i*PtsPSPkUCOF3F)X&0_wQYfv4w7=$I zvlNW?yx2d-1aSKobuHcd(ntVcj&dNkl9+3cs{%pE%dA-ZFxdCER^b{nLt{n{0 zzPBvsr)Zd{lKd!rjVxQ)e0$dHwakl8V^P)jgNL}G3C>7!b?1^oq!sDp-@V^DujG2V@tt1B+6cKgcr&NnB};&wX;(}QYu z@OSzb{$?wkn)a_VWNSV`QU{5sh~@vjTvF-rBI1K2%G*I5lzvmmC%)qTy6>)|P1_Cl z5Kd3|?T)zfW*!+Zm$(2M53y8JR-)1LyV7%!&!lOI1Lu$`sUN_%u1w@pKS zv``c(nCQ*#E{=HyKs_*A;oU4{g@j9ql1`b1@?O?ydXy4%WR!dEl>@|fzrkg(Sxj+U z9;;NyNcNVx{EG5#B7F}1{aQ-~27v98=RP(GtCZEha+)jeDVFIJF&{Qa6Rff{BESB2 zPm~ZKrd)Xlte^F({W{;BlZKjY`-ZiLOI){HJl@MGRO=VjSA(y2S10{ISB$B@t5^?M z)B_~iXlAz`ofAZI;+o2LKM^OmDfnZ6-dPG$*T~<>~ok|WH z6B(vu$YCBtmML1$n;3tvLHrKGWynhPx=_;FZenV|^r3!FPLKGH{y%@Pqm$jOzZIk*gaP>c_H5)}QMV#txsoWBOo<-mN=pXh;pD%M#tM})ol`S5#l|Grnv9IYk z6?9Elv?5w@W|aNAKS@m1yRv^ICg8|OIB+iV?68&}0v|E*87yxy^do!<6DlwgCcn8_ zk&y-tQ1)|BHsF*|)oPlMbK0-VBXj(l^E0G#O`j!aUZLi0mi}q5+?!tV_fxZ|TlW@g zPP~9!kYj0qc)U%WF1#*h%%j6iz$Hq!s=oCL!d3clU`zhGR=F~)+KmiF0=m6_37AHZ z#pBO8t!5ZMm1gea!xJ`uT$r51z&hhR_3eh>!F%GRju6n_tE;PslFVj^G6)go6U>8= zu?;K5tC9m;-##^Dt`~)h%;UeN=8}Q$^z|V9&bvYROCmh`xMy<@;~+U!(coClL%S{lOy7mz|yf zjP`JvWWpM+I(1Hm+Lt*$+w0$^{ZPQ~z26HGymtNF#?vm`YUV+a;{af> z$PGOMe7uYMX-JEYcg=5Qh4=?;lM)j^HX`mZGW3(3{8DmN<$|hl606-J;$w$pdpVTB zIvy8P6p2B|2}DIgD{VO$3vEsdmgH(xn@0wQ7@Iy@$ho2IgsqR`BPS+4u9is z(LJ_4Y<=;BWt`VaOgvjCQC{nQNAUK>p3CZ*?2Ct^8b@Xz;L#5#dA6cNsa6Mv3%#q8 zd`!=Ikflx<)&!eVN&OYBPFb(6nyRub)tpWwj9ZRS%vc?Y0~uMT8m?ZyjLiOCCGo{X zM>HM)&&}E)6-q`#C|5&twCCkk7{A)_Znn!F|B^b>o9v&h#Ittz4+{!tI8LMM0mZWf zVfWG5S0yjDmJFjU(nVnz^Hq@#?lSKa1PT$*`q3*uN{5@BRT)bX7N? z7hxLYsC;dZFY`OS1tEUT9km%alJLssZTfON5dnp$7`wy%hL?j^I&7fx>`(jFn5GU8 zav}D@QYi9^Pd@N)FWp%N)4PUb-b!+1_O;6Go_FZtSotxtV9O!=PPscpcU`A|Y8==FM-VtN1dPq)=b_-lCv z*lwZ|O!Vyapb~ir2`8rx>1(K9JSxS=h(G*Omab4zo=d3w{y1*3_Kq4wocoTn=^Zf( zxEy8Q`k`3*s{A;xs6YGVS0G)@<-wRhGgLbQZ0vzl?wV1KBu6gxTZx~;ib+bK9PSRV3MAlz2@JS^&|2#NY^g_aj!xv7@$u4FEDQu1cs+IFWE$f~8AP!LwG*C0pRBa)blo5=G5dq7V#TEu1 zkzg67jKFW~n54A1fz9SHTynrzql5p3v<@kxK2Q@|-vYA)>tdKvxLLUHvS~ zY!M+d?(bj{Yetek9UhGx>3O6vFJA)X{@V}9`D1|8fO-na>Aj66S{R=U0Eq=8z}qV{ z!CO!;#Cd`*eKr53{-x)=#G(PMP-X%LL;Wkt`c%0;0dgP0K6ZvrpDahHubEg2jjVn~ zAJx>5{fod7nWnoe)C6q-;&92~rYnhzv!%c5+i=#Ojq-pQO-T`J7Fd`Fl!yj^1DCT4 zTRP4MVtH0PWXH5A#@g-nX8jdQchkWCNIL7N zrvLYUj}EC((lHuAX&3?$118-Pg5)UaPH9Gmbayw51}O>Y?hud;1%Wpp`h9(Vzx~fS zJDlyc=kvZFkLz+4D3Kgp(J`YL@i6dpm`Gv!kC!h5J^E zA@Uc2bf@+2&d_y^Y+*gqU6L2JbZD!RxL3ARp#n8kW%^64JZ$)2SzFZN9MA_-1>988 z$XXQT4Nrre+q*j|Qi0)kj95YAyI<*DP@6wVmnsx|V!kW+_*v8P7Z;PQ1dA7CG|!JH z#VA%-lDz=3P&lB9Ht{ard@VSlT~0+iq|S%4u`oye*02N-4@XeWy+g#yeduaK@i_V^ zC2d^Z0XydvgJkB)XTx6PDnOR1^rxp&E$WxVkk$0#(2J7}vAYC_w5MFS1kuXqo%)esb9vl zZ88%;$sW5$Ntd-TGR}tWs@KIq1Y!+fcD2YsJ_6cRYVhneli!y&lc$3s{a(U1hK~}{ zcaJP5N=bm80%39h?=b`|Ra|6}3a$tKZXqg7X|F57AC8l?sEQ^y=t0qFJO>R;;(!LA z10pQ6W_ycSQ6HCV9$PHBVCUQPtnj940U9O@jmrqjjE$g$eu8!EAgv0*5B19hnIvLn z4PN%VSbct%{CVH@SRr<0Rnw(YRt{_Otnjs?GlUO~)hK68m2R899aF3{aP-vZ>nn3F z4>tJI8|sAC(ls*F8pNVy{YtY~OmQHNto7Y$@PD5?AHH{D^r{9>{gzGxA!jybApz^S z{8~9J9;WCVa&kle^%hYy^Am^~*S&#F*6!qc0~OO{j6s1peujE0 z!pwjlC*im*oUbolDz9ljCmIs=RE+(!x8rtq@#uw=D6h4PSqe{N?G^&z;YI#}&YxhyjuFG5Cxn$NN&8Ct6K&2X)WCqYqnQGFtq z(>WOs2Phx4hs6(aF6|H{z>E`kR_QdreIN;Q@%Rwm7j28PBKw}!cp;VOh2K3XBNMG( zaJe@dV-!81JBHBNyfw`%3~5~2Gd`=*SXn)?X^s1IcRX8mEDbAG#h{or&jt3*0Ffq^ z4cLEU>W3OA!>%PFr7rgNSdOS&t_0k2o&$IQ#%mc_xgncgivemfu1ydBbse%qBsvpe z{jHYLDX+;F2fnfwVYCH|FNOlI9wiAX5*&9bn6hs=YW@HPQL(YIUv)R-;xv0@1hyF} zDb8>?TE!J|0uYFtCMPt;C+C3Zpl@V$zhE}?!E+gbalSce$YaYS!!!WPfMF+NvVNa? z4qRYD)v`k&RA!0_Z0O|AMSzkxMMg4pfi^qMfX@=M5G>o786MPS_Db+y_PJ!(*R!c% z-9;07I#k2y2m5UTpAnah<~N;o>(1+&qY=nqe)W0FkbdfDik*w^l$ieRIa4$Yed z`FNzi6i1@I%r`7~2&-@iu7{e7p5HCgAjQ5ujh*byW@HP|&gr(P%!t8lUSTg}Ms|)N(XH=0pG0o)Tqz)D{yIwpj0})-fV^cdI=|@$q#&Di>E}GBh#+j@S>LZyPC|zgLGGY$Qwow)vuo!Nz zmbh^thb>&JElWt(3ReUd&O|Bg?qm)PClGV*q^x}DU00J=r25)*w#q)@p>LSzrLDyE zvD|z*ey=@k_f6qjO@GC>vqy3wPwQ$b_UpV_f1Z*QE&KEHTTS}_AD^g!isdzXH)1cZ z*S$XV=Mp)Ie5D%s4!vN|PK5LUQ9F3I&5e$P&ix&I^`j@g?Mf=xr=8ou#VQsTzCRR} z>9dE$a5=j<>9;VngKr8g6vs8|+l_`qY~spHfXGQ3NZf{)D#J^lQiI+K-B>wUIoJ0U zGP~1C>p6Ky+@#DVvP^1bu$G$UAq2!+R$K^(vf62Vl)vbmvJX}+>3utgZjbO2vXGf= z%s}UnyJ}T+{48N4`bC*rj*Kof#1`fUEDk}OQ%AkRB}T^p&9I&XkB^L)Xh=ocw1hs= zF2-jJN}d6n;(SbWU%O1RRMqj&h?pc1W`zFBAGy2p^7{98yba(moTZ{Ga5Kq%i$d)^ zt`mY_5wEswjQ?3-y<7$&_5V~@o!&Tp`t!>Se}^Yosq*y!83GhQrw;!KX#Tve0djP} z{gMpe-Bn*U6DRg>{Z6oX;t+t5-BNgpH{rW?*z?NGYY%uzqUyf@3Q_CYFyzmr3qRzFOhb-u_wIcZoZ-%K;3*Kqh9uxJw$~#9#3}UF2lOw`F0A!=- znKwKZLC9e6EgGrDX?ER7PxQ9G3xfvftQ6(;n?G1r-`=6}$tEh}{>SOp&1DaHm%)qc zvEamlkg2~5jU}-vfdONv6p!N_PR^b8F|B(0&)9@;wDui0QYXk8Flu2+1h@q8AB%VT za$*^dc6O-t^rkKdCJ>g<*h{0dw$^-@AUc;mwHdc7v)^t@&*m!f5ZBlRAJgb7%#9je zRn(Rzg=byWCM6B>MNM*Zds@2%S`b*g!u7UB^hy z6=J#?yXkQZVjxobZjq-Y+fs2{6jg~*Q1&wo`98`F`PK1*F6m=-p;q6Ikq<;Qdp7Ro zo5PG)<$QmStY?BECe?cGF6_R3v$pu9&GXS;Y}5VE7(WW-88${Ahv<&+)`raamu>#m zguP6UIUmU3jE&E_Sb6S}%$V=+hf6Sh^PJS{O(w9!Nh1aVy}mtVDTdWm3`T!#rs8G2 zjhBTSI?Xo?7DtxiTMzz=`o$3Gh90!l=7!<3U*Iy;O zsKtx!f2W4kayTi>Kc4o{bU8TYZ4{jiqS?kVQ240dODeB}$RK;{FkxrUch~U|y~KqZ z^q@@&(B_+Mb_gjxSw<=qIaF-8f^2dWw2;UQ4{X;=B_Zeo)AxrAh8*EC+v~{=58;|U z>|l=)6{1v8v{@l&)QQFItXW~BF~H6ggWyL#{@=Zr+xq|LmbLsvjl=@MJ+WP%Uo%}X ze2)f&4N1U@aO)I=SQXv1L7-+w>@t#gb`{$XmGB|hth-=Q!X6^FjS($C&g0=A9@(W{ znc0zztSL>^QsF1mR)WU9i2A0WV5w$1+VRqjtMuG8lvkEACGy<^ZOt9YXMHgDYSIi0mA=t!-ErO$3%xK)PzG4Q)2w0}ZN ziE2x~cEQj%?&mgy)5!k%`Ynt@>X3OH_pM01DJfAvUs`Tlm~_kiAXNxik=YCm*h^3B zqCj<1OH73E87!wP3KtlF01NGV|ASnb7(ayfYTqfKXA8zOUZ^VQ?mdjx- zJd^GYZN6?g8)B3a)1mxM@CHvkzJ5jbcKKnD zMA{^ohpd1I8pz!Hh(?3dq{GZXVf0bBU3^R$IiW-c*e@^Q-(8nXQ47dozbh8n;R6Rb zNloG42TCO5tDhHBdEJeiU;HDIW8`Q*m%eY}ne~J0wy;SO!RVIKZKfvd2tj>ZL29DX zd~H&3>%2{u;i|iIg$5T4q*2nmA}(?%>kW6Y7x5%`fsnbo=hRM?ZyL~zzhn43H@j=G z%B0OOP=fZM`fBz{kWe$z)%Nmd!=i68)-G1M(omDv?_(gYN~@dwW8K+>cHY&CTRu4v zDO$U{sxy)*0{@0=3rSMcwx@V4H5n6zqkt-bMbRTU;(9$L?-#4F(M9eHFq;|VlJ z(?f_fA4jYqkOqo#870)uU};KOOht@$=be&@RtHos1v?D99O_OeyZ=*Gp&2(!xavqL zZAG;U3J2pg7&>S@`N~0CoxFp?Jgc*Wo1KfVcq!(G84<8uQc@K;CK;#dFCZ% zQ=>W9H~h5i>bCp&4c~5SWH`F1rJBMi7vw_E9*nm);j4ecz$n}BJ23EBi%+-zRA9XJ zZhNi8jBkz?EbwP}s>IAF80k>Qc2`Fg#)4rc zC#g#&S(1m-THE}`%x4x=;dZ|~8nv)>J-5FliS-s233xR~pVJ3{2qBgNqkJ;8Uc@Xl zd=nrW>}nQpvhTUZ7_F~}9En%L{07K30}CP+0s+b52q`GQ;YyhB_8*28L_$(D-<Juav32~S>SPKr*~UC3!W4H_xswSKCNW^R`0FhZ64{cVe{*EwqHZz;t(K{#anwWE zxbHX2@1tqOexc*>d?EN@YI$bqpZ6!A+&cmwhPk(aY>!&Gwi-)NWolfOosQ8bsr2=8 zmyVr84#8(1ZvKZEdqoDRJEh4Q(|Z&nTw~tpzS1`6_tSf4H(yV~AJ{Uj$W4}EfElI#7S%{kr`$e&608&^M74D_;FfyT}i`I(vSJR2!Jxi?1!m0E1 z>@Ynt(xcVCj}1%4(0B>^k6Tt z%_?Hm`Uz(fV^35pcJ5PEgnFxZ?af;p?3CYbO{64&RTmErqs8#}uRZM=-349Wc>x?| zZTmHv)-^veZO`E+Yyx#hOrdKc%5Q;+te|JhvN*4gLSAnu!GOfhFA&kisxe{8dwQwC z%E{Lm-=*qcUzy|}tnMs!FzF*)q5@16OXS2lZ17e2`{dLJ*Vn3(zRe=Mx zE+NTt)@x=ZFA3zf8XMFd+np)542`cDs##4IJL{6MlXxb`4{mbI9T{SH>Zo-k(d}=8 z@#s4~!G2!3GKW}YrvF$qM1zC+Vo}(b>M*J#&i-sxnoHkz#*@;m-6OS@F*g%DL9s{u z)R%-9>comEK~k^MmAy~caA!>q6pi2@dymbtV;^PWNssDq<5C8~I2A57xsM4M&=SZ;bmy)fLh zH3);_GaqEgotdOTgKJ%Simc3(%xxD$#)~Ms4^d1%gbv&p%K5D>4b#5rjJVKgp`T(7 z2W8rh+R_O)adLMg->lVky=MB|)QbAu={Y<^_KfT!f`{;}{i&~aWr@bEd1tbuvI%4#CYPI;g z80VkJK3x$fvf;$IGtA_c$4VbKz;04myf4>&JTxEmlAY3g@A!&JO6DhL>%@Y}a_P5t zN301ed;pFMHtuRW>B;o`xTvIwP+a&qg@p?e#6S0zXC{Vm@Da>EI*USn7FQRfB&QM5 zzjOAm5G>zx7uvG$(y$U1EEuJDW|$65isTSW!f6?`yx|R>&L3U6;^~6J-g;jKEllt@xWYs|SB?OCi=bE1&&R(o zib(RWf75I>{Dfa`Tp{lT&vjVKhjv;h1{@W0>_t_%&NE(PFb6Ruc2#>s){fZ#7fk=< ztVlO3?*<}osO|q6|rvR+H;bA#ElJ zr*=!cMB5B7NZVP-oBp5js<+pqYw7U%_$awT_{kec!8?v?0kEoS6hB+pq$to{GJg5| z?a;)jAx0_17)-ddu*N=tiMrX@5s$wUm(-!bg3nA_NoB4OoJ8ANHr~`;MMBmtfAYDQ zM#$sBcypuCKB|^tp%J6C3MITwCqF>|A)qNz^U`+4f++DAAZLt>z<4s7;vH7W*NNrx ze5x3Nt!0fM*D093k_;wg$)+VXcp(~^=Fz&@F%##jjp^6%-mcaZTVW@jC@Or$=!f2k z%l?x@Wz63*{q>RUDB0kbcBf7l!(X_XcbLvu zs~q>uYgF4_wJYXlVmWJ?<$sx(gF-Z~@VxxwziP%CTAGIA< z8-@p>gPh%o=xIQOf?`Ev;zgPylqgcu74DK0Qc)b{x1*5On6h|B(Y&5NkBs!z55X}{ z-5MY!5DMeAG=MVLKm>8hE?2x}XMgUW2P&vGItAq-wN8@%^L){7!w?bNdQXBbULNe} z$PNNTncs6BJNY5{8dqi0AMMq-JZ=nMgg}9jLEDy}P={U5oyjRB5{Js)L|0gvMF|QZ zGCNH^T-+r9D9vf&<~+|(pJp2x*D2`RykTuMxt5;05eH)>E-WTaTBD8p%+evmgi^Mx z0uIEoyR}Rau`@mnKr-s*g?Pv34EY=8eQHp^^|!G2tiEQew@h&C0?Fvo$BlNt%O=Tw=tj(;U{Ch)ud-5r$8)Cc6Eqn zIGF#EE+ocRtVQ`P{AcDu73{Cma=V2x2)iMWGiSO+lt~X`#s<9|X3j19E|6zQMbfxb zwU!|WuCPmti6UXD_~A`weflT1T#yw_+l?_fr~K9bbUU~pW0V&M6BSadSwb2u4l4?N zt6A3-H=p)GwmTTFfx3aSJrsj13Rn7~N5sBV$jiwPoJ3Af$`1#8WbJ)LO)R3KqMw(? zDL~|Oeohx4%gx0&_UQd>>F9WE0bDpgUqTUg+65m>;9I}?b~p_}W&PS}%QZ|Fbu;;J z^Mg&Mab_=euYgwl&0H}IX0Bp2%g%jqj1#ci{v zm@fR16|Cm>MGc{mHZTEoQnR}VJ_a{q-m|IYZW^K4*ADUxt!`FlQVa5r%~4`u6r|7z zI=&)RWet@-LQfoz!QCtd{KUPF&c#x{&kgPF)0KA z=foXK!KDs{EKG=`Myps zgEMsgq*@DOXjfuO$F6!}#pJ4FV{i0`j+M|4=q#^|a|4?##QC~}`P=XX(M)>gV)WJY4-R-FAt#<|_ zykNv<%@8AKZ3%Zssj{`HwaiYzXgC$6$8DS9l-W#D*x@4=X=~b+7ZxdDTfbDLmvkzh zk#1#+B7&05?gc8s*m+KAmN~$gt+%!yYA(ovAK~oh*Bz&PC0j2v4Wl$^PvEx-nwxdJch6JG{EH8mSF+UY_r2WxFaEhom)ljpu5j~GKAIw3k%-qD_j65* zT6;Y4Rk8RXyCq=#0c1GefqkR^Lin$%D=nqto?)+*F1FX)Pzb)~$C<$RlQ>U9=Re_w z%KM2K22(y_s#SlzCxWLkeTu|{glG(Iradt|_qiV)Gm;dnJcHg1Hq1sl=QCH^)mx2f zZ>tjGyftG1{@L9rLwK6X28SHKMQqBO20A%GN|1rrg+&oQY41U4bT~6Rx{xb@x|jzD z9p@5pP{o(`a@-|Qh|Q^!z#thZX>nM0mtP;@H~sjTW&w@k{_-{If+z0Fu%MyL6@@LEfT0D&7eT0sNB!kLpy)zBrE8w#{Z z{^OTlWo-GpJC)vTYP{7aN|azwqF!Acz^}<;(f?&Hij0hW{O$Q=yWCcDar|cE@ULg~ z*7hYcc93ZVSE-9cIEme?h;F+B^pN)zZe7q8cFV9!cTPS3u*c+^!0tZ}r}aRV2G zqIJG+z@=XbKLHLW#DAv3?>XNhHrTlo9jmD1A(gP+8pQ=!;;yfEYFq!P5X_ZlzD$TZ z(4rX>4RapV9oUL=Cakd9DxvY@S!_k|jJqbDSX!T*NZpiI4Rd|1STw6RaSN!!>5F=l zqW<1g%NH81=lzu`svxE8Hr1_eawU)hAKv^?ekceAuK^a7;j%~@?g&dmRq2?!5#WxP zzI*=pdDa^nr%bw)Jh)~pf}$Yn?KX$5F0XF1-A^w+=x)&KKMEafL7!`gdNI!( z)BcJ|SwtN%pq&&MMk4~b04M;jR##y?ch?=9X8Q0^+`-47%xIMxavSgprS%yt$>GI> zgu4aqq-B4S%^!;wO-m0~SG&w}%~N|kzakWu2~BPnH-(M7Zu<`tc5K61J&|OIOzcEv z?%NQM>2O^PtUo^Q;`7O7&UVm2KMLE_ShafKo)f}R;de}F+V8|m2Kw-mS-SX$v$1~8*3_k+=+ z(tM?_4vU7I9=1kUGEge8v%3LR=07?`dl)~*VU0n%x!wpWNdP@z)O$^c?LDP&&p1V?dyvU)T0wFQa;S%$n;*N}JF`BP zruy;2czNN9K+aM>D8ytwPKUgaOIQywXNK}ecx zr~_-O1tq=^+jgA4C|IIbWfhd;S#R~aO?n`=pVSf|BjnQDF|=0-Hm4dUnqBh1Zd&nT z{?*D0indvA?T`(!0g9VWrG}qbwRwZi1e?LQ#^~@W@V$xvNC2FiX~#7Hq_9u;| zN-7|JE5uY)k$=doI`4;&L2mcNa;EsNnracGozh8SrduNEav;>pG8Zb==ek&^?h6aW zfc%H8H$+Vt)Zk7c-jHy(CV;!uWS%9`Nb-fC>tNtbVpJprJbgp}^+J`n+srNuF;9NA zFSMl}YZn(reDbmnZgDD%)4#?GO})#y5D&Rv`*-iY>K6+qsC0{2PIkL^Ow_ts;q~CX zDAs-gA=bt$I<;N?c-WM9bDZIFZ44vVx-;ZJQ-`=`!^wae;pP5kN4Ya9`Ksu&oHn0t z*F%+q9-4gLkwY|S$S67caWs%{UCN8hvoi$>Hf=rGp(=Qpd58)cClwN348nZ=SvJ90_h^qz-_#TwH_b48}|m7>gs&{x4Zjs`Q_=K&U=by{VVIC zkk0g?AY_z7gYgVICV~QsBvbQ~8FN+;$6ry~Sm- z5WW&_!&ItMt81T>j3zV$S3%cPt=2`;bgu;Sp`*z+&tL(p#T-bL76F~?TwuV5fFAnS zBW_y_^VSLfe%IT(Xg}i)XO-CmyW1SZ?{1AR~ zTrIoVrBgX3e16TxnA0mF2er8zQ&9rrZ4PUNEQ8t(^7=na3Q{c<{`kuic9Phyqy)H~ zGGcI(*T+2lmP$c2Rs4;1pUv;czTUz8QeIa^-T6R1tr!16<^C8rOFbRc(Ye%h<>FVk zdzX}4dJxhF<9%14{F4lbB~5N>TTM;H|1Y-1l3=lbM%Rk0B~~Zu zMN^y6kbWYC0T@#y*6#EJO=7}FLyi5a>}+~0%aDdha}9J%Y?!nht0<>JJ%2pGbFbYY zm&-q5lHB%R4C*{xb6OM$uw?mbx#g=yTo4CTR2Yy@pEdNVgU0o*sT17E%-sC{X>bfW z?{5+zAEU>t)=%|>pN%Yj6L~c*p!zxMJ;VQAOCAATJ#Fl)ElYY++`9DfmbbI1u8DoJ zaMY3+Rm5Lbpnv!0_Vy&AgE{f@{U7!)IKMmd(Dl92WFb{&9Y7pQKWsZ>tXHUe*mfVm zE!MV@%&STzkyQ$gvL72iBPW*$p)#V}yAN0AjTs#uSKd0|%h3|;?diF1%qI&GbVTEomY z0g7SQt@}B0GogQWhO-?$FlQAt`%F%d6HY$fPxK{0n91A$7f;&D zp^Kz5cL*+3{caz3P=yVb@N2j4c>pqM+(JHBxM- zf19!ae9RXNl91rB0!TyE*=STkIy&URJ?>ryr*&*y`-sUrntY%SWQroM#!?N{8Ov^| z33&6`-!T*n&Kgou7DkTtk?YQ^#MSnb+iM6`2kFOFhv=3}Fmp$lp&0~;l&CPX)Nn%D zsY#_-UsPb#U|2!$X+YG%=mMH%Yzvs9mSOt+VSJfZVL=~XQZkg;ic@l4eimyv)Bp|X%DXr=OieRar%~P4Tdb7oe5a-LXx#E* zi}1H7PE7V#MEgYc=hA}vdJ8U*>1rnlBIFTG2lNn?5;$)E;h}cnuI)ACjU>^inX#am z2AcI{XJ@5ydc`oy+mw^lx`Q0$y7jD5?pcvdYt%tFps0+=Y9FLK`~2Kejdyf#<{UCR z_Z1$ii&|RB*@qikt*<|;Ox3FL{(B*Czv{AHggAj0p#fdCMnNGYq*42ClxccEABhiD z#4mTqK~8hlJ;%-Gy}l4HX##UQIlRDa*D0m;<<;Yz$56BFJH+IRn^9=S>M@;_*H7`Y z&r;V|l4XzL33Oo1ekHCHH(Qq_^U?emv zs7k!Oxvm-nafL)D$@f_kE3MLLZ<*Cq7`iz|M-Ztvt(#e|t??0sde{BzaQvCDmp_F@ zdB?%+Ew6SVzB-pB%{(Io)zvzQG}C}ZDgyCc za#^^N%5HN^DbGumvt>_K2a|}vf!}MZcpzP(?Uyaso{we&cJhR>JLE+&V?N_PanNEj z0{-Eb{1FYoN!&~3NRW*kYKhy3+M|>21N;*;$q5Z&V+EGE9ziby3%5iV4}ri zrwY-S2a*5r{v+4XLrgW zdE&Z8VNZ;l!o2*g3@U59Gxh4vpQp84CGe$troq1X9Rtg$UHT3 z({c;yOs?q6ASPE_OcJDY*o0-h9KJZYY=vDf)DVwTF*9Mv&o4SiZl{l#gfH&!yQ`$j zu179-7Dqsd%QDgYOv^kvws7g>ufSH9{rh4|xb#_$Ft0GJP3mq#;0dH4Rm*kNlz+Ap z2eo9`WPD2rPWN#mA7>LA28a?6D?vApNGgl%gsJ%4!+YNzlb4~4X^F9*MG(Mi!MC$-cP~P-pLpA zID8->7~+a1$?Q_-AV)QJ3_TKFfsV2n%V@5Q*8fFqsmaH(SXNeT`4L_MSIP^h_Wp)1 zjg(bE*hCkKUZuVZLA;EK#t%nFD&e!o4pYd&CP2E%I3R_iN;@`LT&!6n+DcsA3o7ai zS!s<0{IIeRn-T<7kWO48lmFxGWLxJug-#>E1;gx_O~xNeblj4Z-wvC9^;Jo2pK8T$ z`P#~*WJ=KD15n)|Ds&e{#dJA`PKzf0e3P~*8gjHicWaDDjX7Z7%?HC zI)5(Zqw@_)tEFU{{d_1PBAG)KalqgK>nCunQ@iTg=I|=^Qp!Q{xosIT|L8|=Z0zbD z-j#+rp$>p|mD1{P(?2WG(^iy`)$r5}1PLX%Ywb}lvn$icmRpC=N)yN_7>VEI$Qrt`c$%CUP`y#KN;CifNRe?C{~tYzHMCp zuDpIfXT48m5_&k)kMUt=Xsf6`H4`nfKr{9sH*KUVu;eJLSI^rLVmrp2Cq;ku>yYAKdw`uDfl8W#gFn zxg(MV4`O`;7a@AEW5SW5U%< zM8%9%Ffm~g$jTsPn~G4^tTW^>7^x?VVWuEDW4p)iGL(N%ld=dpc)bAykH!w! z@Qa?A*{auMRaFD!e^ppu_$HNUXz+0Hj}JQ->mvn>$jYs%B9{rWlPnIHD@*!7;N;6*$p4 z`%WY}-T`S#bzMX%MTr2Pwh})H8>m)22p&vsIyV2*n<-0|EZv*aWri%W%H{YfO!)0> zCnm9aLclgzQibmvY-W2~c?zHMK`Is?hP3+UA3|xioxDK*-JV;QG4KQiJd{6n9eh;( zz@Q_HnO6?nXpCxw)8(^XtI=%AoJ?~mD^FbC?x$CN}-H6w0U;BQTx+!sD+b+MHeq&;?UEa+d{_!Rt*Zl0D zGwLlqze}Q)c50qY+2is^efjCwgdG=%yn#458dr<%@j4MO&ve2=+&fGd81rD3b1_o<_wYR&>gq7O?#f zfC&IrDgEV4+|D}MVa{abuXS%?;^-{uZUR3sMo_UZNiW}ESKEAc_Z&bMj2vz9U3Wj) zGnlmNr-=%U(g*Wqr7n0VH}M{7 ztABcK)b35A{^MfT$ zI{otxB#F*jrfB(rcs_o%ANom?y#em;{nyCQ`apIo_rwAnlI{P6e{MBxvAh^96iJX$j_GI}MyDt=s8sId`gHatyh z=H?7}CXHYXDzmMtda1qW>Tv0teZgNW%zDSVc^mH?ibnVGcQ4`v9R%@|6NgrQtfKdm{!=gZnnGXD>!pstdH!K4bYCH!(r_p% z%SY%jHWUNK3L)sJBhk?qW5fQNgOu+#BBSzD44*Do#%FLWd>0T$@9?f}d z67%Q&dp~L!u9-kbA*!$v{B^_l)vwoWI6xB&BJ8tXfxe!Gw$wU;&L5e4-I{7=wCu6x z?MSuT#53QmA_yWe#ebUfOP3YHg+1Y8Uo^r%79o{oOpC%5Iey|hwskjyY((krq11~ZzM72vB*26GOtcx~>b zf3{yF1<%(hDeaIKv?q88G{64ggBBlYy-pOn`%=f&t~P!HYS+s)F(eIBZa}2&th$fw zT0|nd9Xg8l^LC}~h2x8>9zboyZVWG;7=Y@*9W!4fNtm5Un^yJ0iMvxZs`$HWM~)iD zbWEEAZRb|#EAe}Rf&+n=hdW!wqSf_)*G)62J} zr$&xNl(I*i?YYXeM3EvVyl3>X>IMvqz*4j;8}=H#V^8E_|9BXI#p5$l!4=TN@v_$9 ztRWCS8)Z9@)WM|p>A@6P8V#S)1)Cqyp$o`0ym1El--bG=OrU^_!*lzcnPI!>F6Kv@ zPYbQGuws&#KDJNGSf`^53}E!2C@c*bIShR4Z(PGfyb&ZM`hdu_nuZ&~gomeriqOE2 zKA8l?f-p2-(h3;g%p)d(nP8%|3JNkUo1b^|hr@79M<=^sBqW7Q4|JISm_08EjlQfi z#wii|Q-8~o$F}L&`yJ)QldxATL{W|~5>@@&oDP^!Yfc2yw_dNcjE^zQA)(8XCZCY# z7}C*GqE7LO$RqQgh_YKVuO{WHfkEx7gZ1^(N&am!G5+hv^_~>-2-H&dqGSRxeLS>E z+|B+KO)^qto}j|`c_z9Ux@lNVl}+R2quG)9evUntysE$8h_}jiQfO*vO55FQL1N=K zvBo78(J<38{<0^MtBZ@szMv6YaHE!C4?1rLa!IHohYpy9bGaMSMNIL3v(4(hkby1V zFpSI=PY9`YL5b*vFh0GU4^V&|cKC&y8O>UUbQY>}Rg#0BF*_^iu?Yzj?WC zw3|{h(!S=`Y$x{6 zURvTw*vWj-H=5d_yZQih%#4>E+KdcDHJ-sM5Eto|Qfb!g4}fFPg$|GwaQ%1+9Ctf@ z{Oc~R`$RHz6FMJ{=K)e@IG)F?u#(0=#}=;Ct%@L_^5|7BH>=Uz{;*Brb2RcFZ+FjD zs{}oBpHqFhP46f0?086IBo6U*pqn{Cm}wPKm1urk!+a%Ri}q7L$@{MWh@*~?zag{s zlu}Ev|3`NhkKJNcmhNkLbT=ka-NW<%)7AV-5&ynZ#(%y;!eZP4f{%QC<_HDphUC(Z z@+G&U!F=E-FCksQ_ON{LPYeS~+>W1f8D*tS1MH=w|_0zz{+(4feE7 zk+?W%8yoBpDcc`0x1ntp>ZsFVuJ!}632?EbhiZ3z@4gQE_gNna;`NvZ2F$_O1{;@)W9-w7(&lcQoytce}d|eVdnPlr~ zU4jG=>{>uvA>s1X-Q(u9(g8>TxN{+PIwInw=-0|q>>6_1xU%2jF^4%AUEJLV=7Rt* zKrlb3GVytWQSKi|kFd~m%;n+QpW}eBrKL7f=m!OuO8FurQBb&587 zL9G|bOc+I=qYLKo`0g$#X~Osj%o*OF3{UXK^D87lbZ;pgYv-{22bk#+Oe&yyrZ`s? z2v}aN*sLtt=)_mrSlnAEB7Og(KS9!N9p@Cae%jY-&|=x}KSnB#yhx(K9d02++UCFM zzGviUpBKpkM8rrmH?JgAeKu*N&}^zIJQKSxeo0K56h7E~PxPu=wC=rqXnH#!+lR=Ucqm$f(D$pYjxzbnyDYSS1&5eRA@kUn*}#+juH&J1w`I0%7cW^ zKwoe7=HIXz`+OnaQZog9Prq&_J3LGT;<6v=r&= z<1QGt;sCzQV-72m5t&FMz5)ny6a?$0Yka(ZZo1?vz>hBN1^+*et~(y;|Bs(78Fw~W zcT!GeZ?Z|YGqN`sA)9O&XH&9QWQXjWot>TCS=pSC9cTPL-{1Y^@$d(CpU>xguh(-T z(|_=iYUE01&a(Kmv|pSMgOGnc84>T0e7DypB6EBPwH*pyFof6gUFgi-c0PGxmbI1b z5{%g?f6JrbxMTqHuV~rt9JBlMB?{xmT4Nxi3OHOy{#9 ze5Y$#PJ}`rj^YFzlD*-1Fp{1O_khN#a{Bf9Z{FS8F1~L%4Oeb&#|LExQxIyF^!^Wh zJKEaX+mxNH>Btcq*?^h=;BFaCWt0+xp{4s^ZgPK~g;4>6Q?2sX}{vBuHhL8wCK3nHKNt#kDGLI@QFs zKD?UyXs%$RHQ@i=IDbXZx6ibB)H z#pKTE+0Iby*X0iBStHH0-sS838DFFQw2|ShlX)rvO?89{GVAfHs>zMkV|~ycOO|4y z>-~4|RT8n4ISo40_4!p*NXVOTJ|%guKr(4e2UO7r zd719yoaI=@{>ok5<>E7x7i6tNxUkhj-8El02Pd2&0)HckVNfQ+d70Fck85XNu5J$| z4a0%&p+p?*aWZhG)smi{7f>?hGfEuIYC#}iWMX4HE+}cAH7S3DQGmXoj`#undJ#S- zU3Nv9_(MJW=&;67{nk`@xma~Bo%ssi9{-G4B9=tLQ==6mZ4}1mP4v_PtA-EyKKm^X z)^6EqqjQu@dx)-)-yi?x{`THIfyxGl3P5l*zbqV1xLBn+Dt5QVdS{W~azrEdn;?BE zj6K9UZtq(n`)50Qk|LKOlVU8Y%8&;9T_R#84P72hng4Rccn39_1`ZjU`P}jluZ(90 zELPMxY)WD}yV1&D7fZKWMWyRRW+^KCdV#95z+EsFJfU+dD986eDm;+FvV>BoBPL zhp0q-LR|N-@ppQm6&f(8KLK(*4w<^s3V3dDq2MPvqz?2FBnA&L#mMcYuKZf(`C-(y zV1xgXu=F7L+a1t9NuufoCcma?gOWUP*@e=H?Ph4whA5xv5H z^7+fIq6~4@iS#mYLL7vN>PveoyFf`h-jB(Rg^RYa>h@*V+v*`niMlLg;3drJ;X&W_ zN_zFajDg--RSOxBcdM!u^W3Aovl>=cYiYHQo2^iz^$^onPN@j%Wmj|~B(Hzy%$?gf zC@?Uu1ii)hYJCmN0V}z={PA`e=z297WrwtpF;j&wBQ~aoP4F1NPL&0{V?DxN@7vd7 z5_NJl(@Z>v#J0*+hQZdy+Dfvzz2zj|{DdcUk)+8?8?QxE@_Z=EUonqUPH%p^M;7+) zyJ3kS}&0|-4J7ke%C=3>mx2^|)ap*q_ zix83&cO^~du^^E+55hoUAU35hY$RAoMoV@j|7Y;=pZDb`WzNHCr#K!Mw@Mw`frznu zMH=D938~8p_)T{fA)Am~kKE$H{>uc|?4le{e`EiuguW80^pK$f5MQexufKi$wz4AwD$ z%N{Z%a;|9p1or$gdw9`0Y0JB1(J$6Km&J>X`|ZO`=#fb(0pOklTBVwSEbETkQSQ5a ztNDb~*7o+Gn=7I4BoE+l2e^!?>6Z#woPr{?TozFLJav?$WDz$~IVmASE)C_jjjGCg zsmGxVEe_%>QwWb+9A|&qU`0%#z%;-G77tUXfipo190)$rm!h3#t>YZFk>6%uT&A<} z)~wbh#bR5-p$NukBjg@-%4@_uGwaz^jVZ;9aD5#feRAiv)~F}d>wYdo9CzC8qvV#C z@Ld--RfQJacQuQSAc)Rh>5f7v%82)5^0N`6 zYK~qGhZ=h3WOKQ4q_ToGEy%16OKCG!hVG-s*3B|mu#IwY>OKTce9 ze<%fwKr}U7>?5a?^*Pc|I|r|Rc`?fyAB`ON`gnRea6{(3JTl+p9*|JFi3rE@MJmI- zKOI7$WZ1IGX0ahg9Loo;u=CT?5F}6$j7Du@Hj1J zuwARL$$7upWcd-}kt|&f3z%S~G55vg;zdc*S`O}i?ScZHOofn?Ppr8oa zGv!WO4!*m&yqu?)0HE2S%iZ3tlanC;g;Ig9`2vv3nP_l?J z$FALs6;SRlWy85jh8RG)TrXud=ZY-p1I~HU_^ZWU-Gd}E;FWwP0#$FL$?i>r)l1o| zcpKv~FuIduo)@0)7z2v+1W7y`jvsVTg+^59FUHZ+i`$9Z z4Jk%Lp12@n1tW2+foL`L#t&Xup|3TZYCkzcO_E7!2)rWrGz67W@vxn})GddTK0A*6b5$S!#MbF1m+Y97zR&R^hSal74O*J`ATihd?IjxB*31(OmlxHu z@4{^>ZC5o$PTUoDP>PppqThY(DtApK1S~c?t~Ypc6Q+$6PlGzro|?|yT&LxMC;W6> zBGy9II_|I?TPD*F_fo0LLpX8ZY>%TDH8{Va%KR@EmJ`P|*g zd~u6gtnyeb>#KD(avBwL=jEs32m_u9v+5*N8td2xpe{#k-6CyL5*II zv@yhVHW26*lhKyMO1YPPmKNxx|7b*02HNi4PFUKOiZfD~Kc=|ckW4dKCCMcccg1*;dv4Sr_WO{LUcv7=>&3*p^mu`h4 z2-kpY;n+cB6^EDKhFuPrHwK5PyG#Z<9TmBV9iQ6Zvb{mB=P0|Pp0iWRYQ+(Ev9W!V z}EPChPM+}Mu72@c=@}&PILIvmTPaIam zOO1!Kh?wpM4Qr#>`6kRwka=o6Qe(}Q=>-w5S?=Dxcq9pEtK zj4i)RSB;Oy`G-bI#b1m@PQlJArv5{RHluQ+#PMuAzrH*~x$p}7cogGmCzOWxDOnx; zeoYxc7V_ms?VIoo&guQPg_=iy{&bzw7CnAfG9=db@2_v!YFJZ#`F}l;!_G7HUKvb- zLSWE95hEL$-eHS8V`oydHLhHOCH+{pC`_9MS}@8GHNN>neVi^zpGq62uoDNb7fFpg z3usGu7E7F-tixlapv9_s4~nm4v8oJ@NV1ysLatzJDD)zcPy~$@DfV1zK_pA1oGvlJWm)Sg*n+MPYp}!Zc}U|Bq@}1DEv_HG&`i zwMX~kKQj)Q>Hme1eYXyx$4(fl%zwwV+(T4}Z4vZ_gy1G@O#-^P(M$K{aO&9ga*pDZ zUscar#xk37y@_3NPHwP13fF+W#6;y1xEx2c<_J23O4{V5*z<~*&?`K~D;5(rosDt5 zu?l%Mnk0EnnLU1+Nx`C^I8#^w0-w!;^nmw3NCbxJPA(JwhVAe2p)m9AIk5XM{`F8Q zXutz$9T865rBx{$i3~wb!5Y3j+_IOr{jsg$X8@iOU@gPtvMWZ&y`8c28ZWAsEuAPFw}wic zElAG`e>4?64jT=0>C9#w(5v0rX_(1=6~%LKF@Jv;5GJ_Ukn{cG&9>S$Gh3SRQZEyz z^K(~)>_v%s->jVh7crHNYCg5B^toAe?)}WqBs1lYO6_mPArSg^FQKrYFQ01t_AMp2 zwze#H#UXs!y!?s~uck}~qpOmV6qH}?;YmH7!bsf27A9$+a&;`wr`g0Rj%4%O*=4{% ze-|7K*obc`$nB%w4LQ!#{H4&m7n~4G`(w1Rnq0_lrMxG|IBl zO5@onYA0N+ssb<0ZB(Q#e?wx_5P;oLl_qD3wgy)aDi70->oueMNxNR`w>qqws<7Vw zD|j6sRXrIfkwX<_xyMqq<@E%ho8(o1!x$8S_I%Of!X&rb1oLZ~HkwYF^9#+emLtht zZFH8p{Ox6`2RB4G#J9 zrj?&cgXYp}Yk#GxRv>-Of4Qx-x`#8TIsBRXsl5y>NQ}$Kz${(nNQ*Cmk`N!|&LR<& z)3M~5TV*vaSsY@pDyMT>PcdmF?y z6V7d#llXRPqWl~>6hzbTK-1-ZdLTvq?o>0_Z7IP&j<@ddUc-!SMqXa9`Lc*aJDSrR*cgJCsUAM9LSTOsL0|epcpj{yF0_Bbc`?Y{sZJ&YHd- z!UK@nzrS9ap~OCYJM>>RC(1Q+BB&i5`@q^Q6@c6cKu8?~AjHAfXRa7=0d@#rJ>-7f5l0V1m(tG>++;B1w5ooY zVLDTRlh97dg=033Huk;H{QU=>V#}^|%Tb%!3DmePDy1H3(ZuzYoNrzW*cSruoZ7Qg zW?jYC0sm^2ra0n1Lcs(NIpo=d>KqT$aAC@CV#+NXX;P+1*uXGY#oK2PPMnXl4`;vj zshP0j)lg2v$lyakb%-SSNzuM=5;i-w@*j+lWU%_wy`zvYRkZ2Z1_`c zbvMmFPvxJ8a;@iebSaUUePe#wNZXQA5WcRQ5v7rw2+=H%9wzx&<>t2XEO+6ug&Fk+ znzS}GXO!|3k!I^*CI%6wDVutRaJ4%LxcFmg9#As51MpE&0nvQiL#V1OB=j$U%r@Hg zkKqBWuRGWy_vRFo=v`ULv$W|h&)?5%qjLfeMKIRleRbag=6H9beiO=goOm21p(R79 zFqgdlWHy1v%C-8xix2vH`gbGX9XReD&r zEw3J^T)s@JI+OcH!O$NxmZQJ?(*_2k)lsXt$mIgiCGD)sTlDhn6|gLM1;#~Bw=(D` zi2T|%!d-$N#mR0UXGh8L^U`^WLO!JivRPP^^X>gu9tME7Ph1l1&>}!(@5ZFJesZn%r#Ro}R2}RtsrHLP}LBN{-c_8uH^}nO{$lIN~TZe6(XJ|nuK$5yG8lF!6 znVTF)jY=w_B;>dU@0;6LIHl2RaN(ePX|;CgNN5bd$58rm7S6QCCQd19tA9eE)(ptnOTl2Cosy z*MDsZUZqk0FfvG|)<(o2OS_1mNIWfo(PMaz8E-OpV6A#%eM>`hS1;_$E~rliERo7b~UdZ4^$0Jv9q zvr#UkddK>QLyM%dLSrY_`z8GdlajJuRi1l)C~|C=<2WlE;rLo>k7-PL@ovpcGUvML9c=AI@5CF0 zN3dg$``#N==)nOE&u+WmwFMTt#-}tOQsNA`xUxgJ7@1Oww3GQfJH0>jk(0`ReeW3$ z?ORv*COyynLm-!X;N(M^?Yz6KodYd{GL)Ap-C>ZN4iuIN&B(%Mw{~{PBr~w~B;!vr zqABQ6h6?*W8|Pb|`%<4#Ly~eV8=!x-ns_4y1mET)>SHtKS(!#puFr4zXOfApqRtemyt7Og)!z~|YiCR5@15Q^vXrzpZLZ}#o%Ho;r;_i;8 z+@@FNVL_|NFRM7GLBta;zCHszr^-pec(?Cn)!)Y^uZWJ`|1og*_9V~u2w%pfB^NoNpxK|GDg3Dy86EXBlqIhQ-Q=lLoii_c!ZH@$`+>6hf zZA2b}zbL@}kZa#G4?@5hIM&1#Kx(wMiJJy~FXGLd6J| zcYxj$W^eqt7KGK4au6Qmjxs-q`WX56OkvNzulIGrajWGEn7_vyxe+qs+0X}6&^ESQ zYpDcqWkjxtqImHj1gl7pT_(GrQB`>vRxDCRiI{6i_RQSU>i0%zs|9IK&m6I;nP>ru z(SsO=&w{UiGQn!uAxsLVyp-t34^6u$2IcEJx%{HpDEaSv%cl; zzYp~NeE3;g^Ml9E0Re5Sv%xkrFn&@z0jhMcHgO=*$Ts8k!|bj3>aJlE@znq+os~*j z2owqDKs-Y730SH5m^1_NWv3I3iz`jtLZUP2&5`3p9zn*C!zi$O z^|7=m7d6aifnoNrZZ~CiT!{FXg5DEKP@T?kNWCauM` zpxmAq^hm$aYBh&7({*0{z1#qeZfq2(icXyA@+Gm9iq}Lfxkahrve}8a$9g~J@<>wZ zJTUQ-Y%~Zr5}^uLj6m!HYAnqD;&aa{w-uZ+A{)H8GK`K&U|wX^mW&;)k!w?vv+>rB zmtiqNVH|UN#^PvwN0f9_E21#m;<|>d2E;@8S_HWwN zyjD&y7-Junt6YnrL$9*ZlrX+&a7+r!y*YQdh^%5-FM4|+r#_t)5oic)N{i$God!l>4akm&U-rN=8S+?F%!BPxS@DCI@&=Sm7~SW1u)eVg1GoH^TWR zpZ7qv2%JEV51%XaO6U6O-?ZsLqaX0$uPW|(`ak}LIlC&!>u7InU0BLJ5q~Zv1t^zS z)2BNju}kK;BYU3WpPug6lpRJ~?gs@ocOhdh zPex-?(=1@yDE6%F#KjNE6?-kOfWl zVr;8gFPe}#Px^}acx8pqU&e%Ton|@Y=bu(XUfE4C;v*(x+eTgn0_5CJ3kwn>_8FOw zM_j$`5yW=-1nv{eSjz$f51GBimo%L~x^+gpK!N`QUSS$iE#0veGMalxLHv9+g;aa= z8dv7$OAs{(CL%l#h_0*u|hyQw;BWc+Dq7)83CtUefpei>Zj+#&B@bOT7x}2^D-(x{lyG=Rsrm zndlvJz(38qD;|IX0qQH!sjSPv!8b=?_SSiuezaxe_uqc!70ATxG|x-R29$6C+5Sj+ z!KXvJ%n$Sc|7{bH1kIhM8g?|0W}9|&dOj+eFjVVHxBXvF?i;(in~uASwArTi!;YXE zankpgA}~%uN&BI>d7i85q4HaCK%;(w8a~;37(-vE`GBvED;zxYOrYiK)2z($_wQK) z*1=(#kIA}MJOzJi-=+W(H*!xY>E(k9Z0+z(QY3YT$&5Dh@LAl`BEeHga-ZAvZBs`} z_Co<5Q^}-k%|!?v47H}*paE`w8w7d}uC zFY*`_4xVZi--~STpTWxmm$>cqA1!!k6J%TAI}$Sja!eM8YE_XPM3x_hWTGAf)@=%D zybw96l)Pg2mu;=A;4MHw2hse{=N9m8cWHS!@TkI4jTyp&j!_ePtx&;YCckz5<1-EB zMJ3QX9yM@(d2FyD11=c!rChvas$+8|@Q3w==u&}@zbK9_x_^TpK(Qvs9znSmR;+O4 zP$yxiKQg~!M_62r|6=&z>_ai`>5lZR`O8D@h)De5&Nw%a!o$cra0^TmN zr*)4#Z*8Wi_l!$qd41f{NjMf3u8zT;?xEQ;-2IwYKb_>bl>Fy^i9+Q?n0)R)dkK0@ z5@`6|Qcqw9#i9e0MB@gxp_SY3NpXYf=fkMan&J9Ce}2*9;*3E$)iX94epdDW5cQK6 zG6B$Vz=h}aWQhbqIz};#&a)UApgQI$4A@}?;p-C(_jL6dxI#!aO-zFW(2Yh}^47#< zhyF93r+5Jm4uv!VgM&5UHz(3}91AkId5J(6J7a>iG3?~G>Glut|)7T22sBz$N?{gME>8%i}Bsd#u z$OA(UVYcdo_hA7ct~IeMS%vpR=m`?kWRNoWZ{`HK11`8$^pD3AyG55=nh=<`jDc^a zlUm<-deTEwE19|*or7w{Kdkt)9Qtp$yt85IpO7{hy&Ov8Ne~j#*lvP=7M8jwbYIuL z8=5G*w|~OEv^ak@S-9MG2JnntL$s}T&3i$JyKrTHBkPAgv0lfso zuxY+62}pH*QUORXk=vdqPg5?%sx9s!>NOPwKEyJaOu7f8@{42n0JR*xpq6p*G)p}O zMHM&t(|mSh9Hgc<;{_?x)J&A&<@s1W4N$Pyn|9}on;Y(i!S;$#(5BM#Kqpgr^Hk+b z%r(v{)`^zIDvGQy%^2=ScVlzTFJj4elm#DNX2sQSxzgJe+Ipk;$Z z$jVFc0yaFD!tnUF9O)hxJ+k(_OqQRIEvaDQb18Y&+{Z7XP&RIybBwbeAjxIwAaTbv zSVFlGrXN>P;YCvb9#(F_df|hI>8ryZe)kQsx$?`QS}K7mq4TWuZAh75lx6J zLRtd4#z>=Dq2k9khj-XKoczIZFu(OE236GnblfppG1{PO%YW>9Q}HS6q3mPOkHlPy z42Uf=XKW4Aj{+E2|E<238OhrBRC4h^fk5a+fR>iN+0k4Ry*&e5bkcWYcRG6iNe`N!0(IVe#6 zhGhUYRw+NEm{9}jNg5wi07u6jUlfN;Nsb>k(WSn7gH28SXLS!{9yq6-$Ls07+`~Nupt@YGj@H)U6CkX30zt+TI|$X ztUAJ+j`k}$wUJ9wZDsmwdshQEiuZ;88ZgbVg}LpqvH)pd&Zb*a&&j%h=9XEZG~i%X zA5nofY$mOab8J%7Ww9Sht*#vXW^H289>|Z_e7+g5zKL}W=mD}kfjE~q{PjW!9JMVk z%!kScK9H37POYLa!12rxE=O&8mPW7FSxBE=Ci^jtK=Q-yHU8EZnVuL-5GZE6JvxLK5!l|3As`s{uBYE)a?x!VFW#3rUIoHd1P#{qg_)U_cNfPg zUq=1Vqe=GmdMxQuQI1P3@H>@akz?7Pe?IMlf;(_K>4(0@D&vVn0Rl{ zcUR_jK(`HW4p`E=!xKDv=<`@u$F|%ui2wHnkRfio3anDW8Ak;hIm zFYw6IROAyd!Rjdh@{iH2I_r6suZcR1+EM`zpY@IdBMA&h3h~OB;ZSAngKq6ni7x@{ zsUzGd9NFlM#V+4*PXk_yyxd%svfL1C&7#KYoB&|^?C9v2h0RzJ!~3dIoKe+5ZRWY{ zFB`nF%*|cxz;eF+ewEE%z@PZG#Hs~AFUzFxor*$Zq4+StPkum2s2gS4gD1bfpAZRB zx10jX7|_3)i)BEeT#-^08<=-62{8rM8V9#Lm}rHnGjyEVvZ&_GxZa!3(b}26 ziR^0c%3(0ezn+B2rr>2#J=vaO>?#KFAZjq7s(ybnyzyfXj*Kxj9TbAsN8ji6wxVs$ zv*M+m`+TbHpxA6Cg^W@!sf}yu)qIiN1ZlSS4`?O2e#1~*XE*Y*P6QX8_S{T?9U734 z7|y7FAsgx$SQE|VDWNaK*!wSDou1Y^VZ?~Zi15Wf5+3U85lBrzrAeVFl4DLcf~ zt~lPx((VGA--5SDf^(K;nXNF0ytet}OGpe6@D z-|ul-Q6AiAij35FIYz?rOGAP_H+-2gF*;crD!!h9(Bv7?2Ac+zPzHtx1rlBPJ?nLP z!>^eyRiKXurw~nZ5pH`PQj3R&O<3Iu@t@Sohkz2c9QMIvG_tMrBYR!h{@a^w6YQ2u z+&@0}?Y$Ix^Ir#_eY00@#aI2$PR#B@ylJVxA7KXDln5;gS$-v%7bz2t-6O>JiD(6; z%h`f#zH^c=Dp`fw<4aXUuRo#9k-^RX{=^e4H8gr)$+}~JmbvZzvOU6r=UARW=tyig z(nSvG3S%huJx>H$1#D=OZ_Gj<|Ar~*=Os7k<+2RTn}mg|33KaMzi@0xAN-xvAdrts zelF=3^9Pn!iCgK1zFnD?2~mbXqPsBoN^->?0L-V>+mw!-9R|G$aFh-Qw+wCkq;7iF z-_!ke_ZK|&O~vpD>TA}6N9prp9RZ6kC%v|>=nFmiX1#!cbndS?u3(b6Ic@d#i%a6K z4PUc*2H-+NoOG4sP&Rja+JJZqp!V5>?+D0ZT)y4^Z&BjuxCS(|E=|Vp4M0Y4wJpw@ zRgMx!CF+0QQ%TaWYia+PmRNpeJ%*zdE0Bt~YLDgXv=Va#0P0J3F^AhkW4UufN*nR3 zQkZ+oV$k(J%kp+6dE&)6U;q2FNU$)0>|K0`^DG4Kz43xYqV(K zC&@mst)`S;UJ!cT6>`RO0;`7!;k#Ai&yHLo=ruYO||*HM8bA)(Tq zMDlXPL6pmf2Q{;}Q3v=Z+=`ki0m{EN?j@d{Y+r3Zrc9r84C;}RmX=6gE0dWelbQ;; zhQi#n4VQlUvRKERP`XaG z@PKaD1e5l!u(=7_E@6ANTC-lkSCe^voBVQL)5Uz(R1Nfh<)Bh$mZ+cXWor7-h3?}O z@yzWDsk`Q}ng{3P4A)0qJ)Y=zl<%vct^5N03K$5^$EHKIr{7bcPf(r-`8H(CAeeRU zE6C6%vCouifrkUP{0U#W%WYFM2$FqD25NER1c@e7-u!w+#^jTqodgP5 zBz|1`!cJ}_Rp!;(8S}Tx%64{n%|H4iGm33HK5aX@RL@6QF3ghovK;LwwHS}Hyb3I! z9B(FTeNuG!^So{A<;NGnnjWpaF*bI^T%dtCXr$kSv$5j{;iqmmq4)QFPez=4);>$o zN~hedU9wp8zP_%LHpF7H%DS3xY1urm^o7E~g+xfM2H=DNCSB+$oKQ2T(K+KVH{^8* zH*eHX5qkk*sn|g(M-%4m-?Hc;B5>+r2Jd|^sjz!p{;&OT!FJQ#roO zT`Yw}AcPf8$h?OtUgM0W9B(_b8JmelRwZg!|MY7EYCo3Ehwvu_Gwx+a-DV>>XKUX$ zsg#w!K}RA%MvZX;ex?Bw{(d!2KN9s^TwJst2Ieo_k6n{3@$5Bi&%h-lUb!|_V84$_ z>n6n~2GvvH`(3DCZ0ru+zP1>dG}yB8S8;1L86Msn!AZsIkoyW-ZCZ@xWJ&9HJ6#48 z(@XS!Bbep>nn-LhiM@;6*xz?XH|7Oh4SsrU6<4Kr!Wzck?C%aQn%NZnvy;We=M$Kf zsiP;e4qi~`E4*$@1^%Hb40l=uMKW3U;FvNRzEpqT`)X(4I`n*|0Q5;LVU5NO7o>-b z_MlCc8-^1UlZ%0qr?9+nC6n}W;At+gB5Lb8KM!UVK zUXc+_JSVTBjm;NFhS z;pqsx!|r&ZOhz?MlI-42mXe#QEu~;kvMOi%Og>CA5}|y>ZcOqqC6o6+KlbIDm-9x0 zgij#<0-CZioMf;}Nj%7b)AQqUHYNwdR1**B1Sn$k^78EL%J-S^poiV2^I-CS?@goa zuSly|b(IOgh}CMC9rFyW0x66EWtdKi_wwEo-h)bG zB9#sX!b`caMBg=amVPh0&;30+n>J44ZZz|LTwHjAc|0Ra6%ZJBrc|(|N?y4uMP69m z_@3(w)uFeD_St@h~nM~ND+4Z~VHI|uzF^|lo!P}r-a%R*JXN&TjTJI`#FK~G#CbXkTe9dd$#4V8+y>> zm32JcJ^n&L##SM3pIS3aX>gMk#)$F+D*OeptO_nhTO=}4OhvxhtbX6?^|FgUX{dZ z>vpgkPxI7lCmaR+_lPeuF``>nn_e=rUb82?_szzy0O24=<~<-r&O&dbAW&7tsQ_>T zyaIfN0F$Dp=_^(@li7uf8vx$VyFD%OGQGQ5zFQO{H4~M1htWem z_{!!~IfDJ?p+F8zw;J_A$KXjB70ocNg~SW)b}|IW6jj=49u)YmGbe0)w@N#_t^|5M zt8Vc2bq37H)F!6}k&i}8-()bq$I*pM6k7<4og@+BdPN?}(jF)BCajG~-|bzC5vhSK zUGws>dbo9%u1J~Qq1Szp;R?9+Rv%b8(=)8`VUKB;(E$3oRI~!se6@S?59|!<|ysUFVh}95!~KEMti8PP0bN#^3cX= z<*1Pp)w(sc&Tnm%#4I2s@{tW}GbDaS@uv(u9`h%AdrBF=2YGSthS%k5*XifSy~7fy z-&Z)ZnD-Y2PLURzw;E{GSS+|;SH&=D@A`ReG+Om9oj;!j6^7g3x712&s#2$O7 zt=;cS?c7JgalUCnUQahdwEk{y2R>{DfkuxkI+fqVf9x-g9N-e#s(niPy$X<|K zhUQrVWqqxD+k3qlF0dQ5ho;ueHdhdC%#oIHm*!%RFxzg}?--;^kn)4%_u9w>T}ueg ze0rJXUJEBTUlOQ>!Q$Fnxk{d{d2cZ5ysVEGc^msT`oXsu=VMi`b1l6O!brpI~hgO50 zQ)sH+Ph*hJx1C_0vJ&7SN?cP?p&oynu035YEC0Z$G9OVfBy$NxYRprcGM)~r$F$B! zsyAiNR$+?)i-iyAzw6_h+2vcQ5fLSQJ#u=yB5HX$q~TZE!GfRBmbG3PC@{pWeVKgq zddCw{DYdfZJO^GB;G8l_aQQ&@e0agjRN7l=E&oxRLg+nGHXM-RY7fr6?W9jpcnP8; z$T0>CIpeoH8!SIJ-@t$F3H6hOR!nQjj#zCfg`Xl1aqBB4?}~qg8d1+vrcBwo-hY?& zRFKRY1Gx2!WY+0i3l_4zLXE}7krVabpMEF2T2@4#1*d22ek{p*al5YYl{y|NYJsHw zLc?LVdn&m&PUe9ootm{O*O3&OJ7_Sc3M2n}Wx_ozWECA?HkW~Je{tTH<7!oTPOg?{C{pZ1o z&i5v3!lGmzuyn2K-D7iTqX${yI%NnrE?EvquK!WUm{QL)O-+kfnV=;+j?q+>iJF9+ zm%k!9lCNI^KlBEgvZUE1%P+n2pQl zeRE{iD$4e-#!rY^-e_d05j-MHB0ujx<=^GWbhEzL>+zZICx0@HDpIcap$_NO68-aKN2UwZ6er4)X~{=)%_ zHB6|GJEqP~*W7o^gud3)1lXqM;{G#-ykR^c;6lONpua3-hhzs<>$J&jL_@<&VJfld zM7~!3tn}yj@jf|D0w*kStQmIR)S70WG2H&^hrFcS;v+0Slv{zvj|1!V=Q~+X z1Fp-nxAz%ptl>*7v3#4EH(Ez>_!q*NblCIgL5)0mS+a!}Qst~p@_0BPauRldCycyS zY^3WRQJwf8`g9$eS9S5B^CIC>KZ#?kNUT8VNH`?o$TKMG0LK5!BhF1Uo@B<08D2bB z)P8b5mT%fQFeoTvi{QKh53uf_+Ll$TW=r~O+GgCCR3lmk0}UY;ef0I!y4o~)p+}_2KjSCc# zfg~;{FN^6m5C84+JgQ4gF)k@-?s7JSn|sHI!y;bj`<<(%M|7KsAgkYLUoni0k*G&s zlA*w9OelwnS3&n%{;v-LUfe5Y_N+=o7OBk7=F!VN3#jLl*?M)ZCyLZQ;Omc7g z;tmZe)$c>3GYmy>H55kUjsg~}?G4RvLclFhG3?K^Q3raxhRs^Fy89hMu|6zL$$rgc zTy?AuN-$?{BH8_)&{X^kL|x{Q1DV^!DY!L{S4z*Xub;xqXv9gx77tEZMcaDdfqd>| zSrzc!fo|UQN+9-VzS0y6SYh_jO?Q{lcO@biU}Zz2vyi|za`lE>K;)k-dLc6vXkcbi zu3)Kosn-=U9^aln!=5IA) zA3B--WNr#NI@S5!?LYPh6H;wO*e^%2ZVou6Qp6EvU%^7oLg=VX1>w3?MRcyi<$*DB z{6;$G^``5S5PVtLat9oBg&Nfmrk2x59_fxn*GW`%`X~j3oR-Q8|~Hpl)lo_>Tqz#+3+G8>|s7rq^MLgodX$8P4j} zFO?YkwfL)uCOrNhMQ7pH)Z2#f(G44+fFm|SLg{W{#1H9GKsp3QcZVVjL_$$IrC}f- zok}X*F;awqbT_=`J%7MH=W}+R=iJYIU)T4NfBu}?7Oc2=U9G+6?+A=#((JJoL1ab+ ze>0yCfksb(AwuP&=E|U;jA@^k>agJ8V1NHBKvKJLrLU?w>9+y2ZI%B5u1<_}%0Q2_ zR^(jU^`J9mIsO!e9tqg)3zLHXsgXaW~b}MeWI(b8H@Q+Y zduZgfoj>g{hqw24=u1Du_T}YN7&TqI zb!(u3Pp*>=@}B%{l6ih2#GzuO%;Jpz;4#;sC%+X=)jo4sA|Cj?5H%nj6r?{gIqFJz zymyu9B~R`Moi>@i`AwS=KvsS1M+O5OUSDTP7ER7bE$@q48xfR>ExL8;mEQobxZqjW zXtCqr)rQ}pG zZ2M80Y@#gz;WnI{D3tTygY>lJXIZEBBG1VB(1#SAH%;49718opHUNxSj5{alH$x-U z@Wy`)1E!E9eF=H~O`F;fz+?5_XmU(t@n0p8L2|;n4WH9toyBRPk>i?*LfRKn?0-I@ z_petczc`}n{x+itARpZ6wT4{|XiJli4rT+31$%2m$4a_ay`_8>1-xZJ!Sx}E%w8rb z@9HT$jq`>kj%|m92ezfoL++19!kDiH>Rq(@Pg6PI*kc9=HXy3)gb?KS>1f9saV00Y zJ0o7^rWHmi_p$ED_Vsnp5J#lK%ND+ODvb8k)2_7eUCG{a6X8Q znr@r>C?ifnJ0ozIGoWYjkbTD1VaxxE@X5~B=E4sBe}5}ZYuAhqb(2oCW(&Oja!5ec z5|8n41}UH>PjwNA6{OF|Ce0^98(*T-NX^i7v5rRW4jd{V>N+c#M?~f`hn_}Lq0sPy ziZwtg;)@VD4N1@do-8QSvHb}ds}zr+`sLki3q!%Ayxe!DMH9K>*gnmBzzO9wsO-!d z`%ool@m?_XZI9gT)x8qgI*nP?J6B;nCFcSU9R+rLvTlR|t90oME)hQAfQxhhF3Sy18t^lj=Z}3&I+2V8kY#Kt)x4B$vJofnIpl>1hIviWyKE0fhp2bV@kPe z;mt^nbnGwA)t9{OobhwQ+9FU33?3h@g8Vn72dX)XHh#;Gi|6)L3t3g{zoZY5|LAG# zL<|m&DO7o$DJ(Cn>n2u3J2Z|$Ueu)D$Nh1xBMGs|9{!maz_|&slKgz#B~B_L?Wy`dOkY?ULRK^ z@P8PmTvgp46^Gr`89usC$T{%KaKt9*K6znYU$dEnt}hwpvHUN?d?}N!t)8?f78I8Q z1%v6#>FK%)M$DoYP4tc5(_X9ea}E4YyV(D0iLTB=kH;}ANrwtrH+ob%n2xm+tSah0 z=I!F9>KlxNzpID)i@%?c(L{k5)8xA$dYjk}j27yP1JmKxugX>34L#)MBEDaQNX!bA zp?@`n<$x54KO%8v9LUHOk`y=f64L?wIxm15oXaD3a}yV4*B(>)N8Tdx7fq;T#@JaB zTM!8{D0jus+O#h3ryRfSGqPvy^Dd}Xt*s`IyoSC{_=b6NVa`?`a@YsHkA8AZ)pUuS zdya?ieklKf;4q8>zVyRG*ixZa!CA*-hq8OnK}}Qp_mgg&+%JWxFLUxoGHtFS?ok;# zTlGL-qvzgp;&}<`OZxbL@!hyi?Evvz8lfmPxn4YbL|n=6a}tcv%1W=85eebmS>cAa zi4w63PjX7+Pnb8I)9E2ND(X*N?^mje3+`FJE&s4m=2FS1zejwkFA~)hb@IpG&3&z> zGMWDK`yW$KzGL_1Sf?Fa0;RINuDfo_{42R^G6h(89u@u(L zR|k+xVRm=D&of<}C#E5nhgYE;Q|RUgivr|TC)bz7UjE_dr!_TGKG{7w+P-m*>FBs=>kn@CEu__oGN5omO)8i9d=fgx!J*~IX%3$T z7^uYqKDm`&T-X&qU-@o7>vM%(K~g6Lj$%@a4JS}g^8auZX~l%m^;3Yt!C?#xQ!Zuj zwy}k`%7^V=-*wzz!`eo+fkK>K{`RC8$TPmVN?4BDJ2v?%mNCtTP~TukY5CkEoNmU%fmpD>5~^iwLY`P!8el+&_a`yV$Mk(gIw+NfKri29 z+GmP~SD0#%Uh(CZt+NgI4$`b12osex8%}R%t-M{=EAnLE2YstVj(JKYL|%m4{lK|` zxdsR(7Q3ZjfF#EgcP53}WDascp+U@V;$j&3IWyndEf7c}KyD@u;=+-8r8QtZne?;@ zKGCm)`zO5xh-@vUn+&Lb$g3$IIn~3FtydOqA6@_rlZ0D^XNC_(*s&Ur(sJ?41(lPt05MQ~m+Zma3!+VTMbI zUiruRRt;Hp8>>evH-`Zx%gb(ZxiG!G^HX4b4Rm#_jpn>MPXl`9PXjyy$%WzEmx;Vl zMy5ML=Ftzij0dg~R6Z;>8mMedYjBqc)m>7L3Lwf6Pz;doS=1q=?iL`!Pr*ax3cZQ04iUVJWXp<P zS&6)Qq!fJeWh%Ud^kah2Mpo*WpOSbs?s?&)52nbGp?7Tc1>36!Muv4)?uYN_W&dqe z8s!beHVAf^tPeZCLK@a?-R=Tq>i^^K%9w!IyxWwpJ0NcGw85g|{^GDpDUPRrwz+b< zxjz!F{2jyvXcRtfBOe)-OMi&C+M7&Zf^(tr-pC6O#=CHr^PqG)XWzN1nI?TbD=s3@ z4=KGA43NvoD%gA%tUlJ|H>P6l%kat0OdDICb~#v$89NbmBOIp1kW@MeVtNo9-5G_dsx8qvqgLT{EAPg35%6FE@+z2AD0^FF4be zx(#VolkAKrEE{j%^DE$W-EIX)N0pX&$#BAyXrR|3)qmenE06rhAvvz}%*@&29Mv!L z+%PZPwPrstm|>V~{rAni{Ez5>)qb^44hC7W!fZbu3hPe==y;7h$=zp4dxVQa=@~=5 zJUJ^HFlvE#-pF=@>JY=)2eer`S>eP2%QHshChwtHNkfBc66)Jf zdb++77yv7hlHTkd(_(6EruKUhyR3#j5j18^W^EueJYHKJi(fflI^%8=tx zXN|lj-4uUq2&twQ))nYV4tnrgghH@4AY0bbDjR!zsfZXEdGREa=Wn`Z`RWBVI!26!RYxt+kKgu?#RB0R>NU32!Koji$OC9BkG{i9!IMqY) z$tl>@2DVb|!o#z~4@u82Yr%@&j-=$Je{ZrpZ=y-RR!}tj5T74|!dEF#LpV4Tq3hJq zo;IF)4HpaXC=+zy7n7h@x6F&~?$GP^ctXND_%98n`egS-hOb`LnPI z0U-j2hgb?S(SW?O*UD1@vu$sIRf0$2-Ld7iR!|Bo+naTTji4DTIaV&$Eq$ zP#lcx1mltazn3^kifPe%#8aMnG#DW7`a-u^o2b$aM0fCVD#faev^ByW*ZyoC^~;2#-} zbxL(R$LwyrqX+e7+VtBn>J#g2pKe`lVa@?w4j{{2>O{Is7r11UxgP)Vdo!_Oa4NU_ zeqzp&v0yO%XEA#(34@?^vL?ojI1G^;RzgA|HrC;ql!wIgW$rM_}JVZdFk`)h< zv=?KMKZ5r;;HIYaUuYufZ3em5`;)p&km<+tL@$vvN%T?4{{e}}qk`-I8O{_6UK6D~%2blR|>K7|( zk*kVo8iJ9wPQaQsp82W4BHu2brG)}i4{)TO2@;-;ry39Ef)~G!4#POTlZ2Ojn#Rwq zH&UNdcWkkvZu&l_%MGW531Y2K+^eqMKJ>mUj~G>DfYQcjzN6-eY4%b0W&oBM|5NW6 zf8=)~q)VY*!edUNudPT|LrO6zXl?sw%0Cy(#^v4Kt#^gjJV8F7AI~xgmC6cs_tVKZ z7Rcn+S10)%YXE2E#DrXTcD=Dt5F{Xnf$g#P5xP)Mz7O9H~X`&Y}ri z8y>M{@jWmwYfZNBS_bZdk-`9Sm}U}^c*z+PFRdlkb4ol`EKtNWyNXhh4@ws1 zKA=$O9+&7&oLmXDi1P6EQ+YSPznJVqha;@0#jU-jCzk6T>c2wY5pwNoQU6@6y(H-T z>A)Y^w47j>jlZ+Tfh~nN%_@^*4K(&#Cof;$r=^L9B^t&_O1<*!;71HcvFZybMULW8 z^at=e`6fdoOdQ=14zKI}Xf4n}mXq$ycJ~&~&D$Yk6#)rQMFCuPgr4vxasBN!QWl;E zks*)$ZnkV#AM-?pI6m7vwsnkRg>unb^G`D;*0Z2+7}MUnfpM>R;332RQ6Bmc6kYc* zD`~yX)2UjtooA5fn}2ZQE%;WODWWlu+P(o zin{D{wa7Bg{NvWLGKQ$c3hgD$x__K)@3qX_=E{=Mw-<+NYVX|8UuxcgiyUc@l;+Rh znrt^42`iAI9sf;`0q!9MPeEq(uF>hMsmTb4GzuKvmU7ni^+tSs>PBO6cVavlRm zly6gJ(N-L4d%UYlf}WNdLfH!izYUseY-pq*w2rfMm7J^Ro?<6G_F#=2hE>Y$C-X`F;nIRMRpFSiXD#r z9Q=DkNeC)R)K4fn2`y{5_wj;;(@&R7rLP_WWeMjO5a?GM`A=^wXy<rVv*AH7g@fK1T^*EZa#@mzzu+hcHlx#P9{4HJ;Dg2aiW7lI67>iJAR zx1*mS3Ru|z!r$0~& zwyVW^8}-9Ihh2~MsxoUo2d?;pmS2d8XAy!n*Mpw&^Uq`o>wurp=?S=t%*O8q75}<7 zs|q0oM4E4nw2EPo?mK;EQJ?s#rx_K) zfP{CC`~o*5id98R@{N&KNn&mtDcJBcu?;|#R0dDcf1P<|C{RFuEHN6DVU$wNikoVz z&sEqYLc*X#tfWCYiD+=@v3o@-iOtlTO^Fy-hr`bUTrfOAz|3?h`dtPzAc%h6eoKPZiL$9j;NxczB=}JJ+O^$;OOEYdvjw5V3-612jhpkV0tww zPb70j8ti~x;@)*5KWO9hHk|)vyLf^dairSC!s1qQt99^iHDRPte?JBnx9WLmk3^42 zfv6xWOkdP5z?a;}@4Ro#M!*X*anuSQ+oSBgn9uHFiA-i17SqSy+v-FamYF8Awd!8c zqlg*zGJ=SoDJg8sxGJFo1o7Cme|;dfT}dNW`Nzx|^@xYyR~$@8A7S7~M?IP?ucWeh zI$!bH;4B^kqD%VrSWIFu$gO#+PPo1yozu{e{ZH<^Oz=`4ZQZm>`SnhvjYfBY6VJHh zpX*o>H(!?F^pdQ(^M4#vu|6XL19LxJ*M1G!S3SQS>yeRbFK;2^h5Ut4D5-O|-XE0A zJ3S>-#P3Q(DHKFWDEe`Gj^ZJ{*9FSSM>5@(WPlvhxjFd;{%MyEkW!-S+moLAC!O+b z6eqAC(Z?Z8i>_z`6=tn20gdfu^;+PA>H-A12ONAZof2K|%v1)v@)><@Y;R(0zH$q~ z-{+x9O^Bs2$(IgZtw(_qI6<9>em^Dz0@Fqphxm3Vhud>bKW+6%j2v$X6+)*z|40hA zD)%X$B3Usvw24D=;}BYld@S=pN;2AjKsw06`P|&7X}gGAYE*9~P}Fb zTweAPX_%yD;KiKA_QfR|>r!sdv6n)4Wu%bq{o%6(V~2)duZXwNXSa_xzGSdQATSEI z*vni7g*SFNqo&~$!1%mW5^E*3h^?Fc(idT~LT`#x!;j>qdiV#Oj0y3u3@G2-T9@1q zi7U2ysy#9TvqA$hMc}>`z5uUu;AJQvZ`$}Ddx}*(+ z&LrC}DSbIN=RqZ#kiCy7K{2FSXeuZh3Ws2x?Q5Pq@;%ECT4eDmT6?{txgPo!r!mhL zo76DfIoj!8Jiz3(K9;iNEzH*Ph&Dg=5mOBRrDf1Mz7l_*it1V=h|4IwcXL!(04?{K zASXRtQVF`cTec?UC!XL#%Gw(H;mWc$Y-WeW)JjgUU6J+*eXjkaL{ynjC_fEwFg z4NPM`Skb3`EsOoGXOu4x=a^Y@wqvH)EeJI7e>a0y9W~aHfe_7UiZQ2BCyBrMjH+}d zU+U_tRm>NdI*bfe%YhlkzU(P5gJo7c7{1B@NJjN3Q)5#m+sk)n;-J(bKrss_ueShK z1fVIODAty9@ru4H6IB%X$%j$KGjMbz9EH{xybBj8(wOES64B)5NSOjaXTn}=uc%f} zMeknjw}F~4@Z9xG*|t?6=0r;$(5(acoIYRM4`l)Xi>v(oO?Ab%=yaC6o1D?Otw8 z6xTdaif3V7E@YZJxQys~bU7Ia?b|-xFuT{5zx9^Nk_)>F?pRi07nrR_PgV+h%l9Xb z1N$*FL`Zm4t^owH$UtJIeLgJg9$^DQ&-WG+SeCss;Q7;;uO-QqvaO*tX^=jiUAd92 zt;{}1ePDn;v{%UUYrKSIz-rSI@0 zab~?qX0;8My-YmYj(KX>i+5`oD`a7OR~mAx;k#MQX?ge3k(=-EAu(_Ed}$@I_VAig zZtD?GT)f#N8n`-l9Ge!~JKLE5>Lh&zl)%zUT|zDS{`d*z!JA7jRk2xvB!O&j7YCjv5)JoGYa$esoI=JHvG{# z52cg)QQmp9T+N!Bl}!SQtXgu5#y{J&0>FrceCPk7w(AUqP3zP302BCx2)PYqgs-%h z5Sh~`(<3Q0TU5x%dTMw33y!H5`Kp+KKHM|)lKSYd9IT!*5dgUXoM zy0rYoF_@cEMaZw+H+YW39Ch)3#`4v6)!+@lv&Ldy=kQ*AnQ$Z|rC`w0u-8W`_P4J# zX0p^x&lh|q% zUzU&9Om~cC&&J!ZV7bHf0>}pmTH%sCdJ?rz03=R%#cSvf(3{zR=2+T65)cpnnU=Xm=%tp zm8j_P3(m;MR7rxeUNYZGw9cile8##o#Jdr{v5)E`4iAr{{D4aJ&nnW|RbZF787`s9 zmkzVBpi)3gDi}gQajA2{mzf718f-n=UVecbY(OwAzj0Jqth#;4nV1ST0XelC7Q(W6 zCj7PRTTm7YmVGB$)BGATw!a8|)3%}Ao~Ny6U>NzLu?zcUSZ~u3^#@0~)eL^WMq0pkA z?St4$cJkeqe;)58u4meXD~%hc1wi{iupYjYl~KGvupoBl#h;g}*CZh_P3NU)Lzrg1 z^7?!p?kIZUXjoJp_Y@B?lp5zk);iZd-|_*$$kr-*VF zBEl|g;t-p*L&p1*E`7A|`iH;<2n?E<;JuG-2G%_TQ?qlz{?bOR{!Qx;Q~{mVi*L}< zL@QWYY7Pk$AZY1zx`W?Fb*e-WT25*tw3t)ijA?7BlWE3Ae@rc$rEmW9Nsb=?;y4ZS~{+tsbwEYHy%h=#x~fJCcVe zf{?0ciEz&<6Q^*kZIHMM0sKiFcS!^_^WtM@-N4uL)HH@>rt(iwrZl=fnKajG6UHoM z^W_Z;AnxWKv6wvWeNd~^q7s3EnfKRh&Mlw0!Eb%w4s>*z)4eo7J3pb@fH(ma-nLIP z&57OG=U?Yi;e>>szqw@*(x9UU!V!z}vu(UVKc>$T>M$dryH(5_Ys?Q>SU5ofG*dNn z9_6n2e6s(R8n2xqP|QDDD(%evO(Pi7aahAH1Xf{{E^@YPlw5^ZMHj5ZSFc-^_fL9v zX+W*w)$<=d&fyiRAJ$1lezR~6cikfZ1SD50IWEQx&3tvP_SD4qX%amm;NF_aX@o`$ zuEWH9fdMYPb)c-SU{;hz+lZ~ex)rLlo4u#4oDfTRk#z!GfQtp-p{aghvO6jli?<1t z$4S&?R5l_0m6T&7`>i?xPA}wB7M`{)6TkLL)tuRQ)FiFwS4*mo+5WjHps4hGUD2?y z8GF`gw@|NHw}`lzxGMEsfYnQ>UrfJyAu$zSY!@b}i z?fd1L;^JMp`umMuTEKRh^04nKz@+(7|MWTTZ19gWrfmLbm1)_%XE6@xy_v@o3DSE~ zoVecK{2wp5a7&oD494Meb3`Uu53hkhXl_W4NCB*tp0MFw1qvtIRWo~#WR*JTZ03d# z(R>YhIYSp0tow{TrN^8vMe5#tO~4{A+Ug+Yyygkarv$wGz508xP(aKdvmesg)WQNC zS~0ElQTQkwq_Dn%vh9fG`7OBGEx%JC>!SFmIZr;Yv4Lp`Gb9C(eDQ~CIVed|;@vQl zfmfb2e-qK9^F~qKQLj!03bo+!dk|rDslSThWh+`Xn@GI>$^ntAffFkD4J^_q!+ie{Ym6FQ7WOnWM;X=&{rbK6Ki;Kh z86%B`ebK^sL+8A2dKI{j92wP(ff7m^F;EkHbwc8&+V-u5Jv}Xj(82~_-Ow+*(jcE| z0FId|L7~ll>7?JvUuR{ITE{4{yP6ui4v6u)@M@Bh6-HS+ZdCE9}Tx{_gL3wd;(dc5TO1x#LnXP9s z^RBZY+sQtXQVIMnbzFhoY4c?;W3&lAgf2NTH7T`R0r^yiG>_@LN&dHBFUE(t?+|E0 z!>{gcgCLhTm&ajO+V_t1Q2@6vho=Q&pj{*OysN&F7lhdSbljj_-WJm;0ajeV0b^=e z1;T|AikdVKyw;}${m)gz9 zldpG8r;pDIh$YKfE1Nz1qNd^TRd^2)&D|G5n--k-bMW|>w4}TDM{#HSzxPgf-eU?K{-G^9 zOojdl3Q#3fw}I01TkCUUr@$-wVT034n<$P%8?}UE8pr;=1z2+M9=W#H7)KsODA|jjY?E|NzX`XZt z=vfE6%~U6GdN5Zs4LirA+sb}v9=Ry~%7-fdk;RvxA;u7r!;#u213*#s>~wV%)(chy zO6ff>>E(SvA9O7L(l}2oZzAr;Db6~h_#lzU_9dBKHBxkG>UNX0S`C$bxIR?tlS|Ki zm5oJzRqf&NWgO@{CB-hSPK-q$^lsY%;sKoswmWrLk%?QI?XGvxpYVVt{o=l-Iy8&p zE?j7CvN?#PUX0-oBoNg{ni|*F++1qk11d$+26^wAcE~TuuacGj4&zWLjWq5)fhgLq z!s^l7fU?u5r8){kLu0Iur(mh2+e!Q6FS|XDkOsCm1>>Fm8w*6i+#g{r(8(#7R_7@{ z;U($l%#iQ}-*+&9n^l4lZRbM4YGiL?^1y_J4Ac!l@8WSMEkAOY|x* zm&FD?0lAe4xAH~0JxEv+yC4#WEL$Gq(1SqUu2GOrIP{5L``$U?)R6hr*SKizH(+9? zeW%++&{z_SVb85)-u4rU2ynpuZ<25s8`WoNezC&T)G$VdQLSn*0$Zt%<7YmKI17%@ z1<`N(q+wjkM7NAe#8qE3KnG=OJG%9%!7bO9r7wkR8|tn{w|0X4Sp%mG%cdLbZv7>1 z2gv+@1YDd+W6T|1yY+iDq@gC1NXJEP-W6SEWZ+{~Dt_JzwT_i69E6oBG83|Fr!eiy zxr*K&gjz^>Ute5oYE5A39cKd0C>NJ#3xUnp`&{*@rFPgAwmA0lTkPXKc$Bh%P9u9^+}u4J*^G;J!X?4Q72zHijrg^Z0ae8j0e zYXdso(_V+;K!Au`C+&Ry(5coX$dx&0ywFDFHaZD+d_O+U$EW6^YBS zH*kusToWXEUYRC~%P{Xs{oCuxdN%Wa^u|PI{@eeKv|i&!fi=vp^^2c2Nq=@Y-QUQx z`FH;xHx8rsix6hdGcuRRyPd|9gaT8Yw6wmpCIRgFde1{=i(X;?et&rxP_R*wyl=#0 z!PD|zeAVduR{bHX5R@X9zohTo*^8jHX@h@^N@q_?)#z~Oa6xVouL4MNyKQsPbVVoi zS2t^fx0|?j#U)IB^SWLR4dch0-L&_V!b>eGN|toz1kVIIQYwsimBDcenNG?;7>NRT z=l4z5(=ly=JsD*sc3eGHId^+Q!<^^TGz@y|LhxO%q@tTFU8}IDGAohk8ND5*#dGJ| z1?JdmM;Vu;Ua#et7XBuQC};mYOvfvx~|4$Xbgx~NLgPpdA`L2V)a~sN?b^Abc@74-fuBQ)(mbp z*G;SUgooq?Rvc3%K*fPz*hHY+39QVEfzV`J_X}i@u>lBh25UJgATfj>)2HXxyJGxK z5?a_pZetJ) z6~x0RfBDL?a3B$z%?mB+Inwz)XnNhGp7WiOCnoOvdK+Z+DbDmixxgenacd39<4VEH-G*fVmv}J|WD&@eeBjqFnGir9 z^$acO#p^>)s8xDgnuY(<%wl7_4^EMX42S4yJR9pF1+A?3Acxx6_s{3Nwr6~X9A&b! z8ZaM-r2ka`sv7bgvJCltg{oS=u-C%JXEgVr$9E<_+B!d*pire2t`L2Quy6J?rnEi1 zRGqkL*yt=ucO2Sui4JWa*te$?crUOz?i?HD>r+U56-Z=t+*IP$|acKf=Tf(fpO)|*XZnKd3 zcpe3h6i9v%@ADF+l^+oy$*eLjnsYT^`Y5*k21^jH(jt;1=LoARr( z|7ATd+*e}|Ux61#Fqy)JT&-waqkP}r&?Iwd=(yF;uvr7ETF}YuF{nf44k=J4{H3S^ zYnp!jv@90kkjwjK0{~bvkIyW3Y=f{{iwV2UeCuKN%lH2$r3z@V<$xTF>oG1s!w=~3 zV~l2eC}1w8vN^!XVd?2Wjf+n@CrD57_7X7luMN!jn7h|DKcV4F7F)*BsMT`X!Qfw1 z(pj_)uVvoLpm~Md>g1}*BXb98guLQQNA0a0UcG4PaJFe)=xEcJNM^0zZ}4KI2GqIO zj(VRo>I9{Wb*x|5)%x02@{vNfXPd0H&7)jG5y&&L*j##i>n?KgoQ{x|q3w>FfsVK? z8?=wwe@a!V?fb%xgpU* zHd2?ADQB=j(17ICz@V|MYF2SlWr%@BQpA5c;(|Z*8J~3Ny@x~!a=%@lG{G>s((^Dx zfdz1>27<{_Q{RXh|0etRH?udhOTcy8_x6_8b$xZ!s#QYQEla!~+l}{F%XHBMnPRad zAAAEEGO(wS)7B^63hhw(oJsAvOC3FW;Mi zUvUop>DaumQJDM>CrFSS&R<6i(mZ?(=6YVvD;U@PQuCh)@q~bRqpP(PKZjzhpocBj z5{5yl!ZMsVN}b%AXI=*Sz&kGN1p>^EAO1jt82kgj zg6j)8ILsxY|XwA5h}(t>q__xMmwL# z{_z3Ix-7Q3nAc#mNR#6!A?d6Hy&&iro&#w_YB&R1KDR++Z$&rjAL5R{vRaHF?#rjo zM;HIu^?eRB^D|i8EI11xjT+4hS)j0PmGuk0xmZ*(opQ!tSnl0C1IX6RhJT zMM#slSv}D;mAhhYg_{FD#v4Ay^>xne+kOqFj?lId&y#g;sY?~Q-2Lv-b``s9t;apd zdJZn`S?Vr@qEtyLk)V=BBKY}f$1*pLbiTa^-dPb?b{Qeru^L%)_9oo13=_;=#u1#=!Sx zzQ+vmibu?RIEaXSyY-DRW1n62b8ezp3Q)7S9%WQuaGVbrqeT44W>f2-#mtm^>bCHZ zb2c`h9l*P|9NtE&ii>*7bi2JBu=DKhYCsGuwCMz=c(s|!_Ke7XlaH)X&A4*pF02@M z0yhDYq2{~qBlXK{Nc)&6A&?Jvo@x``=0GY7R$&Lhgmy7|7`?Ui;fxW4x}h#f*WG0L z{-%_N$*W&0KWtI)lA==$hEKW21(LgxlamB@56guuGS~(#3KY_sZEuUC^&cn5BGosf z>F;}zKSIi!;;NH@Z%wq7!7kFprE{=LpmWb9MNg`?iL&P+_a~0%y(y43KH-ujcX8)2 zzPK^51paOl$8&Zz{(Tye?3FUU(|fCl4c={JZq;88N5)bTih)>ucXN;g?1q| zzT!PTpb9y>gRYMCbjdBB;NjTfDuFvj<1@K#2_B^BbbfJ49qMZDy}FjDE;+^a^ug(=(K&ikmHN&gqFWVlA>tYJiL{lhN3hL zMkRHV^Yw+Gk_kuO$r5>N$UtfOpOu=^fb8hw6`$k7L%_obR16OTCD?bRUlnIv{5?22 za1UydGHP_eBm$%YZTZk-Jx`5Fb0iPZ0kxw0w;;@e%?!EOZG+{K*hv?*=ow&Zm$Cd2 zk$!zEH{#b3u+$a=g+wHbs{OkgS-2QgpgVJ|)1%%< zLQKt<%G!OA`;F${$Zr^T7Q0E72Zm44+M2<=tobX}D?4AEefp8eDHvtdTl#nzoc$Ms zaA=WWufLobG)je1@G&Xh2N>($R3Q(2M?>=Pv}1t^J1_V~{nY-5Hp*Kr$8BcreQ8=@ zGI=*_2>kPQeJ?lm`Ri+gw+Gmqu)BlRw?TD%Ft+?!`{>9Z`zsSiJew?X$$kk&_v}U8_W|>N+n)YixeO z*x=C!`1LYE6c18rA8B7?U__a$r?b9dG^&OIE1pLoyr~S>H8gNWvC)?FbohdQTsPJk z$Ox^CCP9l-IuL1tlK9Yj-Lv~Jla`=x|(jSa2+`RGANbOyLu zKX7V5ymiZvdpvaTmb|O)}D~(AEmvK3`WS zy4F3L{({ry=q}*n1pvZ85GzBzCP8VhR9dcWZ;)57`5{f_j;(-pFp)e}cA)5sUS`u? zoc4KtA)lqM(5PtwAFV#Qjgauj1BqKHfIDzHFr$5c+Hp4nMBTeBcRVQ%Y=;xY#h5AB zvO?W6nH!^3yuad^39TZvd@o2(cg7JzMX>J6#MTEsmH@(~upxOGK!6+;L`@1%a z7pi7$30_9OE`znCuK;Ns_;5!4KHuy&{y>eaynJ->FqSARdi!)w=u`3$#+4F;>w%LL z3gOtR8-JL3ERw19xO8Yxfj3tuHR#RTfflb^*^);ZlZ*@n$-~7HyK>q}oTSVlD}m@2 ztGptlP-`{lpoWIc$$$Tm!ceKS4bQv5Z~9r_4?zY66kR$6x`iQ}9V0&Ag{Dt7j8U{+ zM_%hni%NW8n`C<;jU9Hl3rrF&uP-5F@U0XQiUlG!WCsJet z5DK#j&rQRHAIX_`d%O`pSQvM|K42>EZ}<9VS3E6?re9bj@ynyg5EXu$aFK-ujSgyr z<0pdq>07C3q8(Uy6dba^8jzTRh7EjZ3^AS;YPd~-nv-Kb&WK$A|E>CH;rkNHjmS^; zwMVy9A?Mo#JQuDhC74C2h1FqpIfXvT29|-H^4VX00Iy)wgD$6k|1+OJ?H|GIZy^k< zhze!jKTQ69%b2zGQlq))=46xiZwR_U{o|dv@4qF@5w}VD+_CH%H`kqAJWB9(LI-Ml zH#N@YElz79(w)O$ML%G;+($Q_pX~qm&USJaX6g~c2;5ES>^T)j`rVHsDwQdIfxw@Z zf0IP}m@T>qV^@QlEQw7W>u5`r3mA#=xeTt}a ztyF)8kN2#5@suU)t*K2ny^WTjitYS+a=6z@_n;ufghUU046N|SB2TN=Z%6hMKBtt2 za2DA*VXt_Kh_b-8Gn&RvrQr(`r0B3s7q5}b5%#}zpACPq!+R^FH9YYUQnS2hv^QBCNp`kr{b)NKa7to7qu~|Vy@HzhzgM}WLMxdrgeR=%75(Y^JfNuK@vj}=7#+iU zKxK3~)cgUaGc4o(72oV4*EOj3o3Nb#DVs4epET56%bFlOaYC}-_|htK__!W@JVhs- zWIP*3p6+C+Ch5-SBMoKw)tRnVYV&&#f}d*r$33j6JvfX~ZD#v9dOhdAPH(=txg?3f86&pr3Y-K7LhOrR*k(SxM}j3zyiS6}420aRj2aP=_2jX50*tgh5eDQa)ie!Ia{2MD#GY{PGLX`bdW=xRe8j<1T zMTN8=re0bbE<3tX)fNhr3eYGO#AZf8m3%t_Re9X~rXM63(W!b9R~Ccz29XfqPhI{6 zds{`oy$tT~%)J$)Q0ux9KWEd%I?E}2Z!opkG=uyqZjSq zlafl$W<8ffsc$IWbmm!p5EZ|%9?GwQKif6pyFX#(J+WmP|FesaS0qH!wOU;L`RsoW z=M9U(Hd|trET3GHlYl^ULv44Rqu^9jTH^mYI`4R@zdw#&BQ6?kB#_Ri{BS=Wllx-v>AGRmr4^K!*a_Pj>9M)vRY^N)Y-kMmwQF&8`>|Ypdf3^@BR%c>5t`D3=%$A%;hx1^Exy zDv5?`H~9XpZ;wN*1Bi(5rsCV-$<86_Ixm+gBiJ7g6T4v3($2q(Ha5~-eD4|XmEL~H z7=HL!yZHFSX!8xP&4Wjy|MGalW@81!~xo(jQ^9oOh=OmzPmemcNvv@Vg@_XzS3z3_FefQ zHrte`ieY>v#lc$>^Ex%Q)P)k%Ol}Bc;kwZ|$fm|3#0y9GvRVGGAL3^9^Er+XK@*laI$<1xQwUrVIU6s?l&qdk%GXh+DMjk7k z-`^?hQ$szt7FjXeE$cNOfN8lDgBvZ(Yz%^<>dx?fY?xTLOl|7+3}DFf-&Iz{0ECKx zHb|0m1F$Chrz_p>E>i%e?Qv9|N)QO(1Ebp6x9*BXsZLxscXi+10&KatE)t0bLMaX? z^bIv%e}6=@7ozhF>x)<^{i#;D6B-aBdUj%LY($0*PZ^e=Z6~FvE=Ah@>-(p8MUkDc z;pan)jzp#l`oHqwWK-)Nq^)&irD;5pJ>o-Jeti>>PQU0ewIOy0@0%q&PP{EGX<+Tp zE5|tl_drBB!i$qok_I|xcFb#s6=h}Jaw&~T7+Q*zc;(hmtAv^ht8j7fK+_H)YHAM5 z*Icy<-uv6_UW7r|%IEcAs$H_*HUt%?=qs%M^5X3R_4|U_b!wKh2~l2u=4|SKk^;La ze(&D9T9b~@dnZEqe71T)5bk_me%CI_*Xab8s{6Hx1be*3ud#+lzl0*_5^`;rA(Oh) zpaX^@y7gYUx~N(+O1uFvj8{nf=O}qxDG>%i3DSrAj29x|kRfs-mW>TQTpc?kns#2F zF+n0%UAf3B7RhCw7AI_76aGrN?VK0_dheQ8=mYIy*ZJ^#SIYXxDN4$<014%EbqKZ1 z-m3PKl}}StRqD+@(yRBMVXY4752;7~Af<(xPQEwrDh<}CH(NUJ3sI7q5b0p#qH8#S zgW6elyw+t4ZT*4E5|m`!gkc|8a{#_J7&_i=ERjyMM@g!))dF#Gj`T`VQcSEb#S*LC zmK0mc!&Qbp!31LNa`Em^;2Fk$6X{o8cE5W=tfC~+&5A*h{BCO?zTR3Q8m3+@-{)!I zNcowTS6hY%4=3CjHr5ptg%)TTxm0jTdECPGmynH z4U}?*2`MqeHr7^>{ggUgyI&pc)0Kurv`$yw(e}$6>C02;qiJ(T*~sSc zdmxNzrM- zv*aX?-ms*8%q;_vt4V4=wBfN1d)M}XuP)HN@PYE_L& z_GOzQPDM)5Gu06E)p)7}h1;xEA&h65E@T@)er_pmCPr5aWF03;x)8k*&tchukx%jF zK@3oRIt~$-a@G?r_c0+Zp9S=Ken(&3&PaGb`I^G>LREo~VQyzGBLg}|W9_a6qhUUN z$fK{|X#XObCDe@MsuHYKZf>f9ODyT$-$^7%xeGqJjYbVRY2Kth7LiTIR*saHbK0)V zcPNfgeXf-S7&*J}Up{p)3T9aJ_`Fb2YLj201Y4V0HC>lLV4J0LXs!|2Y=5`y>dt!@8>%@LPR2~tA%5yMCR#Kgcp}!AimcT?@(fb_iq+qRO zo~yvbv0**-)X_%w)#JmK;~g;T1iG>zKHp6!r!Pd=`~~ofC9ge@im7*EDyje0(|Zua(fuf36R*`{vy+N@|3Hn zdsYBm1m^A!jt{8EdsM6Ay{WcnN>D6F%FH&zB?sATe8%(Q}L!PNfn{la9^)E+dzvCr07Mp3Qozh&4-#W_E)0u^0^qkHn&Z%)-2ku)s~A|4uD2pjh|zOsE{@!l^V^K)QL12J{;nPsBaB5LwPPadwhaZpoiVEI^d(rV&i^)@O~N)$xOR2@;>?5%pEC@GVp5|QFKmEyc3-fR zObAtgvB+V#%BD!Ju(A+7l{QlzNeAfsJ=D#cDM51#AD4 zxKs2DrI%)|EYsK>1!B=ijb?@!c z7D`Goyv6+RfgM}9s%Un1o@6ticy%2qk6~m*VPD_6RP?4f;yRB5Z&x6qCMIS`_yh!M zqm``j+vp%L_^aFk(#b{leB;bpQ=DRPjHp~^Xbx6O?lm{r`#&G!b0@?l>$A>0HH9L% zP6(j)$JatSdh-3AazOIocR$;*k))obx>eougR5)B;E<Q$7OkNb|BtmBfp4K0tf7Q8U2@9?~ z_A(kf+-W)p0u3L~mCQqm1xtTEXjNbWpJW}+t$ge8Y5!<Qawi;)^86;0f!y;}! z;1l_+dq-Pik})&|A2S<0iN_{r=xI!<6~Z*ihas5H?GSdBFnVi-=hkr-AucLcMmfK` zAGDm`Znm8YiPRP3Wn&MTyB){6cht>1Ot6oo0?UIl)DL8q&JEDGnNE30gKUN4&+JNc<5vo(Y*<^ndDPT>tLpLI9eY_e)7$ zzu)YVeR+T|2kKPQvapW3IWWdrSVj-I27X%_Z@eik|Mj7LIlO^fvdo| z9`E-<{u4*hrc3em^Uq56;b!K#gj?QE9#ZP+DhQJn8n`!z2O>ig>pQShHctE>o~=0j zB|mOfaLR#wcwAv5-4Wg0h}9k)ODbvov((jn%~j<%b-Z+BWD*v>GI0NjZ9~=bQhGe$ zdRlS87(VmjZmj`N6`NH{;LUb&T-Cki;YGWqRLm^cTOhwZf|T<_>Mahzxf9eun|a5y z5dNsoy0#&ThaZ$Deg=72bc2|TD>w~|ll>ce*Hykq<$CgKKtY!L{5^>pbVM2GkeU0n_bUXbouy3J52*{%p8=c5rQJXx8zwawj3&yZd1!&9weT#w@g=Oy1XrBKpeLva2JIF zyXWtNx~GW(%f@>!_jRI~GsQU6AIClU`HE*|!P!@QRDF~>{e`$LTInR+@|9US3~fpY zgaiV;h7qa>0zul9^Z7Gm2YVllgWvu9nIRb@JocJnH+IZ;xejT|Ata>D z;}YMvIb^Uf?D9c##V6aD$KmCIbmXK0-W!ZvOrK9EmRyO{|9LJf25!f_81|+8zFBU! zd><{E#sUeXZ%&LlSn8jjLA*E1-fq9!Xj!EP1)0BY@FmTrv|j0+ma|#w*;u8>80BVA zU@4-hgY&!lAlXwddPd#X@8jqWNVYAPCu^{qXL+n+}@KCS>$w*E%esbT64Ry#1pWM|B-S>-yWU!{=@81 zeyT<|pRo%}K0+GRhQ({HQv;Zo61g0N60oY2U_ z=VEZS9mw6g=QhCp^@^LhtAw`X(`fM}qWfo-w-UG|MC0$g2IqN%dTXnLPnHw(GdE_k zHT=`b$0%LQcvoMUFa!{;SPXj@haT4#UD)_~-+L+^wZc#%WI}X-(lr?#ymbl~m(O1< z^bug*uX`imb0aR~xt!odhSSJUElF{SLPf@r%_8o=>j!4ouuLKE0uQ2}j)dnn9WL~i zz}qBu{ohUn1bwCc(1YKRN$|^|Z%6qZcIRsj$KO4ex7CqteNw~EdPb4375cgwD;q(CBt)x#58GO3NWqjb$X{4HPv!3cNN4c zE5M?7eCV2oVZ?o%%&-cUV7SB0hAb0BaQ#``o@Putd?C3WM$o&ysC!Xi#XmD$6q*R_ME-V`jd7UFCl0XMt@21B{bn=NU$s3gz z{;TtM_M4083YNI$de=_b$n1s@;rl`QVy9=0f1t?$_Xaylnl>SS;d znMx)KMwZI;&>nZIW3CYHlhp(7pt-W~FVL{A9QC+r6dlqQs;Enc!|u7aOz;gvjT{zr zKl?#+BJCgY+Z}+z&9@cWLZ8ERwur0nYK90fcjc11uua7ek zQrbvTS}WXGP0MFm%Z4(?Ows4%o{DyYGVwyV^H*_GCSt5!&4Dtl%fl<|J-aRw1cqYwuPKRLp@CMK=`Rvfis7d`_1xi65)1c5 zQ6DU1#0B4aB*9bFFpc9a>L)zUI!pft5uRs2qr67gTJB97HXSam+>3t9{CMZMu3F}+ z$Ky*NQXN}byM~^6S^4YXN8Xta#C@%r{oja-E2@zpg8tsmWGF$^un)Cq7h$hN6eT2W z1Pkmx`ky?l9;4Cg$QwmB5M`5(X@hm4qgBnv>-MqV_ZFTC@csB>9^Ie#)s*!eG5hNq z0w#BNXQpdJqqi0QQ$;b_tECTh>LOVbZG@5ept7R<$)Vz%`F_2qeR*{J#6Wh+t&&a! zvfi7wcTV!g+o*Cmp76v;=~g={1-ft6Z=9aCuTRB37f&HPac*ksL z%%B3+n?XknEn{yRyyf;3V`uj2u&$^*C18nqg2j8zgYIQAm;meLJPUF2yTCQa3wcDT z^S1aDbbT>O1jGq6di~79GAEh>M6h(NGBZbQZQ=(9G{AOR>dpXC^E+TS{)&8J-ZYWn zy;|njH?j{55;Picq-@*)%|pyouvOq0j}|fTN?HT(N#%!CE!L)y5%0ieMOtcS%>OSk z6hd%;Z%DB>X61sMDk{wG4}tw#r#r7lT;z=SWppGoIYO(SAdC=lJ}LL#r{C|kgGZB0 zvk(nxj~+gWS~R@fQpG)BS7`B5iyJdLx4ZrOXl-OHAY)Bk%eWJ=wckzzqO6Ndc@G4mTkkURX%39DTD{?G%H1;_ z?sz^3xxWyOf$#YK5oHR6+KBiUW?-wp%S8G*N; zaH2_SRPwl6?)Il`8IhBm!>!GtzTVWd@wTF$-em6bn3;qhX#{MC&+NcBN z)NXtjliULLBv{}B<%b*cJKTApJ#2s!{e(kEbpT0^;gFci8CCm8;>N3)#qrt^Sv-0P z{dEquK-PzdQpABn3l_n6y+BZ{;!aI22odR*^Uj&?ztrHgbf}KfM8M zC_cTEn>8v*N*4SR42#CgyEfi0C${`U0^O{*=^$KxEFVznntaNg+(UuVS0ir44nN(f zE=B0?oRn%#pgy zQpq}nu4cMh@=5fK&zhQ$0MG*AViH`9J1dC6NxV+={FTGFQZNnsc5_E(;HF8KhITCa z?|6q43K5QYcgxYZ-s38@;u|Yj=XwPF5R`kh+S}uu;Kz^dI~C(Kmzq9rmCU$pUJugL z`}A3|aI#aC+1}RljL4LK2bY|a`&E+Ol#_QflaCmu=fYinoNHBDWi5@?No?NBoR-!5 zbye1Orb*S=Lq6JtLavR>iLw!7I$d6Sv{90GW&H|gqDb=I%x>4GpE(w)+5RFQE>qrJ zE|z6$lhq@-^LBwVTy%7_qde^;x21Jm+d#)Gnq6 zJ$3mEY?KWvvK{Z)i)IJa0{ljvjt?-DL89o@V&J5BvtGcn6QO>lV-1ff!cgSeg64=UiFqAKD9W=@zQ* z)3P*tbR52(=CY_7Ajq5~(j%pMN|iHa zc^a2NKyDXg*)HLeHDA?s;@FZreaHhzSiM&+_A+0Jb0kG8>&S3hTt0yCxVsfJxQiFS z47q*o$<5{1X-T9Pt`}Rh8993{gkm3?rtcr_(fU;GkS;9y--h2YFxaK6=V_x^jc;c! z$aMnD9@sHt?>`6`HnUTe^eixEcJVRr$k>Ct2h5-|m;@wAPvPVJE7a@2G%S13e>1z; z{c=&@;U3S|rCSGE`!X^IbfNCNq`<1$Ur%II1_2iBTY5sLERTz6dQKGl&kv#PTw4Y= zkyAkijRU;i7rF{{eclWUvSG9$F`sD1n~8@HN2bBhhD+60m3J{``Y)pBIR2N6YFB^k z_K1MB`_)nmGE&hNajmOArKRyH3(!%C3cBIsI#!wt%fGkc3T`+N$lU4nO?@=UEV?t1 zJa3V4nddHUtL#Mgd=}u?61XVEZ2Y!Lw@Uw>{>Sl?LhUDZ|adFCY-;nOhh&!)#|K9xjQf99wm3 zb?oqRz!hgL`g<+PAt<)p)Dm=&Aa3qZe($X5*t?&VcT}2DW>6{I(t%r6@seWhUh-)hRlCQ!jG9GZ6}4hi@}0$3n&dG?* zuiVH;=(D>*r$T326*&m~+5yLFYV5eJDfgTaELE4~iU7*J$X-e1zIJBVL%PGG_Z%fG z1wlba8-Lfv>nS5SGy7HoFT3mA!l)zvS-1VZ4tsfFm@Y7_KaxK$hy^t_Z)BGFWXN?+mz!=MMx2X{eB;~ zS6Zc(^j%eXf-WMGMkYcOLt5sf?nS8XQcBjry;I1?@z&s3vNCK{5n~+O|c3&EqZ*=9dfudH8dv5>dq#|zCh!?CiTb>LG;i>=&eWF+=~?B zY*4y$i%jMGvBnu|9?`YaqM8%@^ERY(|h}I`sg=L)G&pTe95V#s* zt4FnMIjpKW-Z}2d`|JL9w{45EP$dw2wB(^&IEJ@HQ?E@d`)6*PN1Uo zKv$2m?JLXMUuzTT6>5|8{$jSN5ec=bf{gvBom6De`!w(2bDw1j!^9z*`R7h{6p`~= z8yT4C>V(7aNtzcB4a|C!Rnc1Ni|M-`dfp`R(ivJOR)y2wdXLh;g$iN;i(5;S?Tgb( z?iyQ+vPMH|bRY|cj{ejodYN`zmar28&QEFy@lf3X?hRkh_Lj0CY`V8&t@w%BOQSq1 z=cY0r=-KF+#pt$))IeP>bVqW+`86avqdo{$M)H)=hndQpzro1BV5q8bjkS{yS}Hg- znWV}r9|N_?d-)NyAwI9X&3F94xRik3V-*QlkB^~CG!AjS!Ly$u#b{(P(&1dMLyns--zF|GGo&A+uG4GE7rlM8qWr&GiP(P3=R$~=Tg^l)Vix(2IUIXF&S3g2s6=w zHA(i!DeTO<$Smvxis@p&{|3x|;_?4Z+$#XzW8t_`(h+b%=KiM%C5z_$V)D3{QqgO| z`%6K;;~(i^;8QNoR~sX3;A}(LJNt)g^~|lZS7jc)SRvSr>EMzy1rPI%JNMhvCmeiR1Bc;I6<6A=rE(hsg2YWlvd26Y1>CfT^Zw z#db_G!|S2bZ+!k1amxI@eC2Nj-H^iS;({N^tzlD%`*9n0iUFF?u zIc8O|7<%|(+!c2ii$V=IXURd!!F`8|0N;M&W;e@rozITI^oZv|T{1m6OE#Vba$jw7 zUJbQ@@3U<5?DeY}mwPDb=xv@}RF`%0Y~tfuT=wDN$25c}J8rzZWu zCpb4>QG|^)3nZbeSf$U74H;QsG#dZ1u??~MN*he3puWj0<|+*GqQ0% z9l@+bc1D#AAYE#-D{{~jK`{*y&oK4T0URDnDa^5;N%u)gW@}cp5d2#v?vxB@z97D5 zFn%zfeyW13{d@Tgbz*CR$0bVLLDyo{<6jt#hC=G?PVn#kx2 zD}`PH=am?rOBa20g)~fLvOgd7Rt1VcD7hs^$HB3m-iao>a8bIrFyY6QeEQt*&1VkG zj)QmZLgDn0T5Ygso9YHWfkFSR&f`KEX2@C3A+wJ6{);{oyf!W^zrd)|VW=Mw0l%l^ kS}L@5eJjL~ogwu2>R{aXMsSvFG6ekG($vFLs#}Nu4@VqoumAu6 literal 0 HcmV?d00001 diff --git a/python/nano/test/resources/train_image_folder_png/cats/cat.100.png b/python/nano/test/resources/train_image_folder_png/cats/cat.100.png new file mode 100644 index 0000000000000000000000000000000000000000..0703edc2381dccf285a5a92bf3e044ee44109571 GIT binary patch literal 279929 zcmV)ZK&!urP)L zS(79?mL3S61%Rq8?j8|$$;`@1R<)4RB0W9-|2NFjJdKduL(X&;S(SH-#ogRYRRNr3 z9-ykGW*(6X*|;Ho&CPYea?Xdd1OLx|`7iDco3{SiJ|H3h5D_9GB3O@zMAR;DcQZ4y z7-Rdq-qmhQ-OV%C??q&N`1+{z0wN-!_CQr_uh>3ey%EXp-rIgMbL*s;iAcL05q0^{ zM$~?82fMynL{wEEySu8ZI5{FRQD7u@CqNEJ1c4C101N=8GEYc*BN4TptOE%lbX^x? z1OQci9WJ$26;aiBU_Q?V1HyK8+})j03?alA*Cl9Uzgxt&6LR|)k1@7uwsYm~u}j-| zZnJ@iA%u0gx35bnF*7r}yNb6l0#Hh6w>xxQhlt>S2xd^Lh>t{s$fjThW{!x2A%x(d zswz@NN~x1bQB|pvnY9kIcWwKy9p=2CN=qMRqTTlP?L1%%TcX<1&vy7*j%b9@rf&QE z_2=3EAn<;EFSb5w`$^ocZc zthKfkB%<~}M2sVnl2JvPHLbzE*l_?+YOikwe}trpQkwe>-`(|a{eGSQ^+RY6T@i8ng10v#qHiv{>)iTiB8oAF5SV$& z(;dHUm+h;tb;k*7`zqY{W>3InFZ${VZI^Q;$MsR$saZLenH`vjRGkpXktj5o3$ZPt z0T}ppweATW79Lys(|B(c3AY1ucihnAmI~WS$+Fq5yYcq<+n2V#Ze@o%@nO65ZKrx& zh^?IaR?OMn)vj!p+%2fPgI^_dScV4+t=+EY4F$F#uLD`9Zbh!U&Lbit-!1fCAO?8L zf=gRxW=kP$9#*;UZnoF=ZaapEfCg~SFY7!tnrQ-idv9wZvL(^&-m|VF5y98Gv86z` zWuhj>uiUiW;@?0n#D-Id8_)y*2L&g?rOhVzZ+T&TW)te? z38%T+VSC;7=i2WZnnG-9PHU^WvCdk1eR~;f*zyjEw@01hq!4(hiM z+&YdM8Fpptm6NyQb#H>NZ@K+$4(~`vzd>E=NLTdS63_MtZx?R6UP!*e*3wx600ef> zjr>AQdk7({;0PO#+U_xL`Sh+2+fLrm4BXzk9on}h0st0eu{B$zusddHtK9UhEyp!l z-D`^y!)!wo`|LrU_m9T9j^J5p}TGg)Y zeBM1|eeWG?-Cb}eb0LC=tO^%6tq-YEo9yp~ix`UDz?W{+nVAic@oK3sGdF7@z$zo( zhzvUGA3%gojojUqdA*9Uc-8K|+8Ud|0_DE!6Af9`nS47T-)e1YZ;_&PYn^ifz!+}? zx-Ii>CD4_M6gDOicWT-|Bky^Ez#V+HW_$&+zQtvmo-el$fNics18A#Yh*xZClSV|% zzT95aWSSL`*V$W@GsJ}hm$_Y4lP%#Dk@oVi?ew|^zLBKXpRE&XSGIC&2(zHETBDm? zTtt|^LhfyBcq`LxCzLjJqLqm6)^eT^x-r@M#t?WnsJrI3(&Ea2TUzm%3uXz(%-t1$ z0Nl)AK`!(wC~pa3`|5@9tgT`bUDw^!7t0qSf-i4eZrpbA&KB@3np@|vDIM#0*ITxJ zkF9L-SGf>3wgp%idtt5X0@YfXXR!`}8-=)glO41j!#aGt)g`x-jEmI#)k*jZlY?NeY}gF|VHO+N-5vC5sNyERMA!hmh^+H-<=G8> zHN-%?3gGJ~+9Y2+SBQvM^=rrl8wzOfxxQe%KU;1H)x@i|nZpL)8qhWZB_em(LgqS~ z&9}H^cv!e=UAgtkb$Rd>wh)n+o8Yj@bnW(4f%+;vZC4cF9;vqz?Y=6!4(f0N1e=Aql>yh+wJk_ho6_D~dsm~$8+#KXt_CmKntr|}+iz_yZ)a(p!|fQ~?iuss z+!8(Bz799Abq^UgKz&6nL?oOUa_uS+t=R1D4$jQPt5wC!mwoXn70>Etqq?s$n|tG{ zd*s45vgF^=)^XuRwym``b+rxrt-imVko6}b0vIA7(&n4(mV6@$ZHKmH99U3hyJ4&h zvW|C41Mr5_->ljbt*cgupm}9eR;i8q)Dh;{Z9TAAIVea}&xTU1K z358o!`sza7HCiGV+WM`Nvf2giNY=d}T3gFTMk_velR28Z(3@f6$}I-NU4@G~ZJLPa7P;oRq8r0GY|!d0bKZO&%v8)g}1 zG0!-Fxi|F;H}eMH=Gb+!yqkfx{j8k!E%w<=i*82F=GQeM?^>RH`;6Kc^h&wl(g0A`yPcam&=N|W(IJT?O`R81%s-}I%7=}Y?HG( z(YG{Vx87*jAWoBcx7W8n=^KMIY~+qDA#U}Wnd}yULRI^|Z*K`9aKKdz$64>I;6xN= z;T=Pa0ni(oi&WoiX)8I=Vrk;l261Z=YPayxRy4mu?e@*O|HB4!?zYK0T-)}fO@c4m z(n_Cqt(oBw3E;+OwY4w;fcqkVfFfcGZff?HCKI%8R-#=Wg}1=E<=MM7U*lKlb!DP0 z@7(=FasY3ZO1}~#nze1p+_A-5aSTMjDXu(dZ-Oz`?|p-wD?7A)nt>T0LNoKXdz+Nq zRF;{W-uBfuD7G%lIyRM6meKalOqkiMak#ph`xQ!vcnd4*Jg$z0m8x)~if`YzWxVw> zUPX)>Iq{CpzDi7gZ(An>Z@+97bRuFxL~q-U-umFSECILJy8Y{af0zCq+s)t7XfgfX z>FjsQw_bm@Qn-oK+xB)v8DH6dLtV{4ON@v?ReTYmiD;2?ul;GL1h<>{bxsgq7WV-O z5s(PM5KR}kZ1K+!B2in!EylqcLb@9-E&l$y3Bj+4y_=^P%(}3H-_L3r0f2{q>u^?N zdArJ2&ff@_zWG%bJ93+~RTYSIuV`-v{+BurzLBtK5xW4;mc_gYwbR8xcSi%;&)YA* z+JU?2MZ~y7y@ahq`c-Oczh064HV9Bvn{;*;W4%7)PU>FUx`(v8R`J_54wSzzn_H$` zAHGH2ua07w_f=16y1K8Du=^EU+5=YBX!6#!M~Fy-%t#39roImUZmG61|Jzajk6_=o+E)?k?#u13zS(J3?eA@;HbmNH$-$xR_ltN0h`>CXW=kpVZG&9_ zbYA5cBQqne=2XGW=C?*1-Wg$ahuOip&Tkfa+s}O!2$XB1ZT=SM7R$rJIkmIT?#uyoCj-g+qV0U-F&h^Ss z>%(!8^RE$D*QHvf2=0f2Z#Hw^v~LgSf2iG2!uoj~*V?yD|L-N2uQ17+5dgTE6T};^ z6OnNoiJ=N#TCZ8H0RYepi|Z>IS1|LPGj&VwZzll}uOX4Q zYr2B?-Sr!t;465CxaO$A9Imw4+qTUARRY+q&fTlG^G}-q9ot0J{cR-F21}b?5t

vQgWe}8on-*U^s30IkJ`?SAzOIZP8OR}3aMu00f5>nT7Er#A*eWo)pGoWs} z5`OE{TajyP*t(;$x81!N0^ZEn{a^ow+sZ0io`hRj_^wx1zIxLo+~lv^4s#BUYx85s z0MLM?nYwb$L=Zw?yvZxv8mHFhE~@*fna;V7QD7! zy8FWF_nut8qBgFUSVR=5BGOV9xA9PS_ilf#f5pvCxuyRtT2-6Feg95-#anH0-DTY! z)}Vj4xA^<2Xu;N$uh#RI97V&o8z=@hjBqeld2QRcyVdu%T_IoeYG2Z=uW!1|+Fie2eU~dUx5wVadEN2n+Yc*p z-F5r!b=O~;tq~D$V>Cr`w}!dPfP7KW*7F)amAex%0+9pEM7R%_{X@obj z3^TjbyFcAQ(Hm^Kf9%^m`G2J0Ehn$P+HE-4-H7IXUInikLGqSln{tQX)U?Z5D}wcX zs4`3O+o@}ACL(NNulsDw4k4*3q9M+eB<4_Ty4k3k{TdN4tzzc5ayuk}Exf5Yasas;D+qAoN ztI2teMBQGn1;v}i08ssU#^O4mE0e$7^zK%1J5zs^SN&_?^;etU?(MM1o-1-O$0l>Z zqV%b%Z!D2gZ5`m<5Vn23;?y^KhrIhaa+}@%l-|FFZXd7MWDjjTZ{L3)3rLJ#L<6nKk z+C%r2lzy$8vmI0uL$=mr*Z|+#x3}-th}kumvoTuq>tkWVeg7Q|5#6sQKX*6RTCcu^ z6-mr&i(KpOPD`MClQ98cbxo#}zPk5cqaxN$zy>eYFYz`r4ggHSt)*|Qktr>QmJx`M z37K#!6E+8`wmtwr8!a;fnz@()=vC)8Mt4Uro*5fj=wAq-8Jb$xtWA`AxSLIyeGe^` z9hYu2@1VPb^8BTtr~|ZxCva@~-m+2fa%Vuqz|4WS>!B*uYf%w%2byJm^#I1iEhkAu z!JQE~gi?x|&*GkggPH@VF>n_;axya$M^#30BLF96@(>KMfI28L0%Mv| zTMoBHoB=>Y70@bL1porYKJ8T1m28&_3<>reF#mQK&a(EjsA$qr_&zFguVn7f!c0EG}z+;<(v7#h$KI#GZ% z9B+wJYZAa3<=c?Db&qa6Fc9Y6-ZB0L0^aEJ%jd7#GPhG;Zz3D+_MR=Ptn9Q3AzL2v zjURgL%=Xv6ykV?Bj2lhot{dxTTcS}`Dg*%Ys}1;V=hwHaeOm@`BaOb5N&jm5>PGtZ zuXk&-uE=$PmOml@6SdtI7l-JK=1q;BcZOM?T0y_v>+UUa*$olV6cE7_h>!rm9MEx& zaYJ%&KrlB^X9l+bJlk?xZxJ2mtq%aeK@HT+z=?=s;1K5EUzt5W?%-&^4(6`b;=EcH zSGpubw>iB|7d&cPmRMQlh7o~qj>%jzlkdo^Rs{e+^G!0Qf4eN-O7_0Ag@gXm;`*D` zHcr4uOS+h9wIyeM%Y$&WkzXGLi#WM0g0y{UN^||c)%W_AivGs7(j;vn;LUWE>0;=b zs!KGxS%|{SAcl=bfXLk&lCGUx8E~n}_$zzqh2U;UJK{f5o@tqSdR zif?((%vNjL`h3J$X2LfW{_bFFy%E1rVTrb{2C!Cf;jRO>zqXKNUn+96xij}w>wen@ zTudh`1;oV~z3r!cJEQemZL6BCU-~z+n7b8u3uo)EH$$_lU1^kwTwh5#q3_$+)m)uCHglZ*UjL8S0^hh z5VNkXZ!8hCvA+8z!gEE?C7x}TqTBb}94>&2L@j6p9q>BKxRD3|;H|%To>|+{+?>k= z?npP0x9gfUVYOKYh_2;&8ilWMvnz0{%G&KPJ#MI>CHkz-|Lf%NjrqR=>-&+uY3nWS zj!VWeEdGb~uWb>yq>nTUHxW(axIT6J7Fsd{nOn&5EUpDdaWJzs;}jU(49H!{5mbQ$ zLm*}XP&H#P!{*eO-Ld8XP^9Ks%rvCXOgl)yTwIpU1A$%t5rn2|-RW@RZzGuzfLrh@ z5fT71F>(mO2|)p_X0J6H(Gjr;TsT{E7N7y&(*L?5YyQd|5Zvx|_vTuFZH2-n*jfl7 z#?Wj)h-fBjg7>n(Uorcx+_xgv7UotnQ?nnfG9Nc5BJjqEee0 z!%KNDn~ebts1ad7?7uSqu%Z4odvMDS+il}3$i=I=h)$qjZi0?PWMGC4j7WrmC^9p{ z_?2l{=My*a(95e4R9zeZkR41N2++Wg0GyB zrr@o+>k<#;%LdlOgq73osC^YK+UJ>`H`BW!S5s>j1Pm=zz|3lmwnR<7MXn|j_!}vE z3#_z>HrQ$p|0964MF7~~SHF9>6`JM(ZQKmUiwwKq7u4D+de7VE%C_rxapRx_fWHH| z-cHJ0cek^!<)}p{yU%EC#&71?@UewCMC35bYi6*7y0iqzoA9~y)o$PB{!O3P8|fSJ z3vTq&h34Gd+ers@n-sbZXElMXvUi=v)uD5D@!!?n0_ENJZ_)g#$kn>N6~7xztoDL+ z^5-o2+b^%kwYV?l%3|*3PDCnh3O1Le0R}J=F>MCg$bksV#OwUn0%$I3s-|MCtcbch zWSi6l>OWRB5Lpm-uxX+iZ=tED*+P% zgP|*!`x0Zh(!)}D8D>M-mOgz^Z?~p~E&boY)^_#HY!=`VR|&2iAks*s>9pHL$EEn) zTAgB{*sJx{{AMRx5yUqByLIVdsfA0oVGvtJ_#4}n)3-#}&P+kPM({QJQYmHW_BAmU z01T9t;Ec6=5Nrs3g{_$%mnf^h3Awi2U26<%w|dwh7j6Vscb}bfa19-7)mxa^K^IfY zCOH#tT?x0ivu@Gr$iGzz&+qavea^NuySZVV-?qS8lh&4Gw^bRoYv#AwJ}jHCZ`>5V zNuB-Mn}1`6_zPO++Tkszm0Cgwk>+|tW@c*UMjM_~wN|hV5d&Q7YOEh%DW|tenrmW8 zaaVT(BcK**Ra`^`04VmHIwn+CZAU!;SI~tj03m)>@dQ6hg#toSKixma&+d zo4TSCGI&{OI^4z&E&=P#nUOdbe8r9h#nNr=27YxVAo%9rx7unh#O=g=&DXK^Vf*WK zkChB)Arc@XAt9JMxH)K>&8FL~!uezOFdGDySk_%fQX&k zRKyA*8g&=}(5$MM0sx_VE7+zACSVn36TPaMt2wYMtAdtnW=N=rYn5&DIT(hK$2wN2 zRVx56G7%}Nk%e}u0~i`Oxf!4v0BBK4DaA-cgrur1R^6b&V%VDH6K-?9uB8@eLv47%?9{2qI-*6gn`0MC9mx~tRq@qw9bZ^-JJcior15{8}h|x z0dIVBcQd#y0Ajm->%dyr51~Cx^;+>(m(5>Q*3BUV9>>vVG4fg{N<;&NbwzO_2DfY+ z0BAlAT&f-68~v^|T1*uii|O1}YS2yP()AQ6S_19bF0s}-+f-OYyp_*!YvwcD(&f5r z+qbX1wN19;%0kVw*`$vZURTj<-|tj)ybO7qT68iZq5~tTsnppifvxUU2-7q!6R1td ziZR9*YpwhJZknd^`So}_c3qmL$xIap1DN7aCbryl(MbjC<#HM7MZAQNhRehpj=Q7i zB&O3eHPdaLE<~`~bpZ71uOIjOJwiOad}fNf-QEl?mvid6bsZU@lu<`@_rCA5mVV!l z<5)#vjOX#3Qre*tA%Lq^H;ge7kch;T4*S0AIx~>k3~FT@r-!G0yD>C9z}-`ds#}hA(GeXb?r~HPNUEqi#g|7v$sqG^u^H5 zD*>$ZcIPozdB-7AJ%v=0>+ns0ti*&HC&^C|P5izCI zwl&xwD!Ua@yOSz;%jmS0z;d@wUf;5ObN76>-3+!}T|cZiwe9x$*Lev< z!K_WBi`wGk)lv~L5(Al4F;i>Tl-jtF-4Ps2Q4tWl76(TaG9yy}U^gaqM&?kpybdP- zdwM+deTp4)My4|5*I1L3%H2+_h)PEh2m_;;V9k!|tnS{kM-cKB_h6(I4Ld}L;IkcH zX)a#~g!DR|a?Z|HtRP~SQh)6B`#qH@QbfcoMT&XNEtp7L&zCw}*xi|82tlN(=5buM z7T)ggE`lOU&NDSnKjD&Rx?P2J+E=<-*JVp)>n(@~e%<+C0S=<8xKxU&YEy^XUT$Wz zgx$K`6Up}BD`DJ`L~EE7 zvcARLTam(5#aRVL*pzE;?Y<8Ee|N*%w$5XN@>_Ol_DI8xWpsOEYq%uPY`3*_uYjvU zcMUn*j%_1?y6RO9e>)3z9lk@Z_SrXk+B(bI;jg4Kv;7i-wze%$t`C7*%(_D^c!OwL zK7m^lTh*3r-`w4#V5vmT5`uYmRyEv6h$l`SYR-dQcbcN)>@{^GoYe%?g}b(D$Eff zxQ8m$ea=cG6Pr^Nh+`xo1dM3F&@mXOidsRh6@)Ec5PKqmI$9QI;?xCBeV2-A&e_bm zt|v}_3asw#pr!`0nzs=V+X>sM>Y!*Fk$EOG+s;4S;?#C!w>uN8nMPL8kd9jO5*$ZW=JLfN zxTZpTEvvq=hS6{P*%~bNyHd(Lkgp<_zY!?@mJz-Ma$n*|w|hbxi8o`)EleP`((vBw zSIzUH)|AJ&+Ok8KrAJUeoAP<>^VWE;HkW(%wMxA!I3cCG{;bHdF8@ja*aDz7L;8exBJ`HKC%)75!tQN-8UwWEii3Ayk(fT>$P65CLsjkCCSts9Vnob>T1Re z0E}cvrdCCRR8vA`+L#L*fC0#aoApkL)T$yt;Mpl~57sjR(vAtsIJ|s5^qo(IVi$sG z7B{32BXiE#r5Zua=Q^B-K(Vv}K~9X^$JpU)KVT*vFC!6gn@=ZFv)U?2P;l$WoV}Wo zX3L|p!*nV-y0^d|NZcigOxh)es;*kZL`Til%tY$(c+{m#lbJOmEaGBP0o|zQVrG%U zDlcr;WxMHamtft)Hh+P&ChAHd?bQvr)(0+sS_QX7&HUOz!?)RX%bTIODQ+ubf19mu z+IGCK;W*zUoT(L516_`=K@$n?-uHcD0ksT3Ex3W1i4!i>`fSO)1AsM#V}+D;EbYTL zPtf?kuwje(dGqe=H)~5g7oS#bg^_0z$T`DeM_A)N%*<`hOMj4MtQm zZ|<-q8escgB3kR&<3_%NjYRg)_f_Wm8z;5`yE9DyRE3T7E|}FU)!(2Hyh^k>z8U`s99Ea zdCVON-I3z<`_)<>`I@n4OS&8UMc#i)dwzRbPir8l$+P04yrr zYB^qpmjStv!knQB0Wee<9jbX*ZQFA{zgF1C7wh{v59^4wxYI7!4q+vT^~-8%ZZ=19 zH+|U+!P`Mh3*F60Tlf|1gIg2+x1j3oSG#T3`B$9}pIP|?z|Ge4gx2!h2oRVZL{;l; z78C$L09SRDCVO07UqmFbKO7(Ia+cxjuct}Jdh%Mv@%)@~Ccv&L?`l2`s@PYL$4BX6 znT9D}PN&bW=jWJkWOzA!A)38{gF|nW>J5gCI_IHI79yT2C z2KUVuv*zwFb92hzModG*QbxQzad);9ZbIPhy0jf8zG++J^4o|WTm-}I&UQD+-$E|f z(E57U-RJFP>yeyywCGY*#iceMDMz$79Dum43 z-coD5oL^OS*LT%k&4g;^s;@84rwZ`@M)vrKoBiFRFESd~l@K*VifrA?^bMcMT3;<;@%!tEk9wP@K+R^*~>y!dk0Rl8hw`qkpmHlUl8y|q_t335fQEuu9i zT|inbM_cm#7VmGzxRTa-@d{h3iHeghCRboqKr@9#V~89==(?_%gqUJG6lx6-ZFAC| z)*@QB^L!_Hh4pV+X8YeOZm30UW zdADS}8g#E?d^5>!k?X6ga7VSC;bNUY=imBk+gE!*rQ&GMl81 zstQ5Sr&7x9d_E21X&7JK>u$$rE~TE&=lXq-K98441i-Ll`~2x+fYEXuV+4TX@rWWR zB~lHDxs-CbG^4wSyng;XO%qt>geoqkMI^=;QVji0OTLU#br91dKBj($f`|}k@0%h5 zX3}O0&~nb>IEs`ILf`kh-7dyxn>oK*)3-~Bi0uTnjVL@L*W0KgTr6K(YlAOEX>KqJ zd`pn)PrT7RZ%C`5RS1Fn{?y*T-0ooO&K$gQNAn4a2>X@zTi&jjl~Sf@A|m%bgb)G} zVF=vBk(EIjSFS48`f$HO|5dgPYWE6AyMgJxDsKD0e$74Ja`k*NLd&O^2iAnEjGL0a zGn4xoYX@e8kf>GVawroZ0*Aia?;uVxoppMZQVHw%^z-G@=OBe@(P6+^V+jAB`0(_2 zeE04#635HS@ciX`K5NbL{r88%A*ODe@|uO@Ad!yBA`_yjUlL-X)7a5?9m-v^$f&JPFf_p zNemWTP?S6HQmxeB$QS~k6xC9{d(S`r^4nC)FXzke{^5_m`@=u{_UZX}JRZaD<i7hu%5X&V{4#y|wOS2_T@7@WdORN8u@*;)L;+715Hk+H!haPJHv>nh zb=Re{h0hF4tKr=M#skz(m5RmICtQ1kio_uqg2`uaMa#t8BC_y| z<#qsv8oYmhq!jZwejY!)oL`UuO>i6zq+xnNf~Tj4N}n#5L5lWi*L6EIFeOC(?blCJ ztslPoj$?9nEcx>K^7Q_lReN}NaHN;l&xQH%-3JcoJQVdr6aq1GBw(vnYn4*q=&EW} z+?|OSTRx;iEhrA+7$e5e3cN)Ev4;xTl^lc!0L8)7GGi+n-a;aXAQ1&Zs_E6;0ccJG z0&dyA);t4_NEA3hh!~=COjZo3ngf6V8iM#N&f}ZEY0+qm{@c%g22qY- z8qeq7K2Faspwh>1!sq8c{nB;lrZtOHLTBdx^1uD>hlgW|eO47yPF>&ak?EiQ=YI+z zNGUmIGbJN5FO#s#aB4j&3$@;RcgPF_XF? zm%D2tCIf(g~=7=1SpgNj@`RmImF#4#!{rrnLe)#a+u1n~krIu+7>VX3zx!L*krB35m zUPB1eG(4YP)Jo1{N=c-A{`^cB+%d&n2>t8nTtwXv>;;ipdQHvp`uaLe1qcnHlrohZ zri6&Ol=CoM#ysSqs+`Z~QtMc16?yO{GaH71un>T&DXEkyB4x^P7h&<(npw^{=Mexd z;}E9F%xbM}=5Et8jpO+6-S;h&5n5oqJ2Mxlr^{)#>xyL2y5IN2VCXS2r9Ka%77a!y zAi1hw2p(CK_Twz(Hni;9HLv##;+(ioMmw)?nLoOn+DyzD4 zF$powbvOu9w>zYVc9=9WAxGGH3?1zzqQcuO>6_w9Td0|3F4|-jY_Z1PM9|H5;(X91 zw$nJ5X0p^hx!T=*Yk}U*7i_R;16vKb?rRlw-OdC3@WT(6FCRa>Jda}zAsn9`&tG1C z{q#9KyqoG&&w%a#9vEX66EK-#RU`|5h%sOcNP*JNKmBV60nw$@(+SQmzx9339NiQt zzk58Y*T>z{N%FC?s(G4Ez;y&i3;+@FK3t zM4a$441LaFOg$FqQu_0M`^3ydK~;0kc`Vg5#!f_DE|>E-sAFU@y4N*nI+nkmtHeYqM)p)-JudA)6{r=WBu@#QsYOA`I3B_CQ z#vK9@-RC638*||DryGOZ3SLBn7=HZm#}}q@zSPUeG4w=TNUB;(X_<8l4qz3aoD~3m z`Q^8m62+uI2E-wy-9DuRkG0=*ed^FvL_Q-OkH^QuF^ht89WSq!Y5Lvi<@NIYcfT|6 z_}$JyRh7A!I;)zR(x=lWM4Y{vP*qFSYA&&rn_ZFyW=|2Iy8;ss0ik1a&(AIuYb!x_NX6ttGhxeH8&GUYs$I!<8N$JD1jn1fWT z_3^Nu#_>P@$N%_;Km75Z{^_5(5bHR8`spXH#dBs=29#P}^SRba1hrN*0k9we5r6sR zUt{dtAjDnY@24r3Ec^XY_G#@1|?R+=NrHY+j$ElbHPO6$-UxslInpE|4Iys@L zSJP68x1%(&G#Yh>VHj#IA%vouQo0O_CHPr~;WCcncOO28NX{du^?l!Ui6a4du63#t z03`-9%hNzaeczW-&|OQ-c{+XGi%dD^L)Yt6zkGQC0DYGO#na0xIr+OfemRZfsAD$5 zn7VGidwD&D5PtZlf9DAO{`mAhMULP=rip3DrDinI7E0ERg0F%Qtcuu@UloV}NZkVw zAUcxY%0Y9!o;iR(<8S1aDGvbcGXl149RPSXMfg&vY!Tj|#Y+O9fs3n)8Ukl4h)4_! z2ry$Rf-#_&suVNhc1kN)B4DIJ^nAXQVU#SItJdm8{`}wm-0Inhh&#rVj>qHSaCrCb z-Q)XrA3l8O_d7KZHwsK4iK$Z{BA;cH#kh<&GFLNlHRQT8F)bLfAlF-I>&_Fo{p%Lm zwxIy)(^mm%)ASCx+7@w#Tr2hwk%EG*;vTp=7(PFr_PgDm{^_4P3ZH)dX&eUU@b3E` zr{_;vrAh`hM0Yc(rB-=59&`>}YQ9HxQHb=@&;NSZ?H>*YMlcaD&6m@(+yDOe|6pcy zc-ArJXATC@4=MIv&Yuw(DaFu-q#SmEqY+%j5eP9LG61@|8hA7C&5}9*&GI@iBEWoN zkRwVfNWW|qgziqtMBspms%~2IL_DYLP;&-)MVV0eF-CF8!x;#UT~vedavoF&(8Z?9 zFuV?yH4vwi`tBkIR?GkG-~GFve*5^J{?mW_`M>_Fj^pot|NGO&kIx@}({Z$t6H!90 zWy<6x3Sc6lRtpm|Bh%yKI|Q1hq9Qd{*G7FXj#)%%(KVHQ4w!VbDNnU1V2o~NpUQO_?e=59dpna?aEF%Zs^)l;__*>rgM}Gh;X_lhf~tG;~t3;^~h$KAe*N!=xv94B-ybr1taBxFP|u__Yx(P|aZlu{Q{ zN=d|W&efDK7^ndlw9pO&fXFQro6M+{A5%5m){OwhrSuizHEW-JQD0!?#HBt8fG!L; zp)}+QNQ90E;7V>&nLrI%5fL<1YspWfdU-vczC52!r#j@)QYrHsL{i0yR3O@Q`@_4( zr>Cdg;rRIYczk?HDT!*9YAwr}8~LNdjcBq(f-P#?{d(0I7tt~_g?s_C^)W~U?qJqX zAC`Y>8QgmrYU^jc=>cBFu|-YX)O*;#*8D)6ZPHCRA#1TupkhMEF@5;qhk*3*>Er3s zmr3%6_YbeHuleO7H4{--0!DJFSya@kV@vG_x%5-H)WOxXRsxSmDe$B=oz7SUfJDpy z{tfE9;S!H9AoOcbnJHz-R`mL4+c z4M0;+kX#87J(Hm!E#6TJ?YVUtT|bVpS%r?pAVk$kPxW zA4(}L_XPlm;Tl0=csiY@sifEekSUb0I=NIAS;ETEO+nnC1q5(V1eGdo!&!-_1Fa&p z)(&>e3{soRoQPVX=y4ofK}5_AYBjKsVkw20Jy25I7w`L~7vtP10b9ATT>NSmW*>GoJ! zye5}JGhq2z-`3riN_E@F$~mq?Z!m5ncHLm@`azq044s)B1EQ;eE53X8VK~3!DewFJ zKm6%JC=@l1Ep0*c}c}dlWl8 zzdWRezVCYxhlVr~wO zV6BXc&jvvRjsSqfgaa@AU3?EYLW{RpgG5oYZ-Gf(GV!iXRVMMkh@jWERKoD2@u_D5kQEsGiz#*DL4@s zxB$k!GjI{fwU$!4u47VY3Nk`9&9yj!x=khbX=mWWG`V=o=o8aCPRyKhj$LA?Px!-k6!Acgl=?t%*pZmT)gm(r~N)>b!GjOATt|0_; ziU>AMA#mzahymSlo@y0e>YFh$BG0xSBmfdE0ARCqwTW6yyrOIQu$hC@%rP{90L*-e zR5bt+Hvr>V5WuB2Da6H#@8;@Crf#Zk0i0Z{c$QkG;dDAZfBAH|T+aXcaU93CorDl5 zFuNPzaJf9BZg)H+4iLH9?>s~S-yaSQ5UMFs;8v~K3DH_odlM}QL!cPwS_~aGHmEhU zXpMzlb7@y|URcdZcAJWICs=I^Tldze)vI_XQTOLBP3&ny;tG0ul&AB)8?bTrtu1uc1Mk5CQ`gS8=UiVzo%gxeWPo9?oYeg^{|L z5(h-g0|9vIyN8E^b>aMcc|Luars?(hQ|2_Lmq^hSBumYrA`zvJ>8H>C_Uq5T93P&( zfA@s%KK#qS{L9Cme?kB}?8W_jde)NnUEk7NN-3Ai#obd%oufMw895kGJMoJE14Vad zq}R*PbzO{eHf)wkL?K0FMuw20sEX90GM0>ixny9PO9JG2dOht9dpBslCnBm^Ts@>j zM3##>8ju($I0LF#7ZMQ_keTb1mtGwz!sBTOR@-lOTj;zY-`SpkIzppQ+5JK0b zS~4Pbq07VQk&euXX{@D4VIo5}(_BYU&#oUoeaUK%@7|k2x8J3)6F12v5bt(fO39gq zQq`?RAUH`;QRWz9B$krF*)8+Maxm-%l zSPGOvVu+ZCnF0j|$KB)cyB~ji_rs6WMGUb!?E8m9O39m}0_IYd3SPxZ9`^e~Gg>?G zj1EiNB1Th~nq+!=)oOoV>|CYJC%*;`NPz9&VQ>HhbYfhgs_8u{*EkE>oGP}B*=--T zP`GNf%QdsX)qM_+6cKl0<`$%bOlDF_8MT^vtKp6zqCgcViu=cR#BrLY?!!B*Bj>~Z z(AQi(fBAAgy}En9-=EGej+AuH$9GER^L%KXOcIf7tE!NG#&0z$rXWmnNUa z*fBG@NVewcYFi~C#y|kzYDF_y?)x59*^QW|d>Mz=h;TW-PQx&qPt$oYkrZOz_kB#e z@NO7};q;{szx(*xOYGvy=g+6pGpZJuO3n-jhAwX6W+pcN^7GGK*VU@eU%teU5c%E1 zpwlo640t{d?ta)EnR%L~X_^|T zold9Y!$XYm^UHHcQHRU!a5xW_s`Bvgun&FL#}D7XSJl^_UwiHyVHk$TcOQ;^9H#NR z58szkhG9q{rWj)v-Cac#+>wgt;o+F4TBvYJ=hvaCI06EHdj4`e9v%-AU3VbV{~JGh z#$Cud;|>+sBhHQ^*%=j)t(>y=Jd!&jyEs|Nx|30ybr~T?cIG#G?@brN?|%2^{d4!Z z_r0I>8qa+N!NQSKg_f(z_$K#tN*=T&%fiJusqjqM#%Dky>?%r$Jn0E*v<^{`&YjS4u;`D7Tqcb|pBK#SkUFT%LrY#M zK%bSLL9rWM^-DOs&7NKY756kFjMIIdxZnnH5fp;YVT^)_N} zBQq!V^mwIr#b<*c$y@?WRq~6wU1mk^9jF=`LH?QVTAYjLJ~#K1iQpdAJZT{Vu7e; zKCmFm`lU>e4X(gJ-!^icC>|4oEcvWOPp?O5I7twx=YmoovDfd-6(5}5T36t?CBpLX zVfiei?qB}Av!%zJ+LgKsOu;6#zv?0Yt zPGvI=eALHOS9`OF36DU`X}#?2I$dr&@fJ5#a!m-taDJ(EWoGRG?rYlUbz!S)+3U>> zpF~G-&#nR4@i)S+{KOQf@|MWAefm_qdLWq(%=l7|)@5UPQ+7WE+ zB?F<-Tn@U(e%@jdURtoL`+wU88(gC-)knVM`|ubyi$FnU(EtywCkgewz-!;Kqpi!6 z2vY)tEJ`|GkfmHdv~e>ca$10tgWU_g)rS{!tUNjVx4jF9X=8>XsH0hA0w7(ZB;XVH zlqlF-V*2D~Sz)G|xXUJ_tGi3xEh#>I)zR_M57RNZ^&oC^nL*E3#3?x&vG%t z6PihWMuVBXWLAbK3(3L3m-xB8z8pgd>YIXsSZ)yJcBdv8u8?>8=6`wjZRaXte$~MA z;hp@w-6Yrf^%?=5@40Q?OH@C?K!Q>DqfiF)i%2~)#K4G)5^GiIf)93N*^Ik+)F(I? z)5Vm1EwguH#YJ;+sb%FY1q4++CvY~SeyoBHBR437W=hlbl-E4K!)U7cqlJcf`WoDS!^5nWcw*SzoCtEm5GNwuav zt5Uc?wL#7k4sTt~E@kOc#sR>Ul~JTnMpN(z>c9gXQ>_zt zwC~vTKp@akt|kB_#56G{CEw1Z1e$%c+#wABY=;0Q_eeTopg4&ih;iRZw-Wh>Zgicp zAV+@o^bBSq{loFFZuXf_CilQo6`v#spJIIl} z+o68yVP&iT70&XrGq*r*hYNW?Rm*G$dR0^Rk`4ajtGwHO~O! zd^3QlxP!$4JOGgL8lzAuEu@?}G5z`X@@fe^GL~N02@`v8dV0FjTRO5Kf9=zRV<9Wb zmcLXgUt?U9*f!$AHOL7s(!Q#l+F_xC?>5bxvkD}cm~gS7E*EA3r%CA+i43^20v%v#y0iM;azyCgk6PHhq zxTB2#C@OHw-{p5qD7aJj>WISonypoy7wc0rlF{p$w&$8Y)RCFq_?5&{c3xU2a~8{m zP}qF!oJQ>AhmfKd*6uerS-Gq`jYMTBph@%$Q;qyFiOjo6f;h`-hEXYmqMM}-!j$v} zGJOB9&$4P_CcWcrac1tCJSZ&g0oDba(qSWVlbnX2H?)-0Rn=5gw*%-Fxyl-tuCMaVuD`phknusxnEJ=xSs{c$a%Nr>`#ggtcf?po#z2~-eu%JP9 zMkxV?EDV4?F{6Q_s9FQ`q(5H)RTKfn)g+4mkgO5h06|&+U^>v5oxpR3;>I-L~uJy=hy6fWSpqhTG5LUi>2&YKdSjDKcPLv@4rLHU$7Snc+h4MyvImu~GxS0N;w~x;Y$g4F>NuTbL3MPsiWs>+ zT7BYu+CN6>XYL0*|L4Q0NoD%}2Ak3mEkIg{`=+Z_V8yJloHsD_OGduN1-S~D$Q(I3 z^2N|fdzLBYw2YcPQ;$qFBZW(uY10!bSy|6f0+Hl7V6JEsYh*O7n%w}jdTIIr%OkE}t3ujDe%)R=0b`$o*XUHZH1 z?rNT{FmrhEB>qfnHZNIzD_Q+MRsEn}I@dR)`(e9xA|wR5?^W;}RHM;KkGXaHr9rtS zn)hzi8G-{)PSb%z$!(z+sp*vkXeOoAM=_|bw%5LQw$DIJ)QIGcqgIlP|1gJquY3|? z3BA4`VA0hzGh5|YeB8K~LJ=rsp|*CAnO|x>;{}#lS^=8P`P)wTcbr=I3^YOu-+syj z<d;Nlg2bN-1OV>C}<>V^FppT{?oLrx{ zpM0W}>4z}|dC@pK`)K+D{Ft|6DK(L+_AHJbI0XJtu70?Bb4iAlptcwSp#|{^*XW_w zw6^SnW|uLc*GMA78IF0>A^+uIZ<gn; z8R4uoaY`T>A1EXJboAo})WhW&@DaVaO3WDd5O4$AA!}c5Vy}H+>faOp7j9EH^@K)U>ivKJ5);`w zc83!5Dl?3tMSY|NwYl8m4BNi9q0&_3lGgPQrZ{`wLlx$wWu$HoH~|lE$6MD}24>(S zJ>an%)V&r|-ulgVL6)7hP31y~E@IZ`7xjCN%NgyJCcBa@erw$Yt8qEp(7zU6wder- z{!jGdY&i*?(=1f?Eyo@WPD~G3D0EUmo7mxZeck>m+zg*%qoLMfdGa6MOe!Xc_=%A4 zVgP?5P}o?;FZZ#i*#rIe${3pTp1z^LPUUyKPiEp@OEabkA*N5pGWr-AOHIMzPtdlG zyhLwsmudS#khANZbo8iA2pj?JlYeL9^>az7%jxmwtOvJ0bnPX(sW6o_W9i3VKHExJ{m}oT3T8AFm}rbM+VvMGH}s{E*=*Pf;G_S5!Maz#*33%XvK#WVxGBEr4`lZCZXEv$ zp7?+L5JLe&re{r98&r-R_XekJXu%PVH>;|oI!+C#N8}r`(F%khCKvxj8z#}Trw>7v z@*E(V3F~XqZN8E+^DR@nlS;0F3=cpJxa7v`R+^4UmiKQq~KJ%xrBL8j$}R zy|P$q3D(Ah@Wk|T$~WPe@ZKQUTF#v{T?HcQ;^Q3l(b#;&Hkhjk!o>!sHpPy-m^CUc z%IH$P*0B$CqNlJ9Hj}|x8Yp;EelB~)M|fYYyJ}A>IGdDxBAM-U=XBZDWMr}%!S#%< zyU!pMo5hO`<}j}}`&uV#NBZc>BegqgYiOiN^92Z_bqZkYW`VBaO*I1y2I4c61nFsA zT>xci;TC}tVyx#>A2ZXAXnxr9+@D!7(?9G1dIuHz+EImOo~iN|-hUU?J_OZ zLJ}{Yb))-1lbC!x7<9>f*NTcQ-^HcZQ5wZWK!3&TNj$u##*cF3qjGJ)yN;gf{04d*!{qdjt z6*L{=Yn7?x4ROKE+T}ZndWv5sDeWIKUD!Sirz8(N;II0Bn`pro(Pb|uaUr|wrlntQ zJjAQ}YzBt5^DH{)XE4Qui-yS1r6Q-zgDQlS$FR!}ulvtv&-v&6XqY=O7ik=|mg{Oz z*&dN>3>A28ejKDuP}~}8nL45MC}8PMO^t&Cx=sI+FX@&6t*RP}aQ29hifcnyAu{(h z{q9@R?2|c(BS9c6PT?j$v3F~0E9Y*Rq$4iAtxu+lU2ENp((BV!8b*c3S+wnYs?N>! zc{r{5DmeP)3G(LnFUMTR|cUDbxr^$H7`Z?{Z1^rvYzhvL^xxbTH z6U|IOgC#BDn%9Pe=yd6PAD^DgU2^o5E`+J)Q;j&(=a-1KK5NfI>s4ek4Ia{!T7cfc zOqWS}t>+g4R|^XZqFsOY_w6G7Ctojr&P-&Hl;nFI@8=Esgz$|p~ zMxtcUoAKnd%Oa9h>+P{k)kXD>h4mM%qQx0!mbYG^^#9C&-+s{#+o^hLZ%endOSyLQ z$pEAZV9#=MwTo~+G9oE??gUH&7B^%Kp}NHnc8wJvifnC<$>h5A~&G2f4D z1Mc3Ix$vMXez!EynkJSJKV=ejE9t_*;F?}|p{+cClD<=o$VHiqMsMC7a}%?Ir%+&6 zQB>DI?|%C)Y9*qLl(Q*ZL7F2s*O zz3s$3_KwYDL^>%`loegQ<(LlzG~lZJZCL?pNB_s&(nJEn=XL@J+M?IZ2f?LZIFeT! zxcEEV^;ri8f3!y}L0&tQwT7K(jGAa%1_@_OBoAI4T^+aPMc#2KvJE2x*n?{$E~;1O zSNnsdy>eeTaxZ6E&xuzT8dsj=7yw|Ds(P-d$}iA4e|p*&RtNl~-S47RheBybZUbcg z%t$kvOKN_OcEZk2`V%CzX)o!L|=wU_6ZSW!7lKoKQx(yDHg?3JHAL-m%A_t+LEk@2_{pPp8A zA;;w*B72s*WK~tB1DBaTR5o!bD?!MIdGV3^%KoM5GkE9l*LP4dK$17xe5V+kOq{#e zWxd)z*HGUUXp?#4(K@$Q$^rmp-A2|3;Tf%B7DCeKe)nb585%b#pz@ zTsspo!g1uMoG%6A)tug8VNE8??1nWNnp1S>efQy$GPft!9~=AZLl;?R{N(H7YlH3% z{;9?X)3l_iLE8X~{7ZA6SXycqw8?a(aG?jt-}R!pri!W-W*u??4;w8B+755&q_eC{Q9Ud%A%&-f2wgY zag~m7YIn{(Qj-;s4m8R~Hd*zli2< zs`>xdD&mxrmO75g$@#TW1g28>TPexASILUENX~K9~uTq{J>YzOxSEnSscH^a+t1B65<#jNv$1S3}M^oH?icquI zl1)z$O6`4DmvdL=Eqim#=lg$mu?cEQ0Ux&l3Dagt@4G+0QEdIWh3sA6uR(s5Jqf8( zwJTSzt+S|Vu`UNzo_yuJwtd#ZA+u_I)?YRqy`D{9GqnT6HcNF??lp4Mx@0B)nbG&2 zihANwU?a0D#5UzS+JGpni`5ZTk#3>c^(}zGtRz6i6n1 zYDCthi)UfJPp8hXw*iF;WSsQB1pn#E;e*5u+?%nvp;%eZ=AJUF>is(~^wvwbE>fa2 z5yJqQ#_JDQzRYGU$t@_02WaO9KtNpE=T6eJf^zqZGDkKxHsZMXj#XK3CtlM(b^|XY zAst5>8NRy#=Rhq{{3DlDjmr^@@XLR};%i*N;v<=Bb707yaM6_AdP{q5 zk36j)3PJyfLXSo|99p-0C1iXI)eu+@4@ul3ybC8Z#*9{uNfgs7KWcN!HGrDUQ=!%3 zd5F{gC=C+1zN%By6wQ{ERm=Ac!C{I8yo1T=Y-;K2QA;!u)l3nnTS4zPOMT^FnA0cv z6aM(oMIPMF~v>rpz-t0~Q-P`m`W=3n5%%<>GUewXb zNwMLl7k5QRcejDtA@y{ zi$Ed@rEzhjAw$eC%M4iFPLk;YhVDDWGPiNmv~=y2Q!nwRk8`y1TrJ+HLtEGRjr@5O z;mP>OK-fAAIpK8&d8&u1p6n{aEKn2-|7(f^J=Dj(^I?Sy`8=!jgc2_)eJZ?`6gebr z@bm$e*BIZ0+1V za(c4D%6i|jZ%ICfC}@HBPwmfMvdQN2lsMIbVz;cK7X<~@=a=-%uw(&^2xf|3N#`w8 z)aLnHrMx&EKoRhEk=c*7@1l<-isO|5iQ4{xD0|IfJuH4r&~<5r5~7R*l~0Leil3aQ zZvUA%|9g+(-q#Hq_k8QORtK{2jnqlY7Bw&YcT?GIZ3=3UosZl(VSw5~z8$aQRjaK( zwu(l@#>#Jhe0W+0b%EG#Wa#aPQwM9DmGPJ^iN~(NU#~K$KtYlrmOovPW78`@!?nftB z*nEW1ar!DiD|N0toun`LZrPTd{4HCki}Nn=2Sh0|3_#n;UoHQk#nYt#*j|#>Id=7A z)1^go?8B3pN9<3hWgQ26nbmxkR{&{d`cM2OtX8bz4Zn?@*WJcnxNCb%2Wd~VRyGj0 z)2bH7_`2}_y8dxR*lWAhF4=hf|j5xLH@0_GQ2Ex#Jw zGvB^WTW3V^fR0U?got?CNF`YP3vS>?rVkle5+==2iACsV#YrOzL;<)rS1MTZEkK2Z zQgrbi?m=vrsHSsacsy$6aw#*JxD-)8#gY5xoIe%iMy@))k|R{6<910-sc7JgWHGg| z#>E*g(h|EAB$^flWqWJ>F>7Fn9Y^Luyffnfbp1O#^wGNyrDPfV$imjd0^3U>%D7iR zaY%ZJ7wTU6A8o+wV6I&HFQ1Yq10Y5nop6eCVbn0<64P`=aZg!K2=^tLa+b%>8$mhd-2hu5I=_o%V3V%O( z3#+REb$NdllSr*I=medQ7IQ60hF@KfTF(#u&j)|VY?T4kRC&+FkVRb`uW6Heucper zH!(q><$>e)f+oLg!J4YfZ0%_~qb309B@xZ$ zr1C|qU;N&e8Jp%hVuf_=ySU5uCmxBALV}=R<7M(=_`U)z< zI#)4!v`y|nhxe{dHB_6=BO-%Bu67B7QiGYjE@0mZd=eYQbu5}&5C(@cFMRqY)?T^Q z6!XMlri46$=~$TgghM=i;2N0RM?lfBjxwjXzCW_if2at*MfZwstsf)lK_9yx7D7%} zJ&w9jP6F7)4ZlYfdSrC2$+$d2Hg0nLZmT^tazE}}IyG}MdNXaCIib$yg{ z-n$YVOPDdX9IWD0Bp1xMJYO0)i7`uVugoUYg+oI|i-w;Yk^Sj6Tg>LxY3)=xET<)r zEX>?vQF{BvpEn z{G<}_Opv?hFdFLCinU%%gVHa&K2wAlZLcf{TmJdPabb^`kT zp(6jTO$HLQul6rz0$Z;x4-SU=Vi%%iXzmO8P%*-@rfqeu@kOov3TvZYXQ7{F*=}I8 zIih=-jC|^YWS04??|9Vy{bir0V0B)J--s6jkBC;FDPG4qTWF!`At;8TOsAs;L#DjX z>;koJ7iXx_>7w-t%9}SN7Q?7YcU`nvvnQ+e>mwqcQ@Xc$$h9fzO987L?yPKANK1mS zQ}UL6(u$O3a_}yXiDD1st3K@cE^%%ri8Xm}kgUh)ZN1!TZTXgWKKdhIKdCH1(3epr zW}X)EhMiDWpl&%*NG1dy5CZX!q#1MQZOTQ@zUj70&+VLo)jgAd7$hpr>J-o*$P_@%7 z?da(5WNW&3=|*QC2#bsrH!x0 zD4FT0Xlm_G27&}jI=|=T@k@@pD1^N~kOe{sCgzwDClSoJ}#QfEjDp9U78#O{je#Pz%ita~t-~f!^x9q3SnE_gu6vJT*|)Ul$EoLJAkhcj1xx>Yz^8c!5J~w z7kM_4kKEX=j=bt1p#twSWf%c4+VkubJ%DR*LQYhf#kaYj?VtIfvpJqS^l~pLG_faQ z3G>?*RB%h%^iNoqJ&$GV*no@cj(v8;8I3_qGcG=ZyOI9+X~i4!&rzS?H3uy>_+mdo zGI1@T(;d2_zqNYoXsf2|JUyn{=NKJ4Q{9;mPuBnxya<*3&XhJjKJCp^ummPfpY|~T z&Wu6@7PBwLu+LQAs30e;e3QXcsw~b5vpcxkTK3a?A?yYj_BhK<8(^8$I?+01DZ<2A z(!6U?E%4LY-N#Y4OGf=|KO9R9=EE(&!_wHGsW9aftyh-ChziEv{(LkoHuAyQhbg|a z#0!V}>`G98jU;Ch1%O(0o~#nc@J%M0$;VFxsf2Z0_^7L?sZq4`W|~)!onlHxO**)= z#ERi86~ErAWLb#PsCL$tFOFnfllBvpPB+gL1`G7js2*QLEPCfis_epa%DZGrTIMvs z?iJy!oHDKv!NJ}W#L`>vu<-DH7%6;?bg{KnPWBL9j+Uw#1g{^=XiG5qNq}~DI~%4* zh&0O|oeWOsf>jogJH9V}N0-MDI8rH#jbR3Isq)MEkSuG=Az9kcsqV-mFu}l`B2)bL zFl4hy^3CWmR`cPZyb+~gB~X+$Uhr9on3GOtU~8x!$qdmooxvk#dg3N_-pu0ULBW%3 zvga$)rfjx5GB9IO%0?+IE0i6#jn~*czhJ?@I~kfT zSezC|;~KN61lgj}@lZ!JfLb$%H;3j$O$UR6F%8{|?qAg6!gTZiO7>M-Ui&Z4@FVE% zS+Az-$`WCyEEQ?Xp?2qq)r;SS=vw8P;?bPmgP?!=3+I(v7XQez7o|FOM^_ez0+Yzg zqzXH6QAH*Ey>g>N5K-*05u5RK>O{gnrzbAxCth!*6ScH(_>Tw8 z-$CeD(xx9}8Ffyos1u-(EH-8C6JOQY%ySYgp4(|^TU#hp_hy#qLj3G2%W`)u>WGkn zu8nMTobNd%N=ZpM^@MNt+k=UkWDGkXcFd#R04$l=yCq!~FP3k-mLGY3B5-v~CVQyA z(adUZ(UATmYSK}?$8Rw1(R#-p@JmXn>inW*E>P#tB}-qFaB;A3zz0Y)$KngqKYVB) z7(C?o@1!}KLNoF2%=z9}@r@0#w?*#eZ8LHkICTQ~g8tUBI?ID(h?R6{mh7r~4gbmR%K2%FZ}V#7AkDm3Z-SypOyQhWx?AjICqk9MlF> z$F^T(uGsedMjC2Cbro{8L8h07H9b-A$|;exjWyFxv@H*P=$>2h8P^YH(@H^*J4!F= zf|fmvh~S72(xspBMTf@4f={sjut4NrioPRhV{2AIb6kSn0C$Ox5`R96$(SEkY-X=# zI9>6mo0xYuJwh8h78FA2 zVAW71GvEIn`ze9X$oa=M0lduRJqP;;wb-{#pc48*VWb1(PQm61eHkx*sAvvcKYvz% zr~p)K?uGHtf#T6Bo#?()7-T;`e;-m4 z;znMT=6$|6zOm+_ck~@Y(+zWMM}rINnqozZBmZ@1d{d3Q{C6 zTi|N%0Vk7ie(~4DXD4783bPWhyj*$ViQvzSY>h!Z=^6TMtgOlAJ!9~;em^lskz~uW znS1Y#R{pQUA)gx!EkQ*Vfa1=$h6};-IBlOloAw!l`ri+Yk4n|1=a9)*WVHNYam}8~ z_xZeL=fm%%(8~BH!Rik(O5DsAZ^p0Xn1@JHCpk+R{~m8XeQhDVKS0J~_=+?FM2GYK zz~#qem*X6NS(jpKpZsyi(xR6MkkI*}LxaE?6$nC;;AqDZOCyTuHVaDxzeyFNJ?wcZ zwU~2usSG)#s^pR)vR|warNyIgXAO)|2J2MD-mUXn%8w|cLlUuSuxz>_tJ(lq&97@R+h_j1@ z#O)T>&9`JVF!AZ!NR=J4_T?)zoSv5Zy2XY9E-h>v5CzC(9#MclKZ$VBvKQ*u#dW({ z$eB&Te%kTxfArRkk2JHxBhfpblAVm6 zX~*9fAz!keQkT(Fp{2*;lEpxpeBX+XE_M^vsqfpGxy%O|$A4|6-VL|2l*$AF`U>Kn z_ZhOIvu>zNoU!~K-M3y?J-ajWPNzA}rn&Jz%GJZuF65jV;ML{|_w##^UNKoR^=<48 zt(Pf4OfL$ANyS*0>qBBXJOtqqA;O;%8mN{_bIGR-GQ$;ruI3<}Ryj(wDlC5nb{%Q^kjL)t}-#;fFxFjY7+AdIRk zDOmrbQ%tPQD}DEh3z^$WC{qO?(KJw~e$QLPJ4s<-o+Fq)*-@L~n9FPcHnAbg0ueUn zDiMdGA|^zfT;)7()t5hl8!iA!IKVUXSt(E*Cj*&mnB)Czc-wp-$LE&Fze8+l{QP+9 z{%G3bxp30%1#vFIAsz}%BTxlioX^9l1?(bb*VJIw%h{?lf52>hmpD(I>KA8F) z<(ax(0VY{a{U?t61|k6frWBy3;$mhWf~bl_oZ2dir>K-0-9H9&zQbl?;qdh)zQVDb zKLoOz4o5?oVd%39$LFvWsbfIPW_P{Z>;eLp)-F_*2N-q;z)F4ICD`yei!> z`2c=gCvoqk$u0}h8ScEwz5C(k8NSExtl;WO`}gSPT}%Iscbm;BIqBx`6u@*i>)AZ> zO_RJ(J4^h7>e^aET1vo!zwP|v&wBs3&J5>}ce>3@cp#ST4Ijk(SB z;~$mDGnPi6_vjU{>^G@@)q{~mKxfmq{{^yA@J@BOi*Vnz;#;wTp#K}L%JXQ z5m>}LJwj{UbMXa>11(=g<4%Q#|HO2x*QFaQBbco4M?95dLVBT1YawKV39(Y1oLT47tdig9>3bmEV8}$B5oh=_>5tWiDKwEghs^AcvYJjyp@b6UGh&=ER2H*IMvuVEJY1-$EY^ zY_DqS;}7BkaPvfoiZE+RCu%}(vO`>*iM$u_vHQxyLAMv}zp&8Wp-eQSZ^iy+!)^Uo z=SHK!My0v_Zr%~E<(N3F?;>u-Wxq>`WA%II+C3%y9zDmuNy-{2X}dX&kphyZq>)+X zz=w-Z-{gV3yFF)bBX&`6Gbog#d2TUZ;U5$@(G-sab zDjaqW({K&nkkhu;e>_|j`m%|xWUp^lkq8ybd zKx_%2WOC1e;I#Yuv{S^}|7My{b~N_);D;6wXQv%i`5FopeSXU7CD8u!t(6sW+nOlN z%0Vk!%GY)cs!`jHp$|+kz$%AXv4%pGrstB+7(umlA4oRWt4YsTlY1S0mbzpI zN8QYA-isRO&p@{;0O7)!*OORe)PvD&TsmC*(@^6ce716QagN9>R<~D!sc|pliB{4g z#u%#?*}cO_jTy7Atn~*@S5{o|IAu+zW|RzMUTp;va>@fgm>FZ($oo)T1*yS8i^})j zul!$Qt=3Y7uk^qF8cvU7j>);EbzRCO`)BQ|P{4r8*+B;x^kR*?+V>JCGc|Iv6+|)V zTrfsQN1?qEbM*!SmKJhTcU5!L%aMZ1RW(Kzi9YBbNk~Dp@&K6Q#KMFm1!>=Q%?12N zvR#wM#TWO9GJ)cTEDIpJ4>-soi@@<;q!dbf+*!%ZNXwAxjar%WypK8%p`R`;K6))S~vyQ(gwn z-X$o=3$~(Au|Ujx<0j6{b@U3hUuTVG$WK0Jn*V!Zj2k5Yf{FjPJ)ulSUwzXX;q(KlmA zA-MLE(UmM>$m7quJ9!*ddES-S>%c=c{@&VZ*{gV4;CQcuoFHGR?Jq90L(gM!zGSNJO&7xC-;`i2#| zTnL)JQ$e5r-1_OuVsAfq6J0iQ5W!p-xGdk&V6AeC>=ZQ@ z=hAG`beEajGAr?|=B$Q~5^OEsO$)ZKm{TiS=Y z#TxHr>h8~!Aifb~b9o?eb?|=~r1|1+68(oYv-egvy;*==hMTL+#W|sSj1=UbxP23F z*E;C2tt*`R>gLxl_tWN+4v#OX_dBx=9**^{9FkurOS-~Gsv2s|&wF4;T)(~zGwi?G z&PJn`B?W`2s_Hkg>4l({#fsYhDYB8*bNsWrs=r!MPbR!-CV@WA9CmsyzOwTSo(})5 z^wVczej`ct_uXdJ)cN{~m^-f5(y#M`RYolJCOs&A+IRB!;q|?g-Kxcz9c7})<3DJC zB3hF*slbaiake|EZFo9lVz$CkPH2i;oE;vLMW1VS>Q`s($IW|FYlHbxnM`^iwn(W=fZAmTu%%BIa_S)0O`DvlSbJ>Av;H1cJ|2CAQYN{I~Vfp;7`WE=5 zjQVHwzlEMe-*~1Do>Nm(?c8z~^h=C&rA)2wpL}IaqYbQP}FI_Mr-}F}9Fi)or zp%tF<=Nk?V$FDT#CLyfB(^66^Jj5KNn82h%CGhYVWji{dVP8l6>D#fgdE)a@kK zrk*O=@jUf|G|4zfyt1?NEq@|^E*3!Ve^=eJb zsh1(^tv(TFJ#H#mX)~xR<-MQ5Jkac2bzxOVWq=)Aw$0%Mg5@632&Mh=!|Se!MHbJB zF5b`6FK^zz{CnQOvMaXhNg+I|Jr+6mXXdh1qjzWSYCvjsX+o`VrcE8nB`S!dti1k$ z;vUlBt}rijwvZQ~8E7Uk;uoZZ*?l!P6=Hdrn)bB*BlyKO<1Ce6yLc>k=re!Rt5=yP zi<^6S>M{5BY{#B2XIy{h6BdL%%J+E128Y|J?Qz1lR;#6VDlX2>?#Ovg1#tKS*PBVt$b5noAO4-3k1o0t#Nd0#alg5# z)3864y*@T2Hu@WumScVQ?OR9-&CdFm6z;AJsRY*y&4Mbs*5e@-V+wXa)}I&l&-EQA z_JsRBj~i$9w(KP*?~PuBsh1=4i=AgCD)6-C-v!xaAgCdvE90)~$L3I=75fb#Eqf|J zgII1{(E&V?U{F;#f$okj&cH5rI9w91_63Z|y2S&?pgBfP_B^YJq6GAcnuce(C~ir) zc%EMAXD7hv<@V42;IAE$V_ASX7>f>N8n$S-b0PP}++}a;WN*mjzmr3s|EzcTuTg=_ zdUDvV{mj~z*1hNGPff|p_1`<7Tyh-+6`<8}u>a*U*BqqBb~eEF(1$5|m*+jmyPunG zCJwDjU4#?ul~`>;CC^=51l&9eZ}ZnqKbN~yzb$n3hY&PTl}ON{do-LK2vKp3UzMA#~JwCCP5i}S0AL-8tIU@})2;68A|^)+qhYF74E#iC$wsan|Nv$1m8X(4Xc5=I%yRrd)}dddMzm>4$tuXmkve zIyxhTO%{eQ&ob=?${;;(Fquc9!qd`a;qcswWu6{OVgy_@iQeg$BB? zWWgxndCQdh&D~$wDz4s@%|m;QxJX*Xlt5pUxb&YSG65uvWoAd^_nx$kUK0U(t{)kXY z*T28y=FnFNDyx=tIBTI3Jv4+S2UQ2_HH}MW*w6Z~b4KhwZD}3#u-j%o{)+_M2k53^ z9&KrXHQ*V9juG`x7AU*a!tz&+=iHzov-ap)JA_D7Fz>URLTajBO2m9u^X$Ch>1L2w z2s@86=zS%s70sJ2<^JnsTcp0{s0|!eg3UJ5_|me=oB^#u&NK4!vj&camjZx}61Wzh zv4GF@M$waog_KBB1R~!2E`1z5l>((LnKXUvmipI3U3gwaob@&un_cJi#N(#8B#X9^ zdBo-E>EC~0iah4}Rm*x79P!}$^d8KvT7JH=U6F04QbzR@UaYx!0}oR?x?xGM-i@@k zW{Dz@Ptz_%{>JPW%<}#P`EJ7uq_FI|L9WR@RWpDw`hG@2WFBTM>8h#4EYm{z5G|CT zc$dE)^r!_W2Pm)415zINBhi^6C1u+9R;)?!tV!{{88*H6g=F~8Fc^%{liNnQnDtluKRj zvpbTb8={En@eHH{Q0_P50tV{rK!24+&KG{P?*w*m?nSq*J89M0ogWq4JK=P}s&qV* z@6Ryiu2jX=TnfJske9y=kL%0D8#87g5CU$F8F|p8GCIpZ=Z7p${_xx!9v{{s7b}}6 zOkrH|v%8w3{nAAl>+p8F&2lBVEMw$7t4^GVEibWf9)|d=#F}kyrvHc9NE#$rcftDf zB$pjCThOWvcfVVI(+il9A?Xfk_NvKWOOiy_YZW8vT9U%FB*6PY57q4J zwG$UN*D*UwE^}5`hqH@N80rP&QcWW(XRd!`Rl7|uAzc;prEdS+KUDe^0(lY@16B)o z;Tw%35^e~vK2TPXnPy(w0&aOEB#Lx;qnhiDDTBHmw_+jQk1ia$AJITPXN z=m?z}XN-f0JRDUZ5y;&gmO>i3ThsZ+z7Eeh{?q+}_WTV_J)5nmxj&kk2Y0!g=0yZN_K#kw*7LVOY zC^@h~c9vO@Sy9y|WHmHgCDvxmQ@oIx@ZcQ}Pc~-XUvZMtHdJ7DCiF)+AyXO@kj2LU z1tb8tl2hW0b=9Ab0GDp6Q6TC)sKLIVss#URM0_KyR@(mnH?z zB0m|Edr}m0$3z`oCKHw(?x&M>mY~@E?CXx?f$DB2`qH5=_^RwI`qC16z_T2auD?Jr zANn*nR^|Xm+Fz4g^oc{nzoVQVifcS&f3D*usyRP^&`g9#{jR%kfak4~>E>OI`JPsO zbCJ6;+!4D=upB9q6ySeG6KNdg`qk5B*~|OAoUcUx9Olf~wRy@2F7c1M-6SXi*JPj= z0DIp&nobFX070y`i4n9yuH=p2^Yl9ffZGnKsvoXh%cKZ*bN#D!^f=E6do9O1dht{( z>yeoysc-!Io;S6zGa-=2=c%F8HB7gkZE$|2+8rcf10Ny1;*!TRxLIRC&a$b&^+!;$ zbcVz9BlS!IrG+UUC-_2g3;tzfIM%qfbsQBomAiS=GXd-;#FUliQ`z3<&UMuPvWCy2 zLzj_>fmsl1@|To7@I0>>?b{s8L>0s$E1UA3OBw9=)T_o?;zt}1LM|Sv=_`zF1=T53dit*+5bd=vpA%boHC+vGUZv+K z4S1ldIU6iZR(7S#25-T$`izitZUp-=Q|^Xu>4B14yZZNSRzIDyoyOSO@*_NPLGp3W z#QcddMJ(4`PLLwFIKiJzB-rPp-Y;@cBOkU~x%Yy`}LV084(UGS|SSKg(Cl)GB z;#0vaI!j2YJ0g3tkBk*a^%cm%%5d^nwB!5PPd7|i8Pv8rK}nyaJKtDs9GEI6Cj_Lk zVxS(BdFGAkR7&g!@%iw>JR&xt^?15F@+n^uhXyF|KKbQ176iMdL74JB{J&dsb(O}P zPtE$j=7h-HJL=v(LD1tC&Ot`$dvDDET_m2x_VT^9uEI#aloM+yDm8Df%1S1AcmVZU zp*PB-&#I*)JxW^ezJ)ZqW)#?knfYg=TtKhCEH)V|_*UdeQoYys`QN|DK-GqZGK>#+ zuKDwm^ppKn|^2FE;n7%p;=P8FzVE7+KmTr5!nsAU04Xtu@d5 zn>RO`J6Q@G-Hg?NhLC=d+K$?X`hp{#HhjO&54gNOj|;kE>rsjUpnKN%7TbF?`+U+6 z+Jwx%EZxQ?^U^zdfQuPwGQWw#i;IbqGmkMp>rf7ROmI2`*C~8WI(SyJs=JdN!CTo$d?6+<{CNrYL{dn$L0dr-y>`z zv-)H;zWOs#QA6fX^}*igGU#CL7+iE{)y*N;4;|vn`NO5q<3_#rP>G09 zh&)$6(~>P)aOm#9TKd+ng_j_wXeu+;6^I6!RL?X}?Izu0f8XkP+rb^YdS`*cbC>{_ z2`g%m*7yv^G_omWVrgWjbxi$x$J@tt*|aF**tNA}iaGsDK6+DiySuxN3FM%E$2uv!T9*N8 z1$og*N0iJnr%ttok(iLbEhCB3c6A%3al70=oFH2m$?@E>=%44ZN`z|Huh) z;(6_Mb4}=VA#D=h5t;qQu&!LpGlqE_6OlDU7b71aB1Th54TWM?F253ii;g2l zyx?+Ah89M{mYVDKjP!QCGM_O`&u>Td;I?&0|6QJKU)uLv*n<`o8Fu_e-mCMmPB&YC zTS-f=p@ToUhA9l5&FlyrE@ir_{HH{yd zZ=W7S^~DWgo|lV29E+!h!cWEyni%YyyKo?o*=$eg2`Ik0GNEF9hjD_Rb z#y)ZNN8xG^h0G~t49TIbw%*b-dxxhrz5u$L`Xcl$)V9Q-)xkJhZibdwqsM#Lf5^2w zB7tIq%mzAKPZ!)@ULM#FgEIh}NJf!27QKxY+=SAQEzv`|CcH&47i|-V_P}Cha+3C+ zpK`@FIW9#Qnk*(A%Am=ayyDBfL-PA*J$k`>B(gTg=VEV3)9*f%QDR+o{d}3c(>pmJ zt2hs?&YJ|#MNTFr>2v7jkO<|SGlG5bR0G-I$`VVuWa-9^ow(a%2cmxjlO4M`t8InI z8moob+1cm`rNB?KocdX=OK475ivj8^B6p3n%cJ%0=tyCnprSH~NkE_r%0yEN$LyPX z94Fy!E}bop0*XeFm_|vTDJKpvg@Q(+vBAmG#d2pAopx?jm#E+^{$xNdi^1JhZ18Sn z&F#UV?GJJ_(9L*NqX$*!6tTH z!s$Wu>E8!NvHwnw?nzzj9kM{kJTKXVA(ma%VHbsWs!_u%=@GdhOK4v-YKV7%6Cw~O zKIr*lIsXy;Y&cdMXka|hz_eMRsI}kaYy>E5zci_1Jww0P(W1_=xZp6x*v84EqTD``c*>=(##p?-fzYL<2na{_2AG{K>q8!+}yi$6whx_X+VeR z>Ek9ZTotC15;F!J@SnXNh&|9RowLOlFYZi^XOZ9I8&Tj)o7}G*t#dxVM}F5lpueuB z3mDXe*Pw8G6q4`TY#O%IJ$NJsW1aIW-BqP5;L2Rg@Xv;yL<8eq-I-qstJ96fIkj3z z1yKcCm1085XfFzbzS>AKZ{E)B0f)CZj^Vk#;nnG$eni0xe zRYR*`ippI|DqXP=Br+L!NkUr04pf6gaqNY4E4RR>Oir;ea2`Oof{cJ1T2@x!;^|j5 zkzVPFSh)L0x6u%m0|?;LVO^{ZQlW=1Go9TN<#V2=b?g>fp%5~}>ZRij&{G_KlGD@A zDjz<3vPL)y?VDr@qSe+dl?$iulI6g2JDbleC5I5fU8&kJINw_l=I&^;v0r}6UgyuP z%ZN**kDJ~7!96%W&$Rkd7Doa?YX~`elyZGZUirBi2%xRUD3UuY25J}Wr*InzE6nDq zAM>=~Sxc;XaJiqp>_GNG&{I^J*}no19;oL1>Zo77Dc2M(9>=Z6&cRKSI;`~p`6*Io z(EbH547WjBgiiWo%NJ+rdYv%=C1@|VWWf=e1<6(U9q-0^lqFj=o)f2hN-wsUnLab>2jz{4Ww zg0etW4DJ%pXPAa!bcn;fzw*Xp8(Wp~gnptRdKNXy1BBK!@1P2jv&^!TO)xHcg0-Bo zYf#SXU%|=|&mz?YA1~FG*i@UZtXcJ`t+!bZ5NHQlrs`fF?)@9_Uumb``+LrG40zeGgLm# zA2!gxXueoPQi8N|vJx8fehZu6*=_*%>IF5B4`tb{rAN10^9ndWpk>(_+FX1|Zl~U0 zMZqv{%Gc{SF_})fZqB;J-ImD0)9CRdhqZra)_)cje9z7}173CgwIv2<1i=Sud4fQr za${}8aBiT#HH?B*tx>&P=9WH)5XbeCcN5Lhz<|kVaO&eT==fpKhJ(me8|i z`qMk=!+C;XFh_v-Zx4aw8Ob3)4LUT~+cCwIdJA@wQ#;<^H z`4e@Gl{EkYw$1)h*VCm;+F8mqa=K}q+chjn=FzaKs-mehB3z}qgi=F$3AVE9{Xy^OwAF*pIlKGX7jw4m~ zrWGrN$BHPj-K&BjtAb@g9l&o4H8(cTpZhUS`COi_Uwvp>x8^PQU1oAM9sLtpT2^Wf z#TOLU2h|t>nbeiY5B?=qK7vRZ!7odXn9WtMJF*(nOVik55h*Ej0z$p6 zB>eX2k6`Xk=k+MIU$QcN-^K?@te<+JW_-4Nuq{VEM16y zC-R)V4smY@gvW$;C;5>;Za{ra~Z3%URh*NcQZ_M87 zx=@_g3i7uwyLgp_Woq$#xDdL@LzuztG@L2@>_{oU;^{p-W;kt3<&3Qli)CV^<470i zPtgj?2A*NIES%~PNU=6@bjcm@Fjcz3* zj0ns*QaCF}I@1gy&9z;%or7*3j`T+CiaawGfqeG0EqMRB67$rhvt1_z)Njb|yQ04J zF*rC>GlC!}I6h~B@0(%U%z#0_Gwx6avxhQGfFV;kCPuS> z?bjLrT@e$GjO)@qw!tsBWi>cCrot7+Mf9ZKK?Pm>8=`OFrTXmk!6$o%Z**0&4;src zE@UViBoD@t@&%|P1MbTT)3^UbdblRECbg0^@dq~HFLpzO_2Iv^R{jKsc1vqH2MHo^ zZ0xY=5Gz4kEp7IdxMi+IBNIj%=)BrvxpJ|&xuz2;5va;zQgS*b5o_b*1O)YWMj8kQ z+At4ADTlMl-pfrzAawKFJGEcW!T(bKU@CbB$_2nuEF1qZ69ra(o=4eRSGyG@Y_}lB2+k8> zkDdp|`zEU!Qu5&+cB(t`9aq+f-;5;U2tR6TIxG!;C2IzbLfzh5Q3;cfW_Mril`DQO zZ!qTr_zRX)B(i&`PUOXrFH>@D5q3RL?@dYFnc@dIIjKN>8t@gFm0WDhGQ`5G3Os4? zA%@WIA^%x~Ze-@>EXTiNOaN4dt2W%amQxt9I~!^Bl2TLU1=$H?tw!>NLO@bEA-^(# ziT(Ulo;EivP{&g8Vh*)=M&=@d&Aup$$yCIT&b()=#-+Z#V)Nrq@iQcDDqCFZwnowf z70k?dBd=*keZsN|69r^zhEPmc>#d5(V@%(9zk}j7}!ID za&eD*o^l0D>XEC$;+WH*>|6}Comy~~n^ga?-(~NE{(kN2dv~r+6ksZU?L{doeOuo~ z{n3VBy@JQ_=`-QmpQDv}y2&!uDH(p{ZHL7yg^go2@dGZ3iQmY(ZN=_e$dsXCr%4H) zhT3oX9`@^hDv6;D(iIodPr7DeyPPpewZM%oS&1)U%-d( zRcguZQ*doMPUXK>&uChS<*(*7IN?xN2)jTAsn2-+fFVVC`PACu#QBM5xhA`gXHi7{aj z5)ccqbuvraGIFOwvHbI5A+>Vt(d=D`lT*Qjb~Z-M@a);!$;p9PT;haxl%ge}0>}Qq z6@b@VLABR_vVs8Z;myRrsq2rpEqNOY;f=oPf* zD-p)!>@c3T+J)}-zmC?|XTkrH&Fbgphv{f?QP|Nf*~o~9|BHsi)9>P(oW>cgSn>=z zHGHzK0>5cY2}jAI&St)qS-52(^+kH6DpM3vSD*kKLDw3dUxO4C^h0H5B`19+>(=uZ zOZexAu0p54{I$YEWTbh@u#MSovKqrj7lxg)!vwz@LmDWma2-BUyAdZO&YITw3G%p2 zLE99>r-%W@AU4>{Sk*Q7@(l7BXmsx|7Mn`liBtIYdE9f68U0j$KL4SdNPi)={Juc(M!y`g%@W;%`fDehk=HOXZ5r5(@`rR- zoa;;wZ`RD;DEP+=oL-2U_ElK?8GgOz>*^)X`$ZqV{p7UP9zUgpPyM?ZExVT&trr-|Ll(D77*u|(*sPCY)LWlhPw+_d+Sb@V)b>)-rdE4 zLVMvwatj5vjEqKaOf?33uFiUR7tRh3$4Vs9(n^ z%kjzH#4nBT%S4HskmGbHIV+7pv@G6srhBG^OrYIF4yJ74WPha~GLQ(M?jmZ=*@M`9 zad<`=`vm{l-2e$MTKB=(dTY+KB1>x)VRNr*6xpbgBcFQL-_?nUh)^ti-GCeQ1Es0D zp8E$eLc_{LDE#>gT;C3B+Wb04(Gwl&k8FvvI;g14B^e;@+7){+pQ?d@B(o4#z zaU_C7@C|*zF{zxCfz0=R3LuY!{^%M~_va=yyj4(qXN;v;%fz@5hDe>H`@W$wv)(uo zdUh`IK^n!s;05f>Z_+g*El!a%%dtON;-vI-qEzGt2U9R)6t}mPZ3jt#v6A)*9 z{;-n!^%hxwIVG&a9%mDv_r~YP1zdlmEqtA%^*EEC-lklxdCdUGfDGXt^~%DMYU{op zNhbpl@lyy*<~Z6C zrM&FF|LvyQCjWlv5X!9~vD8SfhNKhoZ3Ib!ja zFJ7ziBNQCnNq+kPqC0i(x`~5v4+?L(4>2QB}Ad*e(N;o&GDQdTUpp2 zf8o6H2WR9K3^I=9(B5_dYbqews1owaTUS`i@7u4qcX3t9IiA%NwIQ`Cz>4c3KlUD6 z?K|b3G>TSve#GV+ty;8>B3e+W<}soHF*yHV31+fG?4Fffg;Ze%_Dz8@cl7^TKODMR zesKAq$T&T88#nij46>o~XXv z-|zPA(KNSPVP0^v9-9-t_$i;_u!4h=&|J$s-1O$5Y#|)=lyuq-0n-37mXzd}SFRToG^>>V;KVH5f zi~31cZni3ktlx+362x5e5y62K4oV5+dME!oh=|+~73K;*BAH@+ljjyD(rU&BN9$!0 zAMIbeEI4u=;`LQ?m%5D{1 zhGak75U{x9HCK)t+@Rj6nd5*yxhoPFVu6|gJ%czrXlE6qy>oS!9)u3n z;&O?Af7r@dUso0rF)1_gvB=x^&*itxhQn8SQ>CO5llm6MVyKk~D0KZ}GvAqQ~o33V;3DzD*Wb z$gWE}Z{X(X52_xo1Af?h#M47>=h$smI{MW@Y=;SsXTw9KHx4Wr_2`pcZkE#LS-w?K z!JPA>lTyPMU`9s2tiq+Ll;^eKemOgqxZ{YOPNBpBo7qJ12wZ)UeTE(xg3gM6l zT%=F)&TUJhNbl_^)yIE`V=WEYzra~fN-S@bfGUsyd2adcj$P%qtl)4L4`%9@t@A5P zS-!QK+K1=7bH2|k_31bCtq{P7Ykey%Y5Vonp_=iWQN4Eda`nqXT2p?47ta>b4bEpN zE^x1UIP1R@bWbpO@7%NT<2E7JDsZiHAKWPW%?=*0zdZi7IH5n1VtO@9-GlBcpuQ9{ zL|@e{wQd{0(+X~VL#ZBH8V$yKt1~htVqdUVs}7^ zlK?%P&wDlR{>V8I_rtF1<)Dr$+2hs-&5yS~D}0F56!7nz-XjE0=|L%`Qjr3jK{^3y zc)Qr>cKep1v4C+cs{S8%FS$6ESZ^-4Gf#pcwJRRwMRj|^kI!pPYtVtB#IU`fr_iZ0 zqG+wcka}az^FukH&DNULS{stE_pFAxotErO?mH!4tZ(%rW51QUC4IF!z7)8!o*rEGtExEVyA z2&1Df1xmr1RzSwy*IY2#0u^B{fQAu@g3%`k>)*7&1E%nTZ^WDDu|0u2oSCT+!HAk? z9k!TFW;A8!-c6N$*griWGvm{Mnsr1U!EOO5&kr`6$RjRDB$aE;JzofZesHDgt*<+4 zXR*DgGHy82>C<7?bYzAycVqPhWF%&mJg19GslFGYm@W633}TS1rz0DzCC;W@IiB~z;JZPwO>+g0uHL{w}zi*V1v&eCQG zbz6bli`M>M@@vXd0BoLYEF19|>05NCjy0g?svNXSB=)EqhDom>TEm6f1Qr~NIlu9J zJvhuXqC;^cV>JCqC%Z5FUeFTdT=eFUg=mgY-?fZ zTG7Y+8A|XM0h91?XzEO_c6w=}p~&u$VVW<~JD{0y8uX1bZYoigyK*Jz$I)+|NFSe` ze`D_WT1BbW)PV%bkq*aV|9W43YKC4ovf7$Av_t?_1%ZIc#J|0W_7Bae29s$|De}w9 zvrx6NRHab)an;YH*$`;~H4l2~zy8v&hE!~DEjrK`Lb=H9FN|3||Bb>C3@z{^!iSGc zCy9vgqHm@=8T`7?J9eZdV8_`acNG$#1&=x1)3mnBcE>46J2^TbfJ@;Lz3R%M!xOLCo_E3(y>pZ%+wK5hrB1}$V=gA{W3 z&tDM7Ubf8ECf|;*#}Cby)fU(^MdL})SHXst5T3JYq#3I=JC%X!by>a6vRaA^-J7m1 z9GQ>AJ%yWTlOxo2hli=p{S6P1EYe0=NlXv`t?Rty@95O6E9wCJL5)ja*gN6|7O6`s zK0gWeYnpg&7~7NO_wN9IPp+}^-Je(C`<&D#cEd;9K*CU-fMnN48Q%xDkqPRuKgE@# zUrzgU4fLd5zYJc(s#q@&4la)7P30jlu0?HCK?Vj!L|+ykP(vs5b}S@qteg-Z85fl9 z9~fxCUB%6vGyZ6e%wEj}UTr%~<+6tH1Br93D%YNf?-uiP$6WkZ?~T`j7o`3xxWN#? z7_#wUM=}v?>!*|;t&R#-R+xU=)J?5xEMl~*AdzLzcX(ni1zvo*s2QiLXxJ2VN8OE% z-UY(CHk{=+dkaSQH_jN?K8-YEx9!m*pH&A%?`*dzoX63Hb_I>In0CBp- zYSGz9J$i89q`9%~{+|EfHwuHrB?i$EbJwq>URq|^^TDt@DzL76eM0QY`16DjR#e0; zeunL>p!=PhJBe#cmM14{N|n_-0vLa5P>#$6%WQ za7F5gz23%<<#6^8=$uh5f1Ax%t6zbVPab4Ah!^8GNrERNiRg~8PwXW|QkIQWX6R{~15!D=IepoBJJba=DAkiG7wu(+FZaRKS+n-$(m z=f32wLrow|ACdtxoL^4w#yJWo%8*?g|4UM8dAjNk|hO*dB$RRbVc;N3chdtFSvdB zG-C=`?t~V9ck>t&g4EEyHhg53?d)-18c--GaGZJ8YI|?@#T+U;l(ru2bM`_%dN*a0 zyy-%095$@W+$jHXy~ZXp4ay@V63Le+1PDs)vy;=!Ol6OsA7F~}+z78ONU7fH_RG4V zXJBmpUQ_YCV)eEDL6KDFtlT@HPx`)1Dfs0OYB<||Wj`}vk8c72%Ocv2EXzn6Mu2tY zdnp0(@%#_tYBXL%H+Nw*n0k$= z>?P=$w!E~!!)$0)U%Lh8k^)b*DUHWHjx6gqY5%o><;RL*_Mh(1()RjO$1JxE;Y25_ zElKWqSCiYMw}sxJ^-fKS?e5wFz8y5?t@gU_m=3RGw>xXkYeYl?-hedUlWT_F&l7E3Uw>Bs@WBJ}IR&k<>}#*nPBf9y}v~ z9c!~GWx8yjd-aPC{#E9y9Qawo&Fk@$%4@#>P;eF0y`eTQo-TaDkwd_6eH`?|2wfXW zV@Bgo8XCgH{89+S+bJ0O!s@-QRfwds{K64z&DkeKcF?1R-?NBKj^f+?@}_!27)Ta5 zrQ>RX{QByL?PCvGS{UD5g~%TLbn#TG(oo#-T>k9{Y{!%6t4o!zZL!kO*1nH9z`AIO zL&`z)Y+SCpx?VHZ$Ei;#`I?|Q1we-nLcC4Oq58vp+cfT|Kl|s5Sc$ zuwYVmDC2x|5z$T1+9!VSv0b8Eg}4D1z?lI{%9~Tbe!k_je$7 zoGt>j7#xg8~KVyaH#-&5Hr@*_rrbj zFV2tk6M(?4sh^J`6r75R-_ubrEN2DQqC=UU&u)M63Eo90o3LzX_M)h6tM&8iu*D4{ z14f$^AVIYHB7M&|cvVFO;qxkD(%hAs`@jmghvuI89{uu7iq4V4{TPAYQI`yh zFlzsA`}z$f`>_DEs`n$Wb-`*`5LjcL`QOT#mfdK4tqJmzcPf4w%j7-yVz;r!qbdLvg+1rUoV4O+<@bI&4KYD51#EB~6rYe%-4XBY%qqGGJUdZ$Ie#J2{&M#_9c9XwJ zxsPKWPUKh<>K;YS*1*6(_wHFY zTh~!CAVs*pEpw}7m-sqfPz-S0*LOO(ufEMq!~?GV(-*@&5%s~2f@Aa_wnE% zr6)iCc*{3!FkCwgb@@VbyryJ9iEI@zU$BQuK@l-A;(@er;YDOx7a@P9{)?*eMHg3kPR@247a7FS8vKssFolKp zN2E`5KBlU{shIHY^%!ez$2NGBGIf8fik#@_mO9*r=_FRyG>=tQyTNlSH;LE=K#U?! zgRH^}-q=gev(wXqu_4{fJ^$Q{kDO*?K0R3?!>%ygM-f_kPxrOj#y2zIs$|?$V1kX7s3?zVlM);Y5DTkpTB>f*zSi`IX|a^e|Vj|KSvtFqLET6%72JLV~-9P|9j#@ zGH{SM>)0W0J2eDkK5vEmk{mwTf;Jik>ZejzyaSl)*W`z3559jvK^dpZh{z?!PrF+Q zHZLdv*BBI+yO2og>CV|%?_Gz)@iW;!S`s3wgBm}Hm!}7-^2L>+gP4CA=|cjr+@s^K62riqmJyP$%r65S%OTu&(OafDFDW%QSY|7T~l_IC%bWI0xW6djj*EA{q|I9&5b zfu1}gt)S0>-6RKJL<4FY^-88a9F@d+H<5;7s@+gT0KR*4eqdkF&|DKxQbN|h@3Mtx zy6BF_#cQM{&<}*)o$Cabhv#M6Km+w52TOa6yGr19JiMba9MlGto;7F=irynjb%=(m zYlL@Abc!Wuc+ZMrsdKw>Cp>aBpB4R$zdq??mzmq|zwewUiy_EwSk8~*__Xh>#53lu z%bNlCEh9PVuD&g%a$m_grm^3?oM#J)jIEzT8`&n;n>t_Du`O06#@B?rJlON?ToRdI zo%G0NcNrE*f730Ps%b$@ks;hq=?ZT(9N6E(9ZBB-g^I*q2jlI}XaTz7_Mhdng$2P- zbA+h|J@ogFx=ljs-1W0T!HJXM!)i=N(T$An*9UW=1(#MEA8kX^pz$sY%>{}Kwc=cC3gm=_jvB&Mg7Ts6 ze;h3Ac4xT9Ei9U4b&;Cz?Us6_EPjA=ZS&Ym z_bB_yyu6tS2Q8tIU-p7k$nV%336>#@WW~`rxgU<)sdB!wXO6Q{EzIr>aa?y-(v z5YkwS%JlP2lB0!1V!-6V<8h(8k@Y-5iy%tjRG!e)zn2L#RWI<502)-@?dG9{Sz*8 zVOC)Q7HL0jDq7N+I(#P6G%EyA7;9{~kqH(cw@#IKqGk+rUWr1!TK^MUybY|VA^jq5 zZ+D)>6#yxLFVy8|xHNJB03OZ2X-RfST_YbPzk`EM5sZK(v?N2|uD4M&HLfi1-420L zQ%b%e;yf8P5lG?wUngbmbAH1t`D-#dx^NN6Neq7z61ox3jIO~(dwC9t`O?d^4U33Xp zfnOk8k850Tq~LnKE=0^b{9+poy5bGdB`Re-tT`qwyRbF?x&QzV=@))->05^ zROGD6DhvIaYxcT?-s~x-@mSfHRQW$X=4!g5zSGT8`!W;kpHcWvlwW0@+!QU+z5&T^ zW6ZAHSj`PM&fv`zN>Rqq5i1g!%+^A91T6nxHskDzdac$ zyT(6=30E%j^Ri@6KYP01(VwaHLej=mj%pjK?ABkewYTJ;Ezec6t4YzcFnof44*1R8 za~AqlY_t{q zl`augdy~_m1VkD9k$#`t@z;Olh=$chgv_+hk59>tD@^&j&-OTt@v;&GVQ%L5>#F_Y z@Ie4HNRC=34M4TKwp9<7+&4r3T*VW-}VuQ$=9e$fJ|R9#T(Dqm9& z-lf`;2$14_t)o>zUt8;Ntg#E@Z`-sCB(uebs3n|w5L7K{du}VaiBGdUC&!Kt4D93( zx%wQ6s-SPjmtzv-Vuq8YssS=SHvH~RHn zZqE*KNpg4Xi~aUF-e~B)3VU1(aU7WP!Hu8V`4;Ge{j}x$IP-I7VKUSw#qOWpUBIH% zy%N9EUrRiTaW8gw9GaQ-T)CK2HOoPSZrkB-4Hm}Vb!I}Fl{J-otYvTBZ+W<>S2}h& zw#ZW3K0V@wvR-?_SV@x&#I>S#l+<&io_t*ylwxai*4B^wd2H)5<6}qhew*osGy`*Y z>924*t@4)R(|}D|3EX{E?UB4JRzSX;4?0wC5@A+;JXYWNRb0=FLG&0ek3Sq2#6gh)SBGkOMl*i^R8b7xc#LAh=Z`Q6p+TVEwb+DjRz=C+|e6GVL9I?PB)S)JxUeQy9oR2c2l+h1(c;=%V zymu)8Ie{s0%Hfg>)D|?fbd1y*blYD+)EXIim7*@Nt)}G=ydC$#%ekSaA7FJuGpNg{;fU39_zj0wyo@3({s z8C6(bhsUvmx+JJErV|x`Ztw)j5Dq_E*&hC2mc85gBUcxTNnvmE+xW3@e7qfNI|1qP zB!A7&6>KE3-C;WP!I6@$%4G{`ueJVDW?BmCbvbjCM8Jv?Xb7AtmXrI|x9XE{B4R#7kRC_GiD_o3Uj!KrA%TQ5=VqRZuX zLgD(C{CKdsD})MV#Q$e6#s>QqkomKJK*NF@Vaon8P9C$E6;By7F30@hgt<@xbmVko&9ThT$~n@OM|?a*Dvkb#lu@|;52q<7`aGJ* z23p}%w)pC)!@{QA9bC6pYt~9wZWA=>3)xx}Syrgg=$!==A!nLTqG&6FalmJ8uLJXV z{F6Ncw68g2S47}i(Bt8|E66UIS7CQM)Io06JM$J#iVCYQ)IeK3-{2^&1Vx;WTJ9Nc*`@Dgft&q zi(PhTtb6P=_uW3%|EP)#M3#*X3OuJ@;eylOF0ghawkl%V0Eas}Rha5opMQ-f+)1*O z+Ad?UN?(IaO3=agCq&;QraOB_Mft0Q>jU)Tc;cS&sBrOZ9GzPj#48^D^l=CH)2ef~qqmu|P z$_3w&4T&wlMB}_%De2KoK0%t4P}(i$2(>TN~IsXd~nOlRF8|-M8VUqAfH>th!qs%Wk#i0 zjRJ_*r0RD}xBc4xI64o1D&PN)A2Tbvvd1BzjDv6_#0lA%8OMz5z1N|{Aw=OIt1>bU z<>Oe#-q|CE4#yrLt7FgK{r%nlfXCyU@e z-NVkoH1O+~k~CxpL1M?+O$-<%vF?P3*1V+x)L@NoVIBd5I#94utLOT5jiS^V)!$|e znrFSU&z@3W=oRx%rm@hEf;z%Wp4Gqc7Wu?z1K+h~>(Je;#ytFwW2Kk0HDTp&v&Z=( z4Wpg^Ud#X)E%tAB4po?){D39^fW%rJV=w*?)8T(3>ibUdIx!?$=u7W&dTBoxEIDTl zJP30Zh50=*Pewv?m3PlwtxE=p9bLGJK)21@0l^8KoLhxAC=B>U!uuv3h_FYgg0$}; zzn5J$d0Be7=ou*$<_J?m?uhQpnM4?Wr{eExTfqiA?iyg+&^10)iioa@U*7eV0`Xl1 z-Exc_UFZHFX~XTb`shKE-AYh_Ul^goOM?PLE9WPe;TOLbl;-g z3(jphqw)HoI+!t<-DK#og&PJp^WuG0)?v=QXAhvMdj)SL9iPk|aXVah|b8yR3BvUI2FV6E%-k6?3fl#_gK@-sf!3TmE7TajFkEiHr`; zcmXvQT=;U)8>BxR2NR1vxL<50O|+c@x}c` z!oa}UMHS%WoG>znjbk?mC8r3FNgm#e5OGRQ$%gIKm#8a|ihl><4vo$>|30RvA(A|3 zt%)5*6PP37`O`V&VL^aXQBLSvUUnc9AMaj|HQrDPKMTM8!(WsE%Cr8@Us0-I+or~$ zH|6Tz#_3zfvrc`)$==Ql>bbgYMF2FT5LmFs1Z5SxI6FIBeL+tQTIydB5TJ>(yOwyn9TES zV}JcUt~#GjL7W`-wtM+3N4yTymR_!O_)~eZrYhs>5KIfYOj1K=^9wAQ`uEmBenxi? z6&a3j-%qjIk}nE3D)A2wZe+1v#@6|$8Ny&xHwr;yoZnUWpijSP?dDK>Hln6G_WSx5RJGQoC1dB_KZ-iFEV3qRSdtv;D@rP0;A1 zS63~bah?2M{hv!pq`eKEuPu6Dlb^qJ6so{|6oHTzOBRQI-WwpUdiO`k{K1A~$2hlE zOxuT6R-Yl^}gTbMO*;UO9q5`NQiak!gsE^fwPU}b;Gsg&N!sA z?$c1M4WQ!nUfE1J6^4Kd*q!lPy9$qt2)_u)E>8R4VCdC+Ja@Nf3Gg;XzzGqv6#(2r zEbsVmG^S;@x!g2= z+Dl+d4^h=1vV;*k->g}1Qn5S&uZ}i%N3oHowtEB8Q-4v zHxhO7QrDW4pDL`;}fNyT(aZ8g=yvh}a9c#kH%}>o@Bs zx|zq^!GvA!GpH&dG`3iw|2VBu!(IZJLg z^t8^KU{koGgl9=MEOFdUG-i%8YThdE#{!1_kbqO9V)dGDhaBn+H#zqRk`$dd(zy3*Sx+xf~L1pQBp;N=Z$!dUdIqbxQ!rY z9n+qNe7_N#oK8khwCg8U&8&ABW=^3{Y2!7(SXw}P?ifG0=13YPVUuqO|Bo@Y@Pj_k zk+^u-xy)eGn)5JcyDZ&bDWChmI}_!y*&KIjto7_!XH9Lb*)}YdgPnZ8+W2(Tq*b$oe?DRO={7YdF$xzcc#;|774afFDQdui1$OWl{Q9lp7oc~{ zTAo^(WYM}VVm`VwjX8NyY4vTE25$mjS5j?j+^&oaX^67OPmY(@4wcWt2S#4~3wexa zuBhV8zso%^m#F45jNNK@$l(3PT9-|FjV1jCRH>MS(mg_&Rx8c#EdP2V-=j{VHL|Ap zRQxhZJepzCspT?ritx(eDA0@zv%QC&mn^Uc+RwVHTr#c09{oiOFJcD znyfh936_pWra|vyrFDzgSJvS2Ei5Unf zvN@fpUp?u~td5Zj?X5%7O%ikud#FARdZYP(0rfF~Z2Qf+c31GW7pP$Q8da<8_CfI3 zI5|XV#n42WC;+ai+}z3fZO-T%cqf=-`?{Cd0>XL~z->nfCRyq1Ogo-&WeHTg!3J`) zBYlaR4jih+sir%wuC8lBfJp>G?1e?s&HkFyVH~O`4}7jGLLubqZa=o@sr*+f{A};J zijJGxK=lJRuc?c*xzQ;=l@KBfs;xkUD4~3W)kvfCfk$E_u5-~-?rF#A>K~{2-(JwD zuqyO%9#nf9HDQi({m31=;fZg?BBtLv*q-+x^_RUxLaZc3!Gw_2BRu{%1#*=lPLmrRc{!^`|qSHkq3ep=Fr7^&*^l znmv!G%}I%#=6`9^Bwik0!q!}Z5f^D&U2PZ+Xc+wC zzd_8LP}0>YHQBi7Ko8yNYD=_w%@XEk!a$j&)v>n5N4Z0#qW6atnQv{)*w6(rQjhBE z<9YnsZ?`}lCUQ4`HinIjYN%GCZEH|r&yBLwV3_?U)52^9h`2{jI45#K`Chz9It5H!U7f zI0Xs@un6)2gA5Sa4FhuUJ?!o09aX=7B4{<6;W1TY@{u=N7EC6gi+8c0HRH(P;$+u8cnZtbMsU)U~A8dN6m6(aHlH z&D^A+-MwzNDzP8E(zj0(LO<5+{@@pVjlCZ_pE3aV-~^AinMjoQH4id5_c?O(=h)kV z$=`UiRFIH>D#{_mj8mPvr1h@%C?|&hw2@biyOqClL5Q78PT_R@{p{w;l6qw78;|C2 zjVE`Z{hseZ^DWr0$VRV#MJ5T1?(R-pFLrf<3pyE)WGkStH@Y>*LO`i82Lu^O?lmHR zyE+y7D_H+3NNZB&O8FD6Br&jhb#*0Ur(@~MgW^M+BPbubMf9y!BiMpl@+tws4F1?i z_WBLBG@7OQQJ~Rt>dmsbu%!5@B4%ly2WqZgK#Xcy5P|CddH;}Ht?{4&aRNCtj8Ky$ zUV>tnU=6~4@XNCY(ebBbk-zl02e8F$sbq~(H;krTX-KTkeX`=&O__ikR}V5VMw9PE z|N1~bh~XHhhySUzMtzF;{YR>bzl5;^r4$6qoou;G{31bUOd zyKhkQk5j?U-%9eBwd5N9X%C01Ix1a+BU3ZYz~Z~jxVnNptkvHt7{bdk^dBD z!SSEx`+3uKuUI;-PKPEZ6E7cd>wbg&Pp=5gx{0}AN-qvEl7i!w+ILv5pQ)d2Cs%D= zxpfw|Qcnp)RC4$jut*vNdpMGRj}^6i!_LQ?S{5Du%XA%s9^yjxzow#$CcULDs38S* z)tz;(HPoW|{46NQ=)DW4YVoT)pGt zLvm4EhRM`lU!Sf5W)I*AoqOLI+4>0GE1GT?ti;=TzKl3I4c@@3{p_rt|2r5M(v-gw z=xDYMv<|_-p?ON-;ih|1bS}ap#1C;yYA=e6lBZQM#V?@@7+LA>_Xi+FxNWaF#JdOU zs8=jzd$<4>%u4+Ol#g%IKxZLnfG@RFqp)6$}w_FFg z{w9C?fhvD+LN+eSH?NE@+L| zGnGCge4P077AdDj6HKNSdDyDaHMIX~D@COXO-4Lww6Wc?d_FiBy35H#lnb~f*Mv16 zC;1URkj;1ryiEdmQo@{<#GNUf`{;#i)jc^s1{I1wVWRf6UuFyNKn)m<5VxBCv{9W- zrdna%l^-&S2=%P3*;n$9Mz&gJ4D7GEu*LFT?a5W1D_@`bE4BPB@nL$A;b?w4V7iZ~ zt$5Zx5#qbaT3Cv!9Uj>52lupPYKKF)doQn0sFLdwo2z$G@Zpm4{Rq+DW#wn$H`TaQ z_zUZCwZ9Ji&10Zl?)+9=M-GKP3PUBbQ&?=ooIXbUj`S4IixDIU_Xa3fa#rJs=8PRIaOQotTvGO!W+%WF=W=yYq-)p5x&FW<)0L!18Z zH%%NNyLS0>`f0Hb7u2Y5E(4{HfO1h9rQjbl1c`<$pNEO;UjtevAFOmHDkwR<9kzO&O{iaxW+_z$j24rmamSN)zeH zN7KB4552s?det_aH64L?UdI-|SbK9nNKy*Y<@oB@ws94;iV{DaS6dw?ePm&zpy4l0 zVH#_U@3X_3ZS1h)&OlBM-(TT8@psDn1#%fGwJITB4G;JwO1~MlCiS6Ah!ZUR961({VLkatW4Q9l zSBbhlQz>7cM>u^+n1MlU7f#3w@vCrJ3xLlnD}tBygTsMtGz8*$2XU6KS297Im^kmf z0YXADfU>pbjmBCrA4u}D!qmjZ3JqVsCK{uUVjxymSgR!fuW9A5_UrZQvsUIJ7tw|9 z4+etHgF+Q{d;S4&x+|bvio^j)c75<~qkdTVV&7}o0C+7~gu2~1IR#>5S3lBTz}r_Q zHdY;#(hbO773DdAyH*v~dwW$5cSuK47jId&T>o0u5X}9kUTwhd0&_yvpFtWfP!GRM)LDw?G~cIUqm5NC2U6W zXg?Jx&^Gv+NJi2|exBP^AZp@jVgk@lxpB)E`!w22Wg)E=gRR2SRhs63q?U|T=q}Z3 zNTVhC-RCqH#XRwIm%5qlwq`NE-N)VXNkzljKYuqeGTnF}qU%`kWt|A^LH0G5XBB1E zp&@kUCUv_$AMxdxB&4lYPT;o8*bn0=|5CWoS_A|UohAA@>P>%WseRc(S62+bccM-- zAy(cfrRj+$=%CENAy zx2<}QMkfy_K^^Efyc?5=f6sg^S;=l%Ip#?;qpAA;yga)=m9Vk5yTN6-hcZn9%}9qI;|Oj>uhh3y8EihFtvgp^|H@YE-# zry*etQ#oXhx>Jxja`)VlAb}r?eH(KSr>l1)ZV%Y1>?rCUj=SYs^7Q)$AwsM(2EyB? zH>(R0WbOB&7~^|as>Lm)67H2^j(uBqx56vmWs%2zAeFSEIs8Y>%q)9nQe0Hfq~_QU z|~$>vs2qc8RR93ImW%T-XcV56g%Rfb{``xnk;C$*u*5? zzNQxvUgGe5pjccQ{FU*wi<(rRHhU`U&~&2L@}QsV%SQ!yLo;*Br#+d>aRslkVhP34 z%zWK_5dZYhQjD0IIKLp-)LgrZn})Uztgo1m7DnbkhCxkmsT#bktsoBk?mEWiKGMFi zZM(Dv$CL(ar$dWbXH(@HSV&@1sO;%;NMC7+sDmXj0l7noB1o!CCfJSXY6I?>BksH% zBe(HS+0@S;ET>GQ#4#~NYZ}~4lPZDOwY_}9uyI(a|3^oY-&5Ka8I{5grF6Poha~4> zqlZ=fM>?1*?41A+d=6R}qRx)W34XU^EK6o8B^@);fJZDUU81TG4}BYe-f0_(RJVg5 zZ3_3kLsPf;O1p`&TKcX}RG1&}Xu?Ho#J@5n_svOVqkw2H&3u~)darcM&4yazHT6+0 z+wyS`Xvr3Y)<(l!Z?5QQ76i1kt=+!wRMu=hPuV4b>Bbw;Q<`D}ZuT+u^&vA2{asjJ zYX_+cwpa~&6{)x4P^)R+I3F&XAC_q(acLCTU!mq}>%^xpkTV;wemEqv$*;-$65e3&k$I7bs*d|;H%Z@=^(5MYwxMv4-ZqhjCZ-HN>z5pZZ_NH}s0uI^* zdQ&AjEru)p8;+wxHbeNPfL}pcqU6(l9g&;M2E_wX4m2^i3N_LPkel(O!>mTbzzveg z_Ya0tByt)}=<54?j6bksB))~YES`-I8w6PVx)@<}W6{xELrh$CdYyG9OEb5x(L9Iq z)t>igY#wd2W^_j!p17&J@>+B7O5Aw&VwZ!zMg6th`w7m|*+jZaS06v(!wa`vVuGpM zUZdU8y2>qUX;7W~uX&wFV*zp_M7NZmwB*0M6f!>csOSKK(^NM5-t?yEz>;p$k*#D* zn*D5SY_`^}u8I6V$(x;#p=UoD&)0&Soj`03$xOI_r!enXg=~`{ZboOsbngdCH=^N8 zL!Bu%Rs4gFjc`gezOJ?qHv2@+dSp7k=5C*l^1lIK>jZA;>sGp3)LAbu$ySgS!=CPs zE+HqQK$6|Jpz@jBu0IZd7X|32rug=~UH%;o&-j@>OKQhQuW8#y`3GcQf^sMyh-}p$ zqDu6Mz9HAasByQ>-{m17BT;C2d(>#IUu={JLjcsyp8q@UN%{T0@!U6TQhvL&;wDz0 zs7C9G>?euOyIj$_gIuXEbSL>^U?SyJ6DcO=r-toCAwk=lWo`jMfIE4x>DDg)L`ttB z>zcDuZugKIF!d(e#RV{Vz+iOzB`KpS`Y0p4pVa+~yFG;EW&T)isI>|=Ya8ot!!L%0 zKh^XahCE-FAAwRxGs{UUB8vLgvX|D53z$bAm%Nbu&zz1mouHQZBN4hjyNc9w)V%v% z=Z(MFe|~WC2aNU!4<41ml|2!PosbatX zoN_G|rKh4IKXX-A+3|83n0KbR2)ln!cdMnA*m`7tl1~4#%4YV}GdIht&8))^e@3w9 zB7?NoSA&JAVd+l`81DZb9!yXENIcDtCIZ6XQs-jg{hD_r$DQG_OVJo zfzVy>P}oEx$9P5v%;!%s4P9C>XlD-AR9=DWO1(=SRMi}|;zVfT7f2=PklA)agSF8& zL9)c%v=8UfRuAhxwWef6yg>XcQfvvkqpC21VrUIl$Q36xBBvX1Q6#V;EkSw_jF1n9 z2<3ODGH8!TQaSIcOdb19aS+4A0g4wvx>Y+ck(OR@_|9BS&mn$^U!TRN<-wu9!o3y9 z&6g1gr=HU;P>Fpp0TyQ-#M}!+;z3 zi#J?I`8BI3VTMjerT&fIh54@@?#32io-d@3KAp=%er56#rDp0jo=u^S*Qp7LKjstX zQUoJ0LX~}6Qd6~jHzBd`{^N6qqZNv{e-Mp9n!fFMHj(0+tl3g==gT)zK4(wLg>TJ) z2a$cpww2?zRDNZwR#$ErI0eZ}wfSsU6HZJ%vdh7ds#MWrOd!dFIe5+ABOe`O)wmUR zD@_)KXr>_2fI|?dBC#%yTg9IymP&8Ov#+hdn>RK|kH-n5NT=gv~+q)p~+>F@Cr^vS=>NQ&8vSl*`G(#_q%TTGJMx_+!~7F}iPUz70ZmzRR~t#rr~rgiddYP7 zC_~T<`gq1kTi)X$8U;WD`^u?MMd7@R=wrISOoEw_?MWXZ4u(64Pi^(&{r~BUi zQ8_^>zxgX~#-%~{RNnKy_%tcFzONW0%_4n6Q2HK~U3rK^ny=`7wN;m~)w#t`Xn*@K z8`4jQvGfR2o4}+!hzT-zdT;w5o)Y#Tsw<&*f{nk3QcVBtjh=>ermO-6%rHA{9Na! zPH5FK+*iZN+l;E8wZu@86*(wkp`ykr^He(3GXmebi=R}?;8N{qO91gVlGshtsLV2Y z(?(1;HM>qaTVlp!?@^OB54Wj$`;*<}1HlHpAd4PvPg{4-T%TN+HB6f&Uvngn1}wK% zB?&K-ZOr-~moHsq6%Y!)C3#1r?#v2Lx9&yEdFU=*XDMKZsk55c#_ z>^Ljy;AKb`8`H8#H`pBlM=VS*Em$dESxLD&GtMUwvi_N_b0OK(@d3*>^E#D5kq$@m zr&)=}&^zYqy@Z{`8Re$%;4J&fbtBGVSX!)cm=}I${A#B6e0_pCD@RRkSf@;>HHR&W z`*pOjpS$%VxkJy-k_wt$_)qtIG(L)Bk{TFOYjJK5|7+udQXLZ{wd82rRkS_I@W$S7 zs1)BCq^?EyY}Smd4aQ1yqYwDU{u=83PN)k9D!ZHb2U1x2A53Rr2ma_-UHXsglK}^0 zj^Uul3TxIccWac;W6Im;%?#h38Dj@Qpsv)320BD!H=;qRP|?8B07)z3$FH4)%giIk z9lzfuGXnT$BaVI|(dXkc<8)!BH*XRa;LpG_6|N7ZZ#>mu{az}&FXTZHZHpv#h0pBd zO*uZVBkUObt4|Vh1gnsP32DI~Z&F*KK%1e#_Ec*2_)(oFlX51sbl(37sLbG{-g;Ru z%Ht88nOjvAX1SMD0an*LXQFjy3S^C!o2I2e!|Nqp@PHDpG~(K{I7Mk#=OD|9HOaqn zy92)`>iK;7W`k$ZtGb}-w+ua0<1^!-=wIiiz8MbHrtJn{ySnQk5C2Q+Z)~j(pc8?T zY7W~WANd2}PB<_tyN`dM%c5qxUz-CIWc{Fnz5;#23=?N@jY$bK{E`b|oSY6^y*}GX zMZIisc1TEh=x%Nb2-1W6EBE+E{5MVklusG)IqGzAXa78CK4Os9MhUJb(TgPjEQh?} zU!`1SVc}kHWo}q)@;zTaNmQ-1b@H-&YI~-ia3>p--waZfTitN91Ghju05zE>0&Vo3 z&4c+$5ld}qdRq!pwR#gWbe&daDp70gl z5MYUu?S7J^w;ljWjG>Aak)H4js!N;b0A3J5>a&UkRTI?qmcvBFcbe?g4&JCVL%)Y6 zOpAT|F^N#YVIV^TjY(14K{{1OoIuwhKQgzEjWX?9jTg+m;s}w7`I}8Sn4X^Y+fSnS zUwPyfv#B~0#J9$DxV}6)l83z#JhTj=aj)H}8sdQMZBjR7wUM~rmuFlI!4__;9Se-lsA6Ma8=cs(Wo)6kIhblc%ceh## zSDKfnuxh;xU1uZS9Frx>@W8fE?^r(;Y*{NlnLdC=vNe;b@A9}PrY^DYPh%x& zz!7eSESjWM!vwnazJKy)GV6>#&RHR}pTTjO78mX~=X$KJ8;gy5sk!h*O)Z-AXqq zI}V1eFX-{0`758&@y0y8lIxw;YkhzPnphmi(l76z^7H+&>Fj@=BhGsB*R&mzAw6Bu zStN4DnE^L8CQ9@X*Sx?e=o)doOH(HPqsw5)5k8C>@SYwTVAlTW2;2$-vDM3Uu9{=| zSYG8wuHJop#2E`oF>9a$|54hv73aD2(-$+O3+n^*^SfJk;7xffUPcqUEKlDlSy=yI zU>SR3d;6TX<#GP|o#E7LpjB{e@Jix(>iWk~WS#^dVgksJQwcy>6 zyWrn42HFLOgs}VmZ9YcR^ZW(s%jYeOT>jiv?Spi&^?oqas;OkP*JOv5b&56zY!rv| zb*AXAm-zHcjPTAW&*kdrgGM>iF4KB`LWL#ezd&XG3t?BITOj&goL^rYOQ2AadjK7E z1q@_D@b5$9hPR6T7H@o49=Wq2y8ixP_mPP8a|VN!Tg=F zQk2$T^AojTRn7TPeO>&v$YmMb37JABP@7Fs;^0O8$5O-%=i2MYhjk~1_pUI@mwu`T zG%xq=F3pX)s(q>J&Ief{?t4z$^osIuvOAAD3EH%{HW?szWlbj47p1WwCaG)SBeIW< z#=8a?-3REt$&@u?GE@V@bmcwExV}jHQ9YYe3&RJBgr_w9Rxf@ ziL$Tgw_FC^125MDAMeyD_Sm%lWLs`m>B865UP9tteMbDR+K$!N zeGDS=^Yguu4<@xlfIqw=5kr!CGipqB5we!T5tvF0e*(J&-|UE|j|&PJr0nk!QFpmS zq4L9a%ix4j{$jpy`=$X%cOUp4@WV|z^=#$TvbX=Tums+Yyp9;o%JFZ#lWP44%u9cR z5_Z7)rW}yI_PAsh%Dp^0`V<5OCXKxVt2L)4j~_qIa%QVf>nODUBLwCKk;7mzqubhU zt{5EPY0l}8Nz3tDnR+P)Qv54qvH8GamlI`%}iud?b-7>Q++(Joye#v7bNfzKI z2-`#tGz{zOpSPZVI%Z%fF?!(m!U7)pzYr+zZp-I3y^1o-{fhhDqi!q^$-f5ALy&W7 zrIaf*i0eGbZ)Id8sxVa{m=$i4UDZy={v}MxU)^>uikw`>GX8@*vm%AMh8Y>TG{g+1 z;r~4`pZ1=d#vdDivIszVNdJ(Pso!38Q)9faJl^%VEH4LNz3oL@wVtfnRcW$zSW<-2 zQXA>ib$W6@ft<74dzuRq$zw08I=2~XR;I^YNn|b7g8?ZwKvsAiSR&n&@2w_<>RQaF z7l~tVM&P?u-E>7`LR^}U8A+q+vE|stgDG$`oL!X)OQ2_Z{{_4xsyT-TATGet>xjtw zUtMTxX;;w@dVFHq{}ch<>kWgLKqkl=;%P^l5qqr9rlFBm06b`d56$3E zN+t5P(^Ggkg*1Oqi>?^JdIucYVpaNzKm}#*mop8=Pl<`;+hNn8Xv|tp+=m_?O%Vtw z69p;kVsGEbZMzyb&}4cmxiry%r=MZtB$7|p@<=gZ>vrQZzoi zhLgB74fr(wB!`c} z*yN$?t3A-pcsm-4W#`II277SzqU-%R>6#u4SU7UhW|1Dxz3*KQfigJEZ3kjIR}QFR zI*q%wSA*U0dilc$fU;;?=Dk`|R!QD!jBSIo{qOQB4wDDt0R0k$@DSo1!V!qp$`RBl z>wDSlf1aViA=&x)h{&evcCqxP@;9qFaS$GQZ2egfLmd84Qcv@zFpvI!92|e-=}QI_ zry6kmab&&R5VS{nt!h{0VqB;0r4fhG9MZp%5nm2RlpjPSfp7fAAN zapB!0g`H0dJ$AG6+{KrfeoA>trLJZo(6WOZY3Uw=S;44mXzvGKJ$EgamBha022SJ` z0s2lENeqg`z?jU0Vrd(1pBU$I5A8Mqvhp2FCH?qjlDMDFEl=WSc03p5EF3LDF37lD zOqr0#rUGq*8qsmms!-9Z_-Ea5>e;i1d#V#n%3^KNdMnM^{MW0qD9e&V8-hLOdT3Z! zy^lETB6N5+-`r-~^1=eYQ*{8HBR}E%;TxoSK`2AkDaYLZxj7!+kpaU(Jv^A$t>b2U z*u0z%y`s=lS~hjt%yoB2!!A!e(*ntc<{o9fY52(eoR0i9sZ~~X*lsfhNB%KYUd{u4 z&zHsrl!!)e;p2sw`IlZ4$8~;AtA6nAP`vrrk!ccV=0Y1ve#}ZW;9d90K9RL^!&hmj z#9xH^JAvLoNDa`peiKuYOZ4Xv*M#ak&>!CG z#a=r$y>(IMes5dR&fw#LxRO?@E2+fAxYm#+#Pq!G>FKJliLk+M-E$YSTOFsYe%2UL z5=L*DkijyGBIeGWt6?j(1e)#r%dc5YT=|SXY(S(Ph0k1$WYgkls_WcY3=iM&_^s=p zN5UNn_f3SVo7uhm0xKX-9JVbJLiUm0T~+x}T!a~>guB}EI9NN(p4pKGYx=ZuIwXRt zt&%#TzPleucm63%>zVSe4 zh>1IP=wkK-g~?ASb8kUBDC*-^p+Is0kPz%{5Bih)e@|5CVbPtxXVwjNr?A^*S#_Uo z&QDJES7kY5YQ03ij3ebtO73qZ-2Z8cCLquQm$2>b z>_T8S40|(;kYm)^BKeW#nj*0UgqgPWpvuRM@ ziPhn%llPj($HFNjq_i_-XU9*=(OPeN)mMQx4l*cld2z8slM@@n=8(wPHcAksNPOEx z@|&*a=+GBR&}T4;&(Bi|2DvEdI;ec}ppFItV? zPz-$=0J&v8LnHt6=}j7Va7H^x#)7&i>u({kr3$Yjd8&wSwfw${cuh}ijl??>ExwOT zAO3z3jl6=7Zk3xbUE^VEGm?yRuv++x*q`MsR-hk`*_-pfa~**yB`eFP!f=fY4`kpY zbXg>mEO7SsxTKlFwg;*oxx-PK*tj zhr(}MRrVSeyJVw;z@kG6pQ8fGx2qpju&Sv@eJl!~tHI8Yy!ndW0#;4KS8IcbZkGKg zLxvJt3hP{P{8d}7PPUDl_eh(pGLVYt)x+zA7fNCyY)j$L%OQq&#Mwy~+ZypL65Pze zRIloFGgoKJK*;1Fnfi2+QL>9BRGr6H_51Pe4%1%(HcqP0Pz&j;fF5kRpJf|ly++($ zy{wzh@_+5&OVh)uBlR4alZ>i$*>t*{MSzFeWUPiO@g7aw#%mO_2{?eFd=R!yH`?aN zxpmd(p(UsZZ8tZhF{<&V@x4T%VhNz^(YD0iUtiI*M^9bJ-x74WltKiTMO+?SZ0uY% z_cb0c8p=qo$OC1Sh8rR=A@I@kEMFZ6BfF$(!snw&tFb4~vvN}EjwVCkx!g#3M#nFE zjSZ^aE%SI^p8mzP~p)8_k12p@$2-i$q3~a5Nu&+{&hOes-D~%*Q;QarOk< z{M!tkx|IIq(o$}}wNY@h>i&V(a{Km^R&j(hMMmXv@jm{KzjTi;;_y^k1hS9LvfZk0 z$we$zUh6KrUGcKWl_U9w$(~<~wL(q!#7WaRJsnil14?80EoC%>OYl z(PO>lCp?e8C5-UC2W83^@GE#!E{Wqm$?UhWK_fy>SxL8#&qLt$gNU;Yi6X9+%cINu z>w|sYN>o7BI;OU=j6Bb;g*OmF@4}_G#pTe&{0JoVZIW;$nS7T#4M*4gM0Mr8KCgY^ zKqEbNBRB-wtQbHIJK31fkb5}g#WVMDA~cJvJ(2?;FD~}>zLhoC)*fA-@LpqEjsiU5 zI!S(6mmkq(h6o!`*=2FEd6g#(D8Q{b@ZU@20BmMJ;Ti&acsJ`JB+gH<%6 zJwCVPsM8pr8kvN*X_+8QWZE7gI2>v&BRNk!;pg+h&DC)KE{>1rfsJq|R)ULORsws; z+w1aTOQ#PhY>y)Twitr&?K4iuuV{`HZTlIyLcBbN^krs}#v6sybp1JVJ ztL2o4xO^wGH3VIYpx=y0H@DP?xm+A|jR+0N>-krQAa)vz`$B;tnaO~oIajab6hk~O zJ+MR4VocR~ASUVg!%&|4l?i$vfeY5v*rWKjgbZw>w`WiM_Y$@`oUq*R-|3CowRUe;f@LQEc9s5t|eR$hs8EA zkda^S%-q=dfy*}J02ciBwj6z!cVq=`3M#hnpTCj`yn=;rW}go1PllnnnxO?xlgfqHeVES4&U>JzGA zL;c~O%}ToZA=!s~{(}Ch(q{KNzjHIHwv}IZxHWH9O_@UXv4tyWYTIv^CC(|-$c+g2 zb?89Zf?QFF+>@QD**#sJUxcdMH?18Ta#dm40t$^*@Wn-^f`pooUPLa%BzxIDcK=mkZPt6tJ~Dt+NuoCsI0bL^L+dsy7Hq-#hsklLFHy_wtOL@03#Fh z9VO$$(I(aF_S6@+RYyU4`oZ=#x%=+mm)>-2(5^O^N^{Zz8- zWD{*BXX{5Na=RF{%6#{Kl~L4VGTu5O&yEL96Oj&CNOz^&!5sPOnOJE(_cA0;yn{W{ zq9uRXvKyTFOY`r;R>Bq1d)bPU0!uv~DYh#`20lT%Mt0{)3{X}g0||Wt)w3yEVz)2S zzwMcI?S=|A?|a?=MM#&ei^;gZaGN^Xue;&ZEAVedFv{jFdftihO6%rh(nt^W)M$cI zYTcfzQ|M}toaHNNvT5}r&j}G%o2iz{YOO@50oa(VD&nEEhHdigDE8yHMMCG-EFhq< zfAFsTDza&|5qt@h$}|*(1;m@$!b(9N%6rFo|S3NT(>;o1glc ztFpPF&k3J{c5lv(rndb!3bdZ$QRIzHbD2dI!Otw<3Yo*}r~*n*YUloeucFd!kfZPu zUDt<#t=O45%)c!;TEl1PL;RHFcQ*O`M%fN0dccC_x(&?16?63@gfM6jK8&|yJe0FH zH7f{9C^>M4hwdhO2eo>k%V?!DK18?wJdPuDb`nCYRguU(h~9)h;(EGkMX~%WAo63- zkH#{ZZw_q#l8o9gKlO}b?fRgf4gvt}_4#Gwf189^Enp5Oe{-1-*-P>%=HpQgbA#HH zNcv}1GY5KYWsd)PoDJ*NNQCHgtnYW_Adv-G;)8G+WP*Sh;I7XdZH+1ACnKK z2)7H}pzo}}KXo&E(|t2qK7A~zjyfD;y7hYJkCT(5X5s^nyB#&OiFW2dkA)ug(6@(? z{}!z%4Z(wRD`bYx>silz4u0^s6MKU9M2GA_@Z_e?7P86Wzb59_BP*1b_j7w~KECm+ zx$mxIkY8Rjw^uO#c-i#XMTXzyW_RW8P$diV{CFj&HK;8%bFcVA5s!_+$T%K)pJk&dgWYZ>-v_(lAYXN?cRxLS)C z_;gx?8TkQx^Wx`91-g(rPWvy88@f}XCKwhrW(V+E4nB&?N=n)(5aj!7in;)dn}n&- zh~{c$W@S3VGeb1ZwQ1lEKzBHk*+wq=SE!xC0+p3vmb2?mi-23bQuVYNq&S&aMS(k= za~%Z}aD{wlyrZ;kX%tY_Y!Qe5RQB!RhOX;gUurCWoXXA6{>4SfynMY8ecmzt)i*WC z^aaJ}V{2Xh^&CSG;F%aJ{Z!1)uBye9$=xC2l$@WVN!JhLlSRINjW6}|Yx+V)_FeZJ zE(%@eBBhS;Y{tDUi6(6f;}~Cz+{J;KbR4H|7Nyy7F#R|l=J&Aqj6bVswd6c`kQ=7e ztwMIkcs?h|I#*NnjTg(q2wlc3k~Yt}u;U$qd@>qyD&x@R!etZ>3?#H6UZPR9@seGd zkmSx3hjF1qnfv5aZ`(;mxn@aiRb}bp$pjim{N*IQX%{pTq7v6O$2pP0B+iDDH#td5 zQlt#|^wyf7oekM!Wzp2~vK`rms_HUQKV;?Z1h6m{7-y1^xynsqs!U)iR*WK!lsEcZ zsbFj=0_#$BpH#W0ow-O1}m5wM}3f6c_yc&!9*-i{Qm(^L9f1z$cTDZ26G26r$8yC z6e*=ax@`m5&@IF;l|jMB-IR)!aR@x*Fb#`<8WZ$(0bfg|-7p?@`;sQZxBzuUe-ay}#Q+Ln+aNv9kkBz1iM@c!|5VUEaD@|cGK znK=ez4AFOYQ}30T)@F3L!PE#SyiNUZ7-*1WEzHnL%Lip z-E?iTw!XBUV!X`D^UKnkHSrjRkg`_+cuo8>bJ&vI#)5EncZXnsFvfU)*uQ`OUVHuc z@uQhdX&A?X0YPoKoK$<4Ra?iwN(uqO5E21uR|T~S7-I|xD6j>!76=(I60!#rb~9HB z7#K5$6akq4-B4Xs&;&Gsc}@ucV#pk$1J%~+Wd(q{@5baIMcVo(n;1YK^EOvjgl?>u zfK@ghSG}|vDI(bM(~F9zO7H#Dk9Y6hxhtSCQ;HZk4PzOGgwcpzO<7F+h9F^^UxN%B zX!GgW@L@aPTGI6vH*Pz9<8K>W$gez9@wV&DZQG4_jSwAvRZte*hD&dO>91Jod;wm! zC*EL^-uTULbHji8#NUbjh!F7>V8XRFWJC8GNNsFrL#_>W-(uE3i@XYYvs6X%ThOYi zw~X(m(7#5D=3uWz__ZzTy0*1?b5%u#LdpyJ#*LD5$^z+Nh3nC(unWd6^ zOFlY4AYsW7yyNEI2>xNh%ip&@$sJ6cDr2}hi)}Q zV8)l1Cv%Renhd<;TkzuWuxz1({P|d`&qj_M8#DosrTCJ`Inu!RQ9NCZw zh|B;G1l3W%5ksN?F@;nx1o9O?P#p*(0&$2O0<)?a5F#-}3K3jEsyOJ}-JN4(2$7a`u;pWi~= zw}RIOC2ybg1!?-?!52VzlgT!s0dGj-jcsjJ6m~7X|4i$?h``@4uiv@;w;rwb86P(Y zdnL42{pn-yqwgUdS91DM5PoAq+JaFnL|nx zSlw6A`@7K%g?ev_h)lGNcuA>@qc<@F7XTtKvGe)3;914Nq^}o-Q0|huFPDpHWzRN@ z(t7oZZXlFH>NGgAaxGRoX(*I3hG?x{3}7e^4sv&Qr>mO6)2Guf`?S|Xm|_PNP!k6P z1rupG$4I!I=L;rXTAeR(K*1IW4!fPZn)i~%=el;OTY`)jQcBBGZ!H;t%e*Y+$RWAo zJg*NAn1|v0x8F2frB#Q1c(`kAMOQ!}9o&5^!!QiPG$AvAKRiA@KHj%JQzX^yuDkuX zzuV30Wvdemsl>2SU?NiW+FMM+IPByuIs{XWc`9Xy;*N7lDd#*4!!!)zI5N6PPbme) zl46W85U%U0s{8%^Fy15L+Ag9sc2t0F?%Elg&;h_ztczCdwShuPdv_EObvGa|b|5t) z4Q^n7F~$@}h#4`eg6YZ394Rv;;=mXKMMAwzdN&V16gd({AX{I=tSue6JCZuZRG4Yt z9G#_g0H*+`-WTa4lEV445CS7JMR$oYj#CMND}Y)j6Qr%4QP#e48aG;c+m&1RW>-xR zudv|Ptpf1+KL>|v0}yPjzOA`Cm_-hV3;_COzAC+O^7x=c(uBPxBX1@LfuIUBb+Npl?PD6}@ z<`T9EkFS2yPRI3vV9d6KIWbWzAx2)7%ZCrYEXxvO*zIYCmd+)ZCV1_Q(Y$xH zdIk+lL*~*!Ft~#g5pG$(!g-9#EM4k8kgM9LmlFiu=beSu`x%FrnLmB{ zfQUI~jg{$Ia~I-bhKMNz5f7Adj_31HM3!YCA_L$QvA2*DG9&Zu?$D%%kWwll(p8w_ zIF6hC-TPv?l_fheQsl%55o1o98Lz53V~i09Rdre9n6TIN;c-8XV+b6|9(moUI&pL) zGh5d-PX37&g%zClfb#2=Ed^{n* zdN-|?b6~pP?--%Bj)?cuA>@SGg*b%3DM5;GEneNw2@(CZZSk`VeY>e|oa+nZf;Sr0 z_FdXW2QVT!Y(DG`{zVLL_1cW#Eh)QQdWq!jj@xJog4b*UF>_Z(GYZLpHdvwHvs@ zj!K9aDT_$!0>G-hNj2|gVxVfho6Mj7>*N06^!(w&fBg$r$Cq56TPkCSKrwdHwXF#) z*@z2Vo=%@!MT&HsFU^65={^Lit*4yhP+FJs`J@hnq4(a@s#yxr9a?K(u8781UC;B3 z%;$Oe`P0YRB~N1rfp_s^{fRKDf{+Rhe>qa3JU)JN`gB~hJkO+PB|*vt+$$bU1EO19 zd|uYm@*V$ie!2|Ln$3?-mybXG>EZCrec4YB*_3h~9)b24yb7Y1u~hDt%OwYXR}Lfw z`S9$WRu7@TE_R@Fc&G>EO?%&`4lR&^wO8N7j{{)d&TP~Ngs+t?7@ehCd zhuwImDh>+}h{@^noKu{JGUO83B1J?@>_5DJmve4aer{`%*2J9>XY4C9ook2he)t>f za%n=u#4+Tg&QDKI1^%oC1Tn@EFpCJdaFB;@4{Gx=AD`zZpMM(CA;!W;Yd@HayVXue zK?v8&;^)&i6=qIj^3=N3bzQ|Rgz&UJw5Et?rXqE>!`9|MpFSQAhgZ3SVNnMLJ(!s- zr;pMarlgWMCE~2W$JO>@iNh{WB}@Yj$s;SHBXH~5R{;#67?2urB#vy3xb>i@H~@kZ zA`ysP$%>=9Z~a4VZ5x1;Ei$;8nk$D8keHCr5rN!ZshisRidF+|_?{cv(e(1j*{&B<3w^Z=$Z-4dpIvV>kT}1HTu%XW!%|1VV3srVAX8cX6 z6cHWV2??%#+rX*@0QhTACI=!q9glzc>6e#JpYG!5?#r^gJfFglwRb@aAxvc`rLY6I z_O`|tL!hXk_O-S7<+7;ix8Hw@fn!X}YzEBH+^1;*Zw8JO(m*jr0k5qAhnRCnky28| zwF$>y;JN~=2=3Fgn;v(rp`LoJweaxp@K915OAdhm9hrwA=NwPJoY2AD2{DIAF>@dQ z_~pY#X6{{HPA7!4-#_Lwp;HQ-WAG5TFb^3WiC`@8T#xVWACiKct29Yv2u-Uf#tg>e zIA$I)<%oI8bwBQY`Q@X#3kWj~V@X3s_EOUO_urh)=TcIPv6Qj}cj&m^@3*gw)YfZ5 zcV;eW0GK0(oO0#BNF@c_?Q+icI^_gYN;vv7O=TR#WoKN=3&g!G?^L^ZUJM6Nbbx?MBjb)ov98fBXSBUP;~cso*y3{MO$mD_OAsJ`vy(4+fnTW^(Q$`5dEtP1-u0mu$pTD&G z{a#f~tZFlGcMP0TOaTKx*|eHkm)Ed><1h@HHKx1foDp%e+l}Kr0c@b7-L{()U7-KZXwFm;PbUs+}{!tcR+ljJ74JzAgC(btdC#KS3hII?cXmz zpCj2;RWoXFB#9W!>Fj6QHgi#CGkU z&AzUu{`{iIw_m-znA`c^{uTfT(bb%Yx1sjn?x_Cx?Y};7g-xDUYfJB~xtVrx=>+KR zCVFWrGL?t>aV$*K*QNKq&dVlQ5>rW$sc^tL(sO=^YJ1gbKY43j7TADB_-1|xa-oC0}NvtcBN^I?EC$sF58W5+6~h*#l+fp z7>2s6V=jwWAaV{M#z<_CI83RWk-%igyGWcvNHI>+l!p>x-2SdjY8OEyM^3RAvw}-E z5n0!@*4D%>^TLdIw@*0+3f4_~EknvB4mp*?#4wg&8cK-Nw2Me&U?v1#*ZT7E^696K z0KjO>$iZWbhr?l<%IS3Ets2K%L}D}!SW0oIftfrykt<+~De<(6F#<8UZDXh?U_c0n z$itwjt{q&MHuHBwj3o{0x~g>mKne^_&S<3Sh#X_y^7UH*v%8^y|7j3vb2}}yquf%PC#hddvASR z7V3@AI|m?|Qi<;#?}@1PB`2;ji%YAkb-BO4znrTW0(4P*IlXX5yWOFhHd(#m*a$22 zwRKrNrUb#OlEY!Yrx2@}>cTlrG^+YI?jolg^NzH{Q$7wf&5Tf=#9El(9?jz7x25H?yz$|mf9EO}rh^jrO6hms zX}Qer?jK@=m`XFatZi8;00j)pF@UBJDKK+HAXk?r2p+i%B}_vqLmG0rJM0eAE~O+X zmRM4ZDo()jytKB)iE3TCGyn;a+`0<%-XpzQQMx*!i!=Z-m*sRs3^8bm(@+Wnrp)6M zI0#^GU1Tv;g%D!M%p6J}anpcC0h9=sBLlAM+Nzj&%z0BzL`=J&G$IO|%)GT%la$0XvY)qm`ON`aq3{+1y?vJ1@06nl`b}7OdvHa}&#wwsJ?jf@h1YcE z7nJ9V^;?wv0s*(vZNJ(6d8J(bIsf`4a>1AB%NvsZK61slB_u(&d&cJ;OaQo!cqqB&v^BFyU`|yq_f#G%s zI-k#7RyW!2$LTQTX-tx1WVMu2Xi|EwXkKgE?*jPIfOjLGFYAYwkKH|ugRoOT18>rk zMpVP@zVuxv>&vD0ewYqr+WqwDsV@r+Il<^`IhKearIbr-H4>nzaKeINtrQqiz!ZRk zo#vCdMTkT+40&roMZ_WHno|iu)`|{^nAE3K?sr~7no7Q###-x`i@T}nJkQK1Zibdi z+58(Q1}6rTR0di5%ge`?ms6gmoQE9hP&lQ6h(jI;jG1@CjtK7dsmg03a%YZ{x$ZZ~rpa-UZD87(xUyP(HxQBkMznd0T-)YTF|i_x&7I$3cX*Ol;F*U-EjSj^KTdSTavi(NkB8iZ9bEM zJE*%lsJo*pnw#22yM{b|`tvW#r&CeR#N&{`@a1?`0%8ixt*`R3TrL;w%~Uq6Aq3t+ z43*YQ%7K2cs`y)-Q6K_Oo>y@F$N;p-G>mh zTU`|)u?#+;v)it8M5I)7LRsMYRdDQr8_RLxw7JrSmi zWuV=Z)|!a`(WhxB<4{UUDYs_BdK8u*UFXu(#c)6UGWyvWJjwy$L zv;}MY>X!=ygowi8pbz7_Rt=n|ahG!z(^~u5*30~9+NEKaN-63L&Z?*?%~m2}0yc0r zbOItIM~>D_#H80A0D(hDDJ6)nwd{l#LW+@Nh}X)YZGtQG-n+ETI*^9~=!#~ljD#2f z98A#++`!aStedzs*KQhm5fKa_i9!ZO^4}IAZ9B^W=~oi@_7HzhdiveZ_(<8j5kojYg*tf`WiS90d6P; z?#RrJ*Di$Lpw(|-@%9$J2pivA*mk@VfitV<+m9UG$kf%xlxZt)2ec-_6fpGG6d~b|wXV9bedjDb{pt+m%>`?SY0 z5D^l9IrHXsf@b<|zi&<40>_NNW=80V=`4!dk5H{vvq-JEOZmweL z(*wtQNA1KzCGL4()i|0{!0FTJ#s4&c@6&P zRUyRB*4I~g>UY*}G5$5$?ajc=01=%SkPw*BWHZ}x19Nh-4XsSuS_ybJjG_AT>EkTE z8~5ZEhCJ;@jP!h5mvs&ZB}dFL=Y5yf+S=-BE{M)d95AJ{l?yWhAQ*s~m}oG{7|Bs; zL*f*23ZYq3Q&ZLUI^I#cZkeW_*p`aOWz3AeO55-D-+uo?&Uszu%aaVxA`D}Fa$~|3?;=F-BqP4C;%|JqcghJ)f5rYyNGG)9T9R$`~7|xQsgA6+N`U( z2iQ_(kyD5SNMfi)2I#!ZHBu}or#PDUy7Vfl(g0jUx&%N|?Gy+o)Fx#Jm*sLkABkbN z8w199S>}082g@<$m=XsHsH$7i?)3Z;hMd77W5AR`phU?dH*zEg+`!Im`|fjxH6j2x zY=8N;bm1*>z1@R1!TlBzUJ1>s9Rs~NI{<9s=eJG0RaXE&XszM3%I9) zc!D>f=oc&Crf``%AtQ4@bj2W-QwW%NFow3OiN+8)r=QlV@B|wfBMnD%_K(VFd$-# zOu*~9yjE$%*!sFG3xFl&?tH&LgqVR|PRopJNbYXESJnRb__!>!wU*OxcQ-xtzwAQN zE`+?UvwDae80qoh!In=y{q&QI{?GsO|LytZlbKEX>2lU}otL^0IwH=eb4j73#D}|O zK0iNU2qES)O;gAb5gBmly{#3^-@SYH<3Ie```f=&sI@Lt1d*pQ z5=Jrl@$%;w_e7*!Ty4r_m#2h$iR*e|zK5H=Uf%^t!L$+zx8q&JlVx7cXoQCpa^?O9^JO4jtnK7Ebq&_W2G_cp$Qmziz1 zA5{%EjC^|vc%u`2vA%!_e>0@ouX#y-lg9i4Q{D!rzlP_l0nX+oxPcTJ5y7T+zB$+1 zi=bcib^dDo7Q6t!v~Qh}K<^vBaY!jf27v2m?x@mwR|1@dF(3s%G$%zyVxWi|5NcgX zZJ<~<0u#DxYo;1#LPzb|psDx@i@KSaY%|qe4IIFUpgD`k<5*}@M*#vc2G}Z2I8bvf z4&yl9-yOOs1#GQ7ef$MoV~h_E4^tjPO0~Au`nh@GIPFFutPJjL$|aZ4cd{PVQbwYX z)3D#~mt{VlWEgXo;wEYH5^y5m#3`2?5zv&u8EK>5DvcR;yOEi5&dgj&Y3r)04~IKL z=)I%caCIfUT#oO*`KEh!cLH#F-7~6s|M(DcG6mv|+PBtP@11}Ghm!ME`Zg26y{$)6|&RJ%fFts?#(rm$NAdiuNjT z^UlmU$3P6=2*gNE3;>Qqi3!k*TurK|2;`7r41l}cZpbANlOnKzlZf~$6(E4^WJnlO z$}#2-=bv6V0Vp#Y1VmTUZR%}|p$vJOpyn%?+2KBwQrx}Px^g7E*0O~_9A35eu5b;e zOuLz(141Jtq5#Z}XyE8>;0#yv_>D62yOiSg__xv6Tk`aJJYGAl5fHZ&U2s=K2yj)e z-Ue$M1G+u7H+_8g=KAn++R|=pYx~tLV0?=mU%Z(wu5$b4feXH}P6xW`=^I(QLe^Vr zM4;dAtbPlWZ!bW5zv33Sf%-Q7rxTO4ZoQKTIhc99EOoh@&u4HaqPn!+bPBnQIp?U_ z)U?XF_Eo&&MitFf7mh>>2Cp{|jxof1et8aCB!F9&Dem^u;c$3)UIH@%yPBh$xf-B? zwoXJP=MV^)M78%`&F|lR16SuoK-}c&*803&hzOYAT5P->^Rr$yCchcQ1>%4N08N$L z8E}`gKi-EF%UI@h$!!HO*M7I(-yI$@C!HIm{OQB<yoH*JT!w$f1<7f7}HQrIe83wBLzH zSKZwm&gWxmtB4FdGIQ_sbUOX;hwo)+DuR3sEgaAx4ym=ataV#{WI20)06K;hztza*P5rao4$Q`(7H-**c-Ga zu_WtOO}z`M0ciEryOmVjont^?H|{F^e7el*NnT=%M7ZtQoFfsLORbFv5VMLAaSR1Z z000;xGq=mKv>G9#5Oa)U9>t7+!BD%w*1)3R;)D%4dKXk+#1JCK__`n6$=%%WS_AN_ zag4v&k9I2*z5c}CIE>Pp^}7~T+lU*yDq^pk>sE1oTfO(U*aUAT_q@fR+fQzi*6o_! zR)^iF+3&uq+f(3LpM|gfhjv3QGy8qy@_!L>eg1cYX;gPo?_FMfAz~m^u`Z@9xV!XT z>*aJ7krV?*t^I;chLC|s-H6$|H<8vik5D&o1n>^PL<1BW!m&;sxFD13MHK=crg)s9tmdN2^Cba_7W)K_*a^%|Ad6_TsdYR|b>E(EQ zX?=~6=kw)m8V~#F{o~_)oOaXj{^9ZA{(g7ByWKuUzKB)rrrKMpwLd*QF=Zl7u^=c> z7|J+~yO;U_0M_Mld^x9-VvN#zUDy47Us7moMKbN;uN=)1Cqs8vFxv_Pm?$uOeEM+k ziKfBbW$EkE*G229{ys=^7e%v()Lp%+Hj!!}q#@;y1~*>Uwyd?UHH;VnUi}WQ0g%1^ zWS-Yr`vwQ4lwlYyu!3VZliDjabYvhE^Vd>QbVgNE5v(1UIA>!-HNfCW$czCHQLkmF zaIJ!L`f{w<@9xJh)-Acm*AQ}(fHo+6(^Ipd&9(M`D7?j*8>@nANA%mp*Ev0J!RrgO z`r<-w54K3~ci+tKVhy~uaq6#;>w4s`p7#6F+t(^!$<6TP%_glwbnk* zeOb-SDY(0W2f~md5p`1wXy9VKX%p?_PUr+?9epu(@3N#AoUr%arGEeIcOp`2m0H)j zn)tdlX6EtH)l@q>IuizB4otwJh-j|0w=T`xF>nas<>^dBDJ5izIi!>vlo-y(<5-4i zH<;N>*n8h*a4EZWakIz?0|S7x-damB(q>kdBjq@PceLKR1=;(17wdzWeU` zgfU?9RR&JZftjbci!s(^EqTBcQqH35SO#$_rQ~tAdw3us;ut8FqNGj`87#$NmoOq_ z<{_mq6-2B(@5c!la?Z6@BD#Ng04B}}>gotFr&0<4bnCtMQp&g+h{#oY@4*8QkZB@L znO|0M1P5(fkB&U+sg#j0Ue5LD>7_Ou%Cy_<)~6#Ah_rb*n~j9H-TChK2N_FitMnC6+)Y%?t&Ew2 zn=(^DhzP)pDbd6IJu|G!GN0!8QVDa3|$XK^agkK}zZzIsxGrtC0TLsBys`Zw0sr~cF^{WfpZV-T~B5K>s z5t&pv01$&VT`s5j<>hidw{;a&>#Nk2h^A>6$B|>`+Sk^aR&i4|F#*#+5Qu@W>(<0w z0bEQ0Y;9FcyGm%8-VX-JKjXX$T?b{IQ+Y#G5FZt67L(P7WpIu0k$AjD$!L#u6j9 z(*TdSX0b#C?X4rCyN+eQeJ3M_ zz=5x!VhDkPhCl|;dvnnLA94TvBiWXu31TLWxWe5-qda=HIM2ny_y zSa4^CGu_oa)m`;es!;A%9A#$w;YLJ8=6hM9Q9FB(NVwm)epQ5hZsN1ghOvt=M&G@B z`SR7}%P(HNx;X52T|abP41vH}(|Mjn#CuNxN-5F#u{&p#T@%2lGlQ^^nKj? zm;HXCrJj~G zx06hL-?O22KE^)!;DMO@{WwA1vhRi2mssUou?iWeC+p_8Qe=ne-w${PuclI&(EGryLg&_e|jmOxPSD7!U|82 z>k~5SbUGcc zZ}L1zt;jAdb4zJ(VchTH(5q@Ia$Kg0BBq8arU(Yi>%5%qj+2*?a$OUO4!+-a!`StK-Ex|Xv}#oxxGqp&^c)=bK2Y$(#j{tt zv9j~SjxfCV>I+>efZq4h8ZdwZjL7Ic0Hl_u|`dD zJnRv%)Y4LRU3A2%+I4Z0x>YEzzWn0)`X&y2sHMzvZP`bsVC)?;m!h@S-m`aZ9EKRZ znGv9f0+|o4)@mRkwI7DBV!w$3J-@s>><;^Jbi`X(45Ss2XBU_AJZqC~+#&bJ-2;9}mts6^P#7cR(Nj3LtfX5vA6y<^}*NNa#R7 zWa~Y)yPJ1IA3|`B7z0azWuBHf^Oxrg%t)R*0k&2_g_+qqW)D?UZKWxoR+W-l7%~zN zkw{f-0K~)q(3%oa@R6cpjsT0QG6nDY7I@uzlPgTMYWGfmcOu`{_xS~f}aIm4_iJ81V4hdhlteSll1;aQPo3! z>ggdqhM13%_m8#6N6h-)2NRXgI~JOUs#XG_4CUxGcl9R z>Dsg+kj!lR3sBWJ#Dk&=m;w+25|MXK{q|=xn@K6T zl&%Z0>!ejrOpdW)YfY<)hxW@zrVSc+ zbVC=%&JWQA&q$kVb2uCpz*HMKHy-v$=in?LL1_$VrXY%H-Pjc=Ip@B0z6+|l&4@i5 zoOdzCj*N@sT&o$F0Rr?PDneUl+eUCQ1H;CN**veav$G~LdhbI(tuj~mAaCHr+V*GW zz3=MV+L?P$L&5` zH8X_!mP!02-WL39%EbfWvr7ex8-hk-rXE;WD~vkR1e1|;DOJWx;pT98RZU{GbKM=asWE}Tu_Wk$Y zIS;ihZ%<*(YgN6yUBCPO-SH#@L#ZZtuC=CG6<`~}CGV!z%2!`~VNwa9559}sckwXp z_PYVbFJ4?ebJkBk-o~MOc6ks7cX`%BzRIQV#~Ns$c=d-j{dh48IJ#29%a>=~4Ts_R zE{^zlv% z_1EjX-Z)cTotbJ`Qk#fQ*5c3ua=!kt$)c#Ie87t~P# zElm5a!+xlQ(i*Z?A_FsGFl+{!as&{XwkF_1d;kZu`;SAw ztB(dN`jKn)0VpW~0z3HS(O!SuW%vuJJbOaI_!A$0h>btlVo!2}e+0QcS=*o*N9>3V z#9B2(Gs!uZTGHL!&Gq%|?QNMR2j~b}LD2?`-h1}XnQrYvt<`nSspODDDMhmG zcDqPUtA-HH&JTxSB;fgYQUzor1VB`Ts-j}YX)_u~TI6tcK1~Z$ z8h5+U#awGM-R*XO6+v5*T++JEDJg>UOzgO}!W_K!sto|T;A+bNu+f!cjFxkZJ{_}} z0@CJ5gb;`wa133nz=w++u#zL^9590gFgC>4FSAEy;mj)eFwlsbj_*Od7J9Oz-HD;10c^K z#L%jssX%R#0YIPvikX#C1fW$FB_eM?&Xe~(M3-7Ar4ZhaSyRf$Te;U<2&4oRNbdShI50jxk?f!vZ z>YMp9FYy<*f1omKmxYL*0L{mg)K7xePmcfd|Mi!4_#@Y|MHWPaWFT10v^7wxxky^p zWvQDssaUI-z;ZGGaKW=DBP^|!R#kPK)|85vmDV;%TtjYRpi1ZvS)_W;p^IvEJl+A2 znzE?_045|7gl3wXcp?}25CT||77Rqy9S(;zFKg0ySvfSbs7l1{^2KuzQ@KO((_F6Z z=4F|hvb0PMN%_k*jDA@zr0t%{CzhqK`O)7??QhW*YYsbxzI z(Nr6lFuM?3HJPT#jeCc%uBl1!o?9whvw=)9y6DNVQ5R$M z-kWzutPi=Z&6>!b#$cO|^2CZC)9rtX4)vIKdYWE;+zt=%<5Ro-ui|q(py_rA008um z!@?&)>AgP0N8t6j7k_g5=a$}2X%iDrsZxqb1#MsgqK;r3N30ENopVm5thH^abaRwS zUh{G{ol?%s4CEXWIP#?`f^(6`gEbRrYKUw=$7yQ0O{dd1>_r5by;SzDG|3w~89Hhyd(82XVpsorfeX$~Wb_2S}U$}0m# zAF@=>&Q!Y)EazHEt<_W|mCd$54FD*1Oj~<9sR0~yW3AQ9cvE&n453pi(RT(`a}`NN9>&Lj^h2)#A_f7GTo+^4b>2rr z9EN^P^Rg5*+Zv^-N?^|bct79OF)3qx>Sk`c}+-sJD;jF1LWY5s2Ly;GdC4% zwj`~^w5_eKOx9F>eESyAo?Si-@$iQ~MaH65GfWiCIy`C&i4{OR5G)iK@N7T38j zJWF|68!A9EhvXYt6^*7wNQCU}S*g-;YYmyIG)v|P6cEwXX7iNN?J?K78xPN)Ki{7n zxQ`5s>{K!k3PD4$0N6QIsa)RQmT@@i`@W)~g6n!SUD7R6Q(anJ)heMOsc9)0y7Sne zINs_R>?lO?w&q#1B7#RmCQz%jrj%@2 z2z}p&Wy$N(=C!0EZ{Oei^!l{|Hq~OXr1f?>9j8;yxf;Y6#WbabygTL6b)9OPbME>` z215*0+Sm`l5jb!K$sn*pz*=&xZDV5ub_k#Xpio6~ZLL%@8{^>2yGU+uS;~^;;M`&8 z#9C{u<%|YRCFhb`RRjV=15;D0Y7OqyO`3D0&Z7)*9h>wFL2W&_$&@Q z`doi9TD;F{Jq&OD1i7Ae5N`SIZ3YMYSM#~#DM4d}x)9o$LD&jZQo_o? zF@v%Bp=0LZY%gND$@}Z8_wTN&%HbAkweNrW>DzDiOR4BPjJ4k%#$EU0>+fz(*Ji@r zjpNAVQ#+lXUp_P{I_F8uFr(!>gdnwQ6EQ?$1I$HFQ_4$2o_>1sX20KeyTOlrINON< zARCE-G!%2d9f6TLT~qa)Dscl~Fk)7=Tx;-6TWMN}q3b*&Nkb4-Gh(PUg8+c$yiW5; zr1kqzO~!FN-d^pyI4?El69+2;e+>bTN4EYq3ogQYu70jKlu! z_Qz??*Vo6K^Y4HE`!%Iz3Z9+!hC1an=Uhc{BhhALP1%adUGi&1w72iCcSFZW`*BDq z?T67hw;RSUcb6Lz*t2Wg2#|<6A7hM61V&X=3`^5_O|{nZZ_oQ^=7~DjN+U-=CR$r7 zW&)s18X$LFAaO<7*j=Vt9w+rR5h8NVrIGuTp@N7S68h*oZ+aC;g-a>5 ztjj!y5WrI@rAk2}DPlwfCZ+;n&J0XUp;mCrCMsyy5FG_V$Kcr<0pY!%GXQKfPm%i& z>8YhZynX6(ZJx#Ftnx99^)#6U4}yR{M=pNwNSr-!g&u(b{(3%_e4OV3y3cbxroG_t z_O^P%XYyQ6*&lzzPkcnN&*t4!wKgf435iivT3XU{(pqk=uHU}-NplUvTH2P&Pf(31 zNYjc_UgvcYh0qVnvZOV;)|?Af%ptW3IZHt6x?#858`6@NRH~>zEx^Qq**O7A>yla( zK{nr{idhqI3?Y(t&QP+nZs-A^sodS2mSr*G+nXaZfB*V>fbiWPuKWFq>*MX+oUiVV z1<*VTbKLKV6i~Ze?7A+D{)>zARMt`oI_`$(z2BOU=Xoxvwo;^O&83!Ftd-n64Y}y+ zx7UXM6vqDi)r*~7qB4iT0^k@KFp%GQV&#Dya#cyK?uMSw)n%!zu_q#IDz#>_8X^Vn zC2Or#t#q3#Z7#%EYdziFYT)R@RF}((i`PH=;e6~)x3}xMuI;$*o(Z7y-VXz-L*G>- z(~7|X05Sk!RcTE{1Wdf^ODU5sZ(hH@xx0Py=1nc~{ZDUw45=1D^_@qe+;Wvh3?`wf zk$1s&Vy%=?YxCaS9p}7c({}4Q&C9MIoTJ?^;ujX3AIJW%-$x%jJI1XK-MUXtT{(2^>n)H0Yh+ATC0Yru^-4(mXuR#Rn5Q%%%F+^NK;eXc(&{) z3KF(vV1U?Eo2t9--L%$3s{nustR*k&62iF|tm_I4h{$F{41k+lA8ndq4Vnq7G!+yR zKz0-yhZt)N&Utb}{(GkC-={a6$AFY>vb!q&DPk?RrK?t{vQo|*p|KG*#v zKIuQ&s*U>e?c;6WFF>b#%>H=#(??O(PeSX@3G!GjIj^l&bO`8FYEH{M9kt~7c$b!y z0g)XLHdSl3)K$SkiOpb5Ddmcef$8h7zL}O)RmBk3tg5xjGUry7moHx;yET`2O~&lJ z2j*0ZXN-(ZHMKgei*FT~YD$PkXwI>cS7WtSQ<~;ujPb>@m+QLbRG58}gcJ>{W5*GO zeHVJiYpp05D}(6S zcU?1Y=!~&7MaO8Opsls0O_H`!t%-Bp_4QrK`THN<9OvoH+uOeH!S&pCB9_XUG#gp1 zdBc%ie4(n&b2knmby_9^>3qN)fa!d?JIxCzl|IE7q1sRI);V{6I74K2?mW9~=2mM} z)zpm0g{~tPFgU;DDNnhS3>5>RXGN_wuS?=#Byz2_d0Bwn;9OC=oliaQO;NxclA@8a zp&D|nm6!m z=!<<4eEt!Mu3tXRedEr3>!jhs?AwQfKGq|)C6Bqk$A|VY6GUxo$+jp(CF7{?tV|(@D91%~aQ_WjsOHA4frzP|HgQp9;ZE_r%;^LEMj?2BhF zzx?8G?1z2#^3|C^eIe%*orPP|ZyUu&BOsw5pdckEDBX>eN+^;tm~@Sj99<%e_<=}| zMnYnQlrU1H1V)bzC5F`KQG@sU{sp^sUEA|K_kGU!oG_`pl7A@R?h&9?4bxUrGP zieeY~_J-e7M_W;%c?!MU6mW0KeuxyAThO2y!s0vx&v(v`%wGpfV|0Cj%aICOZh zK5B)nD@0+`F+8VCu+O`-1vR@4$TB(_v+_nh@oLkJx_Vi3m8ILQ$qdU&i?V`=toiH0j=oO@fhXwP-Sd<~X+gC>%jB@ns>pDu zooh39ps2X0U2jMeM--QCQ`pc>|Uq&kWvTzq%JIFCXp`D~HCGQY+sz zydJ2~sOKKn!Wa*!7nRg!zkY#rG$4In!jEh&+5)`Wv`RdR_LBCPf z1ZxNbQJP$qdSLG~lhfg61r^)->yZOaEzS-hECRxzfdTFt?mzO!k%u zf~hly*d8tvowP6CmS0Mi8q{tM32i9y8FXfJnm}9~9A95-ou2w_j07zo1y?^s#Fk~m z@NheRE7UK(XJ|X8X}apxMEzPtu?#u^{XK+E79zF>y;LtBEiXwKLdZc~RP2e(jy?<) z&MfSGfq-l!8AU|;=P=nxHClrW$4VKjv5hGc5;w=Ey2!)R#i>OF9{FXuwrhz=l<`VPir+1!O>~C*PGVyo8|*&Y znXL|hEv-Ii98POSWYYwTVBxcY;NRkkYIRpETrDS5fwfg zXaQC1@Ous5@gHLeGIHLEV)RKFv2fy-K2D;XQXl!gcwzX!R)jb!%*2V+<;Rh~At66X^Y>p-p32Ic*S6{X+K$Wke@(sEug17||sf&?9LVto_O*l2ma@S%0 z$Qn8d&v>15<`D)D=>(d1O|aPqkIka>RW~>@v+y{=qBX!@+~oNd{(6=ffB>b-nW%ku ztwl|r?G<>Vvj+3n4CA2t)z;{LIBSGlTmuMpZ0PmPb#zCm zimZv*YtY5T$$XRh(z$3ZyPT&^B=S}Ae_aLnz4_m!plawej8vHl%3c)ORa4IFrO@gU|6s?vH}My0|@6(xu%WpNe;MK8VK9SML4U|fspU2%-#rIX&*81 zeW<~nnVY!K$r(}C@MWx%A*agCuy!{HDxM1e)m^ zU0dKZzXxi_O!-~=#<{FP;L9BoQZ8(vJT;cTnB2K(IS_~SdHaag4zw@cPOw9yc}i z$C{Z!dZ*>zqEA(3l4q4ADS7wa)rZro3X5Rby`RfB-Jx3if@R*0(MQ43>fSGP3cOZj zHnQIgl3q8>$bJT^>kFDlOPk=96mR}&JpJ3`Z%mBPfY$0i4?`wNR=i3|;RYru7yHuH z+O%Xf$aHEKi6{$IK0c|R>$`(Ksp^X;=u8h$gXB}lblf{icJE=RS@>u6`$XJwI1~#~ zZe)m@j}NS`^nRIbdGUIGB=kpHdz)+;pA^;~{V803qRiNm8Rig6A3a0&@NEnegO;w_ z=4kEw$>N{U+oF?O)9sj8QRrQ#PtGLL-G7UvG@dOm?NOSky+`Dxohh)`g zyRou)G9JG9XX8Q+JCtLPR9sn#GQ*ii7pXMAa{c?!q>n$n*!WIgJwkjxZ5}Z%hh3X; zA2V`Gv!ylF!35@%<|5@a#~PbFjL4_;W-O3?iK$TP}H*o48kBKPjS1 zHd&};%wPvNUDw`av2g8cuBuL2kcVFq9n^@@XlT=k|= zfea|E!~Q!5Dfuik-d(=geC6h>R+!j|q}xpl-C4;qAK@@f#9ti)NukWn3`P}g^n^rl zJ!o_G{MuVOC!sW(w-iHTO=HEvu-K9|;Z#eG3c1WyQ@F-p$K41nOJ2a*CzbDJWUcKG7x4r*yIhydz zNL%atQEqO-IQ~+6B5UOZHo||W@4%$v+@7U-TY7nTaJ)~{WIOoS&Aje-b+$V6L>g7M z7cH&a?tjUSzPrB<>max?n=@N;T+}Wzj2>*9o$dyDZON7XS5`pXTD3iV0s((v`s9cf z1%0g*2p4_FWSsx`su~R;&ziXZV4;7yr8=iB78lXKw&!jK=dxb)+wjz0rH8yJvRx{9HD$FP7b3UGwzhVvm4=Y0m5hYUE&f z>3h3P(G4+N{t0dLQ``ox^(&lYm6frSr6*F@?`dRH41KAbM5G0@Sn&Tv8@6 zGWM5>3ZJ#Vpe%LPJZz)*0$fq@gslr|^i9RWhHI$I3qmSVEtDCMxIyo9fbGk}a z8%-IkAPfb)%OZ#M4(a=B`bJ-8)9AUL#PLyWXs7x4`b+r%g)T56lHl|8^qbS=Ycm_5 zS!nu(x0ub(VXMVY>NI*A_okr}f$)Utdbx@jacuTUE6e=s{voey{%r6|ngV<2_j~)l z!o$MjbG&4+Sy@?5i2BT1A|eKs)%h4hh8`!ce8&wHg@_k=Wloujb#mCu(V=G0X2w-X zS2Y!9n7U59CP;-GP4>7UGl0WwsSQV8mMI>~MMI!{62PG^NN4A1Xar1ZWoZE|D0#?o z)sZT(muZBL2Gq6i-6vvyj_AdnL9Dh`_be^tUJ1}=)gHI{dZ2lQj+wc-{ruu^Rvc?y zRg3``EsNhLQ=TJJBM2|UAk&bW>*4J5CY)Q9yKURTd>z4R-#G2xBl|7+X`jUs^--M$ zB+5$RVWI(IxT|LQ_F_JA!?40g%$CD8>qzTSXN93o6CsnC9+@@{)`;*QQ@6Ees>#fx zYJ>{Vg31|hZ^cyJ;=l75o{};gW=3*HgTzyZVQ5O=vo;40Lo84jEpGXgmm<~@qXR{5 zw^+lsvZMQhZ~9A@<5S=)`?ddN?AWL^D*y2cwoc6JPPWk5QVza!ay>}=IE}3$dFW}D zn11-DN0DRkp)CFWTTvpA(lLl|fC^R^%fopQu1YQzpA_dCtEy~7^PAPI85yZD2gWv+ zcY-T6^h=x=YF8+Jx$4<@t@R}VsGiP(ZNPvt2Y#mxr7U;Ude@V&yRWyUb z+TlsLZsjegNGCmtU`6rExk{HY$!wxE4Nb;mq3xsjFuGR$@veLIGL8JPTPO^b#=SW# z3zNw<(&jJr|7}C@rGU>|-Lqh#^=dKnnxgA?gD|hJm4smqoxzvuG6Y$sO6eLTx5&?r z$q`vi*i#WNet#xyGTk?Cpi4#~`$A_WDSkEu!BQpzcsN!2URoA!)IV!5ezAaOXC=~0 zCI2O=DY31}@^|!8E@*0$ILpSKLp3@9wz}UE?7p|8XV@wF^(YH8X8dePYh$}*X<l;d)cY_3-(+o%8M=t_?+Dk9zyO|wlmTEdIn{FnE zjOCY|u4SI$<1PN>IOahE!3J~omT-&p=D?2$dplS9F)Mp1-y`YsAPOUcRI#JBim&JM zk==d=(bIzp!OZR^x38k}Le8!i6Xyo%s)eemt5dgjcRe6p6$p&)|f5y+dzrv>8F|4NH ztnWp1SJm>7rX{gvJ)bQ0T;img@T65A`ntRv{law^MQ^i%$_;aNChOdJziVs2uQz4< z)=XSwz!c%yojgQOli7whcm6H=2KE9nao`6aVf$7YvY06H)10~X=&%l#4%ID$o20sJiZ9?^e0RHsvLy(DU*KTK^ZcWhVqle>44B;<7`Twh84P)%|6__DlQDLVzMC5XH2WQiS*7ZHzN zpi!H$`T7$j<$RqU(QvbklY(}5OrpKi z`9`9J#LgJH!$}UmcafBi^R3h*k{!D}>WSIU+HRf(QQp}-P~Oigh?`9=sCnaFVi``? z=ck_{sbZ_niNZY!)75We{mT63i@q7uF=Qp@t$3R4?JhPiD<@s+j`@x&Pr|=Y!oN#t zH5lh@#3V{2FRi$E0~+tD3p?QpzTDb&r?c29KRQW@64>>%1W)^L(2fXO4E4Rr!iyqU zr1H18z14Pp(zE_6J>i)3FHSkk@d{b~!$PT3vq(sqLKm2fo!m@JUztfS>#g33mDY?MVOCI`%+Z+N@{_Xc$YK=qglGC)z*O$INvM9U0+Wzr@xll`?%Z} z>%abW&v`UdaC=7))+R037u0~c`{`n*=X?~uq>;FKa5}@8v)hZf_u<3PV`|IDVMc|| zs|#D1r%C-oZc79|84hf{>haH@)4$EP*XT+P-0+ppCfwZ&z6-iJ3w}JckVbpJ-UK83tykZI#2pX^Q5uz3y%(7z$LmbiH*`1QnM3-g^WKF%Q{Ln z7!xH82l3*F4LGV^yW`t@gO60cMdg!5zV2Q0$enHg9c%)cAM|LB{?qduVsVe@PdSMs ze?W~@4Y*7Ch}be$LZ@Rubpkb=EQffY$n%~TVzNIQ=lKR-tA~Yk(sA9(~f1{5w0pYJ5u zc!Nm8)DskCy|aoytC-~|`*QeDZYINCT_m2z$LemF5s3O7`4E8K>( z%FGKNnPciXdeH|gYNCtyoJ1nM>68SRftDTOjU!0qWiM@J8BogLtJKRvRD==G^-Qz- zJ^1~lAg?f~6{9} zSGHPhnXu}9^QR{+eQ#=|a~ezpDBBJ9R9X6JIigTmtXkw@^gqwfdA9=c#7q|6rSPUU zRWEZUlunf06D@Q?fc%F~2$%SdC0w>w!2?jkBs8|E6yWs)9GzZ#kZBD%+1fe$dH${_ zrXPSzP!_Dqjd(%oUVPDSkTM85Soh-YbAjW**X5R+OBx`0QN8E+D`mJ#mMF>LVn> zsd~F1I=8<|@OZ$H9tPEZ#9rXy%~|c>Lc0rLiJ!{zh72Cs56eO(b&n~9FPasmM?QWp z{T%+ILB4CXLzkW%6mvFkE7$fHN68;WjVe0TL*>9bp?oUAN(vz}f5gi!xrd2F7TBybnZh7cKJqar1U~$6q{>H1Uo{x95fQi1 zRfk^_4D&)G_{#=1x?7O#m3)xR82CVVgcOgyx`ZP)=f%61RPc9N1C}G`OVoMWizKm* zR+$IAK>6A~*gqM<<;}>=VODkfp>=UefE?K4^J@fV@x$|c$ctJp+ex2?G^)RR+Dwyc zN!0Ho`L_t(Vxs>O-$r8iswJ&;TNQmyw|wL0+o(Y;9^|& zjQq$J3*0C34XjC;u<7MI(ZX%*3bh^e9{z_5y-egU7WUlh>ynk;0g4L!X%vs>Z5V5T zNOeIY=D+SE>&gWk`h%|GPC**YEwL9S)q3*$fzL9mJ$(G4>8}zZ! z+0%l5RJFRA!@HMoh5m1kb4TO_Ah)=lNpKx*+#%7Ud4C?F$522*{9LjQO*da8YeXh# zH2awWWcYx=pAT2)Mcbvby{^&P@@KP1aALi#8h=R$y~f{ow6qiQZrpE#Z_rgJj>pdhgEt~8%yPQ3r=abaV zmKB@*ESq^UEJ+hasYiHs)z({z)B0}i?xm|qV}@izltVlsZ$HQcB$6gUjIcTLIN#i}jSarY z`wG!Biwg@n6MsK^YnCujVT-b2i2{nItCC!lck=QQOJ?h9ND^&zTaJ!oXm(WjdA&{5 zX{?%*<3yKqA9J>&)IyP@mx7k5EZwy~XU?+*Ig5oCZyJ zrXrVk;3G~k5|amTmaOn3ehhbhkE}j2477&0SF6>@<(>N+G;mjbC7P3X3^dn3&iBNAri|6_zH^4wPN_>Qzr@VQ_;upP!96;H$HOhB>@zN{ zU{8s$H;aY4KN6G0luYwT#!I7v1G?VOT{DjDIAni+@p47Kgkgc$( zlPx>OdLLcZQ|qPnVaJsHE*Vyxjr+b#^unh-`e?5B%G@N?`pdH)WVt9OTbk3 zr5ptTN zlVdf)AG-7o4z?d9?1iGuokACQGSo~{3v8ttEcJw3b%?;y1$AvU()Ds`=JckK8A&eB zW%u>iELbmCj(H|KovrPBJ06}*6#LHUcYIlj_0(Mwk_G81(Yy44JZhEDVbbhUAzz5} z!uy*^S*4x6?hq3wuPRo?FBGPL$YhuXXHg}Q02?q~RaKBPReb*rlLq6PJ|`tK5ZG`6 z!&L{1x1$tcSo?$?_J&vPwq)e{ z?6Ag!xx3?jZTNsetBW3UnzW7;b~#y-ZVNpuSpO;`Lu!Fj!nf4Jxp!Q&mrA~pG-gGf zUa%MH#YXB)^SkIgNYv^I%Fo9~a`@GM!0!%5*xkixmvxKy;eK8Z0p8*x2|5TzHZ__8 ztFe|*_jpg0raL4{YC~2ekekyt@>xrO6mGWJa;s8*P*-c?eJ1M=6v;_%P61aj_Tmzw z2Bqc`NkjjZ0mVoSeB6=yGYT$wPy@n0JC4#VjtNZGh)<%(8%yV>>!FX7tvIG6*yV2g z@=Hzk1@G4jaxQG_w5 z3;Mkku(12jCgAXbp;dhtJRxAcsS3!68;bP;Wl#fpM$yNdSzChdrhO8AOjVNAc!YIs zwmx?J+Yb}!4PjO-eMVJ(7Mi#UI$ z1{0qz``Gr8n5vpy-@PFC(qe8^uA(wW=$0AO2^}4`-!oWUn)TKGEc)g+vF-7M&-*uv zn>3)0pq^w5+A|gabiuy1s@-<<);p)zYa5W9A@ETM;K@N`$%)DFcy; ziZe1MBg~IXbVy-2?5Hv}4aq@HjW($(12^Gjv0DtS{}M~s}Tv2mQ6(^b?xzEAT;@jKmlB?U-q ze&!Ib+z@@m91JMcZB5F1UcrZqE4K<4ai{)#eKwI#DK$6SV6AGn9{EB44tX9+=d?7J za+)pWm8ER`cvn3ZnP4W(x0}Nnd}b#cO$Yf2nUKjz#=V5`$9j z&vLvgBjv+zSFcBUih#`JWu6s5+$klb^yy^|4)|laU2S|gE6rQl*q(3#kJJSpY)u2B z4UQqY0yZ%#Bxn4SmMBA(6k#SVS#ytjmyQr3U8KM-#X($Gp#x5h_up*TZgZiVc^)xRud^5PSWL4%_VdiYzDyRf8ZI}6a zOq|QH0=a>}a0^(zMAIqWmoU(q;yd2IJnWLks$@}$xg?YF3Z9={xXHcVt}l`7duZT_ zvZ~e;)l@(Jm0!~E%2hur({Dt-lVLa7qotWAa?vgOjTi0-xpePnpaUqp?2i0j_l}E; ztHU0J^{j1va%pb}`af3Z;pQPqdp=|;kYHOsUEDm|*@*{f{qC;f7~xL-u4dn8uC@tw zCj3B2z#-{ge7J17QoBVQo;)Q^j&?A+c+uO#GDG20$n+4%k(~&Cf;29xjNiQd?!L^HD@j~0lGVYP z;m1#MGqZb#YIn}d;*PM$H$hc>|M6*Nxo{~K#kowxQqsx0a{d0u)ZN#A5<`7-<{Nn6 zEi`W|?SD1)wX~^BCge^WF=$+vGo7%m#2nNfus?8`ha-POKO6Xlyl@512BLrNEIDn} zl#anTNor#vS^86SJosx>`r4^^oHws(>L1HvQrW{bZLO1;tmH9wJQ!dD?VkPmhL@-=Rk4&!&L)EY&q8`20Qu~08;P?DR zb6fK~dvYAgKdEKGYB>0&*sCffg_n+T-sPAq)tF#~koN0y8Sk7}pq@;O z)N|0OEg%#>wH)$^9R6b_V&g*o&A8ZC&fPuLZE`U(i69Pf^dIsLe|jf|YV^ z9xf{-u1<~l>U_3Y%JlU_R^EFHi#D6p#nCsV#lYxj!0$iZv%&&cXy)gea62MbyVW;| zL|Ns&5SV|6%4hvAlNg`iWxQE{bI0H=8kxbGk6+6m=3Whsv{6U58g_Afe^1<$!@OrV zW>UQVXe;kcj?ICa@Vt%s#gBETVdY?F_9%2#vnTlJ2>5lvVLN8igobU?cUS{AMb4u zP<2OMkLGjc;Yq>obkGFK+16&(Ce)XN!a|iM- zkdrg52h-ZvqCV+*WJpHwqWKQ(Ftgihym>UAZWQbd#_}yMiuE7RyB|ER?4^!T6OU zssIC}=>oiZo6+?fDaLOehSSsBR--ItB>jMvwZ?IqiHyz9o5|O3m86`a942;memss( zPagsNd?X}9)~9SFbevUuJZ*UyPE`NLjb0lW**Wfd+k&GtG@9=fSCY}G zH@erl%yuj6zo-~{`wN!rJ7Ehbe!35b%ty_Xq31lIf`-?&Qez#KiUDzSO@snB5J=B# z0#mMH04m_)pO&>NzI+t!LGqJZmIJtJ0(_p>s>TyuibnLUG*i#CZePZ7$Nz)}+mzIIh0JC9H2eMyc%G<&Grs!{zyW?ut31y zbkHLg=SJd*l_6RH(vrkhp_m-y2O}->S+dxcs)3g|Kdd0b4~$>h?HVwvIwD0^XJ;Rd z&^fNv`|)wWS;^{CBy=T*}#V}aQH z!$a#(78uH-WBGQ@cAt_j#FfSK39^=7TpUb;?p0F113(IZ?iD?3Lo9a$MeXg+M$TVC9mw)+&^>sJRTJ`;g(a)$G*j(lr&#Zc9sjStIym=$Vif_ zK9@8a6jwk6g@2{!SvfxM(Jm8P6x<>1R7{ZB`nqic7dqQ&^V;(OAKBdbXQ5Sn)!cAs z91%*Hs^n+(5?@(W>bQS)QCA5caeaKTzMIH$_0J=>#VApIa^%u{{vc23YhMsX_~A`5 zb#F9x=RG=i+NHi*GNc;xfJSK;W;maH7$Ao{T7X*h&fwR|N}h3}_Bc&%sZQ?_;}ra{ zC=U)v`CIu2rpkJ)9b#BjTw7d9*Gs68I%QWheIwB~SA#I&Vhk9Q0(C-$Z?-h1=oZmC zEB^@DS9jzqp0qP9aT5Eu5)B@v5wwv`aFCZ*einV;>!#|Wd+3K*p9lhPJd>bdAaP`Z zY%7U`9j!5Zf3yCcL7a_pzGQ z2b%5T5 z#G8G5sC4nI+iYJ=(N4XsJ1CLkgY%=oID%i}gUkL}YP3fkdG4+77&e~5qjub-WurJL zESQ8*D`|e-CpVb|CG^mjEn6Y*^jV~ccXtR|3;k*LCfgL)fz9Ajq1S9?9O`um^1^Jso6DN>bWVwi==-+Q6rtcBdVu7pwcs^{9k)!2F$ ztF~L7(mn7bykhzrI(wnlPNK3#k!ANckTZ})#lzxgBKkeMLw}z9ny{Pkm0cvj^_asN zIx96b+Jn`DikMQl>_D_p)NGMF+V-re2*8}X^@Q=CW8`A$&P6j|uH)^fQ0 zlmK$dpQ$oFVQg&Cm8zXu^uEmivL5`1ztK4B)xF0_KQXiSf|@u^o18stEpO9Ic(Szz zT?5s>RjzF^JA__=4{SM2a{>qwH6zpSMT}vsRuFw-7F2sGm3?l8+RW!c*?YW_np8V% zVln-}7ekfBd}BP~FT>M#vofcIR#^TU0P{qr)d@L6UfTg3D{D_Qmv8x*6myp7?`@o) z|9NU_Tvwv|fYHfPVDj&e<&6Lkr^wyAfYWUOKS!zyY|q?Qipw1w96GM2mahN{_HwV= z)xT0{_s}aRxyAo##h)Z}GSG#mR#IP|*VS1NE-K;%kLqgYZ@1Z~d>0U$HiVv_zf!@! zs6DdbR5L1SkBiKGuJ3|;l26wT6EcU~z&rf~IU9ur)LC<+SfiiUr(w!diupt#o?^+^ zk#B{rQ48DO}$7@{C>;%bYsqaugRR&Fs8LRVU$@gMY@$717z2(;)^zYe!CG zANlXtT+E)9x`#9#U|BCH*teDc*%HQ^m4f;| z+u|Z0$kZHPiAq?ZePMp@OONRWV`qa8&&Kt2on7IY>cjVIj@UCi#WT|4Ta^Y~ytaTw z%53mPA|EB)x2ew- zbSO{N?Kr;}Jhzuc|FZ|basIT(wbL7XY#FJe@?^NO!JHbX`*uPE9YLmjx_V~%SjO@( zP`9j~w_`DSdV0k#jSL)-Ic~U+jL<6p=!Lspy}>DWgoYMV5m+fa^NkLvz<5+QkaY<< zSs$z;%xH`xfGQ`y47Lj)_hN6reNYyf9`gjr6;Q&-Z2Jc+Tcz0k(JbvVVNc zXhFLecF3b7QFCGl#fi5 zs>ZWj>@IiziKq85xG)a3#3{#g=I%FNtFB#;)tukGywPqQD!tWFD!MAXcV7SK!uzrt z_DNBJn2Eua?)kUE3{V(#?Sp78O%4)~NgrN{9T~l8Js-Bw@W#>6x+oSl@jLFM6x$F= zxb8++&D}5nV249h@p#UWCXt|wZ6WC;Qn<1?H zKN6k_De+aqiGy!cr~?(<(s_B2IT*0=iR@L6s^qh$dJ_9%mvnVljpfE{$W?iW)xuKB_yCGx~Wy08K8|1P|{+;!H09Q_1BlDne@T34@TVEu7n#iWA2A19t1HzWI3+-hBxHJ$|mfCFcdt z2Tx4;Uu?xOTLcrfuC7E*#~ALsb8e{5{*7qh|Ei`Mz`|n#F2;QEbcMUD6cH6@aI|fM zAkK2S)c1AzctARR@_d$&X@foM8tH**Oq1^d?R4xQ;{iT_lD`VJ4K;L9^64JtJSLg- zlZo#S`^H)Bn0EB<(DTDg+BsAfZJ~4{AcpZgR=Q(tDe;JYj9lO?;%-dhKg0|2smHpX zLeT*dV?PglFs$&uxP_A}!Y<$x{RA6DN zp@4S&nAn%&n!Sh|x3bq+xJ@E1r)e!xQDO69PJ@oSJ#Mi@RVoWPv;mKnk+E*rAu)RO z=nko-w+)vTO67;c}!t%@4Yr@ ztEgL*p(U6~=KR(`pXM81cg^yDR{ON&R$U3cZ4U+(fLu~SnHRfyp7Bd3(SJa=1+qb( z4;*mXuYzB^1*E^rFS8&f@NXYoW3YHe>3}@*S6oX1>r^aMk{$#xaSo$42$ z1}dPhQ-{=2({fm1-Cz2w0H!yU^oy2FbzrJ?n>&&=RYkA!yB4trh`DA=>Od)`o#|b6 zVqRTehokF%ZulCzLMf(;Gc(Ep&3UtvQd#3&I`-pDajq}vKP|=6iN7G-aeQ{Qb zl#p(oxA*#GLn4E$KO_XLpiy%a_BJm{cxBRzUun%G-t!(Po19Ad%0Vw#>Zx?<@9OhP zXXUo9Py(C2;wiQb$Wjf03lNTrj^BpwGm!Zj0T6W|$y5N*s({?C6i{-x?=d_?|L64d zHB;+$bS^^#Je;WDdBsGFgu>K#r=g06b=P^1mM`d{M$p_F@I9)U{G^URYv(i=+yy* z`Xdn|Nd^&TeT>hpI*Q(G0t|hXnK_`RH9V+AN&uX^FQ5H{4(oZ9=*PY!0o7m9>hn(p zj=%v_nfhAH1FrDQNemb*>sQ*aF4>T6CKTTG__Tofk}#lGYH9?*ZWms3avP3Z+B~`1!60~7h zqu0ZZnAKeW*HW+@fq)wV0=Ch2nO}tU=6B-1)@A4jnHwl=asnWj_)0X5CL*$B*;6vtx%3%N!O-iY_-lm_VR?8{=OFoPi5RP7Mt6cgIO0 z=A^18=#ye@;nXw?H0-%astUuXghJ4;@R&8DZ$MAG(_q7kE+tDM1Y6FUR@d#+kR9f&xS7jw|ny-wEP{9&jbzgVjWAg z#Q#Nyj`P#hWE{Fh*uBc_W*-$QM{zmaS#41o%g&&yEl_;M(l3wu`$HKQTx}xTrVZlM z;LC}!l9}hYMh$i-?(`K6E-o_RM|V*%_lm-pFUL==_Y0rM(hi;&VR_RG1RIw*Hh6Q= z4D6&OU&`XsEu=b}C#7c_wzgWHd8vLBBJ~|m;~vuKr7HjV*W6<}38L(~_xA+14A`NV zo7u?LcBc|+l*LJEJ`heNIHu>boFFDM5O3^dtrw7T3=O(L&D$tp51N;e+NP}!goK@u z-72RJN%g-lltk11l;?+6`gG{cJ1G222m5EpEVoavbpYm z@FMNyE$YANx4gmX_6(3|$ul=LeEG$f4dzNa1=K$HHr)5HrZgDbiCYqdpELoT<=rMrWtv_PTc&f^ z{7R|d`z5jU-CW-IEWXFzxCe}LXdux;3B?V@1w>&sv|Jgt_S-IFI{VkM=Qm5rpI9Rs z<`~~fy`?hJtNr4s%S1#8d1?nSX_ujRtI4Adn)5^sX~vKB?T8yZ^0dbw8Zl8z(_k~? zkwP6lc$udL&kz;fHlLUGAE7Eyw7R({=!Z-mramo*>pR1IKLt0GyW?_E+1*u9ZJL8N z+F>MDvEv7zSdQFeIcV+n!IlS&^zYf0HwX=W(f3(1a~^*OFf}rG`1GAkOmMFaz;%;| z9G-G-7|bgqgchnBD_Et+Bf<6a=a|O1B0E50s!H*c%h~Yn_aQgAY|rZi zi}UlBo_MFfE-h>5rM=mYR;C|Iw`jXKoe%Z#s(Zg&*(41_EJp(+!UgLm4U1zLB~HeY z!oPuC$;Lfl#QAonZmZh@xR*4zHGhGRBsk2R7e}M%^|iL;`ns#O3wbmHQpnFN|I%F# zS)`rMVQemL%9y%2x^4#oZ*#nARp@i?^CMe@YklfX^7nZPNp#px$mn1FqeDefcX_$h z9&w+x*P+!xX8c0E{U`|9KYacD*X9O#RkjN>ie;fCz%>kjQ{4?L{m&!WgktVO#E>aG zrWups%-|Xv-$n6ATM}YnK_pNJqOzQi^`N9Tx7-L^s($A>Zl27&1zb*PNiWr>3)1b0 zqu^A%OFc6kR7>h+{0_|6QXe2hO|Ht_BA;VeP}UlJ*|_~v1Smn?^NlvCeoxrSavwpX z=?}8HnZ4wUa!uszhc)KwD<{k|f|)fz-0BQTb3@|cDVU|!KvY#`w79t= zxwK+LB)S%_10>eq@$mQ@HR9CRa&F`|OG*e@w%FFbjPaGz%q2%?fSJ}{TxHyl0l%}= zMmrEcY`{$XOPSc6LcN;I+Z_?2Wp+<5EH~1s5^ZXCVxs44ZEv#@7c(S#$&D~9DE7&8 zbZ&ZgR5IpXT(7_4vRv@W`EOEhf&4K54T@g8?jX$n{|gA^6RKG{k4bU`bQ!3;bAFXHvEad+f#uCoxsyf{h3x;PgS%sy~-~Y8*bmv$i^=PN?9^9 z4n))Dh;x!X)z$tjh{UKs$&7d?g#t!IJ@4B0Vi5P3+!Jo~7fl+tj@?$9_F}F<+iqs7 zeJwXK%JCX{Qtp&F{LEV^(yHp8ck6n=1N2CEO^R*QTFxKir{rO-KS{fycg`(pboNjT z9Ni@p;@32kDvR%f?loV7{VO^!2cfx9H|Ioo_Q%TD4{k$@>XH8ql<+0IN^Nz})bK*H z7KZn*M0=!Esx)pG78fD(7&5c*%p=0LjwLv>$w5pLd;;ZBIzu`_obDDML6czWoyJ&C zXTb}4oA@47XPOuTXIW;(y^rLhpDRm-)>v7(!nvQ^`GAf7@)~I9l;5Pgz6&sbdeh;o{AiU3_h!oS{o`3Gi=qG8NaEukkgA7ab)Gup*8ksc1hYk-ZsQWV4ZoYl-VclYGkjh^S*r}Ye zNE3=bO7{MW?g5PbLFjFnQzwY$`(lCmUFTf5A4Kazj@o2kv#sw=j&qibh&V|GKiB;N zqVgEsQnMq&u^${2nRLMBvl5=!g#p7sC1=8M)qw5bZ}N@uU^{V(iTlm+0`_8C-<_lS zbPRhTTf562(sftvI=KF&b<4pPC5YzFHEd%>=b@Z&QW(%CAWfGAQwA* z4RtEXG6XXXA~%frz3XF`Jy9=&B-JaCve1Q%5oXHI zVVMdncda;jmGpA%!oE?WDfsRsIU&;OiMf)c#c``43vW4n_dZDZefN^hW*5r#vR6%@ zi$1FExEAw1m|c(iA4TUL&(#0N@wv}^Lz%vE8Hy5f%M`gS$^Blg<+ftE=Q`!il-zPB zB9~l-)Lb@qavhRPVMF9L*XELh-+t%M^XEB_b3X6)>-~Dap6%P&1&B}78-RIC-WH(x zKQSH$hI73{;4>^z=lxq3m0uHU-fse~y^hgUTEnjjk5H@Podttj)s0mx{$r||AOW<2 zlNy+bE*n|;-Z*$X)JLqTV#bachymt)duaRdp|UTl%`4Ju@&qx`&@R}upNAkvk3a9L zC*oeBxBtpNr=b`;up`v)z+iMB?Q}UL+v37so+vBr7x#ONQ8Pr!=pA?0WpS=s@jcEf zZ9dqtK#ioEXa9b~k~&V;^XoePuIJ0sEeF)Px{9Vn_`x&Nhh-U`jQZ(gHT;B4nN4LJ zM%~qtMMbIF-B?CpF)!Or+aN$#8=a4PUE<#p38PlFgO8CnW@XMGR&j zs{pmy>KRa|^E5doz&jU-<^BL5F)A!>4Cw zALMBxMyB^n-K~*&soHo#E`ynM%LWrsL%qPw&E0F0xZ0nh9K*l?c^oF@k5~eV!_$-! z&>8CVE!0t|6b4-IXCMc}Ks*Bj<8U7G=yd*Su0}OW4-7mEbOZQA2=BONaPW}u7Nx8_aSG%wHGD%#6L9nqrn7o5?8HvoA5 zSn%>{`_)Cp$d;{V;=HP~BjL7Sv3W!&vA9o~bs2<_z8hPQ0z}$7JT)YO$EnCFf!eAz z>)^m5N%P=|FIyLZ;sqWXNaT5EVnI6d{hWx?e=Vgl%3pIpx2tWNwA6{+M6{rK9H85n zVB66VA-njcXLQ(O6NQk>I^3Q+>z@l(y~w5&OzZJ;Goe>S@6hkPK;1%O-v{wtNkhxtFtU9qxH)yLxUm&NTL*UjAbzgtt~ znUDUd$fvVyV?X7MHsu07#$FrN>D-c*d5|TQVrg=;SW&cP%G;l>%{`>7>l*?k)v{}2 z{0e`QKB>H6gNyL`Y!7WNjQCJGo*f@7-?C(8>~ZL8IvP3rd#YM7vDWbX?D&vmbZ;gL zE5Ck`<5xkz$V-Bf>y=!VYm72{fBh`uUoIbo)N|B~~#k%SzFTYUrXMgwB zp5h}Vz~)*)tVo^!gfg`}9$3(e_ptgX;*8LWEcdLkIuB@~^jHC`301Y( z{LK~OmDj~GT2<#uzrd7KbYJv#C$htdhZ`Nv5T1np7#U_19|0S%+9q5eqk#ea#N6Ee zP-#~RPReBXE`9H`60;XzFlfd8%NM>>F!C3kTu`d=&dnVH*lBez8NO|OwvcjmTy*xY zNAq9LSqnA@O}n_r5J_&!h#nx}?TnY&f(3EDAieHQ|GR0F)=;GkI%Ft87KP8Csx5@(v!#T1g6zgki73w@VIi*o2vUKVf-l7+$u=Dr=h3e)LA@+l4kCk!-Bm zTPAqPo3SE0N5~pnl-C`}C7$E$ueuME%joOv8uAoA+dnz`)=0^w2znXbi4*5`6^g+B z`_h(8x0k0>E|IQxLgHuF-Szp}oJOxj#mgix{uMp@^9V`Wq6YnFEW2kNxP9;T#d}Oi zPWD)ofyb?+#5Th9>S?cSlW$EFf?|N>#eRj%Z(-E}t-B4_mUnRx&qD@!e7OZJhSavh z5J&>FMS=l*m$@0tygB{R!3?%ipvG@*PwlZlZxfSfpyso$zP6-Z|ALhQwlD8VfHbCH zbADzsYsP_$xE2`uyTj+9;h@N;t;15=?8Z?KO%?44O@Hd1RPU-{PcFMs^RG`eeG)yj zk^E~@x;Gp-KP>p**VN%F%@x(wKgVSa)LT6PJSQ~4A6?bIabXC>W`0Cs| z8F4edZ@z~UYh5)T#lV5vXtsI7FaKvHj-CC{JVkG7urs;HxKBlW&n)Jf*xix?A7E>T ze}I~PwR+_f%Eh^R>i4hB+Aj6;@`?;4B{-)dkZp}4wv4@kI=I$5Jt(D&WWJRGOo?RgVsNOpsE)s%~KqUF0MGr>Ku=7Wi~JLyXw#2-b^ zHe$K>-=6LC@U}G){iDB=aB|gw#zvriMWdR>NVOD1z3gyfAr;?{`cS8ja!lCnSdknM zc(EA1Autn7;66vx%+V4vOA!p8QC%OCCs9nPI>(H%`0Uuz9M)l$jiX2Z*1Kh);# zu3tp`o2kZH6+@YH;JaXMZflkoemQdyW63XazZvUV-xQlA&z5J*c23g2K-YxDU%Lxr zLhuUhM^C(;AF5jLEOz}9{@oCBpX0 zZ-3`o@;i@gR({~(8H$9$f$DX5ny9w}0b zM9!n@V7`DP-`Ny)&L#>}T|wbpLe&K|sP5Fdsca1zw3zB|6o3f1r{|)l z6nC&EXAQ4Cy`eE1@%PUHK?mFwSuML{p4((dksIEeQLSyW{QeU)HS+h!|7f=&pmHU(gqGherfFgWr2H_%iSZ^B36-+Fhl5&SS=C} z5iBMgArlIDHJ^Q1D1TCEbX!rf9JIdRQc8}B1gJr}ioq;mdX8huYEoe5W#NYaIJcr8 zt54Ck>G0xWqr_woyk_XAx_$NLTeaBB=To~-4?I!RYL`WNHnb>8!`x4W0t zHBK|bcP}okojzS=7JK+G!SAs>6D_ph|0|kypv$>F-j=M}m$7(Tdv`RvD8A^{l(Av$ zhMbOsk@|VbA1^oW1B3g7yTn;;dV+dG;0WdwNuA-K(2x-2De;OPzZ;sfn$J(e)ih*N zcvt#gd+?Y3^Y-Ur*m8dLgo>*ToSBYBAy$>^Q$4Th=9Qbr>U-81S8r~0qyjH-@fpM? z^GwTvn8Jfer#lH}!rN=LUG*x?w^mbX$TKGn3&-%%;{ z$0ycd=M~4g&l~Em{rL&qa!aPDmc zD93|s3Luy!=j}OO9n5t(UcdwhXlHPVhxXouR>-=tMh9`DaX{YqtgEtXdwd3DvNI}If4_N`L^IO*UCUP#F2M1HJWs<}s?J6(FtID>xN1GJaA3D5&F%`geEpI zpX8Yp4;_H$fb}mp&od-P{M(7C;+fc6-t=mEZaa$b%^5y9sR(Ov$eK9%7ozrNv?9UF zv5h*{aV#IJZIr>+e4VHJ1p`PMV)#3pKGP~3HST`jNEVVY6Q=pj~ci)A{6O0gO^e2$mM__4)yh=r~Z`NA|R}B$AwO@ z07yq6;>`cYEqPB~-EeMuFoWhyeaHRd-O>ZOIi2?%=f>NgwM0y8b?7HYctkuyon^0& z$)};tMju4qlaX?X;AS3xW+bMq;{gEmUwVK}9Qa>jEvn5y4tpa6sI5vJky{C{S4GAP zY$b~jhSgG9Oz9AZm%dt;gxn#9Yt-e%)vac1cB9IsNrqCjtC$kMD`R>6Q1rkOu$@X{ zhB|A3^q5B%iNp!@n+Z)`g8%rIi*B6`5JUARl*t#9>8;LrK-l*F0%h`3FzQWYzeAs* zufa3&G|AzUtAk0!L_=XVeXJ1NKj$WMF-hX(&WN+;-F_Jj_2*AiTG6o#c3PaPVAfBU zkihOXOBo%+Cp&(AiLl7r{K~(qTF7E(8a-2GE@w!1OT4r1q535)jUpzq$wq9D?!?yPjD{t3BF!(uJeqt_DQH^^{4@4~ zFv2&qb%~NVAfcF!@io!6I<_hQV6=#PU5LxmpYm2h6lkybLwat-679w6V8N z;&i?wAY22RS2ynO5nd;$?AZ}wFn}digd$f$u3DzLo?!+c?yJ^BR7(FqOy!;X8)|9I zXMWv|{?z{r%H+RhI`JS9LKX})PJUFUEZ)xl@T{mdFyh!}`P%7Ba`wjY@U;hu5$Icj z>g4R~C!*mCI(oe((9X>61TllCsh=^A@^(J^-F@#8s}_A3EZO&E^+SQl@KHB$skREz zj+WT;SEMecPQV1*EF!(%j_T)@>NhUsUx||drM-_q-`6_NT&lITFnbO>^37PtGF?)Z zSvV^8nS3pmIA=wHfsCZOF_RKwWN4F36Ny-J7$nQIw6palpE~|z4JMaU^+L_Z{ zex^YVYlAv5YZp!U)-)>3Y#r;sRA22SGLPxyV<78ugVuWb$D}H>b9!O}WTetwqwW9O z_~2C{8E0{R5HmZyzSVY!uZesE@jnl}KV^N5ffoT)6UwOyrkQF!Uqzwv@Go?K5V8~p zH#axkTh9I*eQfeFQZp{Q-WCBT3txI`eD{lhm7~4D*PSZIR3OAYg!+Xm>|rPG+2Za( zxc{M_tT7=}G+r^eCq?u$-L`SRLA`)uVuqlq+P?@-K3qZTm?vkLLAvf@q70WUT(xYAw2s*O@Bl3WXKVrH2B8d_TdvWS_Z7N~6~VWeSf&&+NPU%u%xZ z^mP@2eVN?8zv}a4!+M^l;0XNU{Nr>vKd)&v;fZ0-`jgbu*uE^N%XSm%^0dH3)s+2) zM7N)>Fp(I`3;iug*F_cML9n;J2GBBop=Jr#--+~(Z7c%gsw$~Pf@?}5n_01?8ZY^r zp8qxa+CdDO9{c2-avXbxDaX}p(rU%r0dnNF=(bNC|ypSxAd8;nJ&_Q(}=_vTOuhVRpyIDVFSmmU5NL~E4R?Q9HlrVZD7&3yH&->3ALrn731e8nyVd|($VUJ^gl_V;Y_Xu@4DU)RGVN#|VP zi>_C)dudEGDlS7z-iPL}^!aN?* zTi)>tPpxmLU@(D>+55eWEpK(lSfm&NgL^Y1oxujkpvtRzFIev?a{85MNyY=3rU^~- zT!Nx8&+AFH{x(S&U*V=&kl1Cq*Hf4tz!e#1apfm{kNx%+hBxS_;G z0zFL|gAnR7X2(2ZH9YF<^Xd{)TZSqpsDb@!x2R{)yI1E~f(Z{{V25}9xs~o)WIJlDMnU50!E$-wlzZETxKj6HC=tl? z{yOU7H?x{ppdpm?j-|Xvy34~tqo&$rlQ2K8Y2ofDVW_NCkUMlXuAI*9EjCIL2J_P$ zIV(>VwkUQn=O(9>ljxtC`WTYn3Y=#EplQjy(d$4n>B@DUAH}Rt{Z66;Zz9{zr=boj zeg;-K$FVCZ=VT^y_T_WE)DA#jiHS)sS`&{r9drGZo=TOuZ3HP~m1XLzxP zE(!)^#yLa6QF83@NxIO$$J6#ccn^C9KupP1t9}tT?M}`iEjM0D+{K`^AdfI7 z;vH|{Nwz~xz}wNZ#$YP>Q-Vx>0#G10cy9d%oldQPQGA=avm1V4IK)yB>?eureyB49 zmDFL3@)g4nZgzIT6qF6-`z_DZ?_Z9fl;Y%KN&W0 z6Mb81$O6%j+(;QnJyWsk`^JoPjM!>U!ps`0aMtkkj!r}onlRd{si{!{x+8UyPkSF+p+5LQ_yGkr8DK=k|qlKA_ z#p>=qUv&!7#1M<4%)#7*(3@WG@)eUjh;fu;hZ(7o%Tc}e>7oD9@7$xr7Y8zURE6JO zyqT>P#nMz;>}scc497PFUmPAFjZp7%{8%a7giBufw{`GLuLl3-Y~dYdsW`P1nP>eb)Ytq^eX~g+g{)$2Eo!nAw5LjYlx?2Z{eKsKublmTa5}tVy1V#b zG`feb8SjSt+dJCr=}!p_Sp;dG?%4`GKTJ4dj6K^FJ)OC6P7u!$PH1f17dO3;iLSrX zcW3XFx_5tihiKC&>%^N*wZ}aa>{#BTxY7{)MThd2*Rp?Ep;!Ro#Rw}*E^l8dMzX0& zxPX>gP`~h{Vht|-{?4mL?k6$$>Zy`*nobll7mg0o6zzR2myG)- z$Hr>mKC{kv5peK4+K1Pt^sS^P)@k%IRpn8rrYNV!=jS;?&PFJ5I!euG@xICM6&_oQ zpAP7W5yQ058~@hs1pYUua({xm{A>2akgcfZ8y}y!V~+jPuH5pYw-l9WaZ?LLKYxr? z4U1a#Q`H)g|E?p?8-UH7Uq=lwppQ~`yEDsso(3x|B`;`sK&nt;*UOVJ7y%#y;O_e_ zm1ha!k`D71{J6AOJVhpdHtbuXUfRu=&`UeHM{4pFcB-SqXi9#*ZY-c2k^5V$tdv+_ z64J`}xe2tCpZosI&S}F3IBhe8D%6!#@xS!xWj!m)_)b(DPx^8#xdr-eeQdhv}AoC#E3zhkr*#DtKYL$ZvBny2q|8`oj%<=v; zC|>3%7neY-msCk;VKC=6*-l|_NP7U1ppk zm$msH9~_W~B)peBXEQ1FmuywFeKUI5VJOGqhb_k4)=mPfP)ERpsEe$G8YwT=NU~^m zSE9{0O-2h4@d_hWUz?k}r6fzmU8`R_mRgA|UN6fK7hT@nRGu5L&-B1>l{9xG zElP*h?<|*l<51Mikl+LiG#$R(UF0fM9(rr`w49JuuX}&nATm92r#k$+)4lJUnPE?X zj9O5Rhp{Mro`t1!u(=ZZBd9?oz*Qy@687jvzE=pSzL(-+0x6`c!$=)y6&`o=CRS?I_aEj-4%F4&kl=hQcmX#o$*o&RPptLzz`l{M zP&5R*Z9#26LwR}S<9N~`Zy-W(3~?}pN}+aifRQXF3K?rIsqpIv7ABnzh!is)C9e6~ z($|pa3r&Qn>=nONdNo50&lyb-EQ}9yTnTkBrG`VOHbStwB+4P{H(CfinY|7DU734x zPaoKnkefhEq^~BtnDtfAx^v#*@pP+^JLGp{*0;uXhFH)Gphxr7Qm%o{C6INs4Zpsy z+d^-Kq0o>P;+uw~;_&SzprAAIQabsrAI;qsCZL@Q^=rm8Uc-9L&9zDZxgJT1-I3%D zh=rEuoUg6&jDo1W1YYCK&$T<0%Sva8w12;zFecjPm$Tvr`KS+F*lPTCDe>EVW4!;p zAY<=YPcB-L7!q(XhrdDqh=KE5c&2+z20DK{ zEtRST9!=wd5g`ux5D>hqw=#)Ep_!fE9KZ^LvA|%gjFAKN@;e=BMK6!2Gac;E2fENd z^l}Qj`>Ud(;p|?U$swu^j7WM!euN~dIXH@U&em=`%z5W~Mdo4YLk3REdj>Kp`&RyS z(>T@1s_cG4mF<0S1TBA#ZUf6o%yzHD@$_gRhVk9j4h44+WI1ZsbM?!^l%){v$azt1@D$~0A+P?fU{UmZpqav2_S|@ zs!4EYtoEo7=h^Yx356^H!hn8gv{qL&;`ixTcteQpLi%@Huvdzx=Ox7S$3Av&m+m>a zY(vliDa_}*c7cLl$$8ITss*ifn}0vf-;Ff_8yROc#8Wo_kBlPJh+nq+|5u2buj&QQ=B8< z1yH9?TR{};YIY`mRs{tB{6-T{;V!t`o=_G?aU&JgntT=y0|?>nkGeKDy{0&V^etDq zLbe67!Q-k5+c02GOpEO7szJkah%|coywdwC)t5|MtaG&$0h0U zgFeOBU%2^oQ~hMpm`e*5<`_+*#!hu)j_m(-kRSV__`w*V-igo61@fmw96^f3=Eb5O zPh$_;aqAkR!&#wPmpDX*1-@0zFTe(7e%Q@Lbf~R_7Lrfs-U+ow+)chHZcw}hqPP1R z>tvr^dgAw}!~c$0XAOtSl63R*3?6S`Z?JI>B+3*^SZz{n#xcRgIUdQVvBl+YpUZJ> zO86n+!(qQ^FV%ehN7r$a50G=Pvj77$lSq%jTZ{9F>orhNfiNSP5A!gtCRLlNv<%62 z5B1)eknwt*TV5lye{|@=2d*TLv;J2*{xk}~Ob3xxa6k5IZTBcfW*BZ#F`UZ`p2aaV zX!$PZTTn!NZn|IadSL~#Jv2lKni;8QK7qsL3w|g0Yl-L^oP$b=YcaDrb3J96pyco? znM`DqH8b!_4W>n6Zs*(v$vCJJY)xd}viVokG5IYQ7!@wN6!tI3N%i{P>(oz7E9FvpTk4nLFu)I9PM7F5V)uiv zg2Cd5Z#HDI%S+0DMkmC%@KJYkucQq3>&qAz5!z{hRkL#WsYn+_Q-O-XnA$B{fdWPh zkROahxTQs10Soog%^3PpD+Oung+HHtQVl-1obTS$^5sE}qba8hl(9#FQJfhpZuWbA zYblAVASi0gN&h}b{fh#zr?a+1Zo$-7imoC1TtPp3Mmx6_G`#$y_61<6XtTzrcu=t7 z6d!Y>`pSnzc~|CTc496(waWUL=f9@yP9k44L|d5Djn5BlX4>%k6*z&#DW`M-vIzZelbg}g;XMN#&*Q@@Xd@fED(!kF($ zUN+$}>TGCYkzIsk`Uzt?|9XeJS3ywHeWG9OlR{2CG#TN-HF{s91@B7{$a)M?=2}_ zmcw7+mrA^5yq9ARZY*~bO{CMoS6JM#Jn|$TAq9#_7cPL%IZmU8YAcIY5RV)m3-&Fu zNKg)X-|MwFL6}FOyqCYi%?!$+IG9gbC)qH$BKQDw-?2gcN=#~I%@~zG4Pg? z8R<8FGo?cEx|tlD@$oS0tC?QSttYmepsGC|6?=Oy3i!f?KDbXm^dciwmMG-E``8`Wp zDTjin!0?+9X9pt|O^##w0uY=T!_YvLY;E zT?gESD?*ufUeuWA_vz=E-=su1flzndg;)IG%L1hHky^gua<=|&^-UCWbR1sNE-C;n zf>KMkT#y*;RlwGJA~83<7^aP_!aJD!7Yn1O)1_~1%iGr-U!$Xf_ztniv}yGDDij)h zHIV$ZptHwL^+TigYGgr_G~-m@#eP4s900a#fpgjt^|~T8_-i3tT7m&*hf2t>2;9Vr z{00YTFE~T^6s~{v<_kGoK2GSrgSPVGPR$Wwx%k$ovKEUXILOtSe?_DxN_PXg53XlN zHV4P)r774d)jgwfEWeGHtxO^|qTS-PBwLcSq}MBkg&-CPsJM11te%vbo^yL9%U9As zGL-iLr7g56za%9)0E-&>w>*+RGeZ}kzG+_+=)}DFEXkgOhL_}`sh%VTEk2Fgu#aq{ z$C7t{KfFe2zkxKe{hgy?+C&Qp_O*I>t66`L_?1Y!_{BUO#`#hv>@l3F1PuycR-KD4 zu((iQM#K!vqPG(~82{J`?tHF7vrC^EedX8aIuB*ltPK6!xR23d)nnkeBj{54340Ez zGjb7T28`q56L$z)9sg$!{2d7YQ+-r9wc0&V6Z9(Mxs>>+RJppV(0@HsH6I@hQ_qQf z=%jJ&aK~)@vj258iJu;idCzMa-9-5`)rj@vHqszEb}hm&p7wG@+5JVG1hLS`eJjn= zUo?X1_R3L6i^JBoX5({)EzC}NYtc$esTFDyXC9fmPvK3_oeQu6Mb&#~UdwiO6r!cN z3*Yg*`M@N5;@NpMBUi6#F?X3oxX22fMd2@!bCj|oN{{n(oFsQ=ib>DI#t;Z?k|RM} zD?Q!9puOb~d&_&1p+3b`C_WK(vbVd`2dpf$h?iv)@g%u?>RIq@agfXAd@VcNCw;b{ z`Hvx5^wyixqZG~B>{}VSGaC!13XCh`m{8lS1lppq@pgN}f8fKS2!q5Gzdw7636)C- zpeNm{ae}X%^mDvt{W_gW>Np$miTJaSV8Oxu{NJB6pttu*Mh;{)&^BwE&H_^B;(vep zp18770)!_(Z8Y@{ePDu1SF}(4tezZkRQxhkXS)%6N94O1$`M`H)Z~*qVV+;OB?!Y4 zTF}P_1eG^rG6?Pt;&C|-9>g6m8^u$s1qKH_QB=}qe+Hj;{_T6JFWphYjd?#jS{z8U zhJwWTrijD*#a-VUZ&ouTN-A^p3_^vIr3Z=SM{wiU5*x=Ip)_Pc0_M9-O`tmxT7xSm zmPz6^!*4Ob6-s+$*a7cxl$+xVzBybsBe^Fa1J)P+?5wquTaU@U;fuYBD@Wet%s3x4 z^4805hj7+mieOcx1e8f{o}wn62n&$+y7M0^b7Z|^yjY=a<^$#~Mu;fGs!SQDIP<~J zzRT&Sws84Qz}p`0*xMj89RW#gL^ED{c|JKAAgK><3nh>3`{bV6{l{Gx^7qXEQ^<>) zpD$mX>WCK=a(DyU8FT*_YIofO_W&bswBcv%>PtcjMI-Ix`&wY&%ZwnZMC7ajQ9hgV z9r)p>ZT@7S+Ox3Mv$YLwO}bU}Pe?6}6s9nTcd~nMcJyy?b9dPnF6qMcu=iritor?*pk zUfjWqh>ZIBHkg!{FgiBZ0RP-HSXH&xcyc_qw!wgczO1JQed@Bsb>10V@Kt%lLwdS6 zK{Fh$srikDr=L(|h%y0=ug|AV44bYS)8ITaBxp@ikE!T$E&@YjK>*Y{%r@)SzmO}c z!JW)tBlaH$VsF{J?T331T-2|l6mM^y3ApX=SP#00zSJfj>8V>z*Crce*$uRb9yxt} z_qL^bo)%au<~PyS?KORH&@e5wdsE2iPKB3Zcag=BUbc7PDY_zx8E%mmC|d6N1YG>9 zyJ<0zsQ}izoYz4p6xI{zjFC*VeExcPCV1P^9@S>lqaV5TH~+BX<@d>4iC@ln$ptTN z>R(Y?+3HIgP&RTku*I?Ge^`+%h`rBLLq}C>h<)C=mh)d8qj+inV<6?5x&VCO-JLpC zweh{>a^V&Als=uSJtTXiS3G?6Qje-jO*)tJ+34*`ncNMu-0kQIkqN0$1qr5J*9k03_d!uYk@qw$= zE4H2+lLv2vq8?=FX%-z;*Z#~-mJD}xYEv9(k8G(>dBRFy*xKd$79rMmCd zwyB`Vhe#36>63{ghCy}DZPwMT(yE8BS8PCSqeMh(8kd$vjpe7d57z>6`_huBcJ@>K zzZ#yj<#Q~ZM2bHGGRhAh3Dccb7rJ>xpDR=8RZ6;{r6cNdWVF7B!XNeIb5N*kYkm-| z+-XGHNTcsa3#9q%uWx8L(PDWwzS77euU|sRW5&w9iDYekj)9Nwwb)|Di(Hi$3|yE; z=405zW5;aD)elNcu(-;`D`|>~AKp4YAjF9`{C8jMcBB|zi*7)m4l}c|gaEJmV*#VL z&LhW*jn{@*yJ8uoU4<-`JuVPu@J>UWP>1-8>rZucy-4p^p?9~>{~@UQM1yW*_~|Dm zcp$g}1uP`kteBe{IK4$~lqLZQ@JxJEPhl-I^eI=MX`F{Nw(umBn0&InvNe=5+F8&XGZ}H-mH?uFbF`t*v-7F(44tC6|u^K`0}#c z%~Tfh_cbpoK#SqSBZ+R4!JDAX;k`V;17=wAC$@J7hjbfs`)BEK!RzDuR|*5RifXE^ zT8+!McGnd`ot8D!>3=fv!mp^Qj)j%v`mtmB{p|MA+-)J=c}TKA{jkRdI|usOxn?$p z;p%enqOqBw7-Y-k~e&UJD_BLd;#DGU+x6B9nxd#Z(Ys}Nbg*2tM#P!Fz zRnzo2IIlY+AMi5KyTO}OWxkXY7sVoFVP^7zAu!+&DkFi4%}1fye$dSwHV-#O)HB3V zyV0tfk*okR13^kSPSM4Gvax0@mHqzY%&+J^4-s=d$qoCt|A1_4{ISJqp50X@%qlKS zkG1rb2RadZYkzRER-v)y)W^~91S#JIL|p3rBa4UB?C2)s0!utLx~Jg-NCyydayfT$O(mR zF_`blH$UE~&e!@fTZxkK%m`v@la#I&?;TzsOGnim@r?^LZ`V-uyIfk$D=*bg7#Lsp zCUjyPGa(W|wk{!hJ%TlwpC3#VpY89aH-%kyYfqMy7a|DG6rs_vY@K6kB4Zp!b)D36 z@&~j6Q^oIf(Da??Tj;@eZ5h1yywH#lT3E8%{7csnZU=VX0U^8EDibYPN9Sx$F(?-X z)9bG;+~-$=!(UL~-8Uj%5~V8^eo9KVxW&BpWCtOxtGFrDh&Vfwxx!5!H@MI6ar*ChpAQOX{ft<= zf9rHd5Cb<^l)Oui0q^hcr`QR<)K#|RdOw{{U|{7lz{_4`C|6JU7-#4M%Nn=)G$I?v z6^a+??`bX&Wq&9C-+Nmxv|Lp@Ax6ye9*P{UWXXz1vAZ|f=q;!xy*OzZlP(cm;nEicD;S5rwO zcXHNjlo^p}Fb#fLPKt?M1`*=$iTaI{3&GnsZ(loE^)#`_>rxd3tP91Y7a0Yh#Eauh z65UKVk9oQLWq~b3rwc-|B#AKoQI|2AUpoZSD*~OJ$aXPh6=RMCkY^+%N(J2)(A4e=$YHfY;K}iY$R)s>T5=~Wq6+$=E#*3^qdVt# z*XafSXT!@JN%xv}{#|f#wE=VnEPv0|-;-(w02ZcJj?U}yuF7j~T65fP{W8+bb1JoR zPLoL#{Q1H*fjm{nGPnG0 zTd<`VV=N8tXkw@_)$cx%`t-@eC1(aJc|n(;Ps4af3;kpqZZr`3i-O}OB(XSbr* zOkfh@SNsU%V{sOfx4-Gw7&T#LFv`y&CQ$NZWP190oIjsQly_8{7X{t)1y^+=P3&cT zhN4R;X#~Lc@y@zVn>k25yj`%g5P9o&K??h0X(_+0cBw1sNaQOWcsaKh)=O9k#=eEK z*2%H_4H^jPAE***Nnrzieq3u!Xkl?nS0@^EJqBJ_24Dm>#(P?O@Nd3dOHJ_ef9h*+ zJLgAd?fL>aQJOYDY`Y!&k=2+W?#V2~p!C53#R(Q1b=MbA>DBhXB6QyY=)Uri5ge?J z0lL7ZP=}J=bFZ;2X>q++SqVXirN6t$&$jWH1)bklF%|ty!3lYL>cMN4KLMUsnE}mZkCy)0q!iT!hlh`9N5sMF0KXp1bDpjr z$a{UteoISVq5GEm?$?;6`_EK_A}2%n7I~Koyk_2;KI9dI)D@nOunTj zg*ZtJ^YS8pYV`HJJa^l_n`lpcBsrGL^`11Oe>=ULc{z|Ay^=C@4JT?!iV`P3xfc{pzp6QkIcsEVM0 z5bUqh!+0WeXVJ8XY66U^$GP7SKDh4> zNG#3ZkP?S3g&!|JcDhGM>i4)|63=KA$xH{nL4X4vByPCGn)TuqS;YQ`OzHY2T;^5i z?D>*AFfQcYA;Tpd^=HXvb6gMo;`3*U(-DUZU>=>F{eW*-9*e!a#Mo>gsk3nn{zZJC z=x1Q^%~rL}!d4&Clk*)MrX7k-7YxRHclgYlwAr6CeLENxlwUi#>VE43n8;1Ne{D}HtozDPai4L;Ww|>*VfD1ei2K#Jn-HPx`In3@KqZ_x^`hV^Anz-iG zln_ut-m_n=w&||Xw9n|P2pBh|K`e6$54)K z!i~|?GxMEufMz)M&+to^qI`KJ`4ED``WE)bdq)eqPoYf8(QU0lA;BNA-OZH-J=$-U z`oa9!9xXdwTMSYE{j@&tBkuVP4Y}clKP5=%O4z&3!nA4=u&>fwhJ(oSzfW&n^Ut4k zvCl5^=GjH+BP@pC$-0Bepyz*9B22XGa3qSIyVb1rY%SE(F+aq zI}u4=u1>OI$@e!`I9Q)DBlHom&5yJ-UvGe>Uery^*Jn%r3@F|=tX_@#vHx-bo#Z=* z=a&bYDm0MS!wR{@|FO5^JtJS5*4{LH*SAr2O>gX;slKM+)3zFhhHS)S@;d+^-AT2Q z%yO=0W|$iKJT0I-`pZkOe9rN`sX4o2zIbSfy1ggmP)HxjxpzM*=s^xXj)@>}dHdN* z2+JJuCxic#>CaTS=%uaU2Xc+qXT+RGZGYc4Q0v?~4Q@KD=%=>;c`^on*G_4hyCF9o zcw)y@n0N*(Uf~j(>&`-g4|xKVp{lF~6IL*H%&XD>8^9rlk;ka3Nl(Q!KNGG>^xKCb ztU$$&$J4pZ8It^2_Mh#jg;eFU%8Xty3j8Tzf3rbZGdJ*;VNY!pRu#$9YV7XTedJx+ zIw%%(LOU{L*d#FZo*ZOrp12c(jfYIL4J-CEMYjLz4W1L78Xl|(s`yHx*!Z67UJTcI zaP}wVOiP;bj^s47Q24vN$Vj6RD{pUbRE$$waH%HtNX(x7a<(1gRV&78gx|Vd9dUSk z){+#E>+RC6F__l>scfyEcGrh8l~ClFv*;3vef58hAP=TVDxCA(*urqZuis*2(FZ~SK>@?! zu4~xyg%VE)pY#*z!*^j+pY78*TrD{>Zg&5J7vPyRQvk;ut=WM*OL%p9?nqhN(Pp@9 z7BZ}raBTu8>yxle!Y3N8JD>wBvyJa6_~QQ)xb-e$aRh5QZC9*pcInj{gL`auxRzt3 zL0>lUwSL+5J68c36^UD%oSj_)@J0dR1~cMAaF4dO&PE#>KJDF|-@rgKU4+<)6ighT zyEA4T3;<|{Sm)IZGDJ#m+B)9UYGDIu1U`5Pw48mJ6uEjg;WV`G-qkaqVy6Yp2&}_u z+10~;E}A=`XZL6hOoAQC+4FHXMyhqgw(BCPN2o4U&;hj((pKBt_h5Oi18=g)1CXJw zt)0MMU$x_6fB8QEBthH0u=$-NbadxX-{b38xQOIbH%I8E2;O_|rbLud3U@QRzrU~R z77HRxuik@0AK;rVHRQ$5~48O zGZ;;qqY@FATWzj}=7|^q*304%mu0zJF6+9MQqnja@p?DR9nB)bAyl=scDY=Z^To1A zdKjiRG##zib!|UfermP4>vld5g$q*>q{OV=F6-&xjL4-7!nxLU+qM-iLL`*jecPHx zZyp^X2q~v2mvp!hwISt<0tj8JT5G1ccj?tQ_Kv(^)_Y&qOFN(I`q1kV(3vThQMIWn z3YIaaF-u}1ShWQ+1*uyAp4u7)=Vj@G@9{%LgC6Xiy@OC1hvaGhJ)4HR?({+5*1dw- z5Y-$#h+#L15djn+F==K->&=|H_aKy%OUcBbHWNe6*=;9|D4Zk611wCz9786y5;2+yu6fGnF{brgnGFpo;rL) z1cE+a$xCn4pBh`n3+f8KK-h16ou98CzJgG%!0Y8u1q4pRC=d)@7_}}55nP8s07nNf z^0jWuwr;Io&Ohd1`26rV*7dS3B=T@NE5yf-A2X*?N>0h$nR&=%p659Wk<{92y$~Ab zuByw%l7h15f@RDpB_K9MGrzfgr`mf{4PZg01bae}2s1LVFrtVjNSVw;lDS&nY6l>R zs|6mBQi_OLD_a+E8HPL#X()x5Q;O(BA|fVPaWjkMe5tiZUrzLR_om+7yq)tR^J>mb4o4zhaW$WrQGImUF-RL55%PulI&zyq(~v4uC8i`B+MUw_;H-3l#;jB z*XD7qk~87myS)diTWGgN1|ACN-CEbSZChQ}x~|$bB%{RB?YN#=Gn?nbaGX;i&Z&&$ z{{G$sclO7TD_K|Du+Gz)aZEgjK^Pik!7NTl#FRut0?5@=10slos!M8f( zPTNHg5Iu^Naumi;Yi&E&d zW*>k02qcnIDFw$_kOBchyt|t*$? zg|J`vOaXiEWrHh*IK-Z6zNef4K)^4%mc4?Th+x7zbKtZkL85RryMt*zF+^X<)A4Y)xjPQWDUVEofQrN^5e-RKCjv93QXa;XOHQa>+nP5v?|aeGo_r8a9_~%e+!Xf|Ftxo)o(Bam zq?8=O(U)cE-MY4i`_F`&N{)F3HC>lYL_{t(LoUv_01_tRw5Km{f6{3{!X3;Y9PwY# z{`4!4|5s0a_!o2+{MPuH;J!{v07mf?!3%p%TE)Et`l(t`_AG=i)oH@7{_3l%#;0@O zFYOxluNynhG7P^mp1}(M2)-V>e0kSip5nP{*~<{Y%bmYUguS}a@uIC8{PI7BWk69- z3a!Wh*t@UW`ry_#y__$P=gXehKwIl;?H~*#v1KfYz@=nZN!`xNMO*i75D`sXEm2y^ zSt}t3(jYu08ALKALee|}i*`+jo>EGjhH^+LU3v`TIE`5Y6v7>L$$SGNOvrmrd$iCV zYSDe$_Ea(<;*^9aL{ce_kB_}~AQ;jxj$6bU zeLG#Y$H!!LDEa=w7CBMin1(`f%qgj9$2af4Z+*Lby!Wv8H#g()=5f9J_~Ao29-u?* z6+wxFSP%&ye*6A??_I4a8WN$TX`0@?eS5sSDaScY126>cSy4oo#&Lkdh4umR9d70h%`T@9pmF5GD5Y@DyYMLhkq&em5@8+($%#}6JeVLcAX@-Z1|SOy zM?@0NEHWAef;k{V8b+b$>>Xl$cSjK6it2=bKo+Xq_ojATo0&0EO3cg=cz=H%PUwip z2trH=n3*3w{3Yicj(d&HayhMQ)m}4VM5KTWB<@~oZw+(SAu)DD4iv(@(NusdBH`YP z!2uA1;#G~!zi>RSmtQQz9d3TqP>=+$Db!(-}I>-LGMFb#wSkyX{r02>mk^?(NGuDUGiILs-Pbo+iBD5a4gp!aU!;QFM~5y_Z{ELqj~8#3 zZd+R(AAb7tpV#h`v#M6}A#l#qE%T9?YjdUV#^c%0V?#A=MyiLQ$ouch^hVuOeGjSv z^6m}B(k~C6>$VUfpfdsyr6PHfJTQ-J85~uE&;wk@sZbv?+`yFGx<@Ah$*K1|3}ZQr z<2n@A(}e^B!C1YC>-PfU~atwiIa0l?nHErnq6~zJb?a2;l00RbHS_H zm2dE3zcTrriGj!tMno9l81BAXB3F{`IsM?dZgEEz_+l|Xdy(-=@9aw(3cg{pJn#S2 zD}EUnc)7p(&#}uVrtF&I{!4&=KG$pS^%Wky0?n5ZhF7QlDgqi1U;qISQs!YE&8CP8 zXwO3$=2Dts2agosYc&+?pr`E&8@1LPVy#OA{{HvB6-t97A&zF+dSYSbw{PAvCx~Q< zwQBEOLy@%gE)Yt(Ud+t4wU5J`^K_&G0Y?xfK_+HG2=@RAcU$Xb-c*}gGw<$>%Z7+! zDL~vjwDs6YE+on&i%3q8vxyK9x#+nITx}b3PRw~Mhnu^1-~ZI2eh!3@@LPE13_ zOr68Q6&(Uv+gKPKqjp`JSB=_XYa5xFwQXzLhKP}~yVtEH&JlqGj3B!T7$Pv1aZuH^ zwE!nZ01(E)6yc$Y$k;m|c<%@dB%uM_9Mq8j$j!)lgn|=dxWUfrUjvBXe`AaKzs}IV zp#T3h!~I%m6yb>EVDMxaA|gfjp3DKUTa5m6F0o!;UgGnG_9wJ@cI9@g*^wx|7_VZ8 zuW;lQDBzP;bnNWR^-3WkTz}5<`uWzxb44h;nr$~CzC5Xj0Km1?_x{)chzJ6V*mK@q zet3OcT*P245FDvL|~+$4D-Z* zK{yQgINv0s^>p^u_A6hOg}ET8cBoa&I&rW#Iv|Ef+Utk*WIYVHJf6YA%*!;6hq(+T z2vHj+YSwD&U9DS#U2dO&gqb-{V;;sZYufkf`}5@@B553W$iUp68nrPX0s#r200J>( z$@Agncs%avE<-NEFa(6vEnKVW`gk4#-%^t4m{Brg8H=_0xO@id$J@j0oHy*NYrCw0 zBa0Zuw(6%(_YV)JP!pu@j>pFjADKYzZjaNP=Q127o^r#0L^328GO&kf>%H}DAOLU@ z%rs&+TBC48$RUK}t+#E9OqAJFx24IL-Ca?4ZB9gpa3bv9l7%~M+PVdkWHyFyU_c}h z;gY35kXCKpdtshNnFnVx-I}%tWr8Wm$efzD+UxzPJ#M;eR@(^{M6|AEQA%MH^uX#| zA_CA|!Wo@OcQt}C=29$}2_TFi_CE$; zMC?ugdy}hweH`$$kMgx={J+jy4BfTi5SSty0ArVmgR9aX13EY&AtF-HlR|_%dvmYM zs9Nij5%ddh?urOs0Mjq?yuR|ye;v!d`u)EJf-j!~&u?G3lV3$N_y+F=zl^26{7AKX zKoKhQ-G(xW5MasW$egw~o$ep@=Q$B=+fqsqq{qic2_(d5 zUM~w-NMU!ozrW9Un#NSf8Tqzt^t6F^8k;Jk00+plu{IoQ8ROq1S|+( z5QlkO>!zWC$Slj&&g&@;3E}RCPahvH=g+6pAOHEEPN&n&G#^u`l>hVZ z|Gu{M&G7~Ue)GHcXRAtZ_x4RESn5&+a)@p%&<}^nwq8=osi5uMgV=!rJgeu-iqN*j zst?DJ3X|dbc)`ph7!(nP1P$D+t3~zzqHYnKC1s~no0%GNNr@2H9KzfYDT&X;ORDU> zH}jrz5|$Lv9#4Jk?XubZ#UB9l(D2=d|Mg=^$+`l--QC^&$4^-jxF6>k0Jh7*%)>k}bMJjj;-=n7h$7UsjS`KA zqlKD-c_(C)#LNi%Y=7X3vA-3Om!CfU_%#wgFCu!8Hx5s)Uai)@)mz+yp1WJ?ds7>C z2kX7>m3Yi3NRA!=Zt<*tBipvX*O2SW7jUhz{<@9xFF>Mr+VkhPU)dji;dQ=1`Dgy? zEB@fiE5D-8i#cCjE#m7IAkTIde2u`Q!U3jet_}tcB5BOS5mbj#O&2w^%>(vY2e+Xs5HZvuoCx-|5hk3O^$$=aW%~VyBlX(P-Sp*`AAZLrJ5y!W0{{4UcgDw4XxvaIR8K7aWW(ZpMg5IXR zc>oK9D_{$t?kL?HTmhS2E+7B+=RbYEoG#ifRe^+1A^^d0L2qc2@qgpeFbz7VO5(^{?XAy+G)Mbyd$If<*^GqUv ze6FyU`Qr| zlLLU{AsNKldUUgx(kPU|13WnPe%=<^QyFh=?s{#TUV88B=xS} zEcuvI(rxYQVqJ4ic$}jvstfZI-V*``glSiGv#8N=k24`g0I}q8OvA9QD-lUb%mRR_ zR%`3Mrw&}QgKz4byd(+YZhp8q{NX?Sr+@#O-=FUv{*V9de}DL=f4(fs)?1;_-V0Ms zTuQlUQ}9%>hN;#b2mtJ1;2sDZr%V0m^XZ3&PsZqo^Wk=$=P8hb2FcTWI81ZQ3A0FH zIZSyP!426%m>EUH8aC_2K`AKqt`yJ|c3+ouwHgl5dvl>=Mv#$(Q$hwH4-7yjazaKQ14BO4VpJu9_<4 z;2g|EX-LGJQenx2Y(2sw0LN*taNDKg5K$PI9D$jq!*PxC;^%IbfSHo22b!w(-T@gU zIg%41NpG9GQivo731rdcihbncULTE-!mo{=p4Y-FTj~Ek*m{Z?2JIpT6zD+!A>=@N zJ<`5-pHe%Yc<|SCi?4DBUTu7S`z7xD`j`LGM*9|W#cM0-I(BRJOA8Br3AoS5_3OOb zzkE2gJ@;U5CWAaN2Z*S3f;lH4)Rw})hD7Rvp9|;2EanbkGEz!q7zVFB3>Yv=DJ%q- zh*{|E?c2I_Rj75Xwyx`C+ZMCF)n&abrk)1MG!&jQjS0shzO_o;ha^)@Ohga~sD*_A zqI-l{bnUJ6-kapa#7IPlZcc<*iZJi@Cjbb^J;EF#z}?ijpj#~42Euea5@7e>q1?Xv zZu*`cKiz-2|NP^_=Z+d3-by;-l8}ZW^B|XAog5^G!z$8TnJ6fcTx}LW(uiv2NSKAj zVY<1!L*Zo+k$D(JMCOUI@GLMVCn5q$j5!5E2J5|g*S(!$^FWY1B+SvxEj%Aca$!U4jn)3KM`^Yq2c- za@NQD^I!hF{P2;s3eB+tHh07BV!()~CmM!nzvoCo$O$=6P*@eayAwzf8Zr*z3Al|Q=xnnm=tCwQ?qE{JWK_iti=ETCm{Znb@zYY zxY}G86bJxD0Jd8y7_rx8y8%GJHQ{4F?224pnjT*w(JL$F<@R5>Bj2hn`quuq=g96p z;ft6iIKmUB{_~%wxt`6)=OmM7pYX|UyrL~VIh(tu9ue@|RQ%^`wV(e~uMj~PK!jZO zY+i5zADCz1v0H80n>7GW84kk;0No5!3(HUn5h2n%PQ9BTP2*UkxY{0zFl()E%cb^y zO%E*k4=a#^;PRC6BFSl+(>?mu`x zEvI!|<;P`UAQD9AppTam0#`&;U3+c4XOuaoT?`A6mS#;sffX>!y9bI4c_s@%*;&+K zp0uQN7-C532Err^M2@}-Rhn5Q(e7qm(lCtknDVB!wR)HYnH`YLoj8qCnI=pUgcjyt zfDY_Cy&?6Ea&+00!=Df?$^tVFcY8n)+pHA3xVmAMNu`{o_yY>0Tlv zFo6gQu>dkcLd5fSA)+)C@qz>_OnJnV6cBqyg8+%lQMfX6=n%x5dN&9n!Z1jZd)v0P zHB}%gSvVCZuKN?65P_IM2+;d3bQhGuYXmtWI56{$*8pVyk3+6~5I{UP+Sw(10s=f7 z93ZX&q5$xm&t=}fIujzj0+%NO?ibMW;_cV3=w%Z8U-1*4clw$?_==+-<8$tKC&&D9_ic~YmOUbE0T3WSoN`DU z`34eO=g~|dkcp7XG)i-~W$g_hrrYCh$PerM>EV8;oB}Lj*-ky|)5B>f6RY*!FXwYx zx1pq4?L$e4m@pn5?NXZ|fd~i~k^(vcBLYAvVa82O(f|05dTU2PDY>6u<#2l)DBZ z2qlq37>I!ZfWgaSXs3q@KAh9zl2;uJozu*W0TF1(2BN!^9|og@khyQ#1=%=pI08aq z5N1pW5dxjDVh@1t$GKNk)szYcXzdKqRBLS>$OCgKAgOy;m_Zx_g-EinK!BMVCS;P< zR|5)U;&5gPM#P^r5&j=D$mwNb|IdroJuuwe_m1Sd78v+54=!Qf(&yU$ehnh8+{xz~ z@g=f+x!X$!`?h!aYXqC#tTyrpW%#y}j3_?;$r8HGsmr}+R5geJ>Z7;)aLh1JQxGdYUT(-3hX-Fwe<1rUp zm&YP`nx^;P{qFAOH;e~wYz7~G{9gmeneR`Jzyi7OIHGyb=U^I-4TuKF4go+SL?TxK zaX=9SbPRC!?mf)hoe_};5$(w<@8Qfm&vWxVJ0FLXk7HiVtX4!Q;}8O#AZ^R-Jb-EK z@!|8QsCs`s&m`WpcRim^>tz{Js;x~Tdyl=xOWIlsL@pT;JL6?rKdtq~0|1W4qYge! z(}^}0iUh(bl_3ObH`8EFhl9{CB@G&Yj433LdQGe~>STzB8M0tbEbJVHZeS5H4TsBVQT0+vM!9Sch!~EUqP;~Uec?xzxwD)gX>$B*zngc;1>g# zJR;0Okvfy%E)ZhoKq8!ot&~!%ctoJV8l@CU5(p?vDXq1-yX3rQC%C(-w$@aYGSyz! z^-?z*$HOoT^E@X3;+UtLS#ECb-o1IB_~zl``jxv!_xU*mY0O204sy)mJQc4iKwHoG5w6{5M zGlJmEQyCroyqw0Oee6NnS~H4jwV-wLYW>{%);3c`fd4OZf7&I-l3WRbvZ<<>xqCz) z06gGrOfr*XRd=&{y8r(l=;`irW@@U)O0rmFXXab5#o~)Cx`@bpu!ukaczF4eWYvrw zoHzz|b2arvugP-dN+n`9G@q7Xwxt3QA|lq-_WS)Z=6Lo55r<)Dok@zSGZG?^Pz)g$ zJ2phbu1lLP^&Il;m@tTQUGy;JrPgW=$i*w76A?)OVkdAyBUHBv*4!K{+A6>e?mDC( zhU&;6n$ybWMnnWgU?4QLR`NWZ=4s41Zw`k#O?7{p@-jpjgbAg%>pWL&xmp2lAz+`< zb{qNx%VK6tO}#Z_HdRxowYDbMrEViBkO06c(QJ$i0IC|&&`DhQ#2lD8#%M?>r5GcU z000t640SpR8aHaLxz3AahK7+qxB)vGG$u*L4UktFrT_m^g#OW3DW1CejQ7@-IjF;` zFAV_jAg?O5{j{z0Q#KR=fT{T>W|X^!@RQ2NgA-N)5@RhD&tGev7&s+IB77H7kr zQ`koF&@>lYSiIZr#&W9Fc4wO_EgvZyS?bfL zoA3Vl>WgR3zJ2mE7>&25o10sSNj96!#YMV&YP+-Fjmw)K{`UISo8>q`RqVF3z*!jT zG;h@6VEX-)4826*t`~@nidNAOAqjE1n4Kg#LUlJ{3Q~iWR-M$ml2wE%SlQJeyW7p^ zecuPbw&dux4b1FT%X~PB0QQlZ8AzM!(BRK@zs_g>*fW)z!?!QK{PER0n(OP||7bD2 zo93cc%tc$Lxd)c!sCL`$nw(o(^yYNYg=gEe+r)0yU4D`L8+^OncW2vizS?d2Fd}6u z@kfE}l53u3iMjV2Qpz$H^@`GBjG|z*mfCn$a|#GqGo{xlH3co~SH41QEJ=^ZoJNRRj{Jcdt&cJOAqP z1@3lN?{18^MsEhn0jX!&Ds+(!T?!moQLD8^K(SDA03&JU&AkuYQ;107gbj8xygR@A z()ZJRd%EB)H87{D7WyCoavoK)1ny(%T$gxWt=7qi)wGe;;MPm@50ignySQXP)RlX%?rTB^WKQU{nYsgl(%M4} z(aUyyxqQyc``qJ?m-2WKpS8K}Icx9;wmxD?_|)BhAL(s<5SfzC$hy*_Yr!YB z8+>TQeQH8KoS+UsEP~XL6L5ooArPNdwgG{=GUKotVq;|ZX7j}|%`Goi*YE0Fcik2M zHk(cCW4o$`4Zva)5h+VqmgTSh-Cx&vx!oUn4o@zh{Kffiy6p}_*w6EutE>O`w}1Ec z%~hRCLcZ9Xo$YoO7~O}VSLQAS4m2-I8S8kn{OT|NY8cWuj{y4g>#sSb>%%^zfGle! z&L4dys^;K^AaEybjKB^s3`303U31a0ER#2HwdG}#IzVOzYppD~&c$j2pvy0w?{Yia zU8HVP8{8h3tK)tir)Mv|Je4xdODpQ`NZ1WsA~`5X6bap~3(t4m)3agMNx*h_`Lydg zaB}xmydNloaw{1C=3K{l=2(HF0!oT`n%4&L)@m(_H#I_IPAS}3oPdZ7tD%V$LLdo1 zVu0$PfaK&rBFsbxKE2fQBlTBw zU!e8Tb=zGW?IDkWz9uq!J-)ZN?*B`xj<>9pV~$|%XyEGJMJ9%(83ni7O*3g)w171& zrOflFS%;92Af|A2_4fAmT6*EYyYtKLyg!aRj-iyPx~pj)`p_k3IvftiynOfS_4VmA zP75LSeeX*fkEb^0U;n$mGSjo|uI3qB=eFFQuGek||Md_5@Y3fmzWF9@;=I3-u6usA zvw0>`fAn!734)sf8X+>Rxd#Uab--mQtu=QoIhRtvnrl;D+3@_?#q*2pCK6eG_UxGm z)Ka7d31YPwIGfDno5EudQzV=Pl{3YGBt*0O%L<)i=1VkYc zOLMTQW}v_XLJZ*MV0S%myn!}r&C1;JRHmuUC(8@8+FGfq31yJTY)FhlMblHB^E_|; zyziy!H<~NdkVuN!aT*O#y)DH-!DVe5!kl6s$OBVM6eETXNq`9)DUd{tU0@%WG8b*i z%y{n)7~z`dbn|A4TE(oW&xd475Liv-+Snn_-(GA>8y{W0IR;%I|hY%vUYn!V>a=H+<1Ow-a! z+zdSglR)EmXl*$=+e*Z=?XxXPt(0;+9)aP@uV3=zWx~MC%2dAp@zwSI_Bc%-;j|(b za6sxXJbn7?%P(HUlv-;ncpCHRbWm^m!!1YVAaBO&Mt9 zhQMpMNf`uDSG7eqZOz-fR02d?%NP-nm?I@J*Vgi!>zKCOl9+i~gpNEgW*Yc5tQs)0e%;@XS?5ycpXVb~;QPn)h^>Qd&+F%Sp$yDYip z+=#U)s|N#h2gC#f074j92;G^;otcABAtYfz0Ah)OL*fttxiobSAQ9A!nz<`7lUS>a z2;k_*Zmy**v*%ftrH#itjaC){6HxF{wJwK>h$$xNCDOUpW=)O!_2DWrguKjEDW>gq zyJ3+)A)2ZdEjc%+5HM1oh!ByT#F6f-LM9|eMD=E9D9C|)v1Sz*hM{kuP6)2dvFo}h zf^J|w*E@sD41_VTM2>5>$HlA`BWT%S%9<2o>>tO1e+kxn80P*UI`)41JQ6ql7uj<3 z5WRx6=i(iEOC6edb5Jxza&Th0N3PF73H^ei@?V14c#kqG1^=OH+U;{F{sCs;U7hs< z4DllfB7jHTaz7la3l9JfCe=NJeVhUKBkzLQ-5YpwQ#U{lf{5IwURX=c>ZMuH+LnA8 z$35r{T193NLLdTFs}35|PSUPF>#mP=o+fRMh^b2#UwmPumE*iW?DubO4*UJQLLh8CS zpT<1RdCue*gaa{{HxVKbkkt8Zb9OcymSgeUp6)v6I!wvH=Xst^V_9-@U6u(=*=o+I zwF&?+#>5ek8Q8Sdxe$>z;ULfyRh=ri1(c4Y0UFW3Q3(Qx5F@&)A;ZeBUs+C$gv3A* zDXv6^>MklkY=DUD3{IjD7?l76VjyHQwOZz}PN&n`cXb)tbOP7JB#36!OZCY@6jf6I zWOQ^3*lo7sX*!g#Oyg49W_#ZCee8$C9JjrhiPfN02_T??=UR%XnW7s3usNEO2qMmI zrDX$c-fGE~mqFsN=_u0DYC{!*K7|xwt2s|Iz^b{p8jN%&#Z!TqYOO^T8Y&3@i-Z`u z{I5VRzuz(+Z673h^!%SVoB-Fqk6*sR*1gRI70|%JteM|w+k6(ee&(G#Ec)@m=e*1J z)jyw!H+?)m;tITXn1#5))=K)vyAE#m&jR3Q5Z3X25&*cbWL#Z|kJ6Nn!RztYw0^bC z2;3Y9SulbGOe~4PTdg(EZEjYZRc*_%Or5LXE5L9`A}*( z2~sIKF2}Lvah{fCS@WujxRRr-m7H_V**1w|s=0CRG0r+GT%iGmm!1nxIy7rV_;N$Ea&-lf1TzT09w~` zkkF?T21m=<7Oi<4<96_sV1-~v42Zyvh`b8lphVHY6i5Qc7}JnwnWj1em|1HI%d*VV zh$*lHeQ$g@0RYi^%)&>3M*56W zn*S6@6#$Iv1OELZn`L_#JrBy@!8Wt>mbbTn&3ZraS72y-?x zWr=+XT}*wSg3z$(o;|-Pt;N0@$RJQR4A-|ehr{6yH&<=WwvYt6aTMOz~xO3~dNED-lWh#)W1X@BUzt&cJA)2A0* zjAc5uQbPy=*lOd%47hS-MM2dFfSSQ7vTUpAqRg~bG+|g3*qJzxh+}XNb7BbwtcJwG z!Ukk!RqZS!)n>KUnla~g$~tD9>xP0$Qgye&j-=p@1b|i3AT7jZO*V1TT1|~zx+n}A zmfoP|Hch8emZ94mk0r*?`Os0K7zmLN2|4wh5j!(dL=h4OCMJ*vPV+*_Fx+#`Mrq#0~7UMgj^q4jPUt5m%#@%+9Ndi+}w}W)o~uDW$XeF;ac6^b=}}y z+iW+(KtzJnTH9Y=AIIA#Po8*QQVO9z?}ovJe)ospfA`&Y%cdiBFqr0Ym<}c9R?QG{ z$$$3i=fC>ZukxJVe*feC&AY1~-oAVFI`Xr_c-$z+Chc~^#j^|QaC`CW`yYPzi*LW# zr2fZ0yspRTt7l(ykPdGT-Vd&Th!mqS0yrTdIhuo-n}Gr#3CK##l(mzvHAi%BRhxnb z=2A)^?l(I~W<}eay%`{GFP@bBY1uFJG}qHSO;edmS@P|8I<+=xThN^8&Yv*bZBigk zB9W-&vOnBlE4fSCO?q;E#uU)dnnw(Y6FLNP#z5eVBEZ$m3`kMdp7Mq(xe^$Wn1f&t z1i;mml#V#56FXDrT-O>BqSbn{s%p%=R@JGsr9fVMtU6bl+cpgDq)x4{tGcPXqlc^4 zZ-f{l55v%%4K|PSc${kitp?N&Lm&I;m=Adm<{@yktj%rlhMF*~of11jBXUFsQgAZ& z=HN!u3c<9D(>MyH=x*MoIp>^T^bwds3Tw$0Iv{)3$75BqwId-C33D<;RSz7ZkTz{3 z9AoUZ#9^F9><9R8|NfHk(Y|=tDUT0+Ar$`dfq2*M=J9IR`Y>1{^q;ed9$S(R(EC}0 z`i$DL#NZ!}$KK_=n@sP0Z+uVYdN8jLk=KrF@cw)NKxFr}l2aa=PCvQw^=}|1qFU>+ zECA4T-AZOzHCO#^f0>!7s+l=pYi8EK9c^u(r0$x_yxDHMgwyHf_IBUuk^=R8-_=lK z+V(??@zoDMPSe!)ea`vn_U6~W`OVpri|=0j@L&GRe>qLFx?h|>y#-xLKHiQn7ic=2 z#+3Rmzxn!a{@4FyKaRis?caa*`#+qnZ>O6h7JYJYp>^1v_uKRB<%=ggNW&_UO!@h< zuYUi#A8+5h8&JM^`UP1y?U&KZ=IL@(ujYB#J$NFJz z;pvkd=jlnG7Hg;ZwB2saY}ll}@4Yn5OAyl9PKP7AU7nxa-X3bLA%!9FGUe8qh*Wl! z8bTtYR$AiNrRWM}X-WW$7)2sepk{uYXAL`MMhV1;I2kaY0FWsOOYFqJRaKiYBBkWn zvGVcywA^0rLP%yXduVVv9z18QIfZz3_9Vu5Jf5zvuP=A!`{T*j({3ZxHrp+9;k3-o zw08XV=ES_c*rK9ZTiU$oc4_E2P;Rx(*6+66W>}hn<0{-r%)~+=G6m4FPu)$<-(SDm zI&YAoNC@G0JW>k%wvSz>s`EJJc~bYT>rzTYRIQd$kun6X&3vg4yVz~}O)qhn=8VB4 zL}z~3Uyrr+KH?ZW>{jCU8ZCI=9adFoRX+J}i~Y=a*zX^mFCq^LfJjKh?qF_epyq0- zrsleqf#9Q%_kG*QPdK%Ym+}$5{^WRA)8qes^vJ`2M<}_1&I1@Z%ZE%+EVCc6*gc2N3 zmDRwsIYUgHA~>T7atxtM6q%V10)|Y)wiXXy03!rZb6lH{qPZEm0UTq3L$9c)x5K-MF zCPY*hW(Fp4B4G*KRtqo}5r=pd23uAJ3Mlx%6$b|+D zT^gp6wYfVb>A^yo#(A8I%$iG4Gc;kA&F1|4oMSkDa&fai91f?P*KY!}p_@#%xAibh zw}&BifByBiUp#w0&eOYh*Vl*r+{)W`SNr{59E0PNGfBOAD zUf;Z1>b$?X`uD&2t7V?O=&M(+>FM)+bJjQ?r|kPFlFT$M%fhGA_Swbed_yq=rjoOB z+oZw4TbW2w+HTbD8r_hfJG&JyMKy3hBqSoM(#(O)LBY$soc4$D>bBmVa$Q1-!}%s` z`Y{(IAVLW!AfREJ1Rb2}E+!{AET!Wx^nCzg;(5;NmlcYTAO%h-NDDJm5fOOwauOw)o(C397eU02F7=fx^Avk-SN zc3mn}MB-iWMS_F`5**Qx@4iKeP63dbgQ^k!v|aKuAn;2}s$Vib^6l;}jRt5R*3=LH zAqavYwso^0(A^IAxSk{*P4^gj@R1YyN0;?D;{N!2d!IQaW)_h(OBG{;pYj%!)&4Xdj^mum?csENb6cy5 z#N~D@fKN7OfBirG=9?E^zIpxn`#=0}bKGA~CyY{ST~`vj?&+9>Ao16dn`Sx^kJRV07NN0V6y5y6B zUp{}9OJO2$j}(R!t=8lIM!J6eCAlj(x`As$Hb(>?R5xZ9YE{up&A z)VR2*t%@)ybS4yucbi@adxrC2!w9GOu#JKeurGb=cbgp(!JPxb=GY9=aqa*C002ou zK~$=t0h$@OXv+meB&A-WhU6hqph(O}frT)V2ljv_R7sc!u>)i$F=TW`3~q!#!Yj9U z?Pz3xb{DhAS;ff8kO((Yn$=~xdRJJR(@Lo!U}7X}&6#M}h^cE^iK*>0<(wsk?QR#7 z&=5GLwdt9GE8^HiZT0vC}s}^I^t9O@rsIAp(;NVj3`6Wa_T~+l(-5n-{qb;`r+HZp^E@wQ zncL|&ACEHvr?Ysuz4-R)Z@zu`)h_jKU%&oO|KUG<|KpGS`7UrQYRLTJ#fz6;yy!^0 zw443y@ciuU{?+{bvDKUsJb&{1<@09<_K*MY+sn(#-~9Ez`|7K&e)!?l-~H`>2ByFG zi{EH&$HQT+OWdSOdOCICbUK{!Sn_yr`J~%!y)D_YpdjcxE#3LKyQ->MQ8hq9Vsdf^ zFfX;WT9#?b<5F|8ru(aRIcw*w(F2M4YVn(JZ<6~kenE=uZA1fn=&LpDGV zS{t?k1Ovjgr;YM~%fCS8VmvzkJT<1cHyUkhG@1&ZxR$bjOq;AuNY;Dfj0Gx$lO2n$FfCOYn&A|0BFIG5KHb!x+d%;EIR5)-Ea*=fpW0lHum9+9ZB_&bLaSj6LhK|+ zM8WUt79Rpk@EKhE+3P*J+#{rWJP-4oSIGBc^%GY!=Tqdewfo+Cj}GqvWrY%20S)ik zU;<0{=wn?keIF47`moX)tUVwp04&S$5xPA#GZE45;SX1dd`CnigK5d4W(wX&i&+JZ zX^2BBn!2twb9WIbOFJA6x5op9IE~A3oJw;dxj1`*6#BmZn}7fBzxv{39;e^_r@#H@ z|N2iiH#boP0lFAd;LiJDx83b_9S1Vs^}~_p#SZWGKWw}7&DSqFqU+axD0$vS_?y4} z_460c{{A2S_P77=kJ80o{pvTnXJ^y*x7W8<+3K_Kl+5SjVbW#M#i0zFlrGNZc^MBU zp|I)tA;ucTjU3$@yEk-nLq{;i)>cU{Lw2-wBQVRKhHh-RP;wYdU-gbon_8GzIY4awPz zrnw4-&N)hQAz$q;CPXzybOK+|Z*9y##KhhRkXA;c7A;Gyd0O&zGo0`6m|Gc7b-E3q ze);m^?W-Tb2}>+qv6oH97$_v3Tmv&c>>RaLwAFfix9>Lt0(XWDpf#;-ie$OQ7^4Jb z&ULY-h^V6c5EWTzsH*@R5;mwyo5uyFoI>onL5PjLfDuyYB7$Rxtrd{a0Amn^wJ;3< zSXTvS;?(tp1sI*gkempGknbqyKV|!Tc3=OSc#EI6xomA_`_bv(jBEArDn=~GftZn6 zzn<|~;65Pj(n3#|d7y$|8(RoPsDaAE+c^6h!ShMN(G5bdp zFvt6j+#dzt@lzZ4@mp8{YsJvD9Pe>+qsPd#W>;5kuOot^AtDhNBZ3>2Qu9<)y$hQJ zDFC_Byi6%}+C0yTfgi@h_4W0bm*X@~b3ToEDGDHcH=Lbqwwvw6_WXFW|Ht3{_8_}_o!U+hnZZis#Ct?BK%o4HN02}0Z3>npFt4QnZpqqwIX z=e7UlT64hSW^Qigy5iQ7=e#T>SG8)Ys%^>88X*TpA~G{$H-s2^%Q@5#qjbc;)FCFs z=T9zD>JW%Xg7o8(TZ66xA_78WKp`MijBM3R)scyTg;1CR#}vArV&pDy*GWns!5NJZ ziC6^80m%(<<>aklWp{7YeFX^x%bJ%w9p-twy?OKX%P%7t_;fnHUiP;=`u6PUJ|Q@o zYt#8S-Zlb|SizdG88kOhLx+F?s4=;LYpLV@)D)qQBrF&}SV*n4B~M5cV?++CUJ#?) z1)R)G)z|_7IrVAiLq|^YJReSrhCB>i7&;Vgu5QFJ3^6qC1*tg%0(DadVnihL)k2NB zmd_v?qJg`cHv@BdY}`B=KeNIA4MSZI`|-VffDbIKhu!@^+jInGOvE5U5`>wVcI!#< zx%$4xrqk!~^5X@4id-uaee~c*r#+5yF|U<6YzjatDgok$PU54IpEX?dU}QZ^{#c*O zKgpIpPDkF^c8~Du@j98v-OY`N0Laadh`>__(>UjxLA~z>3Rb6O8Ozg@`o3@0ZV&S* zkMC~Yy?Xs-Egni~kkIds)3UUb`j;qwKB?!FTdn% z+UM!!cnC3`ot#v_)YzB$w?fTu1R?7eISHJ!r|Hr@ifBlF5 z@aEms<%=&bUw-jV-+zC-Kct~QzCM()Y|l1bOj_#o+jk4(r}6UQ?ELEL?W-TYJKtSg z>@FOrvu79t@*O{=5E!< zO>5V69kItTN=PIxpY9OZ4MIqOp|#uW7Gp#r66Qc0g##E_TajyXMno%L3rQ%%(De{w z+U+O5ZYZ8y_taQ(LX7RkgRGMpO|i_Etib#Ee~9dla?5 zeEF9n|DHU@b>G)H-&1Rqf{-0HUW0H9a4NftUp3C=jb(3a3!bU8NnRXIORIvWpb~l z{@B{NV}p7DdFaAlD~qnD1s=&QlBZIFc6Hiwl*MYUcWctGut*O6OwzCEp9?^_Tq&`)ljcv+Hf%iiJhD506idKbwi< z3bW^hXLtupLci$#*L}wyYL?KNg1_fGajZD8&#&pqu~%ACkw3+fU)nJ+*96dr2sRec z#1kxj^kfIsOq&ONOU4D-!5DpllpB0Ib%ur9+E4jkQiS4meP*tJr6F#n*nR0N4EdG3 zbhKIofVP(as-b$?DyGQg53ELRpfX#| zhkYFI#b}(^m!uDW@z%QhAKc({E>HaTTd9y>?`(Ctth!w!b4t0ne&pWUxsuN0XAyFS z;vx(6c$4M(SsUpyN>>~+@fthGu*P|8rX=hs;oRRy-R)`84)=DD%)(@lX~&`kB{C8! zusoBiB0UEjTw9oQ0RDbF-NtiyWOKb-L>03 zR%uvu5L@8{Tp;8Iz5q2`DAhbVN=*P6JzAUUWWo%^WoCoZu%qKINz<}zK7`a=llPkZ zVe`{au?Qngd`d2A`sbr7dC=K6P!E=d2AnfUS_28vDhV$xeh_Bsa(nX$_=k)2LW4A( zB=48agueSXoV`x;<^kZ2%j%>)iK=HeIkf>p4JJ)TI)4!+6czQ;Xp9XUZdVjlU83mZ zjYu~}FVxfiXq#^^h7=QkKMSM|>n`~@`-FtZ>Q4Xh&D{ScpmcV0C`yVwQgc^QczIP zHBzuEaALgszH%ky*HPAe|JNeVZq0=7d*tCz|Epmp{;uG&|2)U$yfFNKY_EHI=p(5O?8#!tAJja=Z-|ai)eHSC^GC+PQkplS{(CllL?rth z?5VBWW*3I6l}EN1nQ43aGjp#zqyJS!kPOQBM2ck)HV`p0V#CQL8UTgT>(r*fm$eID zpDIS}5I3pwk^^j$cHlhKF8%9-)!6TIG+@A(HYX7Q`;%AKPodCshKwcp$6iod4vXk| z2w@6#;iMXqm}K;P{GhK>6(aQF0wuX(ZedZd_0DVIK*0Z&twdDRkr+(Y3p_j6x_x+j z=4U16;d811UhV#Kr1AbOm2S$dXvZmMxJ(PMC{F02s3HPxjEZKAtBJdS4DUoUt#h%m z*7k}A-;czVR4SYpD}-)S>8)!tg4l%E;ica!3^?P`PosY&&Bez*kJmh1Pp*sjFvmnn zuf~>C4d2YAll;&ZSt=Cy=j_anXRJe~Ji|aspbfLo-F&D~w0*XO#ZGuuYH zi~!G-jOi%-M~GWMOCE~2_eA8EgEE)B;%QGGb5K%oF~l1vP6xnuu}~mqvh(}zgAts^ zrICY!N&88Uz!UBdF8dOBNi-!Y_Gw>sCP17qU82uHFTKg)(@O00`XYrypfbd4{**LV zKRkub2brS)VCv#%_zNDBHv|!~eZq6FBSlB2AzeSyoweVm_~mo?-lH3LFf%hfu49bN z@VX*#v&85pnvku{@0Rh5k%nUrBD~qyKKE=60Ml?)gP%8T4?cTTxC?w_U!_%!cX22b zrJ*Pw1iK3;>Ik!ezg360>IlaIaJlo<>~S9SGL(Hfm|R&vB2Z*S?FazsWR3M^@-ei3 z(|$^q#~7yxc@CuLbT$;r64+#-*0dh>+cm&>qe%hGIg%r|$aEDNJO2^^ElbPi{_sIerevseXRcdf)? zaVJKbbNZSX%0#lII!s1j_08pa5YO0qI;)%cuU&2J!Pl&J)8VRryRQPm{~UPMEn}CK zGOD3(!u-91F5SXqx`KVq&StHihFsB^nBa(5K^&`KW(u%w{F8FspEvu}!^6`cy>3_M zOlX={9LDf{?!ve@X7c%|ltyU3^TzzlAIS^AGLXK}#n7a_L?;!LBV>nE zqK$PKB@z;3ptPI>ctFZDVd1|~P@9h+dQ+x3)8{gJq$>8}ult z%i!X_KL58`OQ!9teeT@{EC0_aZ;KRL?4U^WkhkJmtpf)0YEn~ zxOL~Wf$)N6qk>U6p1u#|OC~(jwLxQy|2Npx?UhPa7rlh*J4ewE@fNeeK+IaU-eWYG~vChd?c7Bp8^lZ35U-(7Q{pn)B*K3>8oA+a>S2epWr*c`<|0n#m+~}_75C^ye z=TO3It$uNDOS(Jgj7%rsbslT@%{MeunT~EzE(L)??jM~=hTokGUQ&GxTUzN_l$Z9@ z?hFpu1#IW}G$VYR`Fy08EFbhWKUl65l4G!Cb3GLcfh&@aJj2fh6RC<2LMf>=5eV9k z-kQxrPjKF{CM7$r&podaf3-=N!3L|RU_?Yp1U?sJXd-Ti-+%?-OZ%1x5Q9v^2!Cn} zP)8G=n-ZVe&4J>9Y?|=S)A@NYx?sxPAtx$_x2;q{OgG*s5sr4K+epwlHRzP+eGlew51}x#!@nSlpo`2+)g( zAQrBLLgB_3AWaV_0okQ*VUpDmt1v#xDhrJm?bJ)N7k=KMJ1-{mjpT|V(20(-gG@i$Atx$t`n)1MqPJ*W(Zar3;~VAKMPUGklAt$?32|7ok>^fO!m8E;fQvw>h0)w2gc zLr{ame&5Eye^4f;-z|$aqIzS2xmZo!2uLHx!3&{*SBQg@>)pk?xpJH`ycB^OCKmza z88ZRARye#PUMQJeH0<`5$?WXx*YmI4S62>n>yC|=$Jd9Me|$qj2MJYJB^GIj#?Sst zS6YQ%p6py1&BnFyjY*-j8P@&h+`-Ndf#w&pl{IpDaf^+Rob9kVF)?AYp=n>HgwHZg z2muPMgQOm}WI#;cK-9R@7xp}Syl3%BD3{arCXAfAz0j0*rg95}%T0F9P~c0@zoL;t}n{AVeqD6sW-5z!^mIZ@(2)Zuu= zGZp;(>F&8>VnKB`D(vrIssAuk*60^vm}b=S1%tnxmdWub)M0X-2e73W>Ef>7bcoRuj5Z<0*hoGM8EBqIw1UR zzi8jnGqz5FnQ|jgYSBHuI797&M5v5xNCVd6{lwRm8{8aOIgkjglXy+ZBy-^5Ny)?c zq%isV(s!EcvBWccx8^^nRwk;J_*XTB`vRI+*wBo#XyvLS8jmy@l>S_0FPNZyYn(53 z$-GjyCCQ>#z^3?9_X7)n&CWFl58wia7ZY#xuvFC(xY{)VSU_cgY<4719(mh5A*LZP z|5G0CI`&-D>gj3Rp#|D+|Iq>FIR&Cb!pU!RKF^e=mg z5u_=k>L*I@4xY+Wl{E9R@6C9g*@<2PslEL*vY2GMI2SUFCe0*Y5z8Vw~Gbzlgx$J7h3udTCZ%; zE&4=J<@L*#YIJc6xu2GB{5ix>iAo2PIY+Um``xEay1=6@B{xl4E78FP{)ygB@^9!$ z%}x^^kE#(f)%2QIHL7T@P(J8QP*Bh%Yxs3I^^`}8HV+OP+lf208WgzMnhg)~2?-6m z+q}A3z!oZmPmITRE$)Zi>q~~x#jUN+|C;W`dDlIHcpSivjm@%)kl)9x8a3N=j2wB} z<-4}=4Sa*lJLwma!y}CN{_kj}@yEyM=x|h zR3isKL@@`xkqOF?2#D}`mGE;Xja*y zPctoAn?R%HONF$nB`oPTi>ATvT4^=vUIQ(6bm>{1z_X6;o~%1_GrC58&V$a2o>NL$ z5{pq#>;$JeEfr)Y7aQuZ*i@lO!xKA~1B60^gxSf4vkj`Jzexi`NRiHdARo?4XSRcbS5 ziQO|KA2TnK4-IY(`RNy3jk{!A-HfWOTsZEZZ?$Kf4ANPFrULVEyr)^kAiC@B$`zt| zgd?^BZ?Ddd&$n>rODK8xHS4{?ra!WqvTs8QEa}Sp&9kNdRIuAwNp8WZlI62H z#&dn1#9KnQYm0E8+LVbfLB&B|Mx^mJc}CQn1gT{D01-#(I0Rth!#wRM^~J85Qr$gF;mf*lG!4oqtE zO7V^PeSnzwr@6UnbBvR_4hPT^5dpkhJsd6boZYwFa~wVAzwe2gS@^Pyi3v;xE_6dO zFxK)8u5f*9#bI+aIf1T5O*vsQB*vWi-K2m(DNeZt8*>mqP}?La6JoxvZNUlz1dH4y zZ&9dyn{egL_gS~TGIcJtp)0f12P|cqJT=B|_LOln8m`F8AqS@wH?g{#&$8;$;pq&! z-4J65$g}y~EmJ_bprsQD#5g7oJIj3_0chS10bCn7w=ubdI}fktX?vO%Ap_eyfjjNx z0%IZ-xJHJciN!EpO&G7?DyS*bNM{C!M`)+|hKzibWqj91q zur|;}4bPCsON{GDK=^pfKhCZ-jj-`b{o58sR#~4`TIu!IvQBT$Z*Lm9Qtu2qO~TiX z;!acodkW&>u?3eyO;&NK2M@FNM zEP5XyB4}L~bh5m2hQ_&7l5lZ$mD9d51;9>y6`)Xm(kf)2o}wIQb}^)Y4A7gowzSNS z|D)@^Ll0gycCk`kCn!z^++JROWTjg9WKiMFQ_k@$bcNj(?uVQ|n2=c9KSE~-u5^Z+ z9-w(0b7XCb==4ftK&O#YC?ff_vPo=u6 z+TM8&Px!Bjy^`&|!A|fKryY4+>20zBpiN-|?ow}AvE(=BciX@R9))>RV94i52k@Ab zVie&B^_{2#eET_&R9y_;TEtUn(IEFt08cxEEggf*!T^U}K)EtV6F&q*A{5}o;V7zT z4`?G)40~wJ`K0tcTTE0E zvXjXOkV_6Y{VLT#~+1PY##c4;-{6&96d>FlM?sqJR9AmF5_S2N=-4$5pGEt=1}!2&zw5?8XEE@A}51OiX?7UCqCf{zj_v*?y$OeVxCjQO%AY z0?A}DF&0`#mhe3Em&DGG!JzN3yc$211viQD{H)`%yu+T8kEajTt*u9TGq%jv)hiqX z;2sirb-P-qxQ(#zIh-{0a%XH?nEb0{XIrN<`K=InbY9WBzU>HNp*2KO?oblmLRyp8^S`UZAlzW8(D=L{;hy3i{BEc8DI)6BLE ztpP}kcQpdF6{7=DQ?Jg%dsI~&0@sEUP5JPULx%RLx6HZxW{6J%m5cX@zjyrY?;!+I zV8>r9M)i=fNEGCiR7JBhXb=l|Bq~o>e;yf8VTw^?WSn7m9`kF0#z84S&DV~`hHNTf zrk7Ds(Kl!WnD!MT4*|9ITafo22JY2)Y~c?qAFHA zT1>vK)L;F&obRg3-C%<<#i(*{syn+Lu*B!QGsTgFkF(lWILpFH+>gPq%k8}X?plER z3g_IcUpMs+S^}-Yj@s5(@2+PlyA9vnS4DI_1k^9d_FsGx`gf!r$@)g{dxOcaqh+o* zk;S{7@fVCu>I;gDm|F3cj;37xEX1NHdr?u5Liin1?ebDSWnBOJ?X1^y+v2Wu<%|-2 zw|7^*G9M5alw;$fd#2pV@-n7FF7(auuPi~IiIwpC+hZY1bMvsvU+*0R{{K~oT!|ie z_va(-pb>8g1b5w>DrgJ{7Rtf-t-3=NGjH}ytnRNQ!^4`f84hxLE0z`(wZFX%t^s$bwJ zX;#Wh-#$*{e}6me56Qu0c_k7iK9cSSiZOHJy|5ZMCR1CNy6NSBsgJrl;gk5@HK7z5 zL#}7-ESVK3d>c9$Z#@V-9$twwxWNmQ(?tEk6@ds5R6+=hdHMSff3r1zm^cve+qJi@ zZk>YaI*K26UgBN25A|_YyUp zjSWSQ^izS?d>D^5LNCC$CZ4CalA_3iuMUoShwgO>`?d<6ukLoU?*27KTr96&=d6T; zQyzF&vg!r?C?5Z}2kw&kW@dxD@qI@mrq$dzb z(AG^tP2; z=*5A>vH#imJ_<$A=qYi_oRVv;iGBRGV0)f9&gT7Z>DQTIrca(Ipgv+4oY|6*jaa!9 z)!J8)4B1UTJ6=YU=0XM?<@t#UMxJS_4Q%YYH%in$48Z5!>mhW4i;LMQKY4BOOC3#U zNcISZ*;*y{NOC!F9CCy9Y+7%T{YV9;Ej>5tOh>Y-;lCjg-7q4eqc6aOsspSA6bTh0 z>U?MOTWf z4&rMfIU>FB^?w3_a)q@d2_=WuG7es3xO6Pdq~0aCu_G7&c{L^qh#)KPI&A+;Imp6f zY4!l51|;xd^D-bXa@b4-WlJTpGbL-JeL4M$rCRa4z8f4YM>m8A=)T@Yz3fSFg2U7+ zvJJW!q6P%YX>1jteuF!@yX=1JXx+>B%YPrusO7)4-RVkPvoZ%{1;*5e%?20Tp;C7t z^=DJ0_YKcyH7GYMi2jr*sk%xwS0ttk^}Da(emla}mK_2Bz?>Wa82A#6RR%_Au_e*I zV)wUI;R>vl`1|uBH_OrSR4!#2mv1P!(nXG6PD7M|xYR;(zIIF&#=tJF^MVQm1q5&z z5DqzA*9S<eULZ6Zz{1}xT|9%h2J^#D{w7(n^Wre3@r+WQq zBOLT0Ze!8pYiuQ5`9%4|ZXQg_*a&>N{$8-QIj{fDSH@j}QI%=l*krOoCpi@vjQq3|oWfFS=c`L!z4C(ZaDSU9?Z47iuH&=eRC z!D)EV;~cSKfAKeUUE!2^zhm@xepFkKH#eUc#Rk1+#jK30 zuxF!o=HchmxTCrCaHIZHQyM7doxFYUL(z5Y(>tZoIz%&B?)`Y!&B)f(C_I!k?@p4j zNEsFVVZDgA=q}4*=>E}*%gGP=3e15i1H0IZ=r9L=g{v+rs^v!Raey_GzA8S4x|<%# z+z9SWM}W18$}z>S82GKxt!t5(26hZEm-QH+zwRm;3Wa})Z?EG;4NdPa?(SmMwhwSb z%i(h8e1UuS?Ntq@{{7A5JsLaNj9VvQypHX%i(gzOwBcv-jjxNBT^9Ez?oVdJe^E*v z)_qS`$r$6(_eY}qHC@^=s8zDJWyiluBauQEp{#ZtxKC$i=;@a3L{Q`WFLO^%s+N|P zZ@g93)5(-u14qDgY+I`{i#|FZx5qgku|snZW#BXP#dRmr*t0wXlWXc@ z)beS<6%G)^6Di5dk@OP@;1f3yQcG3j8;{tscC=WTHU4g}#HL0XY35VI8MAtjJTMON z=j&y9hLF3u8#VdwKQ=|42Lz96YtJhX7#R9LLA7npqM>Gc7u}a75?|4Ixc141-xI4M zr3R&P*hNK54MnX7=<$i`2zyqGcW}ula+A{{y8mj6wiq527Fu}3QoN)e!<$JchdCo& zN`RjPh`P27?pI)7uEu9G6b17OWAa&UFj5_06zBo6hVkgBOuXi--y>(!OlQ-pMw{@3 znjPGTc9S?LyZvPC7EzCiQuvz>FO&hkoPBmT%qY)+?$&dY&kAT_sW-POO-8Ikp^BF<3A!r#UtXrAa0qR8t z99q6TD=or2D7?Tk*21L3oVBi~>#c+c z*V6fbZu!!GHk(p!bq94Gku$+z0y_sx$|)9;@#N2IKe=Ff9)axx_uKg0H)+24og{?_yB3)fdfejk!}yOMGVhKU^mF5KWvPCEQS1BKw|0{l{;7y;qvH z5T>J+4`vc8IQ6yp`?75#tak=~6tGKH6D7hejfJcO5VeaoDga$(T5*y1zD4fd;Nd3b zOwRj_``g9$T>3>%Lt=Sw5v&rcFf->bs4U|L13l3(5^t~VeS%y`*t+Q_tZt94n#Q{? zFT!=*e5TuV04$sX9QdU9dcPTR1El(Xk|@>w3cj~*X$w15Dcp*gITTp0e`-V0{}Cr! zynU=D^qGx^jTFLOZ1W)!uOJ6cO`1MNc5tR$gYb!)wJUacC5n*2YgYL=10M{M-ULb*DYslJm zNB88@JL%Y+A{mxIaw?sxl=R{wNDurG_U=pg`lL=*VQW~FQ@f>yxD1RX#}vLL7wL*p z^jL7ABg#dXi)H0%^TM*0T=>h?GU>~zx10+r6M7qu92;${-7U37+nO;r@N00;fr)LK ziIdH8Q!Zu03BK>Y{rdImg^mo2i!gzgrAjVt!a!oW7_UGU`#5aP!=rM86kKGUZ^Vy_ z!@9cqKB@x5{=fwgci{xa({ecs*&l!SBRgAL{~e!EZAjH!%}jgeoXjk}Da@f&B_9CR z(?ZHYAn)##hXD5SjOwus@d;^NP^4H|St5ke`5CdXB@j}Ej zQO2lRw3~LqK^C3ENb(a~y ziev(w8xg%$c2tL|I=L3sae3{uCu^#DA)m_bdy+EuzTj=ghpS!SM0+jzMe>l+R6?-N zgsTF!@pTrof!G{8wI}d^$EMN0zjiFcM=zdQB`erZ@ku7=x@6Jq1-16eVH>3B;MVh3 zF|V=>6d{>_k(wA8=tHu)TY^QdPYF&%jhSK8giE(6UNQ=eoZ)LXtOZBX54Gg8RBmzt zF_Tt%YxzYNOsq{%7Dn468@Xi`iuT>ZyWw=URjSikdAYWyPKzxViT)h6suvtTnuS_T zAs7CSr+Q=?M(!e?G6G{aT5uUQ34eIX22FC=`uF z;@RyOCX(Qm;z_2$ry;Z|lF8FjDdiF|J>;&&d*|o=vTo2pvR9dMAM6AMLC~_cl+(8% zOWj>zTxqxstImU1#_c*a=EZj@!D=7N^WpzfL_vj|s%Zb#r}XsdIm6WbO_|*7u(7eE z2zlVK+@x1!-0NwPn*T^ifPBh*WXmu^6UStwW(Rqc^qhojKP`-ISjkt!aA{ zuH<6nt#{+5@f^yDBlf(L(WFy$)Sgnm-o6?3T3|2T_@^9_~(N5NVzC*QtaW7-6y9X*uIt7Pr>NzU*q`gX~e>2I>5ydZy}v@?7qGiux!zSTf9Ze}Qw* zc82jnZ+|@aX1bxJZLw@=#XR&7>KSKKN0brIV?DDTL6KcxV&mokH~Ovx6`p}@o4;|t z?w=Z;fy8)yGpmsEIsGXjJU}mrlvgEI^?u?v0&g~dADKl9UHGt=TSx8VhKeUdHPQTb z0V;JOa#1tvb`M;PsM83(7>bHe#rBSazx`JVHT1RC@B^I2Khi=wU<|ZS)h_VjYM};G z#uSHeMMnk+QtyWZ%sy~L4z(@7HcEEfoy;SiP;Z>Ny~$&N&EByN(-8&s!Nt&JU|3nm z9 zY%?a6;6Z!+!~gIe%9NN|gyZV!tHj^%mQ$8f)FMf*b!%qps)bj})GMKAWPv<;2h~`%JNVogdf8 zGnU*l(cau}-o8k^+jg%L4?cUvI4uA2Bs|c*z+RVsqfEKTkJD7)-(S{hQIm$Gc%gqT zU*Rpr1=YxC@#$(rhBM#v?{Y=gWg2ux(;qQ`4V`$E9sD9ht&i!@ady<-+loUOp*+1| zn5*f3*=n?ke_(5YKNEZT9o+Q_H=i893Z+2+ao$Nv30 zEHYrj`_Ls+3*rwI+jeHsZ!^;9(WL(jzFCVqyvG?v*7o-HMh6E|Wqg0PgwmC%)**&6 zqCWquecu_WLLiY4gP&Y+pt%`u*`&my_{zmF4-_=|DSigB97eq5Av%Cq@Wb8>~)cFZU>&^){Q>uSywN)hnb0m~H3;}M)Z8_xTaDSG6 z2c#^QFt92gu`m(bN!D6^V=+IXx{8M_kd{(3UCeqA%Qjy7qbUc!VvZdu5^3m822f?P zy}y}%Z)OWe!Ov7!)lTHL)C4C5X_nBnpA5*@MSKb!wUr5J*^x3NPu}3!tBVrT4S?`- z5RTe)K6U-`LC+;^$Z=T`X^q@JtN5U}6f_8M54dAo13z}a=UlyJ+IrmAeySkPp(3|* z9j@oI7P`09{p~NRKd>V`g#B1x`o=TXJY{MpYtH;|AFa<4et#*KOMA#0!t%R&ccuJS z+(%aZFVz90G~r*#d&h4Dz8DET`3UuOc|hZj%Ls}xo{yv{G4eE3&a$m9ZPVM3!sPbP z(`LYxdEQAEKRgFX|71uvoCwOX|CN|7r1wQ<`33fZC#ALTZ z1w$LhQU!VljJ#j}k3o98n=90Z>UPE13kMGhiz~6t8fQ(WwkC7eWWGfdaeRdX-U+C$ zA;UYLpxoel(>NH(O%oG8LcX`0V2u8`U@ii!%#iXE9-5(w*Nof}c@KwS#B>dT4b1Hs zw$`L<_LN56!q5E-w)YMuP*?yVdJ2#YoNJPTzo?4^V#vvI_$Xc(<=GP2B*PFvdf#fQ zVMgz%qpL6uxhGL1-BfH+(0Fk-ggDy4<;8&9mqNYD#as7uHYj04wPg(+K8dY&9AMyL z%GT(s5x5QhboUFyFqOuRj7KJY9_F7*VIILrMyT$in#QK7%Dq*n%?nS^tdS_(;>y~_ zHtcTM!lYGcS1~W#aF&BlKRAux(gaa#sB%ylX5kH`a6Iy(EQA`E1|x{*(w};vEiP)m z_Tz=^>nYIARR|Zpt^FxlpImudbGN1T;2_{bDh+nZ4|}0|xCapJlwrxGEzD;cX7~f;DC-uIT%t^{ZuXSFoWH!f4G6HQtCqj}w~;dD3(Mpr zTBCSFL$+7P^X_MHwida4q-|=Ql~(85bFr9W1Ug?D2o!XmUtv<#D$zc)>|Q*iT11(8 z_{sC#Iu55A#tRSq+{t~WNX@a%#-YPqwVSPyp*nsl=hK(pDB}^p=%`SjN64Z^|E@`5 zr1WQQb5*sBXvoL`pw=n=OGSiRjiI@Y5pPd!s@42^y^`mr#=6V{InlE}6JN^^5VCm% ze4YMpO~DLq12N)XFeWQ)Z_hR3-`G@6;Ar)!in|R{n)8sT+SH8aDV0W=*5D{TLUF<- zgnb`4(%1%!gBG8(kLj`T%8G1e8W}yR=BXVuv;|LX(Ss=v;?K+25Iblz%0DEQ20f*=W4K-ub()El(>g3FQCMhKJR*i&16H^p7csAXt*aMRI;0&`fKj)NX(E7FP9b z#K7PIYtc~)BlFm)$H7E!<0*I(Uve4|VL?eGdlQ*ELuJ0GuvJbt=FKxn-`U~3TVMPWTrY36BaQN&EV;&gj4E2%0z+qe@y86RT@)+WLRSRgV6PSid?LVzk2Fh)TFXWt&6>wU?3%9}%w0t? zHR@X$sdxjKJ5RM9pPDG(u_!k|Dg!+a1Kx9i@@8@`lhW4}a0K>ae0fl#xbFbIy=7CT zHzhH?ff9pQYYQ^J0s@!;MpPDx$8f@aORGv-_*zO0bS zrc?_<8irNeCpCK{YUjUPPtKMK!ymK;>GT{i_M;6r`7Y|ec3D!<=hlA=T!`CV((YV+ zX#cPgn9D_ykqUD?HTWU^jvo2UOIhlF(|AOc;JWa4j4Q#4toK$nr~nA(B$35Rj+}2m+;1q?~pkuWlnKBHg*CPQBdB_d{@}V80E|oqv3O;&mBUuE6v*13D1W-fzdaW5W}; zGQI(!xc^smR+_T=1;sb?uZQopN4r}v+!GFdL6#JcFn08Se=_JmlgM`>Rp|I6P3Ua( zpccN7uxco|N|w*%9g?*cm%;f<$jzrJuVfe5I*$V(CZ|&+@+^RFWti=iMIfYDUTdu~` zd2x<;*eOY2AJpGZzT2%<`QCxj>fMB_ZRCbnJKvcpz zwcO`^5fyuSeB&&@EN61=;>>M_S$!fTnQ!9 z-5wwlGt-O>TtX;Dylmg~!xe0v%&DR$$&pIw6BS>AzKU?**W3%XP7;9YBRuI#c4yju zKgcd=CRnWSS#o#+Sdo9OXN^tJG!AN{L5lft-9_qy@`GbmPOo~|dH2c~Zu*ap_WFD6 z9uH`UzCH3_74BHF_#<%j@8}zTICqmE@Oy+cfK3(3MBt*sx3J)x^GGR{68VZ$WI@8q zRgX}0^`I@oeBVokGBnYGjR{|ws}!E^F(%SNAAobomG`fC`Lz~iR$*lHG}Pi>XxbMJ z&v;G<`Uf)US4KbLD4i+C&V%em9Qb^k;R8LYLc%VnwwLL@9dWaR6t8SrZad@W8JhnA zRHZper)pO@kt02o*#z7>-Nc9L*uCEW=#yZ9Y0sxkhO`kB1BIo8V?ioF3Xhq{G>*X$ zs3y77POP0l5m@~%9|ni0 z+xi#7cqc^TTWt!ndin^Te{$PTUnBL-(MyuwmP>h93@0WGQby!z3*-RUy;Y6hxL2qp zOkokZE(AWFlz?4+YHo%=M76Foki@v#6^CSyKs-IHcenv$ssX`THJfizSBnEH`?M6< z0EwH)bXocaEBp?1?X;hjc98oA9sE9Ik-*cA3^~^^)t>S6zJLpB%$A;E)ptyGh*QLa6EvRp`i<=!Z7&dN|+MJ2t_-8&m{Iryg8oM;vW= zh4`IOvcK!$;biZY3)g~d8KcZ*Inff6WtuIqJ=YM36KV?vM=yHobHFokr1Gu=55P)3 zXg!r(!@sWlobGmaR-r;*fIeko?p-o_O^f5F=a~-Dx@8)ouWdhrA_o}2_yP5c`{uf- z&<6x$)^vtNxb|oc62}AE9XsX9be}w4KL{={&|!jt|5>F6{N*HO0jLvg#O4mgGXpv-5KKW40bIMwTuCJz zB@I#&c*90rVOcRlgd)oZ6TzgMAif~oULZAg`(()DP7zMn*>c3iH%i6s;D|b zO&_R7G60sCurnra{Rw@PdvMTb`|J%%^RlPI1-j>!P4RBUM^3#?c5(lHY_Z6{CEaa4 zay4gs9E+FQp`vqGZt&lym;Ar7?(W0Ou2sHzRB3(6(a$-C9w_1;^^Q;cU}drw@*ux7 z$VrDITLYCqLL}IhBAcR43TWv?uu7eU^!>Z{NBFFQ6OX@M+&*qxr7QVi)xh{q`ld5t zO1oS>>FM(^fEj(xHf1?Pt`~O8WUuec{<%8mtzEOI;uvP)!^1zX`K4tRdWCF z8jESz&CQ>;IW6wH)qO`>cLhT4*-FnaMvuVR#j}3g(;4q`KYo7EFf0F}{rtOt-#(sK zcn2=K`5I}F?K1Bu`XC13QSs_$J-Q1M98OPJ385%&v)D$pxXZ942L$kJ^NDFfNjwYq zL~LlKC`nXlKJ%HTnn&}fC2kxC(o@;|K-z#vHB_f9x4ZZrKvu8`&ljq5FQ0#xPajQ6 zE5C3z{d!5ez4H4aGEyhvi{oP!nys#Ydw|@zDB4d-2a504cm(&rSqB$}Yy$7M_V3~( z5JpeaMcD|S=!HC}CwM0f7LG!CcD{)DNI=M?OgNg9qsRav6$VJ-M=+CnK8AupHpu{+ zHQ*NBLSzCPTOTR@#VCGYU~9@#mSvkBMA~{Hj33#740%fA?G>rkU zv&A^K6u#kMZy*eYQkEEp9HKufdH$9n^IWhq^BWvjuPxTKup8IXwCG99Abpr#W|zHE z7ZgP&{3D{LlY@2bTVuOreZu$wDo+F5?m?NV+ zsxW@}oAqO>uv1ytf9q;pb4$BFWxIChyH;oJ&NzeK=Fql7vX+UfizIzMRw1^tI&)aM zZxa3fvT2CvCbQ6Ia|inTXOKLd5DzV`L*$je9EY(HzO$=!$7Wi(b_!&DUGC;5cM3($ zR95OQUZ(Y@R0@xJo7OD^Tm(u9W>NC%U+ynT&N_tUejRH*t1zp9yWpw{!~2s!3%g9X zULKo+U>T-LXX??h1*RCO$)cj>y0JFM5NtN?D~nd4!GBxsmUN?|Qw}=Vz8;A8E7oKc z21<@n;?rge3=y{?#|M?X7Gyv8qmZ(y#)a3>mIz3c%Mtj}M|E69nmkTK&mp_7L!bDd z=cPrKrEZ{mFR%SuvQR8^h2aY9FFu9=v|VLaV59z4|-{mel9S&5?`?}nl(eb4wB zQ^gk9R)!DT<0$3Y&Zm}wV{-acn_|Tz-X|=Gb0_%^8!v~J@u9W>YYhM|wl_dpT(p5w zl*kkNiWsGx>@c#^(X}V42gqNMqS;`2*9)M-Rt#~h0E=7lfu_1s{;^~nHmV+oJ?oPP7X_<58jO*fptuCq(AY{UH%K8n>)5kiaU=V=W;vFlLxPl$F~>43)q3ta`m#lgD%1?!ImG_^Zo>WkCqwVZE(+x zkIjhN>$}{)mONbP-nVHIaQ^-)JKzyrW3$^>hum404ky+0^OmrjuaueTUR!$xu`grO zsl#vnZ7AFwZJQ5mU`d5>HdRf%q}8pjDK#x0@X92MH+3ufm45#FyC(WdbHtZEh+cxw=A`k;nB7#w#amoEo%)-J>2LhC^CFYfBLrujXqHf~epNikE zL_Y0%u>~Qmd>KKS%ABJr#EBoDoc~ZghKKAY3dsesdGIa*9YR!7cWNw{)#gwzp>HVg zj=>Pq6%^rBSW$*kmAoszxf*ZWz4?K44mG%bo2F5#0-sjtA9h7@_J}PMiBhY6c9qs> zX;8H{QKNQMv?4}rsh~z|Z{B~-zvuHg=Q-!OpZmV9>swbnZ9guULETv1xHTONSju3R z!L*8jUw)!}ZboGY{8Oj$ijDrgAQ;*+{@42A(Hv%GW?9O$>hYcGfv%#izMaoA)~A=j zJ9*GKp!mt<*-qftl>E&!g@W{(?M}X@!H9{TNL_qq!tGyzew;2M*_l$F%eTNUBmiap$vjfpf)(fN@(JZsN9Hkb%tv0wNH zfORK%$}x+q$vjxENW(ie5oRAzKG9tR9L(m2D|{useiIpdH5w|;c}kA4Y-}W(i`P*4 z#;Z#%`M1C_1TTE{@V6J3-M**dM68$G>tM1r%J;#*l-^K!%g^(*?HkjVc!hMMSZzt!raB9*+n||>Kfsrkne~PKX&;F|AGPr_WP#qX- zyQbNnEG0oGpcNBb&_mwKdLMaJ-IcZz>4p_jd_ARxe@i4)8KMlsklNOcgCwnQmcVW|);g%4i5--Vz= z(@TeHS9d>dD%IW$s@U!x`Oo%xid@YtKgjWH#;|4)BLFPZT8Ax@e4bzaMbeAZ zd`EBS_YAgC8I+TGyM!mSDwrni8XM%zTC9-KKRWq9>@LL=1Qn+lAx9Ci>BCw15Tr$0UUBU6|!R@{7qfC(F#eE_)>Nj;m;Qsr4^} zDXh=!B>v{SLoHCrN2@)#=zX^7Cf7A<6{^NTUqh2Lr{*$i5K&D;6sp9{7g-p57ex<1o_l8?aMYu>imcVy@B{78@E zgX2BlD`MBp)B)ellgf~&@NkA=->!=z)`jS=HhJs!#`o7N^bW!c&ByZgq?xtR=)2!+ zEDVt@yaBqVj6W=IxpV(JUK;0Djo?E%YLy<*EQLA z(mRp@)TGCZzE`;Mbqd}Yk&T5jZ!ay~jIf~7pkW@Q=%FvJip>YHN@fRyl;#jbnbMs; z^F~Vjit$&wWNU=}$uT~?x+^ooNY|b+3g~#0s@rmoS(wD&wi;#2GM`z9+*g}3g2*rx zGO!~*v_agn4{bZw84sdHhfblikr(wjr06}9@F8YW%_|BD)SyD(#4iaj^BhLa>A=9J z#@(e5oQp0X(NP|3ppRx*m(z{R@s$ct;a7AK#$F4zpRZ3i)sP{5GyB|v^PN#%mBYbd z$}yeQ$?WEw;A%5d(qr4A=oi7v$(bME3$v3b`5B2(Yt8lt7LS@95aMQLRW|-?gtqcm znE$*DQ@@$$A`V))+5gW^wL|IV{E_#0=w;VTFuAPYmbuCK4F3WL8c+Azt*&Tm3hsYo z`D`?f@oh}A4~FvYeCG)~#^H3fKB+MJe0LpQQDgJPiU0TC(WQg!GwrtM7*(}WnATg~ z_#haw8I6kDcbViTVoNYC1k=cfEYNeU@TKnBq=fIC(|@;IV#_=a3EXs(cQw3{RciEQ znUMZMpkMWG{J4?c7Jr)$y$mkNg^;dw==lM%ME?AR0Nrvfqj2Rbv>q$(kLQyg?$B^% zWI>BPc9{4+nAN!DT{w3Xcf-?*EDvqIfV>{^0v>pjVxBzt-5Mw1Te?*EjqH~)xEG8KjOyq zE7Fkt$A!rpB|(CXzd~;a0~D*{J*5(meHMs$n-Wv?T0lVXO#(X`0?Y;h zk`;;Z0^Es|9YSi0AOI?&v8hQhMwhNbVgN9!ompJqQ)~~p{hpmthAm@WC_pne441vH zj6w3Vu=@UUQNG`vk*RZ<%jgG%2BmE5sdET-rbanwuMzl{sxCElwi#hK#$BmXhZ+l= z(A00eN0knMKl5|$x`AkN_AydGo2vP&v3ioD%@!wkOEpECr{ABedd3SKoTIcVl*JAz z&rEblFND4?piqPody!)FS?1S|=rjWu05&tzJ;H%v@MTf=#>QY=ajWEbh8O1LQ2cxv~QD$~P~G@`lkQBL47PXhg3oC-dzhc|19zjC)2M zmvyp`N`{mpWBayrw*1hhPSlaXQJ^GWlhF5@6n>Xj^sA}F>wL)i@|pFYRFSS2iGj!cai`X{ zO=y8dlW)65m7jz(zec~6^4i~*O{-A3d!r&rY4YMiKE+QE9#841crGXSEhqE$aAM~% zO?2w>e})!dCzsV!Gbyp{VX4Y{00xpH|Lc$~=J#!|<3=mE8@=Wx*?-NiehKLt(yFO? zj9rQ5t#kD`EHC-RDDaiHNVs#x_wrWGo(ZsK)uBG#bJ1NH zsgu>z`0l36ET%V&u39U+TJybJtNYxrDPNShImV%WzPo%_cTk<8$|OitmIeFQ=sm*3 zH{jB)UNk?iZgyOtK-ynbb5ZsAN=XHbdwF*9uXJgIEPpZyZb-#X$OGlRfV#pIeQ=Ff z;a-dE!USpnZ*T8W=jWHSfJgR%y{(CP5ieGH3x&(FQ2vwT@X;hxuW_8A7!nivT8Tbq zrElfl(~`n+54$F}ydM0NS*Ct&V|m<h;My%#4Y+Hm<6A%e zP8n88t8FVCi4;t+^$l7+VQuJ(k7)Z$s8~+hj`}%?ZDfQKYe%`G%@cx4C35odb`SQ& z>nDEfh<(a*w-g_i_0Z=zf(`^bgWT&QgkW=y_uM4@#<;y@ENVL4B0MnqEUBNFQS9tm z=QRExEvzunMO18HkE8d@lzJ})#mdh7tm*eEB@5YQds&ca;v#HJwGg?8z(_MI>(_L8mEgZV zON$vOJx-VwWt^xvOPJjZa|XOeU7#-$iE&ZpjUU=t6MM^|P4R0jkhQ6OJntovR^XPP zG?PwW2^|0^h#>Jz-klTWy_qChR`4SGnzp58v5%Yhlgte24%@WMr}?iM)J9#P@;J9D z0AMN|Mu+$t)*sqs5;B3R^`oJI_CcIsLM}o5fZ#yy)D1f06qtZdX{@OhZ(M4Y78|ad z0Uu=`lv27@S+K|DBCd~u*$RXJrKQ&kwE;IHtVYD2n992U5pc5%2#|i3_=O3xv@0LO z6F4NU_Ts^KNLYyGJ?kABwZWH^O5G-S1VU7|-N(ZQ6)-@evR9urkcEzm=$Lg`ivp!BzXg(c$%B zfdj3F`vc=&EQ?aV757dZhHH{M@)5NycjflD%45==nHhGpO{)C?9OV(abboI=GMS6#FfVR|UT zV!?B&dJ6zS!Q$fk#wGz~Nk1J0Jh>7*A?!-j=b30Tczf}Owgvj4OH_~SKQ`=cl+X5m zntN-SP=bz~u#KbYcw(5u4YyVS3jp=F5?!p{59Z#y`P{emjS3V(t4t;E!|7=K=@%mh zFsL$5$In#k**8&<%h$%oTiFywJ&gVNQINhi*H6$= zh=*oQxTtsTCOz=Aj(Er|5z6kbukcRcvw}%Ce4}S>p9*K4*E#H~dt7h^bRa63bOOk% zboxL;`n=(w7b(7glRvGW?mmnRKc8rNwWwp>g{{87vKORS*gqx8J=)~f;dEV0nn|qeXTzv7+PmDu zgWp1=Y5u337Y!bd$-q7kHSUF^h=PeO+=}ESbTpPHV(@~3)BAU1X)$1ZUL{_RQ3gS< zyF*!r`b3(g7dFr}Zny(w&)hZpKIp~-I>fv3KzP&Bx%Gi27}8OR^lsJrugh;h z??EQP+O?jKy+)j?v_F`dPy%p%q6}6I1!B1tX1?Bmg^FLPm7NuA3h0#uRdMFUD#a}{ zb=q{`+!MY40Vo|&bub4(c$P!-HaxIiJG#6MnF)MIFp_-~UUZbP><=~lxF%Xw`YB~< zbLm21LHWBq%e!uE6gVMCmIEv%>J=@AcS!c9KJD78;K za}HUcticmXrh^AzTpiVfNT-~zHm725L34Tp%2?5O`2s&Vb6FovwLh} z@^ce`-5_P1#m0NSlkJY#Czt{LpM;Bxw?aID;iSv$)y+s{QwzS{t+twlFl(1~$+V5? z4z>GH?gj%Mp`>Y*UPWG0(!d!;QN~c0h7sT}Bhh$wt~DL$SyOHjaWJ3v5g=? z_RSY#FGCZF0!tf}MG_M-fNp@&h?=nslK{dDQ*{*g!liAdUz#aeRjX;M+Fmhq=RYnT zSgK&+PP?C;Mtp&=0t3B3fE=|S6xe$mNeR$*Ykq60@E)+8!wA~3ihODyEK+VMD?Enb zTANqY08of3X(8&En0S4fS!)?9vNFr(74C|duwjuVP)uM(Ewv>^Vuu_J)RYuuJVLkUUuFV8#jaaf(}E7yw7c0gb)~fg z4jki?QJ{AP2o1pA*|m}@H<{~llvJ`HB{}wQWA}QVHD)XJRjW6)Rg`BXI`q5fw~%Yn z?9xhv0y86=_Eb%8=yCPq(bqBSgO4jqrDFfGcuLhCo|b|9`44AmJA#|9Ks__TxAf`1 zR1b%@d>)?O()C#%f0vlm6pWr3mcGgUqcRG=CicLp?Gt4QWy}T9;>rOm)EXGDgv|2b+nZ)a>3F{RDb~EBs07tUWu6$3@Uw5T2(C~>P zZYrcal0PxZ3)c?NHr!kq-MhR@w*f@vP!Kyv^9_yrtN$>_m|u2J)@E^gmo9IwhgQrV z68*@|G^*y2;jNxQd-Q&*L(KyFOBAGkU)qTa2B0=?`Z(%oO@_lm#e8j#e)AZ-t);ag zz_6{q?V5@ye#FvJ95ozb`=@ShkMZcoL;9$yD}|F$8+K&Cr~VkIS^N(3A8^TJmCp}V za{!#Nq;%HthO2Z!OE959A(rf8e~`$&T4*FTzjb~57jS#LZU10*+FNh^nPnhFQ9v1yjO|4sG$sUoY3&<@=p z1KV>y#@;rUb~9ibWH+mN(BbubW>+BC&$WU#t{L8&A@*A7As@ZD{aL7}P9}zUE1PYv z{88`?ZvX?7quYliF;1EeH!BD=SJbd32bTbSm^8BWuMKu9$0MK737Y{mj3b&-cZH+E zV$tijnQ5SlQm)ktUC~z!oa9~AOnT?3C8YQ5ttNHNeA;`fg~LIEVS{eG1KO%nSPJK7 zLOWVfg-U-bDo=(f`l?!bkEtJXM0@8hCa-rf@ew=Q&VCB0-PvXPdu<4!s~i6CN)yH= z13!3syAVHZw=Pp_sBC85VvXq{8N*lJD@mNUqgAlN7fF6N=CD@+(50gzR21CPbtq6f$^PPm|*WV?GcUqR+PRcz;~uYHPKZELHkh-~=utHr(6MOC{TtqI^w5{^@{ClbxEzMQH2Q?p~k_-_Anx{l}?p zbJ%;SBL<>cR`z)F6*_XC=S!(jVcOJ06_KWi>9fiBx|esj;{C$#UtFJzG15-&$Z!a8 zF2r~A9P}RP97SUz8N;Sb8?zsjc&{-+)TD)0eE^#*-oMn`)zXfcD2Sdv_joe^bY31 zu~Zq(S4P_$kE2ryS6c671`U2#@o0@uJHdq+ZHls&tdTcILr5J}X*tp2n9zsm3|g9E(!D@`0s5`3h2gP01bf7A%KAuOk2+x zJmu6DULjbJDCg>;ypkoi%49J2HgDhTeY(fT+klmlztnU1KlT7QGfmDv0}V3`N+q5j zggWmsZ!aPXUC?TUM&&}e369zyrW3@<`Vh@soq0P-|F(N!kjT{fiJr?%UP7?=QO&h1 zHNkFl9;ya=N!4=OPf=yAOg5#$ztzYM!j>&w2>#vCrt%|ItaIkbY3Fb9)nx!}S4x-D z%!w05>3)@e$K_^neB}A;98}x6J^aIZ`OaSG$NdfOgbh$X)e7J2BOk3=sDeU`*pI5Yzc;g7oq5XyGwceBku(g=-IzW!zYPQZ#mE@*-qa$Sk(iT!nOUb&#M9HYSr zIdk{^C0^Zf8zW67K4JEbU2mQrAKNrk-3F~{gGqLTMms=IQ>*vn4Z+tfDddGXmZ|8L z-iMc{)OLn^SxH)V%WInEese8t5TNvoUbQq6p<0YRZ8;*e!eE5S%6FzkvOCdoOrj~X zUW#@*!D#d$fVXa?@QycEvAv@*Q@NTs@=kbj3e2A9sHS( zU%#8GjkP?r0^>csV9RwOsCfN#oj8(@I>Oh)_cW|YtIwqQwXpg-)wS5>HX{wle$nvu zG$lYIrC6E(7@`0VIa^QPf^8X2cLU*aYZ}r3sPfxXhr}10sU0_#W}RQIZ9R%$rdsny zX(V>ssq?g8;%4A2tpu{mtdX0&6?SqE8uvfELK>9~Gc;O@rhLU~-go^}O}Fq3ugW92 z5@ZoAy{b4xKG@Tp0c2|gU_vtZCdVRAvpwMAjr&N`ArIPT@ZVhE8Kb|a)LA>to0-AG zf8U#Aif^_go;9Jwk%Fuu+^n+3eZm^K?9d%&ijDD@7qGL#TmlJxa(YtpWGpA=^b@9e z|7%f1GAMTZ+UJYWkj9;4CrDoZ=*Zcv`VPhsmI{l7>Zf;g-qaF3gDJLCCb8eFbn39) zI8K1Z(rD$4w^G+{Z<`o6eD}*u^pJOm1;=RQLh{)H>jMA%A+GPY*d^bbO^-m>c7^GGe2o$KV)ng{ z2%1ntm^JG?j@a?+&CAflXM2vawlHO#nxA1?0f2{whIE?lC2b$HgMgrZV;!$#Izr{F zT9#On*;w8eRbEy*t80UT;o89r>wZ)mC{8?cedc?tw@F?Cx7u98BRy?8jqXC4;aBBL zTQn@iw`cFNerO~45Y;B&l3WaSUKA-HI<8n2CIDTtQ4$a~;WXv10i*-dQ|Ua*aDHXx ze9I*S!~ibtrYpYj1tC92K{=GfER&9q$t9od#VQj5e}KMAFxOUEGg3jA;>)M+WdidC zhFcw)f`ufPyAj+LH72qYQ=`9nDJcfyrYE)11p#WZd!oNV-+G`SR#Gf10pBz%hJePw zcT+S5EN{j4j;vH4-F~l0^Xr=#MW?gw`Ozsf@N_tO(@4HMhlLMuHts4Jsw){5LQ01( zTv}13E{BHhN2_*>VzMku#lqq>i(A?-#Oz^^rgPWPWmnr*DbUs`;kwKWdi^6q&FFTJcN)kv|S(iXJSpUb48!*1W@2i-K@#Q7a zCoOF;`$SyLLT5})V%-sq(gdZN^vI}8oIv_--vWP410CKlr|Jj$CuZ~fNRs_jFgpI< z?j_MrF{5xj;V(StVdFf` z-QL^FZmp)1kB_)x#Di^!0Odz`Wmp*W<}pr8B$?}%%(_(hjDj=LNsCDavLf`RnV-36 z%@p^#-Ov((7~6KgSdbC|)RtD%2B9G3@fsx!1;E;@SO?yQlim?gI-Pt{7*U%+^u@Jz zYgG)Z{`thzgnz4%_cqpu&hFWwW@@clk3Q&SxxCA|&RJ66{e*R$7!mn?t?~2Zom6c; zWHsKMFxpRP{AA2cU4Swc?r#~X7Jh+qnYnINJo6h0KDk5a?E(pRqFS6*6CP{zT^28f zvaQIchP5xN27E8nGni}K;%u{I1A-Kl?L-qIDU^5HJhra~f0^;Us!CcJuAeY^+U&*L zA1H8@V7#O7?DtLda<-8!i1NqinTSjr(g&yfxH>HjUCnwD)W#}UL%6Ngl0ril4;hh6 z&&}230!b87#6F+toBJTiv9xkKGdCd}R&0hSk&Ft_6=44fmxJ=Lv0UkNfZ%mdE4V@EJIdnkYQzU6`-c+C}a z^}MA40$O8rRnmI#Vb-L$7rO!ZE)e2td0MK2s8*ZjQjE=Q^YsP*>E7-UW`SJpC<-!h zJ#}e?C6?YtO-hf(c;!WEYA?hoap)5=FGPe2T%uK;ZlsB@q|&8-&fNOMPnS8jA+EZ@ z6)7g8|CdszK;?y9fvnlg#%zb3)th>6dkB*>vB}&T<0rkUzi5h=AK}%e`-;$OGMNTh zWNI(PD#_S~LSzv@4G=W++^6yQTlmeup7B8OW4!sX?>7gON|$(a65AOYHB>B2wcNi~ zo~xEdzg=D%UXp_QSb3Dcu}kfIA>MV-pndvB;_;K;Qcd4#*zh=m((#JPaXiB3fd#x* zqxss8o}&_izc_W;cM#4F2jpAw>`1W5mcPm3&WRE;7(4MVicNo090qvaELbc}-H1JU zveQ9$6y^1kbR&Y_e8_;~v&TaFy(x|3gFVWxxowr}-TNnVqeYTmX_Xi2rhW1QiL+Hl z0lAIC0qJ0bmwkqyFrVvl?_@f#hbIneI0kMWxkI%uogyUun>T$k6uyP!T?+3ghsI8P zjxBiz2JZ>wquE=8|70iyK$D+u5zo3?K<|ZanK5sh$UP8d|5qoTdbkt{H>PeO)G=3_ zA(uBpW8hc2W)aJUr+4i`9+^bGoYM_C_oPG0X533NlCS1!%vq zU9A`qlz>7YC}4E;Io!E<7Ju;ecq8Ozr6tDgAiN&HWs%Qz`|bC=PzM!(hvbCA*A2=iSWk@|T%65J z@PFd2Yo2S4Gv&1Kk6_Iam$mxg2ei7O+t{kfF^Rzq&JGG80K94btWg4?P?Mbw zbF3!6hIurp(fgswO!m*&DfXMD#FnG&Pw21S*U?6}`L(9X+ancI7eUR|$WHN*P2^6D z=(4ob9T4Bnj*#K^lgrD_OMC^!rEX*2A!Aw5_zhD>8s1XW!^`eJ1N3rZJ@$N;EGKMk zTW963&RN~u^l-Gcx|n)*8AS5qDQAm+@iW4=Ly8cY%hwxOb3geh?1y;+hg{pVIc0nT zfai$L$ZZ5ep0KytEf!!r#0Bl^(R>vc`IMc`aCKhRcklQF{I|D9H>=K-#&hK2>YCdm zhW|X$>1)?hfGZGUsnWriw$|HzuB9>}D@+>b0uZ#h628DzE5sxZ{oEic40Tz{g7|Yc z7bp9ev?OT$^^gRE-EJn8dir|3mIi~tPa)8izHyJm*W54|tl$oK>1(W0yZ&EUMj(ia zT~E62GKCIA?>^_Dh!QhPFU?muHeq7w?WXSqjOEzkvBBPG70M5Y@SUj4zb=R(q-Cmb zGRCre>ggT*p)W4hjxJLh_P$9`j@IwV>I2|C+O`DiFUu42hn@c=6ar+5N%J2HH27YD z%n?--vS97>wi!E$rp!9O%IE5DN@6x8F8fcX6*oiB+(-GCwa2(et8K^jGUYG1dWf$(i{GhbHW^)o^YRrMU{E%lY1Z zML6D)+2$oO*_%Sx@&)h(>q;b17V*k`YHqca~7zEPj)?zD*h^K58 zq>V$yk?AP(vhnd{uqR*&*T3yc-_5ydJ<>j!$D@9ty-eJmn$A*+-rNT<;~GXEr~S&1 zxS3ypULWm_6~_KLu-OQ*F?1J`O+oHaMK6X8rXk?(ye?2jsGw*@YK;De_+@!~KqC$1{C)Wk&C;-eyVL$Bw z)J?7jjKDY`pj(n6k%Fr>;tym|^4@FxJ@Xu@_G`1N4XAw<$atWV)OTbMd)aA4sQvDK zF_EM_I^(QY6B%`2ReGxB|4YYlbfmt6W0Rg;PCY7owq;OE=A#vSo1E3X`g3~Jgn*q1 zE?7Sa$7dgM=vkrd7o5GYKJm)sWJ#m$sCni*Zi!gyT_6N5KHHOQoS&fqpc@7b-w!u> z;}jO6ya0sOd=D{t1|2w5@o{zuhdXb-d1mPd`ehhD3wyx;R)M*J^S=Vyo#1L_l%lpq z(A01^{a46cljt9O)5AB+Gu?p z?IkRf-;I-R#2)VxMFY7to$4c2rl~l(AK4LIw2qzii-{2tRm#Z%eXt|hon>(Z@+2fW7b$6UGTGRIU&+OH^j?U2Q zN-0x{E17kmKclUK@2Yaf7ZU(qRLU;dEmUHZ0y-Wsls^090r1=2b6ASAR!cB9+_@QI zU1yzUQ9<@BdwDVU%37E@8I#(z*dAiiO`p}H^^HG-!E#)dH87J3Z`sYPFQ zsjC;o9520WyE==x;=kO#T2{ZhSh!Nt6CcjhUzm$2`{G^2vg+q}pXl+>3q750u4UYS z^%sp_;j(=yE(`!Sljif2;?teYeC;|KK5Lh|aAgVs%)ETKQuOmkC19pvjUJe>5VoNz z1ii+`;bw?bHK`C=k2fac5S{Gs4o2J-L!1wr9udN8e>L)yGfdhuPu}?vCo`j7<(ZL* za9CDM`LN<$)KTR#J#E~_E!f*bH=$`HiRi?pf`mmNm9w^73F-8`OaBo#bz$vHBfUP( zy0CHv4Fp%chzpA-fRldYB^VJu@gzq?K+;0H9Pq-OkQ( zyoUE{Vw+_7=pznYTI<->UZ8W)|L1>|2OnnkPTzoRQ#nVB8_UyL(Lhc6>s2qN##zbYu~E)aT;KqyNK`R690mHu*9k7QmfnQA;t! zf4P+=u0`X84$EvD3;r}Yo1w(|5{}6<;)Gk_=B^1_cKE7|7~S?I@&k`Tvcn*NioPk2 z{2DKm^j9HIA3j}>el%L{&XZew>}=o))htz}g{$2txot-iN)p9eI{0zSMb}jqxk9@- zA~V3r_`$37oy*#bnX7*>;c(o;obQt`{et?sbTT5 zVgL^wE98+!XtZ(Lo&UgIunp?J2iO(FRe3bufp}Imj1wx+QJ3eX9ejzhueH}wz_?(?nHF#S zNVuf1aGhDO#1^3y?e&oYpev$(HYx?>>R@_`_;roW0t|hQE%K65C}+Dr<>R>kK+x+` zJw&-$uPh8{L<@q5Cm9r=+P~525zj*`0)%-I4M^D{qNhbmwSHO_O)Rt2>8L?yE*m?_ z*)T(hoEK~WaOTRpN?mn&3w)@NTxx0nWwZhu;_*D%UD}||eJQWK_dSygd zZS+C$P3^hX!%<3%?7#c4b&6YqgLE!Fo_v1{HBkE9;4cw;0=?dxi~r4&xmcv(i8q0>|m#}#aedpGMS#dB7b-^s0( z_;LTrSCD;$s&z+uDG+j2#ls+yBQ|VzAYq^~5fs3&;TywrK?1E2-NUikmKK$6zJNn_ zI~@PjR!l^AI2ko~#=rI(bsR9C-QqdmQFyhB?`pxEA8%r&%8FQ?@UeI(8a&L_=k|O8 zyFnJOP`nL)+4sCpY$$g?;7$I^Po5*|hCh?D%==4t>z)E%pux^UJ@}tepW$~20H2B2 z*gPS60g>C9DBFOsPpGUH`hnhruoef&()kfVU>vsf?Rbve9U9)m4Em+fs3quN(!}WL zEbGWx2KY^Qvjt9ij^G+BtO3;mQNU7>Z3pY>2wFJ8}2{P!>(44~&W z+Nx(ipyxJUbI7!P?(1D;%a-+CUviDdyCMFMF8V~4bTUgd#GbIOno72p1G#)0nbm{@ zp0RBEOlLK>ZT9OIPb)!w)q^3jP$9p&1Y-3pkO~11b=7;{@w!TC5_?tnA|v`TN)=8E zjDyNM36qHG9XZBRen!|0-Kbxm=)|Q;rGURyBmTl&s~=yA7ERYh2XF5lC{y8Ol8X+# zL0})5J7&%5eS?e4x5>1Xp&{wN^}IjMF~px zRz6yb$auli&^WbsIeJzDoC@f;UnHd-9Xb;KAm?|h4=bqx^J|n^yFdmkNG}JZ+mlZz zIwqO{K9c4mym7aT;m9@x_!n=9p8TloOSIhfmlc*kf=BPC&I^=KMJEbwm zq$*ReO(o`KkGGTiLpTX&TV|jJx`HUBTU)ffG*S9PH;da^k+Dzy9?sLTra>4=0SCBa zOD?i!rr`!7QjsG44W%%m8*#BiYYv2@Bk1VwW>JDa75I8?fe{)jrii}6_t!4UBK z65u2*MK)z+Ab^rFA5I_xu0B`J+Q}p7W}NPt^Bt*&du{db{VGTxm(ckS@-M@Bh3Sk-VzVH^+_=M^t z9n-f*`!=37S*tG3JP0`ES^{yX-!Zzhe3_Z~kH{HTVAZQ;Sn7D6#$!Q@Liy5X5Skt{J`Jot=p9agBKXuFNK1FLb3NnhXi@Q+``iR8;+7{c8W}T#C+g(b|9z za?N)%Jg~)Kt5v1~O3#bKHrN`R z8yo-Lk*u?yVkg0P;})C8@4S%~{Lf#tFx8F6ekag=B%Jm6*37o@IH9o-dwSv#7OBd< zix=P{+G*vMx&ILr{z|7=)7a)iB@jgRRVA}~=Tx-q5k~rY&%_U2cTxg|+I&Z#3BW;z zyS#hu(hLy}{5*+m5{$bNEgEU>@Rx9Va7r;m*I6ST5_dC7m1{^N>k8Ef zsCmHE)`mOQM>D^QmIf3;+xq)sOWOw|sS%xX5}@1H>NCkp`5um1Ge|6)(1w}9X+yS# z-CdNP=_Me{EC|>Ln@}xwZI-5LH5*_z9ms^aP|#NNc0y@XKU;4?3F+ELkQ_}{=}VZ3 zT}+@3w)tdsIv%`67`Jgzcx{@odp6>4@ zA40wMWmf}nAMxceyI~M#R`xtn$`0qJZPifs5S4Y{&OBclyFCG2O*s4)niu%K+A7{g z7x;mXUX}IWU_J22SZwXE?(o{1Qsc1P{wXmNB&OhOeE6EE<)Qp*e!du9;pEcV=QH2h z^~~Tk-U2iYozr9qi0Nh!mqS#C11kwr*X&8gZBlJ`h>AKSekqW$r);3pUWr2an?Yf2gqL}dAj%Ipr{Agb>Vx3 zCLJB(niM&m{&qefiZCxU_5 zdZ7{`$jl0-0=WQyi?Bg07?mZpBG6GgLUiN)?&mgZP+)usm|?wS#owkeYL-GM zUfU(z224;oeHubAr%qDRge{oSUJk{29 zsxZ*RTKK#}ymKt`^+=+*#;CNp;YRDvZ}zyYU|)KZsUSF<7Tox=5sf`N*_%=wN0HS_ zG3N>~hX+gS2Dh@&!O&UeiTVgHw2%Fcqae1-NixUHwh+Yb6)GkOB5U~r#`O2rE@J@} ze~@Pu1u|vwru35qJkHcYq{%zuH1KXrxsyzWP+1}Oya+jnq!|`afF>gFlF^5jgAET^TZ=njO-_L~Xar(el(H{Ii>HWi6>)qzF z2T%oH?f@Tr5K6E9I*<*!A9a4YeSzC(6<;Sk#@{XIaWwNpkIwpu_H1A$e>?UpE54Q7 zx9vZ7S9Ka+Uy}AmluiN&kL+WNEe01gZx?3pp12c(+YUyBF66hH^q8a;u4JvkFGi%| z(iV_4j@M%z?4W?45rilLkWrM^*vqS6p}B@HAgoeHZyj9l) z;L+o5PAD-jp(ZWx%FHL1jLVDZII0kU#!+gJc*H(x*aST{j&4`aip?hIFwc;^55+2q z?8*mQql5fXX?@XZzA0pJfQF}!{L?3ve#K^9jW2-ym*=_kUul;mE`{Q<$cB1p-~FkR zcJCY1DdsG0p>p_(W-U&E4DCfla_#`F(9f9Z^|4={9#y}5zPQBgLY_Br7(JdNderr4zN!obYt!X5RP^WC%<&v)ZE_2?`3 z$i)S|X{uxeQG2b0niE<%ZO5!Ow_cbwuq@4l>OoZd%+k9Q9$kLgp*-T6h!|BOIgtw9 z2z^&t7hig2g~anj>0?#Xjkrl~hBFMf!h^g+Ym>}yoC1KZ@w_`=fOm>TAW%Fje(D;`~AP<{+w&V96C17tJ0x&TWvLC`R zsQu78^Z2Ac)p?O=+-gZW5bW>Dv_a9`hBJlik2U7htN)2@Dj?27o!wKP1!Q}o8%$_= zEnis$>)kfcVA5=@?{KYsNAHMxglm`F4|7zoF8J=p%d$0bS3k<|NqytAm$w+Z5kvQU z_^NX$Lb`bXJdrvwvp3w+|!jv<@4I;;ZLxj?Bgmq@Id$f&NniG3Bp zEvohYY#5jM7VaGB+f_7vU5pZZvAd+LJWuw>vHy+2h~I;w_wn+Pg;MfaL2{ON;Ub8F z^p|4bG`FXvNk;Ah>Hhi{X+J~SWO8H!ttVb>SM_mHi&vY2B(6~y40p_FNnp=d2~wZ8 zKgQmsVRHNIO^)E+SgY1H040i%e~2U6ki>`9=d82ukR;=r6N>E2>TqVZr0kuMy;sgYB`aSX5;8(ZWo3^O%7`u_ zoRJmBS#h@C{r(1T_j$kG&)4(u_!UH{Jw%6|lAbH4e}#SMS|qjG8umBw6q>$}-WKrS z(PHe6F}*pIlG<=+S>4DF*-mA{r5iL^RmV4H$r#}7)K1(|XBOiqm*B}r!E=H#WtG-O z9#VB>#F|R&%Nf{yRIpwvYcNbtm(~5q3@&b=Jgb_$KxJj$c^brY^{F;Js-P^39fVw< za6}}STj;ixd0`l_1hZ{dRzOUz_>e9zjyqQxh2{mQj`9uJeDiUh-to@R`z}T+o%;l( z0TqU5JL=UpE%EgbfZalRVi;P+j8MlO!EM@Cnr}@4`n&yLm#qXKGT&3c7 z-VGAtunAX^U$?jZfu2jc<+zD+a3mwEC1NfVrzZ7mzoXXW+rthtyqIG zN{|~0aDDGKo2FbTwzj9v<$6`97c^ArV`No!0N~2bTX6Gl$&KhIu;T}My|&hMYxTP# z|3cR#wBpYK?83>@@ph=IcdhI*l-aV|4N{B_ZCBWc0=nAL!X}<9r?{oEXQW%C{u?LvK9JU1+SvE zh9|e#d`P#Bei8`-wq><>@#TU{kjL@FIk$C6fWPtTwLac&z)7&1zfr#Wm7bUj@=ttx zmw2VbOS)&{bb)n$2@mBf>O;K0=N?8rL2?&w-HSu&!*o(dkm_yGkM&2h<-_TN{@7=J zVg^{{Mjh=e`f?j3EuHWFBQ(u>RQD>rl^ksR2A4@`4SuuiOtP{{TlBQKW?elo&G8yU z!JijS1sFm=R13Yil)Xu1Fs_l23D$GFVu6GS@XeCTlFMxl(kveOsZZ5qW1c_F<$%n{ z1Uj0gQ$W(>K*D&!xwL-tF->yR)sndCBS;Ly+Er2nCw6U8P~a zf$Yn-No`f)0kCi0T;>7S@6YNaQi}7(elvziLrY)@^=o=mdISeWeyq1l%JL&qLBvCJ zdD5cd;2--W|C{cn*sL96iaOpTin~s=4w#I-Rn`bu-L}uqf8QD2y>>^;QABm~31MsR zYKmQ-HO(pSSS{)3Kk$#iRJ+&@IdNf&&Rf%LwO-XBQ{mrWcKmJUq6}^IS0QKR@xT9E z!4WHlk&k)WJi(ueQex9*-`}Z_(`D4i7=q{aQLikn`z!tQ1CwTg(%x?yRp8hEO@{c) zUmn<{RX8=w8D#XEW=ixd_Pg&*oXTi>imB(~>M?~R()bWdc9^HypI$CE)12=GLn@{h zvjS_=6m#b@ewgmv^c7#l?5x4`VEQ|6j{GrJKGThe;JUft4>Iq9%)H{&G|>YF*dV(P zrLf%m;AO>EuZvV|N9;N;Ur;4fmQt<;D*wJvN}vMwLe^?rQBEVr2vh&}RAa~CmDf{n zkb;IPkby=q>p=b!zkK9pIG0#N9~Gdrh=Lx*m{Sr~*L$D!mHso0FTL$)lfN>h@^3S0 zi`Y1VHGi4vtbdOgr&>En{5(W>)4#hQ@hh(Kz1hF)SkoJ~sULgtvvdR0*j*feoUG80 zv^O^l(E4gRStxdPDluaq7feu=QNy`pxj6NG6_vq%87UZw94iK@BJxxuL&zlbm!(?f z>&hhhST!9QfA!dpb2ywFCqxM%T#Rb*;s?hNJj9j^AJ5OyBlQ?wCiyP^jH@CSus8dC z=T*}wE+4jjx8pEG?$dfay%99lqnf?c66njt4-y6@zs~}$65D8JGatFK$|2-He#>xd zlZ{t}!g9ge2tIW|2m%a_(0`)I9Sc?NF?0)}pnS7U0lgy+D@cC}q zJWBzItskEsLczG$-EF%(xz7H`v)kDrS(?F5Bp!$}O=6UHp&7 zHP}u7)u~zsZY$Y{xZH@(y=zz`wbrl)miiPKYKJt9Cc1`COp(Ti%#SVNt*$Y|i zK-xpD`+rk1!pHxJz8WHuooQzHTkgX(T2z0_ z2)wOA3Yq)T_X|Cc5~Cfo6#Q88l3&#LLxqW$^y8*Q5P(^q;rPP&xpk_RhFB3o?M1$(VOvNxP+?m&)5qIN(eP#==RCI(p!rWZH zQA-3%sjD*~LBJ-Ef;^F*m3inIwLj#If}=EB?JyE-ZKbd}EYT&zQ1%B_wE$y$COUwb z9n#Oi)>KSB_o6+ekMY-!%%X|v5OQ`y~EVYp~}E@rmxQr{rZ+0C>A=}Gki`{C|Q z=2*;S3^ET>XedxYh>CJX-KznA>8Iw}Mc$=1Jm7Q6y_C3uaB_TRF@LTWbGf^JoPT+g zp7hy$KA&u5*RB9~oLzmhiAvFh z3Q#LIfcDU$J|ln>@Hhm)@B9lu z1oZXeaN%Bqv-&Yo5HV##my^}=5PEU`{5bcciJBmnz3#nd9NJwOh_Go|CxHO*#_aUv!}a7Xe9QMVDrBSz>QW-KR4m0&!5%ZarhmI+hvWudvMPuN5I$;#a3)ZIrOBn7kbx>0S zv1#0Zi?^#e1D8R;Y+IvN@r^{6uv1q#;&7zo6>2{@qD_e4I^hmOY($+RI#^U~u3d{s zYqe)6mD>Rl;|*b*qLS?k*#6Tr`Cw{SlDc+&U<=k0;)l*z&qMUionLF-N$|e3A`TuA zEkcfCttIcv%T6-^e-MLAp4D1)SBH9Cx^gGxd_(qJH!j(K`DKC0n&eG!sJy|rIcmWI z>Zgf8c$anJL{(D2P0CoylM2__yC|IKA5=_@KB>K&6zDupO-OP$GP+#9fBt2BB9E7@ z*jjS!y?is-a$HdGitY$cd&68fww;VyFXi&!U;UF6aLtJJ#^q>j)pSIK^QpDn2xjlG z9%k+LkRH3jMvzdK19x_Rw7%HZgs&wFy@RWW(*3*A-H&C|>nCF0bKcGbwm-Vk?XG-t z!NnH&H?V~opgg9?D!hBJ_iMTjwLKRVv>4eDIr>~Ij-$c08qHhbWb@3+tMI8*hi#5{ zGDiMvcb#K%XXADO1@K_yNmGjZ=o55l^&f@)WHzrH`aaC_m2scnX8B-PC6P|7B}8_A z)RKtD_h{u(OJn;jWd^~1}%l3xxnR^7nzXOVZ?(U8QPx&W!VvK`m zCan*3cV@wF5E=8RXauFiqLRd8!Y)#tG61M)-y2>{f`mZUxyA@-$of;q=7`O+h^zu# zMe4D+77dt&@4gLl9tJap3Kk3pP`<^3Ez-jL__<#tJI1d~H%zg&R^Eh3Qr^v!(y0a4 z^pRtMb@>4i84f;131S~1>aMWmii&2C*9Qa@M!g9Mj9$p(DA3 z5jk&bw!)qtHWzh32Z)62yg|t_%F|Auu3H{LqqSNLuru;q;T>q!>^^GBfcGx2h~Mxu z{Z@%0Z0epi-NZw9*~6O2{h)gSX=birZK@3So><&E_P97+N}CBvo|`xZx=vQ?udI)0 zRLEpy!b%=r9{rkj`S0j#Yx2_JkI((*;b%sd7nk2|r3rRTPd>2Er_l`v{ox;-gD}sx zxPooZR{`LqZhI0YeiQorP=dA7*S#SQZzuuGuCF4ux7d>@{He;xE}qS2?p$7xR#LR5jpgEX#EK;}4ZdcvdOQ7bg^;z-TNY4XZS6e1!z^k{!CjXZxg{Pza`6wfoy zO=1Q zs7ul<1p9$R-m1BCie?dP)+Rpj8TQk}LKpb}iVo02l#<8lP?r665Oq+(#B`t{OW3nF zHg2jN+lvB|FOV}@PR|w6?S+5)@+vZ7KTZ7lsXOv9HFFSPC4Y#cs`#E`AYT!yWNyRY zqL2I8^zlxtbvBz%DxTn%M|?L@_wV;23;2rj#@{qge(l5hL#^qJ!=px!J@hx!L^67W zri3NAPezA79^6{3uEuqx4qGey)I`lGr%U~_4VcY{-#X)Zs^(B@%~a!V;59O zk~m#OKVccbKN|x+R(v@2UC=R!A;-nNz5L;?#`5Wg!ZqQZk+kTy4BFq4vV_49V&iyO zP0e>-;~%EEUkTLc*KJ_|ioC250ntym06MxI$E*2sr!7DE+4E#!B$J&+dBDJC?6Suk z_GE&Jql~bDb~F&z5YxaUO55GO%1q5>y8kGs+ExYmQ76P|W1^l~<933(8T!QkS#gXlYpR@>S&3R^}p$7jCHFP0d_ zKYP8{OCm2KfSetWowZZ3C(Vw4v6cV;0V2pgI0J|Qa%e#30PefE4=}^oWW@u&QIB!v zrb7T-xR9VCC|d;qyCo@4cOvlQ$kUoUsTgpp<7|SW*IAjVq-wc;p?n(g2Hvy3`R{o{ zTw<#b?b65!GGreFp+d|j-3i8e#{>QnKZ_)yMeASJ`5@~jZibCuaAvt=ORTx?{zU#U zrM05xCw|qB z6a7!FD8J#m(uxebF1DJv?O>S|yZ-qRn~U8mVHWsOAWzZ`gJjydhjGx%9eU-|#k{-; zW>XzHfd_hIvWV-&ZohzJ&dZN&(ff@P;Wq#%Q%1y%lI*;Oxsol^0|YY-z}w!tfL(}* z6l=b47&#!G9H9PU&N4ozf|FFJXHwUJwerz{Yp@4O-%Z7Zw?D2Gu4u(Ogmm!<`|QIO zF-o%9UrHmR6=wxf#pGS#&l`h4(>~rF5{LUhmM7_sKO%aldH}v6u8#37&H0uUHp{Rc z4VJ?77(>z=RWNw|sh)Vzu>tQj2qr-+Lr1;->0C+DyRiV1NuR6qw2pv9LtPk>ZYI%3S-`VMm>GZ5KRr=brpW2lwY)J$yFa2DL_= zU$HZ)Tv+=++ZK=zz4!N+ry05Y#^B_jcsi1%OGuEwbUQ?#Lk9o`55((hh!u?^#*?Zt zxWA$Vfw|4w3Jh=df%@j2|LCTRtF{axurnlV2|b6-pBy7Q)&R@{+f@w1s8inHXYv0$ zA94t?|31CQPr~mm9+hFiyyfxyZdCl`RjreJ9Qi?PngEz%mpw{qgt>6mrd6T3YD1$v zP?e5Cn#V23I1sJ8h;w$65-9P76@R9#<4|zM;GlWyN}G6Hyn`D+!;zu{$r2T=jLONX z6oE3nYaiz_CrgAU^(SS16>Ql!-d&tNEI()z`d#D6k%SBU8@&N3#uolqbI?ia6%*dF zvGTztgz1Tw={NJ{#O;6Hj=1wXd!6pWKGLLL;R1-Se~pZwV?1 zy`0_F`V_?Uw|Ij)-PBPJ0I{c91u(OQ+>@uFb$C2U_v+8{7yuxAxgf9U)AQeUl2Ns2 z?wIKhn6L)w>2yopAI`Ed7y+cK%GpGCV z_VAJ&PNk_S&vV9m+wO(kuIi<-|L$8HSDQ%-odN6oP}T~#6L)9sGNsF}$q>}kEEOU7<_h`qCp1J_*C!;Ds zDj9^yeG(rx+A+NYrp`Q`DU^hP)5jK{W2gYadP|uBWPm;ZIWWXpNfr7J5dsppqX;H5 zS0NB+(3TBO(^Lr#_&Gti0)f=Q8R2HKz93(zHWe)nB&cW$+agl%VyM6$j6`ycSr*T9 zGzxTHmexkZhu|kCt#ps21n|Z6*m2^tm1l!tPoyI6n#;EohZ^bs+J;;dZISU+%_<@Y zXXwv4Rq2XHQ4T_9IpA&p^H`uSu78CQ74M{Z_{jIw zcVhJkQ}oEjNwjG|`}mXlcUAvf*p!k_u6#aoVJh5Jb^7)ij z2tfdv5K`~&3_~=v9;dQWuxGDjL)nON*SM)zxgZ^J6MP@k=got_n`3g+$#KoaN(t|% zP3;whQqnnYAnEIDt+4#J^E6u?6$wkYD6ewyO;IaTFm;KE*PG206X>})q|8upd97@o zuf~2>B|QNQX02thgSC*`?bRivyW@1MoM9tRS~by2rN3EnAw7os8Dhm~W4cxN>+pZ4 zf8)V|;Fg4zChvA4PWmMpsI_f1N&N+m$qpklO+?^XYcQdO-wP_lek6qnc2#B-{FF0L zdJ3Cu#Co5~413jb7q$BkmFJQ(%L^MyZ9=>vo{GqPS4x^T*UkU0AR$1vGE1o}(8ex; z2}JOeP19t*;r9_s#!gHgoJ6)`vC4iopM3v!a@-%@`nRJ2Yef)-2{Xp^(hwTMUwTP$ z8VT@shACd0tm{q-h>qyniiXya8KDG9aRZ>S1qm`!~&EK6k=D~dYeAg9=-!`&M4T= zvg9jtONmc;LX*La!X#+I{q*2sGzBo&3+FiQ`vzKC@EaT4qb&V$NdUgc$AbVxbGWuH za1&=Hut1teX%r3vc5X*sw5+yV?z4(r-q{I{E_{o7)aE0&g53za45F`5SO}GU$T!bF zwk2|tD^h;`8vJGG^_kOrq5Xxw^d%}g*Hz-c$$~&ZSXW|}bAfpd@g)XxfQIfA_xa0) z&<4OxW-V*OmEV-Rdo)?8>ws97k+0z*>M1(ze5N6j=dP#0Gm{b@s1X;wxr3!um05V) z(pjj7t=TmWX6-HSEA?uDH`0+4`DkAl;F-;EN(`pCfe)3KD5?7mpMs`vG{QY!0B$eY35N68B1GI<#qoo zG*^!fgr1%WA_#X^oqFE|qr`gB_uKRXY)6Z|tsL*^*Fh8VIN4WI|83#>4^AviU!X0U z(4v{(4Ak1^E^0&T!}cEEN`IQ=3Cr!qfMF$KB*Gr#cr!~T*YrC~Mnd*e(}b;0j-(`I z#JmV3ctZWYIrd?d!FJ$k+VDa%K|Xr2-#Z7uz_jUGra z%WREfSSd+3p@TW;b!9V~|bzXj~qz z8V-QNbgxNMgdl=$p8sJOP_LPl_IzQ>aJ1hKCXOzUIy5#mg(h9Nj5D5}6|Ku2nvU16 z1bRiZ{_n~`&o6!-Du z-m*WST;<`}!+`b!lgp8%9}Y~NFpq)BpMPs_Nt!3zH(Nu}l(ac5s(k2(K74Kz!SwFA zkJ6Z+6l~MWO8J+0j88H*h@Ttq$;MU)s87|;Ii;Wf#0M)4fl*dgIznP}KrKGMq&+Ab zt|^Z2(R46tGf3#5qjqvIyxkFn-m!INkGm(azm5yLH|$v(He%0px8-_0S9@jx*!cUS zTQbtZ+OMQi5HzmA0xh>0Ibm8w<~Yig zSQv1@s@ks5?rmWqKR-qMxSKk(|B&*!A6EF(TodWK*`DTY;ptR@lVg(EpFZT-y)iE1 zJDG@;$2Qp2S|AJV-xFR1DZH|;_uUG%nzZtIYbP0f@-NKi?l2w+H!dcNtrc2rH|8;! zihX5ioC(jnDmW5=k&0FvV%yJRaI`anL15Pm78Hl202otn&-G{B+{4<&B_Bbrn*5`f z@&A$Nvb)e1T+=Yv1_4KzrjqaVGBmq~K%M`DI;khro&FtBtl(q-fP9_*lXXFl{RFnZ zel3~{W#Zt{-J~Muqr|>#9hgU0#e4ZoPR?AII+d&uC0cBghHY0ec`2` zCjN4W_vg&3i06_3!7geg7O4bNp=uk?tKCwfM8z==8yy3*(6^3hg_B>dJvlIg0u()TQvyy(!2&)!!sg&oJ^K9}AybvbL3=Z&%% z=&S;^H~Xpw-tX|kcxGfobuCdsWeZ!Dbvx(5mKQccukyg{@?X-A-pi%=%Qt2xB!$jL z+wKu2Kux=N3yOa@yk_zseYJzsseGSxX(_Bw`*o|=b%@u@;lF=!2KfE`eNl{^t*r+( zjcq=qDIaa7w+=m>YxH7j7+fi@8kD1ynrwQL*aRks!_5+o%-i9~ndsjp)|L)GtXw5jUF?ydpo{kkv z;Dr2065{c0dW9iRlMm{G&)YgXRPUP|Q3w-DVa=-1zu!ge`=%naqmPb2-f!(X9zS-> zd+-fsnRE@3e~6v%9-GZ}VP6FMYP~cz*Z6~mJ9rz6=W8s<8Oc2To{Y|fN;Nkh@{oFY zi=nUJF6z#I7enVoUTQ;g>GVde+Dni$KFt47iJ?^cG~?7x*7{ZUBBQ$A3hQ^~Xm2OA zS?F%t-Q5`=M4RY#TNuD1%cieMkuPYs8*(uRb0W=6>12|ETuQ-!)hiVxFZj%Cf@zC+ zZ;?22jDy=B|5AW$YO%HIJn+@#9j1|FaxBI>ktdXuRBiVsCz| z6~F7SzD?apx8|w8Yx;etjYK34Qlj+Ds7~7nV#{s3oKcdso`q&9WLs&@<)FiRW^m>3 z`1tE=YfR-fHe$RcE5do_z;d!YOqtprJIrTTk+fDdT_rERkj(Io=`n=)Bj5#?sUlqp zfaJMtSrc*r<4Fqu6gyB_$a7;jgp8a{K?%IZ}|zF_6-ijU{qB_= ze|ck`PzVm`*FOW&X-i@=fq=je;01oB2lbki)TAWNB_n&d9xnW1PO!PhCS-M>TGRPzEC5!8@z;|!|ZFY z-^%q9{QUQP^S;k|nCa@*x_-4*Rc%MPeWQI{1UALCwp3Qp zewnc(2es4w#VwiKncJ{!$aiqFFc;DDSA4_Yae6p2e{tBJDuJcs#(k?OZX42{vf{y3}gH;hVe(+BF2o zpEJF3Sz}RG9VI~{C=CqvY-!qB32&aQnVfmi!~YjLUEEj`HON;n_4chDnU((SuXFq$ zt>8+L&Y(thpYR8{Bf)^_$s z_PV_iQb%SE9Hp|-1uDQ{h)qS;i(yVWrq{_zyLYuYByj17?!RMh_Pjxqjz%h-=q^5F zCP{kbe(x@xo7`MH%d`nBk>eRFTS=$AA_2JN2$02cq# zcDhKFzrz>*+Jhq`e`~3h2mxS=_JFdc145@s%gA6i!GMXpKtl2w_}NI>td z46a4_R5i{YQm%#-!uh`&ZK;d%N!TFc3b*nG@34`W4uvbAB;(1bdBQW@d4sU$p+}*i zp_?SFZee$7b79%fAPozO6Q>lqO#w^}=kh9P zjg7&Uxp#yd8|JpVaZ?d&vdq)BYa_4#DTCK)n_gyKV!MmJU_nF@VfeicyQ}M*cP}ah zo&_~Z3e#mM6M0%YVdqmzbFsUWH2?g}8+ME*3hW@_cFJ7EdtfQNkipNn2T}9nM|Vr~ zIiQ~haK|a=$-$GeGiHFx%{r@QLoGow<+Y@x-J_kI;w*Kf#0ezeTUJ7ALr}ZTSi z`E3~MGMy{8Vfivw`GL8D7pUvB#)>#I&9LbIghLpNS=$TE6Bz0L(jcBbrNRgXf`MOc zREEJl`mAqY0KfVQa<~E@{SgP;ivGZ*EJX){2_f+O2e#oh~s>s7M9M&Pc@p zk>QZd*aV8Xxbn6#v8(Oqr)Y`gWMC

`qPL1dRrc`~jA%PY9 zTMxN5L<@rm!}N&ey1sZ=sKP%(x|I5CmhxFBmLn7C_Y_&I3St=`dMGQ!0pPBP?cGC>r<>syW8R|*8|CW;>O=b( z_y4ryminqz^Z#kQ*l9n@SctfY9En4y^P22OU7X=p&myfx+05P8PfnI~9$)cpvPanuD|=KG8EZ zF0D+Eh+E{sF+q`f-)o4qv!hHuwkO#d)j$*Q}(?do%%F>ywABsf^i7m zzrQhJH8t5V=^Ws`mRQF*y(ILL7mz4C(SxEZVZJiDetTu}bY(vRE83zV7NZ3F+5= z2Z?W1Ac|Q9v6?Nn7jPLy4zYz>pG!VVhd0ENME9K`pV%W%;Ygh$%kNxq_E?~af|x2uQdTU5qtY-9lLP{H_UnYmih49K7*qEZiDhH})(Dx~W>S4u zKL#nqE)%~{s$l>ne^YciN}G!XBMdQt#R=IU121br8J(cabOM1ufQ+UAC!oOlFG%$7 zbJfrGit}9?yJ&BdS^!G1-{!+wqtST9^&eiVAOHf#oxYFqzQ)^u1f+CU$qFBRa#Dik zztH#`NixtF3HigX-XFAxuqx@#e+!(GEXH0?-}bol$&;-!;Q{S+e1>Lw zvaW*6Q}e2V4%}Tx$)~X91+09f= zP<%z1BrDUH*KMD}k5v!jevJWdOPiJ)@vEV@W)d>Kh7Q#cEP2MUW$Gd%YjhK zapP4p#q}lS|1)%+@oaWs9F843q19-Oh*4U#_Y7hytxqCSUUT`Q_xC^W69Szb=sRVB6}^HGR5zh6PXhB`68xOwo`O3Pl6K^4B`3H(P)kvt;=20wNy#|mYN zW-vHFlRT?~OxD76C*@=106KKjFYDWnT6fx2S6xyMIaez3kPrWHJiU@X9 z5e+R;XsW~sAmplNSzS)6GXbX%3zUt0u_$N78~lf23un)SWlPK~tH`7{$H0HlOvAh%=asTw3h zJG|)RUUpmO>{^;Xz7zFR((LC5*Lp1#QGx^@!91$$hQ+`)&;GB>IL zr*F`84ZGLC3}uX%_Ut_sThr*i8D)KvAOCIPi(;SUCMk`Qr~B%M0OYVTZy&9K%(NAc^Hl;(KMNp z!IhN_Qw@z(`H6ks@|n^CrwJY&dLB6^($Y-{N5sOcDJNf3DGMrt;@MPHetrR`ZVtrg zLw&yRY(Vx)a~U>&)j#9f2REopzO}0mEXj+N+%0V z!q^?O?8rqaWP6i~#uvS0;3DE{z1nCba}l#2qbVLc(pH&$yxTvV%A^Bx)C2Y|=KCgA zmwK8$G9^X$7B1aaH+7E_MUy+M3&MKoz==ItaonUmly_L-lj-Lk0lLK}#2z70B#S8s z{mkRPVR>oBQ-WKb%A7<(GUbzW!!DjI!!LezjC^`L(XWi`C)`BxTk%KMnaQ3NH7n|--%P@dXn`a8UGba zJcNqhg_Mjm@z^L=MTKAWX@%oT4*QTJu0;gwXKXM5=05`be{!w%?C2-v5M z>Os4l#ZlZ(;gNj(5M92%FR zKl*|MAKYF)WA)U`kj4PaL|7thZr>K|d@fW%4%a~k^EV*)H+CjMk#`>;i=tZKB8K>O zdGo4@9OgIP02&@TDkBk1#;SkBMKeh>5Dp18U{?-a=ydC@CxG#XpTfo<7EL4n29|28 z%4~!|>;ouYmWdy!$-D~nUc1fVLb^Gj%j`kh`V|nBjM^YP5H`$J8})u#+{UUoQ|zUM zp`G}o=PK$&e-y-y@SAK?=PzQ>iS=o_grHbzAraS@J~5jmwRzV~)X?KIcdY!Idq8_L zREs;2=}bu#e|B5QKE=CIMntl3dYlgNpu}4UZu-#*Y_CJfrFFZjn|>Jl!;}aDED?Oq zBA3PrcvaIhx0dFzKW3Hkm&CgtWfY<$y=jd+?wJRQ?^RUXg23;-62GG*EnB1jsi&pD$Fl60*c2y+=5`P%NGm)I%;3iSpSZdX|SD({yMYN#T;Jj+z$j zuRfCUH>)A#&6B?ax*IAmA7APZBgEI+&+tdq2MU)U$nhm&u=* zmYQ6HQiI7^KR(_mtFhPrx9_McsSi|KoG7%@S9_cA(UgW1%F%y~?9}{Wj;v#5B{O1T z#5*T~AyIU<^ghp*^*(x`V~&1$e7^Kj{=IBi&MM)=JJk18DXCe;d1ss{-~Gf`Epj8u z!Ml=Q?R0u?0cq?x=IP_Klm(%HJOByXT(!~=5WVTflwrvUmlMaOLdxc1l@-j`B!Po|V1#=9{w5E~ z^lXI-YQ&b$fe3q7`f;ntOtCI=e_uK2CPgYJ zD*hU3X_$Af#AgQEoD41DYdREn_JEL9YPEHnWEshzS8;83Qoh8B z@aMExS{E2ZJpFUP5jfN)e7F?$X-1A>vbQANDLVku^50;U`fg>71=$l)nKpUJmAk0w z_*$e@LuZut%l`mSl0aW7={LaA$;m&{X$`lhm|l*?1A5pb#OoYMmW$RX0fAoDvf{LA zg{_)rW=Y~qFhZ9dVI( zusMMt)n%1}BKo-rV{`O%R$~=K{3uZ*lf`a_3^%avoq0mHY%@BI8{tr9 zn>U4OZCU_U0O@@9<0BTo*7;q3)aCSU%e?cGn+pA=F(;#<>|ni+HQ>#RMZ&+W+F52v}LUIXLaO>g$l+jdNm%JFa~kJnGuo9(;3s-F}a~ zK>t>JN|M3-!@=bXwV6G`0o?&hoy24d-`(bd%+!^z;3xjwq#X98ZqD@TZ8_3!94F>J zyYq9KIUwftGSeCNx0Ni4*49~1Z4nX8Wfi4`$yfwVRxkET z_-znFhsHejd7g+TshCka@_eg($psKGrcaL6MhQ?b(Xn)EJqVzI=r{s^M_=Pa{l~Gp zF+Ey{_CQH`>VfR+PmM06QJrk|4b@Ifc7Wg~ozr{sd%pvRY8Uo8=WtJ)^<3RGT8P8) z+$9o^@>63g)pd{vMSw=$BQjoXZL{9TQzLV-)G(yG>FL8qzWwU)2Ntih;$z<`!mXyP zzz}qHgh%zAM25~hHkrKPFz+wfz~~UVW0{0#4itEEySJ2ig#X*4(d&hxyNg!`>;>sSXH)G|<{)ZP;j*E0T^-jTTT)^Nf%!$nWcp;S1R5Ix zFAjcS)QWJyZ;w3Ev4jdkWt}_rD}i`IUfafelmG9CLCJ)n#(08D`x|nw~_%w;@S09|LM|Ik#Cd z(g*W==Pj(0%2v+Vh)AC^zpw};B}H8arMWE4s*|fzTO4j~0f!Slygm#1#wLuZE%22k zzPsI&tz4!VLU*_H~m&`A)MDp?d5h5Xm=t>Z5lz7)ldQwyj_m}4j zU~w@2$=RZqw`04Zc1{mr)E#SF&*>wK#>1XyQYH2=Iy)9(M<19!|68L zEg^gvy~A{|PeopRx9pn<%aMy25}|UgI-(-aPNU=|AkmmsS{M^4knVmo8j5=MP#~>PrCuZnvIKv# zbD-WX0k7CsVb<@_oM*3b=8gh5av*v}Gr$w}-m%;&Ax8Qx0DHiuXGgL;*J43o*>xcF z$dqqpirL>U$@y6&%t7aqduR*Ze@rIG=ExmbVDrFBAO_;rj`&?`>B9%N%{Wi$xVXwH ztDi1Ae(FA!8ZcC=WLt`=OLjNgn{7Ou+J7+HcA* zn47+ZtM^{HQ;*_5yH7bAWr!q|Zrc=acx8rT@jv8Q)M%Fw81tE zbHGkxF`A>Px=cl~cW5?o0MW1Mu4OD$!HmC0@7z+_3EW-_Qiwi}kWOQP!Fm9{8v_*b zxg9K=2YE+^+AQI}NwG7e09mdtD!{rY&OmX=_8G?ELpr!6(LQH*7fs2umX6(+3{l{P zZh>+42Rd#Jl4cLW0t2NV54Bbk_c|XrPEO9Tu>$?F$&Imso1VxsMj}vePknWxAaZ_T zF8pG|cI1$-zJ486-EcCSixD~ddj>ch!R=0$9d1mvC>C7rUo~-F|MR*&S-HXAwC7K6 zl=;u**Ncj-`rM~>vmTobQBY$2wHJA4B2X8~mXX`w4&5qcAz(>n+O&ke<3Pa*ne)SL zv~NA}rBb^mt{r+UVZ05xj8guI;j~gpz-u&iN4zNt)0x7)|Wx)=0Ofd zc4Bs9z)onX{T~MYykIjrjrss~iqC(VX}HB`HWCE>)Bl4D#cz}J2#$X~RqA{(@Ii03 zQPwQ+H#Id#CvS9YuOsC2=(4$S<0j~=)Wd|R4KtYa-3^xB+grEZ3z@;LT}ugv@<|jk zG`o?J=l}JT{$xETNa{1_I+)6uMam8OE3xME&BrOOl0%hDAQeGQ_Um2<9OXYUJQ89W zsH)GSUM&@@Hsv>E&CnR8E#PGm&%AFpz3Q3+HmsVOG;7m=T5TXwyU=V(FIVLL$#M8B zn^smn<<*GXmwWW>5gdG=Q+1Y>z30zznYMBpm#Qof;Z+5-i6r^vkMCYXLzSzS|#HyoV| zYi=JXv16jA>P#2gkl+fq)f?rUmB{AL2y~kQGE<9-^1$@L6S%o{>`c>ZT<7xNu&{_A za-t$q;5!$`m_hbo;Z{|dIJdR_hTx}$4BzV>Mr9kUNB&9wrvUkaXQT{q6R%!aVpTL)ea;|%rOiNNFXpdfu{-UaKg<#^LR>KwsByE-(xvoouD123xk1+AL?miHE1Y>ww z)Peon^bWsPRIT<6LniyfgdkoRLJr^)Q=xGnIfTC)??l}qs>URYYO1e2EH6Ur<3t$m zZIxa-5UYF*?g$AJNssxd`q)p?50F?aF%^jExW2}WKekJnA7>&jN_p5ybp)gh@ZK%d zK%!ojxD)Z$x$9A^cG1c1c{Yz| z+2)mxtYwgnKz!i|dJfZwZG7fl#nP2Z#=jQt%fJ^4=hIg=yjws5L0J2W6E%F@i`6_o zvYDx=wFgRJYc4dJUbyiOdoH7aOTH?@ynpFf0t_?3zgV8 zGq3W>&b?`sFOS5F`XzkLDZfA0w;J^My&G!MtGN`Wprojz_*k6Ty~#6i!@ttGtZFpb z&;uXB&~^UB3QL=*u%Krvl(eT5b~RbXIf{Rlc*lPZYw_1pC;N9(h-)kCO|hAQ4-5N~le;bmm}aYW ze*v|c4ik8!%0CPNiR3ZKcGk}>7OOAYiLl-AWGg>EcI2crfyme-yakEE%zO9mzlO7o z+=c%8{kL1vNP6EtSPMd_vg&w4zL_Z8Tix^<>wXYI7JAhrW=vs|b?g;yT_a)KCfBy8 z>mG;Vp2@AA9fx!z*z)XY0D5C6cm!yjgG^nmo8I`-?<|^4%{=2~Q;3z4c`;nX0Lbgn zrk9X`bd&fN!x3Bo-c<`XQ_~CKH&bOB!hmfH-x=2$iH%uLG z4RlYAJa!;pYljop-P)y0c)@n6y;V&-=|=X>jvA%TGohKhwlxK}T-dsnm|_z`6U1DY z@_0(<`Uy6UvEDTk`NF$PXP1Yss&~BuFLtMG2O^|?=q9ZkoSzY)=5fS>R@t=%9VEl% zxi=qvdG<0g(ChBdwH*m}dXe6cRR*rs_Br#YyRz+eum|4R2M?_R?sXGzVpYvyC?ldgG@-( z2kx+m6@54EQjwabH(8Q*NOYH1QTEpkcek*9;bChIg?yqS<;_G5dN#-x#>AduiAl?+ zSc{tfF5t`Nl-+eNInqr~Q0F)XIHj`%D_WN1BMgug%V-%I2+^Obaa;3tvZsaa562_2 zKmhY{7`Z6PO)V=MM>IEr`el8@&F6f)zeuk%b!`pB#!dZFh*68pQmb?`d)sf^WlYb+ zlG{yC`NJBqT`W!uGG&LyVJP8tsm z?~N^Dg^01RkXYEs&Ji-N^zMjx;V^zj`UD_+zRRK)es|>HJDD(P?m6hnz>w`?=-zt8 zV-qs25hDTS$1xk`j;qguQ7I2jIN-&EXPVD40k?V)zlT3O6aOAu3j2H)=-A(>8ec{n z1ZtaIb5do3o^*Oc;w>2pS*rWE{O#8vLe4+d<=*9`Lb+jLqQtHj(l&qWs?F12W21m$ zW<}iUsAuVfxZmam?#;iiw|P-F-Zw<7g`6^u*%F2H;BOZMORB7Zp^clXt@V==zgD|J zgALylpETVE;<^DmaYSubS<>By8!}aWZEqwDIF%!=&-Mk%ti}Dt2~!`Yn_oz&ouo-_ z{sfQt%iBsxdf`&-WH^$D`Za|SiM?_?)Hwdx0xrC4(V*|SG<(Pwi(J>ac1+U4qudyP zb@v=5Bg(Y*O*aKuavrkO8vU|X3upRfrznb}jtvs0U5Gh(;_kT&$m@F@VbxBl*9)3? z%j#mb-|L^2NTMYnvWvsoC4eS5iP#a#bq8Di>0f92g=1es00W z<$U8<)Us?9`K zMQ7sdc)lVhV?fVBGw!{REd(PEMd4GWqRbe9tRaC~W8cgv&^rL(t zn_EI}Jm%kxNZq$SW-8xE6I1Q}N z+z9@W{(j(d4c-lr&J5=DtGrjOGq~EDb-Y){%}FSQ*=)$%C}UY4XT%^WJ6WPlJy1FF-<-w8(fQ#n z<`k!@SnUd(Of54<#H`k*|0!R4>sm04ak%iGfD2aD1{Xc30vAz-G zlDhM}b(Q86fbL-}wrhtWLSZ#n(66M7G&Z|Jo-|Danmel%M_~b|S)zMxvMD?`qWo(7 z2JbOFJ01KhW7_rLQTdH+I2#~2udajoJA{KtIO(EtP%_|6A|;&LIT66_kT8^p>PPO_ zaja^HUfM^c4FlPimuCj(@6(X=B+NIM#L=^fa(!&ZbH8u;{cKO?1sxYqh0dqt#lk`u zkE;a`9T&s}qn zHuqz2wP&{B0+!*n0H#Ruu~XA5j(C@X2CJ4{Nqou$oHS;es$IvXcj2Lw*C6X zzpVA#*;%JL^3S&U5!aVzlM#`X^FuraWn(i}xnAF>v2iP9-}weQT7C^()O3U>u;p;&)^Z*9uo$^{XAlIz5wAiiud9f=OZZMyDXs<+$Cd+6C=BcZTp z6tWzm->-dnm$RT*gl~{~;wcn7vHB1YvIwp3INC{i^cK}W@%LlyV4_0#<5ij@-e3q@ zS zFg;Zd=*H3z@EL~@6)%l1?6b>i!cM))TGJ7P$U>j5QUIKtQ#@wj5TA&(bkErU8{via zn~FwHmb`9uOmx2luWSKv?3yU%v0{rgjn3Ith!hDdj9rp7Ffv~czN6QEkA}IdTeTZ8 zSMOzM9e_xD<;BLbgv?47$c>5hAQ+@@zT)S7AB^=UDa zc=K#M7dJv=4u%I4-SbDp`MQoJiLuF%R%}CcFw2{hgE9kw$%QN7o6L=}>*J<{u*>tf zCGc68oab+*GTE_IvxbOsAM$3c39*5??exE?cc6w-&!nwG_ri0CneaCp6>My{@ke1& zx487!^fUn7Q3$*B{?A>iz5IGvs5(&3HY7G7J;CvV)gLyJ5X+CB8V3o!Ogwkf7So3+ zUwPNK0Pvj$ugys|M64zs%yH+=g*_ z*Jn!PYP#Wedn}Wf4h-hPQV~>gEt;rG+`uL(NI@x`@BV|Y*HlOC|Fmu+877lxa}@`I zb-)P3fo|Xy0XS{rs8G!F^z>m164pI@R3ckoNOj`0=FTr8?FuL0h8tf{WFdTz`!kb^zn-3MRD4F^4&I>-hrlUlzg z917unV4ZW^r$g#_4zI2=UWWYJGIS_tAN~@0c6m-b2Cu>{A@;|$5r@aS2SJ}3^m#c0 zE>aH*N7)DjCB-mv23;e?0t2EuYWc)@V7uXD-EDI!q^aL2KQImrl%5hI&`!*KVtW+P z)@E6dR1n$R9$CX_7e?< zX-TE&c=lrLXPWbe%g3`gs9yKcwlqU7kSx7T-{3gvzTCEqFHk*N%wV)vfxsZfkEW0 zVnJTgz%}O&+G8izvEja$ywNY~dCV`L_>Bt`-Lpu(hb|3bTXP~&?zWeqVU)W zHeBc!CE!Fq_nAt5ja(p^4P4jk|2%Mj4r)_W@bFZlowh4a96_~Dv>L%Z&thch<&}&y z9e?noKk5BZj_%jKRpjbb^}cS`Gk`g>kweX7eNH7YSO4&>_2$JTeL?O1DzEjok_AO2 zgM91cN93OzL8I%{VRD#HsCZF3RUHn;Q9={!yBvxtRtG9T{ofKiAkgS|lMRlFhcFdf z-lwu-SsuuhkpgE~d&x~S>LZ8s~{gw5C5 z6eH|y5eV|FW6Q1f4l`<-p~*#UkUgTU(e5mV@@P;eQddeX?6@fM!X03jEL6>r2?u~n zH}?sZjIs`+=LCWVRA^&kWs)1fHrVd27wlR@KnySH@Z`BPv4NhllMf7!Zq#AOu4~uW z89=$8syE6lkJc*^qKCb4G-`Wq0nz>3jnDeTM_QhYrqXJW@=0(O%aYa*)ykO-O>U{J z6ku$rMQ&oOt&dQN-l5g(k74DTOJ7re6!G_aXN)j}q5Q3r`kvQe>%H1#t!zyMF4vvf z#6tT#-P<_TN>;;rpJ|9oqCHy$F?qg!cCr59K|vv5Va6UrW836f*v#WY0`4lp2J3G< z7B2lU7v68*;`AdpKXfmyhe%OmJXDCRH^>Yzu0*^H?2wO;JvppMnk_3ElL_|JhUar8 zmB8D3&x1zIGndm#}G=SLgH*2g@LA7VW+0a8Zk$E3IElLO(2Llj@aAu&?i7z?I z+V~}378YFo>SqFXpDW}ytl0>~jl{H4E~D=!sJf!#ynE-FnJB3BWK&l=`BTyS2EIT@ zMRpNu^wNGztovn>zpps(i8w%FciodkVXYI6P|?wH08Eg42~a1%vDWOs5#%E_$rQ|n8o#|h+3>7QoUWvWlM=`Z;H-2*CW(@G&(#b;OEb&z$ATq!wwPwxVW~R+}T?bDW z85MND@w(U;y8OyCU#iD-rC^r#3eWL31i#x!Ej)5ZyyI?$b?DI9_tEp+l^<`8 zO?Dw&Bn&HpV~3(t@TCLy?I>H!=zhX zoht|F;Ieello2DYzy-TsHN>nO>tg|+Bj~^e%a0{dMbLGJ_|c{a2ImY$mmq-v5{ggU za&fdV{{DDS)Z$`$T*G?BdeOQyBH1;kln^_dRK`s$JS6r={dF6lve3E8>sxE9EcuE_ z=^jG>4hy3HoCvh)dnh9Hai)H{q5@G2Ni~Abbwn1FZF$QOuaW0OhmxOM!?9sgRq^H2 z#l>u{B0xb&Vc4f`=+x!&y^OYwcE#21jl*n|*Pa)mzp!Si(z?+`G*>)-`s84UF+~Di z*%a7BJh!(FuOc$&GL1iKRvrJ8YX-}hDdh`SHB{w~_HB4rXR{)+`QT*faFszVSrPNI z&K2*^zrxG)qS3^WHL{}n!diKCpuQm*3R_?r`LwYjD{SSC>4Sa16S%W8qn-B;;6SQj zIt-o8>_*<4%LJuyws*EsBkQD7ofsyGR{W*L_e4z-!L6g(-_7I#xMRYO@wR&nl%|t( zd`$rNyc?ya2OH^NL0RL~+FZ}+@FWxfFf|2*07wz=hg-1iQyN{$QGZJb^vgQ8i$wP@ zX{XO<9`k2HHIQ^M{wcd0U2Gb6>c{r{gzFE#oHcl!XeNP8hqcCWXWOov1bDZR`H6H~ zy`XbxxbO6|boa3O{IlxmR;iG}Os1$Bk;{%5m-CQ*WW?on!`UkuvbVTpcl(ek&7;Pf zki(kD2P(3lZn7`OiQ;oAQ3 z6^3!IkHJ{RVAAjMZ~V~eL>hn@ppo@a^GtW=Pz1D8sa~Z5K_P!E?r1YdKNV49WngJuM&mCW|kq{##C*-Ip+3#^GPgdczNPfKsC@|fk* zvemzX2#buo*_~bx`LWWo(Kyxd?<6uj@P-gXe18>kZIy4!j7I&$sP~q5nwZwfc#IKC zA)gxv>kEj~jn?pHoTPkJ-{A#Op}SQUcyW>c)Ko^P#^m_G&)CEHEt^(kF$RID6C(tI)7Ayg+Q{| zug@P%PR9~K#8Tk56M@zCkm?j-ieD_~eF_~EfiC^?wJs((ITslDwjPAv{heQ)x6t0 z#t$vr?GX7(IdxsI|CYn<&un!BTi!YeDoJ|E)tKN(({mXi$($(E@0;xz=#rScU6FbC zJm^=e^-RFf8O}DmV_~mg%mA!~)-~q`5~Ck0dA{v=kMzbio|z(_OqB=%M@1;LjXB4( z$iAL>_6xjg6Et%c5?_l(=u7;h;3oZGgpz4^R^{Ph4P=7Fk?QCKVK6V(4U&@9K}ixL zz8-hs(tDn0G(xq|{8eJS7!BuGnaAv3303KzPvUd_EW#MIkN|iVnwt(iE;~=eH+Bj%K0T`;fCNRceT3^F$Ktx-C)`{KJz1Fa#)^Km}ivi zxcS^$D$J(Z)Aq>;yl|2pyEr$a;f6fjwbzfL=3!238yjJs zU@Ti#KgrXdVn5$$d;|d0*jRu|uVS{hlYa!ni|9U+rqKsV!1U0XdR_09fJo1EQ6x~{ zJK*a*uC$alw4_Xa^<2dmLq$>$Ka%Gi8sG@F2NU^}G=r|&?bDu{T*JfWWd9lm=~A+g zBo39CHvR}7L>ADVCN708u9po}r17HbbwWC5DUD%`v(5Fmdu5Diz+S`Kuup!n zFuP4EMf9zxw|DP#rEjz3N`A5FNiU488lnH*Ce6sDa?&=j(?VnU%a79-@ubAuGZ z%GU6Vdw)Kp#$uXy-Gn}l2XqMlkP-^j0P2v1kU-w=zvCSrq-C;~?hG%k&){lOBFJNr z)%5kgGR&fPQDVek(4Vz+GrTiLER?s>4ODS2mR1!~^fkNa0TBt*P>o0@wn?*yQcUih zhjYX4vzVBfa%5G4$|;g7`_Olu**zE%8Y4DG{|q-`4Kl}O8(pX7AIMdB+~m z15OU--Dt%r6?Os^!j5-GmQK#kI`bF;FANGeuZhy`i@$4=UXeHQ(r1_Zeo6%k)3>|e zKV?Yzzpyu{^BHHAo;!C9wP+Tq?5E2f2DDgh$1v#&q=}TR&NNyKDa30DQ2>)>OR{aZ zot>9$+^G)RhI%ZFrYzhiB}&o_b=)ia`+>p(zhnD>3}gvUDR_Y1yAHf!q(zJt(xeo} zRegmgz|wU4Y&B@ieUbk@jDNwBwYpZmN9)X(BS~Og#}zyF_R(}8%0%bQ_jG+u^GnBV z@KbMkkQNW=2e3U7vR*nvf{y7lD#vJW&WrZ*^N=E1;^ZOST48VOgI8b`6&xQ`v((FB z^6|}NS!Bz)cbwY;M|EV_b}Zc6rFB0+d3WE##*5Tn0n0=y?^4SNL~|7tqoU!zRn!p^ z-HNx(1LCf(l?-Uft*H!Nb6p?0KVi5uANgl;6=L~)v0!K78p9CP@e9{}we(Pd(aX70 zxvxc|NnQB*%l`7CrA}vOJGD(TIynk5<1DZNS1zbI`9Q~K#r zF7B`N4PhbG+oQ5#J%Q+_*vn#0CKoA3M&-KJ&5J@e^d^gE<*I4H>jV}sIck|w5R&?& zm^=0r07xq?4@WSHOK_Xgsi{6uJJr)l*j%3gYK9cWeso`vAVH$dppdZ3%R2P%VOwAL zsKfll%bT5Y*l<-rH2mseO-Ysci5tNMTqIh53qBng@$>a<)|8112~s+J{XUcT(U6`$Zu{%8 zIQW)!gNOZSwkIQ%>M~+vX60aDXGr2P;agmzjnJ^dn6C~s+m8k1$CK&AAZ61IFqK(M z6SXW%vxEjVa6GD)Pc_IVfr?U6l7dUol@;;H$pTS98d2om-sdZPehzsKVw93}kf<`< zmSi>*`H=mQ^^4lkCQFesIu=9;uyki35TZQWn=}Lr6r40oCjs5|49}xBVXQJLMsVLi zO&U_?P9A;8L@rkll$_)%$WZ3Lc0O-$3jf`04Zd=c>kMj_kFMT$1|AS16$OnZHo)Lu z+5Nk*M1BecIJ{}MI3ik?TwDw&YpV@@ncfpy6Cx z^yizr8+0-rl9hqWtH_NRw?n~+M-{eY<<;3y5rR6rPKPDA$ zs;NwC4%Huy_Z?mN9vl?JUjpb=Fi^9p_wcXEEhZ<}oA$#omUK-98UQL^vSmATk&X&b z)%uEqjThxSE|wcvzT~&!3SYmx8^8_KVB&>#Z&O>^w(&fU$h& zuZrpk-i?K;rKFqL$n&_Q0uE0CK{?#U%X82(5-!NFjflL@U#EBH zDd`gr0`DDjh9M==PT|b73T$4F95{G&($YECXn%#{?Y#7N^A9yM320n6DhLM zBygeeZMM+>vV2Jpfx23=3LhX`l`OGt$tykKAn&q&wyd{#NvYbJp$f07(0fb|gq zqCTpy=-tmm_5;75AUVgl;w{#vx64DJ!6+)2IkY128W;B_pUcRx%!S|vUvhziJp1*? zxmb9mk2xc2W69II{REm8#jxZzRfop6r(uK{%gx@>0c zK_&vgML<`FiF9!wSt~0B!d|Z497BZy8Zkk_%$8j2h)MJ@B4(z3CD;S4;wQ=HR@GiIga=Wk~w75L;M{u00D!;hEtt`BdfI*oHf$5h8` zmG*S(lLQRrbAb4OQB-1Ax9kZ1oT;5L6u|h#UGIn1qc{m+ZKhR+;APjiNt+DCHOHzN z{}F6V=P(lh?186NvubFx$Th%RiS0&zwHc+}iYtiJ$^(AD{ySeOYe^p~Q0DXufc-s= z3p`XPNHW0Smab1I^8wieBZuBzCPB2B&mW7&P9(@^w4U(xv~xw32PT;Ecq6XNqrJFy zDB_P$CB^Ee7Y#L*@BfA=RViQXuS{-ePM_^x#ib8o0*S%6w#oU7tLl^3T1ECGSAy*V z8^9C}>LNZE@zdyBdbz^M8E_c4ujh^aLcGFX)!Yn4z7CKToSe)o3k*D+xLi7%jJzJZ zq2}+vV3c=GIEh=jvf|~!O+h5pE$wM+4i8zqmACT3lttv(7UOAP;G{%&m`n{`Zo{Wp zF>j36<~m!d^f{~SILGgXo#Qxf_GgKniJOza$Ug1H6>5M|E_ML2e}pw!p}q#tDI0v|)V|sooIlktEj98--8ff#-AM^sr!^y-R#n-wy;Rj` zCb*M`LMgl&@Qc?<3j@Voc*^I3Q*+#gO_@yJIfN$}ZglW($Q zWxjv1J+|>U)WNaO!ku=FC%xCQ{I0y2(XPMrg4lg$@6+A->`x^oLI-suf927Jq5AIE>DthuHaem*y=SJ-50TRDi|b!Tn5 zgIl(I5ixgJ7Hb%$_W9;;#p50O)%02JPrpZDL07+6ttSj50F*~Wl0&CRr}u#C%%jYW zj}-*ZfIX$T%#z~aa)zZP`Z=SZzaU?eI}DQeMd}n@UcZD*A^{G;pC0@77VtSwCV=YT zOX8SOY&^Y5E;YCPsSDxJ-0)*Ai>Y&O^~Mqa;1@bZ%f^N} zWP#&Ye2I-vrl4znz}`YfM+@d9Y9)kZ_2P7|Kk?;+b7c|5<3wW*qC=@?a&l$zU@KUi zAH=I*KDPR$fRD1x`h;^Hw{0wCw)e|R*sH*a$33#MbvxWf@JSdV!xoSA&f0^_#( z+8V!g8TO6sW_RH-_jGSTS@FAp^3~DS(#FNbMaPxF&HC8|Q3(7;2=AGH`(MurHz^wE z($V;FaSkQruuhP&e+d*J_lDcS=^3Ze#mPQAP!sXh7bbj@l=+2Ydx0dU)aZ9=GWDP? z5;5h(w0o*N*D3}~Vyiw_GCjzSd4Kk=B@npe-3d=~Rdp11_q1O$Nf}_z6Aa=}_<@|K zTH`QMrQ%^^fgZnPtheH0%Z4UGegKA3&Bp0%e%v`K1RNnfCuSj$NLRC`j6IKvZjncE zP4nw8-gh+K(^4oMO`%_n)zC^s8zKv-9So^xpHR__In6JyWjJxMr7igfwB}#?@tPzw zPuY4}dENLJoYjz6dOi5n(Z-o(8g$*@)Aerew2ZVL-}di699H=GZf-$afE$fRdKJXs z54eDwiyMKB2T&xxhm(jj4oUq5fmbkl-!5`H&POEfWBC3VPjZB-NmKKfR8;jubI%wRNfhN%aS{P;0a%B`%{^|S`sO2 zDuv9h0+!YJP$X1uTiB(b}p}{6&Ylx^* z>rn8U_UgE3ReEm(q!Q;U5t;PBuo)2B<6*KKN%)e-MuW0y#l5n7r4;m7kAlOITBGQV zGAh~*Wwut!Xi!n$X`ss0($N>;+MUweb_E6GpD?AG0_AX5%iXH?NyIQ1v0Jd=Z~f9+ zt?QRXmH#|@j@13;@SAFn2I4q=jh83rsfl7jD%3iSr49&aX~87A%ZPk153J05(;%fK zC%`+Y1b3gHlsf*p0@$6$2#5_E2{a0$2io(W`I0gleUFE9&RM#9V4(bZyxJ4rNJTY( zC~j(`x{Tp~I59GdgOWmGlH(^@Ka%hunorO&Vut1b631R6aYjM>1z5Uc+ ze#ttE%1SO+bYJszE)$)u6AJ9u1z$njEyNbwrU|@%2c!W-)WbPf-xNYsv{Ztv#K{mm z+$+RpPJ1M=2D)85LYi=v$>M5d4i>FmdL#U!Y+ClfG*#t{;>o|M`ZJcme|xCi?+e9r zZ{H4B0%!r(s|N?)%kdsvukXxn32SZB)^yLl=5S`AwC_R!p-~Br?NKND&zc(YO+D@* zBQMXq%>t?yqEo~$D)`yf>d@vWI&!n7Z%VfM2KUj#F*JrDg13y`kdjEgFutrT&X{TACp&)$ z?i$q3;P)+0IsE+cV1>8_iKTs57W3;bt~)H(+dUXZ<(K#iC%&dU8Qy9FJKF_zSo>Zq zj{U?wb4RKLH^&|_g`Ei@rqxM_^L0}?rNr+2{dFRB`(`(ek#p{NYinzxKVMj8(&QbWCxZB8<#V?fYJcwpW36v513hBwTHT(uqbc$5%+=4@*c%rk|$X}ra3=Z3bF}XN} zv2^l(BDHrP+-;+rW&)>NG;lzdk+zZ6EY%@Dq>onzUk7z%0>C$Iqu06*rVDO1kEh$N z26FeVRbR%~dV$ap_Fof4=c?TC^XzPYK5=hb_**`{t*JjA!kT|zU|?WpgSV(nhn4VD zTGm)jR_rc&96mEqWGnhquCd^wW5$#nU}F3qk2do%|NEz*xRAQ>Ue*{B8gXBk&MvJCTUt8Ky>Ey&i8!>o^>nLbllR^4#pzBthI?1UwZIs8)^l^Z zcXNG8j3%t~84L^T(yG3(;T9>$J>}%PfXQnIUbK6<+c+0#iV1aA0K9jhjBEY|;p4EKhWa4_S!?DJUtM zPc!Z@G5=4|xyLj8|51Fdja=qZxrX5zNhai0%r(hf$lNKhxyxPdmyv|zo?E$%AF13*RZ@>M$fA;u%9(%uE=XK8WxD$S&-cf0K@;KxWBOGV<0QO!p^bIoV zp98Qnc25z2T6e2hA#*yC`iqj3N$O5_;26Eg1k!eUIn_@(7H#Hxa z^Uu92|HVY5;1sB{MQ^o+AITZg_Mx^6#6_>(lOHJDmfoUleg2iSsP$Qdznw;TeO&9h zEdF>VB{?5H*6{BZe(PWx;dw9{+rB^^n`;ZCgD-%v)h9)KiHNj$5Rp5ymf(7V0l#S>%4kFp-E=!M;5tLD#{NNVbKM@$WSAy%4}d|6#|Q zgkY;mytCQz>R_xXy|suH(JfbT9hG@>bhK93o~wbDOEJ~eSN{Hhx)2mTgOwXfu&yHgfi8LTIy4?@_7WNxS>7@!~5F>>uO$&qt z|BGL|`1kJeAC_{DcbNM%=h>;-Q#R?lr!8_P-gQA_U3|$oKP)V81RD9h%8%|YIeo~Y zX1nFaK006J^@7+<7J>QS|C+;2j=di;Z(QU2K_kYn@d`96IIs-$nt){c-ay|f*8Aq6h^^l((r!dpy=szLBJ!!6wZ0aY2I9X>h#t>i zf*uPrnPpRlR{U!o_bv8XNdhx-S5$OqQVbE(7$^QxjX}WR*-~7LOSfx58t1F>ZU<-= z7cGER&r4DaS@Y~oKvNP!tm5O(Ngix$aV*kq13t9LD~{E0(YLX#Nt=UQ|D7gHAM$n& zF+Z}ROYZ85q}r@`$w#*g`UhZIZSzi67ldS(2-1pQ`3&*zjY%t`c)8+ZqUA6EK?nbv zRN$Jj!w)Z$S-Jrh?->HW&?ns(U&Z`qP33X6zA?@IyJo@nE&u#8=CdHrgOi;nt49)l z&Lucj*2i9dOpFa@VX(qdbt7n%|so)0AVS{Q2f{3V{BQwUW4me&JB>FY=J z^aMrSlL~F**KdAi0kq$jYpT{88X}xDx{Lk>E#IqO8_Ax0s=!vlKpPcJ7bEkm$^86s zaIQdQI8`JlD$CZH^oVZ{kTO*D3$?6g`2OC#o{~G4;YBHSL)REzd`+EMW(qke-$GUC zi9`fw#$e})P-phPQ_8DQzkcP+_(ei*QnTIs4ekjqEj>}z)pmD_l@^ypV7MF-BO-h4 z3CwSx%uK|t@9v$8@&_W`5(mpaE6lPc?45%w#|+;M61|ynGltDQcgWuu zU3-d`dnq_8z_PLa=lAhX{WBA0sqH@(y~pRLWU}km67E1Cr5zF4_u@5Ek>09@d`RLk zWN*m3Anr+`aSz4Go9V~xKRCEC*yP!<0xel! zOH2v0%wXNik1_568xSV4~)I}o}fQ56@ew6ygA~<;qhXajn_=X!QRpw`F ztZh2k|18ZL(OGihF+qZcvgF-YoU2f=9yRci%XjWdc#Dw(f-0CLUJ-O} z8Km_@LpWf31>QH>$B=4tSBRFLS`d9*>~G;a^GZcFyJgz2XBVgR#yLQCOzAGJD>7vU zi@w2SkAWe+Di%hFA0Z!_c{0-SMO93GU}n`>qK)g4^05YJ+izKx19d*5Vv^>)V_Ur| zg~7Bm>Ni$~pz05B6qU}&0?1W}ARF4ps9Fzc#!T53eglPz1p~UyuU45kUPGTSw83!GoCPLZ=50#6x$oCo(FmKnpNXTh zvYr-sFvn4R< zA_a#YE8M$><<$kq0yw&b*a)qhLv#HNjVpJU22hC}$$ILo#V9j(IBqa$rCfM?(fIySv;>|rV^Cbu*JjCNUr$rP3xhs zj{GCVnhn#iP-_Ajyf3?R_`=UU;OG%W>w{jUq$K7ARv}%LxpPkx1xPlHyj&Hpr*r0T zINasg<;6R48yP;=diVM8vFsa;hE{m%lc;X~L5IaH$JscAhXr>r&k*b9*myC92uD6o z&p!E`RUS$mrn>}fV0<}>L~joTGUC&D%eN+bHGmFWaUDvGzDZ!PO$+f<>f;Hode+9w z8e5E^q6RMFnVGMG?%qNQh;DBXGn3t{m5)^fB1ENN z;ms$2s;W(?Y=6+@-{0QhZMm#ug~h6>)RhAsK{uin-* zMWbf^=Jc&xhTp9h)toYAoG%MqaBM=_!4JYmwkP8U-%0Ch=tT-^D5*{pi>SV><{%;+ zIu@*)1Don;kJj9x#7XV&BVQeKR;cU8)4~{O6 zE(dE!HfPgw6gJz@ZvmADZB_*G;>GmkpTnKGx!!H}mOO!N&?IqLLg?hru#{bAn50Nq z!`XRLXV8nzAMM90suyLKFLL%g@0`V79)~*=laF6fvadv9*y;ZMaOZ_UqdNtK=$RjO z+40LGEFici(x{25Y1(cf{h-YPIPf2~jez-Zt!X2}bN48mqk~de9Og9NfIg+bg?#!i z|1^akt>;Q@1gR8k1uQ;@1=%jPdmpiO2DOFDaeSv0iYv-_txy0}7!_t%cv0o=gv+7MP^)}9LK2Vj~aBY5lPQopdJRbH!+T`e5ZhqIy!M0jl zFV7z7`bhji>8mf>!Vp?F*}lAQRA;pl7DEz|$TOuFFWO({VSjHt38gdg(jW;q&EL{I z zq6Gl{W8AcvPP7jcwWW!ul#?Vd-uJ64J@`iLCsTW)9VFPu^(h36L*Q^l**==l5u3CT zq@}GTxZhmkq-(;batzs3u^_Xswv$4%0RR8dpPrrFS$&EiHliKe)NXTuQhJ}LUl)u} z1{JH>0bpqyf@#?p!tC$n8*Ijkzx@4da%Q%tYaYcD zJ&umTDRWO}2qiMHm@oukDoH2ITG+k5re15V0d&%p)(-GQ>*Xa6I_bLd4a$2iJa@OM zI6YUHW>?qzoSVxVV4Z=cq+&(;TGzd~T*99jjNzt0DXv**Mds5KSMq}b!tiKIb>_~r zp@2czdr;-p&v=rgq)NYmKarUxkM3i+xgo6GjWMKs|!j*BvV>Th29c7QY{fy1xJ<6FP zEUpBzP(|?Rr1JL}^I->ea}1uxU9dYwA~EKU+<^dpJv!;5xyL_7fRrtaYtG3BAG`=y zA)-amPZKb>pg?`H3<`)9p#F6EVz3OycPPDPnBlBygS9!f?)!U8vy|vN9UjJL>m9Hk z=MnhP-rC8Dzs+~;wB*qy!RaYKOa@r5N-;w^=K;SG>n=J{Ac=q{kt(s&SAf$)M<@*J zZaEHdV4gM8kOMc}%UE*R+!dk^psF>&q{A=_AUX{nM_{U{t_{jzq2a^eyIXYgjip+t zTkAflTv^K$6&bJ;_<*Gs@Nntf%v5=&oz!pyFStk%l-i0nkM;(-@(h5~86UT~&rWcX zC&{7mHJR=_akm58X7DhW1Z8FM@50KzEyORJqC~~VQ#!c@?I_q$O0=Kop(~DumAQtV zpB$|FSU$-w{kglldwA&2#{mvII}8bU9UP1*g*N9kQuf)^+xPW8@v5kP444elIhgb8 zXu)EiIJnFm{#!nm9SRI-exT~+ms>^MZ=Nf|@Mxj=I%c-Z@3o3*XLzUkoUERvwamzp0B;+?1%rR_oUgR=~6w`DD`|pB)4(l zPvS8@dgnnW6OWRUknI@x^xnM3HVfM89gMFt#ovQO*dJn~WN0@>?XSb((8u^Zm zl`ZlTpj=VBYRWFCe^6i2{5VN?-{ux_320LZN1lC&I^ody{-(aZm-VR`gsvVj5!d;{ z>2Gn-DK@(obBC6WnGaWKwZ~hh^GeQ{M(&3duH`Ow)vZ=2Ur(T~Z(HXazo)|HtS*9? z2;#d|RdGMb$;8u`U0Lg~$=zxas*okrCMH$0JAvj(HITkghvjnw7)(p3!oblf_n(@f z6IzPxghl~GzWI5p9p#tL{e--{gfv~sh99>>J^?e~Fdsy}&0_e`O$}OZ6H514wdDcL zz3OeH;-T~y`Ya~+K=UY9!9!()iHaS7M40#zZlnZWP2VpyuPIJ4^gW>6#eM-H!cYLl7p4e8`nb`6)EI=5{IlQ$R56U5HyYOh06!Gx?K2OEVD;64X> zyT+51$q3%PK&l=h0+Q*qMnmRe881%;vb?8>yiZl&aekoD$fbJS&*<8wLbH$hy*>BUOva9HKmw;)iKl{@NF{aU zkNMd&4qlI(H5oYPZN|m~6#bp@+!d!Km5r@4>++-oDe%+BQ(NZwB0iLoB)-xgk`dVM z&aY0ZzE*@Rv7YZ z)9l-1h$ek|DNPK5^Ht}snq!5BGAHl4e#$%?FQBTaWV}$KiOGDtZrDvITe%wc)c3WL z>*Dx@ckJP()f|s0ziAedKk=C@`m|cx`J$)D?gQL?_V%+cCXJFBrjLjOxmbH=9IuQr z-sGycE?HArEbu#NHz^AtS@vRF|-7`gbYu^6IW@(Zq%PP^n8qeo;axO)330p`3BG#sBM@QfP4&!H!7o;4R zNQceGOXi~w{u4~eG7u;bjyI9ef*@w$g31LqiS2}wf{R1d%*o^Z!x|4ymWQJD`5V$D z)u)SpCSay$45pMv6AAn-I$-i`a4=q$h5qmI`CuNR0#2{@=j7l9ZQg{!*cEp#MNhVI zz4nZppnxn_ zS%lo?%n@4gr7tYUu*YJ3ky(5V_dOW(mXc!#izyz0T%Y%Lr6Hz*-iIE7p5jGUW~#nu z7FNp&L$v7`zj%uB9!kb>iz#scK5LFA5$mD@{Ka_Oj2++2@6w>QcN4g%fc?f?#`>7s zG{unCsB2fQfYn)lm229=hoUs?H(fjymSSzD`UBe^S-p0*VGv<5&|%U+Qj6$BA<-BW zh%Z2P1l^)Itb}{q{>=O3QK86MIa-NTyhLI2QG)Q_`*U*sX~DVpanRTARKTwoDzLB; zxD#MHR{HP#((~}-%7a0wh|63_Y~6n`{6U;L zyVwF@wNPS7>9O>+Qhl`GjPi)OV{A>E5TO>APU45Gn_J=R;h2isEozIGt&Qd|iE3sA zYR1-nn|AJc}!Q#IRT>G3mD9$ z*SC^JFQ7goRrv6s{Ih}75gbubG*sCdNB!xbVk9Ubn>rBKJ>sa_LTVQHCQ1h7Ac)sr zy^U1uxa?TN+{wHgqrj9dDMQ>B%yf+%Dh5|*F&BIsZX-)aL#qcG_cK_;iG)u;R= z-amBjqUpI%_vs1(u5?xn*XgFzbexe`uOAc^D$> zSw5CYPANsOmSEK4R>MN~eZTJ1IS06*O9X*QJ-EL_Y(QDrXe5#}D_7??EEKb}peskq zz8lLz<=0}l$4w12GsAa%g9?fGYh0%{T>v1@B3kOGHSs9obW9L&{-l{jK4j z@wvRO$v_LU8h^yB|^W$%*-@FoVPC19j8qi(ysE^1hf?J&D)Hzs z&s56sNVj|v{O4uLHheg#ov;P-uSfLzeWU^VYntN8CRNwuulf65xut$xNQMKvQ9hxr zYyC}5>UQFL39$&jC$u(M%->$DljaNNymyu>x-{^U2fC>h29w zz)NS%qt@l6u$w1cv*b?wp)DVkH|$a)&6FlmfEoYQ@+LP1BocFS10?+4%~x8 z*^$-H9%UO>!D0XYJ(Rgz*_k^Jvm2%dcmua5ze#(6T!5v0e87j0C_zDKMj2^k+xG=` zGoOICvl(WpDjK&?_72D{M@;|_njC|?o8H3`6U~AwL|NHd=iZA_7P|j;Mc3o`)Ty%O zm+;A*t<5DsMAs1QFZ)or)-Ni1^yBpA046g4vTL4s2q9_4^}uC1Fk9HCQ^5?cEH%;k z@gZBBVP2pNjZwg_R-NQP$zBCOL}dBxG$X;It!k09xJY)ZuP~k`JOjOu-BitMAnuqh zZXovl+&HM@N(Hr06qq_y`U+TR1(CWhAWx0B9FZhR%8@|0lhk`OCLqWtO489u=L#bSTtfQ( zvUXk^$X`;nYFjmzxKM@pWJ<7^jH4(Ru_Kfpb9v}_@fKONee&-gRCBKJ47Yfu4=idh zN9(4b)8#fSJUg1J5Ux9ihlj*d)=63EH_8`wm!6lI(?^sBw%obsR_@%{8$_-EQ#ChYv*q`ZEEldv6FjaaXsuisk(*RW^VFZF^{zY57b#++bCBug z-^bf?^8svhkHKJRg|x^U7H(5~fZFvmXH+TNtUFH)p;ilet~xd$hZkygnt>ZSzYY$w zCyA}FNd0%$2*JNP{wUet2mTmd3KOvoVMbdDXj0d;el!>ZWyR)BlY`m$6_^L-dY4H! z*|EMaW)=~GJtF!a75~1;QBu5Fqh>c@$vK65CW4%TRW_4;nrAG?9E(gUD)cgr|g1LNCOW$RyH z**yyS*hzAIwxW$YhC4-5fw?Qq%=n)drPay2-rYTUB!bYEb9hX~xY2my_i zc3-Q!qmhwEznHXU1~TiTfmAx>&MlmhaNSKRHq7-abTsxnCap7jtdm?b3=hEcGB$S) z{>FF%`D&*E=dsup-l$?5aqpO=HQ>DIO%SpsVZYMTW(_@I>&4{wo{)`)pCM13{m(+?yUl09K$3YQ0pgh{*T?iyHNm!V>sX6WqqP6+h1*i1#H z$DJ0UX&~y9DOiZn`)t425=$D9D>SlTdBiSIP%}zr>3oLwsLGTjp*C9t+nI=u$Uho?&cid={ZWvt#+b`WU2G}#|MQrL@RKLkAaW$34KKQ8QoIb1fx>a$Z^JL|4 z7r*z7u@oNL5Ztoa>hiVxg=(^&jX*{cT)bbpG-s}tOcTW`q_!m36_X5A?-_E`k5{0K z&8hpViYfr1^bfqhlN&yUcAzU?ep~ zSAJ?nLwA*mH*dOx(~QOAKC^2luZf7n3$7q`FfGmso7dCqo6zcxB`yH)#4dmhh1dM| z%Cz90d{IBqu>?jTW2hh0SNbKxYl7Jmhv?Rf$<>Pc6zZqrj~$LdR~>Pk_}ZX+GYPgG z|M$o~u2hGG9f}`}qPRBA?S!3fADKyNog5zR<@{_~?Btzz)spAPH*$8ceOfm72y=FR zx)^?r&-G3tuLs?K^wYdmlK!Z)_1fU@`Oci%+-fhkb~5{PYhL39@8J3Fv)wl+t4hr~ z(uGzCNrc*@mdF6Vq<%s1Pmy)0oM}X%INQ|?+%P+KV-Tc(|5nN0SwIjewgFISV z$3#?|#gEMA8S5Zh)|m=pHsSU-eIZ6+EKQEQDS`#6XEWQk`$W>Cl(sL4_^COaAT?H5 zulyG(N8|N*iyj5Is~6$(k^Y|08=78>|7!Cbu`~GGyv(Mo@lc4y3sA3O(jf5nLBUs6 z^rXD%-hbZ>128JNS{Gi!hD_%dbsAxT2?cx@+mzw+E2tYC`z^qkig$mQ=k0XJn?plA zBedgkuSP~ja(IQMQ;H)!a#}uA0{xMX6%t+e)+le6EWfTFK5@2y)4TJqNq1(KHnmMY%K3d&$yoWdPagg`j7+I|Eew)|C>JZUn57 zm;y?EFMD;ZFw8IinU!?3c_c_wJK<}g%;)He3^d$Wo5B0l+ZmoT(*PT0S!P%H&pNet zZzMTsK1upuV^&g?_lR>9#RT05y(9D|l#M}2dSjYryp=}C<+8&9KPl+q zka+3)l@~m!9CZ1+cTV2U1x@w{ygZMa8qM609Z%W(9YY5-8kxCPcsqmCl2JM-k?()% z6E^$pxGpSErM+_MZsrTgSaAwmBB!aepLx*xS>qGL+cta#r3ltNs1pY^Dh6{fysz!^ zYO$s)T97WYdxaHnCub_nNB2R4L3&2`t5DbRoKGRbI7Ln8CXfHV>HX57pN^TyXV#`WiV>^v0%^n~(@EOzcv%7GIdRxkgn5+ff$O@S@gcD*`>s4XEzSZY*7hSBEpT z>pv3Nd>D8&7WC#mwB32B*l%c4sckf|t2(#Tqb;$I5$;TD`(XwDmMA;0v{`DFYB}IA zzeftmv!ihhdHmD~?DLf7oTjSi26VH0WB149tC7~j>B|xAG1XKqm1_?#B}84$zHW5f zC-I+r+uNTZrPj)1Qkbhhw>qmK3>-2_uY$dsnqCHI-@4@=sT640UYnsS#^0&u5b%S@ zx2K^uY`%(&D2KDqqk}8QZ^M;U3pP>?j{?wB2W=S>)PuRxN5^Nz48ZR|>}nN_U=hbI z>!X$0d<#E;IPc8OS4zzHTqV%&!y~M%u2kh%87gv!XnDJA6~bQ-v!Zp_uV-lTHMUyH zX+4xGE1ML(k(t6(qLkAUraFk?Ok*Nb$mRoFpxas1mCPefND*BR(cZeDC4XBgHX7fh z%->h3CHR;;9k&Ae3j&yQ$!BkfLX1l@%#8xFhs(mnLnm z>@~wOk-~f^80!su@B3e|bn?>(Iz&AQ-Po2fBJGu!w$jybXZYvu3d|v$ab-zJs@Dzl zl=(=Rcc1;}Gj|M(h>(`UtuI zU>adgFO=w4ay{N6-g&JYd(mTP)4nentvc}tZ8zQoM$P18Zl2- zgbGGnOy~|Qs5}3ybYKMsC;A-xX<=Us`M3-=x#PLTSLWTMy3l5@N@WkR3x>Rk@KE|8 zr zh=$BG=0y1bb;b#6tCWtXooa;gZ=zG!v7U{cMh!%qt_m zT_)`Az>}3(zLadIq@L(tuccC$u0)B#v5)OE`J01%m*0%8wmggQ?QzK4$z@R78qCUu zh8iY5JX4llDqeZlkTu<}wY@Vm@Ux%;PoUldTKT6wS6D~lpdVW`$6IiLJ0Y!#sAys8 zH&N3QqZ?!&$}=XO+FkZ~PTkQbF9)iQm|7Y1LVj&{GgES<(mC?478}vGKbVNb!L}(w zf}aB4#qi>VyZhntdGzFJSf?HNy1LgL)FEAj%fbFoI393P*OTS*%mFWF`c>ycB%h8w0l3P8X!zL(c^#1G6Z z%*^&3c(O_=4tI8rLv6=0H~xu<>n1Kd^<~!FZFCPxR?9M`jph9$=LUN+39A<^@=nrt zsKDMd99Pc)DiMODCLB$-@S*_Ph%?_&2 z*LGV}fFv=o_N3cZyRv&Xq@?^0C<_|;VogmZpvS_eue5CZoRP@yUjgk_i5zU+S0lR~ ze`uA+LJBhXm6t|n)OL6vFal29@f5u?i z9~TZLZP12tH_j?-kHa%pS*<<7Gz$UE1nVW$+P$4`+8P0muzxnXKU(VPpTpTAyOyS= zw922OXGiXIG~cO!ONwr|ZN;xr{44Trl}Sdqn6w$@sv_P4s3?TmQ2gUf0RmYx=GB9F zCia4YG1hmKihG%Inqsi=%NrnS2ZTG$Dk<7aaNK;Vsud=li~U3m&WV=4Qs1SPOARP* z@vG9_Ot_XJ!UUX0ZKUP4qQLA|si8^Uv;1bgtG%v7dmh%pS=fv2AbED((QQswy?ghf zdmKJABl{S7A)uf;kOv6_gd}+m`2d0MaPlUB}A?G#>L%g zAwSGHKPuIH7 z*B0~1$)DWFCijrjbzVB+jz5L-(AuqLJxMXR%V;Y2o z?9qhY4pzafa>mIf3sX_(W%Bycr6^o3qx3y|Uek*bF5wiAiM`-_j;X0O6l#hRq%9gVlKh_s_+b+CWWOqtI=But?p&#}wom{6lJu>%^u_Nn|ohWXl)d z#X*Sfq=~MRv#FN_t^F@7mJSY#qcF#{L+SPZ1~px*o5p-JwplhvM|l(WdzCD`Hy4!=XQPVQWWO7G)JF-(N4zl ztQ%JxC8+ON|W<$e1Y^g+#_#Yt|{cM9l$r^A>G;@$At2h9q zSiPM%G^qe5=lSB8nS85bY}CBXUes|&E3k3BdbK^&KZ2~X&9=@U+?-%)F!yj2I8b{CF0|HdLDkbkoX$9{L4-ND08(y!w;~wsN_V0IZP)q2){pH0=*2^%8 z%VKUY{wV%DYntM>=!)ZlOZKL)3{vc!8u%za+84tmZdMV1Zf&kGqyptnp{LOATB>YZP<}gx=pne@`(@5a&y`0i^GQG!mr2e>i?D^R zD;K@Z4O%3-g3f4=q?Jgb8nsN^=8A!B(GE*I_(xVGO$X5DJMTAqJ0sUPI%J{vQNzT> z>NqY3$(6@HGKXn$k7FtLQ!wp!jV*aPOY;bU#ols_ZizksGcYSRRkvVDmq=?P(}k+Q z`J8OU79=pbdD^^ENU#puifuXgUVyFMj`gDbL>Nj>F$T*#!9>MKm$>E(c*mr4{bLNB zHno|M2O?s@ zqJy1~i9*wjekvGE062cO&h}Bi2E64#a0hL&`N5y%RdW2EpJWmbUD1UZCNR(oBLq>;+O| z`OS1yjb;zm58d)pe;qVPy{HV5!98ZWRaKys zX@2~OkPih?lXq&rlaG5Z4!zp2L2cHfnLY}9>RcHnx~647acYpk`_Dz_%$Y&|8TQ%N z^Ow#+;i@+a{tQm8A7B1cy*Ld&-Ma{$)i23gcpWIP@+?BTco~}!+`4>aB!b6d=(*GY$((2zz;d0nvL!D*iDmE1=;^MO`#vit*fzkX~U|8%w7 zdN7hUfo09~g0DKgP z@7KUcC>Mu&OMa0X_YVs0qqoLi<5OR^=SxTW-2!hG2Km9v1Am0@L=f<1ihJUJP(Onn@^MWHSfNedXz)N#fPAUX=~TuDD+o ziDGOFGPqueM`EObv;1#1rV__p^An&Yqc{@9gLA)y6vqb$5>+ zPq#hHZyeULlBqwg4JY^C!1mr`%RQQ6KdSu3Rb6cbT;T0_hO^P+1V7TXi_Omg#GqFX0 z11!*pna>|YV+Ge@uG1l@sUuY5;q0L+(bF!u z^O zJ}yEe#p_m3O5f{Qeve%b50BCyLlOP%uIzpl?1gG)YxsX&XLP1LszHAWeyp93tbW_L zCxe+)Hsb$;+@ushi}iejAPqaCGPB0o)R7QVm|8=v!aabS)MPz zLc+ob<~MKN1bOF~NPbH?3Xpf@5-1R$N|J7#A^&u34UuD-l+}Nb{7yi?M5h;`qM}Oa zHT26*Hr%^ACmL_XEdq+pW{b3x@YDD^?^99I@L&uZ)M+)QXvLrrS|8$0*!6f<&O^sS z)-08e53(X~z2J&dv1maVS*?UVJyABsiF)N1Uw8vKhs{5EBlo7n@10Q|V|RD?`#pxt!`%0l|Fk@R{H$VZ*V zAq>h=wlp;NqevjJ!l0I-)=YEMeKgKS3lbNpH(j#+!nCrF-H9{7MM{s`-t5XP`?#CW z5|^ev73;9+nmOr-JL=)TM@%Z<;!n?aD$kcQddr#NW?t%~u^{SjcJqt*oqcL6Re@J3 zACy($VXsfP?f9@4Q5w}7qF0tO^FOq{&wjn)d*E*$-Vw19D3|+4-uD&+V%PKrPyev} zJ50PJ`~$)Ue6=Dv1oYS&igCx?32tszoag6sCEkO8BQg1JzinRA=k7*jLlML`ssdlYH7IQBoxd~a)R!4M)+vadD3 zz3K4g=rq%@55nr4+NCDn>O5@tKbU~Ir)tey&>k%@39TZut(}Tg#g%5xui^;0vCag? zBpYS-gApkDf#(KeWXhW$`WRS-`Srxdaf+qgc?i5UFUzA#S-KZKWY}z4(;PO}Xg$z3 zpczZ#XDuO=Vp<-mJitHTKT@Xlgc+8-X{=kQD)=;9;t@t`Z4<8DTA;rOe<&p@q>&x>F zblviv?v@SfIjl!b@*29U3dyTFe1K}p7<73=D-TFK2%R&H!%v}Wyq*$kGt|-U=xT8) z7k5$R+P*$E>@UugCCqD2{${f{ z2(X!Zq(@7_joY17j_|jO)BWuKD185Dl;7LJ*?+SybQ2T7p;UB$kK&r zsOFZC@s#OaYI=Eu5IqY`-*YXs30Z;N-A>@@f*SnGY?~YuO78>RA(UR4ya|K^^@U=XAMov{)t%ASx*zuY(fCy}IH zrH??T(v>CT!O^d9!bPyAJd0Y2>)QNw99LWr()jx*W#HcuYL5|$^C&T^3aYSb0{nvC zmbbv0OP~^vz+kdTVr~n~Z@HgR1LHjYUw$;YkByCq6iJNJBmffazoN}|>hxIF03_UZ zK1_C|3BR$t5w(^e%WSA+W1k9rOyhcqk>g)Cl^6$F)B10;Y^Ll_0}cjYr`k8MOuhK| zTq{3j>HEbH=3ZG{9V;+?GsqRs`4o}F?&OBf%N))a)3Yz#4Zhs+ z?=09h0T_2vNpGYL{0t0W9o_wV)_`A;9tDFwF*$jSqcgZx+j#MJrYY7U6((1l6%Dhg zS6Q1!Q_~gM&Q9!ArfPjCS=)&Rl$vcy>4h9jpZCUDePw304smMbWyIw9BG510K`B>T zP!h6JQA&p6Id~WW@VU6PIA!bnTsqYm^-xQqV;&T`he#X3rB zZ(y_a&YaM%VGkzz`00uUcxiYN3b~cCUB|RNHtBBHo81Td3)=7Q0;1Ao8giN@KqK>_ zaXPLTJZB!880Y|1-IPn!{Klj8KY_?`wrdO`_DMH|AA@TlI|D@G#i5_^>}^?7FClh- z$z}KO-;HVO0vg=dqr3l@8pC4VT~3hCzG__Tw^^J9UEd0mD{yqHVbqEs@I!h z-p%aK(CT?LFE9&(dF(!vBTl2;f@YX$6yzi*L^?+%OUwt!K52-rUJ}nZ4n8cG`Bv8h zX7-_*fgf`q{G|`XP;2R7-ktX>tHOiFNu)=Gzt&^W3h~?+Z~n$<38VocrQCUTiy=Xw zTnUg9eTZwOF{^ddmX1*nTxoQ7`g{%o6|Uigo>IQeS2RV<&ZYHV2?3QuTEPhY``|({E5?x z)}(y_5@d~vHe%<>stcK^!@yz6g0PA%DodlN_W*hToF2GDALgfPpDtLfO>6%=5mk=E z7uHAWy|IvmT*1ffLN`es+O#-ScIrXO4_yO#F%6O81qNdo4X69`6mz@9LS`B-8$GvU zk{?AQD;@w@Hb3)wwEV11QAjYPY+8m&`ksV1^QS&4h1>U7C!4XnM@{Me=@>IJ;pHy& z?yvl~Ce;L^aBJUgf#q4dIE4b?5rKlOje?CwgBXo>1xX+(l_wG>=^d|eYyJUE^hkg< z0B*1gHCWPrgYfgZM{7YVT29GL2ig2-rBwWU6Zt%Miao9^6QOtB)P~K{8${SnYgrS9 zgOm&Ud?j-;c_#4T6IE&f19V_^O<7OZtN{fsj_%my zk#UzcAlKot;E3uy=4(7QiS>x0|R4>+6sheN7O!>=COSTb+!+?`g1ipI@jX4YLi6tr>%qM5utk5g8(KJ$1Nb_mC_*MqvkN zD&SV@Y#ZD%$v+Q>?lbVamB`Qss$Ao3i$YfzR#zuIJ5DNV*q-ZzKmuxZA0)eq`&$~U zhRqUxT^XeesfF_PHqGyhw!|#$$pF}+|*v3x|00hFz(f8(9GRC6WJBW@)qav z?N;!~1D1CCB&Vm9Lmz#vAipJPd=6sD++YQFkHExRsuial^2AvP;7AweYOS^TWX?*m#$PnIeNs^r%{_=vv2>asZK+_and1t? z{5kY-uLmXD*gS@xiWM#7XfFSbCvt`}>4mePFRYCZCy~+Nw`(`Z??XG>&pg%d*3vuc z00yRb16k%PIxK3$4h$;x_^#0kL7)4Qp<#Z*u?Km^yeYiwQAO8lx00tueDT%|L2_5t zh~A;+{Z`!c?DP)wQCpbxn}z}rU`MFq=SUje+NIN=CKXRuj$a*vk)6)R`O0@rIfti? z9e7lRR-9|{|3MFH^{W|pR z>RP&%bNPu~ORx0Ru-ha<;lEj%@^UM5fp14kO zmbuFR^!V*t`NAq=@@GT-|2R7DXe#{wk6(KvT*)Z&+Ct{X-WNC7Gkaevdu4B~k-e^+ zY%a=PnHgDebCv5F2}xELH-z8){{FpxIOpEuyej8_B#n@BBW>)o-zi8DK2FE$T%p@oJ3;+iT}7*Xk)16K7aXj+41@szm7`piI*+7 z8Z%A8;}GawQY3~IvFtH})?DXKqK9QCz{%zOr~8s<<7gT^+=6Br`>aJ`G&jqdWc8*a z0)KzAiEN$FVM^%Tqy>Sh-dydfB=L`my=DWACvaxN(m2w%{-iOQG9u50se(B>=!Dr5=k0p$0x#_b#Q-!w7P?k+4Bn(xa z+!>7&$tV$gPmm6Cj+_DcSE{1660>$E^HmA~vJ|eQH&VWZVtP3j^mQ)oLZd9xHBwDA zz4D;;e-guImv^_%?Bnn^?wj_>OiP;+i@9^g+yjzR?U?8daqToQ*A;=2ikzoyB)@fXhq^EsrR+cww)O&%R`rTL{sK+}|H?u@Al3 zDI7j4iM~1%r4sXXJVkYRH5`y42mySQAO@a`d4b9j3oi%Qfs1PO%~1@oR3U<%gxv@L z2zi(dHN)df0klr(?#_6pCer;-tfJ=2n~ceu3}&Cq|FJQmx=Spd{FEF2JgWgRnyv|R z@5kJii(Nua;{SUHHt`KiB{bo#&be9NRh*()?#|B8(YWX<_KJ#it9Sj^E6l2%Q@FH9hp4#+RWT zJl(i0yt}%)rz&T4^5-wpRDMV@xW&gah>@bjEAXs)#m^v+OTy-kUfA*!dsh zq4k&9{}@l5|A~+O{s-^+ri*{#C`~9v^s_JYfo7nnziJ?%xfYR$6E`m<6|tK~WyaC( zyoIzU#2LL|P5&KVbmLmpN*@&sy)Xt<#&h;GY2u$$&FZ7teFa9%lFn1IG!Ecl{7%tC zM@9h!V57{nEdsn7XTM5%j+sfb5RP=3Q$=(^An@=kDazW)lam;z#_h;K0wg3KFNhOur|o(2z`%Thu*lrawsUu;>!s9`43%!_ z-iAKOIeqlxiRV%yx4YZ`pxG#`j?6jh7TKH&x(Y=cGxG;jnYNczdp7i^5YsrY%o zES4#b<8sHpp|jSZK*qOv`m-Ae!yftU?iygxwo02eg*e+z#OBwH2*977Fsp~?Q}xFu zviZAI9BO$+f4n?pOQZrm4y%q9C6zvz1gI#wQ_b1fAVOER4ukNzE{Z`#K>gKAeB zJb59*=q}ysGHKw;PTueRV*MO(Slrz>gO>1QlJF;aov6r_p&zW zzv;9j03cda>+ej)-NpQug@}TRQ2qUe_5|}P@~xoL{kz{>)<0#F%uJ=SWMu05I?H{W z=Nj?5Pf3+TrKgz+V|FZZKjKeqV|f4ub@&Pg4H?)drfc7NuIfshqzL+`&65luuPuIV zA-%9X{aLZC$Z)>XgL#)*ti#Nt#3OfDuV?vuYfGIa881S{_xClRJalbd_j&esn){Gn zd~9-MHxO9Pqa`Bbf0gO!Sa3c7;RwDxdv{K)$r9E|Nn7^Of`6R$VVwhcI@o@dN!JrP zQT&9kgitn@xj5Xbph?Opq1M#%Ij1+Mw&OhW+g}*Hd+@%;GQ9!SCxX6=_|`RGgF<4r z6?Bj0b$an@oGdg-DY>sEZ$f}I120K&PBE!ly!p~6^t@P;TC7&Jq;QGpg@MXQBSj+r z7?Yz4bw;*}^4zxQ?^;e0Rf7w$gp-J{Uyk%a_XJQ&6PSUGg^imd)|@05o+o8WpN_T9 zjGBH@H7Pd1Ifo@~LE~f9q}zJdYUJVo=9StKIW85o;GYg*8Rlt>njdV1VP(w3_A&K( zv7bpQk$xt{gt7!0T#{<`pbYvL7H(hiY)62zFDw zQbQF}pSjQXnU>EU-Jb!|8$GJx5})KGbU0>z+zZa3S4(|D3P|-U51GW=^4`B*ZOh{o z3zXp%CDRhs((1nBJqr`4gemEs+`_l@yY`~L|4_c#;yc2|D>DLQ++2eQ0uOE(mD|@3 z`^>_2bxRl|4#qf)=RAtL&F)b?P|5s}%~{ErEowD?{=Rv6zpy)GTR}M*zoUB#tC7!E z?hZq}y%ALc1S&|g#wM{G+8HK*ifIa*GcbdBau|Oo*m6?iixlYL!?gLOfVEkhhphw5 zga?me*rWkJi4|3h2AxTQcmq{EKeMdG6d9$0epe#g5hw|3b#hFuTBrW+*CcoV#aq!g z67M=k$vcHII!Z;5N5gutMCrM03u6iQyO2f~O>i3#MzvNKCny=+3FNErF8N4!xrBs- zhltmViw$n_?K0_+rIr22QG8rIX^n|74Y%3DI19A4O_sg%{p?69J+!|p3w0-I`}PFy z<-x44D=R9bjUR6)5*LWwA6k`jd_PpJk++81dm0P^b>p=c+6tSaoYjRcbJEhN?H3FN6)d}oiAl8zYUxE3y0@1@H+Ef+gx$jwXl^Pfl4 ztQwda3MJ;)6f!A%4JW#Hc1CPU!Z42U4vSLi-#Xkf<8JokO|iTAeu%vMkudA*tO<+D z$r@il+;>lPe^g79{0w)tYjE{K(Sw#>Bs%ZHgVeiWzMhR5O>(j?pF5~LwYB1; z<5eybsp*hm&%m|HrcaHiu01B`ebSfP3x9jX5zGDZW7B`0`9KXBh%cKGdne$r453FO+63f@V)@B zy{m^{OgdHpk}Zi)v5zvry#H)REzL_TsHR)lT)fBm>9$Gwd?Tq_V(3)D%dfDr5jP10%1wo|6ihfk{*wvx^`fr z?oRe&xx?aHcLU(mlkl$#dwX7rHISai-xa2j5|g4->achLH|(&V(tPEAkVmql+`+?+ z){#s}!%pNZYWDu3ER+1>h#yREtKB|)Wa?KCOgQ<-*cydJM~g3Q&&_}A&Utv|+rD7d zm7F}e$7hZgnWPWZv`6E_rL^OJ8_}jc*s0p&ABj;RuIEb!1{xa-F<9SZ63Vr4#aKyb zI9s`28Jb;^b)A3N`h5>X}hn8+%zR}zJqkj zbl9fIC(stX>wPGU=$t8b`m*GYV1CO%Daz4E=;pPadg=w+9vEVsV2%6o>xiD403a*# zf<~KHGFzz}-_jy_$cymGlT4&ffHKF^vIR-sM@n*j^&3+6nQ80~h2n#k zSppM~*~#s9@MTL6L`Z9<{(+N-1|91d^?XKlt>0Z*Xu5jXzrSsz^Lj#@y4xZXJ-&_F zO{-ZimPGs!-VjIfDMg3;7B>=mC>8ne<_}`nS7T8?KSqM{9r11oh_aD#dw8s`Me&`6 zK#x2p0bTM`(x!)$RXa3w&eENBmX>ywM9%6%=#yoWd_$Jkw26@3_U)U~Q_dPrP3x71 zF}0zo4lfDLwvq0t=i_ioc|EjtH5IQh!AXGh&b*pO zP*2R~64t~x`FJ?PhUGddX^8Ug%vRfIRc>}RHlHIqFE4dzb6*J~t^h)n;h^3hUieY` z?$WmWVr7@MZA~j&1;C(Y;5PIAfAGZfnbr?~zn^dK;P1k|mLhhy-}4 zU?yi@FMvV7hyd_2Ee~k}`qzE(sktr}pwjV0HcNT=#sapv%e~WRp4zzS;;)H8j(@4W zU1KX+Von}=I5v5l(cSuVp^he##wYqTzjvmn!^D>niXV28h>~!!Ir)d&rzgDUmrj^O zz{x@G^i7=5O5ZuS&ziwh*FlDbdRrz6hrwNAPG_DDNygXJ@zwec46;s2rsvdxOER4v zP5xN@WBFxx&C?X@L>=p-vu$Sv{mvBiTEh(e<7_!%-N!%60&c&cN8Ka#{qReVe} z5(u+7DBr|n0f@v|HwJXK>AwRqYVCIw&4@Oy9uo%Ik#V`56z0$>nHnj(cK@uXF~V)* zgP2jp1q5XePbnDZnoCec7SjJ6UQgbx;c?iA-3;r^a!9l5oVgdq<{spQ8jOEaHxZ8~ z3kV1hIh}3059@0XYzyYdaVAKYQ3gD?U2c8jUah z2*10%vAa9Bi#}gAKi-Je5s19pDa3nel`5R_rjwgqYB1&EA)f}nCg_qJifU~b5ery_ z@mPXmZ|{RXo?XBXL|>e=2KIKEa8***>|jS|(B|*%++!{N)qxMR2|o;d?$*VZ`b0Lzvo z?~93Ya4u|pl8;*=!vX`Z@4YOzW9o|1ngZJ;$j(HJnp!UXY{+<4t zZzGU)aMhe4?Il#v-hXq;Fv2MX=Kt84`N;4~6S3-&%ZycnP3m63OJ${J_*}B| z_03C!tsXu_Ho-|cUfeGIFi!Lhcn&SDEC1C);_>&&xj*mE4KHMP9sF0!%#C%q#opN- zCD6TuhJ{6WdlqCZ zVf2==9Av%fbdy{)v_j<+hAV6yRlhshfo8^?QM|E`Xs z1HvOrQ=goi`jVMOn@7uY=zO?gzx}n`bW^f-TycjvXuV+iiRpinH>9mb?3uScCmita z=|%)Ql4W9L>=gA>tU~$7+tv$xadG~{?L0WoQ8vkdeTX@U)7M{3+lZ??evFojDK3RO zn;Eoby1i?ogRQ?cbWE9aI5yGQ1iy%gs79$w zA8($CTu&yxvdR0;(|%>}S(XHlQ^*#D9hA8L_P&S0kptRNf}?r;x#KruY;1@gfmw9W zHKC_%`)$tFn`&wgxO~+Y7*dU$d^y$XNgC<+wVX%U796&IkWv%U8jy7}M@%}#LQHkV z(kZ69c0!vN$#es=^0c=n$+&cm9!Si~XFG^Zv+*B#$I#GZa-5%-5B=MCGQZMP7(>Ct zQhDatzhXlP;J|a6BJ~B`CC78>*pEO zFVKH0e$O!-^pOSkDY$7}NkI0S?9lts~nou4#=D}IsF0UcrxyUDY zv;e6Rev?lXlM7;wT=*ju^kAGe&hx~&%j-bZ2VY@z&eXB0J?F;6BW4I|R&;#4IBn$# zmdtR9cC`|L(5i7T!G`B(d))=BF4%gOIvt;~KC!3b6g6DvJfvne_hB~$K-9@&3D#6u zzr3O8B>=1u^jRpMb-XsSnqS!{hw&%zgKF0;?{Nl~2?O6e1|~8Q49AI`Z^8QI`!rJI zs1)1UQLFZN*yraxJn!YG#QKvtzVwf{_$xTM2BIeU4C6dCSXfM5%&DRI3)&ie3&_e zt-zo@E@xC8RrqIb)eITyeq$}dye7y>c*`XL??QE120lqB=O)fpX>O#3ZDPjGu-@7I zTybz<9~a#_d6X{jv!K1f7C^7oAk{%FdnCnDL)tmTU1p-=+V$EvIwI&U?db0EtF2opW&O2ckFsxctN7>RZ|SdRn=HK$=l9BCwUXqO2oh}~ zDnw_O^`H0Sq3KSgwF1=75Dn0PpB2n6NGL##v&(}hg^XNjDMt<-x)31`(=_SsFRvf`<&x%+MjzOIJLDK-V_e>SvUu0zxS@yRpX}CiK7BBsR4Q$H)sv<_(`g@ z7uJ2sRpT&lqhVoVv)h6V=e3wR;=Zfy`h6KU|OpmSQg542))d=x*>hrbg0Ja4MQexK^U@gCY( znc=xl1d=!6^F=)OuHBaTD1#?@krGg+I8h~-5m68WKRbe{zviECkJ5cKRLhZ3>HaNF z$w`~iFY$39dq&x~g5$)scrE%uoC^SU20&Ob>`h(5w5Z zsitUqSbHKO11mkC_p@QyF#q+nn(m-J?G#wTvnQ6uzT{4{#7{cq-})3v0GYGwBotMz z^GJbx$eQkj4L=Stf9U{8Eo0m*h~+h`*fhk4m$Y`hIJ=SZ8~b<`7s$~Qfuh_cV5XRU z)7jP1q$Op_pU*lxt8EAcEm{`ttkG+7LTS|dWas@Js($_SXlg2NAlu49B!Q(r2+H35AFQ*99Fek5cuvekDbfYNU#4*)!CI5{u9On&1?#g zJO2JSX`dm3Agv)~w)vThml`u&k4F#2wk+;<|1Ua;k;%r7wVZUerMdS4&RzY#cA;WtnKJ1Zo4h0hIy)w0_8m)X$YG1!`hRlAPUH6QX*6 z%9@Kqon)d>c7LMiz`22cuK^f?dw?i~y}Ni)3C_116e$Vx1TCR$4`afIpBAH%HtC$w z4TndpJ)0CwYH`^9*?MJov^M-$WLy(W_%0kApZ3H_$*VbwxE1dF@bm<&7IJ)PasS;is;;J9mtN5H|(DofTdr8*oU z7w#SUu5?kT#x2zAp?`~QS`kSk*z>`Wj0vUQH_z zzIr?~zz($|N;(s2uNO;~9kVlI;p8iYggz?E=BDKV`&daxa0vBQ6O^F${cZp33M)s+ z#O5&2%qEW=4h|U2Ko5Nb-X8tD-90=Ah3w1fAhV3r#x0-F{@yFIdYrszR(AdwAW-mod*q-#mJw2n;NTurR&h<#YpIbUd$nfM-73J_O#)(s+y{Rw!N*L>~ zdv{kNdpzI*?dyA=;|%WUX9k#B?8c8OD8xO_fj2iVj%|0ek&qIZXt?zHAB$HU9fcn^ z2Ki_<*I6#m;IP(py}Zm6u8*E4QvJN-t`X(bERxi%mUy^hbU^60V%a-OMVR05#<*fu z;YEZ5YXkh5gZ5N?x(TnPOnR$BPx^Q!E=rKiUrDuH2rW?kOtYx>)uQA2V0hm`=NC&ZWWVb%YHZaT zQL)?Z@tSRx7X+lGaOgq0Q9xJ-#Gerizh%vO=KuMZ!EsyVku7ijC_>7N5C+~0{gm;} zIsBOF&3L6-HNl4!@%-XX9FLSMj|eDC%fKZGAre1cxrRIe3HjMnIOgMPeuFWs$f(|O zwsoDvrfN{@fyR#a%WbvvYP>i9zmCj2kAp!y9#G3Q!xUt;PQ%k?_tUIGzOigYO9&n( z)&}S6nb(;4vewo0VDa+k2Z9CxOYTRmqm;jnIlYml@ww=%C}=3tw`593hC)-wTZ3Bm zGzzs%q>So9i+P2SS3M7!_){7ovGwsFKmjBsuZH`H`<79Hgv(L`-E&LxA6e;%zVEep zQaQ3?VwgY0J-temppd1rI`b$?zf^k3AY&(P5${FM6i6NTwG~1xMx3CQqwyRJD-L>x za!{OJa<`pFg)CqYnIdwAz=t(F64|5Gt>^Ey#ENw!1c)riIdIKJa4ESyjf2N=WfCDL z&E$SMF1EW^^!qi}0=81+{I2W7^SkaExNW~pYh;TZd@t;sUF*iBW8DY;Z&5n}`-}qJ zTaH^(T2ScP*-Wm|+Yd$LGDj*<@$9xzy}G%lyS97ZYLvYe6m2|1U*XI=fcf#(`4mjd z6*@Hf4-y)4JYH;EO(@Bg-3#Tg>@keXF@2Z~B%sW$5@hQ-zTQ7t->UiaS+Wyl<*5eu zj;Cts&DIjopc@R4PC&|dW)o;TJzT{pmidEwKZC1#;iO?u>P1TXalxwpIoR(>*tM<< z1TCysTFK-rDbBvufsJX+EH2vAH-`JfZjLJ~NCgv`wh889S4&k)UD{FMO~wrxo22uh zMxS3EK(FPdO9+RgvfE3mRr~^`w%%Q#^=qu^9M#gYoZHSgTln zaeRP%z`u)4DaR|GI3a(_+MS3#1w|zrk$`0*?c?y;<7)t51$Vq&B*iO@Kd#t2eknZR zXZsNOgfVhJp?pJeZDT{f?jsSyyyfmCw+8Nh_}87V205vFygUVv?{9!y_7*w0yjzXU zgl+}mdEQ0Ov?*9+d3ej*Hq-8F>u_%p{s!ZvS5h}fBHf@K{+ac!XX3w6>JN#5PH*)_ z4N^E`dSTTP+&Nz%soM9B+vb5e#uuwNfpB$V7mxy1p6Bl7z?Leylx-gfR1*i=vA+2n zqcs{)M2 z7+4zc3!mXF|MyGAZX<4kUYPXNnbK5N0t<&?CHixM7B9u=+(Y#cc3$aR4R8;=gG=1G zRE0!t-dnu?Y-OesWx1Q8VZI$qQ19TjS){u>Tm2#%WsBv7Az_33X?<{T9oQ z|94m8)n?_1z}%wm>qhr4ds6)^aMVGo%0-~x-y2- z>G2Pp&OB)2<6uGxpd=M;3G+t4@E1%DfOp|%G~` zYVl>HCGrC%nS$xl|ux$3CQA8t2_iR*#JQP7Z8~j3e%(98n3>j&MB6*Qx&V8q2*ZM&2N?c zUVW%snCAUK9{dk-PTlx;(?Cm6)Q(<#)dTV-omZCkdsVZ5n(dVE*h_TJ7ma*J?e-7d zhgWNsR!6%q2qEA7tY3zs7gGxpab!5S>%l<{J&-5=3q@rhG39^94~%M&5g*=G^_P(C zTK+vSz%eOLT4Hg97az34Z`Q(Xt3I-qys0>AkFuuxyYr?K#kF5%LApSVx$4cx&>i?~ z0nQtRrAq?D!;T2F6_gZ#i*@=HBKiwHN9^m7*K?O!N+3KGFAv~HOU)u+2&M{V&@>{w z2YX+V63$%p-lEK3cV>5ozRL;Nx4GNDyH$>T(5WdvC9KgO&Ib+yw#*n6Uj2)>8M{_i;KuiC zv9Axn3yeAb+rW`K@B%a2Jdu~Wjr7C!%oo*$;dZ^?b>O%` zwCSI2f{t6k<7IQF%`V%Bn$uVGns$Oo+ltb}*L$Z+qS9=ce0F4^aDgienbVH~*HROn z2J~ApaxXaEe$(}7Lj`mjOg=iwDmYNR9A&B>nQ3XI<65t^lTEC5-Q{cSNfEem|QeKAa zUcKL;hg~h#c>Sfn1;BQ6NAEyLNRp$4jsXUtN3@RI?fVBpK&xGtNC?Y>{Xf=q|F0ap ze4HQa6&K0>Yk-MM#1B{B+t#j-+K5Gts1{V6-U`Hu)CE%eCJ+-lz^$y4%Q@ICjc4+Tb$0(bax2czsNqYg!H z3YQSRsVrfFimv|4qu-Iy-iN<*6YKfmM?hdpo$+A*jO8o1{+XW1TlzuynH%@5-w)r> zH>riRua_x|CC1b0*-UGSKM6Z6dyQ+us~HoffsLx<)$G&6l)QY^S?ZU7U0RBGI2Z

lhHDt-xA8Z7)+o$CV^cK!~#-sBg1t7wOK9j`stO zcem1Y=%6k@j_G{Uj@d|`!4S{+=~AQTZ$@X+tHHHuMWrz+qH}f3->V_wg$aTyjUc$? zC-I77ZN%(LX1Hn|WJEb4V*cstrnBRe{JY*anNS0?mQ@P+n-cv{(+ACx4gnH+1`rZ6X_HHsp` zFuU$@!miJMBcy160&*CPKAGvJu9NVr)oxU;83VP^cbIScY^kS)zf7j{z_m)ugdwHS zTU+2Z8u^}nwJW#>M-fB1LM7pY-c|Y!#NhC#>_GNJR$YBd#kV@S!|@>^kr-P&zCIuvve$*}^1@<5TU^TP%5T?-8_U-RkBL;b(WR9B0CPn+)Aaxc@tb5@Q^v8ucY=tBh% znmDE*Pn2D7aVq|wkCnO0e)y9rwTMC67WC~dJ|NvMPxp+HJCP>q@1`)fHaScNxis6? zy~TSL*^LTAZ~n&-17(o@Xb;^17^WtCGC2qg(ZXA>5Bk_%unpt$c*oa$Eydu?UXx_O z!WzAWLR7eV5&3-r$t{8|J)nSwczz1zf$yV09!M3FM5gGM%df@qANT(F8uf5ZT z8pa6Sr=la)D7zd5iX?lfD z6#Lh_yK-=#)?mJrf}VwE>uzye1Fn-+fN4Utkm{SXKN3 zlU#VefU|t0`4j#AG@&f;LWP*-0tqIKb8AFKbkF*vmMc&&>u{$3ea(DTA-$-q7*!h8 zhZt|oJsUceyj_3u77a}(8sXZEb)o*Rh$TP&pLLK)mEP77s@tu<)6>cv9bCIs{&X>H z1K)>wR}Q{At5YsJKGJ+7KQ`i&6U!gVui?is*2PWRBzp$kk0P8hM>PEdi1o5z- zQGIoxYQzG}>5}bGlxF5VK(;nT*em|cuEU%xx$L@2{7AfFj)iS@c2vma42=)#Kddc= zQ+=w2z@{2Rq&K!S1GgbGiO$^dsr9M5Kc@(-#hbfP#MRhmrkDU5bBFufu_g)G|4}g; zwy%}Q)?`TrJk)MS93uxHi|crHrqDr0sh;7%zrPKKiYW=t;1-aJ^HrSbg@B>}*IU28 z{0JAhqK*Q~T(~&QbOzs$n{HWjuzzIT$?bYls1&fty5Xe~4(}~aprN>um5An_1(8}Vw1&!3n&I`whl&cV;EPDt zg5KF_=pbi(URCGCdFb&RhG?2tMl1r}4uAMR-xfnOBCLx!yM9KQ9HXvHN6w$tTXhBkKUBYP?lowGtl zb?(1b4*&dX9Qi|xBB z(d5)NA_320QaGDqWBThti^!e_`Wys*i1wAHDl?CBjj06fqO!@W>v;GV$NGEeU&p|0 zT1v$B$E5@CD(A2L-CosHHag`ve=2Z=M>5dXKcSjjo#Zx>1;Jz5=X*_x7)44#X| z#7%-tG5nk>`@kRmrV-*PGy z0OBAgvfWY_E3t)97LcdLH$0XC zGaqlTzY!g=<2(4D?H%Vc5h?|9uuRVPq^TaBORhis%wL6Fa+z0LwR<{)BW)|#y!vU4 zn8}aBi8j9Z#pZcQ(U?FEO9_0@*ym&lS6ptW^A)RlXz*rN*b&z^{ZP+lX?<-sSvnS-P81b%e?+J` zk6@Qmh|m{mnGgdGTrzsWZ{$hhR<56!nJExqyE_zdccXl_aEHa#>Ce0u>NMb9+G{k< z-rL&~8(dR-|3IFiut5(gx$Lr`NUr2eo;tsiSNf@Q)>wh{#U0g(`zm4~1ZTwPL=7!) z_ZfsQU=zCPLJ_(2>t3&QI4`E;$|5|yHWsnmSd$>ZT+$|g+(aUKL3pe|I!j<;Ry~yS zH5FUc5*6EQI|X$dE%4RfuD4H$m9MRz+9MY>xjwOI?lCsHpM875v(-*?an;M%P7-5! z570f!&1~{wAF6cK0e1;Z>gimU2R?(Vip$AQcs%-&8Gs;7Wi%``L^y8A#8Qn5P%!L5 z1X3;|!un@v1kn_BB2U-wr;arTzWeRg!Yg-i5pjN{}&WXaK4$@eb{ILp;2~pW3 z+UaqUA$14TkEB;3?B3xd>xTPU)TDCOB7L%1C)Mg*LVu-A6t(9O_w`uD3`DQ|=M2OH6FzNE>_pWx?AO!ooeq80dh!{haS zq8E{6bv3DSA)b2RT)i2i;$C=cBB&@4_5y4g@`4s?a4o}vd^^mtx%jCCZzKDZ^K8ql zYbEj)I}~YN_$l|_qwQMg;0n$h?T!#OJ=t2Om@LsNwl>8&D zqR^!Vy_tQ)3oquCm%%}~N#6TO5>xqARZ01?Ehg=N?PQL?5sM42mERA+&WhipxN^m{! zrwxuF?H`*v94%}?6?(5U`I#`;y&jtHn^b8OjOgIHsx-epxgH|dMh&6@zAJ}nQrr`l z04*7LG|+$TMtI=`Iz=9;yuj+O0pviJX{r!ybzg3Y7qCwqYx!Yx4$=nX6ah&Zw>cKK zcrHw~660Ywfd`uxQ2f0f_}gli&7M)$WafGpF!bLozEih93dCdHVu_ z=62)I8;+Yyjt;v%Sy1(nzs}3qg;R?&=u2iQIz@l5PE&ccur{T}^sTwINe_Lq|8L_? z`3^eRJ$PJrF&Prw6A{s(;4?TDpF-Jx&ATfxIb_2J6tGr+Wv5Zr;bz`y3Z*K4h~{Dv z6aqLMqjCCo_%y8cfAmatxS8ake^&U4o^L@L1v@6kdufQdzh{%z zUoQ^(=uQ=e-s*j11B0)oxl(oDTkhidm1|=o#v*?(S0?N0Jb#Oy)g{k0uMHj7XM?O| zWLAO+^&9;xo+bu0FgpB6?`t#d2JG~??;1gQs^|y(i{YV@w-6JP)>W-6#Z*|ORTq#1`6?Id`ry6m ze(X(IBUG9#$Tih!BZ_S{4-7m~TvH>^J0fUrTbF*h9-q^MdjHd?uIT+9Z3AbC-dNnW0W zJGz`SV@vbT%;>CTA8XbZdpXpz-Km|8T{*Gmqm#1f%cCGjqQQ7<&%t0CbpcKr!X?o@ zRnI?7TmYfoi+LbmtTMA7*oS9vjqmusv1||_IxmKU-U~f1+d~O~;AG)Jmbi6!)#^M! z)1DW<0mE+rg;X>-Izz(tQBvkVYvR9aahc}B#7DU_X{CM)g9xXaLF3$`OXp-n4 z-zXQ0sJ_d~V`k-m-RNzmgTsJof2t3*ttK3n-S8wFT(5dGZ1-wv9%RzlB4p2i&&e= z-;(2>$ zrSGG`M~Xl%Tj|p{GAqL9r)QBSCd-y&JYq#DT3=OF$DO!*aqmp)$Ov-m^>I_842Vv8 z&i*2CF}miFIA3*ARSOt1*zjvS**DN>7A+%?(@j%Oi#84*PM80war1=A?-;6C#+j^u z_aKTtkSL+mH!j=5GdSJR!Tz%~U&QZDB7}|Po(Xj32}sXyZ>c#Obp)YTY)lvS!WWmu zGH+pTp2#kh?1OVcHvvC~D<^+CMA{HW!=1qQ>D*>l{JEUi9QLAjIEQ zhZB)eG#P<&$OAS}B_rB~X30Ok(!E_QPc7&vv6t+WyKNCa!S* z7d^T7)6Mg*DXAeGE))8S%3)D>b~OB95PC`eC0|Bd;@e*N_M{LjguBo(k6&TV>(Dao zL{b=gdHdnZFW@woE1QV|3=&ryqt(Snejhsx?2V8PM^JYpoT8|=}=yA~eHqorGm{rS1#?S{oc?Dd);ns(Hd z+DK3S6Vh8VCZ-ySV+XcTz%UC?-7%!&itZh~@D!CR$dJd82J8=I|MF)xPr9*xHt-ng zB}X9u7N_Ol!`l-CkHp4xvgNXNMoq!O#l#k#gE;Z0b;xH2^omMRe@%=BzpO9aKboHI zEN>}f2RSg$$!Q+2pMLO)$6lO+9{^Ie0=L@iWIZV0%5(PtF^+-V)PmKIs|1tfpO4`( zvmbpKWopb5cw;tyhB&J=@Ye z1=N`G%ImQWd7+l`Gw)wateXMne^WeXntH%G1tjc1(NEK6FAZ1(5YGioBfM4=(~+gq zAos}irJK1MbwgQ`Ktsc`=nEQa%(~d7I;eG_jc@4ZtPWHl zu*P2|&!y`m+fVpweyn8`jC?ZTwJ8WK_Z-buoo3r&A(a{3?{4C|ZBf1xwH-=U4sVZt zYPDjwoocVCX{mYU9eI3RFvuCn5f-;rzjK5~#oA)J$Z0Vh8ZKIfm4=6$USTwaK1^SBa+ z@^XITQdE#rFeze&A}m~MRBIlIhOjA-QTXZC8m89ii1K@nWcCJSLZFTIJta7P?Dxfz zV+5i=o^P3(^EF$;Gu7!lbu&!14%P@4i9^9VwS#bbb>|*R8-s&G;i7Q<@sc_|jSCE^ zry8@F{u2sUl^IDoMCB%no>O{W(C5AJX}3Q1rl_-TH?Igaq|F|h@?$^$oSmg&p)L&P zrV|pF9nWEyC7zlJ5*NrB%Y~?HKd}YWw7z1E+Wz}#?}}A;?MxJw1(5fIiRZ-nEpG8@fC` zZ|lzWB#?*YO)o_t*|QKcq;ES{c6TBwwI)k5Cz86%ZMQK+0uUrY21f$jn!vu8NuxcG zUY?H^sORUlnjCfgVcx{ugOJ1|$jo2{F%u*Dfs)~T5w~#nQl?wKA&r8^4S@37@JAzk z_Tp;&?lt?-NJ_*B&5Uu62*IhUnx$j}hZE9c_Z<<1l8Gr}q)I_r>Xa-5qCPA=ODU8j zW&`mU?j6J|G)<+}3g$dW#xrikP>_J*KPp52g;)FgUE^PWG>tfDXWe_G0m%%4WJnd( z!Xg9Ga*Abowloa;QzWnSv@dU<;K0LZ-DdD6CR9NS#AC?GWA86_5C zLC#ajD#-NKd+!5@D)3RKMInExZzu0FLP4*hjx)ohEl3x!f9NhrXPRipyFH?7&dXAo53w6`WY)oDqx02mCW z$dEixoHv2K&+#DP=r#Q^g%rb|XHtd?+-zhVw8>2Fk(?x>BRUSDaQ6bo3*B2L(Vz*% zPdRjvne?+gJf9(3pZ@~Ch&%m##!Dy-5<*~LtNKQeQVJ8ndRp&$bN5s&s!AH>9xW)f zlFqeTOUm$OVa#9>7AB%Lat>ylq(mG!posWPA2!e#Ao=Ym)ZcY{mFN0h4)~Ix|EpV$ zuIoBiP36_QyHBdo_UGrPLICmJY+24iNf~e6y?6I+b8FXh(lUY4+s4AvRQBiV-t6Jw z&HcmM-MXumx$bT!A%KqsiKNk`top-;?`A0sdVKo$Fx^d}rovGYet&ne)>sfu`@X+@ zcr#5??>lM zuj|_OU76GI;kzH|>D-+z>w5p@ou@6!0&oth`_}It>L1?zX*GI#I>p}i?&mV)zJL7m z@$}~YbbrSuu7d(3B0wTSh<#rHByl7pZ|g9T2;$G4*Q@bGvMFEj@M?WQ>JC)s0Ix?f zB`9CI#%}%^^J^A%^xm1JXaz9L->{mQM_bEFn(M3O@zuCxu7|Hry`(5ffzw1tLOq@WBPbk)94pX98u4aAx$q zQ$#INiy(cwURWq87{N)DlthxC^st@6Vf$v=rw<=(D{WgJ?w2CUB%C0O1PYZ>GgGzJ zI;VMfx;0J+Yg%vN*~xo`CkSEQyq~8jMzz)V4x^+}imE*CYh+rud89icyhlW|b&Z&6 zC^JhL#j+Q1J!a@}RUJfl776;5W}8R*VKrfLW@P zCAesrPUkmunjIvfuz2;|2W9@x_muyG#_wDDpj*SkuQ{v^$@RnWBXT6MN?Dkc1x2)$ zIu!xn?%b^hl|;B@r9-Zrd1Ad=PSLH_x}5Hk$e(WR$Abem=!O5Pq?^_^ijmou5+FIN8O;hGti`22F=73a;>RW<166RqL zowbV83{d7&>a1now)T7_lFQ}l)|NOI+owj99oN~mefKEyX*v^EW*%yTIqb&hU>y43K?ZB@PJmVq9*EWk^!=NKZ(LipfF=kR?enkg&)Np=szW&u9;C8nLD3@#yb!?S`mbsQvltn>z^C{d_Rf_iT4o_hg#3-$iFp#(r z?)BdHJ(1oUkU{|u>l+2CxK!~l&lZ_1bHdi^rMHb~QYlQ}5qf&SL9}~M6%agZM@G8c zo$p1Yike05<~>&5iAwM3p1r&8IS^??a>!eS6OjlKG$`{y$)L-6Ev0A`o|Lk8-}b(H zPgXKBbEjk>76vEM`m~$}^?MF9glA^D=PYG3g)*}gg?nZ;v$_GC~uiCjqBvWGK`=V1u@F5(K)FLTzkL-~iq{QiI zH01jz0Vo#{V$fQa)CGyUyt%u7E4nl@KyG9RK{?pta_#4T{u&FYy?18z3<#4JA{HrS;?wDbk$)A*%rJA~o%x&7EZ)`489LQznuHkU zo?upxSir*L2t3k_Mv0dQ$UK;l&AjhM1Vk+|RrRegA}LdP5(ipvXCc74`LyEXE*c~t#uKeriIEuDBgRoqFPE& z+TP7Wpjx$U!D*lo2gN8zd)o8k3V@kwnQJMPM2rf!R-KA!RrjdnpI0-BJytiy0yzn;in>mstn#Q ze*LuX3}Aj&ecK6;I*A*FTcT4)Ap%l}AxAm!^Q?GT4DsSRjh8cT=7}f)q60&53tMgg zcrem&82_8CdC??hd{#Lei+_;B0f4_E|NFw$yq@>@*FcXi4%d%8tgoD}zXOYO!yF|8 ztXdPEswrZI2pep$EL3W$L}UwcDJ&I%kSc?%H@puzqk~$#EV~oRd^$g{o*X*J0mdaF z2$2hA0+KO0Fn{g}{H?$1*ZuebnYo2)U%6dB&K|zbZGCl|&l$7>&vnQ)|MCeOeTzh- zsz?qbl`5(ZiJ>JQ48W2aq7#~1M0kb=F;OymWN*C?R~C|k7MhtmMOu@j^dtt+K}COj zv8qHusIrzq*@M^xlFI*7`}1Pf zGJW6I)>_*-v!0eSkr0Whwzhlo2vn81OhEK4+`_#l3ot5RX4b*Z8Cq*n?AE+nPfH3` zR-Me9iI}-aQDQ?dq7aYYF2u z1V;&=p_J@CR-=}J#J)Ol=Wt!_}0_>MLLFG^oK`=QirbVb1*E^2@;V(VbPk^d0`P@B@q+~ z!JcjJeWocB8IgRfiqXDTWJce8LwFBk>$`hr(a6?X3(qpui3>Z7poOcbs#3x{!d;Yw zm6Zy!aVo>wMwIW~y}4X2L<6v8+qTuR+?{74>CKUHd3lS0)iWfWBK2k_egZeK=oIxV`oj-D^i!DSvMd}P>*_sqVzLH{ zwq2e-{vcXTWty1_lY=@!KznooK*n4L#38lemR1Cm4R3mD(m6i*!V3|~7F%+lI(eN+}c9<%g&BzRu_ScYBWn&$UFq>0}U% zYAYoq;ja}U{;`9;9Dnuwe>D+yyG>u1yk8hchMq+@kV&3>Xe*SdhNH)5i-sq9vLOcY zBq>?BmbuPTHXBS>Dq4zA5iXRR=*`TCW`Hweuq+Xgh=g+y?tM3FVYWu>+uGYMB{RKS zOUq@Rr&?nR7L=leR0uhu#Rv*2Dnz<&mvy}W@%;QOLeun3Yc1o*6PX2*0qmCBwu{)) z(-VMF)IB%f+P1kh&X}i)L5hVer3iWN?%wx3SyhS@EktZ-5#i0Q>vPpI*QJUoG1V%} z=NXoLl!^zxzNgC>l&`-gfyEq9%fjIi*89Jx1@(fNe~ zKN>+PKiLVFjGuu#zSI_n+VZenFWBxw>@HvZ9ZBUIqt>xY|LCZ%Cn#PeO1>($|N8Me zXSjYe_I-zDebt(tnGxCAHZSK)U$@J?`Mvi@yKEPLR8{%2{)MW}^W4kQEmW)O^n6{D za|XUDywVN#B=iqubNe4kqyzT zM%f6FDm{r0V0xBa?`r(G{6U5Zd-FkhagbpazNwhtGu#Ml`-ytL6 z+kP!VWiD5{T-vqPTJ=r~1Mt>0sfuPs?>pN(b1klj5FwuBdR?C`PxDlK-$?7JOcU$# zb=$jfQkm-Ya!p{KXGgBjPsV&Yo&NLvKT(GF-quyY=Vekx@B72uY3=>~;UUTdk>&nm z$orFiGc6D2x0?Ft@%sF9&D>a~_wT>4=zsp#|MB$cdxGCDRj-f!@rMV7hMddY{rOw) z{nnRD7&KYCefs!MfBKekfBfMY^=>)8H`WA~w(_B(nOonwn<|${BZdM<4%8LrQ4L(s zUvH(h)5Vs;>j_e3V*`v^p~fMG`U*l;_iF(yFd5o}CEN zJp0~bN`MQi#TFnT8t`Za4Tujj3l|~oy*0DK;;q}d`nu<~NADSd$VrQ*MMP_xZ=1Mp z+eXr}Rv^t9n3#_(*WLNxhL0f?2G7Vy#L&S$EvLeh^$i#lH`CNiMG&dX%A^b*ViO)7 z9^EXdpoqw*lcr}52Cm`lAxGm_nIAdDmuCG!9vJd*T8#6(3a z3)NCSJw6E`-N&7?9+tf|`yc=MuhaQVwXXT>%skIMvF`2NfBL7(<%y-W-b)#k#--N! ze7)@Jy6^i`#nbkEU-xU(!f{|<&1}6sZ6GSDLCaLE_b{h~f~@b&nk9>{yA9#z1dGU! zgc!P+Un0DI*8l%S2Pbb68b|BVXAdjzwGWXI7U2Mk1i&l@J`NbZ-Iovkfn%hTiFugE zWmybHbjaHQ7(&9Q(@8|qKkKB5@Ukqe9XK@+0T&hmlZ9K@a(+;)05X$rOZo_Sfgy+b zLIawu!XxCaUGVPXg}_@qXInt_}c83AgiX;CgZ6naOm@Yc81D5k+- zl1X7MDoGZRL%|bZ9_aZzJ>CxR=MsO)2G8G|E!qbwZC{q;{VNfVQ84P0NVBmq9d2^yUXI0T! zUXp-75YfPT4MLz=CoOycod}sZy!~L#ppm5+!m-2xWClf2M4APJ0VWWV7$jg4nyLs> zWeM|8{OON>eDlpWDD#KQWk;SL-ayOabxq`TTQeF+=lN8olPWjw-ukv(dhhdGn7O$P z6uMJ-a#r?c;Cm@m_!D!q4d5@_<#G)O_B0^#o7-Rl59yzUM90ygpM zCt@7CD7~1in-!x#=FNT$2*p9@MiiMoaMi>ZREFb(ZZ_*5l<;|NshJsIh-4~Jx<~j9 zGlC2iXty|>POUZT5s{Satt-jAOhe5C1YuE`z#xKI>}h|P=Xst7lw7`cqhe%&GE;Ei zy#AxcOI-Y7l>heQfL=fTHAhocC8BT)0RRACPKND$2o45y$^Zf6#DU1@eQ)rqwat4A z>xoEb<76R4NJI~U$dHEh=sjQ*R74RVN0<-Nbm0Q_u#Dc?wu<+@Z=6xIWGPcE z@E}f6Dz(nFSlZ|`P-TxH&&`q6&B6_EK%(c)JCTJMAQ9a>V5US7h9EFehGZavGlC-1 zv%}p+-e<%{qbViefu!SscQ=c?75FjamFu*$$nL$H8#(~X3??S^LytB9V;<#jMs_77 z1Ve@vtO}EsQp$DRuGg#O{&=}utv_GZvYbe?+@1D)xA0nOSqc-Lua~E%r)iqDbq(Kp zl;*kj*27OK5mAePB0Vn8kLUWW66T@`JU>1P^SRExuGec%n#z1~NmeZ)Lp~UT9@c$C zaY>_YF!ueS(frfNi=UsTFQ$q>KJ3u#`PXJ^0G)yeG~61_(EtlbzLrNl^6dWsKG$uP z5iwsbkL&ZZA}U2GrO>G;hjjpLH*Y3K61OwB)~dvgu@j5d3eFzZtwHMPbcZ@)q>X+C zH!DP$$ATd(P%DUe#2bnuT-SoHAb zl*ka_Iw_i}aQDt6L^+f=)6IJaQ&sLAP>8Y{?bgk^GL-ph*KilEk**??Zf%cki?sBx z1S@kIBgrk(g}}_s`-n+Cvpotk%5u#z>b|SeQvf-o1O*J(T57r_vloP)|y8oKp8{yA+7J%^_xFRt$Lp7etEin`Z!B5E+0N!K73fq za=)DJ=>5aP!$X;BcWc&>q^cx0^vtVJ<*zGF{PY+^U;}ziuhK4GnJpW@OI5y6Qw-?1 z#|XYf07k}90Hyy(OeTQdeJDt2QCZI9jS(#}dJH%+gFqY^Zq~M4L~bQ3BH`o&$UxJy zoX+>qf)=C&M}R3W;4>oP!Rkyzpu7=bzeJSyM*--+8h_`Pese62qgf}S-uEOdH7(7;c54w)iVQ9dQFS)} zL^=5M0jO$_w4}{K2?8P-D#oNJSt&^q1Ndch{<;+w^*UzU*swnm3&|Lj&2}rEr-iQVHvf*tfM?T%I2VSu!gq zDQIu0-k%>o{g1zV_;|Uye{a*ZOtLJ?5_w3ju_h0=BLhgvTdNR8wec%d^vkyS^OdIY z{s1M8cE#7x;)~h3VJtX_6B$Gt;h8Z&OouJQYi5hr8MJ@5aVwl20JmGov(_4Qu0pj) zM1nx!%pMj(LM#LjMujJPYp4~J$&#a<01gm|+&{cu=DSg&P%X$bH`U2+xZKx;+LxNc zab^Gi!RPv$8i44x-!+XS5BNjpb;;!F%o=H6{}Z@q2%zApQ0YtQb2B9Lae$3}r~ z|M{O0)b}pTs^rb#kVz_Ox>?`Jqn5HTIMds@K0Zr0TVxOQBp*3&j<#oS$wX-)1r%nj zn@6{!IBtP;>w9BhDy40kc~j*&s7MnaDxySGSOSq5L();Wxb;b7R*h`klMsA}_Qkrc zr@NEf68}Nr83zg@vzs;BGm}KCR%QYX%_E*UR(ev2~y4)AM?Hdw(aY z>1F{4lknE|+V|}Jy050w<-^D9&3D9h5%<`xMrh{S#~)}7p7ypj`R6|jA(OCC(USys zv!e?qN59E|od1Q1^|O|X0`yW1Jal3JfCf3li`mLs1t3P7cfv<(PQ*m7p%=hW&vGQ; zbs+igFkZU@5rGKzO!rcXsS+XrW=@txM5XRqFeEc=H-@X$1lqn64eTF9ME7BvnONUE zyf5=I`l-QX!0YESAt(rxe)zpal#=lL@|6Bz<8_AqcS+L!*5e!+*8^^P!^{}c;YmnD z-`i#1F1@eocG-ZtK2zi`~xi(i7%ZDywSjn@2EnA>Q4iHOe?mWft-6 zx~_X)uNhzqwlH%ER+hrjx7GJ8i8zu$j3Ga084eTx6bGnBS>`&ePgk?X(5ylTN-%R_ zfrE&I*?R2zPT5N--u-SqO)8F(VL~8gr{qLy`{MJsb}tA3WV z{_&5oV zWA2-$cTV(m`}Bc5sc!x0Nl~ix%ev13MUopwR8g(fK&|_$M0g}|91V2wleo}dT1z9( zMYvVFzv%0JmG2+V$Ka0}q&NVx069|Q`GxTU1`X}6pZb5t80}DFu?$uh3QD)kFru`r zOGc4Nq5hi@`75mh8IuGcG5mLmJUx42{kBSEQ|9I;+6Pal4mOO182Z9|XK zG~b_2g-f_!pX|-~TtqXX@7wj`$KHE-2+;sKT(9fC?L~{F{oB9(+llJkd3p2Z&AxBn zeRx*l^WE9qE>G9%^~yw5Q+&t%{NNIiVec$%0h}0rQ#Cw!U;e*bYi^`TD7DWujqE!XSyyB|KBzIpJ>eZ4k&x?I+`Z|(eEhe6 zeZQO)d3n0d2o{R<8rSWcyR-N0^4O(LOmj9%k$IjYaNF}=Fr|t38c+B#eP0tZy`GMF zP(-8%2^RqF;qGSrI3^49#Tr>C2Yw6XAWq#2r^Z8V7 z_@iGt_I*E?HV#H8hzL`Kx6|pgU!E_Qr`($&;ZaK&oPuFlxFB-BtRgbkNmX^4LYYe8 z)7-*R=-vDG4{zRw1y7Y&M}2j45PuHFBS8xIP&e%H`nP31LEaAxBzxTJP<8>6h#N^jJt1 z(V|mjN%y^V_f%qdSl`{ed-Q6iDJkqk9q_7M*SWd*y=rNnhw9`J+ ztvOLxCY7Gpd*6GvaL=&t42zC~&?X^nyW6(69xk5Y5TPN5H?Hj9gC_7aO_@3TuaSxO zK`h%FWeSM3Fey!LQwW?Zgk53=8kSK>6!iM8H)Vxn3;$9LnGeBg$0`KPKzkmPTbbk^eY}fwu zxV29Y@Ba8-|M;i8K5wlD%XIfpSq+(}#0M!&3@Ql<#?VvzSv0~|v-Z_I{+tebSy6c- zff$szH|*lyq9yay_~nEA4aaB3t)26B@*`$!qrD=KL?npFB)2S5;xPFG%Lz))NVrfc z4)`QPby?XY`EDaB}zF zY`gltW$!9b(LzY;YunN@GizZpYpr?j5kWx!F;meApx(P#5LCDjq^NVE_rC9YZ_RrE z5t<08r&;N0WsMb;_`MXaa_I;0Tlwg%~A{HiH=1Gc@Agy~On~{iWsUWxB+wShF zimYR2*gRg$7Bi20wwd+gNvcO$HVKhoQ&oW75KhNcVo}xazyD$1_gYJ>YHlVZ!qCzp zYhuJA%{z=4befmz;}d{~`}?uQ>fefm(Qc~(iHs?zsMgfmkS?Y$%7&9rcH-n*I= zlKslIM{6R zcIJoKNz2A@`B<`?ueUR$?8si%~oL z0(bgTSNkio^|x8EU+}qJzu|b>tBdz5%Zo3@?)7`dW&EwDI)*Y2;p`+D5K8A@%1{D> zGMo&^)HCCVF*hMD6uAaz&!1!8IJ^KX7(PvIZ}(3A~h&XJl*%+M&WSZTM;FaNehSO4W&WC zDpg4>ZMW`7Dx#vC@NNtwarf}F$lQ9{w$|OiEMk9pe`W;2(v5@=gFMGyR>)ws#mJ7i zGbcyLktWU;;qj?1X-OQNBOJYVq5#plclVTpgSvYVkus}3T`se`i1@Luhf1C2d42jM zB22_!n78YS^g2y-o~oj4yQ&t|(>x=hl=5y_VqM#Hv+K^w>FK-g6;UPFTidQ5{`_YK z{`u~H15w)jGX3T8Q+xb`I_o!W5fSALqiQn(Bu`F?NJ`vTMt*W)(X0H+%iD+uB*!!k z0*wSbWxm+8QTdP|<2+{f?MmLv*6onD$1fyVUn~jwG6D4Y1pdD9)#9WVwx0+F1_pJI z6ZPidUUf=8g05kNI1z~OZcQTLg~_xdf~71ocpi+{<0sNBTzh@ylrNCrFNiq4`mw*` z__0a)Di84a^8SjL^yTm0bDrwmfMIholhQ3CDV&uf!}s0V8etjDwuly7;ND7+AAb12 zcLE9sw8|(6B_A^Nkr{|YFeAu7E&yeQ@35UI9`2UQhqkpHnLK!&(=w4g5KvYwoS&(J znMe{buu(%eCwiozE_K_szL}Y&A&HcVsq`TaWPb ze8v!=@$BM}!)S6?r;H-3JWbQ2su9uAYpo+Z1i&C>1p|timc+=^8_bCvoJ7>NXZN&@ za0*wVAemIk!<%>A_EH4MXk_NW+1)p5)Oq5GRnmtNcxTdY-@gChhaWzE_m^p&&&xDT zW$*hBAAb0!fBL7QvOZsmo{P{l6-q4AJds?UFLvGgbwAHbzg)H|TkcOEemJSz%~P1S zZM!`05AXl*?KkiK<^T9U{P`KP<#i=9Wf78Q9jd?puLT&H0nYLN-#P}IEis@-G-4iR zJ-W>VLtpN|%3B{30C@4Rc!ZU=$1mNBzzrk&;&Xjzvwr;Gw-)h7HfJy|j^0cn7Df(t zA34(Q0(cP;DFO*c5caj1#ptbx%-(xAx2<2*iD;IYN-c$$ z)vF0;o@WZUJBhLucPC+Hw*G;DI(mtwsg#mI?mkUZ@4dBVHc(v$Gvqwa?rxq#BbF$c z2!efU-8@5>>$pNid7A2V^VU5wL9tmEB_&QGsfASrZD{}*>Fxn2Xqie7BEoyusmf@A zi!7qv8lO&-P!?-#UQWKQ*<2&oe3ruDlF{~UDaH0{+gFOTzE4%>sqE{;E>F}fa(({r zeYv}<@4msbWKY&|bOjLt!IT*p2%?`xA-v9Ejc1OTos=0bvllr4CL}DNk@*JmXR_(f z{?r#{i*H`o&1@Yn;RR*(Wdi8y74^G&CAC&IPC$f-*^;80ceCc1PVSV+2_XU#D7BP< z1Q8JwrOZIOr`#&8NkPJSt7E=8;a|esepN8}_2XB*|Ft1IZ0WC#7GK`*`tRR-8xGQ| zOkxmlkeL!-yA6b8SPuG^zFAv++qUb|^MCsGpNZ4GOOa_VW}VWL2%HQ~3?IP}P6z0+ zb0&Fn+mS9x-F6Bu%G1z549j#?MI&ZW(e4IEWRx=1G98ov$r%|D6c8?Tp3L^%I}xeY zNezHmq+4s2mO1b;-O@vp%4`uI*GC7zl%K^b4B2`T#&cX702sI9U z(}4ng*>-Sn_(o=u2ZwVw4S09ND8?~Eg;Q9#P>}Vb7GoA`+u0yWNvuVuW!9?Fn1Dgw z$IM_FZK2!a_3r*WE%WyuKYaZ7VZq&-cMqk!-`ZY;+#Q)_mhRiOy}3ufZle0;boaQL zBE5BTk7f^d_kaHM{q^aIiB#+P;hUwF58r>!W;5k#?eV+sFfZ@tlT7n~i4Md_4i1Pz zPwbawD!#N_uV0A>BFP|TI;i&3L7&4N0O_wtrH_N~tEv8Dvz1?c&1;_vuU7Tz$nm$X z@R6Z^$#dbz&_hMw&Qg>_AQJGtqz5+e;BCv%bRA4c5EDh16A6)uq^I@RdOMdRg_ump z;R@VhvCqHl_FMBPp8w4T>(`7gtko-@>oY{-S4WGlkNNldTtjr80Jlub?B?ClW4}D} zh{pSt*4^6re0jb+&i7?~ru+Nz^Zi}XxlX6^c^Qp(gH&NKXo4s*Gor`dT8p+T9}GSs zq@`A(R2}`aec!ij12F3x`;Fl!+1&%g%%zkMA3k~Oy?dBvk`kYmMQeTf_}%4lsdYZj z(eyX^I1QBN{_Z~ z*1NUlu@Z3qaNe%_U;gr!z32OH{#2?=BCNda{d&2|1#m_7vq3P;ecbljN8N4S@|!#EdZQpKe;4+FDL6ILqD33A|fNwy|^ntHlSIVh z_0izx`x7eGPEA1G%{+-2a(Uh!KYV(6dYtE4C+524R>>Kzi(002c0NklKJC*8J|Gvp79{;~<+rf33@A6sW@!Qk4Z>G9j`i`VPGP6iw z(y$)ZOmD@An{MMseGxE(&0jF_Bt(>yP5@H^%)F1fPzLCb&mm?N5~hLS0+&z2QXGn% z08uc@NdFLxI?xNBU%2OIuPp;dS}hM^oS{%i$;?P0qM+?%;k|xvEU;S@@})gx%7}pV zSHQtS1JUaprdB#YUj3U)2ThRd_e);cq%9KQiFd%>skwN?vqwhCkG$dr^K`b8e znY8lt`Mg*zW~RuXWR{eWQYa!Qw!R922;FUOyZ43&5@s$Wd~lF3mr|x_nx-kgJUS;b zF{GYZODSAR+4sHoUNc4H+FIMTa2ZTk2+GWT-^VP?fV&_1(k;4m9{O6$7=+w(1lp(5 zDXoKoh{F1MxtdVl+o+jZ?-_7LQYbTX^)!my*6q;|mY2&~Wiqq7`#bBsg;5NS7aO$k zj^gg_2Rw>1nVCWN_xJ0%jz~B&i8vD$K2IlAWtO5-&dgjY&qNY9NU@WWiRFM(X1EP5 ze%(4zYcwrdTWkB?!C9sp1pBZ9D-wZ3iZ z8r};_DTTUZ!Xw<6CAE}knrf}BcTi-0rpC@ZgyxmGFjRHBUVHB)2^6ih)|zlFrPNwE zoPoW!QVKJJ5SHnQMDW4L9${_o7Q2(FGHVji-Kk!l8buVL4gUQ6SXs=w5?h$NC6EdT zlZrA|sX`p)E%yvUcv<9}%jt9)de;##O7a8cFf3^#TSM=?+?@4gS$gm0;ci1`H#sx2 z)=JY7dwGfHy-L~~w>zwZC;jOQc6saskDP;<9vjC?@4B#<{?xuyA1w_nsp6fDK7H=Mr z?#@XhvbFyB>C@ZGWqS7wClCB307nK0#L(AxtFLAp(`fX`A^kAYG032o$G3Sh&ghuC zF9Ap%F(VGR*R7U?WPscokr06>hBt6iMIK(&=RDWTx8Q~ay;(5&Wy|%Gd9iL*q%w03 zzrdZDCoNSp)B3iHyJ!}!AdY}Ys%qg9mV-r&nd|8^aiytr6;`l>P$pSqGqYMN9Z0tL zl@R|g%Mo9k{L9b(eS^x^U-bH}{mtW6Mg6sBs2PR~gb&ywWfEmjOw;7nnniaHWHP02 z3VX{8G>>LpXI51%`o(O6)97x?goh9#4sot^T}6aEd+(7p5?Hksck^zUK;$SgCBO*F z1Tw-Syi{SPh~9e#c}gi#NuYxCmXTFN0Bbvo7M675gvX#rWeg{pB6uet%!^bK84Q7N zcyy0$;h9dPDV)d%Ohv;hGr7B28>zGwx%cLNNNjmVW-yBu&P1~wmg&q%Ie4-?r=`02 z0kRfV9>lI8a|u9HS;Zhzsnarv$kX*g6iP$OjR=Qb%Q=fGv#2T!f%4X)hmc4>O4Spk zd1j2-%eA?*?vYh%nU~@emP%^o0xqJ&prCYkb6Z=jGq#SPDz#H#r)VYti^L!rBjm6@ zqLf1(PM2NjIGO0V1+#mR4^`9HhZe$s4VB(@aF5=r4fnGC9>{NdYgZ2#2POJ=ycgd-3c z(_Ds^nBk6YJ(3~{>%8!!$w&#JfFjdczqZ!qd44(Vq4@Vx=NGR}@5L~^n5Q4xk$(@L z3tw(FfAbLzoXTN>f0g}G0WmU#Mjg(Njl`u&B#q8y<|0%9cp^!Nl{iVDwbW%UrGPVK z6wN);Wgy=(BVsA+y>IItZr0jd3w7VNEi+kZcv#)`w(l%LM19*sB;0%J-Z}-jHScYn zYpp6$=g_3kLJ(wT-!H9SHHoKLCBdk|MYL^^nSmgdB=LZaq^Uj4vs>HdQbly{wqkvH zx(wj{-aBzZfYjs+p;}9XGs2riL<}f&Zy!H?bjxK~h&i%(I22Vdp!nnQ=aT^33LM!G3)DC{r$tn z{c0`tJ>7!?L7eEl^=|MGDNMpDtb(DaB?)HYOwIwx-S_LhuMu}6?sT{|&ThNtg)tn0dM z+cXtcp&Qzmk9rIt3CiJ}q0>}m)*ju$_H|v`ekW#qQz;bAh@vRUN-$>4%#^^uHKunr z^Fkz)l|)4dnGxHzJwHEA)09MYQW6#!Oe#Xm0hiB=E5~2=jNV`NH;W)jaB3ANB@oDk z3z8!!3C!BNCxLiO)yyD9jGC;OSvZYCk_RG*XeyP626Oa?hBC=4#M#_Df`r{uOC+Zx zX_9X(v{o&M9IJm_*GkeNlNik1!4_@%g!*SxdmzJ{PGBE@^jnx3m_$`;eJw zo!t|oVLT)yiHgqq_S~9HDyo_p&CFXXTEXNi*7~+dscl>Nd`BWlu`IP+ytRh(-uoyw2uqL& zl9(V`h$#VuF)>pqB_sB>x3)F!-L18@N~bj8lF>6<06|*cl{uaGR(4Y2;0(_5ywp0S zZ%WKVKOIzS4GL9}Qi;gZ%x&!V`(=lRinw^v%RS_vHnjC`Gw%*zRMyRT)5)t)81_Fse zPo1Q~B8$$?`=f0;mFWAfs-+Zn&oU8|STbl^+o$IzTRWf5QxKQJ;Gu7#Bvq%0Yv+Dm z?p#Dz9W)b(+4}S>(~N`^fzHA3h>l3Yk+&a)BEPBvF&*@$@yh)l?sx*32NyBlSmzUX z*|uYFH@`^593=DymMVt{12AmXtN7xxSUf>8b408+OZrmhO8OQ0q+gjW03__lbnD%+ zTeR-mzOUEapM{zNvkE1XkMb0s-@dU<$c;rjX=@(j6Vf##D2fnQmPsjLVV1ogObH{+ zck@7BV(Ql{K%Y#O=*e(czw=bf#aLWir^xhLbT9l{L(zb2b zETXKn)>;8rH&;+1(#S)YGE)eD`s26mez`ndpD)+#!pvNBnU>fd0SsbS_nk>SthIff z7qUF6KT?@#DEAdr8l+i$AAohKb;umlT3Ls1&bGFF5Hx!WB0{M*Fq){U7r(o^yId|2 z5sASh7!>9vLob6lNdg%N29F+EE-I?qD?r0G)W)0#I z5fefgt!<4cReM`8)BwhlCc@0jiYKsb*tT4^9-AMJVy3NfF|hiG;S%40SqTm0lpz;+ zC!)E-zaex(8ayU7ZWA{jU4v2QA==Sf!};Pp+jaBWZY6Z#k)hAu7T4~3y!m(dXO|oE zaML|Q+?}FRg!j$YCEE2^Q0G#MatZ-RRLE72m|uI(THybWYsMD!lL1rbftVKk8)kH^F5 zsLByBN9?V|A1_+V$ImZY+uoUqDTEKFW2O@YGnkSh9337E+OCU&X4B>6{PRygMMT?{ z-nZA+U}h;&v_3x_;jyfjWm$aP>QgC2o4X^JC?wwdhaiTil^X$MhmA*u44V*pve14U( zOH0*SGV^jdw{0_)wHIq$#eV(xNmWZJDg?&nJ=;1>({jCp59Oq0I+2k#)6JRX%~ znrbOZMDFh1l8~OSmrq61Otcm=+xog~+fP6J^!I;1zYn@z*B}1$XDNl!fBf+WY+a45 z$gIXZSyemEgZ1>fttus2bA%n9rT5D(zn&w4_~*m15_NYeqQogO9}oPp>@!au=o}~q zBjcFgnF(TM6`|gul%j?AmQOKexck6cjr`PJ040pM$Gqo5<;J-_7UbS)%RIokxQ(%_ z-qyn{0ZeZ`)NTuYXI1ZPQ}RZayl>nDfl~Eo)Aq7_`ux||PoJ?}eL15fnjGp>%s7-& zglJodX)abGsrlBs5>y(d=c9)fo&x1^ZDdAVy?3JFd*+Tm%giKILw+N@Mc*511iF33yyOzBL%rtu*|$!OU^XzTDld z)_cpGWhUKH_4i)X$H3W01cL}hSWN455G|Q0O7lF=^DMhSS&)dTDrORXU4uM99AH(c z##?h_48}Ne>b+;*gn&d_uj%o0JQ-oTT;1KKN|_|g$t(~N=)mUiJWaxs-FsVoYf#R< zN0iL0GNge^?;VmV<@9c%$i8j>KuKi7iF<1T6(7Q7M41$i9ySLkmuux!is8s%=@ecq8kCDs$P%?dIXLe`i!!&uFw(UGkm9hHPmaR24)4AwWv}7m} z#44r+olvEgdOVRBStTjNmZU;9s?c{+vY!-g?63EC?iXYKz<1fJ0W>np&A;bx@weKZ ze&02_S&+AV^Os{hSN@^T^`98M4t+fagw0IEriqA{d8qZ3TA5ig4Ff!_$}5MSt@bqoX)~sk1Ub-vs;W$a){ORMiEt6VnO)`X-JrSPnZQc5|9Sj$e-nN^LMn|BbLf*qi6qVC=^ z=E|V-jLkc9sjRF9fFfIO5y8qHt=0-eUseyF$V!<=kv%igySw`|O`31{|QWU91Xvp;T5F~zK2yPi%0b+1I(k;*%Py69aVIX@& zGVwUAapM~NGIAV%8A4*5H4$S+m+#rGFS6Oc)q=z=TqOGAHtx|o@Qk8LneM5Cr{hW1 z>!<5S@4@Ma7MW&+ib^$M^A<#`R!Ie-Qb^}I9SD?#Jp(%->6hp7 zTPya5##e^yF$?zi{-eY7^`9PI$k1Z=&7d!xzHi#8M}P1?+;VMB%p{^Bx}VSZnU|HI zv1@UeFe_M62D_uw=}GkPbYkYsyCmy~4tQYaGQ34}Q&Qqt3vs4L^fn5V=v@+`QdBsx zu1nuGZw)}zk$THgwU$W%WV#1Dizpa95`Yj-HqEtueBIV{^>s6rS(HUU1mRLNc6{32 z5DiVHbR-e^c%L&VI3?2m^rt^9%hKWQE**!d&htFi>2DwZR;wvfi^xbJnWkx)4oMsl zoA(~hrgN2fu4S_RI+Wj3Wsn346$1G3dS)H~u}Q=-eZ4NPpI;o*+KR~5+9;~F*w%Gr zEtHsQO%l^ly=CwD%cqabTtr1gg@p$(uI%>?L_|`onqZ^m+xgSy!{OlBm&>(NkG7pD zs&FOQvS)0fC77~XZt6x_RY1&AjjMp#7O{pW#Qf!D`}|Rj>!Hr75rF#;2l&F?FRkbAdwKtGSmdxL@P*mm^2DYKGFx<|yn z|Mh8l@~CS$2(-g=v+LknlIZQEX7 zuOd>+CL2DV0>O;y_3G)KkqQY(rptD{Ua#|Oxt>1rtg)@IG-Yp#sMIob?-^32IaRxd zH{vS25rc}9%1XRNx^Lkg<@nrRKfnI1RKfdqr{i?$lr8R-Ym6^CpNRzIL(+h641B7? z!^%4dX#DzUDVT3!E`ScrO zKQ6OjG!q!2Sivk-ODU)MupMRrAva0I3itHZ+?_#so4lDG*(oD3d|Q26GxK;jnCO<>w?;y$ zg@h2aZoPZ5=y1rJd-NXJwe`@m73>SRgd!_)6eI#36-Mj zx_;bVGPS79MRhJkCr<9Y&-0u_*R^fFiLoJj_jO$k`4gFrB9^;*(nwVQ@WT(6*Ut(N zVObVqI!sfxjy+w~5jf0;r{ifV^`15o=GJ^V90FuTw%(7=@0hu&)KaR73L`zD2a;q2 zkwfMi&HLK6XQ`tTh>*qP-P3Woh6$I-zIs5EB1Muw6wYzoHUV)t@O&_pjMVHEOyS%4 z^EtdC%KLZ6xt3j>lfIcS+S9I&cmf`eq+LDXZmn?R4*F_Q-M4g$?Ggivg?og!!v!!r zF9O{JxS9Dz!uMOB>zOZq&Ozx?GdzI-$|fFH1Bd}q5Rq6i1)NM%Lx<_T5Bm~YrbclO*uV!SHj_NK<~@zSu&Spq2aSYqjyS7)1<=Nx}2{o zdK0EvE9KCDao^gytq3Rg-mXeqOlq}KRaaqVH7lhKs-(4hcqmKI4iL=@VIDN8J$k21 z1u-WfC`1I=2Si8-T$PAPW!p9eAyOI9H)6@?=hs)BI5Qm0sNQ?x-TiPnF*6BTZULvE zBR{HOrI=Z=a6lr4iQJ>^Vn%4!D>0W+N+~ZdFG}q0aByNOQxSbS9+z!PkKSTgmg#UD zTPE6b)IkG02KVT_d*2SHxl}De9iEcL1Wreu@}R_{2SBFzFipK(ib*LN8zMa@m=w&( zM1lwhIL(D%Ob+L8IC7Kzs;}1zAt z0QaWX+p~OE*0>M3W$20YCUJ%vG=D(GNW|{hfzY9b9AIL^&IIt(CXI;0ZEc1vnfMHylNl<)ntHKS&5TnUrJwIitiocKXALg zXD05Chi}GiU%j|j4Bl=DfI)0|^SRI;^uFy=i>+d!|8fcH2{%G2>`ld`E7Vu z>5-%=*?Ygdp0C&I%k|UC%S-b%O^2tacdU6`&e!W!j6@WSh+af$t-Ta?f3Ed>J`>UD zbja?XKb;Zr?&)c&WxHOM<)X~b$K$a~(fhJo2GM9+H%do#WKL5#9*(1`V`l5{Yuj+jSpkPZ8-oBSJ*Q6h2Z-tfC|=Rz^T{U6JzsgS*QEh3^ys-`eW#gQ;=l_;b8 z)*Dyii2(>wcDnkJHJ)bK)=d(1GU8J$(R+Gu1|m8rfFMa`VF?}}{JqU`WN0{d(Qm&+Uh$tUdHYR6 z_AMv(i<2GY0z5uUZa&vn(c+GN&_8t8|GMX85f~n|6d}&&ZQB-aJzJ+9+#$i}=*zZV zuI;+LJ8p>Ozy0{b|M|mvSvOL7nx5D7T0}U4$Vw?i9C7ycYx*y*>;HWB-l=V zo*6XTLEO%l?fh{gFV7#2Pp7$WuZv&FBCdSCZtvdzaD6>*9mVSB%l7W4_uH~=OYaUd zQ>Ma+?CWd$%ZKO9vqgiq77MV2_B36yVOJoSxGRGT6#;#W|7WU4$r4Re>r{`%wD!x^ z&Z1#;c=XhB1Lminp8xN^{?F6#csk5X+WKT9AafkskC^v~1B%cG_Ae zZ#J`YTzu(BDt0;)B3k0YcAP)_uw2&9p5@(tG3>P@J64^|XWFnW+t$2Khl8n?Kg}Qe z=VB8%fB5h|;x)RPnU&d?TH6ZaaFPgcZs}yB3A$$`l&SKEA0S8xlaeCJbbR-`w&nHm zDaof}IiBjlDnll%2dCv1v8of5w)C75mh{}E$2>DElyvK)zy7c3Pw$UU2YmGF&$X>x z6323wsGvbXvrb!V%p8)*>5L?DMgpprHF7koi-?dBliFBU1VF@umAJTjxRWdx9E3ao z7RpRAl3tVoZ!yPU?EoAy0`(z(0)t^z!^%p zH&1?C^Rm{!aVl)Se13hsY<)8&RIOGgk`}y#bf&DTN1sWN5?l;M3w4Ts8c4G%(=^vZ zk(tucGaOpZ>*s&>=fiY*=iRLwnakGQtnQ~tPH!cKr0)cJ_i?{Idnzh6zBMik=t&|8 znCp(~+zrreYpyq}7BZ20;}&SA%18nrL?!~Q0%5qas1AvNp$xS~Q#njh;c+@11vyRDd%s*R+q#I*oHocv z&1{;cshWTWQgJAQ-JMthMsjz5OzpIHVq#hgGY_Q(NuyLHArc`yuZ5XStsJK+lZXK^ zyq?~BB+Mki+vXckTKwtfpAU5^YU;^}LMEQsee_zcYukE@0T={C^z*Nu9cfi36`r-$ zT2!o-9{%+7G<-b@n&;U)`?}q$Bbp(TmTG}aA|LByJjPgy>QERGDTs9UYg`o>-IaNQ zjaF?o5mQpx7c^7Cg_M~GicLvW%T&YrreY>0Bn;u?=-$Z*!K92x_l8cCh>%Pn&PWr3 z2uE(`tG=G~c!*ULWMl~H>5{BueF&N{B}p<7W%B3^ia>%e8l*=DPX;2h6dtnVJj_n+ ztM0b&?jG4#QQvq7kKppQbocBSzqL<(cZ}7Khivg!J@=4uUoe0WaU#7(?=4dY-+6dP zi0NJe6*4ns?2TgUB*jFOSQsqWQD;SMa2=$Fiz^e6s4;;?croQ@T6*gf_?_`PP7{6M zz(T%rDIPy_Pq!0Mk8MoCw>QB=Bw{!BkZ-2V6XQhwkW2M1x?Q&OR}wI4uuP=FDX%ZD z>+5BCxr)=Q)4`^3a;*o1trk)BhygDc&!|xj?ya}$XZE$X)l`pqQr74cJ(bj~&ZQdL zmOPhgN+wvZ=k>ZoUrSX`)~OWHrmQj@4wDjl>!p;|+O}=eH0=b85u)5$>lv9jGxp6hTT>>}XWHd25Xkk#t=)Q#`$UGS#*X8D*6~-&*8`=9*Gi z;NhF+34Ps`WpkvJ;t`aorZ!atxG>wE;LOoe9bSt8xGW1L9j9laY^|N1s+p10pn!=M z0h5+O!pxdMds$P2IJ4t;oG7s^i}$s8P+}?tdy*SEbVnHuBoaW|mQo5L0*;=Ms)d<( zE_0AGIM#G?BePPBbW$y%MVK=p69{D#HtC*}TD6EsZ|&1Zo8}{n_IWO|l^Q)LiPczY z5;+IS4!ph_8-_TPh;%$EMF2|jJ+Y%C7_E*&H)TAN^}g1SgGJyQi}}9fZV+w$!VJlG zb8y&A1JfH%4iSV!cw~JC&@4fjp2_5t?rm${3QLA(^hmn4=*p%-l$;PDRWlQl!X$)& zK_gLNE@DO^;L*G7lq@owO(P6sT{~ool7kmx=*GNJ#*xtwAQ+@iRkWbRP_Yj$?8B9HcmDFUaW~BsDRiv1hf|DaC(ldfgNQmIxg90KGRW+V%PPn(o z9=(g1iI9dq&68EtqRxp3MwY6RmA-Z%DKbx@5$Wf1&Y$%#>-999#8kG7?jWTuNFYRq zz94vT_A^lsv5dUx&3(d9gq9dnR`!^HffNGW2_yz><6fT_479^3cuUjVYxDQDA;LHL ziQnO+5+VZaW-Ec=qV9hiK>l?1MjjDfDTQ+gwkL~nFb5&m>(y0?@bI=uRUuOb#c;9! zBveF)q?ATDi?9q~X3ro7LH4ZQnCfa@M1tQO-!of}K{S1l8Xt`~<10IqdN=_HDIV3M&Z-3y34SZ*HtC zM9CteDGU}OW@1PNc~7C>U|WL!#7${>LF<>kdY9gk1UvR*bKIlVirpZ~@J%?yf=3@8cpjA@#T zGR-K6NlX-=VDH-kV30DhktvTRAyr`;bQ=uTyz!UT!(?>;8iNTYYN~~mlZx0}K>-Pt z(CyOuIqEd2iZFwaa3V6&$}B3COp8>e9^uzZfBD#|MLl__9-zc6(-97WFb%>iP9l=g zX$|It)x$_&jrd-fS&wD zm9$K%aqsU0goK9ScwD9OeV_4h{XhP^2ijjbK6#7g_JCkyuWs&YWF#_fZxRvh3Qu4G zy=j{7s9+Ml=@haAsq$CH@OXG%9q^Z*yxomkp6eeDyBk_}kMJC&oXh}-xz3XWE0mH+ zl9+8i)M?^sAoA=%P{6&lwiI%Uew?QH^c;k~UD~?Exid2bu?sui5h)`riI7`?>M7?*bvlN@?5XcY`rp#$)w=kAL}d zqI<%#J3@EiaxbQ)wblws@0+IP`SlFf)1RYJzSrs9vrcv$l!m1?44BfcRl{ILfQYq9 zG7%zOh*(*fNDO&{U+x*XxJ8)*XgX?zK0LEQ6Y$kC7|T!Xmg0NMiw*MTZ?A?y;B{L? zqZfh3@gXBK(Ob5yZL4ob>d@kIg#Rr%0AB;DjYhBmf=Ry#rFl#YVEuwlLNik9h)*Qzv z|MvI)Pf8)y7TJ4;FcI~>RpIPCy;UKurLDYf+w%HC7d25LLXc7_DC>mdOT)lC&kRDu(mNtYGrGad48*-{BiEh-_>j&Cjzpv+B=+lk zjV@vY_MVWmX$GRT1r&9fgw0z^k6Pxc>K>q8wMdbOK<*vzX7%&y%T#Ap>8&H9*4crz zZKwKdr6gzgh7M2f8C!3s(@9xGI6X*MlrnN%*Hf(^>d^x(uo2{E=Jk55buz6CMg*uc z>sV1FDnxzg%>j|#TAQW`+0iqQlEl1=i}mb|QEPV*Q5H@P4+RmA#wX~&8&9)AQ#?l6 zc!-h+0^WxZ%CQn5ys@GbiEQn)*I!@H@(PdB>3KfReOsJFYuUE8i;@x1Kw{FU(L8>>u&DT5FL#BA7``OnI~c zU0zNlb?VGpDMMDAf4f>C|3&+s?hO#T&Z&vi0jK+sq?e4=@WU ztH^BS{Q1|SRFqYUNeZWuGXM_ZQ01q?9DyNB&!i$U+LEu^%eJf{RBKfemMW^F?3kIQ zl(9xKyCjfFkr|0Wj!e|IKF^hyJu;Bg!=snKSkccR!Ix+LRrxSq(F!gjgc~AGg#Fm+B zYXP^p)R+>!tZOEj8EYaX!@Gl7IZ;iDy)`WZ8SdM*9S#!>iAQgbGlQ!Q`chS`Ru0p| zpbSAKx?kGn>sGaVc=!I@yZ7=z{+b^H>BwY~9it}>wDoGWa!%d@N>e#>qt@5UzyEEC z9^HTZ@#ivCU$;)6`VETY5q2O3< z1Vqx%B+qTzM!}mEDqwHdc!vll-pX`g+{$)Z?$a z%cHx!0FVPLH+-%^P;+yO=;pWX!Z7>$xAs`#h`3XC5CwjhJpQ+TdVB{Daw^P!lquho zg+)b#nN*ntnT|v$16Y7vJ&Xi^1w>3$A~4KE@0&+Ak?ZC2WKhm+x%jqi=T~q2bU3!o zlT{+L*wTB8KyPhZOsG~n6rt?fx}_dUG3M}$fd#-KSbWpbkD)A4$KiRka&KQEWddcOYf%5`ld zBx;e_yg$GDAce&6?)ig|3W>Wbt3lDyvyb_dB!iUs^zJzb8JLcTom(pCbbPM!91*_j zYXB6g`g}Tp$;uRoW!biMX9LI}Eg2D_WIAazH5SB-$QGMGD3vHxQ>8>?kXtRM z(`he81E=TXAh)<)udlDKt+i5gs#R4XwrwrCS2aYEN44otYO+g;DBH}O*tS)K57Yd4 zJ=;{9Zy|J?PtFwXr)eh7jx=(y$>HahUsfU)O8oM}pZ=^K+viWO*UN|h<1ZZ1ddu$f z@h~P&EtRrU;4sgx?QFuVMnstourYI?tt|jkQ0$Vsfbd}C&ZORBN4;4g;&ol+Hi&4X zwAXcY_rtM{JH;fsm?M7g;UC>q)r#$_Ju=7{RG5&_uGe;c;p_z-*{_#Re>*Qf|NJx3 z(fhIL%Xz)7*Z=zG|1$789<944ld9%u3Xw3gVwM!36lE2SjMG#*C1sRSlO!^?Z6hK( zz01tj8fCE7D#pZ~ai^>P=D3HGxV2J^Q+}uk-onG|^d$!3tKjbOox3%;ec=8@8o>>I z6X*?znY^)^-3iX}fyL^pd^@RWKv5h)k-CmBt*i@SWH#fbjWcdos+KXns9fA zJ8^30eFy~1h>Yw>qH1PpmZZeVU}QRccedwwRyG=1sQ2a~(7q{tqg#W3Ss_p@Px7S7 z?ru6*&k=Qwi16Nfmjr7qWUNC7E=6gaW>Pq|HS+nxbDgIxJu>TIj)>SARE|%lVwFsj zDG)g*$iV8Ix2u!)&(E!Bet2T$36rOnES$*fl8kWo7S8hs=LhT9&SVN=7@O>nQxP_j zvHZp)*ZWGmtw|X)Ab4OS80)09CL(5}VtfNM{?21uK%8J27@8OxRe})QTdr5MYc;CE z%-IRCT-FyOfFfCq!zB^hN}0qo2p0*F86>Q#d#H>61tCYVn}odeU<{t^0oeJf8b}WT zSLTpb{l>tznt0w~!!H00ARKXGkiA#NOx6c@Q_v=hA)%v zP1H#wy7k^N3R5y*#6`Af$K!0O#2Tr-L~G5e>#}+#{R>1VxlmvU@~OCWJ)= zku5q%RD?}c3T2MChcZ))?n*yt(h6K z2=CAB-$$bI{ojzf$`qthDA2dXmshk!Q>}0!D#zx%UCscErPRroQaIcrf^TkMcqX9C zR!ev$^?(Aw0+5WTKO%;-3K8+3yCeae>o9^cIZgU^M^hfht#*!kElnfyf z@SX^bOpgrcPAI5O}~75O3L)~LnLLUSt*73)><@0sig>XKp(2rLYCX$y);z>Apv(Mpl5iXl)|RQ#zccu zW0IPbH~^6fqew;;WgA>tszkE63*}U*m5EIOk|HJoq=aP8OqX5sJA`dl@MTT{BSAwj zjHw)SkjwY40MgIh&KNI{fD5uAMqWdk7@ zs!$M#%{D0;G6z{mg0u)3smdg%njV4m92x!BQ)9+_wBwNzs-i`!iE=_i#il^Ejn^%Z zR*OnC7Dtk(r*jICs1#NiRqq{zOOz=$aMmaYj?D3HyhQeLs(ar3ue-yAhfK;>LjcWj zZ37t`&KX1mZ`2p+E46EH$x7&bCbu%dlK?Gh#AHQ>cDXy74*xKi!-G^+MU(zK{_EXHJqB_2%wW=Z4D78xM7#42=JC!A zDz|Up!H|uB1_J_d^K|L{S0-V4bAcZZ?7qSKW*@EC<8l5oc`h2h%Pn(9Iedp*0%7>u zK!p3&wyn=PDT@+EAzAWLbSL?oY-^jbr zKKm#V6#OOzL;KZXMmU?Pst^;A-jqj0NP>g*D4CKYqDLDbk5Y^zKP_i(S79>96QM`< z1c{l6r8M`;a#5QIohhwYRh5vW$jr5ExrfV?9*JZTo2r^IDG{Xy91KHq*1HWn4Fh91c*9xakod0@F^P}~8}{C?8fGB%_^SxhEDjALF{!mK%R)qj1d*D=qTZbVctAwolpk5xQnnx(1w0Zn45FhfbsF&` zu@Va~&}$qKR;CdfMuh%zsrk2Y2aY`2G5{=JfOYSm=NsM7gLg(mlv&=k!~6d?{z&XC zQ<*8kkCe3hz{B;`?j8?}Z};eK@BXOIrK5Je_pE~eL?Y8l*)RWCU~6A4*KO<5`#%Xp z_FhUbf+(Up&^dw#j`RewgHu(zFYcZh${f9QUmeYquiI-jPj>>8I3bh_lBt-83Tv`b z67Nad z?#mI|4kgHlY(CO5%qlk;ObTy7RBRfC+e8M{ILlD~?j(HE++qllNr<{{px}hBOLSKj zWkzI~51Vi6x}v3tDX5e(nO?4EWb||f$xhx12%XkcOX{A{dj|m!Z z>XujQ6s`G{`uVzvt?T9@r&45^lwn0QOA3%ZgD-!7*?Q}(K}|qKR7FfInT(7SB${A= zBIy=AeLb3*(4qN~h;Z~4-Z@%EP`k#uA~wz*+tu4u zi_M1_D1B>1l?}bEV_$9CO06Cl4lAm{Lwsm-pavxp3?zuSfG871l}JY6E0dC#h)s&S z9|p{rxx0gXsMS53@>{fc*J}OkdELMA_>+hlzonKQzXpln<$Yyt3|~MiZkCG(oSeC9 zk&p<8oLSTRZ;$7X`&@r~m=F`ou%>%$dPIwv1Jex8zV-FGEZ6I{^*jZGlyq;^2YE*K z&8QpY$U*Ng5oRs=`uS5t7v{MbF<7%sRz$YTm6=p^)S)Rk0422|OiEy;8wO9A#inVR zrpc`I=Gi?nSpWo}6pwnU3~&&%mescuuC+*!%C*j=t;@D;Wjd-h1H!4#r4D6DLt-5NTl=I<~|K=F$DebX}HWrZh?=7}TT8>>>in zL?8sLqN+UjdEBF=XQl;HQ3$b0oEJ^z402D0CmGwCb|@7I-7DdOl;{x=ok^A3wh3`8 z3rz>zoW!hc(l)I^O=DYI`n6rOmCIlMo|(y{R+)0>NUB*Ti?WIKQR0gnEMCC97RCcE zwCDE*&i`Sd-U3OY(Fp`+R@yrszn|wqGBFb}GQ*>7>>JXhw`fajE4dpb2#OGRclOq_ zRAjbw8{KIUF>7gU6Au8kQzvpbAcIdK0Vo)V0FT;HB0EAPJCB03v8ngQ%)@d;IM~(j zJ4n>;SuXlATD%(z{a!C51c2~EE`NN{hQ)b+lHS~|8*FtqZA8Kx%{%?ZHvP^E-hX|p zPW-_GZfj=;4WR9wHM2yTLC5Y%gZLkB}X=F)Z$`KG28C18*5<+Z6 zhQ@}c3zSU_EQNKj8v01rjS~)lG9_X(fhga2_>`sJI@AQjM6I>n-Q7iv2MGky;bQ8e z1B+QqMXOfJ^_smOkCJTe+uE|4S=HcL3aLUFloZh=hubfQr{`_kmbPIJL(1UOuk&Q0 zLgd^z;nWyJk<3Oqs_`O(k);vh>-9R%bFGyfZBU_^S}ij05du|(fSJ3G?)=P%fMgN} zWd|n{ksX8wZwOO@!O0@bTx*?bEy9?xuuP@SW<>OF=gYWyRcqBUf{%ze9HymZxC0vr z58Vw?GSXSh%)|;x^$dp0r5Z~zhHOHzWS}EDA}5LH?#;CzU5O?YF10V$y38pA1XF+M z>&x{*h4J|UVNtay#nfxd!@N&?GG=k&jNUUxI`gZEzK| zpC&sXJtH#+0g8+OBIfv^*4kQ&)=Z>(@4n3Q48zhFW(FZFTX=JFk;}{HQ8QL$?`uRL z>1Zljr*(CukLTCp@pxS>(|lmd<=s!HQ*Y}_IwVgV2i1v+f;md>y{Q~dhwgsv?Ql52 zm#{>n_k^&OsY*ra%gZ?l(=@TMKiT}JqnTYUuTQUaK2#Duugi5?Qc{XYAvzqohzL*u zKvIufgoLR?B)iO3l~p$!`{^L4VcuG@YZYh&wH_-A@S>#@VI$Hawb+rn^uTNf4Q=bS zR$cow`gQv8!>3Qb9+#QrKtdU59Md!rrN`Rcqi>vXuv)0_3Uf0v{-6u35KRKurK;7b zS-XV8M*dh-Oo%n<5SeAEy{s=dJ{@M7=GLyuU+15ng0|PIy_*@U0&&D5rwqMru zulnf&n@XL-zNc)OzEg(Z)#`_uw(0rs=N^7Or`eMzUDt$^9zLXnb6rRwtxe36rnVx&V#Z1d%H~>6 z)n@NgkTFRD#$hE3Z`wXaY)?;rp}d|yr`SnpS}$FUTfZ>pZ~{b3s0fvT0QkZN5et1` z?!Pu4o#-3x&v-8KJs!M&C5D#*5b=K9Xp})gJjz9!EQ6020df^Yhaa z4GX;=!2kjzA^Cz{pXDDTCPD^ zyDp0)G*D}O|L*T^6&3$CgZSb}S*g&BQGczsBMqHR=&{(qY9%JSZr+Cw18Mf(Rc8Nf0walcB zQIx&4+{e@O^v}+rZ_aO34Wfu>(UxT)T(vCgx}1o{O{9oyEs}cgO|LYSlhUaglsq~d zT115CyZwgW9&($0tu<0z*R_<=demC!c0ukl1VHbddslDLJ4ajBby2joY}*FvkNw8` zJp4Lz-Cy4=*WNpEn?l*z&dHO2FpViRrWqlT5W3wp61*+`u}h+iBqmD4&2o+E%%HgV z=V&Cpx?hifdbC!L#_M|zj$yfWYsGTE0{_&_`d7yP57wNsGolKmWB>pF07*qoM6N<$ Ef;c8yBme*a literal 0 HcmV?d00001 diff --git a/python/nano/test/resources/train_image_folder_png/cats/cat.1000.png b/python/nano/test/resources/train_image_folder_png/cats/cat.1000.png new file mode 100644 index 0000000000000000000000000000000000000000..cdffc823e5049da0c651e27e7bf5cbfa0b48cbaf GIT binary patch literal 42144 zcmW(+2RzjO|Nj_qWgYo)BJMO$!jkdcgJXPmv`M4Tu`GP5%-+1V>+)m_5L z-p6gYBShEI=p0HUj$?W0~<<<8vDJC-vtEiX> z`TR1hNYqZST7RVqnf{wjqi=W9_e8=@MjgTyO+y#ej;HCp4WFhD+qqiy<+6N{~kl_aM1tlFgO+fs6x!)d!cpX#}jK$7yw_T<6Nh= z3dPB>d9Qt7>Pj#|JVpNHcw^o3$%enbF~FD6r=m5nxyx}+j^=_kn|BY^%7av|#_7=Z zo5W*Rf(K;3+a{YD+n;X=>=Glw@1$v;K@=wKABpj}AJfdy>ypfX?Y&O;>-k>} ziQEkpa>4#|24+g$?-kemyRCb1(??74Cw^hSZ;WI&goHCDn_l)c(cm?A$DS#9$`^m;`2xrUQKAAhFnqqHPS1Z>S$O**E!w|gw8pg&8UP?NUfUIbwHl*zD z+d~AcIj(n6w4XxJE^Sh|B{CiM&%@14&usAAcxa##;Y}oL&Z{=jXu4_NyS@*SkUW%O z$9z5}2)UEW|60yBshSlUmC97i;H_I!^}nsgrACh;la`Y9ZO=j6RnsNk^;b$3RYhS) z=549bGR14!PW`D>e2>^jBvQ*>mozsD3q?UBl1=)Y=Y=#m;noQ6wsUzIHED)8ZY_wS zHJ}l?a969aGz!Xyc?qpxQNM?KsWlp-ZN?^ZF>0hNrR$()dSc>U19xtr)ZL9sxZ5!c zSJH{E99)z5NB2*mm<7kJZTU+JL?UvpMCT2-BGnX?qaew~$%~3Pvba3jA1$++Vh%+? zrsemjRaJiml-2eaK0K5V^lNrMp^P*}o=|AglZP|bu)J*gXtrYG?#xNEvd-?tlHw1N z7fGT3x!d2g%-n0MDmLF+eG`G|QN;Kn3n%Ul!bH6`+mEL{WdEm?Q8;~gVo~0Da5r}QsIKpheC;}P zrTq1MQl_z>#kg(}MltiDv5&X62mM40Ex{gVpJHl!GyQu8#`pq>ECqC!TNDte0mc^! z@SAH~RskzF`e2s?uO+9TBy8}^fHRgLM|vb7IFrFE%*?LNCOKqo;u9&3AA4vlfsgHk zAdJqK=`YzoeixN!#Kg*Ape2#hFA2vgPInKkz%;0bTfxEqBK=y`V8nMx^YQB^OD17& zt~JiiY;aA5Ee<=h?h;uu^*#Lk_x|=vYe4Sn>nmk0Y;AoJkCoL!U=I%u$!fcE!K$rZ zi(`NO{Bfsm$R}5uovy7@Zar!}$)o=U%TAHsr<3*8zazR#q5BEPtfqCfYWq{W)WKeB znVFMG?;kHBl4}zAgr49ct6Nib-(G(Fzj7d2R8xpQ3{5SZRM1yDSdNT5&Z`+9I=XoT z2AZcB8xQAg*Mzo;yhLDs>@pEUVouXU`+`Tsmm$mwx^v;tX2uVe8S(&IpuXKvY~|C9 zY)o16r1;hD|i~#CGiKw(56Hc3a2>DSJ$3{F9%q`Ub65Y z`pP~czNQ9NkQLY@M9p0;=^1BARch766u8a7t%YCya5UX~Jena*jPH?%oByh;((uHD zjVORMaA~%bVgUI0`8Bq>x@@~(E4znUhQ`L1u)iiJk4@>TYxZVW3!L4`)qFzi+-P<5 zZNU=flR>o|v6BRPG=1%(V3U{KwSuF=*5mKr+X2U*ZOSnG_;ED69dA<-f2B%@k&Oc` z|9q;NVuVI&ql>jaPBu3WI4&H9TJbsX&x;(s%Kp9Ecz-guL9%~fj7YG!AFsWp0kw{a zfaibfOs#~o7&bIF(!fW1V{p}StV>4tb?oT&%6Kr`eQ*X8r54p z(k;B{+n7bYq^r{0B-6&-aU{N@LOy3;VuY@*sBj}_!I%J?rxCi;wQjr{W{2ax4nwfN ziQ*JxeNkh=l=tE$QJj#0VF9eGHJJrI)C@uxV&42HG;yxKQey6@cIGJDQ2$jl@}|DU zuLZMHKV)NbONvi>X`P0@u&`Kz*6XY7FeyE;Jg_<12yzcwFKwkSEMR9E&DY=%Tm_ei zmu+n4B#>np7px4n^ zxo7U5xS!Nb9~_Ld(xh8jP zhQON;ueh8mh?1iXSE^Yijp|)IV5@k64&sOkCDyi z+6#=nK*1Y17~m{aEI5g;k%QjhfIDMW+@r8 ziyOFZJNDBBWhKsT9P>@j7yhj1hQWk|&vT4uPweh+agn4*x2AQNpIE-;E`+ulXB9MW z=!i_e*QW~4|M=Bg5bEphHghWL)Ufx?D8Cid>^XiZ!|(8T)cHL%YN)p4N<_|Pg@)QC zk&u5CMtwAuX)39{(Qo6GwiI97ig!b;95WC+FF36U@z1KOsqxv}n0z8{Ts|^CQH0kv zymz`g^`&9-VejnhY+V@DB535f-qf+j=GX(yIT_r!8BcU%TOB9b4fodmQe~OK-t6NG zan<8__{nCvhibN3@V}2ev=UeGf|kv@5=j>k*x#F*;0nrSQ6&oSMm~?bJ=3W5-AB0} z|Eln#{?jt-O2)KZw-~9x8@a4!Ac+7#7zUJp@Wc*;WHO)8JFRQDL>65ee<-2{u|qfo zpBKrDBbJ$5 zMg5wc%}Ly*s?qloK;V1-q*rZ!Ku4r`e<{5!N8vS1g}#t`< zL?)j`g>6e8B|Fp&s%_Pj8&_Q#{KWrqSM@;Q)!eZ2b{u_H`nWHQcC|BJ&pspagY!8T z0!Hdgh?l1)146R@Iz(l9x9Gw9!no03b7Nz(B`w3Rp6oe|O-&Qy<3aRePoB!Sxc-Ju zU(4}M(B|fbs7$L!4Yh`mO^DjlP{Q+EmcycPdS?UpsHWTj?foqUAW; zNCx(=nnI;j=&v;{ywEEEtwVP*ZZsy zM@PH%_q+0t2i8nps{Lt8wWM+Ffn1g}s@abVCQ8I3 z@#7IotHY()^rIjb!8}>L%nCt4!DcJ9Gf72R4PB|b>+&tinFQ6tF^4&C|M{gG7V^&- zG#RjzeK&!X8!EqxQfHHNI<3xS3#&5f8?uj#u;gqKOhyOqPosY>W6GNv>g)Ge!&dm| zC)4ypum=tH_ku`ubaeCv+0q8*`usWR^5DfZj_A$-Yk!|j^jMJEinmJNT7v-lNCxHy zg5+QWyB^D5%5x+31BO!_3vK5O@cenJK6S69weQMm0SU2m3LlMQorn{+huKA#S@il0 zJb!a-&Lh9Z>*e-=g=bHQNRjS`s2^uM`P|&Ql$C=H52H)9hVKOUAO3#tFhXyq|4Z1I zZ0rx-$ms90wF_E%PkpjCO#icKKCiHQm^a(C_-`y;P`c$LDj{_5WPJMgxtgBs^xaNt zPhaG+&t|Z#42?*}~52m~`jc=!#plA8)z zg*<59U&u&Nr@}vvr>|+%dTU=>x=T`V`8F`%X*@hqZuJ0tM?zj+e(HFZ)pPk(VXE|H z3onQIKIKksQ6a$k>5c3R0@qpK_ukym(ZVvw!`{tT#Z@KsKK-ftc15MXpN0E`97cN* zx^ttvWvheje;Bg8yd$I%YkYZA zs|+(|Dx;uQw&QOu|GsXq&2n4+^koukJzy%AA#e6zQBrK;0gJ~SZnlmuB4U#YD_w>t z2{A0#2s!|ONFkA5Cs)6Fr!y8#*w#bi$ZdC}Tbf$w9&W!t_-4*~xw+lE_TXLf#@5Nn z==XG&s?py!vI7DznB#c)sl73BsIf$aIYCNFO8-7zMx^*pN=4aqr$ zLQ&9iXgOGp%og~?B|jgz#J^2dCuOPr3j1PCQqKIU7=79)?U`!!uF~YbH!*}f?|r?w z=kLCMns>%rEG#U-=G~8KdRG>AcI4V+*1y8?!lPB~M#?Q$wi~oA zo#%z*7#oV!c!R9qZ{@1QJdOu#&Wg~+@@RhMXHK8g$nehn@_<5|f0J07PQ?4a$lkO1 zdYlUFfaLI=MF9YF5Kbp@U?~`B*uXxISMPmaymoMO6#Qc?Ui(5sbHk^i!4(X-`G>5R zs)!ox+tgZl*#7b|yuY%fBr9O4Uccx@rhd8A<8qfb7c(1p$ZCFzeqp;o^P8bZ^J+VD zB1iAD_X%Nt1-Wx{dDd|(ergAw$2TX=v%!S!zbhS${Bztnh#DF!*vEvO(BqH`8Mkp| zHJSPmD%nwQvhxk!UTh&FUjmP)_tJ=lLVFB;Z3gI&=?CaqM{_d@07n-ipVgRuUpgCM zQ*<_feY-(d=#X?8K^y*SM7Z2w#Q+Bk`bW(kNX0G6rjx}r%q0M8-b+x7{pZMPSJ%-; zITs6HcB#~T*9tf@fk0S)Jc{7G^^%73YJ-?z5-7$w)dR$HY>3M-{`lyLgh&}Vk`r(~ zKgJ;XRkglTX}rxu>lc`Ew6qw87+GT2!mk`IPD3uby_$o2L(Cl+KAV|cachQK<8JC* z&|LqJY-AnV(IXgtfQ1*hI8;-S1pTAvk`+QUV4m|&AA5jz{ zvxQ2Kd^UJdSB5!fQ={l~L}f9?F#^8jFRuB(IGJ3}d(%T(*K_;e0V(ylyfMne2NWSz zR$greWm<8eM%FIjlX6px>lJP(J8Fi8P~Y>esF0_QHtLiNt#JYWW;QV7TVe&SX)FkK z#7~x|0hryXlhOTu#kV#xswgMJtp}#92Q}FOI|pD3aA>6w1z{I14ZEr$i3$!MZfp0B zI&bV(P<24q=d~XGAKK;dnO+x$91LLH2Pxo$#Z-uNW${ln(yfMjO7NXu6GYDy-CDbo z=6fu&!zhAa&b-8H!=70+(XMbBVW1&&uV~KJYwDHmd%eKs4`n9Cna#VMe$`&OKe;Op zMyB_!;r@Vsc6V~KWt*hD*>t$kn4Z4Yy2aMIWkFw+-`&}LHs*PI@SqFcbOmbG6ZUKCLBEDOnof3B*Tz0ra+ zU;FyUQTDCg;EH#wYeBUK5yYxM+eY#&2|=<$vqmeQvz^^SB3lIfA4JCI{k6;dMcTsv z3jKpfBp!GS$;RIVY)JL@Ki54<*1A->xKQxUssXIWUHdCwBMkOh37Fbm12K55{%fvV z32v@sr(Wos&S974i!-(|4~L?vy3QzE1S{=#honL$nHPfrQx=k6baDo9Z)9HYu(-T? z_W&Cr%uLByU!Ro0NhIXvpJis)y+s8@5o=b22j`n#w}0XHRrKuCX7nE9>c6#*<5KnC zUGTrTe96cWfip1?KRh|qS&<2(eXo;YX+2qM<$P0GvbtV3EfqWG7fSVO$h1nShdaJb zNFbbIWz{jeoGxjhciHC+lS4kEjt(m%rzksAu9S^Q2>wuz|N18mX#CGJ!Ct$hm6hU3 zmK%Kvtr@R%(+XQa%$tD$9^sJG3`~l>Ku(P3{VQRz=K_Q+EChWUSK_2wTx%>XEJ|Z1 z9{qQ4Vp{>Z@Z?T&P4no=MZ*j!T7_S>q!cjIZOKB13@G_z+TINpd<^1%q8rwCX=D%#g}Xp*yliKy2C5-%@^18 zzYRJWv=t@Y)Y5#5J~JuerC(boPkvprw)TcBKm<2`ouGtj;OD=&oqxQIbp5;nofvYz zHHJ`=7QI$5O7D5iRr+q|aq8cD3r8d(wOy1fvz-em%3B)8pggJHeHb%bCZ60o&w| z?05FVvW1_#(8bgj$*-I#f_EEIc34Mrm~V^Z49<5>7$FR1g2MBYp9TfpU+iCWR5<$9 zFWu_5`_|VFY-meM@O(?<5c{Bb&+Gs!2EzT2#g1296Hsq6Pgwo4DAC3|{&{5l7PzE* zk0Q#|Kl7hrX}ek_prBl86}rf8(tlJGdaxbsM;}j6J82vKOKsg5tV}KOYPf}-KAu`A zuppnbNQe0ZSS@&-Om2p5WrJG%S6k%%7AxlGtZs(ullq3V@_PkYp*#CYRaN4`-rDIS z*{ZqccYeFvKd&16xkho!Vmx?nM);926Q3@tzctCE=5x$Z()1pvwF}sZ7r^EZ598*O zm=IVbaW{l2M72b(WY9wBbGE*lGXq}l-Q3;zv}|xTI0QC$x*Mhz=)E_Axqr0)j>Q`h zVE_;`bZ1y3pMSoJe$cv)fql__*m_XfdNA~$E;d(P_V@0gN{p%5Jo}IMd_0@s6D$4F z5(QOdlsfrY(rEDTx(DR(;2mP(TZ%dKrT2Bis}siP^Iqj8Y!L=Lr|jP#99~pqyj`^X zWKvpHRW+=#=AEl^c*wDlk)3@5WNHi1PUZ43xjhU#FLRI|oX+L2BZjyX5Z@H3-mj6` ziOv^lD75{Fv_RzfY@tmN5mL)G!TliAUolJDkmu!_Q!`KLzjkLaBq{D(TfI>_^AL@z7G7QUsKvfeT!b)peDuuS!=Me!67*_ z=6xO?=1XgurHHd1_bvv^RK&-}FN(6wK@d-Ix{bG^VRr1dZ87j4aka&wr^ z6w9!fN#U7M;m^CzG37HuB#K@{o>RPgdI2wL{lW)azl^cx)KfSl$lwi}P9eNV3as268HX(*qMCRkJ-f^(s5J^=hr{SYJW0vMub~RcSAG4E zIWv2~1NY;4(HwS5tPchB1T5^|8+f_7aln7A_=ueU&2Rro=JnvhG1v4>l5fN0vlckk zj12&K()Jfqk#ehr#}}O|hk5d9;(Z9`bA;^%o|-%MTzd-HCrQ{RUQa!erq8TFa#YBt z5!gglE@9C1Ds@}EPBh=Ej;bWo_-#&w9QT(q0kOGTAtH!>tEJ^j0imIx8;2+9?k)RO zYKK*NgL^GxMXpW*m+yHkn}>rd3-RuBQrKUylRSO-3JPQNIh#V-(bitaZit(cP?2N| zQ!8H48*p;Laete4`pr7aa>+Uf^Brbvs>O@FX3<7H`uFeOPO&e^XYjRA-=ipojPEdS zYd`mH=L?#Xo4&}G6!Yw$!IgiN%1A!?VNm<*bP=5A^l{X*!b@iHB+s+DHEtbKRmYn0HJfGfQh;|Q=O;0GmM@L8F zBEbtH`>;OyYTPrW%+TYtC#~Qq09owxzH(SmN##Y|w-*b~G22-3YCSe)p1u5JyH1Y9 z&|2$MenxyZ$T)2hvxE{do2$MRm>8s>m%Or)%?P}3&7KITZKjM#DUojx)zd*!aly-J z_psR9me7LS^ij-w%-e;98l_w%ui1T*Nome1e;9)#(e9#sU@V#XIxgqCt)R?&ez^+E z+w=5|@{=z5p4!P`=KA`6urRjb=JJ+zuv`{~%ykEVQmuscwuPLT4DGz%r|0tVDa2W} z_1YR^RR_nUsxr98^z2;L7yZZX*OitqnwXC&xF8CI=O^sYcMzIcRFVcUhBCG~nx~V1 zG?@g+rV2AP==VskU3$qCZ(%2smjfegxO6Xk2c&C`v3{!OXZGPPK{ld9r6+O4f5 zEeB-H-=WZ~@OjOZb2j<{|3aPpm-5j zxCil^-PlGcJ{D;u?_-uyBg=`xAyLf09SK%Iw;0B%`}1^X5AAl01OO2gV#XrFeCw^h zbqc>Q;l0X;U~W^gmX{wnj3nJZSxC@xco*4jZD*HPa%Jfh{WxrQ4LxG|$#TUzdw+uZ zDRx{{=2KftUtv5`fz-{sdKvJYyt~COH za1mM#f?z(M^EBAAwh9Dl-hDp}Bb1!jff?Kfg?n>}Jj|z~p=k}jy?Jo6-g0c6j_*xx z&6T^kdMz=9i_gMd6ms(#eiP5-6`jkt(PmOVOoTX*h!0SAcwY)Hj{P#W+HVorL*7+x zbH;saXSQ<%$&lrA@>}C z2!X<{<}(@IoY#p8v}$ql-$T10=TG%7Hi^H(YmoCig$!cLjN^Nho1dGR&i5`79Ms4m zn+x6*VIEsMTvPRd$LDB!W5k2xQd7{Q-1%hn%64;k*~80h!hba-^}eXfrzfMywEp#Y zA2$w789g3!F+cWmwf(g!6!P4iWq2(ooJ~@lPc)h`Qak@K4u0btqYh+8{Eq-fu(Wof zStMlV?U#3#LzvC7hBNa^3yS--?F8l(=z<-(`N zzI;(&o42MCx2s1keX|(<^)J^y$gg~G?Z(F04I~kQ(0wb57Gy;hU_E0#%6@j%G`p!a z%9~utc|)2p@matmWL|59c#zT^DcKL=_=MN5rKH7-jhO&(+Rti1;^{@ zA$KMF1=6jFjg$5D^}M_b2rTtQ>%KgF=Uwa3qE3P^0-KNToQ2v=tox=H>JAP$_A_N) zzwa_BTiL?^KjNFBVe|!&3;IHyKYNMzb2smxM;UL z@h<=<5QBLHnHPGQNU7oJfX*4N%GYe_|UJR<}&Z~qP{d(^3F@IjN z>@Q1D1^%ZbpGDpCWcy6%Dwz$QCFMJg6N{lU0A~c8{k9#>&|D6sYj+KZF<3$_aibXW z$2uzCqRj2pFS!0ZZ`LEco{gi_hNwMSs;{g~`I#UJAm>klD}GY!wVY&SIla`r^^fh| zLR4=a>J3(hgyAY``0VaM2FwKqccK|k5X3^ppt-!>Z3vQR4fdG1j5;f4rRIszQBRV9 zU3H56dpq`V&MM5Ob?<`~wR8sJ2Wmj3ei5M(f{5!9)q6mwp_~JLtZn~Qi3a^LSJ`FG zmuA-rJhdkOjWpRCZir-KGrux0pZ??%QZMcplgcII$0>{D;X*BY4p!b~QsX@Oa;duJ zyacJHaaz!CX%R}=-&cCC0M?$ns6A>~e>v{K8pRq3* zzmh2{K`X~eMB*1za+Tmvq3K%B<{TlUU3@8~rmmS-kV>nr~xv!1{kb zhtiCkIlTCksMU<-45zOFDNx&M;caJfGP@uM$VW<>Vl?vIPb(`D;-WbS6z&pXh^hU3 zW;>y7BtVl(@M)n6edZ(@Cj5mj6A

y6%b=JWKAI8{ftH`LqE}0lTAc)8qThnLR|Etdavu)(F;Bd7=Ilq@T#peQZ`Py3;C?Z5Ct`^0p_ zGWDfFLK>6rnpS9tMiSxepaa_{DP1p8PI!?&cnPauoh*wQLKlaM{V?aTWqQdH*}mV~ zSyF;SO1JF&)$pbk7dUdDQ!~P$v0| zgkBaE_)<45oS@xYd8L}y!9mRD@d!Geztn6r_}|uy_jRe{H2g67k;JptZA~ON5zQZ9 zVss{01f(_M*C-=_3+<7tEt_>OG+d6iW>49%+eu;_l2<)224fOpOcYQiU!YMZIg?nz)+X+|-m|?_+1s)+~Ph zj9DGC{UUUCzOeY96HK}QZxr-1BLQ{9nSyGHQoji&3ZZ%7;f1dU&)Q(|xp0VzU51g7 z%VLJppc2Fe8t5rxheLdA9ykEOk(6}G$fx&`SFr|0eGQ*DJg|wm;-N=kR$||RXW7a* zqq#mkZmg|nIVRS%-q-c__HJlw4D$B=Qri-jsHfaJp|AP)awh(E8DNB+Rz5Eh-<>jR zZzmLOwwRl^aFsjytHOg3YX>oN3z5h5m4`&ZX{CvsRa(ltW~36-Lk)ze!&_d-U&rf?@d00 z*^#*zydg})m`w~(_wng9SjGP%mcmRBJtWU5_FEI66kyPp(r(4pTjtd z*@P~bJ7`21v+6{zr3;dqw~ofu{DU4QgYqDgz3lGi#LDMs_O<5-yHRgZa-O`Pj=J4` zZgxuLDuMsL^8gJvtxH2x2?H|`P&fl~?n-FdPAR73*qOkrZueWpmn0!^)s7j_2196$hPu>eI8T-lwxh&56ImIm zOs>Docz{C;h$Cb@9-U#sGq5nmWJUpS2_Jm^HG`D^5HnytC4XG2l(>3L5OUs*UG%TN zuHiGG4bEuOPpbbF-4fP3ZXX*XC0EOP#b|tE$O*+;cbv~BMzlLN^NGGv#vxq@$jj`V zwM8sI_|~5T1N2j~T1%-)$*cXAG1@Lp1eP|`at3DIxe)KGl=-}FTw8!}4fvs`kom3P zM@AIzp#hZOa-Gry<)4G0wBDzTh+xnPZ=H6)eSdaC!al_w^p4CZOU;p1BhHHf!nqeq zoz?5Zv3{&+`>{WbTy(hGtYwy8QyLnT*Yxt1k_rsN?WSHmutr`t0aI$#z?c~4Sy3_L z3&wfXGA1?djE?7J#spT8Osd9F*NH=@7In_Gm-%nDzue#2+O za+(2lV*{f$(~h(}ixpm7Px8FjaX=g$^JsxrE z6zjh}y*sX1ekp{1-&yV|-UvVyb3FY9;Od`Ty%bM)%7kR-2>?AI*j1HPyRWx}0rhI{ z{?P~p3(JiE8kI-VQP+7*q8MK93-_0vvgcH|nt?orLd(o0VtbTka8*(Zs#x^(Y+;3@GqhfzrE^14}D7q+!t9()3-D=(#(i9=_Ma1N~53zG(m zHxTKJ))RM_5-WavZhJyXVNMKHD$~<=@tZ>~L@jvfy@Rh@dD(V!#@u4Cr%!Xxbr;kb z&`o`2Y^-Gn0XPwHF3Je~s3U*`Hk*0Sq(bQv>y!t%Hh@sn%* zwiJO0G#U2WP&p7Aa3@F!BB)<03!VNh?pw>0lyvkN$=lw%X3`(L8uc{fFtf92#oJjl zBWFhUtiyV0K=tI#MJ8`zQb@bvDQZ;pU0^rM8}9z z*+26=Vgf$v3oxLv*Pg{a7dmJmL(W8~vw*;`($MWYe^Efos&CShlf@?|vxl3(n?cj@^IzP`%w75`vC!Kww}c;7 z_pTl+FJq0q-73L@@vI0VkK?fvxwl9jEf5H-S5F-roU?OvtG40zc-7M@m_;a%y6$({ zTOF;KO)da%&eoYmeM$la44P;`Epd#A9qlxltvz*<2~atBy0hYE>>UZ{SX2we!Ln6nw4{k`AfE@Wg5#|AZ% z9>|wJ8Ey1kqY$>YebC5GKn(E?7M-Cebap4N{ zc?7fVQ`~j5@_hJ>wg}3Ls}jPu0w2mT@`SFfP?H@QNsQUH>@wk4Om@&^&OzBs+4p!a zx%8Dn50aMQMeGlAE|y4Hy){4mkZeDt@k3TF2!cwb>gebM1qJ!s8W^*^S6zvfk*G%2 zeoEiJ!o?`9_n;sOB#5rOr#fd74ZbRFtN8l+|562gjMIp!Q8Zc^#$Jk6if*MJ`}J7M zw52lYHQpu#g;B65qBCM6c+bTkvy-meUAb zo(ov$mV^WFOT19IaZOjg9qT#$?%czg%d&m3Ka~pNZige`%;{T*QzSBV~vX1sw&JwY5A2PUK7G*(%zDzB{es{gw7Y*p^Mf(l1YSd8wQb zMEK1%C3c23D8QH*MS5iKQ1$-d*ly>?!OX{5w;_VX3Pcm#0xcnytnTpmpZdW8HtB_4 zBsW)A4^pJKikjN=m?f7=ZVkm;R_~he*l@-(k!QeKz@TK~i zc!v?JNTarIwbBT6U}X?zp04fE#UN{>920&)yI6&6;fI#EkP4#zTEcg8+g76p15F6% zl7bZE{6+veP9_&r+--O)SP7ljeKV>24)e9#o>erzmesJ9u`vV`_$mRrW!eWmbWC7~ z9btO&KsCBe34nG$0oHqs&tW;~f@@k?iqsEGSU8y$c+6?%s`x~~Avk*klqo+!J%Dgu z9UkT!jJb%d06eB z=1C*w(6E+qH5_I}cchpD+vY?SLe&x&8GzE^GeF#F zSL(hDk$4BFz%+=Peze~)%d4jFp^MAr0l?)2wB)-^TH*C;rBH>KS5(hCpEqgkANa9C zqHO2YAqe&i2Ix8c5ElZI-0OLT*UGvoLOZ7sf-aph?D20YtF=#-!hN)vm;r!OVHt|z z4LPXuK%EEcrlxU2v=d6 zWl5Fdv#`jk>^p|sdse0J`aHI(tB);Ld#s5UwHDW3eEoxA!y=2OVSWt}c4Jk*&OCN} zrFY*|Re(KE5438GM?FZakqp3VBM4&Ix59`>Qy85bql{6KSza@QbE95RAD~0p81M#6 z+8+xHT$q8v)au|{SSS{v@$-Ph7d>e1V8+KRjQyO+aT}sL41JkwR62~7J;VJ%MhEJ| z#Jw;PH}~2o*R5 z`g)~GKSRHt|KnMaL@4J_dKJwb)7iu2zwm~Z6sWMg;Wi2HX+C(*UwR#*4I1kqYlk98 zV#}X+cYjs-el~r-%k~BGKa`e@82M{g*CX9L{43Gt@fgm~z1ipt8XRI$%0B;%;Q?<~ zv`FZ~kzrfKPFO@u{bS0d20!{kPuTbGt$8!V~P$yA<*#fTtmIb>PH zPYZ}){WYLhxgH4n8NL7dzDyT-Q8>Y|s^r`&Xzcti#(WOk{NBO#8r}$U9t*XZ5&B`S zVa**uVL&4;;B6BN|HMhFy4M6BuGig9xc@qZ_GdBGhB2df?Wf}4norslm$bs-%o-Dm zyJM$K!@=eVd}wB`83Id^5lnCP zg9Q;j%~*%T8%@a$6Nzd6^L6mhzq8SG4pm*GovgPv2;UdH__SVOWb~9!A4{&xhngEF z%U#i(yZFkabl5}T=vDUKtHHq}8jU9N*R-^2XJ=z^aWPnxJ`>4R=SN8|WB?`t57LlJ z>C;D-)9zh1OA)Y(%@yzeNoyK#{HiD&i-HV11IZ+aF5eu9ne$(J7l#3z8wkRXmm5VH z8KKhZkxMtS_rG;Ub7f+TK<(H~lVSHR?~^$iOOX|~cawuw~9 zZU>4=!J(=qeOMcX_xF@CIk5(ic7+`LBNy>bw-;qtYZxpKf4$St!OjiL;n$77mwu*Y z__2`lb~aL596cvPP64M;E@40Wh-nSuYr?T-+{PYFtrV7a1jZbti3x?lxA9qFj!X)b`J+(b=A)lTy@!GOA@iQeVK=s)w6T6ldXIy z?lN|s1#T(D&v|vua5_>0YH-fvyNWa$sUGvm#}A#>@F`u}#JB+OsL@mC#1|I@%^hF= z-{6x_QUNspwY*;@e)V%pdtJHJsn-MHOwo#LY1*4=N@?J5h*`IxX#6Qaky=^yg9QM_ z{v*Ol`-t<^%?uC(_Vx9$Ycs<3%*ke260;fSg@rP*gqiYmTq5CHs`UDisbJdoE?Yxm zt(ho1fY2OPn3<$Hjs^#w;(2l9zW64SA77TrDmjv$`%-WV#t6-5=ZjK@H^pc_il=^n<;j z2SK2YZccu7gWy(_9;rJb#ZYqOEE6^m}U>vL^rHFwXs|Z$Xs_2osj{cj-VJy2@Inf z8uCJwaP=i@1dYeVU0!pijKdTp`x{Emq3fZH7t|%3^c)G7@GdzD2xvrM@w@^BtrF}H zrkE-kZwe>$t5lh&9sm0d&NZ$V78OzT1|P`C zsBy%ZpecIEi~TEOA6w;qTQ&Nw<0EWuwt^tr3mF13rPr@H<>e_-($a6A`uO@H)8$Ne zY$02n(a|Cy{*B^SsMbZD-TYMp;&*=Dar(0_9lW!=9AhS=^@?(x{Wcb1XH`BDu$XKb zIxioZCt1p+*2IDE2S4lUzuwu|$rb+_!GijLhb53rOK%l0|0F(|forGou&U=hy@su1 zdC;Nq9M>D$^o9}MqX5o>5WMx`nySCQOQE>oG$1$i<^!_uUP_s@GuIKC62>V7n9n6w z00XvD;6xx1P-=Tcebe*^$cJ?kyzbwv_u!PPVFu#6lM3J9jqEb25}~V-t_9LE08ss# z^*lCf;P2kvm*O)BP1u2XEEqTh-Qd!ZoxsVV`Reg?(0=JVmYK8p4H%9osaL-kV~Bw) z3Kz1^b>IAiAt^?N`;cVyCT0J_!ebd-+%9O14vnQXT_MzPGR}t^G`zWjelJ3#8G8nX z8V>2X%f!CbH+g8ayl5OsTPZ5m@hg8$HhlnuQv>|Mp_<($18}FIE#K z0>d-F5V-VpOev>2(Olp*%W;i<5!2~wtIt{<3CwSRgAPX+X;(M5(+Cx93vEa{JIXP$ zki9Oa z-ZorH1J)92NY?Qmc;ZdQ%(?}L%(gfrquam#P~WJuhEFw=s!?{kqpjK7K*ogo<)q%` zATElwh3-US$l=DH0WEO;GQp%Y12eTFKGyIa=@saEy+x%{LC)=>{AKo#dp`ySddrW; z%6*hFXFrFnkJa2)-G`wRgg6mj1qCgPzKCDU^xk7VX%m6PYqK5}fhZ067o5v%=&ZRC z(*edkRk9(E3GV;ki~)w`k+GJJja!WL_YPUTG$G)#<)@$dE;z+oxBN(kXmhJXD$-@$ zU*#F@rA?37#ixTqqSAat4mvS!_r3ni40OuqgZyGW0WvRE#FgBO zEj*trYPnUPPY?qMnEdmJaEL$7f+_!@ume_xDv3=PZou6X6BE;6Wo@=%1-4FDH}@Gt z*jd){YI1w>#544Z`W{WQsi@6`e#F%zIuk;qAEI>)T_&x=p*de2XV|OtZ_f)Ep$!;S z^v8eam6VLvsqyL=^0DQ8?b`Qz`!)Wf2`$R2cV%TC971qD-Ql6}N(?JQL|%rXwBB&& z9D{z|9pl#Ho!yd>`Y&Hx1wIf=^HR@T#v#okSG=9ld~6QBtV{MDNJS*EP32A$S3aH;IZ6zyQs_ z3Po`;j>f>ZRvT3R|MqFEa$JwK`2>sx*jA)k=|7+NLQ)jrj4mDSV>_ye*IPPed?B{- zOp+APWBvjE=sSPZfIbtNf#EfGY|f2_TwZ0n@f3IYcm|oOmE6G<-un5BtJVc6_@D6K z0a@?(%bYGo_{)5N4OEsPG{-H{Ibo=+_I4z7LaBg5*2E_$w6?6jQX3*0@0=D3<|syg z{W37Ow6X%Ho9jycxyt3?!I32{)&FG(7jpwW=*#i$t@VwKhoJtJOW@WUlr@o%&6L6; zkN;>T-f$|>Uc?!)Av~VefFn0i(2%W}c7GKU^!H~~@jhf)6C9$HsXzG8Egq~U_Ao@X zrIbbGtDmA-6;tsr>V23tWn?Y-mjHvPNpuUXdbot859_ z)jhw3SzH~CPT0VZYEoc!5DkkoCIHe=+JNiGF}U}i_vXmeE-eEx0;%-Dr-O|8pFc;cKZ$v4xV1`uk4@&lY-GJ zpwMo>yYC;&nG^IUin=*G92W#*JelMoeD1|p`S?=fkkC-Dsr)t0iWIli z8(auj*)J=(;_~6<);`th;C=SV*#Erj);)f@Xm9$&xzAIZ0f=6%!Ypb5(zv1U>y=5! zq@-zAvLHEh(IIq)c7`4qw&&8EN|IW&k{&f{-9KR!q15ahxkwM8zQ>DK#lHP8!r@S* z#mlASuDEO%PJ};K@jhdjrN0>lhOB0A@GR-R zi+4U*HLub>Nc|s2=i$g^+lBE^)!3m*ik4`#_NMlxXi+0-)ZTmVtyYOqdsDMwk5Id| z8c};w)Lyk`_@4L6e~{#P?)#i`UDxjn__qVf6E7f|{pPq^Jxd9wDfg|vsAzzUJvKtG zt(~ce;Q_YV$UMi1%H1X;QtlTH;IQa$)t*u<9u#~^+31 z`_Xxf0-IT*H$cAQk+^SV@+-xgL{E?a2R_INg{B)Hbk=;Q`p#$b;L2J%hNJy)80@Ud zrB})>aEo}R_neXN^Dooe&ZrT;P2uI)SzVwVxFX+Sc8ldiG>j+s-YG8xqkM;z@Re`Y%I>AEQJRUsXuZ2*40`- z$$fg`H@{B?hME{bNZ;!|i=yDQd-`*Kx5!EdQ8KJO*AbE|G5FoWRT>!26v#sJBFf9{ zOiKmj$V~t@@Q z9J{lec}5&&BbI_$f^fxp26atH1UD8Fi3o~{i&9I2Eo!e^g`P-3Itvova7i#$ht5{7 z{AQrb2mIkQ+M*6;I6XgCzwXe-h^n6#sP*m_agg#Sg_k{)**2Sfs{)cxoNwW<31q$I z;{lK%TF@rCenw|E(~6;&kpV5Zd%Arz`s>#h4W&T``SDAh96z9T)_+`JLil^Gjwb3g zZ&b4IQ(4va<@O&xEZ^xet(%H{3C^vMAbYgCyNgrXM1j#?y=hBiRvPC#_5g}axmJCL zYOSG7m*7S()IoaQ_GD={KWGu8p=9q;Z z-VCnSAelpy;;%`bXFh)>UN;5QBM=}?o2w?JpaBo2Yt5_-VWbZFZb|p*!H~ci=q35* zm+#MG(J5XgA17_Krv~=)8j*>DVqyobLXv+xgtGD&gXzBSHnGupL1E#!pIh|@v<(UW zVFZ-MkW4f`qpqy$wUW*BSavgsIuE#YeO+%pAN9zR6$5d-XW0{OdK9HUEx@`q&#=Rc zBfh!x>abA*hcR2H2T+-`w(5M?4I%Dn@g%!q}(DEh4hJ?Taig!lR7pl0lI(Q7{g%$MywAU`B#Fcc&m5+@lTN*G^cRl{mCi`21ULZSEv zYi4chKbhOLmrQn*B9T^95%1%1K|7;54lXV{3J=u&5_K$KMPy0GNd!U)R1+cic`t#n zL0i2Cis#dVRMkkIth-oDXKbshGZ49WJ$`ic%W9lgDng$PECpJrcfkI}ax$b-Dw6=m zhJIW>>$yUDBBelR^ZyR|H(TCL(tfIc`D79e!?f#aJbv2U8CJphgIvN1os{uF8I;gi37xH&jD$!)-1Tcneft?zp}YNv>2 z@*eprM&0C7dQqY$(6J55DplQ2sH-TwO3@bMh|II1D({%eZ2&&?MNp!^E?HGxRGb>u z`OFImFfiU7p;S|3GD~+IL8F@%@02+jZ8Wq*;i z$l}vlwuU%^#;)dSnZ4a4uq#tD=iR|{$KIbh{x&G5Xb|_m?^3ZZ#@J6_5$6i*}b-Y+Nl+S#L-kYxe7LwnV_`mJstH z@g?3G4d^$oU?DFeo>TF-Z=1l8Sc`y^J|r%AY^-xx64LFIS;Pc0<%EL%s^SEx@7g@T zJ~Ly`EWN$8^DXcVGy_d{&- zRM=HY)#`ZOB>q0<&toLSjy2?_Ax?OCjsch>`}-bC%#p?-*$a&>78e5&HoD1lk?|}a zD;67FVkIK7y42*uO8Wfdm@LFLJ-q~nHG!nJ`)0xot-oUf(iTeeTx4ds6~(o6HuXj- z^)y~zdIU{*a>82clwNk+Qejh=7$40)W|8HjE&olo<9ksu0@o&g_@%N6TKUmuJ`O_A z2E?V;kfEGn0GpGF$YIGBIldhum)}%I7qV8XxJ?DK%L?w*3NQQ#l?H)iKkfEqUcK%% zQ>=bimj%4QLXV(l)=WrqEEo=}-ARi3p;+VGK>)cdV2uZV10h%e8UuJkOEx7bKP8z84{z-H@L`EXS;+!v7P&}f;I!p#c5<r>2IhS4Oq}feN!tIUJ-zP;U37rCd1F4v?w;WENVD?YaSP zA_!yzr6?>P#FmuCYY~`jn39SJjZ=b?%O0K2M>TsK4p9bj#l55(kt_~RCfH?lq=>)` z>FBZL-#uS#$28o#7gH~_w@=8?d$wtrPUYHL>~7l%hJP40Q_J4p$NTbh&ryWo-d=LB-->lg??9k9djs9y$9uf@>hF6yjJi$g^q*5qV>yQzOqEn(m zA5hTeCQaxVA}d!iN9taX<$f)o%r9)N&oOT<(XBX^J0Di`Eh6u_Pwetkr8 zfECC|r;sS-=@@`Y9X7W})hgZHHYN)L)L29x*gIkHB;oUCz(&>ZFOn}^O~>?Pt?RQb zN<>(&2?*pyjff92@+a!kC_(SMr zAl)GCQ!NiF>_2oMu+{@oGAXg6yqp>)+LN03X|~4QUmkdp0Mf*iqwnnM>dLrUK7MOM z{iFZ=w7<8|)kyolo@xJ$3rvVh4A+0YFK1v(=zoMcY}NZ%R1)yvFTK+E%P$GMmyhp% z1-RaK^!*2W3d{G0c2~AF-HnEOhyhNhi;LJppab;sf-%@OZ-JVS%s?G9H_)xKd*djS zHRTo!8PIeiyMX{q;5UM0Sqk=U5u)Awf%Fv6noL1B-By@LZV*7*f3cW7|nTnH#7`~hO^h&2IV4$G;hp&@sd`wg)M#cysxzOWNm+6X}6B+m6CeI z%p+Rg-wJO_;pV)y>90n3N{|5UPIr7(P-^S+)YatKCw+A9+7U3-v8aX5SFdK)^I9Uf zV2%L+0b)67iZEwTVEvqaxpQo#d960!o%oI`6$nlnDIRZPI`-n)BBtYBc%jWduXPyR z7ZeY#9}Dp|`fZcWN*KR5j9Pc4k?S}=-|$6iyy)zAm^N=nD66c5!{Ow!+p}qqRSb#v z&k!C71lRaBePof@1+oPg#XkA9DQ#bwD2* zBUGa~v&|R!pe1r}AeHlEw_t1h-+8w*l}TI=1zjLv;EZ5y7T=V>BgR6Nvdh2Ms?*nh zuU)nJLch`Cy7G%Y&a2pE2DdIQ?tj$3-T^2I?TCaw!0f_Udp)}kaJB$**OXErg?Rgh zcVLQP6Z&`YF9AMW&oT%5PLJ;s*=^fb!0_t$eWIBH*eS-U9OX~=xb6Nf9E>IPVR7a> zFq2Uyi6K>q2iCoAmQVzNO>Dn>sj%jY+9~*EZ<={YZ22dV>ul~CU3(oiwX) zZ2x}D9J8ie+$CuD4%?C~x7zw zA(x!wmXq>sQ_~NDn~l}m(>p=Q*X1_>D9k^8903u0d zt;k6Gd9An^!m309(wVjBR--eVISy&6du(wPY*MGC{8W~Nb_t$`%ox*={u0{nl__8$ zfNQnog(J-HZ4_;t&(LG)$7^jADnP7D7LDRpq|cA8#(xaOktBbJ<6=UHMHr|_79|6? z?-QJtjI;Rxi+JHweKZ5a7|aLs+f+%~`cKTN2L#S?-P#T>yfMnf;}r)}X9`TwE+E)G zt0P2bCgw4fjrd~j^%qLaXfy%Lmgu4+NH;#uojAp;1kMIu1Hyg(ngBahEM*CaIZ$0$s3# zuuIw|%ko7Ow{E#+BAFEU8|ZP}C#Bt98PX&q{rIdInd>!Xsj9vH{?9j+r-hyZds=IF z+LI`8XgT$RI$SthB7)Qrz4hyoUY9+27Ks@hx{%BN`ki{jL=sY_S&A%q)6md%Jz1HZ z^FH~cg>@&dpYe#RW=cn5bZ&0$9L3W4bg*@FV{&PZm_+?Rwj0+B?tAU~&4pM91VgNO z+6koV13W6Pt+!Tl>k9-TIb7z5Q8y(f6k1_%U<&ex%kekXBN#6TY7=eck#?OpKo~F) zs-`&ac$qI&kKzL6ka;NdNs^j1FO{7`?A@RH)y=!5Weja+#QmuF^;LE00Q@IEw#7#*_`CC}dAui;Z=*eZ7S=*!;y|y@}xoFg$`dKa2 zd{#ykiHPEVqpUL0_k__M-$Prd^3k|y zQz|L{7V#)Hz9m5Cnp-@#V2@ zJKX%?ysWA-2;8eOf*@%)y!mSk(6_y&R zUsV|C`i;N`i|vnPf1ZdGo&oK_ptD#SOQCN& zOGtm@C!59HGu$q|C`TX8l^Vu6H_23UZFeUpQrV+iaU3*#$o$(`{(p0(HD!YoxF>!F z{D4(=5(x5hlB+7>S%g49-yZzbCdhR8b99+sFK1o8zsB3~A_o^8;L?Dx4CKwxpVDDq zDR~0=2~|%v%ifHRhrTw`Hbz9t0;VjI=SxJl{W@`Gvq{t(M7W>nzyBY zol8*mhuXZ6Cw~BLklId>ReI4^H8H2d{iyNr@t4tqHMY9!=*F5d;4w`gJ{57`*QF1M z=M969tvfR8F|&}?r1ng0cdey)dagcPuzX$jwUdzu5eEE;kigffH1U!opn|$d9Ug3G zpH;eh^P+O`izG{GMpT*Lx6;{~@$G^|v~g0UCW}=Wik+~_BjB>bwUKa58S1vm2-rP) z4-P8G?H;HIf&?~GkIw(Hm6t>Cb*5Lz&Gb#Zrf>(P2zHL>z|c*oNnmd1Pb>*h`=Ifu zIx0zu9~2f=Zw!1emU83Xt*;M7&}ImxLFek!oSZBX@6)ZlJzJCxpHhAS9uSvWJ$d_j zYAzvR9H2@)|M$z*$=BE0*LUa0S4fXueq`uruNKlcZcW&T@J!LSu+z#y(ACl=J=S9;eqdmviGE}$um}5&gccXN@rw0K6LEI_$ z$K*ld#9JCziFSEAmEBtB|tFKTPy@3VH@p& z{!--Bi}gr%rM$?>dQ`wNRdWu?&QOFSwFTIo1aYEk2Sg5b#Gd;4I*Wc$+kcUucDHrl zt40-OTD_h80p1}Wec7|*<)t~U8Zi;?)FQpj%)@Z~KXwj>^`@M1Y*KWDQaDcO zMZMm=P;?iC2hpu1C)879=M494_~w3UKq(stWC?43sUVlHnrxPNQd^tAtCY`ZonG`T z1!#%bVAPLg2n~(6PfO*#%O#^AN~$(yR@p2dLdO{`W44e$ows$tAzI}jPE!r!u2 zx)!lO}#EF+bk0sJCIvxykCw?1}k{zBG(PnIpt+t{tQQ+Mc$Emt00l$hi2qA zSp^;K<$zLpUuTRk3lT^{HCfRL6!=-f8vZfqJCeI3c~+5Xm!=@`#)zi)+3wK6K@yyu ztJessC>WF*-zm=o1N=4NE|sEh!K8p4n4CgjMkvhvwpEW;MQvAeEJY1at{Ud;QgR{4 zb+u`?MP8!K%%NlBsz$znxw{luqU76|W>C?D+x6;K3EzaqbCpOj8hjDgH1YBi$_Xx)Keg`pfdfJzOxeg^#6V~mxLgAt>%5pxAF$eo z$V|~8U>$LWfpija5Nt?ZBDYjdV2-%XJ?djrG-}=VUB9IXFpC7FE-l*VF@f8^U9`^u zdPi!Wa`7~J@PQ1vFGbDTTx>SnHsUzmVFztf8W^RiYBb3WE44eb4Z6nN!E>}sWt76@bR@7XeZW{>Ua zoy#Nj%BChj1?dQGg2t#!2m$;LU&_N*xk&3Z+bHbU3VT;Zs;2P5-V}f{q?X4#ur0#z z?zbMTRD_M{Ra-l%wzaMPS>I7O?P5`)exvh4<9eNPfVqWo>d|+?loa0vb_3V~hqB=l z5e|HCH421{pzDqRfdRy!ItWxnjHSfr$LczE|b=f-auk>wS z^XeGiCt%5JF4*$_*TT%bcuNW!gxeQHbp3Jmz5dhDGu?g{?Gf;=14tYQ%-uHvfqa33 zN=+|l*kBL7BuC_G4Qu1HsJOU;%w>r0_;mfx!f82?%{>>!sqfHP_%JZQ~FYz`r zxhQZtjH|0CJ}CYI?0W%CnZq@+lcgVTx;l6`L=HIt-&uHgABgHf*(6$EE3*gs0x@34 zVyZrM3}osXGgV%#nbm9AmRH7l&jss10k=9t0-scBM$wM{&9s~iwU@S3AcuweYN*n< zsVtAqd>NIKw

+z~RCg;+b&d>6bqOCJL62*XH-qhU@5Bv0?%E0Oc`Q!TL8YP#|3B zPYWDot8TYn=7N@_3$c8xBPDNBHO+t}zmuN@I>_gKHa>RETJnzE+8xbX{tIMV$C)P9i>VbM~k zv=bRwaMlCxx2B4$&*cg_5Aj@))9CVE_GC2Vc$MbN@B z!-K=a5sJYcAiw)X>b#3zCV7OH4`P( z$tXcg*XOqp-Zp`+1*YkQYTGdN5Qn+xC7TTkMoKLccqYHOI5_YSK62F@*=STPu(pM| z3NN17Oy}tRopc3GsJYq2Mt3K{)*qSDf&BdZ{+AtZv-I!#0&ZElP-n~6<;yh`pYD$- z*95#B53|+MIjMS_AUMRJ=g=j23reU7LC{1bl_FDdrs{iZ!|2-IR0Pl8*0`I%f^%{? zV`smG(1jq>R3K@uvug{NM7H&AEhiLdO5e0$ znrpBAzPXzfA32$%UcRzVFM4$Ix>iAw$u2#Z=OHL{WK%J!#~5a6hv-cmG6%9MieMXt zsCvkwU_(uWvIFIvi`KbgqKFy)B7Z80Tbu| z`I1BmsYw2PWi4yA54espOaz&3QETJ)-vghh)RQ0I=RHnc4)clCrr; zK=al|c-||UmWF0euQeC~Ae$srKLXPFg3A4nO3|WnrSRqLruaPZ2-6~9-YO?~t4b~j zPEsQVg>wc1&?g`krpBVjM-=?mX$5QlG5&e+_Rm8s_Huw9&PIG!Fy~|3_}bhi2kgsQ zBK^Er960QsNg%3|8Cu%fwEwaYb~tn433w1wFl-SSLM{QdY|N5SNqrZok$Lm)h~y>W zsEIOppCu&VF*~Uu;3&Pn}ZWtG_Q+5BZP{5Nv_Dk`WZhE&6?9 z#P4m{sAfE1fC9p_=iq0OTP2M2!@_#P!?7~CDFB)ptmTsp=!u<>*Du4)^tMB;NT zQAqq7|LO$%5`kQgAg;uWhM8_jmSrfw&WWAVe>$LDrL==Cx#2D6BSEB}!KUSP}{$5_1U07P&cNI#U`Ts3eb%ma%m6Pljq?8I# zjVr)pG*nQmI|l^NzVHSzKaB5D^?=*xq=jFi!KB{q)#P#fI4@CfVBGS2e%DIXz5)QV zr2{uDvd=F3sL?S)3YDmx)V&>;h*g~e=Mzb3qX0F{guL~w=u?#QheY_fv(AXI}z z4c!(QW1TiXX!l>WM0Vkv(BYRH7(sFq5)!69igE5#LSA`LDI(=O+$`VHDl#6~>8A??_=dZY4wHM1+H zb2j!$08%e}qfn1Wo1-0+eY%wcs@)d`fg-qY{~RSS3|#;$3B0wa9&107x>SxF|Bb1x z@QVf@jPNQl?+kU|bLj7RvN61jK?@r}>AP!Kql_I;I@&ft(z2=o1iBM>{TZ5_rM;3l#L_EYswM7*~SMIJKC1)`-K{Bm1}2w9UH9bW~w zHw>e>ltN+Mz2MJ*TwqDdXCV~fM)(L6u}U$4p*);9B;rF=rkU&#=)LcpYSFQ%BEHP|-uNz;q#s)CAC1z`r196ExYI))z`n-0B`s zS7`K3-qw#*D#G{{=c7+>rs!;p3cqDWsMt5NTosHCb@JSuXkdiP()KCtXGy&$I+6dO z=uI#L2A7@mx&>AZ&{_xjbmWLL6+T zo91Vna?~YH3MVdp)ItQnG_1$W|B}jdo!`1N3B;?JYWE8I9c=J;We1%8+AlAU7Kou0 z^Zv)%jAwMYdADm21o1yB74KrSj}0c1(pmw1TR0KeB&GvGGNAhW>nXuUS=DGkB}3i& z`m6qgx4nYl|ElX$Z|)v1-6c5xds|&sUuNsE^0?*ZhzD~TP|xI_F;Vw}N9rZwv5iD2 z_Ah8cr!%?~*$I84Q%~Cg)n%Dqc?hm@K{D?aea*a*NQvw;hgkRk{?;!5zXEaNbYMO9(lpG*0I-w=y0m*?G{H##pL2l66 zLM-m*rV6gE@c;1BubJl;7Rq+7R(=^lLf)po0NB{V)yZTw zuvPo(s_#NfP<&Qgz*M&FGAdxDy{>ooRczbMzQaAqmIItmUj4qI{m#5Sg%}q@UrAM* zk;kn$_TuCZW9Hr)_x0P)^M^>e}Nlr{K|Fm-5-8qLKg$* z+ZKsQzK#6BUpQ;uAtI&2m}UQ{I%|~O@Vpc*ieKdin2mr!Yua3<7mAQCeLSm7C!Y5pAFYNic|sa+2_2qd|+ttq`E4>pQ_2?ZpE{gZK?%>_pR+t8SlL9;mbfS0de zA;3guaLw}XbAmh}5RI!!XK$>gvx>-A0_C5#ANB%vo5j;{f&-=>uB<`M9G<`w#!3dw z(%n-|@K)^k$1%_8a7f;M&ig(!eKX6v*@E1mkCaAxWpD0pgUd}-vn|YKfdTS5kQI7X zHK$b$#{MZ}qVdOtfg!k}#5$kXwW8kz*6r+HvVPxQ{bo0bRbi~l*6E^Fwp)+K!CVvW z;Z~g?O2sEgCkQ#seX|oS{`vl$H(f5JV%*Y0EP5s~GFW7MMA~-R_QB4;K-s{$eRXEf z8y?ZiF9~1M_yV!NL`C$^#AM8`n&Uw>gp{e6rdses_>_e-dyfj)}eR62GyyUpH#MKM}w0Yw-Aw z(|vzJDSm&we47#A~$^5&qVeKIOetq#j~e_8`PJt1Fd1AOOEsEZ>Y ztaRQGaCLh9tMp?f*ejjbP8XorjhK|oA#0dcuda${(Ew0?RK;mh(_;7h=!okY4-ZmR zK_j&!wqT>|FEsI_vZ__bLaY0|U#QZdeat}=M^5GRBF6rHZ+o00r(KJ0-}#j=5sRYhTN!$x@YfMM z4^;2uMFtJ7|JL0e*(3`WbL%ej<#{1>2sd9Jmm(GUd=78?PsP5|ef_1-=hV{Fw$)Uj z%cnQ3fr|QtwQ&=EA%EO{PkTS5^9oz$5?FGvyxeBiJ2<>iwyPO^(sk$9kN?N7j-|1& zsd2%zIf{rHNmQO;JwAo>Mj<(LHoN3YfTa*3Y2vplBoSK!alfpn`ke6Gc!z7F|MqiK z%Xk#>$!zF1RR?Z^WZ@*?(%|d}VBG)*2M6up!Nq)*i+!z*kNEz@ywWB+`x}Sl7Ja_S zj|P5DK0Y_+{x`(|=PDJI9PcPQYJ$zmWpU^@9-+L>-q{B8G@dROhI*?}emcm+cknuX z{FOsTOJDYN1QWefF0>1v@6W#x*Qme>q~y?V3Aj;y7;w4rh_*L{!yrsxaj6wgtkwJM z*rWYsv8PA&)Cpz%l`iC$P}>iI6aU+53vrI;)>{ozdnSna@?i7d6BG3%cH_KyyJbPI zr#E3CgC|8bo&AUvUv(5;`o6m^9R9CuriqXk`?z>S}AQJG=*UyVfscYKcevkJ4vv~D;4-b#c3?5Q1 zU`@r&?apxzCBbQgMwE>5T0}lH=GD>5Ou;8cq=Np*iGrxe91#(C_j(ivV)d(Q_|Nym zF@Qke9@~&)bJx>;d8nCn<8YsGe`niue{FETJ2~4km@>cek9L41M>OENHYY%P5_r7t zj8Fa^@2n1I&`M1V4!+wc1#VW#H`5mv7mVIhZW;CGOE-5@piBRcTwDc8AHU4BEgR%i zl7xTm;5>WGW3il5Y5m|!q*okaIHqd;d9X^vds8IRsiMB#xIXvgQ?*2F73Qar@o&S^ zq=v{zbY4j%3P$(5;D;~9nw+-n3Rwus(k7(vVigSzM$ONAF!fLxh|D{9cpN&6Ol8RW zI|s*!qik~`LhfSbi=Q9kd_4NXsscZG%%(A!f4hD%Javv`@WDx^%SABR;3q} z3)<1=<`0dCSP`;jDYiF`t4?sYBIgqmm^3ph3)Ts|sI;^mQ#!*|7nfD@OA65MBOrJF&H%M$|xLr0w0e?-Xt!mv`x$w)M@ z%1oD=Al7CmU;Q_Cs@ogRToce!B>hGWNh%bL6=HN}1IR^PN=UtzkM z&ya0xA_4bz`yDo~8s_);N3z*=S?D8No^HvwMRp0ZW4=^T2nSr_y)$g=KXcjDIVB+_ zh_9W=UJW#PKh8_~y>y5yhuWm=?s~WR!XZc0=R5~ctu=S^ujCXQMt8>#ptl8&& zt=3}X{AQ8&wEb?9gO>JiM04?*je-A7ct%DB#y-QvytAlcsoh6c(B}E`=MnhZhrTal zOx`bF@I+#uiir{?+A$BS9}vLbNN8h&7!Bjbv}9#q?_tV;ow#YFbeQ4gyQE1RdSLiU zJ6pbg?TpeH?IT^I599uFI03@WTnbl_!R|98`;#B|O~ggS#Oim4_wBA%&NBM?{A*`<9@9^ubf=JpI%&4J(Xji= z9r!fgu98JLyno2oz256+^$MUybny}i&O`I76B#0lQhm{{HZysW)IAk{@hXK!q?@G| zIb}+LaUQN0f6py!B&*8xgD{1B5U5FcqF@!H^~Hje0?zJ?s{g`t1=Vm{bQ{SLyV|iB zY4`WXklg*HP9t99+`G{)*+$#9c^<9u(ul%eiJEg9fg5FHeyXJ0X}olnna=T_QyJKE zIm+?RMe%Qbf7&6<)WEOg;_u%UAW}O7-rl)o?K`yyIFE&Xvr6yi15y;{8yi={<~zpv_WrvGW5O7PE(i(3(pq9P@>y}XDz&LZ?!2>NtbYxo9C5+Jd>Arw?O&F zsaQ(-lJYD^C1#^zlgOerM&7g>oL99J*+0d^N2QWtWjv|+vae+?hpzmNFL*-)Ti40e z6zL)(9u}yp?KCgi?~&l_%q##asFv@K?j=9K_xU$tNp2b(mO|sg{GFeKLNuCJ&5RLC zebCy7BF42p4)$n*um8Hbq?a$I(iiPNrdkq*e=aqT@JDO&^+9=@<&-4w7@1{Y(lQM7 ztUG{(gbPWJ{jT`0f^Myg8DeI;L))?m{ueql6T{-$y)1|zuo6Tf^X{_ zE`VZ@_GV=w*UwlrW#;cGi&*=UgsQDoG7Gkpll!;+gP6uJHyJWv-4r^*bg6&UEruwa z!k1G5CRNCtsM;sB`%QfGJhN8m_|KeyDMCm;l9O z7DAUU8it^ksYx~QAnIQdmOD)k=}5nD6+@)KL+{@zlXQ}7uO%95r>IM(xVWPWMPezj zey=_=L0Ig%L6peeiihYM02ghz`0YVyaj8RF6DcYEEf82vXvbh;N3unA8y2G|IeHUh zKjGlL%Tq(8et3cvtj^_2pP|+vNMxPd>Uljd620?U&9n5j)ip?P4ZoM#Fo10-jW8p?j!5^km@#Uq}NO#ab@K=N@I^9Kr5=IwD`7nh^ks}rEO_@tK=MVm)r&^A+NXM>R) zcDEiIp^{U!+~${059X3}h+b+70Gg9F_B}zdUL8|te7SbX@h`a`PA2QJtgqzr$R*Le zsl}%EQ3_Vs685B{$Dd@n%_wQPo<)FzqjGAeiV_uda&6~oD4gf2V>*Xy#cYEU3qm&X z^hwi{8s;K+ZKpDZ%`d)Ke7>rTppn&@^N&5^^pzLRY^x+cQqkLfoqnN^di{$a`SpCg zl2IfVIejPhsi)AAF`fC6HG0qODlAz-OiVmmlhcS zkD`<*oJcG*)r8E&wkmbe9}Ei>XrT9{QQxYL;plVc5W*#veFlFLTliF#k)=ExYlA)! zzdQBd;CYx4y^9q>_7#pq7}A)2zOK|!1GA=vwB9y@VXF3uxi2*g|E=s@j#5(`v0i&H z_pGnH4m#jfN_TNmMKVs8^cg)k%$n6f0=R!dEVv;}X;G@4kJ& z-^y%$4f@alc{xz^W!9$>B(I^Ipwvtjl-biHGsRQ`geh6pwLy&A+qSGcBwc~8r_U8w z-)vIFNX!RgO+4lG6Okd4vU2_)n>U6H&2RC!xN~Zt^`fJh*AZ_owe)j}-gZov8NION-2QO+d#6un7`3-xOIHpRTtA`1>!p z=_zg>8L^6fsCTBHyJ*FpaN93dRr{Xxit~3amQ=l+M5pz?*C2E5QoZc{UmuF7bfECNgFm++N&K;o#v>x@r<|ut5S-Fem%OCUY-Wd0@o@rXd zUyJn9Mhg0|Uw+*tLd~FHF@;lK`gZk3fA-Gf7A`rpe@o(GVoQsQJ=Je|fvo}gcL=rsP&)e-)i;R<%RX@rN zTqoQ~pA-U|h(Cmc7Dbc{cTe|AKQcu!6L-0`hfccND4JsBg~D{;>IjC3PswsIC3f&9 zK#e(HUphP&J1!98{fUnugb5+YE6F5rIDEFVvf%3wzW;dkraIGd38O!OWPK3cC7@r%Wl71AFi#esBrL? z@{a(=x!hTi(r4-P^vox{%YD)IOj+u~ZQ0W4muOKxE$$cVC1-mG=DS}KDpLQ;WM^;yuzGlIb7H=~BI^fRW4p>OkaKB4_jWJ>w z$f$iy(rmixLd)}B9goNuq0p0#530eh9gVfK{F>)GL{?4Xuc)g(n+S(>1&*6y!YATs zs3-pwtjf!*=PMp?Zni7~8=T?rEB}i-j1TA%*yRjV8dsN>Ky1+I&y!sQuSia7CizWRhF^hP*uW& zW~}Ty;$nOL_xrSIUbTWBWocn7gyR$SY~QRhc2-vYsAu0mBEj#hamh9KdL#N+x)Xl3 zrry=t?bb83&e4;(Fn>0zCX6za0$~HopZm2X3IIz|`VA~sbylgLS+SQ-%AK%gk?8QS z;p0d}O6V(0t`Q7Jz)Dtpv>=vdb2IctR->os)-+~AW1CWz^{dXbn@RiO&E3k1QNui6 za-^L1A;XC1aTjNz18avh*ofxFD&09ohK`x57b0g~fsFhuk4$r=@v+%oT$EmpMS9*} zRxL`ORPEo~osozukMqiU`YxdC=3SXyaS>vSM4CTw%{mu`MWdz?TfS~InrErSn+RP_ zWdCux;{IB?J83=1tAzNv|9<8}E;O|9dkcetCY(PPsZF1+Zro3$MkRM<^xgW^AalQJ z7?BE859OarZyo5Kw+-bc=8*PF#iU#(GuTQ1rfvJf}PEGpSUi)2@%X|6~d!+rOVx(Eq z`$+C7Ks-_vexet{Pd1@$zJFHC#DqM7IneO)7si9*{I9tNdF_ zmvQ}9#xsvLP^N%~nAtiz);J=y);G_EHh3k!aQL;|H^42e|3Sq&U0J-(N1RX^Pb;E7 zwK1sfr($GLvPdnLITY6-BVIFF73ZCSVRjJ=M}oy~xwL?jD1NnJ*`I*!@X$jcAXa|f zt`ff)lihzwZW7W|bK}3$XGDV$hzakAeE}gEl2-gZ(RMhQ+Tp5q@$Vlb-oTwU$rPVP z!B#^hzi#Y7@6Z+0W!gwr8u^b(~zh0C8 zB{=T!p)~=+y1_EAUQdDk*;i2#X?P@{cT&c+PJ&@j6Aq(xB6R0q%AreG>dJ7&XBRqb z{J0vb$I>{5K}wON#uH2;o+R%LiReJ`Y4@oh2{m3|CE))DC5An_mOOLTK41_b zePWYElXUYy8C1<%o~3B~E94hm(DmkgUcarXfeZJb6;o&=n2RG+jx?=^NlAJ;hVeym zYNH*C$q=s++RWG$4YYTA(#k`&Y?|;%)Y%N})Gl8FVfYP4+mA%(bsMcH`GujKn@7W; z(7RqI1QII{J)$iW5nZb)Q&CaD&qg{*p2x)=`UYTmn78UvzLd0jCMAQ}i);SG#uLt) z;tNkoi+1s)fvD;~R?_Mw0m=XD2frT>!~=0Z-4rk-g}|}sR_SGh?2v4I^}__K#wX(hYtL+M#C?6%@|qkISZ-Q8znVdLXeq=1y2dpp|(U1r!>YB)3jBEz1Oe(n|fRwgdb6NYc>FW3JbDlZfc|OToY~A{P0urdr)8Z`Tr}M`m&LS}Mm&z*V0-fu9}QevY@()BRsX=l;ml z|HttW8d2L&BgMvKLgtoRE;IKCv&t>G-<4}Fxg?jl-xf2I`)w|{gd$4r*OKO%B)2k` zkd)i!d%pVz&JR21bF!k7<&HDSLpo-;(e`cQr{I9YQ`-E;TP0yk&rpl)|Ux zWRFLeKjuh0_)*>9y6P5Yn!#$P>snpSF!dD5!T# z@a*=-{XY#)79pGEw_5IGUta;tw?#!ML)-8JY0em{ySIGr7`j5<`zs32oXh*2_J-=t z^*5N0nHh+)gW#HeNC->3S`YlSeSG1)7m&|F)wlC%a+?F2WW8o3JS;&ft)Fvoc2aa= zB=z~Kv8+U*lFeXHzQELWkkrX9?LPlb*sH;T5kjpSrP-@$u!kLMfCC7E$`b-dJmrwM zu__T$BD!1x(_=>l8yQT$oVygA@6yD_2m{t;sU}ePElm zf>#?%8|bC%5kVG7{{?kPr@WYd>1FjTX8+MF^&{zzrRs`a;zcnlHnp>iA>M`@f@H5A z8&?s=v&26!V(PQDErH%JCIIb*IGMlvC_id}w>Xsn^_~43Fn;$`!SptjJBRl>T!%>r zA)d8I;N?x1kaa#W4_B`Y9?Ar1s9I&CE}D49h%n5*>gFe4VgU{y$!gqD1mQaypq0-e zRQhsm(@^GgWZ-dH5$EBPX=4PPiyI002MpGT-OaggcMv7}0QEQ(U`lb~^^o@&09zkI-GhIWRAddMndteljyt{8?|4_i5n5icd+knY%Fc{!77_ic^acCsy?)`I~3n1VvhV(MnNhWf=(tG#_v0XU= zZ>9gxg=8is$lv?*@!+(0uw1_QcUX)Rbj2;LlE!J>{L@#`@Tf9>4s<_)?ET;Y@E5er zZe;8$CHc&tB3^~W|4IgxZ^lz4 zVWK1?W%-KuD=3c%8m~!k09ydWE|LAP{Kv$Br;g6G7f(~P8SK3 z>`w*l;q0gwO%IQ&9yuIA7{PdvY;D$VYUAZS+K)75CJorzA7AY6?1A}|!op7~RL}Kh zr9uxUEN>Jhi05h$xcH&y`#d;-zU|Gq*}3D5x=?A(TMvL%SNin7khk-h7$DbkU($jB z10l{vrTL~HXapQt^wlH+74-P=dh2MYpez~dU&X>nnAMLRyXDCw%c&;a=txwW=I|s` z-5*{HGjUSye3yyoOYb&R*U}dWFA5wQxw}Y}ln>du;iRu5lvB&lNh7NQDdA3&l(Ep4 zEMW!(pOqbw+oq~Nd{7um{p#XEH1$hJW7guLJQck4))Xh>h{Fouj#vVTg8sjYhh6Hw0` zYSXx(pC!9NxNicErh@OiA8Evd{{~7B@&fvMd*uF+(vyAkp=q2#CGdfQNEjbMPUq^R z^1en;bURT`{}yTYOH`f(>A`f(91ZPhL8PDz%n0&qFY>Lu`S2VWop3CB5&b5WZfStj=C20=8bzl|l23~O!1x>)`if~cDo3Nnu{qT7~^N)n&N-8W? z0zD*+0ztGFC7gNF$2{~F<&1FA8lwt0$0T{DAUmHp?V!>K?}3lmaXnHN9z*L2TjeBK zS#Cf=qgN{v0z6x5P4zh`_O6W94R_s76MsD0U%$CKIocf1z&tZ#2`FZet< z+)S>aMEt&Wa+C`g>2!C(*>A4!>EZLwO?nu8{3vmn`?pgEuHJRZ6Jq|AM8OXy$nRZA zN>=M$!J2$jmOgV0_Ol$97B)6ioWozeP40O6x&u&awBF7hRGuE0?>JH`CBueupMd6$ z0*<8%nIozB58y2}lh13eGmIwXMp~`pjQ~@xYr4zs(R@H&;HYLYHRzy*7DY#LHTK~LIHmgKivI@rhIq<+1av=B@vk~M zH!y;7#g64}t?ljYz$U@*{K*0USc?9?)XCM>TC0g#C1Y76dAJKW#NUzd!vv>E8-gvl z{8qlFLY2>}8BZW8MSmvQ&Udp}#@Z_L#fO?EC6~r)vJvo{VIKS-Q7O8TMq@CB;^Izk zU0LyM#itK*ii;t2j1yFb_8yD1UcM3GcJc}+^PzC0`=u}&^uk8QDx0ilQ^1_Xh|g4e zMMJbJf4N%*{OsXIV4G3r^=GUjO1j9k);;IO%C=|u_Os39N@h?^==*aUe{`V+6B0;{ zB3d~0e2?{~GfuNPi*+lHJ*OpSEM|;wB_D|-k8A%uOYMfB!h9Y*+Fzqr`K+~-VR~;O zyQD=1kOh%6GR`53^n=QD(ex zpxk#F53tE@?4@4bY(c!ax~OZb8p@Gz`BB`^P(TW1wX!YUZHF$RU3*^6A zA4I+feDO%CnVv;@a?R%{n&$^QxI2S3C?ksiNh>=@t`b2I^D?*Auve+$uh#h4_7HG~ zRy+4zb`WucEobiZ*PI^1pU~J-LD5O)%~M9zJf{PfEr0d<;`)LMP=m|Dr*)hdVuq<+o}TTxu{p zqQAbb@6Kk)aoUhG!{tx;~{%zg(rUC*UUd-nP~-d_Ld-Eu0D zDhdS+6CJaTt*n_ZGjQds-X30)0gSy?FROz3=#`dmE43{xG7pOqgCjcn6 zxm1p3vsKDucL(8;GIkfzSl4RhpUrwIpl(wdhx^t!D`4BEXPz*f54f841!ZA7yofYc zcFR%OgO;D3_jQ5ZOaCIxd3YTDcD_*)g8zHkTzV@OngdCXGrjM&jCkbGa5JqZ9J#b|$LsZm3lD|!+cB6G~B`Foueoma&x7ca+^J?O`yQ==m zzBAY2M>6^)<=^&Pkou7CJuzPJQq@AJ_yKTWK0eq3*t0=XTYNrKf+C>=r=mStZmile zwJ|~CHcrEI#G-g8B`z4JAFZN5Z7n`kEDBV6 z+&{{5sYkdTxKFRyb{yUM@25JDSvWF6FxLaE0CwZjjZLDuY-r98MzdLCVAq<)QKnqD zZ_@Tpr7zaD`41Gj>Si|rC%|5oAPrV$$AUEOBA`?`Dx6|2D{G~Yl^jPl)4NH~mH5iNb+;-o?4Guw9vo^~ ziT_LvNMZt6+d!{sP6&L2J15@cL%Z4;IP_W-RQuk~p1iQ(on1!ipV87AwzjY@_1S6b zo2yfawsXH;Rr-dL2+|XIcc1@!xkmH49 zi%KGEGizSmTaYH|jay4Un(b1n@}5zs5;)to=S;7eRCw35m^+da7Nn@bHuUc?zD8x- ze1I|*w3Yb4!nmzRMv2k?uy7!7bfl(zA_R)OEsZf9EtONabUOY_MuU(ufqW+1`DDp4 z2hek5`v&^p+SN{X;Bsxv8@Ih?<>zR}B8B_<-mKrWyjwZ9*fC2~U_BxMpAgP5uE^!! z!-B>2Hu!j6ZFcs%7q6G4=X!>0iW9kjEeFJv$pE%ai+hi}HilY%w zS1$OdTHmWS#V=R5dFa2kYAKBiRWa@%V;b$US4>4`0vM(5mjR&tfm~|M16+wnRh}Pl zcen;dAf<(}?Fa~Ft$kfyF7+EFiRvO@5<7S#GM4H$s{pL8f5=x?SFz3hbt6_IR+8J2 zo9hgM16_Tbd=lKo#_?RY zMB>0E*yU6M4y1)2+Xy(SAU2X+q{m0GyVC!rnU0xA^C*E!rESJqZxD?-lXTNO)~oMt z7ZCS6S_G;zPZW*Dz1z6L{d-q_I80Fez;^J^r)J9cqZV65&6tucGLY2CRVRyxSM6sI zzvI%0d45i`RE!RbJtq;$6Bn*3aF#d~U?X}*v^8~63`y|PF4;QVn=Y$mXxEm$D1O;^ zf$8<2Kak?}ND1!{!SaIQpYSDD=J}2KE?eX27O2uRKToR9&FUp+5l5Bc`(Az)CzDg| z8Wu4Dn{ZbBWkL4|D)@7QX>M*A@aa-@5r9LL(W>pV^;kjmdfqEySc#_@KEW$Nsuko6 zemW`BWHcH})!tOH@`+ug{yeZv|tSAcjS?zM`i{Xjc9! z&YR8(p@Gk~!~NNOIgcNL1_=1$T*e^L<8a#lbaIf=2$%HWw&q=nNlj`{>`O0WjuQ3nMRlifI>)k1uf8jz znaDUY!Nw~3aCd&6#)*7xZ4l@BIn1+`ghCGe;8b#bkbm+s|K9T^pPAISIA4FLokvBj4M$Knof&y?5%T)?_oeDy za$;iQAia;}qxH~`(tg-hib(Vs7RSni9=IXShU&qdn{2o0Nq7x2O?1Tf{;d$tP+py2 zY#!{`H(4e-*b@G%KW^W=T`?ngF!!ghq{gM3Ho)(n1x8&F6yi2DrXurKaP~hEXDsZ! zEkSyipsmW^b*vL#;A{h(x4KU1YDRgC8Bt<)|E4%QhoZ>%yx}OKPeM zMRN{UXmG%y90?4vrsLgY1&DyKVnFnEZbhaC6IhzfCv3P<{Pi?jC9`6(joJ z_n&kidHIM_II#@F`K38vwzaI*yzN^bq?qV$bD>l`tByviWy}N;j9p8^8X@fCP%jj6 zN^w;n^3QRh96+0uPbv0MSY4+&G`GzmN7?S(S3^{>lKCTE$ml>*OyXq&*5C|W}BO5)B z0lsf&@H0P0W(VN;E~^yx7>)}EAl3De%o-qO4I;SYE)8O-xZ6qcrT6YNm;+ zsR(alrkWMJ@Uq4Ev33GXJxb*>BsI7wkb>8o0opANfjEB(lz=Cjr_D|%mq#n9(Tiw16(r>xd(@O1}c=Ed8SY2dd|$C%;Q z*q`m|GR~QnX7>Fh3=~ly`FnTXERpU1?J;wJ)9_OLmaEo$5ABi=-`)7*w60MACwQW2 zZ)2kk{(gG8mF5Zfetjd(_z(d@<=BV4BBDmxZVf9@w)lS5KNi!Ljk?{0$&Z;;b@p!~ z;HQIq|CtlYg^0(BfAzIR3TFl>gLhft$HAVIKy4hl&znwL3z_qpNOqqX_Bt%pyQghB z!bTF&=YJFzS3l2fYSR`M*EYM~!Qf|^+7u)NG+7#r18Oj-gwaX{qH>4Fc9b%pJKydH z)9;m)vVum3q~Vpdwa0Jgfdc;`zpsaTGeB0=b6$|_H85jKwdHlhkIN5a~Tp-{|4={}+=q7L$>C45-5Sh@DDL`7VAgWJqwE|E+%t8`Zi{!W!c>P2m z(5C89zB}pLBpD)D7;y>eDH|B7lMh^sNr;2F2v%@D!(u78u8}87BL1n$Xc_v8$#P#b z1b3H!vj^iBq(owA%^a`YXzwY{8foV}SdFZl03`ik`+hjJP|NIWQ~|A^bZtP%5M1f5 zX5X)c8<%TpYOZDMHty~1HDTn80Z)&GqXLZ8s9=7#{POe_p`=prXu<5|AKh8bC_+39 z0p^>09m{og@%P4~YxoqGD({G?`}%$S_b#0c6!v51ZtSNPKCdqk(hX02^Vd=oKSLaJ zV=~V%<=Q(u1rmbE-X=KRggpCn(MDB-8v%1=8{-~j&G{gGv!WZ?X5AV5uR58dtBtaEYJG!Vz@;UEr47VAa#Xv8zX zCXbjNx`mb?GFDwM+!Q5k!7^%*xv#{DKHD}Y3?kELqXL$U^L>CxR_&P4%2~Ls99F1@pivyoZ7qKX~HwV6u24jTw7lYP~N$9S?a}T zp6u#{?75Ids@+N%gHo$PSzkQS&~#*xkRbXQuMjo3(|P=u1V{tz_+NIaxCk<1i#i3S zD9xS*LO-USF$_CHv!t*6`~}LyiaYPHQirumQ|A|+sdOy? z$~3jHTUg2cX65D@gc-vvn-0dFzaJ@JKVbRH@8Kw)QbvROK!XhQnT4BlcFe+(!K$Tt z!d~Vrc6g_y)wEd+ezh87ROkq>@48=)E?aN3r@EPsOem+5{H)3AHBGCl7om^>h`moM zpaW2_=LrDm*NRSXkQ6+#-Yr0&1Ftd{_g9EsS72lKsv6AzC8b$fn!iA_j7;@?CTMcS zw;Rgm&{`|IdR_Vj%RkcL*yO0S`s<&E?XfRs_YvAdUjl&m*QMTtqrXo;@%^K8OUbY& zA?DrZI;5|FK9R?Hu;77ma)9XBWZj~}YRZtas6_B8EdAOQGOC2KngvO(fD{SI5{F;y zz;>i%Ra)-Zs65RYP_C{P)>>ACAKjhgg|eDsotk?Tm*Gc>CEsf5V=>whOIQr56X@gc zotU!|J9oxM(*RxehExiOc21NlmoVHW+qI>DZnly?^Q~{ryX9U}q#!Q`h=q42U9MX7 zSvK#Ds7`J9_+j3fvY!e-)#QHH$5n|c`r6SE70j_mz!>%LSBMuDGUuJQu+AS|8czHa zk&u7I3L!$csqf6M$z)=b5yn}76)>ZyXyvTiucc;OWwIO#WrN^!s3RzdjEP0wvLueh zV8Rk>2Xr4QIW#(xPd9x0^yjvsAk+ePCZdQamyC%mN0X8`h5O*~b<{7~BCC3iM-V)z zIJRPVGwO}tAZ2M$kNGJ84r*(A+s1_;IkqrO`^!i1Bb{-~-B`zRJ+Y*8974t3yr<%P zWs&IBSkmFQ7}v0q-vfy*tCM#EX1{Eny!oGQ zfmTFB%Y)h5MM-h>Zv3ZcBKO~c(u-RIeU2fQk#<%fzI!)VkA=<{zRaAvLS zwHR`o3Y}lL_2qu=&SQi30q(s+S&kKYGFNHNEDlMgil1tJc*^b(jfE9B1Vg6O1hSl! zNm#1P>Vm;*mzLSR!#KE3zuOKF(Wfb=IMC&6T8z;sL9??xnY`Ya)Wn>e&eRDX;9MA+ z!JJoSEl$+sA@VnpQq?{(^AH1VCfRj>?eN$%Ul`6Ku+3(Ee>u%ZfI~GAqSiKY&md*Y z`MG2+2<~Du=UFRN`@nl*^Y>aA+|WXIt}C*1NY}Q?o9^p@f*qQ?mZ7F zyS2%1xvr?7c7A?^s#Q$R>`fAjcC2Bga`%}>vPh@e5it$*2irm`nMS)$j6(jM-wNw| zbXvWdqTKBUkA;Q2L=puYs0W%j`<)Q zna2&#nCUV%fOO{GWF|#rc-B;aisWMo>dkYu8xe$jR|E(mBzSH|h}hq2*|qt7ZAM{Z zZRgEQ_0y8cmaUHe7WU>AwUPtWKGj}yAcKhjfs1_h3(Ei{_J z^u*|w%P_6c41-Z@3F(~6;04ut=}ih(kjPaMvAWXWpl!GvONFL`t%->fj6J&$6MAh` zPE9^G>3|6lT=ZIy6@&7{=bZtUbr)HfBd%I{tN}!@&9#-v%8Rq|(1nfTxt14i|64eo z9XNb3I%|fh_soW6CoJ)3fy3s{bpg|qv#DxxeZ5+_2t1T3cBN+V+CMk#=tX^COA6Hk z*CLm(ix(TZWmj;fj|6-BCblM0Dw9fhDD3X{1w9x)F&brKG!AK)Sn?lx`3aX;4A< z34#y*XX4GgnCqH3@jd6hKX`vEc3J{3~rsy+lD(*H{f z29*<75JgE@hJvv48w>pCh1>vZ>*}7)wX(8Uo{7KAuGk%SKZ$Yjf6n|amgiiF8#a`Q#d3S%Lm*n{=#QeK z6KZo4d*??-M-4pXJGi!b{j~T=jCPoeWU)N}UaBi>gMQD2@4erh9*X(+Ru+Yt@1<^y z#q_BS2REO=lC>!z5cjDK-`-ot+Llw1y) zyS5@4J*FHr(U6~;xX3wy-s^?h)2&|VOx0?Hp4}D%0F*9?0C3XPX2A4>p2xorv{rFi zC}5eH4aT4mP`gmakM%D7@CqPe{)D4hI+Zsz|PN% zOslx5c3L+6^3<^gQr6`vXLHDmXcWfFGzvVj^K)iqX06rrs6(9#qD5rU{nD5$Klu7! zcXxMZ^j+HL;^iuD& z4`=$B>0k}NE$yV275H9LvyNxHoX_IGjm z;YQ-^^50$jfifd-y)BD+HBQ#pwb61*a3)~89AUw5eRZl~qapkB7K-KOj{=h(o%{q3 zSk_|@tF_+cg&J}0?IvP-SgiH}&-n3+EjSFP=*|>77FrTo#Hm6;x}Qj?1#WL^jPNnBQVDjgz%wiJRhB^e+sDFTJzRUXYgPtI)Qkwei~Tuv2U zF|U9MF`fZqWjZ6azxBP-Cfu0FW}GM()_S@aF0Lci%?|qK$R6$d_%~Sk>X;b;gznoO zn}<@vW~oeiGYGpbT_~bz;6==_7{Bu0;?L;4?`~3AAzeEyx#_3>NQ7soTy{iYI8K*) zOq^#dEA17y1lA&v;$|JEeNzEZUYG9-&;YbS9$rGi)jwILme~SQ*L|*j zgJCQd1)n}I;*6^`tI?DEwo?Br{Ih`CoIrYd`qkN%S?vv%;odoZD3c|%7oxD+@|WC_eOX)w9xw^qxOGosn~`tGxnJveyh7gHSl=1~p zG5*`!Gmeqce5jsMf#oxNb`%V$t^H#{l@YB?MW34UIblkO50xnEM6^2BZj}(7N#`a& z4??80N;Vs3WC#C#8U;e0@#K&ZlSSGr7v~-g%J4g`7?wX>B~9ve;%*Ef0$RGq2d`eCC)$+zw6-^WVq*iaHEhMP;IFD@gA&)PE;1*XJ|a8zf1>yC2I^9%9y zHP}r6q2EV?@QLU|&PB!YZ8EuMk=&irJ{0xtzCmpQE}f%#i6^&%HsU6Rx9@}QygoKw zJ=r{U(QmGKL(8_znABTOvDo%fsp-7OV#j|7VSTnMgmHB3t^arP=xW=c{P_5ZB(ULo z4nFh7Se2I#o(Vx!CqsJ) zgPG&N!yOaVuPm|^o#)N{O;^m~n=1jXf%ZiGZL1Q~Ix6G44G#2cq;auLI|W1Yt)ogG z0sNE#PJS3AUkW8Rkw@O%{h>w#B<}e!(L>mU($qnr&2pEg=OaZ-iF9w^DK5Be5^^)C z$t|-qR@?Q_j;OqPxjk@SI5QVJNmjCb9IIY@8X6tZ(UZKo@C?cV&LnMBjMC>1Ur)tfBO;jvv{ks|on_vDs6qV( z_FDKzfyzghRgK78?_@?kD`c9^;88!%-Y|>6QuNt8djQBi_2a$`0PXAYK2V z(MJ3t%FM_m0DyRJlo))jkL+Hpt{t1X6vbeS`Zvuh%Tp~;NC;%lHLdZ=xmkbJl7YwT z(ED_$}#O)O|MY?79w`yEPNH?=ueGAZ@;#pEemE z@cgSK{p<$4@i96i|7ZWnYvZdZ(wjTy8|UQ%^N`^V+pd?>L-C_KMVv83Zc}8NyY-VT#l5({ zYqE__qXdR%@R@|KDhZU{N|jZKMw@?h&=uC zUiNu2-CqaJ3cQaIKvZytWdQLQ z)ow~Ftko|#$+uj&)ttsC*R3}XHtbQte@}nduC~mFN`79hY*h3X=}70`$}Ylo-60{Ho#m>L3uWL8^de-s0*dIZEH z=Tl-O8Ohj(hU7#fSQJjtvU<3KieNOK*q+H~z4N|I%TWI;9ttkhNe5F<C<-& zhtCF>c~34}%Ew!*kJW{JeH9D3A|9YhKFeOlSAcL$-)u}9cW5>-71wT;MV`7ah;RJ& zjOXkFq3ghB2G@Ym&ARj5kuz~;+Dp&!yrKMZ(=L9C5{LLK=j`5EA=BctVc!?P9xS^5) zD3q8AM?@X(QLLy4Nfy0w=C~Y~q9)L^z&JR!u;MFk8yhkYC>ZtxBk{rPT533~5oMKn zN55aAKj`f`GRgY>{NDB9zTj$8Aya{wjN1(4dm29POorVQIVo#YDqphu4vLc-Q3`Ix zN_U-yf@UtW;gl(#+EB2HBJK)4fEx$ErA#SaoS=>)jN$~imcO=EIwZ9!hUf9~*JNX1 zDbAV75n^xTRJf+k8TNv5c9#*+D*GU68Soeu2S}NEG1^L!wQme6C{2b#xr_cRvlhwW zSlUiCP9Un3h}@N$BDq32SuF`AZNnYPr(S9TP!b#{LS7rdpLY4i$DTm1xey>d*65lS z9OUckYv(COziVR{=~Zzg#*xcBQDjf8s(d7dc^&sD!J<3ZUTILgB}}aDHQk5K9Ow(& zcGFT4*+aBUuX{Bu%-v_syW~saQ*^HXv({asNW*5GT!Qb6)J*MiCD-?^ zSx?{C+Gnj!^OBKxcIfcyi%3ZY|NJ`qI6rV!IKYl&<6q+=DN=El+5`k&Yi)tL+^a1z z77Oz@*K;%)owsMpoVVv^n6{Y9%0F8hLY0~g5{C(?Syi{LoUalIsOg|-fH;NXPu4g< z4U>PlVlx3F3rnxxc_ik38`)~YYZaIBsf1^oXPjBm|6*jm~ z^eHr+L7`-{bwM2Yc504UVkqUq3WtGi!mEEjb|3wl?ESlylbg$GNrgih!O_IF)3&RD zR?{QrF!-_A$UpeEfyW~~3VhgrT|Crf~j^UtYVaWB*9+Gi`daidm=O$I!R_oJu%CC}1X z$c2`nyu9zPyPH4ulvY`B9>%Ca_clmbJsRQSuh zF5OYR^K=hx`d}+1DlzKz-j&Ukyi$Ejt}gOgIqf+w|x!Mx``9r ze||P>Uc0n?(d0mXcDud39VjI&BJzCja&B(U3(dmFSiSU0E^2$V&bB%`i(7G1kc<8Z ziTeJK#>xqMJz1qccaWNSzMgI)%3W5rS~EW8eKisAofKtdkHs1ZKETa--HgQ^9+}+* zHyb_uzGP#VY=0V;tH_rt!s3AoPfPKWr$m7rwB&Gs0GR!6h!;wXOk^tPE zAQbssWB?p%`&V^LN}5PrlPYFfbxB~N>`uWFqxQfrrl5eUIds2UAJ9VBKFYul1D10@ z5g%)z$GV&b8!-_iQMYn0B}u@WHk=5{358a))yHWeONb8NPX%=g_W0jk9{P)lOC-pYMfqvD)h4&QdwpSU{m|iM_mAnP>8uug zhSn*M1|v1ClF|C*Z#6nfJP3j+nKB8dU#RS=s@0CTS=>M1%u1*7ut#C+OK_E8$Ia#K z+0Cvv;*_Z^(8{?jsPEw4di&mpqIsXrSz!B5u6-g;d(Ci6uFU3SH$#hCRG# zuxW`g30M&hY6fcdcblSW_}AC3mcj1#is7TBrP$#jp7AS%$1oTop1R9J{nKNwL;=tf z%0#(Fb`&8VNemv~bTp?v$RV!@;C){VUr%CXTTEzU(6xFD3Z>2w_0I^?mr=vPEqEI0 zI>XRs+kXNTFFpbx99{&3B^F2)V_HTeP7<+Dm=9XsAqZMkMo>ROgd;Ai0CjB5cthp- zq_VZwP)7WrR+6ruJc1&OTm?h-Mh2x>C`|DL*(vxtc3C4pLCF z@#bVS)S-Z8KWRN5dM{_3zs53dCcwM>RhHwuR;Ac@AS1<%Ez2`|%G`Uk-MHt9y4bw! zL=^7CbLN0b{P1s+_A5{~-@)UNO(#a0~N~U>7VJ>~ea2e!{VM6{@M{b2|{4j0vS*0Iv50Nvg z^XNP)r3=dTVP1Ljo2lcE)Zl`&6ufM^QSRjd6VrO7_uj%HoNAx0L4$*eO(9_{%1Tc z8A?7}8U;Q$V>296P@_UY(l>1JZL)zYXtLs*e|Ic54&iCwGH>)3x!^W-R=&3Rb z6V6k9BGlD?q1`eW)XsUvHCcsNQ_+qC{hlhJ%pdu$7{zD>4s833`2!h7rmhIZg@aJK z7yzk6+O{nK@;7)4qcSGHd$pW8G|NjiA#4a4>?mLPTJQIL5_Nru|ik2xI(nvmyZ

%)~@istFQK&%Oo6uebbaxm$jl#iXp>yXXz>wabe}+CGXw_w*Q5b9dEDS{L_*fNQv?=7xf{vQLT#9;sthwAdUUw_**)B(UbF$@U)f-`4 z_@*$GwbN@-z;&cur(8;AXww!VCexLfnv;_w5pa3j|CVqo_~v9~Wo48l(Eep*q06dZ z7ZX9B{BP2B?S!d%D$0JmWN2Ci(qtmP=lXkk?-TteC`oDQkn>+fSnRN+R?Sj9@r22L zPC`<=zZjF7cyENDMKHcgD|CcdM2B@>+=gdh22+Qgk)@1oQv@a2hW(t&c{JWgXnmc@ z0?N|be|5>b|1c>`#)>oyV+|rD%A9P2@EdQp81*l+;{&Bn-(krj0Gu$&N~~}d4W69e zL#@vxI9MEFBfa93!zLOw01@W<5Ms4WBR2my=38Yki!2@*j1N$0JMUN4kWLBRVgDi|Ue$}mcZQ^08k+54Gf=acT0S1*_^h4ARjxw*kfo_ZPN5CAQxV#M>X zam7hY95sYBF@5@B)Neb7fkic6GUU%)R6e|a*wX#1pUN`*Yh>l&qhWAT)QhqkkAGu6 z)hOC8%KwhRD^X_GB*u9`BKJFY+QE%!%n6gP#r}k|TYcKDz_b`dYt_H*6&YA-?^jGf z?`iN|_n6dSP`={z)T!Jd`NYXTN3)7Xg;wbstNFU*ZBO#;wH)^fe%!5)adq~a^O{x+ zlk3r$v-l8s!`XafK=xwYvCg08^Fy!O@y-_^2M`v>y?Mezl5zVjhap}8ji#HHqK~(2 zTp1o7F213VCHm~akSoA3-XJlZ={J^%I~A6Uw=lmNC#=@^@&yKZaQOG{*;sFvL@zix zcKB=mErqs<6g(keCE&~>-{zS2Wj?>wY2YdUSaB6+<_^rwsws6#kHnhjx!!+XI zH32>Zpv1CKYeS+rinJn1fGpW~OjxMB3eFhglQ6@|Q5119295?p5=T-DbHoBjG>J9m zL-8%3G+F@qT*L}b)=~jxNz?c)_44l<5-^%aFFc~^l}`fbiRkYOz1k+NQf5*NHjZOZUpr@s>XBL$Bqw$yi*{gzKvqg`IL`fM)u7ClmE74Xw>qup2k zDrdgY{^w@!y6%xU&&bphZw~uCF_-Zbw2vR%q3SDD8AqZIxsu88?>@Q zV!8Bdr``AZ`C0eX=8{>+?f1#-;^N}p{wJ9D{wVnO2v-?)3$G0!U4apfMYZZf+AOV? z^GCnF{rIA3>Jc0i8qAk@=~#QF zCuV|@a^+y231A2Fw8c=;_OV6chagad5J|{c5P~_nxXh$_tQq?~PqA1_Cqn(Rz6M81 z`ZF`zKfXfZ^{bZDo3@eHXWbmigkK4tKka)OI&sT0NR$n}A` zIPm%8={mE0acV`!Cr^-rIvRyY!-HDC{N^?L8w5SLNk3baa1FL@V_)~4R?lsl^rxs= z&F^C9z4{_TU3>Jjp?Hm+;Ks4(j1}AA{@3UFS3>L6fB;=M6SIn~Ir0K4mOY6oxi>RE zoaE&xUv0h{-5Iv5H>x40polF(-kYC!dw&9@{(X5%(Q|!R)!5iL`i|vU`uMsXM;{_- ze_>%EP|7vyd!F?OD_=63Inp~A704&v;beWx_wD%bE9@#0hL}y|2SPc@I9Oqi-LN=HAj18OCqp()VDlvV_{a58$o?N0M;m@e&~OF+ApQ^@ zHx!l(Px4tlG*Tu`x%c&*ovb3wml}1J=`5WWdz8j&u3nov9SDJjXtT@QXWU1kqN1q6 zg`eFk!d6rXc7*a;;JOH=-05_mGs*vc9ob!-N?%`W4~kU|8Kf=1r_Eg#2AS=m&#VQa z>ZGdeh<3$atPN@2Ea8h)DaM9v^yYmW$U+o6Pw`;1;u+UQ^GF3q%MMCi=uZfCg&cW` z*$^as2|Q&K@N(n5;;lLon^yA(OX$9$mmNxE6jvK0YC73uUlYn3+}^)(qeWVWto;aG zRL1Fkni#w$%n;tg4SQ$r_I9d0wQu6Q-97PL`Hg6XN$jEXC+}yZT{Kh?+N8C(VvP$sm)wxkIxx^?O8axnKSB3^0 z6<#q=!Oe{g->>TpVZ;>4(x8xX0?*DflApb1RfQ2#)9DtD)lE5!Fb+qAr zE`#CLNis$b%->&H;(3LWC?^x)`opVt!gHVO2+RbeShaB<9iH6m6vz6r!|-=&L;fwK z{7e$~d9ZaSCf8$-zB)vWL+txqt|T*2>H8;LUrcBhG5En`m)J!4Diq2*d78DU4N{Pg zrRLx}yfqI1eZYZ-;(@{7Ii*~ziqp;@=CFhK`RC(ic6KEYz|oiO+$l(gLg8rv@-&*z zjcVxWS3ywIcI{A62Zd8bf%HMlRIHxAT6kP2Q39Ao7CbL>DJoON1px8=hYAnMaU3I` zSqC$RW3zag5Mj`|XWA-Y**MnHd3&6AB~-$Da3UU8jzdRmD)57|aw{%{B zY>yWtf)w;IW2a${P=SGfus<3g@oA+31iNA!cL}tJ4ooG^J|O}pV$FExs*33&J^sYc z%7!SgOCx$%Amt~E2KOn!NkS6^*exP0`Fh%3ZvOdVkx9uZ#*u%n?A?@b1kS3Xd>p!O zMY8KJDJBx=fe86m%o6zHP7#?P53(1dw|Y`yuNG{v#=%4opc+Nsy}^@BTOj!Kps%EN z&U2J!mbB1`=&NKhX)8%;S{3uJ<5q>0lXS&SYl5engjZyy-XqLGtpPb3elv?;Cyy23 zO~0MS8}aQQ8XG33OS_LfG^^pAG=gtoKVf2`OvDxlPoWm9Qo1lS0^ z83<<#eZ3TwC-9J3M~$s7ljCHOdlav{RXd}NCPKdl;*|-VJB0))mr0fE3c}XxI@dV4 zqn#a{InApy(NT^IFMoabjE~tWBWa5sFm})|?l$as0t+RA`XBi`l4km;eJRhTw3CcCyl2G`}Kvf-V@R{q&*gzHEz5#-qLwu z#q_u0}W-<-qYmr;(n7SDcCs9iF~Qb8}~!s*h%<%nk5{pWQahI1_7x$Wat5?gwr3fjeEDK22!| zPYt;dcL%het_vM!ZR)yISufA!nbe6RkDcC}Xjp210L^mh28{`B`J*ea47NNxXh{qT}F*Kh=({dNn#O_)4 z;oT->kkc!!x(7L;>v>U)Zn(GWCDsb(+g+9L=e=a$`S4KAb^HfIlOqKfC5}G$;DHP- z0UzUW1DpAS?}?zy0qPkM&(b{kx^tUxk_t#@CZz-6uIO5Xut+m*wsb>h_fR2omy`f7iIG;2a>xniz&OtI1L^49A-1O`GUDoINfTOHV{tR22-4E zDS8B!ljwKow8)5h<*D!yDOfDD5M>{i>YaaQDeWh>c#{JFeW$H&Y{G>CMFCIxF15GB z5JX|p9mm(l05r7mn;neX(My{;M!V+4^ZMj-ZlJmdtK4Hk7`zEEVIE=r=QN_a_tlWu6r&ivw>ZsF8)gmDvU^c zm*#gMBJ4;H$J-bh8j;{Ycfk4+2z55;-c^>y6&Be6-52|;{E7_)90-I)XexHq$j@rD zo7`;a(GArVxrdbZM-j}cSQ6X)&rc}UB*|s^s^L#<9{@%rndEbzK5M*nXVx!G59aKP zM?$smOXGjYKc$*s7RF>|FonV{3WaZkQeZIe(*$HFgom~qi;*w4gh46$;!JUL2U3&a zP4*Vn-~ZtLXBC_1XjNHrJ>@NBs|QmnLM0CQ(m(-i#=5ZYSgS56pG+P>=Kb^*s>5N>1FAX=#tJzE@OsKaM911_atXT&- zP`9ML^YiIf$_nkrkG_}Wq*hFBn~RKx!jw}SE7+AyYM0La+KpbRJ}g|yOyJ>wxC3!R z9}^soJ=BUGzFQMu8Ns}flh^h@wl*qiCxvxA-dj$RwQkyo!Gyg6B}LBHN~SUNAm5cF z2)DeHoUweYc4*hn+$xVK*<>O(BoLDcO>njy#;d`Mj+a@?GM!WkPRXrHLCBW|64P|w z&+4I)(kgUkocM*bKYK?ADbHVQdfS=Nu{IgvuE9h{H*c1;FB;alS zOCdg!v(~V)r={hk&ZllyU)}bW6^Ea)%rnfCN3AlisNp2E=5R!IJ|v}GkeG(lH>UO9 z65f6wqh*X)ZYS}aZEGHlRvmDQ zuR=?tq-dLxo&yuY%vzC}cj{NdP5o&2(7;iCPi+7K^IPL<%1-lE4yD3|#Ug2~X5p}<+-_;n z%^3wK^wkrPM<|D=r&1Vn*=XJPSoMYsWD-DX&wma{ZooPoq69t@M+*nBxV^c_@k>Dh!zFPBI8JO|xi=4-ES%P+Tpyy*~m&hYj4eaxC_jwhW?6dVM^uLmj-hK7GZ66@+##PhT?lhl>6B4cg02p{jG ziZHZtD3mPRdsFF!;2T&H&Xe~2K;|H&nW4`~1w7+r4km_~7IDOj1?ejp<`v@d+o3@+ zosXM2jG<32aB(zW;3^j`!+Cnx|2GHm5Il<{R*Qc}`cWjw_sI`zR^cvV-z#TzAiDD{c)jVQD{nC>y zVnqm9^V$Rjgln}HQWG(`{Gt}AsoFSNI(ng26wqT!m84=!+uNV_5Ve1|I$|{vMNT|9 z=J9G0=7bkLU+GpTCye8GOL_e80qkA)(CwEcL`5X6r8dIdy-KEeqe$P>;MWspnlXEQ zlo755n%W+Jm>WvaV9%+lGLd6rm`H}%35};WS)oZiD;OnLJuzpRQOMg&Pmpls6%Knj zEzo*-3?ynliY*rpqFum2G8ogyqjlos8up`igj&Q#MCjvryrly)hPa*kz|K@G50d@Q ztVq%5-{o*dMuj@7H&L&Y#iAm5a=Sm2_4Z5j`zt&v{l-Ya&|6pn5}q%vA?O z|8FyR>Uv~LWOucUHO?^Y>Hi>ftGIi$a({n+cRal=_we)SJ48&Q91LbTu7$;NLKqnt z!83tU&yMbm44!FSv}`GslRJ4rMKw2hEMtXqx+eHFCyX_Gz%SY%b+?k3+Ge7s<7Lr6 z_ApGS&@g?WgOs(P;-}@|M=syjsE?SEkE|f2!f@r>A`d#YB$%RnNke*ErE&lS0-CX5 zrYR!9rR4wpeH){i(2Z+QP2~JZ@ePxYV7`^5n{3DEm2hi#4}L zlM%p30A)+nDN(@MaxgV7@2SS{_Za+n+4-hyOhb)yizloI!NJCW8=I=_B#WtemCUpR zmmh!lN^82r14xxJP55Jn5kRBUcDEHN_Rib2?)^fY)4z_*ug4Ims8pNG%Jv+Oou39+ zA@V2Q30bqdSiz#EAo7ldZOKaP5NFGe=-2rWEhU2WWL;f6dBy@5{>)p;iBPpU6fp|U zb^n8R1`R8+8!9CcHlpG}cYwszi(0b3lvXMSU%vER>MKWMgcz*(u9izaX&2FR$j%9D zVA$Cke4W+io{NL=;jF`XWcU(dIl1~vn>SUlv0v{XaLckBK8x<29w`Zl3{{Tld};lG zxlW&$z>7cWW@;1(PpdCJxYB8FoE5&{{{h-m_)+uhHh+-n-;YQO z+t}^e4<3h`4;*J`AX*!*Q(ZH0R%baf2wnZ>^KbUv55$?dJb0D{(P z2cMm_c@Pb)MkEYQm7cJexEb3n^d7amBo(G?QZTD=@OAyb9teh6B(O^Ns2O5N#-g z?lUNJe?6l%Qh4KFJwkP6X+?fzrcLhK{8_@rNBi<4x#zJdm%#xUR^oZiTP3$TaeF6i zM#o+*-8{i{ldhQ!6xQjVRbFdGVGDSJEzBUKobf4?;Co_~EoxnCs8;XEFW!5A`a#d~ zp>)Pd0BkGfiy-!E0&6BRhgs*h8m zu2}9Zpn`qNdS4H(%VKes(3OA^YMpR+P%~D{gw<40$hsO^LgJiCdy)_f4}X}fkCx;x zCzP5_d<)m9A4&bNodrlD0-_Q?THyJa1hNdKpN$E>ZoW`GjfP>lX0;;jB}Ca*+QakD z2zR!fgp}Vv-VkZg-1D1Cd)yd|bBh5Oc}#T(OS$S%w!PUO*{Lu6;g$2-n5|34ywl9v z&3y0cRITvl`oDRPfV%j{u*rKARl}LV&Do>cN&a1Uwwt5DwaT?j+t0ha=tcE6bLZn{ zA9_OqS@2xD{!8$_oV<~Gwy~wsY!T5FiRZ7oDz@&EkM{0Gz?3d36za|3C_$UB1 zBFZ7_byh)TN}?enuKA$9_miAVN$bF=nLl0C&d!d&)S~jsuAo1E4;rNdFK?En&8Q87 zMI?RQ_DF8||VaM9Qk{ zyKhRVC{b_%Ql_vM>BJ8J9?rth*P~R>Bu9o!Q=Ubh0{6$T``MvF>E1T*?yzN23Ol*V_XEF2o-?gD zM>3h)Q}w){&?9p9m^L5#!Eydr@V)7i=Tsp4p>Qzrvpdq+axQXC_RBiL!;~HHo~pl^ zYVR@@AK%p$zts$&m-9?mVqMVRGrQgR_uu93?A?XG-}Zf9d;Zth>7zfs`_X3NbYV?+ zI$v@;M_N*{JK**8W0rqSoH1`d4GoyylU;h~ICnmJ2I7lyMtduLr-9nmm?PeV)3P&M z=MY{V6!|BFI2_%K(RThu5bjl>gTaW5{R(^F(R!q4aPPn=4NMFj=f0x2oh3=~$K?Qd zakD}mWU_i?<7mpD=a5CObRW=#*mIYj7*%&H#EeMdYQYEo= zdQANaiAeR2zi|H1fQyndTDAD?5|y>_AHQ(Vhu&GUUYeDt*L$gyFwT-xN<+NTGqR9} zU2a6Do^!lWZ;No0dH*{rgY-fzNW_&E^wU zbJsmZR0TtQ1fCM6#9VplS6pMkz3zwg3d1INJ!Lg{s`v1cb3VS3eU0|38We5^%bJ4I z0?&obPnO-F2`W&p4QP@%1N}M`O~Op;FPme7v#I{sCXL)Co97u*1)hoT3V+9)h5N0? zD6UH1hSBkCb@>>sAF}B)#LV4r6|4qoUI@lQmG=XfzHHy`3R$n+2NP|4Le3F>z z;zyb%la}*|m72*0!;}Up0AfK$$~V?((;19%-``iH3BvR!tRKzLu?{NHew-lpV@b##)xTJ)?=fD2He_=f1{>>a(3Nc`O{1llEl;VbSAVa5=oArGvPpmN~y$L4n&`Day5{YC?A`GX} z!)nrk^qZ-nJ{LQ?Za4~wZ62D}<;jLl0|i9N^6|lDVK&Z3-ZjI>k*j6(% zYQ*G8U63|Im*aUfgWD3hwlQ-T1K~4;)EX~PVYJbznO|t(&kMmZVyQwaADM`uhXQhWdxmKcEo3zoPH9l$au{0`dmjZ&olV~g{!`2n>>qgbU-4Utgz~8T zz+XRi*BZyqTbb_`2rav8DCUw=h z)1eO!nwj?X#oNROEjL1}$;COJ>!%xr9W#-V7WWCAyzDrejd^eiTZ_q)?-X&EZ_7Ld zoX*u#V8okbh;;q6&}TBRP*#%zkMtwYhh*Mhfj#TS2Yi7L+6*yh$yP8#YoX71*j`TzO>n$GsD}i5 z?_wmx&${i>ZQ5({-{S!Vyq}8HtT4S2Xz0LmA4%@QFcAB z2q{x#m@#y=%D$tLc%A*Ju$%=SAZq%PAo(w$!1=ULh`cY-NjsgZIz+DXb;lZTlS4+7 zVgAbS@;9j{1HF>1Wn4@sN;ZS`-dWd7=Sq#W?Rhzs9?K^rQ%XJ=SZ~0ap002A|A{CY zLvK&diyzyA_l5ADylvpAt*w2kijMnv{P+0RFTU_P10LOGwb>KywJr*dx!()F?IoUm z6s-F7*D{Y)V{~VwoA6wLnkTFr4l^Ii&BdjZIXdGc8YB&I)>u}x1;j)NF3R%@gNw9~ zNX4n+ab&nA@jc$fB&W(%vVR|n(ZP$orqnGi;UKt=P~>GqTdJ&9V0>47B}!8bjIwn! zs);Nuo(p7_zbC#}fb(Nf)K=Dw42KDR&gEd%<~Q~^9trQYq9WQGidi}|smJSGT zzx#VU*-@EjNLy9LJN|2XFrYs2$Z}g`uV24jUmKxp8x8aJYz%qR zUqISln&2X}s0mXX(rkcxU|Ee>{{9P-9jO#~v*Q%VD=HnQ6j9b0 zZ!<*SN#Rn}7FTdL_@7yrf3{SUZRlbtAcCUa>Z+8{lVV5Qi~qEFvr09yz3Y*+oicdA z{@Gfpg2VCbeIQrqPhV|n{DSjMVZ?E+sO)3%!#XdE~z=K zX+dL()rb&beT?tUN+G}AOt5up)u!R6+uABI)pzP+1$K6Jo9=oUna_s|vIXA)Ng_BC zTvzR8)wLFVI23AnCVRYvx#(yv0dId!)1tL)H-op)MCLDeL>PN4%C8TUd5GbTQQ{jG zQHiVOkAN5wr!X8&fkuwu)N6}qM9Z7_YH`)?KkidAQw+K`acU77D;L8?i^BDkPc}Oq=#;#8>B-Th7yo2krD<5 z5ReY(?#|!+zAyjD0vBuDXYO;(K6`&o_uB(EawqX0agu0EJQI28I#>8z(Z?h*M(B1D z;t$HI(QTAzMFdk{LLvH>*CFHV643gvW z(%|6eEM4x>Gz)C(D3ug$L?xa6xOtvPPj@d-%G4k~*A;dcIv`IVT5l+$(`hEbVZH{n z9Ao^TxCU`blIRHrjuk&8238rRs7!*L%;;8@O58$PW6_q{Dnbwy>MsR`@7yxDhti$& zZ$wXqt*spz_AZ?~J;g=zzhsI}+NJ$t(Iq3h;JXeK6*I2=3)?cBv>w=fe!7{p=975+ zVBsMyshky+nAcM93w99_cpSfzUp0%#dDT5xg=ie2bl%b6ClB&*BKM8h1LlJ+uL@s7c zsTH7Fsv|9?bPMvaQeLkHE?54j(-f2c>r(D`KvNO+4Qmk*Wbjv;0pg3s=Sfbu^Bfxy z?=cbeAWpRE)}#_4m+Vtc#Yz$t4$=Km%36A}{-+69$i zcKotntnVb29&j}#q666Qu(IQ?i*gMKu}ESru7aQkoos2uFXJ|;d#&X0ipBGe5P#tZ z??)i*tfsgjK0Zw3uC5duhiHunYZy#QP)J3j61l|X;^g|?IBj(e+@Sv4^4}WR1gtI@Ov+%*#R+A2TYD|60St!K%Gdmp zpiITx_KQOgW!S|}F^|ZWmP1z7(xI&^GH321(sDRD7P^$nWBvji`DvLP)p=u(E{^~r zk5`DXQ+&-}`##A!OF{0M16C3A(;x_~~7OuVuvv{2z`wtFIdhc{59Az$~tdOWt{yM_mey##z zF{&=2Y`1pR!CTPksrxlJMwI^db0!ogdVWywXk2L2gu}VaYd$(zY${b|Z6@0fdsg-M zo$iYBbReA!GK4-4Q2``HBPzOR23KRxhFW*{J4pK z(@Y22`k=Y1603yN%J=KIadlZ~TnwdtmauZM(32cQ|C!8gIeT@0=n4;4vgzwZHetFF zNtSod^S|MM@0vn&9Rvpt<)0mAPVXfzg9;jCAiLfA<^!bELh=00%6nfB!bs_Yp$)P- zg&5yz@Mgos(e2QI1hrCij<0S+Lw1kITmN*rVzxIVl6M_N`Z#Rn zH$F^TA6`$U5z3O`VF}*wefEaTg|kMEHrWa+ydj*+57>j%% z&&xQfnKIR5r=-^0c66`H}PM6)Akts%BYJFc?u@=NH(mtG(Nh zlXLch_seDlf`JN&!sX$iMU(0Hfwh}v?rR$LA)Jv0wohniUK$oeoONVz-fsMjqqT|D z68C`rkyjsc` z)*OBDVb)T@yHS&>CER~y2=F-?Jj%Ik*(|yD36T*z!6hTxOFvZ;5b9ZC_Phxh2iof^ z?i0v!uje`9;<|1WP(4j@XhXBw?C_F7(tpZ$v;^LBT%(63p{L=0Dk1(9ZGKPBErc%r zLoQX&m^eEelWpLZvAUMifOX zE*X=9G!M!UlcwZ_R!HYHQe_haNQwQ3Jrvb6!A@Qdd?ku$K#C&m{+b!wC}8)Nf01%t z9X%|}5&?JmYtCI!&R%0n_Jq^e2(I53KQqY4pl_ZGh~7clo*$SLs8ZVrPU@R+)U9yV zZo}l~r$27mcu`aD1e)j_Xuh3d8CQ7TyuV){`x47a$88~)<-PNxsQS0-+LAtjvC&7!Ma0op#!jNG{a^Xou6OUei#~&}cBH zPKVDp2#92gcfPDWKRv|8Zuv`^bqDyh>F4IvnR0RsaTWONP_MR`36Fmpn6nnipc7#b zD$<#aQX-u}#gOLMekDqeAdN_XKg}ua`bjpSf-MRBK1E5Ig+8iSUc;3 z+7sT2Mp=`7Xr66-toN?(N_ocAO^&`97)&BcGo3?8X5=#^*}yEOQ0d^XIuI%g4xp}E z9`u~xd-Oz0`xs2ycXKUQyh!i2)cpG8S^UA~g^J#U%TSLhB4eSFtb?Hxnp8iPDJ`#^ zf?xVs7RVjHHyxFm>L-`cu}sM%r+qU7-{OP~V=#OTN6bzsDN#>WQXacYO-@?C=2WP`hn!1f0^2PqT*=(B2b<{z2`ufIbGtWjWNSC5CG-cK zfEH0s8;7>3L?zphNS*IImm3IU7MK?F2nRJ*k0AScoZzauQIl-g^PJTRXA*BDBO)tr zxs}pqT~s>Ma&pQp6Hh5m@jI_{0(1kGm|-h7fB)B({!jN;#fphQW0v=-zXaf2RRbKL z=iSrq#x($-qS>$}*FANz6$%xN81H_R>*6e*+Dm2DP}k51dO6YydMW_w?(rVMXGhVz z6^X758=@c&VR)=RgkWwId^9*KFFKOnc=hkl>=ePt zhJvDI$v)_aG74>JNffe*YJ!1|%(Q_6R=@8us(Yl&65^3~L<2jYByYI+IxXm@mI5I4 zTIHB)iAQVee1jxR6i9N|q9y)d!k6W0c=r^edEYauIhSK7K{K?`~o z1`vLi!SR%(%u6@b;20$K@96LFHmZh$l|sMP4_+gZ-|*f^Da+C9J{|q@<2{^Ed-v9I z0H+q?2irRs^FmIBzDWtw5Nd2F7N$CF|8(_;gNdN-jE-l;KEAABHHW;_F(03zYrkbd z#zB*!^#@XJMR5lFbn2++3R`{O;)VEfjf~}b4ZmhS<38VrC+p+RN2mLClcIN_u{o`r z{F%HzJ=K7*>KS1#ILe*_4qs;9<*@n%=q$)-QFfjwtKxi zJ2d61mv8L#C&tVVpAF}|mo~1}qYFQPRYBQV_k7#f8rY}DSh)5ZIy*Z7ZWy4|qP}gM zKOD;uLfw_u%IqwvJ=k(hxfxBQTI~u-yl~X5#?~FZd3QcO)nOm?CJZ0TZ1*6j3%$ z{D&Mw!EJQTwbx?(8kf3IBC`SA?h^>Xrr>XiaYoTKDJFk}iI+!Bz}{bC9I=)6+fFwv z{B1S84MWSCNFtrZXm$tU!LJ%F#E$;97)GOmsKwCucEE2S*CkNO;~q1&77o}+=ZZ6` zm>BXQ!p+Uj-x~d;i+FxiqmQv;J@K*YHeS|8$ig7@nhqw+;F2G;usaKZ9#by8La5ww zNm_zQGBTBGF9n@KJT8l(!nKG9stS}Pn};AZWn2E^^6rD|@>%cqrkOm5Un8ZJ(9%3g zA=*|iMs%fa+V%P6hcXfA#{m1$=fI`Z)PVUYwG>USsOI9t3PXR$#%hgx!n`BHtNHj3 z?R<%SEfvs3*wiO$+l`YOK^Z!#s98UX&op7*7N?DT@pXg4C+yhYcd3qfzUOS}4)p0B z#8-SiM$=H|3mq-=a-N-)x?kH!l)T^VEmlNqXP;F@A-G^Ss`=13xp{xSe+AQ{-b17D z=jV*U|J5>{Uo-m&p(5L+-f)I)G$E0szWURN1b!PkQZPGgaEg_;$N%|pZ>s#@>i%qF zD0O)A@%)c*E@{cLNW-B=feS>ig{@JE^y6iNAnksAauusK>li=Hbg0WWH zh4u?BST-f!CMe4>Lk%!0&^^&)8U@zWt8M?_6YFa4Ia*(hmJPC zn3N!Z`DsKJEq!T1=pfiv_5pn!i}j74=&pK>vCYinRs(fw=(4tl7AQEvf5f01A%fPL z^B_^Mq6)CWxio#HLEZahB_gs6y)6r$00F#%7=l=Kso#qE^LiL#KgC%INOS4v6|(LT z2{3pr4qe`;4RYXIEByUvk%0@TEXbIcPrA(Ygl$5W;!PahKyVn|r7z$_f5SRQnb@|7 zK;W1%@A>ZYBg8V*5F0}SA>~I%vlt~SqDQ~9xjoN!qOLh_S;F3&y^ciU=El75zv5?a zYy~5}K?N|8u5sJ1i}T*8nouxiT`yHRbC%a{NB?8-sD+(HSfT@$98f$ zj1kfu51+dxey;*;Fs;pXmnJ?KycX1T+V|Ct6R&3$e1Oxk4hUX5YY}e@GYskrHrXdO zvNsyPUGUA4O}AFb)KSeeciokq?sU}kJiI7_hDFKILs%J82NW?B^ue32L(7v=O5crL zh{sHOf_}2YknMCoSR34nGiO9gr~G{w_tT(++sA6!Cry6*9kJ0&UGb#b%#Pb0YovfI zq4PQ40ReI#UsR*x|5o<#G=MU{*3#u0Uu=Jfi!B;XWb8=yZwRB)So_u5_g{jp-pq^{ zoV^21*XIwEK*cfmuxfIxftO4hVvfmT5=O;?&(lKifg_teIqzLXN}8-aWVgz*%`mbd zMN}y&ZJKq{Ai43PzXV+?mUDy?iBK~Gr^_;)qamibV9qZFM|74q$EL`9V@cT_Kl!Pa zg3$3@6{OY~+Vplgwp9!t+xg8L99>lUadD2E{?#D*K{LIM6P2#7a!c2N%}1}d+s?bHZ{d4=H~G0sQW60!C%3l%fuLKE9Fq6yg352t)x)7hc42E^fw73XIl1VsOI1GqS55$=a;2PKA-kY z+Fkw|q6xa5w^f^ZO+5y%sQM!DU=5GGc!As^!qWW!1*PRTKXszi{cf>K+i5OG-42&o zPYe&-rg6BH1$I81sz}j+J^u!wZ|e#W@~+%w*k{o;hCLwMZWPR4*&?h(%-}S+D?r1l z^bssVTLU9O2vs>La+L1FD26gkrV(pJ2cu(1B#CUs#f0KBGVVRyI);9)K@i=`8ZJ(% zC)r{q_?o6_SW6*x7>}x}291cS{%L1Wjjw|;cB-FdXkzgLOiUHG;8Q3u9&IsnaEWcb z`dcD=e%%5xZ$`W%`5V{%PsL2&q9f`bF&}5s7Nkyy?f5nn>cQQXBZ`H_{5McF`uDG7 z+)0BQ@9VsQsak0m=U>K=&tIe#3p*k-=AFc}&rdL>o}Bs~9tiH>D%T=npl_qGN`zVj z`{IXG1W|?y2%rhOo!m@qL$eQb;SS3qQ>C~`;V7@Q6h2!;#6j3pi=bbeyYB4$T9CN0 zu$RudN=zfM`d(TlwIITFsBz1|E%a84LON8-ujj{FZvvh1?VoyD*30php2l)19@1HY zDMnU9kG7Y>x>P`hU_3JD!h{64n=1!kaS&uytRA zf-Z!l9w*B|rvNYqQg4`0PGl6Uu@W@yXtVNi+BaZv)<^SvIuJ|7d4WeQe*1J1Bqfqr zyHc%JMZznj=6-X$jCONOLj_;SFW_(h@OL8r)h$uKm6(~Cizh1#YutAvc$lJj0ODa4 z6#90^rM^4xZkyz-WcSTlB#pnvn!=Vp4=2;BZ_2KQaa6Wr?evrMFe8J%v}%N`;F&pO z0VB_CI1Ysjza^S_%Ed3O+~B+bU4rjZR@y<5M4H-H{h>e8jhbMfM$Ml5XA>4RH;f~sxR6)Y2l|_3pbH9#64gjC$=h-I{kb=-(u;J} z$Yw%jUlIx#SzNsBsmZw+@(7C5=~AQfYOFTn*m1sz(mw`IA}fJdMlcEAPw@lBSAF(< zPvt6PWs5krP|P&IK-u61Y_ON*6Sj|GQS_qg)Bl#RHv{&Mg2%#z+ z9XI%8J|5q#t1t22HW|HM*ttC3R7Y2WXlnBB5kci-rLn%Mf(YnJS%PI4E7Le@N=JRb zksCU9cUz(banwP#&lX*yiYR;;urIX2fRNBNjz^srbmivRB^-3|_m6<^PTAC+hxjCW z)!)(_^@H}nhij+&ovE-eW{vz#1G6{lKo1H0#5llOX<)Z)oVVijxjv)-ZTd*&{P&7- z6zu5iL=N8aC1L#PS&B|-lO6^2%+eN>l9Cb?^?kU%0)CzR%v-QSSWHY(QZQ>6c&EKr zu{QbYd9^$FCTmQA^8%iX*l!hw#cJH?)7aV_SZ~-B_~aYtQ%d)|nW{k};e+V;jfqE? zPU%&M4v2{DEy8;Z5Jdl9_^bDbO8v4+egz1@Ofo2njxx?*aE^j3GSd`^*beUyj{*Xt zvNjq$Ba?K(bT^}CzQ=<0hsYdE}k-b{cg}qrG1je75Ss<<<<#Mx$*<%DCP!L_EkP@ejkcGx^#)U1(|<}6D{j< zroj#Sh~td*qi> z8f>^Vm;FDs+~L&WG2aXl(4vq4(PgXynX-sj9z%$XOpU-m$v*lp2{UtfdHK#o)yrd! z846C&xyJJn$=#G5@Z`yY1oNm8hU8W3B@2229t|X)uHyFp29I{ZtR0yyw2*6kgn&rO#R6%s+g!WFT=Zun7GD7MYNt$4$%; z`ij%AVanrKA2l&i?KH7_@;<^&Y)Qn0mXf#Fm!LP~K?!SJyU43Xk3zD%oOAUvQY0>4u*K zYB{owbobg2KH*ha=>GBb&n}a3NWI}%-tU!`*EjuP55&U$kiL&mp$<97DvLm;b+^V|)RJ4UhWNM6lOD>4tIrWWe~lN|4>1_vlj&&^6K8CU1c zf7I8V9l(DkU^@*sTOKdVrA#<8B9>W8fg_Zs4#486#eKaY5K_iG;20JZApLY}1Ki;5 z0BPm^a7gO;>?H7}A-I-1~MVHOayteZY#BEoT1E4&t&K z{XULX1uU;r_*rRhbN6;-cu%h&An@t>{DP}TIE(uHU-YGy&fjCe~ZUvJk6)DAdF84!mtxI)T z$Y5pF*}AN1r_w&@BsO5;g>-I0A|DF{w`qqR@(G?)Srt}6x!FpQW6a&`J77)BhUnBfk zuCPV^mA=!l>7#dZ)@J+ht3sa+NUc_7PXGA^J6Wv;Im8a}M8N4;_H6#~y7RuT_z;0Y zz%Y3F@>KOQzHyVUdQuR0ahT(p{hy>zKG1r5pX1#9bZx{$Zhd#w`|oa1s4ic^j3YhY z=YH2Uj;BOl;EEE(@<(}NU-1+XeKr>(a+uz64)z!u2yJ_7x9L^7%M1z_S zgm0jd-7mE6fyQy*ej0W8P2YMuZE*mqc)Q61>XZ-@^LuD!)sxXC$^K^DwrYB+m7XuA=uDo8y~D;}TUg^3<&ysS}Ub?WZncAp4Y z(C9zNH`Uvo)Y4fTsz}{DIQc2I;S8i2!qs!HvSab+tP?8DuowH=9#5{_CKinQ1_*_4RkT1sXm74r6L6>J<@@ z;fu%p{r&Uv^Y#5d){=Domtl#qdz-#((%YW8`D0u&lLr}GIl%LZ$pTIfpt)bobaXXrif&7Qk4 z=LB5bth?KPU0mI_>+`WYKl!uONnwnl+oP);K|xK1GrFFEp7whNgFZYn!}Jc4Y?_`v zP1P9(*mmO7RC%wZUe*tMyQv=@SB5K@dveVz{}i$4yzfb;(99}qV)wAIF`v|D=QJJP zQC5-I65DYL7LQ0O8fXI#{jT|?VZmA-uVAh$V)(b*(%7XB3ui)_*r-dKkPdq~aa47s z_ z0_0?pS=_6s1q|G&+S9bDfWbRSFo-Q%oRBi1GQkoIrtODMFCLaADI0pZ5hR10bOIHr zLHNNT(rJ!4L|>VBQ<$%#%zgJ5=-mQ5Nn|TvP}VAaA#&^LHQMnssuWubl+VHV53$V+Q6LU{IDGYXNCe$mIur~3{3 zs1^B5o=2pl_Pf!s+YqjvxHHrKr+;H|>->K0c6VkA24puZ!~q`)&30g5ninL}mo&Sx5@#1w(4 zL;5ZKxH~%xKnZ=g%F(N1k2wK)2ZeBDx*9!T+s$R{%Rebe-^(bFhz)^>P_|vl{f`;)^fqeSJ6&eBw^;kBn@nsZ@i`6QAe){QD4{(X`Ta z^L#c8yp=9BxK{G>dL;aw#;U^hZg)>lC(#N_O^l!Kg?dIC4%b$8kE(=X6RF1)ii^dD zHj1zAGBj4j{Q{Kchw!9?|2&ROk(s^xi%lK};EVax+q&56IW#W$+A8%QZLZZ3-X@EN zhiRGeFyK@{jr*I%8!f~~6h$I-yJ)N_@XOL<5g9UU)#Vr_8=%6h%p9F$$z=}Q(NC}_ zx5kl-vJX!C-{-5MiKV4e%RI}hB6?!jWHFR1D%PU_ToY6I)YjF&zfJxamKaK&8kX@3 zPhh!^qo7fZiv4w~LIW3ioZfsW+ z)b~lEE~#@~?mhHxQ-ZrF@YT>lF-tJRk(mZQ3;4z4z5duiOCQs$?#LR#2BilPWs%9L zB0@`kD&Z#0i!daXD38qe`UFT6jL#kjut^hYqG(wX@w!pe3)Xv;99}X+MRnHq%prrw zdDHvU$kDP?BNFmBpw}%povHT-{W7Oq$G<@T)?D`otqUT*a76Cam;rM_Bz+pU6+{`{&8RhZtYpD}yhGAm01+G&R-08RY~q%=-9=v}fhNQQcXxLHWfiDY_PSc;*cUM5 z_S>0?oO_-iqEtX>?WgMD>3Q{dcTvRhygO7Ny7M7+3!uLN8#A5;Nm0e6D&o6ldD~uO9!C;5$UaAMj|tZ(O;3U1BMWk=E`d z2hoNWZH(3jlkz|vlKOwYCgF(=m0$Y>Uz$jkW1>Z70>`nIrU5>GL;4}oS0n_(f4rNU zDNqgn$h~Q@xsqvW3_jHOVY2Q%Bly#bUf0P9X>^K2)TH)f231_92;&hi{sG)=m#iIE z8}(}4dMS3()u=6A0}a$Kn2XR1e+-~BMBZIJxQAS7e(EVrL!}I*>xM$zg=|;m>_4ow zQqOZ}N_c5qw|qVZqk-0vQ}SXt)&nSmkd^r7mx2+ie6Z-H7m(Xz4`06qO%xLS;L8k2 z`td4}iQXbX6;4M#-%QsIVCJd*4=vqZtxcq@vcwBZx4@Y%*@wVbJH@RsA5gtiArX~rx z*|O!PTzy#^etD33?5Hypf4Gx+!jj^AJ_tIuO>a5|T$X$sxe8{6h$cPmRht$;xuaI@8uw=E5VLn9x*Jd z)-+$^8TE7b4JR?IBdSul3zK7f)gg_Vog@>lC#(oAKD3bhnol*AT_NpD*v#~RKh>72tL=H5$bmyAycS6U+%_&W+TiBFDq&OvuGuWp1j)mjwz6-t!u1%kS|`IcVRqvT&Ri6*s9ucfF#hth|(j zB&)oX$lFM%pp3OvNbY@`zJlON4r*nl3D-yEH&_{ zI6MOaT?Hg0(#o=I<_|Z$`CrYb5osfr4SVjpXy)!wJ#kz|4%Zse};+ z={M1CcYdE+moze6-T0CH#&Jnlg6+Vx0N{Nn^J!|wgmFU^{;QthE8SyxK_`?p8s57qsHl5fGrfJ zq}%k$w9D7?WTmU;X|bot1Q~=qkm&<#uxmaz^L}j!^o<{8<{huEj~4Ny9#7ZTfwFGT zgN^>;?lrJQ10ndzSAsf`M$*T)2Joc+u@%o#KUwWDO5gIWca&{yZN*P6gEq2v1ib(b zWx(;OSnym?x$n^CunoI&Aj!dWhw%HjdP-&`PzOAIG$mHk?FqcYv>3T~G*)^4#)npQ zT}K)4um->3oJ%E&A(We1=!(OGxG+E!lz&^X8EfItDJdq+w*u=7!bM9r`*TPru*%cO zmHC?rZ`sd06CLvi7_yObgQE!uL7Cr$3KijXp)8lDH}MGRn(5whm(P1&XQzW=gL@{% zZz)D=QTFY_*ZGTPBmU;$fP1bF;+lx({TQrJ^a>P~Uc1O%s9HID?4`F0sA zGMFFdopvw_Yk1n9Ia@+mq7f(*s8za*)2lqIDV157GaekX6A82|R1SPe^hCrg=|@LU z2r>yYDHe@~X=7^m5eYcP<;UwGV%0xfk6JjI2*j!`aLVwu!#u>A*Wo|+2b=q&T6-F2 zAr{U^jY9^GE?s?}#5PJZuUEhHbyhBZsl<~0OLgEI)$)NHYf@@El)qlN!qA(elN%Ml zvihE!dRBKoA7#{W!ly5um=h%*cZ-2Smm4+U>3R&{a6GQ78R!9{z(AmqaO=9?R4fR* zom9k=ygMDJ$_TnEcplm^d%6K|!-6>^iN|Uu1qlkp?68$|ECb_sh2p-{zJ~(;-7Y-O zfEIh5tNr!!tAkm!sinNvcg2-w@W!ydVXJj@jdhQ6DgTRyHO5F60+=5SP(>@!pejcy zHhKhH!OWghvH%s&4B(!YYZL&T=Z*98w)468$zk5&Yl7|Qbm*VIK?do-TfSaD)BK}k z!7A`0kB^T4N~3u4w}`DP%HLVwRgEzf(2C25U(B<00CF?8ar*&Hp|XIHa4y zspK1gcqc3jz#M?@=BhI3>dtrqqZr^`AOLQ^jolvayI;uV`4pg-BJP@Hyq8_Y@$S23 zD0KeczpY0HUvKVwYW5EYWl-1*lnqw>`xtJM>v4Czem{1nYl3v%d-gA2XS;3{zT+CQ}LxYK$#S%0vDxk)F_ssuRyxx5U!{rZeOgKxCS_KP%$n1c*7J{;tJ_SL;C<6Mo zaD3S)I%!uI5&rYWDgTVa%JIZT&tpd?l7;Ybo}`#clJ~$}q!s>q#*<1m zTpdbaS)xwp7$;0g>S%cuxVIX$s1PQE$aLtY#N8V2TO8B7(YIB~ij&k8L<32or3nAT z?PB{8WVH>HIXTHek{v-K;` zX=yS$uZNGZdvh4J#G?8_bQ(9_9esGR^)1c_$V=(_*fyJkmGd+{ zR8}A6nOiB}EmHH5yf=4x63c9>^U^Q;8r5`mm-SehmrG_1sh2HBK{1Ms7Bc6c+hUCj*ny2CFuDo?K1t=ks?h)H?$gu% zDivTck=hjY$1eM97z_r5mZ70ZTGIc?+;30E$12$JdETF?ay1=I9+}BT(3_2IoOSNJ zDYREbc8FeQk4`Lf*VSc8C>XP0WjqN$FSO4FZU`M+fr4?V+cl&MXN&FsR*Z7StOI=o zMDlk7`=IxY>A@L49E&eiNJX8s%jaTy%?|PcUk2Li?Hr&=KLhIf=Z6zONr$pvHfHKf z21nx!5ry*Hd9)b|+4~BMpmatw>I(o^Oyb(~OdUnZ3-m+sU~+>a_8cGT*O*y%4qbx9 zZs5_ZM!mp0s#qf$UPH9;4K{S7eN+p0|}{-;B=neI<0YB2P2W}0=#lMXwOp8`()tV zi>+CNgq|*=G?Sa^N+AQcfB_j3O$ai7W%DZ?8cBHHcOUJEVfG6OoLO7~-cEHvq-K`ri`G=t$#Syu;Z z_z}Nf#NLxVZWj3bm7;bTNZC(p!AmjPWx2ndvy{7br{F``Di|iVe4F}(yZs)|W4Zb- z3IZ>82&bxS$MNv+UY>ehZh-#qVSYUDdN$~3mdU#-=;axhc!+_bc=mrn7Q+W_XvDv> zm@C|&;`1n+bhS|do7jM@3Y?^|TW(p_6o{{>sk;N8va<>#{2;!*hUr^?Jk;GSnB|#n z4K%A=ra7%klI`M_AYNMDpPrr*BWKF6C}DiELvuA*eAz4PD5_jRA>&y|X+Pn~2v)U2sW zUe~vrHACElz<)o#2BUmGzI-MB`cWZIr^K_2+h%E#Uw9&Uf4eeYxf~%Zae3+0w-$%?+ zvB$NvU;OSM&mHAvPxm=ofiL&-KJ4Xq$KpOOic)t&LA-T;0f}1*Kn4I$@#g422!9G# z?obzWRscNKApaXUKM-gjk!3kx6TF%qkVo%b|D>mTwSSZ&v||hSPP|vCmU@C-?8M

~8nsxlyGx*Zql4ATdSA>;)dgOOfmK=aNT+m!-Qq^|UTR|^9PpeP!<1d$sbv<{$pJo4k=Hg650`-oebMsi-8pPuFg5=hlY%4>rzj_xDE#nV*1_&Wd27Y$4C5wcapb?F8|F z(5kJGIoJLA{pNzc; z>N35-Km-I(Z57g=0W0jss}l0you$~Fx(`)bS6AZUz%UX5L_(lYK#+giDDmG_V^L53 z#BWqLi@{%Ut0g1foN3kr6qz>iA2mFQ%d@vAG!QIAsYw!q-wY-dsT9kgy!rFnHkyS1 zsgRo=H>@0WSdni$75|m{J2Wr?qYT!3IA}D@kLE&Em-7Qa^iec8&g6iSRj80vFHvMq|Fy!(s2c?gp{ip=m~44IX2ry<7_uWvU_PMEv2 zxw3IgoOaMVpot|WbaK8jwJ;u$i(Xq();?QjuuNG7*ke+NaVWu?^~vvnlL`E~P)-YG zQbk`_4b*0)U#kwjM9k?l@ynzp$--5^Z`?rA3$FrF4Z-ribNSlZ&Yo#iVlcq~6Kp=q z7+$8BLe9!Vgr`mi#mp0u!j4(mGE&9l@NaXSMYeaq6jRjTC5Yfpz*)_SEX0Sxzah7-1B&NF>H5P z_dL_%1~;hP8E~*SX5)K_e7T(k-i|c~zP{eKfHMNnGW0ZnZ0dlXV$kEx%fGtk{eqW^ zpg9WVJj zKi&c1RbZ)GgE1$t)B`w^v4F4^e5k?tSrej+vAeeE=p?8#dit3jczU0vstxC}yN6O)FS%_F~#NhA;= ziVleb+$X_D^uphN#qZ!_+rwGZsDmJD%a7kg%#XKcXN#YL4vbd!Jv(0QfPxo>e%>VF z;*pV&Vf?fb#|@*+fcexOteb_ITU&q8UfA;hh+bv(%}zURBC@}&cqtNxGB`~doH`6^ z)R|Jbv88}{KK7%laHS3#;G+u)3i9-9`cOrJ`Q^;Jv%NigdK;8f9c6E7MqQlnp^8C>E&t-p3dqgJaa(y6n>|I8+ar5G59=XNN&acBexVX3< zGSX?peJ*dhUlxmtBlvzejkmelSGjtmY(!woah48JV3qFN$MDutgjR85BQa4-Rg}8F zyD(uhPAPWee}xhl+*@|go?T|$V1S@9D-7}u!P71}R&>JO4%siE;zpgI^u|kCN!;IlYE11M7xU!o)J-IX67B0!Yy@I-)@^>LDiczMNSR&{%_c-gD!6kJ#SC-!S+}c7(7kJsu zkm~Sx7z4hT@u+=?{C57O+B<6&RDEr@Gkbj80YsJ5;md~r42(X;xtTQ;Biv#L+liz@L7%e)7B)x#jr0_Y^5qYy5OLOC#0lck;06lV~vb z{QL}5S3IPG*mA5_3!m!lx&ce^8?`S7zWK995av{6rc@aM8NnQ2U=ZXNBA8NE|zwG{)TQ6OpCZj;Sq ziCrqWxz8q5p&%r|mzRzJ-{m5Uj(PKD4A0e4dP{=z z(+7mTOvrHF@T6OWE6KVrB@bJn)8!w1!;VJ$A4k_65B2}YkEk=sr7I)hL^*q9&r8PH zmynF?8QFWEnH5Lb^UM&kGcrSVb_iKn*_+?{`|GbBJv=>>ekr^{iRU2lALJnyS*gb~F+6H@MG&z}xNx)7XjGc!dA}$;?~R%6 z?IliJeqp0d*_Th^Pw05dS5bDbh{&G$T}1=ZEx_qSD^8k!>{J0PN)XDXtoxJ1w|xn> z3f{EF_cQ)|8`dhn8+Tm+iBIyx7~c5*dy9%t~|cUXT` zj%m6~bDKRstVw^c86y#j(RfwBjz4sB-JYpiKa_vj_uZm`y-HMJltImaTt3W3(k!;*Gk~VTfY7e5yi#DP1{uD#abNGnqE7{YacTF z-dFM+Iv1_Kb#x3w4g%;4Py-xXTx|H$3*Y<^5ebP75;Lw3#ahJ$8bbmlBWn}YZ;~Hb z&w~&W@Bn+P^go!c4u%1t1Ygo0$2wG&e?+IZBn1Q*cmMoh2F@hMIz29nm-njGXvwcP zZBtyoX4^D>=3A#nAJcW>O@Eei+H|$muDdDQ!2Z@v*E zpgo>Dn>{MRXD^<{hx=GLUCie^PC&mHKiGNAfqkv~)M8Axq`JD?*V+5mo0`5z7MX** zwBgR>v1NRj7aYltN;8^=V z4!XRHX3I;nGMJh>o*!_a4a#S@FLC=r<#3 zbg%)}Zp!|~oWM2|`+-jIS_p=dylf#*huBnboTrXHB`eB`0@K@>bGag)#pSv~MvkvRt zVapbH#f{r|Gvp`*9|g{`4(9Q`eLNk$G`pg3pYkbltdr*1MxNp(`8$p<$czEy$<1(L zaY*L(j+Bj7;`)E%%rAWB)vQFGpDwaBod(Lr(G#Evm3`bCd=7F)*vdG(5C4_EVDol9 z>dlYJQL=A5T;80%=>ATiVQ4_7`q*%$Ii+{W_uyEuX2#>X9m7CGMr2lubq{EpQ6AAv7UFf35cf6g*v zTvha;QtG&E?qpQHc{x@4@4=i?N( zT~>I}Xwmjred*EJMP(&F8o45#F>nWUdLdb4#?ea~C0G4Mo9*u0kyqpF;o)HdyEd>T zAFkzR$h|1E2EJ2UtMlX68JMQ8UpJ;kREsy2^PZwqMhGEFLa2cc=E8<)_bE<2`ay!8 zpC{j{v-a2+xP5A+bY2nvgnwh3!@7;NvOE|A;jU2m?epxnXj`#l6W{AUVMt>3*Vx_ zr}Fg+;{`Mkxev|IAkj@KUmdFu+KYLrfQrs{dBWc=wY`s8Xl0ScNS@gKeHcqR%k6&_ zJk8}1nzl!yL}AVUNzHH$1}79Hh6#5+XJal>N6vYif7Z#caZXOE+x0s(yt3dlF{nS7 zDol<}4d0wH5Mv#9LBkS%*+L)vbwrx zhiuJA=j6-EoyEDUYyBrfL+_2K0hoC>JUk41$Y9Et-g+%emx#tIec-x{5>4x`Tp92K z8@rZ`Q_#29N3WZ~>T+;6{hx&Bcy(xOED>r6JbfQ}Mc7S}%_YHkbga*_FDY$4nYz#5 zvkyXfWFCF!`w$WFzUTaxrD&3BKG2OX#z-0H=&ZUn02mka^z9Im2TG3!>{Byh;YxTt zuFybd+jn4|5KyrZpp7iY6k-_-3N=~rIYX@49GYwi^KZjm4pvn?QI-L#gh81QZB%lf zdCl|_l{*k;{jOb2T=zBqJgX|*5>2)zv{B&qNa$Vq&pGJ14^<3_k-Bs*OfF392{EGE z+M2AE*r3RJ9%>U}p_bB@w{n_n#D*6aDj2oaF4fo9)jd63FQZDlXRp_IxH^k z|GcC3=w{8N5W?%HmEyC*W>QTBO$E4Rt`4Wx*Qw)b!I8o#BaDD}VWs@~jg zzG_sDFt0=g{hPLLdXl8tFtY_7?CrURny+8aJ7&t9v_de+-AR3R7Di?#4Ij8Te}Fgh z?B%6zla8Lg{>AQA+DTpA+*#usXw@89`=W`Y=*j`-$5ra&V5wuYv9WP%zI$!0a8q=< z>CCR9XLS9bvF#dt(Q5&T4XKR7>n}^aaug*qM)(tB9YxdF5S)Eby77#O4?==zJMBqR zGrrr6+kV@{WzBZCJOfDY93=p9B*%3Lunwtv-KK#{8co|Nc_KnWZYR4-#^pLcmR$6~ z{o$fd6&w;-4&npL#>s*MVSt}C@~{?60mTGZ*uEPFoOFVgK4GnC`>bisRWte+XLr_(kYwXbQW!5X!2OE0*je%v!AE4xM7-@gv7uX^qM_SRv4~Mf!HKtaPW^Pg4~4*cbg5q6a4R551a)DN*`Re zWGVY=Dbrhi*$a}>b%jsy@ILyeKJ+!76)_~-{?pkQ35!z*Zq?f7%*3IU?bujy;TS%A zE=Ir_p?LQii$h@aXcn3!ufaJ4Rq#pMVR3{9heqlP!C-F}N|+?P_8AB0MbYuBhfBYa zC#3Eq3s;FO2`^t1X56kAaP?m>LiP_-@?w8{?PBtlu|qigx~}|$&BFmh$;mNFz<%+M z?Ac%rjWMok@KGae^_CVo*ht|nS?gfZ)Z(|nl&MyV^TNgH=nj`6>G&4-yfjT#fuL1g z=|ZYCZO5#RtkGZ*R`$xwbeJX!9!9ysmOK2Vw)Hf#K$lY#LG*mSuw0k&Hx@#mDiU>< zo1BKCP?JM8fav|=T(^f2`So44Ee0a)C?dsA64MHQKgngZ?r~b+=E%nu6HG|RV2)fT zG#<<>yU)Wy*g0-0ZH@1<_%0o&$X<}tVSD?(U3O=V&{9aA9aEXC)r4H^qBe)N_WFuH zh-58(`v+QpF+HFm)~+Tc?eOpEl(R4)5QLOAx;};SR_AXHy)|>F!xX~N&-(b-_MALB zfo>-Gu#v%AD!yCXb2k^P5^_1J`6kqIXT@cu+$9Y3^s1`5OLc#jUTuG*Q5yMTk!Iwz zGPrrL9BG?BVpWrDZ_rR(S6A2EJV$$Y?KT_UF)x_QJkRbA|0KHU_}}FsMUws$@K#?9?WS|zkdCio*ohC+xrTbMDQKT%L68R<#OK3dZUpe0wf5~P)ACP zQ)22dsY9|&;&GF^Op8J`bEYV9Qyf_$cw0lI3bW$9s%;4=}ph9sfme+$;tUt zpu{(&79Y6g4?c$V1M8}(v9U35^dH5))X|x!JGwvT;G%!EFO<==(|P{{#S0A-GxR+; zT^AA>8`GaVYUGLUeo;KCRs4jO-0xyXe!FRpbKIq&A#DdxY zLL3CoBF-*&1|Ql?`}HG&@_Hmsk2EDpc|;Q(jACqn5fOr0@dHyU){-L=u9P6l$n}qh zZAyl|2D)59<~RTs_coFK^&Ju^L0wJcjZ(`aOL#Y$&>cI`7{(vX zE7mFsMdqrbjV#JO7+w6Ti!hRC(jNZ(;EJ{^Da2*}uz65CxjlWEIcuBbt2gKG7Yb=j zD+Wz9aj&$}EM~d&7ABu&$2wkP{MVh|?BV#OW=@p9_?eN_B*y4(ok+0`S3=^44{6=| zITU^_pyl2-UyMG=puw|5P|DBUy|%jgutVE$)^ptz7DD`-yKjK=#rpL0bOL(m`1rVE ztCYOWx(d{n>W%-1#HV!j&)QLk0HvEj*|?a92*I?N=jp=Pwy&3wRkuLjj`J>+-wu_z z;G%LTvE7%cP4UsWlc8-8@Dd+?hyFVx*G7zHrr@4ZonF5{pLOBarfNCl6P1y5SM)*M zH%pyQxX0=7oc9913Cv3a>*vA(Q{dY2@-kpm!07;v1Ozki=JlfUsXeT-aA{C|J$^q*)O`nbF?yIdDL+C4eP_{;}U zNV@MHuNfWXV$!GQ=9;~ik|nBQ)cO-YXpfJLRU~=m48<}ZAC&p|0A>YnIC~p^zUXnx zdM+-5!(oXXEpp26yF8hin##}c6HJcuJ3I0_0&x09d|Wn`kX42{qz1x>?-nLuQ9wQ= z2*@O*HWPhFfDj%cA)MnOpW!pp1~6bNwR-EroA-G-)gRySltjqpkgIVsa(gzx6#~(h zeRjHpmfKi7HI6S%pz$3xHV{)}Q@?&{>DRijm7cmM9U%Zx>CG0ISTsYa$5;*{&!&;l zduo$&^JT1T1TEIvtF<_Eunla*F#$u@W`ShEix+ZqJ@s7D#uOQC@$Nc4pI{>VXdLj* zhlWixyp!c&7Ay;{XF_oEp-wg>^C}T(y(-Z>hDWh5Gk{82?qcY*aGK~+xjOnKItl$D zu2-D|8jQ(`MIfX}@I>Tt&~Rw9G!}!kgF;eyqxkx3JE`BX@M;>7nPjs*$6j}kxpysa zL@*Pl!FJ)_WjO$4@ke^|)Cpc2J6_PRO}PR2*c;}Miit3{z`~x5B*WF&FvhnjeVFU5 z%z17Ro!Q(*X*xHfoN>|`KEx61=Bt3}X$EJsQL4>;u}?lqiTN8x6u5) zxr@2;q1FZCh8cqiy>r0s6tQm-&O6;#qR0@ZzaIae`Cx2Q=<1K!rKvF8vSXc9rPoqV z!j8K#^nIj1o+9^X=Ki#bg@DGlo-vU0A}lQYJN16)#I#rWTdTAkWefY-uV0u0--@PP z&e~t8!iehhR4aen7)S@3Nx8(uJj2zoHiO@nvxR3aMs|SF_&phP8HTC8m@3(RBaX-MlqK|7h>CfD>DXVN9W!Bo6|Ke*PHmeEy&w=Znd z#|Qfsyv;<@VIXx*lQm2o22I1ts@qvb@^y+REPMH~K8=5=bTE4P7Q!f;<0<%*`sUMa zfy8c8Lb1pNhEnM}Y;c3s6QqOXg8VUT3Cs|k3Hp(g7S>I*$90Z^&Jus` zNEDx7{?>DaTMMYJA3T#H2i@muFWKX}Z(h$iZ7qU6Mr$C5{97JByUhZ(Oh%e%*2^Mw z4=9o0FeBZ>+o5lNd-(q4)da<$W2W$cf>d#~{9%EI;ae~0eAy%JD(+APLD29eG?FQs zj&7GZY@w2~lov1Sj~KudHN*0|$f${nFT%!lBNG zjVM};S24y>nDlH~JV_k}B-$q&18s(T-Ko4NTqcpT1R#B04HAHfJtM zM(%iKs%lq)runaKmY!)Jxa@wE<~E@^&L3WDrG8Erj(pwnc{6;)BMH?svx!y41Wg;g zz1oU5^x5fNv{17tCTn%-OkX6rrH*<1izgD5HMr(nF}VRyFLhQpd7C@1cT)g9-tRo- zYG-Y24KR)vj(hLN?x4iI4mWm}dfV1=)mit?E>;FI#%-L~+1bIGZ>akT5EOeGC)?-$ zGQ^eEneE`QP%NY_o-e0Sa%(mi+} z5*_p@Z2Jt4>2^r9t`bdC6%rD<*#^e3mj(uQHPa+4d~~<=PQ)L78L?V&68ThKFfb~h z0^sAEA&}vG7GjpzW7b=78G1E-wFmr4q_6Fdh>iA#$6UETa~sxsuXJAkt&8v3^5$y+ z&P1irI=)45X9;^_s>=U*)X)2Kq)v0BinNN)K#{RaAtMw84Tm z^~U2|ghwe8Keof`_GQpCBDxx_u}<>E-0G>ARvqW@Af^aH1zb;UDSM|3x8@BEh6hXn$^Z4eZ$NYNns?0IFC=~V#_0U9 z-eVMqENbV9p^BC0tatJXaik!TCXw+Bag4(B5D?Cj%~6~Kn5JmGheTv0^=(f$We518Roth-CR>xC{m?-b()t^p9to&3xh{IjNEZUcYx1XuX3lg<%5&7k>WGB?w)-xR~?aIqCoAE10n! z6B0k@L}!iC)2})w3m)q8?u>KAMg#d86wX7h!S$xg z!+z%?UURiaA$cSS!LhNL8ZLX=(unlUD-njG zdHPbQ-k1c@-`{^vJCQ#ra=jktz2Y&B{XXFE=-g`ij41@iz>JS(?g+>wMOGROwwL=- z#KgotrpgL^1mvLD{;NK7Jf@tSzc@v^RIl5-GCA+SvoRqd0afTeZ2POxZ+kq$d!x40 zD)~RjP21en#T<8HahkcsRmbK0NZQq<-%&F-ALCakELIvLBnUL=+>Qe5)A;&{W1-XA zF~EOTSNqo1O+MK7nRBCCDK2a3S=q~q8(D)hz}}bZjRWIz3ipeMj_mb%Jb3BdCu+wO z{;zDAx|UxiFi~-b|h>AtqFLday*9zT_RncsF={o4`o4? zJn5k{Y$VQFp)?@~7Bt}ql%Ss)#)yMF%q%b_C>Y$+V`_|r6|O1&{rUNe<)0Qk;(H%R z5CVOWFw$$taPL}JIG4ZU>5ex|nUyV9q5-0DD%fqfg}$%fq>Rwt$&&smvieSvAPfaB zXn!sog{Jz9JQYV+js-0*9h>P9Jj79!@-vHX#h87JdC2q-+b;L|n8M;!<$HemVy*&aUbR@- z*ZrHXXg#=Fgf80!7ZkfK`8Zol`3&67lYR+s+Y>{3+ebq?6vInx9N}MT&O7tPiPmUcFjK6K}!k1 zjJq7J2YowgI+DO8e<~?K;%8s^0wq-%eTMhd#kR_$#lgKA`>%(i*%=eEPzZ0UX?q|Tq%9VYUH1^FB zt{Zr-jmNr5w@z>C=q$X^*by~04)89N(@OA4t={`nJFo27g%A*H_j3s+PaXzv{iJ%d@y~QCManfEr7@C-hf| zU1{&#Ch$0%b@_d*tgLKk0I;>|;ZZIw&x)Ro{=eOMW8ceC(j+-Ap%?zHW61JLb+sUU z%n-*XP&UxJ#6x~|A^8`XfN3(U1EOY-G)bNXUoZZ z|J3^=|2m$z1!()el(m!nsZ!x__cGu8KVqK~Fp9ieb<4)ehSz`p!B>v_5#iL%{HTNk(mB4`5gEYGf>w+ZH7We= zaj*E%0yAY72{Rxh*L_OUrKx z$OV-}(4peC?Qdn#oF*~rln%N9-3|=Wu#h?=%i$^dNt!C)HSm}r7asxInMSV~O*Z{H zN4##yp{In(2wMW_-&&yi$QjDSNqQVBj30pU3C!^JWn7#m>5oCn_gd4z@}}RJ{nh5& zSq_8StIeI)$Dz$vH|!e^mQDF4r>5BMy`x}|AoK?h00<+wS6}MZFaz|6L^6e%q9P(P z&ClC`ZVDhz0GanZV1v*xFyx?T_P*vub%L#3RaLb=T5VUWQy|i@FjqrxN100+D2)!k zWtM0b)n)SY=XCX(Y1f7sIj8#qAw&@Uan9b>CM88Otv`itQLoMU4?JfoI5c%Yfso&x z{be8c(SZt+OTT_!?Xp|yWQp7Fq;cB^+z;DRRW{&u1gFa-v0zigDm`H@7+=T6QvX-X z02cWZ?rmi)dN1a=TmYs}nve(uJBa8J2B}CXZDX3Kp20m6^HVcNvkhG9kFq_cg z0<%e_NokIaSld_~dRbv|cnlx;Q>H?As#B3ew!>XmPo0sY@%Py4k|aI@d#C}vu+o&| zQZTf4n^sJ6v_EB(G}?KJjUo4-nY{BuIlao>Sa-PBk2gZ?HPO(=+*9_B1&JT@@?>gb z3Q{X;QIEcJ;hAL>?;2OdsT28!>3GY#+{6+xvf)~fu63o{>e@23&N%#R)?hNt$o%k^ zz>mnR&@759N+Z_0>aD_A$$3F}Vr5e|k{RtkJyx8!d+#W-rN~n*2gCd&AdeT-qx18l zoXp)c!ie1vSq|vXFZpjLlXp~p4U^nuRG122zpmBkv0F$szH9~!p|KEpH->nT*}iy@ z|6Xb4N5O;s9DYSL4vdr;n{Mb&(T$X2H~J_Ks!RqYeOA0Ox`TCIJa*V+CP!nhxa){Q zl_EhVKWqE}Mb~imy?5z=J&k2Wn6Y9tB^@4!e+J^dvyDpSNBnfR=78DLR?uepkjl{e z8}->dg`Y6ptvfPVFea0aw2Rw*M-sQOA*Bt`dl`O*dsS7V+>&P?HtK34evUf%a$~ac zKiw`K-g@*^Sr7=QVnEubw)WKw5KIE#ZS3u!T@4Aa^L{|%zrD`HwSI5ZEn#}wQ2MG+ z6D&4|hk~Ae?!D^(_ZHB>Eia?-m@IX0w>)c}nVgtlUEXE<^V2~rEgI7!pz`M?Sobq7 z*D_8qxE?%UU8FqEYo(e6l2o6;d>eMFujNQPja3Vs4&3rx37xZ-obhp3nVp~AGA_wZ zIn9-#bDlbw^3ER}bCJ;g4%%4;Rshfid2M0Q_2m=bAk$R6+P>}V?7S5w56r}{x{9WO zpQ*QE_e323(?wrk`I*IUkka<$3)cImDW6PDl4D)u8*Pb!VlA?il$1$5E~u5LzOo14I1n2+aJey1s^0((Pjz)QZPayIPGG0^Y?Kri2SY9& z7;?Wn`%Tv66wa8O#U7sii4mDa*{U8Gr>u^G-b;)7sNLQ}-d|U^X7@YiN)X!Z?{Cgz zcBdQ{Nn1#tib12ks`nJAU4tn~v*E*lKv*3^$u?w^pxz%=8c-@gD8n2`RsC6H+~%MI zm6hqg;#g9pHcI}RK_PMO!}O|WjYG9F_ubA8-4~qY znV?8K@xxvuS)e}=6HF3{435ppB!%Wj@<#z}1bg5vE-ApXqtItcp=Y-z`G8nh3%;+(6|ZSltDwE!Kq+U>k`x=U#quEenh#W zP-0vVJ#GWcw;;Ag$EF9`_i@Zr0LU?xj0@*BIXO7#>F74-Hl*Cop%5@*g)ZL4f6*$w z*y!c-86~hoC_Yfj1 zDLM9^)6hCuknR@99s!_Q?-C#1#^O#bupeFzLRHa)eHL5&z$^rX;NxJoG~2V{m~h$u z=yFM)%Hbd#0qh$Qsz_pWJr9`{B)HF$S5)YxkHiUEXvk`#+c>TJaj|G5I;{T*PXMg- z!Pcn(j3@wxMt)O>^wz`#x_--w4ZKE7j5&n}=xx@RUcNbds3i`c5S=0;8TwfjbF=Af-=$aZ<7#bNK#P2M0`rQ zeqpSdkN10IU?&qHfGZO`J;2hCk2uNUh&<7^;HRntZ9ebq>pL*WU-X#0Z~v7yDbfxI2`6k9 z;;(kP80_urZiPO@YZhvf2EKN3auTN(0->1s7c;YM_T^k!4i@wgce0GAg2l!`nSi7&~ zg3benVidEDzD~~#PX{tCmkxVG*VL{?GS1h~O>w+G5~`}a3*Fsa_l^DREw-PTu-VPj z0}x2^ZLz|W^JpTkLFilG3U3^pt;+d#s9g9m zJUsGkF2g0O=C8BAn<%kzad82^k*!+8jN~JF9Fq|>7{EdTYCR7?N(2ZXj+ymXNn3L+ zvYM*27{J1skGz`Q+}(xgW7>GyWBtH3K54tPwdD)+{HN!81M9&Ri*<|Lxv`vPdm?Rj zzfHKlk>}sXN`PlvdcFqwv~Yw~Q`=!mIgqltkGj{I*0t-Fp4#;+a@2VB6P29AZhtgO zZY&B)5TQQ!h=bi}$qwCY!9Dv9x;e7&e4UXQiHyf8b+&JqCe# zH>s=BN<+W%^KZ}drOFF5y6*)OMy=D1;|Y;>M+ADF$h44o8)BQ_0dPr$ZfFg8PV8Ou zf)s}Uqe3MsBdM@iNZzXutG^+g!WxW$v4jMLxLDFM2Ur9n9L0>zFpbT1P#a&>>_p0D zFKqccWD!7VI+Yyzs7+Ek9c*9=ryuJ&q_VPRh<~X36F|zhKac9SB+*&ia_Ia=!(ZDe zvZAr_g097%sUtUA_mypZlxZkAakBBFD_ra`)4vNIs4O));PH!jNRGqhu9W^rw#58< znroizi)j&y@kFBxF?zS*%WO);>;!iu?~b%^j z-TlVac4KP`L^FcN3usD8N=jyCX8iVJE;UND#Kgsatn5upuY%0t>NnRQo^>0Wn;PhM z9mZwjXKQMH5f|)RvyH{;r>1;`qt-x)oTi$uRh%_M-T4)R{*!ZdaS1etz#RUnWDNYp zc6xSd3WL_KcP^Z?mGn9UR0v|`m#|1$N(!h5OR-r`tuQ>_SV8xkYr5a)J5a7;tLp6D zODo$N|8$-^vz{*awa{mMtVGCe<|pSJjz%6sVx<8Kr{>e8P``^0ejvF={idV0`+j`J z`Jdha$u7_FNVBB8m#lDBe0^!oK3F+xSOcxG99Ew^{}v3YbAq zIA!w`MB-Cb^yLXGL;Amb-(x3#ID|qE& zDT}PPKMrJ1I^7>FctzuSUZ<|nE{a1@`BD|~BVA`*vu};v7br$mN8{0Jqz)n0#gKK0=<`kK{nzR%{7ooZ170(r9(sZ_a1Tj2CnuZ0g{$GjzJIv zw(H54HLdJ!?Sa~4EOSyC67pod{i9Yh^Q}fSQUOMz;JAcLl*QM`#n}}LAp9Wap-ww+K~ONNGQ9AjUS|VFuhKGD!)0> zIP#D^CFxcAIDg2~)>3}DhC2^!zNkg_>4C}~>ttTqsQMDs-yz`fT^#y-dayoed++6j zx8!#H7ML&AA-I??t5X-9}*LjyYlglC%^xFp5AnANSg_|NH};hcl2f) zQGQsgTf>!7LZ@}l9twA8DO$+b1BvN;Cw>#SR)qt@WNznEQeoaBkb7~@8GoKj&6w>MoY+lLh=seYbD49Z?;J&#ZEAbH~&_MI)~PDx45mNX?R5x~QkoVb#VQX@+I%iOqSZizXw$^zh(tY@Kj$ z0}&Eo;jqA19A%n~;_pw`Ovs(N=Qu(jyzD7}uwbE}9ID7tlUhL}M>7~5EMv*{9GwB9 z42i?gh7b~{hdJv+9lDjdHg?D(Nk|8r=pk=Qr!Dy!Oj9HqvCP-vm1UN18Z3(k_fz`kKowb^HCqe|4`tc!zLFYwk6#z=yPjCWby zXVS((vf&i>n8U5bNi-NCPy!NCTsGB2kn9Md#=u*o8FTnIo_kDf4z!EFj?O1CvSt{+ z$5+hHM6`{7@qK&#$6J$aExF}%!@JNjBKNaQ9EN%HapKqZ%6Qz22W~4SzhG_iafS;2 z#wvrqM9hNz$HrLX_~8D%u!xrrlj$Y><|n-T&0N@}_m%Q1hyILL`Cri!IoObZ%0pZ!e$ zW4zz~kURjr#>NshE4AXC3^?v}#goWYop{euTtnhlJPLFu0$X~_IeJ^Y?jI4%lZ3l; zIq*EBo&3i(x_#6zV^^nF{YH$4h2)RYv+=!%CCbDsWA-;lGs*ZFomY<|RUh4%KlK_g z6dw66Aa?uE>+q-5lPRUDj{6_V68cb2Qn;5Rni%SR86lD0f^?2l*iM~7^;n4TpN-8e zN#ApKz9i)bDZBva^=hQ=7aQXG=(HsqZA3k?CJ^5(Kzox}k-V+ty7I&q3J4Zfr)S4Q z{R>pYIXJu=5kR!JwzhuzrctOlH8%&6wWnuieVZCawr@7=rEN1h_<_=IAS7Od(1gr) zcXxxp75SXo^c9Q30OER;lljBk$jHdaVQFY4RI2u3YkuD7Jxz!0J;J-kl8f(0Z>aK$ z7}2g6!CYu?v9+G;sSg%%3G#~=6QC$XEP0T$B54F>@S_Y=1C5o*#2H6E?{{gXc_^!h zCo*}z|BCckAtCsJ^N0BIfxGzB-B?znk0BBE4uW}vT#Ec=n}U17&mbnm#US%Z^WIl9 z#t#?bf`^NW;Qb|aKYt=0Euh9^GTy!aCZo*`KU~yhuiKCCI-OK)5+i*0EP)Bq!$Tg6 zKK}N_9)Ox-iaB0mt~W+F)Ng@92VaU)e?4*aor*Gy88-eywjxzlloDB(V~`Jt{NcRn zzMx=iLkw~7std~wcv&3b~;^?)5rgO)7+aQW3%3z_C|D6X(sH0OBPtc`7&aCGgO=12B5Qh;#ogIjKI!LIkdH^stVNf z^!;?!U%_T$?i}5`O_0J{S2}H1Q@+V&^2347^j!Va?4?$T7CTi1z^DHGdkc_T5QU{( zYA7aVBMGwQ4qw^N)H$g^Wu7#B`2wt%8InFd)Ri1Id`SRq90Bom6Lbl%z<7U5p|h)N zHHe(~{F#R~iZqyTtNX1mGv;P!n_d&)byI$tSM6GC6M5I|pCIpkvtUptioC87M>s?+ zzo6lGyFxk^_A0@c<~Bn1hW;}vSw>Q7gmhv56RVey*Os0Jz3xr+UUrPROmVl1c7R`k!#}!-&5wQBS;dMNF@R^0~h(wG7uP!PcCZv`J9sCLZQ-dn`)#R z*Pnkm5B^xhgw%=Iig`JuRrnJTyI01Y8nC}Z=o+usb-dD5w`T%@Nc)fL zvQZwehZ_^2!d5&Uj@waqMQK&eY=fo}L((n>e>Q%T$cr6IzC%JaveeSf$IMeunG%_HiZh`YqjPQQ+pv;%rX7enwwrP#pmv;H($ zYo%7hw%ViWlKYSF;4SZ-K5i}0Ia0V@U#C|JmW8$evzH2Vb){bJj^3W05pDh}9v`Vk zTqIKZ2qBK#)LJ>U#AlgJhf_9&J_n;cJ;|8A0F~M_sKF#XZC0d{^4Ex3@9F}={-~&^ z9ryEv4ze#)Ak`&xkBZwslkG099Sk{(JbE1W94vq(?MU)$x38oakcV?kjzDiSYn;XB zny0gogJmWaUQuR+Vi}H=Q(kHs<%O#@@Esp=JP~;ZP`6>d=yX6fy40AGWsTx z5TJCJ7}VNMg-q)hL%U(X~PJ#Qg zn!`jA**XjA2ym57fEg;(18%VKEq_XT&JVc3!kxHfbg-|11) z|NiwFW*_PwM`?eH#F{US{CeY<`{0IKN}9Qwc#F^il&IG`rsX;~j38~4b=BYcEmOYvRJHsOZ^<%TtK72*Jg|-5U14ik z{Wg!Vuw9sFn^%YOR;5K+nUzFRRsc7a>(bQJE~yCz`BOA@v0jsbG9p{A{%b{?P{pEM zGp3|If7Vj`jhknWX^0(XS?OdbnA*UAbNHl#ZE)!IF^IAPi5AA?AG^(lo}O2Y8&tkI z>zK968G=9rM7}kBS|wt{%i0jx_5c$uaJD;r1b!-`j8%hXHKgIomnWlTevK6sU}hT7 z?UV5f7+EalIDUU}_5p|$l zU4;FP%Y&i)K`KH#49iQ%l&b&kr6zl>DECuoB}f!sn!}(o>Mm0)6cyu+!vw*K98l2r zXd=X*rC=lnTH|jrGanqoHa~?@V1b8CRHb213(PbK_imna_5^e!Z|@HhMD0DVg*-n> z^-8XNDiRvgAEO@X9oOn%?2Gw&<53G8wf%{V&&a>op84INlzSxZMLWqYyUM0YLxx~X z_aqN(+j_b9XSHC?Hel0?WB@XP>L5lGy!nHB+y4<0fkk&nL@xMTE5fP6!!-Gdq@H78 zwnK_g#dsZ(^VoKc=WKYm7WuQql#eP4(D0TAT$v{)Jc2LHgC4@*EBFT5!U;pMmr^&w z+~@TC56JF>{_*1Wx1!8~MC%luiMc)q^B$lP@e_m({ye1;{OGO0m-@Q)y@5eA2Ulta zbK(6)mMAP5n>CT?JGsAKzgyMSK|en8?J4|0On?pPFE^XD(=1(tmd^xJU2J&$YV| z9meI}-mYvYMJPMG_xXGZll_rw%^Vmq-n_Z_jz@p4OVE)z@?pPYzY>1<*RND=CV3vX zU3MtB33>Q!JUso?FU{i7imo7A!81@X4gp~;r&G(Fjb0Sssg*jjxY`)Xc|2ox1iVl^ z=8nKL51i(f`k);b^NKaCy-0iR2#8a%)W0t|mS`0ND9z4pw$5p##HP$etsmTlWsL`$ zi)}e6DbrhTG3=VAsS&xT+s_I#vAnmKnOF3hMq+R9c842t?%P9A@+Put%20&a>KT_& zOL)JCHjb51Qv>oFXXM7aj2h!X;`M;GYQpJJ0C}rAYr3p121GN2>pomV_dAatl-p(M z@yS1*{!(GYx)A!vN(0-RQM@jBwAvUErD&l(lF@1t;mvq`#Qi6=LtchM*iww&*)E{K z^q^74{%e-MfZ=yDS}NWAF8{~t>CT&qNNw_K^B%sxi8I%Se`9%j{5@8hxVaYcr9Y2y zDunH3uy(~3ey9B)Q2%UlzIHj{k${tRmbE!X;d1Pw#0fN-&T7ty$$~J?^ujcc2qH|$ zY@n|wtTtK9Utm=m>U@(o!1RHFzAfz;L#~m81}mPM|1$ZQUQCPShPuH7VO`6bI%g=I0jtsn2(i`0kKD_`h<+8b?R|ShFsTmn_Iy3HzQ+<(1 zn@_w-H{lg^|GlJ!4uK+XXUCyf=`Sk%0^f6bdbDj3#i5CaEjIhG4NP#e?Mp5ClBqwaU6WyRZga(x&f)hdxRWK9&ikm>B9RW^M2O6<9^HnTPd zb;vUp12AZl?LD6llyHEOh)c!y0*fw)8~^s2kpPDQvrd{3c`vwI!b*Tkoe&6_6dFKu zLK~0r2taSax}Lt0&}+8g^k(|-t#JFR`bk^ha4G@OBy;D>LiWvPB|vNFSa)7*cmbrc z|BtM5J3PAl%YEg^mjp<<-?{mi!qn7MtdTnygZK`Eg07+0PESExG_gMrP}z)^0ucG4 z*zcfhwr(e6#e;(!NFQJ6?fW$AaYW0TJRK1k{~t%^9Z&WDNAXL@EM!L^<6ha>RNN3Z z+^ZPU*nwTk^Ciag+vT6 zPKx-g&N<$@ndp;Ls`G#zEs4e4pG8w#cKr~&cQ-GQfgs4(tg4)a4edflumomX*&?^Ic5Uzm z%n{wUu5Eo`-LRztm@ zsXub$8w_7IxHFznaF7e4A;#76D3DSS_Oiq|AkVQdtPp=-<{{h5JP(4vrhiXHo+o17 zLq;3q^?DF6RZTw6&$2eex%-BerMy@Sirk`9JLCPzPGpj%aMGFl$^$8{6>KK$SCbKU zUopm<5g4EjDeglOL%AMwyr}4b7(qjzMzU7l=v0)E34wd>KN+CQMs>n8$S_L*C$1u? z%qnV)VvRBG2GFO-hSQU&H(diw!iEd8Eg3`WJnPM1BIDD5yYR0@&dzx`OdViK))N0C zU41B&yhncf)B_pB`ixEHCTvZ7$#V_lo#Y#QZm*koY;Q&FoF=qM&Tnt<+r2e0o`s-h z-0HF`;P36*qw|0+2fTKI2dQm|C!W85J*)>mu;WHFKPtgF> z54x$ox%rlpbz2gSsI&h5JgWs~|3@ugIPt5Edr1GHeJv|1 z1FSJaZTLXXb9KH1&WGdU5r@VxhL58ow1b?=fWu6OIJ*76Yl#uqztUjg{YWSq9>-eFql#d`1Fk&z;Dlg&ZY@h1f>%qP zLBt$L2xDm+?Dt>r?;S=0X{RTpECgAo%8@AiJM0`ZrOm-5si|_6Lc0l3>+^$zDL=iq zY?XV%{<%{BB`brVg4WdEWIE@Phm7YrbXpA(lLQf9(3mh9_#hk^30GF6s*1$#ApR;j zj;2H)w7lf~-=j>_Z!>{KIF?U)Curhj!8Z)5eeX`pgGh9-<7*h_*JzD+Vp2027DV{V zMMRH~5B_tK4bQA1Yw|mfFx33Td#!+DMM!`~ll2y;K%=c3oc*<5h)LYFH{k5b2*rL7 zp+t@Mb6X{&l(d=Lb~OdwMs^GB4ZnRXX3SctM+5N+wV|YU%Yv#WL%SY~xqZf#+)g;G zOuXZ>@?QJ9xu7x8{#8MJh0NPF>+bTKll^Kz#B38qYu_t6B@to1T_s%#h={xgtQx0` zSPX}e4xNz5y|f>{HCJ%8&4A?0@+R4VNRILx&(zM#sYLf)HfRPFfwo(J^lb+y;GBkXRFn;G#K!|jU zp;~v8QN;7Cozp@Xms5;>2PfCLryarkjyqU9Fhz%9oc;X`(CvlRuf(#4*6HZzPEJq9 zsXs37NypT-w?6^wTygPt%8LM;umD^Kisc?flWb_mQJ{GM-aktt1dJ!Zx%vD2Jh(%j zsopoxnQ^;`!jlCh%*-#~3^(LUxyaYZMgF%GbKjYt8?def@ygyYIFm3)5l$N(ewRLj zzD5%J4Un+F2y8b9fE<8JDv%lx)3^5OBOyeo2Utje*vj5RRtZ^ZQ2DgUWe+qK&zOM^ zjmLcK@GD*6xP$68Ojp7G`eONc{lt6jR)84-eh;u?Tt8TN^X5&W&l*TpZM**G0jMdc z-y46GChOMLCU z2%?0q!S5iFQ2nI}SShCje(yp-Et4oz+D{oQrkWtgDlK_UR|z9xWjbHPGy_XS5yosy z#-Wl{TEr=cje`<`gU#fH*DlOSbeTC3e^afjK8He$dRUM7C$h~ zshz^AP8!_4>2=l12|ppwW92fo(yL}nkx}?z1qxsx;>#VIF;NR z!WaGXOvpaS=iVMeLC&Kq_MqdRCPiZdq49x*mjVf{o%EMU>P3H)+&sA?X zA9D!xDVUH@kG?FLEcuB+c1q4P7jAj?il~l!{jfWqwbq>JzK}8jkQ;uDC;v1oc)}`*AtIjZ`E9#D{ve@gF5vQf}3=1(6XwE+232md~HnF)iRB z1~DtZ>^;$GGCi2VvV8-R3}WNM@b8v7VZXh-K6$&`?su`7$ZhC*K9~4BR?@s)H~Pao zSlIzL(+xwSCv@NS+q~e7HLFMI=<0d{tdIS|5-2MX=U4w~a0boJ z>l4${|3$t8;I4qs8d#{Z!BLNsmIgH#Ad+NcOG`mh#y4$Z}qruKE$4LpgVP>vGk(#;8 zFc@KMY2ZZQo$Nabaxl)9!<7$vXpu-=4neI-f>T$96tooz(JA6ivjt5GgNJ4dfkto*axID+bBdX%(<8S@I>6>&HOhX!Z3zrzX|Yd+#8)o3&&Yi7ih2 zw`72!?q-K#2!~KV0hR$Pbqe!jl`u^%ybDHwQnX)a!euj<@16GLuNakSG&K7wQyE1i zeXL*a>2W9m*FVu!JBsJCcKOKI!$99$LFVYMqzr^4__7-8Bm{CsJ{1D+POIsvs$czX z`qb~l8thHohFzt4g3|`foIx?%_z~xYL2ciOcF+2TNIqk-biL?c0j8@%7MDgIy4C~+Ns}Tx7@Sv9 z^Y4=r_r?Xkqm6NZHgxOqyx3{HPPsmMy6nC6UlUb-cSc-H49Gk#F6U}@Ud#ZRBDg17 zLFM1pHX?Ba0$>2(667ly7~Be;{ylhJi&Iyb^!;~v*1FeD2FX-=?v*8KsVe0no31FJPhhH} zM(1HlXWjpO7yN_TiZx5^kWyA<_`z;wsGa>uWT^m0`tz>HBLZ1PLRxv!67-)-BN^uC zn_UBrB8=oWC+|&?5ZXA8q)Yk#@QTq)t(>Ay`9)8cwsVW^C(z4PtN~Z?q#$$OzQ})k zgAOBPiB{nx4Dz>quZh1V8Y4tC(oq7SrxlL|kYOuc2KBl?d-E*6!^Yl0=AI^ko{r9U zxsNmr*#7I2XthF+2-QiM$B)aWY(RxRt2~+&<@is>0<`p&3flLfNlE6sMWn$Hig$#?sAgWQ zNNwudfQyApi>>_FC*FU51pNuoEf-?~!#ZQO^f!8-?|fx& zlR(~?EA~OHXk@Y$ARGL1krI;93OD*qs+B~SHeLUnS6+LAxUKz=cPW(yBv(MZ@nia~ z`04H2;MHhZ{EzVA@V#jtfTnNe27p&>zzk5};)H;KMj5P)tH@}lgQ>);cx|cO^Vz3V zV8r;DHjxD*xAMQzu6ox_Bb9H(mY)is@EU1XuZ%(Q7)o@p4K=BK^q%s0C6rD1+=VN zK#RI)Ws_EDtvf305-KxUuOjvk-JcSPK^>i-5TG`QkeqH7d-`;bU}^pU!K#p}eWYO3 zI(mM7ZY7hIoqhink$}OcV$2{9O{lNG3%(ufnA1?tUjc5EAF6FJje@OQ0i~Ik&dV9}7Mh6&u9gIJ}Pg^6o?DVefN&HU` z%mHSp3p*{Ejmw(=6(!XB=}T|P2b7I)Y8j0!&!|#~L_nBZHgP*ot`uX4R%QCe?)!AU zzWy6hxZuM;AvOY~5+FXPp4Ac`;82Ezet`*3K{ZK51_L$l%|xon(gt|Q&I=Y8Jo-X} z#(uBc6&4ZDVnXFfLap_UHtc(7@Mw8*&|#5*?6QfB*M!gqPTDG{TCeZo0+f)zxzNK| zKeZDvvb22qKZdP;{(c*Y6MtO5VJ96I?)6;sq2L$1KloC5;d5+?Cua3Ok(xP9yTu~z zx27ibmGKrs)(*za0=s6$&z)Sz?{OPw^LPa+rL{u_1O;trlTg?j5OY|&wDJwMnNW$WeOj z6bn%QeVVeNg2$1|ohGRj)wZ;FySml^Gs;K^@=f%gXfGqgW{RPI>KG2s$jb{rJRUs% z=Fs@?m{6n(%ug7H;1NqGKRgKj7?8GzsvZLS@VvIZZM`l4&3Jj8ZEPq7Pq;lu90L?> zNHO2HP)cA)@bU4{&4Bj}bPV>+cXq=4tewCpCvl8~jq9Xt$|hHxduazA?)xNr%~t9T2nYP~f<-2JF43=3hybZSym zD?aoyO_W&S)@*+`e#KbJ%s^I7s(F{rXMRz5{O7MEY$Nkco7;MBOrZauby zI;+`7@G41#{R_kC`%NDHQzcWsEom*=!K`-BqkSIw2+4*vOX%MA|9S6UY55&^I3jI} z3jfPQYA?;oopI599i`{r!DmtC)L3R(`TOy&H`5y#YC>)B1R0hU0(bNy$xST;%Ow2H*W#f$b^?0!~#v1vt8%0_WR$0dHKU(OZ)-V`WgyEemfo74}uTV@7 z&E!{-z6$b!WT8~lqT)kCUcIGHN7A29TR|in33KDZPDN#9Wm)AlfWM@g>m+X0E$rl_ zkSUVOQ5r4b=W5Ql0R#KQ*OvgRweM-S4pbvlzzy8u_Aq-+ruvjQ$^Chj7a^+AwM7~HydQB+iQ5Uh zdG*Cj(j9;(nOM!UlrD78o1gvUw5AB{S-d#C_t=|;$7>yMkCr}(Ru_T8S(@Eg-@%e}$!G5|MD_@bR^Z`4&7ihnCDP)1=KOJLHRJyF9!LQIDhB`?^nC*I z3DQSuK|Kb-qwQQ=4#A}Z4+l^*F#Jbg@gZ+g8a|&Ns8vcs&4!8>x@6pQkqj|uYx}eP zw9ky9P_(vP7!FS^!}i;gd)+_owROQX5qO$xxC|G4;``b8teBjHB2zKE5gJ)4I7~LY ziVIEr>}mGU`9w+@UYYF}cW^viTXj4qg(Ae!XCc^V$E z@B@w?obrfY%)+~;99V)ufwI6PCuMohl|O6ZE1%sXm{0fV3%JJP?zp_t4k^N{M&+I{ z&)=SF$h`45Wuz2I6>M{lE?bi^44o47rdpp^3Y{ziT>vr4D*i!$Qy^!JKswu%v2 z5?lq7NtCxQFcSqmXlmhZ@ijN?VG%j#t#NyqvSv)!AueB=-qkI6rAMO^7y? zPy?7U&R9xiqQh@i8E9H+3f8LMq{M;<@Y^#b<@TRHNO5u)cB=*0J^sU{0(~uz!C=Ks2e`8Zd+AbAR3<;U?=a0A zgR`x}DDeB0l?Zz^ih=Kya4LD)7a&M8z}o2~?_DID6Sf0EGD{yGWcd_8kNJkR1C-JbkK8?6tp2h`app1H$6)cnkm?|05LxS_J6&_zl~? zTZxc&8J)Nu9-IK?e6!w@$B!4a_3yyJDoDof;3v?0ecb^PFPulfsmC&SN%sCtdw`hG z^TX_G@K@%%$~Iik-lMqr(bjgu0O*>lq2EqFf$Rsj!vqZQPO&hvWZlb#;wO9o(MYwH z(gL6v9tUAXKjJ8B8XDZUeyM6s8p`c!nWVNDp6 zJUFnp-})JB-P53(`VnHaPN#eEAt#1|8jeNYG;-{X;J*q)nj=f`*1iyH@{)1#74`b5 zhcVW6UO41-9(2=aWy7IZWDXh?MBUCIm0MWysKPoZ`&UKc^Lv`>o1VW(h~WD`96{B?v(w5_HPA6iVML4TZ~UmS}a$@vu^DP{N_6Mc?TCQjBd>xQfawzh=k%Qm{G>-W*Y z;N-8+C)_e_$wa7{3(xhk+Q z&bQylB|z|gUCi}6KEC_(dh0VDVyVeJe`ZUl7k`1}BC#w<$Sr?iPcm`SRV0~!T%R6A zisVVE>$?ynmSO7#1BvVZ;*Rfw^(&Y*0~}<;PC8iTudJ+ma{}~6N@W0+=gz0v<@OHJ zNqz~XsS3gYRRK#MjD{4%X9@j6|2NAlv<5mX?S=9!DQ_b}xwxVAs@J6zTN~x2)=auB z>P4VJsA~Lua3HKG118zTK}*fa=DAe)IIq97s6qGq)G;$B&8#g_HpQ!s_w)h{jg4FN zX9y5iVEFF$iB1!`#Eb0I;?NK3amd--^HcnC7- zf=@^baI^kJH=F{n{sEL>xr%cu9db*9mw)@7RwO?JLa+E!L{-yOxHg0>Qgx7Z+R#()w%o-LGvOU~ z5bm`kGLoO4k44pwTYYM%k=*onv9+-DZR$g(-cu~i)ld}&u9M}XNkv4DDMZbeN@8D9 z)#MG92KQZSkPJWEau`m`g!;z$aFEzi3oj`F;z;+n0Zt@OMS@mVge7XhNTt@vcpVG& z!Vu}i^Dg1$TWF0;O~~u8EOJKELSy0HOV*4Q&BS3}BLfnv%~L#fmirkuPs!Bx1NL+O zwemg^BWe_TpNS=Z7I8(3htXo@TNf-?&2af4Kdczs{UM8WGc)Zmp1hnu;dEyM{8ARt5>y>1kpH3gp!^po^2Z3 zfT$KmsH_)Y%I%RLT%C(4^&D2==azhX;Ez`*)RssN5QEtg0atv1S@Zop@=!19pGvR3 zd{i%C(COzK|P$KIhjx}r!(b!YmuWRlhv!TnC!p-pmcZ$68sOV^UFgWSn zx$j7HWM^lC{I1>T8N+Ax$KNkyN@-o1Gu9+LxAK1f`umsRA_v%CL5d<6KKNa&(`##+ zrCC%ifO=<0mvQ;n49bJ8p{_ykJoUenAn)2fzRL<^64cJK_kZ{a>*?w0>dR#x0Y?1t zsi^9+9uDlc(qPyP37@AfEl$ABb_;<>Q2+ z6BF|VkFb~|gtf6h6^{Rq?0CoawYZr0TwVvbR=gVl3*2 zBxnA_9;NSxGBO$)STkuqm68emXMXXrl&k7&wJ_> z4EW3r=+?e}Z7LuLhaU)Hap0 zaGZ@r`-ksB5p^>5t}!oK8X`4`Fv>7S%J6C*#q#4J8pUsY0$nOwwWM+(F_Nf*={jtmPFwr>oa#cKg z$^WAKtDA07kyoMb@&mXtYx|bdApY2?zQ9N8Tnqw%p94kO#aJ<5eY1obs;l?^bKo7% z7>?1B0pt3skKocp0ETH^q4g|l2@+bTTaGCC=>_r-7$gmV7f`p{#D~<3zS^1Ao@HFB z5DA@a03_4Eku&lxFjE7|&R&*pU!}%R{am9#@<;x7xJ%a1`hT82(A>&@0hV}65)#8n zk>??J?uW9uqh=xii(r>2S~V?Vsyi#XaQMmN^Kp4|v*fcz3~I#vqc-n!3LOcXgao); zKmlG}S=pF2VpHEzQ|IgGhy=nSgSJTj4KTN!tTU-DhL1D30x8nFwtv9BT(8S#<^r-H zdzMBWz>0KkPVPkpXv)F0^MdDuc0iX=gEuoPi+^}5J0nAD$XsnRXJOp!@zIA-&t@cZ ze~0UD?G~r1*2ac9#R8j*j8J2SrlrmCY7BMa0$3FV-T#p)ga_8VZQ8jQ8@+tA(SvD` zWrhxK(mEyK+LVNe{!zl!VU7syzQ9GAyc`6p2TyPulr;)J@GcsIg2do^$9)M;K=j5i z$p#9;CDscS2{i}`>d;%~|DepLT=v(Vv-jg`>YdGBJY25{wNvWvuMYY08qLr`w8FdP zvl1V`IK8jY7Pw8HS4S52cbU1(rQjHkwOWC@nnQ6=Q1uq;>I)dy;y{%-lOLD8{OcR6#J+?V% z^@+}3Ozi5^!%o_~%Zih;LXL8fOSiRI3J|r93Fxv!^}WJ=;U?hWuJuO$M-lPAWFR?` z@0{H_Ht^Z02R?R|+Z0`ZK|(lRmz9wLB!TSu??V*;#3S4T01x>2^X~FZ+|s5A$=U=} zf(X;j_BK#Q0zxSeV}U-fjEka+vNpb$e@2ADrMd7ekcnfp+6=_q9*BVWg&!>-!wIhh z*y?hT+IrgBZ{B>I0qKcN+EB2eKVOB5W#?o53jxB`G&RlI)Ia(ABLvJ)t}hR>1x9Cp z7PD|I!k7VkgIxfBoWzZZsJ`id5#rkSuBQwyfg$y=|Ix;0K+Ng`w!|oF8RQ&8%GfY-?#xBRH{EeIPVq5FwC4Sqbq zK@^h8$2mt+D1wuOl*L%XRtaiO4vnLPLg6$oDZO)1q>?tsDCbVuB9*z({+cpc+h*C6P(uc!4&>X0K^FBCLyQ;fWZBa$SB3}g>oiPQb5lhUmHCmtFGc91>2 zeq-X!CLa}@Q20$UtuWLWlkrVtL_`%%vqak2CsxOXRD<;tXZC9O4DWC}ch-oAzlZsk zyp5-feQEt0rI4dOVSGvs;8PD{)pU4tpK~7xRv6@$gx*1jQlL1ZFw|%cG+JKv6UBG` ziMDVq!<2~@4(AFo`rDj-B$G+oGSQO)aZ#30Y>>8tD9#7af)(k1$4R*!G?ba2DN%!r zb9p~H2RP8h5{!WhDkqc_CvLsXa!G}!JIZQ9!!K>TdpA^{r{$i8-QZd=yZ%$2-nc8% zw5Ph=Gk;mz7Yjh&@NYOAg=!;72nN%_LR=ZSyYUlg z>4$Xh!7{#Ozjv#c|GIqnygVcXtWkO;$1H09n}b`D4^ZXDyUe&<^~`~3PPC$_aeEl^ zV-($NMH=M+059Ao3Nfh;w>jE$jr>(N+j?_I4i8dW2@eP9N=;1wa*`$E6`ZGNM?AIT zKCUrQ1N3c$)?iQsLIQOjnlk?CSo&uq*c5;RdUufPyb;s->C<Wo?g$WbB}Nd#N4;7X??v~0 z==B91tk2>uAY+?D2YaZobtoS5Qfz1_;m`B0vch1NDvZ6JZsFyL%ze$b|N zjy#x^_%TNZa64uEa+aitNQtDKaU%;gVF&}-%D}v#k@4~I@)?7^W(}XcCT+uNIOw27 zDmY(CQi%)|PbnTn6)W{8<#0ftFgXY^m}T+L<^!~nv%e^|X=_l&?PE9J9k|}u8kqh^ zfj>bUr$U9#!vR;0!!OPwh2Lu|UP+D0lc(>D;~7$UcNcy~uV(bn&)JUr_f*f2?Tfp% z-a_I|nd7cy!oP-M?`GUy5#X(i+PDL0rhIoLE>Qb{B@c4BswSJU4~Yviwo2pDHw;Gx zb3{3gKG+I#nT*8X@y_#ebtXZ1WA!Wt$W!t{t8`NT@%~U@gGJ;YlieVSFn^>xf?vR_ zmB6FP%g}-E5G5N*Ntym>NMhSu)@1eRxfvX9CB})V=U5qb$Ds7c=;R>P_soRbQli6; zT77ro;qd5KGwYSf@GZw(G?&2SBSRTQHu=@#epKd(1-ZJ6;Ip!qO1*-vk7VzRr0Gy# zf3%2^d@?SYv3BLUGT!i(=NS?d7Z*!g1c`HrE5!h=wlH5yc$QslE!eL?QQcpEleEf?d=6x zmZB*@NlZ5Wt|sK4=+%7cYeWReLE$wqTWAo#QS(6@wxru^L7TP8pW~jTHTeQjS9r zvntoxD%OM%KNI+KDaeZc8C7P?knlxrYnUw4cvdTDPknm=9&O2&7^keKC)L%+aC1Wa zYKS$iESa~`cjG?ty}^#`SSHrk;JG%<`;L6U_YQTn;dLto-=%59R>VFUbS z%D!~ZD+6~gm4J+b&OgMZ7Vg3vUeR)TW$E|4b=}KU2>5`7!Gi>+9&N=+8xcF86 z22$11aDw}MCTcqD^4+r9b7vN*^-5|ulIa4-^O-@~I`URTQH#qGc$;0gHbt|@}x7_q+z^sJPRN@augCKw# zgW9L5WXAAwn4kQ@4$y_m3(vbP{vZCKj~)drCm- z1-Hf6Czn$oRmX44$MsqcBp8?pg9WB8e{1_PFu8UG(f?hPTR#5e`{Ze=3Mex4_06L| z3Ym(jua}pkY1;DHt_S_^V;~|QJ1orKHxpKta?8lfC@U+I@w!9{xz`0ag4zoF&^UsH zV`FP-_ZskHjg3j}egPjx1Hsa#qo?N!;2hIQFMy#03C(Dm6-Yw(Nc+5|n_(~&j%)w( zj4BWX^_&rjIh=W&cXPG=%fD~hF=)4Y@RdnSg8?1_2*;Jbg$-fyOIOjTK}l{zm^_j- z3epo5pAo@=PCyGn7}U5kTplJ?3#SI-KtRHZnaqCSl&=u04acX^{W;eIn^;y{G*a>saF0jBLm2>(7pGV%$dgGk{@z6cSuuH*J#f!?Sp+1(RE` z^-2iHj(9b1b4IbGr|Y3}dz+AvP$MgDewvq&VbpM11}Ut(DYIq?J2@5wCsh;8@K@M< zDD)>{_Vb6rvji=3UL5m+`i+U@>UgeriC@$|M{d0Ssw74N#qYR-vqsyXk(3wQ9q&}`N2p97+bdr&YA@I&AUs{ZI9R|eWClOtdMcS(NxjT~9o0pZ9uDMjrI_hZ9GkIH7K-j9yguGt=@hz!+T zs5LGx1XQV$`xu*Z^5XnBJx&aMl6pn&kF1+p+O>XjW$w$8V#9`d>sa>wTQVN6k%gsX(;}*5I zQuRaYgN}CUb{^*TrU3zlLuP4h28G(>#FH+emaPJ=nxD_-kHP9G!q$hqpT@Z_TJvGH zXw56)8n3VJ_?1uZ5wS~HXO$Z)xiaGbAzgbm2rmx!1h-7w2^Lfg+q{p#kOOG@`pTy- zFE4*jPd~r!_p{ZY=CvwV<12(~?|iLD-ZMjQV@Id0}X+BSV2vu{<`-oS-s zh*cEB0{K`OUSYET&PaMu9|dQB@A&5 zFZp~9m_E3Qcs|wV@wu+dj_>u)mUy-nQg|>IKT7Foj-Y8V7NxAH)|H;5L7G#53W6vQ z-=v~H(ySx>>PA}|3ebAW5aKQ*NstTv`?(*=tFWOOj+Kfn1f)TiS z5oF9z$cjPrN*YJqaC{-I#09*TUA^ruWKzOJC=s^NQj^cu-nC-4Qe5gIUkyepNgxe! zsJ=?Pl*P{_$-6O0h{PA!{W)pMmW!@+X{yAKcd~{)qT+za7L(96ItD`h2@&CYtr`R> zwC}MCD4vcZ;yD3UKYY6Oy%)-&Mhp!__5-Z6yhVky7|NrCj2%Yz+y|ry1619egDIay zmc}I-qT@4r_>Ud$4Y7aMA`al#G)(K&jGCspN_?setGKL>D4mTCcJ`1uG2xv}oIcSIswvR>3!V!?a2;2+7k3*@Tv{6RI_$#ga8i2@vbfSrDuBudUFm3@GPh6{XR5h^s!2!-y z_LNFc)d_2FHPc1d#S3p*dYasKJJvS0BxF z=t%3NGu3jDj?6bR`(D3#CD_jrt=Rt=03lh4L7y0Te@6o_$8wQXK72zp?Ux5b@La(J zu++^5rOowa(bLTA?0};UjdCe*26En^7Pl<`Jy4L8LHw%jOGJ@oPE&L-M02I85cSCX z7fHB$IkF5^TebUuG21B*0fiw{vJZ{t&fd+~y8!;csPrB|d-ni(OR5Q0p@9!JN-)X> z`xGE^Z*FR7kzydnpgu^3cW-}}Wd;~c(7*sz2ka$J5;5DxVBCmB_G6!hPoJXvhjQ1H zVBpJ~U$s-=*TPFJMR~glF+76|i7UhYJbtyDG``N+4_rCxY|hz&XCpYtpYcFd&G$_D z)2nZ7&d?*3N}vKoCc5{zV~FtNFeV~eC}h<$L!w+;>Gr)nzXKh6uj+p`A%=CeSLK^3BlafA7*O5F@e3>vtLfc>~Ct4MOHG^fH9>LhN_) zFhKeis6iK0^9c!uLtg+xCJHmo;FX^>sjZ?8dv?r_g}nDrJ>di9lTE~6Flt*2OY{CX z-a2*D{^#&ASHx~8<>tM)0U>-yQB=H@=q0-%lpFKt-kxC3+rD|NwBQbc={xrx=%*0u zSM3YmN|?R%h@xHOl{ObW(E}=Fi>y%uEP=-yA3}|QkERwqPvDBkRwKdU^DT9D|n#!8|h3LIjE9J}aEZBW{bpRa1%?cO*(;kfWrDRG4vl3q%5r zjz%`kJv}|(qy{78r%#t18d>_}vIuPFuoDo7oZe(zSJ$&`gG>O@$86MYcrA6kyA9^V zVAt~dH_s~>kn(**u=M>qxK>0|k6fNZ>h15gCRqqGk}qj^d|}vfZv_p*Oog7pQh>EJ z2{y6N0f5c{arw(}Tw5x5YR`{K3aty{JGAVpwSnRAxow(5qpi*~h__@j1j<~X5wmVw z@MymzXJg=|R1yQ~EETR0a%y22aUdJ?yEp-6i`sD_2rLwnfLw2yTyflJy~PID-wqPn zwoH%6Lxde4-+d`E&iPej>^RQdP+z}i6}HVfyK^Lji)t8XTxx=`2&4?*jbC+m+S)$c zP?H%ugf1{xstaLC@^^Dj!s_5-F!Iehy7fBqdz*L%XMt4xl zmVkDq1AD(0`YWXWdFYD)hDio)B4eNfm?RIe$#5CXDB1DJ+Tp=!QIbCckvy+zGjuYB zrg@t6xcd0U+(b*E8mK|ph`68$2|?2K%2C`3f48F}-Y%zF4Q(JqK~{nH5g`$RS~f;E z@8?YnqF^bbJDwLW+5dp*;e$IrXPXsagd|?<$_H~+hWWSe1lJL=(O7+u8WL%@nn(k6Ia=*`{~s5tz0MQN5i8rzZJew(fh7i*0RZ27f5w+hzFALk zi<+(5Q$~@Iy=G;ygXZcJsQa;~&3fr9oGMo$2s61}+Zjva7)UroB9Ajn)6@5MtDl>h znOVFvx7VMVBFq6QxbF2#BB6re2lsbnzEiE-S%x~B|D!aQ|0~E&5K0q>c@YO8!S`!z zZL!^&caHWuUUaXm9S%v;(KdUP$V_Ufg2-ck{C#maCNsbE`SIjlzQoL_-$dE$!omaV zSEREMj6k`w;#iD9Onz&yFBe_kFPel+VTqq6o9m<%qCbPClJNP-_S~ah0IBgl`3c%o zqWDjSAmzp0{y&C>=Z3^6h@5eDcG>k>3JH0BWM8p`GENpu1H?@M!*=GQcOV%i_WskQ zxjEPbNR_oJ2SSAYR$wMlD|*Bf4N{tMUdiT6WY^eQz^CBzULqz!FN*g9 zlu<3@2^`f0un9ZKbW`?sA=Nz!ZJt;Rb5nDAY=xz;!dyMASac1uqmyOBjGI7L+H?Kg zJXgl?Jmje+dMO2eeITY&%iC28^=ocbJp7tgT4jy(OHM9U!W9yvj<8FKoCF-Q=t! zx(VU6=&*f~e+y-B%J@mRQ`IGkf~x)XukrG1hffnC8zUcBAc48lx;)tEaU%xPcHCTr zsgDCqpBz(>*DulqkMAQZ5~`CQH-1?>Y1J9UX&41=LhB_O;~s@%;n*xE9nJM@B{t^t zpT@&D2T2otAqU+S!6*P3Yk0ygYgROwt2lV(JIat>7b4Vq>end;&gK@|h@530H@Ct7deeW(6+pod?hR(m zxt9$FAopUWGe@VVUonJ<2+QjShI?gADsgGpImwaZUHLDsUcP;L?F+cyE_!2!b<1cde7`mGzuC5(wQ`+73sh59flv9GAb^0{N~O!MX3>3%ZAZ_^V$5 zEQx?7Z22^zCrTSoIryVAbbgIHbh+8HeV{B`u&4Lp;=R|mZ{=bO@Cih1?yYVh-5P|! zwRr4h{W!4`d&5f?6>)rgoSqKMdHEc)^`C@$r^Ushg#KFwpxGyZX!PseG4E|=he z5K>B}yv82Kw!~;?vOpl%T@SanV9^KqJDVcCgn z+3y#>7~Y1IkM%d9r0AMo&?5HAJx-Vy<#p>0wd+4;|yu+|KT^&YKsmLz&nXj@jy zFr;?Zv3V*Ln%0dG_71&2!VuC@S0v&ify}+kVXM5V4X;7_0qu}J}=WLVNJp4*I zmokxe=V(60eBEnz-rZPaCI<^IAzaH^L%lMV%GPK?hjTm;XTCKfCJx4j!2B=t3)HcG zYtiC#?+Gb{R}{h6ue6>k^(*fSW{)na5YN%()$2AO?gI ztvEsmw6x5EhFOJ}M|$X&buyD%#j0k`)s{75UItVTv1CZb?aV zf78@h9*TJDt96&+icfALukuHI1bdal^M@0)OcgQ_{c**2C^QK1m^6RA;pJyGW;>I^ zzGyTK)cjXCtb5~;(cf~Y?Z}r=awwD*JCurNHG8W4Wq#vh>9f+*HDNSM8IuUK^0}FE z8PE0fN#i%q;9i8R2Tath>A-` zw=r;>KhEV}>*tHs>)(b~ANM`=l)M1i@V`eS;2$nPjT2!y41m*iV)ccYU6atM}WKwcu+@; zC?Mb+27b1!I@ca?pinUqWc~RMl=;c#1P~T-qHi^>ApG~^YSUVyhvMCcKkv=0FnAV7 z%r8-KYRPwcVy2ni&$ZQM^pBax<*v)NKB|7RKDj$DqiqVVF$5b&dSVRR)FwvU7V@z%jdi|P^xRANyO)D3VWZXG>y5c7&X)!03 zTu_rICwL+}ia-72X4(gekK-1B@@_^t$vw@5_r@a!8JNVEGw5C#Ill???lD-?7uWne z;oByOoqhYI_uCs%-p1m<-W;J&7$ueD%eRAaMA(`0XQR!EP_|y0`#I+wU5yt8;{6FD z^{QPfn=9|(15ncDNnLahU6i!D$V0kX$-0b=UoC4A$Q~1|+bRhhV~1du2qqLOjvorG zlX2kq8_RmUFgxoBPRwkO52t5n_}1PY8!P!YzyNFy)@h)iWnp1CYro!UTqXs% z=PS1-MMPf?jx*fp6EdH`9eOeMt?e^Fbjzomo9pZ^R?j1alFb?0CiYH&u)7$W2)F2S zfjox_&<3@4JA024Izb)}04^^>Hd^nQh_&nDwX?xi~pdDNw!Cq|!hr z)8gBRrj1wxdIfyt6bCOvG8U5z6y*Qo=)A+Je*Z9jiewhDS5~t3ri_ryvG?8_d&{12 z$UL^JV~>nuk0d17EBl*0LP$2h$IsP2T~~EE`h1>ozwi5X&n+(+aNf7kW!HB3gE@mu z!{E`;uYe*vz%gF)Vg=BYZrzVNAR7#u+*LsTkaXDmr@JC-5O<)*=DvA#af_Pm*UFhlj8A zHU_KY*Zea0xE1RYEU}8U7xV&44E#?=H-lZ%<7A+#ePr)Hvt}??c8x)JO0=K1zv|^U zehwiRJejyW{MY5)nVHiDM>#FoX?KgK@P zks8wafGV}U-w7lQ7}zJQ`0sz>KlpO^1@z%jH6 zCwRr|P=sTjx|wrjGE~Q2sVV}6vrExzSa5zXLp4yNh3k? zM1WAQP`|m1g-@G~Jl|0)we$4V2W&#G>hdt9w3pS-B1_t&cLo2O?m3|89{hgN{ zYX`2j9RkPgSJ~}Py$|uONAf4VTFfw<7BDO?m;TxDf3J564!#5D(@qnv*uqhXyPR!r zmf*tF%NtNLcyi(~@%Cm=Er>oMrqB-H;}a~yGM}YffslT@oI>(>7dS|OOIn%Q>G^qv zuQE&7^C!`aLUo;;fu5c+*zeX=$t=Oh$&E@P_*Ps+6GBV!Kyi3-0$)0jvCoblFw3Rz zJAhT@_wOHGOD3i5&CO^37F-6Tn33%DIS)4dUQ<1>owUzlG|bLk@_vkme`7`Qaxz;X z&)SVh1%TFbIqgoaclZqTKHlt3GR9sl(a@Qf75v@Zjg`4&-YG`i9<(1a!+1ZArVk-C zDkgVwa&wa+!1lb7{;MOeHFQ{;-?9a3P(x|fCp+G+d z=NeM4PAw&$NEI_thCOTjCv^^*xcY%GFf z$#~_JluQ2?R`lO|hBC2%0u0xM8|~RYL&f)PRbw-M4a?$Z2qDZjDt@HZ0t&8Uduc1I zC!SSM-bW$4dtugz3EG>+x&aUU)pC33K0KONKVvYj^(9_HgXkEm|M7nX`HWA1w?#Cr zc@9xONOoRw^Ws;5-Ne?Q7VPvh4T2mcDlRqreJ`9i5`V7%}38eS-lz4j+$(t*e z`r|vx5AG{(j53B(G>$o^!}Rkc@k*@yj0co@ncvfAAMKyU%ae|deyC?$^7s94)(H%z z+rtA1YmL-~EMD&J?yjzX0zd^uj>l?HYg=1AESpCzCjmfm!HgY<#ezr~`DY9Y$pF3C zxbPa#5_^A@nI6`Nj|SXci)>wkauUFdX2f`#EPHc2#I@qLMg6@AEd2L(H~{z5rUDov z3AwKn&8oZsz-<8RB184_YrOrK@!&8F7Io2}YzD9lFM!mBvMrQI1c3*Ug$g&}5ihAD ztmU@gb;7DA;y%XdJQ$xb8J`UtrMy(@y4a)~b%bhxoV*9pED^Rr9PGE)xHEG(pR}xXV29yj=)JDd`zv3 zrInS@`c+_Lk4=91`aEP_2e{7dV}M_*MbE^@sG|;5+H9dz7uRqybnaU!(Io8SOCIDl z0vp;7Q`lrHTPw!6Govo(=GEO>a?n?_)}GJ*9ywF@Z4alw+dL@s5u=! zihhVP^?QVfPdM_GMHzuqcwG!Kr#Q4xJ&NJIsge?~M#^Ded-Z6AQp{1AcH%5;huGh0 zn2OHCH7g>>G;LN`ctTDO)%iTa%g&f$YHb%>q75B$x4^#HmPkQf$n%364bJlb7c#+5 zT+Zeij&|SCU5=Xt(a}i1t<=K_Mp%A+*FDXR~B$(O$C;j5`*J|CJq1KVkxo~}bYJIvVI6{2@Qb|X z=xo(J^%*nVH1H>;G|^H{`ZL4B!%l($Te&_UN%SoM+)<5;mwqRpng7_D>L*D=sm27kM=5*@0lk9^DiqT|YUdpT#Jcpb2XqBZZX7ihPf9^D0C^LVAX3HU)cRv zK$^U#1+q@@svrJE`r)P-S$0+jyie&!M1Cq$(2^;tVi4bmqRyPBK?DbO}hveIOO8ffQBy~Owx3d7QB;_0dtsOod0stMC> z?8sR(dW%zy+7ba?H+LV$U~r8A{U>C&C6hZ6mv8U3)$abPf%GO|rGZrLu+7)gpND6r zUGL`hy}=Tq_JI>W5MWtpWnBuCH?>1;Dxp&ef*xPWLG4K>I^%AuYO9_e!ZMGm@yAo)sF*Y)yb>}73M;LwIV^ly4A(JnL zha(s$G|QTr<{B%m=vz-8pn2e@R#X1#*{+(rYTunhs{ z0e)}{p3I=*+Q4Q*c4>@zxKj%STyx!CXI+t3do$E=$nY(9Ns@z0|9>bbJ*9Tx)FA@H zG;{e>aSD$)!T$4eePUgEi0Wp{b96|#b^&pvXDkobTYP5Z=L~E8D(4ez16{Qd5|UOG9i)W=SD4}I-k2Hd8}r6IXsD-u4B;T&GUAa#1Rf4Y8$6&l zF*Vg(*|Fna%>goQLxYD~pls3-2<(jp-+I*N5q$uaD6!?FLz%Dza54)Fq=uwwfB{0W z70-rH$1Q{anR?IT28*cMLru$};7hf;owK0BGox8#MigWXA{l9voI$(FL65`W45{`? zX0)pQ_TaC7CnG$j^FQ#wvNBJ7odR)yC1*l8frpGH{h&+-mFmu$2__NZ=?H7Ux|B69 zOr8JvQ$9KM8*WtZFZ!I93cT6F@;zx4j~eFS_}d96# zGfWm-niu-b5dBckDkS{~*MQsMio_>&I;4(1;EA4|3RuGZ>w&HyB-9LEzKdVc&&LI~ zL7c5-52zDezDE`aLUWei@s>=}&C^O_`FUi3iHRhC>c3L-~Lh?IF@^0eX>$D9O0%iqW0*^2l+GDfIx z4iCMA#sB7?F}+dDz`{F;h|%&W_#?Y@#3v4O!)k9^ag)PS|9fWT^0M4lakY3J zzr5&Qv8!WqPToMi>Av0TX3FX33Ob7OZtn!9O%zI7Q-s12#J8vi=tN1wJjg7|PCdEF z@MFPf%;tS|$*LWHmNJvfnf$jeX8;Vjxf%5>=>3p2K;k38^EzS38Mf#*1XUFi>_f+0p93%GN~xACAb3!L^^nq z|Kw|ZNRxZEKHjvC*g6I1+tP(Y&#R>xYkj{g0Hob5qYrmpdbKMH9XM1e`uja@xO>K0 zdxm*aXv+_#{YU6G+INYais7#DFcVQiom*S7hlS>4BA3kXY+99tja)~P`0%MX5)j-u z3Qw{MENH#TH;mg8{p0_{(l?SnnqkZRL0nj{N!Lat6OlZsC{(D3hbJUSXZc94s{a!^ zDJ?A3nXRmGc&d<~-#sd%OkhiVwmy-trH<8kL41^>SEX-EnSBO=z6dVMc6wwN7`doSmvY^W=g1MRJG$wKDgtkv-|KSInui*UKc=PcPse48f` znaNUwcbY`PS46l>_cpw~kQtj09qOJf>hLid8SR54xRu$whw1@z(yco?!r zyYi??k~~UtjbUH2+q?Y?aZTL~aCm`;!O)RZAI!_g)R`Qk=~WxLP>?7$?It$(U? zxOElDok6k8xL=-&lheD)ozk=CQ~+rN zfYgsJ49Q#+o3r+q)XT~H4_8(4G8+LEeS{@H4)KV!zdO%A&1o31EpUfP zh4WN+YpJMdK0lq-2}V^sGSTP#a)hQa`Vyj5g@%qQJN{iOg7k&pD|eCP-P%K!p7Kg0ZD4zlW0Wj}wjCsx!ObJ1`wX|m)H7MHr`)(@ z<4$EY5W8dNHiXecTw_PbG`hw}6{?Q5}MC4Z&^@#!KID0Ry9S-t>R^pC9^hvP+W zLAOotQPa%<@a2FD2~uP2=={&0KY#yv{xmI}*m(CkT|)!*i~uWV_8yx^gQ~!g`mc>s zXPC=A3ur8$A|`RF?(UfD&RsN@N#Xda`|b*eMdoB9TVTxO>*vwg-Tf3Y6)AH(_m$Sv z#gN=`%7Q|69ALwwUZQ4zRRH4p?%m?ySz4+-fiAP>x$j*r&HBuqv^l zLz!K$;<*4r-*mBsKtG>g3Ac@zh2e(K(7rAa>K=M@At zLlX1fAr!>f!a|UENWUHXWC2A?!BYrauDWb)fNZQ}?Hv2TXYLVxJL}K1Q~T6b(4Fi( z_!ygDXCqqkXOf4_c5kTTnyX7!ie#}N(n`{H24;Ht{@R#|v>|Ih^%1SkapCE5S>(DK z2%adyZBc|h!XUwwLl-k?t2K&Ow+y9|PiEN!W0NId1`FVjMMy=WEY!D2Q(M$!Ki*uK zfvBEak+0uri13>-CtoH?d~_2LAI8G1!Gk8al@qtPE_r|p$TFzp$nWe!(l{IoCALgM zB_`_?X8GhV%br2VYKq6F=_?bk2REjWe|6hFNkUG$t2LxNod-c;6#aW(_2Xarlbui_6*=a65a+ zEG6y7zB8BT>7o>^#EIXb%QD8m&0n!;y|6)3|HGRl*%Jxl=`ZH&`wuIxHr3+Ak~o^B zq=(Gcis83ENw#vx`Ai?hfRq2eXylaQM$Mk+l3+=a3niMI?IY$LgEqT3JqNPpk*8c; zpyZnK@Wa3tc`+3PS1n?ma@|H9CKc-t4z$8#@P^zXuZ1ci)R%# zKdktk>>LeE?yWtQxK^*f_IH~*^u$SvJW|}TI9wh9qLDLaEsePEpCz9auN%R~zwsUe zkK4^S5Ft(gI~Uxr92dNfTi^M;0~E;5WLHn`j%x2aVpC2U_B)r12HBO!!pH3rtw9Ny zdvI{~>iOUN%_y`C@jJ?a2lOX+cR8@Vjp!%Z#Ghs8b4?e zN{N%|j*lU2n4YFzoL>NB8p^YSt#9!>ueWLrga7nC=k1ZLM7}!ek3MvEa>B(3|9JO1 z9cT&QdsH*&>IlBOZGr(i5bMD;*WK)$Ve>*D&c5$Gn09;g z_nkxeexo4>jOkbF018{<;FU!1(csEeKrmO#v+A#%oo`#MmToUkopw-xgKHzcM#cKJ zW8EFCL5;zQQErVHL(KDrrap^^pNgdELZni+R}$yvbGF^f{?|dyu_oMg>SBCi6i zCy*E`IX@I7^CT(UO~x5l6Pu&QA+DCpU8<9UU1hSqz?hPmk)F3F5+94X2(SDp-NU!& zBT{C1R-e@!kduX$hSWrrmD7Ls*6wD5Z-dz=Eo(lGauvXQg!*NZ@NPy71J0VGDId3= z&CUpY+UPT*ujVZ>&<`cHpoZiVen=y9;F9HGo5M4fIyKp_jb@#3?B6U2O)NhnsN#L% zRfd@+YJ9Sw7wH?yi;$@g_xfY^gM&TVq@e|3qVoev+0(Kw+yp!MPMlHCayO$pyWB$l zL@2oYZc3ahHHHGsIue1g?HvK<$w{MSZF5V0*(Wz?a3V%-Sr64JD{^u3!I`7logKIn zqTXoDx}Mr7!ibb)d<}Okz|3!a@Q4c64xN!vVk4?}Jn9k))4ce>ScqUgzt%}Ow6jGPeG zM?$k06F&X(_7>B}FQ?akWRJj*`ZKL-nwyjJ_wAd~js4#%AaD94`5e3exz1z=++Aiqi7-@NVzU-`BSVg+iugsg#DD zn4tizIp>5iK_d7chucfgp9{!P0MBnJOm(|T+8wi&VUQAI5qchtngPTsC=vAEB1GL# z6I85IQp!SqBD{du2kc^B>V5;P#L;|z^z$I$?v}2gpuYfZDK;Sk)k1euYmr0KjVXbv ze0rd1FE`mZ!#*^jE#KpCoZMbEP?A*eTM?G~sqiRYi%ZI~oJp#7mKvfSDzk0pj4B{_ zogUZR_QijClA7BJ3G364%%FSt##wm@mo?&#IG0)jts;HR3FEZ%nnh-DQo`hkE!V() z&%gPZ{>>RzN5@YMOAGVAaRc%#AooO81M-eHpP|>Gonql>TMUQOj^nsDRLx= zUy)6R6BB|CYlQwiFPOFPu@nu7eE*;NpILw47jmVwXYod_>qq1 zTZfOtYemt0(g@R55&)pz$@f1zIJEJrtW10paOQ)r_sG&Tnzq3au`k+twJ(*Xn@U*eF(4y4P82kqs)n94fa6E@rXzP zO}SFplC$Q51F+N$2<57R;Y8Vzr#Q`k6j@D}YU#5_zkq%LJO9F6ocEnENrB9cg=2;U zGlofJt?%8ueRH8J%3y>eI~;1jGKvbw{r66k{9gcA*uUCB3?2>j_0J2dB=?ezK6;aB z%R6eYRMRck&$xYdrH`<;d_p5G*&lQ>4TjFT;6CwH#>H*G;oJ&!)@I&?nz@7mu_WZ^k1!fhQ=;%CE; zUG`_oN{IDE>?U^gko|D7J?#4)S>;@YicJEpr}10llyuDt6dVHaG9EAVHlF2w(()`& zXQu ztQuVXgHHYqHh`4f&pDI!?gi=(&#MooD2W_QURX+tyutX^%1p^6Vhev1XDt#^Yxt~+ z?tT1U9>`P%i9s*qAtjb0`~FDvptR)}WbH=<)+fbQF=wnw?CQ|@NMD9(WfE+N6Q`3o zG$Z82T~>J=b#2R0NoJIsV7Fj7fNQY!4JGqiD$^ES3sq(f3+rs3_cN3V&6XJ|#tQhL zd10z$VR(LuB`ug!zFvVX4yhe2T%e~pT0tc1L}*iYmoXY2d|lKii|76(pZSOY{cZ2D zxKIq70YZUZo)4);ty3gU zHG}W}zs?+B{ajlM!yda;{60Jk{x@g)G5Gf4@OtDB{1&9#!Tg~=Zj4K&K$YhZV4XHk zc9xg(QtKyo@C}EKj(9A8@6~oQ*Wn`bH2SCXI_@RyC|{ z!ua7KLIsFxN|QuzZ^I`UB=pz|yQgg!T6z8YgvO?MS9#Cax3=;p`EyN->9@cQsogYK zX;nRa%Qj)nH{quCcH<+x1;txnY!3JbBxz$}zX6xic51q>ksA7lrNu=eSyE?)30T5%%`)q#;iN8&mq*0mX8Y+UACalwy`2!Rcrr`ebtA%2DzMm>)AbA{KKk#%h+ss)hZSB z6iDK*|BM<9vWrlYLpasNYpt>Rkuwj;(WsujMaXD=N0ct|UU7hz>Bi(BdR zX6NyEK19(NXGF-oH7#)Z5Tox zhEe_5HE|v@Ikwb=!6@|Mo6jWGY2$JVR&X3Tk%t4yBvEui2HKx2kZGy)V%r`I?k7eL zFM8xz!kND#8v~K46K4Dw>_eoL7t_BtZBSzxr}Mec(lp5=a^fK)drhmkliqZWm9uLd z-SofXk;xk0RcqVz*a4eoA1sUfD$O%OAT<~V(!|#n_qHa${8KIM6?yqz>s}j>ZLY5u z-Eu0K=?PvyK-sGA+2g~}trH+>0+J;7!}0NWn#3m(;06o6K!mEUjwe$Yp9i0z)7AO+VFrT`fJP z1my}`%}U+MX?FvfP(VNy>gC1_`|qv9`Jc;UMa-m<9WaO7*f<3khxVt-@&?sX&0hbO zJTi&qpUW3TJ{XQ?^2B;b>%d*LTI)=aQxfWop|<|vbcz3oPLf^P`>4xdt7kh5@H(AJ zB&*y?@`oyD__Q{+uVY<>QGCzDCV#HFK0C)LGuE+doDL5v=#93fb&eqd)zEqZViaGh z65)=p#88-i!Y3^Q*Z7C6j&A&-8lrd%^ER@tUK<3n=DN7Oxve!xURb81*i+xrdwmey=v@)}!J5Q#r) zvM%C4D%UI1vDa{tR=$9_wY=cfUVrbzves6!);h7xoq&noiK*LlcqZmJziKg z-$uGAHl1khNC5D`H3wfcgVK_zOEAhn+AGd^Si-IKzbXpms7j{qzH>9IuVb&-NNHVxy&?6gni1qxeNVAUGwQg8Z&F5 zLh6-Hcy&AvQ3Ib2PcNkNB*k6On3ATwz`pe%ZjqZ(av7eG`~ z2c@X4SCwOVf!@H5BRpxarRY#Xtbjp}k4;6O|L8!lKZNtwqJbmEH{q9V!i zOgE{aNo<>o?l5bRndBd9?$zdb!OtN!OxKaU#9B*o%QnE)-+B8uFwVj32?3N_4r$0Q zxf{|Cu_d~q)huCeQzuv{EpE!p`~|~!& z`^RoSS>e>_aQl24oL7d2_%;I@Aj}f5u1I#=JYh}ON4>Aub(C8zY@P_(A3g50;EceNa zR|y}UN?fd`fU#8f%3%ug+uB*SSPNU*XbU}!QwCqL+&G%Ty|yOUhiC=|=KWKD@OGd< zew=9I3n#4kn}Buom7yc`BPEy>ok~^-$BIC|FoFYT>EEk`)uBFig=CY`?3Zqi+aL*s zth3hV7&W(ZuHjl^{)CPb4zRZR6ny+meJG1P$D+v-zJ(^2l4q7bIkN^zp(pYgeArVQXQt7f%YV&^<{%KWZz=!V0__VK8cAsyIwoyr~ zln5|a!ylMSX&d$JlE5ESH9pJU3D87AJ1pi@w10e!wZP44$?dNS+-SJlDDr0XF^S-P zT2@EjJgz=_j`xOEN4_Z2oGN5IN**-L^Uj}$%RNgIXS0(Vf6XVy!VS^G>2!dl>SPr~ z=AhYrhHLS1CTmmWP@95UF#tFF>ec~-^_hhapzvwy>Q?*-sX4bXC%2W8;IEv+g9C4j z8{uK)@3d;9aeZ^A)~OMy1BYh;FK4$9dwOkp3yZ>J|H9cXcC%|gsH$18mrRzgID>51 z`LYFyANR3A=gq$beFRiOb;u1Ur2BDkI2^vpCTg zVSrc8^47Gufg-$VAKTo;3E(b^dj8CI-=fxhqoW77OliQZ9~EHDmf~u(&}7)ffn1Lh z+0ck9#O&pcfA&2nM398^@8nqrGiHBp4|wHMH#bgf!48slcy%9$xCM~!M^gCFp<6!{ zh#pENAq_}wenB!_?;$GL6y-tZy#NB43xt^AVU@7NX!Gr>*X6#^=DvY}U@8cn^&;lN z>j^O5I=iwbI=i~rusGT?y=z<9Z>_nSF?0d<^!>x%fU*^Ief+1o^U6Ndalhx+v=7~h z9(mIa6+a6_icx4woO$Jkd8+L8corhD1`6Zq)EctBy~HP!0e19;C9(x<2d4c$-CH9Z zo|{oT=Km|~GJCpYnzh$yzU^WvW0}$SN%4>SNZq37o7(A}tK76fMn-ze&n;3ZN}1V< zAOsNq?Gdlp4_9;y)}g(E{KO1Zb3>Z_Y7{FRA?HDd{?KQ!9!+WN+VTgH-yHK-5nVgX z1*?T7Yk*#)CtQlqU{y{z9S*e=VIDsc&t-BjK%ae-6)^0{&NK<=Dn!e3shic=4E>avBqQv*g9|gPMZxvo&(EzK8Sc z#(0oJ;1gDh9Q~PQG0Y;23exSf{uR(a8Yj$>)<@))kT!CiDOtaTrtVpXS)t?a>w?-sEF+cC4SYY)M;+$h_xAyZT+|5bUb`g zy5ua>J@rB2-E(R-XrC48e1|KTpmdN|?{?-x&=AS*xuaj5%`|S@=1G2eLi!v|qj=V< zO*dM^QGDe@1{OgL=a+qxW#2Y`IkXx1hI{uA>**SBk>Ct@%_C?v9w=5| z_T!{5tb&?6a9ZaaUaG#oA^_yhtk>x1XyD4><$gMssr$&|@CS7>=u)46^%`Uu0P9eZ zeRdz{B6?cHLlly-4eM^{RXF0Phm~Q`7&zJ2qZMEDK3wlFBLp zXB7K=S1G?>Lp_9bHOWwq8F%(DTmw`9f_nrDabLmOd}8K)Zgu6JQuS%?+oxf!k+=aj zoTOE=hoH5x_6uivG=<&&m%uhdC8u9Bq1Vq z(>(gS8;eR(7P(2&C?QHqQK5A}Z<`WTQR`NpfNNnm@;5lG?~$_={EgV?WA!kybQj*D zec62E7xR7(>IjRWuGCR~(ZN_&;!@0ma%yKL%0;T8 zFJiq^LW)~9a^@SF1x*R1Wcc<#xB>$th z$M(!fs(+VK?EHK`EOMpJe*gV*y<7hr4mXR`AseX)C)dj;CKq<{2nOlypX)2suRQ`@ zKfB(zND^{dF%&xoU1T{m&3e_qN5if;P)BB2$=_w5yreU9=#}g3h{}of%X&dpNe`7R zjJ2lXEaD!SWFLckMiLmiR-7X_XKtt`3so*mR3_YdCz=ap)gaXd_KMLG z&0XqO5+Z3D2?LgdZ47;6yD|*#tg=c*KT52CY%c;TJ)lWhad;IWsBdUk1p5x|%HrZi z)jFUMMX?-UE2y57_nESsg3CrtHLR-E<;lIhV{gaP!SG=I=PdSS1JYVQZ|}p5;QdY5 zA;U_4X78+1CvGeS7pj=~k_Af_<^*eNYA~q&G94Rn`EI2&g9sAC6k3sQpM`*6(E)>- zDkr@r&T5SQ>iRYNV8h_Me}kpn7H|;`t}gjVd@v|%5o}Ub>+(CMmO2~=KLT@mVB#c; zx;JQMQx0t;&-(zcxicxY>;g{mb90PU>(6PKrc1(g29$8xlup%P3MN@Z=Cz>1jDfAb zje-|tiXgt}a7;_9Ida8sn%<;V(b%o9qy8XYv4>Q)fD++^?DUu*{mDFPv*lGI7ib|PuP{P(Y{;sH$|F> z-UW&MX=^QVD+AbT4!Ga2J2qV^r-KhGQU*P_6zT_c49YSHDC-z^*weDX=Fe1s(4vdJ zk%2K!aqKl26g}WXPvhq>%Sjbd42BGwFqZL32%xx*ws{Nn`I=?P9EfDuMS>jHaeED) z=cAZ%i7;r9jQg*wtQF87DCcgvIAQH&M)(Ar?aZCDTR}FcR>M3@2A0oPAbwSjN3~SS z+;r&NB}n#IX>#KcXmz1@^@0gM#khMbj|oRi+4qWYjqi-*7%58j_z{+3d5&ZjOrKY` z=V^V3A}HGzc)9oom{nHkuP%3dUXpGtWrxnfN)2ojH|y=r+A8@wjDA>Tsuh{FH@{N* zGSue@nz-vTnvFd7Wl5Ul0{1aTE&y4;WZ|P~X(cx|iI3>(dG;Auo2kCva*x4Gs)136 z6)J1qxOBWqUk6*L=fG1B*^H@1Y&DAa>lv5*rAlWg(n&q?rIIC3zy#xyZ;|VtYieqM z?hm#MVd4B5dwRCgFY^G=+12HYGAQ1u>+b##1H9_&Mb`TFj-LO*pT53W-})GIxdRqZ z0ABnE?2*$Br@#sX*vI%YtH7h~Ms>HV+ua7pp5O_an|ogfLO-%lPcN@~T3i=b1ZJ2d zgX=>5D<5GhdRtJKsLl&KXpNdg-UwpZf@A6IA7-JS!?_QvIIPg&$`K2ZUI9G4Dh_$ zmr%g9<%%p=m|tcRr5?G+U%=^A7aOqPz#Y?g?-g`2>2- z|GOEhw@NLR{Vrk)`{?0}OXN2)#nD#K%ADn0p73K=#TABKW{zrt{*A(3f0*Mjm25%F zYWfDXhs^q9kB>WrxS3I(@iOpU_@eRAI9>2T)=+oTlk@L7l&q=66WNa073#7z>?0@Q zsChy8{U&iU@=&!@9g6V~-rCHK%QVs85T9R^ZP=8uUb-=Q2nzi1PYaE_^imv+)LP%AZBjAgcA<^5!QpubSc>J167s$ zXVu{q97{_CE(x-HsWrj&_04=}u~yk|^N`OXYshofsT(JsNRmC5%Kd;>3`2vVB{ZJ& zR9)3B{0EP39Uli&muPi5!+9dy3rough{%4;D--9ZS=f;gnWc_n;EwWE80cm-CJxc3 z*H7-zqg8u_mVBEm{o8dc{vf{V0xn8~rK?ilFnM?$2KQM!26IHU1NI|+1<#7@0BBTZ^uN zT~p(7Eh)8r$@v-Rj$3UQeSHOiVA-k z&eMO=0pwC}h$?(tTEh5n5U8RO+n%D}xC1tTJK*-Y_ga9X&@r$OHOzrb8~A#D`zKn4 zT!*L+kUKH?RFwq)TbDdVLqq8$gi&AAl#^KxYT!YaSeAjDX#IhLVzgSPtd<@6FirX` zoJBkC$rdyUo?J{EHczr68($-zun#mDM;m@QG0fWnxt&?_u(t{wxH>OMXK#Y;Kvf)Q zWD%$NFkrdRRE|pflyP#A!=J`Ha_Xe!s1k4B2Q8M(+qwYd=WwlJv6ifnNq#y}P~rJ1 z+R#jo-}K)?Zo-m##-~IcYfb0JY+6^blNlcm;OYQYaa?|;bNwDqy!Y^lGI37CV63akrCpCd0?)w4^Dh#^}I@9SKyVKN)m*kF( zVnV&%5Xlae1dq=k7W6TD_M-Dy+v{E9>B?G1h4*wLt)I}bpWq^ApomR5`=0N?n=AxK zMF#N&bGUsunPmcuFOHHwwn)x~>~WMrQZ(-*RmAM>-?m?@V?!70M%5&72`k>zrDN`C z{mH$o*cl@|B;G@BFl=W@Q+9}(TRC!?mfEw_X8$#PCVuSl>N4B(`eUw%P{LZz{Q34} z$C5K&|A7w;jj8dERY}lqoY~(aj#b)I7#SOo06(Txu}D+xowh=$ zaWSY?&mM5L|ijzoJ>HON*0JeQ8e0Xy+Y<8}*nN-Hyga|D-gYo^samIfL%TjyrW{HnFq5?kQBuT; zg~02-H83kpil@lEwNGBplrod{4RUO$n|b~utU+nOOKOK1H;#T1gWV=He$p<9$gjq_ zWSt&Y$2_FT4z-XwMhfTCwR7%pjx&@l>dGwQTt3;nNS*1I(Lr=gGm<6gC@@(cEs~2m|p(7nG2LulM0XqaF)Q0Kw%( zp-K7_#WdISsyh$-*aB*{{LG06KAeFp;lcjzfBpa;G%P@d?46GF-ESUn)E?rVFNp*Bkjkfkn$NaaVt9Rgx6^h&H^9CWU}b@1cq1cJjF|87vX3#WNg zhSC(L?P8NvlC1`1IeWjr0H8%1{9x6X?D*ne#U96J^;N(EA2lvxQD&w3k}zHFN!WY1G~7vx-0p8S^6sMJMlNfC{NZ8+6oPMpu3Y)Mf(hLTyAOKtzV zIy*Q7(&p>=W%C+XfFP@(J$Ypqxi)jF9!KfP#pydyQ;t!lS)oY<^Tw@aWgVxgYL}W? zm-tzqrly6i75j@JGbBDQho*kx0!RCXtRK_3jJ_xp9>}F_>V<%;=H7X=HWf7u&2oS% zzHE{!b@P(qAp_C1Kjm!i|ZaU$yYLdptbg+*kaKb`w%Y+a;=r0{JySh9Oicu zWXb>^ueYB-Pxs+j6WziQFEARsI zp~Dz5j?O9LPmXW`kg!%jf1&|Xc+Y)ctAk*;j(sSGplWt+xTeI7DgXvJ+KehJ`JHWT zyC^XreCZ>`U*~<2gN(Tw+5hQT4r;`PFYI5yC|iEu=;~#;4a2SAj!|HMsWkTsgS(`O zj>uxtz|qDsYbHwD0k5BoVtKy4GWO+!WTw5{i~%wgJ+I(%CHW~1)2=KYh0&ZF#7;jQ- zZi`<*INyv)_Kmj8Ub2zSp6FH2gW~aS7O@9-Y6A;wTM6$d^+?2<{Kse$3>j}YlGxynNM`rqwaqcu!I4IA{yeykpS zY(xEA#00QJO^M+j3t$yQcXVWH16Gc@6N>OFn9?_Cd?1*yWM zmV=j@*Tv>qvwwI{>7>Xj2?_8^M1(HrkFXK%0DZApmNn_D?!cC2VU_SN}n`3BxX$hc&6uxZVNBd7({>RZ- zMn&1RVH8jhkdl`MB}AmV8M-^98|f~IpO%l=bJ`={kMMZMuA1IRdx=?ebuHvVu z*oVtCbyZ|54UEzj?6ErkMR$s~3i#1|9iBt%U2SdpD#iK-o{OUV{E?eWP5+i5lV{+v z@p7dONi27ldnNmfTe%ctSQ8oxnaq4+xHx`xHm3H~zZaW_vPgb6ka>wnBXS*kY z9gI6SS=6?`l9T?HIsrDqF@{4?ZCjIks+B{5@ngFSal3lpJQG^t@7w{#xuRrx zBvjL|f%J0ulG!>(6+hzi?OjN1nsT*3P8p9YPyOMohD@QTAzWunk z-k6P=#kl~ zD7CkyV zyuTcx6m1Dqqfe1a(oi58FnRp7+kL-DnQ(mkFw}iLxp~oe5g+*Y;~$iSrxSX4I&g8} zLA|~47tt7Z!G$$2EFxm0_geN`*P(QIoU|UwZ4@i(&!DzFO74X{)H4YBjMpPBjvbFh zUV}nH(x3frVpR2;5R?};Yo57Q0VSQSh{x@cZXaONJ)3znN{0Qb z?*%giS!9ZWQ{9?kK?bxAqN=mMu<(r}1$Mo#P^_U4jGS`p;H*$7Sabm*Nj@OcndcBY zm>THPLWP3>2c?e3hubg61pU)4j;s4u|7z9*clB&tH~%(s-wh}E>kW*3AHN6h-kc4b z-`oHt0UYj*f$>=Ef?S){KOU4zz97AFm#FO&+#Qz$B_9@QhEi z(J`y;lAPS2Z;YPpcf1|b8vU`n<>2M4`r&ZkLW&s5 zZ%{$^-*csEaHE#iyBYYo=#&?VCrve5qf6-8d?+pMcX?MIP;!7_wc+RKf4Q*fWv#Bs zJ{1B(C9=gaqv`9%ob&nVv+CpN(dyvgfaroiT!Tbc*G!C6*CF12K^iOWr|yTgvuiNB z=Z_64z}t`&z1ypPys73}Sa4K~jp%dkx-MRG>IQtV-M1x^-PcEzeBaNxxx#-#1CqAM z*Luc1Txxuev)BUfO|QOJ3@4X^ymN#)lwxf2 z-%HIyA_?!3e3a?2-nttOovM8W6KYGnVeusm1^5mNVmt-wMJBS&lC(O3{2)m^Kc5IVCj`R+x7-qu%Iu&7@F{_h(7I^CAXsa96g^kL5S>D-DL+Ix3Me?DeyrX}zhbvorU9$<ljv@j*3>eb#;xGxx4yg% z3fekrOfOWd>zCIb4%bzj>6|-GWSSAe;MjjYt9qEuXTVZdNS{g(D=XyNesy^TkO_5% z+W-L%n!6<<4y}=O+&Y}GMMR;7qg0ro+?NpCr`Flz+?Af~vj>2pM}S%{)xHMIEvW`1 zKs@Il%TDw$$#m7Zc`e=R_(y;kP=eU8(lq3g)wKOZ^1$)*0hmKB{PTV+0mg$!oUy*mmIkfysLJzYK^q)D8GJNU$h|G!$4e6xXj9JVdf8L4P==(EDth@Am$| zPoxB=Yj-j)@a|+Nwid9V{&7!TvjKUM5ncy8#RveM0p!z|?Xf25W(1K0dqfi+A6%;i zlfvvO;&nVt2`?nCc8m&b~Ds zGAkgA$%8#8MD@_x=nb3S!1^U%t`&8?GgEfARg=tv(i~BRksQ1fafXwxW6fE?T|Opj z8AJhxuI}S1$3b_LKmA_?`nYoq6e<$ync<$Eo&f1O2O!-&%Om^q1K~>b0qyl*xaj6h zd-lF^w$agmA+3b;Z|-#2u-lMp1M5$xi$Q%pZusYe=J@f0G-JdK*`O0+qBK|I{pd=y zI>`m!;%;Sz-Rwj9I+nhUnsu_PmD1Gm0X$R&Z*6sUQ2=qwk~~d%#!V=}{u)GX!WN}^ zOb+D~P*_dTE9QDj<5Z#JY2n`(&7=#=?^P?%>QRJr7>JErDP|L2s8V+p&|_*47xGwD zB;T(gomj8Yk=of>#aHA%pV3cY{U|oks@&%K##=S*u4B&BD(Ys&9+L%4PRPc%`3qTS zYDZUiR0d|DS^=Z-IEzwL4kn7`B-%Ss?Miq;(6pDL;00UbajErprt}Vx%|Q4`r%3*3 zg4Qmq$}wUw=RAN z#3CeNVPWF!;?N?oF+#Mh3q1tMp|-Yj{_w*173KY1Mqb0q18`#k4@Nn=-07Hevu_`A zvs7yTo-XE?LnBtor`D!fC)Wu9+4ZY%M@uu)jbV?q2N$va=&D0{SRrjL3(;amhIG1@ z{zD4?Da1K#FxH&_VjN+J{&R^_QGuE0ZTkfCAmO{dpNZP$^%sL4MlekR>A6ubt{u11 zM0}vIhP-;G99dW1KOY_mw#WqjAUFY;6b0)FFqoaGHTajBT3lH84oHx|`yLPwAK7W_ z%un%(lG2{#U-%V-sSR1{nSoz)^JHgx8w6pqmFVU?RpqnTv0j|K1+y*ymVdnHept-i zy}xvR^dyiDd~kjo_vY{LIyN>DYcT}b7~|}`V%ztz@B7{1quWo}DVlLv`rHSk?`o$f!DDAF%D`RL%q_G9 zGZfHSZ+!%>Zw{(``+7jd3H(piltg?UL%j)ASgV;s1r?ej>^aBjr|nV~G>~hxEdgIZ zyu8HSHZe)AbSG^YNmAX&opA0TM&}-pDz>R5D!_fp;xVzHnYB_pf`+sKnDIsr&y)94`ym`5?%ag- zg07wg9G0#}S}Q#(MCxN7zd@x_s#5XOunCkUwG*DxXr!~(9WlO=o~t>1#BKsUHFb+;SQw|2Ilfzu}5EDdh5jbNTEXLH)q zm?7I#`+av%X;*TYyn4YQw_!E4V%w`5pb>4>2(6ELQMlnjM zpYM?4f}1ybrpiPz-g#w!8z=1!TEmSpLKF5fsKu=i2eR2Zs1>_w4QOz2FH+N za7T(N9QKr=82YE)UReROL}CA%R1H+%*Kl-wgj8-*90Tw`o2wF*8^zl>{gLGx^fcV7!c_H!#h91@SotoA<7h zLGrnb+r#>jz(3Cd*`NLw2IwpSzAlds;6=Kg9&%=EXkQWFi4=SKG_=iFC^w}n(}@+?|vERrAFTN{rvH> z?;|+%n6>yER5k}(l$+=g1p*z}|C=`RAW{LIqmvT=)J&iRO>J=Y2et~dHHSO&{ zg0OVx2{>E-=sw_0<_`{5K)?FiTGaP!?aHWI+Ak||IH@vDh7Q1nN#Ds-YtYT$xESr7 zfADGaGdqk=;ZI@&K;*2f1E-;^aPW32XEA%@I!2(6+A>Be@StKnCfLSvpkjbE*J0O&kZf~tX!xe2?qqdZOCTT6gBU`*XoSU|W6ekck(Px}hz z?)~*S1dWMEw<2ixqNL23sq#M>lyvz=4S8&VXa%XS#=R95L%h1iqPDnVeZ5E>n1KYb z)d_<}lL?%{30#8;KGIrmns{s~v8nK`ogH6{GnpXdl+pq#Rn8e-ra2+aXBAhzk~NP# zY$;RIw>OKTfqQSi`Gkx!w}UIEprU2=`4y3XWACLRLjmTJo-JRn`r^C3rVK(Na|Q9Q zqfat!*&KC+-PSje+{iU9=r+5q&rAEQO$B!A zb!W9fx3QJ;OmMqW>*z6mv46p?=HFlE*jL zDOC?v3Z0h_knrjC%$6IWjYhpKaw;q1sL%;|FC6NyLZo(7hSooLsj#sX z($ur0lWqG}5;PLn@GBG%l1xAYA-~Rj!;?YF9yF3&Nl?XI0$izM-urhD(bgWeF@}p- z>BKD4M5T^T*M=Pn#;1IhfQK0La>Be(&131eB12*GcIt^eBq{nYTng8m7Dqn7G}A4? zsR#lNMU()xO)LLd9Akc?ZH*Np`||ewA20)CJF>D2q}Bbv^ZU57ewX!c75Gf_uedq^ z*tUa(I)K%KhxA{Lo0xd>#GC&&=(>hUcB+8NGvLh1Cj-@SVKt3wA-G&TA_8xn19!aw zK9ZR{UOb)z{ytv2{kxfvULRVt3sUI5I~>KyW75VJ@|=2M0`@`yh!ezEJ=Gbsy|H!& zn)}`-?dfht#;i>(;|v&sypu!gwbYKO#krDkYJZbG-ua{+I_28o#YfrT_)IMD_B2y% zM`yA-t;=*ReZ6j448Yq+hFm+EC}AT<55jI?^#gY!^(EkiR6?)30Pp1G0ci8O0c!>b zZNlqvayr3V!+fl#C)ir3A8P7a`+1_CJ>9iz)~`}XE?peq7UIpbmynuj)IE-rF{?Lu z#=;sqGlpM*fnh!@@GL-+oE#0;9PFDiy_|1?f@ z@g8U>n5i@4My6`2>F<((?nyR=3MNjYmNnK~Ia702&PqH_Wlc?`g!}|_^0?ya<19Q; ze*7gXw!{ch1@bo=W$ctj^qGSq&CiX3ecykyhUw_~WvQs@U%%XeMv*6eG)N!*Dnreap6>E!lqsFUK{vl$c;&X7r1Gs1u-y zC+@cHLO~VTl|oi+sc9S*PSwY*H<$@_;fJI{@qI3tZ;eYFIhibo|6%3$3P6%7=k1_= zejU2Zo}SJBf>M6Bht1OJ=RU#;3xOvC0DiI7{b(bYqe7Rmwr2E8M-a@VA&@&jUB>?h zHXH%Sqbak zSF%ov#NV|89nBqQi57@W0&9<&Wpkz>#wq|#qLTZp*!1Ny_cvJ1?Zz3c#u1KyF18|E zLioLcd24i=+*mYDf+gn99a^D7(VEfeD*6$Qk5Se~=%`~k;y80I@yp=AK=^JmC?ZQY=L{9-nj0Zj4yt(})XP6E#G5A$te_Or`dA^yu9$|=fHgMyTTGe~9B2(g= zNqekcH7=cO(dSfs==1Xfz*85_Ye12eY?uW)Z1f6#pnM}3V0vD{YoH5rfl8x-wv~J0 zy4IG>=V>bvojf>+WJ*bszef;m8Zig93Woxk+k|*Ieg+NGa`ECfg0duwYE`edyq&&Y zz3z3)=DG8po#Q|ME)PUGeyaJcYiAzIZHYlpZU<}4DzOU?Smw~%Xiy~9ea?@*V3S(C z9!J>2t$o+*5uN=*A@#bD%T+2=Nr54VHl9S5j-Q)uLCV%rYv%?nASNKduZ2N$>EJ!&SMS=t%a0@CJfSI&KMJ2~sN zkVkcnR?Z)ZHl4bvN29&S)9Ep_6&Q-#^|tErl_~$r&W%KS&EIVF>_ri`e9Yb5{(tbg z;`-(7ZP@WGpjZ6Des*>Zv4Hw~URA1otyww(+$&0<@(>$D*BYV!PGotC`%t6)>gDaB zbPZ#}4kw>p@|K<%x0mDO3uvq@zI}{)&7lg@IA>pGlQ)I6BSJ!POCQwgjB-sXw3(=% z%A?Cya)vQ-APhWA86wYyLYR-v5uK8ij){0xfeAUWFoRB2M|HvI$U$v$@G=-#V) zZA3}0$n;way{7u+TcFBRDUqD`4Pe|TmpUN~pSfr?{oUOqXGqyhC>}Qh?-Bs=%Y~l; zM^U4uyqsy#BOK4yw;d#@^wP0EN-j3mGv9)XD}dx;99E^e8F@K6vMA5y-T$@*A0Xg? z10M@18eL$J?A&>=@Nf28RrmX6Pw3%zW%If>z!5bkEefMBV1eT!&Qb>-U*J2`d%NNEH8f){Y$-Dcz7e+?r1ROrVo-*x!vEF zdm+s)rY_liUKB@RrF0#<{RBJVR3F_~wS}%o94&PQ1VS4>sg%rR^N?U?z%A7YZl65^ zDF3eGb{1z~%58ESa@Q#*L;@Cp?%Shw@FxIBjq_h|qI7x!S`j4njNiYdy}KpX#numD zG){~l&w{2B$O|6fwjZkSKKDmRYnI#CGRtx)t36L%HOr>9&Sc(8bK4y+!{(M?AQ{IK zkR1P#Dnq_|`@vc1wd}NEY-1W<8ne1%YN=3$7RpPHDVf6b5CaKxO$`NkwO>0sNMuKE z`66zLJ}ro0;3DOFW+NWb5HVX47Pjw$w_P~?2uXwvh2eQt#e}%Af3DCb#fGncTMs(zs|AgWy8zGwG8S;A5P>#U^^?^EuElh0UUFKYKb?QvomMOBOPqP&) zzs<6Aq*wZ-p;~ZzhidN|Qw{)p3T zDPoOZHjR6pg83h~G47p&Mm2t(EsC5SrRAIHQ-&P(*6}#xgEdGeS^Am^5MN(1}m^-E^J$A1d@ZW+0fT z0@=RyHP(?&Ni`H4{lF-KxZL002jH{ibFjaM*VWOiFE5MtQ^kYN42pjzEHL@i*_|y> z1*7QqwV?2^ytHH*3o5bz1_BlfJD@Agv74Qk^wi5NR$V%2>zqUqwB;vXM^*o$n)_aU zK>+U`SnT_)t0sp{$jwjx6Z?Fw=ig1YoDK-*;3dHV<%$3>TXl6k*=auxpZ>Msmo}un z==}J1NOYpl+tU-nHaLpHYseQN+lYzGwM3KB4vC_iD@z z{P6Z_Drm0l(8rdV>kZ+q^^i60y7zyvqmvT`YY*4^zy`e>;C?(1^RD!{T1#tFK-=ed zna#_5FRn}n@Etli1%SykfGC{ag?KNSKzy4pCGBY5(A1j+{?PSI+8*u)q=P!{xX_x# zBP_()I0cPBipM*`a*$ENGU%XI=q?4mI@zD2&&R72ZfCHpv;C*fMz z4D6#eZ{9cxAM{#v{>kFde%X}bQSAAyf|G>+d9KI%9^3N1LYzu6a3gI4pGH4@fTUdS znJ&!3a4Xt<;%T!ZxjLc33^bT)(W5bf8Q#G0W?t(F#yb<$4t9nTX5)lvb1>xqrx{k8{g9DPaS3oOQqy?v$n~fXuy*c)LtuTRJX!{?~x#pZl<-;_6F&M z*nd~stNQQiJg1|O!t_n&oKd1u{jWH$CjCg8^TV1?pY`TGoqECkG%;dF?k#QUI-Tj< z9MSW1vhvagW@Yq~D4tcEu3IWc{C6azNMatY9j-0b_uimsm#pVI^j3KNs`_mTv_>DH zdqqO=>eWOtyC&316SFQ6Pd+nb?z23*1UfRhX;8t>F_&T%?fe%k&718s_R{j%BNCib zGR?Q06~p1?_U0>{RftnHmr9Sac?%Ihp)9{H9Cn*L10k9w$NwNnQhB8&+_9VUcH0cq zU9t?8+FPt%z%kOuNRtWgO(tSl9I+^DJo1pqJ`3nptvR8UhjeQZfcG|_<3!5TB94=s z!O)udG)!>ccI1t`gz2xJE^-W?0s5LO?qB;4@qPaTeJv{ts}JdgBjQz@;sUWC=>8DL zNUK`_;z7z=83?Zv^E&`Jlg?=AR+J}#w0OlozLXa zMc7q&%f_e?x5?F~ zZ*Mdy*$e}Y&CmqvIASObv@LFurXbjw*a~Fq_V|VZgSfMNSU39yi3) zYQuA{RFt`jwu#etUs(5x!uiZbZC<(-`2#fpB&i%Jc#nMUp*m7%QC5KbC|>n2+9VHe ziC^I$HjX&U0XOc0!F_mh_iE!!3}J$tw|CKl%x?J*?lIY%3$j_GK^|~^N zMm9Rwluv<%TY^o`ez60h$!N&UVbXaU3aIJHv{4)0t%mq2bn%=2uHcZ$D@$%-I4b>uS{dm{qr_BFad$$||#Qa=#s`tsff3KjR6?F{&s%8eBCSv`u> zo1X$rJS1{%UfjnG@Xg)3^ee$QsuOTG0So~MDa@7B2~7M>q&qP6o9a2pUfrFnJ455u zZ*Bm&WBbI)IlC%(6Mn`m$JcJYc+ox>Sy+=qIx(Z!1rZU!waz_sbar~l>RhPxB^AGk z1sQu+rF|3k*2$SF|G6G!6_d}y-HqF4Gz=ZRTwAlubmBJ*G;l6kTE!4NY^zTpywIc) z`S-CugT5*!ZTGmPy|HDDx&Gke4_$UMLU;(&nYKzixmVFaB69tucUglVzcErI9b%U) zs{FRuG%bh?2taKw$4Tl14cT+cQDFG1__;$#FT8O`i$ugC#z>y<+|<+!cx+@Zgo|@W zesS){v(&$IIEz*)wp}Qo@pSZ4Hdp73UpeQS=E**j*qL%6X(Ub6u(J$GQQ{?SM`Ktp z3RT-Z$=YREMCs#?6z)zV7ARcNLwG}>$(a-l932gd5!6q%2HCAhR!_t%^(II>Vs={0 zg@4Vi(L9=v7U@)d((>awf1iUKjETLk%ggzszX&aI1$~fYlXm-EI;s-ti)l1|vCRM2 zxa=*7MJz{{azB=Fl)jh#5%;{U! z$Yerk$!7n~nV7s5jrWyfj<>B}hwW-lDoeFT-OCqg9JKK%O#Av07F0WfQaSmK1~%5v~EHP_mjGaq~<;dm-oOJ~bF5BbDXahq}0e5`l#(Nf(yH z_>NN@)k3}yGhHf`;C+Y?6@SS3WrpjXk8zVBKpDMFH@AmTECN9gw+}RzEkltyjFA@B z3QN^XF18&Ft8qj)Ld|OCf@0knOHxX#gATID%}6x-q>^9s_W+NY11~)0y+|Gy&8i*c z?hWLmTgOuK*u8V*h%ggdfWSFX>i-j_7S3RXgJ0jaB3C}bH`^N#H3+T`CLR36P{2EGMp(uIi!Sho{yo_Q;oa^T{q=S)XOLD*-Nj2Hhlq*@ z`&@^($B%i(w>Nj58Fb&a>kL`PNPD~cyQ0?bn+`ZEp6c^9FZTUl#_nT==&u?+JDa#1 zKK9xy-)zzdn{Eg*Z-%t5;pgyPWkXoG<-C^EOD_%wPP{VSBEL&Nu23p|Lu6CPz3*|O zOTw&XIRJ?(Ndc)db`lFcqA;`cFRaa0eE${3w|2FuY=$AfqM&e*$|0_@yzcCR_e+eQ zQME}B)>WHf8ObC`dv6D_O&)IyQ0!kFMgBW=&OVsER}&JuTM%Y z9CbxC=^<=J-#&)SE*M4jE9LVf9&qg=9H&#iUC{L|;>KotW98&O$#UWo{)tw|3F|v2 z=X-bbJ^Wu9WecBY*fd5tp*mn1B=Ga8#q#zR8OgzUTW!)1Uxs3J zq&OjpyX_YZ4SC1W-4dHmjzdk3eT3#p-YPxNveZ#`eJd>GSK;FLtt_}m{O0-x5P5<8 z@o@8HTX0M*O4&4*8k=?S^yebOs^@=^UYTh71pJjrc_dmm=cH%D!q+%^0~Pq|&re~g zyDggHKeO$_8bCQpOCj^Uyf#0UWOUNU&1n$0<+*=?#E?oPEuQ2ZE2kuzYPt=Zs?ul8 zI9CLbS!N|Fr?Z=pR)Jzt0o+BWj9c& zsxpXYO1!we=SLSF@}K}t-GxKT)fxGTFE|DS1)BBH}-lXv~b7MHQh60n;+24vHg6w3RZL{)G+>% znq%5h(-`T%RQs_cNFo&M39>yiy%F>u?As4gG#ER&*d_Y^BZP@5)p4tGAr(cxUp>TK zS*!Q<^c}}i2Lql84xF?ChU)8uQcY`SE=`XyV_#`^zQc0zrNd%`-vF1|A&4>P2qTy3 zy9dYlV{_6a+G-2fJ@G&*`mUL3+KAs_OcV$|F!O>~I8N_>xR9*mWAux|f@F{)^wNRz z868xmMY@j5k zuu`Qa(H&!DO!YZS8)q;{VN~4Ez{zk-h!}xlSSfB_B4f79foWr1n3{%9dZ&J6M5l;! z%it$w`DeGC;WN1h`V#buFg%hMF@I5ZG91&b*vGVn?`eVYAMHVXE>!Kn;M$AM6KJS3jIxG_z3DM-g&K4c=VVb_B$ z3(g=kPr%XoY&m*Gmed>|!@m20o+ya#LbVkB1AS3i`B;vowO=0uHILtnB2)M8?9iRi zXYOhHoYgCZ>S{!*$ zp!7dQF_R%hoLvWPjN?Ma_SWWSF3?6zh0p~XJY>>Mblx=aAD4Co9NLZBQ-rlgFUuu< zql^bG7O6R#+Oy3yl)%6+;dL`JV-Q18qu5|;4bv#GIaYb9VC)P%5<0z|UNg?@r@H5H z6mD;77X9!CNs74lH8@%}`@c3<^By$vZRqmK{8_YnuDlzjHTHdVllw7R>&(e<+2q8m zdMtS-BWs1JOQWpQ|N1gVfdN}41Dzo(Iy_F1^o{FVxWgofZtadzZ9!SnG~9J%3f?CY zU0-i+(-`XIwl{TV7Eh;GtFl#BhdUC|^DKTQZ_tB#+HRnLq%@h5HUSoLgVGE=8)i+S z{fI!bW0o!THdO-Ia>3JW!m;1fm=4^2q~s zBhrL#w9+k8%9#SFKtKxl_O2D0Tc2;Fm+o*viug8-c18R5f};m%G1A-GtrE0Zv-q4r zSZTtFt1?Yv^y_J>v^h6b% zjv1quVDF3@=l;<2rFS`b5bjY5n}}$82VQPB4+Vy1{Xm)So2r_(DTIlV>?J8&GiUXzCZVrC)q3aE)iODxX(i;Q790CjNE6 z&nd1oaJLJoCXIobI9^Tv*;fkmZ6<1z^=2Fbx%W8#q0C81P3YQVB|i<$57Qv~q*rFP zTFw!FXCh^ait75Zq`o(yk?1K1RrUxxlhQx%QF4}?XkK_}(Y1joEOw3h_V?<$6v~TdauEv#Jc;Fx}2(;}yZ|BW$?AsbA ze*Ibfwama>Z^qSgT;r$AKboO)XZ6n72U3cxboN25c#Tsw+f~v6rsn+^T|d3~)|LNj z5?VCmY5rS-;W5jdmb;+;&J62GcZJ9P&3hlqLeO;mbxRNZ<>{!sP`}1=Gq|-X|kRLH@U5UnVf#BpnwmFI{q=-^W z@svl*isn7Z*ui3TAN%V1I&R31U%y86R##R8MMXi_eozx~rKv^eUb5Qi4xBo!JXV5@ z;1rd?(&xO?m0<}5-aBv?4h8JXo$)NN$^L&a>I`qT$7?;%-|h5|9sZRs+Tp-FEf?L9 z=UK(-tHZ&2`~EDayfyK=-#&UUHd9tl!zn-< zdvtnx2542ilh^|lb^5aCt*s49y_UJu7zjhpj@EJl%16ARllQg5$#yMgoU_$FqSYyC>-{<}Bi$g|zj_$ffWRWBZR6!Fh=v8B=S`##hipn|34 z@uj3@?WHhvk2VEi0No;ln&K*;!zLB%TXy|RMsY8z1{6dC#O*S;XvRmoqXkK08%q`LyVDk1Z|yAc0)>521@-d36;GA$p@&f; zsY!&E!0;?8-XsPsEMZpINSXa2p$x*B&Q!&Y2@pBrlF-G%YurGj-_A{p|h ztjgNxpr?a+QmORl>T(QUG-ESQf+D|>a`t1Aw-s;6rf3fm1|jWX%I=9#DKHu6SeGfF zBTFriR5rTzW3Jszv{+UZNVuVhpM3N1G@5h`17U#gV3KfqBRS}p6aQb z95J-=K8Tl-#F~+#U({7!kFsM~oyC-u{X>`0EYYYEoq6Crt=|VSiFLimUQmge7}(C_ zpo4Roj{>d$wN&+@fRNC>r_f{`Ww!|QVdL+C=p%8!*;U}>g3YzZqP?*(le${I4z!fZ&W+y5~!BWWjHwPXA)_4wW3VdsZ!i3@#g##`cSgFHGci_^-q~@iF zL(%J-<)1xqmUTCKDGBS)D@2ruF!Wq51-tMNYi4yaLz*l~rCPMDtxaJf{r=upc#ZU? z$NBN$F?+XliKv_g#!~S%{Uw*@# zrD01ZY{8tyz;e)%KbUH%qYuLq$CHCa%BdL?KSL3tCeXQqjXSn>cUEG05nx4^J7E23 z7WwP`c>j3z_*wiHs)`q(4Sm6At%RiniKdp;GTGnb0_Kr4OBf!@EhUYZ1#Kyn*jLx{ z=)KQYIsT~8pZ`=~dxq6a(FnJyg`&L>o;;IMPG#i9eVi6H56Rp@nG7wvL8z_ zh!d-DoG2{WOtMXk(|`D_cPr>faT~F*5p(Urf4b05wHrG${ZW{P57R@*+pb%eXthNQ;n9-K0Cfe{&F<6>ex~2>gVAYVrpbVdm}U zhlK<}yj|4RR_$$!e#bs*$!zC`x46_kyMnaxnj;XIFzc;I16oT-tE|b|aoJkAjsnicJX1y)y0WY;(=)Fp_9M}3;jf^PY ze3Yjgh#;}=Z49O_V|`s$7~(rbI~GDxqc?jpAmxl-6v*Y{NS+nBq?WmO^Qz&bK<hMbce1>^fSdE|)xxmfnL-z|W)=knf3FA&0%+^*JW#zf0d0%8*EP9` zQhQ^FG=Sp`%~i?Ya?+b`Er6XIn#O9YuuQR%FbhN4sPCGP9a4u8D?yu2*WR{k=UQ#5Df zb@sQeEJSIMU(GI7JM34G7)K%)8dxeDK01_f+-dL)F(KI<`H3iLy--))PoPRY|MKUX z$!<8_j|z`41@uxvOzH+7{~N|%9v_Q(&bK#yZ_F`#6Z5QXHdcXm@av7U&RRAm_jv#K z!r$KE?Q`jqUMZa)c3#9E7d&0D8!e~1e18(*lx>dNZsT<~McD2{*s6=#2y@J~L` zNK;6K-U5Z+?Ks>5iZY<`2*~+ORL%QmtLve>KYtr|GUKQgG$W9&zebLH$UR`ZC z$=(8NsF1HOK>QwXWAx#LgImQDo-Nd%NNBE!q2XQDN0aXz(6jdAz+J?;kF{6BX3_7# z99^_LQGbuyAA18jlMMypbMi9{#GtEOm6OGbg<_}4Nn(A0UH?2wZVwGvi6szeE;b+t zB}@tfMXkJTeuM*=ea25RI$8fOCk>6ZW4#)ty4UM0y?0Vl6|HW_VXw9S!)Kv>V@R%y zBMX^is;q@=x=1k)qKcDSo^s~f?1#L~q(w)GLJ`;d8S8meb!8&FiVib6NelWAVNNk-Z;VhC39sk%!BZPr>vOQqGi9rk{`O3^v zUq|~!0Xbk#&#iy!E;n66kgC`R+#`(fKHT1SFb@hLKsHHFuLD?G=MH7m`?Mr9ZIf6C7g$es(ABc)KHG#8E zi~sFqhDiW`C5_ercA8}WHASih^XypK2c&{VyY&|t3j7A(x$XP;=kCrB-4!@yr3*N$ zI*q|k9(?V8V0^=w!FDWTf9S%0?w?02Mj+kyvxXUd;tgcqKxF*5m+-Jx{V?Axm^bOX z)^_o0o{3=_TqRjY-;T)|RznIiBTK;c!+UVkto$qAIWtC>iZyxAcRM3AybB`Tt%SOqL8l}Xk z$O;^BWHT;7@)ktXvi7ZFj1nuN@2f9t;m&#B#+Lo)cfc; z5t%jZyrL)JB35`JDkW7|yNE8wb86J^m~wg}-2{h~4)mdwMI9K@ldCy$6*Ff^yduYu z;?~+GGq!3%ag4X+JD+<@)ftMT-ujF`PunAgG6OMSCepAc{F_h9BwZ6lT?m=X7!g#S zUDX_1?nCee-h!hZ=%%xyI0YU&P(JMD0b~G)qqAzXfR4llKdVv`Kh!L0Vc)KXhnu#t zH&y8u;~U5+txq%snHyA#5eP?RMs>7MQCfYAA>$X~UBCl~AU}sBuw-yVq}=ewx0+U_ zV)lUyU==aEj(L{5QLTALfW~uzP)o1R0ahbc%t>*1>ONW*ixCa_Nf~h1F7)3cH4Q8; z8w1bjOe~1>s0m)A!%7(spc?4&M}YydSmxFg>fP(s>hZii(S>pf``2Y7=YVY+1)+EC)fJz!K$3OM+QzuVJp#B&uDh3`; zJ4)gc%T`Eyi3HH>i{+T0<<&cc}S%Lww;ro6em{P*tygY5fNS&aKQ zghc7P_K0l1q$USV=su%rME9%CWx>}_gdwFpk*7!AM)Q_#qVar+6Pd>#s@@Z)Nb_7u zZE~(oh)`gKum$iL4( zpZL#CZ4_CXaXZROHteTjRQdC6P>#$$L0#)n9sJDM@}nI^64X*=WgU!6_Ss@;z4})v z2@l@?s?oN?zkU<(r=@?S#d;I+8 zy4H1F^{)4wIcJ}}_pi-Rh#m$NGGHSdUPOVPL{3upqdsyTko!pK^iy+^$!A|jvU$&!VoFe4+%TI8>hBf_ORL+gmHq8=Vz(H36# zf&OnLK{_f-XY6#_;?8@B+VF9u zU#0bGvql@&fmC{oR5moV2ZHPyk-tgD*pV^mk!>{Q`$|=r#kWNfXacSN;?E;VR8_T` zBZfFQ+*Lt633$XLh}nM3PHeCl+bR3N+M z7L-Mg?X{yQA+q&0n#z|Q8J$!`d=fR1K9bis%BS14U$7zaj9SYf>aydzaL{p2SPf2* zo?i8K1|L;;DE@n!t(Zv1>e%bU0m*Ajl?^rd1I<~F#IGl(juz`5DjUST@0_aC|^&mPl}`fl2MN zHV-P)qgmxbaD#g_b9O!7Q z5r|9a1u-zP0U`q8@4QMr%n;JxdR_PDy< zfkVj$m%ER!{vMWt%}7K<0UpE5A~FeH?eEtBwo*3Aqp&G(3~myaZ6ndQKC97hb%!dN zQ;}syl)cqSjYgZH(7u3I%Y8F2VbnEMdyh0497NM^Smb4D0SR_si zasSiKXMM!?qnKAx`2&vUb=_}cv*85G_EJ8wLWfYDMv>Y^PZ2&{Z*g?Tc>#}UA9kMp z3!9}5>Jl`C7>=R~-!%c7=zRNE%-g(`EeD-XCG?b)niARLw1_OCf%{jhu`8ShWZxe&oh=Y*f~{sNQ+}Lg$%XZH6yd zN53DSnj9QwFIeB+98@b!PQt9}*z2Z07lK=d(C$bX_e5fkTaJjo+x^y@(NtE{8We3@KYX)f$Q z5s7|AQSk|vE_^a>NJwkDwGo#1EGVym7lvM>Eb10d5kl?+r?k}JOP9Y91ZW9R_>nu= z{q%71uhQdR1>D5sBwhzoR2Hqbq-26iKWkKHSy{;Gf+74sI&sdF3#sSQ>*($gR3~~0 z(nfy;}-rwXtt^5C-GkTcIAA9`6=k9hh;U+8*GQV zD8kN3#e*-BHO2qx!YwQ1bW^kFk=47bDKXW4KmxXiPS7z#@#nP*)v0=ES)#rZ6yeff ziXTCPdelGCOm;So9x_&?hyn%Bgcw-) zcy;157Vuw6_O>j-njJK)Vz|VijeS(9XOiDilARyV0kf_9ypK}Y>wa#K(%;7!`q+y` z`EP>WGvy`6P9xe)tKSn+z_d{;N#e*A!4Nu|+q|s4U#w0w&&C%zJ@G#PtQQ{kY(vGC zA6)GjUV&?S0J9LC4@}n(Y>abZBgNf>i|^VTa8+g$!t=Tap)-K?B_AxO`;7~%REEW) zw=_}tU?*LwGd4C-AppCB&Fe)r_tIEX0ZbcSi_)*$gtD0NMZ6tYtgWo0o*0rU$;Hg@ zWuURVK}Ree-$(*G2M~yW^7&oxdXw@A7zYO-##kw#x1cQAu*-);veqS4Zi&3a<(6~O z#ipD!_|nVwY4xsP`m-pnW-?vn{asz4SA`=rwb0-^l6ItcNZWo^g8F%Ujx_ATOYz zzn}St^|vxusW+=re{V-QdyQlkf?$R6Lx?X5n+w!M?*s)v+X*Dmsi2{HAk7r z;2Sz*NKj8R&dA>^!|up<8nH2`CbKHWK!291(~yvNzJN0V3*HkpEM?oWzVuvFHp~Dn zhZU3;deG?lz9z7`tXyLNlNs32rv>y#)pkOWFgHqBL8=b8Mox;LSN%jrK&ZSkVd^oU zhcm^M4C~+J;2T0M{ySa~?(6`}p!!R27IrO-=B9k^HEOXbUJKFbgd8BC!Cz`W6YQ>3{!Pf9Mgj`JIOr=Bu!p_fHhvPTBi9V zTSSsw5X=*YeQb+}K+pW7>XDF86V_Nuv>iS%nsC?LfwZ7g_j=nPVYl_<&fD9294I)CmANVvw(%DMi2P>Z-rRH8a!SWOi6&Y{My zyBf47m#F;IVNd(VnB0I;%$tL;Cy~yH+=Z;?+85i|{}G-$=ya{O_BR!GqrAGybYxhu z1hljFnaX#axP-4y7f68)BH)7I{qO}Zr#=_Zj>FhiyZxU${B}J+-SvBNY(Sg@G;eT1 z5t6fv{zmj?HSZwYFoSV)yN;>OulsGT5`unWg}eK}E@Sn4G&H^Fq~qxoO~$xo`K8b# zH#YTr^fstcZ|Ni-IP+;W4;uv+Nskn|*Yy>5TQFI~gU`HruEElnT>DogX-Q>ei3{JQ zWF#+zxZZ9ab&}oPZHidQ6vQNJRlS)EAHj^ruK18_JJJYiVy$IHn&QX(d_xC?0z|W{8OEpb5>1KrP-yC$9xO9-n z0CQiv<=W5G>k{+JXUb`j5t90SoXM(|GT(8Lt~A8azZYMq*SB;dWYg84V~YyKE#U<~ zFdAOG+OrGIe(3#kaQh7rgXtCeE-PN+)EI0^hPhR=|0`j`+&OA)G2hcV(UvUPf^v2T zE|W?@`KxeX5VzPyd zMBr7b&7%niiSm+EmE$>M@4>cVesCt&6ixtc& zC3QVMc z`e+yIQxA7{eTx>_nUiaO5L#dE_V@Sj;C~J@>gY1o^h%jwDYexM}sGfx}X@_pVJ zu@XiwLF(ubAVw^DB=m*h+hZv-{~ z?&N6%ZSAeVVx!JWXkbah8*oZapMPdm{w5D=Y5LPn7WmXx$$NmR%(Jm^^{-z z1D=ERYK@Qc3a45Mw6}X!bH+fe^J~~!O=fyXX!5TZv0r6%Gmf^4GorAAW#Z|0DjDYJ zj|SOVdC7IZ@s^mT9F__A9wP%F@z)U&$Wqyyi$}4?6f1r{SF$#Cp*2+|miHf5BzYwF zV0H62Xl`~3nQY)fBSM^}ai-F}WeQytji# zM3PFW=|Lg5Hv@^^Q9CD4K4YKp`Ev6u)@1&Np81WQtw_{$fYj_!|*&ye=4K2(*Bbg)#qs4`Egsg3}W+;2yw5M!zztH+Af^JcTd<6})-KdwP4 zNFd7Lg~%dR4xz*O>oYlO>)&=;HGWN1!}p$KjbwAz_k8s`VI{ahGUTybIlFca4sUlE zl7^jIT_$c{Gw z7GVz8n~vIrFwp2hCqo>Tb>XOW1uASo!Slt%#frz;LobfsDWm^69g@n z{?;`)iJaYW%Qq0|+2n)Ej;wcJckaHveq05IiKnonR$x-(QFs`-{thW@$uiV{*6Onp zMTaIPCP?S(`hu1FX(RXHpU42Z;p$jNYi3(JGHO`=oy>jK-iruAuMTy{2M;d2I_g(q z-^wIrWoJ|8pLg`Y;XH6*CApL;n0WLMj;$v0;_DfC%?YGh{#C96TVf9su>?ZU@Uwq6 zUF~zznU?(tK72S_P^~V;{u-|*HC0aLE%a*1?zT>O_yX4xUPX+DLz) zpS66t#Oihg&0#)JCG`;89|YEv#DJ$ydqNDn6Rixzr~y$~E-(iNzkBMF80|B@|lA{{Zx*%+H^ACg+rgG*ogg& z&`!DK)yA@+#3B2QZ~sH~VrP0P9YKjkG`gouK^Os|McvvmHr5j(Wj>o#Y(*dm7+Dc5 zjyi=!yj4STO_gG}%thoRle4n*d=6sfk)=%ab*$}YASnpim1Gj>hTGOoauBNOtfpfNqM)&?!lI<2QJ2NYyX&WfdP))E&##AJ)3Kew^Bi$-x<)98>=7Q} zs9T%j@haI5!>Q=E%{XyX%tr70{!aTpZYn+fQHpcFvZq*S^M87bCFB45@HL)$xIRg% z+Uz~o&DyTuvs(wzQ8$CK}>yq`#))Jt|EFc^?0Pjw%ox8Na4Z zku)Nu`OPoVOH`d&`LZC=bhPg_nLDH-HRGrbHlfHrCl%~;`xJsbJE^lbdBL-kvpb

EOeGW(0t}<=33X%os1ePH#Kh7@Le;$LMnnnM zRN@jRPfw;0ZT2*+dS*S1#i4;kbc3iiY-}>9{(-(;@Tz@`%@+;$>5@WwWQe0E?-0iR z^MG=$dv{%+F+;QI#;n(&IgH+2l+N^Y=$=iVM-w79rO{P1Z>_h>%8r@0p>Yjh1RIDY z8~8$}Q@=!PJPGgUH#MZI!aG-euGF8W)cx0EZNUX`G_69+ClMU za0f6qV2hw+y{2(Ha0Z0)0OlC@22(esXD^~B-;LMVzRQIRq%^vNe-K;e>^}W9dOGNS zXnwi^1#9J6)n72R?__1iqFlK<$uI5{1&(BxEfOo5nx*_7s-MyTrPAH}-QS!4Kub0H zA$VzeeQ>~;=KoyFb1*3ZNjBI2@z%6+e}Ic0!~Mz~_+UVH!X7qV-ZvJtn*ViY_f5gtuVed{<#g(Xe?s-ON8$Qb zHGyOY$iL{13p*w1^7IS>?eo#o5k?KGokdP^Uhv5y z#C4z{Km^qz5jOyH>Sf9d&CKuY0zIO^|7_DgEOquZ=OXzQNPKWm_fu}n9=NhnMRK<_ zbKlJyUj`0c{Yf|vle@))5Lzjza_a`Qs?^#A%TZ5p~ZcD9(0n-loPNMatf~5jK@cfmERjezqv^rCkXuPbDrs-^@2bAS8ffzQ@=a!k z*U8=cjy$b1mnJOh`#L)X;_6w!luVmXCj<7umD{Fr%*PJJ~JQSLjUKVXr)0!Ti ztGPHko5r-d++8^e+^3xQJ{=w&f-wrL!m`Yc$-@MAa5@X$*G5*KDu^Vahj|ndS;oDy z?Gp+yr;;`JBj}>^_#G$@k0lqcWvIm3bgQc@J0HlRUXPZtzE&jG(|uumLZrR*aeIaL zXM5(e!%RzG_#{6L1l6GXEE{GYg_Xm)fGaRxJiw#F{L`danb#*+0M&x3(S`6}$xHNP6pI^E*LtlBtWd^ypm2&N=o{Rkq=J**RP(xBrn?xu;gAhR&VFgK!FJ@w6 zk#W49$o_9Kr`_jkO1L`DPT4|?dbrMf$%?1xFLe%exSV}o9W5_%{P;jI?8a0r3{)sF zwhkNfRWylHBM)cD933^!J=WU?@uDc3#TY*jXnFj;;m2wn$!Ibg_F50BG9jKyo#cCm zh|ns_zF+nh7w-dmQ7VM0>J5XbaC@?IJ^6B1Se^<>nHwkRFSYi>&jpN|S+sN#7s>dF z%91)71W=2iA}Hub!oCMK1H~5%di(=1O@sYJxl&I00`(cy*KycJ9m0kcn$dIn*^1#h zIDMRBS~A^h)ny|HFN;~qS>9M+G&v3&NEq_ts!9qSUJB_2Bj718YQf8gfr?l0X2&x_ zo?)VO+F0A>BO?Zn0dA=j-#)mVL^gU17Q-vQfD9&I^kHl>>#A`>x8i_;4v~KgDi%bA zpjN};&Bzx%OWrS&yWUHDQTc||Xb|xTI%a_Y8k$|(Y$>6>uVF6s327<>Y=qIkV^H*S zd)uOVK2u*IT_mVdFr0QE8T1v&LZ^Ok72}B`K_=l|ofK$16RqGz&G{U-;{fsnJf+)r zfEf1f4e~v>bG`Zx$d%zI-A|9MLJxq4$B-nGvwMBKvb3`D0QY~K1U9zAeHlrK zl*ZJoOp7n&z5ZI+ zDLowkscNnP@FtK@5z%;I9au3rW{}#!R5lWiU_$S&`nnZ%{0y76ve9k~n+!!h6bn|r zBZv&aMHU~)XX?pwvG@O!7A>tR%jqEedUqdzv*>4I;7X4pKZ{1;`Pfa@iT7h;+KYe) z6#~8 z`d``CMaZI79oq>0j5wSf{O3E(>j2I&w#xtT!M1UNsAk#+Y1EuOb=eW!EdAawO>2(4enh2+D#jCWorZ zGRvT+UaF8`8^*crqt6Qi)8_XgVU+uMuBP7g{EAEyIYB|u4<3$;6| zvuPzM@{dI^o9u_P%c$nFMA7U*q!*)-r&6GT+1Wtg&o$}*c$y=e<9`i+OG~tsmyQAw zKY;koqu{%irmB@<;c6y_3C?!kInAkPud5IO(ph70nquBKxdQJ$!h0O|K;Q(QTL@@H}CuOuDcb= z`^N68c?uD)i^-?C?ug)H`0gamm4j~Z=Je|3=3sf_j)@R*WCS!lrC7Hj_5J;HwU$PT z)<@Jc@WoiAN;4zf*+b-Ho;Y^>-DJ60vMAJ(CZNp}@hi<& zJI&;(LujBQU-sm*&}Jd7`0IEGa*@kR%-t@@8!V&x{!Hk-3ADR07Rg6?)BIX| zx-}#X{UYz=#}$7adkkoL?*0;WiU_w+Xh9#mPe#k4$al#AG&_EcSO{jrj zqU1{uBz$KXmS!$em&psg@X6JBd*Y;)JV0zy;X(-EMk)SHg&5d%$}1ZV8UJN&@UUCO ztg;sZxlj((Z{dY~peY?k!_ZOJ0#1H|>7IQxr!Q%aasYw9i;TfVB8LuF?bo%GmTGLx;CN}zv-fK+K4!C1hO}O2*WfSyN=bem!$#m6AB`MDjD>wd+T3b^bT!BUay%n{^qo z6&>CgRn0R4?UVG`Z861QpJd=O)Sz_13dXpC9C4Ss(U{14*#@JH%t5R?|azr7s<+>2?zJASc4cJtPM z_oUFu%hU4)tlSr)9Nir)EmZlv#<f8*}d4oCCvV< zjhR%9gk6mcEy=GhF~VQU8qKBRVH9!{C1Ur8Rmz_}dGp&*Wd4TUjb9x&`1MNZ5f}3R z6LBc}^_Q|`dTm_595qJ7fo^BOW<}CLj%$w$#}?xqEsjIkTSUUc>mmE=qh9a7 z)U-2=Dt~0)A~u82&^98`Q^~NSrYg*DmhCc;8um2CwGnUSfkF1?nQe}ytuT9|{3a}K&1QsO^n5@O-4YcLN2ZxwuRstL?@K{q zQo#+d&C{~epGr;n)~nK+DX`G2NpX<+xpMeAFgnr#W7+Ji@aG7;!@UXt^1br0Y{*c7 zgTsiuTgdNf|GeSN9~~Qyflb>+Lsmj)Nb|*(^#QL(1FNTprCwBkkD@w6i z<`7uu6zlAG8}dhl7zG=~mVP>A6|8zpv%ts-A6ZP)oM_dGM(CS8t1y68=98@BjeKq4c9V8pG|sA$lG#l#a%AX#fzjS%p4 zkhOb|Hw(Z`iHlU1C2D!a6tHkjCx7GwxWM~O5g^O7^!aC=V>$ml?Lk+KIl=H(K8N|L z_QwpE)*J;YqR#jCSw^`vWerJP;HRK;_VM`S@i_N>IQRY(fH23`?#FT7-K;y@$9A9n znJxg2DG=UncXOZqb^052dcBmK1~`?uxjCPA-7PI3c{?vBr``&4^a*g7D+Iqt zL*okE%KwR1)6>&N_tE~fWo6*~C;-!j&5izFb~wU{OA6WRyZfT$EL4FT_(3U<~0LKd(xE zQ)<2P-Yl>MdscB6#z>94AV1e~;BBSIuBzsgrLK{3jLb^h^9ff%G=vo$1(6)-9=R)! z*Q@Ud;kACVz!YEpw631n&@YvEr4~$b7j)YmCqJ+~zzR#5@ZIN?Y5pn7{ z*eyWy0J$w{sQskWyqHio{{rM>5^y^TFj<`jeOMvYvG8ZTkGc~PY)$PiIrD}fgg=%S|- zrQkbdnDei;I>-wCIU&w`kiiL!qkz&Bu<--AzXFR094lyX`;pJsrX~9>O z%KxSOXqhToo07(VfyRGyAv=80r5YLvD&u?T)y7j9oD}G?AH72wcA&_%bvp6LF*eE8}A=V zo_^#FfcM2%s_E63xC^jHEt8>&O&z(1;v&A}*!(}vpnT*VAY5Miom?Zn|2yh`%VA_- z5c`of^HcP~vLI2|R}OX25u^m2DW<`Pue_G;^y0jbD7STfMTTC9$^_2@BuAu55+yTs zGNBi#D(lJX0fW;{CZ1}0)GB7?&F5IRyJwkC50W|SSZFo3cR{@8H*bYUTD}(aP-OJe zLBu%So(nA$r^Ez^qCrsEgs3Z&W#>q}A^8Sd<3&0)q2O>27$<(FQyJd#jJBFUVB058*y zNBX1J%N@#_USgw`+64nCcL_Q(j9BE73m!c+)S_>kQ-#~y3eOXF?}~Wol)oYd(8L=pO;c8edEleS*_{;6nK>@BqKsavPd8iM*wesT%6hnP zP|kGxLq*9-b_N%(y7<$uQ09D&ZQ%4)L;)i*zf^&qUo;+Jw1?;n|$^U;|l&2P4(kL_tQ0iAmgN{_MQFG{Kx}w zWNvU3m5|9Gw?Cm+X_4eNDSr;Y`(dhGoSbLA|2bCvo%t7g>CKSy4S7qyXbcV^o zeHw{4?Z};W@;b$ElBj$k>)w&rgLd03Xlg+pefKqx6j}Bv>es)- zQ^dZAOGd)@BMf1F*;G)%%NY6jW63@db4Y1xP-a>gZm1)EcCe@wTM;q>a%$eRv&FmZ zzm>$FrfFs&2x)zE5tc81a^4k-+@v0bImmz1e;N`Jbye7y&FFA@gFzyhFvXG=Fy7ms ztr>(`P>HxQXL@zMVJ?)*mgq)&Q&g{!y7--hsvHG1=iIeTAA{PJ_(x~lg-CE=@*y^w zQjSOp(|)mnCV}MFqKm#EGS?fIP+ThHrD9M5qctHRbou=yA_B1kkl89Dy{vVc2zOaR zJ=KtLt=^8+KAbiFUZfq={QA}uz7L1s7Rp&S1mI05RMCGf%gFw^ojJFFlM{JaR%&ui*m_r#FU?XjQ zjv|B~8i#lt8cEG!vAADlZ1w#Vsx3_BhKfsfQPN3HM{Z0z+Hhwkx<$Z@HC}LHAkus& z46*%Quvqs@G>}aHKbU3&?#Hg5;ES8`;f9iJ#Wn!!D$wy4&|aOfUa;j&d4NrkI|87`iO zgb*T|kDE8etW-e2gdWhtOMXiZwt4hehvv0z|Mo`{z5D`#iJPZPqiTAZIyzen96etl zuWMZ0Q|Yr68KXW_lRc}9C#TNC9U%Uh5sgZZS;v0y@Ebp23b4<ds8yAqlge(I_0J%QOxAVZ+4aLdL7nM4W9Z*lqk(IOO%Z-l&XG* zZ9!5u8@&oJYm6~iG4Ypi(4OUe=R^)DYtWD}`GrE{bJAxdsl4siJsTNyFPX}iMpR#? zdrM9Js>()@mPX=#&!VK7+tpsm5~xr^^z~@8{`X_607P3maLoKmVA0?Hc%=D5 zIsYab{Q(TyQ2}2C_X~e}UgA2bpc((h_KuhimBD{4c_D3z5nH(E3Fr|S@5)?wsqrVx z9gQUv63`$lXm-1i;dE+cMsEvq_tsIt2)=yZiRHt?MxPe5TkLKJs&3}Up zje@6`$Z-&fN6kNpqerrdr`vhGYH@gH(Ae1tif%Qxm_nvxQFFAw$_D<5N(Y!JjD-#A z=7PFJpC>=$^oVn(vh>k={&wA~cqxhn)nszZYY49kl4ahmXT_r`K=&y^p}}iWkYbCE z-njpXoPdj8IF*{5T=q@%O!!zYCl@)Wyr^Wqroop>HLAFf9(V5P{V!r$-nNYAD(JtH z>tKW$Ixp!Rtae0qZ6SDC(UN;({a#LPek2zKAl!l5$;mDe7%`IcPIaRUNYJyaq*q}D z)5ZeG%u)G6qcp>H!jrd%i}&lqoT#_HWLOHEPNcn;l*+$2`eC0eDyGySE-)@NAGXenaj|iq~H| zlL7e~ES9=kaHehbaIsJiB3^s$*ErZa(V?_nxBDron7FryU&i=K78|KHfQFf;G9f73 z(#w-%LTo6{ezMwig!UEj`KGikj}D%7HPK~sKg;vJoOvCkVr^X`IXzP9A{SV|hi@4) z)*@0g%J~GE0ge;#PA{6m zB<+k>7ipXn}+`>JJJ1^-^}|hLE|inI9=O5CUWk4IMM<6#9d6VJ(H{rSw|c>gDOC zBO@<0$U{=xCe$OOW2Gq|@h|WbwBF!r@4|GV;_^9d`5hJJ2PA{{b&J|oCOf?O(}*Zx zx`_%(mPj{WNw@PZy+n-ILlSd#5x&m+y`1ML3a-~;BpTIe(hX1Op4;1o9^bBV%VgIV zOhd#^d_?$}L!4N>@gSf`*e#>bN|sl`%d@5#z-L8@cB4^*gaibbqD-5#f4oI%I)vo; zR?7mj_Ic;fvt*dOczL5GYr)AK;*|C%tCn~~=?l;xj8KS_OX;Aw1?e&Nu}9?&4^*`V zwb!V@tfLX1YqeX7rzhQiTtD<0|D&w|2_2aJ>x2G)%j>(x8#8L;(M_j}vFr~yEZlq_ z#PCsEc+Uu_Ax4lae_{#xu+F^rzUIOWEDQth$g6b*CL8LmG7qzB+wEu4*VJ`bjn zt}ntB_vX$tFPL_(Qv{FjN!%b(TpFN-0@WUj8xU|~O}2o7WIgfo7N6dH@k}QgK0)$p z6&>_=k}WV}Kuc4IhClPLM48rVa}K1Ba=>&4-hxsUF04+jor??CYz8ov{JWX~EuF1( z&boi>4Of^q+MQL#JFg3H8)k~hXNrTWWas33RFRMXqT)Z^FAg>~K=lr8a=GAx0<(Y2 ztxZfE8t!dhK=%D1GQoJ0J0P&iP(jh>6@CJ`!gqny;rQ)|>s|1kHP_*DSH?q@T-JR~ zPd8`8z7>;dEyDiAnY>vAqSr+P)>up!GL5G$)TLE=JSnRT-3!ZNmA=hv)OVTfICvzJ zeS0YUO9n_5_SyC}HNr*b!W_}x5j=BtZT>64rq8n(;Z~gs5}y#Pi^c1~LSf&H%I929 zM(T2ZOn?c!(Xn9T89RaM{A`t0c*&6RB5PG+z}O6W6IVym4D(X8USp)8e<^DcQ)^*S zs=3Vv!`w-An$R{QW~csX+IV>7P2aNTS{5=^fB0ve^Hf-JCh6RkZ_oZe;UoHmw)q44 z=xxs9Q-TV5I%m=;JoLiFSln-%=dS|jwX_SQSxSUu)|sXZ(@V5<^irX1>@z?9gXo)A zvH|D<-avv2Q-~^liY;76BIZ7CHynW_mgbFMyDS}NahZHqQ)-r)?40#H3@_nxUeede zHp+}X%AAi<5_Q;-Eoz6J8obz}0*t!c;-$R0OPm2E8g_3|SXaMPb`E?sD#0crB*%+= zYo@F8Y1Sm^vU8Df)F`o^R1UvBpwSE08BwW3*L#h_%w`(eR0wU{|t zYCb(AQ9L4_wbI2xN7q@G${F#~^98RUb=hH3Dp}^o7c|^ed(}-syb<~upM8njwb{E1 ztwsp#Y-~oa!G8jFat-G9qGVwVsGHkMWHIu=vHj3ks$qsp;%bnZ3}pSf9>WE`rOH?W zq3gF@SAzwe{lew4(n%exa)=&KX!@?8$->EHALk7K`1oDU`#Jjgb+xza5!$!vXGB2) z!eb!o-fMTOpx%PHsKfv1ad;S`zR@u{D4jcfd797m^HkwypUveYb1DVmu;5+@@!c%tb5rN^mx46dQhG#5qfAkWtaBe-643B`_ds{J1d!$@#Jkt7eY?wSOJ<#_ zzD|4SmosO%z!&DO{ci7)qDQHs;_S%{vK4$Poj=t#TqmrKUQ;*z z6lan!X;tr4le%P(K*Xmjg|YeFeU82382eA}+2i>8+t6;GY*8fr!9%Rh{LO$I85HZ@ z*n{UiVk;vKlMaFANY(4;*=t=$d4z;x%bgXhn2_-hAKz`n1#=f$_1T!WT3oJJ1b6r{ z!uN$S%qVks3fbhif*~WMXy(KCn2{04q>_aXQ&3bH3}#3T9!`R)?rsDGx)a^frX;E$ zmGZU}9TM`V;)5tN0n@HNahL^pCXdp5v}?Ay1@oqT9*4&Ocxa$#sa94$LTY)Q%XH2ha74g@XlMI8Vj@V1>o8!cs zdABO&w~$0tY3|fNx)}vsk`Tk{Ocm%u9NAr5rW%HLgKMbt{J;ver%4Yp6gL!_GfC)z zEdp?eOvvk%eYD*#MEtH=q*dn*KIyPy4E+Km&KD-W+dkXz6Gq)TH&ycoAR`M$_e{0; z#8}dGa{bo8>^Y)Dc1xXQ-IvPI)r;eq1Pop8uHL3iK&Q`G9?83e zLEi#NLE#SEhIaZ)KTNhbggu;%J5M7)!T`DUb0tVm^(Gy=UCsX;{d< zI__XzCGYG=;D%Nd{2&^h;+0qi!OELc>NP~U8qhw3LCugPXTxU3UKAhkWfz)g4($H2 z&@5GfXNR#BjnUh-0*fP29W`?_a+q~FSJ@~p(InQLlTsw@%S=j2d~+!?a1o0jfxM4V z#Fm;byqqaJ#nM;aYUo71WTt_)?oVOzcVljSaa(rZ2hY zmEK#7R@wEU$ja(3nN2ED5L9Kws8pW+eNar@F2-y?tE#%jPf#5ZL_;>t4qdE+*Wj4O zS||^Fp9=XL|6(^I5nq~Fa{Ol)^7G)l7yUFZMJIGYfM~XCxuBd?u%z&(DwOG4oupw~ zepz8F>Pz&mjaNQvS{bwqoe1&kgf=Mjh>4=letuq>H=`XoBS_rzB08aTw(DSvsFKs2 zSbz!gkI347)AMh?ZPC+PW2gPW$NE`w^`p<%#;yznyffhT>iWn1o*;Zyd3iAV0()@7 z{uBR{Oaf=37Y}Le*xsC1lY2@G5_(5s1(PI!WX?>d;pF@(AgaH8+uqXh3v6+&-edBr z$widN1B(kjhAnk=SGrWaZ>(TmIa(P}SgFY~)Dcl|Y+TuV0(G5ZPBk@aj{!V9J5O%D zg6A2QhsFew5kv5IvwP~Y&;uCT+O>|My)Xr6F?Dq%2%}DWl%%Z~J9Xq!su&xrMMe1A zwaY4`CBxju>1sa0U0ud93VHywqHjIcmZI`ky>zI(Qe+x6P3WR&qVtWLhK7rBfvMN% zCfffo_X+O*ch!3h91G<4H@v4o?egPq!Z#bohle31o}dDvX*ydG4FWsh4enhc8V|O- z&o4?Q|M~`H{OXD?e8%=u5}iqyYtG$~d;9X+?f3Gt$u3|~YW3DnyBjoW zoBAVGeO%)P64!f#ZUGf3<0?zxW?q_2p+Xx@f=huyQCGLy(DyxOD~=W)^Qj`6nsCCT zPro$4@JX4|cN&U|=Ovkoa18CLql2ystQS6E3?(>)VR?A}*Fni9Xn7+V`9G4*IwwD=6I^0!xE*cXxL;C@qboGzcuQG!i0$#21h*0TJo$hUfZw9RF~b(b--1 zbMHIOd7V)~Bz488(0t9}Rqd^f48z~o#7WhuSgAU(f}=Ydjv8*6uJ;`2X|U`3mxyq= zLIPDihpI{@1x=X3YA(G)DPh=*J|;2^Y77PptPM2NwVW#(nQ-d~WV$B0G!gD)JZ$U< zHrEz`V;Ty6`6RX5(4sxa3tI?pNscr(1$#AEWU<&VD@G&|7`4&Cg^NW z5U$pE&FCwZ#Ot*8Q-~%CPSlN9mw z0Fu9e?F6q@aI+JYS>I%2iRz*;#4YP)&*5IVE{s7l7yS78W*kAdDPNIej>y|}U zxZla$ym>vSZn7J3!%>S;2T*rskj1-KeDo4TN%WC4S{eFyRJ^jGsty2$_yXMtPlnH4 z3di31IrNVS1r|Eo&UyN^ukpLK?w00e;F}Gn`Tgg?zA1L?W44597il7az|If-2Q<|q zcTWbnXCto=1I#26`I!+9txQycrIxS-%SJ~aD&q~1g2@&4j zKK1^=ft8(CaIl~I1lXALfBav}=i#rmQ{WH=hT-EaMvn_FX>uR!MO3ux7^bd04lLPX zJgAJCComH>VggCRx8FaP1%Gh()#%j+lF!;K7O4AW8#KlB0qsIjmpehr6-$Ow?wNaKLBmHQr|031VrXFhOnj8qJvc=-mDN0&y@t3JN7>@~d>{m~R1q<|Fro&T$38mw^2YBNYUXV(R(zT}+h zDHJmdRVn?woSuWhTQmh)H2tEKw&zW+)TS%~XuhgxX{DbE?7NG{yRlSNauN9qe&aYR zL8RB)t)3t=fj}VirBjipbEO%4&rtfAYOX1PM|_Y#Fes9AA4x_7g%Nq8%7esWNXjW-3+JsEKvl$4u4olmE=9Wfo>TK0wM? zD&*E7O3l7;ieOO3AsP;+jY;S2J+7mcur6(>G?$!gXWQP+7EaZQG*k+zX-X8PLicxA z89wc(U}C^DPMa$D-a0jjv%+178}ezD6RbULyj!P&Jw3l`y9-ddbGdnW^|gE6Vfo#= z#%jCJ4qG&QrIo_%ibtW0?|G&m;isvYx(eKfX&8c z;A?$rOZS|AlB8>6-F@&o%Re`7iy@l<^I!^9^p>A-`D9E{^ehDd(UI7-JXgaS2vhSF z2@dkV`MoTfc zNe8>}j)#evupZByJOSF(%En*Mb!%sTJvTL~L~w{mOG|zD6cXg>dUW1U&hNZ(bMU_V z?DSJec1@}lA`u7KLj0v)m-lZ>2X_NIxPwL_%ZfZ!rSoF;TVN~0uY2XQgwftj))2z& zyYegNG42UpS=*Z&FS?i&GL+svnbJ==PGt`_*jU040Xn|=xgGlH!cM!|_dQc^30uvi z9jr3G&!~T^9g_I;nR+8=n(^qIs?aE9j>z;^R7M31e4b?&;>Hx!5kbAzaBu}JBv)FR zJ&{#oc*aCZ+sKzh%%RA1Y?;$KReXy-r7Wr3LOv5j;v|3mbSiS$5gP4i73wCfPE#O> z`(NhnXMn_qm@ruUC&VbFeS;oYdSBPS*giz4#GUYh*hg?NSm=^W`r$pCM98>(aJ6{5oE^`@=5a_6XDjr7O=!g3}jHLPPvuww#iXV;+Y zPc5+wHN>AJvPICScJY%Zfcay_E)kwQ>T{QAfVD=yW6AMYqJ6C3Iy&d1~Z8$TffexKsXk{^HG9=cs$z-MPuFVt*^< zh`F}6uyHxX)72X5+LG_p19{vKWekBgZvp@@ls)KbX>-hYZ1dE)gO?O*qU7USW4A$b zAD>ylAF~-{pEvysb<9R~6ERlyLeJE$++0|4~t>bpEFOJ*Np-MIaYGk08=P23aTZM>7^izE;t#wE3zj5c4+(HWthjes#CkG-@Q)Mdw^*JVXY zs}6uEk?yp54bHnR%6r!&mZ`}*F-sGNicf_WlIQ1ITj`}-=;@-ACVe+thK2rw6{QrW z|o^h)KEE1Ekj z!{S~hxQHW{657JkPSP$!RI2>b(6dV@D4aT4pgygML$z+LUMfwr6-W@Z+3?V%Mfau@ zbwt)jRuT~$6rlZ@657JP5yHuwx97{j8y8ZV(Q9WyOc5zoW@y$JStaF82b$&8IqPt_ z*Yi^vcKA|ffP46JH7J2;zT5Kr@#mZ+mMR_t66A9B57*Fnm_+YWf+u)BbKwFzm$zVp z#q8d^MCq&VP*44mNPpKEmloQ4)tWi%{FbrrEh8v)4NF(W&p|*__0rvXrpbkW=eX_n z=KOur(_c~nW-vYT6pd0`hYGS5f}76c!0gW3gWEVpyZy~;7A(W;nh4cQs~XX!(Vd$Z zP2bav(SMSblkRJm2Okr$hi-r{F}7*srL?jWG@(-Y2Mz>vO7AqiE zd%k+~autfxmOCXjgCB1H;rpl$0m818UUpgaT8~QEQWnHCeicb=!zBx#T7c&MlYzs^ z^})Xw>VMw8;Y{|Bv#RcXXrq>vsP7St(&Ra!Y~^VLE}l-+v}xn{zDvmMznli38TJH} zo=;%5^j6FjkmUf>iY>0*_tM_2Pvi->LJZq=SpaDCV7c>gXBupk9jNiyE#0-H?v8q& zi-nx^t&Xdu*nkJR!)-Tl=4N89abSe0zT~D;+#g9#Bb$j`k%%ALtir}gfTyO|hR?ti zTl1?}XT(i8*}cgYO7Lora3hhCZ8swy=I((KCVBH-T^1zOI0esaj2nW@?0Y&X?t2{)J~Ro*iQl=4xux#W zk>OAJul^zzJ1ZcEw+b&HQEf!kqvL8(dRBGNDI!M>4y#_@JFlb>S<-83KOg~+3$Op`w8?%srzb_JPjIO15Z~eU=EQ62DMLX;C{uJ-Ocg9s?XMt+_flWuU!TDSRPhw2ZocM?E~!v)nAkGxu%RM z-z{;(-g9UNmvrd*y-~1}Cog}Bl9c>2`6nsOm1#O$O3U1?pt^!Xm7IY}GZMWtmAsMt zPBki7W;aGbGK|lQ&8=_~`HZCx=~u*EgBL+W~QX z_C7w3+moBUs{^DhO}xfzv%RYoEF-p!o!ue28R+7aFS>;U{k&Z3S`0^9n$ZmmB0#ui z>~`*RoQqiKsYsLVe61w)ovT)~Ib}ZlMSG(V3_pQq#d-pL#h+mt6V$Q?y z>u&x94d4U^jB07M`_aR@gX!u21YL2$X)t?zh4NgS5>n}MMu@>{7u-5W*B|(KN^hE9 zQp45R%Dr_#7`yIhY~4oVduedE7vJxfg45m!CdvO1zZp_ggPe3zQ*Tuoommk+7o}3|G z{`lYVX6M6cOvAnk`yAAH0C!J(FNkcQgKED#4hpGjD!)JIyAI?Wlj)^0=*a}}CU@I$ z$G)2vonQD5my^|8)7fh)&-JsWJlIV-w|*qcr_A+aoN3pZ$jV(F{tCSL!;BXfcehj7 z%4FHqIomQVlrpo9Mns>Mt}{OXb1c}EdJAp*O~3$=stae5Z}ZHmgZYC15S{{aP7B#@ z$-hJN-}l4x+7RyPg+oc)ZHyW=?np9fdHRdm!s=x6zzSdJ=tCcI5UH7#ka^nP6jGHGWt0n}%fp@8^_-^68R`mJE1 zT{AXN<$Iw97o!JYC!`F_=)Fu5XCAyFz6vomlQ!NOl7gLyiopuur%%n{!U_yrdd zLKD9V+2nHLuoO0!Qt7T&A%;*)iaE71-s#Ib809}(XB7L$u9Nr^sXB!R!-D-NwWuPx zM4KVqOOr{`>N7RU^%5A%KkK?be&?)ii!h%=+Xh;NAN zD?)xkz80(!cXV~~x373Mwa+n+j+j92L)&SMX z=il-s6_(I7oh-Fvo0$->_z%#hW_AI`6u=Je-#jw||2{ZgZd01xGB zX_~a!)DT@4d*mFTsJP{R&w~SEivye&U`3e`Cl3)mfLJ^~U%!E`)?o}d^P{g7*(8!i zhMB6~I>W*9G@4pI=0d&n@Bx_D9!k5)+oze3*&UJ|smupi_&F8_7vb^T5 zo^KYkwygNiz^rL8l+XqJx4T8gZGvIjxSW5L>f|4dMQy#)r~aoJ)lEfN_s%M#Il0Ha z=90}VOAE7~CXx~x-#hMbApZop^MacN&hMM;Fp&j3|I-`&YmA!kCr2C?L&W+yGMa(D zbTbV!&|_~#&?XNptbbvG9JfN^8C(Pmr++TAn<)H^9zsY#b+V8LNHxeTq6m;#!fm&H z51<}|14=Z0ay+k96e=jmDuu!>{(zhDtE7B?rFt@#p4LYMQ)S9^ zIVB#--q)E9%#-Yl%N{BLLDzhUqiguV<%RK#-+o~99S;DTE`DB zfUfR!VnD=$f7^sUSa@xGw6f*a$%xt7*@+dafM?M_CAk1kO!VMz39zyD_Tn{Q2Sdy4 zUd;a=dq9oS{$nk#$J9Y6?@8FuX%!sq1{hr66=^covJBHi2a3H-aTM62ut2cLH-P+3 zEv+a((VM@%rdQ`7_kP#A8giWZprBO=mhx+BYhQd0K}xE0yJ79DU$TD}lhR1K3)AAZ zXW%}2X4SR(!{BQ%U@B-0+T6Imy==k)jotJ#(Mz*$9DlEGS%&>Pm%(xcxDUW$9~jwM z76G|VzGc!(hNPzPXhHnD$QTG&1B3LjYHBJfEiEr^@3K;sSQvl8y2w~6%Q}tF*eMCS zairOn-QmuTeLz6YBGIqi(bK>t{o`-ehXRy1pTt5y&bG3$@|BRFO#RFW=@BSx6F36` z0{Vd-fEy%ft$}yn|8cR!rB=HNpvrz2>0}{T&A$Ojg-7r_LeV`-Py0(NhuDK!_=(iejU}%&WW?-y@vFlxOr-iS{e92~(nd z)l1lrT~Ef&)uoW@`kkFyS}@mLf)W7YAsRv%YT+kpN=%;i^{+`Nr0ASx|6QkXOTT#U zwj$K)T6=;HNEcja79d6;kLHWEz$c#`Hk8K=z-O4N> z8D~J(K)Z%(&_5^v6(45X7m5vGj)XvQoNzgR@{XVpz*lpWn^Kyl?3KnDapiGSC!rX$ zV0#VXN>o%B@S_LJq88#8(0YSZG1N{sIU4g`Z>iQjvy=2Sel-_foc*}rcMuhZbH}69 z07VItAVf=kygNJJGzsQZBIY`KFTB=kz@N&MKK4R#YfMM6a!8?dD3j@8Mw9AOM?Q;n zFd=%LoI{@0x<7E%%=+a&(UxaKLLCE0uaXJa;OVJHX$O3K3uN#BV&jY8tJSq~73bwP zr82A{4)$k!x-e!3R%U_6+Q+KLoyXI|$M?pyKun6T;>~&+xbOCL%a3<1!pSAu3a8pIUrvNG7Jmg@;Rd0pW3$h!7|MBQ-8psUp0B#^;0FDebPvOFu zzMc%|q<%Jd0rGJZWs8$1I4GUTEWWsrzB^|544!fBU*rb>X-A?4DD_O#b38^UtsGcJ z83HMVhgn0x*Xx;TQ)pj>TMk5m?*uS9KwWF^;_^fkz;;>|y+)@`g@GNmtw&J`Wcc91 z42_IfK~LHe(AfP1A`<=rhT_A`{XFP)9(qDn%egCod=>6v72Hs|D)d$8CLNxtQc_0c|lUee1px}@kjns zFpzJs8EanxMpid9LZD^=JZ10_d>`(v?*0NO<*R{C2p@R%aZlWQ*hB{kAYdE-qgdioC9Yjqo* ziI03-&K_-Hxw5uxx)%0IT~l)Zcv0S=(2Z9Y!8;qIv#u4oM$I4F|xKnl1eOiC^PPfL4{mZDhDUTl|4s{?-uu9bV3KAtm);IOzKy z1pe`&?7<<(rrxu@OJw~N|G9(1b(E-?M;t8b(mM=CAGgFFJ@euJLLL&nw2;i(XTdkB zN{`NZbyS|32~dBgDWb&Z+NBHN{SyC$s@zhY=-PgKA-RB{m^FcOG|W!>Ghv1r&Z2NY zr|}^`b#3G=5;G`Gg8jJ2h=8KhnMkd>HjTG%UU4rVZFGWQkKm&RpJ)apjZ)327{64G zq+Y=vQ$`&GMGSvFA{Xs z5S0^qd(Pd#_t^#d|J4pb3FPp5$Sj5weW4Om863bT9sftY{**bUzT)GVmt^la7u-0yTuthN#mVs}6|-oNw6zKI;?eGL~t zO?4_~UTSu)JYKYP-vhNTxN2!IT~@wOyju+v;vP*z@11X0b0&=%LXq{$K8}*Ucc%QI z(j8d+)Tbyv#XReka!kQ&55AGb(2~K`zbOc7b?8ImT<-W23g=MY+?;5Ei%r$eYr8nx zQ0yzr$D@&q&89G7>|VZEmwEq>*x_EbuwVz)CY)lt6DQ`Axr$C^zS`)}t@x(UQCA|-8(r92Y7uAlakq6Dg@IQ{N~hz<~`lty~khGA+DjecLKMS2!3H8WRPha6pvO1E2G zU$`lZqCYh7{Li@W&+X*;UB`93=Tw3Y<9PTCjOl6Qk!OvBFg}(|)_D}&M=36}j zOF=P`eBVk3Mi`B`8MctrVyya8;xI`@RE4K>`l6^1s5+K{Q)}Gr+)BGc#(Qim4W-8b zxXb)df<=zdoHa(i7*)wo6`}$&4%gFSJD@zZp*COixP7%?_lH4H`EUD9--{E0!HI9= z(;tL8ou{^|Q_ku1wEs39I=whhgir7o)oT0Y6TqG^Th3^_AK-qR(I->~nke$!<|fga z${paL1hoeUQ;<)FlSPksr57>zJc$4sMlfajeHn5X(jIinhi9bxDqpkZa2a&u0+h{# zRZddB1wzpvGAhm03DtkeP?361NZ)P)`I=kbygnfM68ty@JTY81AD_#h ztdD({_viksrGNFial`e9CL2jzUVtQs^P6N3RjnG9CI7rjp@8i%7gdv!Nyp)zZ0y#mGzu3;R=|#u7Ak@2-H%J^Biujg?;!^veoC`F0JnFGbjuGzy7Q zB||0NaR5$dRrYLVIe-{%W&u?3OUCtoE#%|si-%P}0~7-$Kj#(@5CAOR7@ueSxw1G= zJ{;a20_3!NTj0@eU|S{n5;koQh$Vx=V2+tWC*Qre+fU_M^PdOe;{IO`RDQZ{-l^;T zWeNUZ+H6SK?F*tLsyZ+L?5|V=-o$od$Uufx*pv~k$a(rRr&cG5H$qt!-k3Cj?Dk9v zA}-q2!P$Wg$uNYc9mV+}XER*yTW}ZIulax1mn{_K+e3JESEQ+osD^vs@V{(G@PJV? zgiIbgT4P0jy?@O|2*Z84R_$u&@!3s64mtPb-_?hAl-P0wF#EKRQFNC3ZzND>7Qz7} zWI|-PYC^z3S3}4EDqoEz!tc|Fj$2!L5~+!-h8bb$I5=(f zicegM9&gwmU4>!hS?%kYgx3Je!%n0ij^dY9?ACSR7p_RdM5xb1UkXzrDwU+c#m7y` zi%C%z(x`h%pt(kcR)EHwtrJA0!EcCdNJ(puY4tN^CTzA)=#lF+QduJLN2KW7mlkkT z84*-GCY+n^8eFZ^a~Jj0%OJcrfQ^<9_{`JyFH@lcCY_^~_QU z)2L0R%BYDk>!wbepYaCq*5b@*MYP0^TO#p| zv`=X4iKsU}ZgR25f!<0O$J#OIxUtaH6|6Du4t%vL18f zUG@}OUlsA0H;5)$7;(_JLf=e@{QF|pqISs9Dc~W4)_HAKvcQC%ROrsX*OpY>6c%<*5@yJDN@cLZJZ&o z6^1c{Nc$!zDz-#HEnm1Ca2YXvWzVrUw(*koe-UB-eShS@df&V1-wE<1Az!_H9HoBj z_0q{Vrit`g5-?~}T9%i1e-X`7uC1-b18{<~+k0RFcMa*KHZX**^?Rq&e&N8H7f-7AmxE022(WBrq0?~Xj4HobLH`ErUysYktP z+_Jf=867u5yLfS**sl3wEySoC;g;M)uC(43xRxfPD2}9Skh8RbcH;hq`^UbE)WyEX zgWXllVb6Xuwqu|Yo*y1QUlEV*ZmIKO_SE@#^QWtkT=gdc2e&Rk)CMjlk22pwDAw&H zf6Ni)(M3wzc_dDn~eA5Kx z*X$7ePaOurc#B$*D~9q*P_g(}$4{TOMQRnsPD@_D0)wlu{Qv%F%_SNe8}YP{v<}sl zuL-EAYtUIahxLmEzIeB4DU`=b0u)_Qt-7qk9WwnIs910xhf7_#{xq8fc8Bt~fb?h(r zadhF@Akv$c9J?%xJ8^Cn1%d>D$V*9RlJ`El#JX}FIcT&ZMfY2!HLn$xOMzkvoywn_ z^li(2cm{WBSOrp3q$EvE?R9^_)4F(&*JNE}4SGh*nmVo*(*ApM*Mf*!o z0EQg>M#2*X2t(jr2lc^9@a;bM+YT}(7hl24I5Aqgo!o95w*@(o8=z|iC3%iZ7jOrR zo}YVt84o`C9rkj1yK2^s(k5Cx!CPvjrVBUxq3@7@W`UjDFm4KKezKrFrYfrtG)#N% zZONYSPAMu__=@RVB|d|+^m7wr=bT}#a(0&8^X!Zi0l6OvN+*@3C=#aUktNqN#np*y8J^7d6C^G~FUqQxuO zXSBMMZBUdMS9220Pbex7l^tBQqTk-~jtXAl%O5O!G$7Dz1|0E=nKeEb0{h^ufsLTS zh)PYyw;+;E7N8=n2?zK08|UAjiOT1|>F27I1jSp-G^7Y)_i3F1e!^d+`n6-J8PMaW znvBWfHQsCcdI*LDUo1Qo<;ThxTzKB=-SAl{Feu2WI$>=N86*aYjD*uf&vNSEO6#r5 zFWB?F1QJR@Ln+22RDx72y^H?_hQecFN^5*W{X9G-?VBzx{)ze3qLo`5;7)wx2@L)h zXLjAQ6gcPrj(IHNViN))@UVQmjl2BM!4-MRw(j3)-L0gY3OKhvJV&fgx3)m$K=;FL z_XCJN{d5lq3cY|oWy=FC0>rqC9)PeC6%hDMbiL~8>*iMY53G{7CIU<=fBu9=x5jpN zb#1)Rg(jE&{MayXM+rY48Br=Gv(^h^c>X*wjxrqb9Os}hyR&aER<G9*=ep(&%0Jg%2&H?-!cQ5XCqA-LGXeXL|mmfZSAc>RiUJ(oO8&aJB14{e)kMaGO(!{U9gT5^UJ`)MT46Cg2 zdG}6*Oot&bDosz)uB;j!UR^%OiHqYILx>!nwQCtmsr@CC5gg6gt-4=LMAv?R%w8h@ zAVSqr+d)j#5g|k2O=u|y?bCW#8zuMYd))Ae)&W=y&rKG zRcEX}{{D9{YvgIOW5038ZpvFx*KjYAVu>A zIchW(#P!MU^5))4-Z}&R9$vt^HTbR^ zu9rFJ_@>Fy=!f7Nf!x{*`II9qV&COJp;IcKx&3sBv_5%hzrQywDU1+-A?|)UJ^Ofx zvmx8+-q;a+8Okz(z-*RCpUx~%M-cE{tVi0F-YSbP2kme%zG7y-_2 z@^})AT-dRU;2utN$zGKF_+d#jk#*<;LU!3GF%@`%i40YqTv?6!@S?QI?X+kxpvxY4 z^+o+?nQ>+FziTsAua!&L;&EbIb2GBBQwU8g^j-cOZ@d*qG~LLY-#Y&lzPzv%X*N(s z86SiukwnjUiRzaxoY?y*GP>*jVo~MIHC>{G@f&{-G}HZsv)^k8^jD62p8yUu_0wIX zT7mZ$C;0l*Kz<{OY1>D(%1V%Okj!9MYPjCv{{Y1b-?l7DgVD|mYh{rb|E4IaWEdxv zY$oGLOX{%6DU&xl6H@f?@d;YTpapJAP3hC(ak_4wWtrs$Ct`^7 zjB3P>!c7z>kLRCU#M$^i>jqpqFp5o=tiKcWm;vC@axQ0U4Ki{Vh}MA zVk@ZjJ>3a`sYW?bpoRV#hhS7BQ;XNx^1I#5WV4HH!fl9<#Ti>8Uqj|9-JH|UTS`UM zI(Z!n@L|4*YhTz}`kNw_`smTNhQsTG&!|2cu0i6zrJ47_tG+E?aGF?$GBpvK#&q^B&!p zZ^>%|IC2eLJe zH#|92qGk5*l4N+@)rG%y$hskmMQ6l1m7fCS*=P zt=NMCt9P1f`|_R4hoF^96n6QU)a@q1$O62^&3A05(KDL*D4H2x)k)DU9f5;lK)|?p zK}}nz!?e#F zT80(K?wj0>2Z5k5%vG2u;@M?DCf zSzwK;tGJ!hisE%Ma;S}T&IF4wZh;VJS~w~>9mL^jV}0&-M=M1*vv*7b=v`7OEX{Z@ zcxGPtHOHoUrPxkQomnn^u4Zgv^UB}wODWx@Q%10ia3@&m9jNN~IW0Le)xB*<4rkqk zH)$I%s|hfaNNG;UHOir94H}=`Or!5|lwix%p;{ULwQJ91N6<+tO%N$CSjiP7G0=Cg z*7Ui^T;BYL_fqK>8pOHMnz6KDJ)0W`$0e+Sy!eSL-LK#&)YFmDaBqTGeiF=G90V2t ziN>#{dUN!ZLGR)<6bKHA@UH^B%4(}$qeRqBTZZSa&^Nt}43`W)b+++fei24#iFMXK z)$70%N*Fz;L%d>~TsP-cnT@uM`9@O_pO_j+<)MnXa{0PJ@3)^?gG7J7{2w7@pP#ad z;Aqt{ZlQusouonIp-_PJ4JO19GNMMT`#tQOd1c!O_pUCu>4}IYqyjm=6qZQ$FYjGD6DO79&W0uU82JAzqo*f|w=Dt1#urJ$yF>#8Ki zd=CT!sK7<5WW(Pu&ts2TQ&l3x*!KEw8TCo^)}=702LJpinoPj?;H|c0-Jr(CL>=t@ znXR;=y*;+50+?^*zu}aE&Mn>b8%P@hx%NKGe(#5oaegia)!h8i1Vx?l^R*6KiesDQ z#MNC(#di`?DvUjfZ{`2UP22L29&F;LfJOe@Kc35R|MCq;!--lZ&1myEr2fS9>di^A zjfwV``gzOSUJ$G{xuOTxmQ&pjOr3uc(erR%j7W3{fWcXlk|xqI(<7~m88F$81bax@ zQ`GGxL3z;N*FA0b1!=|hKbw(X$aqwrv5KbAo6=#vu&j}+xqgQ(JKSIjuWR_ggTd;b zM>_1I%P4FV1deb^l7BF{^Al?WNvFWuKmDY?NFNqPTpX}nE7yd@!dPg&$hcKVpau&2 zExCF*+XWP&L9Ei6W^nCbDB1>!T2@#kR%$7Xl_HEM)dLU$1zCAnp&AQOC~BWRhUr%jWXzMlDG0!85vIv3TKW!3Gm~BrEzQ~z zOzkaXNO2Vxdu1iEBC=YzxT38Zv|Zh)mu1<$`YQS*Qk5;VfX1`~RTYDFYJWO}&{xsH ziDE_?PI0bXL%pkv)ml_OVvEe@7zlMITT4-gK*J!fqGm$mDU$i5EPt&?nb*y0SO;@}Y}^^8n2BLyxI zhrkuEYRTVh>+%V9bK@q5>r{C+^eH{fNt2QA4Gc5{bdr$6Dwpng?UB$efzgZ)1N2Nk zs*4zaqSP`TmcYj8d+He^rpX29?Qre2k?HACXMUwwxVIc1 zE`Hj}bahlqxY=Yev;|`7Df+~lvV7e3>~_gYWX(ON5ZxH#>qzsyeN%f!$63$|fbOv6 z%K=n(zyrOz9t#<5c+#I29le=x_x26RVjj!I3!AV%a10KJBmLg|`_worf6p1-z( z*zppoakpNJ*=((v9?}>EK}@x?iHCs1q+DB-Hql?At5-BypdzK2)xpG?gs>LLTmD{; z{Ob_ANB6aBN5b1i^&$E=;vRjzxF=yQdIyUbiC&Zd9^mjL`Pn_Zn~s(*?L3=KnfFFP zgcn4dbL%v$2g`3Vp-0J68@c_@=Yz&9)<&Ml)6M z(fspp@!VdDjQ$zSkP{k$;*$2HNycEx>HU!W>>EJbQR1j!f7PrS>9T$ zx=YL!<;*TySah`1{n(1am$tfbhdEsR7WwcxzHb>6g7VZxu9mV=F~4;kO0D{nKM(sj z#E;Ja4-@(RP5zNlEgqkL-m$w--dO)`)TEO@S2^;iZebC(kQ8S_Onv)}&-4hL0zCScmn|2q3Q#ttn`F1z<1tI=OkRcWznh$!d7&+(*5`^ zEY|~nYkM!%VFFgSW; zit*Sd8zc!LD8PxMaFoDLc|c7i!W1`j2}sfboyvL{`k8jwH(dsVG^lcnRGFDeQn;u! z1jvGZ#wkjGDqZyMOViXGz#p9#Ay{0Tom@VZaXPqrg zf}egYs`Mw&p*U)1Y!NrKJ%laSj3OA2ACZx=jO}7jlaUq<%8+~b5>b-y#LqHZ4BX8byaQX7Xd^tkljF2~nyY*!-EU*+idsim9!gpuTN*!xmTX6LT@=Jz~T zw1n69B#-`tjb=j{bIj{|`1$5=wxWRs8k&0KWnZzwZ=MO+$hb63K4^gv+{#pJH) zXwNTQQ~D>6W@q@Ar<&7b=p3qBm)I_ON2S1@)5I^u%x^dsNZ<4*Fw$CO+g!}V=5dzs z=sSqc%?}Pk*-pnjd15l@$y7gXBPJ<$7Fo%BwSRlQ@$>NNb6?DpF!4`3Sz_N z&LRe|1R;HA_Lo!x_{_f6B6kx>kMx*s0>$QzI}vI!FAQVcz6tm;)vRY0xY#L4(2`!J z*UykgpehI?s{h4coIY^Z?j3w)LJtf{lk)rv%ik(7EsZZ%X54SqI7y#!* z43}0`Y{KgPXDaQUowZtTKXBwbHL4xACDdaJ%A#k4zLVH$cg`=_hcY=w6Ztje0e{RoO%vYZVeBv zTZ+D@wT>ArDJO>yIPeSAFk}8Uq)&&&kXO&>Qy6JR&3%lWI#w+ZZ!{&l6lY`!Zz`Jt z5$B1L)%l^OSoSgGk?93|@72gGN|K4clKxqJ3!;RlV;hP4k>#TPTx?5(iPP!Hzn{L? z?`I=c4Nrmzz=U8-pXo5wU04$Y5VK+^?v^XTsry{`&X}%^=XK~1)V1p9lZ*@qB&mWI zzVobfVv#yQ4|~1e{7aWb*j|5TIy*A3T|-A29tWpQ_dSJVb^7E3d0!;VKC^xR7UPsn zIy-;voGUYiNQ|PSMUuk5OfhRB0o?6vkH5dW@XDF05w?B)hxLE{JQmSG{Q7uDxldU>7v{bn+=A^xQ(Nk(&t=+cY}KAk$ti`{dV$>E4*Fv$IBUOwi(s zs}Zu9f!M#qHS_vbpF{9iB*;W$?g{7N+(vKsb+xq2+VX(CJ(zP|vx234%VHmPY4(S5 z^~!1cdS;*e&4&5-MV5X^=QxEEjASwa1G~Z0)YQ|{)8S#}S$X}DH+UexCWjT68t$}{ z$1;aY(@u8$p^--@E_{Qkt&mVTLq9@k+PZ<$jzu!2^_#A4!@mC%{ciRG*cWofHuoHQ zd~e3b@ovSvK;oOcDWUL~NB)anVBrhuD@rW%f0G1J(e--l*^@;Mr5x;+I!yUOf2{rK zG>uYBKFG?a3?#hDTNBXBV9iu~#lkrpZS;+-|2XLJAIab$GvqoGXe{iqU&)_ zaCQxBWUy77Gl%ILXid6a^ba$gm1Z&`D7WhHEG zi!e@LdCfiT(daFX=oAlSW`W74^Of!MU2$j5(C9IuVh|>tlzbf-`rF)`#E`}unMM

LH9AEz7Y| zMR?aB(s93OB&svI(72f~a&qB7Mb_<$R@uBC8G+Pu~2XybR`8!-zdEgFc})-|P1E_uE1TO~MH>7WKw{ z`TPPRp##ib55w#a51=v$a=+O5m*v#c6D9?cCy8RMi#hX@!9wQp5~$j`yLDN7JyJF+ zM5g1{u6iOsBm0j@4gR36kb&IY)@BxT1a=wa;1DOpBl}_M4@T2nU?btlhuN#j^zp6B za?qh!1JNsaSi#dqcUK{bpp(JZaqDMWzApwzw%|I0z3*>Vf-aYqNAD#_#f;^j`P|^6Iwm*8n^u{2bT{tf>J?3nmi8)t%pEB1{@0FC8 zQ7z7;w@J5eF!+M=Z`uGkO(NTg_e;$1-*@L#2o|caS`(3f8MF@vwU0Zaj~Ct2aBH8R zE?&ayDkhIxvOovA^}l>g#K!Nd0jQLut567JUau-*Z6YJ6!UR)D){Z{jSYP8LD)O3+g$r&%nA1}j;Dq%lTF{3oA=V&3gXmW07e%M7b z`R5bK6{@(0vjz!LmMpkWEs(?`MZy|s(m*}QY@t}?!9|>$W`U>5Ku^RZt1w@wO{8bA zGeqtHcg<5pOH9(s_Gch^Djl8uGX+K%x)vs-DB}RI)l+gMkys@f>FKX_X~NwHi55OK z4m3bN+-0E0DUeGSYq<+`8S1oQ8l^zl5s3eyaK+2sB3P8FD*ZM3z?hONPv3z8Zr!?q z@s&iY?;?YNJ|z?xGC)fYKhJgsWfSS>{o5{cSeW|I`7fs@y;4s2M9}2Vj-QqEQaJMo z5qP#v;uwV@IvCIK@CJ3cq$+0|)YEzY`%y8(WelH*vr4uuNT@P=pY|+v4A)WS)nZCk zbkYD)VWg(If1x^zr&xwiI&62llFKS8$EiTS%}FDlJMjNVI`4R@-~WvtqK{DarjWhK z-ei-NbvX9O-h|AI>};9IN;vkhN621D&ap>!$CjD#yZiqB_c)w~_xrwIlv?ayrx)Z62#ZPtRQY%gpj8tN#B=`QbQrkDe`W*^kpC=vf%gJI&XcyEWGvaa9Ua z?;Ich@SsJS)*>6{T_Ap-@RFK%1GJq0gnbtb0+|2b9}5vGu1Yh(<0q-T58*AE_+QMd zGl(LTewCGeatS<10D{1`?GYR-ou}!qF<>#`h&T@V_?o?XoHw3-^rJOrNuO2sAtvN7 zB1tT8^o@fc^}@1|lPSs+ZlLfDR~U9%Q$$lV+K%eIXJS3-h_J$BR;O@Z1oLkwJTh+{ zH2$4(C?5<^q=kp~n!OHfG_kuK)&pj#st_a6KCp%j^b!Ba3sc4{IQR=`1jS&oJZ{8}PWg38ozY z$GP?kz?7Ha@M7^nKq_`RMeaHhjrhm=clP+qof1kz!lOW{TW4kbFDVmL!$1WKqUPW? z)78O!J^`e{GI!S}Xf!%J{GM^$s1T(!7lwmrilRC+!#{BuREu(DY4wVi2fGJ#Rc@q; zV`bgJvmooH&C3vSc=%i`i(D~zqkeW2(?8#^M0`ZR{6d93(%lq=;dAvz@o9WSpKrxR zn;B_DC;I3y0m;t<1T&RobgihJ29AOpf{_G*qlGLITInva7&I;@4!|7V7RO-p5+r0c zqIdB$N!L%$)Hh47qA+5RqsMt*;PlX5BZd%1mYuK!^Yc&r6x&%xos5!$fGXX5?MJv6 zWe{eZNX8*uCVH26WaAGD)k3xsZ_`XJd#C1eJ_~M4)mn~UJ@zRdvVTuc&l+ll_}y%B zeTl7E92|K&c#*%RqL?mhYYFeTn^k5dI5>g*MrnP7y;lmR^#1RLYeEMXT(S^|PJKX< z*c+IutAu=iY}36SZXAXZu9}6kF7lQ_pi5vRMH=%VjYd4u&r|T7oB929a8UO`CC4(? zb^;{p9f%6>i`Psx3R38`an9}`h_TTaIF-y^!m?r%I0>?z>I}aCq^a0uvG-8yGps(M&=F?a!Mxh;w zL9YcE23~{p_4U4O+!5R70K~a`cOr9l0!ScT|L(htk&-o0|HoZ>=vlItCRK_4@B2^M zC3%5frv9&c5kJ_k)M%133dFp%Om$-+Sw7DhW63!~%>}67+FToXQJCVVT@7*L8gL>fH{T0uT>0s{r5# z-6LPtwdwHx4!~v#WOxmW3V#efTMM~94xwG#xn4ts@Xt^s_ESXP*AdJas~er)6AoK# z;Ry$UzCzU8!elEhMX0XeTaZ#pkMm)9$l>7d5XgFfuKm?SKJZU{?AXp8+XN|kgjgV$ zbpRi{sz55hvuq6jEn?B2n|LaX_xtQ`&=M;#4YQDfreuz9@bgEjQ`Ab*Pck+%_2*#C^u)M9=~J* zEgf{<+;5mOBLo!A(_G{0^$aUUb5n)lk>S?>=!>c#D-Z&L7i_&!)NXObHw>ZOGgz8L zUjBFFIdh>b1I~~me8@_lD4B&@yH+~pTT?(_v!$$am;v*8uZ|!a~^1g(ab2TQZ@4c1qHHY$`_UNlIy)3~t`$ zzzhfyvw)}>nYEeD$q=c%jst|HiEEcfR>4v<@5$>=CO1w4vSszfl4j(??^~w37vE!J z=Fq}umzD@R`pwLEW%=D%xCn%Pr1n{@{@2|oydm2^ATyyGCE~AA<E?p!Q@{uWp6umQYD>Ms)1Z%G4*i|}|3(AL zz-78IlKK9L%$ei?05;@iHGWR@2$3F$24d8c%ClO*{<^Z)I;0*%2fMpqKrE6owl(n) za2`$rI>4Jan&9%tIy5KQYRP|9)uL#UE%sk^i9(^kF3}+y+&BdD-lIFQKSYR*?n@gY@A=eeM3c>}gr(VpOb zbvuxC3JUg}-SdSS$KjocCi>e_Kma{@c9!Z1#<-GvG&n-KE$B;}1*w4&>GE%-)+P9K z@VUz1Is;DFJ|H4(5x-nnJfG1|kC6op=&PFc3*08V&_Is)gi>}+IY@tsFuY91?sB#Q z_3q7Ico{#H*6sDLE)g0KbzfGTyAU7s`W=s&AlrkALRn0{Mvh^&SlEUtJK(5|sJU@P zN_{D$-#!0J%T6s99c_@qHS%>%@I`u3e4u!K1?+PyNsK(Boxd3yYXZr4U+L^U#DBKg zMYVr#EIT#+8}qHx7<|yMrDn0%ZtRg6jURR&%b-|3&5EQ82<@py91b zCyr#Icfux;cwK48Jh47#BoMV>MOZt(M~ydUoD!NM_`3(hV*z-#Pmq_FhnJ8kua&7@ zit*nd5!?oNY`MHZm1i0upxTx&|8i_ufvRVeI3!~f5;Echx(E?s8r4Kco8&dP)V8_o zHv0AY9*>5smz^4f&!mu@PTR&O4-?7Pkk{7s8)TS6TONNVVV3-__`70y6>`@3p~I>p znc8Zak})LX1BHTGib7hM23;w(1^mXBMqRG5d>AwKMFe9dS=;aF;)ndnYl8-l;-mc( zl<1pcoc{I-$GCJIo{!z~W z=Fe~_rp^ektGkSyh-}SKv;L9jmHY0xsV@c{eKjKXTm!_qCCPhd!v!_320Er4^+Q*U zKPc;h`P#1G7sM@S_HX;TE+~KkGh~9=1YAs#|2;m!W1x&!Z4ByA`=CC~EaMi&Q3Nck z`};iiu%Q`8qG?24kNG^yt)E8s_V)gBitlCA_2B(+TWYuD=D78PbsWq2nr7a!`aHFY znRPdaVyjw#$e0jkTeiqyJ0d`#hzhTI2pa*R|v0 zl*<(p(kiYIAcC*sS_Wfs@cut}@d*Cc`=V>TRUx`HyxT%N&(nY0=_6NTcr=h; zzetI=cXcuTVFE|X)yOhxBzg}%GD9l!PXF~qsul}L)FxrF@|Qp}!XvPR{XCTwLpSii zly{pTwlF%gV#D*&j_yr93dadG#ODpuf5uxO2q?zxQelj-h2S6);I&J2AjRT}rH4>O zu2nF)OFigGj-~zB!0!*GG}b5`+7FP*LWD=F1+{w>qza@Zj*Z}&x)VYKd4uo>qnQ-S zIUjAzx21G-zRa3$TH4ER>%?ffT6#-}@z7rV3zF&yktsi(VymvG8g`<0WbHJEDYI%y zX5Vth#ntig@g+J_-6)DMkM<*o`F3}Eb$)EM%E{_3r6pFSSLG==pd)Y4DBph{3j>Om zQDXklG%rIbv$cHCKvfHSQh-sW)!;8X3}vaJ3<1>yVU1#dS0l zSW8#6&Xui0BJ&{#F=wOKSkXp*zZ>eM8(4&;%at*GQ}oH>QAYQ1yih_b5=QD&tTxho z;c>G|J@Pc${vAH>9#rsNa39+}pOpQBgC^=Rke@}o2b|bNY`-KjUJOPMF1lt$8&r@v z5qiQ|BUUO$^`QI9S-G!+g6Qr)G$>}oG{bU17>}7fi)B;@JlT9m!U||Ul!*Y%Hfq=s z!~-@(z_U*2U|EM%Ey|A(&L6ZC(SWRrp~Fi$I$G7isPDTx1ol45v39zp6PrIz@t?DB zaFOFhsg;-iLS%D|P}ZHFgZSUVGHCkH(2cS#R%=?e1tK;bIJWmqVV$0|Tc;IMY|eZ8 zwDJARmz@&k8bQlKN(Z}-8vn}C^qqjGLm)OATp!>F$maz^n3i|iYW(4;RA}e*Z&NEE zIQZ%m{V)4&bJGrB+O1ehY?Q^9&$mu4f#|%dstTy$ZVw6WzMr)J!#yZNGr%F+b&$Vzu z*Gi7THd=Jkk#Dhev)eL;aw;?{xbFi@o*If_>o z82?CJz0ph3dFm@9p<-i}BE%FhuVy{d*IKH0=?poP*$u@KlOArkOyD^?+7Mb5{bzyy zRY>MGL~AJX1*hLfJq6P1y2|kI|4n z!i)p8T0P;kXI0esm6_-qJs;po^SyB`t10BEFuy5|vfAA3aP2}t&X|lyzfpm)Q>v^( z-Hv$Pl18QoN$kn}FZbD+3bn-_eR@i&YROE<_=y=ynM3(!J82D;59*!;xB1)lDz{3q z_Lnu6HqBWDu?b6G8}TV}eDqs4!adrAmkqY(f6$1`IVXDhu}9sRzx(@8Q1ASqLPORR zwav10cogL$o>(AFI?mJBE@4+X`sFOtX*`i>YXE&}@`*%|@sGgE1iPt|ijm=LgSHMM zE)aK3h^3hS6oeV|d2+I`#TF_RC6}^4pP)sGCEoqZRs?TTD$^>$t$(&8t+Ks_?h{}B zn?Uc(aEEG?Thn~>2d0?AZ&tP1YG7Mh!$vlba)7TTSXwRyNj%~`f2Q&_dN*jeTpy0G90dp^oy2LRE^9xLUK@;{>87At+W-(B4=dVebv*lU1o z&_a+J>LfLR_coNEZ`^(P{Q1ObuWatZ&5Rd~qT??c5TqnE@pl!q{4i?o$oppU{bT8% zTe7#kGiZ?bI5`=H1hg=-u3&cIC=i@}Uk)fn!CC<9X>Z;Th%fdlKM7j@E6pk7`53o1 ziRyf0<9L02{qI2+&~|7ry`pJfbvMdju7)4~_=>#Fo_`CnP!IB6FR7;3NItGt@g!9Y z>})``f9Z0m@OVDB_NP}=^nJnQ0+=+M1^`nl@cMa+Gsi8sI>Miu|1QjhPYI4dK;Sn$ zj=YS;enN_XY6=H0+%u`5yu4^Jh(V!k66EC|VEJ7EV^?_&DV2s5r=A;Pu9Q(XuJ5a3 z`RrMC=F-wpIH(HULQ3Cv9d_qe*LBOkRnrzVKcgHCM+Cj4*7SM`Yogn~+78kGorVzH zf5E+~FwdwlJ2#g8dyiQ_mb{D&H>ohc-1xiRlaDrevM$M%-1g7Ev1ADk;h9#h`E}+l zYV@xSjd=^JPYLcd>#FUrZLqD1J_;zOJ6`0D#9|OCVBjcAL8=ftB)d#%cH*V!?lwc` z$_4IYsiBfPX%wnnxiEy`%i127a}5z1zaanWXexmHk`WqS`4_9FwlOx_vSO+MS*qw* zPgxR89af91nfD2D$BFS_rhBwK@7})HF?W34B=^;rvu24vR+jK>bdwkRggR;?4I}$& z;Rk?0n9#s0|O{yxlUuSSeL(0TRs~9sf@i4HKTjb zgt%w==b~8~!8|O>ne&dj+q;{S#qOlG(~geCJ}aGrAU1E>fs=NytBhI&06vL?XdYf( zKv=*29CG!K9u0UmGS`Rqlr0s!O)oP#ihQ6M|NPP&ZLA?Ts1P_}X?Aw!90W-Q0WVH( zH=io;W(P350YAW=Z@Ml>U&_^Bao*Q)c?VMcu)l+ta=&Zgh9c5n=&P z>zU;H7ga6OW##3Iu3oA=zEw^cNAofBMD3n zx7XnD1)|m@DcVPL15hplFu#5M>SMt#7LXm1Ml%gCde)=hjsiY$Dvei!o9pW~wML|^ zSDOIxINfz+2>LzAm7vSxfAC=Z`ykO9Y|NYy#^yn%>0ADR!!yl^-GY!6hx+>3`w=go ztsV4Fq?>`v?51GwSYluRs7ye~C!8e%$bRZS=yAm1ja%4Y_&Fi4eEzCPU14-Mw&hG0 zhY17XK^u-?$!-SkmakrPKo|r3YtiMeJ}{U)SZv$kV1*8xOnxIL(~DAb8ODwGDLJOg z4Eo(B0q^rRD9$?gYiq_J*R;4VXfl!6)%nI-QvA)yVYpTu(=y$zo$7Bfp71hEM;Z*i z!0;5FF<}s47{`SXYI&%SRA0G2c<6@xRt^|UAI^F{_vF;ZBPA~oP|UV<$WK!;si|=8 z&@@QBbgs?LEcG-MS=jS_yO;%!R&g?g7ImvJ(4oBOUbVm+=_$HZ*m0OXrK@1xGjOa& z{!nTf@c62e&fii}%;o(_Q9q{>8d|AAH$bEt(CMvB9E!~Wm85&X^{KJn*wg^0fQ}VE zh*aLhBu&wbJesoX>#3v?Zww9>w0zP6B75M17=FK3I-Nn#OHlQ4qdz?VDdk-yqLC&q zAaQfP1uxSBQgG}e+?LOjYOC@i6Y~dVi8J;?WsU|542~cDYOv&vi!fs60%D6Se5fFQVsMmk=p$gFw*ej{_cCLpPyT{wOuiJ^XbTn2 z8M~PAbYd(C{Ig9&!&IC6=ifKJU1ya6R^XvjRb!EZpU}?y+ zWgVC9O@|uVQ+yfuS5nPEh+6+sy7rrJOMSgd@fgG+9sbnp@BRH%Z}B|y1h9d8|H=RK z_q1c{;0dfyazO_FZ2^&Rzlb#>;GixV{LMKHkT(4X4*at*68yQVt4-wN*j7C=_P;nf zwU(S?bx7!(5i!GTIR{q~x;1gD#J~UO5hN*xj2-fIuywjoA-l4%v|%nu2t*n*45o6y z8_4C2u;7T~@qmu_sbaQRCN1PptDrF(mfY~mb-b`o&kb7ZAhGobBARd}xoC^-b@2%w zO?%EdB`1dn&SFk7TtQ4-j!43790g_!!P#MhMlPIB1%(O^AE_vZ${Qqk7T2_vm%r7= z)5eu|hcI)vlinM=k6+ZTomI%;WRNe*UQ#Q}i|c~_x9-T6K5{OR)*GtKtu*7*lY*R& z!K<$K&@V@vK`52Bd^|QpZCotly-n4>DTCjb^P- zz6)JTNvK|?hfapg)gC!6Z2T2C)RD&I6T>HxajBZ0??G-f&jA!9&p?9nIImk9%Qb~b z>!UPxEaSw<%}r31Nyq_6FLt{-a|3NpmdNquSkAN~bA%uG->=?EpHKihwgunC#hTV* zaEFS7oQQz6arl}j5ZMjg-rfRO56~68r3ZanysH=Y=o7mZm;eE&vQ@`IC7#{QTt-BT zuGCetF!p;a00YEuz6Ohb{wt_&U|_p3T$BogQ@!JaEh*SG7fjl(`;V7gxpm8_rf_Ad zm!s5PRjQ0fyk>DrvlYwJEUL+2K6ytVEh^dkS=}v=wSu;^@0)i#P3inIE^e!7IvLx2M%Ws{+2O%dKhCuV+xwXi38rpW-xsD@*OOhkSqQ@T9FR!Xy1TGDESu>xVI~ zotH8PvyFaQAVJf-OXC;ydM-@^hwQ>yivnV`tNwJ&>dD`NCN~DG=B)BK@4g$fj(Q5` z>axMUyk#3bk$mw3t7g(34DoUO{rw}!gO)P^Z`-vRg-r}KXqeyYSaNss`|j#PyMepf zT%3d@r%ymINLiQ8PX@IFhwu;T5+B=#L=JVO90~mG8&E1!hp}Px5}~L{^qX`c)oh+M zADwh8S%s3xEfG{yWg2n%cjGPbrwRief>1+Z4vZ>ZY_TeJ%rE772J-N26Ha(eg5N>^ zsFtWM2LrT;gHyc$(Y?t{^h{u`fYglixSdeA{AKdjO0MH}s}d{>VhlJUA5Bc;HY=|C z>Da%ySs+F@sy8u7S7oTv*@W0PZ@ppja8IS{wt;5&0csca(0K$p^8u6UMc5OMZ|0+h z$T@nPIk>A%I>bUFkF1dnVr?B=mLXE_+}NnPquRCeCOJ*lg#Jf=5+i%M1&I?WtoUTL z*O(wa36nGXqNXj?p7yp46W+^(VeVp3<)u-<3^SmevMV|&g=CPcZ`RH&W+>*6}* zcCUWxIfQQPaBLbD6R}DtQoo!|v5}SM-Xy*rr!KspP=GUHw34p|P&{Py^OmFimiC_4U!PZvZjhV%uwI z$GY3?rdws$!22gn#_)d*QqT<@od1=%ntadW%JV_4*~dn6Dos^D9*2d#j$Jt$#CX}E zKjhqRbgDs?I<%Tqi#61GHxmL4@UyVbQ#+$}&mK#H*gG`ZY2w6)IPS}z1f_dKEs@7T zz{Trs33OZ8rS-EK&fav8Q1-n*_Gjl9DoaxPF!Kayzk|=JunaLU9NFEEW`e4-Lu;DY z1Q?2C^~rsuKMowJiM^G9WcEu^ZXL`xi!Zv0Bm#s}96hRbh2Rjtn1K$(dS38FmcC~| zz!fkGYkhu+Y+BV{0#{v7(9QZ#D)@?BprHfs1{1hJdy$D7t3DY7^oKwQcF=W+JdxD& zS$NFA==RK|)~D0z5^4J(UxvHO4z$ZUP^YGHd@ioh@_k!rzJEFSH{_QUG7Y`Rrz&82K6!ED@>r6+pxQ50CoG;FT}pLrJJ z!X_xKWTt1Wq_f7_zrLS?7fhE8YNDJni#cHr%3#Q;23o|VT$)XZe^AhUNEy?-jXLur zQP10SqjcMB1CJ~lXhyoCZ39EV)rQw|J-rR_7e-PKAr1Tz1MmC|vU~h`&KZ~SKr0XiHD$K>t*|;Ie(^38cRz;qP(Zqe^_+Pxnx#?|bvYROJ zwD%C-o};Lvx)i5xrd@I^wUoY$8deZ-%zByL_mg!*MzOFflWC{DwTDB>)dq>NCL zWmge=`+e`A(~Gy%O2>A{hHf6{fAuoS8Qk*<*~sJi-1+DW?TS7Xw|4@MP;|r2!zke4 z3s7&G?`Um3?g>Tt0+EBV2IFtvHb4r3X6)_lfhvzL{Vm=SSgP)M0ZTWf{zbjaU%>S- zAqQIAkiUCPkPE=z0@s?f{k-X?^78(*BT#~-FfC|_Cf27ecKNV{G-xLFFh2B6sYcNg!$bvrgTc6L(#j_ALI zRreW31acx3FyC`0yYBvWg#a1K!R`LRE$Z&d{QT@JV}!X&kunOPgn>Ql!?@9dchx35 zOVD5G(aoA$R6^##5gC6!(Bu3P-57MmPtP$0IrAWW=|AY>XVmnV-Y^ti;D)$7n}Bfs zI9XzeJHodmgAX{$s9ir$V3Qi?t>7BkH$?KcWZVzhQB5n8qr^|+r8iKM zW4{o!v2CG!KU^m|BeA`Yf2}lPH^<*D-6(LozeJW}hcVU6I)W5}613w-NJLR%S*?2@V@jf~nxBAu17k{I z8SdkbI%6i>Fj?IP1IQHuju_qVQNg`UQBOb2buZ8`GTL>TIKi1A3?i&mv-W=S?6NO+ zbYGPv2t_3s>l7s#Vm|05Rhq98D25FrvBW`o7>|ED>g&sk;bO%G6u*nc9FlVoDesG> z$70(^G2zT}DIy>6NYsH*K_G_m^8yR!?OYRCKR}q!F{}gxnRm#2-0Aw^(_A=F$Hg04 zo;LbfkCcLeec6N2JCa2qC6PG)r9QJ5aT*o_RzbrGrTlXyIak}3h!uSlcW)3)95*r2 zx5(-lzG9t;rBDqL5;{{^6KnkBaaeFT&57_Sl)RrJl*T4%#6RqM-|-%w$Jlt~{YL#? z!+CCvMf=~4i{vtiaU1G6ayBcVS+NGqxp1D+qUKQqeS(56TS<1uzMmw|@ij=jYRanO zUDoI?I)B{J#5|2HXAuwO$$XVbF)9f1-EXGB(E3~d0)R$~CcdhN*3bTX;OiEB6jPYw z2pf&z??H70S`1JYW(H1((72R-!;Bf7Y-fUHA9O3Pd<66~B**Jno-eL;Vz} zM6xve#JtcK)^1;Hj~v-1C}&OQsINgfq?S3)lZBf_-!<2+rn~W8WslrWze^`}+mvtr zy^{b}2>_QYjhz}5yaE0Wr_DfKmrg_t95{7Spwh++aDm+&KR@lZj zLm#h){(BPmH5Str3U(Ov+w#?@it^YSr5Su1-~j>Ktcevv^PmmU@yW@_e?PcFymqQ_ z7;<}kX~FkT1tym6$vdc=ewVTXpMSAJohxXZ2?Q%)TP0{cOr>Di>^e~>H~H#P{z2;+ z`l#n5Mg>!Np1VX3N>xHS9e%cpTj8>qsz%fFMENYuEC;pnNxM3*r=Z_((XXJp6vuVjvR zJ_Q?3VGh|Qo0yikbyc+m){e9<4Ya9Vrs+{z%Dzp?_@45I_H5+m+@`)mW%{s35GzVa zMei*>CdUIds&HL3@kat>b*t<@N!gvaU2>VeMMdY@9qoPX;MUqY5u-T1zH1U8`r=#f z0w0SF>_}Yf*?aikyLZZ#PYWNWrc7o0P@=-qJ7G^RujJKJZ}~(t456pT{lrwjs7x6a zy>3K|RD1q|@G>mzNAl?-9EuTLHM9M?g|(v16T&Ra;@5BJl<*B|_+$y$rV}}^=WOa3TM{WK>W`*9>qd#)(|}(kj3Upy%DC3a2`l41pw3uztboZ_C*nQjgz@m7Hj@K=kzc60ir(H9z;Jd37=Z_9Pgc03G_`!FeyA zq@<+HrUZdM;EVP87nXA%wbu_uHdsftt~w`ZKI7wgj$3f3cQrlxXm<0DiP!OlfDC_J zYil{^@9ys_&)3zdwgB3v%SiI3uF|M_?Tk-P4MZ0Q0wrMY>I&dW0>j8DIunL{f*62L zfHzG`4$M{mMahT!b-UXOx#YdG6$V3rj?;iKx*uRJFkA-|5*h;+?_6@sjVg5&W*#^{E@bk4|>JTnpA*l7Vx z1J&B`@`g~{$s~HBrkB!m4WC?o^5cCnqZYbP*18yla>A5V6JwW6!qj2=LK~Sc8menT76T3&Y*Oo(L8l0xKMI2VX|0TmPs?ZheuW*TgW)_llF~qC2S#R;lye2sgmYAs zY)jSJr9N@}mv`E_;<3d7{YgTJ;q!~hbzI4g$#0wVoLz*clxs4K$a8d@+OaD^P|IqX z;gb*REG0i%?6Q(RiKHc_=&GG_=ZLKdjO)O<-sESpoaz~NG!Kca_2Gz&j`IR3(b|t<@%%5^pNGt+w@K|E5 zMGdzUrXy1=h@*9~1%D1fz>N~6heme-Tl*6%tgkn8Xi;~Z^%C$38DD**a*jXpp(1-u^XS+Rv9$?B<{g@pw$JM6Qn-7w_0%))z>k&8gK z$^%=~rbQ*n$pCw>4!B>)*EZ1c%3tC?pqLmL!0duLp&ukfY488-L?AtK;o;BFv*UKp z%--5^tlJ3e__l$G9*EHc^6wLXy16a^IwTQusk=W@%UPP6iP~{}zc0$b_4GQ<5s~Qj zLF z(?H%7)mHe^c#B7N{lAC2hq3{>_buy*dvHjaL5HZC@2`-UW`JgjZ~xkaOc5roS^dvn zzaM?tUj9q;j3zjPylALw(xEctVr)zZQn^`Slcnu2Qml~~GrcacKl|sQ3sU*Qb3;fL zq$fBPV>}>E{$$4Z`%l2UZkn)+J(GeQF30y0b}BJ%Q)PApiMX*73hwEiV%wX%! zuWOL&vcsWhU4qTbOhj#}r)Y^o@9wmaTWPyN&47g){&vPC!Gu{Iq2*v~;WY(-sJHA^Y`4SECI`F-_$fvqrZ15VaMgm8LQrCb`h9L?Kd zV#oSDJJEH|D( zC|Dk(&WQ&IKftM?$`1VN7WIKou;hS3!f_Z#C{FuwZt@1bMoz`^C*p2;m+VuH zkBmwu7r1hGpS}5{$2)+5-%(yML}N znY+sgIO3n_5zv#{HU^L7EQY`6I!b>8;c+)NH$Y9BGxp)bongqSA$YwX^(I}GD(ci= z2G`I2NqerK!5jdpy;&(3PAwdr)z*KR64FP=;rd6dF6 zbT594GF5-pw{AdMZ1)udM%tPY21#Du6d>Q;E#>eo^phrMB{%Han>ySTpDzKspgrFvVbWTvw_%azc7 z!Ji$@Uiit7GenfQW&^RqlCI>2QRz|D|WOyWZ>o@-xx1PP<#6(a5n<1hl7Tu!e$aPBV z&`>UFJTB#af_t;`qup+}OoWnJ{NuJRd#@^(3U$$q%gZk2_vVm85{Vd2Z5Bw(EZ@^QB>?89;4yLN=+~FJ z{YSA64&wUai&!C-og)!_^oPKGXnCca!5nT@UB11(o}4y5^?3p$ibt}6y^?4GZ^kp8 z#}bQREZ7G&$06Tm=jY#C{-M&pny%jh!a=Z*mFUf@sNh`oLFl#qxUc?SB(mQKx7WQT zx3%iK_-j*!h=eulcg85X!~p<8n(`o=w9mU|581<#CW6;DP@-3Fmr^MJPc?%hPV@8A zG)7az-ws>i*KMYp?s+9_<-jGGpd_Bmcz&s()<8U)GZ{b*7mFi@PFluN0s z=W1#H>9KIJ|5vrbi?*TgJb1V8I|YTISeE?A`^UH7xEU!lTJryXeRq9%w>ECw-YE*) zs+pNDOMcpj2OJ&-7fm*|P~okXI*2`#{_lLr-`z7fz}M$; zKM+at=yAw3@P|K6P)bl*yZ*BawuyiL4d|WiJ|NT7{XSVb_w8w~@RyC<<^QFcSR?8*pDnQkyQH7v74$*o;&Bg+t}ThA%%&W?QTB3st0U78sPeNMC3I zYw3Iy=p`K#E>Ex4wJOrjZJpRsVoIQ&+UXn1 zl}LGL)AWOxK&r+CGSjRm3fhN{&i?j>caI+}b=NFs>aeAOxXiKb&7ru=3G+_BXXbN! zMn&*IXtF6O-cQU32x#C&)CQ?33FQe>khc0D8Qu7~jrOOfxh|gE_v92F;TVeb5U0&C z#HZ2VY_YzVS$MK92s0pSQp^iPg}Rk7{wHu|9JW^<={3yoNg zFFqLirC&NiADbOaIN)I$PmG(~z(b_dqNxLzc*w&qliR~(iwuQThs_=1u^?4UHHB3F z3&Qmh~b!%3Fjl-f4ZJoZ0m{k!`PV&zx$4w9HH5%@a=)pwW!N5X-<-#PtpmK> zAA7!2OikpV7;FyTOzmznTUdN{zbC}TPCT>0s$Sr0KtzqsYtE4rQ<0K1^r?8FwStXt z02Cx|Kg;R!SpIHfX@K_zpqD^@({(=-?7{2yr`<)C zac7SX7ijmxmu-u7kIgb{oxhFhL#(&2v$Dj8ex!GETk78aG9ja0K7Z4J0RgMnOkT5R zJl$F=R!k$fC?Bzzo84D3G>_gDRrm>9BJ@h968wa}+Y5GHACD9jwMCr0dlKBz)`sVJ z1PaSdCmwP<##ZpMSkG0%d>emU^mS|Kr=S{BsV9% zqfZHwa|jGV!Kg@%&>jw8Be&b{Z@1^&y1?SOXtHc-=MeJ{5aOE!oe;*7siHFfW6nOY ztkSDQRKEJ!;afk8|CWA)IQS872*1Q2B{&~Wz(P?p5jds^9O}7Ds!@vu5~{=#55+sG z`0a#q!um8TOl@WZTLzIHk|vdb}-y1d@ zI)-%YKa$I*v|p^OleG)dhJ9p>cK8ZH&M!30GW>Nkka%o(%dhDM*g5GM=J4a5egBxr ztX15jgq6{#jjvBeRfNd+@rtC19%tdJn{bXlR8QY0C}>Q6s>7_BC_+KdWb{N(y#zrc z)9ga{l(92|a%4kCX1V^FKc3$iw^|aZ+yEdTa}Z!CPAI1R$?ODKh~0IIvsKu~XCf(@5`AFgvjiE>UQM zSl+xh#<0&rVl|Anv2`%GLCnt;V5p#lZ!+m8+bwo~&2B+o0UF&;jF+E?CVfmZ+SotD7Nm@Ws^3&(AdmRO^9p(Qt!`Mb{I$o^4R=B^0up5KΞZhT9Du35sVSf; z166w3pYZwccB!7w8$xO33JHnwGjsjVK0`}A>a*cn9 zvM{2KvL;>!2EOhv`zO6zoCv;B#$k+QEQH~hWH2+{mpi=LxLjN#*gW|X*}Qxp9dx?a z`}&=Bi*wBB?(DOuL4O9{TBE~)LN;83FYTS3w}3o;!^-=yZsF2T@)i`OFcc6wSD`_{ zZKxlKB{>KHy*{~2s)eo+0CXk~#E(;2GB+K|e>>=L%sK;yp2PnEbHB-bGpja?(4if8 z5^{ZHRcldq4FI+&r8^+WB#xi&IAcr9oYMHDvce_?TXP!nFA_MFK55oE<0lnozM z@hsarUEOr*?pt-*DsqI>$0Oxn112~!Q!V_L=(wHMymFp zu91;ZS6)1^H?_rOwfUx@#(}!=JvFCq8p41`jnux|x$Zki{HaF0B-b`kSaQ?-5y`;| zVR~RGN1c=|G)Mj7Y<)YCSRiq&?{yp_=T+V#amPj&mz`Iqa^NySVGWgH0-xaS%6%Gh zEzBi5j*%a)i8x6Ut>~G(=?vf4LOo*|Ul2eu$Q%sx(;w2_b_P{yd{Z$uE*9;;`Ryf{q7tG(pz*C(&-!Xi4i8lJ4n; z%aZuMA~`Smk7)vQ;Le#RZ0v026)m+6;H?SsR$*+yg;Tx3>p((oZ75P1F&5b2Xd# zS#lTKx^2<5XtHL`-CB<{a;ud~XQg!?(8|@@!9_}sbGy-Xof&dO5po3f$3VCRsyilC zpg3unWY zAjeFU3{!+q6KMmJOef(SLa(I?t<1pTz|2@6KS1c;x>r2p$TSj<6E;?6nSk+pZR7gtYHq!R(}wWs#AYlVs~6ehCCMpA zPt(CrNctmFT9*3>x}wSU%?(3L^q@Yjt4qgWo|b|g$t)HGgAZX0CKJZ_=(z@Z7o6{U zbAxJ``Z&9Ua^y8w^42OQP6;WFGrt!kL|a>0TC%hh9hinyM0K8N=%p|{7}@svRcjN3 znl?UhtW}Vc$7PUBR;oivvx80aZ~V^SR-=a~`6p@{wA$*;>x=VZSG1NUbo!U8#`q}IWiobvVQ#ho!8!(p&zI#2{_$2?@ z2p5s~s={x#+oG1Vo+`H(L$W zD6TBx3!W7#=Ga0PA-lc#4+W97WZX}`(ud08g+`#c>W$}h!-X5mNDJl0DU76y zVZ(tVxb_I;_M%;))4~GC^avGo>W7^1;HR^*&=jUod0$B_oKFaz@?oEKaszD0^I~D7I4tW(!gcGq z8Dol+cg=I;t1vo2oazjnH%)C6WUr4-L(oy=PqD zq3`8!!kQi&*#9wkh}BGX?(y>S8GZdt(S=K}O;Bl5^3B<$Qeh=W2@Xe11y@X}LMoXW zF{a!3Cb$`1n-*Gj$#gyjdF>JeXY+fF$bc4@r^DzNg76n;kxAs+gKq+)KR$WGIjzV7sK01}0veh&r!~*-4?%Y}DGf<)c;C~%g$Z3T+xO`lj zpifIsXrf6!Ng&YM{b?y;#tqv!kz12`&NI=cbnsEwxKvV4Qd&Cr=vU-Snf3i(xCmjh z*;5>U4tfli5|@%GpV654?3de=p#wmkV^@ja$+`<;16mx$zKE7SKyvy(r&eI5UGphr+8la}=iC0Oji zepl=JLqQuYWlSnW!ZeF!TK-l46-%D)#BC8gk+Rso2BX8()l~q-%-{O7^_0=1&oh4w z<2k9KYuzSV7{B+QZwn+q{Tl}wxr}+_e!Ttj8GvPg5uod$k9Q`nh)k%5?;0rLV+;RH zCa!L7fncNMo^@m{a4Z5y$A7sGlKUw;FZ?ajS+m-eMr5<>hd>5r`mD_`RpOf$OKNLvrYVB`>$c;HBJbDVL+$Dsm)-FHE}*A zh|MfHfhW#GX|mq8;p}Y69^npS#pw`F*3+*uW-pnU3#)cyF6OsuyQqJ-&DwJ~#} z;=bW(q3aQ#WFe6ce(xi})DO7pbemfEhXSPL7i z|B+QsYAM9O-+?kr-uHzZWm?PGwF>6~!!iCTx&Sh+R!BI(R^@wf$qz;2>V>1=%-;#L zSjFO+WvkbIL?MyA3?r_!fyWK)mXgu0avkc2NCJR=9V-}|y3;)>2Vf`%$qb7$9|Up{ zAmp^T52|p}^)2Fm_V?~oD?9^5cH&NSKo#k z>`5;stV(?)<_WcG%3uX!vm{v>4$13)r`ASmXcX(+nqG@cg1zQ1ncsB}rd#6esRbMA zr-P^v2)*Z1TM-z4O8gg(JOrX4(BtQ@6KFG}p`pphFd`Z7a|HPV#ZhKXSUHIX2WlG{jcl6?ZY{J0}KX& zi5_ZXl-SR>c0HQl+kn+?-uTuJOoV|29aLg{5c>Ak=ly&0sO6)1yC@40WJUAUwbUue~}+IydKzc+PsGXQ6Ij)(bKX>#ADBWgM_mll+f$InzIw^n5nB9sTl$P}T9VAn~()5>zYn1XRS>Z!58_5Q6_K z#_{kYn1;mrrGmK1(}Fw+261^IawrO4wAS9&>Xsyu7f#yf0lQB>YTldljYIF!=w+Iw z8HaCGab??{s*n!i8Thsv945+{vns$|mAb4~=QIir?ijRMPikmuCy4t1+bJzo{`wUS z_DJtvTr==V zRN0TFto_=XOk!qjysRp&?Gx+|OAmDnr~GwP^iQxfk}x~9oXV#nSCh0ffm;~ z80i3516<=y&d$W4S$vk~#<7!*x>#Y(p(7IaFk=6s_zdNOabEoPiQ@`z{%>x-Wny;YTXr%A)$g#A<(nK$2 zGf`0Id)rdU&t*IzL>fV)80IyW>}PU0^dCs2t<3u3^|TV(wgoApWt+&Q8wS1R+?fDQ z>QqL(#iCt-R0Bsx6bV;J?r@MLzm#4$D@ml%>~#2L9*t>R%UmQtO8J@%50rV$+Fg%T z2D-oQ84z<4&qJ66ZVwM`MwFnAp+%vnxPM_T=NUFkZHqbn=_og-u9v9grJl`(ns}Ie zOX#IXA03~~LEV2Sg!f~9mYl>*>u{py4H-#@-)|d%nF3`c9JJb6^C(Pd_6C*jY6R-F zS-jB1$5@g*yP*~P*DO4jPVD@3MFPZ)8eZueM*%@U5+ZCA8ACtW&O1GeTs?a0_~zHe z_%Z!@fpg4Jcff$FGb~n)kd& z(wLlnLE?wG%|`UfVafnWIyz>D{Ku>ZyClJE9AuwK@@w?sQqwYM7g~jBRE%1^TkEUa zQ_IIh8jlYG)mn3zu@V4Q*oS=Y0m%(ejq<5irN!w7s_dvdyjy9y4_ggm3o^On z6lA)Woj_~)hO5Ko#iEP7V9GCW$ME;^YHV-6AJ4c$RD<_AXLNJ%w%PwY<8IvlfhjHA z43@Zc{fMfX*%WoGT>8dV5&Y*y_cHIR@c~cq&2!N5R}en{%EeGTK_S2d{MFWpH`t6` zjT~bIr}IswJ?qa&QGD>7uPiMo*v@V5wtP4Q-oCbSCAa%w`Lz5Ac=<}D3ounGu{1wB zC$Bbj^mp>R?2i2E=y%iHzu`y@)IbkMM=-ce{w1RLATx|S6nX+px^R5=?x6b-u)R9N zfH>d*@D@8e%2eaK?qJE`8TwPl_WNG|BHugzvv+N)NA6?AI_1+zZy=qCx^hSsO;ElB zLNFGzg+PZp0jP8>Eg(Tw6Klcx7aY#PpC9dK@YbI$-lX^+)82LavqhK+P(BCMRZB}C zCGD*L>ztQUg&}Uk>+N|3#jEO>4r3XE^HS%d%PmrW+tIA~nEfrOWk-g zlG$RLI?r7d=#t9=$yTp)imKV!hgMKV29U~MWuu2JYNi*zeuE786{?|>^&8HrXH!e8 zRNHJ|N%13^T6Cs?Ub#*-u&DpXWevDT2%nOyxl70FP4T& z59KWplKT|gDJd4Aj;oS*Djvn>st+|&paQ926qK+FI>=F#4Cf86WB}7EPTRLryS4Bd z3QgC&4W(I47oikBOCKkv&l{pTIDHdvl);1u>^>4qs#Gy0QE3M${La^3y|*u1smd0R zJM-dLQcKq_GSEYBBg$gNrBvHj8ev}-)7syaebe%b}AWM)_ zO9bZg>y=)-MLZxZY;9R5=7Mjevd+6cw#dQ|6Hi}XUoS5`Ei&u|z+j1t#A}{;Vs0iH zQ;Hu9&RU24wBSkyviHt$tDTgL`%$6CQm(MDUYEvpvjz9`io*azahHG!7{UOqt~%*k zrZR0#0A-5pd?{{;C{VncDUi6l0Y*ZQV`^<}rET*+!uCJF1_59pgPWUo_h0Yt+V8!V z+V-zgS)pAodq6iiaPgfs4MHXdY3|GSa<%jW&Iak#{X3tkmXnx=*_A6M67z+;=VIRy9)ppw$}xHN)l zABpRk5kIh6fy@t}`1cq65|%A{ASnu12>`Bbp?;ueS^B%EC?Xpu$#29A%RCF-H@;o7 z_@tr0-dswj-!Fqet->tZZpw`ww%KFH6SI$OYY>Q>j3^ejwNHMRSBB7VPCocCY$ra~ zV0q$egcGR#Qdr+Bx$k(*yAib9Ktq)j*vZ(XxP~pBSVvREyxhEG`VC!G*lOm^reo05 zYB62m=syK6$LIjXUslS)h z?p&IbGMizdWWIJ|jJ-dIHF~?h3(gk{ofT7d!recP#py@4g`#Q|6NguN`9H-ofPUiY zNq%9i@0^gg`lI}>@Q0*pC=&#kUN@cUjy^&&)G+yGYHJVwo66XT?IK2uGOdzN9599& zDV==)()ph&$vd54s1Y-$Z?k6zc%mP@T0&AjQf64`nuX?ke=#-q#U|3y3AWZ@BQytO zZRoZ?osD9qJx|BIy9mrBm7hw>RtDrrM~j~P{S@9x4J`!@JFsqKYIM245H`u@w@vL; zD>Yt^G2LvvZDEWHOEr7^SG~?6wuqF&ALrYX_8!5j+eJ!)sXSlB_$K6{Wh9x15`GSv z)+O~Tu6P>j4C#@#i};@AjJ&vAZ{D9Pb!e9Wqsm7uzlq}tCN-3hJ|$J)cr;`zx!oe!y4PVJv_yg;MdBTc;Qdi-C%!Q^ywJP){xU#U*5$eev9= z?L%vGb9)lAmchujpY_{@O>&sAOEg2PFt6&3HIG8puKA)QuW&>XbGjn zI?5%>bVWI?n;qzmxGfBrccp0Bbi0{lf@JiYS=0bP;{=D&y=SrM68?TJF8_aYfG_|5 zt5MnWs<@hgkPKi@fKz-vsbd7@n*>4$$Z9ynd|C0DcwA7!@HJ@ft`-b03MfH#9iiN{Ys?*rIX;@a0cv)t;@2xN9x! z=E`1yV=L;I~(pY+nPvj2)%xVzJPi<`TRHrV5x@>^^o_Gu+s#5@_s4@!K4fSTtknB zF4^axm7DaOLC9T9#(dgfCTWh4%-u|}0n;2BZ{jLn_6>nZTOH+W%kMabL?(< zf0};Rk3_474ORJ}s=8C!idp8v{#mtXOwy!Xs3w?nQknC(c6!#qAxcX04buz!dql+d z1I8M53Q}+7$y4r>74uIV`^s}wvZ~QK-qh~lmaEQ0XG?Bkw+klyaagNJ7i8cU4Xm%6 z+7^tYccLAIJM#vzy%c+uhE@na?;gc$u$>CGXY#%QASGOty;G#!U26Omf?PyYIDoEnIYiH^!qv#Nl;RI&zjtk;tiL zR}zIDW8D0wQ56@6B&v*;Xmj>Wdy30x$Ju^Q6d4SM8_ayTFG8^m%kt9IdqH2vE$u@@xCpXLvu=>xrGV(#5p!Lf9;#*0iiksZZp$ zw6g|Afyh4|$qG~*a7UK7IeEHc=cv6cJ|c1XH~uS+k>Al_KREN={+q9RaSn_}4NE~9Hfb-}X;#iIM6j5Us`Px)>YwVLL8$AGnw4v)r#yeGZ@Ng`P{z6z zM}sRPS(MLPUecOMRXu$n6@w{b>bu6;MAs2iD_evhWh!H4C&fD)-Ji`k^aGiTfs?U3 z+)g{+c$?mGJA10=y@ZTDiLSU+uwp6{%?f?3TG-zx56$xVIdlG@(nGm6%6Zi;r`G*G zKMki;0}qQx`po-B(!%~Wjcr;j>}BLfE zhYYLH&B}iu=E}w&TpzAsL-Klva0O0o{Kit?t9mOi zp?V1FKP~~dmKJq(36#*sRZf&KRaEIyRnr&1!ps>K)e|1p&6Hv9RsUCPp|;8w8(i^~ zsZ|6gsY2s>5qF*SR)7XBBEVS2X(T@!V#oBIh6JLIP_0;47&-7^8{0@+FK* zXRb;iDD*lW!l+}|@m5CL!d2gk*gloi;=|AjqRMK~-m#lRPL7H8LkPNmZ!q9W$8b^0 zS>z3>XeF-Qx&*TEtwpJa<|VksKyykfqid0&tAUIN-r+!RgtL4Ua`)N%s=^Z2##{JH zcG<5~WB(ikOI@lqKHTgms53vxf46Da`(HSEHfqKCBwSdlHG@e;bjy7cD+`9v_4buC zLuu@or+A+^_CCQ|uAN*)2IL#JOFDEi=_TG=q<8&{O=%9BSTc3lHXOrm%?;Y!>F?K3@# z1;4rvFE&?3ho;Kq#ueLz?TT1%P4xzNA-@|aI?3nl!*{dsc^IA`;TZNrbzMB+s}18% zrNVr2k#rLKXJ_QZVCq7gB8R}rffBYSX*=dN-=w;VSOud}7wD?fO*vBc*>g6Bw%%f`5kVG=d>(uU*>_GI;mnDIDdf)Qu}ptdp9G9+l%?A)oe91G;}cGVR8|N z8kSGpF3Yg1;PhX~ZOIK{l_#EhmofwDuCu^!)yhDxvz8z{u-3R#WgHg3u*?&4TgAox zvE$JYKTiZKJV7ZjAwD5i_w~2{)LwlC%Te<~eo~&P1p1;--ig}XU$wh4|9haOJR|{l z7Fz%c;g@%7TU#60Fb~W9Z?}P}%B1+9IA`31Wt2~C1EFgjtS&HB007{ogvp*0G+O2u zjD7d!|EoAB<7&<-Eb#*l^*+~Ur5gW0S`Oe8fKC%t39grzQU`!rzu+Lx?hC@2merlA zJ9a|AY6u`U^3*E$4&l)~&O6s_$Cp+Dvn6%Sr}<^JQr}9*2$QEuki$q7Gn61+{8hGt z7Lf={BXSFaNe#VzXilo;h+~u>O5$s2c3wI4l21hK(OE%wv5`ggR$3~%sjELs@X)5( zoVJB5)(UgUF;-1@jzWG^DO)eRskIM>_-4p6;5QI`SzDS{-R9t_L52j3G}Izd1vRFk zF_aqiKRq{#UG0##`l=yK5$doFLmReJGw=EP^*L0Y;gQ10bZ{r(<1YRm*sG~ece#k< z+17UuPo>#Wl;2_NqD(4H*2*bsxVi|p*Q2m?IiTh~mHcF-!nevxB3X)5Q70LBdOnx-EY8HHsOvvxd?t z%kWxkvcmJt(!o{H+WJuu?=#Nc^;drSI*c0 zxc#3@h2LFAFsY|~pdyWUWyu7Zo|d-o=g*&oh36oA49;We8@G4o?e|-1{@&i+9C@%4 z0O)3r{y6(=8r>f-aD?r{ob1y#tkPv6!2A15L?r2#W%n}Cej_-$0Y4=VsJRC#WoXR> zAQH}I-U0U^a7=;(_2aH<@GHRq24J~?PlWp+^n5UKe=`D-LNqihZkQ+vZoGZ?eL3T< zYpx~O&|b3Gj63doZ&{>YH~At`F}OFeMyH_H3gff)I9^{yp*TdUua2X zD}Rk$+B80?%I471!gBtLdDpotE3re6edpfX%MWs~rw-d*>663OW*sD^P`5;}r6uCj z^+jxtv7k+?K(pzoeiRf0Z)cj8fq4KnkMe=uv#U8&ejj!dl-U*fVNrBK=;RV&R8|u` zfEEjo`UL3S&=>qB$&OA`$_|*a6T%ahFz%ulCrwC~&uBQqIqs0NV|+=b-s4vKiuI>JT28qpUV(jdqrtdcNGLsXsny!rDUlQWT&;B%CLqgw^ zNE5jo&UUVIZ+etE-k8QLl8y#&4G}%EK)G9LQ@G$E2I>&IYaa zAuzSldyIrHD3rR9$lZ#J!uOFWlOK4_%;O;bftq+xJ8SG+B3+!@>g^tfCphW#*=&Cn z7xRaD!8Z_vE@t)6Q~ckfF5Z%q&^Y8_Vx_8wu9BW`bRDQt;ElnZBb7ysC_jGhE;5%p zW7$gas9&A*a(+t5#Oj%Z#Qv#weSN)xs$;UH0HuWSeL@-{ zBks|>@Zp6#+S=h^rAEGOB|{iB3h1LWq=M!j{&N{Nxe*f)d98H2ge~C-h;NobU}z|7 zP_e)fBKg4xatK0?{@2^>cYsjyx!u{-HMmI!EQwlpmgFyU{!#cnu+cu(hNJx6Kj(7; zHGMz0`sVO>=x=1SwCJ&(h=&HG03^iqcWxz6VfXF3?_ztSp}3g8pY!3w`K`ykLtMkM z`NYJ(6;n#>rEnB@tM}>PImyObbyBZ%?@G5}L6zFK+*`qoVFD_|8=aiw)zxJiQA4jF zq7oAKpr_;%IOa>i8Sr*{NbdA^H4+yMqe(0>fZYe4PXdY^1D1Ky!LqmB6yzq4wPM)Q|?83lG|kr-#`5F!xx zqLwB8_cJv;{e{<&`HKukp-Y#>8?eEplgj5H~jB5JQMC7%8r4{@`%;{h!wgB zu4HU^*s-ept9#!j>Vp)-KkJ5}n{PTr&l@B!WO;{2rKa|xktiK?(D1l1o5wDW?jqMq z1LKja)l#XM$jOD9yWW&3N5^s@JsX%+sZ?C|7OJXU?i_W@P`m#aFF-3(_VgeZ5KuMbvjpu4Xz!CfaV!{o%un5+0Xc@+lT zO&uRjbR+9?MleZ{yaGI>x%WC#r{QuMoG6*;k~XhNWm~0;1U0nXo;-~``hEU#Fp_Je ziqyM>$`}F9fxUM8U4Rf$az7_nOBKOJsQToHL=m!%k9X(H7{i=QC_i~@*i;+t*2!u* z7SFF!p#LXBU%P01W8xJw3bVjr%L}=u(!<=Gz|_ciqs3AT=bYS3))b>57*-qkhOWwn zE**`|9W_Q*YKtI59sj^0!tg_iNcoXv4{9wT<*NQlusk)<8cGL|6JhEYE|*xA71*Fo zWv!zHX9HYm#wP?-7>DfKuDNl$49ouuqpy1JOCLf6&;e3(;v*vZLw#uYEZs9RqsQ@h zC{{JICeAlc;5!M zRqJV;t$Z2hS|m4wIvkxi)Be1~j993HI7r9aBw>1AJntec0A8&&^ur=7bWq!rcQH**JL8)bJ#DW%%K#CKx8l4 zC)!rtaPcpW!j=2fQK71e8GIG_>_@;ElPYx!?t4LW7dPkr;oae`cYq@V;^mG;tsned zcR!-4T8gUL=2$H)IBFk)Y8!k&FTd~hzgT-fHm0XNAA<`=<90WcS)_k$1MjiKh^w?N zG!(dnfZ`&{PvZUqbqV0-L|`jyT!!M|NzeUml5{`pi|A<{U#^lPDb$P=#=Vp+0UbB-O#W`c9O@M<2)Qx=t;so*;v&ir+tNA~1=CeK zd%=LUR86@jBDM+DgXg!u&qXcZwhIbR5^CLv`iFxCUs=B|O{h)ePiFeW`r$LkKZx|^ zM!Tw}szl7V(#|`}Fh!Keh(om__02 z5gP|ZHX$^)_x8(+C=+fB|6PqtwvAOhDWe+vYdtfG4KCr zed$3DaJ|+2X}h;JwP+|z5Qfk{#Lum5M+2+{8P>xcOfZABNMeXrPb2pxum8x68=Hg_ z`hK-qYVUeMPnZd8rzFx9O+cmZ5CuG+kbRIXSdRNMKMxfDWv6hX@3nGQjr-SyWH4reDc>B$Ic$7Xg{1ATO zKlU%vL@nhJwlnPhtiBzrzl+F`7uP#F?N{J&`4GwhDfiqSw478?|>~E94DsyAEdCi7nirUz_D$y;F%75QQGe&Ti?{5 zdpOTO%Vw2=Idc5phe>YMA! z7nNHs*mh#E+L#YEw;tjY86WO0w^PoITKo-2=Q>yg$ZP3T7+v!!r^dEVKkEjvah}ms z@G!&Q{{ZkfrVxyD$EeC-3J@6{Y}hA){+v#8?Xc>uaaFak53ihU(Do#w4dLq}SxOb{ z%hj$z7|Jx0(-H3NyFY_T1}26D>x|O)k-}n&Jb(Mk9b<`C)VK-DrcZ*w?#`02IF!W9 zKCeV5>i6$sYG#^u(mjSeF`>7Atem|3+h-kIYUuUB9- zHNFD0b;zTP`z84cxyo02&fL)YXwyyocO{jCD7?pY!wY4?4Hm~$#zBCZ^k(X@coxbR zbUC(&@T@y!iv-&O%QV+PE0C&T8RAd9vng@0!-@mt6;=8IDdh@40*4LS>#R?fik^08~)ghPFEYwK-4&;9A$S=mYPF9)B) zo$onJ6ollL+&0;OF##bi|lhe{o1 z)DiO>dB9zYAlZ0Y%q2f*FKOUr0(OgR%O^G)n>pyfK=Vbx+Vl!KMmGSs-R`7cw67sh^d4C?g zvFCKsP*Tx?*uPzV9{oSJf zVWiOqaKo(!Sn|mT-LXfjfWb8W$)?8M5pN&BDC7ou4`;B|(rNn8 zYkz&L4$jp{%%krRzuz=I>#W}Jzn}9z^7)eUo*$Z@vb*nh-uycFekPVih0rIS{oPKr zsx5CwMXFe99!qmO2@d1Kk9z2FAc(bvQCwHk8GC(kGqqHBw8Pbkd!(R_Xb>(gdZ1 zBKG8_H!6&IXwRODyhcvA83+nCl_K2?#`1q^5@QZ{PSeKP@fqCHtC8W z3=|rF)1v%j^`rP1m?H3*;-^7LM$nZc6sHGlADQ8(6=mQT6R17$Dd?U>+o6eL^K&j-zC)PFiHm8!694%n`& z3rv&%W#!@FfzX)fE7~S0XWoF}_8JRK8K^JR_2o+^=SwY_$Rw#2DDuFcfa&4B`wXz5 zu!bomgR0Hj{`(7B3-sc(Ga&T9_05X9L#{amkSO(XcVG9S!KDPwIFM@Y zG!BR+ct6~Ye4oCNB*D-V>%4Jw)xA12cAh1H_MO4#euZCeR>5Nws&H$QhzsD@40`o$(7e53e5l) ze}slyZ}WEXrFX)~b=&@i#kYKGM2~ekKzByv#{&!b-$dco^D=sJHc2FgXPEZ( zR^mSAY26oo(sZ5{0hRkTkNd%>#}b#{(v-FSH2g?8QET(~U7a)ChQGN|S4W94WEHr@ zi^NrEmc-?GA%l#0x5`w|-%{qfnK~}>SUTKIMt=~(im~9W@XF_1I^D3SSz0CHsnd0A z>@V3T3FBB>YS`oKB*hhZA@(R3=c6>#?aOTjqi9zE)ZG<=4k|r1@IBDd3E$4Os+5g+ zR!5M}8AnaQ2ePqBKk;Dsc?vz>xgNdQi{aJwZbK=e`d%cq`O!I*Yl`z#r!%9-e^Fit zlj-jfxZT7fQ&hu^*fOA(ovB-D@mOYD04vH_PJb6=?Kq8Ij_L^oPtl+V)d|DiCuvT5 z0*s${3@jn1Eph5O=FFVMEv?*Q((}J%Y};{N`RugMA?}K}>;K4M#&?`VX}IX$Jw{ zCML^q)TmG0bRZb`*d7O5n7g@#jiBQ)d7_ zj8MI@=U9Blhw%$>f;CyzLMBC={U{oPVjQ@Kfqi5G^#M+V@&WWs2$G))0r6Ql%yFl*}%Sv@aYhfBqjjR z?@3t#Gm00YqI%_1A)go+34*2VvE582zvHKYMS0O|p9tSKPL;5xO0wRx>QkwxDx~;huY54Z-4XDJH{+0|> zTqXJlhS6gU`znLvL@gqj)#c6Z*mfgQK(cHycya=uVyLV0@Mg~AOB*jYA6F-*C6uG8 z4z8>++ts(Q{ys|BtMyc2m7F3<_!XVK>ttH8!k^?F0B4 z^{W*~(QPk`+r&Vz?az<0OCL)cy9v_7W*CWskEL0>`NMS?@_HG6&kI_`ci3g}#dkQy zk{Hy+17fZA-=tEHMp0ERQ{8@vnc`j?liKepPhdD9{jjWAo4gN;=WNv=0(doThop*$ zzq`6TA7dDCr^->@^lD|ZOn2%trO(b4fa?!`ItZtuSt~?tq^`!Qj#TJHsi@1-J&CL; zdwtv~vJFAW8J1d&!qBpgpZv(eeMXVT@&wOX;Jo$MQD0YE#@%aH`mfgwOHmdGECutm z?EiYB_#Wjwrg*oTG_UZq56VhvK@g7akkBu?T5||pZd*5>dL!r-L$Yza*m7YuIiZ(c zprtP0W{+T!b9FuGG_waTgJTxVZ!3o7$?7`#XgeoYg?_44BCw zk{1`imfLZTU7fYzC1O=nD@Xh{26qK+?S}7_W+(ZTV0kk74J|$eB*_?Ke|831%LiY+ z>oABhex>vHqAgZ@yVvv!lVbXVO)_i$nvEjVkFFu!=}L#4%qcJqW#Kd!?H%ZW5pkNp z(OjbH@Ce$cWi>lY{AY^lMTeM86F{+|Y+T4!B>8Bv*px`|3%NZ=>3Mw?H`;OD zMF-2~RR!$0H~|ZF?MwIiZ>l3zD}L1msJf+Ot3o^)>{NK7G>&gDJ3tSyE2?qxg5Sxg zyun{@P};fL9~;lq*woYoL$e@(ngRb=_ws9qv%7mJE8Tc@;(0G!s`nJC*UMJ zN&(g#qkF+2eslXA>G_*Ip_5?4iR~~@EH`>{=jZpVJ!O%~%sY5=bjF&Wrc8J)y9~dw z$Be}`h_~e56&OuQ_5FALoYi(}9UH>7>_U;JT>NJAX!SW(#uiD6ymM`5wuy#*8mbP5-~h791{bIyPYQ|etl@ucVu#=ewy;a zNo!X@q%Q4KuRia{u}I9S=ZJwE9U)xeGhvN$>5V-!)RQqYH=SF_)uQ2s9Qp; zR->MV5UXzWk4UsMHTqrdPRV|W?1*j6M7mVMF%|6wX!NHt^U&Xw<|Xq7a&~N_Xyq`W zcuFmWp^pAkmzFM%i)N4`S%v6gYZO;;3>D$t1d=ip7p0`A7$(d$6`JRyS+q@gjjnu* z#ak9HD@DrLeD8p-%=&TX1@Wt%<7@6b24L=w&){EH|#NWE^}J~5Tj6$ft@0gn!|xL~@ermC9FCY`1r-da3~ zo0SeY10pf%^DIz&XLn&p5ZBOa+j>{wG}Qu?@yt!!ewu!Y%=R2R3Nk`DJOwommFP z|C}hrZ@QnuNikOgEOWu5^5v)BO>W>dkh-x&pe-LQ-rt=kr7)hEmX;QQ`WrR~+8fcY z8h$s2{Z?i{)d~$qo-Ar=YG2|4iHS1U{W^H@3&yR#<=tOO+;2$S9mW<<{#$7FP)irT z1hbp%_S?Dk+dt=Rta%URReUG?Qhy^*N8w?JmCJonYmlJ@vkWjH0gfUno&(^$j2V0a ze)O<$dY4~Mc-j>=8-nrz1XoryeLUZj5_+R|DyEZZ%_vahpg&Xk2LJ7lVGf>IkA+nS zI*y%(@iZg2_m)OqPNS{`G_B||l@)Y1(X z7oCS6ODK=mSW~iyoPxPg`Pceaddm6Zpp^AbrqKY_;m>7Hfq$m}JqVJQ8*W8P;Df5f z+!WuQyU2`EAEvbB3V2G2lKXV*+w^f|*$IV3&|~Sdu-~G1F(UGTXnFX*p`m7Z-~ z2)JOD1KfeGnp3AvB0tS{;ZIFw2@jWY70xZimOg4kL?ibqs*x{uO|Z!{nR9uGl&E!? z-ehWV2?1`?1hG@VdbT!IaW3kA{hiAvOP*rIWmJ0F5kV$M1=)E&OR_NXvuR3bKkSd9 zxJNYFuDl8FR%A!{zos#g!i04DDEk_1B%5tS{`}wOY_@gB_7m}P4VScCM71AGJyr)k z$RID0Mr8U)5a8)F|B2e-xGp{Z2St~LxFIOba4=t`z05Mfo~OXw*`BMH4(|ZrFNJJR z0UB!6f(Us)ZM9~lWDa%~-|vTIzh8#gf2sZ}4NTvMJD2q9i!YdR&C z4)hL@O4GL1a$4ln8mEm_nu%gD`%bpT6=haaPAZAMtga+HM3x_+5XIML{N-bb%pZ<4 zw@J=W*3V434h@-13 z%E#_`6X$K`zzO|~9wR?*d}|^ns@~z*+%CB-`%hi4_S6Y6tJNRo9k1emmF@sN4jcX1 zy#r}BC;QwntE;hyDaxA*UvLN1Pm zcmUFmSC=sdC#5j;SlIDTX4Rh2O|Q|zK4qYIn6jD>wcMu9R^ph=*7fvL#;w}@iCSeV zasMHpzzSuI+zp=6&6=>lJe)Qr1dM>WUTlO7o1VwjieiaIf&q1g$o z?m<{6Qlrqz8U*$S5?Q|~J?%9Aho^>dXZcWrH~+YJ*YfK-0m39!_oA|?NUyqHh{)!t zrJ*QC3 zMfqsdQn%m(>IR|hcc~(oePBd#VJt=_{d40OoV1s}r=H}0FT%-_-ifvkBw8(*v~i?f zgNfiFGOuAKq=K0h&9?9qYcBu8sHNTj=S;34InH$yh~O~CYLZ*JvB?tEBeKm@%zg-2 zD$!~Z;*O(vTQcmtoQB_9^zfn>r=z;6RCR zew$PdvRZ=bgregnrJ)a%dOXNt`$X-kN+cRH{rOu1wu(^VS9&J7nzI_&IHPj<7C1Sn zt{fQ9e^Khj(NB$^a4fyfoK2DwYGtb%M$MCkEI#g=X(}sj(m>z@%yHeeP(FY0;ttT>oO?*onNdJt znpb*QAWb`tHFnynh#c%@X9xe1#xL@kmYO=ZLRw zunKs<{6A6ZZl*m?Q%$gWEs(?)`u#DQo)poE_j3%V6_GuTPvhgqlQzyi&D)PiHjG>U z0CyzkR>vnGjoklp^gn$xlHu>^NZh60W=@^2 z9}gKc{n^!SEATZUTA|eI{S^CGTpFUG5(Ton?*%A*kADe+l=6dOs8g(^mk5fl#c9lL zt{!?J!G-NFGd;$UCeTZd=^%Zpe5OW4TK=ugDU9!&MPL9mbm%#MDpC1tOn{(@{v#2P zaZfghRCKy{Wx}}r&iaF2tsrW^{nc}ut}b}KDBl|3^?1eUp|b1?yYAEYnFwIawJF`- z+hd^Tb<9I(TPU(W7ZtTeytPLb|GK)ryT9?jX}^=WcQn+B&3JCG;Beh7Rj1U-OBDDB zA%Yf8DX8?(#M8@*;+%wQ2w=WwVZ-2Gmi@^V@mV+v@sEf}SyPN{KvXzmWacS9O{_0> z*&l-!z9*j5x`Ha*;%&b)G*J&fLd2~HZoS={oxgm=6dV>QPS}YJ?ao2BR#8vz((B8S zg7wpESQsL_4%dHC6+|QWTbkv5T>*mcUp>){Dz0wImF2mYI7e2$IfpbR1?dZ61LDIi zbU%vXG?GtD1>OuqNh)+i@xv4L2xiz zjMKFcGY97vSu7kJnA?dOH5W~nfqP*wfAL^;8F0nOeXX)a*p<}5fE_^HCJM74%?$$| zwZCwcknN3@tcV#6V~ABjX%!OwPZFc8pY30c<~GDF@W7LF&xd%4V{W>Mk2C;PrKL$b z&IUJBgl{7=uXR4=e~e=m8UfLCAH1FNKyWD z7{bEK)r|mE3JbsynuNecMn7v<{;u}h$i`?+zl9!)V$h86wpVz}q#IZ^AgQayF6;2N zpfV=*u^Xuq_WKg(&l2L^J{tC_YT{hgank$LItJ6E7pwTZd#p@6a_S*Jl&4>%y2cM+ zJoQ#4sPG6ADf&{MXh0cj0lMP!(Im`PZqTwKGo z327g*6zO_pHhZX(>c-F!kiJ@XNdHWvL5fNE$V`WBsA|n+04FsgL&Ez6a_tAE>8%Y7 zo}k>&(f?ZCoIqR4&CTs2ogb`!#yt$h_UkiEw1(MD<^_T1IxNSuL(dQX9{+`!NKhF6 z7VabU`ban{c?ojBU@yg#Oa|$!p1o58b~AkYwbIiPgHml;-y7c^*N2`0fbLxPO5fG! zgA(?b49857#8jBg%EN1Aqm&|dy)|G?(&I%9hQp=EK?Ng>yNa)0dxhrKB5icYw(Y(q ze&nabsGeodQOW64=wfpsKUq#dhqlHI7&tDCsj3Aa+MnTf zwFO!&7kX7-Gu>B=Dw+iZwx^HJ8(Rc(2B3J&a87m$b}HLaEY^UtcpBYiy;6!SHQMKG zBL1Xdqibtxw;$5~lg-JkB`-n3g_VxjDwb*lhf&-XqxU`7Tik}T%Sr_MC=WY+mfNb+ zb&LgmgW#B2r~Hqk^Khj4|G)T^b&a^9++>DaB0Ja29@hw^xJ736%AQ$eU4%=S*;&`V z_NGe-A$wEVD?6j#>+}8n3)kEG{d_*pIggY4x&K@almIU4tU0ETWA{xgk>aL!{D7S~ zKSwvjm1GH7yTI=TeDGgWzs^(;ctKfbM1?HWBF0P}x>8tMyOY&wILl=5r)t8=)+AZK z1a)7M)8Q@LoX#VWQLqfUK&mDQ> z-*QmAj-C+KCw`WVimBIw6EMOXh`^L5_xqx@3KH~b!*fqB%)dXN^nVa}_HX*%kp$9A z`6iKiyEwhGCC1LuHNeftprQH>#`e*TmcT+RwW9k+8`wzq2G8JrVj{gmq4|kw7|E^c z3NWCK5BMD0+=wQf!de@>MN-@YtT%Ost$RXTUP})Zf;FDs>m~*qWbl774I=Fz;9oPo zu^5V!lRFA{&?oZP_7gUThzgbbN>SSKhTkS2X`pwaa}a;3NrofKITxVnxOWNj+5)C-Rc`2=sR!z^ow2f= zJvri{dMJSr-0fKM9Er5YXiEuCx+bjc5Sa_g|zmT8j9=M_xBiaBNw~F75E67EMH#>6|u7aX& zt*9}_(Tm8y-t73={tR{(y1csyF{|&{3b%sY0O0zTFMYQKqjz5KE@sUq8W^kpa{Kry zheh2nMm&oMp|G=J-UL`+G<_}@I(2yfq~I7Kk>ojwXkf7q9@v1GWNLXeJsLd8!@`T&m9Y+8bEK94v|rF z5<+UIA+UMtFcB0OdN`h}nWs?>oGB)6H@I zxJkQuM^ZZ6P{Ak}+^h`YOEq*a3qF_v7b>UL`N7(wnkvOY1|Mldkk0AqH;-(P;UF2r zjQ!o*ECtXdxlRcmkm2Ww#Cp^zXh^$8DKPh$eRI_Ns){m77SET7x@|9>dz-qe!_?jX zhhU#CeuhiqPs>gKo5O@A8HNnIi`)hr2Aj(I`FwkJ^q5h)cgf*%`y?xA#SD*BST>&K zieNtRDyr;OzvRf(;w1kC_96ldn-CN+$su>Xk0y z0R(adk?lPx`jCB5KP6!Bt+1yM6QC3Izx>1$IY$w(jJt1dzZfX|`tNy4G1CjELSgnd zeJV(7kn0#9#TkVtoSJOokO=}VCLjS{X~it#wWu_O*kGxFjAvnTzb)V&S`;pr3%Zg9m`^JVb**mxhX4U z*pbP-8BX*>sF&B^Ye$)>CkG9T)F45WCKBSDV(UDd z;#=jv8;U5b!}Nq85Qd02K6wn0yg&@MY5ShXBS;XL<4u>ik#;wd54UePvscxL_UI}@ zJB}Lezcsuvu;^uWLvfLgyjHJM;lVaQWy$*eIRHK$OCd9iPdYt$l@n-IHm{tpYfzh=~I2STW`^=`)M`=`>-lxrALq zvFLu4%-4g31WiZowl0^q86(X4XWA(bG<;iS2E(?&39delu z2K(@fz*FwpPu~Zj)5V4rO7?3ZdgaS-)BHFi_)$Tl=Z)JH#u>4{ZV~#6iwj1I+_ulp znRz(CO3?c6V#glGdmW+icIC!Udx0&;xtLde(7FTMgT21p&q#ie2q|?-veaC*d%gLB ztTk|DDnW?w4-!GLsYeDD7O#0>)fhDS`kI8OZ$?Mm?kWw_mrBtRoS${9Yx>T1?ANtb zp!e0%VCqL20=2!v2m)tFS!pRLxz@H8jQJgZSD#aH3ofXP%U8)|)1B1XNq#!*R|CI% zV#fh5u$N73%Vz#qc!w|y-!KnL+8y3{t6oXn*>xxX+x`r?eIEkOtZ121wR(63c?7SoJlanTcMr^tiC&-hTYc_T+5>M zj~$!(?E6NR3IEjoRz+ z>Xk@_4k%xUISOalts;;3o|oPK*Vb3;Vc&Dpp?vDO zF<{wwv@VRIFwDj4ci+_E{q;>!EoNj)${tZa(_7Xsw53j|tn0eBx@U<@u=<5vdlMgT z*J8rR4l0yCMc}oAzOAnr8p0l%V=|-0pP=vi`t=}OX`iHY z3_phG$9z(J0Q=nCV>IrOxpJZ#Zx*u=X?_Pfzg#po=t2=JYsk+IIE|yD*9k1t7mfKT z7Ga(4*C9ZP6&7WoCM%5k&urN20LnxJoT&%7ri2##?>L@HBIg-;gkNHzo$g7t<6kyS zN%FpOXme8sK%x=nPVW!>jM^liU88%8wh3GezM+?fJvE5+oZA0nDKo`F?do5?En04m zo}|q>jcoMDQ*kF@N%L#s@osNxe(3u@H5BE5zoFoTYf8Rf4-hnjdOLdU> z?@#1_k}s<-4dH5xYAsUitpAQqPU<)AE#y%kr#B9N@1KHepX&lu8>=9?6b{-6>!tZ$RYWKKKUwR#_ zx{EOn?CcE5F>*ZWd}kS9$q2b3A88e)H-v93#mB<5gx&3Tzvm-QC~NfAn-;dy6KT)! zdlyQYJW2YX;SklS-ffoovV5+sZ^JI})5V-nUfXsR zSFHd&P`iYc%92M0Rga;sj!9PS9KX^GcKr$`4%H`-=e@4%jeC8o=7+o_IGM< zZU_VP6IN0YgEv*0mri2|IQV&5-NV>|bdCAL%pH5}D*}c>kmN9`(z{$Nn!1R3vQ|sD zb84y#9YuPp8nHtbs04Rqma=TIj*nN*(p|qcydjU(Nf*z*Bi~Nsnrs*{ohp^pdE5mQ z_G?z@!+J2O{AC0>dBe}1yd|(zbG*dhwUEmcR^O!6k7whh2-C7p#T48EU_^u(a}JprYZ zE3~7%w(+KMMf+4b9^E=I+rJN5E7ONyz?3$=b1Zp0wt;i|ufKYHye@PO9Q$_Q8ecm_ zu8rYQRN}JmM_)nvpa-D)oVc9-oYc^}*LL~$#ht|X&?>$($Hqu=+KznWQAO-p`y>Cq za14wcP+0UBvK1&>9Wf^-mqQVpgKnEaCjccg6_Wo=#%el$3^X^;{hL|<{8y$9Fp@kxE4|{6st%fF=(+69_s#6KDt;@t z%RnfX5bC{3u%#O%*8%b5PXiP6T-^PpL$dZFRi>4SM#7_Y^eE$YBoO%6rvb@?4N!qh z%8w70C`?qme|ddm`GT)kawvHcCS)p>w#4i#jn`dnMb$}@{7T^-Lof4Zs*$c5*^y)i z@5+D&MJSnK9&SNl2)M$$x3apah*f!yh%v&U9gy4=fJN$2Yu7HpDsFune9Ks$8S?}y zdQW-oahm&aOZ2B{nWk1CU^i*(r@vC=H5k((Sj1J`?oU46pY>h#$z0`@vq0IaMXKl^ zt=U`mpRLOe(G>f7PkTta%v&GH?6-$dMyqHo!a@o>vE^B4{-#TrA%d8FOv5_EO&&tzr;}oekyK0}&bnZi`k*x!8`~3_UAlf~$3ozo zKPFx@<15bdBngU-9&zI_G_zDGH6DCqAki(X8lb>041sxP3d)#ggfNvV(F#y@ORA-F zdK>Azlp*_f^a}veY7oxO+iG8SEj^(z1H%m1$j&@Qp0xfulp!0_)3d%K8^>T-O_L={ zT{*QB;T?7mkp>z54|^P3F~02Jbp)(x|Mp*BKKx<$FoDEcp_;#o--1m9b34qGli?Y& z_wO#DO}L(I>XHj(0DJGaF&cW7XB+h*ZWgVDrr?;c>)8~i?gF?jOMLDkou_hT?Jfdf zO^s`ci$M*hy#0Tj=Ko2mEhe*BB?Nt}oRK{kFW(^=rnl}SQVhs0u*6-XQGm_N&Vu7Vsh}Lc zy8)mWn_UB1v!XiZ@e~End5@0%p6sqJotq<>Rn1mwj4Z8E7&Q`p;GdU0nE_T0#J;&q zhB~j3^I(6w8ugfYkQ@@(J5oBn0aR<#J`S4P|I*rUZGvG;$`cHeICMm8OTW+Qu0Bcy z{saBSte3P$5?r$kergGA!wKa1SSOXeS|be>Z%lYAh!&IQloS-LvqPC-R5VjGT*tuH zb5j_440QJnQviXD%4yegQ6AnkBx<)2YpkzFnu$Vs`*Mnz`U*epM)-ZDZ9!6B1c}~w zn_UaKz6KgZe*Zms2?BO?&s10U7U`I7Nm81@?UpcZZ|VA=>qGXIi56l1?q&Lux4G|S z)Ck(V`6i_w<@9E>@!~DB74zlEV%%jOg3?Gr3p!7=JO8zK#uAY5_2&%8Fce=K1O@Vc zxK@w~Gca?%MN^sLxUqhi{fi57jIKh;%j;w?RzjqHaXiZ2kqAMw&P=-NCu_%IsD`n) zA$b2Q@(?qG9!E=}3K!&yT)QDIghF;zD67hC3{#O6ju2WIp@u4MOitJBKr&{!P8n7k zIRYgxF>!y<{m=Ym&%C8UN1-VCI6Se=d4Aw!HR|g~V$XjBjlUOi$JbOMuBb(DU}1bD z_j1G&b{dEnpv9W6*qNb1btx~NCJycX{DAyTK?cz_!1(`DMLkP4Xx+G)NDwCk|2~p? z!L5&Fkf!wL~-U5&W5X%S9DA$4V@^V)pW0FdJzNm!%=za&C;K_E!|2>%u z$5)MXocv1b19=3V(7lUSqA6Nc(>`qyx7!0A(z6@z1_&14{l%~C^?Bse!;E~7eAt(U z=4OT;3LE30q(H4`F>waT(dg;Se`zZ=DX)BeTPGcKqtD}BpK~t#CLl>-|6XLfm6dUG z@I~e16v77+xmaU!FW;3t@v?1Q{XB9gwZ-#^dh-+8g+a<`zIflSLrW2#v!ku*I5}pT z#2gz?ek?xkw0$-2{nHmO=Kpo<+W2x6RCj;|dls@y-3bnk`K6YL;4_eLJOCOPD6Ihq zg8yC$Vedfp0wCA*rGu&`;~$=q$9>GRk7S(t%Q7QPD%z*_c0RkbCz(BfWtJ9wHR=G@; zP0yyB-X%ma%Bp74DQfZ{C_Ixg2N~1XD5KFQeWTJh5&JgtV#5@ppkrdW{z+?V(Y^+>%ciB;eRPFxIvnRf_0t-DZ&c)E6P z^g>}XJr@h@D9NdrQqA0Xq{>{r(I{N?ASQpv>Jg*rw){@72cQx|Hkz*Gb17Yg#RgqAQF@2&gN z0}WWpJ(F)j$Q4z-qN7%a5LZ-@NDR(#QFuxHQE_n*Z~m7oHf0VD5(HKo*wdGYp6rC* z^b=!;bW08J=7vBCm4(yCJvBpUH};7b4lHs5BTPo^GG%Nvh90M+=ytWq*!1Pp9_cOM zz+vOs&^%y2~nfqz%OWa1)OQLh+pQZvr!S2zKr&LBx{|7q{k)~%1 z^KW^MjIJAg6US8QNN0VeR2ysBva6QMWTsZjz1{D^a2vzv9zdO2QEWf*!^nueI14-M z5n5YqXvV_I8hAMRR7LOG%WB||sp1FUi^d@J%ZGPj)Qvyk_8*Vuuhw&W)5fTBe*0Bj z)iD?EMsq4)*u$|dkl#H+KhjQ<_*siB+lT83NV=2vJPe*uTbc1}Qdw*)%MbnHS3a(E zqYYa%zQK_h+Rv8HFkmg`ceDxM0};HEE3iU9d4>qVsbMQVN9X=>ZRKg_2M0D{OpbL% zMq|N@uP>YCxfTI38besCRpkf+*vGWb7nU;GBh84Ys&B#vF0= z0e>dMB^yf{?s`k-(_vS^&0!SX(;DYUy{YGQ8FGSYyu$Wg4|S73cc%hzjW=dn%G5B> zgGHrO2f6pPN(PCn7#pCW0iD(W*v1@i{(Paa?1Nio_e)D5ESrwK%6SSE7SBEX0GU-Ig=MmPA#B^TFMb(x!zo$OGW}gmk8dKRVnKe_xBr$+CKjsg> ziM#6XCRt71*W|!tR8M_q<|kQhHI0pL2_oiZwR%%}GrL9T4`~BZ`e8}u=Kb2r2wl#_ z%|Cyxt*kXoZohtdTXKat7vc9ZQib9n6%8}&O6<@t&x^|=kaoK2*VW8t-y|Vx#Q&Qx z6}orwQoL^LvBC8_!YeZt&({eb0xDFO>C$BgFQ+= z(}1r%>=(i*L2ch~P-sJnb1&tecaAw&wD^gr#%o+CD!-G2v@|y`HL~gA-XE>|w($E9gjv_obl%ZA< z7wgA2uAcEQn5%|rJ&8~-)9U#-1YbV8bU`Y=TR%TP{~XO&*vRTl1koiREQ~UDh95(P znQ1l}3m~2QK`=r?DW3)IF{)<%Mm9y>VB?c*VqrFaER@EZf5NE_#Ee0P5jp#JbFz9| zu(hSFO+r;h8GAl77F~i9dg!z3ll}=SQy4_G6FSx()5dDRtgZqmQSZ#{o*g zzMacC+1#YJlkUg|>8eH64#2D<@B_Rff4>aL?y=Dq$eNENXjQzZdVbG(j>tt7g3MKf zHU~>uXut*fu4i6Za8Jw!6U~d}fGrl9bT`#t`_)W_TW<{vxSKy=i&N3OR;gs5_Tu=> zI3n`coN<27gqfNQgwVKEyqZ`{w$j};CzcaZpZvTI`A)KBMzVTMcZ^TZ;_R!s{WsRP z;v`TaMF^B=06}CfX2EayZFwd;pEma-cW+O!s6^EY{pj1|vRoi7O$-@?SSR-<71CUQ z2aBKaqlz_DgD?}VWLaEpQ^X7WteJIMW_L%un4iP=5&t`P-6T(z*GpD_xv;{mI}DfQ zy)pEi{JI-51kx3?#L#i)xzd<{7Kt)m z0|Gaqg{bu$t|f-BLwDi1mYfNOx}DV1*&pDZvnmHRDikC$X1!D|aaV|*P{Su(i!c9& z30>g$2dt&XTXpkGJSu<5edpP4`XiY^npno~sAm4wy{BF(ddk2f{-Q`A9}HZyvV)8= z3t|8|0#d?e@RuH$1#2#1z54($t9tqCP!W1RMJoynhI6-yZbsA4T4M-ky@%H)WZJDZ zAG=rNV@ZC_Z+^4xCMH7xdCQ78W+-DQcRgPp^^sZ6OcQN-$u3%>A{YX~Q=~bJipncNHB2bRiH_;Yq zh7&%S?~|@Oa_{fTwL7rb`%=>wUw`xYh8dQK997Q)2};|6dQ36pme!u3;zfrn9e2W9 z6nS~luaHuD7~RcJ$KOb?pg}IP9!jY>Tpn!3>lY^j^Sf2JpXt0mW))9$j&07$I4i+diLz>nHu{e z{B?d*LMfjB(n5Rm2iSr@Ha3pb(7`Uh;!6Q~9{E!<3$_np=eGIM!gC%i1vGxy0x1_} zjvV>TO-*T9ci@x945~P3I$B=t8=Y{QiVI+P8HTx*EJ{11lCc!-P2c{|4)40@JM$1J zON&!s8<1GM2+MfrkikKGE&c|7Qe!ucU(+5`EdyU4fz0B)parZu5+fAxv_w4r9ajLy zd2unWRuB95*Pq?piT#j*;WUo|ym9&aku|YR9`L4T3pf~>o$Y@q{bK5nP3~U|TZPH% z&i%K#{SBXziZ-U`L$1A+VdXq5+|XUbhY36qtNRPE{jM{>#r~ZNcGBqNCdHY zkm!<|@K%hk!cU_sF^3e=tPhn;dlEwzod&KrXM8Gg=qWUndpcu5L;dSph}%HWqXCg6 zcqP+ws9*E1rc4~n8&<(agIm@&9(SH^Wik5qVPeun>l~Ik^*H8w>(ILTdS>n4px+^ciU|X5^t6X?pB?{% zv=ae^)4YR@9eexMK~wLW9;Ii6+6_}qbH*=t^L<^f`9)qS!Ei7XdMrTQNQiE%2cbVD zv4#ngcM8D1-d*`F2mjRN*TVX;m3D*aSYBsr+0Q_Kd_BX~I+A$VAwTbB-j6^1hi53w zk5r0TmA;k6#2izp{VyLGT&H6;3X)?qFaTcxhZa1+Fh9E3-^(pB{2w42uCz(lFvgL~ z$&BZGgm{EQ)5ztXVfDKmVa;8sjLV$nq#rzP4=6!9cu}PzbiPVY*|n}NAqT23Fkrq+ zy<7;+!zNqthRMegT`I|JMJ##?Xi$y1;>T>Vz1-cyCZt%i{~vk=p0O35KyPnvKfgY> z65Z6SQ}_-^k5gP+9Bu?sQ^vJ&`(XK3v-ker^^u3KcYiJwJyF!0+&K<-?Ytu-Eqi{x z-(>Q@A$52^Shq#O|eB`%yXIava_8 z{2rUEZ&(>8bjo}!6c+&&gdzzOiil>5)p2z06(SP?insG7>w^*BKUY;byIqdGZSJlJ zrS)TH9xc#W(uSD2yLQ0Sbb3%l?fZ0d#FKg=;{70G#M-9XL!=Z&Z=_JOC>{?bos55G5WXm zxrVigN$bU*{=TmW0`jx`7fEsNR~a7Nx;+%!B=r_7-Qz@W!(dv16$7iUjzGsu)EoRL zS^6!E-;Zj2->_geRr@%qm3=LV#IqV;=IOtP%N-~q7}xb^J|l!F2Kk|KL& zx}#mP?w%WeJX?rjv4?s4(ucYPWfbp=&C{J)annw=A2m9iN5Lzp!UQ^Vq8j{HxD&>0 z0I5llpYA%)uwyp$cpUG*g!5=mQAwk_2e-h$ln4K6ly9iIXYinSt8h?`a~EF-#Aw)S z;Fmxp0|c7e%j|{KCBLyj;m9)E_p}z*9B?(nqh(YI!&x*cA=nCXKx{Iv2h@TS7t;55 zkAr-%!i|IMHTq~iO^)?94AJ5Q5dl|HVcxEI-zh#v*_qOlPIBiXuPpo+6J{Tat=dD@ zg_+#j{!_pdqZ%SI{o{DQfsqV-f7<<1%$^Jw%5ScqHxBJQUiK1U7`=Wc^iFi~(;BDk-B!>d8+1!*r3nY7z7f%HLtbPK0{5vj zQ;(U}zJr7;{PB35m4VA(T+X7quj$oW2RtovyQaQBKNnQzuCGSsjcTaQ(7zF_w48MG zs-Dz3$z(hkXyDTL+E}%qE5I3{$Q;r0kH4O8(6)p!2Y(|t?yB{sE$3W_zj$dF1bVLt zX8!$QBS&hiFvyK3gS+*Y(={Ul13ke?l15>%2hh|4BixffI5?z7x2vfP7TD@?u=k68 zHDXW+gxlbvi>bSIzf^zHzw71&T9%3n2u3_&Hb=+>(zqDNiL)TZRB6&Cu!^-=NL zn@_i@9F2wMMF}=VIqar4Ann&&{36{cNsunbS<%r(5drcHgrpu-EdJ<~>v)jIfUBDK zYh_%A5=j_LN4&n7|GrPEc+#B#`k7@Sj&3Rd48jOc-ovG-3MGF^5hmD$wvfxXvN9Id z%p9igy)!omzUJ54YJgLT?D2U&v8cbS{rzeIa^>jYx_ykWm2p?-+hSKg#jq70GUg7$ zxVKipJHdlYe8{UIrr$TtS|0{O2|}S}VYfMY&}qLV%@c;V9VdNyWw$8?d{PTs;lhgf zqXf8hOs?NS*CI8^`jx*SHKq}sW2+ZfQ^BZeG43gya8BGof4_m=)1FT}de7ay=9v-w zqN7iyP)w&a&9LZJ=e^2F6cg$8w5JM{Vo|+c6$EsPRMQZY!m~?gOjSo_0xzvNVLcF{m zSdc{p@g}^E5nio{w2=!`fSnXR4C{gW$r&Y^bxezCOJv1$B^KJ^L3tOTl741)o4fn>2(%HO7d(K{{P1qETSq&MHLbZsDMq&L+?;k{HT8^aY6$IZa)6Rv z5ym;Ck`=u({`aV6j{PJ$oN%YUk}kjFup6JG4=9`Xp?Lk142+?)?%%ZPx8CX2) zxj_@~phnGBW&8Iej6Z;aa#LZLLQpK4s9ENe+y#T!hFsfPq&QINHK2(n7!1-SqfY<>9U;_^i#qTiox+?$q26Xks!!>s_ z!)IWBh6FX5F!$K}tU=>;E#~ItESqYQ6ifZKLN&T80*dzjZM-Ob`V5s*RfrQ1HgHI_duBP9 zX-x8@+5a%&E{>uKVHP?y&$iv&;fZ?yS>)vG z%tQW&dQve0yBu}d9r`D{Ws&qpisgb#Q$67t1Y&Ow1&rJ*}7-JGntB^T~D z?L8V36(ITBL%&hGJZSUYTLOd^z(Y->*!;|!pCH6nYb&OHFLp4thy20cpVcdInMWZA z2BPZ6kgKxuKhQ}cI6UZ*{k<`Y(iGo3Hez5f_(F1wRJ(^CLO+A_1i7M)9R0Pd8b@M! z*gPx{`58HQ?4R8OOW79;4qg#3nDcN4D}HaC^^&-)o-fndNx#2tb<3~o6YuL)&^Z}gSg{wwvx zRd^L{+;;BHnHjSMN&!kC^TOWI+pB@{>zl>5Zn4Dw9T5Yz+#EP}v>RK#AFEIiIgb1Y z0X*Upwzf+TTpm%l)_ur=5s2#$)15$ve)~AJ6Es3PX=+0!zqIG1C3s}f7FtwSsz`fHuv$B5YHVM?{Lb?NX)?Fse0#yV%9SWEGZ7n!m(pEnFh_O?w zBwFpK&2+=r`pdSqhW6jXUXC{Da^-x_QG@&NY}Mjo1YJw#A6((xn5Shsk`^Q8;c@iG zL~^w*B0dwGTRB2^DuDcba^fw%UCAW9gKo z#ZI$4L0-Ls@EsTLmQsZ4P%q?gl>7l@-i7x7F?#;a;s`qwNU#cmq>$D)Wk0DD)|S&3&Vutw42gLwj-FJCzg{+Fe|t5+8}k@QmUqGs zvkqSNyc@=uW^S}aOvvuI@46Dp;f<{mE=p7tlMYXQTwjN?eayI~__1)DWeXpOa z-O?voB330sm$!ZB2iD#-K(`m*p1WZvdNqNrjLgCTk6xTC`$O#bzrN|| z<o==e6cHT)`N`C_f~d>Nm)G&>-A?{GHw>i*Fn4cJ#$kKbhOE#V7MA1F7v} zXE9kMo2;bY!Q$r;TW6R1(uce&?GCxvVeQq@Y-_{RMK9s;^Z|g{GDrMd?CC;$8Fol<()?6s)jCxw zP|ptaU*vXOI`yXw6|))DMYC3d1c(D0s~*g=X)>8p4w*G+D9zR&=2Te z`S%hhfLsrHCmS}7r)qQn&d(ooNtHHx@>z1)SI>Z-g`EU-K18)N7fCy225Y_xoYy%+IOG%w6*c5Ww1?`qi? z>UPRFy&-0qndCNC#M}~p+|ZxJQSE0*dN|Hw^BnsCrC33LjZ zsZ%g}l|Ct1bB_{fGk$2m>jAFTuU>_`V?=9#;XW6^o=gSs{XHAxq6OPa9ru`@cxSE% zFxoQ{fr?lJcHGCnan1THo+hSvrtlv!b`MMq(9Wi~;P^}Rust0~Y)El3GokA|)?|)Iz z-41dDbIcIz*_gNe4VB8N?Oitxxf|VdO9MukhrTV1Wzof{T6cQ8y3l?iDGQSGTlZH* z-%Pw0{_1>nNAD@f%du|)*9w3>U%CX%`S}>|<#!4-x3qOdH$PxDf1T+jw|ZEOY-wq2 z2z)J`#u{d3gXi5KFwRvxK!mH*YVM0EsY?y+FXyUFsC9JQEYC0T;%JH%_v;SY8pJx~ zEAI}*4+Cx)5Nlm>0reiRr!TD8{0|n7-)ig}EbKaXGE`>mgqP`IC2=*$gp`YTEJ{L~ z_{Z%0{NPhfD{6}$bP!YnPtAQ(7Ra>%V1b9YZcVRtnlq9&Q5WFvmN}^fc@Us z>Z^0c4717kfM7#;S|_Rbv)}LQ#tk0bviM{CRg%^>8>?SjfQTxvO}~#FrpbOqQ~7V_ zuNhXqf+k|RZ>lrXysf0J1ppnFzu3$~CxTxsNB-ey?EHDfqsF$+mk;UKTtBcwG5Q!` zYkoXC6j*4?zCZNsQBwns+r&kvUUwK70e)q{jqY#ocxDF@1B#zJCckVgUG_KC>9wtV z&gNW@e$6)7W%3G;bpITd)Q#^%7gH{)hu&SMY)GDW)t!>~^eptzn28+lHy?it!!xpQ zgwb`j`)FXrM>k3|IY^pJyM*-g9rxWQ*j(b0&||3A`@apcnO58^w7r7<&3ZUaKP4wJ z8~rUB%s>cOC0FBJ#zZ1I-#m%o8k4jjT^5Uq_#HD8WH>^6=9w3}kkX_4O)uf8)lUlV z?=<;aLaaITA1ZR2fJu0K<7PD*2M4Yr1a{?@c~(B2oso5Dc>0v`Axnr3)XC(e+8HxrB%|XN0D|Yt*}i@;qJc^ z9Tx<{8S_yG*DgSJT8e<)t^arh07?sg3-EUn`lFWY`LXMfL?W@#_=OPb zL!PEze>dvN`EunhSIbICD_RmEMOlt3i+b$IhS~_*>&Cc`!a5vwKwR zbN+dH9E3*7=b8cd94op`QDQLqLoWDUu?1-C(A^uBgC5LMNv|14Y4dUq=&9OqXW%#7 zkNZpBtFBqc30|8gAK?EKrD*i)c>1vERDderL8KUDwDGQq%i!XlAy4mXZ4XQ^X#4b= zBV#rerPcBjoD5y>+-OQgQIZysZVW+swYaZUWqD7&+AJMeD0fSl8w*Ay3ZWn0mH7q_ zX$oDO(t+LpJ=e0YFK6I@mVe8?te3RZelm#x{$6tB!)2EFV;IQ?iySyt!kFv4dkVuq zq^Us|Yu*PN&OfoqS3v6;#Wn610JKI#->o&@8qoCo9Z9#Ykdnen_@_YO!5$6o@|LKujH23|G8w5KgaHawo)(OhHjBs?&ZB zMDzL6idjiyDf5c#cs*VyS2Eh-V^l*J4WkZI{s zb`z1h^mg-vNv+Olkax;ZhZsAoB{XwkqQsgW?jxd<8Q_#pOHSA8jIODUtUU(w^50B? z&UVZrPsBy2l^*OOV4L@sCA#n2_t{mv^Tw&=8r!hNE^X8An{wu1ckL(LI}~bu3)Nq5 zVznd}+7u+l^;MPj@hvC{%{}}(vS=g4+)o&p^uCQK>HCVhPC1p!YLyA=TnW*mB%tk! zI71?fOL-F4O2(bHfA3d}r!m^{*VCu(1(5vyxBTzKkJn1zR+e(88`{<$nm=r#2o04@ zj}aU$s*zTEw#)KwMsD}ED;DX3y{4JF#%?$guS_mZnXK>AWE0!t0l4t>nH^mPV;1y2q3$#ZJ zSA_bG>UV+1nwf)iCEe*K8W+!#g(rLH{($jc92foL1aRs^qEB7;vNgAwdhdk+CV$^Q z0Axrn@2Y{uKeDXwjO4PN4(hGxnt@G4HAP?6FS<;9M*F>bwUn-udS`&dw2ofS;>SmY z?_=;#GG}B_@HkwgN!XIJwiT@X(Vqd1nr={mlBwX8ad}&BiP-cTGvsPq>8yH@^++gt z1SYSQjl=|@oP|9I__#OcAE>GRT4>dA#4!~Y8boit#D$285~d?j7=eFFt{Jm33X2zd zxpO1>BjY$Zhw0$H&|fGAc=+8xT+Y>jDtC>9$tujO{`oz`CqaG*hmnmiP^J^QrnZ~)?~`uFDV9}^+V znLVj@`l|j{%Cg`#`1DymHlFXme^%XSlv=%ZR+x}lsP7@F?pr)JvK4So=wQM#mX?jO zmQ5;Sh^BM>n#OqSiv>@`TIytXZ^|r>_pSbCmushvz2~_@-O(+CCr@tK-cqfvig6pz zTSu4$k5gT+a+lA&PQ2bwGbN>e-KXppK~mSeL+@wYat%2Vs$J&$=K#3nLdWV0#=YYn z-F1aF!KAPI%Pd-XU@ek2IBa8C4CB5sNx$=os@!N#rYm-T1J%6_IJkjwbkb= zbZM{>2gcV@Bp+(|^|c0Z_(C$ew;cLiuFn+h+r#3jx}4d57fp1PMCprHNkD1eF6XXw ztSt^9)=x2hVW(lVzDhbNJ;?kYP&Q6!!%vv0=wae`{V3%5QnX+pk7Wi=m%=@Jy6_*H zXE)A&Z$CLpamrSCZAmu;jin!m2L3fHlXjzdCBca5LeQ^3eG-b&N@JK{W& zMbYAi)5q_pw|pmdfLENjK({|_Kq;KOlpbWMJi{tJ$0@uMJ_3GNkc7&n z>;zmTaP&%0s$fu|yPa2HqmhTTO_kCtD=+WM$@F)wn$6SxEK%}OM(R(%gF#XDDcV9| zRd0T%vd;^wm2f8JkNJBp@B?r6>_C0REmlIQ?sh}c9N_PzPem!vRV@W#B` zYU;di{wpp0@}qxLTwNfuK5`LLUsr(wrWesf1^6%h;jA}`Ss|xW;nb02M^(>L7 zXA5(!L!Wu^`qypg!}S(UAq!{@j}_+pA32GlPlzEwSmmsAkvNMp?Po3Ti_4>imleG` zMMhRt$Fx7ADh7rwX83K%;l`S;yT`jY_?rd7U|_9a_OlM}ss7?V%r@|LS@}fgR)GV1 zGZ&P2G1x=Z_SY#^e+i=dkkMyvfy4@Aw+&LQYZ%9>vNMIWRo-MdeT>+@bjkXbIo9dV z0Tq=p{Jp%Kl8j$wU{K(yQF_%xKA->X`;`d8$95%IEfkRF)0>`(P@yq|zy<(Z z>!F2IZz}mxRF;)(G!|RV0x2QLedH%F3?fdRCUfgVFBG*el%ZSV!Al+o)=bqBdk9iW z>eYa)qtC{pU^P;=*Y4F_9eOh1&^0-q%ND@J%Kkr=&O4sU|NsAo>|}+G5X#=k-ejJ# z9pjJ)nb~`my|VY-`;c+S%HG+Ly=V3g;rDufzCVAkCU<6*2 zY%%jk|Fb>6{(}88Nks06sH82}k7qc7gosiLXw(Em*GxD<+<>%i?zdx?UEQ1#G)N^g zGb5eK`|do+US^u<hLYjTl zO=Pa|Pok=r1~oPULPC~;riI9tE@1fe>}gLjH@@p2A0oyuy5K`p*i*BD_+*c!#OcTp z^d`oPuDSDbl;gqnb_h{Xe);Y!;;R5ng%=S9X!+>sEN8f;GJHfaDu4cCD099`z}E>QaAyYGg9M^$-Njc*8DuBx-eBY+h9XDEAb>F zuV9pUxO9Fz9p8Cp&AYu11zN9^oVXDtdeRmK=LuW&6Zifl_IZw8F{z0wMm?`}xbu8%S5+GDdo=lE$kZl1+tmjZycEsX9h+BRZ}U zG>F-Pn`m9aS?#1ONc+gWz{t2{!P+WyPLD-U*4yH#!En(&DH-EC--Hu;R{kRT#(DG) zrX}|jCb%}N!yS(YIKb4JwmN1LM=^cjVQ^cOOMYsnnE1KGWiW1LFwLw}RHmCw@w=#o||vUS4l; zu7J?k?LQIi9$=VD+})W~(a^r8DxYwkAl(R7$!c*w9yg;xbFFRs(b`&BEgI2mu>RY5 z-|+(+KtV5_!=H4js+mN@fFO8W+>3&zTl2TK z`~Cfj6Pig$U^QosX%T#Sk+f5%I<)@x2n@Z%LFOoU#5JQRnfS5t;XUwW zYsb;+#xjOci~US?yj&($WUFZ^D{J&@G&6JY+~n-kZ(w^ugmd({71GXj_tfNx--4$$ zd5x2fimP)>1uT1#Y!wp| z^L4*U^S$2ktr`|!-BaEQ6sSb4zs2j1t7e}#Z(dUJqh$_1JNrvq^5w4bezUT)^t%53 zVhQw9%3A*=0Zp2(Nr)zUF!I8j``>2e{h&jG*U8bCL&JRGRcE+-P5;0Cydvok!ox*u zdtN~(N79>h$pD6HQJ=pGuoMhB2r!p|5RYoq z_e(lfTtiKX?QK_+w_?vD8>zQ$gD-7ML|McAU~)rrQv;sw^V7K1`tS~uUzs*CkBM~4 z8WazGeBo^8=HcSu)8+gPR!?S@KpI23?G?yiMjL1nL8hzRl3vQZ#*O}&KCB-*`N2=) z)?#vq-T4X4oy*B}`&IWuZ&Xw;cHOMg4fTp{ifx#?&-uy;xEz2C`1zxBCnRcZBi(Tk zZUl$xxXYLYGV{I0Mtqj+iFx23#NU&8H~6hX2e6nYFA%C>G?$bCBs; zQD1XMLW6-IvcTL`q(^sH2xTdGBpH~0E0i(P$qiQT_M`Boz6nOc(}CHTa!nFT zYMG2wPmgqCY}u}5sIA@ozBstO4~v?Do~Jl0*4{AoN)pOZ9?+q;wh-kW^0bDBExldx zRI>8->E%S4tM;K^-tf?eW)^kAa%wgVa>&uZfhA@QQ}2XzvMQU9A+juV>38FaXT@hF zv<^#D&FvcRj>HDejqlRPvAfu%j2NCUsVOcKvutZLtk8w@I~SrPNCpLF>>Zb3^2I|pH&4CUdm1Ll9^(ZO#GyWw?9%< zP^2-Az);!dV8FvuycyqI+HnH2>90W-9%8BIzUP0)QY_YGuU9&;W%92gZ*Dx_hj)>5 zj7EnuFM<6dYnWf(Ub>@0pbv1(g4RAuPPT|j0m+a%Nte_|YThRB!}{ioYGy{830DTe zLn`gOaq>?B&>(F5-voVZ|12f}SeI_h?)SLv>hEuT56?xIn2HV;TZMd&)^hull_Nnu zeN}MagyD&`AQi&u#{Ym{FD@s7k8|X-I)BQrLWp}=9oC+pMK?vhQZtnMo7%}|))PFV z4}ETKz#R|{2TanpPKP!{jy^t%AlsiuOq*A93I^_~-vxSrx0*@j7G&Ca1wjNY|3VfM z15G2Ms*rLseT}xbSCERp_x@(dRXAhPvLG;j@|i3#g#-)8-$F*6I_CEI`-uf!J$KSU zZ{MZ{1P~r0Bx*auQC2r`N`a+QFt;$vF;c_yg&>jXqBhxc;izG|F4Z0S&|Gfbp4!F? zn|dWltr#-JCS-x(MIMxg!**B*cXs(w%@$EFMMT6P1u@aRxVycXF(JV;%I(yO&2~05 zfUJH*fuWmjVyM)7^KU0-o!x3ayH{{yYr@S>KF+6{6PNe28S*kEEXzbTT-4knXkW~| ztbfO1lG1~jGA%>pZ`n-n_=JUul=vKB|?z`L^os~}~#y2#*CFN8^n^LMR`H~awo?}3$XKml)_ zMtm#pq-qVewOpT8Gw)xi|Hxbt=MN=oM$x z^hpCr92)eoq`ocKpy?J(8vNWu3dZPb%383!C@^`}>w0M(MypU=4g?H(oKpY_Q^9J? z+GJ1^cf2>2$8@;pgX6cn6KD3`_x2pLH`kPe@9eHVrhOXUhglRpXl`kFB+VQ?AY7Bo zMZ9w7qn@Azo3U|yi=xxUsbcb3Z9a?&x|{%X`~MzR7pv!N)PZJ%oTa58C!m}7EB;$AzaJk5_-`@6d8$4|Dl2PWVEn!wjjVVqB^#Ll9ZqpA+G5dCPH z@FZ|5^%l5?Jdd7O)k>=Oju|Sr*hTtK$s+_oCwd%TnIHL$bE{QeVcN zO%dTLP>)g0_q2g@ej(H2-RYB_WD<28DWWh}ais~T7AzAIw_^WX=7HQ0CrIBwb$;ZN zDfa3@v$6|(>sw{Ps9Rc9UYcTe46hNiywE%H7W*lWEx`wz(8@R))SN!19i3F{aW#4c z!~O(ZZM3N$ssY<@Ke#0rgL~DtK_En_ab2mu6&RW-`LasjaukyJ=-R2(>~9QkmSe`n z!^n=B%B=9vA9tIzHjw)&H(|%l!Lh?Uw4# z%Ra`gL#C6VRr5@?%9#ua9WeJgy2elY2H1fqZb5nX&L+0->aWi%P!qJQe5~OMa-D6Nn0QE(tL!r~ z!8Af-FZu&jp^2xg5~bNzH}ibhxUqimX3{cYYxA~gkev+!s5PGye11w$_~cpHfcu*0 zW}uiT=wkgol$Zx!cEm)78b8R66edFC4kQTQ@bG!y2}CS*-*cz<`wo^*o3eLs1(4w^b5( zfv&EWkp7<#IaA5L7yX;ai6^+mCk=v$&>;#^h4%TB3<0zs{C=@2l59~G65(+aMEKF- zbqbCOM~kd|7;Y{(i74UTyOgspN_#T-f&WF(6l1mk7xlr{H9q2$Q@Q$14JmxTxGqmM z(&bMUr5&2-kjESa@qsUg^deBSgk=l;RhxX*sqORm_BRyMcj5E&V@^R|2+|=34MftI zPcbXY=}4Z2?!tyy^q!>is<|u}!*GffCRc`SXCJ%}?>Ja|?l7_gxAYUK?B~#2<0p>T zoD2WdL-8=4$GvL$T)QokQ^V1+BE(`nNveJPkY@|HD(Bu;fcj~#`Cs!l9 zPWIN9RUPOG@Z-3{q;uk5zfUgZk*ixn)}&nlK!URN>6L@M2=z!k8zDZ!69lcF>#uBw zz5wi=&uH9N{?IT~?1u-mZJAYCj_sr^KEwc^5OS@k_qik7(+T-8%`3;!5Y0o^&YVcrM?ufn^iIXp-> zZvei58G z25sthTTY)rjcLT2B}7E@IG-SPm5aWH^k_;S2}Z68=JrV=J3z-BIFiHuKAM(AG4XXF zkmMFVbgSq8een@vgia_5M4!DbEiK)T6I5CW&c`Rih&u+A#f+>BUf{E!M4_1iWG@mJWamuz7JH~WyJK$~YeXB>ekc5OplDeGK-?I0&VJ|P1_tKfIz@DgCD%=MK zL4!aPWu!Fg{!4BnwXAqjM*lFJd*N-P^aHXJKl=X^_4<2sa&iLr&fCk;(QLCR%=-ny zuh1%|5bB|j&l#j{x>{4JkDlbCZD8x;c>sW{%%OF6I!-6IC4;`=R=VurmzY9fY{m!3 zyirfvG3}WiEjR)`;}VBiO2&lLq6%8E=BLCb47!!_<=(>&iU^Vg@=i&JT> zwZ=1v8$(G;SVubp^O4`_f<4BNBf8!4`e+;UahvSEeNc#wh|+{veZg^*KcFLozwX+1 zc~j05wuzywe%GkaS>j0E`%;+3{1GIjw|RgN)6~qK4jGAl zh@}oe=MH}s`saCmF63M+x^d>sP7O`D5h1ZvCfd}|wRrQK_27~BQ@6+8)6Tzm3Y*=i zc=+=1dUoDj->j)PGHX+6sg1+riO$_(;>Tw+dfIi~u+GVpWvfB9^r`AsUBdnNB@3RD&-zE*v9|xz5zi;;VqArXBL%@A02OQ7XIG#2Jh3r2hQ2dj0bDmVVqwy zn4n_$B1(Vn7qWJoTpCyCt4i8^d)vaXl$D;Yil=K`Lm$CE2&W2Yaben;yYkAcm{N?z zZxlNcOdBdcFPlVse$@Flu4sf}fOlvv<08uUY{&On#rJmPexr5TTI*1v+2d?`%)}*& zUXgUr<&rtVQ1A~$9C%-!n0QTRB%Dz<_lZEgKYwnA;U(J%tV|V~#_5rB%QvO)xYycFm>KT?abl-!{#8@%?Bu z)fFVSh+Tt12%n`(T&>T5%PjXhN_pZkslIGObS|GjW&*=f9qs*_SNlwRXD}_H@|)>L_9uG>F6v4hQjvsAr=1&%}jAgaSRNB5WRd0NbBMohi(o z{q;gUNpr1iOinyKWIT75$wYY-e?Yj{QxRwGofgJvl#{1!72)H@HC zOiyVpiO3dybI*ylpAXme&c5yNcn$TKp(y7GN$ln+InD3P?#sY&pcNbP**?;KQQ>#> zEL$R@A%$_<=U3`jl1_{MuMfYx)XhV*K|NF)KPv@s7Q}zqOn=OxilY$QMIrL^>p(SW zTYh|F9vZD83FGqe@@K}w@S$P!(AcGq5y^Gl9=4%8`iTsaeM9RnGYDn-&po_-eN}YV z|Cjurd%4IR;)*{=n?3qAvVXxQDOM0>X>P6n1J*$~X%q%AfTqSahsMVizpGUw3Zle7 z+jv6%_V4ys9*}qZSsQCodZJD$ODYTQxu&|q2ZioMB2fIoHC#x%`wH@vDPIGPyjr}> z@z9V`<06O{)6hIatDgTZ$u!Zw`S-}aRU&Nd!lMC(utv>_dz`Je-qzpS`}j2LxZGV= z-nW_&M*ip0u;NU=te8+b@zRJFq8RK}#CcmAs|@|?68Te+-KmyUEJ zJlEXSL;6qtaiX9=s>^FrwcNZx*OiwXe^Nd2r@ zB8)=?mhe)2cJhGXu@^S6nc&CNmg5jxx(a^*TzU!z0@GpF4nch!D=&k>_~J!<0~Er( zXhJR4({_iqb_(Rp-+z@0*0fsGf7j}J>f#~64=1kmDCM_k_eie}&3|-q^zw_N^DT#~ z@FD#t+jPee`Xi>X7uZZf86@3Ox^uiWjV{40<5X6OGLKzS1tqh3NYQq$TxO$JwaWT2 z)4UZRZ;Ld7-$J4~RdI2jl}K`3DU+45z8O))X>T3@0vLTcHBuQmh?{YS>siMKl(ANM&8~*&4R5u3PlzQ|0eOgC|6j4Dh8S$H}-?d&H z`TQ_=dPbyxBu%PjADoA$@q(yhYWQbdg0>jsi%$`B_swmNyTp)Y$Vgk><&@hYbsjz? z#cLIwNY%kbfreN2dHgRqtner_yQ&Vum-q4ne48)FHH`S4*1XT#w}lK9sdWcjm%Qr1 z5tfnJB!)l_)^|tUIs3nNJr!xUs+gsdcHyB74&eHKj%s z4`1AMt^R2`^ zip=JeE}Ik<5~7IeT8WZnsxS{F%AG z>W1n0-EG>Zx;>h@y1AjTBe$w#&u_@#6)*8E=|_Zeo&KHAD@zA3jLMz2W2}%GH5;%& z+V%9`#k9uwCj)lFW`mRcSKWYb;xM-Fl9YA^@P|%LwYtdrgTp&c96xZpFsKCXiThQq zyH$z1;QRg6)m0FOhO1>}WlaeBIX4NteLfub^eNpAWVF_7`=i6Kd@#1}rK|5HxF-yK z1Peg`t0Zl(d@@wBeu5%j1KwY$dwrbQ6EMNH24U(6I6tJoOdIGpS`e zm4TL?h~Fy0R1mW3Qqq*y7sN5AKK_`AqadJ!)ruSE*UrvP9N1Q6WxeVdXQx!z+8uI1 zA>wpCLW6`Hbjx`5j@}l;t3R3^r+0LsL(=D1x%QB(5E4@Ww$Pg4Skj_{EhY|OQ+*Q} zJoUBI*Nl^v#F9$IW3QnioJ>o-GR%@0XXpl2_GXY zN;?{3W(9R5?{trj)??a+7I z0xZ6VG8Ll&Lj3n`vfUrgq+wt9e{FQntw{Xn5^`o>KP;xyD>wqB{3<8XJd%9dpJ7v^ zqKMZLy#7dKJW3izdAit@=8~)zwabW>` zL`vuOpHWl)PYZo2aqoyQR zEqCXYz8R(zxGy7n*9Hd%RU~fDu1>e?sYbRO{|o@6eO=$;4Xx@B=c*+cX*ufFTp+g_ z1ZjgijeZP+pd^Ts_4oG=4Gn<k7Y9eN8{dCR z_gRV#zbvS?^y-w$dZRMq;Z-_HHD1n6^MhgChppF-cm~fIw$jnM2_|F`t9jEMSwMNT z1d@D|DYQj5E#)_bnB__%iJQWGuKwjMrfuuvS3SA3dzUx7Cbh{!Tc_@=OA)}qgt6nU~nIu*cR?*H01)k(4H;K~MbyO;_H!+As%kP%<4$L)B@rS8~MJzO4c zaB1>(aFfwbegAK7>;4L;+`2!YK_qVeowi=3wdU;GIoYWunl>$ngX3O#BIrH9_Pv;Bi(Hk8`a3>ln)C+-e`jipxwqh(kG2WKiNhwm|m7qD_>L zm+@~bSn`;|Iu+h9Kx#E%EZu?Xmb}ns&z|x0fJ}(*b=tig*Dr^b)pX!JKKIC|)q}%l zu#DBJ`=IcHz9wCH3vxV;B(C9_s zVTwSJsKiEB-8zQ4z5^dE5nL5qXaCv~|5VKl>}A}RpEsYrVtu}m0&rX!URq6!12r$E ze@}P87HcyOr5_Wo9!$4Il_B4C?!W-b%g5y-A87up zq}$eTV4m%}h@_3QI0E4O+^0)V|C=2ke+Qbgxuz<0zl%~}FuA*3y7z{|1Ll|u`p`ph z$V}eIOp$kmfNr3syK1mE?^7_sL9lf#0HH12>)c68XE!&6BOFrtihv>yi||#!!pK4@ zb*c?c=u6cOsklU=Z?B~ljhexnLrUSh(AGaN;_dD2y{vZytt8MSa2*ablD2%6HWai> zY_$(2!b2ZJ6q@1MWrC`d$Q0K{&XRUZlap{#S!`_V`uh5aD#1ux7tfRRe((=O%Vx$K z82b8BiWD%+%gWebsvX>|e)-PPZicjOP+6o6 zlOoCf3GaeAGVeQ?OsKt^yR)OWr=z21{fsrHwi+q65qklFU!U&0N7KD{7$PL?fV<^) zt*JU|AquNXKFWx;z$AU{IW~lVYPU4&ht#AH)l_^t7JA|GXH+R#*-RTo+8%$D$9*Bk zmsFar*<8sLOY5er35<{z$e88&4t$wP#M1W^Qp{oCn8!=>z~sV?ov&JfxRZhS>+ zNx?0O*D`wGX@?%C-lKlFl`-VZ*Y)AV)Y>EiM<-A7ElxXx$|KUw+Y^Wi^0gXD#OL*~ zr6hlqL&Tn-=SSq4Ymkvs@bn0cRR2tz95*WWSB75Zt}p+qowHGy6t~_8AVRQ~xP%2S zLLCTnY!kuiXrUrcE6dOum(Mfbf|!@($|pa_<88?dBvm-vZk<<>FLyHdch%#AL}# zYb=FNg462ypJkeQtnyHSU?jtnpKDQfSFJ?fD8(T6RWz!k38lAN2R&>3Ce=f)!ZT$S zn5B;RvIbtof2l}!7J6h<4^g%Iv9PH~YVgj0=Gim#OTeMC&mkZnM5$vC;nD%nTfe1$ z&|nRF9y0mDah%Me5f$8P$o(4bhTDnrD#GhGBidpkf}TA+(XgfB}B5~Z9tEx z$62xc2kZ{)G*AFx^UbyJ&m?&7mES#B^?D%fBb-l`>E)VQ2` z+4FfnsFlM<=BPMFnj%9BW_OU$`BVUG(gbxc4$9Ti9dig2ez`jMs;s^7{d-OMp=n%` zm}MM?jv4iyoqVX=H_r297CnueBS9ag@F$WzY4AU1JD`h&FWp5h`hm>Dj1vsKk2jt> zYb2EWdU*`|qG&*C=IqbK=D|Va2Vv@YPuuVD()b<=c`LL_%fc z;$ByjJ48w45~rB`d|4dv%0f6}q|)YqNH7leM-?;1u`ebMTi&C1>4~K&AG4*UTBrBP zk5VhA_iP8aboo;SB^bufpo$-2UeT0o{8_%q$FGyOz5lKV`I{QFZDsW9mbjGBz5d*g zcB}buJ;oDsT_)x03>!A(_;{+B?lIi`jd-SmIwBc8cyo{+OUMMW&=~!5<%Di~!@3`q z>cwk{+l+C+Scivx0G{>u<(Kuh&6l#N`^!89`C3DpCh62FOg#Ws7u30eTT2Ah(uX9q z#n2f;`p`>iqf-a9pxwgJ`ia~8rs4+WC5(oPYSW}7^mfK?iM9nJQR*!=BB8Gvb2kce zyIJfj?WI)EFJ#2ixcI0YNtVoAw1qtzbCmARdvls+D$?YYjpsEZ4j-(lWJ52qdUKL{80T~38x$^TVg$g-HmIzmM z9M<};EK2ZhWF5*S?W3-)D#qC&xDzw~TBivL38+#H8WbuL8QpO*H**+fP?Vn4>oQcF zh;PiG&jE7r9d!`qi9MB-F^0{mvW!wNWoE_Zw zPTv+b!@JXHaU#K|pMW63z750U@8+&;i?P6O*9>JY96#Q7d=t2!fCltp48LFNQ8FJq zDftzbP4~GF!a!sQQetpf#|NYX7VuF6w*36-1ta!!`X#{v3~oeIhOMNYHQI3mO&nyS1C3Ex|_+B z&5xy^GIYiIxEuj>y1Kf&71ad^p^Ml?sh73=LwDi%i&^tS%VP$f#49)M}KylUc*e7;(tkf+2^K43V=<;zrDfBuApH)!rkZ1z(bUNAU|%#e@i<*1&6yyhh_R zuM#M+*0AroG!1UVc8FO-5H;2tRDyJk+BGwz#rOU$sr4TNJoW4aMDh{mwS5niC1?-< zkJ(E!e{sAB{vJMeS6ecDP2a!&y=lF_0hFBOcK<6ce9DIW$Gi8pCRHOVl zpcwmz)+SFRKfaktNk(mg zwr097GlqdBaZS+Y24n}_lcZeS&lw}CCLTJqfs(9%_Q^sm23=a@bHNSlXRd#yW0@rS zI9+LW#1n%S!-X+tszM=8e^d5j8e0ozi2EEJo$mr2{O``Y&yI_hs8mIqadj*RY{1-< z|1aS$9-ngZmZL?Dvz*+mvu(Gg1#m8(neA~D&yw&y$!CA_N3qMu4a@;O4*ttP#NjHH ziTmCGV(Zz0-Yf!Yd}Yni5Mh!XsQ$J#Z>k16>UPKf%36yw(79q^AD@uWiE6{NN?8CF zzTUKmc_&As6M|pXiz9(FFJ;I7waV6!>fOSDer^7E*ozlud8y>&t*(22- z)rSP!FQWK%F`lT9zI1FE%Q@c8P@T)p5SD2wFNybEV)uiE$FDQ%%P)j-nTdPtC2WB0 zG=qv78_KzpxAPEQ_uUAl8nlsURxGo( z@OtQ_O1V^=`R^RBK?E44{)Z9)(smlCqB>DQ2{?&N4xOjD9v<{u%4V8<$gt5^vSFb^ zr+ZL9VO(?0qe!yr-Lx=ev`plvSB1{7CacRajx;m3B!tt+m_y392q=u_MeLSSt(w81_^=_$XL>{FkZfneuPXp#BKl&y#ySo9;ya2COO`n(E|U1eq=MPV8i9 z>JWc`EHP>GMLg_*ezEF-k_9#m)R({272fKTv2W35rF&2!_`nKT!fK4~C?2Vo`O<1% zcmY&#b6FeWryFQufx+(s*7u@)us|o_jIY$_J zk7*@6d61$f1Lo>bkDsQx6P6*^`DxJ6t75q3?MCf+O`5hxT>@p4@2+$-U<^`z}N9Wn>8+J}acsN-ghSuHP zZ6|2G@ab#)Gu`rB7nNRG;|hYtTM4hrPd_)NuDww0127g5wefuy0iU?qIrq$Z;ZG;{ zcBV$Bxkqyyngx8%l3&}}U2^tSOCJjH$efWeNQV5hO~lV5oYS8K)CmXN6akh+j2 z4UePYYY|ew3#FW5;{#S@N(|Sq+v_9=77r)UpTZ050zl>I|bFEBiE%1;)NP z!BR5zaG9Q{@;i~V*X6<>b6AI<&9{fU;6R;F(-r+vTH!E%Y0ScNf&NW>yfcVY&}tSk z1Ll;9?{ypWbZ_RB8T&>JfcG{n41atDBZq(^B&&HJ>CVwn^s3;-Y?pD4St|k=qFRh9 zY}XtH^PZ@+zhDBxpO%r86&HT&A*Ky0EU_Pk7{bZaNa=(Rt4hNr|K#gE*n)K z3jD&t?;O<%f|MQpxQX2UbGj3uUPp_Q8$cM`+5JH(IG5w=Dvmwc4uR^ERCOZ6(bOyt z7c(Vru!8Y-6?IMGbrmRY`nP8<9b!vyXTAmZ4lx+)4bXipP@mo0-p$O+MERWMiH?-8 z(SEl5otCz{TZiYjR$~*-ygYBLW`ePd2f<9=Rb}o^`1j{zh<~VV4tXX;3&9jLe5)`l zxh?51^w|^ROh=a~`SRadGTpCvc|Y-DgQBKCmNdBNg{ifTil;(bKRvg^PWwdBc?ik4GILRjlRA3t9TsqgRTIH zf8!G@In^laMC365l5)>0x{S*7NM#L36KZIkKvdeamsk|3r7}TZru5kcxdlu&taq%l=C?Y(%a zaXMM($aeXw*P7#NWkAGQGLMs9id1Fru&41SS$a99FB<<=(_cZXke$?aBLzgoebr=_ z^c&5Wyd~4y?~C+)NdqWo;SPx614!aJvi}qSe7AlXdYzukfkAEU#17FMS6D7UPM4LH zoqKQ;fWd6d%pUl_xD8Fsg09P$00=Ju3K1Ti!l^e;N_u+w=(P_h5_flF!d;ZcV84}9 zmm!jJNtzSp$WUpZKSiN0!#r>QW{f2dNJIJ-a5z7G~_WrI{K(|3$Sp0idd0=f{gd?+z*sftq&6G%dSR-S+quVeh%32hWwx3@5bV1N# z4LAHj4-VAEkKpTTjV}sr0ww0o!WmpY{;u}Nfe3*po^+|()ULBH6pSnG_LuJV0RqjP zdkl}R_M64F8gbo>wRxOjM1e)B8O2x9pIfJ3n~Jwh+B8@Vkg|ZXlP#9~Q%q+_g0A!@ za8;7Yj&b=MKSaOmFFfDgHW$-t_Pp&W5Q@Lp>F!i3D+{!~#4^?0gyW*;(#rI+84s}g z-W+s-A@6{CuHF_Usc6It#_!SUcHIf;(oJ|NOJu7#1ZmUzRu2jO{=m|T>l65W@&Jap zrH{MQ{(vVq;xLp^czUS7_-oTaZbZD3&NQW$tjxPz>J-FPb&7ia25SH?PJwY8_-Zj2 znXkauV#P;A|9r6q?X@qPY;0-rJl{O8uQcEsN8!{)pko=PPa@={w$t;lVSzEI#5xxl z>j3-hrzJjt9u9GL%1m;$&yndqttzLmnGd@Zcu`N=p<5Z}!+wsw*G#QH%Sa%sW}f~( zRCz4w(XugIj*q0Q_zB~Ej`!<_p^_2)5aM=`;HEPKOE+p2b}dWyM@?zB`PJ0p;k_23 zHX$ZccLO~y-aSW_&vCX`1%IS*I8C!P^Y|=x^Fv=^5R9YA4^v=cEu%rXM*{^n5Uryf`u%fZi$+eR zZhF_x!9~4@U8zd>9XgL6N)O-9oO_)f?TrCvp^a^isnH0CPt}Zj*hWxfq_icrE!PwELtWjfrd`YrR`#~ zkeC<}w&yepvJ-QOQ+^R1ozIdwJS7@v;xdoSxfND{aC=H}Wy`3kJ755>-K({f%45-n zr6gjidg<_o$9Y8(6|VSq5BIBGiJR6(gyN?}%^ppn625lc-ml)}KR+qE+CR2G9JIaD z)BRdzDV~w2hOs!OP{zG8O1YbEMg{z&j#2Cus9n1UQwJEr&DxYn#Iy!Fl#0X9)av#0 zhca{lT4ALkN8kNjg-B&%8cKN+&Vvl&XmJV$ciOMUN~c3~gxc$VFeAx09_=e!&Q`}k z+eNtx3#5F#oK}%(Dup=&u#t~77PN4Y9nEm1%84H>En>pLe`;g*9a!k>;X*%J-*Ig9 z2Fo)Q^#ZM{D5I3fJC67NR?=#)pX7~YGglAIsu$J%wt3m3rKV%UAGf2tKavlfTlY2ce?(_p4XWXD4Fn#-MK zdUEgJa1{)3^|kMc zZT0STh)PQR%6;y5V^fPCm+MDj1?IY#aT*|p63ck;e z*Wv;fc)p>;-#qGnH{H|de_9mz7zMx0{T4O zpm=xYv6w6V>veW zy`IPd7|m`cvR->4`InE=Pmi5gzhivPYiD+pO*D=DqFOz%1LG{eoMS)7m%?MsO;}y9 z5wK*S5!W9_YUqz`YmmqpK^F@q7Dica`q;GfW&F_!_F)Pvrp2}B2Q3oY1Po8UXnxU5 zefUwoV6e2b+*KB?H3eD2AKR3}Xry#a=Q975I3*M#-fum4SY>a%`h>Uuk9iL#`;r1` zj4wD9_7>b8x?~Hjxk$az%~AJ?!3Lj%KYh* zr7Ip|;Z*I+VN$i&@gQS4r0ru39mi8vRdXRsDbDioX66|8Db$it?8ivpM5z*Tu4EoX7E8bz9`-fmR4%;P z7!!UiuNWKfaWH$#htBq}G~UbgMVqH3=i!8fITQC(w3>r1+{+e%VtD{Z{+Z#`yLfQx zlV2Ex9BT17x)U&>{!uCLYaacFfI>I)m30wV{YE3~)!t;{8juO$jD;HKXf_}=Oex@# zb#JwI6NsMwIge6NA&&V~iwg%Uhq6f+3|E@Fr$-1E3Q=D9aw@s6H zev)`q2qbj%cttUBbHN25Kc8GZXUn4nY}A)}AnPcko|Kfd))YF`xB$jERQET&_lGp1 zZbx}Ex7&Mr5xgegWe;p9fTn9jIa|nn4wQwt_#tft))v%3M)6(oljEeOWs|eSlu(T? zA@0?(-^h{nemC6s{;CNDT!SPjrh5oJQf zRJvmcS>=%7j`TKb2W@t0#Hh~mO!ZyUDxJ}wg&n7mFqj?!jOD{tfnN!nu5pz*eJ*Cq zK(|F8EctTx6YCfMWl%IvVIkAbiB(rH{ONdx;nFTXdS3$7SbjWGOW(4P@m4l-T#P%M-1oiugaTQK9GX&WMBS% zL2JDD7zIox^*FuXu!v2e0qTYB7n9+(?_1o(QieodEBx*e$w|-*Q~c5*}o-&*bqzZUnw~5=QuCVu*7lws;W7Phfw%`}u@ovxfoSpu< z$nN7T_!zOjy%Mr5#%&nVar%xk06+TaO6E&)oc=El9+==;urfu0!1J0pXo>x{3eKF} zc9FNaS!ApkdAC5gBoVSURMfz&C*lx-$o(AI*-dB~G@W@YRHmz;Wt;>WP@68J8bHditf^Fe6;qxw;?`uwsbgIbfjp) zqKO(nUAtd$Jp8~(f}3jM3AQh6VPnGRc&`h{p4=)6jGU_O?Cx4c;v*j>ELI4l4awOb zudlqYWLkG$JRs~@JNErccrOKW2WUbE2WejUSf8^+UO zoYS+8xMx%?=}m@cl3IyBg)pWM`vq2-STKB5+W;ABiX^%#jbPh?T?-cpBOTuRMT~hi6tDOqW+MpjY%_b&4G$M4)It^MUc5w^B*v;UK1$Fq; zuPMwWX?l!A4*ZGIK3%VzT+{fRrZK|+*(@GwU5eCxRX$sG>uspgs^VdIec~%OI}aCo z7n@UcMRzYdD6{uCCEhS9I)^~IL~xGpq|O}+M&RL34`bc*41Uo&W5R!9=ZMnh{JqO3 zraNC#W@G5|PQ+c;>eTv;4DO*=2Avu0=U0alynM2XllfKTP)B_2?z&Qd99xWObu631 z^`~WX$RSJWc(sXi-WJ-DW+@pq@134 zC3q;NRKPRcZ@br9&saDXc!X&G8b%^R3Ojr*a_Twk*!F-rIvfvETGZQXrS1%tX-U?@1?6@5^kW#hOE3gX%FM61@#Hs(`mh9qH61g?-X@@0!6E7ci`iMVdH^#wZc#htU?sSP)l!rG@@z2y9#lS#rcDC)<6dkO&AuXw7017Q z@5PlGy z5UM?%X6(WvLfBg=&dZ{wnK5;*J}=Kl(+p<5H;xH~wzdLLaX)}`faRuOh9a124OHqVMom~wonPY1Upk!1}^ipL9UMRWkQ&F%1D1HAAwFvozf9ar2L+Mr3vEfn{&(Gg3CV&pjlbHW%3>C4ZL{kB6(1 zsoQSu_8XLc4Kf5A2{LawI_-JBNx+jBUqAizY0iThl@{fh>;($M^3_Wa9lM1q@JlvK zwo!l20iFaJ;f`g`HiIC1JPzh)j;Pj8F-|tD`6fM|3F)_s;-oHrQc|DB)@!d61ouIi zj~e}Nks4!3zXZluYC-XYi6q5{!;Q*$$iuirNx2r7Vv1VlqY`x^deaO8l(Gh5OGlQX3J<&+8biHlsxrtOA=jD0T=T86eI?MFMk)RvY* zz-n5)E5O{^|F?d9nl;8kA^a`@z4d1d!n4R8M7VyDm89)E`h6eu{A9+LQA#9a7F!8H z5{^T}iD?V=7m%&nMY+BhQ=f6-?zfn7-55t+ho|NeyEV#{>tm02W8zn|Gl!>*s~bLp zsvOcUVxhrB!23*^lbh>*(NDzFzi{aKpGFk~;|-gN@9+V2Adr>;=`}w;KXKe(NN-?1 zWa4gmdKzR3J_3*Z!-35ELZ`j4U3o^jl-MHsO1n;%4S4ek{_3Ic&Gu2ntTj0%HejRk z_pGmf)VB3m55l~1hGo{KhJ6(d3E4r2csJf`W?)H<-40_w72{y*oiFT7&YL%8V?hnB zU?6I^b8dUs9i9#O*Fz$dHyQVr1`@9y%Hsg=^<&(fIWcLopHZ)emk*6Q?tWI*cKv-hNNf`;25qHo>ojHZX0#vN}U%XWjQ*o+3SnzG_%znc)TU z&w`ho94XtFU=QjQ9(6A2axUUEppGQHN4~JppoAUkCcdEQ2I&cG_=wf2;YxMzT!8sf zeBg6YU9ND}Sn09rj%ze0HYG9FRcv%dwg2DlyiBp3N*x`QPWN*g_s;bY*Vo+o&?3rKeBbVHhN$2w z`7oT~H3ge~pS+A2k`DqH!5_|%L~+8_ikb*{Fp7WI{oWulTRwFKF>Jr7RRve=FY6iV z^`GBPR1+#t{V0FGl^t|I8cOV6eP#Lb*gRzPa@d zVHpiJNE@NiP>4uM6?Ms_O?311{Vz`#sI6)ZVf&b&IJhQ_^^2ogsrEuS*tHR){9m5= zIAkL!2t4P=M{bicXZfoPA0pu+uf-0BTSQC-BFTr7xuy^as#j~TokVJ}~)?V0hgM1RfNPaG9eY35coHzH~gL=z< zvR`W6KSlTDpM$npyC9J`R;ZNNv(R>^4QyRWXJ!doHo@ZV6Nhf?qWV{sQEZ`{`@2>v zq1R+&z$h87>lxNJDlBXVY5n>%{3)erFOQH3On z5|^6B0dF{N*bL=7*ZPJ^Sr(6mAZE(i)5x1-x=SIzG;;G z&&iaS?b~?B{Z+j}E!_2ajw+N*-qX;@RK)PS7dM~BoGVKSei;!`u&^R87sw|o^{#O! z-NF)9S%uVF@V%2|A!}U6;h&?2$6Lc%E{)XEL_7hV4Ii5XC+toVqP?Lhfl(WmVtQwB zB{jRHqrzNnw~4O<--n3bB**R)uby2Att)rr#1}>2K!-#RRZ$8SD)PC9FtI>$ z+hRh{r*-;HlI(R>JuCkLT2#p3ivMr-E{;N9KUuG9z1Rc1#(P@NIn?e@m9C<+<_ z$u2w_MQrNSeK>xFSYpIb@)by408&F>6LcfMt+ZljT*!onLgX3qguUtwYQZuckc3NT zzb04ATLYAZLD^{M|hZD?c4#fA&Aq9Yq3>IAX#t`UxqLsfMZGrx;ECsiZlvjIQC z`R4lL5z%B^97W8zMfIO0QV+lTv!vJWzX0t9(dhU(6O6qgWv7S0Wvl=mnOz47L}2*D zw8>`q_cadI@CR@9?^7R`1$2^^!>B5Up_kt+zfO5!AVY|$FDb|w7)#<4F(%`W><5VF zCSLf*Iq%#;ug38T=_v@d2GVJxMmzIG6;;#m62l{a$Vfz@LKr!uH*M2j>QDLtu%luk z15q>|NgUWL1HpQ1T%1+1l`ut;3xNWz z%ap7srmwBlzP_(jl8mk;`=?goP1qdkxRM-QvIS;*U3~&vtO85_1+PW?Yb+>h32+7K+AQKIO16CZKwK{kjyCKf9;NBK5zQcVdD&hh6GfM8 zl0PH1grm{XmzAqor}T7aMBi~rlBKk)ztX1ZC9D=2Zz!;_=++|bWV>9Pt>5oC^O9ml zJpO~DM2rc#Zlr2;jsMQG7$EKyMddVzsjaOAPtH#mAfAr{9l^ho$$uxHfwpUc7S9k2 zax(JHjM4sIMbGo^-@iwn-=p@DRkA zkY^UrSKyavXlMxhFF-paJjoA_xnx=A8VIJ0hq~R9+pt~UY|Cu~BhZv#~X%BDjfpRq9E-heL@sdObZ6bwm zLt*5$4bdS3J>~k$Uob&E9MFnQ^|VR)>Ul=hO<)U9BxqPB##D`zkIfiPYzt-CYj z+F@Q^PCt+vLK(fY^ufc=CkcMHe0a#T-ueejGj8tv*pRS3cu$0ChH*O>fZMI_KI`JK zbS!6Ad`O!tmCtTNO#7ZKzV$gI)JXQ1EO%O?F3Z6^LJ2?ohkI=l*s!IQ8dJPoWu>m5 z6djN^K_4ijB@|CKvr<4%w`40fY7TLd}+A0=4ug(aPmMClYzNxp}#Gx$)*IT?(^* zbKWRB9z$@M`$yBOP`86?SsnQGW|%T>;>x+1tnP1=aM#2uTt zaX{ChiU<=okt0#xvbt$Toe3ux!6K$#t{z%lV_&oR(Z3bey=vQn%3|N7oz--ki-D$& zia1FOwTYSLWp)i9nP#5eT^U~pja)r#HmW#%LVZj+pMG^P`8cG^{FO!FIrFEu%MH!P zHElHag?bk!5H zD!EUn$*s{705K=d}e+0CV8207GoxzPk3_xCe78Fu_Xwkr`b` zS$$K;AUHT!FjIitm<-}S#kyCIT?N7UceSBnuHkbyviJQ%*>jL?w1b|(-CV9Xk)b7S zMLU`+lwo#SMzT&rjcJ)rb%lq=*D>j9QJ?k71ud6M8@5yAyD9}$~ zKL4RM7xh7_&i1>I|QPZ?wKSyKRq$%vWTkS(gPwGHOCZ4j@|n$aZB zNCJ=Pn?D^lhGkX}z4{>dOdlO5*eOyIvFT?^EmE}acn{Us7iUv+(Hw)?3MOM2<8j2Y ztJu2N?uR&aTwNp3aq28$bpT>V?rTCEgqawVa z;tSf#jWb!xT+VZ{FZU;&8IU0XiCra`3aXPCRy%p>c0uizf_~((d%$0%%g9Pcok*4X zi87FxF@*Afw%Wd9*1jc)fqLp;FAFfb``7>YH|uI@^_M{C1iS~QxI=T`*ENVW74nsV zk5BiKO_x=bcABrM2F|};UwYRRuUvWt!{TICh5OCln)qZN_zkJZ4~rn{H}~l#4$5jT z6|#K6Xj+h$5i4(^Uw=N6wOepg9n0NZ5woptSyEgT^K2r4)561^mQt6-m8O{1ZMcHL zC9S|^%p|{&nup{abu?rv{mpG&(q+R;CO{1cb%8x;_rse2Fo0rnsL6481GjiV@jg?z z46I7~OIf0R{{1Tmkogvj%>QN5{03P9-edSLKO0mxVTY(H^ouZUoff+}0V|nnOk|Ex zT3{HFgTdq?v*cV5cJPn3iYhXqaPE@i-^#j!4i2f{R0?pssG2|O0!Gw-|GHLf428)$ zJ?~FfM+t~(-pcY>ZJfJyz&tzjoK2wzqUhlxnvoRLY|jQmbKWB4L>4EAf`0|H5wkpC!vSor9bH{D&0bZ(sthlXt7IYw+VlC^i;gOW zJ@RuMdNS?os{QWPJtlKSgx^OF{L=h+Ztl?d z_?M<;+!WeJ_Mekeahq#vQxMX_T3m-+AQ{y*Z&@w5YXV>qv*l^abhuX+E9OoNCw5Of z4Yf-N9252yAZPls9Y_x?$>rH0h#c@b7G@Jnt~1!}0!`tr>9^A2xebBIDXQl%zk|a+ zDNW+2j06_Jh~wNEsT<+nODmlX?0@$A;egfhw!D3EPs2s0n({I|fXRHws-i}KyqLP$EO)ic z^cqiFWL=hy$-3ot&BjFzI9dS2`Z{hrTY z-S^nJ-!uP6qT&4^v;LdbP$;d8ouXI=YJaV zdegpvO+we}bG8yO1JVSH=!gL_blZhvMWGTqrZhrJ}g9lUBXF zB`p}Yv+gM}=iaQ1`F4gp6*onHeq&yDQ`_+QaGa^vd#KtgX+GEPD`HLCH}RRqXj7a( z)G*Q;WHlW3l~emlaw)ZZ_o&NbU!myLz!P&{L$fw>w@s_T)=%oFbOaWV$T1on)3#CA z<$_?W;;<+)5STjYE=RR_Pk)Gr)XT8Hv~5rl%9)t8$iw*^pVt4MstboKNQpkltzAvt zE8K{TvD9kFjDuKJH9y|N#YMuI@Q1KfDCP&Xs*=R)kUm+INA0r3GuNM{f;rL=5jplP zAPQr3b=i1=Kh70Lc04cwyDBB{fPA!hBH2LH*ZcP$N7p+2noDDDLEBrD&wT_@r|W+{ z5)V`EuG;bn^8-V4dq;I@bl4UXW{4@UL!B=0hXb@vbaeFU>S}G*-=nLmD}W{_=H?~C1{H8+ z1?kP-S5yLACis8WA5gqge-z-L3zOPSg(Ryxp21#Y-475ue8f4hZ?T!0B&w9EN8jLWvCh#6LJCBspgUQTOKJhDbArmS20P)F)n8ze={kL5z{m(lXOR zqY~y|YnxJ+jU8LAuU}}dcfBt2YLXIhJUdoN3IZve@{)x&JBLY~ zPdPCrPsLI>|C&hPrJmcpdM$agZ=R5VwN67NS){DIiY(&O9`|EyZIj^UEn2xA=eV=g zhJp7|Js)yG1g`Vf!281bjR^VMof{BJnZK#>T~>i0!U#W9o{?Hx2+oEb_d)CjjzImL zy9Cs`W@9FZb4wq!LXCuREU_FLWuia!9M{qh#H!i4bsAL0aGqDY*mGr6@g08R_(svD zDc5ni%I+WF>B%q5pDYMSJW^^LY=W?V5u+`xR`S7Rj zZLMp5Y#eq`=*RWWhqL}Pxo#hC&b4BNohX3WbJ{lazg35rqzW|-;0u3xPTMlp1`hIh zon*P`q;;pc7!Wd>Y^?FszM77&-=X{A=dWMCb`Lx`h+~lZ{97tRcZXX0-rx1Jr(e!q z>>eGBkn{;`qU-%3vyLJz!Jad-Zq>4lqeg>NYafG>62KHdFee3egb{J6mAq>#kJ8`{ zr`QJ`B|u=3E>de*rTeJPXjZ#eSpiI%=G_94_vnDu;_mA0?CP3qMq;HobY$ss1(OJ%VZTsrxIbY4av2leR%s^iSV8?Q!BIa1akVIhCEo6iCS{D=L`eO)#JD zWRuV^+f*~cZk=$Rqb^NLGu9(qQY7QWP9;7^iHx+3odN!jk0f)_nyx;FK~W-99EHcY?wCzJ7Wh&;i%R3g^(;DBSpEUD zVO@rt6XaayQ&3t5w6?6E&A7nVljBSDN7G`K&v4|--xu>Rnk2DnXcn}VeGf!HWlRX+ zENw)FG-_|$d39xU8{V~a5xfZoi9C!^!D!eJ;pBW;d85wS>=rT-2@59eR56^;gjlmq zDHbKYH0;+UjtA{5SJBJoX9+c$XRy}FXU=xpPBNkmA7u8P6$-Ra7(|bwoeuo6u{{F z<<6H;leStfioGBNqMJo2RFGkm)7Eh(_Z#XPZNig8(P8UJ-9)@bPbUwwQGFEc6%}Nk z_E^y&bn+JVZ*IMtB6t2=^|s*zCzHSK389rUZ_w*lZ*C$kaEYDE0hxknLLwq1%Gb_W zHjEPFm>n>gGC0v-S;2x-YT5C)Dc_@`8?V7w6|Wq=(2qM`3h5|LhmTsR0x}fkxrwm+ zRkL>tztu;+ZJzJ;AFZYtZmz)^{h3$vbv>K~kFEs^u&Wq*|8}e%Cr!&`5G_E5!+vt@ zVv5Rr^ADvChGqROpZ6a&)lMkk-|2^pzg%Br`bu3~d|L(8dM0I8X<|~NHMG^xO*W0aL z1m|8l+^0f@pZr~|tzeQ|YuJ6W;eu9Ep6=8mP*^z1Y%^A=E>GKXyiW2oYXiPVpD zXqVBIm<9{MW16b!DN+{NnV2CsJi+I}x9{-9tzmiZ>tT8q(zA7Km-uT*v<1>bfE`NQ z8)n(+DCF35<6XZxrnY-+;8;@<>ix~>4aJoR^%~m*52JaKQmNI-7P_rU;Yf5GI>UHg zN<%?p7IUHKo3MDO-6$q+zFBm?wQyektr0Ou5*#50`}*(CD$`xo8Ul|gT_uNziokBg<94^kp=~xeQBy!-ylW6`-U7x?qCPy6(e;bh8>O9>;i&e>m+eO$KKsp{%XH>gs=b@-KBO1CDOoI@YU9hX~ zd`SBe(^t)@N1OOHRzAaMzx8mZPLOj-88lbG;R_O61VQR)XZph+?dqgPFbD0-a}!Jf zeo31cm$NK~E&(7f+z||ZvTV!=1hHseFx-j5@6|qA;~FC-r2GK-`;WYgXcl^WC%3C| zm5bIdgkZg&a3YMl#?3*bluj+*Di1qGtt4|43~Vd_Cp21IUfJd2K@cM$qEk;tbbo0Q z85rCg*RN)+-u<-ah?-zygQ5&BroBBim^o~u-?>;GK4gFQ)W3KC0=<u{*7^hq8>dH8n#j*(`Bc*i20#nFEWbe{zd+zouBn6hx%M z@4#gP10LwQk^l(=EExZSRO5*@kP_~iGlR-WO(@mdhDl>K8pTb>^aZamxiB^L^wwM6 zhCB>)?9PqR6eqHE9$E`TpEEq%Qp3d2edVjx5k{uCXciVd@l+PFrg)3Q*-mVYc#623 zEi4!>6fxGz_f!@2#)&j>P$$r~V~Ql7=Mk(vB2Fxp+UmZj<*R)I!zjLABE~_d8tSN- zzAIR^G62xe>}MQXzKfwn>j)k~@*>{-9zP*p;3|FJ3+blIdET;QeoTNryukAv#KFQT z+{#AiC3!lKFzylfmG89_Yf4#JAs#-Eg^nd5+EKK`-Fp~nXD^2StA{S$A~%^D^|^-D zeFX*C$l+Y%5aZmFC3|*uy*vET zHGTf8w}uPPDht7MJ)tug0Rh|>(D5OldjA)Vh~2THTWI>S8(} z)RQY?BknzB`0s$bw@)r}73R^Eb3awy-*dwCo$mgtJw8K&qbL1;GkUWJMyR8OYh|Q) zDGSYtpMkMdo_OT8brbbGc_sxU5HJ+i?P0fBwiu9ge-&pn(`E0bnkeH2{qcb$Z5$Er zU!oKgE~dek_LYxs-rwKBnj<{{#?TY{BGxh!A80E{=kk)0l8TCG^a#MKy;Scq@A1R} zG#Y|Az}5VC8F0rAq{Wv>c@Mj1XJA&e?Z}^_9B@_iI)^N#stNGgFttcAHaRfS>WXu5 zoE$PMcw%?Js0L}RTEe2k6PT3K+x5yUrIeETy=az@w&$Iro}go1g6GUn7A`NNhEG@LG69e+T)iYH zremC%Ouw+e{BHMo%8WP#=3m8=E8nUn&}rxmXR9Ql${~J-Mx*MZUNP*wBSYVOM^d~%7A4?43=>LWq} z{Em#R@tD&BHeWQdF&ODgl#)Jw8ZLE;ii}wSi%Ap0_w@?8!mB%Z~Zc5l?<(KUIe}FAiLK^DUljTXNUrZJT^8qpkVsRZ%>4HJVzlC ztYM)YA(jjG((*q4(xjJ4pmhi{)Fi16=TZ7vU6zdA7f<+lnq#Ohl6a5WB~n*UgOsLe zMh!Js$5i3`qeKawxGJ4Zly4g4&_|CC48y=F^c=6}azmp#(j^wEy+CY2jKW6uoO@tt zzX;p(*2+E(Kom7_0o(nei7hFwUk$(viTfzt{_ z5VHxp<+f#^mG-P9aoagTWiTsbdKlPAm*?mAOub?Hn+kS62@wo*yRW;BT=Tvl_s!1E zf>uEkQus^JIRTOCwp^5fZOTW7M3laSN40fCmaULOGbU-N1*N$0Q(D;hyd$b{%U&5=L#E~8QYE^>54H9!NCW% zQ6v)wgk@mI>JM&iuyuVOa0UY#S^Tfy_gn=D#2=1yy1!E=i37buz};S2_GMq+^y;Dm zkemKFTkzSm?ym-cX)};Z_YS#s1SVDR%2Ch_O;iWYS?~xx%|2Z`*;YShfxRUV+jw|+ zp+UY|EYl+eDtAZZjpbh{*|cKkQ4j;<3j__C|K_N^f#~mvg28&@8SKp zXB7b%-5=6gwUC%B_`q?3 z5|$N+!>99=I;vc-A8Hh6OU7oBE10!g_AHIER0ZLkbz>T*%$6iNQesjc`m`&m+$IEW z)Rxh^;?^}niidwOpb`qdnIZMug~)&9M{QcrQd5s34-$a8*CBBxtA+GIMD~@e&Bz9KIKRnm*Pr)wC$_$eVl_iw2v7mq$N6Zs?@6pf=1poX4HFl)h>W#8M)( zNY`WY_8QeFHST3bBAd(@nM2A4wD9cuig1tZ-!YT|lQ_i2Iizcb1{0~#hpz2%KuT#v zEzOt`RFv1Bc_SIo)btE1rx;lI^77tij+Lf2XqWAfOz4ajGP&WDi{ve>uNMV3x)UL+ zIhhBho9a_ie~na^N%L?1-O(gqqA6}&!jFJ0Z2MKM?7y|#_Oe&)tY5%g5XN|z zZrR=XpR>OCdHPYX-!nQ8xWr8N!0u%JsAh|+_uQCAaIF-G-r0ptxAPu;yV(5M{I?!J z^t^c)eHtk!ikh+5IhUqEQuZSY;n+RUe4*2jGxUQrad}yCCi3v$ATDJ+oFHNr$lB(u)e;rB4d-@g z89&M|02aDQjB-l(g(Qf55YA6~aCo|LFzoa+ilO>tTBqv^l=$6$`q=?H)Vj(0;wO>- z`M`5v$9Y^|2m9!UOTb2#l((uTC;N{Kt+k9*Y9tC(Fv6!RWuW^-8-q`zsUyeuo#SB| zc%DxV-+%}Kv&UzFeSLkW&hxwJsi*j#Ng|ewW61>9&%jgqQyUL4$m#6rs!`VhSmJ<& z0OX?zt~(cmN62K|p0|azu16Ivd8O`?zQ%axHYafRH<1a=PJG;X-b3!F*EW=%~BwD=PxNGhdIw6xWlDUf0Y+Owaz{qtnS(oT=6RAKG=!P=1Ic41@|VK>#EQeu<9w@#PJ+}6elPwON6!QNkPqdR%2%Ku zNF7km)}bJNiD#;H#!rDil0~^;8=`s}Yr~M|Whmqf;poCxHX@r|N9$8%(9>%e=N{N* zh&|kxrbr?Hr}8RwH&B#1fX!Da&pex$Y9g9}=_&Ka*+^0RiWO^MTtQT;a1Zr1I03ladKmVn9%~6lcs7** zBZ`Bs6=oWBR3U#%>5da#KTO+a>Wl(296v^H%QE*ZG1ti~+hNGGLpsOQ?OD)o%^L zkc}p|9jaNYjXvGj@auVmqn=S}MzBa*VF56L$Ca|WMc7EkKueIbh5Q<7Kuf77cxqim z;L!rq1J$PGGXG(Vfta$(>y{5SxaE9P#P{Lg=Ehwx4~b&4)2l(BZPryOJj@Cv?k-gg z=J8QTS>wq3Dv7kH&j)<<5))s)gqY>3(C+|_z!{j_?ySD*A`->FdIX4iNK8y)_H2StOcWUytF9TWb?48+uxgu-d!WC* z;J!kDz;5yQdz1xo7LXxdkf$O&B}DV zmGj386np`ExMVipWiM~(IjE@#9$VkKNY0yTpnrqT$G%faGk))77dv1hp>aEH*dBZG zOz+pTDf(|T1DaYO2PJm?A-zd^h+M7{&Bb-kmCVq_m8V1X@`WRy1%0{|X4hx^$sndH zI*^X27J2TsMA1}d@a#3lOF$LF_Kn+*yyhp#DQ-%wu#(?k7nIPldD58xWXWaUE`!=x z9;r|AUpYR1MdX|ywXoVPULJVJusl#BGpE~hIlNCT0$K8?Vt7@bSS@gnjLK$7uDoH&w~icSjU73~C9MpX z)SLD@$^q?bogM92lVvHaA>aB5Zez7-s4K`$NJV4mUfe<%T{w#tl=g= zGmGP-G0BY0a0X;~^&32Y3%}J;zIVl!iuJAkm!+u)tzV}#I88-cW znfD|tmw32G;Pr`4u4CRFs8_7W-+`6O1!%o6<ERt+_g&el3%RAF=K+&PP#9ttV{JkDnN8!$>7l1A9wvnNENtrY?rJ3|0LybAyck+if%NjH}Mu|GwV=P=l?U z`SK=dNFJ{mR7NZQ{@MUH2~!O}P?AP=eS-q^#mShbKPRU{ym+Cxn=ZCDkNrDMW~R!6=WUW1}?7 z_J02efkV1+@836xCET{!XIvVWN(uUQwEza|%CD28WW^2;SJ^o#*ud!(Y{6dnop0;$D^>Sm`2)!hA$ zNhO@{jLVu~WRWOnV#DVXh$!ESjr%*#6$plL8BdEs6dJzUf!tF_mzV-l4BABb&wa`q zgmVE2lre7C9tbhQ{xf0}%&;16p^9xyVrl3xa)Kp8Aihz_6c*R4sb&=$e?A{>34^8< z!G#cG#T2jl&6K5sky3V3M_h4n0h(W=tkA_fu0ESCED^nS-<7-a`AvwB63*>r9+%xh zK!XCWS>Yx^aOl#hB8XF)fjSVEX1la&jJK(i6&J?T7tmi66DF-EeYBggZpVmSj#D8$ z@=}9;c3?gt9wdRB72g6d!e6Mq@zXckxJ5LG|72{$V*Y!0wx@%Tt>30lxYoi-;gD_u zBj8zbp|{KJW{-~6S5#PcTyJn=tKosTtX+-nGVwth577$Ee7-l-Zn=F%Pe(`A9qaCb z>ZQcqil@5)OKgnap2EBn*3zl*iJTl0&b_S`;=_eZxCwvj#LFe+NGvCp&IsGizk~(&US~TbE{6~A zcX|w*M$D-)ZO$#@Uyyj86IIeBDAk=|&Cp;RrR$;?(wB(Z;`oYFlTbHt!hU*N9B99h zywXk`P_452$@S!aU;UZsj~{=>-D^S4-6>4bZ{8CL29Dn7O3i1-ZMWuYr^gSc_0O;|ucjIHos=Va+MTzzbIhzIUN$yFQS^O;xnc;TG%Gm_YR41)S*B}L50 zXnlDFeT6zLll9z{2~2*17%D`SuY8?9Fo?u8A@H*EBCmPjEE~}>wSw&@9Z~MH5OT{- zL%Cn=JMx=ypGn_fNF$HGT4%v4Z(_)l6=znJ*Gmo?q|1xb`G8*Lc3a6(d+K^+kBqsw91}FwX#5N zm+@)}I_&pl#u3}p1`U(sh=FH-Ly{c=AwDThsxVFqm@ZpRleCK;x7Olr59E)1#a~+! z$3l(wPI(AzjyKK4KEpl?z;{T|th__ThiNGn{>8>AdM&tjlY|TDq5OKdHfC#QH?CMgiL+9|#+wN^a(=*Q59vACN4_>R zS7-p^aNChx!ufzJqYyIwf9zK^bL8QIX-=v>)j{zi+Iq7vsf3g;?hVx-7Lbk^PH|Jo z=Kc3ae<7~~M$zY$@l0EF(0&B%G|p;ds=PNy?YHFly%9HPa+H{`$0GN826YCYKgK0M znQ%Yc$kY5bvIOsPxN#FG?a0p`K{|_LcB+=kAE%|PQqmYtw5F*yYV+d^f(@%-^E>)&P9 zo8zZL!{mi7&&PJ5gV(!e4pVOjgzuiFuiuC?(<)?A(o+jAIU>J*w?Q@V*1}xNECH)t z%uyzgc8Zb+M%_dH^hO79`T)5?wg04IEX^VEmip|V;XcDwT??1(hvK?@=UD@)=N}RO z*o?0gnEy#H*aN(7e+7=nVt+IvltiUu>|d-k1j%H-=hD~1K!TWm_iaslf+!ak(v?@? z(zLnKKkhJ`6Wu(GH3aE*tu6g4ovw{h5C}cB zGNo-$45#o4gl3otsE+Bj`KcR4O>vPZIRkNNCB?)fx#+WWurk~n4*scaexi2@#u1m6 za$GPkzn728dY6H@NEWvfgNP)c`NNonRI!SH`=w6cXW}FirOeQ3=UmQ~o#mAM!61^_Mp zar>f9ZMMsQ*)f_AhZwi9tUej&Er&4N=TRj75F?03r;_GnZ* zbxE6%cG8ByIpzr^C!v3{5bA7KoOi4RjHv*Ytqms3b4C+1CG zE3CVPK$3|Z?lT%uE;i`k0-eb-PCc9ZccDRIo!eWF}JE11cVw{aaO(Tj?FQMmMh_ ztw(xAa>u&zY5mb5P2m8YcLluQ%MItAyw60D(G0;fxn(s-a@RV3`txUrR|N9P1RYM$ z*C?oh<#}*lYSms7^uxSRF_~gsN(<6Pebfrz1gKb^WYilmg}QD12ssYQ<;BTLGlUp6;$s8$G*+k0iv&df;n!E>FX zLZejoj01CFW$a($>;wDxAh%3vVNx?zdg=YXzL0-B)&+u2vIjZFi24nt<=RI(eG9`Mdy&))eW)4P{)mNJc2g2nqhbPy)Q zRTQI1++Gl{^lo!w^E8Vl;-DIV5V_G*eh2-yzjgPae;J9CtHk2_mYZc#3u@HbWhmDBN=1Zu%)tfXn@vrqNqtC*~$5(5b@ z5g?|Nb(UKLoUDWx7PiL9iB%rW41q{E)dc_{oc)<0jH=et9e%a#Oh##%G(wxwUg$Ej#U&aA^q*qdY%E!hGI?r78SHW*tPXeF7_?( zAL-N@R_j!@S5P|n@MBB3no9mi9Y~_W`>cozBbTzzi~AsXwQ!N#cI4l%@LbIl+fezn z={^IQfYEav{IDP#+?(-fh&P-?4P8zAYC$WkHV|wEoZIFSW4`OFWx>kQR+LN>)$oZd z;L-CV1plU*2Ac)0PWx`|whx`wAfbqefB;HsV`xfQc3DZF!0T~2rR!w1BSTHYT^Wz7 zq~FBD+Xwesx_vY?F}#X`TYk=1iYEyg;y04!|Lu~6GP1n6d%L>?eG+;a@RH0cb&k|Z z_{WnpHtreVzpa@YOw#v$hxV?%KBZXi)zNUhedhl5J|9y zUgAE)PoHtVc#=ETZ1O7F-2z<#$?oABja>Da4FNa((Y^W_n=rkbdswbr_O%UXgQI3} z^+qNoP9Z(nhEK>4XY6ZP>G+p1@!}I9&Zv1GgQcegy7d-DYrFsM-`92ve5@hIdnRbY zUpmyj^UH0qtIOBz>1yfbqrBC@->GXvT|%inj1-~`5r+;AOvb{NpE1(t!TIyiXkl>L z{6<8#;g!I*KjP`7U&gJy3)-z_tp2K=`1sG$*;_kxG`-jgfM|^lys>n}mq??FL}93J zjFef7t2q!*DZfRIw4>r$-mlf!Q6(jWx|~YBB+ZE0rpEhQU`J|mYV7wjTj=aXNF+Ok zC?=ygp31ujCJs(S=hRFcBML){@@e9t@e%=>ghPQLMKXO2#Y)zMh6kOf9st;fW_Q(=iYnH+55BQ=*k9XFS_eT7Z~E9oW!@kYcK?;VynO{r+fAuuld!C zN~HPGz=#W!6XvxkW$clmEJC`(9iN`ZMd{$G%tt^U#ft5RosWq`x0P8DOgdsWD_;fL zWUjk}f)xOZYJ0FD=L;j`wo*odq%@#Ok zEHU_ReuPlw{(O7jDDPQm%dcEkc zcsDyVjM%brqJ%>6dFy(m+E8nz*K^s=c@*P1^-KaEu@48EAQJYnf}zS#hE(}w&ALB- zifqU~ei&|SJg29?!pd&uRBSZl%qj4;x$5>_&$(L@plVrh)vqBj=PahBL0qyjlt{0e zhTmd%8vSKj#j{~5(5${o`&kP<=+gN-E&oEz6NwKd&N?fSkJ2k+?2UvS-6Iu76EJM|6 zYvf>5LT#Xur_k>g_8B2Pu{t=5nH(FsHAE}vwh|dOwnYVeDO>@tD0}ZTdoDv&#rHfz z8{Pqr0(kY1z8P!3hwqclsW)`hv{YPO>!p)F#rQ8c?DjHJ;(K;C(!vQcwCkml6Q|}% z0bU+KMizAG-|P`#9Zy2SP8=Q6uHXLE$iU`1$1ZcIu8wSjNpM2z{+wp~BX85O(6M`r zbTC!hUsb^5<`u{_42dS>;SYzLP9IV3r91B319Sk+Q6zMOseMY;v zNtEtQL6xpa9p9bxn3ETOi-hEV&t!tTU~-t4yK{d#sTsQc>odfb7SZ8u#*=+fb{fUE zdtL*~3#R~O>RA`9%=!jRmoPo{rZUAdXk8}@7CE}|w*CumZ9!<1cT|dcw{dMk+0n0` zZEfUW3_(rm|rDg2zc`F9#LS^=8{mhmx zo;QO?hyTr9O^eY9%F1G4q=G?@7b!9TvZJ`D_}Z(fvr(`6Z%Rz5i%7xqk&!i0u6rdgA?^QiT&CI3MlazBf*Jn_&>bQ?eZ9|*y0;s8$arGQUs%S&t z>pmk#+g*eO)=f7%*4>7cVfzGA+o)$mX!PPp3cz}Gxv+S_uo<7%qb-seUhSH58gmQeDvF==~rVvEhoUSvKvId=FckYg__g{u_Ke(X3v|VK0)l#rULmh_x_p;?+1N63Kz*qsr1ZAYyhgaCgM2%E zZh_5KZMXxHHV&F(FLmyMot<n|%u%g77+79dcwQ&grG9s-={`pdpl*0ca>lTH^zd8|o3;bGo=qe=S+sWMiS zRt<$jL!{EC-zw@&snapoK5uTra7+R>H#)1Ze}J>XfObLH4ay&|;iDpw5k(Xw^8A*9 zYD+`b_Evn>v7;HU%IN4TFmPas06a8*hBo|ulZRG1oRfSf&}>~s26!K-Fl0F0nq*`R zSmhgfi=}GnOr%&pWB9yOIsI%*NqFQx-e+1o}|Ep8_HdLJj%c_Kd*L5hmf`B=WO4XkkEH9bo>iqEZt}re38QD40BrzyaVv6B z^S|L(&6EmpJ^4kGo6Nd7ZNaW*Z+B?OPnlzk?|M5cUTy#?AqBkXXX`x2i$Ah`a&sg0 z+Da@{WSCRr)oqd+LRkGXL*HSAPmbf)qE{;~WL2nN;fjF_8D6I2VJWxWXK_z4#64K2 z&4F^i_XN5*Qq9(K05(2^%~`RKH$lNV4F+Ej8WUMke z9cor$&(tV@W(Vo0`{GDN*7lpStj~62KNB0#L>Y{7*bVm5N=#|KWpL6BK?Su*R4p6% z;vu0%5@w~3KK71sMjZNg;j)P1{#N*b9Q01Fh%h%SU>%jO##*7577s>`%?1y<+YW8P zXQTf8D5cp3fu;>zPzNhfSfL8I8WnqKqqlZOxZeJ*Kya#pTE9HT0PpJRDkpgY#Ku!n z-)@YKtzyx6F-Gong?n%1$fnY0ba)ai;;0odRX$8-v4zVek-y6<5LbB~|P7K#|+<%%hyI`=Igc^_aFyFv1ty>1a&_gT> z?nL@JKCsrtRcr*gB9lCHu%C3vSU8HNJCy3E{`m2BAjRLGkeUzS5c4ny~S+-zqcg>JRBR0#?cY z)RYO-Gr@W9@9ei=iF{SDc|YMR0y__RPa;noVR4&|lV~8nPSf!PvPuc**I!R7^4$p> z-VK+`T=kJyyiSSXvu`pqH8s6|DA)*jy68~AC!Eo2k2Kh$cvX`Hf%o*@J(N5JtR54Z zA-nyDvwQxnTZEn!ec*+I_=W+02M~s}YM(9@9|q)k(YBV?7btXp$Qz%H7y5En@uj}N zYRU6ou6OLuj-qVL&598AGvE9wc~KX`(P9~Oeyr30$QIz|pt zc_N}Hd{der-3nUvz3LhK2c|!>5fpZ!Tzi;Vbj`LvcQ~uorTP`v?BeI>lu3($VGIr!?WJK5H z^>RrVZs^sJ5+AZGi(98Oi?Kb351WCKM*|I&;Vq^0)F-0 z?%Q2|U~qsiXt)p?q>luDYwFvo<4EYIqJ^8kCJ5XkPl!UtOEVg}05_q!(hh3QGOa2N z>_!{W=$gp^?gws7l-c6M#96|8M$*;#A?MhT{MPD1wo4m>2F3&OK3_;`S?{(NP&>I8Xn0WEY_cVe##)4-}#b>@mcBVxqC#1?cauKktWANlA5A;M2A@bZZw{f zHih+{Om+ssXtWGtSA`D{ZQ?XE(Lpq#FYEwOAzCyt75yqy18U2z$I&Y!v;Q{+jZVG` zZi?0i1g8_MwId(TAI~NBr|R&T*Xhbp3NY(MtkZL%pm9Rg0VtjuaKDIgn$LzHL7f$t&aq zoX#21UO#T&v61QP>qsSd-Ws}obY)u&gY|8Gm5Byh&u81=?E4V&6LC?r*OTYb{0FMu zv2YpuUHQpsf4j*eMv3sBb<@A9s;3>oaQ65uz=j+CwDCxbG_ZJxdHP5D%eJBCamPc0 zJQSZfPs)1P*AYCr?n>MHc!?-EsxRrdqaOlF9w2+W#GQR`;%r%Q6XSXT!P2@+SW@t? zdHx4#sl2LJv0WN|HBj<&)B7lWKH%}R2K3%8irlUPy7bmCUEG}B5Urp}>g(&r%U0bx z%$AKD?xaI{It8YM5U3up!vG}2hFJfGT{l4M6XCO%{q6>2a}GwfExKOn3REfk!a6SW zuYXMGaU6D`Q&+na!QP`<%@#BgYNz*lQy9x2fFH0qsEskie|~uUy5{~=mGV?^lJs<0 zjzGbWELB-+h4=#!7*dH7n9!63lZ##M!dg~OpH;_fG}50P2q{1)2+WDZHB<{0eEuEW|T|orB_!&9xj%D`1+@$ysEkA z<7A_>8SCtEQ@}oiyWiW{NVr8+=nG`hRY;gFTdl14>u+=aTqXc7r;3*MTx^ z_Ttxn?O)#LTnDwdi+#02)$rq1g=TEC0F%S~{;WpGkP6ENjm`oOvjQxY`Iu?yt3xNN}(KSp|=glrg+;T`p2l~g%dmy6*vDqWbFB{i}?mpI|y&C250yj{AY)?r2p3;ca(zwO>=^!9$4kq^>qkcsy)jR=Z|A>1(cJiEmgI?RLEYvJ z`gUd)d`yw4W!0A%dc0_PfwfAL<*(1`NAD}dOtCvdeHe02Yv0Ib2&i9LZ84sd7bcQu z^y_qi)c@M#f+urKzZ=opq6@aOcC!cug#v7%(=B46^x-7Eqw zk<1@JA9*DFegr`JNJ>jTp7lP4b(-Eb$MQ)_%eY;|l(Fi+LiRphG=xHK>C1$a6cg|0 zDZ4g2QeJOCqD%4Ub$hV&%(Cou5&&7lc8ZMj>bggXpPkXuu0ij^|NiXPm$9txsI&p& zo!((7|S&vQ2&zpMh89Go%mv;Z=d` zt(7A}pFZ8W^WE<;2ssWL74L9Ye;~)><>eAt6wjNjg?7~1HjuYQ3zR(TBD4Y$18N_% z&VD#1k6&CV>b1FbAy9wI|CYyN0~e{8nNyNa??zSTISoRTx8=voY2{*S2Y1ccJdmY2 zC&bk#YID~kbDqSs?%_YcTkX}|!Cg^)(o>{+2*r3#l-rkxp2oa<{8J(!s4<>))76h%!&>kup#f)Vlls@**0agfcy`Gy5*RI#9MaVUx`N@Ije#stw`imLjDYOPJDe6UaIO*MM0_kEI2 z(e9evWFzRm*fbJ>S617@!gh3Og&5Gd?bvv{7u)5>cd5~HG0YedsaVAEP4o;Eg%Zt& zCULG1QAHj#&&Fx}1eh^5@VIVp<%qF=YV6*i?LZSJRv0FaQt%-tJ-%}f#~0P@5%2nH z%ATr;PV-P`>CcrDaXM9m=m?EEciYL z0V#NA7Z;oFSSR=gfBLSS`MD&XD0&9_UscIG1@>+Tc)N?}Wx%cq?EUBrN;6`aXvVPP z(<%9tb0OFXb%28* zBmLK}%WY*xR^am<`%{$!*nuP;JV*8UC9iS(@+Y|$7P|b82L~z>@xu_eTJ;R;$%G8q zt=y%F9U$Ma40hMdRa;ZLaeTFF%S?|Rfq8C=Yf})mpC28IUv?s0QT}`8r`}nar?hZB zv1Pjq3o>>p^JU9_Q+r1qHb)`Ld?uHPaNFQIETUeFAoD_;9z(wy;-KkhO%{gl0MP1I z51rgpLrT}JhjY*&LIE6ehCN5sOLXfyQe)AmYCt2!1CM7@jsHE0cyux zyUt8GRczB$x|oC&QPUDl{!0N~Z)KuLwC6+#bl?ap)Fmj(iV-cH)SrR;CuAF&D^2ko z_T!DG3Aqb+gXX>S6VMQ`Zc#Z6`+UEGI6vdXreK zz_1eD>t5H05-2Yh{axWn93V2y(yjBR|9*r z_dWp{-7Qm2*j7Aq5Sv1Ewl(S1-luD;t~DilOS zL8QeIIbPEB?c3_{6>N{lM)xMEnX?zL4G!!D`1=FHezRpoJ)`3oNQ_lu&nUAsnzBOd z130O>>+##hv4;nc1bk0KZ1Q?y2jEOcf~CWIdZN0js_FzD4SMeZk$lHf!H$;-Cv?G^ zIyqPjsD}`#zY?FdE3}tdJVm) z`8ksqJRISp`M(xRSJ%M7aze9pgfPK$XHZ*aF2lHW8-@a3MQlG#9KN9r9oCYHiY>0@ z(mUL)fyc|u$L&2$%`UfP|N4LIbQIc?QzfjQn5_h(Ep074)r@7{*@r;jg%*FP`>CwK zY4GSmf}K!Co$)-2m7PNgfdm2vc6q_1lSK!;W#8hULYpKABUKb#Ed9ZPH@dYw6C;y# zu2`HE2#ii&ouec;ze7s$9=i&I_&N8R@Le-5HN7_8%7CC%z}_IV_MhO}%*twNYg|w4 zZn~?OwJ;#cD)i=XY6l@=;K8lI&0#^mg*YQ)40+f^0kx?2MNZArX~Wa$ga*6C;Ez- zvx&=}6z#F^XShXx>kjEGb)IH7%C~+0&re|kzuUXsm@CHW`ub!$200TNzw&NS<%>y< zL?pQC9^Vl&wIRj`@IU*-;auIYW4?LA(P1Ps#gWwT?RK!#omVzod}#{$nZ=#Vkvh#kkA&h?)Ua=x40DGVt>sa3LNWqidy33 zgS886Dm-qUOTU<`Qz5)2Dc^9`dE^wmZ`r}NAcnDFsn~B62);HdrhV|$3k?fXfnHr*0ddpn^R@MAT3`E3(jZ$A zOp%G`4#-91nZZW)!zH4_!lKBGYeMMtQ9;<%IVVU_uYUY?@n-05@pgW1Mq!yt|Bc*l zNqZo<^5d}Xk^^n_v%?#5!Fg7k+y$|BOR=T9pOEk&>$GO^lSnVeQ28dz&~#rj2dJ{m zdOh-7%*AvG&5-FCEBW(*=GtlDs=bMen)$Pgq0@_@#UErbYtnvFxS7K$n3|=FUMqjVw=oNPEn|jZNCqJ)0RX({^FDb+N*UrX(;gG0q$_aMa2KuSrQUD z)Urk*`3OC|i-e6wpxwVYU!@6rZ|YgS3PutgssY>Va>j0k4qIgq`*A{rz^&RKv8O6_ zWGvuz?iQCYD!xCZ=)D12o|hxLm%AFKycv#MlUOQ1cge?WL>~J-;OE26Ko`(}(+$s9 ztUd3AYZ1mM(2Tt2O)jvskHI3sM8lq6FyLeuHnPZ$@}`~#&kr002LF5}`@St2bAXqL z(#4ohY0YvN(rR0%z_1IBsJjC7ZU4NY*!$B>F*wS}wwX9qm)}4&0s`K$ zt_E*h0nF-u>%hYJcfB(EZkt~3i2dF$(bsIgU0vs{KfQ@ST~C5Znr0lH4yI?JwEp`a zhjI@X!=q?0Q(gEOQ<^A#db%7F6kV1`Pkiz^B?KpO@-F_I9yxiKnEX3O=d%%ubXR}o zEz4BZIpcYsZBDwn)zQ@i_T5krAtf(ik=hFz6 zqu#*w?_!is^i6W=jIqAnSy7qpC182WP+9)DyA|K1l?dOn z3z1Yb4vI5!s$}>vbTZ4LFKZOc$QVJ#U$*Z70s#Ud2rrgK8g`v$o4^}~9BlKSV58#+ ztNq)aHPkU?L36hA0Sqokr;Y$_AT4o)L13;J9vB#O;_ThY=}ZOX6>XlrD|^>9Ang$}mM7B} z)D8dtyf~Pzsz)YzyCDwY&ysELsD0BxMkKcyD&6@?`=J#^E~09)73;<#=Svd*CCf2X zi6$m7x@b6M8pg?v_KU;(y?rs)Y2lBLfiFtEZGS-u_D~1|18iQbh`1@y;FJ=7&2)K$ zFktxl^XJd$eOW!SpDfj~ZGEs$ox;?J(&Rj}ntVbg^g49eH{Ok0v1!G1dS=D|xd@22 zLT6r+r^Up?+@W+pN*->Foz8WaR< z8y92M2V;uBAXWj*8oR{OF&|4&S?~57$MooY&IX4GX9;*@QCAJ%Q!TQZdw&PASi-`N z3K7RwgoK2+32~z*0Si@DiCRr|7_th}olFr<_hk=Z;Q(1N(6fmXW@m;+N3>|?tLhUL zR(#u7pR+dWo#?g^LrM~RCOzypFZ%4W!7n@OkCD#Om|L#3nD^=oQL%X?#r~N+UTmbj z7^~jsyxbs9)f$-^doqCIb2 zU=iF11wE4-VZWWwUUPNfCOedAqcbtt%t9QBk_87dwdVybv1Q*T5~L_@M^rcOJsW(l z_)~>FYMhia&8%VB!NGygFZ|(ThA^dMVq6dJh5RzXcSB)TE610cWyXN{r#;*^=yCUS zacE48Ire2tJ4rLsTW*SAw1pG%8ZE3p$G6SH_?dN5lWlx6=eI>8t>L8vMbuSttbR=v$Nfn_$UcpQ zS&yHhVy@|C+TRpf=|^Pm{U-1Y#6_I)!v`lz-0QtBzYxl}a3qU-_ zcQN&~81D``J+sCqPmx#%gJ5jz5M#M$TVG=v1~8^(+dDR%$q&V;1`(P+`4m z$M}?7{=s`0yuwppc|}HPHiq{jzm+B~CKE}9=uFO=)p7W2wel) z`<k9n%wH1!p2laV zD`U~%n07gG%>YlYTNI+<@#OfCD-w$H#z5EIYPOO-QrdMIiLyGLU?9GK)C|AZe7ctj z|JLhv%S}c4Q8vUWfjSO{)`m%3=S5Z3T)ULDjPk-M4SWJ#RXA!Pk!89M_Zq!59-Tcf{q<<`G|SaY=sVdtZ76B4%_2e8}s-BVa-amY;7k z{uZwg%Xj401+R^^g22t8hs}}_E2bgd?WWvCP1tX5th-J16u(bWlYk^$OLVY9H)6tO zu_Wt0zRYK&O3xK#`N#xyKRARhB}0l0QDS!!uDywq`@UF=T3_)1jHnN@7)PX*=#VTl za*$0P@Y7(|bB+cSaY3vJ`=(c5+(OYWziy}2vMG+1E=L;?yzyvi%$_wy5*5XQ{;fTX zR0gYtK+3#UUu1JLZLh*4{&ux1z6Yk_4b$B9!VV2H+;*`9wLs9NiYJJ8?NbWkwLyRZ z`N+eOQ^#|*z~lccL00H;x*WyxLd`sr7KOSOf$2*9lxvD9N3*`}!u`A&T!HbAc7h5P zs+Gf(vuFZ=JnGynieKUI+r@^ki(N;c5Xtf-)5d3u5_F9Ab<9V*KAV3ib))H+@lPSN zoqD)@^l76e;a8~t7hB!DeP6MUux^6v{2IuWu8N!b((&axcm|@Qs8y@aA*aSb&s_NP z0Sa9zHUp-_WWnleL)%*OUD*SEy}gpX*N=C=QCjp+1xblf(91-5YDQqqk@dX5T9+;X z^3;Ig4$$dY54Kot6F#iS)QG0XSNZ(DCt;+LzB#hKx~7@4WZ7*+2Gr}ld z$@`v}&Xv zuXb79C1VFXDCw}d((^Yp^qPh0(NkZ^CNUh%)h_Wj#;OQ<3J2p`Tp4h#9{!!|-UfDf zPm9?AoGbg#vR%W@h{KY+K5m%XhUng?(Q(mLk2z;6zY-Mjpb=LCKNN3aphh&aP6Ix+ zpr%@if{cuojHzbx+MG90t>kugb!6&fXuS`&1rVFRF{F%+Hxh%Rz>Q%Y;(zu(skTQk z%~3IJid%4PO&NQO&~!%7x|GD*NjB0WNu^geCY*Oc5$mwB1-T? zVr23tJ(Wgg5C_9_UQnB9!!q=p=-*<13U;K?s6b+bMb5{a9xH2+ClEKr@2ZakB(@12 z>ml9$cA9J9p$c|pT&gk2d2ORsb4sKN0e^p$NQYgYtZB}4)xoZah!{o}`mzT6@>&ah zJI`hW^%gF&z4NJW%{}pRqlf;y1=%Z({$=z2V#K2A5w<{e=YoMM)R}w;d62?bS(kbB zVKK;z{QB!!_ftqH1pa6L_Q-FU`pRtHr;Z~&`CYDQLbRAOMKeb{AFQ=d)dH>Wz_}<# z^clM7#RzO-M()JAiO$8_LPBlIJ^$0>9>e|sTfBA0aeSGYR=vk@(RYidLlK8p1#6cF z;g3%bIvX_DwnmFl{&q%1W~L!$4;>BRCpAR!l)u`g9KjUw)#HlR^v8*w{VXh4Gy#1W zg}S~al-HPu$hMKz^+Q&$8F4!P8%X8c~+~c~YnURNi+LDd#@TbGY8eqC0(y$JzB2pL~q)W@o z1=Oe6cu^Pm?QRG^e@W_;*N!n@d5QC1C1{bu#x|3wWztGR1HMpF5UDP!4qZ$2;_vF( zDZF=r2g!2z7oVnQ@TXI=?-O^GLe>Wb!&9;uPoM0MJB;gn{=1?^_yJQs*OG*-OS~&b ze0qLdyyXc+X)Ze>ec!s;bj{@wVB?I9jRAB^_YNxUi_q{+=`J9X6i|f^9T;tXlWVo* z%**g1ibf+I00S)nu(m2u8s5ZYs^OQ_Ve+@K#PtSmlxDR=7pLTx5wkxX#e0nkPb8dJ z%+gy%slVC@I80KMWT5Q;aqKv(2 z9wtZx^*Adjye53Q9>|-WHfVUF_&w2VBh$KO2PVnU0}nbwSUld^HsQLxQos$hn{cBL${RX~7^u%)H@-x&2>_H2;H9F@io!U4Wks9V2%u^n+)UM>-9nOaxV zzv$RuRyiHS?ZWxzx9usP^NJJk7`!B)V8%q~cqnOFYT2!2QK~~)ZQHu>@q=IGIv+VC zD~=Z*$OIT-{Y7mf7{LUJhS1|OT7J-S)B%+>zKic?=-4e6q{K13gapJugip0fAs`+f0fklqvgZZR26Uz8etR>S?K3+n!95 zEe%y}o8N^dRXPmHMt0fM)Iv~6-pSZ+M_V-|0BmIQ?d^q$0cq$4GI3Y5KoXDSKwYzc zKZ9kuWdH4N()Y7Lua}rMQ_DEM3E8Mn>h>k9V%z!EsU+i5eH=Cf96cFTj)`sia;_Ug zkWc@5ILVi;Va9HN(6`L_*TWvNgr?+ZbTCNk6Z7C8u^yU;Vr<$iISd}s6?k!YdU$wx z`fXiV&8;9}r+nQ|Apea)fX`2b{en$MjLhRsjLg$cOeiAvhr#bY%FTriG)!L%G7UF6 z>Wo8!@*KaoCR&COALmaNj^3b~+J0b4!eU;x$vefVH=kD`8A5607^M$wO~nSO&#$hn zJs<)5MpyXNG531d1)w%gk$Jp$*waMKQcS?5&dnfe(E`zETj1KZo`23h=h5d#2-H)C zt9=`qPo8!ekSO5fVdg`mwZEjEzC%TyeklBc-+dLtR^e$N>QJ~?hYn(k7Cry)d(FC6 z%J5B&Lyqn5lM_lUVdUZlkk=Ica5=SyOo>6&n{w+Hkno+~JoN`X+&eY@a&Zma*))DAnA|_9~EYcOyCBv39%y2qe(P*w`emf@l_1a(mJGXxCg2=A@CZSE)8r+2+@Q z__7KRV)cEz+MkCD_-Q<&fDU%fcEmz(`&nIK)@Gu~X?ejasQJ$;ZWriES|k9t1jrBs zx&=Vb1qS;c{*;Hupk~plNm+fPy^i1jsFcM0+_32@ZGR7acwrMcBK~hS+6tRW#n6T5 z;)E!Th^#k;Xp!3OYaVBp5~bMKxHNsk$-U|KL9-~HwiK)Amih8OpJIdw*l-93cq4z; z{1D5*#i#ZaR)S+vd4l-+Fh-1umq6!CiN*uoBIG-aVDAGn#zEHKE+ zw^_Pl>A!UuHy{+_YT+Wd{H^YX!L*K^MmFZ;NqmJFE0Fvi>OjQ6!l`AW8TRtJ$A$VQ zyq-Kkkrocto^OjHL)+Jpj6NcKI*@ssmwA|<8hE(%(cwOfCC*bMruq!E#{{>AYKU0m zKes9UQeW&A`qbdQ0Mna;G^A+#RmEYcCB(eoXL4 zUSMeA$)7*a&@4FI)yK;(y+yoRg8B81Ec?I`U%6iQ>}3UnvC0u2=n^!a7qsYDz%^-Q z2o$bDeg?V_PBr~9Wq7U3Qc3R_n~sqlTbNn#79B5|8b_Jd<`4IUuMvBI-K+Orp-y8W zuS}erzrG+y3i?c)RU^hWk&f2-y=PU}xQfG!E8NR4aUIm{));>sa{G+Hw0+mczE1D%hqLsm=2V1c{b=T}6Yy!bx!> zIVuzIyE|W{DKKa-uhXxII*am#Wo-cuyx_C*b3hz^){}w;9&cuH>EE0cR9BwEs)_0| zC42jVA67n@oJa#6&kzv|9Tn!FWxv-}BNi}N>t_8hMUqFYIhWHgHRP1oguLs+QT~Yg zZ_?3GIZZ)1ome4Oo*JL7+|iQw+>^04OssBxd)sE?iuF@l0uU$@PmpGa-NY-*hoAbD zP>L{Nem*jqh? z5I*Rfo$OxTZl1JU?Oq_}7gb)$`-{?ylE#swwJZ&R=K!@EpYtrGHDu1N z{C+&2V#QSxP0Y8et4_kRY;0? z`r$BJlqMzqTg`m~I)9n55}T7?PD+LM5g;qUw4WcOqcR)fa}%zbJOFf8eCGxR6m|z! z2nVQ92`3Mag^^1~*DENU&5}bEpqHRjk(m$hsDaOZf9r-|*~QQnbxC3r5LBm-=dGMZ z(K!)&eRAUM&dbjgUFJ=vv!*q7oE0~8N^T)ZeuwK!IZS11Nu!O!yIpR({f+K=CF*V0c^qbnYla~YKp zabJ07RF49%oUH~o2ljZZyy^sdIQhw)!fnwDuKZI zxm#4F@t-3_kD|{`xhflkY*Zq`Zb+5V%viQPz5Kbu?wSQ{bD?}}JfpWaitQpEI(bR% z1z$5nJF3YFV#IegsRXTUCqSalPRytL(+1Cbe-Ln& z&_Lzz&rQ3wM&L()lp@H&$dWokJ2&CvqJvyj=Pc)e?;&^Ite_esLHQlBP2pJ&>E_8$XG10bH=o2BHBR;|{At~|YsD3? z>p~aW>kbuKQ%rXHdq0HhQCgz9jpA+e2fOQAQCop_5aCTIPAl+_wD&N0}_ zItfjK4a4#zdQjFn?57l5aVL97x?LtD=!qx%X1)r66ef5$i8^r}e#Bm=pekYrD@FX) zXQ?3lY#$XpODqmjRp|$JH&9%tV&nsTEMK$Z<0sqE;12eOF9I$qb{pSkUoqT1Qfw^Z znPmRceA)^BMSDbB;!ZxXHMNLi^b9Je{UW+r;a5_eg5mWKocz&Oi(3X8Ay@tYQKhD) zCP+kdZ#I8$HBx$T6zh9FH>QOF&d%8yiWaCWOV+y)w15V5PwtI~vtS_c6JYA;HkXl+ zsd?#|ucV$^H03*pGew$4Kipgo=M^+_DI$@#QxEv`5Jp>dUi%*6bY zf{9d_evm%EW_e=KtMglTFNwTPfCahS*QAf0?J#5(2O0ewIw8YhH?V0RJ+VU!QkG>KhH=47x!)Z7=2T|3BT)1gY|v->wOP6H z%M$#0TJ9YtZW9Z)74RId|p>_K~U*W({0 z^30{?-_Y*);dw~6u^2y}lG63OMcDO##pC_ERht7fy=w4$v%$bnUQ^DmIYTW1!X-=CH=VfsA&5C8HlFUa3>!|J}wBfqfl+I6SLcOJY z^3Z0VfLl*=QbVK@9?6fgC8P}tWqNxIkEw{MbK zEU}0W!gbU-Ly2B3eTubZ))N@(Ja`FoQ?QZ|5AH_#6`nOgp1wh0J>qdA*-7-ROdviL zH3hXkws*1#03WoD(s$;L+sA=o?{8z=z|Z}F?XE;T`NEmk|WgHF;jrLP1KiOwYHOhe)ZcwhMXifOa3Kg7f}Imm>Zw_2vIjbQX?q{&5^XIm|eH zOgGcr-8ngIy1SWe8x>U&zk?LIk4iq8moJlOej?1XSrb()8GRRX66k;rOb-Q~(W;bylmZ|Dd(s_rCU z-^V~vl|8*M~foC(T+-WM;^IK*aK@i8uBFoDTEw=PjG$PR>B7RtP(4m;+P?+Dq|2Gkd^ok^m>+#eO%PWL-&<5KHLqr&9lN`LFwL;bo%&_bDi;K~Ml zE=oc?ZvXxo)Ii(@5ur?i4{sU(yCX~xO+>uI{{w~=1Cioo4?lO0I9kNxX4ZbypEO<{ zLwy&muns41jP`Ns8Lawe3G5A(Y<{^&t4LnEHH$=(cYjA(XK_j|^n4d8JGZ`K=WRACOmQ$PjTz4}Fz}%y#hHrY=@*tX z)f%^lc$L}hWmo#sY1BdgVZs}PVcX^iC^F73tL8v&ANM}t$Mc)_jGu5Qzz~dNe9zQ~ zS=1QWEl-uianZ36Gy8xaLH=JKe?1IWjdhO=3xs2s!@UpWJB<5%`EXW@q-H-8YhY^q z(nOM~1p;ufI#!_u={S}uHQMZW>(DK-J}rnUp|Nz zyB2LjX|0prBLt2Ph;3w~Z4VNd^?W1`&T-+5=sV=aNKUHdwp5fWRsuIZ)S8Cv_v2xPJ88`XW9HRS?KEe(0 z_xCQ4xiU4qSTAd1jQY-}`2Me8UI&nPu$x(}y^=T7lm9hgz*C6b^z{ZNbP`HU|MDn# z2#li0_~EBes?6EckIfnO%1ljf%zxJllDeNt^%WSSKEi&ekQv(yfOxSC=lf9QGP4w& z$ECKg=#kW(Prh^FG0;k-+oQTaz01vLG_%mQModk+RQlBV%tJnjogd%hL{~S1s;)=X zTDrtyJq{m}??Ny*TNWIAynS<&edQ{B#5M%14JnhxN||eFj2c&K@N=!1KC-%&E2F8h zm*k;8eN|q{jY%9UqkX|AeZ@7CX?8%89~9i_H#V%hzHpM*YV(8q=Ny&GlQquyDg7rw zQ{H^_?5S_P5-;ru2gXS!^a*e_l_zS98bkQIwzJZ%v{0FucW|{R6S&hW^(ob2X zG;;ztuN$GkBn>q4oAT}p*alV6vK~8!%`Ca5)?eYXrEAVL6U=eqvDk`Fq z2!0y$V^;Z4e{%JQkOlb6d2qP8AE`DupOuY*tEy>w-^bu!js!~^KC7NZtbi>aeF!Mr zjW&jj&Hm{9-tW~r$)G;qF42GTK-tS)vj7U}VO8{AQ3fW=%a z`)TrEID4j9&Dm>>aBpj|yeq%e$2mWRC9*=fCzoltI$t}fav<6L_M@1HWde6i z%-A%&1>?G7yCkPgfoZ8SZ`s`9i1oOFr0Y^o9UqYbkW{h%JAd`Q*;AXQ@)GG7%Wd_w(g>KN_HQdoV;X zjw&WlwqU9o62p@W#z|-}XGP@f$Z84_yq{uDK`r)q7g3$}D+C#H3MNeXj7s}cGZ53Q zFFaZ&*QWetb9T9Eqs))6L`yYr1fG_X6203IJ%wa1XRj&Yaw%Y6XPau_P&xcD;ZSz^ zCb8MI`|S4D!NI}#_U-q!9jK`usK+^Jtc2#`cz;Na5RYCDeK~Kr`zidUsD9i0IoXod zTtLXiB>8jG%OD*07!10r>Jv_&-NnH%mjk5Z4}o+u&X{fH(;vN8kbI`WC^VTZWJXJ= zd)vrXfHheLGPJCB##Ow=W*G!`bZ=}7%W4s#GPB0LC&ZBOIy3JN3I5J6d)FB9v{>To zGbSrXW68+aR4y>j$Md=}am_ExtG#M0c}9JATT$aBKHgbJw-p_u05Tex9DUmL9(^(p zV`RjXQ?LRqU`i`OmJx5gO>Yo&zA-V#Dx3mxM{I_*gVbaT_FFL+vC-|jDU`SPf_CEy zyvPvsWheDu@8&-^YKwKB^VwLf8ecDSvd+Zu$tWTb2{-Y_dF2sQ+^@u96Ut=l*M zwl*XrR*uw>ldBtnL_@wNAg4GcCr9MO#;Wa4y|c}lz4w*ub(NV*Az2O|C-;r+&g-`B z?fH>j;hy=~(MU!6<3;{%dIKOuf;YI$(sF52%NEwii>qxMTtS5uBPB(9*0v{fQN|3( zMhM~s-;*YAhoMS^qS>-@PqPuvgK1i%W4Vzq92#OJc^5(8Ximb$mga+rvT5-!wTYD< z4pa0hb}fOB+`#^_SA$u3%UL-hfd( zuz*G)l8p{N$0x3|nDC*aW57ROLxjS`YHE{{AY5*9m9U+~S@=!Hey0;9lx(fjdd%>V z7k>T^{qf6v$z(`kYq~=p*Tdn%Y0KA>AM;k}Q8`)hGOqlQT8tK2dXZQ57=5I?3zBU!Dv%aC{EGj;?yo|IbZPKofQgt!v`q+DE-2gV28aET2`aS1VT z3FjhQb>5NRU*U|~sK^m1hC+s_uV(O9K`pHO%b|9=08&=^qOA|>i zMBjvoSDiGirk5i#k}U?N|COY^$2amh{PlYD;3TyGObwMDj3?6)H4d?zfDhU4qSq)$ zs_7yXMkQZDovQ=XG^>y0SV?j4fWnNqK)c}kj*MMt6rmq{~w?zJ)b@2%Ktm9Nj=6#IT&dAY*M^Q1z`Fo0woNnU>BB|K%4Gj^4Yi z;Hw#2U=^}`Uv0vnJ9~8d(B%q0M*My3*aC9e*S}+s0r-S)1z{Iyl-$?Zo zz@Ay;8nPltuli7bHvG0z_zI1g=DYg@;#R;kcVRZMn5$M3YCzC43lI}ke#=fRyXdb* zW>ofrpC~L;%%Q(!T*V)&PEM}x$uSzRT5xP$BwupUHT~PxD$^Ttq(mBIrxM+^xRmRc zo9bN55mun^!uF*a>=$m?pMigy*BJ#@bKz~+9lu5^s%2^NdZwSJiUc|k_u1BcI2r{` zyEmidRkV@?r^bI^3optwCn0mwHQN=dTqDfD+1kWV2V{m(-v$e~cwBUekFm#4Uj$A4 z&zqJMZ;U&0=>X13NiSGGG#jzic#|M}j!3T2%=o*cxTwn?Z~VRQO>1*saa+V8(T8yv zdL$l05{A3>>G1R&k8^9yf+>HXglWq8+5Gvh>t`+#oFPK;wNzm@PN*aCw`o zR5?m!7^JMgfB$1lTtH>I(gHK(bb9v|wbZ zPf`jqpFOc!r=9&bQ*wwiEa8{-ny1Ztk>RLzZguNAsY6|@%gTC$?iev2pZ|g%sA&lG z_Ip;*-FG*JZ`Y_v<-k%vV>G#EFqI40K*ss|&5DB>I}11$D`0KblM=w7y z%8b3Nj6k!MVWh=6k!e0%Hvm~+Myl{iKr^vIY-(-?Gfj< zx3^61F}G{uRpq!M1e5Lr1qDMMPfR5@11&)4%Cw-Pd{qqNRZ?5O%GFG|CTxAHm!)tM zJd^}J_XXd@cTu4?T89|^|2|Lf)bIf`t};9Jo{FonoR)$_$6Ifg_~q=%ewP=Xm6H3B zDBoT@&y9VsF9tdYi6){h#??)K!ai@%NI*9RT*_NnGpWA4Qs3mu6bVlsom@jpArNr? zS>*CmO_fMPf!0k!4@3Q4M#?d(8D$R841{rp1h5I~acPL;xi$RBu+o`~N&-nn%i$$+`dI^42ok8P9gqC=7SZPNq6sIl->ywjO>K@W{tS4T?Hnh% z$9Ffo#K6^ft3BhUkL{iH zaXA@~i|Lt*1)_2oniUMn=t(VdMKY|yk(MtHce6x?>e~Fc()`PWbZCbS4JBNac-0kv zbK-hhNjqCg%d9H9m0llE{C0+Skw;qH6d#jp0FFUc0Xt(u_7S=!2$GzN68A_6?)0fa zj8Z6+zu~tQBxM8yp><7*Q7uU_M|`2<+RSo>D@r`H{BBTVF_x4RqxAVdiM#cD>x=!D z^ZDOg$lqZqAOw21>4zJE4UKAY!yoE%#^T^RIxw8W;&ptPq_!`V@9jFER;{jDjm7x0 z?n`!&$vJ|ms6Jd6V>95qivx;t#vs+s;9?J5~LD?38gqI)jNl=JMVDhD+_9D~NyD-5Fq9^P85Sxp|b#$|_$M(4gY zT?`Y7sF=N1!b$bsA;)^vdEeHfo=k>0vkCX7`RLwKy?d>2Ih(afI-d4m^VK0Z{QfRz z>rJ)57hY5-VE20LMw#HGCes8vhGS&AEZm-LLiX^_F1v{t4I2Cc2n-@vOqA>#;!93_wxk2XjvQp;UvFeMI(oN1o{@#VO>?f*n-9}5@!?^c9t5-TV zzuWe3A|6NYBMwLtKO`qP#3jm?~7j& zOQ>4DZ8%mKi>MoQ#MMD=Scb{Ae1q|hBM^~EDJJFey}mm60+_x`OH z>0`HVDSeOTSTBUR@rWxf=u-YLu?Ec)haY-B z^z}cX&RkxfIzggZuV%QgL+*c{KU{9U2UctWduC=oo46=nQRZ^+WND|gwzE>6Q(jyB z+Sbl9(9=6-dh5Zb&Ie@+r6Z{5VvZZNKmq}VHN0gRd;RObe?N8^y^#%>?v$0Xj4M^5 zi2`Gt)GQmxwMPe0i5&iLJB4zK1S^){8grf)YZ2BAmS&+wL~=$!Qn|n$JA^2jf7@n; zPhR<5oR)w3OkKa8Pm?H=N+Sq+L(r+t2(92!sL$yVdHh$(tgX^ntI(0`@^`m6BQ~0# zD)DW?%p%{Y5)wn(lo-FBJtSC&+#?36Q!h03i%-YzTXY$*>6xg8U9_4R3ok3Jjv6s;HxVXl z4h@T@iI=k1%nk4l)Hm#Fsl($&KksU|h?41h1HG(xhxYXYmwm46qQu4Ro%8>h{2%T= zGg0u8@fazEKni&1R;ZtjQ0t2czNFB|zg3ig)rS&Ot102lepTinju9%iJI0S0Ad{1_ z9R(9GZjaQV5HLbuM15Q^vz;C)4_dTVT0N7yp7+|&aHU0Xs+?7_3^cxU$=u1?I39y$ zMQ^#oc>D2q89*(WK0mDXKLLwxpxrliST(kB>(?K!?i{w*QiCP_E@ikxTJ94wx*^P- z-5(}&%;kjyB2=2?y&5RnpTU*8bEXU_8G}}CJJ_UdJHhp@T*Q}igHB>S(x|_-=_8Ez zg^v&>Q199LRmX5G#U@rP@$a*pGd5RQb5wm%9aA&07jO>rXkD^so6`|Ip7OF{Ep%nD z|CqKx^a@5OJ-XvkIV+Cid@{eh7Co32Sr*^I{$19d? zA1OHs`n>8zzmm2ErB;u3w?a2c`pMUJCA(Tt{4KR*Vmg8%cv&l~X`B$}4VKb9YdcnT5 z?K4(jq88t}-fg)jj-Bp3U~M>29ga76tD)}!eM11Ln4{Sh6AJj{H;qO5yFx%YD$A2T zK_^J)*@XT5lR@tM?ES@Z%iLC-={8gS4^B4G>7Rr*l45ysb&`%tnY=4dnZ0|qhemTU zZizcAzgOA9ja{4VWF(KtPHi(I6NlA8uvXe63DK#=j1wz&dfnbYsYI3tq+ZT=nZ-TA zT`6D$(sY^(pxf$}jH=Lx@iIzy_2=zqbg}UumV}5!B4No8B9v&Fn~gyu)6D&~wTipG zsAmcgn444z8YGCl4Ys)I>5i?Y7^119bmOAT7hEx7%n(I8ps4^AUlqiIbv*`XjZj5b zg0;VlzN9i7RBtm09*RuhP8Gr{;XbMpOxIQsQC=mbCbxAxroO;@TBJ}8y_~vV@{>gg zL{hi8JB235Dh((w&A`l(zp@!`pX~3P{`+@xvRbIty`iYF*;VomJ&^gt!PBD!fL!$W zc{=4qSB(MDJH&9&(?`z`lB5EZ1&{oM9jwEA6k2q=g9dw9$-RW~ZA{RBuF@33s*r{C z>Kz)2SCTb*7Wug~R7PDYw;b<45u*yLW)tlXz3K!fHNQG@d)w{(cecc_QJ<;Ms;Qd^ z5406;8b4L$A?jJMK^3~4S(rp>vs^ony&g$>ISoI1a&t-pA!Cvrs^X(tPJA6;!77s? zc#y{D<&CY5j~oSdVZ!=*c+FJkc-1FlOhhh)=uG>}35<_XP8M!8dPf@bb?-d0fJGee z|IHCihFkk=<^DLYQ8D}VTS>`C2$Hcm);qJ*O39eMhPSL?-v&tAI0M*RtG?Y%uO#)pq^b4J&Eg<0U;;FloIw7<4IUj324q$(pniCl=;L!{AOwUS)tYmT#Xbi0+tM;$r%INpZz1oSc zxlD(!#JWLvuPm^TyqEq$CFENd`~_aDO`hORwNQ3ZM!cdu(IZ`|7QqIGt`&RZfQP}_ zF`EJEGJq`u$Cb6WQA0yh|0Hes!Ku{`$TvJbK9&eq((r^U?~sZEg*yW*7JlP^BhdnL zNq-*p)iFZLC2B#ca;JKa^#`2Jv(yuM8Bt zO6@J6HJl%X$UpRo+-^kY%|#DOZw9x0;P~VQr(9j8jkJqZ0Xyo!m55{mWItyyq0V;7 zdVcNge{HKl69GY!23xAgWb@PU=A^_x+p27BX9Jd5UJe&y+bz8MZEdRUqjaA!BQis% z#;>u&h`tdFTxk=Yfaxdj>)zPd*xfB_VUopQNl0%Wy6<8`8o*eB-`DVw6)oe?sXB8PPT`0|4oK9|s_w=YkoUDOsCsTZ(%IdcHFj%C}+eG38=#(UwxaPl}$tZu7L_6C^(N^g8Edec{7O zr`tzO3mm4Vi!Y`}Ace0wqg$XXyrnT_Y<~?LCvqKRjH6@C_UgU&*w*MLQgqNTE!kQz z2#}DI7kB|%Wy-jStuAr%g*GJhPTrUk&d8Q^Qag2ZTvVD2sK6$OuN1 z4VXE3$$v>Csj#M|KM@n<3^#|Y(|DmeA`OQl3oy`>rhl-Y`-%Tj6+|oR)bmxF!SP>U zROk7}!+Z}*c{xY@Si=aqn3&I2Jss$;SQ>x zmNJxg3s1wLz!~m z=pD;l8jSN2>nK!B=|=}eO#18?4N$`rqk5keQ6nQ~AWWIdM~4C2i2V4)Xyjqt+fUIhTMjT)G zc5Z7%=a-nN;VoE9XYc}H;wLs;=K<%_sTo|8g8%#J7WdfwS#=Nw*=4iuHVO|TFtjY- zg<%ha6&M0!%_96D7Eo?fGPV)n5-qY!1VBRdGc9zPotx!I922PE zy9OSBy!JC76SYnQ`*;-WvFYy`WZuhk_)YkWhD-!Nl#XJ?4&9SWZgTUHrek=+jfRJT zg2_Z%RBW5bX2y(P$~|Z#>ZK)TMQc2ShH^dca}UXPPQW3KP{I1l)m0qW}9g(Lu6zj@p^+kI@lFs>WGN6OqBwpzQv zma|n?*)#d>A;e#7qsY)YtNf~%Pjog?C)aDsOy>K|Vb&=;E6(51d}e94L10r-T-?x+ zb9NTWjd-L4WlmWy-i$6R%iE{4Yt9vgvhMczU1dU`C}!?b6I1oE3btwBA={{IkTo}m|t3|nzBt!rT~zD)UO->b>-jn zDwQNAkoXs-xOqeP@oacLnkJE;h@CVJFz*t?#50@rxmCs%+}#G2$d_wA{_9hq!%2Nc ztqP7T|L1?T&{`>JwRiOXW4`Y;oUV}IA zeS%fuo!?HBp(X}&K!%p4 zz|2xp)ACmug)}1s504lN*wUkbK(c#p>(Dea=gqN?pI;er@twDGjnsI^DN|F+4~zq; zmoSD`-ib9Z{x}w_WWQh2jI^-J>qH0@jmAD!0k3^kZYu);zN0ig-ZyOxjK04EoK*ByI0zE#|Lh{vKHRYta&yc-sQM}DJZ&hzlz3JcKRG+9 zT~(F~d8)=J@Ns6f@qZwYe~8e(-{*>dhx>zrzrHla-9%0*WK!GJlb^#)=wwt&J@KcW zInA&{%fCcZxRd06b$rR2cG{FV1$$*EGCuU$6Us4htw94W0!28Kf2 zH@>Rx*=yG^ge+4qP$T%AWzE&X!{w~tE+0k6B}4i;yF*UbOAI)Z-x@##1xB;+u^J|M zBEZ7RYQ@N-H1wfQ-^FeQ156?BiVAUuo3_qVZ|u>@_UMHWfA1$2sQCU(Zf;DA{93kQ z5!96Le&*=dS>g8|<)HZo?h5D%Efc$j6dqy}e`LD30N3|^{%h?eY4UpoZI$As z^y>a18|uRt>zSX3f4wdXA+yXwKO$h&ENn#ui8dq3#RN%;urEvm42T*4pcy0#UF0QoL>w|LYUi3=Uc5~?0k zpwuFk+2n;4CQ!}msRBeRKZ6s#qn2iBBw_I~L1JGyto6>mivIS)cjmj4X+5!Dvs>lq z19JQHhhXiSv!Sw*O2<#Tr8VPg+?7`xFo41|hL{uF*#6!DsN*g;Os=a+-;wf>PYMm1m3Es8v#X4Pn8vz}V z3lx*)b6n#=qVT~a>kNa%eknw7=0 zPwVwqg-f-5_5}DF@)o|z8|O_Z;-@^+fIqd{0SN)%-hY1X0yuh0?z9O=9qXG%%ypNJ zauwdZu&U>-yk%g<&EEdC{f*Sq+Z{-y=_d#MTD6JhR*V!oVZf0*InIC3{o}VPIF(KZ zBjq==O6Y2?m>~dV+z5KiS_%xh!L9FN9DJ&|!_D}Hn??smN{ycqHS54plT(y}s z;9%{ENhJ~V@A}tbvon&^=$1FFfKizd*{rZ>21~_Ko6t)t`#6n1ZG&4&*lfAv%OZ!% zQl60XHRLo55)i~ue$T3Rr7tA=^1T-dv!OtUy@p9^XA%75LvG=P!bn!wTOLP>ade-j ztw7E-DgcO?Ro0M&Z6GSBl^z!Rob`PET*&Kn=3s3zT2#3oQvLU}WsL`auCCn4mjmI5 zdEhI-E=h84TFISQ}U zj8=SAp!?kAoD}obfb9zjN2PR0UQNvxqNawa9vQj458PcZC>6Osnhxn2^TAa9&Rjp) z)!|43nUJtA9{@OpGoy2n`B7H`!rz#@dw22n-X8H)oHBPFJqC0CbZ04pSnEOP=eRqL z@hX4_uwVJim=@q`wXmeq5R+ZfY*db=@2|we{~336wbc!4ObQG9=EP42*$LV%~$ z*`TITm2=jVmUwH^no`Y9yNbWOpguQK+dUd<<7ktg++)aZi4xlGyo~1XP*K1}`UB#+ zmK_p^=mroMCMJDdJ#|y8k=aooS`wxz?g&0qGj}UBm*kXlH!(@5IrsQ|cc0o!x=|iavW>pg8YKup3bM|C*U7J_54Sc=ZIu^X1KRH&caYqOhTYUaP}ssf2UdH z)aH+0F85K~s&{>ExI^8Aq_N~h`!H9!_x=VfI}O6b$r7ad>JqJuJO1_p9YCm#Kkw{{@2*n^V8j` zprA6OJ~ltMg!6H~lzD=aU6%r+o zNdCG|&yjC>=CguEqa{@Kh){Ymq*UR<;VV12)a?Sl6${LSnHGL!A^VE&*s?{Bg|MWD zV?-fC90pBnG530=O?_KvE?pA_eopD!XXZc#f}c98H!}&$zds#8xo2Oe5YpAH#5Xu5&ZKcV>FI%f;Len+ zJ@o^Gb|g@fS+8%Kk;~5qe8TC*wDab$$0*;qw z&~M5nqLIPYX*P-8^DB_kC=b?p)^{f=R8sL-@tsVI>qU?CESA;$8a{4Q)3vAelF{pp zVxutX{3&MzUn+Os4?iw~{rPvZb3f?$92jY{)GFE`W&UiAXH?IB$7*UQi;B7$0)o8t zgm7s`S|llj7{QlzBSt(nOQ4+8o>7a-vyLqZ@^-2V*KYGBz3mSG!a=%}i`UWhcs`*? zd6xa!ewsZo$KH}nz_IPrQmNrsR^FT`M;wnv%GRlLJTz}+feapLU|{RcW_Iu^*BP!r ztYmpPLHy(0VwtLGT>&q|j zr>TpO=V8E~6#Ud9eu{w1o-(X8|H8v7Abo@Mqc^5$sP0dtV-+U1+K>UQHYZ8An_VI| zYXhj#WP?qoXogUKX!%VrFROD;yi8tEZwZVo?}_G05B*oYKnSgqNO<{4hQSpb^P zrcad{H=;g{GGDUz#EFjBfBo&P+8mt*KbNDk8cLAxp0TT6>)IZwW1--l?>b~KApR)Z z8l9^-#zc~)kDg-sv=_}yhgC3wdh$kJc3?Wza@Bs%gjk+P2do^l1rri7-6MDcnqTCI zE))HI9*BKn0rwN)A|z7A84im$JajF1gBm&x{Nhy|_!c?ZbgRs(ckV+BJbo-x!GBZl#GkAF33pb8 z6C8!{h7~*Mcx2Z_N$lqN5uglo&k4qSy!zPfdaLdm=;_yH(hz9qE9241x;T-aQ=B?$n1?O>!W0MKqy?#d&gKSW3JnH)!Qg%J~NU0ao=egVyWe! zTa|5sOj9U;LKDx-$&C0op*IN8dwcn8x>|H@)J}gGR-Ds`$i4MC$ll$|6TDDQ&2fs; zCBno=W#%V4LhlM$Qaz$LCVHT$3H3)Nx~@gl^_FJl>!N2=j7f^?%0_J|DCgD+j80s$kl{KAee3fd9gMjm)C}RdYXme6$`rY zYk+00#5rfp@4}KkR_P06ZSDuEV?}*8jkKRFk5S5aJ}ap@G;^CZg=*F%%A3n$p>a@J zsxk`ni8R}Fa)|wG>0$G*O|u9t!GZ6BLj)?dk*zF7&D$8ePu85*tVK{Jz=c8}(|al@ zJ+#=`2Ag;j@|M0+X|eKwyXeINn(QD7ubZ41R-v#${g}VGj12WpyjoV(4D7bM&8VeS z_+9Txq&DkEH;#{E5iq-+ZiImznkQEz5A0)-T%nS&iI_OKzikOCHW$3)4VE5p{Dwct zClNylVaa~%!qt;k;liTb{?F$C8I+}@uYU`)cz^#A)YIQz#Q!FeRb=*`oJw^ilw0AJ z^*ess0oO-(z)KgmDA7`T{Ror#NBbpKYnx z(qpk?pd2EHb+(eVZ)~IcMD4=;^j?<iQ(C6duu5V4tNM;RgYO~0fDLPnJYE6 zi%0WTaUfQI)_x{(_(Ajs?TYA96 zg=z#kES|hgB*M2!u?(-f*r{sOQ9z{s`9pe-r<%rDn1IA#lel}eIKo>gn#Fd0ld|Hkg_i#^r z%}$527a~Dk`hlU4oWVar$7!sa|t`v<;2}jdvnKF2Qfo{Is z;Ro{kt3>3iN}+Syb}E3U)^@wSUBUqv$nn65?OVX-oJTMrtKl}c5h+agJU zKb0~=SZI|eFSW#uQjDSrl#AlIJ>(#8#BT`6{(fLi^7N5rB7KSz2iWmBR8dCAW zSO4xFPi?1cE-Uq!@+tCm+wp-S~pP;xkjoGey6GM=hDN7Mk#$2lZ!R#JR?ysiez z2pC`a1RN~PswWkAiWFC5qtCMJxE~4+aE)u0q}z(h_ZpmZB1_E@6}jSG)m#2;;3-pz zga1!%x(FJT$|=;oTSuw{>tT*-sf!-Ov$w>mXO+hR5$a)}X^r1Gyx6*1>6G#&qAh4B z^q+&PzbBqKEPCYNs$YGbINDk@ms=@J83q1(x|iYWhA>C`x!1I5t)SR>y9y(;WqQXg z)#Q1CBSi)Se%0GOWDhcHl9jdGTn)FPbsSw!Z4^zSxh$Ob7RO48BN2>~25K%$z-L0Pk(16EQ_= zFVv>Z*uUE5m79tZ3liQCTXxR(bXe~O+$++5V5*bhwY1VbX(|?Qs}^VLF=nGhD0#m#1|SE4vMKO+x;JhEu?sD|im8RDEZMG=C>P*ixLg4| zdG(p)L=Rt2;4}W>Up6cROsVc-jx60a(ZYx4+QX!jUDj%vlj%V+zL1vZW1p;&}lu| zF$w1D2$HaZjiL(}JX!bdF+c-9*?1kbJF5#pJ#93oaNM176;_$lJyvRY8ww(5T0i{~ zj)hXF_4m+-2_!1vLC=`{QO%03vR|IuVjc#Q3gfL-Q6uEUs?bzlQzl1pFj1+pD!8KL z6)4$&fJMHXkR|3cUB!#r(pGX5BTfZ1;Vp9ok>0eD(yWCse`Jo|MF5orYNEW}v}UI5 zNpx>(?tFehKKnt{)%^m_;9Oi~F87r?F2+n<1C#}ER2vSi>DbzO^~v>E*zNLZ!}+%p zRBJsegF}U&ERLHwQiZ#?wf$}uJsyeBQ-~a=d!yk3{`{NdS=#P7UB+_y*Zx1hpz2pd zk$t_ry`8-Psi4Z1srJfmmBRIWlb`9`E+$PPx$*TT0!%K=rmnU8DqxEuk^Af7{_)OE zpdkkUuUhaN#9(@_M{OIW+Pmdtm5AG8$m#6iZLW*&sKj}ip zO^2-tEmr6{w-*^Q52${y7Kh1@R1A(IN*o)v>V4sY7n@I)R8Nc6yzi$E%2{{_LIo!6 zdc{wHR5Y8eo)ZoNG2@>+6xHHp@_h)ImQ_xkb;wTDLPa5#gto9-Lk#OT%VdrNb52ge z9{tDse?QS3>+9oo{+<5&^T$J?D8&7C4_Jls@M>}}_Yr~WY7Xx2E2KE6Bm%$o^#PLP z`qtV}WKfso%ZT{V(z+91*nRII?3yd@zJuLy@RxL`6)hL3LFx?u{g%pg?({S3^ zRNoHrG6%A0qJoG*EttV=-{er^AcH85+IVF*9H?WbbaGM{gZAXRpbW_Ot6x8rO;luH zH8uHEmELLhYLUvKg&H#|9>Iz;=a0$hGG4Qjpv!SB$C+J&H#Yh)5g5~Nv$@yzF(P0H z47I^e+(e4jFIKP;s*|}Cnm!fj5y_yPLiLS^K)TrvR(|eXOK0EkiLdS_MZZ3O_2yW* z!L$I1&cJxCr;An>7dLPG;eP5}t4)|L=;!EthPFfRW)v8BCK2$vuHR6hzwH-r2%VG_ zMn?x#P!lKxBhbw`AUS^m@Hiv30vUw~!Fhrk!-x9gurJt)J2o}bux-|1rc-0FvHOF&>{r~=5&2ie;Y7@O9VW9Otk!7u}9N^{wG0V})EkMOi zlcup_!tk=aBU<-WlvIJ0$Rj-OL_$cH5Gh8nFsG;6T_!DcxHMX7jX4>=ep_TV>Ysh6Gn=Wq9~|dL7RVC>QU2w zyQ$g+twX7Tl7i%lV9K#J^fFNpVIfYLkS+s&HG6$ECmG`5ZVxpy*ZPHwk~Il&&iDN0 zU$*4ZkvDfZ)#_(ZAkq8P|NiAQ%Su}gmPG{+NHP)B>tpk~ZtHbTEm-4ZlJ6h1w$P}Ys3Fm5sD~$A zChWwPR2N*=D^cKbK7AHz(RdY<1fDd@veOL$ae@8BVCL{7Y28k(Tp-jSZpyxo6FUaQ zBZ?xTcmQ&bGDPLfLaqU;w*`z0iCP!{(E3B)KKg4+VLi{`a*Ws76P5oY;Jkm?KR� zcW#d@;V%c#w%AC#?P;UhPqLfX zR2SlMh$LF-=jj)B@wanTJ!|*BV7*p6?>$U^M<7O_)AidfCpt5Ed zPSO8?Ll)z4UiffSHRHRVUvR2EFP@#0Zm6D?g5xLewr__d?||b%-@kwHKsbh^1X4yR zI{Crg!T987;K0w`G3Fl;5LwiyskE0DK^IPv*u!peWrs8W_O5%*qM~Qc4|XXGNRZ?p zZ^xY_ol!E=7zub~c4}alUP9_%Wz4M5U-<^LEhKadppdJ(kX3JEExj(T1eTZ1SjYl#{D!V? z!ev*3@6OKC6QUHMp z=WG;y8R`MZ`hQ1PoB#=pp;swk^;U>+MYr51#{G+bKyE^t4_NL+7v zV&G}f_??6=jF1;3(ST9nUF0X%WDfIj!;rpk>lywjf5CbLiYU+U)(21?6NHZj>GBG` zK1Zi-GprZ++61)qsWWyL#@Fv2-9rGsOw`Zc&7>x3TJ6jN%X3!m7)6_O0+llvNHVN# zl>miihLK}9-?NB^bj^GvoMI#+he*#bvrw^CI4kTdqBsWW!|wtL5nJQm5ua&MFH*** z69~u2`0j-wMhxGKS&>=pJ2~*a_c&%BZxqV;@L4-R+T@Ft`XeCBd)Ble{vlP_$U*#4 zmFwQS;;N}{J3HFf>bhmpAAD1`TLRQx4H*|~Z4dR<$o~gR%RIXUyJHOq0#kw# z4o=OKotv50v>b0NvArMNJKzyN<4XVDo)x(YygrNuVoe6q7T2H7a%5vrXu+5jwA`Ls z9VKe-bt8dX^GhTx^l7T2>zkxS{z!Mz;m4e7$B>>-C^ zrjO@I{lH_i(cEh{Yk9oNucGQ~=quEq|As}F_)YOI6KvwY-4V55Jgzrn_1;hkBODF}lcP82Gts%MYg=@VFyoPVHZQtgCV8 z-vS=e@w%wtZR#)j`G1Xc>eKP%tMSL^nflmC4^-uVf^Sw<7666^jwHz?bERt`q1%)~ zeAaq|@@{>i?VCpGH3E@ZWI;|f$7^X-rooqGtDL5@ho0(_kmb9_Xn0cp)A|2Ls0Q5b zSwkumy2++OEyXb{9YCW%SK0>5gH+sdKjHq;o~Wg-Es3=NL>z0u%yAZN2AMF_!+M|~ z;s3Zx73AmV{*5o;2QQx9aM>p@Z;zQ$fS{DWAxec^~LjJ_sLxq$)P~h?tF%~ z@k$0q{Np892CzKFuYJJbRR~600#2-)NiSdDckci-jhnw!&)7D?Y{@A2{~${_2>J zAo+1+fc9VdkXu3tOVnf?U+SU?)t=<*7t9?p+1j>_esDDQA;zosMeG7cH;`{o{{>J* z?m#q%e^JE!AmFW)0ia>F-yZSgF<{D#G!vJ`*X4S$?R#?q)?>CLR+bs`*##~!y z0pR_|U^t(;mK}&JYI{3)gD+u4eo@K#J&oI<%W7ZXQ`67Ih_6`mnO{$SCP`Ia$m!}S|`V#sPq9K2u9@{#`3|s4J(Lf)m zaHDUJ*Ys8?wzhsy%NtvWziA%>A|)rIHCWA##xXuZsX0jNt~C2(0fJC6;Xc5n2=9{x^oiJf^>IEr*!A&ZfOJoX`~xLLPWY- zy5YJ1&tCE2-Qxbmxz70>;0qjoiCpCdY2#w#&GvxrTMP5dYlcR^C8Z>t^X{A2R9TAN zyu_!-$Z3q@ConKD@c;ccs~$vq@>n1eb?|Z5r1iA0seUE1ckGa5(t_s}>T^f?7g=It zA^C58T{5;!;#5ky?9_CKQJ}h8&YZVsUbHHCm~q1b@C85mPnO#{W8DwQS;(@P4ypE7XXvmR& zctk^MMs4brF?K`_WgjVrdZFS>GUE%7Mb9FgL%H&UfIu~Eb|?*xLc7O=f^Gb7P=p@w z*4gIa*6c6NuA8aCjA6wu_m{ox8TQw0kEfmi0V@NhIz$m4tTsBDMRD{=0U@}C87rPB z_G(UsI>as#?4}y^QGMP`C*F_OesSdQi>zrL8Py2dWvZ)XpI5@T!GD`xv4`+yjhn5r za%*S;qu2~DzAQ?+JYsKmEl)}7G(Y#`4@^98o={xZ_t<$8UU5IrcY@E8ZmJOi8Jsri zx$J35zksE{n-4f^URFuSKv5fQ>4IF3KbfmStdD|LJb+KzuWQlwdxzies>M#*c_Gl3 zl&{d$;(mH`+_NN+@!t>#3)unWY$`QtvLn_cqy8ihMV%5^h!jl6^YXQZJ!C4SpEhJS zM~)+(832z@=h2+~{Uh3)Yk(*eDR8y$`YV7xVJz|(G_Io%XXEuY(CQ|l zrD3J+5V;bUAK3KPY+;jV6+jx~UsKyg@J#r9s}!nCRO+Q3eMNoO!4wRD@eM`o?`mL}xin^fcxP+8E`XQ_}8BC)5_cqHeW zaKnlHOo$|g{bHe(WFoU_&Z{cPB%Ihr63K7YKTF!NO6X*1CRkJ>C}74tFZ4hl`6&fy z2@P&h0(qkfpX95<*8f|)zwpjiv%ixi_6r0rag9xIius(LRZa$IB)km!L1Hdpqcs~| z)3H)TY;w`b3I{P)a1SuEsG5KKS#GP=N0t~!#5mdfUN9s-rRKo9Q5P}=pubv`qd4{9 z{)!!7ReO$<4xf{Fzp_U6;1oZzkk%^Y3FQNw-~)6CjK>heG(whwO>J%vLzpY;GdUV8 z@}9?%PWt^j_3s$0P7*c-b7_^o%1F~HEjqI~H#O%{SaIks=G=p`-bRxZY1Xj3(kGG8 zL5u5X-~@K$-&JA;e>=Ncyv~o0fId^Ze{TBt!kbdkg#NdOQRcDu<*kWgNAHK7V&R;* zKclvm9gpnT74_#ED(u3R1xC#mvTI0SE*CKG;0ovm=92C4_4SMMhLVMoYh6;=H2wL2 z`-4Ser_rt9=p93fcuI8OfvK)w^n7yiyI*=sou`*;qp3Cf;Q7H&U(_*k+fwY_@8l7v zI?nqzFXrV9Gel%9p=(@S*p(w%TYTonIICBL?UtEw1D~AqP;_?|hfsP!yJYRtQ{pi& zZ*(_JnuO~)Jfj%oX(+!bPOP3lE{X&WRtAx3H1oT6SU@h zhSysXXzh!>@~*VsK8=sx9~o1BPGt5(7cYV7&s&nxjSkN2b%%V*oYBPr4Qy60ZA-J# zi~!3OCJ`Ok-I~*oV{piMglT$j2SqPyiAx~?fp#NT(N-PIKT|>z5)^MIa>vYqqWoQ$ zN6JK1?BWQV4MX!TijHo!CC_q6Y(ua(KnuL4n2)heZ-btGR^EvVC&M}Acu7-4EA`935fii;Mf zm!_T9T&L6fa%8~i{IoaxAS<~+oi=k;NfUu({hpHh+r5-AuMObvhg|B~s>$xDy~0$I z#85?MB=dW#Y9QJ;v12Yq6>o~>LT1sBafnbG!HV{aSo+!zo+POZLf8u}UP^znvpBby zTP$FOUloWqdVTn4i}In`jZVFp=*5nzsxel9I_I~k{@)E1P876|oRj$Ko0(9)srZ@C zh@v%PAcORMtngYD3%4;2MnD7F?a7xS?J2NVB_q$e3sC`>FTzFF@f7^zF+sc<^u-zP z5lnSVmee4io)|SPj|30k{IUddrZ(J;R+iNQ_69)_$B_zd8THvvj^A=8_YvTrL2z_o z1Q{_iDGGW7AIL;OMm6Q-OnY`t?L^wFo?L5J7dk#c?M^MBz{8QR=9O?gsn|HsP$I38 zuCpbWB#8pDzsNQ<{uHc-Iteqfw#dXP8BG)OuGVZ>@dz!?wAi-0@mxajEbO@)BG&BR zJGZn9`6KXrsonKV{d_I9@xk|L;Y8Ty?qSif>qv;N^|?swu6OD9-_zqgpz`|xbqUM& z00!l0!@aHZyYMf+-J?6>hLa{hp7H+`CX@ic_4b@4;P0?=Yd+R!4o2_{z4NpaJJH6$x&N62GDbay9` zL%HwCh>%fG6kOv?EHsFQ&1<8+Sy!}HR6L$CE2v%pqqtV~SL*P&w@yEcEi|hU`*E*f z%;F^)EAG&c^pWW|vCf)PGYeToGuaHz-e`}mzj+1y5=RSv1|q*98*p%nu7J{V3p&vU zn)CQPZSE}$?=6jok6Ub*a1C$;ii3w%gWi_@49#HU9?(#t6L?6^l=QI;H~u$gUASe( zakxJn2bLtOO1Z)qD`_2L-9mwK6VAzm;zqs4onSHjrUrD^P$>oO{jE+8eVx^upj&$0 zGNelY$tulw*>i|Z7s%Idve5qB^K7ef(!fA8gEYlNJgk*Dy3i;DBR|HX2VB&I1IpeW z6r_%Ul`^--Gp-j_S(frbOx{|NzRkkjlOv;4Q!gk+um3)vP_5dem~92N&je+weEA!o zN~HEvc>NGY_EtKyHGO=0yl1!7nuH>y@a!@=&$+nw>J`S;!hS=6ha;b3YUO`xy-v(& z+sU%;_ZIui?Y&>4fHmm^XauiyJKY>^=z4B5e4Y(>V&D<;x$=dD1CIj2e4HXzSR~+f zzvcSs>FMdB%E4l53$Pr(590Xpr&AHt7FSnm_Nqj|jw#mh`t!IUQKX~$o`2I)Sg@=O z@gy}!P!L<--(leQFRBxZ5p#VQ-4JsyI;$2IIPu3I|4=a=S}2)`2D$xvI9rLS^bV~n zjZE=}&(mK7oE&{VJE!n<|EBWU32HF@InkD{+K(~lz>KBNoMQ`?vE>QEHeivFPb`Wz zrbo;R1}aX5f&CV*nh)-;fr{{XeCm^vrtl72gJfu=n$&8iHm^(4=Id;CH_Ka+NR}wc z_a{~Jk`=MBkywVT9Q0u+MC3{sG6I?qGYT##Hnl~XV>AYCb^%Js+&7xL*?}~R?qk{E zb*Lc`sz^=twoyGA3?R?Vz;)bkO?BI9l)YA)v2OEP78d3gE8sTI?lObJIp1vBaR;ojEl4sf~b1S)37uw~tm=q43)G#fqb zkf2VRLlfK13=xWCbg21(VJw05bCNNxzl4Q+T)wYdE|Q!kE<}pj@8mvZ$2Uypac%E( zSzIVAXS~8udf8Epnc45_?Cji_UR#A5(r@CgmA+j^?XdhGf2~7R)Aihc5~3YV3P|NP zjqR~8cY6@uvazv6rMZ53yx9H1~n)3X)`BxO8bC;QztdBnA`E8#6;l~ zI0Kl2F)S0M{NclnYY3h73sCSYa_WxjL;*VVA{*>Pc@(tn?ru>W)q$B%@u-FHp&nzF zYyqf?27k{SD%b>-@$(dbadZV5p4i^oJx=L{njxsj_FHqaB-zHnh37C?YBJgoGLR(B z4|K?SZpO;W3f8!RWB8JBK$pRa0~Ow2ku|G^ZWg!GCk|;@&`aMBGrDRFIv$p_mQi(< z@ozjD&1cyiUzI5nUiz)@#wwakwyQ<}QFi@#0`kp0Xa4fM@lumbc@`{|?(}hsg#j}# z58E%#-G~TGOd2P%IAn5u0#BSTQ57e!}IFl;i3I!SPtiCJ#T2@D|B`xq=ptL6MESmXXMvh zE~wi>U~)+Nx0~Ckj#iy6!;_|F508nKC5N&jCvX4K*z(xKs%Z6rp-PK*LsjEd6leOfi1+ zNy39;V8tnt#Cp3@wTMED)xuY3QSB8wOylcx{@h_0gi?@VU#V;y{9(r-r0{l$2_ufM zM5&{}9|mhv@2j;e(@0f1Z`R*ckDPP`bgeXoX5{41Xv$dTWlc^00 zhik^J{NK)RT>@r+*~e)8!(jubsIRYB*7*J%eZ9}w#l_X}W83vfV`M|9JJNNy+veHq zK_$RT5()r_k=l1Ny4Z^Mk9dhO9FUa8DIu*WM8y|oaxt_foT|nvjti;UM!qeAA=j8h z)kNg!ZlVRDlM0g*Q}yER?m$SSU)>?OL`EN+b+lOFRSAHMka+KH*NF@3xx9SyqElEX zeuDa~kJ7$bW7Gs@yuSK;{kmj4KyX+xdhwCs&cOR;{Ge%>Z1^8<%MNr&O_GNGzpgVC zTo^ScdKmChFZ2Ll8R4fm`x{8M!G&YLh?SZNC6zY~r0>s^7+@zc*MZlU{8t$a)hi-* z)qrbiGPu1txD+HE!_W|PC-&wvUvkM|cSp#khW8;5n)nddkhtUpg+O075;*b;H!*=s z4~8%;Rg`>r_+@59@xa#vk_NKPSN--(XlCytIbPqXM6qvW6DBGpZ);XCzJzM)k7WU3 ztXcZxPUUQcbteCJjjC0&9hyGLSH?OY@N=0Kx(;Q{gA;4nIo4L5j#fIKoq9o~?10Hy zc95+3KuI-aT^MV&TD8?WbJk=n0jc|{<6p$EPw_AKz7ZlZ zOH(3Z!}&{zUE}ycpuc6&?w!VYJ=HHpU2P>?T}pe$4#GnC@L+!7(i-@)Wdk|+iBKIe1UsXnK|)=y&q?-`lI^tdnZ#_|g;-`6}*c$1C}vqe`&GU@M!F~U{}TJG)SVnZ$Wh1)HKGhFowu3&;BEJ93xUE`(? z2+rUhJHC&#Kr8kappG;!pTZFd-4|H_*61y?qlb}HI7Ut&KlXT=YfgIK**pgf7w*16 zWlDNFaoQ|Bm&ktEht=26)2H3}RX4I=2QoKX2B!>Ts(m#`#DlVHjgDLJ7;+f2dtYuZ zEruUiI5kpY53Q#bImud z`n3v?1d`Q1t+cEL%&JV(EI=>+&-uYo(8BmTU= zV)j%(OLBBHHVB->8YTPRFJ&Wy*HHNJ$erz;BbU7t))N$r$;c>t29&qA1FXozT0H?C zIO#j$qTJbWARe5S=l8V1>3?x?HbNzYYlWy$Do8BEr;Dx!fTPAeQa-}&_B)5(T|U37 z?pNca0<7Y*03)Oaho{0(Zg!r$$kXFvhpWTa4@1W6qiSelv?dugSYgIiTG$G=fS3t8 zA0Hn-^+W{8Lq=;3zRlIM1CNB8%XS9}+pqTeNQi{vd+foyG?|px64lxVbS1J7E1gTQ8j1u3Ov*mX-coglllu zV582}P%OnuCmS~r4t5{x-9P*m>##^!@bh5fi8J7IKHxe2*@?OdVfELU8_&8$5B^6S z4+8oGOaXJa<8e?6ng5gOMmnWN*dkogU!rvYC$6i70}sdAw{5R^o-RBBPKKk?PF9v4 zw%Z(^FI1i{0I@URbm!Tt!RIDf>~^C9NGV+{KA%|vqj44ghuQhOr6s=`mFN5CfmM>| z+u^-K>VV7WgM((A<3rz2LErNu&iRAzoyk1lJi25AEEWfMBI1YP8*6nRbH=h=aYRde zWp_25!pf})NXF%i9D)Qu-M>#ZIxxPe!8aLS1uU2*%Mzo+9DQVkphZSAg@-W4$g6L# zSjg%XI8TVz=MgxLTx)4It~iu&1QYp~6DHM?a34wSOO1NJW>jmf`o0zB>KEN!zl;$7 zcwsbGu-+X8Moit>NTU|d4!N6Tc>R+ykhO@T2t>koL5byK)$}T{X_{!kLYi7QMrj#K zt`x~Si{OJSh&(S0(+ppCPKy_>jJK3ueVPcR{56$2(u&E~q4*+3x4#!G4315>2_y3} zAxfV!O8USA0wv6HFpGNc^IN@o8|NOMt!!7Rc-gduR)7~({@8Izk?}(iJp?Ho!5!Mt z=1rM3x zCu%i=Ovd1#7X}GSW6SaKZ;GYgt@IZ%e^awx>^xY3bnm&idAZkM7Xfj>!6t@2wO1uX zNV>6E$8}35V#`bZ?Oh)o`S9WH*%hPGsfWA!B_Ibu<+W;B22Z>= z!4**nXqAz|>BICrICVo)smQEIv`vfl`tyLv&!=m_&cV^Kj37kH5Wm2HdC!pNL!%=TPE!@E+6+is8Ea!Y@s2ibKJ$wb343P{n3^ z_d?qIn<6h^{Bkjp4>^xUvX0U1s#MRo%>9nX2xo)KXd5+Swg)JG#kXDP>yuG6`rkwR zl}^7MQw$=yzJQxqu?xSG@0>4ze4U5xfjhsRz9wgGtyblk=fycgZmvGCU`tVm`nUT7 z^>WRIZQlcaAGx@_A zk)W84=!#+89MV?)?s>4jItm#<7cJ_&t%v#|513cmaLX+f`zHK{Ot$)#U4&u%P=sN% z?UmkRwvLFkdyLSv-J^h%<62A0;=MWZ7vHB(qBlcBXTg0p7L_D6{w-pU;9#7k-=_MF zj0krXauIT@t}QN>hA~L|>&WPr*nRXBpUOrHRGK7X@guGQ3QbiDJEnmlQ6sZHIc1|r z9%)D$uPhCLdT*){Li+VW+9abeqhba@^k$I|p8$?mii@DOXpLIh2=K#da##Om z<5f@4E1U;-kp-m5+EL=LzB6N0!wF4kXlSXWoSMYnhe}q1Pk&MYKNNeutRD;b&D0W8 zMn_k5E0kBXU%z~1A7Qt);iexY`*jTrvWVR1rfU#?9GV*z6wiJy%>D$J(zfLlNr7~v zLKe9R*4#?M1zZZ1ib3>URxe9t+&fy@x~`iH$1>}GPUeZ)j#RK?IZ{fy6SWmME?(U| z0qKC>z)G>?-mz5Gyf}di$3*LU=eMtsn-XaSc6q9f<}3^d$9?h>U-DT z7f)RYTm>O&n&-4MXL-ksI(8e!EeK8jyCK^H>rOhdobs>&ZB#@qN_e_z-*V$L|D#jD z`1<4qpxx1B&?5N7nIkg*GYmYp5VUwFHrO#56XgWQ4>A zUNl>Y0&L^1700W#A3B(IR@_p&dnEv_;LzK$+o+K$*^nS)3Acv`?e z!{_1uPTS*eyYQe7Yt&C`fMWFYupK?QfD9>@ViThqG|4OaKry=`jNYkJ)C|LL^qif4L6ObVdhP;a-}T< zqws|N!>pFWgE|K#E28)+&)f$+!)MSy(>fd~S*96a$$2-Kf&izS!!7e89vMe^8M-~` z+Y&MZ=W9uu{Ipaq&kx+^`C zGzto$M)*cP9RV$bSu$k74h>AF(aZHg2=w{;m*u+_n7GCpw|TchxOn1+I6!V2Y&>Y4 z@Up;jw)^XtIY36|u55tJ;#Bw>*P8D`c5&p{oh2^EWQsOt?T!XlX%L6l4*t{X_-%q& z9h+lR-~EI@gd#kv{^!7r0|wMa_Emd<8q3yaasgznhG>bb!jpT46*sBpfB*gg#~F{e z^KFj?HZnk%?tVfCZBA*HD0D6HXNCRZ*d0xx;qj8C#Z*`i0o{vNE&4%h-*tvW z>X^GqR!XPru+Bc*)c+|PO#zvk>tq&Mg+I-&JXmzCJWpsF8XD+j*C%P=b~%AKdJ}%eCPb+86tC5rGf``wy^$uWEdtq z_l*VzPnzp%e*&ceh0z~rrdBaPj2Jj90AY;#N6|&%Y|z3Ojz8~hu*b5Gj?Har%>iJp z1YUN0(H?%5Imk0`Tf}onrYpOn#Q2p;xT5FPR$g7dAFkES048MRGIMC=rOkwQF))>};KAJy|tJI*tL_6NBR$w%6k0jS#q)+UM~+&xqb zhD1y)i4XpydQ*ry?%EP}UyaaTQ$ zU-f4uI-}yG++uQs?RuoAc-Mdhu*bL)ZK}HrG~MS&nnYGeVNehw(+S2++(U01cIk{o zY=^ASzArigFi3#b)v(}j-+7YLxGYgX%=5B(pCBJY#Le%7UlIqQ7JB;2U@&GLT3D5S zSUba_P$|4X+5P1RId>lo_;$Orhv^q4%0yUQ}>wA*1D zQ249#_JV127O2w=0|zy@H!J{P7@XNJnzrh4xG1tay+-B}q_nhz$WS_CRQc>s^}lwl zw(7=1KGK*Tan@$;Bo+eRIuiK3`AXz!9OD8W0Rf!QiF2TN{^aDoGwrLAfau0%*C5$q zCn4M(QzW{+YD8^fFM4bA{QR8E)D+A1H8Po6L^lzQG;-fiA4oosCPZdg0F5gKii!b2 z;*GV!r2Bis*I$LEG>7~48ZfPPEmB^MknPPqezr_@^KSZ>ke~iT#5-h7y|YyDKJigr zej^{E`Vrvyiu!qa-rzqkK6~uOi9L4Y;T)aLKi_yXlnU#-R^BcHt_6FG@tUd#T}LX1 z&f8g^gV(gOSIxlY z1)F&m{Wg~RlK7qZll%A5aCN3q_!A;onnr&r2~qmOnN@p;QU|e(Wig1$*dp}Re|^9PLk9U|~UB)d3ks+^P8*gm+5WCWTvQO1qH1&S*6x(!K`c=^b~Z8 z^3s*FHp^q1dX#OHOo}*=ccy3{mYV{N>0Ck~1b^qb0tt38Q~L|!q@yuWeUj7iVv9G4 zU&{}>6dT)(oU_HV4rhQ~VMU(lz9rL)ZgW{%Z(%C@qeL=Rb6N=}D657nTLkPzYTm&C z2;@3kF9PV&tE=M)UF;}?T7F~%a-CG%1aiU7Z}gq)qR$AEmn;mjy(hoei4tci-ecEs zSo1m?$!u7@kW%&LDypDCX)Y1Ex< z81OFBC}~VRgmKvxcf$m!6G7ro8C3Cv&t5QD4AbG*F*8e4>6kId?|vt{w)iYyCYyEX z@GAYFLioZo%|ucB0AF#HS9TI@6@5bQ3ayk$CRL+{zXV~xPBNL>@e_U-?kDkY%d&QH zqiJcG6|48DGkw1CDs=APzFFxPulZqExG@=mR-ep12=?QyfLi<&2OF!VpU-o_{^;z2 zP4lAtm4}$usX7Z1B;%YiUq4f;8}&;71L zdDKFcLoqWdZi6$fV}c@<9bBJL*2^k+rJ~im1=(JnR=Xe)a@0@ftdkB4G0pwuFCzq` zl2o3EUZyrH6g|(tCefuhqFL5z5H^{m@%b0>`H=896fPcudvIGPgQT;Q7E|=B&DTZ8 zKgLGeO()9QOerjOVq+r}KLvG3(b&90Li)0)u}{uyHj&pJM$=g{rERe#j5-69U$YKw zz&JCN^~s}m6cYMy|FnIVb<%|mX}_-e`o+fL_-M5%dJIq)e>cyysn*Jn!nlK|?No&s^8{oT zde#dONfsK;*m&Up>3P^5OAUpCMYNn{9g!ASUduGVT-TP)_H}n*+kA)Sxkb)ZoDy;zJ1Jhg4DKmahPXkSPu!-ZIQ}0TBPrX0d+W2j&g@>ZXxX8_GLq zol9=*EuSLx;*x=B+910`fxc*5N=Rpnlf56NAlrLR>W8e$R$uPzSz{PCFK)${aP-uC*1(S`=Wd`&n=mNPG(iGjd!Z`TDPIlt2B6PM}j)FRe8cF6b(r|xU~ zfT8kz*7fy$HO^Uz?j-vaLSDipCTSL;!oTa))ATyi>`psHo?Dvr6JW(#33#S#qvFrz z+-W=dH_rK7vt<9%ZM3!Z(UV#bqls;_MXUzj{Z&!G)hBWi2FQ!64($;mCkKzedZGe8 z=TAA@aB>@xST$DUfZN%4$F8<3LKD6cqC||spZ;M(qNu(zkp0ALLPiWvt)J>juPyYV z(COc?Y`j6u8LJQ|TATY*TJA!K$|xQdQ4AHWEc4~!{*-%(_>glb8@W3aLg}peDbpcw z_8XD9WlqYJWy+24mbkLLCt8r1P)QA&@?B(u@x1Okv&kD?tv^^xVpJmzDO$ip zveNc3$9S+1ajc5g4-=%Lw>``qiz2*AgxPtTfV%?yHzeQQXZ~_Zu>XBZ3QTtBqh1{! znsa?rAz-F%%h2h)n~{QIDZfvQ@wc$#d4=Na$ns+yN~~U?aHnCmik?YHkRFySA>ajU zUM5-Qd#HcEcXi2aFXJs9UHLzPMhQ!Z2Cb_7emjc!-u!#I6GWa&#XjkXU$e?lgR>C_ z3cbadmQzn_NH)J-wO=vDXyqEQiv5Ck`QRybHY80?*CC0aHai&UtFkDE*y!c@W?y;mhnW=ir&G`3)q_^PP)w*au||)jRFxQN4lgkygqWL4gXqPeH<7M*!*Mjzr(~wWA|feqSSOEme1O55ScENBUdJ@1 zvp~&Ms5yawOKvAc&0JH)B69x^aIK*M4PxjLyS;$oMTnv%OqiT*vncWZqAyECqL8ER zUClW+(Cl~ccsO2KH0-+NU`16Lv0@_h6ir#gfr=x?x^fHqxc%ayR1>F|COUlJghU=o zN&^_1?UhddTN}TCYo_N}vD<^LhjZ!<|5NRN%X6Lb#waYm#UrXY>g*GaTs_~%OKnaA z(fg~%X23dq#HajSa<@QlBk*nf2_Yb&a{*899#!c`YYV>Z8|E)Kbe?hZ z*i%8`ve)i_imNsA_w)mAJQ%JFPYw{YLtAR#y_d;KR|p%&t@K7Z0cxf0V>9U3^aiF z-G3Qf_}dPsxQZyFrG@$l!pnQt5;7k2G6Vlsk*F1Lx@gY7p@qxjZ>~u9j!c~|SITG0 zDfR!neq8-~!&I&}vr*RzbnTvKjb}%gg`9j>FJzO#f!0w!nxs68FXQWcj6{XAM-zTx zW_u&8r=J~JJ&sFZGVSt5!NlAvq@La5zDNzeN}6M1+&CBAFG*10a#7WIaOvH=7@5jz z6SEO+4BE4EF~CX)Yf^`I_}-pX)^>Vd9WFcox``4N9a~`%FUDsil=a`%{h(oW%Xxg( zl4C_xtLpjirSN#_6~FddrvQ&OFEal8DI}fEkn!DXvrrB^J)l>hcvORBtA{K~wE}ko z^EDEEps3F~SjRWjREGOeqv%XXa1DG-Dvc{|KLn-(wh zn$8DySzN9lArqsDr40mM)!a`G1Ri0))twKJPRJ9yh^qRv1@KP zNi_}ftuq~IuuLnfV0F}wajpY&-YIn2v)+Gqvz35Zo$?MyjOIN@2V6yWJ!Kg_tdjzE z-;oi4aGX&K_s$PFV;vgh7xA%%1aEZ4rF?nc^TKy z5b4m}Lgp+r-xp~S@R1STiv2LE$j=!>G>KXm;G9>|v&iCMH(OK1$L235M&9P;<|dKivXu` zR0YR3_u!*8|A^*x!L9pKmUym7iey?Rm}`eOetv0KQt z!xaiL78w4g@_1qAaDI%P9U4`CFA7LnaG{z}*V@#Fn<_z4+n#kyp{%BY2_=O$jj;iiR8WA-lom7N}Bzp)657eT5Md_4q0^HKNalv(ZeG zPlwOf)IY+blYf!TbpO`49A%p1n3&u#_w)Dm_VS91xU}0gt%K=_`mLM(rM%Kqcf9tw z-23h6Wu{=S)UWL>QOMzC?#8p^kXvVFw68rTC-^kSdso#awLdeY#gmu3CwrrU-O>Mw zmR17;6x?8s0f1sju+>7MOkzZDe#(*huH`7BMrG$upYGARddAFd+KZ+rD|txs=IpNg zH9*gJBN#hIaxF6;7)3&8YE+%V&-hvy!oMyq9EhC@d7sD<-3<|-E#S^n(~FNv|KWNu z!DFnjH8f2qjvWdz0Zq7nVaCFW!5DHbwvs~Yix}dY%16QAI12qH9-597=R9qj!?XD7 zwUl(QX>U}OLet!>(Yl#(|BOtslNQBC`RpR+B$3?79o++f8tOYo-w_$X?yT~U5s=eJ zB!5#~ojBA~R-Bm!xBw7v_QGqv??E$wB-nZ!|GL4foQD1rC7Cd~HHrN_kbpA?xN*96 z3fOI1#$%7JjmU9Rn^Y&7b6+s+)<_#>@j+-xi_(!R6k=jgGmY>6qT}=1i^pG~0>)M6 z9o)U{0q6S-(W8}HhYjZy8l$r~n(a@X=LaVN=cj8$xLPbGn(2QKdY*BRBHXz}FdhHN z`xKfYC;b>OrC{t!Hn@i?M$x$I zzOBnAARxfUhi%iIJa=2gHHKelB>;ZIbdbc8>Wq9cjrwyekJt~wNI|MIPbz~X`(hP@ zg8?x&;^u-*ZTOy&1?}wSl!!CzRgIQ@O!1j4?PM*!l5BYkvCY%HH%j_8=Kbpy8Y5uK z+Q<^aF-i~=@V)E#ESlO{>3P$#>sjo<CXG7N`Z;ZWAv~!hYzURC!D> zE1PR@cTV8Xx%xoevvlX( z5*c-XRs>@*3(LyGpGx}Zrvi%Wf&B`d6686za;E#FKiK(#&}hngzz|9bt{4zcq0@fR zDuFPXAheiDF~rWnItf9bpXcVi{K+DfgG;2~8XV#K^~0vmXkuQ~2P(WBf?pZFT+iHj z`StAci%JQviLd1=x%+?q5cvAng{f`n7`V0hkS^x{uNq|>7Bj2;J9ac|Rjx3M?r!Do zPdkzrKn`vZFsZHp`Kl#{`Nde=&MzZ;{F0i1v>{``#R$PB_x*w$9uC!7l{YTZye?d* zd8#*DYs(SIKmIPf9{Pg`y6Ac~d|vY9t8@c`aek`2_0HW7@Aos=XMn|L-p$2Q#_s1-Wf^^ zm4r8@#z7S0M~^9o z{=4+LYMi!FhZamqO)UmdTygTH-eC-4YBA&Vk;FdUru08Mc1EQth}T~J*@v%4+;lWk zpe2n|zLssGg4Dr2m+M1LdyV|Fvu19LlN#*BJ!>4}g=Tm9`2e&;O{fkK@SGdi$ zK6^Z?H8<@I?VleLBQ`pdCNYCGnxW+taMJBMf1qi|{$qI!ttTy;HP;~u^~(jRb7iDvXES!uer=`TOVSoD#(LW=VV$^A-0 zOr?3w2EHgj(o`N%VZ`suiU|0vs=qzayKz3;cNeX^{@A-g%219X3)+^K2P}h;bR?!m z7UCeK5F3Mx>{pZVRcg2J)bqozMXtM>vsYQA#@V^K7U3+fd{jxfGjvQy(;|xI4=zo< z@{5U0vg4hVFW52Krsv40r&m-}HOiKQd%oIY+lSm;JUuPrpZa&DtW{=yVI9#?WqG59 zl=NQyMZlp@qF3rZO9rpCy(7A53=E56f$Z;3wmf!kvy6ASTlV`^iqNyePM}Sl$4$;e za?O=V_i0EYk+v{w8V^QPdRymD=;?MdTa9)#R~Yoe{l2F0P(yzg-u#2QP0Doe-(aw% z_>5H`s8P)fi#IBrJm!iF@)`J_bjhXZyRYf{zW>y^)V3$es=A!cyieEr*PIeEh{>^5 z9Gd6n=cn@#B5&k936;4mZf(?a(p7X7-!K)9_*EPa%|;=*DQ+?RGJgzR?V$&~!|X^5 z%J3bl@290*!R2z6Y1mIglgX?rieQeoT*q8AL^Ci&ek_aX%zfjgni>i%w5q;VWkcm1 zTzV=KyS)nl@*Si~Q62YPPn7{32DN~-{qa0SSKq)Omm_t^h>Xpih~UQP+W_z>418Za zLIwXT$Kp9yP|Qnk6neMl*vq0b`($M3Z7#e0?LA zMfjhg17zg1G~A$bAB@V*nb zQ|#Ee3wz=febF~boMY!r5wPyb3;(ZkeCG>aY`&p|Y7i^V_=Iczn3JUrgkUPL{m^&4 z<0_r&uiAyV)eIjgz;(8XlyjQ0n@2~}I+RDuYpgvk~V~QN&jNpwcmZwSr;HY0>$$h^tgJ0st1Zo3FXbX+V{=R<7ZKX+IVVP z3Qw`h3~Oh63N98>LztH&4E=fA8cs=;ByV*}>c6z#6ce$USTJh9BUJ<$^@HLub82G~ zhX`qCW1Pz*Z7Ot$VjK88<4qzXSDRaL%SPmF1nPxo|FB9qo$(jGuP0>>2t}}oqE$B5 z>}O`!9}kfVGlhVQzH9W%6d{lcF~I3cx+2Q4MKoA1!AQhO(2^9qT=EZKR*<-OL`(!Y zIA60$;r&j8;t(tGk=cz5l{D6LK;QtVyGkcaMIFwn6kjCGl;FCTop8})Z_eIURjt7r zG5OV?wsEd_<4bU_^6YfOYDBTobi0}Uw-B?>sPj}k-% zVqm27z@e$7SZluUhmc_&L7m;N!f{0db~|{eHb#6rXJg z*T8O2EF?mlMFxw;Bw_-6fTmP|Fj3MeS(#+Jn-&?2MAbkGlchIW1qUe#4xw{410YH> z_wUyR3H;^dBm10-zH1_YAs~Gk&n1-N=9!w6R9TTo5;gM~o6w3uo%EAF({6(Jvsx~D z%`Y%@#^@Prdq(w5D5Yk_>O`ApN6FaxRQ>k!GdI>0ksubHxkRZLh~y`V zyCxL8zcUqu0hgRlTgJ`LWzU=VDk|FNt(eOIQuv#_PZ*mJ&P!{pw?tlDOTfN9wN=%! zSOvhA(i#9t+vD}hiX+v2zp(m6cVB-hNKAQu?pzSKn%SYa7~{~w*WXD~5D>;y*(@)# z6PbKEw=6q0j_Y1W9UptlTadK;pw2X+qE8gFY9cO8gV-^#Fr%T>B%^%8V`}r29O-W8 zhZm{oFJAbk_Z1o}jjGa8BR?$;D%BDVSFRV2_(Ev5Kxh0rJ3E6Ls*Em9FYyWfOV6{8 zE03zAx#GZ}X9A1~Wur}CwO4%P435}@jI<|yPNE?RA6-y^eh4bbO)@j5eBfmANhatd6<7q8WcIM5kx?a3|kn&P+dYVCZ?B#P}N^Hom`O1fBTgH=$9 zk55jzZqDQupx6@fiUV*Z79i#LM*vVG(!IWZ*sXsL|NIZF6`W2?Ydwp*;ftgEy~WIA zq*P1x-6_5dL-0h-^?ffAlB^frlz?2Ixegkh`ece)M%V?%jRN%~2A)bOVyFqLo;Po@ ztd*@YGrOw~uyP%A4L_dp#f1NSp&~865h1Uhvi@>ZqH*r-2T-UW3zY%z0}Tr}5IV_K zGD&%BDZ#a*h>#Nc0W#>Oq{GSMDy8XJDa0hE1Ra!sbt*M)qn>KMupmL+uxtmw?|{Ml z@NlHkFyXi=gNw$3HGI~s<*iDeB-MB7u+|y?*tXwF6$W@KTq6oI9}~XMG$v|vUgutJ zWMm8WpH28K*o7$SD86|o`gI|c{Qeq|G(>af%$Rs68KwpB{$_S)3;Q-)#H7SJB?#Cl zD6ov_dw!RB{PsE7dh}Z%Rh_KJ1PRDu*(DeN5)ZoA{o}9XWUr-&C*o5R$rF*=-E6v& zk=FVjCs+4SZ*`LJ&aBs@9SvPiS6Tnri~PTL9JNk(lY*ItBY*RO01j~T;#9~JOWwYb zJOz%keB<1*p!Fq3Bc!UFgUxyT%pvYrQxgue6}nIpER2>DRL_WPNuMlxN#9i(qUWrk z9vJZ^_8a_{6BGTmr97pjT1Oh&U_WEzjKZgOgWCQsl2%@e)Kj&r4FeEc`Cjh0MG|rf zt=JoD^@RI9yutHxS6aZcEpoMF^7Tu_i1+;Uqe%-weS2L#33WQ08*j3vJSbKY1OJXSFqUS9a++g$0YwV}*aa=jgP8Mp-X(4jdP zRWC&21>)V=mVHa1A`3&qsYy^gY__M%`x2hdL`%TNCdXYW*qUD9P(K$oQ$eCu{TXKA z=6|(0oIDC_iyMF}*W3)iJ*qY4-P*>rWby2%=Roi0-4>gusLT4qu_$Dk8p7epL7yD; zwXk{19U;AA$tJl|T|{xSr~_d$(ky!<&gR^VC?G#eIT=V1GpY`oDk6oHY{qSAI%efdV!8k$h1Z?F7o_2`6tjZICKbFXA=)tnF8OyZoVW&;@aWG#pryVm_Qf5evYs}mywum=K~} zRc*my@ZiVJ)82Vq!fh6L)u*v(KO*yfJ^Gdx3-&JP?-lTHDia&3F=tJ-XoQ+$mZ2(` zpB7=#BIvYpSAn|a6d8(Zn62^7062tDgSIXERkSbdK*B88(3o{?{jJ$~0@TfUSo3Ve z^*c-2(#bWH0eYrG62|_&0PqSA^_Yxa_`!+GyOxEgZQF;32O?T7m#3$vN~wp3hvVTO zXFqEX`F5?fHjf>1d6iV6U7RWbd5{t0Ny&_S(p{qIrb%~>_~7sH3(rrK5x zSbmS>RrdQmG05@>A)H8(RApt%$9q3ru`GyLea8vvjR~EA-y(I(nuS%&E&!z}Y?XLI zrr~x>Nb%HKWH}SnKtw7PBa5Jf&Mi((mh;8Sr_X-!lb<}ldV&lZ(D9I^Ujz{%F*7hm z;KA9>4uD8TGr}+o9{IcPzWXnK`!~1ucM$*;NnQ~W9d-sNroqZ@`^e)HO{#aKc#bAq zcWP4Ko%1MN*9~qYu89Z|K?*I?=!^@hla6alkQXITE9X=pEs>g(nu)(6ACy+){4Dtx z=Z%iPg>O;=*v zIXQJKHAPuqIAy9s>J1KKua-`FGbTQt#~RZZ!CDLWhzRGS#FHX|ALu*TQ%GQ77XGY; zvhtEGaKk5T!*c&pETpxTEOD5$Yqout&FWO4PtRAj5Z?RHw!(w|^y$-QS5KZic~TU` z`r#pjkb@_jaKUE8H716>eB1f>=O>LZ*5#N#n7%lXd~nK+WJM(IZFZ;f)vUFGN~r`y zW9qi!G{sw%PM%uQ3>l=#evAZ&nDDd_a8Q&!N#kM(&>RtDedRdm#MdhDoS#Z;~*u2X5mhzjYLd=sg zCZbO;A4vQ&W-^Ff{%Qw5Jz!b0XhrFs$ADM9x@nrPzWniLuV1g`i{)(I_kA23A=X8i zPl7B+48Y92C@GH4dr+iNb;GdPZ9m-K{qi@zdVg~*Qx_s|@GdaxpEtx||iLx=wqA}mDC*vM54RjQyBVp*p#M_>2zc@qAIprf~d7)-DC(X_f`~zb3qWKsTk)q z%+6zuDj;#cWP~L9F*&y+HaG({*4i*(In}8a8z&6EXy`pOC+Lww=WE+n~5cKk7=F62JNf{F+kVF-S*5jl{r|$UwB#-FEN22j&o?wboZb8xwk~ zi$e1C3K1EUkQ5+!_& zpZ@fxFP=XK7u+zoZh#PrsxroCQd(^$pbJCr7#L zMD7+sklbCIlZNT@nY(!ZF<65=E#$O|jU>X7J_@=3{-GNvg4qN(e$%KnW?>2A6C z42#K?r8I8pt@C7FU_Mze@tk6el>ChCcKi12+Zf}E7cb6EPl>25D$_v5r16zfr1j|UC(RyIdRk7JU7JE_;E0hqM1~-Z zj`R7vsw%e{(Zk-^wr!Ozlv2Ta03<>U2$7XW3F?r+ryb3kn*HT`swq`a0*&e*fX__RzJFu$j+T(co<7Eg%#{!PaMW5GP2KID@%%Eh}7kKuB$zJoc-j zQVRId%bu=Vetexo+^3@K0(k1yO*Q*el>mUm^*@l_tgpq3lzfKV?Nozgguxi&C(QoY zy5~3O2}z4wnKfmuP}x53y)c#_qF^98F@SpSD^u7s=C~|Mxmrnhe}Au10!D0!xOhI5 zT4#3^Ckc=2JL4n(%G<#_f(-KbYBFS;Q_R%&o$Q)P!+>~Fp_JD-Yr3gIk0}ME62o0q zpjk+=*9DWBnXHAxS%DN;4#QAXrJ6JVX6HSB>IcUy#wbiO&bjS&D|@7sn$PD1BRT%i zj@WEjmX;sexwDUwb^6qD$X5g(vVNF8luI1TSyA8wUCF8-?~a~KW9I~R$=7E65GFOF zh**}TWPq7jrl}0W0GOxcO=&vdv3G+eV}g-mM%d-7f_ls_AWrgiWieK{!icjs%M~Ij zKm<^LT4AA0sf`_a(d-DZGR1PSxHvtN$1EaD*oq;g%k#2M-84={*n2NfzH`p?gM`B-Wh!K!vX=o9)QBhkJ5S(zBP~~^NhK>! zoS=LuIS=Cf$-0@ztZ+^vr>AM8FliK^X~lQ4%GqzqGB74Nq({*?ah%Jtq;ka5MD^Eo zEg;MfoVy&(>ssenN2iHe%ILbTMRZhpiC{?LVM>A~t1$%!@=+Cul9kt5hcG(4#*A&! z!T@8(VdZ0JE(c07%qkU$`c%v+%l-q?k{2x|tV|f(riu(wB#{kC-m)CK$X#WnnYT}+ z_Ou*rh;hn??gvU!#BXRjVZg7eN=m5D&(DWpI3P)jlTu1C+ZyOTP20ou_9xTU`DW6;LT*>%ET=Od8bgQva7+L_ip!63Q#7^tjjme_ilsRv<5G QBme*a07*qoM6N<$f{@4}ZU6uP literal 0 HcmV?d00001 diff --git a/python/nano/test/resources/train_image_folder_png/dogs/dog.1.png b/python/nano/test/resources/train_image_folder_png/dogs/dog.1.png new file mode 100644 index 0000000000000000000000000000000000000000..8cbe26abf2a965bf4b2b652ab5a92ac0e3324ecc GIT binary patch literal 217431 zcmWifcQjk?AID>d1~FRHsKjWM+MAj&TC}w*YFF%4BUTkPV$`bIqphMvXptJRT8e%N zD#VOYdqp)?&0l`^ocxiTn|scE?sK2#{dvD%pQJ|*449d&GXVeq=6i-PGXQ|ffbt;& zsVTpx`x`m{r~rU_Fl~#_qMmb`25*}zfrNjJglV+|xmieM)uykN0)U;K32H3$_No52 z<5H3QkJdm|=`8Axz6!0_*6RBD+j6_@Zpo;-HQ$H_pP|1?HhW=)%)SlAYDY!fho5K9 z$-|mQ@i$(a-@Ht}oVh%h=sd%p%B1}k1GC+|QL%&Ly(vTM=jrps^=ss4PUlkH+#FFO z>dZ+)BcdcHw^BwHE|N$@V3!YYg~)EwA->zi64|$Oc&_B|Ldt$7dlWvPR(zksc zc@jMb1B~Qly5tc+!d~Q0jZz&a_h!AL$UA=L%C9p05<)p>ur1fkdYU8hw#e}L^L^Hi ze`T6yueqYhD-;Fq40uyV10Uif$wv8ar}qs8wsiO5!aXok^YWA z4MaNkVzIVtDi0Ea?ehQmURv^v4h#%*VD7e&?(y^v=>ub+5%mG6kzSHhKe zM!?~z&r=)1&w>h0o}@0gxL7aI*E}b_7s1+l##ry}f5Kch{^DR!o?3A%^FN?jhOUmjD&HXf^~ zp>eU|aIqrw;$UKUco<_9?1j$9Ru$xPK;Kv8m|YxQ9=!NBOc~ z&GQ3?OAOEKe0y=l%HrapQs+O?Qeq+j4!j2gyv+`_xd!0wlPPbU^6ogreWF_9CWQ{N_4Z7TUxRD~&G3ld3OSu0$DqOjkMHf|wlWdx>O=_T{JQ@h3S&Zg0Yy5lClcW!1bmUc%$?Q4P;k)G=nA z;YSz5!c)eLy}i94)$@Ppo#Ds(9%?K%G|!KZHx^e`@`+GLz9gA)Kf}WhjebTn?fv;1 z6l!B{Z%7NsSXOq^O8$h#K%CJB`_GhRJntY54~d1ZX8O`Kqt5;a3B6NP{0tA``zlsj zvDKRVXlSbKP0@t9969Cg4-t)a^-lHnxNNhZX~y5n#}PyCQ2lK=xl2?4Psw<^ya`MY znD1Vah92kq-F$C(#8c4{F-WeLNC}&H_GEPfkC!v#_Vdd;4pomj{`1bt(vrTB`&sIP zCyg?WVq2}XeH)DuMopXV?@Uy%7&F=w#PTM}`yL^y;9frH_O`aOm(_3CyS=rl@St|;(8pk2<9Vc1f{FXPWCVkzDk0wO6LkrT-HgXH#qzK-)I7S z;>oH)4;F{eVh+~Ou`TPj3N@?=qXWP~kP6ym|bZt@-r;Q0F+}`@8 zeV{lKJjH9&YW5BAJqiW9rNZMV)H&rR_n$OOSjmr+u#WPrEeW<;GgFwb@)CfVC9AB3 zl{~Wt;OvZSY#(^puZEGX{2071hA*-VZkpM5kd>30Xx--;g9i=`)>J^h52~z@T9iEU za$9P`h&x7NbNiG<;orZ1*m;eyC%u2HjL=)uo)X}=!2+zO_(0I0C)Nt{Cb(b?&uJ#% zB>f@G9#wXIxoD}`34yFLG5T5iHM{NSztGUAuyA9@talc`6J2-tw8_M)W@S|-YEdY9747XKZyQMztNkF&CTPAYH z@Oh%QTB#Y|vvzk40!A*=Q8K+w+|m9v*A&dhS-@<+h{SVTp%Uc~EANPPP6gnIl^&{; zSGL>9#tNxrfn{K#+nDvq_>;XdqGw z+L8mmsuCx!ypUDx`ds}kUpj-_R!q+z*tDmxS#)p4WRgi=8-n-bGyFB!Fixmg3XimB zP2W<-omh4JOJ?la6irY2#_q(f`i6J0YzFZm{R)cL>E)TieMIU_$Cq0szK zYK=@?F)~aH70o=pA%(<{hO3UqXWsgN{f&*i=Bjdcst1+b^$%L&iYELUtb?IXwH1Y9 zxkG0^xE6=Zu)WL2m`z}@Gf2PruU04&3hsrmf^KH2#C@Lfc20%RTHX~%-)c<&>i|hZ zgSVRJca;j6q10OUe-8Tk`Z}%+Mk@2wTJh&MH7V`R&G{*1GMaxV#D!vKR42jRIzU9+ zH5vx}rfV6{dtey@ZK$ZzAb=zZ^?2R?t?f;4aqbT)IL?u|m?wr02%!@&`?v*L0Nnzj zh74iR>n_PK&bT-kpq{8BRL6^tNh(Vpba5XB;vBK~o)R%=LuF!bhjdQ!ex?T`xch-mT!L$6hyO}Ne`+hDkz z-qA_VmdEo8E(dc*fAaJ~XTRTs!j_@+Gdo|qm3uj1BSF9~uXq`Qu=6b4{sj|Z_Ju79 zcQ7V+kAj?~#vs05OSDk-OhYCr+*Vb-S1>tYdCP36d4OpEUg1%Y_7FdSSH0`uJG?vB zjOodXot@kE`>G30{eDUm?G25DC}FHdOn}@@Hg0v|0jTXvt&#pJf`nrMmpH251WiDc zXGYlP6g0eCt>0ATYSf?WABh1|6)}!z=e-AWH#j@eDp3b|5yn16TRpwzFAY=DPW~_HxqcdB0%nsnN&!oBS7_aZPgRht9PREsM%^1L?utd#(=asVwTbi3A^IA(j) zMx?F-RMp}J!_8iyNREiX+4H|%)reJaMp9B7YDl1`-8Qtz4e6gKaG3ukA~8yU7_k2` z-~7JA^Su>ZR5}YfyTHh1qHUc9EH}JWFVVsS2(T>83Co`dc!9Tj=n#4ir}ezcN6`h5 zk9(S&W4w(&Rq=@X`x+tN~-XlJ<=CRy&?WD8A`Nm~M%*#5Md;j#B* z-qx`vh)Bs{UqO1Jh1uF$-*Cpgqz$3v&_{xGMBi4_(yt<{jHb$$Nk;az;+U(#v`|sAQ5E6MfD4$V!{H z1rp|H&?hJi?FQ4Bivt%@Q7FVJ3Dw-$84(s1roy)h?>ncg;yySGVJGAYfRODX=;Tx7 z%X}WH9`agI&cMRbKx+{ki!u~sf$(!v37QB7{iC3?Z(++@>QpIKcYK(iMY3wJrq@1VPcuP@tEM6pGNSs zCtQ3PdcqQFwAN^4fyqDYaCs>H173&pfPVKV4b3va`T|h(iciOXG*d1zy8>iwg zdTh$i2u5G^dNa~>ol2Uz6*Jn)Nh!3c0N;SonQ>x|+|ig;c2lO9U*Ez@KCO4HuXm*x zYoQ48()6N;SVXkwU_amN{Or7bUqI&pSZ6`H2NyEAt@rL z%#Ot-^y{YbfAwjaaWX^@c0SYm&CSgXqlYHTgb3z3Z%VnN*+!*7GQDYU6P%9!5+f|1 zhlRBeNr$_09qJl)$RGgOgYCT*s&+wv?w&R+q}Oq}D~QjTFAL26nDg0kkJG|PKd_hz z_XKnJm}&$sP1T5KmAEq2O7!+UvW0l3%>+!Qz5M!peVLxA>do~$9vZ>-Ho7ocmv6%A zy^l}r_(!~sQ-QI1gc z!?iboq?;pTv7592wvCNO>@fLke}Dh%?8$u@8uPrhV4EC)($g4}m`DQ)L!-fp7mD5p}f){3S@ zt>^V_TRgL?y7Oa#(4{WZ)LQEma>UJ(#QR8BfBAt9>hE_Wr<%eBeuN*-MWgdRG2ni++0>Z+%n(G^|&j!1(&FHJ=k2%3T?;@V0ALrK0JM1rb=7nRQko5EvOVEhdS#s|ZxH#K1-n^nKTROru%8yIPu9c>v$h z>0P-AD?a(S408RvUwWSqlqsu0n@T2%A~nKspVXHz|?YnioC)%zxzhP znR#xF$tyd{3J-V`0LCfqSFO}Oo8n4b`X6}q;e}kB;ZNLyt5zKswrSgWgE;Dv5Yndj4&=^W>y(=i-orn^AIzVUw`@2uw|isKm|vUodsP(ty8k=4XCVDYL=^%iC@dS7odl()wy%{Qw2TCK4bGG^$W*zM0cf-t?>A){ah9f05ln4#bkfgbZBdp@ z&{{6gAW&Y((hmpsvTU!i!dIj%kXX{(7g!ueF2gEZlNjkN(3n-LpD%T@0f6NUGbTMf z7;#wrYmWMrT+^qF1kd2!0X_OL)?U?7K0#b_hmgJP6H02yjkF)yoPRM@j( zc$gm6I1K6`GpHI>mcI}PhIvuTz>Tnyr17MQVtUs7KQ$@EKVbQ%D`IAJ(nxJLM)#*NT8wn@-`;DuzX=^Bho?^o- z?TVL~7lDb9#N!(R|8;kFN2=a?8z3i3Zmn$xG)#F%o#0Djw?dE&d?kZ?`7kK7MNOGv z|LQ=aAVp(^3KHlL>(TC%s-43KC{s_SSUu2GI4EgtjO|y~cuk8!uvc;yeJ#Bn?y4sZ z-P2{y=~>RU7b@T>p4i*aqqlB|R6Y@XMkYyZ;U8d~;f$pC$73l%YRB+`wQ!0(^ATm? z#Uki(eH4Ku=6aeuZ<_hR?NSZU(#~MZ&n$`!ttRG1hKRLC#tupJN3R=anPqWD5AlQ^ zDexyA-F!wjNt%247zimJbJ=dK-D>U7C+3z?ImEYjx{l{^uAVJ+9^FUwt6Xj(HG7C@ zE(K}$0rCLY^^;{@2o@(OThFZqI{^E;L!TVf1UdFNR*=$eu}s6of)$h)m3?~yfT*Ic z5#=8_2V1yE5vmVSS~`3gboqu*nQIPhPSu22Nzv?k%39DMdA&d=_#GJbHb(8RpXC(* zgmti>h9L|Tcc)a;>eUnjt;;%m*=BzM#@q6_Fi$X_6oXx94QSLOOmxj@1hqt+Z)z{+ zT)D!agGyZ`IgWeEl&#alEE-C7(D2kG!x|NUZ2ZNi;vQ0YO7>_7_U73-%lcTs0q~Kh z&4j|*sKfAJd3JUvU!QN+T~8VEUg&nFo883Pz}}xf(s7e?3H<#Bu04a5VJwu>T<^3H zVhq0-p})f_?%E4Av|tdHK0vk+=CKje=2C%=MB<#{cvwk3YW7hv_+|fIsE#nC7Q@E= z8t|n|yXm2pf_2jqUNC)Sg^601h_F-E2;96$V3yc!IUXW;Wv_g70Ln5lmV6ttnV&uY zFJLzf-c6#*azuS+0?01-A{(Z*AN0B6FiT}=k5K);_#>ou*11#+Ss#3K>oxk8#K%@> z>zJ&IGHsQa(eoKCl_;{&ig5Ltqd8Dco2cxp^%`oyRBUR0P7Fs1syZBdxsYlm$A3NP z+9Q zn0ZxWT27QHy2FQ|wZh)(&8@9gfrnBmYuWGPhCfDd`!~%%oW_r=;nFht)a?uV85 z^c$LI;|}MRLtB{jXYogqbS0JeqxlyvkjSPP->ZDl8B;O$KV`>OeQFV;a=P^?4@Q`|(FI-uvE2iO zdxk%duYHy>+%?WTRweeH7XXGL8`Be_L9{)C55r`*j^>%Mx-(K#p z$pn+^9d01b)&*`n?(OU2QNTRtZQx8of0Z=f0_spVnmTfIII=-sw=JI6)p0Wz@Gl)ew1O!ulndDE~}?xElFzXx1Lp- z6-{T!Ttt5 zC0<4$9qCmawdGcK5{$Zk7Fuy6Apj3xW?m+HwU%}hmd@-+hU_Xm+;-mzXu6^rX1qf% zo)`kWy8K6GdLJcJ;{eRg&vWQbB!{?lPI-r05nju{&Q=TkoKtbOVk_N=tT!~&y_+^J zE;=N(g6RbBGbC4q7v)f%Y+hC^jS(kgHN%K*nI(@0STy!F_kp-Pz^uU&UU>f_zYwj? zGu^HVZaSJY6DbLuKfG%>I4W3pu8#}MVW3w9x2r@ULKhRL%Yf8^GD zLd2|)5f|-GPW)^!RWcmW!BT;Amk4`6RAgw5)7kIy^PQ+l2_~&UQbzSd*V`tQ!TGu6 zRz4L2WLhqtDNB3NOrKPLxqiv-U9Zso9HJ;NZ$^PjHM&TmR9(uE87RWijo?~x907ft z$j@wX;JrtC(|{g8^VEk;Ldz9R@_Y<*jaJP*djy;5SR zMu_cX*9BY`fH_*Ep7ysEUtXL_beRwYay|MrRe`p79pRY|uP`*AugK_EZ2yCIX1CBP zKe&ay=l>NekI$$H8+vm~_H{moX{v6b4@y04DK8<%=uWxSxF^4n)IgTmv~Qo;L@jZ* z1lw>@o}Fgs=j$6THi}a!#D!pA?Ljq6x7HWn&Km(``OrjeyiQG(KwO+2ZG41wqk#p| zKu}o59~$(XkPq}>5ro<)&fe!&)Xf6xqQ1-Y7S2H8C*kJ9Rc-FWi{{nhLFydPVCTnM^>ebt8J5VNCL4cO1j`kxM>0X;d z!$Vk?mkF;5J=+GmID$p&87iY%k-@#3Gl_~#_r4f&bla+nSg?)t>TMY?lUx!o2-}+d z!blABRUBv_e3xuOI##)1<=>I5iA{14l2*1C=R)NfZ9Hxz+!t*3rVo11JoE)Vyw@ow z*dAD&&F;bkS$+~xe(v|h<#MsC^T!}I3&&4z4vzN-<5o)Ielm_uEu?kaYBe6H@$Hw8 z0e_+Dz;y;RL4~2lvn2+q5N-A4mckDL*kb8HZZ3m(%8FP9 ztJqi9`^zItRH9jctFzF}u38c+_}T%JtoJTMf+HG|757`~1B`aru)nr#5v=@vS6>?= zCSyC3sKuK(;?GNZk-@@ma}1e)Wv7oz`FHz{dGUTzdR zRh!+-Z^2l#Bb*q2I{OP=?EL}xgdZUUi~c_GL$=L-!}`Q^_E(DRenmPwCmT@yv*gQ1 zSP`3WOIumB3%4mrXlnX;e7Pm1NfAk0kr#Cr7n&yxni1h=@^|kJR(iB_hAG_{xL8do zoQcT(GOjv1S5$#7^qY$}IX))t=KVXjcJTeW@ozF?S{XEB_xJ%vCXTHA{o#KFF3{&T zuf3P`pGwJ^slMzaJf6sMNDG+EO2}QR zkHDtS%=q0X!4ru>0O<(R00Bq`ms|r)O^*oeb?K(8wT)#2l&#J4*Vk(`Ho}j*OmsZP zzcoR%Gu|;R+x=0zJI>h^BdpkDqrz1rej|B)Zti#UPxRG0D%DxNPM=*Rr!$fmOwC!^ zrkG*0pEU&p5+Kk%54+;X7o96{jyQrsK=LxV;0xiFbD%H42<-@5a1~q5)A95&>K2Xt zEMo?O#LLIYLsM`R6>Bk`JsMClbltvbj%(^}@Hjo7si{e(H|r`O*LBB-bKnVE^QQ;+ zK~7M070120@5wOaff1TB+H*YZ7#?9q-%S{L#q*!GE`1zp`oqB}tP$v7W$|A~CqHo1 zm+NVUv}$~XRUwh&QJf2jmxx9di?!0Z64#Ro_FYH$5K4*pb1O}5Na^| zaNsUW=p`%e%yKmvwnn;UE#F3{)q4bj3b8SA6bn^xWCklIa?2!iMMI%%PAF~T5SO0X zHcFH|o-~#Uyyuc?yx#)XS~I&w6|BBofgyyIq$jR*FaARe4@#sBfd<`neLekiPD6UU zL}86HI|3T#p9lY}Mg;~2;uT)n)z?pX3zTSdw6$gQhhTkzYZV^ivqw9YoS>})DYiXI zuzh}dbb8*A{S1jz1ARKWUAHwY;(Bg;a?`KL_Rhg|J1o46tW?ZBpXQP(Q=Y9O2yL6C z3JeOAJDsgwb`Pz^B=?BJu{k~&t=StAF;(IE==Sdt(DgjCDvZ6Q*14m0hAgZZIxj$k|efEfaj=?5OPZ{=@@08QjZ&I!D;M>~6uXODdoF zS+`u+8Q94IZfR*&a~~B37!kg@Py-CEJ&*O_ubqj6iUx|zbZlNo8cetu0c+28W-RofHo~! z3m6TF`@jTX6OzMB*;hw{Q~tXH>m>QZx2#g?AkmL#df zd3!|?dR=GxJ<99Vn1#tZ%rzJ|Gb!b&|FYbj$d2CO{>BI`sG=D_y4tg9x~32-$k5UV z6@9Q~Hr^_?s#=iczN_V>Uw*W`Wk!X#Vrt$fNY^*spTCn1XI1XUcKyci|R6${o) z7Aybsb*^`IXJ_-G)-6hLD&|C^5aX!j-c5 zR&M+`K2tX_A9CfM&Y#y1m$O1q)};A)d*RX*{SoE>lid z*E$_d+ACFnyDUS&$1pBWSeg&NB1i{rwz6_a%*a=Xu4 zg4c=KUHMPjk4@UvyER_M4|ujfQ6#SL*#sl+-SLjOD15bnTOPoT?$?U3M&D-jfqmz> z=51%}BgfD62u*<;SFzRpp0qjL_Xw26EJt!?YhDUOEkosc7DqSy z3cD^xGQ7m~7bjhS;F`OwQo#x1`KAk2~e>BG|Q)*~f?$2eUSM zC~lZBM)P3$9cS^k51zrn4MRO4#n&Ej0a{;Pf6od5=^9(b_kV*=KMz4{dD?dwgrF3@@)^>1rnl3^*jKnO72YTxbM8MoEY zV01EeIsLk>I(1&^Z*ZXC`|AGfdFKyY-BJA2m(Q(CQu|qG-;$q{+NV(vS3z4@y&zbS z0*yZd1t_{V%4RW62%iyqEpGwjPP|2(dOragXZ%B!ErDB6_qsEbT(Maml)GwjYGQr& zCI^&y)w)j{bIQ>*ww|B>3X&}G{aG{bsG6+gdYw9})`Apj?Y~(ieDnCX%s4Z=6 zrGM}P(}>E0bO=a>u3NAlBnBoFwO1QQnAAtBu|Cap zke}_m9%?!Qu*5VnjlS1i>l)r;HubW}5y_EN28lrv zJ|QHtq~7X5xigvA@vpl2M0?dH3(7p=y!lMWnf`3hA=~QocBP>1{h9NyTH8+E6*I}! zjtjq@_KSLf;k~TKtr2cZQd5dT)8(pAV>h34?8Ip8f3KqaKIO8Tc%k1lbyQO?T1rj_ zN1^h}%!0>PNfXn-O2&~ZPcQXrE?60SM1mP^`rl!;FR9Eu^=ROU1yHkRl@%POL!UlF z2G&hKL*wLuuTk>1ZV4C&h|pU5D*TxA(vOzum;NAc7z^h&oWr?;6_yO_%>y*%FrO@uQHDO{PBR3;YIxKwJrH0Yf1zLLB?ot|-xXA}}_0 zZd0Ldz3L9Zq7{%4Bkx3Hm2PkKfk-K?$!_oiF1G`S0YP1NFgpcpbQ`P{v^z zzF>BuIs>AHBKyN1jNKZ+xE1OSGu1ukU$YliX@IP$1#V1f{T8S;zmFdX9L~}KVrq%= z2kXz#I+wOC2QMt%(gTF!m0*v31o8Y*gd|d^fipfR{g687o7(!0i)y$oc%hhwN)ZB~n?!Je5uTT_DE7!&A z&jkhL!{I8^rCib2oeEZJ8G}`DH=t}g1nb28f?W9jU{T4TBSP~AtLZJ+S7Y>5z8|dM&>6({#j6!oj)Su=@OzL zOr2lViQ;}?^p(EtC|BGDNLcS0!`4Ai{)nRw&9662mFeCM+}0B|@TT7!0ld@J^&%vm z&_0#LtX_*KNnhX^jQ>7p{H)~0*vi;GQ+?DSb)8{7OG1R~GX-wBz;5z>mq4Y|gLkjB zgD6SGPN4mB7WO(y$(tUlw{2tQr?!(bd#;nndam?h#_{rQ-r0O`Pad2~G(9HpQ>7kj zpNBn}6k>qV0{FTE?f!{B_NsdxGzl_{GtQ3va0}v}T0fKKw@YC_=2CTO$s^_cJs`vA z7*|6^K)9}lSb5pV0|=^LISos!y_w_xQ-qR9`bp0bw^P1)woZ8)eyEmmPSqMSPWhVF zSFxcyTuD-+S}6F{Qf(vG;5+Enpj5bKK0IhQUXW}E+%{PYBUm}7DoCB~)V|?+IlKKe zQ~IF3patyLG*ei3?%TA54)u^b{MP~a^J4}%(70@cR6cZ_S5jA?E`V~TU8h)L1%7>+2XM7&=`}z5uW>0~?-g;8O}BI-p%+&%Oe0oK zDTJp;z5+K&(#W#D$l|{nwQ}+O(Qvskp;)+{GO}{tfRbQQE>!nYnSu*Z46l>c8wu~` zDcLnT7&~(h)N-5A4Wlrz8O`_TLqSL$v69R826GGGM`9x%o5%`+xrCb2utgaoQ_lyQRlprKu#w;W=)+ z^2yE*&#g~A@Mm87Ag^CHG}Sfk-pJ$#a5vFSE(~A*xZ0e$9!`$1;aY1)-NMsk=6qcH zeB!Qywcfpln1Pe{Xn(XrmB7s4@Y;dFZt1ultPE4owZm-_OU&>?ogT!dZ^%_Zi@0(D zMk{O}Xtsxd zaH{Ak(#Q1#D18w8xQ$rVF-fACo15>nFj!Cq{oC8aS2BE_p&;f`ohPfgFAjFp$Nf5@ zw6mj@Dm<7Q&PxTuC9d}zBWE#b{~)NFt--Ayb6V- z#bB*H2h4H~-engBTGL+JA)gG%!rGrg`IU+v{sX}{j~v>S43=rzS7uWP-v?}p4b$%A?mwesws*wA27lck)s1}T$$CxjGb)K?7`J%OrZOd=>qyk9m02CGE{-m(!U$_y? ztuz<~r4kTettuFugxluCFZgQQ!4gS91rvI?MR<64%A5RGOU)F%LV2}l#td`2S$4EO zr*iFO>p{PC@oHE}<&m+#av$h4a(4CLEpB@-qi(3{+@dFXLLQnE{r6ABFh(x z+pdb07ZNpm=`HPy(z+8|9yP$Ldnl+qy__z4o3X6b!Ir0}zrG+n;+nqyxK-oT3ZHGj zY&(X~RN{{eme*p*)P0gup08MKB%U$R;x37%dQWK%qIxSYv_JIR-A5r8FKdav@ZkS_ z542g4d)6?uHeF}0A28y|zOF(gzr}U{kLuL#Bm)@Ep8U6&a2q;ErLuqhs_9z(VwI*7 zVw}7t=MP-`Px){4KlA56?_I1sO;cdh{*=Te`th0OiIx%!`A8?o4PaNliQmI_58!Yn zsY){5vjrY2(cRK~YaFU?Rr2s6Uvkkf8FzB4`?X(ld@onA0T0#VJ~KPfb|XGKxo#a- z`$1&_0+km{)G)(4O{NY;4()(lQXgNZHUtJ3ZoZj0HpzC7y*QM)j9&ES6s?iE}Y`o}iE`bzfr3F14 z8n}3VzRA+-IW0Vu>Nvi=cCvKTnZFwH0_ldJu*D9~55|3)%2G)G5y|zdBGv42iT#vD zn}K=>qms(t5=xImei&8+N)gBvtr96OWu3vJhhf@5nGGC=mRF(>JZhiHayuL-n{tIm zRk^nf%DsVyL?WsGmInPdX~T%1@4;L%)s?4wr5F){-Uz+AoeM&ptMK*rAjq}6uIsJz zz1%{1i*!%f66NmGst(WEP{2uF%-E(von8u5I?*YTOiH2fxqFiakuWxCm;LhPsCNFl zbl^*XValg->AvS(dPIiqm_xb<3c|S^?Q}%snXq-n0R`6 zx}(e@Q^DdDg~S2= zgZKkLAlNmX43;y#qDd4ZkiXyJL~iT7j`N>Ylj6)){HC>!n|mU_6X(^?PtuRJt6C=E zWh(;!(XKY&eWS(McbzdHhnd3jO@#&mM_n;#ap5`m|t^(p>h#eAc_K z@m}{pz~;GUDTTB8v*?mNf?V7nQg&x}1K3Ry`4gE{vnHXkCcXdc$gR3V)Fo6jQ02$t z97{`l^S1_0i~8|Ko~vilQfQ3yhwRZk|H(X_3>2s^fUj?JWRWywLrWhmQ*Ht*#O0Yi z9lQs8)ucPi^2K7sx_GY2v;V=fJ6>z82(PQD{JhQV&x}20pP~B8kABiJt!phjcLv!_ z3uk!n89)mruCj?Fbigt{J-i-2;7e<5Rpa0-CHc4722Px=)5Z?VY`*?M1y~$y>gLly zISE2L4GmPEN^hA=PtFUt#075IGeY*6gH$KK4hYu`J+XU99#3{~V|roaa(F}8;ZlhyMH8GSiN#Drq$YB#wXK8-~p9islN)V(;sE(PiIm|-xh zRTLz>Ejw$J;?mpJv+Am^st*UwGk)fr_bo$N1VisbYPU8KGQIep#8bg5v7;-HY;hZWkF7w+ZjSF@$y9vDm8qZ z)w~n4N0`U8c53ciiH)qtJcCAe${G^~!$m<{gtspXs<}^9AHFXPSDGeFx0unF>LNQM z5hS9wTVkyHAcs#hNs<9Tbg z;S=?wfk4d3ls9D`zwdHGcjg{qDg{I@0s%Wv;|ct(e&wD-nMK#U6Oz z!<@W$N|S1_%_$K%pA&n!bI5x6hxPKLuH&3^I{*Cg{AfP%oMFPWKJ@3Z*?mghbaZqy zU1EEnd9kE<(V%%s?q19o9r4mB=NxYs`yhn_>*PA6^h@rj>dk8IUavcYE-PLRdfVk1 zRsQFk`g|CIWQ7#)FwLjVIB`Fy#P_A&Si{GmD4iW9DYZtXniEmsmR4P{fV7QSKVyOT zMDF`)x>n7GBN6j6a5$1O7LGE)q7{M=3NOGV;ssbG72*&w4xR%lO>J=Ta!%@XjvvR< zr#}V()~{aQu}m4rm3A5c!h0;T?+FT z9jbRP$RlVg1cbocafLyMLzATE$MPK{knS4*5|hbM36;F zs=ClSamKZcw|%F~`{!=&w-cc0(f?RF%djT@w-1ky(1C!Uq_jT=(jC$u5~3g=qd`zW zx@&+EBcvrpE1fEg(E~{(B}R_Xqr=hp-2dZv_IAgyH{0&}{(i3Wb6z$sI?XhaYR|HY zSǔTMvsMzSn~^KZ&_efa11B;mgR#a(P<=hs*3)^S8kgUG5v%}?2Wmc57MH%G%E zS8k`>7kTCTZnXxEjbHLYj#Bj4LkP2b*B-fEC_RS-@g95Gs#Rn3)qYA|P%G92hn{ua zLth?u9(JCjxSgvCU68UTB)r6gTCADykQV#y&F{FnvghUX?CcyIE{Rsgl%zXI&b0P+ z2VqPaWh*gIrI#f{QY70U7t5#*1HK=wO&*`a$H(>R>2ke;i{?y~UOj0-O0Ico{C z@k1(svA#zs2jfA5h#x%5_A{D5uV{Wgd0&25&lB7%bjKy~I~$5~67QarH!exvuv-jiy`5Eu}rr7)zjqD6lyh$4(~f2$4`_oddx6JKu|VQ zh`q=v1lw91JZ2Qye;Y$j`#fCc_4_Ji!L~QElC0#)OM`oBS`r-3AGP{*~es#Is%u5*P6#;jZZJKmYSxClJ(VJ36rryIJ3=L03uu8ivz%A;<6T=244mU!Oj}c_mzH)dj&|{XY-0 zi|#GwkZ!JK(x@oPakl+>QR~^i`$Zt7|Ba>aGqgCP>N-s@qWA*)g6H9j65a}KAaApH zA5Lwi+M+U8tztOoq`B?IR$)IV^NwT~>BhcjV+B+&5^Y>bt!SeQmMt2_XPdEU+9mB0 zfqTd~*w1BYprz@o&}=_sk16OdeAS7@Nq{H|BQ|uJ=oBPT8-n z#+Ey8_Cs#am)({(r`K%ryRI*K5$WE!#YQes8EvO$LI52M*b8=7H_m{&f*%Yzk4_Px z%6Z)OOEbx8E%^6wH>!H;z>ieeUa)8ZuJ*S4XrjJsGU%reQo+e#%4K&C=Q0Yi-&F{w zSK)B;-W-%=<>|^WHT>p|(sh^vo02H`QWjM74yTH`Ln2g!3>yG4+}qsS#2zu3OzSTY zq2^IV_TtyFk1FeUoU|Jl%k6o`Ygz##WU|g+)mgOfc}ya*3*@b}&mrEODg94ed=;Z& z?l9l5fG!+IjY(lwx6gy0{+=kO9d_nC>o26eeQB5Xdct>L ztGe-yl}+k(;i2gC#opn%!SBUntvdTLaiRWyL5JQ_=UKW9pMA^T=|6TG#DT}DXu%!h zi5;NOGIAQFEj}Ey;v!Usx{#VWKi=`!LmG)GHj5{C3=|P*mDAm84~35!lIn&H&{P3T zlDw2{Nh-+ENx)dHu5;0C0R!9&L!G;nNf9iv9DeEi(hXrOhnb6|q=O(G08IvKMfT8f zj|l4zMCz!GuYm2CyUyxCV~S%0w_OH?3_g zbO;7kB8%zImyV(5w@uTEv{$vNt}+jcG}I`vNBsf=eQ$PmXue{|97T$GI&?Zd$@N%! zA|}(%>Za$xo%W3Qo`C8>v(wx*(zE2J8uuT#pNu+gRsEg0@7Qfd%BqZ|PYRDvVU*gX zUTx{DGPD0~wo&14w3KN)QBVMc(U75@Mi<=TOHT|n@;h|ezWB)vTn(XO| zrHJ*-Jlpi79QYeY0VJmz{NViaB%OJ$uKDHIaDCL$Zsh)G_q4CUlCf&+g;3xY++r=| z`knLu>+n4)g&TdA)FupbAGeqoUFaRNAZtVC!3c^bBx7QDA1S92PN}Q4&IhROWEEI? zin*>$FrwRRFr2*fcT3tD|oz=)SN8}h&~ofcz+Hvb+Q{Q{==H#Z2m`n*cmNQ z^lDJsf-U&bp<|qt-9|z!G3Y0)o{S!$yt9>7?we$v<6I~=@f~S{!Z60!Y0Q964o?gD$ zyj<)g7i&g0^K-xdw!1CYE-5!Drv*_PN7avN%U#3JA%wL-;8R583MQfbh-f%tdyYw6 zU0Z(h{0p0PY2lxB5~^d5iIQlat9W=2-yFGyoa|p-p4==pyQ3@it)cITVJB-Y0*#F# zp4C+tK6?TDV{6uDC{JXzN+M%^10u;yQ=|~iX;a4fu}kfjA(Du%L4^66480iL?r}nB zACCy*X^UX1dP`;`{c)`OnFM23(4h&h8FV&8Nm@%eq}7 zEfddamt!FQf3`7v#;qh}Z{OzQafsLmwB|OWa-y?)`7eW7U!BhvMJzD)EYiww3xP zPD5w9tgnxxbFL5dZVrL}AzDHH^0f1=Vx_h_e#hR=^X6#q8UR*qu8l?OEMl8 z(UXFzQhQjXX5Dk~ZGrLS^^Q-8KS;8G{QUhl2ysnQ$3RGe#?_EPc!{N)o~uw;cR%6k zrb=S%7|&w)SfMPqgK;)Da$;_b6&Ef7g(*IL0cc=^jj{9FQ9H%svNc=wjM?A&*S6{2 zK%E+2(=tUWoB9Yua&Uq>^C{A3KXnZ<_E6C_(!5!qi5sDx(Cr|iOgtQ|cmJGXoQuX+ zV2_)`Hm_J(|BVl-!a{D(NJv{c6av_Q`U2FnN1MCJoE0zAV^+Emn=saa?BUHiSy}MkM9%#m+EI@d&a_NPlICw;`uds8Zd;Chz^7ifq&J*o zafA{-K%vrFxO$QhjZIliM32v*7<^}cF97qOE-oJaliymW;8ds9H?`2__40kO+|&LS zH1{zBV}vnt8ca0AQW;K|F%b#<3Rb%x4ckcG`%<;4qu4-{5lG^QDOE`oODdi3ke2Em zeUh%P!=OAkIYTjn$>H&<*Wc7HFV_OQYj(XsuIOkEO?7ux15%Zo;)qOO;^I+#H!m0P zX!+hOSzg8l1t_rnXAtapa~ykuo7={@^vESiv*{7U9h|Ler=5GChFO9H2eJ8hhxK8L z^M<_9z3-^$a>P0Z_L>EXTsELkD8IorlY(cB-w)hO;ylGp=6H|XKl^d<8gSV$o>~erxy&x zldDS=i{8>y>6e#$#|>*M#;&8h2}O6mI#YqPWcc4jSxMg=!Edr4<)2oK;tBr%yAc2kDFg#PsraVPIUJm{RYj%x5~OX-(Q1Y>$*rH1QM!6OR?<65=C#j!ojos`E{_xq?n)tx)3p?evCf5j&eBO$(kC7?;Tro zgC}cd1lT^69`RSw(%TLj_J-oyWIi53cpZBNy1+Ws@4moFIxZB_mEMQPSH=%=Om4zi zKQ`4Av6htm#{pa*@8+;eW?9#6$i3{g_Uo=4MYb=eKLuMvxdi1XT*Dam!VZs|n)Dz= z4F-D27rC5^9&cpszHVR^Fnj&9CR)lLd2s4P^UzlQVZzh9mXk>C{YhKOn-{vwAk2{` z$#0~j%NnN=g*BYBh`y`E!D${f%EsU}x4o43f`HJL%U;`rWt1c&=;3Zo_pf^O?h27} zw>5{LW_J*d!~YeVAzo>cz@H$QM7qN{wZNQIox*P5$CdUkfXq7Tb)C$pJxUKE@yX6a z%jH3DpS32QSk?2{`n|`$8kl_K9vaNW0o?b6?;fu0=HKzP#Ztbe`D|U9iduiN$x_RT zorNJ5Oku~`>&_ZYR2R1OrSR3n7OZF&78rO36#VZRg@c^{7g&nkkW2K1>-Dta#okSa z;wd?B(7X9(dEKvgmZAu_?#{mfl}(@Tlh7=Dqo?^(&i}~NwsnU1Y^5Uzn=$YYai< z{zsh(M!34$#p$vF*xSGhUWf})xHa5bQ}>H{_1rKpD^B>&9;3abQqnu|`J*P-FssXo zOg)S9Nuw5c(8~nF`_9|9*WVnI9B6V-ZXd!%c40I;yK{VeT#v6#|22j`3j>WPd!Q;5 z@)|HRCjtS$O+loEKVlo^_4qG``pRVkC@c=7;+L|;Z!RvHZ;w__d*mOrgx^| z_1QjsWgvXjHJngp1LqU*Q4mX$f2*C+V+;Mh*Iqgu0~K@@WPmCo`yO#=g}N9f^_jh) zNqnC5-&$(2XtBFk+)t)^VJ=Z^pR;Hvtx0W4A3po6z3~^ENs_?b2@7fdU-p=_u|WY^ zUvQi$dYcdLivtxqIwB?d=g-$1ggl#DTe}<123y6ocw) zz%ALuXVTx!V9jjg7zD{T5wafow{es|d?`S*XLjp+mOkM@SNlVUcN^;K1+{-`m#kb= zFFZ(4ZRJbxjL=WF`;AQ1cXTSk+_E+y)Kw-3k!wLjwc-ZBG*Q+xG*W@`EsVdUsne^p z;RltKT~4U6#xW|Y0Dk%714xyG!MZKSk!aN~HX-yY!=2oUZ#WAszz1k7c0WmBD|R^W z3ihbF=07*8N#k&Ts^FTlovqj_(KIC?xtI$(Rl3H3looxbn$RnzSN(RqKP{%AlSkf7 z)Oj0^1A?iI6K~FSuCIE|^6=}X^Y`*@rG_u;dT6liR!Mv%QU@_<=2G`2==CNH8yNYL zf81+jysaxy1_=9u}W%V^4v?4mNzR8LQcg#oDJU$<^`Q#_Z*$|5>n)%HGd=V zpF0^hSz08{Y+9}O=>X3`^HrLiDW&~;}Ph#PT!jzvJZuS*= z|5%jO)HKWpUZ6YnhR0y$4KU}ar;6woY8(9xSzF!nL(8z5xbq2AQy|3de;TCpmh4`J z4qo3K+~|Kj?YBBlP-X{`T=mBR-zhKI3>I#NJ~e}L8Grv}Xs0n>N*wiDgSnEor?J)5 zonLy$=dM8>bhn9t{N8cz%WV{!gNQOX zxut?gU#CxmlB+^raC2S6&n1ohT718@{hrL)~B|87pFm5}n&c?-T8>2F79Tey*=Q1ju z^P<;37tiQ}G!m13VD6!+pVTlus*?mkHNB{n>ZVEhL_(Va_)X@ZvvIaB_dQoBUuAp} zM|qnkz16{AVz5Ls8Rz|nqnUj}ZkZ$S_;jTJTBs$tK?UMf-rwo5ME<>7HGKM$4xBkA z4Qa>)(FcnXI>Br`f%CJU1k4&-nzy(AHU&tTiaCzJEv61m?{=5V z`L>3O(ofV^WCfQTCu%Hj(7_EFR)v$TBw?E86EC)G#^}5|Bui)z7uJYaM3*KbxS)Qt z1a&W`L_9`wF_cF`htz}tdl6s0JJX*5e=K4{A-EC;dcPi1J?{(M_KK6(S7IZl`|Gb5 ztMFf7h#^EOqyu)v<4%f{-F4UeH~5uIJ4nvzNwQB|3Zry((qhVLY1 z4gt-{8uA=!)PF6;BH7u1z5XY3a~^V@esk%IzTQg5>$&cYuGy16$8@)IL@I@j9jv6m zO-%^)j7+p_m1V%qy8;m;nPDvaPyWGT?Jocs?C~@u7K_ccb~IkA!xT&CZT(z9n6haw zp-lgKP5KT&INw+l(OM6DR!$;~DYt_dnZAwIv6z9Ov-Av_2XF#Kh05o}D&V{@L0bR4 zaPOvSnD}1heeSS(GKnO|CJ|nkU4UCXw;Rr{ShigQgh)B8M#1KP99MtdqshfD&)P{& z9i14%E@EIcAx&J|YTLPAoBMZZ12$u8ijVuf+1YDh=H<{?{}_%Ix|yEe7c zhiXl7v<-!Sj3pQMav>KdZ1#8=$(%cOJhb5jSub(evyP)~rS;J7Pm)@~&Z-XkunRTr%@nWxPYNj#V5*u*jMI9ZwKw@pMKi z%LJ!|lyq8$kFz{Z3g_*S{$3zllIo%{O~z{bbEc=xgv!Q>x{so-QscuIep52_ zadlrQENPssVS$sJJS*(-^x%f@UkIQH)7Mg+;Xr7PW6q_Aa{H^Crgz3Bh?UP5SmQ<3 zB$tb@!$O~zL^MKc=dg-*3Uv%48rM8zA5uqsI?M2k7ikMb-2{eQ-fg>F+hPy#OYE%9L@0QPu#=c`zw zQF0CKk+);;RjJc%a7R13B(pQNshNdfI5~%&VXcYw4T%r_Z2rwGP-1i4QS9pL%1^}u z33F7x_~^}`tpqn5E=(L+!9|&ThYOhIUtQNviN=3ZxffZ7{j9N|ube@w?|D+(rRl;~ zb1bqYW&_UOWBA+KaeU>^N`xD>B2zcl746u)7#u8wF-}?9;9dR z7yDpm`l}v!yfH>XdGGH7%ODAL6TAcCd1x5nSc>gkOrHzNpucE^hWKL_y}nkba0}~D zLSsp#@g4L>ofe%R3aFo4^+p~;ym$#KNQCeQ$*lj^@W^-ZHES_mO4A z?s6Y`pj*=M-LIuJC~r9#*d{&xm+Bcu%Lk!-k4ubR|YbgCuoAIG)q#j58E1kR8YN0KA+{DWq zvu%hH`)^oPR0p24Fe>jxNAePNmsWPKHsZlhr&3J4px9UWxaBuh!pd3>celHhe-z|X zh?;1J)(;V;b1%9aYS%)G$F~;Y@tWgON$r!;TKaKd;>3?Mj^<*-UlTJW3@Yu(@_5M~ zba|gCLA4j>!3kCtnfXRlNJJj0fQt)6D2AFk>AqEZ(RJT8E{s+lu2u{@g(D=!pcq~~ zJ9AUz$;a$@kLJbguQ!HaKC-L|K{r_r2>5$V)gcMIe4#l-!3qY-((7J3oqGF8@W`PI zN7v)@mBhgTdfAOxwo9kJ$p=p4D{i|N?>KV}sjy~uqdxtwt^!T##;V(B$$!uwV0<@k z2QD%mqolH~sG}oAlp(OC_RSZ^zPJ+!ck0mnz$b@W zisn_NofhARZ?>~=hlrP2W%piPCH4=IVpXaogLbUWE+=ZpPS9ySt+45dJ1rK2L750* zW0SzRW$E(@EL%-~%`=1(C?sd!Q{K1U@>{?=q5!3y*wiE0+4TV8^xE^AiqF?Iis$zp zB;jf|lSpJTN9fk?HSX!Z4RBxxr-i&OX-j8KUutwnYS4wWhTOp7tQi0S!pC0}TTS(C z{wUJkm#p+;tBK-2{xkP}YNyF%E4l1MvzB%4eeGF> z1atl@TrO+NuG^IKt4sqF4!(-lfQRY&AcUaz5YH}FRlDg%pjN8)MW6mc9a~eWxA|WT z9^VCgudvGlKy1|}G7@pNc`|2Ro=eF1D;i3|UO_^f`&U=54?PKA`HtpYA zr^P&F9wIs&Ov1WfjsFf!pJT1%yI$F1<4Bk-fGD)Esz@@s7W}sUW2uaY@gK2f%(Z-& z_fRoIP{#6>GaEgc68URCV$yuEUKQ%#VRPUA8Y2C~EmzW#~KO0r`cei5$2EAML8mVgI9 z_k|7!nc zU~zmA#&ZpA_eiL|)?Z)c@Bv}T)& zd8gzrZelN_#2gsfsQb)|)QTTr5US#}7SE&zv$GcfoKrhZnS%K`sJ>)ht30*d_^!h1 z(D%!sC_%nvcsPnZ1V}PFqkshIA$wj$?~KRL+1Y`}JGN&JRpd8qd~z4!Ln+p*$f);c zJcue?(Sd1|mxD@rp=iZN8%JUAkYzpd)KUB`T25|*JoWdFM_dKl?V6P(!nGTZf`*6h zT?>|JLUvE*tk#V2OKZ{oj&A7;S@!bquZkX-Jnz#96?t8Nb;(z@MN>Z0-Jj?@8V|q6WcV$=X zc*bs1WG$`@UVWY@64O69L8<8)YcT{_Kp_0|Erv<8+J_O|J7^Eg1h>h z*Ic`=t=s(V(2q$j&l4yzj)_k|qYE(say|O#81Qcb6mLf~U%I%vraOcLJf%;z`uh)7 zSEY6PU24ZL|Db`*9yO!WcK5$XK3lN7Izj<#D*AF&2z_%M2)1>q!T>xQ-Q_>*MJnB| zD7&E;mqF|yT?&z)E_=wV5$(v0M$#$bOM z<}j$I1H@TK0;C)vTbk4Dj+{VN_WC1=A$UNaMCSW|NOn)9uWNe>McwAIxn-O%GH|JX z;Q7n|#AVO(?wLMt1W|vo)>ilH7VVRfQ_(>+>%a>QEC#uW0s{^lvA7szevN>Z@)+xy2hhS5KQT-pa-=846~x`^PSm)b)VeFpD$%JBDs#5ZtKkJ9Swr zsDaJO*`jF1zE~O|@lIDd*3`VQoWFMZUC;BIn)FM?^rNVT3tnZ>WO`Mi_g*Nle{Ph2 zMnbOMdP!35MkL7aOY}fo3|t#~^}Q8EYnVfh#hWFN(tT1f9!yH6(@WUejmkij3z|3G zpAj(MnxG`L5TB6JIW=;Z2M#tGQN1O`R8syavy5|SYN3^`(@p7UV}TA9v4L+Vk0^a~ zb3a#OSfxq_@(OMmYI<3z$%iXIBT!YH;uAa(##D6PNJMOADT^STg+?1rt!-m&trEg_~YqK1QCMl=5H(u*}a zQ=QyIhzQ?-KRE3wjcJcu(3nD2*LNzU2!o6bM9^O-Iw&aFc0G3FXdQ~}F>`Ht43vn{ z@=ypf^DPw)`=xc21EmZkTZ#4=fC{f? zi@Jifuy@*{K0{t`X~6% z#?VyGGJc9Vl=A(X|Ay(MIsbXH-F5sz6TJR)G|pkrVfT2cmh&dwTwVC;ZB09!ew^sx zm3{4R|J>r|h5(I-I{_$gR1Hr4a?Kb$B=gC{isaquduMND@pqRGg#ZNH_HA=Hp+WXP z5IoF`DW19(hr^NFQuESrLd{kk2{N~Gu;A979@}uK8|$jj0`V@M;9oo45Rb|Pt{a2) z%esq8ZyYk|f({AG=EAr8W+wc3m6X>cvw~x*PMFyLmK1aW~iuCy5Qj(-i z@V9vxJL+X(VQ1a)I~a~}h8%AW(cn}aBTm&zN)kNr3eNZ87r^W)_VnPKgTnliv8+&wv|u~hufym+uNcHjoPygb`D5Y|=KC1Pe&i{$@K(S|6b0Oj+& z*lgqMQXIyk)SMM>*3xe_USWBCl?;{4j#`{3ZCw1e??q4)t987Nrc8SE143F!97dc% zmz*N9IP>_{BQMoVWZJhcGlFa4?KbxvLXde9kl0wj=5-nkbAs!0gE9RlyKRZ$V!J@rNZ;15sZ-@+*ep*|Pe zWSa*n|0emX8vo|8KdAMNQg&)S!jq;gtt~FT7?AGX@mZdR;8zE?CULX ze@ceg6@&e1FkG%j<#Sv=Bkyz7-3GO{%p2I-&%Cx1`+z^Qv6}fHYdI9PJn3oyJj?j3 zkHG?&ZPtl_g#`U}g15$>-z%>=E4t&>-6<~<3;G&$mahKy0VTA90%$IGPzTbEhQ2jV zmu9SLWJX}-v_PP@zi4ENX+CPTT4)OjfYlzig+@_;L^OExrNo*huIfLshj-i|4)vCH zzt9vG{tFFxtg^2#2TC;5RpJZ}<-GMOyfIDPG2v1H3FDepeMzxDTic zz=a89>bSiV0X;n7VD42Y{C$l;mSC|erxw1%m`+#OXnPXUko_U@MI4nu2bWbvPlY(Xfl!`52 z4#wRgPELh7Nf`RHz6($NRnZ(s>^)&jZR361SdNIkjPvxlnBhJ% z>a>hV`M2ms130{42eFo2jJVdoNdyNMu{V}Rg|Ec+_h*VknFmDiy~|$Gm;Z5XatZc} z>wRJ36P%>}>%#v9Ua>*`3dEsiag}d2@IaVDbm7eHypz3c&nCdnTgu10p;Iuj`D)Eg z8pscTm$~3AN_vp7l{b~<z^xm3fI@my~BX3%4E-sLMu>(8?SIS+fM~TyP2!r3>533KJ^!@Q} zh&1WD9uX}cLdcj{+R~m)I!eiWvPg}SPC|tuSgKHv<`sDERU)GbNNJatOP{5gX}P=)R)>mxwRvhNmC@(b_Z!yso@zq~JQQk0dDB#`P}$ zNL~6+Wr}IN%D33abmRSxuo;^lLN_4=nyAS&wG6%|*=YBDRxcWE5*!c^BjVDtlO^Rk zu#Ii@g;7N$>TgG@SiH^Eb^_VwH(>X4tu11ddAD3X;i0Z76{lQ&puf?o>k^H|KkYN{ znV^iuttD~Yy^w&(ZsEb2n@uS+HD^l`IP6n;g1LLYa8e(Bu zk1EkYe-pJ^)sIZTZ*rcxeKx<<3?r)G5@}gzJ%ofSv5rWgzy(C493KqzC)w^Mr!!}& zZwh~S{obVfwwq>L{_PkV+t=~tQrSWTpISNgyCN?73z!+_x5RKOc&WlDvm`h9o+3X7 zC8?MJ>04+{VFuM;p`bsc=u%o2ceRAyWHz6xt5%EvEnbS{fogQyLQqOX?| z6v0X3ZsjHkOrTI=5&Bwo#g-f3{mw08VD(Y4v2WqdH7rDNyY9K^rWY{%EvoN4Vt68)jYd6cmqx z$w2^WU{j~>IN1%fI(^w;%#u%m-pfm;U|IPZrVKzH89|fyJ{NBX5%B@;#Lm5+9}MjB zE;L64)tdF@xtCw&H=NgyXVvf{GP(5?^nfoa;ApenWbsXU<{356RPp1r_clj${UhEl zfmYlpkzQb~7DNwh`+mNYAZ@>U(hbO{%>dHWb<_KU6!L(9ELNFqvh`qNaj_sr z4D_Xi?^>{*>vkE-ngb>0QLt-%)dYs#n&F@gQ>Vxc$A&)5W1F99OZo&KH*%_=up-;T zg#ZB;@n2v!)%i|V)XYyumfpq5{|#(HjVls1z8oP(9YHEgnU@e)S+$bs^Cv zGeVg0({SshF$-e{ABjsi{nD4=25N?miV!Y(0E#LQM5Y$8jQ)I*C0y!Q{RQsfyj96c z!gb*T_QncXw(95mvd=3jJPkS7ky^GNE8?*M)Z4+h5*XuBC@WQ2S={peHDTG;5<2I7_Fm6n|xZ6BTlLrSDaXYxO6rRZP}w29@=T+zL85Fjwj!(>KharEGt{7jiD* z66N?O+&t#(V8%K9^}k76=Fi*O-!$5-WVD?zy)<0+!r&7&Gn-zrX^n;Lvz#JfrW6X> z_2xW-XP1ZLX?L>7R5Umk4*dNZL+wR5?~Sh^{*bPxR$*oU*>h@oTLS0EShLzhjBI%E~2yy5O?4D41e@hTy z^LqFC`g+$Tq|q$C1b?#yEZh6}#K7Pnj?bR;ao5{O&+6W>!$ZG-0NID}W2^F$e8_<2 ztxbw8O+7W3MPf~gfJbo=Q#*sMxDy%CX*u~LuCD@#&je_=`6cmlBBD_=SzPU#ul)qd zGi{X&+F2wbh_1TD5@W{gOos5)S|n!5d+w4BeB?jNi+%s^|CJgRjLc4krN({3TI zF?rx85AjhzY=rg=XbIR1mcG(5s2au-uCVrU9}+hRcpwo`cq!%5^nKMYBt#LM^W@oY ze-#JP0T%}p4>oN3)5pf%z32V-(}-TR3iHnc7FqA5JpEYI@DT-T zbksQK-&IlI{>r(8y#g?)wrS=}NRhsi7I5qwx3MS4_l{ruRu~?Wx3j0t2IT`qB^Zu= zW`b~bKjwCRsJ&OL2HE~&4+c`#H%oV94t9LDXqr@w0vK@NPP`$XejY?noA?!wuAebg zk4aoNwvSNB5M+unv1{{fHp*&(KWvP1zOhWyA=iAra69glZ}5NBaSFOdg&*u#R-6Bc zZNG0~to7GD&@k`*)&A~*pMs?zK@T6#)Jc{z85`a z>Y{zVmzmtXj)NmFZy~CAqK?vvjy9pLs_LN=uO{})(bvHNlwG@!4xn|@QTkEy$4s3O zBs~+oh{6TQb$!QD>*Xoxrr+b5$KftyX!j+m%b}pdEMVyZje~67`JloPJN znx}dtcbiptHg3Fo+#9>$<8V*6ciHQGFG`jnJ|cyQ+MHb9j?%!B6A?w? z4?Z(z|EwQ4_7sgaLy6Xqdpmo%etXhdt?y65t^#a)iP{RRW}tp;?vRSZcIj05xs z$H=ojhW3QxJkzFMPQ>D32REG+{ObX`(aO(tDaS!C@c*vv?%|V#2$wrT^Yc5B%RUt| z#M}mlVb2)=Zi5!N_PTHU^F_1q-IJs1-Hn5@{j=pgr>Y`x2he^5V@M7)tfO5uG*!Qy zc(+HLR0-)&#!T~yb}p)}OhX;|lt1io2BMVjmv=P>sN(o@MTu-mIG1_U^4s#;D=f#? z(_ZGN=@0R})NNM({TO!~8K()=;Vf?3LaB|bG|ei?2VYC;8UV4E+q!;3U~o-&8_$49 zvEXBsCvUz%MzOOs>ZZ8Ai(`QfywuBg7OtzN$cxZ!?ZWUu{Q?AEnBcItpaLSJcx?{(Y@- z=%vYEcKmVQ=7G%WjE#~Q(>-62x)}3zrDChcyn0c6m*3gInw)hQCGFc`G7pDtg6MfU;cb(pwk{unjr-FnIwV1>1F`B&*C`cdNCmG z^YRo_j+I*Q>R$CPl3aXqHKDcC0igvE^nj_3Ti94Zf;&2Qn&y^%K69$_V;Hndn&d~` zulXd7z9W4gu8SuCg^%#Wj5A9NauuF3fA&eQTQ4r%Nm=RvZ13B!8m>j__1T-(D?%E5LuW;mUy zR#?~sxE&s<3ZO$-Nln(x&4Ea!!ypAc(Jf!gS`o_e_n|Fz+jIaCtWtg8hfrK9z@7;P8%9vU+~V~-Tw-W_e|V7%q! zQ2sTE(ajDLsU#91cuRZ$nwA`^MapZCQjGjT907-uGE&5>dniOJX9|kqnG$+^IKjmQ zhUJFQebCWArAd8Xtlq=jDM&ut17VVBe#Ba{I)Tj%`KykxkGYU`8c;zzO?11a>Fr8r zkM*5sgq2{sJv@>^-}HA?^ec)OaUnZs+7p{cBJm6L85P-@pXVm|2Ja6X`N;!}Y68pM zyCpeuQ&2Z#_df$0<6~O$MqJil5AR^k>8k(eqDxEibMvz{|i~EBi z)sK8JIh?YJqbX@N;YcowcLEh4mwHN`<}v`O3OF zzi7c~Q*|+MqF8Fn6AzKGWb|X+3jxo-pYR1Xt=n4dYg0rXRr*?t5Q<={kt$YumZD9T&-%*F>}agh@8uwIpe&`CBO#Szbfti zJ=JE?+Z#>zjHgTgTKD(#1BKMK&+#{fa>2jp8oz*0tk=okj{oJgUg}+EEpFy|Uqo<* zJ7fm^kSSW3lj|K=)KcrNxX%SbVt(lcKd&ekGW82kQNp>pZRS)hBNXqA?U(2towMD# z`1q4Ew4m%yQ;PcSKcx9Sh{Fr^dA1jLf%Ys^4WQ&{ijO6Wh_sq1|A=|i&!rpFH^--| zJ=mxa4D)Rq7|Oh}=+k?aXB=cr_p4$`TSZMm=v$Q}!(Sz#`%aiFL(@ecQ~Kt5%PoE$ zd(w!TrAL_aghZMH=;4C5DiRqvcX8HWCYT4*E8Uy1+XkB|rTBh%fm*ni>MY>2kRB!+ zC@m)*XZ_I5CVY7GqFUNq5Po`u!d(Ih>siRP&`lotDhr(mb%u}0zMd@w{H}gxWb0cw4;Yg*^jWbW?^K62+E*l31npX5{fCTX|lSWx9OjsP! zXy#}Rk02?Y>h*i1L-9KmebIg2Fggsvm#_6jU%gxt>CINm6mQW7oqPLzNikharRm!DjMH}-N%IBBtdMBmNOBHFqHL+4%&r{9s_QU-61@|%&}6PVnCcs^d2z>7*+#%J1*Z6K(nCT z8C3(5(H+D87o7xbXm<6;C7~lhnGFr=MRj_{xdWyt*|n2P8T`k-PHC5V2dH2)L?gHO zJO}{l$FjZW=8!Elgk9I0iTxY-n%qWZ7OW)$Z(Y@oH== zrQFB;lAl>_tHGnJ2*KxA%crjS?`%w2t%nB0QK3oE-lF4)>#g3(sH(Sh`C4AwE@GJE zPxOYTM-(jYqN=rFz^3kGQM?+IUBdmIi7VZJ?4{_PB$O*k^TWteeb(CzsBhlG7eBqs zEYVD&|3c1t^Atj|(&pvPvEljb2P3k+5PMx@1` z(8FZw>+1`6V1P8HYTh_+_#L}DuoBGIx7ZV$32&-XT`(#zoZ1USb-(#nH%;9YkV(z@ zBpspeCZ;+(;d?=DzyU6%SIt}cK7co(1yc`?ow;QXr(+uqs4_9kx2Q{d3D%keQ_w~) zS|x&=ER7P&-rcVy`uw~L2o}=iAGmTW3guT7uV1N?ala% zx2JjzCIaPSeoD5Ijk`zr*?q9l`U%}9KIZu-_@7{7Pf9^EqN>)Bflar|OGGE!* z3e-hvF3^+DMzS(MdRN&upyN;ITts2l*w4CRHEJ!(J!nn4t0{I%>Hrbe3U7MRQH{Ju z7aVpysQHSP2^5wHf&H(@4H;EFqo+C4idTmQ7Mobh>(Ar=4o;u=2TUD906#BK+qGVA@}~UpBy!yQlV(I7 zx2zBg+Panq6a{yz(ym4tsq`Zm{zB3veuG|F*~n%al}4)6J(3k_l;lDjS?Ox!rf-b@ zqnnnrMeQ`#v<0ZcMIxto4IWpOzwhS-)yOe|f+u&RW(B)SO2lA(#t2ScQ7+NH(0_4c z5RVwzL|Q+wcoFY(nXKtWH&CdORocu%)8AM_T9aFUstxXNmB+IXWSJbD4GB$vM!pPj z3NG8bNQayleKQYXI3JlgxiI#MLNkt>Z=iYcM;RSd*Nwp9z4JSh`Zr#(0zlF8R80-| z@c^C!d!I%#7j3MCLdzWGk1emsUMLFG@?`|oSdAt-8?(R2yD7MB;SzLxnQ8f$e{(Z_ zKf<4S$l*e)yL0B5-`@}xBW|wi0hxgie__un!Yuz0Zuih!MU6+OV3NpikPqr4pc_-s zBYdyHsU_^=Pm$2_P%)$Vmg{nWn+O5~$icXqb1Af%DYjJ`QuReiFuoR14ww+1K!)z6 zA&9nv6zJB^{2Z&wel%A|Iu8#g5acaweEkBR^rtc!Fw%z666Nm&mdOlzI!&|+VTj@^ z)E&3@R41VQGGd<1%%XEfvG8@55PUYC_d%nL_QLfrJv_6J7~W@Q_!_B2=PV$^%W^x8 zQs%Sex^< z>v32Ied-9KPSF>;qphtbYPX{;Ig6pFF?XoF>D9ZrXKqvu_PLI{U0oG4KgB%<9Dvi3 z;|Am8bxjK zS$`8$P6q0di~H#{-TdovsjPA^c{My++6JQouJTKj;rD&aiU|%lEM0qz>j@5$^b}>Fz51#y@U+;gV^eW@DLFGLQE)hU zQ@ni6=|7tczDl`q+ zI@fb3 zqTcpLI`UAVyN|2a@u7cj`<0R}<3fO!i4N#coD|8PR{E@3A}U=suFK=g_cr=5jON!q zHO4lb4jhU5L9tkSzYT-DkIy+zC@6v7J@X5QD~8m~JJkWgkq?6==Z%hB-1~_xvKw$0 zHu!IZtV<2yN%xg7!xRP2ksLb|cx}CDH7JAe5E2)>`1skx@_dkcn;?irl6qxQlL#BV z4Iuq*Ji`7j=V2bnf0}DJHRV27L?1*8dM_?6R6iof_*wfNTxDfDYm4@I&cnV?+=Bn{ z3x}tf8tco5b;*KI?)J~~PThu)qwfHv{5`Rm1Gm&9sVLpyj9#)4taM1Kw=QUiW?9gK zv1jYe7LW&ez2^7$;`iS5lGki;rRR#a4*1C3Q!2~TyHi*-j@)}d zy6ve*YfU}j$oDP`p{}Uw^rd2tQ4|ByHg|b7lOPYla~;>dsJyJ1prGah<-R(ow!s4w zoX5fy1n#Yr`ZeSA)!jqA-wd0?aG>9od1ss|HS)_S;_;xIS$4zrF$}sch1IKEPu|+`2z3k)uMCYpvB) zNh zE6|g^a4hVf>Vx#frmDXt@O?{UVzs3E#99W^hfv$!EUD&l`~5`n`t z0%C>DQJ@j~pb5knEiW&xx=rtRVXUqW_PKSI-kP-oQo!V1G7z9AG&F+9Vb<(;qKZji zL0|+_0Rjl_@uI^G6ci+!w7FMQdcW0Ra*9^;%%mioWJ>z`J=;(X*6;D29!h~|MZe&4 zS<@WrLQ18dS#ooyjDp6KDmUX`t+sNUa9HKjZ-X`L9AuPsn16C+D~t*c4Pmh0phP){ zu1xsoriZt;c+BPgiB#r?(tcSYn&0Msg_C!q;ffn@`qw;)O0ehV_$LAQ`mw`avmDSE z|I+#~-42@J*8OnN66!`#pbSBT27Q~%pNUs>*l`+7%`Gm*l?ah&QRpwi7o`H&0Fyid6UaI{-#7?9|V2VEqg3{)cC9 z%8VF5k?bNl*xi1*@3xOtwW4Rl?Xz&!Gxn-~0mE3!D(h=0Y!`VRg4)bD&{k-2;bC3% zCR@yc@>w#aZP2WdH*afyU!6qY-QrSSzar4$xz%-Q$h10ko{jgNNxIz-2voR%$rpHw z%^(6K_zyB;{9zfwoQ1JXizJ_Bwg}>ooTX9nXF>kc(71uoi>=+nUZvIPa4Fp$7sePYF z3;@D*y=BAXoqkzS%UuXOZ+&rU;dHG5m-bJNx`_$yS?*pZb+a^=%JsDg*vU#$0fN8Z z;|aq8QPBC0%tgCkOzF%|A&)o^=qB=MArUVL_Xs9pC)o2{C3cIxt)TVw-~}jL}HG5^us_3^Ese`QXjf@=#g21jf6}!Kswcu!`df`V^G~%T?rSPnZ386fr_wUsNU!-J!^VsljRzDjL`G+ft9$!~5{x(K z%9Z=;SL|LU)%N)ssNfSDeyNeTtCVb`Cl4Wap*#$5A^qe(Nm7zZTOQUJ&zDvXG?p8d zZqrC-gd7b@eM~16*MG2fTHWFaoCfePC^e*H&2T+Kv4-+@gX)U4o9jU6%UU!apry~n$boNZh8czlB__Q9T7&v$<;|R(*4wHS^SH`!e;41Ly8w zE&S@pu|N9*dIe6nQcR!aBoQMP{ZbhRmThD*_@z;V_hP9htZl$yB_#0MmutX(+}&N& z-TkbHamBj%q&l7h_I#i|Aw3`QRVa8+lqC*^gz01W^_UnI>A%L1Mf1JK^;6W8) zM5G-zZT~rHI~K9qjXPc&3hbaAtaK-Inf|YtN|h#pzAyT@ zJ#PV7VJ`A^V;x6MW!92COD^xdaLg!I8~9Dj2CDmg6K&>FFxTSl;H zcVz*`JzU8+{5iUDo&|%@MPn1%?DNO}rdDkCOV2j$g7Tj1EqW5ChrcZ&qrRSBFFk;yq}*G(1o=V?XukQJWIXrdN>-?;h>VAkI}k4W?Q>F+JBEuI z7H33m=UX>F0Ne_v)A>$f?KV#;@@n(p-%qtce}1 z(_vd*eT^ir)1tRr3z!llN>E=_JeK2{)K!JWO&xA@IB7d1_Azph?}q<9I^1?{1=O5% z3r=So6iRWrI@XNX9j8AdIcBPSgc|PeCTxtt)(LmfGYfla36Pj>Al7=eA7)P(#F3v+ zoC1!J`@xlLkWWDtY5ox_Bmjp>=sTYJRmX!D^NtPhxhvemy|#QYp#RLs5ifZ!i#Z>3^#b$#e`+`lPi2M z2Po~2+<_Lhr$YZNfG(0diGh4R*MI2$4hlbR_q?Nc8A6O8g$v%W;ewZ_qjcZefqXU_ zHsG)8uI{&)RMJQ2DuKhfdaLQayVM0frRKbsfWA7pJci78&N~IR&PLlD`n&-aZf4%1 z3?TMAyEGje*&n_G#AuwFD@zx@qs0bn`qLXmtW^I?1}J%9?1+BSG_@K`kA+{-@h(E= z3sn2h6MCeqsno%72#8c4MMrjcKTPj~t$`LBW*n}uaEHwSriClB+=f~fP~<{QWUbI$O0HGTv7XFJ@EXA)jWq;PkFg$%oE@u5fRY&ArA5qc-h zLdL^?LDi8Xw0CpP#!ANa{g98zrbqi2Jb@zH87khFax&KH_J(m#D?`2eY z&`~fkuA0d!yvMfN?dowi@(bvw(e=;hK6~#f2>{?N<0F=uo9Q{afLuHi(2@V`Oc*X& zMRt#_y1l#=KB_6xoXzjcBG+L^A3Ed-{x+YMf>^08<^ZjXsF0%?RV>MGEqxS!Kw%Zr zgM4sS%?c6F`8Y-g9uBJfWTr2`3CVa^uKA?vq0M&1-BKQCEJ-Oz-YNfNZ;jvi<@M|O zv+Ro;&%l`t>#S~uZNomN>gM>bh|iK#vWbE(lz%r+$_UP;G_@CS@6A_i8yLO(XI0o9dn>tci&6baX10AQx*IJljD>aw2;XYt zcZ7V>byWXWv+{UQ6_2Nikc1*hU`h%!;BauXpx0%!8bNmvL?@pt|7WaP4@=O6a?yXJ zu!lsNg~}G2FpN*o{Nf4mOssEFqx$NgCJucDC%M(1GeCSiaaS%#nqv0}WmBJ`Tr}T{ zAP8#AxvSrX22o#f8qYK;oG=RhQW@#@>=V}%f3{nh2U+SjaFWj>)R{#=jhN+?zh^c^eVqQ zYcP^ej3*d@K@lYIg#&(EOF5qNw36KhPSJe_<2jJ}@Zi#xl9=G|G~L|l;%L@cVX;m3 zXMAsGegU1l+92B(hEqJD_w-SGxGRGsKK1kLtxGZNH>R)UxDS}5`eJ!V1ftX)@jh1l zJq~Mr=e&aXj3}uwIvoV0gR;_+%E|zt<{6sLOzx7k74r0>rW4fFIl3M;RDaqc9J=J$ zLO8q>aI7#4a75#*?YIB&2wFT?YwOF9BeA4Lcp zA|jE;{f8o7zDV?Jz?_$f_^sIqL*8*i-Y7#pblCzYN|sJOnv9O4qh-50mrqsu2VS=X zcbAhB2~!M`4(qwuSALTz$kw3RO#m)i+&{2tj4|$78816O4FL@As9v!_VG zB)zNY`mter3}>EimS)i?+x6d3K)MEinEBW5ayp7Juj#95hv|-oMUx1Jlqfskna|5) z<0q$oKfYhdA*f}m_oE6bzu41avFPflJ@+7<;q15H9cQSWdTM81Q(*m8m3}n1Z5#*d z8+sRFk$5Cz()!@Va#Zl{^D`@fK#Ph~nP+<=@oW`kxiGlS)o{Kj6t^4O(k@^f)}of8kSDuZM)6v{0Zy zmpz_iT9DeG(tqLEE&tY?MjG=Gdmapmq?~ru2|MGdki}q!{S(>0AxCZ{`6@5R3>VD- z&*Shm*=PuOeBU1b8f4EC;-wBN#;Fy*aVRx%Kk5~9LXgG8RR~t5mVkvn|%fI8^&y>4^f)8TyJJUkT=MQGqEMzH3x+{B;XWAmuhO4zXefqv3`>#CW_ zk6(R!+)*?An&iu;X<1z+5l@-eqmz>qcj*iPX;U3bGE51+L`+LZhm;`g8J7yO@wVK0 zqMbn`0Tm<=PgyuWJ!dC#(2aV2|7>?cahuAHtkOCoWk(_4zv2Y6wBO0!*@Y`zRFVOP zqd+8Pwke^g=v84cWoaJCm{xQq4}0FeJv!JSuKvDGqg(GJIXStz3~~Ii8ta67J~X2> z`2JMDQGcJeyS)B31f$4)<61ckktSD?XJ3eYh(ak3ysl@}YHmmB$$sj&EWHDch0kN1~KNPTi=q%yVTQ&zKmwAdlu0^RRU;DJu;8a=k@sN z(mY$${lg(6$J2lGx0Z^?Xl6-#ONz8pxtkw-FqzbnwE<{QnecgM({QNREEp7|oI+Ej z-{_KMXP3*uO zjxZ)v7ca+mWWM4ON{uQ=Y+rQ=kS1-L6+aGaUKm_Apf@Zj=p*PljgWFI6_s={o(F88 z;qpU;Gt{B!e#gDerp+_&c!GpR_fPh3w_XM=EL{I9bSk0rH7qmGV5W$nJDYOu?hk03 zKE8z3v+}%JcbrTeyJX%EW)ycY&@6J7?k&QYjAnbghBIg2UC zJRpfe)aWLys*#!0oopvzvM{^e%b4+<{{4406$zH|94FPNXqc^N4EjI|2_V;dxPNtM>Q0UH8NIn}7ryjck%mQn+Q6#K;J*dhNFbo6PVlO7oaDOT+Ku@J6xR4G^Bsy1 z5G+Y!&?eD2cHH(rm-Ly!V@;}@g4}0j4BKKT@OA^gk_N|^p*`N8O&$XO4p#IuNHSCz z@NIZhGz$2r;6EDKZPtd4P5v6MSWiU>dn+KQf3Ree4Plrm{mV%}J$;mg1iX^hyrHx& z9fv`56_r#;Y~%!_q%4n+*z^cb7#$S`g~pyGs`eEmG|)XWMX9Bi8#lDsiVcnew3NfG z@v_O|WuWGCte<#n!e0dxwJmSXY2pE;2w;$7`|zw|qY9YuOko~R7Ja*7U9Up|U>oBo zhcDAp<3_hZKt{{U$hT+b%Zu&{$DjW_<A3L%k^HtZ}C zp=~6Q?;$hB_t_ZiR>POs01CfgG~i2!K%UWJ`0xn;9)wNC?*x!S0II>2rQkkb%vlQG z(#%58<_$MAO_xg0q{b>xBe-_`pgW;_HhMOCQ0+WrG77nTAv1RQ2iz^s3w>cVOD%f3 zVIP%NOJ7o3Ab~y*D$kFfZV+Ze>$pa%|J{;_fRqi!&&=E|7Xso^wpaQ<>a7QL*J}G$ zrZ;VbPEuv5c!D0kVr?q9rG^R(hNnw3f3LWwVVMK9B>BYH=sG9%qjc?{ZF&FS-~WBg zYyb-1IUvY&a&JS9Ml#KA*||JWv%zFR`tA>PkuS>Wl#ZX6xYNaLd%|Nl3|oPPi^`yuIk!K)3FEo211?P@-2Zfyma z-~30u2JE>Jr9H=i059`)HgJ{3x^u3`vcx+N>k4zO^yEhQyUhONkxbrpFn_}Vshlu0 z8dn{TcuHwQLK!U^90CKmwJ|C{zNHz%+!}U*eFf{vZ0m(NpIRTb;d=gbeC-P9V)=#| z=UdW?q2UnbbgAe~b~(#i2$^&_qs3m!NAD+le%sk*4qIDw7uFWKeMfwOf^2}m^&olg z)mmZ|d8jHPuMMe#uqegW*95T8FK)W%E0R-NMk2jNYkb^k9AP_>>0mFfpSATX0)VGV9y9UID97NQ7>CY{fPWBD5yqai*r{ zM{$!sq!U$2V`vqWpq5@&3em!*YTLB=S0^LPEUS>XDPp?DLlcK0d!YbnhXS-lo4|Fq zqjg{m2BhK5I@NK60@qplZ$If4|I5Oav(06|ojJ$h$U-ohrj3&DKs|UQ?F(kV1)=M;)+H6@La{%fWK6-SoL5stK*Z&HMF*}PeEyxy zTf5Km5hSx-g{StsUh2(8qh2g5-&RBQ=ZMHvZ#G3Q`^-*o!Y@;!l*+oF0n1mzHlYBwKt^vL4W`#=zc+UUXQX4Q|Wr*o;PV`8c| zmGOzELMFGmd!_DRFXsz26&Bio0)Kr_v$r~w(L}_w5A`54wsskq@&g}<7&?6n-~pNh z{>;&$+pX8_dr;TW+shKW$LE39z_L?B9r-`Y+WlZ+!usZ@_yW1JbKAIIWyn`)kp@pn zkAhc`IJ_c{RepaGFAC$#YuQWwQ%3hHCbL}+q+?G`|FJMK z_=VoGxoO|w4x80lOVs-+{`W&TK|$y@ZVnQSSC?P?D7(`8cirk&3i{k@4bNoF90m?p z{dFwY=qw&!ZSWYxcv@In4-91JE|gtbYv}N&zq=Gk0fLs4Dn;|+NvP#Wb4Q2O62GCy zK|K#?v28XJX}~%!*#Px|tAGv~^aIkL|2$Z!{w{{oMwkMKl3+jn<3fTSR#N^m`siKa zlki(Tfp+%SB?*Jqzlx74`^RZ^!jz^R$%#p^afYMTc5olbe#)AJnVM1_TCzM4s2BYY z?WCguOD(j{T2DYv((u2ex76ECaJR8aEJR~w0e$zfHl!n(KnL1saQYMOI#-y`ri{rJ zIMihwCis0%m;K|2_3&rJ2@s=nF!Uq@3MC8PaJC?OzVLNmjyy=rI1aqr65{J#_TA8q zHpzYr>L<$Es`3-3F#+;SXb2d`#wG)Jeu<~$7A$agS#G)rBw88BQ}C`^*!8~etbW+d z7WlAW%nD;!%DglGEgon^Ea_Q#lzbsj*2;5j2ROsD3+wAX7KmyyShe zTd1Cc6nDDOS$6x57BA0I{w~FkAqMrwA{o83sQZ;7h`N|iQp@|%B2#Mt8{>y*+C2|o zc{a#W8i@&dfr_UvSIFnqH3Gj7k%vF<=)8~lT-aetQHU+p``{_aWzx^9JDd=B)ctXA(ZmUq;3!O|^0H4|sj^9u`&=f6(@ z)7^hIzjnMv%_|5II>Tv2y+oqBiaE<;2Ex5_0ILm*)H`xT`3erIXFhqEFu;ACn&>n1 zu)mm&@S>E;WJ-lOm@*8H-l3Q_Sx%vt!0Phys7%21u8;KPj_Jj1)e_YTo3+5OVSC<} zxsoaio~&>I_tf>?qGOp(#=6G0aP${6dAh~o{5&c2&FHMobBs=ZG^zYJH`h9l8d|S5 z+VrTmeZHBEFjrdnr!jv}L|yu|w+?ls1FgwA>kq@=w`H4~GxsvF`RAP0H}tYT(gNI<-EJhQd&tg15QJ6GY?9p$O3I z&mkw81{~6LQn*x73?yhsBJc5!DzvU4QVm zTRn9;AIW9W0+4S6q*~0&TFkFPfQU#U2rXH#I1B*3ZM7!4=baq+KX8ak!NMVe4>Q`@^VgbqR`+w|= zwSl4bvmPnqrv*|?^+ZmI!7tvO0Ve=B$f-{FOZ)G|oa0jW}?p{tB600ANHx!oHt#1ojPGyI0BsJgk1?>=WGAEettQS{Y&Dg zAVE4zOb<@-9crg=GdSHKD|sRk2gPFEe1n1{ut6*#WsYEB50ecGvo1P`^+z*|aykx< z!M15%;^>$<^sgkTA7}ht$abGNZD=VR zy~6O9=M9DdQ_0j{F8Blyrn^atdS&~@U{h~b2H_EL)l0Mw_#I3tAmNxUQXG;KB1#~T zh`Z!psbU2%&@deAb0l9uCyF}~H|wfD&Hg`|jbGZjq$dk!ICTDQTVJe>qD~G0?ck&e zkVaPBbU(WO5x1P#t3!58-mbjRur^!kVR68>H2X9rIr-KTl>F$O2?CPLAma{iM(hk_ z_)4-t1z+Ls)6Ik3b|4YOyY>9!;irVSP z<=Vewnp6;BZE(2ZxQJO>N+J5QPW!k7d^H|T64XxY1}5yjS*#J1 z2JJ1&Xbh1CMD#g&1Z*6dWO1csrz7GCd^Ou)TVUo;JokPbr$K6tm z=HI3-4r*i8eOoftcPi_483I>LhP4f99xV@XR&0ylfvIxm_g!j@Uyli1^Fd+`bzY>= zDNq`J?KGqijP_oMn}bSFu!6!qIJng=z-P_(>R|KZ@>a}>jnICwikBe*# zei-E5!!;m!2xod2S<#hJG*!r{4vxAc4O~Aa-XW{TND^nq=Ud?`_qm7eDQ%^y>r7hB zMB0k=&t9E1A@y}@12jY3a^d!JKr4+JGOR7G(xGey1%Mv*J5e)iyvXG!&gj%7`b{17 zXe4@3EDA@a5nR7w=y1AO1cY1wt^)7C>&6ieOz(Fn`c+4P9l|;*$wD;@YuE_LxzIC3 zMVj614F_@IQJ(fR=Rvn)x9b2;&7G9U)Q+=6atb!RAN}gyV zf1(`^^=nb!Ku>l81HD0M<<`yDzD)H7m^rKwq5u#g^kQiH65FA}3pc{&oMQz*z|;ho z5tvVb+CKK!)vjmb=9r~@AWtm|I2nshE&h|&|a}n;cy=G=}-)#Ogezjon7vJ?t zzw{y_WW0j0U~~7BZmjr|RI{!FviQH_VS}WPD5%PNR{lQ1zjd|6S>1(~|*JUrX%GXffB1HnvQ6uCEWbC+iZcfg5EIKkX;+^{C9j zpgZMEoti_#oq}7|Nj-|rj#7;ysBF6RA-4avAil-*jk>A|wSZhrAo}1+@4k479qZyf z=A2vvf}I@#(X@JSpaK!mb!15#0J+;IOV z*Tdd7O6zv1{K;q|$1rFE;~9XTZ<2Jz=Z1h%;9$<`Wx^ZlpiL^7{~9AZFV?l@-*bkg zxIYCX(khMpy20*QS66#iqcT?qyWMQRvSawXV8I~pBMJ?(sk8xNliHmh8Tc2xrp^AN zVLk(ZAuLOS_;vb2dy;&HEB?JI!WL?2J5-2ngwE=K##nrZL@Re|wbFv)e!ejhR)D0) zdxXu)S{1_awP90E;e%io)8K#d{swmebDo!P!(B2?9Rj>RjrSoXD0-Q(!OCXTdOTcE zJ`x-ZmQ6c-Q(q@+)%LWswk1j-8>(erMo{7HXN)B4;&)NU&{?eTV|Towk+=K-6_t}P ziW{~W9_#Zrr(be19%H!Ry5RWD{zsXRPBgOf^8$_8?GHuKIB>8Qvu`FPh11HL#(wZk z-PFyoPfS5r9MVm#FFN#dAA=56+>&7xG4f!eVZ;u_O)}Cm*7s9NPGv(hOoW0NU1Mtt zCFx?Mk)=;lNllK@;OHuAY?NHIqMYamsdQ=5|M8#^P3@no%1ZC;)tPM>eWm^Btn>B7 z`nB3v$pueNO`!vs9NC)3$21{Qa<`XFdYbrEWr=9kJE4Yml^D+|##>Z_hoEF~>WZBz z1EZ_ekGR&Q)O%Omz*Zn!oHm0lpLe%ST*?n>ojzUe4f_)JsP)9Jt?`e^K-5~}IT`+Cuyx6LuixWL<+V^+t)$*m zab`GE4_Gu9;FO9yU@@{-DYLKKDrmdndDg5O@>Uw>jDoqtZi2_kTlgrqVU_lvp z%Fud~<7*At0l(5&16#7$0JIs&c3Iq&M1`D$5fACpk|+*p)^0||Cn5W2^5B$Vb}grV zkfP#vJ`WV+!j2}BU!Wt|Me*{m|Dt9t3)VpE~};@u)XG8BQoEE#Y8tb+Jj6F_jhMg6q5x zNjR)HD|-48yPJ(V?2Th z38NE)#+MLE3AUC{t7_v&+=_3Oz`FGhylBa^;VM7LLx1y6B*Oo_^ggJ;lQRT8$GaV> zwV0RqMH?U3$ZeAyb{Sla6$)(-5Qn@M=4fT5iPkSYIxOR)`#hen+@Vkk=0fe1^8kH~qx`+W}DANvRv4vjRJ zJZ|i--F^%rAI<|DlB!vjKt42}kXv=DH|8hM+m$t^d*eA?EF7C`rIaW6rWQ4+4Hqf0 zC^84hYhz@S|G1!UREbZ4a?I3+=+x;uLO+MZiVn|^f#fn}Eb~fpY|MhTBcobP>hR{_ z52PHYchQLPYr7o%lztt>G5GdibR5{z+pNe8&*2w!n^N_v&P%sLZy(FBgG zkGy}cRdg2Mr>hJjX%^bFgCl(o4y+2Y5K3JK#Wp&ef?Iun-9*@8VA+Qf!rdWR`x0wA z(e%jYWr_}bqaqOi#q;Ie7l7GNyY3#7Cm=FXE}VfF^OM3R>0$H?n(}98|7pAck0;oH z5Ob?Rg7P=vy@~rd+Him#BoFZ|k-7Lg3P|ntu1>=D97Mw`vzHJOm_ zmrp)3-@kt!C}RN*n$%#|O;rRiPC5l_&z~O+QAR_w?F0pM4NW3Zx3GO$^lNqqRo!vA zJIFWx{lRkMLff()jOR~GXZe~nl^docG->ezpve(${BW}{&st#7oW&xGN;g{@)W+9Nv7MKnY=i03C|x~t#e z)|HAE3jW*b*9)jvrFCC7FTlQ?iUffvH*XhWAr?*bf|_ zbdX>t=hSt#OCV5bV_txCgoU(+OKGrVp-{^eS*N5&g;!2X- z)w)B`Z4zKlC~a75>{3~H<-fdjV3wnZ0Z(Ka$O|dPIOqs#^GX^YM@6NELZOZZNz3~# zsTwYdteL3a+H$z|!h!Fw$vQf*{4s{fdp5U@70g? zeg&RwUi#Sxz8jO8 zqpf(`iInr%)341kKJ%^KHP0&}i$Q|wqdPrk3A3ZiPIXVR2YaidmpAZVC-rz zwN6ZY1;~6EuO=UCH&zhtO3q2gLBE#RIlip_b9`>}?DK3ig`3Zt1{Te3sn+JeRVFeN zfW?R|;LoEdCa5?7h1L~P@+sm41NsKY*djVHxg5m51)+g@d(?idP;kT{`{{LcLh5k1OiWK^{sH6K59nl zZSRgYDMs{C9#W4v-@fh^_;4Mwf0*4!{*n&j^W*{O@3!eRc~$5ACD}{HrIdz><)L?g zSomOfKD_(LfQu@n_aqGGf)BgYBP6{$nk_P}Vk0~PXem4t1<=5S@&1K_n+6nTaF$hyHv0H9}tgAM)SJ+AGzpWK-re z>cgMKuxibFH--N7kp?IO7mW)6Cnr^6+lKgrh9I4Pj|hS~ww`TN?VrZ7ul^;^iZ|}| z7w(*Yje7s@v>hmm!+XmSzXA{Ku-AWgi`ZmxzDuihC$O_)zvo8bMeMi=hHAzE>6xXK zd^;br`>n1yqb)AM65eDzNom$B>ofs?vAmpuWu5RcScB)4jx`j}$xiPb-@bi}C?KBt zNImVI@;^H$c|O166(pZ6hv{}*aaw+wn>-CNE9@f!%>prIt3GeilHH?+Cl!G$j?`c- zm3QTC%5;!mA#_Q+bxvA_6%krm-lp?uy`-cAFZ*X*Sp<@a)&}y=l)NM| zzRADT$p?|adAaIzQ|ia&j!h=Rq0o|hvj3audMyjIap;fgma#{c{ni0fQbTO>9FiZv z4fJx!i^?Yrjmq#~i~T<&Wt6Sqk3BB$PBb6CF(Nxc?x>fY6_gIwel+A-7kJ??DHm;S zU^rRD+kP(hFL)_nVaO3}VG$|)RMAS*!bH0HiT&G1B=SFnT`WGSyDD5thO{<3yfV09 zRH*U=l$P+n+(|!}og_4$AL%|xqtN=iy^SN)_3iKCqyLUysNAevobJtA*2TSvyQ2-9 zoVo>I?+0?B0%#%if|5Qi09pwg5eEkcH7iOiNHQT1RB1~)&4Gi0g7Ha!e;}|(yyNnP zbzyGrap9()uXI1g1#!5%d0C{IEw)Z~LSKG31;oWS<7%t2xWIh}^%z<<3nu5?Z4iB*`MNiOr3&hrrE?OoTGNn>Fx%;5m} z?!(JuC8^E1FI{Ohl`onixJd5@ctmcE);g)Rm%WYRqL zh_js3QOjJSJx;GGy%!66Tr;GTa=O1<5b+!w*I58bz4yZZU)>{BgtR--v_{>0N2a;U zAgP?PZ{{{HD7Un^!sM7`2+;-NFKgxZJiT}4-Rz4s`+|SuRr07jPbGIw5;&YA zZF-bmlnf+$sM*O7?zoBoB(FZ4FxC&d&nf(Gw|}N=>SFXC9)C_L9ee!Cz7dEO&Ugjl6ePT_LSo6E!`HlW%XZel$M5AeC=B$E)&mhXAcyO z&#|*7XGB2czLhTI;n!~Ea!wuXn?JebFD0<*YY-DYs-rIW^54J9MorEC_~;;?45@B{ z>YrO8Iak`?^dOq&)a9T!a!yKRWl&k$LlSZ{x5dkOqIBTZpZ`cB0R5Zz|J!euZH{HG z%C3RxQgWTqQsx27Hz43*$2ss^=2G)Ih7mXvrA#yQM+`aOyGl1S{AkN~G*l$B2k&Pj zZ+Z~79~CGUBnF&+pS%To%5x@NkZ>`7bahAH82fE_IXpbvoUzc(9`Bp|0S8|KI>!T% z!7r&=i$RMN--2xN5TA8kU`Rw&og6Dldqv)gBR#YpfrElvA7vh?@|6amlpYyr3B%u< z|2fx_K&IGIovk2=AZ^O+&Y?lBD}6Dj~^!ws^m-=moOoV&^=l6Yhur9 zo(gvd&DGKMt9^4FIt1P5GWzYGsZ)99wNI7d4UiLS^+M6?!|p~yfnJA0Q8#vMg07WF zmydX1#dhs1zVcEs{~xVVv*)WTKezM6!?eoU~`CS(8EA4N1iN5-l{nTmuzq%(Ma1)vKuy2eP ziBq1}zpeLvE3iU_*&o%j#;n_=Wy0QewkLpYVvMDrU_;<#HxIayB6~yJ619DMysVkr z`&yQMw5g_h)U9O$4YIV9wV#t51dC9$=G$I^U_pI_+>G_)P2J!6QV$rG!Mv`bp3GhmQw|G#~Bta&JLF$5(Ef7%w?S zLGrNxy8_ZZMId)%th_3|K$N zTx=*F3Q|iBo?o^kEV+fm*eopUktY2x{{jgpNI#WWeE^; zP>fg_n7S{lD*jNyzhNSyrS(TZdNI#8N`taeW&VOkR9^)<&VH5Oh@}h0jV*LK3~q;< z^yLejXib8=%Dg~6x&NH8xET?ODW|F2pRO$7xqA)3<1S71Z-cdKp!YRAzQN+;W$`(8 zUTNCt)xDRZffbRYdUhz|Www4Ob&>Va?SVt#Kb_Gf1X=|MqN69~`o;~r3#Glrss#^| z4!XCCg{2n7_7oN^>KeS)-#(qyCeYOb*OEdga z5{1bPj#zrpcR1bQI!Puss~F`!Zer@p&>K1$tmwa3YEbPgQt(|(lUl)D&S-E-{e%>C z=3o+*@Prakz-4-$4fvU?zSoWOEpI&c9_kIa5Sfu*fRzYudjF8~GgP@AUqE;JeuLhI zaSxjErL`3{U@zXxYnvGiG6QIwsZITIe;O*N%%g1gJ4QuJD#!@Et4rfKAC1le#+A9? z3-85OxbY&681N$Zx{!L8n#m2P+w!g^=-I+weff#TxYUwQ!1U{kj#0f$-q?4B?x9ka zY796W8A>U+ZlH>v<(h(PGDy2{q6ehO{$2_F9gUF4&h$Ewpi=|`i`Acqyon*BO6#J#u-mv-4 z+F>U3vhMf$*5#kkwx3@XuJ}X-Qsf2Efl73NLR)deum%dURR~hQalHnFH3$Um?_36) zr|YGU(G(>34kSwoi~xP!$ID&4sIi`*$&Y@ze!n-H*21vV zh%^qDvT-?%|CBrEcD-h+_I|baic%{~9^-r@q*yhJ%o)%6acGNZ;8iC~WF$}g^a2pZ z>6HP%B+oaiXtS@calT9KLI42Vz8$W4Xz>?5%4~_!AygP0BUKR&2&$l-U(}~JQrEfG z`=Zq&`PFL*0XvO!Xt>c0D<{B9;GF*Td;XpbkTRB~@t^Rl%xeXzm%-hPapk~HAoXSg z9q4SLV@Ptv!K{)$&jwL&|Ludk!t^RKrnoJ-vI@dRZ!-z(kkjKms=vWC+=tua)mlA_&@3L z@qEz|-0C$?*W!QU@;?}+SEA4vpmz6*FL+#*KDftu=Ul8f6$3Nz4%bP=I;fw0QTb(t zkvy(+Ox$Rn^0vw%X1sG<4Hw&=;4C0}VaRFB3Ze(W@eWir^lkEGBh)DnB)l;n&*9HYE6J;nF=y zAys=1)}Ab_)eBbvUiZ`lA&9)Tj618{K7OJ*bt=fC9 zn5|tUcD1orsTrGw+HLI>DrRGrimjU3>-T)Vzx?kw9MApS_jBFXb)K)2#T=Fg@X)A} zlk;N=;uC(F!a2{DE2KQ{?gogcHT7h#o%m*gy~B`bLu+W6DnrZ;wM+OT(8M z?U_12L+FDmjc?aF{7uImCsjblyRBzI-~46o&c|;7Gm3Iu%;uk%MhG_bfe^mOb$$}| z1&c6sXZ&Qv-P9uyRke$nuV-wn*4$-U;x#ca1f{-zL@yGh2=+sPva7c+&oykzcf4ft zb<&6I>3J3w_j)+;j

tqS2d^oz^UFrZBJzHS|6O#rKb8vr%-+-kE1>^HO?orcg_9=nIeCG8f)=lipq1MF1_wAZ?;jvg0nqDsHf47h z7jT-mliH-&;}yq;)lp2;hA*sdc7}lWOCq@;933-ROKe0qdAR)x6hJI)J*%<^fN$_Y zOicIFIc>LX9%;YC1E{UeOd&DQv&o?co-$@>r)-@;XIICaSBsY;i%F;@~L z`@8ZNm-EEMtK~k0jdGp3m*!dgU?cF_I zVbFPxXUA)R@4M_Z<8itdx?f@c0x-|-g_K^e7Pl`%14;=81M{V65+VQ!et#mZ%zfY7y?g^R&dZS!nL{sBphC!3MN&*QB43~9eQdCnja380QX#K+%Awu3PaHlN7 zf=0)NiU;sKA)5#Z7mWAv=|^Kqi7?%;1Gcb}A`+A3Xr}N)x%yxD*vMTQp=AC&-^aiJ1VDsdj2RMhveDIBps{ND@s^u$+nc-RrT4|c zHif#rU6C0Gi*XA#_$ks^vU_$jD!-d5lDOuK@R$?z-xH5q|MgOPN{*)~87Lg+liadJ zGGH2Nmi|g3Zk#dXfkTcC;cK^b>1k$0-A9&H%{!*pw^%^*nKhq27V_TE9@ugIMtwtt zdaUSsXlEF(3ibRzhbQ=;_TKjsNZ9cPDGVqRuKvX6jxs4m2@A)ByAY~d7d?gg3R?ofmKmEZg_;$)sF6D_E!=!M)>uy)y3+_4_-Rs#(suk^xkdH? z!T@LV^2%9$UNZ6Q>`c(zHb*LZxuR)Nlmq~e^GAzUfo1AWwSQql*RjdILhE#97lF!1 zg>^HkM$P`lzA+G}e*IHU|Bp=#cFh9VQD0YCsp}(^Dh{)G7kYX8S$zIDPgQYGS6q7t zqSG?pZ1H9{P`p*bu7e_pXYzqi=?xQU4%%8|Yi1*r=r~k5%yLcE&0oaVB@y)w#lv1n z9|+PTCGeS}m?O)bD?TRDEOr3>(i1QF{PZ@^HIdSZQbEWmLeHQ5dkwV_XW!0_-ZT7^ zvA+IpB&J%QGvh1cEGUB-l=v{>3GQW1yrH0!~8})}Qrk4OYC7qlfoGyTW3&`dJtPBcQe=ea? zD4K-X&scj0R|3y&qe@(*O%>U}%n?+v40}j6@{ysS=Od!();)^v{p^g?BaeElvRbNH zu1)a3e$EET(%Cl6h9aBI7COVOj>4`8fJS>Q2Y~b0UP-Fm?bCk{d=_8RXKkW+*2Q&M z`@a3Fi$GNs$xcz*yiIaI!S?oe+)?<@n-*0LlFXQYFe)<^9R9L`Z`^NQ*f%w}=AQJg zSFQbWwQ|>kWC34()y;;G>dSVYc>+Kb~y=D!umV zw9an+Z+ES|JQJ8uR95kelc4t zdGu}6J!jNi{O)0U-y}5Me~)ul%yq+Bb$b~3n%bR%!_MB`VJ87~AaEq6$PX9(jYbSU zU;;mJIOHuPyQ@|9 zI-)y=vq_yy(Y0!g)`ca}`_lc!QvTDEdddF|S*K0Rx89}qT&v0A0WEQ?{52<5wsj;Q zK*moDzc%<~$%yYpF{V7Fi8B9_-8euB{;praO9tv;U(4yZxO`H%aWFC|oO5?~?`mYR zW2hOT=ys=1uvmQLncZPjO5Fi7HCZ^L*&lgH!=i)-uGq_FcV%sv)Kz^BgFoyXK0$)RvAR;D5VhUgc^xhD)RBe`xvtef7mhTDg4iQ7ik$o|hw? zSIaR+otFps7n>$wfVh3}a?LZ^+A0RcinCkS1K<0dT2bTr5BKfo5XW=)DH9k3Dx0V3F^v0^dJTgW{swBU)p+fb%h1Eh0V~Qt{q~g+r2t?Vz$QzmyGANyV{cg?#qnDVIkvyXYD)3AS~v==C%ELMw|EWfS~;{0Ii zl|&HiXlcoKq(CR9A}IFi>4LY_&D1!|M~a_E?MDEnbi|=yaSvE=Pbv5E4^zS3bwUpo z=jJ|}9}QVvj=v3*$-O-0I{$HSaDZP1%GAxptJbhH6TJA<>2%mD*+XqS2YqQH$$Z*) zfF7V}Zb`c18Zh^KqP3^&WsM1q67y~ZdWeI84ZU~s(?fM`0cCQRe-6wkaVVU@;B(g= zKdiwhXI=k`al3|D?!tjKDQ%>;_x}9+y$I>==1ews?mXo!UmM5P8Q!mE`D>P9@Ow4( zUCvHnJA?(QTiWp?CnwqZ8T)DuGGW-Qp-NPj1Y}`Suc~af;QAb1`eiNluxnK5Js45N z`V<*wo4SXc$$r9`OjB8;2*8WhMv6xVy@-lcstq$I#U+zp6CemBKWOmM@9F%8l#VfV zGSH9TagX`l{ag9Bak&5M@2iH{BK|KKy<%haN<3CZ2kWB>=T8p~v&Pfx|NK-Sa8Z(_ zq@uN+n|yT?>1+u4d0Ki)R9ioRn=RCjt(AKU)QfT6fR0hvXiD^m!y0$#Nv@mf;-`o z?W!k?rj_&K!sVgV)kxTx<>Ere)74S=Gg-B(oq*2kDK-gRfgncW3;4KnG-F$sVC1$& z(u$M(&Bj%xl=nbSVfSgRHBp*Ew}i7~@-C-&9A}0uPF)Uoc{!co%E+x&?ANZH zsZ$rk(hBB+h>-5*vjM#eWO&<&Du94W+IY`uWsJb6~ zUIACf4VBZ{0W!JDzO|q8H8UAN)Jy%};(s1+5RS<&FE}7D77*YFq7CERd= z$hmfQ9+X`tL<}E4VSA{j%l?Mo@iPeLen%A?&T8D~LiiFNAD@Lc`|B~C=mRS@?UNwuzSEMkYJ4v}h?f&KDlfvbu&}AQRO@4A$ zH(Q&{0)OP$snpFlw_aK&MSk?$yL!GHj*#F-!PxI~+oeOi?U z&=>O_3Zt+aB=HiNBpTK(u<%N&3;mPr%z=j|aC#FmE%-y_lqi&)q-iu2OXWu#%;J+N zHqUiHV)~uYN^PU^=Sp+Sk`NZK05ArO${7OaZ{pf%K5dHtHTY*p6ia;1_Ce3kDgMFB zn%GlVm=LwY6INyr^*hE}wMAMAy&+{T<*Ch|QV*<^x^d0XKP%*O8rVn^O*s5{zOh=+x-5Usv^dGuru=9Hzm)RnTT=WbLV_X+qVgIhq zO|DiHu5>RO{=_Rhd#>CH#edsbqG8Ct2o89Cl^uVf3m!aRH5=>ddc-`xQ`48JjNiRD z1o9bpNj*K3g`+FA^ZuR-!9TkDayE0AB3}`e-*|5Y)Whbuw(n!>#&$|y19^c`7v469 z6uX-~_Yi#+)c)$P4o-*wx)7kj)z14R1!k!c=MH{{vg?>gwF-s()qn%VZh&m?bEj*` zKds;Qrpku3h3V0Y__rK1#V|E2n$3GjarJph931#cFC;IQ*#G0Jw5OMOz2uO!zu|lb z()vB}d2bGb0j;5_i%pNaLa8&y(O>UQa>nvZZv4%de`6}Ymol^&J< za0Ev~jH(D?6XF?nOVzxgA;tp!_cOJds@=_Z-W+G41MeW!#+C1aKDpcsH!3JJDl15C19o8KHuA1d%Jj|XQMlt>~xmu!y(LJ3hyXZBXFJselaPP1(skzp z!;r#Jfx)1``W!pV5X2gK-{PGWBb!fO^Y6?2S3D{FrPE?nw%w%nvzKw zr}()>0SMVI;SnI)4WR2PTx{qrwgoj1b~2tBRnmBMLnNXYRa$*%w^K#b7O|X<02Z(` zOydC{NaCbV2C79wEX3eEFztAB-gkA( zrI4HCoFgyCS#8k?la~uUI{-SSlU(|(luzG3;x%$rHQ2G{NVX+3PD@FV0|)T)inSbD zLWWg5@|(+}75#;Ug=oeEb7~Mq&voSCQroNdAaOXet+>%di8Je(q)pZ6-w?p)O!DZQ z^=P6P;^eP*Ey!=<)3DryTEDE5a`_>y4fIMm`c8oErt$7!h(eMWL{ghH>zF?~tyL@x zi8$ENa3y9VB|DhQG|E-$GoQuGIE`HY;CL};>WcU4uXYC>+8rc-#5m{@7P;5(pE7g* z+R>Z!+2Z%@ou!JWwp9ccjja$ z*H}CX;8KvgSRJ|uwbTHYt9hDc>vM~8tdO>jFdlOF9=e!PjmOrPM;)#sSEE8_E2gai zL(cd4i@zX4!swNs$i8D$mlJ&R?|13CauR zh?#iagt@Kk)U-p)@4TfG5_a2lWZ~)kn^3$s@0fpUdrM2pmoHwp;`>yQ)(%5gB)i>1 zME!(m$e-*Hz^#fdK%cc1DpPbxcZ6MqvPU=m1}+kJ!Nn7db3Gn=vVgX5WHBVb&+i9T z)@<)EdrXq&n7#1z*`a%ww8i!jL*D|pqoV~a-jr_gk&e64PgZU+DE$2`?oz|zVN$T{ zUBbxP-_wWUJIfP`iwYeJoub`ka$!7CXdvlvX2<`5X0`pS^ps1S?0jL-qwAA-^WqGc zpZ-;<*eTIb$m7b3WiQ2XcF3j;#wi4jL*^OK@)>L)CA`*WpXb z!lRY^7(Gqs1N)b6#K>>fBa8IPbqpGTNa2f07r9oo&3Kw!ne~V79i^)L_$W}^ESbKU zp%vt`JXr`!mRr&P4tEHnou|RA^M!n4<9b^SwX0z>LAsXGh2xt%wqf>Bu5cr;BH0~s zK2xLy1xlm#oaKGcdvZt`tQ_MgHQLbd=9Xn;u7L(O$FBzRT^P;FJ64tr4s^&n;@b6; z6t5*f1KEm zYy#Wu%D3X+$(FKDv&}TTaQ@=EL|f=B@SA&ZTr+)_B2%5%ll5x%JD?d)@2rOcsmvLf zhS4kKQ@53=c|3kLqJ3W7rgLEpOxa*qhw`E~(K$z}xoNa^Fa3L1VwF(Nw>MoPV;;X7 zlJvj-9VemP@E`9WvK%`VbM{8##~?1~54}$&GSfbdmNwOmHvz6yIcydU+n~K&%JN*_ zr1=Z7D*o}(_mr1n`6Gp@sqm=uID<*{UW{bs%e&^_AwR(Wd)U=Cs&EzDx|6VEcQtss zV$wP=a}~#xBb6j16AUS|IUN)#`TZGl`t3#Ds_Md&U!TXDM=ECuL>G&k>dR~7YN+Ev zYK77*ge%t6T+as=FS8wACRhymviPLmyML$J zqokK`x7&`*nw0IxVRqXuU%m)4#5qSI_w(ZcmBY1Sim=igXrRh!M_~uKob0C0-)9K3(zWmU6vEvCO0`3c);yGkTZNq}w z+OLj*^1Nk9LkBoJ|7{Bj3Z@(fZYQs3;jXrcoxy=eBE3#4U-=?YB9aa%Z@E=5WZ!d< z@)X3m{pEpO55Vu!S1%pKUmN_D^1Jx2`#rJ}=V6#@2^CPYX+3JTMXQmA$J%ZQhhyLIYx}n9qWX7mL$f`)?1GIm6E5IBE|DD;X&(J4mtx zLb6J3fxJSZG8C!nMMCVsMCsnGlxbPbi zTZKmT>ep+O)km?r%~HNNGHMW?VHM;fmhYR*ELq`TSrH#aB&|zcTro4TEK$dZi;K(C z)3b??4|Axczt5&*7==LKk0oN{uDC3g7 z58EzMP5WsJl8~0}zw?n+I68RXRFD$f{L8)3NFkxoJQDN8UK2&uO8(8Ln&JT1MX74_9|b_fe}}$YOkkI7cW2q8J18jfk*s zVs-#7r7Xks*@J0Qag)hG`iNAN5_auZ2oZ%5Gb2K}5Sj?1!@J71j%kzpATKOIUiPkh za0_R$N_i&7r(}0G=Jfe`8UUMj;}fpnq`ek49scL&KqVaonWU``G*C3E8)q5jrBu`d zQ>vzFlvC{)O_n%o3;+!lU^@$Sp3nW~V5VW{V5|=vMF=QVBl2HgBF!qzzmXm$OsQM^ z1`X6Ox*2klBoJOn%D!FY($k;apX4U(a^!D~ozqDwzx{oKXDKt=KiVIEUE56G+JTrB z*(WgJ^=IMJd09``>^RU6Vf{ioCn^T1X^C?RkfI>UU5m;INg4AG%^~F|7A-*l_83wP z4gugqtGo!1RaS|24l&4>l1ScQl6ny;Wb-Hh3_chh9$cq(DHl1OcW9me%+GCV%Ec%V zn2?fdtEw*XP4DC2R(AHo-YwV5-@2iPo>zZsJQRX`3!Jk&!u$p(Fn;U0+8>u-!i$IP zkM0`W-n%Q^EhJcNj+c`K+KGFX?nj65Qre#?8o|P;h{{}Bhc4#ll0ZyeiJg^%WJHGI z_*i~gBrDn1To!)-h_>O2!R|JhLx1n@?;oy|sssYLX%8M$#IAW~l?<2-2DFXW%du2I8K`k{0Iha`)F})da$A)|i;h2$Kf_>Wm z1oPIrC!cL4Mh0=HS0s%R!MLnQtL<^X0L50h+!Fa4L=DpUN7FcY zhGAUD@4u-C7LYOHL+HD@z4Sy8F!#)M)q6}*`Dc+rC&5{Y&WnlccK2)Y!vb;antni@b9B~@#mg2( z^MG;T$M*}AzTu>nmz0R$9ok90*@6LCt7$*U>-Nuo3S-?E zRyKS$ZC0e+T1Lc^ictgaJtgmu1HD8h2&f2_xPn5Nq^B>2U0wJ$>}44n;l+z*#Cbyf zNiA1_eOq8RJ!|~((j?pRQx;FbH*}adZkS_LW`@3gIHPg8tUv89)wWNY?f07JJlVbr zDK{&i#Z4&Tta{AK*Oh%y{DuxI&?+j>=dqPdSoIXkKhfE8y;w5l%KpX!i5lk|Eju#M zTXg!HuVovsQHKHJ2f8cLbSepV!NBqIRUP*llmZD;C`ykyDg*~sbHaBzAIvdMGMYW6 zVm)~JmE{ghEaOEXqe|6m=FhWP_K>{N_@VZLiiGQ2Ebnrzr?KB~F2gvg-i5d*Px!&f zK&x!3V6ZoR4$?uK2jU1Q;NAu*%`P`^6rtFcEJ9YKBz0D- zfh1m9w-ZiXJm|pG7rJ});$WrbatRRJijofea`2V3BHt$%a4?D=h|Ekd*6^<^ed$yy zdwc3z^Xc5AnAX;peb13F&OGr^ike3Am#(?Uq`KygpyD_cg^OQo3TJK^c}b-;K&sU| z!L!u)rtWlly7~0TN)g7L;NH~A$IVh*Bh*^W^sAbXWV^>wm8}g;HC0_bJ)QJ#1tK5# zo)`a|(Sm=3fP8tle*x1o6i<|J;s>PGsx5UuKt8IUNtXl&D*rdXL3(7-a+74+fvW;A zggTfs0oa3=cz2C*_pE|F;0vk={UMd7;w2L0C3_b*E^ZON4lj@n4kSReT{~-q?k_)x z18Qo~Zp#0P zSJQWhC-yuaw5)Ypl_xqy+UU(ceRs$N2&iT*5Cr{anN5(-J;ox+e%a%V#4NheU zc_Skp1Y;{i<_$0@tJI|gF&#TJ@M-4C;g;zL3eg0C94vWAA@tG76msDK-u*vs=RTh86-LOCCruO#4f9f|rbd+z^)D@oGwt&6`lj&gM2;@=~M&M#jAfnX3o z{XuZ(l0Rh7?F;oPh^=$Z#uc@(d}H69&JVjhJ`Mz=#6U5gfO+NZ{q5UPh^M|b+)34- zoapG?PWhNRy@0T7C_xO$AX>A#jKDt=+=Ey0^T?ezPs`E2Q2uOe(c_^Mb)k4lB=YdU-mHy^b7rC^&;o7W3mp`KFTHKXm8Qd zVprknJdvi+28W`ipwN6nqtJ+{#^(Ky_d8H!PFZR*zJ+%RWQ?J}1*=kkO|tHB7d}!W zX2$p2hKMRb-?}eR#i#k;U`zsR-*i0%UiO&}zhnWh-g=0a{ER$0 zS4a;{9UDw=qZX3N21W=mT`9pJVvGo9G_;Z6(cnz*Zy-dEma8E=Zo<^HDe!0l?Euk&o_t09fCWg)h8}yVtkTMI7x$uXT`;fBp$N(>~lx#ID*XPAtc20PM^nVFbf5G(o-B@`LQ=tyFR->Zufqz{` zE4E{-SeH~yp2I2XSyG=j;(P=c4=mfd$MC5+>9@%>=x# z<(rK=fDz=V!?X&w8!QYfiW}Oz4qgiqcYbT9W=fp!?EEb%$rjT5QT|#qzp4;TupD>? z^OliuYA#6=l#YmozI15J_w+?QlpONz=)g70l-{0M-V}yNFbWa0{i~gosp%D6Mxr99 zBewjfDf(b0Tz569yUHhMwO5pjW{yWn+P@?~N3>UL>CmDnkou=|TEqPmANe3WUuGoh zTvNQWh^6@b-F<5&PNOO#WHn~}9*tf#ibKWA|20B`CA`ox(qK6>dv)5QBrvI;ctHp!h$H|?g^9OB!7k{8BVp@EC;mk_Q;aRW2N6N+gwM@93 zH0O?}q_%KqF}9B-D)TUVM>YYlnz2VavlC{_EiJEBA1P#(`~wb=pp3SolH4(hx;DAQ zdw!hdGWTOXFZy>W!kOioc+`k}-WXv%^XphdbAZ^W64zNbm3!_`l8Odlry}a?#$OuUVr^CgkA!rRBL^3Ml8U+3n8xwC= zkhdL~hKch%F)8t&bbXn--Y(@`aCSz>5sT=4=mC@wIyXWq2C0qx{wvYat0)sBdopTZ zM;;!OcLF3yWHUl;elx8w+Qh+PRzviRK;O}$_UwYVYdy73g(%YU*mj9qf%4M-GTUA) z+>+Oow)2arH;oOl&Wr^W%Ji8}Qy;@(gDlJNJB?j8G6=S~gJa)}5*URp1*8XNu<4$0 z>zRFCErcgmDpYOiPV_fYFN|orv_#`F^OxOpDssi855-%YpSFSfjtTK&YWEereX++}tnQ;jWsk<-tWP%9N}9$Q@;D;heg% z|6Ie_mUMcMU9!}CJVjeAzyodVdOZ{&*>r;P-;&etCRS(lB}{x<+-mqNKq?!;FSEuRNSO*V{HHJZ8RWVAU>;&I5hi5 zwEcY^_8K>8<3^HQ1XzuWi+Ae2)8BiNg%>B#bVR*bA8Lq=J-4Z@9_Gj>DTc&@<&XyCMk!|h!tTH?C5|J+xG8*(3u|qw5b~zD)Y`a->cwTYxNSW$p+-vYr3gU)dqnD71Ej9^msY z|NZfO^`+Hutz$29HsQb}Qvgu+T{l#tKxhCFzu6v7D~ct>Tv-@-h;BY&YZHE);1@yv z=3M0NG+R|+*LujJ+nWmXp_G}Ls$f++8{=?ib|Y8nW$}ZwMH3E)BKRo%61AX?KqIAq)u(ON{kuVqWK|Aua6CgeWo1da z1ynm*zmybluAY5u`9Nty4!?_`X$uNr`Ma^3o^MX{eveRSS2)POx#;Uo$5TsU&Z);| zJs2=6c^2%KxENK&vUT57LizDanUR2zq?+n#M9}H!R#?-bCHdEpgU7FJ))1~Kv{r_t z+1V9uMC{~cds z8D$1V0pw=u)~LcGpMDB>m7|am3*(mAT+gnI)r6V9c8tN@XH|E+x`%QKp>J8wVHUJG*AOS{xB3Op2SF|^Bi zXv8S|h!O5t!NZJ3JK9N)TsQ|J&XscwpSibcBM%%H46$hxh4cd=@(Xen<`$660ME{g z)2*6hl|{c12Rtkds@o+TSD5UM|2zj*e1v%+e(ei;mSj5MGnueT67ZtE z*YVab{3fc@8_+gj+~Xfy-riQ%zNUH<0PdlNSu`R#^w-z-*EbQNoh==w+iN8XoD1z2 zyS_5NWq|>Rf2%jP2mu|rT3fvO-MF7S?fyyOY;%px^3o1>vQyfcc*GMWQ*M7&y`{dq zvC+M(Y$PNo#JO^DanYB&_Xb&G(M}G6bFKl*bASPVd(lIxb>72APWTo-U3-9Lz>dX& zl7kPu+~drx6&KG(k9pq@_Xhv=jO{0Hd(rnDC5_7@H%c`Wg{1Mb7}_-_Jd7d#*ztm6 zIpHr^EqCr}aU~5XqJMmct#td1C&z!uNn;U(S+MET9w9XO6iQkwo=l&TGQt@CN@Tbu z^)5gt;wB2TlQTiGnw9~1!d}PK?#4#!8!6N-0Qbx4vc^lyH@Zu0bCMI$xA!8CO1qeD zoBj87&~u=ts?JKC=}|?-(CjRHnun+Vw9n7}RCJXwC3aLk1hpO*8>}qNb`A4ewXY-C zDm`J~de-t9Vs|aa=;VmXp?l76{^tF-AdUoAfB82#$Zd1HxPN;kf)n`++RWH~^ft(% zsRqM`(BLUkeM~O>u+E^873&>BXw1_bc#m*Vxf9H!vSt1`^Br;lHhuI7`3;?wjT0lY+SMSuCF?bB%f#xWW0I7;x5AnF%o6T zEdkT;M|K?liXXM;Y`Hkz@Rdm<%_XlGgvlhSDA4Pc+TxlC?X!DXAwvX^e3T&ws17@B zml_Thn$>PeS_U7k`34*vdpSmJnl}N03&4pK{3Ur|ajxxl-w%xuv;*kRK!NQOsZpfp zzlB{>YBK#>15bRC=s4For3?AUH8f0>n&oXXw|_=6hJV?tSJXGMYH9o8BVXo1@QACY zmErP{H?qI>EJN}dsB9{A{0$#y%VjD~8xA-LfAS|%E4o$>GjqIM=iqq;%=1Sm97zJ= zvsz7s^`qE!w5cNMYd6^r{m;BvnPmhmIpO@6j=TyRIfecoSF+<|Er3ftgd#kCEiOd^ z+P4Bk*_#1%@5sZ(T;yG9rS#NyGLFVESYx&493;c(P-8}5@a$-UU${|Od&fZ#`zsz~ zk<(%YT^tROK3{iDJJbyu)~h?UShh^SE!O)>u(0H)l=R^77XDM480^nciIGR}+kLU9Myx0$dH}(D%xVBc*Tq@7ICfCHXa9nP7)khJj&W$OfKt<6eeN9VnDCBK6 zgeKGq%X#Z_32!5zE$ZuqgAh>cXYS6A4oGT2^Jz*Ny5PIhu={)MXc-3EtUld61pr9} zbR5&rtvkI-yj{Yz0UlVdVIZWPRgWTxXEjG)rISDZSpj-@n3m<+oM}VDY>Rd+;}7zr zOcr*(3mw~9ZiwLdBuRWLL2`d#>(_lVvV6xCAwCfc&vVGMuRJ&-)*bw74$Y>I#;%b; zMBQwmZ_P8tWs7Z)!BGe**8iN+$WZv&j?71`PS_Xr*6UgOICRy5JXeZ5pV=wZi{`$>pc0baB{r+G9zg{yJ9PTzVQi19zu{uoN^FN6xVqw4}WNg z62bX>`;e>ii0A|epaF%f=_6(OZ$>OeBju4lGv3D;)>pBlXcR0xmCJJkk%2(Z;7W!w z1-UEa(T(s2W~)zVZZuv8Np4zdi1`qU%vo~PanNZ*s@YRG8WAMR!F89RnVd(OoA~|j z0-TJS4f$HiD)mJ@Id=tLB)8}rW{}Z6pO2i1PP>|P+tkwG{G00-S}U6qcv0G5hB^wa z57z^63ScNeFhByljSjdzF__KG1{Sdq*zlE=UziWQx;Vm%N1$G$KQYrXwCFq;u0(=G z(h+gIncRAI&$o|9EtNfr?P|w=Akyht2kj|0zNNQd% z%F8>%FDR%Av+x#Qo=@!F=7>RP2xzQpXuCHxw5tj>u&H%B-_=U$+hq1%mla_Gsk1)L zXZ*qMSE)V(<4W%H&e+cJNz#M@=`CH|AB&XBjQ#?qB26ejg8)tpR<2E&x)ufe`|2Gf#1QTBMhLm%mpk)v5WvU?rnG!Dx<%- zco+xE*XfegLAf>EOnE~pq{iUI0UUydrzh8*N2zoDy@fuf>xy^u%`Ge>fsymIe!cGD zHyP&uu*%5dkCB(Pnu|b>wMsagJyIyhqVVS{5SJWDMmS)!M}*AOL9TBz=!R8+Q)Zv| zWtDxH{s)C49ihpjsg^uNaCk<3qOE$wCT6uzlZ@-re;}?b+b~RAv+#?b_*GIczGLqYOs}*Vb zF5yw+M6Zp*vxBO40oc_XrfYe<234;}r&%G}T<>ps9KE&+02GxzY_YK=I8?3pTN9ke~`!cF8?r zg$Aw}(>OfL|BGA5u8Z%6@p_Ku<~@)uu7sZFcm!&O>GP^R_! z^ubI;eN}8EA30S6#EP1pzo=c?k*>u^6L(8JBQG_!GJ^%GM{_F&`K=jZR5S=+Arn?A zjrt2vjQ3(bcB=r#|Kk6UYhGar2kkd#G6$;Gp8h!7@qeB7xPjEKSHW8TaVf{vib6Of z-t@6Z&|aot5x*bV8>B~GbPtizcn3wamSEy9v}ws2uIR90%lRY|WknGX?^1DKiQYY! zqnZiTfvan8<=fZc?xBBkd*KS!oBgEpo`}UP8LeZB^}Ji7RE_21eS@&m^Z1&}^UaJGQ*6xgwzt0&ik)gYpFZ z&l=sw6E4JGK67SxvoeN#dAKW1k4O8)i!_S#dTXd(SMa`VSoEAMrVJiMFYxLE|7jYW zzwy1pDDH&~V~^REviN-}ZnmsBhHdit){wy9v#!3rE~vd%1V^Be;kN`MO+g95pIgf> zOviM7_@UyXN+yJ?nD`x@uG74^_fV0g>V3rw_#p(zS8Qm<05(jap7DyXQL(Zg3dbo$ zR3aX)LPtq%C=D<-Vo#1c6+zFC9MIIXQZA)x3|gA}F=g07^Yym%-nA#s33WLJ^B$5v zM3_>=U&e0sb6c>0XL$-m8JUz=>Vt+SCyfY=Q!apAYB@d~^V9d|_s|o^dT8Jji@tPT z%0GoXb%CELl3J#wQ$laJ#Mb4bE;+IS$U%R#WXwhO-j!0=;$V4oXZdKKJ-D_#LddhuBwjPc@< zY;LWA#w0lodjC?}1YkM`0|72K;K$JQ2T+_^vjcPZg9^(FjC16XnMO-zXUNem5W@K= zrGEA^Ui`B=&^`*jIGr3B!5+;W^>rxx%;^62tF=0av^KIeU32vRC_3+Ww%#_3$7-!2 zzoMuarKr7w8Z9lU)l}?JTWn(1RwG5NnxT}MQChpi*4m?*nkl75YVWo@KUiH@P*blG`wTp8YnFlB=Bo2| zDyQ!T2L~5-HL0yacfYG#I9}7fNsR4XIXfG@+870Hbl4R!j5GKAXx-xqh=jYgPuw?M zK09LryEXWSg?+}JU(W445wyqfzph~^+SqXS*pbwjv9e?k^S1fmih5_nD(9|Q#8jy7JSTV5RsxzD)cXxt|0S8>~aLa)1V zp;;-}P*7@H)_fY=*`lYbxlqvU_%(Virz4C)P+&ZmTho40)?)pm?GsQrymsjxl!E`p zJp&?hR&St~ZSk$DjVA`r$E{grru^RDhs2IJHNur|VKXJyH2krZU$ps(b?)i}qVl}aqCHb}BrhlFU;7>!M%+m+?FLLNF&D6M zTD$=akYSZDJ0}F5KOG%>1p3#!bzwY}FuO&avRPc(jINl3ix=0mPhqMvH2VWPDHz5G z^z9-Vu2!jzWcO>VvqVKd=WT7bpI)82H(usnmR?VDUG3dD-I8rPbVh$S^f^HI0LcBs!uZ1nmw11Yi{pbc-S(w_I7&=Ot zuBveoeRQ>7GBYA%o@q>cyr4v;*gJa0&>hI?@TU(DyGUwUz2P7lUfAd6|4CqaB+ApqJT2339k6hzsxL`>Ho}WVxd(+HVL`=MNDvoz zH=2BU;G6mL9=Ohhxp0pGpzf!^|Ru63W`anoDTg%!r%1|h^#0aJaoaZ0_3!$vn8Ve=D@QX`q{8=0bWX`Q$9iN@;Q zBdL3E4)=2B*v`%O)aOZk)rmAI|^9E?h#k zUE3%iaK8MrT>#Vpz!&+OCE@u6FL{-z=O^^h;E|%eCf5kbZmI_U3*3uIKT;*8A?PO@>uX zX{ihnMAw#9I5lLl5SENmrsF*o&lGkt@T1V@2NEmP;Slv+$HHiC0iJ@#u`B zymVK6Gcs|SayXay-DlY31l&w*N?}k9 zdpJodhB`mpWJQ%xbBkT*KjR-XnF1R2>@CZ0bEV=Ms;PLiyN!Q#?b7iiG`w1NbSP{2 zumV<4q~!RBVu-xk+sQD4o2^1(s9z~?aB^-XUX~kf#?GygI#73#l`UPF&j_O!K{>J} z?>rJ{_IFPfV!3RL`H$Rk>nm&-C3{jYgH4r8abaAyIAxbKUk5KFaq+7=L-7){ z_>H!8P_Z8<#dJfhG1RBHUl3LZ+X*^yh)iV;N^d0`Ta=+1SCZE zhLjP}XV0u}i};iQdIpV0MCl`P7`#Q2{(r>+E{&fWR>gGPz}QpUqyyTk*(KZ*?t8ap z{ymyfgNu{X2CzPVFX6od6Et8+`-1dL=P?dV1P?Mk^29nH1B;M9z-Z!5ssL8vBs`fz z`L*u^nMbkheIDghqIviA#5M|#gl_4Qv;n1QuN8vT>%odaB(7^2o+XJ11lz!wGoIHi&rlc361J2oE zop1zES0M_<_|>kq_}6FP<=N?atyxy;rrPQ(Trnk^Hk8)2Zp3aBjonBA*14=^f2SYD z^V)AJLI4hn*!KW>g6)6qrUBe?W=};(hEN(c>g$vh*d}39Y4mbIlRBj=e%MFp>d4~? zqYC6R%wJ!GU8n>I+{Lz^Esx4!db_*F+s;q9E?#fX{o(8}cbV<&0Xom620Ax#nAVh% zSOb8UQ~Y?5;h_N{zAlkq%^u;c-z2UZ6ujZ8;&^$lf z8r3)biy53QURtQyTW}2!5JLlkw5i)#kc6f@QU0jBI@<}qXf5JRuR0wr|ty*RUvB&BJh4T?eg#cA`u zmzV8z0@N3)-mj)?cO;FD65y%L811W z&~{4=B7DnU?1S=pizkj@dq8$edwbi|^>qH#D$paky0p6Z4qT3tnE(>W%hXBvVeqT# z!}INzY*!at*YCtE|0w0fbf4IrCrF{h5qGswV+pd?C2w^;#+5S*$kTS0 z_TqrEPDnHE;oEs7v?tgVdAzq=`nO-mtsGFz25H^`V zk7lTSDfo(`=P5ddBqY6~&Am-XQr7!c&EHC*`l(W$FVHH878{iJE>>>`jkdhu(Lo7D zTko=TOZAzKW7^beu&=mg-~)vhjsLh>T?IZJehF5sGv@uuH2inul)3){JIyUJaEf6+ zrM?uaB;)MhNg9*1GxnDl(fP=7e@*e03R@0APq|&=iRl-6dzpWAWluaigQ77%$>U4! z^a<4&s+)cT)DZ4C*52yKHUO;c6D+IaqDNF3ttR+b`0rfpE)?WHVYpRb?JZawnCBu& zURJ02!j`rWh)wX?2=UZ8W)kh-vuzYD3VyeECuKO)pZ{&vr(@O?x8=s=H2~)nT^BN^ zjNJ7|Z#r&6Dv)(VF#svO&XsPp)!UL%Zy87hmeXRbH5ZkV$kGNt5nW92`7m~ zUMeiL99vpHf75V1NWD7RlmRyYTr8j4ug(@0fZs1hVMA7rMM#SVkdh0NgG}AJqHWs1 zs`1cq=iUfa5H$_NR)XICfq2ETvwDZ`yo;(T7@_|3TPMjlR^Qoy=u^!sPp{s(&k2?p z_D$C^Q&`I-v1jWE%*)vnjS^3KA-HUkzCbaiFli1GEF!-5vRLN&^w+FrJIVhs#OC(( zzJO5FJfn414>py!x@H3qO$2NZx(v)LOGUx(yQ?Pw$>I~L1-ru<@2H%4o(?M!O=nA{ z1C3PX=0>1iEn7zm0CViW_3`iD9crh2EAP8noK&=54dY5nhx6Jm6NV9luRS<=EmjZ-JdWHo7U5DM%(XddT*Fp++ zm<=bKIgbtwC_&It=y&MwNCj4VUn1l>=X2Xh&A`R#(ardyONVpE>z#c0ZQSrmtpB!N z>B4Z%=r!@qWc%r`N81%J9tTh%E;I9C7rkL;XTv#LD~HF^O)QD5K8;PJ+`V&(8XP@! zvlc2>;+YYNT2DDyH25m+rKiydA_u&K*;XQXw;Y6C2$?P8O3zkmAAB$0rWZlw6Q|r} zHY>Up%c(8Q%{^?0@5@$FZ$dTHzQ+hkTTcp1;^lLDa9+7$%7ktI%)o)-QIXySYx#?n zOOLiOuF!uT+Yu}pN{SGH3S8cQ-y3dG7q2NzuKvm-C9f-%SJbB{Q+3&_EDKlAK~4HLOx^iG zAP@rca8)@mIQ;JGQmEdtd&?wscR<2O5Epamf7J%8FJORe&0^h!Dd)?E!F%3H$>aOP z{_2e1jdLD%iRB7Crw2bI71DYrz=87YIh)^0dqY?6Wt$vK_sp4ZR(@=F{cT8vaVI!= zUnnlBf}b@v&|qBHmNuGxIA;W_Ar6zy62neUJOe1dA!8jaT5BUww!Is z6SC2*zh?+fPB4&;3S5f|{i^aUG(A7+OkJ{h{{v)%fbt56m_Tw!GZO;anp;>)s=02G zY+@#KnH{QxaWN&-=5{Z&a6Sp?8E?5jj>xqeMVP6H z6gn;&oF8h8zkli+9{N5@suRxFHElIn$Tk+#5SwI)zAKh;D$GPpv_gQ%Y17J&A|L|# zB}$Td!)ziOrT5Yk z3lVH`e%>qG1Bi_l;ht}$xd+Cfv*Y%3%TIuo7>$4iHt?d$?3pUU(U zfIsaas&e0+*>NQRrCV+X0yq*nlR}S1PCX?dc0s;@s{%`=@tTy@xIYpoPfIG5x!Ffd z>bWS%U)GJ&J{EGs#Ow%Tb$CkJe~fB>YZ)h$MuCdGxA*D0_prd)`!?^_en57Y@G^|=2AQ`{61{KK#$nvv<}KZ=hYJ&5q+iNBq>t2q zy(dmTQ351oc6=o_qZii z#1Yr@Q*8Tg9!lowWP=CtF_?FZbUj2u@9+2(o6$VZtYC+TT3#j}wX8SfEsdju#P=yJ zOW%p5OVK1rSN}HO3Ruv)W9dWwsBqAOc2*9DE7k|oN^#pN>2V!V0vbM;R40YJvqi6e z4R`gay$s3)OOaJ3(hruRRV~WxxTSrzn@E&{Tqhx>#c4jizP@<@?cLj?RWh-9^R)~Ru+DUFU57Qo~&*yTrK=x zcp{h%r_^`hVJ4bG`ZU-TC~iDDAQ3W%P8~2GG|*RzU{Qw&%`(o6P!RhRRkZ=IV;)Bl zg6YTUKP817c9tyfyxvfOefA&Us}9)_<;;EMzc~}PRf11ex=Xn@Q6m8KRkX+8=mN-l z$V~^2rcyb-&CmMG&RsJ`PJx(dK$1QkjUefGeNPXqk7{T^Y*ZiQX(ZoK@sqVMP_L6I z)1+`*z{z`;oBG`Yeeiuu4(bW0C0M|pe2hm2a7C_iW$obl0&TlARcXHe_}Q`QBk5$} zWNN_+^Lpd_2=PcS0u8=nlxxaY+`{}!Y{Pfn+&6zUbM%%1kmAah{OMOb3R*L9V{{;E zBU$qOZN&mbkIJ2BR`hD}FH;H*QnE6Gz8A_IiGszk1l)P7na1xrU>&Xx8ckB~tZp4G z_9h7$sT2E&rq0qUz(SS7~`(I?|bMWrgmOBn-5LMz5LYtU{4yKG_!SnIK4wjzK;%)qC|ED0%^@Y43S#X`rs--X~jo^4_I~Gy&X<2p(N%DRsB-rFkrEQzSRm z@J1TeM2NxEw5ool*|;Fi86il`3C<<0drLAu?5bTU{RT#rTx{Cs-HDpJLGP^A|E+_xvwa+_~2t% zXm(U)Q)hhH6ZOaZcp9>?%$p=tkN(Ko*NSoPiD8n7VFEGCS{g-mVDKYfQK@q!Dut1z z5tMQ#kl2?d&7r5Ka~q&80FwTO+0B^1Gar9KKHY}6pP62n=`&0LGhTKIu0(fyXZDEG zy%gGbSL_>V+^>;7keR%^V~&-wBLIt^%8ADCY9`EUl7m+9rFt0^&jbPg4s5Gx9VRFy z;Yo~FrA&SH97HVo1&=ePGpK)DKyWvxN1E-?Wpe~=M;hGXk*;>bl*FuAd~I)k%H5ln z>;hI)EE0-s!uGJdKDPPxnLqMlQe!?d#kM(J>Gwls5V{&&1a}^X-e+Qefq9|8d0tuy z7)Lbs6Eq{Hin?rW4>ig*+=6O1jnp%CG^I(X<=Yv8i2XtO*f_lWe*&qt+Y#P?TE2a%idrulknLYRkz&X$qxD?*^D&pi5e+7{WDs^8dRn)MUHwX6F5GP~96J+iN~q7*M`!6Ky2g_&2?a`vkxxj8mz z`WR6zgph(v)O;Ft77WxaVZ>sK4W2lb#v#6y!8Ibc0gEAmAH<8`!q&a3EuUe6og|m7=B>wkzdr9sPunCk!cS7MR?{B8FKx-KE{{NEbR1Yj;9!Y{*NbyA(DO=$?jUuU@QFucO2SC zc)J*oA(}QGBwl=Qn(5(6TQVNQg~f*dzv~d=)w&Xd&&bw1=pRQ!TKS)WLrLCDOwjlousV!&} z0nBg+^xfjxh|VCfmY0WBqW<+O@UN?;HkcHMp6 zTTN;}fAY=hA@~2qMj=58Y zNJDD}RUTCbzdQ@ys%BFcLzzNdoooyR+Qfw;hDZH=9!VXVQh-6kk+nJkN2Nb;{Tu3f z$Doo*Oqon;{?r4kc1Q2!gTJHpN$4nse+kF;tF# zNHLERkZ=AvWwC)X`dmfKg|StbZpX%)U2IPNQP5ASaqF%a74^qifaP^gH_wv*B{q21 zQ|aigRjH)no3UP&xAQ$iIc_eFVV;(-8w__W;`WTW2AaSnKxDG z#0P;K^?mw=8h+7lf^{SY`>;QoP%7m~*W==v3%$OY1;*Zs@_Zf4Xx4R8swzpr`jyl@ zKb*!%$YWadPc`fxqe_%+bS_r8sXp7OqDriHVg~J2xe?^=!4dJs+%j7FhJ4JgW*juu0vyb7$#& z+QsTt!O|(w>TIloG<@nj-&P0XDds}$eb!>IHb00Fh| zo9uX}Nq5+Q!NTycOKIBPFW*Yrd=y>V^u`gQUgfa&g&H8t((F7n>{xEn6$Ba~$E!yR zo3Vsre}6;-nLz}0{FD+T8zCDIGmR}uQ_W-y;Vepr$Cdj;I8}Dy8)pBQZyn)8C5J9| zcMg`o&HUe{gn#kJQiGUaS{ls!9zc*0ge~ZP`O3VRTD!kmaO~DK7lclz?}%UqHDJa5Dhe z%~@HRQaEXa=K|w2h5!48%hS?O;B0sea>DnHFJDwABy2KYw-m6^cu68YCQfaw5w_dI z0ss`TE+`ol7!W)J?PZah1r{p5x*tiJFj1Xr#fwt=g8^enZma}7UWw*F>cefGd{jBp zuP^I}VgZ}%9snDa27P(sx%y#WgIsmAIFFGaw45@4E8C*qo9%Ym@BEc+{7z&0=`0`! z55Dm6$8bxjg0?v*o~DCT4`>y~r4G*Qp<1I)yj07I9u-9hN{Ul!(Mfo&bM#=|A>I!D zm!fD&KBQ4oE{}StqH{a^y~n4n5fqdCibZXO@9u#Cw>LZ6=u&T>X^y@Ilkq`63D}mh zUPR)?-5<^N{6=H|N}xnK6S_=YI3#S}|JE@wMkl59cuo<5c6-hJe>wWwW6tmm6@frGXyX`>&cYTpCpH_Lnp&?fML8|RbeIuV$ zLG8MZZ(KSPiwg_Z?7ivoO`hKkWQL`{(?6wSUM@?wraiWMB-6=p#E{k>?(*cCAHwTR z@Xv<7{q*V+=)+;Q$ZPbtT{jB(c?KR=`AU6hVSz{9%A1pDfC%@@`}zL?P7oM9Hv9WG zLv}F^PtBKg$$*7$la1xBmFLr!E=a51;2!WRP|)$KPQ<00ZeJ~}e*!yBFx2VuOt?xJAGiRW=QE~))=vk?g4wZ1{JL%tF6Z>9t} zFHh0|;jVW5VtDhA3l%ZC&-LpOWrTPDZ5}(7W(@V1)SG;mA%dHH$(-6*;>$pjqn!hh zieH|_H5}^mfy!3o1&}@R(4oRK^NS%u#FPe&VGf(D8w=z&P`gK+_Dz|Ve@0*z1{jn74U2p}W`_KQ z-=ndGY@d!;C(f_sj0nWm<0j$2k~V#QiS>^qNW6L5`4*7B+N$vZaD=9(ved87%(~wu z%~ZNodb)LWw7crXM$AP`x`mzXTU~4r!;CB1t^?LF0y04{82qj@@ac76Av|Q)YyaA! znpWi|D(Y{Zn&?1nJyHgAHy=J0zotTFuDVClQVC|V#@kh<*n6%Fe`y0i%u%H7y_9>rS&Byu=}|_zPk0_ELZH- z+(9p&)DGnmo3?|V(iY;y6BUf_a6_E(A~tLj(7(TJ?d?ex=b|3o{)MU%Wa8)FtN)h; z55tt2Fm^zl8-pk?D=_hL5M==Qc6g_R=-Cgt5ZZYLXCawVX~9Za9r&8OE?%0R?3=zb z;9)F1$MyD>UY&0Py7-*N(nTdB{qT8`hqvJ%G2tI_Bsr3ynLrLW23)cd3H5-|oW`#v#=X00pGj^?ybRzn~;!2XFhdrpa)8P`W% z$+qVIN?I5MR#InCewPyFOUH5SGSlSGLC42rC-!kY2qT|BKaLy&ovQC&t2Tx{u%WQ| z?%Ug2oNebvn?MKFz43NODsUY0Uo&~ZxxKAi6bJ+Zy+%h(nSa0WBWMr)#gK0q$>J=k zVJEM2s7K#h`n2a&HQ`sF(2ES|8~o~tZ~tt{n%4N|$bMsC|ENa3xh7J<>)@rTsrCV` zL;9*|Hp`*ReimgsLphTu0utwbK5|;<#@b)|*zyz0|0Blp@_<9?OoL3~O~6$|S1uZ0 zM3b2SSz^f^y1CyX)>p5Mtg4S;U@SO%KImOz+yQ`-wvu}B3r_C>w_ADOk0`lRLNF!J zV)DYFtG|B>*Bf*;0O}tA@WVVCX>u47LibgXEJ4?+^UuA=i>Cta*tp6f3NqfLK*$vL zL?M4y?+wlOEBcJNEJlD!k@udb4ruBmbOI=F9R$pG>Q?FMs^|Ie|6{hBRgU&IFS|Dv zhsYmw&~9ul^oh?*#gl**!X(}1@~VkS^`0sH_}1gw0#pv?Fu;vThxd@N!} zXbKvujfJ}mLpJoKKxNrG1Cc;?DR%!fZ~d=r^h*VCAt8_rDC{6V+h!z~Se@a_eF4?kFRINq+g5r|u3BjvkCmK;;@Ft<5$cbK?53M; zr$0W7>}*tg^-=D*1oVV9?Tzi@@N@j?;$|t3D}%(iesJQzMc8H3Hf_>c8p5g{jkL{b z%etN0M~BzD=@>IoWP}wPR@SH4l)vOGt>&6(06lF+mL`b#KviR zn`;HChc~D-c6r(Uk|YIYn+F&$N0_E$bk=QgxUc`E%oiH^Iy+2iXk_1wwGooZYq}-M zoe#S;LSuWos{~0F8>*c3yQ^e2q`CDPU|F9JZ?|0@cxF}blkCn%I)X+;CBbOsTdHc- z=09x3WS|o-uY$7kmclA-AiO#GLXZdKexjNGum)#A)?(ouz&W4x|4?9(y6=&(3*T z6Sl3`6OO_{1Rahb*s$E{>pAh)I{C(ShwLqGm${DXHhO7Z#K z@!mZnrl?lJwjW-4dO?P^dy0Nhe&(cZ0!vA3!hD)3|LQGkukoU6t-G773N^b=1?&zO zr*(-&?@Ivpuf)0gw@ZMp6yLAp0}K#aj~Sep%4tWS{3<&5RX`gcpZH=}gWP~wjzV2n zOL7@Ggg1sUCvrw0#%{lGCH#0oJa2e^X3ZXh#9)3%@;$NXm=7k%T1@HyEo#P_4H=(- zLbqBauWtb;YvioL&VdB>{e`U`_XR*p5`rS6nFn{ch)G z_EypFzk3*GB zDo5%nlb}ninak^++mDn1Of!qDRp{w>j-1C?TiDky%nHzP++B>tFL>c89NY<8jwBFt{y z6~;-^Q?E_e>-=l}iiA|GAcss*BSI?_6q?^XCvowEG2KAa;1GRZ*LdN~?>P~-nHz%+ zZzs-T7-kykFKdfC?VZnT#H}X>Ru!g581DaHEf zENM99uQb39pcL@6nj2hDZX4R#oGEQwAaE_)=kcrV_|Xc-W#?eptiVk zVRg0a5Bw1>3CE8~Rp|?p<0XywKWTiRB-IzpdPb2<-b^n>|Koa|ofyn$I?U3tIxC;_ z=)>G|pIkT-_~|d{w>F%>4rRcoTlzxA3jhxf4{B#7JJr|T1icVoBry{uWjj$Jn!_rw z#cBNNV6cc|D>qmR$rkgUEp5eGR$SsDP?i&+Z}F4@b8^1ET|1l?eh)yR0qcjK{IZBm zzsPBavf!i)xb_<{AdmyS>W%DjgoTLaKTJtbaEsA*7FlTK${xIE>UNd$0Xa|3SAE9p zM#3Ntr#Cs7dAVaCjp`}?VM=ZQ`YFKwcxQFIqQUL+ zqW|%t@z;ozv!CtF8MD#o0IH&6t6(r-qqPY~4$*IK6Rp`e2Qw%0Ig)w>wro)24Pz;N z?sBwby#P)GIw`sr`e(y*OaAHVJA>y2+|S7+(TPRU^<#=eRc9@!iY>W_Y&A)e0Ef}v zQ9A%v;MMk@*0A$~gCZ>8I~xzhA-j}Dqq^v9Uv@7smVaRTNSu%Vg~a?K39uVAbsiSH z6_WQ3ebU$Msh)=Pt~cC%+~{^LA%1?`$VDGZCZJZX$}USNPZJA0yL~U~StS0pL|SU! zaCJI<)oUa$w?d%C!*g8?nW{j1yCj_^SWHU6mM03iNz{)$o8saUc)Hz=NLV;&iIfChGbr1_t9P3QwZa5GLtVq}^{{-ze1;_!a%^W}Jrp zn2ZGw*Gh4-`4*k(z=AIN(@oLhGJ%4?iD5J)1WfTfR|@}=Hre9eO4BcmNBR`4Iz6Q@ z(6?c_;vyw70R?Sj>ipR4av(Vd@0G`lVPN<=5Bte8im3j;`zHe;$n_qT?(`{y4OSAR zMm+*d;^`!?Hm;5KqOrZXM9|-c_jh~2AWTVhwy6bB0SM28J$?uUf*yr2!HVpd9&In==W#PV^4|5Nv?bh%HxP1!(Sk zecfd|Ue06g8_Odf9ekOyAm1?R&v9%wV2i7owKO*mK8bjH{nEu6FNK#m6|XhZY2dIC z>F+UB&w}serPQYZ@i$OcAB#pe#k)3USM0j4eZK?JDYjgc#FOI|3UkU*o`6wZL}h7N zeJV<#Cw`FQXC~0^0*KWK5;gvP_jZQfMx;+9uFH;zt^y(0#Et*rGSWu5gEqe_rdMvIL5S#WJF8U{(I4K+`hod|z*%#%{Uy8~cg7sqDU z-Bc`eOYwRtmlwzF*O%T4$T$!~WWG@wgSeE$C`S+mg={7KwmXipHfcw9lqEW7c#a2x z9e2zpolT`PR(IO*74_Ha%v!hCt&HpBj;%k+Kt4MX{!tmeOAN`tS*IS>CqBMASF(=w z)qiKEa~n7wV$Rd%&LM!15267dz%!N_Yz%&+6frZoC2n;J)2p z@0OIOg{ge)+)ksRjyhf6zz(7%22IHU1*hb`eg7KmnsAZE4}K2jHImh-MY=SuUM+B4 z%}=MN{uQU&l}(r=W)k-YcZavH%dmjZ8G8X>{mK0_)k_AAVA&-n7ZKEHh?SE)u(F1!2-D29sTly`>7Lz0$K$}f4cRHIa+?={3t)#lj z#fpMNK&x)45IAAf%tp!&!U`V=ewWJWu8FnrRr_;165X_#9))(5+wsG341!8rtXxk& zn}&=i#Rr((v4`f`DRee$e*b9fJK3iqMeGAe{O#&srWn8;+{fd^?fxU-&%!B6tv|BkiQ`c)k*CALT;AWLu&QqxhxVe$ zP2!&C7tjG6u;!8D8?MW#A^2#F?BhI_H`SQBCT~Q&-+Zgv=S2Qu)-iJiTs^t5qwJ=~ z3JQ&DS{H<-!;9|WvRG_IW#Oi9A;D#qMaEj0F$mD{DJgDAzulj#D< zxv#jSpXJJt?W>t~ybfGkYT z?S+gEQ|Yc>)0`+U4yq#JCqL)61)tVj4}0ju`{QQii8i^(ea+uM$#Ayb{xp-%lNA~( zSesg6`j!nbhYBW`iWjGCBvgI_N3k3fy zu194dWxyP0IqRFB+-;tb$7B2h(}0yirZ`Z~7u;X0q{NBEV%IR}+~W;)k+%a)v=8{w zh2@{|0&|Ku%|t=H=iVkSu(7e-n||GaZ=Vb`8TZHa@te{OhBDc9S&TS3Ji$<4Cu-N& zC4;s#7@0WaOCu@e55D*8PR1%4^`>spNyo81V_#(G&HYvW`SnfJy()tHS~Jr z|L0eo?H|pK5NmBW&N&f=S#fCH(tYXF~gX!&K z?JJ+45oks#%NHudEcy0BA#Q9Am1TG#lqjDtIk>zWJmcO7V5oSM0=B1WZ_}k8^DvAS z);_t*K>l2!jV;KiglYhS>TD_eBdZOEOB!Ya!;vV~@?RObmF9BWLKUw+{T3?OzH1Bk zk=S1(_&s?W`36l{1QMcD1({Nm3PG8fo~NgkShK*i01v%=WzAVuSCP(;t*8cf9uO{H zDnN>GGN(lUlNS#pvI81_i4_81Z|g@J`t2D!8~Oa75tf=8jk0)6=`9C&m@jQMTJ}Mo zQb`bme*BVNhhwH>Yo?i$1cZ;PE9yBA8QPo?u|UDkpP~Nvsdz=CESSwg;*+6{W_*Fy z$DX}C&2kVFIUrxI>Mq$7+TM6!jl>Mew)mu@(X15%Jdio-sp2#i+q=!@4;oKgZ@WUE z#DBS`^E$hc(yKI;>DIt^+7YQkb>O7(w>(12uuPlWqi>uaUXcU zh!!)Zv)B$$anveEtw80oYCJ}Ic+6<_>V3RNrY&{jM-hlhY9T0bbJ3l}J>-v7VyX2% zaEF5ij2T4E4elqUC=_D;!O*~`^Nw-6D=Ja*p56$>z%r(0Bx;vq^NU51#3?Gy+U7mFi2bytkqV|uTGe-ue&8+f{22OqhQN^itQ~e z>`6oo~Q{c-X4Rz5~ID$#?doD#9!w~qB?!-?Kk>b4qEY?>pIlmEXkEmRqY4XI) zCd9h*PUeNYtL+OmSEuHg@U@73g(-m)l@*%Qd^3lQ6?XWt^oGWfMfmCL?|hNde$X?e zw|MwUG0$U?s)qC|F61A(T?8ZFn=|YYLf}IaqYYq=8+LyXK55M!`_&0KnEEIC_v8?# z*~`lGJg5K8QQJl&Z~7#7v3#=mK$n4r{_555LvQ_HP-PLU1_Wy;-AIt;FdtECPFGi-%|-!s@_;EIwgFS}=v&=-$uAZ|M>z|G zkEL_w8>dD>w7vvTTRbB)xCs6QkW(~e;Pyw9m&-zYnQ)fQA1~qWdn_HlI_Ukuj4Mo_DEHUbzn8dXtusyz^xIi8e$i~pgA4=?Ku zAOm9KlOOkSyR0LK7eXayv}l039mFy!-#e7)cYcZ1v+7INZZ~m!eJ}JQX?$vmu&}fu z7i90?;M4I!ulZE{zMQ{wIKC3}S2wy`RmBd}|HeIb;GGd@XE{Uer6Lr}alYylPwhqyx zUzd2bA(2=o91abR_)um?XzL|UUsO1mXDI*;b(Lgn4EgKTuQ%5#eAJK&`rbYC&NM&{ zZ`MEA?gVdy04?B86ru)o0p{74w-Z| zWuJEpb+%I@A1WzH&pIHwjJv-KCESsY8syi@8VmCKnsC#EX5h*kkxo@ED(W1n_?jWa zv(XGdb}B+LYeC8WhpRqkXNT*{ub5NP3!dLI`ICzlCGKy&GGr1dUvKX2-r8JTy!tZh zYO1~wyWqWPyEk`UGP{)PcK=DjSe%yGn#)I|--k+eo0VKt|KEz2t7oQ}eik?3N?icr zJHI_RC6#++*x@xgbC5!-iAo*d_N${*%qxD>`MyV8t(I~Dk&B2b?~(5;Dq&aby>sf% zDb955TqE_R2u-~S~z;*$DF0<7TF8_-NQ8A_Xun{#jTo(w%X zJJ`$Fo-3Vd4?17)Zrh}zpEDXQ#i!N0Njqy6iXOCRsbMaskOOCzyy2*-_ZQT-&7z?W zPh@{6A@QF;ge;T)+dpoY%@X9}oxwuEyV%F-gpXfdHoiO_IAinEJ!@-Jb-rV(VCeFr z_HNlT--pCjJJRm`NOe-vD`d?*l^%0gfqjlQrR6<A|A`m62{H(KQ|c=eys!>>A_L@d@a0(WTdjD_p@RoOz z{U#(aVg2@Fc_ZV!)) z>(svgOZ&_ixM9(_CtWV(D%aCTeG|Twq&`OS!y1WwN_R5dPPM(MCv=_*i0iFE3g_G( zN-65`L^V@c;CKMnN9ju6Vvp^`iKR z8eww*kU;ozUeNpy?egMoZam$KmbnuAp=?Pcx^HeWU9&)Ay|mrJCLxn>C|e}hEr97@ zI#f;Aut@$NN9W;4_5XkIOGxT*+S7j!Rtoa_zlJ_TDqXB_Z22%eq1c*<|l@ zv$|OsHzC<8{NCT6KY)92->=v6JkL1~@4WHT*El0(095rvII0WF$e%>*oo6pd5mR|c zmNVDr?7P^k_XL&3;I>~dewN^P&j;fzX0cVtWo4m>G4jGoy^eqMqLgpQZFKfz@xNfR z`0Ckq*%nZCxb#P5`03eU_ulQAvg<3QPUrVpRgK>tHh z!J)kdxBh+BMyTRC6D|yM@XBM>yQt$a_nSDxd_q zA3EiE?P=15uDm+`>!Qf4u!wY#w$c@I-iog(Dy86RH(TGHd`F~wmzG)Ii#j|wg|qBg zIj;^`(BkLYz55wGauGiQJ*h7x7JufvP;L2DN}EQiFcz^l)hguMdwP7>dB@Z&VlT@0 z(7JK|#!Mz7n2OE#{)3pg=wY=h{Jc*xSCpW>TZ#C%u{$o>13Conzl;*0(XW|MLODIz z(UxQ0iH!PfTk`F_I?Z2B%lhw-`yc{wT9j_tv*h!DT_SZ|Sug${L|w4bLj7Drbs5c%Ek~`MT)z82GWmo%jCb z?XATy5-chBfBqUy=tP(hWv6soKus0AL@!lK%DH6xfy|+>if$2oL%(^D4<(#-?zJ zur@ZAgjMVcp8q4|SpoDMb>cl*0Md?$SU(2U>%FGW@Di$GHFb&EZ8C2XWKgjfykc!~ zTeX-h(;%{vKFck-MR|}E1H`Yp@n#j=*@Ch|vH^$gVh_wwb>I0aDhT*#;_P2I?*8)JiYL zNtdlabkMj$H_9VeTk1}Sbq&hUUdtj248GU-aZj#g|1pT9-^@^*(IGjCdQ&|GJ$~8A zA$<=z0>>Z`bNKUscB{HM!8z{Cw%X+m2c<00-#K4J2BeIz$~%MPkyrbkk=J9(*XNu! zE87BPBpMFmbJtR38k6n%nNB6Yede>+Xh}Cq5^~>_$=+AqVatM;f}+xL01(Bf#QJha zx+=_jwCdTUMM*1@w@<}sEVXb18jtH(s7tr zn66%Xy4mcm<|9;3o@EayNnw8Vd0Jv(?Rpi+%UaE`bjy}k34RoKawx^8GHxzWoj`#Q zC0%}XJ1XoSn`fVB`HJhr0Bp`LzZ{#d|Jn%jHOC%08^xGZ4;bWjI%8^0w|4`!FC%tt zGdM}mWub=;4}mbD)!c$~)2znwY1z2#j&_!57P_sCDJ@;R`1_%-np%17V!AI{UTuHo z8-M~S?e}h@1!Fgvh3DvrPS(7(ho)?7_y!cWOugtPSH8e*GdpBpn)AQI&JNDcyRTYq zW{v*s_gw-KMKW$bC@fg<`0Cl+H}>X@N^8%ezx24-46yMGl7Ti+)GVjIG+; z{MmoC9<40E$xN{p2BK2WM0H)%vRSRZWTdf?j$V2}I7xCX4c%iEENY5ulK4u!eF0r+ ze6`VxS|no}F(-MKkTCIg>b$iSU32sSDhk^_Rk$xBiQ&Lj6Iq>=Cvppq6 zo(K4f9;T`Gx7FsC=a=)4eyJBg7(mHE=4P;T9KwFt z%N-oxJOlqI<>psa^vc$=C3}yoxdxwWN3G+?{+)NIo>9>LD<^b-*-#(`l$uxGpvn?+ zdm4)cmE{tOck0Vq1E%n{^_8n;gJVB6Xa@eo);^FJqpyZ@xli?&pSzR}PCrPHVXLvZh^EH*?IWHx6PocMc@3R2#x*8+jk{*9{C zXNic@tv#2QS`>W}qzsqmn5tOY2velFszjqudjUY0ZwSAu{vP&PsxDGqu|gj5JHLhI zg9PKpahv;P>sA9omCAy!!DkQ07UG)p+B0qi)d>gFXiaJ|Oz9e_89#02<2lWx19P|9 zG3UUHgkpQB6-uI0E?Ybci;8rN=|F?tADwEoaF{_*lWx z4u_uK@Awf?y-6LM(Ge1TrVeTqoR%5DY~;i4L5PE{tX(reB;XOz%^Z+z;QaiArE%nl z>}kU_T8(=xwZJ2M(hT7}8N1`Wn28}tXaEsB*9pz}z~#9!JZ#>*xj)sx$NQiJD9kml zUqVP{j*+Nly$39=@$rG8DeidL2YNOY09Bi<;$~|9>;S)fEmf_kR0I?d`A3zHXqC2? z4xKvfhCF*3-BBj~m%@Ic^*#fLEEQOnnRa8Vt4C4q%8HHsSK6%ghR6GCxbC=;FZNlT ztjxj}c|>>Obri_{eDw*fsa>7H5}q^*Rp-81R{k;O&qJ)P!l zDglJq&WWi&t0zM53$>a^Lhqpa^|2VcZ>YzI9m9ZKEsP_b6neWf6!Y>R^G* z)uB#2%mm&eQC|PN^LgUd4#xMfAtLtIXIk9TgL~7ZtZXgO+iVXQ$a`Po$gx(ac^Dr@kDnIwmZtk88jkEQ;^W8mo!C6MwxlrDLN?UxuT&64Ezx8YXY2J7z>E30@iiLhYV zlk9r@p92Z!7A=p2tA5#L|4Ki!Xq=rb!4lO-qhsNqk)f64BX4!0qcHHPMu3rJC;<_y z;WX|vomZ+vBk7qd?MH*+_O=cSHbSzU}^Z^cGt#GZ?)+n4NyPmUOtjO?`KCPF984`hn~$ zK)=x!x6X{ggKlM(*bf*X1n`gKKYqMJXR5k^AuNzaRc|BU+L6vy^44?i2p7%jg6BeJ zYN;bn;8^J{N1}0<+-JTI&t1X`d<)ZSKD{vF$;33CVJAqFipd318=!)zS*8f*M|c2R zp|cCFh#1AG;x;iA42`6*+N5FX9aIqVor1dO)ur3D3%kvxbG6%*9uzQczh3d+$INwx z^m$Oe)YzUS6LZj=A+S__%g~wu$?uRr#Ud@D78zzSjl~9bQb7G5TAL1nqEG-T?CM}l zxB>_LEjiFSToQbzQWtl0@bBn&Y)a2YDO+L&px|&WDF6`qpi72ORo2;W-m7F?ifU(B zLVAH52?-3(UY+>N8S$6(XHU^k+2PtwmRrtzb6#}Q`dkXCI{ICzeJ}EC@mS1pdOQ2Z03n)1 zEcl8RF2nJfqId`E(HMH@+=8cB3zD}nHlT#0^SX%b(`s25%}^*&c6&)DRZ!NC1Tna# zIEeT0)iH(*tY?U^zUF1T+p%j~Z^e-G`YuEzYAE~}zIoA2g`BCuqvbke7nPb@jeRU& zI9}meWF0~hG%qU0`5O@(_ciBGM%UpV#gtwlyPSl#41fTDW@E#hj1L=#oOP=H*qmfT zc?~)84p|xW^yg+Mz<9S@sDxSK$5MM&$6H)U_ykMD{*qQM-ir4jq`?=i?*qIWOrkCG z5E3DfuCwKDed&N#%RYal#$!O}t!p*w8!u!A(>fNU0tu$f_c5dlpbHlg&;yPU(L;gw zhVqTs*#Q*+{r2DZYMt!M&TYC1-FYL7517AQhvDSB4`fslLdN;dB zDPBKG<8!PAc`=&zp%|B!P%v}6sB7cm9w*-PUH7Z36S0{H~iXuloPU-HXmc^qFb@5c;E@ZA%p@ZeND6Qv~m~tP=o+)Uqb82 z3VVnOA;JCj$Y+r1vrJn$rd@g|FEuhm{2UH9*S~3a#>M~xIvAq@6$zGN$5YG4)aD`j}x(JPD zdx##q#qPtfL&*$C!B_m@Kd&2`c8}7zuW>nyjJ6*0y0~>XYG#QdtC!_Pnq5`DpyUVe zfCS)#aQUSmny-Jr;xF=|<{t@^tuxH!bS(8XDO71a#TT|DBs=&C zf!0fke6l9UQbh*+2W`lpA zo~Qs^Q2^&{)3qtq5!nI+yY`k8{?NqJ{?0LEL5_%5Eh^eyeeNb^-cvZDAR8Z{@-~nw zcxxaABE09Zo&rBbtJ%@c^Ch><|a$nWellQU6& zbGP97uf#+8)#){$$gq3fY;I$>Wu1L`AEL3khO$M%Yem-wr<_VnfUm>?6>)mfr*uWQ zktmF?^W)_s6ZBT0UvLI8?uQ%YZ{WYM^YMmZUOM z3z#?lJtAv{fA>uPB&k33C%;PBnHK~dAig`}zk5YRYQZKh9spo;VZy4%5++Ddb&HMd zA3ww`rotxcd7@T^6Mm%ct%x#!NJTtU1&|buR<_OU+m`caWpaKsQlFIcLw$&lgTUd! z$z(Oq9vxePRhjwb--+K{>Il93`&VE3)qm??aS_qR5sW!CW#D2Zso%e25*d&Tky-#T ztM7C0?{b#%63&+bpOkLSr>3TCw%Z~O{|-~1wol-tX~J1vMtpQHXrkeR=}TYkDROX7 zFJ6BkC8$Ny7MxDL!MW5h%}z0aAA)bf-w(DIk_as;GZh|o?$_=Ac)$EFj{iHc-6Q)3 zPZEBZE7*pVF?e2yjrx*#$nINh6;P5LrR}W@0ecq)(8xh5sA^C4fKr(5o6S=xo$1$C zGAS)3H%CCW-9{1R@7r|(TJNKTBtby`p8g}Wd4!>YO!nS-|U%6`qT-SpD63Iti@J zB~l!$q*MB~{4{yv^(o=eP7%!3HtVVeK+;TmO=7@^HZ}9=aIA)(dBtR=<`CUK3Dflz z{M|Nr7fNh6`i@vzJGnPN;Z{OcdU@5^XpKks1pVIj?@Lf6R$D8i2CZ-8KK#ySl0{K` z2!ym(MiM$0T_I+fi3#j1ufO-4AQ8#;K{e0RCiM{`ENqL>a>f^Fr7^@r%$io|P4l*! zxQ9_fJo@k^BL;osC@(+}p6v5#&YTsYjUN}&mLjc+O=$eWE39JW8{=%0Bt|*?^J^7Z zuzN`z^fgRaoL-vuK3VFjsXHE{>(|AmHClnwLTv3%tga2wsJD#bDs$=xW9L58mSHO% zIJT3;)9&Cosre>FBi*FR(zTaR<#Jb+x$(ayiL;T+)w`j~zKgcgDI$FATrNh_YQy#= zn-foXx&_p<`}kqOgtl7BSRG*6X2-T2Ak&`{eHry7O3AqK zy`fay4*IIhH({mXS8Y z=p8vfKhbxO0Mm~2dF|-aWD4^J4fQXjrbn6Wm_)aNFg5XM~ z>SpIt9vo;Wm-%7!$o!qh@N6Uv1#=4eX*llH;1nh(aD7CuTyScK7 zIPKfh39q^njd3{PSfivZ*3lB&V#M~?EiIiW>lY#yM#iGFc2}_m@&mNw@ z?$r_X;Dm%<=jRKB1;J0PHV-C;a1p1)*r7Eu_{rrXT5F-~@@+1MtQaj#R+Y-%7aNC~ zU?aO^(Wjz-??u&0FF3)QPG^`p`U`CDD#xQQm8jVL7F}6R^att*k<1L=QMAmIF3ZsU zG3}BxIjQOHdvc~aq(}sphG`TtIJY-+St9ca8%JdS!ESZq3aF9zY2a)b_9OY#tqjyA zIhf^6$EkeM?I9Nx_@+pTssXLCnpCkJ0+#7Uf8hK!_6(azz5>*1^iFG&Psac_>v?e& zOswO)qgP=(!fLnLK?3~nHx@1W zQ{G#$+KX;leoyBJ=*#*jI2+6Rf_WIqVEUM=*S`D~3#R3(!Gtw$sN2x9;sffyXMA8-4xqQY4C&9H#y* z11XT3d)d`u*ggxcUMt^d$}8`sfH^Q~t3Q7(4}7^v-D}UGm9I~9V}k$J+o~qkqe~6W zOds?JGFWz-rXOYWV-px5f#RGLUVhsG9&$9U7p8QfT@&p7EE4#UdX>yiSSa`<(w!L}db2fDlrr>Ib$G2fxD@$QW}|$mJ6xWl$f|NF^0a>Y zyWOY%`n+Tdfcw?uf@HCR_{@86=H;&N&^GIcf9Dl-!b97~tPUJ6dyQRL6Pikx3|=^B z?lTRI`Fr(eJ_UL8#>+-MOY>7Knj$n-vT-bXNZs30h@(g#w?ZV&y`$`*e~YW<{GhU@ zz};3>r!}_nYN0XQTkeW4#Lo2BZ>Bj0zA^?w6}XMb%Z3za$R5~~)a_sGUPXrQ8gmBv zw*(bh)JYSn2bEisHQ zdsq19vIRqg9W`|?EM7u&s@!YyR{lzE_@ZRc{8+l28JRpn^VBJ+z#_>V_7v3X|10WF z5`#Y9gJwNA4<)&VK(j>Vd#XVv*|MngVKK93_nV-uj~R(a9=h2_Xg$u44xBD~1Ljm!YE8P%Ll>z9uXJ?<>^V_NhuTR2Y z*9V7yadY~Yj{h#3v_O|j9-G6is#SbOoY~VwC>NaAYq-vzOOX9A?7`Dc*{zu!d^72Y zYX%t<`8A|`?&9&?&$|?h*(g4uX>Onu+J!Az(}&EkNC%d(mip5jZFD{naI-nlw||60 zKnN|1)Fckcl`CK9>#+?h3HDk`I>&0HmtEb9AtxRuNhwq{xj#ix!j*cP@wSE)GTQe& z^YdsOFtn>*ZQ6DLSLy9J5?-$q`w#1H{38`}^&?)(0N2=w+ObPvU-MzkWeXMQC$B4vmQ1{U^Qumel5zSLY4FvdgrgQaXnZJ4X0KfE$Pg91Uqp&)8qSM+$Bw21I9xat~Z_ zsU4Lk$H~c`7W~H6HlZlu-{sgCJ?L9+omDQByP(n z@Wp51{K#iKI*GhpgptL^8HjT(`n7lMIVAJ{>J9e$JPJoWv;ImD-0*21%t6tn8ICG5 zV}x6&>L0u$y$>E@AeOf3OSw4o7hthlmXw5)rxSg2!DH{#^AOmhx;o@w`rrJU*^2dYGUxx3pNB=JS^NMCNVGH#{G6<+8r`oN*w*2TwNa zvaGFG^VeQcXx15H;2$cdSTd0>;SV+e4@F=Y`oYd5@u^0(o8GD;BH1^v``P|7i3{R6 zRIr#5J@PDM-rE-PMvlZ8>|ClVp=8Sb#X|u!x#C}}spGTUi3i()71GjXpeS~GJBPUy z6>#qIGPwS>8ocAbH9E#Uhu7~QRnI1Q?=bNw$5Zhn@Eh;P#Qd1R%=n|`{87PBbR5F~ zJ^gbbiF^r&EOnPp@0&I3mI}Gf+{?Ba>}Xsr{Nwk-P@W1g7vjaIVGLO43DtZK=~YdAPLQnf+dL z@bLVHP2JToS+DdG%6(h8JHtnz=&}b0)BI3S)K$|ZxZ;gUs=zRv#OP|Q2R19(eo_}k z+YR&_!vG)N#`&R)I_PLAaF>a3(0k|do*c=u?U43G_YQ*gaYPF)y-4|TESG#+R^$-Z z9bWUidtnkgX*Fj{?!ZxUE>F&#d&KIpxw}MQpYMpdw zdP?1S=ysR57#T9f#=Ni2a0Da7bq+}eT0SKQJMc7ST9Q}nr0e8j{Aeu0 z7*v(9Rzy=$|ro zlk%+16cMJBAnKVlGiTD4-DVOY0wWAiq*l%~G1B({$Gfr4qQBLQUss+<%%i0zPu~=_ zqGUp4Ll&*tS*L&B9u-s9f!F3(2afE^)Aa_lXhIc~A}3thtR{yD6(ve?1dbD+z@Tr{ zYehEdpi(^DMFl->$y1KBHH#|VG5Wypp|%*dHq((fO#bxaczf+u|8dA87huu&WsNY4G9XB3H!Zc}VB?(O(&KBzp@@{)VDW{`8l4y`sYJyB-<~G=MQOGP$aOVamTruRjXjZjw&Axje8f$atU zkNvjpnHe7+9XSRL4#T8O1t29VD7K;f0G102j887m<>Dt>z+mbom67uWcA}V?eeb@^ zV3rEqBjRLpQj`9KTfb7pc))6ca}N`dk*DJid<`9X?A8T5IIXt3Thfh2MHmnVF__XbYRKFNXOoXCw$2+!>F#@K1~ zhSwFM~9Vp~``5WF*|@m5^%n6$q z$aw`twzgW|@u+9GNM5x*H_Y3z6as8y+Qcg7_=LObWMQ$i;t3^tnt{d~s=WBlcfQMn&M>vXK=&psAW|Zh|@faQFE?U}MqD8Ok~o zs_VJojA+Cfqnhv@tJEP9+z;+YD9OuzDJ`*!=#`?-+sF8JE!GMP10tUi?<;6Uhp4UfxzG0Z)r%XGOm_ zCk0yCEIXreU!E?3(_H+lbsL}IHhj(xi*7pb0ao-J)a`+u9GuJYZur4a&-N6pH0Gg< z^mQbVfZOs(yRv&BVIj;#C#y_Xxc&UnUj5MW37Cq++{E;E!TX!eC&EdU19`SUcX)V0 z$4;bB8CBnx(@yAF$R4{?ez`w+8AEjbA-QSD(B zzs!O1N~+MB44`t_Cwc(j82qEOdmf#>leeF@%aRoNc329*2p=N-Cd1GE0>vp^_$D33 zA7eNZbvNmIK7vtf#ImQyXa5c>rbHjK-+9;?Nfgl|_FGav8yN$HfC^Q5oWxy|>-R33 zNF5wEvZ8H39Znv>8;B}FTvm8lb|S01V;sT&NqBvZ6-a>$8f_Qa7ukeI_A z3UV0Zm^h0OD0Q>deyqT4K83wh2HI0f3J2Mg;ZVVM1W);^&IM77EOX))dCy}ia{>lC za$MM+#8@iZW~{T9iZw#-hrj8cr2Fp>LelT(@*G;H{iO5K&eSiO#4Cpk&ti8Gfd0i2 z2ht)N2;M5vJQqx5EQu;3+v#|aiu^bqAYj|DxeWYH*iTS6P%35Ni>C+5&tQpI%y*4g>*I#B7^xJ_3Wwp}XgU*y&5>&UBZo11IHkZC;OcUb%IveXdJjd3=ommTkZ zx!5l4uSUQdMk+Sz%y@9hneevi1S4PPTOD?WG9TcCyTcT%mG%JW-{L#y^}*O3&{9Y6 z;Y+D4nQQs$D}vJ1;@pf!OK8x|t+{q1v~n)8?Z5Dw18CNJY^v{0=p#Cn%Jx~H@G(-c z$g?H?_;2TF`=U_U(Z7xA|L`(G^WM&X%WfRceam0XH4MDSKs z8%p^Br67KAN5`#(5zPMV=kR;fuz^*#G)K-+qg|RH@`w%h{n%tQ?j$ z@&XO>=2kEOk0S4#n$m#DI9UlH5#O)<Qj>af%}gk1w+>5}S@M>Tmmm2Pal zJ0h4V7URU~V8HDIs>(P3_>~0=$(lnYmY0@@KSMcUMkv)3CvF7z_xq?PzFKZXcxMh% zDDN$XTzXlfdP&wcfzSmXOR_2DN;nT1Jz5HP8cZWW5S9A7Pq0~ACr#zSdkn8Q;!wool^Nlg`RAcrM*L+gk~TeS?~)@diC%=M0~>Hr}p^+jTDHAMo`O& z^8_=kcW%XXd8y@1GUYQ)V9gRu{!>CF4d9Q;?2Kcl>nG5Xy8sB{n+~ORg4(J z#^lv5(@-WRlz2?G8jS4*$Z@{T9&=(Fs>QaI2WY^`q159$+vo&jO8Z1s!i8NPl4I@T z@?ODVpGe7O`p{~?mcZOhFO!ou4p6fE@Hz;9Q>%wtufCv(Ji7%pnT*#L_r)9>`@lF} zGZ}C$cQze!U5NnG7_hE-?qI_m`vDl;Oy-++fx<}@XGWC|d#?XsneSKGb$4a>v|1u& zBiKAzED)q1q!3gSqkPAxS5(rfePM@Iw2Yx5y;0j>kh_coGCHoTk03_gd#p;l*Xn}Z z^)M`ZHX-iI1V5C2s{%%W>A)n9?{PkXlhE^{AcQc+d2wblB*ljWd#DDBM!oVhEc&T+ zYpwHmb!jR#(Gn!i^_=PP(;EKTyfz@HSzGY-dE=9>=|8`mt{ElVkD-YU=ZS!i0O62n%5(m}Ja#m8;AdufHoK!s`6zh6ZPw?m zjulC^*To}^%}#0K)Hv6|8G+5yY&B}#y^Hxz!w*b1Ttq4CY6aYHz2WXOMiuH-0BC5J zU*3tk-fSs2eweXh(4LxaWFiW0v}6X&caY`yU!Vavkh*UcX#mg7I~i;1!k-d!T%Ul; z0|ZE3L1lr;A3gZTg+H`C?bz)$s}^zIg2VH$&J%aTCii@IcY(Az0ktC(d6V>?@6L_E zVW(LMfRYHfUzevw;uh(1)drAI-jp{@1*iEnPTJ72@;}Mpn(&0uLnhNo0SG6?cPTEQ zQ8LvTAjjB5jBEPL8tGF!lp!UxRW^8}jCjxJ4b(uFFL+9STmIMZ%Xfh9g|8V21rwMU z?#jyI(gznbYOp9_MG-*+OnKh=))Is4`&BZ$AFGc+vOZiK$$5Rd$Y4-O&7no zfm-QBZkc&E3-wW)pJIW>To?a|K`H$17nlibsE`zl27PbHuOx3yS=va)$&(X}<;l@X zssIf33VW}|P~N+u)st#!8SwX1OqgvS2zL9zOJ*k^A2{~g4n$L6d~fn}Nt5kH$NP6I zxrSod<{3o-HL(b6oF-|~mvD&qEp5QDnXD;ZEHOii%#RuG8)n^x)cZjI_4foA9u)T zEr0Nx6cwusw9FbwJ-%xqy|=mW9mv;vwZDUk8CU>^F%bA}EmmRwQNG(j5F|Dyk7XeL z7CLyRuq$HgvdcYxulojo)a=E)t#iLTIKY&f^6Xk=qnE68%QD>CUGjI2LprVNxW5Vf z3YSDBzBpY1oSx-dFIS}oo{@DZ?6k!1icl#7E7q*fX^+|sP!|0Y_1e=y@&kjOx;(VRrVDN8h3CAq#z&SXYg7vAtogbJ54=s+1YTaIMR=-~C z&@yW#-@vd~ueCmv-X7$`GisKYAJrOvo<%G-uJnY>DX1`j<$6y_pk>UBg3q_$Jf(-( zR-41dhwZ{I_bs4np{h;kL-wJ6$3^t!M?3mSM&dIiwBRV5%oC;{E$H~XvHzUZJF!=y zDTeGA-==+9cr1$oJ!DqD<(^00W>D)U^7>+~8=wK7cAx*U#CdRf`f08GHzNBLBKG>* zOMQnp_eGJ=mYy^4@Sn3B2uRyy_JHnD{$W!pV)CBp=)%v3P+_6G2<30ksojPY{lPU?5JyQx1tw7IYI8q`|_7*i&BU05r^(EmSp&J0c%pA&BLaG zq2|(F>_y*hdF-sIbwS=L*XC_@U#q$H-AQBVO|>&75Eg5IDu#V9Lfwk(bwxPkcw}|l zp9-kOwroodq&L#Gw;A7k?i`;C9l|7Sz9Z_aR`Jp>M~K?S-|e*u{UxgIC=B5u2KtY< ziJ_ok5m`TxyM|>v8&Tx`m9$_kJvHUW417?5o`_AguPy7OZ$4>^wJntG2=siM^#Kma z?Y6~2mg=@UsI_6a<@)6dFTR`P3IN6c&Pfk>FkWk~?#yt^0UW6O^|yXPGcmqWOjydL zO9Q*S1Ek-ljMCxndgh2*ehF&M>30MmAep-=hCmh`ux5ObaQDJAiZknxY);UoFh*zb zaf&C6wn=%30v^at$ia92_N0`LB7~>Tb}1>tX}b}W?(E-XWhlo0M-=l=y*Pyh>gqX3 zh@?D7r0b)*N&{0hyUabEk^xH>Fol}-sc|P60Vc979qD=P@`#X=Yvh(Wa`3RO*v6n< z=awb{@n-$KxxNunx7Y>E%Ll=}c>2zpjnolSUXFjc0ws(&0i#bssuik*kWr`@$uR_` z%{-Gv1D6ANdBqDq;~5+d=Z=xmxu?Q!7H`*3Sn+_&F~C;wk5A{UdmKC}M=RGJgM6PC za^rP&uz{0F2{;|+w9zBrcbgaX1X(K_$KsF(mtU9VE>ihA!bv3ogj9TG?CK6#fB zf#6_&^MLHm_UEJYpTBNtYL@Y2#`KU6N1?H?$^Flc59Fode|@jr9Ql#CPK+K9XNoa^ z!__u>f?nrn<0N#XUe`Q?n!{H;j@@nbt~|VFUst3%)i4G?;lOVK@Z}QB(pZ>U++>LAdtqMu0I==d=G{Pq~XdaeS2c*I%g2T!+a3g_g14 z#)S_LW)_A_xZR`+Ol)IFymBx4JUovDU>6tX+dvin@69704AKbvW|Zr0syAK==%Cry zIM<83cteR=Y)0W;wf-Y0U9SNj<06|$T+YzyM_?7054h(xHU{MGs?cI2pE>9%=b$Kl z(E>UVoys%me1L3%`{9vo#|-Bg1TexcIDla=z@p!5y?|&p-fwx=WA=pUynr$ zPqt=Iz43+D)~b@1xJ#Bnus8`ym8&D1aC~2MXHhMTEC|N<76O`S0H^ii(O@fRq?|c~Y~+1p$*p z=Y&<}?Q(=Y^SL+=3absXl04q>(E;c&0aa9@gS8$x!E6;WN7&yANit6q2L;o?tuA?t zYdx(5`$bM8+7~g=sTnc8S2C#;`tdowhJhMJb4^9rF2wcK^O^P`85Dt!DjVKguZG^) zCB?KYdAKsJxbSf;48ERlev{44jD0@F!Ci9RU+Yq$VG-xT_T$PTA6Ax;waR3ggpqXL zRO5E@e{@1EUjB;W*|V4o$Og)6vsThS8;$3SzwGEiHq1@lCSG+_`giT57yjz^;pbh5 z082DLxxkZg3IL%4LFk73KuHFjt+#|WN3UFf_V1P5{40;{R;o2-Kx90=0c7-$-N-{c z=XFu!`JVgYP9V+RQpAVWwCqDz*+=(wZ*)t<-5_QFFo%09^Ah^K1I5Hyf~guw`$BMlt&QReBr{yUm1nkb&Tw7fwdGs!3z8o(T0D`NC4CI7107g#`?mg<5i{XU~&v72Zfl1RvPQ4@oL-5H94R#i&O?NzT6n0>ICIvzdxi6MinN$C8hQbL{4A{<}IoH?D zkbtJsk>(HT5A2O+d)ub6Z45V2SzpYTVp2?LbEDWbHBEu{?Q(gPO$9p1Prfp^?w<%YgXU_6k-cWK620P@KN#eE-e$PDVLHwZy z>no4ID!kc-MD^0-=Tb`Ct49f#L7)gZ6G>57MR*7ZK^?BFdze5(P>_rlxx=W^_0hSf z(vAJ7-P*hoiQ^_9H3F-0S{1wp6_-)BnLiIFP37lfS(&LLVByOZ}%78Q;MF zQoQT8KMt#Z=}r{J>Eq2rrp)KXfdO!+5}N2T6>DsmDYN{E?x`k`I6i1q;b%+voliRw z=Ij)pI~f?le?I@ywmj02TwxIr>Wrm2dG@Rof-E`m(_WqE1P7kdl~2@5CPZLO!-34x zWBicSvdz@h-;KkmbR1{o*=j(0Y?0E{A8Dn3o6_A|KtIQp0B1A)kC#lcVWe!qzkLRk z1n-M~8%eGm{>Fem$xwSuA2rh+xCcyh9{A=@MyTHB8ql)t#X)Z>?vBvY{}sRjsp#E+ zrBIRt(L9-|iJwt7gv^o<(C3k+OkEGN(&?(ucYfZ|AP2sy zXY%nX2Jdl_4=6g1D)R>qynf{ni0*?#?6pxSiR4ORKGI}(6xmO{;o^blC~-QmPu6zPw2wrpO#HR!`;nEl>`(;{fHtzCm#&zG53I3JktqW%h}?Qio*)kXs+#VP|5S{; z(-bf+>oK-;h6av1n=CR- zo-s}=Q$eKjS!#!yAO4S{^9*F`|J!ivQF|3NV^r+e@l#uC@7SyMuDw^)-m_Lv)QVMm zk5Zdx#V9FiD@9dN6wmoTUV9Ode9t+b&wXFlY16!bkCDbwMOzftQRD@!cF{P{(MC({ z?gOivw_r60%#AGBD}4XjPo z`wur^_AmZNNAtWY3+}dRLUrrHGJ!raTQ{PW%E-g_NqlL>NjY5Av4ez?xY+l7li; zqRlNW)JitTV2>`0dtm;mnsjQx`GO>t^LJ;Ox^E`w}Is8{+99I<)@~_EaJ_d zgh@W4H_H!?we#p-z*ZLO<@!84CplViM^Ckis0QRM_Lo2TKC2)I4%dUk$n}!w zY2zurhQ2Z)Wqa3`A#3>6e1OT$katNUAh7%Jkl^?PQi-sI8;^E>hv7bspRR*%>(4eB z_3xeehR#lysF381J8*vOSpfvs2(8S{oah>Tpbo%gR{GZEEM;4@&;Cd2k14ZPy($tW zd3;@0Lcbc|bM0@U1w1cGEYU%kjkV$`tX-EM`_$+Xy$x}(YWiLsHr7TviFHSEpCG-C zUwOyOZf^EoZRpjMmZSs4URk*-xMjM*zlXcX2h8BV=|P}Nx^aIgDHnFtpU~z{z5RE0 zIiDL zE8jyB6R!S^%iRM$o}eJ4ZtkUDpHo*o3&B}v~O1EjT`>%A|@COEdPH8~Q96!ziD(E zDKoQdQ_jL*5qCEi*GK5%fLrT(G+@YzxcTw)$$6t|lZu3vxjHb6g+A8@VEJe%!0Rr> zv>txt-HBOg7N!yceQk)CRh=Gw9<)ZoekHkt&mO(qjZR!pn)M~A;9@OjzulA?dj z6s50zcVUzi^L_R5c-yK8;6#K8sHhCxN>6qZW9e-R7rJ#>2m+Vsn9ni9uT>y)FZlyr zy@{hyaX4`|zr`cseWgMs^(%w8Y6nqw}V+B z#hp8!m!XhXRM8c1du7@V-vLhvx?jd$`pFE)q$>5Dkz!V%jJxa7EuaY|BURVgSJ$5T ztWL2u*z%xeDR;JZHRiJx;TdfQtg>E5podpwCfr;|#SZ3$P@pQ_1srLfRURU^vJw)e z`O&K*@xA^s$(6!1-k<6;w1S%t#f=`O2$z@DP56GL)^9ROSo_+w@u(Aah_W|CF{~Yb zGVjfF^YIK4iqyCnS0M>b{dyHxno#A^vi8dvqi7SC+^$+?Jp1_6nLGX+y-<7s|D;x8 z^fpE;2N7OFU;EzWkUcG1wN~lZ`m0W08-A+|?<_j`@#9D2zx0P^AD)bv8^qp4M0S_l zUsyl*$taq?`N#S|AXxs*S~i>;pnS@P{ryf6@$YOrXTHh>7=?enlKb~-d z%J@PqPavUWvic^TkfP(jAR_1<@8M)tSFhjUd(M$k1FZ9ENeWfa*0mF zp~hKB6hacdt9-Ao4w0V1{SI$$JPKJ(TRzV#jhwNQBT@2-`DhyLqDcUD9UfCBG(+7M zCl2?+pX{3J9^*dn2JuD@Nmegd@uA2?5ix0*gkp~sK_C3{QifZ27)+^`VexJ@{ZG8i z9=#QJt`jt%jhdXveP*E|5{Blzpmt8p9hLZrz2- zd%=uk&!W4P)Zh>oOaN2MVGyhsy6<6-zMS0kvs3mG2Ph6nDyhRU{_;Stv76;7DCkJy z_Jf-RwT7BHe0k4OQtSeaUhcsFB%iMVZ@}7=?e2=-Hz#48E8VBP8*H{SK+rj233FWy zK#tr0ok!k}T!#$+%GT3}nqGi~dOIC?=NO3WjsGmPOzuwoWzFS|`lDbk_ z_J8;@_-_P&4Q+_c9xtN7pj+B=RI6iNc_IW=gx2zQA8iX=V)B0Qph&oAb$mUF70 z+7#SK{pe(`SyRFNB9XJbR8_2+A3=iS=RrUEb|&@VZvHc7#Sp4~i@a1>4y$IrgUpKF!*n=eiNQ!Z$;cSD()9_Ewz z+x?{nXO=$xkU-cXYU=M0|FNttRbNgUITe+Ao;$0%hfv5SXU`jsQD$lvZmkPj%KApL zCZ_P7SPn1;hIBhq2Bv3EwwNmc1YzVNwTX`^dq!qpp4kD5K&xt9V2p0QzpE9_{o(s! z-{(2k@ItNR^$WL_56%Kx;CGpiiIn#(Qo3aqT$0c=wh48PC3!M4GNj2aNY@`BaQJga z2gd+*Lt6H#8~|GZsP8_Dky2U;<{|XW7l9~5@v9_NW%w?liT~x<%jeDrX9^9hZ*z>`~~5Ui3fc0l?6A5f9Tph&!>e zg2u53<=OY<(=FP!9o%2b4Ft>Woe1efX6H+4b<(MSS^$<40#3AetEX3=G*Q=6hsnZ~ zF*Tl($e$4+?*Q&G5Z(#EPy&q|w3z>*8IDf1%T#6Qd`|Lc=3RvKdjIrD=0Z*>2!!y} zLfaQ>UyB184X|^CwnjN#N{w)*s)sWH!zkug$zp&ESLj6{2l5 zo$Zh#LLDX0)nB*X&c8aZj63bTKrArZ-=^a0s%G~1QhYG!hfwg&Hv zD;*2Z!`7abq_&j1!uAVOso6oue1<{p^CN>c&iK_;O8cYpBq*ULbu1Xbf$jhY^c%iB9A5YP91f{d!OR~xcEzql_n z(g32`agml2ywRUzC!{jMOI^HYDU)~IXBcvI#~|{-tzYEjS7S>6tT772KT=FNfq%3F z`+_>*gx2zk(j^>DFsL7UtSj`wwWOv2=_&NTcQGmKc?G1YRX4(ajw^v4|ncuH zPOtrZ3QDk7?6|CIwslzX;7Y#y~+b zO2Uu64OX1{YSgMTXD9>U<4{XSGUH8koQ=?z!Hm5fq0jjvz3C`F81 z9EYrT5lKtCp98IZQmlS5>XiKgM(U?p`w4SEc)-NZrpZdg7iKwzJ2Mu{nHjCuLEv|p{=?D)3)%=OMQH<=XiVI z62LK3{|pXSbMz zoffAKMSl7+GaYigh&nb93a8eJ*@1;?q*<`$=fV6#WZ)zq*PRH>yl;rl!1U z&q4NbQ`(78N4wU+@}@x0Pm;JFeYv+!s*-#M*OdpY{@ie$7smS$lu#idrT_6HX9p)0EYbfyjtWr9akZWl*H0V6s>I*t2&GU{u=2MaVjX3x|e<1jQh_pwA4uB7Z}~St`QL z?rMb*b8%@Wd}wC~AF9mon(bboUDwlk&9aOHqyF*W~xd@8sRcq~*ajF}g2H|J^h z9}=G7`(C!x@AUIMJz>tROxTvvgqta!d3*-Jbld2xik7$!EkDt{{cq)2=%TJ=(Ji?_1Glt{I zS#_t+m*+lvXOd=7>1o=j*8=ySwMp`p%V72JxfzdEXtOy?wj04`BUEkxc@LAC`8{~o z^pGBT9r@6?|F_od3iahYZ##Lt=f*3f8xG%9Vk@sNFIT4FjsICqL$v?wD^x3=5@d9f ziZ^8Hz{PFQ#^SlX8z4}b;GU@J^U~2i9zeUhFdC@O0Nit{o5yHgau&r5icc@j%se4b ztQ;_eTjFP{{VWRhTmWe@&$z3as{U-bj=RWi)xRxC1*7?8*d2)s08j!~xBi8nVRDbX zTbxf0-L4BgbU%5g>9@4t013J}dX5|tFtPH)4swbBpT9tNs!*os&5awGTl~=?a#sEI z@>go#9;+s_Sb^{@F*PXa-wTVGMefto;8a&5=dUmRGn2-vp+lSqjJkKc|5-rUMvP+D zvfB{pw!Zt;A8B^JdBNddus(3*WP}c|wA&-)5fzKpRvuF`YGQvB?%)qT!Zc1sBSE15 z?4KJ_w9LHFm_*w2$RJh3s&j0UD9a%fdx+V$>JS%3pqrQHtL-8}p5y!0qVEdkg>qpk za|IqlU{Xhapj*3sEaL#nKkVVGcVttZ1TCh(hAa;0Tk{;pK-3%iUd$agKDoIE^r~~? z9N>ZA2ggEvqu_7MeOC!0BqQhNfE(lLHr#TVc;Tv_H$fFhjr~@GQ(lL6!(IbLqK_IW zZ?JG7I6ibd6S-HA>gv36z1zcPEf@}%^v%q(!EVZYAAYV{y7DB#clLU;F$rvtY{|k; z-!7kwH6KZ=hwfLjODlPe-AT#G;W>gkSha%{ff+Y5!I=9zJ-#e8%yDY_y=K=On z;^pDh&gKeh7AzHZr|vw(lbPB2Eo?lfXYaM&?8PKhCA)q_dXMJ0jCZFJNQrVA9r>VO z5Q9P$p&4nzQ=4}=n;ZQMlUV!xm8 zt&79IJCw3viOgvQCofv`N7K2Hx#uK~-tP-}{&^dAw9L6P@VKA*_$bdJoOqt^t4*Xs zDfV>Rn>)e(>^Fho-2d7NdcA7G{eX8fH+bXXLMyu|ChjTlwbHGpN$&FT;QsGO+> zoE@&UVQfR@k+m{FY1jjkOLjC8x3a$}QD&oJvCK5w==$*c)x+h<*{_pc@{iXY9&+I~ ze{PRi(>k9{=yQ?+$~K&LKw3JTRZf=GTHR>>-}W{~_W#HXVwC`|hemRM*mAMW^5?B= zx>92JiL7pc<4R|j7>n|2ym8r>?_s+NQ4~H?;4IPCfn-NLZ7-fInJ_mFWmcpNrz%ZVecXa9*$q#?^@sEffNbgblp$raD-&qgF8iU0lDWF`)Kh~G$@2nhY^F5!YGyj#K_=0A!66r{k4p$bt{ z1s(h*rnT%p(AXQ0oDlhNyHQ#)py?BUQR0l{#??Vs^ppUsIGqvN)f--ihs(3;(>x+} zj>PR*N2kXdVx%IuS;{_B+0I5hV6L&|Q*W(=D)!R!>Vo=mb>fZ&V#-&Ye3YzY1YnVH zTZ%PziOzc;hPcn}MIGwFjPaUW+UQRvhNErO`K_)=27+etNl8KE>u+*#Jfsrr=TNZ6 z%2`>UQ460m7?32WbSBJ&f5CUd0{th^Xi$To6xiqs5BoP82pC3;a2jiC?_TE(Wwh3N zrPL|PrN(6xy=SAWL=?c%4CH^oi{mn%iTiuBk@LJq61G&OuHWX z<(J{5h6z0TVX~zFwA(owSgue)$;v&xzHwN7D#W{1SMrmw&gsASWqX)VtDv&svcI|u zoK7VBa-F(|-gneXiYZN~n%)j_@eSG;-JJuV60nAF^+~A2;>Tx(HI=!>X%n*Q0~nP* z723zRw$?w$9%a1huHV(RL@YIb=*4pXVEJk(N%`z`wVr+w#39!9l)}zXV8m-0)_JR) z{G{ozf-`bRS!nr4_|gL-q9ar-%Hz?Au~0{M=uhPspqgvlXeV8GA#adQ#3+LO$rUy| z`W3F&6!kPqW6=`Tf z7A$Z7?osd>FWv+5v1@dx5ZC?1QQoGXx;K-qtlG^FpZniJ52J497vnk`$E-~NZ5O&3 zQ~SfLSL03!6s48xmg1`zlkm>rhZQejw5YtEvc5K#o4QK^I*nWw7feG4Q3s?`ju^f&0#2-E#`K1-OnD=Ar8X@eLW7aiJsFgr8*H1oEUmEdFLKmJEZo60<9 zMYw=V38~S(;^ZfJ5n)giY3TacM^Rzw+8rup147vlxt#eJ8*^uV zFx9}+sNgH01@KLIOYbxh|)!ugp1_f3DvU;T`U3{idu&ui%y&zGCAW^R4E zSkA;*OVILupeXzI9FN>I(_XJx973h{Rr8rDuw!-k$)*u@bCk5!4TTx8px=_jtN~)D zIE%!zo6PlYDtoI}ukq-ugKGyx+_%&RnG3}@@;6>v$z?iib;c_S8b+IlIKCcW{bmL- zvF`}$BFfEefmI>){r3Y*Mf?1!8~)~;(b+q`wm`09eIu3T zze`Y+HSY<`c8;U3l%Kx?mwyk}#fu7W)m!}T7*KrsfWEM{Y_WJz$m8s2-bI|brOHFa zHj%X!&gwY(?_u23*L>}yk(W*l6e9&no0yqM@M?}-k;%W+o|ECXHc6Dq%w##ibpDu= zMW2`$ClhAEMQgWD7@dHq7rG5vg~9ZSjL(@g1@dN@Mz=2+8!kHHa&L-00ui7j(Z9Cu z)+FT~ZnlzzNd3QjLu_+0F)o|N(?Iv!Q`Ba%QBThXkVqFs2bR+b5S5*+8j?WkH(F{B z(a^3;=dga3u>v!I!%NIhTtNT-otwMTc^-<@%sA%%@MbyG~sAVrM;IsB7) z7)JujgsprM3|ED9X&^DJal|A1nLqngy({C=9r>}eZmW z$0%)YvGMo~j`T@CNXE~?Tz#^8P(jh~ec@XwYf2Zh!wa|mN{5);US(Yxr8pk;6i}gh zCVe%1#hy@FeCR%DB8)YCf0HMX$E1Gzf91ov@6hWJ~_5v#&oekmrtvSB3OliEoa$luY zkF2e&y&C+dZy-D4!W6pCWOJDd<7}4SN@+%M7_F>|y*yrBTUSt7ib4DqtI7%62dLq) zL_-$DygVQezeU-9E{yv5j#sz}e3#s~ZUPJyUMPO13PoBG_W60e?}%sLSPN63ON#bf z+PG0jB$E^NHbR487Dj2hZ}CPVod-XkH@o%{c;$^3WUx^^&L?7x$ce5`=lRNO&w^1Z?-S`p(lSQg-+C&A`&P6B6ctM4U%ka808dHcOfl+1YL@uucivwN0sS{jL)K z)UlM5+l3#i@GOi9G;GF}I=&9mQD(p*0W3%DPuTnz<4fk^NAB*<8i6j0dm+%fDCXlB zf~GE&T?MqdJY4j$9tUTTU}5!Qvb!il!7EuvDqf+J$^Pe88cPJyOpCal6Ba#S4Mz^ zBJXB(YoN6+p^%3R+ND5LD4qC;o`h?T1-nhoX!1|<(A!rsI5>hy|4@O!4w?1x;k`3H z25e9Hzg;DXIyh=Afy<$yeHO}n#w}LOOBv$0ieo=!w*PmhD7s_Xu$w{ckl=mtoJPA< zrfyWd_3@-{QfS$hnYhDbmIfcuaZ72QMlq&w$=pn?-t~{&M@ML~&Qm0%atz%eZHWAL z0YpF)6pC|Ua4??Hf*AjDxW_|58E*N0)jW?XSKN}$o10-SGx@aPv)<4PQz(e1jkpE7 zoOa_s0bi4;@{Hy~#nyA_bdR<9KT%Yne=Lw!^Gc&p`QsEc(u-)h>ZdGkXz#YjyFEj1f{n?!@`E9c^7L_5nf^^@z&!u|dA=R1~{o z@_Pob&ZB`YOf& zmfI8F|BG_dffUa<;pSJ>Hd?^p*}fyf6so&xG-E)PNDCMkf7m!pbS|t)zEQ92LGrR^ zQ|$=1a!{3p@Nv0PP>xW=yI_M`To5VH$(Z}Fo{)fJkK!1m5>6kV*L9zws?ouPRReO= zE5=P$W#X`L3&ckY*jTNb!0ssrywT#h24-2Kh8if=*O$H2#y z=+#1?WR1$v+@fq?B_~P!3-kUxEU(AM2QDK-Xm11YrfMYgs#jG6Z7o#)XVXM zp_AF794MHGU5Y8MIUJ7FN{4*D-&aTdf>MEvvKi{IeF$l3`fsBH04IO1tMYVfk=T(> zo3P#E;oO1l5D})9K@h1Qx6gFp@Dyqg8>8cMu$!wgBYM3l>z z4Ac!o#)6DB6epC&LcMa-p2rt31rX6E63SM8Z+}JKY?l z-_M-*LTG%@$fMm&$Tmgsdl(*d+Lb{skeUd+ofeXml(a+}!MPN^5*Y$(n;A%qO}{va zJR#@IH%|-_8TYbH*4omN!4B}Ahx@B>GuSaI!JZP`(ToUx34a^lyRtId%ev8b{i}o3 zR+ZZ3;2n`2A*8|$*-(ulR~3H#`hY$7Y5q7uv_+-Hgnu<>i-zM#i-t^n_ye7Y=G_ic zhkxfr`*I$pF1bP6Qd>eZN)?io5XDL1Mv~|pX;N5ag8fRYHXw>>m;xkEF+PNA+i=v; ziy=QHs}99Uv`EVm{UjZv%=rn5n>P96C>5Zp{g(a2M0dbL?1f^-MRl!Mkf0}Q36Hy3 z3$>O5LKP%cYL5iZ*b)YiqRDO&Olm7w3u*?8O1K%LWH`byB*QtY=r@Zy1kI9T!X{uo z5X@wy2$C~n)Rwrnv zvu*Mm-LOf^O$f?1c}j~mP<|xoE+SMmo86BPnj$Tqj0cRci!Miw*(>cE>Qa6~RgM!H zikpy0d45&Cs8!zT`W^co^{m!CGg|R7omS-Gtfx6Vw|#r-&XGf6zkbxd(!`2|htZD7 z{K!&eDTLKl(3#f_FfD00UH^@$+)__~scCu>LODGiUq2xb-06M(nce`t2=t4A-&wOF z$p4J|PtjMb%}}ZIpE@KsKT)vpCBLn2BM-`5LTS6EYYGLbiterDtCvwWHpx{OQgYo@#JPO=J~QTcGu=UL!a zq_2NcXhLXLTK&VkkD?{l4 z6EROn_h~58c#aaraPrq#(6>O+OjvQ++adZ^jAw~lBnx$4WMJKFK19o`;cTp29@8=8 z8~ov*@PFTiuED|1+zpRI3jLJKdf?3@iOa9h7co0cB@Ce}=6F__!i6?p$~O8`d$j-l zdN`x1?3Bx)`P0mZls2?xq;IgN@3h)LPOL#(et%nSuj&tjGK13Rly?$TgORs&9hyzq zk|WnZ0rh7Gs+)e>vv;to3m|Ovu5mB%cjAF)@%IrEQd-nW3y%te1U*OVeFZp0MfpDH zz`S<7mk9RMIiHTO=qcOIKt+-X-o@5CtsEss8Nzq_r%6zSK$`|g~2qvW^dt83Yg6T?G zBHBk7gsaJ2LkCL&VN@)E-K3~KDJz|8C5I!b$;Z(|JoJYr&Ckl?Y7oRgFJEoh2QNi1r z{NA`aVk&b?<06P@0EG>2DfQ(Y2sRjWw)OkBEBZJN>TiieIu>rZNJ09u&3<-xqyAGXpnYX(H*9S+aMq;*pt>mDy4PPo-EI z$M``pLmZ%lr?JDQxBD9w+KeflQxYDtkAtLqwrjm$k4a3qno2olYZd1Nueww_e@3(l zSLakAs$7tTW@zF=HiUNdtYc^-Kzr@G8e4TK%n$BeL-}NhiuM*1uZhEp_2o$4pyBVa!GJ2Rk|u@L(=@>D9AXM1v#P*ujIe5 zNzw^I`#b%iyo+wFwY>(It?6L_nU;0_eg+kDo-*Dj;pF+JNbxji+BVNtO2eP*B+>RI z!NjQCzO3liFN)!tEoG4jZaIcUewbp{PjGP^tVr7ak{xv$M{b!7>kIE2_ zJiTv=%ws7-%#y#y#1A8gS(g}ZMA*UeLJyLHa%U0nUQUZ4ynW;JRH6ud$38}5OU#a& zor2ovVq&iq8`B<0LvN71GA^cI%D=8e#&S^BE=b$(ZuyjxDHtCnK$7jB*@3)!cCQ{ zTpV9X;`??~u8lXw^~oZ!vas^0*`)`R=jw@e3K(A>oLpY+OTO|!sg_Lqx%(~<;PFUt z-{923w|VfZUoXw(|p%5BmoJ-YADjS9DLK(x*>_-WLJu zI`YcMg}2OO5KZ|B#5~PUWL&klBJJAz$1JD6JqN$d?2uC$>e~zB^h5$dNl^2n)_2%t zrcS~fq)Mme#u7!)s@nh5f~Brjx7;S!lZNyAYi$!c>fnHEKpkD{CYx4~c536;D-+__ z!4sQYl+f1>1?0kiZ_sndLn{nDYD87l5{>_OJR^ojW4Kb}z2v&Me-uB&k)2s57kp(0 z0?~kJ3MvIT7(py?t?f3<*Q*`@n7dt-&b96-(R?nU^Pw^L?-_E8Bi=e&c_q7h<_sdr zpmWXYS!hJy`TgEq@3|NWyS`iyb&l{D>~X{5i;Ay)lwbT7-Kh&i3EyVjg3WjriPcck z^2xsyzeK3iE5E>zXCS&!eUHT&G%?Mb%lJVi?dKo6rOakerWao*ePJdq(pwyM(H{<5 zh8777(|a7ieZIiZA~+!(x~f~AhI1|xQhs`FF@;aap0aHMU*{kHSMyil!DT6ivaLZ* zbcfScM2CdGv$GQ+>f6(Q_iOXGsi-QBC1pmwqY$ZPA%Qv+*jVT9N7v|yA;?A z6qz*DtL;5Tj971{{;h^Q_A7>R@4v^(3z~sBX%Ua%79}S`KIOMuUC?Af741>`q9LxB zrEX%nkTq|Tz`($lh2|SRGuy%kvPZWp`m62|APT#M+C0#A1lmMNdCZ#z!Fna8bp_Pr z(86OyMNs-ufh3NI>P!v38m1tV$*2oPv9ma8k}GQzUZrr(mJy z5zT{Indo)O;HHEO+chhsC$-Bf${tx2~-b&ke|K%r@Ic zgq?#fZ9*K#T*Uq?hC#l_cs_R<#al^s{(bn7!j$uk4F!oE@5y5KDbcSJFC6~%a|5}Q zZJ-T1jcy`HZQ*soIHId_z@5h@hrbKt>9l*-V%ga80nA!xs9T<++UCvkxzA?(M0z#u?TDW3x7dn-APNpcd@qij4~lgFo;mJ+yHHce6bK! z6x$okAckDd@@ba43hcjBlMQgbc+neTML) z2SUEz7O1WMjyNNGe$)84l#9gum_hB}7oyAk{z{G%G+1wj!NjkGuv<9O4Zj+mPGsw4 zETtJxWX`BDv@iPVek_6Gv_wVK^Ixh5(u{Zab-y(L4Hb;|UWV(V-c8_I$sIQjf*kim87f}NEVOGWS!)%Y} zHJcQI&HhNkg&iiJ)%}tSz5MAX{p{;u`xim(`$XF)+9yQ(QLXUc))ZDMEF6to)w}+! zyhHUbuF@|i<%Q4h)~XWuD+u=g`E;`~S|?apgHW~Ezh6cTi*CdE$u($ZM^ z!O)g4o7BLyv=dg{(`ao4=+XaPEL-2+$M~K-w@s0+H)`8yn-PgL7B@t7p92L&e)`*p z6orXT8Gg4CmjC_t7@ISSKW5@@)GqH0%Uh=0^)A^g+dA*sJBcDPP;mqO`yZOBD~xZj z+213!J-=l2{Cue%qeg&^g~U?ivyah_LB~|1c36o&xTdy0)e&N$o6~ATC+ix__Bf1t z=j0UoTB2R)t4F)+6rmb7D`y4`&WHnVntf*%QM&;@}G?q5v*IlJwmdA;CS9jjuIXW32( zGHsK3THpbPqs4NcHIRJ>W~+_pB6Xw#gT$7W{4rP6+h;*tcfU^7dOI!wC69qD6Pet< zi(7yhM&X^(+{jExNaRcGjhmA->D$<{R7GQppgEyatUoF&sCH40+CMiP4T^ZmwHV7A zGxIIr$&p(k6riuLG0v9mFsz`k8KP7~H1P4k^NHAfAb&5SWq3xC!si@z5GYw%;qffj zJGDen)y1zFBBq>~FPVJR4<(*p>(N(b`Rq&Y|8ExhG6||}w@{_=in&ZrzXyrbSC>Hz zh=FnnYF`$L?O-XlH!rPv^gb({+m<@%u!O^b*K<}4+Xv430XY%VSMH8<0&EpnoBYZ$ z%jKfJs1@FSbd-dNB=pJ7fMIBX`W>;vBOEn+CHbYoDz|`JkKVNfe-F!8d(+Pizu1pr z;R?K2zZQgwAF0M9@V+Z778&!y9N+(54J}m(gBMj+sJ)dbEn&zrBLUG6^5h6|gGbMd z*YnDICZ^=9Wp(1-;s&XGaH15UGnmhEq6wZD+T0H7s{g&@*8lvG^Wbm)NERR4m*4r> zoKwOXjkmML{+Z;zq*bXg%oAFwZfx6;o+SN9uk09*IPGa9Y0hyEm(W?7t3T>t*Hj{0 zUf1b70}M9jUrYP^g{%i;n927?Ht$b<9W}|3k9-*X8s~Cw`2!XZgv1~vn8YB- zV$r5p#gQ#%23Con{y-LD>j-7>JK90`_e6119`*=oSC~H6nPJk&^3_BwV}PenmfP3# zg;md($d>-Q@S$a`a(n}jy!5K*E68SLinX_;v<*nu-+yvQ6QU$iLW=?CvPVndQAW9I z^UFetW)I1(^5VAC2BbQl$^LmT023;udn*v`x%*oYSfL;GI8~an@FY`YS-jdFW^8Mww$&44^O-Ud$ z{q@DqBv)9WLgVTM6aF2h?dI5LebHTB60`;S`l^)kQU<9Jw!Qg zxoT?(=x9tYuQu89o4XY^DMj<^FE$*%aO(q^A60o9mdhA{Ua4mQh?*+MnAD#N^?^OP zcBB3Nrv;+rZ6vla@Xzlzo29ju=HLvmR_z|sFjlh8mxz4R$*+eU+xg0$*vdGbaN#ja zv$o3cULKUR4rEDTfj-Mjg-yYHjEwQ35)(%=a*?%lW5!zT{a>vtEj=WPy~!v=m4H&S z>LaB+v#8}hYllbd<*)h8NC>bf@#lvjBQj8Yio(Wo|NqiBGQ9Cq6=p;CfsNu8!dH-w zn~-f)Jw1*`A%ajbQKAT7xGg(`arz|FyM@l%+9*uUImyD`B+Y0*MCM5KRH8I&x^B+n zeWdwI@XBGpDU%OkbArazU!;#cG#KwE*5oaBlZ;!ecK9WKC1Bw;p^Eu1P~`)0d+{!; z>Kl-n1KJ);;{a?IuqgvZ&6`#oE3ExjgLjv}mHv7&o>e~|82S1d7EH3c29{ln8MW6T zr-G^NzKLC&1&PI#$PLPttVvxupP}ch9GcO>Oa1~99Ph+CE^`&;HJjAUHm;&fmfOjS z`8VTOW)M{5BR2lpe6!z`Un6|$EsTxSmbF?v!R5@~3m>dsuqKDhywzM>FeR{H9FGDr zB2y&&+~qzfX%!27ZOH3iVp=feeDCF@EMQ^;PDU(3J}i~9Yo${&Rwf&^SH8@A9>=Ij zI-+ZwYqUBdRha_-j-}Er@6qEoh(Uz$U3J$)WbSkBqu)hDjA9=m;pcC2m0NilA_T9_Zmo7f>RgX?A zaI=QDb6|UPo!%PkX2c$@nrncVwiax9SK3%TLu2{MMwzHVA*ZxTA-=wLDYclJlN0U# z=uS^hvon<5EXgnYd+({tIR3@a08S>pMX`y9%^)puFhVQtOFa#ud2#%9`z4y9R^z96 z?Gd-A{YQQ(MXxv{lCd6M`&P#3g?L}G0e$srTvu!Ma(17$Q<98qcDmF4uBqeR^bd6yGr}Z5^>fUm(^pKE1m{t7v3D;%DK?|L*#-S?(!JfH)JQ5Qx#{L^r>Uu1>V zT{CCXhEO+8>RW%w1Gi%c_3Y#5iEsToSn|bl*dvBD)HCXTGFhNwVSj&pgkAj}ZCw_! zVze8$J1@OIio9!z$B=vM7W?8CyYSL{7z^6p^0QV1+32Z@I?A%7FE1M_|2f%PyDbVF z@Ie*i=x+egCBjw}bqE6<0*4pz+++DV%*aiG#K89HV*RNv3iaa;Eb!nIx89drpowdT zZ&UZReVyS^Yy9G`#}+JhF8E$3N+mLg(i%>wf9l4nZF zN+|j_Muci?98u!5dp{?RF>4>!NcXot8)~5!NQpG3cVWubGNqW?iX%p(=8S?e=k8`u zn!TWq&^Yuxw7tmVxaTzT`p4|mpf-o3AVW;#{q2Ue)#VFO!odc zEWE^gcQFUNN6b&G$Jugyi(DVGdF>%SF9yt9ON}0P*d3)A$daJX=qXmG+JpUABVgkr zlUzjX+)*8ky;=s#;*=_=yt&4P_NBt-%B?Hy=Wl1vE*&F-p()o=-|V=`$me?ZzobBk zud)LrwAfQ!7%Ni1vwF4pQJRFco;|DP&v#OK!z*TWvPAbjLCp5a#)E>d!Bx0h{9lEE zI~4$9!i;O(O{3GRrP307KH{O-U)%5h_|EGom{2B;2=p14qyOyCDy;JEId$*HocNG{ zy?c8>ga1d-S@<>iwo!Z_AR@>>q(ow(faC~eh~!YDrMo+Y0n*(iogJH$bxgyX zdv(UU@K>53kK=h{{s^((hn%3bsH{@acZmrUIGX^$RWuyik=XN8kb>niSCUM@h2+*b zK8ijJPt`phZ#lH+=#M6Tz>&u8&*_rl(=-~bkqYsP*vn6w*-O2Hj#KQ+=LkKWmPzLz!kUy$daUY!o9Oezaw1Y z;tRSV9Fs^6l+v`PQ_S5vQnHYK-6u-@^Pfk+K$taR#7u5rV8w`h$OR#0F7d$RDzqsQ2E866cIB3$?b!_Ur*2|>jH3c zw{U%a0&e%L#1~hgB2wV~n|rhN@y1JJ!NAba@1Tb!YuHls^yUPdTo>(^ZbS?^&703P z<2;u#jn^B$iPN_`11v8+PFG#+E~Y%Ki*FJC>gPje99^)Rfm4GXOx>(7s>V(PW}9gl zBY9+4EqLmJqkt_*pE8VuFvi7Lq>~AA`I#9^xfwkEYCcuZZROz9){t7_b>C&{!cQ); zOcGg61<2J2c1kICa1wbYLkvYG?hx5Xzn(Vx?wSJ7VoHH6z!bIf|76GBNAaV+rd4Q*Dl3{}T4|j!tCBAdP!JRb_ z$Y313-FMmz^y|h-qxQCMP4=^sKR%h4Je@b`KT3IzVKvPWE-mb6X06!pPhm-^?$j@S z!NM6_hA_Y%@_dHKOJT;aF00{z$;I|M3S?Hst+{X{R1kkL0*KFL;xtJ#&yxL z*{RT1?QIhclst#ew?;s;=*r6Vf5oV*F@_!i$}@O<`T3rL#r^PxQOot!zW-@+WqV1P zJj>NF??MkPccOV5;`6g#q1SI&Q8^~&9NJ_pUBbF{Zj^nS*VZw`t?v{82rV1Mf*H4dV-%*ptlzu(%`g!FM$)ix8xyuTEz0?kw?G~(00c*lU)|mzZ6II=~9}Q z#VAE+BB3O-)zY{C6`{IL`Fp3QlQX~p^w|lV@HIR1(LKkyDnfKCyGuu8@uyDXD6IXBYo)PtN!1bE3@iFAq%6ciRmA3H?B5}@W>RurwC`8ZAvWxX z6fy$&)7(_pj3ELjOHHN6tcPv$_Rndw(zHf$n&s~92>5;1jW$ki%oCp4tV&GrFm1Mc z%tbdKgsl3jOTH5wBv0SsHp?;cjbl}H#h zY=BJn$fzmJXirv^!y_vcJf-aV6qWZ*M+(}{vEk$WZL?|beA>eF+e;r}x*5k=H{geT zWEBB6r&Ji)VA3wZxFtTuxS~&R?Fpd(YtiRm3H?dPVb_(qY#7GUCmh#B43rT2*%JWQ z0-GO=S;zG6x|NSDSETRY~NPk=%2QgLt?|EhaQ4y8PSmGv zv!SpV|5rsUCEL2WW}t{O5UC>Tn!2msTs29bq`HjAQy~_y5*&A(o2c0ay78|4g9Oj# z93{Cyg?skYB;l|}UUv&U{vtUakY>!gIp1H`^RVZ~lc1G|?^7}p%~hJkm@w;APkT`h zD{bcve+*f-F>czFHmwed3@md_IZP^W4tfodfDl0OWX8!xGRzif2?K~JLR_fhFF~%O z&HrZ4N6bLq!0&Lg6uPqMy2VzzY}o2`+yfYN!%kxR?%90Rvb$RY6Cd`S{I1pv&={3i zo2ZL99Acvny4?8#hE7hzQwc#wj|qPU-kQ8|#k$>@78UcYM<7*rOHfEb)kM?H-QVBI zccWDqp;2EoiDjrYl2Uw?6~gRk5?fZLcKN;fEFz+B$XQ6lb z+R#tfWx8w9DBP-SwC5zn7T~Redm1syC7u1Ql@duSWnX+CUI)G$HD0DO1IN>RS_IbG zA&|}d317p5`q;rv?78LGE-0|!z2 z*X$uEH3$Wsyf$k+C|Uf%wCGGtmUGE3XfCzzm2t-iL ze{eMBTIGpu%A!2GQ;f5#bM{DUi*&|`m)kMQya z)Ev84CB+nc6u&RL(fYlSqS9u&!f^&e#?mdk8THqg zWW#jQ8TypWv5&4ZBVj-h5KRWmw7j*42$(iRcIoqXok>S z`e`zJ&ixYyThIjhEB6}ztMjp@L-b?Wou+bbuuPfZ^nd!Xg*M{=ewSWTpcJC=xk-Pl zWjV<}j29k~m6bJ{)v)s3GA?U)ee1lS;%c^`{j&1rxZTmwQS5A|=4#NcXPcQ4^>==Q z*3of~v~xDszp=epfPjDtp&QIXVEH)F>Mt(~g*r~hj`M|@p|>5-#4Kw-MbXKLd-LvJ zyrD~PKca~cXstPSXBS`c9p&?y^w31bT6fG!g2hlZs-j{We#{klA0t}iBSrfp17ibn zcn_`a2ri*N$-Gp0RbUU)_Nl8~c=&H0yYg0Xw5nhkDFirK0My26-I>9Sv$Vc?a(-@}mw_XZl5+(H0SldvYO)9gjuC0SP|R<%S$r}+ zNq`BysR?c;M&>aW^)FV=p-I?7X_k=HpNV6w$ss+$(!?S7*68xtz2&Vf&@nqTH8ndY z)7Rffr&!)K!$60Ox$G!Y=^P9y=q2n@Q)sB8mXshSqtJ;*j8GvSc*n`#FmSPRQ1$a$ zHAW4j$C@OlQxF}v&F$7}jpV-azV>Nfm>mH7oEbQgd4^WDO!F<0f!#=-%`;C9vC8rf zHbY!m?4>#{daU~8f6zdMf+~C*vZXOiHe6iE^p=|4H?Tr|&^daGzGrQf@`xuH;>{_w*}Y}sJI@zBt~B)I7-mg~y*+p(c^TO{PqQrGQj zz#98PYfWN6;iE=xn$AF{nDWp}9SE#225E_{Y}C9S(qz3&+{Qm_Ytfm3PRCRCqU|G` zgi<-_=~Fe5)6>(9qsEG7LZ2^u!wqBLtUs8P5mrVzR%JwQGz^MpQA(@!BrI&xu4EK8 z%|Zf3hK3(kJVdgeEvRc)#0=^H!d1h9(YV_YsG0G*I-c@P$8DfYPZS?1z1l!xIhO0n zvr1gxuUV@mk@XQ*OYbGjzRbi6P3p?MA1`31k`I%L& z@AcWl5shLqd|pTegQA^Cmt$2K4Mbrb5hP)0X=TltA5ipkcz8m`w-5Oi-!^yw=*YP8 zSlLP^zB>+u#2n)ULz7j<-QP>~>p){r^@lWRX}m_QO)YKw^Mz=N@ATvc-^Vmx@kQ_P z$I)mxw~>^Wkn7h?mSw-!aasIoo3$27fS-!MoBwz8!|B89*wvD&$Zk`rbF3ZGsT(NGHJR@tDdF6YzWy`rAb?VV_MH@*_$Yq;uZvu~<>JpZ zIKsgFlG)=!3)`(p-O_S{J9s*M0jto?gZH^+{I~>(S@pE$T|vc!8m|_F(%>y>CB&L_ zLUOohPlPO1*TGm-qE5uvB27?H8ktEqDyye>O69pHix&a6)$G=NsaSRLU`lA?^Mw;b zuOOj=&zpgO7AX6Oo1b0TYeSqC`uF=6jQ1K8wVdw-_!2Sn60fS>d+{Qi>TIqV&7FqwR4aUnLA@sF z6_qb>{9(VbKLi!W94_GINJ(JBvTCU|jdrJChTMvlSRaC+Rb|zl)0rv}1V7nEJIT1v zC#Wf~ye53L4dLdGLzi!rRLrbTM{kVeh=bu@$YFbHpi!F2UX z8jS^0$u2)qmRO4aar64$9MnA>ZJT$C?|wd%Hy$MAOl+K96C}Xb(Z+i`k^{OqH?1ng z&Rm<@T~o*r@FPJPBG1D>x6~Hnm-%ikbYV*|-#J9TD)g_3q9DWcBo86Z?HEB!!pxixjS?~@5L_Yan9LJ? ztk4N|i zVWme$mY)D!45S><<>c649V+wr@4Nx=w6;ceFzSy4@~4f+ijLr)+WA(G7orJ##uf$N4?YLRm@@ndWupiMb4^>O^BPh4MPBSI!K$t1CVR z-qYcpHNWO+cD2zV8%VQ7&ciXiyqF_$s){yf=6RR*O<3)nq zDd!Oj>qH)WH92vSR9z+QGYx2Qii?>wqBZ01)>waShmS^u3DcR8{(=@Q{ny*?u2@dQ zDpPD{SI*QWOd4zx%Dh7GMI9!5p8L!^030$5`-S z?Gb*doW_HV9*p;Nb9k2WqYJxNXv6%fvC2R#;ZwX1jCCwXr0WUZC0Bebix~w)tfXDP z`Db=IE@JC%S*oy%lpuJ>s=%58Yl!t1n-|SjYuC#IFNC5MIFdq=Q3ef1zy36O1X*Y6tQ0TZIHI1LRqXSc$D z3|6#iQ0Lw5vI8qbJ06usACov@vVRlSRn6KIy~L5KBQ1LQTUHIa8s8~=lCdI8*6@MJ z{+-JjvgHT;a^izA+jueET5Z86723zO=YbHi8Vm#(t9pgBQBlBxA@D*__9~Lil^~-!AAar15MD#* z2JNOLc#dT4;NUP@@p*+)QJa|e?#ca8@==a@CoXjtRCGfptO+MRZ^eu%D3sG91>UbT z_J-0p01;}*&wq}RVsVJM_?!Z93o=dyqm+ypGauir42L(ApH>j1OBoA}*8ZR9ays{6 z<2nL@jjszyH$$5;G}ZTbj)wL=(L{C+TaF3Fx&UlefJ)?jr7O?Orz>3fz`31{?M7e~ zuwbG-I*UPa42I8UmDv9HC43BL!^J-ihn5}S#0UcMO-Whs-xJ{otjo(O9;={lClVnm zN(MRvA0H8W%H6+TKx+n`*1^HS-JZ4*I~T)#!Lnm=Pkw6kZ-qmFFXXQn#0c~X-V`MV{@Vf-nahMb4Flg zUC2SS!2qS4M$-`zLkLw|mjxkG_$*lE|pZ7o11Rt!@V5g5FqdE z;s`ajrepsoe9y{^6tcl^=LcOheumn;sISpm9^rhMC7K_kkq?v95N~zVbYBPYkBpU2 z1^!p>t&v2p`9(NVb|JU+rpiH&Y_X0`ue|p8kLsbw*GpyNmva~Z%ixG%QGhET3Gh4P z6WK360SuGR0_Xu~RqIMbXr`2+opX?X4i8;wrh%+V+v(TtH2Zwxa%-Y=*m466NLy_c zz)}i(uGlry@S*`!QMozDDix_Md>rD7`>y*zBwMMstt1nDbl~wcvq+hl)=Yw+pZ-ZZ z=w5d%6U_@SkAr`vf^^Ut^LDz%4z*c%mX79Lit;AVbi|C_*}Q$6R{c4I02+9QPBQAZp@$-4U`R63 zMUI8YHyOUxoft^l;oaWebh%SAB6hJ;Veg5Hur=vK`J=I8UqhF#a zQ#Np0uKOK+>@nH$3cEHk;)m{&z6dM~B|^?-r+GE{)i#L3?UBC9eosJUWp&GBZ8=PE zzRlMgg*qBj7H{>x$w3$y0jbLnVdZzZdBK2D7C&1RzZw(2xjt>LEO~9$n&Lz&K^yBl zxt@3nY_GLJt0efNBc3W_nPXPI&ALu&qFvT+4>(g=XFgT9Pl5NxKd`WU&QU z>PHO4ntj}Tot=NWXCI`zzO56WMnDb0r+|KMgr6t8^D^BYk|H;}L~&(52qWf(MlE)# z5JcVSN8IgI_%aqGO>COYC(a?sQh?$i(Z+|_CCtq29dAz=`5sU7Or$*RCnT^|(imE3 z^PrXs#xP2+F+OYiN)pLbv2~Cz`suu$>v*f~cH6Zm9UqYz35!y|?^e)=z`<#puthin zZ~x@3hxE|+t0({O%xB)QVg3SKMsC)4!OpQL*EB{$K(pqBmi5oP7f zKZQ@gfiSBZGLdOhI&E14vhU;gpeOpx%cIx7t6!SQ&=Sf980XMP2E1hyRu-%GP(I;? zQ4-U_1g#@HpTCSCHhr>2h3HLBW#-%vRn))qoI7J=jTV-8D$QleK-)aK3xZ(Rz-~cr zW*P%phKC^je=a0Gc@mD`gay|BAxj2F>WZ#bC!<^vdd=$&g`Hv}SZm-}n77P&dUXzF zEEc9q&`68fTHG%x(_PNcxHYt|+Wnt4zqgxmI#)+cv@W`R_EwcYIF&gOsEOw^J3dqu zWw_^mwLZIleVJA)?tk9T2F~;|_p!E%g9}iw0R*5Cu$b7t-rEP-=Mxy+Pfum!ofQOQ z5c=os?V|4Wz3cZsnP%&=P06W#R{wfWEB3eQqo>Oxk}vL=-fUOKk6yHm9KfFmk`X0v z44l=@I`%iSb0M4>3~C%`E#w0GH0t))?y#UQ7F*lf+OQd`9U)Lu7&|ObBA5t+mt7{1=|$#Rybcc6*RgcoP0el*uT96k zZC>V>*}sbE&3*U~$+sprKW|s_ZVlA4%wa)1`#>a$nMHS-^T&Vbb-$r_%N`Qcx4sEr z)gRpMZ&OmCTd6njKE9kT%Jn77*9Ny6EokU{;g+aaGS-=*;*SyM7M>(=FDYKKSpL%Bk|2cq1pq!8osTM`;elT8)d zJ`?PCi~paur<~ef&OZYE9Hk3B&Q%Lu`_~&cm$~33QTlolJf5LqV!URlxgJy>4T!eU zD-0ProAxN;Ad$g%Rk_HRxb|nBT)K#NLtOIZLLtWGRg)!w&x}l{-93&K$4l&x<%30* zYN)|;z2m!rk$L5p)pBr=eYqJ;@;WzQTNPJpGNlTdKlRP{S-kPiH;T2%I%^l}t&API zpANTk13Mw7=UH7#5C~Wns)^t^bny(dp=-=zhNUPmQ@8G>2WC7GfUvU^_x+clU5|X* zZ;L=`vMUALJ`+TF2UEl+Q@(~-O^l2-%0{iiOP_vTem1bu=Xo{qkz1h5wq^SxURMd> z+LDT>6yUt{jdq!!wz`$vxij-+c_sKeY$TNgXh6#HSGw9@I66dD@AK)G#b z2zP2bTff-9Zknxazu1aS8_8Xm^Z9oIsuWIdHvGG3bH)8DOCIeMIwO|NQ!1;* z3U9bt&qIq$Xp`?NFLO=g#E4~VGC(Pe@rWoU>MEbL6HvS^YyzngyZ)-m7u}ZuH1sMD z3|gZBIg;_xVS)ZlM?i!ilIQr;sbD!!r(u1wwFT^ToA`5@f}JT9-E zh}o|9HT@f(@i&qIQYiCiY_U#!q6ZO_z$%+v3pu85B!07cYy}>Yv!3GzWwy{Yip5dv zO*chK=!x+au6OxLyWZ-YsFl)COm9l4=r`J~*>PuoWCOKu%1Ukpwbb{B4v5E#g}-GD zVZKWc5fRVi>rJuaLbldO0~o|&YkczJlU_%IQhoU{x6i&XYqV`I^&wton^+F`)%2Jj zbOyI-K9IRz8cNp-JW3F%0ZHqOGY@|-F)Z91eDpV89~`{e$Zeb7eevPLi<^^Pe*piw zCJ=7mDFw?K`Yp=8V((m@iTmY<@;ti({-}0~mX0pof{}5ll5Ac?2E=ydnlZ|l#u4e7(2~3Uqoen)!y8VT>*>#6f&a)UKhQJHxI?Tq}YP)cDHYf zRNi?ADO?oGR^mW@zWpj2^;~}a&_fg$LUrsOoHi{0cUBajqdLy{`z|nj?6t|v+*O*D z&qgr|z&j=;Gy<=_n|&(cwv&61{+FkQeTq6TWIFOwwa`ymyBO`kL1x1z&m@o%f)z8n zJUOi(0y_cR&ZVGeEEBo@cXgmcjOUlAnE6#z_=mJxjC#kSb`F&0(9s-gqAz`la1r@Q z;2dL`Zx_#2BGA_h#{VYh5fFERpHXm#&gY9((!3v`_6Os)EJ~or7i%g zOwt()%Y5lMwI@*iZ9uH;!z+hnI^9wUe9QVLE8KVSu4eY7x`*B(lSPebiS90=rNJ0G zKSjL0w|DKj#RlbW$_a0E zF-^CIUQ{MpmLG?7XK2U3^VE?E;F3;^jEAM>n-9%%wwa0xbiMC{-qB$~KOXpQt=fq+In1h7{O8!v!T;Acfi$P*sjNZ zOHcou538$JXR-<&WaK$1PnXmtiB&G^Jk+D4`jKDFnZyEP1r{gcUEH7)w5SUpUY@$ zKH7AzERlZ_`5yPsEVc?3FEHh17DM+Yn2%mrq+n@p-{*h8=&&RP5no0EuF^jocw|3i^UCZ{l{VjL1GAoIF z+pCJ|>2w{5khEt4%U%8uS>^lff8O!?v>u0&LV#{t9{Uof{{C|5DjZa8tY2TxBi=CU z(8@^i@*lbKW_%se39P7PR%kut*s=$8BNF)f7@f@Nqc=z$KYYlR52Uqo?Kbne1+t!x z@9u`o*Mt1~HO$J!7MFO8hmT;(WlMxSGRV9~i)&*~9v9pU-95{i*(wk%JBIQ9U0HfB zI-k4kKIWEvdJTHby)wgqShIb);QKOkX`++{bmi=~U7vqYEBi}=bym;GZdsn>&a$~e z%aLcE*U~2+CK3#+&A7bA%U01*2^M_eePNS1t{MlyvOBdUuHg9TIDaU!snBg z`y4$C+GK4N`2$~<`)e)pySkFE@gU4kVWam19_+O3BQV33T>@SoClT648WCSCwtzH4 z+euYdNWMN^%)<6@Hiv8|EUga;zZFTWV>IP9=TL$@O3v z1cW~v4c4|lPC=_d@|J5qzv$Cvo-Wf{$6S?6mlQ{Qs%bLShp}IVDnpmwPBgZ#D^&;x z!UdpRom1cKr(mtrJEo;NjQT!iF~ng|^6;*U(-&aXbWu8pu>nHCfyd=Fc-5cJz2G1P zUbXA4srIYkja^IJH_v4 zDPoSbjE~fecOj0zIp$e!rWP z1V)kS)~o|<4U@L_wGAzEzKn5BYMW()ZXg#x`WzYW^Ieh6LbfKVb3mtK)iU5l^Sa z-80_H9d*N)zmIwL&G0hEC&xgNb5~iE3@dNEYAQ}0DURxfI?QG@PS{|U?;*wz^Zt;> zv=Ai@O==VlY)}5JrRCoFXWQ^-8HuBl2qs7j)K0C^!s@QV-;%J9uDF6EaFaNA8%BDp z*{Oc*ccD4rF|4q?I&$h`t!UYXwKm`EGU@iZ926WE+Jdy?c<8i&oTLFOxrbN_lyBgu8~rn}L1K zYHA90Xma_ZP7kMYW=a34CWWuf|ASeq|J7lkVTl*rvqy5Ynye}U6SKRZFW8vut99ZK zU}he_-vg{4Ftzd@&z?Gg#E+3aFrkULe@_Eu5@Pm3kv!p~-ol|H}Wu_BQ%a`5>55lHjTwEll#Tf|}y13_H7);X@Pq|Wp zu%KfWvkCAY+Xj#L)nu_R704gm^eC$6-t-hO!|;r8T(FvL2ss)4c{+(*r>@Y@GTdKo z;07v)>VL7naCJGBg+6ke)s-hkXZ)mQqkyJ4NAT@J7Q4d9H=4(0fwi&C*XToeWhO}c55HJ|bO@4Z zY5o5v@7YcRju77Ml4G3k#=bJS70-!eYQ-trSpZvP_wv>sslVA~f|KKJifu4R@VKK0 zMZeD3N13@zwA!rT5aPAlidCn1o!^s<_KK}y<-OL8P;MkA-M*NQ%8ild#(46s`^&j8 zxzfB-jh&R!wlc%~8Xj`x*m|S$p?l%MxaoHv3{sbb9qA&=Jo(4xBhjjvXSpe!b~qKT zClv+B$G0MWD_QAS7kaWV{#bT*_E|5=U8m&X+I)*NPe`ub(*-6TJ^@IO>}bW)o}gh} zh)h5rbl3)Mj=j{ZD9NrYuk;}6*E99OhV7>rVepOrPRFvC{Ctk4t#WT1jd zpgcO=Y~MsgIQup4{?P8xg8p=$^*vkLzXpY;Zn2L8Y@+`Bi<%;_zYwVo11PzJaqKuR^XC%v|p7b!p-b$(damfwiC(fv|m)V>&RQ`)$Yk;zb${2ukeMKltW;734oLDktD^!bqkQumye@ZmK)W$HWUPvIjYTF zBK}{r_5Z(WztfH1bjR0k4OhuN$w>*AXO#TI#^00gdJ#ORq`S~_rNS<|SZ*BCwSxYl&^Er+z zEp6Bri `7^$qN_`d=Td>A|DTc)D9*d_Z&;@Wh54sJ%s=3VsQns!bC(X;cwI#uC+jL(URdVPzhj;B4!;|PNuN|zoz!b8Z%my2E&HBP|VSWN+8*VPRWei{*@Y@gv(nxuxxA zfU5RA9U9cI+RAhxxhqN$N|o+O0=~F^n2K32?|yqo9~%vvkDg0=JV90UX9!H&zD)N7 z>dw)-miocPPlj5NoW-D}_^U-Lrh{6F^?3Ca!FM++}#1{gMp= z>3cTOJqn^Nv|k>0LhbwUhdTdiJLabA>9`DW3cZm)VpHC5SkM|F zD7fczX=s|0M-M#8YWSqMV(VTKA-W2x`Nvdyv-g*t@@8XPY;x6z{o%&$RUZ zM@ik*{>s6Z^0=?bBtpGZ+f6Mz$J}loqgl3^l*XuMzS+j3>!c$PR&W7T37hn?Iuf0` za4hYIG@9nQm}eYu5;|{VLF#abRfR27OUDcFbz%Zg6^@}aeCAP=pEs-DzI#EMXvr(x ztQkQ_y>GBTPJ2s!#LN;36^tvJx@6BM{7!1jl*Cl_>U|BLU)R9hPz|!`h}brFfz!UO zwkhun%kh5LctI@>7h)q`pm&-^;x*EYf*+RmWhHTcf>J!xdShtvdqOp` zZ_^bNTQ2w?4Zlzv`~SPS;u=T^|MvZ_*FD7AFIUF~hxg9UT|jJgWTTDg&M*d` z9(y^SDYw!m!6vFdUk^r!{^TOJRn8p~-2AHXf}Gmw@nU%?S~@T!voG7`bTC&-Lu0B9 zEIb@EO3H=iFUD+?9+k>kdx$G0>3l;zm?^e<759l=!NU6<%WrdXox`P`v(JQc#bYsU zm6g*y7j3)==`&WL7{`;BR`#TlX|i?NF@RU2seNtZ+6i2h+!{^zk2p4c$?C^cJGk(3 zX_(_3C&#G9HUAXQ3}EQOz9Y0W>LWPwo#FWT>vjYX(9Z~H%~=DS7moz~jNT6@ z*#^z$^x!xgF1f5q-3pg5u9EtAdVVWYrSeETsx;d(#8T3jQlgKGD=`Z*?btAOCRS)J zb4NqgyYb!93_gkDm+baOaoBX$y#DdJ2fi+gW>9S%l;!ik9op)(U(0ASUE$Eef4g=G zJa)&d6@Glze|X$mBba>rnWX;hf~G?74)gW6m;w*OnzH{j zCReIq`lVIb6GhEi;4gNNf|~fDj37kb`V*mu<8qj_Rc7Sb%dfmxqCCdbN4OAWEID##sj(>04ZWyD_bcYEvj=o|YxQ?U9jBt(kP zP3dTksE2ssSR-W%JZIO2I5-{_P)J#2pU$5T(B8@`WM+lw_*A(pmYg?HKh$~PAx2$P zN4zv}s{y6oy5on3pDjg}7&hnl0VG6^u6Bj76)t0ZQnR>UZct{5?uwzLheqD;ZE-G9 zmFT|r)INe2E{_bfKqk8@T>SdrWBcX#`r+Z>hxUuP?WxL(HvgmbBB1jec!=nW`?t;- zMfIKeq(1pFcJkDcE__}G>{G%$^l-)N@@p#DU5A%{@vYs;n9|QF;L!mdnMc=K85b3* z3!~oqtqve$`K;W2jUi*~%xCgT7O_T{+XP8<;Sfs!;Z%M)%C{F*63_na9Qd|a2p%CT z((U7kHF&3gsFa+dhk8`+vUy(}hZmc)YXeF*72>KN)_o7EbTAtQwyxr^{bD`o@ zI@;YfFr;dcbIaJ$q#U%Ih_{NZo80=ZygHVI3usNGd>XRr)h|(&kX9un9VCPdUvf-P zV~*Ol7Zv4;t6J;zP|k?V&Gky}5vI}8Dr?~B>#feWiGy7bqq=HN#SUwy?N73Hm4^}w zJRX{MSKJEd(XW)u9{d{~F9l;v}yXpyW3 z-DUpNs-I0WEFO6I*q3fB6i9kPupO$#L2d0ck>^a$xN0}We7>EX%rb@f`J7!ww_h~3 zdtYC!{a^X#1x+dKZLPjn=i4`z{=Qr23ox~MCJuPZ*=N)=Q6qHieAS}+Jn7{lWpmLp zoXxec;)lgAij7|zHKPfCuAyxwJ1st*xmE2E$D!BIf>UYzlo)T_Spou>qz- z!QtYIw{Bza{kPFaeTZ1l0h7+67((|m#74N+I!3LH1(I(1$3ZA=r4iKU;xwf2HPOdo zf<4IqJ^=DJ83L4f8_Q!Yf0tlH2OeI%%pHE2* zx10FlnTVn>6*kN8+Pcct53=*R)w0H3dl&gmyga{tt?)Ho-Sc_nd2u&vB)!AHHSz3dL>yp#+BfmJ^s-hjeh zr}p!$(&SOFg^s?h9YZcEHdK-jtQ9Oj#WEKB^5A{AdTpNHsJByy$UA3#6abVAak%XTkHQ=o24`)_1qy5iFEX7R&-6mpjjFgFkzn371J*?OXP#e)}ZZ-RE+X9Z&| zzy3TwG>C4ul6zWKo4rK2Z~doWN;J+a3iWPAw_W&6J%J?-5e$JS)dp4uBwNOb#R^mH zVeEt|ABtLMaCHt14ouA?o0f-82YZm<8_Z@Lt|uA%t-5U(BT%_WH^_7-9W7Li=|%={o0VNOf$p!knZ7}T#>9W~jNrapR`-AP_5V(towMVt7f#n# z%KoB^A9)R1{>|;TU9Ki68@6~ji)4>eXy_z{e)HkZC0Ev+ZqWo_ash24>w`NqjU*zM zvNFV|pk&6+nSDQ`)dYDQus#1?I?2_d)9~ib-yYLj6IOH67#$jVt-rmUE?k5VaI+|~ zQT^K-^+Df~E27V%>2`obccK&nN`WJ^N<7v~yayah13$fJM&nHHnY6UNdS;Yqx@n3w zjhUZWIgmxp?xsE|Rkbeh0=J#suX~i?g&W0QBi{Fb7 z93mRDO?uE~=|4a4l4mJ^4|tmYq^CxjN&NUudXW+ReiCLn>6PzG_4nsOki00xLUsKg z3e#Hsi8ub=DDAIa)Zo5tH{o<&a?jSWI$QRqA!@$7P2%bpLSR#6BQvZClM^jmBDj9# z*U7GQ#>sMi-I%c~%T>~ut?Spx{fac)o{&bGm~nOC*F|t((Og!wm}=}=_w4Qo*yxI1 z{T*Yv`PU2X)_)t?uL>My{cleFFBWbt{CjEJFAp33FDFbH`IA8g(#arpMNP7g%LL=; zTku;)hc{ALekt=%Xyw;Ym%9Fv@(f|d=cM(;A$nN_#tJmt3Tf~mcc(h^m*WPzHHDbz zXO+jOFv%!QO=dLS>`~c}DoJXgL4yHAFiS-t`uy|FH|I@vNl~#U{cP*ml z40cRoCL7^lM?XM@oXr(k3qJOR?^JC=18OjH*|4QjdwsJLc_LAAqvSOjFuKGfwktFciEyB%+{8ySG(msMVl00d|m2l7abgt){idGUij$ zdjR_+p_lU7cUaRvjsQD%elDhTJG$Os<23~}3T;H8tl4$`5P(o^}t6aD5>zs#;d-FR7H6N60;S<^hWu;0J6ucxKF{Cp=#bK-o} z;@I^?g|bj(SMpm4<|OB^_1_&BV#Lyc$2HhT^A`a#2H=xS<+l8v6}cgPbGAEcR5Ejw zcC#(+dv&ZuL-W#1B|l+yS9AZ;Lv)L0gD~ZKwKzGVLBCYQ4J&HQn8Q%*Vy~WMtCm0X zz)LdoRx?`Qoy=X1sERh+#FrAew=`6I1a~3cBy14&SKQxf-%@!$kzad#U$#roW~`cP zy3M7X-jvRagv=r4z{Re$%O(>i?Nly<$N%318S^J07>qdM$;M8Lf7|rVsn!ftF(J2E z3{2I@N&I)sI3xc9vkkOpj=MJ0Qsi61x8<#z5NL?m!YN6k8_os=T3-{=?L2=l%b@sb zKYvdXytq{+WiU;t@$(ZKk4^}*{6p3-X3n*#Uclq0u0-5r2OSCLT?J0pfw9<&hKP{K zpet7K%lCOyTId){>1P{O*G&7qcwSqv`NWl@x=2<8kk$8+Y%(uZySXis(!@~ugiqHf zqdbFqG4jW}yl)b0^}p#)@LE8Kpy_r57LNup_n&2{CNh9dvUy2m^!N24VA>fpy1oQo z_`(Z^3AbSQw>d{}5c&I_?rHJ=`QU$g;eTG_KVvVVG5chNF#1%Lr_`R`6P+;i$XGKt z$iw6^u_r@+dwor5;|=hxZqFdn5O!K*kFE8|2B!{2?`5k}7E4BcA6|aqq8h9z!Tj~F zJiQsZymr2Hww0^lnYzJn=UQH3SsM?J2E5Mi{Ib(SXy!&;|u`36UBs?g!N=IpIT<7>=L6DEu z{7l-?PsM=#J>6rLL>7Sc()G-|`Sh*nzww#I02w{soq`G`sE>LYPnvptgP5zI=yPI- z&l<(gbUG{>F$>$a{xJZRk-S^%mPK&Q)x)9d%b|IN+qo?a-6dxLGY(f;X6Br*}? z!~TLS5Lj2pt$n(D_=k2z(^0Qb{#4%+qW(`pylJUd`5aG1FzJ6BorOb_ZQq7Rhp03I zlo*ZFXb`1FNJ>m{C^cGXq#Fi|l5RwD)I%$+!02v}ZV*tA2Knyy<4+jd6~F5|k0Y`$ z{R~T-*hjvn&u>a2eioKVuAD9MN=iM^URe6oC|yc2#37>0w4-^lHvbFhk?Wrwd8aSp z`c;%TnO)hgq^xKjbWo*8M8(qk7BbJK$^mP==i+dAd_|2l z+D&%m%CE})Wchid(#u{K-7A?q-z}2m_ZQM<#??mhO29uF_`?VT(^8RG80oS}%;Nvf zV!yXn(t+N6@!r%m;(b#jkc5}n0u+PuZ%wD*N#W%f91uJPGU4&GsgPCO5n} zc~TQej&elIZivQO#y?QCYa5;V0D)}$yz*1t$*Rh+sM>F~%Y5Gi_t@aY-{e~uHvaVV zb#}X}d8-1tY$6dn@U*hdp@)#V;n4$06<953F4|q_1ljH*wX1~udBp*@^Xg(+>271t>W?xrfi2I5LnX4!i?})sySO}74N!_DlHvN zuNIjbM0ynprSKVb|M2?`!6mMsRt^f0T(ohhv@)N_NCiFfs&!n_H5-2t-JI7N z6izo(n?%#y@X^`&?^e(yXXQG$hDK72Fm~1Cez2l#6fqeqDi$ge_;1Mj z`l#jj-(uiRN5{=k%WyG3MO?S=^1t(InLk)CJ^(;)5#jUkbeT+s-)L)j^HwK*1jn;I z{D^)d=#XfjU+88|@RiR6hbe^BIG+x`4qf@2*@H%b7&}svN0@Dor^iH2AvaL%ZhRs{ zHsnU!9F&@nkwR!%^8|^5XHV3CvOQM$GaO-#qF10qlEZ3K$B+@^pXAwVHQ@k%#$590 zQHk9Nc4pC=G67|1^PtBV;VYKj%#xXqR3w0zC~vB0EFx>*FP2L>ocZc;)n+v%iWJbT zToH@2G%=9cNvD~WQiR#0T>r107CtE{K%bFx33lg|Hknv z{}%7>4x0j9MjD)arE{o*cp#Rt(7Nj6=FRc5N<=Z&ECRp9)aYu**rPJA{5UP!e1riu z>YrG5XXJUs7ht7Ertv^d&7cv(`!qhmEthCTz=+6F12HPo0#$yh;V;9Ka~d5o5k3<@ zQu{{Hwx!^O$b%Xw2(OG}QQ;Cp%bmxphaQ;~FHV-BMS>lA#@I?87MjfJrj;#%D*Y+0 zX^H+bv-Kq9QNJ$AAe7w~H^w$s9MI**W2h6sQ zAWH+0UGn9_*MM$f!29UHFl4m#HVVZqN3H*#Gwp~J4nsXw$|f5T1F1;Z;V?j#PaGsE zHqr8zB|#qlzT{RYn4Ro>Swv0A#UlJ&Qrnh1v8zx%k}E+F}$<{L#(bF2b9;}-K6JIGfEJxh&YDmflhG+x|q zVDI|$_g~gaWpB-Ln&zAm~6& zk)WZSDrxhQbPbMP0y(*H5``=}Hejp9^!+3IUjm`SgzmXWt!|Be90Wk@(f zMM*9wP;SH^m7!|M8v3rXh~TnSoMbD^T14iTz(;((XXo#p{{6$7SLnY7Rads;+x==L zN4CN8h;=yBh%5@4iXtY7t(ieuzD8!US(24Z$dml{<9RBntqa*{v?fB9^VzDGj)}J8 zoOE@6c~}ON1J|E@x^;gmBBgTM`vvopkIMhetX;_Z-5lL5cHG}|-1i3F_5wk4U|v*; z5XSO&M7gC9MPkTEywI!(jq7>`YOCh^T#eCHpIilqEmd|$ zNAE@Zu)Vn$qu=7ziqwylj35^`cV}m3$T}R5)BcPIU(ozl>it+F6shR%ag%fPX6_vs z#&I}VgUp3gBP9il)NXlKHI?G~kUPb)uPbeEkgEH?RI%Spb610p%od$ZJr8`!sp-ws z&#oPZUcIH1=V{zxlT}h`HneqnCy!<;& zgO5|0Pye4qNk9Ru3yplTuMN@|{gz_;?BUCXWe6&DEJXgPY#a*rog%opO5oob(4@`# zx5xay=|wghMe>9I2wb}i2k5hww)>Go|BOxZh)=I`)+76PC4yoU;k4zyW<3lGu5{1b z8W)`qvs{*-una}DB&bEs2VNtjQ4t4~B5Xqv83!TVt=1p3UyebcSv`J5p`4kjOWr%9 zb*z{gSYrk&bz*{TBBuMFG?|43z6B1#O7cHQXUuD%>;2L6_};0M%OOc zOhc18bwUaO3GFARoK&ky7Z`GT*I;K%kbu~1S{(*6sUGB`5M%Q|xV;gj=>-F!SGV`` zVB_bJtwXHyj^eENWO^e;n(n+(I@~osLlF%#app|ows1|Bb%kgkkoH+h9Z4a#9?n?L z{kP#ulQ`jNmH$tF*7CpmRCtEe_#iOM5OaY`k6+O+Vic?kntUz0lT0S?8S?Tq#rn^e zR;rSMbV#&aX2!E-ef+<)YSemGA4vRi?eorbW;#moiaVUTVm8yr;VWKdB}cZmdnn>C zN8&Yr{gkJtwtSNu;--~FyMynQ`9cn<8bvXDZJ%rVGoEH5iP7+2^^8W)_#Vft4!^%s z*uj*)go1)w)HL=RsIh@>3PJ$Iw76ya&}^8mCVxsTuF4z!n{Vuhmp-a8_!@}ik*o02 zr7``Yyu)x93<7-a(Ve>JBzt_&@Aj&!;#>&v!!B~m%;I!iV#|(0@f3gyjdeN6Dt2X} z7~Lq{k%hd3clx$L49-CBC1yS&whaTM7$3w zoTzMG86oIs&A2&c6uvG${+e0H#e7w@m3Od``sFq?0bfUCd5Db9INYjVoz`Cb87q;x z8>u1v_JiF7wHePV0Z75L>2ss2M_+%WuJUGE1#~|#{fC(fGT+jo(pgoZ=#}0zx!L^s zqjL-wl+h;6dXN=kBiQlr;-7)R?9Sf5~cE?0K1k`gVmK~n3+>FXBk44YLaB_Ug33urZ^((_01bR|5ryPT^ zr(7Le-5RD2P1ZGdmGu*VtZGK+U$EWIT!kK76GKRVxgWp zEZRH&+yN9YzQ&|G^#@F}6G1zz*7x!BT~jGNsXmulnw3Yr@|XLQqc`NsxxLyi@BFA# zsGU?J}?2KiAy~EK;?ewX>wOYJmq7ai>abK!FUPd4gKro5r z8n|o#DuLYai+0)g5vh9k6I_v5C!0#yCy%mgiFc?;@`b0a2`L_5G~Fy&!{H^9Wz4>E zf7`Xty>59%>EZIA4BN)A=}yKbitd|#8l2XJXjqD62>z!Gw>VJPP`U_g6Wjk{E}h0f!EAV*U*Qo*bCSYH-a?p<)4D%H$q--9s$auO$vLxt zpe*}C<>;qcoH9qX=+AOi4v@j*p>&4Yp{uh&189*BrB;RVSJV@M{G$eAo_{?W55&}Y zY%8K9)z8#yJ`)RtDpFF=n$uF}TWjjyC4#I7L000p_~4u8$9L{e94ybXjx;qAdhCr< zv)6C&0gfkbVN5Sp;?-naZt2nmnF=|oXN9JyEBuC&MUc)yY*ZN7DM zR4*P%#<@uDu*Z>svk0){{U{bgha_iNEvEcs$Cg8+o|Bjqm#SM4PUWWroFIe99`THl1^=us& z<7_NcB=GW9qal=lS?;^rZMb@u#=KHb@?Ni+LcJqN%U`LD4K=WhfJh^%4@6GS%876J zuAj+Jt8l1bU+(u0?Rs&HXTh`d6X z4X-C;5&X!Z z!}T#r@bO8`{?;&a(|pkAv1I75!A%xo8?3KLSNd&psG06;pz;q34v0Fm==y=^#vZS5 zf*Tn?d&;%yOh~BkXty^eH3BIN5o)MX*HW;*^}tGKO!W@waUN6_@pSX#VUiDq+nvM_ zO;2ykfUQMyR%H{&X$rUJ7brJ^<~%6UBlHOSSr-Os=kVa5%DO;DuxXi(s= z{1*}}M#hc>bK*(mn)+R=hRiHrKFQaJw;ERhx)%$fcOUG9gB^2lSteK9rJI6ap+AB|pt~ArS%95*B7okjeAWV@VB~nZ5 zbZY}+*3{8Ir4Q!x<5M1x3Uz7g%N!WqIYqu~@|bQ|e6{eO!ly@L9$i9c_D}0`l+~d{ zQ3EP>Z$@nfdn{_+DNlpf#~Pn6Hp1a+LJ18zF2t0uaYGg4WTPT%4O_d`%sNX>0XuA& zsU@&v`wvyCKn*ScCB{?IWS8^8x68@9Z(g2BnOMNVjf*2h4-vay1@edZafxFfBRtSc zvbb0YQpzM&#Qs;S0QhyiCn?5~0B{6@88RQTjFqt8Y~xe#ughGhw{(Qd7sra)|i7c#*yGCqBZz ztDDg6)`;!Q^B-}~?k4bzxge_dtPtI6^;&wkYu#_+Hf$J4JzvTM3nZ+RI{4E!3Kn9D zqUfl`v!b;OBMB`L?Rv>~7_w(qt($CLKbHl9wArMJ&+tioF_o9w)R~{GVnI>7f(L)7Wj@%tj}&c%KL9JTl;g>>kh_NM!%;$a|7Qn*y0S} zKL7RWS2+uDEgJs>atO<}W*oOBN{YPy{<36Zaz}Tjvu6yj;6`PN1kLFwoi+P5sh7<~ zaxbHa@tLW~=fTrk_T&0u!clU+pOOsaIPlgvJcrn02H=EAep9NXux{eMilBTQHg@&o za*ABBl(OxKzvycVblHLQ2SDE|;W%g4T7`vS7nu33sZ1Ox=`uN z)$1tfxjg%P!Q{R<=QphJCF;pwr?zKS4G3?Pj~`Mdz>>rb2%=xolS&Bj7~<<%d`~VK zh`TOT*SpU^5r-5jBdBTQm_F+Wk>cw-<-$>ee(MKY_C0alD~&Dw_)-2o^7>Zp`iWoe z+qIyzKgyBb{d0~JbFA{hR&$25_#zqAmZMAGFQQPf-OM~4J~{&x-$5cUHRz)id~=4U#t zZa&nln06kQo+COM2F77~w}S`4WyD49Q1?vN`80mqiZ!r6BNolf%sdx!8s2-m9&B!g z3FO)|+t|d`75I36n@LLm1RsqpvB$pe>ZGNWUk`GFzy&B6fu+V)1xM^SV*#eNEFm2% zult?k6^oLUVuVBF6HBXZGZ;*F#bbTH@T16xk?>%AeEg(tp`-mmMg42gyLq5fu=%hg zhnGrZesl09J%mRG4yR5sdTt+pjFVU-*ts!XFrQMqER==%CqT|#<)8*@v%qEBG z1*Avg#-HeNdZlO4xAaZNsvN$j&Ptt3P$_BD* zI^OEsWywb$&x$h;(cez8Z*RW)E^QW{h=P=G3q%k2GJpNt)=G9n?o_8J7@YYqDh~!I zgzmF$Oq$+1Nf;aqmdDhc?r7%`94n&p$1ZaF zBugMc#9idDW^-ba(1JojIr#~?qh*uIH!3QFn8u-G*oL zn6za-A__&>LJCdFbw6mI<$&x2P@e-#ZpzhUiIwS~ttKOiY9%hynS2vE5Dv zVo+7as1mk(ftamt`FC<6tV|O%qu<*LC_=Ndf7;WQlm-2r$vOXjN=TRx@%V#_x4J03 z07mlhLr=Nw6MI0LW1!L`*AKPctAN7BEK)#>8NxbqlASiha}` za(w#yAe*tdeeuvjA3vBxoXs2^K@QN~>0*>f{Q=8y+5Y(Wcz=J?1(41>^ca0ivDL^q3ZELDgP*5a{p;>J8^m!(i^VVPJuoeuwQB*+2 zjvr~~6|JiZSG8)rc}JAi^CSLi8S&$Gf)ymOt@$m+9t59e5srKITCCs`UI~m^v@kl( zXtMAt)Fdse4T=OuG+x-nPd;hv&tYm}_cF4cwW~WydXRR_JfW=ojqv*rewx{|?u33c zE*!qa>KZ^?H(&9d0kW~5fngB0qPPuCBKHVDU>!ax1>(fR+xFh> zL+CCcU~EQC1bPi)X1knw9Prz`-EfrhFfs^Ea8cN|T@)Y28-y2?wXk_u zFO-=JxwMqqQvkRRN`;A{vMotM%|9p%2HgPqG8s06of9PzbP?4EihQYD|7FR`&o4Wm ze~S};`&?lVem)}RJY zH*e3I%iTW)Z^;u`gcreZcJd^s__|>U83qnF2fo3mQc!=qLnddpZBVW7v)KHKHW5Mg zMXM8y$EW*5Co$2>?CB8aEfVi2wNyfoj!-vA!5~;9OEAxo$*iR81W~u4(=4&IpZ4i% zav6s7Kf3l=T&<69Ee@lqTAvJ|OUY*Y8~=L2UUy-q;UabLP{$BZDZ516ybxEkres6E zZ~XF2)hRX|(nq`V}fCj;uLE)Ig9`y z5?b(1$jKD_2Z-ai{1^j1rn3x=#g~y}^6Z*)2tHaYSeYTw-j&KZQ;yJOs)#k(o_F7Q zgo9jx*>8Wnd6^9|;c`i!?eut$RNvWy_oX%}#<^>4h&tUeAc}clvex?}Ibw8*05n9) zqPZwWYUtWf2hU{s9G&nV1i*!0y(@)2H`u?X2>sIh>4o%4m%+2Qu;`zL0YYu#xgMu! zx_E=JiBE}qZh{=h$7@8ogHci_;970tAYwE)i%xr*i&ylPL6+!obP^X!?G9+OgZY=~ zH7MqbDAJh}6N?{8jGC#P$rL)8GX|X$K$WYZv1Yl~(S5vi-y#g_CP5eG;04b?Rff>x zNa|>+hk&8WQQW}5z-iIbY4Hd^>E*r%_y>Hz8yl0=%x0~BC)8KG4=Q~H+a%u#j4_B> zp(`;1(vhfqHqPJ`$HhjXARx@K^j`b@(l`GLB}?F~jJbJff7jjL!~6VKf6x5sXM{R& z5HHM>UjCQGYVf!%2)_z#$sFeRuu(6c1x)Y-CMTPR*pab7BQrzROXSzfrw63GJUmiX z_iXBn>|rh=9xY2i=DERB3ZPo%@Hr!Mk;Cz4#FUTO>QS~i4=rOQK-aA3>K5EQgi3^~U5#zUKDL_kxV5-fE%!U^^5cOBs|FWS zt3n=uOuix8(*;vCMHbV6i1A~+UWoDn<6xu^PcaA?l6Wd`)6c)?e( zaK;vakYQFWYK*Y4O=zxKF-4FeF8;2~cL~1qJ>UI)(SE<|u;P8XJL8lnM21+Ja*QQK zu6#beahk=N+b%X<)L=5F|Ge7TpYDgkzTFQT__r)p3C_~(7|@Lhj^tJgjxIL7FlHi( zVkM)Xp2uNO(tqbFcGFi(yB|i6=5QY3{2V@^A8%cyGZqWw3{7$iMp2fSQ$w*o%Ycnb zH{h9zYj!lVokAfBirYh;ByXa&S2c9ItG|Ymtc9OZ6o!|02h0#MeDD z4D!;$AJVcW_;uIJ+E**@$U&@L1SU@f;y9lID*=mGR=Sgdf>ZhBEuSM#_hSZ^=XR#Ll#O9^;|Fh7O8>_oxl&Tq&1@5axaKcwd&tmEEZ8cV6o>pwlm!&AWmNlQ!fo<(~2-<&(`m=>Jus;oL>?oTKX%ZK)%Vsb&j3gpZj z)>@xNdR_KuAQn#`c3L2gvaT*byY(t>*f1pWh#-uLXyov8Zjt8QRxh$X0W?Sj< zkW?g7h>q;Slqi)nd>A@!zJE&GjWm~gCzo251Y-%|)WcJ^Plhb#W?NaKl$br*1vRRU zHQ(^gfA;%xt2Eb36Q@*SkKDjR@?_{ka1^K^@(@K^RJK$Cx#3?aW*O6Vi@kXVb0roF zpI^5{>usy60$a=j9<65KU9L|!tt_3y9B5;DP)^bz3vt#YVVuF^Qw=fedUj|O><CyK23b2^m-;< zZ<5jc%K5ETZFe$;Nx$aUj~Svzr+{LWX8JyD0kxKg0Y?w?H-Q9IExYJ|GW$%EY<(Eo zrDeHLxVnCoSJ<5QlGsH7GBM=(62_v8w8UaoUCo@9Vd)_Aq_xhWsD9m_k0@emcp4I? z^|P@n=_EcpzH;i}UO9vg*831R0Rg;WzP#y)qa_vrG9R zhNsGZlqASjdsZ`7$t+fvKDEZ>>u1;MGXD;_RZevq&p}x(u55fcf}nY@jg$4q^?koA zKE8{hu8&b*%d#IewwYP&y4N7kP1Npf559pb`F^^YWwj6TI$XCzTbM8FssLl7VZyxF zpGS}9aTVi%0BmB3t9ZDo7=4m1tpdUo#y1M7yBx3tAgD>;&}T&D4~h-5dYA#e`F?zz z<2chrb+`RTA}wpMWHhh!zAleHRi)~^ehpCpg;B+aOu}y+K-5c6u>2~zoSwy7{s2je zp(Ll)@06_SPTX5;_|tA)mFWYV7~&za(Rst23#1g4l?!K_IF$R&DOdC0q~F>eq>{kW zOgCKR^Bb>daqNOf+d)&?l-xW`TD3W*V)UsiUl<_|5I%Ix*qcV+1i9D)4aI?p37u zZ=8tF2h9kz(rya6aCXdv4m&2(2oWxHNnTMmRDAXi2-ydEx?Bx7RL892Cd^Z2h~D)| zH6h*tkg0%xc8BLx$hzrQ=~ zN8J;Fkld+CA7y@_t+@fw1!^P63;zyNf!PBO-UrP8tAp+JWQ!fXQm5FiDW=V3?ubXyABx>ZFjEJbzQ;b>x`+tWe?~{H&!wZ_l zsXEYou-O+VWAK0a?B$Q<&N@l7Rc5A%7_zRKi}_vuLj%e)kLvve#F_ofcHhLH{vf+=-Rff$MI*AIQC9SL=+1V$y&42Hg@ zkB)jhCz5FrG38Zu#xgPIQI6ZF^v1V)e_g5qAmnTNe|lz&%=l@Srbnh@lfZ3U30Q`Q zcS(_NRpPz0jteJ}J!d6`O<@a-{^wh>*mQewZ7mPEeZYe{3%3||wL(oqKT@E^Y3*Kq zTpCR&PXo!v#S#7!r!bY9t%o5L-bJ3BgdP59PGJ^OYF9Xz#<$^faBSOzh-9nDZ-{D^ z4at;Hp&yY%utwt#`#FN)a8cV$D^2t=5$*$Cme0Jk#qUV}`;Ygxv&$dJ>!qJAnvx>O zuRs3=2+d#!swaBC8~%vMK*XM&U0KVzt9gohrR`|*eo>ep2+Z-I8Yh<95L%3)>AnB- zr;kng8OZh%9>_8{HO_@w)8hjuzuBNI>q|tD?cfR2zo(lOD_MQ8`+}`1wT_Dn>xIa? zpJ@c_Xxb|KW@KzkV(dMtdM6?@V;nST?bgLc_(sj0$vpJWAnCn6js@;db5;k^dyNc* zrxSp5J%=&{hv$2kJk&9^2Ga~IS}h3Oe*lZYWx6&O81zhAh&${L2u(_g6dp&u-wjP; zkArA}iM~|2-;vvTxp;Up*XN3r zDKhFP5@!uhGrzp<+USk%__tp^?D+r$oEWroWsIt4tK0r;_xD#$PEI!46SG|Y*JkX6 zpo<9b_x>KS-`v=J>fHs4aE5;HJQ$uG?fBZ<3Y*&4+CM*h4z%4H(>rd?pT{%H`u$!F zK3)Vujg(9;eDqCvVc7r3qr!GWVLH>rMnCj1!E6=7hQG5&w}EQuqBy~*B^tTO*EmC{ zA5~`kj1w-lrw9s9&ML)_me?n3S{OM{(b`w>Xl$FIDOjehU0;2PLg^u&Y3;r>VovqU z#$aoXS->`MxZL2r(C}w(4fE^qHclm^8eS*W)*K0V%6Je9Dkmn6Q|8eCvl+n$Q7uDv zLc8s+$igivZW=`)Axzi(SJUlh`HW7}L_5sR8U3MeU=QP7G3ycj`LvcZqE2hq{JQFW z-Hg`LBs~+G=u7P!)~!5j~G1x@*Og5^mv669IWh#H*$ zqxbFlui9gOYs~k@DK%v}Wq_2?5I(^-!0x`#c=()|+SU~x#*XiF6VPH3xmvk}Kd;?A zBP5N>5*jft^l3dCLpbG2yhJV_e%P}m@T2Fhx+CB;$E$n*yGyj9L=W7TZ>J63zPSID z1cW)qyuu{w59d#9{8zsG81v{=DQSAYBrPp%RQX18QWE0oN!PqHnu{b?AM>|w;I>3a&PEE>d_vW=FLDu36#;5;p=k?Yw<-2Z z6}o;6;@%9X{--xFm>{Zt>l-OI1@Zqxyt}~^B5_aIc;bFH=VQtyOyGKDKe{KPGHJ{m z)pZm8ZkdB;4h9c>*dRQdI~(t89*7e{m0#=Vz_SOoOI3Ni5CD!L?Z`*Lt$$)=pp?x- z)0)}(HT{Li1d(YDeaqkHYFU1QpNMAVplqg*m^C{}B~sWpzL&-%OU#sA`J^)^q~AhP z<#`fC(G5?q`z#9;*bsN+oQac$pXzr}5bc^Nfl}9-L?Ob5lp*d*uYtNgu$!(p{SQxE zb?{mgYpDJC>v%iM)GCJ$L=~q{|3y)Ec0Rwp1v>P?ZjK|UxPPA*xDX#0Svx3v(V@cL zRi$WMYd&`rS7f$Z@=QAv=MDVZ-@9_RcJbD4OQ{f5+J4OR((iY#Y{1ulC#Q(n^R}j$ zqbdQ9t|cVn&DwXmJi++*_bbi5z#PP+>8RpOdEv9>rD=dY0&s2SE)a;>y5cfk6MA14 zZzQViI8WC9{MURP*W(Ul;+mDV(~UfeOThgO5YKGtu5WLjB)|Q*7H$)tS4aY5uw3Bz zo#{K9^!j4iMl+?;yrSV%_vZcxlLSz}wUpzzhW%IAsY>LILdk&@enW{9YBr1O-w5;v zjq|ow)sEdMw>gHLQdb8TxN5;w!{ty@T7?v_^vjr5&knmL7Vp@P=hpXjNeF4X-}e>w1`bVI zyJwUvr~7@^Ew)Y6(iV`RpijqmzsuZBNf&z9qV^{g-#C^4*<8!rX5r<`Y{{MOGc`7vqONR zJ%yN$O{~1!Sk-|gy36_L`==Xj(fz`Y%~i#AMvN>q;L~up*znJJhM*Qt+eiSBvy=5E zsA{GKL5Erbfw00EqwW7r;+X^8{e0YgT-{x3g(J`$;*#zy=l10o^jL3DANwD*g2#}q z*K9Rvz~G%VJ1_68si{*{F2I^>T#QDEG*%r<+t(QL(O7giM4Agl#HWeu{G3nXx^G=f zpYFG>F%{Uc`vWYRUW|(RvmT31i%XB#Jw~(eDJMlWb$u`k7k55D6Id_KbX$lW6h^*% zDh0jac`2>M39cOO2V)NyzL1BM!r+Crm72SQ#c~!I#Qudh8aeuf_O$~2(V0~yuoPmE zXch>UlBTV@V}5(Yf)cEMCRY^YLIuOy(ZhakvJcNDYkP~n+HV9IS03$LkVsvzu12?9>Pr3OX?-T5O;3trpARbM4NiM<*cCrC-htQhajLRX)J6 zrUV<@qMWBRk>8K}&#on={0!JypM?o!SGPDx=!;zvhr%HH*K6_2_c%t&{yUXDCoL*5 zS9doTV%VPd7yoNrcs@sxvI-^Pp_~pXQmUT&UT>cK`i8k?>F+(biK%}>Hs{eY+*3EW z(@0-4Nr-%D%*W3vFL#E2@|}pe^?aawIgfL(u(d#p?RgMrbY}#zbJXMn4*(?O&!riXnj~cX_b-HB#|+%Ti}tO7XfH+>Sz1e<)4iu zq_2+;5KYLme?rxzmsSnsENleuxeozvWQ|-ZZBLql^;8hp9-UErg>8-7;D|!*8>Ng{`-T9&fP4%iVJRlI8 zJT5UMOEJ6RsbZSXQReqzwzz&-V1IPZs6`Ks58MZp*Hw+IUv5Viz4hBo`twxbT_>Mf zw7|S%V%ON3rvgIarM+yJ)9j(~%2~iHa6@fdPl|a45rJh3?sy$_e3h0G`-Q>yT<(773cTH2 z`%+S_bb9W!?4Fu6AdnUV%wUg!K|JtMqafH!OH2))dVELjh@iJb0#4TWH!&t9y>&g= z7KmrPoJiJv#G_j@ z1}w2pTdLIC!4w>p;z?npg+XQfNIk2vV7*|kxRF(T+pRtgPaN_@bCnq(HVc}In2B_(=B@9$YO3{QoUZvoKJ2GhZHo#U>zXQBx{p-Orb&BW%`7xJ zQF$t0w%tb9t+J~Q(pAtm!^osx=c1Npa_Tl0`HamJ^eZ>|Oh3J4vxZ$gS{i7HHMuxWCWA9Q~lGB6FWK@~Pji$W@v~ z#odsRwDRzEy0x6Ha1);y*oQ}8l9;*4OWuZ|MvHP+?aKVD+dBUQDApG^~Rlf z%xa22aJ({xKNNY~!d030*oqI^77 z<8{xbq}9Ba0q!?5ubjR*EpAm{7}I{#6Z0_;%5q0T8{9_zv*`Z7BwI6cB)%kYxxIOL zRcF(3R&2yyhd)J<#tC)yS^YJIE^^Ix?h@8MNZ2yilhqr4_-Wck&G5H=>;pYRHCUf6 znxaC}XB=E-$cGgfSZ7)~o*TI>Q+i!REo+pQ_FkFxoqi#-HmF})#w=FZF2_RkF_X;4 ztF5$Q?7x#V0Ezn{V54oNVpf~^#gijP)(Eb6W+1~MHk6XI8_p5pbL=gB}j_KhW zhHK-jVMzB#J}kB8VEpJb{ayWl#j>KI;QBlBKjL2t=5r6*;~7Bl$3MNd8fLYx*ZNQI zB_K?BKbt*vj`lBid+6S_dL1tty!EeHJj`}`+pty~^K73I#~qP|Tf`b> zPHaA4ew~g{vBYNz9nDl5tUtvgRBeW6&@4f*(eu3y);Hou~24;5iy z3^<@eX+3z z&w=$qLwcI4kX05qdkTozWfgr*-2YMCfh)fe;$Rw&==r2)t-W5jYsXS@w8W_bG9cZx zc{1eJx8W;pMR!B@(C_Z@kRK$M9?7D14#Q`Th2vU51wl9^;BJn>+YoIJy8wNlX?Tm~e>%4a(ceI&RWQXWth=twx z7xA^9DiUKeOe3>RSJ zZUl4hw=ZI@_JKpw{gLlVdFBE*G~#KCfth!YZ+zh>lkC-QlD5Qw6Z-eU!H33|u1h?>(Eh zWX}rfx554~SM4Q&;LCyGAsC1-2>&pI7{~!AfaeTj!9@4B{gcDfDN5wMIt5&gH;5;< z6}6pqLY#QGyeQ&l(W*{MZ<;`3`qSQvz@waF%hEEBT7!2rF2t&;pU~|MvgnV7l4&I$ z2cX~d8J;!_n|CX+?>9n~P9}7!!X__y@xno>!V%|eANFJ*pX`kJX$ZOGIBj*x(f{@wVcNAcebsqUw8PJRXT%1Oc22+Q zIS6!{c{? zB2tS6pv(LrZc6Nj(qkQTPm3UBkDx8Czbn4f4^ zppz%U0Q~;HyOqHE%e^nahCE>$ih9ia;;Yxi2!OY|3~W3oUje#K{&!a;K@<9XU2>pq zVKO`h2X}m4%^d9VjVQ_30u(P#PDfWWrJ>QaN|=`BR_Nn#WWdLh&HlYMy7J~Vf%N98 z?HJA3z(3Qo(qVabn!wP`qoE87?vj3{? zG%`L9_!7uo_r#k8`gnUc%sVd)#mCRu(5V@Y(v>H%Aw4~;?FSnam8&Tgg>2jK;VHzd zVJzd{nlZ?!XSygIMb*Lgs8{nJSD=)T9&lTv&bR2P85}k6HGqltmZ}h})>_z@wr^Mr zXa6;!k&)f1s?%kz z9F5n*3C?NRgRi2&mIJcPrCURb|1lQ~w!YZc>N9;of_A*x4BReZAF#M#7%ZTck<>gG ziSgx6DxT%(ai+zWn7%wWc>dwI8@)t{yPs0Jlskf0sM--e>I(kl{u6J z$49fU_!42AL7S}9Yyr2ck)MI04EF1)|8GiI6!-U(VlgNG-*;mTPP6APqoGIpuBX3l zH4y&yJ#_IOmi{eu1YX|XwA}xrJMOs83;07P__O`=e=0SWwZDJU6GuizD=XvT<71&J zDqOw2P2{n-f&Y4D1(!ZBz3}zCx^;w)?)TI#p5r$g4}Vzs)-&Yz>fyuvmet7BNOA=- zj4DhHtRE@gJl1GN$lTfup15Qc#TwNnlGUY=x4Z}D(L$Tq<=If!t2=ScP|QKY+v-`T zCSV*&Zvlr-yM7ybme-o%c;Ychp-guEij?bt0~+G+TM@-TkqpIJIosE^>78))6Q?_> zCnZhKi>l)g%de%3pHIJX2@tzPBd?8fR^{bx16;lo^K zP4mF^ve8NXa~7TS{@Gx1h$3mSIK?aZe89#&n}Pypu?1_uc=n&*{Z_~A?T&2V?=8B) zt&_^8d(&Z`Q*XRonwUA#Uy(K=2s^@g&KmL(eHuy2Jeit@G-A+NE^ z=5hmiu5KUt7uB_y*$#2`uVmqk+Y5G~E}Dd-{9w-=Ty-B3|@Bp(h( z2?ph(>+Coiv@5gtGJ#P_1CVHB3Xl@UyfeORHnFjV_KmxAPA(a}tIz-BsAlo z5v5xw@}mutkf!4BAcuM7{^JHZIs&cATou_D0H3bc7v0In_un1ixuK=2Tz1&Kx^i~x zJ<7`j)d^eYF`%b;|KzZyUHNB9wsrQf1Lk;Pj#y08S$*2VJZtp%N!QloXnMN+s}~Pe z_uqs>fT?M@v;7dl7uhU>4TBNgdjl%{xR%K&;!(lf*M2(Rw)qZshOT8$s`7U9b34H{q_xjkFpE_`C}hfm)1x>kL}9m;=bD>++x=&vjru+z)b<{Rxo@fpZOoX zshx-MXO7@-#)73^Ob0}A9*D?b;-TU7bM_5Vi;8GdXHQRRIB>lVum5v%yBzNNM-Pi% z4~(^+j<(3k%Esl0ZC$pzEh`N?U+`ceIA6g3I*lZ?&tc>sHu(|6F=k|A?MidrbEaX$qQ+Yg{^x&n5@HUm= zoSuFnK0B>>Fe-o&rKag;t(p4K^bW;09z0L~M?)ilIYLH@aTb(_j|k?ZcUes$vUa&V z{nZgKHglEoJ(S~_sP)wSy(Hrl%H9*;F)BzZKW=^oNCSh_V{GA@dL6-mcaH_n7#G6Z z-BL~#@{aBLfJ7V0FEv#Kk5r2k{Jssq>@wDnV=Rwh&GptFvGSd!MMJLC&k@dGr`+=6 zu=UL9YMjR-w?=)f`xKwwGD9Wbj*dKVHnE#?jXHmsUXdz9dWUajl&lN+&?I{6Iz-Bl zBAS69S^Z-+4oiTPK$}mKy{NQ)@41=&^u@OCVyK_%C++H)&!}%NZ`Qt5`yG$v`0Q=A zCB1te!fnoXTAOpbxZr>5yS@dizHpi4L!J|)^M9|LeEnRltl`tH-eNk!d`Ivga8(VY z#m&3wqdDvAdz681yBi~n1L$-Jchz?wPzfFBoEz@%=72zM0=d>RquG5zZs6l{o1%F; zw|KQ1H8D|Qu^w=8Qt1s0{{K{k&Hi`z%L($?;Ku*9FB}M=ckeiC14zB_-$0aEL_fLq z$j|JSQ-JyYtpHZB-8a8_xDyJZtfjPr?}`XkS^xBbFi22M@$k}@R*+C87_suAR?yPD zy=kSYnM@S>*Nm0Zc$tHgT8l~hmX|{GSH#^t6=7kim^LrG%)aeouj@a*uQm^(rhb3h z+}w8(sOD5eRXHBls0xY=PMNoeif==#=I$ZzyZa$zk{Mcs5Gm$*tdNKJ(PJjGEmuZU zfmtop@AACLXqH|-OZV$hj4}$u!pg$>jyD#i*Z)|T&XWZZ`zSObEQvNfjt5MEc!Pvc zJZs6Qnjf~gQ%Pp{WJb1Zt?%8s#pABu)4$JWXQWO)eY9)ve4NslL1tbjQdsR6Lyw~s|)0zMNv>|9;7R&5(W{SGr1 z{QzLE$JJRGP6KlcB%Ca^d7oCgFSI$Yt3;M}5fD%mTalx>UuM7oeWR9({U?iYqV<)x z=RCufvZj@K&olw%S}&{RbjF4-8gT((Qi~?(jsyqv3fc5g z1=LM{P=vSQ=xi$MR4|wAufUHx6|rqt(qQsn3v-K#5IKa%dXU3hy$Sxe*E(HcY9|XK z?*T(^a9#bbxBqlhNK(>$L*r#7Op?IR>X)f0yIp`R__;5+f`|eZz%TkAOYx*?aU>P< z_GPM8hzFD?NYaz?3pn~d;cH@2M-+c?ufq!k6`|bTJHDPptU<{z~fIrwyVsHII&M2yPB=eFMFJx`i!&2K+n z=l``WRlqC7Ugm{lZXP1Pt~qHbXS%hbO{ zVTg<$_7r{k_#in^A`1%u@q5w(oc{9g#g`-!%~dU}tv;tM)$P|Oa}KTNtbA`k!9vnB)cIp*X|pMmf5{dM0~%X2z|`I! zyZOz1^Lu3l)sT&2RTNPE3;-vAbAAQ1>JHJK>+=nx&#Erk7i|Rr3?N~E&w8c1e(Nnh zhmX-bkZ_RUx#8AQ57dZ^xB%0gR!2p_`YN2)v{0Q}*2zoPh_P~yNEX8B264eptA)YnihZoU?mDDise4w6cNUp#VudjIn^>v$C?nx3`Xl zF`~{G>fkVl;U6<4)BUGkr&-b$3LU!l-EboKnN&&4Z(DQW#coOhyk&B(}0mB&8FZK;{_Hz>x>TqC}o-tw$oMAGo zV!f=|8;<|%Ml9nBpT9a4Ft>dchrnlEkU>SON<_@wF$5dYGvVP!o-KvLycm4N3Ce65 ztir+X%eO}7Tf{GZT~@apY)!46%ZrAWWz1_op<@7MQ!R^tyQj(Pa>pGQQZw8PRr_CG zogcRE{)3&q1BlYzi~iS^h|?~^=<_v++PNxfBUc~o8CSG~N#?BaM_qv^Ge8at0T}2t z#GtL_^f6#@UuV~$#RUw%^5pNmJwQtb<1S$LIIJI;ZCv=xi1{6j*b3@Ya|iAEl5Wp4 zMv{DT;gbR}zKCe2#~O)cj;tvI@VZXBCJ8TRfFO zqVmH+P9OAskEEcmc}n+0C?Bh-Ji+h0o+UtSghIs-P?9CUOyLP_oKcyd!b7i#K|wPU zqvWx{hkS@abaO}`7QNil)8p16J&D7C>;4}TYe7r}-nCx?UP{zA${z$*r8**z}(L$x81!XF7>4zg%zEUqrfk!om^%c%N zAH?7?Y%d*rb2G;@XIz6Y$jS58&C8q3+P;qS5r_7htxRAyaO#_>HUDF4q0O6PJqtf> z(;Z}3yrIsr%m^1ERH;ql@7;r%kNis62g7iN_-15%j{hP{~}O%lKT2XmY0XC)~-$>1{M866cI+5y<%Yy^jqNHtUKG3yL8Ul z24XRMIwA3I6XxudQ84UoID|mDYw0qt&HJ{;-o37=xwX;9&+){zt^EjB`Ht=--<0c9 zgyo*?+mBiga~`r3aJ=Cn$qj)>PdX6gFk5|b{V)yqU|=C5=Hd6{_+l7Ct| zoF9RDAM2RX_puVQte~w!LPi^vHWqPmgZ_`9sj~z{`Tu=jR>V+~!SRTL$)~l!zpk!c zhB{>yTUpD_{XL(;yl`2Hr?|@-1SWxJmoc!h_ef^Abwch8-S_D5zZ5BB_Ld5oIQMq0=iMGczt9(hunN)dr6KY zS<>)qTJx~yV0vew$M~16**4C8?eKhjicgv@tdl%5q2 z@5?7zRpr{ZR3LOKxm-NsHQ+2?Rl_a_-)B;1%y%UHY#Der=V~1mLK}bxtH2Ae-%!t@ z6bK{>4v-v9FQf1{(wSHM?hqbuJy@HlvlLdPB4pH?wp3sgDIu6y*KLSx;w>#^fOE(x ziNd4ngAk7N2QN8Lsu*S@TBEKnJ*?Gc2flAqh=&Z}<6&CUd zvCsAwD*AyTorUUjx)@CL#Id)oo<3YLXtGAVO0U^|Ps*?O58JvA5d}UNBzf<(!gpK2 z>#xG=7wxyesqq7Fliqtv61+QEm4I{Wblv}EeYf=x2ywbv2JYW5b*B6kW>2KNCdQ-JegYscDXEKoa}4?*-GpHLBqHp~ zA!9BkEDBr$INhX3HL@2vQHehNsRoETl;^SlQ zg&;Xtuu1dZ8s=tw#0;Kg%zhCZ>AFikojKIl@=%f>U~tx~&X_u0h@cw5!xsfYM8*dt zQqoOmfglV)v1#|ic<9vMY|DOtQGRs{;)3(Q)JaU!a6pAsYpyDASeiHrL&~rq@g)dv z*%U?^kwO{YZLPJ@3fT`MJYBdnLn>xu2VYrQI{TM_OGg1mhrLFV z8A0A;^Ba-mAB~^Z$v+*ji$Qu#L6%=!@iQ9f6f*1BLpe~Ru0;x|GP9c{(Qe(PbQmT^ zN<0cZ{g{oS-mm7)$8(FnMG_Yq>oj0Jp1$C14*Y_1 zb=<2iiGsapRCgZs6!xPZI!J3s2jb3GHZF+6K--gH#5o7Ei zfy|#*CQ6G|IF$rqNv2<~UV0rQQ-mU22R{czmZFdfg<;r!bKAHwqHTC^3)E9QYv}%DT z-QnOo5MOLhcd$CM88@>HP#DY9mEuYQ*9PWDVSvAi6;{GTfCpHQtOlx;s2A=IJsT4rHgax(vDfwHn1T0k%s=YAVAN{sw#d4Y z^^S^L8J1re_&PiHR3KQ5I9Nq2hq04)0(#Xl1}sT)va-6ZMtMsI(TE>z*~QOTK10KD zNhrpt>Y96(mzU42Q+#4@HF@`Uub(gaIJ>$Mg08<#yI%i0u?K^S*t1u+N9F$4Gydt- z8URChtmAZ{1^0t|XKTY08U`UU86FrI;Bi5jaW67aT~S$V+($Z2g8*8}6;=~-wm z5u>?ca;}l^&}+#|Qp)v*AmWWBrsCnqN^4*UM(?~$Xyv^t{cCjw81b0M@VGkH&FV&Q z(X+vdN9t4uz#zugD^N-7V~C-cLxy=3?w_=_3b+vH%_Aw82~^GM2Uv&^7wlrapp7jB z4Dia=?^Nk?NmJ){vx8?MGpN)Y76X5D8Y{_~2NX&RW|BSd)JTgTqf{~fOZtkCrx&{I z@%_9oN0ll6adoOi3Rz%wo&*%3ElBo4X2#1AWF@GD-U}!l1QS7G-i(^FmCrh<&Wui4 zTCWXa0GPy(`mFqqmiFE4X0flI9SL&E$iFOa5?=G);deGyu;H^IiK5GgvPh7!1eTlN z8`t!z=dg&RZ)q9Ul*K4%u@}vFF$xd9Dt`GvsH9AT!)mj<{K>9puAs76<(p^ky1w>6 z-<1~tp7zmnK!?+Db^rv2h6Ak)kPm-saC`hOg9BIunz?=|<;)!P?frYU+3!67M$HV| zfhz@ga{xnT?6PIi)zj8{FIk;c!aKtu0ZGG}$f82|*VHii`vgv-jJR$vF7<=kUtlqD z0gS%@r-)|x)SQF=dfkkE`Syi7!_`d`cRNpS>-C9_``@3QH&?co#f-@WWr#^crESQ{L~I@lVg<614a3;S~^D)9mQ#f&~J`QeG@1H)9?RpySRAx668~owTP1|H3%OgpOf+^#W%rMAr#^95-yAjN)0=$%W zPJH7y%g-s^gb0S{my={*DU2XRa)b5t_G0DRKhE=NQGiVLJ1wsg|)POlebc@#VPL#1Mpny)X?3f06qz zs|$drIZa64*!*iL2iO=yJ@%H4j)cr?fchpI2r9Zg*_l&T4~K;1IoGSSC3W$DhV2)g zRgND&`X#cON|Ko_FAy14R)$0|uNtI9#G?_kE2`sH%#x_wG+ra(Cwzcmq*R^p9ec=E zx31Zwl7D#wJw4KTGk}Ie{~3@iE)Hah-%S0Bj{w4`j;V7z! zo|0}SYQTbcSxjQ`+Y|u^iDe7~VP|O-0$EVZHc2Un^wJ3pN*pH-UhW{8{68jZY4B1T zJ3HU=5uc_b=KYmW@637oxw;LJi1tU_hNJ;X(YC@c3lx|Wagn#(lm`KV_?X~q)5X^L z#ZC)Q`htK+#BF(lA4Yc1)fHhNUodsm3H3u9_q z*LPh%w2kM#ng7y=X>|^npu(6By`!q2rE(v+46S(_CLsX<{9Vn1whe$?sMi1do&V+4 z;TZR`(Tc7q>*LWE4%cUW;$8q0+;NziY(e%#g34q6Y{Eh29*N>0W%KC`XSJAK9`bTy zD$LWEw+jA*Maj`HI72rj(-S;6AJ#Hj@-uSb62=ZMyuljnX}((r^idKsJOB7YT3jlx zUb*kA{Y!XTX1Mh@{kzk!=ywTxVs(HZS-Z@d7T``R&bWhzk|c^yWeLd@EBxk(<>a{l zkpvJB6w(ng)0rqaNg%=0>ScUEg%ZUR1P?*-{U>08=*VPNO=g?ebQ9)pj=IIx(Ih3o z4lQ_)%;F}m3y&6kdaT>T#o%wQkz0u=L@Y3E|CcuBySUiq`-JnQ-xZW%ci!Ksc9uP+ zCO)^fzmGs@u--kZTj;nX9djbUFNdw1^vLm zXIk_YF?fPeJ@8Pri6_k5xm<&sQXrnuikB;gSHD-+O=PsI*w%W9e`ywryyb+jj*F*| zf?j`&1_7~FD-eaRycz2`m)^*c+K#brQOSTzrXt*|lp z>~#XI&7Xq@l(abRx*K>f5Z{`>3xe(R%;P(U0KfxMxktDa3Xpz+6->Sm{GrLpbQrpHq)uq22f zBFo34g3GUJhJA+Q$xurhrhws$1N4M)S23kfB(Z7==)NEW7{4!1w#$T;i?LP$y~gq! z$btPfY;n4~*x_}v(glbX%nxN%U$!)6J+C8)v{dM4D@-_om&F-iibr|4;ia5(CdtsC z5OzvP%Gc7cNjpW4w+(#ObPjmBvk~7B$I7Jk(ycZ71ENcJH4#M? z6EBz-p3J~uwZB&6l5G+K1~ZHI=by=lAA4UWo6BwH*k;|=dlf&BN=zZlY|=k#y0aA@ zU=?ryEBp{?Qh?Cc&ldMN+W-n4>x{K|_eG%pyjTh*fB({x5?w#HIzRbr@z$1)dLe?_LCWxvI65yDe<_G4C03xP$pinxIF!nMS!MZ&2c*8m^ z-)uIk7`~i><1wRaju+!jB78Gm^xtnP=4TE@2Q?q7fM$+<3gabAf%1o@Hm3 zS64r^b*3;gJ7Tw2hyIrh{+B@Ip~gBbOtUQHFbiF7d!8C*WT;TtlX2H^t(ETKprJI% z#&&|=q-dt_;b2N60v~%SHZrR`?J-!l9FoPwX*KgSv#`@7TGesp1S{WTw0H{aUVz5a z6HwrI1Jk0m(}nv92DcY8faz{C*x=c_e+{a_N~GmXJyvT*SgC& zBKf7F!4+7^MRy9(6p%O6Po{a!=WyK;&o3Zfj|(9Gg)>jz6*+% zCqusX>{c4~)(Yu?J;%L%EB;IOeQN!U&KP$QQez1ixe0{7#-#LFKD}e4Hv>R!CWhCd z(iM(ifRXPMH;n~Z=4{YWgka?0GUQGl^-g^k39C6%8F}*sbYr@hds}nsuxX#rv2$h` z3a?Vu?>~O)I^FiBsOT_r5ExP4*KVPj=s0w(n!R7wdeA@hAchp-X51-UP)Z;fT~WHw z2k8d(N<84R(yDsX(qrC+*e9PxqGNV0^eKMSyQa+B&oKDywE%-61K-2FCjf3_;b)uo zE-=?!{#0TZ+yJ=Ye%Ln*0h%S-j5Rq7MReW39iwV~FfW|mLRVP|83Q(~+?m_)5p&p5 zC}4v}bbbgX{*;wY@s#g{4Vg|Cs{{uqV1tn^pH#KAE+yo9&r^|^$zC+%0(L}i6WH%r zjdYvoqG9kkU)nsRG4R^GD}G&zS5ZE{*{11toJqdmbvV)isJ4JC&gigjjFHr!*H40o zfq^Z*uE{6)SYC-xPPkH5B)Yco7~zH(Hku93#Ze0ya;h>2Bz~OdVRlc(VnaMC05kA- zuo^P~ga|~|d6&XyDl!Jkp2$=#rS)9|PB#U>x}8kEf}oNb(%|SY1Z4<2W7pC{>CRkL zjHoq3FivB-Ff9H%2QhDa(9!%-8x0-fy*g*Tj`w3W3@wapAxL1@uRIBr+;7)QBA@=6 zqVolvg2md(e)p7?M4I=0bhCUZS0y1V1qLO{loXc{X-!(OQop56?$^+L{^Vh^GI%HZ zodG&1s*2B!>b7<95A?h{h-JI2ucvL^{<%vfACHZt{Swwwzop9NF1C5@92%S1^E6}r z%WCDho%(Yr)MEw{_XM%Zgy|&>a?@7vjKBt;f}l>_ZyTM?2%;u#+x@p<<@%j~F$rG6THk6-^{S%IqT zvi%lNQ6B^Jh!v{ffshk5NvjnO8sDzn!jd3HlRoRIdl38M@gLc9Qk7nR+P2^?>ou}2PT z4mGFG4bo~Zuc)j`clkC}9UxIbU6yYKEYGbSMObB>TBS#yuL~BcAwkI13!l!41?BcC zT=BCAC6OY_0n((a^wzSnMLEC_etk+V{x2-m+0E^N$)D#f1qB8FHrD?X zvT=`#vY}ytCzr0gdd1_O^LJI$jg6W2%a+UR&HEpG(Q@VsB2%&)@LcB8Tt@%R2!3e$ zosJiUa|FY%jQ4qM;$ftj^Cw$dgClXfZ5Ka)w9at$f8NKAtD%k~-`gmItKYyESWz)+ zJM~*yYcZFSv~Z*{qY);8 z!ON1LQhODP7d`#{S1_H@34iFbn6>9U*XSvRo}gD*V_0Y~T^9xeER)uCXIwijrhqZZ z#SqY5xCA#>b)2^CUYFb+y&L(1!rXUq&8MW0->&q6iYrcYa&;Y83BM}k>MX>dr?g*SAv$mCt86m=m^{{b1P_vkCJ_VE z*Sp?lWj_f1Aa~RD3Bos*Tuo|XH~YCMvqOlIn3lCm#sr;G&oI4xY~OI0*O2mN1gONX z4z46yK}fyKkrN!bDgoy@Kn2C5tdS)cLCS3FUJ8r0xmq}W5)o@lF%LFvXMg&p(UB3} z-6kZt5Oj3K(U|tU8U5vaYCJ~`&Hf-SFBEW6ycOqvo+c=w6GFy%0R~( z8!h!1@v~JoX|uC)_;s56IU7!eoxL>8E0m1xkK2p7(P_-Yuz|-u5Tx26uJcIT@9M0J zT>J`n+sF;7o&nA3h8 zIQ6PecA9jvCYl{VsqaovigS)d#h=hcr@iUr%=T@RqX6wKxX<&DcCn+n!y!DYRAd=A zh0u>2DTGGM$}&|MYa3^u%FfPKj?Vej1sIkK9te&Gf|GpX__JS&{FiQywS>0zBD%pm zI2LoUHs{N&-##zOXj05V_&_KeKNz(A(tBJz`87IzTkh>`sd9L){!jfCjA!483*llL$2NrCJ0bD&Xz>Y zG!11gIGg&60g?1WIa0pYh-8JCRD1nr65A_lO;O-EeJgOtC_eud=XaC@7~n^?ZYj(-b}9JTpY0(=JyAH*|HZ?F)1~kwv&IVhQm|5+*bGE?74NyJ{|VcD zjmPZ7LF`_tGGZbkUi+)>7Cn|@09(tkPuulMLEP;=!!01?1U@}9z@gikotY#;Q zW$3T-$`=>~mbCFvkT((9tLIK$yb8O?ruTSEAX$w>$@1QaR8`o(lF`~rO)?UbacI|< zQDJcu2C(C7H;2efI$i=wR$vb|<9|#eemVQf{E$AYRwHg!JP8+}({g^)ghGtS{fJ_H6$wZ;q=*6kV}z>3 zVGl4?^z!j()vpFL9<{*Tyx3gj{Vz$~=WygF^<=%>;;HP=lqDU-?mnAd`MqbacYnOFr|H1>O&}dl=|l9CJ6``p-Gy{lP5uB--Q~(HeuiDH!zM@6d%6zb&a`U{XEfK3#-Qx8@ji_J8tq-+ zL;$XvQOy!VI}p#ZuFJsHK_z`cllxX!bL#NfSj3nV>9Uhn24L4iWH@_{!&%f>O<8}l& ze(@V*eIfA8VwSF%sF8@p7M(#)EpWs=#HdbMN=6LElrgv)M`>yrM)irS#}o`}D1hmg zw5z8pUuaRI1h6NtM8|8$K;!Cu0_qHSZ^X`)+<_7BzQNV%7+~`fg*P`g8m^&699s8# z?Q4~3hIbC&#+ApEe8T2@#E}u4jSSGSFN&u-<5#d7;#Hr$^PKn2sa{#LX^wQ}zpdl( zoCwVw#Z>CMpjV15h$d|gu5qBvdfeM|I5=!;$N5X)mj?5f$>(s`nI-s%MbN2U%Q8oF z#ifRe{GfKo=|^x`stBi9eJS>jii8-9$mS`DtTm{NA9_Q^$nHbKsR&UI~F8^oC7Ipft)c(T-v{Y%w&F zuVaQ*)i7hDqe>?4hE|tl0eacl1R&h{ylor}G>m`a{0|CteRsV_n}C%gS-V|^d7X%L zi=%BFuGq#+?;Z3-UIFvgtIpu$Sp^wCl6rSNS=)*_Snr7Woqh>ZPnWBsO?3+)&TeQ@ zs$5z2tgU$^ZCEF?AaT)6Hx^o8qN13Le@{#S0txL5Umd8_akn4KICpEgUS`V?_dDBY z1KQYd@hhVVeyb`%_%m}plVPv6R35xwB|K)#&gJ_*u5I?I8=H^JNS+sGk5+}_VyRtw zR3qJ<{&>vWwDP3n{o%@A^WXiX#QiV=NxUUy_blffueK~ry~S`L{;I*y5!8xqie*&# zwuo7OxFa}Ejz<$}i!HJk+Y@M+wo(N})&jLK4nmcZA_dQrxy zkE|#Zf*BBIKGem50?cNQ&h)W(?K^T(jXpDYpxTsDy*Vb;xzL`i!i*7M{|0bxUY=bH z1!!}pmRp#p`Nkt3C$!Q`_IJ|BbTEilZdFSbBP|sBnlheY$Y6 zYO^|f_ru1$XN&0KG&hlPndi!wc9nL&_YhtC{lMDn<(lGX*gM(!$a36P^SoP&as{QN zTw@bpwrm9?K^4}(6vqkZP;+k1*tk0m#{YE)u!I0+T=Tp=^#iqJ+G3jiSHzcOrC99c z^#Y8f#8`?NhBmc%I!7`~Kmk&1JrPxq8RX7+u&#{_gHGhW4TTC2#o*(`$K3B8DjxwD zKTJ*4qoXtB%SosHS3sqa5M4Ulz;R&Vd34Xinn%gT<9-KQt+70o#dYnIhF{bjVLiXZ zYUegoQbDGJCgpiedbJO`!rkIZ#GK95v09emW`llFf+PTw03mS-Cvw{`==+% z(i+^TKgd56!2Lwf!kCQH7l+E%r#@ei_Gt7%)ge0pMj*y;SRPic3}>dm8i@-E;*>Fr z91q{%tjY}im|C7;XO=m;zthqpyRJNu)-1?aQY<18WBL6%E}i>9ADb*&iEa79T{O2` zH-#>M1pxd1R2%1f@N~k=r;s@>c*C&xp(uI>3|dZJ{*$m)E{CW0pyc;) zb5+FI?UnfLSo>i~NrU+9m2=<8+1VK|&Gx?BX|OnyK{eZh0t$257_cmS&ZG5Kzpl6W z^*6j=%#m`|9Ro5#K?-#Ff2^0Y0PTv91$%brpKeJbn$=;Aix6s=UZR+QAaJ+tl>6TW z6ZPx;I0o{3?w{W0bu$J4NTR`eQ=P*`d%Ca&lHfSltwb;Ko^jt${Gh(m@4XCL5?4uV zjrM5Sa%QyNft|>>pxK{a2Mguk zZpvC9bEKk9l1mH}JyJ}PE0@p6Sm-VJ3X}J>arv)Ov8bXU74lkyoOY1^s}ZVM$q{@;J$|L- z4&JRTKrm@5$l$*}@UQs!GJpARCj_*6Eg9U4fW*esqYv6Jr(Yn8-o9sp8+>@b1x=5MiR@nE(!W(ewOZ zKWWzI9Jo2!j;D3xe=~P5t`1ylVaU(TMq#4jFu|9xjdV6PeggPv)HkmJj&$T_fmokr56#*jRt|? zNkpT5e)7a1lwQ^8Y06dNL?>I)0c>e+)^miUbzr5Ev{!@H(UO>Kcug3WRt7 zi5XsJd%ylIQ~j^67ymsA!20tq1g!*!3BMlQ+yMsvS4Sn{l{mnyb_9GW4d5@bo{xB^ zB_ml~vPDJ@*!ZRAom(Y-3;gUBVn}$p(LW{Z_e?q+y_ULdrbeiSMu+AiO7S6lO$W`p;2WRVAjSMTXLhE3sPbU1P_XI&5JM6@WT6U^H%SNNs1=Byl z{LD(K9g3`z$6-gzSb@O+|80{<`9zH@J(*60wj0p zp1vyKv4m*(Rr)A^~h>&G{@vM{KaK(G#_g#eU|;&&r3FF)}KA z%_RvCGu#ct&Z3L8r_pa37>O*%5Q(RSrWCmAU;74{+_>Xwsz5J{>B0i_44!PS|w2X}XG z&n$oncz+)eS*20#+9IDfuN1qfM;fevcTKjGGCeu3~!)9+YVX5g~qPp$7}ROeUdpkia|1Ozq*L(`AZuNl?I7v&$#dT3$Eezl4e4d|MQT zERrlNe|#Y@8+sku-EwxS0?oJjugiE-?m(*XgTqQ;Bou_)VlupcxF|o85X7*sOrQ;t z4c3t-W!70T3RXt1lQY2|XrLmc8W-w5`ti?|W(ZLDjHm^AgeI;JEt^uh1Bd}~fbCc~ zgJ0v^&X(EX`Fc*y`=2)sw>!6%8EXuF=cj<0nkcssPZDu+A;={wCf{7RZ+A;S9zwM(V z{%#>sBcS#&OQDl9i5^WG=9x%2O&RcOa_|dFOs@UZ%b}U9&n)}J)nI+`>hEKdKS|gJ zu6N;-OepxSFE)^iHpzli#@M`uea7iE3OD$9<#?aJp~CKxgO^R1H($L_n^vP7aJMQ9 z20T^6e=b8LDFI`oh>pZ5%qaPeOrePHa?ptAS7#ax;&E~%>(_f2EpqdOSl#129e8yCHUMV-@O6>gyP~>?4%rL-z z0*LW$4jgXQ9P|O*7BJETl4y#yi?pds-rS$Q8~utQ=()7>VT%w%z#p6*5155jfr^r#!P3zkuItbR94Rt0g6S) zAWep7G_r0QIv#o0^*z%5Q1FHs(cz#$^jeAVYrcX-tA57IAQM{^R6)M;@se+}NXXaI z1@eTnXh%8(Css4GA50+63rK?iOCM+=*@cS#u@)Dn87! zq*$;0eGg1pay=lD%(e7)NBKd-*s zi@Wu0F9j5?m*4tsZvdO_`l9!C_O8$R3uV5zYs1-!nYp9_br6XdA?%T5G$n=_+{pvI zm#ep@ADUICOlD=S06sJaz!@Q5Co!l}n!_bxIP3?(XAVb3{xG=*^lp1~ z92VPegDNwzR1x2x{1o3u0tLF+e`5o+BuPn7E{{kg`ycaw(6&RaeJgW#xKfjXWT(|2 z5D_ufQ#V?yt^yCKr>QQ&vjT^pNcZVxNI+nYNE-17zTyE@@_aEO41e}6LC-XQ zz^>)j$euDvX^q!0j06cNt&{{JVmJ@{K=vC%z|d^V9t_T>yBj*H2&KP@Uaap;Tb{DB zFlV%k-VA>VwMh#iExVcI(J4ej@X_ViphyySq|#^QCEJvZGz>8hbq-^0Tx5TeLMqfb znK^L12vIS!ri6V!K`V#AMbJz06A{UjLl54x}R{1P=|mK`J9oAAOXQz-RUQrs?N=t|M4-0bpHas)pN6^7lTCmcXZ%dWZ{%1=r$CA1fdJWet|Jd8<0yeiP|ufJU??Phv#HskA|M55Mop%H z-ot;_d(j&hom$k=)($j=62G+d%8=X$TD$qB`-?O*BJm&}m8{4+j-V@&CbZ1Zd;c!X zsqf{Ed>Plh*=W4HtA^xXq%fg%(>>5q*0MzOc;)IhcmKU>pcFU+WU#$c(?62;P->Gl zi+CqbgRVY#9_Q`V9I&0Y-Lrk#W@Nmgvw@z=it}kNO`hQrAiCN#nRI=tCkwbF=eBD~ zTz`$EYt`S+~bAcqD5jKJI5+?L*4$ z;-Q~&#Ai`k_rd;PW+(?uMYA5xL6*w@#?)R&XDg)FyU{t51CmFc?S-uF44&jPL*~8w^)VcTWxcTtA8&sLrOPo!dDb zw!1nGVMcX|F7=F;B{R1rLYp1L-x-WeCUJW&z3yp+Nc)_LFKxA6BYIa{m>K%qDJo!e zCb09r2$=-Oa3thJYX9|zXG*J<1s~c|l{vK{WxiiKx`PpYr}ScqQ? z0V@&ULmVQLY767Q(ft=0!=(L04lBWi{GC|t94zZM6+AyauFjd^FfrwX@H}7J#NSMC zntS6_p_I9B*^~b}&=(tTn)J~fNa%3dcAFS3Fs5R8v|h4EmAdNpPRSc7x@zD${$1NN z^Yz>$uW*V#Y6jH(hQH{r|dMW8>uu6nfs z+sCWGi*$bOIl<3)20$|bI2aI}FgkYohYe6AT1VblS20ZA8e5{aXwuHSy`k5kFk*!QdM8uuXDP^N%>~>$2l<;x!2~N%E6|WfFzpII*zeH;=QkY8>%7 zVcn&@-`O2;-9JtK_`Fl2mx3KE^ACJ1%uH0O-Q~IVb52v_^>mtZ_PR>gcdBV_z40HO zD`*UK^sq~<+{7bLMXNsy0gVht@4W`m$+C(=w9dnDobr5H<+v`M+|9eHcrLauWmYvz z0_kJJ&F17>=7~(&>`%r`KMH-`a2XQS)FnP1ZfZ;T^ZbqI?J#!=mM1&jcRo2kT)wQh zZzx7R9c9$))LZpy6CrDr5BgBzl`PdvSPG$-WuENeRFjV zXe0qYzLAHUo8RBI4q&~S2|TWGMQ$Kp58$#cMy5MG-0f9Yabv>ISJsVDe}Mi3utungQE$TG0<54!MbF?Il%dG5U)od( z@AEXOEIhAUgISK>?dm(3YGwHfe)XT_K{}nbbD_nKZuKCLRWfS4BDaPDMgI{WH;^MJT1aTa9oG zB99$CD#`4M zlj}7Fox9!r`)sY5uIXk^=bf0h*)libY-hX2=pX?JraJPj|3w^mcB{J*lK&Fw5E^a1 zp0&zsBvAv;7D_J(^ta;a?jWMCNo?I289e3SYcAX#nkwD^|<#5B2Py zE?)lnx6P&U-AHq7n45BaWb)^7L3sFQO7I+tBPl897l_@mcp>|`R4Vm))~Nrl^2tRS zZP%IC1kaY^Tw{RN3&UWS!kYCX4yr&&3(E%j3Pi7V!1QpV;|2q` z(}Cl3L=BCJ%8~z@8hZ4 zbI)l<&8gJGf6=!=e=fya&vSBfZvwSA#up1ug^uT>u2i&`ZsGr_H(@n`6J!`-gy~~< z-wCh0X{|6d0Lq<>s~m!{MY^R7#1$1l@AVFs|2FO&XtuN)oB#mub)@c&b|GJ!sWfqq z5KE%MS(BM#fQ^E%=>tSJS02eG29Pm{8VOI@Lpu7nHRAvxO^X4%f(@IN4*4K8zh<+9 zO#N&dtJRo!hBZ4se<$=!AnhNu8)f{YktdK+Gcl1#Q;Gp@F^hEzC_Z~qSEdMK#Ftp* zuZk0O%q;ieB#&3%GUGtV!sC>Is881`AGhn_8@OuYj91)8SpW)*g(0=4SLPsPi< zCb)@H-1X*3n@D;Ds33&8=PUu54nke(z0-^`i++7Yr6gA2ZcJ26g3U@Y(^<*c#lYn& z%De07IT0;A0(_UG#3Z$;&%E{=$-0y2J+3s;cL%^I=JP+|8USl9qflU2mScDi;^}x5 zIce2I{S8|w*S&=Ww{P|nfdT#j{7_OkJE`-eL7k%6m#WXWfhgD3^#Uh@1y71M{g$(V z?@RirX1Fb{Zi0ZEYB6q++8JIvdA7BdUI^ zc?#Onw0#kkwvH0UG+KhRF(bEc)DV;Z1aW+pe64*$j5bSvTsBr@o!H?s6^USrR1QTz zY2h|<=qfG>Fk5Y;NC8Dh;Z9F=D%uc-~GhvMo7l%Y&5J;ym?g*n?tLSR2 z!8(YxbnZL=_7gdyMnebGlw6&kb9)cJay#=nA_W+Udd}=Ds?|j_{XdXRoo_{bZ z_52wspNLCnKdkd<=c69(w$EOdQFgG!*ySZ5vv=AgCxSQBDk|HgqDT$ZIvp;Fj50(J z@`<;|=>jdnx^C*f(FK<~NOJzx!`6#<6Y7^Vc^jFvhTTRw+F)F2866Vdk0-Kk&XiAz ziK8;+FWzJzSGsgv6H`jB45=uWu1L|+u>c`TI@+?LCG$m(D*yI!;h@yjTCbQ{3Xrw} zF%;RJL~{izsS6$aW7Jl48YBH_%fG9_`m@RS`J(fC-3&zR)#D4Ik@JQ~neJ5_z zNO39&p7K=~X3s*n_r#0S~n?x4R{}-QA58AToq5wBY8cpFx=1!zEJz zQU~?-MPsVE1jO13vzWZmL#VPDcV{w7@E?w8VyZ88!=il_5*?Ggb9z!t#B?%=1vT$< zRlb-0#a0Y4-{b{*LLIx}`VXW*w1wW9rI2^+S)#}uwDj+KaUZ^ix~<>JU)LxQNLjK~ z7Xh3j^}i15H_gI7pU$?Q&w!%QAGg0ZodGwn!7sVQV2xBZT6IYSlgSF|`^Ado`GIl& zwA#!|V^K}T_I;Ueu4;=MU-Y^5Dw~B43EeMTx%W@};n7m-C`et5|j`N2&rz3on5mzKE?E(mZ#FHT?a1@t7|w8u@?lqU3KMH4=idIjmrB}w2tF}g>K^-qug5dPs;fe3dkT@N96nB zr+f+Z9omNzM0)hJ!T)uhUQX)UDM%i>n}-vUZc0od<$Me#SOuAn2^dkn z7s{h~nWJy8d8h56fNE1ED?=Qkij1wpSeiUcPihf0|B~4}8L$vtF8s<~VkrN1(RnOb znixKSC&lTKhoU=-Wp47KW}Y?ET?YGD>x?JZDF)@s;&<6^qT;;62$h6Ldf}*CR%f)N zkJa!dLe!l$@A(@#FhBQaLjb8cdV!3Lgrk$-%9QS?o8ZvkKTuZKm=kdMQjMA)?c&W{ zV^iq31#LfZ`h;OLwcgQa2Lg zyGz0`(i?!8iEgrkx0?+xq5y%>zgFYZO=m6MK)m8%S;eiCwIvFACmEu5Y0-4V6KYd} z)tW_Qq^5I`VW}9cEJ-XzCyH+pw_|Jn4T5U{7mRcgn}>W68-y6ZG3o8|d8;siw$SFJ zeZK=9OCc@YqIb73SEWhgB`qfQpagkb8G*h+;(UP~nr@uaMsmsV-R|CB;54lFOkZnI z0x3*%(yPR$mBVYLJ6Isw6I|{uXT6iQWDBCU8JrE2`JMYq?7eud5P2nRFA|3W3<;5L zxW(Q#t7Ib*Jp__Y6zXI)3`KiH5cr%gD>>63q6msOI-=<1BrSWF5sF7e#dguLKYkz! z&gh%Fqf`H)P+~B4YFTyo>Wa>?@iAk&bzuVZ^+DJ8;?vioOxb!HSaN~TxLKO{Q+1vi zxMp1X?O+eF?UrRJ&K&oI3(zjh`qyFEJn#7pJo}b8W!BZN1{4xGtn%QvBo`C#@8;C) zsR%=hm}}x3>t5u)Ta7N>wY7&(IEv`Bz=nqm#m$ASiQD@ z!jv2)-)bQu9chP93>Y7TC@$9Tr-Q-d$p(ciC1vWbxJ9e{x1-H$YqPN3q!p;Fr4{me z{>U%Xxx>pm8T&5YGJRk#?E zg&gCBH8H$WgVf!y?6;=Le3N+ukOa_Mw(>B;Ia3FgH6)Y&lyn2;3jLe}+X#~!%3-~Z za*5Z36bw~Utwu`Yc1U;DAEl9s`l?MUvswB<`lA;$UR}5z0-s{i|F$Sf>VCOcx9war znc|q&k{o$S-P!v`xVNjTD_gU4tCeNZrQx+B1&VJIsS9MOa;O5E1SFv0d-hKHozQUT z*DZg+GN*2-M%Lt2HA9Z7-nsbAVw$Gg;PM)P-FxIHWt z*M1hnVY2OABF@X>Pu(WglSyvWzLjYFzLWj5XOpGNc=#pPFgX`EZL-g169_SqSzUWT zP2jTR!*`ui$Nt%}yEDk0O59EUbzAeG5VSB-IhUnPrA~WL@%qI+NI?g82DkuKH=`v8 zZt%Uukuot69gGtXvue&*-0!NBArRYcZaVHy0=?i@@elb*7o}`1l#mMwUicQx@0*xa zqcZ{`PZwnwyZ3a37G_OhF%DmR|8?`~3HOdMqM0$<=(2)|oA`w@$QY_efSDjZjTGER z3$Dt%b~z75GRhHNLIY2C-Hu6$cWzl3{7j<@-S@NrtPs`Fg|)@=sBb=TLfilIY07gtvc z{+7#vA05;Yh^7yY?n#&=Dl@c%Z`i(T+tBs*%ajGC#5UTQIO%*)tfMq!T>_X|K$)+AwMLx1^*;Otl@?U0VCzl7rS-NciYb zdC{A#Cuf!m4w*w8{I2I9D00J)sX?pZ*gBB>4P`K`j!r7@lvFHiePE}dIcDx*$0^0@ zqMTStXsL0!Q*h~QU=Znbz^A;XI5B9nS}rFEsqinA`9L~mC3oN&rhY-u&NF{&cQBTs zPJQ6I?A5{2=(*n(FDIv{oM7`^QQN}H(G`=Mt8baVVH#hoeHO{f8zf9|< za9s`4<*{=BXHz}*vJ5LzA6^buhG2D2W7g;uKKW3;>n1v1B=p=Wv*~Y(D*$izdhanT zC33iSt!hF4z7iz3RzOQhtj2@+!zq-Qz#bh5icOVat($84pCKkMqf3Q#jCQeyOheu8 zIGKYgRXIW1Ei};gLTxp|a9YzzJNGiSJN_kTf~_g}YeTBhDnUoH3k%gZ=ct)}`-;hr zAwDZsU}C{zEU4I_943PeqDru%hDDKpfOzBOtcVx_0pS@2fxv^{$-@T>3=AhWJ?mI> zX$kzCNFIJ<{=NHfuA z2gXKy{Wgkx@We}cz=ofhM4>Y8ANw7!j!j0B@q)MX8RSXWPEMPYWQakskS5zYL>sGE zhSN_H1D|XGQX-Y{Dh{DqL{U!z z_PWN`S|0!g-+#bW<8#drq?ug|TC;qbL9f=`1#mU4CO!v=R3iz*M!$ks=}z1GKiHWt z4WNM>bfXaVS#)6XmA%`8lPd?&V@m#v5p6V+5cw20=Syu3LAAjx2+`bLIQw$>#pLX| zxd9^~UVF?;%N_l3h=Gm})OZ|+iXPL%8BUDeGB8qoTObU3vos>_=4DU7mv6W71v4LE zZ1k$rN28O2QP`j#Nl=>rJZm_VGUJ1V;TfPeZ~vjR#-sby?oEmp5&R;%4yiK2A$xG~ z0fi*aWcJ-?o0$F91!hw2Lt0eYMiYV!-nh&Lg@6I^YG%(PYy!^1b>)z>2>&qywp zCmS6}(uu)a6hy1F%cnrHKk#ByX*y4`)8pwI5W3#ELIp7;sTdfH@6SQF%nQ&wPAeD? zm@c?Z@7kM5`6&%z5@N76kVxjc@3yvbz3s?Q`)ILwidRq)UoEJQ?m4z}&Zlx9$+UFt zWfc6#y=k5?V4jwMGCA)!+%hz7SlQ!pH8>@v5@DAd0wk&gnz%p)rGnl^U5}qWxcQgGTWv7_Nx7!Mb z*|)rtG5qopw#hMw9U)caVe6Wr=CQ}?Qs6wjGjs_+eV_No)6Rh&ga7#x1S#=R95tJnZk@n4NUwCTFyk|meEks=VclHqoSwK%|Wha z3@@yRbK794rboikQAe$Qr3_X;Ekx8|>Pjb3vxjq}ftQNw`oXgkPnRB;V#9sNc{ALr zQZocF0&6;Ax!Fw+D)qz=@F^B(nvW+JTRwc?&dAN1X~;7$Y=$a5fM2TQ=@lIwVP|0I z->CU6L1kzado4yXTIsID7MFL7L;Z8zm6sINJF95q*v9jvIqb?j;Srha8q zg-k1hWJO|UVpAc;^J!|YcM=~G{#E__$+P=;XMEbS>)roG+%!+KG#27we}3Vg2iKu*adX0SU_$jIkvK{oLmi9`+jYrnJ^E5_r+IEQl0bFcHc>#Q z<2(9Jzo8+9yShL|c{l-;RA4y8&QSwwy-FWR7x^9T(h(O3g2r=X%fSc&0T`MhVf8Of zh2y^jV`=(i9@s4_u^GURS`bzXVWBfC91BE!2M&dDQs~!2Z z5)?n-ZbIN{pw7XCfa!HCul4AXsZpwxGA214CdLmgREoJ2=Y~kOk?-n;-3bU-42XSH z29|0y&i8uAQEzVb(4QfsrSVlQ@P>Z)N4;$~H}C!M)P4IwAV+h0diq~e8EBB)Z?>L0 zTYd!SFPE=lZ_s86s+zqmre70n#SJdZ^Ev6rz+H9P#zfdE`DgTT7b@7Nko5#Mb*JGrSB^Ix!pvq}~4y4K}!E{rWfERg~ z1<4o!RmRi_mDewv01Sm1&JXWUL3AKwT2v&4gcV9+Fb+AGcgk>H1v_vt> zU9nT1CX52txA}eoK8N@wU0zagP8WA~Cn+PRGSNxb9xO3r4{!DT>)iYaY-eMQtz)^|4*if9x){H_PlAwS-j;e;Ek(9`yHx#z&QbHIev|m{<=3l!R zM7#k6$Zr5Dj#lOM&UdT70PL#W|Mut3&VSNC;YT0xK=V8h^atpR06dkTo5sArKR+UB zsV3(d#u_9W8Gv`#J_AFH+t3Vcj!GiVdNTgqm{Xc$|1aA!qhHAxPXgOz1u41>eM zgjenF6817la3+wzv8E~s==gY;^0utee%-I=5rTMqO|h>J`VAU#KJ`zOqrJw{J&qBK zrZPLTO(XQX_U)5S1ZUFHT{>e!7_vZLgf}*U*lP$pxA_K9hvtayuw>4FAq$l zpUx1{VTx0(Wn97n-l>&=UFU>$_ZsJQ)kd4n8m@?vOC2@YC;}8HK3|1tfvuVQ(8~|$ z&wydBydh_4RG+%|hb}8L7UOhw-zH(Dt)3KZ!OCQrF$D8deG7!GvDxDAX~iigUgF35 z%xVP0nQ=aJUaO+Dgf@%;R;2s7J<$5C*pli2?_$b<7iUDr2jushu!O=R!OS9WqJ&8- zW*Pj>l3D@=pb`6iQ4j(jL9C5hb%spO&g$q8!w3pL&$4rW{?DkosFlf8;~Of7q=iUB z*Yp|a?F2pF9Rhe!U(4RhB{MgmxVj9v1OVdXaNtaR2jJ}-K0Obu-aS7{$+#)Vz>2y3 z9#8P~^I?PfzDy06nGLzg-e?YOzK#z0+u^@XG%281W?o{hZ`M}LT@?p0%|4BWf512w z#YTo8e>uTo;Y~Xmwi10OYp5VmABM^(3jg7nCwHG#FTC@jVChJ1!4H250-q_1HN~7I zpa|JeQbKZ(aKZt}tdT%s(fC?8D*Q~Bh}L{4=x2h9&3dxEiu1pR^4z)1H_uXv>Qu(E zvb(DFz@L)VyAmQiZLn-SwDmCLx(vE`z7+}_9I?C&&_pRAac!|_^X$lGDydK5V_9@e zY=hhE_2uEO8%h89S}mAvTxv7x;D*Ky{q&T-BuZ)>`SFXc2)Z?aoI|GK(;_ro+0>qg z>IK7^AD3gh$ghq)Bjr3nW}rP%3V`~g#Kk)SAk@E@Jf&OL2X}XO&kGx=yuZNG`{7Lb z=^j9PDq%b<0o(BLkPwttdGbtY;RJ@2EL7-{h631O5vF(+1fD})J<=JIeGo-CKDiN& zlGYNQPmC}=IS}uBGi_3>?it?L%=6;o9%>qvc2kVva}?@QHd^SLe3*4$CkXOJ zt?15>e&>Koo)&~2&p{4%`ACpo`*!Vh!OHh{@%u1>dnSXdUr1;)M*5de-F{-l3D$1$ z3Gl8@9epLFXQ{4Ot-CBh?CrWSu}js&A_`1BRyS}@tQHhNK~v+;k>SA}bo}!$K>A@H zM@J@7eUF03g5-EqenRfE`O*#nUp#VN>?-Pa9M=mK+0@I^iaMsvH@h)?gH4D3=B-vPHGMR#89d0>{F28BRQ!QF+;1(iVD61 zBb2irKYo}2c;RvB=Uubf4dtmsfH$5Nf#J_QZ5eR+9oRVmdSG;E<1ewo8ZjQtrUj_Z z1PoL5dJtV%5bZ}pV)wn98+_n962U)B7hTlGuqgx$-6LP)Qcj+$BV4lkR{p(k`-)E{ zMmE-gR~1wBI1=-Vn`gQ*D0%+fXya=SDnDNL$f9r9H+xw!^9ls0`i+W`&A55XWr07W z?uJMd0f@v$r4dIWorNxvm5cy`|RH zzklLBrShA~uY(C&mW-Cq;)^fm7=&L;=ltLy5F+0|QPqqp{4Qi;X~hS%0O>EZE+PNpb_ZP~!o0?B_tT7_SNppEyT;^!n0Rn1)Nx1ygCEfPOx zc%$+ueiL7ezC5nSxdx}?%hSwMs$6hT!BIbg7*_oMz+)(kTTEUeC&XGP4J*QEAmHc< zifRkDZ)Z_*U(n(f7DS5JOHC~!{cd(`&63r=-(6h^W>kFW$_`0F zq%99F2W>g9mPgIesw4V%j-`d5S^?2>bU3X}DzC$1+8etR@Keuc)_qU2ucJP@Dq2p)SO?Q}0LWxZ8 z2htVPi2WhS?BOutel$~-;ToM;WWq{Z+N)@mK)=VeRuuSC1%Na%q&7!e)XCd~CxOAo z7O5uhzg%@}UaHc^IIx4-WP`Q>-Jaix&A=M1I)!eI7M)z?{%DdTkJ#2#?(r#UVZo-K(TW=b8d4MNJjSWp|Rm!Zfwwk>1fOa%Ht z7s}+M;1$vp!Y;Xzyh9igU$Q2SZIRWXyL7`a^RGEK#oin4sIL>M%n($Np1hW#6@e`> zG8~7Z$SP9K22aA)OUZh=9tw5t^LNf+9|t4N-QBpQHHFetRixPt%yR8NWSB)>c%SKh zw0`8g9gKGp6k+54_&&eROeatja!Bu1B1ttiGL<;Z=}+gZobqx750A6~_C7K*P)D@VZuU{+g{FMg~%mP;i_iDG%D!ilSg#gw^KRByOuO@?lO3A2U$GdBpCcR zmU3V=>z!5h`EhFFl|o1se<*E3xoL)*CWUqAUeC1e3icxPr06yg?(ebBA9gk8FgNJF zcN_wIxHuHPs62Cp&OPpKF3t@x&*2QUo~tdhhJM0zhe6{yMVEb5B_W^sIy)WG|5iiz zi^0Khpmcd7+gA2XXYeUsAg#+0B-(U z3or$NdBPJ0v4_G5H*n)-F= z>wG--^c0wq%G&rJKuGGnyPDSmzW()qfcn+hx?6AHb1b}%<)Le6`8z|goCY663)Pjw5IiKT~)$n4y~U~Rws4?Ukp zyj4P8_Nm(7#*7kl}1Oiv{=B`^f|(6LbdL z-FF}Sh@m}*_|6d6K4~Fb$J}rnw%`w`^%FWQ`y%~N-PvD)R5!UeUxN&?TjGGg?gMh` zT-8oEB_k$ACnjvY_^rQu zh|G7wmRtoh$R54$cNFa34AkWBb2Z3oIdi&ejBGok-&>`8@JeI0eRQ-d7{d%`RyCN3;~VHz0suJ=hI$*dT#s>g9c@kaQ(^T>fxO@Y#9dBF?;M~yV({`TioXY;X>d>L?PzNiasu{)t9pC3e zo52%rLwi5fK9(XoW6F*h=0OTAAOI)EmgrWY6A_xK4(lj=OImR;(VX|#`O5Gf)9`q8 zG>#(Dq`eE3{XX_$5p~9P>~X=)Oo-9~RQIu;fKf3O65r`FIZ?#dc_z78br3n>Lm)R} zYfd4ScSKozR_{kvQrp#Ho}XRRh)r-M~OLeV@#FQw+LH-n+6} z_NOVMTztD2Q4x`unD$XyzPxF9OyJ*eP4oOKVj2jO0gt6YzsZ6;4ySTDYMNVAFn{ci z&yNq$0B2Ob9L;UNhfUz{aCx-We0lj+ZkF1#ay&X|KFziB6(<+y>&ySk8dUcv);^Bf9 zX=R>_@#j4C=&7#7y9nhUeqxhgWqQF;a!;wst|a*YxpvgJwwH5N9w?zKyg;p~#w4PK zu^&(?9*gi$_7Dbq`NT|nO z{SgWFJHsQAT}5BM;UXd0iSBPu`;U`QLU|?eHxth!F;GI|oKeNbqr|Vu>1cU2f0=Q( z*j+_%Kx*4gU{nS=8Q6`6uZIs;0@7CkW4;k;ne+rvCv*j|F~}DXool@8c*U}J4qH@g zelFHIa9ki5~XBE&}XMac~na_hGY?o+%J_HjLLr%Vt_ zS4eg;sk6cPx2|}&j8`9pzWmXXKmJV><1YTe*zL{w z(n{=A6K;-~!5^XRwU6{zUoaJ4)snvE)>4*WfA0UJg8BNR zc`YgVWhxAbZdXV;SdLQiE~4IV1MK4)Yi4|lV*WS6{!>1(v>UpNen|gU9P7IOUVJso z@58*CA6bn8@y#Vwy1@i8aqzv7i1$-r`CIq`X7mX0PUYe8-+$gBld>rSAi%kzngJa@QqrO$AI>y1<9tJ-A? zG60*`=r+^zfaCYG{QE@l`424pRO=dG_5 zcp)6%l3AOz@6_GV_8!jdvt>LEt+elcV+1o!7RbVfm;fhP~?X@f~{wfobqVEfl zSWGbvGHf!OBxwuVAr+GpZ_aSP-OAbxC|nbONgn4hvA<)IH}M`B$wnAY)S!B&8=y^W zhlZSS{U*Im1KQ3--YR1sa^4ENZNUcr-MfJ6uLW1yzrN@6hnq&A4s(8X_L7Q

7< z&d@*vK7iUdG3{iZisf7olFy9niL+RC+zdE3+x9^QBgHG^*Y??DjLnfK=oq9n8RQg) zaxTf4r1DmbY_>iqm!B*c5j#b83xyEVA~DPF{W?)ZMRKuQv72lx zS>xw!BB9+^c}}b{^wGkqfT3B`Z%7rur)z->EyN(YDlzpUU7=lza=+FQF&Tx$}t?Yorwx> z_C9<5i~t2LxMDw#YvF=J@JCaVPw;+Ov282N!U8Y2iKx$h8k`6$(A>rir>%I~9MM}- zgR7^6R^1XYtHa*SSGOCd54=fSN*xEl{>xM<3f^|#|G&xq0L-d=Ep7qb)nA}k-tDoA z&foj)w~bg%ZcYx+3x5EP3xeRYv$G+{Ek0gFg&f0h?vx-V6%+)~kL+l+EpjL9-Y-bX zJ!VbdEn=cEa@oH(c8-;W$B*>DHB0twf$ZPS%!*i`+{fd za(fzyUB8Pctx;9Zry>WBogs>5((k&3MzRM`m6ZjgKJMn5xHyRxRc5Ob6j!j(12zBm z&Bpg7-owFlAEh`lFvAvPs54B~`C0x{NV%Pgx!Pmn$$#zYu)@n3GgBN!CTH0Ch{PmK z#L7yJz8dU^Y4+4~K^FU~KpJmL*J1u;SD-aET3Ypi!Et25wn!k0MuKhY)yS}FB6;@* zhkjARL$`_LM37=jtW%_|i}FfKTz*phu^yLhzxF1VeTw-9<)l8@L={NQWFL)DA7h{$$D47XyTpG?_ThVi!JKfWTQxrhn}0O*#FhkFqQad6W};df@+czPWr9^m5Y4_On6)i zM7UX9ZHZkarsGE{&+C+ptgz;1AGwf9&c(XoD7XEn{fjt6QGfwezuNq6od#Z${S=}v zZfiW_N5zV`dRVG1W-5-iJyFL&dY%x&w=*hb&+LhZj#16-(17B|{9PX(v62g}0eBW; zM%6hOFOn&wX2nu0JJmRS`fy70EVxwa1eoXpgR1ABzP<1RKsTW9TWR)HxB0()`FYS^ zU{~IG-t&(Ndb(VT5^QMs9kcH(S-=$%V9ZjqdI5$$lrTcg)*$NE4+Nq)uFKvDL+w4g z>T637l0mvqE0>SrDaBwNy>gR`6sv7XM`m+6i}rd>S9Xxq!Q=Fhn_&1U(H&MXqW^J@ z@pI2Iy#N7PVX6qVL%qr)#yv4R?JR-t45bmr*5Q2}MbVsq zC&3!7*j#b(*$#)R)Mk>K=RKCMI>fN?^E~q009NH&xfC9~-s#HeBGa9yJ>XXKPh$np znoyk2fH{huLCS@#r1aqi!2jv&z02xtZ)^K=`}cYKAK;7^)D>`dadcG1;Uq`-Rms{m zroW6KJS={#rrkXIZHzud%@(I&TR#*|8V&e?fe;w=e!3yvM9hT!5rJF?zBVkgq?y3O zfqs$fyl}GkK~Ro=g{~E9l-xM-VIM)LAhY z)MZ_$Y15W3a+9J-^9v>{E`BZMO#_T`voKHil~upR|Rqsz@xVOWcDX}{~ljl@gYlSp2%nZYK-L(SPe#5egO)h%(x zZei)<0I8^MdcWlPDi^K$Qr!mr*F7NU4$i>*?e2l%FC@9w zMFkfZ*FW?jAOLfyH-0=)k_O0&AFctro0imFC}ON3;O4&!itgybKmUIG?+L*d5Q>RP z0#ns3{|3WqalT*m-_6b?3ou84O)3`73SEp(0F5pzJhFmkqlQFZ?HlK$rxsy0IxGlP2|LKuaG};leIF}jUMF+0yu-0m zsx=(7DUutoA`6Xah#pQo>!JU4lATNe*U#LilfjO#Jq8A*V8B08xp?x=xdxzS{DJ5k zP~QbSsQ>B?z=;ds)ps2H%N_xXZSGfHe|CZ0Ro>Houb~%5SxTA9XdsU6Jkp3~%Dz^h zCt*M*|CbHsHRZwMZ33=T>%vDCM!%;1+Ed(OBM6PMib=qs6t-2h?OLPQjo~$QCca;j zrd3`lzCH|fBQ8rC2Ctyv?;BT3p}{A#B|4=aygNt3ms`^dmQIek{8^^ikiW{TDJkb_ zRk2{B>|_rR*kD%fd)6BpN3sPVFY-_C8a$IQh$S|5!hesh$7!^Qer+nWAuB@B(U}g#0u&@)(--6qF0cj5FJ?U?F zOp4!-#~|}660?mrR8+LMHe2)CLZ5hoD7wX@iMQ&LO4+}iWfMm!Z~6GVVAFLBrV;`3 z;%&-=3zcMT^~aM>$o=9S?#TT$!>f(EXKj4U}k_$33JluDG<>Z zrgQ_gR=0PT%xP(_fs5?6>7`N7wWTz09kdc#UQ62T#w7eW2hAd|ou> zH~!}@1nLJ55C3RR+)+D#*k?MoyY21a)c*dz>G}hZLVHPV&OXi;8mvrFNi6rPG6`aW z3ezw^2gN21)HrLs>WbMCKje@T%t7DoC+L-Bp~abb8=Ootw$(^W@9_I_$=^&wur!Qd zNva_~pPfxD!!<@(?KON4(`a@<0^r5vOucD1A#saTMm^JBFo=?_UF*~ zS$yLP2E>|0~R491RL%zi%*GYr1Jvjp;uj3}b zD{B=d+jjM$9=1pg6vi-a_L7Ih<}2Be|#GtK{h)(`>#9=oT#Hm{JH^t=*1S9 z!f$}j2kiiQ)x8hCe}4k_h9zRQe74WJE=6B;L{fTBW?zitcZo;>tuAiwhfuh|;_FjJ z7wB|CJkB-iO!)?oo0DxCCNOc%LiqdpM?^%}>7Y3Fxs8x)zG?)~o=D=RM=CmgxnTJe z5Dni*#KCYj)VlCsL1BW5Rx5vIHmxCh&De)AM(J)A9y(#u2SJE4;wZi?E`Ua|q|tW#E+Xdl;>c zNkYH8TtPNLC|#7|x%!}hNw2Lqe450}?r`pAxWL`l2`&>T7?f7+D<2P!34!Cl_xeV} zxVX{8L8*qq{{g?~YOVYc!RsXHo@1^x!-~n-2q*__*WeEzqt60K;;aKr&$o zkR{yzYk90zIi?D~+4yr^8x-j7{@vWfU@_4APw>V)U1LfFfq zbTRoY#d)@TOr+ypuwX@95*IwSIW^d>Yylp_da?%54hiyO@U-2>ilO@El7whF)|COu z5o_X+F(^m2?D8mtC9&TL8}2Ng^b~Ume;UXK?ob&iO&=&F?bKY`^9v+9XEGJ!L32OM zd}@z;V#*4*(BD_$fNoUfD_w6zmkaAJG8pXcRw?Ercx<#iQ#L9FC1yxRob+a0ABeYh zM|bmjdoe>zV{;db*@v7!1d&*kN^snS$ShA9*LAM6@$JV0;nVATKxg_ddfM!~+{9C7 zGxU%2mbLv@a@qmh9d7^aJ^{YEWsk({GJb+eQPGs>>lNN$S$1WdkqJ}8CdS{%FBsgf zq%OC_F`$L4t3$NwB8S+obHE`W6AY&4GQu`pbNJ??Rd24Q4KV}nFj9zzA@YDWC<+0C zVneMo8!vM-BqVZFwZ{2Hloi$N2ziV-Fih4!Aab&Z9VHP+S+cf}MpB&^oIcRgp#*~v zM8{N&6>c<2Yk3>Sp|{xCPw_*ujk~MeMDFh+x?%FYgJTn2z~sc?<*Lg;c4N&(b)s
)hL zV+`1&ZbJlkMD{akC*8%T8W%=-SS<=L7qO!E4xorSIQocLxR>kp0Ar>YUdD8@IzeaX zT1+!Q%c@+=GK|hcCNuXlDV=dJf9mOwoRz`+Rr9UPhAZ$%}`45kcg3^WbWRj#qupW zG0o=}MVZ!riGr7tO~?plxh29{t=4L$9*oG~K(Z(wp&WAzY^||Xx{26nTKn;F{A zZQvxRJn%P!-EtFB+?Zwnt&E@K}CS%hzQ1z+tq2PrtY#3nd$Yi z6-p!s7nnw7#xNMxw|%YzfRR-#naO7Ue1HDO!Tsx!TI=caA2SgV+g3f?&w=XoE&M!A zzmx*CEL53Ur&aZFbfEyfskmk4ah|<*o9%%Cph3#OFIX9#C3s}&$c-YR)2#Ta5CsAS z1S%#?L{bz%EWMNrHd9HFBxedzp0486{R%M?69<%m(ML*FyC7Vr7IJH59Z__d$%zz! zW(FjJ0MEi|j6oIC2Jr0BCw)#wAkOn~9{rp%x)UeYKy^J$VEK1{dDc~d(pL`wm>S$` zEdd}=1g`4%mHho|rq7IQEg=zI37lqT&e9cUe^!&n+}zEq#x1F!$W;A6X}#9{OhGco zn9bLz5onoYL36Btp;Bfb&-3_ro<|Q=HNBazscoC@x3+n6R~5-AV-h9;vHfcQ^WL_- zVi1=>orr+kHV1H?XU!R!iP{FJ!8%e!U7MJ1mNFiBbhKbqgmZ3FdpWeV*|`!0U=)Z1 zczM|Iv*R(-60BrS)hN5BPlS|QmiYM2{#JJEc8+&pprzfZ>ynNh2j<7 zbJPN>J54fjWS3kfApkB$4}juTd|A4=I`y;9Tvw2pB|xN2wIg#P{j1etX2t7|=%2-< zq$0BqGm3hRp#?CZsTk?`;P{y{Hi8FSuE|DAk>psi8&{mx$YQe@VcfxU+W-UWKsF_NJ@dAh|)Gw z)t>R}=XsuY$0eB?s=_Hqkr1!T&M}e+LD5b)D#Mtkh(K#-17&iIlL>s9ETWn@)vzN= zxQZgO?|ZqquX7l^_kKpZ7rii`fJ|sD)Q;oGsY&T-Y)u>!7&FG0YbYiy4YhO%fv}(= zy0GonBwH$;rE3}%GDP}B$eys>iC5jCz#^ghf)R$Wz`i3ko_FMSWuYylu5`$$n2 zv5|Zp{o~{8F{&(mKA(Sx;9p+&@|ycwRov8l(rt=rD55Iml?rOXXjIi{3jO5I>8Bzf z3!UW#(*y=H%9k7{h_r~fSzD^>CF{QC^9y@B?<&d2jvk};J~C15ZxZTit~I($Fp zeHN`Fym~>;_D*$vItWaQf{r25(NCh zLzV=fIrrK>1b4H^JTUv5@s2UH?I}n>N>*Y=a>Af1VgTi3Ck6TtDY^j!6fuaIHPhRz zyoiHLWL*25c}ht}qYQHt(;E&VcI7*y)U1n!JUJpD(xiOLCYR_u&vBe2U9Ch!;v7f2 zp1`u%t6*A=MkF(`9ClRw%dIK;bq!!=B%L=*3fyBdMHZ)h6v^{EuOWJkVMIWYF~+yo z*S1;o6RPLv&+{-dN>;ZqqDK_7!GfnXACVAlw|l$alk#|;V~ln)NV&{(HDcdZ=@vXTf>b2F96OsyYVU8YznX0j+qFE1}K zw|oR;av6Xl`b@V}GLT?WRQ>lKKS++H0}aaPagK=m#Is*+80;C=uDm zu-)Bsj1l9c*=^tVeg8lH@Bfdw0}&CT@6V$XXL3N2j7TI4O0>0RX6^Z|WzpD_Wbger z`gxvl_U3zz;lRG%B7#SMxxZd1aHTk09NG#-B9bc|uKRf+h)@EeZr0Q^k&L!)jyT5X zg92}6WeMc&V~jD*Dx}@F))eZ~b2~@TTOHlQr1yeR6ilEL;M!7To9uu2HSX0u9eqYSz}mBo~_Q^UXlv zd`R|w*5%YU){q9kmrSTt-VF7v837$5pT|Q)#EoPhLjXc9>%@KEm(9Rvt+m_j z!B2AHr$NlsnPI^)zP!Hf_BE0C6bJ(A$ToH2Sv z3Z!Jd7tX0ZPf&V4IZHP}G`327GM=$gHlb*-CkLZqY)o4=2uyvuZMXCJCQv3~#exEN zUPAEXY(5qzs{XmAl)+H|%0NuFu{kW)nv>KeuSGlwidB_ngkBOBocnx?anmLcOg+>T zeu8u6bs^VXHB>1-4S?+G7g7++kr*S#C<^^jI=tPcls~1Wr02+T8eu{hDF~tjD@cx< zs)1J=uT_4#T{!n3XM<8z-J3PHRy^|(kE#b9l#;*_%(2XsJ+YRfh^V=*AEisQulIiR zv&V>xy)|3$u5`Uc#7j2*(($ipJW2964ua*sHRe+TDcipH>e3A447;!2j5rKsm~woN+RbV9#AQA$qUyJ*>v0BMUSFK-8cq=F{~TQ8%a* zi`K9OAxu@6LDL~R<>b)Rv}sa%lngj15Y><8gVi1}b>Aw>f$A8Ch_qtZ>dfjygu_Ht zF0QX?O)<88zhXjlUg~2TN&*Kg1|_OZX9CVzDVHxJlm#gKoWM?Lq+D3Z3dRC4)7lXu zI%GA3KYMzzWJ*`Q$s}_@`##Iir2@cd&G{v=)nCMCrf=@%YGzXX*g|GspORdBy%L3#eWKBnJ5WV4QfV?{DbCIUF8)I!3n^X#S$o= zlQBm3WtDNYWv)oAQv~C)Dg5R7mt4c=KhFk31$SVgWoGZUM$9XGlunM>~ zptQv)%cV$F>yC-Y$T-I_Mn{?XwY$1QREY8+R;XDsYg);fs?-csn9PnONX0yW zO<|PrJRfiG|NPJY^1uH1Uw`}i@1=RrG_BhAGE=4Z5Hs7Gx~qt}33h9q=XoYaDFTMi zK54`_&vS;k)p)5dRk>4(}zo6YTx@8oAR&my=z+^hyO zi%wL*T>t@T)YQ0$hueHiBH~wq{Pj-Zllu7iFLi5dYMY42k{{D!_FI z*U!Nm*cMbXSS?V3*^eo&TH!A%EU+NU4}0GqU_PSh~2t!Xvt1xWOP^ z1O>RL0z$Z(!jvTowo)>Rn4(OF7d|!S3K&JhoAfJ{=B+ii5<_GyWW-4I(t=$=p^zYR zWU2gh2|{M_t|-+ADxQ5rLfurY`QChQx82NQf#*>AoxQ69D*ybqzy0#dFaPy_{9k|h zVi9 zx(TenE9zd<0(OMxz!;@+4AdEY6*00pq9USp)xNU2Z$Z)#X)17yQURSjdzmA6=g2eS z%f2tM@v1LL0%5jHFvgH2V=E%LV7s52xdsYLwgj`t)5-Sei7GELGD8X!h=Q~Q z*+1bAU*gQjFs77H%@xD%BO3Wr~cl=9oq9XAN4zwizQc#yF4TF~+!UC9TXXLtLwhN-7Ry zW=fC}rQNNU3)RfoZC!6>ehJnB%4`zj6CG4)e}4$y5s~qE`pKsjnBa?WRaHI!G9zU% zQ9o(EOEy~c>$6-xR*v)BN~1@W#qwA^jjN2P)uwYsR^|wjlo5o`iw;o=w7ruIYg1g* zTqLB8D;T4RGP>*&+2g$$q%a;DXle90>rZMBSJyR zDc{;!bMw)Afs`{- z(@%dMvPo5FeYu1 zb05*QWTl2gCkGgc^N6*%K&q*i(`k(p)rROTtmnw#7rf5rAU5e9j-5_!`vLjc@@bx}c zaGz+I`tQu5OV759ES$Ssa}5!GZNblh4@v-iE+P|0-^l{|pD6_Tv%VlyCFS_lx9GRB1NW`G!~ znJ9}$Qbxpx%*@ix8dJ4)LI@J{CLk`{PA_uypqN%7Uk9)(>T^HdRCOzM-;4r^B|=+K zpjbZ<0Vc%@uN{c;yHF98b-EMGocYfnDUZj4lQ95t%42Z!h~Cc<%2k+1y1KV^e|>4T)qqLFRDEw#UAae4 zRMgtG+17sk^%t5SBalMzRCzzo@6YGI{q6g2zy0?6@4u70Z98=P{^Q5n@84^7d!D1X zV8`?0<2m;IWBiD1+qTU&FL7|TW_vTqp{l&?P;{3FnvAY-ZNn=Kx?U@`YJL%zWTc3c zy5_8YK@lbgQxn1>hBg3<0ntsT^mJtQ%q$><+or@h-R479B9e>HLe8AfY#MefTnCdm zA`qSAi(a3tHWo!Bvf3a8Tr&A26(-cd*|}xqF(MQ>&b|&6!xfVO#%0+3hY)|IP5)45 zTy}jkD_c76R~zRtdaPG#QutX!#`0X6&FX5y5~AV?GZ&GvA^{XKspB;SSq6<~RgodG+WR|Abn#Ag?x@eq?_xJa||Mvau?d^5{X(Ys&L%i+9KK(dO?Zc%A zKE{+05qb8Jk00m0?=Sa#+ctOEo4vm5CmPuITVpe53*<3Ez!&_1uSAl#Wb0>qmro0% zdD~@9yQ+ynE5Y%?0Mynu^^!Omrwms$RfK2;M|ASc%+^c_vFbKimy$`2<=ZxwfxBO} z*Xsq$L1kt>(N%RZ#&aSp%f-oRIy&$C7-O~SMBGf*ZpzHeQF~;$vx-e7v*KcR%h$xz zrdsVKIH;gxT!21)Ju;=B&-IV;so_<-sCrXba2m5xy9OzfVXl+b1Sq6(tjWhGsMGCo zj1gm`WEr*tDWcwtqb3Sj4g`>rv6f8KZLj|J6+)X_EU-UB_SV{1M!~X5YEiZ25e?vT zxnBj>b*WgB&-Ft^B^9M!cr&U0cpTf5+K)|@K= z*zPyajF=#kU`9VRw8gYT;N#LAzN*l$b3!fGmvd?RKkctllyB z+1#|H@TZDQa(BT7W!rAiBhH8j*VcT$ZJU`fMx`(fpy_sddHMPE_VR7~{$1OaCT|~)zy0?6-+%k< z<9KG`Z~yZDo_)L+Z*T9<=kq+z<{MV}BNBDLN=4v25TZw*A19CDg15u=&3=A;(M`+u zEi*e4$jmRID~42{pJ8T(?jk+O z7u)NeJxbnv<#ts9j@bpeFaG^Myvi)Maw(BdF-u?8SNg|SYg1Ta2?);2toPohYi`$a zFfhH6Qk~o8^YkqS$%>oHA&aciR$Q8XPILjZ<`uY;f+Z!Dg&(P2aT&KxGjY*%y&~1> z{|xS*KeiN~%TVx(^pKBH&ulUbD|1&9fNJ{b2o$-_fJr$kRD0dfsj0ravx)~)&-W+U zpU+2Tx;ry^KdD8lQ?uPj1*yafHiIafktgGM z#u#HW-^};dpQg-$6;_ZqPsVXi+ zL6JsoKU?PWIjG@R(;2ie^2!}mpf2vIfPlwwoaZTG+qR3y=sb=yRi$mdHOPp-Kthj{ z+_%M^l~GGDMLIQm-tODBZJd|{i-_LG{haodq(lxvRkf*;8JSe05@K3np_zF!(jl!- zFw9g^70p^TYTT@O6Ol1SKl{)3|J&9!OY6NKA8%uLYi-;2-S=^xeQ0a#X413waWMF+ zbPjTKch6*xR8`xX-EU^?IQ#b>@8`g6zu)dJP`O*64cXM)Gk6@wqd(6aXAYRRwy9gn z*1M9L>CEqKQ_~ibhq{;s?2xnf8fa?_o@XC}%(hA6?>}#wx7O6s)wb|%Wk^e|vp- z-Om%5`8?0(<8gdEBEm%5zLgYzjuh&>-P4jIj<;vGU-}{MXOJNpt?7&C9e6*#|2kfN z{dPC#`O$GibAA5y{=f2{e);L;wr|--#CdBkx4pT^z5G!b(a%1fOt?6k{`5MViiboh zGjXb@c~U4chDJb!Z>j$KgMaI9Z~yi%^?keDZuk4$b|>V-U^BKYt&xZXDQAz)-j3f5 zP{5jbc|Z@2o~jB($2cvZs&23*E@Rtf%}5F%R2NmsF3o646tCI!Ua~*Q4ETrVwdF5aU6C2)KU3-K7;8G|LLcnVqMN+Tg`sXb$YKVEHLMF z6J#z#j`_igmk)reADRi4(UN?2v?eWH@q9je?{%16#n6Nu)KqZ|-Pdiu>ZNIvS3iH> zH@8iuF-@-DVvLJVP@g9%y_xwDt8PgyZts!#qUrIJ#T$u+N*&sPA0OpziMxi>K< zWQbBU)u<|jK%C@2$3mb3pE5(^?CF_<0DOv9nwf2_AiDO&yHEf$ZBu|RMO?1zb@ypN zI+DTx3d1o-S@goEXRMra9C~eZk+}-ds(TK2`9if^^gRReyM0dr4s;;}5%Xy}?XN z;Ip3Y%gg;d&yOE(-@kuvt=(Sk`|aLZqgX-^1Z2Xls}Hj^uEb7Et@In$P@O=@f8;2r zo}77RW+4R?*3@V&sl z7&8;0a^c@=yH!FaIL{OAuo(a+cyEl#W~;E9gc4pN*oxb>1G??oZEvQntreFM zV`M~G#Oo?C=b(NjE2_6 zZ4jeaHmEr=0ii1IuRoU(10bebK;GWpAJ6mm?{DAVKmPXj-=62;t(mJ0L8dkn6;SAk zW+_unB*7G_OYPgb_)Ptn2d72~2S%Ld(I1HW%`pag|8~E<+}hSQ^Cr;mKYo~~xxC)) z+uqEq`hZRU-~sz3z8B1+uMK;FOlF(gXb>wCQ+F*_Sa&anl*i-o{_!CB>!1D$`k1L{ z-N|NJx?4 zGEsGvR$Z6gqxzO}6tWciuHYLHimYC!m}aRIB$FXzg{0~QQDH0Xi;?88TS!~sS#Bk*1_ZZ=3l@} z<;tz9&$Bc%CI;hjX4&?A1sAZO#gq~o0B}a?SOUehh+O~?ElA}B>(ug=l!X&5C1jFb zv`KBPZK^uX-jDNmJnOKLq?WF>q(P17N1p>mpZIMNLD`O#{7wp^x>K2%Ho^5}p9nHU zl?+iSy9H$Ab#_O1v(NvkNkILPV}*PYQ>#fK#F>;*FY~76SoSeBdZOrnsA$TP6Rhcy zLNJH>RLK4Ql8_9vmY}iK-e#}L!|{cf#g=J2t(ou=dLIYN?cbHqd_>Y{l=*nFyTkUEB( zJv$0JVaMoi51Q(I-`)4B_Q!Y@80o&vg}Sh9{F=E}&Rz(LT$DpqlU#sUD3Bsv#k?@p z(3-!z-ex3C8O(F^<9YTG-@kuPh-quL+qU0ZYuk3)HlrCujI&q+IWlrYYYnLKu;K#> zyjWHB6hJBdM<0=-yUCz(Cy@X-KzL=9LGqKY;$8%zeAdu^j*mp9B-r|;DKo4l%Lx~z;y#OEk`tWouBAXLfXgBDg5gOs+~XhKQZ*EpRU}IrnXY)*ZxQT5&$QN2on!8}GBXFtzj)@m-@+J3vu;O=(xen66j%!qM9#C7{P ze%$0ny^%6$^I!Pq|$(Ce2J2f%G2$9llf$B&CrG8$h8@A2<^2?tdkK@Pt+v9u`&HMHB^_%&> z{OjMw7!cd;_icMXY}>Zm?RINfi)3?mHAYsOOiXprP=GSOlmslUmPkF~(MM;*G%nDA zxtb;*B$oxbOe|T^@xqpG+zcbvIB+#WCZbwUJ&G}g!&rd$RUe3mAQe)HLd~ybN&}iI z7EJNXjH<$k^7Wo0!%`{EyebSWRXiXoZx+)!P%9)7Bm<(6UDe9eLFT`fZSUl2Eu6%V z3@~DxIjJNFfe74WYv!RbNhDBIgn^XkY;AVTG5CCq&mN3lwC}IM9Sf=bHL0Iz`Dbuf zER-}*{VdH1RMLf_gqVs=$rpeIfhh?AvJZuPyTAPO`pxHkbu6S*WL!xB1bvc>s&vZC ztf~FS$4Axg)s1PbP1hpe%Hm7>#7E0a-@aAzBztMRO-+dN`7E={5iIL5tLIpqRWdy| zz1@?9`s~J(O%En@6LBVW6oN#lh!nBScL9ZpgbB>1fQUf%>70`D4=N=Am^W8fRWT@L z-&v@ty-qMP6v%RYQ8JhrWC#-wWR%`^r6e;QT3Z53r($J%sIiO)AX#>-GB;p|s(JAe zND`$UjVP2})2(c?Oe7{n9u5^`#=xMek&IppS{1L~gxZ?r_al^jzo}|#`?l|e1c+cp zj+ngmi0u75j^lWqV`N0QZ!h<6FE20qexE=|$QWltm`iIdQzMTm#BQ&*+wFF{zlumE zd}}{_`{moWZ@)j=hlz5V!zC4{nVTkrk(@hC6C zY2s4_>eMtQlq-@})to75#xf}>%#phhrm{Uh-lCs_*vyY-^m__SC|@gSRI^s+52%E&GhAVf4Se0jI+1ft*TApN+enHlhKEnsXM>C06dQ8 z#?2Ps$+xYN_vd-c3}OLbEAh{xOOmC3MG`D!{C-a1@7DD5UPFy&s)qYrBY-PCgTGcrb;BaYrbo;{m@)C~IP=sP0Z#LV&wcV|7gxg|x4I%cY_3{-Ge z<>;A|^vq;-k!;Q0fB&KCZL{5OnRp!i{q6aDK7RT2PvTFKZQBMbrcnit$3sM_Q*_(* zn>Q1!7U2qH^IWNqwY5qggwvluT)Zaa>p+sVQ>vkHWsSXv8r5nuv*ePH1g|KhE6^7) zytv$nR6-T4wT1>2Zz_UJHW9BdC##24#WAzUp_)?NtV9MwZS_SFYiqCiF8fFu#ZSagu=JsD{(7u5iU zitn?|144hu8k+irm;4%$zi#aM^K06Ez4NnDWN{y+_&-Tdrv`K(b->J2fw;LKkspr- zN7y!vggN3o&#kqs^g@7~I2jRBGaJ3DYV#K0Wz!5`qQ!yU`{OujbbgLr1DWYToE0G> zGc7;n4q3%-HBYu}D}2J{ewBfldA^^K_3P;FEzFyW1cV|~g^np0Ue9}LFV%`XM{;nS z{XEa;V{a{!0%VG(ib$!R6pnKx(VrG;Z|~nDG&9cg84(5l$8=x$y2DrVG6q@BA(feD zIa>HKio0sEdTPx+&-IAN^)yx7ZEL<|W`;5;XsTNQi4VH1 zgV90tZA$A|>0K|-6sQ3zONDrj6#8_1?39_p$!%U@K_&p~`+mFKzPhl2J_VYY@FSCal#3fYr z`(0Jz=wd#C$2mxmWJBM4=AJU;OqLelf0T#SY%zhtmDQL6sxk4hksL^54hYTl*PnkH zJ&qCmI1{3j+wIo&n}``CRovak?nh?2>+>0mJR*qN|jw1@P_V zH8V?0e+s$_T&_%Fsmgr`CbLn_C5Ay_2?dL$;DUGoC_=74(nJ6WfJh;rbk){cO<7fo z>`1qAiUo?w&M8|O(J{qDP?c-uAfxHJ+ZEgq5KJ5K+B}?dsw*-28Yk;t0@&O|NXap~ zYb?>oO-s+5QN2^La?WjY@kQ}rMpO|qz$h7xL4uJCL#tSS{q~8yt47EDi6Z@|&IaGr zXTPF8*9iMFlz;h)nSCJ*3sKOhMY*BY3KkJ}j{=*M$$UPasnSeZYeDS$-j8RcNToD= z^ZBclQK4y zky&<;93#l1_uhOTarWM8V0az3jwwt=F*hxo-^`vUR5eCsDoR#Iv!@7riAG$3zB(=S zQbXx9iHKWMb-+zcw@R=r2Kt$1H+2Pd^eo$zq==-qrdACr^`=NN1DUP@NQ@InQ-f(l z%obB7lrAnRT|#n#`IW-VQQb+Z)Sk-{m>5Dd^BNL7AMdJa*4*9A-RiA<4866V_U%=K zzS-^d_S4VbZu_19G1&X_aeh3{$K(0__&A>Z`)}{=u40*ZKcD@KF-Eg?JdTJl;v}hx zm)nad82JA8-zuLl)m(^SL`IRWZNlcrsjY+@)e;qv(oH#zwz_ z(ahs~zTRI{H6jG!W<}UGvy72BFj;jQA|r-w=^! zKU+BsXr;O`mY?qxYNS@PG3QXlim*~fG#PMo#c%xsY|N|$a7Ug@Xm z)wP(rSDfVR8cJqbl#3nN&-JST5gklqV2l_Xg}gQ~(dba&%<&6i1e?kf@|lI>%S`(k zAg>;ph)5CH0t<{at&6LL7m`&Gd{$Zoex9gDmKrVy$V44ZN#XTynISQs+7yx`35v>1 zpsV>-dJrNa+va2tRmi7kwVkX~Xd^-V4RW~10k^BG?LF~Q+8CJiylBLK3RH|xD{U^Pi;&uNwGj|{EnOQaH=hA9z zBTX6w=hC;j>6iJTDuFS^d9Kz6YUQMemO5^ZiH~8{06$AimO9Z5YH2o25k%5m#~6`= zAyk8?iKKc&WELd|T774TtMoI4Wx3_dEN%F53hQH>z5CY0&D>jS+Whf6W~niu2f~OR zXj>ICSp86{DiPbZUFU(S%5u6mj!vFM{Yg)BP&$}gd?b%ZvbS{ z8d_!sIm3YIYC;L1FomfNT?@{VR{=3U1AyshHoeQy{Li3_E+qfakoWq_)#UmN>-ci9 ze~j!{R;xmtipw%b7*;)W(Y2pz4vE$p)V6Jleh|pgXk}^q2?+`c8eTVc(uGU(T%_`* zj6hAh&y%G36t_~h=krOjG*ly^m!wRl9-*pm7cz|3{Hph}&n@~1^B0k3tqvxg<5@tG z)bo6@ND3UJv?f|U2Eq|cjts^~W;ShR&PdY&H(vH|6tUE5qZB(EWR$jWDJf(YqTH!c zfW*F@Z;h|84og>K4YYy$^2i!4TfwCgHcygCjJl5SIjojXMZF0GNy%B#+4Qqlq0Hjf z_b8FxF%m)}3p%4(OQ6imQfj0$+ibP2>N%sv>r1a~rtm-^CJXy8dzwP#7$b|k*YKsC zbv-YCjsk^=K{Un1f0(+Yz_$j~=h?sicz=6*R5myI6F>xy$8mf-M$&X3nK5uNd2}Kx zGU?_9L!yN0W@3jW6{xmSD5)n$a^459jb!WQzLCiYORKLWL%3V$kLZK$Ai2oA=|4=%Z6z+`L4mGH9Y%z5#+7 z`Xr;lVY;cMCr6a;&kTsA+;l!^+V*YV_r}dmS5BF^=;bWBjMV{WF-a9Au{O^AOW7@0h^N zt6w&IYKziRX*)!eqD2>}_o2Dz zk>zeILKTN3Wz8gRx2kMndA3y?JdQ&@y{&QOFWO9|I+Cn}rSfQyDYn$iuy&Uu#GqMu zcx7c4S#r~&FqvdA5@ecM9oU@yBddsrAZF1qb5YC@ut17Q3`Ti|kgQ{-@*)CDzYj8Y zP0aw5RLDSt+NNqY05L^IbA7$tMMTvfZ-33q9*L2=nF}-XJRT4OAf_%}L$(M4a8*~& zOkpTW$)XHMCm|k}PACMx{h@oF{7}A~INN7MW>A z)#>^a5sPU%3EMNNS2`7{T2Po;&kP}-k7@Wm7r$Fs=+sPZO>=p5Nh7)7r7>GTB&3Mk zSQ%2PxQMCRbbOL2c@5~*FBdUFp(*gm9~SL|LE1z!tAe{n^k)f$(ssd?J zpxqIPXlkWyNa$c_1XaDBXW=boB+W19Q4&GS-pEo6ELKm526>V#!r5pP01|g{_`nGP+SGA_J=&DWJi8&qdL!t+Tcd%A-e2BviGM z5>i=ae6x{b1~wWHB%0eSZpSLkNr6UmQ86fyqBM00Ms_QsfGX+$SA$wpDWNM>XvP^M znVlIizMs!(XG?OwwO4s*=E=lJR{+GPY=%nGn_t^g#7K@YLQ+*Ths;jw0JI?Ru1(~! zB21BDC<#_`24o_a*hk74sFbwIaSBsFnir5{=~gZdK2SwtaX+szdlem}Cm2XUCIq7D zbI)dGROrae0$dbS(5f3ylU{~a)DB`2v>{X>G!k7#MN~5_!tW*yP16RdERa0Jj^{g6 zR6LB={UFdk#^W)bk2sEhm%0BTZvRK|{olV0z+%IHew`#zt3ImA1odR+iS?-}+#}8! zmZ?CcM1N3RFHs{4q}rGCL}s!WPe~>EYE0MfjxjQ`=ORITGMibZ(&v{;xFX#IH?3|S zrtPkjI>fr7mR}TjWY)*J{>?IV44g4wE>0l=49>2ah$!Jj=ZQgPv*Z-o&00V+=|Ck9 z$vR4ei5zR8OF96ELYXa&Ypf4G>mm zMb`X8XOdOb%~6S}6v?$>?V_MjTnhA5PbXmt_6ez)Y`+kKNX7L0Q~A8-z|>n{4%7@+ zX9E%}K?2}vhynl{C7D$qRj8;d6zq8>$x`d}rfw8u^1uJ(&-eEGUt@g42}wxoW(FG- zG?)B0p!0aFtFOgc%ee%X=M-dCuB-j3_q$qi04T)zzLin+h!YHmz~If=Fp?#fc|~?J z6TsB8sJ)-MfU|l6pz5B3))LW>$SeVVh^U!bpQof~P^Y4n{ote;m8eK&=1rYfd?SL)hy?d?#ID^% zEGWII85t|CkLq5E7;`>c5h-NF3ub?iDdr3}?uaqHAUEW7{@pvB6YTL)V;;EG)dQ1kOZq1Y#l=gH_|xM5~)t z1J%zVYOc26g#wi80e{W)s_C|fQrEvgkQdTszBw~9XFoK8s@hu1Kn#itsGw$GWT42p zDRD-`FoL2Z!xNzrj2r_@QZ{(n&=S8CnYX;cqiJ<05*V2}6CFKzc zTRj;Qk)j+KzCdh~D1X;ECYdw$8Z4h)6?d)l)oPQS_?FAr^BAAw|p*M9S%eT1PO}&c-D)BM>M@N2X!Q#S&AKeoe4HC8^iqJQV6?RDlL1 zGe!u6Su;6klwY}-yQzl0-ZuXF%lr0xp4kUa7X8gAS|HDBg{yjA-nl8&cIU#F(yk!2 zG@{Ec7l;Uvtm$tpB>;n|q=~9a+cz-GO(+GExUv%g%96Dz;UYP+2z8`V0awt{X-TC* z5{fJmjV5Veh)nJvb8B{g*-3jmB1R?xKHX{LXxBR1gl;ZSX2^I zWtU3fF_AXVdq$5DIgXEa!988IHFZ@*eN34W;$La)T&DsykE za<1n}=c?C3)#*>0UydjhL1Zqt?drt~u-u18B2bpP^4SYlRf<^J*Lnt!R^zW%+B z7YL>fKFUv-v0{$yc#964r24M5HyzlzDKdfD_ao zqb8*_6HXCROI4`uIYw0|1YN{T-A$#DWPHxDnl*PHtN3!ZQxe#)=Hg}k7039P49NF2eHP=1zJbNa^_if+z?|=WX@B1%5 z|MdF$`s3~V{l`Z{ALQoV2|(w;uM_xLh%}j16mykFs(N+as~9RgMD$GN7-KM@VVU*$ zy%j8^R5sOJC!l-Go{Xy2{1_(7hA~AINu8GQ;1^FyRW%gX1X7wwOAWEgrn$kBrP}n>J63qbC`RJp1$cd~Tee`;uJmV_;;@kx~&ohp?BGo~c0Z``%jP7V5Hj zv#oKAGsZc(igVghPqatz!vS0tE)|DGWb}!B%$yc_g^8*`{%3%iVqP;wNins*gzVa+ z*Q&D8YBe2WtTcOOFd_oz7(qtnAQBj&RU%;D_uAkB100KcNPcI z?JxH|Bl7I~zF9ThB7_k4G^?S8icm7tD2y`Q2FcO_!PxNrB5{%+RnwvqPZ zaqQdc?e$Olcz5+N#_{op?EAhql^lneMSp%AsfyiO+iws?=FipoJQ5gCHIvP2ePqOl z=tOAN9w^dazoI@l{X`hKz21;0Y-PQ%V1>UMi;FKJ6Y8J@+3v=gh1i57Gcq&e@vY8V zCZqQt4eo|iCI{n0BSaEU758oLV!~55zU-UG9(^FmP;Xl^>v@hd<>amH4$Cudw?AFy zaqm~`_RMUpHEmR$=b6dk%6{|@03t*b_y#7{tO>PN?N!56?0W7Wx7#fn0aQ4al+m-n zjoL;EaHojKM*0|k_aOo$ow&DC?&D=+BtFhJ^l<_rn^;Q~lqMYIEO{IiiU-IPn9)c3aE{x*L9@qBmVc*Wy9(#FfT zZH%LznN-FuPjGBo{`|51)?DwKTYHgENM;|1uH*8u z#mC2a96HkB&F^B{8K~2RHaF2Ia;MbX83Bw^8#Yy+3P&SnWUqA#SmcNj za7Py3u!3s6?Q9##65#-#fGGv}=nt+sxlYpwkG2H-Jjd}oRrRIa0rd09829_VnfHD$ z&w8NsyIjzuIzndIRGUdVF*p)L4w%84Tbakd_vStZ_#8^!ZnqXK`p{?YrtZ6GL$xLk zlEdy*b58%mk>^NcCRES!e0zSpKb}SOQ8U}^9)Ch09{}KAlSEbJe`0iUW%2*nYu>1p zn_ur=Y+V6u63U>N`Fgu|Q!{%!9&bNB#&MQ&p37O6p^JqzkL7s~R@3^k$u1W(VUhY` z)p1zG$8~k8y6?MhO;shKAQakMh8)L{92Iv^^%_$s$8nfX{B_yem|sX(m8mWnF$6p^ z^VgqWMn9j==a2U{0PcRf-F9u~dCu>tP7%`0-F@^gs?b`~-Re4Juxc@K0n3S8y^AFT zSoa}kZ9$X?SOstea^Dug50GgWxtC(JR=)+RFjZ>L>5)9yBg4chU&~2iTL3s}^~7dr zvWj#}FiCfvmWnf#Hme%qVt&_;O3BQ;-EL;4>LF~A(U=lSv0#HJgyw7(qDUmT?SiBGrzJcIElBY{ ziieRb$0CvP6xV`i)nOo|jzpVk+ZiNnwHBr1Uj#*H__u%iw@ffqVGZuu%oV9ko2ilP zy-R2gh|)x(01K;O0xdW8W&9?R%S)g_`M&Hkb8$H4pQs=6hi2)umvssU$hGjyOD&S+ zeTu1nY5AE})`prLrQ ziZr+en>F66QbbJLrC5F?L{_IytqNVT4Q2*g1F$c_)@|z;qxVt1od8u-?Z?~OaU37V zc|4yZ3j~qSvGaZ{kJ*;F+G-bk@X{8LFCp^(Z*X6?_>zk&=10~11ZAn((`+}h80XvD z$M^5w$4C(wV+>3?F-1xLqB7IWoHg4dxm=*k<-%M3vza+Y zUQ+>}JVz%n5;D3Fj8UKA9DTQDZcvDDyS069s&L;#1oTXd&Z>s{h`)dTo#e>eZhLFZ zx7K@qjMJC7VkNCHddUnN)fyGmLPsg0v=VK$GK)Jf9r{Ge^tBd2p^1uCBt^o8PGQKp zc4G6{TPH%bv=_zQ`*Bc0N`+)q#FF&wE(?(cR0yL8t<`jkR2gH;?o`C!hzLZcq)7?p zp&lh?Z$QfvwGQvVSrammlt7h;;QX1Oju?Yuwr(|=ZM{!=dgW_eJ9(|i&xJ~I5*<`I zf>W-i76)_%l54?>$jkHw(7{2sOsIN5KAz8}RwKWJl$K5=w<+f^9bo}Yn`m@Vl!R5! z%nX$7e_wdhjLZ}YisXkvhEQ@+glgNHD9CY~hx;bxB~7lg;izTxl55<4?NgOfzN)>b z!00oXFtt{oxPifR_nM&28#V{&`;eSdj*@lr|!$qddNJ#s)nvQ#H3NxOJnswL3au(Y8_+RB%UwXgbl zVB<@k`epmGymNKhaQ5C~a$-L-G(o}L$&!bxU}9>MMWvJQ(KE-WhR^I4PO2{|#MP>` zT6m)BrrO|cQc7e}-y125w5&4%yE}DC^n8h^>@gRthyf#$BLcw3$MZbT(`*JWQ{#UEj3Ut>G)*=Ab*nW|b|XftQckkmpfAIGt`Ed$TzGtQ{8 z=seG}Q&pQf)MjsSZQW_&BCs_H1aK}Qas_Wy?J-vsnOJ9OF@r^9fSMT`J!5Wp0Rd4Z zG_q$35r}j(C~MetyKNAv;&$6!US8T}&G)*Y{mkAYf}>~e&70e3eI(JJpUkaI|EKv1P>pm{wb zB6^^tXju(w)na{SjT9@cX3{{@Mj^)Ny??yF8{rg*!0qm>x%=)-W=Zg6%atI{lun)K z$fAc=xGV|j1t6KT4G~e(ZS&Ub9OFEW^Lb!nGmliVdN~o<0gGgm0a;EhB|(~4Mf1r% zU}AaQUC~wq?z7ji@B6;jRC7vnR6m?wO8i`7%&*aT26355#=N)i5$AP(LF%O257*IU*vFr8ii8f~p;BlX>9{ zX7*q;Q@2@`Ns>xsfGB%RbxwdEVG1&%hNg8hLE?CSOz4z1wR}7t$MZOjejI&_GBc<# zv&G0j=ogTSe*xi69Bw7wlPvrh+2>VS6}ZaMG0Z$-Kxj4yP8%p zsp_`n2t=>_sMVtqkw`LOo29gW{RHp9D?`?m5F;j1Pzu)ioy|q@VCQLbw(a*@l^Ks-+or$J_hk z`Ru*d=uON@s5%rU~w}|Es>REQ58PprlqabUobX?h>>PS05#&H zAx1&dGN@iDgbHmISc4`GnJ!MCE4}VySt4S2lFVb-VA|$_oO`%za=n?6mIG#Df*{Lc zWph2EM+`I1EREg)GNsHoZ_5HU ztmRSawUrUyNywq;byr6})62>|_hJfSylXKSx*w@nuPd>JZp zt$&$B?s=f96!2;4{#+oR3jUmT1CnNTyWQ@`@$vqC9FOyOHuKGGq{N`D`6M%t>~hvn zMup~z(DII}p;}6UC^ev(z?C+#h}`e@mzS3{BmKk_)y>ab5R}hY^oMGD-LX&i>UxcS zTqF7m4M~KXt$nUvXaRE3_h!#vX+Vj{Y$+5fU%sB3v&Ty^GLfRztUQO2s$Q8o3%j19 zZnw^k=vs$GADLb58h)VS>Llx^XIni+8*z0zG3CG}PWdFS8>k0kg^%V)bp3b_{E~tTOiBAhaB?6iW1vYJ} zo`dIkp63}RDh4W{AY>t625T8XuK)-DFk2hstHHM_Iaw!URykCR)Xp(&ghaf#H4h{* zC|2CC#ac=xkhbpvH{Dv>+qP}nZL@8oSTB;K84c3Phc4pKcE9fss53ev`|~l*Qy6wK z)l!Ekrs4pr$5bU}uhU2sMRr&>v+C1QbQTfRB(L;PMH3g7JP^pqan?q?85jJ?`GEf%j z21v=G!b@@}B(xH4krnz&)zPKQc}u|8s?#;1yMpA>_LwL@RkeC*AgZWh3H5N>wgTx# zriH{+|5v%;a=UFW(VM9fnsJVrqgWzzK4yi0Rr;}P7EogX6hYTqRS^;tP}Jon+wHy= zaP$6hdwIRfa_cRimMm&+_2|q#RYG&I_-D(pV(TAS8Y04}V9rbTzBYG#eVq`xN`Wv| zzpH=w`PDO75R%oApR23Fj%dlLMnttStDU4`=B7G*8qBOmsRBp{30kQJOnqa{OIVXg zlBRXu&|H*^h@%uS19LT~W1&$*q;0o|czpEpJfHoHhzywez8hA=J4f}b>cIVyx!b}! z&zmUMHUHyF{(sI?E0_F-Cm|v>IWm$t7?|v7QREnBzoJ<0{X7RL5viJ86C+9>qFPzi z&lD7h7gsW(ZKhzE8KcIboE_e&$-qP+A%#MVuBsGKH<-dr2%69YYN^%IFS!7W;LTJ_ z#Kc?M_PuR8MEe+-FfUPJQ-_Ls3(?2>ae}hHyzZN0^!Fcjnjv#{!|3%9i@98P%hDiG z9T^Z}VlY)`5; ztd#RQ(Pixx6|oT~VCC3s*4)#~_Wl0oj~a_kyu4_65?9XETDxt#h#bdJ>FtJHmsCg~ z6b76^6v{-))EA;q3BizHH#4_3FF&URXaOWHeOH-M96*W{t(2g$>D0nrDI2ceUTxe} z^NFjODJdM08DMO-y>8pfX1%vm3udAy>6uJPCPWf*Jd_kG#T|httkr|)&qHUyGs{_( zCc>h>S6g5WV4W~zyf*^|1gG6-^aU@h`JeWTuL%dhfkQk}x&Xtr*v!aJG(#B$TQt=4LYq z&K%WazoMd=9p@NhY^0g>oZjE(=oQ(Makb-BwP&1>kK_FK_^1JOwK^y;RW}cWVbr}c zGvJfq?`D56-o5k(zp@0c$Sxw~6Et7fTWWMXH4#u}5WdvN)$dCZV2L|t=6Q~?T12wO z_nY}qCOe`btgd4DS(gP?WxG-e1cJ|_UMEdCA~}gRm0blUO{uhf<>*~i)lDglOmF}r znUIj#PGTCt6bXnn6E~P@bKiH{wixWu0i`stCfQZYeQQbE9@cQ=F=Fr_7;V4({QB*u zmzO+`?29M|?X7ej4X^mAOvh|Wk? z8Eg~CY*|J}rAssWp5g~2E^&?WlC2)w+AVbePhyVvoM)CL-uvS_V<5BI2ck4W73cFI zBCVBW3~^>kn8*#gidVfcFa#$Pl6szF!6cfyE46mrGe-5SF=eGIso}owB`9CR;49OK z=a|?8bCOr?l539Y^QU3DDG32}(dmg17#Lu@{rLTMe`VPwxk4pq(WNt)Q4Wwu5%tWW z@EB*#yy@z0LxBBupBpN3#Oxz0Ga}Tk4%=sNACErN3JR>MdNoO?NtP-3bf;G#C9X-= zV(pO(uqwP!{+rX!y*S@;jy^_L)1O{%zx?$4{r$($(@jc+Qe}y3LI5;t0&wCQz)X($ zyG-cb62oX}y0^XU4WrldGPA-C0BCKpyNjPCA~i=bvq)Rx{CS!6H0R-`Gll2eE8)^) zRa&M1G^I+G&sdd7Ng#^cG!ONONhM7gghn)Rc4S2bRMK(QEI)j2i8jB4U0s?Xm)str{~!x%kt zkaC>ozTZnuvG;-LY_a#{9gV^l&p^#jPBW%L_39#Cr{VgP^SgLaHfjrLWl7)~#;$KH z7?w^s^I*@PzI_|xeEa>k=i_m^-M-ylzK;`NQ$w;JXP@bY45CAR`u0sGf9tcaDFq+Q zh@^JfHVwBzp`Y_uGDJTSgye z|9QXH9$>K85e`bgTDln2G9!;nq?x%{mflp26REXD)l{rDo~UlkB)OO}0ALVzPi96J(aqFFj(!fNNscTtQuiXiq!H77 zrvMy<6)`LESsin;&0S{2%#~s}p~Bo4^U!Ur;eowv-V=eqhvpC?cRyZlE>)Cg2pO@ zW2BIq@Av!7#e47lIM4HWp3jIwH@oe-Z>FY5Y};1ru(Vn;vjWhCS9v@hSWWGU4+WJe zJOn13>;zz@IW-+8q;lW)=ldZSsIkxcW{Xy~uJW@=OAb}F1tuv|hDtkX1jw>16`4bC zah;u$p{g%0FB$W9iCcxE7$G9!29YsRt%Uj~rYSIDjB=+K(E-$es!pdf+1l3HR?P;1 zm5Q$1Qno0G}$?>1CVdXcv#M}c+lnG1;KFnrcnQf6-BFhze3I$y24`M(7qAG6HsnFy2 zFtgro`&M1H=C-LCpiC2M?$@zw0Dk|M-#SJ6Z=0?|dpC*)J4aZA&|@}PNQ;&5_(*Mz$!EUScy+*mB?y_{PgmAo#GYN%l%}Y`Xtxi>phKrAs>*! zK+9b)Ra_9@SfEar0`1l=e01iRL0nQNJW52|%nF+!BJyNT%@?_{xvDef-~X8Es|8^-6WEa_KXZ6rsEC zZswwbX7X~|w7LnwvvZKi;1q+XF0+y>7))eGh6u|H(l@uwtf`n5zP?9R`6*`Bnv&jH zIa<{!!If)F>E@qSC0fXtiGSDC*(;2exwK~Pro3NJ88Z{n3CUff*G18t{ayz$a`aIr zZu3gThN8qGcwr%xDajGfc>*jM^WrpAL9q&BncGxkzujaNj%&c1D$@Hiau!Uacwr&c z2G@EZ1V*VwQniROf<2Fj=;t`kv-fVzBc}IU-KhK`mT)E9R|)9b+Ij+?`Y=T}n(j9x z^Hq)cYP0b3e_z@%TrkS(FaQ4ElVb+@+011*+-9V??yaWWsfw*hYc2aB>!o!ih5SEH zmScb_?CJpV>`wp`tKgo?D|Kp_Au{LON)|+=f*q0Pc`Dtv)_ji$)eayN!jcW)^KrJ@ zah`sj&$I3Bo1lSV>h7K?5v$Re*?T|F{`U6qeE)bppMo@LN9LoSsApsvlEG68cj8G!ub$_%xOMHa#z(wz@q?J z_FmK2#b?jA6x}kBn@wEvECW;{`WVBvR)XfGR<`;fdTVP?R~)?*O(8@SM#N$mT>Atu zrIPDMEd{oTzsy8gR?X&&>b_$Y@;%ck;a#Jou{5$KomEwNj=UPNB1wfGK-f3R5## zoM^n5!sxAuNYBEifFwsx29R}@QE>L&`|)D8Tp4LZP^FYf7DVK;;I3i=*$(5x3!4OR1* zsr#+9mK?q3dOX!~$cv*B@u_%0ExgHeFOX@Khl{KP$adQ*{U=8dCN&!rXh??E4PcMe z&)~HJG3}iJB~VZA7=4V<&vEp#g1d8@)tJbYpa{fD&k7*38nZg_wbQwX1u6T7wEZ6< z^*_AkW&gWOpZp3s{2JdAC?PJ`@|G2$tL%GY?)Sa9Yjb6qs%$EM3e>_rl}REFxR&;BH+ zZc@0$$cWK{>Ij5nC}^gw`M$SAoM)gg$XP-zP8~=^B#V$!fR>VKUWrnnT;}i&S%q&F zM@@Z>mop+q>F9~+8AFoB98(`F$rQsbZd(g5M?W((&pKHLGnf~5ph|dr0mA7gqSS&O zl!y>;^R{^-VJqE6Bx8&i zW2|2M<-9?X?>{Cz&0H}7*apeod&D5il76ecOXffbP!cY4B;LolY#uJ4h+fHiW?))y zR$NS*Rx+$od%_)8Qm7?_Ivw3as`Vf;dWaDagTinTXa>*BjO>|17$u|aW^MCS%E-|p z1Er5fQths0qH`=!h&~i4DaIHhMwXAooD3~|m(Gd*%KT(HSV>9==MWG|O_J)Q_pibF zis}D8cK?U*T|VvHcgOLzLU*V(Oc}|Lv`Svdob*pzA6O4Gfw8fmJKW^jQW_TIrXf zMklBH56S^ABVx$y?%QU~Au<@rsT0=wAvI~|RIEE0XAED+>Z|X3?T#v3Vm|tc(Mr?x;|A@+qRi)sZaZmgZ;vtXvx#oPTE}Eecv0&K#ozdS<5=E$a%R^Xe^KK z;$-Ak zsua!Y#MBe%F%sM+Mbsuo0eXxc5u-Ss>(bD2H&OVj!~D!9o6crG1R3QP4jih3!+eQ)1>dhw^v=R+E=@P2b@Vv@r6Xiv&D-z=E z)#c2Qt}1uQ5d$9}$Jx7_n-5xP_xbyekMo?qtYe&`hXDJw{qpUnecx|;6T#@mdG3r; zG;;Lk$B)NHWR5e&qo1Lvsu2ju(KLb`fdxg#-HTy(o$d3?J1bMjDqWM5kd$tfvlk>Z zdoCPN^HGDf!eglcY61~gF)h*aEdM(tv?P|YTGM3@Gz-gq$W(lsJ?>cs4qma6AfyuO zDV>oERmL^!ddPF==n|;~5JF~wqM+(jq>|vkSx;1C)HjYXuzJUtdAr@V&97Ge$^?k% z1h@z)suU=x+m^E(H$O~a2a9JfYGS5YzjfWQeJ|t2;B2ER&}MpZyepH)Yo$k>qinXN;S= zij>cWtVz2Rd;Us~mZ~CvWMDZdD$!hbZ{p2xhzZ;vHX2(ActOcls4fw4w`RJzmnmyz zmQ@Z#lx8$I#;C|L8{#$9&dk-Q(M#X7UOwA^BSwZqL?7cEagH%c%gfytpE7{@IRvsg z(PK;yPK5}Wqk@9`is&bzYDF(#v8aexV4^5`YTd0By9%sawn_zaltSXwoMq}wVbnaH z8VZW5nX03Y=Q?Hxp}DEJsy=>nF8PomtrmKr_Z}mAjMWsY~BnuImt-0E)uwGtofBDOwOW}BXe~vL8=W!gz^El2Z!Y|7jVLpx7!%z_!j}yg) zC41F16iQ{XljB>#eaYP`$V((ob>b{R+}e#!B+fWQ#H3Vcbd2o1lTX1e*g$v3@r>v5 zImV>Pnc248?=Smbe*G_IVlGIIXuY@S&&(nF*mk?!{Wu8`C!c!F+#wR83vW98x~3B@(w(8W<{C zsa-^Ak{~mB4=O8{9-~)Jtj5%(A`L*sDOQ{tokk3)W>N<8Q|?@V$Wl^0?M%(^v6f(U z&mkg|kuZ@gTkpyeHeS5*Fg5oI7NV-#<|30+b3rER-pIs*&fpHfTPr!I`~zs3ZQm{* z_E-SZ4K8lAJ#qII-CrTdJ-i;;r59 zhy=8Kw|9|Z*AT$a5!ZytDzzi$@I%$9S+F7`2PX_kME>ZRccmQvr@=GV z`~S0KzcLifYKzzALTMRhPC-F;{r2sftC9TqXCD3h`1tto@&0%`p2yKM5)5XK=rPVh zRoBdFjtW(aLRN=MPO*a`9h%vt^G?n)f5fC=OcQeROt`xcCypHBNHGlnw-@K=5z)t? z`Oq|WH4A~o88L>qn~S;Kw(a%y`s=U1=I9YY4RffXBhE7(9}jP`zua51=ke(MfKbhb zu(^6`O6DnXMlH=V!qouU!n#nkv>ADIKSVRL@Gj?h+SWF6F*Qn2ESUK?-c_}=<}-Cd zfSR1w_~cb%Qgxt=8pSbcHUIY8_t)1~ zcOU&6$JtyrH$>iFb~6<##C_=sii^h?S9(s;)v9<`odc6Xl1zjNpsOQcg0>meMlfM+ zB__lJBLhFm~E}KKKgMSA0HnXsJuQC5!d=L)m$VX6_KrNQ$k#H3NMedBwC56&vtcp z0qhMm#4ZEbw^Zx)qmIXmG2T+I(H&fY2 zmcz)kP(?(hI!z_;p9$25rKp$*;6hk&K*%`LM?U*`K6^=>G}DOqNJ(+KfBR|Q?>~*+ zk3P;Z&gb)^PgC!y+8m>2=EMJ6sG2AYewNn zX@`o7=mh-&HjzO>l`mKc)r(IoUdXADJohKZzmm?>V631lKV z8_@+@vzvQOX^2s&$5w&k@563>+tF}@iQe}{#(6v=N++$>YYI$FjQUeZghT|FH&!4z zI0zz}x7+PT!G-GG{Vswjucs6QMhFnhfz(n=gx4wFTI=VG5!$3R6OkOFk1m6?MK<%c znYvc1N1he>nbbhJR5iGqBZ6sWFYP{(NAC!z$oJcC)HrcI`q>kZwzU^^|9BjtuBwzI z0!ypzjK*S(5-3d@QxQ4OE@hmMIl5JRiJ|8C^n3T04*NJVK8}9ApU>mA`CtD0r+@n8 z7j!bEskul*gnpD=OQq^f%|$h-U@xIqrfiOo5P3-cx&5Nl+jeJT?_Eah+6M8LKmX}p z|Nh&Lx3^UIzKH-b!nlQeyX$Cvj5FoPjk;Nie4G!6K#gkn>TOlbj zM{7`}$XS{UZ&kjyVr#ND01xV+f=ED^ioqPJL@Pv@e$+tJbO&1x!3c~LZ3?JRbfg~O z6L^aBhCfwQH!-DasM0xGWTihqVpMmgye7sNk8>2cd0=~H937kOU}p?VIHj$^YYa6j z9trYDGEHQkq_9z-1xVVo@nW_zKk*zq!`RLLpE$gMC=hGvVVNEh^Sb9p3mbrp3jf-Jojy%q?=?_adl8{ z+oqW5%ornAn`t(nO0`CU_Z}~j}+k_k9N=+Q^*-?pt#;^p91UA|@i=OmvJBNk!orK~90j zo2n`Z

>M^X9%i9*>1L$)!M2ZLy`UrHbOp*CyO|vuS2jNA)E3j8i5kEec>G&R&O^UBnrdY@CTn8KKQ&Lv?~|D*d_Kd9EBMIm;m{X>t|qO9(Hc zx<&=fG#Xv@z-c^r<`^f>wpe?!c%T(8F*D0)>l$ZWn?-fDGDO6`0$O|v@cn!qkB_(a zxAXao=kq*{KmGbkiNBise<}O3CO48KT@?IuF?RqFBr~(BPj7QZh9i0W>t<92zT@J%9zpvwa1y5MOmk1Z6Xuyp5fKUh=`IB ztznD1rw`AO>1HNNYwh}ad%4|inXr*mQkuCib8;{O+(=fHON9099$*S0xm;Gct;?sE z*UNQ@zx@aEvu}F_YJ)1vqG~-b1!cm#M>5Gw3?s?i#u#QJiA8u>^!wu=am;yk)KarJ z9^cUi0f6M;79dqEuwVkyDQ~K_QiUg$Sv%rTq9>vQLq_h-u*k3xVNRTpZ60-+-M3)_ z0BcD;Mj)QUW^^OciI36QY=Q%a(O4GrtIYLCs#{6t{*iMg#>}ORUuV3Co^8$WGl`bW z;KLv+35Z!`o_u}4r^E;YA{e7qXOyab5kORQG+rpPuX}7Xs*cR$03YY5f;^_p?|F_K zbNh&S0-bO&nhNH%WnB3C*q-0|{@kAT@B4GJv19n_=NHQWlTSVLB+bK>p zddw5|NiykPK{IKiIV?Q_N}_J#CDH`J9B!V&JRDxaHGz~01(1@YF8_C*pje=pw6f=A z15?_EL5Fw|N$IT@oY$=PzHR&F?j+2rmCa>1aSBMK@~=9bqw&IA6#L%+&3Q^~6<(Rn zq(RR+Nu);$h2|8hQmW!;RL(#7LPVevi2i**|J@({>(?pYJvM*|#v-bzw`9OA+%rpv znZ?qWNJwH9%avkKTds+aY?c139sqSu(K@69pB1rgUZ& z0E7xEO;uH!5=TUsn;QUuFu^j+GT}D5^`7QVo`=52b-A|7^>Vo^P0cw=dNb^~IkOW{ zPy$T6T-HT|e(UMhtw-N7Efq9YT77A<4jwK}QEt2}OsW$q6Jb8W-HK7(7Pf73&k`9X zAr)bPrMIRvayZ6!wbe4pF=L*@hnPlUf81MJTWd^mV8}p3t1zi#P2u23?rvinH6SrF zX0-$n*@!X5rFre@%uLHMsJhW_PZ_2Bw=@ZfG;;?b$6Oc!4kyl;twc(Jb}+TWN=Z%H z(q>{*%g8we#z#g$Z z?`iJCdf$9>%Frf`Od#m69GNz^kP_1;N*o?#vu`Vv_6A|9EEVF25ArHAkrXBbi5b}` zcMk)KhEcOTs$g}<74g*O%*cQz#dK(A<`P-rV=|i&W|8LUhzQ}vgb4KB_uefW30;Is zIyb;fC1=XV>$qtVMGl|wxul#0NZU628s5^Q$&{5 zl$DwFpuW4^nVG-@@|i3}Qqq+8g$NNB;;BTnjS8GY0T6LPQOb_wa1Id3#29_w_kAB9 zB#UbVJx5$MhENb9k0EM!7-+)Vk1ag60{-DE`my|zY?*ug>KZ=i|Z}ieSdD- z^Z9(cZ;!X$J0eORGEBx8URa+uq3GR<+`p5U2dQH_Shfe`SkvHzG1t+Z7eLb-A5NC(G%nGRw(D-Gi6Du@yVrB9FXiU@`8 zMU%2>4k9AlR-jOl5#~=s}hbFjXW>PxbG91Ce z-1zZ$%pN^NkLrzxbjTvg#KktWz2i{x7p~lobe?1+#NDj5Ri#x%+an^}vpQzPsx?f= z40Fpgc^6?8CIw1CkIiaB71ln-7>~#Hcs%SFFP|w(emLWWs)Y~-U2rmZ4gdYj2>AYTFlIi zRc7I@%sjFeU4dCtScFnE)ZgBo85ZVM?WIKuk~BR$Y>NlLdJ$D_3lrtKtd1n7-qXBi zx+se*ndFuh$*ihNZ7|jrF$z>xrL5qK%h!+N;94g`Jj@n3(CmIByO6_A>^xPOiMu)A(~!MXr7}H`L0JsxB3V>FaLynQh3%)( zU=Sf%7AeA%Bh4lm3P1}YBF|jgdb>o1Y<<|s^gv-YN}v4@jrh3P<~}0ScFhkRr*roa zYhs6RBT4f;0>bOEyxeY!Hcp@{Sc#N?O7}no&QBl{oJeHvrG=Lko-RcYU<>?l`|5u-MU&;%y>ficNUM2)W|8lN(|nH=MMo=XtSIou=COZaRI zBykgEDI^R|o$8usAo~w0*4AM^8_6dWjZ4>hFR}p@71#c z(Wlo}Dd{$VyuvBs5KCZYKrAhz0uJf25Y<{u;^y8dINZ^E8m15*MHHP)l?e76``&xE z$K$a*yO$gb&vGq~q-t$#2KUGcaX4Dv=WRI_(p1g-pakgb5KNw;|~gT4?3uMH$ML+!&Z(qz$+LH0Bsa1Uw})o+}l*ya5dh1bpF zx%w1;jR$m!E(6%_5T|6&^mI_vbb{1|UQVQ4=k<(qZmqkz5T+ z03z&rNj^aY%mvd-B5KN`hm9RTF>l?oqfv9lVJSf8h;mS%urBGU2JO*X)2x5w; zaVzi4mYIRbAdgvJ*M_;eM#SK}2;*9VhqSR5-<~4fMiqypWt(PI1D<>8OqC+CVkI8~ zy)i~2(xI$tQ;e}WDa%!t&)1i$G!2BA0xPQ!M>=JlP?O{xF%{kqo!IO)Dp5~Dm`iK5 z`P;g*Whu!+MvgH6ZbD7OA|rz`eDsk_0-^;p`Lq!}x>x*TyhjNXxIHtQEHqX#;M3wS`A*ssJJt78rSv;1^B#0=|`p9Cm@fFz(7h8X%Ml>qN>ExtfQC}XPC{Y zH|2ewzy}*e8ZY7W>S0W76cvb~TI(?xA&DR+Q5N9>M^;;AR)+7b_TTl~J%x!;#xI8; zLmu&chU^0g&r^Zxu;M%3;~%^}o)hO^g=%bSCslVi z?w)R*aJQJFXNxF^MP)5Pi1mVk$~4tgTY5a7&;9w-#`!(R*ggHwI4u5j-9)O2Y>s}2 zRU`-=JPLSfJrSI+ZeBC_PT4dLk#us&vcnxs&@b=#N3_2t%@xLaFfyGJEvH<#00Bfo zSt8RsJza|>FT&b9K*$KHfeD)vA>kq~qwj^3SX4!1z&(a=t3F!wzEy*QGlH0kK%Ys8 z0^!i)C!gU-xCF;Vb=$g!n_0JsgRVk8xtUpJkYuQ=+8(?iqm^4QB9lEFnCfcY`BJp@3btJxluVx^A)13uVlE=I2dw8P8FU{xIm1H2 z;PesZ{`t#mH7=-<`HV`6FZxpbJ!H}GWp(ivY&x{>d0K}ag#M3A<8$StPyQ{% zktih5!Gi?=JZuIx$)hcE&Mxxk^_=m&CPHUPT;AIS0;vBRQ@XQ79J&922bkAK9L!)| zRP&PQ<8+Pq^y!mlyuH1tYQjI=UNWYr5D~4*wYAAUZ%xLKYTXurLb%@AGQ-V%^w3z_ z^7{Ju%j=ik`{y5idOn|TkNf@k-1-jivRrmAOJZ7-n<{|x=sv=HjM2yDeSbdhd*7JL z(~fSJ^->*ecu)X2_MQYcbNAL#8Mbemy^Tm8TBFF;+Im~Bmjx8=eLVN3d|6AfdJDtj z(fjtz_Rh@OWYI=bEX62o7MqFQyFIrthK=s-zBEV|R*_bn(2_L>+SaW(+{6MeOZ>} zcK!7F`AahW_Wj#)45!PzKmXVN{?ng-{^jSN-|joFZC!7lZ_@VXwryL`bNASNSC&g# zw6!1}iQj(vdRulL8guEmQ2Ji<^iVU+uDOa)--{b6QAwFo!NrYd(c zbZ(_zoB4Ws!J^?F;l-#0@a*>^>*0v#eK#AW>!d8auxi9%vXPO?(u7#ZI>~&|nHi4T zx(Y}Mlo@-t*}gq{2XgiB2Mc9A-(Ws@L^ZM1KDH=j&B3 zmzQO^T(2*$FQ1oX1+d2=ot*gVBY*jN|Ml(LUw--ZZ@>J!5AV^guYb6%S5n=c{rmU6 zZT;=ByL&Z@n`kQlHt4#n4gqt9XHseG42wN>Rc%s|GL8(QWJ+oaDjlyuPF8A8=D{E& z23ZaS;3BG8?i#})Y#xLKE@{gd?=mx|Ay&8>vXG253GhM{1_3;)#gx!4VSNSU)Y7*UK%3WdSybg^Qsr zEX^OA=|IXrdI^BaqIZCcL(#bd#8XD8?9N%}$d5 z5>aL$V&N7uog`*yCm+JgM6}ofz%uIMAhR#y?va5^c$kg!pv+USAu}^5kU2?z zC8EVdgb3TM*#1OPDgy*^ATvhyZR>sa=W~w&e0y!~2Nr=}HtRbREi)7+; zQQ`(}AO-2p`_eAOv@GI#X4>!?z?${ZYkWu7wN>(4hNo(h~NzKK+t5|F&|Wh@87@A z4go19?uwG6OdC9j7~~Gl1RObrRS`OC>S^6s{v@QTudlB^{`lj|%L~=4tfSTy1OW~Z zNz-LnE-%-Y*Dqgwe7SvkdHH<3yb#G4qvgxf{QLd+w_oo6pTGR|&wu;tZ*Onitt}T` zM5KZEx$R%Se!G9W`>@KnN^Z^AM)8a8C$;49j8NM#QC6B7kmcQ4--S!ljw$+tG1ht|TxhGh-M4qTs@VA96h-J$+QK zzlqRAGgp@E{tcEsxig#z1P79MiZSq^h!^hFSU7Odq^(s?jG32#4zeWoofA#k+LlGt zhY2HeZcipsWg-tjSgh_}6$%NuGyvzB4sG1=)*%?!vG?Fv9#?@6t1(K7Pl8GtI1MYAU zDf_C-FthMdgyqA4ZI1d-AF-tCDVZA)ZQJ8)%T(WqnTb=lWhT3qqz{;q5XGnjC`+s= z!ksdMXguvbvn)wSqBgP&r3h9TYytP&H-9|#eILDBPQM2tikinW=ZT81 z2UbddV+`-3pqZkgsw&DypQdtucTYI7)PUv4fG5}~;T~>oq#(>rtbRP_KlW1oyT7cCcMBk}w0a|M!bkuj| zA~i-K=0#;)*L7W0b$>pq{1FjpWg7-2VM;UieGBu8w%hB=WogQCy?mZ`41R=Pi?rwS zUP8S&rzlzK8|$rB_BvL_iW;%4RHL;=NcFZ7(mMm~~l}+x6vk zy%lbO655v5nl_>GNQvkp68=yB<6myK*O!-1m-TwNyf$6kGctP0Uw`@aKmYvazkU7n z&%gfs^VhGzSZ}xW`qH<3+qS;#kGK2wc8_7oa=E_97=7C(KBxGIz~6rRy0&Fmma??2 zdLuI<#lp+q928A>T@)meg$&6W)SC}YfjI+A0v1G+Z;`mR%&m7bui@2oU0Yj4NFu&| z{Tol|g0;0q%t;wRR1zaKMgqrS;3NVr+zq3E#3D}YYojE1kC7uMa-PB=?eF)LA9T=D z=;qYqA)-=46XJnL8$u}KUnmHy#0Kj?1E5Tjs>+FRkfvzv5jHc%)VrqX$%16ILrGsz z@`_*(HST?qkcFvU+7X_ru zWBtJV15U;qq{R?WIq z9F?4&2V)*2#(x~e|GRF@fAd<~nW2C<6Hbu{k2GCG^m4f_-`{c@`?fzsP*2=2%a=G1Zx23HB#whc3Uh4WWnPsY~%et)VmmmLlS+C1-5pL$W zZO`xDzkmOJkG%e`|NP(o`!9cf+P1qzV`8SJJ;(R$?fchX$773O6wwx0R8km`*=Buq z%x!YpcXRJ7{g<5qQgG$hf<~Yq0 z>OvwV1O+fj7N#0tKq!kG6A_MFq}nxy0CmEXBGS?*Qw1c<3eEzQ#`U)3HasINK?tNp zc!V7WM1NNqS6{K#aP9U&G;3)Ti%MHHg2|on=o`0{D3zQD4j%+z5YeSA1Eo@iq@2q= z-*2smFtHyR>mSMmX{`}x2Eoiy%_o(nEFz1FJFlS;2sfg#$;l*Ql42hU2@swzw@jbI z)Y8%_G)SS4P_1WAN+L(1xaTy%ha^nPqH9|q7+$00F`cUf!AT?@<>k_t09EQ!Tf6MDD^;&dx!j?(1o8kbG!IwIq=;1ICc3tTRiXsF(+>=fa-A*As!g?d zws7oQ-}f=bFf+F~17NOkSRAu16tAm*FUdRWA(=8O4ZbX| zpKmX(x0lz;D#QgJ8o3cM@zFx0Y!D(^E^SuvaY{S`KzosJjN!eH+iu<5Eh93SDt(xe zfjK3r+xotrnUqr7eG10SrdehkVC!-nW96^O@z=lp{L9b3ZA8EP`s?@Soto;)vKZ~7-}kr2v2ZyS%AcUfc5V_DJO+%ce0;gp*Vij)8tI+(p3B36zGFDj-ipQLUb1Lw!L7bvg z!NoVLbE1yon5fTCUGkWflM+zF3eOZE!IVTete8pZn9P@MMJC{7Rjegd7t?gGnmz$2 zi}GdFm)ml^wcmcb*cc*QsFV3)YYx;XBFHk7K?&BXGpa7JKI`{L%UNM4Gq2b6cDa6j zd3{+|B~Eke5l$>q9D$Iea;}5)>9$4G>MW`*Ro_b@m6-W*xvk(5)Mn<=&4{of=9Ef! z*uD4ud_KV`+DgY&q+OO&T`Kd-KoOy?uXf zTYoc5HoD!OLa6T~{HPkCL8JZnYCc$`!uQc($%hRU54( z)=6fiEw#L=71%MssCdr&Q^7S*~?> z#$j$*+pVVVwd42*wGq+D6U+%uj|^~b>EuwOg)~(>(wLH!nsQ^xsOzOESdkG4k7^;7 z*Lg&Yu@5&QGKxMn8gf}zCMg;8*3MLnRh5`6Qbd+zSw^$&SwWfD4fq&Bog=Gkag;D|SROO~Lpw#Jn6XlmNG1iyEa*@NU}nXF zWYC0C645!Nnm4~1?nDGasUOQBwZa9%h9f1hESE|CGKZOGm{~^fG96HvnMpF!x?A*Z z7qf1b8O+qcR+*hy5v=g8h%h^I#g3mM$;a0z4nL`)%VN5!x;8CbZ)=U2%u2D~=wVQt zKe=w#i_Ecgo&(i0vj6`T&_BNTaAwHN_dJ^@qJmNJ*=pETADc46tb0al?RL8@i?-I{ zU@+E$86IvP?kwdil1!{pz{FwJQ!gmaVjU&KDwWOAY|AhO!J|AO!{out5lIjsEdU%G z&*xKw*7frG>Gt}1`}BId-{0JQdrWmY(dwe0x;d(!EF;p*`tWYn_q|6%(@#ero&;-E zQP%6_vT9?<*nPJR5nRlvvZyOg9RY{Oy4kc}KRN?hk&hQ*)`$|n!dxD6L6?k5Sw&HG z@W=jc)KJ`w)*7?^;in%W%EqA<{y~4&K_u7f`sK@)AAbCNyo4F@cqcA)yEa-jGzWI>*OXkd6wGmNLj2MOHXu>J1tUYq?X7GeqaDc+|<>e&~ zDCFE|huA|+Xea98Xq`b7Dv@sPBg{c;1MF>~WU!ZMGY`fhj3($)-hh}h0>^>zcl+AR zh7D$RuBYkyz7sg3Y^0P`L?o;8NyNw!2DmJ349^H{jk$3)kZNn?r%|aC)Os zSAw=Rj`F{BPv>zK+36r0<8WooOyD}Rj}tl16XR75RUn^MP5_7oaVRGwv&?2hnN-Bg zgwXGOq05DDG~`CF<2hD+0jlYnnVu}%2+cz|1)15Sk8z(jWdZjG5k;~?ddyRS+O#9V z*?FLh@W_aGC8AlLPsfRfurL4Pr(yfn`@Jwpuh-9?ZUFrGGZp!aol*2+;lvoO+87tJ z_5SvFy!mrCckG`Y>t%htUYBL5_;n=y%*??sCZZKU)z^#uzJHnPl7i|Du*4yp&vRu}ZV`tL)w*B?zZRPVln8gfcaP1^%9WB}8POvSO^>#`(Bc^pMg&z2&QuPjx96s$s*QRK zk0L>X2?7=tP7Cxgy7$biMs#_t_`6Ia+M!v$tnB;&QnV(Od5qzL1J$+Snq3 ziC&wwwlbnW%`uv=EbZB|#~#5?+dR`fFd*bLX10e1v~eqe3Xc&Z(;@;FePua}Ug0*~ zN!ZaNBdH)}Qh9n|;mk+z6SXu}1wrBaH4AzJK$wDHaG>#v9H=lPD=$nO9@y_Nh-dlk z5@X4T$c1C_xdW1Gasmv{uwN@rVo@+jN`Qmdv#gjklfrX40S3kX*D{Y~;z=KC-o(;I zHzQ7MjhKsk*;>mzgMt$xq*>B4t}H3liYFj5JQ0u-;eqCyHn!OPvbNjxlTd~m+?w80 zh|A&>i4}HFWnD5iyeLW9Ji1HSzYP8bx z-g{;WXpHfB`GkYT7gQGZTuQ7Xt@qx?h%}VDkZNhv6H!dIvm=~{^ytyhA^Qhm7H;zH z(OOs(W@grkq_AU@1s82dDAL+EGQ}+ zG>WPraw@~s+@ASRiOUH{h|NH;kyX_;^bzNRB$ux%d-hcV!mu=e~kB8aBkX1wPbpNTldQOBgE<{O_6`FDo z?BEUo5w+GXgx+@x56ic=w>pe#tbKr?hS>nnd%xfB<;_%as#{VzMR>T81|gY*`Et1c zus#5g@p4<1%W}C~`u$PYR-xtO!r%jOuU>UXU-dG^*vB@8MTA#i0^e>obl)u6D9JKr zD#b~Zi+V&XqO{UPMA;lEpBX@100SaKkV&@8G`Gmk6x+5f4c7~|wum(Mcs_RWI4=+I zOfs{GQMlv}W3ZaRfVo6;r?yGCA!ezWe`pN@s2g55G2VR+3e1Odq?p-6@l#1?d1j^B z)6JZSJnX!as#-t3HpTmqCLhPY=iTPV<1rt~wDp^Ij3|)NOj(n8n#GrUX6ZCu^m17^ zGrNy7_|)sHaRCq&aF|f89Y7{2Hvm)mHK*+ZrpzU*7IV&?iTS9^?PFA|*UQTb@B6U* z9pzHudKM?+bbc^qW*%b{ZRz9k6OrMgT%2ms+ATXf%FcIO&8HFLxqy0i-^Z=RJllmDDx!kBl zitMMsF(&~+CmA8=416E|U}opkt^#GjolwkDGb^Y?gEj5t>Lo|aF4O?`}g~|-XAR0mIxLIQ_+~~*f6tFGNQzf|Mb%jKm0%ut+h{| zJ_YOTcFWA4fBEg_pMU-SxZfY!ecQf$`<`HkG6NK2*!S=E`~Cj*?biYc%q9j;ewmr$ z{iK|qsP~?w>EESqK zCL6!|=VOPzKa}dtO(_qNh^7Q)ejYZJ%q&8>G%zW+_fCvObY0g{>ozkV7C3lV$0n@5 z_1DbPO&SXl5pHQhb1Z0)&zxD#USSW(50eyP=f+gbH^XLX3nJgCmg%qO^>_ zKzJSr5plJn(3Y0PpnG3ob#5Maq_}Vaj|pHX7gS`m5UGlxeH&)!o=mc|Rb?TP2;A@A zBdm|DfG(=44XR=ptF~p;wn)N9$z>3Ie|u|f{rvg!KmU(^{pqKlY7I0IOEI%we*5h| z{^NiB`ENh(y)#Ra{P4pMefYDF=kwwI_Sm=k{r-GD>EO-AcuF%q-*iw1^*WA;hZ+i-HBI++$ctC95_S5^&QL9Tp5UmZltm zNETK0+-y)r`2z<6fGAyfQPoQ4MH-0Csw8 zHmlq>n7JwPI3Xe;@-*zGecvPAM+@h(@DVeit_T`L#E4vUtuT_)@%X@u(SxbfxcLmg zTy`Upje;nEtqy2lWB$w~996usDHvHY_;vFfOof3DPeUbq%GFg1@`I8&te91iL1wnw z=%a_*@fig&`Gx?--a)vnmv&hz9Chnsj71wNO`E-p%#3pRb~nr?NFq}Vqc{r8;8Iks zyj_(d&7ylJr=tFOc@B6LG6F#BuGYH|m&z_>QO@!Ve0e0|0OG$-)sI*5=>{1d_nzpFkB1Smsd-fD3!^UIg%2k;Sps zgk`SlWB<8lIG8mP;Ysga^2kKrsybyBjY!LBqVe_XSNCDQgKP#?_RL_IwsyU)%xq@i z&dlp_z1?1a_~C~ypFXQ9aa)#UjPdpR-IKq4`}U_l{pnAC{pcy4Ij9 zGP~{A3&b^nq1xJuNMw22Y-5b)-q~|)EmzReSXw5`gUqhiD{EuXCY%^aEDTPn3U67K z%sh=-&u~98m(qy^0Q49Dl4)WknM#h%5JJ3)T%;wLM|e2JNQjEGCT6y6o4ZR}+9DEk zR18F{b*L~i$oNP!7!sTu93_O`F1PjC+M0pBTWhi|SK%>}x@voJN>dV!f&e4(KxmT? z6OUuMex@~b8B+*>XXhJ-9rnnfjYUB{6T!tEAY?d#Vve#b-5^p>5FBlgS)Cnqa7Be{~PIZYmMq>FMaJSs47}W5Y{4x`9WVVt6i%j3F8zLnKm4_y$2Qr-` zNr;6aYMfl1LwLAH7CoLz|3{TbJOl3Ox!dU7#uye+*;X~>%-+eH+TIB>6aau=XvdS~z5B4j)m)bq%i7zTA6}7BuA8C~S%)DepNLQn_rMhUK0c|`ff+#& zhH3j(&HAcP(kI!enxRu_+H;}>v^swjm53dC2FG-}!hYqjXNC*2aQ9`IWof1Ap5d91 z(1UzIn2IO!S)?*CK;g8To81drGDdeF3D=gYB09_Pq)4dy$m9rTmdoXGx!nFA|Mu@c z{`dzG`R&)Q``gnl>F#g$?XSOl`~Lm=*RS8qX}!EAle_nQZ2K_p^)4mg6*(eJsL=;L zb7G2NRhOKIP*$|Z`YBdD3U~le8U-a0qH!W3J&_n$(XCw|NAz*eF)WRawi`2PQ=!JX zkO*zt7~{U}!^nL2&>`6f9EtEx!TWHcz2K*<>DrcM)qUSj-2LB0x3ia;Q|22)R2@Pl zN~-qjlmcJ1sSvhRe+GcO=RA>8cNNCLfo}rG8(Q<{a3UcJz|wsn(%>O@O&jvUvx}>$(k2>B+BCwhveMntTtLJuhsZji^yhLE9XCZNvy3nB_6;2s!*3ZdGM!h?op=3T8-mbR?Sys!jIf~>|(tintlon^V+)|cz+%k9&;UN0}NBmA*V9gOGpw(tAf z+uNW2^7Fp$W4Q1_jotmZ^=;og!U|jzqa>lyhPy3t`LII*)0La!$V%!DXD)nO1Sm9Y zmZ?S=R_@PxfIVYGNpVm02yRjiJ)qiXGd7vKvP4yv1lFcYCh_R$LC?)ce=aO5>%!a! z&%FcMgj5*G^`>M&LT9)PY<>m9|&)4gfB1`_3nN`xtJ`u^Rl}8QVS%gIwQHz@H zW<-FfEtgVwRaGrR^xB#;6$h%!sT8)>Hi)9S2XQW zqr3!Jq$xAAj`vO>0bvo8Y3cjEhmDLl$E3aYG#|_IcWe60M`~f6ORb)(lpx&igZp8J zjYEAc0IIwjA|N8iZ1E8j7{u%4*0FaBa|;U28h}8tmYOyfT=#h2c@v^E$|@qMFwA~E zgL~PDG4s@)`%t~k`xjXjqnT+I6AuoE(7Lv^tlB^vW<9bTta!P!8q8j=*B^fP@y9Pe zz1%*jE)jU#pZh*C$pXLq_S?@t|MGY|3^FVuSwL`9RZw~Z2eR7}0Ohi_1x;L!*>9#7 zRc$`If*g2^GTJB+pLJ=~rq5Dsh(gWgFlSNX3HgZWXI&zTB_|>un_4CPk%2nf%Eg|D z*5#ACyLB7xl&(qZ!Ym{6D!N{;FV{sSGX~r?w;#TIiHIU?uIq}T9q#+@+6zRLA^??O zX1M?o5egqJGKLdFS=QEG)`i@1bXa=;B?6b3O-NJ;1j?ng-g_Ex`LIM-1eq)Xf=9^- z)b^{&FR)|ZpR2z*FBeFI+t_-SbMQ-<8N`l^i~yI)f@k#+3wOpBZ)1#J)h$azcp}Vo z5lW<&hdn%P?45}CQ`NSah}qgmSj2L<0=~NwXVJz>nZGjVvmcPtD9$57ginC-30vU9 z=`*R=34E}QaAf-AIcgg3?!J*aE>F|3DW$W)FYM_NQ; zibdz)wOB$%(wgeBcQ&(qAGNv8Mg=!@q?b^EN?Rn8S+KTijcVe6!(}Ej_I(G?7Fm}C zfH3#I188lwe2Frqw{bGjvg+&W?S~&ei^${q+vD-JZCgaJGWkfiq*i=XD`qnf)*tun@z}op_U-%k?`9#QZkb>Y>%%?9w(pPU z+rDoQRE?Q%Rz6x5aUP}2ye!L+!5;!9s;XK!U}lQQ{(LKo!ByLGyJhC+-n$V+W>L$e zDVI@oMIIcc7V(bb!r3_~UzZ4Cm8J_wdhgvL;Ym!QN$kMZZE5>+7nWQFOjR48a@9hH z-0%0gu|?fF*K>t{!jnA5b3SNhCM6IZ)q9!~$)XzTf^bH@TrYj^Z*Ongb5ADKrLmBT zN*f-x+pUF$SYOxmMBfq58+a6Xaa4qH+}k?E^SrS0QrG3vzVAN#a=8f0wtq*YUfXa=_sgyljbYE{{qD`(t4ONMskNqU8fjIls#-3X zwEU^dUDnRqR9T-=60Cv3j27&=QLsh%#pz0v2mnnfX#&$Z=RFg;`ceD=B#sZSvmU(NZf86? zzbzt!8-a?XEZi#frYTUtwU?8UIV!jxp@u&;l4J-Rd~QN40HJqpPZB4%b5&9atQR#oL;+vD*Direk-<;$0!{_y2?yS=`AR@KMj z@!YyQWzb)K{m>{wtRf{>iyshAdCbuamBw~?zP zAumhoDxJauF?s|`rW1`}37!mk#N_YrK?tZaEfa|%G79sYP4!`Yl%fouB3V&DIb_V( zIRUqXx;&=x3(N>7K#vSMvcv+o-VrnHBR#@ayDb0%cA^Zz92zIG?)|r=9%|BpF ziC9F=Hb_RC@!EWLKhB(&`IvZ32It2P7+FUclSrD7lVAmdX=M;Nh?k@#QZrC{XV?}g zmEFySL{ew@v0N!4-J`mtau3fTo@#AS{2^4a+PCfMVQCyf~^( zs_yU7YH3xnXlCw#$fxg=?4Cpv!-=H4I4nDBTSP@9gGP*658=*PhDPCN|&92(41oc7(B`~CYxwdr#1{37F&a2OqL zF%k0249d#mnM)9_QV~w7zZ_Wl0*ewNOGgQj=>bsvRht}bk~|R+WrCHNg)}cbc*T)Y zpx~0fX9gP5iD--w=9Rj(dAj5wy;mA@@6WrilE-T<+nODz9wKO%gWT%LEX(5VWxizA z3z% z^Ur_#^|9~!HpcJ>c%(&v1%`QS+jhU-xBW53h$wzaCZg1V3W8k*qnRls8eZK|qRgmg z!N-Z~9$CNY{tB0j0~?uD zW+Ud-crL0)kKrSHSHbnxoY)xz;>e^Rk#~cn598BwZJaCQ{8y!kGMq$*TF=Q+zL=Cu zSquZBmdV$VeQ*25(Jc^RoP-!KbFf5G(qZh(%qm`qKu+Kz@l0QkNY1cY3@aEyAkJa% zNajQIHRrWWr~@k|d_Fz6MCPj8!7)lbQ@`^j%uP`qq@g}JM=YRn(<7gAz`3>a$hPPm z3i2_yi712ZqVztD$cc!-jiAE*fw%Rp z3K=o!K(z;RYE&L^=RGtX9|Na%A0PiKKYaZ?sVAz@qGRShMK4~~b+NtkAp}u(L|G=F zC3E7Ti9nG+!P0B-7Wg!AU^@?_Fy>Sy>eBcewjv0iM1{zRD5@SaBRNsM_>73@n?1=a z&hR>7Dp4Zhq@+aHhzQD*>8CLQr4Q|(IS=`8s3wX8mkD0#HqBor?nxxvyZ3$H ztG2D(vOq+5U3I-kYc(Z~jPS7g{rec>)2C1W_HY0C$3OnDipS@3yWj8I)>XCX`gm-= z{rc@sfBkDE0$>pq+k3a&EmiftZ@q6@e{TD3W^-T7+_~5vOvGW63)a?4eWW7}&d%6L zM~q{tRvX4aDf3pi#|HuwB~vO0;+{gz0DEelIda`Nso_uzi(xT>nU%T85)mGb2(GPI z9)V-*?(Y8R`^#8|n}sBzQZPc+G_`<$2fPPo!F#UZGuSbbK%8JgbqzBq-X-foL^K)D z1QRXG+rFxlO1oE^DYC%kSplnXSjTf$Rw5!nK(x7ln&`sR&FJ7nRP|h=WHToUKKwFk zht-2X3g*sIW(pZ*kvU0Ebf{5QpeYRBIrgka77_jH<_Qm`NYzCo%t1txI+C-KJ3~Zr zOx8Nix~I0ozVAe{jVAhT4eGgjx~IofcS$gj)ZD;5ElWL^8JtNWDa@=S3`u0;GFF?a z1+1FH7Bv_^M2glqr)n%m zotGv4CwAa>Gofe1$(*RqDz<7n9gB(1)9H7ad-D0O?`Oafnt}JxD1iVez+{b^Rdrb| zy%l8%M=hCrUP2Z)5JW_rs#<{ekVIyy`UiXx2G;`ZHot<`}6D9-^^TE>*LTIopn@`j~B*iMl(PJL}H_R3>1)d zNK2!1gMc(hhvW$98r>bzNJ=+It0PrP1e6-h@BRKboWtKcyu0tc_j&H;5iIFfnyk=6 zJzSj5#y7Cme{oLEA^!B79x@YieUX?vXcPWZM`IBV7b9XcExvXYcBK^d#!b3btl<|H zD(d{U`K?jciWTcPKKbg0#FRlO^b^`A{;pLDEl={CTX`%+_@|^8FV!NNarO8585=wW zJh}0^Nh{J{Ey;s8L@Xtg%fNPB8>BjqwrOEs^*CzvXN&k4I;hIdLrL?#1V|?d2@%?x z#meOPoRKwNq~;N0JxTKT%YcMsxPndPbH~GAj>Yu8$K>nOrcB&>dE0-Lys6Yr_+ZFU zhKTa(pFOOoGf2_CyPoJtTnRmTToWKxL?&&*#w@ECeEYp(Qbwu0cs>`JQK;~^E!Lbk z-^G0VEwSJv)+fFHUhjeYuRm(+Hy;ImWQyDhxld4a6{UD8Qw!!l5l#^n5A(PRb{!YC zfUq6+Is64%|^xHo)Q>C7=ftL+s z<}RUGI#dpaZcF85-I?} zV}ua5W*b3d0`L9-g-_=8bpGIqcmv(mMtp@jy}0u^?>_cU$Yi^4bzk#;+O%_WcOH<5 zB7Yo}^Ze!B2kn}i0}>&(RH`K$aQ_0wm22EK*fA)hPmfm_Auy-=0AfSOQI%xx`iK@L z%#8(Y`&$5tLAz%U?5HHsA^OL=Dc)Bkj%!z~&}?;Dg*SHnk}r=|YF#QmD(;m!?EmL( zpAj*!kl{i5TGT+QZ9x2=H1zsL?4uffCcG~i;bRSzn`KW28>&ev9wA6yf{=(TWFx{5zs0wPDfV6s2cT1z{`ZL-M2(8s zu$?nb&Qnuo>7Xl}AIiqxLiDS6(~k9ZW0g)o8nt$h?O_|+1KVlg&2NoCMJepkQ7Le# zO#HJN5l;1H?c5dicHWr4U*TQ#E9;;ShSi@i$r=jp@#F`>^NYlq;@G zh9D*%vOiq>_jB@hnL%WP;7HxYg+J5tj~;e&l7^ftYGq+Iy0(2ade~sC!u?}cP!g4w zc%;1wPK=sG38&Ux2bD02GMo=R%U7Ecny#I$`g`OehycTgSo{P%{s?<<)Oo#YHRB1^ zb^`b!UU`mG1tN+HGZVnUnY7@-$A4#Tx-cMUr>nhXze2gQEKbkK#A}KzHsK6ZPg{bp!W${#gss`)V#)_9vDu{P9?x6Bw6p1&%}h5<+EuW@A}8hD~dm_N08@kSgR?F(pWiXfqCk zr>O5OjZ#h(deipr*3HiSi{4o)rQ$Zxf!M6rj<)#p873|dIY_Z9j2#X7!KX};IN3zV zxc7sJytghx=<#!~Jz-qt1a~&JfcFvKH*KsOP8GIjyy(flt~TU?!4Dr<*%8ZA5=A#w zRlPi_y~^Lfd)Q*>^M*P}1~UBnLOY^$P2&KdQK8EIDQ|0&!wh) zbsF5=OR~r@FMUxYANvC~Y#@+dD*=Hp68YOt1jZqbP!l&Qa~3e^Wk-Hc{|IEG2G^n0 zbkAefUowXZUtEG!)0gzy&e31u!-^!O2fZTfZ*84LYj~D~29kQCt|9nb(hk4*+!Vh1 z$!yMf-SY4N3y_^0ceRLrRwhh8;`NC2`XiJ$spQF`DT$6dqLrU;b}IH~WC|M`VJ?w} zc;-1p-uBS*!1Y$UBlK2>kcVSqY6F{t;g?5C#YO!cb!I&QcLU6KfA{9g{Y-%eun9ag zwB8Cp?DLI$OqU#+T`>SF;=^J3>M-yKt}|hfO34-7RPoOpYNY^S9n+o%-@oAGdaVX^j z!^p|W7exJ(9x+HtPMZ=%wojp&jhu@uD_1>Sbs0}mu;d?~hlR`1|c2nw@| z@NrQ*X0MDNkVn{KC$htF9F0gsnOhYZQRJ;c(!vUza+ZE~GO7 z7MbS!j>_JLz$X3I!XI^7(@VO`3q}h4m&MDD?{9>gEES~=v$lt@pH$$R+xBsdJkxk= z^}9*+*sf=<Ce6`x_F!uz#ota95&n4Lr9{ze10Ma$lYZQ^TA{-15WaZ%&VIL2 zizP!|6Wb`b&6iWMuMe>`_g-3SztFHY>|Trqe@I`h)(YCEpsEU+GS^RbH@B%xE^5Sk znfSZEqnTY4>?;Ng+lC>?&m9ypk8nmBlI-6RBIO@c@x-!tJnB4iGVmHaEXCy$f96e_ ze{q&`jcXb8&?$+Ar{Tl5moyM!3fprbIFx4^M{#hswQj%B6q`Y6#1*1#Al-d;5g@(3 z*5ibRHf}ua;>)m^6&^iYg=XUOy_xjAKMU#&zS|rDcGax7`7Zku^@1}_`<zhSAI70b&?W4Poy&4OI__-F^E^fAXf^sWx-5Ubzq!A*mAM^w60`|I}CX0&8^CbN- z^7cM98i161i|=9;iUhk+g~A7+fer)sm98;}7&{48<OR>1yYvEW`X>okcNcR@&)3)r^KY?QnCV( zTYg$6hq7gq;(1mdQJkEjMXj#TW*lMXY8_S97}u|$b}rhN%*7^v|8Jk9zM3+WD6U}N zh>ZeDq`y?=#KyD~mE3omUobOmb*l=sAA)DDHeMrpT+gacXpVb_l+E9Bm z!vyYXZF*bPrO{4gwrXOKC7@FEOowwS)>_*!+T3#c%dC^69^j+bZAh%&znU8P+0(Ms zYHB)lWL1*DfXfBBVtPOiuc;(vHP3qSCy{*Z8!LoIMLvQ>lifnHjj*XGnw^@PpXA}9 z_tlyLxU%3ufOrJpo^Jskhkvd99r3~+a0Otlvxo04PbfU?HTGMjA~KPZbCZ>iyrFtQ zrT=o&oAk0Z34Y9p*)&K)+E3rUHZpQq-ap&9HA0lAbTE%ki~X{y*p3L!|9C&JY(VgX zE@6SB*F`!&X@4A?Ha)OHA^1+cXu603iBt$uq?S4P*CgZda`?CZ zYdS*sLDxb;v}EAtiToj-+f7M8YP^W?HGgiWueva$vi$zR;Kov>95~h(VHgVCD6JdH zQtQ1t*a3t!bN8G`x20n$&wzWysmF6c(% zH8h%;vD5x?@{n*-khgb2(?D_KI|BXo}fTO~=GERLH9`MLr}jHiPw zO)?6ksIA41^8FLFK?8>?uAb!l=#KbT9FU=CXPREh0?>RWhd6#-;U=bqv~eK#A%p2C zqV#tPTV|QPL}^R$eR?jx88LZC%sR)iq3JYld*4X#vd3= zv-b^-c}l*47BOTiSbvkorYw~=7%UXja`b5fZyT7>AgX>Rk&K2W{^`Rig>EDjPNyCp4d8`k-5SCJW>7%94LMDmr}!E>Hg-#Ov(bQ0qKITJEVWP75K$Y$gEodbN-W)})DfL+ zI=-UsV(_@RGV7)){j^H3?fFV8cp9?2-?H$!IuLssa@y`cCW{_uirhRU96ZES55#)aR_MhGJC<5iSQ2r4SDwN2R{3 zGJr_mU4Zs+yGH$}lVFk%FN3dPATX}!+6;;L*7qAX0lkO%lwlrb)LLh1njck!<30G~ zzLz;t5({d&KG7~Uw=OCv z>d*a?-}M(uqtGC>?1*bu5^+}y+)47Zar;&Ld}-cGTfaEScq(21%~z%^ZjiM-r5HBq zy=NAw0~VKlnK0anWol9ttpYHv($^-O6PA{2Sd*3vtG75ISotXjhD!xpruS;dh%$|; z5jj;3r;cX)&#g`~PK1>COK`d#%*si?Z+x7aKmo~nzwnQ}u_eq+8wtTvld_QQ9Bfaf zv{a^nt5&CL4=ZdPTW-SFGPu0zA|k>~um32L*5>@^eZVYDeSeZXJz8xS#fh6b$cLEx z+KO|}jPn|cB-1y}1E|fIi~l7 zvlm})N`5Zz&_Ni6+fEm1$;oBng=h*tN6AkVvr?tv9hg$pSFpup&roQ6OH1Sg(Zl*& z#u0C562-aCLP@edkvSQrc@Ij?t+pXa%TLvI)Rbcz9zueI&*C5HiEW6r@i(=$>JQJS z>g~LRL*^1sYO$=xHXe`kQoi&K8~ZplsZ@{rOl_%%@XShj=bQg+!6**K*ig6cEBqI` zz50tAV|vv%q*=g;JH_4U?)0#*gHDX!(1_Fj5Mv_+M`xOOd&gkj`_A*iNqCv5xF5Ju zXsUe*utwz5kf~{BcJ0e4r~^WENtbvk3(v${dIlPZh6b=SR<9N(I;f=9Pg&Wmy^+U` zq9elL)-n5w^x`(RoE>ty=#BzDUJ#0VB&6*)^Kwyq>jT#;s+VMi8IDPy_!*Fk+PgHf z<=dWHb475#*=4audyWFp=Yda;$c`wRe{$#_Uzp0!YEU(G)F)R_?fmn8s6$e*bhv&? zJ5-mNfU z$ci2&zc^lDp4mx3jI8~~Peucpo^8~apVJdt%81VswI4$IKg=!|`2-i?aZ}bIU5hY> zdA4eJw92}D zCE@G~^hp{>TqJR4K78r!<{eu&(O6R*%7nJHl_mg<^L_r>20Zoq%k|6q9bA|L@nP40 z6Fpq2Al7M)t-;lY!_)?!%MFX|zPLi%y^Bgnvx8RB*gsYY9F}FFCj0s}Y5wq7LdQNN z)61{Wsf~s_P;%Ek!jThFc_#t}vlF~l9(~*H?ew0WcJpt}9sDaMsh`FgD)C7*a%;Cx zP{*}0eGA2-Z`6{$rEDV%H_->D2t}4eb*4O&(RTUMj z=E%kF%r%r2yLl=6a`#&gMUHYASDBiVo#>&$HD4XXjcs!E`m6tX$m@)V@Mt1eF+S>o z!1d)bz!ilPjogHhP80!E#J$^IN(Dz1HxFf>mZM3Neuq@$?psV$Q_){)ZEiIc_~!4; zX0HlpAna)o$l)ZvA6DD(F^;Iwtg!pQ*ZHXHsB=t`;=RCZo#w?R)+oJTrmY=RnEU$c zYHqwHP6&j%o1gGQ#g{TIJ)RgGEUdAlf=`ljPOUn-n0jg=*E`;L(*Oj5VYB4%bchVI zNbsUgct-=FarVnHQ+%LvHHYz5NJePubQ&7q; zOV;XHC02~WV?%E&2qMbebWt>5*q+y3r0IP2ZST6N-Sokh*(4``?XI{O; zm5r=hcD&{FSGJrg8k8D~A|z%t<_VfCMfnx1B4xEK(^FIM@v_3ab1`Ys&iSA0DE65m z0W#aamDd5sZ;@PcNZEcqkTs=JsY%^3+Rm0Nplx%1>vxY7bFP^o5%dQigDh$cQ3V-c zA-k;8wpV;a6*UiX6aI?44oB2?xzv>tJJIj(OiNezWL&tr0e4$ngXb0(Hp*nm_ z65|~HH+f<03Cjv0L?vj-u{n%|Z~hNoGA!+)!mk8ueoV^t;sqtbCocCTES9*F$hE(F z&bx@BO?rmza?^{zJ79EcM(7oF2=rt6sT{a1zV4-%Wag}4Cm%0Zi%fpytU%C?qwr4! zQGsXm#dC2R_ig-(O6>3xom{gq3c;-BmE1pl85L5%&$8oa*!u`%!*xHI2-uk%MG>E0 z?oi0=fMmaChI94t$`E!EcgK^r>=f`~NxKf^UBQ zw*nsk?y=P0e5adO`{^PRd^^*7V}O=`L=KhD$Xt9>wQ}a^U~@|Vn;QV}^T!!Z{?0%t z>*w;F@SO3iv}C2e3}Wv3v&*no6H&d7W5U3`(jA6U(?dY3KxN8KoilBFig(FXivh8%QOLKQW}fa+S}Id^_=d5*e6&3D(wb7|1~xFd+So zS~O`F6-3{HSd??Q6ZM7cT^&0>g0-Sf=ENWMRYo36T?3q&2(^)s;hsOp>JvDDMG&SEn8qE3YY*F~slsL$- zAj3<`1mmQ+ZDaD;iJ8E;gaPd%eTBSJcAhxvsB6nT#Nuh+AxzEfmL#fqNO(b}fsAbUjERZ-&R_sJ&WyEI zc|CqKqU?P^t%*w>O@ULJlefVpZ8!SNuuLFe_>enZJB`@m>v_ zz{Bvh+vQ^noz*!G%vEyz;~3|I`5B|^ytx;~2}Rf8?H>9+ZZ^-%euz)MKL3|LePOD& zPvqvC(5iQo|0Npnvb$9?$;QBtnla_d;Pm!RX@_F2U?)2-O>j#*8Fjj4=%CmfLy>zW z!DdUZN|U-Jkqd6+BY|VdO_o$fKhZ)_KdIkkOcG7#HeU1kCBwwS{ZPWtg%9>3*x&zp zE5mH+3fL67TWl8y0Ve_N*FWA_-)dTF87{Z*9|JNppv(X-^PY<19$@|5>vTOZ8PucW zSZf4{P;z?f&s;s?fesT|`_TXZu7DdX4deY$+v>!zE$|V2uGt z<=lH1yfoJtnMOVJO4)2GPNRev&srM8s-9uf0QguZL-o>axJPR~C43A7K+NdfOi5!eaza?y=bx_Yc3 zjeZVQYKS;jF$iZ@(i;=n_B3{^w+=nLu%?`65*(9KfBVkvNUzxA8;xJ*d8B@0dS<{*p|!Ey2LK^mYFINpR5R@!mv+ zpw~#`nbX&%g#Q2+tx_lF09_y;SUPCb z*7*eM6Xu!Fvt_h&86AZ|Nrv$IDrAL!xs{S@-u#XhPZZRdN$khI0&?`l1>K~v-A4-Q zq^kIEAIw=BKmC*IBz09d)r!zU~Gx4=Pj5 zSWGnjZFF}uThB|r9?#g{;_>??r6~XXD)=t={^vg*me)xu=>X8~w3Z?_%`u6ve1R(w z#g3DQ>i%yc?B|Br7mE~ml>(N#1D9U%>+`FDgnh5dk;KjJD^ZXNo1-n2jm_C;iIRTL z`jP8|8Y}JA+@eZBfO|~L`>>-IV}Q}p#s6qvQlCHa$;$27+2Sf@7W&1rJMhla{}NHF z{N?u3`(NkBS8ZlN_<#SaI2yrM^Ai)#aK%wO!yn9TA3-7MHr=%kAwWFhkd}%KcOnt6CCU>jX4E*^c=TJP06>nv>wlnze@W- zVCwvo`9FWZhd*MlQ2^ZP^ax(-*jbYi^Z8I@=v&58nYLb)LtK)YlL=5#qI@99rTi1_ zo)Ci!8*cp28~M4KlK~cbog9#mvF=itH!(kg0Mu0SiycQe;gncA;8@G2b(C z<%be%@32IYl5wpH#+kGqv~7!w*eL&P{c!`EpfzK~>2fi=oQQPvw`()-lng_(&cq3q!XT1q0wKd#{y z8#z8qs_Yb>^=`@4Opb2~a*Z6hy}rzPJPRhI+>Ahv`VvqUgzWBp(^vi?eo^#kmzy<0 zdHLAJ1*@aOuE=|4?DuJ<8okoR+l{YFm}fg)_WfIeDuZ}sNV-jV*aNXdEFAohi5#9n zYQkGLGX&m>oGwTmVri}9@Z0sdH?IxWaK{r8oSH`mQay}0Bnr9!IX|;+ge~nIRLw+n7rqA1Z1rkN%ddL`5dUU<2 z?x$#aXlwdiJCY6c4Pz%huD4TUw1RoBc)!)=f^H2C^e0ioQn&7IMo&ak(!yuO z{qOP8v@7_q`jeD-RQrmBKi;g_9tB{a7dODcDMCzF)O9y&@oh(0jSXC{b>HoyI53c% z?95w3K+NYFwoiI`53Yf*56OHC>|J8)mkvIBNh^*r&wbk^Z#cXrzdmDPXD0uCFaHjR zKRYw)4Y+JT;21greZu`unY+^o!QMaDOkk_mvWlU!-OXiAkDY3N-*htN;n-#^8jjC#@Smz(DHP~gzEipPCBtb(5YUlkhM-k!1P$uOm>U?U#; ztyvB#z6nkLlV^2^BTq>cJwhf%AN^=Tkd44vY3=^9`Pd>QC#+I@idue{ z-y4hZZ%>Z`riqhGNu2zk8jy>V&|@Wp;%F;HPuHsd&DG3I?T3tIBkqW#7$hHi;FPVk ztGD*3k zc_UxO$Vj>Nc;L~s6E4! zZt9H3RToZDwsyzajcZaH1soY3|G8c}taZ8FI$#FFy8MsgobfXNgs0#78o;?o7asQU zy?o*)DJ69aAd3t$wkU!x2J!)f(UJ5mR+E`o&)K0B=_>5Th@4S6i=*}&CdJ0s0p`lLS!qn3aBii)~pLp z4^O#&nYa(CQ|Q<=x}UmRbQpZPE^P^1&)`yyf&_%Ou9>;G2wJWO-*;DaP;eq*k*BU+ z74l^W2)FVL*ql}*92@o3R+i@7KiHgkQAlJ_6#=hd1)+ctL1-1o1t*oKs&d}~tjinD z)@&%U5Q9oS3V{+U?cw%qV(U4FC^EO`X6?qvx5vSHy4NH=(G*rztoZwX0~Ran2?8>| z{2D2Wd%Y?pk9s>f>2-8zR~UP=zru? zc|_J7rwlLH(h{TsJ9X_F)MgZwd4W;knYu<}epp4Vljiy+Qtgj#hE9X5`!Ov9X1;#Da2s{Ki?C(ojgr?DZ*{gAV$b$R73t{f)4!MVFhCnE74+w`1`PB?d}T7D(Rc0g#H zQ>;2`_yVo&Y)O<2ccBt*RO&9(GljnReE;-UJNdoOn?`5(I#Rc8|7>m&#&Q`N^W%y0 zFbr%Om9=#Ndm&9A<@rD4BL7*nwbnFH@Q`(6%b; z$@qK``z5X%VKqGqb7PaQ?C>+*ji+Q}qkDB;3k-6hE$IW1lrKwkv+4ZT)fXmdvvDa( z2F(hiW7NE!r%{0?8jb5;B`|Fotyhq`+5M4busl^`;G^ir`-(WO&EX_TfHd1R@7#+U(x@|JARvhv;6b|r(lhy5SE_ibSu_mjS>J%uzo!W?q&j$rr5k7hV(vN&j0FhFCn3vU94RO*pgp0=LVCR}SXAEVtw0*)0WB_$cTySWmCBXpfI zN~MjLw98G)keR>df9gqN#k@TJbd*ghuMD2=$V{`TZ0T2&D8NDdPWBBOE?PE-;DT!7 zk%qAzj$pZry&zL=rzMP7wE(bUePGmsMCNi|zV%iAeDz(lIBOp+lXf8&#!n$Xa=gbwF zX{o^!h~@?iT_YcGK_O~+bMce;?zpWN_?rM+f=3#9?LBOL(&vJhTsO}xi>-pa!G zVqeL4c;-_*-&rL|cf}=A##FHOZQxbn+G?{j$>mJIAV$MYKl#Aihv|QDVMf8Pv;vK7 zGB+0f_;D9VE7_#qvcX}PUquhNhje4BAe{VVrQyh3AHmkfv5$Nt4X1RCB9`aISjhsrh*QP5_SBsc;_RP<^^^GTG*37&J)hWN)Nf|KUO?PL*-G3eDWGNgx# z?dX>=Xhg+v;^Vrj!)xN`jyERFOUaZEyc6gY>wLo{4WN^GbiadMDKaR4K~;t`-K!XVrWft){_Y%c+^?! zI+?31yS=HBrNgBkjo_#^-;dLuI3xXXli%QI}hhk*AtZ9J+#Hh0;T zQl@4=C*Iw?+Q=HT5vs1HuHi=#wuae;{l?_0{XIS1s|Qjt>H!MInIFGRVFC^P{rQ1S z+-#?ej0}Lo4Cp;*sl!~~ou4nA=b3Of@q}Lc;JWs%W8N{(s;eO4md++;)Un+#mH9mC zs&b7U4+y@Hnpd_cz9UC1Wr+7>gTzq|{PExN$+RbR@#9J+@xhxCqHf7QQ}kQq1~)1^ z-VyTH$mdz;u>5)YlQIbrO(e9l#v+HmNa6j_Dw8NVs>_K@Nj&U2YA$O3|h*gh_b{M39Cib+OHduOI^?!(F)&)oweBn(F$%Bri02 zh*Y_G_36J`c`^$Va4(WB;w%R4&|(^*mAHkzgD5l*C6x?%WGtz3^zsm_=R~3n5sA`b zY=)FY%Og=CZEeq5Oqm6j0K4p5lHcrfl9H(7Kss$_$(-NliWlgLogNKyWim`O*%up5gZz0| zi5@78VNGZSkarI@w;VY6SGRBo>yV|o9v#vpb7aFkvbZDEoz@W*xGZ5qbR~+iWDpJo zCxhjKclA|Gv;%7RsR(cX?R%gx(l#7h zN>JUY8fFLvTo|c6-cdKv_-7{h`dqR>JLwXDMHY`W@-Y_%-5!D>um8aC238fR2Q`4& z2Oxqk*Y6J2^YVHok`)T>|K;C*`~IdFxW3r0H**)zb@VDadpYX zc_iE#iBpSI*zVU@I2@A64H!v@$Y;p8m71PEkqd8GqGQTS6@m$~E5Q?CM1 zif`ihuV0QF@4I4g=Mog`1`MKms#!lgo}3)Vm;JoY1kPM+Gl;81xz*Iso3XHJPfmIv zkBe<2Ioq9oTNs2&s8giBMW{J(!>hHqk$E+2YU`UDr#*ItY(KS zvMi8nF0Q6Bnxn<=72Y+!C672kgn|-#9YUH>UItoBOJlv?y2{V3T@BL3Zi-K0 zfes$yZy?ZLNAU0wB}i8-W{Wz9yIJkB7{)MSU4_=ojQdrWp{?yNMHny4PM7@}vt+cc zw(lD}w_2FGw`DDXCasbX(>)D&UJWPUJp6=#`Rz+j;zb`~!<(jUMMG+yJ@7>`^#&d% z`!QR!r;@6?@be=oY4$+P*hO1_h_R8>&2FAV-{!Cd%9R&JSojxNra6Nyo>r zhHYJEaw}$+pX=Dk#-l~UKEekQ`rc#qTYK3sruBOGjj!Shk0DEKIpW79n!A+b%sR_H z2I=#Hwxe&GMmKX)@s0vxA@kv{p07v^n()MIm$h&C|)-FL<}MH)yZ8w>#ka95Bq&L|qSD?eROi)Vr*k-eJoMPaz&##i3P$ z{i`2gJ>8JwQEmn?IvYkz&#n+HD;S7(VBWFQT9CEry5i7q1nRS1bTb{JnHS)zS zJhS~hX)ctrDW8t~QCjw97CRZ6rIWdd1E0Tul{SxMe>9h(r>6%fHYhd zrU^tgBF7euy1L?hkn-K0A`hhbUp-~Dp4}=mwz}v!YL7aHe4l^u)C;np&7-5Tc;Lhg z1Pzsrt9a`z8Dea{JYKN0S)9xMyM)fp$d=yZ@ruq?7+?k>;VBZ16M}DE)#o4i9|g7n zBd)?j2{2bFpLBoryhS0&hnK?EBWq7;y7Eu@5XPUm1*fVn3l^wu&o=t*W8-9*e7j7GZie z{oE2N+e*V?IYGZc>oKhTC3mU{pIUYpKU65;?f!jSISf4 z+5)myf}&?@D@=d4I<3;5&DXnh56v?dP2V}dT87^CAvbUT?csDdEGP)RdhmYz(QOc# z{UA#bOV7*1P1(AG)wkXrY$#dDI6k@F=7#^E&pp9`mfU}u0s&=@Oj8f3B2OA>!l+>vnlIJeSH!amwLwH z%D9%=Zg0+(B}^)HCi$N#Kdh;lWq}qf6raz`)YR0> zOfVxV_`now?^S zhS*8Yv=i6(wzkft9T1>VL172YMuYIo;A%vSb0Ww`Ehpw+W`LryL}TjEpb&^hkU8G z`Vh4hR~RfOHS;GO6G*~+Ej|3hB+fF_};uH}mv(lP4)m5@k}ki1398 z2c_9Z@tfc^|3_9n*+yUKBbbidF2CXwr6X63OEwDs)TkUWL$60Z2H_cq*OS`uq1=_l ze*^0UW^+bt7NPeG4t9DEqBewOGGC9+j!WvJ2ZbXU$?#_@IiD6@VvFWtOyv35+r#=` zTLV~su6x4YEEaJr>Wc1mR}a6bT|I0=A1q<`A!_Y>Ha>y^D$fM;5INQc<83?Ft*ef; z_g60W3BC8XK={>GhK9!a^OCckBXnE6%gp+ksae~Oiy+|HyApdZHNAVC{$XgzuvQey zd%e{O-5UM{r^3c&yy7aZ%Xh?~w${lC2nhDrt8j>kufh?tGK~*Ih=z5dE}MDc3V<#A zdNk7DhutFSPOb)3=F4sOzh83X zJV~3Q51$GNmbNS3G0tF;R$^wU8Eqt9Z@|CU&NBK!m&on$s_(2E)LcVYsHvDljcF9G(fAmX6ep;~>Q&?m)FI?frp) zf$tB!kFp6BNzeb}%qntnf$0Xiy$c<*;3sQ{l21|I?0CkyR$}R|V~4ify)ixhAHQ31 z4>_C{vl&xACkub`=_t^#o9qZlyHM-Jyl!gRBRU)H%3Dq>4pPiNc725frFR1tY}~$(1tL zyOy@r6|Km61KVd}lL@{Xp|f&9ziCP~RB3_0*dXW9knQky>*#}bs=Q&$J@-G?@6OwT z0mbPaIFuWqo0+M1secOYT)(RC4Z5rk4(M6$v^PjsP@B+xNU1*8#+L98-TF9iKxC{* z=Z5Q*b{W>b;eWlBNNDcNGIctEY>-vKja2QRU zu)WiG|G1*!v&t-VC4awtGJu)W^R_#4qKY>xFh-S^1_J+}!eL!Yf2vUO^#@ZiA@}s( zH(?+k{ z&)WR=rO)3p#VyVLygWO2B2q1@6_d`WmdE|jR{U-=A56rtJ*y>?{vjX6W-ncWI6aIj zP^9wbddaT0$L(#VWSq9v!AauG#ni=Xa&xXQhX@u`#X%rx>VJVp0lMa!p@P26F`On(${=_ z{W}%SD<6{i`+wTPUQ2jI&P!#1`kKcmltYA*Ai<%CVH?R5En}cH5y()jT?|CB$f59b zhfJ`L(J{lj&o#b%XpRX(t~h5i@ko_xQp8_v6`ev-lSbA)(Vnp{$2%=+RZ-4RGOt zrMB7teF&|QnvsyD0M>JDNrEUs@PN2R(Ih}17b|T+zShoUwIo#(13th9tE27*B$%p3 zX~_ypep;CG)Z*YI&Qe5M!}{9yHNu6pR)2(;xNhz#*roe&4pQ;-$tY|r)zIUZ0k)i2NjS(7R!Hfl zLP>=>90SpZ$w}zqeX#=7P>wcybZl`#JJ#y_%ea$SQ8x@?_*{ z^WBn!Y_jGtFXTV`P_~Ymmdw>rXYH&&HEJBmV{^T??@><{usjjcR<=-jaPjLRNxb{w zgZ9f|>13cFwyw6eEyS;%qi=&0-x{h^PBf-g**|!);5q3MgHc^VJ9=w5~23_TE9?a+mep5ZT#pe zw5VnCuvCc~=S@)~o3^h5DrxoS!In|kdCAVVGPafCgk4;AEExX|JnP8S-E+iRBp2ux z4p4WozcLr52bo9Aq1&2kx?~qhw9l4iENs(@*;us9AeL^wh{5kgILQiC7zD{Mx}@6{ z59j|!(s>6`{r_Lw%1XGll5x!|{=s!$@7MD@&v_iNA{y{ycy}uJ1N-%#*5bOk3JI35aGErgyrP#m zO{VN+16PEhZ0j1%pz4RJ=}s{vRNu$WCnkcaKoACIMx}u-go|#>qjDD7TQp-wWz9Js@Ot8C4p&4%U07>MD2INC*)$-QLU-)#a%k<*PS)s0yPU8PiN4 zxz&K2Z=sN@at`WNMaW&w?}fK!G8bX$6Pb>t@H;pci_f2Nc_f1*vMuC2PLQbi{-|%g za{pQt5)htjEwo7<=|$sBf;EtYB9zVC#$9?&{-rqpIG8_GZw#~-%6FF3u)x=p*S9G( z5)GV({X6AssZc@6mQWA~?MQk~Vv>sN3+5I;J+!^8l3us@!j=Rdzqg|}(5JX4NIb8p zOuw+}vaB`jb!~|ww!CCsw$Rrb8p07sUj}$Q0i9=f?P4g*Fl1V3pdF&5q&WDPsAfF- zpeYN=J~hOt?*NIcd2XzPHZAC7xSdR+X}o;`XHaJQnl|%1h-SgTcH@TbP$7rK&$hW( z-KkHgh*0qwyA~Wc2vI#LaDJG~Cxz*Cd-!?7u^afU%zqMDY4E-UqSeL2(z5`&i<1x` zfHL)pV%Y)~Ik-2pQ%;)FCOg*YVj)JiRKPq8K<5>>W@2zXu zbWd!?0^(Km+atfwnrNqOH0Sr|NGzeQu7Xi8rDBR-{@e78WYc0EZY_T_Wfl#3JxM8) zKcAh%yh0gSu9aLMc05ZE!KFWCit?>sRbC&E5T_sVu@}rd?CBX|_I4PFEZH>qXN8tlJ1{^{y@PCtvGX>#H>l|}>(=BkL_ql`ll#^p zQ{`j5mCb_NPKByEs+NP+81!?9y*E5PxOJh_r%8u;pt`3f_)!YO+3;Fcqh~B3UpKqA z?`TOLEsd}T&pJ4ag5Fk9SF}HhK%i3s8@4gk7ws9TPYFVP*AKe# z9@x(=$%6x_673sGh)}JBE%G{Eh*CyIPYIRK5vv! z`pN!5WkK%$ApBX5G5uDY7)!*7wU4_Sl1Jy+ESb_x=>L}%l$lkSZv7s-G}Z?e;~UmB zLImuWoNj!)IeB<<^sxQvD8qE8tsaPlcv(zGmhSHd;4iOE75*IqmglSg+#)w#%@v~8 zhZ`TAE4i0<20bcI{e!N~c8?oe8gasJXPk@|3cBqhrCaFGQ-)1%&r54th`VRdBIQ7Z`jC$RUeL@q0*btEYgou5?pXc%nE0>;Oksul~R9`!O1 zD@(I$?omk7PZoBqkAH?HS)kc=WP-kt8YQXJ5s;ublc{QQGMiVJG=O=EZ>AR>Iom4@ zmCD$^(0gJFQuyx4MjT9jOaKVlGy~h#x6%U8nyEw^0ecUL9Q18^ROi z$2DHq@Y~yuSnt@y$pc{VtXcJA=fN4F+87z}Hl%Mrz`ZDz2a-~1xS`b+A$``MvhCBB z%*pu=PbP(E-(4Pa+^p^2{M#Bo-)W&^VK@DAiu-qg^Kb^LelAB}-dq5J-^YM}PdjV# zXJ5B62D?_eN+X@O=GtZ7;g7Cr0E~I*SG!hMd>g-^(9SJ!VKK(-#@(|MQ9TpGRVngV zmZ7wl6(aTa9`ayJ8vNltbe0EgSm55i5Hov}xIOxPr5s(^TWxXr+lHcfyr@d%<3GP& z1Z7@nQJ$2OIo+3wqP2r-Tn}_NNuo`J#hPa|+{s3$!qve-hH2UfOi|#A1aK0_1+}5e z(UrRzj6)1LC4H4vFR#FS)8LN--_{9N0W&ZXF;1D{9Y3F&K>)WXjKPoqKaY4*Um|NF zBKt<{ovadf>%ny)X?J zzC-D=Hi(_Eop3#h4phUUN#zrt`kNF%(t{y+ z70S7(beT_*(Mwe#J~3fUAa}W2X}6xIsv!kuB2NWeIHVO^gEbS0JExc9dJ0iLoT9=u z&dx@sUGclPhW*#R&)Y|zd$uq1Izr}uM0zN*OFmPslw@x9RCpyb6EciHp7UAY6{xBQ z#Fyp{rWrHWLi1@!@LACQ9P34eZZ~B*9M)Y&s$&-Gj}ntY_q96zL}64Us*CB2zQOZY z36}OYpz*|emrK5X??jWPypalU%cOolD0(UmNtxy+dVaQWCncXegd>`8EtMs8MoyLJ zH9xViqMCpZx+ZqQlzrQuU3|%Hqot~(JZv<`71*NBFU&W%49XwXq}r8}GLBiB#%P(a zb4=Np5MbvT=J6{lr5x|Bx^8xEu5bJT16wNeiesVQ_Cc>J@5o;RrELFQ2JmNMYDJWF zb5n$GZqQPZnX_?tc zL0FE=sE=huI{q9!kTy;JZ8%4_H9_B5K!)lP$4#}^*<=Xr?1M^L(r44@!j}FhiNOz~PA!!{m43hM z2pCX5ifQ|^Dqd8T>!dX+7PF`o)1@cfuQT9f!zQG1e2L2Mu|{W+51Kx(X1nG4q5U_0 zzd3j0ZM?l$o`8w%M>WL`VsclUT&)1Arh?^Z0y|mVi9iAW;e% zt$1BAOqxhjTUsN@NWXgXPjVmqCwoLt|t`-t11UzLu1u~mCWu>0;PH&NAyGwx#4^DePR$a zP>7vwq8sTH6@m;cG_|dHz*Kt|gpE7ML?RFbYq5P-cUab`&ZUlC&yF} zD&{3u#`B(I<;whcfU9(NwYxLu-FiCzF)vRi@uxP3p@mL0bz-yn)Ft+p`e(a~ldW+8 zgA@7k-__$#Rm)fw(A;%!dpmY6IOyNy>CS(h>$^6id8tf;SK`W=4NEq-moMM^C%&KU zW^?@62s$~rszRH8dF{en$U_-4Qs-1(vuhap(N=35g~~~e^{hotW%(vRZT;K$?-S61 zdBQ3hY`-`V;R9u!lR+SaXz*pcJE)RK8^*Xq#;PIKM9@1crT_S%)FhYHuyX5;S^BBQ zC_kx4Qp8gQ5jUbK0M(IJ>QAweV_N}Od_z1Lk&8$KtvxpyeuQiFM)5ntpX~~ zfGj2IKUGcss)@PFRov97NKBqV-ZW0kgi%e5p9Gr@!YNxW8{*_Yv=*y#8c0+JTk6-! zGMH$^!3koalerkiRHrZ}h-Vzv!IWJDA;QIRsxo%QKCc&}Y3-jnG&0O3_c?50n5qIH z7e*bOi6*`yEvXBT0y842*CJ$zi4#V|Xp9N@ z-T2mGd|T&?v>=m11aX@EQC}7!QjZy>L7HbI0VF}>1ZFntkMg6r>cI?D#9@LE@1hy` zbjK6(V6+f;oqNV9i8^rT1*!FSA98K9Z-LRLPfqBFbh8s-wlXIsCnbt!AIgXQ!771! zM3Y7uc@ptb-EV6*MCz4!rd!eq)&EI=1(L<|H^N7{H8eA&|98%QAAg|6XadW1Fk4XIXcFsniK|LZta@MU?IeSTzao9k zsU!L%r1Cr`*;a$4_*ap;eQHi&T^bNRx&pq`?%UauvEI;4c@36s>JQ_MxW{m{ozH-3 z`LGfCea-#iAAX^QOQ($thY0{`7a1{();n<<<>@Tq+TOV*Al~< z|6e)+p8;f}u(CHd|5k7C+!b}ID9VswI{WUs@9T&zB8sSE* zH$@O#+d~?!b7Z0;2=y+11|hBFNSWnMw4N{QYDu;I zzTP6zdjPK+`ru83CMYH-7T3tk%Gxz!GNWu_fpar~1$!8&^xEalH@fC|-8wrL_WMdNf6)AA z%|=wvjLA{dD*IHoN@|X)pB4a(tChT1$I9Qaxb_G zB{M>@mvaUe`B;40zKdSFy8$Glo|=kaR=uGzDcH25MrdCBD6*FY-&VS| zVoID@@AfoB;;1p>k883#>1rao92t%v;=ofSuQ-`D339{Z#4tqXxa@KezpT~8&EAsv zR*HNOd7gxiN(U{$9ekv~5*?6>IIOEfApjOL>PZ&_!sa0$5Rp{~FF#S_ge)(cC;cy@ z)$sKwjwrUF)EfXzUjufk^ei||W!Fh^mlW-cFF`mNAMfq)y-_Yxk+r3IbVvWRvu0tk zwk{*JO#9tEEw?|yL{VPi<63J;4qbI?hMOwWG05%%o`|%bp^Azf`eGA`Uip@@iSS7$ zH456KfR84Rn4AqTTk=7STNabqQq3FP6^X&*BL>#qxlvr;yM1BYijP5)ZRY{F zm%WRF$EZG$Q!azU$M|Mw{hbTHn42D@RY1gQ zigGGPO;GA&I=PXUBPR2UM{p)7)9suA6zog{87#r1fn+#g;_?#JYNy*BjW}p3QD77~3u@#xyQ@z`D^2g^ka~YZ?J^ceVE%s|FvSbaZHc2^0R=q3U z#jhWG=&FVJr}7oDlpER>f^6!rePYsuO6N*m~kyQbPn z)1@aXim-2-PTkY^tZ3h*m(W`fQNNj~yD`KtULQX|VxWv*>-8c8ze-7dA(F*eVS}d? zZ)PVLvTe|n=w>QQg%>kH1}*=%xZ3Uv<3tC%Os4&l;4E&3G-SVW#lJT>p z0Y{J6mbm9PG6wDB1Tm=^NYK1m`({KmhKcj@s49uI=l+rj2|*u&DY`6`?Lqp&hZS*| z7`ET&^RN4vUzSup1po87kYizFRr24a0N7#?zISz(9KYYisfO=9j_R!8g{%@py;k4Cs0X47^;O;c2qXVkaz zoM#KapJaM%fS!uA5gwI127;Tip@z7rB9Aj?0YgNaYOg2 z|Msw7c{hon7L>zAl+KPeEJE6RRb~{zFs5n506J$;|B!UA>B7yY9l8>ZN)vycqBEQz z{ygV%La5}@Mz7V;e2#SK_x9U+%$-{xdQMKy&Q1r9=W#&Z4bU$?IRYcK5SoW^=i|*)?|)YJzn-U?pFcl5#|On$ z3k@3W<3-~H&`f5^4?_6v^`FEM{|IQb>-WtKJO&B9QgrS=d{>c{b=jnkvq#3l=~s1dYa7<#&c>%9%lOV#$BD3_(?NB~%7X zpue2=Bi;Lk@{pZvss4Yc-#f-HH@ZAgx@4a@U`h+WMU<(ri4U%Z3#m`V9%RxmXcV1I zEIb3N`J6ZG0{&!^6fFJw%`q$*Wu^%2=Mw}M(58i~GCeLpmS+aga1q(mNz`fXlg#(N z#noc@C_%a0Z+TN|htr(VE{6FNLb-dTSvJBMvMju#(I(7=qZ0BfPk8Iq#bLJ*S`aR4 z^n__TT+5a>V~SB-!$7rL!~RE$F=z|xikk{hVJ926GNYc4iuKg*qgE1-V5U-YcIR~Q z?dwctE8>qIpFq*YctJhc!Rgmm= zv|@%T_M%LU2eTf0=w09I%+LQ!(t^n-q{!5AOV;*2I%UaRCBqET(b0-4b8UwfK#gqt zYI^q%k8c9;58I;#PQf%Xvr}*Uj*vS4Fqvfoi#phvyRxS5j%$>FJxRjhok%K|#R{aA4e4F#nw3dY;|-QY2$P2>Y)2dK!rT z>^KwsQiY)dpMCCN$c?lEi;I4TdoK9=!K>;!>mCJFrh#b|f}%Ut<&E~jrt#k4R~OYM z^Uo|eK2P_aE&(%sTl;(@(y0D5G*m*EvMY(1;2>|U6h-nIf0iS)v)B6N8G2lu>L6Lj zR$qk^L(7ek zZ!N`yL^kOdD3HRHpOs{3X#7y5P6GW7dQ1gfdjET3h_aG_FpOnHPShULiiOJ~OLO0iUxlxP5!f2_Se=5dv_;YT zg6EsUak_rPiR{3ub_%KzsW6?RgK`4fY&!1;( zvPQcgV81Z5kPpp+_$Ro)m`)bt|5dgb-?)3rN;_%HN3&5K279KQ3Wg~a`EaN#qLG*f zVG7wX&Rr*84e_L_*d>GjZrT4MZjd5@)JU#|Y(`bo1~2zIyptXY$q zjKqEgOTE&nId7}V*y-Z8Fdl1}YuuP_oW}ik7UR4fVkavcC*UIWFCGs5Oj8PVmXwTR znUf=X<7JX=*ZhK`9l!J+ArGh)FTNHE33B}m01p})=dX4K0V2;!=So_rFz6TH4Ln(w z-+S5iJNbiXvy(dR(qCREx1=#W?=40JoKe?c(uUf@%Im%Iq;Tna850<~oN`3ud zFY_vKRkchfyQ|pkUlo(={&&>ZA?Z#SNjvlVja`|~;y2SnvDA+OpMKYPaZ2&3a*qJ4 zYhl=G8uCYPcrtIuSTD`;wkwv#|GD5%76@>6h{INTKlMe&59R&4nPB?+}sZkahe z=LeKNXh012l;dMP6a5Fh=Ta*jKbr#}cTHPQ#Bl3Uf>@Q!qdvurv8IWOzi=NhdX*>_ zT*+9W&cXMu*n=Xid*F2TL$LKDLwk_)m@`;NGjp-^ueUYIf1mqCeL1&g`j3WrAXC;W zDW5aagM#%K6)2DoOt61+zDitU6Pv^e|0#t?J4q0*pw+C>5KP9r=P|J&(ipAJ%@_$+ z6u*~RE8y42FZbD47XiIDhVSU{b*iLUU#wCBJIH z$UPfv9Zp)OA$O>OlT96xlnRlA5=c3riKW+5j>Qus;@6BOM`*dgc(O;n20`a{{$_*!7ndWc;=1FdM-uV25;o$IntLMOojMDIgJreJ_oI|I!y*MQso zXR39v*Q!U8kbq8Yc`>!-Z^7~PVtV%?7^mDI@eQyh`C%9K&g-5l?!4F$G;P3oI1j5Q zD*+n7G?<2W&|_?|%k6n-mHTRvw@8inRUFh*+ka!t9~=FvrK$*PN58TgJlJ8Xr73gB zl2-xTXb0l;q>yPB<}j=B9i!P_yh1QAy1{57mU63{4C?SMp9n~G6#nR&-+NuT=CvL5 zwreRXu~#!>iO2Bk`7Z^sLeBT$qq;qYS^_GHRVlHfwt_`Oi6n-WM7}l2DSn-?NLjo3%Lq$NP$(RWke&$nu^TGs2(Yn*f(Y_+RQ%m3dU$rA5l3Mm<0OCORkc9z2Rcm z!xOymC}m=bpQ5wDCF$rg^3Z`j6ys`4o;Wmw5>Vwp z(1eM)8;?ZKeY?AV{oKvp^0+v3fa!0jLFwpXOdap*7(?dAp}g3A!UW{}IUDiDcY`s{ zEmDJGM-$l=t!kbZB|m7t^$YfoXLklg!@$vHJn!Dg-<*qcb{IZXSB>MbZmzfwObwY6 z<$i)5#4#OKXukE9EaO{WXrk2W-;{KAE;`PSOIbM``mPS4-Y#5P!l;Bx1Ufy_t?&H7Lab&@Exq-1eoczF!F=Th<=YPL{^z3t3J)Ww5nc&UG8l$0#FuiA4YIqc4d zt%X$chn)eRHiABsU}S%rl_VHd>Th9b)J&Me2@i)S-5uvI6UI{== z%g!#4WAnzH74ONuc-UgBG?8NkV^-U4Pjk|S8k|)J>?;JGpER~#?;ak$jmMNJ2L)iU zZGf%#rNY9EU;9GJ{9KDGt_1Kn+BjE=Fp~jF(yTmzCkL(}RGdr!YRf^7_~&gb#6iLc zAg~(^N|Tq(pi8h-qoc7dtMkafT;&>?u8L$kt1{%?0H*1 zVcQ=xB`5u&y(Q5zCT%v~%uRq$`bG!uTN_#|88j#7)H$Z2FWd~i^B ztReUnDZmM0#p%-3?V%3Vqjccg73s?%y=53Adb@*gN^tS*EJtRHVG;Q8k0?~zk zpf3`#$sT(k@4r9a>y?|lS>4!?TA`0%f@*jF>_in}oX(@##31Yx)3;bnk~kyVYY8+V zGZJ5=wto~Mb#_=1EEi)hy;+8q1v2T}#X}m;Mh9B9P3xVi;nVOrCYL$Yd&Q9{OeTrv zZQ}Fqk2Of?nEK5Ei1R6xxPB9N^|IW5j+m^luUyV^zis{Kjvjbfw3No+=@--J`|LBu z>-+S9>HRsID2?sNT=FE(KKeTdr4g^VjVHJ0ZebR?h<$Xa5?09m^63Bn05 zsYdBc>2;zy7$hqB*E##5+rGR7jMRdt6Vw*}l;48YD1tNgDKB^I-V%9eIJdvzVYsHS z4A<&Fs5sHj`xh}fI_z4I2a~r-z|r~(5IWm?n&gOb)(Bo7%0)7>5W@gx((^w)F&atD z>0o72S$q70Nmkhsf#Iu&4(ThYuy-XbGCO=pDmG8;MCKw~hjh{x)%zLb7>usMP1J(} z9$9ACMIl+J?@&K`_s$+1h6EJ=xq%JP-b8HOV@N!Df0?X->< zEI5A>BuNu7Jj2|@$`s!D_jE9WBss-!Ci6+LnBwKj7eNF+#;SK+V&5*Dn-<5~UHyru{B2j?z3uRch zN<%{~&3^2nR15SfL|pp#uJoB&n?)^FZdXc2VIl+sjWDh(i+W&GSbVZWp2p&Nj&GCZ z_K2HrmA7zuZ#tm!^sBDD@|N7&KikFhttM8hlY&BrSy2o0fYOFEASov#3gzqORUt@J zZ(=u1B9d0SD#-6U(JfrQKT9Qw&XzFq!EoSAP`N|li5g3&2|_-Ib$?Kg=Y1B3TOmGw z+Z!Iiy?UuTSEID%x8&5037mvWnApZtMBl$xKbbq9Ri#X%*rFvXvmD+VnsCb}OCcuR zf_m4yKc@LuEW&3>XUjrcs$wx3=$=ejRjyO znitURrLB~u#F=W@{C+Kv3{p$Q!T9%o{ zD`D4>V@*<<{g|5u9QrZzM@1erKY(&Yu+gJ=0->TCFA)q9IFpjaA z^hzv8qsBp_Ems_IqP5GM=obsyr- zupEM_k>?W)u&6NR`QRPj5YQe_?eX-o1UO4qU0nGokk*=NLj*yV-Q!&?>DJ3*q-rsV z`HfTQ>H(K02Pzky&E|l8hJ8w#2H0Ewmizag>GEChJf3y}`YT%<_L%6AG7f2HlD=l; zT6y{w2XqAjiOqSoADIpBn8r@rXwzwa_1drwhj!vGr3NG{AF0%s%HkwB+LMaQ${CUD zRVkcMSrIR*_r&40J7Dg{nVq0dUB~NbS=5DtZmjK8v+6%EB-^>Ee(!!rbl%Zd2X}rj!|(;V2`_ZT zJAzRCfFGLa?`--yzPb^ai;BSHRfWc0m|$Ibw90u5msJg(D+eNz!Qs`7#t1Ga`*;wH zgjC;Ak?4L=G;LB|jmeP2{`vggil%Z15${m(lOL#>bdFgcIkqKxz7~Qy#ZaT?O3+NZ zwtfmt+A#*n_gkBXgxlZ8?=JaB!<0p(cOYg!HwBZ=9-ciEuJ_z(dryplR8Y%%GBuVT zRa;GKYqs?xZ7?#qlGBIWeR)Ni!K#d~eD@6?YB^>-Rc%WEfj;1UKC_VrVgP#XcPMxq4O(B7S>L{A45qX3{NoLl1qtzbWN+}dB%@*B5i z{I{94%zMVS7|LTfy+ms~ggZl?ppxaYr28fsXsZo3ML*~qOA!?G4AJ=(QIO~p@;Kd6 zJeMbpPjQbOn-J;`QY7H|CjM4BCEL9?g?^Wl?v;_8&vKF|Z)JjQuFiJf8ZDc0fuYtL zqhXShKXMwz3()KcI@#NiUsG4XPSEIBW`sJSZ3Q9}#Z{Pj`5CFCuHPkQcr4!$)1gb#Q93>f|B4*OQ&Hmx*T%PBBdbRl4FM9bnAM8XzY4rR5! z<)tjELO}E3$)&b4(5t$kMY%KY$(1DOO|$@o z%1_zFh(TQJ-lAn5W!~#s99ePuF8#8&bvCJ6P9J&?47W4L%71ZRHY8NvUyPA_U&OuW zxy>bKU7`e$%b~`6HG93DJ}B(_=BJS0$em2+V>IL8p6l2xUMVSlr$lf=X7lTU2t}Qg z`1Ht}Z)Oj@A!u7GVF5u^gmBOD+^tC^Rkr&*O4a$%x66Z^z~!uSL7p5-J-0)!jo3y2 zM8%1L#k|IdrAjB|YVi0x1aAq8w+2Jvb2_=GKVfWD)nHQJY+B+e=Ifu+ocY0Iir0^` zk=!Kr*}KMPe!!;T!>@4-xQ6;E`0NvKjurY_&1ruvf*(vnL$ep$zVPyYFMI>B=%jw7 zeZ48P9DKqx_HB-mr_RXY;im+(Bmd^BlX1Np{Aj!FFJZI<5W;(DGV5qsmT=J@n5|%5M;l~WDmf<=xCwe#%^K*eC_`vuh2y6 z%oOFc`qr@sVz?(ehKJ$>a$;6wSHf0o(6UiiUEwQI7Bz2f1AmFs!V*>CQA|+X)&1$3 zGE>N@K!SZCK5~s@e(LI%EPt64*f*c4E==!wBcObKn!J z-AK1f04%3YCPNjE6^P%;lbsutaZHd{YKhV{?Cjk-wv>P7TZI;f<)C!)9In^US%?Rw zRCIv$X!zag$U-^cnvqAG0vSHAU;g^Vz@#7qf)ut^tqka8fv*nHB?nhv2H|_a=hyH% zuii9I5j5625wKOM9#KB!*~~LF$k^dnyPtH2c6_k=Z|CozeL1>wB>Hhg?bNBipRypV z-U?lo)sYkKS*z8dPNr36h18I_rP>4#S^jxrUpVKcaEWVMZOsR;8GFi-S7V|RPvSIw z#s=z$E@V5j2hnY4Bcz47xCs#sjXa}0Lo*8r1Du*q^7l4+mudYnr8JWEh&u%6RV~$@ z56vH52S2lpU8;QWwnIC02;|W)p$t{bYEp7GFw}LJh}=2Vdv~$(U&QeVKE9V=j3*qG@pUfsbuMJrgnI^$+$1x6+GYu|N4Poo z;_!`US4Zy6BK5N^B4~`0$rMDeB1bBcJeFO)n*?L3ooki(lpm~z=1pDuyK#spOy35lJ!MT$bGVfa5zO7(P(vn7u@h@=lh~f}R5c4&&`{_q4Yy&-N5r z^s{AW)eT$LJ2)MBqVTj<)eiO}oKB+bu4*_-{C8?XLsh!va(4bVAaV*Ws#J1!OV7V9 zU(SBe^=OG+x>4#5a^`7lb^>6NP}l_BKk0D5>NxQYi(5nuJoLOC)@nJlV6di_d4aI34{BAeM3Y zG#|{sw_&ju>-t%{U~bYGVvTGELfF7bo-HUlH#YE}A1`M;1MtXajA63T*azCqA|tgl z#a{elg64qX+9&}4t?SZwA>`p*Dx0PI*68ED;$NfN1Z#xVVYFaba@joC#n~SC+c>_p^%f(`;5YZ67NG3xr(Dzsw)~? zd^mtH8OZAZX7s=RD`A^ZI{^}$x~4w=9i)J%(z|s|-IKBonFfDmkOCJZFqwA2eLctT zHtgfmJ)dl}JpY$RFBDc0sG7FXCc}J~M6P#-QZC0plaE(vp7m$10rc?E^1i&fy*;nR zT{q6P@B$@D^#oeAEL9vzpkL{f#5lPtz}aO7FW-U^8mxKYw7eNJp%}Hdmz*Zei77Kq zZ$avKVp{2qM3wA~w2}bP4EyY zB43HQZfJ(EHozF@M;{k8nB>J6a+Pw#*Fc^I$luKaKQI!@|H4z$y(urFlKeRtcr+k^GsEQ<9pfmdCtcSDQ~ z*k`o|Yh-2Ru%`v4fue;p1!S`7gP-nhPzxg}fh$r3zD+#9Zs7R0SQ5Fk_{(Gp><=R< z(h;A$3m=GGtQ&{)a+bE1q{AoHF8XMn6RzHwWvL~Cjj}D-uYQ(_8_#J~R4LKObhy}C zXl-EwKdjp#8im$-&(rX@W6yK{>2qQBr@cd0bw~}yKS83QGU~+xJ#`{6#{`*nrI2P< z^s*v&P0}d$(SpN^waBB$@PoTXe0Mk_3_4Qxq4m~wNj2YMLENE|$D-Y-zbHTjDYw)p zC(})h>hS_F#c1lZW$r**&sO}2Hl-#)7hTjUSY@kElxu;>rGX2OGt;e^p- z5A!Xk^qR+=+lGamP16n1xWYv%UzF&|pGV0R{Vf%6p1$iktmO~Cx>rdq>?RrMO z#c14NJXtd12-r~#lPII&I6!_O=-mXR3EmDgiO#&+tus|$we||vfDan|*)WAMt(s1L zY_6+Ubw7SPyRI*qc&h73th zz&K?sX*sG0Y@M-HpEBVQL=Qe>lJY{D{_Np`a19V-rh63db}y8)G6$&IoH=4Vyx{H& z9iwQtdcuEzjD&ZW9+mZefx*G?Lwlg zFs^MM@M&bp#K?mOUw|O+Z@rz9RMIuxAQUo*rs&&oqdfPBL885$eTnp6m+UZ(CiafV zN}!a%Zi<9fDbrEV^y6DuSN!)}9wM~|8ltgfGhHjCuWgizX})8lJPF?U`U|0jPW`bJ zAg(HWaKe&5BZ$G^E{KV3Pp&VxiFnmR7s(lG9G$XiDd6B&RU$(wRgd0yxvntYM(Xj~ zGg_u#Og%a!B1&ZN=*x?AyRkZ(=tYe&hX*}6E{!JjIb&BsnKd+l)IqA|B=(s&8}FZV zM;_GA9v4~X}d3} zN{sE5`$J14`lBXouy5eLvb%m|tN`6u?_QD^vCTq&^HyA*8>8XmL0z>@gpRaX#;(h1 z__X!famGt-YYUNucl)oJEwljRHhw-KwtSJeL!6+#e1m{o70jwGUYkLol3n%bxO}x% z8&gTA!5T+XU6J~84ys1O%nX_Vz^lPK0rW|x1Lt}lc}@+E9eGKY_ImmldKstT#-HjP zB`=OY-h50*NGLg7^L~drH)_4NSDEneWnOzQ{%28kzdKgZ%egf(y}p9Rx%?J(7YAI_ z0f@$jH!~%<>THIG-a3X6C+{>h^l8B^b$%iDJ7rL78?5~{UESTVV3MlgVl`bFL~}%7 zTA&Ba5z>F7sV(vJ<|0+%PFVB|tJHP~p(wc0;BLO3{pTItbB!}H>McwB*=d{5?dQ>z z=ycO6KmC)o@*%n$I=77St!S9Xnwt@$t>(A5was}7dswKT<+f!)_?MDo%B7;m(oLxo zD+*fYV}O|Z)>qSalI~Q1tTv${B#OlfHnKNjtmwvL0&$IZEL$a!f*D>;K?>j}Zi&xO zZm(+vJ450}=7@^RWQ++c3+|!Gjq&Z!a5KyP=^gs8Dc4%1!e~ zbaff(t9L&vXU#SUMSt6m>=S9ngs2tIBN-1D9*F~^kR@mxG3O%(K?i#eCXyTHmg9L2 zLqsULkVxrh>F_YPT^t|Y1Ges-7#reXmJ>j}LL{=hfqcN!Fo5_iddV$yXVnda3*XRz zxc*={wEE~H1OBM{Xs)CvCdbNfvlMY?YR<~@!!t7^7)_kwX>b}eJ8!IsMdpa#Xd}km z4NQ!kD;c7Zj9f-W+#v-!PL+~cG6|?Jt9gaaN*hA{{;S-xw`pf4ePT9qyJg;mfv zhRHdIc^o!Sa#y3|tc2DYg3wmgUk`cL2X>hKSMYS8=wza3bdma-8Lo~AQfO)v_XWoG zJ++~M{7oe{k5eju?}>mtiwx^OH&%A_i5F8_tUqbCSK&-Hu zmPI8Bxa{L`-}h{R_C7rkA>i=f!OyY@7mSE84;mURX9^J!!;iXo9{fn0yNWr>qSxzk zd3-2V+vqPhwYyT#-MjbJnue89v{VSH(Mp(Ep;FY$B8K@=ilpqW*dQ>lM|88A)UIV& z)^!CWo5QAcvK9{aTKB`zS`YtL)$->*|M~0JueaMRDLJG2x~}*8{qgY=Npd8_mEy@n zQZd4P1nDAB!zoK7Gjl(K_mq8jZ?5sMe)({i#}CadEjAZK$K%# zv%4!o7$RDVT`reT57%|A|M4IH@$}sf%Vo9u_U+rtZTt3myMKLtEwR)~`RV6Bn#ta7 zx7#Zfm-W&{zu)ii7JD0SZ*OMxa=FNlg4;3wIkLqvRcAvE1z#qSD?5YhZ}jcXAf7$V zjJzMUp3mlChWg*#kaIs|XwGH@nM41n1j;d0EqPL+@~}NHQb(6&R-h_YN^Py{bh{TH z(R(Mw%sySd?|t9yz3*SuuoOH#UY;J;r-#+kB47+gJH}JzxR#SAf%74`z^Z27{m7%e zYgx}Q@o}rK2ZS_^arC^0F;rw#rH37kVD27}qPCV|q7w8zT5H2Sj(Lo`Cot~A&1Bx2 zOmb6WXLvIeGgql13Yo!uZyf;!B0t6>Vp#*1^#Mtd9h9uvpcLz^MfW!1W#3-*{W<#0 z$E~+8)azw68$DiMU$3HOlM5>3d)j*VR{}8@33SX9)rW6=^nr2UWfMfyyZ7Fg^(rFn z?m_oXG6AQmDOrz$@P(t46inX)rO{^#ev)h4J@u+QdOoq%DoKBrb(=X^LiO`2?>F2X z(gD27nN1$4oxxoXieK)Rw(lk?)k-Nz;?3eXBeI#P8N(w&O_ya+M(f=Q4aDRso196>djKk=WZJa0}y~bjrEbFpf%|H$hKp-lv zI{+w*nTiU-4q=o7lSmPtsI|SNxoKsMyeG}uzHegKx0f(~`u^d|mtS9BUL(RVUYY_{ zmj^^z7>cP@Rih)E92L5P3eDVXl=j_s->vHdDeJmEeE$Q=^1SbNxxT)1#@M>PHvIGR z^Zm9181lT`@p7A`Xjv{4_2BL8K8D`5ecyYn4@QgJ~iv&R|Ta!tBZw zQTNuG`{2GLM0b6-0rSHSO8LxM^zYC1)p`!-y|%toC8CNAn$9Kx`aR0B}TMXSuwW$GnZ zw$Vsx_6jfSQ$%>9!Pj*aTin@uTbAW^yS359)T#~O=1n0yeR_C$EJg4A<;&O5@@ZYcSc+Y3DP~|;sEC%bwpv$R24jD{ zX)V<_#vZW;+;OAqx)fbksKADmQN$+KS1|387)zrtE$xKLhZ<2siQRG9es&FKXE} zF<3MDy7g|@B> zlw%(+!`szVqxsN>um0q53HQGL7U8|Fby-(c8Dn^aRHL&9OJpzo8W7^{o~oM`)Xza#AQ(x8C`@AOBInZhI*7jEkw|V>}_<1t>togDD}tN z+ndUDH7qo={xl%rMGQ4l0mSG=7Sry2fBp8u_dgOYn&zvXJq;1Lj&hX(0sQbtGqXia zMB0dVLdU+lyVW&EwC7EkhaL~G-7zmc|L@0_6;1!G2#Ub0h*mPpQg>fSw$|R>Uh><+ ztEz@@c3@(}nY z0W>wCxzoDGfqDO!P<-qHonJq@Lm&UlSAtM=+6~WUw}?0-GcQWW z6mAv~`SRsUwX!bD!+N={m&@gHsmtYZd1+gAUPPt~Aj#l(eEgKrbBvZlnWQMmQKJ2B zIg!kkabESIeu4AXln;G8ie_soKj*BFa4;Sd*aaX9YUfC%r9su~9E^*|MHAYgwX9kz z17ozk_wd$x4}=8A=z~#8S!_v&VX^@b?sXJVQPx_!dt)Er!^f$!@%s8Ie(819QcEc; zB{BNz<+7+~u@I!8Zg&o+Q@XQ_5i+EA0{79j(c9=_1X4Z$s%mQ@Y^8o&0Kl|U$ODlw8OX<8?oIY7@JC^$(K04QmJb-a+>X&8hPQ$VB?t7gwbRq@?- z-{psh82sol9@X32N5MqX5A$P5q%;xSy|umfzLbfIDW&M;Qj3;S1Yw7sATXVLa@k}B z5C=r(tg4q~0g%b>3CUIUb@^m=Dg@*}IN}h-k;6zJVQ=#en?%HZzi+o&wvo#rxmXWt z3WKO>tyN=77DDg6xff6*EtG@adq5GI5LNdvLTAHJO*JCMkr=(ayv$)NB~E2fp!<8I znkLyKDY~v}_Zi4~@6IkFW$}lHhq_G6Vll~pE{a1-a(c7MDHaIEF_%e5H^~T}AR7@g zsYQG9^t5ZOy}sU5c)ebWng~r)iYR2MR@GF53>?I^F@`BFm&>}YmmeN6tJjFY7|~kW z_x-lrGq{h4T34BBEm-R13>jmL``%t(U+?$(cHd`rn*fk|7)SjGpomI2q#Hm0Jv>ny zxwj?y^SaXhcV9qinXbzM_v=xD6V*?YZQF2S@7 zn|l zsnZTjEQRy{lp>^1B%4$LAK_z+ZSONI3eIj=t@W*K@DQSi_PwpO6g41H-OxE)bF5@b zsd|yE2ZCd78?8o|yPHs;wLsw_Ii2r>3tcfp8hLBI`*1MG!5HB!Tuf^%FpYpX#B9-d zRJ<<<&30o2s9$4n2^18F%>&1!hkcVMKpzn5P7$cxz5L*kk7HG<)h$y z9LQ`(1!j7*Fb6&7>;d8B@^Gv>gv==H?kBQPkFIZalQLbB1O=8_iz*S`&qc1PiCW3r zDie&F$0owrv}N z>$*Oy7k7^wCqNQvQPh@Fy$uOr9A>|WU=yb(ypIT)u!wr-7z&S&dkO)i^j1%W20`zA zDppz*Ir_t-^ohWp<-OHKY@RS9(0lL0d+&J;v-kV`o~;Ja6XEjvnc;+6zlV}Z`EzLi zX3rm@N;SyL=|=B8<@6A=S}*H^+9kJrDP>tpRh#<#grj55PVMsW@cpO9>-EZNxiZEW zZ?E@nFK;g|FJHfY{paUjMI?nq_x+v@sF%xSSuUB9x6xZ0+qP}n*7klp{*uBRPB*RV zQf>4&=*wkYW)hUG<<^EwuS*rl_Bo{b`8p&l#B{>0k9m~gKsa=q87MyFU(;#x!@kW| z`mOvf#N%LCe0(&sS>N4~>@zq2AqdCmp_$L_*?J?T6zrMcK zS|7i=Vz#q|3{Il>sC6NQA%dC-w`u|wj&cDfd?LpT3BQw=*s@s-d_1(yV=3uZF^MYaJyLQ{XP^@Ch|xPp33g(P zv3IRjOys^#Z`JfND5~FGKWVWR^gh)6cDpSRm0T|dD;@|?I4E?m3x^p2S9YL9-1fa6 zIJ6K@u#{Ss8WF?Y898Eh{w8>zM|e3H#WUp1tXax|aA*2cf$ly6P#FZq2mvW0I6#`7 z{Txn0EmDB{V`Z5uN-a5=@A4A)VA9&hV;qp^L>jUbC1(Z=Y8=XA`N@4WVA7GY^HxfU zs4W;AMXXvWYKl0y3lVO$zyJzeolISXdG^C#N$G-WA;n$B0BN<>Vkh+^M5K3bZOou2 zQ&QMWZR+rkWD=Pf#f_3=>7pi}65+mYTdwsuSm+M%F-8v+h}yC&1+Jx7RlrHf z|8MQfiOfiMvCME{0Q4SU?it;CAuTPHJe+|j6snI4o;9$TO^ATC*%TN!+`aY*D*GMB zu`J87EX1>tRaLXBacd&^QU=CVN+dIt`khNq9XLf5 z+j1kKZC`yams{$+zrNf)fByXP@=|p8??2vNUtjO{zVG|DZ{Pm-_VwGhZ|k~l$7*Kx zyI5;1t;T3-45ob?@9*!A$5wxcPG9;4*+_nmpsD<-Nb4E6*?fo4#miYb+n@{?BdWbHu-X_HPOJdd#gDTU%61e0t6BS|M=_e=Jr#EpP z4j>5$A|>q_2RxmrPdyVRN7m{dCczY?Bn43-YE87a%shM-;C{cqzP_gFzHYH^63~o} zx*^4D9Rx^;(M2sv$ZFe8L1Lr0-#E(!mRuWQWQdScqqJ3Ly_0~lC z?GAtKC zN>|JQIuQ}IRtX$ASvvDun%c4~tu;<4@%?^pX6Y~k^CacuQ9W%`#kwr*mtTJQ^5x5? zPoGF>UDwCD-fy46@$LQR*RNmy{QAe+`e4S(%gg=sF zJbwOsU)Qyu@ZJ_%tiZlJ8Haf>I+&TrRF`GB-EL!ywL%OggHuGxFJkUDwI$RjQGbq4 zIjYv$>{V$7whBjL_Hk!F1@Y%7{CxN4xyF56#j0H7WY_SN+?h*&O0&x*`nddbjxqQo z=bUjCJS_|f1b%u4bu6sXxrYXwZYpnYkFs2t>IhX$*x3}SX1efPUeELBm_$w}sB)7| zwV4m}oVgy9ypHHPGbH5%g!B)89Ar*d^Mr9D0)EAg6cJ@rQj;qeFgJ7dNQ58rch+pO zwGl2PY%I-ylY}bFpcLiEpd6G;YQn8GZHov;MEcxJoLz!8y35U4BjIpGrbw_5Wq>?$ z9b+Bqw(st~_Fs=o7l|oYSiWVv9e#J5fV(woQ-O~EeurGw>-m${m!vO&j0i`PaDY7{ zko$;z93#Lgr2!2rl>dfg5L>2Yglroq>oSE=3GKYvnghvWenK&f~tWp*Ktk{AgBEvAd@h2-Q9F6?27EM3^|rWNx-WWfdr3X2B_! z&kjh$fXdReiDra~+&_I%;&okdcsCQ3-H-Rj13({UU{O3P^{P>aS(rmfkdhed&(t;RcWP0@<>(1M@9b=S0L|Lscb7o1nh>5VM zm&CLe3Yj=|@L$z111X&-p}kxS2_O zef2Mm2!O7}lY}!V-XgV0r@|s6%52b6?~Wh))Jv$m;>dE0i%gM;?_|1r5|Nljf`ti` zTw+uJQ!p>4Bm8iLA4Fu+l{pb%n$1xeLsr55Tq0D}a)bjAZnMZpTK8?;M-tl7nRQe& z;Ax9k2r8dgMq%rmM0ln<+umjhU4|0F9$i zVnncL{$&L9boQ=WPEXl%cW!)8m_`aAU-gacV?{WjI5>dYso8jG2;KVT9f5~Y%{$((`=Qv#Z!EH@>JSAIneMm8-Ion0kcwhFf*d8#c`I**{fA(pkgTik?Bkv!6=`^QYSz}gc85r?f|yOmf^Pa)|!uF zUsnJxFE79S_S--H@edJki*!btqpWnXy z)4Hni%gYNX{rvg!`}eN}BZS=ROxcNBG4p9VKtj-_B1Ux7dT{b7Y$Bp(JVituor$uZ z?v4O}BZisWZe5ilf`pIbu*c)_9l+~rji4ib%0f-N7gi0h#|s2-Mf4_phhus=>v=o)Z1?D zu=7=ZEFvOeVx+43zFtS9r{oS1sa;P|m1HHqmo@XBr*~F$laoqB{6BOWu2WZsuPjy+ z>Z`4xVtY)lre(qVE?f><(jSX1b6(2n;3`6=Ir%g$gEIk|DxIg%W0t^hyR}47=SXd*tQig`RNON>EpYNY}bN z<t$klu2MD6MH-b>H#IF1~?w9X(Oe(THj_T$#= zh#?~X^&kJa+;7(V>+5T`_P##WV^6@$V&gym{Pz9(cRgCwjAM)+KYmEFh)~rpzyJH& z`;Q+#eq8du!nOUj#IgQo3DkX*j&}tVZGlQ_M>-~OjYMD8wZ4ukH zjqqwcRh5X-DFRB^-7hI&Jr{t9NLeFdL}ZqJHG;Bq(!s(BA}>Fd%slp;1m!lvV!bhg zM&$AK_RieZnrS0R@Otc(V8$4Q;Q73=U}j1ZQ%4?SY{#+rwvA(q5h!q!#_;1fl(=rZ zaDRDusWD*J9ux!?=G*eJAMa+?M45TysUR&PBA$Urk?D9-jzaH)0VKhPbEeVa5!*QG zsbQ&YBWh|W+RRk>aXRUY84+Xj4m%?XjMvG)08xXst0w zo7IqGjFE)pemfo%5s4rsVvZ5ov}@Pi_L0hFW(*x(PsDgJ&$0oWFJ@K|h}g0$W=4>! z5C^brYrQu!vogx9ryM7h;`#7um{Cq_%$y`MqGje~)4_QhhiY(^@l_4DjL*bKr(z@( z;tC1`iz=zI1TsDR7=9eJrKZhZTKo04U+(uin6$NehB@(gy#IWA`~LmMkN3AR+_;O9 zlGgFGWdY30wryM2_3_w7j?8o)5lO^Z1tCB{faPR}ose>7%YSdw4yw!F_|q~1~u1> zbpgj$0^qZUy^_ZSLs`!Ic;13?THMV+TLCzyq(;3iz#Y@y2Tvklb6-(gtBA}hU?Q2M z%zfWI!!tb6BX76cvWS@~vGwKN=RPV$8cV<3k7GY;&od8+i0Xc}L=i!!zi!PqsxPh< z6{VRXG6KXDQ&W8o7-!^~d5?n9J&KC%h3iKG#De5<1}@HERWn?E%JT}H7n+%g1f0Yi zEQA~x7;q+ab~hi{_EDH8Gc!)-MPX);N+5^(5pe{0V1R%bK~%MCd1jh03-e^G=QN8K z;;Nlwd6@~4z!76AGETo!fILZqNI1bD(OQZjcukJxcnPLTEOTBQMNJI?II{5ELYo1U z0)&tXu_c$%q!C9#Kpv3@W<+pAu_%Qog^lP`ObYUm`@Zk{5!oU^Off|d!gByQj^f>8 zdbk7Se#bdjNQpwGmDtDlgqc7nJ>Vj0v(^^MT%RrqK9#c5o|zvGu~cNYX@58gwE4j` zD3}@_4EIy{)ps+qrB@ohzTe#gsZT2Qy?0d=FACOTjarx1*qX5EaUARBBMLX$hsOYf zY~8on<~9>>jAG@^k&>{2&{5XBfVMtc1E`Sq7yKDDLye*5jq7c<+A zJ&FAo6;rmi_3IyB-`?JmuvpDwi-Yf1TyyvJeSLd-e>_%)pOq}Bo#LwJ;53LdXlj+Q zmQjMx++WSiTVrM}59oZd*SL(?=ptrhNvlkc@ROdEQRk`ZU8+UtN!cZ$5znX@T{Qxq zJ=2A~#Hm&L{GHe9swU11KbwH2Pq4<*D%RCIU$^57g-=v^qIoPhfAinI4ViixqceXD zMTw}afA;P05!aqbYl|$gEKBbe5k;}>y|>b?A)*rmBTh9|vw-C_%&IvTDh3LG;a<89 zbS{isGLv4~cq{}A#NAraevlddLGg6aL`Eb_GNA-#Dy%azQDJq=D8p>RT+AjZwl6EO zlW;RGh4`3DC^@3wKLuc1TM%%Ah%CJeaY*d@PLVKBF`CXj z9Xm^t#yw5@}0G__{=hBc_CQ*P(k|{F+IebzlnTP}lKihYs;zbq?$tg_40Z670 z1*q&^9aNIMRpX0t1R_AwiKMoWjv}t)$cVX*lKE>PM46@TGAkh#6r|&YBIQwAHF^rv z&t(x}EpOX`2A#=L!R*{!= z2gVffu9MH_7{91=LN&$!0=gRs5HKLIJ+)8K>w3M~sRR zLgikr6%$*Wo;wHuBMYs3UT`b5RT6@B`(OX(e}DP%rL~q!eK8+3%#ydaHxk~q{q5~- zdw;;O^sd&%wiiL-IB1Ohh-2Hf3jN!*jo}btE~gsKkPs#i5?a$IOr*OIJrU{FQ?jcW zOPop?0*sv4j+_*iJl{W_>SZ~f;5;L8p8I1JA{%S%G|zy0YN62CSjiKQ<@t404Oeu= z6Wn~&?9Y*!x$3)*;QpKy{u0yAFX`X@?$bxtCA)eB02!ru`D8p%YyG!dL{wW{eu3_3 zw&ArPsU>rY`l^b1(a8&xnELgzV1bHAeP>3v`$r*iLRfqvT35Y4Td86@B+@eoTV@{@I-{=2%M056dfcYONOk-6od}~BU6A98OW#| zJ4kU`s-)v7xtw-wbizw+^BnAO0N_d(#DIhRito%sHG(1{ghPZ8gv7oNhKwWJvjVPr zx;v{dHDV{4c2odZaJyA4tm|WU--e?`?~4#o3Q5k1kH~q8lNx6b6NieBh_Nb-S%g+V zy`Iy|B+9vVBO#}*WNb;Ze?`qK=99wHM!W>t)mT;3cxm_g>v8H-kMPPgb)k_|$Yb5s zCX;P~pm2;4?(XTm_jSZR#-L0pjaG7A_w~q5BnEbq4`cTX3b->j>lBkxh;uGjFC*47 zwsS%zCRjE=Sw2daIuBDuQI=>LylLgcoDX~^2_lFSh-5MXr=F|W0wmKMB&t|FL8U?j zqNTOl?e_V%&%gis@2xHCx*o^zzOKh{^nUx-zy2j6`+ofR@qO%GY17wz#MZi~C^N?J zk@55A&p-eCbGhFlf>~SZ%sgngJF=L;#gi$LQ&E%<;Q&lq4QEW0NKPJnalxi{j;d<2 z%&h2g1-^2i5j~rp%(y}!6P`8I00>XjV0_$g6_DrB^@q9DM|AxgVLeaffGd1oVfy;e zNQ7mIZ4=MUlBaJtf9X65`A~fN+Y3Y)ry@8J1w|GqxeV(|w3TZSYD&I{NOdmPoZzOE z9%DZWCNWV0d<<14P?^4@2XPWB|Nnrd=QASbSmMg8#av|O(s@_4jHs~-7f=QQI40i3 zoGipeUS=WDDk5oCNVO-0ASk1dGE9@S+KXY3Fr`w{`50&#n2EF=1D?yVm0Rinf}pAu z%wef(!yDNddEY_C_V-Nq{5*v!jos`*QESGsFD=0uZ8d_|(D0O$|7h znNVyb6G>vbiV9{Dq2kvP(J1lPsZ+*;V9a%>$;5l^hlq%z{6k1Y#Axny^L+NGl!c;r zUousOeX8m+7e-nCztRR5qtWW$KuOH2F!PHnQ|PTid(N4Vik`f00@DcRV-Mu_Z-1z^ zrrpGt8mB1?$o2h*!ZIjWlQn|ea|FcQYhiwf$(&}|$bn2I=>kwy?yUTQzZ22_1FJF3 UVyXv46#xJL07*qoM6N<$f>}I!IRF3v literal 0 HcmV?d00001 diff --git a/python/nano/test/resources/train_image_folder_png/dogs/dog.1000.png b/python/nano/test/resources/train_image_folder_png/dogs/dog.1000.png new file mode 100644 index 0000000000000000000000000000000000000000..d7d2c0e43d690470a191d04b95989ec931a02ad1 GIT binary patch literal 203464 zcmWh!cQjjXACDbs#AvBe1Vw9B)gC2UMUB|8ilRd8QB|>tQLAc?Qae_wYQ!u`e`ZVU z*o_^nl@j62o0H^^o1Ak`ZtnN_%qPLv@G%nuHv<3wV0xkpfdT*&N90#9kdk~v-9Y{q zfC2z`0(ofmsvwzU)9`XG>o#q(%|TeQtn}_@Z{Ji!-@^jc?b=!cxA%qH5=u<7E;h3f zs1d3uZyDg<{@+aYpjhfSx5gNm&op;p0M}WvVoj{2-kJZ_yVrE+C&em?|3t7SC=Heq zmSF28<4T0Ych+p$i4*hwb@DSg&dHa}4PpBpbH^|UxEEMNkQeaj291A+=C zuy*>@7=VBbn@Oe&V_#4+;_`TIbo8mDOVeB}u@_4rpf|JvyH-W-%o)S=B@?>iaQW7u zdttPp-n(MJbufPct+TpzC~n`q==s3)bmNR&p19XEu9e7XR2-n)U7TfmjKN$7osf3o zyD#o)5a-5kkn(h)$cJnc5$h8K6ry#u=;ha4l%=%vc+1R8L$uF~4!xPbl8Q<`LFFF0 z4M7?^68+YSkI&ylZH^Z&FRNujK_T{f%LE1~Vml8(x~Ti@x1)~6{&KPSoObmSR6}=| z^^|k6|5(_v%7oF?&#T=_$3fE0oXx@2j>FmJ)kbA;*x8PJ>@91^77q8H$KQ5aPs^)c z-TzcH&VRC8ebgYGrG}sW2(v$Ll1o{g8|_)2p^50qOem+0Fg0_exq01G zo^$eeu|_BX8!Rm?tqm+uA)e<;6NsI;Z`YB`D2!fAmWyD7+jvpJgu?Qhk2QiZ#09kI zrcpa)f^b*o4yICR`QSTN(Y?8;sj2J0L%jLM!+VlPs>2UXW$9_|#xbKxR2BllQgL$W z?5t23rnaFBle@15-Q3*zWO8l-VnBconNF@F1`yYZntdBV>d8iYyzio%|Jd32xw7tdr$=}`G#d>i%n_*rmT-+sJz^mP>^X~89hK7ckg^VWlnjP0B z)^w~_cBN9>1R+eILjCC^hnB9m|Zqo2UElrL2=k zB*he^O_+yxdFC&}oLqhJBUN}@?B-*vH*pC8Yf zR#;``{{Y|v0B8z>DYd9A%tay3A=I99R1qqzC1^_p^M)GC;#8%rFfA1<%Ly+iOhq1s zk;yL&E(uZJyh7}8g>XS1*E620>24DLMe*f$+pKGtqvqs&5m_)83;_5*I9PA&wKM-@ zP0!U4ixB{ij3DBF(O|V(rY$G1vrTt}g{x%MnmjqhELQnWJvG ztNlFqO5tj^JPV6B|Cx$7KDY?JQaR{28ha;d^k2u-@ubn!3jft5q5Fz-(l~ceNxn5m zmj_o5)2yZxV-y*)jf$_vuQgPi8I;ua3R1 z_BH&!Y;AXUMphRn0oI=BCZ1k%35L#L&dcU{kSIU+LgonB@ z+s#l=%DoHS>zD?C*-RV-f~u}Z_~nZEh?|PTi~LPo0TJPU9nP`d4fWr3`rxGFv!>wd zMIFSY)9oYguCV4gPxxq9nQypv$eemu6x;ku0X7`SDH}R-;uIE+K=h`ClomaW;r%e3 zcQm4wZ(WCBnrHKSjG3R7WR<%Hri{qV&6N=XunuquQ?ou1KMwP@(1B2%gk5d?7&zJ- zA0Hbt{E6@}G{)VU=`M!u=m>vEr3$Xff|vK7mS>*!^0+n!YxcWX zk0ei&V*?ut{Ka9|G$*vY>?2`Wh-TsLF>sFM-;w}^;-YoYP|?G6<(NzAW+cd}PECI# z84sae87Y9?Ljh^+{l{GL4)39|K97X2dtZ#Y=bd+1<|l}4>Oq*g79LYY1cj$ID5@*Z zx6B86C@$fvk)U)9WoEYn#4By^b6HJIND$TIeiEL7RbPB>DG-~)DV^o=5Mt_K70}MG zzX&N43?;e|qmXwv`~_&}#L6W?ayLb0{KCUE#*a|DsL|7l{ov)Zzp3FD#aGeG#KPsX z!h`UCOY!^&E-j>l(hNvztnGlfKX>+pBamnF?Duf?x~fW4|DWjr8^M^*#><4TdW8=&=%FC5v6=C{8&!jks=r%EE6CUKntNXW&Iw+Qd`5r z9>i=3Ao5!6!X0c%Z;(7P z>#GF}n`yAD9vPNlq;SLpH)Idg;0knvW73du>PlKe{&ksPJm)P(M5TjrD5i`9Z>rVl1V*yLgWNG@5E6!C*VR7txV!8}ufMv$K1L}`z zKFP^JzyM4hW}wYi+Eh%3#obibfWpLp=QD4QNO`|O%}xqF4hY%84}acJMH5Cklr(JW z&PQD*nMg8>zO#ytA$gnD<~9#3v=r;0tkxCPbsPJqZ}Lvxwx5O-w@Z~%b3}?M`RNKu z4i{LW%#%OK%={P0Di8~Ll%oR?j$p5mR}AgIVSB6V-ULuG{2Id2OD(J&AKZHB+l~go zs*CRWJu8)pZU583uR@QSlJy5G@l1QB`MXAA;DHFl(sT&0q+KND*$wE1fQ5Wa56V27 zO%4U$Bf>n;WBZ>2?C1b3EiI5tfBUR*yT0#>;}*L&ZYStM4F6#Q?Q9Ba@}QIvnNYoL z@y42{yK?WM(QzQ?PGv`9K%%07{)5_`8kk2Iqm^Wg04rr1y6~W+H^!V_S-KOu=LN{sQ>fmbQAkeObKl{e_oVM6a_RPuE z9@_u*S03G1GA1gBhA3kWz6-x%m#AZ-$Wej%-U~mT(^*v8JgW;{dVPRx(HD;d7XIO~ z=hf8GEY*&o(ge`0tmg~;{5>fOiRodm*V{hKs0gY4?N&^>=w#6_j@l0*{(Q`jU0fzr zUTufZ6FRfcKBki4r#Z=GpYJvQrQ9`@8p~?!;^9 zJ(Ro_O5S@qDO&vISLMGYl>^nb)(dnv;h;W4V2uUwCz_0nSli`h1E*$~oxOei@@tQd z&hYc&GX$@8Vs$<8OyTlB@1nv&ik|zDfQ=M7y?rTTGZPc~Cm`#zc{MtJh^Vj(ojgbI5SRGX}r3OC%7Ig08o?x$;lS~80zK#wr4ssx^g$E*#Y#?3{#rVVm)+bi!k!6Z7MnRDS za25`P29h12phmZ$j!C8h*h^qu9~UBPc|p^0oE3q|Hr^xG#F{ z=H?A~$Ic?Mc$O!5& zlbnHi6zbc9f|u2sDSFHY0Z&YMCU<{j=E&Z~MFCPzeD{Q#)y8mAJ#hVUKcfatm3~8i{w~u8md)bpti6Ac zd-3`+fqcIIxmF6yKyPlh`UZiugk2xqrhXs?P@oFdSUq2r;=LJ&zM%WI+;5?=g6gG! z^WVC~?dH8)-#iVosra(6k3TrD`8EZJMb3vl7*eObTv7G$D2~80+YX(b0ds#DjW^yTeUhC9|-waB_0iybcdPz|YLgZ0_#6HiaM%A<>RpaT;eojQp#GYg72c4xDixXv;G* zUt2cS?sNY)AzD}O^)$-ghmN|@#TpXBRqgV<0=TuY*vNsc$-mj!D)#|N%^}S8QIi9L z)K%0i>ZZ|R{MBHYkBNA;-bX?N7q_dIDKhq*ot+^HDmz~M4)8w z`^=MDLkTxz%r~qBC_b0kiW#cM30E|R&;FgPL{^oTvU80cX+i9NBpa9{RV#Bd_%qr) ze+W6cV@O?rv$zJ1lJ=UDAgiA1;M;*R^uI*(epKr8GTqd!!e3rKJ_skC?|3PU22#YG zoZ;x959`q~1eG(Rb_Y#kgNB6V#WG@aIZ_(ZqGz8@X! zBd%ngzC$M(M{g^Da77+iqz`Zhk31?cFKh+S%c#UDWs_7Roo!T`E$VJ+0&Aa)j2IYN z@L@3qy+WnAf_Xv_)sfZtDoRf8%56%qG1;dl?!UWDuIK2)+!0u-+0|ajaJ$U_4*c=q z5%p0f0nPVvdMkz5L`%;-)R!5cb5|zh^tO3)R81B%FDC_Q@I_+ywl11`Keat{tgZa@ zJ~i{9ugXoZ6F$B~1&1B}96-iH<`if~Zo-bBv0#v$rN_$xyR^mYtU7D0Be>?wUtISA ziMq-77pVy^KW@C02%UVMGK2tiA%R0!ro~a7!+a$Ghj}pd^FbSCsMdd&5mO z7O|jj;TQZ@gX@T;1A?g5+gG5IRN2d?#=(SlGXCI$eNjWFq5OBMhXjX<7XUD&0rTwG zSq|!E({t2F_4MJ|KoS-&_&M9o2ICXg(~ zF8ktrHEteflji1P`#CgiH~X>oQ+PAbAJH8r8*`r}h#^19Ya*kX@>OJ>i7pbvKS$hc zP3K{KKQGVpCGiGR44anb#?f#kEl^jmY?etdXkQPz9F~Zm3-P@rc50`(YmuK+={Q^F zXsT9AXrii)p8kP3Ek~mB?gD)>W_g}*%%#pe9Kuq4G@r@18K839mu7aoFo%kQyHSAN zKUZHbtT>LIASEjkMJbpGH4)c?G1=KYML8B~F^p(^qmC4zVHGqJp=JFVW-f`lUI1Wx zLctgb%$&{Xkw-g+4lF`|06**hzKn-Fk_nc?)Ud~;onuZfI;LUrJOcw0*A4h0LAi=D zG;Ft-X-tHDfQDA1J(3>NdtT;RhGP+0yd!mH@}|_T0>Te(3(zVEP?w2SRlp3O26h}z z$$A&lwNQpwnV=G-8DsF5)kXsaV~zb?!7@i5Y4iM<1to?L|1=s^H#DcLtETu{B~Ycy z(Q$5M?L;|K`3f6j;E6a_`~RHY-mZen2Pj;78TJ|Yfii+JC$GjrN6sG{8cc8s~tcDmKVN8qQwZ$-gi1iP4 zdF0p#n$K;SJq_(9gf0`eh?Dory#m|iMG+)L{H+i|et1z=DEMb;>R=4%Z*EHjHHNX{ z&W&avx5rwK`!hDla9BUnw?B$Gdo7Q@Wnh!x`${XzMM;pD@U8fZ0Qj5ON<5k|2Khoa zLNR51W*N~m@9C60)1u&S59N&5>I`sYt0Db9n?5`r_rX`ZH(bUJ6c`sG+gdXRvLBxyFeu+C5`Gh zJ+@v&Nq47&DU6+=yt}J+2b;jMh#r-lYU{XLNq0}F*0n&QZR-r$q|=$!s4j8zI9h8 z59z#!73ib3?VlDhrhvXFQ(&jwM?zyW1|L`)nJ1>+NU{CC(+T6Zq-mb@LDWB#@l%zcQA!b^=wHYQ zpxQ?ipBJr=@)qk}d;M^T*WAF9&t&7ke#EoLO)g0`&#A6?S6M-0VBX=`Vz{AFcr-S3 z{Wtx&(K5UkMUu;=9Oe3Gzo6qJu;RN|=p7f2vg|$@CpDSqntO~kXNgNo{`Y{U!uRyl z;wh&xcJ!3<8n4$26OHF=1`#78>#EY_X^ko=>s9#tQC9^#>t7tswHX#G)k#`HyKfOD z{aTjCVNPQ-D~N*W{LxI#XBW79@A31^fc+OrzYp*xZte_=qqU?%#W6H?thORE zM*9)BKW2}uD(MGnYHl2HiXjzg#pB*jQaVN&_f83iR>5Od zewlMsdiu+6YChJA!YAv-fHQ=Ff5{#^2Cl}K+`7c5UHo-n|2GaxUSxu08O1kTG#Ga| ze7Rn?NJKg(3sRS9ha|RvbV`w20Gg7BCZ61zP6COHj8f0^MphV87Njs;5n2@Apw-jY zWd{ZN?(i(7I=i#ziWj|&M1xGX5|z@qGPrMG+FrNJdnKpU%yfQqmDaBwwnbwaIsCso z)yv<%S+cl%z9FpQX2_u>+CGg`PFEzZ@{4@ zn*li7&2Hc(-S2V9h1Y@Ev=uqdfn+n?XOGzB*{rXxrzR(KErTm9Qo30}L?Q6K&LXA< z33#9QEEA*!(AH_Dl|>r9Fi#%=fnt#YKoet8n$VF$!!P&`fvSFLU-{Qd@(J)4Da{}5 ztZ$s0^qq`emZbXn(!|{HT6wQ|GykrubymE4u|c*=I#)#FK`5&`+<>T&m!B(`ygs}2 zrq2S9Q&BB=qL@M?A-b1~C!TlO+ep$q+M_jK{;=X&Q30O^in29tw!Vu%!qSmk6C{DD z#l=NHoNJ+{%oJrpbo1TaUN490Z_|J2c++Q&~TK$U}wm3xAx%cPUo zMc0_InH3&jR~aax6+B}pZQt=?9tJFFE&i{qQs(H2<#X}#^Beb!2qZU z1q68FnV;b_Rq+SYeN8hCVW%JYNohcI&(?S9N`u{@>&yp1uMvh%z6**!t98{ESK2IX zo*PAOot_p&soL)2Z{cmkU0Qf%c28Ks|LOP1jD!e+32o-K-$U~J>sNX>gl7@=sUPcx z1rv7QTyb3o4cKg#gx%F`GHysqcazTZ$?*jiG4J_+>~tF{)E23Qp0398TX1#fWr3n@ zcD&l$r$hvuJwKRu^5&}VKprgrcKE)eTi@C1=FWV681!~zb&?}Sq^{XeAA<}1CKgjK z?=};q$+!hevKUaBal+A>VUnWzbshPl@c7`Je|-m}`S9ItQp;t_)zTFn<=xfU*)2UC zY)AU-aQS=rtZw;oUE}g&L)ZCEQ*pQ-dcn@YF6>XVgZWUtsQ1AsalEVRyg3|a$+>=p z8U8%AAEa{k^50}bTz&Ve%e_^6*Mh3-s8>jfRnT|E5r#h86M?K0Gb|%eGfh4ljr})v z)bd}4Rke73s>Y@Lvx9c2z{sC;A>ybcQw zUxtX9*VT9~9j?_jAB5tJvv-z8Qd$@GohSped*GNm$}-Ocz}l&@UsdyRvt8Vwn#+V# z>!wgtZ)yC8z4g|E4+|bec>9B+Tg287KW4y!y3}xhLh8s!*lVi_hwxoV^Owf<=F4FO ztZ$J;exSS%>$tU(t~kaPPe7|zR3yfM@OZpavfT`JY3Tta`Mj`wkD^Vi*9`Ddb)2b< z+wz6a97CgMZHVmytE}=*paX+)04`pT0RAT=W5}RbYBm$>!|oJ9Q9m1NKO6GsdnP` z-jT%i$h!c_JnO))&BEaNdV5kff)Kp?`Z6H+2;L0CI&{~&tk#N7FE4dJNdo(=Q6tRG zC%N>Wpq|Wws!E(28`;PNdWgpu& z?sv5;P#D2CPqv_A=BS%s@?goJln(IxSs~P>C>`_i$L4{;)MGg7>uE~Zmt&Tkn`&{b zJKLxR#G_2@`;%(%M^Mr^?y{+&t1Toa(ciXEfBxbRmPPOwO$s2&sI|R0Uz=-kxE${O z%M*Uma@pB^#E&@pJ+s`=1zyv1M7m2{o_mFVo!@W~&YpOg2&1uJyjMFm*0O>Gc!}vk ze!c0~Eg`$fZbs7YRwi^N*DQKxC#L9uKl1CHbapeVd0saulMyVb28F5-%yqtRCQNGx z5oTLVLZ=>mwqX3cgKmP|bqo-CJhYyfgs*fN3EXNaIHyhlyA;*7m(|m;IaS|01UEyQ z_~j?RC{tNzYCpl=nX`(2D;cWk|Ep1t1fO-la?6;kObmKGV|Mjon7praFA)^g4+I#B zfqANG23VbR*%WW5=VQUt1i5;+{MWNNw58dFVaG53;!@#pWTAB9+bxs_?!DjcWn_Y*?vr2zo zT&TtX62JM@)8s1Z>bfIdA!Q#Z2FO3w6@NK4R^QMtS}T83e$s81*+ZnkLa4K=>(=0> zlcd7J!quM#Ki7;uCLXD-2d75g0Art6*F061H4PLCDbQnQ&$|-^5sYE@9h2a5TrT5F zqyQ+pKa_ka=BgQg*IgIVPY!U69{@ObDflQ>-_o(BcopO|**rQnUnZS;U!ArVcVFNZ zx(<3>@+!E5B6^G2CsP{=p!Evh&bQIWRmlneI9qQmQv(!{hINJ$hz|Je(v#P}@!+y% zkaCn#bLK7H3BJM!#scH~v(01d$BN*Q2Ybg4d6^=0WZeHu(PwDgO%?<&+74NYvGg?OEFNLZO>3=^F6RctMtlXVTwbnZkR5}c7X(B;(x30ZQ(!G1D zfFYQGjz7Wix=YJSl_wu97LAS8C=~Y8;%MO87WqexRbq)3GKM7wUwK@&8G(md&D4Tm zO;ehxX6_8_&7mE$!cm>b{Nz+F#*MiqP9;kP#*9T@>%)85u)4rRO2CKMw7fnfO}x1n z3^SS@X&d9rZL!--Ss(A!!b2lKZOvo{6nxTDnTuwC&IEodp-)nl6b%w$)6Q<661sN7 zJkTjC=X1zC8$aKelh>WFx^}5)FpP$^mO+k|VfRGYCxE75C+4k&AIOOgU@DveXH}!n zn|`xqE80g}bTO1_hKm*|H=1Cm49gty3JMDo5yulk>C6?hjVGX}Jc^&>(4V&?=hQ9= z_T9H2YmW5*kLS+9+B;~ds*bErEnZ_PG{R?}>O{ZNxc^TR^`PaEBy)TUm%1w#)6bf1 z>7oA~Zz-Gck1+Cb7t|;-w9p-JG9<^84Su}o3W@UZ2^Llc?xgONF9CRZm~Vfxct|6s zf;<=mQ2^o^VRi7)7_UHa>ttP~MYWhe(ihjRzG3(K_%Ht!ABQzuKL~EIn<-rDrC;Xf z^R^GM6(4Z$8Rj+mCk;8>ab-yPfb*%Ej)}{SP78@bRL8nNYTEB}6 zVNTY#SY^4IcOZI<7pXPDic~|iASg^tF!^ZszcH1ATqOu`ytyn{4nf>RCgR;649F2K)D(b*_ar z^7hSa%B zfl$i&Y^0UlaLdY3rq60G%DryvMA)kn9o=s%uH43*lWEQBYRU9D8KC322*fad(^?DE zeh_-j^Tka*$2;3er{(zsMVy#Qs17!ZR*0GJj90s1o0Tzu7Y@i%jFAWpi#h&+9hcsP!ECEmOp z#)QJ$DL1~BrAJ-=Xe38Vfsrcxh3wsvs`{@x0Yr{(yaVh`sRC(&F+Me~=Y$k)@sds4 zFOqUiA6qz7q?6D#;w_dlA$gnb4jwxG!Y#16>7ic)(rfe&C8NpAcCYSb2du5eUNBZh zk&=Ehq^+}i8M{2bPOh%C*INrN%zJfZ=jZc4JinKWx;DZ1xBy?ofj$vL_Pd_T@OA!M zLi4LscXD%W0)t=2>~RR2TbX6sYPS|<|Em4%^Frxj;MTBE8b1}lr0M(PRo8IM2)Gx# zrXDw)yas#`wDp@T=~*cn-{rqUGXC82=?y0YhZB~sST41Lz3u!`jaF2Uh!yH?m5RRMHn5{qCR^uPcIQ=i*MjKc0 z_InFAh*I!b^#F#RFD13WT)v6aSKqv6Z4QY;5)@jF6mxGr(R6)zow?3J0lO@{!n`7)6);p9nsvFhPXAsaVv={FBo2I ztvss0d+EKvTOJBm~}Z9Ewi)g{rOK@i=#~WEjNIw8!cb%0KsVJZ`ollB@HgLdC5q zScd>Q)M;-~QEk0Yzbd|TgOI!mkr_(i6q&K%&k~q`>YK4Bzpm6V4^5s+>2&-5X@@r%;(z1u(W&a5b{b*J!LoUC6DLMRs&iP4y&&`r z6T>FzESKDh>S=E0J>n%tjM@9Wl~19JR>#<-+}rVAm*I=uxrO<~cIGUUQ6>*;{Px>X zzA)Ip;af%j4qe2TAMB310xEDmYQX!6h%I0<@>RrlZ628%l{_U`#hf`rx25E;>gbW` z_|d)b<`LJE%gaj*;`n6~nIZja$e&Y85hquXFbUax(riJBzpeJ8HE8jJ(1q>GOH-ET z)O2i*;-mB2wygZ-JS`VS8Fm>bHaI}vXnHAldwU{5ysW@Q5n5G@nfXs9j$R}B_+oWx zD$pmsoj%fz;{$7kJKvk_`9SWZ-?771gfnsNqDfMui%tD_D&NyrR z&LgejdRcYw?#f=-zZMdjtpg9Y`QJm0}D)^(*mV{KLDN2T_9Psa|z zkC%EcL@5DEZm7A=`Ulw83zha${`R-!w5bP5)6wyaj&PMV=)2W#yAyFad9 z7uXDoY{Cngdp&|d4z~1VXq4bKQeVAh#zFTL^?Cy)s=np zeD19rt1R^)RPW}XiLhRBO?;b_CbRBN1)}j)_GeHmEOFXtZw2nkZ0|u6Tac8+U}2{u z%=4NSAn)JM_DcM%vhUWn1fgQx8?1$on7=T8z^&2?7I=SEmWG(&l#UBgV+6FZ-IT1G z-;2JXH+d#nbiB?ZN&Pkz08|B3q19#Oqq7+kkk-Msem zQG)K^2d_ahoUHuqUr(VI3mw*V1-nhrurg)b%uZaFXe<;`ip+87veoCSiczi` zkLw4aH+a=ali`QQ+mi&1i$C3G%l{O}{fU2_8vjT}crR~qi(}Izwa3yj*rMvOG=T*j zPtQ0`ew-g4dz$Y+Jh@*@KGrgBgp%u!MH1hiV{Dt;pRmD4?O@}wQ7&@w4Zw_Dtu01o zuNM{{=GU(GVoia=pI@Amj!E=qv}0Ntx?c>W)$ckbBQ+_I|8^U?CB9O-#;ew%9~9Q_ zwl_GamyK3T>mAi5bCfMG^8E_UHY!@e4tgQ`Ku`t4Vd(K(QYGw8d!D(RMZ+^FeAM2) zc6_?Sjx9)9jml*H?6dj9hH)oWwM2@mNQ3g-LXxbbHq4!ed;@tx`tiPp4aivG} zQtC9HJ=O+NyE1~O&hJoFfGCyPqEOX4P^KXtKET53@9e3jbYc@DGNOnX;P&=z!I5Ih z@Mn+-u=n)nvxH!bZ#Qc9?tQH|9uNSd7t&CJKQ;cYSt{zV&)}Dk7+_&Pbe{BMX>iyZhRUp|K|0mL`75pE2?xeq*QYC9 zFIg%VIRInRJOEI->}FF*Xb!&2>9(T*_ZRxEoo!NiX^2n}0S zFr6|>su6NZBn4==NlMaPS2XLat1>_N#eE|!C$}d70*o~Q#zr9h3}5C6fN#_(x%n#` znRmC78D&I18a)c{wDZRK-YHk@o>j#%W(b!Sov_80>Cbtpl(8&@{OwECxIlIP^|<;G zetdPYu5oc?U+{7%ba$ZAA^hvj2esv{*kx>TFfl$}T<>*j+fvcX;{lPWQhA$PV@z*Z z65igUUIJp9RgU{MLV70(nI*V)Z<4eh#b8O%4U86ipOD7FV6%HVa?~mdo(V&&+2tTv z*`)r=UvN2Ix9_FaJjzzAzD;&b6Nq^%7F)t6_Gl*Xj*WPTb=EFX#jVPVg;-ej>S#`s z1ua`QjfJauKG>;Zp6s~xD#%AV;{22Jry8|?w}zJvx7O8-2yJ7AhTj&t9zzru1!b=R zBG@B>-34faQSy_Y0&sY?==Kuv4r?<&galVqfL%6J(fVKK>)~qj{TL4Yx&2CC*k_?? zAz#hgK4x`q`U3-G7Sx#EJj%hk4xKrE*_Mwy4Z$=P}<#hfIsQx0klZ zt#uCdUkZL>`@1)kRuuVUzUM$;H+9xB*>da9w=iGK#A@_Up(6e2{d+gk=1d}j(0mNb zOG~AHyjpC$6?RYBU%XsZEvJ5JO1otm976u#jDsfDqs!PhlD&gN3-TAG!8<*}lT&uy zzK>F8gF~Si0_U?-fGAo2XYSo@v}kvTGNbpWkLUIt@5NkkLE!CuJ{t$uchrB7#3h+s ztFn^^gKbs0?fo(FZB#E0@3f-N9RVBbrZ2nE_6`4c%{9(HYQP$1`a_DCj}!$joanZ< zJ(S0A=QPi~Q`G=p@J*-xt=~QUSLIdr?(qJ>$Vd+(^LnPdOm=(!ecqFLs?Y45k*`!; z^(!q_-~E?b_u=((7N$9xC;riaI&J^}t+Slcm!1bE^*RJ!UKT7PURQX8gL1HSs^DhU`2ZG#RO%ZQ$`^SR3)nwY5?ni-*5Rq%e{z;`Q0{ruiw@)oS=(BHrH5 zP+Y0BdYWNf@$|moN5TUF&m^^8S`^9lixn<^4*~Z zvG{_8Kgg0ip>HRLv2RGzgUg{Ohj}xc2CBOQ?V)QA-l}lLt$Gq~tcaBR!tCQm;T0iP zv62mtJ?i&Bv(4Ip4gl|G9W1PifJu7KgZ|i z-*br_?JjMmh5gAk1KUHT2Ew{pmTEu|S62H*jov-+C(=e^OL@VtVZ;{K@UiD?>*SkI z8uOWuRnk^H;_4gT-sB8%W$7G{@XLGz;{sN|*yssh&f5_Je;~V_kFExAT zw+MUn)GwPZ;MW^FV2!P*Co~a;w=~`qwHG{u|2Fqxb2jzwL?t55lBc^y9TW?Orc5 zf)Y^k;d(9C9qKa3^Cw#TgVO^#Kw}MI-4l)6x1JSi zw_d+!`g@uGP%pkU#gZ7JFj{gxH=ts+FMah#p>HLO|G`2Ffk>KDw&Dx6G&P}#uyn{& zl13^X?_%KQdP{FrC>aHPRxqx=@jChn0g&Tq6>s?nsd6qZI$8iTFHe&nHJyKex1ceQ zKK*Zm*Wg1bd?H8eV%U$PFC!-Z6w#_G-@c&P<5aOapM1M_wOaE}#mf=8$Q*xr95e~M zCR59}*m;lLG=JCQiIweQXPz{Mq98KzAs=7WG%SggRVd2UY`?d2-M3}gGv4W)L4a>t z?k!vYqVJ^cbLp#L@`jz&E5(CbBO#sJ_*=IHrXpUwSP*{Z_eeVsV*a-7-f#qUgjan9 zyk0PFLdSfxqTU>0@U3?Q8Su-E`P_R+P!zZwq%h!!tvg3&F9(<@m}tBT4V;@BZqQf` z!YWRCk`xr$T}8lZV^{NPs#7p&DJi%Vgu{x9#o{3)b=*Mb3wu499xi3~cWV@O^w=Lu z&$(U#9}X-a{O$b^rLddB);NG7@Jc@a=Xl`C3z}RDb(Ks-)cC zu=KKB1v_a{P~COv^LAZ^%`ine^xY;lM+07=cR*vnC+DD{r_Wwq`w;e%t?ymXbij`w zPiMW|KoesqRV{v}{H%jFO&3V>B!{vQ{F(3GbRGv&bzz;L6ih{XV4*FQaFH4@kicOg7dx|!%U?GsP~X0>p{Z37p!+F(#M3&J@UNn~3fv+ng|?Jtzeds>c@f?T|A} zLNT2*=+4Xg`y?wcmmfZ?bK>)p*(OTxR9T>V zH%3aQ?xd`-a4IJ&;udXuAm7Dg^9T0d8uW^y4-=3N@<+)WO$5d&x57%h{+x8@9!%(Amrw}cn96QJJnLq zd1OzA1V=9)VAYI{yfj8JgmuP#-k99ZTSZT9#+B3vy?4r;8=9A5D60Wn;G}q)RG(8) zQHoLrewao?$jrbkcsR@<0Ta;d(hXxji$iz&3U>{BWyh=Kv(|=+VXtiOc)EBO``d%6 zme}i_O6SlXh{k$-Y!F(uW^5{@RuA& zvtr(Tib6$G3}Ei`Kojb6D5%oX73rvi>4%<-X2aeNutvrrO(>s~fzk$SuUAoeY+b+m zdY zYMwk53Qu}@>48?f^fDmPYX+YlL++L?5qxjj3d3EjDU=?5_>Ksn5{8(!a_wI20ML$zpJ_<1L;q2XVtE`@pCS_ z9rQ4n63|n1&64c=_#J~h(<(dxLbvX|f<(nL+-i1$@+2@4;i}UCaUiD0SC$dSI|#MPb?T7AJ5YdZr&*K25~T>V>5)dp%1 zk2i_&e{m2};nq7$spTQvU1U&s0$MqYtdA!QxyJ41%azdmgw);@~Z z7}8zS)YX#J!>bwK;aiEqsB6+R$YfLbe93J18to3l28S`_AT{7eWUlmhyKg~7IpC8e zcBZp8W@zWpVtMA5al4t=9@$5l4VaNUKS@I`-5#xs8tb3}3`#ksi$Cs`K>3$!YEOv- zt<9aGZx1P7>3!Kj*Bcp(J=g8UGMH|7$$7#JVsfB+rT|mNJ!b)kkCwKF9YhmiMqF(S zp7%BM9Hv56HFHMgw-w&F60@LZLOq5ayXS%g8CuxxK6V7B+6r0opS@8`gWRow{g0ya z3}^Fg!+6Z1YLD0iHEPx#HDm7)tEg3b?=5zWT2)(X)QZg?B`CFbmDr=L5f!7YRg}KZ z>zfbB;W#|GbD!6Bp1-rBh(SpdMCoSe?d)6PyH%94HL%V3`PghJgH)S6>*da`eK6v zS`Q|KpVT_vjAw@>>_}>s2Fy;c2>f&C$Zgy7j=5Wox%BnBv+eR|e|C%SX*CRGcJ1j#!xEpTFXm&WOjPV60pi;%szno%O5ZNGx&v4i;|Z0`i~V?HG-d%OlD9zn$ z+2NV{h6IX-_$wvLlG8ifa(y58D!w-&YD$qexOT9nG+Y$t#4_{L zRI17fEf{|a8VDs$S9ffULbH5%j3^_ir0!+FN!>o?mZNoH)vyzr>IfAa8OV-J$tr|P zQv{m1xDZZf4D!k{8apVH%zFenNeIKztR%(D+~4X*Bxl_;aWUzy0-;6juNU~c= zkQd>BI-9}RDQGk_a8-DeD#ipg$ERnQPm=#hS!sOT*Es1#bUP+~&*ne$4#{l7vG;pf zL178~fPVCDAV_6!ozdShTNkXuJ>aG z?7crYep6R^5vlws75sjL($INm(PHLx(F3k@EDbQBBYW)UEMs*P@JyP9Tve*LyQ-*5 zJdd|lRj?Y_&H7))0@7^(!r*JY4>mcJBNV(%zAY}I+~~vh0LNm?*^Cqi%P5DTqp=3* zKY~yC$=NR3GWxYqA|^NXwLUwGKgUVjoL9uX$&1JrmLLyhT!kL)0(LJs5)|xPe0)J6 z%j%^4$C?E;_vJGuu|!7+)QD$jpi4aSdz-7F|3DB00#;|AZb}v@IZyhJSlMAo-~EKIb%%Gb*Fl6P)5wC$h5ekJ5>eF0kg_qIxqc8AaZC49 zCq1;GTaKFeHEy;c=`I{}NW%^0Q4+C}2&Pa9(Cwau)1dmXj@w$AGGrgVvENp`@Zky` zAM~40!-{WjC}x?=t|EYKc~w^Rujnh^fC!&fVYn8Alj4CIl9v|DOU3h^NNGAwnWEL~ zxnVt+f>TuUCspi*Wwo&m%vpb&x1hykpeZdq0g>n|=Cz=6dHcZG_MwoD*%$@i=s2Gr zknUMtHgBA-zOKwT{z1VNSC5A)n#IHbewp~0`%tN;qkcK()y?LI;ymEkriR3s@L5fl z7J#bV*zjWCgfZy6MZgC=8gH`+EIUquZ&UiPCZrz@sd}CoCz66*wGWlIs_DPFJ*LQx z&kHYz-#b~UW~7>@BF#R`sl;FI+1^y0Z78B37@#aLf&@_MlCpFWE-%SGq0r`YhEF3? zT^nO+ytYFQ20Wuba9Rz9JQ4|HeiEv3ITm=aE0m0vCVn$Z&PHBE~Xf0UK&*h7AO)3!qSYj8aL3AYqXiH#@EDUiXW>z>I2)|LG* zeR*Jmh;~O`-Chnwm;75`G-JcNBHl|u^rO@ccbX0Gr^#*1lN-JFR@QF*M1jc4yJ*n_ zt!ck^e}6*iqx$x%3pm@s%*?_ydAPC}tg@2=Kcy{|hkx?slTbZR7nM6_EBWwOYyYJq z)Ksi_jjGfhe8URtWYmgvHaFKp$5%qa6#LI+AU}XHiu)^VRd)Y z{O`uSnCHrXW2_53up^XU1*(Be(M$&n&uz)q%@DOJb~fqT>)Fz5J-_)7RExYy2- zGNV{+R!z1=E1&Qaw{eY4GpihGen6@;R++;3K76Pp)b!-aO)4-xO!j``^5E&?6lfV2 zJb_CcA4=#75-fHk+v@z`*J#I2rmGGW(9d8_qyyV=qNy2mAM9?b6g<4FI#lY z629GCDQ@ZQhzlh-5rq{UM&o{G=hGzNn_`Y78Xzjv(o~V;VP_{6WmMPrltrTw^gMk( zCxF0s=U5@L=UggXX~?}c;Scjk0q|Yhxr9U$+f=p z=wQjnpU1P3Zf0ITW*HkS{=|oo(<7616GpBr=1@P1q}Ld@esI`4n$; z*To-dvCGy{CnxN*Xj?;1PQ0If0|HmloyTARjibX?JhOjvy^B zK6c-JZwNVwZmUw5eYIA2TN%Z`R{rk?+2KxzT0(}SLqgg9TBw*W~8gU zvMf`Unnb!(@DUg-1u6MV$vCiGJS&J?MG2GNZUS>O^A7 zzKdr_et4IS;o2Rpi?gJU*e*BV@%ExCgn^nnM(o5M@&w9-li+~(B9WQJ zZtERI2Fx;}6czP?Q(q#zYf`-z2K{GA?hnyjG3OdFz5yZYBVbh18&<`;a}&ao+vtsB zhHuB1Ily?_>|^_H;@y&{!FK=OaOSbQAsf+;sJ?%@3-4@g`YNOars8ONngZU}<@W4W z9Tx52teOvv4%F8Pzu!+%dUM>;EW}BL!|7Y^E&BCSRg|Q^l%X#VS0K;8S=#aO(8;~M z0}02)ehUrL*lNQiHX-5GE%0kWk99ibew}o|>tbWRH|Y13*!n0U`?puCPvMFAUCfM? z=T}>4a{{o}>~8J0q7^8NtoPiUzEtyktuglUFEH0r`mojz+ww-(y>VpB*i`l?nf3vW zsy*&giFHrO@BOyTIv1YPhy~*&#XI+qnB-HSX@=7RC*8zD3>W0>nK{(UVaks$4P>fIbbf;ZHS zul@t#b=wh0I;HC!`wKmT346-}2L|!P9SyjIdG_GnPalt;)Fr^GYt~8k!JEJDpy_m=ip_XZqsz2Q4g?+n>F4^zh@}~WPo;mgn?v^

y^Jnoc=SJFoONs1M)m#uWz$Xz$q(hRcmMv> zEF4GWwCMk^XN><8dNrW>KqoBRk1xZE=~cCZl=l;e-hz}I!ql4xG`?Lp6>4nv$&Xwn zau*$2X}0hld3jMxg6z^eRY3@D`#e$s67^1I1}Y+h%JPNPZXH(LxAJqMqxRsYCZEDE zejN`wk2=)}k_5dbx7XzRGTYBKPJ>aSKZ-VxWJxMi%(X@NRxR1RoCT6T#zgid7=Ubd zrQCo29W&KN6h3Q-YSQDl-@59*O8&Q-nO_H1k~)W>8#Ifjq6N$k{SL#K3LlOk9ozqt z;Pu&bfiFsTtk}!k7UPV3RYLp$5G!{RnKAMW|EKYNpNI}QJO6zo6@4t+2Vm@6EqmcI z)C`7vBcOm8uYT{@?$Utepqd7qOu5S6e*aXYGh%77g)XM8J!ZO@VWsh;Edo>h`d-ne zZ2}31{j*-zy6KHiwag*Tw<*dour{rkOr_ac#^ z|Elj47-99@4FQe-mv@nWe?7PgF|X2n5fgT$p?KZiwHGagE`l0AF z@R*Xm{r>G=gq_*67^RGT%b+;U-=irlEY@5FG%7@f!Lh1mVpi+i zB%7xBFAMkEc%3Re`x#^}weuud9-Jq_)yZ0kslx_jwv22ctlp~K zP3Xwgz3K2U*U<2W9V?`+g3BdVo2ZLtfp0)M_)6ZRdg3wLX{4n?`)E847!r&xFuU)Z zg_omw&md??F07+@-K`Gd^L1%O$CfW~h6O;Qd)CmZ9IV!sn9^e{}m1nY01yPX> z@&jsvk)>`;IZo!crnIwW<7KGk;)OUjN#^8loIwLxneauil*~k%m?7EXNAD2uz4u#R z{mO9rwPwffJ_t(u(PAW&)y?P1b}mj49hJ-}KX%VKdzxPnR8)O-mKNRN9^h%LJ=Mm% zsM+YeuSu5t%Aoir&bOAXL_tNaXy38WitbD2>t~9J0NrjPyw5w6f^BQcly*q-P&P$_;9U@g5xKO6r}V(NmV0&KngnGnDXM0b`ZO`iil~V6+>T|-)0aNanJ+e zReZ{|Qdubz^ZdVp>L1^}eH&hV^q8P}B{@2ttHW%uM5%c-A{Oop!zsL#NbYy!5OL1H z*JRAxL&)|m^}#25RYa~vb5!44TkeHiQs?RYma8BGH8v*nlXyBw{<85PqVl9j%* z2l|K8p{`a5m5m|^U)`am4J9Wiy#Ms8I{SP0J)v+5<+N^&Y<8WgY)Tn_=73f;YM!U! zEhceM7IQ?=KP_3LZ*f-F``hkIi`I!Z*wT(3F|Lu1J1X81i^TX*biZ3R-}rBg;UIN0+a>lLpQ7ayXa>3=V%HQ?WlSyiOYa-)*?5H4oR@Q${8P8I<^_H zTf*`!H+L8e07C(jn%aAbdv;dm&$siJXMd+FpEvR+Mr5`QrmpB>g z7_VpX%ga&*Z~Ok-tox`C#r_$(+P2Q#01w+e&4VhE}bM*92FCUg;QBt`2 z5yk8kKV$zbSryx!{|RO0>DBuX6A+sI>*Pcqn)eb28KKjkMg$K7g<=9LlAG|z1mRgC)3aU?I!lYa9 zaGKzeEr32Xz7tnzeG)R6Shc z3ywpR_$xhb0v9mMfSHefB77TLPmxR_u&$~+zqf0zOL4(U>>(j4WTewspSQw$Y(ztC zM%~P5Wr#)09^eL;ba+DWQ^}gRg%5B$iqNEH#Ma4r7a?bmXoWP~jy&XT#$- zWrj&c*$*1j(S23~q~lOsB>^`(t~s9~ye$-+p%R9no-2(L$ek_X?#ridM)dO*~- zDQf$wPV8q*=xVb$(mfBCNKA+Q3XFGA>S|eM*4T}eRkq4A&fCbN0=*hRHF17g)Hq}0 ztPv7%Y<$B2i~2zab8-H*Vipvy^GwL!S+dEw`oYqVC098P$Wmc()fZC#`MSkcGr^Zq z87sPdx-!eTL8WFmyCUAx^V3$D}w{;Lrc*1-3CJ($^axif|f3Xq$Bes(%w5>d~NL#aik9$_-xk!uXZRgeszl2V%l znq{m6<;>YM2ihv%xF*tb7^=c0$3L5?NDmqY>sT@?L3N>LD#L&?isOE0XsAtq`8r|T z_lPB$^aQ~(xMuq1r=7TGLGlDPoOcQlH|Ib09-Q{w z9(~{V&3k}P*5p+gro>L;q#h+EafH8@`B9;CV2A$K2OV>2yp#j!ypya}mZV+gt^tA& zL$LtCj+YTtx&Oy8miz#s>lf&$+%F%dJLu|fYGQ6`&|eQPW`Ky^7IVEIFKS|+>AMH~VDkM}E1=t^TcKhHS(_spWo%Q}&#)`uNF99)>CwhRgz%$ZRiZS=rU#g#%#$2O;IBi7UGk9G@ zt8pJyKx%A?&fB`@e#C&N+785urIeyb4wcr_4)Byk1>c&Fik( zEE*3Aj9u+2%%+~=Xj9A*O+8gMD&--MY~cf^-`1^aB8O(`^Ucl8!4;5p=#Pb*Cu0dO z!}GS1d$`2$9EBZpKnI1K4p5!+XU504rkdHDul*aG5J&8U4XlgkR~-N?g0ZMY%YPU$ zIKH$l$T8FlHblfj+!@X)n2EWj(IID_IAAs1wH(}>A#>ekrD>{6a&G}xH9*}U^vcdpI&}l)4|^Dg=2wmvB^e^300|b3*d1VYG0N zCam&}J@}FwHKy9kxS?2m3`*YI?3Kj}uv4>Me2lhfeCS|LqTIGbp|Xl%XfR_^S6)2r zeG`F=i6F`X8xE4*X8lm;lQx|k-$&Q7qY~B zp(SeenX>u@%harXh}Sun1&!|@S9OvZg5K1c)QT+Y!*M+H7*W=&n!6?X7m>dJ3YS}9 zjT^vf4Ez7rL2N+?YI6oHkpoEn@vvD-?^AF8x3G%~Y(C(^^IBSI(|tK%-rNtXwDJS@ z6iCk6PXv{mi3_Td9{;UcO4V0tYVumqnU9<%EuNeu(q$*o)z9Ov==y<+m<>G~BI}lx zX6!KxL(KT*VDW67gK0n$Vq}@Q6ex_Dsv-C2D%{p}2ND*|sQmx!bn_LtP-?d8?*KR&-@-QHkJz-0izAVk+ z3!`FPpEg!gPf>YDjS6e5Gl)9ObBna7wsx7@>B%hnnuO0gt*lBSkWruO9fFdbahyHm z6*WCu40&?-_h@4>!E9pYsjBT4+0sXD6go$g&$7fS8OoO0(xHQzGw^guOB?>DCcb3O z1R##ax>FpPPEii1{(B4Uj=EaLjjhZb@&9MDhUa6`Dgyfw{$r{skkz-Dm8qwl8gL+- z2@xlaM>ZwltTD4bUeM)@vX*mgteOVU&Uy<%`K#0Zv&vMI%kJJ_l`$F+dlS`D4odb9 zD2{IT#>wgLG+f&G<7I{JEh|M+?j382N7}Q0#(8F^$Fdhb^Vc@M9*+eaYH3RoL4?J` zrXr0?!$%8Ho2L*L+eWEIr)0#d_FMaS1YW{3t5F}_Jg!(p>}pJ=m-+ELVEtdINt zR`_y24#HDscAaWy!rMN&1l7XXkI62MSo0jbW;LY$WzzEE*Okg^@O z`%^mRy@+sew>H0@uhS+nylQKx6*ljJV+Gz zDaspV)5V{L%li~kBnQHGcBYa!niSNt@U)%RFdc~Jf_RqX8!>D<-ieFNpD zM_h~1G9TLS4_+-j8msm%O@c?37VOWn-fJNp-)qsHBfm5(B~_ zdWm}Rno)wDTfFg19MkTt)RmXuQCCecZ$gBUPr(ywV~4Xy4st-t#?wD3fnWQ~&86V6 zCsQD=y$csXg`+ZJ(s|PdWGtiAmrytd;p@q)PM2dr^kN-J;}Oy|91tpNvQOZg##i~-u=SMuK zcW01KWT^;<6Ol7*Hp-gIDFokTKfd$x*!RrGwIx|B{ShVjuFXUVmR{Ds?|P4552mM9fCJDV(Q&0}(U3&?aE;PQlAn?)|KnapMt5@}q3x z66Wy?`_uEJNZ<1X$9qg>c#=od4)$lIJi0A;X*DA5_Oc}M0rp=>#x@~TcnVkG|1sNy zPX_NiXFSp5%TgiDFZuu^x))nhS{l!pyaDh?7vD$9z2^lg30}5MVdi4)S6wie=Q+}Q zH@mCrz!vx(AZCAQ8EH*7vCSKSv(#c`Q)~)HFUz^A-r{Q+Dyt9{& zp^xBH2xqv`2z$T)Xr8p7yof_olY~|Umxh`vfuDdo*K#!tk&ZY={JgyOI4)e>-aHXU zS*8ujV*W0_)zG#s)Lr`f@9tE_<+Nk@W~r;+AXOWOkNSvc`Uq5IxIPN7;e-J)Zd4Ro z!2Ih^W>C{Jkw9*_96Qwn-K8SEZcXOJ?%0AS7}a(Qui2=m`Cv<$kmTE;@*h1ftxU^} z<_@xyXzQK|OtTgn@i`A?=`*fgVw#v}+D-7u)Ik49k2hT3%)Ih;Pv*Pmv!MH04;jGU zir{&3<;vr~_w3K%#a+KCafgVVT@8mcPJ6eCi5A`x755$_YoVoAH7LGbiEl7%T!94!w3V+IRZT*zkH!0oz<-cZ9t2B7E zcr}hq4sR4(Wa`@<@>W1I;;)X1_Z)bgqcyd(Q@Jq}{6krY8?np5*5*(C-SW?UZu*&$ z28}bRO!EDg*-~Qb9Nx>C`q*xz=7Dd6UDrEVzGrd2x>X*4pm4{C5Z0;w<3l1MCOAPE z$hGG_NKI3AeXBN_ml)i3ddibvN%N;;G6l$sr81f$1guXn;k1ch?t&7#p&eW>p2p@ZyksuYFSW zUH@~z0_Q5EBWm9=LJ8_y^ze4l$V#1_E3$SXQ2X)XS2@bU=N;i8At6ywrB3pU!c9#S zlq`yJFYf>FD6V_X_Fk{d*0JFyoXf*hp;x=!+y|tV+ zb8&I;eZgS+-M)4h&V&zF(_Bw)+VP0#nM499u zjNGBx+R-HUDH77bpwdC`^1&@V@-apcL0R=m_-pz4uxI9aFd2s#0@FfZ zLmh4u5*2eKx0qmIoPy-fO7UO{ZiY|zQKsHL@pXv)o96PJ6Xp%TRr8F>$XB%N%puvnUU2i?}KsL(8(Q$LGc} zm|L539H?o98TQ?I-@H-`JG*I)x?PU${CMXYaa;2)#)|Ie^|zDNu8#|SVeLqvKf}#_uRVbGV3j1xdE&IsHwf*tLW=sl+D{` ztH;3udLi;}A6BpU99C~1vo+J-O1%0p zjqw@>1VRnN2GOi~p8}U)AWYvY=Q$?u^i`rDa2ZoO7H&cYA51>x6dbX`f5} zx@b$vE~`?Scv=q4O4{%69R$mbH@!zQ!X5Qb{wiID71+!$yDm0 z=k>)@ar^$H*4XzQj<=0VV;c@o@a`s0rp{5BsoU=^Qy4eC_y{BW9!&t{gk-Pt$+V+? zH9}Jp!3+P&V6;ivt$WF*_AoVJ($eTJ?M9~(p(z}9oJ1>@q68Ns#B$TmM#zd;WMZeu z&V48w9Tm`L+nh4d9+=ED-Oc&J@8^345-KznX)E= zR8G^Of|@E9h9ijOKzxzNsn&4dso^>ubezorStLnyFbg-`sOB*q#$5kwtme4r1Hs_gFR1_JPTh_0Ux7D zP{PS6C5xf>kQ--6R0_v^^;?*r1uKhRi!DNoi}F^NI8Hb|%{r@Wug3N)FL!0W?F*3Z z{CE5D7N~W&iuviWeSdO#w{`lD{=PH@qXBFbu`}1#t38`ub4$hR>+8>bEr-1FqpyFr z0|3;(TEFqTtN#1OyUid_R&GeB`3>n{ET+6CG-!G2=msS;MpV8kUjLAd&ZRhQ=mgY+ z8<->ai6X0Q*ZZJ@6}7rF!N9~$?m-V<=ISi zk!m5GRw_$#FGs$$F@dGys|mEpVT`pkO_kEL`A0@K0=8e{;g4C1h7tbf@fgw?rqYhI zuR$X4k4QdeNnm>MNz>3T`<(U*QrWW~)_GhVx3U>ao%c5>eCRcW<*GoFq@N#R`sQkN z;YYN5I$8v3z?;JC4GFS=GYffm`&PTZySpwaw-AS79GwP(EyGMoZ^bYr0fUB%o53+5 zU9Uu0$pj%nML_Wk`v3hwonBq*QSFxgKc{*wgE|xx z0?e4lo>W8KOoe7-3X^VIH`&ONH@&;IPf<;0;@TyzGbq|EyDZPb^WZVHPGPxq%1{bg zWe7h4w$~`JdDZ7vd)t5Y7Hh9nHY;Oztv&cK;5JZ2(2gn!9|RefN!vFzPFd^pvU^Iz zi*LT%5tYtBt1xl&GqCARVa+}F05OeR{-I=ttF5UIPcI2bI(A3Qwwy=LE>f;CA?2Gn zPJzBU(ctHQNaVO2%&UlyB7f>7!0N(yF}gg)=;zeY2;}QLAt(ueM#{4$+kfcmvw=jM z(rV!suHw}|ATRgXSGj5UqPBU!U`K6YwMXmXR$WLsE$LPfjDPZ0@NbiahfuZQ+?sMQ>gq&=W~t7;Nakp7=R>z&IHp?JsP6}0W#@4 zDh2R??0t5|wLdaumc*boo4lS^Bi)Js)FZ(0AIoG5i6^>-fEVyuNhmuI-=I75sb@~H z8AUkb0ua}5OA<9S*T4pEsb_Xng}a{A6U;`C#`-1h)`R9FC0M$a>g6aGSxVfH5|m%7 zO5>n(VePDyexM*V&`3-q1rMU+eA@5@SP)~d{8tZ>5^8Vn%aH7c2z=tyuhR3r&B(m$ zuv2=3GC$>fwR%&W&tbS_ViGtNo;N)`={b6`bAseo3DKlNy6+&3C;hus;CCkl0q=|t zcPQvonM~+Sas`NvO0Lwuxo7>S*|w)Vn@{tS3tGubKhX`Jzg@!a_4}wfby(QGEhU%H zh%MQ5=x8cJmh~X5nuJx=Xg2)T8<(-b{YxQLbNNwq%x;O}Zx}4mki&s*QrB6=fwZv+ z>q+6)Y9Sx?dsxP$hmFv71|2xf)6T>zLNu*2O@}B)$3S!4KFDO*qE8$T^P`n{Jlx0W4r48T?!%UQg!mkQb zvWQfP9^>^|2q2NcQeDYI-+ZN==%HNu-;o4zrztc#{IZ$xvvW+X zv5HEgm$8M-Cs9X?Q?Hanb>H@A|s?`63gq;=jE)^0A#ZB!CzgN_>a{0)E#=M-8@ugn~vo7UF!> z*-`qg>y|K_TFWK1Fa$T#5pkL8j~|+^B{Z#IK)D7#E(n*OU{Qt(M}?cxF6{`U57SMa zcrP@tWdwoTwC)+19*w0y_y~p@a$^^@K{fEN_V2$$JbtP2DXyZ~yE&UBOM~QqMaH1X z8A^|#b1Le+{@qu}_VlcEPoV zO7F2fR1v3H!;B^}qXEt18AA2KX}yi6YPw?qeNPlUaI8gCba zrxsuTqO+@BT;F{o9XGPy_}lWtK@Nor<2ORo@|gsJ=rQH3i`HKU0kIv&^Gsi@Gtu3qXook z0_A7K&yebuJcyy8%wh;7O%WzZy_*U6>YLutskNfN`rAg!a<|2bgYKwZ$Wb#UK&RCT|o=S?9bfXU{(rof0t8D7*NUtFL z#!#n$dHB`&14}0!CujBj3R@VC(RnOErWbjtz0ngDqwAj@FPO+ehD7iQVQLy6k~kuw zIJ$fdbkgg^v-3Gw^0LWz(1*ADL-G{1<+->CKSgFGf0qGFC>0_gvyl0&?`-e^+uzN9 zfBs!6{$=|cV%rNOP51uYpy-Q(h?HUf_jCm%_c=zNmWIfPUpUO~TWAAR<GivQ(&k~y$d!|+k~gvX%$k0BXl!O@{zRRdiZn5fACI$ah zuP43ycP6Wi*Q^uyP71yH4i0waijwiD+6lxE|wMMKeM5=Tg@r(|gGk&yjQO7 zro%gZM4R{vRN38?Rg|I39l^>Jr946jlC-z`GxeQvi!(VE%f%xir0_3Sf|}k)D_Hrs zo0F8Ee7*hNF3e7M6vMo&@Hf4+*3D5Db(afP(7-7%{nYp^lX{)JdTsmOPo2x@@rrJs zFjV}G8nfZt-fBDE<{dsDM9y6$4t&J9#J6s9b5hLL6k5osRYKdJ?Us-y$Cs%F)5>0! zH}(Bm#ULGWu{5-?U;lFGea9E={yHn=5tjn0tSmlrJ1<$Vor<8GZ=aG`D=x@P-pt|` z5WsVAW=e8FS5P^o%LMs>leh?=|9Rc^XPOuDq{1`iW|y1EL&kTR?M-yF>^NUS;@)eg zN;+M;V2%Quq)uLM{xnXNd7LrFI67u889go@vWEG?ba?n0;0>IFe7t=P7{2pA{@o+* zziN#6C44_3ta!64w|%n@s7S9Jz)}1A&mr(@b)DKHBeJm_;g5eB%%?<(MjU?KTwwM3 zq?yUmkP6IXUWc6HcYc{$eLelHg;Yfd*b)n06|b}krc{-{h@TP0rfbFu3=>M?DEiJ0 zOIDpL&QkSb@navrvt#R3;Oy$UM6)TH+G=XOn4YH8_oi+BRb_FhWsd)uIEeies8EwX zs~~xk&pgGRi0uHyH7~*#WB_wDa3CKgI zy5kTeRJ$FjRU+#=2$;*S`iqA131Gd=w0&0@2@o-&g8nbj2iUJPgJCa#jwF8EN3&_U z;0O&(CH2)wHA533ChXVegcXR2tTbMuS$zUt)yK3W4W#2nlaN2?RYHmo3uOCDFe*kS z7nrwc6rbHkKe>Z*k8!XDZ(h@~WsdE%-tNAh!}i!V&n0s}98zEQ*^&>nY_kQ-VuPT= zd;Y|r88!9_=Wbs9-2&$JkjLlYB#o)=%bmar*DtQtQNT|V)>s?xFnX@BIIaW3GOg^z zpyR%p?5<>E^|l1Q!&Dkma9SSqNWZk`Tyc*ck)%4vJxTk%1DRG zEbEbzGN`Z1*bbv4fslzXAe9q!Y9qk{CIVTiQL5TBOEUQjz4loT)(is^E(gT{(l8by z?wV>1th{-t#*NIBp^sgYn}XA7GTCI*O|we44ba5!Pw{JGL&+hUP<(LQlPXs}h{Jbl z$S0bT0O`^PJ5hPxOVYwT$BWHPb8-S$ll1-B3OaA^vN);8>aIw0NK#WZTZbPtyh)Ay z_a@thb<`n9K`&c_>6Ni|EZ^{4=~>rzE{MZ4F_B*M9dP^k0VLo65Y11GRs+=S@!sAZ zdCcvcAlCUK>feZP^shg^b|^kg<@=BxY@KvY%MXVJ1?@e4GCblS3RGxx9(H4Uz`$ol z0=((mV4@w2)i(1b{9^ffB@`dzrpq_|CEt_P9M&k==|2aN%ffTUF=i(q=(Iif5LWH+ z2KoCqD+C1~^QZWz_^IFL63qBf8fI(w;KX~b$oQvk6}xFGbK{!Px7G5;!gI*#5O9|U zdO%+^vnodQST*v4`IFO90}hw}6RM)Hi=tW;G%(-`ic$z)ZTrgC}YM>wrY2I)ETnm$)uP~Gn-8jh@+9-Wtz*Z#xYK~ysFF@nukUz$IRyK zklw_g7ZSTG=8s+`g5PKzRu#6?(vaMWu3WH33@ zaNJ3MBRq$Ql}hdHE*OA&yKwLe3u92><^EM$ecR&WLUqkt$-r9i(Prcan}E`EB&K`B zecOY-E&NM!G4*Bg>z#xvFAPz8T^==@Myo<*kK+wh9QL-;U1h-(%ATV$iN9hRRFSLA znuPN$E1(HmG-N?i;JE0}$__cd2u5wPWLN^VV_XGl9xTsDNuPm}I7XFW3=|@79M;H> zN?tQF07cUBzjcX>{inn5ef5WWSjeFX;uFnyYw0Rle4~RRPcS8}bOba)RhkogRSKg8 z)DVn-F(_7JK8AKb(lwKNj3*!1+4YfFxNF>Q`fcK~n}P_$L(E}@DkTWr8tLi3H2S{5 z(v1Vr7ue%eLwx&^H^t>eoyW7n!ism_&&U=2d;rcE*6s=g<2N?(n)$MAWa9$g}R-K%7 zkg{IvDvsP~9a(g%g|)Sft&L4%5o0^PMh5#Ec~*T9pH9E2nTE^cA{{6ckneC+-z^>5 zqOZ)YE~eSI3&Xxd<@@0s(k6o|n}$D<`wi~OV7}8?saMP^YI+pwadA)a>ERsGG=>xc zsk%a@v#@eYPX`WPVh5`4Xn;ol?z)GBL;A&t)Rh-wlaIIU*7n_mu)>W{VvCjFVfFj( zy$>*vwaK_PImb<>YsP|k%t#Balpf@M83LR}D$~txZEc%pJS!~J)5LiS@&n6K5|>*F z9PYKHh~jE$0~e&K+`H(QsHiAl;x5k$43W35Y*&svfqqC>1lUEZgwJj>j^fkz1)tLhKgsxukZOs0PChFskpkZ zKvOeF56zueJ$%BWJj*~dzcJ=iGD?*#*(s!6sdJ^R({L3oBSOE#H=}pI)hx$tx;$6|`Fm0X4_K)wZ08J_*U1 zVKZAeF53EV>w0n-9o{z^SJ7apJ#U>_@h-Dg6!ZxiJUlT{f`C)dU^KO$?uH}-8$K`! z^izgrM{T2W#4gQk4prqtpc|A>(3r}OBY7|k@*1I?!9ZY=&PV=M+XlzPXd+osh%W77 z)e_e1pJR`h?KVsfp9dy#Olb81`vYS(hQXUIakXF!*i=ud8;R`1YV*Nhv}o`3nq0*8o6MNQet+>O<021R>GG0D{`X zwa}1QU~n^LU6aoGJgFfi-7$hJHvV;8lp9TQ_CRBjLK?!?xAVI;A3MK1&&ZdKMhSh# z^o#QQLovl>(95E-KxZg&+N*iYThS3?(@L7NH3(d0;=(B(W-o^TAX zLUyCN>#${3J(n$(@2Gw(igcES4_W${whTggAy;}D1w)r`==a1S% zGM2i5-84aOSqx@p>vCtDhZHuBGf+eG{+Rw#th1Y)pBn}9o_MZZ(_h1HVE`u4+ zhZ-(Y-v5+}5jj7Oj)(y!ny$A+_4Z&;Soanp8ol<3u{UGocK*ZcArmxR6sfWxo<>np znFEEAS~4an4>owur^n1N|BBqS_Rbp{nw6pF&v*D}D~vj^kQo(?61T2ZQdS1fNkW0W zw}#j@MSudsbC%JU%-4CcMVVJoY*W&&xk-ZeEyVDxv|&}!4)$uIsDnFvFGrJ;AxesDw@5fC!`{rx+-=H0)&kAL(3 ztt#I0=q*@(9+@hZDxP^<6&My47SgrJLN@od57XyUv@f@PJuC7*iq12ht-lT9p;oPm zP1T4Qd)2NPdyg2cRhz$2yY?pb-g^^DQPi$lwSuamX051EwKvas^2#edl9O}p-*sQt z_ZpmAoFh@sIKXtuV9Mm?ENNvZBsYU%%Q}>OqN170dw~mPqC2#j!>5~ulrdWNy^%{O zse$(M6Po!tFI~Kss*n%*+VI?i0~uxfv{pwKGFb+xT2~mvCTgN$gDEovUX}e9)3ATT zB;GJt$b|&}4*RY!vort2k7)B5!E8VxBv18A*0)^DWy$z6<{Ut04$~^$FX+=MSou zxH+D#O*PdIMHy)txv$|*-ZJO|V;0rQq0i+daGe1szULJ4yJR(ben#_iiamBQIBk(6 z-Ipi>x-4|kA1hp)@4!H-^Zql(lUHo1()G&1|iJcJ=xVmJ0hhBux`&-zXxhV9sY8}KI& zd6^ku*E8!CY;Jq#&S$DDuDMO|^5vMs69Ri%X)Jze%3%)XjK&?kG(AC+yldEush*nR z%(X6+?cZhpqf%0@36yGSn$=GZRu{wgb$))MqC&@>6fGWyFz8~T+l*<|LX>vG;PFpz zVnjU?78NH5VGbY;{G(jl&#Kni$Auk1kXXi()e2 zJrL!%K*iV9)y-MaNHMhg+K+!U<^8}lw`05^F!a-fen}vRv`U%sS2xq23LiT=#kK=y ze<&x$0{(mTpy!~E8_zD6MuP!*9T+v(tYTbPL)_{%K=J9(mI&Ot2aoi)L|L>?KoVSK531-`h zv9%5jQ{=(AU}5o80rsJimUD9?VQ`p?%_EgA) z5X1ZS6$!KyE=^2633y!WPdH}udLpBF?$@8)P1<3-x!eNE|E@!{tvNIn8hX$L?ZMkG zFC6KA3QnYZi6Gp(Lr1gM*4E~!G!G-VDP7j`pN4s&KD`FVnBqg&QtL5+N@&|2T{t7} zmpE9^SXrEPwJ&jh7QG?rOxfje#22 zDkdqHCp_aM)bAI*0esevy{ywUM?8Kz!Q~Hb-7gB|eMPv-WsITv=JmzS(E~ zlW;KBB0nUAhvI{5XmGVgmlt*or4paL`z|)~R5ic&v$u`{o^4)c{ta&=LLiK^lZ0DX zeeCk^Fu3pN*F$=8@O8x6-TzHiu19Xt4`;KJc4XG3jq`Pr!#9;dYa#c~*LFMqYOYmh z00UE@CsX9+agH8o|9qnb&@!+NxY})kfyhf@#aRR`*|)8P(1jhTxEC9iqA%va`J3@c ziMBkIyL5{fMBREia$+umn}}&0UV&li+qV5*4gX>6l_B&HIKnF%re2J!Fhdu+x*&L@?mRX*-sC@mmytR#$a>1U zxjo=@@sK6AZ+p5mTCPl$ZI}24Sbx8uIw!@_)%+a~S{H7va~iXnHKtrdze=7RDq?d# zRit@iH=Mfi=k9RneNxb-S76e(_m*$*^s39N5o)49Q8|j7&?F&-J)QHQi~@M;NhuPU zlcu^R3;hBUPyUOC)LCNcC$LnMr;5qY_p5O@+)N}(D^pFo3`4+LREUd04)fPc`KTbd zPw7%Ap?Na_@}$$Vek(D!npyM-kEnFQCm`eY;-1F+PA5ulvqnQUXHJ1dohVMqk+2-_ zG0@xavtTXAzjf^|5`84xoiHNoHIheTfFVat0O8~)0dy2rTirKYt#ztH3V*&IMD+?#r=>nNH;BtMZz*wozI?uWKTuzOHYyD|5CK2Rvf z7MGsDVH>OZsfL_d$X5|0zrkOHC-Ktn_o3SCcTaGWgvXCbSy`LVGBM5E5)dO(rk)qA zTdV+QO$`+f!O5@!XPz48)`B}QdW6#seb_kF^b9s(@B&x*@kK`#l36INHRs$j(gN7Q z@aR>U`Dm=u1T5SPBbgV22W50n=hTEZZma7Y(;NTi^OOP~lNta{twbV83Qv2HR2}=^ z)Rb5{jH$ZiKn9J{A1$+x1`bs}4{sA5s{HsV|No?+~eA{@kOr-T9(%XC$bk#A)Cu zB<_qqX-Y+(E4VBVx;x6&PEb9=D+}5=Sy6iJ%wqP|X+4D!8;hdZeYWF&s;kQ|D!#>5|2Vks{%lj*9H6{Y1z$3*6(CjcnW>tL6 zEc9lyb^pTWU%Uz`7sEm-q`jizHT7J;Gsa4C^l$QHTAo^M2bpR~)RarFr-1-?>HX?B6+DAXecaYnZGaIf#i= zM|4tZmPO1tb!F-5gPgrY+)x}sU*@s_yL$2U4{GZyz2=>Q@!kAglBSO}cr9dLbb*4A znh8&q+p6ij_xHq#KNq~fP11LBvf0Y`vI0CslvN5An` zB;ca~x|N-Jx+M?lwX-S@+F~Y6Lfp^Kj%V%9J?3(uRv2Is-7vpZB1eyXZzwB-grmIS zByyOs_5r^XdmY2x|D&lqKd26gJoj%k=ktwutD-~RL#l>PsFp`z1-0D3aa_osS8B}xU-xGXyrn*_OwB~(CyeGilU!cIW zfd|zRqwS?I*`a|1*1W2xG1JmZa9^X3*(lU+`0UQ~1L%aF<&M*7HMI?e{D}K>|894E z0i82j)$k$!`4N8ylT#ecy=R&ej32sPU8Sy^5oe_vV;ZAo6|Tpk6ux8C;#H(erWWhc zs7nlu5t13!Q34^{fgRcX^y>2Rr^DY555Rm*OJVlIBjtEco?@?4BtS1>=I~NHpm|+y zY%XF=E$_X)M2SO- z7|1Zvw!85;7klHU?^X>y$ztH-SJHha5w+)Hpr)yxsz-`fww#`C)+sw8s5w3|3$eHQ zd=%KMJ|WVYCVdt5?ffm7ROnGO`}*kCOim7!_`MZZJC9*e?<3v$YH#l5yI8Gop`Y8< zUy(?l9QYWh)cThkau+qJZ9Eq>v3je6FQA@PFONO=?imQ8rk9f&q;sOk{+ivI_WPgh z-7~)e&)$ZFHV5C0~;W9b6s}{|IkMN{;7uWoxgZ>F_d$&LauZ3FN5gFcBHnUFxdj(FHeS z0m+;|_~^5w4=BSEb$?u5{OH{DZvSQlh<@D$Ro5Q*FfQUkD;>b)lW)j~*d}ZBoio^E zS{e_E(>bajW%{_Iz1;D2cV7c6p1!w;+7=fpZUGv;M4-gsG$aM5=l&i3pCDfcK0JG= zX9(QdpXwz&)f1`u$4!Z+pW*8b@04RQf>W5LX@5m+Z*O_x25)&Kd zj0A|cIYN+V601>hG)oksSw)7>$#5nLoLV3nvxI3%9v2GLSU7A|%14a0CUzHgHbgLD zzdtIp*J~DYMf&%ci-IW>+x$;{-!cU5KVSRY_t5N;%*u*Gp{pFKrWU7$t&5!k@;24*Bq^rH z*D!dRuNi;dO*(XX-IUM)$5dpbkuy$ooCWc*iTxq~fdt+$qOl7>ltUe#N@mi? zb2~sDF9=5fr;?ns>^aaxgG5rN$WzC{#R_)%P6tTAna6-b782#aG}l($;byHBy<3%$ zk+N*Omh;Y71_pMdj0ae}yJI|*1Ykf&{h%`Mtw3|T$)q8FfQ1(~^h+-8rtNMk$ESeg zeJ|{0SV9{*Y+oU zPD^cg-F>FO+`fO*d-(w9vC+nAVP%~1%>TLL8)M?6hjT$OnZO)ik}3gX1twWJB?va9 z-9;pdDg6lck(~5g@3RK1i7qD1AN1b8pz7b^x%X^n zDlK5uok-=xq&u0m8bl(q;Mkz&+#6BvO!R8atnp2ttT9_Nt!6+$233622YU7J0#Dc7 z_Cg@LUOxrwFx;Z*&Tqi#NI26TK35I0-lMRy_eftYW`3mi2+$E-+W{xDBeja(T)}x^ z?Nzz55sjtEJSz1PMFQG{P2o}GRUpg5U#rnrY3Js2WTpP0=$;?t(SYr;NKJO+ApH$P z0rGKHynf~f&gy(NnUX1UIrp?BxNN5T!MB-RHZ-1H?`KXyP`U&Tq$#WZHFuVtLAsMJ ziH#y?C+DbpeB|%H>uk}c>5~n2-gPrF!*};L6kAv4R7b;FP9xXv(EI$m16U~&QsVyxtj|t#lwGRxzuisC_E52%3uLL04>+PTR8D5rlg?KkfJf7Tm_ZYL4YVjw-^YOaT*Ig+nFhi2*NMV8S@36Nn{8bnK18Ia_f>3~V(CVpzP z$zy>xN%E@}EsiKBH*PN2hQ%zmgp7EQw}BQGFE3YIvX>6egL)i#AI_hNp3faU_A^K@ zTC5MH+Z*Md16h{vmohi&FSM822XM8EdHCpsPWrkZ(?{Md&U=UvXKSx%>H=l&4O7K8 zI~#2)6qQonJ^3qt43(?A9ns7S)WJ}?iJ%rL)FG$9$VkLdK&Ww-5febTsKjahd^*O+ zk=o%o&J~pb{UgjH1veoD!y&2G@P>oJ zfV#Q@^iw`s%VdeoYkKZ0uqQuXN)cI`6P|Qxb>fIup~R3UUd;`}XVSFxSQY0;W1;Q& zz3^TtSFdCDD0fZ!{W_V*z$7C8=>B_OZ(9%pKv9 zQka379BmAfec-TkW-WZWc<5=EYL~YXqgJ}+`{!JbPD#Q&RVR+7Vkcji3vOE2W&U#U zjRQ=rQHY6J4Hr3E+241|3|%ef5@?Mq2QrjSqhnlQuzZ1POgheNcgnxRj?>0K>aHe! zilYmg&4IbZc6{*D&~L+t;XV7#1>BoNEk-dA<6E`Z)URc|fQkQuNc_+S!~H-Ao%I0# z3G}1{sY-l(d2aPo8OPhj{;Ml%72%!-T?$1BV_sEwMtEh79&s-vI8ryF9T_NF?D_D2 zyV<}07?yh;0ud|4kH3p=+aG_fAx&IeT^mq*(HYf*M&uY6VawD{2nl6vm2^ck7u_FJ zl;uJ!Nx(k68Bf*omlJrsJ#@vQDCZB>XS0%N1`S^~G++NgN0)E*B!#XBgD`hN9 z1WtUu=q8lP|1L;l!P}|U_znW`v07LdFeE?M&{q!>hvw4>#(wTDQWPoKlWNfy{K9&|uRt3m z9&$FL?0ZXWMiv#XM^lYG^Vt}(eWuU}4(G_?V8rBSW<+zB;N^wsBvM4iag-gG3e`Rn zbu;YaX0J1buLzersHEJJlVxHX@GIJ}O_hToo$g<4JDlmw3YKokW~^E@phM*Pv^gCf zfNMf_sJuZ~VHR%p+)j_NSn$qpxXB{J(EvXfb|4N)82jLTEmhySELFv$(P(iyHJE zz=0O)318N?z3|%|;IEKgiXv2KR805}EY)bpb>z1w>}?jao|6lLw>pHWzW&+pm44ai z(^FvQyUcOz(jTO>b&RzxF}XZ+XSl|8Juy258NKg+xN(fk#y`3{HEbbtQ3)c<67AD{z3nP-ybiffxRmKF9g45p% z_r~~Ude$H%m`*^f@gFosNn-(cjo}wz&CQh(}1H8W^PnxL(4Z;KV!AgABW!Zsw3Z$5cpqYzd0xfv)alx_rHNP zK}gw{KRzAe^R(g94>Hrbcnadr@T((B5|F1*hl(}rPE?IpW{W8PuQ>ep&Y zKh_ub1TS1C5bE+I{LXUyC)^+vs2a2+YEpdMX3Faa3dKsmL85818`~b*FJ6rxjZ3(x zMFrRH=v5rylG$rR-~9Mk?aD$1@#zhBL@d)+n}s89NMPQGrVJ|)B~*u~=zIkul=$%d z8gY{(9Wk`B_sF@eW zj;2O%^b@31uX z$y!mr{(v2@W6ljM?}U;n^a{Axn?6cEihc1TqX0$9XMHb|H!}k1%>VvVJ`!>_64E>+ z2{{Q{41m?=S;`hS=52cbRqI*fwlX&0-7Y{A<0}$Y;+a&{!+~2~Wc~MSV0uKZmHN<0 zMIqrh!nZi^;?GE(qW-Vj$ElE;$D8D0nQOPlfXBp^ZL604g7L%qp9kaEIuzLamF9Cy zP@#nfCM7sZ{52L9=!mnX>bazA@fU6>CthSK!85b?6-LOxfJX;mFL1py;JYZwuNvE#0?tib!bApHnV=Sj_kMWTx+MXP^a_n>@seV^OQ< z;1HzL{q{3Lr^^Qe_9?6ODgI8usaNOdr~Wgp%qv(gFT_V40!by)`9xRbj2S^hKrCWD zNoMqhMYV}`MA8S9yQP!lUa-v^2_b7I#2EVZUmgL*d#f|+mn7n8n5w#8Ess~lFNkwV zW^5W=is*A%5&jGdFUrUjZH~TSm-shXS}an~q|sXg+PE_6)q%!7Dzj-*h7foYKTXOI zMraJbpIE+77MkXlZgvn+<^;JcT>z3^$U>fMOBAZg-2Z4_hbOBQ%4aCOUcM>{G(0H3RW)ZP0=oTqmxq;;%bM;11uyT1Tt;)I}c4ysEsOf;T@3g9Zt-=K^f zH6V|+*CTqwF$4asd!4+IH*OD;eST7Hz>yKI=D1tt^#XM8Y}$L}{&2FKU3=a>sKvN8 z&H6MTbQ?KFt*OY&o=n2mA4kdg;U>__{Ll ze0;kZiPV8lIAe}y-F6ZR*u#T>LciXN{+J2l`QLkc_+aar1tS z*b4`?dI3jKB<4NcMKjduY#yT%ED-m+4==F6av-Ih1r;xADiV_gfv4(~#2P_Hf5PV6 zMt;0c-(@P9;-Uv*M>8Yo6^@4R=@(S}0O_hSWeeyH zjOt8CQCZu@yyAN^X6u0FP+)JQd5lVr3?kn3JjiFha+JPJ*)59PnGG2P_lJC2M7LHT z%W$Fty&V@H`zYoy$a4#h^j|klrAe^R>lJ&pb(bRL@meq8VNC2A)p7FdPp>%=PPsHO zTCeNdYj^qOLT2F_kWHM8j+lO}Dbr6St$LFfn$_P|Eon!;*+9E&^?Z6czB;vb_N@^s z7|qxHeQzvn{yWGQHXSVm^P4@mhAexVi7xjh9X54sh8#{k{tS6sdvtra5tZ@E z4Ds^rkfDp;`1{cpPC*JW+M|rC@TGX75R)NpZLV$R+%CA4&cl*0R6!CtNsV(G z@;RmLQkXHK=`i3p0QbM$237G3t@~y%6b}UJPjRjmTe~DFfLg;1XR&v`d6f>IeJTN!#NA} zqTXgluF)j&sB1ujpKunA0Vd-0ZhQ1MGgDL07k)xucj-+BT*56orf_&&i!wk{X+KCY zg_B#ZGH)*9O)SLjTSFiU*&xj?y0AzAF6dI#6DA#bStXDzM<}yrD7kZ)Grg0b>5ND| zCP#SXDI{%n7dqe!B~0*cOgPlBlO`Yr=Z;vQ1SHtXZdWLqKiW|z5@;xs+_t61cIs$d zWd1f4Pp!V8?W^QNVp(Q#%Su@2TLRwrPK&p?q@Ey7*tGS zQ+yute8D{W>`RDDqVM)#3HosG0^>1EX8)YT!i6u6G%c!EAS#FC?rCTU*8?X|*s*46 z!8eaab#$88M2XpuJXdG2Qt|*rV2u(~}O=g6Gl$rL)gDl*^sLo`VCVdUBl<@`pAVUS8kfs_n(W6(3gKh))^sZaLdGS5P^I#K5 zoyHvXoL<`(8&o{@JpS!_bbHJXVW#)B2)vuI>yftwi_;NHs4WRe;?mVLC^jR>RiUw` zyZD0TlTuGYRo%~nWPIF-xVf40m@xH}IKh@~h7JAmCaiVQ8NBiraM?%_Iy?GkpAJ*v zfzSr|fkimYQv(%})B_HBvnQaOqrX{kVw62A>XM1EiV@tET7Vogo6cW6FRj@URGsr zaAKEw27`&HTJ24`Bq7)~mkwvOVZ+um?*gVQ__U>UeT2Sz^u6}1g*9RGG9`G1=0VzN z#st>BQ0QqsH`yI;_htO=j>gaEr}##5dKEdO6={VMVocvgUE>_oR8R2A^W#Ou*(@1N z>$AjCjYaZ_gl3bh6L{Zy5F!XSNAhjG4>wH;>8*u7FH0jmQIZ}B?z^syfc+Ts>+%)& zWqh3Y`=k6SK^L!EN-`5x{ud$?6gVVUP(_w_8;-KHG1ZFb7!x5qx>$A?3nNxTMmuae zk8?uZM(g4?Y~Q*eZf7mzC2LH(^?f7Gq&V%=TMO@kL`H}>#ygnou)~8PV1K^+L@rqw z=-YP>yCqKz>jn~ASki1EAEsO=idLBN>gLd(3U_UePP$|v@l$hUWTlDz1 zO0JUxjAEz0YPwuaKV2;@EF^+*lC&II|Gfo{`TVoTE!ulPB9^=RKabSjWHV91+MLhR zK7H{`qOOF3KfHqiPOB<@FP{2_T%m*a%PsB>{+n9!yDYR7 zR2zw2`nK+1_or-1YSXW{U1H0nYHWa8`AJes_8gewF%gVep}*cC!)>p{oG8JecVlIO z7Mwx1Fm_02%;=#XhVf0Sc_UyFqreO^qqB+yM+uPf+kdu_;qsAl-SU&(qd6kahyfJ41COS|I>3V`X0ri_tg8LF->OLu zPJChaB4)Y_Ny-#Am=V1okcK2hq}6iUk4L8zmc9x0Uu+S2dizH>Xu5jGG47u0^Bvtb zdfk=pIdcZ?u;}9B>+opy#hW)2PXfJ- zM(P&usa#zE3n@vu{HniQKGYjlO;@|+d?>^JM*XNIfLm2*?SGK66GoW=1_@G5h~fy% z55Y{q2~+swn_@rz#lg6s>+Xl#j)bgk4Sii_ip)BV7k-?y&8;_Jomgn(mjpH!$F8=c z+iLB#YoJCj+@fUc{{DVEo9}V`*+YrBG_w*vL4ws7##TYV*_mohoz6=HT?^qrN^{Mg zwDPQ@s;)jSH1n;B0_W1HWWV;+reAE7>d*J18cYdKt1yFWa#KqfaU41eh~o*?3;2xA=XwZRA#yk#P0zIl`doMKs-dZ&&cCl-vw1()!1#1+udS1d*fR7Yu=wwyMAspNGK-= zpAl=%Pp|J|zT@d2V~t9}AfeMMd1jZUy*r!!*OL&ZPZN#x#&601aw=fxZcF?(Tl{#? zbGdpuAaggsOAH8$cX!?Ux97lOv=P7z-Ma<=X&2`=@gl%I8W^~@g99u?e*^DQBZ?1j z2D-Wa4L*FccKt^?gcD*m@BS}ao59=c^6F35+WlW&3o|#(lE(*rP}1;F) z9_YMs3T>@KZ@!XnXWm=~r?P!g1e0SJwBO!;e$)L#w$S4uj|&V`&j#h;v%+k2$Y;{# zSorY@6Vkn<>q6zV!VR$SazQ#+ag+umEK>~)ND}$&ySDY?vf4C99dGJ{)Gop>Z{mNsLe5J@jqMBQ+>liR;urDq{F2BdTU9%JFvfT#_+TrPE#35#~3FY|gF`Ada%0Nk=<^4b6+!IcX-{`(!Gqas7m zfP6umWh9}r;paZ9;5BE;0QIdOeq?4#<#+k|AqyAKX5mWgh*O+LDW;9(F|g{5)^@ z8i60zuj3no4*wHvcf&iI+4h?iW_%rkWN9+EUuZR(R}B*q1H8@gbBp`)jg9nu0Y2wv zE!p!f0qw?xG|%3R)l`{@(CB^6KMcFtnfBT54EFFg7mrydk&DZKC}PKv%f{?Uag=_E z5RsX&`4EwQKnVH$t*Grsa(KYPzf%;L#y!V9UJ910^RoK+lN_ZLV>D6vG5n*>OG1SN zfh#3#TEihJ5`+h|BcoIW$LkX`n@&45ypUmUe3ErFD_8nCyt1uC%F*pV> zZtIz6C8>S!uQ^v96S(okQ0vR!nvRzDex-ivEvzVqsdEN{sgy7(5>V>!U;%0h%5U4b z*`2l(JzvnWWO43d@aq@#Y5OjB=NXsILq@O`(Lk<<&>Y5T#doNq(ulY*wxV58ZYo6n zAXO!fkrRx`#Ax2AsAAG(tW32!9eFDO(siVt}tRUcI>nTRo57bht zM$BTpvlZapAEI_YT44!D zi!U82wR8Cl(t};P2d7cwEe=najB$u$(b+~+`$K)#q7T20?xJM+UVlisY-e^MDn;J; zh8#W*IZM~3V~+5N`QYiw@uKAY)T+?BSciu2e}(xL_)(VX4fy%Cd3M_%6$%e1f9Hu|Iam`&I#-y!deUX0M*BNy4HAmHwNIMX=)9|Q-|pAwFLE9_vSG{A*1gRi7B` z!bpR;m1TxoCZkoEUNN^liGo5neDlSS7|GKJ&h%$tfwChAwj0mp4R+dr(L9{IT`!bI zeKmvKITeId{OJCZ|Cfqc>ZcSp84S(zcAO%7h>4^Hs3fC)rw)^$&o|R3er*h6?e9e? znH1k9R+)tmj4j@VA2yUIk^x#2(;ziXY7apva(|dsdiV2MyP0o-K%9*o`LR!&u%`7( zJ5*CfMvLRN-l!v61d#c^4|Z!SL;8mVh!Q>#5`~+=;j_Y(8J+l)gW2P(Wuq;!cy=Oh zhFPT0)G@m8RTYrYRR&$n$niuuOS%Fa(45s72)!?U@bzh^`2E#Y2iE!-RNcdUL$+e_MYp&Pw?v+~b`lD~_HJv&6br@_WNk$=IPf zp#I?;SA?=&jGUfBSSeIfm{o38efwtH4?Q{pPxt;04dkx7Zku4w z%{7p+R3f9%fYZ=Re5r1nePUS;;#MzQjYLe7-&4SwArUoty0$NkAk1_V2~s@KAB7kf zIJot)?zXcn4B@AfjUnYn8&km-EuQAT>@PA2m`W4~WKBLjw^||!D%KDy_2yxn1{UiKZ|mf*VI z57}0JikzV6zj**elQ@$=BhD{=i!#gA^ zFzF|ZU8KCYGl&bGywI^c+17jhxh3j9)f)|48u1h4FYkLg@V7)wdse#fpINarUliOk zoQO(|3pf=$<-lFAn1qAU4c!$Tfv9lI4{|ke!*}v6gEjlZ0ev-linGF{_$V^V(XYgd zOa#9e5dDKv-2aJK4kBX&N~!@gH`Ilm7>K&%bU5Y#crzsas^o3W&iianJ0aoGQ8E!g zgo~=c+6U^gQpigXz+x?)2CPC3{W+qn$SMawtf$=><3p=&aY4}|g$+L%VHYutSUHjrf zO&mjVv2CeADX)=I$ttPkRXWA3iyLoCO$Afl?D#RWq7M&T0NLkza`0LCx&J+YD~Uf% z+!902OSF?XXIO1`X)e97`4lsM4ayhovlXefaDE)>C)2eYAw#-$XrO35hvw$Rl7lvx zg=mPF$_008l(oZ6^mWgB_bQSF%q{w$;zcgNrfri%zQ&YNoId%ot83*q0HE;+p|f(m z&1m7^2Qe8)i=lrcdB|6<d2Z<)Vrbp3nNzpRG)yc?ehD3ZuoNx58Irvh{Aj60vL>9VW{38g6_ z^vLxkD6x+K+AR!lnO>S1%xy<`OF$#ONrB|fWGVV}t1bOYmKFDbv#mh z9qzOtrolJ%y)b%+qz1(sHAhr%(nxvYN5H7N0dOw&J6Bw(Jc0MjTk>oo(A%5BI>F3d zlqtU6u#S4usa0=o$SUlhS*69aKQbh5*k!2=qyvXD5HL>m1XausJK7z7-XyX1A!X#k z>)-~gBXTBy;)EfPZmlRn_JhBcDzP?B<3N-~cPw3jI@d%`u3Sw^ot|4{ET?GKriif; z&r?^PZUv`?;|U`sJ~>n^eDNyi*}G}oo3`hOB(cfMfTP0GN^&a(&hXlh!%*5SC*3z8 z-0ILOzA&-PuRJ18vk}$({y3~>k$f`fkHVU0)X~46qXL}Vz2_DZ zA5sPX41XCM!zCGb%a%+I8`M)vCDk`%WYmHlUd90EwcyM7-gM99)fJw-y}7y;s;S}; zYMGnGDWNau;x=G9xNUa7%v}OSQha%o9Kd4@Y*mb311V8Bi(C~a1_dpW){Y6MKN*A}ut<6u4eMT} zA+s{{mVnskQnmegcH`EZmS-X;X{QJ1+ z(ef+gSR%#MR}yV%7GtLyW0}4%4ro3|WC6hhJ^2Thd{15zebD$ofv1fWggv8zd{Pgo z!$-+wlZGiCi|+IuPWkMR#o9E&X| z4Xj5($h1J?y6A~(kT+C@c*W5_o*Cj&8?jd6F~1ew$0O4cJPQ@+LLk{5oY+|r z__iGBo$)l~HPiJa7$DHFS<^Vvt*dB}OSQI3@aX79QS%?&*<-JcDfI52Yk?8hParA!4p?IfAG@8m=nVv5({+3n z(m`ipni>_rA?>qdTapf3)V^-aGoOCC!X)K8TPD5XZDbmA*#Iu@^j-2P?KD}5-9tgv z98HFX^()YzEDZGtZuf@Ou3#8uiDa_G;-_JZMG#@#?JCRM0Aj#SLHsyRfW0X##)Nnm z2_mm-JaAAR1%yHir3{I6!!qYukmqhM zK*V_`pI$wsNqq(6vh=iEXreJX6bIW5oz44@NNHjw(=FFJV!(FkyC4)m_K#90L%@;h z8&(N6+ml`0>8ctegoM)u%Mc_!knHu-|#R@-{@u zv_3oDHmKL!J4V3}nns2P=En$E-Z3%W6aKtE=&%(#E9hpY&HU`OvgKQwRA>v4%;d)e z>ye2bCbQ_Ir(cEX*!WckXKEHKV?|0^?rl}vq2Jp^sI39*VBhiNd+SH<5GfL{V5~~9 z)tH~p14{f-(L*h^&=b{Sy#pW|Xuof0(V-5W*?#x`6dk=d^aJ?nbr3S;xSTo10tx>` z;l!3d)wL<2oH;As_2{uQEJ;t=aP2KZD`13WTVK5R&oGK#z7|dzN&QNQD!4A(6m8iR=EJ5Er?Qx!; zG2smTF1O>*#0U^pc8*WE2Ut?BGfzSd(NEv8f*>TTvLwKMSB(&WQsZv)_z^6vnwjb!pK z0zWd_(Vu`6Ezt;)EBP=-nMS2I#hSB`DQeKw+Y0X-Y1E3Hwg00U3^29pi;hppJT7Bu!QqNH=w+u?fVaHZVsADPv?e+Ui1MvL z5dUwBF0H$Q@};WYw}$o-DP)}L`c8F1)R;Irm6_gxro_t@uAYT@%6_kPQhm*BLe2i?rQWMSM9l>6rF;9zlP^wyH3DE8 z>m-;FgQ}`Ie5}CRQk+2hvz5;vs^bd=>{MVe1mlZVIWq3xtrPdsfB)m?yyL0vJx6kMO zKIgn%=lRMf6bK$zZ_0gARRc}?kGW(%vvEuLQ+_;TMBZG<9Il`e5h^V?X9a$Rt;ifKwpw#xwW2lK@ za}a`6rgd18j3MJKel9P&T)(^Rf3QehC?y+A5%WRE!^xeBX_epDf?J`bKae{7M_aQ! zlgeLHPLjbrqx*~S<6xj+x%L69{1sA|t8dcb_g_c>VBq?gz0l>BU+~Gz;>rvx$LB{$ zD3G4iZLuQ#))`2jhCt@^OFhZMO5j+0(ZrgH8Gk0f667u&8RpB;OUC>Gf#5PsTx_q& zXsqZ)LF`P}_N?Y)EE31j%D8ppoNR*DG=k_bAFt)}$g#Cy{1uuQ>y@v(M8J!3b^wIC zE>2(nxjD9JF*QhFPvxdNEy{k~@!DS=sIAtKPtnZ5jHvQzu{&sKsj6dq`-7y*JzsAW7( z#A5#K(!OG(R|b1hMs_bVO*wk`jBtKYo)g4!1f3cU<0EiDiI7au-* zp^xpf;=g#15+i-N>~Tv1EziswfMv~4dp^C1#;r*~%-oP>qDF8Z_C z`GmC^*IOJR`Zod#O4G#t8x<$86jRf(th_1>P^0Z0iH3AtpnglndQ$6okl4Zp0!J)9 zS~Z>VpACYbI;pC|qlxS%!dQdLMkkVe6jshX>JMNi4*?%ag;Vy|t$~DxY?V1NKu8g6 z7#ly4qP;>&#CaX`i93O>UiuRSXp>SU_b6$qhitAL7X+!i+0%jgF5r>^tvq$)MA! zB}U*N`=TN6%s7~_)A>+|{D=8WNcmk(X?tknJF}C=MIJ-{X=@Ej^QKGn9-C`+Qchu2 zoM`2LBic5HSfw>(dds%VD=+Q0TFgF98I&R!7qc`nu$cWiVABNg1MV2TXTmO)2k&V1 zuyQSOayc$8+LGJPu|B`mg|sw$DgFADG3-9n9tipfWFP_l>>EQNkij~*{qRvC*H_)L znwow8MN+`(@cXMzRE|bO&_+!)uys0T(IP%9iVMJ%w3eSH4A^wPC@wB;G>FO3RW&KE z;d+0C!18v*WrKGA7vwEU-;m7>S#GD1;4W|2liK4H7Ky3%dVQSvRqJZxl#J1E3n$-4Q*I_M$@MrdD` z({tE7;>*HN5?5{gz8dSD-}FR**OsL}eIkwJ+j)T@T|a}5lN!;nOj3tfgQO&1ub*sw zFHq5V>ZqbaY^rmnh{H?dQ}i1=k*Uj*9hA_R7nm7?a+N}%zLO;+JuLEUi>p$8ca`9E z>VpFxj5X^iGfG-;`l*@NV^^8N`-J%d4n0M7I_O{n_>gk4H$)JGAZ z3TkGn)zBq|ax`DHe|eWpDiw6<*b)fz_LeI>z}_kVm%a$Uzpz~CYS>%Ic3EuQ5_7=M zM>&RRt4nlOHG+0XMUKEu9~lBrIB#Q8k~wAu{tI@kQFc`xw?!-Foc8Loah*qYS|I9m;jxv6Wt#Xg{ZGlm_xuj~U+AtwizL&ny&QSLfy8N|O? z*APuDJ&T$p^rourT~m>#hmqd;5On(CuizNwEQ2Jm;3f@wfM? zx;-CjOnTYh&zJ6ALYN_(@AA(_%}8;ySavg+zTHFg z8YE!^Y}GY2O8T$mh^WVq^r@5t3LE;Q7L#VIT)YmxEbO`z;wkXR^@R^S;MDQ+uqA+tpzC^0-GCN2U z>{nJke!io=Ge?|_4=aV@7leNWhiD?*oCBrswwRJVCv>+V!5{1VtE&j>$H z3BTIq05s#3OOd;d{ic&K@p4}uA3Y2}B?|CCNz)3pocQ&iug;lmR7vxa*EMk>h9fpl zpHv3x9${q$qyG5t4UUfL;ZJXYfkOlaAGOP%y&})jy_U-J|2S=oo0*$mAzM)~&R5Bb z8)u`wKXKvZhyk)Uj4pry##x71y`jVQLyak3FGU41gZA#8VYX~aHZ-N#FqCR;^54`k z{D}0f&m&~pOnz2K!>6-1>TKb0fZ+z^Q_#)EFvVeLe9&`-!x;DG`R8FbmjFZssM0~M zBO5?vi$`Ew0dNn~2CsX7r@cHANMNz$xAwK-!j1wkk=Il&W9m+G-J=gJRiI= z`1i5^t)bVfA297`(sOGuJqx`D#-xCT3ygLHmIJp-Q>6Gcasa1x2@8dmK|p8WMqQ+c zh{(6%-f$luWPl0+V9y?pd4!#8ef{>-1HeEFcaDz&3cN`L-Ddq*@<}Qohzy5tt1k2* zM(P~yP)`X-qf&7FP9Xd4?T1bG<0na?+Xdr#26`A&Zd9$44fS}~>i*-yVvr*4wWm{4hy3`-e6p&Dp>Wq6BQ z=Zc5hd>jO0uRlS$uB-#WnkVL4-Ei27du{Sk|AKLs(*s{A*Vqf8&+hjWT3yQ~QIdAS zt{B3@Hu>XCfbDk15)U>vuXmiUYxDd&aVN%Zs!uBL#*3ScM@aaoGl2DA(oV2<0jB%xu?fAMKV>%NyVsnEzS zUhDGG7hJ7ZpTx_Lg^Asl^(zN+LOWXyDoGyKaFMM?8)``GtcKXxXp`}o5a&K=VkQ-^#ez1ATY#vCKWi( zfO+@Hzmp0|QS;B(v+#ML%wiUrc4gQwy9c-jfo{L6yY>!_sblD|TT3g+3A}x?~*cKaJ8^C9$e8*jkummNt*6vCo2~CXn;V z$$qUxQL5)#nRlMNW`-q6xY--vXlAiPa{+Z@!Mnidez)QM8#kxHPooL1W8sZ$)jErY zyzz-C{OvHCjn7jBU)L~o6Sit2+Py#}fTLMJ3(7g$F-YjM9Xz*kPs*0hy))7?sK7Ne zFxZT`kRnEo?;O>;V>RXc+~h8|dXvjuAuuk?2`|qae^&hT&DU19@Z${W`p~(9g94UA z#a{y{`eb?q!qswmXd0z;1RBcMPk}WsMZ#57F-^MDz8W+mTbQn`*za9(9o#|8(9z?; z@>#d1COr5!X6ni-eRojQ-w_HOUK1y%L9*ItAsuYEbUrfKQt2{Nk;&*Yfqm7%V(`?b z3(R(JDShS{+`Y#irZjZiy_|EGzPntXh#s(+ytoZ&7WBAml@MPe%L5X|;!lqJ74bNr zS09yZ8t2*m3N}wx>*l&30y{bSN)1!|)^jhPq*kO~MdwK4a+QdK%WOK-c|#F7G`GxP zMqKDr>YvhuYu_GSIjUv?8(Z&jW(k8NvF(Cy=koCKM{7%G0QhE+F`0V4STZ;!UB_wu z2gyfC_F*VJ(^^pXOk8iio`WhsbM(7sJYe5534p};=I*;?N6&qDtyNV>qSrq{Q0K~F zF`(7~Z`xPudd3OALZ5~jE`uaKf2T|+Ra%Y>kFKihYE4aI%#(PNk$-fGHq_;I@5R~5 zU7_?nP)B<Y@?G?K(*rAsbZ*B`W>&7 zPP&d@MqHUP8LM)g0F6~^QTdyihMKyO(V@AYw>;ZBtC!uYY18Y5hS{{Vt6sM}4!4CX zK%U*+^dqEt;@zJON+@(7@iiurS5X5I&8*2P@o?BZF|9G}IR zc0ire8b9aK3}j_@s$OXW6(}p)X+1W21%}XBb4+x{qx5NEFqit7$8S6?3b2k7(}zb4 zTpT~epddE?~)cQhROV}8W~E`Zc6jeOlT zCo7kd_|Y;6t9c`k6^9>aY67PIaaLcK^G)MpIQYBNGCJU{94nr20S`k?>USm6YHyU zqdpOXjW3{KMl;;@8Y}xEd#P&^?=eX$mr3DQ&+az?)ZWL#Z)am;>mFggn5$=do8(Oa zC345qvCtBl|YZdT|(xcKYu!QodfPmpbG;Y`d9khPzO2{YGPPh zC23=1+4NQ6%x_`&aX%`lp?GO!@57ovZ~LpToy!d!>p(G?5aZVp?{MSG%B{|;(9tM} zVp`2|2I{CkTW3+byV)D`qvV-8iMqW-sM}Lf{oaZO;HAX$7%?j%W<-CG_h$+&*E|rA zOtP5O_7AZ#E@}JQ#?tXEiCT4GG46m zcxle)lgfMwcJYAF3p;eIyOXeoOXBmN$D;nNd*%JW*CqFwuCv+;;Lh zOu?Yd&myCluPmp88slp*dlm+G?w5dL^E9;A(E+#-XaPWY32oteZS9jx-wtz6Jumgi z3r=l=^Wr?<^$DAy{NP%XrLsW99_azh=oq63;Su_2;9;jKBz952&7g{ZN-q2A` zC8wZX1_fRJ>Wcz)0165Vfxx}Y^oE5XM$U(vM16z(DAWuq-cwOIM3VVG^YxprXSL5j zzXocJ4lPFueInq29*GIB6g*G0v9!^VQ`NcY^;j~9SKiblx1Y%-+tZSwP!aqXGaJ9b z=@vN#Tsn|rr5piRYxJ9@1 zBt>GzsG)_`buH3YyOqF%xxuKdva+((TWdL#5JZ06|6^a=F9_f^G4=(2vwhVcVayIr zrJ+ztCO~JuvVpRdM>y7Y@XhRZSa;70O_#Vi^c|+ZbTPM4S$-7R`q*U5)~2Y9Vl7>$7mra%De@j2q0Of= z45HhsfEU|sk##GI$`rrUD%^DIh{MH)m{m&;Q73A&JnUZYGN<5Sg|)5fxW7ME;4Td0>Cx zsD7|jb#OTpQoQKGl@d(~Qc`$f7Vt8EB(o{1pD)nE)5m4iki}MwNDGnEFqyYgQ&Slk z6fXO_-pl$8D<1L?w?%LD@~`XeZL$nSQlT9^?Rm3je{O8&Tt5;YHZW>xJ-b_bc2^AY z1az=6ZIVt?kCAu2!NC}y9{vZ^1IE%D8)b99PNq!P%bCGOMtpyHi9(M^8Kf3S0gDKg z_Vf=w3m5gX;j1fvjCR*~e|CTKY*>!S^>uB{g;OT$g&XlQva>U0Y>>y;P(Stya2QOc5LNl?4swQ`(KfB zxf*-FwQ`dW%ohd6T!ks3i|i)*($tYn9 z6F&iK6RQlb|H{4|A&7u!t#5or6Nx-YoAi0ZVm}M>QdIL4w;Sm#&GN$&5~@UR%{^tn zqJ{L4F*7?xzNr%rfUq%nDr@GM`v_6-h^`iKFP~PK5~)AiOJq=UdDyX*v;LeA>DQ3{ zGfR$~LmBp&Srv^=q9i4@$`uNPuKq2;y!VOPsOIF9{)-Gxbz;$@N$V9tKDG7&-fZ>~uDn&}V zsKr+^)qcAjTu4&wp>m9&vg>C&^&__3k;VlL%`od;Hg;^mkL;n%BiYbcmQI^AB(Kl*^&G6&yn{T6-P*#|P za6uF8HcA2A_4o*GDY{gphb5%zYx{+bK#eF2mG~?4=fU}6sC4uY5uDF->N~wR%jIY)tKdm)cpQ+B2YaAiV>&6l!XK zb;#M;r-sdClBi_-=M3^2JDvO2Gxn39(VqcfEj?ySh2v(y{xjM(U^T?2gRYWx3BE}<&vkMbn|_S z15%1F+V=rj-o1&;Z)pa}7I}$K>C%Oz_$56xjz7$hGm2PDLpi0?cpqzAh^VPd?D)?D znO{BmVmRCBL}%7AE@otc)CLpjswx7n2p9{m!s{blzTp_Nn1>2Nbl7b<*mfwswSS{I zPO`@&DxSUhUDn{U3%E&Xu*kDXVTCg?C>`A*p=J+{huV@gq!Y<8LqtB&t z1f`rl(IDe*4T=(c6e&5J@;{(n65T%ko=fo+k3B>JSMMzvBl%Y+8lz91B{s{)Bp|@_ zJ*@YcQ(3B@z(`syt=z^fFOmFv17b&0*>BP7M_MJy!)UHF;%HeH z86nSG8_9D&kf=8Xr*|8Ny@ZUaJ97#AjmCR4L9@gKdcPN7U_kr@AoBMt?#n@d%~?q? zv3EaB`O*Iwb4y+ClmbJ}t+D*jBhdSKSHL=3JeDs80v`tK0sdj>+tY_ZA>f@Q(&hha zeO+DcL6d$K)50D1fTd~6K@%Ydag2_3%#Zyv>yHkpp97D=No^%pS~~kX$e@`JT&_!S__u@jaZw zT%MN8#ijv8%NRdom<5en*yP4V>=9v`xDfRB?+`$g`!Zr8vY@w`~1mxdWddJ-TI za8efDowh+y@`Ok7w%8oeQN|n}P-0Z-z7_1@uHsV`$Ta1J|L z8w!L}WKtti&Fy^jY+Rm}rQP>WpZ$_3Jk6Xnf}iMSiXw((M4{DtLBjbhZncWtQfZMB z?j9akR$kpjWA=1V)1so$hiVEUF>@|LGgnt)bh=%~uEGx%7YFL{#_SFB${kZPTF2Da zZGX?e(sHy(6cRlw^oBJDHy;|O^Q3p0K5z`Md40$b4Hw!6!e`^(eQ z)6SLK2A=~UrAGQ@l~lnNc&;xnz_=HnJTAFsUH%3(?tp#0wY9#CuY)T$XeEcr%EhrT z8oZnPoA9LOXF$~*6j~S!*tlqQJB&vgmg_p0!J4yeBAxDxV|fGp)-nQHTtwum-=pU~ z|M)Vo^s}o8QI<=1Swsc@RV`G**g8X@M_Cpn&sd^)Ag<{(u+H9J}9j&qa$q9BpKyuJAQ zG9B0QL7;uk%WKvLMjU5X1wdT zT1F1MW%xk@=tLN_5+m>}caAOH*v6r7V+con7B-6p6$9gSV2+THkwGqFWcRi!|2Jcz zSR!(7V|hyDoG#)0`_{Q#Hb;U1zfXK(z7br2Jju72K$fP*?|eg7nnpZPr>OoaBYWM@ zVERXMja!l%IZpHCkUWa7-?|PCX4Q;C+Q@t~c&ZrTn)vtkfSRZo-Be_13g0z|J5qnv zPc?3k!v+*V?|?I*$|Tnhlx3pZ5Id(grY_AZhi5;k%Kkl%oU`d|Yi3VaJ_Ck=DZ#H( z87B$&z;)VC%_tpMGxWM67XSk~^AQR>)B0xmBwDzflI4*~Tc8RwNw zTXHo{w35l7_kXsoBPcWtzh-DAjg8lCgJo~v>CrKM`0Xw5Xxt4;UvU5~)bQ)#yUOs3 z!@DK#o|m3sX}Z7tFO`IbMGkyQ8qV!`1*UYTQ1$ zJuX%djm=JwM{SUB?N;*$$kX-#p0+M3CqIyJyf;zNYl6O%5B@|DN%R@|)kREAuo_+T zB8-NzL{APplV?6u#Yf7@$`->4^RG~596d8a=&zMs#1q_-<>o4k;UkF*vnqlVqasIT z=ddFB8^B{Od+}pJ@$1}gB4h=5tj*_n;B4i&)8ceXx}|^3onIr#(`|G6nd4h!G~vID zH^1S%Zq`;r@Z*7SjRSi*jfnUjY$m4iGLd7$-4P_*Jh&2q*Xg+5UVzK#X-G*~*X{f& zWIqcm*g*0-8i?qt-yu=><#>bz-Rs~v$zbnN z8OX`Gs}qq$G#Tz+Onxr$nz6N^p<=#k5e{z86mEr7@CxTLI?&Wt*S0z@jQKOZ)=Q@m zjjnQ}wLcHjw!n^dbX|9yf`QB}SLlo$8C}uHuk|Tqf8A594E5CEiW zns^~P+O={BOc#Pv7-{~hh$?7&7vFY`rZizmd`hyM_=gyp^A*o81dtne}sHS{RX(>U?>z1QZg% zFyMN&(iJjGrNZ+nQFgZ1{Pf}CwrBDD%W}MOqD_*qOA^31Up1#klqLESQ1C@EsR{mC z&dZDXL`S?zya3|7-}b*X(Vg`J4jEjNU~Mt@d;h)HfR@zVUHEPLw@vNo zW)~ysXRmLeo`YR3DWd|!<-kIWqjBOyP zU>FWZUC0r$0y@q6k#(62SXCuh*E1fHQ>U0c6VamzR!#K_wp>f!_DioP2a#WT?zr$UWSb-W zBC5iv=WO5O835aLpK~>Yvu*uqGh64{Hn%Hei%LfrlV5xR#;h+_%!cgq+{MmhVqXD& zFZGSs(Qa1*F`a{pd=^a{g=xdC0)mU@kT!_6)Jytz$xyvF6_nRyJEvXtNW06Sq7vnO zp@~#~E8n6Fu%8u5D<{>(bEo~X)_LocR%Jej%_Kv2TLM$hkZc7y|i)A$QA50;8 z_jL{d-({68l0lwf80%vxx^p2h{>P8g7=O{G8Rv49+*Cf@hmjXARKz|;%Ic`A zDy?=KX_t1GJ9$%CB4(R%`0X+hv*Z0zt$8z8Jj2l>Tqk?1I#+c(#xoT7KvJgQ<=n+G z(K|*P1EHB|6;1+AT{VI7OxOdm5*b?vGb;)DKTd5GJ2xtupO~J-15D2w%2#aV!0i^* z(GYq(-X85Vw4SC?r6J)%^5qi&US!kssB7!W;qQFp^$jWcz^ zYjn?n@ZRF|l6~`T)XIT4)TD9N;&5?ses{N8n99bUfL;z2lbTD5<1_2mn)|RJ$*YL8 z8{MR;h47Yf8h>V_orHYfDf;erc|H2EBx%briSYfc^C`$w)f68Yp#wF zrN*R;Qgt;Wyxo|CPVuBhkA_CJfOY|{4G;IiEqoPjkrUybSWW0S{r-0r3-qRM!%%B}D6U`dOYHe_4 zV1V`BlN_pql%#|m%6lY2<3?lURAne~(bzVuvNnBHfzfkcBr%a>XV4EaI?)vo{gfGw z6T>DFgm!++<}V)FC&N{qZ_uEI#A}@M#k_M(5-0mJx12P@7U>6*R}o+$KMOM;_nTRi zydNALNrPAWv>pBi`xOX>cC%zsR=mP1NqvMP$;-8_tmX9(qQ5}D%vs4Bv=i;#db`=V z0{JCe^bCk&_-%jd*#bC5Cwa>#VW?M-J{(s0jSlxVQ=nO&1sdV*puY#5hp-V|-@P+c zH8lpwAi?6c8M=BJvKV!15kn7N35EWeA{I4Uhyq-Swh#)Pw3l|w$%U6VCWycASpPgiPyTinZ{B{kVh*@=8TFfI&KG+x?=0P8mw@5u5)e}`U}J5D zxxW;-KcwFKwO%{@=tJv^(XjJBS1Ho>CAlqMc%%DRY?wToxn8d-g=0i|@0X-6YZ-5L zq;F=WF;U?+(y`ZAKg_0z{H?d5jjQ^W!z@Fgq_~rpKDG#E8E(l3Kwji z^PjW>dEh-f^z6rWtv!j70K?!j<{I0dbl_# zJ!b#8FB*4?*bZ@imF$07cCKqMpznu-UR~p<;8Gf>8;Nf{Q@=ogzYCfLd$Lrl zh837^Dr}WT_csQQ+QcQ$&$F;284p7fnj7XGr?Zsl!sIfzz(eCw3^1HcUMs$dk-X#@ zzJRC2czvQ%lG*QUNNkpa3v}3Qe}v@D@dvZoU(X+R!`a-_3`eA<*A8%J_xw~>A9l8& zINuEdcW#zW>*9g#DeEV`R}g4mGUnZif3P=oJ(m2UJCF}lDSfZMTVSqt(RVVz;=JN8 zW1}YY_~YDX8WzMSY)=^xZ%yxbNPNY8%iTCHe^N`&#kE~2A3`oqs*lh2`P$pJl;UtY zLL$(XU~CX6$PPTh7im=w;*hKLSL2S)9-^yV=lHbmP7GoOXKVk2j-4?RpxCUUWU6Qg z$DVw?lvlhEcd$X(iO{Jn@RT+bwopgJS_2PV=hDH|$u?kCoLL}U!60Zi66Ar_Ekvq# zj9NNF@4)uTXTK8a(~NsyQ8#smWS&3 zo8<8?YV+(Yzn=hg-D8wpzvRB%t;Jk}A;<510nip-9{={GKp3acax03Uq!%RdBnL`tw1($FvR64>L z152YiNiO9SDd!bhG2}nDr-QHIC?kLJTt~5QR*<*<)F{m6rhcV-P@cW2eGsnw>Vp0X zW}NV+DD1%}ofjX6rm6{Pg*8VeFIcSr>F*b?q zh>XJ-vF{{JtFPU&$8!V}X+WgJ%6q0c{)}P#VFJHB^zWZiyj@(ufl^$dDaX;%cW7Bc zP^s-|vTs?r;JNV+Yiun0lAEqnkat`GFJoIDIRP7!FatGmS7r_$fnbX6Z@>6sjqSrr z0@l>hda`G88g;P=E$~sbNJ93K!Dk*t$ngAOD*4=uh_gwwm}Rz^njv)cSTx=2L;*`- zXZBRb<=iQ1qLWNCw54s>Zv|U3PWSfhJD0dhEvNLG^FX81N)0KIl zWGTCj0)car5j3y6nM#-5sTFC=od-okX4Wko0J&i*>wWZ{c6B0_ zp=Z;gK52kUq+0iX9Oiq(TvPpKvOw>@Uz4uZ}y`v>{%wT#@^D8I*(5(vgd1 zWte)0P4;tkE=n{`fx5j;!h6VIsbp9wRJtlqG~^x?br2I+X;y?H09yor>X%$BO_OkwJB8cX!|kP(?{& z4)6EvTY`=^CY4rhj;`&`uCK?Y^JnwN&=>FO^RGfQvE^4tzx)!jSg8vIGg`MSlgbu_ z`}u{2-mb5*>;`q)(8wnU)+!E2j9;3j4)#euLA68;c z62uIPa-8d~o=j&~PEV|enVRWC?Q}PCc_T4tU4AL5U&ta8GG3BkXHK66%5 zHK%Bnx%vBI;3cxCIv9^b3ovx;&t70M)7j;Kd;9bs7P&a&csQWvq!2Lg z_1@oMfTI(jNA>>wyAr%z7$Dra4}5uiifSYX;3S)og);ZoEiy$P0U@a;UkA4fkQsaGO?Rr!cu=2J17+@;=fBxXXPKCAS=CiQOGaH9F(U8Ur{ zehX>Y)ffpKZ9?$a2%?QY(@ott}ikioUsw z66j_@(vd871p~GJU2so5JcM`g59t=-RXtyqLlMbEN2V$+z>fHUnf1jp%jfqA()TC8 zM!96@;nMvnV4=ug1QtYBOt`55Q|C8#!&>)FKqwcJ=0hVGyDsE+s6Im}nj`oh(;~^$ z)c?p6@exaL+9R9iW$0xOM$Jut6hhJA=S^#DnL?}YWw31%A*(rx<9cpyf7t#SDh+ky z;e;j1aDzUVB-8P73em?$MtXbN>u+R`G199@@++>m3C=*!yv>t2Tu6u$4c-cAr(`jY zR?IsJ)P5Hbf}_Lm{N>vTZY+B+O=yMg3ERlaJ_y>xo>ojqjA89 z{sH|Q_pvJ@EB%h-unx(&lwzAl+S$qd@NW%m1F+FK|%5DH_iMrV))#y z-+?{)e?c>t`=dwJM{nL7_Ph+;gFKl<9kQ`1QZvWuw3&mOV(% z9%mG0JCyd*jtGU`pWZJltUWm@i$Qq@4p=*E&wh0c-!~u%r#j$E{+a8q3*4$fkJLB|8oiE>)eLOylgrwPy`KNSuksyExy~}2tQ4^ zn{@WW+#ZJC+p`^p{@2lCayx!_$B;?@Uvh69eQ$aQxDAS)zQPgs6Kd+#=pho^LiX>K z7Xendf#QQfDg=dE7}RBC94skzOv~IvN)@Z&dBIhke=?i?bkr6cPX^#EFn(LX2kI|T zq0D3vsX}1tei&^!w1ZtnnVJ}7jjc9AvpzKEi`Wxe_(Q}5n=wY#zm>HmSL9tdK@?Cy zHLVhx9BWm5=ORM<{aY7v$$*8jbIemH&zrB-H0c_6(RwiGhv*ohM=%*AoG(3Jv3dYYmFBnl~ zhN{qmxxY9LkVS$B_#!7giJc(ysN#t zM@VuK;3eD{79*tm|O?#ac;3JkBt0yVBsyFP&9DG=l^%!`g5tIYIqc^f3(W>bGk2Hj)_w{5OJ0Kfc-#jEe8b9 z1XK{R`U{X0w-c7Y0D7Swj-e4DXm|~YkaVKmZ&k{@qB5R~f)UxA=%ufz=x2Gh5|zi@ zmoUBIWE3ox2Syh*i2>HK69v#a){l$^XwsN+X&!JS^;cB5&;|w`?G9C%?k}^hEQ;xA zcxr`b8ubL--2|^-mc845?r(^MpDd9Ai+Lvw-+)N7OBxss3tO7hhx2x-v>IV-PGzDw z%mi^5B&`l}{Pmebvn8BK>(xFwr`7ElwZM{s+tTxwpH8lHQ|szsK0+gp0Nj|65<&&S z`?!S9%Ck(I_JJrInWX>Uo(fEVQNjn8g$BkagUv0fOZ#AW2SyLmM`Q*T3XwH>0&A|= z%q8mUyyxb1TuO+RTJZPzd0LQ102E3E`Do02%)LFWW7L^X5U08!qU*wM zpK`O>RJHQ&l$NxlB*0ke5SNs6%i9796rfq%`whs4?#9I1Rpq8Z)7;x|V-+E&e zBsQO1c{fkJ5V&}q+#>8Em&1O|-Co@fO-onGLvi(WfgOpN2(7UvZ-A}My|p-=2PN6< zkYCV&U-}*8Khv&W7@J=dyptz2Zk(IqXpj`wXa0KM8oipFQ1vti>o&EB8U+#Oq``eKTj?hz8 z0r8UC#FOj(;M;*em~@FO%oWKW^A_`1G!!Ubi06;Cs;;l8f$5j_k;;-~Hj5--;-gJm z3}dZR9bHU8?-deZ@cv28(O?ZS%HQvSvp@QrjD9Yx1Tw_PJoXtr397GEWSeyU+u~6#W-6ig4(1)YnqwgU5|vm$J& zlVE5F=u>5t&|oqV(-qA^ko^#Jz9W~dH)WY({;G>Q&528=a!BmW_qk)b2sOL-UuAKd z8+mKh)}L8k&1B9B)!5YwZICrFjV5|+-Y(>;G)9*7s~ErS=lS6H;4>2AgjfRd=S}N8 zwExSDI~n3HqdxfPNbPN~+8ry0sLzC_hh7&2uNmK$sr(84pye$i{aBz7s$-YzYup}a zO~7+8nNn+Dun?hb@2}qe1jUn4bWT$$7jX8SP!YkZ?=3O$mtG#wT=n%|X%ycUv2#lP z;XC2C|0~i{+2vF41w&}pRgeA)M7FgUKDlhc9|Nw@70l5WY0QZ;5`YOHPah&Adz`0naE6Xd*C~RQvMUF zzGu=&6PZE;XIuQ}XV56}B+3@j^4*pOj~aK>fC7n9d~V`@#98VTSCzm(g(5+za~=3R z{~C!UxBYu&UYs4pr@CzEEP)g8nSrWGe5MP#KgT8>md^giKGnoOR9ammOZdpV(J1^f zHhd>`huLsk(13+re&rF8I)UQw*1ydo&qP^wx_Q5+_-XcxAw4a>4Wagrhd+|GyBPza zRXV$t22}X#1z#c5tb#<}{T`>aAXD=A;&crR7F&}DMt3KpV*eQzLQL^{EqNZo?eOV{ zZ4E{uM6qHWrJ_)Rex=`^AGXPF+ZO*&mIHxGUs-C7ejyXopp(Z|=IW7yWcmGm-|8oD z8mc$DIEIJIP+%E?$?&*6!VjNn-48zOeKOhy_|J`I&&JNq2BQ9X8RNW2&EMw*gDJ5XZN@z{tyY-$*IJZl^;I>= z(mp&DPWrTUvBF^k1`)jO_8`B1Jf%^7`h$8EMt%^dFjPGbGpUGdnY(1w1(RYYfb8h; zlA1K+5K<&|nw$E84b{h0H64k)^jX1-1%y46m4p#$yI>HEM&5R8)g7TfA2k+Cwlc5C zBcP4`#M{c-;sLGDT0dhOU%m*wr9z`Nl()%{lwwxm9BUXC?ce;&hg}!t>BJ$Tiapuz zfpxWG;;+G{;gY5~QSF(-q%bRu^W(o6mXhIr=NqLMZydg!B#g&it+BEjDmOwZIZfZt z>i#h1RgaGjlU@X);K?j((ZaD#R5v z?XtWnCY^i^dk`V#*dIChKZ?%75z0S~<7XdT?%gzU}X>^-ta zb_m&9cF77EnPrzf`#rz=2cEmftWoUSM+Z4q{^ikxHk;E@(xug zTR>`5SkBVh??2fVQl_=JXI8pTg%euRON|N99N?%4gtbjPxp8%KUv%+K|6FbL}A3397e zz3&cKQdS)a4DCw?M7tC$gQbHe+41B-4?ospvRJ10MVj}1=KY-($HEwbK?ngpwr7f5 z?lHNXh4n+TkKTAbzTEHKh>c44VrG66K(F6GH3DM*cR45t~qI6V`)^}x? z8NYo)oGC2Ar|2a~%u`oO84Dl~XTzd~G8Y{6+BpX6<#3=SP{Gog&Dc~#OyN`ST*U<$ zG<7BuE-nL8Tv}KOs4v)N+=kpf@d;^2#0V{XJJ--Ma(Iz1QV!WqUp$D5Kc%C~>`A_! z39sGNqXWcaKod!H%HD7>MeS^|Mav!|8QGgXl%2gmDhm?03Gi-TfoLbcg9P2TUXZ*k;p)me%)k8(Dlz7 zAO#WFhro+WY2Waaj`LQ2vi$ZM6$#sf}+;~;A}(9fW=iL3rc4v-N}UE(}F`C&O})#bG85MDUk1#i+8zGNG`e;26`vtd;mK8s=*UR^ORGaqulfVCNJ>PzPG!MP5o# z1b$WtN(5Ou7RXVO7q|^&27>EWI`ROnYW=51Zxl=J-isB!v{y`xU0VI zCYi;d=C&!m{nhh6JY&OAM>hhW3_F>B&Tuc-35@pFsdE*!6v);v+)GH4?`)P0MZiDj zT4It7@Vur4ur-mpZF3Sq4E(VX4ETRXl^l~6y;#r_{v+#-y8H$8s;hoHe}100Sd)}Lw)v-+a*qs;9Ie}(wQu?07dAj{g6cF_6xHd@ahkzZr`(^IbFI^Lrzmu)74sU(>f(C zoS|fd;#`3=cDOURO)6?UaWOgNodzkC9s?&J|H1acHkZ>~v^UL$pZdC21@68aveV!fuxj9_mr%V%b)M*20C#OiGQ6iW7+~;|g))4>T(L zrI6Mkp(&r#loi~);H`&C`&$I;EcT;t1071Mhp~cD0c>(*O6)eRNg8^LnL{J>K5y`h0u0oDOy`l&w!--)+G9AOUpb-#2_$+v1I`4_=dBEXNqcbNG zg|Z~S5}dWy@3PL7b|4WoQTpoXx-X#Ps9hRJd z+Ywi|Tj9nTQ%}A607>$wb&^IB$Jm{?fgK#U*?AbmR0(=5LAcWoR#mX?*Y?tK&Ud8W2M-;`SL; zso((1RLR@pFXp#R7kRiJOGg$K7Q)7&_73zGX zfZGZ9^Dsin-R;-Bca=TD_~Pd3>bIL@MfBFnMS!;MkT57FNd$|!??=jsqN;_(bB^t+ z!ZvED_PG$f6N+9<KS%Vo$7SZ4T{_A0J=fBl zl_x}el|mMHtqITmIV?8l)asB$yDIbXJt}CT_SWvN(NJ*1-&bekN0u~kOta6{r(NFY z(8|tz(#|qad!T~rkQK+EZ=-6f#PU6-f^GhT%GKapMZ4Q3NUy8wmV3%ilcWBd5=Q`>ve}5K2=92 za+AxXEPJxinU)kE8>;h0ZxI&9SQa#ig{MOd|*x?B|Sj+7(+CBn4=%K|_o=IJrYrzy~pm3b` z^v^%#lx;R`@&sJC5h;9)C7oTLH+>FnZ+Xr6ANH zt(6?(zuJp2Lh4PI0s4;i4kzd*m1UyDEXwbHNV6P1pyn=4Ga-8hTI077@*XDwN8S=Owbzl!Pn!9{>FVSKTDR6t(cZx&k{uD-P2H=on_@ML70-6vbi zUgvs14h61F^0H^Dm0bUkSt=gyAw_y#>}LAKh8pM`C1sd+#hPsmJTe8kJXoZcVDqZh z?H)kRcN7F1J$EURSL7&_c~8^hqYgkl99y^IbFmEQ;eZdI_E~HlbTpfkIGt(}biqtf zr(}Lj!q(NPU4(>P6do;>lLAOSQXa{Vf^ZO+1y_bw3e5>N(N7ES{Zv*td=ntKPZ@hM z`n_|N5o%OZY;#biVGo`c=t zHI^(4$G9>ed?zP+B+^2KFP$KkK6qrHyr@*38Hb&KMq$|=DYa=9t!HP?FK^Rqr@U#) zk>J{ArZIz$z6^AzaaxRducrC211^u(W3SQW-aL!I<#BA8Gh%u+#0LX%Z{*S^Y&Le= zYM0o4&z>F3R?UL|08{_6yz2c`5rBm;=gbX0mDSgMQ~+YjCpyj$$sIR)~o%OnLs4tfg!#_5PdKu|*OoUJ>V?!iQ zn5~V)8#YQYZ~%tsxZnIPl=39@VuiFeMn^Zc6+9C=YVhcxkqD9U?rVrJi@X$q5r@)l zLso@XP!$&gw%B(XGx}qjW0Q_syXv@DX?A8D31P-RS5z5-E2bKR)KrSHFgS3vGC^cw z;BYCc^L~j^Hi&le9k1660I2@^*PRZ~=vRjwU0(FX54hSByLL&f8XBzoF&a8y*V?~zQ}#>#6aGLg@XP~noZ3}ft(<1 zqAqMn+Q9FLF^bc$%5DOdd^kD~qAhufbCQnEllYjhDgr3)4%E;F=UB67-_Focj(8czL8vcg_V8A~Rn zXM|qI(|XSyaoVv^+nF%e>)Ux4#Z|C+cy4#FH#@uG4ZK}Hed_AGq6@m)S-o0by%{{= z$?NoKT0Z=@^82Xf=I5^O+2u%kH-H&RA7lOqgxj?ITZ(#IwP$?c zbH%+QRo=BioS2MTn%LvWI=`@a$_&)SOL>q6C~(x%B=H2-mn(z6(}{wlHwqHDrkz~& zT1FXADgb(gPP$wV#1P;%?FoFP1~1wQ1vFaa+a>Q;VYX4lHTJ#6-k(pEE4+9IG=YRHP1L#MBqjanJLZLTB}n4YMe4ZrAf~aJw~Mt z=!1BQX;$_DVuyz7}94}E^9yLn&shwaBgEHDwISKu-Wu7PK zkAJ;EF7V0AbG~S&7bor?vAp`YR$2Kg%)?nCU-FaxTfYDg_qG{`s@7d9mQl_D z35cjTUbGthUAQH8VzE*b8-&@3MuYa+)MOLTCs(&Xc-_zbDE4cMRgt3Kggj9u{DjBp zq}&9=?0>^4{e9n;0mh8nun%llQE=h1z9-)-)He;jLU-Lnji5=q=d$d&6~;BGi)K?1WyvQ9dQK25%A{=OwI*QwgNe5 zMm67R`+rPN0xjsm_B-#V0H*@XWEow}TmS-%)4xi>3<-Ho;_|ZmR4Qdt9ui!?v2dKlDV*wL9C zk@bz6p{(A74g?v=Rz6`f@~l?K6cO1WHr!tE3knFRN?M0tt1B8j5({VKC6+_Ry3tGb z@`QrXpu+oUjw&W)bwL)jc%Ez+vR$c>SIb0)K}V3P5#$ zvjd171bCQkdRg8}2kij*jJUtUbjI_A!=iILK`?NR&rZ~!IWx*9MYNh}!3{~qNhH^4|Bc{5TOF1GDhmxGj~wid_Fm*eB9 z(>xrcgZ{J)`NZwF8Wefx_fEArkt_n7i<-#i)60jH&@r*NMtlyWzYl|JOXS1N ztk;o_=NZR3{99G~QX*vfO8o1fih7cYW6)j7+%pw&Qn}RCC7RKR7sO#k5w+)S(?G)7 z4a4oOem8Zb^xouAGjKmpRm>{mBMqrrX@2e{9STA3vhiH28$bV{Bl-~@3nc=90#?13 zOoz!_ed#iPz09Q$eDL%-!C@=;o6l?d$55z(c1GIU6iYHT3i(7+Tl!Mrq}1eKUV1c^ zMFEJfi1ZOhgq|)VwL6@TSQ;ty{zx2JD&VYH+A`BDLBtz}NY19{lkgp`@s=C#1N_i5 z0ir0?tXlSOmTkqO;i;ZHJ7QsXA!sdZX#J3mt~HmT!rbCmEP=g{FzWd`NCabfc%lX# zzlBU3se%nN1rLg?iuan5s4z1(z{$k?Ts|ZuBGK?Oe{YF$=8Ct^stm&9is=U8%Vr{y7)NTRu*9C+MK#iQH_4fSg|9sxV?Q4h75?{crn__5;R+-;x za%C!dFYEwf!lv@sACrmME|Uf+fKnR0eAi+zpISD?N)3BuOtn;U;E&&=ROEe_)4KMh zz3X(s71=S|7asOCkl(`W(r2jh?wZ+_c|W_@+*na^dFe9%)rO)}q92h;*jHf46Af1KsD z4fsklVS(V3u|HRSx-2z+^0j||=+zF;g#WD@>gqD!GW=axbpm1rv7~f$6-J*E5X1cc{o~qsbmR0L{O-nN*QEL5yV~{j`Eciu)~$ow!~tjZcs$q9CsawgJwP7 zwY^Bjvk}-VUOn{Uthne01YTmK+_GR8Do%dnwHBE?xL>MPO-bfEFy-T5{t08q(Ag1z z)%h(_up|fgJ3im|8Utnz;YxT#WJAKhPB+<4KuSyx1_UiA+DHjz+DK{Q=vf^+3dS(< zmmZ1RqKs&;TmjvYb?}saHj+X?Xe_e;n>AANFAQagVepy!YClr?D4X1pVcFBSEz084 z|CGfhCqKWIr)}||B)gWCY8U;-J39e8X%&_$lRUfO5)OJ34NU`$_*#muPm91%rPGPV^8= ziM)uq!2PHKp*N@LY_+9)L}U0wel70bNXB2@nvrgJV9 zdb_?B`nI|_s|YMiY43fRd*T(zU~I!aWH4-m^?3#kLPl5i4^xAb+0R!#5C6X$-Q9}K z*)!hx0Ss~2qt8PCQwARnns?yiqO(U%tZWkNzSlM8;BTMN(p*cUp@)~dyPKD8>Zcg{a6UT`KHHaKS}+H}5V-r23?g@-#_z%)}yEhS(TJPhNN4^AC_m{?ul+zOF*?N)mU#tN=(LJTc9 zZr}Yn;}GEjiTj=3`gc)4X|$}_$_#WVp;e>IYLl%@KfEnES44M`Y3q$MuJVW9`rlv& zkE}j=RY@Es{qL#FahG=*G%2>x1Co?!qAS}hNY+^MguIBMw6|pgu8b}DMCqw?h0nmq z9xe#M*gV#A(#YmIZNQKJ$n#tB0LBc3RB5FZEoE|LE7P1J05vXDI~mhy19vGf@Trzb zO$!kD{_lSP`s%V%;PrIAgx~r8MuJ2jU^d#l#)q1knwv>@0avvla$=mEy~&$uu-U*M zdV$3f3^iVDqj+)!A9O_uytk|1T-a-?-2oIqtpG9p*Lj}|l1(@*d0J(1DT)9}DmAQh zs_o916VHQmT%u(Cwf~y996jTHc~UQInH(Bz)$QE}MX0v9R7Ez=VSl}(Pa#42sy8v0 z6PO+G*J$R(L>KjA5LpF}%EO|FgQoS#5D+rf=*@uc73cXqr+LqylNL$b-pWc!*0Ge7 z#?)^>GKGGX?wV%mugB%7{+kdV!JIAGP@=N=&%Zk~R3yAi>Aczl16n$}x=foVw!&gl z25;9-tq=|vw-SLpqYroK%P}r;FGp{07yV_;_F6pcE!H^ZOq7*O?_X2M9m&(bKR&AF z1=RG*%gex}qF^ridl#!g{{SXK=qVsvo}U+T4y^p5MkQ3!(lnPYeAB7>%x`-ocKUz` zFLOAQZ_5$3Ga&LA@zAu61|JVRWkf?N_cYY*~r}mRd_ZWU=z9IH}C&KjEfC zEoIJOz09nTU@~|#2m&tb8-soA19w8{dvv5R45ds!1+e|l;t{I4QgoxP>3YV3T_DG#Hax4AhPVAYjVt6 zsiMZfMeRt5T<)0gDg;XT`xE6n3BdO8?U#RmrW%g5wYh zWj{KPULfK9b4HkuDVIFXr|@+cu0Q{rA8qODb{{gw%6AD&fG*?V8}>63O&SZK`tvQ*8Y8^ zCeAZpH3*=@0`C``HNceE*6ENNL6J~|fEP*7fDZqkI06Wao38v>U1fHBb?jbR;d^O3 zJLO^8b!khrjVWz^Y?WWbV+(BCtzAW&pR}P=GHg^d;$;!7tbAoTDEl(iL2`9vGTta@ z^sAO71pgS;U~X(57K`ZZho4>CmpGui57W4hp}L2VR1Jx5)!SNGAzw(wfGoa(f>T~&Xc zBs&s<{{;b9KMrX6fC1&N^&j-JbF*p84da@diak$Aq`Ee_Y@ZF`)qu}oVwI9vSn1!7l>OOk$?+ z=Q~H!nwz^G#3coz{wrIZrdqr`iqxBCtDq&Xsj-FPKr)ayia=vB>-W=_j8A`RQ1GQk z4x_a-MpghqYx8^+5DEkHfkl>bij)$bpCJLe9!xj{;d@!bX^^|1TE<} zFC92_zR$rAKF$B|M>CRPVh131FMIY^>FfV*oV|(7znh5yl7j>yu|iO=k7-<(H#K%4 z8qY#xjsZSD;I9Stb52{^PUH_+=)T5p)o8zL&W)*Xl6WVbO)--y`A8_2B{+-B<(<5X z@Z*PBCUpiRtTtMB**!IDu;*zBxaFzUV#8ahp_FA+s|1F3sR<>i)cSiFlp_&T*ao4k znXOhR45y|wC6xWGiI6n7P-o|vWRdTlC@VmX*`3Ha@FE2zp0^3IDbgaC_27;u6>t*TarsrVmMn%g>~XJ^2^k8Pp7~Y$x}3Aip;_nKtJ%h# zpre++lVZ{p@AS~WRI_CR#lU~*+<8Fv(g8)Dfy#cvXl4H>v1_DaI$=k71A}Gn9Y^fA4Lg z^vLAvp=7sE1ZPU4f07*T=b?pSZY5wDk`mKz_3BTeEjaT zPCb#=LBv$C{rt>I_-@hY82F^(9H)gcySWW{R`^4<7Ci-I5x~((A_F4D+Lw4e0Lqgo z^V8JXJz(S!%hUDt{#H&~mpY4>p|Iz1e!zWk_g#C?ZNk4l7v2Bv0P_48022^90z5gq z#8tJ%oarZ9uT>z^u3mo%jy48yZeq;?5e~bRZg;#L7xy>%`aH}{41>Tq7t~}_NOl%* zP+C!`UuX70?1kZ^k?*CHm|`j8`Er^SY^+AAR+ z=Yk=;$*6-q6J2;wwSww&Zaz@vRfEB^zSC|7R^e>pN|lvbEax&X{Kx{xuAwbPmI;QS zGRjD3tD8B87hU>V`L* zH<#zj$Ip)2d&3$}ojTb-s?)B=KyMECad%k9e3m%3xj>hMTt}w@)W*@g4;*M4Ittft zW7t!7{(xWVA8PxqWaMlx4T^dmyM1i`Ofq&sa&nrg7NjV?)yXcvLWNQ1yV8S|Dux#KLD~o%Hwqt|Ul?Vl!0>3?VN_KE+vg&bM(2Xjxiax$FDfqTo5;>>(SV@2Pm*UNjW-icaKu}AY?7&@MT~BIN zMjhl2%~8U9&lAH7>>Pn)>Z@TeL9VN;G-iDDlXpFIvUXr8(j0vs+|sy zL6NCsr4)uh*s`+x06URJ4A}pzsD^kcZz8h{Z!jkxG0vvVAAlwWgMnleWzk`z6OJpB zwl`fkp!9*h;8)c(yUorHxF}WYN;^YhP(y(*Qd_^~sfQYG4i`5MPhFSqlPKZ1uJjn^ zU^v#H!t*gVMtCreo1C)JAU;27e-CX=vk@odUDt!Vp5CRBo;wgpeZ!giq>(9yVVN;J zy-d_{ZmhqcVjz_r!-n&%WG8I)C3)+X#p;!dEe-Ss27bydIZ=SSXPR4< zdL7VWR1-W!P2Lf7bz8*)phrpn9g_wD;*|gEN_C)5zFqIbwZx~p8J&En%dUJpwMaCD zOBV*T3(FG|l{D9DVr#ICQ1oJRllE_7a>qo3{P(vcqb9_ zSAflw7+zE#il0RG%JM+c<73zKgkGd+di5)q3QHLej$&42<7?{S-o)ewud9@cUjPGL z>aTABmwH&E2l93faDm>&C0MSW<(L9W+1jiLlpYJIxiI8`C zUs-9WnIL@iDSbzH$HsFSxImj=uFU-F)kj1Kjw()mL#Y!ze-nkYpRkgwr%wIs#Yfyj znZZ#paVRmq*n8sqX=cR*dG@1ugK6g$nn(-G7!?j1<3y~7r>SfxBvksZ>1xi3cL2r> zVSKw*cYl3x-z)j{`t&vS)p{(?|H#PuSU*MrvhlW1G9ZV^5UZ+TUek#r>zD zPwoB*h+Sb-I54gsR*Y-$N(O8n$YL8n%-%pfD0`gEx9sI7iG(|4cq@uU34cH`r;nd_ zquo!$J=RKK}RJ+%_bO1Ecyu?5n zQzv|t9e;xDi0P!C=8U9fO-rkNuKo)KQ66BjfB-+l_U$du?an_Bh9u!t@0?68^kJ7l zqAXlb45yxHEQ->{uW6|kV)zx&Xpt}?(4sIll<*;cSSEH4_XUQU@zGBn@?837goIE%s0wMtWvx3b-upTwHYNdNMVNS0TNrGB;~#vF!Dm zlPWX;NPuau%z_@Jqxyji8dkut!D$#K^8Q^Z0SkeS@LgPx6z$(D+4c z9%#D1oTDE0>ahzA4|(}WZ6)qtT_AET%1+Lvkt)}NXdD4x(aY8I{O@lU67CO=?tdlR z|Edc-z5|jy0(Zm!nBE5`v7;!}<0(?wA|y+cj&9RMKnw;Qu=-`^WUvDY&Jn8HIq_L) zrewAGdy0&J0cF9Et>>4j$_7Q*-La*ZEdCeYwGnaXOzTlQ?}7K4ed|qWlE3Q&y%X88 zyJ=m7*<86Z?Lv!lZId2sXUd(^^SD-W5SbEImEmRe@sl)a4qU$o(bcLL{T7Wrs4n_Z zWM@T>&6aG6r*K?xS*c1n2}rgYIwFr;5kszA?pBOxMnR7Bj`7*9*dY!VXTLss+WXd@VFv9*b37e zI!#=t=z59y|HjcA5Mfm$ihqB!%!l?)>h)wfoIEPWO_uFm+OKEX3_zVm< z$YXG+9b`PNAVvJt&9lQbTJ%Yg z!z^!wKmG5gnaB{8Rfv$Zr^~rtBfxhbmEy`c7|Y2GMjB@RI5o# z8@k?_PU_}{&698Za&X_4Gf-Czu?$)9V^z152v~J#S(>$PT0I*37z@U6wK-H-GzO94 zu49B|`1XAiLth2cI-Mo7^s zVpp@jsbZyPj)h~0sTavULS%S#o*9T3*9qDhl6_ z$;7Lf&*xbLqK4!vYpTG|0eXQ(QR+hx?9ZAF#WjO^D)Fk!cH3u2YRpr~o=?@jGAB=Y zkxX}8Y{_9o6KS(>#_p8+7%DK0%qx;)pKIU|>U2uTin4E6^-x|Ru&h^{%LGC_%6yL3 zq>1Wl|Ia%EtcD+flyX?$?FL=G_;neY9(GW<4VE^ZIAtisl^T69D>yI?XehPkNp0G1 zn!aZu0w{_b{>CVR3zrT`JmnJ8Si5bVRQ0W6ICA72b>NTjB%XgciXN+b@!mi@Y~z(0 zlZ zR=zXlH>R`m%mRKl7Dx}bM(Nk*6$>S$!5@(B3!oWg*Wa|z-V5~hRnBTw6D`LHNO$~B zDI?C{P6i_6a^_tsD}hQI0A}tqHUpe73+3NHyK1=c0~ikg11_PUJ*zpRGgE?~HqxN=q{u*( z$g0gJLV}Fe)eq!xv7VK$C^5tsmwV8zmf=C@<1;b>YSOla&1sCMd z17pNy=ZN2XwNjd@6~pirp8aKqH7UE2Wvy!|>_Zb}4Y5R%(->~}T!GWToTW~~1tj*7 z7PUD2w*t8wfpZKJ3Fu*om;9Kl<+|vya)b#rhZW67*-N{Qo--iddAJXKS9%l7I+ zI**iLVejp^UbJ=maJ>>fha zBwrOm?h(Tvr14_(G-Z=KmOh+4&g8YzQjDTyJbQX)D0)1q#)L_DyT{Tpoo!%W3f2oJ z^a1ryBu&9kJ;HK%!bLbdCjuS(Ebg*kjiN7n99BK-I~iz2S{bvP;;!mxJ1s(HEUAa8 zG0Fbeic*Sp?>w7j9G=-6&h0IEwyYjeS^R+X%^B(IE4}SB>jp*<&MhEb^Y4JT5qKjV zbyhhbiF--=gLSYt3N&$XYjBcErn$$#7*2XnE)yI@L;~faa;Wk23ZE*q;wokHtHG-0 zf<{Pm`gl$U-4^ELld=Pv?T&`FIg5F&jPhj5dD1Y_Iis3^m+a43V~bj4X-UZVpX`6N ztYUyOe;G#%M$=f0Xl&V^x#GQ_;RJrfTuaM55M$U}>w9@zL=(Bxo~CHO)Rw@03E9XL z-AeM8lnM2D!Pv9x>w?~njllYGbW!V6TcgIqpe!w%+NyDG za>oPo-BD1i`|*P>bzPOzVgo;&p8;y1vxA9$N8NYxl2<2^egN5C2*7ETC*#06xMw!MZv#_&VV4SKxtj65;_CI7Pguo@#vjF#|bzwY4mfhfMzqmTH_e z_ATqtnR?N6h1BoO90fHB9cbp8huRoeaMwpmGm*UVY7(Usgdg}OjR*<_$04rYuTsen z0&&KZy8(}(gj19XdWKsTn)RM@9x1Q^U-UEL7fh(6w$lN@h6fKxRG&(BjcJreyp9fay>zkexA@o0wJ)A& z5_1(*yXmbb{c7}~PXh0&=#H)r{qk?`GxCF=IJz)S^jjC(MnfWmyd-|Rp)j)*Ra;&$ ztCYk0+C4cj?RacD{M_(u1-A-HX+h<0n{@;u5)6q+8%9i-Uf;7daBK{}@b;!XcB_Lw zQgXnqhix+t5xLU6K|EM$Y$K1; zOx>hYZeW+2j-7!)*e|=xdk$p+C(W{R%aW%B=7MVWRvTBbo<7E&Q$Ohg>OPpnSx>2v z8ipE`z@0q-eE?wWr7y=GTVhnL0nWnLa*{x6T!1!s(%P!zjhGYHFNR zMJ=PzGEphfE~X%Q_KL6(cn>{WG&A-ZLZYd5Z+g(Z-r)O>*Y0gs9?I+r)~xrI^PZXY z+=AX_vd9B&RjO}#MtJ*&=RU6T%mPKz`y%waGNv02Hgo_D)xD;czNchM9#PCyrUYTSKhtW3aDpfZL=?^6EK4~KZ= zI2q4c94jVkQIo}D!5gMXeOVsDv5G8^tacq2kA4E|AYfs+;&TeX_CI>ff?0**!}t^4 zJk!qU?C9-bc=9==jFyN{>$`bU&hA&$L2VID?KT%ATf=t)*u@gEA;CPAxjidbbSMwD zaZJScouPOv>KPVj^e2L$-%l&@72GoEH^+Xp+x1iWaQG@zE4lXQ(Szp$ui^+RMvq9C9QSjewIW%luJ!{o7uTpUpy<3`K+p9VlyP;vdBkB?dN_ zm9=|iL=2#9)6^DHwolKzc?F}>NbOhnT$9%<*A9GTnX#+Q7`X35y8m)S})GXg#7>DJ@&{0Afe{3 z^jF5}^M!?VEKD114OP_vu3O^*#b3A-Nk~)ey7S$TU)^G-V|OZ|sq!S!;%oFb+SyMP z;m@P0PdXPI0i*F!^K}UT;Zm2l-eygpliaQhg)+Q@^OsPhygdRs>eY*>i>g^5!wMMQ zV4tpBuro+m*Q6Dg1_}^-vA1ND=69eo$&~~UbK^MPMwk+l062!G`a-t8O&>Z0j zh{nBt%D6K6Q}dC2=#iY`nBp5(ySl<1ZPJq^dvlbaHG>=ys?UI%!t7W0gid`Xe6VOA z^Cx_4m|9e=3-csm?1Sfc+e`W-@9C1FifT0cDW5G97#zOVUaQ?4{B4SCwon!~_%3kO z6Gi?Ekb-rD#xpa6R4RgBtR0MmM_c8q+2m;@jbw5CRWqo#A0p6spMEF@*P_I?K3a#J zuh3BE)XhyQax2O8I=X4~ZUXBWZvtI$tQ_hsW!*9v83C0c`hxc#vBQ!-9*7fBeJmRs zHo8EzJHPet@bhQ{Le>l_R4bP_>3-ib@^6IOo43DXR}YqCVVurhT33dzJXR9_O_{7! zN&Of_j9{^0>hsSGAg)|1fBGHB&M}yOj{U~;aUbx$!kBVW_VcQYf3HyY0ehs#For1F zNfHy$12E=9W_oPuD%xH<&adDc3l#?3;d;}!***SmW{b`5bYrhZ;Fn&;A}Ez<0e0r6 zR9JWvanXuM_Sc;b$_zvbf_=&{_{xQOJsQdl3r;92kn-zl2hF`aG(2PrN3Vu6=Ke^E zI2h|?-vB!TCZOa<7TJT42hi-!U$$$VE7#)_SW^@Jo?H99EOfy!SalkWY#B1#Si}eQ zP?!4|;{_ZeF7M?%(;7YwMKu-@2tbFOJ6RI@4E7i zMC4!jYHYGj!{dQhDmUs9SDOoC8UH(00Fo5z>OGL8cytlyBADaZ&nAldkou+7M-d}% z;ZfU!&B>vddg{47Oq6FWkwbr>%qw{>jqs~MT zsI2?P&EwHOiS!p2!6Tn^>tvdpNb`rQ2e*1Mm}iny=F`wJqHm^Tk-UuanmVgaup(E& zg*_T=WrmVk8fm`BT<2KMV$un!QW~ML|ivkqj;u4BT z@c!Z)pqer0Q?zQRuy#+i5zj44R+=l^e>KrF%`{dr-{GYD&I9|^phg(gUUvwyvhpjA zi9aEQ6M-#DMdd;SLhF{wF38hkM~1}wAJb~>4QmNv-hQE)grPp#;NDZJ zALMxnq{QgikmUNuNXVU;KcR~F``RS1b9w)Q@skW!?}3enmzPb=?7=(#<6u=|5q`^$ z4=vfzSfJ>Bgw)HNW{suh$*i__v&R1i&wi;Bd_uhGdQ1ZWX(>5Ug=FD){72DQutn9b zVRYyafuTD_8l*!IkQh?BTe`bLkY?!a?(U8ONhv95q`N<)OV2*n#Sb7hd*1hX?t86G zKf4t{w*C?r6YTkCl%&(!;F1irPu_hifCWbrQ^mXa?Hy^+J`IgUh4Ohin|^0icv9g>9-S&E^8Cl(P3VOWDR z9_#yZ3mP7OXyX>X(I8^>sf(&8VKm!CccuZx=W@t?IRfeaYkczERq=o5#b0Ha>5mj$ zfm(G=^?>Zj_w^aziR!)unx|Zk$EfC(V*Ess=vox~DF&?8uy$_c<`I*MkeVwb5+LpViRs7TGRfVC~b3C&iR;e&+%G{UnSg! zBtrF7%$hT~&}$1x97O(X;HnZ>eo8|4d7=xhF%ZTGElGLdTJo z0cH-K-7-*Ig-lc;aAb|M(jC03}>ZW3$w+ zsq|AtMr$?mU6w=>mWAUC@=PBrshA{6>={au#&{@xno!o3#gDg%wm1{Y)ysnC7I4B`$#II!<~XJUeoGRaf*%_*zQ zeF8w1j-91*uyB;QnpBJOmW-knlaS*_5X8RKb4(Pt%&4ktumo`5_B5C(rsoj4(4hg) z5CuIk7MF_F%WmDoMVKQhD_%}*afu@dD7KlwH+%?j)lEK$8#6>@Cxs+Y-CPMGt;;^z zxmrMC3=9RBqX^AuoNKMXWM+2J!#nOiYHeJ5-=9Qu>F|y7{;u-IUa%7Gq$F~Z@$zKi zw$wcLWLj~By-S8CY{`n$rqbu)WZ}G_O(^tJwOD^&y&&)Eu_o>iO=30@gAGxL!AS*G zTLHXL-Iy!DhjelO?Y}`F`d=mVFD)KncK-Y~Nr<(~j2?~UhCM&BS~pAB=qwK-XIf3O zrUTcouIA4vCFOL+Lt-m{nh{TJ#GZaV{MCr`&39&L30_o$3^P-q41PhcuS2Qo=3b34 zW!FX<{HZ9=5{5jk5y6k}$lyBy93?19dx=jD5&*w+DK@b*7(_>2^h`U9?>S%n7T3=>APiqdE4zEjCwa$;WWOv$b-0zjh8d znj{xVkSf&t>@5XyCz~e9KPgam>J(?GkZjt-tj>3#0P)Hi8|IMf#naZdaY`4D=XHDj z?5=+iD?|)MTQXYp6kEZw_}mc>8JwN2#Cd?gj(C( z@U6|B-p4T(Y#U?QCMSWric2}w%M9$vdu;k4S3i$#4471owymWf zQks3BBeH>SXcN9aeDfEWTZ$iVJMiiI%qeWRS#20Wal@j@yoW^$X;;AJc|+v~q}$KD zmOcOVI;T$#e;||=z9YmErkzZ&AC^qJ-7h)*p~(K1UBZ*4Cfc0CjZ$K3Qj&M!Jh>>6 z+2{jnxtSKl7@Lq8C`h-N{^=XLJ$XcVfBkQ7n$Maf)Pmz3C+66NU*Wa)0YGe#xxNd7 zR^{UOZQyWTZq-;NN!R`0%?^cBUH}LTU_Yd6*F2xl7gujR*WVV}6pf$(?DiM{-CF+o zevIrxhvGuwJxh6@kp7&j&Y{5xNpO9Tj+6$xMN}bC*s)8*tM0l z6Qmjp2y{I?!b^iE-5a1qO}_l;+E&JEDg6cDAy8SWtF%+9tYVYnfA=68{Dyjt^H;RB zJ$5lEf9!F0SNMKq#(KgBS1|-cnkK>^>E)lY5F98pE$HXn9X2?9icfK&#VudW6y1dL zXn%!}*rJ_`zxtE49BFJHzP#1$Tb!4!YBIDBptNFF7$7Ej}3-FLe4lUG9nr%(LC z4bIcOG~0ZxI;m-0tt>IhKbtxq zSAOQKJWc>B>e$~^Hf84Nvm~&Es;XYo!pl~NtuEx?sUD8o6co+G#^IW@rZ81IA2oTo zlxDeLHNq!ajQmN<#ntSEb51M(J|Eh$a>kz>doe{KLlTY(oEC>c^1f^-rYxH2Nk~~w z2KPfp8o!CNO<8~=TDu8E&tsbZzD)HH%!$7ZRt$FHLLY##u3;LqXXmCb+Wn_MF`OQ*m)0Pw|a zJ@@;WVa3vGnw4NPjNx;K2YI*1kb-bY#4&q!95+S0AEp34@!x}A(;Fh)3V78iEYv!J zv8kr$3J!*N$In!=qeD4>ipB3T!g1LPwDpKiE6<%}Lh0D(#}` zZs}Y1X?7yk4v;w87>d^?T%_Rdhy>qg!~=Qul77=$g^R4c^X;nct~QD+zwo<^>Zy#w z+P<)S*#ZZ=42?Zk2|2qYl9CP#uD@8V#-vWUnq6EFuXrmUA1yOc#!M%Vne&o?|MP8q z`R^e^|Ju>-%ao-@x0KZ4YI3ovp7!E@<6Y zb8q`sr&MvNH~zojSAOFy*a&IAO>OJ20riccNs2{&SL zWM&ywme_=AA>-sdt>VaZMP*6n0Ep?W7~@v2|FdUn<-zsi&-Fo|s6KrN1Wlv$8H(6A zkSaO7;&koKJJ_hHs4SB!?*l}0dmryCWP3(iZL0OY-E2n-Pfs*WHW^>wUM)6VD;SCi zeBSJSL*E$q+4n{&Q2hb+%t8nosGq&cxG2RI!51Nylr+Y=ZU1Ny?TH4Fus`W|rUrEh zRlp64sN?B2+(0l#qGS3Qre?oONlF2Srkjn2kkf9o5(QS_B78+VM;k?+&Q50 zB=`izG|_bK`P^V&SDXI{o}<9<>v-PBagqEI&Sfc_>Bz6xQ1U2hh7Z>LZXIi=Qc%BE z2a?rFDJ4bo#LoRQV--dM2rv}Q21Swk0bzt*?cb;ULjWYQw6vu8i@N~dvHd4eNHdxw zDKj`yHU49g0?hXNe`=ip@lp879cScNUg~FpUy0=lco9GZB!ppJN=TL!lO0UgJ0~w= zkH;%tMW>r4^_Lc7Jz`;ds|i|X{!5CAcm|iXC$H5=BJU6_b&0CT38h@z+<^O+r@Onm zmzVZb`MA%59;OQ}R^fUW3^#B_=`XGX0+!|PDn|gLSx3&Av~MV~v5d}H@`PdP*rFN- ztGKS$aRP>B9{S2wG7UD(Y<4G>%|?Bl0qKW6Vf`~5PA>FxX;qa-KRLzL+j^Z+(mq%6 z;zb806kN{<)#W|$)JvdAm3d&R=Ee<|0hRcMuwNBbAXuGC9TfVc%j7WR;dl>EwVhCpY z1!1WupGof*haM>LqtKZd$ZDZG23A$+SW^B+gY2dDBPgd`5x!AVH#K!~-c`v`+X|U6 zN-=}XSy?g8t{(Y+$t0pz?;gj&<(0vJb(UA$bsyoHd!}H@^*HC~idy?F)gqk{5dGG5 z@-k8D;3=dFeB<|#=$X6Hq>EX#@t348JK+RY$+w-q2|wZrh-JuCdr$&*Rlv;GuRaCTW`6)yQ2xlzgB{rZnyv|qnSVcMPw1YXyq*oOv8 z>?EUV|ECfg5{HNA&RCCM_p}jdlmGX3{pb4Q%Gkofg1b;Dt|Ns8mkMHng|4&zk*Kx6 z-T^YX1odr6%VMpQ!4kfFm=PN7dq2^9NxQWwPo<7h;PL$_u=y1*@O?4p!`KcfkUdu5 z(RUtzlfNSpJOo`p{F!@s`BCw_&>47%bq&q=w|z@fPI*i@Ba0-V3rmUv88k?;AY#Vy zwOdr3n7P9rjMMp|&wihPuh(Pci5~Rk?Hsf1g0uVjT~QSRkk~nLqD+E6VBiST)Xbb! z{bm29$NLH>`AqjcVaJR`aeHkDdK!cYxzMJg$|Z}9Mqba@c@&|*R$B8UXDJSIJQUYj zOsZpA|T_aSXV)Na*2#wPLG{q_Mle@SQMv`y4QoMz#?ROYqCv0H8 z*w`6j1>3?B`gb-6r^9=X-on^~NvBj7pHk>V`Y;5BwZw~=jf?9Wn{a@%0Q{-G-kzk! zFdC_-s4{7Yk0&4&mD%`}t$l_eCd^33#A4jcN?}FVNfTNjv~ygRn1Aq2R#+Y0LRMwpJkCp@A39M~>+hDG{$TF#Eq)CXup*4}2`Fg(j z_k7SaH5PL;h1i)5VFov`bHdn|CR9ZbVrE&YcUfv{e}T-4YeUpnyUu^xRwdw}eln@o zUD;;UjNc$ksm#l4Dc(95TS&KbX!RMwJ+e0rquu^j;U=ST%)5l^7l$4^w{U~KlX$-& z07QJU6VeG{yxR_7>oe7(AtBE&(c{lu(n^6aal$uTvR%h7btx9fW5^(63}mP_*E`_mqT8vuikr-EB16)8xSKWe4X7kxlm?unV^>(c7zr|LBnH!-JtN%t zu_!P=xX%#e1$K>gfI!~|Q1Lu_%~MAS75EzJ`sKP?2!dF|J%|3{fk zx-I#I=@Mbo&k24VSbZyRcaCyF98FXENti9DC0%7so-42;K@UqWJJcJGR#M3I&ju@9 zln|ydr=sWR#xOpjYDEcaQ@vNnJKJMUz(i^t^4Cup^teFJMtA@^miG4=rN}z2wqFOxtb$IyiiR0gS z`)0se+oVTz>!Pj^2qyb}W>Jx8Us}ZNda`f<7Xo2cBLn5gE-ZIBv@8no)YUqe1rtbo z4-yfOU-fJ;d>>t9s4xAWfr_Eulq@q}X53mCi}e0Pec$_uraOBD=6;+A+zEUh_hSOX z0b;*mcl7^o3Ij43N>#fg2*0D3%-gD4WPV{!U#JltZ=#Vyw_3ckCQqhD?WS4zkKnII z5PRTRe*|}H;S@iUl#uaTny#{_6yxW;wVBGsx^~J}CYlwD?hNnU(s&GP-1;Yj30aA5 z7kcI#lV__gyMelttdxCkVFn0+Ifz4!I{_zl;l&ru76qxG8Nk+Q^qz?tj6){q$DSo% zDNrGN0}C8HEv&|P7YZq`wt_@ZtyG?V=q0yC7C+Oq-plzoGD97)OtAm-VFGTURdP@9 z)cY3;T-vGDO@422_n*Q6uc^qQ)nyG);e3v}7FoVg4BDzRRPy9G1rJ;#vFyQT+ON_J zMh{7FTtN%K#Y;~K9pRnfJ@0}sz;4D`==)7=i$^NAi&N(1&UlRPX`Bau#w%XwFMzZs#CawzU5V3 z>tbV;ln^k2}smB$K;a@b82>*5qh$durkc7eLTd#5+{$_pJXCX=TDR5*k)>chpobY!ln=dq5Gx<92TCiI!DS_)|vK8dk^Dvj9FmIN-FAbo3&&8k#S1rcPU$ndZ4 zA&^=(mOU%Wt&VHiC6D4(21Fve+S>v7Q^Zys_K74I5MPcsOb4krEzQlR5HhRC79Cx3 z_ZZ8ZlX+!io67n!To8k1;A@k5Yd}`L=H3W5i{ar@Bim&-e6-f zx-c?AybntVh&&-TRL^m{UJ@p|UzbI-LS+vvNL@Eo!g_Kt@ON^X&-Coh2L}hOHGXey zN}nuDl9Z7yyUvGZP_9Y_u$xDGqt&rpi2?FA220w6J~t~uqA80WCY4f~*-Rt#z_Y;ijLJ5ZriNJm8C6W>DR zpmkgiB_ODz`XfO!fW7kZA3x3A23<0t$xC?49M%qLfx{Z0P6xy%yTyXXlZ)rKzzuA0 zP@AJq@MA8a6gbe2JClN%I$ru+sYc!*;IzNH^K8`NbW(fsQXSY3B_`*nP2 zbNEOWJ2jnuR6~>C<<>(y}jmjEBP`x&9~){R9ErtD%^hx!uKH)rPZ%UiPtlsfEhA!m;l`R{A{9fe|qD0}uHDIv4 z_Tf%}jF7ot*4!Xwyv!y~*rzD+q)TFrEcL!P%t(hA@hl`J*o7pskmI$1k?!qYU_*5A zFe*qP925$Nov(0xhVjKbuTs0eO{U8~m|ZI6czZ&?>OO^Pg$(xjzPoFdoSxch#7P}5 zNuUl>4ecEdYh7GTGhE~SxxQ|gcgXRjmkkT7s@Yl$X-5tYM$SqabtwhIf*UaV-(w*f z7UOHk7eeflCX1<7kjfZ^-6GCrAHtqpeaGLc`w7h%B}o3(w&6AR{cN2edyD{sEv~ZH zL6M(%a|j3qK0ghA2em4$L6Ho!vuwY;?&KQY*b%#8yM*Snq>4j>;t2k@w&>Mnm0?g5 z4BgAul^NAf|&}y`)=i` z#sT2tH0wD^7_9rk0ny6s#<$noiOrV-0UR70=|fr%=jN3o(>=*`yiL^JHOirpvhUmBe!Sq%oIrmrnF##53-?x_@iPGx9F2DGk5pv@4xPj{sqWZ=W9!Q4|{ui0{;g4%nw%19@@VI z=qp(w9zhlIHMal!*0(Zjeg>c&As!0j3T1*~KlY?|At@iA@+_b{3A?ez^+oIACmxa4BYQ$qrc|+V^7y= z;?|Ov3}p!DR#C_Q$duV{b~sJGjpB`;q##y!n!AIc`%mZr9lT8Tz0jsKsLhAp|gIyqn1V10%k7o*!x{0rY#0p4mu}A7PVN?YFScTT-?(F zWM?T)i_k}@sL5F{x_jf2TRq_hl+%n$^_#-*E*i+R(y2?B5iU9$^(snCs>h60toTm= zj&-1D=QK)&@)=c%u^vA%7p%V+5cA&5e@wm4PXgZxf5415cwoiBtsC~s#Zk~aM2)C-Z;2^`iZ zJ_I44kgau0A&7;;y_Yb(0q~8(aO3PM68Dn5Dzix5KB8lHZI$o&SELd-Qiqnn4@)|B zy2uT7Q4w^@&pXBV40Mm3D{@$IoQ~twOkr%PoUb_Ec z{n`X{-K(mjgM&x{ILD?LSO};LNV*`)q8w_D%QoLP-Y&M>60$YVkj6N zuTU#0G{>#ZhVZ;)utFyM5$TA6Q(j&6MI5@}DE!g#6REP&$nNgS*QE3?e%*zmg#&Nb zc|Flj-CsN4*@5G;!e8%mMMV}y2z*enYTeMqqH3;VjQx&Z9~6J=M+ zVeLRDU#Xn9_|e>xeG2Jt_fHH+jguZcr{vXkvZ9Grdvzwu$eMkz9duyBiWQRwg(11iRwOxXQl6HsVD=rqC~LoNV{EC# zsrFk?CO1d)JFsJo)WxrSW5&77MUo(N6dJJ~xDmNB zxhUR-4&5xZ1-Uo$-(?FKIZ@fV!~0F7eb(EeZ!tk>(YL<3k!VO|j;AASkl>PBH;@rC zI-WkK_j23*GOcpYs?>qXY01@DzpQ0o>GtBnEnWH3Uk5V>L%)2*8}lyru!-_w3aoA4 z&Q=GDKfteMY3+9Gc(!}9moK?y+c$vUU&oN#JXx^cLDkT)Lsv84C=%3+io{{07ZS%~E6F&2XEdEc%O~l~!vygm5 zlFQ5SiJ6lHRn5|tAV%*Su_`QCpv$do5UJHsp~_$P3oa|vHqRslby#~^L`7@j?$`S) z&;0pgu|Wq{2P^6w{KIO+#ZZ5FHRlKCrEAaj9TzH8R;oBx;W<64RrKwuRc<$R{@ogH z->W?ncvcOO6=QkV_moNdCi&WQ<6zleJZLrs-n(akRRb|7$aL+ENlSq2512;?{FJUrHk7=YQX zU<4D@vfvNn$UrRSR9aOMDpRX)HB32Hh{Gu=df)5aT2+-WERfS(&K9E|%l{Lzg>)4v z@}YsIX0K$NDcp#>O>_|Vw?hmq$9EQK=aO!x24(dwMT)n6-=d^YOt=1AG{TLd<&sX$ z&mEX9j&jV>wmySEhvcOMkT7@QV+oj7Ty@F=lA$h6t=04p&3DLemd%0$>6R(rJ2Lew z#~`)s&(_l?-muGVbI^A(OiNV5Oj+F0KPa=IX@(dL-M8xCczU0R%i{!5{<$b?y|)Pa z=ViE)sYuN`rz`(7;lBBWw^gUqnjds;eIu*w&Uw)PedqJ;s^4WQ?3C5vsrMM|&Jfe9 zOL;nTu6<~~T$J(dKlDWsNGvSwu@RC>g zuUu1FNZR}R1-5}$u5{P$+tvaee(bnPQBprhqy0FNbZw1*3)L>&4=*2Nh&#;`|LM79 zqmkmoYzv9t9j48?24l3Jwm^Qs%h=U|lp}ZTi#2zgiR$`kh9Q%rxU6CA21$(^E?EW- zQUrP&T=W0c)bsQ6vJumqc}|l2{5U5yuf+J1-Z>a#2<`Dzn%l z8lW$@4cO}+))1HT5t_TFDJsWD+eAvgDfGUoZQTjH@#bD*7f9K53b=msZ8qUm`f8!i z$PQTzWHVFxV(to^=Z)poQ{huoUuq@hA2H8#ncB|+0mr>CGLph_#ovV)-YUPOlm$>g zC8K%VHYeGHGG2x<@WLPkWfpm6Gn$E}EAL4viszZQ4;n0?Mokz_kpn-C(Z&M0I&NpY z?+Q(Y-aJb`N~?#%O@zOzsuC2)cdtgt>Jn@`FW8#3Ofl}3)dr$u_@DMTt>~#0&^RY% zq3Fuu;BNba5-A45<;9P-W}F-Qi2cea)oj%L+zYl~1WBQ9QCwUcBJpJ? zh>t)X6b{S3h#nUZw}@PdRMOly^Pc#&Etjsi zg?48!m8bkM>am)}OEAnxT+#ci=Vy6|;>RxrE1tD(t_)oHxWuoiN+8A6>#t1;C2-6x zTW9a6wNaxfQtb-YLhiwTyR7}+v1L^@H&u(pTJ2)q{BH*gii-$XYNL|C`I06H&F4_F zMn4LU)QLF}!32ETpg;fBRrI-iSret<^Ce$mgNqvA>ax;lu51|}-e-Ia6$H%)y`h^h z%2-*Q*(L^N5Wa^w=t669RwEfBow!7qO+pv3BMtElmy*%0aTGzR}r?%=egp|I) z?y-Bjs_pvzu^&Cdq|xJ)qnTE~@bfabIxuIA^qJQd#CElq0Tt~C7sTGckvg$_Q%CRf z!Ur$q8ENNi$ImR5HmKfzp{@Fm#&fV*h1v>uTB-3(tc%2!!G9VN^v!kHMvBMD zN(d`A2p*~#u=$D2|h+G$qL_@qvb*COI)$EM%rrfWZK zXYCh%^UDi=n4XHfh8W@MY6m5_r^tD6`U5t)Q{?B2Hzb%PH=E9!USFNDHoEm|_qLB0 z<(E)>l@dnG=|JiPoi_h9g%bGE=u^^Yzwowh zN}4sqbzB+)kAxPQp9r&peHE*Ubt;D8d?@jO3N%>|X|$YIC=Vhe88XL_)jkY*X5IqEc!Cd zhr7Zw5nS6^5!fHxfH1nR8(u_KZ(<;!5>=-dpo-BVP5FI62Y&KPf5YeVB6j}TTO5Ry zwYJ;3_u~#mao57y3~vm72Ju43N~09;XKiQo^kstHmDStS#?H#y(3sfdUZ2$E`IjW*O5IT~FyveAlfD zN#rLvjpa&pqlFQHWtodz?^4(e4BiE>9XX2?xk$eK+k)n6GAs!e*_Qpk`~sAS2vU=W zY8)cpH&UQfT7Qn=DcFI-&5AWNKl&C)ImhcBY1w33!bMnw^RG_)QTgTnsEQ; zCt%{SvAOQN7RUJR{jPKd$ia^F)F3kgXSTmMBlz;(SO`l<*tqXsH)74YXH@AJf*9VV zxIy?l9a%YH@iU}T z8`e~3LgZGCT`rDQO)HcPl)~NUY&~RAnnJgKfHqCt^!>Gd%fHX%Me|h0Y8PDjEzlr= z<~a~+2NU}O(u#I;^}Z2pQHdP?Y6DFM|GJJG{hr>_;}fmYayviWh3l7-%qQQYgcjw< z-AG9_xnA#_{EDbjVFvm<_o~Y2XIVy;qzEDjC<=O>ZjH8m%vcLXtKhf)uKsT=|MsH% zckENyFg*Ut^8D?flZp<*AOJJZ%cnpj2nu?+kxdPkprl3f9{ak2 z8zY56hx4vBO^|7~AI89faE_!{G40Stl|VO5QN|4FKHzXFVNPIEoTBHXg(-e-(mP2} z(UTFUnTOF#!->li?GYnS{#NYUwLo7*6($4zq|%k#BOD2HwL+46~3ZhUf}75f?!+ z_I7@A1R(UQV<>{U(2T}7t{l20OrfqBv}GjTvb`6*CHfp$;3<3b%6VrhwKIywANxRI zcfLjIB1$bn-Ie&(D0inujfE&=j}E=++V)8)BSV2i_tPM4R(BrWXTZcR*lR0k1B;+0qN{9Z*Ho1$js8KuGU!;_RZtOyjNwM8t2(u!%b&?#tFtk=IWgH_1=IEJ@pEACQ2 zY?HmD->+x3DHtJ}=f@oG9-C_@#f(*VwXw<4(BaLkr-r(|8HLa>_1K*B7+FzO+)5Q9 z@Uf^?NIpE)f4f~9wl&-fQ1_(5CYtp@f zpwB>YbP=!NzuiWz?Y+XS;0_s*t8*kon4~LwQVCj^AQzfgRTbhpb(@?HDG0nkW`)YZFg2hGGQ$pUi^L$JiDz>F8-7a;6o;dC80+$=X z&7@$KxzP8~Uhk7#3SI#Ton4&;8Q6i_bq!%(zi~nM z9ilM`Wb{2*JmoH_5J0~MH;p&UNaV2e3j8+>yDXP~H1oJI)#8?fbk)sbQ8I_Ta=`ge zff*oXYY6!+sHiIhGevFjscKcrVK$wNw@fkoT<576}+O`uLafXh=Bk9xGQ{545p~kJp-ml8*4L4XTt|dnQ z8f0|$s-vk@^6^;t+tX6tOO?a=}gWGiAY%l2Hd%PW*Fr_$1la)b1l>EyBW_h30P~Vz#xMI(b zpBOLt#Pr(&=APDQCr8h|8oIhhWfeY&^} zN^d+1Pm>PmMKevtOzGO`>zgt8n>bVm;708M;Vd&AD{3iHDDu2KGs4Y=&mpePyKoFH zsqGnuwka5i8QG74DLr7uzX8{GSe*Fs?`dgmA0nXPBoLY>#<1&p73i==MNO@5C$lg7 zwcFD_R}PF($iIC!&yxnBc`&3_ihNbXi;1Io^Gt;GyIm2dgY{({=eYTW)8;K0)oIzqfKS(jsy0%gxB7VrfvzW>~y@xeF_hAr`r4bQj(2lEFQ( z)MN!Y58&-RnNR&{CHcB_|205(2TMlI^!tSR>L8s#{F6M*Jhg6GtaNeX*r$NIori}H zV#>dO-tP1B^KLSrrYYlzW#JhQb0zd(?L-YMK<>NL=Tt2Tj-sDivDHBi1X4)8cPW`; z=G^(yisC}~%!a+)hG94@lQR=&2-Z25fA~}Gzhe&e)GDq;#FyjGXOV#}~?Mc({QvY<-{K>%qYyCerb_=NikTVwzP} zRh{ymDWTcZqoAtT6+RhtdD2ppel8t5l$_hR%-C4~N?6lKvw3B>*`IRxlY8?UU1)oG z!4FFKy5lZMkwjUmr=eh!@z5Jt>*_A~V$Om?{mC;y*&+hlLz~)E=~4}qRe;wO5frS4 zV9VgD)q>DS+97REH_C(;iB7ZNpf1o`X9pT{ub4TLw#c)gi2sb7p;W%H2JB7L%5uVq zZHR6h)1zg_LEJoc$x})*r0C$%e{JqoV@86Skrl*@y!1N=44rf6ZCjrJk=X#XLA zxXK`axm62GnjUWN2crzxHc8_3BWYdz=F~2g$_*`?^hLw>=)O+z9LT~sc*q@PlRL7b z>wbV(jzC%De!PeR(8B5x4#NA3A(hoHc!$XJgugiV_n+%jk2Ev?D-ZfkRbME|PH#vC z`O}Y+Sm9X7>yR5;_$sP{zS+DJ(T9!BF|{m~3BzK8saF2@0QpE~WA|!X)x=i*%LcyE zRmNAsu#XOs$SC`epZKS?R-YVMtyXbjz`nYTl(SDo_vXdQ3{fF3KZcWQ_>{#G+GO9=17YK6r#^>J#XL>T;Y?Cx z2C`AcXFaGX6@!G#oZnE)(GVAmm;jLuGs{ACuWtPSv!w+bCn*7~D8FGM9U@h-p>brF zZ$j*xj`+}m;BMNA+>Pl*ts60xsP^VAl8mHl1Sug(LXEh0+kjjEfHnooVD>vrQlzMr zYBY4xvY5F$eRp3~In8w(*2e>_~?qf@RM7 zy+lolu$i|4ZH5tqyC5cZw;EmQfko~_X!*Vw>JDs<4-fy<)YC`9Xip+D*V^7knHt39 zxb<5ZBM>IhhQ$$vgtf4Q6a*sCtVFZFJ?w@#Nx|qOVDq=_IAH^+OtC(BfCr7*u)Zn0 z?JdZ17=Q)W0o5$iMxFX4S}uq%e?!r%H9^l;@kQq7M%w)xzvnv-v-Ou7g?(LX`VXJ7 zVz19myFXq!sK&U}jyKtB0lQj!a|2N}Chgtb$Mc`-R{5__#9sog7P6PL#P+3r@uOC# zjPUcXThC>eNFgU34RU5EWfOoTG*Dya{6DLiDd;z&iC+YpM(VPRv^q+bXVlp%m_TOq2U*h8|8z3oerL00xx7mFniylY3kGbAsEw7xH4& z;+JO7$5+j^e_`GGfrlbf^s0X*-+jg4MqOD zA*C@Wa0Z!F-TYF1^id_@odz*_3aczt*DFa>G=}Q#s<3wKfW#pKhm{N?R|ikbCu*GgYnL-2sQqhK*JRSRldfojQ8|TpUy7B7Oi?y- z-}3qts8_Xr`uBJX>_J-{hpbYVEn*gN_O+Q+YeXJin8jm>&Eo*GFH6@sV8J6tiQx7`#;b}DiJj;t_SrWett%T_dd&+ZB@SzO04t# zvS8$Y{=yQOBJ@y!O)QM)eS~kGRR)k90#mqY`pf4%zG7!{3f*WV{h);tr^AFV6DX7M z{)XWE_%TF!yt`Oj%=bXvvdE>u9$-vxxQp-oDan=dA22q(7He69Z`ZrDMW?(2ABn_O*n_+^bBjVL~a`dz5kr7d+O+iGNRp^$8>h2(*fPajM$w#c{Lh zPHVVOXNja5>sTP;6SLM$r>E~BQ=Pb0L<~cF`x=>7DQ|M$RX&b2T9_1%H{IB$4gdx& ztL$M@7oh#RbP8l|UIq`R+1aP!CnEK=RR#d-kYjb<>q7nC*Ugt(;Lef} zDAcy5M`p(p-7W9DxQd;ATJ<)xy!B0KN3c(StCpV`a_fM~bpd`5{2EikH0K<{RVKfq zehS-j|5=uIX2hBo)1&;yu1iC_*K-|PELRj3=td+}NaUTS7y=C^(&-8AXUczM=*$yM z#*q?3#-;}uGQpS$$g)AESLm{`g?-F4ETzI8xvjZw@AZDfn2%a})?O2V{xmuBFNNoD z(#9|*cQt00Nk|3e?p2YW!elG!qN1&)1B1=~KyoR;CSY6@xfX+9*!nl{&)slUj8=af z-ZL7!fP31@duVC!GPZ*t-MlV%+h{_Gi}F6+-P@&=YFJU$hXi7(os0(tCmB(?)vi|%9JZzus-PiP&ijTyUax_qc9zs}H? zVG48`@wuu~8nmDAB{M@b%$F3BBzlfA|LN%t;g#YqALW>s8O+UpMSi3eM(d# z1=Rg%PdGmoKBAW+9VUJ1V$;zP@VuBXXYyaDY!pb2xe-s^LXM+T5_e6r>34zx!`&Bf znlx~Dgfp2;|K<(4l1&(ot{V`=Lr|+&n3^2OgXtt>4*P-76D6P}6ueVP4EXa5`HJIX z?v1)U!Q(Mw^XF>d?i2+ zw%BTXee{z*m&9NX{`|nVs!$v@;aA_v-@zEM(~FhiigIDsOz9duA44u)I4Hk4sO2>L zZhoHpG<9x-WI4?8GWZu-h#YO|z2b&*VD0f^Ki^dl_+UT+%_$z%(TixoGB_^k2U8aP zE!9BXj~D-N*Pki&{@C7&@wp+V{{@Fg_KtL!R&F!^7#t>b+|&sD_4qHx{l#PuNo@IF z<3W(jf9OTBR+*|*wkHmTD`oFx-XVXV`M1u))4tK@;IUz0iQyV`wz#T8O`R0EpjWIi zDlXOcc}z6lF-1BLnOjDAlej~X*a~1w@bCx`XD+BM2JmssM~u%kJM+Tj_A=9 zxck*ZT@c@azszr_O^;buW%gd?F<6ojDgh#Hj|6|rxg0mtS+NL^9Ak#W4P~nN8ri&+veEB2KOZwv8 z0CX8A(PTR_G_)YJ-^UVX09|66+7I3EB1Hy;EJYfMCxJ$5yWB=4PV0^=UBZ+5cE|yD z+wf;kgZc6*r=>)RN(%djqu~Dte;hpDEO8>4_?~uQw<3HXxYjD+Mv_miV4HAQ%bSds z*1b5nJ6@T^`SKzL?p&i+jGN{d2}|{D4X?Udji2C#b`CEfNPx|Qf+bm*ma84bM&W5W zQ$4Vl4h=p0*{%&eyLe?K4I_ii^+!y7-XFy2n_sW9{mRKUTS!|`SVd1tkr^bBzfVQp z=a3)Y!>mr9&CiN4m~Z-g9#gq2xUwqcF)94ghIh8Fy}mRZP8|No|8`rZtMza5-*n$g z`M;Zd?gy=&b*-gKs@|9L>cD^B%NhcRPEcvhel8d z>1IG_36W;#R8r|~>5y&|rMtU9>F!24MmmO0hoNEUc+c~HS$>2Mu6542uX|s6|29kX zfJKA$jyNKuZA-x#M9H*5wN=R^077Sh0qD1~M4UsFBFEo7p<_h9ln81`%_ZMJL?VDB=>4igDIlenr|WqT)64#E+{J4colI63$fR@nIqR4t8dV#TPa^{Z<~;?@LslC zS~R~$dgSC}W#yh6A~O=E3mVf_R@P?RR%N7S-6|wY+qk*e80oAN-9N#@;%PprwF~|E zXD;>faFS&&(eEpYc;r_~@NcSZoRIV1C2Y0m3X$z4CGjDQckDkMh9nv>kiRnvX$Nt|yWb(PHu9-}QE59&E>hQ*R3Lr&@gB^Xh*f)w@*VHx+8S&kJ^qM zj#nQ}?lV6F!^Vf<=8_3zWlU*3TQ~O`N8UBs?8nE^5SMBA1<(m0 zl;sWN6p$`0<%rvV|CsH9PCZ4kl#8m6yB$}(6hcs&P+dSlOUEW6H!Gfodx~J_y)v=Y zixX|&dBp!g_)qG(Iz}KAksTkT9&Wq`iIm2l$0g`Ez)^XfA9>ZQHoZeb|5^|P7-vt6 zmGcSPujqslg33JE!?N3byeztj|?)MvLN{*F-V@rn0nyUB#tU-;!bv{ zig)QHejEH4Am%CG`eC|>%2eOmOj>BU8~K^Q+$LI3lTUTf7oh&P&TLH!d$uv?KKP=j+4n z;mRi95k?-Z*z=`_<|3m)d=Ab_L$MdE-RUMmEujDcHjJ84?`7tumxUpl} zu&8_c+TQvFMv0JCAs;ceOjNB$$5RZ*<>(=+b7tSk1y@mXy+9i;_ncQ(0Sb zesgf-?tgc5c((Dl!*7&g;q~p8_W`-s26pP`hv*`PK-UO_RQdk!A` zz-g>!v}>V8^hZ;=94N*_P=mdH&fUzkVLcO-YgO_H4lSIdI%deu$|+bQWjP%RDg^)j z2K#RA>mxo?Yz@jcsjdTj7z9)}U>ws0xIur#T+7^?14E!qolb+&>jUY+mGaWkTDL7b zv}GG}{L4AOC`jvjA~IcUO<_QJ%&1gjX-QqWbhXelpLXEWI6^;GbocJTB!K=WjI*QJ zhvAqf5Hl3tC@#D3!IbCp#81K;-Vw*mu&c)9w?_AAAo=ArKDn^+Umsx;v2@{1W}XEj8iO(@z zkq@+~FL#}!UeZvaxSkji;zP@|Q@{H#PGi2`^YV9bIorM{AB#xX+8P}lZLE-txf$|f zaF*m$|MQxJpgU_NCt+j?Wqq?lZ5>;zepSH$3zv|NPvc=n%joIsoU`k87s>BfMGCXz z&9K-R5}F&E;g9=Qnu7l=w5S}AYLn4PP1q#3CD^eK}3uaJ8uhGo`oKzB#qbaZI@~Myy}zRW*jo zB;cq$L)#?(3+?_@9YTf@3sxGi@m8=;i|w0T!EGaS;l6fh{&WeC4JC4=dzG6R+Pj3Q z)LVmwg7IyJm^$|Fs-#FA!(8#0J|*ImP^S@VFyDbKHxFGESso|?6>A}vWy zP7Y=Rd}U<|hF4ayUe?%yvwWD;4LOUxKK;nD22Uun1{Re^#*?L@R1CHRqEo9sSa9jY z(6Eq{U?xZ{Kg-#=9Z*nvkE>RM#v%|+A9eX**g`Ac?nwasJ-kBbRHgmH>)Gm+}Njy`UvSq-|aW!^_<0_^Rzc{p6sGgRAY>o1`ckT7}64|h!4I1iIW0irBlEic2 zI8o~7?_RyMK!sy)AN2_pAlRwDV^^O`hCgR5>@}yT=32Dl0r+Xw%HBL_A4@*jD#>H3 z`n7xm{a%*qDcQ_moPE6|62hQq_}vd7!8SqUwGgR`$Ylu_vOVt@Miof z-G`rB0f}yAzDPy66pF&IZ?*Fr>?>lw(cWcEOBG@U_IZ71It5{1;wFC^U4N?($x(~v zdnzd55?N|=-TfKE-<%%rqf4l~&98{(5Z@E)T1TmrP5{CHt(LdA#LoI1DF&MULw9|B z|D=o_EK20_C)HhG172QyW=UnE-}(7{m6N^lOE7~!+FH{pr?V-uguP9_f_ za!W^PC{Ly|`)GJ@TgRC)I=|L&8xe4IOsk22_3OqAxY^EYl+`WU&t0`hBz?}KQ<5&c zn^BYjt;-~;OV}7jOzM(){XSR2{wMu86nMh0Ej}lLYgfv7Pc1uYm9mI;5KYI)h$K}= zmWn0mE2b|Pjc&F$pAuWXTC}Y!6rVLBHTAd7L~?RIN;v^JfzGZjfcy+FZgrMds%S|5 zznS{rpqv>8bRwh*5f}wS5_6my`x}Y(>J*&4y-OR^Hp>B_B-e zs#mrlDmSJ}wNG~%;>WD;I@d4Wj|!bZNP7Elold6&#Wv_c(_IbH`@o=2tA3o1PHl!Z zl}aMt{W@vq3Uh1?OX@p9CiqsAFVM0Jcb^RW@~BS5v)o@4iFGuu5K|yj|58+Fh<=4h z^Q;*?BqyztdG+Y7mlG|H>S{V2Jnc<;EGT{1^(wcdPuP4RL|AO6V1ZN-SBo$X%%e-X zw7nEe!k71FtB^|2wqedIIi!Vk`dzLp-j1FcKXQ_fD0G}Zz8E+Srd`bG%q8;EJB@9u~lW{441gw-%}Ux%r7J5dhvz%_ znSKvhk9JY&>#US(LB+2?y7H0`aISY;qY2TjOX;8fGn4Aj`oy<7G*(+?*~#E@AjQFD z9*U%ynVDz3>+HL}CsN1f{xcdL+Mrb5hN{ka=F=Sue=+t=9K70+oJJhZ9bW%N(t;VS zfYCOTd)Ydw42e8knRKn!g#eY|)X*5dS@#BQ@&p#}rN31aT&>njhut7oKKx+8HpTgy zdJCrv`trL;NW4UXL%XX2m)Za~0X-J(D-YDG_xC4s0;2NhCb8Vr(5sdoA?ru+%D6s6 z>zA&V*8#eYjDE_A+4&fB@k_=$Sj*fK1N3uZBdwQKX#P27TtUZr(HG{vaAQpM4{9Q4 zcNjwJK2f{ys_$YCkimla2<&bK*!9W4wbm@`wIEq}91z*6m(WOO=a)@LPb>gV)4Ihk z73U%mvhR{UCr59*I+Y26NCEGcsj~#V$**{U1*U{y?+$mv`aCBj{1hO&4rk2I1GR{Y zx{esEh;xyrb@=S!>F^gJmQfd;G;^7kpuC&hSBC8Ktna|Jx{d%sf)ps0ty_GqG&f@p z6ghrW8!&7qJ$~?RFt&P!f!Hy1_7TlWEE$~sSQc>Z__$*9I14rEymvR?pT~LJUgQgbe|GOW>F6YzsDWDl zrinv!k{7kxRovr?_%I3jY(hdlbzPt=cUP_1-3+jhO#=cQo^`==0tUe4*PUIUyo`cOf*|A3mTgEnNC`# z2C2|6%(b3ZjA7*Rf~veDB{quMo%QN3E}z*}L#z9~_VUV_%2yG@)xVN&H}MD{Q9XN>P8FpciSS4`{tL>-*IaTW1@ya!^a1LQJ9=SeS1!0iq^ zY3QiY#Ik+J=a89~xK{X5UPh!|zJ(iHtepDE7G%D3^TieI2~Yb0r(yvfT>`>wKz8r2 zt1OFm&hLDO6xF#c7U_+N!?OE2}PsRkCRXvU8m<<@~7BT101f0O} z%EoJJDl1{F4zPhbyf^H|yUYJ<9;$WT?wC76WDL z^|kE2GO5!ys0V$lm*^}jX}ndI#u&1ZRHJ>oGf+~ZkvqG8E7DPdv`9EW)Ucw z{fyg+D%YcveBt(lK00i+mHmT+3wvHZrZ6X1#-v^9G_9$^w#lsl<8*phTUrw@V$ru& zpR?=n7LjoNPCApCnsaSMq^30oY#V2&!!G+xzpG#}8l^3}d%9_la^|t{ zzM9cw>aC;!oI>@0<=_gRbqBmrWW5LjlJKNFG7Fv`#FIs!W@|s2ISN|dSP+Li4bsvO zr%j(P``CfeA_7D|4-+`6BzunaMGoimQhC$H3-?+O#^3tqNL^Y`$&)2fVMC6E!5^-( z+3=otQ!Cf3e!)`%a`ZNlaJH8>Zw~_O1RNt->e`3ScsR2S4Iv%v=}}A*f|vdx6hDd% zK)nrw$qJ13311>jWl1pTF*^73Y_y^nbO{+*#7X323^uR`j0x1}|ImyBUTr+SJKr=?x&-`ZF?oNZ9uV>)>kK zG+}R*R)s2WsZH9q@T~)Tvlo|shVI*B#6*lnoZGZL-awy>smdhlV_z93wvE5^Vb0-j>uBy%@+OwDriO*@uTI(M{7xfJy`4Sri{@DCP!ce z;o{$ldkU~wXHmPaH`7I)ccQ$lm$U^(cYxm)^mT(#qmTb5oeJ9Jy=g2HnD4 z{9N^l{yw|=wkJ;W9937`y=kq8J(~U>Rc(NH{-S6&Ckv14b@4)>uT|e6gdW>9Pts*! zHFG9co)(5U^T9-%&^u!iem=cuJ2(hIgWOLapQFpC{#&m(mfDar@jjXD63da-xn36m z*RI{rPcghsa`pSJXd|yQn4qS%J0Kz`csI!;Hgw)zLAkdKKUFAJO7+^e&7v9k2~oh? zWmF!x%E|zbdu%ONv5^Kemd9dnVQUbw=H~@^W=Ea_Eb8VZKG4bRsG}+)!+oFFz0{Oq z*}-_5?cTjhOS=*K;HR!7yHR_C=rb-vJuCE9H7;gtxiJ-4R9X*CGY0##M7KcV`|b;6uFM$9r0r`$6BYuPj>@ zl3}F<$f&Omjz^sS7zclUY738C7)^1e7x6%Vo6(AzL&2z>Wfg?!+cyvO;||n#BI;vp zlEK`)T)xd$H1(JyC?qOFLw2I7Co!H9B$wUUVWM;(GyxN)108e;csb+uU!X5MX z6bkKbv+f zaU#c^ayxTg+ryL=8iSiy7*zrB^m!~*vQ|2hi=4g>of%N(PIsIP~bci#wp>UuQUWdTB4S6qVa%T7X>~b~DXv zWmFL#3Batus$P@V#V$Z2#_5o!k`{p4AAH(Y>AJZcO(>XFYUpV5Q}x^^>#}*uBP1lkmmt`@!zOSn6Evx|2!_v#$58YAi+KRv^%URR1&0^fWYF0_hV% zRspICF&Qsixxtxv|h11|PEG)2dq zh1r~gVkx&ed~W8m8rPD;*p9)S>u*G_KHa~ty6`8j@|37w7}%%XsPdB_Q)rT-5+C7^ z^z^x2{oEQe#`2JPZL=wrbHe41mM~h`n!A)pM-pTs&URC^~6UZ>(h?rF#3tvQn0jQ~2a*jQ_QiG_qfd>FVl= zAxl(5r2Y1Gbk@?()t4s?`ads8v*jJKN=(nLadiCl0Lb3C#@p(PF1K1L6^7MM>7s}I zV#fJ_J;7%xrD&dPF}-*V-zieNJ~-Ro@mNdt;uaLNS*S?2sneZ}=-=Nkaq&&u+i~wO zK}5fi+bb!VKE3wu$n6wVdrm9;2SDyiO4R2?HZ)vb9<8iAj4x@bNq*nj+|zDDra1>m(co?!b$5)rH8~ z2(K6eNsBHWB^)_!tM?YO!nem5)7T|>yxRP#Q#~?)y}9o_Ux0^t$D;dvoDKhaA^rg3 zs<=h2L?Rl8E*td>ZV%j5ciwGf;UiqNIxcHA7|5izd)^)ZrQb_T&WPrczVBGo?M>z^*fol*MTytc!$iX9eduM;kq9o;*{q0$26<1~BVKzO;|o zVt$%mzPh^zy_K6jU9em<=gWZZ=IJN$2^C7ew4@8)-WcvVBr4}0YezOPhFSMe?RCV| z==D9L|M87oUZRTWpZ&ERf)!uAa#oJE)6>QOv|9fjkkk~wQ?5$OWBu)?BP(AnZQs&aLUzCw3jc%x{6>^-oz z8}@wO3<0}rfu}#*yq^T z8gt>{A06$=f{E$(2L9JSJ$EeYhU+5Fz|iwWJ9m++tcmE6|19kGunDWWe0D^m#2@z2 zrAC>{X6U@Nm@WTnyrRSkt!1Y1AD>eatFb+jGwphkrrSf({g_F~8%}}edLf^;P z%O81g^WVLH_!*aemzT%+rFYwE?v`cCv*7eSuYcN~H8i?Cz!Wiq!1d_-CiC`R^S~@{ zI{7tw?RC7@ndnT5k_|Z7q7@CeG;h$ZwkMq}UeT0^dq0~jUVZ$I%1W5(t+j@tLX}*R z`)poK6_TWp|B|?8%KY>4{Ldmh$EY|8fCsd;PX__3|Ckd(Z}&b%@g%L?ApnV^65?S0 zP}G_DSqu(lrf85sF(%M@-)LRY@G?_NoX# zV>tbanc0TG=@dt-m#p^r!UTii^FMwlP44-n66U}G$}nI5ZSHEZxm=>}*2Wf1o!Fm^ zKd@;)gFzKOkSe5-M>v71;M~>8BH{2ZDA~&pSAp!~Af#vTun3hCPD*u^UZKx>8dQ9W z%w!<67!zZ8zO}U_nQ`iAm-u#6o$}NB_n=t50kWN|fnI_$B?UJxqL(%{`9uOp0awAPfDRhGg23|N|Uc%ARA-QRRc{<%3l-AR}hyZ%nQ zx6@p6bhe)+cMmgqWaX6O-(%YGR3~AW8PR2XS<%q@%Z-m!)%Fm8RWME#*0P^-kjNO$ zBj=6$P)>3lt^v+jo!DI+coewpw>8a=+HkCPcs`1Zo~w=-w6l|Za9nW}>T30L(R*() z_=DN!U`cFax7VW)44{#&VQcpctbBayaJK17>`V{}OZ;4Ziy&bjaF3h?J` zn7!P%T!{%dc?P?&x#7zA2+(888{Xu+=;l{O(w}yB4v=`fM!(~_{O#2d{8FLNNKu!} z)WH3eL-=Owb=pQp#T#7;p$z{ez~4w!W9y2`j!mPA@`33zfiEyQRichAf5epqyxET1 zZ&Ci$-Zn#1S~B2h0OK8i0G;1J0D^zwsG*D(uda^x16e*071*QZk2Tgu z&&~ZMdnxb75fNDU{p5V;^Bt3ie;E7d0yl_7J(zF+G&QUAShsBP26&dEv%6$Hnvr99{__=|!SEY+VHtMj+t;b(wP>?DT?6DXH1 z7S$Zn(kaR%Iur-J!p(Z@8wcCapclfR<4d`FC^?vKbH35KW4!hwkr(>0D6>uG>*8** zW|h%z=;MMlQ?jX{_0bAqFrt{30a5a#W|>pxAy)Y*`G~!n1q&zqK744Q9(OcTywdHj zbxj3lE?<{Ol^A^5w+u1h-l6h98Qo*DAcdd>2Lvl4%AU5Z0nv{)W*rR&XFNuZ|GVOr zGXnwZ{u?!i;O@E%C8k26di%5;cFVRE@a7j@!nhq57Vv*k3g^VLkb7zKpno&J5lcR_ z-gVqxT?^&@JzHC~cAKO3LE)q;e3xMHZHS#GojNKSxN0I~K>yaB5CMK39{gF#Ptd2k zr9Em)4X`CdDKpjG#Q5LKRT*_bD8@7Gq0o^9?jH6MgNp5{ElP3sKvlBYVaFT5bxzBjY{u#xwA$juw* z(W-XtrJUI9pC)%+R?C@>5az0?s^PsRL&5pm;+~y8^A^GMM2lkO;u`_d?c0o$HR^E{ zh`+zW?ag9;!mP`~tW}ry*N2A5i>*D{2gfJalO^9<9*n=$gl_&W&K_;3pul^RE$)mR z!PerRj9xOg%frkUg_wE-lYTZe`!cd1@qE?0NEGy7`Y(OKp?OIaSF1H&*JV&B)Pr{f zs!B(0Gy{XuvR+>ZF>G|wCIok)7A8H*%oT>_f0msgXmMx?CG&n1yd(@ImPkbH&aD)@ zq+w_Nu+F3>2HwKiF2|j*Zm5-Qve#3V8xy&j@P>jIc7b#K^x*y`B!_)&)^KKP1CIcG z>uLQ;oFl2lxQhal3fmhVH=>lt_#$s9iFvUj-@}>fc(?>yYv?G6ud!~x{U(on^dz_v z`LXLh{L*D=s!7`QIvX!C=N*1RK?k1F&rk^h+;)!T+K9u&D?EmSw7{*ey;UKbeOFX* zb6tDiF6OVNdh{^Qzj|&J94iI*5j6+wt@q(MPY=iysa$8NP(FYElH=M3Bu)^xmz(r$ z$Wvsv_5hwRTFagT@uR5h?C2299@m&}`t)7>)O-IG=Ql1If$>0UOa>o(b|N%Tua zn8a=BH-RMqt?HnytX?e3QH>Yx(tAK|SC^)`fqe+f{A8aGE)*$6tw0`&S1S=>> z*?&7jS39xW?YIVH}~U3 z+gtpvW*xi!8u_-Ye9jT+U>y~ix8X?(*4@Nj72LD?K%d;Dce?5@5#!qSQQ@_z^8M4v zrd-%pIxqR0TJwsA#;}z>hNPd#g*6I>`nN&q*^h4GIa%D~1l;Xf=4-Qq3fOLmNmAO% zo-XUOc_xN*L*tgOH-z$MvMv9nqC#mM22YuhgNg0;IGc% zF@W15DEsPER=QY#+Bu?6L-`~J86-Xi0{%mwK`bL7x=?KQknP?vGS|3OF)CEuw6x3P z@U?80ETUzsDq+Tmo>S&V|0fQ(9Q9+S>FxTr+%-HD=#XgP-a7T{(DynPexHHeJN=EOSYwFo~} zhCFp6E{M#~8CYoz2dKHRvJo+UyFvRn8514}2oL`Ea6HHpvP!n@6wsQ|X>;@e9P)S> ze*yfoj2~ErMr~b^<~(f9W@vRiY|cl7wDp>M6$4wCS$O;BS{(-daU{ag|Eq^th9VnF z-Tl9RLuA0PH>;y8oi=O|ETd*FYoSZ5Lv!wT%Q#C-jfoBdc|(sVXWCSfT4rYMnLCQk zX{RDv37me20_V^4G*#ZXbw7p zRLbWOl_lykUl{`Ay|hG7MP$C-WWUx$C#;mbbHfp){rf1s0rI;f;OzCI6`YRfh~=TA zBOs5G-7p^-I0|MD{M%_PP;pGaxTP!G9mP=iaJ}fM^gO6rS{t9BYb}5^nPgHB!xC4Q zttixQ;pxDb3-7zcK$>r&I9`nBcmX5Nb$GKcn)nmV#j??|ocVTN%Z~@Qr|dzOO*}EW z!Zp#xi&g3b5I%t|hj2{jOxm0(OOoyR#hId{L+cPgq>o{eg)0c+c(^I2Z+8r>9O3Nd zRu^DO=%WoqYtybnQZukpWSU}aE7Ww!qZon3PFKr+SVWjz>Gb(I&wrjN$nCb7rD26; zM87GYaI4YVXq5@b!^J1wA0t2q8r(*2Wp%aD-sk+t+pPi69;b9^X>)km)*eIF=u9={ z8vyd&Qb?O>ziRhlqb3;rK7sb}dJ8XyJb3N?%+ZobWp%aVZhVyU+_cudY~0VMsrjx? z>~;|=%J1e3*4Bo&3u#(yWg@j=ziFy6T%CX1_Vjo2wwr0Uw{&siBPUK6{_|x=r?k|R z&VpxZ@r#I>BKdrzEJJ~OkQ}Ih8K9%TlSJV(K#XK3gSy@N>1X_LsTbB*rYNn{$wi8^ z8;lx+ktytr8g1}_Qf8rtFH9^1iK86%Hg@>XH0_%{r5(=<)@ATT9}zj&d88+9 z%1itbf3J_d64#Rn4ySjaSh~vWFZrPyHPiI(Sy53WH`qWf#UOvi_&bJtcxvaC^#KFD zIw9dJb_iPJ7Gu@bWCiZCN+fMgzIKA#cOJp2ZTu4YUU?4WhS)t5el+H4% z27SkfJ`e0zeqc_Qu5w(?S(z5HPUvmRN8`Gl{?<~=zpArG)m_OVcl)a+ayTgcveT@) zOr7uRHGNDcn?cr9$L1vNlt&S-lR#sinN!*Yt9>gsX5Hz(ujoONn*Jb7$peHRDkwbo zs2@l52L&}~@Vr1x4btT;^me3{J*z=BlQEF2kCtu*dY5B%i4!G2l7M}5luc!+ zylK`KU`1~B*~1L)k!z2;5uAH+as{Ii0GYuCZhE~oXn`R5p9jm-*?)2h-oIIyj1~BW zCBBz1;tXdk9F4ORiiBgw=>?A%aTmY!lPsL-*+dO=)ijlmM#TV4I3vohXw)Q*l_LF{ z+pqA7^gsmPt$^ges@Ple=c+jjn!xc|i|3?s*&=;E^Eq9)$-^AYIiy6a1nc55=enEj z&AYnmdOgD~Sen$KEWyhMrBuODC2Zo9Y+@5fCN;nQ8qiVC1kELJvw_Pi#uNVPcByaP zXx9RFN$>^=#K= zSJ@j>$Gpx6*xe4S$#D>YG=~U7Abehyk%B^y$C@-uhAH6NBZ~t2yv!Fr zDR%2424g>R`LU(H7oP0Vbm)E#Ofvx|b^mcX023e_mZ5i9z3TAdW6@&!?dExFB>~Xp zD$>!88WkkBYx5cmm>q%IG|Nv~i<84Owo9-O8H4w&WJV$_>D07-cZVKs z22#pOFz_9izh!d0#l1mNe9_`5;y`~Rq|{#jB9ReOkL9gma9eXf-LsF8$+5XLalOVv z(Y8dV91F!e>)2WYA#&wwaSlwTJeQ*RfWP0K;oAYT1U9Jqcs4f*on$on=iENv_>d2m z*nl}S39Tf+2|Nkj6mP2vOW2~u(kXi!N`OFgP*mtrrxn_VG4ZnOpY$0;Bzh+)`I^V`=Hm_eurT<^xwFyslW+%V{?*f! zz$P1}D$U-_m5Ca>Y>Oaw7Aq@0kzT|6<`=VN0^C(d0~=@OVms7cZ+mxR<7o4uyg}JS zYk~qFC3q2|XN0WpKqhu5^tyD3t4>`$5d~d~Ej&=(D*oB5Y-`+wKk32yq2_;a{>lHP zk?HaQyzt)56fm z#uQSON5s=?0Fv`@7MM3vx<=%7CMOBdl;0>KKC%p-`}*_hRne&4d+!Wd@Qol7&Zh$& zZmgOd4JpvAU^{(M4m_y7-tHTjQi%Dcrh^{As{&xFzJV8@om>FO#aw49kjD#P6(%Ss zcr&cxU)vf0)ZL7j`ue)JYL|EV-%R;D0S_PCb^k}SvT}u)6^Cc`$2X*;q-9!0t;KVS zNivYENc~U3lj-6jxGu+^4+w<`;yk}R;JzZ;ACSh6nWp3tQa39M|2z;PJsxg}x&NyF zzmo7$XcC;y|JM$pBW70<{-Sr1oP1H%G!~RHgv_fm>n_S?#wT_+#tp>|9y?f4vN>H- zd#^^sBUvE@`U2x)+m-O2k8AIJG1Y8B#Vv7B=nRBhz-W|;=ngHfU5 z*|ZIeFB5S3{RBO`jx9LQXY)gFCRj^9XG^a%3eCcSOV)HM1)E)FW?%U+9OPDGVcFfk zI>xH+R|zeX@0b=~#*t=JjKIF@)cviQ2WhiJ2^4|4sY#zKIW!dxn6#q4dfC{15v6AT zyqwINA9Q6gLSNIr)j7rnrWatc)c&)ms_?qOmD*ZNm8qyuF7MrQc9()LUZKd)sg#%4 zmMWB>%8R9yneenIaN_S-SE~lD1v2#G;#k{$Z;ZqGH1TlMGm27w@o_b;e zTwHiFqM2Z`xA)iR7{yy*o;3Y40hXA?v2<*)>NWMTSN^;dekVOxJ#Q$T>0K1}Zw=jJ z!o0p(Kfw|qeb4R6Z1{3)?He^3wAlksfCa_bDp~bhmM$MK_+6cx zoP4LfH}-dRQI9TbY-#BT0O~aR`W|8+59WGWPa9Rhpj2f9eEOa%_?_-(y3b`|J8~--9n!RkuGk$=?^6V zX8^0P2kSp;A|76e+~CA1@QZm@{v8!b>SA}3cvV$v4|q@aIZr(*kn8oEkkN7P2d4mk zZ+{PutKHk%GdNaCiptYE9<;rC{P4l0&hPvdNqc`$5^$3efb1JRuj;xrZP9(akjq(J zSrO^-tu=*x)^LcFcQQbW5LdNVOTtpx@8cdKP_llWKqU)jZ9Tx4mvl7zq=?byjZc!b zCL@H2in|eG&}ddAA|D?{q7l+sd6T>CS!iR1P4NBv%{v+T9!W%2LnztOn-G(3$Y>Zn zD+oV*yJ8+;9P+G&5pl@Z{Kt)mdDBDo(v5imFeKS$b~=6dtn#u+6kUl`p)oq30wL5N zA&@b1SDf`#V&ZbOu6gy-BnC|wc7&E9{a<6QxW$*AKAaCN*HM{&k_M*^CrM~cK3wAc zGvGDJH&bgm;hokq1A#Dj-OEuMYUXOOFUed5w1x{`ypTA1T)32f`&*J?-omMSO(vQq zQkCXiZ}%f#!Wzu?;$Or|qaKBFwcDpV!=42%8_a}E>jY~lcJyBgfInkvv7Kvi(oTpq zymlxOjDfltly&tb5`H!nyy(LeN%K89JPU^{PQLb|md5K%slyU+rj^R0h(Ni`XHO+2 zmfV@jEJ?&ND~)d%rN(!Ivbyf}H-OI$>!>q?HEc-d_=gnspV`@vD#H$(_QUOM>Agi@ zZIPQEIVgT{8bUj(;nSoWHwDDh0NE!a{H6{HK|5BCj%gh2xAEeiI9Dr{wSZ8@MDM8xTGZ9M|m=45qu#ZzHPrTEQ^7|7ZZNH!H8tBDlvdV|sgKQRI;W0CpXLP0~PhroM z_TviHJBykIYw$Oeq!}Zh3*&KszN`o;hYc~|;mMJG{1rsSxW2CV;=7TS#=80v(~E7) z#>7PYEiErvHYp7C1M~ZO$lYr6k3o-Jdi)SZhS$P36FxAddI1IVDJMr`Q07H))Dk8V z^GIMaj!xQr;>AIlAmcYD#yD3#Ovf0`@W+}VM%NKD(0jKBeKME`wBl?2XYA&e04L;d zAYtlnU`in_w*a1KiPxIy7LtJN6Ux0$v=n0$o&w(D6~zyRa9GG**Pg-mcd4oIV7+gv z=>%0_Tg&4ba`-Es&?+d+u)FEZbuPCL?l!IY#m8Q+uA~m?)T?Y zv{Zm0=D)y8J6W8(t)DI1E-Tv^a8wO6D*grrnzWCVj;jVnT_~Sdf&Y@2kf@)F7xE0I z#ou(b=KFxr?9RigC{@Z^1Q7zBrxrtzWj3|W2A#^1gquZpziG|&?M~Cv3GJhE@whpU z&Azuf39v}x8=c+VcmyUno#d?SOe1|max5G4NUDG>;RqH9M+Q1*!StD;2j_FJmRGFb1);A zGp5c)k!sX0hh1+TS!<6X{%2dk zbEtJ;ue`Y}bBwlpWz{b%Dkww}+!hi7RE;(P_?PqPgfiNv>p`*GQ@sARolVG0C3(`1 zxl(&Qk|sk=z(5NuRB~G@F}(P)G~Mn1epL9!;&ZL}}JsU$OQd_%$d)*c%ATO%Z`AQ1i8e25>V z>;n8ZYi$0PdcJA@YtNHh@8kEc2RDwwKS|vUj0==sXN4>uIE@yKF+tA@Gdp6EcrVR{ zg@1q)ykq1@TPMb{so!DEJMEie(-Ll4VeW+u(Y=_E)3eI_O;q|4&Y=9+S3YqqEc6aX z0t{xSxvVWqRG)%hwaJ2=8#bk7zHOnQ2{BRRFq#y@e_?L(tmCt2T|b%y7KYfS)6{b2 z(PrN~sQGjKbmaA?RN*KRXElu}qL(89Shm&5%-6q>I{%x7mM+=xHNlG6N88yYeSZ#* z_n(AS@UN!t#w@3e?8mI|7#)P6cC`|x6%sK z(#}4ZHfUD>JFonu$Nf=GhZLYQ*~ix&QWTe-V(7&X~k*hJF133-CjR)|!8lF2wg@(?;kmv?Kv| zpcQnCX$3jfVXVyJ@)FX=C7SpPFxM6{`S^nm^embK2`tTWYCJ{BofuOSU}_W`LMw$P z4GSm*f!2+hO^g|?e!Gu&bN~0P=ojKwR41!?RvlMUAtBFC=%sgLy>VwT64f17IhtUH zmUWY89FWya_m9zUBKjWq#B+L{;AY$`0ob6LL9cXEap|5vb7)3J#WgWp&)mlh!<@kR z@`D7$bE!japY|{b2u6*`^DmS-P=u07HcLz#-Tztai3Ud%#~82m80VXj!dbLsbkNfb zwh|oIWaNq*sWOBA3}>@~<4V;HaJ)kpkMDM-k$$wbC8Q-okm`J@+VDOYp3#Nq^&{jW zAhF>&V981L^N+NykTBu3?gayB{+jV!(pALHt$W9>U}{`XpY!TgBX+lBwuG+M&fk=G zIrmmi+a@iw*?`J)*%a`=?^O&HyItQndfeYX+cx**&Eh(DZ>wyg+%N-Cw{}N(g5r(C zpIs~C_$CmFqaR5~iJ4g)k)5;J0xNI=FdSp(}gnt5o(0z=xiz6G4Irqz@hm z9h^6daEG3epw1&sSAFDfg?dmqw+%Vg?AYe1a7{Qd*kNqKdr+v+cotkO{8AgCA7L90(#=6sRZ zT4>=e!dzqJ?3flduSyvG4d<3+*(5<-k%eyn8^j}g8HZQIx8OM{{v^bxBjCh`Dfjj= zs5O>xr6}QNS56l7l{0MuI>I%!*>NzY>O%-W2yq2>KcQJJc|cF_M4{<^*rTJ}7mMte1H})|mf+vRQ~Y1b3o71O#Fq0fxsq4? zW2KHzlC~}7l23l#(anf%i>$gipX02QNl$Ozcs8?K=P5TAsss0d#ka)Qka z{wiI)oaDV0wX0Fgd4vcj6bck-Ubc1J^si`+ESKhxjW(sl5BH;0#nXan7J~`@2|*C@lotE+>8&k&pQ6v79D(GEON;A7 zT;PYcYI+pZZvaP3v7#-G$2nkhxbt(O!?wJo^PbDG{)-Rf`_J4pOpQ6qe@LZ&IR=G1 zh%$#AreAF>(xZe|R(>8TeL;Q)2AjZ+Hq5Viv-bYVJr>49m+h2jL5?kjw+r@{ z71D8j3!~Gt<3mx?{TAbmzsyRb!@`>7M3J3}c$X! zqewIPBJ$stk9f}_QCnKEua%|0QGW&{lmNbpfD?I{Vp+?I0^GSST1=17dx4aMQ~(`Q zUCm{>n(JN(a|2!;=a*Gv_qun2XwiqiMafuB4<{Vp-bg|%e|XoWt?11=w|Y({4e#8{ zDC_GJSl6v`XX?*#*a0>OX9qV84Ny1N;G16|LbOT4Y{4drtvZMMfZf>5r!iICJ7CW& zk|ZG={CE3%>{Sh)Oh=Y!UViT#&}>3cHS+^f0mzjgxcSDq8!!zY%s({`xp>cLZf^cz z|HZ=1&CE#BzdF~A?!edEAId*}T5j{PQjyHNX z<7D^j?Sy97-&uFW*PC5F=;cWb@?rIH0@`fIy?5Se@b~oO5(u8S@0acV){y&r8M+BNV24pyF{^%L#4}aP?0^?vb~6TIE7Rqn7n%1 zq$E%(Coouhvr0m;Qb z1DE{kwfE0rh~M+;g%>6Ew=H8!eQ`BD{d9&aU`UXDddOgc0O!UKoKJq|MrVnVo$QpH zivRliHph>az-U$0-Hc@BK4Ww94A(?RjV%EuT&s##BP1<)wrk@+_@y1m+cAuQt3GXU zkakq1ocK-w6X4stwS`6`{~dipB=~#ZdQrpfrj3nf-Ij%g znc{xFp+Sbu3b7RV9L}U$IxrIxlMoVcKxpy8mjcQ8#a~bDuuKWHxvJA-fBxcnwClCZ z61>`kRVP<7ju-qQ>c{l?0q?sOMV-c(e;Gj}Kh4mTQ|$ggOAQy0_{#cUI5sQPSj=+1 zW#B5Vajs*af+>o8DW|!jf<3!FCBDf>^e z2cplJAyVQ%=;72k_DEp*{%18h5;O=gF$j}M&JsOgvk@JH!%>SFaZcdP1|pEf{QXRn zZ==~gAIoLAGcUYPoW$1BGaO$CEr2y37 zvRBqe)u>}D=;+l{G_JAx13#ZI5H^g>iG0jL)OB=!bSaj zT;}V|?d60!AMfPD#SU;BkGs9MQ~(6fMCmS42TQy8b-8ZlCiqJH=AR<;VkEXOu}OK) zBUN2(htI1B6#LhYpHMl*@M((0CW)z%zcObgv$(AB3LOL zmI7fW6JwQo*Bw9{%0vRz={b^2e79oFc+C=kWg{@kKd#{O z>fZvY^wD1kHE!QbjmL{K6tv7k#KK}Z?)HpD{+3JH;I~7#iWk$oRY}O^H)iE9Ir5DG zl9?9|ed9Ir_^8xZTd;3{E7BK{tSF8rHiIU^aW?9GpB)c|Gs0}*~w&l=MKI#rROUxD>N20OYpez1S#g)pPcmw{lT%|AiyTY046R8f7H=m^0a#Zvs$I z@8jXg<-SFTC(5JH?)H0v!)M7Sx1;Op>zRPl(!`iMfKTA)>+c3i=W8q;@r3bd6Z-j# zgP&;;ofp_XbGuM(`tWtwEkh%qh2z-d?N5doiPd+5w|Xwg#B!z2la&z)ah}PaI3P!U zJ+g;92P8)&Jj^QEau>(d|Ga)ii*>W~P?KS(rsU5m%DQ83_bljRdXkLzGVF0k%Zvtb zD)%K=R0f%#y^O6KmQ}xxI;~F(e}a=jp2=E|u(Tq2+FO`F=dRdYvy|>^4kp~v7+Gny zQ`TW%jpjgW~! zzKVE10A2-Lh@qXTLy4C{>VS6?nxk;OLnrAs04eIdng37Dm+)Pxk)sT^$kpWYWr%c7 zS6A29uUQ(y{rPt;b!@GlZydjY!n^8Ue8K~9Qv&<4SNR(6Yxk5TGP4a!#rZY>r%G<-_O5#pw<|Dwo?=g=Et;N$ zEmNw+6t^tK)}}#l$q=$_5dBOD#cCs~B!-nnKlMj++OMsOaPG~L zjgN>)DwFZPl~T!JUTDck*)goo zp$7TZe80$l1oo11PcY&JG)TncT``T8-63K{sU0n6;~TFnk>|5YRRJ`tufEO7lwsg9 zGEi}+$$~&S3@Ju={O|v5^7KD5k*?Im4qyBHQfUbUDN-gWUdWMcQll^`*bNfmbVj5z zI?^6}#bH86m;W796g!MPmyxyl>F;ERlt*FOEEi7mnUo>4qiVxHN>R)bb2=Qvgp0P0 z_Aw3rm_yd1(sqyu|H~AL68|G7rb^Z4gJZx3WUOiVUAdkE^92k-Ns_ccEqL9sFa-K2 z8f%85j43NikO;wYRiut9b(beZmAr>_RF0hFJu+=)(@4H-UNHXTyz19&9!q4pTCJn< zQI|#jN49n?I$qqh0YvhmC3h+J$G70}v=IpP;#_q1KPXJ!5 zJ-2(}9@6PE9rQ-4U2XHmPyogQwCU$JYt%dRpV&cd8T#eSBw{=A?XGl~AvWXp_xHfo z7H#wQZ^xi8fNtj8YF^*&3;$V8SkMXV+0gn%W{xi|03(xHMz>znDSXl0LLb>vf7G=y zr+B_oc=zi?V1Wp*R0a|-m9DXtFkCjS`xH}Ecc^A}h^+=l-r}Ru#yp0utv-ws;CR{2 zWo!r7J}Tk7QI=a1)jJ~7g~&99w|})55_F?fDsd~_Z#WBnFUrXE)ZWN|9P1_0+Gbm# zgeN++JZ+6GwMh~qoGCYU47~|wo*UTJ7uZup4F}sK8Pth_NoXW+{on$ye{Oz$2Mq8v z#H2SNd?_xCP644LzSC4_l=48#W9_Lgu3phVD7Ms<(kH^o;-}U9er@hxQzbf|xc)e= za)_fW&l^7u^xlKG@J-ZO;Q<33yGoS2je#YBfs*B*rPOhWSBv~-A@Fognqae0CS#4> zd|Q2M@dxQR{`*HTs<~o0{g*WH1sl37FTyJ$_*91rtny~IrK>FaMfS)$`-i;1!q}Vr zmD<(4ViWt>64ud{S4Q7C7rYjmLsZ$rVIGAziUHUu2p;XmZM(3*6j^Qt8N@nT0zQZZ z7lg*rx0|%Yk(CSX?Fc8ld=%kDTu>!LP=1f*(*y#WyT5r|dGs^iNrA8cNPq$qVD*+* z%;%#&e0^dfsrvLf_-(!Zt3lOR{aLU+rfXD`CAPnzPjpbdiygHr$gITiE^WaHud*+- z{i8~T4kSMA-Lu%fVk9c{?@mSRG*5buP2zZkRc~*1H)D((#ng_EwC4W#pc(nR^_R@- zrt-2NwH!y;ks#^(pJ{j;Q6A#JcXWZFt$|p^u*-HR3XZtHM_nTky^ojz1fh&eegURz7Z;7Im-ozm z16Ij-5ru`5UJn|pFj;5w-UpjBHSo!^IJ+(?J<+@jP^qiZ)M!!7lp9R@c^1H0Qg1@#m^QTZ z%R>J9tK3dIP&Q>8EEHjaxSlpOY(7y16Rg#vhOTT?2h`c;v0<6SB_$Ll=XNQUmp^$a zA}c#rXk2AGLq9^@KHoN;4LTDpcxJcE?_H$we&sMQUo@#UnIGN@GC8VpjC2r&6FZ_tbG+ZqwfrUlUN>w%OYXlcYe%A}bD zrxecZv?fnFPDFBTpy~~|s!&t=-UqL2zQ_`qw&HLWCNA{nFz)=KkGPq|*d_h(!q1~+ zt(JU;2Rs3!%r8`Bq@R%|ay{d_)R1)&;AH-TpFr;n6BL09q1?j(RZ~6xR-(X7#Xf|Zic4t7O7d0_EJMa_{>McsIb#QsKcrHmM&xm?8 zKi{WIuCLGR$ba8&XHjjqpSCxQ@D$!xt?f!E)^a7VRFTEk##tFtUQYW^u;5aD}9B zKh9<2Ev{AjDorg)LuuBiBu#J-jcyh9{CiY8B_<{P^{%D;Di987iuI(1^QZ7&zl>7d zOh6KC0d=d~m7h~1cIb2K!S6~6E}2y>(JQtXmDN>%bJX(5!@$<2{&vf4h~8x*29MF$ zPdiWas&92lpfc+x)-TmtWDQwcbeab-OT$aY*Zj>A=LBAQFAhii;h$n-vnR5`1f_#j zt)kuy#W*`C5@2y1x=F&i-! znm#50B$QzIe(Ut~^y)M~l40JkA2cXA#3pokS!Bn*9QhX? z?Rn}bM1uxp4fa1`3h4-O&F~X|*=od{K2iu^C;a^igo}rC*_0D=P-T@nL6^{e1c*QW zs^NS1-usX0KWl;eJ#+j@d_}#P^P9A0rp9f~aCF*#jm>%9ZH`~^&H=aVrOV0?A@!|w(wVtk)I~3JLX+hxJpX`I*G$ue>bJM%aLNsBR zD{h7n4V~utRvC)hSz%0hEin(dcFG}m)4_g`MK{IbU~4dty*Vi*V_%*tDJ|UwIyv9s zHTD&#dqV)9U|ub^I$HrzPPyPc`9=Abxbs-O?r?gkE>Zbtq0yyqm3}3o zRkY#i($#jHmmmTCcdHG*P{SbIV+4>S2#34QvNYp%z3o2zYXduK) zN5jm_DCELWRYfeiv=$#5P~XH_oOl~ohu7!^NyW^s;>kU6l+%}I>7x#$?#dlFeYiTk z@@Q##!Dh!o8pRr2hh;qJ7IrLO+ZAYMa}_W<9cLHe+YZ9eiNtDFdELLB&h{?>!Pp`)_04&qp1=^O-bkX!j3pLi&kiF_@T`tzU zGk*{IxO`%1PWrh&itA>?+OPRQHAiPQ;uQgQtsxwi~jt>gihA@p$Ngi zS-{0_dy>vjU*vY()IwKh`#vH$&&bC3Nu1^Y&x=NU4P*TkADW6F>7CtiGEkpn*@Rg8 zqhBlTPz=U(C^DnO;H{wlS2|^386SNjI;hjwiIgRy`8M%Y2^=obY@JONL>)7GFj&c? z;6<9Q6oG?DP8~s=kN%A-BRnQtKoAY15{AjjZr$gUbLy5HdtX#7HZ{6#BK-lG?dJy)L_a^F&%%Ot0( zkrHAN6Atv5=C?WR6XBXBJ*i>SG`+53ZtwN>I47gPe9m z90(8BF2`A9Y}6^s@MiDSEvNh-gctmr!`XY+x9`4+>*1=|Of$GabIQ2t8wbob?br9XhzUI&sktLES!tNDtq8G$Zc+I>=<^ zl?Rf4ybo)nMeLv5gcK0d$nJ%VxO+SYkqMli@67Hl#T&c+YC!ZpWnNob!z`Z#%ScI; z2Cn&Pv-vZB4Lv{H+CjNL{6jVL+#M(INf(wM{M$R>C$I>6NLO~}JrpFvsIGRgA5?>g z)$x0RKXzwe7{k_-Fv6x(sA>%Y>?ggw1lvY=_Wm?$NE?SbvFO--anRb?aCvW1lrwok zx5KgbJJlDDR{xb)NBV-d>yyJ6T6^L39}go0irCV7JGOg*ct8tBO+T#u!@uv6nCMfF zNEl+-!m#_!guCwq(CAKwhjnyOnW?|IrH35r(hOH;>04tZOU55hJZMW%1ZO1`bRB- z>W=zU?@+zhUNfJ~RwoF@2KA`i3(2BzVL1{niOGtz@9L3pza^zf923XqPW3q_I;0?MWMA4 zH-{Y$Kc%daGrCVnjX1;W#H(56?kpL4x9eGW6HLX@>uQT^bY$JR28>witU^!`fSo7wn8*iRpGK{tlQ9YdQg__AAOJ-_+pg9u}v5)n%eM`iQ zS;}^u_1hY+Gh@WqHnvq8W&F@*T-Fys@%0ypg^zCYBMiprCZ2z@+jrO`ull(7RAd1#B`(sKcRB%Kg%i=0ZcmGn0%3V+W9l58>3f)= zN~SUK;bHaJqdPMk;n7{KVH~EVk3U1|UeC=A#VpP97NYtjKjrfqj#}MuYhU%2hme^7 z6*?CWXC3M`B*m|%E@9tZ%F&J7iJ6Y zOzZq*DhT2zj`mH<8_WACq*~zFivZ~VqWAyV`b$g8*n~KZgmY*KA>=Y4ZFXpkh!_ZvxhWf0nmbAJq?F;4Q$^NOZTQ>6i0^q&OSjeCm0T4Df?zSCd?jK^2SE#~t zpwkTFEIYmynGGq}Up_Q2j%BKhA#&tTTe$dzqq6wTn5PLPbAu!EaN_2yOR*_GLUH=+ z`lNcOE2jH^C9;Ko`?|g1&-J!EE)`zi{whPs)k($da@|K$3FTkL8mZ3lB_qJF-IaG? z$*#mT7TxhDc1<*G1%c7mNvW`$__>zrnS}v=Lx^cuq6otxp?f82AAgcmd3za1}WW+wV?BKA<15eTQMs7^zOWSd0mh~dl+y}y|hP5<~&2Tfun0lGB zxF?jdvidxjch-l|hop??AP{FcsVLF7AA-|KM*v#y`vqSi5YHG$7YtD)$@28#%C8bi%9s({9(7&Gw?*Ida zukc{31(f`Amb>G9i~C&Y`6H_R@d|Nw((r(-Y)vH}3a1?jL(k@>=g3fOkC|;I;M%DJ@4dRA7k`EK5DxV(fX?sn9I+8`wIvPnI=< zy{1BR2L>|~_C8+a#>T#2*{(nGk6qvT1CWD(@!>wqozjnsZEW1&Pm{tSZFP&%%f+Z` z8{38L+F6Gxvk%XfGlvhWiQw-C_U2wTw_T9WSjT1@QrWqRYv|#954fdE!oG!<UX^^OmhM3c|hJ_9+4})H?$Eo-!;YEZJ#wO2*t->Jh|hT>7@A@t%U% zjXqv1=t3yCRA-REt0>uQN4%e@KO%rO>c_YCfgeq2ku0R2-Q9u%O3!|_98Wy1PA+aS zgxz@w9~?C#8Aj<_S?`6mm&*AtFD)qXvUowEy+^UspJ6`qr${6K4+Tpy#GYOO)-eJ@ zJ&4^rclT2_clTjWVdeE!w!*g`ZBN;fCmDtX_5raYm&Yj<-;YpO<%AJO8F_lCjvhI; zV+@8P=Sy_KmmDzi#!{oc14(H~$vlmU9^})Y0MYzm&!KQUu#99_NB|~S<_<_N4ZCSB zd`N(vO+n9^3mI;u1(PfGC6mV!i+~pTZoPpB1%RE|JJbGRnTW^J>)zY>hKB*@KR{tp z2)I0~e+NKI>5!mg04>k=BTo^Wacf82XhN^npCTRrUDUtZqlVs_QxpB;*n5Y<(1%^k zu!r-)`@evq_dDDlkmTas3J!=Hw3_Qr?f+;01AJ~mghyYLJnld4JQnB#g^2?iJs{3f zU?>c_7W-eoIrMxHdcO!(Z!lkfJ}X*w9~J;)b8LCwwhA%{rAENWP24#uRaatSnN1ma zHLE;p*FbLX={9|iaG%Ui+W-CqYX=M&mLS)36fFcGDJHK_+ak0s9meTVp=t8^Wqyk2 zW6Lu2;H3S1hqqOsOIj~Q-sAZY)oSI(HVLaPYh_?AIGJXbIS;DB-%RuK)-dhO_OVal zbN07UTNTqR{xFxyR$msghlFUn2#EWWfVS0n+H>c&f$9Ofj{AoE?f30V`L3>ZY_| z?|#_6(DiFc`VXAD?hjY)+M^Z8$j64qJLv7rTZ{E|`k`}gR$xm)0h5*r5igt)-n>Tk zKJG9A*8~3lZvO!oDFN=mm(+~eJ9?Ju(8Q1XfebO-)v0}rjA1~TB%kPgxNLx4!x461 zfvX+ShZDey5_&iFxO=kd|F~WF2$&B%oJ{Rm_c9s;AJ2%v3$eCWM*2k?*wzK888BoYbyi%iyB)Ar>u=SPD9 z$}8M#xi<nz;QJM{qieY9*hoL`k$&&7Pd+6E?bN<=1y@9mWz`H+xoC@|!Xl;gz$sz&^ygAp`BVc-RvT#wKb&xE$%uMEM7yNO19Z-V|%Nmvg z2>9epV{;bavsuZ4eNPQve zhIr8e8oG4i?(Pm8MeIuCCIICV#X6}`N;)_dnk{DgPli;unmwo659^3_zKf%$eA~<` zu+p=$!MF5sv&po+goVY}#yi2hIwWs$KV#P+Jx}-J#ly~1prLwTd>rn*Sw!6ZX?0cs zn)U$kLT0x5KoZ#q`QIG;=$EMVll7YaE=EE_Pssh@iO16FzsJ3Xhy5v;$2*|Vg3ACN zdY;A$o1_8$f8V&(Wi-`0ZLz|qFJ&+8LLP2&-Fu;r%oHd7!5`N1M$`TRZ^OH>!jOlX z{m0*g-9eYt$l#-TCk3O%4+gE?tCl9egv3dxFjwVU)miklJ?fh8j=fRG zyydN$vXH5xfuR}BB-PP6M)rl9sr!*0LLRbny3tpBo&!eSxsO^dp;)eW6m@TsU5Z;< zhvt4MV0IhUhm4NBZk!6l9**xo8D9XAvj6MOqJI$fb(*dmf%wa};Ql z%LlzP?#N-@9UKVed_p2+1wDERDO!Xz6S|!-rzJfR&dW9s2%{(Py!=R_zoX0}iiYy{fV<676YVS-So$qw!wz1xUg2y8)?J|9jfk zyR5d7ugNFs?5vo@p5s8G?xhP0{lR76i9qmpzKSw>R;i?n4D|kT@osVai#JQi-K~f_ zf&qvZ=sHTPn(uLoK0QUSE0!$9Tox| zQm^}joet4`fXhIy=K*gpMx>uaa~pidYw~u&pV4%S?y7ytCM@K?V)^f2wgM2^Aj8gc z5o)u`@<{G(U1x7gq6E~8nv_;?d79u&Cw2}33|ZNQ=^Hi8K*!_Yw5gPUg!OnsYZhzs7(oKMLc#(6j1nfmMAM zvBATzCHb36*ArULrLM0hg3H5~Dvlj36Kl_U^}44X{)qn_(>kl(j4=DDN75MFa%R`= zy=GWgkLMz#`$x=NJ}reVVSLV z%6OF84^@Ae=utV@rfS-yn=73L?kya=#+Wqpr^fSMGZXMbHICK-7V9b*IRzD2g&S5 z*-KshafW3Nls`S%09^nY1wR4nrK^Kp93Q#H#*4a=$m+$5)7+J9W> zN=^Icd~Ti+4ghexyW45O@P7T1a3j`Co&GI*Ua_+K)HlhjEa;=;X(=30)NnWc_K3c{fDr}e{k*+% zWN+Bx;^Ue7_`D}Tzw`E&Z$K&>e0lrO?CV8(~8bioh~g8L&Rt2F8)v50!aLQ~1&kb+2Vka|Fl?wjQEi^~Fg5BWvxh|Z> z6=@w+{4|2t2yL?ca}`kq@a_p=sPNPfo7$1wXWv1{8Drvv?ubk){@1ZUBN#?pC{CDQvE_LldMl}Tc*CX#@;lwR)c=X;TdA4 z2o0K#vX&uH1>qP!B{jqyNc8(y6<)rHX6CO};gu~$Zcigqs!WXQODfS}Y%07+!f{HP zr?lvoD>EMlBrmH-t6FWw(xV<9^Ha|B#gUZXEzzQXN+2u9!LAh5u;%# zV}Y*1Xl4{z$2@Rqcm-;_*Gj%f2v%0=m0nz9OQ`7dHWz#tTQ zGjtW8&rY&_f2O?)`N++yW%sI-b)reN6-pd0u?qp`WpOVuY{!R{LS z^-n3&ybAXEF22KokjtCXt=ZYMzX0c3JgMR=Z|~iVv}T}yp%Fn0W7lwOEO9?S0cw2` z9}9s%Ag_bPLj3^@WA%jKJ%C+-56;d6-b{+ckxS$PRpjPERKtH*>Z~i7UAVR9zGIni zoWO2(>d#ZUrOP80rMlPF#v?zmY%}qMy=b5C#3z_XB~V|#i6wBanyI`GD8#(VWGL4%N|9JMt5(0Jj@|D}Zz^leYL2F-Kn(zFNw#B{}rflK2A;z_F@7*+w!FDG6WTI zyuoBw6DxC~*UXzWQq>4t$OBnQCPfxSvMA^$L@MC;cN=AUHHc=G@uI(!NjkSncBbG5 zvfvDa^6^_9khfjsh4b^%7*!x<7%`_a9yRy0bQvpd(Hu$c0#RjeTsT$==0{sRmXNBhv31p~ooWt0nxk(6j)86_Mr$DwqlVwd zN=Z{S+)K_Dvw6xJ_gSpx>*D6)m*mARo%qAERCC6%97fp6l=Z#$JjJ62%Su_Y^@z_THPuwr@f4U8Sny#x zqGUEvAkg?2ZuZTAPeyfqC8wjDu&SN`1XGU~l+lWFnG^De>B)Dbf)A?f$E=LhLLZ_w zgphK=DzRAvGuID%(8KGG!bL7qS;>qNrz3Z6n||`R^umx{sga|l)@MA%5<5}Eb1F^) z=%>6^{{XAY1qw}w7dWRGR%$%6Slniq#kvA6>st@={|Z>+0Ykt|S}BKsfZ{?nv3}A| z7)(}KpZNv5LTF%6rLB;zD%8H^R#GX>#86#(%qx`a$FSlzg^G4-1CF@)fgJuxQ>#28K;_L z=XBgQNAvyqYQyKwuTPF#ZTM@Qt~VYjB-87qOMj@)l(74YTrsQfNKWBnWhpe}|8(!U zc~h$+6l2x?VzXzpJIohX6llbi@;ILWj{z#0^-!;%Lo#1d#yY|xcoA&zngqPt-zkwu z49eCE-@N-TsPn+b+goDFdy-WA5M6VDFNs{)hy<7Cw>F7h0_UcxG`0+0$KF}TABEPA z%MTGG7_XToch0F~P@H8K_p#TnMI>bkE6xvU_@sN`aNT)Npe=NR{JOXRKz5VE@~r&s z`Ht5Ln5%pyzb5{)Xnz48^M3mmKAz__l;dRGL#$7zgl(ldsQRkEl9O2QkL1+$kX8p| z@|R?SESPOKO*h-CTCfKsivXI2b;;zC0L5Yum`;b4nD)6?;Z$h5 zUd^d>)XKyVni;J`t9ppIOx5?pc}mKU16z%{-+P*Wq_HK07VHa5fF~xNmm#$vLiy$= zHn?k8NqS1@=|gS_xXw#IbIYAvX7UXM?R8spBk*1Bd0BAge;T<;#bgE}d?TQ1+nygsB@ zH8rF$-#DE+IK&q2QAg?0N)6nMCIq`vt?r0<@Jt-4xJWc~$%G#*nu%(q=dMh+yd^EF zj_jhYd}eS*Q7x{n3s#1ZL%hTa3jxsZ&)LqfXP&AynW!uc36q1`LA@dxPZ4BL03r)X zvtAdhD~A`6YtOK7rvB17C*{6)mT4~n!Gu(cs*{mjnX#?`ekDaYSTI=(G|qVA5lfI( zG`g%h-E^k@jGx`g^S3H2vMd5Obq=SQ%NU1FTm0~Js28z29U@zt58DTf$K zmAx_V>M;*NLO(IAJOl4_>GQBy_N%;lfiG_mk(y0IOim8a_R=k`&zABx@aRB$%fEh) zWw$@(Kwri59NiLkrE&Y14LJ#!^is^8!k>{~SsytOg$>0!g0-T$sBWSC;&3xMGR)}7 z7HsdWbRBNLjrZup0+{Uh<7>)G!*%AT@V-O=^kybbM`mX9Ls3CKHU&LxeUj=Rd`cIr zs5$R;q+e`UF9ON41)^Pwe#KNj$Q~#*e|O7zdwmk;64BH>EEaDfW;_IXmC4}3)Y+@y zJn)n7*=RbNVztEtj*V3A@fG!Fm-I5@uiiF9z6+5@LSe7{uKZ{{t8}}o#ZgZs@xpqc zf2!K{ZHr42k{t2gH8GnvAYUn{=q3`ngn#aiT)=h<4pXJ)scX^oJ{>;t!mT?-m@j^j z^>d9cZ~eLwhv!`S(rH)6HB3&w7^41lKCl4Tze9nu9f*A@a8XvNTN^L05>${3XXZ+& zWQkp(;tm((Q^kZSF;$o7Krj__tE7?*DA`~3d*CvcVW!G?F>UgKW-#$}sIg#pjxpTv zm>iYP)3HrIP9}dg05VTL6Av7P5nOV^YqK@qL8i(oB+bt_o9n~96msd%L-XFydn)LI zgK=)*Qe2Z=vcRY~m8W-!IjaqVR^|(glDVWvJ|31uQdNM3^bF?MKA zL9REib8eM;bH(~e)MW2TrO`9ydbhggwgTlh{mB&f;{#nN&U zqWao>Y%dm9wOAz=y{Xcls;X_x+)vywtgpupEvSp&=a5(I4ddx`k(cJDjJLt+=|)I#_k7Tv-&hJOdMmGyuJG^*a$vFQ?K$Mcgz^ z*HIoL-Ck5xE1?&GA}c8{nkjZ%ozOvOS`;egb!ism%x*E&Z-*E~>P-8G=oYy^XUoNyVhUlJmrs=e>>MQ8guK26qrI9n6^dzRn zPqNu$r2Kl|l?NMRf-PaC{Fx=^+|{oviKposSvXZ&I@`}Gj{rAs9acJEjZUqv_e#}@ zqc&cRSxXzlq_19yuUEbL*4Jt01GSGjyb*hWa2tZjbV_q&CVEP++-x70Sel(K{`1FH z!W1I(@>5~h&GL~aWrIRAEaR^tBz`pnz4jsZ2>Hbc747mWE#RA?T%WN9Q~f<#TZP*w zq?TuhkR;KAN9(6yhq?TMM{&}VUgp1ETf{a1O3ax?Js_D}Mt%^JMhf#Wqmob8p)p#r zC9nnoTX8gW%4-hVwSPA_dRW*(!NLQANiDb^1|#A){8n;#86~{2S}xS|>zxFikia~s z{oEEDJDt8Qdd)0u0lDlG;Og_PqzlJ4%_Yk#6;4Hq7|F+gLiV6$EiH=z=kXTY97`%^I@9`E>m(s=k#!U|4D zM`sO(aV4{{)Wgr;`C*aN!39!+hSJYjp}5aUn27<6g5Rf*I^hIAb=SNR8KQ(5#M@mk^^a8rl&Md^EjuJ?dbFAtPoOV_DgC1UZyrM%p# zU6EPh(la0!l4)0*^XugNNR>|knvQ&86oc>a4a3|D%WCZP;i*X|DZEZ<3zy0!l9V3; zVJ^`L<{%nKIWOkMyCp?+Gj^wy3%+25%^8g_KnV7JW@5^Y>(#r4gBt5K<~chQGL_oC zW77yQRUh8pcegG!b|}BtUpf&!YLmprU;-CKqUZFf4klu0p$%@*w@~ZjatI1|$_`2s z!NaJ;oM!;*BVj_&4>6NgMv1Y49#$3^dxu8>TRy_S)}KAz9ls#juJch#qMLI0jQSr( zXBpPy-$(J0j^5~!7$IHK(hQ_ybT`sSN_Urnw6u)wZcw_rO9VmEph4id{}0z*a9w-1 zd;7)tp7S~C8*0B{Rj%m6`{!)t=#>$(Ub&SF3g5P7-KaOE<5Xo56v+|u#7fS+@80>l zcKFzp;w556C$`Kz9!@zLQ2qTnM}Y=v=~`GesSUFbB({H6{+w{R#{JX%L&4ov=LeUJ zH|B?;=Ofoc`R2iQe;nVVfLNT4rvQHA*!rwexECpw3xS8?7e$9JcN`)%@BzerDpKks zS^{0pFKWPqM@pJpfkIQds)>_`|AArMF|*Oqud44HrNLMn>FNH++SAD%{tW6F-BELPra znjfk*CH>aYaou+M{x8b6b$!$2=VdR`Z7#_eBL+Xksq!mQ;S{7l)3im5szP2VEL z1S5J@bzh_u#@`oqZW72wOEafZL&5d&m>RKuR>6P1>SC zP3!iBYqQ)V_DH`SmL*qL&VU;?wu*qlg(HFT?@v+NW>f7i+JD9og1E-F0$Ph&2kBKr zhE}{}$uef3tn{VjsbQ9eP_o6G@$aryOxSFQIMuka5)@JGW^m-OHh~UuhD24ay3$!g zZm(|wb!Y)yo21HCcv0GT^iJZ4O6eG_|<35mE{FZDz8L=gu2dy(mAu zBLj7i_5rg!Zn9X*Z@$RkbuBQF#uxL%e3Pm;Vxlox{4&Ws&)?3r`<|%?a1mjw2L7tO zOr5MtKdcUQZA5>Iah4}tsp|J&D^eg*WQfG2Rk6PdG7fKE;F6PIYn^Xy6{+>O|AHB- zg8A-ptXJsT)_;{i9A`D(s-LTTM+#5ffUYEkSXYYc-()xfOdLzeQ@4>EIR&_cW@gZp*~-n*t8hd$ttD2n@gR{Y1A#}-N}{fMVaJC z<63>x&L1Q;y><11r;*Lu<`p!ZOrjjluNQJo5~L6&NyNOl4#*9^%-b=I_tB ze1A{>KKQVi+(_>Qf5O}F-D;BYWvz-=0H)G-XT|9^=IXk5$3KWJ(JY&J7VS6=jZaO; zs_BWp8ENn!!b>)NBu1*hT-rTNh2f*-Z$3&P0gjlixw8z@Lqu*(XtVI2E(T2!T!a%`55O+Q}a+%SM(ba4)>DTq8s0fP>w zy&8f=KDT6;fOgnTdr`W$_4P#z8j2L5!oi_PJVp6c1jxXqeDxdDmK-Y?8lt4BAw?%+ z=3ps&cBZY2h=ieH)0ZibVm^mx!Vj$LiZo|ux4`hjOlRVybbDEQj2&) z!ot9ANFS3Fky86L0yHTs`n{YI`#S*lbNfE{4+VpmL{~6%{`*eIy~e&aISFF~7tRIo znTsi27x%ObFv65MQA!5riV!yypEh3)%5xt-SWykmy@e2NZcL6EFm+q)kmT+mm8*Qy zNreyCu*mP3`f3Vy=;0tKoQc=ip}C4u<{(Iffzp&i=`p{0-iuYgnshrm15_Fa9SHSL z>C4RG$fcxc4LJJPp=U?8c*S?2?SS*0eggW4{vGBa^0|BA-&*NFVcl3)0=if7?1z17 z5QyPN%D&sy^C;s6*4-#ez+Ibal|U*yMGb+`LuJkTQbPIhlYK#-J3UD!CNy!d-Iww1 zAH7<+AB5lUmZd0K zaEXba6M<~fxO9-K@(F0#XESHb@4RJci=k6qgh*w5w8SLPGi|X~q^5_B|9Xkuy4MVm zp)AeuP!lO_QuqlU5c?5}Lc&v;GlXn?$)Hq$!nVjX6f!}N90*Y^o=o{bhnvaOY=+BD zf`x*W4PmY4e+L-)e-JLi6ZAU?M5$Pi#bOzTW0;OslC?kWYdbqTKU13hD!D=^eM1UN zA~T}`-7<8X;FBJx*CW6O+#&#{*6br`Bu4ViG-F$u;DhpCrA0! zvhzWio^%ekn)A!Z#K|dFBaI%-66+sG0|Wo!rAVIb*4psK2gUGSZMfJEzOFqX(^_vl zYDyL2pLh~)n3hWrRu=7;`}s+`TakE z@v2Z)T)^_Bty5G7h-X036c-fwirq{OE|QUc6mXcc{mP;#rPieSkfHY8K@`Q)L@#ZM z$$MV(^2ECna2o)gyNB!T>2sfV?+&`9;R8o~EIz!Zwamezp?qpqU$m7qIpg5gwaTo% z`n7Y&uXf7*3gacwKpY^f*Qa5MTmyY{v>i}WkqwmFHAdDOw*)z-TK=` z5(8AHx|-pYd7BSXO0IUU&#mT;c#YEf)bYn#RC@=uZp!r{ce0;*Bq`&yb3$RT<;p3Z zi%qA#Q_x=br-MF9QIolC9!%&L$|lAAgIA7 zDWfqL&L~T_b{eHu3nV~$^8l4^hbYSem&sfCQ;dR2;V&3bjFPSb7@#F!uszZ7xuR@r zP{XbB4qoQl!9jI%4K4fL3R1HJ)Gy}&X=V3MCPFly`PGZ)e)TUW$Qmc=E3%!6b`3J@M?P3m`8!l4&#s5Jem6Zj^U95gjQ>Q_<5 z_OVUcJMRG7=BuZh;|l|&Lz&aYpH)Sq6vl3#8ThOf{xcBktl z<7bXW>t>X8lTBaAWFF0N@u2~HPTQ)(@r<*ri2(8r3GW2Er}zmOk#KAlc8EDTjE;#T zE{tcHCt>_(@%x7u^k%~@zIXj^ySn8f#U^(Ta}bA_u`9w;F0IGHJg=>p2^Vm)u_(sM zbczi#5n%G006%KLzokacttAR&;4b^`xu+I5EK22ZSuO zzw+2Lhn0p7&!yekCr(_ZcIDvM+KznM-{o{TeRlrnZuJ$#TW#|yUlPH;;@ii%@O-Rb zv#d_7qb9OK8)>`RXaTdpqTKdH^uI4P4Ukt;c8Y+e4?M{3A6(ve{!BuF@wxi?8$QV5 zZtj1w;a<3V;z1+TN?iKU8F+v4*P~mM!%s8z-;TSE1+u#it5ABlhSGmu_c^@DSh~@V z`zA?EdG1>L<(ej{cA4R4J&!*of^5yg<~oOYdgg0k zpE1gs*MIef36$Jt?>A5;NCtcscM|-(Y47C35PY}x8;#ZiXQ`U5m!ev-P<(m;{oS3r z5w}(A#%;pMML$U?N(d=y6I5p|Q=9BFdWhYmVKQ*+nT5gmS;?SH1t}!gn*`D149_?) zbRrRN-0f1+W6$xxp+`TuWr>D+J|Mj>7{=U!K>KX+M+8_u+^vp2%fiDo@^$ECZdMyZvZVsNH$(-9T zw2v)21s4j?tC2)mNELf4Bxy18YiGv9B*d6dLnvaQXdFz5SWwhwuqMl#0tzHV9%IVa z<&c!KsZr5}hmiJ9d%E~>x~sA?h$=~$Vu6?*Q0BW8}8YQupZB5stuXn@9F*T z9cV)Yf3K;%eMzV<`}P%Rdo%u@wqi?aSi>01D9`L8rqR(C=9>QedHnVDyh7i0$2iK{ zu9~INAQn(dW1K=o_IOVG^Gl~X8X2(mNF0R%eIL0#_PdxhUK{fd#buBQI07uk8!od` zMAu{)UA=a|wt4F*nyYr6gKAN1&ZOUWvW#1A^;HTbi=PKQ>Y;vGq%>zO9UQZoURrHl;?h~(cs`e8DFHa0g;y`$$3`IB~QgCC9`f}Q~X(0^TeNAA_0vdF7d ze}8GK+-3Q6*HRAH1JiVW0(;6n@4-fN%%Vb(p;ES{=kg^7nI8|8DXSb~t9EP57u{no zrvjz5lck@WZ=hVysj*(|Q-CZ_EOGbkt5hw~qVrfn8Y(U)WYQ?4kk}5aOn{`3Gv(w3 zhN>8Fa4wp_Wvh6c|72_64H$l27GKJWnV8zL4ErVX$@c=5dtV!%{=m~h-W+;4es1RA zy)#hf+jErS!KK>tNp<~SaT^UIV$Kl-qx?!@K4XtE9V#Mb9qZ& z_j`0LUJAWQ;Z%1luZaA5T!|rFzQkp?&XoIey?N!E4^Eg1r|+M{Q~#$6@za_c`U}qD zlGI6gqAr>7#SE(bUvr@ho!YNDr)TrHZC$_UV0z$SOJt5~l3AFF3JjBLDIx!!uPx!( zbz>?jnYeq%bMDmddd>~vz^+gRivR3M65euWrqZPY-#NZ6Z-1C{qrJD*7kn;n{rD_NUO6PG0B z=fV#QdJffOPIHgsO`%DJJ9mSI-;t?FLjLX6>D=n_a24f?>KNhmDG`GQNKw3kkiUIp}MFhLv#iZr-~eG=Dcalw*kR3QH&}qgFUc>yY?`Esbx_ zu>_ZJhz=%uzkg1v5coiHw%QkG!ujj3>RQ0ITKFF>3Ygh18`lm*D;rDCBh~qd zG_CQK>a7%&;dOc=?vRFOV~nKOqag{DF|e0JsYOf&DMT>7Xo|JSo}O*S z*c}=Y`Jf?%4My47*}aCP6z?(a=9GveHd@c_p0X!x^5D{Yv!=0{teM5=&ApJs0)D)u z>hCN3JUq@bfRdDoJK)g#bhq5gl>!V@0gKoF<6Fh!EtHLVJXCJ|hlMe=vxb%!W$wnz z4JkKvt*Z@HKZ@nXq~{#m|8P~Mc(5S{l%R09QVprF$ZB-XOF+kTqel8INV5*DT#GmE z`0ERk%9S4d4IqS?ZMFXt&_gGkT#c>Xkz?s;ZhO-J|Vqx@)mlr)qZ6 z=&AL;R?eRcMJ01JZPSB^4%DuJ9O!!|w- z$u8s~%g5#7mBBeS)jEmncedR`bj7f6vf@Uj(;S3yap~$LS{&+HeMy3AD-lZDp51c| zX(-Y63_M+-n+9b=J*E;o#ZF3CFO!`GG`0DYi%@BzU6?ba4p1wU$W@MkbC;*b%KHNY zWCT~((dmv)um@Eh;{iq1d1|T1WuV5{ROu=8tig=i+rO5&bp#-}nt$vg<1oTcl)-&#gGH>XISfPrERxz)d;%39($Ut@rA z_p0^D5Qr(;0~rUa?ke#}$Da#~0WSd(&_{u*`<-ao%-gdwE+jkzJ47K4BQ+uh3$=dl zJ5;?$4vrh^Y5KifONn*5!){e}kj$mg?52yrUSeN@ zO~A<}1&`zU)xcj=a~vx(IvAJWt|e$tA=#UXaEF;PW^&j}wX+awFftcGp23?S^6d4t z3skW3BcN&vyxBS3)79H6lpXBn*T9HoB)iUFv~~1bKPk^3@n3=FlTEBlt zeC&Sjb8c!NO61IK{0dQELAav5xiR6#5O^P-e$mUWX62?36wF@QDjChD+vP)V8h7qI z^ghvLrRVw56$%o=tnF@~{jY*Q>NEfB?{i!5V^Qk?DY$|I+wE6lFIFn&zm%bdOw_jM zpl2UAl=L9M| zlsC2nS0uh-N^U6nJyJoF)nI!5-;LOQ(TKPgpe9aD^|IMk?lHx2&o&;(Id#F(2EWX} zWLSUP|LPSgcmke!&(ov7q$J?i&JTXPoesKs=?MD!@1GN*5o0mS$#LsfYKqM?SJ*5} zm&+?xo9y&e#@?nFC}d2|)_tDs`;Qot=jqIZ!Kc_?l?ek#KEcyF{(&e`h+8TuVH$CJ zi~|PPhAGcM9Xmnirp}altDT7%-{nflKyTA9Djh|6yfy&`a@99I@Xpo#P;T_8^vhk9 z<7s_HKPgrWI7#{uK^Y|Ge#QA4BHj+Vmn~t?sk$mC%^=t=sq(2(eD)K`jjJi1S$o8& zp6QCU`0k*~>=~u*JU0KM%mJP)h8wA6##2D{8-MfPZ=@^#cJ0+~{L|`0U(*(kyli(6 z=lt}p`F%HJOfKBiwr%v@l@_U2=OZ4K{y7u;ZM%uqP~(x;c2~&hEj1-~;`V19TURW! zFWMo`WNF>+5$N*UPrKx#2 zS}2NU3`ko*9zRYF6pmgv=#NzZdT#ejl+5G$a|LN=;Bt97Bom~}Dm1+Fc*Z=*aKeDG zPu>-ls{|pa(Oc0KNzCZ5^ruak+zh zTKrDL7!cdu_f@l(8L-dB2jWq6z-{36_s+*iD?;eY3QDW_^V@_NxefM4Fob09(?uY3 z(X!-CRYpwa^`_xA=ilfWxaI+bEpbalvqJn$OJ?Agf)%d^KR5Ca3)XHwMR+YiFWqw} z3Z15x&G?TH{oDnnJn@4bTCVPPSx`ueQZ?S4DYM^{PFiM@n7FO zYKZ;n6x#Dp{j~g}#Kx+k-I<~ARRfyLEAHTazGA_IgQM=#o7|7hZC3m!r|Hw9V5I*q^aPIJuoEDp)2mwgs~DU6;QSjB^ULf7FmCp3M;Trl0s?Y zrX$yxg^3hB#qs?epfv zrViXJQA5b@+u}D)%cx2Wi0y*KG{DGIH)>8y@wad41!v4Z5M?};O$4nnU;D(zjBav` zyjp$wVfKhL!0iFJroH#T>-y<&o8dbU68n!xk+gdQAe(xEpB@2*TTB)6G}V@brJD=mrft3m&tIjF zr{FdSsC|(ql7o`~!$9DBVu5vmFv;$9)&0e1i%+V818NRTffbDw_q4>=?R&lUhWh&X z{Vk^;YQ8n(=;glWyEyV`8sreb{}i+DYnrLiJ-<|^y*wVZ?#$?!k5f~P|JEk@rXB~1 zqBn>3U8$vvsGSbenD0|Uu=blO(}HRLiMx-g{fz-QW-zlder^G~RvK-6svb9IBD<$` z{CBEvGYDerlB-X}Kj}4rBV$6i8F6mVYiOHiW|SNClbyN+@AqC74Y{p79@}a?d`j2}9&KJe z;CexkiTt3g#HPg+Sr$*(n2xZ{`^%qH3ViaV1t!Y?+_QU&mH4-$K#IKvP6Bws=1cI zZ48$JfmQ;|W*ak?e`#G-nwRreH@@{8D$N_2)oj~Ox8`Y`rms*qy{Ji?Na9BEd&T1< z^RjtNVmTts;n{E1gTh{8(^EZ3Sn-R3|2M~)G}4POl9^?O--F7zLv<*#)zI>efeZ7hf5)7#vZCWzbLyOT#TQD} zf(lwPZLb|e zTcIuUKnrGzux6?`6`CdQNB$t`2t)x=LFx?vLhGU4R@{6%Q;Ih%L`wAS~Y!I zHOyUucHPKkz>S&1_KOeKWZ=puzthIQzUy8bUvsuvIeVs?g;O~K(i{3XIr)3eHiNmg z@Zs@J*{;s#QtI`=K9${<=O9W`RK0)&TCZE00q0jy3r@W1w?!FEk}sT>X+ctPWu(U) z^2}f!BE$Ngi3yC5h>&C!%%M8%7ivs6#L&s&kRmoYlIy_IiuA=WVr=swAd#ibES#%+?bt^g*(C9zxBP1wH|rck>(wv?o@| z|NQ+hz}2wNmH}U1x59z-!|dTm{3Y7R?Z1B)7XU2_h>V#e{Mp~f{6RR9I_@N#=h4yL zJ}NSO-qzH>);N1`-v86;T$}mazf~ARpyWdn)GW?}*#+duB%?@L%2E~qm5C{2z_w&3 z{!qqV#EML{)nRScN-cDVQO2*xcdIT#DcoV}0oR66D!{XdEpJA980 zci!rg7O%-?=!|_YHt#A)g`vVMpVg+}Ba}8F5KS!UWGPv2U;=$aV@1)2DLBa5z`cS^ zHbKY73SI@#!0(iHY@Y?f+Qx7i8cCQXwO=l88ZKy%GU@F=$b2TbkLY-SXomg9yTIU+ zs30LP#1QYKO>y~g^h059XS*ad{_}Uk{ndKgwwR3y6@nzKgXMo@1gQ+!R0H!!Edio{ zyYsVpz{RS~aNxwG3XD*61E*a6gIwUH07o=nHC@carQfqJAH>#SZWbCMj|CM}Zus!V z-%G^BT%XT1*K87o4BBP8m;%ju*4H5))2B>uH07-SQZM`wpj{$`LgVHa8@_49n=5mxtFC2|yhK#UEdrFyXb`15CW z-_d5{^hF6hXHrcG{I&ny1K5AHige>)|EjDR=!+xlmJA5bKSiqz9S=4VvXet{%0r$1 z%=qjbdh(5@4SQ0y`d=QUJ@7Oo|2`!$I$N7tA}n7^n2RV8bgs%}ykvnH;AjgBw9F)C zmXM;x0W@nG-9`FzF0# zl1an&Syin-n_A@hDU1vAo0Xep|4wKJ`!#>_f2P@@9^#Mo7`DJEfOFN{qpIYdeY>K7vpV);lSt)iH13dXsOoM|sk|5%32{tRrO zq?BcV_B#HYT`qt64a&+Xe&PGLOl^3?-DXZ>+yFU2biYe)DfvgbYjIFP#ielem?FoG zCf6etoCCSXKWWb=1tXA{NyvK`imJw+u9oR#HMN z;2%(0{r5H_ADJ3-vlD%A@cz7MkC#(Z5gjB~j2n339sS8gXiCe(*T>Hf7%7p@J2g9B z52FohC;=IPeSHaP{7poSh%zf!N_F6Q8RGNXGB)%2UJ!|5#?aRgB!_MPj}Lqtx9q+S zZpKyJ-x0r_fD%)(Nv*6mQaTzK(hc6>EsaiYT4&eZLEkoo*r!cdEwp8=p7T%XWMEe_ z70Y>d^Bu0;D`^cC%gq_zhspF0CuB9lqPDH52d8P;_%$h{aC&Co;-*#?0{Kq0>Dxb* z^X9Jnor^zRbw8E{yP)V%x>ht7+xNE8`WcH=*>FWQE12qKE!#<-l|-=tU}FTLv7DL82I6?M=Ei+DvXq+r+sJ3e)M0Cs_@;X05u zd>Gnt*_j-pfh}Z{^}u#ay{~I(Pd2k`prcAxG-QOPn&r4n*NWEk`tt(pFlNw35rron zawPZhoO4Njzt8aRJX7rP+DC8B2SO>;UR>&GvPv*rBU?#oNV?y4L}(NxiC*RFX0*3_ z%+qL;8L_EN#D#FUZT7)z@t5%i{od!5x?qURPs?HI4&jr#$E$t7oWE{0iLAod_kBE< zd^*oLcMZN@FD-omT|Fgr8B`fBjZxL=x~(>U0VE#|4h~o+W9+_^ zAYLwutqiH~md=k}&59rcSDyRX%M3pFzgq5j#U$24QrVeg`hOv%kMRDqQZH61cR z&D$bYg25r_zF@6P@A6XEoYS~>5<(2!GHZ_W_V>!F7jcU!Y|R`>U2)u}=EVSRF{>&-65vx0P4khPz?n77C8)!Y zE}@{)3)z<&eOiOwpPm?Ep7Em-+VJ3X8Huz7 zIv{r`-r8XxBni1`)IcmhittQ6pWDJ_R&*YIuRolg3r*XT4b&e0S$LI^c>OZmvGkwq zfwhVKNWNK3g%L+(h917f>+`E1Cw;w``sNu6D~}MR0#35M9$7d$ER32nK}K|sz5p$T>79x5e(1vzF{}A@DDs!%^1J2os@yvzw@757zE=X`jVLNBZ zA<-2J)brv6NM}@}QX#nr8Dz%fsiP{=KATlhO>m4i)7KHmbxXMT=2bi_W^4l0I_w4A zFddS;qGSE^WVD~B=@IQ%@W-1kHr7x(8)6tL%gjxEg92%@10OT=Q$n%_GadZomF6O( z13+W?cj*iwu(bk!N?hQAy8ZOv%E9QF%?t%dl)tvR?*W6Fy}&EHs`e68y6^l+Tt!3V zofbUaE)vcJ(fJ3yVz~#tJ^o&`6}CASJdEX!Ap4Y0PPyIQz3Uq%i~T%(l#@>JIL1fM zZbnnqGKw(b+)1Rl^{zrlMFPJVQJ-J08BaWH_24_IXs?KkY-8(zq8(#h`>}HV{IM>i2(c{d5+@t@ zj&JrHC8RsT?$i0qCyP!S-#K{udEM=LX&dDB4SxRF=rNHpRMwh+M*)#8XGzv?a?@S3 zS1Q2JQHa!pVu3;Yr5`Jbu!+!%S0Z~F_km^{UR+rMJH=xUHJh9ee`a)M;t)#g=N~fL zsMAAu2^lbo%HN-Qb9?Ol+^gr6xI;e$GPG5RV;Fj(Ws*vc<&aVY&Jd)X2ea#F zrAFYaC@^Nh&qKrL&7KHSOi{3DPCqw-^M{nx-F5OSGS_lSDAsD&7#uwgHC6t?v?yru zA-+#n(V=yH^@sxhZf%VKPr!X@^e|3}AuUVlWBYgHQsQV)SRXyi68E_jCg>}Z@i3m& zd5-%O?-lQbZt1nBXm{ZK5+!!U5rDswFJb2JKO{%vgHIef_3uyctJ{@4<+de*7TA&c zyhLHrfH`ym~X>Crns@q{Z->%9H)_d(jbeEgD62R8>-g88Yzq$aq*8|dMbtIfYI z1$I^#0)+I zihtf1mp&q7VUS$pf*$ibFvm?6fkgM4zp?skMe#7O(vDdXS3wE>iJcof!k~^VBz{j5IHnQ-p~ttJ^I$h;!E6sQyK8Lwn-D; zs#rF2(r1ZHDj53PgHnhvobz&b_w&=CQ$b`rDMi^*7$*0fes3!uA*Wx{Ua2bf0cP;O z1}bXgW_U+|v;k&I#ng&Bq2K7EOU@DHywOOwS3n0T^qLio4V#cYWG_u;fL0-NomB+C z!wkx{{G~!*JWUx?GgjhCN4VsqOXVCGe!|IrgLLFHrOH~q^#F`mAT{dK$7c3l++GHR zOB0JHQO4&j2|Mkrc#hKSW(ZoIs!JM&qVk+hhVQI6A(wJ8w0MPtC}rkQZ3~b*L|Yy& zG>V);(Ml$==H>mCrN=^w?8$XtaOXBa%)jN&o8aPV&qOfT4C|PCw)}Gp>htE6cU;PTd3uZKs zan@ejboyj_L$f;cgDKbN$JC^%If39~HyJ^0uz|X9-3udDaNpQ3tCM@A;rQv!mN0TY z#Nb2Eu!R+hfj{EZOoO+Y@i9pSZYZ@dLFq0`4UigKTH_0! zfs&E^LuQXZ%=9}eU3(wWvk0QmUQ+#f$#JZn@4kjxz-~P?0Q@mWkV!IkCg)H>_85jd(`#qsPy45okV zCFPQ0IW6uD4H{HxI7{mt9$*<$yBWKT-xEXaoDUt;>F6qyfWa1nyUdsfL&d{biAb)x z1g4YHIYIvn4JmCfYi!Jiu*sB|OiOTKsvMIQHff`a=@<_#GY*xH!P~ds|Jd@=0o8_g zzCi(8Ov;Lege)JtbzjpRh(tg-z;HQwMb@GUN(;qRnByI^kNd8VXL`Loz5f+vo!&q0pR&J6T+r8>I%v!>4WC@b2umbTIsF-zlb;tfMDJ@h2J2B2Jl+m}*wihMOTmMiF(zLb zIS=03Qa{$7KI<$LAZ~fTw!-@$?7S6EGYMl)9xk7Bw7VfV4(*MA2xqp;k=E1-I-^&7 z8BR8#(&;;EwUf(dF#U1o8U@5(HV^l>*_yAqX$_&X&1y52(w!rZfn=W}DI)xCDgAcg zt)m=8GE|>J6(*D?Kpw+68EOfZ>f=^(s4(4i73~Ga5(TiKlrrcins?zgp6v4Ab9z|C zUJ+sJ-R4YWS)7zGK#!w!aRJ=)>an1*pEdE*R!N~zCNE%2j@AHn5|0i3ePol%+vMjx zBrAPP*RXSXB_Y6LGC@PB{)5EFU;LcnAxcD<=pg5oK>w^S?|6mz%Ttz7Ly4laygIpc zAB-I##qqy`Ts)?jkiK~dn3lK+x^vvJ&qFHKgad^V)jyH5H>Y>VEf|cwCqx4+OkjpE z!Rmg!Qy3uzJVDVw$QX&g zwi*E|t^DnO82kGxP8iJ0ns1n?KNNve`$^)B(UR=1t9sI<4IM|mo-{w*=9{pz^5Nux z&{>vYo`an~L^X+V)$lUCjvxb>w;pyj&Q8|;&fN};oo4gCy-fA!$Mx<>2&|W67Z~_aMXOpWyub2O>%#34VHdsw3o(|F8RziHtb&pw_3> zTteMgou!&^??(w<=ZG1XQ>+Vr>KKba3nJyMul4*xtacN_bJp2SvPDB|Yh36xz9frE zifZgs0q^#GyP0S7&mV&DEBC!!937)~DvK-5JG!+yzdf#8`I}~xdt|y$kZ9jvf*g*> zTi+Ks@D~ilrXu*w^G@8nEJI-HaKolX=|P80r8eeM0?8X8>Y~K>5Xze90T0S!Tazjw*xViy z7x=6}OyEzdE#m<4RIl!&*ap{p=^Pc#EC14>y<=3BMKgZ4N4SPWaks7;;=x+5^el~g zyzSq%WkmTg1$p^#6uD^ppEstcZ#!#fqv+i~pjKl!CQr(Ru|zq`Q z5piB?x_u9IXmAK8Ybf~QVdgUCX1~-d+j)~~Ik9V~)U6h17|7Cg@xDs*T6k~aAAv`q zWa?C1Cn%T7JX9}Nyd;Z;*XAn5j|Vup=z9dH_&Pz6|7~Xtbe*&`y`$m^__;Ia6T+-b z`9?FI1+mHN+(HLOlPSlgPPvSe%FgXV-4c^jqk_BZ}i5kdsKabqG12B^tpS&%9=L%wQp=j}l4!Luoo z4O*ZTW_Uf={U^|ib@BTmRSrcnmLVGQGe8XsK*@#ArQ9N_49;b6>t@hqnU;Pwq!qx5 z^p>36!~zeM%7wF5Ym3Wek1up92`jGIeqU}NVl{q?!u6}gJG2E5_p&JWJ*I~atV|IC`yo?wX;OPc?C-E_3 z$!{uEc^RhZIta9299HPg(WpsgWmp*_)iNx+5>HM$_g?2zo}L(%-wvL)x7YAeQvMjx z?P5ws=u+T4rsr{=p^(iyqBmsNyG(-?HrTJBS^Z;ZA8FO58@RyHi>%*=RE3E7JjmKo zQ{+^SEE~e3K`G3my8$#G(?urTVoc1nCFnmQhdc|U0L&Od$>8ADA`cxsMu}^?I}hlB zuB-)k%%=|zzS-)#xAVK_yN4nf{JF%jP)$e!wlaz#_HhKPhbSh5`4Y5~@=Pir0=w*6 zLHnxZebgrvq<64U$qJX3@?ugc2vq05(jxK?xnFfhZWyNn)jC!RD(rbhbmtVV0{SPkB+l6FqsP$;^sm_DF4S)-&l)8U`g?Ud`?PCOQRrtkyCuvy$JMM zrhj?7xha!_)Z@^*gXtBxSf?Rhyq+7$$9qu>^Dcb1+u95RX72Ct=Q$z1+qBVprg8(f^Al;O<3BItyOhq0hxSjbqSyjr6!ia5LyW0^Ke^BUxgD&09vWP7 z^zJIWiNiR1{nh$;xZVj$wt41NqMwb)U#hjBVzuN?+b!4k25Itb;lUv_A%G`~#@ho` zt1&*qJAQ?FcS+=>&_nkcPZgQPbI=EFlZHB|_ZkMjQgbk1PT%k6eR$rLr;=-HaNrgp z@c|OwF3GV9ms0|xFe(>{O_jQ(=QOdK(27DPM;&Nzk?X9aX$GF6lKFYVsf-9l)oJxz zv5Zvg_q0(Df8KhC;dVX*_4=<#<`40bF)`z`NfU!mP-UW+aO=5D${UVkzB^8;07X<8 z4@{gWO|)h8vMr81LBH6X>}6k^-cB@&`Bkd5f3S(NfbwcL=%VAVRu5J^=hU5uT+81o zfK>13@s8$87wRkPfZ#a?6fF9NRbQDt$~mlQxf)BO`N%x_Z>dy&yuQdX`U$e;XQ_$@v+OQpTh@j_S<9aWD; zaqQFpw>O$nvFS#z{9E6vKYzksy)aUap;($6bawj)^u>uBY+97xH?88tr#WH#9-JAY z5}I(Q2`o3lZZF}FhBk^TSYm#aD?CJc5<3cOFH8w1T^?2R*A5-tB_5 z;mis4=W|!~ZD*jGM0AVx&gBu8q2`9OE-qMMJf}=^-C2<+*Z$gI2o4BUi5yf6-b+z7 z{O$~i!u@c71*Ve%@4TIVzv}~Z1gMw(2D+Uzv9y%3n_2d9@~0TE@i$GebMsy&9s=sb z;+tPCXD+hKo771I$y5IW^FR#0)2tQ{twjY@Pz3-D+_6HN6(3er;MJzo)qsY4w$$I! zVY{jpJ+>hg-xrs>u4R8;)%Kjb>eIaRl2rt5NG*b-3Kt;I8Xk8}m#%NxnCZCjSHb!+ z@2JxxnvqaYmNQ>?o^AWg%7KlB=MR+Xp*5of)f#F!O(MLdXK1FQ3V?u%obC&278tvX zd6+`U%xNy5BXqq{ojHo#++-kSwH{s2zMO6jwhFqa`^(<-ea}cGgi?x%=6Q(|yT8Am zrs;e>+ha{;_X4*rfLdr%N~tYbyt!Og_I`F^$Z(`teK0c!`978dMsiMG|?RTXJ9 z5(g`j9{kBJy+vB$<-*O)jiZS_+UfNg*Mg$eOJgbk6zq_I~JwG|ThzC(RJN5V0{}!Wb&aUH$6K;#4~f%{KiW5aqqF)Z zZfM>&+L7I5_WII{Yu>-Sea)8aY0PY~MzXdax;PJw0%5ly1Rn+w5wR&WeHXKeh#)TJ zKFe{7ow}`{orniJI_0!CwCh$%Q4p()rj~=dpaVDyd z<0VYCl#=C=licD}i^$MeKdfW5gO3A(`?{%mdTct@^R_+IwqjO$av4MQSF3s>go_$v zkDaE8FBPgU|LQaoRDhUT_Gs!}2*ErSYSPfMUDO%E0$L*|Sd?JEB#3I27Z-b^)ghL) zJd-`phGMxDmP=?Q2|*RGX$IIX0j>1z8)%tSPe32|u$f{4~>PC;|YRuh~F3#tI9fGBhU5DA!oS7Y|+EbY%d z#sjdFl2vmPO3#Qz)uLFI?FXQ`zDD1$WP6ufiq?Yth)5w|dg%0*BM{h)A#Q-i0k;5~f*8DuRd*Fe#mjjPuyX z*mW!t=UGe1DHXFyl`^YlBK4rbWz+M@0w!W8p{fNy5!LKz6=zm*usg}1x*J`JatN8G z7i<^+;BGl-f)xSa>FJ5;)dHPEJl7*{FCEJW1#reoS%wxFH^7RG-8{(3j^X3;3=4SOz{hyQn7{@ZpDw5j*fjux7DVQktc zBjQEDtru0>88q>c?MpW+<{Q+$!gqn2-HrmAQvCGvbUvR2A%x)b zSJk$kq|XuRY4Is+_HNOd^T0fvAkJV3J+T!+&ElHc7$;yfQDrwG646UJ+h!&O264`Kp2ErO*AqoH{+nJne zg~o8{21O~!JkKf5U07mTNEf=YDlr-;#@NNq+s3x!$IJ?lvy@WIXBfj$L(zlNZ2Q_y zIE!Bz092>**@2ZBXT~*9_+!gDMYA;Vh@0#Bbn8`2dRd1vww`(aiPLIJy7o`BcEC7J za+4fKjs4JTOm67e?sxxL@kU)?IW<9YX)PuIfC>aifQ}+mODqvlpcizs_H?x;WJ8qg zui7o&fK>ZovsFGZ-(9bltuGyQ|0n-dJ1SiBoAx=I!Czmx<_G?KkJ8)hM7vwu)TmKd z)#LU{nrKvixYlI+mWMAJ83HviPELqUD4Q+w5E_7`JB-~NntIa!P>t6}|t z;SZpcq>STyKA%-E_B|2xeLtlVV+>uKQy$0D!@~nJ-~Qws5m_61De!bYA>w|2C~IZQ z=}McTq1Ri`8Wt9=R@K**G3Z+7TM-Gb3@rYEYo^$naMF6b0;kHn1l8swZagsn>0@Yw$Z>mMr)hpJ=WOJLY#Zv$vUfB}-QC69<-S<)p z5yKj(OHHGN#`=937+ys63q&JgEqCE+r7d!^r1Q0|ty0sbnK|;MT!DxHif>(g+QpZs zs>EfQrg0p{aVDbSaJad--3Xhv&EI^JQt~Yq zFjJkX^%w!9tBx+P=2EU@E*uD{6&n0zuPOm+5a+8@9cs*@ux2x`&RlDSboZ}p=%I?z z%kC4=(lB<}4HHl<)uX&D^G1`da?Gl_MkyswP*ou8VvLcAAzAJ>FUW3~z62D8;4(Wi zW6h^UU23TcuBwV7uDXOfO%jpkLJL{dhZsU&LLpogw_vuCHSW+NHWJ3Z3tiVCgxzlE z-5R{KgG2Lb%T#06*#|&`8jO^i)~3+Zry?S29qya!`{iCr*WRv?r$u(IMfcj{2>^nZ zz1BvaXz@^LKxDDfoQ|5DXQwG;tRV=W;Bl^4naib$JhLrsyj zR_N5)q9CPY-R$7X6nw7M8y~HoJO6V-OFwqlVU0ZX!)0eTByAx^?CVd zNmm8tN-ajeM2N1H3=v@r0HC13Ma&D`T6@+d;ScP8fGS!L5M^aL_aU~jaEtgR%UmNn zHwGyhrQcYbDR2w)^aq71g*vyz$XNGE|mp1Z_$8sQ|)L3jA3{oxW zUOR*kH3j7XU_K!#q9O$XNhzPk@qC^D=;`@6rF=LXcKag$h+I;op3i6g_^}iP0FD6> z1=JR+s8SPUSUr|67XtFBQ+24!H7LS`#1`w(-OEG>fB>b^)#YEW>T$OJy3tLYq`l6& zQ=lEZjyqMY!45@4tb#u@*^eU2ab;$UBxzOW4X1Bqb|T8z_KpCcfTC4#rE0I^$_t5r zF2!RU*&t9iim-6Tf+zeL1EGQkq8L^4eg;m zXakFzb@?$&Y;m*2TqgxpHDveEw@U^MbhaPbH?~XJjt3hFt{tTR7uVAIpPmRb4AxKD z^7F>x)Q*tW9%-MQ=%qL?RAOq4-Tw0hd8Ig_VQY_Trs*r}zqJP%^(@AtbM?vIBUV@^wlHUY>Y&(F^( zCFW>wem7*rdV4ZIekuEpM#)UUA^8ZlI> zN8J}^^Q&vHLPX}Es}_$JU2|c#6^mL7!RpfY3}Lw99JIDiTmmQB!=qQ7H(tcJ_O!dl zz^`^WHbC4WkzIafa_!{1Ije0wVyE@ZL2Ad@uB?xz1t=+Tw}+V^BLeRz7h zySpo{pb#usDC=0GmVW4^Oc1D0?u+IG))Hj9#a_XxU1UTwJF=|RWTzirVW*r{I)TRQ zR;$0JQqz&lDAML1NNu=GL;#ncw^Oz{Bu>myEot4bLuQH=*IWdYkeE3O*n|aEz~s|p zX7D&SSzu9%xHt?AZ26{(h)5CZsUadFTr&?uL`A8V{c_81eGY^Wh|L7gAp|S_Hl9!C z^BEEM`@JJ35%qoV><~>+2!WX`{M;v^m2rCiQn%088Wt(76z{)nG4cAaZ@?WYFYzPn z_|r;AJ}UVUXCxwqupGVCc;Hm$uk)WuS+ zko5-eqXmub>9HBq=<*_B$z8tpjw89Yr-I6nZ0N1M5~|wOe&a7KOC7is3i~}R^tZcM zuCnj0H=zCbHKzCtwp=Z%#I=Oge!szYpZGj4BxtVf(6p@bFWaGVtWIxL1fVL^mRexo zB3>aPqQxEm|2zAeZpm?7TNDF8DPpb6+It`S-j?nEcDt|P_G|d}dUy;s?mhOY%3Kj4 z5pV;-)Fc(F&Ux0TkrgXKqDXv9fFKAarrhSZl6DY=I8Z(Q`M{@@Or(z9*-2B-_OU=ZnxWQxvl9( z_o_pkXCQbjlzn`Be0+THJ_hjA%tEv+bu31Bc&Mr^Z6ifG1i%jD4N4+Ho%h21293%# zGgohtoVe__fHU#?&)+h}`Vxt9AO|?M;|QmSP;C$oUgB2DuZNeH_w4(=?Jp70Caq~} zZ4VFE<+f_8bVgHKT3Z)HHGA)w8~#LiTsB(@?>(l!vdcuZ-}( zw6E)Wzu&OwB$&m!yOsxAS6m2A@Uty%pfH2fEN={Id*LvLE8+-uB%cy3h-sC4HS4c z+Zm(UWBCGf0!A;}$rTSD*OdCDBmk~Pm3N}z#Szi_lNAh>hVx<3c&|;&HqJ$&<^()T z$bZ+{P{;*+n|yP0KuHwj@|iu#{01vOe7{o;kBr#%jR-Gw=e`|=6W=ekEK9Q{cCdrjADD?k z5g3+Fuvo{cr{Gt+AyFlZ&?dxDW@yRytPfxY>q1u?!O;lCfoo zs8?v4IfeW2)7H0r-=lZckY;9f|M+oq($toQ44GZdI-n-UtMj-FrX*G=`rfZlC3>#a z$8kcTw$@aR$oQFlQ(?Ww@Ds3OfJpBcnFNTUlLO4Ou-aWG{fO*A8yk1;PsE<81L zQpu>qc*(o;8=;vsE(p`aGyeB_;nzvRa^V?mULR-biAWYxL?^MT^PL=vol__w@cOrZ z`?qV*eA~9Dw8fwdnRFb0>~J^SeYM8g1sIAr)2)nhzf7CRA=55LDM3+1bN+xO2pr{971tfJIg&wBG*;R*=j;u;Re6Dla(tHo!eBbx(o9iI}nWrSy)YfIW zx%(b&a%N{ngpV5u(HR~fj);Ozwbs-mIxPAj)yuLpTMuY2_s*-Z!MfhFwUBK?#rS`x%E2vg<4i zXfPsov7YrISc)KfywVE`hcD5$oL@qJk0`6%^Pa_huZekX0XTN~%TdlvMTR3r#ha3g z$6A3&B2z&tIZ5i2rR*pcMu5K)vQK=STwSR2Rbx{wd+mC-f*@b`l5-m%UX$p^Xj-{@ zQ-fFAFtu|Xqtg`;h_A7Th{&F&CLQZ=dgQ}|KmPhNdA}@6@_Xz1b9>#Y*r!vJACi4? zwS;D&oe>e?b>e7iS{i$F`#}H9fF}EeM!Uq4P3cLBwJLCli0PtB?t^XJ{U|M|)I<&3 z$51a;M0HFc$-<=(-Xv=dcs_3-S$$Vk%f=k7t+s$BEGKv)3nVjfVk>){VCcT@{itIT zz~#Q9Mi|D)hT*jCz|*Ty!94iY&ujCp7G~xW&*w8#+IqXK-Mx0=*QtI-Kl@O3EFv;` zzSG6|^=Z{bnufGxO-AGjYNGAZRxeaVue>JNz4vY3JT$j8skpoUpZ@KW2PhTkWwJA! zm$4lAL?t*Vif!|YSjBidomJG%J=HP3`pr%>{2DRf^5i|Rtm02dQQ4cb<~BO4RK5a? z&XJ;q%wA`PM6E}{IW7QKGh6P5&0EAnBvhrTF+rg@=&o0SGlMwn6@U-#&JX!|Opah7 zd8Wfu;}uS(>2-$@6CVTxF2l8fzD**6&kzf4_Pin9=hs8Qm|_v(ei|pfc^WT|B`yhA zc8T5NN~dh1n~41J#~+DzA0Hn$0(jfD|M&m%|ApNjN|Bk2InbQ?OpBD^I@;ehl!4ChlB8DPJL*g)e8RcR&^Cq4cBZ+VqP}5G7b%32uvK@&pYN= zBvgilQ#{KV0c>J@Tts0moY+lQ?70W>jEW-8o^UTBnc0~Q=E>Um_ND6XVg@i_0Xpn@ zZkg%f=F+Sg4v$^e^zX2ilVZ@f0&cI2y4r)9ClKMZ3Ldqpn%cUoM~GnF&!v7$sWK-G-{&;@P#2-spD7l8cckyZg4=|Pa8Kpp> zyRYkw9c>8NKAvQa8!!eXUjxx48511c9ngcfqu=}uG_bXSZCHq4-DBp#2=NUOG3^eA zc|@^*2`oTg26vKAkr7$YVZLxbZUCC_yzvYt7=ALRCU{qlG%AiJ?;?-ZiyU8>0SB}WIx?fKMEr_Wq(!)8uL)quww_Q zsJYZ{+esM~EwBR=drM>p9=rMcYxjA634${aV93+=LJ)&L)7w4VRiBL|brO1U8vA^l zXKYSO?vpIJyPLWmyl~e&T)On$?c>P#sne5&umF3Az4zFA3QWEl#Sy0vXw=*+NMYl! zG`;sOv0Gc2Q*0NPWm&9c@>fC#V89dmUQ@Z#>T{mdX} zwMQiJD(J%W!YW&EQow1{AZa2y?n&#iS8*XFUnH-pqw%@gy=rb- zCH{j!ppqSGrVB4S@Gwxq*edMhv~&sJV8);w?kGZ{;2iP|&?XEQrWGvc`I!wDvjpb^ z24OC4V@zkc(GmTbrjWK$^Xhk#97jx&q^9io$n*8|I`$o7F|+>efMR#gWV3}%j_sAC$VdV?d{sSn zDZoK!OE0h<5jlHX=;SOJ=k$;SC>IQ*(8f%uwCQ77R+da9w;$DsCs6O8gmKmxqh(*< zGOGd7l^9IXPOAf(KuX>zAY`T{9K<)%PP54>;KDuoc&;fU0xLWoVfO@gM#Q#185htW zG=*X2hc=#b$Zaa8pcH$auc3YT4jD3Tl@yHpfmfW8q~sQTeSL9H+F3{9aKWO^M~A71 zixG{8MsfDuyT>urUDrbcRgb7nrrz(bP_63oHUCI?@FSArBH~q9EWQqEzzL6t3@&Fe zHB~j*e=8#hV!SaLR3Ll5AhMA5o(Kg;xM~(MUnU+wHJG7bG2{VV8icv$>vs6vR$0Fk zXXk1Ak++v|l6~Kq`{)U#$X`b_i<-0A;c4bBq5J)wAC@DBe9(8TV>uVKy$&7aU(Jll za-ZXk2;^vG4e$eBY_X~y@%j1VZmEG609sY60}MQO$RR3l;VueO%9@#FKhNDY6_#m6vN}s=dlvj z#~G?&qJFQDKvR4r=ow^JSU94%7i|_H&0tW(5hh4`BTJzE67sp#!)FRB%em zwQ*)GE_t&Wv#yJmrssZjy`M%xO8kfW5=bs|hnkBG@qbo2?;XK*+`yskc}kPV+q8}LId&2*4okzw!U63bWB(f)o%rr!$nMP}-5z!<1zM(TcJWQASE!p=|ecKLj+@p3pwRLT`b@_2? zpW2tlSM)8StA4Eea(FzZi+I}jCSu{C+SYXq>DIix{AI6g>+!JIm%gu(VYpC_EDkH% zYwNuyNaeM(<-U9_x>&b(d6!L^_wD&q{nx+#)&2F{-PX@_z4hMH*4?+QDM?dzeUE2HHyEnF>v|!&knHu?#}!*1ypGT zEGg5-oH8|d_J;}4vMl>PoS}o6P1?e^OyrY3pf*B;S~<3jwgN;%vNOiz&=}WX*F3MIVqA;#(KP6i)UdhMD$HesHyYKsVASf~~qDbWiFa_LQt8@j7rO>R1`3OSw zI&%lk!J%n7oY#oQuhX?jz@p}tF}iruh(`H|x-H|>HLbqLAyfteuq?y&Z*G=l)hXfu zCCXO|8f8@RW(dpu<4M)RYm~-_rUDn(|+` zSX&TK=I%`cVSUEZ)zqeqTW9fjtNd=5Kp6vS*$v1aLTo&!x8jwp;jk_#K}nYLb`|Dd zqez9(4rAn`%UDh2e!sEHpW>CSI((2u8DQuGFGHyg$?Smj9JyRz7ZeDZX7Mnqjar+j zvSyw3SRQCKRz;36Ss1;-Tb4^TU3~}iSG~;-oz4o!9GUNr+L4d(<|vnCflT>G2gqun zfa)dp1KqG%Zpo)OQ91__}EDy`aMGLx~UV;!le0_ao3+PyOwNsio-aA1G`d}4J(hQAypBQStm_8NhrmrOK;WDfvYB?%U?9re4nABP;a)M(-@avf1kA&i;Bl1i ze2~8H(4Qc%29=?uY{ctbqdL?y=f0t}2|e+Bj0?gDK?JSfNh$fkP#w18 zqQNBcu_N?$yWj8k`~BnNqb)~dRWi#zBV#F_>b`G#A3WvVpZ!Rt-bG#2MLnWtco%m{ z9IwmuAr4vOxYl*WYCLkZMI`IVa6NadUP89PYoS~_I5gN>&opQ4gKI1V6d5I(9Arj- zpt+`xv2PZqv~!p%^xpUV)qB6+?~u&A3Nah_i0-kPOZk^=Jb^^U-BLm3VObWhEr;+J z&STpqaxj7T_0W7J+Y+@pH+4~{_C?-&bwZg;Q8hqVg;>F+SMaZ_{NL&Dq*45t8LZ?i z8>N4+b;;~2{wWveF_y$x_Jhz#yhudu_j}@Ur;4c)HMmPbjt2Oh+ZOU&7)$mXo4lvR zeH)`pcCL`05w>hO$7NJKhDD|Q7px}V5L9(oq{?ldzm$r8&8qoh~GtY}=-!dJ$d3LwX)}(~siM2_?2Hw`I8<(RlKUF_QqI zq`gU+HyIOAB7EPp?;k&ZoP)~zw4?6wz+bP8GD3kSp~<&a*1U}=1K$Y=6P!meL6(#l zxSrdxn9&h|-U`V~E^$K{Cx;cI;Hml5*_A?haP}1P?m61oXP<`30LR~;Pv{>(HS$#5>SwYs{l+5#ybmiV)sSB zREWhzjq<8y&t@734{hoLuFN)Ona}T$8|IYKif{6sL>Z+E?Zfw6ttO*lp$L?ur}>;;jxlg;+bS{t4|q$5HD})xq$(a=Z9I#B-X(^H2$vqOjcdaGdf0jd7t7Vs?m8X04nQ{V9CG-(21U$3L)8~_Mn7h~K89k%26 zx;b&e-{ong%)v!c@&vCAkya5@t93kBi^t=k1nbZ@OdF$O!8|SH#t%V?!(?*jjb;_- zWXaPU=7wCs1mY2QSDF1{Cht`wvZB+}I14-)IoQ!p_6f

JckYRYrF1j8TMVm_c74 zLkHjDxS(tKNFsYw|M774t&TfAL9e@Df#PE~^8SSx*JlT3X{fChf#Ya zTZ%~T{1n>IdssC&Q1WnWX%>jlA`e?Hdb z{^Q5b`{$4Kc3VyF_xtC^C(;lz=Jxsd>3&WJr5u#6*Xw}1W$C@=#--NUuV24Z^`HOz z=YRk2|J}^evH0=hhg1QEtZb7v|KlJ3`18*{nP`j~ARs#v6RLpwy>k|6v^t~a@pwR; zq5keJnnv`Dg61s2%=9dEd6pqddy~z2?V-J5W(kV@%lm%j!t(%s z{`|>41w!z)4AmXLlxf-LEsODLeRoaw1ZDCtO(3p5=H%de`zE?wjnBC^GRC+kx@A2Z zmK?1$i41&+h}4cytNI??Cd-djn4VvuV{jm^PHEaf>noq1pWAlC;!w~r<@tJHIX#C9 zX<_G-`MdACd!ka7c}RyE5~A+uBKFs7Z|x;v|MuVhb-Uee>v6`*zV9N(F(Yz;1v7)D zg-CaQ)!|~A(%Ads`Lu1z_L%bMplv&Suwxi^ab%n?gG4m+3ionTRV?^%v5v3pV?cHway{%%z}Sm5vQ!8*9_ zdbM_ALdXTaK^Mreyt}4pfqswa7z{r1>;BUR@d#6PYU*#knLe`_o6v{NEG|$955i&b zBdJ4KQaj}@Lsqb+njdg8&$M8f1odMNk4afHnVweB`75nhg7ZV3QHgj9d={nLl^Exb zc$_CLMzu2&Rn1H&5@Qsl1)Z57c`>Flg_|uruVRdzWCmo3akgs|C&bL`>{y&rE)V>g zR>upYPu9mJ4l@lCA9T!H<@@LG>x+w3DXv^zHQsG zq&I3F3FUi4uJcMc`6t)}jf2EO_VB0f=jTbJCrxL3Ks%reM;*CA8^7mALrdFaN zn_Yt8z4D`2NZ`bH?Z2(cV-$!{a=gk{I*?7$nmgrihH8ML32m zBte^-xT$>P*RNk$U0_+3JPq}T%v|gl=#!AqY+dZ!W5KY_(ROPgA|YLlLmZB?wU*m0 zLRFgCvMwL%{j=Tfwk&z>!qPe{%DUc{<3#mi_*o||$%vgeQt=riK{B371tDvu+mJfR z{t&CcVZhE*fYV5W{_*DO{5>-f-v|X=YaZDr>$ofn_Mm2ArF`$c?W}i@QVg?*whC>; zW39(*p6NC%rAY3EWvAe+Z4mGpi{+s9=LjWLO*Ho2V`_kv34 z)zmB8%pF6^?Gk4Pxu!QdMJM0(=Pf~UUFb?5qgt3Y#$gVRfD&M|HY=u-2kksumcge$ zw}_pxqtaZG1lI6tI3T*Z6e zGWx+0{MuQq=w1gn_+?ISm7E4G&O49O4GF=)Aa|dl^U%Rt)wWo!$R)MH!&(I0*!N9F z2#8txdT|DtSWS_ndvL$kvw&RfrWdEjzJRAa-h<*0@8OY><6f(l%+s^Qp zY?k?A`yT|v$d^KhMr+45Pg;3HnGl3QG;wa~WbmDWxlFM}DJU`e{!BW&BF~Pv(t&5S zKz2l9c=}i`8ju=RP07tMD*Z9FASC)a+O2AT2w6e}up|x|zI{9{{&hqKS_2-I5VDj0 z4=x*fl-D+40K5*L>%AxUKlEeYEfPAL!M-w2Au8*-e*9SWeSf`PeUH95K)lz%*10m7 z>!>GXj!pXK^4PxbW((Jp77>a3-b~NUkO(9uvSU3kS9ilyPebBa8Pme8nI@skRKz62 zd*AoWm(L6?c5z^UnRLkkr{K(0eu9@nMY0lrkO2l$aNERpRTbfSVaFOTiNRT3zv;O_ zH@lsxdcWT%eZT4g>o0l;;@1s??9&0BmPzO3lGAeb?Hh~2voo}*<381^&QETykEN1Xk~*C{z^x(L&?;{%MlW7z zt<}uj=efm1c3dTzJKxXa#W(m-^vh*g!iMQA3m(%dN73cXGE`Nx{CGT`&&R$W8>){J*pEu@ zSAP?$v$rhR+xymg?>3yv$hzNqhaunhz1>!=8fWZ-PpKSIsX;j1(<>OQmy>_2Qu5sW z_tzJz{Dd1mUQ(Vb%K#d5n%}TE{ihe-mU~bEu&t_XV`ZDgCa(!P;N#1fhmyb6LG*HL zQ^eE;X?932V_Aus_3+qW8|0!qI$lDMq5lVw9WRv;B|#Ci_EsyVc7o76DwiY1>mblr z?MTbLg6LfPChw3`E;YXWSiKz5Fq|S?OcQ+WwMzwX$w>m7;bY_4V32vxTDbxpCTHin z&g15&no!=(aF$hyAxj0X5U8rY&iw&N2~`z2Z2-BqDk4sTi-a7x%r?M_bvfr8OmLV? z7J~@^otD2cq%dg?BaVnjtw5UPehBFN=`UzwI+})oKttxd&@d{qg5RvTI*VmU*etH&$JbZtOBr91{ptFpXv#!o-UYvC* z7eHgNVRJQR!5_h~IN5iyscC~YX@yL&?P#(kj7ty($kl|aIXXu6H9|9p6cN3LyGFFv zDm0|sSc?@PrRIRn8wN^Wl_ynIG5g#8PTBYz+RU|QP*|uHD zOWJ(vTi>4B&!0a<#MCs@!pzOGUcB7gJ!q0)YSxAwybv2zT~t)NhHKbrD$QQqd+*D- z-qw5iAN$MG;a!$x+rljN{U~W7BJScI>ze9)DF3olbWsZKO*BHStGh*P?%j3Yym@ae z4TGtw+up@9;4FG9Gl)yHuyyUdzg|xZTh~=hw|%?+{1Fk)ejc9L`@V&TL^SK}`}Q(3 zyRG7_yC-r@^KKDA>Kg0zzVEH6#A0SD&0QtJ+N~`gVV`aN++1Ie#~;h`^YdT#f8MO= z^Or`1y1?)1uh;AO^?dxbJ$^OW{`jS2QF-VqNcvN-b3O zE%dkO-fZpr;(gzg>%NJIzg~}&ZFet6Kh*Yn`%F#mudVlPA+0TD*0;ThKE59A{`2Qg z3-N7V)a3Kyxo`fE`~BW->*jqASMRF2_r1T)<4w%Wn%&yzp<0NFwzf!Qu<+$k=6tEy$^i zala3)7(;|YwvL;UG1+T+(PT5G$N!5L&!F+RkM&Qu1N>4}D&%+PFLKPJ&SP`=*@zGs z9NQ@G3EtoIYPN;k)U+iB6GqN%U*(IdQZ^tAQe4h|DYB?n1=1NH&=r#`_2fkGC zIlciZr?hN1xzhYb@b37gPGQEl-jY$qgwD6Ln&_ru4kAytc{cO+a;V3INuJO6`ua*6 ze7_G4;50fOBew&t@sz?Sj;vHauC+x}*u~5HdsP!sf+4Tkfw7+panI-T^~~-5uh;9f z_m>~_O#=y3#EUm{V7X#XnP--{AuFm*Q6BcRuIv4FCSqd3ly+;|w%@;hV+k>r%6p;E zPNjfjT=!zXN+=31R8|;4abmDtZs2vSL8JIO@Hl4cwDm6Lj2|`)r*Yh{| z*~h>-{%x)0`R}jSi%+zsTYt7mznwM(=zWh#3b-X>9%d-6O$B)m?&xzyT zu^ZHGZK(1pgsNHq6ciPdK>{O1&U)8cW?yc%`~8l{ex4ibbt?4Z@%Z}sdOm(5&c5xB z@U8ckjsb@fdl6EWt2j;im6uhVx@dUFNoy@ndGyP|K>9;}{`qHy*E5oSyWJX1RnKYP zH+k5q+QQTIK4yV&Bp0`IK1vjp3Qjg1PP^oeNoh3Gxl1qHnwobRLHm^0qa`LHubIIKB$KjI}|ywX5aGIUwK9iam|u?HI2YD1$Ef zCG?yZ92s_FrE&@2Ax7nB;>;JGzPRV<3$x?|t%OhXZ+APlfBp6A*RP14+s)nm%r}-kXwy^9UHX{YEg6k{ zhf5P`Nn!W4?TNpWE3i<(2^pmkmPttX^H*2`AV$4$@Q>s0q6V4FXS>i&CAJ|~Rtd?s zs(yZcGU4zn%i@0pQ@9a+FuwWzyEd=9$qNW*V_bXEnJbh`L+G@lJ&`>Z=vi=ebTi2+ z98Q+1?o^3)>JI~lQq*fA5x6543hQMyAR{VG^pp|8lqP^j7WuoTzPm8_trouZ=5);Y zVOsHeppfU)+WGdjNcgm^!9`%UL)60ux6#Jf(e*OR39QGF-`(>x>U?e#yx_U$>Gi9N z$nAFf^UwcCSpUaA{_)2jf0S;|q?q_3Q;nLDDRBwMNtw6G9q6jHrjnpgdwE)I=UM1r z>tm`He_Ayo_V)F zfBaj9n1BEN{dhdG#N)cI_xn8oJu5+C_#gV6>GC06zDMRD(lvD(gcXsl!%ihe zWKKts$}ac)nIlE_CO>_Gl+XKzoNHk2lyM-VR_2E?TGzb$Qw(bPHr&06l*@S(6Ax7` zfGDL)KvE`3P*qYHf#7fOCB}J^)k5uc0Q$G?(%Xw6X&T6P{Sk`0owQSq)k4b4k<>S+ zhe1JOjSfqiG#v3YrW<~BO0?p+-u0!|fiq91doac;eXdMbl)b!E`V`_XR4XrJx!|_K z{h_)qevNUS2O7HX)uf4tn1mlums}uKRRVXxItQ@t?}cZzCL&q2B;3VaJ@Q!LWVW6x z*2rZl5hTNgc((oc_3Qcjt8dTG&(A-8{P^+l(R<(am&T$hmIq#oYMldqtjL6jNF8n@ z5(7ur(z9F2wCz25WqhU&Vq@wQ^fc*-gc)}a>6xgW$P2R}$J-B+jqIkBe1=!%*MXk5 z_p@!UXdZf%{r&y>cOGZjX_rRkie!4s=jZ2-A3rkDQz~c1EB_VSP$2+tFAi|`Glxw` zFSt$y;`<^Z>)>y(kXxlTK{wq!y5aL)2Q6Sh=&BB|F(6&g`XzL{Q8K!jAxk|dG9#y% zIqi@xR1r)_KXI|MgI`tkIE%fw1&LwAVc<}5iuqs29Gey$onIkD*vHcb{?m&s4O8$t zOrI`DK(4I|fxekVD54EQIE%L^O?{F|Q?y~a@Dm8gKS3_1z(QOiX54x1+f`Dh;DeAK zF@ZY|M=xIRQYVCfK7s(^1=&R;Wac|=o5USRRIn0lwrGTuodSYc0P- z)o6@J@tH*~`@c}3aHhjM#$kRF|MP$T&)4?2-|s(v{xq}a^DB;O1YO5@3=7bz(+2(Ry-OXCbVO|Ww(tAC?XTBsd%Yfy$Ls5{Js;2K zbMKp|x7M^tcaQyyV#D!IKKI`1D5XKVqRv^K!e22~%P^|{{P=pIA|}#9way1NvpnR_ z-7{;eS1A>ilivI7b^|y0y*=yvw90`i$jYoXF*R*ss_xQzM6~5R_n1-0X;#W}tdKJ! zgL%r)_Mo!+nXz)rTndGwB5c6)bo(RX-fU8qGv@B~IzK&QKXx9-dE_8u3!$#JvNeUz zCat()%~dh*0mJ6|(V7rKeFuntb8$XCPh@oTHLjm0u%4nZ)v=iF=XMa;5USt-WH5Wvu%7t)QH9Z^wr&NtA(n>c%TTniSl z&H@QKRBJ%b4b-UgE`Okm;@yCydcsw3v_VMxYeEc4QEhDnC9X&wjf{FP76b58JpU>Cp zwXW^;+OsYI{y+i0U_U06%UY$a-QBk>^CL51X+MDe`FK8FkL~O0>+$Qa?e)mhJ4K{5 zGY#o2B0{=}wXsM`Hj)bo*B)jLmD!0$QD%nbVdrtX|NQ6A-1v&ya$FD_UM)8>k0SzO zwek3hR4h3u&u5ZirN=CE+)Y#{DIy@Booe&3ZceZ3w7Y`K^G3z=)2t$sCn=dBCI$+8 z*4k2@*HKqz^mLD5Rsjo*Va z^*SniWj|9ZWV9lz5;|UdLyCn|1qnOLLmDKq*99Ks1u1#e&LC!`5I}>KQc5_z0za|* zMV+&e)XG(?Ao;^sK+pTFq z6URWhtLkDWPDS_L{kBk@bf;crbth3x`O!>GkN?~M<^P%sFpu9~%d*_po5%k3`>%DW z+Ng4@@QR4iNtikPR*_s0&g?+BOniBt1RnhO_;^0QT5Gv`;87>mf=6cd@#Dwm=Vz{I zZ`*eNSko^UM^V(c~4L?P%6lWV&BRx5t~yzM(Kpo!cT$C#9I!MwV}=UCg6a^ z+oif@UOX4!Axw>62-CZOfJVsxgg90FL7b52oQ3H-^Zxc?%;mR0J#>t0yj6Q0qbotK z?sxBHp7QV~rVH7CW4WeKDVaD#07us7#X`iGcl=%Vl*~_7G|N`qz2I?MmYb-HWNXJs z$`R3SO;yw+#6z@uy#;IY39-5HLnWe%D9c6l$}12aB8~;1jINkjhW(-Vtu-@oR}rx$ zBA$123L>%)0-ym004aGVt~|H1%0Uv5`~9DJqG0M(?>)m8`E0@m0@?1qw5AprM=Bdo zQ=%}j9I(YvQaoskDJp02DOTGPo8lCtgz3Ew_|-aQ9j@fEEX_WGR-{^;H1PTP`TO_p z#5Fj)s=C~7Ecxug;7?FeRXaJn2MWc3Goc!G@U@5ca1U1v*Blj`yWBv5R!@=5g_tV8bc0q+0B*E^@c%n85jpm_^q_{OS=Y($7a zbU>CFzORk|_xyTI_p_=i74`x?UUl9mYB^mz{OMwLa1Z%K%w^;}RXUMCz4YpOve z$0#HM9xZLo6Ng0MaprqS4+#m_tq?h9S|L5NWiaGLZ+O-zL-ac@L?}wYB0UK_Um@-h zQXWpl@eiQ%*g5|89e=NH!T0BjF1%*#I}~`ay$4nT_kC|wH&LtV*6nt)w&a&j-?wdh zJs!{JJ4|;WX zSJ}4h@p$Cq?Dcx(Py4>F>-zEWVU#7Is_LQ26#zJ1U5Z9B(`(z(!YdD)dYRwz*gz?~ zl?h~2&5l)qPsKwfRx)zp0saM98DS@FoLD$13=ljnP7wEh;x4y8VIn$-Uk|@&_$l8t z&vN8%uj^YNXZrmfNPGbCuW$Sg)4v;UelW_|#Ymx|nQ;s`>Ak~e$IaYZgsOs&PHBb# z*A*>H9Ya+Hl}7EWh{tj0hfBQiDkeHiLUlsp5xMuY(o*#Yj=5=ieDvQsR=J4iq9*3O zcg_7!-TP~ch%jB2g)L3?DgRsA(kpe))KpYdL!!&cVjhK!YX)d@-}mkU@A&n4iMF;r z?w>!SCbg}#sC}EJecO=2aYPP#@7uP8zoPfJzk9&tMu9WsQO<0K&+xfMiYxFaZT_MpGhlCW46U z`&m7kL-SZ~V1zWYkB<)kH}uHezrMbZHj_Kc4hy$;IjRm+_aa2(`Fv(Saw2;g-HN*` zIJE?Qx>_?KAeeGa_$MX=e#MF_*o^Gjpp`UG*~u$m67!$mv5@f^D;qo(C7)f!eU(10 zWa33WRO~u)A&&9Ndjw&GnpE+>c%?_f{h8WfQm~ZZ8;;Lhuc_P9Z47@_s~{Q3i&{06 zLwwJ07K?J)0PqDGF8!pr5QET-=>zr2MW(!wqiUE)7nQ8fRaugmAJQ~44ONMEDwHIX z<;6jhD4WVQiB-Ui>&p+j|C8qtIln)83!gnMlXP|iPiE=a6TBKNuzuC7?g+~sVe2t< zCF@xGqbivmFMie zXRY0Xn(7g;tQNh6H1S=Mvh7Px6<{T5j!elQ}x zm1Z5cY=8v(7WgT!E^jMm1?AOFKTS5D2FM6_29XbkOJ=>DD079XD>WuOl2gbGlN%{~ zgrkQG{g3gQL~6P~aEz`#pQ$60_h-kkmgli;W>#M6NnsZBR@?US&rVy_gA6JoAa_hv zRXM*@DiB?|Z~MOQ`(|eD zwVc?uh`x)fhT6h}QpE@7gbOJ+Ui6Je#UtgRiHO8LeIHf+*US(rw#q5=vjgWjV(>X+ z1O%-ZKr!x(>j|dkRc9p4p_oSb>CBgAr4koEUl#K`+=bH*tA8->Ktd7QZ4jg>`^`#qc4N z`pu!qLVy)?WsfG^$e%EUM<-L+wr$wA%>tkomtW*K&)sRYqftuol^V6h<2;8U1MXRa zGa`UB`Kyg*RDn=`LkCcUHHzc5BM04RDI#HA3fH!6i#Q?)Ni@RwQ<kpNFXBlIf2hKX59`v8=jyKC*9+EDJf0UXlaT`?hWS_H^GhdPM*6 z^9R!kjw8s4;pc34S%(t>@EKfYB1gnWoh5{C^3Tk1Oc~C)iRl)+I(DuNil4O#*gZXf zr51RduAl`%X1%*VzP?xuL34QGlZd)+EIF_-*XZaWkTy7`{F$i)iv|+)FYd!ZYQ_ zBV{^7LglFIq?%fvGYd@fW3-SwJlNhWE0HEAGXY+&7xGLI5Xj5pb@u0wCyxk?mc}OC zgy-{>dh&XFrIaLXyZg3hJ(Ff;&CdF$?%umMTZIy4@D{G57PK>BUG7$a$owXit@kdW zkyShLDVMGHItF1dp9}#?$-EJPIzJ4!(3X5%qFClTuj?8i``)v}dAa8;Ym{R|z2~N( zH?}h>>cwgXiEPa}ZSH_wZ?$_qDXEoOv+sw+>VAm6wPt3w+lo|*jPNE(zV30s#8zoZmOMW0y0gZvdpNL0SV?Pe;wL|+%X-|zSP{U4tntLhqkjcDEV@$l}^ zU6xxDS^Hw`W4Y;9|N8y>Yy181eQ%4}E4s)Yv2XHxs)w50%}mGaYCTi088Xv@Cu zDi)TiwVIjPV%8py7pzY2r=$Au@zENNI@*zJ^4kA2v$kZxvN&cF+xCxt{A2H17C39I z?RDact)l8#89yg;Y9bNKs=fEUZ?PX6i4PyY&Ikhy^YiU?(^|Av5z`P8Q_b?6*GWKH zj_aiw$I@v?sELVt^xpFrN*a6Ky%y4H0dR4g`sC8+z?U-y7K$X zy~E?zZ9({)dUC6h*^X)(#cVFxg(O_86J%3H;SsdM1LQ$5p}7C~8D4&6zt_j@Q@+rDHT>wL&{ad%~e&T07~4K*lPYnFk1xPLg94s{QrrSN8u~ zWS=JfI$qo;E=+goy7Ok*qJ8W2zPZ7P{g>I$owG{L&j3qjgI9+Q$lqx zu;|CwG)aX}NS&D*5o$J#=q;OY52~#)vz+Si0OMkW&L|L^h!$h@9@Z`<&U8U;vye90 zM+dTG)pZDUs?Nz)ym4x0Vmb?0jL{^-BH~$+N#-c63(#phmJ-j)O5VLB|!lAC>6Q!~l!UD=Cmd-8DXsF=UJwwIYTa|DIp zTS%P)vowO*ay(9~F9Cp`E+Ux#y5}+6@BBi}q+njK{(dJ6d?Dj;l5>6|^gSY)4&A_0 zcrP?7BV$(ktC{kuD&mE8S<68pG{SP7z}?>$AOtOZtXKvjP7*Jp-M)?=LsjwS~F!?fP^O-9Y()-cj^)m1f-gjZ?uhPRea2xM98(0OzN9tR|h z9WnS6AQTgMqAkCg*@VwAqS(X>$gz-h48p;@5GCMebDFHFF$5z%n^;4ec16@DLR=V>z4%=wm_~-w7WA-$aCw5f!o=NHv*EDJ*4<8qWfl zSef{T>7p=yRA_uWy1ec$(ZJ}nPuw5%i??*7{P>-GBAzy5VR>os1@YT!M3&&Z^B z_};f|+xOSU=W6HfbkhPFHmIybGCI*1BP!@Y2t79<9j5Xq>CSqQal-rku5z}y#p_rM zsDdb7+o83#uqtid<^VgoYLs${YjW@SC!y@^>6+GW<>K$+I)07ePW;=b_)>~NE_AQ$ zoP{9K)69HWz>z9`7yr8dBl|y81mjMy8o&J&eF{p{#W%f*zN!sbz4JKh0uiS<1XO}g zWQOwFRDG085%Y#M&Ex$^7dD2m7>mE70~3K8e0yWK`N|Ap9Y3 zoR}+Tu;Sbkdo=Yui`h9K;1~`+3R+@e7Qn!;FR8Qup3TBh0GHF^E=%NA#)<58W<_dN zM8scz{UstQxjX5#?|W0b-ImAW@b#khvOXTaU$0m9<#xM$e0;3ys%jEv9tcu0K<%rW zh3~ao0Zjf9ns4_dBE0afL_?p?XXWTGn4^eDE)RCvD;EXUiTGNup8k6l-GmuHA9zm5 z0ttcP5yd-?3LU=A&V`hbg-S$xKjZ9j0YDTuwFu22

& zvJ$h5)xarIIWa3g{Ed$TXqZNU52<^38~d<&$$w??Ca#5;h^aP_P|adtm>_Bz9XSfc z%pMGjI)*uQ53e=~8^)MFrWa!~H6pdb%&%kXHSzS?Pjp6AGp?RMmM+5W<0Av^c_v)% zE+U!lZ>kBZuh;AKdcK}t+qU(-X&f63B66}QmK(il8&n~?nOSS=G_+G5s5l*dimKGs zTFV-T6I*c}mRAQ@ZigvOJDl{CSTG1qK&P5$O7&}bQZ>OhkEK)pW3*3isg5SYv8mzQOvSY#DqLWvh7z*Y4p(`3Pec+l;6ZG(g*wlHFzE7WmCQ2M~aB*D#2V zNX?V?E5uLiU=f)VZP z+eoq%RFLx7TP;BI{du7d30{in;WT~8S8Sz$3=>f5!Q#<+kSyW=KaHiUNViPCLJNM8GV*d7{Q7!MSm6vd2Ou zpNZq{nS{#UOv7glR8<-G(DFbgqD_6Cgv!RHpp$Dp)RbltO)t%6AKMFUYI%iNL_u@{ z@0uJ4I0_RY5PdS*xu|L;w%=}NfMPjLYV6y#WlZANuPq|H@8PZ*X12DrNB{cu=)Sw} zYtz2@uY;Mm zGtfKkzfyD#?Jos=Lm3?*zf#JNFf&wGX1%g8goLTkM4)^t45`)%KF8C!1Ncx$Z)cVm zi#3xnnVL_bOnl~4@NAlOLn$1Q5_!Mhv-8;an^=FNofmw68#mwq4patL<(x&*42D?? z#8lel1h{9Qse9g>P$8qqJb1h;sPAlJLMlKc2P-uRHM404878^VE9M&en9RYboNILE zq*XtCqI?%A$+`Cg-@psbC~A~N>3!bH@9wYH>$T^p$2U3Zi~>W?lH%pWy+|`{Z{QzfQ_ua#yPwN@F)p;)%PzEJ;Zj^{=TU>H}!V9cy#y-63Jyw!}m;2O@e5bKpEeFi>r?cg@QC-)cCXl~GVZSnh4f!|^N6urt5H-H(jSIvwc<6zp58q)` zn}mwGNvk4AjXuodb*vS$6+K}mc(G$=;+gC@4G+pEtVOhn*s(D|!kd+isG2#Yg`2#%+e-+(Y9FSaw^F?s5>KQ72aoCE{f z9^;v13Dz1HcaP4m zDEe4ySWRu%fDHmNYruM|ppce}rZpFI_pF`S>|jJ|E$32ZN5IGZtP`y?Z`LXzmh_oC z52m3I9Km2riiv?tQADk{oL!EA)ykUZ8PrSo~Z@7NqxASGTg5M=^j{=+ zS%qR6t;uV7mJ@tkzhO2(41`Q6M`fACylstx91`e>7L+;{){{Jh<6 zx7%DgCGvDPkFD_STVz*C7)pOFiw$9T94JS0LNCbf-}s@^5;jK5ei?f-lZC(szJF3D zMgjbrM~hwWzhclqvVP_;1kEts4(e|%_yW+hR_8HZ8+qG4R6;IvmP>v4t3kp z;(ovD7-Y*1qwWnJ$FKZeT-WcpC;7U`x@$!z^K-2(D%1Qwo!m^G?{zN1?^V@2MUNYt zbZk<;cG>a@pVVkuEaX{fuW!^IeXq0v^pKr6rz|}l0{F$uExo8bl9a33u7&;p2u^B9 zZ)&=(ZL!tbzQ0tC^~R-{N)r>+*r?i9gk*Jr-eX;l4MZZYdTu`7U%xYu=4Tq>(tDPX zIZ2A{wRSVJ&9^EBuw<5Mez?jx=Q6Nd8D*qo;}8`N5(ohugdEZZ`dlTMv50?d?foSB z$B!QwXU}O<`p2sJ^YcT=F`eY?4aAE!Tbr$~*K2!q?_d2RvRs;|X9elZ8O^dsxsD}> zv$L##98pU>DK5ENfoC5=E(w5M-XRXN0XjUrQAXuw5Q%YefF8=_BoMTNRd^{_Ji0BX zqVoL=_uR2uxng+&k0-iCExJyf=MPHW#3Vz&?&*=KpY5`D`mBYythbz4 zYg-YA(Q?uD9uc7#bl#pCvaYS5$<0h6M08n}+N>G-L}W$v2EMq@!Yokk3atKvDRS{9%wlG?A(Lw z#QlDU)4>})r{oa1fXE?#!N?jFCQO2*7dM|Wd}u$(?V24MFN)WM0S=Kttzidfrt$E;DEkK z6PRs5{zbbK=hAn`&XOSoCaRWuKWGY;Br+V$k~BI zIwT;Y=NTu({p^>aZ!E9eB_(j5`iGycJ~d(XKM8QOf+QlVYhc8!u8V8ffCXSuR81^lISU8 zgjWjPawgtxt56P=6#BDW=PrGGe1OiG)REpdQb(Ais@i*I+DSyH^=w(ie*E|$VtGVr zzBGd;ziZV|^Ze^&*(CP;uvqS1Rlm->Qm|%S*JjB+iJ>IGalT+{&G#qrPq5#c1rij7 z-PNl4lCre#{rP;h*4#_X^pw3A&%EVkV_X2s%WQD$ScX|v;do7kYU(o!#&^P#7b@AY zgCZh7KJF||=o=@YNckx4zO1(_Y@AN^=jUhA>esJdPC+CnzEyNc{~!PN|1oJ)j5;+A{NwUw59(VTJZU zZEGA>^N4spUpk&5gt5P6ty652Aq=O(T>nK6v3iE({wT)~QH3n{&6HfwzL`z<*Dg^kuc`?NeCgM;+J^~K2Wi~*9ZFP+YR%S+QRKD{ zRTZj-2jXND5#8fxA&U^IO&x;QjuG{~*^%zKWxTkg_ajEz$*sk>p;3>-$I~qUt$LjY z6y*Rdb@DrN1ns15dnHY0E<;2p5!q{kwXFBs?e^NMj>>w^7M{=N;@Kh@&hmUdWVAg=#>eB4 zgVL(omAN8LK^s-`gOaArj-4CL*7Xh*SfnHLz)L|o8e{Sp-a`kY6Z7Kp#|O3=c;!UB zUN1PZI(mmrArY3YDH{^ce%?;}=O`57lyOq`h3 z6|P*z_q;IMlWVs3ljDMxgRZIbOlpWkwtx6W6d;9#*!TVGxt$I6?%q9)vW}L!)x_62Ww434 zd+*VE?|WpScKFv>f(B@1mPL3DyoOa~A)Iz&D!J95OpWLq=en+*fm0jeemD_`6-P%` zItb}d?RFv{XXzLP$B-%xze;3~u1S^H*M{fE62-U#e2~k<+Q;(1C-zK&t+c>^@6Qw? z;@J@Ie>)_8fsCFbuRO41v&0)+asudFx8be+Oz-6m_?Q$m{TqZt5uwHjm)OmZjUhu+ zL`1SaHMsz%=1iJ4(OoJdwa%BBYrOIOY^zyPaWjwNDq&1vW|@enp3%ej&q85Bz5q-S z$o00LY?67l%gBYEYaxzPL?x<yXhtvZXIN z`)1B3-{c^HM==7l)Y4RpB{X!r@H#e}{N+ULEB%kq<+|n^t1Y+J>lGpH{(8M0uRS7g zC`a!(`SRX-R6zWc?aCtWz3;n+bfxlJa5zXvh6xjUFhP5^JMAwCx>(cAn)2j0#NB1| zkrxVPVlJhHBzP$!yO3Vs0}p{Y_}ZShp?YPFQc)QRxj{(sBqx)oOjBoEZ+-u$QyW?9 zSpu*6zSY@J3(vBnENbt4$~l>OIrUg;w=}<=ry&z9N;BnZ;W=+fNN?>dOC+6^54s&S zn(-z2bJa{$O*g&Tcd+faUr*08_wlP5kA-y!M9?va&)*Bl_m#~u?r_OITqiDrH%AAv zPOu^)M(4$rm7n5N1immAL2*907!nYHp-L*C6y?E9^oPQ*4=ZQXxjH>-umTh}Jp1$wfzVCf|?fd4@qx<3g*)rJjI@VRk3}c2B zNX}&Tt^?5#jAS;SrsS^W5=)3;ZXFTpZG}SR&Qd8CYT{jeo_sj-*L(TEGVPLY5(!G114mQIOf@y#?uM7~=qfkuSny9Tkdx0w5u5Ch9R)<1^M0Fx2ftf_w6mBK^knV$)4vuCSKr#8W>vUEWg4jjV~R; z$#q4CjW&z`5%zs=bpjM7ZBQ*$6*-pLkPWG-2(RadlFxHlg59p~E zG8L>Hs;Y8s;tfg%`p#F)uR-w1p=Jh68ZmO=IH)!WfS>T9NxC{^H=tuhpVQW+2(fn< zm+(NNEznVpCIU+Meq!r1gc!pc%tTg)xXhuwX1$YUDJ2E4Y_Q%Px2?DQ^DKBa$riiJ zS0UrU|f5 z6haD~p`&kr-K!)BsWJK>gpwGAQQ=~qgtqbgBY3#^Bhy3=9~>>8YJ-Csc;XCROuf>v z0GV8rFbK*)KqlkFL8UMn?hG>g@HClU>VAgG0q}wENKEVf<8`>B_V9m(HjziKYmf8y}@`R;Fvp~s`7e0t@Z>7 zK}kf2z!N;N17a++%54;^$DlAv;AV46YDAbx>FrQbMycK`%S@v($oP_F67NmCz=$S& z0jXJ-Cth=h@3$KUZZK(nAYG!cQIh5B^*S@b@_Y?5YexzRgT=BeuA+8k@~rE6zul4R zi9sbPlJZsUYp~h+&gFy$&2%WIRcvWTG$uUaJVY?|9f{Y8wmi3++7}Vzzg_>G*mOJ| z!%z5?r8>XyTB4Y=Kq{OF9pQNu*MnrhY*ded{o1%((*R?ftXNsZ1<^6QgEdeHIa+ql<}HF^3n3ItiJ@h|qBsH&VQ(zO$ah1X`;Kn4i`EG&Mv*UQf|l0_Z81B`m-Cyr{s|@~tnm3bytA*bFRYST<&Y>X zNl%i&4B!u66pdk4u_XX;BVv2)DTRQ3&^;o$cYs%VAIq|~DxnF7+Wk!`6^jL42N6e} z_#{aH^cXF+s;12on#A=kuAG3aQ3VQ|=s)7bESxx3=gZcn5q$M3<<5`>`f8rNrbIpU<4a*cfMf6J^NPIJ*Z8 z)H6%+-1K2OR14_6o1I%{nH0<;HniM@qe^9Pnv zjA9aUFq*%~E3ND?E;Cqq`Q~b^iMU5|Z>=?POJUEw+^V{0Ypt#8eOZ>0m6cW-`j~O!pH4;I@;W%phFs z$WLPdV6qIeSHx16VhlUSS}Y^D9d9i1%0--nln9OZAy;&!5?-HOVYkAEjM|f*LLf4& zD88l6^p^a9o_*JKyF}1fuv~@)aSx- zm6!Rc5KggZMhretegA$QZDzF)67FYe4Wla4pJhO~6_G@M&%P32vkb>dQqzG!r0Xs5b02G1PK~Sz@5fM*+$Os&KKkfvU zp;nx#ia;W3-=nf0F;sxTKx_~lIdnGVxa{w zjKV2>UTYI$&O?t)E@c%amx?WS5Wd(L^y>`P}!HhO6qjtQv6CGS0@WA$;eg&W!0iWVrB7+dogyz2ER&xLNML zn5u}@P(dCG}B0v z$C=dWIAdAn`Fv{SJSn+^DBFbD$t0%J2gJT70Vs?(d=Q~x3%f*wzSrCt%Zc1p5Usv2r)$kt4HlT=XUl{ zcz`kDtiJOg(SB>qY8`Ih_sovq<}vDtII22jIa6nXzhw3?ZJ`wIv$z61(SJr#d}isA zs&0ax&WJ9h{9b#)G(0}bImv?z=!lB!gy9_KVZRxR__TJ9D}6&9%`ywz+_RE?q`{<* ziF!3ypIqpy=cy5Nl3_QmMtec6t}E|P-(MM92M~;iOUhHorJj3UecV&Z<7FOS74mIx zX}YE_S(7=Km^MF2y3;|)nEAT9$0=c;9`g=M2&jTE#VeWDL0OOa>$Eb)gu0F;XmvCy z0>G*wqxqCGJ9Cb+x7M=ZgR7&)PCA2uyk!}_(V7M+#fz~7r$-<~Kof2sf`lkUkY7KM zf*+sc3T(w&WCW~9HvDb!lFRLOi!=5*MB%pWnfv1+LgbWx%`8bt4Qi??VG(nNYf%qH zF3(;??F&JlA#$1Jd}U2PbLWkQAE6>)W+tQfO-#glJwuR5$*kU&E8`3k;Ck$8U6X$vSdRd?4;N-mS+)6$S7Z>wHB3D(W`2nyGvENH`9o2a)d?& z2YpxyLlcp3?S3L4D?+cm+!LZzRpE0!#M49_Jwmiu?uGRuPODvLq`7{X87B zX)*7CN(`wu2J_>Xp3A#^5EEsE|J+xGO{KW_JtG7%9H^LAn=VDB5Z2`Y8U z*XBOwDm|lEKAz=>E3;Glh%SaLO=Z~Eb|_BFC*GtU-EKEdOXEuj5c5I{;QN(v7+v0$ zdIWjqres8Xqhwzu5#}x|R}8b002I^D&r`>yDflrKsOmuP zeUEr;$AKHmI@ndGY&^NxAyswNudhdnf-URzdhL5g(p!#k+qTE!kwA94-Bv5oAGT2C zC}c7C-3D=qg7T1bcbHFfnW?pkKM92(4{;3hE?&SYA`+&e|DXTw|ALkBaJ6WQhN$I1 z&P_b_=j+@DxGW1?Z!ND7Rido+YO3nT$H(n$;k@EVY^}Ff(kbRGR$i#fY*hp1tk6 zwFc2moxD}?4H1cAXwK8!AE7F(X&T)Kjc9e2c8aj?=i=C6vpuW#b`dckb*O~6i>NM( zpkRIPPZ(g4GZUkTX0LvX3Zyo(*7Wi9>%<_D=G}dZzAw6)nu)2XOKcv|#k*{hhF{$h zPhC|--8XHmEoIT&{rTLy4o22Gq9-${w{0_9aV$whWST!p2_>H3yeD`6{e0+niHIH< zAraxyyXg`U`?l@-9_=&F_iN2mj}$2p-?m&K6SL(w0MNa6Z_D7VC#^lQj&X!Uh=(}B zu(_WDcnyKXf)?nHe0Hrf6lJv8updsxPNe*&c^#;E|NBbmY zIN^##M61Z;BFB8KbHEF|ydwU!>ufRbGWa!p+zDb-e~PslZNf2>aj!EW;#o>ws-hDx zkqL?q+4*BHiMMngk{MQYubct`@RwU_N{Gxh)@el)ST=Q)FRnewADCS5(4jE6oEA%Y zuv$A=CKljVew_>fC;^xA0}MHMikC*^GPTxnqTrQdljm@vpJ@F04J0i^mIIx9Gk1)M zdD226-pv4Tk6yg6tCXGu977^gPhz&mXPn{4B&nOR@(lUvurj_lELHICiX_K3X`E4HX)nY-gk6T>4+Pg zL1tr60f9#*@+9_5Pk4Ay8p`)DC}meaK0W~J$ZUx^#Sn~8@*CdLncqHrk3nfUI)NSL zRC6u6^yi;{=7R}GsI0&{LxZkfEB}j9ew&iZ)2kMv(NsMm+|Q-n2~AjMCTDG!FVw6J z)(Njr3g%8c;Ax8urQ?RhE4+?}lgo_H=X2k8iDTQXk^wZzGr_ONPv4x*#Q_jz^LN#c zAzsX2Et@YFRrhh_4Jw^|W4bBD#NnUfb<^V*2rq9ap;b1VLWWR}*1`sCPwMvle$OSi z>qM2+j9JOiA#~Cx=**6Vl3{kF?d$1b=mgUjIK$`m?&R@^5E&ksgn43$i0D|&0>iX1 zL=)r*eJ2lZFY4F;RTFbhirE_zj`zj|@zWl&{Z5JH=@<7O#hjT(3}-14Y#z`!10uUw zOySSZPxve82&bpTd$!@z^xh?25lzHJmWXp>3hoi|xAHw04x}__9gBM_1otwo%`mb% z#mYg77!8Jy0^*u*jH${W@Y&thfBlQufwyySxf>EaLNt+`fl_H1je&zidDbWz_|(2h zC}Ps*Q!trRCnq@ok6|#j{WuvQN!KU?((7nohPmtFYIY{$>C@V2?>(#1XM@k@6Xgv! z$e!2jcDvnfuh%vK3hRcxgZS&8pfg0AdCm7k96q~^L6vLclX!qNxaasV?kmQ-8dZ84 z`lCaK@l>x}T~ZRozU_DN05LuliaHq=T;`D;|Gpb<#5|FO%{q;Ez6(jFjA9Q=$Up!1$3LbOH6JT;poM${&gKG^ zgCCG=9j8LsMEZ<;zmHKGC9@_KZo3Mrk{RdD!EuIeFlBkMT16R~M6ug? z&kOGbrC4x@&8Ol`?j+s_ShqS+waXsoIt)>`I@eT(c-+pI7yrT8gsGpazFKfPWr zi0rb?seV#M2&;BiQtM@6xYIe1IYCLMU|HH;hx2BeQtKss+_WW!V=|mbiT-f+m}N|S z{Hh}&vkVwjGKdlEga9!v;q@3X{rVHW2?B8lQtteM=0pE^4|9x5!QaXDTX|sFRx)f# zB{8^H?4Po%3XlNTKq$W`Lfs{4NQ%rjt*W>6=5(+vdW(H36UA=J$3#Y4K=a;vrrw&_ z2Q(1VFRTj%H)@K;^O{^a}LT&Fs6eOA9thsYXT;*Z`m> zY?l6u*Q8%OiYo^rB4pUQf@iS;4<#yM&J_!Moarjf@FL=4wdti(huMWvT3~ca2s!e$h_1^Kmc-&#z8zseh;+fLVP2Hqk#=}hWACb_8me+^K&-9;8J%XUR{@i81$9C; zHY#t~CxnUDrS`2O?rlbI_dfq9{+{MBO@5mP%59P62|TIsWJqsgS_W!Eey z<+Y*1X%5)O9Vmc+sg4B3c{QYndMnvsyo;M<>;ru#q9g>TY+0@5Kc7!5I!~G@el8oZ zN?>J4N4HMCxqkot4N=G!!G-a0QknhF+<;bj{a6|Q>(_t4`hv(>zCgPi>B4oTS$Ysh zHahH7Dtg9N6Y|F>1Ig~)Sz%Urc>K z4#p(^$n&%D%E&cy9648bX%a0{SEU`Rrkn~Hbed*V`OMtGep~NB7X;}lC&`Fye=!PA z1~@3$Cd&B*2N~9fJUix))c$p$3b#TlS^)H1)9d5js#fXG zKpdWTm|4ii3+qX{$K%1`rDTDcojDTE=kxJ++-|p@KY#YxTdQO<-90+sMlrpU5ojXB z{Sc8q{`ho?>vh`EGaU@XtYxn!`;a*U`Ax107J}s|RN>EKSXd6hYlvU( zJ>}>5eD>b+pi$fdspFMfHsLG@j6~A{xKP~RDpmLSd`1=4clW*z8x+F#ZTIk9Qo7vx z?%~J6AkNPdD(zr3w>Z#n+l~WN*LBe&{)s+_{rh*8sAH3FfEmL$I@-%k*8W+@NX@`bGBLFB;TY`70Z6{=^rJNEQCn8oW-e3WH z$_Kxw8=&4Klg!6`KeKiHl9_`>fMVPBetr;ziEtZPetaxiX`@o;<}kjife61F2^X+pWw}grGQ6Ve#!l%sTCQm&&d&sDQkjI z0c)squOrQ_7YLEJL5y-FNq2C1r^8eyiD4acDntYFgPQp#PTl%hS^-PSPxX>8B2XSs;&q%`6y|2CPajX=5(*SfNS8u` z=u=QdWvLp67#-Vq_`fM3O0kqHOn`SXB0C1iunr}GxD(M*QI8IoS?=;RvrJWX_q>OF zUM8}9<09mrd6b0F;*DNl)M+(nSsR?btkA$NEzC^zC!JOv4)gS_FRW(`#rKN=jJB!H zLMx46Iy;invup-f#nNMFmvlO($S3Q*lFfrO0z=T5d@5;$S;EO>A2MVBh3WFMJVRU& z-eIlOF)YJU#H_@HXF*ILFBtR|h?F9GL6IWZ*X$VP*(~|+^fPfotA;CR2%H3Z^!-d* zKv$#CKPvqZoP{}D{`t>;DxoSsWD+URMK3xm6r0_(B zfoAe4+~;jjc^mIGV$SW`ak?G~q)ccF#vcR8_Ov{Xh;ZrMbum%#2s4#B;y|n5dtA&u zLRDTH<(&d4tRWL^Cwk&YBV4S#Ogc6~_z8@F7Z$Vw8SZ2X19}LRHVW}WV*yN1KZNhD z8pWLDFEKzcu`Egi8A1#MGQdNagqHt|(Z+xXG9o*E;xJlJCd0C>L5E$y7oPY&y;nw~ z8?{r0MJ~JH<9vdfg7)&Lo1SRv?%liZTY6eq#P`sl{q@{mDcMX$SZg$X;6m;8fCj}l zGv@PG8Iq74d#GC#`0G`aR8`ZPJmhL=@cA$(9Zw*@OGV`C>kFZb7#&0*@Z}COYV#0S zR6q@eJMb>X;&h#o#PBXi-~?Yn6DCIR3DYL?skJxowa>+HoZ_6YB~tT9s~PaJEKA0i zoKCf7wVy_)V_Zooo8mix23gt6req`pXHXbMZ_me?PK;M%N4-vPoL+qu1rce>xx6L>L+v;Q%s_e@r?f0XL2IGV!~7;)ZD27(s?iD$ zr00t4uqcpDNDQ83kI!n4Qc@2daj)t?ob||bKn?)JfPFoFFEe^lP7#Tt&ea_2Gqa-} zVT8LNHi9Y+3`t;L5c1|goYq~K4tSi!58Tfi5}tDPua5!fpzQ*3xsMOlpIXYrgAyA_l{E@^Vc(*o0cDV_gRcn9GG)nC%~^#lVBLUUOBQ5kJMO z%q4ET`rc}F4NXC|*$a-t=XkmI2p7?;Fl;7SD5jQtB7e<#P`BF$s~A4}xDI4BgNNi# z+BfMbj|E70m47@1phqYbEkH<;7nzp`d7GA_WyEAWi%I0RJ##Ee?lZ*frqN^&GQ>Cq z+b(0~0537Ko&$zYvKS|K(*U%x0|8Z?at9lw`UTA)0AV|A-&&WYRYiEEoF}L& zWIC@d$9vIpP-fy~UQY8bzLTrF^onJ}GlInis`8pUh^%@5X*Pv@gE`k>9~^oG-mw$G zf|3`WWz>*@0BrT{g**Gc*RJ%B2|Dv)w_QgUCPo`DUEGIklZ{q>D5kd(>ml@c9cGNk z4%GIVKTWKJ)5`Wfe}Z3@!_UcI>X8usLbMvC!(KIOa*`5q_|7Zof;{D*^^b_a`3Imm zwBwlC_x}6w0Ffs#*3m;!S}Ne#AyW@ngDc|l*`Rqa*MmZ{R@oozOu;&IZP7Da z$!fxg2Zidrsa|&P9R>V_Co4Mcbs@m%i`c5e#F~+aIqnnJ@Wg2fEE62s? zPq1Ivtd-fof}ueFQAtpV4*BO1OX^SjddG?1w)N42pbI(;6BuolG5Ji^BI9a-;lV%*F;|YspL$tx5zA> z9Khq|MpkwVAVB;kW#*nhx)SM@u{eH2=q1Mt7h}+L59rfs{WAkZ2*O~943A?uYDiR z2*Z#O4MT6iKKy$HcOA{YjGFG0S;QUyuc_!*H>FjP97fauYJBJeMu#<6Y#^vU97?^I z-Q42{!MTO#-s5#tL^@7TJQ%DhBA=2fXm~*&(D9<8cP@PgqPkWqN%2COYe>& zduj*CP4!&Uf9O~SzMMS_kyP} z=#_8Vd;j`+JfBBKY^ut~$7f2|>y@W}Nfp@14}IHS)a_IiQ`H}z|B^z(ZSSf_@ootd z(FhSJOyaw$g!Mg8n@`Umsf)O3h``?;c3J^vt+qj8qJ|rm&o!dI`j%f>9xBw<-;dvB zrl#)x+Kz1T<+iwME;v9c4!n@#FjkM~6^Mw49!(_DUZxg2U(apZe*XL^A}+_N7!e|} zgv8o1nt?+EQ{G&4$&MF{9wtEboILI2V_C%Z^~G>}nujc5y)4VJG*y%1uv9_%smRCe zmN@A9-u1bxP0hNy=vjxo`%^@uA8NR+>*wd^{eJ)VfB$!PS34Z6w%F^nZ`easYG}+n=w#@9TQ|asOP`HMeEh+9K3l_TBg0x2=~!+HSWS zb`_jj953naW2nH)njD)yAfM}cQ&rL3y|!6I*2j~GxZOT9L_JKcwTjsEKz4QV_e5lZ zseB`je*lcI-MZiJUdvY*Wz85e9RVjW5?1H^x;qvu&Fp@^Kl>2{oi|W1uCY@IlOf~m zSjw+tqXfhUlsIMJWn+l2L+02n$$0q~Ghy2Wqn%$iD(t1@TxSF(>7sq8x~UXQjB(!D zsdm+*#yXw>#70cqUb~v#_!uqZUnhc01DtM~ZniO& z?G>$JWOOPoj;8Rh)3lXM%YhA}H?8AoZQKEt?YtmelN%f4{p3BJ65CvI!En-$~q-7Vk}nJve(%@=;5Kiv{o0CTfC2z z_}=14(A*LASO8PUPqL5TFxM3j83owRKx>c%MFxjfmdjG-48N7wo%W_e%+_@cJ?VzI zlwP*KM9tT&9Z|<&Ln)D?`b~p8df(^f4xe|-XNRN${mWxLR}_Kltof{p-%01BT{2>a z&Alkt#I~IN6)bCVK|K{p7WRTF1W9% z8P%C&7aijvgAO3RQ;dtXnpK^pTqvE9=?|L67!Ak2Q9FI?a$4*@o^vbb-0L(aqaw-l zMsAHqT_4^Mho3^Xxe>O7aYUsIIb%v&E~Yo4OKft2F6Iu)+dHLG9rvKJ5>1u^@6Ge1 zdS%7HLxiACR9&cA7WWhDu3GK|xI^zc#J*E5mX-;8KA)NLI0<&I3`M6U4fbr{7O1M} z5g*GPSd$7Y%d+{Aj|-8;SU@N?lWBpp_VmF%@nD^pSzbAJ zN6#T($SiM~BF{#)tg%t7hMV@|w8ez$SWaHxRS+3$8xRYWJ_b|G#V6ePdttbRJNBQd9P|h_y zV#vshsm+P$s@mFucfoJ<+8K8pPWqA{PSiSKR(5NYB~s#axXg{xYIzzOBe+8^m{Zr= z*!31F=9PCEZR^*IF>94iDK$0Epg;I%6?4aOB{m*kS{Xg@EO3?(e) zS2M(9v5qO^6%p1>FO}(!S#Z$AmH`L|e)vk?REApUoK?uQSNJbR3mOsLJ93j)ruT2^ z+Otkh*OfIn&=|4X$Lx_S`(pI0^YQ2NnZf8w z2YD-;tXWbO*qONuoA7iye^$YqAbMBQeHanPy=#2T@_|@Hw-Ur^aaZ5Cd zRHrQr3~iGdoUe`{c8O-O^Mc)cp;-+3diZ`)UAfddhX0t~>&0`2#52!?)SGz!=F9yk zbvvi@vCvQc5va$+L4VlD$+%w}*ve$bMk1kvJm}SR2u3wm8&!ovE&+O&KYe_BU~Nik zNhdeUOcawtV^8OiXY}x4W%Q(s!k5L2@oT3~~QMD|f78Qd|!)cU* z!3|!Ul{30>uGX*ywmN!u8crfUTM-B<5Grbu;kvHR=dVe$^5P;>{oK|kI3sQf1ERtN zx(buP5ex)C*lp*?DnKfzreog2JcY;O<6flj!>P(0H$qv_LLh*WZBPofXHa=2BZ!}U z$T=>M7n`=CJd%r~9mtTI=1z2K^EMZuDVL5Or;J&`;4F8HrDfmu{@UaMs_vczMO5`D znR!;?EAbd)tHdZ*ho!yklDp>fuAL3;aXlll}{#gVxOV zU>L3Ff}Wsoqar-%hJ*~uP~0U9oQdo;MYXEDrnUOSHVk_L+#~-h4UyKC*809biL=1j zFguqSw6;Dm7~$W%@|*zBkO4);>&h>tQh&GYMO88DXUc2k@24Ok>%%ClIbDqG@Mo=d zc*$tgV?&Le0s0h1OdJH>%80Pc&vTsJ!x%7j)t+NFq9K6|^}GV7qVmeTNgy^^e$bs{YxzXVN^nEB`9@tAt6y$7pl_7iXBxDX<3$KIpeu3Z$jA0l05x{8`uYAz5Dn$LI-Ni5Kv0f~I)r6)qs-5~nE&NF90BDhyqy2-n1f^~d9tvSP*l8-hlr^5pm z<*)cYWrknjl$G>WLZ)x71L6K-r;LpL!PE|D)y8F5^r2T>CS1T@c6E&|3>;=AsK!el z^JORcz|N)ZKe4Y%B5Y!AhQwH4x(vs1!Jf%DZNg6$LxLcBn^{ljA=ahtA4i?)5|s1k zv7*>XQg_djZT31k7_T^;?s5WBG9Oj92T~N|gM&37h{aG^5_|*!IgM3SF;^Rrox}$X zo$xW{_wbwKa`Fb$HR&s*jkwbUF3ZnLajFAQ_~5=;Rf`z&qGm7H{rI3=sgVVZ*l5)YB@CmqdW%yme2>W}K5V2w0PYYb^Xhsk#%k zM8)NoD~N&UdpKTQbZ3RW{BtDLAE#i_GPME=BgDM#hvg-A|h-M zj8;W-C&J2T$+V^sLh(}c$|>u~{j9R9did<4s0;*08kkj#F=IzL^RNmnPUV$OI4Tv( zY@bO8RkbZEGYI3#Y-xt|80V}Ds4yuE%OVR|m-ECHRjpv+vQy=5tmHUlMd( zFEd7D3%wEtCznJA)EiV{x`EYl8WdwU#tzxa@up`15eYJHIW9(a<{21_7XfOc<_gRq zmdk0p5s?)|&=mX*ejM!saiTUELJAoWy>`rkUlS!6R?wdVva5v%=i8R$25~RN(+FMH zLF7iAIp2$S8AX9WH139uVF$lN09f03P^@72SjhY-LqCG3@D3|@3{LbEz+=FjtX-@m`2jgFleQQc5+2=j1{13aZf2*F>1MP@ z0-Nczvo6YYon#K0$(}yC#5;o+1$>M`2q6w52;7{NW(@gLt14z~%nHqGT6tz_qk9-~ zYt7rR9RoKwWxwIlF>jq;bmh2`-2(c+GVu%mBINE!O4Cb4AQ>@Qk7&aX3PU&G&FPpK z86z~~S^nvev_eL706$4N#S;ZnuEArhy#x}YAYrxfvLz5e+4m|m1| zq_zZSF+mt8iSF#Vs%E`I%++}L6vMPJbE%Th`v*l84f<{)I<$@iLSPBd6VDEzF~gDS^>B2I;#Ntc;*r0E-_xq8*qHn zAoF(;RF*3y9o~Y$<`7=5c?ZKZIeSZL{*wQkNR4kGWBkq3rMwu~{V3@=qa+VO2(_gQ zzg1wpjfWf<9o1%3kboCv*mhb4D_U)yK)1~hlsG7PxVLc_uvW>k6XxWFrWQRH!`YR| zXvQR>5g~f!8o0Xz@*Tav_tali)n%yVz;iwp1Fgx>o*W-+!a+ zRB3fGwW;OdQX-PbK1f!I=_qhPHgMnfeD}HNL2HQ&(*S)LwZ*Fp!d0ea+!4c6nCgr} znjuWnzd&JzNDPOku12v`sVQfYFX86n<0Dyu-E=4kFeyW?tkT3yyt=ja`}c29Nht#t zBGs$bQqq%s3g%H}m32z1VjeMl@|*kprsFzxe$O@kl;ZXL?*t-O(xnJZOwY^5y|qDm ztTC1=YbKFkYloE^cKU#lK>U3_b6t#rVAuBZ>Quf_$f&UQ9puAeEHIGFKHNW9bWjW) z8RlfnJ9%e6m>}2BMl(fF2HfW^bvlJnE5{VEGK_ATov;RB)jjFM0K_DM%d%9JQx=|y zl8kq~cZrkZmC$u!Lhn6nurvS)9j|Y%Qpha?? z^Z}Y~eRubJARaMrl8hs;;Z<`*B(bG|Jtv{Q;S_Q8`Z$kzXP= zVB5A|zka1LfaC$?bBV&oAb{2u9>!=rX3z$i;PYcM#(1)!Jgm{)=wV6ynkfM_B-8Wa=l3XQ48j;ho-p85> z*^Smd8dV@^ZHb7igOHA$o{nu3v3>6%W_G*Zmu2}V=D2DzTPyY=W=lkO-&CC^@12UK zvO?n8wmhVJ(=#bZw(XT!8Ot(EwB0?TeSG{7k*}|>$K#pnOcAne=gQN^#~o$IRnNHy z8!9-DDYg_K??Z`6R&!m125sOe)RTJG` zTc*k1Zub;U6_*GVQE~UaZ_nrRsp|)b$@Bzg*|&%YZOu1#cL`rqTn}{)(@el`b##Pp z-reJo(KCr?DwYdL*=MACRi4CVKCJbRR1;B#+~8Qo@WJNuGU3O?fcw zO5Zcz2fFJeNmgj~#PMjoTx^rkF*L#!7~Uw_w2zAm!_F zCksQcQ;-MplXs>O)vELvw8^7WrVRvMFVhjPJXDUI@IWSB_!wBn{ohdlccQvLHidCv zPa|Z5r4+Zd+OLNWE7?mQOL{ovcuX42^ErIeD6i})br~nvNjxxQch6ib63Zb?&?~+w zWcKh4ut;eC@FbJD6N;lusjbY=EAb(yz0ygdiZ1v#sl^U28G|3}4F|8ERSbh!oO}!* zJ4ONxly*3V9_N@7v(^}5i;NdR7f1|3>p0Pyrh%+-EXUw9sKBBOKw%u=x3gBrVNTDG zB3K@EdYM+>!q1qpyoDGZ7sCx%_1-VJ7Tn4F2{jsFic(^}zP@sRlf{O~1f9`sqc!SI ziAnMf9D}Sy!VsKsmQKF#I%WaLB{C3-#RDSC8mk2B*VlnQMvFpH8qR~gLdMh@q?+LX zo@~@A#am9{5pc6rI)GNa#@FkW6W*O;_2G~jd43I@JZ%5%aS@h7n0y2ikSO4`0^MN7%NiAb&!v~lsd z_m0^&JAFcC1{MNi6hdJJ7nV-B#Al}~@pO_`Xui@?8=A#it_J(cvdMn4=@J4dqcu8~ zuA~yH=)}#0+NDm2hrKLM`Rtora-2G)dnIeEiU)uf_xn9})<`XLq1%AVsPJ*}okJOL z?R$U9hu++r zq%!t1>MvB9D))N9>qJv&KXU-kA|2D}nVOTaf(%`Wj_-NcxH5VGU@ltS9$-Oe#q#l=)P@XG5Eh8ea`|IngRmJVRKTAum>nXKLsUR78%Sh$t=Vu1` zC)wo=JFSZ_lx*AVy7C7eX^=hR#`#Wghv@J8=PX^tdWNJ)D)6V!#Tbgm8QFP^5Fwa4 zJi;?4R(LoBxr(092^`n598q4yP6w^P8bz&uVRFNWS^XSNvPTaQ-tjt#UTIa6WsJwm zdhwQnfF$-IAgk`KHEYz-6HlK#1L9mmCgbcvwi&JIIKi6Khlzi+5R4&{%&>+^&BihU zjMar+XHEf9Gb?(+Vl$IC!>Y_9jDkt+?mP$L?sdvKx&~SnVeit5xY=QfrIXgRgX1|?_7D35& z%qF=Yk&f0y=~OumWP|T4wD~XU)krO?q@JI(@kTl18Dy)3hkP{@3>XbC+Us-$t0_ok z3HPc+-Yc0Q*Y1qwNqPV6b~*u2RG?dS45ij=j7>wR&}Vx$k70U3*EqkPolw_U=rpD5 zKn^Nf^IE6TvNWve{JJ1RViaA}aRnkO8DZ)SGZ)t+qEg6yyWMgM^>{qeNVC(17ft#L zu~bURWO2=Fp-LF+Nd#T6^b@H$Ld&dinn`E>@ArH5-!4%sRu(L5q_m*&;DT>Oj(zJv z>qv!eXFqzMXY@)=hgGZ@s|{XZCRQ6}`GhI>CdL?+5RNbfzk?(eQ&ttQjWwCuVPtXl z){c}cjEGxft9!%&_uXs5Dg!8YTWi*-bXF=yQKW2xh-8huy^1ET>jDvh2Lba2O@ov} zjL=C2g2^k{woP^A`$45^@R4l;zJdNjH(0A9B6n{KO(&U6%s!<{2p!NGRFS(X7ps#O z+&7_lF;jbzI{Z0FW~RUb;uGc5Fhfw8n>qcTB-UybPDs^@jHg`r7?SeJ0>J(0F|g?3 z{x+`Uv+hjl24MrzF%v}sWPu`JM$YbedUCJ*IlY3p4Bf#!zz6;&Eg#bd(qeXI(6bmQ z0jp9jgV(Wh5s?PT$wvgjoKP_86xz&e#fgG?6w+y0$6QhJ*;(R2N`@z)okx?NPjNtyfYFw}=R3K5cJLvN zAQ@Bwq3zpk)k5JYC)(XVKF+*RhCH(cn7B{{#C<%O7ruP1h{Da`tkp^EQa7+e0dL7) z|NQgMZ2ar%>+9=FME>~WkI&D~bv>8J5j)NtM+j3!3=q}?(To!9gVtNDwld|G2fiO% zo6g)}n#cDR7mkmgle~lP&`I!!xrX;Ltn-DRV5F_GX}L{&X4gKioJadOJq7nc2lv61 z0iq?fvmQu-8n%!?gQDu5Gpa(lf>zq2{Go083bIU0feDdQAo);f)-NH0&2(eEoA|{< zkW;UPB{_wS4^zL9L$Lmd_W(l1B>g~ za?YP9y2Wq#{QN|aEok!LqXUkA^PE+taNgEx{og4Z8WA{2!z!8m@p$yhf-2~SP6C&- z=B9`6{O|wU|2TUFXsaE)p%f}r5s~IZSNkF=p&! zYdE`I$&N5UHsHEa(J$(4DBGS@$|(5X9`Y z8`0wWPLaY59xNL?j-Fp-OlW4AYZCCXK}H?D5dPOGegzd{8DJp7bm}}UPjxOT@ZAWN z{3*uJi4)vc9&KLmPZ_LP;dyL)$vDdfYZ=Id>X@T5*Y2@C+a;rwMloKrVXY?CeP|loCdH!2_D*Rkr{fj5R7O{e%lp4U#O4NW%+6Uik8kY8dYd>JgpMcJ-CyZQj@ zxEX7~PJt&GfrI-Bac7x}%LyB}a{@9@99u%5qEWLL^pJ(P?;bW*6p1iX!yZMCaLNB9 zG>U*zpcWr=G;EXO0PQl$QovY51i2ZkhX}~Xn8md$i`kHm1f-Y{hfxgDgPu%i!o5{} zQ!*tV;clY^o{jn?>cyH|FelRmx}mpv0s0uts*E1SgizBXxSw4uC<))jPw`Z`)2iC! z`?Dl4gyDwFa7buo!b)WZPrQKl&61;4Nup?YrmKS~K?k!j&@czDa}E%swKGcNG^}SSEsOaX*7_mOUmW3r^x@(3DZ5f-3&(R33oy5c;@@9}HT?L$mOcl(FHrGunP9 zqx`3cpaKz)qCjLgHyvZQakDka#+-mW>C7Z2jj93&Id|7+n2;h({a4CZ1qFfvaWP)< zy+||7VIJg?BY``$lGucbbo=GN3f!FMTS7~NOew1sG#NnIDS=vQ6D;3vN5;i0nfZ3? z&rk}AMok^yZG0!U=+Dv?Xo1&4=mdOGJ~EbGSpf281WXvITWeyLGumEfxO<&zF})>! z%gFc2$f*j4OgFm$?H3wo(4#H_P8=J%rW0y+LgYkxLf(5~G zc1F)hLqr58T#HD!sA}#(#rVOhsU5V&D8oca-XJp{E^RLE;o=d3l#~9vg$8!en2y0` z3{urV$Rc39m<0OIVIm=fV*WYJkF;#kLTAIX#$pvEFT`Owt-d1u(0g}(!AVE()G2{| z!XTxEH5S(Eh^kk^qzVI+(L(6y=A3gQt#v;L_k?%uo_htj+3CS6h4DryWET;v(BfB+ zed0Kqv8#yku5|7gOE_AH5#m__rxD@Lec!>FAkQ@O*%8Ws$h}@?HCGIZg=*pg?^t2h zO!6kDXSe7G>nUk}{qf}_%gcp%QDPV9*EGcXb6E*5s8xj)6I`DYLtIoSNxRh+%!Olemd-YjwsNa>CZ01 zmF8$oX~C9x&7IHv3i-W_#cUCmfZ9h@NsSSvg;tN5CcTb&2#VVA5P|?5j9|@=ajDJ8 zpUrQk^%X$FL}Vz~PA7t*8>)(WK0&2>eB8Lp|1x&-0KL3v^l-8_Oe^$R#|8H3W+`pA zN;DJ-HzlwxRO!ztQkOji-@%vPzkf?v6pYB-R|=@G_r_Sv!`WZQDe%lYc(t%e@I&!9 zw>EtVzN>2QC&#^#OoTEcdTs6A$FpUkyw5h)2?tg4r4!n+R8O#gj%h7&AU1-9;7P5f=MzWIM2$2wqm4Qtx-r&(t{Dh(KFb+y1N&h_t2H-RAb>v@>7tB}G9(qV5 z+yrJV(+;%O5v;E^IVe{626ST3OdX+_`O2D>J&f|gI+fVveoolI0)b$T(JTNG=@h)i z?7?W0u~2+zav?K^0aAjIYB>=s9BjAHRUfV)p9eRfG7)2?yWehWyz@b>l*!i9C|rp= zS*(e7pd0LgSBL;~$Ui43M+;~W&ToE3_2vnKgytwIqLYHii0rJl+;iLmJCOvOqJ^Nt z6u!t05OS6-_N=vlKBIZh=M%f$gQ924vbfu|V_wlMmdt!nr;NQ;2Sx%Su_`8n*XVx| zbE(?D&@ZJrXxJN!u0kVxTzWZVERHrYGl+^}U+Jb)eRI-Q>>y(=O;w`~ePnQu5zV36 zxb5o4Ky3J>pgeFK>Wo?-N}$RDHX+qp{fe5Sh{&>UL*A%pEX#6h%f7!v#8ed}fm}u8 z@%&BtmWKVcZQHUO@c}HJ9GTSqT8{R6CcN$a`TY9Q*l+8a_o#}-Cf?0#trA4hv`6e7 zTkk!NlY;HGs91>X-F-i*S*qo*UBp9r#ECB=Dq#^SVj?1;O~u23j-ld_)s*|TZ~MM? zIksbnYKYt_@!9J%C^N%Z99JKoSKCsir&$toAO-RH$mq&>0)`&d;PQJF*$d_McDqGC zmej+2E!h$GHKE7dbKZBxaGHv4``*LXb^U0cTi+kQUhBF_Gqa{Dq8`$_@4d&e=2*gG zi+xKz-q(-+_>cc+t$i#_B)q?(%VMpyw(sH|ss8Ck7d5g{)!ok2{$0d#EfHtBE6p*1 z-A?O)5u2HqMQb4vs$r(?`g(ntS;B%IHqt~ac2PAo^7y)Y_tu(vn5tOoz5BL{$fBm} zP~8^{TB=@aubhoShMs@`W{Q}IW(`p@Q)R{# z-S<5K)ztQVf9y|HU2dzH-EKFnkjS=e;fw1nOuFyVRkl#mRa$GSMvvI{?di0JxyKGz zmm`VZ*0qTo_`dI(9^1;oeK$4Vz3;n8U#$H1+xmJvMVgu|7P?4;M}NK^+iDH2SZ1uJ zIzN8>?yqBGLHFo#q_N+Ber9B;_x^moDlO{RpQX~gyLVrfWoe7VYPvta9#4^VUDvim zJGSGxd$SfE-Fs{8e!C|)U_A`wqw~t0O1<~TFeEB+CV={&jArhj) z>RjDuC$f)mbEl-b!(Y>L3Mu3wahpOe4_=oIVlvxkZeG@M@7F7g_?yKIX=73zHa6WSIyEzSp#A zF8bCgolZyfJW#-gW0$j6(vJ@JIya-WCUt6uq8J99jrV%JdejaHS_fX*4eFybBJggK z*k<$bG7}?M)J5b~J^|Xq4ALqZ!$O#qIno8>N#0VWoS^3Q(@_mHp$meS(?O+_>j>Sp z&HE!4y>rro{eVAy{J^+^W(L21|7JC`ib~}ka;t+D@>$jexEHJ)Oq^_OU8>e&Oj9s9 zgbDF?&=FArCNxHlcUYa2b~s|hHb7pw47^Vfb3z(^|ukal||*XxG#M8BqxmTzxptiSLyx4=pt_ zpq5`!jQK6NEM{t>4-AxN|A}r8cr6PA%cMHkOSynM6ahWaLQiJUbu6dYiwG^jG3loF zg7w*$P*yeC;m<3hrIJUwoEtApB%hB6 zr9qXEj<`%}oFxFG!lOD}-ZKp`=zR^YU%*8XFPb7@njk;G4rK zQ7foc1%$Ek@%4Ow7g{0p^w#qOZ<3dBy|7cg%tY}$ssYLa>8fFTUf^? zCFBcshixn#39^(-J^su%a!h=W*PVzU?^J!lEbiU`c4o${sH7xyIby7P!MeNO?&q>c zQv;_C+t7V_FT`}9`Iu+Tqh`-QYj&vP%9T|{5F(X$T@Xym2#ZI9otQfcFvBjm=I_j* zj+uy4hanjJ3iw&*VT$*@50%s)#*-V1HkqW{fw=fjdaTPSFdZtXqi4r>*PVnJ6AM+% zwZ}f}7m;x%j!+?&`~5D3+D=ukv0&RWI+pH%nyBYnT5Ca?rI$%I8+*BIQk`xFUosxW zt2M~Y3O67AQoUm1JJ9aUCGS8x583-+*<8>UX(8?@_qTLnN5bdUk_0}T8^c2Z02eP@j_9SjCjU>Z1Ul+blH z^-W?N3)Uo#eBsx2q>%y9Iw(0^G8P@x5qBnF!p#B%kwd~Zsu{i1iG8g#ljEtB(bO#V z+GLDEu!dmxF(Nw_{aI&PYw7(#JYZ3f;}K(Q#3|ncjjL)R7n4E=-=|J0P57p6Pqz`9 z&dI*Wj7mIYDJ7hU0A5Vr}#cJXw>UF|x#906}B6^%4uH~7s2kEp;#NA)74MbK7?VE|9)87FR9)_SkAT5q;c}~9IuIgFqVW8d>N|^pvF4(0)w^slvN6L!Cy<)kE-m})7%di^k?k%{YsSrfnT>O5qIC3(Wr>0i)(;s~NHu#xT9 zU9#dEv^g;mU*cbgP~N~!w2b}K3TBq2gGD6TNt5*1l`?DbBA#bNSs;v#Z|+o+HSt}(SzDU$q>p=CCoPlAXXi9K z{D!P7z8i7|rg77g4CT9#8`x+Qt*UalLA=P!)KADxWy%beQyCJSLp0DgntZ7+IZ~(o`$;1x5wwbVCjgLhIj_*CJ5f{dlCtQgtLNj|C zE2TulvaDWfi4YjoGb9C{ApP#Azm&Xq^mDQh08`%20cK@L&CH}ufwr?6$^p~Z?23h1 zFuKuU6 Date: Mon, 23 Aug 2021 20:25:52 +0800 Subject: [PATCH 817/823] support tf installation and separate it from pytorch (#4535) --- python/nano/script/bigdl-nano-init | 8 +++++++- python/nano/setup.py | 23 ++++++++++++++--------- 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/python/nano/script/bigdl-nano-init b/python/nano/script/bigdl-nano-init index 7b6ca5397fe..3ccfc0dfd6e 100644 --- a/python/nano/script/bigdl-nano-init +++ b/python/nano/script/bigdl-nano-init @@ -91,7 +91,13 @@ OPENMP=0 JEMALLOC=0 # Find conda dir -BASE_DIR="$(dirname "$0")/../" +if [ $0 != $BASH_SOURCE ]; then + # this script is being sourced + BASE_DIR="$(dirname "$BASH_SOURCE")/.." +else + # this script is being executed + BASE_DIR="$(dirname "$0")/../" +fi echo "conda dir found: $BASE_DIR" LIB_DIR=$BASE_DIR/lib diff --git a/python/nano/setup.py b/python/nano/setup.py index a78642e6711..d857ed9f768 100644 --- a/python/nano/setup.py +++ b/python/nano/setup.py @@ -104,14 +104,18 @@ def setup_package(): f"-cp{py_version.major}{py_version.minor}m-linux_x86_64.whl" torchvision_link = "https://download.pytorch.org/whl/" + torchvision_links[torchvision_whl_name] - install_requires_list = ["pytorch_lightning", - "opencv-python-headless", - "PyTurboJPEG", - "opencv-transforms", - "intel-openmp", - "torchvision", - f"torch_ipex @ {ipex_link}", - f"torchvision @ {torchvision_link}"] + install_requires = ["intel-openmp"] + + tensorflow_requires = ["intel-tensorflow"] + + pytorch_requires = ["pytorch_lightning", + "opencv-python-headless", + "PyTurboJPEG", + "opencv-transforms", + "torchvision", + f"torch_ipex @ {ipex_link}", + f"torchvision @ {torchvision_link}"] + download_libs() @@ -122,7 +126,8 @@ def setup_package(): author='', author_email='', url='https://github.com/intel-analytics/analytics-zoo/tree/bigdl-2.0', - install_requires=install_requires_list, + install_requires=install_requires, + extras_require={"tensorflow": tensorflow_requires, "pytorch": pytorch_requires}, packages=get_nano_packages(), package_data={"bigdl.nano": ["libs/libjemalloc.so"]}, package_dir={'': 'src'}, From 114c1e8172795342dfc87278c0ac6dfc3e982de2 Mon Sep 17 00:00:00 2001 From: dding3 Date: Mon, 23 Aug 2021 13:40:46 -0700 Subject: [PATCH 818/823] fix jekins failure --- .../analytics/bigdl/dllib/utils/python/api/BigDLSerde.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/python/api/BigDLSerde.scala b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/python/api/BigDLSerde.scala index 62f4a6f49ba..cd842b377d9 100644 --- a/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/python/api/BigDLSerde.scala +++ b/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/utils/python/api/BigDLSerde.scala @@ -99,7 +99,7 @@ private[spark] abstract class BigDLSerDeBase { */ object BigDLSerDe extends BigDLSerDeBase with Serializable { - val PYSPARK_PACKAGE = "bigdl.utils.common" + val PYSPARK_PACKAGE = "bigdl.dllib.utils.common" val LATIN1 = "ISO-8859-1" /** @@ -108,7 +108,7 @@ object BigDLSerDe extends BigDLSerDeBase with Serializable { private[python] abstract class BigDLBasePickler[T: ClassTag] extends IObjectPickler with IObjectConstructor { - val PYSPARK_PACKAGE = "bigdl.utils.common" + val PYSPARK_PACKAGE = "bigdl.dllib.utils.common" val LATIN1 = "ISO-8859-1" private val cls = implicitly[ClassTag[T]].runtimeClass From 943646479acc582e1f7612ae97e11f4ebd603499 Mon Sep 17 00:00:00 2001 From: dding3 Date: Mon, 23 Aug 2021 15:50:35 -0700 Subject: [PATCH 819/823] update path in python scripts --- python/dllib/src/python-zip.xml | 2 +- python/dllib/src/setup.py | 14 +++++++------- python/dllib/test/dev/release/release.sh | 14 +++++++------- scala/dllib/pom.xml | 2 +- scala/make-dist.sh | 3 ++- 5 files changed, 18 insertions(+), 17 deletions(-) diff --git a/python/dllib/src/python-zip.xml b/python/dllib/src/python-zip.xml index b7ae153f09b..6653be76549 100644 --- a/python/dllib/src/python-zip.xml +++ b/python/dllib/src/python-zip.xml @@ -15,7 +15,7 @@ test/** /.. - ${project.build.directory}/../../../python + ${project.build.directory}/../../../python/dllib/ diff --git a/python/dllib/src/setup.py b/python/dllib/src/setup.py index d343be18dc4..801b273c87b 100755 --- a/python/dllib/src/setup.py +++ b/python/dllib/src/setup.py @@ -22,11 +22,11 @@ from setuptools import setup -TEMP_PATH = "bigdl/share" -bigdl_home = os.path.abspath(__file__ + "/../../") +TEMP_PATH = "src/bigdl/share" +bigdl_home = os.path.abspath(__file__ + "/../../..") try: - exec(open('bigdl/version.py').read()) + exec(open('src/bigdl/version.py').read()) except IOError: print("Failed to load Bigdl version file for packaging. You must be in Bigdl's python dir.") sys.exit(-1) @@ -45,7 +45,7 @@ pip install dist/*.tar.gz""" def build_from_source(): - code_path = bigdl_home + "/python/bigdl/utils/common.py" + code_path = bigdl_home + "/python/dllib/src/bigdl/dllib/utils/common.py" print("Checking: %s to see if build from source" % code_path) if os.path.exists(code_path): return True @@ -63,14 +63,14 @@ def init_env(): if os.path.exists(TEMP_PATH): rmtree(TEMP_PATH) copytree(dist_source, TEMP_PATH) - copyfile(bigdl_home + "/python/bigdl/dllib/nn/__init__.py", TEMP_PATH + "/__init__.py") + copyfile(bigdl_home + "/python/dllib/src/bigdl/dllib/nn/__init__.py", TEMP_PATH + "/__init__.py") else: print("Do nothing for release installation") def get_bigdl_packages(): bigdl_python_home = os.path.abspath(__file__)[:-8] bigdl_packages = ['bigdl.share'] - for dirpath, dirs, files in os.walk(bigdl_python_home + "bigdl"): + for dirpath, dirs, files in os.walk(bigdl_python_home + "dllib/src/bigdl"): package = dirpath.split(bigdl_python_home)[1].replace('/', '.') if "__pycache__" not in package: bigdl_packages.append(package) @@ -93,7 +93,7 @@ def setup_package(): dependency_links=['https://d3kbcqa49mib13.cloudfront.net/spark-2.0.0-bin-hadoop2.7.tgz'], include_package_data=True, # package_data={"bigdl.share": ['bigdl/share/lib', 'bigdl/share/conf', 'bigdl/share/bin']}, - package_data={"bigdl.share": ['bigdl/share/lib']}, + package_data={"bigdl.share": ['dllib/src/bigdl/share/lib']}, classifiers=[ 'License :: OSI Approved :: Apache Software License', 'Programming Language :: Python :: 2.7', diff --git a/python/dllib/test/dev/release/release.sh b/python/dllib/test/dev/release/release.sh index 7b7ec4f62d1..6a353d4c6f7 100755 --- a/python/dllib/test/dev/release/release.sh +++ b/python/dllib/test/dev/release/release.sh @@ -19,9 +19,9 @@ set -e RUN_SCRIPT_DIR=$(cd $(dirname $0) ; pwd) echo $RUN_SCRIPT_DIR -BIGDL_DIR="$(cd ${RUN_SCRIPT_DIR}/../../../..; pwd)" +BIGDL_DIR="$(cd ${RUN_SCRIPT_DIR}/../../../../..; pwd)" echo $BIGDL_DIR -BIGDL_PYTHON_DIR="$(cd ${BIGDL_DIR}/python; pwd)" +BIGDL_PYTHON_DIR="$(cd ${BIGDL_DIR}/python/dllib; pwd)" echo $BIGDL_PYTHON_DIR if (( $# < 2)); then @@ -32,7 +32,7 @@ fi platform=$1 spark_profile=$2 quick=$3 -bigdl_version=$(python -c "exec(open('$BIGDL_DIR/python/bigdl/version.py').read()); print(__version__)") +bigdl_version=$(python -c "exec(open('$BIGDL_DIR/python/dllib/src/bigdl/version.py').read()); print(__version__)") cd ${BIGDL_DIR}/scala if [ "$platform" == "mac" ]; then @@ -61,12 +61,12 @@ sdist_command="python setup.py sdist" echo "packing source code: ${sdist_command}" $sdist_command -if [ -d "${BIGDL_DIR}/python/build" ]; then - rm -r ${BIGDL_DIR}/python/build +if [ -d "${BIGDL_DIR}/python/dllib/build" ]; then + rm -r ${BIGDL_DIR}/python/dllib/build fi -if [ -d "${BIGDL_DIR}/python/dist" ]; then - rm -r ${BIGDL_DIR}/python/dist +if [ -d "${BIGDL_DIR}/python/dllib/dist" ]; then + rm -r ${BIGDL_DIR}/python/dllib/dist fi wheel_command="python setup.py bdist_wheel --plat-name ${verbose_pname}" diff --git a/scala/dllib/pom.xml b/scala/dllib/pom.xml index ff726e4477a..fc450d88fbf 100644 --- a/scala/dllib/pom.xml +++ b/scala/dllib/pom.xml @@ -261,7 +261,7 @@ false - ${project.basedir}/../../python/python-zip.xml + ${project.basedir}/../../python/dllib/python-zip.xml diff --git a/scala/make-dist.sh b/scala/make-dist.sh index e2f5c5c7426..a3a844c06b7 100755 --- a/scala/make-dist.sh +++ b/scala/make-dist.sh @@ -53,7 +53,8 @@ if [ $MVN_INSTALL -eq 0 ]; then exit 1 fi -mvn clean package -DskipTests $* +#mvn clean package -DskipTests $* +mvn package -DskipTests $* BASEDIR=$(dirname "$0") DIST_DIR=$BASEDIR/../dist/ From 5525ce20ac9130701147ff22deaeb810c7a83ed2 Mon Sep 17 00:00:00 2001 From: dding3 Date: Mon, 23 Aug 2021 17:11:55 -0700 Subject: [PATCH 820/823] fix build whl --- python/dllib/src/MANIFEST.in | 2 +- python/dllib/src/python-zip.xml | 8 ++++---- python/dllib/src/setup.py | 6 +++--- scala/dllib/pom.xml | 2 +- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/python/dllib/src/MANIFEST.in b/python/dllib/src/MANIFEST.in index fe4bb943569..acf1748209a 100644 --- a/python/dllib/src/MANIFEST.in +++ b/python/dllib/src/MANIFEST.in @@ -1 +1 @@ -recursive-include bigdl/share * +recursive-include src/bigdl/share * diff --git a/python/dllib/src/python-zip.xml b/python/dllib/src/python-zip.xml index 6653be76549..8924787d2c5 100644 --- a/python/dllib/src/python-zip.xml +++ b/python/dllib/src/python-zip.xml @@ -11,11 +11,11 @@ **/*.py - - test/** - + + + /.. - ${project.build.directory}/../../../python/dllib/ + ${project.build.directory}/../../../python/dllib/src/ diff --git a/python/dllib/src/setup.py b/python/dllib/src/setup.py index 801b273c87b..677c6e7e4ca 100755 --- a/python/dllib/src/setup.py +++ b/python/dllib/src/setup.py @@ -68,9 +68,9 @@ def init_env(): print("Do nothing for release installation") def get_bigdl_packages(): - bigdl_python_home = os.path.abspath(__file__)[:-8] + bigdl_python_home = os.path.abspath(__file__)[:-8] + "src/" bigdl_packages = ['bigdl.share'] - for dirpath, dirs, files in os.walk(bigdl_python_home + "dllib/src/bigdl"): + for dirpath, dirs, files in os.walk(bigdl_python_home + "bigdl"): package = dirpath.split(bigdl_python_home)[1].replace('/', '.') if "__pycache__" not in package: bigdl_packages.append(package) @@ -93,7 +93,7 @@ def setup_package(): dependency_links=['https://d3kbcqa49mib13.cloudfront.net/spark-2.0.0-bin-hadoop2.7.tgz'], include_package_data=True, # package_data={"bigdl.share": ['bigdl/share/lib', 'bigdl/share/conf', 'bigdl/share/bin']}, - package_data={"bigdl.share": ['dllib/src/bigdl/share/lib']}, + package_data={"bigdl.share": ['src/bigdl/share/lib']}, classifiers=[ 'License :: OSI Approved :: Apache Software License', 'Programming Language :: Python :: 2.7', diff --git a/scala/dllib/pom.xml b/scala/dllib/pom.xml index fc450d88fbf..5c6d68ba9ae 100644 --- a/scala/dllib/pom.xml +++ b/scala/dllib/pom.xml @@ -261,7 +261,7 @@ false - ${project.basedir}/../../python/dllib/python-zip.xml + ${project.basedir}/../../python/dllib/src/python-zip.xml From 44617b8edf3fc309005e0ac0f6ce3bffac26360c Mon Sep 17 00:00:00 2001 From: dding3 Date: Mon, 23 Aug 2021 18:33:23 -0700 Subject: [PATCH 821/823] fix build whl issue II --- python/dllib/src/MANIFEST.in | 2 +- python/dllib/src/setup.py | 12 ++++++------ python/dllib/test/dev/release/release.sh | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/python/dllib/src/MANIFEST.in b/python/dllib/src/MANIFEST.in index acf1748209a..fe4bb943569 100644 --- a/python/dllib/src/MANIFEST.in +++ b/python/dllib/src/MANIFEST.in @@ -1 +1 @@ -recursive-include src/bigdl/share * +recursive-include bigdl/share * diff --git a/python/dllib/src/setup.py b/python/dllib/src/setup.py index 677c6e7e4ca..f5d3c315211 100755 --- a/python/dllib/src/setup.py +++ b/python/dllib/src/setup.py @@ -22,11 +22,11 @@ from setuptools import setup -TEMP_PATH = "src/bigdl/share" -bigdl_home = os.path.abspath(__file__ + "/../../..") +TEMP_PATH = "bigdl/share" +bigdl_home = os.path.abspath(__file__ + "/../../../..") try: - exec(open('src/bigdl/version.py').read()) + exec(open('bigdl/version.py').read()) except IOError: print("Failed to load Bigdl version file for packaging. You must be in Bigdl's python dir.") sys.exit(-1) @@ -68,7 +68,7 @@ def init_env(): print("Do nothing for release installation") def get_bigdl_packages(): - bigdl_python_home = os.path.abspath(__file__)[:-8] + "src/" + bigdl_python_home = os.path.abspath(__file__)[:-8] bigdl_packages = ['bigdl.share'] for dirpath, dirs, files in os.walk(bigdl_python_home + "bigdl"): package = dirpath.split(bigdl_python_home)[1].replace('/', '.') @@ -81,7 +81,7 @@ def get_bigdl_packages(): def setup_package(): metadata = dict( - name='BigDL', + name='BigDL-dllib', version=VERSION, description='Distributed Deep Learning Library for Apache Spark', author='BigDL Authors', @@ -93,7 +93,7 @@ def setup_package(): dependency_links=['https://d3kbcqa49mib13.cloudfront.net/spark-2.0.0-bin-hadoop2.7.tgz'], include_package_data=True, # package_data={"bigdl.share": ['bigdl/share/lib', 'bigdl/share/conf', 'bigdl/share/bin']}, - package_data={"bigdl.share": ['src/bigdl/share/lib']}, + package_data={"bigdl.share": ['bigdl/share/lib']}, classifiers=[ 'License :: OSI Approved :: Apache Software License', 'Programming Language :: Python :: 2.7', diff --git a/python/dllib/test/dev/release/release.sh b/python/dllib/test/dev/release/release.sh index 6a353d4c6f7..3f1fa07e418 100755 --- a/python/dllib/test/dev/release/release.sh +++ b/python/dllib/test/dev/release/release.sh @@ -21,7 +21,7 @@ RUN_SCRIPT_DIR=$(cd $(dirname $0) ; pwd) echo $RUN_SCRIPT_DIR BIGDL_DIR="$(cd ${RUN_SCRIPT_DIR}/../../../../..; pwd)" echo $BIGDL_DIR -BIGDL_PYTHON_DIR="$(cd ${BIGDL_DIR}/python/dllib; pwd)" +BIGDL_PYTHON_DIR="$(cd ${BIGDL_DIR}/python/dllib/src; pwd)" echo $BIGDL_PYTHON_DIR if (( $# < 2)); then From 0ad8ae969b1c406cda04806766468e5a7c807712 Mon Sep 17 00:00:00 2001 From: dding3 Date: Mon, 23 Aug 2021 20:01:16 -0700 Subject: [PATCH 822/823] fix test pip failure --- python/dllib/src/setup.py | 6 +++--- scala/make-dist.sh | 3 +-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/python/dllib/src/setup.py b/python/dllib/src/setup.py index f5d3c315211..c30929aaacb 100755 --- a/python/dllib/src/setup.py +++ b/python/dllib/src/setup.py @@ -22,7 +22,7 @@ from setuptools import setup -TEMP_PATH = "bigdl/share" +TEMP_PATH = "bigdl/dllib/share" bigdl_home = os.path.abspath(__file__ + "/../../../..") try: @@ -69,7 +69,7 @@ def init_env(): def get_bigdl_packages(): bigdl_python_home = os.path.abspath(__file__)[:-8] - bigdl_packages = ['bigdl.share'] + bigdl_packages = ['bigdl.dllib.share'] for dirpath, dirs, files in os.walk(bigdl_python_home + "bigdl"): package = dirpath.split(bigdl_python_home)[1].replace('/', '.') if "__pycache__" not in package: @@ -93,7 +93,7 @@ def setup_package(): dependency_links=['https://d3kbcqa49mib13.cloudfront.net/spark-2.0.0-bin-hadoop2.7.tgz'], include_package_data=True, # package_data={"bigdl.share": ['bigdl/share/lib', 'bigdl/share/conf', 'bigdl/share/bin']}, - package_data={"bigdl.share": ['bigdl/share/lib']}, + package_data={"bigdl.dllib.share": ['bigdl/dllib/share/lib']}, classifiers=[ 'License :: OSI Approved :: Apache Software License', 'Programming Language :: Python :: 2.7', diff --git a/scala/make-dist.sh b/scala/make-dist.sh index a3a844c06b7..e2f5c5c7426 100755 --- a/scala/make-dist.sh +++ b/scala/make-dist.sh @@ -53,8 +53,7 @@ if [ $MVN_INSTALL -eq 0 ]; then exit 1 fi -#mvn clean package -DskipTests $* -mvn package -DskipTests $* +mvn clean package -DskipTests $* BASEDIR=$(dirname "$0") DIST_DIR=$BASEDIR/../dist/ From 0b7216fbcc14380f37cbfd4a706c4a73e9f9605e Mon Sep 17 00:00:00 2001 From: dding3 Date: Mon, 23 Aug 2021 20:41:37 -0700 Subject: [PATCH 823/823] fix path in test scripts --- python/dllib/test/dev/prepare_env.sh | 4 ++-- python/dllib/test/dev/run-caffe.sh | 4 ++-- python/dllib/test/dev/run-keras.sh | 2 +- python/dllib/test/dev/run-tests | 20 ++++++++++---------- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/python/dllib/test/dev/prepare_env.sh b/python/dllib/test/dev/prepare_env.sh index 4e3194c7d04..b6ffcd8ddd3 100755 --- a/python/dllib/test/dev/prepare_env.sh +++ b/python/dllib/test/dev/prepare_env.sh @@ -16,9 +16,9 @@ SCRIPT_DIR=$(dirname ${BASH_SOURCE[0]}) echo "SCRIPT_DIR": $SCRIPT_DIR -export DL_PYTHON_HOME="$(cd ${SCRIPT_DIR}/../../; pwd)" +export DL_PYTHON_HOME="$(cd ${SCRIPT_DIR}/../../src; pwd)" -export BIGDL_HOME="$(cd ${SCRIPT_DIR}/../../..; pwd)" +export BIGDL_HOME="$(cd ${SCRIPT_DIR}/../../../..; pwd)" echo "BIGDL_HOME: $BIGDL_HOME" echo "SPARK_HOME": $SPARK_HOME diff --git a/python/dllib/test/dev/run-caffe.sh b/python/dllib/test/dev/run-caffe.sh index 6ad394782d1..d2c08253a2c 100755 --- a/python/dllib/test/dev/run-caffe.sh +++ b/python/dllib/test/dev/run-caffe.sh @@ -28,8 +28,8 @@ echo "${cyan}Using python version: $p${reset}" export PYTHON_EXECUTABLE=$p export PYSPARK_PYTHON=$p export PYSPARK_DRIVER_PYTHON=$p -$p -m pytest -v ../../../python/test/bigdl/caffe/ \ ---ignore=../../../python/test/bigdl/caffe/caffe_layers.py +$p -m pytest -v ../../../python/dllib/test/bigdl/caffe/ \ +--ignore=../../../python/dllib/test/bigdl/caffe/caffe_layers.py exit_status=$? echo "running caffe layer unit tests" if [ $exit_status -ne 0 ]; diff --git a/python/dllib/test/dev/run-keras.sh b/python/dllib/test/dev/run-keras.sh index 300e2af3f4d..388b6099874 100755 --- a/python/dllib/test/dev/run-keras.sh +++ b/python/dllib/test/dev/run-keras.sh @@ -32,7 +32,7 @@ echo "${cyan}Using python version: $p${reset}" export PYTHON_EXECUTABLE=$p export PYSPARK_PYTHON=$p export PYSPARK_DRIVER_PYTHON=$p -$p -m pytest -v ../../../python/test/bigdl/keras +$p -m pytest -v ../../../python/dllib/test/bigdl/keras exit_status=$? if [ $exit_status -ne 0 ]; diff --git a/python/dllib/test/dev/run-tests b/python/dllib/test/dev/run-tests index 0e77a96fcae..34605089df5 100755 --- a/python/dllib/test/dev/run-tests +++ b/python/dllib/test/dev/run-tests @@ -33,17 +33,17 @@ do export PYTHON_EXECUTABLE=$p export PYSPARK_PYTHON=$p export PYSPARK_DRIVER_PYTHON=$p - $p -m pytest -v --junitxml result_bigdl_${p}.xml --doctest-modules ../../../python/bigdl \ - --ignore=../../../python/bigdl/dllib/feature/dataset/ \ - --ignore=../../../python/bigdl/dllib/utils/tf_utils.py \ - --ignore=../../../python/bigdl/dllib/keras \ - --ignore=../../../python/test/bigdl/keras/test_application.py \ - --ignore=../../../python/bigdl/dllib/examples/ \ - --ignore=../../../python/bigdl/dllib/models/ && \ + $p -m pytest -v --junitxml result_bigdl_${p}.xml --doctest-modules ../../../dllib/src/bigdl \ + --ignore=../../../dllib/src/bigdl/dllib/feature/dataset/ \ + --ignore=../../../dllib/src/bigdl/dllib/utils/tf_utils.py \ + --ignore=../../../dllib/src/bigdl/dllib/keras \ + --ignore=../../../dllib/test/bigdl/keras/test_application.py \ + --ignore=../../../dllib/src/bigdl/dllib/examples/ \ + --ignore=../../../dllib/src/bigdl/dllib/models/ && \ $p -m pytest -v --junitxml result_test_${p}.xml ../../../python/test/ \ - --ignore=../../../python/test/bigdl/caffe/ \ - --ignore=../../../python/test/bigdl/keras/ \ - --ignore=../../../python/test/bigdl/test_utils.py + --ignore=../../../dllib/test/bigdl/caffe/ \ + --ignore=../../../dllib/test/bigdl/keras/ \ + --ignore=../../../dllib/test/bigdl/test_utils.py exit_status=$? if [ $exit_status -ne 0 ];

-CmJj_pF0u?80G!zwz= z!jHjU$8aMtQ6q8q6j;k9L#lv^C65`Szuc^7`L#tVkKVY(_5d;a(+h8oeHHKB`uhQK z4z*q_aCws}$#~_RtfG8#5dZ5T$p^mcrt;sJHkIinjUI|cRG<@^Bm+5ED8pJm4j9K# zwHN1v5S-!39|T4sNc{?q;pOOyC}3atj<;J^pM6#%7MS67P12RgM+BB3ENsG4wEStL zP{E*WYag{Z9zj#4WP)S0pL{)8_&`jfI*p`p9gpY%RmMX;I2&ruw%bRz)a+^ zYdm2?JGq0`zd)} zwJyQYXF{lQi9|pr$AdtJ&<(RFCa0fhA6Oc#GJ_i=I_XQ%=;uBW~>yL$;-LK!cNv=^F(Rra)y)xeogYxh;s6G3mSiW@R;{NDks*1E);k)0+vK@b)4=I+ZeUR7}FIL8EsB^ z8xZD?yGG`1DL)5V(>NU66~54#n$womGTtTAPd#7s_u_pFQBgr)pe^|4es=bEQ7&g{ zEN65_uA*a=2j$5WEhZ}M*RRMkrAVPabiJy9AlA_!#E4^XIQjlQ|3rrDNa%IbC1du2Xy8CmmV#Qm$%$VS(c9y0Y+;-~dX888?M z3mFE}zCX&M!M`>WUTK?5XsJ-$d!fY?{-{R9@gnka??1mQ6Yj;$3(L3J)4x@@UrLYW ziX6>BsaxJRJZCYSffd_-)Mh7FTGDn4Gh;}(u+p+l{1}QRa}u6?w6nnEHyT21CA1nQ zd^c62WG?2Kpm2nH1x+*?Uu!Y&tHM@a_P)G}$smg^?&)z|2k7oR!yS%~UoHtu zj;C=8Wi2s|TK^i4zQCeAR(Pu|@WIC<)wGMpv1{+%GdoZeNfKS)Q^|1`uf3Za zx6kv?W$F0PP+DGq4UA)~c_75cn;hO1_#pHa7*~(AjWsldi=aX)PCG6+`A=uVSft(& zS5Hg)A4gXi*3{d^2LcX6X&50f5D=+R(m6sJWptNxcb70yy1TnWy7|*7jfB#vr1X1y z_gybNJA0n{{-uj)hfek9^M`FC76ZgL*P#E}=~a;x3n~`1A?4^7r|+3di8`+Q{D@q0 z`vR+9vUWMRn5&Ig^E#SMRIQRQmbbt@Pn&`mH_Ws}8#5;%96TN!J!?p6Qsn7%7sjt5 z=9bG1VWE5-^zp+X_SkK7&;uM9wRAYEwA%26^`;KKwqr_@YdOE)@{_Kp65R|WAA@B( zhDjr}f*%!U9@^oQY{_2#Zk2U7MjnH7obxZyD>d%eC{M(~T0>DFVFWz`+3pNYF;phIu9c7IDA5*#CDk{@(1fTBzD* z)oF9Jv9nWpK_lpMdjW*=3-a-$%G3YLyROu3{(i?02hU{jc*{^VNNgf+$ap(dHE+62 z@8aPw)9L+KVU^ruPtk!SoL7|;3*Nd-0c>>7XzZnHR4z?xUcbo*LxmS{p=<8amT46_ z5$s>O6Jngqq7p@Ls2)wrzmV$}s~aEwOn)~8>MXFZH~#K7Avm!;^A1xQQ{6D8 z*`=4qYNRxnW7on$79I@z(yER}6A!Ry@yxL~Bf$1l;1{=1Osh!b_d8;~^KH7YX|0HF z`_h^D@!~fpc>gi$!k+h2{X7mcjjAzi7<;$NF7B^uHDGSbM-e?mudtXB#gs;sS|k!p z52Se6^36CdqkxuQ#)b5*z@*kIrPE`MP-~+4Zj**+^b^FB#~G6SfG<^ctb~ne%5Z%x zQ%O-#_+~5qpAX>Z=;-%vF zD5CLqpM+V=c<9&%ot#aKtkrGQ6Ys`0r~80|$(v!Cq$zwlpAk0*2_Qt`NSJdbM4a$c zo@yZ192*H0@(#vLo3GD#hxJQ#1qOLWW4>4j21_GV=*Y%#V%9XsLTo=^8oPLBI#ubs zrS0uB#22N?AR+p5#E#r=8#Y>T;e))98F@2C(P^w)p8ny>bFVAKf-1SjU(dke`tjf? z$iXCQu>Y9ahi;7Ta5R(vWK1MZk5@DvZlzNam^)u$Qf5@-kgT#1O8(^W$|=L6$uT3W zy=CTMB*Kw0QBa=dd~q8rv0~cb#sFG@Oh}MP`4%f$l9W3jja%Y2qt@~kWLVb62+A{9 z=~4^J=n>o>23rWnU6%CD4{pSPFY{o(yF=si)H4fB0{$u!o$Wtf~U)1dTc%E&Aev|u1{Kr6X z(v3az$Ixm8c`1`<-;wbJTbA(MKj4*hauVzFVHqg88&9(R8cjo!yxx=4!Z0DvJjHa1 zJk_H^uUQ!K-vIkGU+PvhmP(I3MCZ^lvxoO(s@;!t<=Z6X~fp&J9X+FXcm5Yrkv-M5VoP#+q zRIyvpcMVc4F!<3+#X-ogRcHR!Onu9ppB^R}M-gX}f){*>@9l2pI7vrhJxw1gsCGN+%> zL5i_;rZd$KISm8Yo+QgxmE?g1bpi8qL2EwG0n!?XjQ}b+dn_Ao_m%uSPuGM%Q-f~* z%8Ay1KCrIGzrVhQ?E&NUzkDbliQL7(K~^Fk7tZ;OwAoW&r^Z|dFWCyOv@AS|>k!bV z_}r|l-xHweYkeY2YLl5DmB3$0l*H+r=l}e@{!4Fuy%NLcNE8kzK7=_4j0<_mW^5Mb zQ$iQmgu1q9H*-@cQ(dsIfB&f@jNCHP>onemD#Y7Z!nH;^d}xp-y$|I?CW|kXVL6y} zuYA@YQ-eU5+Q{V98bSWd-mpm>c6pNxpHz@Ie+GxE@z64R%7KOXX3hj8PtThg%1!Kk z8ujBlJC&}P0GnsOlF14g9dx)L+ZQ3qwmKM>c8YJ}p7l8PhlK&9Z79aa1HE}t$@RjA zDJ|h^fvbdy{tM8Bt3`kq!P1T)8tCwn|9$&4GmEV5!f=ByxM^&x?2p5|sR|oaD;oJe ziyDvITCe5yj3wg?Z~E-)|4#{kLs`4K@%9A{W`G0O{`pf}>>Q|J0zSYTBC*nxrtfjS zGY-sqYn(sqWSQvv2O%tbsaG=V+RMv{UtFq~M879f&qa6vGM^!#>_>UJ!BqTK;tZyf7~HLf+v3qc(JlK} zF44%6w}Ff4Jh4M16p9U-_BT9CO$-8xp7~Mq(wAB4UwMR%lg(nvnh3;sf!g$oy-nas z1C9YeJN4?-t18_#3<=)~*v`MbI@jw@x&Fh!`s0e9m$UPCmmdH&_gR8`ukysDV zR68AfcY|)hYG~@^bAXQRX%nDne&MWTE0*OkQa7N94k{jw@G;OIou0BmQ9z-FZO@@; zd6}emMreEG8mFhHZDrPJtaS}h3=P8AKF1+?vaOgMDV7Rnq3>pPIGp&0=wOz)M(Fx4 zAKWHj*UIHUqel5`eH-c(c*Np&plfBg6wSf#s{o;ig6o{pG}nR{W<+$EGd9RaRuU344}D= zDO{YTk4e?Rvw`943cyn%$lu!v`aKH%lW!Xzb^qVi!S*u zag8rS3bXi^2#(P@LR>mDn{J`S=Fwh@ELXE<#$|n zi{w_Z4>2Y*Gl)-gYu6TCSqF`k3&_ltj<8x)tBUQU33)-$*JG zCj$@rPn-RnX@3tcmFVBJQM6GOr6kb?E8uKLsxPads{Z<@{MDG#_D7 za*MkWjW!V}_kVr%QK7Bv?Seu=J{N#YKbh@ht>f-$<?L~Q+eA1 zhXWuFc-mqR0pLF1la@6&=7qo%7YCd7$FgH}TmFFNeIgA`H8!S3g^RVg`aH#d6>j%A zPvSmic)B}P?7Y2~($y9Ab9H}!yf6+c<$9-qr_%V|XYlj35&v5(=c@gAg_UqT*wC1n zmUJxQ@THczTKmB~$zj}x84x$>7XX_Wsnf2iF@D}V76PS3gTwp$jj)(mV0|0xME#>)PjqUN%3 z-fy4+O^wLHE4i_KL_Wb^R-v*J%g9hy{6fV$Ml(jM4K^XA&Vh%Cf+nc3{#%sza*#+( zJ&l4F$W4vyAmi3RbM?KKFZr_@5=or!1l+KC?e2d7oo9Z2-#-Z_Q1$SM16Vr&kw102gE_x<-_Q!YsrwQWS|wh*EIQ7ztoiBh*MEK20ATAY)Z*)`kn+VCE=HqD&(@n|d_kWFp%%4b;fE83EUM3uP|wb) zC}7TadT=d9g~>D$SyieXqfw8@zNg1e7iF)FMf?11J8oZ2SS)FSFOJ7aF|lm$YYxvY z(aOE$=t`{bNF~P&f>07$_u}#1`|@mibihc3MCdQjaz`hVr#ah;niEKAf`Oo8F)jR} zCHVlRL1V(s`%8uF+>>~CE|wNDG=Q6@V@mrCU&K`VO~~t=)Oh@*rokww(QBu>c6nh{ zBhN6p&5f2V=65{V*k(O=w0Y0eV{;2qcgx8Y?9g!XPdQr|FyXWB$E-OMFvh7>9QHOg z4D9Sq9UUI`i?j7~bbvz|C=))dY+eJx0fApg_~Jir81T4%xcmT|`b`Bw!otFQd=@J_ zb2#Ew*#00)Xc+{`2t6#KIr8eKMZ)HDj9)3Nj9ueA?gw2zoju)8e(k!!KySAzp0(<< z;<{E?5z5WE&Pp4EljJXGAws&zP@h4+8{7efB zo*2!MJAT`qZpo=ZF1|{gUVQUWRd}*rOU})^DXwRfTELlKDQd_w{7m&x+@8w9)-kA259a zM>lRSU>u$00%TzlCByVT*ZGPQ<DWQSV^b` z&!8%TJA0BSl^aywjG3;mLjaOVstHF@#4i!#G?Qx^TOSKMw@Pi*4+whYiL#G8r>VttAi} zngsw>%sh)UPk!!T!{R-eS5pl(U&wMz9~tZf!p|pZ<8qbWCDGJacFBu!Ccm|d;?n%WYQ zdIHLlJG^gZy!?R3K%mC_?=B$5^A* zN0H1_GPJs_4D@Hg@Oj={%TIBxLr_s%(MAsFZGwt=*Mlipoj&+rnuK%94LrHwY) z+Isf8h>}CCUfH@DwUEa`%V*8=VPdE>s3%+;N=Dc@ErAs1$H29=Vm9Cn{zk->sh_Z& z{l*h*C2k1E@N%yeUryS!pUVs~0;DIDE+ke{YIfn5{Z|p~X6rg$>3Q>4os!gKC~5sl zVg@eU5lhE$OBCm1`;Fl^2sSA7Y8X`R={L)@yZiZ{9Neh(8*o)B0rGjbHILnm4Unif zcyt?ZiQo4GJ^xB6R91EX91Z_EGJq+OOHC(p#gg(gq6`a@CeI%i#DR($j%10*GLmdc zDl0>FT9Ms^cHQXvJ&8PB-EVEbTd=EOomfz)_w%k~(uu5X-qCtT7d)5<2>26pK%&S@ zLjz*mfT5#4%i5H&&$Ii^JS*=G!Za(M{a06IyVky+VHH~BvF0VlK8Qx8@$MY;gIUA- zzQiC4`tQ~)#K1iafq!q^tEoKS`kC>VexSacukSR-3X-FsMSjht4bVy zq@)V=01j+3s9$4&04=n)`!y}dAH@EgE-L;j$lJknrpamt%yj}ysq!}3cbe#^&Nx0% zY9}~KG*^yR-EYSq+1B4l4-2~OTaj#WVxrb;_c0pftA$|xZqwk`(v4n8wQ?HW5{^?o z^BKhxctX78KUp+l##t(sRMC*52?k2Ngxr54R+`|>fFA}}6pth4nxbGkfPl#lcx!<% zoI6wTpIyHL5IMvT!=pCID)QpmI~xyM^D6LE{BfxR!4d4BE7Bnt1|I83FSQWh94!YJ zg7l}yh0%!fprv-_A{6F|**u50)esbM64Dk%EaQw*Ey%q+6c3YEDq7`F8I#<0s(A@> z_|00+<|Yla$wnYm9#F#ech@Shlew}iw1Igr!d4Ny;dkQ(c_NN1uJ6CMg~0dYS<0R# zY&wzI=hR?u=2A~5u-bAwpQQ#c5#CuG7v1WlWQAE^biD_lPfvc!<>SncCQ+~n8roo* zMRIt)qnCg<<)n!+am45pcW*$QV+-HMs2bR~G=~u9(|+jU^j_9|h~LZCWACLyiuf$L zdwSa5{z{}Bt2#&ri$y=tHc}%wK3?AV$*A?#+vzXF=C~PuEX}c3n?bEBPHhsJ06Vy2P_V9d*onA$bk+ zq{hrIfPOpaU)DcvTz^oDM(SC$Xh<-5%NNhKmVgv~Jwy9LGf{ND)FTYQFZmshMv)4$ zu07gTc;g!-&@tITddT>BVH6Dn9}@aWbjWjkn^~@Mwrt6)yCLy&GIQ0ij}Sh|tiy?H zRvT|j&=;di%f^3lR^S*Y)D|$(Cbp^H0FufU11nf!m?4d(v68qb-1wHeEKn?1PLAgX z?*;bf8(=t1z3(MiWTB|2_Yd#< zrx?f^8@qeX^(H51f0b`{Fh%`p2RR_!YRc0rKyfN z3I_Zm0g*7S9RKnvFEgoQs&jQjY}U9HjTG#IjJTsC15)DH)U`JARatn1n#HzWB+pH09f~T$4yOk09R{f?M$_vUm;fUyvqI%ClPe=R2VXfi<|FT~deanDj zk9UZ-9Y#N_Ioz2D4mP33hq!xnUUB*DD168u)QA&6pXuCxKO*ROLxoZPxpSD0-Q@Hq z4R_3#(Ngkz4*LEJeQsnv8q#K^p+_YH@e*k6r*UDqsn{0R%to0N_eS&u%u5Ebk{k{kHzN@N||vsi=4V_xQaHpr@}? zqlcH$yJ#*#bO{9O;jxS$YG#veh$CvZ%$OJ?VOY!mqkXN0q*%?gB;&}wHHT2^&B4)z z%vb4a#Sdl_p4FoBy*#l_pGPz7=_%7kWOwOdc>CUcnoqA2`<$4BQFWx#quX0hBAYr$ z_KA27Kl#0pkU5{E5nV8?Y>s4*R*Jjztsa9>!GcXn91{gc+W>Xjff8MCacSb1Szv7M46@AS@h}+ z(FHnxDZe#9DUo4llUtdio~hrj{dW#{Ify+ z8%zKOKv7AFjdfvu8E`}aCccdzoHe_&9i_)Db>Iil<#Arrs5WlAFLO~6M^@_ebMFf& zNPInph1dAmpQX{{YrVGmZ-Nimeh=z?5Bg8%o-OX%gMZclHV?lp-(*-Gc;%rg&g$lHW)V$hpyk2uZ>XR9#a<8a4*&TKbdO)3bWDr^QLYTdY|BkPzyS(Ws<63ipv;rKR0-O4KCIMEDu z(#i>n){ZVaXO?N4FL@?)X662iCs=UCj)75a-+u^KyO_{~qrMe-Tulnm)NE%hA83i- zq-_X&UBv;t8cZ2BakRHr#wXwV>FMGz4YJ-?)RLR8^DM!Hg|}iH>=!UHhgR(ehOMH9 z$)jxD|8#b6&{G@8BjtR*c%{1C)tf^`hbA+vMH@xGIWu&gJUDE;j~^h~zqJ%-s?(!?x>*ot-DX9=5#rI8W(%4L?w`YgYXzgaiuK|3tQOUOOUaQ7LRdppLiIq`Em5BDN`sYvL+D$4O&(* zk6%m2D4@xp-5x6tQff=aWv;Egr+ClEpO7|a^T87SLdpwFnjd0g#Dz-4tsA9h2|P^z zz4{F_udlXyoCBrnH^6RV%INoxpe&y3d;;v(hpoPsbLxHoYxA0l$}MvtOC!xTFCtr> zzGWj*=}RPaP7Sqxqj_YQFxU9O!&eda-*d^wp4TqgpL2Yjy6uc< ztjwqi!IM`Z>z;BIc2=Dm)PmB1Q0p*IUPsUz%B!>vTe||2>%*Y)(-7j5J?m0!=sBm> z{t6jGK_#VJVz*aIP?L35&G_Ou+bl|POgFo;NR*~-EyZ7W|8gg#%PS|^f^nR;Mvop7;3XQya=d&JfXhOWdq5#4lvTWoP5x6Pu zx@Uv*Ri?ieuh(H^rAGu9rOyrYR=ZGO?2GTUvf88hCkeovO%jSA#$^4ANC)DYNjf@O z(JJPbTA?OCOzQl+f6X7-t$H6fKKvrD%1WuTo}6HxwZ>DagijEM&&~52-$_a4weYDg z`JUi@_W9g88t$QjQhPmx*{G`#=n|AW`5G6SibnE@3%r{K)Ey~y~}ZC zu}vLoC-7O7LrkUB94Yx@F4y%BV|I`U{U2}RU-lo&*lOSnU0r^F&lrH2C||$U)zhQ@ zIs{OUPM)5(`BV>klaEJ}dOnwk-P@m=fcOiKsHcILL#6*~Vv?kWn~m3ye}kC_kk`&MnxiJ}y>y5vI+ zor480yoU{iUV95ipOxrTQNGrDB?>YyU#(JY8vnufsvu|~==WN7Mn*=bx8`&$rQa(j zt*fqTE3^=HVzfaBmG^?n?0!Ok^#TkDtN8Pxe9p*E!z)?@&hJIiLV zj4)rtZOdB69+d~ip8puVmi0SGQ{*; zuU}Sx|42kwxT<{Lvm=^@SOUZW^0&Xplw5uH0&jC8Rnz$os2_XG>is?$l?1hcm5s&T z&GkC43tg!fPwKC>IyeEIFr5q$KUarZb2hd=XHc*kYR)n0*xN-+BdE^TsTH?FQ${kLR@j6iM=|(d%>uz^$8(aIr^o z(M_|s=(!tg0sL(Xra!o%xO_eB1xu~03bd=+F1NQw1h_7bN+Jj~U%j#w^tckls2IIn zk^U<3_u+Z7{*xwx&?rEq!YSf1Z-4$S^tsK7tO=eCt|)$w7ggJ$SoTA;BFY=q(nX(W zcP9Lu1Qias6&b@1j0`J^r>kj&0@XV>FWU}W+%cB{a|NgoOsY}rKH+b~jOx!b^J3duYLo@~i!p-4`hdYBnXt;Q#lbq_2Qt;kKjRynRww^OqTuV5imIcdqsK~8j(A@xOFTU>nPzJk3xWh> za@Sk@Xt51D5{T4Yarv>dlxpO%e?+Ioul*(%2yZm2&r8cG^O*0Odg)Cq|4q`Dh%j!M zCq)sXWLU+sV!9B=tj&V;vxf-Z{tsp1);y&-^WVB$ls~Xn2z4h*nvLe8Qvtsw@B!Sn z0FF=XJk!*vo$KF7-1zm{v$B$|*U}VR!c<^4Mbtxz9)z0&0QhHTb;rCMY{Vo0Z@wM$ z9Oq=ci;dA>h%fXdYWJDr=15+s{+lx}91R2OlZ@G|Vgjp@?gUFKEsSm*Ww9yz@^d`8 zi(c44Te7wKSJGonYt9+xoDB5EBT6ezCYkJqxDiFBqbPrd=QtBHEl!2ZU7UUc<^;IY zP$R?~+`i=-$RC*O|4CR5eTs=?K_KPZ4{$Nn>MmhHK|y|g#|hv*W^jYz(HMK0lF57z zY5##)sxP{=?Wjo%lnU)CQTqK^D(-q3x62m1IhiPLPQ4k9KLu1?f6hr5fUsKN^!_RE zGSy^q+3y8Z>Y2N*ucrF;S^+)y@C}L^%=xI#$RU^Y=LA`RMw01*3k6D)055;1$9ZB$ zM+dc!!+aiNMZoI*^RSawNf^03%)xMY$!66`YWY}KXBY_r~(4Q+g2 zKMfX29?EV-$SjZpT%`L2aAJHm?XX;t#_4=Ks#iK)_mhKrFSN`ILgwPMk_uzcL8#(W z>^k`G5SrVZ^2l5VWjVg`+1#UQ2W0HAc^M6&F`+O^YQ0- zem*U9Mo||aKYAoIQ_C1DTy)!}XluiD#bso>emOP@3kgK%TFFqmuXEElXW&_^4_ub% zR<3M&B9yKHU#IW!?d6hn(}_4a3aG3Mb=^9zM*d9C{`Yl4UpQ1X5`VZk z)75mEjRCQV1P_n~m(^^9_X0+MB1{??-p=*^GAhpYfy4P@>)A-qG3l>mP!i z)5OCel`9)eFT8zA>3bzGz$S60VDTBL?a>7huemRPSaD#sc>Q0r<=UOw`ol0VkT#~5 zh9f_$onXwqP+8A;*MjAgDuyu{;Ib9Djr)(e`wi68+?{Bnb$a~T31aIyXpNR|`zDe- z8IVdcdTHcfaU`my;CPdVG-q6inB`pOjxQ5^m21;lRqs;zjzKOE_lx#=rMBNg;?3rC zV`~M^6gXUf2$?i`ZSK&vsajpNu6f{OlJ}M&bNOUr~uR$aK9*f{dMNg1ZoxT zE?W)FSkfuPe^a#Jw9q>yWk^Ed6-Ds{cGq%PC{W$3?Bj8Iiu=#mzV|>;i_*w}0gvfU zo;^4ab^aD&I_Dg0^fJLE0O^AwK|Mf(h8w24R})*7Q&xjUXEH^$?g|(pxCb?~7xSD= zb?4)Q@_%&%Nr-7wEyzouErlw(PeAWNS5M!`?69sf%F5XdlPD)p9%aOl;_-1-o@&V& zrn^Kw_S>YziO5Nq_p=Iy$t<`G_a<@;|%eL-<`*S*AnbH zxp|tKxT1Mq@q=o?{;Ed>;gn+x>km2Rk(HL~Jm7im*&X*c2P=$J50Ez!sgy?wo^YP)m-~Cz} zV79q+ic~qbXc*0)*FD^Os}6g=x*JpG6uw6A`+QeP*!u|wr|Z$u?&AW&;r8c1V?OQa zgJq+Z4iLV}rt)*m+4ZE6E@br86UGE>}Vw+_dwB8bw z4!_T6WvDRQiS10VrRTUQdh=KAagi%kY_fgXvmpP?f@zvR%_rV^9WEfwq2J)HT$F3= zuIds^gOCUtA^Zjd3XMTxiW)-MMPSC zX>Y5t#wXcG26>5bQK26lLzxY_L0`Th1WZZZ8KkG2{5%IoOE(vmU@sA$&FVIVj zw`Y8VKB4#$Br8RX;XyYQ>}sz!;Hn9(om~rDFPUkaGvQQ{kq$@`%t!|yoqnN_=E$4G zxFL9Xf9*fC#7xuP4JJUoU4vT(>2#TZZ>dZE1y_OIqEXPApIPIHM;H(+PKKnAngF~f)(mtY< zl;^{HZqf7V^8_A_Ye^sS<58!NTd;Fc#+x;Er!ga3LYC)l_{Y&85HCdftbhvuw&^lL zBl{!$|H}fRKr%AC0U^Js=HpHF(+vRUzc%j9)lXY^ zD<3WfkRrq8aN>Nc6bM# zA2BA&&3;s7#JIAxnsm$gFoB zjDgP8EfSH{k(h(eW#gQx^yvRil`}`^#^lk2nk0`AkSY+q4Hr*$Vs~0`23CE*PK?GJ zyKv~DIq&n{2subdX0_F}h&`)UskVI6;&E@nUw(ac1!Ojpiu3Zf-~4SmeYzHTe0civ zv?S2!@&bM5++*QOFXGt)5Gt~X%NxAW#K}rP9er_3A?HNNeVf>%tIuvSK-U|4+QP#( z^-T7tm@%ApE1+0ef1;hwVmG%swl!>Jm_aa47L$P{IOAJ!AL6Tor=9F?J=?lm&l`0+ zIR|C;2B*#+yN4&Ke<56lqy1YJroJnDy*Th{Z2&}`*Mu+ z!K_h`*1wgR9^v57q{EhEYIrz;GJ57X-DERcfxD>nDfx?&z3FCB#N4)H<3(n@ zNpYi76;s9=&}MJUJBD)$BIAo=eyM%&nxryIMFoV9XAxO-kL#Za-#37R&+ef-FJ=s(8Jd5c-Uf#$<* z>!u>76!lI27Qcu3$_6S9E=K^s$AOmuU3WX$$q@(>abu(TN>scaTuhMd2PeLUt@5F2 zot;(5ql~NVbe`ocys*_DEB%VIXuNW4vE>@qd%ef+d`=}oQ&A%s;r=B7*p~Sycu2gI z=E--Qi4064dFFf^yPUNkWx|1fZqC{4$BT`P4dB{m*KqLK4NapG-3)6?GNANHGhnR@ z4FOr;^|aciAv9YbZYs3dUW)0qb}Tu2w0vt91TsI&)vV;qU-BC>Al|HgcY&441v#{| z#!T@S)G;Aub|6LzM!I9g^-BdaF>wG3cJV#TQYWDIn(z(ohCz$fE|(BjhD7e}BsM-cZJ6tAxRfD{pN zk>8ehn~cbim;hrH)rmx{6QdTL+agJj3!9q_`=GqIg#v8l#q7QyTu{G?ID&|}$!1qz zo$DQ@L#>8F@#UJe-kfA)u}JH~+8veeJpd*=3}-)hEL2T@N1{Phz8kAzwcGUn=lX|&gE%chCWDA(InzY=CB+$+n+jPS?lm4F&Zh8f`W#Ve zZUT@ZqUzOvV?pG~k=lMaudVg zF&K{q5B2DH93Ue_M1&;!AC|m7wtd)C?j{w~Wl@WGfyOkljE@Bm;C)-loC#`(DrI&> z*Zib9ONM7ml4rL<9}>pRTRoCjH9Bmv(^1{FK{}lnKirSFth4YjsWFeaH9HJSQD5vN zIA)dkP{iFMRsH*F@$F(6S|4cA=G<0mkRDp-Cu?Sx7t z>OQPs1oe{&p{b^8B~ET~vuM8;TV${tXs^a-$8_by<4RcA%h%Djg=;Xulr%6HycRmh z83Zx*kC9lZV9U3o^1j^O&Gz&3K2+39ASoYQ6d%r4j0eYpK4ASilPpBsE7QNV<}8_h zBSO6_8Zy&%wRL}dQ@Z_?A=Zm)?uXdfuN5KWhxt*Cui0`1Xic2w_AB-Rp1RR9H{}G_ z7aFFE$VM-_Yq&rAeHdS+)j-+De~PYpcxSi z|8J;QOdOe;xF}0!#yVP8Q7i_whNYj9A10M^FXN9B>%lmG(+_p-&g~zZ7If;^%Hk!* zTxm`FF3#r0&kj0X1z~e$PMu^2wb;IZ1JUeKV@y5Iu{Ggs>>Za3fy=_GMI~jj%tGCWAf8%A-@(KZb_9nm#L3jv8nLZ?n<1h7foCLeeL?OlAhV_ z{ih2MDnI>tx}IyT($CP9*9q(J3rOrGK#8Na{`SQ>9pvg}O%gqOozdcFn6CDSoAp)E zkfe003_jG4QBA?}gF5x(8L7Bbekf$_;cXpk6laFlvh4A9@P}5L=-Tx2)vOh8`Q^d* zq1K>{aJJRJX?uZoCpwg|2Ei;IvUMhSL$CQZ<>qXlzPjuF?C!sHgefe0fDv9p7=~n?#!<6`wO@jA$P$xd1lsl zGgt9R-MBqI$6;o}YqSF&s^wHQ8cj3(S7<_v5CT-~XOW<>e`>sXE-igpBjp9-1R!a5 zxx4(tyX6W6K!#ZRc}BK9ZDKNLtft5u0@`>g%scd3XOkySS3BHlFR1+P|FHR8J}ry1 zZV&P9LO#%y@dgh0=dpHA3rbhGCySdFxDnK6u4Fr+jVko>`Ha0WS%vjVsBy7&(fAoF zk~qq*bkKC!G+Ii#C(vgUCGR9TkuBz4s((znpA#g^sDWX;xtU)sW12tQwrngrlns$r zh0;PpzH_rMXC&)?AQ-PS#l)2WX9L|teWChpx&R!irLs6>A!Lb_1q9kN!A$NH;P|m- zjZi38lYNJHolbM-%OUGZmJA2Qe5O|14*N)4Idv|Y@L+)G?E7^%kvQ*+hlzoYWE{k` zOb+O&!vcvAeu-057$mZ}qiWCwn>cIH9YFs+NBb}VePlOAy^%ZbQ(JLYd}CPSMyK~u zfSzDj_2d!b8B8jFt~VtU*cJ;z3!KSgVTx!ZhmE@s=wDL;v^-ogax`G=m3bY=sjMY| ziu~D;JzgZK!mYqrT6B1M79`T{w|nvcxSDkasPT||AP)0`B?HYdhEl3A*TS{wrf!6@ zJ^a;grmWPjQY(allgWzY@1Uhas)~%~B->8pxqD!8lqNxs{RQ0-^tX?ZJhfISwZjy? zQc54xT=Ly|8vcoaQu~JY<=~$y=UV;vRjgP*;X?*2WYbP2{7&uki5hg>|JMntIFuA8!hp@E_clGssqLZhc#3FJ4viICpkvxCX$Sv5$t!KQXEBj2*a4n zuHyF?`l#FNor+oei30w$sgV}UGRVvFLK|{w%!L8Em7<^_`-P+c2jE#v$*E~S_sL%K|!alV{~=!@bLELY|z~$uk18nA(fCF zVzcKj^nCg)W#V`B!%de2{u{U;0h}c4OJQpTh&s4N>wOjDVHl3^_xRpyz(k|AWFgu9jVgxqtF@~51d;;P z;y`aLQO4JVhGJVr!E)xm@$#CX;N~!X|MXhYq{QPbSSLYi`fVVTvL25nfyX^mhKwe{ zAQ*32lOPZ`6=J;L(jEIT@u(>iLYupddr+rQp-w`=U{n}p1)HfL$1eehnL?xiVaiGM z{GlS0=nD?c_N>#( zB#dVuvl1Dac|OWKZb#wVs>k3y-u_{WmeKpW3u-e=`F$>3zSCwufC{qwX?$lp;3cM1uG=Q;A>ZO!_>y< z#}o85eP~kZaov1v@L-Qm1Fz~K`BY>!bYHiz>AGl=<7WQW9Q*gsz{QSt%Y3g_AtohR zGF+p#Yadofzqz0F^Tic40 zjAez7kbg#SG?W}-1VY?9$FCn4>I;Pn{NhPGxDd;|2R#_CzSR~o+$IA`bQyf|7~)dU)*nkc&SDjn0*wQ_}J8qW)LL)Gh|cMoHED0jKUBLm)~Y0F!ey z5E^?a%DcY4va*s+MLFm4ntz=u%Bd(zadf{w>5Zn`1d*g+X1%20C_XwG2ptUi5(62A zppLIkxIKTKKVVkuSnPMkYps~0?=-lByB2x z0W8QgWFOu{h6HsN_g9HdIB&b!Nn64D|la*XZTO+kt)b{n4!bWEh6 z=D=ivM|fCS)t22bF^wVtk09{b_^m}QloLz&WzamP$ul`G%0Os=`RtTnC@Ke5*ouRu zMIEH!&(azu5u+K+)Dp|Ad-IDgaEnVw%oBtm8|Cg?hY;?GG_yF}H1CA<+md9~Ru2xno zcPGFG0|cs+IV52lgdTjF9^wYX(2NpD-QB2KrtZ1fo6Gd-1XE3=-PwIr0mbso&phee zHVaZ&#M_U5{`55F*D;e*S!2=Qq7b4|6H{Eo7bFxN4WuAuNFA^rL{=sq{fsNHeOMTu z=?(2sV5bZUWTm5^q8^oe#!QnA6h&=J;7cGsKf9addzA|x)OZH6bjFPZM1;Jm??Fbv zRfpm$1CWtkF5;Jgz}XoNo6t;?FRMCyX8f{+&XTMMpr?Smq=slDF#Z67?>jHMHECkg zH>pZu9?L}-i{Nm4J~;uU)Ob=(7*-P}f%NlGa+Xl01^Mm?E*j8W03~j}CMq1tL0_k;oc)l{^&0@0r(iZzXtq9i8cJ z^oj+dVDU>0Y2QJ=fm9S%-Mf)ZLj+==A#36@H1-#Kfag@;l?q(evGCml8(YJf1@4-F zV7s3B`Ge1)j!^peD;xm)o>}2h>j`o?59&y7)^EEUd3WCN48!P(?wRZhC2NN$X%*zV55K-a5ISNoUf>>+y+3tX%A3obS*6Nlk#YThKMG?2zcK6P`3Uqvu7 z4a>?m4M=zPC99Ron}Pku%}c-ts#QrTy}eH!Jq!URM`)@R7XSdkA~^V1tRf{ zBB8!ZwoFOBZ}PqwewV1s3TP^Ol_z4+Ho%*ZLy;+9qG}c_Q|I%BLspY?*je;ecyZ7P zViI$&tBxxA_?a@X!6kO7V?CNa*^cqT!>?ej_~OHirF$xsoZauCh9ryfjt0oIqkOq0lsO5~nK)l~OXTaOTc4+TgFLq~jG&9}zET_R1N@B8+o{aZ#YQ^eE!vdw zUPXef`|}3jr^LFF0_PJXSRjcS-XMe4o+}adY;Q6IJPpDHV_uCEm@rB6D9MZcf(62g z%E|zF^h))Q8|w#6kT9=stH;BjmD?$?BrH>YVnPuM0I5ast=qu(8Yy3BE0SK4HM`kw z8J95BKjcgNvVJb7JtrxIqK;nov!-dF!XCKL7NVo|TYNsd+7F3WlAt(GkkXl9;77)C z$&vMRi>Rk^qY#)`u2664&;*Ps>uS_4LkNsj^mr>VcL-`)UPXL)qM!CP zS2k02&s0cNO=FUra-2nD{z&1exY53Pp2_P9j*b0Do%2x(+o~<6qP*|24;YkQ@_K?(fQjeLhe@dWcDN-t}M&4bez?@*D+mLubR$BdG*fSH{W{W+QH$;$;riXVO?p;(v%hj zy1-sw?K|&%6-y6rbbNg}o2}Z$y0V^zJmLz}6mGgm& zXXaQ+SuD`b`EH3Dqic5u)FHKdbCMTw8V3MO8?eER4IDs3ilXSe3&9tKEsBzd9(({p zc0Ow_7}T)v(X@!DZCgvAWtNM*Jpj<7eemtKKmFw|LkNq-qAJUa)f!!~zqh})KZD5Y z_44fO?DXa7<>h4vfeaiU-n@SM_Ah_=%gJ>5yWjo(#p%gxf8W;A>o;%g9Uj$_34<6{ z9vmE$broZbXvrWsLNtKJ0*N39vUVQUT_B5E!$i0!imvOl0JRx<6q~I2bUFZa(Q&81 zvBlw~m#;QOBecQ@W_B#y=!AQzI}Z)!aXp^ZYLP^^q0T{njnpD#Yi-*mI-PF7p>A1S z*ZNscttg7N@knCH#Ms3UW_t(Mj;??E@L@jzYa9TF&~=?4lx4YCUlf+ObVPKpe*|h@ zl<3YEqpeQcf-t2}=p>T@1tEnoGxe63!WB?z%37PNdRji=q%F6?SB(y^+V9C621F z5o0vA;Gyn+I4A`(Sdwp7KIZNfvsPQM z)2Jnp0Ng3Kg_-avnJCCvjWi`s3D6S^S!)4Yj8P<_HI=P|4Ozp?NK{poA?TX*;-YPQ zhag2&6tRH8QczYdcovZmyh5I;`Kougb3+Eos;sJtnU~AU09dxZjSypO+La(U=@HKi zElsW2glDGBNo&k^GT+b9Ms0!Ua2pRMD?KCtKmri$N}0OG5B;bNM2;bL9Qxwqm|f|; zBf7KEOwE>!Dv{~jhSwj&HXi2mXjipPc$>>&qm`yR`x$B`0ze*-yj6L814-MaYyeC? zz>U>m`$|@v5Q1X0ri=X1pZ@vd$KNg%i-W`Kuic#=9UUDX9hPOu5;(-J>(=XK=sHBK zs;Vr@$!xy9Sk{wC!H*7b4?!`SheSJGarh5>{hY6h8fw z&N~@emBZjrdL^k9IeMUV>^-9t#-Ot(XstzSM1&lbScI32gmI<3)D_JVeU;Vk|HO4?{`_li4!;b!-IhTh*n=)9KoO5+u-?(-C z$q6?6C67imhriX`BjHW2Pk7S&& zg|Vfe3($ffh(Q2CtEWh?bsifaP})5poTh%d{cR&^a+QBJ1QqvONF zgR(4(!dmBsJtwsY_->2Wr7LaJX#dl>qzz2Fl(X*iEB6tEDS0;PZ%E0})0e?j-#c)b z1o#}aaz2>@21lO<_Q_!VPV_6GIt&1p?u5a3t}0tjFVea+$0=vyIM{vi879VH2os0c zumrVd^m!`76q7ORUys!>Mx~C{S~J{ktzE3T!)rHRz4OMcTel?o)oNW7MNv=IZO}+& zL}VcW6c!eVARe#_opWU|shI_Y0z^d1U;)S&Qx+P=Xo)ELfQTThAuWa!hIA!8$_(Yxa~eMa>%=#5ij&?>+ZEAp zP%IPld8hSS>E&q&z%4EXz_j7;C_g_Z=GN%KCD- zUUzL53iGHHwHf+|tjS*~wFP^hD76aWv4KBWR}^Mqsi4fJq0-iV6%^{IA7o zM1-NDfX1dON$vt%zwyd!J}qo&ZlBli6yu3SDR-Gcy?|jCIbDF`Ev?UP1n53dsx8mOJ ztyrxEB#FosG@>&3*5&jfZDx97PL$K<1PquZc*d-JO}=o*q7pgNz;lUmyujfa2HX@B zOXSHyHZ(zEV$v5*_*YxYthK-)Ct$i+&NyRC*ZO9CSyr`mh40$sx&=mNU;z;1C?13% zQj}IaBQPNVI%|~1*~0DZ?SUYXHO3KI1P~sy2JQF)3_<|gHW*zu8ueo$Ad3(Z0;xQxV7alA zVJ08i3OI-u5!KcrxWblo_4@0tmjw~XYIzyj#t=2DwfEk*A__AI5OpC$Ve87WngBou zY>X+Y(z!Azeu$lOuJL^@PAy7XmSrwc6T*DsebuLsDY{| zh?-*;0eSx?)y+GPut!pGt{d~Bvt=S9dmHIA>L?Ko9Sgft(*bqk)`z}MV+(Y zCCUn(rsGck4#&M^0Mdn6V9pI-I&_qca){g3vLC{0Yb}R?FmrGBH^wY4mInt14B{it zTxm>bng)PF@CMM3BL^rDV_dD85Q48mS(e6GXIxcI+Sa#iYmBkRs$qkfi6lkv)@9jS z#K0i12(spN3{w>xV&4G;j0{1blqIi@KqqJ5X^RJx-ssgi6(61DNDPDRe#p55rB+$Y zwDfd}IaN*+U1MF6MO_A)S%U9`MJlt^PC^KP2<+98$pJXVgFCM|3r6JP{IppvKxBEj zSg+Thb_(3+6zZGsJ`$-)zu7bqQO}JA9aWP_T~}&vbIw`khO(LJ`ZfTtUP&?#Kt{Q6 zTh~a}E>@=#$+tH>*b4$ZR;x_65p;9kp>A?Q>Ct3H+MI_t!%)9wwt+E(5-Yww#7m3owwtK5*f-OGoO&=3Hk z@u3^o`zAAqkWSA|jWJ}5<_d=pR-K3CYCfOGZe>wyLF?7Ih&YP?GMQBQ*`Qcc79vs< zea}Q-fR<{iHN%z&j8jr40G(*HAp!@jvJ|7cfwf7Djeik zJpj1@0RToI5HNu1>@0}@*i!4D1JsJq(PGj0C5si%7@nbM6u`P!S!=E7dow6xmSx$* zo{HrZC%1z@M&(6>TR{M8RF*`JqNZEt41m!v-_AE_kjUNtQ`xue$c`h4MFc4+v$8Jd z(!D+1JH4|#3&XGb;Q#-G!7gAJe(?a~2iw@0*{MFKsxng|Bm59#Fi2&cwS|K`Dy32) zgK@tR5m`DP7$Qmyqf%(xJf*aySVNe|Wj7y)>v~OT>lpkwU9ZJUse&4@ROA8BG>ga1 zGIV4OgrGC!ZF+mqL$yn@I~4D~|Ni7P5IZ;N6ha^%v+%FyuPT+%413+3t0GUC0G5W{ zFx?0#W3L!@V}@HtpDpC|8lpzkp7E<_(sMsAgCw~$z`e-Ye`UbeKUEvF2bwM*q#_YD zatu|*=F267aR2ZCayV^=oUi}(Z~yDhfBy4vTIS`LbDo#waCbUhuIn{j({_J_BH z&Z5yl6xbdfPccTtKmbl6*Y)ypetUa+d;Ii)DTv~Oh}66Yo5da(JTcq6^jAj1<}#q( zscE03CJZgH%#6qz*0TRoL{w?5mK2*jG~8I%b>j_h9XDk)G!2<6cNUunVxdFZ4PF&7 ztOmK72{vD^Ul@WJUMXZsWQ$6i^AM+y(wg%oP@EhU7AipsRcBuPpzU!sJtJwVPB;%l z<7xn4wi&zp0UzfC*E>pixe)*nsWxnIBqEJk>yFc*1n2a##ZfYBy6s@;`CI&OOokZwJsp&m8^zo-Nv)d&0KglJnSkxIwWGt2$Fst@4`TcsE0g z05nbEhaY};{@p*my}i+|U*mF+ZJQ3qrw}R5=kxh;xia&7h}X-8z;}0t^Z5$EpFe+u z^JU}he0{sDmo>}V<@NRT>-q9JO;ZBQBErCuMT&RYXKs1W*|7be#-1EDHhdfSZv|1; zV-OL|z#OPrsi?7GqT+>h>*dk1*#GZj5c{;ddLco z(}XUIS})6@e-hC=&uZQe5Hom2<-0@>BE?9|ktKCL&H@12R;bj=dKEIxS{q@&}hll6q=XG6=hxvMW+t#;8 zPk5Ntw7p$l-Y#z$!--%%&WGcX0RUKKv;c~*qR>3I9fdr!^}2^}S+t$Y!+R5*E3O=+ zipJVb7M}~Gx}yydffTn{2mz}DTs3f8(|MkUVgcnBMozHfs^}DN`$|(2>pEr;KvL5w zAOcYYxB`GMN(~HyXUcW0=XsW1m(+0{4o43hy(i|JbKY{^GA9w+O)5ButJo?OGR=1$ z%JyQ5mzo4tV6E#aPAB)cYuC}$A(N{cDJ%dG0H!dh6!jnz5l+4jd5tZXqFLjsPFo8% z+*;D>TAkQ*>7_<&-o&sm7rUr>biMa-zym*iPlp9}`pyA23EL3J9} zh)5VVZJ)qtU9SQ(&&S8d4?q6q=j*?J`Eps|`B}wSj4>ch%i-bi@pwEU!j~^!=6NE* zZCk&-Jb!q+2f>Zk^Ln0+_xFzvpT7U}@zWDaV!}SlkfOdvH*9x|L>zu`w^}iCLgr0yZ));|{F)X*}553Fz$V z0ijLzwu`xD%_rLs-L%9FTUoV%-89W*`{@^NMszmLS5$RnL1;4AT#wx3!^4So=a+st>_}P;1I#)=m zAu)2*W7}4|b(l-vCdDP#k$OZEYgM3yQX)7_izT;B(`*p)R-SU$X__vV%k_FSVzC|L zvF0NZQ4FE%L4^#eiu?RW5h<2H-5@fhPVKanu#0+lE8pL55={5{7_eWeW-B$SX>Jl- z0}o<-=F;-I%GNUlClJ)YF1^``Y*7te1{$;Lo}>rSN;T@^36EThUqf^| z-@kf*2|5YveGv8>=@0#3VcVoFpG!=>uN=Rbt-WF0Yq!V#YvWqVGR^kBEQz)1O|iZy!E=UXG^==MUd~`r*ePA3i*7 zoM!+bBq?fb@zi8aG|E_XJW;%s)N?KaHP@Y$4H?8W64oq+#+Ri%UvZC!j>n^|xGf^0 zAqpU3*^5Tduc?T<<31goCL@;tz~|>xJs36|2nP=hV-m~llcH5LA;#_MI!SV88 zNcL?vfd24o)K4(GA~0|i662gw0$3sIma{An7+_hZ>-C&;$Hsu#!ufPM=1ljGAMfs; zu9r6q^Oxmsm-8zT(Zh$=moI{Fetlbx$EOdUuIsvP+u{EH>GO9VKYbSt_uqa0<8S}r z=a1ig4pW2>2?A2l50OPu$+0s0@t!mFs(OeivDwbGh9@Rn{UWwQHT=pEyj`up2}GI^ zbiH1!AUBkvNYe!CUNl%DZT8;jZuXdIOBP&f8o#V8b*O^qxR{7AgXZTcfmHNX=7J)a zvuGc8MBvi?B>+&)vTd0tieLz2MA0;Mk#o*lN~@Goa#0_+u>t)Ff+|8`UK)8IA{Jj7 zuBr2ldxFG1{_c^|K!Avv*_OC8>oJ!S_Po}9uE%!1h;(r#^IQ1Jczej;BT_Ez3EJ6plkRlRP~;@j7k-~Qu29Z!FGczj%@@L%TR|M)-ubDE~* z?(ylvcMl&w$HNK3{BWE#z#$AQLIQ#~X+Ne4mcb=q#;sm)wA48r>?H5H6qdPVMntu% zMZ~BYY6KWrY!h-s^qqa7`R6(`*biV>V$X5>fc=UCl%1=^oQNLV>-KeiH9e0rr?Oo! zX`v?uH$?mvRqIL7T|w92-Cmh{5ZKQ>kqi*AbDKa2#L#UW?3HIDZ@Yhfc6L%7b+E%BM2vAZN4*}N8f_))hrU0V^zOg`wp4Tax%NL$dx>jv z6%QrpH4!5TgRObv{Aa@w2ngyh%K)gs4n!adAOu1}BnW_%^GZaZP4oze!m=C*Fihe6 z@^XjqcfbG7cMqQ==i_nt_;)}3{-6K(%a}i{cn>%G3G^gwvRXuJvn0qYHD}PaJxIAcKgMX?TFj0OL3?2#)U0wF(J&Hl zgl#Dzew>JyvsneUQ^n(`%A$sA#*5Pt#=Njg>1K z-q^%_-L;1vyHw$BX{3395D`SRCuC-A>njKhNJ4~xB5bl1_MJJU1R@;5L<9*_hUxh5 z`R*wm?$75lB0}b$fA^cW*9!yOpB{l0mTgP2rMIW2j}R!J0A&&&gn$Gnk)qMC_bd)f zYCmxU-N5GdTn_+{>Y~(e=w`iQ)FP<1jBy7kzS*8hysnmyy@*3TRoLQ~ffx?O)S z5+jiI!p(zk_)1R=8MJYv8YV=9B*>`U=XB@KImEb8&XI_wDWFz_kd&O*+9_4jcOZnE zAm4^?cA0Z`vetv#_zMM{{FfG^2 zc+S&ucL;F3UK2wAB9efSfT7d?3&pk*jc440zxTbvbIJFRR^eS9PQ=`3jEDkKh~+%S zm}=>ml9ut)V)BUBsEZ3}pjeNh!>S=-jR06+vKWbOs9+>LOk{wJM`vdMo}}ybYDTP>(Tiv$O8zTE)NO-?i->4FD+U!^y;BARyGLwy(iowJ@r~e3 z-)>ybF~I@jy|`|73NWqnPSoTKVxiiom(hDj85M(ZDzaR!giE&o1xClB0 zpy|Tk>`kaV+OzD!ePS~oOsoHYqH3v8YuV)++wp#OdzUs~!-cxs!?*hgjm^PSH4)E! zJ?#LLea|I?z#<~5x0g5>5pgbk4!HJq^r(Z?r)7yW6c5A1)$U!EC4>-S+tNsXet&;I z3`i%a^@%nwBRuns=|xS&Gp{DO-YFiQ{uRUE2P6X#k13@gHfV_$AqEjyITNcAh;z_{ zBLTt~X4DD<5W%ozAp*u}V+l#-7?#t+;o-DQ2LW1^yTl9-7yv^Iga{(SnHeR6B(3K4 z(T%}VIE>#T^d2P)u+WQ^!d}7IXiy9ss^K)%I7(z|S03?@4>~=maXYSE$226X8o1($UBq3|lE*yHJIp(3OQ81vWElus zL_jQOt3XK@fJM^jZjoHM7)<~qoFp+bY>5wt`4A@{x~>T@%*!!=oVRPv8N(DpxaL)% z6)1uv&Pj4Y$w4xZkgI>fUW4U6Z^t8&;fY6yy|>Jtprt?6BqVYV54DbRUDxC3#5MFl z?x;6+B(3;7@Xdi2+{E*GKbhqfVgyzmr^{NFFIP{V=V(~>fwNaspChNd*?`SZX~ zy(%lU2aDwc+Ss6DRjo(AzDV%XG^*+ z9Yqk=OSwPB5^E)*EnPfd{fX4LtJkHcY0^ftVp@JZR8Kh^4w|u}+TRrCp}DoK4(vz^ z02s)=qS~NNfB5$HmQq@lrES<$95G;Y0)1)$AS{y!6FbelUK7buc$(IJ|L~f`4Y5 zC#Y1d%N1PLB-u$JwS;W^Hb)-`Q>LJ|$0_-nnI&gNO!GVw1pus-Jc!6dbXe}{*f?jX zm4Irq6mX@qrc?s{dwW=46k-GiM(NEQa+%u&o9p(y=u&iu=crR?Y1pgoQd9p1%^Z5&OsdTHZJEU-+?R{GHdx#P;7HujAwaJ&g6B$Z70c6xpg_O& zAvRgzzd9a|cCp92ta_Hcg&9OxWUH}pp&Gy=qRZvqwZ6*ikq{m2}C zD~u-g|7lRVjr z%Ukg3im6*(lW#aw)8nCbDKhVxUukd(g4_bp;92m4muym$ap$+&AIVV>InSA|9`OUX zUSGjsl1(A!ER4d)8H7cnxrp-hA&?{ z5~{gjDKzR_(2Pi3j=D3{2)f*N?XCKdek_YMdA zLL)@FcBY7p_YYGR-LOnb+EVe2?8a>Ppx!dJ&x+E%HG%+SW+2^V5<0sVi}Hq=hgkpF zF^FjPCn6${hgOJtaFA>Ih5Z4ReCzZp#oe&$BX2j@C)LF4_8dp^mQ6iiq1@UE&ohEO z#J_Q1D2c+wz*4`t7nDu}Uter85d2E$Gt_jy2PvP-H?% ze7#;Tmy3viP>gZgG9sQ%r{nQtnEm0y{mpez=N;@Y{`K|j?`LLh@c}N;)k3dOEjwF5 z>@6b?VJUuxQ9FXVDmByq4~{Qln*%dcb8F9`J--`{Jx?h=Df#eipIxdN z$8f)Xs;IOVnJ;R{RJ9|spzf}vV_VmYa z8cZpr*SFiXlS6fin{`u0u>d-|50eqyez+9n}TKnpot6=XH=UeWc zQ)RX6x_KMBB$pSkf6L9K2UkN1rO(kQ=MhWksEmvW3|2&>LbohEA{N9}TM!{o)J~nH zT^EYv1jxc1f^I2@A_6P|Ni-G7^+~odeRFPn`%h$TV4x4sYa$YQb!b?F8(2aF&bea5 zW45AKw*8_8Z9owtXdVa=S0|H|XPH~Y-YeB}yQgY-Kh7tvBg*jUsoK&}uxgmY(?ldr zP1E*_%;rVaLt0!6+SJkkO#~1KGUvq1f;AgiT_}Jn5zh0ZjmN8ik({%p+#e1H^&uB# z3dL7%Y)JpKEw$ZyMMMtEqfJL|(D^DCu-%0eePfIu+23!AI6%0aSH%{o^p(|A7!eU* zmduhfC#kM--FDN^!$4{opeBUTDQoCVo^O*tN^RFC&oz{)?Zjgd^n4ji=dZFBpp<8& zl;$}MKn8#aAVqj;T3?mbwKuJPl|f1p4j=+@`3FtuE3*yggGjZI9VH9?f8tnwMrn)ec8J2`I!sO5s2t3L1bRqAj1X=r!iQ zWoEv+yPKy26qYxMcKnVJsYO=o5D_idwpq99a1|$zv06Ra-_8tas`-3`VL%2w2D{zg zQ_#W@D!Bd0bKRJ>c*{2J3Ot8?`*`bQY@pVY+_wYq>>h3%`9`Ca?k9WMaEq{onFU(A zJ9C~bOGna0;}dyJDQ(-8OVcvYK6N=OudPKLwQB;)>St@K1yye$B5M{UAP7@^Yb74Y zb+(pauO5?>b6&!1P&0>_^rozzumQ+}OfRnMsx)8~mwrj?`G$5LKMdA!nzmIp-wZit z-E!b?I8dB`b6GR(;IG1Vn*4KaqMhgNly4j1HdOnz$dE2DWIgQw zU|rk(Bt~P*Z2NL+5`L`vQD^Sd5PP)|p^S;EkC#;8CL+tSC^0X~LLqQXUnx60)>jxMa2IMA$A<3k;{+g^dZ4Mo z**XeFrwJG6dF}uJPft(FviKYxJy-j{gH&`e>J?GGY3jE?Hm^1I%jfshCZs{vnU zr2dNn|BoL(n_tZ`C4s_B-FWOUP1OjdkfV%@Pmis30=6VyrQy5dJlNj$G^{qf_qKRW z?Rky#9Nh3q_Y%R{Qc7`E$DsFcAGr22vWNKBAtF(0|E@2r7-Buwk73=|On zi2#;22ErO`${+&TDj2i6uMhy#XqN(0d%8Gjc4O$2bX{!%^@r6YhE~L5&-7e#&~W@c zx*k52%GS~*UfgHqy2-k34XovpP%k1t5JA_R7i-f^6^!i--jLCg zl$bfisZ+H~!c5D-HWq;z5y&}v9o}=Sn?XT1}L8@wnjpu;AyO(}@L zoO7zCb^_GzvUUhI7y>}e>(ruU7SROLa)${GzhH<6eLu50T-IAC0I~T(JC(+8bihbM z@p6R-d(LXNH9B!W@m$u!V%mAR(4rUG0J1)T)Hq{S6%=|mdvZBXo>1SVST}h8lRcK` z4@0OhuVyYdn$&x!?vM%qe!J_!Wz-KMr6kkpngm#Y5j3J*dVoXILI?msNEmg0VwLUK zx?&8Tg(5d9k5DvYjm&}LVxndawKoAGRaV72N3pG=F>wvK6^H9py}z6BdOd24J)4kM zft*vR*i$iJ$l?_}@1;`&F#oGvaVm5tHX6%dR3y)!e0^b+(6py4O_Q4R&4t6zUc>%t=eqyp`g8ss<+Gk$SUin=$4p z&9Wtvz&j;-eyCV^zfm|igOmHLHH9T;n-KtXzrxye9s9#C_I@N>dt9Qo*moSk08Sx5 zWmSNvn~<6Vf@)HSU!$8CV>9)z6%kDW4mnb z3<{(`2tv%sCJTWXm{j)mu?d@w4`*0JCWH`XQmWQ0cl*|~ajF}>my9tA zelYeHpR4EBf|h8Y_8=8!+@j(uf15G==UFLum;?cf2-k^F1%3P@kk9)ab4HBUrswW z>N1=#@h&b>V(WyxdNU*sG+ZGk)h&d(GHQDb2^@z@+p~!#5pkB_Te{lR`jA*KY8L>a%Eax>0i` z{SHAkND<-fMyaNVXeWk_H5b=F+uzIjWENQkGT^6(U~Tw`)mt_Yv!As`OT%p+>{!#A^Sb(x z8@_Sxr?BI6I^D!WcJUoVZLxv?pnU-VJhuhtD>^+?x3rqC$MmlV00^p1gc7FMQFS3m z0J**5-DEt?=!N>($qF<#8&n;m{O~qa6IH5q?J2-yTAie^?E#byQn(x8e{c-+Ih+iK zmpS{jymd5(Wm2tM{rD-lO(!AGi}WgF*}M zTXTpuuiH<)d0j*R!Brg>*v|8;+q}eyJXQNF9iwWq62T7%0P3DF_AYjDxSQiqE;a~o z*zW;aUmo7sy};IGAZ)68^};~a0HuW$k;Wx_QbAAE2uAaJDBmRPvf=MeR@a+RRgU7b z0U#W(s>*5zrJA{?YTJL@=`G))YGaW$)ImOX53eVNTi$EZ=IEX z8-VJ+{{YScW_KCh+sN7Y0uM@ZW2yCo$DzLW-kaC+JDeUtx9fB4+hM0?c7yElkd#~H zao;}Hq;|XESeaLhkp!e9(o-~#iQDMH%j%Nd)PY3+Aa+f`MBrT?c!5t%EF17};8v=T n!~0ZCz4U6Iz-E3*H30uFvPR7ETjXkfs;trPZfxxX45W-rBS)hLQ5<# z9MxfYNv%mq@mJslNM-1n!CzL2Er-cQY) z2BPm@WBNCIZ^(V}?Yt7Q=W^MJ~0N;1Zaiwz8rT(wqrcXyAsruOFF--WcMfdIEzSnV9 ze@V;wv$SLlkN)!I%d_$ITr68Q2?CR>yVOFX?WRr?)}07i!lBH{dc!l#s_0;f3qI_N zRS6^(i&bt23ws|6Bp>l3h+1e;0=^{8@H4J~0} zd&lH?P!!Uu&{=*vXASuG(v7=u-B6O)`t}HmErge+cm)LH3<*Xp#&^!{;LFp#HdQZ- z1|Gu>&d%DLIF*w;_>qPY|FuBDsbLvshF@p*g|V$c5LqyHOyyNVZGGpNoTMV zu6=Hpq+2#F0l`g5zua35%qG7?3etmXfTeVdI?gtMxm5H8ZaU&{5+6*ao=U2#=ufP= z9XFa71E?si$DzIOklPN<>e3B{afJkqa$Xw&)=VYHuQ#}2T`FC1VDTopH_cEhHd}Qu zn+khzH#A*KTKDxarmQJYB&=TWwN_r}>~Zdh$%tv8RCqzF6DRCo0=IT%t~+_We$4zY z%O$kj;b^DkbS)l-bDO+lumJ*jWXe_1_mP~N_?G=Vabl1WT4ddv; z12AKLLeP}RYzm~Ufh8y7OAi3x38#0J@c9&2zi|0?tJXFFGoG;gA9-ZX{rK@F4LnvW zga968nxV^_oC%9M2pe>_@?o_=u|qo^-y_l6)#;$?Y%xXA<mJmVmbUU z1(pJc-lST~_OzZCd8{!`a$I^<>I8JLy`cXRkbbpd$^JhPCWP({BnoBcifN~CUX*V6n$TSp6b zG%uzkcJ5}alITMZcXxCHoz6Hvg7nMO0f&@rNK1A3n2SuAzz}C&`IuyBb~}N(*){*P zWHvCGU&nsFGerSZbK8{~ziE5n-pvG7L;sNEQ@8v&zxZE$F-)vuX79> za8RBzVZH4Wh_VH9BZLvvEVYKajP$l*er6Nj4*q2ss*K9Y)?3r?+c097>FXIV%J-4N z8scaH&tL2ekF`?L=H5EbPCA9mKYc#FA$>dQF<<5N!&BuyE3cLh$T#;kxwY5iXzIr= z2-T$3#_vF9hj7S5+s}pt4XktSLFn949oCwfC+6nn*-K2y9i8vrha7KCWLk!{E?Cb0 z_z?^LS;bt*+%TYubBTG)ac4&gse(c`pMJv`<>1dclOpC$h7<}$)&uV6Sm6rw(6Puj zc=7Gt^t`pZ;WrPfzCy+0Hd6B05hvsQ@Y$xhz0>6wr=jd)HGv;$DUmsz3kvIJfwvgVeJ9t6T;~T)ZeA~m%S->#XJL}+Ww~t$$HrWd$mi9; z#H7r3N>{}>Z9|YPfs!6uJw2^&XkjH2y6sREM|mE4&_Kp)_5G|=RvDV*^qCS@oPDk3k{+ZkCGf?(6m>+P!trxqTiY5iVY(Ghsr985xeniS&;yt&S|x#I!P7f-Z= zr9RXcj`!7F(gVZq$E7PaLqGVf5BPoGpcxCk>oZIQ0f?FE>sLSFXmALd-lrov_8}7Y zYX0LBh7vakyOm1a6L&*rNv;GnD%>m`G4`wU%k!Iv;uA*zukru2XY)bpl@=3L7ua~T zeqVznA_{7&STM3BYNA;y+OWg%rMnoqnF%v^`h=@&vCX9uhBfXfZc%D_Hp#iREaz39 zvEwAh+y5)`NWeSg3gBL%)E;IoHWIjg{C7f`!t((<7H|D9!r8B+q8vcN>nJKygot&L zv@|Gb#uSiaI5b&!edblZ>SKUNF!T;9OhX`mUpdX}R=iX&FBQ`XYxv*Y0WYqBvApd* zAwmcfTe)_w&&dtl@i%wc)$!B)vM+_B>NaGFfr?ymbGTU#<@1$>CFzcqK$t?QWQ0m# zvSH-I-r-;b0`0y=%q~fT_FW)^Gki=45970ANBPtcsfOOl^QH@?&$F*wH-A2bX|(^4 zS22~!e3d>3L&dF2ZtST{;=|GOrhb2-tpVWl7h_BhK`*~0>_10uMZUbxy5h@MMNOx! zTD-$(SB>#{%rLkCK^Ha6iqe0JDsqB8r??h5UiqrRBdeXA*tQV_z@vG2 zES@|e%v@Yt{NC^SVqmWfa3u|N?%fOgbt86A0utq3dtR)lmExo@yzYaqM#d6lF+q3o zNZW^p-h|Tj_V&)buzyDd?H$Gwr`TeyN=^=K&3!-tfyQ#xM z+BU7FdVm%#Sw4Uhs7M&+DV^=AIkrh4e35`4-tNeaOKS@AM$NfD zkR7njdWgwm@E+Dpb05nv+}+!Aik($dRFtQdPO)%W$<4Nzddl^jXklr2V`RCfhZ0Ho z*5h(wVnT0^48?eUO5{@T^zV=Bc`SSgBSSPoX~6l(kle!GXK^xig$}67!YO8aCAZZY0SbHv zpL1yzV59*0la_{Omj+1@a3DAqJoa8jorqpMUk^<=SK&U+Fa0Ew@R5s+VaJvW0-S{! zQO|rguZW;R`<@=!N+wnMhqJh(J5Id!(!3IXd!;rnU=sbX@#Vdz6QB?0Z(NSXCFtG5t5>NiO2?p^7!L8h-+2TW$#FfiihZ zo6wg^)Qn>;k3a>tesFAiQuZpyiI<9h-n|(&**tgPUnNI$o?zQJZP$_%H_ZxYn0e=R zH*r}SBjY(HjhT8X(yGbqj|D=-5uz5Uz)7Wc^PDelYIkvj3gDelM|ce|awN;gG2+2Ea^57^~+0Q)W^X z%f?Oz4Oyf4wl{0E^=GDRSo;yBJD!*HE~lg_46FI$!v{uzd#|>(depEbIa(>e;-4F4 ze=f95Btxnqt=NhJpBcCA${F>P4byHp)UK>-@AzRat03kTUkXG=t8i26-Pq5+vK36d zD}HEe=koBh0+BClRX@xkkKFz0--nK!&K76H(4yq~`%%1rw{D;QT|0eS+=v^m4S&A? zylu|i`l01xS2TLNZCG`~{Y|S?<;cU#k?KUat!%yXm22fhb`}G)m_0kCM#Be396quf zpb5uWttsd;e?quR=pZ;?Ltq$R$}MYzsrl~xH*X}>*I%V+4V6e;$t={J5trK9HBkzX z=FKrbWX@W|5;C+y6WweA5(}bJM1`BacH6|eYR4k3i=vX8w5%OK<_#! zlphb7S1$zj&r*$y(HyV!I9OglQqDRTc5e&!dgmvgY0sb0bK&_Am+r2MEoFhmOb&{*LYruP! zVN){mnUlVx!(!DMvN@6;Q9yY68FQdF8d}erTnMFGgsZn}#l^*;V>RqpTmN`-wLK&r#9(#g(tSX+5Q%4(>97<$5G0$WzftjE6f zNXW8i(){7O){sHr;B;V}G3;@VnI7+vi40G&bL( zx>_vz#g7|(6C@9+r_Wu6i*{$0olK+X<0&`MMj?=9O;S}k$(?fl441X8k*0*PsdDlQ zp&T|%58RA@8tRD&7QWTLZ}j`y_fJoba^khkKjz0XOFX{s^!IQ4Gs2DFJV3bQcZlV6 zrm(g%@YBBi3N6b^TJ6%od-y*Y&Y}G{_Uii25(>JEWY87P0D7<=Zo=t9h>OiI)QN)K z${ZbDvVzq+BjHd38Cn05COXA7X+Zxd+=VLc7AigPG&~7}%cDNE&{1sL_V1zzCLKp zMiqBb;>?3WNDy&reXmw0QQ;vj>7S%{Cg5K~a2?(!O`aeP!_&|I)nB`iG*#vkG%Kz- z`4)MYH2K&36DJJ&<&4|+#m9Gz;um20qXT+#E01GiV3Boq3~AO;d$a%RrqYyQ>7A~o z@|5F{vw}8IIt*4Fob#ty0;#iq4T5nc&Lg1v!@pdLy6vC z-hLEPT$T7+>TzGR@xAKP1ofdE?*QYg$05A~c0UM%`6>Sq0w-OMezh-myvDvWH7vhc zEoQ8Dcwo?zRQHv6E2!qC`L4-*5mR`hyIineMBSD;YY__HOd$HF*jupB z9)}Yu9%_pgqNC^p4axIvtT>&56%Nrcpn&pLwiIqRUqc#=z2&bRgJ6*nl`!Aj zHU%xV2D%wrsIsfqifcm0;D0)e@ z?*C^Tg@;DSzvFEh9ze}n$Tl4P$wh}`XZ`7mzbfb9p@u~l4<}JYS^1{Bkw|3W7=7zD1bT2irkmQ6ze$bm>_Pq={~sHxh14(iJQ9~3 zac(`OJcnjiyGU5*sJCL=3EMN5YA)XhwX^g5FyA3d0e694oI&Mr_SGqqIHtcTB+d6R z1m^Oj+Ay$(Wf=DM!M0-3qv8BtkCVEMdmtIVh3-YtI#!1P?D*fCZ8#eGG}yYLo+~wE zC=H6I&;T58RQwQjFe>YLEP)VfZG|wIK;Vi=Tnf+!NJD4e%ZZ;|);L#PsKLMBlml}Wd@uoP*Z$ah0PB$h(jSq}()i_kWm3;+; z<*!*!CbZ!^%%#RaYsL-}Kl^Sf7nLzSzEDLE0+~Z}>Ip!UMpW-u@|$u;+6W!1n)Hug zR}(b$7RJzXtRce2PMz*P>o;RI(kN+e7mvr}9<-wD!fo*3R#rT2S*!N5O^7G`*|$$G z-cEo;%Q9?HXufv>k75RK8RIFAUhmA6gj#H78h0uy<8zNBbQ;`0{n_;wH$i?^`qRU6>Nn!ujpWox&4vE^hhLj z@$(&z*{0Hy%D6g#eq$~>O|a$ZXn<9rn97(j-FT_hc@WKBq47s;C$3G{`dd6^6A z&Fb0X@#UmXPIO?PwR}))gf$%xc5vaUpfHwUK}t)oNCDEfK-XK>Iq|i|TaMZkuN8k-&_H260ArPtdaf z<*55YR)x+0T@=_6hl0!BkwQy5(?F)BG0vQgAOHn0hFlRX1>wLO!HJU0WUGMT$^d4a zY#TUV^ufK2Ygp z%O{JD_q@k$bmtgXd?;N!^SDsBHJuS?J7Q*EJvzB=j+`dkESPh?mt({ObbY*e?5bV3 zEopsSMV?_j;QFtM`FIGMIvU+jU_(l9PKP~*C&QT|cs|I%MZ^qM7b8gZgi^o7DN6ESHat7b|^=D-#ADdEI-v`@oc{ zHZp4jvgH!6($gI6&@5>zk$CACB1ILPfx(50 z5~@-(Mh(T^^%uK(akxG$n8EU&fhe?OOld#REmFsPI7QH?;GO8>WKZ zAxghLs1gMi;=>EUa5#nm1>!;A*Z?a&O37O^g5f0>PWk_O-bPj%13&{cPHVW2S|VFY3isM?DTrru)-Jh z$iMZrYb(k>w??+L-jMCAdn$+dpBdl%QGU+h)*J55`N>t1hc~)rPAn8Df$#3~sI3Jy%43INfW1W+s-fR1r| z6KSrd!R>CkoA;Zv@8_p`0gJ*n(Igu|k;PD5w^dbA6od;;H{^-RiQn)}<)WhS9+s1n zyP*^;RPx3;*x#S0IDYP*uTsE@V}F9!#3*W7+HJw31MuN0vxe{O?J1wTx{ogHb3HJN zfx7SB%-^ya)#)B&WpMC(zE|MTj9cVUI%BG1J`wkI0o#e^8ywH;zO}(U-cD}pBql8V zZhmI_k7sUbnM0iZ5+8P`ad7!cClVi}2eRizSJ;#*J#x(q72NkXKpfuj{BMvV=e$9z z(=)j>Yhv7%HF*$4t(6M5wNlg@m&Vj@WKmL&f#^ulyt2I!D)1v?@5CV*5Ih{uugN&b zhcM*rg|;wAqHU5!rIte(=n0zI0y zGtUI?B))d0)mWB)V);bIjm0*B7y9#HiEM2?yTcu0EnhH#V9t!b7t>>aR{SgDFvLEs zn_arK08tPZ4xe)1u7Cn^zdF+~4IL;GJ6&4vj+wtV0WyQcEgr$4CAZV?5IlJM*a<(K(7!0(*6%G)oR};7L zmz8%Xd4P8orVaAiTL=W=3m*Un0kl~aal~vl*=zjFDWg#}ThUIO1NGO7!zERCq>)Em zHRW3Kdo_?1UqvM@#?#YNfJgX7(tzSiUAuR_##wtl-T%p7lXsHkg`d{Xl-J`~yVGt7 zd{RhyCI3?$t17R|HIOqK)8uCBQ$OR{R>K!vv(;Xsw&FwpKfj8MC&BXJ3hI+3ag^-P zJQl~2vNkE*umqpy9fX1kr5>{OSfe)elzTF)it&)m()wOy(Xr(9+3}yA%Bp?C<<5;; z>@I)DR)U^|`a56Q{Mn3+Hm{_&@GK^N-Y;(yF)wrZ9;l1XwcwH%=TMHJL9NXrg-|GC z{(XK#Bxi9^VE+re0>VgQnFP`^G>wpl$`u)5CnCc^fO;uTYgt;#%qp=LmGyE0J|Wcp z@CW=FXL(^U=c@<~Xll+=ZYf!y&f!0O3zh9}SwtU-;=no`c%u3&lbNsxarhF&4*yil-g@Ebb0^o`!y{?HR#+`Q%`gr(GJ{Q)MauEA3X!%$ z5DZmIqo*NLPp!wjTyN+;e{j{jDo#V#sF4F$l-4=$gz!qt`B^m2ot$6%+uS6&h1?a6 zTHM$7SdMD36EV5|Ytnf4Pm2S2hIf4HR`TJ_b;EPNTBs>?erW%Yp!Lz_+r-vK5mzkr zB{K0??I_7vlnVd?NpPjRlW!~)iqyY4Vr!Wy+ic92dP$@rrl$`#Y2Ulo<|ES-D?~&j zK(9qDja6Zyg5^G;-&<+MSi5yFFX*5!DGHu&^6=TKNcGW8*sT*xnQp?Vcj2SMaUyg? z^&f2wRa7`Lsyd-U-3(Q5$d?F2Dh0I^wSMka1nVdSfRUcR?Y7#q|-8^045O1$mm)Vj_o2;YMnnsdq!^g`>f1qZw&*it1AYayF zP2-I6VC>M@X3&2en}4S2s~mh=LobBB<(uS>fs{qHPf@sh!^m_I+kCypzsI;AZ@qFJ z8`61e9=rovqPw}tHkDNDqo6WXU3(41YTTV5K?M@LN}0^?t~^VsUZG{~+dXTvF;q%L zjg?9t?P|Ns69ZQ_^`$kqE?QbmC3SDZpQkq`&~k}Vb1wqEP{<|cbve4V-8 zi_7Ci;=uC3zkl=A!sxV#tN@LqlY5}#2z)WY!QFgaXWUnjySC=`v-WD@Nl5$LvD?-h z8_uZg&M%jm@V=s2I|~;@E>_bN7A*hPc$%UZP0q5pr&Ab-9un;Ku!_U>!lA^%>{yk^ zuQXe;Dy(GXH1@lkf?et0wj%w(#IKBQ7UPI8hg)lwl}=#KrxbMpzF*Ha+Uy+B164xrQQ!xb^$bv0fk@xHavXxXuk& zSCy4(f~uqxixJ;P0EGbneM5a?gO%2wX&}X)cQUy-q9xzySbM2goK*>6$~D5H22tp6 z5`~H*f+8c80Ub}e@0c=3g|S240*Ul&1cx0hEiLies!*yUDD{zz2;nkckp|`K0ZmRN zX;oi;TPz&s4B5k2;?D@sXS>I}3-|)i{x5tdWTn-(RuVZky_?#zvgA;CzB^G|i?3*J~`vY2; z^I}EONmGU#`P}JrQ#Y2F6P_S)^$hQ{h$QuJxYhj<51#iMPx@TY^vbH z?;=f)qQ{db{jNbHtYu?If%`hB2^G9!BrbtLg8e=~K}F4n{yD>W#NA>=Y=tATpz4e% zwk%zbN_n#f@*F6yQd%$eOVn|ky}uhfViJ+UL-!U`UPvK>>NidRvnE4PpN7O~^!d?? zH??s9PO>TK&iHuApj|X8gz8(bPIn})hzK8^SrtrC1^WyX3PfbYocA2~caV+553M2R zCuMRT^IW^!S|+eqE>JyQ^E2R>&zRJh5|gdPGrvPNySDw>FG3ei-{0Jb@cg{tue2(r z5Px?8SgAKhx)2_`XA+gASa2D#pn zQd(d0{ti?}0PI$!P6>B}>2cK}=JZ&e^US0d$-g5l28BCGF9fkw{O8yy$ zK-k#W@bF=&RT19dnXw2C$(AcKu%NK5pI5!*<*XCjV3ItdYFz*;H8jOIC83uqRF_pK zS}@vNPzI$Ug$9|AVrtfadS?NBU+HF%EMXMk7y=*~Lqt;ru8z49)w-A2mjC>CR)2t| zr63600z6yZ2c7QD?S);QPt{t6{pX2fupRf!8Cv^qAH~3e8tx;h;L+&txo5}+ZKZFB zvpXk;XPXo8nWmRhWUuFhE!2FnwDx{|NZ+XO?LWhZuQj6!@ z<`=9b1u4#9=-O7;!R3+V#fR(9!G#*p;5ecc++QSs`4-Smfv3+Bc~?|mA3e-|HPAZa zrf&*fXwf|BvxdEAWB&D>NJn1g0r`@8Cr;Jt13tqXS?EJ)jB?OHy!}!5*m(v=gzk7( z0tP|?m)k9pCF3yk^0gv2u9Vm6GboC_QBuw9r!~hqXBdj%SGjcz)C>h0mDVLA zd%sh;unW1%&9$|)o!Mq4rkiWizPI3|#{I^|6v?W1d==49r(5MlnRS9>pvx*0HftJ3Z9bRCp)v`u^xPR_EJ1gIB%_^wA$iq$mC0Jvyp4YFElGc&3o#h zy?dQqL{_#f^5%of6}?a|FR!uF_xSRRQYS^JXLs_TDFN=^3_s`QD>I$_?2A92ZI@0c z=se@s8$a0J-<+t7iHU)N1_}W+wJxsK@!AecvyJzU^yo!9IFL#9<$}4lEb{a2ncKfy z7D6!UOCYnqJ?x_#{h~v0FvX^JS5v2pVt>W` z!KY6+`Mw4fDpT!oZ&6lZ*38yaZH%@5!OuQi!$unhoaVz_h9uEU+q(Fn{U-VTpjpSD zkt-1Gypt+BQm=7WmoXZZuWCyq5TAYj4qwKGy(U|xnOg;Qc8Yw;lL~=$U|4&N&sxRA zqZ%I-aHGeex8lgew099FW$?43eHaM>M4DOSRDjop=!xL?TmE0}z=6 zD_}>S3_!>F%^VYA}@$uOqWbRW-LRSg}lykYCeE7Gt7kW;P zBLp9fx62iiZQZ7$nz~oQ{Y})#H4DEl*dvTS?OI9rSo_Kx&Q$6(&F*~3ylwg90|3b9 z>G0b|8`j}Lr3j<^jU+x6{nok6F=IWC># z!X6$n*TZ1Klka>eIa3Bq;(4RIr1qUfg7_B(jjBvfa%_4k3~0SSY2*prx6@!HpYFIT-S(eoo3Uh^iiB zJJZoYvs$g$rKAUya%0z+u`qojN&}xCd%@VOSRdW-Z<8L*_+Kx*Lb9XBp(=%4Y{cbH zbxm$Q!sSnxQSZ8Be3;b$K7FfPuV@-NIussd=LmixRnBby@Tmx@OVjwV&_-Zdf%5uc zwj3T@bV6Yvr+EwosBK1&xaxZ+fv-mbBvIFL^qmyh`)ce_>veA}|KqqB3~_q|bBtX+z}@1?2UFr9aqpO*iqjZ;FyC6^9hN>~t+`=*x8a}Xb>2FG z$d0v`vk;H{4OQ4wndN^g^@(92l`dv0-;n_#(PCgiA+;xp4pbDGRYWZ208yc2cu@SZ z9|x00bYy(3$hl;{TL{Fm{?XeWX|+`vLTk4uB?ScvZ(t0|Djs`ntX~3vA%dlAgOn@) zTdz6{`w})3FF+e>Ob;!}E8!N)hntGO%I8Wn)Ii*N)lFOnJnbHP%C zeRp)^*4PDYtoTUYXGPwTIAQ=Y;QA3&U|AnZ>pfIV}QBW~AikGDNg3 zIA2uXw_iUBxbC)fo}c~Rq<9{+%6Dmr+3xZ#pkENr&RErTmiD;7X^8NA;!>eHx za{PnkPy2I*cF$Jhcb?5!Sh&jOkUO)&yZV=sCBE``DIl-jX`saTw_@p=ez%)F5m`bs zvbV2!JCRW-bt7w?LSaml-M8ppc?2lz)3*-kGBj@F0gV+^unsLJsBpc%<3+fG#d)X4 z?4o^|u3X~XmC@DQ`!6AeS`k8G$@j4`68m;d#PLh>za3`uxOiGB^%m*JC*k}o5)T7Y zxdQ|jt?hYuZN{Y^3TPT&Axh7^YtY2qyWzBEU=pUcOFb-0kCm%GF%H1TA(Q8qfC*y) zOC(Ce^TqDfU2u|CU;0b*+bZk&!%>AS8Jr~M-C;dnG$ z6(6M-79<`Zcl3R=$t{I8S&dZX33-yUh&_LD#io!H#R`mtQ}upfy7v_1zPx0(WWSbd zO)as1Iw5gQ9e}_6$fbxviP&?-NHUd$oi=~?chdafV(Btk!C_@St!ScM{20JSKq)UPr#Vx#h>u?_ye5x5C%`n5`(6s#XFv zln8ZJmJYkDSAB8bEzPueT#>mGIQ;sSkPOocg(f#b>G;OV82;(2qV?cI$JTpiyuhM= z)7ee2l;}Y2vWXcV{V0vzgea4D<*0>sKp~x`mWWy$7*|~9ziKTkcyo#gT~1>Ml!p_e z5QbIKMfY*#6_u^VV+pu-=C14RRy7>LsUn3v8eSVWRPn3{qxkhYLR)xvY2M?4x-w&Q z{7@0TRYlbR^G&hQk?*WZA5w2HVZIW~88d!BuJ+E(`862D3jYfoa^HCJKJe+E!>tje zbJvC$u92IPTe9b0wp`Git*xzr2j1;778W1$-&9tL7FbnTXrp=i(^7?Fpt6tKrkNJz zgXsY(S7_X#{3mHT1aov(Hz}>itUKVNJhNoXfCQu1zfZ~W5eS4ns<$v2ADLQuKlvdA zW%1?LY8iq9NDi}XXZ%*j_H2LYdG>>8t_F?H@`)@bdHMc_SR*a&jN=y148z%&S&_`L z%GYZSlcd)~89hUW1_E$=H2T%M8t00Od%UJ89~1*^nytqFn4dozd;|UX!*`46Kr_JC ziQ7UD8DDP@l7G#-;llOwjWh`01$(m$j4hn5!*}qvo(3 zm+uZR6-P+%tI}hP?QbE}NUZuC`0T9sY>Yro_Gi!(wOe*Jq7R@^pe@WW9A2+?dmYF7 zb9L1|0W*BW`iK0l4B50Vqber3BGA9dNR&)(cb-*R^+@PjA(;2rnOoFD=eqHd?%y-A zW2^^-UqF-V(hQAl5(M?=e!DDwF+%J{$g`yzCJyio=xcq_wt{9Se}eAbU_51t$=+;Z z2jaA(IC0Lu!*Br^14jS?5B(iLGad7)7ZWgRCIt zhEjCAa0E+anZf8m=C16wrkn1Lfwt(6pHp_r>qkQJpF6I68^0opZVrs`%B_}JPZm&( ztaKWlZ+C3k$Fc5+8h1#nO-~p)h#sy8ppa@Gs=y#eSZaxynMyw@ z*CVx%ShjC+*3UwnIlUO+zZ46NDr1@^W0vJj?%{CV#xtV3n0TB}UsiSwh{LHQnzakY z`>3OE8W<&eANK<3;~%`)Jf)g{=h&LxI<+K{YAfB~MJRQWx^0=jfQYlh3LHV6NF()q z)IV}mg_rwWFp#x4t8y)XM*YbDis6lk9r_V18v5#VMebd_^_b`axIvWy!`{wt!(OWSe>tH)4dZ?{QrmD`ONU`0?Bj2I#c5m6LwAX>(wrxHs9SyV&N8gGbh^@Uwu**kOF%z$<%iTapTSg9i+3X+{v_-T&#t(&f)rnYAb9 zA>D(LBnt>75(#EYDRs(7gEI%DL?CRe)O*7L!_R3kOTI1b?^{~~F8^J``0bJ9Q|SZ& zy$H!DeXZC^r3FN>30iFe-@~ZwtF)=cGV+^dK|%@12``DcOg{lm^GIxC0^Lh1;yUfg-sp6_>+(?Hzvedn=iiBMOVIhU9~ss+d(3xe zuCr7mu8;Tisez03&TBjW_1(UluT{D{T%XE3-`d)0e?`vSCG>WYdbvpQQ3I_-<{teG z>1ebK06el&WFi|`%{P>Obc9_l%?0gUGtuHjslOHYZbWTelH~A|wQhwaE`o>~ON-a^ z8n%vf(pu@X9}nEA;hT>+PXQY%x;`1AYyA)dSBNSEEz5H!J@EUzb#EmX!+JHLFM)>V zca_(t3Y7-pD+bZQPe^Oa*Y6F9-+8ap25L4^2K?Z%x5=`0<@v8XlbBJc{MGMcesQ|j zi^LN940sLAHjmAd%@^z!NAhJgov(?!XF6iiG+=4DT+ySB4joqc-qFCRJM74Uq)kOF zjktbX0Br;dEWCm^{bN*Ao7d=>ZH*It=3;KP{XDnfj*g|ICSm#4mzPqhO!l^@6@9Jj z_k8?PhvH@6tn{}cZM6M((k=-z4$XmrWFiD*KRLm_O6To`o*i7ApL&p)lV9zJ8;K6e z2EvGd?_}xUvh(lENa)FGrX`u67jLyqyWj39zA&&DZLMD2zUP;8A0Py6zYb^rB+O7o z&VzoGN1?_F zm9x-8PfbnLK6!7mW|*e3GHoX;>5j#AdvbIEUpfN%6dYeK7V=U8RJsb+xfySS2Cffm z-3SdPHx~KCftaA9`$dR(E!u;6-4HjQy+dHuv1ajW$xjZ=yBesr_W_mcc~)>Ak?-zM0nwax6Qkr2Wd4p!z3vm(iya zwRNc|bO=jMDa9+Pmm=&czaMoNdy61^I9@wEs(0zQ9TgZpU%_9_81PZx=Y2G>A+Q9F zeQFTKU8V6h;Q@#-s>sX$OIPpqxZ!vCfLgo|Y^WP}Y>sLuX`@dF!ctR2D)g=df0eCv zG`!Ud<>ULg;TuRU#oSO#b8T2h`+K)@aUL$l2gsI|j*fZDu!BB_%&95ibn@TOrbJ2U zQkr~t9`CT7?DD6~AjuHs9to*YbKRJj)4hccp%-U!vjyb9z{OI}6^XmtNu+sLk4QCM z(doa3FSE7uWHvV+q|V%CqEH1Fu3dM*PL$!2@8iuKD*YP#^FOIv<*|Cn8SuU6aOHF% z?2^pBu9{9UTv3lBaX;V~knArnB|B)+xq8cmR^*R8E}ulsW?qRm(Jj|ozcDdwv06jC zLk!=4d@OO0Mf31wsNct@>5XeFW*oQ{PfCDfZH6s=h%{z@)P^NpANjjx@1)ZpG4aDz z3KbSI$U);sP-?mhOG!n&E17;~Ju6X;8BZ%4Gp1cCr&-L~q39z~0V%dqMcJRE+dgmk z*Z3=T&(r^cQQDwU>E=?tFHcoPfYBEjvkb_stYTAEeJZ-qE0R!2>T9JsQLNWS_*)K$ z#4gnoszP?Wt!)IVZsms&Wi%e5R(3>|mrl=O$ny2y|9YBl^pmEy6OCwIx3r3goUR}K zxm|lw+Zokm%kJ3xJTh*i3&tgi_0FOaO<&I0R-bx|BZ5KSu7YK@+ zTX_64#g2AursclAJ~Ch*m#$x$mJQ$>;+J9}Xmx@ifTv=AD2Ik#y`94K7^n)wO-o=1 zcdY{TX;{t6+s3SYk5N6rOGvHs6G;Vp;XZXr&WHaA6{8*^d3fe?9%p^=GpVYl+dMbH zs=Z|jpaD>7U(I#rofzs#4|O9;)uPN86vC1@=_+;HaoU!M|8aDl;cUKb7>`-hh*Dyc z+O;=To1$h=rS^!e_MUAKdzKhQ%^Ibsy=&GcMM+V+wYL)Ceg5zBDTgCpy`8fVGnsj#5%)!<*|BDSS+)D>k!%oucHOuF=!zdc%N zUJ1i&jpzEz<#Cj1Xxoo11{TNSa$!5R$1Q&lR4J+=ltAwiE$;5S81lYEu z=fHpNW9FBCmwiG%)7oliw5QGeT0Sz})w<(K?lHZzb8)8Fm3u6*RcQQ<^u~l?D@^HJu5*DQtUe zq-yMdXIz@obQW)t8{XxkJA^mM(rSJU2?i7U&WG-k644P5FFS0n;V5!j$^3i~%0fg2AI~v3Io|xDwQ+prH$Nc$ zJU|PH$nU+K-?HD#o1f<<&ciqIo_-U(2X~hI zYR{qC54OAb+}qL98EL13mPB?2f>M=LWQ7*ukB1CKueZjn8StM2CPj0g?Z7a%(9@RN(r;Dy%1Cl)Y^NlrQTNUB4@WW& z#0#dTTddyM!HI^cZ5*r2HyUY)B7Xm4e_^)#f;zN%OA?u?9e*gij>O6TcO7QjScF8t zgU8|A^pD2PSCtAYe-qg{{=`&f!03g<>>p7JpZNjazQ~^p7ikj^{#rNrsvo$HS$@gZ z5gc$B?flpeJ;S()0Gt$D0yPlHUPBu-fmY`t4bgCi4gnppUODP7w!K+W@^Gay^wcBV z|8#W;i*?Iao5h8b=Nxh8z2!HWeJs%ZZw;tk#4iop0!ifB8V~?BxnDsp39wf;KV%^U3;+4+L zCU+2hGzhU}U?}6R+^$0|q@4Vfl^Cx(L}SCk>sBtG9>e0>T1FdwzseOpx*O7Obs1+D zLOaz^4QtUVfa9q_dcOSwq&xnncA<0Eam*OaiTc-_$t)}17hkd#2_+Rrz5{D*7fFNo zMOB8V`B>S6SRXt~dQb|&LDITMyooHr@mnICY9O)eQSBIgYWIDaJu=f7{_1+|2jFh9 zfe-X^m9YwHpF6p0Bq6pwn9*=5ew7d_rA`G{9szQ1DhJOfEup;D?X~xq@XJm9jisTbLpJft2bjjN>j-abAD z;+hsID=GvhNL4g41y+G1%Q85<>-b29p(4O6wxHg)h7d zyhR^JgAtU%(V4}SO2Ya$1gFcf5qs;}WWVYhLg{yHN-|83HVi+CyVw`cR2)z|o94p0MPj+2~X!{tx_@8^VG0x+)mOYhUmdAyLN`no# z`aelbLe?P){8f2_y}s-*Ju1kUN_0!lcscn|Jd^JZ#Ja8RLt-FK)cjNo|oVn6e|GVs~MNnW5Yk3EzX zwGwY`1y|bX&*lvRY66z9e+NT{;o(w@CG4V)RlQ~wYln+tuw{v4{^rjjDW#j10ORQ4 zf3v*^NA@*^uD{T z-7%>NiAa_bw3STT=sv=^cQG(`OfnkB%og!eEFm{ODKm#m^i^|JpAnkUv~4p>Xgvu# z+J$9)cJhLOTy%7Fj`3No%f00G81Qx1KKkI|wrhW_X*c!_eTi8K_Zl)S3cT3l05=kP z9B*j>?>h1MqPj6b@SBL3ot06>(Gr@6|?$WwrnG{hF>(g=X17FA6x$m(m}&dPVqY_BOc5! zN+3v$Zc09Z^!5{QfBfp;*?mal$U?)2j^Y*5YO zSXth$#J_D5eIRI&gac`#y5<6%E{;^)&Yu-AK-;V7skr{Z&x0N%M2g;*7$gFl|IGQg zZIQ=spoyTt9i8E>j&qa5qs6F(i2P7$r?GuyZl7!ONpdQ#qL`R_*_%)KDbzqOaanC>=uR_+ z8rb3$N8T_-4yQbvLWUgVP7rIneCzpZ&UsPVZq*Y6i1OgPCnZ`xXo-Ak;$9fx+v6*7 zLt&cqlS46>b~HsZi+fTGP&;uXjNnwE{2s_#iqrrUq&0_a2RqIUK>I+jwh3eYoAV*0 zjE9RSt?~>u3!~O2?RnB8e^Fu1%sz-%{+b6a2_O>{aQ8___(Qis4c!xzpvAZ)RP#!6 z@iWjnz0v5`k!~O7gjk3!LnGGCpHc}~?-QG{ zDH;FdK#q6`yRhWYne|iX?;#WMSsKK-LJR}_DPNn;>|jL!!JG)WB1Fn1Z9Alxc7heg z!;$3_n*oCICkSJ9U9fpseb|>dRK#?~pB-FdY0EPz!#?@I&O;I({}|>40braKyN?`w z#rLCooNtBjEyc2E!v^RWr_RZ6bF&2*!6JfxSo?=E5BCoExbhHRxbvP%e<0L1Vi#A@ zVzhNMB`0H}uVbT5p`QG1fQL(J2qHSB+UBg zLtF-I5Jfk86R6#|Q7we0_U7K%fP*Ft^ss#U$J;U4NyU{jp7$uHBN`;o$x3wwc4fKj z7Aa!zSF{jXUHTTsq38P#%l0;pl%G;bW3J0Kgo)V*Kxpsini&{BD$HP7X~9%lN%)Q$ zA*@2f?_9KXPAFPlDjR5$ffi}FX2@XL{3+d#^&@q(we7;>V9x z6G`mrTLr3dOw8_wa=(^Vr0hCJ_$e?x`JETD+G6b8KCNIY#tYS2`7 z!^&YZ+RdYb3M@Kl3)^8O%X^$CxBPcu_5M3M7)&6&@|{(#n}6MkUugOu@edm!1jFD< zk&Fu0CwfM*ZxPhuBK*+h2T|FNB!?A1j;!3IFL(El3%2cFPR@2 zGoV-X=27iD6$s9t)C?2JWiZ~%6KKs2%$raVtD<#v?(kuXxczDSZb1K|5pI{MiEi-0 zLR!T|9FlGau8StAon+K0eRs)KEQ7aGCUW+;jlW(H|6xEuot4zgPw5m+B4QK07BK!yVZk( zt$=XgL+_@If{^(8H_sCrhr$;HUeww)d~Djc*#L77Iu5JC`*Xqp!kX#Fc8B&+qwfY6 zpC*;6vhUH-J|UILkI5^C1kN{O+|XZLVP#Gp>hU}qdF5=S!od%0{km|Z3E5%nl&b^_ z&Tv(dN;}cGwRx{qNr#xM+G$+V3YLAhV7USV%7(y}fQ>21{bB1b8Q%G_@w!^SDZfa2 z2txOy|3E^0I3KamLz~^u%}m0Gl;jzXdTWTL#(u@Vq>O>bET8+HxA;_f+&fRz zg`Tc8Oh)Zl%*BimZ6G{Vd z#<4`t^|^Tj%pd()6cSw>HZ9Sz)#M>t$71(^V%&d^`@!qi%ew1k_WCYU0s@!yKNKF> zPz_MWep*WfwK|DCq#JT# zJLQ*(Bf8<8jg*e2`L3zI^=+(usm9HAUQ(ON&_1WktjwhCmDwvzm)O2h){G9>VutPb zp?{xPoIR@R={}owT>Do_uE}2#-4i&s>xw9)HmFQ`!jIK8^sO;4OIn~&fggDqkwBp+ z>Krwr`iwmG$p$$xkl5qkU6)98N-_v!_Ko#Bf(;zdGQF}|`{}l;#xnOTfH}18hmm}*i2YEm zmupyNc5t8z6!Z6QcS7VS`t&r3$A2x4(zqoVQ3>4Vg4*P$_W#(1^QUPUFzc4bpsu*> zho5Sv=TEwkiseu6*u&Y42N>JRbWy)w+!L`*=r_}vZ6FMDVzFNhkZJrr;Va0rt0QW0 zPzPapF|6eN^HeD{Ag3%Qqm=sLf~!PYygxIpe$<;&Nc!H!PY;B!SC4|0x*Nl^!e@|z zf{jP$ljPWwGDr)<^dKVvx~dEYTI`Xgb+hZi2isnWK;d0&P}0S6MQ=04G0{ARFp()wSY zxP3S?N9~SUJl^~9Oqr6|xo-)6={ydnj#CMfVu#v60KK#P9*^=0&r4?0c>Rf(KmTJmQq+HIW^Q8$JAS6O3R;P$>x{Mm*6HQj?^e=U2CSI0u~G z^lB`=gdmETx0oT?_E6_?ryMbP&M)JnvpJj><|~(>uE;IZ)tPsWa8>GAf3+7M%K_b5 zbI|uz;#8cof%wPMvf3lEx*htnuzE~S*v}FD3!1YDUZb8T*44#d>kS1K45VI!9{I8t z?-(WL8{tiMvKpIdC3hk!C(Z2r8#k2}!#=XvsfoqR_5EG-@DKNv%Ws27h6dNISgaHN zl2PH!E_Z^yuL>Rdm9!q1L)ZxW`mRO4FfU5_0!st*Se?H!qBbcGpH>4s%s zbhQ+D@LVCmx|<cu=lJ-p5f|+I0x)WnGkKSIpd7rkXJ>v{dpSw5N2+76TtST(z^nh$z_Cu0Q zR2v~)RYpbwCI0WQzN{?37IDb_FeGeA#P4~d_q0T%kazov{&F0`+1g#X5FCi~3GpA& z2JZXz8lJR4VQBUBwrLL{GwgVy1qeW~ya`T>$dk=VJ zjPGXVLm7dxon&s}eZW?XCu3%5$$W6|JW}DK+Z%Oh78b93n5I3wH$BH@nm5n)uN&3W z5Z`>M;!4&RA?!ALi%q+WO)jRmd1(pjMEcRzE#2LMsxZxF$;)z-M!!)1&vR6U*91h$ zgUZ$r|8YK1@K*+NE)mfpkPVSmKT)1{{=+39DUr!B7%w1==wvb={Yd#GFTNxry~GUM zx-$bVTF=6<(lQ|B7tO_iK59U0=2?aXeOP>{iED^^JhGYr^AU}GgdWr3%mYC*~ z)83wVfUXfNZNZYgqkj9W5bqvIFXqi3=Z>~&%^M${@!I=9FSeLKlLu;&fn$?!bFkE& zhnk&LMMXEcjWfn%wg&R`nm@)jI=`ZV)-Sm$6V;Qddm6dJt-dCN^!egpI4X*pdg9H3 ze^x#b#wo{kZftpQ7Wx71Da zD;(Qk8FF&M(V(H>L?5#X9CwhRQ>lCuH-|lCw&5Gxh%ymrWHOE$E=VuTY&7DBhof>v z(a@0%Mh%wAn)@4bfF}5hi{ss9iqMz25xOWg1)sU6br(m;Wioa z?C94q;&FiAc3QSFqEKd2LBF$tkDCCFEI|*yZh` zwKAs;UU*Jf(<)46w&=x)REaDQ=?km6+a#Ey-qaBz>7u5Z$TVbhi~a9s4}MG>2QreF zc646Dr?IXl(WZ&(WQtYI%2hmjvoCAS?BHzp5v`>m69!<;H{d^j4{r3Mf4la)B-`-876*^eOE|9(Lq%VgfKieq$=;gC~urM`B zRtI1+|MN&ov)$=>PZ*@hMJvyA?xa!=t1^P6aq{F<9pBNkjLHex5y1 z2?z*)RGaleQbg&!L^JvwmebXf4B3 zjuK8(&WVq#K+DU}1bcubgeEE>^cL;5qE2RDBCPS}H_r z+i%q1NSLC|kFU_QvZ6ZZD>5f?Hv{cUFHld$j8us=WY9C@^m-qQ?GuA@>QRE3a@YrQ zUUUSOK(mX8$|^FTs6xh1hr!7+dw&0k-74AC?uBM6qMJfja#zK)%h#WZ<`Cdn3r!bs zWWiwH5n7I$UtVrHRA`O0tk zTL)*}OYhM<;)2GiWTh3F;3CIxv#Vq6jZ(wmjMCx1J z@MrxoeZ38@D+jcch?c1r#sCQdw{U%KC@djcLOYp=MXnvrL=_e#Mmgt6h98D5J$tRD{Vk;6 zlQU%K7>F}@(zA7GC{wi*AmATF#7u490OKnyEv-t%Nq`jT(0Z?nG4k)`XqNo#zqX8% zOg_=!qU&aJ5|t5#0YuTc-B-DQE~xuoe0RXr2?vgut3P1W#N77fvV<|} zI*e~t)plKPr0aU+%XPFRK)9C15YwI_o~ADlwI(Zpd*enqM3NCb-8w5ETnlxpK# z`2g{pr%Xa}y}oruk8&8~HbFES5K4wW>;D2iX_obB#P*IIP9sjqFhn`6M3moX7?npL zT(uL1zE$)(OS*Qc?ZLShVD;)oS+x4PY&j|W(NFaG$S@gbZl7KO-1>6V)xpKl zcdml$)ns77us~Cf>$(8AUrw)X0NWc-t^|&VCb!V*1q%5aAoTdZDZBgjyw<4k5W_i` zcRFtw&H)`-SJYheO~qrTh!VRuBO&xvuRfOA;0*%tQtdG>?u-Ze-LOhl`zUAASSrFu zQROK?8``qmwJ${#fF)2tVn3F7`3JXjx9S?;fGCuMpK?ufoP4TYlIO8!w%el$0-CnV zoxxl->K}K(V(J3CbO^2fgovCTGMg0KO0doRSL<+J>HLXxDIL_=YF|v*-$CKd@VjM| z_L{$KHBPqZ!J6Hi=o$eD3N9@tzEUv0^UtBe%AE`ms(~roA{@HPkM!UE`q;qhpjQ0= zCcj+7HjAPWm zDAQ((v<@+jz^HHQH`qp8(evx|4or>G&UTd;m6ipsWo3 zJC0@<2hfk#4VQWUdZzSTJi@ITF=B;pq`u_7Cc{E7a3{Z&=3 zajor1G#G^Gx)&v;gYtpX!$xR>-E!QU+ab|^17x2c6AqWC+d;mG{Nx_gMi2ZH5)pnh z#`xyc$401OgiGMbuO*p+$>!$fBrPYKesGRJc7M;G_E{i^U*YWu)B?@)FZXUpUiuGN zbC-tPEhW1pgf?SuF#ufU%!nwGh^TruXS51CFi_)^qdl*qjND4eNOvbp@ywIu{}iz{ z{!__2J*ilA|(J zHhxoa9x$fx=(GvOO6w*f908vpjC z+s&|%C}~h3j#%Bfz-y3DQo3LjLVe=vQbYxw;gaq^X&xn|fu;&y8dJi?aa*3~Ku@U_ z55eHR->ybh(|%{IcLEu!?m|iq1Z~aRcYFX?lD)zML!?9p){k#yw=J<*cbj)tXyE^_ z1SW3GHof-NF_7sDJe~5F@9qqw&2>YGg{}bT_VweI1y`Fts1j{EtCLj?x!!N!cxRqX zS@MAqBxj(I`#QYg3TS&I2xn<%euUr@YW+3C|1q3-&07SEd*t^tF*ob@sW&G>of~@n z_~nO4Se&CtwRrikblyIS4=nZWcdzA3(}~%s2CBNZT)$|6UO|B71bKekQ6bG(qfC8D z*I|6@hDdRDOFm=fT(Zl>OI6%V>7K`B14h-~R)|gFt{0;@g=~&m4K41-%DA_vB*yOj zxcggh>qG>I=h4lzE8+kC&Q@g=gq(N*Kk3`>BMN{Za(Kvc5zI=;{9;o2ZqGr!9dMVl zw&FzmcgI~`-Yx?myQ`4flTkt7>D)C;INsTDZbruvd3Jr^ zseICY9E-fN>2CjJxp%FNZeqk;gehFdkgt5nJ3S7#Ia=)lB-6nC*2zo+QsPvy6;nQ! zy9DUf5H1x%+cL+>VQZ8`frUBH$aiH<+P*w|ZMX`YG_&EymNx}Cpu4>SY{Ov}=D?!` zdo^zvifs_I?7ju2YMl54Nik1*J1qOz@!~S7i1KC>N#FMper*`L!*ABFNh6H^zBO4! z;DUJaky`{Agyr7c=XD*U;KHm$LC43Y;={R2SdNX=?~HH>s-lQxOm^|Q&3slwsfOQq zpa((YHPR2O7rLD6l8Fts>*jS24_!LH|1xU&sQ8B_VCj7Ac2Cqpv9S=xSAfeaW!`*sivjEzLp8hdcQ+_F%F=94 z4)`z*Kb4AWfcb%#tJ!einpcD-b}o+c&;%i5<5*Z2R|@ z%Qz;c-AO889qFyZQC_5dpRtHi7_MT63-=k`pLxXw3>vXCvs_wcdRVwG0bcWz1q8Tk zHmu}h)UlnjAX~&1 zFSEwwc7wLNrE8I@cjb1g9X8_b@9BG0?O3rG7hVEJD=pNTQxF`tD=^S;0#A(nL;t_K({0mtSU+XXe4gyb<&f<6o#B3WFYG zV<=`JISfi)@JU^X|6WT3eR1KmG41=yQeGg?$@+^vnDJ2uJV9tOyHdoRkZ&Fr-6&Z< z$Nzp@b}#|1+OHKatoG`S{X4tfPZ7CHR)LIy0O?u4PtejlK4ZU$BDJFkSho6DCaAx8 zIj*s>AmYz~N4T$V6e4%L%nhyJ6AFAB>Ra8y0^@Pe;)#`ASI0x$w^iM++3`{Ac0t%j z4ZwfieYbkPTM&AE5+X;N+V?jge=uW4NZVk7u@`}ZXcdbTmWb_p$H_x6WRMTcp8tt)P8>gsim zDRC-{7$eKcsIEU+6w|KU;*JLxqUZ8=n}WCVcYp7;YQwOCE1kiT+UXErnosH=aN>t! z!ZE9N6ZnBLRi(g5;Us%^1W4+yX9dG=ZiDsEZvQ1N=zW@0^b68hZp1tbC}OAJB&-{ZsAA)Lfe2-pw0gLg&4%Wh-ChdrAK0z1Qgicii9 z-cwaLrGF&OKlW41oa!xt6;`NjHCQ;1TjPutLzFp>{kmLoZ>AhCM>#d4ea-ztq*j)g z+}+)?p`z~1=-n*A4yuFAO%6a&l8ZTAI!x@4}X=mx|F)BZaAa%%~* zbg*{2dX@{gWp?V%WaQbfv;*M^4H+?t(uvEro9AQWWjP*LQ&CM2S<& zVxO#j!Ik?92`@}QM%w`0^j{_j_}mZW?@mU0JQ+v#1DcmCS%Fe*j@m_Qm zPNrL0iZISl9hNr6CU(Sb!Ed3t@(bPfsB`w)s*Dk_&6@N@4x$%x@_d0>ra?w)xNpu* z*o{G=l*KtFLKfl|pl&G*v; zd?iCe7)5eM8tmK8qpROzqbP7fc{miCi8)dng19 zy8@nIZam6x?5^eOX0$vOCGVC46l4IyFW^NF!|XQ?j$giBzP-+Ujp#F$$6T%4o_Ake zh6i6J->w!stmCsE5!qx1^GNjT1HSwSXLHmBc)GA8kJwv4x)k--=jsHtODkyE(bgI& zwZcq6_(MX4DpC=EF8A~kX;NEsv|OK|bekETVIEUf@umr!>|W{fpES8$VmgP~vs+ib z?(mEDItVltM)8Q5yXFWy6t;aEB1<8?9Up&mwOz0h((>fPMEjBeSLZLw($TyjC#nC` zojH*3pwo-ZD(u}F%k32~tzoM##-ElJ*wAubkiilX?Pda2CZJ+RKtBsjdD^+dj3D53 z)F2U26*5r#>TJoTz>ph1uBIf@$5V>`e0x;0U%bcEMV?o}#K=2;D11(8CP$5y)?Ozr z3)W%TF7W%R(I#?0zizf}t!HQ|gkx>t<(naazDVXPyHI(TcfC2Poqw!aU)3-hTdalM zo(gT7y`@b3W7ewQ4n<1Z&JH~oN7OmxEQO&inIghk0{w%7F-IPE>&YusJDZ!EKurRB zvKbG=^V7G5~CdVE2tz`@V7^1Onb~*;1=12sxdCsyk2x4Qy{QGn4?Ngi9msxJ**mKLO{wC z7LMIn8yFSrx(>R*`jw33-XOvBTx4MF*09lb(Y*8X}%ZT0j-tOf9 zHLcvTMjrnJe~!1#Gvjo4*I{9rtjN(XXI8n3}_2yEz^z#d+U?@MFMJ z9@d85hY#g*7k@*+BX^5qwRuLya<$j9Z}}Gc42Qr9~Y;Q&J% zfHV#mVAp-K0Zf!xKUN2i?fvu6;c=75%~Rt?S>Byn-m%ORJQ9a&@9vA8e5^yFon9;vSZ(@&984JzqqULNxpAx|C~xD^=LUdFE;`7<-_YqEC5 z^=ac19FVHWk06R%^mq#$g&uE!N`D!TF6!OSUK9_95$+&__Uv|@NJVO5iq&DCd_V3LxRS~`*nw> zQrM-#u-uuxdCRaX-P?_Un9I$j_5$~hf~7rDBT`?{ZW1nww^j02>9t{3|w4kdHP z4M7P?-cJ3BXONJVWOgggw!gWP@~z8$Y!rVb%U3!&+ws1R!rMK6wl+o`cUx#DwZo+RY?@sUb@0fKqkFNUw$HBH`7y#6X?^-H_mBmWt>fYXl zpUE>I%DykshVyDYBLOZ4#Mc#z!6|n&ag~QZa357>wLYSTkqwLu)f}{+QnP)08C0;o>1W;1&5im3~l+b;}dRCd|@*Y^dbBWBIVl z+x>A2g}g>v*U#{)^GRI?BI@=g>{^3hZLOu2t=wqK$|3M~!J#~`6Y8+dU6~pQ6q@(xqusku$Cmv_m`@Fd%@!T??DXSp!Pu zSB0S~RH@~(n6B=Qj*hmr9ZVPTvHk@g9A?x%Sxr9Ox~G`c_QJR!tO?!D5Q7Nrc{KQC zEHL-;!s%5AkV*hIitgQC`6XA0{N}LJt?{L$|Lome|8}a{+n>}n-rNAVCE(=FI|Vo) z`v6U-0dseJUZ7piZsNxp5L-#_eR*V+%sIKXbT@7tO7dqCG#>VyB;TWXV1$>LEY9h+rUYyj+GRz*mDi{L#% zosxs~vCAY;jR(++P(aBVU*zHH1)+PU7|6}+6D1 z9t_fKu5(Sg>K~BDFfOpj2l+IaFkL=zxjHVc8i!Mbut77gP`Yy6Ex*W>0aWAp`Fli~ zwF?QHq4=LLZlSF*4<}u8Soofwvi~1Ij!!YPKAp@nus{@Yv>cy-Ff)$ARZOe@UCBE# z6RQNHiEaAwsZE|S`s0fGi;2{JYFvA62Tits&&J~yA>LtOb7Jost8-KUVwbw8WMw{t zln>dFx3cL&=U}r^S^1)BcFVPM!PT$bT$9)e_MzYdov^q0=ZOg=3(bWLy-fQAPsV2= zd9RJ~c{Vqi+|Yn5)ehg>Br}OCiGyRD*$xuTo?UI5M(dykA_U=n-OFptT;jIC2jeJi zmy#z4ljSUaJZzf~fzpBLqvbGRDx`97S(^~>?njEAsB*ed8%Ut#`~R zqw~I(FAltdw^3$nQ@o1Zn4LVBEX82Dz>s^ zKRExw{#;wHoMWv2C_!xBL0I2`-?CqRG1dknASBDxm1P1XqsvryzHbFbXI`eUk^BKG z8=i&f<gy}G&@@XPIs@23v5NKw*E^O}`<_TH(098Bh&K(!O*898UVJLcTD z=-vz!A^lJZr?v$qf|9cdiN3ctq1F|71q+_{HwSQkPoQmWZ6klK$9`H+6VNU1`vF^& z?Oo6;&^v$5QdKFEc@02O$$_Z0-}B-ZN=7O#&q@tmuR_RPBF0b7Q^}%C#8v?r0Dt?9 zCzTui^b{XV$sKf<9XQvg02+zWG*Z-{rnyeC$9hQH-G3x78N|%?4uk1z9B6aTYs~Te zKqSh50|gb4z(rE3^YZb|ZFonwP21W6-jHW%;IDN)N-}two(X9|PEk|n6sE_UPZJLl zfa}kYC%a3^ z$ctkjb5(5>F7in#b_+5TH@B7gx+9IB&J%6BdVn&Jz8PVT-nXp@vQ&J?{5u1toaho6g=jDL>hO%zlXe7b z270a0ex238OGH-JYida^eBT6v9_YVFo%FM>ensAf;MVRc;M5c(R4J|QEiOtU^Ai4O z@VJPI$?Ai{ov@CAhD5}I9vhA_bfz`Lq=qsdIg$eO|6}(j!;Vj@xO8t< zS;8)30KffCRqalFjc~NpD!ms{15YF43RzAMLw#N($mjoGn60?0uDc@s%YUBQw%}~M(phDTyZ({ z>eU_&L8^K1(?C;NdTF&Wo4AIl4^0e>A{5w2!20TOL~KqO?-*kdZmWzSg+WhB`XV9{ zXaph+-{V`FlrcSuDs=)T{e0$JF{CG*kaqm{z~+dv>Fajv7cZB(SH%W)zoc(nu2zNKX0hcI z&*VEov1&Gz6N1qI4%bHAnn=#ZS|ys#+J=W)B?FPBjx+Pyj6QV5S8@oe?cc#N2=LR@ zG;zE(G(2TEjP?;N9vNvYq46(Yl)6;b`raLN1c z9Ur7*5Kt3;`BH5pOl~yqo1F%N4{R2>1kd7k|G_PoZhKkc5cFDHIhR_5^&_nXFlT9*ov7ry&uC`bG*MX-7d z3Ca=od0Kd%zRH{}9R#%O#@M-TH-UqjQg+>7MomG1hx&b~Arwof|DadmRqff2T!c6R>zq zMip8Z=~lIuGPCM&juWQ9Oh2a(DM32rmHUghy*$%kac+z;9uYJdab9ar%wIQMsqbH6 z7(eEDs~Q2`Jt*{5L&|HrTwx;Ylpj_&3p}7@Qk$ZnXM+nK$36xr_{_2W>@{&%xHrY# z%Fwc$H}Gdcs9s;47@YDQCUfCkPmb6FfJp->iYBNyL^i`)nF_rg5lhP&-P<>o&TZB+TpGDsvzFGU(WZ}U&1}f{=KZJiNTPt~qkC)p0 zdnI$JQ{7B5iKR=ekM9xkPxP;sC$XKDiFlt_cdtll8_*1F)Y+UY>;56lA$kIfdwx9R zDu%k)ehG}ua{Wo4Zn+RFxLDUazAz%)Y*7nLaXwK}4YOAn`~11BrV5}KVKA7G)3~A| z&u^M`J9Ohe|7G+2d>GYuv4oz-f0@Za$)@y|#Fn(sPMsbBaNI0v6=(MM$5G|F@>hSp zyEpH~;5q+6+p15$AK07?Bq|nGGl~`#ie-KRdRaSPoXqyFWBhStAH?t>(dCfD{is*I z{ewb7fQQQRC|*4Qgt8gWvq>ML#4LZP;d5kZH_J)UC>G^#J&d)088 zeIJ|C?h3B|30c>=M=DyTW*4_DBAA^w2nKi&u>1T@)?$=k+0$7FLE7M5cIwwrq;!c6d z^dg6edy?yRqq`Vv60*&tEr+Cn`m6bwVwzCgbM=j8neImKjf|+H`12?ei>e=Z*N_r> z(0EIUH*6}NlBXcJOG*`6ArI5!Zeko0Ofo_Z8{L5G1n0MNa3UBeb)ljHf_?BQm2VLl^+motQW-~5vAZOCM<__I z@>S!-o=zJOybK8?zt5xkK6Zm1loprFM)mj+3Yt|;q@g+!+|d)CYIlhrzLQ#ebv} zXX2i#-zU%6##_m~)k$W-ZBN11fjD~&ZB9_&bJS4W+soUAcM+U zZ>5pbLr8Xx2(2p}B9KP^y?w&hTQwQH6Ai7g9gxT^|9=+o7jIp>cA`(iK4lLa|HdB) z`^iFi?ApnbFwU2Ykm}oF398pP!_qzj84|cK>F`P_^qWjb4jMj;8WZbyewC6N6F#GJ zB3C={vYcQodu8L~>#z3VsrlG*4m9z^p_YmsyR%Yl<+XA6#bnp@7Ermi==`@j)O_gf zYJ%M|mTwQcAyP@_rF$`5olj+CW-uW2Mu~d1q)^k!Y?bsejvAvf(WeMpxqi}4xsE{Z zU~g|(#&2L&9_~l1bqq488A@8C9AeSCm7w}K>bGaMi!7K6spypI$A~FA3sXw4XB1w< z#M*}7~b+>g9HAQ#@R8m78tvbw}+yc2D=(v zt2h70(Ruh&{r+$Kn3>^NhlF!F#wp}jA(@BF%p34CLS|%A7Yc1EVg+>RRoa-8ris^=(ib-F}1g612Q+uG+nnn9IkubRb%T< z@&kc$Xe9+~L@!_QN4^+eLhQHM{#DZUC8Crzkb)ePBbth+`Wgj?J>TRiOVLXru3h0A zxc)@*PKj%f6U1pLkEtf7K)pq5S!LSVdZWP#I;B^*mbb<*~q#`-tj_{S5E$1QO| zxJ>1TGHPI}TA4U#zjz#U{?Zdr}6xXir!2T&FIqxC&}qBFivJLISNNm!ICZBjXEXW zPEQ2?n=3c_xle<^&L67L`ko)Gov)F=nAW&hB1*KaH(%LjdASkL>R6HxlNV+5Mlq0u zM&n2k)_>X{-{V76eIuDgDjVQvIW&HIIs|cnVxUvB1j{KYJD{|o1csK8PNJnHWCchi zLfI5zWQ9QtNp^gm^yTXL^NGA??JX^KS!JHWL;oo&Bf`GKD~w}>SX6~4TBVgcbW{)d ztlL`VUX4#oG^$W_{Az@NA`K!$ zYsCu19@=cQeL=1(9CF&0hIZ3<4OS-R% zP0U$&0U_wNJ#m6@s^Y`QP=*7(;81JRlF@YIUtO*|YELsZ)&z6@*^kVJ_>Pld2YVw7 zv~_S9j>DDY*fgw4LJIbx8RN+3akP91(7)E_0Y{?;!5QF_%51gAw;_bZKSyd2;T{ke z^r$Nqf+4!m;z#7d^dYn%1QsgNiWCpzLdDAjaSAebF$e5Az^5L!X3d7fCxD``mOwGD zxtO)Ko}zYN%F|b3=bXIb;OGdqI^&tDga5fcn1LnJ?Z;GF+HW)x2)9kFxE zbU<*W5?I8tCA{#$@Jj!LJ2_y&?(o7|)xL2Klz=SbGwTYxoO$*1uJngi^F^lN9WB9; z=8}{$)4?HG>W#mt`z|=-8=1HVZBF-+2G-q<*b)68Nmfi2`uyzTXIq1*vn0pj!U9J- zQiYc0VXIRH_e|?EEH+Gu8%wcTSutLL|JW!Uym)%1%&cpebi?-KpkBD2)}9O+o=`6s zxa2cq=%~kG_w=BeduhAPGzxoa=mG@kJK|v@LvaX%Y?^bn7iF+ez}3E#C;RjY`4`76 z`%gnTZhjIlooDa0xJnshNkGu`wUu25(YObVz89PIygzqNkV+gY**>oCn}81_Le+63 zG-|T0srlL68bLtaQ6yo7$05Q$`6mz;RXEV1B@k*bx}0N$Hq5M#T%bs_C`&S9k%xT( ze|BcBur|S=H`TorRm^>D)##a5T`p?9G$tE*kzLZ(pA0w<>6n?OVKk9!8l`)dp-j=u5Cadf6fScwtnNf$13bG z{VjQ}mwZ3xH15T|pDIFT_q90N$>mHurz&MW*{GnfW2}X(6B#h-HSmp3e5i48n*O;H z_twDbBa zJ+wwRN!FH1=1(m+;UiA2qD0S8$_u;9gv<{!;hw~pZ4ED`f8I+qe}0buo6Xwb+YiCF zr@^<8OV><6Cyg8V0RiIPvoAH5Is=W;Nn1&ipxS@4!lHjyd-k~n-+}0`iZ&O#i(HZY z(9zkx_GV=3Ze_V2=gM2GgQGrZHL^og>$|0s&&;6(%aPa(G@2x_YWn>4Ed4h0rC0p|u>v%#twzHzZbcl2VaorzXteQgUT}kn0v9y+ zR*`^K4pmeFR8NK=-umqq!%;Yx^x0PfLX@`NDt!mLw`a@zbMo}~iSX4g9G(edYd@GC zEfe(CN}~%XI?` zl)AVT(`#Xt*;40k_OrW@D=*}KiX6}3MWIc$0aPk zybD`RCML8fxnAh-5SRtfz`YQorB{~wfd>Q`)Jbk6qvT&V(}QVfGU6T=87^ z2lV4Any_HnYRrZoPeb9#bkJGx2%ZmNMTRbKYVPWi3SQ6>mZe-CwVD+omAD685h}E? zSejgl-bm$&tgFqDqQOV*K;K+W{w-jZy|{{b419AsU%fY*}mGZM~tf++3v7_9js0iwI3rGMJau|5&@;>g3sKEE~79qH7 zbZ488k2py8pQ(@VgFr0rr)b_|!yvj}Q#^-k0R!F2m+~)0pND^ZZp8yXuUz%5(a@sP zYeDax@5g~l#{*^dNxy!PAVa`Jl=vJ$B-B!%GW}6bkNmfcQYnu=pAIiBb!cPA%-OFr zsv6ab^-%}ZV~NHn!7dwIMsi#0Xf|O3QeoeM+h^5YRA{aGP3uDWKj=#E#uc09((;yV zChiyffN#$0X54-0PL{qERXsaN?)uGQ(d6EEwd7J-#9mB;D zl;_f@{IT7ORWGy`A#_4XXpWrDCT8GH!40t}$O8WF25d0>8fj853pmgv|L_(6 z`bcffVPimEv;+^0SfhYv*1Y4zD(iifZnq-7!A)-!D( z6}z)fBX|mn;w)Xg7L5CoqXq^wwDDU1uyS;c@K+%WNw{VO3w^~4AeCZ3Q>=tUA*Yxn z4&2J;w3@o6wCuVeDjkzq>YfCz2i!2lT!VMIs-u$g-6Fb?t(*j79mP`C`DKzE#Aq~5 zh!HW<9Z)-9GNRczFxR z-a(IHCbQQ)GFLrg*&jG${Mj+DX9U3@S+F)k6e!8I31vu&1&olkUl{_Q3z)ey>XcB8K-$ zEv-dSBd4-3i=?8xn}S>;qY*7nx9>)BbIjFj(~91yudV~Agf6*j#;`CznQl=Zc-zgL zW&JCPqO}}VUE;t}g|EOj@KS_ZQKhERbG+`Q?O*x##@~PbwBr=2F|3*(i$>ahMn&$x7KaIGFGi+P8HTB_y!kU4K|9J9?}SUF#T)@9?7WfE z!jA@oV~%@*r*rInoBPk-O*#D$ksc&Ja*U$B&DH;IZ)PC;cyfqR`upC&-CtpJR3P=` z?{hhSc|qX1M(d7?KijppfQMziP2j`)F0fRtZUUSl$o}{|nW3{pNCz3n-B5A_{r5gu zbpNWttOmIAa=`*3LQW*{Ii>#TLokMIVA1ehMbGzs!D=x_#nz=I+afzq-RR6K+qqZ8 z6vV8dvk-O@v)hB&u8Y+vO^xM#h)|(D5P@EbU&)}Ztl4&gmYejk{l+ctNXbVB0e-d* z3LNsouaZYkdhgK*PlbK#mU0%P!bX(0e@1Zeiz<@`RwpZ_% z32WBlBNF7vdcN3R?R+^#l=`N+OLHVx9#upQ+DTs|!TmTQnxEB(J=)fo2MRaaf#d4> ze=H?y8*0WVguIJ9?)(yEiAUw=Qf0c)sBV zz7cQ~noc$ZWI!h;61Vb3Y7cP!F9-Sj1PBw&9U7s%T)!%Yk^KzN3v=vt8=tBRjq}${ zRZ0}Fi!3M+di*f5BFkMbk+=e^BlUFtYA3y`CFqK`@W*G3h51g72#cslG0;yk z-P6-^Q>#_qDG?QUr6=Vnmg>lu{EG^_@T&wpv?Y*uyr-L1x>f)cE<~1{)Va>1=f#ji z+4JpB_V-+U02nT)+dtof3_}$fz3!V9r=Tb280xWB**m8rpXxCC_u%)tHz!-VnzxWVDNw<^J4eO9%L(7c3*xp6ueq0T-dFV!L2Ppg zxLRcbFFYIP0qJ|ywbwYt4@A*Gi8=Tzm8yFaLfl}S ze5$h|vGuQ=7)AHvbY;Auvg-!sVguewAMVHyW7G|25CwQKjpNP92QO2lxiKc4M=Uw% zr3xMzUf$jjhc#Q^#B}BN!h?6BKTDKHYxhaKY=ilH$dDkC{Vllb$_cVlmE3oJpoY^1 z)6sJlGLo%C!&TzKJbDP)B#G|!TP_;Wv* zF#U_mI;t5YUHC5`9ssaM_r4!5WE7n&RA32hE;1xYiL7w1vr4rO3grjiT7&hvGXtJO z--2J1bB&cH>3Kn<=ol5Dl4`nr&$p3y~&;ZZHeBvix7AuwNd>Y5=j=g1hfXD7icX^W`PvYN*k~}9Njp0BUMv3KDH!Y^?Wt#VdPKa zxt@VRmPU%rUQkBMFV(Ib-V>zM?YEEaJN(?eMJ0Jp z1=6jJ?FT&xNTSjR&tJ@z~evG zX+d#A!k8n(IkKM&p{nYJ1A+;9cnW2wC>CAS5;=N1s!#@aNlKAj%4jKS#$7w*yf^F@ zHPo;ttUwvJtpRY7ay*)OY2DQ6;XLz7aSumEiOuC731ClCz^p`>GfNQ1`$h_=0^%^T z49Bsd4HadC^ENn5(U}_*ngWK0;_?*qIMHt_#g37J9%yux*Rwc281me!E9mCxd>+_N z0gA3TnC7LL$cUDbOR|2*@KdYCna}FtCBzhh z_vLz7yg1~slXdkR42*zc+7AtdyESpV#%xIb6|N3Odj4-2s4OJ(# zSx~I>==yZ2BjEbal%~whQDc1jN}=I&&zC0Pn!f1sCElk;vnn!1iO+ZG3@AvG*QC1sAoLA50)aauohG2;t*l(QqeLc~x zGJIB>SpVOTU?=PxD>r3|Ey-)}wyrw(HyMaNkx8fd&WL*J-4c~AJ)DTdR|*jst-QwC z46Qn@S!I>85{z_oHcz=q=c0ZH<K9|?+jfWoo#!a|()+#_&>08sNs5ZR>w8;!}* zWzF#bhMC^y+C2K}D_iHU5Ul!?2u3C^kB3+ZL}Ga0L1)2)5{<|({A&NXmUy&!Tw*`4 z-*_4KUu5J(M%wzmc$}gIWu31H&%j?C`!J$klVikwG_CpjkdUubEYZm96@Bh97m0XN zf@cxU$AE_hCo3Cf{uMeeJX%k6de&st)kz0N`@g%fRkg8S+^nzn`v(VJpB+wgRc~MP z`IuEcM3khoofLpydT|X!7ViJ(S$++nwE_EEY?wGW%Y8dfI?&zM7lBvlGZgmB9lsZT9<&-NJ@HFuN>i-l;UfrP ziiZ>(-83&u30^PM7(%E%qmJ^OlEm@Lu0L%&93yh%58;L}fxju%$w@s?AA@M za$SMK;f1+5L$atjee`s5;C%CnxFpc-;%1XI;wy6xE;y*gYLuP^HPmzD7nU@NmX$g3 z@fo=!I)j>liYO=2#MgtVlzNE23p{ zkiMf4YEF2`2;>Elv*oL_pXc%9a}MWm{ks1qGC5!L`R(}u+ju}zKHRU_c7GsK#Q^{u zDn+8lB?_X(vSPEN@4U34CjoIPpxVwB7*OJ7Dk3a(?^A4@BwUD~mMJBYtp*tP@&Y>t zsi#ld0o2KeE}(!;Y_^ zo5mU~KUxnDM`B_qF%@+=I^TD_!J@WT_T1LS<1TsS^>UV4%1~|)?}BmTnrA>Bw}yHV z%g<-0F~YQ_8x1!ioIq!V+Sk-c#@&AdrRL(h8^h`{nXXgz<`U0NU+Wa*kaqT&zqsrw zr-O~%OH9uFm~Z{e>a8wJ+=u9OXxt~rJFLO)?n&0-qEO|OhNu7a=j7&|Pp%o(T6+X} znZ9J?l=eReqlUe>?_wj8M(w6r`LsE5;f1Bps1^2^aHBF6^vt3E{Ry#_QB96c43Syf3L5lZ&!8(-E3brrhqgI<%Q zPj%P~C=}sfIJ15HKes_!&HVl;(Ba^Bc;P&RXvF^(#9}Q(!q66UVEJkaBL*Bo8Eh%r z+q^qxfL1bRjE2T4uthQ?yG%UMWS0RVHLq;P^<(VQ>>&ZO)FXld*>6wTwUlG? zJd^dNM?<-I#yus6QGydY9Dcf^44;W-Lg^WwiVu}BDVW6VS9S%<5QSQwy}2mvYhAg7 zq)lPpPMKpqKhOU$WKUy$kx^b7Dtw4D?A=)`r}BSel(3+~iBh8iSsO+G7x$<7p*dgg z(#W2|*Qxxt(3as629$zH*8F@T7RLqJyzF!~vfaVk;~=`MFz{WAqpT!Vcqk;0f)16C zIR|prQ7jnEPFs>SnH?zN)UrKG)|5WqY19I{1YVp}y*UJ2jUSIs^HvqYBr5 zgKkfc?908yr%QSK;u5yc)}DAw$((wZ#T{Sn6M>=|s1_%*?O{O=iawg&k^Glkt^Y^{ z2Lw}^pGpN@Ft9|_Yto{A3wmeClM#GP+3AB|Qod@N%zjU7VloVGaebO0=QN@Sg@II6 zij@esUY3X-&-zA47&$(!{a=<5RIuIV(4-$$vwiW2!^IWwv@X9MtkOmTSb`tT;Vk!4 zwmu5|C8vuYcyOB2v!WH-56QR03E_&HnrmKW@6P#kG|#NBnI@nO(=-}~K}?`jktskK z8ch}f9m_S$y&3_9CJ>{n8U++B*hKq7u-V6RjkXgwWNqg)kJ&{IMFg#oF9EId2cv(_ z?5zG078Rk<(_Uk2Q5sl#wt>!j3d*C#vnPKphVrlHWG<_^E~-*K1BgGH>58)Jpy0s3 zKyfy}j*dck6(Lj*jEafO2?vY87o zxwI>IST*Ld#PhJ;V3r9A^oaTlEC~Fk1)Xu}B_q|-4s|$!B|;1sb^b(NpP&1Zd|SAG zpMiNp^W?H8xm*)S?Of!?w8YOd;bA#p*~Avk%BkP&3$iN7(3A8vj2rehLs$bq*F025Ol-bb#rcZ z9WjCT3N(~jJPO?A)BS*q)Oao^WMKj)QG8gf2EC~98D|_vHk1)d!xFfJ_35wFWi#lg z9flK3pQduFh!Bv2=?Lbmp19BpVo@)QQ<{aWBYsrtGlB|)N0rB!#@X)P(KmfzBUU11 zV)#bK{dY>=v-F@I`@Sy72H4}pE}Z_Su%_B*!9zS4Geh$#N&Qf+t=YhWlv04HSt@oT z=nx$KdXdL;m@kXtyq}U6*a;hqUEX7@!aL|sFK_Pc>n?d;F644t;vfFunaFH?{X!>+ z7c*N0psW_^$7(Y%ZZ0rR%S6sb!}1g${yuh{>(K`+<3G&v8&1&xUGz?udtaJ-+Wpbf zlWJI&*V)M{7Jq@b#H3E^TECKNRHh-PEoZp~1kE39+4Ra9k&4H=a{!=EPUBb6_UjDO z^lyLC{O-JgSB=L*9=sam;T$2rt$g=d0HQDd>Kth<>*vEer{Oa{zRK@Dc}Uv zec_%t)p#M78|YUigN2X8zZkPQm)y4D*ED*ue@9b{<7)lRvO9s;0BfzwY(11V_v&{}k(G-g9I(1J z@KuEwO4gpP?C#wF`svHr*)hMqQ9{A~FJ2}(5+^2k)HKf70}Pz7q|i-qPlcs5CjBy% zF}M6`%=!-|@nU$x_ArT$$t+;QJ}*nN&U7y0jpjvgXi0wh%l~ra+lv>jNgqSXm5;mNZFaq+N;x!Q@_t%fF-r4*qUs&4r@L)?p$;X}a8(fn&_TIU3ee}G%^YUAE zzS!`KKzf)!0AB4eG=1|TNL(nTTqTAJ6iP@qSlwRgF+A9dO8MQ=o3O^eesX>wkUmk2 zFMD_}4hz*&mu}F0*31|DQGioM?Y3Z3hR6 zTZs||R&SxQGENrRu@lM!uO$kY`ZP%@%WhI@)U|vf256^S#~wsFXnWC-3KL zrT_kD6UtV_J1H=ZP0caYyg4;jT#t*Ed;pP9cybH_F!w`<eEKw1$iC~{T1=GvdLHt-FUGb08Rs1{Yp}X258YFmg?HT*6tFMqK96ao0EJp{P9#I zGIC8-fSfY#+h)0opPQu2v%Qhz^vCYr9aQpPrMj={mX1Fe+mP$9JP+ZyXL^pNx+SLo zKbzc_6b(dEBk&JHv4o*YvYFo5@c!>K&a-=#5DL;y-%c;}qs#1RztOS+t;Z3C`%1LX zq8dBQQ7<#|{9$7wHBv82EIYQTW;=4X?|dTocx2kaqj@2#$SLyQ-px%Uhbgc#^ZD=u zSQ^0G4O`d7=wrJhupnf7D>1b8n7}^bjcQhJX^w}4$A}}LuR!aZlMr|CKLh)+;E3eq zZq4MCwTIBqPP3xbc5xihYT-SLyR*Kj&YLt;b+2kC^sViA?>aB}?fctS*t6P-Kb|(w zwGwfw{aWs6cR9b37xU*I;KR+6!mHjHeBx>bB;mXKj~nLqEteKLg4&n%FHo1+K9$b8 zs?k~o>z`kZDEw97rI&h82re7F3b(nSfe@$vXU7^}0Fe{TC=I7Y($~RQ$5}Hk{@g%&BzUY{BVz-$Ik)~#jsZ>whBbQwHM)F7Vt4mh-FGHS38~)qJKMtlCU~)oa4z6z4?e7U$_9Mx3ZIt! zg7!By%$Us9mA0{5iTBEeq#5LH_semG2&yKX{iVyX7uxucmqI{BlTRk`POQw>WMez0 zm6w)Ll%N2)b4@ER*#BI{1Zndq>BXZC-KU1*yOrrO{C94E6qoC_3Z6RV+$y=em!g^v z1{m1g3trL^6KE8GuR_#z0^5D8 z39vnaxP?H^2)XMoPjy541$$3kZ$@g(hU^^lxn$XxJSnh0+69Ed_LUnili-6hKgd9v zJ^M31_v2N!Mnj_IldGinblsQNd&k?efikq5AaP7&q3HN}Q?mV0nq~B`*gG!j%<4pu zz@j!prI24sU)Vy6%>-0H|uW>c%B(sP!brX2@sff zm=cBbjQ|^|c3+?GOA&=! zZaRsYy1ZnGC{B5-i?&S$l78<=6`7L+0#X zo5uaDG&1YFefx}0Ch%{@;g)D)+{u=+5Q|)0Z&;4`s!v^S+SSd$BY;r0uGUz=veuRU zPa=lL^e!iCgkX0pG($Sx=WR$dgkYjRIX1_veGul5?}8mll$ zPGWD<*A?W|K&M#`5uf@1g(wSs)5jA**W1&6{LwSwwd6|I>+L{{-JgddaOvJ}uLW5! z=ZQ4nz-=P;JOaA^ly`4hOlt>dbya<-#m92yB!X;a&2fm424|y~tFeDiKBzXGs`gwAzcj0FELTHe;IxY=I+ziXh= z_i+7W_o>R3d*fow%kASfzLD6!Ko;`v@rNa<9>JQcK%bpBeugq$iW0uFCy(e$iB(DG zk_ZSmLe=4OED{cllEXG0Oqh?Y?L;Asjnm&r{>1XIZL#aNt-(t0eOwLMB=q<^3$9q- zvL+2CNP&oDO2`I+PXD)03%0>&O)nv#Z3ZB_<&^h zW){rOWoOMcmDi96l7+Yvu0|Q=ZST5j)FLkYv~q#l#y<5^lH9Dh+d~;3QT*Dl`b4a(HcEGp|CLa`UJXu)>(DnV^ZJcp+PS5`c<$nC z+*Gn^AUQ&YKTk8iQ6xVtz==FnSISq0I)fr<^yVZ%f4s}pb}@a;ncu?JS18|!P!L2o zifyK9Y>xhMa)&z9D)xQCsQ6v*=gpP2hkK_3kO!gUS>AUzzzlt)%9$EdBBjM&0n^9u zXV$Pbry35KAEu^)Mq)eqqV*jM3%CE~T{{5|3_yla!N%PODP!?Jbfb}>umU;_={{O8 z$lxWcbu;qQo!=R~4~lqMXV?^kip`czeJPr)GR^^1uXyZoiu>Q8fMbc#^|!BXj)8~M z4L7XZUcNbbiZ@p{W=*U-MX1g92wBon^JhEeaI6jdYceLfq$Dt5u{?v!*&^20ulZ&* zc_L5B&nb}?K^D=u)YWdt=#4Sa1-d$40&JAz=0}zl_{K8BCslvy9#a{OAm!xm4@^pO zI==pDYOcf>P-hEs?_Y0y!|gmMAo~0rzS1LCpXqKo0{W1SoHvGq#2iD5b2Hp$CQ;-Z z(8(cBJ{Vbd5pjAgGQ*YD3Z6<6a0>HuXK-?fG{069%{6JD%P_HsCQKNVAm44!uMcp54S$aDooW4r zpiQKDy!XKW6I*)~{%f*mf%ph>P{Pm8p?bORl*&d%6UHnccXzelIvLVBOpA&TAqB(x>erSr@ppGIs1ZUkL$09rB z8;lDg)&OxTcPyDXPmBYg1t^J^wTJe%4S8TbfUMl?jh{vqE~v3Vi?78yz zJSkJ>376Bzo-fd;`rXdke~~@^?7mBE2w{q#MvPpqy#;gbAD7|tqo!@40r_ ziInIt)(ta)Pk`L-T(A&A)i&mzs+o{V#zOZu(=hlfe8d0R<;_&T*4Y00qmg#vg&yAYO)k zdgk>qrv4MoJS58MrqDJT8*`v?QVc2(hZY^d_?! zob>12o9fXd7FHtJZI#vcel(x{t9HnfY7-KS_0tlKmF~r2`wj8Y(|DJxehFd!XLM2h zX!`UJdHpCJI8i_3-bSU=SfII0zdDU5eG-ZeHA+c_(nrwIfeqwU2+&-XVSMZ&WB2|z zC_ehm5l~ix>XmAKW$7-B@#R!L#QrAA*-s+L0hn3&;{gbo+@ZA->)P~S2lrrei6g&I z$}gu++}CBgIWqQ>qPtauny)PSOG4ZlGOq;4=qO*3b23dk1jZo};YxpgEUf$)O-!e4?Bue#P7}hR7v~)B(E5d!cg+7PdNUU=ia>1^LaE5=EDehsSi$Ewj(eYQr ziJQgP-{PcPHC3s4cE$u&fyzalGjQ{a>qcob7=t ztOw`4RbHu@b>`13wwyz>VRvQ9CS5pTG=A-Vea!RKW2927F=DcZB>!IX2qDXc&B>iq zpPLr~Xw5ru$wR;W=~Ok#q|Qu_C1ivO3ka=2qkP>PfmDkl5WSFu_Xr%azXiT9l770s z7k7H+_M*A{>6wF<@A2yST=OXTo{5hW&68`3C!Uvu`#1Y*7ekG<^D&1&sb|mkrJeov zkKC`dW}70u(Xem)1KQj2<^~erEGwvAK(-a2Z;oH?jHfPM!ri@_n`We`hBJh}BKON| zRknYwj=k%vHWdEU+SGaMAN(==31Z)z0GeE=l0OMrn~FQMgUV1hlq5RTOyzheaD1zPlG6T;p87gDqKtv9v!%9on0MVI0r2BYe~G=p`xHgX){cW z_J?S*445jZbUrhB4-n9nk6jw)&N`p>nm8-T09(6uul*3tvjVjB^P!<240I8&y4qlN%h;nyA$qqgIKE1F@3 z79{u*Z88fDE8)csUBXc)yOFexR>|6)Bt>gd(49Si^DVgkdr@6K znVtM5g0QF#hk41M_~|Z6n_fjn1%`mJ*yyq{V5NKE;qPaUCC}un`5^eDoaw^g`z!55 zbwO|9Pm#}m!7F!9%9kyp<^vi`4|FkSTkqT)tjZ0+skDJ)5?iajCSr{*H6G_C74?4X zl2oA4gIe!tfwXg9532q60ND(k8hw16Q1>rMSzQkV4HKEYmM@s>y*fwhs48m1rz?i5 zLC{hL9a^{$&_7bEUv;>PA-penb9JbBQ+s<6{Hpe)mx-pcf1q80ZSBncID#*)vOyjL zxw}N{NVD9Xt@M#-tgrm{>`?U>V^#38~hqB5!lk#ZDx~B_E zN{4yi{~m2kQ4ty_y`-tEHdjSFNi17_M<%3#ozkZng~fruT>S}pLjQ?HB%%|vuzDm( z1#m}7a|*dTGFdx!bFCfiva$_F#UBkDwf2qVvt2uV5B|9xga)aq{DN**ZC_kn1^Dj5Gy+wD%WIxwzz+~cv)tDpo)PE`0rD0 z0~)zW;`}DIqDOFQ>xc1ItKzXfT+UzAn(=G_2`Cmp0M3?*j~}9UPg%t%!&X5)A`s8;OBi3`rpPrK=U|fpb>ZTq^(7jN>^)a}ca`x~vg5`+a3eg=IYc1TnX zC(BIylro-5!a^DG>6P?@58uekV~eIFtq{NIkQ3p|!5ujrGWKwg`P|k=2%$#|*KE%I z+>m>n&es$e$h~aH0H2!k{F5eO-_?d2jQUMRCZB5zmdZx5xdc(9tYl0t%tB9+q`4w~ z5-3Auc>hVPNfj2;T}M?Saaw^;kmOK8v+h?_+V}}zjsYF@ ze^%ys9c}PDq|czeq)_m|;=wC4cl!7JQ;9EYp9XO3$O6^XVZWEtfcbl zydBI@;EyG=&-2^F&J*ECm|woql9~{p#A~B#c%xy`gT?D#NU^=-bAnlOFS?b#au#jX zEiEnn3^)X81p+Qjs0u5$q5YVp2Q~@!$u~Ej)sx6OF1Rx(!auMb4CJ3sg19+t^SBAj zRVq|a)P>fs`)x|cgT&1b!dJ%13~MPdv~nQ$!;Op_%8SyI%aB@a&4X(Mm_&RBewLEx)8Q36UE z6@{b|M0}MR6S3=tD?`IV>o~4&_HN#EQDVa2E3`i_F*C?}N_SP-1xXV~3&lqNF1DQ? zWF#l5yFQ&vCIl(df^-y|<~WPUdQ+5Q2}HRQX(KI^u?4>gLY$g(d1z3!n!i$f0SJI_~Uv73>T zEJkgX5K>zEnpy=imNF!oK|OQn1IP0XHtWnnx6$a%W?s_$#)07z3KSu&Vl>rJmW?h# zHE*}Xds87gbO=li=NQ-#VuWGqxYbo61@6ZcIwLAezKZ|@ZpwBv<@2}}LHWTMki{Yc zvjL4E9NkwwNNLsVPh1beY8UK#%icK?M*|4Lms;f z$iV8F=|SdpxQz3JU^$~i(PLw>@R(f6#dyUjs)K9Ky@C~^5 zPPOT|6IT<0(lTVV8v?{}P{q`e!|3;JgQV`@q9Ro_!23$k>xX_8bg^1{dr*t(tG)T> z6MXn2xDg){FYPDh=}eJVr=+Jrs4}Rbu^baqwRCkcq5JJRXo)>uQTr$`{!mBb=37h( z5X#U1(8xv^UOx{j+^Bj0r$y%%LtU4wSnRr+YMK*3=|!c~`;C0PJ8F20IP^iIS$4sg zYLh}@Dq5WbBo0}U!Vpd+>6od8!C4b=BKlHb4zkp|g>rnUG1sB+ea}N9DV9Cv z(Bv?Ga@uuS4oT)8#>@&!g*B1XVX?PInzvTC)|Qq*woYLC`C&es_D>tJwoDuzczSqs zKDz@_f^A@VrxckL-GYLok#mClfCZLr9k&fUhsH~7nbL(DISD{++zt7`ikt{p?bOyU zmNt4=L`XD6L?=B*ELM?k#JvOB_8;3`VcnvLv2QW9Hc)bICsBZNHYjfPYoa^d%GWUF!fTVrkyQABG zAzWVfRfOYv?2?ixs$*0;4G)_14{-&i>8(vY;%R*tAW}K(ZEN3TIT8fJFJ?DBy~1d7 zh`ws$CyPm(SON9Ipms3~P^sZ4oSBt}I5pD*K^9gs!f1p+Y`L2tNK-wsaaUXu*UTH) zeL|G?tj@&bVKw;9r5bg6vH5VU-*dfeWAbqKv=t1XAx*+8;gY%-|sx z{x2;t3xS4KP|#_$^xZ`8jpoxXAYI5rnpylcO+?-lr0jcc?!NfDKmQs6lovVIPkqOU z%MaS)0>_tior6t`jRn*{@Tb03LS>5$EhS*^kt3)r_tNFQ`V2WvC_ zdB6dE9WbseAFqrESuoPl#9DBV#(io<8fd;+TnQ|$Yp5w7RBGh`abCKnbN}V;0!dVB(mo=pdwuf z%h2ceA?8^HvkLv?%#T+J->X=*PdGcxs7qSLRZN;~$Ww6=amt0jD-aIc25sw>af(r1 zLkPzB@D?;wzYc}kXUZ(zcx!?{cy)9ggonyX7us#Q{BQs5t-ljT0O>8C$c$yuNfUO7qUMRrziVe4F@0cu z5~i%04_KMf3UrxPkkK;J!`xVTN+8^k(6Ju>?tAovP0DYG?GF7LJ;D3w4xKk7YQ) zdg9C|&Nh$K_WHi~)+xJVz3YlJyXr)d^OGxrQk-=xJd8 zJ&yTEWpu>kOQBdN54^X$r9WihDy}Vtz%%z72SoKQeQ>TEfRuFET#riB?Vn?xNdBVC z{|rZ?YvGx53YF#YjMWcgJ*&or!KEq5)!b+5rlf%*{umxN{wr;Qk?W()s!0BOgPlN1 ztUN&%;&Y*aet&^+9lw0*#9w!P@@gT)qMs@`W(JJth8aqs%1DypYc?rXdPvN_ATlj) zZWZ(pVE4}xbTcL$bT=XWxFtPA1^68v|3*pIEF1(s)IYtlc?5VC)hPe_>-J2Y)HSKb z1>DM5045P|yG$959nwet$a}mwIx3_Y)Osz&uRgiR?^@zHX?Re>mHYavm&(V zmT{la!8eE{g?YK=)aKOMfZIa7=tY344Q&9G1yun3V25G3DvMoAO<`|>Q}ER-&_m(j zc&B2iuh3kppXNn40xoIDnoA`>E8#;=0h2&ra`#{>LWa1~n#7dOHOCymPYIUd*E0qV zxi7Es&>=K1p>fCMMs4_Dz&q01AErA@f*<+$XZ`kY^Pw4aZ2OHQ#3xYzOtL6es*;0| zWCn}GQa6A2m{33vBmJj{?^by*KJJw&S0Ap5Erq;`F;6g-{ zl8}Gr<>fCrWdulWlK{eBD(L=t^J(~L>2kg2&gaSbFyZNXo<8UzmIp|5c0U{+>;j-d z2|qQMf_chUUT<7d)z4TCPMqIv&%T4R>Rt@F-S5z!SnGuwV9`xu=FqnMP6kIp`2H2qM>5FDkVhj)wBjtL*@yvj0;U;c%rRQGTv<+v zUqpxK-ExUJl~xfi1#zCo!9_MPoaSDH&_p`EWN0AMPLWOA5MF-J{1AhNroJ}%9tnan3z%X4y*y~q#jM>>HuF0NU zj0!eI-&j_*cQK{G@5a%KN=s8H-+2#2h9?8>g4k{@S1#8T<0p?|8U#Qg?dn1SJ&;(+ zO~9)FV6|l-QhQR9nS_6~lmCKHDIDQRQLqVDa0`YXoh2vJi*(HCsE#YmrDZH? zBrnEHRNyDqRY>M|4$HTw2S+>e(dbC^qRuj z@U~Y6!lH*Dh{jawErTvBi4gd@sqiD+KaXd=93h=Aj+5FJ4v_HPAX;xYN~k>vR0*Zh zvP(S$0zeY)FSl-XfPS+yHi*wy_Z=e6UF5UVYDH?2EF1cQhJ+p7H^!$uDof}8Vn-Sk zxjeKVdiNeqB>2ABMVqkZ|5--E;c7`*9SDyLFTIw@z*V&p5$v^ovU)DKXI1F@(Tj~C zH#ri9%!IM#N>Ve(J7#h5zb+zTWOci^+&fPY1{MWt%{>ntPaK87@dC&Oys6*+zDy}i z7j%(hD(uz9y$|TR_B-f<0XNs}0nj*n>c5zMCsNJCU{Owv&u+u-d$v6~^bIITPD}v0 zC6mS9B)}3+;N)tT!*%aSjBjLMJYF4p#7bb%odW=N(d<{G#+%Qj(5cdW+Wz@3nel6< z{7IK|IB*<~gH{)>^jYm&eZXY514eZs_*CClR%&18zNV$b;~KM-YQD95)ERpDN{K$k%77E9mO-jy^cx=62ryu5!g#z$GlWq$Fnfi zk?a_Qyofji|JG6?e&S0E8s>^xTL-h6wF1=H8}wJp_o zxDbm?t5NN@^2t%P&8w%K&HEfR&%|HXtDBGMyn#1BBj6#b$05JH{_*-Q2p^#XIN~fn zPm-r`Ro`njQG-04gndtHn-l1zhq9DbMKeTYfx3rx2V%bN;Q08sz3z>aYJRr3CI6fR zkxZ_~UZ7uMKs5b=e;i}68g6b%yKzIV%~T-JDnG%9RYD+IxF<@TE7Yiz0gWZSI89eP z-Eb0lY@dOhl>I>r0Y(_k0ouh!Sr0o5S29W~5R~49=}9C(vES-5>%A)=rx* z<#9GU{crAopLjL+e(Rvs1;A08%{X$8%`amH=)MdCv!NBs;iEO=YP?Gq&mSe%6}-Y` zPx%r#?X|viMFm%ZhF2|o>%J(z()eussla=*uY88 zoe^R&=i>i{7)cE$=+pg;^srE*OfpHW3hsJk$BXo;VfX*Tg^EQ+QoEmc=4;RZG~b4I~t22jH3vwkSKLs z%#lU_E>X<<&lHmrOg}l?c?beoE~?`8PhYH`ep!8adAH?u()H94d<~FeCIARu7I+Fq0$V%j`ei38J@jRv+>s0lj zQB1Et_mKK?yFXsiSc+c`QU*%WsQ;qNA1v$9*(d6fX3!}y+1s?6U&tQ|p}%1jK&lK3 zQL&2GK2;1H#zF5c)2wsUWUa8Y$dyBR8+@nV2D-X25N;S^9!O^WnwePjF_rLxBX;lP zGa@tNA4*>Bt`S{r_5i1gV6QLj?Zpem?eBIkXXuaLcCKCo1F{OCfu~bI^YHjcH`=3%jRk!3?GP_mROQQcywr*!;0lrB<*Z;Xedtdg222tYT zgn{pYmygE_J)mJNVD(jo=d{yU-lVIu<8=(Gr%l{*Yj98j;c~VlF4oDtSZ(n!vcv7q zRUUhFrKgBD0PAi;GL}L~N*iA242{%9If@hA*qR@Akrc}2H1?%gd_+llL(eqVGo5c;r2d$+FYlP z>`gH7eQ}#dU8Ls-o_N<<{fiGx$JsP+iRS#V|uYuEQCLK9o0D4CJjq zHi|mw3g)r`-d)ho++pW&UE853EurP&F%fX?+gJmJwszVE%+c(&D#p+y(GlUhFL)daj@RB; z-y~%JH`CPg8|8O;S;mp?!UH^b1Gw$S)%ul7ErTmJ{G6kRs}FGVxCUh92TVZF#2_1; zYrP`iwa3gVw+|?LFVmBBlZOT-oyW%7-{uvV23_2~AD&EZwi;_RMtGT|9{A^(?ASL0 zB=9z(B#%z2fRq1XA10Qjc&&aG7ypQ`iKR|G=550)hgN^HA&96sp=hR`en)2RwN zWE^sZ44~Xso{^N2UpevRQgD@>N6`|MwmZ8BL75r&+pZ$p?Lxv;3Z+gR=6}n)?&R)%fSTcpr>^r;FcXb$a=TPODG!nO=HD~AOAxFu*FdB=S77zQ)tU0v*EZy zD>dB`>Iblrn3~0du_Hh*0DP4ydql|;Zv=qRf1hOgk}?9%O)c>pK7L5Q@m?B}I~D&; zHfocPNLjD>_2KgJ?QxLD-LZ)@Eq*eTJKSw_BHiZj^TJcmjoX<_Ko5KK9GxuibUOd?r-Jg-3L@K1N><_;2)pQo0W14-Rf<=bX~t>a1PXdH4Wz1`U>8L$EM>@_ad+c(2(7utIH*bx|NMlTq``2$khJ_|XNy%Qu zSkC;N@S(v$K3hOsn=#@GNA-6qi@#lo1KSTvMD?bf*E~5nN;64ISd5+0X~V$MHuvLMm|}Y?w71mDzY?;o%{O`M9k=UHYRpS z(T>XQf4pvf5-YPZ)^?fv$n~BFx5dI?YHG<&ML9)ni+7;mz$mGav+HVuhpz!ToPfnJQ0Z1M2R{D-&l$xE z5q67ZFYnBQ*A(dE=hnMB{4RbCK6K=iyC+QY6d1pCSx`sp+|+}K-}H~iFo%T6gojjy zXC_sFa1AqFF@j|=)k2bszGizdrNUz@apVDAcHKcFfcEi`Xp#RL_H9z|-Rn11_CAN_ zvgQulJUqhH{vGQ=2rijPY6gly%;(CCWxW3Se8qh5pN_Kv5#};8WR`KO7dt>h_y3A2 zz}Tp_XdQ1O%tLtU{ZISli(h=oBbe5B9K}vRK6N)&9*CTs_m--2QU+DvfR6^t9 zOlpPn^%FYj?}|QdF(fDK7lb6_F3?q^fBKXCN0UyKmjObm!1{wpJN@r?dWC3mrOgX{ z1X`k9&<%KyIn*M|&<_u4EtZyb6yZf?1yT-66lfA)B)(t=7o(I^Eb$+BX>Tv5>CnQL zuXz=T(bC91mdmd?Pqqqo+!a?jA(ET5mH>H^yEQ;|;brKb0xyR{dvbgo9qwe4%9wRaY84_(k2B>AJSs-xC zVTw#n1JK&Z)tuK5l|F?-4@HuM{oJE#@KV$dkK-8ZgLbJ+=kGtVKyoOq(P^Nv@BrAN z0HDW8S^v*z`c-v9W5?D-fMYt}?(U6hi#~s;R-DYtUT8U3BV81RTbcDa90?|gMrR9U z8(>yu0W>5M{nvNSovZZTS5B)J83_d|hVKtsc)qs1vtwGAdEF8)5?;m_GU zr;qHA+h;G*Bv#IgL4E&dd^FFdV2oxAYcAMfv@>KA~GwergsRwNN* z@`g+1e=%JHFEXi3djd|KsaTSrC!t50PKuffzcm}{h<5+Ql5?r^A_T3dSc*tqSmsW6 zMjA44JWITS@+AB=582%(HJZFYqyhZpBjj3%dF`0L-*$#GOkB&B`gaU-j=ZDmv^Jr+RuxvZE)hU&9tTrjeX4u=F7y9P z_a9M8k-y6{bc*`si}YCCh2%r3_*ys)nh-jA5Vt>f-)N$6fPyoQxy0dWQSi-?k95%a z1TTJpWQWM>^^~J>4izBD$6J`L9!HErofyaBFp}LVLd|P4syfX~>MI9M(vnS9vVbKq z)9CyoDS15xmh<^@dDP$#2(Mf?n7ApqSZ*;+C>QfMIJfuconW2DBNATpYTi82{=M7; zeEx0?b;zUo&0AN0*NG+alE}is*piD+R)Ltd-Xs(a6ho1e=~* z*2NjE5XE6d6N#3IDi62QGfwkD%+1@+*9aX1ymB>xCVRv`4_Iu%t-5a5WyTSs^L{u6 zBEbn)tt_L6^F8-BJap1Q zpN(q4GZh9@Ic^070k@pxjO9NEWA&FwKW_cGD>{f=b2z$tJgWBbb9BEt-UzyfN$&)tMuMtB_(i z#rU;f9SQRQ+$1BR>WSML=gFmZ=eX^dzJ{u=y3b9R{(P!w$w;`p(^p4`?{;7RiXC3m z2U=@y9P%3CJP)r>1`DssCR4o>=bU$=P}ev5FDW-(N{;y_ z_govVETNA6j%{m6a6)mV7uSF|de0qDcRMnS_$~jty3(AQ+JY+KDwsdr7CZqy^}&X& z;In(>y~Cj3C*Z*$@O-%5rvrcy%>yvPpr^adf?DAW@|$k)wROst8r1%;`x^TD<4=@D zfM@0!`nmsyd|dTB@Ng2@-nmrA&-t}Zb=P=z`0gHT~1o5z-;_~(wCyT&@bYJ|M~ z!HU>xoIYKObXmL*XxXEy2ylv6|5W!)lg>ok(()gEyFE(6!ZI9%0=FH+dv%>9lVOI$ zF*agnk=W7?MTfJq%E?xVaFi$jgw z)o>$695gqK{nzDLSZt#+;pRq<|KsU@s}q13CCdZ2L)!l_ir#yCl)B};{yIsa&|oO& zBwhN4#b2HEw)e__s#|RH5dh*A1fR{=H-9r}Y;+$hszD98r)@G4cln*`S)foHOy4Zy z*axVX3T1XAsj<}?xJ_uI5)11N$Vv}Z%I&qx54*}bv$G??jz9lz(79-7>*)zT-%;12 z7(p`=z1yS3j*ELL_=>ALylc9~G+ByhoU5~pZ9VIobY@914`O2W z1*5VSG1Q$WQTcmp(6jJR?6(q0!5?~3hxDbU>r9e|94PI1KlF}BxyO>ITG_z9hy$>= z*A4~w9f$vpSDIC~zu_+y1aD@JW!}1abv@I{*ejOs>U5yusUVK*EzM6DJ^?PJ63wA^ zRQ>AqI{VpVNc1u=G8%-kWE#iU&LWn{ngR1NJmW*xKfA2Tw8J6?DPYR*r~QPdKdadi z$kV8(s5lqV=r~wicV7Jbo{$UtIuUOJ{0yzhNCA&Hk}RkE^wV0)k@@D-3&BuesS|m6 zwzn0vwWpjLnb6>n?DL<4IC1ve_?(HUY0urSqXXf3V4WWDA5H)H^HU}Rb@>!R;l=tP z={4zP`NL^QdMKIwBByUw_n9#@6g=$08XY05RYx{S?cBT58V zgtX+k5iytzA%Ay{a-?r20IJEQerxL@8MqDOPo_3(Wu04S@A#-=_$~@H1jYPyM^r#3 zC3XMrK@(W{ZUIb_g5ZY<;Xr#&AK#}_9WIFIn<_L;;%vmk#jN$VlKK^g;LJEE704NN z-=3*&UO{Cbe^$u0Aht;!m9ql5$HCX^rjrEUPal7F1m8ymu9mLm9PgAn(7aPZ#vwh( zW69KBA;@)LxQs}IJhKI^QmKp#4!Ix*TshCtOp+Of1u^9jXoC1tmsiYMQFk3~Q7&Hj{>)dB```%Bf{y^tsI66zd zaCATo9|rR=@dw0%fZEe?QJ4UlBZLecWymZ_=M=;!EY6NrscKFYtFm7muhi2NfBB+2 zzK8aEBRB7NR+C%ny~cb$SZ-kE2djk>W73M+h>1;%7qm}}OJ;b|$yw1938E+(bSR0h zD@=w#9-UH&jR|>c%;|2cGaHG6TC6`#r90Rw)*=tJV6cHuMe@6w{m!n)txjDT__TTy z^aOa{&tGi@-5ev-40PGBWojUkm^3~QUaKTg!WFa3d~YLrZl|6EcWL2bd6P-#=zv$i zd-Ejn{04p59$LKAy4W#k{*P$(1~Ap^?{i?v!DMRvXexj!GlQI!)n_)A28H-gxA7Zb zyDSyx8~pT$L1t$;ZJfMNed)g_;Di*IeL*!iGG2^}w1SnVt0EX#;+f$D;1M;Dp(xD# zCf6QEZ{E!AFf&WyU%Coo-T`l)l$Cf3JleT0x|MqBGg?|W;M$nVLUU#^ z(y$o#)m`<-w#r!Zg3NTw5fISSHw|=a%hjj=!u!@VI;(4PoTij zT_7_l-k_ zXjcyEu568Hr1PJl14$CRq-=1SE4>JKP$o$m@tbcuMdd5F58xEu-}fgJNCjAZNZ-OL zB=|XTF|q#mN!7`U8DbR=MQ>!InsUn#vDYFd*5NBvLfTf^m+4k%-RDAKTA@Z-nBY8L z9Tn=x(^Cnl{7D^cjld3ZfrohcZwT^Aq^JvRh- zcA8$(sWlpMOSOkK1Iv`;vDsOj&8vcG7&E551%G^oNdfTI9RY8Fx)666@GY9-OQ>b4 z1T~}9H-?wV0(Se?+YRz0tSxrx0AO@K@VW_jo8e?$v~T=)8&=^^E9i>WbG`dP4^8kh zHR2~8eqpRe;Y72@M4s8d)3%aMV{vQ4$E+r?9R2R}K{Fd>OeVxAT=nhc4#w|XZDwp3 ze^e?f;b#}Z(|IR*xx;dM%Y_2z4QXQya&_~VQqIDU$8A_lRCJ)w_}i^A(4{|2{!pKo?{)-~(#=e6c$@YPmSfi&=C1wnVK zn~LvVqBEca`*svZ^MrtUzpGx7P-IIMqP%gg+d)nRUcR;Uu(h=%5NAW8Ce`%>viEV> zIE0J8GhuddPmusV;_UiWV)n&eZ{#J^s8;aig zSs(FQLB$$Vu$n`3F(F}?v6@Qg2v}tdIoF5i2o~cYLols6?X(w+)UeQSiWbB^?%GK# z@>v8cgc57*%*9)LO+|l62q?G9#=@lOrW&Hv)fmI|TkBnb%scQW?qeF{_Wu2>KoSyP z8;0}Fo)hv5L7F=4NYV_p?ddriFIKy`zW)1nk13OdaMr3VuHt;_uAn>NNRWS%woWDK zgg%&Cp*`qo|CR$=4-OL$*lZ!*TsF3)$m3YhX9ULbKy;Y)$H8J#aVO>dKYHc77B?G; zgC7xCY8j?Y1TtUzqQzRiZ^aQO)uSdcJ5&cP&JUYtSYfQCQ$d8pdulO*Hvv*oQtDev zL!W!vlpEUG-jnC7pYlmbnAF&5D2ND{4Zs3hpMD(y8{LP?#m)Quot?;^_*x3PV>GIY z{7l^NVGbuG4qHga17h)H)p@btAAqp`@H}@?v%;=DcWRTLf97`paGW@M=hxrC&#GD1$J2_FtMw)C_4RqLo&$P#CsK$#8$B~5vyeU0o*Zmi5|fjU1`$&C z0Fc>%fdrt*X>s*gu{6`O(s%t!HpU0TpdUx+@0=|WgAuFtg5;jb#4?z=7iD!>l+-i6 ztVxxzESbwR2v-zK2o|fA1&$Kh98#$~2?@th#|JV&2^D-A>S|OKm!CTP9#ESm73b3? zi~Z{dtF5N-^0M3IcE`?d11|e1h8%!&F(nW>1S2{3Yn3WmG4&tZQ#NCFoEpwW0~Lyc+!q_8Mw|DcMvI^z5+b z$i{>sk<2-LA0N|hzd(h6ha6UtYPkt&E?0c@MGC8axezjWOn8#=YXj;? z)!89dK`5UCgfY?4h7Ty>Wjh&y>P5w)3}OOp#R#;?N;%*$ZpXWr5Ck(H)HN_-S!uk6 zf(zk*J(roQFBCGN!imnIQAV-#^ON}h?Z?Nn{qk`W$Ddr_@0U*UutV7W>&#KdHm%NX zl4k1ra!M$2kSYQJ%=&@X)Z?K~@SXJI{@!Uxw*R-emgh5{)A!J=zeTP8JiaZrW5s9( za5tCF65@|*@~mWgAyPy$l6Na7SJ0uHoE*FC{QP`C=|N9NXFN2WLtS8q(+R7&*2I%J zGN-m@d{SVMO7 z(kZi!qDPR87FqsQ9dW2zh1F=Jab!no(DCo1lkV^rxBc{wX40SnD1$7@$JFF$ee?b* zr}_ZE6$M`(p8$WflLXH}@N-u&>7ZL*e_vQtRJo+*(R|P%w)4u8u=;9QU~5mi{cSg( zp>5hcxr!oO=72z7-4E82?8wx5}a zG!AP2?rwW|EpWbRb7+=))X^{P=njg2(a03C#hDF)a6UD^2Eh;l&%Q8#vQuu!rVhAF z&+N2_*N*%(aPY%8)M!!$E7f)6wdU=|j~kd)sKMyFMxV=;NY zdTxt7JVyHJdsxMuu_Is0w-}Su(8r3Wzw`zdE{_Mn@8^$PKBW;=w3^-v4GoRc1zlIv zZ>)6zaZ~+Z0QZrT!&A5MwmbB|RbGurUS4 za>0M>gYT+0I^XIHAS%MrBhY#tonqSFqO#W4)q46&sYx>DM-Z?U%luX)yEeBj(w^go&v%AcYCdE(^-Yb|2iXP*CtaaGZLay^=o8*>QRNXL@fq>r*a4n6=WPe3V*kfUw}@#o&cw z=&)eO9jthNG-2R?#8}uk&Can)f7HR9`L8l?HR$^9&y(Q$7t$}A+R=-V)!WdJ6mA3` z?!oj2id&`=4Dbxkrl|Av;X?n_fBgshTUk_OBVZZgXj+q+NKnT*p}lCHhUl*F{{_q! z`#bwIgy$_Ep1$(VDh}_WJ?;4A80MExumci1?SL2daf7v;<4DO+9aJSvOgLQ`A5-8W5WvufNiw}-T2-+7E2 zjDnrq=pFPdY<}Dq2Sx0^%0qs!qiJt{NC`gen0+{v-h8Sr$k$kp(zKksdz@eFc_hbv z?<>)@Y+?e?!z4^G)$!+WLdU1uhCv}5ihq?Mylqll=s8)+X-3Vd7U;-a`1|+g9FaQ} z!}Pk%6j-`_GT}1qO3~a=^iIwy-@ezG2p@Zu^fyXI95}qcTBQF~NCPCzvkYGW!e~Kx zOa+)C{L{3+52`P@mT^(*z=_K>=99VFYWAl0p5J@41Ipk_%~gZeE=Tq-TV!tgBW}Zq z7%5{&zq1Rxy29$q&aKIiGV zGH+G#WBRB-e$K~8%+)Zb#-=<5`2KTb z3&(6aHnd%#Vq(M&84*s(B37aE>BviQe@hs3uJb3yx9h}{gZ&J#d6dHqE&J*e!_Lxd zPu;7NO2(*|&5r%9oK*7}`zO5B$qvUI2Ysn@_^#Uds`8IcX_51u7G9>D-c;zg&v@>#YOBwvOJ0dY092&EogRAdz(L)#D3H+jZ@uWL@$S6_Eyi?7}bIF9D zEI&WKPk<0Zh#=B-0BIF6Wd9{M$6d&OgWb?@%t9?1gZOzP>gLpZ$LFNmCVtl7Tz5!f z0)yK^R|a;`AjX{uSxuzAduh0PzsmD>wSCa7mzDCYFYtwx$E4FQqX`tflVGS(>_PzT z@a$~KLO}B>{cU`-1rm77*)1_)z;?iGy+EukzN21zm-T_`V)Ry!D*0VqWFMKOS|}8f z&K?EE!PkiyXtsLxJtk?2imqcHsEcq{x zYm#4U##f${UK?t#fi&xNrFj@WnF4|Cfv&>#Ug|`mye)k5LOi~ML!}fKd(tBc6%&JQ z*3BP$7MV&X5619 z!gD2SB^8XdFA$~q!(y@B6?U{DhuU_Tfy`6h->RaGIPzlp*CIJ<*aZrah zCynDR@5QYef;hVcdN#rkMD7koJT8zr@-P%fRu*^y)!V!v@ix+2L{c&gb$;O<58Ou52jV36@EOtMaQ z`BqZ?i?@r1fmpf|Fps)f1NX@xb(JayAg!r=I1XkXz>c9wYO;?GnZU64?6?Jy>Ql0<5cnr7pJjUcOZ#CFam5;fK9y>b|aLC9H zY`<3rg@?A{lG-q62tRTPpj&nFMqq%UM+d-5Uw;p&f%=RzYEBk5 zlU;WGw9*lLz9qv|Jfr{gWr?PI7T0RHd6{i??JE6pBhT$b3IMop-9eTi0_ti@!h0Tu z^WECu#t`0G-go{^h@6)ba|yMd2Qq+vc_$B%8^A##2~sSx&}9lNCT9IaG9<7ttEgqK zOcFXVK$r~tXFE(Z7{cnD%doUi_Q+I{|FGo>f0UGq)bT|zXd zU=NFJh!V$wPX|x;wR>=&$pojXLwXeH{C6SaNF4(2+rRyHcTv($!cW)?RJaB;AY8O% zu=?~X!Lj&?D7{!zm{K&YnYku31yE)hQ`$I0H0$xG zc1ksrE$`z5y;JcD@;MN%qO+XUj)b~cbqX;qW85&_sT0 zPD?Wkh<)YDp(Ih7@0nwJ66!jP+w0w3-S0n);Id&0G;t)N-IQwy3d-~HRf7x$Lu4j$ zvs`SBU-KJXo}SKB{h8c72Zodxra^n9?bQdlT;6W+2ix(&UU-qq4bCOEm!&~AjEv>E zO;y>lwaIdaD;g?7Nn~HtNW)A_UIKealf`Dpye>1*?C;Z0nzO!-fB!u`4hQ@Det9~0 za*|kE?|S?dTb;A0VxBo^4d=&o>LCbGYdVl#^vH5Y1NF@bH7J**j)}N-@XHhK?d-%O zGk+0D2l+hh1*({2VJFcr4TC=ru`Vk9^&hc_b5Q>AYCi4D!VE6w=UN**6}wdmQ)im% zo*^qJA86OD-B@GmHw-4Mzt~~2+nA(28%`!17`p4*7zlbZ{3);&&8ln83A9YJ8n87z zOmQ?|7@!N;pFPJzoW{SA6 zIL|QHGOeqmyf??&aA4FnvOYZ;qYCbf@9Tb{(Q@4sGkl&({$ySJscZi4RA>6%NtkRA z2{BDbrE(ga$Z69ZlRGXlTz#9&Hk%bnO3E)=8OM_R*|Edcm?tA)Yif(V$6sWvQ-vK% zMlkaXZ;;p;!UWm(;U{XBoF%Bv8lcKcp9yJSf4tN)7X2F_LDMK~J-}RJN8&=$_^A@N zc6rjM)wc~jN5!m;G+pQ>`^eXk+6{t1Ec!Cxie5TC12}ciUjkIgD}w)`lF+9<2OR8~ zKFk`)2JY1DOuzE#ik#}+X?iDNnlH=XN=*URA+FY`RVW`7NKwfwV@s34j2N)BGijeJ ze6=8z;(n`OC3l8RIvE%An6qN;75EW-Vge^DB|}M?#w;ooI5>Ep4Gg z^fbOB9WOCnw1qbHS!&cwQ^oq@ewtF;>B$u30EIGoi4viMdZSHxP$W9)Z;w{l+hfrQS0H5I4XHp??~9)h}N1Yq@`sQNXJ;CzK^ zp2A(%5dSv5Yk@<@{bKI5$oDo4zK_{W-^gYl=pTr4vN>pqLTMn_J(3}ZSV?G+ZPgL% zT4oGJv#58bu|IxCh4J$9iPC3eJ+CCf5)C)>&{v0bWfFhb@&hYGBxVaW2OeGy$l6pz zeq0?A&8L9f$t0nd~~!5!DoTI*W2b$*blwkD)hYygqm^>cJX~JiY&SZHDj;e7|&5 zUsvbtq!V+Vwu=7%oLCAf_Nwz7EhD(n9-t?7?w!Dor?dj2Dl7(SnVd(N=zBaVy z_WLfu92P<~UEFG;jWuBSlQ{9^#=*tz+Al>Jbi#3HxNT^_nNylmuWt^`xG}B6?SJ`t zN!-*DS(=}B34N;|cmIo z{ui=wu?Fh6ZrSDMr=ITvo4AgQ`P|?5I-b{ENca~H- z^FAy@2E@!64FbJyO;66IAAScv?8zQg6A?}O1pZ1~l4BCdj~cBIvvT$Kr{$TwPd zpDY63e@4@*eYP(4xiH(vaV+zKFk+TMMV14eiSE#%Kc$Us4v z;&Oqa{Dyf8*&(&sgGa6pG=>zJ(YC84t48ntLJMM45Lq>3spI7YQA+7ZDi}frFOwu2 zAWvp~#p0m&$l=LM&h&{mCHOfY&Z2%njm)A3m!x2pFulP7G`sK-oBNA#OZ`DcWfdTh z#wMnh6El)2Zjb&d-mTGZhxb72ru75lE_)ho^_}(^9|p2-jBm;T!={B}!;?C)w?l?? z>O1!rsR{M=3qy&UaizBs*`}Z_&{yyTGBj5WtiY=T8UTkyd~ui<&;ri-t0t<_!M?)Z z@AQ52T^_f6>U)A(9lj92ds%DyPi5mQ@EMzJzSxqzzS;(1o)$tF%5A|GsHWZ=0DM=> z7>bY;un57`;lih7vWW9(t)JR$LQCTWf%Qgn-f2c7Utn%8Bz;Yua(vm@g?UgfV{NKM zSCNth^kd4&(8b^3qlZ(?jCbfWVlt4BA`}9HmdkNXKa9T99K;w|NcPiqGI=FeU6y6o zZ)H=6C^e{JNCQPzDKjQ1z^c&;i&YqK3cvgvtGuz;S^H}yBqxK-KyCnt4P!Y)s4%|m z>`+1KY(X$gT>~qzou(J)VbCUyqpSD1D11N8M2}wH=At6d)G57wVqNj_ihM(apX;d} zgu}=o4TUh%Xdl_OtRyGar~V`;YoIAzBQuoIw&1VPV%j;uW2WHO_e)U~ zYswW-s!{q~c(f4Q`*VH0;Bl{>Cu8jC;^{Iti7()MFejwApJ^c$onaPZW^Zo>Bxi47 zt5RhnLTGqtO_&La`5I?ti_M4yem9+85KW9u#g}7SYp6T^DNK&N`5St>dW0f5j)u2w z&gj+U`t9fk^`Mh31>+04%n8Q`F6Wm|Vyb(9q+1EqL{jhE+;a^-6 zqY}w_$1HRN8F?|#ODF0*KAQ0Jd(gcxu9Qqv>rxe*k`DzX_V%fT9&rupYRkkHA%7d0 zL8AW<(giXcv54c$c}-(p=bo80bHs<#&cj9}t{4oF1US?5n{g9w);|9C^CqyMAa6=0>?=!#IGWTPe;8G@Ie?akm$>4TwUFCn7C|9#!}PR?fNgP_Vo);>!Tlg$| z!;ps;0wE|?>qVAlHYJsMu`%g2etfTJ;C$FH=BTfP8Qd&*KLy^FpY|yQ&bFw-+3hpY z2ugTC7?I6Ciew*J43$1F-EW~dS|irSkms)VX7Mp0@jL9JS~LY3H~YQ$DEW^2@{Ju6mi zYOkV-+7!?EJ;`#9Nq$qLLmxz5h zybgy-RIlkKWnansh6_>UffeFlHV28{LhX2Yi7~>ITbtlD_J9A@d@NGGR&h#bwIlms zl6%G#bdFg2??is_)cAIYgc4~=Q+n83vyFz9juXrr8ZgubxD2GJmv1)#bw&5Z!b<$(^PiLkiQ#g8mX|7H^N7oeNQjp0QN8 z)w})W*|HAPsr&tx^^mQs`-Y_|<}?1!7HgB`v{M~fG$+RroSX)O$0!ob9)>qDeX9~q zmk4dlYAC;&)AYZI{ZjttagIHx8~Om*7 zudQD1P*_;1w5pJ@lTiEl^K3HeUtJxoG3k-69wX|mISq$WjEIqbA|LYEj7v(o#&FW0 z^h-p6lYT0*Qi&a5DY~S{LG~`Z#K%76FfkrKmJW^ypnen1I$XMj0_8L!#g!QcNvaL= z=MJB+q_llu7ZTs%nFP1^V(^)s!}=9<+80WQTmmIaJL51v_{ekiMa$3?+6G~J$Rkw}4zP|xp(_A2-z_}+Pf;XS zU4)AQ@p^sIN)-izH=@Y8yVV@(j4T>Ophq!61n5#*LvUKzce{rPl*{S(;rfk+%1VfP zsn!C;^sg1d`()~LX@1PTJ25Bn8HGa=wmats0@kO!|C*$V1;OQf;@J5Lp=@-B_zohN>NP{Z)|_Pi`%q| zEy0lSejg#%1&{6=m%SqTFOwo20XJv)EonSkgZ zM7R}o=}QVnHzvfjrn~#uN&nu+6No}^Ot_MCK1@CwXQh~Hw?sTWBYg^j zG(MglD7FBnLfPM=iwJ>*j+!~&1q-{Au{4Y)I58c?h~_~{jCp&4gN^4zkM2$Fw5HPX zYmK85gTrOoLyIM=P0NhvA;Q)MdO%>)WC4v*jCRV{4<-B0y-{BLN9pgK9}!R=JDrVd z-IWp&yBlpJ-){8$K0)B86_ZckKy5~Qk}A{aNYXGt75gH)5A0xRnL;B#!gYCVLOIrW z$(TmQuO=W$RCD4Kw5?#}SWvk#_msuHjJWZRVHRdrOKoR{r`5!2b@iRG6|MZ% zlhfed%$H1V(kJ0;pZ)BiIkuLk65M3JxJ_s+*`81>_+kf|mM)(uTmKHz(#-ZZbUHu% z-(d0Ty1OzRM$>>G*MPUMD7m{f3e6OfZ!&D6{-93;EPDeajoU>KQ0_=-neK(9#*S`{ ze*R7FoH5pr9B!4@f~K;Cwx3CX(!uT1c`Ds^#By<_0|_AyrtgvkMrmPUKz#PgHp&t& zZXO2RJWq?cWgz%;(S2W(;Dgago?O+tHm0Nhs83pmRv5}a3fCSQho8jmuX--#<91RB zfK9ku_VlW&8F2Rp6QP^BgWK4NM@3xM20U*~TU0?$X-iP-Kk`5Dh_WX8*+A)e4xL^aNx z^mqEl(AMB}87oL>VLc*Tlbz|Z+nZHK$FFiekslhC89fAwSk{Z5Szs4@#kkOrrCqiC zpGUtih{I~qY*3uCFQPAKT2jkd-_pJlWZRvoadJOQ=~auKp##*rF(QW!nMAAN1!#E z{&jA!hXdDpR3vltix6sz9*%WOOJ?e2Bp zJxbS-We{e%i`i~7_Ijzlg7{8u-Ql^viPq!YC!2!QHbz*0_tf_78(03|A)OMq;%pxP ziEriTTUH0obJqmY+Nce5&0Yj}zrAkVS6P1}dlCqwA!d2Lbg_{)t}~(#QvWA`D}R(y zwtq4>T<0tQ>zd~l2}>X}{HS#`QSWk>&+g?HMhI8F;xc{+1BDalRv7;M3BZuWK;jFF z!RF@ZulAwz1SXcf`r9-vEKh1LN&IJpPK&br`STlpr3WJDpK6AdPOSoCdK?mg!=R0X z;3BSSG5@}vql@|cBYLh11|RkK@v3n?kFbP=(=~Z-rOurJ13RB3Ap0UbTG?EGJj#UE z@4FqgsB$;p8MCIxaE~}*=PJnbw|adyW-wZ^p;#tb`*X52JPa82eZX!|)M<_rv%eiQ z_R7wQ!aNC-RcQ5_ujdW8U5F1pFAi>yThNK8mIA;J3E$tyvVHvI#$O12U*tH&DYo@H z?H9JjbsJ+mh6Mox3>na!4AmSXVWc5}60K+g-I{dfk#t^%I3>H+-rJWXmbgMJGK+T~ zkAL+8bT9OYn1Pq}Bsu0U$JDovRkq8W&XvCz`hka;hhfpOFHZ>yxh0u<{a7GsQ(vOI3 zj2Z5{VHyO}2ovPb^SB`tMt zc6~*8L$nHBUJ84=ivHOcs-$!;1PU*gC;#Zz#^|iju^9GqztrW$9kdYnwCM4Ht+VGL z{L7A>{9|3}M~HsPWgm|}+#fp9YUcT#-UGA0dZO1gAz5e8ufBf4KEa2(++Mjp8QOKk z-T=72+mT@4g~+fq=<0H}sx@pol2AC2?<2AgUdJc)t}#FDtGTAvOSWSrMv%A;SM(Em z`+K}4MCCY#6Ez}G-S^%Z0tk^l;`yvFTcRW0p)sVA zVn&bB@_BKj%7?^$*dn>jOmZsyFY86pUs(+eE8zK{5*b#XddIewrpbNfFcL8P0bxt6 zHPN2f1nH3Xs%+JWbQTvUv#mM%Q%kjB4LeeX1UhnEF!?jXSARetU+upR>_vW#!cIZa z(qD0#J{<^XzaNo49FA3)AS7~g0R1ua!g7=F=aUXo)M#dsr*G==!c{6toP!sV%KWbf zTAx;*+|#cE@`by>gt>fl_({6=f~nCdSfrA;jxxeFNkjA6;`-$FJotL#c2HO|_@=n+ zqlW}bp8|7+h!Vlaq+*IXW-CjOfr?uPNiE+;161!!cXC=IhRSbLPRv}F(o`b_hfCy@ z9_km<9o)vk%CBmj>{fmL3(HlXrK-boL+#Zc)e;x-hkEBk7Nveuv6*)2s zv8&j!9i|b-=#{||yx9bTEo)Zq|MYVzCMrY?i^k>>ntpl` zlNTT9@GQqZh1ggLNIkNJn~|LT_M2#kQ+Iym>sGPg*edGp_|Qo%q_D0j``|>5DOsjE z_io~a8>Ma*4cH})4!@`}vomu3nw+M5`E!rj6vI#1wWF`r6j9u{{k+}bm;_CRgOc=* zNadB5GB9|G}=RTgbQfU}^|36?`6f-PaF$TzGdB!gK-;fZtUYx#5q z(7zw$tb(_}yi+BXxF)ooQcNsL1wotY?_bHy6pK+Z|JRYUf`9G7=ePae(9+tz{Gqd) zrO}aZD~kr}*&e;Wiv6JXeQVy0045GGcxAFm?X2kq|JvWB!TYa0=X24H@5)VLZN0lt z`B8DngfJ3)rs32T-^^YO_|H16r8@I*Nf+YAb}Z|Rte_99YAJ2@6(1VI9njkY9ePKu zVi`9Qiv%5zo%D%zozx{;ja1wfBf{4(9>hY#L0Rt-2xR|Ef`(+&+hOHWR8(0BuNV^V zz}oE=g6f^$%iV7(#?o3~pxuFGN16(p;fncrx^i48?cQ7~@XNb)hD4eCKpoWxRSa#44ohvRwk5SkgPuWpWlM?8Cj z{l4Mh=Q%UR?{oZ{?3>KDsTWRc_?M^A&$nu|E+79F-C(JSFJ&=sN*C7w3^s!o1%71a zgD&H4YGII)@+XcLvqj?h%UFq`g>w`7%_|_ zF{T1R2}|`7<+LRPgkZylnwq%XK-El7*H2m&POaol%#Dpi!N>H0%5lGJzCF*mSqr|=d3^itsC9o+q=bnhh0o{t!XROZ!mxHA?&t$Jwz_tt3%w0EOfI8vp+5xd3p+v4Eq+>hb8n=&5h8;{5Z zPp3XZv{^Un7?aBz!MGt!kd}%8inpsUuIs{AbihS3aK4T=uQbvj8^+fj(}Nv@D7+*i z)|d*#So0MbR$zT!pjwH3)!3m^zb)@I`W+U#Qnx!>;!t;wj%6Lwg!-PQk}d~zNQ!w$ zNDz|qnRunBqF@#o~XuGw)I#1gmzV_ykW(+@DX1hJ=CD$pZnN_hQqRi+I5P?RvZ2 z_HQL(Uzzgi`sVcX^3Na1IfK4vM-nvJ0EY#6qUIJyo+2nj>~M{06|)J6&zxUwkmVsA z^wk5WsfzQGt-1ev&nRNoh4+si|DMBl=AW$i8VS+Bt?3g_p2?7x2mk)J;l-aiXa^-l z{Ljf6c!p|Q+6N#!RxEvkZ}>_%ZhdG$tdPpr?Vsb@Ngp`H{&{stIn_f(W8Ccb?a9Md`CTX#P3z1X_L4o$pa{Mgn>UO_N*vH|WFGzzXW2kXPRGJDC^7`p{%OnQ)I3?g(jwxq8&>Wre8f$+1SCQTJoMM9FK zi!-+Ofp>N|({Eq}%1+9G?8J%ccsnqp#BUun8tp$OrYAvzS7{Am%4akqh0yva*Gd5L z|KahsIkwB|Rvs3%xO_LSz`316ym^j0UxejiU8dy&&OA}~f4v4ZK>N~6?L{7FFwk2HOB{HU(|}1btyb%k{dc^H zbIAh7iS$ll@89b$$A>qh%KRMqU`C-3s9>%dIG!7QV8w5~5jEMXfas4oN?!%aI8Kn!tV2VdXb)6Jz7+y6`V`9yICf z{}ie-Z0^CF>)BqxHZ@1e;5`VfFPme3c?w}rv0i9o3=v^I8pn&_RYO-8^m``1xP}i8 zOdq)hE1ZFr4^mTeOw1M1M8H5FQP~zQ=R^47X}m^&$xh|ah)Gv!@DL^Y3#DN*&OA>y zxm1Gc@Tv_lAte?H>vq3`e9Tb|sFcT@<)NN@R@~>!u6dzI%&KPh+S_@Jr+ul}i2)MS zV30Lg*$b1@TDSBc%bvjwV$LU*+VmitY_1W&hL3Q&Py{4v+r@1cySG8Nhim`)lU$!) zQ%m$@ilwaXP-ZuXCb}|&{761B9^}%Q1%VJzAEx+;TZo|qE-X)JNraJ9&T6V*cut0P z@v!!fhIPrE>>xA|U7xoaFBN9wxXMWg&o)_(ueW-6EX~1fDyk;W77xgi^XwMA0$Byk zDeLGYlxvffQ1dgyAnTxdS_DgYhw`1*y|ud+)n4BgMGAJhlIpg%mpiy*YH&ssnlMAp z%C$nXVdLW7$ytu^-%6A6*%%m9sD<_g$C#GQn@BN@GLGr38nTh%fGaqWc^@xn!%ZX^ zNWrlbj**^kZd;0tB3t$3`yG5!;|;*U4p@J_wfbew|22i_V#_Q3_07AJ5%7#p{1B>Q zR=3zuRLLj09gz_p#b#H9%zT)|;1Zr3%Kui`Gc~oJZ0kbkn_AP)+*kbZWbOP6RKhxY zsEWro=W7-K)bet>E8-5i%L_()i4P={jzB$O7)&w_`?;YNJgFx<1ZR z(w^><-S%te;0(yEK8edHD;SX zGG1z0w?g}}SBed!ojZ%HOmu$VS{c>v%tNn$GFP4{9$&{|c0)eknU``Z)0N`1rVX_cE~Ukyf%}xy09q zYk&{Bbf#8-1QDlbm!c>0)k|k&^h-K+2>QSak7*7W`!+D<>Tv@F)U9fuzf3lY~`AIiF(WCYA7{un)T@5_@H$E z>Uz=;s6+xa2p$_N2=$Or~J?>R>G?Di`_i1M;u# zAen9gEUZ0*GFf+F;6KQ+TvJNy*TuB&vJYPPQwCWzQJ!u#*Lu(3+cn);jRdL0Gc<4) zw|Yh#bSe_Cqb~$zyOa2okn2k�;rS<#7#)#*e-Y+H~}={MX||D{J9P z9NYsfoS=8!e5`JmL8qNch>HLMEhb&2z3fw*O8sc`Rdt2;^~q@0>}P>)Z*w=SbP2H-t}k+izB!ucSa_UrgxR;H2dvtHy@T-jzs zq4Q{E2fW07Qk}giZGl_KV;BSPJeIvx6{dWO@7kY|Fw&Zmn)wu(`rF z{{DeJo)0+6$l(m>Cw`gNTW->d7~XnAQZWupKY}iegYcb_;0GD*NUAzCmR3Y(e0^+0 zRhyN)0^GVTi3LK=@+L=aboT3UIEAl5$S4lO= z42trfpO>|FM9Wz`aPM@TzA^2Hp6B?L%HHwsG!NF!U4IZGjyNXW&h-4igc2#$oKkts zp`t3>qp10EDEFv8V}Gx6D}C{Iv6TLd3+Nx7#`!}1<-h#SyUM1AiJrerk7P3+C3!Wh z`1rGqjG5i6BBh7qRJE(*U(l4GKBEK=Ymy(;Y?KT;?#TL|SxS#c+p!}=gjG7BjdU<0 zIE+-F-mE8j`V_fkK9O#uG7b8CDz|Kht;U{Qe(L|&_$}PM>T9`c!$b3|FI|L1R>{0# zoqI!*&n;S;8yi=;@jp*X9pS~sl*fkzA%GYO=4-bcV6XAnr98Zd=VL$Q8H`v3B`KlI z)CU1#;$wO1UHqBWl4~&Qq6F=}Oq=M>T9Wlr{Vt}4eeQ2wTE@RFj%VZG2r=IWV3Mtk zb8LW>{dU1;2>^b*^@^}x?F&(^b{>5D4Pc`Eu1aU)?_0ef%$zW4C)wUo{V%ZsYivnL zmizlz34<;>J-1q-J}tBxNAo$lI*$pX!0F0X^zve3D01uvN@j#W0G+6Vr&&b;RZ2o=}Q8`^O^JZrRAG8wa1{A85s8A7ARUe zk;#L+J1^3xP;(MN1_djTjT<1<*QDXAYipl^H7X{z%yDyG<`3?S5N1}*E+x-5>X%%9 ziIcLBc<}=_>7h#ud7)k(i4URx5F(O)q~ic2!PVZ^V0lVvYHIGcL2u8cZg(th*lt&E z4<6pG0tAAE#otW|?uZ66!N)#&#>~UC8&k?8t!hwT#aBhHdKJxmKR^ahaO&14iP7X=@eR;q~E6sK(!=n8|s4+!ydM1;C}Im_7--UR_D*(%Lo2HQAE@w&;;Knw$;(x^ zZWbB6aQN!XHu9g4tn;qZm7)bpajIKXg^y}CU!f!O5~=6=!$Z+zQ&;LM01OXAhFk+S z#nmCh)O&&VA-yXBEtyUgTB%Ak%7}-dWGpp9$=a4Gk>n5Jx)srsw$A-U1Tj>`20pwY z2A(ebMfEPta;blt+^#1~pa10vBk67CjxNhF-Z$XgxH&wJe;jaja6HlCcj6{1FUMMp zj1UQK3J&mZRgCEprs^66_mPsBs6<%t6#6corcCc-PThkF*t8ZGKP;Au?VjCnYjhaE zvnY=u!HE!?m=JKG>A>1{7{)v*adTqU1YK_A^(*m_(8zxejy9We(34xUMW3p_vF8-B zVa_SHX;h4HE)k$;X3IJX&`1fwSoe|S*LM!%m?c@VW5|E!CuFm7n#cQng+l+*33mV9 zuaEFxU528E5ZhvqjL>IUl!a+@5@HqJ3RLmMYsLRlCdb!K&QX>Tzy^u2NM0hHq(IxFY>C#9v{o5sI0&hdPdi zSQrDr84uWHUe-HVN3r|1KODISg7#C=$I21P4s9ACJZN$KC{m5_;1e!MY?U7)43?l5D?7y_ zZbH9W5ex=yu4$wwcGJm_sHBIxOXr~U>;+XQx7UhwQXQGYSN^>jZVYpS>l_jbJPVtgs!YWF%w$29y% zJUd*Js(6hS(i86ELY;SUelQLks3&`6P@7z%o`DF2ov2t>xZZ>=IdlMxcAnCe{AMQj z*#%^YEkb=yw2m>K_9JTAOwSVceX-Y5$nmyv!~ciV{b>%oE?;5ozUyMpiNxgP@qq)n z5eMZi443G>%*fPrE5AV9=0xjm9Y7d_Hec#H-EFzzf;#Aal5V7K+Ad2iNznM^c z@{H#LD!oY54jIFOO(y(2ZW}eSlt72xlJHOhWxi4!krk+C;;&p>shMW*ck-<9l@TFT zt96dRmnJ(m`Z0_lm0_GmvfY#Y-~gO_1f4N_#RL4U52O8*04@8$BhY)IUSt$sJ8^Qf zdxa|@{s=;inM&wLY`twXQ-=`AnGd_nFPSX_&B<4YZFsfTUw)BUcI_;mj})LUx9EQ5 zvmYOB=gais-@9aF`j_;*v#Z@*C;Qgc)*GMOKTFp=Ik%{32_5P)0$b>C*Wt6>FihYQ z<7CXm(`RX(u7r{wlNuq&5ezm6`8stc+iGUH75IADl)6F~Ui#RTRmU z68apQ%-*hci@13pp==T*pgRw{PwyyW`rAReDc(I(YW49)sXB8h3#RBgAqgGQ6c=U1 zEm2Ofzhmb=&5=2Zm&O0^0{+P_JWq`FarYwn$Jft)s#}w_3x81lML77>f2AUl%uL*zyJX<5bf&D^H9f0?0R>A95R+$}x*>qGA@T z>F$r5;&=UC#rD68^iQn=hBoF4KmT|~NQM{fdb>2Y$rjXaQ$IIfDlTHDfEz_kWJVmF zADjzk=>XIT+*{cpC7#7OQ}bMb1J^MS&n80DTva zU2E`R<^+i&5!T#FpJmjTg$ejy?1f?SM=J0x%C7UpL2;}?kHxM|xxhme#$slscDSgK zfQ|;i5-2(<42&mF#+q^&hdr%VghGYz}9&Jmm~12-oZHlCZQLry{Q zlRveg(jH{&>~N-@8QFu2ZQsQQmBI%>Z6h*OiiQ@hZ@%w6HqFOd*=AS^{OOhF7xZCZ zWSE`Rmh8z8gfbBAHEFA7js6dC2z=3J5~&NXrKoi9Y6#Nr_^a!ehKf#lOyMiy+Q1zD z@bFDbtKYY!eW!{KMnDEmKH8QvUzeCJ3h4j>i$g#(o(`W>wFh$S$wCS@GYLMsyvTW3 zJ`68+T};6-r_!IEy!6%Ec)FEgamy{b>7K?_6X{urn-e#W8A7=tT>yQ)W18Sr*GhtC ztC?V#AEtkf`)uhCL1pr4gD9bRdR?Oy{bV-le%)`i|J;?HZ_(a=s6O3b))vo|FxKs^ zQ+5{-wgtQR-6OwcqR{q?^rk;Ojp}|(N+MfsFGr3B>gw}{zW`=U~v8~%Ohw1E{oRez=-r&xmRXO4Y;VOvg;@6 zA4LHVd;n1CQRnsu5)zVtP!HWY*Q(I)YdK)?tYUDu?di)x;)6uB(~&#KltY6n4p(j z#<;HdpTSkr{2ruhJ0PbH8HuW;KfLR56gCPuHtQa5QIN`v z^MfOu)QAY7#n#2!OK=8nn91jU{7i$!-rV-fdu2GOR|2Sd>o|3|xIqZuf(_iS=bP|r z9iZd2tLB^B`ON|u!j;HM_Kg&zZOztky%q!nrvjV0B4A* zK;phEnzhA>yzI>jp5HQ!_nQkmJarAad=hUn|J#s<_>cG1+C_PIA?}kGF$oCD-vhPj zN~Gt&%;Xn8L(tjO*#3U;c$ZtG2#+TiBi>1_V=fW-A^7{~y711<+-`)%#p=#zWWeQ3 z&dtBC5nu{}0+48|4x0{=TeTL;27oTMS0=(vGko@&xV6TCD!n$$-t;2oRU29o+bltzBGZ-lSlbx;7RbIqJ+8(nwO2>4JYE;p;yW?Ip!; zsKrZiZ}AhIPz+#0ZbF&^uM=y_yyNiI(`Piplt)&B)3Pa(y~t_+FL19-srbc_RqHs6A)*q zxc%WSGvOt9k|+$WbMg1`>QRGBNLuE13CAF7H|5N}i%*yeZ2=yi)_oCgeF-6SgnRiI z9;`_mW}-)*z%=@~1=;6_RkHrNm^tbGQ1$nFQV9DmW7O4`2Q;XPad8n(=cM>uh)B`8 zzZ9TYSl~*Iz4wWI5CTSG*R5OB74Lx}XaDh&R4nP%m`c7-h&Bkv!*) zMbggI4|xH0NS(e2q^MY#;V25Hndg#kddEoJh}{EoE}j|TKH)C?bMg1tx4x&wQ=C&j z9qsmCvc(6t3buHa#m;O-UN*E3WWQQy;ae>4vOAlRq@^|u4{zcZnAobIam@;6(uKm` zy}NqizKHzGIjo47>WKUF8`iBNFhrS{5gsTOR)$U2XoY5FwSx9qM~JD0*}x;lozreMJ>d>-2X^hPB;0=_wk%&DUqe$G5cc! zc6uW(sAPH)kQc9JKm{M;$9{C@E9~ZGpKV&GlYKv_jj<0;-K%&JO*OBnq2Le3MWN{F zVl}(LLy0Z9IHK=Fpj>jSY9}4Dg^>iO1WQ>KAQl&{oBj&WE4&c)QNM(Z z_!s4v`a~ohpDx%jdk%|;LPV1I4=hw{H|Z`s#@tyl+Tn0Xol_-@oYuIonUdvGF-As* zDRo35lhMGVMATee)}7 z-I+91=d~lOO(uktO!ZymTm>!txkPgpMS_rO$+iXKcKC5nkrqXjYb8)Z4Bb@wzb*zf zyvK`J*-+Q}Rit;-B=Vojy=cjr{>RHDMq3e=k?n3bD1DnMIQ|$motSa|`|p_>69(UO ziOb9WP>9*M7(W7%3PaNyJ0X9gy{o>Jpm3&z_$kcdB(vGdPjE>m&s%BnJ>ls zJ#A0*dU@l7sa>Y@-RO^4Jrb=~k(7OrrJbUWJ1$ZLkV#_C_*`5&*CA<3Ak@T?a}+x${c+7sI$MWYC%$L1ch10Ry_Yg} zfRawJj4nf0fV#`Ychb88tvN0pC!?ym&Q}*j>aC3l!yrD%vf7!vqkpG}S6gRFWfSEP zuc6C7ll_P3&tHl@FTj~oDuRjH{IB^?H;c!IYZ3FU`$QW^W8xvTFBmY9h_DS=jdK$w z?b$HX7)Ts(FT~n*Of*~2n{{W1YW#u2-zn~QP!wm`8q7|!!2G@iU^c(r6eN-Kw|6$V z8z~Xjx6SM|3}IE5?(cU`0rCwTJQFszsnp z!4!(nw!QT)ndh1|LOnZN&X8WDqK@X#`_X!V5VV0}nPK!q1c0RJ8V8WO3B{wwbPZT- z8L8`g5OOt$Vl5wS-;(2(S_6{%$c1&BTD%kgE;C_eQ0UUnQu zzU%s1yLilU`8`~&dg~j#yCHD2CT&_v;L(o#v{Jgl^*>q;abE4N3*a;7u9xE^lq4YK zYD#D|3KWh+m+%fH4-RCAdeC@cT&zO{ zbHa>@j+r|l|JJsLY;2d`cYO)Nn)?0Bgn}qp{~_tP|L?tYa#-1hQz^mHq$*&2m41CH z_ONQufMPjc0!?03L^rq0ux^?H!4O&;nyZC6mcC?$B_n?>}lk zv#YA|8cK@sd!R+FFd#ys??3~atpng^E1ORbPlvysPo`S5d~3emj@Vad4d(IVTjMm{ zS*Sxy{czC4bQ7ympb~{u#NQ=p41Gw~^yBG5ui441jByt9t|S2%uA7@yEr@z!v;WBu zYFs#(JSlv=?lAdppu;nl5%klOAK8GI`h#O`glM?tcsV|j-V^zz?V zGn7eHM2F?9#q^LuZ&ODMrpNxO=4O2!$pW(dSlz-FND$dP5ZF0OD2_14R4@^qo)Eh8ZZ>;r~p=BF*3 zIBg}Hx6q7&fM@76nHTqu7!kE}LB^kV;<^RTmE8^10-t9tF)PVw&dWvh`^OAervRE02w4;~x?Gs%y#h2RW2SBr&z04SwQ zsO{gS(9Nn)+!i2iUDo@3Ovj3aos-+yyL~^B`V-kWl{6Zn6I}GYc^8rf^D={ zee`6OkLE^SUu#8_vM_mfcRm@(YiF>B z&E~j@4efbwF7G@)b*Q(fmGs`h3a<={6VcW$En>|xFIo3h$kKUencu#(wz0C3x9$ja z{yJY;h3SSN|6hMOxVC09E^e|czMe6?0L0#Zp0`+dtPqkH6}zRO7cFwyoCS$jPB8_Q zPHZ0&U*EXZ&Br&g<_w!h14PBH2xYV<6yy}E#E#OZrC z-eq2t5In`__r0v#KLbk#7yr@SO%~mnd9Uk@Y}OoxvU+z;c^XGoQW)jHRJPNIdhig; z!@Kdh1T{47tDw_o^FT19+Ns193vBsibFzU0>XJKlOaI->$+gx}Tj1H-|0y!j!zLh- z2&;lUMVnZn-5I@9Lw2*B6A7|%!JiM3l-=E#3P-Lhd)oiK*RT?cc8DssSc`S+P$?pm zZv+@0GYqLVBuJ1MZTn?r@xZ36MR&;%7dno~AeEs!jT}>51-~!KV?qF;OYNEhSb`80 zYhuOmq@%PNy40UMbkftj@pfrNgKr=S++WmyGJ+)0Lm^vx?Y)iL(&tuvA3!~IRn$?} z$$Zn;zG^&kVI?s7)|mAUO=D%cRBIWaljPOcwlZO(8{@OzBu-!aem{K0SMIcV7l26i z*_;eU6CW{^kPIO3hu)#^LUmySj(6@X*MFm+03}o(Q)~NmZy=b_-IzM6{ZefLA%@qD z!)mY=e>Cb`X8IL^R*3o*qdyEVT%8m_4Rd?@udDK=L^JQs0*KFzOrg|6(KzI!lw7gn z57R+$^>o?J>%6w+=4P4S)K@oUUtNLT>yl20rSO@L)UfUjAoxQz*?Wyru=mzI zZN{@4hG~5j0}b9XMM+^R%h8?Rb%aKUxcII4-HZtzTg&`9I>-|#jr5!=x7MG*=e}MI zK($eUqv{eCba_^4IFta^#A|BNl_b+xOAsNNG-vhAsISc1?)u55vRLcmWcQ@t;q+bi zZ44BjLZX2m@$fXp!ib{7hceol>|2{2EfU^ z+!-&{c?`_lF65|~u$jen+@zzXQ> z-NfbJfr!EQdsk0xf4D+B$WOT+I;YW>|<)6_9f7_yPq;QM)d)=UfrmQpMTxAVk9o>3$bCY1# z>;|m4@=>Z>EXRas+}duD9W4{-gQpLkGI8gQg1nmpGq7Ei&9c8wuWqV$0jEE&7P|zU zpibX*06}i$0@u1AC$0HxfE<>W1}gjb*z|5?(DYIM8(H2L)p5B;(OPCOX?y*afw<9#h zy*@kgkjgx-AjEOw;HJ1pEDxcYpy~V4C->@-0T{{@UvSXTF4QJQYuHB6MuLig|J?a| zzl4@yA)nYMd0J**2|-b;$kL+V3i#0(B7pR!E;!^{>`T;;;TO#Brq+BD344?^#;P|w zY*e@g_a0n>B`BzGY;C-n&k1Tl>3jad7DZOSdc;L-nKV-TaD8xGf8;0YJ;u1_S7m|{ z=HuR4HM?`IPDqNF-29Dz*2Y#rnfNX@g=}omr`JYj&GKfEs8anDGc7O>WwQC!WOFoUvdvxphr={_3Su!r`lZA7CMTbm)FVIow{|4^6;-EVq9v zP4Xtub2*veh)iR9ooD;~)Bfy1K4Mi-B3&^qk3Vn!Rv=%65h#-=i&Udfxh;8Lk3*ZM zOab5@eJNP_cyHl^)h40)@nUd*tO84XJWJQcIh}e$H&{may&LF@Y)zCvjX8(nVRP*t zU%z{6{-;jDCN@Hl^G)w*cKC1*X-6Ar^%G*{il+H2pDy2yyP+k?ro}PH!r_>1ZeXG( z2vwSQgRw@i$6u87J-wIbh|PSu^aPQ~I7etK@R-*pOc%5POMg33$jdmd}|Zsd%hxrJ_h_us}3S zm)?@}qLiZrN;!tah!q8_{P9#l8rPnu&2}ule#n=b5tHOFWtdqVg<%i@_X%D|(!dBx z#7Mk<41a^2uSv)TBI3g+fWct-tiuPk!n>6OYF_d+iU3COr}+uEg%dxO8US z$69~+!Qt)-gisjI_DxMkE{32Yop;*;Nh7GFmaNmDr<-ovVCi^83mr&%aRY=Hu66{@ zAAA#3vU*R1i6LaGZecB+#kHgv8zJ{wg-X*rmES;m8k}CPgj{LAqOi&zL)_?8$6oiR zeeq+&!pi@%9Uy;HNJyVa!amU5h`)dJEc^BvuwoT~*Lw#~6~)J2CFVnE%xQXa+S-D; znspdI#SAB}i1AmN|A6qgMRZ?e7CA39yROEnUjL;( zJ0$qw;`291wq}w-eByi)6SOd zcjeY=yCl+&59r(aozL5__ngwox0mA-D(C9K!k14QaYnY7n zn%!EJi-?uNJRSery1Lz_l#r0PuF6}j$-;+da3Lhq_eH!OP|4lHyN*~6Sr*m6DI@V0 zL=yok`)riGKL3XH6QqcdHH8G zheCx5PG+v3}b3NLR3TP@=i^s{?b0(^3Xpv@#C<6`JX;Ub%d9!Lx2p z7Nej72^UVEGlER3>iBX~#_BFQu+lz_taeqlI-NV+y*3vST z%h#l!XoB0n&3F7mN2Vb+IqKu(>ncc_I&LU9+q@(xbdRS;BlGHh3G?8KdGDTo2Dmb1 z^(;$^p(pN>%k0k{g^<1KtmR5F%R7!8Fgd-3b=fk?E&Qm+W>k<%97|RxX`%?|A452w ze-XIg4J5&fkDnsxV6PxC+{KbOpF&xqzSW_-1jXmRj_N)~kmp*)H)@{lmi^TyL)l}M zMJIhuJSwEO>)K%l6_u5kBt~jB6c{gFfrOB)0@@SWr!IZ^q9Po}t@-L0M_&F(Naum) z_wMcGtk&;xOJF-tcj~p_@p{W%VfiU3i05}}*6wI4@TX0Duf)}`|9Ur>%7Zu+Epx}f zTa!T+$F)ws)9JoHI$y4ls3$(2xmh0^ET7r@KFjNMul#B@6FeO}uD&th_po-}W9We> zg)IY%c3&CRI+ zovNE?fXYU@$37!UTga_(u@(~(o)bc4jVXys!dO>OJ=w^K7WsuV6V))36pAQ#gD;o5 z=ZI>}WpAc2pm)vP5FfB^Ix+&NgZ6@w0fOV*mJM{^2U(a_hF_Gcvfvscd*^xWLU-nG zv?Ha7T!Rc@QytWv{M>uENh3z;=}iyAgTBD(Q6jBd9~F%T%#f7U46N7z*oaWJq|MCq zMr#4wm%l*F85HVT*s|#Q>iz7S>U#vB2J1-eG7IMCNHujO*==ci(e>n7-@ujoCyKENy>_U+_?wM;W{epJ~4uhOQQ4e z0f@@`IitR>{HC1vG( zkWybYbqnmzerzswNz0RMAQC(3`&`09>k78P6*j3T5&h zRx@eM)Uy`+jzS&-HG#L2B&Qi&9|}XW8-up32>_iT*N6AJhmMo%e$G2>uRPmkw0_s| zizv>IonLnxtpU&P4;v&94}Uhh=JrX?TV&tT=gBko8QOGqEtC1m!#>gZp2#@8a?#&k zM^h7=lg#4$ShK8Cb;$5l@+Y=|wRY6$=Iwgm)c1>y=(M%h!TwZW3EPuJYPQq9{C2 z{>DA1X%elc&yS}}dyiv=W`&hKpK_z=1Tj2DU<10>(gW-$H77xOj&<%P#9G%9P6CTx zg=5b$LdhrQ;#WHQTW~HhVZ{Z%%?AH7L!(QemP<`K<1IJ2oby4GBHdqdJ?-sXz+tD{ z6CawK2~sfR(u^E5wbWY!tTmp^1@>u+v2 z<3cCg@OH5YTTax$nMsmKL|=wZ|LGHUZ?$;;0Kvr)X0(>SFF#4TN42;09d?YJc>iB+ zls1wUkK+{j^5;BnuiK~2&X%_H!Bln_2ZWv@nGfncLWAOs%+~YflPaipwf<}FyJ?t- zwjOn&5Rlp~cSthswr&Lv4nBYHEy)VVcBNSh4qZL~my6jl5GhaTOvVPdb?AJbOxWzo zmZW(+$pc71-lG9)nWPkuX;BTp_BL;FnN~LE$U`MKW*t`+d#WjG$7_gIA`68^DGiP{ z3w#{|IdkX##}A|AzR$KQGa-heq0pfg%;Ch54bkzGyu>4bM$#@lGJL?Rqqc`f7K%>< zE|UXJut8}l_Kz5b`3KY@u5%yF3P+k-=ENCoY(cVgBedBBQBr(0@fN2SB4Z7)=5p3$ zQL#u&)Gv)7MG;!>S@#MMk{ha9<0vj1f^cV(jY0_#^U|8WU?2HUdWCuTiNB0 zDqp`vA~#sZ$LIdOKSHD~A0v?y*WPISBG1GQj&0+$m?4jB1G)hpvcHz_|1uU!vlaoa z!cPy5f5A*yhBd;kI~E>k36~2+n4(8V0vl$qJE zhgEHc=#aILuU_1AO5uZ;{*>#^h_c{meBQVQob?TcT`e7*2J@@I7ynV1c>1|dsFIjS zSl8v}a(gcW64=#f2E>EUG*Sq4Re%=9SyCelduw#->qiKXwH8NCzRCCU7JSUKU3Q`H#ZOr0qwVfpW&>kH0 zkfVBdEDi8%+LG|$_@s*1#giF|`-X)4%);MfFH0TWE0F=6>D2-pob<(_0SYT5XK~CQAJsh1pTL1Y9)}j*QQ$^;@^7b7wCQb;ci#h zrD;h6y|F9?R(xrGv9jHpftGPdQ5|+ra4E8k&%a3`-d%c4&k&E%D-P{vNVP9;f_R}V zkW6nhmc?o4NO-w4cn5ea{#ZeOj@0EREHdv^^&p!OQT?Ed6zc3R5ynbN$cbhm5-JLk z?vLgvBgO?&7iytkOaypeTOcMomb$eEQExraoR&hC;9>FM^A+>dBsy*J)5LL>MrqU7 z2Rn|v@P|?C9If6?dQQ!qW z=g;gB!MNZ@_oGqC;U3Is$z=>yoJ_L2m*B!d)ZBd#HzS777QLnu&K^@@Z@ZuS*k{7o zC1WgCNjbGLEQXO?FYbv-ojOCTc&z;!!M)AbEL-S2N_Fu_Pv$sG*>vrwn2-WU!^HDR zgJOq4tC7__tFN%P;rm@1`X~U@pn5`c%BJY@Cd+W%yPm199pGPVk31bD1ab3CM=*wx z&Uv35U#_id33pu{-ndgw5QP!l)jMOMKg~IqW)ngrz1bNX&$QM|#f)(?3@Wh$}11wX-w7%G1Q?c>ry6;ym z2Q92}&xn&))iM_eS;WkGL9~h9rHt6n+LR^vgLA7geoxs^63L@L4`PV-nA>$ynTPC;Unny~w-hDw zI{q~N@|qifKIy%=S-<;Na4TNwb$$EnKgavR=_-K>va-_Dt>atx5LE3Bw%s3;scMON zBbUwl^v{ZJ~26+Qg0aGjdYAeK#z6owy7TTr5FRxe$JQ`s@>gvtTI%xyh6 z4*6Pn;r9tf{<~~%5`Sm{8u}-E%0*Lagw|subg~Pr+E3fAhj$p%^x`Ue_@9vxgYW@8 zMR!*7PRW&y!v0lzcKd~Tr_om%Lu|s0pF$Iri38)~$$mrAa?d&VE=uEX=;uQ((8R|dX}dd-TQegehgj~%tny(Rl+$?IxGsn$J|(iV4m40)jWIF1up8P^=Oi! z51$Mvjgfbq*z!aWs$1v?C6iV#=_0ui;><4UM)++&A z0fpIIHHD`6E5PDLLbf7=y4Y$$|r-9xD~gC zPWByx>%om9|B>m5$R6#zeELveZ3p}Q1LmvEGuf$qv9pVn$uGRa430a8 z=hx})BHXY+&IDDudLHvObM53F z=G$&BqEM&hA`*(N<7AesiPl4R-o zEpkdx-H<|914s65e%#0{mDcq3#RM8e9TrClYAaR5=QQ7dytg&8ZEq#shFnp zc|U2;om@{z6$n1wa+7Ace^9AvAtx$PASK;q%THpKp*B{{Lk45l)&9ttq9Yoorc|pU zhm!-rQzfp+sKN~K2w~aOlD2H8Dzr0l#Dy6|lml(0YPV>&^>)*I@;1>h0zXUEu@f}N zoMVR8#*waQIuL9yUm={w|rh^jbGLbe8IMO|g zpFM3CP92A{YgipWsg&Pnj)QORYooKDX`aDtCvXCel{j26a1)M2g-NAY`;#;M_lfOm zUOWsf$|40ZR#3eqY{SDFKl%6xvwfG%cO)-Pj$2``vC9dzebX{4w{nN`D92^ku zOpZ=S0m^f89g0fVZ#49;lV$ydZdBCp!6@rAv6qp#Kc(~`QnUN!ymxU!p&l%Q z6A%AGrub7miVAz-Gr^8p8Z2dHmChF5X+89kZaS#k`xb1^RhsfyN!6B^qz{=G2J?)? zbxeq}Q?rLcdA2;m9aYGzv*|L^lNLkKa(EOT$IrvsB{P*=#z^$Yte&e3C7rxP5OX0B zVt8nce>g3Xf>^1aS~PDSs6X+mpUj)2{5e*HeZKI8Hl7PK6oF_l2rD+n25C1Pj)A4I zuh9v7^l{_A^hxnl(EYUFA-J4!k`nV{D-x<#7i0?ADhi`%05eVcJNpddD_Dm+B4h1sibOSh2i$p2@NDHGQAGd;u$%2O z`~{|HFVn_5&^zejbTBjeI;TnkNQ?JrY3~XQjreX#)ZU zJb`WY0?z>wssaD%+)j2VTh}40TsQpeVt-9NREpF zOzDByi%Fu4_|#HD?KrH_O8_VsuT%81apEc2Q&sLlg4{!m*bmIJEt8-^+x0tKhlhx| zw?$r1n&wp{Y}2)6BS}nj@Nr^$NejNrNaL2C33|92B=XH6n5>%VF?WEz2@3P zzt{Rix+_hcNcbi2@f=?>(C2^NS#7#^^dTr~XZ`*B-r(Ti_VzYJ!G-)K`s{Y;Vrg_d zpu>lQOq1+rPszWrV?oG~B*^#s*@0@*iaxr@4~BNxDr9=jQmHTInS(1Fi}IJ$FlEhU z6@QR;npA*e6!A3CkMx|G3umxe%0IupIrdANdPS{VD1ekTd^NP8##~W%?*>425ge(#J9^RVX*ihSGzv{PGQbaBh zXt_ehk?>a*U5tOcrKoKiox;#rc3MgzqaT(og34 z&!U%{P~!)l#^t!IJC;iD);tW%IO-~%QfT|YnTBzP_j&vnt#Ap$x;b?G zn3EUK@aLUJ3xR={hj+pcoM)uUXn&60cH%O-F2@Uo3BL|K^c7!gK>(FS`vu@VJOC`( z2ypMgfgoJcQf#NSZ0`xw=)(hj{Ur+qzR9J7rt{!mTD*}{7x~Y-?gjBt|Ltf=OC6PjdW?9$Oi)d+{b9vhr23tFvq z&lwcZFFJU_8lFR@BC?pnPE`epoh22xryIUw*^ortW2gix_#z_%Jj+m+0+civF!Rpi z;i5uQig0Pi3)-LD!;dlk#OlEy*9E=1@aGu3t3`lhiLOo%|X~JJZ;r5 zM#gNA6b0y`>>ns~G%%xIZE#m%Qv*#AK~MJc_z~j|5FBit9AE4@I=<+~1*~cJI1Fx} z(o!qJi$&UcGRM$lCQ;+XzQ!3K^UAm)8}bqGM1tTW0?NgXg!JNet1KxFGvDYSapr7{ z(c|l%)4t<7n|=$;<%*uIA^zVC{p?LM2BTd&FKt_JtLU7*=yloGcmLdd|0xYunXP`W z)_cm`Y$~kCspYgaiiuR3)HUed$0H}PC# zp!0bmmo3R@l|7q(mPD9Fe*)O8yYj7gpfft*HvNNpaOPtDXvFh;R@up`El*3^GW<|9 zIqPvg9&=2FmHI^S0`Zr0;@m*AgDCH(MffW zQjP9Rf#G1d&Q4rwYe$D*X58LkMiy`y1H`1ZW>@{HS@-66&e`h3O0Sn|adYX&@tZ-2meT!Z6vi1!G&W3U?2{oskj;Vv#NFu(R3+dj2^`oxoQ zCf?8er|;k4a@(w9y2LQRQqSpix%>O~Z(G-08yejibiKm91EGp(e{1qE`f%s=@bACZ zAs3fjx4o-(SNcH6s6@!{*x=3R=xAKjnn#DhTwI+1BdPpetBcn7ls7jY)MQ?)+=YRd zL*1TR6dkhr&x-NQ&8_CfabgAk-F(%04Huk+ugw zh~={$o=;Bp!v0F55YC70XXd+JTNFs$YaiMXskr^qUN1|)Ct-2QE+s2{c_dFnkrmVa z+?~)=PeI-Jt@lpHW2CK@B|a*hhGTPwe5QSrg8SSL$vo>hZfkhK+51m*LYE5rLoOgH zJS^rd?3zN7qcBSrmcLeLA2GA7T$zo>gu!vcxuKbt1Q06i_rHlibiudMpl*)$p%0_% z&13njM_~Bdw6rQ2-u;`TAGNV>JG^SB*c&}K>py8aV{YC0ePZcAa&WxOdi#6%ddV#K zFB|mk&$a9T``48Z4@OfAt-_7;XYoOzWwcPxANJgDosR;D2qyWcb;Pg2)#FX6Adq-% z4TMD(BNI#qrH4fR3MC1PmSI_{aSpJuQ=7<~m?zT|@YkOmjk8wVwfWUL;~pZ(+}eqz zDxlgF4@V%!H^Fqt+72w_!@3DjF!QIDfI7%*%Q}awR099Qr}88+Sc!-$tHV88RM9U zAhT?W&Y?W(v@vdJJ#dgYK7yLTq1+j#%K+@-zls>4^ai8?;JV23Y*|a8HZHvwUf$@e zOO&bxU7vOs7+fzn;_w8z&6ov4!U#AJR!D_GfpwS(*v*J?^T5jLx|miw?(Jo*a9q%fk}Hg-Qxzp;?Lq=L-)NA5?+ z+w)~PC8f}$X(=EyL;(mUZ}FJ$Hdz(#U=%koH-n|KXP5He_eGwjMTc@v>xA7=N(!T(L*ZU0R!lj1 z-w5T$@9jvJa6(5SMble^Qy&HRI_7%rYp4DR$||pI$vRD_4xxz2iclcQuF*uF!m_R9 zyGdYHTHgCp{kPscPy#FIgfj^tdXC#jH;gJ)oO1QdWF823arDCEx45lPap~iI)1?!;v}R zJfGA0yQKChOLXTbIgceT7ietXnpB`dX`08;HXdZ@IDFY7P>)E(!ISK2VxdvAsmc7i zn>c|zMTrJyt%-7>)-Q}GS)@5%EPkGAGz7aId{-yi?p6M7Kr|r{$5EGiY0Z{B@y3o)3fuwo@Y7M35ixkQ|k(6BpJ= zV(qPi+*M7n%5dM@O81i+pFVq#BOgaenEUrV#&!FDBReW~&$2J=clxd+=U^Lg9*Ch| zbIPJCpfV;aEHNaFo&BwS28xqKFH3=>Jpk zZh}2k-=+s}?e|IEwOb|vhb>C`6zJJH3~L^Eo6z=e{e;pxpW02NlXCmkE^Z0Ooxa8k z&CdHdO&8o~WCEzL_o}M4%cIN7tvt!t0YPVUjsEP+jvgAg0OH{2-q~P*>@!(eniiL? zyFY`htg<11Il#iL;?F^;tA4Pq7;C=GxsFsK;RkYNlW4u~Pfv5EN zsf^UATRJbDt6tGw)4t=;Ty8XOV~yDr#X^fbjcGfn{{*R|Z=YTD(9f<}G%v9h@xt`Pr2?7K1yUr9{2@=*Jxi|CU_IZc?vLHGazSC#`e zjRoh3I*)bBm~cI1cIIE{vb)`lP|whk2>ggXWv7k!Tx$M0tb+>HUDEKS@isdjo3$80 z@7Qa2*NOi~TC-mwA%^F_@l&l7R&g!;Q?KCd);a&6$qBjG^*P2b{z+0f1jQ(NzBh5? zvM*&|gjW64x14-s%7(u8F(3e!L=b6K3-MrTO2}$%4emN#BrvR+4#Y(C8nKxwrts-~ArL zY&t|YwS+s2IY;OnDG=NoM?bdU(dATx1@dIxi zFJuEQg3sYG)VE=LUIJ^pa1Iw2d_azUl7}(K*p&9z=$N#Uu+4By83N!EWA=tjmJ9& z1_TFZ!0gG&ND(?r$$DZA$-KLvqMrYaAQvp`#_wHGb9%72^k+@HOOv|5zFquYHzeG{ zV-8Wkj|xvtP9TqQhiHpYE@o3hIT)$DZ<2JBv18JYGEye+h;uhLk%YyC*hT;Zhet|^ z7Y%U=AK$F~!Hl|WmGKR-H4Fum)^_RL`bTgCUBbU{8uN&u&wbp&#k@+hcpx4l2%Jz? z?4G%eK2%e5L15c`@czT1e?l zdy^Gm%JcY>c6c(o-sZV&;0`ESw`=J@cs}RMv9{usmO3HRoa+ZywXK}JJtY$nZBI$5 zNp43td&Rx6^-}OVh!OF`AP;*41Yk5J6W|sBBi4<@$hP?x)uiz6=`1wlp&T&bC`vA~8xRfY_v?hv#vB_~GLpUxreYVG5$r zoaRm>=3_FYoW(7=(yZ|eHfdq`I?kNt+&kq*o^yMIq5hnnGI|^};(eDTc?Aztte_%Y zMl7F_zc;8^7GEOSshE_8&nU$S=rHbtGWPV*JeF#a(LG$ET-;k;Phy9r&J-mKsOffc znx2UX2W5sa#MSwT(J9p~8XK<|SMf1&*0M)YA=ekAnp+k?sTaW}f5&HiI+ke*WS%{H z)~NU#Wd9?^WWc4$JOal97Oj$~+C0%7Nyc5|Kf*Ig;dwGbClB)jm=So3AUY)N{hYZ%^iKCYHM2v{)bkw@OdTaJ#(PN$@)##^EU00ZsJ<-)aG2R2d{-wE+YHRmqFQD zR%99(T=Vgpsr-CX(0s{)w+13&C54^OJRbf>ktu~cjI(G`$9uE5#>K^Dr6W*3twxHu zttK#d(Zr=W$~MBBRwwNxe|nt&<+waLxe~``mOfkD-qIno_p`hpIF>ie9$FI=woeE` zKC`=gnh%*1mp5|$QX4?ji>U7dK(NSop|=SSV;yJ}<=Cx7#LaaSkm}6Rm*LAxIT8o8 zG3-$?4_wK#b(L&uUQbiKQfKGvlb&WvdvO_hA5Wc&>ngm4bX@y0QvHpH$ePM%_kIkI zGJJ?9;3%%Hnh&X0XkTdTid*$q~9^-zIW25Ws8v@=h@)$=;=)PZNrP3&8$+WxKM`@Fi}LIJ*Z-Uo+iV9Z>@HWor;iSpYkF=w$6_42RrWiO z>b%7j{IOs3Yh7uz$E3CKks9~INV1>RCwd4Z4R)t54lV>wYelFAxj^HRrGv!K-lV8; zYplM%QqZ8+q54*Vo#Wgo`NS_3kEOmb)8`E$zI_;7B!afLEr>=ZK1xB!Z6U&Ie}-*m z$7bH!B+yYODP_6e(z|8Zw97_NesKz#Q3{QP`*@6P%l&~~QrXgZjz;?hju)x#mnE*%Br7E`q-+#s%o025?q~Z23O%a$J)o)AkNmnj zI(l|qrg{RTjcLdln|4*c9ObF3+1uM2t#hOFEeN^Yp&Y!qUt&cPrEF!dk(3gH`Kw^b*-lF~{q6aoBq{EG2s`UrON7+w<&FZ9bXdyLBNRL45%w*~>I(yP zEbZ8R_JHw9v%@@O-g%aeF42FSYT*)Dmn+BpTL=~YY*>?(;K{qb zeqCSprG*|n0dY>?!@g1v1ts+XDfWHJ=YuW`Iw>__CD8bRnwZ|+wasHUC&jzpzvYHj zu1Dy9jV6BolKE0lKRAt(|2tEEE)fF2;mKCA$=dp z<|v~2nIVf8pZ6E1&Re)Lh;P)fqqAb zG~??Lo;Wb5MNS$VHeJbw|5uuVJBclu(e@m2PPS0a1NC=w2)rg9{pla#CzU_a?-A&& zx&rh&EiD!NtIg$Yld{2Af8UiN?U@2J@Cg$=%|%3?zAL;v^gmW!$CCnt}OzZjF`X9g=@fub+wvPBMY>b@k*D5WA-uadfl)9P$sHO?f{ER% z19XXAgwe~vzgjJR{hWI2(UOpLOw2fW6cj_!A|Ym%f>Y&WI?2Lf2TIcXEz!?vsi0IM zVbmgc$y8gnmD&9aRk(ob^Y@-gn!$3R*%CS9Q)U25&C5i8Dj-+y!Lv3BLleV+zh>;p z4sD0P(7A;Be%6_)aW<0nvlJyVwA!4Vi-^Nxws^%!JZCRQye~TQQReKyweOju6u8&2 z_BT=2J`k1_67c%d-qJf&k2F%ebRe*SmZ<(aAT2ju|MQLi`HfS`0r>HC`E6f|e%4zHDkhMF$-EgD;{Z1Pn zWXSHkUwk5VT0ezcPO2&<`QW#9snC~OJ`VYI*#=fD7>}^^ha!l^wvlwj@k8n#YkmFz zqYa?&aOiqCHga}C-*tJAL&@6YTC;dvTtVhzvUL0BhV^ZE6D#q9j;;{n_b>OCjdv#U ze{XGZpD*utQEQ0n{2@kqt;o-zNaXj^i9?!~=qAjJjRi(me@vc4Xy)D|>E_T=6Zsby>8N$dEr3I_bI-sAgGG8sxYYrRaNoNq3v z;(7u@Y>MF&NLmFbJp@irL`p%$kc4Mvm|9)Kpz!M&u$@>M+N*c>{mPP3(}^s$#zPRo zb?HDSrCpys`JJ9;WbCyK{=f8dcXvPCO2Cvfbr`#HMot2K(!ZVPEiX>kbQ-746eo=h zb1JSF#{l_keMprIUbu`2nfrQD67l$a!Y;oKha353SHy_m9~I#`>pZzN?jgI9shOwO zm?;s3ponSNEKy%qC{#7TU)fUi!|Qli-d6hLd?{sbz&Pj%8R$rgj$+5h@CzLs977>c zq##mRGDwZ7PmUS%uEC)wg_EdZh})V}!=SOQ(A<(|SSLU2Pt9+Z@X^FcOgx(cH#0-~ zP?ECppYMMs53p=iCDc|<*Iy~wacobmw&O|}%-4d@G}pTGGBPqsQW8pf1w}V#m-Yaza%?u3-2R@EIWNp#68;Qa>xNq7PpR?J@z0eO{ zcL;tyFE2a5&rDzE!w4@G@qH3tekk{)XIkw>fxK3)2G?85yZ3WzIk0P4;M-QT?5$+( zznNn~CLv+rnh!#YEt)#bvVypj>5MomWA7BqJS;<}YGdoK6gysMCt3eDv^&a!UK1nvhXC{g=V7$5WB+Jq%neMgE=tDW4{f9+fX(rD9bFV0 z2EYC$`+!`=`#sfz6#`rzlB_RYxl=$(QO{cc{rf*Ug4b1)|Ke7^%jTXB-d=4TL|+_{ z|Gcqyt4M$U-0Pxt@!8%Eq4V~*S`9=GN$of`jQ9Hu50jNq!=XVYkVUy=O(uj!=&aoaGhEbD5MZKkBh~$Bih}dKNK!I3j>F2S!Gz;Zs zyP70kHR?PDOHXEgUK5oVXc%kMFkm+%-`cfplB zT6j@n8M^;^l>#LIj3kQIZCa}7`OSo}HP3xa)Zc0e7qA}YG)-z8hg=-pn#}Q^t^gv3 zCwY2jUQdquCMiI`TDX4kPf&<7iwTOWvFYd$L(T)Fc6z=8#-YVH+VdRl0PQ27);r72@AmnDwA3g{R3sd=bt5MDv-89FW%Y5e`8*K;4(^AHj!5+Ken^)ZOk_oz z!1a_uM%ZUuEBofQp!2>uA@k7C#Ajrt7BZ0mGDJ&~TG>Cv5vZ@%$^4BNB$oivbAK&M zC#s7|rsdkmHW-R6#DWRYBJglbKhUMoy)=xQ7NGRoY*8W_o_MnE@>O}x|DaWIOL?~! zSbCDkQ)T&gM_vTC7&yIV$DuT6qdL0+{a&xI%E-#(VO?sCZm;IME=Rj=_guSfeB5S@ zNl>%KE5PJ{@o|96%RD#*)2}Ho<~hDs-&av+9Zu~zFz>W%6rg`$Hh+7$x%rVb>JU`x zwemEx(U+80ZMPS)4v}AirBxGg&;@sx1|9_|X43EVuKfg-DCgG}zpkd>9B_d`CeaGV zHIrB(jHH=@ zlPVaC+nA(WPl(Qv7a+Uvda~I;&m|aZ(^4_6O{{upUm2znM!ro}lR1C15^}oyVXCcd z@*;A(s;a7|C3AUZN?aZGXZdvIO@xcRrqnAN1!k^6Z(_H2<&YEF`rxEZKrC238K)=p zYzXySh!Nh^^X;H|pWF`kOjf3ip^maDD*U-4VuV!Gmn9?ZN!0r`=z5Y$sLyU2SuMru z`ofuT>|v^37%ah1)MsU^uO=HAHyPuU>M>^|_nbyvUR%uJWtTY@gmf1UsW2vM>0?Oij| z*0e7>ULyD)vBO^_dl<9S5o{R}gQhWA-XMa3dN126ZvPVzu_Dy=6}Qub2HB1!bTOD35`^Z9_gq7DHFi_%Wa`C zC{t$_*y$>vXh*pqlOqW&oTGGy{mqB>!x=s5AZ=*M(3Hv#Fzv^$lk7xqHXB(ismMZ)}I z(_g*i!}@4a!l>jX^0;h<-L_S9)ZCLG3R}$Y{y98Fqhmm7i_h`cTi7|@fuACYfS@dq z2+^0jBqSPwoO|`+VfR(_khCXy9OPN&76m%-wL9NtY7cq}L-${t-)44Qon5;E)~3VE zGmq)$1#goN+LGL68wr-g#%Xla_Z3Oj7JN3yPa;9Fd>{WaadJ&QyiYm)`?sGYy9~6r zI`bu+ajNVQs>`2xPPRC9E-K#CVIx$>3J)O(YPU6rIrDj&p__vE=j zb!OGLV~1&t_$Z)o%Iv+}^j`(DtKn!H889TqR6`aaLj&oLO@%dz*-pz?6Am|a`axas z3!cQTwiShiqA`B#P!6mW0zO>ft=kJe!?1JCK!8P`c)8F3xJow!#r07fOpF;)HpNR# z5ym+-wgg?TjbBqLwJaQAYaYHFeP8Bw=XLS#U+zzB$-}l=zXk8|cMc-M4Zg4{8<3ps z*Jwsyh%x@pkI=8$O#Q6pvTExTuYmJJYOAE6S`QXQ_QHY`fHj}T3!6S;G~qE>`IQgFJ4^=W^N3r=d9Gp@v!Xw1S!)Eu zz~27c4G8o{f5@O;;wN&ldfghL&f8Ml{>ea4VsCqpUgn*6K~9sQY%=#VchK(@*XIAO z&-F=vSxLMO2#7&NNo0EXP00GpsiI*9+N7(cGjvzjM_5>^;OOmjZ;@0f ztwIM+Tn~GNq3oZ1RkR{laRxsWK~d|f8BKdn|1Yo&Dz0f-&KaK_9n|t&o{H;Xm0cxR z^9kHcSq_hh*Lch#MlMP&|ByN>j1QJ#W5J>KC~{o+;mA7U!x?==1c)1ZYA{$q98s9G z_$Ymv@SPgMpH8yhH1Lfq7Gz~Vecr#v zNT;YAKQZ+EHRskx$EeJS$B#3|$|+%62n?n>ds3Y8Y`sDgEGfbVp-$zej%M#?c;Apz zMCY$(%;~7cUCXK`*q>Hxug<$`5L`e3LU1U+;A5Vbg_uIN!eS<_NKPCRbRbEABuE{l zqy7XL1HIlV&+<~CE{@;?S}z=~>-Kcj*x2}cXCN?)SUQ*;_vqjo#dbUuH<@WPp>?al zW!m~&{Qc9$8oZzvr*5C3@4)iUe9IU&(ST`5=pWNR>O>X0V1Px!@yvQ0#-OgrdA?NN zzXgTgC|SD1J<6#XD7&n4UDeq;Z`LxR^YVC?NqjB~jzL?kRIlVi4-37>q*2WS7=$ z=arWxb@gul86aA$S~@s;$QA6rSkTHsssW&0SuZBzKbi=>{v5Q#Vw&I9ncd{W6nFs2 z)}Tj5Lpo0$Uls)|B{-$XWrh{>gE>kd zmQO#b!WJbMdp>PROBLwY=0f7U)U*sMNweN>>Ac(ePd}}*dWdeL{j?M3XqL{p(m6~x zxv?Ma&%o^Nh0KistGYPAWQY*cxbh~BL=FvzUdR>4OI0yVvp`Wg;(Hqch5CUPcx%6 zqgg-?(7dYA3?iv?6*dg#E?pcwbpnXQR}93YwOaTSwZg&2Ijc2InR!_DyPS|ySMVC; z-TECH^*8ZShG!LzDa%MX`C!!V;38Vn?b+^3z6L8SqUl3!*d(s;>=G(`cE05BVoPrO zA=P$#EC*vDQ006*XBUbcL5qnzD%sQZsQFQc6bk*gV%m9qp1C*9RkzBTv%TDD3ha$P z__Qq_-5nQ*lee`m%z67uvmD9X9Pg~5IMeeL>D5y<55tG8Er_c6b8Y0dM8pvFt<@e^ zKTiB|HNT~L>LkA_V}4%gn883C+FAqHTuc+clPdH9bYZpsRdgMWQ2u}Xh$7;QG7|1W zM9#`y8D-0+uR~VIh_kbjarTU}%kGE}G7i}~?g$|viOcG6adEic^Lze-XMH}O_iMuE z-pxHNsy=nEsYGitE1D($dYLO&VFoH-mf#I_c9hi4J({s(VUS2J*D+ws8SY_)JBG4A z{%Y^IJkCqiZE>LF|c<$VFCu$qDf4){TLa0%gvw5AMQ`2y_CBTez zpl@6-T)D)Y-DWmr);uZgvRRIhW$uVY*V%d_^^khJ($Jp&qxk;u#LDUH*%_v*m$)b& zHz{WfR3r~}HX!aG3$W#*PCQU^eMs7*xBJmfIbyuHwRJeJiMQ4b0)m)`y&X*DD7(ti z>IZnOfKZQb;4|mde5^yz@6MY+K^VGVd{?C1yQt%B4N6__xdLz$C3i)i43MzPHtQ6> zWmf4^z#HM96Gg%LskMn!Es@riYD|G91n>0c?zw2%SIj$q(+ijqJnvW3ZiI8OO$ia? zN!jVgNhRy{MZ0elu@aFCI*638 z-zNjBg3>_D)Zl#Z6;rR^{~Ym)f_{z6Ro9o8rbN|LtGuKcv!!JV=o!x@H+;vMHlzOS z*8uP0a^cn?z2ZvZOv-EW*z3Td-NV_%kQE1b@AGeE=X)gT2iyA3x_6)&r+aE3OW~Ub z`+Mc9!c}q~;Fr(B?Ii1-5=L7{@fG^2gTuZ>D7)zU3eo&mu2^8iSp_GU2OZy6jmj+) zk*$X-T|=Y|nu?`Qz*F{r#+x>aS5g?v6Aa6}^7`quX0n(HvSV#I-S1bHAM6YyIWvg9 z)vQ2i%_+5fQ_@#!A&(En%P?j0CO@{L2r}lBYl1xhutxj(m$LzZaU~ne?e>)jTfcm4 z6^scQE(RgLD~2VFX8zaIK%^0y4)RYJdJ&>)M{BTHp~=ne4=tDOa1HKIMP;=iktk-J zxHK?`MtI@1bffwk#11+jt9+O{Q9S{&-s5T@mXT_Sojs932@x@WcS$v6#9%<6yfk?~ zESj=`{_9cR{aDLLr_dE+yWEYLs7I_&)lpswf1N$e=0;t|v>5Ix$)=iR7dpXy_YKts z@0^?lnPg-%ppOzQr2kU?u~HBBNO0mF<%w&7}6| zK4`lnJ1uwlrS3+SMG6C3qLfjhfrWujh*x3t?f`+(g*RpWRBzk(UX<)J7ykoeD%KMj zLaB2+4{pJj{ZjXsV}$_BxXE#Pjhf251XuZHsOqpUmzKswl`2?78nQOq!cA6MTaNAe zexvl}vC^177s#V9RCfa*^F+i^=-eVo=7F2iVeyN01WQha#R^$(9b>xX`d%fBlRbQjTgHi)sEVFJZ; zS{e^LCPx6%2}1QxuS!pJ$n%y*GvSJyxC@#o9{LTEu&>*#Pad$8@$;Ac5lCperxVPA z%uXhW>u|Pk8c8t(4@;u&z}m`BHm7UUhngJ*D{C6g5&@YeE4Dr%zq7$2>pQ=0>u#{P z^t)?V_k|BW*Au0Dvm;+x+!0sklKHCLOe)+1g|Q z<0o+p%^MM8cM;wPqdcM|Zo5D@-M59}o#L`zx(OYU5&VIl==8?yJ~O;O%MMs=r47A5 zr>P^$c5vFTW~XH?E606jlEIS;R8#o0t1`@i9R%K9--dzc#EhRVDs5E)x*RSZe9oVRs5wYp`#&>gPi3I}@6 zhoFgX3oM?7nN{a%F?)&SyOeue?p$^VbDsLRq5(uN2Kopjkp28HFfO-OSHgaWPknS~ zc(@C6B#&DG^o}b@;!B;x5r-wj?EdPcQcWi41$0@HV*8il<++ zDlMG?$fkD697+L(b9>^VDn>5RWo~wHBBb|pmg3iQHj&86nt`WoublfOx?C^6;=Bab zW&pgDVd%?8CR5p;L3t>Dt48FgGzV(%p2!QX364FNTetoTqXW58A0cQ_*cwE739+%L zof6TzTED6&aJ_h==^(&0-^*grwe5Awtv8;klO6RO+WPmFO+6rv>Zl!q2Kqd?JC<1;YiVzORerB7Km>|>WId{?i34rc@JnfizcQAP7;A3MolA; zbME6-!r*vJt>$addl#of{+6jnMv-?DeHqv-7F*kiJ$5s0%xWq5zLTz!;x9Gbs}vMZXqNf(b%3*(-;_IXeFwhkAD|&-Oe^9le)(52cC{ zZu0nYOWzqTkRWs~W_x{Ty=eW=r?rRHvGjzYoUU3k$$o?Krux6!lwvGn%CIXMIjon< zd4+p~W+);GEPLg~xFiV%{) zHBE226%dsI*T|ZG(^6?C1myr6nxe>Qb=WeZx`Ge2i%N`E29Q85=338}f>4UlKF7&d zNPhYp*11)btCjoO)12e$v*dYcWX&H9r1oD4=5oRT>Oyt+9ej*wpDBE;*Ymt?WN0pO#Vc zd*7kAiikaK8)!7MP@EL=1vb}pw8C6|@4q3KFTgkfWrD!JrOaQ{`StSTRr=Swu}A|0 zvvrNrMC#u8BCCy`Z-x(v`Y-xS1L(z~p7m0ROs+ zom<0?f>CeZ2vr;9j_MuTxaTFwnphP**3`)g^(v}4`xDlCJ>iWgx_Yzov1ohIV!I!9 z_JC3reUi9xc7DEpY*36;Cd^xvgDYGUdu=6H5^va^Z98JRI&BR+zFAnzu6woOuV93iXQVHer{KK zsVuAFTRC_;)^&= zdZUF)tri*|&i2@r-Pbx7ya7J3pihc9@M8O?7O!Phu1uSIxIpB_VPT`UF^<}hV>w~( zQJT>4V69&@yJ-EUbkUpB`(f|gQEv-^E=mtiKVG@(WA(9rpENxHAim!;)Ew$<^0%$* zLS>Ntiu^#(Vv1ZMP=R;!;n56rtC4!(csAkaXM=sCWSg4$zW)3A(1v^09aYu=sSis7 zrHgG7n46;mTGGq|uiZ^eVH)RuN1~}j>KQSL3_u0#(f`dII-ajPF6Zs84+S?U)Ou}b zuQCqhw|;aN=pxeg$ET;EeK!ogIS4mIF3AFW_23mMmk;DHx|t_TU`ox5!{8!{@vbt8jNvx z!v5jdtw8C&6CVEy7GGWwu05%xF%eGgkW74%1%Za+3FnX1**lzG)!E9fsK~baG@_5^ zl#1}!BIzrnE7%M!GlCLVck>S`mbz~P>Qyr=7?*<{O$p#=X~y{`5T~RMq+=3$siln3 zXG`t*+gHO~Wc7m;i+@U4>Y9bRZ;%C+(^4|)nFgYIqeH`Yv@>+Dhd+OAX_-;v1TE}r zbgimG=WiU1Y{VhsNnt$NbA%s7kf2&aF-DBhhW(bNg6KGG5NkY4!ugRD{#%b-3T8(cDaT=MhCmx&3$MVsR*B|uVN^soR z_jnN+EXbP(KKVMHTt4sHo?$iHZp=rEUidN!N&+5qA4;}7p8#OMtlD#i$msJT$MCp* zf9Uolr5lyc0Ofi^9<~$dW@n8}TZ2}9{3fv-6scUIzO>ZQ`7!(1PLXzqxi#YJ@YfNc zi?{-8%U(G{3{F_ql0y!3;V$T*1yqaUf{qhlR-vO4)_P5wX%v^r3{EKrOUcrRYt7N` zcHQe#j}sbZNwO{qGLi=Z?0!$a;%6IL(1!?18`6#c%9@l&G3s}HNn0LYOM$|@R2iF2+!fj4BxBsCRP^UszkwFV zLcoaAF~@lOBDbotCZb?E$gIXiIq2O@g}9lx^BFR&eWQX+dpm%~7c;ko4`@xH=HzM41)X>TUO+=lRgDX9F*zNaPfUb&%r{(% zhuzWF`+sqVKt|Bhlp!FcVg^euJBR(bbWj3e@Kj*N1djOnpYvAo} zf@pd%)Th1316)yB_~{2~)eoe(p_83VyMaJ){2d&x7RlD3dk9|z-x;EdA+!fA!wF6M z-E}_B7fSH>u(6N>Dftr)wTeXS5jR29L#8aE+C0qyCCQ=jNIow#p|sM*2N%e1Z=Ax6 z@DDpTs#q`)L&+cu^JZh^_&d*^AxnY61$mvS@<)D>zAFyjdmI~V-u3US;4tq>r-B!F zx{gjU8x<6nv9{*MH!b&)YTqXfEvAj$}?ug*2W0_ z4Ct`8wBkG22-MU5h`|`Yu`y04ww%5+o$R)8gXEu>Y_Rycy7@| z+n(x&iKup!-HiP1aW8%dJrN_`EzU$d3+X*M3L|F=W{)hA{g$aeBMpG*Ji)@xheOzn zQ3z^qTRZtTcK{^fQ6e9!rnS2KD07{i-dNu-&ExV)eFUBFHQ|!ZXXJ>}%|H8qn9)$j z@tIvm|8*@Z>)F`cy?MDv??g@>)D%yDXpT?!gP76OYC4cMuMT@T+e_g)5kY_CHfGhy zQlzwmdb;rYn~G~)!&KDSSpAry-`5TgagMjF#=lF#x{P|X++n$4x2jQB13YLzVOvtf zBzyMvj3D{^<&BBUASfG+g#ozYmIV4$<9`PLIR`0V&j*Zhv%8Ba+X_7nbOU^SeL4p+ zet()J2(74la?oszfgNSs*}-vV@w>yP(VM5L1g-+oKUFIj6pzO%seE?TQk?xvz_FKZ zs@#MsD~u#JNZib+tdwT!cgNi5RA-6Q`;5d1>b~95{VEXXX7AS)xe_uma2|TH2-u74 z{a_9b4pE0G#O38~F(%??H%}*NluOdfiw3XVv!}KF< zUE2?hSReu(f#9xp@B%_|{vquP_QLB!x!{7N4re;hxW#-2PuaRGYATDRx}cF!Dp?Lr zohEFY1N`xEC@<}En>;Iw{oFPXM)f5}yiQrzkxi-3RnuT9OzUSv&BLpx;k^fG#x4wz zDu3(bLVw!P>)`+W*sY)4hm~@urW9=)Wayprmqi3^1Pm)Jek;x!HjhN zGiP|P@E*3E424R^$VkcG(l_@^O0yr`(B7t`M=KmyM``7oWIh*C!?dI+S;U zC!)yDsK=y%>GNkN|40NfaV45`zFSncJrpny*$4?9dC80sPbrsx+r1A-pK<-!v?Si%rk8=nFz!N=qq^;M zIENA*dlUk5`$Y%KQ_F6pkd0%9zGzC}M0xG_n?bJgU#fg_uoO7rWB+t!h`|^|W`&}3 zh^L5Dpup1Nm-=S98@B_}^*x;33rdnoX~uB0*EI)oEsaVLJ&zjNKYBk8=&8?NKcU6) z!Ws4YB;%V*4rd?tweb3?H+0-mEW;}u;~bwmhk@ZT)pE}8B4QjW!#tp$p%b&C^`$xmkSY_CKfkjU ze-QBvnBj@-b2cM@(BllCSN7HbpI83Ce&c!Za&I)Xt=2hpsp;w8(PgWY&W5KpoiicY z4A>$px65y3y{rPv)fccDP^q z!fh9ma!}T!5LsC7P8Ni&RkGTGE2L&)eTL?>W4)?ZGwS7C?FRJa;dCF6)N9o|u1_eD zfku5!&}NEOgx=fhC+f^Q`o;kz?y1bTB3l>I@KG+zSHb?b=`uo|UvBR~hpbZfs1(Y= zpRIa>b^EXEkaGL<@o(?Zg;fiP_dp*fs$jpL2Oyu1os4Q}ipl%m?~HviH}MR3BTsfH zM(9=FPW{#QQ|d#_MPn|j*&~RMS$Q}{jLYyDKXO53i$XGrRQEd`!oSmWsFvDP8}EUZ z&3aSzvC2A84(i^%TRj9Y*4X`S?_qJ*lYl|hvYe-k%M^*HdQWs{p}<(daF3DD?+?$@ zUFBvRXGzqChe|yM1mXWiGyvlic&fZXsSUfH&Evu#@+kZ(C#CwXvC+K3F2PhoV>wb1 zdw{3z-~n$pK)kqDRS11hCXA{;LH%e+AZ1PVrqagz&4x!Ej3GPadWD*5m0}aPk#QNYvz(rn^-|aH0s>t_>+8UJ zMPpQRgmSI5Sr5l(T?FY>FHI83l#!ZzAPA<~^CYLt&+#k6L~j^gKi2NH0kB0Ont+|F zqo>Ej2c_}|;d>l@8wX9_M{Pb$GIt#WOCyGvo)gtbQp^=mr(uDm`9Te3xUmZ+r`m*! zvWg>M4d0bf5H1MAnc_6QM@aNuUa>G(@7I(u!(27!`sB zy3gPnNvUB`F(vyxc(WBg|A|+r$1M;bDpl%kY=EwK+3vAPu}wkJsFI&WTTYqGf5l2+ zMbV>vS$969bvnkf6eED=Orz`CWNVKJ(_Uuvxdly!n_mzi7#`>`kM_9d_y0&Lfk`Gz zvI41B0vbyKg`aw#BZ68nogI?N&fEz-o5;~ulFqyrKu(iF`iT$`cUuV>5+*nZ)7?Gz z*#;7~gf*0>H{Q|G^Myi_;jSC~S>zoi0-q9La({^=dqO)&|0#cZb8U^Zggu{R&i9M% zSn$49=ayaSd`m1Y^CZevk}Kh}{Ff~0j8upC2N73A1h20)ldcSb=|%r7Uy;@|JCTuy zZTa%G`x1_tO=Kz!GTZRUB))bl_#MW9XZ@m|Z&@wgK8)M_Tl{IQAeJ`5Znb3jOOJ9~ zaCJ`jdG%U~EySzZYj6-}^9m`dX=P>5kOB;i?6zjF;zOi$L#%hzG}v1l;v);f@B+5I zK%8%Mwo=^7x2_QmeAgGhaFm9puY~e8q4EJ8oXvnt{mtHzBB6RP+jmPoQ?c~#pF1Si zT~c`;H`%K=eodL39$1U&eUO=1USMUlj+)42gj-$lA>4TNp6+i#U`Po0!(wDlbUB3r ztnySZJ;1pz?!A-N7gOZ!#1gfKW0O@eU4AZSaK;s3jYT&^JVwW%uAlNeadJ8p9?!c^sptp4SPb~cZ>ZrFhpMaW&lx3Gvo=O_UI~t6qgwjI zsxP}3h{cO4tr?9}#|K{xC)x?~?j6sae{P@8O&qUwYvUt1_Q>2Kt&TZ9!8qU>Wd#m) z{P0UBVUNS3!W%l)9~=z6$g3dm4MCh^-@<~e4Z+fcVow>}BR4g6D+BYC8pSw>u$v=T zi7iN{s;>5-pmX0)f^TLjQ>AnJ;@Q7{5h1GQz)=KH0JE-iclGoG0o)O6%H#QYadsyN0)`MW=wuHnyO|HXasJ%G%lV zsv@%S>rnQ`y@tIW(YTqbr5EZ9E@y#b0`6T?1A;lO&C*_}Socz|ML}z0FC)d+nnGOb zKbuX1-)(9|?tEQ5>~6%JF`gSPbQpt|!W4dbPxlHb-%vZZ3Ff*5p;NWc=}Y3##3$=I zM#e7q03wUB9dGB=`V85dZn$lC_i8633Iq|0g*xZXuwsx#mWJrJrja?x5w@P^QUt=U z@AHd1;uQN!MFL98#H0`xhwow;yjQ0Q#8-$6C6&q+CN^0I+nV90EbBf7X(v5+A3o6N z0*jV`n5U7vbl9y|lL^HKZ{H?CpqX{~+u{KaZrrclszjPuk(^>-hQ{nUpw8KoSa_3d zgJ0pse&R~I42Z$#@$b;0JJVPC7#(BVfISbOo$Vj z4NmOw5%hscW+E(@_&Pdk10R<%?MLQvzWim%rt89KCh)N)A^(8ci&6yX)6> zx{AM9P2!MAQ02G?(?|`|YF@N08aomAqk8nZg^0boQ3t2)=W3vPzPW}Ox zYCv-(f}Y_=oYHEXUt}&CkF~dPh4} z3oZrv(~AG92yie}Vg0qyyxje_<;@sPyYUTtMD|!(&YkhpQPMsWnx`VmFGA1+qZ2`pyw-;Mr*enY46P zF40y8`0~H`e&yPp7z($`6Aldf3tqA;!NS8Pis89Q{IaYAwJwtS6$S=x1Pb#9rU$0i zu={%-7cZ#vcH1@pxPAgwIS@j_pWapem#fZg9d4Bu1UKZSleqMpg>j7I1>{0A5bG#F zD~(_WRaD9`e@dJoKW5OkqRD#ibYIGdjbA_j5Qsks^C}qq@B{l)9ij@kfTxNRZpj6) zfv4iZ%_76a;W)LNW5JC@)q$O9p?dEp5T~|}$KC65W8pVbbngmeFVv&A>}8sh7TqEY zc#9FWfWZEqNN6ywN)$c^5VuVZSG)bwWyEv0o<0jZAF%QLrD{DW>#@LRyTud1)N5pe z+cr}UO2*|Mo<@>73U6XMB=QVb<^4<6#fKT5*P+o3PeFQO86$YB(lU9Q zxpilM|1ivg!!g?0Nif)6kdJjA4DN{J>*1eAw1%Xf;*-d_#Z)vz)7Q5 zg(<)RsX+h;mk9j)gO96$Zc1tp637lv5iKDfmK^R}OKG;X^wrmF%J^~Vp?T3MsJvEW zjC1HX+~*CLt=vsODhZ#~RBpme5o^{0U63MT7pNw7e3g|@LdDc^rXVyqRq^$XA@bnf zm;8KqlpA3*b%ML89t}-q(Kgb5;;No1>Qdof9$GAY8S)pt62Htf1eAZP{TAq@5^Bn* zn*ps+&z~11SV&HV+9(s)Alsu=H5Z`a?(Q~sVAp!>F_Ba_^6abz|9_&090H>ZoK0`m zxcpn|>BV3opEsfFyzFM_EgoCa(?PSme)w!`RP}5+okgY#61&`;hux6NQM27=& zC#E}+ViSEO4zk&|ZC$HUnv9k(FknaHh-GB1y1PG{8QD=PgE1eNn(m9z&1c{I%SyY{{)g>&xjtQkb)?Gru_TYUf5VEa&%#p0rsws+VOy_iSjaYyOt&ph=H%Nlm}c zRLeV&Im*VCFzC)4!;t$T3tE^>UKY`~33VZ5i#!}Y{zK(5UA4J&p!)Y@tJmG}%@enU z#&a2}%82`DhcSS5vnm)-`bkrm2qhjwzFPCC1WOZ7R9< zpN%8X_EZ+lUF3CfA-Z&q5#cns%PM8;PZL8{w;2mq*{BOjXc1MI=8bRQPFUUF_qVb+ zF0R#NsBq^6S*BzoMNLOJU9f{;C%b-t&}E`%X>oCBZF*qoYcXvx z*1cLB9Q@?{8XNs2+Yg(sy^rNCxs?dI&5L-(ujR2A)vNJ`^?^GOECRGCy@NFrHS8v! zZYlWmYlAeSQ^l%!h=^5w9GySnN+u&?sv`A9H~!{&DpLQ002Gm-y_4{wfQ=(6YNHMWOP}NLev834tBqVw2mEQXm6ehZDIzS4QS2*yi9>MY3al<5 zFyZ(3fp-Ght6>d-Fc_=5r4c-S*%273eXX%%Ki`#}4h-e*=K3Z8N6y^!nY(LD+5-tN zQAry=-Mj1>ulY4das{>8TE^dW(i6LpkqBT>siCU!i34IHc8xH}Mql~Gcd-0OZp*|T zB4sjZK_=laykJ*1bw5-UcLA|XQJl1we4>>ThmsZ;Wgah{O`7csxd6R>R1<5hpY;wP-`$j7S;x3vLs0%=l4u5!v`@ zg(qb{!sOtXQF=t<7p5@{s8va{14iu%_oyDG12b{?bFDsV literal 0 HcmV?d00001 diff --git a/python/nano/test/resources/train_image_folder_png/dogs/dog.100.png b/python/nano/test/resources/train_image_folder_png/dogs/dog.100.png new file mode 100644 index 0000000000000000000000000000000000000000..11673bbf81fa7c89692d354c95888c5ad0759546 GIT binary patch literal 260259 zcmV*#KsvvPP)00E5%0ssI2Xqgn$00961NklL zX_F*LjwJ{_0L&tyYKOb|74C6nR@X5#Q?%XFLp#04-`-FAeW!PJmzIWlrnb7p2$o#(lfNmX-~rfE7mJA3^2@|RzJ`S<_+-~WgI@E>k( zZy!A#h)C7vc{X!)n86fTIsgE%$Hlfw)@dRl77)cR05W%VcfcHGR#nwq+i7pMs=D5~ zL@XlAoJ1&I=BipsDW$|WL?j|9rTELZ4{76^^Y{qVE%GJo7+Bv}8)!odiYB96IRYbb3 z>-!!CRZX1xZZjSZUFt=I3?jmi-EMbtb2FQ7Hk-^bsFS%}E`G71;l@f1RpZk7(yB;L|GOqn}xy3Kzm3Oyq z-n{+Y@BiidAAY#Ky`Rcls?BP}J>{Ibo+KMwt1BG9g>+T5n$BhwuFS^dV6ZUrc5{|f z5=vK>kFunjo11$z*CIkz=IOW_dg+qv?ys+|9@ScFE$Rl40qSOEkWDNUGIuowh)9S_ zDb-X}4d5isWbUe(@oE^li;IhwFJJsufBjdNkGB8#5C8Fh{D1uqn_gfFQYI&Ia;tSN zr96N9#eTo9wRT-+iu?O}Rqy-W%!TtX4BPGbX0ri!nx<)*?l{+4RaHQ7+sAR7#&Nsd z3h46k^4o8}{qp6jd7gjwyWjoGzx<1DNJOT?G3Wf`@zwqP{qFYe^5P=r>}pU8b0VT} z3CIu6(a7B0yqY`6os5W>!OYcboQ9){WH_mMO5Nqf)uTsOXZ^*`KmW`us#>NAKuSri zOw(k3jK`f)5^l!EZbVeeJdWc$&txu%x~@wp-H#8pmuju6fv$e@b050?YgPT+xMQi{ zViQf;`?hiXheEt0Rrze(DGxS&h;nBj;$7&hJ@%sAu`ucjB zCJ`|M@URKFJ3z#qGMl2-slBo~XO)JjV&=oQa6N z-P>^-+Zv7yq1JFN!!X3<#Kk`NrC)RIs>(wj7jI?+xSLiuG@K*c+#P6F1rQi;hM5z= z-D$af%Z4o=g~Qxo3XrNhOx0aX37*V(-RfWt0ueDWQF3>6i*1;Q7_6#JU~ewT-4Pzv zTa-|ykr`o>?%MV{v)JU~@sYG31Mr?weDrFfVOQ&q>o@Vr>EX}fxiK>Vafa(dUl+h9 zo_&2Wa~#CXPhECAk=0qR=kUoMv!1irY2ZRw;-~dE>tWXr>XTReO_;=b@bzeC7nfI6 zpWNQOzrH?Bv$Gh_7-wN-PGN!O9uFFEADFwYV|Z~dfH4s#_4Zj>9jviQyAzpD^9)n1 zvz9qYW_Yc2cXtoK$wh>O0l2z>=Xs7$ruja1S1Ya&rJ=2Nm|3Z{)|#@;^L%%AL*zpz zFTQv-48!04?ce_J{qL$))hY~TIE!MhqAnMh-JDU<^)_h@J9A)yyF&=hW9~yZb{qsz@rO)H%)> zY9*e#sSy!5$N^`EL9M#y#6+BPmx$dxygxHs41f{?#RS zx`ZrZV?6lB6X(umM$|%nB3Z2b!L!G8uFe|2wh4s=FTYvNjXVP3?RHSrPBYa90g89aDw~dzsx~SOw7W}<{qnvLC)f86_w0~VcX8xVP@n;Yhe`i*2 zZ*R}f&xfHKh9Sc6T1)KFB0}_FZ=V>|YObfY`rxLCklTLi?q==oF)@6-x8#n+N%AM2 zo!8sO+#Z3pZ7H1ZsZ;hPx?g|)$&+$-2@lH@(M)?oP1QjFRMibhGhrZts zByyfQ7p=4F>{`(Yi7+V4HIhAMHn^(8su{@L2_(!yWNxOhxf7U~oe4~;3WJ%eYOQsi z-DR)iJSDi(vX%kVHR&KY^OBJ)>kTC9ePHG~ zuP_vdXm#!@EEAu(-t&MLRyVOa5n6(^^_w_{qDqXw3Ygh_Np_ro*k8fKM8w1(0}~uh z1S8j$cf@lR2SE@)L4$c*n8O9ZWqqJTtPk0Xx;dGP5E1&^xmv9iAR%^XxR#ljDZuOi zB(7{2x|CAnQfFtI7cXA)eJ_dk`+e6X=2!qEoM`p3<-vWgDb8wapSV0Ca{tK5#(gvY zm9(av_JhxS3A1?lTUC#TO3g_9L`dI|2n zD~IiN`}oP#<)cUW{kvk`Npi3eF(t6LyArq=lB7-|n64l-hnlJto2R)F@z8C~&MqEb zUOhU$*zFGEajrC5RU*nMJApxoW!rDNE`_~06Oou!sCm_#wnW5E+YIty4*;_yGJ8i8pt^S}ktskxkb_Roz8EfV3? zu+~3YTtM?pAH9p0EY-Q~FKBQc-ZgnZ*OlOKYpCDL|T^8aOwmFA|foXQd^}1aUn1>UJSBX-AM#ptgP0B1z3l9 zN{QFoi_;ssgfHBs1`Sh~77>KchQZ-ZE?IVm-JgDXEnWKh z>#x81`qkTaZ|=uK*Cl4m^T;q3m{$hnEJVZ&qHLE>OP#m#&MkIYE`)diDXGc-sMCjs;Z@`761_Ga}vq!E{uSNU}59#YFWfU z#LkYihA?Yzgob6O&+REZ==$TS37(>gtBJ2$1jYXz!uAtmr4^K?)2(7X=IQ!5b#^OE z$}}jFN_vICX!*N1{%S{9S&kA z0~0yLz5%eiyBff5#ti{LWV=l440nQ)Sp`_Np5}J$PDahRiI^l&D<`-+Be$!dc8koM znT2J$9d^5Znx_4Je|LZTI#dE(Px{cz@rJm#4epAWc0lZ z+q1`yo@}>UfakfCSqYK^6$Wz&FwRtK^;(NBwKpQLFd&pjSe<9Bb*>CRoB|q%hXuGI zI22CANfMX&m2Ao3r8a@x?H7Po6xs zGP^qq=DEzXaX>o6l1N0N)aGWYI&^1i;73GoI1J|O%#a{HurpbB58KUUF7DM+b>g%= ze|&y^$(&a6C}k2x77~#}?94DSazNRUD2Ygx6b7GInAn-zDKoKfF(PN7L@QvudXV*X z>RCRH9R1YJ)~~C>#mQC?k;US*+xlEBeszlgh~W5ik(?H)aB3tbH zzUz}i4~N6u-R<#saJQ61L+FELP;;x?PJ{_jEftR7 zI9&f};rH}%*58wTbbC*I_xdhrXR}16%T{M*DM`B(8u!|`1)R)IaeCzB%T8Gw6mF4; zoGh@ScmitbOs?+E4(7zn2b}>dRY|B7t&MO(DTxReE@D=NScnP2LX=2EOx0B3-Y(MJ zNkJ042r>fYlx4FWKwj$n{{8#Q%gg7_pLboi7Cec_omAaj0nPpZPvOtJRwoO9Sl|fL zq9wTJByBTT%7#R-XAqy_GAG5=78eKP7wiaPW~zAb$ftgJ_1d5K-+3Nme+lnE4FXeU zEYSbM`2d0PHJ`a$bzp!GSo}v9FX?W-bNAd0PoF=#+wJ#M zwBf;s%*ot@WYbIT&(1F%Kf1W^svqv&$F;n8@xuM6Wwtt-5Hkzht4`CDHN^#|lt{7z zaIIFW0+9%USR^6=svmVFehYufq|9gJ%;@OKAFOJ9K zRAvS-5}|-3cl|I7Kns|vnN8DFW-~Y#K&e&DB2Vd<%pF8#UfmdO4q|dwNo4NxJkE7y z<_|aTfA@QQ`NfNuFJE+tfBNYs6w6tl2G9aYn3;!j2MlBjv`ggiAmN53(jq{N!C-QB zHFqziGG}xchV8RwFTVZeH;MTE?&j&!=V!xacXvG=_jM|gz~owIM;{I$(Evb%gc88+ z)fMVyZ0uN|XeozE0J}3Nt$ySaH+t&X7dQQ|gg$XkpL?hk8gO6X>5pO%W)4zaLwNBi zlWhIV^~me4Kqq-aBtEKEYYh@40AfEoJ2SId%gxR8+qZA8uiua3q3`=sglAdM7G?@E z00gj*NZc8}z8-zET$YFjon2Xue9r-j5uCtD_#6XuUUJ+jE)_QWfI zow$(GpuS!265t!$%m{7A=cw?y!`K_TxIECftBYp`(Mm`SSc;55#2_KAv&X+P5bS_b zi`B+YWe~G~)gZ!>C2>vw!iwg34)RqZHmb{R?e13V+*Sbfk=%^qcz@r(nVj=9)i};# zHUWdUNV1kfv=E>z9i~`NOJ%WvDrc{^;nln=-8-MjZ)*Ihk* z{>7KCro;RF-R*8anOQF)Os=)gnp0xtoOA95k`(BdyAqRtAR@#mDxqpsm6-+1NeDvB z7L|IUc^-4leV-FstrhO`ar|%`569i($B&;qdvbAc^Zxz&2yP_xBB}3(Vb}ne=Xsv$ zG>-AO;|{CpG)?XvyoL-XhPyjEtGPQ945Vt(WmO%=gSu_E+i5=j>GdCnPX6*Q|1vTC z{PR!l)^(kh;%btZ9A>&YN_GHV#;I-UNfzH?pfF7kQB@ z>%xjzIU5bJThiWB6AXBZ7k>J>y5f1Dg=6~Q>3~wosc}crOvE5wKXi5L;Q>!^%=HA! z>cWH+qOR&}W&qV%Hk)C;KfZqb!yo?eFL!r$B9bykgczV_Tq6<8%(ccoOe{G`N~!C* zl=yJil~Pn)M7nMuqIs(GTxzW%1P=VYs5TI=FoB15Ea_xJY~NJL;(%9NP9lvbauqehNH%$bF{exB!fp2KD$sG&l?|gwf+(FD?p>v&9iwy?^0Op&`1`833m&{L9 z1D#s!Esg@BrF6lGB@VJ$Yy4%n`#4CgHOO-~DV#Q6Q$M0V#dD9UGMyBY*5eyJu-8@` z?DT-ACu3%AMDdWBFE20u`mg^+)qeVa{=cWG41Hp@T6CUgf%JXfb=`-bk%BtG~JZFkf9=sTCp0In^@NQv1H2#iU+Zby-(cU`Tulv3&xE0Ne8y4h?>i8HHC z!9ySrHk-3UncV&2;zCuYGS^yj-|cpH|M@@vh{d)T1 zh`5Oy>S)dyEln~aVgU;zQ7|!dy)J6k6&|^oGG|HDN$R@XbzRO`5`FvIzZl2yupf`d z<2=u1aDsH%jlisMbFXe1EZ^mZtZ+)OeMKavMofaPI2CxIc%I&|^}0^o{lhP-ho^RU zH8%s)j41Ka=7WYbG%A=Xd5a_hjtOE+tK~{gDM^Ee(Ht23kQ74iViMZ9x0ba2C2B$TP;>yG+0<0AzF}J{N2p|W!G}M)p7TVZygs`7e zn>;n_+@;e6#Y+G)mnm#-?ObbD?XvQzT5VwY(G6;?sIc6MG)*HjZ@1gN-vssL?Cj#| z>663msMV*rs4Ax{EDAbIbLx8{Vu*0=G6%c}wf$~4m#T(hzTY2>c(=d5zIk`P-CkZj zE_Is6(O@EilvL}*#YNxux$gngYNa;ljuvzq=H@W=*-VI|k|Wdtk9d}FFY~dMVp>4P zvQ|)d9LJlxyZ6`Esq6a9*|VoF4u`|xa1igzY`@=^Qp_x37xP-@+1){&bB>1)6qoz^ zqnSlFMr_?FXG#+LHzbifl*EMi=H_M?hV%3DzVGkv?xJDp>gwuvJTf&JgR8A&d59yj z5fUubc_O4vIEjc%wZyIytiQ+4ufBQp^6Rg^KHqMBeEq|J{!jn-!w;{^G@fsU&CtDi z`TYF+FLLHT{qTLQ^XBZVXjQK|R}ZElnZwDL6S)Nm+g#gb&(6XU=H`y1pEV_g-K+tK ze++~0x+<1%IM`t%be1q2c0jp}aE5=KNI zb0`9+5?XR%cc{b7Xbq#1ble4>iMo^)#*2Hwu-)YIP2bHq?{07Jj>n_B3ug!bR@GV` zSn9q6T(l%QLM@x2r7tFK#7X_6e6nw?7kKKtP7escR}!bWx)M=J8Npr;uj@;3pww2= z06UszkI0!w5`&l>q;Q9;n|aV^9!lQ?Vy%R8_d!x_mKKR336liWy_!ecz?KT<=O(clGx{&;oMvZRbb7hO}_b%$Qxvc}*mh{PXLTuxu%_fO-eNW$WFonm+GD}B0-)(c!e znpW-|5xEatcfL6nkwZB$^VQYWZ@&ABpMH9CdwaV--n!8+Fz0Tb=W!f27cH{uq!SS{ zFSAai?Dof^#z|Om-=*SqIP89U^W$&6el=|S%d5w?A3jWnJ-LCIUHZ+CbH~i;RH~^4 zkzIt8qB4h8*@3x}i3q^~{#Ry1wt{dA`3tbX|9Kb#;09I98_C z<~LJHA{o|LI?F4Xr-WHfJtk4Z-F-=kR)>g%SoN^=`osN1blH#@wZOAKVj^E^$vyL$tj zot=I4)mL|SyZimY0D~(Wrp2_Fb~U9WNxGC0LDY(>dQ~k|g!{hRoS#GZSZb;D_V)eT zZTIN>?DFyDJWtbjFh$OZQX1!CAOky4qT`;xDZ`ZkOXM2+GYfHKs1bz)isl3Y)IA9^ zxkN(cZmLyHRWC0uZ|-(CH#fv`c6PSeY*eilMYS@|buMOgi3SXA(iWc!M^VsV>Z&Pq z!9AU|)>3S)POMI)l!p7OyE`eMUQhwt-QE4;KmKE_^{@Z(um1LL|F%ri&4+7dcDEHY zuC*d^NC_DbiM++(OmJo+SJP?*?EPlb_x;(~*)R;}o4!o@pWeN>zP>&j#+0a&&Pp+} zH}8H{%ue--moI+vmwz$a{QdXed(Dd-JBmSNa1%aHOUsjKQBQbSd*>gqL` zu%ipn{o&f(U9FWU_~SLdUd8J1KlvJ|0G%|2EjL7oL=bEQj+P35=6ME$$m3rdxzAbr zNJ~IX5O;S1k@P*gdo`bHjmIaFnOR5(&de}`b7um>m5gA4S4i8o9L_l>Gc&kZB@Ou+ z5OK~TM(UL)`p}lxQ{B8Wb55OzsFrD(#&MjciJ6%eE)bYV%q*R_>BWhE6o0IqvE8HA zrV#s_pb?n_PAz@$wgQdLyt%VopO(S&oy>^Zf3^+hOP~FE7r|&-eGYbFPEOI@Y^|?5mR_rxVCkxGb7#}2F-Wa-nhJ9&ru%W~=L#S( zoW$MSjNAp@+74>uYdgQ zcfbASn{V%L?tWMP`EWSYTFg{L;N*s&{5VyoMhnr_n+qZ{=v1ev6lOj@JG;2N9EKsK zbbs?pDdl*a*y`nWaL~=o-P^bCzWCy$k$t$m`TZZi?>7DO7f-(&zx!~1`~IhE5ecrT z!2ok|Pm6P3SOpFZJX)b>q`5!u-2G!d%;{SC#J*P#^{Y|#5^1(R%IH`U$!0bDn0fu^ zcqb|BX+0n)7Wgds&z-iiI>Nc+??*0>#eq0 zi4@dIh0y~BkSm-WU@`|&YX&BeFBoTp+KBEX$~mjUYgIQj^A>O{@h_IAk7bKyZcXIz zge+Pm+?OBSV5Ag>QZkrXbwszfEce8xafyfpIp=t+$K&z-{+@_v;99F0m^GdnvM{&E zIVv`-Y5fFFJbm>;_lf6a=0)VG?Fx3{ky8qpfoL)Tu}f+~SM2Rt0YVe1K@^yZ_Yp`&o%wXt!~-p6pB&_qAJ@%3K6D;O0eLs1l!k6owC)e%_4VyIjzue& z2~f9MtE6tT*(5`fjwzX0DQeZ3BxPpixt3|3b|~2g7j{&qdVO=V*=(3ey&5o;d7f=f ziG)eSNz72Hj#HV{+&y(Q3fJ7=L2&bT&Py|NpzRTI*J)O#SVSU8gi=cCxIgRxbbY_w zo_Ad*xv$gYM)Nr3oKq5FR)u_-h6VJ{JcXk9nGPH)2C^Pd^O4oJkIuQ|((7z108U!aI z;jU%xncR7rikX>mvkx-n$}B|63#}rOfm(H4z|<&-AJr0?kju&K{ltBrx``FdUvY-h zg&nDGcpu?}MI6eF(1?-MIXzwVe1HVGh13CmDtz)p|IL$K!E4PV+pgDp)|S#3G#f zuB+mpaV$DdS}V*_N^p+*5J(!;8IW4bG>@gwyZ7%^>(J-icd75{G@AR_aF%l>VXL)h zEvD|^BssaeL&?k-0Y*D$=|>SFW+#wC0diDzax=i4(NISg;&DDam^w+Win( zo<4m#O_QtbR8vY_$|3?a5n00HkZpo9AjHhoO76SudERW#L$opq!fx|C*HW|wOe)UtlhRx>c>Ep|*$6tT_WiW=dSk7JFrC9h~9~Z*08d__ZS+QxU zJ;>5)e&XEeq`8?FlFaI4(_)LKCi{uOo&M&Obm%Aj^eJ@_T%-Bf_WNQF(aP!Pe&Pq$ zRiU-1KN34N#JN4W8bxhw3Xb7|YaLVK&?&^#h&9t`MYd=zT`$vHze7_cBqFG-G}F__eKln)UdA5Cnv_ynds&HSA*zQ5c6UoT z$Nq@5_-rtDh*5N`x*Ij$3&J>#!U;?kz?2(Ixw?cFAcdL&;1=0#Vg|dBIV_qIV)3kQ z)I#P{md~V|NoYD88P1$Iq*&2>Me0&cec#V!=2ctu z62#0x?8L!K3AG_>VUI^&!cvd+W>oND80NXm^Ss~h4SaTX)^)>Xv!S4Fs}e()Sty#- zN|`B|RfwI`z!b3&*d8k6;Q^{eQ$@k9o($da z=+V{w{xBV;B&q9q=BjgHW)Fr>C?b$?WdA}s*KX)9FCU$qoe|0XVLz7fFpe)S&JTy< zoGs^`xEtp(&2~5(E-uceQl?Tk2{VuL6pH0veEG_L!tL(fXHS{Cp|`54MRHHkmsXvc z;$bzjsd`E&v_VoS5qHaphw@M-54MRr{6zD}x}gD-3w^`PR882KJ>YCr#9AgdsXG7; zCvLz%EUI~a&;X&3Tp$O$>ip5-AcV)NTEn$r=>;=ahl%wJZe>+B5V=!z0g$AdXMX$k z{_bu*JA0J6yxZ+EfwU57g0XNCV=fQ{ric)EAuq>^;fYhCj)EFoE16|BokxMQv}h6l zB>^P$-8|3ZwA;{GN=eDg>$**RW+|s z;pGG%&c`k|Ib2r(P9|$HJb|Lj)*L`E+eGJ_VmtAo%H*98IJ;Za($!4e;Z66WjJuY` zIh$EAa!ra{i>8!P>HyT*qD$eNMyHnDOq3h2!^Hg`oV}%Z9 z!`0Q*{rw)L1jKmJr6k>SGaajyI?uH@Rq45O%2E}@x*Ca2mc(k9HH~#skh{0Xsm!$1 zOX@G48gGi%`8aR-%py=rWS1Lx+;gpE9&b2pU0tieoS2>2K;~}378w&mQZk{?e?K2* z6Lx|U8N*p!2#i1d%O799e3g1JFXx+l-pO%)98BN#cbNs|UMibxxJyJ{y-s7!nYkzC z>RiZ75(K>x(D+eg?q=CxL=5OWxjV5F^MoprIHz*hpZ6OO-5>73#P=VD+s)Tsezkpi z-rZ)|Q-Zr5=KY@4hqE*AZ{59+&9xLSCL~BnqtVW>&o?~2Wn)#@dA<=wl^(J`Q< zhIU8CqV6^qR265N`R?6R$Gy}i{maYk*|ZtX&xUihOvZI$ft+`P&N@v;uO^au?#P(S ztg3f?FDwEPlaGYMPGgky$4}D6Pkyt;8f!T7Nn{-jfNSuw{{3l$;vWM+f9`b}*R0dpOBEO1GpIX8kzxaLKc+D@+Mp+Ad=kkaQj z=cD5{e5QT$+QdXei`b(OQFLQBgiVA%u4?YGKt0X5$Jsj7Isv@0)V*n}>w*@5h!_n* z7v^fj%wT5_o0ErtM(cnjf(0wUDlE`^fF#J`7AZpP%B8gSsCM64PxhL0oyNViVEAP2 z^k6g0d^jBL@9!_qA8ohWw16-<_x!QU+}hxK3O!??y#TOQ(DpU5Hq%<2rA01Lb6y{D zEt9VnNGJISH8WZZpX8q!?P@S_cI!gWrHlK4PGhvxcpAHArP-XW4z&`X@F(l&!MZ+P zL`1aNY>vlcDdliDlvx3spP!q-oTsV4-^Q#JUdZ^%VWLp2P`DM-y4<5^ zw=hB?4o$oD22#KhaO3EJAR@{l3JpW9RS$>5&4=4Qcdx#9`Q+KNGS1iU-uXBu$wQxs zL{d6V<6%4&Q|YAZy4-bO7m&{7e!m|NN2@h4G0>$f&Y>C-$Wl~unCLJbtD-v01+MD% z@B4Q|^!0a_&z`-=oL|3weRq4S)rYQgukLONH;Z_>Eee5GU1kvdT0s@9Iv-2)Bcwal zP$|&D9ft;#6x9W-O{2r_OzzYouRHDRj*OMejSf`|6?|`gAC$~)iyqQj;f)lxjw9LVL}{Y9}gQ}fL-%A-QV9I z9vv<&E+l6y)eL=~*3BW_!q(C@W?Hm~myd{Ut6E5eMQC}GDc~A;1`vB%Ye!+C`Oqd| zJ)G*vVev`9$s8YZ@&ERo*M$>hX4Nv!1xT62gbl1V>ukhI;%bFKMj^6V66a2`baH*9%;HXW zH}~*#dHLwcqbK#u?(Xh@b~_(9`7IYFmVFNC8`~& zWXwEGlM_9D^7Q=i)qEU{@aXBYtFJG<|Cit2-QPYwzaYoWyZ7vP{`A@2_AN?l({l^} zNobr;3ip-E??- zu^rCOVGiM0>v0^x-kn`+HX9aEud1rGlxeQ>>{^qg(0b&CaxcPGaDoAzbCyh_VVS+voFt%$FjRS)~d?R;2`bd#3h0Zj!aA-aj>IF4=#VS!h*5>mv^1K z^kgv2$rAXe9iy#;pein^DWAk3pKmHBr}1Ah=ubX{+r||{??5ErOgZP{@pyfGeLNmx zrNtVkwepGj#{$Y*G)&S-&f69$nr9DaH9Y(2NiH|0b}J0glirc%m0 z*W)w-5XsY2ptFcLKUf7(sHt)Q2oI0I!Nl>ip)gzs%&~+Nrf}laP%nltjJ?SYsJckN zRgFCw_sz^!q>0R$uX2j?s1<2~_2&*IGbfG-Cc;c1Y?4x{<6c#RTg1$v|7Ys!O<1tN z)~o!9b6*X04FXP0i<#&7sOl+kO36*lJ(`tnZ*LBV1AxtDLtuBGrZP>(fiH#G$!9Bd zF2yFmf+Cw3%v`L(Jaa{7T{j52uItUPyFb=K+s&hwU%xs(KmYMhudlDKr*YbDw{o_b z#$%mk5m5(-yp0rKYQiqgO%>A20W)WEkExABl|2(1fk=y)Pz6o%{PgMbXU|`J{pC02 z{?prc({X%te(~bT7o_~l@MeE^KOaU(eFl&7-1h?(UeH5^7{MH2WtCXuuV~;d!CS%x zcQ{3?;viuLTQZyHaepAApI`s{#q$@Jmye%4e^rVb7?k^Me>)ZjQYV~K-}fmcgSy)N z{k;$gQI^C+Ft@3wstAYpqPltv?t>eA=(B@~vk}*Vxm4IZkB8TPnsOSxc=r71lV|(; z{q=`icc)O{CUJz|d3%}F+zoNv5#^nhp*|tQeB_2s-7zoJE?d+EQhI2!Uh-dZcdhjl zd7u-?l>dS&|KI*v-90l41uP2)_kI8V{kzw%Uw`=UfrvJn4GE9q810u)bfS;QJfG;1 zEpUn#TME)3XTj3@>8r;%y`r-G=!;Eri*VOzCm-!n!_cpH%-xy6ASR@gRJB$$GdE%; zCQgaVn)I{`fP>`Nb)7qz!MqYA2#CzAsP3+23OBVTE(Ua63K?oMb5(Xg(18o8M~4y% zlL#1qbW30hL?&5XBLYa=t#uT@*MMTN962F$H~(&C9Iz?7h=oZQ;Htwy<#hU1%=EqIeQ zS$WccQ~V+6`NULc$--76KaGZ%lNpwowGSIocznQ5TUCi@)r&v1G5lKRcI*jtNj2TQ zlv2#J`tf+2rpc_bASGr_wJ5b=q)ILhCv$R!5?Vp15^*GOL5@{uu}QewbY}pU8RN0; z?)LLkP6>T|N5tyWc!}_M+DM^UuHh>BraeREzm^y!gAn`@4Vj+rRlA|A&A7`!~OP`sne{ z=XdYkJ-d4B?h64gbTZJ0%?^hbtL)(AQ9byd7RsabaKxg9lVpnVEMTkp(@$?0>B}#_ z{OX%;RPpnVKV9#S&$`VxACc*NxJ+G+zKzK5kN5XPG)T^g-QlH1y-kM{B_e}5)VvKy zD<~l0M8YXXs7zMPDyH=M^-qbq$B&4O@ zy?gg=x7z_oDTQAEBt;Rki97SYG$Jg_(*bn=?b0@78jAu~(FZsuOw z?R~f*4nP75bVPRMA#B%2g*h=(wF1BwPMB#M z7cImHDBsBp?qDVbg<1uXIGK?{z$__A<9RzNf&m$%Jy2sCP)uc6$`2+Bz8tzVA&@MP zD62b~5EwbT%Qk#-CgUfsw@v_mSV+89*iM@Dle&nKbZ8X2{kS66*^yNS}7%Q1YAZj z;}`79=z@h4{ICR=**=;X^(*$jF1PrarAg=4Iv$T>3}8wrot>TaeZSis0+C;agnO8X zlQ}H2NWWnr5~!9|@d3b2<_v)mc@pY!H(YFKe>~pb9bW(N<6(FF+rRzW^Ye?G`orNk zj|bD5h>4lSBXyLv%YxKn_8xi2OFPBbp5VAi?7b3gCN7CmHle3apY{Fb`tJ7qyX))Q zn@Mfb^2_ym%=PN=+rRkjcmMXk{qM7M|MJhj+wbw@>e*B$JS5^C-K6f`W(!Bhi3R8c zES=s?XqY88-gqKbcXmiL+!|F=g6!}2AG({zkDfhw^5p8t^yZfj`(qh4T3~(dhhbpm zS?6h-bS`SO&z+T8%fvOrYqVodU9UQavQF$PM8q&B)X)$HC=;2Imk7vm8m>RwZ}Pi~ z^G8pvUL5vwIqEze8RQla-$`8vuG6$S$xjQa>puirX8G~K`@_)k?>4hFl45c-B7!^2 z1DB88n-)o7Ul7Jq=l=QEKleJd?DaPQOpTetQLe+d-7K^T;}fGTEefhEEu~nA#^Lc- zEamiU%}iR?OAP!=IN8$NX!X?Vd+S2Y3>YVd%9~kQrnRlG3;>6Qh}}m|!Q5f9PYWlH z7(_4+HJ}{hCiwKu03eQ8C2I=|nO9S|7l7AF0yC3)3*2H9YGXtps!xh0$kT@Jky{(` zizs}NgSfSB`pIm+W*l)@4-$rOXS3=t&uSU8&CE?K!D}Q%!$3M~AZd>tJ-WPnL`-uj zoR`>wK-^|Ca-^b5b-*`X>5Yt{0#(6YMk zWUX3D%Y5|3P!{-(gGdJiezL?js@k%kXj@nFMx9E4uV}Sd2tY6`JvH<(D`CB#r?r8W zG%pEZ{D(!lE)mhA#}^kD=llKset!&^EfG=m-iL~6W=_VLbJJZyXo7lG3;tqsm776v z+#PH0Ddp|JRsH^cZ-!rf`Q_QOXIbPAzyICs_51D6Z->6tIrYiR5PITZAwo2?#cC1( zz{JENB4D!004tn{5rl^>bIvfj`EY%6cYn9rk)%BI3U?-}_@Dm2|Ka|2_uu`S|Mp-1 zAO5?W>$~^we#x87{=n-!_=AY4qXPIa2+d3I9(+uYA)UR{_44(2Rr=!e5JhlEaI zkA+Dr4s_t^7Fj6gWUitl0!iKRczpZz{n^>Yt1n(Xdh~dIw|o0$s#a??n6fOmMM!d- zzxuZqUIFgCaKLQK5q^gp5y_~=w%ob4PA?e`-3Xd>95h2DHvb1$L z^lgk>L@#DhRg{oQrzQEo5*{87hqbK+0F2^%JdrPRm)08|_3!m0mq9933ro`-hzZu8*cs%Zo$Ne~-pI?~S@p#N#M{$0wrx*8>N-newCE}u zEvB+Ymvt&sva53KR(pYo4C{P=JG z?Z5fz)tAS^ZanNRE-w0^yMFgR`aZ+C3*woz=WfIr51$*Y6wLDjm53APoVwf(smpWK z<1|gvBqEY!I)=!$91qhtj=%qxKW_WYZ+`RL|LOnofBlEQ|6l(3cmH&@83JD-a3aE@ z=~OTWJAmLuE+^~u^dT}gUH^^~)QXy`k&~%WLDUqPb75 zHciKQo~=~435Y>+iB19}X^rcurGQuvl-bDJe2pdv5}6&7x+f-si=b+$wcg#`{q)n% z+y4CW^78EL;vKy&bJ6MqcVkM;oE+%VX7#JB-DZ_EI(3?#6CvC^=0V%Z@K1c^&{tWm zPomn@iLbw3J=A*q)94;2-hWk9J#pxfMn`Whm+dnq$M7NUwj>$Vpb&pjuNL2e3tX88n=5!ioKsN|5qP8|1qJVi@fD1ri zDRn~JwkQLLVup!uCZd=+V^Qvci-^0f9azkiS_jjj?CKAn)k1YN-32q}l)99o9LQOsoO0j>Pot<4gy1cu)d;9in zDf8L+rXTV+j?-A?>2Nq4iq?J@5~X;uoKkes=G?cLRJ-3Ii(@x;`8yi?^0%Rj)6!O z)uWOG^~AzmN~z0|GO?iQY}g+T%18tvp64R)O_!_H4I!&@gx(UMYP z@>#2awz)?ylWH|gbG^U0eRO$sdHG00NcgPZDC$(kd7iy$P9jKFwU#Nhem($5ESpUu zM@c^6uo!78ue+IXbhVokdzU3R8pmnMS;(l?VqVYA&K_M}_5Dx#{e9OBLZUX7Qd8n? zvr#K355aD$g2s}-L*nx1vF|?g;GX(G0ech0HkzwkxU5S&G9&KFozdF5^i z^dL-+2(I@Re4lcaw4bkb^?&h>pCbuI&$gFHGTIA-wzbbS@)Um{;q8`n|3s5$B#51d zJ0R_K6P({iQ^eUZ&A3eOu%7Wth*(V@gWJK$kGrjLW?qo1ddlVs7JX+aXWZ& ze!BuaTl@NIAc6ilH(ed&3dY6Hhu;3if{TO)2Qqm`s>88gb~c4}m=VJk00Sx}atHc8 zId?Ao>SosRKCB{5civ`-Zl%zDzF==eHAI^%*t_Lt?QX#UH9)D@&;npl%Z;nVVd}rk zT2oVFGROhtGA1p!Cq+0|mxd^@Twy_k^ z^~j2CjKbz2Lsjd&mCqI}ksx^sYMR!e!gFUTN9@{0mrnWxJXpL*r_!Nv6dgIMviXT9 z>EXc5_h5A`p=6lYW-XmCa$L|1kD&h*Ee(;yM6Eh(en!jjRb6g%)^X%4J-oa;8c6kJs3TQ^{_|R#kq_iM>)srd3i2M$HU}Usygjm6gaC*JoQvj6+y0`*L*SnLN z)!y;jQ&U^rkGG8EOq>oX(e5Ln%-&@+dqiQ)o7rDmcyXU*UE>lZMs|{;VQgDf2n(0^ z%@bk}HK!Eu4?d!Z+d zGMYMA<+UqM{f8#kjM@9o297%ZCePjqf+5>aArsW^1ez@&CCOC~4)+w5eA}dbEjsh4 zJKTh(92LibRa+(R)7p0h-z2cf_Y0%OP6}$uB4+Q{Hv?M*Ce18&rn9z0%O%UBB9TX( zPm4o8=T=Tn{a+KFN#wRTq;ur;ChHV1ZCEBy%RU5Stggp_a=$XkVat+GXg67w*-$%X zUmk*>YD^zc~l@@(m0VJYC|LSKjK=*U?&ePI}9 zM7BcJw_(+S5K&_^%+G5eWYWo^EW0ml9VL&V+pG(YhB0XL(2sM)?s*2h^UejD{JFWW zIQc75`4P6TAoItG<3A- z+p%=SRo1_infN&$Wf>?8`var=T*&mG&HsrswnuzQp(jzvcSo2dcpz{1ZHD(7V({jB zaWj8`KmB?#kkDu5A7B@2A|L~yw!~Or*0MRF3uaDJm6Hba^Swch#fxMJ8*z2LftwMb zaTi=qjDTL;oL8TJRMbpTOzJgMc&agtomr?BUg*lo&-bT?UA%dc`nVuB$B~y39!FUZ zD_Li!dS8w2KQ=h%VcXH!8T4=~_@dqCcK_yV3vmmCm9^M>Wz?3)LtHbNSaAT!31``?$5iZ;Sf(7+o@VF$`FUn}BAuC}?`g>d zzn?KM9N-`t zNs+hyX|;^4#b$ifrsPu>KI(uKKb1^I{SA0w%NFg*((@Q(jI##ECw&yo5beJXS*H(W z)!_Zbj`+OU=b%tDpQ6#PZG|@cIs4Pfi_yevjNhj78{o}XL zoZ1h)`;HGB`X216!Z|7Fd+QU7xz?%H08$wSr}l!`uu3W0XL}l^8DNy_ky-VM(i6=+ z-C^GvQT_ZM-*^xJUyFo%x>rvAS}W!pyp9*Dpv)egB_(5<;m}=<-7>rOiQbCCujqk+3I{hk{XfwjfB&KF#I1mSSZP!-)k1n=}Jw4p`RZMekIM z1gGyi$-0s!8Z)};KcfB!KMI?E(_ICV~x`R*5fuG57S!{>lUm z6CNcMvj1KKyBmjnmODeiL)WB5mbgDcGZVsLE_eDnOdB^ef`l2&WEat5B+tx)3+r8q zjv%pSVjiaX(M3ZNnLs(wG{o{vOLK=d|Ce@){T0LE@pH0p50}|n^Qw4a`h=}l>`Iy0 zKqY(bpvJ|ezI(B*`G5e@Yye3aI6K(Kj~ZfJm+Ju z`#TeDqBS4A!hZ6|vq7Jqy|lR4`Nsvey*swXZ5DWOxg|XA?Aa8TCW7|YJDCI`mlg|FEOI6m9eEDXXP=yf=y zOzfo`dMcc20`_Bu4W|nM;Bp`w41~tKqG9Kkm(@(9o7!b*_1sPwI}P$47l|VFP)Whn zCt0G0E%<5B&4u!M;O(HMq-W_f|5MW|p%f?lImha80TVN`BX1-dzrt<|(Y47k3)C!Z z*h{%e(R3;?yX9lAfk^j^Y>kp&VdlmsRT;`SO5*nKdh+4fdS+wWuhs1*yU1ix_*E-2p(|r*gx0?Sl|FuJYI6r?>Oc;{C+_monGGvSj&Z2rdG*T#U z1UjsFY0g|+^XfJb*!Aw#c$XGUjXl|!DU*sZDe5yS$h}M*XkJTu8|x{taa^c1kfZbn zUP87HAlRq!TnFc6D-GW}vm&ZfaWeZ*57vw9-9v%z+|7Pci8P5V7TCSL-*sFNsW4gU zl{Mge^ZeDJX|9?``{RkGsF#-YU(rs#i@!sU*ZpB%r2&78MW4h6zNSdW?zztt&@7(b zs02}uza^&C#fV(yC?cdoJlD3-UeuLvH|pl#jQg*J*kPhgP8QXq(w~&nTw>9!iHmnz z*2l#{Gzo_zo7NheRnDO7OA~Z+T|03xQe{$a5yaxWKV@Xm-hU#rBGIW2=bON8>vMa) zV|M?`?Akc!e%z7&RjVX#VxU?Bo|qtvu~98FTB4LpZO2{{%6Q_Dp|dJy_ELr>W#aM%Dyb0S z@1H7~)VbD8=QxVi$=`5wCqto%aD#lk<}{K%h4TGOPv>?%pG`Jg%6E6!v7hgJ6{ZBIiYUlY>iAauvU5yyB8M zL>Kk z>oGCSZmrL(b-Bytczu+)(Q9ey*f{Fo)8Ri zw2wmTno}YxPCwdsk-0g=I+7(@+2qTWj2Gp`YBZKW`UU@O|LvflpY7j0G3P%y5gjTY zI6ls|e#=n(>kBV~>rCOB^3pg|=s`>QlVt>t#I%6PZhcD{h!qq0jUHs+x>9FbUn z;gQ#z5yBzSUckuu(vnmp8CJ%rl0CptN2;==C8!&iwKMgflz26(k~HKU`ccXSAs=n!v&2>p>z5rvP0?7*qHrDX%Yv#ANpX+E|LENhXEtwQ z&(0Q5^-nuekG&`FO#|-^goDn^=$`UFICbU{MDdoZcZ!bET(v=m?s~_nJwZ$<{F45NIjlJJ&pF6`Jeo1>U^k#P91eTjng?N zUj0{UF$cse1IGz_yC7%H1^wD8A)>qcL;3Ykn%fbBH&wD@(SGhnR6IQ&_6|idk?+z^ z4^lI;Bt)-r%l+^Ssn6ved!_C*(vSz2osYs#K>R!L$@Nu(vDA7XqqW#@8TEtbnsNBF zxx#0g^O;yZ@NR?nWx}V;nSAHgwt~dN`}N&lQ$aUPazVhqHXukM4e(tKS9b3HuQ+{K zUT}2<-Ct2MF%UM9LRIXugsGO1-PBDjyq}*E3QQzkYj8%L7(}o$+tolc_ufKlQNQ^T zbhH*o)#2_eZC}n5i-3Z{>RbAU`$1%{Tgmyg!?oM%{xyM&P|fFv`!V6i8A_4UfJCI zJp^&blw=36`EdzGg_o&Xxvr3ejBZE~>9SB$UUSB_r3(&5`S!PUxD###xwIjD^lxpo zmy4n5P~pnO;LhKIfHBc3>ie;Ko#XB@tJreex7=U){4dHPey%w6^@a417IWJtD;&Rs z#xR_GnwH1;j7fIL0JaC2tDn6MS48-(@8eOBpz|7|J(tBQWgB2(u_1S0{WF4i?!m8# z=fh*L+;FW;-&z_ix*g`0GV(#ikcvJeO$k+`l5uk@>TVp4sxq=RNHL3Qm%0{q z@R<7(Rrj`BCKUnC5Y6+mtajDz44i1f*nZ`80<0UOGyJw8n36l#K|KKl`*%yd>*j#C z8oTIdpsOM5S@=_^2lxvDmY!K8%1xW23T`NqjN|=%R|&H${Fl7sWbpC6Yqr;u$mtFW z_@M^R6oCVDA$N~Zs?GQ6Z~=L6Gt<<0v$@%ux5G(~$zD3NX1p8tCBwrKy#k@nxX#pf z%yyinoe)L*eDF7%Y1UCWg((@26;R|kR9{3%1uR|c0KoykeWKx2r5D>M6V33GA>coVWtK~!3Pf6ku0s~=6}4Itd;>M;lK<|mXThxVaVJyw2-E>| z0R$uh9`A2vZ0`=Y3JxWWk&|4w<2^r(1nC{Cop)^wXWjW(p#%>5ek{o|{M8QfrJwv6 zD+U#$D>lDf++OCSe*QVP#ckH^b92fwGENrCtZZFB;~C%*eKYldoa;?w?Z~`_gnB*v z))Z5cQpwg2$u|r0)X2LI^LXYOn*C4N_iVIHLalRB<(>8>a9%9|0)+nseGa?>rb^Sw z?D^_(5{6z96o$qPG^g*B;MPPwi?c28LA>~mk%erHEkYj}XC$Kti$nr&H#aVHlN{@oVKUaTH*4J)vYG<8qUGUF8_IzRn zUe1x**g8*M%}@*{;xNPVVCIdjy-Ik|1}QwSp%q9`J0^=xOv#9P+*D4KoYlEROGqjy z9 zXp3-wU48^e=Vwm+jU>$2z36`(s;8V;pZiUGr-5$0eD=}WlGyl9`Z}$3d9y`A+Mk6P ziWEE#u!U``@~8a-GC8JgE1rjm`X|JK6nw6ym5A7`c~U!uqQ2!ntakyx^%9l36!6r) zW0HKoadRVd)gL%8^e5hn3u_=nO;8owbR*(;!gVEg7_Ka>hqApuXl?8u^Zi1^bpEDz z9LkrZIsB}xKf1c=Z2RKj<^|js&bUnjc!cCs`Jm%x?&=oRKWeWNxv^&Uht1<^n|Wk zm&NF%6Loc~rAUw&W_AA=Bvv&snLT7yOVPeMjpY{Vz0s8Fc)IB?NQ}ae3B0*c|9Se0 zVw6tGx5P$l|1xS}`_RxQKhd_V1~oK{_&A0@@ZpVJ;(6H1ISx#P`kpiV{`3Y4^3bx! z7$9MdYOO#_+P`sG>^O)#__e0k>-{CLrKKZXEMsk(-Rt3Bw31l#WNxK#yj@*=u+40E z1PU6onyh({qT$2sDA4*iUQ4}49Sx@yee@}Hc|t@hPQ32MO075VT)IlOg9^#8o<<14b|*8Y5W_brNefz%>ippo1e!+|Qe zr{TAg7RSM^!h4bK2&2zGYSwkRL^wLUqPXA59Oh#pjADRw4H<<3n(a%Y;Wz%^+vv{BQzl_c5B;-;z+TTiWz4qcUmHj7Be4# zn2e~|5A^=K02{_CJV>?PFtDq>u@3|i_zMfeQV&~C0(;l%Gvh%2sEqHB@M`Yhr+Qx_ z5hq`zO}?FbtUWCUN3;%yL0264e--l5WRSMn^6+ z)t~ZW-BT3YFE*XS-ww<8TitnzCHzl}litSvlXjbN=o@ z)SlQv{MpH1X06axZvQB)=9!zsYlIStIypi~Tf`98yB_Q1oH1;^IB%od>8JU6Nk*;i zFPT*k9+_>a>yG=_>E#1=FWuc1x9O&2o|$t@LOp)o=g48-DX*QcZU$Y^wJvJd4RzQr zL}_Pn##x^d?~pY!skLkyjEjfVX#uo@+!{+1_s@pCDV_W@3n2|8J0XPx#HDE}*~Pfx zksXA+L}w`(g^}!=yk2gZZW;wbK*thOj_2B zGZo|6x{~$FGa2wgtgW?fSw)m_G^{Qb`i5GtMCQv>BJyeGt@`yuP_x=0sTZBjF$@IE zOm1}TgJZD_;IWEPdXIG7wtqOuKn#Qt-t+Omgpss@vBQVSgC83@a7Y?xpg{-b&Ha6(dWS+GOC|(%+gkAX08O0 zL_%(X_UFSJ_$$^W3rB3@!LW!|TBMrsPDA8l;q(RhY)ye}TpFoUsD&o!?lk@l4qgBM zS`2Ki`V5&$Cuk0#sJ6kIjaY8e&MzKN=vKqcZl#Ox!p)x4gZn(qL>{IyZRmw8Ws3BF zKaV0c=6$3pbL*@Ii}7>gKJ+GAG_?)mSF5-Rw9rJU#sWbaSy+v-h~^ z^7wlJ;0TVEj!d-5RoH)RHRs^)Gu!dAu+|*@lG)_h6CrIhT6iRTm2q=gcoDYoB;il$lje~!2UnY zd*t}4By9(mAS#i^%^&+0TI5#)OgMZr<+3{X|@_PXO!PL@y%lCi>GFwj*JBD;K z7F$y)Vnj&sIy@yCCH8EVi+3`t7`ptGm=yHyJ{e{7FQu#x9zG3U2*|f1KAprO*LY8p zL66(FNss?l9d8p>$M{&D7Ea1v&s~k!yHMDsM)#?U1voAUlc|cF(KST6AUpWm-mB!Y z33(zw3WsSf%tjSuh7Oahfo0%&^u11&vpi)|qd()}ac3pCSBi#(;V84beyH4ko|lfk zEGvK+4&l-IC#y-T=cHW;>+^}1wz4wC+DG5Du;m#=CFsmdq}-aWcvNeoh&`J>IIo}z zP7?x&FcoIjz3IchA|eO>XnZ->w)UC8u}L(x1REAAz5gjY-WR_mr6f_WAGF11LZYH%p} zmnx0BzCGVmWo4z-fgkj0@D7^PlD2cQfm*8^?0s1IF5dN4G?j*wG!^bL%6Ap8OUdJ- z@&~~OMAzby9T+l1v*jGCHywteaRV};{U0?NcAlY5A0Ix3Req7O=4V`aougQ$MxY7S zVAX$qI%(=;>! zd^kPdNoCd7&(y4OhBg7L#K`)7-|P5B9j7yk6i=SShKy(p`#S6>!=(6lqD(@K*)G;+ z%a|nCI@w#pWtC<9JFako6yN6QbthrXJTRTzkdcljapcjvn>6w*jzy0=w#dM*FWw0wRYZzRPuk z-Hq4AK}RS$MNao?!zKOKn9{rywm$QZthq|C%w(n!oOUv^X}$r&`RvNx31So}Kc7ZQ zit8rAp$wvA!ct6~o-uCUYPkD~rV3B&Za;{)Y5?_&2Lw`n}EfQcDll&CiCJo3ZS)IL8(e>bYAkVq?I zdrChn*_Zw0&vj}CXXc@G?|i^5H-dVLiq^^Wu#Fzh1tIjRM~rNRN!{Nw8u}qE@X;$9 zA%^MMxUCt7YND684SUvLaK&(s$sOQRnSZ=q+XLVd3eAGkX5ghUcXYMi*A^qAHD<)g zWVMPj(vw#I>A61}7-95RVzjRQa)G7p8+BG{(ko-f6T^1XC6lK^!Y7H0TA>FZER?Fe z=N2?N!aJ|NTR7d4lcG@57II^G-x46lv;SfP&HJw{DnP3bOQ9G_4 zPiSf>GW_b}o+7X3A3TO07!7RQaLQ|L81wo`>+fJf9f9fmoHI zL7J5%PW!??@99=b(7$fqLK4=JRoFeK%I%HJb@*y^-+ffG#T8J8U|J(+FFcaT(!$0U zBC;7waQG{JW44#IQMMCK{Fh_%Q?UN9?7O(G?nGHYzY==|)+dFOl=@h|>fK$O-%Xsj ztjVOhsBLeY5U#B1_V4pzdPauP^pJF=52`STcGpYZpC6^-F|AyD{t+0_KTg}uynz&d z|7;s9$E3PWYP9u({t2w6oRVQ&Oy7 zBM{EtY@!DXUx?LX|L^~;J%E z<}xr7D_b1v#ggYW4%tvf_DKw*92T6Lde;=AdMza8<2?sNM{RCb`swJe_ku1m^+#(7 z&Q_eExiDn|m}se_f7F0m(9z;%q|WvtW^CNA$-h1yG1no~y(y*gp1dZn8=Xf+9je7B zycX!7oo#oeM_+9J=4~5n@?~iMHwzf>>US*9Rp*?;eF*b5B!Ujh>L8&7KHeakgp@+K zJ9wc`(49*|)wLQm-%j@*^#_H!ZR(b)$H^q7kdcfxXm%@`G2tV*~#>d~IC)qc;hMJ^u@- z?Qo85wu+A^_vv^6&0Xpb&%^lYyQ4qV&s5!w`z=A%T}2ws$KRFl{if6BKI~wUh}YO% zcHHLr16~( z%_PuH+tb%yCHG3?Usu--UB&yn0 z3S94P(TBP>QI*k!nxb|)^WI1r;=BU02?wA%g9r=^bZhR^x$v!tdN^sO+iForw?x!t zi^_v?0r@eBaQU8akyF3u7Cg#gyS0XP&tWlSM4n<}7QbOOsH7Y&%+xmX8B@qU&x^&2 zfwdo75XxgZ<%)6NbwZ}jzVYeV_k{vp6H1|lCecZGqD$I0WvMPLxkFv2=R zn|fq)K2j{PG>tlio3{b;aEzQn&P9VmLV}UFP7qr6JC?WKVf#+ITzzWKcxV?;2;ad| zCU5U=&X7p)2s?7<(!tf4#1@=gS?WB+!!2*nc;4L-Jc~DH;JGGpcy(u=ljY0*SsqfU zY?<^r;R{rg{es<>U>f3B4F&2J{ytCZySp3uW`WJ^>}wwyHZ2&zfM>@?f+X278v9bo zcIq*b#~fNqUenJWofKYaKS3XrTbZ$|=Z&mU?e$(e1WTj?BTzc22(F^`2#+EyeD1e7 zvIV~t>CXR=S@E8HHp%9xTJoSZIKx^?+S!>xO=o;&Z-T>mvzcSH3X!wL7pe;$*;9l^ z7Tc093ekF~*drHAjp0w{q(fB`6%!e*eF0V9Y(Hm69Tt~7yPss{(O$jrFhrj%950w^YcGG;)oG&M#6OC zzZmjp#!q#JD(COJV3o>ki`GO8nDbIbv>|Y$c!;{FJpEYOplp2l!Yth{^B2M{BvhEz zVPdkLm>RaG!>0znakRdYd@i!>{;<;ULBoXUcd<)_lXg|iL80$ch-z-@;XW&W*PJw%l)W5!TDkd$Th*m06%pC?uFSIl7k^sC>jeIRCIPy{RK%jO*{DuS=hBoz04)(GdnDhUL?%4}pGVz8t0#k6lF(O) ziCx^GioHAFoCc;S4cnfrj_F!Od~uvFPf``5?H!A%=(p`$aQ1>n{|faVia!I@7v@Cj1MOKGFf=*MteT59iyW@mf2iZNqw>> z)+!h&R+prG$LXl(W2M*FoQ*2Y8XBrUlA~yiUap6l{1x5Q7X-c6Rn052M>Jk~e+pbj z6UbZ>aq!w?GH))&AzU8Y9vph_VSyk^t|4UtHZM(^nq9DaGqRobHtoNtdPN`}luf_x=*J(*s`%rRRx>lk%`tJl;*rz|~+QKYIZ*>kvu! zrBg&o>hPc!>DWC3T{1N_m0Ub3Se(C4i5f9Z?61rCZ$q+F1LYgy^igE>*nwZSw#b=@ zq|1G|M|v0O7UWsnuUW%IXBGkJ=J_nqqoL1lz0z^SY$Qh>flMRGgsn%lD3?qcZQk5T-FF*wJX``u9&pL<`O+l5l)nKVTrf$g0p{z3 zmoDKwNwq#!DyA^0l422(Y6&&F*+P(3w~9C`1v&FR#e2$imo$i$I5@(-SJd0Zban2EYzjx(qSU%N(Ptb zqmkS88OOkm|C^#)dI}$6Iz=^^wcP#S*gIKwz=RDHv4Fo4hV^buevGEUp zpa*;8!_grjL_Cx|5{B#*IT1S9kdoSkbIDsvTZXJ5wE*%8#V8tF#Y-Ad_4u=alh@;; z(1#$38U8qzcK3TK(CsB58HC=rVCcYD|rri##0Z@{nedu^mq^6K1R{rU0xh zwo$Kc5_jTV7x+ho$hQ7gJE^ew5Rf}fNhWSPt;O?Sd_PL;Fo!EoUAFe z&w$w;-Cdj+ta;~z8MD1GK{{!f@ZMDx9LB~$90nCF=@_Rl!VZYVWQ%IWZceSPdHO{n@4hw{qPV$oy7drBEY^7>g($t zEw+ETIR)$+qkYVKV>X5(Cl77iTmOM1zO;~F<1KzIC!R7NFSE~KJ{8Wyd zi@Ljun&LJMyg=1X7QitRp5%y-NY z;Nv(;+!=A3H`RhKv4U;v2?+_8p3X1NxI29=c2TIxOyMi2Qn=c<00SuoqdR-!50Lp@ zb#*0o46ILL>K)jw@MxA#fe0-HL*zfAmkXnU;P(TJM&n2%gWTQCRzc9&fqwXP|I$OC zXg&cl_Mva@#|J72ykN8~p?X&s4pAij*F2j>86JqG!|x=ow{{JUVx!X8bW@&z}k+m1>FD6LuF_*zmW*=KGQoOd$l+}l_uR#QA>j@|gf8E1V928b?W z^DqjMMMj99?o(&XEjBAMI3JFzKoQB`uyqBUY$KPmZAt<>Aq-^T>5^uNapMvW>hINd zp3mWP3?EgWjbkvr*0FvYb(55V0L|S?)y-kjFU$OU=cL3w-%i7ch2_uxinvAq7gHhU z69!ldGphxmy^!@ji1#aYaP=vuuP8lXxtS_DEgahtvm~!Z5t2XCJkl5a>ooX~{ed(N z2l9M`DFkG(jzbp5r1aL0Fdj!38}9|9ASm{GaMyRImhgYglxp-uB5hf(J!V(X+H>FV zOwAK6u*uA;4Ur|U3g=(!`Gx^JaGX=1*v3oNS)%M3khuxtXAE-?U#a}3?w4zQW3dR& z5~9!LBD7ZmC5fW&uxOk&iVT?n)q?1Evn(q#{6)EfpB&%!S4}^QAq3S~*GGzpGGJn% ziPKBL6KG*-y5TPrF*$0Vsbzu9$?xxT2HlWq!SxX5X)!nmWE=$%CB&J8VYE9!vaYOG zrnF=Jeb{LD2|S6Az^uwzGbss)MgQwhgDPOT7gc0jSm=!vUa{8K#9%AxCSrbi^oUY- zJvJF4(WLV5KOelSuXvcri5bRi3?=9NeeNy~wh97o51N)F=NBK2eJs0-Vwwm>-PX4P zNkUt`^7B#2mhoiZtGPoptJRJ{5D!-<-u7NhmrXta@M7*o{d(jBJF5hEc?F2&wR!!C zcijtF*Fg^sGl(9*z{ggO4=OfvSrHrYe*>cY%QIJC#VhgR@MG%(SGEPRXb zWKjf)L7(L&=V)Cev&gXl)-L7u-7Z8-qN=0rGVJB()iX1j@%tkuMYRY>`1}jRzZ=GB z${){MZ_HS;ug1Ly#Afn>Dh$pkq)E&RWG0Q-XOgU4g}{>D5_<{b5kly(E%0{QpDDlQ4`cL$4GyEL14&;}$BV%nhN&dpaB0`o&Mr}Z z5*eJM;Sb$vM2HC^2MuurRq_*KN~0+>PNjD&=O7QB-ybfN$O;2 znN|Pj@*SRHX%_u=C8_?_%Y=@OPWQq`0>D>i9y`q+}Pac*X@z;KgC%bIHJth}uYpiVEZlE4rm4$y1I zK`oPFQ!%d|4tAv5L<6tSQ%3;+VS*;`>Q3tM&gJQQO+fx-kVN1WDzhNyb~m-+^yCn* z%3)aTLy<;6AnXtWLI+V&)>X>S!2pqvNZ_NhGrt;+MKnvr-%^=Eg&sf@aC$ zx=HZOi{uRj=sVY-OVHh1k3TRlY~vy}DmUtV{Uq*pn>laQIsS0b67XC4aM^Ru!q>lf zeyZp6r@Ov@vU!Q+X>eR`ZseK)n>x_@8{TiIDQNSTtK9)eaNvkb6=*zXn2;lmR3cut zz)7RY_4uqlv?RKp-+O*`Br#f^{{WVFdORBlBsjeJ%bpo3ASnha=`syo$?nO_eU2$j z^dI(qNPmAnA)O?^rb~joktVxxE1wKC;T}QG*{a!_q?tXE7Pgd{lqE1}%<1_Am^Fq> zC@ag5G3Gcyvd~C1(u1mdp%wkN(C4lWA%2}b(0hfD5fIOC6wDx+1I^-NC*_E~t)r$L zPwY^R@UPsuPUfkbirAM_nP6EjB!y?k03~ivC=5ppFk?^#le?)2VfkG4g`msQ5CyYu zjPyzL3=i^cR4gw?aA( zbFyDCLk02=SyP!E4V#GZavz7Wf$NWtIg6+DL;P>~MrDc6ZFk>+MgLhTVH3SxzcUg$ zsi-cW)TdEmCFlJC&zUru@^Y$f4|;m+jUoiQ`}$_nvH;zy+xxRoZZ57YQ9mQL1jRQ( ztLVhhFr>GOZ6pcqK=$D09}VUTb?AfCP0*{L$FsTE%A3>M^kgnSqgIDto{QrK){#;r z5RCB|8u@nQgFLSE(Ts7%jhmaZq6^t#b@Lci|l4Mh&w{=gmw0l7d>^2?a)&~%dI53@u&kI&Jp{Te zR=X^cyMJ>WlsIdX5($iDydF8KosjgJ+VcGhac1*uR|Yu3pcI z86sG|<_{Hx_G5ENJ+;boMHUJ(31EPqc|z&E3mcAqSodM4Y5VQ$Mxu2|YsB@15~ziR zM3I5k!+s)c1%Ot!r^XCfhTL1WkWA5Gpr)|D3Wg000b zS^Dfl>t&Oz4ujd@AEzdD1-tkbTkwnOCPb{(U&}5mEp~smLyKH#;~Zj3na;8?{DHl2 zbOi)=49UTMK0g0DfPtL`yb4T)&MT|PWIfODhm6-ny7~+T66Hq^s5D!N6gIH`C{L#n8*Zm_op4;S!?!qP0+t#srO_3w)2un?N!kA6(IaNt$(^L zSg+bdlRAP*K_&h2Eu~n6~GIKRhB z%{9C9fUzM{4;nyS&P}h58nwnpw_R4i=+XJqq|hMXN%#;w62x;A&rd2uY|_ zVC&uQD7|$$^-1{JzRU{1xRc>s=Hyt`PoK$0%~hBX^oVJ5VoMO*mx>;LHZ??z#}I!I zSURpSyl1>sy3C`DCxZ0fjD7Si<5=v#0c_-CyIwE0=)C<}xV9ugJG=4P!NFBKfJS_X z8orJe>Sx|kK4@`0AqDx7HuF;* zmpI|F{VJ%}0TVu!7ot-(hI3JxPn}*3Rcn4oj{TVWdw%lSrh|Q8N)5z%ml0d$0EcVz ze0ZkuDSG+({GfSYV#4S-aoHHp4}+XnE?PcA?7XHj4<8Ew0eU*$&D=6ix_!)p4rZvH z{4nLmWrkE;t3K~sjkPwKk;4BMce%f7jp^@aYcXe9FHZ&I$C+&3I;e?TBJ-1;6{+YJ z07gek7e*&HpL|qA(-2VZ%zq$DDc?P_O|;I$MW*}5`jc3QEzMsX%uEvKGjowajoD9r zHRK-q;sdDH670VFNZ5@c`rNf#-wJ3O*si-zbwD0KY`#12mBpyYo230MV_DQOcoT5@ z7~6S&r&GlPwM6oKYs)gmxJGu-*fW}(U_tTmu6FHWd&=ys0e%4i?X!CuTZL6So%cJP zPc?#q{j+z*gAR76%Ctc&a@uhAA77-omi*I%?gQLp)S^8m9jxHRVcsE!&R`(oj>cur zoL%JN+s@5brcl=&n>qsK-tO@iD=yaWT^qCgL8%fH6z5cR%Y~YP3dnI7Ka)VA3e?T#p_#j2KM@9QXi4 zB-2L#2_S6ZPkOpZq7gbi?1Z^B>+J|M_{7R?FnQcX0fC5jZU7dZ0eP&}$(lAk&^8yi z#)9EOzrT_3Lw1ftuQM06kh$NhwGjdxY+UdDP~{ZAMMp<-^Cd2#!4?mBEUZ;qO^P@5(+*lA0YeCovgsfP zD8galMj54BHJx(@{OaZHTk|-^((NnLAV(8OWX1g40+UWs*N~8`*?^yUX8nf2|^&;!{ZgjBMo` z;7tjyQ#7N*#R<1vS}O}QM~mm=QgG@d@wn~3>4!+q1?S_`=YL-+LPvV>Lyy6t(d6_j z$$Rn|e7~EL9js``+&K$ia8}zOJ@yY3!9}x|oKGWBo)n7Bk8C-kswy4Fn3;x8i7@M@ zVqin;Cu!j>8{Zn3i*7SE7^s^MLOsHRDXKYEuj}Fg!I3aVUg^L|tIhoZvy6;iLv>lL zdVJM=Og&wcFhNO_6b2up*TMuWek^3g%8B$1^SGlIUoV0!iEQUOR2Y}{VIn}pY}T2D z#hl)NOyjR_6{_}e&7aUOyc!sL^(+koqx$o4+ul?YvcVE&bop!)07_q7zL*Gch|GP> zF%RnrnEE|F8a+k|JJx-G#~=RkYtW-5`b3Go^BVf!*1&y2`hAg4Dsb|GYysO(7!QYS z=H;0d)>Qp42IHFZ`ua2~J;12R`{%k+ME)a$5g0!7nc^^ zK^(K~Xyx1K=_&3h$a1is)c@+0uGS*h8{bd0gj@i&2Gfl1G#-fu$;<&GzD}0=RP!Qv z>)$@$yRB>JAHx->HeScwX6gKQKhfA(-RE|HlXJR0k7kz+6eA-6gLEQ95GISiBdgKN zC}qHw@dy-5@~bEFY*V>Q{g*Te@suLZ10W_qZpfg@#dvm06&Y0`U=?PfQOpb9EuEj# z2Kc=gtDF4QfF4-KceV=e#_bpDUPHEL*l6%E4^p^(#FAE*{awAz%6cJo;u?6m@*WrR zj){>HE14Hbk62#*FGHt{8sE`zivTYg3{9`*lArz`bHFOsUg1kMW&hu!BNW;zg0-rE|NHpkaS)c&ESN~uI3f2zvKv7BbBvk6Nt zx_wWMVTF8D1)v)2CQUTz@ADHJ({-d=^0+@gBGa*jTG<&zL5l@^atiY+Gab%BZ7~AW zbmRxF6}}5lE*RCoQ(y-2nI7pm`eS{C<3|&5U=3JMQC`rJ+4e1@oXa4bQJK^aEAJr4~x)!@G|K z0`6c9^KavnV4~xJKM&BaA3enW3Xf7lhvSo2HboJ9dibGb%LnbV7ERld6AG52OF$kvwZld6blH2oQy~?gKSNnlAZMlq~_ z6)sNYwcPxZ-4N?+o;K#eS{RU7>YzSf?A*s%Xw}ngRR5zUE8-%mGys|1EHGl=kNiXL z^yT(Rp6MPEq%#_frq{~YkT||H&voO{F_snpS)NTD+rCd*j_j`&<)@m!t;+$J->;05 z2``crl-22o&;ML>*h_VFN(=ePZVX;dm1`o|KLBA6kvUwc*t7DUIiAxl<+Pc;;YZ3O z;_OkEyRiyij5dLZj!i!I;VB5DV@^hZ`HS-cKB~lh*M}8vqeir{%tB={-X3`m59}6Y zM8z{munqkf6|`EilJA3txf1fQ5Hg$CkT9C04Isni9~b@BsX7~fN-rFR^09wPuLvXPi_1zxKetP%SB8Iayg8gc3H8jo+Wz0fFYxvU}DD5~EX)tSM}B zvU8od(B?$t)QL2XsE{hftvAE$b+06?1FTA8|HYbAfTh@wAY$OWDp>w<@|CCx=QpU5D zY03>UxT+!agtrYvt0L*W`|f}Ct^+;&;M-0+?L}D*PA*3^5w+m+j$AgX;vuKsnK16` z9^o+rQr;3R;N=8)jH1Ywnb0DEXGFO_tv_2>yuApiZk^&^48GbbtDX!xsH)N;?j5S) zB1v>-mxAuEkx)$0@q7Lf(*5|Vk3H%f^J6Yha(&x&eA3iuk5A+ph2?cHNERb&MB!Ew z()P(eo`>ezs<1oq6orIrlQm<^B8$Uh`N6~e%`^juU}dZ0bzlM+$*5Qge#rN567RQ% zRu^Kw85fjWaXzcNJKRPS;*nVjjpjZn_c-ndXvvjZzs%D=KbqV8lbneE-L&~)-7zOj z$xm#jI&JeWBT60#pH>xf*mS_~VoQ|d3 zHZ?kZD^H$BJf*3!Oor{65pIO>N@spA*oTvmwTT~cTCXFEtw8#ZPppQ&s2JjbMHRve z&Q`6IQ6)4_zXhv`|D{R+Ct2DF8xXyPFhtKjT)5)#@-H5!o_KBO+V?bK*UF;-69q&t zlKnno5}tMUBLFGCo=pFXp~|KCV|smeeIR$ah7({fZDew!gBxd=lKQb9j^CCwNV5sy zy4)o{=oe?4xo=joilZE0(XIFYmSq4P{zofASz^dS%*EZW$TM9(r=3g@e0FZ3Sk+DU zwymNs4|@fHP-#4CW7=v0ZeF~ujHB_%zr9EOx3eR%*Siy^Pi~%daD_CrZq0zBNC_GQ zd|r&eKJ-TRgrUP|DJ*w&zcW7kn%*Cw5T(}jYt(lPi@n}*4+xM@3ux=)=)$RZj4THV z*E8BP&naKoS-QU{?YoeqpcJLyvl|Z*bE)L2$NhjkFHv*PN3~rw84&|OL{GNVL${E$ z_Syfh2b41pHsSyLIkA5==3I_xsvaVrUv96&2WRYAgQkF&278V6|LwbGMnhhf{A;M2 z473oVuPd*p*#FEI4Di;+c1O#f|F{#}Y88X-$oJh6edbGXpIfXRG#74_1f+RF!!m zzD?RnlFk<|=|N%mLhsOEA;ss0r~7pd6@R5j1kFb(?$vGK!Y^zXhJlgdkMJkh7xkRJhDQGSCNs>#OpfZc`8K-8h*zGJj496mx z8)Hh2MK(m^6GT4&l+;*Engudja$Zcqa^?Ad^AlkW`U{TNWv$kTN#sdLj;%cULDRS* z+$nGF`34qE33d>f5BPxoM;rmIO)A*ci9#A0+YNy#sj|ytecho;wxvd*(k0#HQNW1H z_dISxKP01y84w@{Fq%#6`{F-qO}gjyR_uNj^MVhLUulSAQk5=*@vxTz5CE}65> z!m`*F;B~Zaz%w@F0R$*hvBLFx8!o=`@gf7lNudw8iyYVfksT+biJ}gQ>Iu5oknMgh zN;t4GXI7G`_GxX!)D94m1Qm@h4BZAozOK-3djgmnEv zFBbm+M3jB2Uv?@3z5e(;x}2GXd19t=aK~g5M@?>c-$HDlu2(hV7r{&$D;!(Jvz@q9 z`fQF2>1f5tt>C1gW6+BXhQbY=eq(xATON}T{o~00W-PDc{&XT8D}CU1p=V-|k6mSq~d)TbS-tX0U$W`X@ee;#zGND47#zk#Sj*mzWPC*^Kw66%Bs7 zdWUs#c=BR(PJsb-V=w$;S7}}Kp_t`LGmjknG%btEN&5Wb9HnP;hMPRn1t&HFVj>7A zI{&D$6lqDPVf5z<^)beUY&b}6nl0{o-CqbAsd-6~cRK397MLw}d^zG{9(t5RmO__K zw>>%Y^&hPaY!23?h-see`Bdd(p-7IV|Do|?&wkewtAt7s$(?7eG zc^ZbCzxv9jVJg!m5V*X0+k3uvuU3CMe}5!~<7?d8!)|c=iL(AN$4Q2E&HCp*G{S*c zO$`Vs)cdgBZ-9`qob)McOygP zekw`Dx+kET0W^8)ofu*-?M?V4-f)BW860=Ss5Q^=^+~^cFN8=|Ny<0jP&ZDuTNq1z zf`?Pvt`~3C#jt7B9Nf( z;e8Z`BHCf$t@yn|GjX+Uq=FwLnMth+v{;v>pJSGRUwnIr}%(U;SnRWpo!fbj;el5z4$pvAfCJ$o6UPc+J-n$;P;8SJ!I5?bbL zyEewH3OQza%c`*3(FzLuN#5Pi$-IX9lXeTk))1KY-|#7*6Y4fO>ysV?sz20S>KGFKY;U4 zI;>Qvu#Gb6t)L~R7zJI8`fHl~$teP9&WxU~SwuT6EsC5nuOCJL=424~464Z)%{0h= z0MS|y4FM>hMi~Z6GE67cIK3`U)wWuP@R;PhC`qDZV)6eLDq zYV~VyTYFV~hySTMQlGMHawZK|<(7&ddExFO>Tar?WctS&X`}uC&Wf&!XaD&~=B%L7 zAvBTr>yVoAptu~JbM2TQDG~)P9Y4@`n80EBKro8}Y6+wTMt%~)EXvK&=N`aEl1|Rw z6n#@hHptHB0LgikH1Gg|`X~HGuOAPVzHsd&J||AuawII8h5#(Hj=v*12>{YRAlHOy zC>QbIg=8csygwRk1*!PioMRXvAkbKy#;EtR;dfJCZo8@r8%bF=&nxyKmsZ(nJsP!^ zp^tX&;z(zUx~0F~xs12%rL3EA!-li^4Q_#|+{22fcTBu;h^Vb_8+a+`I*H8tqf4400C0Pg+WvW8 z9hln!VNT2*8CM;L&5DLqKm`YH+$F2~ITm{AvO-QPE#6#Ch#knbS$KxRiFc7$eb#u| za^A$@czBNz_I7--#D|ryhfh;f?~CaMD8iU--NrM6=hSWBTts&zsD zWl8sy#04bVaHmjQ@lNpV6?ICc>ZES1cylBNwC3DO*lByR!At`4H9`d`!nmHpFeei| zq+k%4ZJqUF8qJmqG-=4EHQjs^XA^JIYpwB5xhT6!b4U`>ZbbG@T z9^=sCPyEjv4TvdtO-0=L2UQ3F;QSMJ?m>Q`p`F!ERif0jdSPXr6VIB1LLQEEil;+I z&I{1_Fco(8(zW)ONUy$-Od^>Fu_@t-vE6M>apd(vhYz68FK6Ge`h?sN$kammqzC|t z^D{n2I#y*U{Y`Gl>ND#DWdP3WA~ORG|2;{b?8)xAqr)gSd( z>c}8rA96A>TN(f!AsLSDJX@CjE$xNg&v#1VNZ1i)6h_$2k9-PK`I4-p2up!S_>p!5 zF9-@*PX5kIL3Z71m0Q-o`b!2-ZhG*>2)9_5L}gEw;5a}h9x^67`@<3Ope=n%W@f@- zdIE8-ugi{hPC-bWBL$InJ?|>(&zT8<17>!-2*W-M>FcgS_To|5xsK3)KqJc5!E}Bc z*NKSr)(QgTqg! zM>keMcX5D7`foSm8kb5DNz-ZaG2LNW_Ngu(NwrmLF3WIJOb#=%kxwRPae*1VU}?mv z7+CSa-3t}jM;rUy&6W6s+5d9AHp9^9-sSvBv!|!eV`G8jF6ktnS?%obV_o-UwFsao zf&w(-E?x%5h9q$=N#WtwJO8~4VHbrP5~*AUxF;FX78O7wWcM2=$&#sspi0Eh*c80A zcv1{t02a}UC$g+!MDok`v&<;|ykHnH9wRR}zJ~zSYn$)msVd&G#tZIZ!&mY~bpiUu z%Y6UP{G20)7)+tNnEVl&=pHZ_O`m~3XV$pSMF4{UXdc2(r>}^Tc2x~|nx_*)eLK2M zNg*TRHPX&=ckz*mlz{1auMBmbDEgGV-%ugpN^UZF*;k-x;_e9%B$%5&>RZi=YBV=* zJs>>ge4nB;%*L3Nm`po|nQxn^cQ`+EeOz|A;}&{b*?ISfpzZR| z^^5F?@O$aG;JOZ6nD2d+UXu{_RX#+=mfzOFoV?`>m2X`x)AlrOv9q>%S4bCruWju-wim}HhVOu0vtPw=sTrh%k zqFG$>0SqK&I>Un}nvGJ}JMzYLUbMEf?K3CRKpj2ah+Bff-t~b9Fm?s{Z+A{)&%m5O zyP2LJu$P=I=(w2I zFci2!L7BAi$e^>WW4mwMJFE2GLuvQvWdis~yNuU437&jQgvUWlFYNpY9ue4@sY}lTReusqH6aNgb{aGuT ztO!DC&SXKr-G_{eY22BnmTaIx?iGU}**~k9Ihf7)mH+@l6Xl#d00DU?AVE}I-ByGT zrG;~X5b^BO`X-8QXq~=Yg$#SiTw8;8N<)R8ovwOqOFn+WF_nJO(8sN9shQQ|cZJBY zP_$wx(eP!$5*nV^uYVmfO9S{y))XB|q0Ff(nWgk?i>K8P!tN-BE(1T|Z&JO%#`>=Z z3?Enu)`F2#_VzsZOf;`M@RZIMo=fpn-Xd_q)!p{zkXtd$`(vHEvynxarL7f#semac zTg`v_s>R{w4NX#lI<3u`txZ8_qacld{LO{d>MEb$8$qK3hC#!oRP%#ZYj+p!Ev?NH z{2kJaeAS=-n_pxyby609;2Tp>`aK~5cVqo6O;85EFMlVWE#Dp84(*9()nh9^m(WBWPIgU3=m|dKT1ChMyjiMLlfs5vW#Mzb z%va|gyXT?tdC+VMFTZ-xK*B)D#oaf;=9IleT(4h6tjYc291#+l=wXhgNOg||gdrkC zq;;kpa8_X4~7psRP89lqNNYU{lm%>RH!9tFS9vlsiY&mVWSgCj@+3&Mu=^6VU}TK zwTV%g<=4qpf=Ye|uJ&UlL|%=ejPpV%T$(R=^o!Y#!8kFgslbd@37KCO$UPR}I<4j* z0`5}a)+f>6iK0>g0r+rI)@Q%?e+`9(ph3p$20>Xi0*+jI$gc((Cct<&dQK2*tV$m~ z%Ge|QA$dqsy)d?+fsMjyScsT9X%8rvWdhe9kVMi)jq;H=gQxP}k%+kfB{BqkL16xNFqSSC>TCR&WK26fH1H^a`eI^xQd?=w&(jrozz@y z+)l6(i4BBCG8ax_0vc!c_$jdQcG%r&eLH}VuozL_|Axfnw6M8Iqu9%b*Oib1#XjY`#6-zy}4%shaPVa z*;@oxS0wqdBqo&wR9wfc%ml0hHYEh9;8tADR(kZn*q+EV8mHHjL!1^G910#&3#L3D zz=4jd0N%l?v%T3D{EKGh?fxgFZ!QkIy_s0uvWhrUOeZql$0h$tz~aH^CH&%;oeqW+ zS_!t_Y9jFZfV-I4THE66QU~)X+_qOo_BNaJe69zfpZRn*vNq7#W zq~FKum3@Y}$l;JdnsQNesx8bvETHs$l?cLKSj(D{!IVxdJkmNZGQ#k0%noLph(BPJ zJY!!~@|>)K?{W(jL;rmHdzWy4-<&t5Uo$PH?`>^s&=`9$L;N;}9Fy-3AoUN4U;Wdj zM(qm61z;F584o*g)xyY;O#DPpn3z^u0$aOh7*aTtL?_T##HyCZ4u$J0MX*!Ygsl&O zquwV(Mb~%?p$DctA^q4W0>HTFcmyuwMGAfA%%WA~!a-MT!~-Q*`a=(v_!ivoSd9Hy zcOh5BW4N~d*GENfh6t60M>#7L+4-KOG0^c$vQ4fJ&P-%d7afXx0L-W*I#I_(_9$SD z|A@L(JS^3g_26L#$P=@jrHqqRu>wk|=m+q+p_5hp;t2zK*Ej=@&t9Hf&xX%Fv@YPM z&RCtiJ;QItUU{RSlk)wu>s#p&Vsd8};@YDk5Sapalh(>=69m<9FIl)2G3KF07}?3o zTM937TQEbb--f%Gr`tNTU}H!vruGBdJJ0eD7DGb4{LU@z21DOm9#5E+gSxm+u#?{=wEH#01dU7WsUL=VNOCGL3dUMdz46L82sSYQO>M{iSkGp=jP-K|2^?$ zul)(0&e%VHlkeQ3*5)rc1etILs6h<*G-ehktabeTaD_E-8Ev{Ffd?mtu7(8 zSXjgW`SFAdP{rf>@ug6f)(s<}pq_#_qnl?lCd1XZmMxzMT$H>%8jaK4-)ZB4$wq}$ zJXM^?q3_=_FEQkKu{}nE*i%^-yyHR6e?%eW6_Ih{|F~KX>g0B(Tv=M?C7Wt_Scf*Z z98{}krYHaoo(D53^_fH1FIE7yiV7Unmn7%3t!}>|XCz7-#fZ;n;%B!U0jT*HK}7r5 zhhL@Ej`?kmQZb-v+i8rhO^3f2J?c&{p{uenhMVa3XP$cV&rAnyZaY#i!7(M8oW2of zmUo$Fm)V*TQTBdGhBRNQ^aIBh)_V}heHstmw*a@guA<)#~*bnphAh#2S7bo z3E2^`-~;78kf0a&0G0B@9Dd8rV}!D(-&lPsz5ymCyVOBOFvhEAo%xPk0r^e1Bn1 zhVjd}xD`Eio(DTWnW#^yBP=9YNPTR$V>WMYvv+DUsIWkgk1$N@onyK6-qip!rx3By zrL}n5mWORFGtrR3!(Uw>w_C1oa03zm#;38*cFI}L?m;xx@Mn8yHbrZFRe0d-mA>5E zX+4uKU0Dg6(Vm_kUFeBe=zV49Rb>*3>*DQljOo~A4L_*5vRm?l3D1BQ+KAQst7Ks~ zeL-~X(J^|mDV5{u?mnm6+mA+jrS}pp@BA%%HJlPMN+yl!pEBXEw~QItE0et#QdW9>u+uW>U}uBV z*8~~$y}mDpIg<3pjP+BV$K1xv<1F|a@AZQ2?_@k+7ptqcu08q}xP;48dQ1<%wfbjQ zs|o$Ld4H~|`f4B|B7%&Tq=KU&h#S-Jts7owIY4Ge%>DQ=U- zJCr^#VJerS?JR>!%}6jk&SZ0BeOjVvkfHt{W#S>Q5CtQTCWKo{?^0yUdHbpTg43IO z26l{aP2&Sp$kV_K!Dsk#azfAV{)(O4<5q8IdqB}1%l%5pSCfV96fa3TyYkA$p;z{P zGPtD+^ZuMqW7gSqPDWHvSb0hJ#vfM0M>6s(uTEa!11x1NU>DO;RBVLS?vs*88fGtiW-3e(he0ea#KWmutVX4oCQG?ow5mQ2O)Wf~ zyRBiH13$-Yf1&hfwLFF|#87Mc_i6gv1|JSx*C%-p(CZm)Ko9rVYKD08xqU)hB)wAU zWsE95Cm7#d*-%-Jqfq}b#hH3(%SXIol7X5bW`NmzO+6Z;@y-`WNOfCoke>6QFYxb2 ze1O9DC6Q-QVF*}p&pTcBn58dz_Ej4jafsf~x5L7@e*m9De8;|(eRVU<3tz4hooTAC zDOitvEpYlfiKZ#_w|+igE6Omfe7JRWbq421Z8uH8Lut=5289U<;aT|)>lm0(q5=@w zK?iT}S9_u_j)xou=MCG>nNyrgzq}njsXGdLnOy(`^gfocsGXE>`ed^6_wV1M@!l8i zSKF03EX-5=F@IN%k8ub66X~Efiz%PR*;Ai{!?=J2ogPb+z=xSM1NA_tzBpk)M%JT` zxu3n94@m$`O--26AM-k82BjJGlRG;*qEzJ06$`HtQs3F?n=iEAcaNUMYfcKc)ss&#|z~yfJ;Cj2_((qZ2z+crGEIPm6O4{10kzd3ANw z>eq*DZ-leM2fv~QPfzQePROUj9|`r(TMgzM%>_{4))>ZkUA@sf_VIk*Zu_%#D$}py zc8p#%O3|dn-FCl-(~oO~X!d+Vr~Wit+di&d(M2iS*g@5wB^}na=g8!cArQWQ@CDeV zGynphVBe=GYTn=-qL8J5Qg`3Dk8j*gy*f%q|0NpUg>5xa!Bc&tjn{1@ zL|0wC4o4+6GmeeZy$jFYj(qJvGSm}>&Grul3iz?BQ2C{A0w9|p!o&$dP1YJ8pL5Mh zAOQH}3)}qSVxE%-r~HCC`^K+m`fKn1OgTyE|DCwbC=3~(elZHrLmWt1CCG_%9R8vf zF$|c87n`wBtkG`24{7Swz9H?b;A&0G#I510oSU?=+(c-R32X}fbT@>RH6it>fwDX& z)uB6{zWA$V0RaH77pc1)bC7X-0`K<>2N>rqt~vM%_Hk$OtDp#O=JG zJl8NKk2G~^b?9hnN^R-Hl^GnJt?%#W5_Dd!%#Lo{l)QLjU`nEE3kE{LnpPhz~TWV-8DWzbNA7aARJ|koWz~|0J%YQK#C=#111sZ z;TK8`JZTH{xZRH9??9t*GPexizZ;a!&?ZK3xb(2Epa}gH6K&+d*jbV+>$JXpOOu|pmmmbv|B9HwczFx9pNJZn&DA@`Yf}%o)fTa3w0-Ydn5Rg zC%i{kQg}FskaIhD=K(z0F(|mLv;FqsXf^2j%gI&THMm1}`6R+n9m?RPs=Kgq;p<9l zClG_~OW>RucM)|7=44m~vocIG5aZL+8v8lBD>pIlG}Ulk`(4!1H|M z^-#iT;#c>J!5l}xE|llBKd-#@jXoq=FseX%=dCJ|Qy$xG_}dX#UnbylrTr^e78JHD zo*EbV{%aRt2|0j@N6o@L<=~r6X;Tncs_d7oADNUA zvOg4iyjWwj%=_8SPNZeGy3`SY&cvf*k>r!}U$b6l5cHS~{^eku657MK0ns^Oqr{CM z>$+<=ua~r)9r|AZiLmxOkImy1;P%jb%&b}|iO8rMgW^~ZPKnxWRsjjG)|!#P#12#F z1>>$`btz&lR|+|7)sWi#WBXlR(~+~3ayRZlUT(`LZLM=G|8<7Ur;i)Ci47$GKCy|?UP zae@2Yt_LeG>!dOKDE%4|IzKMnle)^1&6zr5eol;QUD0)dxImvc{jbPR$rF|pYH&}d zz4_L>^?-$g^ToS~M(oYeYD~=aY-MGf=bPKR`K!Z2#1btaYyPvb^$$Fm=sH8$Tfx_} zV|Dn4Ffy`p8=sdb^PyI`&RdThTKAjXAugq20VAH;)Ox@3fipjR#}-uiNw$t6a_u8G z7T?+Y1qr}CaArza#$q@&uK0i+50y71zu&^9){{_{NjW)?79yJW46y>Rp?_*CF$ZWu#LYZV~&rTeV@jKH9MY=nmfiRE}1(A2&k z_Us9xp%(qhWOEPeOpJcCiLGwfyB2}u>%V_(tx;I{i7D67Rw=Xp{!9EGK`z}2rqXj7 zMDHq+1(-F4NLu@v14fyk3?H2fIISoxnJ@FC2=rj3D0$^<`*V>&Y8&|X;-B>#YIgDs zRK`4d#cu`vJ?oOla7Com$T0Q*jRo=)yi3DBE6p(8GR}Ez9BH06h$}rBi^mPJ(s?s3 zKZIahdi|cqi9E|$6O_Wj?=qT`LP&+n+|ChM004*l3={0bPXFX#s=FofU$vqm$3!ZW z)g`sGgv@riK^muD$)dZi@N&EWq=yNLN0dRU!+Gj{t~&r=FLG#VxCj zYCxcP98)yzOOb-P)S0>GpF$@KC5X;9j@}?Mo(5C@YDS8myvS2D-px2#4;k?1mM>NW9Q(a4|3w4p;ZcqH@0SiAK6RiP4* ziF6l?K7t@7PwxyY5HLvrSpuUhQHdblU9zv< zp*3^;bg(C4LJ1koe$QY*pJH3{YH89v4qe^uL%HU#Quk?g&zDP-oT%P8y2aj=Za*>J zfuI+QEpMZ=QiBBn!YljzctT`r2kvmiq;%UeJ*_20f0c@-dzq4474DJ5d*lvG=`Ct{tK zLsJ%^Z~Xq3?~yxy;KT!eXU*TMu`3jhuLjVfq+o*Z(`G^cj1Xso15}zW?^4Ye- zLx+QD0NMlU`?wpPIgDoEgB9>-c|TDNgACG1()ANF)y!)uA2~OkAsZ5Yqex`4M1L1b zoEn8b1gL;)zPtY+!Pn?2botYVAUfPn6QPXZ=9N|jTS6^K+KrpaslyYL|KscK`}Kqx z#qPTwUrzA7FG$a>InmV;R5V}rG?c)-pE38cNWvC z#B=37ggYCa0hMe&)^A5!+1b-UoO{lJ#s(5*2dpsG7D;;y$Hn!pNlK%;W@M;ZDWSS? zHo#$f7IicTDODWtC5r%A?0x;?Ihowz3VS6mx}vn}Kt!vCm|>mVZ;|=8Jw1@DNA=Y+ zu9d;tzqj5dS$yY|*Kw6CmA+;z)dlJ&h9p6#sw(#=9_GW| zzssHqrx$f(QqvcbsNERGZcq;rP5)+S@Ef$cM~G=--FqgU0>BYhGqS-=th_j&*8B0^ zHvBxzry~AT?$S(IvDzi7@ae46`tph;Ox@E!c;TE@EfXrU9rO$zzT2*(%i~N_aW(I^ z$V3tIx#Q{>*M8a7**W2By}l6iw4_Xb{FxYt~C=L3QZzo+)>1me!20=B zPJb&8C@T--SzQ~nKSO^d9^NKKBry0#Nv&9+*vD821b(wr#9$D5zyO^Zp?o0Q`ZD0# zGi-B4?&;&-MNfe%t@DWhO~u~s9JCWpy=F{r4Uw9!n$wVpSZ)8==4=?(Bj^{cI*dfS z-0|14lj-Ujx}X>hnOG`$CCSpe)UG8N->Ps9R#Nl#cP8V<#$ zkZ@_v8vKpQP{buwE^G8CRoH(#eg5o?rOvU8WRtUzIp@U2pODf?Tz}|0l|Y^Hkkse7 z0Kku7xva=lPW!Zb$K=B~MzgX*W>)JQE-M>5>r$;h{qtYql~w>&GeH*hSqvIVv_uqs zqKck2xo54mZx93{Zv^#!;^A9`ipMu_`J?=5_g{5n+GvxE+9bmsq;fWBGVIOvdH&7klH5I=B?Cph5_xO7Lo5(z$h7=>kdPq=Ep-=CuY%C?4kP zd4mAUnJ^n_>nm>(<#~Zj_zhekmt0Ibonk;yQVvbr%L^a?Xh7k3@wmd7j)0RZ+{BO< zeEb8qfOdZq96Z+<;$Z&XLb5?qeDHNrL9z=WlA>GR6IVEnu;wcxVjNws&2@4wT!nTTwFSVaZctRHsWZP3|<(GI^* zxlk)he|Givb+q%ByKL)Kh##;n2*nAxGyMcGmF1^5*jZN1nQ%li~malEA7$33fTbubl6Q7(K4C{m**_`5IBHoTTCzbIqqwe1X)b8_ctSp_ewn+O z{L{E3VRGIEoT>oo{*{m!a0z|v+;Y<#=rM3=4Wq*MR0z=x%dE~_jz?p(t6~l9it??N z_DP`E0dyG-71s1nNL)5Ss1P;7pB;U52N#Db0=JnUXP#FU5dHT#kPK3&mG&Vo*Afuz zkddm^rIv)o_^v%Od=u?AF->M*gZa?ANgcS55-#y55YQ#${{Sth##;DKgS9aAPyADr zFYaOw*!hgsiRDG4`8i(5R`(k{(#TQG%q*umZsBd6H`}49u6W98GS!g%)73%=&0|FV z)hrHt`11N8nw{%!7d^GqKJsyyoZp$zNO;quzr6aOCbA`0=;oua| zrmPLO&a1P<>(Gx9O_5nKxCK`#drL=?f14C7C!SgQX$(uqpM&$Msj2(D&fCuW%e;GI zrm&Cb0m4C^5?B+OtaEx#xpC5S3>Q)Nmdv?+20uPLeCd=^hA-Tz@Itgm;bf4GZt!=h z16Ro>jj0f+k!!GOWn@PKwSSdpl}%dztDPrJtcWU{?+Abj+IiEsX#2rkHqYIbFFcVO zsY}B4;QX}{PNbc3tNtHZ#O?mfZ87-vuh?Sn3DcihmX5rQjAxz7>pflAY(DY!&-Ksw zR@y(eWJZk(4UKJiTl187(ni!z&WGS`_OiSXUz{2eBX_g4y4QXd(pcB)nnZsPa2lrB-c9sqsb8QhSdU z6;)epRl8!up6~N}^N&BRCwK01&gWd$?QJNrrcCF-*fmABsIiP&B=g|5KIb?s2A|it zhP4;0GYwg>Fy-^kDHE+vj%9(-2oKm1)l&N3%D5K$t#ra92yb;a6C5>AKRa4-k)e% zgx{X7+~2OOECK8IhyQDWc5=VbGDP8|<_ez`IZNu0OTI*tw;ji+bk3Ra3B0@bb^qa$ zv%jy0>KbL7f?s@p@yw-kz`N@5FRmp!6>Koz$`Q?ba9Djf2@>U=rR4+Jn1 zk*Tepl^@u`S-6z{kioUzyw|8$zIGQ$96CMS@apJp2Qrqy5>13ar22wDfJ7k5cM_n^ ziOX$6{x=$__k8fev?5R%Fbdxmx&+>1{EUFbOrb<>XdsgJk9+fVfdBQ?$*w{7-RSz! z#UOyRjjlvJ2Q{#sLHHlOkX`g;w#s?Q15yn7Q8I||9s0BJy3qIP-Om}SnIaauCY$TV zIS_Mz`^Tb+VjI*suG#=FLDE3r~vaK)knQNoAuyp6bO%> zA}wYiLn?*(3XXtfc!&sA#c3t9%LfJ8`}3Lgqfj~2uhh-B(L_)YP2i1>8hWOQ%8Gg4 z-pD2u`ez3k;7mNp(D@_G9Pv3nFz+K?i%F#SqXbLM27dw{ zALJ*RMIxba7WgoM+&>c=&Xk@=EfzRqr-#2k@A)C~P(W!_!WPYKkY)f6k!wGjTyc75 zY^^F8)h3ydtQoy6c|%p7_J? z0{CMFZtPYoZt)bnQ4~2@Th8&^4ztBGqs#~beem~Ya;7X(Yi#m$OZy_4HW;;PYhUYb zFOa1tp$Sxqz(4rzRw?ajXUn4F^5h}Na6ahz%V|@aZ&q%Jny*71?;k#e7+dmMmB8m} zPeec>lXtoXWl!b66j0u#8Je8Zw%`_l?AtI+wu6=1R41d7j2yTU3Z0{z`pHYXUM@9@ zbbsb02Uaq(kIHFM*aQh?1!>vBh#tL}T3A>BZXRJkkP>MDT&QR3EZQah=9hojrQ&b+ z??(_>u5-=6gr12@k83N9fXe%{k;>}2{gXNET1(RhemWuzlWiL9OXMqbs0|mLRiL`HN zhMsNpGKlOy4iVlCndgan{^;h9Xa%eMj$nAisvy;1L@M=LfuBw9V_9)=*A-iQqIG0| zsRnS0G;V+yG0CetO2nXoV7@+@w7*&PJz|UgAl$xz{vO10YfB%dICCI3$6t5`!4)78 zO8Rd=P*lj~ya&5VkjBOPjUR_Pmo3K0z}A`*GFV39rA%`-j6@G{sqiHJ(-gZvh}(am zR2hpxVHA<1(c=QzIOtOD#rFZLel){r%oT%bFus_4>|9o9>)Kq8sQ+@|$tHw>2NxE# zt=qg>Tcx6(R;jVJ8;7hetUsIj<>}R9`dHoI19LCrppBoR7%|XROPH28@{)5_%u>s( zM##ZW=nPTZ2Zxlat4F`>k&23>h6mfF=Z&C^I0`r7uCy3Kl0F4>QRVbW5IgcJE9b$Ms^cPsY*4s-opo)a*YNB#0x?gq9K%9t7b z653@3yZ(J&rGbChs0z&a-Nv(!LB}VV7!JP)@YdP;YNNkzzcxccq7hLFr7ExXXQxQJ z?Wog62V=-_pwNtQM%jf;M`5VMTI$2`Wxy)MyON)_T{qyzn0$&Drx?{BxoR*W;rkm~wDVkgk|UUL?}&!HPvpJu zq9++DATZF#C01Zcr6_~wXYe;{Hvd0@^1_*ZS=nkb{AbL#>7`yDgLGc(?j8A8@Q1Dtws^i^?RRDV{sopPH!OF1D|H!;^>$~Ye!z~TI z_z_*T=jYA-BF#WjJBZGeC>}B`$iqBz_5Gnfjxs%6*p}15KrwAHD#u2}KRn8l1>|Pu zt79N9$~vIKGR#Pk0MBk;v}VyDE}YvVr$90u>6fJvPX^x?&ohw z{1@ThgYaRqimcG5W8Z>l}qoFDav->jX#sN(_Fc2(16 zl1HvA3t4-{m!xOKFWJf)UL#Ax8REzZB^0D zmixOkA7k;!?K<@VL0if+nn*9oHJf*b%8%YkJvsz`uosk*)J$rA7>FB&+w#yeLbmTR zRW7x_(rh;J&K8%=AHi<>Dpasxa(#VisegU_VrFglp6~JqPtJnOX}DQ*4AtR=1IcXe@*+bbT>lJeb6vVZJ=yqBLxDdBQ5s8sGo8=?@9kpwU7C4cc$S z1OcxkGXw@PIoSR_6yMJSpHWZ%OfI{1cUN`i*NK7M-adM6ZCQO-$(7np?h#@V!>R{_ z$&77(0`|A;0I6xg`v?9h)JuN;nx;QqFjkv}qWk8K2fa$n%P!dcK2}w(5J5mZTuui@ z3VFNnDF>x$7Qm^u(CoqLx;*W~^Emr@a-F``MIu)lK3Y^wO)>7?>{3&+7v$cxb-K8( z!yk+Hi!{i^J}M86JB0ZyriE2j(zaPx|Si2WJ9_?Rc5F~+?-o3+w!FJ6}to1#{L%QwS zOHx`=@?-`~1nn2f7?kvE*zzRz-(F7m7b4FP1#+_d!wS`AqO^%n6t?oVEfqX|&4lEk zs?<(kL63?DAdofFLXp6M8uSKJ#lKJd@(~BxJF{}gw2W70DRXV<9XsxN`Qu8O=P()s zOcaE(O{!p^DQ*i%H~i-(;UfqpRqoSCHX)WRG?W~k2UF-}b@53xT}OjJWSC|6ToQhQ zG76@GoO%RO7FIYuN(|9>NvI@f65Ci6`}WIMd5kemDq1f}`wfcT4L_oP?QkbwMBCwm zV-1ZpN9=Zq+Ud6m%@n2Y*Jl@w&XAvl--=p-&-k&U2{<&h=B5W^!M57_5On2JP!92r z_1?g&71uG^sy1iu7*4z91)+V`VOrFF>F3I!MPu?9WA?)?u9r``3bz)n+vDToi^8J% z!)hDN%^4UE4j7pZNCaTh;!h!*I=*l#9wZqZhf<8UB+rJ`Y7jjxDMt>3MKw0?J8%xa z)3l&JtN(P4ksjrr&faSS{`QCy?SSP3CqsjQzoHfKdUN_$pyIqLBe@ypnnPP|0nabL zGe<@M55jXjvspVcgND07nY*L>y6&5Q4PzRBFR0nSV(?@8)@Jq#8&@P{%s_O<;~3=- zGC15c-xlOWhCUJP7 zv;m@IU+Zgc?w6GgCD1bnQx31*p9 z@Lr7|4X46sw~7O{fm5^7-`~H}!jnP{8=pP(5w{BH8(Uh!&56hni>%H{_pL2v6Ft7i}5uafsH1#K8X6Yl;UHxac*LmfNoBvbn6HJR^OI@%p$u9gdUjH=XVE z9i(4SkNuP)bznGE{ww-@GQH<%&dje|S@ytkI1S8x=R*`MEm|M_?szCoJNM}(h5U&^ z^JvgT#_4jBT2)K%UgGcK3~PJG=z1uuGMnugPzZpxYZmGabrSzQ=Yy*%Prx4)<}m`r zK^{9&&6AV7qAKJ@c&c7TD(KP#{s)SB)(7_iaxuZ-BwJreVZWqw^(wVE{y{g*kbtEg zd$BgyoShx-D|UBcHQm)MioOO+7}Co!i{qH|+zO|>EwN7c@qU=qR-klv3S)9rjxQKa zYyh|ymiGNlM=Odt!%l%r5T=4$1}L?F%T37Hu!z|HznV&+23iawr!Gi<=$ogHh@Njl z&WH2mAcx*1_q=l44A4?-*W0Y`2?|K?x7%AO1TgzooC9EkxvP71-FJTyUl5N3V`qd1 zqcWJ7XVoyI(_@w+EbP}XwwuSR@3u1`cS6IfZT@3$cht-mr@@CfmX z0C70cuFTiqu3@;CwOmk%ap*Io367|(FoC7PF>h|Bmc__D;d1lR#kW;&pLVs(No)fn z&~TwIf60{&5_5}LX^0Nau1nqkdA7kBRSxFkqcxUp zY|*Ki4kb4DZVV1Fx^{Z9pRHYeCN(QHlDq)`z39o%j`qGmFOf;HVhv)E%*gTSbADKs zr)w1e%bMbQ1h@%Y4xlW33d90q*Gp}+&*KD6mfKIC^$7TLcAmSBqEplU`%Bvr9wJH3 zh$Goy_iW{9WKw)bwhha!6SQ(SKjdlUj=E79=GWmgGg1AUUljF2GLA zE^+Luo}fub2$;fb=dO5q$7|WFZm+F2=x52#$tc+%J#YVHDY;HjG*#YI766NASuxWK z3=a;T4JJf1o0kXW_>a^}et+#XSEa6U@f(w`)%TBKk8=yZ-}XBSgjU4{)e!O&tVsy zc*bE^0D2m5x;A(piOM$c0M?e5UvN!Xn$1rHYhKeLk-evspn>}Q^2!Vjr~FO*9OCGP zr^L&w(D$f5MtdJiYj$64RQ|j96sGR0&6n*hU6(uW>qD=ug2bej7ehKaI!vuCs#hiP zY{*^qGw8qSSWK*a_{A22(@REN?!L0oza6>uud?o0lG=zM)pBOXh}J@#;ybmOj6{U*jhjH!u8 zaBwVz^!=STwnOxzW1)^gp{B(BpMbdE^sjT9!0(FG*8sY|mrX!Oh}w{(YoXTqyc>LW zYZm<&o%0eB>tKt%R0Mrp79+SEva$rFIhq-)49F?4CA!Q7EUP@+@ARN~h-R^bH=EB*DaYF#rXMe?(f&Sl7>7P6_ra~fvU7JGhcFH?5iAF z$(2SI8zL|xuT@lrlK{bnh>lcHmd6flZ6#q^=(RR#8#jC5whujwN1G_X%lmD@lZ*32 zrF^z-<)^NNbXdsc%{HwLZO0d8ve$ja&-o7QTwF#Q==5DCHn7P2_1$uq zh5~?d4ga5|H&~j9V%#H+ORW3c*iu(H63sRAI#vMchWgvV97FLXN8uncCzqbhpR1-# zc+XAE=DGb}{)b;A{xd$1qKtszcVF#3PrJIg`GnbDTh{|46`=w z?(XtEy8Z`h4Qe*tT0i=^R0esqsp3Tb&YY#S0xlWJC>7KqL&@zmr`PNEH~kwsm!hKsH#e8B3%BU;-M-AHMd7N|75fg6|1&$d zEm^#g;;pxNzXHZcnfHG`SqPh4o+qgdp<8VJ0XlcM+mA;1UH>4(1aG4kv<#C6KUPUg zO9KV&;EZmm{W(2IN1GsxRWbU-Q>#m2S4aD6H%zFIK$=g;*s;jd?>65{YNIVU3$i>_ zEEFLTziO?j!ZN!8C%)%z(GUwfs*J@^pk^6;ZfV)1Q-BPCkZ-<1IKtfZRgY@-f_Cg{ z>fDTmYd}wgP4)ClkWjfAJ?fg%QdV-rZ=SrAFDoJ-H5XS%J#G2cGOEDatJCVqowYmnblDt&h|kJ?|B*@YGt(al zZY-QB5^r$juGQ~`)#I{-z6CK4?|~5T>*4pIV)p#~ZY9$CJA%^yo!TIRnG-G+K>r|A zGf69oAVxi{2 zr4_!iJZ1bgeI;BSLnoM}S%m6#{aQ@EYywKQqZ(|kw-H0z8& zpKINwZZkxlc_B-seiNzG{DG6DaZ9st>bxhz3aWcOgN;wOy?~BIR?IWC#|sUKrT zHez=})u#;Ns^{$OUG^}HLw#s8&J-{S0V6IJR_@l@{Vimqq)uzGO8~ZS`ZA=jf)x~? z|7#xo-HTiM*HVvsJHD(IwdJ-j3I~^z9P9}pJP56L3zyAR`V2@4ZAK$Je{Kn8Gj-(RgaGf7K{Llbgx zTTLuWUlBYWMiuRvv7ylr#`ln2!OGqGS~9-rdE-RSX35~=I99M%S{nCp>V*10)D-TU zVL$l+YQ=#=AAop1--^naBF517INaoXh6!Q0y*ImFDrb4U;CaHw`?WzA4bcN90l9ysIsUwxgn{ondhmW zmuLE(QKO@BthF-Y23qAF5vwZXGNUfFEEqH^08nyYS>|eu1T+(!;0nY{+&2kU+E_wYJ@L0LNlz9{V{ZE)tGNl8vaa zuFN2Sp$53P02vI$3RsB^7lNCt9v1rH;Sy`AR#Z~yY6OS%RUAc{r#g+jdlrApq~I&q zYwc4m${f+QYEEW~BdffMb$OZQqPwduvpk~{_o#o+rKHrCxunH4*+?q;g;2E`3e)yX zu3(M46q;&z(US~0J{x#W`Ky#5eirMD0=3woTxbCF1#_l$f*Pw$W3JG?Sh=-*?LQ2T z6|nsuNk`2L#^)`i!F%-Qhb|V(A>=*LbkcS9Q2svT=qevL<6HS3GOXy!p zF$W82Wd@GKml3s1C)rHlH#-L1L3h(4-51k!VW*eH_^XT7?H>Pi`>=$0!GcRJ&bwhTL-y9i>N3Z#Nxj%EgV@Mj~DmX&-Pj#m>;J> zt3y0~Q(T_xOx|Asz^zG5CcV%?7i}Fq2EG|zb+-C;@>{V)<8di~)0-Bru&emWc@c^H zKnQ-zFNS{GVn`}Viy=3yuH_Uy44J>$cT)pejprgCFdaCjq%(lUuzL{QU} z%qGkSg2Vf^sdj8^EIS+9X3m@LVV9))Ot8X*z!4_`v-=bCGilpLqFMc@rly;T4*tf9 z$s(R0V1T*4tQE&2c|VfYXzRg+(gev|9>@_z&>nev;c!$nLMeK5<`o(KJ5m zQuC#~ts0|1?hS)6+NT=2q`fKWIy>8${MZqS^}IKp^%JXWd^2QQ?KHb=>=6p7mV;`y zJf(hI5*@rd%r8&f1BsSr_WSh26QY)ZFjuh`#M(K9-&lm(_V`uA4tHyO;U z>lNa?%ei*t7LHoj1mmQ)&+n{P>Fw7eBSzCXWf^ajv*0KgD)c`+^7Xg>T2r0K3B5J- z8PonUIPk!zWLXs!V2`7R84zgnm&*&FlF#bj(|rPl$n0IHEKGanExqjMhoN+4L z3jBOgN=4ykT&V%h5wj!&u2DxvM?2V$wTbz9_8c5%M7RJA1&1QR!#~l^2!%lY!V~83 z8*tT>U2TgE0wVxkHdyieRZncJF9;4|fs;`n_z@7^cx4WYTK?(ijNe>2h&j65`O%pQ zl&8tZW(1cS(DPHR!KI|Ct<^weQgc$p=Pw-_)iY(hhv%G0_hOw4 z#0+{QI}PODK*O638X1sr5JtkfrN7?vOl@>Yj}RJ6sLp;VEs)1s2QMsqB09rnPfaG5 z1Zv8t=2Pyqm{yzCUU}ztn{UQPo|f~nZ!9k5A&WqDywa~MicEeZpV@88GRBp#TXl=m zqsgb(L&#_?m(~G+S|1j>J_$TPcNou*>$4hYbyak>Ho$DGqw}bbL^*a5z`A;6(K$u;`v^?6Pu!=dzV2enk)LCFX4b49!JCUp9NS|n%UOc;!`j8q*UHQQ_U=)RkNAtM#q zW@@g=?MUpNEKT{m55SwakAYluaQbR)JtcBSM1@E3VeMyjW@*{|M`6;-3}g?p0eXGg zS)eqPm}K)~e@n7if$ z#6gxO=KA(kNb?b^s`$z*ZPW){f{2-!ax?kKb5dL!He5KDuuDn)Dp4hK3UzcW3q%VF z6HYUqkKuQ;+55*-1S9djC^vl%e#?;B=47y+DC{qEupZin`cO*RE>Y_J0%bYiIVFdF zppdL6y9%L-&MUYJOw(|r7HqMhp4V9e*^xrtK*zC{Kp!JDo{hmOOYfapT#1W|`tu)q zA_pD8V1hqwwT`NksINS4-%J@NtYQaK!f!F{frUCmLW8=Zhm!_EGihFHAN&NYR9*OA zaipgQ0_xYq_J?+Q&X#NTz&W8N_u7ALLT6M0;r9^OIVXAJx`>A${TVr;+w};A}B}H|%2aDh}%oum&t{-j7=Z{#8q(EQQ zylR_&jZ36+13oD4*Rt$3{S&erUt4h<{h2U6mZW}l?&)f8MCOKJ1t)236+=Huk{1^{ zm{+ZbHAy&hwC0HRs2O&@3111b^XPTM#z6!>6#sxKCz(2PmmxmKibJh;AFY}02|iYo zOIZ&IINzM^@2^xHuJ_*$-Cq*9ymB`V{hV!`q>>1Mr3Lyov)w=Z2*shInj*V?L;G19 z)V)&_VUciw?l>#;_fUh64c8=Uk%=AuV6~1?`|s!HQ%0=EkQ#*)-0WJ|##*k2QcMA? zI2C$}C&z)WX5v!gWN`kEi&Y@EN*p8r#?|?mUr~|mKyJ%3N>9&0cdIr4XB+RVNqp4S z-ab3)*J|2UiSI~mb%vUeU`1m?#0!6JVJEkg+-lxfBK=`JPKN(gx#orX6+nceh`J&huqh=GwdgWcgy?J?OFNn39o6-EGo$RXb4*Hv!h566A zG$M(ng@d~9(;A;sXbfkk2k10 z2tOsiKlBN^TW7jIJX>;!nBq|<$9r=K`RlbmWcGxkUNegimoIj!xp@0Kf9zXq`Yfup zG~sAyK)6{eEcWt8SX*af(=gHgtki;491h6IZQ59(XbhaA_7)f^ExrRja)5mpAMxAE zvGBW{K_-DuFSfk9jz=bST6YU)&y3eFPY zf2wZ0J3i6PQQoH~Yfx*XcV;OmocOXiR3QJ^clyK_ODDKnzCxBsh%bq^8ZjVBnbvg1 z7tm-G@0H^)O9FJR+iF48YJk|@nl6VB1!2jL#Zmp%HRm;>{iAaD5gty6< z^JK3tnZ0K})qIgHPzlJdd6|Rv|l-(IZ5TUk4G|oy) zqDo7L-W5e|6?^%a|#+AIsR)l2io1uB-GuMs}*}NX7=8^~~mZm*F*4m`q$RmT! zR+)v)vE_>;<2n}PMa3Xv*i1S6X(xEC{|lLFEFP@TEV?_ckm(LQz4G+`*Lm(cI^$(t>ytVB zmZPJN3@*nLo2qNVadWtPa@YBF#`!&nm*^R)SDDR5^s!ke=~gt&b7g<*XHN$Ts-J>u zesA(|P8>K^8NIbX8zXa4FkQ9Tjg24)`V>pXG*+gCFgRn8G)~X4S8c3jMSJ(%;gas5 z%nep1cBlFNDt>Tm%pa>;G0&4|{Bnfs%8NdSoCdcnE&KM0KHbg!DMxNB=%h4EX-<&gJX944NmZXm z7|sg8C-Ou^z6q+Mv6mrXL06KaH`9}Q$7iF}BIMn772%@YlD>Kf=_wA<*;DKC#+eFi zZrn24$e8xQenIz7gXV?EW`DbG!=thQgtYs+0TvJ5u5_DD6vP@ALf(1i_AHaGPHN_L zK+jxOQv$N0((vFGyERssuq&Q%bJpx*H0FbM+$kO6QT$IS5f8zE&v9(RwqvMEO+BJq zwy${1gsv#wO77>Ut*20zq8O?gTq5j!9MNhCA}Fn=#)X!&7mdB7=HjB%vdc3pl$Hs* zyVwblek}JQR`xjqm=D5>E=f_Kkr!5iXp6h0Sc5cw$$cnb%S>YWTxjp^4Rkt;sAafVyI->!HVcF9od2c0wz7^!G3m29hQjFPAiuOMjxir%H-Fz&3^OYOl2gr{1G;CFU+ zd`1v6@-%0kLfg&aeK)4OhI@z&g*G+nqsQxW4HUB|v{2SXEO*wT=8ce3Pb?KRMXf(G3lO$aXa=E*ltYGRc3O_m(8M!R0pW;_!XVQZ=xtRRl^ZzC*G-H8Ce&pq#-K%MXI?U{8dZFBQWZS~D zopz$Kd@)-zBSiq+l50c;lWN*4&|wCLJ&iTp8f@5KBNG~Qzjk}O*S@}kna-ch z9vRVfh;me4%B-ZchFDgDEXf#oW*?AB)P^1;9Q6K~cl}LrFZE%C;Of@tdT&73H1-8k z$En&m)|<`fCmeaAR5wCAW*S9DTuIC{Pw~0*^KWsk3jHa zxbCLx$EB!Xli3it8r;bYmDl2H_Pc2Zo=<=Nt5jkXn{AMH?WRScq?4%>Rkjt_NDUcA z`Mz0{(D*cLFVbdWCj&c8`y}ZLm`haGjDdyA^rQ5*9a{T=DTWkFNE9^(S;x}kztp_k z--l%%)!#A~jiE6Sj%(Wl%ztebyJGR+NPg-fgLhfWveS!^pdY#wPF23rXn-3k&&{v# z$3BgZk~)fl8$qz}R%0?c)NF$dg^cs|73X+Mmv+K4ccqblO5V;=#x8juw6ULC^MzYm zST^rH1bJc&&|8}tr4d>-9GPXG9og~(%$m0Zot1jwQS@`HIu}tfEFk736E6bXntXXE zzE(?IR0=;rip#nd|^f zc3{DQ0#;i#Gaoz_ni%j%H4$Q2GEigtC0&)@(tmIgnB3n978Ml*d4LcZnGx)#zZXi0PjAaJv2e>g=?AWaRi_Cps@%n*9&L92y7;Tj(ft2)A!Vv}CC6U8NMZdPQF(5v9HEw;V}pWF8^(c{U$U8Z zaYq#2J<4L-W6J>3{iu`(a@qF`c%op=GbYn@zqbHXiy9TP2j<~-H!z~Fr>9;7=ass$ z$GRbtgo4V7 za3k7hwNcOHf&TE_PDK&0X!1dXbiy?&WwuhNzq$QsJ`D8ih#LV0li?I@yZ6+H^=uoP zvgTXBzQmP*Cl06Of5Lg-{MiS~(^DZ|Ys^m03T&uj77#+96BgYPZWVtjHB2$V8rk3E zsvTh!(vO-VQ)2ZClEd2#PotiH)An-5Gf5G8VXgm?6b!S6?WXO+5paG1-XEhcw4!+p zI>vK_ZJ>*45=|EXD&fC}@r`JQ<3sO3Y4PvTEu(06XLt8q zcd^PhARo;yy#6Aet!Dk+VZX6oG)TzU_To{Zw=sRN^lot?@&d#LJ(o(VQT*SQbVx%M zXc9XzLcVgZ<#G$KRN;Ss^H9B5Usmy}i)-uVbClS_D7U{|DVWqZex6`oyiSr((%kM2VSp{SUcHbO(Bp?}YRrfE2^ z;c*=3I!0p?kEQpzHGLXuZNaIH9LchzK5&85iRVidYPDT?9{_ zZG_Rac;KCkMYXI%*t(Amr*+<_Z!+S26z)k5rjktdL9Up7$2PTGUxsY9w{x|=0M6f$ z5g^79eBcdRI04d=RxS6hM8D-QS0oo#IU_S^yoc7;Y#QAt4jt_w1T)_)XVA2}wc@G# z{8&jWR2ZBzcNuxvN=0W~Wz_TBOT1ag(Ay&>y2_Z2jorcA3hGY$Nb$IqI+Ze)%CB|x zGQ^&Yowr$cQ_I%>sn;92jC1zapRKGx!cbVuU&jX^);R{GGwDVL3RFTloSXN2GR~F7 zmdZYv^1=7lRN^sDx+m~lUYCM!(d^`N+qhQzKG0Gq9#p|u`5d;i^sbN$q0GXC+3v5B z$bQH|smgO${#_o=CN0PvgUQf4KXY^@()5W(-LC7rkz@6w#bwXveSpyB^-y!YCgZ{dePgE9?fl5uBC*M4z-^`l0x7r5Tu$iwi_n)d zIvs34i<@$f0C&83Cb6b)-n)jLf ztfl_IHsPVZUPb%jW=dZYC7*O%9!x#L+!-MKx=?Gu&jk!K5lTNk;la2Ei#JzpcgF57 zG`nwitLyrAyxf6(d>nn%w3rG|-}Er^vrGB>oB`^VG+rB%B{beU-yx`=aat9yb)<28 zd|c0!J$XC6BoltOT^h%igHUGdNp&BF<$rUEV8JQv_x;VeBiQt0!0LAzYnf*ZWcC+^ z0cOE)ASM2=2t%IT7=GV>PIfn9ad*6D5rVzBS)8uQs6>alIZJCcHjgK77Lx}(z|1Bs zgg(A|Fh|Q8Y}@@?J>dFLOQp~E^m$KMb0h=v3m!}rtx3T7r=KAW6q>ug$cj=>G2%c# z0(haZrexmszgSg&GZzzq^YS=AsdVxAmHBU-_?dCbr#y_&{I5i>dcMoqf43)iXf)f^ zRO6&G>!k8;bbqb+UP6MoDj$ZhkHe`lN97tRgFvHVAe>oSaS*f|H&{!1zhW2)D0^-@ zI#4}=*(@1!I50t7+kQOO5<5TYZD@N;D6~yczXUFrtce$qv-@{(f@{I7;o*CA<&wy% zAET)=a2Q`(1M7N|niT3cYlKswLwbFa`^W0xG+{`O@^G5EDW1zAonFnm0!#{)Yivw|8v0=-1}iM~N4bBLUI(ACCt{M)sv9MLDVC|>P9aNd-Ddq9 zBx{H)-Lngk`6WL}2g3=fL^BGLVa7}CHu3b&A28$FLXu0@_S)j&Jf#>lC)X~#+}*d4 zkrt(y*Xse2?y=)S*Jx8P4okl{a|8Wm`wpo4D&FFLQ>5GJe>Aq%^NqDNN~az_w-ubzPTK`!S*&s;BrUiFn{pTc+EYC|cBfj{*pJ@a50H`viW?`QG)&%DT$y1=aWT$dJ zKv)9G?QN1k1(Jfej3MtMra}qge?dsNSZ_T=3?^r0N96KTHNqxtZ`*@WdAX;NS5GbB z6l^X8Xj4uI(Q9UwGNz|uSBLRK_HDsde=2MD>Up+M%f>Fo4xbvSqF=_H_mLt&Q(s-k z(*)xc;+40Z3G5n@95N2W_0%=ve9b0(31RQ(-sf`h6P7A~|6Ys>LLWf-#-$_&SEDHj zpW#L*?A}%z#J7K}th`1}oApzCziY)FetEac zY?2$3qd}k0us|CJ#YrHtX4HCs>@dyxl%r6)&E>DeNL|TBMIp<=LpLtloXmvz*6u41 z%?rqZ2z&%Vjx}pkc=)gwSz-`q3^|}+tV43rGY}zGG2{llhB`$a9!{>cxMfY95&OS| zum>6a%mn_;YqQ;V8*S!*UcGJ$dpUh`6(Ws?sEl=SstFCapG;Y~KRwxHadc!LCz|2= zH==KvG!N0)qh9o_I`zycg_g66S^XG7_kbM10s|VH>9w49nP}?_D8$l2h@6chrFbV!f%c$?Do~{ zq#3=9H}~uP7d#r!+NxPU?Sq4P7efmzPM4=ya9I(hj0ZphE&Trae{Lh0(CeMa3i8;a zH6H`j)^Jg?EZztH2oMaO?kOlBx9t1l-AayVygS`*xeHnu>I4YAo{$bV9v(rKwF;0d zP$Y|s>O2Q7jRDxJ{d^G*-tCHmm2OG138osMciON)bE<`#T_{WhXPeu&*Uhv0?P~aS z%5Bx!jLH}Ua)cdeliBzC&~~q@PxK}D3bug|iqFj+u`6`YyS=%@0&~^b|E+^mJEhv3 zf8wfpCQyM*%pLh9Mh3zGfmD#Py8G`8;m+zhmV2}rlRc1g#miHjfw&hmD#P18&lfGL z&iHzICbU)tDUxE+-Lol4Y527{pN90jorJx?eNxPDDGO5Nu%G8`qp8_zqw1^{n1L(k z;kdG;M*>EzqGY8MQ`@vc5Ez7T|Dh&m&KUo>6&Ra6fdjF?^v~01?+5k46z-ty-DK9; z{(rtL79#SyN(6{pU~xvAo@`|D+teYV(|@)DCYxXRSU_`<+2|jK-12!rAXL>^Cg4A( z6Zf(|UCP#bQ_ElPh1Dmy4WAA?5RpZNehHNo#G4B~k`=TCcu&zdv4*J7!A$pFE~PCW zsNU(XKQWYi%ZP$8hbXnyGS5t7l1Fgk8eC`|uAq|}fYm(>k=CqO7fo=d{0Xv@Sc!j4 zZ2kAMQVvfx37~!~U|*}Ks2F3C;T}n6eF6#y&DjdXXlnj{XsJ6q083jpO!HSY8CBCO zG?9lu;2{LO2U67oSof_#48pPLt`CAq?*=ER?Ct;ZRn93b@h-(h-5!`wZHKMo1{PN2 zJIGn`dcRl2NF+!FuHddsq(NBvBu(N?IhnmHfi@$M@xbMWGmsIAc*Z!UG`C)Mc``9E zofE&ne*1?-PpYwv=P3X(BQe1Eu-uW1B4=6-zuspmDwOUHy*>P2f4chG;^xoemC$ZM z-Ti!Jg4gaTZn@>FAZG0|Wc!9|N^CA9q({FapX?IN6373ct%VC~iYMC`yxF1#e0Sj0 zeX;vEYEpBhJ3KHj(D%MdIkZc1^nCDEKy@NS2@(IG{|$84Bx1#P2?rYdi}9M?XW%t* znn3j{VoBG_EB#YQNM&$awQFLV4NZ{gr6Z}Mz5^R*jw9WF6&P) zp{-x3`b>?GIiST^xgh^My`B2P6`9ba+>zU*y{pYuDH9J}RZ*@HTn+Mg!o%%O7yl$? ziiofCMo-0%F*Qw0k@yucyX1j`3VhQ^Up$$c9A5Lm`76ZLCmB9}TH>oRb`;phq5Uh0 zhbt>{-wj0VfJtIWMz($8m!O0Op2Y5)Jc+5CDqWZu+8-+^A~A^Km$imjfe4agN0&nJ z=_=Y22(EPggCwdU!HiYTokk6%>Nw)$f+=+_4KvD^S4Jhuxdk23oUyvn&598E9x7X$ zHSn{jI-kEHXnasxtOBUPxO9`8pGC`f6cP7u)cBXOrO0 z#fBQcrZmO94lF(lzuWilk-9Is-||0T>ehUm$PZLcgIKSw_3N#Qv7)e``_0jI;}Ps- zQZh^#G%4tx!%81i`aUS9s!=Ov#4KS)Yu)EY^~dGJ;}>?~4A~o!O#>)GYMAHg&3~9m z{usDU<~-4NUi&SXU<0@0WU-2z95S#a!`DG0%kS!>WTKn$EO0g`63!}@u$XC8zP5ZJ zN)|8n>|I}tXm!TnJ6J}2_5pf&TKw_@Q-j+qRt%6>-s4zo2@l?uZHDujQT~1smDUx%onK z>5w)xf5^aO@7@|IsaL51b_Y>4Z!FOK%B}bRC_2wTw%$GrpAf`OP_-K|irRY%VwP5o z+BHgAd)KbSj8UtKP--hJwKuhj+Gz9#N{pA?Cwu; zR0?(#8$B`qu;(253yxrWdcGj3fpNfPHCbnTlLv*S*{-Hd60u8TAQ(?U{T5}`c*eZ>=ZbYd8s0aSbTYm)5W>kg`j@~vvQ9qFbcb<#`b^feY|N* z#zI03b#c9iZC_NLfNHO;7T5U5{RS#|J~?tVnE)VDsu!@gtWqTgxb`P|%I`|IwzhtR zeg3^2r0k227BII#J7fg}Kbp6?Luvj#@zmK;)FV%AIsEj1otbIP(31>0<$!zOb!9N< zuua|*oe(_DorL4!;(Goo(Jos{Xq%Jn^Ve$BOz8htxu76J_N_OTON8F@S~fZZ5s3|N zBJwN4SDs@t_~GG2yEn}jC!FMl_K`$%{~NVP>@!{w5)?~Ffle~3f}kP>Xqnf(+?#wc zO$zHtZYWHnZVzQS#IJWH_g{7EFd2NoB;nYFTpkqFdO{|^gPso#LB`kSN9O8;oc<)93xOreZF%DPK#4=WVz$My6;3mO{l$2cGE z+E&rdUPtmFNMhX(ijkm_UNL`)pdRquulmUBy2 z^5L~iI^GoGM;u;M@%96pV2aTLS}A%NKX`O6_Nn|_LFI`JinpJGhy6DSCC_YAxNdKA z+p04#G$bUaV*JEr*TL3&mdsPWuQs5Wp-pvsIH?aR$n)kVE4E!`==wImN+`fTpgiI> zrz$CPBeopOW?{pV!gT#CRae3JM7W|u*W=AbOQG`vYBYdo>g_mBu9J{IZt@Vym1vW0 zIA3xx$J0UlPMQ9E^1(8IUAKcto5o&$YAlX}vI%o`1ZriMrEeHxp+YE@Ihh#{2QD59 zhJBe3$RtGVaesd#JoNvKOLDz9W(&IP%5a zb>(ch+2i5_dR-s!cfS^7-FsR;ssd<(2SmnsB8~pz!BBAPJN_p6E7!?G87sh_h1*5F z$oF1z&cvVF#RQG1j{f&7Bn1Ldhj2iU1~$FIL!3G(nRqzaIn4L>9h8P?0`Fr?Xyxkl|EQo>EMJt+B(fV^B=?S7N@UZhX3SXvWHq*HOS2=zK?3 z0)HTz>L>cbc!<#BC;%gO=NP$S-gq`l`%=G(OMLNb*&@%63GEDBuoO9v4(FmTN4%_j zM6{@Up^ehW4Im>evS;Ge{mBE&7@A2(xOBx;*@kik{cQQJEG%emzsIpeOET*Iqzr+i zYE0B+yBP$|$vrdjlL3q<8kt>9vv1=|z~+37;3-~R)ozj-&_2WPeUZlaXnn)fS8I0< zZ3FgIN(Z)Fh&R{Hl@k+U=z-C}@3jzhpvIt-t4t{{BJ8=c*w)+}&R!knYmUg!z+u*h zSe!PO+j$T#j3ohVq;e#bLmkT5Lm<5fYP2+f0eYC?7OOB#qP=CPTPbIeITzyjWsDrKwZVIH;-V32I5d(sz z<_C{Q3vkWQjfr2B2QHVgZ(7EFpLc}XT{H=@B;Ud9ATb81_f%00!fvrjm^Er?=tI9x z5XT;WSy?hUU=;y|3a`RBs>Fz4gS9_rV|#Iz&0dcpu6|t=S7ueTX*Tn--y(ww(ufzD ztIV*H;y<6{MZqW^FOW~B+HYzYfY4M4{VetzbPx_C^j%=r#t?Bsc6hqdy?z$b0ks+n0Lmr!gr19w7SL`5pt;B*Tuq{y5V8Zg8vIUNkdt1(7`74#TwGLV`XpP=Te;wyH^@eh8#Cj><> z;G&K7Em|t=GN1nxk|B|Wk#Wg@RD9CEObR+lib9 zgux10tiQ+hQgS%jwzucTD9URvHfs)qK`tDx|vZ&!HfEoNzat54+9%9Xy1u%8na}%s^>9-8Ky=l=+=KKf5 zmlDScDl<_hW+1nAyyHOi-YLmtS~zitf}=O=!;9nG;qCwaeBAH+`*YCi+UJtz&$;K~ zzf&LGr1yS%56DVbZ}5hY`B+3~4srJk+e$k ztL(LxE!D!qU{vQ@XRdA#xdzre6rJFv*1S*VWBrYqlp&2t@HLg{Q2&0NWI?bwH8q8k z0V1W!j7qW=PFngF0@CFcCBx?^LbBdg?S1)Hd3Le$o-3we2G1wHM9}J=>^*BIjh8gG zZKjf|3oE0|Tjz>;>Hp#-cE~{+6~lsCQleXMObM$FkkjH#g%1!TOVj>v1~a~1p_ifl z);@8$o*(i*hUggnlnH$%e+%VDTIJ?cl`=6T;O3$} zl^FE%Vg9e>oSQ>vB;b6fpR0a+YS$xzR2h9u$~i>x>H~39{s1LE_2cpQeN%{v3p2{* z;QnCN$?@+n*~=i+H)%U#JO79@XvNMD;HdEq0F8zV7Zl&va&ZR++g9IhSX1ez|pb7Bf z=DOkP(M%-Ks3E{iAGW!smA0tAf$KE_!?b!hG4W{{>YKr`iQ=UDeidO6?uKJDlS@-H z4o}|xb7wpV^`c+@7XM#hHNR(Pmw0vn+!PS0g!>^+(S2YkNbX-))!XTQr-ATej;PkO_jaOPF2!vka z(a2&uo+473)LrDit@6f3xevDv?n!CB6#!Fc z1^$drOu$Lp(B}|p0fN}-BrGaDfqSo9)q%=@{H<0wsoAyUUUgh8rR)W{{Y2k?Xc1d; zt12_I@Kkj8wV)nBe7**9R3Pd4{@qKCB<>Il#nToZXgbb5sL{xb=DiJ7U41b{$&~i9 z8_|#b#K>u4;QUOUoji;tb#7CPFDH5gVCtK?`YZ@DLPJdJD`UWCU!`S+M~1?K`leMV zd(pmxg;eTz(ehZEYqGJrE+%aHm_2QpAf={0dJavG)nRo+@f>-}hh3e{$MvmU{B_** zTJ8uwADcf>4xA3KjenZIf zlRGxK@AO`bUl*|hTW(wHtZ{w$`?QO}i~n#h&RjKCb;j~?J@1p*t3}E_oXeC;(n`-$ zs1N@c9a>$}w*Wfxj)V=5LW^My3gN*hrs}K_veBq!LAU7E+r<_l(i~|Z!IRXJ;ETTN zzw;|Z(Yn{8kC#VlefQNmQ1bH?r^y+0YP<2bnbFMFR@LTGCJ46|WQ067%0U*>6JUcy zM5hlqT&FtIBM9N>c;{V+s33?XQPnn?RjLPBMc+kNM98%q(w>LQ4a6EOTrYqBGbeQw zydIHzywWj#lQZ{QSVVJn_tf)SLNw@yF?R)tJb=A6esw{m5OY0U3{&v}Bg2Ggn)v=w zt|DT?9~RjZQx~!Q>sMgCVi`mACasB@jK}b7o#E!eoOSk^_+;#`O@cu{&epIE@G}3g z4c=mb<_1X~t|v_*n-R3fC4@sc=Zr|3fypzgmqmb2@uZ1t`C?lEwxjF*im)tPKCcOy z+^+;e%hGp`q-N&71tdx_@(-9C%V>`))>vnAmOq ze&v7$3Y6u*=Hr0=tzs;yh+^(E|k9gzPq4ej4AWH!cm6V)YbTno& z^N|`c4UoVLLyQVTo4OdmR5fDCYtl|Z{{yAH6M~zGM7~jYuy_x#%o((@D&mi_NnbG z=|Q^%ma;h}n(gh{hw5l2iq^a!I+UBzO6RH%AFyn2~J4c5#?8{3yz>z z-U@T`V91?K~dOziAhLrg~s@itUc-6d_?#FaZJ(&peyq))J=m%j`r-}-d@%#r0Kk#E{Wqi-q)*Y{`q=AytjIKgc z=t3ygZ7Lg=4YaZ$qiwNDAyp*(2hS}-56KS{PN(k?BmaTtf#r5ryXNK7JAEmKdK*En ziScIYVMymHMzp`tdnOWxX7=Plv#i;#!D3aJ@b%J35c9oSG-vVRm;ogv--m%6Q=vqy z-(5sjORKV)3L&r3E1>UYesB%#kCQca38xi2s^NTfo4mOpZlII%%XA0F-uXABjF3BvTgC);S&iqM%6@}5FjkB4*G zF4w#b5HeEG8-a2GQg_W>fZd&T#xcSgsfpvxLk`^PEt}gPHz-V{P&&l=Y$vJKtBRQr zRpPK%?BIbuUZ3oDul(%BgVgrMrc4)f60Qc~Bq}NLu1i_0irt8|`=pMT7hbb?i>rR; zn)o0y=iFNpV5%4rk+9C!PA)Djn3k?MTfPJ?8DDfTv>2EOlHp-IA6%d6&>N(?6uRV` z$lJ}|Rp7{St_m1C=0`xeCB9#9(Ec#4f3fGeO*)y*Es3`Sj{aSRqQ|+si#W4u=R$I} z!mO+wK@s8wHPv*?f?DiYx{bAEGl`|VxLvQOfpYc+nbQLBFYhHLHtBL0z;tvV-q&pO z0vrxdqAbq}YYbuis`2K{#MX(z-FMHrT0#Neig)VH3d^@FcTehizW}R!tRYe1`@!QsGEoqeQ(E;MsdF@T-(Qi4x<9BBMKB~o?b!&I z`{-r|8gITETg|RjStD9ze(L~x@?BZ2Ki_Vuqc&XBc#lJ2g^IiEY$y!^>D@ejsrKED zVI!7Da#7AHKp;Y2@a;(3Rv9VNmVM=Oc@E!aj%XNu8-l1FX@$HBk>2b4Fvf;(b!>ir zZs6mTfHVm_?-D&AA_J~34*dV@dqv#Qi1_dOEv`4WI3rmwaAAv@Ia1J5w?QEQ(>NOU zYkU?PnNDKJHN<{ML4uzwI+vL^J=o+NeMk{@evsGxGGm^eey{D{(Z9h)vAA|inQ(q6 zJbF7Zk&azK&1dpJ^IMca%7KI5vy;C*YTGNAd!AKCD8sdd1O?6UXX^PKfaXGy?_9{( zVI<}B*JPclSK_hCrHm}9<2TMXA{gAAmBO$CWSH5^F!Ku@N(^T_H~-@Z(^QKELhhNj z4#dHcSFhA1Wk?%H(*m4mvyj?_@S68!k}*!1={{oJOZ8cDMGPSBZhKoK*qsbz!qQ33xsWvht93mGeXQm(XR^*hQV~#bR_iM^Gn=ayp5yRrb z!u*67m%B-f7qG#N3K5I|GBu|C8Qg%@pwXhS^#AP38-YjO5I$g5c=qc)8+9yxq zcPPQt%yDXz6s`JN&VIK@e)gcGbN?TiD-^=x{pMzHv`&L6`AmF~k9q&tt5nLDsi5!A zqLmz3w^^2I$YBRQ@*MDKnclK{#9!|j-XN2$pi)18uPP=By5t{wZz(s;5$Ya<ZY@3}9GX++UfH6g{WJU`zesx(15ITq%wS&&|G`6I{BSEMF#ChvlO)yq z_D`(=2!h2YcU~OkbN`hhedO;C)I^SNFR;oaV_27nuXw0#re??W|5WJNe&6Bs_ro^P z!)rA)W=FUKKU7+3Ph~93Bx!pwFB-0m0sW%UPy2a?5fT$VakmA|o!WY_RlfBh;@8;K zI|@fnvCiCQ)1_5`Cz~oM0->K824nszO|LCH+xaKL``0ogncZr$jmnF^y7iskzxG?- zyV8<8C*o^=K_kTxTjk#riKj2v0qn|gr0#Xr-RVbepK54pd}Nz@g?rHz`SRL~O5m4cI);>mb`_|)EZJl^$?t6i2OI=WaeHmZW>Cfq@k z3y0z=q+?c(Q7NbrA(=&7l@hKlriOnO@pb5xYu(_```O`+q$?pYj$>r&WXA`!j&kOYBABk5F-8ZdYV|)u^=QdSU#m+BC zZ}6bn@mV_k%jWG0@Oi*#&D{iDG$8rY9Dd69;fQ6Cozz=O-p~eJ=!V!2m_F2PPz8zGgYjP{Umoo9^Ts&>c!nUqkNYaCr|z!OQa{ytd1u2S@Lwy z^PMVVaX6f%D|tb0x(>{U|KkQ7{$DoUs=oR_A3UmTE~wF%X>4%}B{70X#pqsHA0g=v z9Smn_F)ecy_hy(Qh|iYu0`i0Mqr1(!Q8#0wO_3{Oz7fFRGQQmdTz5@dMx3_Ua6vH#wSRO`nWR1n_Qc>;)qSetYK?RP5 zKhSQs+OpzDJ(};Tz6xq9N4WgE+sjhxN+C{PF0pY-@DalYY-&6=oS>`H3wrvb0j(n% zw~%qc=<}J@(=yM|RTk;=cI?HbCuHg&JnlU`YM1cqQ3(+fB59VX?Dl!|MEswnXI$}Z zMjR|US|FxWk|ain0dSBbD=MOKqmm)X;3&Uj-frEsr+@>AXXVvDe^lAY zf5#sp{@&`ga=Xg9{(I~CIO}6eP9PsmMXA$`Z3!6fYjOaS+6tGF+?l-o8XJxpC4+@p z$52c0Fr9jzYQ#MGOZ4ho|0nmc5D{^;6MhmDbaIBps)bpoKCdx+igLg2vYt33YI82G zqEVv>qyp~Wr#t7L`Tf0|lU{LDbb9mH+k{(rCmkw3P)Zk-ybFco(6q7m4_`T}H!|tP z&y;n#9F{Tm`?fapVWEL&6I8Y;GZAF&AhyH+)7enbc)Ih0_4CTo8llyWXg#Uz&B4>$@#%Vh4Y;lJ2F)z2A(>^jqz6{3Wxtk~^7%ZRIosbu=w=_3f&v|SYSEHO2Bi;?-=;L_}qk1w=kIT^vl+@89 z<>&QHQHIV zTPy8lYi1_Gz$bIF5J_!R?w!ybuwzR+LtPR#H`*-UuJ=8voXNXg4X2nNIR_h59S6Ob zf(-v;Bn1s`JD1f`NLvJs#K^QXlmmk)We8()tH{_klSbX(wz_3a4Hb?xP@-Ppp z2yhUsAi19J&_e!IXND^0cFF_ppOmFPK$|#R3H$OqrxbZWQeG1qOx-AH)cwl)97lzO$A>$jz=zU2uBmw|9pU@tPmJeyP=U`EMd3{Muyx^kDx= zdE$rsNywYX_(~zQ#7{MamS@k#CN%#3TluDW8B+Z`J^|FCwI25C^iSQldbQhyVNGBC zHxkBAtJW&p&gMc6E7JnAYtA#pEE6_RqEaV+r6r7vzE8=FJX&n~UD3T0KL@EvvvGIk z%yB**u*qh@M_GJo3ZmjuET>5_z z2GOqt^&n~I=m$83L?_0yVAXNIQ^dj$EX2Z}XVuk#EqvQNGT7fQkvQ3KnqyFwGGyej zmd6+a-({m%gQvU6=7W3L3p}2Ag&vkq$Hx{rfBp0mnidUo_x2fbj`ne5yr&MhY&&zw zB&asWtaIgPr2zy{4;hKihn^*c;Ra@In*x3M{Am#wT)Md|5YDc+Q}a}-Eh z9_V|U#Isc$osKFmdAe^c6UpJN$v}4qlVVroa$wHq5J2(1Z58wi{Im_YU-`pYKmPyq zW-GWg+cKN1+xu8wFTQi~{Dq4G6|NV}yqzBz1raokmVa7QY`xK9&Uf_V+fc;jQ-13; z%mI7L3Q?B6e2t(!T-@X=Q2IjEzHx zmOyvfc`jQCIs3zqM~?aSL;}<3!^!8wSF+*>mO9Iwhh#GL_k9aqlGW@2t!%+xvOd3R zoA2Vs*D~AGP}=$P%A>*i6*)!dF{V^71QN3l4!q391A+AeMkCb2>_+Nlh1~ARhw-x` z4e*cYFFrevrbPLBTKma|ekCACvYO7y3`ICN1rZ_BWwv?m-ytDB@SG@om55CNP-CNa z!9|4ub9BQJbRnrTMkv{G2!6<)cW*_UTwC$f|$&7F*ZBRPNq1Z{MrOzH<$cw3LoBX-I}4On;tdzd=Tj# z`54ZiS)_yLmHYm+_MaXMiY58OMh>!ai?hNByqx*|>mb+b`IOc5bo0s<=jSbn8yId$ z<+41@GUw6HQ9AV;t88IB)lyAYxExnW#LFI zd0Z%w0%vn@aIf>jsKG0Np;?xSamIRWpw5U{8bk;!S*NFE%ZiOl4!6=mKykYSf^r_+ zppsTtSjfe^+O@Oo{NUjIdrLga*T2)U?LG#}cZ9sq9jOx?5o$KyI4hJZo4Ay~;VR~C z*?rMJzwUi8*&ozJ_&xKltT5tof%%qao;tE0uIt%}L;YD?tP}qHQriK)OX%wx8M*7` zVaF<;{Yz7_SFU1cDo8X?J|hcMOTxe1?OMfExj34>tFpOKM5*;q=X?{zg#@y7`2Y=G zCw`7~o!KeGQn8HI5!BSKJ+Q*u@Kqs7VVk8`Y(sSebk;XJYTC(KS>%d>AYNSn%OQta zPxJ38ytaye)->{?6tI*j9BRz#AsFY3zkUW&4EUCDlsz)H!eL(VSe`|^P#hnpbErZS zZBv4vZ;k6$lDz34ekXX$tx7M^u%NaF;N8?2CL=!OapfG+odpH!=Y)EStw7#2>I?pG zY$FpCPefA|65Ru(r=>C|eoRWkDf+#Pnrkx?&=%LkxZ9;>Y)|d}{r;2}Eje+hHEwV(&X@JV#|NPYq zr!RIa36DO8jx2@&LL^gxZ;DG+_rHmJEyZsew8Spj{1j8crGktkFZ z@x<_`pWi-?rRw%y4qYQuNdYNdN!)S|rXu_IKFlGrF(PNv5$0O;L6I{V@x|axK9^zD zoMKu5wn;UOgD!BZ{1yhvA4yEn(KSP8mxfFzSW1j$fZk(sI(w-QV}b0&%;7O zxYLnh(|R@ z%R^X6w&e~kVQTch#VQFBYQhcS(uO&I%e#0wXARWv$VdKb-k{r3^K|!+#`sS$Y~aY) zB$2l@R@P;4=9aR#+Tii&B(5?;&YUyMs*uXh3OkH9u~t9qs$Q6(ipFqMbLsMeZN7Bf zpy}{wX84nF`_{8T&q@635o6LVR@3o8yDJ`U517;gW)Q}ra z;M%8eqtP&jFH}C2C`??djIE)j_2B_P)63kot@X4rplt3=oI=O%Im}NucS&N{pWpA< zHiS>bA81AWTguPMvg@=icVQSXrwEez5ESgS_v3W5I?K}vEj0J_r|F)v$E}Z-g>$4* zeihI{^k%1iMd@3i>N^N7e5nC!%KES~${wucor!fZB%>n%cJtkv0IJWH({1l@Pf?{o zmA5r}Y5U*UsdAFmwdX)?>SQ^8^ZnD3gf%eK9U!Q4 z;A+|hO#+SgIE%}8-B2N4Or!yT9`WR;K%iYcHA;!h@$7lFLfD(7nJbIR*AwSQ5ZWXG zxJGRQnH(dR_RI5yaM>ms-6#D~ z|F5b^{7ke(4}-vhl6H_766N654wp1)V48cqOX3}0KeTa>vwM>z7On2OOCaIt?88`~ zUMnN?4I&Zzzn1dH4^NOeOIag~2q8(tvbq9O{O!xL>&vZ`^{c&$uCBE&Pa&jN?;RrL zdLOCLANBNH|>%S{_0}Ts2hdb$;$`BzLUNPM9 zs8|nFQZ-jsPpdE@mf85gKaG>^mX9GtlNNLKm#7e855sRM2p;mLe<0;b(oKa(={pJ`(C-qyT&7`J(<~aDt#4cJvfN8!Iw|a+V(%Io zwQ?(CLcNBrzsSh+9@OjkW>zd>sS)^ zS$gCu9TR5ym>_r9#4hp~#ak%~F~UhyJnL&92(ENmS8jQ{C%NgUAt`rsBu**~(BD<1 z{LvBYWd=uE8RRciL1B(J)8?#P_b>T#-aHWCZf$Lp3HuNP9g6$#y>W#MxAOe_sy*M!4(k9@rRb14>df@Be?l`#C|mKd}Y>%32FG!Ta; zJnfZ{XnZU<Mkffe;%wrakm1=J6F}1X;7scDo=_xT1 zN=D*H*Xvnz{=7tx^B(Odk{tDVkfscZ-on~0j2o#(>YI+-4gV|@IzUOKNVFM=M&@U< z_&W~2Qog6}O=1vNNU{xZ*+WNpkxW#&Nx2eoC8@L;(VETNUuZ!Ni-@J@;ZYM61K8ab zv{Lz8LhkZ7fxs!sgsN&BasFhimu+b}P2*(klB;J~6+oIoU$uZ#TV%r zH7r&K24G7m&z3gdhCT;?&U2@3PurVsjW-oR-HRa&N6WSgrWN*9l;{>3ErUn_I4L8# zj_zQMS-S_w50MRi59a%`%l7O{t@eu4ekjVLoWShb)U?n1J5F%*&yd_#p_OgBuj0z| zcP*3ObHn2SP%?{WW~dbQ#8tDgWiG=32>^a8?6=9&eg_bJ7RkXG3CHF+22n;oj5;!eFfACkI<|AJ=$;nzjNk(@R^Nh7U`cZ zQ5}GSxooa3jy;;jJ{PG4m2`CKVnD$LFv$<}1H?OeoOw#_1b?L$Zx8YHq%&0DU*N)J zGxcB`Or`y`@!3&iAn2sdrak~Q_yqHwT<)bYB`Dh%*4yQ)PB~<3?nZZGXexY?yW{$> z-jC*A;Z*|aI0{Uq_V$R~rC6w2<)oA&8>@sL!v}rA$c+PLXyRFut2KeC7p|Nnfs2nz zxGlsxLGnK1f>FI5d0_SZ{qg9E+ZZea(~bPRenbDW z^8igw65cd`vS4*}jhV5ZW!07zq1?>@jpVRVH)Igid;1jHBVcBq<9hh+NH|@y*y7t_ z$j!9rK9$Tb%76f+BMq)VQ$;X|{AFS{#R5;)5^u$N&0+tqt6ZP?KZE?~ZC80EMjRM> zjBt(#vac`1F8)D1*RC5(arE?`^lJq|xb1k>6HOEqoTm3kybd5ifLDnT`zBI)5=Yde z?zGEzLq?1xq~>&FKW=fw)!pcH0m%{icc}j`MC3SQS!Y=MzfN&ghvya?_2m8~-54uA z#H(vk6APKqC2)n2gAh=Fb{NO?+LZD$aguO%+x7_yzqf|A-pS?xQW!JM0zO!tB=zQ2 zI#W-x^Co%cKZ!!^s+U|p9Oiz%r%FHaFd7Hddk$k79kY!RXJ5TJf2bC|ope8(ldJOI zIgl11r~AaJ^H%ny*5Bs*>yK$Qq~QJXgHw-+R8U-gZ4Py;YTT4<$fMnnJP$pn_1(OIjW3T7NABa>vs{f?CIc z!Lp4Jqyu0>{Tn^wHEHF%xCjjuFfe68d_^v)6wd=Vxv?BYGqD?5BX06vH@yGo)0u4K z@W4JVf{Ni7Zj9W(8b73_8k!j;sHjWWZx}(_1CCdRBuJQO_~8x6_l;aBgD#?k-xq(s zG)xSViodET(oFyFoX{VyvznI8<9P5Vz}BQa2$@c7m!7Fgn`t|7xPH>Dlj@FWQVI7H z2F@Xy;3)1*N&5P(Y9CcK9hBP$ZC+LT{ft+#_Mmt>OS!Ja-g$>*5`F zKM>jTI3!Z3L;RFCT#E3FV^n=9+fAiAuIM#8e3-BnY@2nirhLxcV6ddzlz8eDz248d z^w#I!3YizM;CJ>$rKv$h=1MN?l_gc2N6rC&J|nj6i6zmK3l12gRCW~Us86j>}5`M0EWgi7Vu{(=13!~AL^0Rd~7JHI$rrxC~ zzv3cI@w#H2wibZs5XMrFvVluSnxH)GNG8ki#P(uV5YxJq5ewvsNF`Gw5WT0t`5**q zzH@$NfQ+ux9bSe^gbg zkJIMe%%_eOEj36ZP$k`p&^2A+;zQT1$fk3X221>=Wr>B1&467vbo{ z()Ur-MF-a9^*T(=+TfCexARFVfc)xMI)(Ub#o}YqK;%(KolFf#pw7d37I=ly(n7po z>ZDTIksJdD`Dg_Pthxaf*Bhgi=?uD0f`%)Sg5}}kajY;#nGH`S441v4Kt;ptp5kop zVO=n;*xig8Ffv2xd29DS-EP0Eu9HLiqG9jca(A)4WR&roXVa11mAjj;yViW=kD47g ztDUD-sHY)i2o;V}|sE?(riYZ{V2fRAfoZHa-c+;GS)>&5g!9wkqQWj{d^ zZL?V;M6*CgmvB6-nMcyL0~gi7wmFUXlUcFio}or}jsP8R?(76eT6GU5eF^jT0hi|K zh8H#}wrs8qnk5)t6b&^I!4@n&bX!9TvS}6-Y^$%z>l&0*-aXt5m-*;#`~DmU(1J%k z(Snhz6Jy5x&fBhy-_I#Ca4udt@T2r)6=b&L7>=}sae#4*IkuOc`N_JNO| z$BG0BqBOz@%fv24v-HtMeP&6E!CR*OS9$k+A{&K447~}SUyzrWoWz$IQ;`;vK7AWS zft1p6jGhE!*@Z&XLHek$$0n~{;DYmAOnxpF*Wi%&v+gW>XmyoGuvd@Ix0ua48__~a z>O4*Dkn+@B=^Ije;Uc9=1kN%9%LyUO9!;9NwX+v!*ZDjsKvnGXH+Tm8XQ!QLmb0b3 zAsxU|gpHwKu?TsoHQr6$Za6D(HTLB$WgNrn3;UG8(<{=Yjf`N(YHV&a==HhnD4bp9 z@s=aq4RHn|AoUTf;z5VX9fcHK)ZD_LH6Ig~KoUR#Mrsezz7*nyDP-%A_&=OLVt3+j z?c%=Cp9bp8=4;cQZu@Wt6W0UH;UmkXB-v7kl*CJ+cR6@$1*owLK*}?aNc0ZW%V2Da z4WD6aRGxkRsASa1Cgr)ev-0`Z$FRSbSNOEnL!F0TQ1Ifz-FUWTsqPus}C_{oP8mX>qJW>I*4yL|dvk79nGU z+xG8J;!HkKR=YPi@kZ@UR@3jnze#&-j-R;ie`o#*Frsueq1q1Ig}gCB(xjv(Mp51a zuO_LKlF4;&Ge>{pvut)bPu@Ak>h^uR> zCX7av!fO@$o6Aqg0C#v8QpbrNfHzAszJMjqtDnPc(C`}KlKNMKUMmo*EzM)8xr9Y8Nh zf+x~c&vlN@PYVH-tzS0lzri7a#NekDo_QNKeRgfhwLB!uK6};Q*)Zn(k^0V@OE-nb zrKhT4I8)B0yR2Ur+OrBaWm-<~It)_IG={d;yxsL}6+hwq;i;M-`GN`)jmvcYRsGJ+ zU#^3H*kcn7ow6>@9qbCUkZtUuExAdYEgyT`3iV#;yO(*VGNI)K>6%s6PZ`iUy4xQ8 zxPVPAQ>-|PDU$&;VlCh(S2h@7H5_ZgeMYEkTg36Zj{fnK(YmJL|6lc1b%<= z=cm%QZ-|U~Q~C<&o7gT&32y$key_oZOB?lsQIxC}Njp-X`Mf?dUu zFTs|X&X)Q_WL)pxi=7>!z$)MM|5${!>S^i1i;O)XD+M^2eu zc7i<0&5&TGl=1s6&mu-MxSUbv7OXU*{o*Jrcs%yAvjSsU4Rj63#dB|(=`EonfmkbW zd!wdGLX_E)uvNocAE#|x$lc%u*cO|i(C9+oMA=UC=DSn>4$`D1;va|bF3u{35{P5L z=+J3?#-`rY%X2i@4|hq#w?<`1&VI(u$&$)2bX2={yZtTbR{BR4we?wU?!l<`?<%Vl25X)D>U z5>735qngf75}RU7pI@=rtr8LP1FFEkgLrm;upk3p+ zC&D*SG4I}OvvwQibHB{__T496y3%K;_ET8hnH!LJn{O7Omlw-O5Jh#Y?KVUr869byh z@8I8}=MX3XYthlLY$4Dxfxvg!FsC?C|8B<}UHiUs4@paJ#O;IPoy5S!d4E)ng<^Q* z8Bt~8x)s=2GoB)%QL`XKhgVdFCn&^7yxwK>WGVE})@#9<1&04;KWrfZXT}^&2CF)e zZU`yLCfp_032(CG#w@wx^;f17}Yp8W(OioXHEe$0M`yiYo zR(|IQQ`jA9o433r(L9Gz1ZHK7GTCpH&5gAdKowxIp_rTst%3JU+D+1W zL-n^Fl=!BRNX%2>T?M;0M&l|J!SdMvLid+qkLMKY;tihqd!oHssjNmgiixCTcSPONk1um&s0YJL{RhxqOIsD#r?IgulZc>J5+@y#da@*+_tG2YRm{2zX2xa6GeX|fNX zzyDtUC%se>jH+92Jr>m|_{jfJlUYo8EOZbZS7oEy@j64rqj3Qsw0xgCm$jrxEorxQ zFHl`NX9bb#FF!svf>4R`Lt(?qQwqy>MYH9X zMZ(xkVeqML2r>=9aG~m@7#Jgu2nj0MIdQs2am#FfEIatH)vIv>$ilNaX7mPBPMhxW zqB5-dVayU4zKm@%c?c!;e5p)O*<2%^r%;on5~JiVY%Pgta)l|q#XNbxaarOn5mR@c zxMDqegm|wjA#T=)W-@Zi!|4n58l|fL@sG%%qj+6uOumq`2VhV(cg9PZnKgs~L?rBB zQN_Z{wGi;Yz`AzDoy1SB}~ zOw>Gj?=Q?6AQ0*GD*$5%gOb;I*-ckc-;ajKs7F|UsxM6$G0bbE8Xuus$CyiHX7$kM zW;Wq$31b-AqpcGfb9GSY{DD+#>w4eX1sNoX_GRAZ5*{XA7amEx1bMS?oh%j zipD)|dKLf>=~&gZ6WKh^(>&GV;k)mDczk-1GSxCSwFo>M?z(vi@DhD!Okl0^{XH5I zfUoPt-2e(AI@sEmGD#BdVIT0)IF527ryLCc=d*SURq?_D2~ zl4OKCfO&vNkW%gLb6K^tq)aB0jBtko27?(m+1PSmzd}OU5{Ft$Wn$NgiBv8-zq>(lvYDpw-{@R|>-vatZAda-qF5dntTa=l(Zy}enJDO5ERbuEp?I`j8OxdsLRv!jQg(|MA;z-@biqt-WiKbOgp&O)tykV!9|7 z;l)%{e*O6uU6&!ajqaIQR9VwrgxsmMA|h0^W_mPx_u-OicH~W&WyTBE_@3t&p%P>{ zCL*)6XbDF+m_eQ0A5kb3R8nvEhiRix9oMGU)%u84xT?7O5PDABJCxyUqD+){i8~N6 zkW8%CO9GG*{|PDmHV^JQJXBzmT$Uu#(?@)4yETc-W+qg%1puvONe%^ixFL4<^_u6oO_iM#>qfY1efLV`SZ{FcDr8RzkdDr^Xs>d zTPe&eZGC$@9%5?kN^yvIGTErOT(+No{w#vq?Y577>yequbxAqjSf-F*nb`OND;3x~ z8v%0?^}EkokNPSEDGWk{1*@vIO!?tLksOBt7b=^ZT9PCMAg1si4hW$};F(npFp`|} zLL_C%o*%H!Tn2^a;z$`;14_lwEU|DEVU%2JwebQ+E;vgW0S-BJ0MpY=NV@l9>K0=R zWGYQmh!lt*NtHTKA~9v^=KudBxW8uKKZNo#zW?x3=QH#B2LL_f3V^`8`cL*t7F;?p zI6Z%hmGx>hIn&+UWP0YV>-y=_r~3X};)I#Kyv(w{dqQcIFNEw?IJu_-RHO;D!L&hyxPe{K`BF4irBJT z#@PRl|Ih#V)6c*B_y7Li|Ni&idic7o-yVI8alO8`%c_eZ8J_#-+qTW2dljlkNEugZ zN@AL$%wv=}|MEE3 z=85ZpS^T#mMP=qGOMQwX=8-0s7bhdVe9*x!U%q_#@&&-Uu9+n(iZ5UOsNQ$$3Z}b9 zjtHu8+JFW{V60~K6br3Ll$k(^sF(&jNI$IC0W%&W$TyA#ms;S79HGb z?E7woy4rF!7y(oR);+_K>7^1K5nG9sC*8;3&}EtYTVwPZJAf@Ki5h)oW%4B*3rB6fyNFmgu9gcmWswdw-G8hEL`?_dr z+S+jcye|HJY4;rA?{9F#jV+4=~iHnpWurTZQ^GRa&4 zj4}4`(3EARDuiuaa`Up;Q`PclXx6rA`Ter*vZC`-J(LGE@bytVuDT|wI$m_rzu(>b z?s1rul_(y`CR$#jqetKUa(VmZU;pKQ{cnT1-X7!k-+%w}%WaG=>)SPHZ=%|aETWL_ z`$*Ytw~ue%zV`5CT^7@Q-*4N))QW{7Co!f}S9+YIwuLfh&gMiO!-GK7reHy+Kn*7n zQu5U*4KJ0?EElK|I<;&lH`^TT(zV5v{(GgpWPh?w` zPoGRd$$*CdlvH7njus(JRaG)PbKm#dcGGeUwx$rIM`ZV~KE>RdO65eudk=T_x3-W3 zL8v@;URmr&M$LbVYSU)RvOrx6a4$Ka$cT=hnF#{fg}kmU^V*n?eS43#?6OfrhK}5N z0^lgSwIkh;)aq-2`?#qpwM=)VsR;t%ku|JF58U@Ht6eWLyP!FGxF7^jCXwBTyLX>j zA4P-DEEFokz0g&XrCK2}MTDi4AsoRlHA@@2kFjrkm$fZs*Xwn;T(z{gj?6KlJl2g> zQ^wpYGbFvRxo66DlqQvoGTOs4kE5Ag>$T<8b@RrSN?HI!16IGkWTyr=|c>_Vad)H3GH32hxsj%o%7 zCw-Gq`c0MT`{Bn>?v&vWt!?%zFY z!nhL+uOX&9AjQYy@$K8UGq$2=VJeq(F|+8S_s!O|nJj93f22C0r5q&Fcd2HM6nJ|5 znAx%{kFAT)s9TMxMu?o3$b3tZs%&lU)dFD-Zd0C}9$;ek zDve|YYSPRoUVN3snnD~oI^0dgRQHd)%GEm90qpz!#~*+E_SLxPK6>|U^oUrQW$y+= zIh%(gHC4~~xXE{_F@=<&lXOr@oQG>eAL>n;K}E^8%ffU3@5rt-*V7*6)R3OdFlJeP z)RCBi)pPBMn%UBprKuod%=nv9>XpIBOiQzE^cwvmq7_IYRJHr#QQhD&NtN@BVis^B zI1aNZN$;aic@tJWLNTpUy4aj%QnO~4w|C4@5+7qIJv{NU2kVkx64Lc}@=@1Pj&I3S zVwiB>(I+|n1n|-`tu*548Nh@5b}$0sWC7MX&&=NY|M(w&+ea@_HtFQdR5yO#AgrISAuLsnVyv z$6Tvb6Q4O~MUXra#mouDRMk@HZ30P5hmMm2ntAfv;qmtN_VSC6ImIwkq_$5T8melI zP{}$L4mXR;^paiz;>3tDDF{vg3My2sE?q`sj@h*3q{z$@HwoZ!x%BWchQFTcBGXzj zQw?UO;7AmIj0JSp4~#kg5$g%Lrxn-o_4e1bR@EAnu1H!mRWlPVQWM*IW66nlU`zGGE_vWXefPJftd)$lhl+MC8(D``j=%{DD1rpxGW0*@A;2^{C&TFy*7)s z-0%0r)`jhMyS;x3@?kW+>f75o#@M#)&-+(Z4Mu979Q@vUhqvCS&1?~X?hf`8jD)S8 zxoz9Weap;sU12(qKS^|t%HY&2DBWXZ@y$dSO=SUP-(!69KJ@L=vf84yC@+o8q>t@( z|EQ)vQW8>%gv+utTbF(xBla;GVvPRrb$>{M%6hqstZ~LQ*5={6`?o#i`pYev(EO7{ z%g3Mp{`<$r-+ur78*ZIWwNJ25x&CxtfBpUKU;Fyo&n&T%%X(=hV(iNFaa~)&xj$9} z!gY+;9=UIK-xUfp^kwYnyNIoV_c2r!)78Ynbl(Zn%=B-!-^^@j4XQh0N2ddhfB9rX z#v?a(SBf=D^1f}hILu5J0R~1SAx+2tJ2})sB#_(iTksBkzI^_3`!jo}@RP0UiUFuC z|M=~X9@+P;G_|xe$V7hWxQH2|-6@cbCIw4dw^Kt6KVmf|T8;-c-Rwp>%<+XEIq zuM0%S{`IzhU6zYp){S??FInmnSppG~!0=Nrqxu&Rfg_l?=T-+NO&}c6xBiHU5!XpD zi2!qWXZ6f|ME1kwl|l$Reb9&R(T9j(rCyu7nY6Z=Et&hif4kp;i1dWMf4V##kH_sU zB3HSx%)D~&@_G6EFUzvtI#WJq5NN;i+CI-LvqzNdOaMSw?o}SCF*!={p5#!X&&B%JaFwDppi*`w#7Hds@;%RM485E1DCNP-G_?1+#^0#?OG zK?vPHEo-~9%-kRQqj!>K)8^0_u*vSN6XELXkD2@HNAAqPdkfzS^5k#>suIsWiQ`NGHZfZ<~yP0Yw@99~BzEK2~r=GYTGtx7B z=|B#%UaW(?(TtQxva~jUDgOpr885KKAyraXS$l1 zX6MA_leuud^EjMyUcdfYaGzg1$1-Z09?#*jEQFt_v`len?cLwr-o_Ze{r20($44bw zC2ryF!#ji8i-nM5jApWL_r57=?3$awp%Q}$;!aK*<(SlA zPF(ku*U$4a7MWN?g2><*c1-QO6#4yiIBS~c_3?bYV&}y?QLh=(&qD98%{xEX>+^rS zI+*<&9MSoe>3P06zEEz?7e6me9G*~>(rZoKWbXU^__&SUg{i8U+C=-v%pB<=)Ackw znUba12Sk}46aph{ZL1uELJ{GSHT~s*VkuR@QPZ#h`rb#x@Hj6>6Ln+`ml@Ng$chvT zK^kpYmg{9ILNloI#u)D5iW0Rc4b||Bhza8Yk`sn2#nJ$B1USal^N(+T z{>PtxesiQEmFi$XNMv>o3=fQbczzked|TR5IK{e}2x12^DS-?}6yt*`Vu9@(mH{%5 zJzPqSOVL7tb>cRq0wJ1X3_na6tL5YFx7%&eIeV#mZbq~pdfjf-W<2-oxna-f#=|mmMy2%g_<3sCqoVl}{K=yU zv~T{?=bwE<(HV+IP;9a4rALH&-}mE06uEu-n6-O3kVHz&+9dEKzWfu-sh5u#BElD$ z^5c@rIDxpA$bSBKg>1}G5(O|@3&&N%!Jmn;U3;NC{N*}7qn5mwai71sj(N53@?%H- zHME~Um6`E!ejFb`6i-BsizwD z|Ka@1gX@Bq{{DF(&Norr@E=~Me(&(?IZFB7>O)30rBZVE*n8$sYYR1zonp(?=a^>J z*khCS-LG(zZ%=E@)LtX|>sfW^^9ha=zEE}Fw~DSPc@h;Y0-#%~q{2WIoNgH7#ct5u zfp8zyCphraB0F!n2PBIZxTJc z4*(^D!(dYLvn_KqBljVLL$GaH5YNJvt&cON|piR^p2m*XQ;$4l_#Io$X|+&Onm zWu~u>+H1vI#}V?*%1KUEp$K0_Y4&|DW_$(q%-puk-I02`-L`Gp9{cF-7HZQcr!^H> zw{3eoZkf3(%et;H`o2G+4^=f*J0?`bqf1UQxfVYi1kj%`bT&v!N1Ig80p;XOP6lWE zCrVZ#CjsW^uOuR|kMlWa#yuljJ|ox}871fcnJI~w6@6L>Ki64H$!U)JUGO}~xq0fb zy~JF{Go3{pb2B!oBt!~nsROc@{0X35>Wo)saIQ($*RRw=E&oa8{`v{?FSEJNe=_iT zf*pt7{2(=SH=mrq)UGScqRf zR^6&In$#Qm`1qK~Rs^CNwZarX^tk;+Ta~1>R-{J8vtIG(R! zj~r1H^8`JLeRY=haTr_UD2D2e1DI465h<-V(UN8m5mAOFA{%=kBSRvYP^C~U^kSj} z!@oVc_dPPx6Oli?t$;+7z(7=F&&MMm69=v;%GV7YNF=^{T4|BXEJjs|I_a6xcNg}Ht;@QaEv>D*E!%c)dZ@t0!4rwP#G{OKIYhj$dQ0DE`bm?JK7wr$ZCsAcf~KV)4|idcC2qkJmsg z;y-5Y=kv}rdLrrh@@daR{`Kqs*<+q0HAVXucKrFzA6`e)0tvFs&iC<=JkmqKX0}v; zO|I6c%3v}bPEesG=iH{+V(N@7RH~$mvFD2*+Ibip_;5&;5e zTJy@82&r^e1Iss)_E+T>9{=wmCA3GnUqvD0LikfLI4$N+Pd5x z+t>#8VSyNk@I8G5tbx;~RYG`2A7hxwLJn{bzx7R+nl8wmKr$(nLDcaY0Z*p;9P}PC zasc!kfCen!o&wL(XV1Xv~3(n|1pr3NuPlBI=`K9Q_BS@@IHHDG3n%o#+LIMyJUnI$(bYNAUK znKB|$qL7y2L7<`q@)%1j*uj?9z0HV`A(>;$#$DBXmARPPz7PnOFMTiS=e?w}Kmoi0wt>);&DWak0Voar8pZb6m*6-1Ai zi^pR(lb?S2DKr26_y5@U{d&F5sXj?jlw_}}m~m-Vf{XWWAHMGjs7mR79KB0C9i&Kz zMg)_dPR{U%DVb8xY}uzzFVeq6n5q%c$EYd7JfN&P&l=~UFeBKR;T~0^*jrDb3`Ql1yr@jQn}9yog)HV0gJ9D5*v8PKa6c-}-g>_1<_Z+?Fi z0eDreeyT$LaMkKIpDQd6PWNB2{PWe%g>pu4cW>74V!?5CEYIig2mD=ysX*$oD1cGD zP;T4iW6#VcVycJ~p)?y1hYKW%a)ViKTRBmMEA@#>dItTxqvxGFFHL4vA$BJCNhD-y zyaY}HvWPH6RD>y{30p}Tw#96YTI-aDY`uH2;9oCM#hT6g>#9l$4!0xJgdEPCBnXHa z6NuCl*03}{mBCP5T)DXiJ(U5DfFOta-uu^&Z<;Z#>*q_mm_CLF#mIz-P$Yy+(cB-y zgOQG1Q{0omva>!`tOpLp20}!ZWsyT4wDQo2eH|ODoK*_F5ixl7a72fuim*l+DVZ@M zf{D=+DNqRezH75fTkwWATx4BA?jGB=4G-^QjDRz9%0gWwORu!|Ei)y$3e^fHteV$B z4M`Q6a9x+)J9_6qyPi{kgcPA7VVFwzsM1@W4xf2=efg1CRn8j`tu<4PRG{Kcl&W(C zRGZn7ff3%jgG$v&%z4>^@j@0nwR}&^{oIDHTlk0M>GhfCw_Y~svlo&FFRljJYUx%$ zX|27#zn6gX$H&)iAEWo}?d_78!*?LB*A-YVmxbidpFWo`=a=9A==)w6ec$#HG1e2+ zJ);N7LM%|Gf&*JXcxE*!g+eC>GxcqMm!FZ&Mo3j$vG$9H-@@;p$Fbl|%= zgFJd0$L)3z(GzJGmW*&+;g4^Ev?d$H2#iXej^3fyW(f6+(2_ML}rTuidEc^XI5@ryV z4RoOLdZr8O$oR0owDObNdjL~+c^a8A*7iuT=#t!3K%h^K13-> zIh~--^o7O3=6Bk5=QbG;@xsnNQiMmbw5{ED4txTjyuu6TFx1T0YEmx(>$;w~|Mj|l{`?N$ z@wmOcUmx53@wlyPyFWf|w~s!_c?QO|-~aXB{@b!F`@Z*m@4dgjzc;ll9_zaD@K@qX z${0r~%I_Z?J&RmZ7h$TkJkO^Y`HwwC&cYyt@{%3zV|!+kH5wiR9>9|6>BA31w=S2_ zSjKHQCZI(~s>yj6L>_&X*ZKV82SlXEZNVe`CQ51Iufg{C1jn+8JabJ^jfn9QfzRUL zbwR$~eUj0i5YtR2{~F)VgXI)nKY#0a+MK`RH8{ThZYgy)l84si>(hQX!RoQ^dknu^ zE~=XQSl3mNA78#L%OZ*We*ehyr5Pv!N%XWl8ozNGl8MyGcD{2tC|x}bqKe~ytafXSfVo(aD3lU` zeH)ya%OHYfUbzn-7F`!JXSsi{nbCbj2lcux-G`6e_if*Iqpa<+uIp-T+yMkLF3q^K zIz}>5032Oo&0uP7gxqxhiCf#j2{Q8UvskGAE9bmuhzsvxrO`{VJbHJ>$L z;WctO7aKrAhmQ!DEp1&)TMtjeIBfHdf%_v~|0N>u*|h(0hXbgYqiS*2n>@GPU%^$9 z>$>h+kBD_$OHyf!v99aPHIyQp_ESE6`f%>P%llD`W^gJxjVD);_wHZms;cIvvvBF7kJbD1ftIY}g z%p>&AKjxfWRTQ(Ksc3vQPV>-XI8T)G?VWEG*@#LD@lSZ}KRo_&KGaNB9GP;Qsd^rI zr$?4r+5LBABVLk@fBGn$DHD~H;qH+H#u#6}es%X{Sz1#7``DC1=7l~rY>;0+|FpJcwbs-wm&rWxAmug z{rT7b^_SnieffX><3IlWx4--L_~rV(KepJnzO}pb-n)H0ua6&Lb2vsVCPjQjmwXS|5q%gv`Xu~tIJoU3Bu zVBJR_!!yU;-Mxzaa|KSy0-%(plFb?+w6(RSBFwk9Oa0#J-g~?*jLNjWLo-exGZb3N znOiQG)sIZgGoH_hXO1tFtf}?E`>jT#C_Q&nZw~s$n^?v_Y z7ISX^s);J1%Tlu=OECj`AH9!{kK6sRsThFavF}~WKRyAJNK(nR#_4^>7c7?t+;QR| znNbP!Un9nOtWN4w#H{!$-nH@zbND1&m?%xWkGhqJSpj4+dJjCCwy!VI^ft*&syU(&3S!1f02`fLDOUGp7cQWZTsVoKlXi>YS;LP7)c5^ z%6&A$CttBZ>HrWZ9vc`!mFaybc3#eN1&heGA6(!gADJBjg@`GN{+CQ3JRKAWq=`X= zbd*$zK&+%@R-tC*7}2}?@V*V8()`FGJdWMRuFVqZkq!r-UcGf0`<_TbO~?s`dS>QG z1=iNyu9vDm%*>2$@2g#w_SQcC%g?`l`}*kHAHV(W{)p>E1L=K_eP_BT#@N4&?YdrU zw=i>r9OENm9J|NJ7(S#AFg*|uvRSJwF#5hB&kVP%Q||61v>nYA06^Ko)$Ev=ZGALQ zBu&eNuo%$OR!vnQOa{jA=uAe~TxCR!yns=%eMXUW&1KOVtpYNAS?6J}@B8o`eWn-d zx}IS-UqDqqqGY}lqe(zC!waGvFAb+1@l0k)3aeg6Y6j>z;o@VA-j|iCsz_2IC;chH z6KBR zS#GzRySLUJkH_P2t4&d-bye6WKwqcT?RMKo01_ynY*H)|k{p>4bsko@gwIbMN8*{8 z36_H96rcCu$)G}C!0pULKJ7Oovt%#xiCmLpFMYmJi>kOS#hZ}xX=YY27l41y^#9?) z{uhBO&y@Bkw&H|ROZ|7^OHY3;@JRTc2Xi%YoU9HsBA$#7yqw2Xft@+`52t%Q{wJ9` z|MED0_OBoNjF|aaCVXEr;-$m$`a0(irbnqC#|ZC3RT5eLjlK6wha@3V)lf}HAz!yE zkFwO^S*C|gXauKu%6SPQ0#AEu&3P<8-?qFA-|{$EGZP-mvbLtyR5YCtNL5m$XqpOP zQLZA<9mCPR?reQZ4QN6WRdb-cAyP7v$qN1lv2#|YjsWn>+uM}2>HCH-Gb&8lW54^j zhw$=ty*7ChT$m#8`(3;n&=`$qfDrD(z4x(!C95atyGJ^*Z#z6BGWr-vAs4l!*}5(h zo5p11H&IbtmStI%8XPK0Oy?d)Q*kCxQ^GXI7)}~6eYlf7ToQ<*>?PcLW~NXnBzhcv z9Q9K;KuQudaaI*Z5fHtsN{;~xlpMWPmJKqB43{8_;vsCUw%@rj3oyMuc0Y}SWD0+Z zNRG!`#>bmTEu+CnhMVi~4AarU!elDrPVh8%i3B6)-hoL8@4G+O+kBVRojx^To`?Z} zx^AyY*!g!ooU>}dAKw};U!42y#A@nWXXWAUfB(n-@6)GG|N5{0s;am9*N@wmWm!Lc z`c$Tg!e;=Pzuolf#|;p%C8c~AyKfhZbRO%|Tp z697^UE(-m{Zx48q-yV<7N?eXA=^2IG3*%N$RrC9Z`Fd4x_}Lx_?vr>vf&Q1sUT@Ej z0R_kN*aqMuiq;8CRhjx_&zQ^;3rQeayVQNDHFMfpoB{py6TP1O_^*Te4@p6O57M2_ z!AoTS;k)M={o%)7ALrLA5E!JO4v>r5Wo?YOQ3^aQJ**~NE-RV)p4ks~7fw~oS|Y`< zhzNp4S*^9!P7>Hb4NB&W4&~%qrG0B{HtSMGrbUi$5itles@?+Nqn8|GFjbYRGq7h; zr%UUxgyu}`qGqDV^t}(zk=5@i3oew5rSQ>*ACJrw$*ISFn6G7KWDL+FGIKj-ZdtG| z=>d-zN9C%=2q-e+_#zKe7al7AlNMPSD*}+xpc|u?ZZQP#Nzl&Bl4d;9;UkQM+GN6! zPwOQBF`kFPxuMQ@eWon#UYw;q`b#W$28C_gisvpO%VIUAUT<_l^M`Wh+<+pWdh8PktKdG<`r_rH zpg*sk7Xw-x-s5c-neB+MXRA4m8%=VAo7rMbRjc$1q`Pn1R>6Hy>xon4u`TN?%rG@c zDp*7xb-EP5HOA;3V>s=Z{}mboJm=c{g+yEL@<7}0V#WTH9hAe!>5#M_kCym(oSgN< zv97*TTo-*xJ^#-$_m`z`wyo=})<03y!^DKIfRnM5VAlBW8Nu^t`#xXrC9;#e+QRai zPfELde!Gm(<(QJCN?k-%kOJZE{k)M-qR2TJj(~v0QZxbS5hH#1u^c&?_2-#<%m>ed zI=?%p3^dgyOz%POknWGo9|3QxE%4ReFcT2?sF?|Bi7;mZ6Js<=s!8I0?-kZ-dNz}g zAam8Fx)0~?S7{>Rw_AprP))g5C<5^qyQbT^v}VdM!B0#3d0n=A_WvHu&n+jAE4r&%4 zz?0CDS!$~UXV+aQN31|5JaTx{nUoZuMsNurJTlWGT)kAp2B{g-z<)IQQ2p=6T|c`TcnM6O4ZD zzvpU4O)BKfRrh`W`Ik?b{p;7?TU#l9eOpzy@B2?b{nU5gwjR+%4qV*%al3!}_R%A( zwQbvSj3(N9N8~YERwt%bKPnF>6@kJ9s4_cv@Zm^!GODs2(bG>9J}Yyt;EtC_^=f|E z-k{31sH(>BkZeEguS-Q=n^Z5Gte&(P0oSXTdH&o9{Sr2P`eMP(=DzDE!zk9~zJ8%agpZLUM4;My^wI(ov&-dT&BFWE2g%*LCr7x4t7?|94B;7KTD_GKF5yz< zID8V-c~p^;vKFZ8j)Rh4A8Jm|=UhH2x-8405@QSmBlfXvzCltO$W~@)<%BVMS;IFeQQhUFZ6JA?%+ZJrIoNxO~CKyJ|ZGwMDKZk z9t~==lUi`>9Kobjh^pq0ecwHX$J9ZO%rSx#iX2J^FE(Zq+eAi`=1h_?Ihl#Wk(rZ0 zp(zsemaFUOC5Kam!dqHoX^>>MLWN~}CXGu9d9-n`K$Nt#viOBTQ|JwKc?QQa) zBC1)43aPi-#hwb$w6Vysg(U_Q@QZ@mlZ%5?^4F1P*0myeihc{Oa>4lH&pX{}J3@Gym76mMMRI za=xsv*Vv6?WbXX&nJpi*MSLIM0gzR;hWjv+7{2fO{o_`xq08l>%G=m_-&I(^bW!hr z_+MnU9mYH)YUKA7EQ6|+m9v>;e`>mwqhh-!0WnQ@;RTUFn$Z~N#S5iw#!n20nn zeLNnSWdT3h_3|&j{Pd|c0{d9RxbORy+t+=h3KAGGj`X%PRVu$;M7UnxB77?~tLy@* zh*65gq3%1wGfD^Eo^H$Y&Z??%-f>Ap#N+YUwk?koe!X7Gwkb3Fa6~dCD0*2GG_&%I zW8Z6*__6gU5et$kCpcat{6r`zXu_z&hDRA&8(UQYiP*PoJHk4Td?PbcFUxZc3#(iR z!1q0m*;Em!PQHjpiP^Za6AABc?Vd%kn@o&UEtBOF47Q%-ptryuH1B z`t)|azW04ph$$xfzW03};aNAQ%+bDn+-{HUx?V&qlRh<(SVn-6JRBJV2nBTkaL~aC z5T_NL*G!w_aXe*4to9rNifOrxOxz!jT770_`V=)&dm4arA3larz?x5-*Q(NdOhvp` zZ;j_At)Ov&D$h8EBYYjJD4x0RcNIWAmSy2V^vs;>7%00;oPW4pg#_iGmh8q9%G#6T#ZtSNNeKKlZAEF@D78O z97RMVvVhM#kip;bL58iyj7Y!v$JTF-E&b6yz7X;G)2F6(zi+}2vHQZid(UnW+Jjq0 z#6r5eB+>)G<#4%sJRUbo%FNsdg^&I2y@rM$B%8>xT(V8J3}aQ*$l=(zC@aapb(V{E-glC~@f8QmYfYik|?ZKm%d{QlVY-T`UrMB=t>1EZPM zfNK=IkloFxNygwKV*kI}l2#WVP1w{dncH}`_lqp=Z|{tZ92C({@4a`sF78f|Pwmt2 z#l7!7NH%SM{_&L*lVyxlE*E*aEB32UZo>>;5R zvh8`l-{X?*9#A7!U0O|c^f;XQjLsPH$Sf$wY;Q?jFJ;Xzyl3pd=J`=htyE($A@);$B(AX;r(v&DKfb&rjGXg-5h-PHv8$l<99Rgp`DMGBIf^Kuov zXcUMDBs;;TNRb`OGg3wz?K{tm+x_e1vRtnE@$muj`fhEtX8#w7{UOHn)8*eT_ITXx z8fvu5&!5_|{p0uF-TV8epHk)bFJJq8`*eBhy?^`tPiAVMKQ>Xh#U`Rurb-f_fP@Hs zWzPb10fAbmq)_zAH8DJP?}OQeStv?d7?HE3GmdzaXf9?=QZS@uvwHN$<3aB!9y(O@ z4b6-6*+*uMjLgZABk4jShujcMBw(bdG!ZjOTB|!g1xRw{XVnrq<1abw`o_MIv_svb zl=(7qYc~MP1#Okgd64uN$8oDb$vsZE_#%_)8WEX-6ctmM7QUgA{vVE@sx;?Ia6bd| znLF1%&w}pt;HtFqJYs<7p*mBl`QuM4{($3t?4zD9iywd*cN9M)=|1*-zwg`Qv5gTU zqY+SPZOMxRAi_mv)qi@s0I2?6PKRJME6PMf+;gwDo5wtT1l%8&%iHk|W`}Lwm7`U? z?OWd;y{no;@9SmGWF0`n`gXal>ow_+L3KcicF8e$X33fb7%{@#!<`JXOTB)=ODALu zxclK{|2(=Ym-6F^*UoqM43~tglXzo11|i(t-DDCZ?-x+vZN9m%i&cNw1j@3h15@WArlNgi1U7 zE=cOhtT@X!vh<@W$tXhIk&GXyUwqfgVsYGRE4FnpH38A0p=Bni5x74(C0pjmk>Tk& zvn`Km%4yP?Km?Iv>{=tYGjlqV$RMHmgwcv7O6Q{N47lE#HuELx?GP) zoj7VuLJ!pXjD!cf#-9yRynr)uv?0&0%rGJ^Jone&en#8>?2#uE^Pk1**Vj3p|3c!- z=Po)htiQfUK#UM;F?uF;k9`~4zO#5&Oj4<^b&c>Gu1&?1Dur9-ao{9?Y6E7`mPv@H zqD$uVDO7Els6uD%nLIna^M1t4((8uWEj3imyWHTXo5(e6mEK6OW zWD-mSD!q3#hC_JHks#Ay*0FaV-Lt8vEn-bfJ!7G&G_z5IKi$XJI=BkrT2mK(YGNg3rUR_ zfJ6kQW7*_8#&md?F?#pm9)OrFna9tMs0koln5=a2aC5+fS-j}p2|CGY z0S1!{vWZm;CB&?$Y6e^Pyp0$fq{FL`YeFM>@1t*HfY4MsNX-D`7^|s5mj_cJnNW(# zB*37q?__`++_r<7S4)C5OUdK@1C{EYVMuhU&zEbXdzNc+E#`e^Qbw!u=LvWcpW`XcZ_}TpFPgtS+_4w zcBVfQ-x4X%WAE)#TqbE=G*P|c{kG+E)yd5eT*?GX(ri- zW@=_1U%!e7A_Q?5O3($phl#Am1o?UMR;v{-$%bPLM&NJ^A7iZ6P?WNn)wn1clx$<5 zCbO&2Xr^B6UPO%UV~>b$L61l^E(H=R#i-_K&b zE_-n!GAK}!gbczX0)O_3S*-cC!{>lX4hIrq&1@7c3`8sEn;## z@WQMDo^D?C;0qo8hob^jV0|Cl`8~bzTtGM~%1lDdN>WATalZ-81D^22$F?(FDa{my zt=}R7A|=S4?$yXLQIes7K1PkM00~^L*L~kftyTufW^1#WxI3%wdhRTZ>k3}YZ z%aUFO=-zvW2ZhDZ7-RU}y)BeH*xWM4E+VQzTIuK%bhx&*n2ZDAhrpqjdZKn6k> zz;MqoB8O*`K}ZVO7DdubL{^by$;GDHN%x5EE?zs&bfJ-CI7WCxf6JD^NQ{VZgab^b zil}4MqV$PQYecZIHkiBTVeTfRN_Y$(eT>iWl++@n%bhKBKy9XsAyA_vifR`t6s*~F)f#$=_3%T0>l{8+T}>DtyFW6Pp8A@ zj|Eud+6vIe51vikl6)w)#UJ+c`L|+r@7>*d=1dZAx7*hDWMx0&?|=V~-~agbx8MHp z<;yn{F$$UbLQZ5Eo<(LRySN|$&QqpIG65gq!K~J1+3|$YNXgYlaZt{XFc%JIL@2wE0~Kw*Y&E}RQKMykMxny zP&M~Z<g zZrAJkY9_*n5#G;sHq3PA>`A3`W46)%LG|=*2B6 zO07#~nt&lrQ6{Ak;-DqTrgm8r9)0#mO$3PHzIPDfv9fH{eHha z?$jpM#)#YP_WK|I`1b8v#2iAXxLq>8Jc?kC*l+jEhi7QBc3Io9ESECvIpi0_it6Tp zaO5bUAtazml`OakaM3X_4LQLXCkZ-5L4pCsIWznkXU{+sFNzfPYM1xr7?DkY;RtzZ zl@sO8z`}>YX}%OMN{Fwi?Q8yeUJ9Lt>{{^4**~Z|a{OhAd5e7aYUiJED2nq1qA|1a z65YR(-hGY2=LdfM?id5@yC>r>k7t~L%FGj9A29=>&$DN?k4h7xwT6_8-Akg6Q4suW^EwLm$tPiXENl6W zJSoy~*fJC;eqC3yoAXFG!!aOfDu02yZaRBoZ4q9BIbQ`IzXqOoIQqzk0Of& za(J%Gby+U^{wT=|)n!dajECPJI|5$YzK)?PJ2H(xQ?a)z$=>_p@wmKSu+eioy7zEn zTi4aBksKNK?SV*1ghfO!6Z_6O@R5-N2&2#>koCC|!!tcSFp?g`v07MDS;GZGLB*&2~Q!wfQqr0rVhd*0nuieL)?3n zu{E6FqhH_Nx6%9lSW;SBG-Y`7`>nmTLSx8dV^`d+cP$i3X3b*NVCB(aJGNgQ^V)Nm z@X(??wy*o2qF}%s4Bw`Z76lZVh%{9bc|7j*Nu=UH^}3TnnMm14k$Lv0>YQlx zh#Z;<2>?kKYMq5Y1gInU#GHi%=L*?969Q?<n(^Yb}&hQ-&P%ojcA0^gtWulD1U z|Le!=fpJFmJoWYt#lCuE=H*%fRf332=^2m5qa-0A($;1sz3(G(>$|(RWx7C&j7)kB zoeTHJHkI(g0cZv(AisV2wytYU*~nqAa$c{xH=)l%*r(QLS6kM_%w$>HWnC(Pt_u|r zBDO5cx?UfT4S)kuh7Wlv0q5N{!EQA6`{I5h@@B=|N#;#wj2rG7ZwC@2*<& zPa0;GP$+7=avK3d}}Z7OUBPhX)b}DrUs1)s=CIHF^*! zUUKTiUUvXEDKW)iPs-@t{gD}?M?@e+l||pGHm9??rl>(QAR_5RjG(Cs8HwG~2&N~N zR*t@1%JjKl;mBsXZQIy~(w1dWGvBs-+n4v{HTs;H&=}*`y>jMmW7NzwaOhQ?KdLHG z>$&jtBXjq$@k#BNbZ0V!W|nmw$Q?(T`P}SNTx?FUs>-+ft&jWLJJi~;EM)fH-$eYB zXpc!I0r2>^*}ANk^?Fs$++w(U-}Y^r8_?Ly+OjNYmsIvXNT?EG>&r2|G-GM98Y^x<+WG@B3a@gu4&lTWd;rNmTRkcvKRqWW?y(-g|GVmt{#uL~Nro zA&NyUFPfPt`TDktktQnJR#KnB<5#FsgsZf|e#-=;G#4ME-gDMRyS}aKx~l5q@hG=J zO8Q|y;oW1n*@DPmEQ_`GPhb7x@z~!l*Ka*NV!Pjs^7eKu@v~2#J}t}g@$oVCt;`-D zk4M$o@1Nd^ZfIuL>uP3&a;)oHW_ol&=YX-`=bt}!iY)K#+O8j$kNd6ezWER^A}&}f zJ_bTknytQfDnz9BuIVZgIc~RIRa;x^ZSfJ?<8i&bUoRJxecunm^zWC3WI1w#rz5nL{pdyL29-dej}S5<20 zufP7fcmLzhuWz4!YE8d<`LZmF=S(3@w25Y>kFh--KmYu5anmysrmM99?wCY1MZuxD z@1t+u0KC1u38FgM0hdHDN%q}yiun*o_W-&FTGKJS_kH+YpQ?(9*JxgmXEKIuioxkI zyjsBKx|B(y%);!5X{tJM3?DvxYMx7yxjkZL2FI<*%u`eIJXB6O_@=KNM`jjlURrBB zhN?ITtQ9J^*E9Z%qBXd4yG?8I%jHtv5*=qa{js5XCf9n%PNupJ9Ez`$z><$wc3_kB^V9U%!6)_N}gTWFRBTEsGFvnFho;G3cWv zKYy2q6n;55?R`61#8l&OAMV(3^hlA3niZTecP98#f1oyuWGrhltL!yine%v?jIG43Ui1qpbROViR?x~C6t7TQN!4)iCjLZ|yd z0#J&Fd+|VWri7-7k~>gIAP(Ac#dicvlv9~4OT=H-nfRdm3`wC;Q~m}qv(jrKRI1V->1&EMhF`>w8 zd=8z?`1#tTeSI9UqGTlMPE;*Ylv>uPMm}%B)0g9n?Nz2kcuq!Uo;VQUN5dwcRLIw` zix*9plXFu~n=^Y8j-NGnyaxC0QLXPdm@mE4A71wSGh>*ULL>a~xPSTc_pg8car^ca z;Yjx=u|KbZtIBXxOpp0n#e?$tOIz*sbZt7K%3~T3 zD_HdIDUQ&VIV>Ot)ydQ{%O==JjsXncYw>srr}Rz8WVd-_1fWugS7xG`wC)jcxm+f? zkRCCP`(0t2MG7#{VOFT$&@tzH#&CE$R4r)>LJG)fZ56SMfymZYI1n^3p{(X;LUJ_? zq!aYaDkCBVc}DhfBT<2939Bij)QF2$u!uxDCgkOqT@O*yMO7bTGc#L_I#ph`Z7W-x zWnE3Q0EEkO%}np(zHQ60ET$|IPbN*(R7Ax0sTH-)gjsc#^WXk|oN~dXB zN{L|K9{|Xb2-bDAW~+$YzI^-s(3*SZe!pkt+Lp>-TWcm70TXElGK!_CaCEOS(5u>t zqW@-f*NiQ)gttUhMVE+CF&%&?G?8@{!Xt&lKoJ!MA!0xZj}v56VS$*er>5@hOzg*w<{>W{MtxK`V_AUF~OaP_o;WCkw6otX6<5Y~A%+j%i zh!`y-(!mr2y^r+h<3U;1w#u?rkx`Gg%XMulJRbMWL@sY@Q|pxd{?KHqm8GPoTOJGA zwWKT&ImgpUv7^<=6wNI5A|*MUZnA2In<(hq zi)ZMsP?(tl2&I}e1!j=UDNM+mTK3RGfjzU_+`J()~3247+G0Yoo()(!Zyi#Sw#LiAgh^4#!&^9Jei>6eR z`Ki6t@;horQBe`8jQ=o|mYOmJFfT}EM)ul7M1;q9-23D1V>FddZkSj9?2g$WRd&P03UVj_r20 zCT6;rC|NSNNI$HcW^*vIv<&Ahm~kziQ_EkzbX8rNF3Yl-iIyroBZ4vgP6aAd6GX&n z*>cTg9Z4hHT{D<&Vukg`R1S!u)u}eKjMy`CjM(>q$cWi~Yr0gki-9qEMDRkTsA>~& zU(Ae&^ys}StZa@B6srQk)V-CUMsZJGH z>qkyM1Mti%N&4u$&*`wDxroTRu8+OvwpCGFJrw|B47)TG@mPvlFC}a-#t=ln6Y%N2 zS*1)VGZ%Yjx|dfhK1R1ZGu#6RFmfNqHni3nM2j4xZ8^e=)c=5z3hLj>Yv{>F z5fOQcaJh5lDVCBF2^C;Q_B@y?#btsJFWLiV6S1(0imNBj@(i#?$Mr?4APM4p+7a20 zy)56IzVnOUqa&|RKt7%3p4RC+fQ$1(pYJALW*@8=KwxGv=5P=)0>PKs~*fWL6j9OfxeA}V_o0ge|ZxS!LjXq-{rC#`uot_@E z9LA{9GXl&^VHyOI0SGuUG)Lqy8cU&lHQ#-?+^R59%gpAT)uvMtNju& z9*>8+FH74t>*)g4-OmU{l>oxh3#(5B910eWI4`o20!e{nrla8Bl9ZVW+JHPh9<|3S zY>pACDzs%;ChSy(^^^#TNbh@zxnR1#1RM&;JtK zU&HR}Uye8Ql%1^c;u5<+>FnWY&jCy?lDT{`Mm|g6*Es&e1J9q4Ob*|3c#h#?AN#gH zHYSuvBZ7yTY)Xcr;rQ*f8|%-e+Iz3oki2NP7rrrTDoIIEEh0)IF-DC(mPW6S5oT*< zk|n~Y%hJ|$T}?~=znY~HVaBE=BF$i`?w-TbJw*^15oX2+stK|bN0}lPRe+1g!DkB5 zEXi+~Q7|Y`*aCnzxsr6u*62K9^~f4ZG(u+g-`!Q$JWh_Gstia0dr|bqNLYCsC0wO} zB5HZfHW+#HAw)*(?m#rr1R_U7K)_@oYx9f|qkHO5+S*z~JS8bo3c$$|N_6~Ncfb1q;WIQT?6y%#t^Ok`lr>t#%aEvwsBUEy?p zZE<~gky(OEdu)Xhx<|SrQb9XODKQn>K~zFM zappN->+?O-F@Ts?Ucfr31*KQRgN5wwNC9ZJl@wF6wHjn~y6cN06_3Fs9w5w_cb-o` zoH|SiJ@0fpqbDLWpdoU`W%3{g=D{93zx9ej1Eo%Xgu9o}r?WB>9Ntn1B*dR3SDa_< zc^aKS)43|t*Jp4)-^35`{p9Z8<=md%kHZOzN8{v9RXat1|q|~P93~nN`T}TCtIcj_Rd4^q(#ch=vRl- zoL7iUE1A~X(yZ7Idml$|UzTMtTULyvwVhN!1nq!{%q*FZC`YCdpoV*z8ad^|rjIG7 z*h69Fhgm_5-SN^cnHe#*(VvT^CVoj2PmYHWXiZtTM-HbTB8FAsuh+L@Jpn1AMlf@% z(#&iSIn>9_a(Ugi$U&~6NEXM#1)7T@!fBr+wmgR2i zrPj=%#s%TF%{^-rwKX^-DF4-QACru3c6wB)Yq!1owqiPrH{^Oig=KXqIYjS5p8} zLpG3=3TNi5ax*A^YUkxX-if@tnP)h~ zac0E#4dHsuLE_BJH1EFwK!EARu2ePECWVlVdeZ==%jhu2B64(c{Xk&M?0Grz*`Ds0 z6IpgB@MmUvFsQ1lX%=xLr)G=h`uO-pTmSW6ero!UfBfw~KK}e;zkMjjJjo>zph_-_ z0Nl~X=IKou(nlX8YjUIkP4#`fh{)sd$jnOxC9s9obzGtS4l%d#1NsHX?ACKD|!2A2<)Ajncu4WePZF3*V_VyXQ_b~$T_P*Y} zeKE7EX=?8KuD;i5=$wnpd+(zIfmDPNnjy?)P*Y7*lPn@f-R29I%cTW!jInzMGN#a! zBU2E9u!=d|Gcpz17+ZETyDsgcKi1aBWa4_ctk)~a*mwJUc|0Dxn%U*{#lPN1W~%nS z?eDZVU7Iv=s=?m&U9N1+GIPYp)>7mVT;AT^{!CVvcW4qA z9bJ4!TbG~=<=$vbLf5`?&!m5y%UYzTW}2c(q!#e*hl4?ktUSD}-RYS~6KSSPM0(`4 zS=AN@Do~T(nOr{q@56V0+%tT==sK_@G~?kzMnuOLpGjUB=!?H3%R2?YGC{%MPJ6qv^;PKAc-f{x3iO?_WN?ZQEY1FatxS zEvxx((5I*ghWZeCBJ%Rz0F2q?<3ZA$ z+~+W-UTkUH_kHVoCVKXsoh4wSsj4)7+4i^fjT!X)T`xKIv2T~lB|U0FwL=FSV!4Q# zJ(e$zOn3x92py!bse7tkLUivA;kp{Az4xYhoWUTq@>lO}tb~xoL}*fT(cH&-X4bqt z0GXa?vhN!yP1aN3r8v{Kyd7r~?3jZ6uatg|N1UXA^CWqG?}+BF4*(^vDLI=G^$?lE zQZL68P~ydmAj){{@^=07)90Ul`YAEKef|CUdk;~8K*}+Ad7hI82>$GL)}5|Qfpc8A z-tc#>QX*moRZY@tsVUYRPx1Raa>kJ{Yd2v@?~XAZvWt&PYegFv+dhuR$$@=+ydL%S z{>>zdSW)CE)@Kpq$b(KgI4ZTN9h*dtf<94`n6sivDO&l~Ju?MF)ZAE_SQ@bQdHB&;}LzFI7L?765{S)HJ*|f46oBZ$EcM+R@~>FnZt*FTWY!$SC^1QB*}$noydZ`^z)O=s33RGC8KYELB_Ex~}WG z){x+dm7kqYu~us@rrXbmF@OgZG%b3I5F;Y@bkFYF1i}&-F+6f)IDY!+7ZFQ}sz4|- zlA)?>B{slf9+H-1vQXIK`xMYOVgH0<9QM@6f;Zu3DR36D)f_cGF?~A<0jaSsU`D)o zi>LU`Nsr`fSbvT0ufhHGFEZ6p=MyF@e)vV4RYrNUWoBw02%!%$C2eVC+>@-HLKdCf z4=`sq6DX6K`adIjVM^bBy$H4}b&)_O3jitpuDeHO)i*Rz&7w_vjT0OeC7#j8s7o1~ zqfaGBS$iY;h(3IuuA%4kIakuWXs45Dl)4+xL}W@rDkXB93p-Iy20<}K9KN0+L>7-_ zY&FD@Oj1f=7h>$#MNKJS?t4jCMV&@qPz|a9?_&@cKE#XjNE1=jrdBliNX-93EnTRCydc8)(wr%^qZ`tuWIt&KQoJTAcy@(g^x^} zqDjxKDk36%?>so~a^QG%5?n5qz5ChS5$d|G9!9bhrH5G@15NjjTb5vyM`RoZw8)b6 zwmEZ)Lx6Tl7BcwKy8-fKmAID<$BY{_Ra-GSjt77`IY{pNez{yOm&>v&QUk+t=1$Jz z4K-NT`@TK0{6uU@% z6_RRZoVZH$?T>I;?dho{hhQ*Mra{d!U&ISM{tWIVjGmE4-&Qiu^gL%q&pGCmz$>-> zpFV!b$Bl9diZW3yIU*9r5P^WI>O@s2A#K1Bo{w?A-#+g5kNd}$k1v>01IG?3vm5}@ z2*E5t0U>pAW2!;#mFDUkafVA~*5`eugu;D)KzN9QdxVU!*Jr7OmnESUjy`)P8G~f4 z6&``6y6{D=Dt(?>Kx6OmJvFLYLm&OyS#@B>Y^spnsCs&Ulweh6qDdV%l|qnZ(MB_K zC#%CT4Jah2u30pqEck19Im#ZRuuzp0jv`2?cpu2Dth}kM*5CmTTU5*Asd`W{-;ejP zKekp@GM*|zC<1-oF)}Svm&woZaPODP1;C-@f_-!|v(`Lg_wn)Z5y;4Ve}AvaiT*7Q zr6ZH44Bol83pS02`b5W&5hHx`$Q)G^E$wnPgD@d-$;`G8WC)9W6p+%M!|Vai{Sz4u zW`JnKs40bt26d+H`(99o2x_%uX36A7j6hqLWmy*yDa{}y7wW3As)%U}`WT+Epc_IUVwwkMm3=bq@es)8fH*O9aQJ?_y0H$kUOjZH8ZQB%HJz`?5$v;2tO=S0> zpqgp*!vzsU1P7Bj@)!z|T-&14xzS({gd&DU2IihBG|6*1T;a;zyqTo&35G-@7f@Aq*)wspB+oMv6X zF;>=0tC^9R!t#f^cUH<( z;A5t6nk4~P*X5+6@EDiL7g_Ar1B|38%$NL ztqT#`w(+2ZMbSJWf=LyF#$!*WONfk8YZZu@GNmkoO1D0y#g!#B;DI41J&};GF!b<4 zdOt8hxd6~iYF?rT5S&B2$xM3G_~xAFcp8bH!F@UqMIhy%BAtoA!B6M&s-4T?zXXZf*RGE195dT<}5f7$bxP}WYF1^5&IsuzCA{Vq_q~~At!a zNT?JUHi8i+Y(lY^Ev6JP)kbC@m!-ud-i$nLjMB5T^9fN=Rfq(Wk#C>gi(Fx5W^+uA zBz&g5K@qSI?;bIH>wE9Llq_r4)c7(B*{yXQ@Psj$EdU1GeTdSJtDS;{Op!T*t`Q+) zmdPmbl!NgpNKay#{t;wT4%GlWy0r$t=?+8IC2F?l*xW>mOo%AdRWh?s&L#?E*~VAs zT5J_ajL6JpSZx)N!NHW-${$%m1@cnaRF(ivaZx z8~98-0qj09hr6E|+10A>(PN4gmndgDihzk`){KLM6yTPn(^o^-!W`)yJ_5X8=0I8l zI49xihrGDDDz;oABEb>9J2Vk7m@pH`@V)Q14+$UP^nuLlDr-{}NSX3cdZPR2?m!I( zhh)u1CX2>Ol#)&Hqbt{KtxfB}X;EWdwa(7WB(!(0gmm8r zA*$NF3++G>>r5rds@tt;I*F7_1^fug+j!)m+vPFRvuBTpwqD7QEF0JagQ*1elItvO zAJIAQm6*+&<8j`OGQq&X=RP1iUzX);h5;u(O~VMGm_npyVDzo;0=`|AM3IKl(=ppA z2rnvnI-?8|wJkL~rWvJdnFMg>R#G>|ofkm5}7-UG>LJ2fW{$dac|NUF2MXFRCD zGzyF8Q7TqZg95L$mrX#vEOhg`1|y!qI3^1S0NZ)v&yybqPWRWxbCncce_(A9p@*1a zW+p+A*6geI#Vp{D{r=_KpI<+|Y}@A1F`tD(5$lcs=ox2bT6ZW98Q$}bzIKoV3RHc1 zuEUIsb-e_<+Ex)MTBEr&OF}6h;btb9h?HE{Rdo`0RiD60jD8L}7pS*w+qP}BC`q15 zLzy`;tVzidj~G#@j~Y@`8$|$yV*oiV7Q;SibzSN%dXEuFPIN*c$`Te!CljNo9a|S` zTim^dB(pkoLq)X~D^wOBJxcwmT%v2vJ%?}GW-^hPI@@Cm?_JUqRM8loj$bi=msr z@>Dv|+sd{uMn7V`kj0?&YCT2-1u|WHvT(eP82cy#xpO5KD4fQe*4QzVcK|7BsKMZf zK}ivqsjBYbH4+~Ydxm&o3`Bf>yM}wl=-YO#W>H+P7i%VhNH?l1O)_K8@q!>Hz?!gU zXoZc#)~MzHkXl1Ac(IwSf0nV=#MZO)#3X|%O*jQAgg>Z|Cs6uykw_6mW(|mrF-l0a zevujl^p~-AE#x7mBAz5rS{-Hp2#jD%?^ae)5}rxS#1_tziNU}$SBTdXvtnFzXkH&b z#HBL`aFQ1ec;|Sj%KhWgAi~Unp{g_1CN3ADL;!bLc$pOKNJ0z(2Abg8-3ME0GN7Ktf2kwi-OVCe@(#bJf8d|zIJ z`*U9Z_{({8KS#69ME;*f_P+-A=eHuy;_iH@`8ujpMa?oCp?QCN10#L^czk^M^2eWl z{_*wepZ$KjE)6r05|VN_XWsjRRGLajYMF+6amz$RR477Is6^&)UmSDwC;6NiZ|FZ4<%LnwdpR)q4@m%+Y&9nAv4nY7K4MR*}86X6sTXQ16|+ zv*y|85iueVBxOikS|iItTL3tiC7?p z0HWwLF&-tK1B6Ho_y|(Gd|%@vx>QQ;>&4DQ=k@VQFgrgWfK?VGBfWyV01+ygm&>K^ zj;T}a?ml`D!)W<3fpw_)MUM{5S^6{13i(r;CSt3*d-iZIc~oSPNyuQtv@ZuLh{th$ z#3A8(s{5W1yy64*8M4YmTuP;cI)(gzL;0QVK78D6H@vXun1PH{wfjiXR;r}{!owpF z6ECG=GCWHG)FaEeQ|ro)F$9W*A$0dz|K%MBM8exrkV8mCY@lEn-}iaB)HwORu@7vyiL@@I1{>#swYfev)NDwC=D=q@6c%R(xW2!6eNthQd(tC%XSG=@k9A2`M_ zVq}pNuRmS)$HpUJN_go`fMq{P2!Ii0c3qhn<4%!n=uRjO%g#TiFL z^gcqGKmwVlj!F$Wd1k7%@NvK2-{0SV`Q;Z?z2EP>Z~NG^t?lwg*j~&3GrmC&kIZ=4hKMM1eW0|RHCU4tg+qPaa z$$=41<-yX|J9Edg6n;$^UsKl~vimbSnc25*-(JsvIh?pGTmSmy>mTqiv&Zd&DVMeY za%oFdAVv`}`F|Pv+a+0&97z;Y1a48HNDp^27@$z7ayW2XaxTRJh1rFnyGZXnE%?$jTto;z z%UE21d>dU2%K05fRa?a zm{jXCA14V=Mb8tGtYZlHWs3JaE%B1Azvjg!GilszkvuD28A}n7$;?X zVuT&mQ&*bgb;}uw5KxB9y~=C?-@k5VR*Jm6y=20f1h4vw5BumVO>SIFOg3hDx71~n=TrW{Clg7lo=v0I zecqeTbvr~$+?i)2Bt#K10M#Du5xeIwL)!64OrhO~S6N_S@XDvRgENO;-8eKMO96$V zK}=eG1fs`aALG0Xpt;q{C4}i_Y7yPrAUY2At^$!-%Me7w`ovgeUJr(*qlkOXm@paU zWJ%rte|`ZdWx-3@dv0V7!hEPG6{@2GFf-Kw%Vd0L{W+sQG#-Qk+7!LXCo(;u#w(lQ z1XN08%*M#~i}3Fs9(2COdEiemne*E6&V2j!ZJrl?{t5sIRMDW+ zS_O1Rb30nQ-|vM|wN$N(mRd@o2t@0`YZ zNhD7&Ie8~P?4_&zx=?-!?rmUXFX-j?PkA{I=3>Z)rxLgO-P&y083 zo`?LHRWk?DgkellWZv;wm*pQCG~tMdxYnl;hcUTXhlPuXmek-b;$@8QyO*(V-Q7DF z>ZNGnhGE`XkBGu5frO_EG_mQ!4m*xilB=~wA7-2o>TLM}V5tmMZy0zHB%EZ?qB;y} z5)d;Oq28gCq6iDssvI&3k3iJkoua=1 z_h|D|{M7s9^sqi})cO6a4@4ld#}VPdaQAK7+~R(}XU=#W$FeM`>~ofV@*;sUj-Yr( zc9N+O2_V9Vobx~xjco(qrz%7D6#7Uq$+?})%)|YkB)}|SY2rc^8S8U`IRXgM+s+4v z>Rc+dD$7VRFnn~QhkrdD3RX(hvXphxnrBf8l4>m>gi07Z433uSl_D~5PMCQ_^mfln zEkcSg;Jqb_8xECGPMJ4%^zIcTi%1I9!iP~os4^+#BrS59Pf5c2nQ@d;bu1OxBJ3=2 zW~83M`R54kFIFRez`0(}!*h?NOYQiTAy+UOL}u2kN4U8!5z!7DJ7pC&G>hKEU0!DI z@-=W6_xxXaGoLy;r4*bKEA!J*B+x<4F>WD}Du=+n-%C_vZVpYG<=(xfj>Necn2o!z z0K-LOS=aaLw$%mZy>)B5MX2hwtVeGxBkvq~%4-;1s)!zLNA#ZkmC?C{lrZNIDn^HK zDoBuA%2rSmRS1I>XlGDq?En~xkYEYXA+jb?(xa{}b(MIuN5)Ongy=E5c}^$IlW`Lf zW?e!`Xa;yJIP0utJ;{z{X07Ffs^@?)1OgRPl7%F~VXdc*B+23>yB7eO^(<&qb(Pho zuqwzrrz1kO<|>lWL#0_9dlwNmWFt;h$#9L{-L0Q1!e6?AqM$^C+stM_?{=sv6;K4j z1Kn(IN3(u!%`F?Yrh)L&$f8FW;3PP><_V++LV?ly3J-{e%YJwfH_9RoN=8`$9T2K3 zg3%(5FhOuhblp&XDlIg>4kx(Wv#->BB>H{_@_$p{3#X)b2peD zt=C%XR+puUh(&LEKkO)_h_GI^%d(NYA95Uz-VVy58l|KjiFdaYu5AON%<18mY5(5O z2bzrC41n1smNSkFK89r}noTOrn)IJvXY_x~n3CAwM8|XJ`TQA@14f)7-pryk={*I1 zfBfUOukXLx-YPu zSzk)+c7&z$nhB_)RST)icAlAs!%>7PvXnI+(5SE?9O1;#Zq9Bn5wYythB%!^!fn3y0JvLg*X<+MOs&;gw>gla)3WL{rHF`9%WJRY+}JsY zm+zc;Sw^d6sZ_YQ!Qn83BYM)AgKnJ>1{LRI8#rY+)#MpHV+{c!G^U1axPscPfdXX@ zic$!bg^U(GBbW8guUq;4eLIeBwztR8t+ZaYZLNPQr5?LK_FK^{jsKK|l#(?}K-RbO zv(MwlfEOPEnM8?HLgn(ZYxBMV7&`A~gpZNZ#8NcfluIej;2x zB9d#q%)TbcQtDF%0FeZwh#2590QcbtCKN(XRWo^^#GcKH^H6=>UkHFQrS_7@AwZik zJq?SGd|B6ilkRx5AGfbx|MPGA*A4F8fug6hOCZw_hcL_;7%1cz&&8P2A$1N_53Y-9 zvwMU$dt?qa52BpCWCA$C1rXI5L4WkTZW#-d@Bnb#F7CaECYb6utW@h(w~R_R8$9tU zRe__oNY#U+eUCDv$+MWt1_Gh#8gIVZZSoNGfU3bx-Og!1rHH@i5KIZ(o-ErRirCI0SkCQg9Iie z1}%g^cZ7TEZtfuN)?3`t);MHeI$Rv8Tb7WXl_MB z%}hplU5%X-6Ct_Zjhqy~L~0m=GC(KGIf#nl<;JJCvfBK-%WXu{-LrdBB z-ON5dKK}HlKmGpukBC^7rM1SHJ16{+wL4W1A#-69upIdw%$ZToC+w7IF5#a*{w4;w& zU!h9j81F_zL;+nODu#xvIGS|a9$y-$!Q#F;;ZO&OwGOJNAtcQla!A80{cqILr!s`_ z9_au_n4wAQjqI|QDRkm6X@am2|>WBs)w z@h`K0v4(OP%|jA%RryKmiV(1tq7-^m(f7AYk@)&P=!1($(oD_Ft#fM6o-_TKk)3DP zJPD>$7I4`TS@OARi)u-^m9Qj}S+q>kC4%WYVK(-La*SZ~qp5N!<+8419LLdn??<~G z_nutcVYL-*ezdk;w#?gKbiffwGi+5K@llNwCTbX65EWE2(;?b;ljVj)4L z(FsG21j5~|_b%v(ftdSo9JQz}b&VjriuMfRbNa?CU3CWl<-izvE9&M*p<51Lo7C9b zblvy;e!t(3U3%KRc57}Cg{3TYG#jS9{o7XNhgB^lh$jsuZ;LR9)M+rY-+%udGuK{f z&6_Wr5Z^pXXDT5>a|HlKTkmzPp8uLRbDRovqa}@lDuo4%MH1A4M+^mKk_dXz_y_=u zYzpW506?4yWj30a^khH>!q^QEnj-cGw@VR0_eRaSYE|WKH>jv+ceDH5Lzen}*{+X! zN?y~#C~yK!A+R7KKOlzpDpOZ0=ZMd6FC*2Dat8F**S_!Po_$@P>2S-ZH&EV5Q7t6j z-@kqP_Dxlzuq;Cu;59SRoaPc`3J_-Em?0@$3}617r#J-3p_ODZ3%b!W^YD3^o>8bw zUAdS>sjngaaU2;F(itKmwr#s!ulae;s3ia(LEpZU@24(HDby;eok7v&o_sbL8iTY< z7lZUc1QOL+ixQy&>bX7c#k~h>kkG-W2cVKu?W$^E?|nZG+xH$GkhL!BWg|2*-F|EaE@i@*;c;>%^sB$D^-nFXfJ4 zaUA1dneDRNb>JQnF%4}2&^^K>PpjUfw;G0MhaJ85-;VEX-zi?->+RS3kI&x;m$klK zw=h3?0>wcNv!dO*YF)N53egX%Rn~P`w(U5McBER+pqi=b=PkKV7hN`a8LnxqA)+X2 zEwv5{g5J9?87krK4ma5Sc(8HXHifp1`@@%IDQh8lpodd+4ff-4KiVUq6_xJoR>%s8 zZf2nt*Wg92Zbv{I9v;%6F$SjAU;bO7RVh}ZRPWv&UA}!rjFV6zWcWTK!lMPLrHD80 zool^Z->&zgMFb3~d!suY*ZokX-y-nJDXaqlc;)g>a(qBPWB&{}WJjvitu%U7zZ?>*hdTWivwp{uST z0V((U{k}g$_4599Em9%o_BbBDe0+$=*N;!Pe!X0B{gn zC`$DsLrhVnUxd@3C)ImZ$=R+RhjxBDI8{eyaq%wvu5sI7fyMPND0FHfunyd z>!KxYkH6QoynV?1Zohs1^>VrV+435pykT{VWAf`kc3Z>*X|x}>03D@Qt1(4-OSsVv!SDeh?fF! zxW~U;*W)#N+lgGZe~bqWrOzd`;5^hHvR#gV|vy z5&QEqy!`bu*?$F{N-6iRPnlP8e&iT1%P{(qInCUAmluQD^Y12PFG)@Y()+Qs786+D z&(jt$w5VFDA)e_K$&))DvrESDR5Ehuc`IU$>YN`iZ`Jv9F#kHHrSJLu*M}uj)L{fQ zH=ZQ)kkpwirI1>y4N8X8&(E8C%STpA&(2#8VFKbl{IINtvpcb_m(d6fS(fF3xZm&l zzCVsfMATYA_TIm~zH|ug7fwIVD7I}|)zW+K*3L&6Q!_Fmw#$MSs3C7GC)0*xWHy|H z1lZSocSK#*qQym<`R#GPJ$4q2U~d@=lw_@|Dq3sd^mM-RtRr_3iC8JA>*5FlybD#h z6^(M{Mm@FviwFzR+^n-JiUh@(4Hg}e)t<9FG;s-uNWC!e%-&xYpn zUqyt4?!GKbT7J2E3LWKrOSKFj%nU6&cn-Lo=~_&5zLe#GNQu-KgAtMZ6;;jFPO7#E z1tAFGu<6JFkW?HnkO^X(g?~2E{Ds(>e>^{_ht3A_*txk!0j#=}rApCSYqL%={Yf%7 zI)?~DMC6zoESU(RrR0DSJOt2XNic;;;TxCkHfi5;~SG>wsov zlvwMsuC*?@EUO5YA|jHWn90%HY88=eGn-qOrBR~RQRwQ7d?J#TS7#)Awu~c!FAnSZ zPo3VfBr~}8e*ED)$giL1_Rplrm}6<@;Su9b`Wf8MaFe?+C+QhrI<*E;m_i|xq=O79 zE8NKxSPsI4G0S@Ity?+~s}>!d)mm#eKXVqJudLv7P9fVI#Xhg<>=>VD&$_M&9f(MB z0W;itM8?_kS4xp|ktBx{iCF;M4dFe6BMUjvPeiy|J8Tf`OU{EZd9gn?HqJAx_fzK@ z=Qa1ot{OUs;X2KX?H87WP30)lAcBA zWm$Xg$I+vkqf<4lKWZ)WR6gfW=)|%t`=ePm0L!wH(pum5BZ)CdpgwxvTMqZ~{rx>} zeugKW2FJC&F0=3x&^9wM6=WrHlmOU z5f2VBGQ!8aJ{c~6a>$bAA8r=#FkwW2P(-YGx5L7VN?rJ9-db}D8l#9i1K#Z*G(_Q^ zh+zyGTuvytz;h%yflpP^Q|V;x#PEO4frOf5%3A9nRl3gg>k+YSH6qM%%C3uH>L%3$ zeBDx=e>ZO`RWbZrGD8{4uQCk-Cn*^dlxOd`qh9$Lk$6tT`Dij$%G5eNQ&|hC)5gOs(=n&RgDH!%ZLxp}w{zk;nkJl~tdODt! zM1F)kK64K6JblkSq0{{2IG(nUt+i7)>^vJ@-h4ofPtpR367f_Tc{bKW1d#j?6L=*6 z;UX-;qKVw~)K_WO>w9aBe6-dgthGId?zCUsvkHvxk+BIRoJf-VgvTrEB1_kJ%^UL3 zqz@M(E|-gd66meLqqTNjMlmWwNq^#m^!a>8`+X2*kYhz4at73$Fv|9JJz9TsYkLdy z22e^_uFZS1w2O@x%Eul7s8Xtk2y8g`5hVLjQ;!R9G4T*Y4_NNo9^S((+`?74T`uKP z?>i|L5ymE{5CPbuC*lvL0`4HaSon}0w(K-!sIn48gyybuQ|JCTO|WyIK$5+aEw@1m zN)^wcCQ_@KS=viQ(7FT2rJf~1@BQ((eSLlHt<{<;QIXL*7ZC{rdTTG|YbKN)5m!R; z<|QxE6LbmqPg9SGA!^u988MD#VMKVGN7z%iCjZLWT-912j2IGZX&Q4_@4buJDZvvx zO_<#&TB~xLFh_f2+AboHMMgU9ZCors>k$!_QpN*k(khCRKJF3s#}~(}i$p|sJ3c=n zBKc#>S~3s;fK(|ODNq29sdIG(UXl!>O84gCq>)rPhf-P2t?oXz+6?Y_56|F!#+@^o zW42OX15^Cme3yS>)SOVpc~YO}u&S0)8E|Xuu|FP%ce`v$EcEC|jehUrXGKIxPsy#9 zk%Zj;CyIiX)Z5IeJ{cFUAjai#(P`|GOE!bM4eW&TeAb*CYwe!CVj1frXcnr^(&voM zDR`T~H><4(=4w{Li!csA~h=#bC^`luoL?oM0HrInl z$uYkD_6SK6diG|i7O<{s7NSQx(pGL@DsK0OF|7BaNRmLAQUv`NA3Td+_g-st#Jbje z3|Zwx@`7blq9j2NiR4d#5G}B9csK7*3v?F`c^rpXfU2dWOQjplqLgLOhmt)Uk$35F zrw=_3N7wLBnL~1h#uWUlFp_fk-;zkpD{wr&yBsJ95D1^*jT*)^a8yXEtdsWA^9n$p7z$53%vwBM~J_ z>Z0zhT@Tf>QV(K9Jiy*NF>EV~P^8@VM+~Ap9IdlzpTn`yCtq`) zQp`iKF*MxlzCUEw$0KZ{&6px* zygq3FXCS~l7&8g|!#m`ySN-Fx^UE6!&5-kiP}Q}r{W$Ko`#=8v_kaBDKdbO<-O9S| z_nu$@#|$G>M3up3fS!55q*|s^P6CvZ_#AkKxxDom)6ZyiUJNEUdon}veI|FvzJ2@l z@$vEg{+@?HN-xPwA5_&(Pl@c~xBRe~w+bm3dDb{GU*kJpviS2d@|y7+qy}Dg*?G-B zE&bz=*+?iu1{MI;b(IOuz836`AfY-=Xw1Etsa#T8UY@b|BOKktOSQyR3k%8Kx+=@E zEJaT00)WrY&vAL4f2);JM2owx>qVCu5pkGVh?H$xdOVIJXmGPYOoK%eVQUA8s5Qp` zfzUDz?0k;NE*PK7b)4D5sbeG@M>GM2lA53cu~y#q$KedBR`HPIu-m>r9*;+glV~YY zR9A$W1)Bo`(X5{>I7}O9#0I3P$gqlLxB{)4*?xaK-qs=sOEU)#?uHI=z|HqC(-fF9 z3j$OL1H=KP1Jdje5v3|Ln^6E$vnJs_x1c|tZ^?sPM&6KFE`UkF-$Vs2u2*7 zaTe`8g*Bs8-L~cJ?LygA^|<$QHO3PP;46m>F9;hds-sO6%}EL(;DXS?rIcC< z%tvq!VGMDJFt?XrSWd{v-A@Qto)6}X^2i~Gbk+Jpus*+d-TAMp#Qm(Bp7A#+jLBj= zGmCz-^xp7`7>j~?gF{|hj0kdw_)0-jz%G5Iuk(7m7471^Z3D3lziNhJz}j%t*)w=wCgfLZRx$|@C$&i-U%h>VQ%IK_fU0E&8&FI z(>KX0GW>B$U6#kO-}Z-1<>X9!%q&U~ky_GnO1HX1%5aWlW?5R1V>Rj=Lm%st9?8tS2M+7a#Y0@gA+T^YGMoT-n4XKPACYb}m&Kir)mHV$lxa9!7BDSJC2Y%Qk? zN;t9sGo^9PM%rYOf(%I`AwN64={QPB*PyvgW!PB8PtqpO;69;(XP_UysUww*m+v!s z(~~JzW{d*hD^*~&y8C_{d*AbzDod3dhoEK-6mW;vfnNW*Zo_&ckxckpE{t?&6 z*${uW-ChycGrqqD_dkTopW{BxvDh3HZmqcw-pH9mkZjEq*p}^jyW-M)Ld@ zl6ngNO*w#9tnH_)A@LbbU^>~<@ghrk-i!R#vnt(rIE&#s1 z?!EiAZR@(0vZx03J^d0SC4)HD&A#v9NDZS>ihHESqXEP6CBJOeXSR^?K%|1Fce5A z1oSx4@h2~Hs^NC$laY#w-O#<%epSh();1=0YF7+8IQYn24~pG?2!)d5tIT)_h{A+LvJ2IaH#R+ zkW_2#hSs7tkARui!EJ!BK%6Z0;AsC_^weq1A>ZD-wVpQRWcmxssAFlxtCH0o800!`o8b&uVXPEh2_89ZCo!a@?FnonIUT<9dQ}TYQ zi{$uEHY!D=ynVZUevgRT*CQgn{qm;>v)MB^B`BLTJ-DtT||~8vmfg{ zv`{tedZM-7`;ihldB>8@0iXyhtDoa|xRy$K%n% zyB$o?H#dO`qX=6IU@$djE;b_3<#Nf7-|a{t!#n~dA<;gFv)K-@2oZIdh+LM78t&~Y zXC_Yq2%^ljoY>=Q!H0-+6QVLmaf%6+wFrgIum_1+m&_I7&ZSbPsI~cr&vHoT)#qItZMt`FQ~V+%qi|_fee>`HjatLwrOBTtt>q zweZY-B9Wt+3gqxkL>$KfvX-@I7(%t~`<`5;zy9^l|MqWxIr`pvU$jo1AC*jV+)@VD z_Bf8my_lb@0eK9nJ_R<+bV9U>n9=_B0VC)<2Z6LpIeV4B|Tji)xnZD6Ply9 z0lmJkKPaM>g8Rq0;|64u%$$gbC`CTr-@BP3YT&-_J=~p%urAB;_V%V6@e_bjhe>3( z&o1E8NoqWo0Mc*SbIh!kl^SjcI;qMM1nB)ZTJO55213(>ryvYI+4^B^E^4{36W}CF|jqx`I1C(dw2JKG25G2&zPtr z!=HTZ?xSm;ib#M65fMHx&U1aR5Oni)SnoB(1c)kJ*DF%DcHeWw%cP*5&-e_2Bs21! zKXd-r>zY`WCH*UU@1w04)_XtNo)T{Po?h?Th2x_fQ zZjZY!b$fkIcy>qr_$=}Vw2X|nwMHL@cOHc2=jzh}P(;>JNz#W}r~p+}LPVtKQbp7E zClcfXSR%*L0+BhOh=zpLnjB05|W&Ym5$QD%uE4@C$?F|1nHMFd_@N~tFc%S5rR zRRld%al5tFx*d;42!ilsS(f^Kt^tznbh;)x)#wf<%=x0_m!X1btHzNe2Kl=n;U)DLF8RVF3$S4uG*FZu{G6dDb!&9A~ z7A73Fy;4gN3R2?uMY2%IfO!g-ba+#i%2I2ns@7+YTjHfudET;@Kz#<0dH3V>JA9eG zd3~#w1@Xe$d-)S*;-5)q?!bNDm%dy=31XCt9**cRuaP{?a9H<#JVqx3nfO0Q zs@CyVRFCN0dmDtpOmklEc3#_w3jw3}%}Y9aBrz$c8FXGg(evp&LIGhG5~?W7x@{W40TO__6q8m* zB=VrUd2i-}$XH6LAJ-GMT9zeE&wIG4_Wc0Rjw1`KS_(;#=_MqhrG&X7Qd@Yo&Ig(g z(i4?%2^WNn4hNW7g;F}*+#^r~%diwjn793C%@A5EYPo1B;m~pNMe4u}0-U8yp(7mh zKqq?A?5z{d+&X%%d%{(HMiZ|f(|Gg zVL>TLzeYsWB_e0!F*z0RA%CAmh6XG`!Yu_I3u)feRr%blWJJ#8<#Y}tzsx#&%6M6+ z^7eMQT-W#dksxY9k)NNRXE#e#TPp&Oh_xySOXci{0m}m^{7TZoQ^%iVh{rVU(3hR^ zhY;jbWh6#ZO#nn%Wu&dH`H38yK!PMectV{v6t)P(aJ^R{w7V?}R0~9o<50?R9Bx)~ z0vKU%X2nj;&WH@=$F#s0I*gg#N({V6`anUd!`)zF%0aLbRgZo2C&p7GX6qvNh4m1) zm2irN<9LBy&rgUK*g6?u@$(DAvFw?=pON(E=>Jo%l0P@i@$$;=sOk_uF$3_}_xpZ$ zZ#@MRh#WN7dyC%O9jX2ccUbo(LHwL?XCTOTJ4qTiPv)<$gk{!smB~=_7x}<5lq4{f z<@|Z_+kEL{#AkZ)_4O4I_xqjg>vFklmzB>+jsZrSr6BXTA?{%ovtjkDZ#W@KpQZ;g z+3silbv`c+MJGvd;6EJ3F^S>x%#ibgaPo6XDQ&W{N+~f@zj17^fQVolpajmLW48cM zwmcySs3p%a%)ktD1`7!mmC;{fojsC>aHts6lh7f(`+gj)9cCdS9%f$HYXX-fms%@p z0zGEE2*D5{x&^%VCc;vN2F92!>0?MYix>ghi^E4c9$sy?YJq5 zbT^2yODGFS$o1{L-HwCDr53{d*jsB@7A(~*np>a6P@YT>8)AwSmC^h^j-#$?p2VXy zKo?a`HIay<8k19^C)`OF5urgwXtjvz7fGrL@e5#x^^KkmPOa(UxOC}-x5*%u8WyK$+Pn}TpJuXCy6$;S3`wI?sRyD?_J zVM?pvCD-J9lRVC)$Qhd=*`)4(r^aL{C4(&7mC<`&1gg|vOFKZz<2DsM1Q9$?IHkq(W-;q`VqyG@WGa*8@#)s&rg_F=UTd*DoDXL3_8~l3I&rcJSbtz@>V0W|d z`{O~lh^P#na&v2LC5}=`(a{3!oyXy3UW+VCRY62tuN!irr-;wy?Rg~Ng^`*eFK@y) zbz-)^9&8; zk4QaC)ueglyTAjK2&agO20}F4J+!m3EX!GbwbslX{w$9;6|G2?DGuuHusNFIp;aSH zGT{jWJq#9}VoyQnm^m6Zr`8Hp5h!51G^lz>j^u@9^Fn9wpTd8QxpUBd5Z1z}&i<<#DS(xRD{E5^m8{ zWL;ztUaAU$5!<>}UCD}r&Tb)ViS7^~vMMUU64B{sXWiE2hM-ZMqe{kaK>t3strHE)< zv{uC}>Mz>L2uldwMq_?US}9^grbl`0M!TW zTSqi&hd*I}=lCKg7Zv!4J^lJ3d6wj_=^N@$8gYR#BUDr&;fx#Wl1|x^<*~>7vr4>%eZ{|+n#4!)O zqmWgp0(!Wm1V(rkj!ue-BvTbY35$rH)Id4|Ow${NC`u8BmXfCzBm6iXiT8tU)*NBh zJh~PtN+}|Uu-4mtv<~2MInKR(g5Y`Q$2Jz6OI<|1z8}5!Qbc(HBx1y2C80&h<`Ev6 zYHHrCTewfYS(>~wvs)>P>JS35e#DUO_k{CBP_(AxIHao7rIyQ9i#r2VRDjmIko|y` zT9&2Ms!L^6hy?w(KWZtpl(629UBjhFRd!Jen?pYlFgFtD%~b)>}{tL() zm=Yo7L^$DbR=}xmq40AH2pp;R&rHotsI`>3NTV!lWC_B|j>Y!gG8!Ml&Vv@g7*bk@ zAU6PR7*+1@+(ibo1fo@7k)%tpqzoy+${?YcQHU%*9C4Y0+l`{6;W#~b(3SjDG|?- z;*8EQftn0tNssyYlg0^C*oz!eoFW9CS|xhwllZ{Mo^m?*)N)4gTAQl&ewgu8B0M=8 zoLU_2xZn53=V7o5@DIgo1RHPM6g7NSy&!|-MhnsPE~+$sfY!y=-DBd*)}mp}dTkN>>K&uk1*2u|uDloZd%0QT_b;Q4bG zV=ndAI5RfOEXRH738#!pjA69oKg<{v;G7T0Ztl6K!o3h?;oYdHTtEaZtZONvrK+l$ zHnt?(c6UlL=bD)jG`hQB$$C0JAOui|X!o9MT9JUHq8A}WZ7KysIG-DwI<7OL^%$iM zYF*DeQ*sZ#rhXaRe-7)B8W z%uZl}HYXyVtjaTm_o;s%BFS$xYsvne;Z;g0+eV@uN9!$mH@B?fpP&78f%;TNC7G?o zyixr=c)rOXI*+A)Uh(lWm&V;~o-so4WNW|3Fysd;y5*hEo0+0qC#3VTLSptF2za-{ z-K{6f2!oFSiwU*A?6-b~h>;so(>4r%s+3YNa_xwCI-XmGp_JvR<-9+rHMT9|qpJSx z@3(V-ox#2Lp7;#1qcu-W0N=j-<@V#V+aXi^w6N4#kKU4K!f>s6t!tW=e9bPF^&IFM zb8SRB&+p6zIl@NPD5BXa#4vd;r7TNvZtLtJB_vT^k zbqm<<5lXD<3ShtAyMPq30?7NHXITUt$8n%FOPcStUY2F4Z*O@*C&pmjnxQq<0v*$5 zpc2C_X}~fg)$+aAS5-}ah_q+e zwyj{F8n0(S5Rz$!nnaX2i3-X5_2)(Y8p=;eM9hS#EXx_(dqfd+imLkD9eGAZq$Y-+ z4o!%`93Rq$9x*sXLx?fIc|7h|mPL?gy^2JbQeG3KL3l4(%mwmt(>#5=g0TRpkqwK5CQcsi>zS z*c{R1927cJNKP7th!}^0zBj1WDyp?=t^GJCA?nf(6$KyeAU2BA?vQ{-~wJvRc)a&*B zxU1^*`u5{?OKZiF@D9bzkK>4aM{C!2SVXOLyB25M9*^E`o&9li)wDp@rPRWEp>7(V zU-k3zv-f_vTx#7sEKi!Rudkz7NL)WIt+oBP&s+PwwYIEP*KA+ekjQYjVObV$mNa(2 z5G8Yp!yi4x5UYry!#sk8;3~C>TW{WbZ{9jfR4pxQTK48;DeJa~fbiS>x3w75K=B>BrP2Aq=uBwpY?mPR@yLF%uszO&?>Qb)T zH$yz`$JhN9rQeU^)*>C?1j@o%>Seh^gh2B+x5LzR*|zVO2mV^D;qN~_??3KvzpNWf zjhpods8P)jZlGR1zWsUsdfbnjr)qYIrGzhnP%c81F5;2CkI2!jB3GqG^v@%_Zst}I2DsNsmcwHA*ihZi@M5P!tIe|^?k6FJnP^X%2StmbY z>MA1Mt^1hF@_s~^5)sW>vmUk9i;9RADO)YFM;vj{Mb|>InMK59ybBN((^|XR zZ-4vy?~ncC<1d%>quqL3!_0bX?yj@j84u;qs*0Sl`h> zffj|Llqw>4Ea5|^l^&8LX?Dq~RNz7<2dWLUQQ$U@9(3kWr*QW>EC5m-65r(zDSTVE z@BcRwdE`MZaA~0(VM|qy?r!dVskWLO$KHECtOUD9BYo84;&9Nt0zIS zD+nrC$RJ##SUarmy&swD7pW?GZ)>Iw%ps<607j%je?;(UyfU-wahMG!l=E-%<#nbl zuPNk96Y`0}#7pz!%nB31IH#FJBps9CM(f#I=4sy!<^WFjKh9o`!Svl z@XPr@CIk_bp%7bZ&E|gc(9Nv5<)~^NSEUpJQhusS&ZJ&n)RObC$>T64Jd&A3-+$)O zkHm$A((@*~q{$JnXvw5>WL54KLDEL1;-`f*eDvba57=4@uoNjeQk(2Oz`gfT2c(bh zNPmR%p&EF;$@xqPP=<|LM4V;24o5{43X6u&5xaFW!+wH7i3UPK zL~1GSu2o1nkfXEWp78J9`&t%BZBq{-lvGOka-BmZM{D-v+zB1= z+duxnRQWlYG9nTD{Q-hVxj}#dYT^@@%L<@-AWTBF)S_HhctZqXK=&{oDMolZ9!n7x z5k*<*+hyIhZCRGg=tU)49a>A+IjUXGZh^V8_aMU4WRJDhD!P=qy~85) z-hF6qs8&>Wk2&7-+i$<+L%eL;$H&LFZ{OapZ?)FPtBOcF61XTA>RYtm;N~(#8z|4wY&(9lETq{Q-`O+vlqf?xHHIk4p zb9h87Dk|$zWs!PbusajB&F_@@5^;V4UVbilX8e!y1D^o&Yoz`|WXIH|IG+J#xs>8> zGyLIlTvXH-f$ zfTw~=DdQ+o4U6o@dbgae%Ju^zVBL)wPrY~R%{-9Po;G}chDQ8}e)!a_oOgAuk3ibD zrKe)=J=`@>cyQ_%L|I) zrI%)QmuHFlnD!n=QC^;hN5Ce|!_j(`QY6$7=HA@8HN!mWL`4No2~??~1VE3RoD$WP z>(B)@X3D663ILLZa_hP++iDh%b|mb+i2nKGTcvz`-Oa2YhX89?Ypu1ey*pq@{3ehb zBw8m)aIMOsFobz{6AFnyAVd)s1|W!STO#6U%`LpU&_r3t;oWLI?QKCl``2gT=s|4jx~^*}0(0-K3{}k`+YE?R#mqWsBa<+a z7Aj6V^jofjYpF6AZ|A z2azgsnGX{ZqdiTL$WafG(2>J%iLg>@x1;)Y=7MIHEb88$fYKRSndgpa%9|?%bZ80l zfb~Wts%)$rlrJ-LM=&h46;Xc9TVLYFOXhxtp_zk^D~O*#_VbpX-~9PMsfZ2=g}csU zN=li4*m&pJNW(b&5CtJREIVRKE#{%8F%o1Q6ZYdzUoNX^`m>DDhIu(jZ_6NFRI6$w zxvt};r=L!EUX1kK5;>=cu++NdK!0X5VcwfJSS!}MclEL?iHClEezw*U-+$4uiq}u) z?w55r@25-(8X}I?NS4|7W2Ur`ndI{s<*zs*^FD+#hVEg0NLW6O6fZl(LIkx)DUwW% zaYy{87E1<&N{A$1dQJeG9h$SA2jBs~yGewAdTJ>LBoKg7%4|McI{<*9d6`t^CgAKiN|WYM}@zFn8={c``h-#$NY_pb+> z3~BA`&cfY~-c>6dzr1bk9){EnHnV<2YrT}DX+$481wC8?E~3g>%Aq}aZ~Zu$_YOF% zJIw{4hJfo**0nOeiJ%l&tJHIK-dcmf zZPuW*EQAzlYO&Fq^&r!Tqgj{ZxE~e~*X_bUn77t$KRzE{w>-1|@|V9<(QVt_-`_9m zmU-i4-GV_*wzOLa-FojGRCnK7oDHqW=KQi<-##vHSCWQ!wEp?q@4x-{{_AhQ-yV0U zyj?%u-oF9z+mFxx^|${xm)tY#cnrd879XI?RQ~g@NY+E9Xk*?O@Knx@FbPRn_)J9> zRC0*Lk5HD$(t>4`CgXvT_FZ)bc#@>-htt%~fmVDZP|6pp8FTYw3aGID8;Pvu@4u5aZVY zJ&(R=snH&fM~2rNY&owPgDU=%SB}K&U*@?=s^Ljc1#Z$UeG)I*+i@Jt4l}ppf3v98 zoRG>DpIxIoW6t0{`g`+Ys!Zf9iMppMQ#Va_5gA&Hc|Y3=j}>G#^qHL&5)cmzeQ!ua zoG=6?CYGm*NV4Y!xrpZlV035FgWZYKsBM&yNi&G-0Y;<_od+@~^GQU@4{HdRFVs>M z!gg-}6f9s6MO6Y-6zUmUW6Z^$f0TKqBw)tEBx2TDixxNc)?;rHu`J87EYNa&Uw?W3 z`24(~{ri93{&6!9s&~~|w{4?p%GGs0AY2x4Gw;nJlm#AUZDe*i2~HsV6wvhE)75re zS3pwrKHP!`Ymg{9(4A6w*;v!TLqpSAMl9TVV8NCJVs?7>2n$O*3Ud}IA-a5=y?1vA zf~xL==pw-7axp|5uwAzix<+sZ-Q4$f+xOf3*7^^T&+k7j z+g3`sUa!~d)lYI^cvu-9c1%!PYt4I0MJ512{r$(MtZTVkNF1%*kK^xu|NDRZ?QdWA z2Son**T4Sd-~KP#_1evDUyu83R|x?OL(c_y#qL0HVorXDGL#5mn1Pu#65YL!If$@ez%2EhVyVh*-M%wow2qp7kQp7DY-P9OG zrj--`diozkL?E4J2F|%Gg}f}wQ^3)OA9u^0TJ)*erC?UPq6#OV5fUO!3W^X=e6{-- z+)CIi;S;6<5@Upns4y-o3mQOYkb2hW5%Ho${(2t7WNQE6RA36Qx&t995fG8WvMh_( zm~0La66S8bi%?|YM0IBTky@+8GdSk`N_b&E?&lrM3m6QUAcDjEMuu`g(UJ|Th?t&) zl5;oPdVw6h_u*z{=-tiia?uD{&nP6SwUnjkbCw5_iZh{b)2CmDWq2Z)!!;r!#*Z84 z5$092Xi3-|CkOv!2Eoo-9muAC5*H()=U@X7GNe#BH8nAu&>bFUV;~R9i0E$T^mzy+ z+mGIQYXBkU?gF7o5SFf+K<}Og zL%~oHtqTPKiG)}31G%SVT07i!#LWS-k;s_eg`6E?z zAOSSSNbsxZ$H%whI6g%sP$){dTrS(T5xCuMtu@<^ucMLl))g$T7ObkvcB$*d9iN}Kulpk?GxY{fOv&Bd z#T}PJ|u6RJD_tPO_R87Th?{p<{2p3d?oj6o_yvW zZe}7*sZs$j1fFffGLtiz|6_@6K1+e~vN*w{6VJ-q?;K;V86ZxC;R*TuD@^7rG03Di zJwrbbWU4(%(Pded{%Gg>DGDhEecvoZ%L0)qg%mSCj;4#gy6$E7_4S7M^!CfY()I^|m^{1@ z*(csIKL0$NUMmcr@ieC)l*Jj0aKZtNaW>mLFi2`C6eknWG6s_PpTRwE%Yj~|81?J< z4SDugGy%g2xQYM)002ouK~x{4cSg@fTm}(v5WPeseU=eH6$xk{B9^lFa0p{~! zy7ibf$E&G`pLDp~E3TbS`MGqdg(awtJK1-hv~VYReEc&M-+@$g>g zg)V5yQbYnR!ivac)yrkQT(`^RlHwM}ad=>@^?JQ7a-E*kxfaGDr>gFe0s!uAtudq& z9PJBdvITSZ?jA6$Wq8&lzf)A!HJ0UqU!1v{K4UL2@oP+oTa; zz3uI=-cmEw%u<9bf1oU@h&Xt;yf2sS(c9nt^B@2Dpa0pc_uiYuwymNyv!CN=pI^7H z&)c!L#pg87NEvW~03Hqt&(KVyQ8k~~^fPZe8&?@-A|jD@5kZK=m>(XQ>#X=QzMU4# zIJ;kr+}A`%0Ca*S0d|N|1j9d!R3Gsz^$-VLBY?UtSp?*`0H6U&!&6ueiw(-K$azuY zBe<_8k-8s4(YAMQt=Taeutk+n44)1fTKW>ElX$XLOK!hpnsOQ?<9UM7XU`!&zsMg` zx$_pEp&bA8@{_?mN}$o5#uV4j5_MUYW7Z&N_(LhRXzv~4+GZ~{PZTY$f90gSZ`+oE zRYcOCFgL)N{-0rsQ-XM$8MA4hpX=1Dv!$U{Vm z$egS26hAuceRE6L6xZYg+5H@jot|7LdXtwS-JqJQV|){VaKS)MNv2$Sn8Q#+EoR*E zfajTjQ>=~2aF{Dlq=-Ouj2bHF;bC5Dt)(V%RJByqq`F_X^>(}Mj|Zt$%iDG9cHAC^ z_jY?c{1)%-fBMs3QU&_q{^&Tz8hb=^MnurN$sk&Q-PybC-P}Mc0dJmyl;$v!lvrw| zRU#HGJp$UR%37+zYXwVq8Zi>}y4LG;+qU)Se)JxJvM$@(d$C1p3N6sX17QdT zODWBnyT-cj_ruJVS}lX!kl38nA3a9_Xf(5AZf7s>^Ye4xcWZqq^}1aE^!+#+bjaJ&7*E?Q>Z~QkYUwCOenM{c(uiZnxY2`~Ud={Oxalt7YBZ zuBy7Mm;Bvry8ui*yVjaAZ70)0Try@0Knd|k`7q{*>mKe)S?8L{$dl1&uJV&OmrB4Q zRCRYwgp}XnCq^zQV1?al9fBLwqpe{cQ!r(--K`ZnyjG*7}h(XnOcEqQRzH>=J;J0(dE^ zF+DBG$zq$z&Yr4Kx&L#xptUoD%cqqeY~S}YzMlaUz|o$Dt+@yjgz3j|ezwQs@p#;> zZ`<43TfTW~&Dz6N!kYKq!nIIlqc6|r?rEjA%e0Y`mwC>VkOpAZBA!)SH_Nd5_Vxzg z>+5UVHUO!25fOz$5jPj#*Vk9b7-C{_55o6--?q($HoBgpM%Q(FJRZ636X?gp-1D%x z-|z0euB*Er$I)7A`_64EwR(iL=I*uD%ev-4m>uwBRt`Px)=pMsPP#73vR$^`##xm( z&{>xjsWAp7O&K07i@TWIvhx@~dab3|1)|mh;n7D$P30*wg9i{QQi}F|oU;yD zteg;Gis1VkhDhT1v48P})O2Pjxs)?%WGFGSS}Uokf0H;VkhpUmi9)h2&M>nUVX6x0 zm+QLpWp4+2yuV#kwcc;tO=`K_?`_|I{P_KPz2=1f-+sF<%M#AV(ITR*OQ}mY>(WVe zL~HHXV~D*+Tv6}$udm15z1OwWWdXx_>vpWAq>7mYJ}%c<>%yw_SQf2|0;+YpT(-JZ z5iUy^;`Z5gBGcZqbt`pg=32|g+qc8~cDr@o5fbj{sjVW*WrJlVFosJLbE9fi6-5r{ zE-3w3x`AX7{U879f2`~J=Rg1Xx8HtK@csS$*I$1P_hnfWETUR9{iA>U{(bq_`Z0VL zV)_~M-akG*PGZ}#EWN*tCWP?c|Mnk$|NGy+fB#-eDeH2^hxhmQ<2Y_VKI^uA`}Xba zdcADhpWoIWKYrZ5ZVa%LjIOx>ioV~kQ~DiY9j3J zt?!7aTB>MsPX==cL@Q)OoXI=KP#!tN z!_Vk{&Ix7t9xqZ?=g*wTK}3K}L`p4MRA~VbUA>3(u-^B_o^C3OF56{EjgT{W9YFp# z0bkG}ynr*o!H~m${W_Ckyd;eOe|}y>z?A5-%yUaA$qj1lIfU7dBdm)HrJOlCUf3Nz zL4O}25+hqY2UC5Dp^pnINV`GKwxfA(9o8pGp2AAg$a`H^!wK`nxuh_Q<)S4I!u$Q+ z9{X~s`Tg^;eYn3)`*>Z~F=W6|P8Fa0RC%!!KiQ=@u#;f8Bhq)KGgQ-I15_zd zBEq{JkDWmY(UD}99D~e}f;^MV(YtpGcZg$U@o<8Uq}cath=#=!OWQNF`}BnhuMSfKPhy97RYCoLn0rBe{{~X{dk@9>+rjsw_peZK8A*9L#6JqLHnHjvpJ28|LzJLFIQhP;YS+`v3U-w&n|LyH96X(|26Uvbh*}M%q-X z%_uyx`zH~RlV#rRp-_s6O4<0P+vU0-WWVpd_w`ba$9^2$?0EEs4kT6D9T6ox{>D)l z19XN+)+uxjjW`)A=r#0@C^HdZyc__}lfZsb={#Sh6i`~*+tDICVUzFQzh~K)Zz#xq z>P8^Etsdfo8__dP$fo zgG=i8&yNmwb4lj_YC)C07P?0dtMT-~IL~AeNviw#6i-k{YYC#cjU}SuNy*^cyQC9_ z2!Rdq&9zMCh33)C_0YSGq{iT@0fCtx?U(k)s!|pWoO_| zxJF`-@$;@q(Nao!&>YtDqD9q(PAHfYI_BOVk1-BYix#bOw5HT*$%90=b#u#8A<4q2 zjtNUi(j+*HGPbaeMKDt#?>kE#uHC~TycSt>ZEe47CE`lA&--n^J=*c8we~{9utwjzP?kd>TnF-JGMb?=8*heUBJK@kTTTit5e)^*X6E#jq=szvLuEDIUkd%%8aKtkU$lx;C{RP`s=U%_{TqfeE*K{mL}#3rD*c~5P%CoI6#T82r@FqltD2+ zZC6PG9=)}pbv%Oggej=6IP*BJe26GaKjvp-leAtQt{_ehrB4bY7;uLnn^FewG{`=K z-jIShnG|PM_ZlrR?3{ss8{AS~i<0MI`klcLutYawh*=F8SC)ON)}yz!@B8DvKkoNb z4N1a6(iMuHK<#*M5fO+uo`CqN{19V&@?S*xKe*gIUw>3RnKv=1OA*I$kWj71(Z0UE z?)Q5F<)tiD*d-YLwk|(De;-~7!&X+3YOS-weqK!Ogrg0cy7QkCd5~Eju@osxLW?0H zn^zG5geq%YN-3mbhx<^8;ZtSB&04pE@L1{)rksLQ^E7&rg|g0iJ^T{A33YSt-X)t_ ziF9`>T6FsRBoXvdo@GILK#Z791#l zKJGJTZ~KwItHU{YP?9ZNpb$EdEK;hlsBVjQOJb0UB)dF2og}M|+cMyHL|(tmA}os3 zT1&0yqYVZioZ-2_!olv=-9n;=HMk1^wBsw#C}rRG){YPoHQe2fBKqxZ+t%{&xbFMj zth;-EeD&iV!fw&RsRAt&fqO)8Z(;6-`EjsyM8L7oYNe{wP0Gr;EF%8){>xUE%evK4 zDnx?$_->D1Yb_8QogNfHHnHB`KK6F(t%)v|>*f0XP3vOjOI<`n*gM;hGK+4AQmS+d zjmU}I$aces5|JAtvPd&Z$bZ4a$QqUH^TdI`}+L6{q5iX{nuZAO}9ucINHGo zQK?#-mb~^e6m~P@+%-)iToDLLet4v_7LgG$Zx1>G%p&enf>a_5jA{7ga#^=+yWJj?6dBJJd^P~%IQD>*BHOyGYb`}aFLxgu z=^0PsJbQfdoHEBukov2C4U!odLTK6@2UR@B(nQ8%&sm|2?}%VJQbj~K23~LBG2KhR z5JXK)_K?}oAKSp)Rm+J0>Y!nyM7s`w^2yP}tjrWM^%#1-V;Cf%pN?Q|6IF^ht$Sx+ z5iOJd&QSv@sF68FghHT2NH^=P^+3@ADGmukcqWen(2NvN5^-X;?!6xqwbw#+Safek zi_Rjo6d-!HWi45`uXX#lY!-eTz4!k0d6?PJ`y-lm?-t4;TAMe4pbOol+ZI`{Q$AE= zDMfXGR4l8OMN8!`AK#WDYh9{Tc!WiYx{}vg3ssK3_i!rm6jarI_kBOM%X+!Iy}i9t z%5n6qj&Ph4TNd8K(S2RBD{=k*#LV#6X%5K!~5g@)!Y8#``6cx z?`?k!mr{8N}2J%HZ#d4&d;&vh@Npe4L3ETTd2K=1u9Ywn!_Eu_H0&3a;& z@9)?5x3@*h@!0S8yB$r+mhIE5fpWzse4lNAvwxT?fFT0SISE4yI;Tnx&l~QOJbEh9 z4Fn^T6OMqL-O)aUVI-izQzd2$5T;=pMk__;ps%L=%ly}b$eMQsN+~h#r-+Pz9HSSS zjL+$o<*t;V6Cq41mCOQ%T+Emb7ff!=*k>wGSyGMJdUH!}yfkT%10yRP&sT)u6cvKe z+?uu4S`bxLq?WakgjtpJV5#oz(Ldg z{@EoVI45IKuxhEL081&QE<%OVM7^vmMb}EvsC2~vjJCF$Ev3{_AQEA8ONn%+?R#rS zkD%72uG_L+G6CqlW44Ge+cSXl-mQCdBXWR8AbGYv8(lA7B>8P@=&?cI5%I%ss#+o( zu`UZqYn`NKLpy$IP{p}^h6u1!6$b?!lUgFB3NRexsleTRJgr)5(b|Zx-rDi_0>Jz2 z2JhQaqN-W+iUj%+`ZF__r((X3SdXI09QrHTyPZf&NT4U-U4}=Plgvitgb+Dq1J8Ix zp@!C4$MTb$Lh}4&LBNw({)D1QrZRhXCjm2v5bAm0Mn3A1U*;3BG+Uqd@R@x&D4dWI zM!f+h6hp|&-7qF{e3>u@IqG<&s#$Z7ZqW^S;Zh(X?|Oa7+yjox!t%@yM?gkc&lQ1e zK2MbE^|*UItnh!Xm6|URvBM%_nVgDPEF`;IJ*QZ6Jpt~nb(wh;8ACo75tp~?aUAW~59@@4iXgh% z<47Js6w(k90Wb@K$n>cbIn$p#siGpH0v?h_T{e)d_c3z_2$@qo6k6}jv<~$)466Vr z+-t2x6+Kr@;%NbT5k!&W=-$m4+jd#k^-^2!>{y_3nBUtG!n&;AFSF$vpt*$P6tC;L ztW~O59IZdP?;=quOF0-Ax*oxFNs}-OjcmxcTk9sU@WYK#YOUM#^8W4HvTVZ$rLO(a z18A*v4{Gaf*#$U`!!2MwWaZ|j%hNsPzn<~sT<#;dcQfzqe&Bd}dxMJ2`Hw_F_o?%2 z2ngk$c5o34b*^bxpFA-s71=_Q;_ge)h;R=P-L`Gfau3@hybqd7Z%4RE<|+&$B>!Bh z>%ucXv)%_p9DpRGfFdlS?rv@D?ff-?T1yyes1uK*6DYNa2+YG=(g9pG(vDOH3h#Il zMs#Y6g$$f0tAtvrBP58@#uOF}BP6&ymL5$eamh^X$J4)+g8S zHK+mb_^%tPGt@@JzCX-cDP>z%ojN2GDWwEaR9fH92+lF`J9LZA>On+K{hhN`>eDhQ zf2^#duP`uiqE=m&Whvu;siHdc-1pX6dTzM8vYcDV0Z2aYuM0LqAJq6Itq3YbDe1GZJHRQeB-iv(6q7^vH&r*^sk)-@f(mICghW zKq&*ne(Y7WWJsjspdBN6>yD5VnwF6lm56S=EH(dCM5Go`4KqtbH8~F14HksZ5QGf% zrW~+xr&&90SGg{wM%ca|?w(TV$Nf?3vMftl8z&Qqq(5#TF6*+sU0k@gZbV&|kL#+c zN`b*z??;0>JZxP`ty*I5YVOfnM~BH$F9ZZ8s3H;?3Ysu|x#y-G8KRl3bzQgX_Wp5w z`&Mgphl8qWe!%SL%`B*ziC2bTw@AWu&WJH_ycyF;p0hjygnuTL=B;;Y?yeWjA*u1} zHa19dPkZkiXU5n;`UfkG=M>)9N5c!0BT@KvT`rf)`}_O1Z{M!#_VsoDpa1iJetv$h zx+p)0s;pR%^_&g#(*o5nJ~&tmQuP0tkZ|&+)y$ouP3QvIz?r>?i@TEp(D%m zm*n_-Ph+ae$IoBRw z1d%8{!yy6@u^y_6mSGQ_4@RWEDo^2sz9McT0~%VPpoppl%&hJ09@19PsP4TVaU5Uv zW8XiI-I{A%*L8t)2H3;N-Gv7O5RXVDbZT1$Hqxz zNo-$5X}o_uT5F}0_1kqV+i@J;?l2AC6|&U2l*JEkZC^xwdH-{*_3P{FcDtFG1QZe; zz1fO>Wqq&ETK3+K){FA8E%(-%TRV;e)@fZlmjU}yOL6r6=)K3fl&S^hY*8iU0z4vU z0YHe|rmaVW06MdS1?glxOv&KD?tt$X>8*n{RstAMi@#v4o z=U@KYyS2`eTZVz?AzGCc;n8|e`AG`(S_*3u=^#tJ)`cEzZ*kvit;Hz_(v%^pWomPN z=MPn-R^yR_E8%EIEOn)YxPH`k6N?C%sOpc)k6EL5I0T|lP=vL&e&p8f8Q=2VO;%Nk-G1D zZ(i1GYmI4*X%xk#65WsetJYc zSu|D+_U7Mz|16~(ZnxX*Xh)9(R1)>pWES3kOR4}V?tR}|r|@I_$jnPjH{O$MK=W(rX|66e#Kd0x7AFCqf5Ay+nkW zA6!h-C7YJk#=@4eMqNs*6f2CucOj|5l38kcHvwYZ+^zXNv%DM$GqWTTTt#!x1S-q6 zmQs56-h2OcPSmFz2jHDew@reVx-Mqs`M61{{`&d5h=^z^Bw&ELLtK_rP&1qgi>N4f zGwY}4ZO*TV9?i~#_|%X9aUsGZEQgSU?q;@kSHGq-bkT)EgoK3~#NK62w3V3rbP}r0i7@K*ID8AX|Kqa1d3$NCHNFK&SY1H zbp8N60W`Pl%0!vl1|Uw}-Px9+SlfQZ_QzC{+7z4sJ+c6W9`O-B~ZjP-}`9 zi*0sXY^a6c?c>j7T~u}7+vD+A78jAXx69p3m65kel5I80znJ{!)FVozorz3m)j!xX zBve7DI|gNABEmUa1bFXF6jG)4{&?K9EC8TIq%gwW96g-HbCOtupa$H)FvjI_d3@b| zeBF<|m9qWvmp}dW&wu^)?c1Z?m%38R*Vor?-@m@TzFKSHCV`G0!lwFf-kB=pQpg23bD!dr3+i<+u|< z(NYd?XXu;tzLIRMWiOUwDbMWNTFXKqkJShdq_CPqr2EC$`LdXUvaf6Y3@PVIetpS{ zB0z+>O^3L-NlRJI${-@viCxI7Wk#p#9E%!+Sz@>d&W|Y~J#u;3>m~Z^5aOR;nGpJ5 zvdt5Ta7;Tj3C1hITvdnV*g)1&nZw*A2Dv4V)0lF13BPc0 z_vFIS-8{luFGX^$WZSmu^_mhQpO1h19Nfpc8J{i#-pkUQq==zm7%{|$$%)UuXx+h_ z@6YDaN&JY2VO9`wLRK?g`?>arq-XT*zV8p3h5bZ=J0yaR5I#t!lNT%tD{w(tPFRbDu#>*M~|Sxhe8zyptaTx+xOO5Yo4xQ z?cbDx9nj38KaLQpr~>g^R4+%7%tp}wPlj;HWFL(JPLj@%soQkj8<9PW)$kVPt?ftK zgRaVmXx5J7P}NdbDWHg9a8a}_q7V&Hhnq`8SiryT--QhufB*6MweNd##C_lQqqY601&E>)(W-^FeF!U3i!d~+5JqdejY=!c zgo^{#eVyH85lte?&@{^6lYPfAw1WUEQ$au=ApxC7cJDoeI$68|K&VFhxe?sat#@m^oBLNr_RK+( zZcj3`)U(YyC?x@uGb8)26Y|eWT2UQ?^y$lpP55pfe@c#c+@l+%_s7B97+BI5?K1_Par}QDqILf2*%(5Q=qlznvNa;k5C{o(3f&O zcVlb)IF93J-7O`+#-CFXOu+y+dhfS=KaSolQ?O{}5PF0N*UP3_tw(EZZ%1(C3fW{l zZ~mB*kUZ2&Ql-p{69E9SHk|swup<8cSW-J&T;jZ&A}?ZyD%h=wQy zn%ex=MS%cYgn^baog|O_*xoMd-zmaMZZ5(%b4XJ*POR7v453m@|skrNo>S&~YoRJ?EwT)a{_u64z%V!`f%$(b+> z+*2^#$eNltFPY8Q#b-o4pY{1gM%jqSQB88p4@aD1Sw00R2S?~Io^LWd=`mtc^pV@n zQLeyAZ$I}*j6F{Z@#IZD2{Wl?kY{I-6SMP^Pv%VIv*mo9m9T2w+@(u^whW1+^x1wrNuH~cv_coxe+%0GGHltf!jH)IH9YE63 zQ(-w?6b0ghN`A7kg7qdT8=L&rZ1njs%y##n7LnlKxibXj>HEwW{Bah#O&(0-yulco zKE-@8eT%32$5hpToM0D#e)eZTN4lN3S%a=3vP=c2)*8bnz{T@%$4n^VWFU~|;SmuE zs%ZL80%LC2T1!$;w@LlP2v0rs6h9AU9izYyyCoWEIWT#db2FVzPu#Yx_kQ$2 zn*fj;+1?umNcCb-gvhZZOSeBp5v@yE*Y*B|Ip2Vk@B=`JT1PbLJ$g5QMb}b_F!dv& zM|XsTAq+m$+&tmUGVHei@7>$9Qj1_lGYM!dN8jTv)bhTqS`{j#l>4K}n9nN_;e&ny z8EaNGdise6(5$Q0w28}kJ`#jPcn;gnoW0aqGr3b7FnCr`;Wmc4u`P?Ls**~;WY{4r z6*h-6x($^yp~%OdKE8eX(|UO~i~Z<3dI3TC?ce|LXvgP|udn;#^Y+;H<`w{ce0}Zr zd)xQ5>b~q%rK(gdm+RZuh9N1YZ)P67Rt-p?WFDNZb}>=njC6A~&clAJ&B27p?P)%S z?~+NJbdiWi2G>c+HH5g^%pyiE5{zhhY7QVLBGI;MP^`E|7la@0Zwmca-S?7;!TU1ZVpf|U4D2_9WKf}5(6*JF` zeI|OZ#l`FOjhHV2;ZXZJ{}WQrAg4W^(e^l^w|nmg!!lNfcVQHf{dh1N=@Jy^1o1iM zmB-stLV{#Djz_@Mby>1U)5mdwC&WM2f-1?JxXUc~4B7Yl-O*y|TX7P`6LUY2ij(P> zD=Z7flsbV65S0XkDk?Quj49(&bg_NNBxH){<|1NRVsKMO#go@;h^3Ugo)c&r<3;BT z$cR$o#hT`V2r+tfabZ+h^PcqHz4txLTF(Z<=;C^0h8t5|FZJOAbbqpCGJXv<2p}S5 zsl}uVMCb{VdXGrzCA7%VgL*C)Cf+I{>v~CLGyu`)lVVisdak9{tY-ih*3ZNzE2pD9 zL?j@Ts;pCvB6FXC(43DxPkpm-mOKwO3sS;U*E-x25?g9rn4=+t)N+C6ZZ`OC`ET zbobW7$aLIMCqi_&Uf+ND^W$jy@1I}$eth0qv#-B@0XTZM9{b*Ye15jpfh4Vlh3^gS zeh#`8=ec#DZEg={#posfB@E86RJo)g(}o>ZHDBoAxHXv6w0SR<7IA3 zM!@^a>H~*W^)sN3R@6|z$jWgr0x0K<>kzoj8a;E|ICE+}zmBpjOGd6qMjj+fU}(!^ zaCvTq7Y(fZV`snd50QO*m61KWJX7o!qpTi2&RTS>iwM$mTp2~B5K1`m#G@z#fiO2` zgh)_C87=07DzXcgH!t-RTSTNAYVSkXy-3FT;nym&ITmwJ)~1wh#uu&6Kq*PhoGpiW zn2&z~D5V%&R8zLex`oIhQkN}ofExbr-ps6ZPufYCTkD<;$-HkiM@V#HPua#429bUC z@yOiM4ps+T7y(R?78DW@MT9~8|2X@%B}tMTNf2cKBC2NY5gB!xneDTC{{NqrhG-v7 z&vaFH3oFmFKEV^RUShAP?+F?b%tS1}4)fk&Var!nR#v zIa>yYM^EbBj~PScVK5j<^>ETx;6BS+&u*$JX3?k+QKtC%gO$wGsubBUM^5zoC~~S) z=Cr$tsX(QGN}r@u6;^B$&JpRoN9t>f1USa{I6iK-Tg0}SvB40SXpy4ZF}eodKklRB z<2cN$mi?!netOwI)mkABLxP1nP2w*} zD5Z)T=`^pUm?EuN0W=>Zzz83SdT8CQs#b~?3rIk;Y%bf!F@E{&?LYqh%eQZD-@bkS z?c2BgF8jW#*~ihoef#$Qaf^gTO~q8Ks0u`p2WP~uU%w|rcDZgZFE6jJ*K57hY9B`v zErJ3}{c9DoV(`;xNJU(ws0tE8rvMIA5tl_=PQw1FXK+TRTB|b#^7k4QBYe2aq?2S< zMowy@tfVlOwWB&*VPA;>IM5BOmn{z%wK;C+nJ`;D zPw3o)N%Gm2!TR$Cj79!Q;x4B~kdlg#?#OWQ;of|7A0xVmf#a+TkI|2#e|&tH2|fG# zv$1BPRy0>#P(Gyf-oC#d4BXr8zy76coHOOQ%<^v3Y8h>L4&SBa$ni0yu2yzPPFVQ7 zjrjy}DDPU3WnwMG3?sUz)KUhAiWD;fC1QAx9x3CZx?8RLR=0~-b)tLx<-h*7;r#30 ze*QoI`+xr*|Ms_|jn>BPcGIu7*VotmdU5x+k6RmkyX?7ZprXah%2teCSsXw=`iL0! z`@OZ>z1@%F<)v+V2_mIK2@e-CU1tjy84L&!gJlTHFeM@arI?Ch)=}euC7uCFL`tn3 zT^JC7scA|T#fAwrD=~$Xk~3Ci?bL-#b3r_7&~oy@n2DL3n=v*NcnnwZexIfBfmoZ$JOtNB{ZfpI8@b>mbne)Ka3O$aax$4WyOG`Ejko4#Ip#VqkJ*{F@1Y@*h-W^BY z6BQ}Fv*$36nG9>KxhJfos`K7v-O~H9mVT-pcp^J`>U6J6|J+ou1lscaH8ffD;hc%j z&&{WN0`%u8LZ64n^U}?qb$1^e>74=>86vHsY};nl7}1Z>`Vm4ag)=1z0=>6!Oibpu zzl(@zy>9#c-j3TnGxXM4U%6#(tyVLueYAi3xBv0!)2BcG`Okm+;~z!jcDtqiV()&x zH?CH&nT-*>-y~AlaLVXq<>tdj?=LSeW_H|rYb|1UZ~eFz*Xw?%+bbm^x=|=YRX%N> znA1qGQZ|w7fsL=@7>Jyx|LGAy$7TB-pB3j?d{vQa=q^R zzW@0ze~cKDs-gfizc1h4&1~PV=Z-jz)_cEPE)kJ-)yHv5&4ye6=gEC8fFO;LUY(xI zH>NHVIR{FF_#tT^S<7BavK4YHRuocfbNAHrC^qM?1UkIISJ7{p92*glx~=%hwTDuM z^#=(4Q95Iy!O3c0r-JPJ&UB=FI>j^jdj$81LSzdx33dX(eQeH;gN6^$;loOEuxq@qKVz>?1_tvY@kvigwzu$ zUU~6jgf;Kl;=RckCZ7NG`OT4|`RA~DL;_x@7~wl*GHR4#QVLP?LWPK z!6%YAz-p51)o?iIS-NKKpmolpr(UaS$rD8v=}>;WQ3|(A-thptwrwLRsRHC40{97P z4O@ohzbgXH!~mTbg5lF_<-!0~otl2J5r5 z%?vdnkgEqk0_@pVipS!X$?4+M{z7U7&4sF^NK*9HvU$Wh5z~4u?C1EyA5AyUXqge_ z+zu;sOE>t(c1tb_$+S-6)ED#f2jfACXIWI3c_2{J9#c+1kH?RJ0O~)$>-oxz_&G$* z%|GRZ=4@1schF?4D_gKylzGc1<4Ycs9VL%s5n&)Tp-%i1lZRx0WI)N{PaaDB z5=;K-e9)OXnU!|CeH?dH?AOa?72Za32$DM!-un^WV6aQQv|}JmUkBj-LjYS76O@RE-q{l&LlrgEvelPQuU}qXUhMVDac}Rp<7i`y`2PO!um9t3|MqWx zyC415j@xmMh-#Y@>0|WXx4n)rT0fwoRm~p5`s>a!-BA}osi` z@66vtkgE6Dbvv_c37pd5>aB4R9g*q$KhW5pU*{#BzwkLfnv;ksj6g6viirdfEZKi^ z8c{?QsfSt8Od|Ud;aP6xOF#{A$x)O&T|j4y7UdX`sHkw#NHT-VH;n%K8+!JvGEq9u z;aITsnV56QqKzepn8-}NC#Fn%LFv7BAA_m_%gB@IG(=$NA|PXsj#^8oGu41571AA& zVQ;t#2}j{7%AOqJpDVcMykYdn@f2HJ`KhgJ)A-Kf+SP%79tw(fG4dGP}QrdugjA_i>Xy|3cx*FeAVRC z@X0SZSDIu4jaf=RCpNFIueK}+u>`xaEu241PMO(k7C}Z35zpczPYYP4HWJCih?=*- zK?&qAi|Lg1%>-qHcd}826xqb&=)WQ>!$44opmOWQ;+6-WIr*H4?XV;;x(0;Drt!R0+5Up$8j7-yQ?DS?Dl>CdVHi` zqZZRl!37~Gwd5!yOr==vFn5>m%qlD|hH1LNoHBJqkO~S|s191XT8qgXhMj9h^B@q4 zs=~+=Vx@~^Z9NAmSN@R6766$tcr(gJKg-k1Mdk!nZ{y5L^Vyl1sEn~DS06r8u_y^L zp4aqguVCs z{f^1(Pl;>%?%2R8BPOI{xguvrI`7FN7ZxDZ?UrF#L@*>c04#iI5@uwYTUxQ=?yV*M znEDBl97Ca*37NUqIgALipd6dA> z*dZu;F*7PtOg{5NclQzKlt?S2oD(H9VG5U^sUqmTv->UH(#v$OTMVbOh`ztSAIA~l zW|ida_MRr0m;Ghm_t)1?*K4lZ>Gjk5$B8LDry%*QYpvUMo&=$a)c$mO^dQ~A5EUy{ zF11|h{!+@uP8^2=|({QULf<1SjZZ6`y8ia~d80x-OBH1BGr zr5GhfTHF}aP>oP4QjpS_6oU$M071GZO`y-y-B8MAD20vz!n=?6+uOa}j5vlLBSjjR z^`*di_+U807zpR1EYZHC~!`VOp~H%|3woxP#{IibBIe^8^Bz&DHnq&#T@QY zR7Ihddr3r+xTPg|>#^X98EV4I8DxG}YCr!yR+ctK?x0LXH2Yj48i542maxQfYj{fh zOJ8L|ewTCv7wC$|l94@s|BTlE*=tsLyA9Yrsu#tAJS=9vohcFpJz;ezP~5t6^Lh&yDZuM zv+93*1Q`)UEz}f72jNm-QWSFkxMj{o62YRT+pfU(@o~mVd4l^N-CD*89QzWWcsjHu zTCGc#;QZ1uBj}tV$jTStDw4wQDpCaLqpPZFC?UWkhE9uDt$8I$2Lf=mhUW9lE_U`I zb>=V5HHT_y6f?^SZ&gKqoVc^~KOwAqUO^H<6Fbk{So3@5Gv#9Y#u8<%wOXCo`uh4g zq*hf`rMSBQB6hOcQyB)@(v!<9H`pAac25k!U?08Djw`%}kDzP8 zrZTIIVg{fyG?-#49?|Z{wr}r8yMMgL@M?OfqLVQpa#b}GPT9^D5u>L@OzJdPESI0p zb>b2L-1~f#Y9fpn!vivj`6;L2?)QGQ*4+EV>-O!U)wWGwW1#ingRS{-?_-Q#$KQYX z<+opc{`G!u5vls?L>Cp)VKO}WI0i8yhD2PpKa^6mNarzncW>z%RaT9Q5Jzz2Hg;dQ zVs;be*o4U2+cz_BS^4Rh2P( z?%9-|;LLd3*GDrPnMeabDj3VtjQ7fs03e~c{DsqZKozPKi7Awn(;z5yBN-gwo}+UU z3Gn6nu`=|0<($pAHBNfovu^N1pC{0cG@3s7Q7eBQ2WFKZ(emxBJn1=OAK&K&6zZ)nc}=1@PblSpo!s zP=aR%I3bxtIWpgKDr{Csr{^ITYNU(lm`$oP>zw(fC)o11OBqV?)=vh=`o0TcNRS9J zn|ug{VOozy%n(u_W>#DcG6~X8`d^udiHIo7Ec>X)i zMdTE<5C}WP@yXy&h?J7K^H`1|(}6l7g%9{V?hgRC57(S{$rYc8hUs$-0!vx$EG%Z4 zX=W*zz{x_+A9;KPNz%jT*zhygC04OD^Nn@dsVajxYHv%4Aw^(F$%8wl`VeMaY`JVQ zl;<_b)J0uW&6%;K3wV){lLJU~t+SMl#I+{UMdf|8c61I`RYvNGiSLuq z5d$q^Sb-|8yrlj67>&^rZ->JLsb~PGLL7krp@VbUazTGTXUErQM|HpUdAS$(*SS-9jo% zwQRbH&XR44tO20%tRW;5TsiN82+nb&-(8ZPuA%D+ZCRe4@cO(Li^OS5Pc5%amTj~= zRaHy0GQ0&Qxo94?67(@Hmuvo>(VQMJ=Cz$g8CEHspPm?{s@83?yk|SNJbDHsbDz6= zt^2tGWStXcHoB7Z-W>pnAF*z_Q#~g(~?k~%8|v)I2kF((~-%;&d45-@>R1-7;9i{Ie(q3;q$Z9wi9!I z%5bwAnXG=E5nMz}hKQ&?a}NU0C$bT2gofpOV(Lvp&-t|o=?m%sd_wRXQB$8kj5vxeLEJ)dd>dz<9^k6OF?#)o7tUD-g=;zO0uH>03}is z4u(057ID&L9ziSvXv$2^q!4#~&X{;kRpnLWi82?>`JQs&e>PI3%Ups%oF{?E z*z#?ZV2^>Agv(O2jm}H?t+z2)FQ0z?`1tpa@MHXR{Tx12Y^xT-RTXDdp@5X+F-(A_ zToN6amlVQ0sK?Y-C~B$|5O=tH;qf>jjUs@I;eFe-*1waq0$;)|?fch*pI=@smwh9B z@T2*?6p>BG80pm2=}Ot^u9_}8Zvezpiv<`Oa}qxYnE3=Ui%uiF^o;S`oi~`Y&S)h+i}Wh5>bQ?8b^kOgr(|EaJN@ox5k9(v>(~9i6}FAoFLm3m=>R~) z%j;$I!`pE?ZU`4attiEd|6*1z+o$Wxm&?nm$mQ1L<2L^7@4u9K8Nv74+uO&-?YQ5q z>*w;neg7|^qr3a)J9*h_z$vY9K%}{hpnHX0L^rkl+pB*_KZKjT*g4KkzP@})#jcDk z?sJH5gQZT+wrw!Y=&3YgjPMlU$d*b@*T290y_oHbBH`|N?gMhWjgR_0zWiz1w$|Fd z@7tH{Vpl6>W~m3-`|2&FBdaYO)=LV9HPYxx8ozC znc2SAh;j6{BISCy7QJwdYd>=S-@Tsy^7#-usXRaCVlzX3%AI8$LCBUKer$EFWilMAVaV)L9lK?ZT%aI+g z<=~z{Up4(d$s8NP;zU>DBrGE_5GGU4zAlGxN{Ila>-nGQg7hIcTI}j!mki$yIE(b@BQzhmx zAw{>X2;tFX5RTUF`9sLq5fnm{XJ-aL3ZJeAY9iSnng??t$bFn5^#fA?l%{zYNGa89 zx+{1e?tX7a2`_8S7AYfK!;f~Tlyx&@cIA`o=|1AYy&)*dB9g<*box{=@e^fk4g2+tW!5;#637DUpym;mngRx7t{%lkJ^rs;Y- z|A(nSlD>C9WD+CpV?^)XdOuvwxnfz*K}F$xkQm+LK75{x~iit2tzFPblZ`ttVn)((f&Z|}Fi z{q>h`-w(0v*Wcb+j~o{3z29%Q?oEYeGD1y)p(RomMku%dPzh*6bf1#(V>IUQojrWv zvjH-eCu&jS6fU0l_4zD9;HQ4hGn%^Q%Si6q6>G*vL~^PDKrQ?A625>7RgJ*yb{rqa zwrwBx+sn&k-*dziR<=NhXr@DFz6JWSr=9%3GqUyG1wx9YEgwW}QY1;FerROFlS)!; z$})qVSPV$T9JgLc64undroFK0=`NSKjP!BOZ@K^(2awDzvm%nj0V%ers!GY88Y4&) z{P-QtaQu&deh%^ff8KMZrH_9GIa{4drZf*z16S0t#U_AzZ|&}IVTun5CB>K-aIO>^ zlj_as@%orO(Y3^dl;b$^-6R-kwP^rM3zl|OO0Q)tZmL$~(O`^)iA#isM^8na9Dx#H z@7-!GW{N7JwWtW(Bb##BBhOQ(jgc$+yg3n%PF*P_!O$53;3C;?OwV_X80j(-Q}6`H zo|F(^#BlE%KD^gjO}E)f2F=uL=`5Z}Dd~;~H&wvWpG?v>9twKReAeLZDcvPPG26CX zRV?LN7-K|lBO4Ou-r&URumJIMwLCH}ST^N@WUNuI!AP|%Q7Vvve6hOdnA;mNM^VH? zIXq;#Ru|E#wppom>g4^f8nM!@6kblUlb8aC;d1)wthow74##gri%#ku11aBM%?O4< zNszMxFCsnqXsh?zhkH*7vWJs_sljcMO-8zi=HS>E-D60!-bd1P&I;q?MD@zZP3 zjRJj|)R$6vJep+WEEq~DSef4#Bc~DO#V|GQMZ~r$09qTzahRdjDo}U7AGaRf-oC!w z-nMOfy$YtSs0!1qW13pWNM;Xkj+5BTHNiO>w<0PEl~QWmQ?c1n1v78p52ViLV153R zKf*8BEP8&)>S#Ux<>S_Dk*7@Ycb#eMNtG|3YlLX>0EY3cbSyc z(e96j@ksAe&wVo6&Tl3n)iwa(JmNT?Q`X&}3QQmnPr{#S)&nDY3%V-k*;{rIk)m6v zT8nB?Nq<|m=xy}V3agC)p!dFA_L-k^QZ4hEo-u@MC(c9?rmCeRikE~#xQF}n|07%> z5e^U!`9tdT3QZ0;SMy|^wdcrPZ_XkD81o1L0}O1}nVSoo267~rATTn9d(MW8;o%U> z@#wROA4{sYl#)=&d8n+zKA$0zckU!20gNQ6N%|5HF=Dv;%l4929r`egRnXn5St+xl zxRttXrPg|HGpu02M5>y)9kl>BLC3z-MU>L#Xoj7~%#L<5U(_Ku(rMtUtn+$K$ z)j(As?mmvOZCeC;8@{+4wWx?4qn)whIdqUboA0`YPD&|##JQd@OK?*ySr7L%dK+d{ zx5QnYg=37-*nNz<_YwC#Cc9TtfM}~Wr`FY=`naBWnmj*w2UwG-xsQ?=QA{iYOGFDs zbdy3Ak}6EmixZbVuz@wcxEn)ScmZ7V)o7-)s|4`FZXiz}g@$C%4XW`7} z$bSC$+_C3nrR3J{-u90_a{m2EoRb_p;UsOTruw|7L9~!8qMDr86l=Adc)^eC48g3M zXVgfksq?IxA(E18FctF&=FZDdoRYtuiYHcbwVm-tlE6?(Q7fhh2^EC<@R0_O0A5~R zOthwpo){@U7~cDMh~JKglKA6l#5Z4h@wEVm-e*C}sUb|Ha8+H|L9T60(8k9!o63h~ zX8XREs!F7piK*nQ3OUz_`;x*?H7nJUuC55GnreE{scAWNSJw4gExSsAgc6B0(WgZu zkX4~LdOT5;{t=^l3g`gfy~WC-&CFnVEHxjwNLC`?2p7hLVUT2=m0<{zo;H;#vMwI( z(}QUdy<~n+BB99#=jyL4MTAqm+2*FBG6dkMOT2mps#@M|NADd$lZj}F78B3dm+8a= z)!d+raSJ4?QYb-m&?rP;61B&Z4^b?dH|*Zq2V`SerW_xIb!ufP7g@#?5T1{pT^93>A_^#IX*bxi^{vIMMj(g3q-xP>wa`0oMpZ86)sAk_T&y3T3S}01<5i&-fh(!kXrya=W5dRD# zJpZTP@nn8i!kioXJZ=7ZP7YG$g(-EMAn4myRAevwfGG!JbrbQV* zh}MO^0UB!-Cc@1`Y*RA@+-E=SynX;{B7iPN z6Y&(Ko}TRlJ-SD5^xkE$RL}L9M`mK3W@nFg4wCFe5%XMwXpkbF)Oa8@DKohdp*VdU zk?s>_lUXoZ@g#F}tB9IrTcStL>EJcPIy1j416u14Tul0C@3&*S?0>lYQGy&!Qi2K= zQBk?<8-UBcT`re=B1HOyZGZf)sMrM$d;{^?JDe);ryM2z6KxA)(^e*5K@ z-#sQ-~Q!a{`mTZ&uov7ZoDbSeC4L@9eRUF+y;ni*pn+%~GK` z8<$%$qN|qI*K4sY4;T^QB7UQ)dH9GiMhrTQBe(3m9bRisv#N3VNG!&S?ly~RSNnQa7$)}dhR28aO&sv?+j1EdnG$3^om(0^~9z1F=n?kh_ zSt`UFj-N0NArTxt$h&vZNuPjRu3LD7yDH4KQpybOHi=n@-~*BK2AmUKgD@;FMUt{< z_!P@iwY-y~wSCW^IVrt~K}AHp+Iik{*08G^hKV>muamuAbK*COI-a0Y8TmmF|;ebwb>>8ZfOP(F6uxtPgR_2ctGeM(tMMu(ho%A6e-`WQo>0yXZg zQUJvhEbiWWKS-)!gi|6J7!hpaxE;5_2;}rRTr$1~AzC;0S%JmykJ}rR)iiY-XUivO zqT5zpuP?3Nw_3MaBgUYEEY&XCUf4)@3ztQT46o+@Y_r0BGbJR`oH zJ{VkQuwZS>bFd7vgq!u2_1@bt%kYw6!O8#o>T&mEXh*JLj#LMVrIE^&u#gf zsz1kWoCN!od!Ek76WK&&?lY1`fH8u89CzIACyXD!Y-=& zzHi&se9a#chA`-F8cMrUShp9oI|CUT133OT&UK2hu}|N{@~zW|N2+PKFxJSL}cG zUCTbk@b1DWn)wigX!qlKy(Z__eVB;}hL8KWuuFiZqlWZ(WyFBiP zNY>wWzu!mqT6Z%|;%qQDrW=$b74mlxeC1!ihhNTO;W_a+-}A&Xi4`I7RRZYT0hQf;008KP7HT(pI@R*k;SCz4@ie`^Ve1ZI{a>g_eBTqVD@V7)m|yA7ls4=)WIaaAF{pTG>@Ws^9%+gs4u(wbJw;+3 z>A9j(&H<0aAP50WfNYsF#$Z-UqA5GP5{^lnH#6!~kj^<+Rb7;#Dd-BIF55^|-OF^Q z?@RFdtbFqe?)$!N7c7or5~zGcI6Hc>e#aR1drM&x01_=nKmlYCGvS3IVrq3$g<6Ri zeZ>9f+ccgois^AKVx~6LiU~Sh0nv0G5-CMZQ^ZA*(IkTS^zxd6-S>~%@FpVrzQ0_r z{o{SxwwKSJKK=CN)8{Yy^+oC?R&2kF-`@WExBvM2&%eIkk3NP2S_;`?CLd80!f@{l z-%C|vIS_e1qPM+!cF{;c{~ifW~n3gCGAp6MVd?o53>J^@WDX)c*L8-5U^!UPE) z42$zi$t-ps6BrPtHeYHbAqWP21cRPFvp*PZ|DFnYM)u#KE6?wFzN|C2pPGfgKiE^A zeYWs|5h6-qCUK>deXl0^(TJ$e93%%6Ehr+9t_ukc2nS?LOgL5~nwib6U0Cx;xtn`;kAQEsnHFv$ zB0(O*?|mG@>&uq+T?l5g;Dl~NB1R{mqNNryF^d-755y4GH6+nJqMGd2>%Q;q{o})1 zYc0s(UoJ0~*VmuE{NeMLFJ-%U;Qiaj(Z@$?|MBy$zx?|3<7mU9d1AAuqMZqR<+zuO z;of`r5vmkZ2t>+R)0!g34&>yhOx01HN)0EEPDmT0t-u7KByP@Ny=)dzO?uH!5#|;%Gt`ZwE^YpqSeUA!vY?>Wx|d3F zc(X!DfvG1a%lx$`T9S2D%6*#|<9VEgK9;|hmJIi&bt7g6wdCTAbPJiHX_2gXto-MZ zbBhF-d~X#sDJE(rqXnjP*#MP6L8!`dW&?oo9H7q=4?ljub3lLEv3RN=e&3RP{5T9) zU???UA|XqNM*))E+wJy#e}B8*@3r+!Yg9~xoCJe|#iqtt*e4w550AGn>J6RgTc85}u0azYR@xKjoBbk9O6g6AUE#LZQUq+uwM zCOQL`YEd;)d3vl&jJ``ZQ4!Ce6YRrBN@tklgj@8UqhzervSSh)5s4fp9?b}^dnI}9 z$}=;lwc@k{d2k}BD68TWARVEoqE!J=p>ob;q@AbM%Wh_p=VD^N>8&wm7pHRZ+$Wqw zOe|zZM9$gN^OVKPs_ysufpN-lFrm{(9=Z&V`*BZk#9R+*W<^TXeJg6_uDyd2Olt_! zD%O46?)Q(6d+VMwu!v|OCL$qDSBe8#OIEpwJ_@N?ik1&=cmg@IJ|ldLaU4gIrq*{P z!2Q!t+x2Su>!985{q6ns{r&CzIKF;=zqjtr>&xqfl#lzHzqi&#qxbHkZClxD-HHxl z7e78eCXOYS|J9fBsYk|6_BkWAVesHgmqs+_ayB1>G&Aj!@_NRR`Q zNzS2b9#|J&lWSP|{rx>ZHydXAzNa$JE?X_zwr%88st~2B?E6cePyj^il#0n?z~`)% zq+C)&Jf^aq+EgSlsCa}0fhYkF#6SdMa4OPM0aKIlh``04M|UQG15wx#4~8=(iL-gD zB386$3KK${;eqA@f=u2$x&PUeQ-Enm&eX;ekqBO|S2g5FiqDBq%Z3cWaE$bYg|@nDR_Hcu7@JQCdNAZ&IEzIQns5wnmx6qL^cz5*#2R z++zS1jF?;&6U}ZdM|kdV5do3vpAdcPR8nC=p;~;Ad?R35bt@$I*O!_33?J?Q z^2k!NxvCdpE1@7yAH>YlX8}4VE~dz(yWj8k#8}IAm129TY6dM1HXrvk0JJvlM?OcC zQlu0yD}`n-6EP5wrPeSQ1xjc;#`kaU@AsRk*8L@<<*Mb>aGz2EOg`#9Rsdv~eZ%Q4>D zxQ&)NH_e@9^*FwHKNthHVSs8FjowN{wd}elXqMV)$ibqt2@ruQRCLv~Ix*&MssQavC=2F$6~r_ zA4zSTQ7dJ6MMda9I6H=bxjWo!&JhbBR&)&;n%V7k%ca@9wc$q}<$jb>zQ`vM8I+S( zq++$$Wq)b?9)UR(M?LqHl-fO5+XvWS*FhtIebbedL&nP*1vO#1u9Rl zOr9aX%gvt?@aMneq#`|l8eKAGPueU$upuYOgELe;&lF{?)cW!E_V)Gb*Zchgz+U!J zwxU&-TAi7f*Liw_9*%&jI)#oRCTgmdRu~RJL}me!0&vA zsacl}mj>7|R8_}dYj;oUJ`eoNuK)>=4|qN?e(qa6|+O~Sg5G5Vyh35L`C_&C}zQf#1< zZOC2ZdpFe-z z_x=05M8ros-oAf)e}B8*?+y{wdu#8v+r4#?BhVJYuvaNs5mpKcDBi&i_UmOgGZho6 zC#5(h+?tPG4YTWMFwDgHOb2y^m~>do+9Gnqj^vE6v#!iYl0KT*lek!-o$sRZ}RKQUL%4$e@^na8Qeys7g^7iD0;MBo+v2 zb|t4W=e*LKngIl`n6j53BeCoR++!Wu=zRg`39x$7L#C&Y*tF=ClAP>_BP08CTdn0R z@gky!pC}&5?juPiv3l`7F*sj!sGRP%d2h{5$LfqWMiLzE6p3P{v!G6rJ46`~K7{Og z7Ps&OE@L*pJ+XKZi5O-zqnQhJ_OeFye3m5$QW&zn_k*J6c~mIaJlQS}(hmWWf#~++NM=%Y|g` z{r2{L$4_JU`}^_B_wQf7e!blf5h+{!?c1BXi)q;|MX@T*u}_hET45;yAxuC^iwhMJ zAu=_nfpl&nfJ7Ta2nk(5im6QC5i`&;u~Jc}XgTG>df)e5L{hzCE(nES>c-5X3dnRv zOt05#@i|VMQ$1YB!HP&JQfzaQjgnd}skO4gXl!LCxy~UZ|AO>dvU{H}2LQwvkons) z70u%-i|CmfC4D>FFkNO<+|M6eXn03B?9_b1| z$?5TtSq#^Q7M(&#t=(H|2|{vkAVwb;4JJ~H&6sjF{$NA~rfe!=HaY3eKKcm9q|+!Y z$6gei_UDhoNn#p$gkLVNW|Db%?>#uwsugjHX|Xz|x)n(@%?TDsXhb=M4+u;OUh3gJ zop$GwJ8x7{?|qFe%)Sl$DFDfd9-a?ERTsw;=?)taG2q@4X8GOuTFcyE$&oTl{B*A+ zJCI|H{F`hhAIGGMnklDZABl1thbT>R2xtJ4_S%NKdul4=snh!;tHf#|B@mI}wU2&& z-?_Z=OC@j8BNO(us(AA$(L9-#pflh&T8q>|FwfuC$I*LdM=~+YY`ZHB(cy1o55N_r26DxpXihKu|GR zNjZMw9#nzY(uov?5AyZ80~1pkW4L#bJ=2S%k(gQ9UbogF;+MgQIF6&Y;XyMibuat& zMeN>=cFfh-#!MiNkDDq=%_(pZaUiT{?=5HSRf`BqDFTB^6cbK88LTxPPZ+GE>T+_P zi1dvDkR%gK&!ZVJ^Cb!F10vEnyCWn6$S(z#Km6ej-A9s?((yYV*-5Tl;ofJ4D$Jq$ zq(kTOi-=O|I@gCpVo6Sq0TzK6%uMoVM|cd?)qr_o`pP+h_4LK_wUui~rU(XejiNp{ zgxc@DMTCJxg)|BwGOfMVebD)$P1R5OnB(yYA#Q@+d-v!^@1v2aZ%E8Qp3)jh8F`2O zX%4Bpyk5_fVeUE;wVY$j2k9cBHh~4zN8*!No2o)f($AL&qmT$ys5X44lrnR-1SqP> zM%+TQ1*_rHm!Cd;ec@kw@c0;YNWVxiGvtJ1Rl8L8;TQon@izMR_<)+e=(cT~!$dA3 zx|K38?jD{;-5{qbg_)U%kA@{kQOoY`9eoh(-Vk#{X|GFpF7$#+WyPJxaUl4*T>-uw9YaCi155t^~SwekMmskF_iTDcHqL`$7< zHB^(7A@kv9^uPO2itbx#VR~yGV?;#JR3kQcIfDoUyVXe6CGx2g2q?o0bof9xNR2R6 z5!U`I<4v~usT3&|!5$t;y=(~rdSp%@g!bUg1t`<)%STY5#WqpGOnSe+y?u>{eVe2I zXgy~`1lIk2|Cp!5=m8bglD@EQnVIbR;v*y7Tx@0*Dj&CTxo*X*wf6q@A<4fIbJeEW z#t`?`M@@*u>7-ML}(QiITvex?g zdR4LR_U-NaZ~Crg zcKiNz97l@WO78}kS-XVQYTG8NBcgXCeSh0NK{Y73OjKD4O0hlOhmSP*1GJQ_*j7p{ zW$W$@&7INbh%(7PfBr8$dhZ=h+cqg;W$)JpiI5mF6>9TX7m-1*Fd_!K1W_Z^V69X$ zZpGZ;jlFvpy0Y${+`Ui35{QVXtFUt9C-4X8XxZUU+P#&#^ z+-t3{2=cal6x%l28P;3(rlNJL*QzDxf)JI|Gx3}IxP3K^h~eFXA);1GH7Q~R={xTd zKv0y7do<yJpY)9KPTz()CX8`|M7aHTL17eGZ84J&w>z|3OHthP>GaqOVaVa z@8dqyr1z}aTn)~$PV5Lw|E8jv1n4u*T~NkE-=0p|AIk7^JU1`-JQ}Je8-8Ao zI8Gaf97C4J#fwO5O@vfY&4floD9VVD*Pnco>?q|Zlb8yOF`dU!q^-54s;VU&7$PF! z7a-N_##uOvND+HqE|;wx;T~R9dqjNJ(k&-oqdOxafw19}FltvRls2<7z{5`pV?>-H z#gL7;VAdwN_0L z&}3XhhM~vT55O`wnyHy1CA}hr9!e=$v}1uYxrWYLe+HC!kgRUWnW2v{kkIGD0bEq| zloiO%MXD)ew}sj6mN>=e0U4v|v^6-fnqatlSD^vP((-gL4_diMt~3fDKADvdA%T;B zCX4WbcqZk3MKhL!!=Qb>*KxdRZkCU^oIQMK`|4bL$ zC#TjF&~RjX8%&oBEmqRvF@>&D=z|icZ_NCf&**L-hkZooe89pvMn9v~G&C0x(M2F3 zX?n>cBu|lZJZ&sui2S$!NRK2KOucB$c;!x+id!s_c)gR^QJeHGOt4-^6V>iLznz)X zx?f*jF8lTR={k{Br5b`9(Yp_;>Op4+y?dCt!2py2(d3dTJ(PNqms1}t;NipFhqvJa zmif1nX?mx(HQRRUsO1A>0Sa z%XI?)8VO%ra0(_{AP`GoT&tQ`fw_wbyRi3^0(a(afGJAa$z!I;5gZ;8K6AGa)!+a& zGleu7(Lk?tDMROw_bVE&iwTJ`_98dM>~?H zCnE3|^ldMB#&6qJRY<0p=M>+^NX~!2BZ(MLLztN`k5UVWif-G((^6K8@?2?8huGX= zqj?f1+ym$%5gQ+7RVpuAg>Ff&3P0{gyx-rqZOfyJWZidF1)VWct}(030^~mM;Z%x< z&IDR=DAlZ-+ho?b*{7r>UyE@X0|*1P7A)PahyXMn*{;bTPUfkaX+R~wSq07`bb*A) zP5v=PL}2ZR=WsOlg>EEs?=&AZ3$+z}xN`7_h?FF*wl1Ytau1q?!Qi!@+VXMxnPLb0h~wRrSV@Pz__x zsY4+FEmeUSF@`tqE$w~@w~&Y+hN!4vdQMCsJW<#Z-YH4#_<=2RuC3?Deg^mFmG>Ot z=j{%j!TlM?e|}|v5@@C;Q$g)a7Um~U`{~viiork$7fBgz2S?0yL8go1Kv3hsGXBSF zJ0~+UmOhRpd$2Xm1r$Fj(k0(K|K2GQi$bgOpv-Nb<|%Wro93uwj?<~AEQv3G5=`0H zY~`m^X^!e-E}6hoQ45_D0amMfI!(@5yP~$0I@Iaj$KkC5NOF8Brn>7gKDRu{7$o|_5&)YH(_q?A$wG(>34QnHlIiYm@NygPlCXoH*%4OG#hMK%gO2sN;Tf`PJ7 ztau_#KfAY|b#VUI`#h_XljO+yCgR>d++f+1bN4p;!ynw?;9Jkw4!w*CZ(oL!r zIiLqnGd`iCFq@i-_7@NN8;I z9`I1A3X7IQ^Q#AwkE2gKlQ&9tTNbUgeU+#VG|2QPEP2WXSwRXK+7JDOF9~uE)sF8i!F4EtU=m znMA8jxSzx(~q`@FT5tll%SCp9JSX`Y>Vvk(-;x$7cA78_TDrcgt* zJTz&GB{rEh*uaVJK(%NwGomCcqiJ{mg*rq)8J!~OrJfX}6s+aF-BTFEY9-mn(R%B> zYfRlcgK`?zscPxX#~3juU()XoBI1(aWBNacP+-uwwGWa`IpKorpwdn+0jjD&sG&?` za*i=8#h{=>u(jLy?6?4#yvZWd()N4`oWtu0fQd44*4Gr&3B<=!6RxT;uokUOxjpFU9&s|5RCU6J zGY=CoBoGKVoTQof;};9VDIgzmk68V%bNeLGey#I)$Bz49W>U<~VV|7Wuz6+=hAcu_ zMC3Zl7?i&{cjXC3YmOSMb5})@`j=lKKU`JnR!S+gY{g7qA<^P+4?$tN%1eX*LM>9X zI6TL_9FPzSvOd0UpXS<&1V7Uw$6LR<`xxE5IbDQOOp5B76Vuas$7lPxh>96WjuD`W z6)UOu^a$?fLH(b{z4j-xu!DQ z#+VJ#q86APztj9&EVn7HXaAvtlA)wHs zyO|kGV4@^s*0^F3adcOA03JO^tcF$Qc(a*Eipb@1x!;?6FeJoM^T*7Nm09z#YXiV> zbYLn&&o^Yv@F37BMVZb3gldjdX<&#LB?~$9+4x@Ou)(j72ytd*J7n?B;DI{XnV$~&i#8{ z)%-&k8MB>`!pH$CQ&lZ7&*wTIbcm!gX{z*iDX?X%ECh0l0Rto>m5MUw zGRss`2+4s+VK}n_17NVh$oT|?DQvJmg8NyE$s%>0LHzl3KJPP^=d7U;*Z-%%eeHZE zs}>@dY>*ZnQgVtkRUVb37D41dGg8C_;fzp4m8Aj9@3M6(oQ<&wNGE;lST$4?zeXlQG<%WeY6|hGKu!%9=+REMa;byvCF=F9Cr~Y!zGGt z+cwdmPcPSeUd{RK*OlJuy5RYtfSspYoTPFma6J>H=m}%6J0h{)qw_Zx{M`o5HWmgrQG|GD&NPw z|MuH&x7+QqzqBz_n3Rx+5JpkmN_p8XTPf8F03TdMqMF!QUa3wD4nS&)=Vj;7otyAf z%EnSfKVe5MqRVr?K}5H0qsP%ZB1|;h4#yZ`DrQ=YCEZI!Ajjw@7p;_H5G_^~<4u54 zN>1|RiE(k2Hv?MM^UH;(q@yGrIKF5Zp=`WV;{N{ZBy-r^g_ilIp?r@4x(#3vRny_I(%O zaU5@NAJ_e5Wyf@i7%(`5_oLaiLwOv%kAoC_k9;4BDSr*Uat|s zoC%Z=8l%T>caQEQ9aNPJ!5+iCZ~JxRjT2QZ8#J8mBZmyISF7JP%3%nCli3R~f2$y-}mPf zu(g(*kB^TJtT5pb+jge2b4GL;kzX!ai)Rx`MN~p!aQJb|0=iVG8|i5v(dPh~sOpx{ z(mjUX{FaLWBCX$BKkn_;d#}|zL<7a9lj88+d%rE^2Jc^*Lmp! zAwUGC*)Wzk$-`;d)|F3aHCe2&dPiNyV})h7HN}JL8X_V#vC`}zDiIno{JCihmB6&0 zNma5eyL-YZ3mF&;L^z{NNuvj4JC~Ya8kf#oav?CLSjWTUCgmgV=ptfjs?a6XBc7^5 zDF{Bf5XnXvtvd$*MYsf92E!-PlKo*7VdU(Por*=BWQeP3DWsNdyF^U6{Lupjrv$Y3 z5fNi4xT!5qo`7f(Bg`fHwID-PYEX_g+US@%4K&FOVp25S z3^E-gJOYD8LM&$EXN*y6c{F}j`*MCwc?!N+djcrierDEK9ch_l9xM);C^QFI`;oE- zD9LS3in^Ta6eyXG8&PsPR+A6F(Fj6 z%-^54(A{lub?D)mJhb;*X=*4|ODS5bs@{9I2+NV8=m09iIbLiBKxdLSvxTf8b$f{k z=&mDh1RP^x5)2N~wa6~DR&yUP6~TG_OulxGHmB)Aa%en8@0J4@GyOp zb8G?NOvx!}dZE(KW)2XZ$VVukrNrVD0P@P{a@I@c9)Z%g>h z{O-M(S=qN$0ekww`5G3!PFGWgZw!bo$R9#b4s=O(5AIkE(Msu z;DY{Q5?ep!?+)!WQ}0ff%lB7se_YFZNkA`BEp&@`BAOAQH!t7t6~(~mWNn7`R- zo#|+9E@Tp=r%p}^eT&dcOR+veU@EFauJm-h$ypx}k)ov(rKD4ou1@&;^-BxqxitKz z!TkiMo|Ex)yC2A=u9o9-l;;Gp1(-12qC93Jh2TTWskL^$wNy(irP4bX(T9(LF$9bv zq&kDnh}67AS_pw0F^d2aj7R$|ONBX6vOE|;@acQaT+b|9oxl6m`OF|ABMzsj(_DsT zhU8bxJ;XEK$dW1Qy~D#q#q4=It>`SOItM-7b4}&Sj)>l)>g3ULH3PHNiKJTQk(1LF z5(=9LrKeus>>}1wZA$_-BIxejBQ;v{sVOBjUDQk?NO5W^8N*49@S$RwIs0Jf&;?#w zFqdnJ3khX-U@+P-#C#C<-rNH!?ml9eUL&3&>?Ee;0#(fpY8tNPKFV2=xpYHR6)M7F zsdzA-cro3IZCgz-$&>z=<<1#ZpYMN)p9GhzBc~g6me09@a(y1h!QoQtwr$tfZPmX) z2@(-L7z7nUOZQm_0wP)vI+^%ct36*ytj0s0K-t-YxQLsEnC9s-nak$@Y*^kvfq=GO zYeaO{Aw$AM!YQ-ws|8vp*>IPnri>@O`*TPC?v)>O?gdPUH2*cHksdQiA~O@uT)%9I zi5tKX-TNHKFtbSUmsO!kW=N|pm+WHXr>8aOiR3JuG5`z*5G(f&nFYsKkuDWbWVI5D zKIU5FkR=dy*>+XcQdNp9e1$bDP;S2m)#$2v{+aIhdNWmNjCO5)tD5` zm#Wm>d*%oyB78P~`Lv5p9IePqG(Z{T3F9Pg1gJ=IU2}(IHWxziQ7h=82ab|$G4 zrYYv1Z3b1XRuYSwg2KEDXa-f;Hrw(LB}SiwL_$baKtv%^eTD24Izv;HOk~q7)65)H zU9udMIER!GMS498z*8i zVj+|4eA?`tX%p9K*QE<$coY*KA0OvCrPyv(YgMziw>Rcc>6v#H6Oo)Pwo_t+P{o>1 znV{Pwis>|wpT=KeA~pg@l(7dGFr+UV#3==1++?HMf@MeS2Ej=Z!6EkV+v~6Be75qSjnw>b)Sj7|cu;p*cd5k85?$=@EdW=S+By z9Nd5Ki!*N?40D>Epek#vS)B#PV2t31iTvbDeq$dD6%E7? zM?3P4nN3VtK>*d%L`}t{r3BF~L9xI7@ zUfU5-ENO{L)#TKEDH?$>M)buRP!*BZ?uoT!LaFER%77NLN+Mj{nOf%OS8eI7-!`MNLJesOFdi7^Kocx=kr! zRSLXEPvf@8W_TK*T(_Nh&@O)S%yI9JO!1!v_vbS9{HLtf?*ip>>Mk=Gn85Ui0M^ur ztRhNPikK*v!q^j&1hp_}+Nz>~DN#STx}Yjh6;n_~aQN8EOx43vgJ^QQP7n~MUhqHE zpdgV>m;p0Ut5M=dPPd z%n%W#smPzRo;0I=7uj>Y z9cxxjmMDKhaKWLX#L{e;N?RDqd`2OqY{?^_2cN)MKE1+|{6;EE6<~G8_F+|T&_&wk|GGc)L`s%lEa%53&2y; z*e%ioHP31fA5_)L<$_fcsHzj;bk5E5v}opz*JXi}_{@2QT-OOa(gCi}p9HU11Esar z-W&iCdCbVtIRcLEWsZ4ue`{p?#=lbc~?#j2@I5 zIl@OI_P|n#LHls`=rO<`h-?vxY}bvX(4E9^$O08_uJ85xg#Bbl3R@_Gx+(Z zpGa|z(L%&Tbl2;p$k%?qTrR4(-EQrEFv2+kqeW=m2a+;aNYa7* z?~GAumz{uu5(TIiVI1C$|WYks9~^F*}QEqaf`-KR3*2pfZj-5`uR?u~J1K{2Ta zyUdxjBC6^pHuHC6(kEtRvrSdq(IZT9zEMIIQd|%M3f#SfZBlkCCTawG^oUX7>+zAC zSgIb;3$^S@j|x=L;7BimT$5#&df5TA`=P3(Y-+sSZcYtKK#CQw`!Gw``g4_p+UTLI zTiqmh^yBCOH%qYV@-T2x1*dr9(n!t97~>+SaKLk(tDHbO+rRUNR+FA`!j4JE)B z5@KeAOBFy`yN})nuy6b2av8^q<30&om7)?#*CMmmK@caFKZGTR(EL21& zCRCwBXqW6_D+0K~KRh`6lMEA6YV!F=RzQ|O6WsMdT6K3<-HH{ekra<$FQ)Z+6_F@m zwk@S56)jM`RRSA=4)kQ1f>KKiIz$2*EUKC;P?O%PYRW}9)SaDlfFfR0Yqex69^-Z# zNALHD&u3C7lu~C!^8WswV14&S()Jyxvpuf0)`;Z^nC%#;&9h zlmeIz6)8JJju=O8?oJW2@;XjfM#@Po;-V;FVH8eF;sg_!6ZR%&@IR9QmiW< zIVY)T1k^oyX_HXPRob`K6Q!Ze3=}HcwjIZTwRfm9g*3C#pj)lAKvpkXQMOI0icC$2 zxy81sCY#M6yH#7vDU;!IKAfi*Z@z&~s)ykbkYXYlBYVCCWAqOC=)nlpSZ_yw7;rcx zY^#(I6>5sHaDK!i%YOXfT)pR?`9woH{~Bi}42gG_{HC1KEe%PFm^QN$i!fD%B;g`~ z!sor3mk<%BT-A!*mZk5>c3tgUf4;&wJ)@da=w|_!ziNz8?2=`92F6lK%IPGxoy!LA zHn(5%Qdj*gvIkR3{OUEO&dOuKKfY~SRhvd=y=QP2n{)Q=-Ac)UUWk-pY9QS? zBm3+h;6Zm2kyK{XT5R?Z;Gu{Z!v}%_ok3G6Qq*exyQl@kgh^-2)tEh)q}9=wwqeO& z$V9``RQrhF5D`)wLGn2IaU8X7lWZzhR8+-55o`U!N-!++e$!(H?jSu%x$vZ5^%0(uS0;i@ zg;qvqnvjaKP&@_@Q~REng9Mqzoh>mSASs|j7i1?Jlj@eMT^}YAIUv6x!buqr%vHzC z943x*hDZQDB8H#gdv(;+Qil2rOp9NP$%>q6Ei2EEghv<5?n_cM5~G;EV=@RL18Zvs zC+0Ix&57igpH6t0)6V?Kuo9gRNy~T&7zp@OSa5L?N@YY!&ShIHl{ib$Qc6h6U^Va3 zVzbRCdJ;6|W$Os1Kx;9x!HD4$DT&&6q`ICIt71Jy41fFW?GfB_SRnVmJajW2Vfs8w z);XE+{?SuiK^s_EHl7aK=c8$EF2Q7qfdOc-l!;jRj?8!E`Zh%b9IZ8FsOSh*P}Nd3 zZoYpW%R(d?CEi-DEy2ot0z#%_!aDXJvpm~&Nj?)p!$~-bYF*twGhu>IL2`K9Za39Z zwpwfoEd*46#AV&OWCs+l#qZza;m)) zz4wNY{CW|dqN*ZdWsit~pk`*KjzNM{pO;jikdRDvpjS*l3jqc+|95mB!_!U*zz8^{ zJMaDG&fan`N{y4Pl#1J>Xr%%}i>CdP`2=%CG{;yp*SaQ-JRK8{2%QOCL~?M(po%IK zR-llSmWiB-6P0vmDHIhI6;@lhTX=*_bresErI{5qRefuX;~^bGviH%)z?xH{6TXvGj#=iofz(M+)im;_jSFa)>AwVPu9dI8Z-2;>;6~dnv`Xx+$IG-W@^l*tcR(DeR0s!YK-YNoyZK3Q4G#!nA6RTSu0ZGc!vn zGP7SQGPOJ(#nxHvVKLliP8msHn@<@)S{&;fHG_zXN)e-xl|A0Fge!9~m_jl>NVYQP zXRLUVT7NSdkF)+WQ_^GhL?|+yHN_v9>{(av2@IGToB#kbJB7@cN|uvc>CeF*0HSL4 zpmp1}ec!DXRgGdPI6fvifwL;?K8l)A#u&Yniar#fqha#kvwg zbVMGjtk;i+j)kJ%1>8y~`sBuk-b_qH8At*}(k%}i(t^!@0!s&x#t+ihE?0zSmRAbg;%yC_YU382w zMub2%t)==AevH7>4~U5Qr=+Bm=@3#%iHM_3<&1<8@}-MdZDzOouq_3n_f&%I;R

8W84Y@PnVx6JoNRCN9T9w73UC8alc|LlU? zDIyMUPI5Y0uYY{kjj&e0ig_7gFBy+Pbd@N`YU628ixVEyNaVLwJ8 zeo&XDVix+lDNwDwBp&@Ni0nqA$BmI1Hp5i7JSjX|bWoy@7fo-7C`!=3dMH=h@tVQ$ zBX!>8#x6s?4O^1*vg1XvXPlgC)$DXfjVIO|xf;gbXH}_#glZI}WyUiJ@Ut#yy+Zdd zt@}O(e`;v-+H!cV1JeJv<<)NMO~mH2n~dOFnr6JLd>YI_IPXUxYaMH0-&DN}2)?$4 zi;*)MxPpq6okiU7xiCpD3_ap6xe?cAE*jI*D-ImdQD-hduyd$^xtY%G>~7?{l!v_O>JP*k3tXAK#tadfl8wp7m{dPZfH8hk`d*wA#`!_%GT9LkYP zk<>w${L3MD!n**l;4ZgU{xvXNM|J+T)IXoEHTNZJ=aL>qZ$W$pAb`*zVA0Btht!$H){bR%2Jei-FG z0&um>M1IdTwaj+0{eJ8_K^l6y;k~x>&UcO+{9N&ldChV>1D`CF19?3>%FB}cXf12+ zVUN5}+$kQ{#MT*ZnH{G@V@xIuC(yLV(yQ15a$8yLEJUELH^pQ{824B?QCa@15Gu}P z86*1AIXT#$c%1Jo@U^s=?*KP_K=gg|-{!ncRc&e`q{ z*-FK_34Bh#`$vX`9@$aKV%hPgy|r)o$&+i8&FHkonNrF7*-Fv-!)AoTnVcLNxY@~Y zZFB=KznPwsOKJ$4yyGfy>(NZPH@u*tAb^#|JBVOmHa|nh={-?TH?|l#;LdYm0QV>c zO?7h;3?iymTd!xs4~RG>=uPG``nSdn;5E_xn_6l12|zI7!$#0o%TE^cs&@;1WQsWdGp_4VM-AI3(IXVL z_??^(1v-k7vLLY}r-xfkNof5+{jL~PmkzTOY;Ma?id)Ge9;60T7p0-AvCNM^K=o5f zcD;poLgYs}%R5hJ`0wimFRW>Vkl&^gqXG{O-d~=toJ8^e(*8K*#@_GrKuk3hOts)$ z(@yTNn9R<|P=tMG>>zpB4X04u`t~K0Iy}@R*?OwIdgH64Mp?JcX-EEqWyI$yQiDQ6 z4H)INY-4U-XN>XB`+jtPIddg4GW2Mt+9&$1!2<B*m_!@V=p`gUUR>jZiyT{bcI73k14J^Q=#nCzdLhjqVTJtZ+*DvFzSM z#vnPOt82~ZDYNj?WHeja(8(tDl9IkV*j{pISZQYK5g6a%PV9vF<6=Z z%Co|prFyhRrEN3FB8xH3r^z}Y(G(~Cj zs{V%%QPWCBlipxrk`dvAn(1Hd?ja$rv(+6n)jWysd<|o{7gBV&8P%+;8b(R15Yrj% zmcH^Zog)N^Xm;cv!Ydx@FZ4yFe|Eny_kkQp0$+&?o0NTZn?tLNxlA8l@zJv z>rd@;8^KJns=gn2;)hkqXtTALCy3HlC`qp{$14FECnjz+yrGiQy?4LR6j;(wt)iF6 zIW?!pazo9*m9-p!xIs;^EZ*6Kn1I0JD$>bd89A>SK6DI&y>sj<0eP8}aT=%h!Cp4l zGAV7q4b~z=vdMN^#`^Zcjt@=IeY=zM?ZGEpVl z(RrNxx-zZ85u_Td&%ww6KqXaAyQvVco&)xOpv-t(wmk&Ty;#oThQ^tLm)a`46$Lw? zs^KBX@X%1eQq(p}fznr3Zb-^V5m<0kjk-%vvFzA4x4hEF({s4OG&^i$B`_R1LU<#b zpzb+b$sF4dRNFv0t*PpZ8F1yx(1hzU5?$%I^re5)z1uYJbQ~*b9dCa?r}1~e{!uz& zfZ)?~o&I>aG-l(`jQBOGUo%I7O%s-$K13UXWV4%MWzRn^Eso|>0z(Rd#?^c!%u>&G zbzM^i#we^8smdFDw71k!lN9zOyVCO4B(c#;5Pzz>^IU=FTN9ls;hKf8o|uP)=b zKkBP3I&IS`vwsTsN=SHFnNJ(LET=e-(p7-nYMAUp3sYaKpLsO7wzMuz#|v)sYwbIn zZX~q+{8T@Kod{Rty_dC=BWc%&eSSumUw;|fPq{S@d0H`*hLu|q-`d)e8&$NGa&Qj* z+g@Ju^U-zU=a0Ijwg2V*-Fap5!e8eS59t&#@Aer>TfNx(^C+ZgbrO$1OPjyM zUGBZTygz@r_*VAaZ~cLu-v+9&UjQ6sZDd2v@&@b`kFS8=5$e|d=eu};>ggn(j=OK} z)lW!DMz{sy2Ufy8?dns1eKy@I7>uTz^ddK%E#otLS7LDUhnxzZw59`14)Ok_{^NR! z|LRBO&ciW^R4%QH;U{SVD*ph~sGw-SXKYDdH)N-6O*?v-}GXE4--?je*e2Mwc5MC z9{p$H+05ibAaQu9v41W@WMYTkJc`(n7Zj);!`b!iwZ0hMmXd{kb9!;MD0LtHW4a?k z(Frg9>~1{9R7A#^(YZGjp2eq`;P%m8%Cs+~;8&&l%a?_QY5HO9TY!4rtH}8cS=j)p z(&;>4d1V3N%K@KHM5egdcY|-a7+M?ahCfZjSp!pR-`u<2ww#&mZs(eGl%9aekT`2O@CDeH##^Uz2+4w# z;uF+me+Fj_aL1B>44PC@3|Hqit|Z6M#K*p)mj5|^NUsVX!WbmoDxq2Mo(A6-!u*Vv zicw1BxC$Xb!2q9<`vIRn`l5@zT)pel{hJ(p=5qV1QPhb=fE+-oecoyL9KZcG;__z` z+l3=AY1?z~=)0pYb2A-F5T`YynCun^gMCRFB_!L-Umj`D{n`18ooj3%)CYw~zChkz zZ%LwPji1eEp~|wdGC|GrxNFe|!~NwR?5ZY3+v8bi#fVlWp@&$EHd1Wm-;UyNQdTB@ zN>Bqu9gaxb8NJx?-HVD=yx12Z5pmkaj&RR>bUnsXK3YbV>Jh>_>r?;m!yTK!R52mOP@pjO%k z&`8U-!%?8ayFzBW`hL$1lAe`&16sOLQ;DO>!aZ)Vv0hT}h7{}MVaWc(k+NE)Fb0K#;) zsr=cmGf2$95*@QWI)$oAk5loz79Cgd2Ds{j_g_-YZJ^i{K`G)QDpuUIl}??x(V`c9 zdfkLS4%9nzFZ2@)XAuNvzU{Ra@%GXbAp0XX;VZK^GfwbN|NZk(@9Wf0b=i`xaXTwr z&885;T0kZ0in{_XpQofp3z%i?U*|4aTm^1>I1WT$KJ9y5^O#q_A*e*L0^SYLD7 z|EvTXh(j8;Z?%2(`eliuAR3l1l!LHioe9TiUuHXPM-Y-@bJ=D5v?isr8;a!aMjowB zcAgUgm5s$F$O!G8#jl*B;9rjS-}2iwT8XV3hEw%R(XT7R;rtV)MPdK)tgt{=Y-&&V z;|Cm?UV36oBJ$DCJ0dTSAFzog#kEk4lNxI%NZW|oWTmI>cDC6ok&j2rlqkOr`-`Ef z_l%#Kkt;b1ADg~o!WM5 z#QsO%yV^S#F1y{n)Z+akXN_*8EO4AS9t#wv`kYW%smTTp(e(YEtH$p9a?V2ty>@Wr zF>nCz9`|^z%tc04-b|KDl|L;Bmx3-gIAN*DUBYQBMtHL1q0XFpy3>i0IC_vUdo(y@i_dRbexD0d zhO&QRt&^2i|K^_emhpW!G0KWc75u?0m)K!I>4vtGK@XCt+!P_?xyh}G29vJcP}Lzb z$Cl@p8f(B+9TrZj>%M1)7M&M_=%a}5U%^$LFAWj=;h8zdM~lVcst1wvX2j~m9CzK7++e1uTqTeo3G$tk zr@>sgZKDD%zE%qFxh7rdPN1Zp+r;d2cE6X6=IvfVHWqQlCG9n*9geySQS;ufpN7B? zBB>@)vSuX@+10Y(x{B5j^JKl6yKlH?NaaD>ryc5^1CFdbsXZ>33UQZ0nB$DkjMq6S z(vgdX?p;dmyG?M^j2W`y#|Qy3y$}zBGMF-QakudRKB9cTVIVQwRGV9|f{tCAT#(${ z9uq7TQcqVCECon>82VV4K{!YnH^&>Qn1WEu0|L}Wr|f~Wrhp!ub&wPtln$ZAh|uQk zNDS%IO&M@wgwruG0|R{S25TElsRZG%Kit2+*851d-?|)pD|xjR_H7{2Cl&PNlA;#m^D07uz#}+vVgG0?q>E!Rk~y-hDrKz`+9cWN+WJ zBQ0j<#pHm@U+-C$aDvh>05A=;0}9aA4(vgk3_eT@Sw6|{;_A9PPM!4UgzvvT;=aQE zid|c3i1=yCCQm=At+jTyHM}q>j_^^M5lN%|2QF;<{$nWiRdC4D5PIeh4tJI)TSh}4 zCh`Id#Ywl-+Z#avWaH5<`Tqgqs}CB3Qg{;r@=5_1HPBTc@*}Cebp2?DLU7nRUSFy< z-ZBjtJ1`%3K2l(%ks2 z?tn-Ur^$NQ;IZ~FwLhRR&1#$Qtn<#7HDzOrVm@SLupOLD8 z_50P+VilpqQWVDS2r~KTE90;57ZLIGz=Pl6#o{wG*o`ZK z;7znWWz`q_^+A)Hx|$MfEA`d3R4T~-V)b4ag2y&X;f~lGeN3$CwH7OR>(r*`S{tzP zSug1Va>N&#EAG=U6_sQze&#X~fW2lr97dt!2$XwAj<{#8tL+_K*%XrKET!We9 zx5`-)(QEEhf4T%WeO(hxP6K%^9+S^HpeO{Y_gXg)Dy{CUuBM->(~#fO7;Gf2-s?BH%N{Jmk!id@884mf)tMcjEC4eqB%|N9mF7OujC~T@ zG&4c)e0hF+aI#*uj2twrNI`?tPU+-b9Qt&`5nnth)Rh6HThJF!rgVTQn0n7xDh#EO zt4XgNK<^wdTmaWKh^S4wNE|p|Rk?pGxA*3qAN^;t=Hld)CvBA<<}Z_dTE33KR}1F({%Y^Kd67C=J$j|7 zv6?b_><2wCl7`{UI}WA91E@N8-)^sNJUX3&S#S8u;PaK<>#GdhF%QRC&=~-IiX0mV zz$8g$BBWk}j9e#L=Y$h<_={_V4RGeHB#DFs4Zx>&2iz*h4r}}nv>HgfSDn>OnlmVd z8LgDt1(n7Zj46djT>Lve-HuL~`LFf2rCp&UX*r{);qs!X^P6};X0jmlZJK~TV#(wi zFfn18VD+b*>RIt!sxm&w%usP=ahPHScLOj13jQBMXCBY=1IF=fhPgK+N{*SUq`8kA zGjpGz~5s4oz`RMN1P_d#j(Kp!& z@jdRP7x((*I0;8to}5-b*Z;eX->?fj;)l+mrzt_X(S-gUu0D|CVdfbs;rN6HHP`Mdd)Ndo5mB&j49C;HX*oaFXlC|_XvUOU~}Ay z*NcyoMyXz9o;X1%ggh7!xK`=&$p~Yp*}Bf$*9``|G2z|W+#-kHe?6WyBs<{WU0?qS zMD)*ubQ)yUcemKtZvNcOS?4rt)@>!l6{opr_l+U-n01LrKegO-e~Ly!y;1$AF)5n* z`N^;uPimgyW}jEMmY&{^9(v>Ev`ocj&4oWbsa7~C*r@pfs(WQzJ>tR{w;4p9&n*>O zch*C{E`W;i`0?;^OFB{nW9C!=cuqVq;EGLF&;sT0ePl@*-HoBycZaF8fWQivSU?HD zYxoU~7^5^39IvS;00;mH1Samx%f>{L61t=yXKjul0T3|0UxP2nIDtdz@70w4;dnlt z_@1jVPyiVwROn=w$tb+BLoLP}wO^d6pYOMudg%;&iL}cu8+BDG`eJ`e03Fj|h`#5a zfB&lmJ2qVG^Ul2yZV%A^|klk2dZa< z{b3h>mQ&hEp`WscjP%9l;ZFgJuU|dur-Sl!t4Cw{Z+_WRF~98W2CL_?<>`j($-+{! zpfcL`pD0-3m!*LK>9UP`Hvr7W+SMU3u8M!=htb4T7xa!VO-2q|>GE!1suB+-@uLb) z9^X0G25h|l&w9fz>*(-L4R27jV^nD9;XfxkA-I9|+87`xS{-Be%#RX4& z`+L&UbK+xm*CJ$~<|&O}B9e_(LC&*s?$0n$(1;;M3AoYyb9vvhuFZd>|7%@^rhwJk zFdyr?*|(co8@!Eur5OSA64>)Zc$PoSw^BXo0E@== z&GL60FoN3tM=4P;&8Yl8SCWbmE@P7{%2<7Z132z#6D`_78_F4e)wLHt zm%{;sO_ZK#bXCQ=5a5&ymg*=`49jyP{kRjGLj9{8<%BC4#niqjeRATUTyhLYQ5!Q zYwz1nIl#_qfLH?f%P?XA3Xbd)~o4Ju>lX2--tb5&Od&DZufiy1=;U?76Bl>-N*(#=ET);nBnnF9j!s z=hVr)I^&M(@6+>ttFHD6g4fF=_9Mc{!_qhRcX{jsa*f+VkMU2B=^<8)&k&{fzM-N0 zpc$pos>9vgh^W6$Vk84VJwdIj@Y3;w=>z=Jll3auTMN3+OcpNg)SuxOBE!iKHmDKI zNSoVcIV0m{wdmW!A3|O9_LtHIU4D_Xi_pLNR(IwU7SvRuI%+c3M`yfTnT6K z8u_WnU&5;-SQwUDo0Y-=O6s{KvNfz#RFrB^*Kjm{w>5K3V#oqsYt3ZY`03q`RE{jr zcfVXp;g;vzrLpp@@rZ}FtiAY1;}&}{|HT^?8qx!Pkh{+KhUUU??~8K2@nrp(wk9PT zed(tOt_cZzTVL2x#Ehkj-Xh#rQ;C~iKG_Yn(^@&)``2AcrAqzmQ$IfzkBvBq7^3^$ zMl`j(Ind{tK+x1L4c(iR-}$*je2M9)Gb9yLO$d%LX#rMl?zv{EIohHB>4}LB{5%PR zX-6ZIy*;3@SLPd4*Qqp{7~i`bDG@n7)e3zO6o^8?ASfHu+ginS;h*H&XehJ!oB1n6 ziJd}J;-!eb8Yp+cwq^0PqU#9d=;ps$y8BOGT)(Iq`>V9yxj?Ymv&T0|X~nD0h_?Y*f9pTJ{riQbu}ZvC41xJgKW&$(f+$jy6UZijag#kgu2|5<_)z;8m8}(i-|p`< zJP23JWGYzsjy-MjtZ3rx+7`LY+}W|nw^bfyrk6#`SF&Qe8otX+d40C+AS<(0c z^(J`Xi{=}f{nte3K&t~u3#+5-qTPksE{7%w=yrWauQ_1Xs=VjAzV%S>zi=ri0yIAci*2WxphFOjCsj`#Lf_x7j^vH&OTUNM7n-zcC3fH97HfcpJK%=#yrpMaU^Zu z>v8Ghl(LZ$gCEiQhz`KIm6*85%|~DzK^EZza1^mbEVIV%*uZ{Ia$>e*{M`2Bk6*!2 zDtN(sLAG@?6v=c26vUtn*J&5B6iE1ZVnGY0!P4J9UIQ*QHiT3HgPWsHcKf!&x1$qP z^akH)*JlUKeQeCV1IkBU(msT22HAvnulgb(F%S@V4b!m#MJawiX9mIKa%K$+B=*JH zxI5hggMUD0T3BxcV6Gg#(tLYeh8>!i%sxx@6ysA@`CZw3o1?`>A!O6&bg5fMhY^D! zhAS{~9w>M+T?{TkyScx&N87%+9bf&9Y3W7*Tf`;U!MsY_(%fr(WtTsR80UfwE(b+x z$sPx7fgXp;fu&=%CCKV}x;vNN&%gdgQ|xkp@^u{OYQ8ZLv_M;Xzpq4<7fj_d47ci0 zZJ-)mrOVZ75&Ht--e;46aevE+ER}o9sM6W{>;L)$SaRtb}yn|3^vX!i^9tEb;)^gT2RNa>G_$Y~AKKVJ* z{pkH>Nr6%80~^U22fo1;vq9UD(&cYwC*+B3@{|`%ePZncA8NdCh&pHalV)ENS@GRN zLt5dZN+$T?a5jLbgll^9v5d0W2nR!Yt@Qr#^77C|GXFna<}T8GzI5lAKVCE7ecN>K zFFE~8wmw&zql^rbi{-|J`bpmRhC>llzk#1p&RVFXt4Lu_525vj$ah^Z>0Q%eME|&y zNg9+rAk*Zh>@9MGh7-!6H(I?sX~moCD6}2hY-%HtrA&dh`g4(dGmtx8|EsSM= z{~_xdBcVq_k@6fAsu;vx@P-B#iuHKl#%sKb)5C3lHbIs8QW177micbwS?dfGd}k>% zy02gKr%!r6iiF^J&q4dVf|kT|`lEDNlBVZrhI3E*Ir3;p z0HI=eloe3ZbX!XRj=N1QKHPxPgISNN?qtMifW6~EXmJ`dkLep@6Dna9_)Xri}n z$}DRQX#`_Xtgs|+n1|!s8uxq#t}BKxmkuTh^jG0E8S>95d6Yv#-gE>RXlMke ze!{0J3tbyZMdgPID&3;zP8^H}kHjQmqG@hXu64wHx&!{C+LgxesrBweTev5Im2ZoW z-8)H9`Et`t7Ph=kTmnn15#-YXbfQn$A9o$IEi#LVa} z!N0!aOq>I5!P__DU&#*CunVy==%CHcmh#Sb`7eep$Zg!cUGVQ{C}$3u72KD$h5)KD zWJq)YJIWBWT7@Xz5E`&{!jI|j?IsF0p%E z7i&_k%XwK5)8~F0%>u%@&otb5339yNT$HFpyE}ayK*Ag3b562i*Cc;F+_K(&N3b`2 zcYf9{=OKw}e`b5|s$hSLf52`aJL+A|AWoZ*(ax;~lsBtL$Z&k%8L2R!Q{=1*CMq5gjG6KEG15P5cTh)*Gz^<-dufJV3eXn9}S}b z+G?+n>XQy5LuugUA$KF6L@4H_53TMU?Am-_g^91S@#m)2lvm+To~Xd3xiyf4{#_M6 zugb&uxbM5!J_dSMS|sqv{OVPCY7eRN+0M?fpVDjdT$ijp*^E$aOh{i=q`q&zv%cJZ zY=b0xC@>$}aAX*f!!|QykR{_GDktm8|5stUK=+E0LyR|_?l!UuLI_*&1}_;>2x=1jSgXx8IK8&hl{y+@oM9WQoV-?3 zb9Ocwu!KP;sgdk@AU!#{>_FinVMn^f2S+CM2dmP{(TbPTIQ3fYlq&s*-naVPy);%Y z()(V^p&*S%$PA{-91Y&WX)kUi2{5Bzoc%~!6W}MF#(7d)3TSTr^&xaE^vZWGtiTni zJwzAk?UrMF7gyD!5SoA?C}vvW%H&-{2=DzF_Z8;<%^^9WvP3$k2)ha^0YE`0eB55E zDx@xq{*ArBRQND0U^{-q{`S%=KX+`1O!Zwa|!2t-PKM-&a zUYIsickqZ@qX?QEYqHLqi4glpak`R#q zD7Uz%Q9hYl1alBcD*H0(xubr8)K8l&Xhsj?1C}BQwa7At#qMZLq5_Mys&9r`!}s(dmtCy@O4N)xpN0 zOUHa6LY6=?nTl4=SeyjynB132I0C}&=E<3uC|fH(3w&hZ#)s8_{b+$JXN4mRpu$v! zBO;?|!JCEiJ`oH~sW@TwEr*I(EzMOVH=1=6d^)R#4ZOl;;pY9`)H1b1M^A!ZGLyrSO^vZ+S zWt7{+vxZCEH3vTP3ep8g#)~Nr$tU6K^UuXGn8G9FjcZfA6xyh*;lK0NEv@AhK_Lph zC{#;n5r&)gPU1W5kdOquo17}MJc950RgAE=QB4_-c+49%!&>Jg>nA6wsk`KIw@T+A zGm|xZ*}uP2?SF@T+BV*0Z2poVO=V2+t6M`gK&VtunP5w^QelAtL@OY~5=zSOSEO8O zZfu#9jE<8Aa)3||-Q?VVu~x>bbxbTJ`20!Rhk52fJ-8qpBU+xam- zFX1_w*uj&M5-UukwBA;vXZ6==yFK2V9Uk%q638lkM}PoRC5h826KM*qNSW41Q3Og5 z5|huslO@vER`vAw)V%i{TrL zxeE@V{+>Hhlabq}$t{w!1QYKVBtDu(nF>RPy}9n| zTUahhEi@$3!`<|VE}KdAhfk%_OjvycchqU#wF}?){u+O1+3rLcpA%XnpPv5J`bYKO z*-CEn|Fa6wJ28Ic8xm6V-e0?3zL0nJ6=?CC3}0ye=&^+&>Po*Ch>Z9<)(~XFU$;qa z$XKTWrib{|BmXYpM;A&hh~3@c+w-zzj$_qPN7>XW##Ayt-}%lz{LtSnvoEJ!{K4{=zxunhH$=Af`F(F~X~&QZ7JW3&V~i0O z6nqNznFFP`IB9epEOp~U)u`SKBjzTpIhwjT zm>Hgnj;7V(fcxXh1)WBkBaX4jQIV0VuUXwn3Miz2ttt=sQY}oD_`4g_7tFrU-0GdBD3EQ3shJ*#I{;VGvLlk@k3oF16D9lCE8 zw@ZHz(8Ag1_&RbGMz&vagi+t?L+XKAQiGA)u}R_VaAYedWy~~&j~ZuN+|>|%ZCn)yI`bw&YX955n%jhMv#JfG}3R+Ppa$5AXE)M%}s;j4Yui!Z^wCPUay;Tnwn zC~3XY359_P;;ODM<+x;t%xH6d!Yasq@O@rGMVKH)Brscvon@SyISW@VG@|a^MaTwl z2QBF??SzLdzis(`!<^?|w)CBpNB`IjiVBjJz!eQ?N}nt|X*b_^m?`WO42j-#{edPF zO!LCVAZ> zn(DRlozn~H;wK-KFMab?c%1UO{=t(?HmuTb|4s4PZwP-({y9H! z$;iHw4(Pfl|H0JU>rsx(ccU9ot-+btLPt#$vn=T|-q~t;bI4$( zeI0M@1=V zW3{!`22V85 zYO=S~gGY4nJ;_~o$m~(&gInXZ&>I@}6U)eeJPg+|zbxl0mKN7HXyc}q!?sn^*ivs-oH`}A{3J#YM{ zW7P3M-?oh1H~V4*XuRv^vfqdW)ttkR_emadlaotai@f;#GL3ox36*u>R|)3YsTJ>I zQ6POzj!>QrfG4%nc%!E#+z3Ljt7kL6fDl?Qku$yO^G4X{Np!&Cma7jj^HSo~tj4-( zmvfnl>zb<@UziCB6jIE@YAyQ~#fO8}q5r=4u>2f5i}JVhoEy3lvfj8dWzV~PaC~I9 za1CnXl9KT3V}|Z_-&Ark;rJQPbHbAPi}z9Saq9MJ=Ldz?w#&er&?hFOt{?T8VcY<2 z9_=Y#+2?KcVN<>?lF)b8W>bJ#h;?cy@~Q)qj7C<#iGQT<^F%{VDkp~puy1jGu71{s zXInoTzk8QQRf;QJ$#o<9ZfuEnf9uKQ+t^qFg!S896u+Dk27|mTvp+wpOu|=GWPZb! z`t-iN5e|lBxXRys&StWpY#Joj(e=|eN+pkfUpl+Hpug4-8%0{{V`J;%zkZ_q<|INV??i2BEfca_RA}_2gCy-ZDP274N2fr^R z62X8(0bzNqsRuy@lvIO})Y9V94a>{eP(_XHu`v~=C&oRB=6##T{>=hIZ*6sP?6Bze zlWP@oJ`{ic-9!zSo-r?$%q*AKw@Okm@}~#gY@>Ne3D&VcFYYB zY@C|80S5t}(Fn0i3GE$=qoH)Etw$xlAa7+~I|LH`tv&CP`c1)r;~4IAFLZ@<_yOoa zC@6Eah5p{(zsjWb1Nl?Fq<$O%9vjLBJ95NQp`!&MJ&?lo@0WHt0wdzCK4lfS7+k(q zvO;c;Ro^%~|8Cyy%yDL`b)q)&H;nvaiy7jxS=}0wT%VnhcC9XSyS{Dyo~QiB=n}2Q z%0E?xapKtp;#Y8lA{^FZUd01rSur`o9HF~Px~$V$>G61|_TR6N_VcZf_TnD9AkRi^ zzBP#)?8l%@#>#M(xrxRRMyQz?Lm(6Py>>=pT#>cDBX;XaqYa1`b-rA>ZPQp8ZmuA3 z_+u{X&&8Jd`P@d(ixwk%fD-TC?tRuiSremEe-+80t{95qt#QZeaQ^^(@p|u>hLnRe zc7gIO@k{g&h&G*JqRO;aDo)&4ObP)8n1K3Rjm3ik0vMtBU^5Vvl*Pr|B)X>*=g6H; zgD?zW0MP{o2G9fmOeeqyLMAOiDkrnlw#3$KV1G%#d_HW(sTAO3Oz7`1gmO*$NqkYH zHpc87_~!#K{YUDZ^aW;HhRw{tN;!)c5D_@{b;TFHVcpW(eEDw6LSH+LUdV#d2pXwz zJRK^l_Z-a|+nQnd>w1a-GcRTZ1h4`W!#Vliz0LH}FloQVt48$iCkF?x4770s4#C2o zcB7|i>Vg;;d+OvgHNUBl9e>9_cPGs(y!mW@o2pdFi>jNZ@$za^a!u?&I$G=JmZUPg z%g&nnxv(5kA^_;w0#Xjr13CKvV4y^zpH7w34SErw%a=K$mMvqb7}3{Y;W>@~h95VR zD}SHzIdp^Y1TkGqZIRtoAz?1JN$=SGbkem2)yOTH-A_nX$zM~4Y;e#ySfrgKeRkTx zw0PrQ&(z}KWycM*lfO5JN{V3n*r^#!ki%j$KC@x2E;dw+vc_0G@6o!s(W1i!cLL0` zR)o9nlS3@&;nFGBB4?^IHZsU|=Dm-M;b|&di?qe@?8ma}eLm{{*0!SzKFt1|EO=DC z#S*3&vr?t{Hd|hpR#u12SM)n_vs;t+l|fIev@T7YNj?NZyA6!yMnxyi1nKAv)f)DA z)C8?pePclb+z0T>c>K@6U?(Wo2$e|oM2(-H72{BXHBR{4bH;>1RZws0X{r2Qgu-lu zqv>1XTWeg_(jd&y=8~MSUhYLqf_|6mLm8n|$R7cEKP;q?pnG(iF^K^p#I4no;spsDKFqNZ#T@aUzX8X@HiHJ_(MsWp> zc4+eCnnuek<9pCTBO0_gb{dY*)4w?uL&xXSXphQ_uHUg}CFujd9Pv4TEE6zm0S6 zeRv>Jc*KY1TT*j-@C-xu$!9ZAo=gldW+-FC&hi zOH9Q%m~v~N?uQ_t#%ZEX3?rBrg!al}v@V$7{4)UqK9{T)L<9faxkz=mF8%ePo13%E zRY8A~PF)NnHiDI~#P_C+H%~4oz!x=$likvW_*BJ&1RhtzN7}E}L;=m!1*fK%G@k4~ zdePKMK2vD_CMNQ9_csAD-^cRwF*^_oq*>nk_t|EDx-%urq)H++pgiLiHz{V;dCq%g zI;Atynm$c3uLs#k!7_(!5O#1+Hlw;D2+3si*>cQxb{wx4E5a1j??H zvgA>m5Cl$x`=RyJM3`3)6QitfB04PfmLL?nWZSf9TQ|Lop^R?e%m`1Ef)JS3Ay!+Tj`J^7YoUS7-?C!c=6f8?Nr{#HldL1cDd19zdY&{HW&f@3GTk% z89L|k@3OS7TYQzf!1m^6$!Adt zpZWa%v(KNK`cR|J|BS4sPFvJKYwJKrwtW7d+SaARCOo5*%cTczdh5OAmq9 ze3$KgJGE$MJ!4(^Av*~SGsjuY%}lQwePzzyXj_N3|2=G^5{-(`t(&Kl>@*GsE$jDe z#{aI?BV+yp=)nE<1g|q%`o8&e15l`QP*df7cV;q3L55R_=c53I4?o`ez;o*DMIFB-xr%idmd_+#N>Sr2Gq z0axh~;tuN9#wMN$Io=#92D~quvL*@antcuxH%L#s4({qe4+lfw+$+-2Js1#%RW~)0 z7HNgzp0$@u>3lN44dddA-tDDPfMD1WUEz8Y?`oyQ7U$$4f&kv(7OjsHUJx}yKSiFF zutrIE(xQ+`VBnyi4-@_4Gda}p6_)cCVuzQLj?#_i2ezC8SC16n^HUW&`}4jZ)!={s zW4e@b)E8xRjP{lL@Sxsu`7vCwaTg zwbDD-h0wSRNHBNNy)6%#QI$g&@Gu!GJ#^dRt2|vY{Zk%kIr0-}6sk-ax`WtmHP`O~ z>!$Acs}+Z$dmqerUsC!^=^sfDqM#Xy|FF`*L0`icqR#@wuGoF1JYvnT?raG`0yez6 zZAW84w7{6c&N(4=?C^jT+^Oy!xc}(JXR&|h;iN|IaIcJ+;BV~yDTTugGdD%6D}$RJ zvao9MfGA1}-i;fxTL%FU2KR3b{H2tnct1y9ASmU5pvcIT?W;o6;+Ynb=-b_bg|s09+CH&T^dgnGP5g779*vVWYT&vAwW-A2*%`hbP^}Vkn)X& zus?&N_7HJOmvn(i&q9uVqOcGa2bhB(J6hWqfaDGTY>TbQ*2XwFN&o=N;Ew_X{vSQB0?tEi^l zIO4kfbJ(7{DDkqcvK6$=R|!>)Yj!K>c8qGXHP@#a z#=jYbuy|dwLu==YcfYPiV0{jyzZ2dx`Vzt^_B@iIP>A9$jy+>*uf^Nyp zoA|I(0q=U?n~(A@vSr1F1JD4vrW^R%f)5(*Oe4Weg89}CjRL}OK395OC_5u^?+_@4L6|VnRa|t}{47*<(wB5G zE^C1c7*hcmQ5dlB%kjzSC{@UPzOGB{t7&@Td%r7!& zFAjbtQ;l{f`$vh;$fHRoJ1QO~W72VECP?lB4%*}HL5}wTq@|i=4%HEfZF9U96ZtPH zk&HqKD`kYMjc2t=LaU(uiFcbenMr-i^gl=ejwvY>&j{VGDkmBwhA#3NFMg!DSbYXd zIhK1+|JM*Qw<;|sib`7P1L)$^6z?xyY`+Hv%e*Kn?~~MH?@7okXY53)e(#Oko(QxI zl6ws66Z$+ZltUu*i}Dvx634IL9Ix)Xs?tug7)loeh0jZ7c-f_&{XBVsV+r+t z`2dK_jx)cKrW;_nDQ0czWfo6YftP1}Yb)&Q&{AEwP)gg*Ja%e-s0$)W`IEo^ZW? zPWx6ILvZ=Fx^>tR*jw;cn?4Yb4(99FDwJdwkNe}RU6dxG&9G6xA`tAVoaV2Vhkt|j zg^vK47oU3xDmZ|nhBZ+(ckD~u}FW~_2Y z5d+pz;qNj_YMWYPODy0{nAQw4()je<2{ZA~2c}>K@Vx()K9|QxLbmVXjCm%KiK_D5SC;z@_?2O8U@dXrQ4yaE&$jlrc zLi+r%Zitd&Xp1KIf__Yx9R~S7o*rh_+f#XE1}4J5J10XA8@HaJ!C)sA!*F+j7v6t9 zKIj6v_IMskUm9WPV0f1*v!CX#|ER^$fr_%Y*s3ZrbO()1am0(%fRo5EK@387$+qlB zO(IQa|MHgGHQ2apX^$AUP!Agvn?e|uHajFH_!p#otN;arvr5oBf(k5~NT;8{MY0$8+wWHEHu)lAO|+T{7*sQ8<&%fA(iz(o}=r1-j#--HaCg;bvA3M0bJS;!L>^{ zND_cZKac!7d$~S6Hnd0A++XB+zc*hAgsz|<`%unfqtu|?LRH}h0qVk>?ezR~d;4f; zWG>uuaY`~K6HmGHgE5vJ@Qlz8KD8}DXdz^Zf%ZZ$gdpFonZwWe+zyOiJAk${&;%@0 z$uA}qH;CCsPuF{gwdEv_THMGJwD?q!sRd{jQbh9xx7I9~G9sN`Ox%4Xz4xl%zvv`p z4XC_G77m8P8A5=+3E*XRY3r-n0F`}lO#teNCFt$ zCFXA20j0@u?$W4m7AQdFlqf-8at1j$q)QB#OJ*xz2qX>Ctm>!tF8$5m-R){h1^S%nouPYD3>EahHoz& z=#@f%s4WoXzVy(i!eJ!5l6DPE3_n~^c-zJav~AH;nSp9Q-HFO7nVhD!zQa5XGWEsX zf>qR{#^s)H5mO-CvAGS;b92|{4$jY1nf!ZFLnjzpQst!UH@hnR9S+*~lEuOmm>}PM zEjs{wXWsYUK4J=>X%Ilsa}9tQ<$1?JEcq>Y;R;;TXzezPqe}H&P=SN>){(d{;Ope% zU6Yw+3{Q#Jk_NCJ>K47l%{sZEH%Ab>_wjKCwWew}^qBmC03AWzag#2Di2DE5&V7U; z^ssV8t5y{h@u^Arb+fSn{PUa3Ksdp-jq{^a8)Qwk*RV&o1blxMe^`Ra9pO zI-RV*Tbq`{{-HTGIHdH9<#3Gx+;WSKPlB|ieev9w>v~1h=8{mRgE+uzRbJ)iV^9#3 zS{8@CE)Gnf`;zcg{MbWJ3p@3Bmi)3t6Cx#i4?lSiCVpDn0wTu7*j4k~$_U`^E}i(?Gs)sw*c*IVJl-lB6zyY`bAGaZ1Hd4U925H9E5@VZzMKSorpP zd_W2cGmfHR#4s~r(7g@`7Oa_~gIdpwfB>@hd>L zwo#1P9uDS6aPW@q0Ks+HaT=N}4A`Aful#f2S3@fOcAJCjQ8#6`EFt@sHvhi7THO%r z!v+v5{&=s-vit!i?KK8N%8sk|wzCUY{Jpg;l`tX40OcO`k0_jM+%IK>+j!dN=8dlJ zw|U{`=Iv-S>W^9)yl4PUdsQ3v@;25y?9bM+eWLcs5G$VqUz_YHnfRE)uks6$VjS*1 zL2;6cNf08o>d_li*Arov;ACVXfE58jiRus>?Li>1XC(wP#x-z)_$vvXS`IzQfXi2X zY6^KsL6$On%-28;C~biNfBNT;!w+SRe3*A`nT$nBrJ>^CF0fo2J6zhGO~!{eVRkG~ zqM!7i(Ae1UH7xX}aBo23#t$Ihq6Q!l(w#odTwTS-39JmY$qpfOXw3U10~zlFI9L{M zWjs>i1vsp{;nGujh{lMFBs|Y2CQ0JNxK%0lsF)-G5CG7_Xjojr5|P>tSyw3^Y*V(l z-&4@oj@rr!z;&Pwz)WikqbfW*Iwr5PgQWS-o0AKW{imc-PHV-Gk_jU*0*wR`{;S2s z3cif5&;!IgEMmq0mWqHt2!bz>2GIr3fj|gEi#ngJ6jLl#;}bA8ycz@6y;2^ji>Wyp zVWyzPg#2l$Zv;})9>@R2lRw#H2|>ebGR(Rv>u=si^K$dXaFjmEEqN51)ZZV6iIE0_ zr3DPw^|SJ)WwkU&>6YB~>MY;sgz4fe0SxOF1+>flo!FxFG?s`iIPlRH6G(8;z z`XYK7D^!?$X9EV!iE!8H0s*Z|xm{Kqx_-C=JE7;b>Ff67D6%qBTbZp?OV8BpGbQ$m zKkXg%*3ggtHIHn>6I29r*avuXy5s}|9?}Q#wH&TwTRdmmUf5jI@*qFDiWm_#7|<>A zs;`c$oS?={a;bo5o63*&!Qy9Y3l}p!7r$nffBLn)L%C&eWTOM+X+Na6CIW^1Hh!#0 z;na9x@yrxm_%rTa)_yS{4}pe&V0$1Z@?B-X;#pB#5An+zA?a}yqanc;zOVXzOv*DG zVx$?wnmjziZu9SCB|CX^(^#u=)`ZJicF1m1r$8G;BfY`eg|e&2l^m`9kD~tcqy!Gy z(8i_87z!c!Q>iTwDvKd zFyNH%?NJl9jgvZOIGF28LtIVh!vXJ-XIO$^PeMS71^OCZ1MKRc(T55~z!}iV5MTau zo3RNPOt7~kGd@@x3jOp2-zzALM}6mnc%;O_K|~CW$lS#O?d#D2$`+P_IsnRmkk%L! z!cyUN#Z^-gM;GbufEq|2cFFlneecQ5+tQ4~`Oc&luLo{r4^5qtUqdfe1bbgtvYu^3 z_F6J=Pjg`XwC~{PeJ3hU<@E4(u0GLP=*k_&E4d^i4~8|e5nO? zMO;Z$h5#r+FQ!wzFvLi`7Z6UZ3}Gv>-BYoD(Rrs%z3hrm`bU?Mu{V~8u}4{BZ(7UZ zkM7x2M9O@q+u&iSEs|dNy>P3wHRlh?cFonZ!pa%wL<8~_hG_GR=_&+3n1c~=L#f;; z0^EYVxi>(wo(c$u5|ng6(8eO;?h~cTzRFx{^_i3AqvxaJ=jU!#o3 zY4@tXcuox!_ijZ>{!hzSUPeH~IH{yw>ps#^@e@f+LZAaC#0~9)WWug#ypH(7u##4r z+03H|666r!$_i5rfQYwz&;Y$%WOVx)_@=W&)SoNa)Rdl7b05&u=b%l>jJIJ2(m9S` zzJKjAWq=4?6=3aE1>6R|No2Vs6Pp=4DrTOgEtsKYrq9h?Ad#ZI2s7~fwe^Fwhg!B> zo1WSKd2qbSVfK3{#(O+RgJn!+Uv?Gh+r!PkEWtH4n<)c@Gx#&YLDEhg@AcyW&uiXK z)cgPnqzE_>?>xdxawz#0*L;zjlfL>eF=wcFe0F*ve`*2`y(F{jdjm>1|Zp5iv3=ttdvvCVL8Os}eTT?v4R-JLXDXF4)2S`)YuNwh zn!D@OTx#V%aGJX1ue4I@oO2EH1TOo++YLDVcr$F*{pDqY7)#y^NXKlK#5!+@+Hc8; zs&~@>XD&o4ueXiQOiSg)f8iA+>t2yjZSVT-CC8@(hwyu=MF@B-|95AiU~2@E!ww)K z>6r6408jwNf<6G%L|2!c>7kJLHHLcq6D(x&V28=rY)WL70YINfQ?v;Z@hbGdM}eN* zt9o0k-ZI`=@(CY081Dn?kDk3u6L*CN$GY)d>Fl(qa&L7JeX}cA`gp{UL0m{wTiOB@ zd$~>u#484em;Z#^hh~v*fyK+xzw(xBtIITKem|KLZM#Vw zf9eUoJ4?6$90>sHv=vK@B}t){4Li9&&FRxw@f@Q1-=&;Zd55_5~pr|4A)p=eEKr7(NdN1;~MGG8;^xBPd@Xk;KmU9$fIHZx8t&v8(}Kd8!d