diff --git a/README.md b/README.md index 59433626747..5d168cd4c36 100644 --- a/README.md +++ b/README.md @@ -78,13 +78,16 @@ To use these Docker containers, you'll first need to install the [NVIDIA Contain ### Notebook Examples and Tutorials -We provide a [collection of examples, use cases, and tutorials](https://github.com/NVIDIA-Merlin/NVTabular/tree/main/examples) as Jupyter notebooks covering: - -* Feature engineering and preprocessing with NVTabular +We provide a [collection of examples](https://github.com/NVIDIA-Merlin/NVTabular/tree/main/examples) to demonstrate feature engineering with NVTabular as Jupyter notebooks: +* Introduction to NVTabular's High-Level API * Advanced workflows with NVTabular -* Scaling to multi-GPU and multi-node systems -* Integrating NVTabular with HugeCTR -* Deploying to inference with Triton +* NVTabular on CPU +* Scaling NVTabular to multi-GPU systems + +In addition, NVTabular is used in many of our examples in other Merlin libraries: +- [End-To-End Examples with Merlin](https://github.com/NVIDIA-Merlin/Merlin/tree/main/examples) +- [Training Examples with Merlin Models](https://github.com/NVIDIA-Merlin/models/tree/main/examples) +- [Training Examples with Transformer4Rec](https://github.com/NVIDIA-Merlin/Transformers4Rec/tree/main/examples) ### Feedback and Support diff --git a/docs/README.md b/docs/README.md index 508409e75ec..a81d0367ba7 100644 --- a/docs/README.md +++ b/docs/README.md @@ -9,24 +9,12 @@ Follow the instructions below to build the docs. ## Steps to follow: -1. To build the docs, you need to install a developer environment: +1. To build the docs, you need to install a developer environment and run `tox`: ```shell python3 -m vevn .venv source .venv/bin/activate - python -m pip install -r requirements.txt - python -m pip install -r requirements-dev.txt - ``` - - > If you add or change dependencies, review the `ci/build_and_test.sh` file - > and make a similar change to the `pip install` stanzas. - - Alternatively, you might be able use a Conda environment. See the [installation instructions](https://github.com/NVIDIA/NVTabular). - -1. Build the documentation: - - ```shell - make -C docs clean html + tox -e docs ``` This runs Sphinx in your shell and outputs to `docs/build/html/`. @@ -43,6 +31,40 @@ Follow the instructions below to build the docs. Check that your docs edits formatted correctly, and read well. +## Checking for broken links + +1. Build the documentation, as described in the preceding section, but use the following command: + + ```shell + tox -e docs -- linkcheck + ``` + +1. Run the link-checking script: + + ```shell + ./docs/check_for_broken_links.sh + ``` + +If there are no broken links, then the script exits with `0`. + +If the script produces any output, cut and paste the `uri` value into your browser to confirm +that the link is broken. + +```json +{ + "filename": "hugectr_core_features.md", + "lineno": 88, + "status": "broken", + "code": 0, + "uri": "https://github.com/NVIDIA-Merlin/Merlin/blob/main/docker/build-hadoop.sh", + "info": "404 Client Error: Not Found for url: https://github.com/NVIDIA-Merlin/Merlin/blob/main/docker/build-hadoop.sh" +} +``` + +If the link is OK, and this is the case with many URLs that reference GitHub repository file headings, +then cut and paste the JSON output and add it to `docs/false_positives.json`. +Run the script again to confirm that the URL is no longer reported as a broken link. + ## Decisions ### Source management: README and index files @@ -65,7 +87,7 @@ Follow the instructions below to build the docs. * Add the file to the `docs/source/toc.yaml` file. Keep in mind that notebooks are copied into the `docs/source/` directory, so the paths are relative to that location. Follow the pattern that is already established and you'll be fine. - + ### Adding links TIP: When adding a link to a method or any heading that has underscores in it, repeat diff --git a/docs/check_for_broken_links.sh b/docs/check_for_broken_links.sh new file mode 100755 index 00000000000..79976896967 --- /dev/null +++ b/docs/check_for_broken_links.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash + +DOCS_DIR=$(dirname "${BASH_SOURCE[0]}") +FALSE_POSITIVES_JSON="${DOCS_DIR}/false_positives.json" +LINKCHECK_JSON="${DOCS_DIR}/build/linkcheck/output.json" + +function check_environment { + local err=0 + if ! [ -x "$(command -v jq)" ]; then + >&2 echo "jq is required but is not found." + ((err++)) + fi + if [ ! -f "${FALSE_POSITIVES_JSON}" ]; then + >&2 echo "A JSON file with false positives is required: ${FALSE_POSITIVES_JSON}" + ((err++)) + fi + if [ ! -f "${LINKCHECK_JSON}" ]; then + >&2 echo "Did not find linkcheck output JSON file: ${LINKCHECK_JSON}." + >&2 echo "Run Sphinx with the linkcheck arg: make -C docs clean linkcheck" + ((err++)) + fi + if [ "${err}" -gt 0 ]; then + exit 2 + fi +} + +function check_links { + local err=0 + # If you know how to prevent the hack with using jq twice, lmk. + broken=$(jq 'select(.status == "broken")' "${LINKCHECK_JSON}" | jq -s) + count=$(echo "${broken}" | jq 'length') + for i in $(seq 0 $(($count - 1))) + do + entry=$(echo "${broken}" | jq ".[${i}]") + link=$(echo "${entry}" | jq -r '.uri') + [ -n "${DEBUG}" ] && { + echo >&2 "Checking for false positive: ${link}" + } + local resp; resp=$(jq --arg check "${link}" -s 'any(.uri == $check)' < "${FALSE_POSITIVES_JSON}") + # "false" indicates that the URL did not match any of the URIs in the false positive file. + if [ "false" = "${resp}" ]; then + ((err++)) + echo "${entry}" + fi + done + exit "${err}" +} + +check_environment +check_links diff --git a/docs/false_positives.json b/docs/false_positives.json new file mode 100644 index 00000000000..2b6b0eeb66e --- /dev/null +++ b/docs/false_positives.json @@ -0,0 +1,32 @@ +{ + "filename": "index.rst", + "lineno": 7, + "status": "broken", + "code": 0, + "uri": "Introduction.html", + "info": "" +} +{ + "filename": "examples/index.md", + "lineno": 20, + "status": "broken", + "code": 0, + "uri": "https://github.com/NVIDIA/NVTabular#installation", + "info": "Anchor 'installation' not found" +} +{ + "filename": "resources/troubleshooting.md", + "lineno": 24, + "status": "broken", + "code": 0, + "uri": "https://github.com/rapidsai/cudf/pull/6796#issue-522934284", + "info": "Anchor 'issue-522934284' not found" +} +{ + "filename": "resources/links.md", + "lineno": 24, + "status": "broken", + "code": 0, + "uri": "https://news.developer.nvidia.com/democratizing-deep-learning-recommenders-resources/?ncid=so-link-59588#cid=dl19_so-link_en-us", + "info": "Anchor 'cid=dl19_so-link_en-us' not found" +} diff --git a/docs/source/conf.py b/docs/source/conf.py index cbb4533383d..a55b3ecfeea 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -87,7 +87,7 @@ # html_theme = "sphinx_rtd_theme" html_theme_options = { - "navigation_depth": 3, + "navigation_depth": 2, "analytics_id": "G-NVJ1Y1YJHK", } html_copy_source = False diff --git a/docs/source/core_features.md b/docs/source/core_features.md index 9e828cdfe93..a3d92461790 100644 --- a/docs/source/core_features.md +++ b/docs/source/core_features.md @@ -37,7 +37,7 @@ workflow = nvt.Workflow(..., client=client) Currently, there are many ways to deploy a "cluster" for Dask. This [article](https://blog.dask.org/2020/07/23/current-state-of-distributed-dask-clusters) gives a summary of all the practical options. For a single machine with multiple GPUs, the `dask_cuda.LocalCUDACluster` API is typically the most convenient option. -Since NVTabular already uses [Dask-CuDF](https://docs.rapids.ai/api/cudf/stable/dask-cudf.html) for internal data processing, there are no other requirements for multi-GPU scaling. With that said, the parallel performance can depend strongly on (1) the size of `Dataset` partitions, (2) the shuffling procedure used for data output, and (3) the specific arguments used for both global-statistics and transformation operations. For additional information, see [Multi-GPU](https://github.com/NVIDIA/NVTabular/blob/main/examples/multi-gpu-toy-example/multi-gpu_dask.ipynb) for a simple step-by-step example. +Since NVTabular already uses [Dask-CuDF](https://docs.rapids.ai/api/cudf/stable/) for internal data processing, there are no other requirements for multi-GPU scaling. With that said, the parallel performance can depend strongly on (1) the size of `Dataset` partitions, (2) the shuffling procedure used for data output, and (3) the specific arguments used for both global-statistics and transformation operations. For additional information, see [Multi-GPU](https://github.com/NVIDIA/NVTabular/blob/main/examples/multi-gpu-toy-example/multi-gpu_dask.ipynb) for a simple step-by-step example. ## Multi-Node Support ## diff --git a/docs/source/resources/cloud_integration.md b/docs/source/resources/cloud_integration.md index be268ca1b31..e8744fd41d3 100644 --- a/docs/source/resources/cloud_integration.md +++ b/docs/source/resources/cloud_integration.md @@ -59,8 +59,9 @@ To run NVTabular on the cloud using GCP, do the following: * **Boot Disk**: Ubuntu version 18.04 * **Storage**: Local 8xSSD-NVMe -2. [Install the appropriate NVIDIA drivers and CUDA](https://cloud.google.com/compute/docs/gpus/install-drivers-gpu#ubuntu-driver-steps) by running the following commands: - ``` +2. Install the NVIDIA drivers and CUDA by running the following commands: + + ```shell curl -O https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/cuda-ubuntu1804.pin sudo mv cuda-ubuntu1804.pin /etc/apt/preferences.d/cuda-repository-pin-600 sudo apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/7fa2af80.pub @@ -70,8 +71,12 @@ To run NVTabular on the cloud using GCP, do the following: nvidia-smi # Check installation ``` + > For more information, refer to [Install GPU drivers](https://cloud.google.com/compute/docs/gpus/install-drivers-gpu) + > in the Google Cloud documentation. + 3. [Install Docker](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html) by running the following commands: - ``` + + ```shell distribution=$(. /etc/os-release;echo $ID$VERSION_ID) \ && curl -s -L https://nvidia-merlin.github.io/nvidia-docker/gpgkey | sudo apt-key add - \ && curl -s -L https://nvidia-merlin.github.io/nvidia-docker/$distribution/nvidia-docker.list | sudo tee /etc/apt/sources.list.d/nvidia-docker.list @@ -82,7 +87,8 @@ To run NVTabular on the cloud using GCP, do the following: ``` 4. Configure the storage as RAID 0 by running the following commands: - ``` + + ```shell sudo mdadm --create --verbose /dev/md0 --level=0 --name=MY_RAID --raid-devices=2 /dev/nvme0n1 /dev/nvme0n2 sudo mkfs.ext4 -L MY_RAID /dev/md0 sudo mkdir -p /mnt/raid @@ -94,7 +100,8 @@ To run NVTabular on the cloud using GCP, do the following: ``` 5. Run the container by running the following command: - ``` + + ```shell docker run --gpus all --rm -it -p 8888:8888 -p 8797:8787 -p 8796:8786 --ipc=host --cap-add SYS_PTRACE -v /mnt/raid:/raid nvcr.io/nvidia/nvtabular:0.3 /bin/bash ``` @@ -179,12 +186,12 @@ conda activate nvtabular 8. Install additional packages, such as TensorFlow or PyTorch ``` -pip install tensorflow-gpu +pip install tensorflow-gpu pip install torch pip install graphviz ``` -9. Install Transformer4Rec, torchmetrics and ipykernel +9. Install Transformer4Rec, torchmetrics and ipykernel ``` conda install -y -c nvidia -c rapidsai -c numba -c conda-forge transformers4rec @@ -197,6 +204,6 @@ conda install -y torchmetrics ipykernel python -m ipykernel install --user --name=nvtabular ``` -11. You can switch in jupyter lab and run the [movielens example](https://github.com/NVIDIA-Merlin/NVTabular/tree/main/examples/getting-started-movielens). +11. You can switch in jupyter lab and run the [movielens example](https://github.com/NVIDIA-Merlin/NVTabular/tree/main/examples/getting-started-movielens). This workflow enables NVTabular ETL and training with TensorFlow or Pytorch. Deployment with Triton Inference Server will follow soon. diff --git a/docs/source/resources/links.md b/docs/source/resources/links.md index c6f921a813f..e843c1ff97c 100644 --- a/docs/source/resources/links.md +++ b/docs/source/resources/links.md @@ -16,7 +16,7 @@ Talks Blog posts ---------- -We frequently post updates on [our blog](https://medium.com/nvidia-merlin) and on the [NVIDIA Developer News](https://news.developer.nvidia.com/tag/recommendation-systems/). +We frequently post updates on [our blog](https://medium.com/nvidia-merlin) and on the [NVIDIA Developer Technical Blog](https://developer.nvidia.com/blog?r=1&tags=&categories=recommendation-systems). Some highlights: diff --git a/docs/source/toc.yaml b/docs/source/toc.yaml index 196b5c5b2ae..74ac6a7279a 100644 --- a/docs/source/toc.yaml +++ b/docs/source/toc.yaml @@ -8,43 +8,13 @@ subtrees: - file: training/index.rst - file: examples/index.md title: Example Notebooks - subtrees: - - entries: - - file: examples/getting-started-movielens/index.md - title: Getting Started with MovieLens - entries: - - file: examples/getting-started-movielens/01-Download-Convert.ipynb - title: Download and Convert - - file: examples/getting-started-movielens/02-ETL-with-NVTabular.ipynb - title: ETL with NVTabular - - file: examples/getting-started-movielens/03-Training-with-HugeCTR.ipynb - title: Train with HugeCTR - - file: examples/getting-started-movielens/03-Training-with-TF.ipynb - title: Train with TensorFlow - - file: examples/getting-started-movielens/03-Training-with-PyTorch.ipynb - title: Train with PyTorch - - file: examples/getting-started-movielens/04-Triton-Inference-with-HugeCTR.ipynb - title: Serve a HugeCTR Model - - file: examples/getting-started-movielens/04-Triton-Inference-with-TF.ipynb - title: Serve a TensorFlow Model - - file: examples/scaling-criteo/index.md - entries: - - file: examples/scaling-criteo/01-Download-Convert.ipynb - title: Download and Convert - - file: examples/scaling-criteo/02-ETL-with-NVTabular.ipynb - title: ETL with NVTabular - - file: examples/scaling-criteo/03-Training-with-HugeCTR.ipynb - title: Train with HugeCTR - - file: examples/scaling-criteo/03-Training-with-TF.ipynb - title: Train with TensorFlow - - file: examples/scaling-criteo/04-Triton-Inference-with-HugeCTR.ipynb - title: Serve a HugeCTR Model - - file: examples/scaling-criteo/04-Triton-Inference-with-TF.ipynb - title: Serve a TensorFlow Model - - file: examples/multi-gpu-movielens/index.md - entries: - - file: examples/multi-gpu-movielens/01-03-MultiGPU-Download-Convert-ETL-with-NVTabular-Training-with-TensorFlow.ipynb - - file: examples/multi-gpu-toy-example/multi-gpu_dask.ipynb + entries: + - file: examples/01-Getting-started.ipynb + title: Getting Started with NVTabular + - file: examples/02-Advanced-NVTabular-workflow.ipynb + title: Advanced NVTabular Workflow + - file: examples/03-Running-on-multiple-GPUs-or-on-CPU.ipynb + title: Run on multi-GPU or CPU-only - file: api title: API Documentation - file: resources/index diff --git a/docs/source/training/hugectr.rst b/docs/source/training/hugectr.rst index dad4a6a118f..5674b5b1bec 100644 --- a/docs/source/training/hugectr.rst +++ b/docs/source/training/hugectr.rst @@ -2,18 +2,18 @@ Accelerated Training with HugeCTR ================================= A real-world production model serves hundreds of millions of users, -which contains embedding tables with up to 100GB to 1TB in size. Training deep +which contains embedding tables with up to 100GB to 1TB in size. Training deep learning recommender system models with such large embedding tables can be challenging as they do not fit into the memory of a single GPU. -To combat that challenge, we’ve developed HugeCTR, which is an open-source deep learning framework that is a highly optimized library +To combat that challenge, we developed HugeCTR, which is an open-source deep learning framework that is a highly optimized library written in CUDA C++, specifically for recommender systems. It supports an optimized dataloader and is able to scale embedding tables using -multiple GPUs and nodes. As a result, there’s no embedding table size +multiple GPUs and nodes. As a result, there is no embedding table size limitation. HugeCTR also offers the following: - Model oversubscription for training embedding tables with - single nodes that don’t fit within the GPU or CPU memory (only + single nodes that don't fit within the GPU or CPU memory (only required embeddings are prefetched from a parameter server per batch). - Asynchronous and multithreaded data pipelines. @@ -126,6 +126,5 @@ When training is accelerated with HugeCTR, the following happens: metrics = sess.evaluation() print("[HUGECTR][INFO] iter: {}, {}".format(i, metrics)) -Additional examples can be found `here`_. - -.. _here: https://github.com/NVIDIA/NVTabular/tree/main/examples/hugectr +For more information, refer to the `HugeCTR documentation `_ +or the `HugeCTR repository `_ on GitHub. diff --git a/docs/source/training/pytorch.rst b/docs/source/training/pytorch.rst index 06ccd8a7452..f6023e5d1e7 100644 --- a/docs/source/training/pytorch.rst +++ b/docs/source/training/pytorch.rst @@ -9,7 +9,7 @@ PyTorch. The NVTabular dataloader is capable of: - removing bottlenecks from dataloading by processing large chunks of data at a time instead of item by item -- processing datasets that don’t fit within the GPU or CPU memory by +- processing datasets that don't fit within the GPU or CPU memory by streaming from the disk - reading data directly into the GPU memory and removing CPU-GPU communication @@ -42,9 +42,9 @@ happens: TRAIN_PATHS = glob.glob("./train/*.parquet") train_dataset = TorchAsyncItr( - nvt.Dataset(TRAIN_PATHS), - cats=CATEGORICAL_COLUMNS, - conts=CONTINUOUS_COLUMNS, + nvt.Dataset(TRAIN_PATHS), + cats=CATEGORICAL_COLUMNS, + conts=CONTINUOUS_COLUMNS, labels=LABEL_COLUMNS, batch_size=BATCH_SIZE ) @@ -54,10 +54,10 @@ happens: .. code:: python train_loader = DLDataLoader( - train_dataset, - batch_size=None, - collate_fn=collate_fn, - pin_memory=False, + train_dataset, + batch_size=None, + collate_fn=collate_fn, + pin_memory=False, num_workers=0 ) @@ -79,8 +79,6 @@ happens: 5. The ``TorchAsyncItr`` dataloader can be initialized for the validation dataset using the same structure. -You can find additional examples in our repository such as `MovieLens`_ -and `Criteo`_. +You can find additional `examples`_ in our repository. -.. _MovieLens: ../examples/getting-started-movielens/ -.. _Criteo: ../examples/scaling-criteo/ +.. _examples: ../examples/ diff --git a/docs/source/training/tensorflow.rst b/docs/source/training/tensorflow.rst index c18f571b2fb..4faa982ad17 100644 --- a/docs/source/training/tensorflow.rst +++ b/docs/source/training/tensorflow.rst @@ -100,7 +100,7 @@ following happens: dataloader. .. code:: python - + history = model.fit(train_dataset_tf, epochs=5) **Note**: If using the NVTabular dataloader for the validation dataset, @@ -112,5 +112,6 @@ a callback can be used for it. validation_callback = KerasSequenceValidater(valid_dataset_tf) history = model.fit(train_dataset_tf, callbacks=[validation_callback], epochs=5) -You can find additional examples in our repository such as -`MovieLens <../examples/getting-started-movielens/>`__. +You can find additional `examples`_ in our repository. + +.. _examples: ../examples/ diff --git a/examples/01-Getting-started.ipynb b/examples/01-Getting-started.ipynb index 514a09a15f5..a11cc8cade0 100644 --- a/examples/01-Getting-started.ipynb +++ b/examples/01-Getting-started.ipynb @@ -58,7 +58,7 @@ "id": "1c5598ae", "metadata": {}, "source": [ - "# Downloading the dataset\n", + "## Downloading the dataset\n", "\n", "### MovieLens25M\n", "\n", @@ -240,7 +240,7 @@ "id": "03b3152e", "metadata": {}, "source": [ - "# Processing the dataset with NVTabular" + "## Processing the dataset with NVTabular" ] }, { @@ -248,7 +248,7 @@ "id": "2ee5c7c2", "metadata": {}, "source": [ - "## Defining the workflow" + "### Defining the workflow" ] }, { @@ -331,63 +331,7 @@ "outputs": [ { "data": { - "image/svg+xml": [ - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "%3\n", - "\n", - "\n", - "\n", - "0\n", - "\n", - "Categorify\n", - "\n", - "\n", - "\n", - "2\n", - "\n", - "output cols\n", - "\n", - "\n", - "\n", - "0->2\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "1\n", - "\n", - "SelectionOp\n", - "\n", - "\n", - "\n", - "1->0\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "1_selector\n", - "\n", - "['userId', 'movieId']\n", - "\n", - "\n", - "\n", - "1_selector->1\n", - "\n", - "\n", - "\n", - "\n", - "\n" - ], + "image/svg+xml": "\n\n\n\n\n\n%3\n\n\n\n0\n\nCategorify\n\n\n\n2\n\noutput cols\n\n\n\n0->2\n\n\n\n\n\n1\n\nSelectionOp\n\n\n\n1->0\n\n\n\n\n\n1_selector\n\n['userId', 'movieId']\n\n\n\n1_selector->1\n\n\n\n\n\n", "text/plain": [ "" ] @@ -410,7 +354,7 @@ "\n", "Additionally, we tag the `rating` column with appropriate tags. This will allow other components of the Merlin Framework to use this information and minimize the code we will have to write to perform complex operations such as training or serving a Deep Learning model.\n", "\n", - "If you would like to learn more about using `Tags`, please take a look at [this notebook](https://github.com/NVIDIA-Merlin/models/blob/main/examples/02-Merlin-Models-and-NVTabular-integration.ipynb)." + "If you would like to learn more about using `Tags`, take a look at the [NVTabular and Merlin Models integrated example](https://nvidia-merlin.github.io/models/main/examples/02-Merlin-Models-and-NVTabular-integration.html) notebook in the Merlin Models [repository](https://github.com/NVIDIA-Merlin/models)." ] }, { @@ -446,7 +390,7 @@ "id": "f3ce8958", "metadata": {}, "source": [ - "## Applying the workflow to the train and validation sets" + "### Applying the workflow to the train and validation sets" ] }, { @@ -620,7 +564,7 @@ "source": [ "Let's finish off this notebook with training a DLRM (a Deep Learning Recommendation Model introduced in [Deep Learning Recommendation Model for Personalization and Recommendation Systems](https://arxiv.org/abs/1906.00091)) on our preprocessed data.\n", "\n", - "To learn more about the integration between NVTabular and Merlin Models, please see this [example](https://github.com/NVIDIA-Merlin/models/blob/main/examples/02-Merlin-Models-and-NVTabular-integration.ipynb) in the Merlin Models [repository](https://github.com/NVIDIA-Merlin/models)." + "To learn more about the integration between NVTabular and Merlin Models, please see the [NVTabular and Merlin Models integrated example](https://nvidia-merlin.github.io/models/main/examples/02-Merlin-Models-and-NVTabular-integration.html) in the Merlin Models [repository](https://github.com/NVIDIA-Merlin/models)." ] }, { @@ -628,7 +572,7 @@ "id": "a6ff4c40", "metadata": {}, "source": [ - "# Training a DLRM model" + "## Training a DLRM model" ] }, { @@ -636,7 +580,7 @@ "id": "688b89c7", "metadata": {}, "source": [ - "We define the DLRM model, whose prediction task is a binary classification. From the `schema`, the categorical features are identified (and embedded) and the target column is also automatically inferred, because of the schema tags. We talk more about the schema in the next [example notebook (02)](02-Merlin-Models-and-NVTabular-integration.ipynb)," + "We define the DLRM model, whose prediction task is a binary classification. From the `schema`, the categorical features are identified (and embedded) and the target column is also automatically inferred, because of the schema tags. We talk more about the schema in the next example notebook, [Advanced NVTabular Workflow](02-Advanced-NVTabular-workflow.ipynb)." ] }, { @@ -703,6 +647,14 @@ "metrics = model.fit(train_transformed, validation_data=valid_transformed, batch_size=1024, epochs=3)" ] }, + { + "cell_type": "code", + "execution_count": null, + "id": "4c8353a0", + "metadata": {}, + "outputs": [], + "source": [] + }, { "cell_type": "markdown", "id": "2a6ad327", @@ -738,7 +690,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "Python 3.8.13 64-bit ('3.8.13')", "language": "python", "name": "python3" }, @@ -752,12 +704,17 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.10" + "version": "3.8.13" }, "merlin": { "containers": [ "nvcr.io/nvidia/merlin/merlin-tensorflow:latest" ] + }, + "vscode": { + "interpreter": { + "hash": "5278529888a7d71bb985f02ff9083b63772563f3bf182683e4d2f66c9c40ed1c" + } } }, "nbformat": 4, diff --git a/examples/02-Advanced-NVTabular-workflow.ipynb b/examples/02-Advanced-NVTabular-workflow.ipynb index 3a8c08d6224..bd3fa651cbe 100644 --- a/examples/02-Advanced-NVTabular-workflow.ipynb +++ b/examples/02-Advanced-NVTabular-workflow.ipynb @@ -30,7 +30,7 @@ "source": [ "\n", "\n", - "# Advanced NVTabular workflow\n", + "# Advanced NVTabular Workflow\n", "\n", "This notebook is created using the latest stable [merlin-tensorflow](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/merlin/containers/merlin-tensorflow/tags) container. \n", "\n", @@ -1065,7 +1065,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "Python 3.8.13 64-bit ('3.8.13')", "language": "python", "name": "python3" }, @@ -1079,12 +1079,17 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.10" + "version": "3.8.13" }, "merlin": { "containers": [ "nvcr.io/nvidia/merlin/merlin-tensorflow:latest" ] + }, + "vscode": { + "interpreter": { + "hash": "5278529888a7d71bb985f02ff9083b63772563f3bf182683e4d2f66c9c40ed1c" + } } }, "nbformat": 4, diff --git a/examples/03-Running-on-multiple-GPUs-or-on-CPU.ipynb b/examples/03-Running-on-multiple-GPUs-or-on-CPU.ipynb index 47ca7fd7c47..8764544c8ce 100644 --- a/examples/03-Running-on-multiple-GPUs-or-on-CPU.ipynb +++ b/examples/03-Running-on-multiple-GPUs-or-on-CPU.ipynb @@ -54,7 +54,7 @@ "id": "1c5598ae", "metadata": {}, "source": [ - "# Downloading the dataset" + "## Downloading the dataset" ] }, { @@ -93,7 +93,7 @@ "id": "63ac0cf2", "metadata": {}, "source": [ - "# Running on multiple-GPUs" + "## Running on multiple-GPUs" ] }, { @@ -103,7 +103,7 @@ "source": [ "### Multi-GPU and multi-node scaling\n", "\n", - "NVTabular is built on top off [RAPIDS.AI cuDF](https://github.com/rapidsai/cudf/), [dask_cudf](https://docs.rapids.ai/api/cudf/stable/dask-cudf.html) and [dask](https://dask.org/).

\n", + "NVTabular is built on top off [RAPIDS.AI cuDF](https://github.com/rapidsai/cudf/), [dask_cudf](https://docs.rapids.ai/api/cudf/stable/) and [dask](https://dask.org/).

\n", "**Dask** is a task-based library for parallel scheduling and execution. Although it is certainly possible to use the task-scheduling machinery directly to implement customized parallel workflows (we do it in NVTabular), most users only interact with Dask through a Dask Collection API. The most popular \"collection\" API's include:\n", "\n", "* Dask DataFrame: Dask-based version of the Pandas DataFrame/Series API. Note that dask_cudf is just a wrapper around this collection module (dask.dataframe).\n", @@ -672,7 +672,7 @@ "id": "01ea40bb", "metadata": {}, "source": [ - "# Running on CPU" + "## Running on CPU" ] }, { @@ -749,7 +749,7 @@ "id": "4e07864d", "metadata": {}, "source": [ - "# Summary" + "## Summary" ] }, { @@ -763,7 +763,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "Python 3.8.13 64-bit ('3.8.13')", "language": "python", "name": "python3" }, @@ -777,12 +777,17 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.10" + "version": "3.8.13" }, "merlin": { "containers": [ "nvcr.io/nvidia/merlin/merlin-tensorflow:latest" ] + }, + "vscode": { + "interpreter": { + "hash": "5278529888a7d71bb985f02ff9083b63772563f3bf182683e4d2f66c9c40ed1c" + } } }, "nbformat": 4, diff --git a/examples/README.md b/examples/README.md index a11dc66a792..6b9168b3655 100644 --- a/examples/README.md +++ b/examples/README.md @@ -1,55 +1,23 @@ # NVTabular Example Notebooks -We have a collection of Jupyter notebooks that are based on different datasets. -These example notebooks demonstrate how to use NVTabular with TensorFlow, PyTorch, and [HugeCTR](https://github.com/NVIDIA/HugeCTR). -Each example provides additional information about NVTabular's features. - -If you'd like to create a full conda environment to run the example notebooks, do the following: - -1. Use the [environment files](https://github.com/NVIDIA/NVTabular/tree/main/conda/environments) that have been provided to install the CUDA Toolkit (11.0 or 11.2). -2. Clone the NVTabular repo and run the following commands from the root directory: - ```bash - conda env create -f=conda/environments/nvtabular_dev_cuda11.2.yml - conda activate nvtabular_dev_11.2 - python -m ipykernel install --user --name=nvt - pip install -e . - jupyter notebook - ``` - When opening a notebook, be sure to select `nvt` from the `Kernel->Change Kernel` menu. - -## Structure - -The example notebooks are structured as follows and should be reviewed in this order: - -- 01-Download-Convert.ipynb: Demonstrates how to download the dataset and convert it into the correct format so that it can be consumed. -- 02-ETL-with-NVTabular.ipynb: Demonstrates how to execute the preprocessing and feature engineering pipeline (ETL) with NVTabular on the GPU. -- 03-Training-with-TF.ipynb: Demonstrates how to train a model with TensorFlow based on the ETL output. -- 03-Training-with-PyTorch.ipynb: Demonstrates how to train a model with PyTorch based on the ETL output. -- 03-Training-with-HugeCTR.ipynb: Demonstrates how to train a model with HugeCTR based on the ETL output. - -## Available Example Notebooks - -### 1. [Getting Started with MovieLens](https://github.com/NVIDIA/NVTabular/tree/main/examples/getting-started-movielens) +In this library, we provide a collection of Jupyter notebooks, which demonstrates the functionality of NVTabular. -The MovieLens25M is a popular dataset for recommender systems and is used in academic publications. Most users are familiar with this dataset, so this example notebook is focusing primarily on the basic concepts of NVTabular, which includes: +## Inventory -- Learning NVTabular with NVTabular's high-level API -- Using single-hot/multi-hot categorical input features with NVTabular -- Using the NVTabular dataloader with the TensorFlow Keras model -- Using the NVTabular dataloader with PyTorch +- [Getting Started with NVTabular](01-Getting-started.ipynb): Get started with NVTabular by processing data on the GPU. -### 2. [Scaling Large Datasets with Criteo](https://github.com/NVIDIA/NVTabular/tree/main/examples/scaling-criteo) +- [Advanced NVTabular workflow](02-Advanced-NVTabular-workflow.ipynb): Understand NVTabular in more detail by defining more advanced workflows and learn about different operators -[Criteo](https://ailab.criteo.com/download-criteo-1tb-click-logs-dataset/) provides the largest publicly available dataset for recommender systems with a size of 1TB of uncompressed click logs that contain 4 billion examples. This example notebook demonstrates how to scale NVTabular, use multiple GPUs and multiple nodes with NVTabular for ETL, and train a recommender system model with the NVTabular dataloader for PyTorch. +- [Running on multiple GPUs or on CPU](03-Running-on-multiple-GPUs-or-on-CPU.ipynb): Run NVTabular in different environments, such as multi-GPU or CPU-only mode. -### 3. [Multi-GPU with MovieLens](https://github.com/NVIDIA/NVTabular/tree/main/examples/multi-gpu-movielens) - -In the Getting Started with MovieLens example, we explain the fundamentals of NVTabular and its dataloader, HugeCTR, and Triton Inference. With this example, we revisit the same dataset but demonstrate how to perform multi-GPU training with the NVTabular dataloader in TensorFlow. +In addition, NVTabular is used in many of our examples in other Merlin libraries. You can explore more complex processing pipelines in following examples: +- [End-To-End Examples with Merlin](https://github.com/NVIDIA-Merlin/Merlin/tree/main/examples) +- [Training Examples with Merlin Models](https://github.com/NVIDIA-Merlin/models/tree/main/examples) +- [Training Examples with Transformer4Rec](https://github.com/NVIDIA-Merlin/Transformers4Rec/tree/main/examples) ## Running the Example Notebooks -You can run the example notebooks by [installing NVTabular](https://github.com/NVIDIA/NVTabular#installation) and other required libraries. -Alternatively, Docker containers are available from the NVIDIA GPU Cloud (NGC) at with pre-installed versions. +You can run the example notebooks by [installing NVTabular](https://github.com/NVIDIA/NVTabular#installation) and other required libraries. Alternatively, Docker containers are available from the NVIDIA GPU Cloud (NGC) at with pre-installed versions. Depending on which example you want to run, you should use any one of these Docker containers: - `merlin-hugectr` (contains NVTabular with HugeCTR) @@ -58,29 +26,24 @@ Depending on which example you want to run, you should use any one of these Dock Beginning with the 22.06 release, each container includes the software for training models and performing inference. -To run the example notebooks using Docker containers, do the following: +To run the example notebooks using Docker containers, perform the following steps: -1. Pull the container by running the following command: +1. Pull and start the container by running the following command: - ```sh - docker run --gpus all --rm -it -p 8888:8888 -p 8797:8787 -p 8796:8786 --ipc=host /bin/bash + ```shell + docker run --gpus all --rm -it \ + -p 8888:8888 -p 8797:8787 -p 8796:8786 --ipc=host \ + /bin/bash ``` - **NOTES**: - - - If you are running Getting Started with MovieLens, Advanced Ops with Outbrain, or the Tabular Problems with Rossmann example notebooks, add a `-v ${PWD}:/root/` argument to the preceding Docker command. - The `PWD` environment variable refers to a local directory on your computer, and you should specify this same directory and with the `-v` argument when you run a container to perform inference. - Follow the instructions for starting Triton Inference Server that are provided in the inference notebooks. - - If you are running `Training-with-HugeCTR` notebooks, please add `--cap-add SYS_NICE` to the `docker run` command to suppress the `set_mempolicy: Operation not permitted` warnings. - The container opens a shell when the run command execution is completed. Your shell prompt should look similar to the following example: - ```sh + ```shell root@2efa5b50b909: ``` -1. Start the jupyter-lab server by running the following command: +1. Start the JupyterLab server by running the following command: ```shell jupyter-lab --allow-root --ip='0.0.0.0' diff --git a/examples/getting-started-movielens/01-Download-Convert.ipynb b/examples/getting-started-movielens/01-Download-Convert.ipynb deleted file mode 100644 index 32f46d0ac68..00000000000 --- a/examples/getting-started-movielens/01-Download-Convert.ipynb +++ /dev/null @@ -1,488 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Copyright 2021 NVIDIA Corporation. All Rights Reserved.\n", - "#\n", - "# Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# http://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License.\n", - "# ==============================================================================" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "# Getting Started MovieLens: Download and Convert\n", - "\n", - "## MovieLens25M\n", - "\n", - "The [MovieLens25M](https://grouplens.org/datasets/movielens/25m/) is a popular dataset for recommender systems and is used in academic publications. The dataset contains 25M movie ratings for 62,000 movies given by 162,000 users. Many projects use only the user/item/rating information of MovieLens, but the original dataset provides metadata for the movies, as well. For example, which genres a movie has. Although we may not improve state-of-the-art results with our neural network architecture in this example, we will use the metadata to show how to multi-hot encode the categorical features." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Download the dataset" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "# External dependencies\n", - "import os\n", - "\n", - "from merlin.core.utils import download_file\n", - "\n", - "# Get dataframe library - cudf or pandas\n", - "from merlin.core.dispatch import get_lib\n", - "df_lib = get_lib()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We define our base input directory, containing the data." - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "INPUT_DATA_DIR = os.environ.get(\n", - " \"INPUT_DATA_DIR\", os.path.expanduser(\"~/nvt-examples/movielens/data/\")\n", - ")\n", - "OUTPUT_DATA_DIR = os.environ.get(\n", - " \"OUTPUT_DATA_DIR\", os.path.expanduser(\"~/nvt-examples/movielens/data/\")\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We will download and unzip the data." - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "downloading ml-25m.zip: 262MB [00:06, 42.1MB/s] \n", - "unzipping files: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 8/8 [00:04<00:00, 1.74files/s]\n" - ] - } - ], - "source": [ - "download_file(\n", - " \"http://files.grouplens.org/datasets/movielens/ml-25m.zip\",\n", - " os.path.join(INPUT_DATA_DIR, \"ml-25m.zip\"),\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Convert the dataset" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "First, we take a look on the movie metadata. " - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
movieIdtitlegenres
01Toy Story (1995)Adventure|Animation|Children|Comedy|Fantasy
12Jumanji (1995)Adventure|Children|Fantasy
23Grumpier Old Men (1995)Comedy|Romance
34Waiting to Exhale (1995)Comedy|Drama|Romance
45Father of the Bride Part II (1995)Comedy
\n", - "
" - ], - "text/plain": [ - " movieId title \\\n", - "0 1 Toy Story (1995) \n", - "1 2 Jumanji (1995) \n", - "2 3 Grumpier Old Men (1995) \n", - "3 4 Waiting to Exhale (1995) \n", - "4 5 Father of the Bride Part II (1995) \n", - "\n", - " genres \n", - "0 Adventure|Animation|Children|Comedy|Fantasy \n", - "1 Adventure|Children|Fantasy \n", - "2 Comedy|Romance \n", - "3 Comedy|Drama|Romance \n", - "4 Comedy " - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "movies = df_lib.read_csv(os.path.join(INPUT_DATA_DIR, \"movies.csv\"))\n", - "movies.head()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We can see, that genres are a multi-hot categorical features with different number of genres per movie. Currently, genres is a String and we want split the String into a list of Strings. In addition, we drop the title." - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
movieIdgenres
01[Adventure, Animation, Children, Comedy, Fantasy]
12[Adventure, Children, Fantasy]
23[Comedy, Romance]
34[Comedy, Drama, Romance]
45[Comedy]
\n", - "
" - ], - "text/plain": [ - " movieId genres\n", - "0 1 [Adventure, Animation, Children, Comedy, Fantasy]\n", - "1 2 [Adventure, Children, Fantasy]\n", - "2 3 [Comedy, Romance]\n", - "3 4 [Comedy, Drama, Romance]\n", - "4 5 [Comedy]" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "movies[\"genres\"] = movies[\"genres\"].str.split(\"|\")\n", - "movies = movies.drop(\"title\", axis=1)\n", - "movies.head()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We save movies genres in parquet format, so that they can be used by NVTabular in the next notebook." - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [], - "source": [ - "movies.to_parquet(os.path.join(OUTPUT_DATA_DIR, \"movies_converted.parquet\"))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Splitting into train and validation dataset" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We load the movie ratings." - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
userIdmovieIdratingtimestamp
012965.01147880044
113063.51147868817
213075.01147868828
316655.01147878820
418993.51147868510
\n", - "
" - ], - "text/plain": [ - " userId movieId rating timestamp\n", - "0 1 296 5.0 1147880044\n", - "1 1 306 3.5 1147868817\n", - "2 1 307 5.0 1147868828\n", - "3 1 665 5.0 1147878820\n", - "4 1 899 3.5 1147868510" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ratings = df_lib.read_csv(os.path.join(INPUT_DATA_DIR, \"ratings.csv\"))\n", - "ratings.head()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We drop the timestamp column and split the ratings into training and test datasets. We use a simple random split." - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [], - "source": [ - "ratings = ratings.drop(\"timestamp\", axis=1)\n", - "\n", - "# shuffle the dataset\n", - "ratings = ratings.sample(len(ratings), replace=False)\n", - "\n", - "# split the train_df as training and validation data sets.\n", - "num_valid = int(len(ratings) * 0.2)\n", - "\n", - "train = ratings[:-num_valid]\n", - "valid = ratings[-num_valid:]" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We save the dataset to disk." - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [], - "source": [ - "train.to_parquet(os.path.join(OUTPUT_DATA_DIR, \"train.parquet\"))\n", - "valid.to_parquet(os.path.join(OUTPUT_DATA_DIR, \"valid.parquet\"))" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.10" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/examples/getting-started-movielens/02-ETL-with-NVTabular.ipynb b/examples/getting-started-movielens/02-ETL-with-NVTabular.ipynb deleted file mode 100644 index a5b8dc1915b..00000000000 --- a/examples/getting-started-movielens/02-ETL-with-NVTabular.ipynb +++ /dev/null @@ -1,701 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Copyright 2021 NVIDIA Corporation. All Rights Reserved.\n", - "#\n", - "# Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# http://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License.\n", - "# ==============================================================================" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "# Getting Started MovieLens: ETL with NVTabular\n", - "\n", - "## Overview\n", - "\n", - "NVTabular is a feature engineering and preprocessing library for tabular data designed to quickly and easily manipulate terabyte scale datasets used to train deep learning based recommender systems. It provides a high level abstraction to simplify code and accelerates computation on the GPU using the RAPIDS cuDF library.

\n", - "\n", - "Deep Learning models require the input feature in a specific format. Categorical features needs to be continuous integers (0, ..., |C|) to use them with an embedding layer. We will use NVTabular to preprocess the categorical features.

\n", - "\n", - "One other challenge is multi-hot categorical features. A product can have multiple categories assigned, but the number of categories per product varies. For example, a movie can have one or multiple genres:\n", - "\n", - "- Father of the Bride Part II: \\[Comedy\\]\n", - "- Toy Story: \\[Adventure, Animation, Children, Comedy, Fantasy\\]\n", - "- Jumanji: \\[Adventure, Children, Fantasy\\]\n", - "\n", - "One strategy is often to use only the first category or the most frequent ones. However, a better strategy is to use all provided categories per datapoint. [RAPID cuDF](https://github.com/rapidsai/cudf) added list support in its [latest release v0.16](https://medium.com/rapids-ai/two-years-in-a-snap-rapids-0-16-ae797795a5c4) and NVTabular now supports multi-hot categorical features.

\n", - "\n", - "### Learning objectives\n", - "\n", - "In this notebook, we learn how to `Categorify` single-hot and multi-hot categorical input features with NVTabular\n", - "\n", - "- Learn NVTabular for using GPU-accelerated ETL (Preprocess and Feature Engineering)\n", - "- Get familiar with NVTabular's high-level API\n", - "- Join two dataframes with `JoinExternal` operator\n", - "- Preprocess single-hot categorical input features with NVTabular\n", - "- Preprocess multi-hot categorical input features with NVTabular\n", - "- Use `LambdaOp` for custom row-wise dataframe manipulations with NVTabular" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### NVTabular\n", - "\n", - "With the rapid growth in scale of industry datasets, deep learning (DL) recommender models have started to gain advantages over traditional methods by capitalizing on large amounts of training data.\n", - "\n", - "The current challenges for training large-scale recommenders include:\n", - "\n", - "* **Huge datasets:** Commercial recommenders are trained on huge datasets, often several terabytes in scale.\n", - "* **Complex data preprocessing and feature engineering pipelines:** Datasets need to be preprocessed and transformed into a form relevant to be used with DL models and frameworks. In addition, feature engineering creates an extensive set of new features from existing ones, requiring multiple iterations to arrive at an optimal solution.\n", - "* **Input bottleneck:** Data loading, if not well optimized, can be the slowest part of the training process, leading to under-utilization of high-throughput computing devices such as GPUs.\n", - "* **Extensive repeated experimentation:** The whole data engineering, training, and evaluation process is generally repeated many times, requiring significant time and computational resources.\n", - "\n", - "**NVTabular** is a library for fast tabular data transformation and loading, manipulating terabyte-scale datasets quickly. It provides best practices for feature engineering and preprocessing and a high-level abstraction to simplify code accelerating computation on the GPU using the RAPIDS cuDF library.\n", - "\n", - "\n", - "\n", - "### Why use NVTabular?\n", - "\n", - "NVTabular offers multiple advantages to support your Feature Engineering and Preprocessing:\n", - "\n", - "1. **Larger than memory datasets**: Your dataset size can be larger than host/GPU memory. NVTabular reads the data from disk and stores the processed files to disk. It will execute your pipeline without exceeding the memory boundaries.\n", - "2. **Speed**: NVTabular will execute your pipeline on GPU. We experienced 10x-100x speed-up\n", - "3. **Easy-to-use**: NVTabular implemented common feature engineering and preprocessing operators and provides high-level APIs ready to use" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## ETL with NVTabular" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "# External dependencies\n", - "import os\n", - "import shutil\n", - "import numpy as np\n", - "\n", - "import nvtabular as nvt\n", - "\n", - "from os import path\n", - "\n", - "# Get dataframe library - cudf or pandas\n", - "from merlin.core.dispatch import get_lib\n", - "df_lib = get_lib()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We define our base input directory, containing the data." - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "INPUT_DATA_DIR = os.environ.get(\n", - " \"INPUT_DATA_DIR\", os.path.expanduser(\"~/nvt-examples/movielens/data/\")\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
movieIdgenres
01[Adventure, Animation, Children, Comedy, Fantasy]
12[Adventure, Children, Fantasy]
23[Comedy, Romance]
34[Comedy, Drama, Romance]
45[Comedy]
\n", - "
" - ], - "text/plain": [ - " movieId genres\n", - "0 1 [Adventure, Animation, Children, Comedy, Fantasy]\n", - "1 2 [Adventure, Children, Fantasy]\n", - "2 3 [Comedy, Romance]\n", - "3 4 [Comedy, Drama, Romance]\n", - "4 5 [Comedy]" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "movies = df_lib.read_parquet(os.path.join(INPUT_DATA_DIR, \"movies_converted.parquet\"))\n", - "movies.head()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Defining our Preprocessing Pipeline\n", - "The first step is to define the feature engineering and preprocessing pipeline.

\n", - "NVTabular has already implemented multiple calculations, called `ops`. An `op` can be applied to a `ColumnGroup` from an overloaded `>>` operator, which in turn returns a new `ColumnGroup`. A `ColumnGroup` is a list of column names as text.

\n", - "**Example:**
\n", - "```python\n", - "features = [ column_name, ...] >> op1 >> op2 >> ...\n", - "```\n", - "\n", - "This may sounds more complicated as it is. Let's define our first pipeline for the MovieLens dataset." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Currently, our dataset consists of two separate dataframes. First, we use the `JoinExternal` operator to `left-join` the metadata (genres) to our rating dataset." - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [], - "source": [ - "CATEGORICAL_COLUMNS = [\"userId\", \"movieId\"]\n", - "LABEL_COLUMNS = [\"rating\"]" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [], - "source": [ - "joined = [\"userId\", \"movieId\"] >> nvt.ops.JoinExternal(movies, on=[\"movieId\"])" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Data pipelines are **Directed Acyclic Graphs (DAGs)**. We can visualize them with `graphviz`." - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "data": { - "image/svg+xml": "\n\n\n\n\n\n%3\n\n\n\n0\n\nJoinExternal\n\n\n\n2\n\noutput cols=[userId, movieId]\n\n\n\n0->2\n\n\n\n\n\n1\n\nSelectionOp\n\n\n\n1->0\n\n\n\n\n\n0_selector\n\n['userId', 'movieId']\n\n\n\n0_selector->0\n\n\n\n\n\n1_selector\n\n['userId', 'movieId']\n\n\n\n1_selector->1\n\n\n\n\n\n", - "text/plain": [ - "" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "joined.graph" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Embedding Layers of neural networks require that categorical features are contiguous, incremental Integers: 0, 1, 2, ... , |C|-1. We need to ensure that our categorical features fulfill the requirement.
\n", - "\n", - "Currently, our genres are a list of Strings. In addition, we should transform the single-hot categorical features userId and movieId, as well.
\n", - "NVTabular provides the operator `Categorify`, which provides this functionality with a high-level API out of the box. In NVTabular release v0.3, list support was added for multi-hot categorical features. Both works in the same way with no need for changes.\n", - "\n", - "\n", - "Next, we will add `Categorify` for our categorical features (single hot: userId, movieId and multi-hot: genres)." - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [], - "source": [ - "cat_features = joined >> nvt.ops.Categorify()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The ratings are on a scale between 1-5. We want to predict a binary target with 1 for ratings `>3` and 0 for ratings `<=3`. We use the [LambdaOp](https://nvidia-merlin.github.io/NVTabular/main/api/ops/lambdaop.html) for it." - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [], - "source": [ - "ratings = nvt.ColumnGroup([\"rating\"]) >> nvt.ops.LambdaOp(lambda col: (col > 3).astype(\"int8\"))" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [ - { - "data": { - "image/svg+xml": "\n\n\n\n\n\n%3\n\n\n\n0\n\nSelectionOp\n\n\n\n1\n\nJoinExternal\n\n\n\n0->1\n\n\n\n\n\n0_selector\n\n['userId', 'movieId']\n\n\n\n0_selector->0\n\n\n\n\n\n2\n\nCategorify\n\n\n\n1->2\n\n\n\n\n\n1_selector\n\n['userId', 'movieId']\n\n\n\n1_selector->1\n\n\n\n\n\n3\n\n+\n\n\n\n2->3\n\n\n\n\n\n6\n\noutput cols\n\n\n\n3->6\n\n\n\n\n\n5\n\nnvt.ops.LambdaOp(lambda col: (col > 3).astype("int8"))\n\n\n\n5->3\n\n\n\n\n\n4\n\nSelectionOp\n\n\n\n4->5\n\n\n\n\n\n4_selector\n\n['rating']\n\n\n\n4_selector->4\n\n\n\n\n\n5_selector\n\n['rating']\n\n\n\n5_selector->5\n\n\n\n\n\n", - "text/plain": [ - "" - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "output = cat_features + ratings\n", - "(output).graph" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We initialize our NVTabular `workflow`." - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [], - "source": [ - "workflow = nvt.Workflow(output)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Running the pipeline" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "In general, the `Op`s in our `Workflow` will require measurements of statistical properties of our data in order to be leveraged. For example, the `Normalize` op requires measurements of the dataset mean and standard deviation, and the `Categorify` op requires an accounting of all the categories a particular feature can manifest. However, we frequently need to measure these properties across datasets which are too large to fit into GPU memory (or CPU memory for that matter) at once.\n", - "\n", - "NVTabular solves this by providing the `Dataset` class, which breaks a set of parquet or csv files into into a collection of `cudf.DataFrame` chunks that can fit in device memory. The main purpose of this class is to abstract away the raw format of the data, and to allow other NVTabular classes to reliably materialize a dask_cudf.DataFrame collection (and/or collection-based iterator) on demand. Under the hood, the data decomposition corresponds to the construction of a [dask_cudf.DataFrame](https://docs.rapids.ai/api/cudf/stable/dask-cudf.html) object. By representing our dataset as a lazily-evaluated [Dask](https://dask.org/) collection, we can handle the calculation of complex global statistics (and later, can also iterate over the partitions while feeding data into a neural network). `part_size` defines the size read into GPU-memory at once." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now instantiate dataset iterators to loop through our dataset (which we couldn't fit into GPU memory). HugeCTR expect the categorical input columns as `int64` and continuous/label columns as `float32` We need to enforce the required HugeCTR data types, so we set them in a dictionary and give as an argument when creating our dataset." - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [], - "source": [ - "dict_dtypes = {}\n", - "\n", - "for col in CATEGORICAL_COLUMNS:\n", - " dict_dtypes[col] = np.int64\n", - "\n", - "for col in LABEL_COLUMNS:\n", - " dict_dtypes[col] = np.float32" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [], - "source": [ - "train_dataset = nvt.Dataset([os.path.join(INPUT_DATA_DIR, \"train.parquet\")])\n", - "valid_dataset = nvt.Dataset([os.path.join(INPUT_DATA_DIR, \"valid.parquet\")])" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now that we have our datasets, we'll apply our `Workflow` to them and save the results out to parquet files for fast reading at train time. Similar to the `scikit learn` API, we collect the statistics of our train dataset with `.fit`." - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 699 ms, sys: 593 ms, total: 1.29 s\n", - "Wall time: 1.45 s\n" - ] - }, - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 14, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "%%time\n", - "workflow.fit(train_dataset)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We clear our output directories." - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [], - "source": [ - "# Make sure we have a clean output path\n", - "if path.exists(os.path.join(INPUT_DATA_DIR, \"train\")):\n", - " shutil.rmtree(os.path.join(INPUT_DATA_DIR, \"train\"))\n", - "if path.exists(os.path.join(INPUT_DATA_DIR, \"valid\")):\n", - " shutil.rmtree(os.path.join(INPUT_DATA_DIR, \"valid\"))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We transform our workflow with `.transform`. We are going to add `'userId', 'movieId', 'genres'` columns to `_metadata.json`, because this json file will be needed for HugeCTR training to obtain the required information from all the rows in each parquet file." - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 3 µs, sys: 1 µs, total: 4 µs\n", - "Wall time: 8.82 µs\n" - ] - } - ], - "source": [ - "# Add \"write_hugectr_keyset=True\" to \"to_parquet\" if using this ETL Notebook for training with HugeCTR\n", - "%time\n", - "workflow.transform(train_dataset).to_parquet(\n", - " output_path=os.path.join(INPUT_DATA_DIR, \"train\"),\n", - " shuffle=nvt.io.Shuffle.PER_PARTITION,\n", - " cats=[\"userId\", \"movieId\", \"genres\"],\n", - " labels=[\"rating\"],\n", - " dtypes=dict_dtypes,\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 1 µs, sys: 1 µs, total: 2 µs\n", - "Wall time: 4.77 µs\n" - ] - } - ], - "source": [ - "# Add \"write_hugectr_keyset=True\" to \"to_parquet\" if using this ETL Notebook for training with HugeCTR\n", - "%time\n", - "workflow.transform(valid_dataset).to_parquet(\n", - " output_path=os.path.join(INPUT_DATA_DIR, \"valid\"),\n", - " shuffle=False,\n", - " cats=[\"userId\", \"movieId\", \"genres\"],\n", - " labels=[\"rating\"],\n", - " dtypes=dict_dtypes,\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We can take a look in the output dir." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "In the next notebooks, we will train a deep learning model. Our training pipeline requires information about the data schema to define the neural network architecture. We will save the NVTabular workflow to disk so that we can restore it in the next notebooks." - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "metadata": {}, - "outputs": [], - "source": [ - "workflow.save(os.path.join(INPUT_DATA_DIR, \"workflow\"))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Checking the pre-processing outputs" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We can take a look on the data." - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "(['/root/nvt-examples/movielens/data/train/part_0.parquet'],\n", - " ['/root/nvt-examples/movielens/data/valid/part_0.parquet'])" - ] - }, - "execution_count": 19, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "import glob\n", - "\n", - "TRAIN_PATHS = sorted(glob.glob(os.path.join(INPUT_DATA_DIR, \"train\", \"*.parquet\")))\n", - "VALID_PATHS = sorted(glob.glob(os.path.join(INPUT_DATA_DIR, \"valid\", \"*.parquet\")))\n", - "TRAIN_PATHS, VALID_PATHS" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We can see, that genres are a list of Integers" - ] - }, - { - "cell_type": "code", - "execution_count": 20, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
userIdmovieIdgenresrating
08439453[5, 8, 1]1.0
17658528[11, 7, 4]1.0
2334741093[2, 1]1.0
323873754[8, 12, 11, 4]0.0
4166559[3, 13, 2]0.0
\n", - "
" - ], - "text/plain": [ - " userId movieId genres rating\n", - "0 8439 453 [5, 8, 1] 1.0\n", - "1 76585 28 [11, 7, 4] 1.0\n", - "2 33474 1093 [2, 1] 1.0\n", - "3 2387 3754 [8, 12, 11, 4] 0.0\n", - "4 166 559 [3, 13, 2] 0.0" - ] - }, - "execution_count": 20, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "df = df_lib.read_parquet(TRAIN_PATHS[0])\n", - "df.head()" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.12" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/examples/getting-started-movielens/03-Training-with-HugeCTR.ipynb b/examples/getting-started-movielens/03-Training-with-HugeCTR.ipynb deleted file mode 100644 index 92e25bb4d8e..00000000000 --- a/examples/getting-started-movielens/03-Training-with-HugeCTR.ipynb +++ /dev/null @@ -1,683 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "id": "716038a8", - "metadata": {}, - "outputs": [], - "source": [ - "# Copyright 2021 NVIDIA Corporation. All Rights Reserved.\n", - "#\n", - "# Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# http://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License.\n", - "# ==============================================================================" - ] - }, - { - "cell_type": "markdown", - "id": "ce578729", - "metadata": {}, - "source": [ - "\n", - "\n", - "# Getting Started MovieLens: Training with HugeCTR\n", - "\n", - "In this notebook, we want to provide an overview what HugeCTR framework is, its features and benefits. We will use HugeCTR to train a basic neural network architecture.\n", - "\n", - "Learning Objectives:\n", - "* Adopt NVTabular workflow to provide input files to HugeCTR\n", - "* Define HugeCTR neural network architecture\n", - "* Train a deep learning model with HugeCTR" - ] - }, - { - "cell_type": "markdown", - "id": "2215198f", - "metadata": {}, - "source": [ - "### Why using HugeCTR?\n", - "\n", - "HugeCTR is a GPU-accelerated recommender framework designed to distribute training across multiple GPUs and nodes and estimate Click-Through Rates (CTRs).
\n", - "\n", - "HugeCTR offers multiple advantages to train deep learning recommender systems:\n", - "1. **Speed**: HugeCTR is a highly efficient framework written C++. We experienced up to 10x speed up. HugeCTR on a NVIDIA DGX A100 system proved to be the fastest commercially available solution for training the architecture Deep Learning Recommender Model (DLRM) developed by Facebook.\n", - "2. **Scale**: HugeCTR supports model parallel scaling. It distributes the large embedding tables over multiple GPUs or multiple nodes. \n", - "3. **Easy-to-use**: Easy-to-use Python API similar to Keras. Examples for popular deep learning recommender systems architectures (Wide&Deep, DLRM, DCN, DeepFM) are available." - ] - }, - { - "cell_type": "markdown", - "id": "5edfe68f", - "metadata": {}, - "source": [ - "### Other Features of HugeCTR\n", - "\n", - "HugeCTR is designed to scale deep learning models for recommender systems. It provides a list of other important features:\n", - "* Proficiency in oversubscribing models to train embedding tables with single nodes that don’t fit within the GPU or CPU memory (only required embeddings are prefetched from a parameter server per batch)\n", - "* Asynchronous and multithreaded data pipelines\n", - "* A highly optimized data loader.\n", - "* Supported data formats such as parquet and binary\n", - "* Integration with Triton Inference Server for deployment to production" - ] - }, - { - "cell_type": "markdown", - "id": "5d2d9c94", - "metadata": {}, - "source": [ - "### Getting Started" - ] - }, - { - "cell_type": "markdown", - "id": "a569fcf6", - "metadata": {}, - "source": [ - "In this example, we will train a neural network with HugeCTR. We will use preprocessed datasets generated via NVTabular in `02-ETL-with-NVTabular` notebook." - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "60d42722", - "metadata": {}, - "outputs": [], - "source": [ - "# External dependencies\n", - "import os\n", - "import nvtabular as nvt" - ] - }, - { - "cell_type": "markdown", - "id": "b180488d", - "metadata": {}, - "source": [ - "We define our base directory, containing the data." - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "4c9d11dd", - "metadata": {}, - "outputs": [], - "source": [ - "# path to preprocessed data\n", - "INPUT_DATA_DIR = os.environ.get(\n", - " \"INPUT_DATA_DIR\", os.path.expanduser(\"~/nvt-examples/movielens/data/\")\n", - ")\n", - "\n", - "# path to save the models\n", - "MODEL_BASE_DIR = os.environ.get(\"MODEL_BASE_DIR\", os.path.expanduser(\"~/nvt-examples/\"))" - ] - }, - { - "cell_type": "markdown", - "id": "d5a646d7", - "metadata": {}, - "source": [ - "Let's load our saved workflow from the `02-ETL-with-NVTabular` notebook." - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "id": "5fb19b6e", - "metadata": {}, - "outputs": [], - "source": [ - "workflow = nvt.Workflow.load(os.path.join(INPUT_DATA_DIR, \"workflow\"))" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "id": "1549fcf5", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'userId': dtype('int64'),\n", - " 'movieId': dtype('int64'),\n", - " 'genres': dtype('int64'),\n", - " 'rating': dtype('int8')}" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "workflow.output_dtypes" - ] - }, - { - "cell_type": "markdown", - "id": "fc61bf98", - "metadata": {}, - "source": [ - "Note: We do not have numerical output columns" - ] - }, - { - "cell_type": "markdown", - "id": "d036f265", - "metadata": {}, - "source": [ - "Let's clear existing directory and create the output folders." - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "id": "e3fd80fd", - "metadata": {}, - "outputs": [], - "source": [ - "MODEL_DIR = os.path.join(INPUT_DATA_DIR, \"model/movielens_hugectr/\")\n", - "!rm -rf {MODEL_DIR}\n", - "!mkdir -p {MODEL_DIR}\"1\"" - ] - }, - { - "cell_type": "markdown", - "id": "bef2934d", - "metadata": {}, - "source": [ - "## Scaling Accelerated training with HugeCTR" - ] - }, - { - "cell_type": "markdown", - "id": "b897e86c", - "metadata": {}, - "source": [ - "HugeCTR is a deep learning framework dedicated to recommendation systems. It is written in CUDA C++. As HugeCTR optimizes the training in CUDA++, we need to define the training pipeline and model architecture and execute it via the commandline. We will use the Python API, which is similar to Keras models." - ] - }, - { - "cell_type": "markdown", - "id": "9eea2afa", - "metadata": {}, - "source": [ - "HugeCTR has three main components:\n", - "* Solver: Specifies various details such as active GPU list, batchsize, and model_file\n", - "* Optimizer: Specifies the type of optimizer and its hyperparameters\n", - "* DataReader: Specifies the training/evaluation data\n", - "* Model: Specifies embeddings, and dense layers. Note that embeddings must precede the dense layers" - ] - }, - { - "cell_type": "markdown", - "id": "c8b855d7", - "metadata": {}, - "source": [ - "**Solver**\n", - "\n", - "Let's take a look on the parameter for the `Solver`. We should be familiar from other frameworks for the hyperparameter.\n", - "\n", - "```\n", - "solver = hugectr.CreateSolver(\n", - "- vvgpu: GPU indices used in the training process, which has two levels. For example: [[0,1],[1,2]] indicates that two physical nodes (each physical node can have multiple NUMA nodes) are used. In the first node, GPUs 0 and 1 are used while GPUs 1 and 2 are used for the second node. It is also possible to specify non-continuous GPU indices such as [0, 2, 4, 7].\n", - "- batchsize: Minibatch size used in training\n", - "- max_eval_batches: Maximum number of batches used in evaluation. It is recommended that the number is equal to or bigger than the actual number of bathces in the evaluation dataset.\n", - "On the other hand, with num_epochs, HugeCTR stops the evaluation if all the evaluation data is consumed \n", - "- batchsize_eval: Minibatch size used in evaluation. The default value is 2048. Note that batchsize here is the global batch size across gpus and nodes, not per worker batch size.\n", - "- mixed_precision: Enables mixed precision training with the scaler specified here. Only 128,256, 512, and 1024 scalers are supported\n", - ")\n", - "```" - ] - }, - { - "cell_type": "markdown", - "id": "6f026eb3", - "metadata": {}, - "source": [ - "**Optimizer**\n", - "\n", - "The optimizer is the algorithm to update the model parameters. HugeCTR supports the common algorithms.\n", - "\n", - "\n", - "```\n", - "optimizer = CreateOptimizer(\n", - "- optimizer_type: Optimizer algorithm - Adam, MomentumSGD, Nesterov, and SGD \n", - "- learning_rate: Learning Rate for optimizer\n", - ")\n", - "```" - ] - }, - { - "cell_type": "markdown", - "id": "d7f6fdcf", - "metadata": {}, - "source": [ - "**DataReader**\n", - "\n", - "The data reader defines the training and evaluation dataset.\n", - "\n", - "\n", - "```\n", - "reader = hugectr.DataReaderParams(\n", - "- data_reader_type: Data format to read\n", - "- source: The training dataset file list. IMPORTANT: This should be a list\n", - "- eval_source: The evaluation dataset file list.\n", - "- check_type: The data error detection mechanism (Sum: Checksum, None: no detection).\n", - "- slot_size_array: The list of categorical feature cardinalities\n", - ")\n", - "```" - ] - }, - { - "cell_type": "markdown", - "id": "b939955b", - "metadata": {}, - "source": [ - "**Model**\n", - "\n", - "We initialize the model with the solver, optimizer and data reader:\n", - "\n", - "```\n", - "model = hugectr.Model(solver, reader, optimizer)\n", - "```\n", - "\n", - "We can add multiple layers to the model with `model.add` function. We will focus on:\n", - "- `Input` defines the input data\n", - "- `SparseEmbedding` defines the embedding layer\n", - "- `DenseLayer` defines dense layers, such as fully connected, ReLU, BatchNorm, etc.\n", - "\n", - "**HugeCTR organizes the layers by names. For each layer, we define the input and output names.**" - ] - }, - { - "cell_type": "markdown", - "id": "16380e74", - "metadata": {}, - "source": [ - "Input layer:\n", - "\n", - "This layer is required to define the input data.\n", - "\n", - "```\n", - "hugectr.Input(\n", - " label_dim: Number of label columns\n", - " label_name: Name of label columns in network architecture\n", - " dense_dim: Number of continuous columns\n", - " dense_name: Name of contiunous columns in network architecture\n", - " data_reader_sparse_param_array: Configuration how to read sparse data and its names\n", - ")\n", - "```\n", - "\n", - "SparseEmbedding:\n", - "\n", - "This layer defines embedding table\n", - "\n", - "```\n", - "hugectr.SparseEmbedding(\n", - " embedding_type: Different embedding options to distribute embedding tables \n", - " workspace_size_per_gpu_in_mb: Maximum embedding table size in MB\n", - " embedding_vec_size: Embedding vector size\n", - " combiner: Intra-slot reduction op\n", - " sparse_embedding_name: Layer name\n", - " bottom_name: Input layer names\n", - " optimizer: Optimizer to use\n", - ")\n", - "```\n", - "\n", - "DenseLayer:\n", - "\n", - "This layer is copied to each GPU and is normally used for the MLP tower.\n", - "\n", - "```\n", - "hugectr.DenseLayer(\n", - " layer_type: Layer type, such as FullyConnected, Reshape, Concat, Loss, BatchNorm, etc.\n", - " bottom_names: Input layer names\n", - " top_names: Layer name\n", - " ...: Depending on the layer type additional parameter can be defined\n", - ")\n", - "```\n", - "\n", - "This is only a short introduction in the API. You can read more in the official docs: [Python Interface](https://github.com/NVIDIA/HugeCTR/blob/master/docs/python_interface.md) and [Layer Book](https://github.com/NVIDIA/HugeCTR/blob/master/docs/hugectr_layer_book.md)" - ] - }, - { - "cell_type": "markdown", - "id": "31c83553", - "metadata": {}, - "source": [ - "## Let's define our model\n", - "\n", - "We walked through the documentation, but it is useful to understand the API. Finally, we can define our model. We will write the model to `./model.py` and execute it afterwards." - ] - }, - { - "cell_type": "markdown", - "id": "fca06d03", - "metadata": {}, - "source": [ - "We need the cardinalities of each categorical feature to assign as `slot_size_array` in the model below." - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "id": "4dedb5e9", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "({'userId': (162542, 512), 'movieId': (56635, 512)}, {'genres': (21, 16)})\n" - ] - } - ], - "source": [ - "from nvtabular.ops import get_embedding_sizes\n", - "\n", - "embeddings = get_embedding_sizes(workflow)\n", - "print(embeddings)" - ] - }, - { - "cell_type": "markdown", - "id": "b4a6d2c2", - "metadata": {}, - "source": [ - "We use `graph_to_json` to convert the model to a JSON configuration, required for the inference." - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "id": "7bc3f7fb", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "HugeCTR Version: 3.5\n", - "====================================================Model Init=====================================================\n", - "[HCTR][14:20:17][WARNING][RK0][main]: The model name is not specified when creating the solver.\n", - "[HCTR][14:20:17][WARNING][RK0][main]: MPI was already initialized somewhere elese. Lifetime service disabled.\n", - "[HCTR][14:20:17][INFO][RK0][main]: Global seed is 2794144061\n", - "[HCTR][14:20:17][INFO][RK0][main]: Device to NUMA mapping:\n", - " GPU 0 -> node 0\n", - "[HCTR][14:20:18][WARNING][RK0][main]: Peer-to-peer access cannot be fully enabled.\n", - "[HCTR][14:20:18][INFO][RK0][main]: Start all2all warmup\n", - "[HCTR][14:20:18][INFO][RK0][main]: End all2all warmup\n", - "[HCTR][14:20:18][INFO][RK0][main]: Using All-reduce algorithm: NCCL\n", - "[HCTR][14:20:18][INFO][RK0][main]: Device 0: Quadro GV100\n", - "[HCTR][14:20:18][INFO][RK0][main]: num of DataReader workers: 1\n", - "[HCTR][14:20:18][INFO][RK0][main]: Vocabulary size: 219149\n", - "[HCTR][14:20:18][INFO][RK0][main]: max_vocabulary_size_per_gpu_=1092266\n" - ] - } - ], - "source": [ - "import hugectr\n", - "from mpi4py import MPI # noqa\n", - "\n", - "solver = hugectr.CreateSolver(\n", - " vvgpu=[[0]],\n", - " batchsize=2048,\n", - " batchsize_eval=2048,\n", - " max_eval_batches=160,\n", - " i64_input_key=True,\n", - " use_mixed_precision=False,\n", - " repeat_dataset=True,\n", - ")\n", - "optimizer = hugectr.CreateOptimizer(optimizer_type=hugectr.Optimizer_t.Adam)\n", - "reader = hugectr.DataReaderParams(\n", - " data_reader_type=hugectr.DataReaderType_t.Parquet,\n", - " source=[INPUT_DATA_DIR + \"train/_file_list.txt\"],\n", - " eval_source=INPUT_DATA_DIR + \"valid/_file_list.txt\",\n", - " check_type=hugectr.Check_t.Non,\n", - " slot_size_array=[162542, 56586, 21],\n", - ")\n", - "\n", - "\n", - "model = hugectr.Model(solver, reader, optimizer)\n", - "\n", - "model.add(\n", - " hugectr.Input(\n", - " label_dim=1,\n", - " label_name=\"label\",\n", - " dense_dim=0,\n", - " dense_name=\"dense\",\n", - " data_reader_sparse_param_array=[\n", - " hugectr.DataReaderSparseParam(\"data1\", nnz_per_slot=10, is_fixed_length=False, slot_num=3)\n", - " ],\n", - " )\n", - ")\n", - "model.add(\n", - " hugectr.SparseEmbedding(\n", - " embedding_type=hugectr.Embedding_t.LocalizedSlotSparseEmbeddingHash,\n", - " workspace_size_per_gpu_in_mb=200,\n", - " embedding_vec_size=16,\n", - " combiner=\"sum\",\n", - " sparse_embedding_name=\"sparse_embedding1\",\n", - " bottom_name=\"data1\",\n", - " optimizer=optimizer,\n", - " )\n", - ")\n", - "model.add(\n", - " hugectr.DenseLayer(\n", - " layer_type=hugectr.Layer_t.Reshape,\n", - " bottom_names=[\"sparse_embedding1\"],\n", - " top_names=[\"reshape1\"],\n", - " leading_dim=48,\n", - " )\n", - ")\n", - "model.add(\n", - " hugectr.DenseLayer(\n", - " layer_type=hugectr.Layer_t.InnerProduct,\n", - " bottom_names=[\"reshape1\"],\n", - " top_names=[\"fc1\"],\n", - " num_output=128,\n", - " )\n", - ")\n", - "model.add(\n", - " hugectr.DenseLayer(\n", - " layer_type=hugectr.Layer_t.ReLU,\n", - " bottom_names=[\"fc1\"],\n", - " top_names=[\"relu1\"],\n", - " )\n", - ")\n", - "model.add(\n", - " hugectr.DenseLayer(\n", - " layer_type=hugectr.Layer_t.InnerProduct,\n", - " bottom_names=[\"relu1\"],\n", - " top_names=[\"fc2\"],\n", - " num_output=128,\n", - " )\n", - ")\n", - "model.add(\n", - " hugectr.DenseLayer(\n", - " layer_type=hugectr.Layer_t.ReLU,\n", - " bottom_names=[\"fc2\"],\n", - " top_names=[\"relu2\"],\n", - " )\n", - ")\n", - "model.add(\n", - " hugectr.DenseLayer(\n", - " layer_type=hugectr.Layer_t.InnerProduct,\n", - " bottom_names=[\"relu2\"],\n", - " top_names=[\"fc3\"],\n", - " num_output=1,\n", - " )\n", - ")\n", - "model.add(\n", - " hugectr.DenseLayer(\n", - " layer_type=hugectr.Layer_t.BinaryCrossEntropyLoss,\n", - " bottom_names=[\"fc3\", \"label\"],\n", - " top_names=[\"loss\"],\n", - " )\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "id": "2df637b3", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[HCTR][14:20:18][INFO][RK0][main]: Graph analysis to resolve tensor dependency\n", - "===================================================Model Compile===================================================\n", - "[HCTR][14:20:20][INFO][RK0][main]: gpu0 start to init embedding\n", - "[HCTR][14:20:20][INFO][RK0][main]: gpu0 init embedding done\n", - "[HCTR][14:20:20][INFO][RK0][main]: Starting AUC NCCL warm-up\n", - "[HCTR][14:20:20][INFO][RK0][main]: Warm-up done\n", - "===================================================Model Summary===================================================\n", - "[HCTR][14:20:20][INFO][RK0][main]: label Dense Sparse \n", - "label dense data1 \n", - "(None, 1) (None, 0) \n", - "——————————————————————————————————————————————————————————————————————————————————————————————————————————————————\n", - "Layer Type Input Name Output Name Output Shape \n", - "——————————————————————————————————————————————————————————————————————————————————————————————————————————————————\n", - "LocalizedSlotSparseEmbeddingHash data1 sparse_embedding1 (None, 3, 16) \n", - "------------------------------------------------------------------------------------------------------------------\n", - "Reshape sparse_embedding1 reshape1 (None, 48) \n", - "------------------------------------------------------------------------------------------------------------------\n", - "InnerProduct reshape1 fc1 (None, 128) \n", - "------------------------------------------------------------------------------------------------------------------\n", - "ReLU fc1 relu1 (None, 128) \n", - "------------------------------------------------------------------------------------------------------------------\n", - "InnerProduct relu1 fc2 (None, 128) \n", - "------------------------------------------------------------------------------------------------------------------\n", - "ReLU fc2 relu2 (None, 128) \n", - "------------------------------------------------------------------------------------------------------------------\n", - "InnerProduct relu2 fc3 (None, 1) \n", - "------------------------------------------------------------------------------------------------------------------\n", - "BinaryCrossEntropyLoss fc3 loss \n", - " label \n", - "------------------------------------------------------------------------------------------------------------------\n", - "=====================================================Model Fit=====================================================\n", - "[HCTR][14:20:20][INFO][RK0][main]: Use non-epoch mode with number of iterations: 2000\n", - "[HCTR][14:20:20][INFO][RK0][main]: Training batchsize: 2048, evaluation batchsize: 2048\n", - "[HCTR][14:20:20][INFO][RK0][main]: Evaluation interval: 200, snapshot interval: 1900\n", - "[HCTR][14:20:20][INFO][RK0][main]: Dense network trainable: True\n", - "[HCTR][14:20:20][INFO][RK0][main]: Sparse embedding sparse_embedding1 trainable: True\n", - "[HCTR][14:20:20][INFO][RK0][main]: Use mixed precision: False, scaler: 1.000000, use cuda graph: True\n", - "[HCTR][14:20:20][INFO][RK0][main]: lr: 0.001000, warmup_steps: 1, end_lr: 0.000000\n", - "[HCTR][14:20:20][INFO][RK0][main]: decay_start: 0, decay_steps: 1, decay_power: 2.000000\n", - "[HCTR][14:20:20][INFO][RK0][main]: Training source file: /root/nvt-examples/movielens/data/train/_file_list.txt\n", - "[HCTR][14:20:20][INFO][RK0][main]: Evaluation source file: /root/nvt-examples/movielens/data/valid/_file_list.txt\n", - "[HCTR][14:20:20][INFO][RK0][main]: Iter: 100 Time(100 iters): 0.145249s Loss: 0.599668 lr:0.001\n", - "[HCTR][14:20:20][INFO][RK0][main]: Iter: 200 Time(100 iters): 0.14389s Loss: 0.569523 lr:0.001\n", - "[HCTR][14:20:20][INFO][RK0][main]: Evaluation, AUC: 0.747082\n", - "[HCTR][14:20:20][INFO][RK0][main]: Eval Time for 160 iters: 0.035607s\n", - "[HCTR][14:20:20][INFO][RK0][main]: Iter: 300 Time(100 iters): 0.177161s Loss: 0.548131 lr:0.001\n", - "[HCTR][14:20:20][INFO][RK0][main]: Iter: 400 Time(100 iters): 0.140567s Loss: 0.546302 lr:0.001\n", - "[HCTR][14:20:20][INFO][RK0][main]: Evaluation, AUC: 0.765986\n", - "[HCTR][14:20:20][INFO][RK0][main]: Eval Time for 160 iters: 0.041411s\n", - "[HCTR][14:20:20][INFO][RK0][main]: Iter: 500 Time(100 iters): 0.22512s Loss: 0.55636 lr:0.001\n", - "[HCTR][14:20:21][INFO][RK0][main]: Iter: 600 Time(100 iters): 0.141749s Loss: 0.541177 lr:0.001\n", - "[HCTR][14:20:21][INFO][RK0][main]: Evaluation, AUC: 0.774578\n", - "[HCTR][14:20:21][INFO][RK0][main]: Eval Time for 160 iters: 0.035427s\n", - "[HCTR][14:20:21][INFO][RK0][main]: Iter: 700 Time(100 iters): 0.177425s Loss: 0.545869 lr:0.001\n", - "[HCTR][14:20:21][INFO][RK0][main]: Iter: 800 Time(100 iters): 0.138808s Loss: 0.537519 lr:0.001\n", - "[HCTR][14:20:21][INFO][RK0][main]: Evaluation, AUC: 0.780465\n", - "[HCTR][14:20:21][INFO][RK0][main]: Eval Time for 160 iters: 0.073079s\n", - "[HCTR][14:20:21][INFO][RK0][main]: Iter: 900 Time(100 iters): 0.210899s Loss: 0.549535 lr:0.001\n", - "[HCTR][14:20:21][INFO][RK0][main]: Iter: 1000 Time(100 iters): 0.18031s Loss: 0.532493 lr:0.001\n", - "[HCTR][14:20:21][INFO][RK0][main]: Evaluation, AUC: 0.783634\n", - "[HCTR][14:20:21][INFO][RK0][main]: Eval Time for 160 iters: 0.036747s\n", - "[HCTR][14:20:21][INFO][RK0][main]: Iter: 1100 Time(100 iters): 0.174997s Loss: 0.543344 lr:0.001\n", - "[HCTR][14:20:22][INFO][RK0][main]: Iter: 1200 Time(100 iters): 0.136631s Loss: 0.525491 lr:0.001\n", - "[HCTR][14:20:22][INFO][RK0][main]: Evaluation, AUC: 0.786688\n", - "[HCTR][14:20:22][INFO][RK0][main]: Eval Time for 160 iters: 0.033862s\n", - "[HCTR][14:20:22][INFO][RK0][main]: Iter: 1300 Time(100 iters): 0.174932s Loss: 0.543256 lr:0.001\n", - "[HCTR][14:20:22][INFO][RK0][main]: Iter: 1400 Time(100 iters): 0.141826s Loss: 0.533403 lr:0.001\n", - "[HCTR][14:20:22][INFO][RK0][main]: Evaluation, AUC: 0.790685\n", - "[HCTR][14:20:22][INFO][RK0][main]: Eval Time for 160 iters: 0.075811s\n", - "[HCTR][14:20:22][INFO][RK0][main]: Iter: 1500 Time(100 iters): 0.261529s Loss: 0.516566 lr:0.001\n", - "[HCTR][14:20:22][INFO][RK0][main]: Iter: 1600 Time(100 iters): 0.138679s Loss: 0.516145 lr:0.001\n", - "[HCTR][14:20:22][INFO][RK0][main]: Evaluation, AUC: 0.792489\n", - "[HCTR][14:20:22][INFO][RK0][main]: Eval Time for 160 iters: 0.039438s\n", - "[HCTR][14:20:22][INFO][RK0][main]: Iter: 1700 Time(100 iters): 0.180547s Loss: 0.513846 lr:0.001\n", - "[HCTR][14:20:23][INFO][RK0][main]: Iter: 1800 Time(100 iters): 0.14265s Loss: 0.52191 lr:0.001\n", - "[HCTR][14:20:23][INFO][RK0][main]: Evaluation, AUC: 0.795303\n", - "[HCTR][14:20:23][INFO][RK0][main]: Eval Time for 160 iters: 0.035608s\n", - "[HCTR][14:20:23][INFO][RK0][main]: Iter: 1900 Time(100 iters): 0.18116s Loss: 0.508622 lr:0.001\n", - "[HCTR][14:20:23][INFO][RK0][main]: Rank0: Dump hash table from GPU0\n", - "[HCTR][14:20:23][INFO][RK0][main]: Rank0: Write hash table pairs to file\n", - "[HCTR][14:20:23][INFO][RK0][main]: Done\n", - "[HCTR][14:20:23][INFO][RK0][main]: Dumping sparse weights to files, successful\n", - "[HCTR][14:20:23][INFO][RK0][main]: Rank0: Write optimzer state to file\n", - "[HCTR][14:20:23][INFO][RK0][main]: Done\n", - "[HCTR][14:20:23][INFO][RK0][main]: Rank0: Write optimzer state to file\n", - "[HCTR][14:20:23][INFO][RK0][main]: Done\n", - "[HCTR][14:20:23][INFO][RK0][main]: Dumping sparse optimzer states to files, successful\n", - "[HCTR][14:20:23][INFO][RK0][main]: Dumping dense weights to file, successful\n", - "[HCTR][14:20:23][INFO][RK0][main]: Dumping dense optimizer states to file, successful\n", - "[HCTR][14:20:23][INFO][RK0][main]: Finish 2000 iterations with batchsize: 2048 in 3.61s.\n", - "[HCTR][14:20:23][INFO][RK0][main]: Save the model graph to /root/nvt-examples/movielens/data/model/movielens_hugectr/1/movielens.json successfully\n" - ] - } - ], - "source": [ - "model.compile()\n", - "model.summary()\n", - "model.fit(max_iter=2000, display=100, eval_interval=200, snapshot=1900)\n", - "model.graph_to_json(graph_config_file=MODEL_DIR + \"1/movielens.json\")" - ] - }, - { - "cell_type": "markdown", - "id": "121d3c82", - "metadata": {}, - "source": [ - "After training terminates, we can see that multiple `.model` files and folders are generated. We need to move them inside `1` folder under the `movielens_hugectr` folder. " - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "id": "c8273275", - "metadata": {}, - "outputs": [], - "source": [ - "!mv *.model {MODEL_DIR}" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.10" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/examples/getting-started-movielens/03-Training-with-PyTorch.ipynb b/examples/getting-started-movielens/03-Training-with-PyTorch.ipynb deleted file mode 100644 index db667dd1972..00000000000 --- a/examples/getting-started-movielens/03-Training-with-PyTorch.ipynb +++ /dev/null @@ -1,611 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "# Copyright 2021 NVIDIA Corporation. All Rights Reserved.\n", - "#\n", - "# Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# http://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License.\n", - "# ==============================================================================" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "# Getting Started MovieLens: Training with PyTorch\n", - "\n", - "## Overview\n", - "\n", - "We observed that PyTorch training pipelines can be slow as the dataloader is a bottleneck. The native dataloader in PyTorch randomly sample each item from the dataset, which is very slow. In our experiments, we are able to speed-up existing PyTorch pipelines using a highly optimized dataloader.

\n", - "\n", - "Applying deep learning models to recommendation systems faces unique challenges in comparison to other domains, such as computer vision and natural language processing. The datasets and common model architectures have unique characteristics, which require custom solutions. Recommendation system datasets have terabytes in size with billion examples but each example is represented by only a few bytes. For example, the [Criteo CTR dataset](https://ailab.criteo.com/download-criteo-1tb-click-logs-dataset/), the largest publicly available dataset, is 1.3TB with 4 billion examples. The model architectures have normally large embedding tables for the users and items, which do not fit on a single GPU. You can read more in our [blogpost](https://medium.com/nvidia-merlin/why-isnt-your-recommender-system-training-faster-on-gpu-and-what-can-you-do-about-it-6cb44a711ad4).\n", - "\n", - "### Learning objectives\n", - "\n", - "This notebook explains, how to use the NVTabular dataloader to accelerate PyTorch training.\n", - "\n", - "1. Use **NVTabular dataloader** with PyTorch\n", - "2. Leverage **multi-hot encoded input features**\n", - "\n", - "### MovieLens25M\n", - "\n", - "The [MovieLens25M](https://grouplens.org/datasets/movielens/25m/) is a popular dataset for recommender systems and is used in academic publications. The dataset contains 25M movie ratings for 62,000 movies given by 162,000 users. Many projects use only the user/item/rating information of MovieLens, but the original dataset provides metadata for the movies, as well. For example, which genres a movie has. Although we may not improve state-of-the-art results with our neural network architecture, the purpose of this notebook is to explain how to integrate multi-hot categorical features into a neural network." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## NVTabular dataloader for PyTorch\n", - "\n", - "We’ve identified that the dataloader is one bottleneck in deep learning recommender systems when training pipelines with PyTorch. The dataloader cannot prepare the next batch fast enough, so and therefore, the GPU is not utilized. \n", - "\n", - "As a result, we developed a highly customized tabular dataloader for accelerating existing pipelines in PyTorch. NVTabular dataloader’s features are:\n", - "\n", - "- removing bottleneck of item-by-item dataloading\n", - "- enabling larger than memory dataset by streaming from disk\n", - "- reading data directly into GPU memory and remove CPU-GPU communication\n", - "- preparing batch asynchronously in GPU to avoid CPU-GPU communication\n", - "- supporting commonly used .parquet format for efficient data format\n", - "- easy integration into existing PyTorch pipelines by using similar API than the native one\n" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "# External dependencies\n", - "import os\n", - "import gc\n", - "import glob\n", - "\n", - "import nvtabular as nvt" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We define our base directory, containing the data." - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "INPUT_DATA_DIR = os.environ.get(\n", - " \"INPUT_DATA_DIR\", os.path.expanduser(\"~/nvt-examples/movielens/data/\")\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Defining Hyperparameters" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "First, we define the data schema and differentiate between single-hot and multi-hot categorical features. Note, that we do not have any numerical input features. " - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "BATCH_SIZE = 1024 * 32 # Batch Size\n", - "CATEGORICAL_COLUMNS = [\"movieId\", \"userId\"] # Single-hot\n", - "CATEGORICAL_MH_COLUMNS = [\"genres\"] # Multi-hot\n", - "NUMERIC_COLUMNS = []\n", - "\n", - "# Output from ETL-with-NVTabular\n", - "TRAIN_PATHS = sorted(glob.glob(os.path.join(INPUT_DATA_DIR, \"train\", \"*.parquet\")))\n", - "VALID_PATHS = sorted(glob.glob(os.path.join(INPUT_DATA_DIR, \"valid\", \"*.parquet\")))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "In the previous notebook, we used NVTabular for ETL and stored the workflow to disk. We can load the NVTabular workflow to extract important metadata for our training pipeline." - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [], - "source": [ - "proc = nvt.Workflow.load(os.path.join(INPUT_DATA_DIR, \"workflow\"))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The embedding table shows the cardinality of each categorical variable along with its associated embedding size. Each entry is of the form `(cardinality, embedding_size)`." - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "({'userId': (162542, 512), 'movieId': (56586, 512)}, {'genres': (21, 16)})" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "EMBEDDING_TABLE_SHAPES, MH_EMBEDDING_TABLE_SHAPES = nvt.ops.get_embedding_sizes(proc)\n", - "EMBEDDING_TABLE_SHAPES, MH_EMBEDDING_TABLE_SHAPES" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Initializing NVTabular Dataloader for PyTorch" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We import PyTorch and the NVTabular dataloader for PyTorch." - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [], - "source": [ - "import torch\n", - "from nvtabular.loader.torch import TorchAsyncItr, DLDataLoader\n", - "from nvtabular.framework_utils.torch.models import Model\n", - "from nvtabular.framework_utils.torch.utils import process_epoch" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "First, we take a look on our dataloader and how the data is represented as tensors. The NVTabular dataloader are initialized as usually and we specify both single-hot and multi-hot categorical features as cats. The dataloader will automatically recognize the single/multi-hot columns and represent them accordingly." - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [], - "source": [ - "# TensorItrDataset returns a single batch of x_cat, x_cont, y.\n", - "\n", - "train_dataset = TorchAsyncItr(\n", - " nvt.Dataset(TRAIN_PATHS),\n", - " batch_size=BATCH_SIZE,\n", - " cats=CATEGORICAL_COLUMNS + CATEGORICAL_MH_COLUMNS,\n", - " conts=NUMERIC_COLUMNS,\n", - " labels=[\"rating\"],\n", - ")\n", - "train_loader = DLDataLoader(\n", - " train_dataset, batch_size=None, collate_fn=lambda x: x, pin_memory=False, num_workers=0\n", - ")\n", - "\n", - "valid_dataset = TorchAsyncItr(\n", - " nvt.Dataset(VALID_PATHS),\n", - " batch_size=BATCH_SIZE,\n", - " cats=CATEGORICAL_COLUMNS + CATEGORICAL_MH_COLUMNS,\n", - " conts=NUMERIC_COLUMNS,\n", - " labels=[\"rating\"],\n", - ")\n", - "valid_loader = DLDataLoader(\n", - " valid_dataset, batch_size=None, collate_fn=lambda x: x, pin_memory=False, num_workers=0\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Let's generate a batch and take a look on the input features.

\n", - "The single-hot categorical features (`userId` and `movieId`) have a shape of `(32768, 1)`, which is the batch size (as usually). For the multi-hot categorical feature `genres`, we receive two Tensors `genres__values` and `genres__nnzs`.

\n", - "- `values` are the actual data, containing the genre IDs. Note that the Tensor has more values than the batch_size. The reason is, that one datapoint in the batch can contain more than one genre (multi-hot).
\n", - "- `nnzs` are a supporting Tensor, describing how many genres are associated with each datapoint in the batch.

\n", - "For example,\n", - "- if the first two values in `nnzs` is `0`, `2`, then the first 2 values (0, 1) in `values` are associated with the first datapoint in the batch (movieId/userId).
\n", - "- if the next value in `nnzs` is `6`, then the 3rd, 4th and 5th value in `values` are associated with the second datapoint in the batch (continuing after the previous value stopped).
\n", - "- if the third value in `nnzs` is `7`, then the 6th value in `values` are associated with the third datapoint in the batch. \n", - "- and so on" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "({'genres': (tensor([1, 6, 1, ..., 4, 2, 6], device='cuda:0'),\n", - " tensor([[ 0],\n", - " [ 2],\n", - " [ 4],\n", - " ...,\n", - " [89409],\n", - " [89410],\n", - " [89412]], device='cuda:0')),\n", - " 'movieId': tensor([[ 18],\n", - " [8649],\n", - " [5935],\n", - " ...,\n", - " [ 666],\n", - " [2693],\n", - " [ 643]], device='cuda:0'),\n", - " 'userId': tensor([[105522],\n", - " [ 18041],\n", - " [ 499],\n", - " ...,\n", - " [104270],\n", - " [ 62],\n", - " [ 2344]], device='cuda:0')},\n", - " tensor([1., 1., 1., ..., 1., 1., 0.], device='cuda:0'))" - ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "batch = next(iter(train_loader))\n", - "batch" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "`X_cat_multihot` is a tuple of two Tensors. For the multi-hot categorical feature `genres`, we receive two Tensors `values` and `nnzs`." - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "(tensor([1, 6, 1, ..., 4, 2, 6], device='cuda:0'),\n", - " tensor([[ 0],\n", - " [ 2],\n", - " [ 4],\n", - " ...,\n", - " [89409],\n", - " [89410],\n", - " [89412]], device='cuda:0'))" - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "X_cat_multihot = batch[0]['genres']\n", - "X_cat_multihot" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "torch.Size([89414])" - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "X_cat_multihot[0].shape" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "torch.Size([32768, 1])" - ] - }, - "execution_count": 12, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "X_cat_multihot[1].shape" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "As each datapoint can have a different number of genres, it is more efficient to represent the genres as two flat tensors: One with the actual values (`values`) and one with the length for each datapoint (`nnzs`)." - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "2" - ] - }, - "execution_count": 13, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "del batch\n", - "gc.collect()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Defining Neural Network Architecture" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We implemented a simple PyTorch architecture.\n", - "\n", - "* Single-hot categorical features are fed into an Embedding Layer\n", - "* Each value of a multi-hot categorical features is fed into an Embedding Layer and the multiple Embedding outputs are combined via summing\n", - "* The output of the Embedding Layers are concatenated\n", - "* The concatenated layers are fed through multiple feed-forward layers (Dense Layers, BatchNorm with ReLU activations)\n", - "\n", - "You can see more details by checking out the implementation." - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [], - "source": [ - "# ??Model" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We initialize the model. `EMBEDDING_TABLE_SHAPES` needs to be a Tuple representing the cardinality for single-hot and multi-hot input features." - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "({'movieId': (56586, 512), 'userId': (162542, 512)}, {'genres': (21, 16)})" - ] - }, - "execution_count": 15, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "EMBEDDING_TABLE_SHAPES_TUPLE = (\n", - " {\n", - " CATEGORICAL_COLUMNS[0]: EMBEDDING_TABLE_SHAPES[CATEGORICAL_COLUMNS[0]],\n", - " CATEGORICAL_COLUMNS[1]: EMBEDDING_TABLE_SHAPES[CATEGORICAL_COLUMNS[1]],\n", - " },\n", - " {CATEGORICAL_MH_COLUMNS[0]: MH_EMBEDDING_TABLE_SHAPES[CATEGORICAL_MH_COLUMNS[0]]},\n", - ")\n", - "EMBEDDING_TABLE_SHAPES_TUPLE" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "Model(\n", - " (initial_cat_layer): ConcatenatedEmbeddings(\n", - " (embedding_layers): ModuleList(\n", - " (0): Embedding(56586, 512)\n", - " (1): Embedding(162542, 512)\n", - " )\n", - " (dropout): Dropout(p=0.0, inplace=False)\n", - " )\n", - " (mh_cat_layer): MultiHotEmbeddings(\n", - " (embedding_layers): ModuleList(\n", - " (0): EmbeddingBag(21, 16, mode=sum)\n", - " )\n", - " (dropout): Dropout(p=0.0, inplace=False)\n", - " )\n", - " (initial_cont_layer): BatchNorm1d(0, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", - " (layers): ModuleList(\n", - " (0): Sequential(\n", - " (0): Linear(in_features=1040, out_features=128, bias=True)\n", - " (1): ReLU(inplace=True)\n", - " (2): BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", - " (3): Dropout(p=0.0, inplace=False)\n", - " )\n", - " (1): Sequential(\n", - " (0): Linear(in_features=128, out_features=128, bias=True)\n", - " (1): ReLU(inplace=True)\n", - " (2): BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", - " (3): Dropout(p=0.0, inplace=False)\n", - " )\n", - " (2): Sequential(\n", - " (0): Linear(in_features=128, out_features=128, bias=True)\n", - " (1): ReLU(inplace=True)\n", - " (2): BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", - " (3): Dropout(p=0.0, inplace=False)\n", - " )\n", - " )\n", - " (output_layer): Linear(in_features=128, out_features=1, bias=True)\n", - ")" - ] - }, - "execution_count": 16, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "model = Model(\n", - " embedding_table_shapes=EMBEDDING_TABLE_SHAPES_TUPLE,\n", - " num_continuous=0,\n", - " emb_dropout=0.0,\n", - " layer_hidden_dims=[128, 128, 128],\n", - " layer_dropout_rates=[0.0, 0.0, 0.0],\n", - ").to(\"cuda\")\n", - "model" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We initialize the optimizer." - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "metadata": {}, - "outputs": [], - "source": [ - "optimizer = torch.optim.Adam(model.parameters(), lr=0.01)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We use the `process_epoch` function to train and validate our model. It iterates over the dataset and calculates as usually the loss and optimizer step." - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Total batches: 610\n", - "Total batches: 152\n", - "Epoch 00. Train loss: 0.1944. Valid loss: 0.1696.\n", - "run_time: 12.725089311599731 - rows: 2292 - epochs: 0 - dl_thru: 180.1166140272741\n", - "CPU times: user 9.66 s, sys: 3.05 s, total: 12.7 s\n", - "Wall time: 12.7 s\n" - ] - } - ], - "source": [ - "%%time\n", - "from time import time\n", - "EPOCHS = 1\n", - "for epoch in range(EPOCHS):\n", - " start = time()\n", - " train_loss, y_pred, y = process_epoch(train_loader,\n", - " model,\n", - " train=True,\n", - " optimizer=optimizer)\n", - " valid_loss, y_pred, y = process_epoch(valid_loader,\n", - " model,\n", - " train=False)\n", - " print(f\"Epoch {epoch:02d}. Train loss: {train_loss:.4f}. Valid loss: {valid_loss:.4f}.\")\n", - "t_final = time() - start\n", - "total_rows = train_dataset.num_rows_processed + valid_dataset.num_rows_processed\n", - "print(\n", - " f\"run_time: {t_final} - rows: {total_rows * EPOCHS} - epochs: {EPOCHS} - dl_thru: {(total_rows * EPOCHS) / t_final}\"\n", - ")" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.10" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/examples/getting-started-movielens/03-Training-with-TF.ipynb b/examples/getting-started-movielens/03-Training-with-TF.ipynb deleted file mode 100644 index 30d019b78b3..00000000000 --- a/examples/getting-started-movielens/03-Training-with-TF.ipynb +++ /dev/null @@ -1,766 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "# Copyright 2021 NVIDIA Corporation. All Rights Reserved.\n", - "#\n", - "# Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# http://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License.\n", - "# ==============================================================================" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "# Getting Started MovieLens: Training with TensorFlow\n", - "\n", - "## Overview\n", - "\n", - "We observed that TensorFlow training pipelines can be slow as the dataloader is a bottleneck. The native dataloader in TensorFlow randomly sample each item from the dataset, which is very slow. The window dataloader in TensorFlow is not much faster. In our experiments, we are able to speed-up existing TensorFlow pipelines by 9x using a highly optimized dataloader.

\n", - "\n", - "Applying deep learning models to recommendation systems faces unique challenges in comparison to other domains, such as computer vision and natural language processing. The datasets and common model architectures have unique characteristics, which require custom solutions. Recommendation system datasets have terabytes in size with billion examples but each example is represented by only a few bytes. For example, the [Criteo CTR dataset](https://ailab.criteo.com/download-criteo-1tb-click-logs-dataset/), the largest publicly available dataset, is 1.3TB with 4 billion examples. The model architectures have normally large embedding tables for the users and items, which do not fit on a single GPU. You can read more in our [blogpost](https://medium.com/nvidia-merlin/why-isnt-your-recommender-system-training-faster-on-gpu-and-what-can-you-do-about-it-6cb44a711ad4).\n", - "\n", - "### Learning objectives\n", - "This notebook explains, how to use the NVTabular dataloader to accelerate TensorFlow training.\n", - "\n", - "1. Use **NVTabular dataloader** with TensorFlow Keras model\n", - "2. Leverage **multi-hot encoded input features**\n", - "\n", - "### MovieLens25M\n", - "\n", - "The [MovieLens25M](https://grouplens.org/datasets/movielens/25m/) is a popular dataset for recommender systems and is used in academic publications. The dataset contains 25M movie ratings for 62,000 movies given by 162,000 users. Many projects use only the user/item/rating information of MovieLens, but the original dataset provides metadata for the movies, as well. For example, which genres a movie has. Although we may not improve state-of-the-art results with our neural network architecture, the purpose of this notebook is to explain how to integrate multi-hot categorical features into a neural network." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## NVTabular dataloader for TensorFlow\n", - "\n", - "We’ve identified that the dataloader is one bottleneck in deep learning recommender systems when training pipelines with TensorFlow. The dataloader cannot prepare the next batch fast enough and therefore, the GPU is not fully utilized. \n", - "\n", - "We developed a highly customized tabular dataloader for accelerating existing pipelines in TensorFlow. In our experiments, we see a speed-up by 9x of the same training workflow with NVTabular dataloader. NVTabular dataloader’s features are:\n", - "\n", - "- removing bottleneck of item-by-item dataloading\n", - "- enabling larger than memory dataset by streaming from disk\n", - "- reading data directly into GPU memory and remove CPU-GPU communication\n", - "- preparing batch asynchronously in GPU to avoid CPU-GPU communication\n", - "- supporting commonly used .parquet format\n", - "- easy integration into existing TensorFlow pipelines by using similar API - works with tf.keras models\n", - "\n", - "More information in our [blogpost](https://medium.com/nvidia-merlin/training-deep-learning-based-recommender-systems-9x-faster-with-tensorflow-cc5a2572ea49)." - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "# External dependencies\n", - "import os\n", - "import glob\n", - "\n", - "import nvtabular as nvt" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We define our base input directory, containing the data." - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "INPUT_DATA_DIR = os.environ.get(\n", - " \"INPUT_DATA_DIR\", os.path.expanduser(\"~/nvt-examples/movielens/data/\")\n", - ")\n", - "# path to save the models\n", - "MODEL_BASE_DIR = os.environ.get(\"MODEL_BASE_DIR\", os.path.expanduser(\"~/nvt-examples/\"))" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "# avoid numba warnings\n", - "from numba import config\n", - "config.CUDA_LOW_OCCUPANCY_WARNINGS = 0" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Defining Hyperparameters" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "First, we define the data schema and differentiate between single-hot and multi-hot categorical features. Note, that we do not have any numerical input features. " - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [], - "source": [ - "BATCH_SIZE = 1024 * 32 # Batch Size\n", - "CATEGORICAL_COLUMNS = [\"movieId\", \"userId\"] # Single-hot\n", - "CATEGORICAL_MH_COLUMNS = [\"genres\"] # Multi-hot\n", - "NUMERIC_COLUMNS = []\n", - "\n", - "# Output from ETL-with-NVTabular\n", - "TRAIN_PATHS = sorted(glob.glob(os.path.join(INPUT_DATA_DIR, \"train\", \"*.parquet\")))\n", - "VALID_PATHS = sorted(glob.glob(os.path.join(INPUT_DATA_DIR, \"valid\", \"*.parquet\")))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "In the previous notebook, we used NVTabular for ETL and stored the workflow to disk. We can load the NVTabular workflow to extract important metadata for our training pipeline." - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [], - "source": [ - "workflow = nvt.Workflow.load(os.path.join(INPUT_DATA_DIR, \"workflow\"))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The embedding table shows the cardinality of each categorical variable along with its associated embedding size. Each entry is of the form `(cardinality, embedding_size)`." - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'userId': (162542, 512), 'movieId': (56747, 512), 'genres': (21, 16)}" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "EMBEDDING_TABLE_SHAPES, MH_EMBEDDING_TABLE_SHAPES = nvt.ops.get_embedding_sizes(workflow)\n", - "EMBEDDING_TABLE_SHAPES.update(MH_EMBEDDING_TABLE_SHAPES)\n", - "EMBEDDING_TABLE_SHAPES" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Initializing NVTabular Dataloader for Tensorflow" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We import TensorFlow and some NVTabular TF extensions, such as custom TensorFlow layers supporting multi-hot and the NVTabular TensorFlow data loader." - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "import time\n", - "import tensorflow as tf\n", - "\n", - "from nvtabular.loader.tensorflow import KerasSequenceLoader, KerasSequenceValidater\n", - "from nvtabular.framework_utils.tensorflow import layers" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "First, we take a look on our data loader and how the data is represented as tensors. The NVTabular data loader are initialized as usually and we specify both single-hot and multi-hot categorical features as cat_names. The data loader will automatically recognize the single/multi-hot columns and represent them accordingly." - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/usr/local/lib/python3.8/dist-packages/cudf/core/dataframe.py:1292: UserWarning: The deep parameter is ignored and is only included for pandas compatibility.\n", - " warnings.warn(\n" - ] - } - ], - "source": [ - "train_dataset_tf = KerasSequenceLoader(\n", - " TRAIN_PATHS, # you could also use a glob pattern\n", - " batch_size=BATCH_SIZE,\n", - " label_names=[\"rating\"],\n", - " cat_names=CATEGORICAL_COLUMNS + CATEGORICAL_MH_COLUMNS,\n", - " cont_names=NUMERIC_COLUMNS,\n", - " engine=\"parquet\",\n", - " shuffle=True,\n", - " buffer_size=0.06, # how many batches to load at once\n", - " parts_per_chunk=1,\n", - ")\n", - "\n", - "valid_dataset_tf = KerasSequenceLoader(\n", - " VALID_PATHS, # you could also use a glob pattern\n", - " batch_size=BATCH_SIZE,\n", - " label_names=[\"rating\"],\n", - " cat_names=CATEGORICAL_COLUMNS + CATEGORICAL_MH_COLUMNS,\n", - " cont_names=NUMERIC_COLUMNS,\n", - " engine=\"parquet\",\n", - " shuffle=False,\n", - " buffer_size=0.06,\n", - " parts_per_chunk=1,\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Let's generate a batch and take a look on the input features.

\n", - "We can see, that the single-hot categorical features (`userId` and `movieId`) have a shape of `(32768, 1)`, which is the batchsize (as usually).

\n", - "For the multi-hot categorical feature `genres`, we receive two Tensors `genres__values` and `genres__nnzs`.

\n", - "`genres__values` are the actual data, containing the genre IDs. Note that the Tensor has more values than the batch_size. The reason is, that one datapoint in the batch can contain more than one genre (multi-hot).
\n", - "`genres__nnzs` are a supporting Tensor, describing how many genres are associated with each datapoint in the batch.

\n", - "For example,\n", - "- if the first value in `genres__nnzs` is `5`, then the first 5 values in `genres__values` are associated with the first datapoint in the batch (movieId/userId).
\n", - "- if the second value in `genres__nnzs` is `2`, then the 6th and the 7th values in `genres__values` are associated with the second datapoint in the batch (continuing after the previous value stopped).
\n", - "- if the third value in `genres_nnzs` is `1`, then the 8th value in `genres__values` are associated with the third datapoint in the batch. \n", - "- and so on" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2022-04-27 22:12:40.128861: I tensorflow/core/platform/cpu_feature_guard.cc:152] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: SSE3 SSE4.1 SSE4.2 AVX\n", - "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", - "2022-04-27 22:12:41.479738: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1525] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 16254 MB memory: -> device: 0, name: Quadro GV100, pci bus id: 0000:15:00.0, compute capability: 7.0\n", - "2022-04-27 22:12:41.480359: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1525] Created device /job:localhost/replica:0/task:0/device:GPU:1 with 30382 MB memory: -> device: 1, name: Quadro GV100, pci bus id: 0000:2d:00.0, compute capability: 7.0\n" - ] - }, - { - "data": { - "text/plain": [ - "{'genres': (,\n", - " ),\n", - " 'movieId': ,\n", - " 'userId': }" - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "batch = next(iter(train_dataset_tf))\n", - "batch[0]" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We can see that the sum of `genres__nnzs` is equal to the shape of `genres__values`." - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "tf.reduce_sum(batch[0][\"genres\"][1])" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "As each datapoint can have a different number of genres, it is more efficient to represent the genres as two flat tensors: One with the actual values (`genres__values`) and one with the length for each datapoint (`genres__nnzs`)." - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [], - "source": [ - "del batch" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Defining Neural Network Architecture" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We will define a common neural network architecture for tabular data.\n", - "\n", - "* Single-hot categorical features are fed into an Embedding Layer\n", - "* Each value of a multi-hot categorical features is fed into an Embedding Layer and the multiple Embedding outputs are combined via averaging\n", - "* The output of the Embedding Layers are concatenated\n", - "* The concatenated layers are fed through multiple feed-forward layers (Dense Layers with ReLU activations)\n", - "* The final output is a single number with sigmoid activation function" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "First, we will define some dictionary/lists for our network architecture." - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [], - "source": [ - "inputs = {} # tf.keras.Input placeholders for each feature to be used\n", - "emb_layers = [] # output of all embedding layers, which will be concatenated" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We create `tf.keras.Input` tensors for all 4 input features." - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [], - "source": [ - "for col in CATEGORICAL_COLUMNS:\n", - " inputs[col] = tf.keras.Input(name=col, dtype=tf.int64, shape=(1,))\n", - "# Note that we need two input tensors for multi-hot categorical features\n", - "for col in CATEGORICAL_MH_COLUMNS:\n", - " inputs[col] = (tf.keras.Input(name=f\"{col}__values\", dtype=tf.int64, shape=(1,)),\n", - " tf.keras.Input(name=f\"{col}__nnzs\", dtype=tf.int64, shape=(1,)))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Next, we initialize Embedding Layers with `tf.feature_column.embedding_column`." - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[EmbeddingColumn(categorical_column=IdentityCategoricalColumn(key='movieId', number_buckets=56747, default_value=None), dimension=512, combiner='mean', initializer=, ckpt_to_load_from=None, tensor_name_in_ckpt=None, max_norm=None, trainable=True, use_safe_embedding_lookup=True),\n", - " EmbeddingColumn(categorical_column=IdentityCategoricalColumn(key='userId', number_buckets=162542, default_value=None), dimension=512, combiner='mean', initializer=, ckpt_to_load_from=None, tensor_name_in_ckpt=None, max_norm=None, trainable=True, use_safe_embedding_lookup=True),\n", - " EmbeddingColumn(categorical_column=IdentityCategoricalColumn(key='genres', number_buckets=21, default_value=None), dimension=16, combiner='mean', initializer=, ckpt_to_load_from=None, tensor_name_in_ckpt=None, max_norm=None, trainable=True, use_safe_embedding_lookup=True)]" - ] - }, - "execution_count": 15, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "for col in CATEGORICAL_COLUMNS + CATEGORICAL_MH_COLUMNS:\n", - " emb_layers.append(\n", - " tf.feature_column.embedding_column(\n", - " tf.feature_column.categorical_column_with_identity(\n", - " col, EMBEDDING_TABLE_SHAPES[col][0]\n", - " ), # Input dimension (vocab size)\n", - " EMBEDDING_TABLE_SHAPES[col][1], # Embedding output dimension\n", - " )\n", - " )\n", - "emb_layers" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "NVTabular implemented a custom TensorFlow layer `layers.DenseFeatures`, which takes as an input the different `tf.Keras.Input` and pre-initialized `tf.feature_column` and automatically concatenate them into a flat tensor. In the case of multi-hot categorical features, `DenseFeatures` organizes the inputs `__values` and `__nnzs` to define a `RaggedTensor` and combine them. `DenseFeatures` can handle numeric inputs, as well, but MovieLens does not provide numerical input features." - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 16, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "emb_layer = layers.DenseFeatures(emb_layers)\n", - "x_emb_output = emb_layer(inputs)\n", - "x_emb_output" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We can see that the output shape of the concatenated layer is equal to the sum of the individual Embedding output dimensions (1040 = 16+512+512).\n" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'userId': (162542, 512), 'movieId': (56747, 512), 'genres': (21, 16)}" - ] - }, - "execution_count": 17, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "EMBEDDING_TABLE_SHAPES" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We add multiple Dense Layers. Finally, we initialize the `tf.keras.Model` and add the optimizer." - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "metadata": {}, - "outputs": [], - "source": [ - "x = tf.keras.layers.Dense(128, activation=\"relu\")(x_emb_output)\n", - "x = tf.keras.layers.Dense(128, activation=\"relu\")(x)\n", - "x = tf.keras.layers.Dense(128, activation=\"relu\")(x)\n", - "x = tf.keras.layers.Dense(1, activation=\"sigmoid\", name=\"output\")(x)\n", - "\n", - "model = tf.keras.Model(inputs=inputs, outputs=x)\n", - "model.compile(\"sgd\", \"binary_crossentropy\")" - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAABD8AAAIjCAYAAAAEDbCUAAAABmJLR0QA/wD/AP+gvaeTAAAgAElEQVR4nOzdeXxU1f3/8fckmYQskAWQnQJWwCIGDSBBVoMsig3yACLKpmwuiIi4V0uRaguodQFZbBX69VsCfL9SEUSgYhUSBCKICAEVNwiBACYmhCXL+f3RX+bLMAlkst2Zm9fz8ZjHg5zc3PncO/ecTN7cOcdhjDECAAAAAACwqQCrCwAAAAAAAKhOhB8AAAAAAMDWCD8AAAAAAICtEX4AAAAAAABbC7q4ITU1VS+99JIVtQBAmaZPn674+Phq2TfjHoCqtnLlymrZL+MV4Buq833J8OHDq2W/QG0SHx+v6dOnu7V53Pnx008/adWqVTVWFFBe27Zt07Zt26wuAxZYtWqVfvrpp2rbP+MeqsOqVat0+PBhq8tADTt8+HC1jieMV76D9yW1V3W/L+H3R9Wo7vEYvmvbtm1KTU31aPe486NEdf2PBVBRJSk412bt43A4auR5uLZQlRwOhx5++GGNGDHC6lJQg1asWKGkpKRqfx7GK+vxvqT2qon3Jfz+qLyS8Zg+WvuUdfcUc34AAAAAAABbI/wAAAAAAAC2RvgBAAAAAABsjfADAAAAAADYGuEHAAAAAACwNcIPAAAAAABga4QfAAAAAADA1gg/AAAAAACArRF+AAAAAAAAWyP8AAAAAAAAtkb4AQAAAAAAbI3wAwAAAAAA2BrhBwAAAAAAsDXCj1pg3rx5cjgccjgcat68udXl1KiIiAjXsZc85s2bZ3VZFWKnYwHgyU593E7HAv/g7TW3fPly13Z16tTx2Tp9mZ2OBTXDH/qpna5rOx1LVSH8qAVmzJghY4xiY2OtLqXG5eXladeuXZKkxMREGWM0Y8YMi6uqGDsdCwBPdurjdjoWVL+8vDxdddVVGjx4cKX24c01d8cdd8gYo4SEhAo/Z0XYqW/Y6VhQM/yhn9rpurbTsVQVwg/Ah0RERKhHjx5WlwEAl8V4hapijFFxcbGKi4utLgUXoZ8Dvo0+6p0gqwsAAABA7VW3bl19++23VpcBALA57vwAAAAAAAC2VmXhR3p6uoYMGaLIyEiFhYWpa9euev/999WvXz/XBCsTJkxwbZ+VlaWpU6eqVatWCg4OVsOGDTV06FDt3r3btc3q1avdJmj5/vvvlZSUpKioKNWvX1+DBw92+5+Ci7c/cOCARowYofr167vaTpw4Ue7nl6Rz587p2WefVfv27RUWFqaYmBjddttteu+991RUVFRVp0/Z2dkeE9LMnj1bklRYWOjWPmzYMFd7cnKybr75ZjVu3FihoaHq2LGjXnnllXLdOjp79mzXPi+8XWr9+vWu9gYNGnj8nK+du4rw9tq6eNLYHTt2KCEhQXXr1lVYWJj69u2rrVu3urb39tyW7P/06dPaunWra5ugoIrfnFWe66Mi151Usf57qf7orxj3Kq+yx3u57SUpKirK4zoveQQEBOjw4cM1ftzeYLxivKqMi4/thx9+UFJSkurWrav69etr9OjR+vnnn/X999/rtttuU926ddWkSRNNnDhRubm5Hvs7efKkpk+friuvvFLBwcGKjo7WoEGDtHnzZknev04X13f27Fm35yvvuHU5F47X4eHh6tmzp7Zs2VLBs1r16Of084qqyPt5b37f1eS59eV+Sh+1SR81F0lOTjalNF/S119/baKiokyzZs3Mhg0bTG5urtm7d6/p16+fadiwoQkJCXHbPiMjw/zqV78yjRo1MmvXrnVt37t3b1OnTh2TkpLitn1iYqKRZBITE01KSorJy8szGzduNKGhoaZLly4e9ZRs37t3b7N582Zz+vRps23bNhMYGGiysrK8ev4JEyaYyMhIs2HDBpOfn28yMzPNjBkzjCSzefNmr85TeQwcONAEBASYb775xuN78fHx5r//+79dX69Zs8ZIMs8//7w5deqUycrKMq+++qoJCAgwM2bM8Pj52NhY06xZM4/28PBwc+ONN3q0x8XFmfr167u1WXnuhg0bZoYNG+b1z+3atct1/VzM22srNjbWhIeHm/j4eNf2O3bsMNdee60JDg42H3/8sdv23pzbS21fnmO5mDfXx4ABAy553b3zzjuuryvaf8vqj+UhySQnJ5dr24pg3LN23KvM8ZZn+8jISJObm+vWNmvWLFf/qK7jrsh1y3jl/+NVRcYTb1R0/yXHNnToULNz506Tl5dnli1bZiSZQYMGmcTERLNr1y6Tm5trFi5caCSZhx9+2G0fR48eNa1btzaNGjUya9asMTk5OebAgQNm6NChxuFwmCVLlri29eb9zIX1nTlzxtXm7etX1jVX2ni9Z88e079/f9OqVSuP8bq8eF9Se/t5db8vqcj+vXlty/v7rjrObU3204qOl/RR/++jZY3PVRJ+DB8+3Egyq1atcms/fvy4CQsL87hYx44dayS5nRRj/vNLNSQkxMTFxbm1l5yANWvWeByUJI+TULL9unXrSq3Xm+dv3bq16d69u8c+2rZtWy1/BGzatMlIMvfff79b+5YtW0zLli1NQUGBq23NmjWmT58+HvsYNWqUcTqdJicnx629KsIPK89ddb7JKO+1FRsbaySZXbt2ubXv2bPHSDKxsbFu7VYPYOW9Pj788MMyr7tmzZqZ8+fPu9oq2n/L6o/l4YvhB+Ne1aro8ZZn+4vDj+TkZONwOMy4cePcfraqj7u6wg/GK98er3w9/Fi7dq1be4cOHYwk8+9//9utvXXr1qZdu3ZubePGjTOSzD/+8Q+39rNnz5qmTZua0NBQk5mZaYzx7v3MhfVdGH54+/qVdc2VNV4fOXLEhISE+GT4QT/37X7u7+FHeX/fVce5rcl+Wp3hB33Ut/toWeNzlXzsZf369ZKkAQMGuLU3bNhQ7du399h+9erVCggI8FjSrHHjxurQoYPS0tJctyFfqEuXLm5ft2jRQpKUkZFRal1du3Yttd2b5x84cKBSUlI0adIkbdu2zXUr2IEDB9SnT59S918ZCQkJuu666/T222/r5MmTrva5c+dq2rRpbrcyDR482HWb6YViY2NVUFCgr776qsrr8+VzVxneXFvh4eHq1KmTW1vHjh3VtGlTffHFFzp69Gj1FeoFb66P/v37q2PHjqVedw8++KCcTqerraL9t6z+6K8Y96qHt8dbnu2zs7MVEREhSfrss880duxY9erVS4sWLXL7WX8ZsxivGK8qo3Pnzm5fN23atNT2Zs2aeVxT7777riTp1ltvdWsPCQlRQkKCzpw5ow8//FCSd+9nylLR1+9iZY3XTZs2Vdu2bS/781agn9PPq1N5f9/V5Ln1t35KH/XPPlrp8OPcuXPKzc1VnTp1XG8uLxQdHe2xfU5OjoqLixUZGenxmaHPP/9ckvT111977CsyMtLt6+DgYEkqc36L8PDwUuv15vnnz5+vZcuW6dChQ0pISFC9evU0cOBA1xuA6vDII48oPz9fCxYskCQdPHhQn3zyidvcAZKUk5OjZ599Vh07dlR0dLTrGB599FFJUn5+fpXW5Q/nrqK8ubaioqJK3ccVV1whSTp+/HgVV1cx3l4f06ZN87juPvroI02aNMm1TWX6b2n90V8x7lUfb4/Xm+1//PFHJSYmqkWLFvrf//1f17Yl/GXMYrxivKqMevXquX0dEBCgwMBAhYWFubUHBga6XVMl57NOnTqqW7eux34bNWokScrMzHS1lff9TGkq8/pdvJ9LjdclfcHX0M/p59WpPL/vavLc+mM/pY/6Zx+tdPgREhKiunXr6uzZs8rLy/P4/sUvZkhIiKKiohQUFKSCggKZ/3z0xuPRt2/fypZWZr3ePL/D4dDo0aO1adMmZWdna/Xq1TLGaOjQoXrppZeqpcakpCS1aNFCr7/+us6dO6cXX3xREydO9Hizcdttt+m5557TxIkTdfDgQRUXF8sYo5dfflmSZIwp1/MFBATo/PnzHu3Z2dluX/vDuasJJ0+eLPXcllzrFw7Q5T23JRwORxVV6f31cdddd6lRo0Zu193YsWPd/pC3uv/6CsY9/5Obm6vBgweroKBA77//vmJiYjy2seNxM14xXlWVkJAQRUZG6uzZs6VOhHrs2DFJ//mfvBLlfT9T1vNVxet3ufH61KlTl63F19HP6eeSd69teX7f1eS5tXs/pY/6Th+tko+9DBo0SNL/3a5UIjMzUwcPHvTYfujQoSosLHSb4bbEn//8Z7Vs2VKFhYVVUVqpvHn+qKgopaenS5KcTqduvvlm10y0a9eurZb6goKC9NBDD+n48eN68cUXtXz5ck2dOtVtm6KiIm3dulWNGzfW1KlT1bBhQ9fFf+bMGa+er0mTJjpy5IhbW2Zmpn788UePbX393NWEs2fPaseOHW5tX375pTIyMhQbG6smTZq42r05t5IUFhbmNuC1a9dOixcv9qq+oKAgffXVV15fHyEhIbr//vtd190777yjhx56yGM7q/uvr2Dc8x9FRUW64447lJ6erv/5n/9xu322ZLUJyX7HLTFeWd3v7Ob222+XJI/+cO7cOf3rX/9SaGio2y3r5Xk/cylV9fqVNV6fOHFCBw4cKHc9vop+Tj+XvHtty/v7ribPrZ37KX3Ud/polYQfzz//vGJiYjRt2jRt3LhReXl52rt3r+6++263/wEo8cILL+jKK6/UPffcow8++EA5OTk6deqUFi1apFmzZmnevHmVWqbncrx9/nvvvVd79uzRuXPndPz4cc2ZM0fGGN10003VVuOkSZMUGRmp3/3udxoyZIiaNWvm9v3AwED16dNHmZmZmjt3rk6cOKEzZ85o8+bNWrhwoVfP1b9/f2VkZOj1119XXl6evv32Wz300EOl3mLmD+euukVGRuqpp55SamqqTp8+rZ07d2rUqFEKDg7WK6+84ratN+dWkq6//nodPHhQP/30k1JTU3Xo0CH17NnT6xoren3cf//9Cg0N1e9+9zv169dPv/71rz22sbr/+grGPf/x8MMPa926dVq8ePFl5+6w03FLjFdW9zu7eeGFF9S6dWtNmzZN77//vnJzc3Xw4EHdeeedOnr0qF555RXXx19KXO79zOWerypev9LG63379mnUqFGl3mLvb+jn9HPJ+9e2PL/vavLc2rmf0kd9qI9ePANqRWfFPXDggBkyZIipV6+eCQsLM927dzf//ve/TZ8+fUxYWJjH9idPnjTTp083bdq0MU6n0zRs2ND079/fbNy40bVNamqqkeT2ePrpp435zz03bo9bb7211O3LOpbyPL8xxuzevdtMnjzZXH311SYsLMzExMSYbt26mSVLlpji4mKvz5M3Hn30USPJfPHFF6V+Pysry0yePNm0aNHCOJ1O06hRIzNu3DjzxBNPuI49Li7OzJ07t8zzaIwx2dnZZsKECaZJkyYmNDTU9OjRw+zYscPExcW5tn/88cdd21t17ioyq3p4eLjHsc+dO9fra6tEyYo5+/btMwMGDDB169Y1oaGhpnfv3mbLli0ez+/tuU1PTzc9e/Y04eHhpkWLFmb+/PmXPJayHvv37y/39XGxiRMnljrz/4Uq2n8rMrYY45urvRjDuFcVquJ4L7X9zp07L9tf3n333Wo5bm+vW8Yre4xXvrbaS1nXz44dOzzaX3jhBfPpp596tP/+97937e/EiRNm2rRppnXr1sbpdJrIyEgzYMAA869//avMGi71fubdd9/1eL677rrL9f3yjltl9Z8SF47XJUtSvv/++yYhIcG1/fjx48t9Xo3hfUlt7ufeju81sX9vXltvft9V5bmt6X5akfGYPmqPPlrW+Owwxv3DOytWrFBSUlK554u4nPbt2+vMmTP64YcfqmR/qL2GDx8uSVq5cqVlNXTq1EknTpwo1+zy/uqtt97S/PnztXPnTqtLcXE4HEpOTtaIESOqZf+Me6gO1X3dXg7jlTWqejyp6f2j/HhfUjN8sZ9X9/hu9e8Pu/CF8ZI+ao2yxucq+dhLZmamYmJiVFBQ4Nb+/fff69tvv/Xb24WB2mjhwoWaPn261WX4PMY9wHqMV4D90c8B3+ZPfbRKwg9J+vnnnzV58mT99NNPys/P1/bt25WUlKR69erpmWeeqaqnAVDF3nzzTd1+++3Ky8vTwoUL9fPPP/M/DeXEuAfULMYrwP7o54Bv8+c+WiXhR+PGjV1LJfXq1UvR0dH67W9/q6uuukrbt29XmzZtquJpfNbF6xVX9DFz5kyrDwVlmDdvnhwOh7744gsdOXJEDodDv/vd76wuq8qsXr1a0dHReuONN7R8+fJaMTFYZTHuMe75KsYrwP7o54Bvo4/6piqrMiEhQQkJCVW1O7/C527tb8aMGZoxY4bVZVSLCRMmaMKECVaX4ZcY9+CLGK8A+6OfA76NPuqbquxjLwAAAAAAAL6I8AMAAAAAANga4QcAAAAAALA1wg8AAAAAAGBrhB8AAAAAAMDWCD8AAAAAAICtEX4AAAAAAABbI/wAAAAAAAC2RvgBAAAAAABsjfADAAAAAADYGuEHAAAAAACwNcIPAAAAAABga4QfAAAAAADA1oLK+sbw4cNrsg7gsrZt2yaJa7O67N27V/Xq1VNMTIwiIiKsLscSXFuoai+//LJWrlxZpfs8fvy4oqOj5XQ6q3S/qBqHDx+ukedhvKqYjIwMNWnSRA6Ho9L74n0JqpM3vz9yc3N14sQJBQcHq1mzZtVcmf8oGY/po7XPtm3b1K1bN492j/CjRYsWGjZsWI0UBXijtAsYVaOgoEDHjh3TwYMHVVxcrODgYMXExCgmJkbR0dGKiYlRSEiIZfUNGzZMLVq0qLb9M+6hOlTHNVVcXKydO3fK4XCoc+fOatiwYZU/ByqnefPm1TqeMF5VzNmzZ5WWlqbMzEz17t1bDRo0qPQ+eV9Se1X3+5LL9fGSsCMrK0tZWVk6c+aMgoKC1KZNG8KPC1T3eAzf1a1bN8XHx3u0O4wxxoJ6APiggoICHTx4UFu3btWWLVuUlpam/fv3yxijJk2aKC4uTnFxcerRo4e6d++usLAwq0sGap3jx49r8uTJ+uc//6mJEyfq5Zdfpi8Cl7Bu3TpNmDBBwcHBeuutt9S3b1+rSwK8cujQIW3ZskVbt27Vhx9+qB9++EFhYWG67rrr1KNHD/Xr1089e/a09D+qAH9A+AHgkn755Rft2bNHaWlp2rp1qz755BMdO3ZMgYGBateunSsQiYuL0w033MCt+EANWblypSZNmqSmTZvq73//u66//nqrSwJ8yi+//KJHH31Uixcv1vDhw7Vo0SJFR0dbXRZwWYQdQPUg/ADgtYyMDKWlpbkCkZSUFOXn5ysiIkKxsbFugUiHDh2sLhewrR9++EHjxo3Tli1b9Mgjj+i5554jgAQkbd26VWPHjlVubq4WLVqkIUOGWF0SUCbCDqBmEH4AqLSioiKlp6e7ApG0tDTt2LFD58+fd/u4TFxcnLp376769etbXTJgG8YYLVmyRA8//LA6duyoZcuWqW3btlaXBVji7NmzmjlzpubOnauBAwfqzTffVJMmTawuC3BD2AFYg/ADQLU4ffq0du3a5RaI7Nu3T5LUpEkT9ejRQzfeeKPi4uLUuXNn1alTx+KKAf+2b98+jR49Wvv379fvf/97PfroowoIYEV71B579uzR6NGj9f3332vu3LmaNGmS1SUBkgg7AF9B+AGgxmRmZmrHjh2uMCQ1NVUnT55UUFCQ2rZt6xaIXH311fzhBnipsLBQs2fP1uzZs3XTTTfpb3/7m5o3b251WUC1Kiws1Isvvqhnn31WXbp00dKlS3XllVdaXRZqMcIOwDcRfgCwVEZGhtvqMmlpaTp79qzq1aunjh07ulaX6dWrlxo1amR1uYBf+OyzzzRmzBgdO3ZMr732mkaPHm11SUC1OHTokMaOHaudO3dq5syZ3PEESxB2AP6B8AOATyksLNSBAwfcApH09HQVFxd7LLcbHx+v8PBwq0sGfNKZM2f0xBNP6LXXXtOwYcP0xhtvMN8ObKNkrpvp06erTZs2+vvf/67Y2Firy0ItQdgB+CfCDwA+z5vldrt27arg4GCrSwZ8xoYNGzR+/HgVFBTozTff1ODBg60uCaiUzMxMTZgwQR9++KEeeeQRzZo1i3Ef1YqwA7AHwg8AfunC5XbT0tK0ZcsWZWdnKzw8XJ06dXILRH7zm9/I4XBYXTJgmezsbE2dOlX/9V//pYkTJ+rFF19URESE1WUBXlu5cqXuvfdeRUVFaenSperRo4fVJcGGCDsAeyL8AGALl1puNyoqSp07d3ZNphofH68GDRpYXTJQ41auXKn77rtP9erV09KlS9WzZ0+rSwLKJTs7Ww8++KDeeecdTZw4US+99BIfe0SVIewAagfCDwC2Vdpyu/v375cxxmO53bi4OIWGhlpdMlDtMjMzNXHiRH3wwQeaMWMGHxmAz/vwww81fvx4FRUV6c0339Stt95qdUnwc4QdQO1E+AGgVsnOztbOnTtdk6lu27ZNJ06ccC23WzKZ6o033shyu7CtkskiH3nkEbVu3VrLli1Tp06drC4LcHPxpL0LFy5UTEyM1WXBDxF2AJAIPwDAY7ndzz//XGfOnFHdunV17bXXugKRnj17qnHjxlaXC1SZ7777TmPHjtWOHTs0c+ZMzZgxQ4GBgVaXBWjbtm0aM2aMsrKy9Nprr2nUqFFWlwQ/QtgBoDSEHwBwkZLldktWl9myZUupy+2WhCLR0dFWlwxUWFFRkebNm6dnn31WnTt31tKlS/XrX//a6rJQSxUUFOiPf/yjZs+erYSEBP3tb39Ts2bNrC4LPo6wA0B5EH4AQDnk5ubqiy++cAUin376qTIzM1luF7bx5ZdfavTo0fruu+80d+5cTZo0yeqSUMt89dVXGj16tNLT0/XCCy9o6tSprNSFUhF2AKgIwg8AqKCLl9vdunWrfv75ZzmdTl177bVuk6my3C78wdmzZzVz5kzNmzdPN998s/7617+qadOmVpcFmzPG6NVXX9Xjjz+u6667TkuXLlXbtm2tLgs+hLADQFUg/ACAKnKp5XYjIyPVpUsXVyDSrVs3NWzY0OqSgVKlpKRo7NixysnJ0aJFi3T77bdbXRJs6vvvv9e4ceOUkpKip556Ss888wzzzoCwA0C1IPwAgGpUUFCgPXv2uCZTvXi53QtXl2G5XfiSX375RY8++qgWL16s4cOHa9GiRcxvgyq1bNkyTZkyRS1bttSyZct0/fXXW10SLELYAaAmEH4AQA3LycnRjh07XIHIZ599pqysLJbbhU/64IMPNH78eDmdTr399tvq27ev1SXBzx0/flwTJ07UmjVr9OCDD2rOnDn8UVvLEHYAsALhBwD4gJL5Q0pWlyltud24uDj17NlTrVu3trpc1DJZWVmaPHmyVq9erYkTJ+rll19WWFiY1WXBD/3P//yP7r33XkVEROjtt99W7969rS4JNYCwA4AvIPwAAB/EcrvwRStXrtTkyZPVuHFjLVu2TJ07d7a6JPiJnJwcPfbYY1q8eLFGjx6t+fPnq27dulaXhWpC2AHAFxF+AICfuHC53bS0NG3ZskXfffddqcvtdunShTeVqBY//vijxo0bp08//VSPPPKInnvuOTmdTqvLgg/717/+pbvvvlvnzp3T4sWLlZiYaHVJqGKEHQD8AeEHAPixi5fbTUlJ0alTp1huF9XKGKMlS5bo4Ycf1jXXXKNly5apXbt2VpcFH1OydPLcuXN1++23a+HChWrQoIHVZaEKEHYA8EeEHwBgMyVvSksCkZ07d+rcuXOKjIzUNddc45pMleV2UVn79+/X6NGjtW/fPv3+97/Xo48+ygS9kCTt2LFDY8aMUUZGhubOnatJkyZZXRIqgbADgB0QfgCAzbHcLqpTYWGhXnzxRT3zzDPq3bu33nrrLTVv3tzqsmCRkuvh2WefVc+ePfXWW2+pRYsWVpcFLxF2ALAjwg8AqIVycnL05ZdfuiZTLW253ZJQ5LrrruN/83FZ27dv15gxY5SZmak5c+bwP/210P79+zVmzBjt3btXM2fO5E4gP0LYAaA2IPwAAEgqe7ndiIgIxcbGugUibdq0sbpc+KAzZ87oD3/4g+bOnauhQ4fqjTfeYI6HWqBkDpjp06frN7/5jZYtW6b27dtbXRYugbADQG1E+AEAKNWFy+2WhCK7du0qdbndG2+8UTExMVaXDB+xadMm3X333SooKNCSJUt02223WV0SqsmPP/6ou+++W5988okeeeQRzZo1S8HBwVaXhYsQdgAA4QcAwAt5eXnavXu3WyBy6NAhSVKbNm3cVpdhud3aLScnR4899pgWL16s0aNHa8GCBYqIiLC6LFShlStX6t5771V0dLSWLl2qG2+80eqS8P8RdgCAJ8IPAEClsNwuLmXVqlW69957VbduXS1dulS9evWyuiRUUnZ2th544AH94x//0MSJE/XSSy8pPDzc6rJqNcIOALg8wg8AQJUr73K7N9xwg6644gqry0U1O3bsmCZOnKi1a9dqypQpmjNnDn+E+an169dr/PjxMsbozTff1C233GJ1SbUSYQcAeI/wAwBQ7QoKCnTw4EHXZKqlLbdbMplq9+7dFRYWZnXJqAbLli3TAw88oFatWmnZsmW67rrrrC4J5ZSfn68nn3xSr732moYNG6aFCxcyz08NIuwAgMoj/AAAWOKXX37Rnj17XIHI9u3bdfz48VKX2+3UqZMCAwOtLhlV4LvvvtO4ceOUmpqqp556Ss888wyvrY9LTU3VmDFjdOLECb3++uu66667rC7J9gg7AKDqEX4AAHzGhfOHbN26VSkpKcrPz/dYbjcuLk4dOnSwulxUUHFxsV577TU9/vjjuv7667V06VJdddVVVpeFixQUFOiPf/yjZs+erX79+ulvf/ubmjZtanVZtkTYAQDVj/ADAOCzSltud/fu3SoqKvJYbrd79+6qX7++1SXDC3v37tXo0aN18OBBPf/885o6deolJy47ITsAACAASURBVMTNyMjgj+8qUFBQoJ9//vmS8+14+9rAO4QdAFDzCD8AAH7l4uV209LStG/fPkmey+127txZderUsbhiXEp57y7461//qpdeekk7d+5UaGioBZXax5NPPqnPP/9c69ev9wg0uCunehB2AID1CD8AAH7v6NGj2rlzpysMSU1N1cmTJ+V0OnXVVVe5VpdhuV3fdal5Jb777jtdc801ys/P1+TJk7Vw4UILK/VvmzZtUv/+/WWM0YIFC3Tfffe5vsd8LFWHsAMAfA/hBwDAli5ebjctLU1nz55VvXr11LFjR9dkqr169VKjRo2sLhcqfUWRqKgo9erVS9u3b1dBQYEkafny5UpKSrK4Wv+TlZWlDh066NSpUyoqKlJISIj27Nmjtm3bshJPJRF2AIDvI/wAANQKvrLcbmFhoYKCgqpl33axfv16jR8/XsYYDRw4UEuXLlVxcbEkyeFwKDw8XF9++aVatWplbaF+xBijwYMHa+PGja4QKSgoSB06dFCLFi20bt06TZkyRXPmzLHtH+gpKSlq0KCB2rZtW+l9EXYAgP8h/AAA1Foly+2WTKb6ySef6NixYwoMDFS7du3cJlS94YYb5HQ6K/2cGzZs0Jw5czR//ny1a9euCo7CnrKysnTnnXdq8+bNKioqcvue0+nUtddeq9TU1Cp5TWqDF198UY899pgrRCoREBCg+vXra9WqVerVq5dF1VWv/Px8Pf3003rllVc0b948TZ8+3et9EHYAgP8j/AAA4ALVvdzu7Nmz9cwzzygoKEiPPfaYnn766Wq7y8SfnT9/Xtddd50OHjyowsJCj+8HBgbqqaee0qxZsyyozr+kpaWpW7dupZ5H6T/nMjU1VV26dKnhyqrfxx9/rLFjxyojI0NFRUUaNGiQ1q5de9mfI+wAAPsh/AAA4BKKioqUnp7uNnfIjh07dP78+Qott3vLLbfoww8/VHFxsYKCghQTE6O5c+dqzJgxNXRE/uHJJ5/U3LlzPe76uJDD4dDGjRuVkJBQg5X5l7y8PMXGxuqHH34o81wGBQWpVatW2rNnj21W0snPz9cf/vAHzZ07VwEBAa5jDw8PV05OjsdEroQdAGB/hB8AAHjp9OnT2rVrV6nL7TZp0sRtdZmLl9utX7++Tp065fo6ICBAxcXFGjhwoBYsWKDWrVvX+PH4mpSUFPXs2dPjIxoXCwwMVP369fXVV1+pQYMGNVSdf7nrrru0cuVK1zwfZQkMDNS0adM0b968Gqqs+nz44YcaP368jh07VurdLjt37lR0dDRhBwDUMoQfAABUgczMTG3fvt3tkZOTo5CQEHXq1Eldu3ZVmzZt9PDDD5f6806nUw6HQ08++aSeeOIJt8CkNikoKFBsbKz279+v4OBgnT9//pLbO51O3XzzzXr//fdZwvgiy5Yt09ixYy+7XWBgoEreDm7fvl1xcXHVXVq1yMnJ0YwZM/TXv/5VDoej1PDM6XQqOjpax48fV3h4uLp3767evXurT58+6tq1K3PIAICNEX4AAFBNMjIy3FaX2bdvn7Kzs3WpX72BgYFq1qyZFi1apIEDB9Zgtb4jPz9fKSkp2rRpk9auXau9e/cqMDBQDoej1P/JDwgI0F/+8hc9+OCDFlTrm7755hvFxsbqzJkzHtebw+GQ0+nU+fPnFRoaqr59+yoxMVH9+/f32xV01q1bp3vuuUenTp265F0uAQEBuvbaa/X6668TdgBALUP4AQBADXnkkUf0+uuvX/ZuhsDAQBUVFemWW27RG2+8oZYtW9ZQhb4pMzNTGzZs0Jo1a7Rhwwb98ssvcjqdKiwsdP1h73Q69dlnn+m6666zuFrrFRQUqFu3bvryyy9dQUDJ+XI4HLrmmms0aNAg9evXT7179/brACA7O1uPPvqo3nzzTddHyC6nrHk/AAD2RvgBAPBLK1assLoErz3zzDM6ePBgubcv+R/6pKQkDRo0iD/WJBUXF+vQoUP64osvtGvXLn377beuP3ivuOIKzZs3r9bP07B06VKtW7fO9XWDBg10/fXXKzY2Vh06dLDNpKbbtm3TkiVLlJeX5/XPvvDCC2rTpk01VOW7unfvrubNm1tdBgBYhvADAOCXmN8BAMovOTlZI0aMsLoMALBMkNUFAABQUf70Zv6LL75Qp06dFBgYqMDAQBUUFLg+slGnTh01b95c7dq1069//Wu1bt1arVq1UuvWrdW6dWvVrVvX4ur9x6FDh9SsWTPL7v5YsWKFkpKSLjmvS3U6ePCgrrzyylp1l1BxcbGysrJ0/PhxHT16VMeOHdPx48eVkZGh48eP68iRIzp8+LBOnDjhmnNn8ODBWrNmjdWl1xjCYgAg/AAAoEb88MMP6t+/vyvQKHm0atVKDRs2tLo826htH2W4WNu2ba0uocYFBASoUaNGatSokTp27HjJbQsLC5WVlaWcnJwaqg4A4CsIPwAAqAG//e1v9dvf/tbqMoBaLSgoSE2aNFGTJk2sLgUAUMMCrC4AAAAAAACgOhF+AAAAAAAAWyP8AAAAAAAAtkb4AQAAAAAAbI3wAwAAAAAA2BrhBwAAAAAAsDXCDwAAAAAAYGuEHwAAAAAAwNYIPwAAAAAAgK0RfgAAAAAAAFsj/AAAAAAAALZG+AEAqLWWL18uh8Mhh8OhOnXqWF1OjUlOTlanTp0UGhrqOv69e/daXVatFRER4XodSh4BAQGKjo5WbGys7r//fqWlpVldZpUo7VjLerz55ptWlwsAsBHCDwBArXXHHXfIGKOEhASrS6kxW7du1ciRI9W/f39lZWXpm2++UfPmza0uq1bLy8vTrl27JEmJiYkyxqigoEDp6emaNWuW0tPT1blzZ919993Kz8+3uNrKKe1YS3v07t3b4koBAHZD+AEAQC2ycuVKGWP00EMPKSIiQldeeaV++uknXXPNNdX2nBEREerRo0e17d+OAgMD1ahRIyUmJuqjjz7SY489prffflsjR46UMcbq8myB6xIAapcgqwsAAAA156effpIk1a9f3+JK4I0//elP+ve//6333ntPy5cv18iRI60uqVp9/PHHVpcAALAZ7vwAAKAWKSoqsroEVIDD4dCUKVMkSQsWLLC4muozZcoUTZs2zeoyAAA2RPgBAKg10tPTNWTIEEVGRio8PFw9e/bUli1bytw+KytLU6dOVatWrRQcHKyGDRtq6NCh2r17t2ub1atXu03S+P333yspKUlRUVGqX7++Bg8erG+//dZtv+fOndOzzz6r9u3bKywsTDExMbrtttv03nvveYQT5amhPErq/Oc//ylJrslOu3Xr5vVzFRYWKjk5WTfffLMaN26s0NBQdezYUa+88oqKi4td282bN08Oh0OnT5/W1q1bXecoKOg/N57Onj3b1Xbhxw/Wr1/vam/QoEGZ5/rAgQMaMWKE6tev72o7ceKEV8fizWthtZJztG3bNhUUFLja7XSdlobr0revSwDwGwYAAD8kySQnJ5d7+6+//tpERUWZZs2amQ0bNpjc3FyzZ88e079/f9OqVSsTEhLitn1GRob51a9+ZRo1amTWrl1rcnNzzd69e03v3r1NnTp1TEpKitv2iYmJRpJJTEw0KSkpJi8vz2zcuNGEhoaaLl26uG07YcIEExkZaTZs2GDy8/NNZmammTFjhpFkNm/eXOEayqOkzjNnzlT4eNesWWMkmeeff96cOnXKZGVlmVdffdUEBASYGTNmeDxneHi4ufHGG8usqazvx8XFmfr165d5DL179zabN282p0+fNtu2bTOBgYEmKyvLq2Mp72tRXsnJyaYib6927drlun7KcubMGSPJSDIZGRnGGP+8TkuOtazHQw89VKF9c12WzdvxEgDsiPADAOCXvH0zP3z4cCPJrFq1yq39yJEjJiQkxCP8GDt2rJFk3nnnHbf2o0ePmpCQEBMXF+fWXvKHz5o1a9zahw0bZiSZrKwsV1vr1q1N9+7dPWps27at2x823tZQHmWFH94815o1a0yfPn089j1q1CjjdDpNTk6OW3t1/ZG5bt26UvfnzbGU97Uor+oMP/Lz8z3CD3+8Ti91rA888IBb+MF16a6i1yXhBwAYw8deAAC1wvr16yVJAwYMcGtv2rSp2rZt67H96tWrFRAQoMGDB7u1N27cWB06dFBaWpoOHz7s8XNdunRx+7pFixaSpIyMDFfbwIEDlZKSokmTJmnbtm2u29gPHDigPn36VLqGivDmuQYPHqzNmzd77CM2NlYFBQX66quvqqSmy+natWup7d4cS3lfC19w9OhRSZLT6XR97MLu1ynXpe9flwDgL1jtBQBge+fOnVNubq7q1KmjiIgIj+9fccUVOnjwoNv2OTk5kqTIyMgy9/v111+refPmbm0Xbx8cHCxJbnMOzJ8/X/Hx8Vq6dKkSEhIkST179tTkyZN1++23V7oGb3n7XDk5OXrxxRf17rvv6vDhw8rOznbbLj8/v1L1lFd4eLhHm7fHUp7XwleUzE8THx8vp9Npy+v09ddfd/2b69I/rksA8Bfc+QEAsL2QkBDVrVtXZ8+eVV5ensf3T5065bF9VFSUgoKCVFBQIPOfj4l6PPr27VuhehwOh0aPHq1NmzYpOztbq1evljFGQ4cO1UsvvVQjNVTmeG+77TY999xzmjhxog4ePKji4mIZY/Tyyy9LkowxHsd7KQEBATp//rxH+8V/vFbHsZTntfAFxcXFmj9/viTpgQcekGT/65Tr0vevSwDwJ4QfAIBaYdCgQZL+7+MvJU6cOKEDBw54bD906FAVFhZq69atHt/785//rJYtW6qwsLBCtURFRSk9PV3Sfz7CcPPNN7tWjFi7dm2N1HCx8j5XUVGRtm7dqsaNG2vq1Klq2LCh64/IM2fOlLrvsLAwtz8i27Vrp8WLF7u+btKkiY4cOeL2M5mZmfrxxx+r9Vik8r8WVnvyySe1fft23X777Ro+fLir3e7XKdelb1+XAOBXqm76EAAAao68nMDvm2++MTExMW6rvXz11VdmwIAB5oorrvCY8PTYsWPmyiuvNG3atDHr1q0z2dnZ5uTJk2bhwoUmLCzM47nLmkj08ccfN5LMrl27XG2RkZGmd+/e5osvvjBnz541x44dMzNnzjSSzOzZsytcQ3mUVac3z3XTTTcZSWbOnDkmKyvL5Ofnm48++si0bNnSSDIbN2502/fAgQNNZGSk+fHHH01KSooJCgoy+/btc31/ypQpRpJ57bXXTG5urvnmm2/MiBEjTLNmzS45seTFx1CRYynva1FeVTXhaVFRkTl27JhZvXq163zfc889Jj8/v8LHaoxvXKflmdy1Ivvmuiybt+MlANgR4QcAwC9V5M38gQMHzJAhQ0y9evVcS3u+//77JiEhwbWKxvjx413bnzx50kyfPt20adPGOJ1O07BhQ9O/f3+3P6JSU1M9lup8+umnXTVe+Lj11luNMcbs3r3bTJ482Vx99dUmLCzMxMTEmG7dupklS5aY4uJit5rLU0N5vPvuu6UuK5qamur1c2VlZZnJkyebFi1aGKfTaRo1amTGjRtnnnjiCdd+L1y5Ij093fTs2dOEh4ebFi1amPnz57vtLzs720yYMME0adLEhIaGmh49epgdO3aYuLg41/4ef/zxUs91WUFDeY/Fm9eiPCoSfoSHh3sck8PhMJGRkaZjx47mvvvuM2lpaWX+vD9dp6Uda6NGjS55frguK39dEn4AgDEOYy76ACQAAH7A4XAoOTlZI0aMsLoUwGXFihVKSkrymF8CsBLjJQAw5wcAAAAAALA5wg8AAAAAAGBrhB8AAPg5h8Nx2cfMmTOtLhMAAMAyQVYXAAAAKof5JQAAAC6NOz8AAAAAAICtEX4AAAAAAABbI/wAAAAAAAC2RvgBAAAAAABsjfADAAAAAADYGuEHAAAAAACwNcIPAAAAAABga4QfAAAAAADA1gg/AAAAAACArRF+AAAAAAAAWyP8AAAAAAAAtkb4AQAAAAAAbI3wAwAAAAAA2FqQ1QUAAFBRqampVpcAuCm5JlesWGFxJQAA4EIOY4yxuggAALzlcDisLgEA/EZycrJGjBhhdRkAYBnu/AAA+CWye/gih8PBH5kAAPgg5vwAAAAAAAC2RvgBAAAAAABsjfADAAAAAADYGuEHAAAAAACwNcIPAAAAAABga4QfAAAAAADA1gg/AAAAAACArRF+AAAAAAAAWyP8AAAAAAAAtkb4AQAAAAAAbI3wAwAAAAAA2BrhBwAAAAAAsDXCDwAAAAAAYGuEHwAAAAAAwNYIPwAAAAAAgK0RfgAAAAAAAFsj/AAAAAAAALZG+AEAAAAAAGyN8AMAAAAAANga4QcAAAAAALA1wg8AAAAAAGBrhB8AAAAAAMDWCD8AAAAAAICtEX4AAAAAAABbI/wAAAAAAAC2RvgBAAAAAABsjfADAAAAAADYGuEHAAAAAACwNcIPAAAAAABga4QfAAAAAADA1gg/AAAAAACArRF+AAAAAAAAWyP8AAAAAAAAthZkdQEAAAD+6B//+Idyc3M92jdt2qTs7Gy3tiFDhuiKK66oqdIAAMBFHMYYY3URAAAA/mbs2LFatmyZnE6nq624uFgOh0MOh0OSVFRUpPDwcGVlZSkkJMSqUgEAqPX42AsAAEAFjBw5UpJUUFDgehQVFamwsND1dWBgoIYPH07wAQCAxQg/AAAAKqBfv36KiYm55DYFBQW68847a6giAABQFsIPAACACggKCtLIkSPdPvZysfr166tPnz41VxQAACgV4QcAAEAFjRw5UgUFBaV+Lzg4WKNHj1ZgYGANVwUAAC7GhKcAAAAVZIxR8+bNlZGRUer3P/vsM3Xt2rWGqwIAABfjzg8AAIAKcjgcGjNmTKkffWnRooW6dOliQVUAAOBihB8AAACVUNpHX5xOp8aNG+da8hYAAFiLj70AAABUUvv27XXgwAG3tr1796pDhw4WVQQAAC7EnR8AAACVNHr0aLePvvzmN78h+AAAwIcQfgAAAFTSyJEjVVhYKOk/H3kZO3asxRUBAIAL8bEXAACAKtC5c2d9/vnnkqTvvvtOv/rVryyuCAAAlODODwAAgCowZswYGWPUtWtXgg8AAHwMd34AAAAXVieB1ZKTkzVixAirywAA2EyQ1QUAAADfMm3aNMXHx1tdhl9JTU3VX/7yF3Xq1En333+/IiMjrS7JLyUlJVldAgDApgg/AACAm/j4eP7nvQL+8pe/aMWKFbrqqqusLsVvEX4AAKoLc34AAABUEYIPAAB8E+EHAAAAAACwNcIPAAAAAABga4QfAAAAAADA1gg/AAAAAACArRF+AAAAAAAAWyP8AAAAAAAAtkb4AQAAAAAAbI3wAwAAAAAA2BrhBwAAAAAAsDXCDwAAAAAAYGuEHwAAAAAAwNYIPwAAAAAAgK0RfgAAgCq1fPlyORwOORwO1alTx+pyfFZERITrPJU8AgICFB0drdjYWN1///1KS0uzukwAAGyB8AMAAFSpO+64Q8YYJSQkWF2KT8vLy9OuXbskSYmJiTLGqKCgQOnp6Zo1a5bS09PVuXNn3X333crPz7e4WgAA/BvhBwAAgI8IDAxUo0aNlJiYqI8++kiPPfaY3n77bY0cOVLGGKvLAwDAbxF+AAAA+Kg//elPuuGGG/Tee+9p+fLlVpcDAIDfIvwAAADwUQ6HQ1OmTJEkLViwwOJqAADwX4QfAACgUtLT0zVkyBBFRkYqPDxcPXv21JYtW8rcPisrS1OnTlWrVq0UHByshg0baujQodq9e7drm9WrV7tNBPr9998rKSlJUVFRql+/vgYPHqxvv/3Wbb/nzp3Ts88+q/bt2yssLEwxMTG67bbb9N5776moqMjrGnxFjx49JEnbtm1TQUGBq53zCABA+RF+AACACvvmm28UHx+vnTt3atWqVTp27JgWLFig5557zuOPakk6evSounTpohUrVmjBggU6deqUPv74Y506dUrx8fFKTU2VJA0ZMkTGGCUmJkqSpk2bpmnTpunIkSNKTk7WRx99pJEjR7rte8qUKXr11Vf12muv6eTJk9q/f7/at2+vxMREffrpp17X4CsaN24sSSosLNSJEyckcR4BAPCaAQAA+P8kmeTk5HJvP3z4cCPJrFq1yq39yJEjJiQkxISEhLi1jx071kgy77zzjlv70aNHTUhIiImLi3NrT0xMNJLMmjVr3NqHDRtmJJmsrCxXW+vWrU337t09amzbtq3ZvHlzhWsoj+TkZFORt1W7du0ykkxiYmKZ2+Tn5xtJRpLJyMgwxtj3PHp7/QEAUF7c+QEAACps/fr1kqQBAwa4tTdt2lRt27b12H716tUKCAjQ4MGD3dobN26sDh06KC0tTYcPH/b4uS5durh93aJFC0lSRkaGq23gwIFKSUnRpEmTtG3bNtdHNA4cOKA+ffpUugarHD16VJLkdDrVoEEDSZxHAAC8RfgBAAAq5Ny5c8rNzVWdOnUUERHh8f0rrrjCY/ucnBwVFxcrMjLSbS4Kh8Ohzz//XJL09ddfe+wrMjLS7evg4GBJUnFxsatt/vz5WrZsmQ4dOqSEhATVq1dPAwcO1LvvvlslNVilZP6U+Ph4OZ1OziMAABVA+AEAACokJCREdevW1dmzZ5WXl+fx/VOnTnlsHxUVpaCgIBUUFMgYU+qjb9++FarH4XBo9OjR2rRpk7Kzs7V69WoZYzR06FC99NJLNVJDVSsuLtb8+fMlSQ888IAkziMAABVB+AEAACps0KBBkv7v4y8lTpw4oQMHDnhsP3ToUBUWFmrr1q0e3/vzn/+sli1bqrCwsEK1REVFKT09XdJ/PiJy8803u1Y7Wbt2bY3UUNWefPJJbd++XbfffruGDx/uauc8AgDgHcIPAABQYc8//7xiYmI0bdo0bdy4UXl5edq3b59GjRpV6kdhXnjhBV155ZW655579MEHHygnJ0enTp3SokWLNGvWLM2bN09BQUEVrufee+/Vnj17dO7cOR0/flxz5syRMUY33XRTjdVQGcXFxTp+/Lj++c9/KiEhQXPmzNE999yjd955Rw6Ho8aOwd/PIwAAHmpmXlUAAOAPVIHVNg4cOGCGDBli6tWrZ0JDQ02XLl3M+++/bxISElyrlIwfP961/cmTJ8306dNNmzZtjNPpNA0bNjT9+/c3GzdudG2Tmprq+tmSx9NPP+2q8cLHrbfeaowxZvfu3Wby5Mnm6quvNmFhYSYmJsZ069bNLFmyxBQXF7vVXJ4avFGR1V7Cw8M9jsXhcJjIyEjTsWNHc99995m0tLQyf96O57Ei1x8AAOXhMMaYGktaAACAT3M4HEpOTtaIESOsLsWvrFixQklJSeJtVeVw/QEAqgsfewEAAAAAALZG+AEAAAAAAGyN8AMAAAAAANga4QcAAAAAALA1wg8AAAAAAGBrhB8AAAAAAMDWCD8AAAAAAICtEX4AAAAAAABbI/wAAAAAAAC2RvgBAAAAAABsjfADAAAAAADYGuEHAAAAAACwNcIPAAAAAABga4QfAAAAAADA1gg/AAAAAACArRF+AAAAAAAAWyP8AAAAAAAAthZkdQEAAMC3JCUlKSkpyeoy/JLD4bC6BAAAUArCDwAA4JKcnGx1CX4tKSlJ06ZNU3x8vNWl+K3u3btbXQIAwIYcxhhjdREAAAB24HA4lJycrBEjRlhdCgAAuABzfgAAAAAAAFsj/AAAAAAAALZG+AEAAAAAAGyN8AMAAAAAANga4QcAAAAAALA1wg8AAAAAAGBrhB8AAAAAAMDWCD8AAAAAAICtEX4AAAAAAABbI/wAAAAAAAC2RvgBAAAAAABsjfADAAAAAADYGuEHAAAAAACwNcIPAAAAAABga4QfAAAAAADA1gg/AAAAAACArRF+AAAAAAAAWyP8AAAAAAAAtkb4AQAAAAAAbI3wAwAAAAAA2BrhBwAAAAAAsDXCDwAAAAAAYGuEHwAAAAAAwNYIPwAAAAAAgK0RfgAAAAAAAFsj/AAAAAAAALZG+AEAAAAAAGyN8AMAAAAAANga4QcAAAAAALA1wg8AAAAAAGBrhB8AAAAAAMDWCD8AAAAAAICtBVldAAAAgD/Kzs6WMcaj/fTp0/r555/d2iIiIuR0OmuqNAAAcBGHKe23NgAAAC6pb9+++vjjjy+7XWBgoA4fPqzGjRtXf1EAAKBUfOwFAACgAkaOHCmHw3HJbQICAtSrVy+CDwAALEb4AQAAUAHDhw9XYGDgJbdxOBwaM2ZMDVUEAADKQvgBAABQAdHR0erfv/8lA5CAgAANGTKkBqsCAAClIfwAAACooFGjRqm4uLjU7wUFBemWW25RVFRUDVcFAAAuRvgBAABQQYmJiQoJCSn1e8XFxRo1alQNVwQAAEpD+AEAAFBBYWFhGjJkSKnL2IaEhOjWW2+1oCoAAHAxwg8AAIBKuOuuu1RQUODW5nQ6NXz4cIWGhlpUFQAAuBDhBwAAQCUMGDBA9erVc2srKCjQnXfeaVFFAADgYoQfAAAAleB0OjVy5EgFBwe72qKiopSQkGBhVQAA4EKEHwAAAJU0cuRInT9/XtJ/wpC77rpLQUFBFlcFAABKOIwxxuoiAAAA/FlxcbGaNm2qY8eOSZI+/fRT9ejRw+KqAABACe78AAAAqKSAgADXsrZNmjTRjTfeaHFFAADgQtyPCQBALTB8+HCrS7C9n3/+WZJUr149jRgxwuJq7G/69OmKj4+3ugwAgJ/gzg8AAGqBVatW6fDhw1aXYWvR0dGqV6+eWrZsWeY2hw8f1qpVq2qwKntatWqVfvrpJ6vLAAD4Ee78AACglnj44Ye5I6GarVix4pLneMWKFUpKStLKlStrsCr7cTgcVpcAAPAz3PkBAABQRQiXAADwTYQfAAAAAADA1gg/AAAAAACArRF+AAAAAAAAWyP8AAAAAAAAtkb4AQAAAAAAbI3wAwAAAAAA/Zl1igAAH6tJREFU2BrhBwAAAAAAsDXCDwAAAAAAYGuEHwAAAAAAwNYIPwAAAAAAgK0RfgAAAAAAAFsj/AAAAAAAALZG+AEAAMpl+fLlcjgccjgcqlOnjtXl1Kh16/5fe/ce5FV53w/8c2Avct0V5aKIAa0EQy02SOoamSgbBQt2keESDEokjvcoY60daupYkglKGVOTwtSYdoxTrIvOiKg/bVCMiQJqULSJLrEY2wgot8Jw0c0u+/z+SPnWr7smsMB+4fB6zZwZ9znP95w3e/aP3bfnOef/xeDBg6OsrKzDz929e/fC933v1qlTpzj22GNj2LBhcd1118WqVas6PBcAHEmUHwDAPvnKV74SKaWora0tdZQOs3bt2viLv/iLmDVrVnzwwQclybBz58547bXXIiKirq4uUkrR1NQUDQ0NMXv27GhoaIizzjorrrjiiti9e3dJMgLA4U75AQDwKf72b/82zjnnnFi1alX06NGj1HEKOnfuHH379o26urpYtmxZ3HrrrXH//ffH1KlTI6VU6ngAcNjp+Hs3AQCOEP/8z/8cXbp0KXWMP+jOO++M559/PpYsWRIPPfRQTJ06tdSRAOCw4s4PAIBPcSQUHxERWZbFDTfcEBERCxYsKHEaADj8KD8AgDY1NDTE+PHjo6qqKrp16xYjR46MF1544VPnb9q0KW688cYYOHBgVFRURO/evWPChAmxevXqwpzFixcXPbjz3XffjSlTpkR1dXUcd9xxMW7cuFi7dm3RcRsbG+P222+PIUOGRNeuXaNXr15x8cUXx5IlS2LPnj37nSGvzj333IiIWLlyZTQ1NRXGXRcAUH4AAG34z//8z6ipqYmf//zn8cgjj8QHH3wQCxYsiG9961ut/giOiNiwYUOMGDEiFi1aFAsWLIitW7fGT37yk9i6dWvU1NTEihUrIiJi/PjxkVKKurq6iIiYOXNmzJw5M9atWxf19fWxbNmyVks2brjhhvje974X3//+92PLli3x1ltvxZAhQ6Kuri5+9rOf7XeGvOrXr19ERDQ3N8fmzZsjwnUBgIIEAOReRKT6+vp9nj9p0qQUEemRRx4pGl+3bl2qrKxMlZWVRePTp09PEZEWLlxYNL5hw4ZUWVmZhg8fXjReV1eXIiI9/vjjReMTJ05MEZE2bdpUGBs0aFA655xzWmUcPHhweu6559qdYX/1798/de7c+YCOUV9fn9rz69drr72WIiLV1dV96pzdu3eniEgRkdavX59Syu912d+fZwBw5wcA0MrTTz8dERGjR48uGj/xxBNj8ODBreYvXrw4OnXqFOPGjSsa79evXwwdOjRWrVoV7733XqvPjRgxoujrAQMGRETE+vXrC2NjxoyJ5cuXx1VXXRUrV64sLKlYs2ZNnHfeeQecIS82bNgQERHl5eVx/PHHR4TrAgB7KT8AgCKNjY2xY8eOOOaYY6J79+6t9vfp06fV/O3bt0dLS0tUVVUVPTsiy7J49dVXIyLi7bffbnWsqqqqoq8rKioiIqKlpaUwNn/+/HjggQfinXfeidra2ujZs2eMGTMmHn300YOSIS/2Po+lpqYmysvLXRcA+BjlBwBQpLKyMnr06BEfffRR7Ny5s9X+rVu3tppfXV0dZWVl0dTUFCmlNrfzzz+/XXmyLIvLLrssnnnmmdi2bVssXrw4UkoxYcKEuPvuuzskw+GupaUl5s+fHxER119/fUS4LgDwccoPAKCViy66KCL+b/nLXps3b441a9a0mj9hwoRobm6OF198sdW+u+66K04++eRobm5uV5bq6upoaGiIiN8t6bjgggsKbyd58sknOyTD4W7WrFnx8ssvxyWXXBKTJk0qjLsuAPA7yg8AoJXvfOc70atXr5g5c2YsXbo0du7cGW+++WZMmzatzaUwc+bMiVNPPTVmzJgRTz31VGzfvj22bt0a9957b8yePTvmzZsXZWVl7c5zzTXXxBtvvBGNjY2xcePGmDt3bqSUYtSoUR2W4XDS0tISGzdujMceeyxqa2tj7ty5MWPGjFi4cGFkWVaY57oAwP/qmOeqAgClFO14O8aaNWvS+PHjU8+ePVOXLl3SiBEj0hNPPJFqa2sLbxX5+te/Xpi/ZcuWdPPNN6dTTjkllZeXp969e6cLL7wwLV26tDBnxYoVhc/u3W677bZCxo9vY8eOTSmltHr16nT11Ven008/PXXt2jX16tUrnX322em+++5LLS0tRZn3JcP+ePzxx1vl2rvdd999+3289rztpVu3bq3OnWVZqqqqSmeccUa69tpr06pVqz7183m8Lu35eQbg6JallFIHdCwAQAllWRb19fUxefLkUkc5qi1atCimTJkSfv06MH6eAdhflr0AAAAAuab8AAAAAHJN+QEAHFWyLPuD2x133FHqmADAQeTR2gDAUcXzNgDg6OPODwAAACDXlB8AAABArik/AAAAgFxTfgAAAAC5pvwAAAAAck35AQAAAOSa8gMAAADINeUHAAAAkGvKDwAAACDXlB8AAABArik/AAAAgFxTfgAAAAC5pvwAAAAAcq2s1AEAgI7x3e9+Nx5++OFSxziqvffeexERMWnSpBInAYCjS5ZSSqUOAQAcWv7Y7hg//elP4/TTT4/evXuXOkru3XzzzVFTU1PqGAAcIZQfAAAHSZZlUV9fH5MnTy51FADgYzzzAwAAAMg15QcAAACQa8oPAAAAINeUHwAAAECuKT8AAACAXFN+AAAAALmm/AAAAAByTfkBAAAA5JryAwAAAMg15QcAAACQa8oPAAAAINeUHwAAAECuKT8AAACAXFN+AAAAALmm/AAAAAByTfkBAAAA5JryAwAAAMg15QcAAACQa8oPAAAAINeUHwAAAECuKT8AAACAXFN+AAAAALmm/AAAAAByTfkBAAAA5JryAwAAAMg15QcAAACQa8oPAAAAINeUHwAAAECuKT8AAACAXFN+AAAAALmm/AAAAAByTfkBAAAA5JryAwAAAMi1LKWUSh0CAOBIc/XVV8eaNWuKxl588cX47Gc/G8cff3xhrHPnzvGjH/0oTjrppI6OCAD8r7JSBwAAOBL16dMnfvCDH7Qa/+Uvf1n09aBBgxQfAFBilr0AALTDV7/61T84p6KiIr72ta8d+jAAwO9l2QsAQDsNHTo03nrrrfh9v06tWbMmBg8e3IGpAIBPcucHAEA7XX755dG5c+c292VZFn/yJ3+i+ACAw4DyAwCgnS699NLYs2dPm/vKyspi+vTpHZwIAGiLZS8AAAfg7LPPjldeeSVaWlqKxrMsi9/85jfRv3//EiUDAPZy5wcAwAG4/PLLI8uyorFOnTrFF7/4RcUHABwmlB8AAAdg8uTJrcayLIvLL7+8BGkAgLYoPwAADsDxxx8ftbW1rR58OmHChBIlAgA+SfkBAHCApk2bVnjdbefOnWPMmDFx3HHHlTgVALCX8gMA4ACNHz8+ysvLIyIipRTTpk0rcSIA4OOUHwAAB6hHjx5x8cUXR0RERUVF4b8BgMNDWakDAACH3qJFi0odIfcGDhwYERGf//zn48knnyxtmKPAOeecEyeddFKpYwBwhMjS3gWqAEBuffJVrHCkq6+vb/NNOwDQFsteAOAoUV9fHykl2yHc/vIv/zIaGxs/dX99fX1ERMlzHukbAOwv5QcAwEHyrW99KyoqKkodAwD4BOUHAMBB0qVLl1JHAADaoPwAAAAAck35AQAAAOSa8gMAAADINeUHAAAAkGvKDwAAACDXlB8AAABArik/AAAAgFxTfgAAAAC5pvwAAAAAck35AQAAAOSa8gMAAADINeUHALBPHnroociyLLIsi2OOOabUcQ65//mf/4l/+qd/ilGjRkWvXr2iS5cucdppp8VXv/rVeP311zssR/fu3Qvf971bp06d4thjj41hw4bFddddF6tWreqwPABwJFJ+AAD75Ctf+UqklKK2trbUUTrEX/3VX8U3vvGNqKurizfffDO2bNkS//Iv/xKrV6+O4cOHx+LFizskx86dO+O1116LiIi6urpIKUVTU1M0NDTE7Nmzo6GhIc4666y44oorYvfu3R2SCQCONMoPAIBPMWPGjLjpppuiX79+0bVr1xg5cmQ8+OCDsWfPnrj11ltLlqtz587Rt2/fqKuri2XLlsWtt94a999/f0ydOjVSSiXLBQCHq7JSBwAAOBz98Ic/bHN82LBh0aVLl1i7dm2klCLLsg5O1tqdd94Zzz//fCxZsiQeeuihmDp1aqkjAcBhxZ0fAAD7YdeuXfHhhx/GH//xHx8WxUdERJZlccMNN0RExIIFC0qcBgAOP8oPAKBNDQ0NMX78+Kiqqopu3brFyJEj44UXXvjU+Zs2bYobb7wxBg4cGBUVFdG7d++YMGFCrF69ujBn8eLFRQ/ufPfdd2PKlClRXV0dxx13XIwbNy7Wrl1bdNzGxsa4/fbbY8iQIdG1a9fo1atXXHzxxbFkyZLYs2fPfmc4UA8//HBERNx2220H7ZgHw7nnnhsREStXroympqbC+NFyXQDg90oAQO5FRKqvr9/n+W+//Xaqrq5O/fv3Tz/+8Y/Tjh070htvvJEuvPDCNHDgwFRZWVk0f/369ekzn/lM6tu3b3ryySfTjh070i9+8Yv0pS99KR1zzDFp+fLlRfPr6upSRKS6urq0fPnytHPnzrR06dLUpUuXNGLEiKK5V155Zaqqqko//vGP0+7du9P777+fbrnllhQR6bnnnmt3hvZ4//33U9++fdOVV17Zrs/X19en9vz69dprrxW+X5/mww8/TBGRIiKtX78+pZTf67K/P88AoPwAgKPA/v6xOGnSpBQR6ZFHHikaX7duXaqsrGxVfkyfPj1FRFq4cGHR+IYNG1JlZWUaPnx40fjeP7Iff/zxovGJEyemiEibNm0qjA0aNCidc845rTIOHjy46I/s/c2wvzZv3pzOPPPMNGXKlNTc3NyuYxzK8mP37t2tyo+8XhflBwD7y7IXAKCVp59+OiIiRo8eXTR+4oknxuDBg1vNX7x4cXTq1CnGjRtXNN6vX78YOnRorFq1Kt57771WnxsxYkTR1wMGDIiIiPXr1xfGxowZE8uXL4+rrroqVq5cWVhSsWbNmjjvvPMOOMO+2LVrV4wePTo+97nPxcKFC6Nz587tOs6htGHDhoiIKC8vj+OPPz4i8n9dAGBfKT8AgCKNjY2xY8eOOOaYY6J79+6t9vfp06fV/O3bt0dLS0tUVVUVPTsiy7J49dVXIyLi7bffbnWsqqqqoq8rKioiIqKlpaUwNn/+/HjggQfinXfeidra2ujZs2eMGTMmHn300YOS4Q9pbm6OSZMmRf/+/eNHP/rRYVl8RETheSw1NTVRXl6e++sCAPtD+QEAFKmsrIwePXrERx99FDt37my1f+vWra3mV1dXR1lZWTQ1NUX63bLaVtv555/frjxZlsVll10WzzzzTGzbti0WL14cKaWYMGFC3H333Yc8w9VXXx2NjY2xaNGiKCsrK4z/0R/9UaxcubJd/6aDraWlJebPnx8REddff31E5P+6AMD+UH4AAK1cdNFFEfF/y1/22rx5c6xZs6bV/AkTJkRzc3O8+OKLrfbdddddcfLJJ0dzc3O7slRXV0dDQ0NE/G5JxwUXXFB4O8mTTz55SDPccccd8ctf/jIee+yxqKysbFf+jjBr1qx4+eWX45JLLolJkyYVxvN6XQBgfyk/AIBWvvOd70SvXr1i5syZsXTp0ti5c2e8+eabMW3atDaXwsyZMydOPfXUmDFjRjz11FOxffv22Lp1a9x7770xe/bsmDdvXtFdE/vrmmuuiTfeeCMaGxtj48aNMXfu3EgpxahRow5Zhvvvvz/+7u/+Ll566aXo0aNHqyUbn3z1a0dqaWmJjRs3xmOPPRa1tbUxd+7cmDFjRixcuDCyLCvMy+N1AYB26ZjnqgIApRTteDvGmjVr0vjx41PPnj0Lrzp94oknUm1tbeGtIl//+tcL87ds2ZJuvvnmdMopp6Ty8vLUu3fvdOGFF6alS5cW5qxYsaLw2b3bbbfdVsj48W3s2LEppZRWr16drr766nT66aenrl27pl69eqWzzz473XfffamlpaUo875k2Fdjx45tlemT24oVK/brmO1520u3bt1anTfLslRVVZXOOOOMdO2116ZVq1Z96ufzdl32ZvK2FwD2R5ZSSoe8YQEASirLsqivr4/JkyeXOspRbdGiRTFlypTw69eB8fMMwP6y7AUAAADINeUHAAAAkGvKDwDgqPLJB5e2td1xxx2ljgkAHEQerQ0AHFU8bwMAjj7u/AAAAAByTfkBAAAA5JryAwAAAMg15QcAAACQa8oPAAAAINeUHwAAAECuKT8AAACAXFN+AAAAALmm/AAAAAByTfkBAAAA5JryAwAAAMg15QcAAACQa8oPAAAAINfKSh0AAOgYK1asKHWEo97ea7Bo0aISJwGAo0uWUkqlDgEAHFpZlpU6AhxU9fX1MXny5FLHAOAI4c4PADgK+H8dHSPLMn+UA8BhyDM/AAAAgFxTfgAAAAC5pvwAAAAAck35AQAAAOSa8gMAAADINeUHAAAAkGvKDwAAACDXlB8AAABArik/AAAAgFxTfgAAAAC5pvwAAAAAck35AQAAAOSa8gMAAADINeUHAAAAkGvKDwAAACDXlB8AAABArik/AAAAgFxTfgAAAAC5pvwAAAAAck35AQAAAOSa8gMAAADINeUHAAAAkGvKDwAAACDXlB8AAABArik/AAAAgFxTfgAAAAC5pvwAAAAAck35AQAAAOSa8gMAAADINeUHAAAAkGvKDwAAACDXlB8AAABArik/AAAAgFwrK3UAAIAj0b/927/Fjh07Wo0/88wzsW3btqKx8ePHR58+fToqGgDwCVlKKZU6BADAkWb69OnxwAMPRHl5eWGspaUlsiyLLMsiImLPnj3RrVu32LRpU1RWVpYqKgAc9Sx7AQBoh6lTp0ZERFNTU2Hbs2dPNDc3F77u3LlzTJo0SfEBACWm/AAAaIcvf/nL0atXr987p6mpKS699NIOSgQAfBrlBwBAO5SVlcXUqVOLlr180nHHHRfnnXdex4UCANqk/AAAaKepU6dGU1NTm/sqKirisssui86dO3dwKgDgkzzwFACgnVJKcdJJJ8X69evb3P/SSy/FF77whQ5OBQB8kjs/AADaKcuyuPzyy9tc+jJgwIAYMWJECVIBAJ+k/AAAOABtLX0pLy+Pr33ta4VX3gIApWXZCwDAARoyZEisWbOmaOwXv/hFDB06tESJAICPc+cHAMABuuyyy4qWvnzuc59TfADAYUT5AQBwgKZOnRrNzc0R8bslL9OnTy9xIgDg4yx7AQA4CM4666x49dVXIyLi17/+dXzmM58pcSIAYC93fgAAHASXX355pJTiC1/4guIDAA4z7vwAgKPYokWLYsqUKaWOAW2aOHFiPPzww6WOAUAOlJU6AABQevX19aWOkAtz5syJ6667Lq666qqYOXNm1NTUlDrSEeu73/1uqSMAkCPKDwAgJk+eXOoIufCnf/qncdppp8VVV10VNTU1vq8HwB0fABxMnvkBAHCQnHbaaaWOAAC0QfkBAAAA5JryAwAAAMg15QcAAACQa8oPAAAAINeUHwAAAECuKT8AAACAXFN+AAAAALmm/AAAAAByTfkBAAAA5JryAwAAAMg15QcAAACQa8oPAAAAINeUHwAAJda9e/fIsqxo69SpUxx77LExbNiwuO6662LVqlWljgkARyzlBwBAie3cuTNee+21iIioq6uLlFI0NTVFQ0NDzJ49OxoaGuKss86KK664Inbv3l3itABw5FF+AACHle7du8e555571J5/r86dO0ffvn2jrq4uli1bFrfeemvcf//9MXXq1EgplToeABxRlB8AAEeAO++8M/7sz/4slixZEg899FCp4wDAEUX5AQBwBMiyLG644YaIiFiwYEGJ0wDAkUX5AQDsty1btsTNN98cp556alRUVMSxxx4bF110UTz33HOFOd/+9rcLD+/8+DKSp59+ujB+/PHHF8bnzZsXWZbFrl274sUXXyzMKSsrK9qfZVmcdNJJ8corr0RtbW306NEjunbtGueff368+OKLh+z8h4O9/46VK1dGU1NTYXzTpk1x4403xsCBA6OioiJ69+4dEyZMiNWrVxfmLF68uOiBqu+++25MmTIlqqur47jjjotx48bF2rVri87X2NgYt99+ewwZMiS6du0avXr1iosvvjiWLFkSe/bsKZq7LxkAoFSUHwDAfnn//fdjxIgR8eCDD8Y999wTmzdvjpdeeim6du0atbW18cMf/jAiIr75zW9GSim6detW9PkxY8ZESimGDx9eNH7LLbcU5n/xi1+MlFKklKK5ublo/7Bhw2Lbtm1x0003xbe//e14//3346c//Wls3bo1Ro0aFc8///whOf/hoF+/fhER0dzcHJs3b46IiA0bNsSIESNi0aJFsWDBgti6dWv85Cc/ia1bt0ZNTU2sWLEiIiLGjx8fKaWoq6uLiIiZM2fGzJkzY926dVFfXx/Lli2LqVOnFp3vhhtuiO9973vx/e9/P7Zs2RJvvfVWDBkyJOrq6uJnP/tZYd6+ZgCAUlF+AAD7ZdasWfHrX/86/uEf/iHGjRsXPXv2jMGDB8eDDz4YJ5xwQtx4443xwQcfHNIMu3btigULFkRNTU1069YtzjrrrPjXf/3X+O1vfxs33XTTIT13KbX1oNNZs2bFf/3Xf8Xdd98df/7nfx7du3ePoUOHxkMPPRQppfjGN77R5rGuvPLKwvfvy1/+cowdOzZeeeWVQqkSEfHss8/G0KFD44ILLoguXbpE37594+///u9j8ODBByUDAHQU5QcAsF8effTRiIgYO3Zs0XhlZWXU1tbGhx9+GP/+7/9+SDN069YtzjzzzKKxM844I0488cR4/fXXY8OGDYf0/KWy999VXl5eWLKzePHi6NSpU4wbN65obr9+/WLo0KGxatWqeO+991oda8SIEUVfDxgwICIi1q9fXxgbM2ZMLF++PK666qpYuXJlYanLmjVr4rzzzivMa28GAOgoyg8AYJ81NjbG9u3b45hjjokePXq02t+3b9+I+N3SmEOpurq6zfE+ffpERMTGjRsP6flL5YUXXoiIiJqamigvLy9cj5aWlqiqqip6pkeWZfHqq69GRMTbb7/d6lhVVVVFX1dUVEREREtLS2Fs/vz58cADD8Q777wTtbW10bNnzxgzZkyhAIuIA8oAAB1F+QEA7LPKysqoqqqKjz76KHbs2NFq/97lLnufTRER0alTp/jtb3/bau62bdvaPEeWZX8wx5YtW9pcArK39Nhbghyq85dCS0tLzJ8/PyIirr/++oj43fWorq6OsrKyaGpqKjyn5JPb+eef365zZlkWl112WTzzzDOxbdu2WLx4caSUYsKECXH33Xd3SAYAOBiUHwDAfrnkkksiIuLJJ58sGm9sbIxnn302unTpEqNHjy6Mn3DCCbFu3bqiue+//37893//d5vH79q1a1FZ8dnPfjZ+8IMfFM356KOP4pVXXika+4//+I9Yv359DBs2LE444YRDev5SmDVrVrz88stxySWXxKRJkwrjEyZMiObm5qI33ex11113xcknn9zuh7ZWV1dHQ0NDRPxuqc0FF1xQeGvMx6//ocwAAAeD8gMA2C9z5syJQYMGxcyZM+OJJ56IHTt2xK9+9au49NJLY8OGDXHPPfcUlr9ERFx44YWxfv36+Md//MfYuXNnrF27Nm666aaiuzM+7vOf/3z86le/it/85jexYsWKeOedd2LkyJFFc6qqquJv/uZvYsWKFbFr1674+c9/HtOmTYuKioq45557iuYeivN3hJaWlti4cWM89thjUVtbG3Pnzo0ZM2bEwoULi+5OmTNnTpx66qkxY8aMeOqpp2L79u2xdevWuPfee2P27Nkxb968A3pd7zXXXBNvvPFGNDY2xsaNG2Pu3LmRUopRo0Z1WAYAOGAJADhq1dfXp/b8OrB58+Y0c+bMNGjQoFReXp6qqqrS6NGj07PPPttq7rZt29KVV16ZTjjhhNSlS5d07rnnpldeeSUNHz48RUSKiPTXf/3XhfkNDQ1p5MiRqVu3bmnAgAFp/vz5RccbNmxY6t+/f3rzzTfT6NGjU48ePVKXLl3Sl770pfTCCy8c8vPvi4hI9fX1+zy/W7duhSx7tyzLUlVVVTrjjDPStddem1atWvWpn9+yZUu6+eab0ymnnJLKy8tT796904UXXpiWLl1amLNixYpW57jtttsKeT++jR07NqWU0urVq9PVV1+dTj/99NS1a9fUq1evdPbZZ6f77rsvtbS07HeG/TFx4sQ0ceLEdn0WAD4pS6mNBbMAwFFh0aJFMWXKlDafn3G4OvPMM2Pz5s2H9dtDsiyL+vr6mDx5cqmjHLH2Lu15+OGHS5wEgDyw7AUAAADINeUHAAAAkGvKDwDgiDBv3rzIsixef/31WLduXWRZFt/85jdLHQsAOAJ47DYAcES45ZZb4pZbbil1DADgCOTODwAAACDXlB8AAABArik/AAAAgFxTfgAAAAC5pvwAAAAAck35AQAAAOSa8gMAAADINeUHAAAAkGvKDwAAACDXlB8AAABArik/AAAAgFxTfgAAAAC5pvwAAAAAcq2s1AEAgNLLsqzUEXJnypQpMWXKlFLHOKJNnDix1BEAyIkspZRKHQIAKI333nsvli9fXuoY0KYBAwZETU1NqWMAkAPKDwAAACDXPPMDAAAAyDXlBwAAAJBryg8AAAAg18oi4uFShwAAAAA4VP4/OFNmQWNaqSoAAAAASUVORK5CYII=\n", - "text/plain": [ - "" - ] - }, - "execution_count": 19, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# You need to install the dependencies\n", - "tf.keras.utils.plot_model(model)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Training the deep learning model" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We can train our model with `model.fit`. We need to use a Callback to add the validation dataloader." - ] - }, - { - "cell_type": "code", - "execution_count": 20, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "609/611 [============================>.] - ETA: 0s - loss: 0.6650{'val_loss': 0.6597499}\n", - "611/611 [==============================] - 17s 22ms/step - loss: 0.6650 - val_loss: 0.6597\n", - "run_time: 19.14878249168396 - rows: 2292 - epochs: 1 - dl_thru: 119.69429393202323\n" - ] - } - ], - "source": [ - "validation_callback = KerasSequenceValidater(valid_dataset_tf)\n", - "EPOCHS = 1\n", - "start = time.time()\n", - "history = model.fit(train_dataset_tf, callbacks=[validation_callback], epochs=EPOCHS)\n", - "t_final = time.time() - start\n", - "total_rows = train_dataset_tf.num_rows_processed + valid_dataset_tf.num_rows_processed\n", - "print(\n", - " f\"run_time: {t_final} - rows: {total_rows * EPOCHS} - epochs: {EPOCHS} - dl_thru: {(EPOCHS * total_rows) / t_final}\"\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": 21, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2022-04-27 22:13:04.741886: W tensorflow/python/util/util.cc:368] Sets are not currently considered sequences, but this may change in the future, so consider avoiding using them.\n", - "WARNING:absl:Function `_wrapped_model` contains input name(s) movieId, userId with unsupported characters which will be renamed to movieid, userid in the SavedModel.\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "INFO:tensorflow:Assets written to: /root/nvt-examples/movielens_tf/1/model.savedmodel/assets\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "INFO:tensorflow:Assets written to: /root/nvt-examples/movielens_tf/1/model.savedmodel/assets\n", - "WARNING:absl: has the same name 'DenseFeatures' as a built-in Keras object. Consider renaming to avoid naming conflicts when loading with `tf.keras.models.load_model`. If renaming is not possible, pass the object in the `custom_objects` parameter of the load function.\n" - ] - } - ], - "source": [ - "MODEL_NAME_TF = os.environ.get(\"MODEL_NAME_TF\", \"movielens_tf\")\n", - "MODEL_PATH_TEMP_TF = os.path.join(MODEL_BASE_DIR, MODEL_NAME_TF, \"1/model.savedmodel\")\n", - "\n", - "model.save(MODEL_PATH_TEMP_TF)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Before moving to the next notebook, `04a-Triton-Inference-with-TF.ipynb`, we need to generate the Triton Inference Server configurations and save the models in the correct format. We just saved TensorFlow model to disk, and in the previous notebook `02-ETL-with-NVTabular`, we saved the NVTabular workflow. Let's load the workflow. " - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The TensorFlow input layers expect the input datatype to be int32. Therefore, we need to change the output datatypes to int32 for our NVTabular workflow." - ] - }, - { - "cell_type": "code", - "execution_count": 22, - "metadata": {}, - "outputs": [], - "source": [ - "workflow = nvt.Workflow.load(os.path.join(INPUT_DATA_DIR, \"workflow\"))" - ] - }, - { - "cell_type": "code", - "execution_count": 23, - "metadata": {}, - "outputs": [], - "source": [ - "MODEL_NAME_ENSEMBLE = os.environ.get(\"MODEL_NAME_ENSEMBLE\", \"movielens\")\n", - "# model path to save the models\n", - "MODEL_PATH = os.environ.get(\"MODEL_PATH\", os.path.join(MODEL_BASE_DIR, \"models\"))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "NVTabular provides a function to save the NVTabular workflow, TensorFlow model and Triton Inference Server (IS) config files via `export_tensorflow_ensemble`. We provide the model, workflow, a model name for ensemble model, path and output column." - ] - }, - { - "cell_type": "code", - "execution_count": 24, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "WARNING:absl:Function `_wrapped_model` contains input name(s) movieId, userId with unsupported characters which will be renamed to movieid, userid in the SavedModel.\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "INFO:tensorflow:Assets written to: /root/nvt-examples/models/movielens_tf/1/model.savedmodel/assets\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "INFO:tensorflow:Assets written to: /root/nvt-examples/models/movielens_tf/1/model.savedmodel/assets\n", - "WARNING:absl: has the same name 'DenseFeatures' as a built-in Keras object. Consider renaming to avoid naming conflicts when loading with `tf.keras.models.load_model`. If renaming is not possible, pass the object in the `custom_objects` parameter of the load function.\n" - ] - } - ], - "source": [ - "# Creates an ensemble triton server model, where\n", - "# model: The tensorflow model that should be served\n", - "# workflow: The nvtabular workflow used in preprocessing\n", - "# name: The base name of the various triton models\n", - "\n", - "from nvtabular.inference.triton import export_tensorflow_ensemble\n", - "export_tensorflow_ensemble(model, workflow, MODEL_NAME_ENSEMBLE, MODEL_PATH, [\"rating\"])" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now, we can move to the next notebook, [04-Triton-Inference-with-TF.ipynb](https://github.com/NVIDIA/NVTabular/blob/main/examples/getting-started-movielens/04-Triton-Inference-with-TF.ipynb), to send inference request to the Triton IS." - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.10" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/examples/getting-started-movielens/04-Triton-Inference-with-HugeCTR.ipynb b/examples/getting-started-movielens/04-Triton-Inference-with-HugeCTR.ipynb deleted file mode 100644 index c9255083a96..00000000000 --- a/examples/getting-started-movielens/04-Triton-Inference-with-HugeCTR.ipynb +++ /dev/null @@ -1,646 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "id": "d813a4ce", - "metadata": {}, - "outputs": [], - "source": [ - "# Copyright 2021 NVIDIA Corporation. All Rights Reserved.\n", - "#\n", - "# Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# http://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License.\n", - "# ===================================" - ] - }, - { - "cell_type": "markdown", - "id": "260dbfff", - "metadata": {}, - "source": [ - "\n", - "\n", - "# Getting Started MovieLens: Serving a HugeCTR Model\n", - "\n", - "In this notebook, we will show how we do inference with our trained deep learning recommender model using Triton Inference Server. In this example, we deploy the NVTabular workflow and HugeCTR model with Triton Inference Server. We deploy them as an ensemble. For each request, Triton Inference Server will feed the input data through the NVTabular workflow and its output through the HugeCR model." - ] - }, - { - "cell_type": "markdown", - "id": "e0157e1c", - "metadata": {}, - "source": [ - "## Getting Started" - ] - }, - { - "cell_type": "markdown", - "id": "71304a10", - "metadata": {}, - "source": [ - "We need to write configuration files with the stored model weights and model configuration." - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "6a9fbb6d", - "metadata": { - "tags": [ - "flake8-noqa-cell" - ] - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Overwriting /model/movielens_hugectr/config.pbtxt\n" - ] - } - ], - "source": [ - "%%writefile /model/movielens_hugectr/config.pbtxt\n", - "name: \"movielens_hugectr\"\n", - "backend: \"hugectr\"\n", - "max_batch_size: 64\n", - "input [\n", - " {\n", - " name: \"DES\"\n", - " data_type: TYPE_FP32\n", - " dims: [ -1 ]\n", - " },\n", - " {\n", - " name: \"CATCOLUMN\"\n", - " data_type: TYPE_INT64\n", - " dims: [ -1 ]\n", - " },\n", - " {\n", - " name: \"ROWINDEX\"\n", - " data_type: TYPE_INT32\n", - " dims: [ -1 ]\n", - " }\n", - "]\n", - "output [\n", - " {\n", - " name: \"OUTPUT0\"\n", - " data_type: TYPE_FP32\n", - " dims: [ -1 ]\n", - " }\n", - "]\n", - "instance_group [\n", - " {\n", - " count: 1\n", - " kind : KIND_GPU\n", - " gpus:[0]\n", - " }\n", - "]\n", - "\n", - "parameters [\n", - " {\n", - " key: \"config\"\n", - " value: { string_value: \"/model/movielens_hugectr/1/movielens.json\" }\n", - " },\n", - " {\n", - " key: \"gpucache\"\n", - " value: { string_value: \"true\" }\n", - " },\n", - " {\n", - " key: \"hit_rate_threshold\"\n", - " value: { string_value: \"0.8\" }\n", - " },\n", - " {\n", - " key: \"gpucacheper\"\n", - " value: { string_value: \"0.5\" }\n", - " },\n", - " {\n", - " key: \"label_dim\"\n", - " value: { string_value: \"1\" }\n", - " },\n", - " {\n", - " key: \"slots\"\n", - " value: { string_value: \"3\" }\n", - " },\n", - " {\n", - " key: \"cat_feature_num\"\n", - " value: { string_value: \"4\" }\n", - " },\n", - " {\n", - " key: \"des_feature_num\"\n", - " value: { string_value: \"0\" }\n", - " },\n", - " {\n", - " key: \"max_nnz\"\n", - " value: { string_value: \"2\" }\n", - " },\n", - " {\n", - " key: \"embedding_vector_size\"\n", - " value: { string_value: \"16\" }\n", - " },\n", - " {\n", - " key: \"embeddingkey_long_type\"\n", - " value: { string_value: \"true\" }\n", - " }\n", - "]" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "0a23cb52", - "metadata": { - "tags": [ - "flake8-noqa-cell" - ] - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Overwriting /model/ps.json\n" - ] - } - ], - "source": [ - "%%writefile /model/ps.json\n", - "{\n", - " \"supportlonglong\":true,\n", - " \"models\":[\n", - " {\n", - " \"model\":\"movielens_hugectr\",\n", - " \"sparse_files\":[\"/model/movielens_hugectr/0_sparse_1900.model\"],\n", - " \"dense_file\":\"/model/movielens_hugectr/_dense_1900.model\",\n", - " \"network_file\":\"/model/movielens_hugectr/1/movielens.json\",\n", - " \"num_of_worker_buffer_in_pool\": \"1\",\n", - " \"num_of_refresher_buffer_in_pool\": \"1\",\n", - " \"cache_refresh_percentage_per_iteration\": \"0.2\",\n", - " \"deployed_device_list\":[\"0\"],\n", - " \"max_batch_size\":\"64\",\n", - " \"default_value_for_each_table\":[\"0.0\",\"0.0\"],\n", - " \"hit_rate_threshold\":\"0.9\",\n", - " \"gpucacheper\":\"0.5\",\n", - " \"gpucache\":\"true\"\n", - " }\n", - " ] \n", - "}" - ] - }, - { - "cell_type": "markdown", - "id": "5eb3627f", - "metadata": {}, - "source": [ - "Let's import required libraries." - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "id": "f5b54092", - "metadata": {}, - "outputs": [], - "source": [ - "import tritonclient.grpc as httpclient\n", - "import cudf\n", - "import numpy as np" - ] - }, - { - "cell_type": "markdown", - "id": "4e4592a9", - "metadata": {}, - "source": [ - "### Load Models on Triton Inference Server" - ] - }, - { - "cell_type": "markdown", - "id": "150b4754", - "metadata": {}, - "source": [ - "At this stage, you should launch the Triton Inference Server docker container with the following script:" - ] - }, - { - "cell_type": "markdown", - "id": "0a350fce", - "metadata": {}, - "source": [ - "```\n", - "docker run -it --gpus=all -p 8000:8000 -p 8001:8001 -p 8002:8002 -v ${PWD}:/model nvcr.io/nvidia/merlin/merlin-hugectr:latest\n", - "```\n", - "\n", - "> For production use, refer to the [Merlin containers](https://catalog.ngc.nvidia.com/?filters=&orderBy=scoreDESC&query=merlin) from the NVIDIA GPU Cloud (NGC) catalog and specify a tag rather than `latest`." - ] - }, - { - "cell_type": "markdown", - "id": "c6f50e9e", - "metadata": {}, - "source": [ - "After you start the container, start Triton Inference Server with the following command:" - ] - }, - { - "cell_type": "markdown", - "id": "bc8aa849", - "metadata": {}, - "source": [ - "```\n", - "tritonserver --model-repository= --backend-config=hugectr,ps=/ps.json --model-control-mode=explicit\n", - "```" - ] - }, - { - "cell_type": "markdown", - "id": "9b7de550", - "metadata": {}, - "source": [ - "Note: The model-repository path is `/model/`. The models haven't been loaded, yet. We can request triton server to load the saved ensemble. We initialize a triton client. The path for the json file is `/model/movielens_hugectr/1/movielens.json`." - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "id": "a9d1c74a", - "metadata": {}, - "outputs": [], - "source": [ - "# disable warnings\n", - "import warnings\n", - "\n", - "warnings.filterwarnings(\"ignore\")" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "id": "f86290af", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "client created.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/usr/local/lib/python3.8/dist-packages/tritonhttpclient/__init__.py:31: DeprecationWarning: The package `tritonhttpclient` is deprecated and will be removed in a future version. Please use instead `tritonclient.http`\n", - " warnings.warn(\n" - ] - } - ], - "source": [ - "import tritonhttpclient\n", - "\n", - "try:\n", - " triton_client = tritonhttpclient.InferenceServerClient(url=\"localhost:8000\", verbose=True)\n", - " print(\"client created.\")\n", - "except Exception as e:\n", - " print(\"channel creation failed: \" + str(e))" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "id": "a2a2bed5", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "GET /v2/health/live, headers None\n", - "\n" - ] - }, - { - "data": { - "text/plain": [ - "True" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "triton_client.is_server_live()" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "id": "dac3dd79", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "POST /v2/repository/index, headers None\n", - "\n", - "\n", - "bytearray(b'[{\"name\":\"data\"},{\"name\":\"movielens_hugectr\"}]')\n" - ] - }, - { - "data": { - "text/plain": [ - "[{'name': 'data'}, {'name': 'movielens_hugectr'}]" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "triton_client.get_model_repository_index()" - ] - }, - { - "cell_type": "markdown", - "id": "23b2df62", - "metadata": {}, - "source": [ - "Let's load our model to Triton Server." - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "id": "2a1ec18b", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "POST /v2/repository/models/movielens_hugectr/load, headers None\n", - "\n", - "\n", - "Loaded model 'movielens_hugectr'\n", - "CPU times: user 2.6 ms, sys: 2.57 ms, total: 5.17 ms\n", - "Wall time: 3.62 s\n" - ] - } - ], - "source": [ - "%%time\n", - "\n", - "triton_client.load_model(model_name=\"movielens_hugectr\")" - ] - }, - { - "cell_type": "markdown", - "id": "eec2d617", - "metadata": {}, - "source": [ - "Let's send a request to Inference Server and print out the response. Since in our example above we do not have continuous columns, below our only inputs are categorical columns." - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "id": "e5aea0b9", - "metadata": {}, - "outputs": [], - "source": [ - "import pandas as pd\n", - "df = pd.read_parquet(\"/model/data/valid/part_0.parquet\")" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "id": "5f696c53", - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
userIdmovieIdgenresrating
032187520[2, 6]1.0
1679748[1, 14]0.0
2413111026[1, 7]0.0
35951336[2, 4]1.0
416913335[3, 8, 11, 4]1.0
\n", - "
" - ], - "text/plain": [ - " userId movieId genres rating\n", - "0 32187 520 [2, 6] 1.0\n", - "1 67974 8 [1, 14] 0.0\n", - "2 41311 1026 [1, 7] 0.0\n", - "3 5951 336 [2, 4] 1.0\n", - "4 16913 335 [3, 8, 11, 4] 1.0" - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "df.head()" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "id": "8d78ad75", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Overwriting ./wdl2predict.py\n" - ] - } - ], - "source": [ - "%%writefile ./wdl2predict.py\n", - "from tritonclient.utils import *\n", - "import tritonclient.http as httpclient\n", - "import numpy as np\n", - "import pandas as pd\n", - "import sys\n", - "\n", - "model_name = 'movielens_hugectr'\n", - "CATEGORICAL_COLUMNS = [\"userId\", \"movieId\", \"genres\"]\n", - "CONTINUOUS_COLUMNS = []\n", - "LABEL_COLUMNS = ['label']\n", - "emb_size_array = [162542, 29434, 20]\n", - "shift = np.insert(np.cumsum(emb_size_array), 0, 0)[:-1]\n", - "df = pd.read_parquet(\"/model/data/valid/part_0.parquet\")\n", - "test_df = df.head(10)\n", - "\n", - "rp_lst = [0]\n", - "cur = 0\n", - "for i in range(1, 31):\n", - " if i % 3 == 0:\n", - " cur += 2\n", - " rp_lst.append(cur)\n", - " else:\n", - " cur += 1\n", - " rp_lst.append(cur)\n", - "\n", - "with httpclient.InferenceServerClient(\"localhost:8000\") as client:\n", - " test_df.iloc[:, :2] = test_df.iloc[:, :2] + shift[:2]\n", - " test_df.iloc[:, 2] = test_df.iloc[:, 2].apply(lambda x: [e + shift[2] for e in x])\n", - " embedding_columns = np.array([list(np.hstack(np.hstack(test_df[CATEGORICAL_COLUMNS].values)))], dtype='int64')\n", - " dense_features = np.array([[]], dtype='float32')\n", - " row_ptrs = np.array([rp_lst], dtype='int32')\n", - "\n", - " inputs = [httpclient.InferInput(\"DES\", dense_features.shape, np_to_triton_dtype(dense_features.dtype)),\n", - " httpclient.InferInput(\"CATCOLUMN\", embedding_columns.shape, np_to_triton_dtype(embedding_columns.dtype)),\n", - " httpclient.InferInput(\"ROWINDEX\", row_ptrs.shape, np_to_triton_dtype(row_ptrs.dtype))]\n", - "\n", - " inputs[0].set_data_from_numpy(dense_features)\n", - " inputs[1].set_data_from_numpy(embedding_columns)\n", - " inputs[2].set_data_from_numpy(row_ptrs)\n", - " outputs = [httpclient.InferRequestedOutput(\"OUTPUT0\")]\n", - "\n", - " response = client.infer(model_name, inputs, request_id=str(1), outputs=outputs)\n", - "\n", - " result = response.get_response()\n", - " print(result)\n", - " print(\"Prediction Result:\")\n", - " print(response.as_numpy(\"OUTPUT0\"))" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "id": "339340c6", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "/usr/local/lib/python3.8/dist-packages/pandas/core/indexing.py:1851: SettingWithCopyWarning: \n", - "A value is trying to be set on a copy of a slice from a DataFrame.\n", - "Try using .loc[row_indexer,col_indexer] = value instead\n", - "\n", - "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n", - " self._setitem_single_column(loc, val, pi)\n", - "/usr/local/lib/python3.8/dist-packages/pandas/core/indexing.py:1773: SettingWithCopyWarning: \n", - "A value is trying to be set on a copy of a slice from a DataFrame.\n", - "Try using .loc[row_indexer,col_indexer] = value instead\n", - "\n", - "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n", - " self._setitem_single_column(ilocs[0], value, pi)\n", - "Traceback (most recent call last):\n", - " File \"./wdl2predict.py\", line 50, in \n", - " response = client.infer(model_name,\n", - " File \"/usr/local/lib/python3.8/dist-packages/tritonclient/http/__init__.py\", line 1256, in infer\n", - " _raise_if_error(response)\n", - " File \"/usr/local/lib/python3.8/dist-packages/tritonclient/http/__init__.py\", line 64, in _raise_if_error\n", - " raise error\n", - "tritonclient.utils.InferenceServerException: The CATCOLUMN input sample size in request is not match with configuration. The input sample size to be an integer multiple of the configuration.\n" - ] - } - ], - "source": [ - "!python3 ./wdl2predict.py" - ] - } - ], - "metadata": { - "celltoolbar": "Tags", - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.10" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/examples/getting-started-movielens/04-Triton-Inference-with-TF.ipynb b/examples/getting-started-movielens/04-Triton-Inference-with-TF.ipynb deleted file mode 100644 index 96ae7306a42..00000000000 --- a/examples/getting-started-movielens/04-Triton-Inference-with-TF.ipynb +++ /dev/null @@ -1,706 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "# Copyright 2020 NVIDIA Corporation. All Rights Reserved.\n", - "#\n", - "# Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# http://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License.\n", - "# ==============================================================================" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "# Getting Started MovieLens: Serving a TensorFlow Model\n", - "The last step is to deploy the ETL workflow and saved model to production. In the production setting, we want to transform the input data as done during training ETL. We need to apply the same mean/std for continuous features and use the same categorical mapping to convert the categories to continuous integers before we use the deep learning model for a prediction. Therefore, we deploy the NVTabular workflow with the TensorFlow model as an ensemble model to Triton Inference. The ensemble model guarantees that the same transformation are applied to the raw inputs." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "
" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Learning Objectives\n", - "In the previous notebook we explained and showed how we can preprocess data with multi-hot columns with NVTabular, and train an TF MLP model using NVTabular `KerasSequenceLoader`. We learned how to save a workflow, a trained TF model, and the ensemble model. In this notebook, we will show an example request script sent to the Triton Inference Server. We will learn\n", - "\n", - "- to transform new/streaming data with NVTabular library\n", - "- to deploy the end-to-end pipeline to generate prediction results for new data from trained TF model" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Starting Triton Inference Server" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Before we get started, start Triton Inference Server in the Docker container with the following command. The command includes the `-v` argument to mount your local `model-repository` directory that includes your saved models from the previous notebook (`03a-Training-with-TF.ipynb`) to `/model` directory in the container." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "```\n", - "docker run -it --gpus device=0 -p 8000:8000 -p 8001:8001 -p 8002:8002 -v ${PWD}:/model/ nvcr.io/nvidia/merlin/merlin-tensorflow:latest\n", - "```" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "After you start the container, you can start Triton Inference Server with the following command. You need to provide correct path for the `models` directory.\n", - "\n", - "```\n", - "tritonserver --model-repository=path_to_models --backend-config=tensorflow,version=2 --model-control-mode=explicit \n", - "```" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Note: The model-repository path is `/model/nvt-examples/models/`. The models haven't been loaded, yet. Below, we will request the Triton server to load the saved ensemble model." - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "# External dependencies\n", - "import os\n", - "from time import time\n", - "\n", - "# Get dataframe library - cudf or pandas\n", - "from merlin.core.dispatch import get_lib\n", - "df_lib = get_lib()\n", - "\n", - "import tritonclient.grpc as grpcclient\n", - "import nvtabular.inference.triton as nvt_triton" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We define our base directory, containing the data." - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "# path to preprocessed data\n", - "INPUT_DATA_DIR = os.environ.get(\n", - " \"INPUT_DATA_DIR\", os.path.expanduser(\"~/nvt-examples/movielens/data/\")\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Let's deactivate the warnings before sending requests." - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "import warnings\n", - "\n", - "warnings.filterwarnings(\"ignore\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Loading Ensemble Model with Triton Inference Server" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "At this stage, you should have started the Triton Inference Server in a container with the instructions above." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Let's connect to the Triton Inference Server. Use Triton’s ready endpoint to verify that the server and the models are ready for inference. Replace localhost with your host ip address." - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "client created.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/usr/local/lib/python3.8/dist-packages/tritonhttpclient/__init__.py:30: DeprecationWarning: The package `tritonhttpclient` is deprecated and will be removed in a future version. Please use instead `tritonclient.http`\n", - " warnings.warn(\n" - ] - } - ], - "source": [ - "import tritonhttpclient\n", - "\n", - "try:\n", - " triton_client = tritonhttpclient.InferenceServerClient(url=\"localhost:8000\", verbose=True)\n", - " print(\"client created.\")\n", - "except Exception as e:\n", - " print(\"channel creation failed: \" + str(e))" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/usr/local/lib/python3.8/dist-packages/ipykernel/ipkernel.py:283: DeprecationWarning: `should_run_async` will not call `transform_cell` automatically in the future. Please pass the result to `transformed_cell` argument and any exception that happen during thetransform in `preprocessing_exc_tuple` in IPython 7.17 and above.\n", - " and should_run_async(code)\n" - ] - } - ], - "source": [ - "import warnings\n", - "\n", - "warnings.filterwarnings(\"ignore\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We check if the server is alive." - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "GET /v2/health/live, headers None\n", - "\n" - ] - }, - { - "data": { - "text/plain": [ - "True" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "triton_client.is_server_live()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The HTTP request returns status 200 if Triton is ready and non-200 if it is not ready." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We check the available models in the repositories:\n", - "\n", - "movielens: Ensemble
\n", - "movielens_nvt: NVTabular
\n", - "movielens_tf: TensorFlow model" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "POST /v2/repository/index, headers None\n", - "\n", - "\n", - "bytearray(b'[{\"name\":\"movielens\"},{\"name\":\"movielens_nvt\"},{\"name\":\"movielens_tf\"}]')\n" - ] - }, - { - "data": { - "text/plain": [ - "[{'name': 'movielens'}, {'name': 'movielens_nvt'}, {'name': 'movielens_tf'}]" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "triton_client.get_model_repository_index()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We load the ensemble model." - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "POST /v2/repository/models/movielens/load, headers None\n", - "\n", - "\n", - "Loaded model 'movielens'\n", - "CPU times: user 2.05 ms, sys: 1.62 ms, total: 3.66 ms\n", - "Wall time: 9.09 s\n" - ] - } - ], - "source": [ - "%%time\n", - "\n", - "triton_client.load_model(model_name=\"movielens\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Send request to Triton Inference Server to transform raw dataset" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "A minimal model repository for a TensorFlow SavedModel model is:\n", - "```\n", - " //\n", - " config.pbtxt\n", - " 1/\n", - " model.savedmodel/\n", - " \n", - "```\n", - "Let's check out our model repository layout. You can install tree library with `apt-get install tree`, and then run `!tree /model/models/` to print out the model repository layout as below:\n", - " \n", - "```\n", - "/model/models/\n", - "|-- movielens\n", - "| |-- 1\n", - "| `-- config.pbtxt\n", - "|-- movielens_nvt\n", - "| |-- 1\n", - "| | |-- __pycache__\n", - "| | | `-- model.cpython-38.pyc\n", - "| | |-- model.py\n", - "| | `-- workflow\n", - "| | |-- categories\n", - "| | | |-- unique.genres.parquet\n", - "| | | |-- unique.movieId.parquet\n", - "| | | `-- unique.userId.parquet\n", - "| | |-- metadata.json\n", - "| | `-- workflow.pkl\n", - "| `-- config.pbtxt\n", - "`-- movielens_tf\n", - " |-- 1\n", - " | `-- model.savedmodel\n", - " | |-- assets\n", - " | |-- saved_model.pb\n", - " | `-- variables\n", - " | |-- variables.data-00000-of-00001\n", - " | `-- variables.index\n", - " `-- config.pbtxt\n", - "```\n", - "You can see that we have a `config.pbtxt` file. Each model in a model repository must include a model configuration that provides required and optional information about the model. Typically, this configuration is provided in a `config.pbtxt` file specified as [ModelConfig protobuf](https://github.com/triton-inference-server/server/blob/r20.12/src/core/model_config.proto)." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Let's read the raw validation set, and send 3 rows of `userId` and `movieId` as input to the saved NVTabular model." - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - " userId movieId\n", - "15347762 99476 104374\n", - "16647840 107979 2634\n", - "23915192 155372 1614\n" - ] - } - ], - "source": [ - "# read in the workflow (to get input/output schema to call triton with)\n", - "batch = df_lib.read_parquet(\n", - " os.path.join(INPUT_DATA_DIR, \"valid.parquet\"), num_rows=3, columns=[\"userId\", \"movieId\"]\n", - ")\n", - "print(batch)" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "userId [[ 99476]\n", - " [107979]\n", - " [155372]] (3, 1)\n", - "movieId [[19997]\n", - " [ 2543]\n", - " [ 1557]] (3, 1)\n", - "genres__nnzs [[3]\n", - " [1]\n", - " [1]] (3, 1)\n", - "genres__values [[ 9]\n", - " [10]\n", - " [16]\n", - " [12]\n", - " [ 6]] (5, 1)\n" - ] - } - ], - "source": [ - "inputs = nvt_triton.convert_df_to_triton_input([\"userId\", \"movieId\"], batch, grpcclient.InferInput)\n", - "\n", - "outputs = [\n", - " grpcclient.InferRequestedOutput(col)\n", - " for col in [\"userId\", \"movieId\", \"genres__nnzs\", \"genres__values\"]\n", - "]\n", - "\n", - "MODEL_NAME_NVT = os.environ.get(\"MODEL_NAME_NVT\", \"movielens_nvt\")\n", - "\n", - "with grpcclient.InferenceServerClient(\"localhost:8001\") as client:\n", - " response = client.infer(MODEL_NAME_NVT, inputs, request_id=\"1\", outputs=outputs)\n", - "\n", - "for col in [\"userId\", \"movieId\", \"genres__nnzs\", \"genres__values\"]:\n", - " print(col, response.as_numpy(col), response.as_numpy(col).shape)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "You might notice that we don't need to send the genres column as an input. The reason for that is the nvt model will look up the genres for each movie as part of the `JoinExternal` op it applies. Also notice that when creating the request for the `movielens_nvt` model, we return 2 columns (values and nnzs) for the `genres` column rather than 1." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## END-2-END INFERENCE PIPELINE" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We will do the same, but this time we directly read in first 3 rows of the the raw `valid.parquet` file with cuDF." - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "raw data:\n", - " userId movieId\n", - "15347762 99476 104374\n", - "16647840 107979 2634\n", - "23915192 155372 1614 \n", - "\n", - "predicted sigmoid result:\n", - " [[0.628711 ]\n", - " [0.6082093 ]\n", - " [0.60346156]]\n" - ] - } - ], - "source": [ - "# read in the workflow (to get input/output schema to call triton with)\n", - "batch = df_lib.read_parquet(\n", - " os.path.join(INPUT_DATA_DIR, \"valid.parquet\"), num_rows=3, columns=[\"userId\", \"movieId\"]\n", - ")\n", - "\n", - "print(\"raw data:\\n\", batch, \"\\n\")\n", - "\n", - "# convert the batch to a triton inputs\n", - "inputs = nvt_triton.convert_df_to_triton_input([\"userId\", \"movieId\"], batch, grpcclient.InferInput)\n", - "\n", - "# placeholder variables for the output\n", - "outputs = [grpcclient.InferRequestedOutput(\"output\")]\n", - "\n", - "MODEL_NAME_ENSEMBLE = os.environ.get(\"MODEL_NAME_ENSEMBLE\", \"movielens\")\n", - "\n", - "# build a client to connect to our server.\n", - "# This InferenceServerClient object is what we'll be using to talk to Triton.\n", - "# make the request with tritonclient.grpc.InferInput object\n", - "\n", - "with grpcclient.InferenceServerClient(\"localhost:8001\") as client:\n", - " response = client.infer(MODEL_NAME_ENSEMBLE, inputs, request_id=\"1\", outputs=outputs)\n", - "\n", - "print(\"predicted sigmoid result:\\n\", response.as_numpy(\"output\"))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Let's send request for a larger batch size and measure the total run time and throughput." - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "predicted sigmoid result:\n", - " [[0.628711 ]\n", - " [0.6082093 ]\n", - " [0.60346156]\n", - " [0.62520176]\n", - " [0.6164747 ]\n", - " [0.6355395 ]\n", - " [0.6193519 ]\n", - " [0.61882406]\n", - " [0.6275068 ]\n", - " [0.6138062 ]\n", - " [0.6202122 ]\n", - " [0.62851 ]\n", - " [0.6351558 ]\n", - " [0.62927085]\n", - " [0.6350106 ]\n", - " [0.61985826]\n", - " [0.621534 ]\n", - " [0.6181114 ]\n", - " [0.63753897]\n", - " [0.61673135]\n", - " [0.6167665 ]\n", - " [0.6212634 ]\n", - " [0.62160015]\n", - " [0.63293964]\n", - " [0.6352973 ]\n", - " [0.61357415]\n", - " [0.6352516 ]\n", - " [0.6211146 ]\n", - " [0.6320578 ]\n", - " [0.62171084]\n", - " [0.60404694]\n", - " [0.63201594]\n", - " [0.6052745 ]\n", - " [0.61897206]\n", - " [0.61399895]\n", - " [0.6196497 ]\n", - " [0.618947 ]\n", - " [0.61561245]\n", - " [0.62465805]\n", - " [0.6257206 ]\n", - " [0.61907804]\n", - " [0.62646204]\n", - " [0.61661446]\n", - " [0.61312085]\n", - " [0.60481817]\n", - " [0.6146393 ]\n", - " [0.6135305 ]\n", - " [0.6233996 ]\n", - " [0.6268691 ]\n", - " [0.6368837 ]\n", - " [0.6286694 ]\n", - " [0.61883575]\n", - " [0.6271743 ]\n", - " [0.62324375]\n", - " [0.61735946]\n", - " [0.63762474]\n", - " [0.6315052 ]\n", - " [0.6226361 ]\n", - " [0.6040064 ]\n", - " [0.6273543 ]\n", - " [0.62771416]\n", - " [0.6178839 ]\n", - " [0.6200199 ]\n", - " [0.6220759 ]] \n", - "\n", - "run_time(sec): 0.057904958724975586 - rows: 64 - inference_thru: 1105.2594010812325\n" - ] - } - ], - "source": [ - "# read in the workflow (to get input/output schema to call triton with)\n", - "batch_size = 64\n", - "batch = df_lib.read_parquet(\n", - " os.path.join(INPUT_DATA_DIR, \"valid.parquet\"),\n", - " num_rows=batch_size,\n", - " columns=[\"userId\", \"movieId\"],\n", - ")\n", - "\n", - "start = time()\n", - "# convert the batch to a triton inputs\n", - "inputs = nvt_triton.convert_df_to_triton_input([\"userId\", \"movieId\"], batch, grpcclient.InferInput)\n", - "\n", - "# placeholder variables for the output\n", - "outputs = [grpcclient.InferRequestedOutput(\"output\")]\n", - "\n", - "MODEL_NAME_ENSEMBLE = os.environ.get(\"MODEL_NAME_ENSEMBLE\", \"movielens\")\n", - "\n", - "# build a client to connect to our server.\n", - "# This InferenceServerClient object is what we'll be using to talk to Triton.\n", - "# make the request with tritonclient.grpc.InferInput object\n", - "\n", - "with grpcclient.InferenceServerClient(\"localhost:8001\") as client:\n", - " response = client.infer(MODEL_NAME_ENSEMBLE, inputs, request_id=\"1\", outputs=outputs)\n", - "\n", - "t_final = time() - start\n", - "print(\"predicted sigmoid result:\\n\", response.as_numpy(\"output\"), \"\\n\")\n", - "\n", - "print(f\"run_time(sec): {t_final} - rows: {batch_size} - inference_thru: {batch_size / t_final}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Let's unload all the models." - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "POST /v2/repository/models/movielens/unload, headers None\n", - "{\"parameters\":{\"unload_dependents\":false}}\n", - "\n", - "Loaded model 'movielens'\n", - "POST /v2/repository/models/movielens_nvt/unload, headers None\n", - "{\"parameters\":{\"unload_dependents\":false}}\n", - "\n", - "Loaded model 'movielens_nvt'\n", - "POST /v2/repository/models/movielens_tf/unload, headers None\n", - "{\"parameters\":{\"unload_dependents\":false}}\n", - "\n", - "Loaded model 'movielens_tf'\n" - ] - } - ], - "source": [ - "triton_client.unload_model(model_name=\"movielens\")\n", - "triton_client.unload_model(model_name=\"movielens_nvt\")\n", - "triton_client.unload_model(model_name=\"movielens_tf\")" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.10" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/examples/getting-started-movielens/README.md b/examples/getting-started-movielens/README.md deleted file mode 100644 index 0e165a68574..00000000000 --- a/examples/getting-started-movielens/README.md +++ /dev/null @@ -1,17 +0,0 @@ -# Getting Started with Movielens - -The MovieLens25M is a popular dataset for recommender systems and is used in academic publications. -Most users are familiar with the dataset and the example notebooks teach the basic concepts of NVTabular: - -* Learning NVTabular for using GPU-accelerated ETL (Preprocess and Feature Engineering). -* Getting familiar with NVTabular’s high-level API. -* Using single-hot/multi-hot categorical input features with NVTabular. -* Using NVTabular dataloader with TensorFlow Keras model. -* Using NVTabular dataloader with PyTorch. - -Refer to the following notebooks: - -* [Download and Convert](01-Download-Convert.ipynb) -* [ETL with NVTabular](02-ETL-with-NVTabular.ipynb) -* Training a model: [HugeCTR](03-Training-with-HugeCTR.ipynb) | [TensorFlow](03-Training-with-TF.ipynb) | [PyTorch](03-Training-with-PyTorch.ipynb) -* Serving with Triton Inference Server: [HugeCTR](04-Triton-Inference-with-HugeCTR.ipynb) | [TensorFlow](04-Triton-Inference-with-TF.ipynb) \ No newline at end of file diff --git a/examples/getting-started-movielens/imgs/triton-tf.png b/examples/getting-started-movielens/imgs/triton-tf.png deleted file mode 100644 index e68bcd4fefc..00000000000 Binary files a/examples/getting-started-movielens/imgs/triton-tf.png and /dev/null differ diff --git a/examples/multi-gpu-movielens/01-03-MultiGPU-Download-Convert-ETL-with-NVTabular-Training-with-TensorFlow.ipynb b/examples/multi-gpu-movielens/01-03-MultiGPU-Download-Convert-ETL-with-NVTabular-Training-with-TensorFlow.ipynb deleted file mode 100644 index e61e4850709..00000000000 --- a/examples/multi-gpu-movielens/01-03-MultiGPU-Download-Convert-ETL-with-NVTabular-Training-with-TensorFlow.ipynb +++ /dev/null @@ -1,837 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "id": "be62766b", - "metadata": {}, - "outputs": [], - "source": [ - "# Copyright 2021 NVIDIA Corporation. All Rights Reserved.\n", - "#\n", - "# Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# http://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License.\n", - "# ==============================================================================" - ] - }, - { - "cell_type": "markdown", - "id": "8fbd62b5", - "metadata": {}, - "source": [ - "\n", - "\n", - "# Multi-GPU Training with TensorFlow on MovieLens\n", - "\n", - "## Overview\n", - "\n", - "NVIDIA Merlin is a open source framework to accelerate and scale end-to-end recommender system pipelines on GPU. In this notebook, we use NVTabular, Merlin’s ETL component, to scale feature engineering and pre-processing to multiple GPUs and then perform data-parallel distributed training of a neural network on multiple GPUs with TensorFlow, [Horovod](https://horovod.readthedocs.io/en/stable/), and [NCCL](https://developer.nvidia.com/nccl).\n", - "\n", - "The pre-requisites for this notebook are to be familiar with NVTabular and its API:\n", - "- You can read more about NVTabular, its API and specialized dataloaders in [Getting Started with Movielens notebooks](https://nvidia-merlin.github.io/NVTabular/main/examples/getting-started-movielens/index.html).\n", - "- You can read more about scaling NVTabular ETL in [Scaling Criteo notebooks](https://nvidia-merlin.github.io/NVTabular/main/examples/scaling-criteo/index.html).\n", - "\n", - "**In this notebook, we will focus only on the new information related to multi-GPU training, so please check out the other notebooks first (if you haven’t already.)**\n", - "\n", - "### Learning objectives\n", - "\n", - "In this notebook, we learn how to scale ETL and deep learning taining to multiple GPUs\n", - "- Learn to use larger than GPU/host memory datasets for ETL and training\n", - "- Use multi-GPU or multi node for ETL with NVTabular\n", - "- Use NVTabular dataloader to accelerate TensorFlow pipelines\n", - "- Scale TensorFlow training with Horovod\n", - "\n", - "### Dataset\n", - "\n", - "In this notebook, we use the [MovieLens25M](https://grouplens.org/datasets/movielens/25m/) dataset. It is popular for recommender systems and is used in academic publications. The dataset contains 25M movie ratings for 62,000 movies given by 162,000 users. Many projects use only the user/item/rating information of MovieLens, but the original dataset provides metadata for the movies, as well.\n", - "\n", - "Note: We are using the MovieLens 25M dataset in this example for simplicity, although the dataset is not large enough to require multi-GPU training. However, the functionality demonstrated in this notebook can be easily extended to scale recommender pipelines for larger datasets in the same way.\n", - "\n", - "### Tools\n", - "\n", - "- [Horovod](https://horovod.readthedocs.io/en/stable/) is a distributed deep learning framework that provides tools for multi-GPU optimization.\n", - "- The [NVIDIA Collective Communication Library (NCCL)](https://developer.nvidia.com/nccl) provides the underlying GPU-based implementations of the [allgather](https://docs.nvidia.com/deeplearning/nccl/user-guide/docs/usage/operations.html#allgather) and [allreduce](https://docs.nvidia.com/deeplearning/nccl/user-guide/docs/usage/operations.html#allreduce) cross-GPU communication operations." - ] - }, - { - "cell_type": "markdown", - "id": "7332a3be", - "metadata": {}, - "source": [ - "## Download and Convert\n", - "\n", - "First, we will download and convert the dataset to Parquet. This section is based on [01-Download-Convert.ipynb](../getting-started-movielens/01-Download-Convert.ipynb)." - ] - }, - { - "cell_type": "markdown", - "id": "e7abbc39", - "metadata": {}, - "source": [ - "#### Download" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "54d7869c", - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "downloading ml-25m.zip: 262MB [00:06, 41.9MB/s] \n", - "unzipping files: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 8/8 [00:04<00:00, 1.74files/s]\n" - ] - } - ], - "source": [ - "# External dependencies\n", - "import os\n", - "import pathlib\n", - "\n", - "import cudf # cuDF is an implementation of Pandas-like Dataframe on GPU\n", - "\n", - "from merlin.core.utils import download_file\n", - "\n", - "INPUT_DATA_DIR = os.environ.get(\n", - " \"INPUT_DATA_DIR\", \"~/nvt-examples/multigpu-movielens/data/\"\n", - ")\n", - "BASE_DIR = pathlib.Path(INPUT_DATA_DIR).expanduser()\n", - "zip_path = pathlib.Path(BASE_DIR, \"ml-25m.zip\")\n", - "download_file(\n", - " \"http://files.grouplens.org/datasets/movielens/ml-25m.zip\", zip_path, redownload=False\n", - ")" - ] - }, - { - "cell_type": "markdown", - "id": "bebb82e7", - "metadata": {}, - "source": [ - "#### Convert" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "f6a7ccc4", - "metadata": {}, - "outputs": [], - "source": [ - "movies = cudf.read_csv(pathlib.Path(BASE_DIR, \"ml-25m\", \"movies.csv\"))\n", - "movies[\"genres\"] = movies[\"genres\"].str.split(\"|\")\n", - "movies = movies.drop(\"title\", axis=1)\n", - "movies.to_parquet(pathlib.Path(BASE_DIR, \"ml-25m\", \"movies_converted.parquet\"))" - ] - }, - { - "cell_type": "markdown", - "id": "bc8da86e", - "metadata": {}, - "source": [ - "#### Split into train and validation datasets" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "id": "cf9a686f", - "metadata": {}, - "outputs": [], - "source": [ - "ratings = cudf.read_csv(pathlib.Path(BASE_DIR, \"ml-25m\", \"ratings.csv\"))\n", - "ratings = ratings.drop(\"timestamp\", axis=1)\n", - "\n", - "# shuffle the dataset\n", - "ratings = ratings.sample(len(ratings), replace=False)\n", - "# split the train_df as training and validation data sets.\n", - "num_valid = int(len(ratings) * 0.2)\n", - "train = ratings[:-num_valid]\n", - "valid = ratings[-num_valid:]\n", - "\n", - "train.to_parquet(pathlib.Path(BASE_DIR, \"train.parquet\"))\n", - "valid.to_parquet(pathlib.Path(BASE_DIR, \"valid.parquet\"))" - ] - }, - { - "cell_type": "markdown", - "id": "6e24ff4b", - "metadata": {}, - "source": [ - "## ETL with NVTabular\n", - "\n", - "We finished downloading and converting the dataset. We will preprocess and engineer features with NVTabular on multiple GPUs. You can read more\n", - "- about NVTabular's features and API in [getting-started-movielens/02-ETL-with-NVTabular.ipynb](../getting-started-movielens/02-ETL-with-NVTabular.ipynb).\n", - "- scaling NVTabular ETL to multiple GPUs [scaling-criteo/02-ETL-with-NVTabular.ipynb](../scaling-criteo/02-ETL-with-NVTabular.ipynb)." - ] - }, - { - "cell_type": "markdown", - "id": "1e7308a8", - "metadata": {}, - "source": [ - "#### Deploy a Distributed-Dask Cluster\n", - "\n", - "This section is based on [scaling-criteo/02-ETL-with-NVTabular.ipynb](../scaling-criteo/02-ETL-with-NVTabular.ipynb) and [multi-gpu-toy-example/multi-gpu_dask.ipynb](../multi-gpu-toy-example/multi-gpu_dask.ipynb)" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "id": "c962f1a2", - "metadata": {}, - "outputs": [], - "source": [ - "# Standard Libraries\n", - "import shutil\n", - "\n", - "# External Dependencies\n", - "import cupy as cp\n", - "import numpy as np\n", - "import cudf\n", - "import dask_cudf\n", - "from dask_cuda import LocalCUDACluster\n", - "from dask.distributed import Client\n", - "from dask.utils import parse_bytes\n", - "from dask.delayed import delayed\n", - "import rmm\n", - "\n", - "# NVTabular\n", - "import nvtabular as nvt\n", - "import nvtabular.ops as ops\n", - "from merlin.io import Shuffle\n", - "from merlin.core.utils import device_mem_size" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "id": "5332e25c", - "metadata": {}, - "outputs": [], - "source": [ - "# define some information about where to get our data\n", - "input_path = pathlib.Path(BASE_DIR, \"converted\", \"movielens\")\n", - "dask_workdir = pathlib.Path(BASE_DIR, \"test_dask\", \"workdir\")\n", - "output_path = pathlib.Path(BASE_DIR, \"test_dask\", \"output\")\n", - "stats_path = pathlib.Path(BASE_DIR, \"test_dask\", \"stats\")\n", - "\n", - "# Make sure we have a clean worker space for Dask\n", - "if pathlib.Path.is_dir(dask_workdir):\n", - " shutil.rmtree(dask_workdir)\n", - "dask_workdir.mkdir(parents=True)\n", - "\n", - "# Make sure we have a clean stats space for Dask\n", - "if pathlib.Path.is_dir(stats_path):\n", - " shutil.rmtree(stats_path)\n", - "stats_path.mkdir(parents=True)\n", - "\n", - "# Make sure we have a clean output path\n", - "if pathlib.Path.is_dir(output_path):\n", - " shutil.rmtree(output_path)\n", - "output_path.mkdir(parents=True)\n", - "\n", - "# Get device memory capacity\n", - "capacity = device_mem_size(kind=\"total\")" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "id": "6eca2e5f", - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "\n", - "\n", - "\n", - "\n", - "
\n", - "

Client

\n", - "\n", - "		\n", - "

Cluster

\n", - "
    \n", - "
  • Workers: 2
    • \n", - "
  • Cores: 2
    • \n", - "
  • Memory: 125.84 GiB
    • \n", - "
\n", - "
" - ], - "text/plain": [ - "" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# Deploy a Single-Machine Multi-GPU Cluster\n", - "protocol = \"tcp\" # \"tcp\" or \"ucx\"\n", - "visible_devices = \"0,1\" # Delect devices to place workers\n", - "device_spill_frac = 0.5 # Spill GPU-Worker memory to host at this limit.\n", - "# Reduce if spilling fails to prevent\n", - "# device memory errors.\n", - "cluster = None # (Optional) Specify existing scheduler port\n", - "if cluster is None:\n", - " cluster = LocalCUDACluster(\n", - " protocol=protocol,\n", - " CUDA_VISIBLE_DEVICES=visible_devices,\n", - " local_directory=dask_workdir,\n", - " device_memory_limit=capacity * device_spill_frac,\n", - " )\n", - "\n", - "# Create the distributed client\n", - "client = Client(cluster)\n", - "client" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "id": "88a35091", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'tcp://127.0.0.1:40789': None, 'tcp://127.0.0.1:43439': None}" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# Initialize RMM pool on ALL workers\n", - "def _rmm_pool():\n", - " rmm.reinitialize(\n", - " pool_allocator=True,\n", - " initial_pool_size=None, # Use default size\n", - " )\n", - "\n", - "\n", - "client.run(_rmm_pool)" - ] - }, - { - "cell_type": "markdown", - "id": "3eb097e6", - "metadata": {}, - "source": [ - "#### Defining our Preprocessing Pipeline\n", - "\n", - "This subsection is based on [getting-started-movielens/02-ETL-with-NVTabular.ipynb](../getting-started-movielens/02-ETL-with-NVTabular.ipynb)." - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "id": "33f1b593", - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/usr/local/lib/python3.8/dist-packages/distributed/worker.py:3560: UserWarning: Large object of size 1.90 MiB detected in task graph: \n", - " (\"('read-parquet-d36dd514a8adc53a9a91115c9be1d852' ... 1115c9be1d852')\n", - "Consider scattering large objects ahead of time\n", - "with client.scatter to reduce scheduler burden and \n", - "keep data on workers\n", - "\n", - " future = client.submit(func, big_data) # bad\n", - "\n", - " big_future = client.scatter(big_data) # good\n", - " future = client.submit(func, big_future) # good\n", - " warnings.warn(\n" - ] - } - ], - "source": [ - "movies = cudf.read_parquet(pathlib.Path(BASE_DIR, \"ml-25m\", \"movies_converted.parquet\"))\n", - "joined = [\"userId\", \"movieId\"] >> nvt.ops.JoinExternal(movies, on=[\"movieId\"])\n", - "cat_features = joined >> nvt.ops.Categorify()\n", - "ratings = nvt.ColumnSelector([\"rating\"]) >> nvt.ops.LambdaOp(lambda col: (col > 3).astype(\"int8\"), dtype=np.int8)\n", - "output = cat_features + ratings\n", - "workflow = nvt.Workflow(output)\n", - "!rm -rf $BASE_DIR/train\n", - "!rm -rf $BASE_DIR/valid\n", - "train_iter = nvt.Dataset([str(pathlib.Path(BASE_DIR, \"train.parquet\"))], part_size=\"100MB\")\n", - "valid_iter = nvt.Dataset([str(pathlib.Path(BASE_DIR, \"valid.parquet\"))], part_size=\"100MB\")\n", - "workflow.fit(train_iter)\n", - "workflow.save(str(pathlib.Path(BASE_DIR, \"workflow\")))\n", - "shuffle = Shuffle.PER_WORKER # Shuffle algorithm\n", - "out_files_per_proc = 4 # Number of output files per worker\n", - "workflow.transform(train_iter).to_parquet(\n", - " output_path=pathlib.Path(BASE_DIR, \"train\"),\n", - " shuffle=shuffle,\n", - " out_files_per_proc=out_files_per_proc,\n", - ")\n", - "workflow.transform(valid_iter).to_parquet(\n", - " output_path=pathlib.Path(BASE_DIR, \"valid\"),\n", - " shuffle=shuffle,\n", - " out_files_per_proc=out_files_per_proc,\n", - ")\n", - "\n", - "client.shutdown()\n", - "cluster.close()" - ] - }, - { - "cell_type": "markdown", - "id": "9e220013", - "metadata": {}, - "source": [ - "## Training with TensorFlow on multiGPUs\n", - "\n", - "In this section, we will train a TensorFlow model with multi-GPU support. In the NVTabular v0.5 release, we added multi-GPU support for NVTabular dataloaders. We will modify the [getting-started-movielens/03-Training-with-TF.ipynb](../getting-started-movielens/03-Training-with-TF.ipynb) to use multiple GPUs. Please review that notebook, if you have questions about the general functionality of the NVTabular dataloaders or the neural network architecture.\n", - "\n", - "#### NVTabular dataloader for TensorFlow\n", - "\n", - "We’ve identified that the dataloader is one bottleneck in deep learning recommender systems when training pipelines with TensorFlow. The normal TensorFlow dataloaders cannot prepare the next training batches fast enough and therefore, the GPU is not fully utilized. \n", - "\n", - "We developed a highly customized tabular dataloader for accelerating existing pipelines in TensorFlow. In our experiments, we see a speed-up by 9x of the same training workflow with NVTabular dataloader. NVTabular dataloader’s features are:\n", - "- removing bottleneck of item-by-item dataloading\n", - "- enabling larger than memory dataset by streaming from disk\n", - "- reading data directly into GPU memory and remove CPU-GPU communication\n", - "- preparing batch asynchronously in GPU to avoid CPU-GPU communication\n", - "- supporting commonly used .parquet format\n", - "- easy integration into existing TensorFlow pipelines by using similar API - works with tf.keras models\n", - "- **supporting multi-GPU training with Horovod**\n", - "\n", - "You can find more information on the dataloaders in our [blogpost](https://medium.com/nvidia-merlin/training-deep-learning-based-recommender-systems-9x-faster-with-tensorflow-cc5a2572ea49)." - ] - }, - { - "cell_type": "markdown", - "id": "8dad9141", - "metadata": {}, - "source": [ - "#### Using Horovod with Tensorflow and NVTabular\n", - "\n", - "The training script below is based on [getting-started-movielens/03-Training-with-TF.ipynb](../getting-started-movielens/03-Training-with-TF.ipynb), with a few important changes:\n", - "\n", - "- We provide several additional parameters to the `KerasSequenceLoader` class, including the total number of workers `hvd.size()`, the current worker's id number `hvd.rank()`, and a function for generating random seeds `seed_fn()`. \n", - "\n", - "```python\n", - " train_dataset_tf = KerasSequenceLoader(\n", - " ...\n", - " global_size=hvd.size(),\n", - " global_rank=hvd.rank(),\n", - " seed_fn=seed_fn,\n", - " )\n", - "\n", - "```\n", - "- The seed function uses Horovod to collectively generate a random seed that's shared by all workers so that they can each shuffle the dataset in a consistent way and select partitions to work on without overlap. The seed function is called by the dataloader during the shuffling process at the beginning of each epoch:\n", - "\n", - "```python\n", - " def seed_fn():\n", - " min_int, max_int = tf.int32.limits\n", - " max_rand = max_int // hvd.size()\n", - "\n", - " # Generate a seed fragment on each worker\n", - " seed_fragment = cupy.random.randint(0, max_rand).get()\n", - "\n", - " # Aggregate seed fragments from all Horovod workers\n", - " seed_tensor = tf.constant(seed_fragment)\n", - " reduced_seed = hvd.allreduce(seed_tensor, name=\"shuffle_seed\", op=hvd.mpi_ops.Sum) \n", - "\n", - " return reduced_seed % max_rand\n", - "```\n", - "\n", - "- We wrap the TensorFlow optimizer with Horovod's `DistributedOptimizer` class and scale the learning rate by the number of workers:\n", - "\n", - "```python\n", - " opt = tf.keras.optimizers.SGD(0.01 * hvd.size())\n", - " opt = hvd.DistributedOptimizer(opt)\n", - "```\n", - "\n", - "- We wrap the TensorFlow gradient tape with Horovod's `DistributedGradientTape` class:\n", - "\n", - "```python\n", - " with tf.GradientTape() as tape:\n", - " ...\n", - " tape = hvd.DistributedGradientTape(tape, sparse_as_dense=True)\n", - "```\n", - "\n", - "- After the first batch, we broadcast the model and optimizer parameters to all workers with Horovod:\n", - "\n", - "```python\n", - " # Note: broadcast should be done after the first gradient step to\n", - " # ensure optimizer initialization.\n", - " if first_batch:\n", - " hvd.broadcast_variables(model.variables, root_rank=0)\n", - " hvd.broadcast_variables(opt.variables(), root_rank=0)\n", - "```\n", - "\n", - "- We only save checkpoints from the first worker to avoid multiple workers trying to write to the same files:\n", - "\n", - "```python\n", - " if hvd.rank() == 0:\n", - " checkpoint.save(checkpoint_dir)\n", - "```\n", - "\n", - "The rest of the script is the same as the MovieLens example in [getting-started-movielens/03-Training-with-TF.ipynb](../getting-started-movielens/03-Training-with-TF.ipynb). In order to run it with Horovod, we first need to write it to a file." - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "id": "99a00b6b", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Overwriting ./tf_trainer.py\n" - ] - } - ], - "source": [ - "%%writefile './tf_trainer.py'\n", - "\n", - "# External dependencies\n", - "import argparse\n", - "import glob\n", - "import os\n", - "\n", - "import cupy\n", - "\n", - "# we can control how much memory to give tensorflow with this environment variable\n", - "# IMPORTANT: make sure you do this before you initialize TF's runtime, otherwise\n", - "# TF will have claimed all free GPU memory\n", - "os.environ[\"TF_MEMORY_ALLOCATION\"] = \"0.3\" # fraction of free memory\n", - "\n", - "import nvtabular as nvt # noqa: E402 isort:skip\n", - "from nvtabular.framework_utils.tensorflow import layers # noqa: E402 isort:skip\n", - "from nvtabular.loader.tensorflow import KerasSequenceLoader # noqa: E402 isort:skip\n", - "\n", - "import tensorflow as tf # noqa: E402 isort:skip\n", - "import horovod.tensorflow as hvd # noqa: E402 isort:skip\n", - "\n", - "parser = argparse.ArgumentParser(description=\"Process some integers.\")\n", - "parser.add_argument(\"--dir_in\", default=None, help=\"Input directory\")\n", - "parser.add_argument(\"--batch_size\", default=None, help=\"batch size\")\n", - "parser.add_argument(\"--cats\", default=None, help=\"categorical columns\")\n", - "parser.add_argument(\"--cats_mh\", default=None, help=\"categorical multihot columns\")\n", - "parser.add_argument(\"--conts\", default=None, help=\"continuous columns\")\n", - "parser.add_argument(\"--labels\", default=None, help=\"continuous columns\")\n", - "args = parser.parse_args()\n", - "\n", - "\n", - "BASE_DIR = args.dir_in or \"./data/\"\n", - "BATCH_SIZE = int(args.batch_size or 16384) # Batch Size\n", - "CATEGORICAL_COLUMNS = args.cats or [\"movieId\", \"userId\"] # Single-hot\n", - "CATEGORICAL_MH_COLUMNS = args.cats_mh or [\"genres\"] # Multi-hot\n", - "NUMERIC_COLUMNS = args.conts or []\n", - "TRAIN_PATHS = sorted(\n", - " glob.glob(os.path.join(BASE_DIR, \"train/*.parquet\"))\n", - ") # Output from ETL-with-NVTabular\n", - "hvd.init()\n", - "\n", - "# Seed with system randomness (or a static seed)\n", - "cupy.random.seed(None)\n", - "\n", - "\n", - "def seed_fn():\n", - " \"\"\"\n", - " Generate consistent dataloader shuffle seeds across workers\n", - "\n", - " Reseeds each worker's dataloader each epoch to get fresh a shuffle\n", - " that's consistent across workers.\n", - " \"\"\"\n", - " min_int, max_int = tf.int32.limits\n", - " max_rand = max_int // hvd.size()\n", - "\n", - " # Generate a seed fragment on each worker\n", - " seed_fragment = cupy.random.randint(0, max_rand).get()\n", - "\n", - " # Aggregate seed fragments from all Horovod workers\n", - " seed_tensor = tf.constant(seed_fragment)\n", - " reduced_seed = hvd.allreduce(seed_tensor, name=\"shuffle_seed\", op=hvd.mpi_ops.Sum)\n", - "\n", - " return reduced_seed % max_rand\n", - "\n", - "\n", - "proc = nvt.Workflow.load(os.path.join(BASE_DIR, \"workflow/\"))\n", - "EMBEDDING_TABLE_SHAPES, MH_EMBEDDING_TABLE_SHAPES = nvt.ops.get_embedding_sizes(proc)\n", - "EMBEDDING_TABLE_SHAPES.update(MH_EMBEDDING_TABLE_SHAPES)\n", - "\n", - "train_dataset_tf = KerasSequenceLoader(\n", - " TRAIN_PATHS, # you could also use a glob pattern\n", - " batch_size=BATCH_SIZE,\n", - " label_names=[\"rating\"],\n", - " cat_names=CATEGORICAL_COLUMNS + CATEGORICAL_MH_COLUMNS,\n", - " cont_names=NUMERIC_COLUMNS,\n", - " engine=\"parquet\",\n", - " shuffle=True,\n", - " buffer_size=0.06, # how many batches to load at once\n", - " parts_per_chunk=1,\n", - " global_size=hvd.size(),\n", - " global_rank=hvd.rank(),\n", - " seed_fn=seed_fn,\n", - ")\n", - "inputs = {} # tf.keras.Input placeholders for each feature to be used\n", - "emb_layers = [] # output of all embedding layers, which will be concatenated\n", - "for col in CATEGORICAL_COLUMNS:\n", - " inputs[col] = tf.keras.Input(name=col, dtype=tf.int32, shape=(1,))\n", - "# Note that we need two input tensors for multi-hot categorical features\n", - "for col in CATEGORICAL_MH_COLUMNS:\n", - " inputs[col] = \\\n", - " (tf.keras.Input(name=f\"{col}__values\", dtype=tf.int64, shape=(1,)),\n", - " tf.keras.Input(name=f\"{col}__nnzs\", dtype=tf.int64, shape=(1,)))\n", - "for col in CATEGORICAL_COLUMNS + CATEGORICAL_MH_COLUMNS:\n", - " emb_layers.append(\n", - " tf.feature_column.embedding_column(\n", - " tf.feature_column.categorical_column_with_identity(\n", - " col, EMBEDDING_TABLE_SHAPES[col][0]\n", - " ), # Input dimension (vocab size)\n", - " EMBEDDING_TABLE_SHAPES[col][1], # Embedding output dimension\n", - " )\n", - " )\n", - "emb_layer = layers.DenseFeatures(emb_layers)\n", - "x_emb_output = emb_layer(inputs)\n", - "x = tf.keras.layers.Dense(128, activation=\"relu\")(x_emb_output)\n", - "x = tf.keras.layers.Dense(128, activation=\"relu\")(x)\n", - "x = tf.keras.layers.Dense(128, activation=\"relu\")(x)\n", - "x = tf.keras.layers.Dense(1, activation=\"sigmoid\")(x)\n", - "model = tf.keras.Model(inputs=inputs, outputs=x)\n", - "loss = tf.losses.BinaryCrossentropy()\n", - "opt = tf.keras.optimizers.SGD(0.01 * hvd.size())\n", - "opt = hvd.DistributedOptimizer(opt)\n", - "checkpoint_dir = \"./checkpoints\"\n", - "checkpoint = tf.train.Checkpoint(model=model, optimizer=opt)\n", - "\n", - "\n", - "@tf.function(experimental_relax_shapes=True)\n", - "def training_step(examples, labels, first_batch):\n", - " with tf.GradientTape() as tape:\n", - " probs = model(examples, training=True)\n", - " loss_value = loss(labels, probs)\n", - " # Horovod: add Horovod Distributed GradientTape.\n", - " tape = hvd.DistributedGradientTape(tape, sparse_as_dense=True)\n", - " grads = tape.gradient(loss_value, model.trainable_variables)\n", - " opt.apply_gradients(zip(grads, model.trainable_variables))\n", - " # Horovod: broadcast initial variable states from rank 0 to all other processes.\n", - " # This is necessary to ensure consistent initialization of all workers when\n", - " # training is started with random weights or restored from a checkpoint.\n", - " #\n", - " # Note: broadcast should be done after the first gradient step to ensure optimizer\n", - " # initialization.\n", - " if first_batch:\n", - " hvd.broadcast_variables(model.variables, root_rank=0)\n", - " hvd.broadcast_variables(opt.variables(), root_rank=0)\n", - " return loss_value\n", - "\n", - "\n", - "# Horovod: adjust number of steps based on number of GPUs.\n", - "for batch, (examples, labels) in enumerate(train_dataset_tf):\n", - " loss_value = training_step(examples, labels, batch == 0)\n", - " if batch % 100 == 0 and hvd.local_rank() == 0:\n", - " print(\"Step #%d\\tLoss: %.6f\" % (batch, loss_value))\n", - "hvd.join()\n", - "\n", - "# Horovod: save checkpoints only on worker 0 to prevent other workers from\n", - "# corrupting it.\n", - "if hvd.rank() == 0:\n", - " checkpoint.save(checkpoint_dir)" - ] - }, - { - "cell_type": "markdown", - "id": "c998f1e9", - "metadata": {}, - "source": [ - "We'll also need a small wrapper script to check environment variables set by the Horovod runner to see which rank we'll be assigned, in order to set CUDA_VISIBLE_DEVICES properly for each worker:" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "id": "c905420d", - "metadata": { - "tags": [ - "flake8-noqa-cell" - ] - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Overwriting ./hvd_wrapper.sh\n" - ] - } - ], - "source": [ - "%%writefile './hvd_wrapper.sh'\n", - "\n", - "#!/bin/bash\n", - "\n", - "# Get local process ID from OpenMPI or alternatively from SLURM\n", - "if [ -z \"${CUDA_VISIBLE_DEVICES:-}\" ]; then\n", - " if [ -n \"${OMPI_COMM_WORLD_LOCAL_RANK:-}\" ]; then\n", - " LOCAL_RANK=\"${OMPI_COMM_WORLD_LOCAL_RANK}\"\n", - " elif [ -n \"${SLURM_LOCALID:-}\" ]; then\n", - " LOCAL_RANK=\"${SLURM_LOCALID}\"\n", - " fi\n", - " export CUDA_VISIBLE_DEVICES=${LOCAL_RANK}\n", - "fi\n", - "\n", - "exec \"$@\"" - ] - }, - { - "cell_type": "markdown", - "id": "8bf8300c", - "metadata": {}, - "source": [ - "OpenMPI and Slurm are tools for running distributed computed jobs. In this example, we’re using OpenMPI, but depending on the environment you run distributed training jobs in, you may need to check slightly different environment variables to find the total number of workers (global size) and each process’s worker number (global rank.)\n", - "\n", - "Why do we have to check environment variables instead of using `hvd.rank()` and `hvd.local_rank()`? NVTabular does some GPU configuration when imported and needs to be imported before Horovod to avoid conflicts. We need to set GPU visibility before NVTabular is imported (when Horovod isn’t yet available) so that multiple processes don’t each try to configure all the GPUs, so as a workaround, we “cheat” and peek at environment variables set by horovodrun to decide which GPU each process should use." - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "id": "1a0b3979", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2021-06-04 16:39:06.000313: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0\n", - "[1,0]:2021-06-04 16:39:08.979997: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0\n", - "[1,1]:2021-06-04 16:39:09.064191: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0\n", - "[1,0]:2021-06-04 16:39:10.138200: I tensorflow/compiler/jit/xla_cpu_device.cc:41] Not creating XLA devices, tf_xla_enable_xla_devices not set\n", - "[1,0]:2021-06-04 16:39:10.138376: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcuda.so.1\n", - "[1,0]:2021-06-04 16:39:10.139777: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1746] Found device 0 with properties: \n", - "[1,0]:pciBusID: 0000:0b:00.0 name: GeForce GTX 1080 Ti computeCapability: 6.1\n", - "[1,0]:coreClock: 1.582GHz coreCount: 28 deviceMemorySize: 10.91GiB deviceMemoryBandwidth: 451.17GiB/s\n", - "[1,0]:2021-06-04 16:39:10.139823: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0\n", - "[1,0]:2021-06-04 16:39:10.139907: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11\n", - "[1,0]:2021-06-04 16:39:10.139949: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11\n", - "[1,0]:2021-06-04 16:39:10.139990: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10\n", - "[1,0]:2021-06-04 16:39:10.140029: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10\n", - "[1,0]:2021-06-04 16:39:10.140084: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.11\n", - "[1,0]:2021-06-04 16:39:10.140123: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11\n", - "[1,0]:2021-06-04 16:39:10.140169: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8\n", - "[1,0]:2021-06-04 16:39:10.144021: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1888] Adding visible gpu devices: 0\n", - "[1,1]:2021-06-04 16:39:10.367414: I tensorflow/compiler/jit/xla_cpu_device.cc:41] Not creating XLA devices, tf_xla_enable_xla_devices not set\n", - "[1,1]:2021-06-04 16:39:10.367496: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcuda.so.1\n", - "[1,1]:2021-06-04 16:39:10.368324: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1746] Found device 0 with properties: \n", - "[1,1]:pciBusID: 0000:42:00.0 name: GeForce GTX 1080 Ti computeCapability: 6.1\n", - "[1,1]:coreClock: 1.582GHz coreCount: 28 deviceMemorySize: 10.92GiB deviceMemoryBandwidth: 451.17GiB/s\n", - "[1,1]:2021-06-04 16:39:10.368347: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0\n", - "[1,1]:2021-06-04 16:39:10.368396: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11\n", - "[1,1]:2021-06-04 16:39:10.368424: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11\n", - "[1,1]:2021-06-04 16:39:10.368451: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10\n", - "[1,1]:2021-06-04 16:39:10.368475: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10\n", - "[1,1]:2021-06-04 16:39:10.368512: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.11\n", - "[1,1]:2021-06-04 16:39:10.368537: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11\n", - "[1,1]:2021-06-04 16:39:10.368573: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8\n", - "[1,1]:2021-06-04 16:39:10.369841: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1888] Adding visible gpu devices: 0\n", - "[1,1]:2021-06-04 16:39:11.730033: I tensorflow/compiler/jit/xla_gpu_device.cc:99] Not creating XLA devices, tf_xla_enable_xla_devices not set\n", - "[1,1]:2021-06-04 16:39:11.730907: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1746] Found device 0 with properties: \n", - "[1,1]:pciBusID: 0000:42:00.0 name: GeForce GTX 1080 Ti computeCapability: 6.1\n", - "[1,1]:coreClock: 1.582GHz coreCount: 28 deviceMemorySize: 10.92GiB deviceMemoryBandwidth: 451.17GiB/s\n", - "[1,1]:2021-06-04 16:39:11.730990: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0\n", - "[1,1]:2021-06-04 16:39:11.731005: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11\n", - "[1,1]:2021-06-04 16:39:11.731018: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11\n", - "[1,1]:2021-06-04 16:39:11.731029: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10\n", - "[1,1]:2021-06-04 16:39:11.731038: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10\n", - "[1,1]:2021-06-04 16:39:11.731049: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.11\n", - "[1,1]:2021-06-04 16:39:11.731059: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11\n", - "[1,1]:2021-06-04 16:39:11.731078: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8\n", - "[1,1]:2021-06-04 16:39:11.732312: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1888] Adding visible gpu devices: 0\n", - "[1,1]:2021-06-04 16:39:11.732350: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0\n", - "[1,1]:2021-06-04 16:39:11.732473: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1287] Device interconnect StreamExecutor with strength 1 edge matrix:\n", - "[1,1]:2021-06-04 16:39:11.732487: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1293] 0 \n", - "[1,1]:2021-06-04 16:39:11.732493: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1306] 0: N \n", - "[1,1]:2021-06-04 16:39:11.734431: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1432] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 3352 MB memory) -> physical GPU (device: 0, name: GeForce GTX 1080 Ti, pci bus id: 0000:42:00.0, compute capability: 6.1)\n", - "[1,0]:2021-06-04 16:39:11.821346: I tensorflow/compiler/jit/xla_gpu_device.cc:99] Not creating XLA devices, tf_xla_enable_xla_devices not set\n", - "[1,0]:2021-06-04 16:39:11.822270: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1746] Found device 0 with properties: \n", - "[1,0]:pciBusID: 0000:0b:00.0 name: GeForce GTX 1080 Ti computeCapability: 6.1\n", - "[1,0]:coreClock: 1.582GHz coreCount: 28 deviceMemorySize: 10.91GiB deviceMemoryBandwidth: 451.17GiB/s\n", - "[1,0]:2021-06-04 16:39:11.822360: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0\n", - "[1,0]:2021-06-04 16:39:11.822376: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11\n", - "[1,0]:2021-06-04 16:39:11.822389: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11\n", - "[1,0]:2021-06-04 16:39:11.822400: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10\n", - "[1,0]:2021-06-04 16:39:11.822411: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10\n", - "[1,0]:2021-06-04 16:39:11.822425: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.11\n", - "[1,0]:2021-06-04 16:39:11.822434: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11\n", - "[1,0]:2021-06-04 16:39:11.822454: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8\n", - "[1,0]:2021-06-04 16:39:11.823684: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1888] Adding visible gpu devices: 0\n", - "[1,0]:2021-06-04 16:39:11.823731: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0\n", - "[1,0]:2021-06-04 16:39:11.823868: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1287] Device interconnect StreamExecutor with strength 1 edge matrix:\n", - "[1,0]:2021-06-04 16:39:11.823881: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1293] 0 \n", - "[1,0]:2021-06-04 16:39:11.823888: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1306] 0: N \n", - "[1,0]:2021-06-04 16:39:11.825784: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1432] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 3352 MB memory) -> physical GPU (device: 0, name: GeForce GTX 1080 Ti, pci bus id: 0000:0b:00.0, compute capability: 6.1)\n", - "[1,0]:2021-06-04 16:39:17.634485: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:116] None of the MLIR optimization passes are enabled (registered 2)\n", - "[1,0]:2021-06-04 16:39:17.668915: I tensorflow/core/platform/profile_utils/cpu_utils.cc:112] CPU Frequency: 2993950000 Hz\n", - "[1,1]:2021-06-04 16:39:17.694128: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:116] None of the MLIR optimization passes are enabled (registered 2)\n", - "[1,1]:2021-06-04 16:39:17.703326: I tensorflow/core/platform/profile_utils/cpu_utils.cc:112] CPU Frequency: 2993950000 Hz\n", - "[1,0]:2021-06-04 16:39:17.780825: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11\n", - "[1,1]:2021-06-04 16:39:17.810644: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11\n", - "[1,0]:2021-06-04 16:39:17.984966: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11\n", - "[1,1]:2021-06-04 16:39:18.012113: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11\n", - "[1,0]:Step #0\tLoss: 0.695094\n", - "[1,0]:Step #100\tLoss: 0.669580\n", - "[1,0]:Step #200\tLoss: 0.661098\n", - "[1,0]:Step #300\tLoss: 0.660680\n", - "[1,0]:Step #400\tLoss: 0.658633\n", - "[1,0]:Step #500\tLoss: 0.660251\n", - "[1,0]:Step #600\tLoss: 0.657047\n" - ] - } - ], - "source": [ - "!horovodrun -np 2 sh hvd_wrapper.sh python tf_trainer.py --dir_in $BASE_DIR --batch_size 16384" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.10" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/examples/multi-gpu-movielens/README.md b/examples/multi-gpu-movielens/README.md deleted file mode 100644 index 611376603e3..00000000000 --- a/examples/multi-gpu-movielens/README.md +++ /dev/null @@ -1,13 +0,0 @@ -# Multi-GPU Example Notebooks - -The following notebooks demonstrate how to perform multi-GPU training with NVTabluar, TensorFlow, and PyTorch: - -- [Training with TensorFlow on MovieLens Data](01-03-MultiGPU-Download-Convert-ETL-with-NVTabular-Training-with-TensorFlow.ipynb) -- [Learn about NVTabular and Dask](../multi-gpu-toy-example/multi-gpu_dask.ipynb) - -See the [examples/multi-gpu-movielens](https://github.com/NVIDIA-Merlin/NVTabular/tree/main/examples/multi-gpu-movielens) -of the repository on GitHub to view the following that are related to the notebooks: - -- `hvd_wrapper.sh` -- `tf_trainer.py` -- `torch_trainer_dist.py` diff --git a/examples/multi-gpu-movielens/hvd_wrapper.sh b/examples/multi-gpu-movielens/hvd_wrapper.sh deleted file mode 100644 index 919064bdb94..00000000000 --- a/examples/multi-gpu-movielens/hvd_wrapper.sh +++ /dev/null @@ -1,14 +0,0 @@ - -#!/bin/bash - -# Get local process ID from OpenMPI or alternatively from SLURM -if [ -z "${CUDA_VISIBLE_DEVICES:-}" ]; then - if [ -n "${OMPI_COMM_WORLD_LOCAL_RANK:-}" ]; then - LOCAL_RANK="${OMPI_COMM_WORLD_LOCAL_RANK}" - elif [ -n "${SLURM_LOCALID:-}" ]; then - LOCAL_RANK="${SLURM_LOCALID}" - fi - export CUDA_VISIBLE_DEVICES=${LOCAL_RANK} -fi - -exec "$@" diff --git a/examples/multi-gpu-movielens/tf_trainer.py b/examples/multi-gpu-movielens/tf_trainer.py deleted file mode 100644 index f635c83e6d7..00000000000 --- a/examples/multi-gpu-movielens/tf_trainer.py +++ /dev/null @@ -1,147 +0,0 @@ -# External dependencies -import argparse -import glob -import os - -import cupy - -# we can control how much memory to give tensorflow with this environment variable -# IMPORTANT: make sure you do this before you initialize TF's runtime, otherwise -# TF will have claimed all free GPU memory -os.environ["TF_MEMORY_ALLOCATION"] = "0.3" # fraction of free memory - -import nvtabular as nvt # noqa: E402 isort:skip -from nvtabular.framework_utils.tensorflow import layers # noqa: E402 isort:skip -from nvtabular.loader.tensorflow import KerasSequenceLoader # noqa: E402 isort:skip - -import tensorflow as tf # noqa: E402 isort:skip -import horovod.tensorflow as hvd # noqa: E402 isort:skip - -parser = argparse.ArgumentParser(description="Process some integers.") -parser.add_argument("--dir_in", default=None, help="Input directory") -parser.add_argument("--batch_size", default=None, help="batch size") -parser.add_argument("--cats", default=None, help="categorical columns") -parser.add_argument("--cats_mh", default=None, help="categorical multihot columns") -parser.add_argument("--conts", default=None, help="continuous columns") -parser.add_argument("--labels", default=None, help="continuous columns") -args = parser.parse_args() - - -BASE_DIR = args.dir_in or "./data/" -BATCH_SIZE = int(args.batch_size or 16384) # Batch Size -CATEGORICAL_COLUMNS = args.cats or ["movieId", "userId"] # Single-hot -CATEGORICAL_MH_COLUMNS = args.cats_mh or ["genres"] # Multi-hot -NUMERIC_COLUMNS = args.conts or [] -TRAIN_PATHS = sorted( - glob.glob(os.path.join(BASE_DIR, "train/*.parquet")) -) # Output from ETL-with-NVTabular -hvd.init() - -# Seed with system randomness (or a static seed) -cupy.random.seed(None) - - -def seed_fn(): - """ - Generate consistent dataloader shuffle seeds across workers - - Reseeds each worker's dataloader each epoch to get fresh a shuffle - that's consistent across workers. - """ - min_int, max_int = tf.int32.limits - max_rand = max_int // hvd.size() - - # Generate a seed fragment on each worker - seed_fragment = cupy.random.randint(0, max_rand).get() - - # Aggregate seed fragments from all Horovod workers - seed_tensor = tf.constant(seed_fragment) - reduced_seed = hvd.allreduce(seed_tensor, name="shuffle_seed", op=hvd.mpi_ops.Sum) - - return reduced_seed % max_rand - - -proc = nvt.Workflow.load(os.path.join(BASE_DIR, "workflow/")) -EMBEDDING_TABLE_SHAPES, MH_EMBEDDING_TABLE_SHAPES = nvt.ops.get_embedding_sizes(proc) -EMBEDDING_TABLE_SHAPES.update(MH_EMBEDDING_TABLE_SHAPES) - - -train_dataset_tf = KerasSequenceLoader( - TRAIN_PATHS, # you could also use a glob pattern - batch_size=BATCH_SIZE, - label_names=["rating"], - cat_names=CATEGORICAL_COLUMNS + CATEGORICAL_MH_COLUMNS, - cont_names=NUMERIC_COLUMNS, - engine="parquet", - shuffle=True, - buffer_size=0.06, # how many batches to load at once - parts_per_chunk=1, - global_size=hvd.size(), - global_rank=hvd.rank(), - seed_fn=seed_fn, -) -inputs = {} # tf.keras.Input placeholders for each feature to be used -emb_layers = [] # output of all embedding layers, which will be concatenated -for col in CATEGORICAL_COLUMNS: - inputs[col] = tf.keras.Input(name=col, dtype=tf.int32, shape=(1,)) -# Note that we need two input tensors for multi-hot categorical features -for col in CATEGORICAL_MH_COLUMNS: - inputs[col] = ( - tf.keras.Input(name=f"{col}__values", dtype=tf.int64, shape=(1,)), - tf.keras.Input(name=f"{col}__nnzs", dtype=tf.int64, shape=(1,)), - ) -for col in CATEGORICAL_COLUMNS + CATEGORICAL_MH_COLUMNS: - emb_layers.append( - tf.feature_column.embedding_column( - tf.feature_column.categorical_column_with_identity( - col, EMBEDDING_TABLE_SHAPES[col][0] - ), # Input dimension (vocab size) - EMBEDDING_TABLE_SHAPES[col][1], # Embedding output dimension - ) - ) -emb_layer = layers.DenseFeatures(emb_layers) -x_emb_output = emb_layer(inputs) -x = tf.keras.layers.Dense(128, activation="relu")(x_emb_output) -x = tf.keras.layers.Dense(128, activation="relu")(x) -x = tf.keras.layers.Dense(128, activation="relu")(x) -x = tf.keras.layers.Dense(1, activation="sigmoid")(x) -model = tf.keras.Model(inputs=inputs, outputs=x) -loss = tf.losses.BinaryCrossentropy() -opt = tf.keras.optimizers.SGD(0.01 * hvd.size()) -opt = hvd.DistributedOptimizer(opt) -checkpoint_dir = "./checkpoints" -checkpoint = tf.train.Checkpoint(model=model, optimizer=opt) - - -@tf.function(experimental_relax_shapes=True) -def training_step(examples, labels, first_batch): - with tf.GradientTape() as tape: - probs = model(examples, training=True) - loss_value = loss(labels, probs) - # Horovod: add Horovod Distributed GradientTape. - tape = hvd.DistributedGradientTape(tape, sparse_as_dense=True) - grads = tape.gradient(loss_value, model.trainable_variables) - opt.apply_gradients(zip(grads, model.trainable_variables)) - # Horovod: broadcast initial variable states from rank 0 to all other processes. - # This is necessary to ensure consistent initialization of all workers when - # training is started with random weights or restored from a checkpoint. - # - # Note: broadcast should be done after the first gradient step to ensure optimizer - # initialization. - if first_batch: - hvd.broadcast_variables(model.variables, root_rank=0) - hvd.broadcast_variables(opt.variables(), root_rank=0) - return loss_value - - -# Horovod: adjust number of steps based on number of GPUs. -for batch, (examples, labels) in enumerate(train_dataset_tf): - loss_value = training_step(examples, labels, batch == 0) - if batch % 100 == 0 and hvd.local_rank() == 0: - print("Step #%d\tLoss: %.6f" % (batch, loss_value)) -hvd.join() - -# Horovod: save checkpoints only on worker 0 to prevent other workers from -# corrupting it. -if hvd.rank() == 0: - checkpoint.save(checkpoint_dir) diff --git a/examples/multi-gpu-toy-example/multi-gpu_dask.ipynb b/examples/multi-gpu-toy-example/multi-gpu_dask.ipynb deleted file mode 100644 index 1001bbd9dd1..00000000000 --- a/examples/multi-gpu-toy-example/multi-gpu_dask.ipynb +++ /dev/null @@ -1,1142 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "# Copyright 2021 NVIDIA Corporation. All Rights Reserved.\n", - "#\n", - "# Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# http://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License.\n", - "# ==============================================================================" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Multi-GPU Scaling in NVTabular with Dask\n", - "\n", - "\n", - "## NVTabular + Dask Integration\n", - "\n", - "NVTabular enables the use of [Dask](https://dask.org/) for multi-GPU parallelism, which integrates the following classes with the [RAPIDS](https://rapids.ai/) Dask-CuDF library:\n", - "\n", - "- **nvtabular.Dataset**: Most NVTabular functionality requires the raw data to be converted to a Dataset object. The conversion is very inexpensive, as it requires minimal IO (if any at all). A Dataset can be initialized using file/directory paths (\"csv\" or \"parquet\"), a PyArrow Table, a Pandas/CuDF DataFrame, or a Pandas/CuDF-based *Dask* DataFrame. The purpose of this \"wrapper\" class is to provide other NVTabular components with reliable mechanisms to (1) translate the target data into a Dask collection, and to (2) iterate over the target data in small-enough chunks to fit comfortably in GPU memory.\n", - "- **nvtabular.Workflow**: This is the central class used in NVTabular to compose a GPU-accelerated preprocessing pipeline. The Workflow class now tracks the state of the underlying data by applying all operations to an internal Dask-CuDF DataFrame object (`ddf`).\n", - "- **nvtabular.ops.StatOperator**: All \"statistics-gathering\" operations must be designed to operate directly on the Workflow object's internal `ddf`. This requirement facilitates the ability of NVTabular to handle the calculation of global statistics in a scalable way.\n", - "\n", - "**Big Picture**: NVTabular is tightly integrated with Dask-CuDF. By representing the underlying dataset as a (lazily-evaluated) collection of CuDF DataFrame objects (i.e. a single `dask_cudf.DataFrame`), we can seamlessly scale our preprocessing workflow to multiple GPUs.\n", - "\n", - "## Simple Multi-GPU Toy Example\n", - "In order to illustrate the Dask-CuDF-based functionality of NVTabular, we will walk through a simple preprocessing example using *toy* data.\n", - "\n", - "#### Resolving Memory Errors\n", - "This notebook was developed on a DGX-1 system (8 V100 GPUs with 1TB host memory). Users with limited device and/or host memory (less than 16GB on device, and less than 32GB on host) may need to modify one or more of the default options. Here are the best places to start:\n", - "\n", - "- `device_memory_limit`: Reduce the memory limit for workers in your cluster. This setting may need to be much lower than the actual memory capacity of your device.\n", - "- `part_mem_fraction`: Reduce the partition size of your Dataset. Smaller partition sizes enable better control over memory spilling on the workers (but reduces compute efficiency).\n", - "- `out_files_per_proc`: Increase the number of output files per worker. The worker must be able to shuffle each output file in device memory for the per-worker shuffling algorithm.\n", - "- `shuffle`: Change the shuffling option to `Shuffle.PER_PARTITION` in `workflow.apply`. The default (per-worker) option currently requires the entire output dataset to fit in host memory.\n", - "\n", - "### Step 1: Import Libraries and Cleanup Working Directories" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "# Standard Libraries\n", - "import os\n", - "import shutil\n", - "\n", - "# External Dependencies\n", - "import cupy as cp\n", - "import cudf\n", - "import dask_cudf\n", - "from dask_cuda import LocalCUDACluster\n", - "from dask.distributed import Client\n", - "from dask.delayed import delayed\n", - "import rmm\n", - "\n", - "# NVTabular\n", - "import nvtabular as nvt\n", - "import nvtabular.ops as ops\n", - "from merlin.io import Shuffle\n", - "from merlin.core.utils import device_mem_size" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Note that it is often a good idea to set-aside (fast) dedicated disk space for Dask \"workers\" to spill data and write logging information. To make things simple, we will perform all IO within a single `BASE_DIR` for this example. Make sure to reset this environment variable as desired." - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "# Choose a \"fast\" root directory for this example\n", - "BASE_DIR = os.environ.get(\"BASE_DIR\", \"./basedir\")\n", - "\n", - "# Define and clean our worker/output directories\n", - "dask_workdir = os.path.join(BASE_DIR, \"workdir\")\n", - "demo_output_path = os.path.join(BASE_DIR, \"demo_output\")\n", - "demo_dataset_path = os.path.join(BASE_DIR, \"demo_dataset\")\n", - "\n", - "# Ensure BASE_DIR exists\n", - "if not os.path.isdir(BASE_DIR):\n", - " os.mkdir(BASE_DIR)\n", - "\n", - "# Make sure we have a clean worker space for Dask\n", - "if os.path.isdir(dask_workdir):\n", - " shutil.rmtree(dask_workdir)\n", - "os.mkdir(dask_workdir)\n", - "\n", - "# Make sure we have a clean output path\n", - "if os.path.isdir(demo_output_path):\n", - " shutil.rmtree(demo_output_path)\n", - "os.mkdir(demo_output_path)\n", - "\n", - "# Get device memory capacity\n", - "capacity = device_mem_size(kind=\"total\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Step 2: Deploy a Distributed-Dask Cluster\n", - "\n", - "Before we walk through the rest of this multi-GPU preprocessing example, it is important to reiterate that Dask-CuDF is used extensively within NVTabular. This essentially means that you do **not** need to do anything special to *use* Dask here. With that said, the default behavior of NVTabular is to to utilize Dask's [\"synchronous\"](https://docs.dask.org/en/latest/scheduling.html) task scheduler, which precludes distributed processing. In order to properly utilize a multi-GPU system, you need to deploy a `dask.distributed` *cluster*.\n", - "\n", - "There are many different ways to create a distributed Dask cluster. This notebook will focus only on the `LocalCUDACluster` API, which is provided by the RAPIDS [Dask-CUDA](https://github.com/rapidsai/dask-cuda) library. It is also recommended that you check out [this blog article](https://blog.dask.org/2020/07/23/current-state-of-distributed-dask-clusters) to see a high-level summary of the many other cluster-deployment utilities.\n", - "\n", - "For this example, we will assume that you want to perform preprocessing on a single machine with multiple GPUs. In this case, we can use `dask_cuda.LocalCUDACluster` to deploy a distributed cluster with each worker process being pinned to a distinct GPU. This class also provides our workers with mechanisms for device-to-host memory spilling (explained below), and (optionally) enables the use of NVLink and infiniband-based inter-process communication via UCX." - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "\n", - "\n", - "\n", - "\n", - "
\n", - "

Client

\n", - "\n", - "		\n", - "

Cluster

\n", - "
    \n", - "
  • Workers: 4
    • \n", - "
  • Cores: 4
    • \n", - "
  • Memory: 1.08 TB
    • \n", - "
\n", - "
" - ], - "text/plain": [ - "" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# Deploy a Single-Machine Multi-GPU Cluster\n", - "protocol = \"tcp\" # \"tcp\" or \"ucx\"\n", - "visible_devices = \"0,1,2,3\" # Delect devices to place workers\n", - "device_spill_frac = 0.9 # Spill GPU-Worker memory to host at this limit.\n", - "# Reduce if spilling fails to prevent\n", - "# device memory errors.\n", - "cluster = None # (Optional) Specify existing scheduler port\n", - "if cluster is None:\n", - " cluster = LocalCUDACluster(\n", - " protocol=protocol,\n", - " CUDA_VISIBLE_DEVICES=visible_devices,\n", - " local_directory=dask_workdir,\n", - " device_memory_limit=capacity * device_spill_frac,\n", - " )\n", - "\n", - "# Create the distributed client\n", - "client = Client(cluster)\n", - "client" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### The Dask Diagnostics Dashboard\n", - "\n", - "If you created a new distributed cluster in the previous cell, the output should specify the address of a [diagnostics dashboard](https://docs.dask.org/en/latest/diagnostics-distributed.html) (e.g. **Dashboard**: http://IP:8787/status). You can also run `client.dashboard_link` to get the same information. If you have [Bokeh](https://bokeh.org/) installed in your environment, the scheduler will create this dashboard by default. If you click on the link, or paste the url in a web browser, you will see a page that looks something like the figure below. Note that you may need to update the IP address in the link if you are working on a remote machine.\n", - "\n", - "![dask-dashboard.png](../../images/dask-dashboard.png)\n", - " \n", - "The Dask dashboard is typically the best way to visualize the execution progress and resource usage of a Multi-GPU NVTabular workflow. For [JupyterLab](https://jupyterlab.readthedocs.io/en/stable/) users, the [Dask JupyterLab Extension](https://github.com/dask/dask-labextension) further integrates the same diagnostic figures into the notebook environment itself.\n", - "\n", - "#### Device-to-Host Memory Spilling\n", - "\n", - "One of the advantages of using [Dask-CUDA](https://github.com/rapidsai/dask-cuda) to deploy a distributed cluster is that the workers will move data between device memory and host memory, and between host memory and disk, to avoid out-of-memory (OOM) errors. To set the threshold for device-to-host spilling, a specific byte size can be specified with `device_memory_limit`. Since the worker can only consider the size of input data, and previously finished task output, this limit must be set lower than the actual GPU memory capacity. If the limit is set too high, temporary memory allocations within the execution of task may lead to OOM. With that said, since spilling can dramatically reduce the overall performance of a workflow, a conservative `device_memory_limit` setting is only advised when it proves absolutely necessary (i.e. heavy spilling is deemed inevitable for a given workflow).\n", - "\n", - "#### Initializing Memory Pools\n", - "\n", - "Since allocating memory is often a performance bottleneck, it is usually a good idea to initialize a memory pool on each of our workers. When using a distributed cluster, we must use the `client.run` utility to make sure a function is executed on all available workers." - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'tcp://127.0.0.1:35199': None,\n", - " 'tcp://127.0.0.1:36255': None,\n", - " 'tcp://127.0.0.1:40587': None,\n", - " 'tcp://127.0.0.1:43255': None}" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# Initialize RMM pool on ALL workers\n", - "def _rmm_pool():\n", - " rmm.reinitialize(\n", - " pool_allocator=True,\n", - " initial_pool_size=None, # Use default size\n", - " )\n", - "\n", - "\n", - "client.run(_rmm_pool)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "**Note**: If you have problems with this, it *may* be a `numba-0.51` problem. Try: `conda install -c conda-forge numba=0.50`\n", - "\n", - "\n", - "### Step 3: Create a \"Toy\" Parquet Dataset\n", - "In order to illustrate the power of multi-GPU scaling, without requiring an excessive runtime, we can use the `cudf.datasets.timeseries` API to generate a largish (~20GB) toy dataset with Dask-CuDF." - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 7.39 s, sys: 2.75 s, total: 10.1 s\n", - "Wall time: 3min 31s\n" - ] - } - ], - "source": [ - "%%time\n", - "\n", - "# Write a \"largish\" dataset (~20GB).\n", - "# Change `write_count` and/or `freq` for larger or smaller dataset.\n", - "# Avoid re-writing dataset if it already exists.\n", - "write_count = 25\n", - "freq = \"1s\"\n", - "if not os.path.exists(demo_dataset_path):\n", - "\n", - " def make_df(freq, i):\n", - " df = cudf.datasets.timeseries(\n", - " start=\"2000-01-01\", end=\"2000-12-31\", freq=freq, seed=i\n", - " ).reset_index(drop=False)\n", - " df[\"name\"] = df[\"name\"].astype(\"object\")\n", - " df[\"label\"] = cp.random.choice(cp.array([0, 1], dtype=\"uint8\"), len(df))\n", - " return df\n", - "\n", - " dfs = [delayed(make_df)(freq, i) for i in range(write_count)]\n", - " dask_cudf.from_delayed(dfs).to_parquet(demo_dataset_path, write_index=False)\n", - " del dfs" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Step 4: Create an NVTabular Dataset object\n", - "\n", - "As discussed above, the `nvt.Workflow` class requires data to be represented as an `nvt.Dataset`. This convention allows NVTabular to abstract away the raw format of the data, and convert everything to a consistent `dask_cudf.DataFrame` representation. Since the `Dataset` API effectively wraps functions like `dask_cudf.read_csv`, the syntax is very simple and the computational cost is minimal.\n", - "\n", - "**Important Dataset API Considerations**:\n", - "\n", - "- Can be initialized with the following objects:\n", - " - 1+ file/directory paths. An `engine` argument is required to specify the file format (unless file names are appended with `csv` or `parquet`)\n", - " - `cudf.DataFrame`. Internal `ddf` will have 1 partition.\n", - " - `pandas.DataFrame`. Internal `ddf` will have 1 partition.\n", - " - `pyarrow.Table`. Internal `ddf` will have 1 partition.\n", - " - `dask_cudf.DataFrame`. Internal `ddf` will be a shallow copy of the input.\n", - " - `dask.dataframe.DataFrame`. Internal `ddf` will be a direct pandas->cudf conversion of the input.\n", - "- For file-based data initialization, the size of the internal `ddf` partitions will be chosen according to the following arguments (in order of precedence):\n", - " - `part_size`: Desired maximum size of each partition **in bytes**. Note that you can pass a string here. like `\"2GB\"`.\n", - " - `part_mem_fraction`: Desired maximum size of each partition as a **fraction of total GPU memory**.\n", - "\n", - "**Note on Dataset Partitioning**:\n", - "The `part_size` and `part_mem_fraction` options will be used to specify the desired maximum partition size **after** conversion to CuDF, not the partition size in parquet format (which may be compressed and/or dictionary encoded). For the \"parquet\" engine, these parameters do not result in the direct mapping of a file byte-range to a partition. Instead, the first row-group in the dataset is converted to a `cudf.DataFrame`, and the size of that DataFrame is used to estimate the number of contiguous row-groups to assign to each partition. In the current \"parquet\" engine implementation, row-groups stored in different files will always be mapped to different partitions." - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 114 ms, sys: 33.5 ms, total: 147 ms\n", - "Wall time: 2.88 s\n" - ] - } - ], - "source": [ - "%%time\n", - "# Create a Dataset\n", - "# (`engine` argument optional if file names appended with `csv` or `parquet`)\n", - "ds = nvt.Dataset(demo_dataset_path, engine=\"parquet\", part_size=\"500MB\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Once your data is converted to a Dataset object, it can be converted to a `dask_cudf.DataFrame` using the `to_ddf` method. The wonderful thing about this DataFrame object, is that you are free to operate on it using a familiar CuDF/Pandas API." - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
timestampidnamexylabel
02000-01-01 00:00:001019Michael0.168205-0.5472301
12000-01-01 00:00:01984Patricia-0.145077-0.2405210
22000-01-01 00:00:02935Victor0.557024-0.0988550
32000-01-01 00:00:03970Alice0.527366-0.6325691
42000-01-01 00:00:04997Dan0.3091930.7048450
\n", - "
" - ], - "text/plain": [ - " timestamp id name x y label\n", - "0 2000-01-01 00:00:00 1019 Michael 0.168205 -0.547230 1\n", - "1 2000-01-01 00:00:01 984 Patricia -0.145077 -0.240521 0\n", - "2 2000-01-01 00:00:02 935 Victor 0.557024 -0.098855 0\n", - "3 2000-01-01 00:00:03 970 Alice 0.527366 -0.632569 1\n", - "4 2000-01-01 00:00:04 997 Dan 0.309193 0.704845 0" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ds.to_ddf().head()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Note that the output of a Dataset (a `ddf`) can be used to initialize a new Dataset. This means we can use Dask-CuDF to perform complex ETL on our data before we process it in a Workflow. For example, although NVTabular does not support global shuffling transformations (yet), these operations **can** be performed before (and/or after) a Workflow. The catch here is that operations requiring the global movement of data between partitions can require more device memory than available." - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
Dask DataFrame Structure:
\n", - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
timestampidnamexylabel
npartitions=75
datetime64[us]int64objectfloat64float64uint8
..................
.....................
..................
..................
\n", - "
\n", - "
Dask Name: shuffle, 2019 tasks
" - ], - "text/plain": [ - "" - ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# Example of global shuffling outside an NVT Workflow\n", - "ddf = ds.to_ddf().shuffle(\"id\", ignore_index=True)\n", - "ds = nvt.Dataset(ddf)\n", - "ds.to_ddf()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Since global shuffling operations can lead to significant GPU-memory pressure, we will start with a simpler Dataset definition for this example." - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [], - "source": [ - "del ds\n", - "del ddf\n", - "\n", - "dataset = nvt.Dataset(demo_dataset_path, engine=\"parquet\", part_mem_fraction=0.1)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Note that the default value for part_mem_fraction (0.125) is usually safe, but we will use a slightly smaller partition size for this example to be conservative.\n", - "\n", - "**Note**: If you have a system with limited device and/or host memory (less than 16GB on device, and less than 32GB on host), you may need to use an even smaller `part_mem_fraction` here.\n", - "\n", - "### Step 5: Define our NVTabular Workflow\n", - "\n", - "Now that we have our Dask cluster up and running, we can use the NVTabular API as usual. For NVTabular versions newer than `0.9.0`, the global `client` (created above) will be used automatically for multi-GPU (or CPU) execution." - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [], - "source": [ - "cat_features = [\"name\", \"id\"] >> ops.Categorify(\n", - " out_path=demo_output_path, # Path to write unique values used for encoding\n", - ")\n", - "cont_features = [\"x\", \"y\"] >> ops.Normalize()\n", - "\n", - "workflow = nvt.Workflow(cat_features + cont_features + [\"label\", \"timestamp\"])" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Step 6: Apply our Workflow" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 3.73 s, sys: 1.27 s, total: 5 s\n", - "Wall time: 2min 19s\n" - ] - } - ], - "source": [ - "%%time\n", - "shuffle = Shuffle.PER_WORKER # Shuffle algorithm\n", - "out_files_per_proc = 8 # Number of output files per worker\n", - "workflow.fit_transform(dataset).to_parquet(\n", - " output_path=os.path.join(demo_output_path, \"processed\"),\n", - " shuffle=shuffle,\n", - " out_files_per_proc=out_files_per_proc,\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "For this (modestly sized) toy dataset, we get a great performance boost when we move from 1 to 2 V100 GPUs, and the workflow scales reasonably well to a full [DGX-1 system](https://www.nvidia.com/en-gb/data-center/dgx-systems/dgx-1/). Although the 8-GPU performance reflects a parallel efficiency of only 50% or so, higher effiencies can be expected for larger datasets. In fact, recent [TPCx-BB benchmarking studies](https://medium.com/rapids-ai/no-more-waiting-interactive-big-data-now-32f7b903cf41) have clearly demonstrated that NVTabular's parallel backend, Dask-CuDF, can effectively scale to many V100 or A100-based nodes (utilizing more than 100 GPUs).\n", - "\n", - "**Note on Communication**:\n", - "It is important to recognize that multi-GPU and multi-node scaling is typically much more successful with UCX support (enabling both NVLink and Infiniband communication).\n", - "\n", - "**Example Results**:\n", - "\n", - "**1 x 32GB V100 GPU**\n", - "```\n", - "CPU times: user 5.74 s, sys: 3.87 s, total: 9.62 s\n", - "Wall time: 50.9 s\n", - "```\n", - "\n", - "**2 x 32GB V100 GPUs**\n", - "```\n", - "CPU times: user 6.64 s, sys: 3.53 s, total: 10.2 s\n", - "Wall time: 24.3 s\n", - "```\n", - "\n", - "**8 x 32GB V100 GPUs**\n", - "```\n", - "CPU times: user 6.84 s, sys: 3.73 s, total: 10.6 s\n", - "Wall time: 13.5 s\n", - "```" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now that we are done executing our Workflow, we can check the output data to confirm that everything is looking good." - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
xytimestampnameidlabel
0-1.402733-0.4124432000-01-10 05:00:0941861
1-0.8754140.3209492000-01-04 13:51:37201800
21.1765521.0599502000-11-10 16:18:51151410
3-0.877984-0.8686872000-01-07 01:50:27141531
41.0457821.3826612000-02-27 08:11:4861850
\n", - "
" - ], - "text/plain": [ - " x y timestamp name id label\n", - "0 -1.402733 -0.412443 2000-01-10 05:00:09 4 186 1\n", - "1 -0.875414 0.320949 2000-01-04 13:51:37 20 180 0\n", - "2 1.176552 1.059950 2000-11-10 16:18:51 15 141 0\n", - "3 -0.877984 -0.868687 2000-01-07 01:50:27 14 153 1\n", - "4 1.045782 1.382661 2000-02-27 08:11:48 6 185 0" - ] - }, - "execution_count": 13, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "dask_cudf.read_parquet(os.path.join(demo_output_path, \"processed\")).head()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Step 7: (Optional) Follow-up Processing/Writing with dask_cudf\n", - "\n", - "Instead of using to_parquet to persist your processed dataset to disk, it is also possible to get a dask dataframe from the transformed dataset and perform follow-up operations with the Dask-CuDF API. For example, if you want to convert the entire dataset into a `groupby` aggregation, you could do something like the following." - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 191 ms, sys: 4.06 ms, total: 195 ms\n", - "Wall time: 2.26 s\n" - ] - } - ], - "source": [ - "%%time\n", - "ddf = workflow.transform(dataset).to_ddf()\n", - "ddf = ddf.groupby([\"name\"]).max() # Optional follow-up processing\n", - "ddf.to_parquet(os.path.join(demo_output_path, \"dask_output\"), write_index=False)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "As always, we can use either `nvt.Dataset` or `dask_cudf` directly to read back our data." - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
xytimestampidlabel
01.7320321.7319532000-12-30 23:59:593561
11.7320321.7319522000-12-31 00:00:003621
21.7320321.7319532000-12-31 00:00:003571
31.7320321.7319532000-12-31 00:00:003491
41.7320321.7319532000-12-31 00:00:003601
51.7320321.7319522000-12-30 23:59:593591
61.7320321.7319522000-12-31 00:00:003541
71.7320321.7319522000-12-30 23:59:583641
81.7320321.7319532000-12-31 00:00:003541
91.7320321.7319522000-12-31 00:00:003591
101.7320321.7319522000-12-31 00:00:003571
111.7320321.7319532000-12-31 00:00:003491
121.7320321.7319522000-12-30 23:59:583621
131.7320321.7319532000-12-31 00:00:003611
141.7320321.7319522000-12-31 00:00:003521
151.7320321.7319532000-12-30 23:59:583531
161.7320321.7319532000-12-31 00:00:003491
171.7320321.7319532000-12-31 00:00:003531
181.7320321.7319532000-12-31 00:00:003601
191.7320321.7319522000-12-30 23:59:583511
201.7320321.7319522000-12-30 23:59:593631
211.7320321.7319532000-12-30 23:59:573571
221.7320321.7319522000-12-31 00:00:003651
231.7320321.7319522000-12-31 00:00:003501
241.7320321.7319522000-12-30 23:59:593531
251.7320321.7319522000-12-30 23:59:593591
\n", - "
" - ], - "text/plain": [ - " x y timestamp id label\n", - "0 1.732032 1.731953 2000-12-30 23:59:59 356 1\n", - "1 1.732032 1.731952 2000-12-31 00:00:00 362 1\n", - "2 1.732032 1.731953 2000-12-31 00:00:00 357 1\n", - "3 1.732032 1.731953 2000-12-31 00:00:00 349 1\n", - "4 1.732032 1.731953 2000-12-31 00:00:00 360 1\n", - "5 1.732032 1.731952 2000-12-30 23:59:59 359 1\n", - "6 1.732032 1.731952 2000-12-31 00:00:00 354 1\n", - "7 1.732032 1.731952 2000-12-30 23:59:58 364 1\n", - "8 1.732032 1.731953 2000-12-31 00:00:00 354 1\n", - "9 1.732032 1.731952 2000-12-31 00:00:00 359 1\n", - "10 1.732032 1.731952 2000-12-31 00:00:00 357 1\n", - "11 1.732032 1.731953 2000-12-31 00:00:00 349 1\n", - "12 1.732032 1.731952 2000-12-30 23:59:58 362 1\n", - "13 1.732032 1.731953 2000-12-31 00:00:00 361 1\n", - "14 1.732032 1.731952 2000-12-31 00:00:00 352 1\n", - "15 1.732032 1.731953 2000-12-30 23:59:58 353 1\n", - "16 1.732032 1.731953 2000-12-31 00:00:00 349 1\n", - "17 1.732032 1.731953 2000-12-31 00:00:00 353 1\n", - "18 1.732032 1.731953 2000-12-31 00:00:00 360 1\n", - "19 1.732032 1.731952 2000-12-30 23:59:58 351 1\n", - "20 1.732032 1.731952 2000-12-30 23:59:59 363 1\n", - "21 1.732032 1.731953 2000-12-30 23:59:57 357 1\n", - "22 1.732032 1.731952 2000-12-31 00:00:00 365 1\n", - "23 1.732032 1.731952 2000-12-31 00:00:00 350 1\n", - "24 1.732032 1.731952 2000-12-30 23:59:59 353 1\n", - "25 1.732032 1.731952 2000-12-30 23:59:59 359 1" - ] - }, - "execution_count": 15, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "dask_cudf.read_parquet(os.path.join(demo_output_path, \"dask_output\")).compute()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Notes on Shuffling\n", - "\n", - "NVTabular currently supports two shuffling options when writing output to disk: \n", - "\n", - "- `nvt.io.Shuffle.PER_PARTITION`\n", - "- `nvt.io.Shuffle.PER_WORKER`\n", - "\n", - "For both these cases, the partitions of the underlying dataset/ddf are randomly ordered before any processing is performed. If `PER_PARTITION` is specified, each worker/process will also shuffle the rows within each partition before splitting and appending the data to a number (`out_files_per_proc`) of output files. Output files are distinctly mapped to each worker process. If `PER_WORKER` is specified, each worker will follow the same procedure as `PER_PARTITION`, but will re-shuffle each file after all data is persisted. This results in a full shuffle of the data processed by each worker. To improve performance, this option currently uses host-memory `BytesIO` objects for the intermediate persist stage. The general `PER_WORKER` algorithm is illustrated here:\n", - "\n", - "![image.png](../../images/per_worker_shuffle.png)\n" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.6" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/examples/scaling-criteo/01-Download-Convert.ipynb b/examples/scaling-criteo/01-Download-Convert.ipynb deleted file mode 100644 index 3b0d67b16ff..00000000000 --- a/examples/scaling-criteo/01-Download-Convert.ipynb +++ /dev/null @@ -1,304 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "# Copyright 2021 NVIDIA Corporation. All Rights Reserved.\n", - "#\n", - "# Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# http://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License.\n", - "# ==============================================================================" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "# Scaling Criteo: Download and Convert\n", - "\n", - "## Criteo 1TB Click Logs dataset\n", - "\n", - "The [Criteo 1TB Click Logs dataset](https://ailab.criteo.com/download-criteo-1tb-click-logs-dataset/) is the largest public available dataset for recommender system. It contains ~1.3 TB of uncompressed click logs containing over four billion samples spanning 24 days. Each record contains 40 features: one label indicating a click or no click, 13 numerical figures, and 26 categorical features. The dataset is provided by CriteoLabs. A subset of 7 days was used in this [Kaggle Competition](https://www.kaggle.com/c/criteo-display-ad-challenge/overview). We will use the dataset as an example how to scale ETL, Training and Inference." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "First, we will download the data and extract it. We define the base directory for the dataset and the numbers of day. Criteo provides 24 days. We will use the last day as validation dataset and the remaining days as training. \n", - "\n", - "**Each day has a size of ~15GB compressed `.gz` and uncompressed ~XXXGB. You can define a smaller subset of days, if you like. Each day takes ~20-30min to download and extract it.**" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "\n", - "from merlin.core.utils import download_file\n", - "\n", - "download_criteo = True\n", - "BASE_DIR = os.environ.get(\"BASE_DIR\", \"/raid/data/criteo\")\n", - "input_path = os.path.join(BASE_DIR, \"crit_orig\")\n", - "NUMBER_DAYS = os.environ.get(\"NUMBER_DAYS\", 2)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We create the folder structure and download and extract the files. If the file already exist, it will be skipped." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "933e7da7339647308a9b3cd0ca4a6b3a", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "downloading day_1.gz: 0.00B [00:00, ?B/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "%%time\n", - "if download_criteo:\n", - "\n", - " # Test if NUMBER_DAYS in valid range\n", - " if NUMBER_DAYS < 2 or NUMBER_DAYS > 23:\n", - " raise ValueError(\n", - " str(NUMBER_DAYS)\n", - " + \" is not supported. A minimum of 2 days are \"\n", - " + \"required and a maximum of 24 (0-23 days) are available\"\n", - " )\n", - "\n", - " # Create BASE_DIR if not exists\n", - " if not os.path.exists(BASE_DIR):\n", - " os.makedirs(BASE_DIR)\n", - "\n", - " # Create input dir if not exists\n", - " if not os.path.exists(input_path):\n", - " os.makedirs(input_path)\n", - "\n", - " # Iterate over days\n", - " for i in range(0, NUMBER_DAYS):\n", - " file = os.path.join(input_path, \"day_\" + str(i) + \".gz\")\n", - " # Download file, if there is no .gz, .csv or .parquet file\n", - " if not (\n", - " os.path.exists(file)\n", - " or os.path.exists(\n", - " file.replace(\".gz\", \".parquet\").replace(\"crit_orig\", \"converted/criteo/\")\n", - " )\n", - " or os.path.exists(file.replace(\".gz\", \"\"))\n", - " ):\n", - " download_file(\n", - " \"https://storage.googleapis.com/criteo-cail-datasets/day_\"\n", - " + str(i)\n", - " + \".gz\",\n", - " file,\n", - " )" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The original dataset is in text format. We will convert the dataset into `.parquet` format. Parquet is a compressed, column-oriented file structure and requires less disk space." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Conversion Script for Criteo Dataset (CSV-to-Parquet) \n", - "\n", - "__Step 1__: Import libraries" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "import glob\n", - "\n", - "import numpy as np\n", - "from dask.distributed import Client\n", - "from dask_cuda import LocalCUDACluster\n", - "\n", - "import nvtabular as nvt\n", - "from merlin.core.utils import device_mem_size, get_rmm_size" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "__Step 2__: Specify options\n", - "\n", - "Specify the input and output paths, unless the `INPUT_DATA_DIR` and `OUTPUT_DATA_DIR` environment variables are already set. For multi-GPU systems, check that the `CUDA_VISIBLE_DEVICES` environment variable includes all desired device IDs." - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [], - "source": [ - "INPUT_PATH = os.environ.get(\"INPUT_DATA_DIR\", input_path)\n", - "OUTPUT_PATH = os.environ.get(\"OUTPUT_DATA_DIR\", os.path.join(BASE_DIR, \"converted\"))\n", - "CUDA_VISIBLE_DEVICES = os.environ.get(\"CUDA_VISIBLE_DEVICES\", \"0\")\n", - "frac_size = 0.10" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "__Step 3__: (Optionally) Start a Dask cluster" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [], - "source": [ - "cluster = None # Connect to existing cluster if desired\n", - "if cluster is None:\n", - " cluster = LocalCUDACluster(\n", - " CUDA_VISIBLE_DEVICES=CUDA_VISIBLE_DEVICES,\n", - " rmm_pool_size=get_rmm_size(0.8 * device_mem_size()),\n", - " local_directory=os.path.join(OUTPUT_PATH, \"dask-space\"),\n", - " )\n", - "client = Client(cluster)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "__Step 5__: Convert original data to an NVTabular Dataset" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [], - "source": [ - "# Specify column names\n", - "cont_names = [\"I\" + str(x) for x in range(1, 14)]\n", - "cat_names = [\"C\" + str(x) for x in range(1, 27)]\n", - "cols = [\"label\"] + cont_names + cat_names\n", - "\n", - "# Specify column dtypes. Note that \"hex\" means that\n", - "# the values will be hexadecimal strings that should\n", - "# be converted to int32\n", - "dtypes = {}\n", - "dtypes[\"label\"] = np.int32\n", - "for x in cont_names:\n", - " dtypes[x] = np.int32\n", - "for x in cat_names:\n", - " dtypes[x] = \"hex\"\n", - "\n", - "# Create an NVTabular Dataset from a CSV-file glob\n", - "file_list = glob.glob(os.path.join(INPUT_PATH, \"day_*[!.gz]\"))\n", - "dataset = nvt.Dataset(\n", - " file_list,\n", - " engine=\"csv\",\n", - " names=cols,\n", - " part_mem_fraction=frac_size,\n", - " sep=\"\\t\",\n", - " dtypes=dtypes,\n", - " client=client,\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "**__Step 6__**: Write Dataset to Parquet" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 8.59 s, sys: 2.83 s, total: 11.4 s\n", - "Wall time: 5min 55s\n" - ] - } - ], - "source": [ - "dataset.to_parquet(\n", - " os.path.join(OUTPUT_PATH, \"criteo\"),\n", - " preserve_files=True,\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "You can delete the original criteo files as they require a lot of disk space." - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.10" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/examples/scaling-criteo/02-ETL-with-NVTabular.ipynb b/examples/scaling-criteo/02-ETL-with-NVTabular.ipynb deleted file mode 100644 index dfdc623b91d..00000000000 --- a/examples/scaling-criteo/02-ETL-with-NVTabular.ipynb +++ /dev/null @@ -1,567 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": { - "collapsed": false, - "jupyter": { - "outputs_hidden": false - } - }, - "outputs": [], - "source": [ - "# Copyright 2021 NVIDIA Corporation. All Rights Reserved.\n", - "#\n", - "# Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# http://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License.\n", - "# ==============================================================================" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "# Scaling Criteo: ETL with NVTabular\n", - "\n", - "## Overview\n", - "\n", - "NVTabular is a feature engineering and preprocessing library for tabular data designed to quickly and easily manipulate terabyte scale datasets used to train deep learning based recommender systems. It provides a high level abstraction to simplify code and accelerates computation on the GPU using the RAPIDS cuDF library.

\n", - "\n", - "**In this notebook, we will show how to scale NVTabular to multi-GPUs and multiple nodes.** Prerequisite is to be familiar with NVTabular and its API. You can read more NVTabular and its API in our [Getting Started with Movielens notebooks](https://github.com/NVIDIA/NVTabular/tree/main/examples/getting-started-movielens).

\n", - "\n", - "The full [Criteo 1TB Click Logs dataset](https://ailab.criteo.com/download-criteo-1tb-click-logs-dataset/) contains ~1.3 TB of uncompressed click logs containing over four billion samples spanning 24 days. In our benchmarks, we are able to preprocess and engineer features in **13.8min with 1x NVIDIA A100 GPU and 1.9min with 8x NVIDIA A100 GPUs**. This is a **speed-up of 100x-10000x** in comparison to different CPU versions, You can read more in our [blog](https://developer.nvidia.com/blog/announcing-the-nvtabular-open-beta-with-multi-gpu-support-and-new-data-loaders/).\n", - "\n", - "Our pipeline will be representative with most common preprocessing transformation for deep learning recommender models.\n", - "\n", - "* Categorical input features are `Categorified` to be continuous integers (0, ..., |C|) for the embedding layers\n", - "* Missing values of continuous input features are filled with 0. Afterwards the continuous features are clipped and normalized.\n", - "\n", - "### Learning objectives\n", - "In this notebook, we learn how to to scale ETLs with NVTabular\n", - "\n", - "- Learn to use larger than GPU/host memory datasets\n", - "- Use multi-GPU or multi node for ETL\n", - "- Apply common deep learning ETL workflow\n", - "\n", - "### Multi-GPU and multi-node scaling\n", - "\n", - "NVTabular is built on top off [RAPIDS.AI cuDF](https://github.com/rapidsai/cudf/), [dask_cudf](https://docs.rapids.ai/api/cudf/stable/dask-cudf.html) and [dask](https://dask.org/).

\n", - "**Dask** is a task-based library for parallel scheduling and execution. Although it is certainly possible to use the task-scheduling machinery directly to implement customized parallel workflows (we do it in NVTabular), most users only interact with Dask through a Dask Collection API. The most popular \"collection\" API's include:\n", - "\n", - "* Dask DataFrame: Dask-based version of the Pandas DataFrame/Series API. Note that dask_cudf is just a wrapper around this collection module (dask.dataframe).\n", - "* Dask Array: Dask-based version of the NumPy array API\n", - "* Dask Bag: Similar to a Dask-based version of PyToolz or a Pythonic version of PySpark RDD\n", - "\n", - "For example, Dask DataFrame provides a convenient API for decomposing large Pandas (or cuDF) DataFrame/Series objects into a collection of DataFrame partitions.\n", - "\n", - "\n", - "\n", - "We use **dask_cudf** to process large datasets as a collection of cuDF dataframes instead of Pandas. CuDF is a GPU DataFrame library for loading, joining, aggregating, filtering, and otherwise manipulating data.\n", - "

\n", - "**Dask enables easily to schedule tasks for multiple workers: multi-GPU or multi-node. We just need to initialize a Dask cluster (`LocalCUDACluster`) and NVTabular will use the cluster to execute the workflow.**" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## ETL with NVTabular\n", - "Here we'll show how to use NVTabular first as a preprocessing library to prepare the [Criteo 1TB Click Logs dataset](https://ailab.criteo.com/download-criteo-1tb-click-logs-dataset/) dataset. The following notebooks can use the output to train a deep learning model.\n", - "\n", - "### Data Prep\n", - "The previous notebook [01-Download-Convert](./01-Download-Convert.ipynb) converted the tsv data published by Criteo into the parquet format that our accelerated readers prefer. Accelerating these pipelines on new hardware like GPUs may require us to make new choices about the representations we use to store that data, and parquet represents a strong alternative." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We load the required libraries." - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "collapsed": false, - "jupyter": { - "outputs_hidden": false - } - }, - "outputs": [], - "source": [ - "# Standard Libraries\n", - "import os\n", - "import re\n", - "import shutil\n", - "import warnings\n", - "\n", - "# External Dependencies\n", - "import numpy as np\n", - "import numba\n", - "from dask_cuda import LocalCUDACluster\n", - "from dask.distributed import Client\n", - "\n", - "# NVTabular\n", - "import nvtabular as nvt\n", - "from nvtabular.ops import (\n", - " Categorify,\n", - " Clip,\n", - " FillMissing,\n", - " Normalize,\n", - ")\n", - "from nvtabular.utils import pynvml_mem_size, device_mem_size" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Once our data is ready, we'll define some high level parameters to describe where our data is and what it \"looks like\" at a high level." - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": { - "collapsed": false, - "jupyter": { - "outputs_hidden": false - } - }, - "outputs": [], - "source": [ - "# define some information about where to get our data\n", - "BASE_DIR = os.environ.get(\"BASE_DIR\", \"/raid/data/criteo\")\n", - "INPUT_DATA_DIR = os.environ.get(\"INPUT_DATA_DIR\", BASE_DIR + \"/converted/criteo\")\n", - "OUTPUT_DATA_DIR = os.environ.get(\"OUTPUT_DATA_DIR\", BASE_DIR + \"/test_dask/output\")\n", - "stats_path = os.path.join(OUTPUT_DATA_DIR, \"test_dask/stats\")\n", - "dask_workdir = os.path.join(OUTPUT_DATA_DIR, \"test_dask/workdir\")\n", - "\n", - "# Make sure we have a clean worker space for Dask\n", - "if os.path.isdir(dask_workdir):\n", - " shutil.rmtree(dask_workdir)\n", - "os.makedirs(dask_workdir)\n", - "\n", - "# Make sure we have a clean stats space for Dask\n", - "if os.path.isdir(stats_path):\n", - " shutil.rmtree(stats_path)\n", - "os.mkdir(stats_path)\n", - "\n", - "# Make sure we have a clean output path\n", - "if os.path.isdir(OUTPUT_DATA_DIR):\n", - " shutil.rmtree(OUTPUT_DATA_DIR)\n", - "os.mkdir(OUTPUT_DATA_DIR)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We use the last day as validation dataset and the remaining days as training dataset." - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "fname = \"day_{}.parquet\"\n", - "num_days = len(\n", - " [i for i in os.listdir(INPUT_DATA_DIR) if re.match(fname.format(\"[0-9]{1,2}\"), i) is not None]\n", - ")\n", - "train_paths = [os.path.join(INPUT_DATA_DIR, fname.format(day)) for day in range(num_days - 1)]\n", - "valid_paths = [\n", - " os.path.join(INPUT_DATA_DIR, fname.format(day)) for day in range(num_days - 1, num_days)\n", - "]" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": { - "collapsed": false, - "jupyter": { - "outputs_hidden": false - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "['/raid/criteo/tests/crit_int_pq/day_0.parquet', '/raid/criteo/tests/crit_int_pq/day_1.parquet', '/raid/criteo/tests/crit_int_pq/day_2.parquet', '/raid/criteo/tests/crit_int_pq/day_3.parquet', '/raid/criteo/tests/crit_int_pq/day_4.parquet', '/raid/criteo/tests/crit_int_pq/day_5.parquet', '/raid/criteo/tests/crit_int_pq/day_6.parquet', '/raid/criteo/tests/crit_int_pq/day_7.parquet', '/raid/criteo/tests/crit_int_pq/day_8.parquet', '/raid/criteo/tests/crit_int_pq/day_9.parquet', '/raid/criteo/tests/crit_int_pq/day_10.parquet', '/raid/criteo/tests/crit_int_pq/day_11.parquet', '/raid/criteo/tests/crit_int_pq/day_12.parquet', '/raid/criteo/tests/crit_int_pq/day_13.parquet', '/raid/criteo/tests/crit_int_pq/day_14.parquet', '/raid/criteo/tests/crit_int_pq/day_15.parquet', '/raid/criteo/tests/crit_int_pq/day_16.parquet', '/raid/criteo/tests/crit_int_pq/day_17.parquet', '/raid/criteo/tests/crit_int_pq/day_18.parquet', '/raid/criteo/tests/crit_int_pq/day_19.parquet', '/raid/criteo/tests/crit_int_pq/day_20.parquet', '/raid/criteo/tests/crit_int_pq/day_21.parquet', '/raid/criteo/tests/crit_int_pq/day_22.parquet']\n", - "['/raid/criteo/tests/crit_int_pq/day_23.parquet']\n" - ] - } - ], - "source": [ - "print(train_paths)\n", - "print(valid_paths)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Deploy a Distributed-Dask Cluster" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now we configure and deploy a Dask Cluster. Please, [read this document](https://github.com/NVIDIA/NVTabular/blob/d419a4da29cf372f1547edc536729b0733560a44/bench/examples/MultiGPUBench.md) to know how to set the parameters." - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": { - "collapsed": false, - "jupyter": { - "outputs_hidden": false - } - }, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "\n", - "\n", - "\n", - "\n", - "
\n", - "

Client

\n", - "\n", - "		\n", - "

Cluster

\n", - "
    \n", - "
  • Workers: 8
    • \n", - "
  • Cores: 8
    • \n", - "
  • Memory: 0.98 TiB
    • \n", - "
\n", - "
" - ], - "text/plain": [ - "" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# Dask dashboard\n", - "dashboard_port = \"8787\"\n", - "\n", - "# Deploy a Single-Machine Multi-GPU Cluster\n", - "protocol = \"tcp\" # \"tcp\" or \"ucx\"\n", - "if numba.cuda.is_available():\n", - " NUM_GPUS = list(range(len(numba.cuda.gpus)))\n", - "else:\n", - " NUM_GPUS = []\n", - "visible_devices = \",\".join([str(n) for n in NUM_GPUS]) # Delect devices to place workers\n", - "device_limit_frac = 0.7 # Spill GPU-Worker memory to host at this limit.\n", - "device_pool_frac = 0.8\n", - "part_mem_frac = 0.15\n", - "\n", - "# Use total device size to calculate args.device_limit_frac\n", - "device_size = device_mem_size(kind=\"total\")\n", - "device_limit = int(device_limit_frac * device_size)\n", - "device_pool_size = int(device_pool_frac * device_size)\n", - "part_size = int(part_mem_frac * device_size)\n", - "\n", - "# Check if any device memory is already occupied\n", - "for dev in visible_devices.split(\",\"):\n", - " fmem = pynvml_mem_size(kind=\"free\", index=int(dev))\n", - " used = (device_size - fmem) / 1e9\n", - " if used > 1.0:\n", - " warnings.warn(f\"BEWARE - {used} GB is already occupied on device {int(dev)}!\")\n", - "\n", - "cluster = None # (Optional) Specify existing scheduler port\n", - "if cluster is None:\n", - " cluster = LocalCUDACluster(\n", - " protocol=protocol,\n", - " n_workers=len(visible_devices.split(\",\")),\n", - " CUDA_VISIBLE_DEVICES=visible_devices,\n", - " device_memory_limit=device_limit,\n", - " local_directory=dask_workdir,\n", - " dashboard_address=\":\" + dashboard_port,\n", - " rmm_pool_size=(device_pool_size // 256) * 256\n", - " )\n", - "\n", - "# Create the distributed client\n", - "client = Client(cluster)\n", - "client" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "That's it. We initialized our Dask cluster and NVTabular will execute the workflow on multiple GPUs. Similar, we could define a cluster with multiple nodes." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Defining our Preprocessing Pipeline\n", - "At this point, our data still isn't in a form that's ideal for consumption by neural networks. The most pressing issues are missing values and the fact that our categorical variables are still represented by random, discrete identifiers, and need to be transformed into contiguous indices that can be leveraged by a learned embedding. Less pressing, but still important for learning dynamics, are the distributions of our continuous variables, which are distributed across multiple orders of magnitude and are uncentered (i.e. E[x] != 0).\n", - "\n", - "We can fix these issues in a conscise and GPU-accelerated manner with an NVTabular `Workflow`. We explained the NVTabular API in [Getting Started with Movielens notebooks](https://github.com/NVIDIA/NVTabular/tree/main/examples/getting-started-movielens) and hope you are familiar with the syntax.\n", - "\n", - "#### Frequency Thresholding\n", - "One interesting thing worth pointing out is that we're using _frequency thresholding_ in our `Categorify` op. This handy functionality will map all categories which occur in the dataset with some threshold level of infrequency (which we've set here to be 15 occurrences throughout the dataset) to the _same_ index, keeping the model from overfitting to sparse signals." - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": { - "collapsed": false, - "jupyter": { - "outputs_hidden": false - } - }, - "outputs": [], - "source": [ - "# define our dataset schema\n", - "CONTINUOUS_COLUMNS = [\"I\" + str(x) for x in range(1, 14)]\n", - "CATEGORICAL_COLUMNS = [\"C\" + str(x) for x in range(1, 27)]\n", - "LABEL_COLUMNS = [\"label\"]\n", - "COLUMNS = CONTINUOUS_COLUMNS + CATEGORICAL_COLUMNS + LABEL_COLUMNS\n", - "\n", - "num_buckets = 10000000\n", - "categorify_op = Categorify(out_path=stats_path, max_size=num_buckets)\n", - "cat_features = CATEGORICAL_COLUMNS >> categorify_op\n", - "cont_features = CONTINUOUS_COLUMNS >> FillMissing() >> Clip(min_value=0) >> Normalize()\n", - "features = cat_features + cont_features + LABEL_COLUMNS\n", - "\n", - "workflow = nvt.Workflow(features)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now instantiate dataset iterators to loop through our dataset (which we couldn't fit into GPU memory). We need to enforce the required HugeCTR data types, so we set them in a dictionary and give as an argument when creating our dataset." - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": { - "collapsed": false, - "jupyter": { - "outputs_hidden": false - } - }, - "outputs": [], - "source": [ - "dict_dtypes = {}\n", - "\n", - "for col in CATEGORICAL_COLUMNS:\n", - " dict_dtypes[col] = np.int64\n", - "\n", - "for col in CONTINUOUS_COLUMNS:\n", - " dict_dtypes[col] = np.float32\n", - "\n", - "for col in LABEL_COLUMNS:\n", - " dict_dtypes[col] = np.float32" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": { - "collapsed": false, - "jupyter": { - "outputs_hidden": false - } - }, - "outputs": [], - "source": [ - "train_dataset = nvt.Dataset(train_paths, engine=\"parquet\", part_size=part_size)\n", - "valid_dataset = nvt.Dataset(valid_paths, engine=\"parquet\", part_size=part_size)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now run them through our workflows to collect statistics on the train set, then transform and save to parquet files." - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": { - "collapsed": false, - "jupyter": { - "outputs_hidden": false - } - }, - "outputs": [], - "source": [ - "output_train_dir = os.path.join(OUTPUT_DATA_DIR, \"train/\")\n", - "output_valid_dir = os.path.join(OUTPUT_DATA_DIR, \"valid/\")\n", - "! mkdir -p $output_train_dir\n", - "! mkdir -p $output_valid_dir" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "For reference, let's time it to see how long it takes..." - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 18.6 s, sys: 2.24 s, total: 20.8 s\n", - "Wall time: 1min 5s\n" - ] - } - ], - "source": [ - "%%time\n", - "workflow.fit(train_dataset)" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": { - "collapsed": false, - "jupyter": { - "outputs_hidden": false - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 4.76 s, sys: 2.3 s, total: 7.06 s\n", - "Wall time: 1min 59s\n" - ] - } - ], - "source": [ - "%%time\n", - "# Add \"write_hugectr_keyset=True\" to \"to_parquet\" if using this ETL Notebook for training with HugeCTR\n", - "\n", - "workflow.transform(train_dataset).to_parquet(\n", - " output_files=len(NUM_GPUS),\n", - " output_path=output_train_dir,\n", - " shuffle=nvt.io.Shuffle.PER_PARTITION,\n", - " dtypes=dict_dtypes,\n", - " cats=CATEGORICAL_COLUMNS,\n", - " conts=CONTINUOUS_COLUMNS,\n", - " labels=LABEL_COLUMNS,\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": { - "collapsed": false, - "jupyter": { - "outputs_hidden": false - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 436 ms, sys: 140 ms, total: 576 ms\n", - "Wall time: 5.17 s\n" - ] - } - ], - "source": [ - "%%time\n", - "# Add \"write_hugectr_keyset=True\" to \"to_parquet\" if using this ETL Notebook for training with HugeCTR\n", - "\n", - "workflow.transform(valid_dataset).to_parquet(\n", - " output_path=output_valid_dir,\n", - " dtypes=dict_dtypes,\n", - " cats=CATEGORICAL_COLUMNS,\n", - " conts=CONTINUOUS_COLUMNS,\n", - " labels=LABEL_COLUMNS,\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "In the next notebooks, we will train a deep learning model. Our training pipeline requires information about the data schema to define the neural network architecture. We will save the NVTabular workflow to disk so that we can restore it in the next notebooks." - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [], - "source": [ - "workflow.save(os.path.join(OUTPUT_DATA_DIR, \"workflow\"))" - ] - } - ], - "metadata": { - "file_extension": ".py", - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.10" - }, - "mimetype": "text/x-python", - "npconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": 3 - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/examples/scaling-criteo/03-Training-with-HugeCTR.ipynb b/examples/scaling-criteo/03-Training-with-HugeCTR.ipynb deleted file mode 100644 index 47c30fb81cb..00000000000 --- a/examples/scaling-criteo/03-Training-with-HugeCTR.ipynb +++ /dev/null @@ -1,501 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "# Copyright 2021 NVIDIA Corporation. All Rights Reserved.\n", - "#\n", - "# Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# http://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License.\n", - "# ==============================================================================" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Scaling Criteo: Training with HugeCTR\n", - "\n", - "## Overview\n", - "\n", - "HugeCTR is an open-source framework to accelerate the training of CTR estimation models on NVIDIA GPUs. It is written in CUDA C++ and highly exploits GPU-accelerated libraries such as cuBLAS, cuDNN, and NCCL.

\n", - "HugeCTR offers multiple advantages to train deep learning recommender systems:\n", - "\n", - "1. **Speed**: HugeCTR is a highly efficient framework written C++. We experienced up to 10x speed up. HugeCTR on a NVIDIA DGX A100 system proved to be the fastest commercially available solution for training the architecture Deep Learning Recommender Model (DLRM) developed by Facebook.\n", - "2. **Scale**: HugeCTR supports model parallel scaling. It distributes the large embedding tables over multiple GPUs or multiple nodes. \n", - "3. **Easy-to-use**: Easy-to-use Python API similar to Keras. Examples for popular deep learning recommender systems architectures (Wide&Deep, DLRM, DCN, DeepFM) are available.\n", - "\n", - "HugeCTR is able to train recommender system models with larger-than-memory embedding tables by leveraging a parameter server. \n", - "\n", - "You can find more information about HugeCTR [here](https://github.com/NVIDIA/HugeCTR).\n", - "\n", - "### Learning objectives\n", - "\n", - "In this notebook, we learn how to to use HugeCTR for training recommender system models\n", - "\n", - "- Use **HugeCTR** to define a recommender system model\n", - "- Train Facebook's [Deep Learning Recommendation Model](https://arxiv.org/pdf/1906.00091.pdf) with HugeCTR" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Training with HugeCTR" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "As HugeCTR optimizes the training in CUDA++, we need to define the training pipeline and model architecture and execute it via the commandline. We will use the Python API, which is similar to Keras models." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "If you are not familiar with HugeCTR's Python API and parameters, you can read more in its GitHub repository:\n", - "- [HugeCTR User Guide](https://github.com/NVIDIA/HugeCTR/blob/master/docs/hugectr_user_guide.md)\n", - "- [HugeCTR Python API](https://github.com/NVIDIA/HugeCTR/blob/master/docs/python_interface.md)\n", - "- [HugeCTR example architectures](https://github.com/NVIDIA/HugeCTR/tree/master/samples)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We will write the code to a `./model.py` file and execute it. It will create snapshot, which we will use for inference in the next notebook." - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "test_dask train valid workflow\n" - ] - } - ], - "source": [ - "!ls /raid/data/criteo/test_dask/output/" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "0" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "import os\n", - "\n", - "os.system(\"rm -rf ./criteo_hugectr/\")\n", - "os.system(\"mkdir -p ./criteo_hugectr/1\")" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [], - "source": [ - "INPUT_DATA_DIR = os.environ.get(\"INPUT_DATA_DIR\", '/tmp/model/data')\n", - "data_path = os.path.join(INPUT_DATA_DIR, \"train\", \"_file_list.txt\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We use `graph_to_json` to convert the model to a JSON configuration, required for the inference." - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [], - "source": [ - "# %%writefile './model.py'\n", - "file_to_write = f\"\"\"\n", - "import hugectr\n", - "from mpi4py import MPI # noqa\n", - "\n", - "# HugeCTR\n", - "solver = hugectr.CreateSolver(\n", - " vvgpu=[[0]],\n", - " max_eval_batches=100,\n", - " batchsize_eval=2720,\n", - " batchsize=2720,\n", - " i64_input_key=True,\n", - " use_mixed_precision=False,\n", - " repeat_dataset=True,\n", - ")\n", - "optimizer = hugectr.CreateOptimizer(optimizer_type=hugectr.Optimizer_t.SGD)\n", - "reader = hugectr.DataReaderParams(\n", - " data_reader_type=hugectr.DataReaderType_t.Parquet,\n", - " source=[\"{data_path}\"],\n", - " eval_source=\"{data_path}\",\n", - " check_type=hugectr.Check_t.Non,\n", - " slot_size_array=[\n", - " 10000000,\n", - " 10000000,\n", - " 3014529,\n", - " 400781,\n", - " 11,\n", - " 2209,\n", - " 11869,\n", - " 148,\n", - " 4,\n", - " 977,\n", - " 15,\n", - " 38713,\n", - " 10000000,\n", - " 10000000,\n", - " 10000000,\n", - " 584616,\n", - " 12883,\n", - " 109,\n", - " 37,\n", - " 17177,\n", - " 7425,\n", - " 20266,\n", - " 4,\n", - " 7085,\n", - " 1535,\n", - " 64,\n", - " ],\n", - ")\n", - "model = hugectr.Model(solver, reader, optimizer)\n", - "model.add(\n", - " hugectr.Input(\n", - " label_dim=1,\n", - " label_name=\"label\",\n", - " dense_dim=13,\n", - " dense_name=\"dense\",\n", - " data_reader_sparse_param_array=[hugectr.DataReaderSparseParam(\"data1\", 1, False, 26)],\n", - " )\n", - ")\n", - "model.add(\n", - " hugectr.SparseEmbedding(\n", - " embedding_type=hugectr.Embedding_t.LocalizedSlotSparseEmbeddingHash,\n", - " workspace_size_per_gpu_in_mb=6000,\n", - " embedding_vec_size=128,\n", - " combiner=\"sum\",\n", - " sparse_embedding_name=\"sparse_embedding1\",\n", - " bottom_name=\"data1\",\n", - " optimizer=optimizer,\n", - " )\n", - ")\n", - "model.add(\n", - " hugectr.DenseLayer(\n", - " layer_type=hugectr.Layer_t.InnerProduct,\n", - " bottom_names=[\"dense\"],\n", - " top_names=[\"fc1\"],\n", - " num_output=512,\n", - " )\n", - ")\n", - "model.add(\n", - " hugectr.DenseLayer(layer_type=hugectr.Layer_t.ReLU, bottom_names=[\"fc1\"], top_names=[\"relu1\"])\n", - ")\n", - "model.add(\n", - " hugectr.DenseLayer(\n", - " layer_type=hugectr.Layer_t.InnerProduct,\n", - " bottom_names=[\"relu1\"],\n", - " top_names=[\"fc2\"],\n", - " num_output=256,\n", - " )\n", - ")\n", - "model.add(\n", - " hugectr.DenseLayer(layer_type=hugectr.Layer_t.ReLU, bottom_names=[\"fc2\"], top_names=[\"relu2\"])\n", - ")\n", - "model.add(\n", - " hugectr.DenseLayer(\n", - " layer_type=hugectr.Layer_t.InnerProduct,\n", - " bottom_names=[\"relu2\"],\n", - " top_names=[\"fc3\"],\n", - " num_output=128,\n", - " )\n", - ")\n", - "model.add(\n", - " hugectr.DenseLayer(layer_type=hugectr.Layer_t.ReLU, bottom_names=[\"fc3\"], top_names=[\"relu3\"])\n", - ")\n", - "model.add(\n", - " hugectr.DenseLayer(\n", - " layer_type=hugectr.Layer_t.Interaction,\n", - " bottom_names=[\"relu3\", \"sparse_embedding1\"],\n", - " top_names=[\"interaction1\"],\n", - " )\n", - ")\n", - "model.add(\n", - " hugectr.DenseLayer(\n", - " layer_type=hugectr.Layer_t.InnerProduct,\n", - " bottom_names=[\"interaction1\"],\n", - " top_names=[\"fc4\"],\n", - " num_output=1024,\n", - " )\n", - ")\n", - "model.add(\n", - " hugectr.DenseLayer(layer_type=hugectr.Layer_t.ReLU, bottom_names=[\"fc4\"], top_names=[\"relu4\"])\n", - ")\n", - "model.add(\n", - " hugectr.DenseLayer(\n", - " layer_type=hugectr.Layer_t.InnerProduct,\n", - " bottom_names=[\"relu4\"],\n", - " top_names=[\"fc5\"],\n", - " num_output=1024,\n", - " )\n", - ")\n", - "model.add(\n", - " hugectr.DenseLayer(layer_type=hugectr.Layer_t.ReLU, bottom_names=[\"fc5\"], top_names=[\"relu5\"])\n", - ")\n", - "model.add(\n", - " hugectr.DenseLayer(\n", - " layer_type=hugectr.Layer_t.InnerProduct,\n", - " bottom_names=[\"relu5\"],\n", - " top_names=[\"fc6\"],\n", - " num_output=512,\n", - " )\n", - ")\n", - "model.add(\n", - " hugectr.DenseLayer(layer_type=hugectr.Layer_t.ReLU, bottom_names=[\"fc6\"], top_names=[\"relu6\"])\n", - ")\n", - "model.add(\n", - " hugectr.DenseLayer(\n", - " layer_type=hugectr.Layer_t.InnerProduct,\n", - " bottom_names=[\"relu6\"],\n", - " top_names=[\"fc7\"],\n", - " num_output=256,\n", - " )\n", - ")\n", - "model.add(\n", - " hugectr.DenseLayer(layer_type=hugectr.Layer_t.ReLU, bottom_names=[\"fc7\"], top_names=[\"relu7\"])\n", - ")\n", - "model.add(\n", - " hugectr.DenseLayer(\n", - " layer_type=hugectr.Layer_t.InnerProduct,\n", - " bottom_names=[\"relu7\"],\n", - " top_names=[\"fc8\"],\n", - " num_output=1,\n", - " )\n", - ")\n", - "model.add(\n", - " hugectr.DenseLayer(\n", - " layer_type=hugectr.Layer_t.BinaryCrossEntropyLoss,\n", - " bottom_names=[\"fc8\", \"label\"],\n", - " top_names=[\"loss\"],\n", - " )\n", - ")\n", - "\n", - "MAX_ITER = 10000\n", - "EVAL_INTERVAL = 3200\n", - "model.compile()\n", - "model.summary()\n", - "model.fit(max_iter=MAX_ITER, eval_interval=EVAL_INTERVAL, display=1000, snapshot=3200)\n", - "model.graph_to_json(graph_config_file=\"./criteo_hugectr/1/criteo.json\")\n", - "\"\"\"\n", - "with open('./model.py', 'w', encoding='utf-8') as fi:\n", - " fi.write(file_to_write)" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "HugeCTR Version: 3.7\n", - "====================================================Model Init=====================================================\n", - "[HCTR][02:44:38.212][WARNING][RK0][main]: The model name is not specified when creating the solver.\n", - "[HCTR][02:44:38.212][WARNING][RK0][main]: MPI was already initialized somewhere elese. Lifetime service disabled.\n", - "[HCTR][02:44:38.212][INFO][RK0][main]: Global seed is 3391378239\n", - "[HCTR][02:44:38.255][INFO][RK0][main]: Device to NUMA mapping:\n", - " GPU 0 -> node 0\n", - "[HCTR][02:44:40.126][WARNING][RK0][main]: Peer-to-peer access cannot be fully enabled.\n", - "[HCTR][02:44:40.127][INFO][RK0][main]: Start all2all warmup\n", - "[HCTR][02:44:40.127][INFO][RK0][main]: End all2all warmup\n", - "[HCTR][02:44:40.127][INFO][RK0][main]: Using All-reduce algorithm: NCCL\n", - "[HCTR][02:44:40.127][INFO][RK0][main]: Device 0: Quadro RTX 8000\n", - "[HCTR][02:44:40.128][INFO][RK0][main]: num of DataReader workers: 1\n", - "[HCTR][02:44:40.129][INFO][RK0][main]: Vocabulary size: 54120457\n", - "[HCTR][02:44:40.130][INFO][RK0][main]: max_vocabulary_size_per_gpu_=12288000\n", - "[HCTR][02:44:40.130][DEBUG][RK0][tid #139916176520960]: file_name_ /tmp/pytest-of-root/pytest-9/test_criteo_hugectr0/tests/crit_test/train/part_0.parquet file_total_rows_ 138449698\n", - "[HCTR][02:44:40.130][DEBUG][RK0][tid #139916168128256]: file_name_ /tmp/pytest-of-root/pytest-9/test_criteo_hugectr0/tests/crit_test/train/part_0.parquet file_total_rows_ 138449698\n", - "[HCTR][02:44:40.138][INFO][RK0][main]: Graph analysis to resolve tensor dependency\n", - "===================================================Model Compile===================================================\n", - "[HCTR][02:44:55.150][INFO][RK0][main]: gpu0 start to init embedding\n", - "[HCTR][02:44:55.230][INFO][RK0][main]: gpu0 init embedding done\n", - "[HCTR][02:44:55.234][INFO][RK0][main]: Starting AUC NCCL warm-up\n", - "[HCTR][02:44:55.235][INFO][RK0][main]: Warm-up done\n", - "===================================================Model Summary===================================================\n", - "[HCTR][02:44:55.235][INFO][RK0][main]: label Dense Sparse \n", - "label dense data1 \n", - "(None, 1) (None, 13) \n", - "——————————————————————————————————————————————————————————————————————————————————————————————————————————————————\n", - "Layer Type Input Name Output Name Output Shape \n", - "——————————————————————————————————————————————————————————————————————————————————————————————————————————————————\n", - "LocalizedSlotSparseEmbeddingHash data1 sparse_embedding1 (None, 26, 128) \n", - "------------------------------------------------------------------------------------------------------------------\n", - "InnerProduct dense fc1 (None, 512) \n", - "------------------------------------------------------------------------------------------------------------------\n", - "ReLU fc1 relu1 (None, 512) \n", - "------------------------------------------------------------------------------------------------------------------\n", - "InnerProduct relu1 fc2 (None, 256) \n", - "------------------------------------------------------------------------------------------------------------------\n", - "ReLU fc2 relu2 (None, 256) \n", - "------------------------------------------------------------------------------------------------------------------\n", - "InnerProduct relu2 fc3 (None, 128) \n", - "------------------------------------------------------------------------------------------------------------------\n", - "ReLU fc3 relu3 (None, 128) \n", - "------------------------------------------------------------------------------------------------------------------\n", - "Interaction relu3 interaction1 (None, 480) \n", - " sparse_embedding1 \n", - "------------------------------------------------------------------------------------------------------------------\n", - "InnerProduct interaction1 fc4 (None, 1024) \n", - "------------------------------------------------------------------------------------------------------------------\n", - "ReLU fc4 relu4 (None, 1024) \n", - "------------------------------------------------------------------------------------------------------------------\n", - "InnerProduct relu4 fc5 (None, 1024) \n", - "------------------------------------------------------------------------------------------------------------------\n", - "ReLU fc5 relu5 (None, 1024) \n", - "------------------------------------------------------------------------------------------------------------------\n", - "InnerProduct relu5 fc6 (None, 512) \n", - "------------------------------------------------------------------------------------------------------------------\n", - "ReLU fc6 relu6 (None, 512) \n", - "------------------------------------------------------------------------------------------------------------------\n", - "InnerProduct relu6 fc7 (None, 256) \n", - "------------------------------------------------------------------------------------------------------------------\n", - "ReLU fc7 relu7 (None, 256) \n", - "------------------------------------------------------------------------------------------------------------------\n", - "InnerProduct relu7 fc8 (None, 1) \n", - "------------------------------------------------------------------------------------------------------------------\n", - "BinaryCrossEntropyLoss fc8 loss \n", - " label \n", - "------------------------------------------------------------------------------------------------------------------\n", - "=====================================================Model Fit=====================================================\n", - "[HCTR][02:44:55.235][INFO][RK0][main]: Use non-epoch mode with number of iterations: 10000\n", - "[HCTR][02:44:55.235][INFO][RK0][main]: Training batchsize: 2720, evaluation batchsize: 2720\n", - "[HCTR][02:44:55.235][INFO][RK0][main]: Evaluation interval: 3200, snapshot interval: 3200\n", - "[HCTR][02:44:55.235][INFO][RK0][main]: Dense network trainable: True\n", - "[HCTR][02:44:55.235][INFO][RK0][main]: Sparse embedding sparse_embedding1 trainable: True\n", - "[HCTR][02:44:55.235][INFO][RK0][main]: Use mixed precision: False, scaler: 1.000000, use cuda graph: True\n", - "[HCTR][02:44:55.235][INFO][RK0][main]: lr: 0.001000, warmup_steps: 1, end_lr: 0.000000\n", - "[HCTR][02:44:55.235][INFO][RK0][main]: decay_start: 0, decay_steps: 1, decay_power: 2.000000\n", - "[HCTR][02:44:55.235][INFO][RK0][main]: Training source file: /tmp/pytest-of-root/pytest-9/test_criteo_hugectr0/tests/crit_test/train/_file_list.txt\n", - "[HCTR][02:44:55.235][INFO][RK0][main]: Evaluation source file: /tmp/pytest-of-root/pytest-9/test_criteo_hugectr0/tests/crit_test/train/_file_list.txt\n", - "[HCTR][02:45:01.551][INFO][RK0][main]: Iter: 1000 Time(1000 iters): 6.31026s Loss: 0.170242 lr:0.001\n", - "[HCTR][02:45:08.116][INFO][RK0][main]: Iter: 2000 Time(1000 iters): 6.5595s Loss: 0.142086 lr:0.001\n", - "[HCTR][02:45:14.999][INFO][RK0][main]: Iter: 3000 Time(1000 iters): 6.87726s Loss: 0.144497 lr:0.001\n", - "[HCTR][02:45:16.619][INFO][RK0][main]: Evaluation, AUC: 0.522062\n", - "[HCTR][02:45:16.619][INFO][RK0][main]: Eval Time for 100 iters: 0.218802s\n", - "[HCTR][02:45:17.186][INFO][RK0][main]: Rank0: Dump hash table from GPU0\n", - "[HCTR][02:45:17.362][INFO][RK0][main]: Rank0: Write hash table pairs to file\n", - "[HCTR][02:45:18.490][INFO][RK0][main]: Done\n", - "[HCTR][02:45:18.802][INFO][RK0][main]: Dumping sparse weights to files, successful\n", - "[HCTR][02:45:18.802][INFO][RK0][main]: Dumping sparse optimzer states to files, successful\n", - "[HCTR][02:45:18.812][INFO][RK0][main]: Dumping dense weights to file, successful\n", - "[HCTR][02:45:18.812][INFO][RK0][main]: Dumping dense optimizer states to file, successful\n", - "[HCTR][02:45:24.512][INFO][RK0][main]: Iter: 4000 Time(1000 iters): 9.50778s Loss: 0.142673 lr:0.001\n", - "[HCTR][02:45:31.873][INFO][RK0][main]: Iter: 5000 Time(1000 iters): 7.35528s Loss: 0.13817 lr:0.001\n", - "[HCTR][02:45:39.491][INFO][RK0][main]: Iter: 6000 Time(1000 iters): 7.61235s Loss: 0.145115 lr:0.001\n", - "[HCTR][02:45:42.840][INFO][RK0][main]: Evaluation, AUC: 0.57392\n", - "[HCTR][02:45:42.840][INFO][RK0][main]: Eval Time for 100 iters: 0.249069s\n", - "[HCTR][02:45:43.756][INFO][RK0][main]: Rank0: Dump hash table from GPU0\n", - "[HCTR][02:45:44.043][INFO][RK0][main]: Rank0: Write hash table pairs to file\n", - "[HCTR][02:45:45.935][INFO][RK0][main]: Done\n", - "[HCTR][02:45:46.480][INFO][RK0][main]: Dumping sparse weights to files, successful\n", - "[HCTR][02:45:46.480][INFO][RK0][main]: Dumping sparse optimzer states to files, successful\n", - "[HCTR][02:45:46.486][INFO][RK0][main]: Dumping dense weights to file, successful\n", - "[HCTR][02:45:46.486][INFO][RK0][main]: Dumping dense optimizer states to file, successful\n", - "[HCTR][02:45:51.203][INFO][RK0][main]: Iter: 7000 Time(1000 iters): 11.7059s Loss: 0.138048 lr:0.001\n", - "[HCTR][02:45:59.222][INFO][RK0][main]: Iter: 8000 Time(1000 iters): 8.01361s Loss: 0.149459 lr:0.001\n", - "[HCTR][02:46:07.359][INFO][RK0][main]: Iter: 9000 Time(1000 iters): 8.1318s Loss: 0.152849 lr:0.001\n", - "[HCTR][02:46:12.572][INFO][RK0][main]: Evaluation, AUC: 0.624589\n", - "[HCTR][02:46:12.572][INFO][RK0][main]: Eval Time for 100 iters: 0.223472s\n", - "[HCTR][02:46:13.798][INFO][RK0][main]: Rank0: Dump hash table from GPU0\n", - "[HCTR][02:46:14.172][INFO][RK0][main]: Rank0: Write hash table pairs to file\n", - "[HCTR][02:46:16.936][INFO][RK0][main]: Done\n", - "[HCTR][02:46:17.654][INFO][RK0][main]: Dumping sparse weights to files, successful\n", - "[HCTR][02:46:17.655][INFO][RK0][main]: Dumping sparse optimzer states to files, successful\n", - "[HCTR][02:46:17.661][INFO][RK0][main]: Dumping dense weights to file, successful\n", - "[HCTR][02:46:17.661][INFO][RK0][main]: Dumping dense optimizer states to file, successful\n", - "[HCTR][02:46:21.006][INFO][RK0][main]: Finish 10000 iterations with batchsize: 2720 in 85.77s.\n", - "[HCTR][02:46:21.006][INFO][RK0][main]: Save the model graph to ./criteo_hugectr/1/criteo.json successfully\n", - "run_time: 104.00127220153809\n" - ] - } - ], - "source": [ - "import time\n", - "\n", - "start = time.time()\n", - "!python model.py\n", - "end = time.time() - start\n", - "print(f\"run_time: {end}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We trained the model and created snapshots." - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.10" - }, - "vscode": { - "interpreter": { - "hash": "916dbcbb3f70747c44a77c7bcd40155683ae19c65e1c03b4aa3499c5328201f1" - } - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/examples/scaling-criteo/03-Training-with-TF.ipynb b/examples/scaling-criteo/03-Training-with-TF.ipynb deleted file mode 100644 index 1fb23bdaaf8..00000000000 --- a/examples/scaling-criteo/03-Training-with-TF.ipynb +++ /dev/null @@ -1,537 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "# Copyright 2021 NVIDIA Corporation. All Rights Reserved.\n", - "#\n", - "# Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# http://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License.\n", - "# ==============================================================================" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Scaling Criteo: Training with TensorFlow\n", - "\n", - "## Overview\n", - "\n", - "We observed that TensorFlow training pipelines can be slow as the dataloader is a bottleneck. The native dataloader in TensorFlow randomly sample each item from the dataset, which is very slow. The window dataloader in TensorFlow is not much faster. In our experiments, we are able to speed-up existing TensorFlow pipelines by 9x using a highly optimized dataloader.

\n", - "\n", - "We have already discussed the NVTabular dataloader for TensorFlow in more detail in our [Getting Started with Movielens notebooks](https://github.com/NVIDIA/NVTabular/tree/main/examples/getting-started-movielens).

\n", - "\n", - "We will use the same techniques to train a deep learning model for the [Criteo 1TB Click Logs dataset](https://ailab.criteo.com/download-criteo-1tb-click-logs-dataset/).\n", - "\n", - "### Learning objectives\n", - "\n", - "In this notebook, we learn how to:\n", - "\n", - "- Use **NVTabular dataloader** with TensorFlow Keras model" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## NVTabular dataloader for TensorFlow\n", - "\n", - "We’ve identified that the dataloader is one bottleneck in deep learning recommender systems when training pipelines with TensorFlow. The dataloader cannot prepare the next batch fast enough and therefore, the GPU is not fully utilized. \n", - "\n", - "We developed a highly customized tabular dataloader for accelerating existing pipelines in TensorFlow. In our experiments, we see a speed-up by 9x of the same training workflow with NVTabular dataloader. NVTabular dataloader’s features are:\n", - "\n", - "- removing bottleneck of item-by-item dataloading\n", - "- enabling larger than memory dataset by streaming from disk\n", - "- reading data directly into GPU memory and remove CPU-GPU communication\n", - "- preparing batch asynchronously in GPU to avoid CPU-GPU communication\n", - "- supporting commonly used .parquet format\n", - "- easy integration into existing TensorFlow pipelines by using similar API - works with tf.keras models\n", - "\n", - "More information in our [blogpost](https://medium.com/nvidia-merlin/training-deep-learning-based-recommender-systems-9x-faster-with-tensorflow-cc5a2572ea49)." - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "# External dependencies\n", - "import os\n", - "import glob\n", - "import time\n", - "import nvtabular as nvt" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We define our base directory, containing the data." - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "BASE_DIR = os.environ.get(\"BASE_DIR\", \"/raid/data/criteo\")\n", - "input_path = os.environ.get(\"INPUT_DATA_DIR\", os.path.join(BASE_DIR, \"test_dask/output\"))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Defining Hyperparameters" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "First, we define the data schema and differentiate between single-hot and multi-hot categorical features. Note, that we do not have any numerical input features. " - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "(['/raid/data/criteo2/test_dask/output/train/0.5891ce6774804b929d0bcb05f5a2558b.parquet'],\n", - " ['/raid/data/criteo2/test_dask/output/valid/0.606080e99a63402f891540e7c01a2963.parquet'])" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "CONTINUOUS_COLUMNS = [\"I\" + str(x) for x in range(1, 14)]\n", - "CATEGORICAL_COLUMNS = [\"C\" + str(x) for x in range(1, 27)]\n", - "LABEL_COLUMNS = [\"label\"]\n", - "COLUMNS = CONTINUOUS_COLUMNS + CATEGORICAL_COLUMNS + LABEL_COLUMNS\n", - "BATCH_SIZE = int(os.environ.get(\"BATCH_SIZE\", 64 * 1024))\n", - "\n", - "# Output from ETL-with-NVTabular\n", - "TRAIN_PATHS = sorted(glob.glob(os.path.join(input_path, \"train\", \"*.parquet\")))\n", - "VALID_PATHS = sorted(glob.glob(os.path.join(input_path, \"valid\", \"*.parquet\")))\n", - "TRAIN_PATHS, VALID_PATHS" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "In the previous notebook, we used NVTabular for ETL and stored the workflow to disk. We can load the NVTabular workflow to extract important metadata for our training pipeline." - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [], - "source": [ - "proc = nvt.Workflow.load(os.path.join(input_path, \"workflow\"))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The embedding table shows the cardinality of each categorical variable along with its associated embedding size. Each entry is of the form `(cardinality, embedding_size)`. We limit the output cardinality to 16." - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'C1': (10000000, 16),\n", - " 'C10': (10000000, 16),\n", - " 'C11': (707291, 16),\n", - " 'C12': (218510, 16),\n", - " 'C13': (11, 16),\n", - " 'C14': (2209, 16),\n", - " 'C15': (9798, 16),\n", - " 'C16': (72, 16),\n", - " 'C17': (4, 16),\n", - " 'C18': (954, 16),\n", - " 'C19': (15, 16),\n", - " 'C2': (29612, 16),\n", - " 'C20': (10000000, 16),\n", - " 'C21': (4553157, 16),\n", - " 'C22': (10000000, 16),\n", - " 'C23': (291641, 16),\n", - " 'C24': (10904, 16),\n", - " 'C25': (91, 16),\n", - " 'C26': (35, 16),\n", - " 'C3': (15050, 16),\n", - " 'C4': (7190, 16),\n", - " 'C5': (19547, 16),\n", - " 'C6': (4, 16),\n", - " 'C7': (6492, 16),\n", - " 'C8': (1317, 16),\n", - " 'C9': (63, 16)}" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "EMBEDDING_TABLE_SHAPES = nvt.ops.get_embedding_sizes(proc)\n", - "for key in EMBEDDING_TABLE_SHAPES.keys():\n", - " EMBEDDING_TABLE_SHAPES[key] = (\n", - " EMBEDDING_TABLE_SHAPES[key][0],\n", - " min(16, EMBEDDING_TABLE_SHAPES[key][1]),\n", - " )\n", - "EMBEDDING_TABLE_SHAPES" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Initializing NVTabular Dataloader for Tensorflow" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We import TensorFlow and some NVTabular TF extensions, such as custom TensorFlow layers supporting multi-hot and the NVTabular TensorFlow data loader." - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [], - "source": [ - "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0\"\n", - "\n", - "import tensorflow as tf\n", - "\n", - "# we can control how much memory to give tensorflow with this environment variable\n", - "# IMPORTANT: make sure you do this before you initialize TF's runtime, otherwise\n", - "# TF will have claimed all free GPU memory\n", - "os.environ[\"TF_MEMORY_ALLOCATION\"] = \"0.5\" # fraction of free memory\n", - "from nvtabular.loader.tensorflow import KerasSequenceLoader, KerasSequenceValidater\n", - "from nvtabular.framework_utils.tensorflow import layers" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "First, we take a look on our data loader and how the data is represented as tensors. The NVTabular data loader are initialized as usually and we specify both single-hot and multi-hot categorical features as cat_names. The data loader will automatically recognize the single/multi-hot columns and represent them accordingly." - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [], - "source": [ - "train_dataset_tf = KerasSequenceLoader(\n", - " nvt.Dataset(TRAIN_PATHS, part_mem_fraction=0.04),\n", - " batch_size=BATCH_SIZE,\n", - " label_names=LABEL_COLUMNS,\n", - " cat_names=CATEGORICAL_COLUMNS,\n", - " cont_names=CONTINUOUS_COLUMNS,\n", - " engine=\"parquet\",\n", - " shuffle=True,\n", - " parts_per_chunk=1,\n", - ")\n", - "\n", - "valid_dataset_tf = KerasSequenceLoader(\n", - " nvt.Dataset(VALID_PATHS, part_mem_fraction=0.04),\n", - " batch_size=BATCH_SIZE,\n", - " label_names=LABEL_COLUMNS,\n", - " cat_names=CATEGORICAL_COLUMNS,\n", - " cont_names=CONTINUOUS_COLUMNS,\n", - " engine=\"parquet\",\n", - " shuffle=False,\n", - " parts_per_chunk=1,\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Defining Neural Network Architecture" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We will define a common neural network architecture for tabular data:\n", - "\n", - "* Single-hot categorical features are fed into an Embedding Layer\n", - "* Each value of a multi-hot categorical features is fed into an Embedding Layer and the multiple Embedding outputs are combined via averaging\n", - "* The output of the Embedding Layers are concatenated\n", - "* The concatenated layers are fed through multiple feed-forward layers (Dense Layers with ReLU activations)\n", - "* The final output is a single number with sigmoid activation function" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "First, we will define some dictionary/lists for our network architecture." - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [], - "source": [ - "inputs = {} # tf.keras.Input placeholders for each feature to be used\n", - "emb_layers = [] # output of all embedding layers, which will be concatenated\n", - "num_layers = [] # output of numerical layers" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We create `tf.keras.Input` tensors for all input features." - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [], - "source": [ - "for col in CATEGORICAL_COLUMNS:\n", - " inputs[col] = tf.keras.Input(name=col, dtype=tf.int32, shape=(1,))\n", - "\n", - "for col in CONTINUOUS_COLUMNS:\n", - " inputs[col] = tf.keras.Input(name=col, dtype=tf.float32, shape=(1,))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Next, we initialize Embedding Layers with `tf.feature_column.embedding_column`." - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [], - "source": [ - "for col in CATEGORICAL_COLUMNS:\n", - " emb_layers.append(\n", - " tf.feature_column.embedding_column(\n", - " tf.feature_column.categorical_column_with_identity(\n", - " col, EMBEDDING_TABLE_SHAPES[col][0]\n", - " ), # Input dimension (vocab size)\n", - " EMBEDDING_TABLE_SHAPES[col][1], # Embedding output dimension\n", - " )\n", - " )" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We define `tf.feature_columns` for the continuous input features." - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [], - "source": [ - "for col in CONTINUOUS_COLUMNS:\n", - " num_layers.append(tf.feature_column.numeric_column(col))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "NVTabular implemented a custom TensorFlow layer `layers.DenseFeatures`, which takes as an input the different `tf.Keras.Input` and pre-initialized `tf.feature_column` and automatically concatenate them into a flat tensor." - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 13, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "emb_layer = layers.DenseFeatures(emb_layers)\n", - "x_emb_output = emb_layer(inputs)\n", - "x_emb_output" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We add multiple Dense Layers. Finally, we initialize the `tf.keras.Model` and add the optimizer." - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [], - "source": [ - "x = tf.keras.layers.Dense(128, activation=\"relu\")(x_emb_output)\n", - "x = tf.keras.layers.Dense(128, activation=\"relu\")(x)\n", - "x = tf.keras.layers.Dense(128, activation=\"relu\")(x)\n", - "x = tf.keras.layers.Dense(1, activation=\"sigmoid\", name=\"output\")(x)\n", - "\n", - "model = tf.keras.Model(inputs=inputs, outputs=x)\n", - "model.compile(\"sgd\", \"binary_crossentropy\")" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAHXUAAAIjCAYAAABYLt2zAAAABmJLR0QA/wD/AP+gvaeTAAAgAElEQVR4nOzdeXxU5d3//3dCSAgQAUEISwhLCFkMqHGjQl2K0t5a7QLaPtS7rXW7sdXW/e6tfq36KLVSba0VF7BarXcB77uLtrWt1uotglJUlqyEJQkENBBAFslCrt8f+c0hk4TJzGTOzJzrvJ6PxzzEycnJueDzPudzzVyZk2KMMQIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAf1iemugjAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIB44qauAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHyFm7oCAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA8BVu6goAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADAV9K6PrFy5Uo9/PDDiTgWIKSbb75ZM2bMcGXf1D1s4WZO5s2b58p+gXiaMWOGbr75Zlf2/fDDD2vlypWu7BuIp+XLl7uyX/ot2IJ+CwiNfgvoHf0WEBr9FhAa/RbQO/otIDTed4cfUfdA75iPA6ExHwd6x3wcCI1+CwiNfgvoHf0WEBqvA8OPmGfAb5g3wI+YB8Bv6G/gN/Q38CP6G/gNr1vCVm72MRL9O7yF/gZgfgsE8Pom0MGt/kjiuoDE4jwPWzGvhY2Yp8JW9COwFf0IbEQ/AlvRj8BWPfUjqV2fqK+v10svvRSXAwLC9dJLL6m+vt61/VP3sIHbOXnppZe0bds21/YPuG3VqlWuTjJXrlypVatWubZ/wG3btm1ztR+i34IN6LeA0Oi3gNDot4De0W8BodFvAaHRbwG94313+BF1D/SO+TgQGvNxIDTm40Dv6LeA0Oi3gNDot4De8Tow/Ih5BvyGeQP8hnkA/Ij+Bn5DfwO/ob+BH/G6JWzldh8j0b/DG+hvgA7Mb4EOvL4JuN8fSVwXkDic52Ej5rWwFfNU2Ip+BDaiH4Gt6EdgK/oR2ChUP5J2rG9y6470QDRSUlLi8nOoe3hZPHLy/e9/X5deeqnrPwdww7x581z/GWeeeSbXEnjWsmXLdNlll7n+c8gIvIx+CwiNfgsIjX4L6B39FhAa/RYQGv0W0Dved4cfUfdA75iPA6ExHwdCYz4O9I5+CwiNfgsIjX4L6B2vA8OPmGfAb5g3wG+YB8CP6G/gN/Q38Bv6G/gRr1vCVvHoYyT6dyQ/+hugA/NboAOvbwLx64+4LiAROM/DRsxrYSvmqbAV/QhsRD8CW9GPwFb0I7BRqH4kNc7HAgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAJxU1dAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAPgKN3UFAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA4Cvc1BUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAr3BTVwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAC+wk1dAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAPgKN3UFAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA4Cvc1BUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAr3BTVwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAC+wk1dAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAPgKN3UFAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA4Cvc1BUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAr3BTVwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAC+EvObuq5evVrf/OY3NXHiRGVmZur444/XiSeeqK9+9atatGiRNm3a1OP3/fnPf1Z+fr7S0tJidiyDBw9WSkpK0GPhwoUx23882TQWG8Wj7j/88ENdeOGFGjp0qLKysjR79mytWLGiz8duU23ZNBbbxOvawLUkNJvGYiM3c7Jnzx498cQTOu+883T88ccrMzNTU6ZM0eWXX661a9f2+dhtqi2bxmIbNzNijNGKFSt0ww03KD8/XxkZGRo5cqRmzpypF154QcaYPh27TXVl01hsE++5+MUXX6yUlBQ98MADfT52m+rKprHYyO2czJw5s9u/f+Dxve99r0/HblNt2TQW28TjWtLa2qpHHnlEpaWlysrK0siRI/WFL3xBL7/8cp96Lpvqyqax2MbNjDzxxBPHvIYEHl/4wheiPnab6sqmsdjI7WtJW1ublixZotNPP13Dhw/XsGHDVFpaqscee0wtLS19OnabasumsdjG7YwcOXJEP/vZz3TSSSdp4MCBGjJkiM477zy99tprfT52m+rKprHYKJKc8N5H+Gwai43iUfesN+mdTWOxTbyuDaw3Cc2msdjIzZzQc4XPprHYxs2MsN4kfDaNxTbxnouz3qRnNo3FRm7nhPUm4bFpLLaJx7WE9Sa9s2ksNookJ8wzwmfTWGwT7XvjAcwbembTWGwUad0zDwiPTWOxTTTnevr63tk0FttEUvOsCw+fTWOxUaTnetZ5h8emsdgm0ppn3XZ4bBqLLcL9N2E9XWg2jcUWbtQ26697Z9NYbOHWeZ7rQmg2jcUWscwCa6nDZ9NYbBDLHLCOIXw2jcUmof5dOM+Hz6ax2CDUvwfn7fDZNBZbRPpvwjqzntk0Flv09m/CWrLw2DQWG4Tz78F6sd7ZNBZbhPo3YV1Y+Gwaiy16+zdh7Vd4bBqLDXr792B9V3hsGkvMmS6WLl1qeni6V0eOHDG33nqrSUtLM7fddpupqKgwhw8fNjt37jR/+9vfzOzZs40kI8m0trY631dTU2O++MUvmmnTppnjjjvO9OvXL+KfHcoHH3xgJJlLLrkkpvtNBJvGEilJZunSpa7tP9nrftWqVSYzM9NcdtllpqGhwTQ2NpprrrnGpKWlmb/+9a8RH3dXNtWWTWOJlNs5iWb/8coI15Lw2TSWSM2dO9fMnTs36fYfj5x8+9vfNmlpaeZnP/uZ2bFjhzl48KB56623TFFRkenXr5/53e9+F9WYO7OptmwaSySi7Yfc3n88MlJRUWEkmdmzZ5u1a9eaTz/91GzatMl8/etfN5LMLbfcEtWYO7OprmwaS6T83G919txzzzn7vP/++yM63mOxqa5sGkuk/NxvGWPMWWed5eyn6+Omm26K+Li7sqm2bBpLJPzcbxljzIEDB8zMmTPNtGnTzJtvvmkOHTpkamtrzdy5c40ks379+oiPvTOb6sqmsUTKr/3WokWLjnkNCTzuu+++qMdtjF11ZdNYIuX3fuuKK64wksx//ud/mo8++sjs2rXLPPjgg0aSueiiiyI+7q5sqi2bxhIJP/dbbW1t5qKLLjL9+/c3v/jFL8yuXbvM5s2bzbe+9S2TkpJi/vu//zuqMXdmU13ZNJZI2fS+O+99RMamsUTK73XPepPw2TSWSNkyH48mI6w3CZ9NY4mUTfPxSHNCzxUZm8YSCZvm45HWPOtNImPTWCLl536rM9abhGbTWCLl537LGNabRMKmsUTCz/2WMaw3iYRNY4mUTa8DM8+IjE1jiZQt84yumDeEZtNYImXTvMEY5gGRsGkskbBpHmAMfX0kbBpLpGzpb1gXHhmbxhIp2/ob1nmHz6axRMKm/oZ125GxaSyRStbXLUP9m7CeLnw2jSVSbvcxxkSXn1jWNuuvw2fTWCKVrP1NLLPAdSF8No0lUsk6v41VFlhLHRmbxhKpZHx9M1Y5YB1DZGwaS6Tc7o+Mif11gfN8ZGwaS6S8dJ7nvB0Zm8YSKS/OaztjnVloNo0lUl6cp7KWLHw2jSVSXupHjGG9WCRsGkukvNaPsC4sMjaNJVJe7EdY+xU+m8YSKS/1I6zvioxNY4lUiH5hWapi5O6779bChQv1+OOP6yc/+YkKCgqUkZGhUaNG6fzzz9err77a453h7777bn3mM5/RmjVrlJWVFavD8aTBgwdr5syZiT4MRCAedd/e3q5vf/vbGjp0qH71q19p9OjRGjFihBYtWqTJkyfr6quvVnNzs1tDTDrkxFvidW3gWnIUGfGeeOXkqquu0k033aTs7GwNHDhQs2bN0osvvqgjR47o9ttvd2NoSYuceEu8MpKWlqZly5Zp2rRpGjBggCZNmqRnn31Ww4cP12OPPUa/haQV77l4Q0ODvve97+nKK6+M5TA8hYx4Tzxzsnr1ahljuj1+9rOfxXpYSY2ceEu8MnLbbbdp3bp1+tvf/qbPfvazyszM1Pjx4/Xss88qIyPDjaElLTLiLfHKyCWXXNLjNaS6uloZGRm65ppr3BheUiIj3hOPnGzevFkvvPCCTj75ZP3oRz/SyJEjNXz4cN1+++06//zz9corr2j16tVuDTHpkBNviUdGXnjhBb3yyiu6/vrr9Z3vfEfDhw/XxIkTtWTJEk2dOlXz58/X3r173Rpi0iEj3hNtTnjv4yjq3nviUfesNwlGTrwlXtcG1pscRUa8J145oec6ipx4S7wywnqTo8iIt8R7Ls56EzLiRfHMCetNOpATb4lXRlhvchQZ8Z5oc8I84yjq3luirfkA5g3UvBf1pe6ZB3Sg7r0l2pqnrz+KmveWaGuedeFHUfPeE03ds847GHXvLdHUPOu2g1HzdmE93VHUtl34vL/okQW78Ll+0SMLduGz+6JHFuzB5/NFjxzYh/P8UdS3PThvH0Vd24l1ZtS2jVhL1oHatgfrxY6iru3CurCjqG17sPYrGLVtB9Z3BaOuoxOTm7pWVlbqxz/+sUpLS4/ZKPTr10933313t+eXLFmiO++8U2lpabE4FCBu4lX3b731lsrKyjR37lxlZmYG7fvrX/+66uvr9corr0Q/EMAl8bw2cC2BV8UrJ4sXL9aTTz7Z7fnp06crMzNTmzZtkjEm8gEALotXRgoKCtTa2qphw4YFPZ+enq6cnBw1Nzfr8OHD0Q0CcFEi5uLXXHON5s2bpwsuuCCqYwbijdesgNDilZGPPvpITz31lC6//HKNGjUq6GuDBg3S4cOHdeKJJ0Y3CMBF8cpIXl6eZs2a1ePXfvGLX+hLX/qSsrOzIzt4IE7ilZP6+npJUmFhYbevFRQUSJLq6uoiOXQgLuKVkd/97neSpC9+8YtBz6ekpOiSSy7Rnj179NJLL0UxAsB90eaE9z7gZfGqe9abwKvieW3gdWB4VbxyQs8Fr4pXRlhvAq9KxFyc9SbwGl6zAkKLV0ZYbwIvizYnzDPgVX15bzyAeQO8JhZ1D3hJtDVPXw+virbmWRcOL4u27lnnDa+KtuZZtw2bsZ4OtuLz/oAOfK4f0IHP7gP4fD4ggPM8bMR5G37AOjMASG6sF4OtWBcGW7H2CzZifRdiISY3dX3qqafU3t6uefPmhdxuxowZMsYEvXHTedEK4CXxqvt//OMfkqRTTz2129cCz73++uth7w+Il3heG7iWwKsS3UMdPHhQn376qU488USlpKT0eX9ArCU6I3v37tXGjRt18skna8iQIX3eHxBr8c7IM888o7KyMi1cuDDi7wUSJdHXEiDZxSsjf/zjH3XkyBHNnDkz6mMFEiFeGZk9e7ZuueWWbs/v379fzz33nObPnx/+QQNxFq+cFBQUqH///qqsrOz2tcrKSqWkpKikpCT8AwfiJF4Z+eijjyRJI0eO7Pa10aNHS5LefvvtsPcHxFNfctIT3vuAF8Sr7llvAq+K57WB14HhVYnuoei5kOwSnRHWmyDZxTsjrDeBFyX6WgIku3hlhPUm8LJY54R5BpJdX2ueeQO8KNbneiDZRVvz9PXwqmhrnnXh8LJo65513vCqaGuedduwGevpYCs+7w/owOf6AR347D6Az+cDesN5HjbivA1bsM4MAJIf68VgK9aFwVas/YKNWN+FWIjJTV3feustSdK0adNisTvX/f73v1dKSorz2Lp1qy677DINHTpUw4cP10UXXaRNmzY52y9cuNDZdty4cVq9erU+97nPKSsrSwMHDtS5556rFStWONs/8MADzvadJwyvvvqq8/yIESO67f/gwYNasWKFs01ffkGrra1NS5cu1fnnn6/s7GxlZmaqpKREP//5z9Xe3i6p48XUzn8PKSkpeuCBB5zv7/z83LlznX03Njbqxhtv1IQJE5Senq4TTjhBX/nKV/Thhx8e8++4qqpKl156qYYPH+48t2vXrqjHlwziVfeB5mXcuHHdvjZ27FhJUnV1dcx/LjkhJ33ltWtDpMgIGYmFROdk+fLlkqT/+q//cmX/5ISc9FWiMvLJJ59oxYoVuvjii5Wdna1f//rXrvwcMkJG+iqeGdm2bZtuueUWPfPMM8rKynL950lkhIzERryvJc8//7xOOukkDRo0SEOGDNGsWbP04osvuvbzyAk56at4ZeT999+XJA0bNky33HKLcnJylJ6ertzcXN14441qampy5eeSETLSV4met//qV7/S+PHj9dnPftaV/ZMRMhIL8crJqFGjtHDhQq1du1Y/+MEP1NjYqKamJv3kJz/Ra6+9pnvuuUf5+fkx/7nkhJz0VbwyEqiTwCKSzhobGyVJW7dujfnPJSNkJBZinRPe+6DuvSBedc96k+D9kxPv8Nq1IVJkhIzEQqJzQs9FTpJdojLCehMy4hXxzAjrTciIV8X7WsJ6E3LiNfHKCOtNgvdPRrwlVjlhnkHde0Vfap55AzXvVX091zMPoO69Jtqap68P3j817x2xnvuyLpya94Jo65513sH7p+69I9qaZ9128P6pefQVtU1tJxvWXwfvnywg3sgCWfAK1lKTBT9jHQM58APO89S3TThvU9c2YZ0ZtW0z1pJR2zZhvVjw/qlr+7EujNr2OtZ+Be+f2rYD67uC909dR8l0sXTpUtPD0yGNHj3aSDLvvvtuRN/X1dixY02/fv1CbnPuueea448/3qxcuTKsfX7wwQdGkrnkkku6fe2SSy5xvvbOO++YAwcOmL///e8mMzPTnHbaad22nz59uhk0aJCZMWOGs/3q1avNtGnTTHp6uvnnP/8ZtP2gQYPMWWed1W0/paWlZvjw4d2eP9b24Yylq5dfftlIMj/60Y9MU1OTaWxsNI8++qhJTU01t956a9C2c+bMMampqaampqbbfmbMmGF+85vfOP/f0NBgcnNzzahRo8yf/vQns3//frNhwwZz9tlnmwEDBph33nkn6PsDf8dnn322eeONN8zBgwfNqlWrTL9+/UxjY2Ov4wiQZJYuXRr29pFK5ro///zzjSSzatWqbl/buHGjkWROOeWUoOfJCTlxQ6T7j+e1IdLtyYg/MzJ37lwzd+7csLePVDT7T1ROjDFm586dZtSoUebqq6/u8evkxH85iaYfikQy91ud3X///UaSkWTOOeccs27duh63IyP+y4gx/u635syZY+bPn+/8//PPP28kmfvvv7/H7cmIPzPi937rrLPOMldeeaVZs2aNOXDggKmsrDRXXnmlkWS++93vdtuenPgvJ37utwJ/f9nZ2ebyyy83mzZtMnv27DHPPfecGTRokMnPzzd79+4N+h4y4r+MGOPvfqur9vZ2k5+fbx5//PEev05G/JkRv/dbxhizbNkyM27cOGfuPmLECLNkyZIetyUn/suJn/utX/ziF8ece5SWlhpJ5tRTTw16noz4LyPG2P2+uzG890Hd98zPdc96ExPW9uGMpStyEplEzceN6f3a0BnrTcjIsdg8HzcmspyEsz058V9ObJ6PGxN+RlhvQkZC8XO/xXqT3rcPZyxd2ZYRv/dbrDfpfftwxtKVTTnxc7/FehMT1vbhjKUrmzJijL2vAzPPoO5DsWmewbyh9+3DGUtXttW8bfMG5gG9bx/OWLqyqe5tmgfQ15uwtg9nLF3ZVPPG2NXfdMW6cGq+J7b1N8awzru37cMZS1c21b1N/Q3rtk1Y24czlq5sqnljkvN1S2PC/zdhPR21fSxu9zHGRJefWNU2669NWNuHM5aubMtCMvY3xsT2PB/p9mTBn1lIxvmtMe5lwRjWUpOFniXb65vGuJMD1jGQg1Dc7o+Mcf+6YAzneeq7Z149z3Pepq5D8eK8lnVmvW8fzli6sq22vThPZS1Z79uHM5aubKttL/UjrBczYW0fzli6sq2uvdiPdMW6MGq7J17sR4xh7Vdv24czlq5sq20v9SOs7zJhbR/OWLqyra5D9AvLYnpT1/feey+i7+sqnDdzzj77bDNs2LBuf1nHEk4Rv/zyy0HPz50710jq9pc8ffp0I8l88MEHQc+vW7fOSDLTp08Pej7RRXzOOed0e/6KK64w/fv3N/v27XOe++tf/2okBb0gZYwxb7/9thk7dqxpaWlxnvvGN75hJAUVtjHG7Nixw2RkZJjS0tKg5wN/x3/+8597PeZQknGRYrzqPtQir+rqaiOp2987OSEnboh0//G8NkS6PRnxZ0aScRKbqJzs2rXLnHTSSeayyy4zbW1tPW5DTvyXk2R8ETNRGWlubjYVFRXm+uuvN/369TP33Xdft23IiP8yYox/+62nnnrKTJo0yRw4cMB5rrdFDGTEnxmh3+rZ6aef3uO8npz4Lyd+7rfmzJljJJmJEyea1tbWoK898MADRpK5++67g54nI/7LiDH+7bd68qc//clkZWWZ/fv39/h1MuLPjPi532pvbzfXXHON6d+/v3n44YfNzp07TWNjo3nyySdNZmamueyyy7pdY8iJ/3Li537r008/NaWlpaZ///7mscceM7t27TK1tbXmhhtuMNnZ2UaSmTVrVtD3kBH/ZcQYu993570P6v5Y/Fz3rDcxYW0fzli6Iifu7j+e14bOWG9CRo7F5vl4pDmh5yInPbF5Ph5pRlhvQkaOxa/9FutNTFjbhzOWrmzLCP1Wz1hvEtlYurIpJ37ut1hvYsLaPpyxdGVTRoyx+3Vg5hnU/bHYMs9g3mDC2j6csXRlW83bPG/ojHlAZGPpyqa6t2keQF9vwto+nLF0ZVPNG2NPf9MT1oVT8z2xqb9hnbcJa/twxtKVTXVvU3/Dum0T1vbhjKUrm2remOR83dKY2N7Yidr2Z237+aaurL+ObCxd2ZaFZOxvjEnsTV3Jgj+zkIzzW2PcywJrqcnCsSTb65vGuJcD1jGQg2Ox4aaunOep72Px8nme8zZ1fSxem9eyzsyEtX04Y+nKttr2+jy1M9aSRTaWrmyrbS/1I6wXM2FtH85YurKtrr3Wj/SEdWHUdk+81o+w9suEtX04Y+nKttr2Uj/C+i4T1vbhjKUr2+o61E1dUxUDY8aMkSTt2rUrFrsL6Z///Keampo0Y8aMmO3ztNNOC/r/nJwcSVJDQ0O3bQcNGqSTTjop6LmSkhKNGTNGa9eu1Y4dO2J2XH1x0UUX6Y033uj2/PTp09Xa2qqysjLnuQsuuEAlJSV69tlntXv3buf5hx56SN/97nfVv39/57nf//73Sk1N1UUXXRS03+zsbBUXF2vNmjXatm1bt597+umnx2JYSSVedT906FBJ0sGDB7t9LfBcYJsAchIecuKueF4bIkVGwkNG3JeInBw8eFBz5sxRUVGRfvOb36hfv349bkdOwkNO3JWoa0l6eroKCgq0aNEiXXzxxbrnnnv02muvBW1DRsJDRtwVj4zU1dXptttu0zPPPKNBgwaF/X1kJDxkxH3JMC+ZO3euJOnll18Oep6chIecuCteGQlcQ2bPnq20tLSgr33xi1+UJP31r38Nep6MhIeMuCuR15FHH31U//7v/67Bgwf3+HUyEh4y4r545eT555/X008/reuvv17f//73NWrUKI0YMULXXnut7rzzTi1dulSPPfZY0PeQk/CQE3fFKyMDBgzQG2+8oZtuukkLFy7U6NGjdcYZZ8gYo+XLl0vq+LvvjIyEh4y4LxY54b2P2KLu3Revume9iXvIibvieW2IFBkJDxlxXyJyQs8VW+TEXYm6lrDeJHbIiLvikRHWm7iLjLgvGeYlrDfpG3LirnhlhPUm7iEj7ovV+4TMM2KHundXNDXPvMFd1Lz73FgTwjygb6h7d0Vb8/T17qHm3RXL8zzrwmODmndftHXPOm/3UPfuirbmWbftHmree6jt8FDb3sL6a/eQBfuRhfCQBe9iLXVskQXvYh1D7JCD5MJ5Prao7+TBeTt2qOvEYZ2Zu6jt5MVasr6hthOH9WLuoa6TD+vCYoPaTizWfrmH2k4c1ne5x091HZObup599tmSpHXr1sVid3E3ZMiQoP9PT0+XJLW3t3fbtutimoCRI0dKkj7++OMYH1109u3bp3vuuUclJSUaNmyYUlJSlJKSottuu02SdOjQoaDtv/e97+nQoUN6/PHHJUnV1dX6xz/+oWuvvdbZprm5Wfv27VN7e7uGDBni7DPweP/99yVJGzdu7HY8kbzY5RXxqvuCggJJ6vHksH37dklSfn6+q8cgkROJnETK69eGSJERMhKNeOekra1N8+bN09ixY/Xcc8/F7INrw0VOyEmkkuFaEnix/5VXXnH9Z5ERMhKpeGTk5Zdf1r59+3TOOecE/T1feeWVkqS7777bea6mpsa145DIiERGopEM15LRo0dLik/dkRNyEql4ZWTChAmSpOHDh3f7WiU9YUIAACAASURBVKDmGhsbXT0GiYxIZCRSibqOVFdX629/+5vmz58f159LRshINOKVk1dffVVSx0K9rj73uc9Jkv7yl7+4egwSOZHISaTieS3JysrSQw89pC1btqilpUU7duzQL3/5S+dDI0455RTXj4GMkJFo9DUnvPcRe9S9++JV96w3cQ85cZfXrw2RIiNkJBrxzkmic0VOyEmkkuFawnqTviEj7opHRlhv4i4y4r5kuJaw3qRvyIm74pUR1pu4h4y4z433CZln9A11765oap55g7uoefe5ca5nHtA31L27oq15+nr3UPPuitV5nnXhsUPNuy/aumedt3uoe3f15VzPum13UPOQqG2J2k401l+7hywgGmSBLCQL1lLHHlmwA+sY+oYcJA/O87FHfScnztt9Q10nDuvM3EVtJy/WkvUNtZ04rBdzD3WdXFgXFjvUdmKx9ss91HZisb7LHX6q65jc1PW6665TWlqaXnrppZDb3X777UpNTVVlZWUsfmxC7N69W8aYbs8HijdQzJKUmpqqlpaWbtvu3bu3x32npKTE6Cg7XiS9//77dc0116i6ulrt7e0yxuiRRx6RpG5juPzyyzVq1Cg99thjam5u1k9/+lN94xvf0LBhw5xtMjIyNHToUKWlpam1tVXGmB4f5557bszGkcziVfeBv881a9Z0+1rguUAzkyzICTmR/HVtiBQZISMB8c7Jddddp+bmZi1btkxpaWnO83l5eVq1alWf9h1r5IScSMlxLcnIyJAkNTU1xXzffUFGyIgUn4zccMMNPf79Pv/885Kk+++/33kuLy8vqnG4gYyQkYBkuJY0NDRICq67ZEBOyIkUv4zMnDlTkrRjx45uXwvU3KhRo6Lat1vICBmREncdefTRR/XZz35WRUVFMdmfG8gIGQmIV04Cb36HcuDAgaj27RZyQk6k5JiTvP3225Kkr3zlKzHfd1+QETIS0Nec8N5HB+reW+JV96w3CUZOvMNP14ZIkREyEhDvnHgpV+SEnEjJcS1hvUnfkBF3xSMjrDfpjox4SzJcS1hv0jfkxF3xygjrTYKREW9x431C5hl9Q927K5qaZ97QHTXvLW6c65kH9A11765oa56+Phg17x2xOs+zLpya95Jo65513sGoe+9wo6dn3XbfUPOIFLVNbbuB9dfByAK8hCyQBTexlroDWUBXrGPoG3KQPDjPd6C+7cd5u2+o68RhnVl31LY/sJasb6jtxGG9WDDq2l6sC6O2bcHar2DUtv1Y39U3fqrrmNzUNT8/X//v//0//etf/9IzzzzT4zZVVVV68skndemll6qgoCAWPzYhDh8+rNWrVwc9t379ejU0NGj69OkaPXq08/zo0aO1ffv2oG137typurq6Hvc9cODAoKKfOnWqnnrqqYiOLy0tTWVlZVqxYoWys7N144036oQTTnAC8umnn/b4fRkZGZo/f74+/vhj/fSnP9VvfvMb3XTTTd22+8pXvqK2tjatWLGi29cefPBBjR8/Xm1tbREds1fFq+7PPvtsFRUV6aWXXtLhw4ed548cOaLf/va3ysnJ0YUXXhjVvt1CTsiJ5K9rQ6TICBkJiGdO7r33XpWVlekPf/iD82ZqMiMn5ESKX0ZuvfVWXXHFFT1+7S9/+Ysk6bTTTotq324hI2REot8KhYyQkYB45WTx4sUqLS3t9rwxRsuWLZPU8YJjMiEn5ESKX0b+7d/+TWPHjtWrr74a9PqWJL388suSpC996UtR7dstZISMSInptz755BP9+te/1g033NDnfbmJjJCRgHjl5IwzzpAkvf76692+9o9//EOSdOaZZ0a1b7eQE3IixS8ju3btUmpqqrMgO+CTTz7R4sWL9bWvfU35+flR7dstZISMBPQlJ7z3cRR17y3xqnvWmwQjJ97hp2tDpMgIGQmIZ068lityQk6k+GWE9SbByIh30G8dGxkhIwHxygnrTYKRE++IV0ZYbxKMjHhLtDlhnhGMuvcOP61rp+ap+YBo6555QDDq3juirXn6+mDUvHfEor9hXTg17zXR1j3rvINR994Rbc2zbjsYNY9EorapbTew/joYWYCXkAWy4BavrZkjC2Qh1ljHEIwc2Ifz/FHUtx04bwejrpFI1Da1HQ7WkgWjtu3AerFg1LWdWBdGbduEtV/BqG07sL4rGHUdnZjc1FWS7rrrLt155526/vrrdeedd6q6ulotLS3avn27lixZonPPPVfTpk3TkiVL+vRzzjvvPA0fPlyrVq2K0ZFHZsiQIfrBD36glStX6uDBg/rXv/6lK664Qunp6fr5z38etO0FF1yghoYGPfbYYzpw4IA2bdqkm266KejuxZ2dcsopqq6uVn19vVauXKnNmzdr1qxZER9jv379dM4552jnzp166KGHtGvXLn366ad644039MQTTxzz++bPn6/MzEzdddddmj17tvLy8rpts2DBAk2ePFlXXXWV/vKXv2jfvn1qamrSk08+qfvuu08LFy5UWlpaxMfsVfGo+9TUVC1ZskRNTU361re+pZ07d2r37t264YYbtHHjRj399NMaMGBA0PeQk96Rk/iI17UhUmSkd2QkfuKRk2effVY//OEP9e677yorK0spKSlBj02bNnX7HnLSO3ISH/G6lrz44ou67777tHXrVjU3N2vr1q2644479MILL6i0tFRXX3110PZkpHdkJD7ot3pGRshIZ/HKyfvvv68bbrhBNTU1Onz4sKqqqnTllVdqzZo1+u53v+u8SRFATnpHTuIjHhnJyMjQ4sWLtXv3bn3ta1/Txo0btXfvXj3//PNasGCBzjjjDN14441B30NGekdG4iPe/dYzzzyjwYMH68tf/nLI7chI78hI/MQjJ/Pnz9eUKVO0aNEiPfroo/r444+1e/duLVmyRD/+8Y81duxY3XrrrUHfQ056R07iI17XEmOMvvWtb6mmpkbNzc1677339PnPf16jRo3SL3/5y27bk5HekZH4iSYnvPcRjLr3nnjUPetNgpETb4nXtSFSZKR3ZCR+4pETeq5g5MRb4nUtYb3JUWTEW+i3ekZGyEhn8coJ602OIifeEo+MsN4kGBnxnmjfJ2SecRR17y38jjk177eal6Kve+YBR1H33hJNzdPXB6PmvaWv/Q3rwql5L4qm7lnnHYy695Zoz/Ws2z6Kmvc3art31Lb3sP46GFlAJMhC78iC97CWOhhZ8C/WMRxFDuzCeT4Y9W0PzttHUdf+Rm33jtpODqwlO4ratgPrxYJR13ZiXRi1bRPWfgWjtu3B+q6jqOsomS6WLl1qeng6bO+995658sorTU5Ojunfv7/JysoyZ555pvn5z39umpubu23/8ssvG0k9Pp5++ulu28+aNcsMGzbMvPPOO70ey6BBg7rt86GHHjIrV67s9vx//dd/GWNMt+cvvPBCZ3/Tp083Y8eONeXl5WbOnDkmKyvLZGZmmrPPPtu8/fbb3X7+3r17zdVXX21Gjx5tMjMzzcyZM83q1atNaWmps/877rjD2b6ystLMmjXLDBo0yOTk5Jhf/vKXIcdyrEdFRYVpbGw01113nfPvMGrUKPPNb37T3Hnnnc52paWl3Y75mmuuMZLMm2++ecy/1927d5ubb77ZTJo0yfTv39+ccMIJ5oILLjB///vfnW16+jvuS11JMkuXLo36+3uT7HVvjDHvv/+++cIXvmCOO+44M3jwYHPeeef1WHfGkBNy4o6+7N/tjHAtISPhmDt3rpk7d27U3+/2/t3MyYUXXtjrv/nKlSuDvoec+C8nfe2H3N6/mxnZt2+fWbx4sZkzZ46ZMGGCSU9PN4MHDzalpaVmwYIF5tChQ932T0b8lxFj/N1vBVx33XU9fs+cOXOCtiMj/syIn/utw4cPm+XLl5svf/nLZvLkySYjI8MMGTLEnHPOOebFF1/s8XjIif9y4ud+K+Cdd94xc+bMMUOGDDHp6emmoKDA3HvvvfRbZMTh936rvb3d5OXlmXvuuafX4yEj/syIn/stY4xpamoyt912mykoKDAZGRkmPT3dTJ482XznO98xO3fu7LY9OfFfTvzeb/397383F198scnOzjaZmZnmxBNPNPfff3+PvZYxZMSPGTHGrvfdee+Dug+X3+veGNabBJCTY3M7J33Zv9sZYb0JGQmHTfPxSHNCz0VOwmHTfDzSmme9CRkJl+TffiuA9SZkJBQ/91usNyEn4fBzvxXAepMOZOTY+tIPhSOeOWGeQd2Hy+2678v+I31vPIB5AzUfik3zBuYB1H04bJoHBNDXd6Dmj822/oZ14R2o+WOzqb8xhnXe1H3vbOtvWLdNzYfD7f4mmro/Vn0FsJ6O2g6H232MMZHnJ9a1bQzrrwPIwrElY38T6yxwXSAL4UjG+W0ss8BaarIQLim5Xt+MZQ5Yx0AOwuV2f2RM7K8LnOep73B56TzPeZu6DpcX57UBrDOjtkPx2jyVtWTUdri81I8EsF6sA3V9bF7tR1gX1oHaPjav9SPGsPaL2g6P1/oR1ndR1+EI0S8si/lNXW0WKGKbPfPMMz0Wd6K5fXKm7mOHnCROsjUx6BkZSZxknMSiZ+QkMZLxRUz0jIwkDv2WN5CRxKHf8g5ykhj0W95BRhKHfssbyEji0G95BzlJDPot7yAjicP77olD3ScOde8d5CRxmI97AxlJHObj3kFOEoP5uHeQkcSh3/IGMpI49FveQU4Sg37LO8hI4vA6cOJQ94nDPCMxqPnEYd6QONR9YjAPSBxqPnHobxKDmk8c+pvEoe4Tg/4mcaj5xOF1S3dR24mTjDd19TOykDj0N8mFLCQO89vkQhYSh9c3kwc5SJxkvamrTajvxOE87x7qOnGY17qL2k4c5qnuorYTh37EPdR14tCPuIvaThz6EXdR24lDP+Ie6jpxQt3UNVVAJ0888YRuvvnmRB8GkNTICRAaGQF6R06A0MgIEBoZAXpHToDQyAgQGhkBekdOgNDICPyIugd6R06A0MgI0DtyAoRGRoDQyAjQO3IChEZG4EfUPfyGmocfUffwG2oefkPNw4+oe/gNNQ9bUdtAB7IAdCALQAeyAJAD2I36ho2oa9iK2oatqG3YiLqGraht2Iraho28WNfc1NXnFi9erC9/+cs6cOCAnnjiCe3Zs0eXXnppog8LSCrkBAiNjAC9IydAaGQECI2MAL0jJ0BoZAQIjYwAvSMnQGhkBH5E3QO9IydAaGQE6B05AUIjI0BoZAToHTkBQiMj8CPqHn5DzcOPqHv4DTUPv6Hm4UfUPfyGmoetqG2gA1kAOpAFoANZAMgB7EZ9w0bUNWxFbcNW1DZsRF3DVtQ2bEVtw0Y21DU3dQ3DwoULlZKSorVr12r79u1KSUnRXXfdlejDipnf//73GjZsmBYtWqTf/va3SktLS/QhwYPICRAaGQF6R06A0MgIEBoZAXpHToDQyAgQGhkBekdOgNDICPyIugd6R06A0MgI0DtyAoRGRoDQyAjQO3IChEZG4EfUPfyGmocfUffwG2oefkPNw4+oe/gNNQ9bUdtAB7IAdCALQAeyAJAD2I36ho2oa9iK2oatqG3YiLqGraht2Iraho2o6+SWYowxnZ9YtmyZLrvsMnV5GkiolJQULV261LW7JlP3sIHbOXF7/4Db5s2bJ0lavny5J/cPuM3tfoh+Czag3wJCo98CQqPfAnpHvwWERr8FhEa/BfSO993hR9Q90Dvm40BozMeB0JiPA72j3wJCo98CQqPfAnrH68DwI+YZ8BvmDfAb5gHwI/ob+A39DfyG/gZ+xOuWsFU8+gz6d3gB/Q3Qgfkt0IHXN4H49C9cF5AonOdhI+a1sBXzVNiKfgQ2oh+BrehHYCv6EdgoRL+wPDURBwQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAicJNXQEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAD4Cjd1BQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAOAr3NQVAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgK9wU1cAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAvsJNXQEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAD4Cjd1BQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAOAr3NQVAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgK9wU1cAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAvsJNXQEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAD4Cjd1BQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAOAr3NQVAAAAAAAAAAAAAAAAAAAAAAAAQNw1Nzcn+hCApNfe3p7oQwAAALBOU1OT/vCHP2jXrl2JPhQgbmpra7V27dpEHwYQV3v37k30IQBxx2tJAGC/gwcPJvoQAAAAAAAAAAAAAAAAAMA6aYk+ACBcDz74oN59911NnTpV+fn5mjp1qkaPHp3owwKSytKlS3Xo0CHl5+drypQpOuGEExJ9SEBSKSsr02OPPaYpU6YoLy9Pubm5SkujHQI6W7BggfLy8jRlyhRNmTJFgwYNSvQhAUnlxRdf1P79+zVlyhTl5+crOzs70YcEJJUNGzbokUceceYkEydOVP/+/RN9WEBSuffee52M5Ofna8iQIYk+JCCpPPfcc9q9e7eTkZycHKWkpCT6sICksXbtWi1YsED5+fnO9WTAgAGJPiwgqdx+++1ORgoKCjRy5MhEHxKQVO644w797//+r5ORwPvvWVlZiT40wDW33HKLli1bpsLCQhUXF6ugoECFhYXKyMhI9KEBSeORRx7R+vXrVVRUpKlTp6qgoEADBw5M9GEBSWPlypW69tprVVBQ4FxLcnNzE31YQFKZM2eOTjzxRBUVFTn/ZZ4BHHX99dfr6aefVlFRkUpLS1VcXKzi4mJe3wX+f6+//rqGDBmikpISlZaWOo/CwkKlpqYm+vCApDBkyBCdeeaZOuusszRz5kzNnDmT6wgAAECEmpqa9H//939644039Oabb2rdunWSpLy8PI0YMSLBRwfEXmNjo1avXh30+Pjjj/mdWlitoaFB5eXlKisr05o1a7RmzRpVVlYm+rAA17S1tamuri6o5svLy3X48GF95jOfSfThAa7Yu3evNmzY4Jzvy8vL9a9//SvRhwW4pqWlRRs3bgzqccrLy7Vly5ZEHxoQE9u3b1dFRYXzoH+HHx05ckRbtmxRZWWlKioqVFVVpU2bNiX6sIC4a25uVlVVlfMoLy9XbW0ta7bhO3v37lVVVZUqKyudPAB+09raqvr6eue1n8B/GxoadNZZZyX68IA+aWlpUWVlpSorK1VeXq7y8vJEHxIQE3v37lVFRYXKy8tVUVGh119/PdGHBMREfX29c86uqKjQqlWrdOaZZyb6sIA+aW1tVU1NjfOafFlZWaIPCYiJ/fv3O6+z04/AJjt27HDO2eXl5Xrvvfd0+umnJ/qwgD5pa2vTli1bWCeApHbM37iYN29ePI8D6NXAgQP11ltvafHixfrkk08kSccdd5zzIbOdP3A2Pz9fmZmZEf8M6h5e99577+lPf/qTmpubJXV8cEfg5pWBm/MFHsOHD494/4888oiWL18e68MG4mLlypU6fPiw7r33Xu3evVuSlJ6erokTJwZlI5CX8ePHR/xBUKtWreJaAs/atm2bJGnx4sWqra3VkSNHJEljxozpMSPR3jSGjMDr1q5dq9dee00HDx6UJGVlZXXrs/Lz86P+UA/6LXjZqlWr1NzcrIceekg7duyQJKWlpWnChAlONgI3HqPfgh8F+q2lS5dq8+bNamlpkSSNHDmyWz4CmaHfgh9VV1frrrvuUlNTkyQpMzPTyUTn60l+fj79Fnxn1apVamtr09NPP63a2lq1t7crNTVV48ePd/LR+X2S3Nxc+i34SqDfev3117Vo0SIdOHBAkjR06FAnH50zwvuJ8Kvp06erpaVFL730kjZt2uTMTcaOHetkJHAzv8D1pF+/fhH9DHKCZBNYiPo///M/evDBB9Xa2qp+/fpp4sSJKi4uVmFhoYqKilRUVKSCggINGjQo4p9B3cPr2tvbgzKSkpKi3NxcTZ06VYWFhSooKFBBQYGKiop0wgknRLx/5uPwslWrVmnAgAHatGmT/vjHP+qjjz6SJA0ePNjJRVFRkXM9mThxYsT9E/NxeFlgPp6dna0333xTTzzxhA4dOiRJys3NDbrJ64knnqjCwkL6LfjS5z//eRljnNeuWltblZ6eruLiYpWUlKikpETTpk1TSUmJRo8eHfH+6bfgZatWrdIZZ5yhr371q3r//ff11ltvOTnJysrSSSedpJNPPlmnnHKKTjnlFBUWFkZ88xn6LXhZoN9asGCB3n77bT399NP64Q9/qAEDBuj000/XrFmzNHPmTH3mM5/RcccdF/XPISM4li1btignJ4cbf4mcwH+SaZ7R2tqqhoYGPrTbx6L9MLz9+/fr3Xff1WuvvabXXntNH3zwgdrb2zVp0iTNnj1bP/jBD3Teeefp+uuvZ96ApBKYB0TiwIED+vDDD50b+q1Zs0YVFRUyxmj06NEqLS3Vf/zHf6i0tFS7du3SVVddRc3D09ra2lRVVaUPP/zQeXzwwQfavXu3UlJSNGnSJJ100kn6+te/rpNPPlkXXXRRUvU3QDT9zY4dO7R+/XqtXbtW69ev17p161RRUaGWlhb1799fhYWFKikp0bXXXqtXX32V/gZJJdr+pqKiQuvWrVNZWZnWr1+vDRs2aOfOnZKkYcOGqaSkRMXFxcrNzdWSJUuoeXjakSNHVFNT49T6hg0btG7dOm3evFlHjhzRgAEDVFRUpOLiYl133XXau3evFixYQN0j7qLpYwI3rOz8gaxlZWWqqqrSvn37JEnDhw931uG98cYb9O9IetH0N4cOHXJu0hfIQmVlpaqrq53PDBw3bpwKCgqc9345zyPZRXNdaGxsdG5gHLh5a1VVlfOZZ4Hf9ykoKFBqairzW1jpyJEjqq2tVVVVlZOH6upqVVRUOL+vkJGR4fxetJRc798CPYm2PwrME8rKypybAQbmwmlpacrLy1NRUZE+//nP64033uC6AM/oXN+BGwGWlZVpy5YtamtrU1pamiZNmqTi4mJJnOcRf9GctyXp448/VllZWdBNLsvLy53PnRw0aJAKCgqUlZUliXkt4q8vr18Gajrw+mVlZaX2798vSTrhhBNUXFysjIwM+hF4xqFDh5zXIAOvyZeXl6umpkatra1KTU3VhAkTVFBQIIl+BPEXbT/S2NjonKc7v/dUX18v6ehrKoHfZeecjXiLth/ZunVrUE2Xl5ersrJSe/fuldSxTqawsFD9+vWjH4FndH5/NFDTgfdHA599OH78ePoRJEyofiTFGGM6P7Fy5Uo9/PDDrh8UEKmbb75ZM2bMkNSx2DzwRnx1dbVz0t26dava2tqUkpKi7OxsjR8/PuiRm5vr/LnzDS2pe9ji5ptv1hlnnKH6+nrV1NRo48aNzn83btyozZs3O4u3jjvuOOXk5Gj8+PEaN26cxo0bp9zcXOfP48ePD/owcxpz2GDGjBm6+eab1dTUFJSNzlnZs2ePpI4bvnbNRk5OTlBujj/+eGffDz/8sFauXJmooQExs3z5crW0tGjLli2qrq7ulpP6+nq1t7c7/VZubm5QjzVhwgTn/4cMGeLsl34LtgjMSxoaGpyMBK4h1dXVqqmpcfqtIUOGBOUikI3An7Ozs4P2Tb8FGwT6rf379wddQwIZ2bhxo3bv3i2po9/qnInc3FxNmDDByczYsWODPgiOfgu2WL58ufOGWSAbgXxUV1errq4uqN+aOHFiUEY6/7nzvJ1+C7YI9Fu7du1y8tE5Ixs3btSnn34qqaPfClw7enoMHTo0aN/0W7BBoN9qbm7udh0JvGfS2NgoqaPf6txjTZw4MSgjXW8MQL8FWwQWY2zfvt3JRdf3E48cOaKUlBSNHj1aEydO7PaYMGGCcnJygm7ERL8FW3R+372trU1btmxx3nvv/Mu/H3/8saSj15PO15TO15bRo0crJSVFEjlB8upc962traqurg76JcjAL7+3tLQoJSVFY8aM0eTJkzV58mRNmjQp6L8jRowI2jd1D1sEctLa2qrNmzc7H4wSWPBdVVWlTz75RJI0dOhQ5eXlOTmZPHmy8/9jx47ttm/m47BBYD4uSU1NTUG/UB/4b11dnaSOX/SZNGmS8vPzlZeXpylTpjj/zcnJcXqnAObjsEVgPt7e3q4tW7aorKxM5eXl2rBhg5OTw4cPKyUlRePHj1d+fr7y8/NVUFDg/Hn8+PFKTU0N2i/9FmzReV7S0tKi8vJyrV+/3vnQ9fXr16uhoUFSxweVBm4WHriBeEFBwTFvXkS/BRt07rckqbm5WevXr9f777/vPNatW6fm5mYNGDBAxcXFmj59uqZNm6Zp06Zp+vTpQWt6O6Pfgi06/zJqQ0ODVqxYobffflsrVqzQ+++/r9TUVE2dOlUzZ87UWWedpXPOOUfjx4/vdb/0Wwhl//79ev3115Wamqq8vDzl5eUpPT090YfVo879VqyRk9ipqKiQJBUWFib4SOzgZt0nyzyjtbXVWYeckpKiCy64QAMGDEj0YSFBus4betLbTVxnz56t8847L+j3yyXmDUhex/pQmsbGRn3wwQdBNLLb/gAAIABJREFUj5qaGrW3t2vMmDE67bTTdOqpp+q0007Taaed1m3OTH+DZNVTf2OMUW1trTZs2KCysjLnv4H3HdLT01VcXKyTTjrJeUyfPj3od1ul5OlvgM566m8CHxgc+ODJwNqNyspKNTU1SZLGjh2rkpIS5/XRkpISFRYWqn///s5+6G+QrLr2N8YY1dXVqbq62vkAv8Da77q6OhljNHDgQOemxcXFxZo2bZqKi4uD1inR3yBZ9dTfNDY2Bv2eQ+AR+IyCfv36adKkSU6tl5SUqKSkRHl5efx+A5JGT33MwYMHtXnzZuexadMm589bt251PoMjJyfHWQtRWFjo/PmEE05w9kX/Di/p3N+0t7eroaFBmzdv1pYtW7Rlyxbnz5s3b9aOHTtkjFH//v01efJkFRUVaerUqU4WOt/0hvM8vKTrdeHgwYPaunWrk4MtW7YE/X/ght5ZWVmaOnWqCgoKVFhY6Ny8Mj8/33lfmvktvKRr/79nz56ga0Ln60Jtba3TH2VnZ6ugoMDJQGFhofLz8zVhwgRnXTX9Ebyk6+s/jY2N3eYIgT9v375dxhilp6c79V9UVOQ8pkyZErRWiesCEqmn13l27drl1HPX/3au7/z8fGceHPjv1KlTnfrmPI9E6nrebmtrU11d3TFf5wncUGro0KFBv/dSXFyswsJC5ebmKiUlhXktEqqn1y/3798fVNeBR2DOGujPAzeUCpyvA3UeWG9GP4JE6tqPtLS0qK6uzqnjro/A65Hp6enKy8vr9ruKBQUFzmeZ0o8gkbr2I3v37u2xpgOPwOuLQ4YMCarpwHl74sSJ6tevH/0IEqprPxJ4DynwWnltba22bt3q/Leurs65wWXg/dRAfU+dOlVFRUUaNWqUJPoRJFbXfuTTTz91arlzXdfW1mrLli3auXOn8/5oXl6e875o4Jw9depUDR48WBL9CBKrh9/bWN7tpq6Al7W0tGjTpk2qqqpyTtR1dXXO46OPPnK2HThwoCZMmODcgCwnJyfoRktjx44NWrQO2KC9vV11dXWqqanR1q1bVV9fr9raWm3btk3btm1TXV2dc1MMSRoxYoRzI8uebv46ZswYZWRkJHBEQOzt3r1bGzdu1KZNm1RfX6/6+nrV1dU5fw78wpPUcS3Jzc1VTk6OczPkzlkZM2ZMt1/8A7zu8OHDqqmpUU1NTdAEua6uTrW1tdq1a5ez7dChQ52b9XXuuwL9VtebxwA2aG9vV319vTZu3KjNmzc7+QhkpaGhQUeOHJEkDRgwoNuNXjvnpesNLQFbNDU1OR+wFJi7d34cPnxYkpSWlqaxY8f2eDPLQHaYj8BGzc3Nzg3DO795HOi9AovaJGnUqFHdbvTa+UZLgwYNSuBIAHcYY1RfX6/q6mrnF3q7LiIKiPSmr4At9uzZ49zoNbB4I5CR+vp6tba2SuqYk/R0s9fAY+TIkQkeCeCOlpYW1dTUBF1LOv9S5KFDhyRJ/fv3V05OTtCNXjvf+DU7OzvBIwHctXfvXudDgrr2XPX19c4iwIyMDI0fP77bdSQ3N7fbTV+BZNbW1qbNmzerrKys2y9P1tbWOj3Ucccdd8wbvubk5PCaLqy2fft254MTN23a5DxqamqctSaZmZndbvQaeOTm5pIRWG3//v2qrKx0chJ4nbempsb55aABAwY4N8GZMmVK0A1fx40bl+ARAO46cuSINm/erA0bNjgfThrIy+7duyV1ZCTwgVz5+flBH9DF67nwg927d2vt2rWqqKhQWVmZKisrVV5e7qx/Hzx4cLcPNi0uLtbEiRPps+ALra2tKi8v1wcffKC1a9dq/fr1+vDDD53rSE5OTtBNXqdNm6b8/PygD7cGbPXRRx/pvffec270+t5776m1tVWTJk3SWWed5dzotaioiNdrEbFPPvlEixYt0k9+8hO1tLToqquu0h133KExY8Yk+tDgQZdeeqkkadmyZQk+EiS7/fv36/HHH9eDDz6otrY2zZ8/X3fccYeGDRuW6ENDkon2Jq6Al9TW1na7geu2bdskSePGjdPJJ5+sk08+WaeccopOPfXUoJubAV7T0NAQdOPW9evXq6KiQvv375fU8eGpRUVFzk39pk+fruLiYj4TBJ504MABVVVVqaqqSuXl5c7NLDdu3Oh8YPC4ceOcG3oE3huYNm0avQ08ad++fc6a1MD7xIH3jQPrjoYPH678/HwVFBQ47xeXlJRo0qRJzk1sAK84dOhQt5u2VlVVaePGjdqzZ4+kjrV2gfURU6ZM0dSpU1VcXKyioiLnA7SBZBO4WWVPN/TYuXOns93o0aOdNdaBddaBm/YFblgJeNWePXv+P/bOJLaR7brfPw2kRFKkJg7FWaJmqQe3+zkvSbeBv4G3MpKVFwaShYEAQZCsEhiIvfCU2EFiI0AMBEGAAF54k3jjXbx7BuJAnTy8uIHXg9SaWhNnUhQlkeKo4b9onPtuFYuUeiSLfT6gwKoiqWY17q260zlfg5iP9mVB3+DgoCoOjerD/Pw8pqamuC/LGJpqtSpyX2iFrbu7u8hkMuKzbrdbFcc8OTmJ6elpzM3N8dppxvBUq1VVbLL22UC5YXp7exEIBBqeC9T35TyVjNEh+V8zseXJyQmAF3H74XC4ISZzaWkJkUiE10IzHYkst9Qr41y+GaNyfHysO76zvb2N/f19VUy9XK4jkQhmZmawsLDAa0iZjuPi4gKxWExX3Lq9vY1sNgsA6Onpgc/nE2V6cnKSxy+ZjqVUKiEajSIWi4mxGHk8JpFI4OLiAgBgt9tFvh95HGZhYQFTU1PcHmE6ilwuJ/w42hyJe3t7qny7Xq+3IZ8VSQG5PcJ0Eufn54jH40JkKeeQ3tvbU+VrM5vNIl+bnD+a2yNMJ3J6eqq6R2vFrfI6gZGREVV+dGqTLCwsIBKJ8PwoYzRY6sq8X1QqFdFoIdErmedJ1kcLYnp7e+H1ehEMBuHxeBAIBHRf3W43JzVguop8Po9EIoFkMont7e2G/d3dXZHQHHixeMzn88Hr9Ta8jo6Oin1O2Mx0C9VqFfF4XFU35LoiJ+MEXiQzHxsba1k/fD4fQqEQD24yXQHVEe0zRJ6kPTs7A/Bi8CgQCIh6QJNakUgEXq8Xk5OTsFqtbb4ihnmz1Ot1ZLPZhrpB9WVtbQ2np6fi86Ojo6q6QfXD5/NhdnaWB1mZriSZTOrKXmnAtlgsAnixKIIm2GTZqzx4y88Rphs5Pj5WLayQJzP29vZwcHAgPut0OhukrxMTEwgGgwgEAnA6nW28EoZ5O9AYsDyR3Uz6SpN+8mKNQCCAYDCIUCgERVF4PIvpOmjRh7xAT17QFIvFcH5+DgCw2WxiQpzqRTAYRDgcRigUgs/n4/EspitJp9OquiEHVMoBCBaLRbSzQqEQQqGQ6tjv9/MCEqaryefzusEM2vlEGgeWx7Xksa5wOMxiDcYQ5PN5rKysYHV1tSEZES0IJyG4dkyXgtQcDkebr4Jh3h7NngsrKyuiL97f349QKKRbR+bm5jA0NNTmq2CYt4e2jtAzZXNzUwTwy+2mxcVFEbgfiUQwMTHBCUqZrkauI3Kb69mzZ6JvQXPn2vqxtLSEwcHBNl8Bw7xdjo6O8Pz5c1X9IOnrxcWF6ItQ/aDXhYUFnjNn3guoz/7w4UM8fPgQq6urePr0KarVKkwmE2ZmZnD37l1RP37nd34HHo+n3T+bYd4qxWIRn3zyCZaXl/HgwQM8ePAA5XIZiqLggw8+wP379/HRRx/hzp073Ndgrk2xWMTPfvYz/OQnP0Eul8M3vvENfOc730EwGGz3T2MMBEtdmatgmStzFSxxZbqdRCIh+rcPHz7Ep59+KgQIXq8Xd+/eFduXvvQlKIrS5l/MMK+GPOZJ4zlPnjxBOp0G8GJOQDveefv2bbhcrjb/coZ5ebRrjmh/d3dXd4yf5sNu3brFa40YwyELDrRzvzs7O7i8vFStsdPO/UYikXZfAsO8NIlEQrWuVHufB1605eWyTmWf1wMxnUitVmsqPtjY2EChUACgXuum3TgHBmN0WtWD7e1tIecG9PPB8LpPpltoFh9AAmOKQ6YclBxHw3Qj5XJZxBXv7+9jb29P5C/e2dlBIpEApfMeHx9vkLbSfjgchtlsbvPVMMzrkclkVPlbZAng3t6eyCs5PDysElrKr8FgkHNTMB3J0dER9vb2dKWtcm6J4eFh3bI9NTXF5ZvpSFq16be3t8XnWvVtJycnOfcW01FUKhUkEgndMi3HIQ4MDMDv93M8O9PRlMvlpg6QZDKJRCKBVCol+p3NyjXl8eF7NtMpUBtELsty+Y7FYiK3AqBui2jzUvE9m+kUqA3SrFwnk8lrezc43xrTSWQyGSSTScRiMSQSCcTjccTjcSSTSUSjUSSTSWSzWfH5sbExkb9Zm8c5HA5jZGSkjVfDMG8clroyjMzl5SVSqZSYMI1Go4jFYkilUojH40in04jFYiqhpclkgsfjgd/vh6Iouq8ejwcej4c7tExXcHFxgVQqJRpS8XgcmUxGDPKkUikkEglkMhnReQBeJDdXFAU+nw8ejwc+nw9utxuBQABut1scO51ODAwMtPEKGeb1yeVyYoBIrh/JZBLJZFLUE/l50t/fD7fbLSTIcn3xer1wuVxwOp1wu90YGxtr49UxzOtRr9cRi8XEIrXd3V3VgrW9vT1Uq1UAQG9vL3w+n5BhBAIB+P1+IcUIBAIsWWK6jsvLS12hJS3olIWWAKAoCoLBoKgbcj0JBALw+Xy8qJPpOvL5vK44XJ7MIOQAAHkyg+XITDejt+hIrjNyQDAt0NCrH7RolCdFmG6jmfSV9lOplKgjNCEeCAQQDocRDAbFRsfDw8NtviKGebNQv10WvVLffX9/H/F4HLVaDQDQ19cHn88nRJYkfpWP+TnCdBskRpbrCPXbaaOxrb6+Pni93gbxaygUwsTEBEKhEPdHmK6F6sre3l7DM4Uk4hTENjAwoFqYRfWDxoH9fj/PHTIdz8HBgW7A5vb2NuLxuAiU8Hg8IlBzcnJSlPVgMIiJiQlYLJY2XwnDvB1aBX/KY1V6wZ+Li4u4ceMG9y2YrqaZ8FVOgkdjuXoJTTnQjulmzs/Psbu7i42NDaytrWFjYwMbGxtYX19HPB4H8GIN7+TkJObn5zE7O4vZ2VnMzMxgdnYWPp+vzVfAMG+XWq2Gzc1NrK6uqpJik8wS+DxBsCxA4ETwzPtAvV7HxsaGkIKsrKzgt7/9LVKpFAB13SDh640bN3gciulazs7O8OjRIyF5/fWvf43Dw0MMDQ3hd3/3d3Hv3j3cv38f9+/fx+DgYLt/LtPhVKtV/PznP8ePfvQjpNNpfP3rX8d3vvMdzM7OtvunMQaApa5MM1jmyjSDJa5MtyL3W2n77LPPcHp6iv7+fszOzgp569LSEr74xS9yTCljSE5OTrC5uSnGL+VxTOBF8uvp6WnV+OUHH3wAr9fb5l/OMC+HLLKUy/njx4+FnHtkZESsG5LnfZeWlng8hjEc8joHucyvrq6iXC4D+HwdkFZYPDc3x3IDxnBQmZfLe7Myry33CwsLsFqtbb4ChvmcQqEg8lbIcTCU00Je9+x2u1Xr1KgtE4lE4Pf7ec0aY1gqlYoqvySJ+UhEn0gkxNrmsbExXTkfJdzmXC6MkWm2xj+RSGBnZ0e0c1qJcCgvBcMYlVwup2oLyXm+9vb2VMnqh4eHRawwxULKElfOOcEYnUwm05B7Rd4oZ2pfX5/quSD3E6ampnjelulIZGHaVfnqWG7JGIlWbXr53t2qTT8/Pw+bzdbmK2GYzykUCqrccLTR+A210Xt6ekS5prEaeVMUpc1XwrzPHB4eIpVKCScBvSYSCezv7wtxWqVSEd+x2+0irzTlc/N6vSL/od/vh9vtbuNVMe87l5eXyGQyQv4nCwBjsRji8XiDiLinp0f4m3w+n8iRHgwGxTHn2WHazdnZGTKZDGKxmEpiKZfrWCwmcn4AL9rXPp9P5EXzer0IBoPwer0IhUKYnJyE1+vl/iPTVsrlsm65pjYJOfgoFwHwoj1C92oq4z6fT5UbkHMTMO8ZLHVlmFehUCggFosJyWs6nUY8HhfyV3o9PT0V3zGZTCqBpcvlgqIocLlccLlccLvd8Hg84pgXITPdAMmW8vk8ksmkmLDSvubzedX3BgcHMTo6itHRUfh8Pni93qbHwWCQG3CMYSkUCkKMTAJYkojLAthcLqf6nslkgtPpFM8Peq44nU7Vs4WOuY4wRoOEltoAABrMoiBG4EV9oAFZmmggyRLtK4qC3t7eNl4Rw7xZcrmcagEoBciQMDmVSuHs7AzAi0kMWfyqrR80mWEymdp8VQzz5igUCmIxRiwWQywWQzQaFYE10WhUNYntcrnEhDUJkeV9lscw3Ua1WhUThLTgVN5//vw5jo6OxOebyZHpOBwOY2hoqI1XxDBvlnq9jng8jmg0ir29PfEMkY/lOuJwOITkNRAIqKSW1O7iJCtMN3FxcYFUKiX6JNFoVATz0/7h4aH4PNWRiYkJ0XenV1pAyAkqmG4jmUyKPjuNb8kJL+Q5kdHRUdFHp8WGcv3w+XyckJHpSs7Pz0XyCz3xazweF9JXAGKRLj1HtPvBYJDbXEzHUqlUsLOzg+fPnwvRK4ks9/b2VAt3XS6XSgI+OTmp6mNwoAXTjdBYlZ7Qcn9/X8x3cEA0876SSCSwubmJzc1NbG1tqV4pWZLdbsfMzAymp6cxPT2NqakpsXHyPKabKRQKKsnr+vq6OC4WiwAAm82GmZkZ1UbSV5fL1eYrYJi3R71eRzQabRAlrK2tibXto6OjKlHC0tISlpaWWJbAdD35fB4rKytCmLO6uipEyCaTCTMzM0KYs7i4iA8//JD740xXcnFxgWfPnuHBgwdYXl7Gf//3f2Nvbw8mkwm3bt3CRx99JESvLNNjmlGr1fCLX/wCf/d3f4etrS187Wtfww9+8AMsLi62+6cxHQxLXRktLHNltLDElelGCoUCHj16JMZpqE9aqVQwNDSEubk5LC4uConr3bt3OVEZYzhqtRo2NzdV45ErKytYW1vDxcUFzGYzpqenVeMuS0tLPNfLGI7j42NsbW01iP1WVlZErJrX61UJLGmfyztjNIrFomqNwubmppiXpXgBi8WCubk5zM7OYnZ2FvPz85idncXc3Bznt2AMR7VaxdbWlkraurKygqdPn+L4+BjAi+SsU1NT4t5O282bN+HxeNp8BQzzIqG2NldLNBptGs8yNjYm1ihPTEwgFAqpyrbdbm/j1TDMq3F2diZiH+XYYFniKucsslgsCIfDTcWtLOhjjArlftTLJUECEVpnaTKZEAwGdXNJRCIRTExMcN4uxpCcnp5if38f8Xhc5I6gV4rponoAAIqiIBQKIRwOi1dKVB8Oh/mZwBgeWfwnPxe2t7exubmJk5MT8dlmMVxerxeTk5OcG4LpOPL5fFMh8e7urhjbAV6M4csyEvl1cnKS88wxHUOtVlO1W+TyTXlHKfZ2cHBQJZ+nfbp/89gl00kcHByoZK1agas8ful0OkV5lss0jd/wPZt5l5RKJeEJkGWteudkOVp/fz/cbjcURRG5cUj+RzLAYDDIeTyZtlEul4UrRs8jQ+ei0agqD9TAwADGxsYactTKrxMTEyyPZ9pGKz+SfC6TyeD8/Fx8Ty/3svaVx8yZdlGtVpHNZpFMJpFOp8U+SbcTiQSy2SwSiYQqX7LJZILH4xHtEK24ldom3B5hmAZY6sowb5Nisdggf43FYshms8hkMuJhl81mVZ0RAELG53K54PF44PF4xLHX61XJYDlIlTE6p6eniMfjyGazODg4EI0/Os5ms0in02K/Vqupvm+324UU2el0wul0wuPxwO12i+Px8XGMj4/D6XTywgjGcNTrdVEf0uk0MpmMqm7Qs+Tg4ACpVEq1MAJ4MchFzxBFURrqBdWN8fFxjI2NYXx8nBdMMB1NrVbDwcFB04WjFGggD4jRIiF58ai8HwqF0N/f38arYpg3Cy2ia7bIWk6EDlxdR4LBIItfma4ik8kI4ev+/n7DfjweV/U7FEXRFb+SeMnn87E8hukqjo6OEIvFhBxZDlqjY3nRiMfjUdURr9crFouQnIwDOJluolKpIJFI6AYsaIPZgOZyZHmS3uv1cpIWpms4PT1tSABAi2eprSU/R0hqSQJLuZ1FAlge02W6iZOTE5XoNZFIiIBQeo6QXAN4kSBArhPUxpLrjMfj4YVeTNehHd/SjnPt7e2p6spVba7p6Wl+njAdCS1yl4WWcnmX5zsGBgbg9/t1k2VwUDTTjdRqNezu7gop8vPnz7G1tSUEydSvsNvtmJqaakikNDk5iYmJCR67ZbqSy8tLxGKxBtErJRAulUoAXrSRIpGIkLzK+5OTkzCbzW2+EoZ5O8hJZ+SE2rLQcmRkRNQLebtx4wYURWnzFTDM2+H8/Bw7OztYXV3Fs2fPsLa2JsQKhUIBwIukB0tLS5ifn8fc3Jx4DYfD6Ovra/MVMMzboV6vY2NjQ0heV1ZW8Nvf/hapVArA5xIGEuwsLS3h5s2b3JZiuo5EIoEHDx7g448/xvLyMp49e4be3l7Mzc3h/v37uHfvHr7yla8gGAy2+6cyHcbFxQV++ctf4gc/+AHW1tbw1a9+Fd/73vfwpS99qd0/jelAWOrKECxzZQiWuDLdRj6fV4lbHz58KKSWw8PDuHHjhkreOj8/z2MujKEoFotYX1/Hs2fPsLq6irW1NTx58gQ7Ozs4Pz/HwMAAFhcXsbi4iBs3buDGjRtYWlrCxMQEr5NmDAXdz2VR8fb2NnZ2dnB5eQmz2YxAIKCSti4uLuILX/gCJzdjDEWpVFJJW+X9ZDIJAOjr60MoFMLMzIxK3Do7O4tQKMT3d8ZQnJ6eYmtrS2y0Hm19fR2JRALAi+Tak5OTQlA8Ozsryn8gEGjzFTDvO7VaDbFYrCGesdn6elnEpF13zGvrGaPSSlTZKo9KsziTyclJbs8whiOdTiMejzfkR5H35fhdt9sNv9/fkCuFhE9+v5/jEhnDkcvlRCxuIpEQdSCRSIj8QbLAb3BwUCSql6VQJG8Nh8Mcd8IYGsp/opf7JJlMvpS0NRKJwGKxtPFqGEbNxcUFUqmUyFUSi8VUwta9vT1xz+/p6RHyKGrraOWtfL9nOoVKpdIgs5TFrclkEhcXFwAAm82mKs/azev1tvlqGOZzWonkt7a2VO10Hr9k2gnlNSdXjJzbn9wYtJ9IJFT5BAEIN4zH44HP54PL5YLP5xMuGa/XC7fbDbfbzeOPzDulWq3i4OBAeCxkxwsJLNPpNJLJZIPrhSTEchn2er3C9UJlXVEUjI2NtfEqmfcRuWyTuDKVSgmxdiqVEu6WbDYLWcE3ODjYUIYVRRHSbUVROK840zbK5bIQxVMbhO7RcrlOpVIqUSvwoq+oKIq4T9M+yeQpV6XH4+H2CMO8Gix1ZZhOIZfLiY47PRipo0MNQurE53I51XfNZrOQvJKsTxb1jY2NweVyqc5xIk/GyJTLZeTzeZHAOZ/Pi017LhaLNQgugRedqNHRUSHOGB0dbbn5fD4ODGcMAw0w0DOE9uXjg4MDHB4eIpfLIZfLQdsktFgsQvCqFb/K8lftPnfMmE6hVquJBagkxZD3o9Eo0um0mKzu6+uDoigIhUJiwIGEZIFAQAxEcBuK6RbOz8/FQqV4PI54PN6wn0qlhCygt7dXSC3leiG/er1eTpbCdBXaoB7tApFoNIp6vS4+ryeP0b6yRJzpJlKplEqIvL+/L9pcJCSTA36GhoaEjIzaW16vF8FgUPVs4cRETLeQSqVU/Q8KCKLA6Wg0KuQaAGC1WkV9kIV9VG9I2Md1hOkWMpmMqBP07KA6Q8Fzch2x2WyqPjvVE6ortGiAg0eZboEEf80SDSQSCdXYFvD5gvVmAvFwOMzJwpiuo1VCDmpzkZQGuFr8GolEeD6Q6Tiq1Sqi0ajoe1NgHh3v7++L/ndvb68IOg2FQqqN+htut7vNV8Qwb4aLiwvEYjGV7JUSqO7s7Ii1VRSMTZJXWfw6OTnJSWiYrkUOftVulGgYaEzGQUmHp6amMDIy0uarYJi3g5yEW64bq6urKJfLAJrXjZmZGTgcjjZfAcO8HaLRKNbW1oTwlaSvmUwGADAwMIDp6WnMzc2J5MWUtJuDcJluRSviWV1dxdOnT1GtVmEymTAzMyMkr4uLi/jwww+53810Fel0Gp9++ikePHiA5eVlfPrpp6jX64hEIrh3754QvS4tLbX7pzIdwsXFBX71q1/hhz/8If7v//4PH330Ef72b/8Wv/d7v9fun8Z0ECx1ZVjmyrDElekmEomESt5KY44A4PV6VX3Gu3fvYnFxkWMvGcOQy+XEOKEscN3f38fl5SUGBgYwOzuLhYUFLC0tYWlpCTdv3sTU1BSvdWYMQ7Vaxfr6uthWV1fFPsnQnE4nFhYWMD8/j7m5OSwuLmJubg4TExO81oAxDCQApHlRea50d3dXrEUeHR1ViYppW1hY4Nh6xlCcnJyoxK3yRrLi3t5eBAIBTE9PY3p6GjMzM2IONBKJwGQytfkqmPeRUqkkYhDlGF2Sk0WjUaRSKfF5q9Wqu16Yzvn9fo5nZwwHycj0YkOSySQ2NjZU8SGjo6MNwg/5mPM6MEbkqjip/f19lUREWw+0+6FQCHa7vY1XxDAvz1X1QJtXlOIFW9UFRVF4LIcxLPV6XeRhIKmlHGsYi8VweHgoPj80NKSKJwwEAqKfMDExgUAgwG0kpqPI5XKq3FXRaBSxWAx7e3uin0w53ihXqJ6slV4HBgbafEUM8+Lenc1mm7Znksmkanxezn+glW37fD5MTk7yWgOmIzg+Phb3bMoZJec/iEajIudBf38//H6/kA9rpcShUIjv2cwbpVwuC8dLOp0WuffpmMSWJLqUBcMAYDKZ4HQ64XQ64XKcjtz1AAAgAElEQVS54PF44HQ6hQSQRGlerxculwtms7lNV8q8bxSLRVFuqQxrha2yoFgeQwc+L9sul0tX0Cqf45g45l1CnqHruoZSqZTKoTIwMICxsbEGz5A2DziNoXN7mnlXHB8fC6/cwcEBcrmccMzJ9/GDgwMkk8kGcbzD4RDtDRK0knBbURSVmJjXczHMW4elrgxjRGhw9uDgQAhfqcOUTqeFoE/e5KTOwAtZnyzq00pftRI/p9PJyagYw1IoFBrqBMksta+05fP5hr9D9UYWWDqdTnE8OjqKkZGRhtfh4eE2XDXDXJ/Ly8uGOiAfHx4e6opgKZEh0dPTo6oPV21UT0ZHR/kZw7SFer2OZDKJaDSqki1RkMP+/j7S6bSQWgIvBjVIQEav8mAdbYODg228MoZ5M5ydnSGVSqkmzingJxaLIZlMIh6Pq54Hg4ODDfVBW2cCgQBLZJiugOTIsVhMiPu09YMmgYj+/n54PB4hrySZBsnI6JUTpTPdAi0ujEajSCaTop0ln8tms+LzfX19oo5oha/0XAkEAtx/YLqGo6Mj0Q+hekHyVzpHydKBz58j8vOD6oz8ygtjmG6BpJbywnTtYnXtQhs5CFX7SovWWZDMdAvVahW5XK5lPdnf38fZ2Zn4jl4dkQM6WNzEdCOZTEYksqH2lnYcWB7fcjgcot9O0nC/3w+32y3aWz6fj/slTEdB4j75GUCbts1kNpsxPj7e9FlAUlibzdbmq2KY10NO7KTd1tfXxcJmk8mEYDDYNOA1Eom0+UoY5s0j1w+t2HJvb0/Mj2ullvLGSYqZbuTs7Ax7e3vY3NzE5uYmNjY2xL5cN7xeL6amphq2SCQCl8vV5qtgmDfP0dERnj9/3pDoe21tTSS1l58ZcrLvxcVFWCyWNl8Bw7xZqtUqVlZW8OjRIzx+/BiPHz/Go0ePkMvlAADBYBC3bt3CrVu3cOPGDSwtLWFhYYETNjBdQbFYxCeffILl5WUheq1UKlAUBR988AHu37+Pjz76CHfu3OH+AoOPP/4Y3/ve9/C///u/uHfvHr71rW/hD//wD9v9s5gOgKWu7y8sc31/YYkr0w2Uy2WsrKzgs88+w5MnT/D48WN89tlnODo6Qm9vL2ZnZ3Hnzh3VxmWaMQr5fF6M+WnnjYAXyc+mpqaEoJhe5+fneR0mYwjq9Tqi0ahqzQCVde38jzy2TbJir9fb5itgmOtx3bJOczpaeev8/DyvF2MMBa2Z1Nt2dnbEekn5/i5vXOaZd002mxVr2Cn2XCtwPTo6Ep83m80ilpZia0OhkJAdhEIhOJ3ONl4Rw7wctVpNJCbWxj7Jr/Kad1loo7fmfXZ2lkWVjKE4Pz8X8U0UyyQLcGif5DcA4HQ6EQgEEAwGEQwG4ff7G/Y5vxVjNPTin+Rnwd7enliXCFz9PKBXhjEqtVpN5KyivgKJWklqmUqlRC5pk8kkngHhcFg8F+g4EAjwHDzTUVD8kjbulY43NzdVou5WwvpIJIJgMAiTydTGK2KYz/MOkmib2vSUu4DyEMox3T6fD8FgEKFQSIz30DhPOBzm/B5MR1AoFEQ+DirLco7maDSqEgXabDbR/qC2iCxv9fv9LJJnXpmrRH96WzKZbPg71Lag3Pey4E97zPnImHfB0dGR8DvIXhStI4WcQwcHBw3+B4vFIoTDbrdb+IOcTqeQEZPE1e12czuDeScUi0VRlg8PDxuEluTQkmWW9Xpd9TdsNltDOaZNW9a9Xi/n/WLeCZVKRSXPzmazQtIqn5PlrXple3x8HG63W/jgqCyTsNXtdsPr9cLtdvO8D8N0Fix1ZZj3BT3RK21yw1bearWa6m+YTKYG4evY2FiDpE/vHCdlYIzGxcVFS/krSS5l6WU+nxeJP2V6e3t1ha9asaV8Xj7Hg9BMp1Iul3WfKVQfmm3yRAzR19d3LRGsXh3hwUHmbZPP51suDE8mkw2CDFoQ2Ewk4/V6EQ6HWWzJdAUkWmpVT5otmm1VR0KhEAdSMF1BuVxuEL/KAlhaeFipVMR3rFarSvyqKIoQ95FIxufzYXh4uI1XxjBvBr1gPG3whXZB1+DgoGpBjPaV3guHw7xIhjE81WpVN1ibnh+pVAqJRAKlUkl8x2w2q6Rj9Or3+1XPFI/Hg56enjZeHcO8PqVSCfF4XLStmr1q64iiKKK9RTJxEojTwgaWcDDdgjbQVdvmetW2VigU4vkLpms4PDxUJUWQ21pUbzKZjGoM2Gq1wufzQVEUeL1eeL1eIYKVhbBut7uNV8YwL9AGv+r1v19WBM6JDxij0yqxn5zMUk4Cot3m5uZ4ro/pOmq1GnZ3d4W87/nz56qN5jIsFotKZEmvk5OTmJiY4GABpuuo1WrY3t7GxsYGtra2VHVkd3dXBNg4HA5d2evU1BSCwSDPWTBdRyKRUEkeKDn47u4uLi4u0N/fj1AopCt8nZyc5DkKpquIx+NC8Pr48WM8efIE6+vrqNfr6O/vx8zMDG7cuCFErzdv3sTU1BQ/GxhDc3Z2hkePHgnJ669//WscHh7Cbrfjww8/xL1793D//n3cv3+f+wjvMcvLy/jxj3+M//zP/8Tv//7v49vf/jb+4A/+gNsB7zEsdX3/YJnr+wdLXBmjs7+/j8ePH6v6eJubmzg/P4fVasWNGzdw+/Zt3L59G3fu3MHt27dZBsUYAhrLkwWuT548EYmytZI/ep2YmOAcEUzHc3Z2ht3dXWxubmJzc1PM52xubqrm/xVFwezsLGZmZjA9PY2ZmRnMzs5idnYWAwMDbb4KhrmaSqWimsencr65uYn9/X2VpHh2dlaUc3qdmZmBxWJp81UwzPVptr5rZWVFJOY2mUwIBoO6a7sWFhZgtVrbfBXM+8CrxGtctTaX42IZo3B+fo50Oo14PI5kMili/KLRKNLptMinkM1mxXd6enrg8XhEnKvX64Xf7xdxfSwjY4wG5UmQJSMvk5eqmahyZmaGk9IzhiKbzSKdTiOZTIp4PBJBUW4EWUzZ29srciCQoNLv96v2A4EArzdhDE2xWBR1gGRocs6QeDyOdDotpH9yvQgEAgiFQgiFQkLaGgqFoCgKj9kzHcPh4aEQEdN9X86NE4vFcHh4KD5vtVqFjFhbxklwyeOXTLuhfi6V52g0qpJtU3mntn1fXx8URRECSyrL8r1cURReL8m0HT3Jtt4+0WoMk/a9Xi+XbaYlFxcXDfnoKV+97HPQ29disVgwNjamcqK4XK6GcyRJUxSFx1WYt0qhUBDySnIx6AlatU4TGhchTCaTKLtjY2Niv5WwldcqMm8TWap9HZl2IpHA4eEhqtVqw9+S2xOyV0TvnN/vZ8cI81apVCqiHSILiOWN2ihym0TrnOrv74fT6VSJWeX7NZ2Xz/FYB8MYGpa6MgzTnEKhIAR9zYSwckODNm3HEACGh4evJenTE8LyAB1jJM7OzkRdODo60n1tta/3WLbb7brC1+Hh4YaNBJcOh0Oc40UpTCfyMgM08pbJZERQkQyJBmizWCwN55ptFosFIyMj/LxhXouzszOx2JyEMTQpLx+n02nV91wulxDIyLKlYDAohH0ej4fv5UxXkE6nG8RKNAhPWzqdVvUnxsfHhRhDURR4PB74fD5V/VAUhZO6MF1BLpcTCxQpYIkCmKiOaAUyFotF1BGt8JXqi6IocLvdLFtiDA+JX6ldRQvUZdFSMplEuVwW3zGZTHC73UJgKQuXqJ5QvTGZTG28OoZ5fY6Pj0V9oH4J1RO53pyenorvyHWEgmBlAZnL5RJyS54QZoxOuVxuGRSrlygBaJSZaRcF0T4HQTFG5+DgQNQDEoZnMhnE43FkMhndtlZfX59YBKrti1Cbi95j2RnTLeTzeSQSiZbJFmKxmEh4SejJMbWvwWCQ+yVMW6nX60in06ogWhrDlc+RzA8AhoaGEAwG4fP5RMAhlWc65/F4uJ3EGI56vY5oNCoSAcrBidvb29jZ2RFrOyi5sd7GidWYbqSVEHl7e1t8rlXdCIVCPGfBdB1UNygZPtWJra0tHB8fA2idTJZF4Uy3Ua1WsbW11SB8ffz4sRiDHRkZEeJjWRYxPz/PwcVM13B2dob9/X2VLOXhw4dYX1/H+fk5TCYTZmZmVKKUxcVFLCwscF+aMSTn5+dYW1vDgwcPsLy8jN/85jfY39+HyWTCrVu38NFHHwnRKydhfv/4n//5H/z93/89fvWrX+HmzZv45je/iT/+4z/msZP3EJa6vj+wzPX9gSWujFGp1WrY3NzEw4cPRZ/t008/RSaTAfBChkb9tLt37+Lu3buYn5/n9gvT0dA8pzwWsbq6imfPnqFUKgF4MYejFbcuLS3B6/W2+dczzNWQnFg7R7m6uirWNmrnKamcswyHMQr5fF5IW7VbPB4X61UURdGVtk5PT/O8I2Mo9NahrKys4OnTp1fOtS8uLmJubo7XoDBvhVqthkwmg1gsJuIqKJaVYisojk8rYCI5Ja2lJVGZz+dDKBTi+XDGMFAsnp60+CpJZbOYiUgkwnETjGEol8uqHAa0L8dxp9NplbQYAMbGxoTchjY5pwHlmOKYbcYIVCqVprGmqVRK1I9MJoNarSa+NzAwIPKnUZxRKBQSbaNgMAhFUfh5wBiWi4sLURdkieX+/r6QtWpjTAcHB3X7CoFAAF6vV0j/uI/LdAKUT/OqOFM554DNZmuIM/X7/ULcGggEeM0A03aa5ZyR+73afq6cH0ArteQ+LtMppNNpVa5K+ZWkxEdHR+LzFotF3JtJPkz3bjrPa/wYGb3c8Fofid57NM8j09vbqxJYaoWsY2NjQogmv2e1Wttw5Uy3UygUdP0gV507PDxUjYMAjWW71b7T6RTn7HZ7m66e6Wbq9XpTP46eeFjetGUbQEMZbrXJn2PXB/M2KBaLKjeaXpnWE7fS+lmZoaEh4UbTK8taUavL5eJ2MsO8f7DUlWGYN8/x8fFLCfrkQRi9W5Iss5SlliSw1JNaynJLbrgzRoLqTysRLL0eHx+rNm2CaGJgYEDUD4fD0VQIS++ThFk+bzab3/H/BMPoc35+rqoT8kZ1Qd7Xnsvn87p/12w26z5D9J41Q0NDqn273Y6hoSGMjIy84/8NxqhQgFErkYxWbCkvXpelMdpzPLnPGB2tIDkajYrFAqlUSgQ8ZTIZ1Ot18T2z2aySyLSSwPLidsboXEcgQ+/J0AIxveeI9nnCMEaGxJay+FUrgE0kEg0Lfjwej0oAS7Jkeq643W64XC643e42XRnDvBmKxaKQvFJgeTQaFYHmVGe0YsuhoSH4fD5RD+R6oT3PSUAYI3NwcIB0Oi0CCimoKp1OI5PJiP5INpvF+fm5+N7AwIDod7jdbvEscbvdUBQFiqLA5XKx3JIxPOVy+Vp9Eb3EDK3GtFhqyXQbR0dHqv4HtbG0yX3kub2enh7RBwkEAqpXep5Qm8vpdLbx6pj3nVYJeuicdo5DKzbWeyYEg0FOnskYhpOTEyF3pU0+JvnxwMAAwuEwJicnEQ6HxTYxMYGJiQl4vV4WNTFdRaVSQSKR0JW9bmxsiPEmOdmmNpg9EolgcnKS1/sxXUUrGfLu7q5oN7WSIUcikTZfBcO8OWKxGNbX17GxsYH19XWsra1hY2MDe3t7uLi4QG9vL8LhMGZnZzE/P4+5uTmRhDwUCnH7iekKSBwki15XV1exs7ODy8tL2O12zM7ONghW+HnAGJFEIoEHDx7g448/xvLyMp49e4be3l7Mzc3h/v37uHfvHr7yla8gGAy2+6cy74jHjx/jH//xH/Hv//7vmJubw7e+9S380R/9ESelfI9gqWv3wzLX7oclrowRIQkg9cEePnyI9fV1nJ+fw2w2Y3p6Wohbl5aWcPv2bbhcrnb/bIZpysHBgWpsbW1tTYwtnJ+fo7+/H5OTk1hcXMTCwoLY5ufnORkg0/Hk83khJJbnVLRyYj2x340bNzjOmzEEreYPt7e3xedIMq8t7ywpZoxErVbD7u6uWFtF5fz58+fY2trC6ekpgBdJ5Kenp8U2NTUl9oPBIM8RMm+MUqnUIGjVrvfWE/TROlhFUeDz+YSQT5YysZiMMQLValUVH0d5POhcKpVCLBZDKpVSJe622WwIBAKi7NOr1+uF3+8X6wA5jwdjBGSRU6sYOVncDVwvJiIUCvHYC2MImgnNtPVCWw/05N16dUFRFG7DM4aFcjo1qyPJZBLRaFSV90xbN7RxElwvmE5Cjn0mAbFW1ppKpVSxoYqiwOv1NoiI6Zzf78fw8HAbr4p53zk7OxP5Kuk1kUggkUiIMh6NRsVYJPD5vZsk2xTjTOWaBMUDAwNtvDLmfadQKIgyTKJWkrXK92x5DMdut4syHQgEEAwGG+StvKbr/aJer6vyslNu9pOTE12vgdaFcHh4qPt3SYCmdYbQ1uw9bjMwb5KzszNV+X5ZOaucF4sgwV8zJ44s/5OFxLxGmnnTkHRYT8yqfdWek9u9RG9vb1N55VXSVs75wLxJ6vV6g1dGr0xr36N9PfHwyMjIlcJhvfLP3iWGYa4BS10Zhuks9Dq4ehsNAMnivnK5rPs3SVLZSgCrFVjKn7XZbO/4f4FhXo3Ly0tdmaXeAGmz97TCDsJisYi65HA4MDIyAofDAbvdLjaqM/I5+iyJL61W6zv+X2GYRvRkr9rjZqLYk5MT3YEpwuFwiPJut9sxMjIi9un86OioSgarrSckjWXeb6rVqkpeSQEh8gJ5Oie3gXp6ekSSfxJYyiIZkpMpisITqozh0VskrLdo/lUlyaFQiBOFMYbm9PRU1IFkMilkZFqZTCaTUdURm80mBMn07JBFfSztY7oFeo7oBd7Kba5MJqNaeNHf36+qHyTuUxRFiJZIxORyuXgimjEslUoFh4eHqvZVs3299hYtRNJrd8n7HIDCGBkKzGpWN+g1k8moBLBauWWzfb/fz8mlGMNyfn4uEjxQPaB2FrXB6Jw83qo3tkXBXm63G36/Hy6XCy6XC06nk9tajOEplUqiTrTqn2SzWVV7i/olLpdL9NVpn85TH8XlcvEiQuadU6lURKBYLBYT41OpVEpVxrVJr0ZGRsQ8Bo1P+f1+eDwekQRLURSMjY216coY5nokEgmV8HVnZwd7e3vY29tDNBoVC8XNZjOCwaBK+CoLYAOBAM9TMF1Fq8S0+/v7Yhx2YGAAfr9fV2rJyWmZbqNarSIej+vWi9XVVbEehOa4ZakfbeFwGH19fW2+EoZ5fWq1GmKxWEOy/idPniCdTgNQi8HlRP1LS0tcF5iu4OTkBJubm6IekGwomUwCeNFvXlpaUoleb926Bbfb3eZfzjDXJ51O49NPP8WDBw+wvLyMTz/9FPV6HZFIBPfu3ROi16WlpXb/VOYts7q6in/4h3/Af/zHfyAQCOAv//Iv8Wd/9mcYHBxs909j3jIsde1eWObavbDElTEStVoNm5ubQty6urqKR48eiXlJr9crxK2Li4u4e/cu5ufneUyB6Uiq1Sq2trawvr6OjY0NlcSVkmZarVbMzs5ibm5OJW+dm5vjtSJMR0Pzhtrx4LW1NbGecHBwEJFIpGFeZGlpCV6vt81XwDCtqdfriEajunOA6+vrKBaLAFrPjS8uLrIUjTEM6XRaJWyVBa6xWEysfx0bG8Pk5CQikYhK2jo1NYVAINDmq2CMTisxmTbeR+Y6gr5gMMjrlZiOpl6vI5vNIpVKIZlMIpvNqqStiUQC2WwWyWQSR0dHqu/a7Xb4fD4RG01CG1nWGggEWFLJdDyVSkWUc4rBSaVSSKfTIn6BxGWlUkl8r7+/XxWr4PV6xUbnKL6N13QznU61WkUul7tSWqyVUQ4MDIik9tp2kNxGCofDnF+GMSwU76ytD3o5yuR8AHa7XcS1BQIB3ddgMMi5VZmO4Dr94ng8juPjY/Eds9mM8fHxplJir9eLiYkJzsfNtA26f2vj7rXiVm3OI8q7SnJWPXGr0+ls45Ux7zu1Wg0HBwct79vb29vI5/PiO3r3bO1rJBLhtXpdhiyszOfzuiJWypOuzaFO5+VxEBmbzdbg6XA4HLoySz1BK8O8LuVyGZVKBeVyWdc/c533UqkU9BRbcv7Dl9nGx8dZ6M68ES4uLsS9W0+kTcetBK3y+ARhs9kwMjIi7tPXeaV9nutk3iRa+aqejLXZeVozJdPT09NQbrVlXSvWlvc5jy3DMG8RlroyDNM91Gq1awn59OR91LmRB+KJ/v5+IXqlTRZXOhwOIeSjY1nmR5/lACjGCGg7/HqdfRqsJQlsoVBAsVhUnatWq7p/v6+vT1Vf5G10dLThXCt5LHeUmHZCA7t6g7wvc67Zswd4MQhssVgaBoNf9pzb7eaFoV1OoVAQi+opIbosypDPyYsqzWazSvRKC43lc7T4ngdfGSNDixf0FhzL52KxGE5OTlTfpWCsq6RkgUCAhdyMYTk/PxcBKSR+jcfjyGaz4jztU/IPwmKxwOVywefz6QpkZAmsy+Vq0xUyzOtDAVwkoZHrB7W30ul0Q3urr69PlH9qZ+nJYElKxu12xsjoyS312lzZbFYlSqZFo9cRXHL/ljEqZ2dnIhCeAoOpr64NlM/lcqrv2mw2EfhLkktqZ2nPceIexqiQSLzV80MvYBj4XGqj9/zQHrNInDE6clBlM6F4Pp9vCKwEGutKq7aX1+tlYTLzTqG+RKvgs1gsJiSYwOdJI1ol0fL5fAgGgzCZTG28OobRRyu2lAMtNzY2UCgUxGdHR0cbEndSGZ+fn+fAeKZrODs7QzQabRAi05ZKpcRn3W43JicnG7ZwOIxQKMRBc0xXkUgkVMnMadvc3BRz23qSS9rm5uY4iRbTFeiJwVdWVvD06VPRBzabzQgEAg2y10gkgomJCR4XYgxNPp9XiV61UqLR0VEhIyIx0Z07d7i/wBiCYrGITz75BMvLy0L0WqlUoCgKPvjgA9y/fx8fffQR7ty5w/fyLmV3dxf/9E//hH/7t3+Dy+XCN7/5Tfzpn/4pJ7zsYljq2n2wzLX7YIkrYxRo7GxlZUVIXNfX13F+fg6z2Yzp6WncvXtX9JW+8IUvcGJWpiOR+/3y2BeVZ+Dzvj+Nd9E+j3sxnYw8riuXcXlOnISWcvmmbXJyktcwMR1NLpcT89lU1p8/f47nz58jGo2Ke7jT6cTU1BSmpqaEyJJkliwoZoxCrVZDLBbTFRVfd+6ak8kzr8Lh4SHS6bRK0kfxOLRPMdGVSkV8z2w2w+PxCPkSvWqFfRynxnQ6evGaerGbmUxGldybcg+1WmfN+TEYI3B0dCTiL+n+T/sU60/PCHntNfAiJpNi+PWeCfTq8Xi478l0NOVyGQcHByKPhZzvgvLCUJ3QxpI5nU4hNKO2j9/vFzHJVEfGxsbadHUM8/qcnJyI9hDVC5La0znqP8jtJYfDAb/fD0VRmkpb/X4/r79j2s7FxYVoB8XjcXHvT6fTiMViyGQyiMViSKfTqtjLwcHBBkm9XN7p1e12t/HqmPeZSqXSIGqlvKl0H9e7f9tsNnH/JjmrnriVc78z7aJUKon2h9wmke/d9B7R09Mj2u1+v1+UZ7/fL0TEiqLwPdug6AkpW4kqtZ/RSquJZrnKtVuzzzidTr5XMq8FOShkVwXtk4BY7z3ZDyPnIJQh14vD4RCyYXLDyMfyZ+h9Ev/xWirmddDzs8hSVrk8U5nWvq8dryYGBwcbyrVWWNnqle/dzJvgVdon8vvaPLKEXpvjOu2V0dFRnrdnGKaTYakrwzCMTKFQaCmBzefzqkEDElrS5wuFQkNCaWJgYEDIKWV5JZ2jAQBZYEnySzrncDg4OQFjGF5HdEnnDg4OmtYpQN1R03bQXuZ4bGwMg4OD7/B/h2E+R64D2jrxMvXm6OgIzZr2VNavqhdXnfN4POjr63vH/0PMm0RO/P8yC/cpQfpVCf9HR0cRDAZZAssYluPjY1EHSDqWzWbFAh9a4JxOp3F6eqr67tDQkFj44HK5xCJ+ElqStI82hjEitVqtQfYqB77IAZHZbFb1LOnv7xf1gWSvekJYklty0nXGqFxHtKQnpAGuFi2x3JLpFq4KKKZ9PXkficauksCyvIkxKrVaDQcHB1cG3OfzeaRSKdVYkHY856pnCgsuGaNxeXkp+hrUV89mszg4OBBiZDpOp9M4OjpSfd9sNou+h6IocDqdoo/i8XjEMQWiccAlY2QKhUJDkgqqP3K/PZVKNdQVi8XS0HfX7rvdbjidToyPj3O/hHln0JgUBWSmUqmGRBTxeFy1wL2np0eM12rHaPXKtcViaeMVMsznUHJbWfZK29bWlirhiix9pYB7Op6ZmeE5O6ZrqFariMfjuglCnz9/rmrTtKoXs7OzsNvtbbwShnlzpNNpVXJoeaMA/56eHvj9fpEcmpJFT0xMYGJiAoqitPkqGOb1YeEr8z4jC4zo9bPPPhNrmrxer5C8ysJX7v8ynczZ2RkePXokJK8ff/wx8vk87HY7PvzwQ9y7dw/379/Hl7/8ZV5b1GWkUin89Kc/xT//8z/DZrPhL/7iL/BXf/VXnEy8C2Gpa/fAMtfugSWuTKdTq9WwubkpxK2rq6t49OgRstksgBd9H7nPc/fuXSwsLHB/n+koqtUqtra2GsStT548ERK04eFhTE9PN4xhzc3NYWhoqM1XwDCNnJ+fNwj+tra2sLW1hc3NTbF+w2q1YmZmBtPT05iZmRH7s7OzPE/BdDSVSkVIW0ncKu/T/buvr0/MQchzcrTx2AZjFPTm3Gjb3d0VScPlNRnaLRQK8ZpSpiUXFxciDpli9WmfYgNIapDNZlWxl729vapYAFrzryiK2EhS43Q623iVDNOcarWKg4MDUdZpn9ZDU34LkhbLwobBwUEh4aMYGFoLLZ/z+XwcC8N0NNeJLc7n84hGow2J77Ux+K32ea6C6VQofpjiYzKZjOrZoI2VLBaLqu9TzBfd+0nSSuJWeha43W6WPDMebRYAACAASURBVDCGpVKpNIhZ9WSt6XQa5XJZfK+3txdut1tVN+T+gixx5Vy+TLs5OTkRMb8UJyn3B0h2mclkVKIUElqSmFXuC1N7iNpEDNMOrpvjVCsplNv6zV6prc8w7xrKcUIx7NQWofaILG7V9mOpbSLfq0lATHJij8fD+bE6jHq9jmKxiHw+j9PTUxSLRRSLRRwdHYl92UWhJ/Q7Pj5u6M8RFotF5aAYHh7G6OioOCeflzf6jMPh4DLDvBYvK/PTvtdM5gc05j67Suanfd/lcnH5Zl6Zs7MznJycCH8QuYNk55B8jj5L+1fdv0nISvdiErHq3be193c65hgo5nWQyzA5sqgtcpVMm87JY2kyVqtVJdWm8q2VbMtlmz5D4mF2ljAM06Ww1JVhGOZNU6lURKdN7sDJr9qOndxpo+Nmjdu+vj6V7HVoaAhDQ0OiQWuz2VTHQ0NDsNlsusfDw8McnMh0PPJgh1yvTk9PcXp6KupNsVhUHdP7NPBNx80wm82w2WwYHR2FzWYTdWlkZETsy8c2m00ImS0WC6xWK4aHhzE4OAibzcb1i2kLLyuEfVVBrDzw/Sb2WUjVedTrdZWcj5L/04JQrTxD226x2+2q5OgktyQ5hiztc7lcfL9kDEmpVGqoI7IsQxZkZLNZ1SIikltS8JieCJaOnU4nB88whuU6cst8Po9YLCaC2Ql5kv0qIZnf78fIyEibrpJhXp3Dw0MRZExtLdqnxXwUfJPNZlVt9L6+PtGWcjqdDfJw+Xnidrt50TVjWOQ6IcvIEolEg1y8UqmI7/X09MDpdArZGL16PB7VMdUjl8vFQg/GcFBgGtUBel5QICdt9HzRjo2azWZVHSBBH23UH6HN5XKhp6enTVfLMK9Gs2B/veNWouSr+iSBQIATbzGGRRaKt+q7J5NJRKNR1Ot11feprmjrBUvFmXZRKpUaxK/U56Z2Ee1rF9QPDQ01Ha/1eDyqvjcn/WLaCbVxtMJXWQRLUCJFrdiSkonyuCrTLbypejE9Pc1te6YraCVCXllZEWOpAwMD8Pv9qnog143JyUkeD2IMjTb5NIkvNzc3xfy0LHyVZa8sfGWMyvn5Oba3t/HkyRMhOH769Ck2NzdRr9dhMpkwNzeHpaUl3Lx5EwsLC1hcXMT09DSv4WQ6kvPzc6ytreHBgwdYXl7Gb37zG+zv78NqteLOnTu4f/8+7t27hy9/+cvcx+0Sstks/uVf/gU//elPcXl5iT//8z/HX//1X2NsbKzdP415Q7DU1fiwzNX4sMSV6VTq9To2NjaE6JJet7e3cXFxAbvdjhs3buDWrVu4ffs2bt26hZs3b8LhcLT7pzOMIJFINIhbV1dXhQitv78foVCoQdzK47FMp3J8fCxEltr5hr29PSFas9lsYg5aFrfOzMwgEAi0+SoYpjmvKrGU59MWFhZYBMIYglZzyBsbGyLZvDx3pt14TQXTjOsI+pLJJPb39xsSfY+Ojl5LzsfSYKYTIUnrwcGBSsSnPaa1y9o4erPZLHJN0PplrbSVZGQ8D8Z0KpVKBYeHh1c+A0hyc35+Lr5rNpsxPj6uG3ui3Q8Ggyx0YDqWVjlVtMdagRmgbg+1qgv0OR5DZIzI0dGRiOmS20raHBLJZBLHx8eq71I8op6sVT7ndrt5vSfTVkqlkhD8aQWtlIdOT0jc09MjyjAJukna6na7EQgExOvQ0FAbr5B5XykWi+I+TWVbe/+mOF65bJtMJpXMkqTDJLaUyzfL6Jl2QP3ZVgJivdwKAwMDGBsb0223y6+BQIDL9jskn88L4SrlodfmqT85ORHH5IGgY8p1XywWxRy4Hna7XeSmHx4eVgn9mslYZSna8PAwlwvmpZGdCiQbJuHw0dGROCZxJR3ryf1KpZLuv0GySvKXyCJKrcyvmeiP26rMq0Ky7GbunutIWQuFQlOfT29vr7gn2+12sVFZp2OtlFUWbjscDhayMq9EuVy+liPkqnPXFWq3kmk3e298fJzLN8MwTHNY6sowDNOpnJ2doVAoIJ/PNwhgtR3GZmJLEl8261ACgMViwdDQkOg4krySjkkCa7fbG2SWerJLXvDBdDrUMZU7qVeJLpsdt+rMEteVWr6MANPlcvFCQ+atUq/XUSwWkc/nxbNFfg6Vy2XxrCmXyyiVSjg6OkK5XBZ1hfaPjo5QKpVQrVZb/pt2u108kxwOBywWixAkkzh5ZGQEFosFFotFtU/1g85brVZYLBYODHqHFItFlfA1k8mopGTyQrpsNqtaaN3b26sSvMrCV1kGS+9zYibGqFxXbplMJpHP51Xf1Q5860ky5HMejwd9fX1tulKGeTVOTk50g9QogE0+zmazDW2LoaEhISOTnyva5wiJODhgnjEa5+fnovxrAxVkISzJ+7TPEpPJpKobLpcL4+Pjqk0W942Pj/MiGcZwHB8fi4AGkjTlcjkcHBwgl8uJ+kLPFlkCC7xYONtMAEvnZIHT+Pg4BgcH23S1DPPyyEHSesGh2nPaQGngRd/kqv4IHfv9fl6swxgKGvPX9kvkY3n8S/scsdvt8Hg8QpBM/RP52aLdGMaI0PgvtbO0MnGqO1RvtPPUJBWnTSsVp3YY7TudTg5OYd4aL5NERm9e+LrJxDj4jnnXVCoVJBIJXamlNvEotfH1BJderxcTExOw2WxtviKGeX306oVcN3Z2dkBLqen+rhW+yvWEYYxOqwTVchJVlr4y3UwqlcLm5iY2NzextbWFra0tsV8sFgEAVqtVJR+Ynp7G1NQUIpEIAoEAJwBjDEWtVsPa2ppKjvT06VPRPzCbzZiZmcHi4qIQvc7Pz2N+fp7H+pmOY3t7G8vLy0L0+uzZM/T29mJubg7379/HRx99hP/3//4fXC5Xu38q8xqcnJzgX//1X/GTn/wEtVoNf/Inf4Jvf/vb8Hq97f5pzGvCUlfjwjJX48ISV6bTuLy8xM7ODp4+farqo6ytraFWq6G/vx/T09O4efMmlpaWcPPmTdy+fRuRSITHopiOIJFIqMaVaH9jY0PEOLhcLszPz2N2dhZzc3OYnZ3F/Pw8IpEIxwUzHUerOQPtHJqe4I/nC5hOhiSDtF5C3p49eyaS1rLEkukGyuUydnZ2sLe3h93dXbHRcTqdFp9VFKXhPk77fr+f7+mMStJ6lZhMG3cix8O3WldJohqOhWc6jZcp/y8r5tMeK4rC6w6YjuL8/FzEiVBMrjZeRBune3p6qvobdrtd5Ayi/EGU+4Hke7TPsmKmU9HmBmr1PNCLM9HG4PKzgOlGTk9PVbG2evlP5LhcrRhrdHRUPBNcLpeQ++nJWnntGtNOTk9PRf4SyuuTTCaRzWaFtDWdTiORSDS0iyhOlqSW2jJO59xuN/eNmXdOoVAQbRm5XJOEmPbT6XSD/I3Ktnz/9ng88Pl8KiGx2+3mcUbmnUP3bcrD1mw/kUjg5ORE9V3KTUjyYSrX1C6h97gv+2bQE5y96rFebiSZVqKzq445DzzzKryseJX8IbJfhI6Pjo5a/ltaVwi5RWw2m0q4qvdKQmKHw8F5MJiXotm9udV9u9l7BwcHKnm6Fj0BZat7td45Hn9jXoZarSbux6VSSTg36J5O93FyRtE5khHTMXk9mjE4OCiEwqOjo8IJRV6OkZERcUwSYllMLIvkGYZhmLcKS10ZhmHeBy4uLlSS12KxKAZtSAKrd6wd5KFjbaJqGavVKgZy7Ha7qnNA+0NDQ7BYLFfuDw4OwuFwYGhoiAcumY5FFlxq9yuViqhvtF8oFFAul8V+pVIRskzaLxaLLQeUAMBms2FwcFAMmNK+1WrF4OCgEFvSfjNRLAll5X2GeVtoJ8+uO+Daav/4+LhhobsWvbJ/HYFys31OIv/6XLWA9apghutKZEZHR+H3+3kCnDEcp6enyGQyDUEO8mJW2tLpdMPikL6+vgY5BsnHaJNFl3xfY4xIoVBQSSypPsiBQHI90kpkrFarqAuynI+EfbJMhjbulzJGol6v68rHZDEsBdDRpu2HDg4OquqAtq7IgjKSxNrt9jZdMcO8PHJ/tJngUj6vtwBIu5BHr08in+cFmoyRODs7a+h7yM8QkvrJn9EGoDocjoa+BwVhaM/xc4QxGsViUTfQVBZakvgyl8s1BC319vbqtqvktpXe+7xQlDEab7rNdZ1xYU7yxLwNLi4uRN86k8mIZ4C8n81mRQCrdiyKEg60GqMlIbjT6YTVam3TlTLvA3ISx2g0img0iv39fezv7yMajSIWi4mE0z09PVAUBeFwGIFAAMFgEOFwGMFgUGyKorT5ihjm9SmVSg0JTff29sSWTCbFZx0OByYmJhAOhzExMYGJiQmEQiEEg0GEQiEoisKJBxhDQ/MLesmtryN9lcWvnMSdMSrJZLJBzEHiV0q6NDAwIBJcT01NCdkrvQ4ODrb5KhjmetRqNWxubmJ1dRXb29tYWVnB6uoqVlZWRLyA1+vF0tISFhcXxevt27d5TJ/pGFKpFP7v//4PDx48aBDV3bt3D/fv38e9e/ewtLTU7p/KvALFYhE/+9nP8OMf/xiHh4f4xje+ge9+97sIBALt/mnMK8JSV+PBMlfjwRJXppPI5/OqfsbDhw/x6NEjFItFAPr9jS9+8Ys8V8a0nXQ6jY2NDZW0lfZpfMhms2FmZgbT09OYmZnB3NycELnyc5LpJCqVChKJhO6Yvyy1bDbmH4lEMD8/D5vN1uYrYZhGjo6OxJyuLLHc3d3F9vY2jo+PAbyItaTyPTk5KTY69nq9bb4ShrkaWu8jr2toJm0dGxtrWNcgl3lub79fUEJkbcw6STvkWEO9eHWTySTWN9IaR4oxJNEBnfP5fBgaGmrTlTJMI6VSScRzaONr5RgpOqa2A2E2m1VrfuV1wHrHnNOE6TQKhYIo+7KslZ4LWoHrwcFBw98YGRkRcX9y7gWPxyPin9xut4id5fUyTKdRLpdVuRS0ZV4vD4M2RlZuC1H7R24byWJKp9PJayYZQ1KtVpHL5a4ltk8kEg1ioVaxf1rJcSAQ4BxXTFuhcvw6ZZ3KtZ7Em8s50y7k/AbpdFrEvCaTSZXIMpPJNOQTp/YMte1J1EoSYhJculwuLtvMO0fOvZlMJsU+SbcpxjuTyTTk8xgZGYGiKA19V0VR4PV6VeJWLtvNOT8/Fz6DcrncsH98fCxcBiQ1o2MSmsnHrSSVfX19QlI2NDQk/AYjIyPieGhoSMjP6JjEZ3RMokrON8Fch+vkR7/u8cnJybUkwy8rGNbLi845VZirIDklCSlLpZJw0JCskiTEpVJJyCplkWWxWESpVBL371ZKM5vNBqvVKoSTdEz3cKvVKu7R9J4sqaRtZGSE7+HMtXldybDe51pxXRl8q3Pj4+MYGBh4R/9DDMMwzGvCUleGYRjm5Tk7OxOdbBoUpWPqaJMglgaUrrPfChpYlWWvNpsNFovlyn098aW8zzCdSisB5qvu53I51Gq1lv8uyV1fRXbZbH9sbIwXXjJvhcvLSxwdHaFUKqFcLotnE03yFYtFlMtlIU8mqbIsYT4+Pka5XBaDxuVyuSHZthaLxQKLxSIEyhaLBcPDw6LMOxwOmM1mOBwOUaccDgdMJhOGh4fFObvdDrPZjOHhYQwMDIgB6P7+/nf0P9j5VKtVVYCQVmhJk+3yollt0n+bzSYkMXKCdL1ztJicF8kyRqJWq7UUK+nJYLUTvsPDw2JxFQVQ6Mn75I1FMoyROD091Q00JemSLLfMZrO6C3+Gh4ebSi0pGEN7np/pjJG4rmyJ3ovH40LwITM6OqortWwmYFIUhZ8pjGGgenKVjIyOM5lMQ7tLG6xxVT3hOsIYiVZ1RHsuHo83JDwAXtSRZvWj2bOFk+AxRqBSqeDw8LBp20rvuZJKpRoWtF4lt9S+53Q6OYiEMRSXl5cNCRLkMWD5PTpHCX+J/v5+jI+PY2xsTPdV3sbGxsR5i8XSpqtmupFisdgQICiP0WrHcbX9a6vVqkr+oZ3X0J5zOp1tulKmW6EEBLLQTz7e29sT/V2z2Yzx8XH4fL4GmR+JzThZGWN0KpWKSvIqi193d3eRTCZVdYKkx6FQSGzyMSewZIzMVdJX+RlB4zx6CeDpWcEwRiOfz+uW/e3tbezs7IixnNHRUd2yT8myeV0S0+nU63VEo1GVfGl1dVUl+iD5UiQSEQKmmzdvwuPxtPnXM+87xWIRn3zyCZaXl/HgwQMsLy+jUqnA6/UKwev9+/dx584dnoc1ENVqFT//+c/xwx/+EJlMBl//+tfx3e9+FzMzM+3+acxLwlJX48AyV+PAElemEzg+PsbW1paqD/Hb3/4WqVQKwIt+sixuXVpawhe+8AWe42Lait44z8rKCp4+fSrWtZnNZgQCAVXfl8Z4JiYmuE/BdAytxi23t7fF53jckjEah4eHYl52b29PJbTc29tTxX4pioJwOIxwONwgbg2FQryOkul4qtUq4vG46v4tr9XZ3d3FxcUFgM/jtvTmYnmdTvdTKBSQy+VEnCxtcpwsrU/MZDI4PT1VfX9gYECsPyQZhyyl1B5zeWI6BW3s61WxGMlkEvl8vuHvaGNfW0nHOKaP6SS0sr1WdaBZ/Lc2kXizuCSOR2I6FWoHaXODaDcSGedyuQapU09PT0OOEJKWyW0kOWaD84UwRkUWV14lr9TGsQ4MDGBsbKypmFV+dvj9fu43MG1FL7dBs/Kul/tD7iO0KuuBQIDzFjPvlGZ5O/SOo9FoQ95NOWdHs7Lt8/kQDAZhMpnadJXM+wjFJFFcEo1l0r5W3iqX7d7eXtFW93g8ou1O+3SeRK7voyzrTeYup/2rJH7a/OWvIqmk45GREZ6zZpoiOyxKpZIon5S7n8os5RgvFAo4PT0VUkuSC5+engqRZav8/BaLBTabDQ6HQzgubDabkFDSsSwdttlsGB0dFfskHbbZbO/lPYl5Oa4rnHwZEXErXkUy3Ow9HktmrkLrWXlZ0ar2PT33gJbrlOPrlneLxcIxFAzDMO8nLHVlGIZhOot2DAAD+oMIr7s/PDzMizOZjoSEldrBZ+1AtLZOybLLo6MjVCoVIcGk/Vb09/fDbrdjaGgIJpMJo6OjMJlMGBoagtVqxcDAgEp6aTabxUC02WzGyMgI+vv7VZJM+W/R32eYNwXVCyrz1xHH0sTM0dERarWakJ1Xq1UcHR2hXq+jUChc+W/39PRgZGREiF6HhoZEPaC6IZd/bX1pdo7qHJ3rVo6OjpDJZFom/qcFuul0ukEu39vb2yB5bSa4HBsbg9Pp5MFVxnBo5cjNZLBUV7SBewCayl5pc7lcDbIMnlBmjMLZ2Zl4jlDQBtUV7flcLodMJqMrJSNhshzcIdcR+ZjqSzc/o5nuQy8YtlUg7JsSwXo8HvT19bXhihnm5ajX6w39ElnkpBWS6QUJmkymhv7J+Pg4RkdHhYSMNvmc1Wpt01UzzPUpFouiDhweHiKXy+Hw8FC1L587ODhQJWAiLBZLg5hP28aSX2mfA0yYToeSLsh9klaB59lsVrdf4nA4dMe1WvVNBgcH23DFDPNqVCqVhvEsvWcJvV7neSKP/TZ7jtArJ2hg3gRy/7pZch35XDqdFkn6CDnQtlW/2ufzwe/381gt81rUajUcHBw0lb5ub2+rEqLJUj9tUkmv14vJyUnuxzKGR06Wra0PreqEXr0IhULcxmAMC0kA9erBy0hfI5EIr8NgDIdesm3aVldXUS6XATSWfVkIws8AxggkEgmV6HVlZQWPHz8W6yL1ZE1U3hmmHdTrdTx+/Bgff/yxEL3m83nY7XZ8+OGHQvL65S9/mcdLDECtVsMvfvEL/OhHP8Lz58/xta99DX/zN3+DhYWFdv805pqw1LXzYZlr58MSV6ad1Go1bG5uir7Aw4cPsbq6ip2dHVxeXsLhcGBmZkbVJ/jggw/g9Xrb/dOZ95Rm4taVlRWxbkAWt2rHaljcynQKlUoFiUTiyrHHgYEB+P1+3TH3+fl52Gy2Nl8Jw6ih+7TevNLz589Va7y0UmJ5jnV2dpbzCzAdD0tbGT20cXnNYvJeRtDXSlBJZYuT4jPtplgsXhkfQblC6HPa5OP9/f0NMdvj4+OqeG45rtvlcvH9k+kYisViQ5yDdl+ORc1msygWi6q/0dfXp4o3JTFlqzw5vE6X6SSuip+4bn6CZjEUzdpCbreb12YxhqRSqTTEkVIMXTabRSqVEvkLqB0lQ7kKSHRGzwyXyyWkZ7R5PB4eZ2HailZI3CrWLhaLNYi49J4NzYSWiqLwPBDzzjg5OUEqlVK189PptOr+nU6nxftaQdHIyAg8Ho+4f7vdbpWIXr6/u91uzsnEvDMoH0ar+7Z8rI2NlmXyrQTEXq+3a9rzx8fHKJfLDbm8tfm+9fa1ucLlHOKtoNzGVqtV5MuX9202GwYHB8W+xWIREkvaHxoagsViETnFaZ9hZE5OTlCpVFAsFlEoFFCpVFAoFFAsFlGtVnF8fKwrZJXrwvHxsZCzyn+jFVTGBwcHX0u8arPZYLfb2SnBNEUuy3Q/Pj4+RrVaRbFYVL0vl+9SqSTKtywYPjo6wunpqe64F0F55OneTWVcLtN0jycRMZV/q9WK0dFR8Z7dbofD4eC2ItOSl5UKX/W54+Pjhtw4MlpB/MuIVlkyzDAMw7xlWOrKMAzDdD/n5+c4OTlRCfha7VNH7zr7raABPYvFAovFIvav6gBSJ1JPyDc8PAyTyQSHw/GO/vcY5uWQJ4jkgXB5/+TkBPV6XQw6lkolFItF1Ot1lfSSBmAKhYJ47zpQfXI4HOjv71dJMEkeK9clWZppMpkwMjICk8kEu90u/pbdble9NzQ09Jb/J5n3Ae3go96AZKtzrd7L5XINi430kAcu9Z5Lr3vOKIP1tVpNtbBdK7bUnsvlcg2THr29vQ3CGO2+VgAwPj7OggzGUOgtemy1+DGTyYiEvIQ2QLCVsI+DBBkjUi6XWwotX6eetKozPHnIGAkKuqWFxdqAW1mg/CaCbuW6w/WEMQKlUqmlAJYkZbTl83mcnJw0/B16njSTvjY7xwHqjBF4maBd2lKpFLTT4le1u/T6Kd2y0J/pblr1S/TqTTabxdnZmepvXKdfordxHWGMxLt4njQb8/J4PIYYO2c6FxJq0pZOpxvmOeT+xMHBQcO93m63i6Bded5CO4/hcrnEMc+RMi/D8fExotEo9vb2EI1GEYvFsL+/L47j8biYz+zt7YWiKAiFQggGgwgEAgiFQvD7/fD7/QgGg1AUBSaTqc1XxTCvDrXTm4lfo9GoKumCnJRYT4bs8/naeDUM8+pUq1Xs7e1hd3cXu7u7qv2dnR0kk0nx2ZGREUxMTKi2yclJTExMIBgMslCIMRz5fF5IMOWE3VtbW2I9sslkQjAY1JUucFJ6ptMh2SsJclZXV/H48WNkMhkAL+7rU1NTDcJXFuQw75rz83Osra3hwYMHWF5exn/9138hGo3CarXizp07uH//Pu7du4cvf/nLPHfawVxcXOCXv/wlvv/972N9fR1f/epX8f3vfx8ffPBBu38acwUsde1cWObaubDElWkHZ2dn2N/fF217Eriur6/j/PwcJpMJMzMzqrb94uIiFhcXed0/887J5/PY2trC5uYmtra2sLGxIfYPDw8BvBC3RiIRzMzMiG16ehozMzMIBoPcL2XaTrlcxs7ODvb29sQmjyFSWe7t7VVJiLWby+Vq85UwjJpUKiXmg2hOSJ4bKpVKAF6UbZ/Pp5oTCofDCIfDmJiYQCgUwsDA/2fvXH4bya77/xUlkhIpUaJeJMWH1G9Nt+2ZcRA7QHeMn4HxJnHWCeJsAgNBAgQw7IWNBElgxwhiIzBgJECAII9d4vwBnt0sDFgNJHBg2B6rpzXdo5ZEkSL1IkWK4pv6LQbnzqnLqmKRkkiqdT5AoYpVJbY0c2897j3nfLwD/msEwZ5cLqfiZCg+hrd5GisHgLm5OdXO9fnQlZUVmQ+6phQKhTYJpVWuHC16LQiv12vIlyMxn54/Nzc3pyQeIm0XBk0+n28TUdrlih4dHbXlirrd7rZcUZLUWAlap6enB/QXC8InnJ2dGeSsZoJWngdK+/Tr/8jIiMrt1O8DXMrK7w0yHiwME53y2PScncPDwzZJWS95OSLiE64rPJ+tG7mxTjfiSsljEwZFqVRSeW929fzovYHGwomRkRHDuzHJKun5yExOLPnNQj9oNBqGGjBHR0eG/E5ac+m2XrNyampKtV3elvl7QCgUUselXpLQL6jm0f7+vuFarX8mMbEuWvR4PIZrNpfJk5CYtsPh8NCO8VB97Fwuh1qtpupok6yyUCigXC6r/bSt1+Dmtblp2w6qjT09PY2JiQn4fD7TuvZUs57Xr9frAet18AWBsKpl3akOdqftQqHQVk9Tx07c12nb7rhIWAXi7OwM1WoVuVxOOQ8KhQKq1SqKxWLH4yRYJdGwLm3txNTUFLxeLwKBgHIgUDt1KlylYyQbludAgXN+fo58Pq/ab7FYRK1WM4iET05OUKvVDG0+n8+jVqspqTCdd3p6qtwgneSrY2Njqp36fD7VTmmbZMPk+eDPM9TWfT6f6gfU9gVBEARhiBGpqyAIgiBclG4HGZ1s5/P5toLAZnSS6/V6jA/2yMCNMGw4lVp2K8Gk9fHxcdvEtxW9SC+7OTY1NSUBIkLPcDEyDbDyQVeSJXO5cq1WUwOtpVLJdEK5VCqhVqshl8s5+j0CgQDcbjemp6dVO5+amlKycpo8pvZOE2LBYBAulwvT09Nq4JYEzfQ9NElBUuZ+YhYg2SlQshdxnwhkhOtMs9lsSzLkSVhmIr+joyPTfqLLx/SFRANcTibJq8J1oF6vmyZn8X1mx82Cs6anpzE7O6tEHGb9w2xbiicJ1wGrZ69OsjKdXuVkEuAsXAd6SeIyS34EekuAlH4iDDvVatX0WYs/c5k9d+nFIgAYRGf6sxWtg8EgZmZmyR/ZbwAAIABJREFUVB+ZmZmRPiIMLa1Wq60fkDhc39b36cGYVEyCS8Np224fJY0IwrBTrVZt7yNW7/h64RWXy2W4Z5gtVsckKFnolePj4zbRq14oS2/T+lgtFY3jzz6UYGknhpVEKMGM8/NzZDIZbG9vY3d31yCAJQlsNptVzxsulwuhUAjxeBxLS0uIx+NK+ppIJLC0tIRYLCbPFMK1pdlsqqLGyWQSOzs7hoKvyWTSUKRkamoKiUQCiUQCsVhM9QW+LTJu4TpSq9Wwu7trKj/e3NzE1taWujdQQSor+fHy8rL0A+HacHBwgM3NTXz00Uf46KOP1Pbm5iZSqZQ6LxwO486dO7h9+zZu3bplWKLRqBReE4YSLjTWxcbAx++ad+7caZNBra6uSpsW+sbm5ibW1taU6PXZs2cYHR3FgwcP8OTJE7zzzjv4f//v/4kcZwhptVp499138bd/+7f4v//7P7zzzjv47ne/i9/6rd8a9K8mWCBS1+FDZK7Dh0hchX7SarXw6tUrrK+vY319He+//z7W19fx/Plz1Go1jI2N4d69e/jUpz6FT33qU3j06BE+/elP486dO/K8LvSVXC5nGCfUF+BjCVA8Hm+TXMo7pjAMFItFg8iSi/62t7cNor9gMIjl5WXcunWrrT2vrKxIHKIwNFSrVaRSqbb5HPq8sbFhKGoaDAZNZcSRSAS3bt2Cz+cb4F8jCPbU63WkUikla6U5fD6nzwvXLywsIJFItIlbb926heXlZcl7HXIajYapiE/Pjdbzo83EZGZySl3Qyo/J/LYwSHrJ37TKSwsGg6a5Z1Z5aSLlEwZNpVJROTJOl3Q6jXw+3/ZdveRlSv0YYRgolUqqffP+oOeU8e2DgwOcnJy0fVcgEMDc3JxBzq3nGujibnknFK4r9AzlVGxslmMG2D8/md1DFhYW+l5rTRAKhYJ6L7aqh3R0dIT9/X2Vv1Yulw3fMT4+rvLQQqGQqbBV3yfvCkI/oGcbvS4eF1jSsYODA9P6m7Ozs23jP6FQyCC0JEnrwsICvF7vAP5S4Saij2NSm+bCbRK0Hh4eolQqGX6ert3UhrlUm4uH6dreL0lrNzWou91XLBbRaDQ6/g58DKBXSSXfnpubk2uDAOATqXA+n0ej0VDi4EqlYhBRcokwl1Kenp4qySoX99F32EF1nycnJzE+Pq7EfePj4wZJMJcKz8zMwOv1KmHl+Pg4JicnMTU1hfHxcSWwlFgHgehWIuz0uNWchU6v8uBOx8UhIAAw1Ozntf2p3j+110KhgFqtZrjGm8lY9e9x0sZ5zX9qt4FAAB6PB4FAwNB2rUSrdM0XwbAgCIJwwxGpqyAIgiAMI61WSw2OkpCv0Wggl8uhXq/j9PRUvWwXCgU0Gg2DhI9euk9OTtBoNAwiP3qxdyqOJdkeCfNmZmbgdrvV4CgfNAoGg3C73ZicnFQv54FAAGNjY6povd/vVxI+EvX1a+JFEJyiyytPTk5Qr9fVRAUJMEl22Wg0UCwWVb/kwky7fukEGrSanp6G2+1WkxQTExOqXwaDQYyOjiIQCKg1TYZQn6T+R32a1vRd1GcFoRvonsTvLbVaDaenp6rN5/N51Ot1w6AwHyjmg8etVkvJIKiv8ES2TlAfmJmZgcvlwszMjOoTnYSwdj9L/Yj6Ta/9pdls2gakUbCDvk+f/BwZGWkLXNZlllaiDAlSE64D+XzeVPjKEyC5RIYWHbfb3VEGa7bMzMwM4K8WhO6oVCpdSWBpu1qttn2XLpHR5WNm2/RZhLDCMHN2dtaWSGmWWGm2zyywMhAImEqV9H1m5wjCMGOVdN8pccwsSLNXcfLS0tIA/nJBcEapVLIVwJpJYo+Pj03vJX6/3/Fzlv5ZEpOFYaXb+wgtmUymbY6q1/uIFLAQrgOnp6fqPsHHgDu9s+jJbwDg8Xhspa927yYTExMD+OuF64zZdd5JsQkd/RpvVZSIH5f3aYGgIt28ECxtp9NpbG1t4ezsTJ1vJfjj29K+hOtKqVQyLRC7u7urCsjywifT09OIxWJIJBKIRqOm21IQVLhuVCoVbG1tYWdnB7u7u6pwMvWJnZ0dw9hlOBxWwuNEIoHl5WXE43G1hMNhme8Shp5KpdIme3316pVa6NrvdruRSCSU5JWKg9MSCoUG/JcIgpHDw0M8e/YMH3zwgVp/8MEH2N3dBQBMTExgdXUVq6urePjwIR48eIAHDx7g/v37EusqXDmZTAY/+9nP8PTp0zaZ3uPHj/HkyRM8fvwYjx49GvSvKjDee+89/PVf/zX+53/+B48fP8a3vvUt/N7v/d6gfy1BQ6Suw4PIXIcHkbgK/aDRaGBnZwebm5tYX1/Hs2fPsL6+jl/84hdqTjISieDRo0d4+PChWn/2s5+VmBWhL1SrVbx69UqNgfBxkM3NTRUD7/P5cOfOHdy5cwe3b99W23fv3sXy8rLEjwgDo1wuG6SWfEmn04ZYKRI00NwlX+7cuSM5TcLQkMlk1NxLMplU8zM0J5PNZtW5fr8fy8vLSCQSSCQSiMfjWF5eVks0GpVrtDDUmF3HeZzK9vY2ms0mgI/j9+bm5tqu43Rtv3fvHgKBwID/IgEw5vTruchmuZf0uVAotH2Xz+dT+fxcwsrFrPoxv98/gL9auOlQ/RuzNm+3mMloPB5PW/49r12hL9QPRFwtDBIupczn8233ABKQ6fcBHoNKkJTSqqaL1Wep5yIMEn4fcCJl5dtmNSgmJycNeSn6Nomb+HPR3NyciCaFa0knwbdZHs3+/r56VySs8iPtcmhEcC/0G16viK87yVr1vPmxsTFTeTcJK3VZ68LCguQuCH3h5OSkLZeXr81ErXr79vv9apyHt2Ua/yFRK32en5/H6OjogP5i4SZhJmg9OjpSMm19/9HREVqtluE7/H4/5ubmEA6HDYJWulZTu19cXMTi4mLX124uKDOTlvF99Xq9re461X4220d1oDtBdWOpjjPVQp+cnFT1Z632Ua11XjPd4/EoySXVXxduLlYySTvRZDeC4W6lwk5FlE62Z2dnJU/jhnN+fq5kwsViUbVNLqmkazEXCZdKJVV7nF/nzY53gtrz9PS0qhdOtcCnp6dVe+Uy4ampKXi9XoOs0uq4IFxUBm933Kl01eq63YtUWOTCgiAIgnCliNRVEARBEG46lzEI7GRQwQmdBgcuemx+fh4ej+eK/4sKgnPMZMskv7STOpMYk09G0jGSyNLa6eQjQROINAlJaxJf0uAcCTCnp6fhcrnUmuRewWAQIyMjSpDJz9PFs4LgBLP7Tjf3KKfrbu5bALq+F3Vam/0sACXDPT4+xsHBgWkAnB5QbTaYPz09bZk8YyeDlUIcwrBzfn7uKKnMbL8eJO1yuTqKX61kSxKMIAw7VlIynmCQz+cNyQT5fN4yEIL6gJ2IzGpbEnKEYaZXOdnBwYFpUFyvgjJJwhGGGRInWz1n5XI5wz2G9heLxbbvosBOs2ctq2V6elptC8Iw0uu9xG5cotf7SSgUkkQgYeioVqtt9xF9bXVMTx5yuVxt41n6fcPuviLzRsKw0+s9xUysCXxyP+kk1dQFsTLuJTilUzELs4IWZu/T8uwjdEMulzOVvtL27u6uoeAiiV+tpK+RSETGZYRrCy82a9YvXr58iZOTE3V+JxGyFA8XriP6faHbosu8L0gfEK4DuVzOUhbBpa9erxfRaLRNFEGLyLuEYeHk5ATPnz/H+vq6Wm9sbGBrawvNZhMulwsrKyt48OABVldXlez1jTfeEHmxcGWQbG9tbQ1Pnz7F2toaKpUKIpGIErw+efIEb7/9trxLDgFra2v4/ve/jx//+MdK7vrlL38ZIyMjg/7VBIjUdRgQmevgEYmrcJXU63W8fPkS6+vr+OCDD9R6Y2MD1WoVLpcLy8vLStz6xhtv4NGjR1hdXZX8LuHKsRrD0MfsgsGg5fjFysqKPHMLA8Fs3JmWjz76yJBvwduwLm8V0Z8wLFSrVaRSKcv5lI2NDZyenqrzqV2bCYkjkQgikYi8dwtDS61Ww+HhoeXcoT6Hbibf5p+Xl5clNmkA6HGUZkIlfelGsGS3RKNRmTMW+g6JKSnX1ypvXhdWmuVsTUxM2MpZdTElLSJbEgZFr7Hzx8fHplJKJ9d+Pa5e6pMJg6bXfpDNZttyroD2fuAkl2Rubg5er3cAf70gXIxWq2XoF1x6pi9cdlYqldq+SxcWc2kliSv145J7JfQbfs9w8q6cy+WQyWSgl6EXIbEwjHR6JjJr84eHh6Z1F2nMz0kbl7EgoV84uYbz/Wbt2+z6bXXtDgQCSpB3dnamaq+enJygVquhWCwa9tXrdRQKBdt9XNZK+zpB9Y2ptnEwGFT7qO6x1T5d1mq2j2ohCzeLQqGARqOBfD6v6nBTze6TkxM0Gg2DGJjqdfN2TcdIyGdWG9wJMzMzql431RPm7Z1qcFOd4ampKbjdbiUVJnnw+Pi4qtlNckqfz6fqegs3i8uqxd3pnFwu5+j36VU+2elcGZe92dC1uVAooNlsIp/Po9lsqut4sVhU1/ZcLqeu9+RiyOfz6lrOpfD8WcZs7FSHi9/Nnk0mJyfVMwe130AgAI/HYxALczE8F8jLc4ogCIIgXCtE6ioIgiAIwtXTbDZRKBTUQB0NUudyOTUoQgMcxWLRMIBtJbA0k2E6eayhgWg+6UID3jS4TWsa0KY1F1z6/X71XbTW5XxcxicIw8BFRJfdDspTv3VKL/LLi6yl6IhgB9236D5E9xxq33TvIekyDc6TVIIP+tNAP/UP6ht0L6QJAydQ+6V7D92jSLJMEuWZmRmMjo7C5/Oh0WjA5XKhXq/j/Pwc5XIZrVYLZ2dn6u+jwIhSqYRCoYDDw0PLQO1uCv0Hg0EsLCyItE8YekiYrCev2SW65XI5R4k9dqJL/bMkuAnDTq8JQEdHR6jVam3f5yQRjgdf8EUKPgjDTK99xaxYANC7xEb6iTCs0Jig2fMV/0xScb6cnZ2Zfic9VzkRwJodE4Rh5KqTr+W9XrjO5PN50/sHbVMBG77QPrN5JJ/PZ3sf0cWw+n1HinMJwwolTlgVrbG7n5glzvl8PsOYltkzFu3T+8n09LTcTwRbzs/PDcUx7N4X9G0z2bdedMzptiT6vN4cHR0hnU5jZ2cHqVQKqVQKyWQS6XQayWTSVPwaj8extLSEeDyOaDSKaDSKRCKBcDiMWCyGUCiEsbGxAf5VgtAbx8fH2N3dNfSHnZ0d7O7uqm0+DjM9PY1YLKb6QjweN2xHo1EZYxGuFfV6HQcHB5aFm2mb4PJjs4Ll8XhcnneFoSaXcyZM0ds6X+7fvy9yH2Hg1Ot1JJNJrK+v49mzZ9jc3MT6+jref/999Sw/PT2Nu3fv4vbt20pSRdsTExMD/guE14l6vY5f/epXeO+995ToNZfLYWpqCp///OeV5PW3f/u3pdDtAHn69Cm+973v4d1338VnPvMZfOMb38BXvvIVGdcfMCJ1HRwicx0cInEVrgL9+ZjWz549U0UEI5EIHj16pJ6NHz58iLfeekti1oUrxW4cYnNzU50XDAYN72203L17V8abhYHA264uuXzx4oVhHjEYDLYJLWlc7cGDB3KdFQZOo9FAJpNR8380L5hMJtU6m82q8ycmJrCysoJEIoF4PI5EIoHl5WUkEgkkEgnEYjGJpxCGmmw2q2JAtre3sbOzo9r69va2QVTi9XpVW4/H44a2H4/Hsby8LGPJV8jp6amj+EU6TrFkXJ5OjI+PG+STupRyfn6+TVA5Nzcn/3+FvjIIMaWVhEkQ+sn5+blqz5TLwRd9n/7ZLCdqamrKslaDXf2G2dlZeZYVBgbPbTo5OVG5UGY5HXpOlJ77NDIyoto0tXMeC6/v49t+v39A/wUE4WJcRb2GTjWz+HHJqxX6jV2btxL9HRwcoNFoGL7H6XuDtHehn9A4j1UdN77w8/R3g7GxsbbxHrvxIZJuBwKBAf3lwk2hWCyq9svbsFW7J7G83sZJ1jg9PY1AIKBEYhMTE/B6vWppNptwuVwYGRlBvV53VCe4m7rAViK+y9g3NzcnMbU3iPPzc+TzeVWb16yGfKPR6KpWfaPRQC6XUzV0ncqEgY/n+XVpqplQler8BgIBuN1uJeXjkmGqmxAIBNpq3As3A2rfvJ4zb8MnJydKSqmfw6XCTs5xIqEEoGrf8DbL2zoJgi9yjkiFbx69CoS7/Zlu67Jf1jOKflzauCAIgiAIGiJ1FQRBEATh9eGi8lgS79GaBv1prQ/6OIUGZ7gA1uv1GgSxfM1Fsm632yCaJQHt6OioWtOAD61nZmZE3CIMBdQnqf9RX9P7nN7HrASx1D/5d9TrdTXB1s0gLPDxxNrIyIjqMyTEnJ6eVn2sk+SZjtPPAVDfR32W+jz1f0EwgyadaYK5GyEs/1krIS2t6Tu7YWJiAqOjo2qimO4x5+fnaLVaaDQaaDQaqNVqppN+Y2Nj8Pv98Pl8KlBkZmYGgUAAc3NzmJ6exsLCAmZnZxEKhTA1NYXp6WmEw2GRkQlDjVVALJ88tFp4MjCn16S6xcVFKRwvDDW9Jk2QgMaMXvtLKBSS4ozC0NJrX+n1vmIlTg4Gg5ifn5dAQWEo6aWf0PPZVfQVua8IwwYFoZsVN7ArfEDbZnChZTAYVIkglBTCFzrOl/Hx8T7/VxAEa+zuI53e582SXIH2+4jdfUNfJOFVGFbK5bJt4TReWIQvtM8Mv99vKoDV91EhHX2/vJ8IVnDZdycBLN82m++fnJxsk72afZ6dnW0TGAuvB5VKBel02lTsR+utrS3DfFgwGFSFmq3Wy8vL8u4oXDvK5XJbP+DbL1++xMnJiTqfFzsyW4v4UrhuWPUB+ry1tWWQHweDQVPhK31eWVmRhE5hKKlWq9ja2sLW1hZevXqlFvp8eHiozo1EIlhZWcGtW7fUsrKygpWVFcTjcXlvEwZKLpdrk70+e/ZMPb+PjY0hkUi0yV4fPXqESCQy6F9feA1oNpt4/vw5nj59irW1NfzkJz9BMpmEz+fD22+/jSdPnuDx48f4whe+IIKqAfDLX/4SP/jBD/Bf//VfWF1dxTe/+U384R/+ocTbDQiRuvYfkbn2H5G4CpdJvV7Hhx9+2Pas++tf/xrValU96/Ln3IcPH+Ltt9+WYv3ClVAsFrG5uWkYR3j16hU++ugjbG5uqkKwPp8Pd+7cUcvt27fV9vLysowVC30nlzMKh/m478bGBk5PT9W5NN6rL5FIBLdu3ZL8UGGgNJtNZLNZbG9vI5VKKWHr7u4uUqmUElhSnrXL5UI4HG6TVi4vLyuB6/z8/ID/KkGwJpfLmc5X0/bOzk7bNdxqvu727dsyZ3cJnJ2dOZay6ku9Xm/7Pi7p44su39CFHHI/FvoBF1Oa5WN0ytHoVkxp91nElEK/oXpcTts7/2wVS27Wzp32BZnTEQaBkzxWqzwkp/lHUktEeN05OTlx/L6gHzODnovs+ot+fG5uDlNTU33+y4WbTKFQaHtOIrHf4eGhQfLH5X+1Ws3wPaOjo+p92Go9Pz9veG+em5vDxMTEgP5y4SbQi3z48PDQdEyIPxd1km5LXQ+hX5yenmJ3dxfZbBbpdBqHh4fY29tDPp/HwcGBerYpFAooFos4PT3F2dlZW+3bkZEReDwejI6OYmxsDCMjIxgZGcH5+TmazSZarRaq1arjmrlUx5bq4FKdXJLuBQIBVReXhGRU15rqVXNBH9XSpXOnpqaUpFJ4/elVxtervM8JXLTXq6Cv0zGp0X4zIIFwL6LVbs/pRrRKHgE7ieplnCO8/lBNcau65DQ3xfsB/Qxdo7v5GSdQLX56vqC6+9Q+6fmFnkmCwaD6Garn3+ln+HOMIAiCIAhCnxGpqyAIgiAIwkW4yokIs3U+nzeVT1jRadLhMtc06CUIg4YmOai/UKBeLpfD+fm5EmDShAkXz+oCWivxLK27GWwmaOCYRLIAlJSZ+pEugnXyM9QfaZKFJtrNfkYQAOt7mN2xTmsKNKFgE5rwqVQqqNfrqNfrXcmXgY8DVFwuF1wuF9xuN8bGxlT/mJiYUAEkfr9fTcoEAgHMzMyo493e0yggRRCuilqt1lUCky7M0KFrvZkYgwsyzLYpoUkCToRhhcSudlIyq75TLBbbvs/lcrUl9tkl/XFpWSAQkCJXwlBC9xU9aenk5EQtXLLE952cnFi+00xNTZkK+3T5ktU5gUCgz/8lBKEzg5DC2gn+JHlEGDacvJ+cnJygUCgY7im0mEFJHHbiV37vMDtH3tGFYYDGlqk/dLuYvZ8AMBVbmj1r0XjXzMyM4R1FEm2FYeSqBMrdSJP1ogwiGRd0enk3yOVy2N/fN53n6LXIjrTP60e1WsXR0ZFB9KqvNzc3DcVkPB4P5ubmbIWXkUgEkUhExuqFa8Xx8THS6TSSyST29vZUseh0Oo3d3V3s7e0hm82q810uF0KhEGKxGCKRCOLxuFovLS1haWkJsVhMxhWFa0Gr1UImk8H29jZ2d3eRTCaxvb2NnZ0dJJNJJJNJ7O/vq/MnJiYMBdMTiQRisRiWlpaQSCQQiURE5iMMJSS958ILWnTBty674EXS79+/L8XxhIFQq9Xw4sWLNgEWl7VQ29UlWKurqzKHI1yIzc1NrK2tKdHrs2fPMDo6igcPHuDJkyd455138MUvflHEKX1kfX0d3//+9/GjH/0I8XgcX/va1/Cnf/qn8Hq9g/7VbhQide0fInPtHyJxFS4D/uxKz63r6+vY2NhAs9mE2+1GPB5Xz618LXPGwmVSq9Wwvb3dJm2l5fDwUJ0biURw+/Zt3Lp1yyBtvXPnDsLh8AD/CuGmcXR0pMSWND67u7urxm9TqZQqiO/xeJBIJJTUcnl5GSsrK1hZWcHy8jKi0agUuxQGSi6XMxVX0nYymTQUwO8ksIzH45L3Lwwl5+fnyGQy6jq9s7ODVCqltvXrNwDMz88jGo2q+TbajsViiMViSCQSEoPjkLOzM1M5ZSfBUi6XQ7Vabfs+v9/vSKZkdlzuu8JVUyqVVP4cXzvJ6bbKj3AqoxQxpTBIeIxsp9jty86hEyGlMCx02+57iRfvJr9hbm5O5iWFa0mveRdOJH5yLxGGlV7b/dHRUZucFbBv91YyS6k/IFwlVm3cSszaKefTri2bLeFwWOpgCo7htWEbjYYSkdFnLt+jmrInJyc4Pj7GyckJisWiqqtULBZRq9VwenqKRqOhxKokWnWKy+WC1+uF2+1Wec8TExMq539+fl7VN+tFuqoLzoTXE96G9brJVnWSddkevc9SDdhSqaTaOO8nJO5zArVZaofUdqk98jY/Pj6u2jLVQya5JNU8ptqWwWBQCf/oZ6n9C68n/a7VTzWPneCkHjEf9+n1HKntejPoRnTdixxbl7E6oRevhJM2r6+lRoYgCIIgCDcAkboKgiAIgiBcN2jCxU5YqYsr9YmZTiJLXWjpFJps0Sdd9LW+j/+M1+tVx2jixUxKSRM4dK4gDJqLCDEv+jPUz7uh0yC5k3N6+Rnq58LNhu5ddM8qFAoolUrIZDLI5/M4ODjAycmJQRZTLBZRKpVwenqqggiq1Sqq1appQCMAFbw1MjLStUyWAgrMBMlU0IjuSRQYQMEE+j2KvoP/DPUFutfx7xVRumBHq9WyTBbki55kSNtUlFKHy2E6iWApmZCLY6XNCsPKZSeKAOZB83aJV/oxSRoRhpGLSJeOj48tg9q67S8ithGGmWq12vasRYvZ85h+brlcbvtOCkLmkj59oeetQCCAqakptZ+exSggWhCGAf1+4aT4Az/HKukd6Jywa3d/mZ+fFzGsMHAoKczJvUO/j1DimBlut1vdJ+hdnUtfdWEyv4/QWqQqwrBh9TzVaR8tZqFwExMTbRJx/pzF+4VVf5IESAH4eHyWihfq13S9sJvZ2iyB3efzqTFXu7XZvunp6QH8VxCcUC6XbcWvJMLk93hezIAX2+XreDwuwkvhWlGr1XB4eNgmPu7UF8zkx7xfJBIJGQ8Rhp5KpWKQCJD0lSSwOzs7hri8iYkJJT1OJBJYWlpCNBpV8td4PI5wOCxFC4Sh4uDgANvb24Zla2tLbefzeXXu4uKiQZZBwgzalmdbod+k0+k22evm5iZevXqF8/PzNmkWyV4/85nPyDO50BOZTAY/+9nP8PTp0zbh3+PHj/HkyRM8fvwYjx49GvSv+trz6tUr/PCHP8S//Mu/IBQK4Rvf+Ab+5E/+RPIR+oRIXa8ekblePSJxFS5CtVrFy5cv2+Stz58/R6vVspS3Pnr0SGLphEsjl/tYGGi2bG9vq9gdGqslSSBfHjx4oPJgBOEqOT09VeOsfGyVPu/s7ODs7EydPz8/j1gshng8jkQiobZJ3BqJRKRQuDAQWq0WMpmMqbSS2nQ6nTbksITDYcRiMUSjUdWeSVxJcwcSFykMK/S8ocdK0L7t7W3DPFkwGDSdF6btRCIhcY4avebLUey2Tq9iJZGTCVeNVVt3kqfQbb6byMSEYaHRaJjGSlNegd021eYwqz1DeQcUe0qx0nwf39YlxTJHKvSbbp93+L2hWzGxk5xnyU0TriuUz8ZrOOmie7PnKNo2y3+YmprC7OxsxxoBZvtlbE7oB5VKxTLPxyzXh28XCgXT7+TtWM/t6bRP6jEJlw3J/zrlKdM1nZajoyMcHx+3fZ/H48Hs7KzlMjc3Z1jTIuN1N4t+1FXl63K5jEKh4Li26sjICEZGRizPd7lc8Hg88Hq9qhby1NSUem6hd2Fq3/Pz8wiHw5icnDSIzqSG6uuJLg+met1cuMrXvK63XrdbF66SaFIXrjqB6p66XC5MT0+rWqZUt9RKuErtVBeuUq1UM+Eq/RvC64Vef75f626guNbLXOtCbeoTVCdYeP246ueSi4qEARieJ65Stiqx4oIgCIIgCJeKSF0FQRAEQRCEzuiTRPpkEZ8AtRts1CeV9MklWtPx6OrJAAAgAElEQVT+biD5HvDJgDpNHNEgOk04cbkeTSbRADsNRtIAPJ9gonNpYkqXzwrCIKF+Q/2O+hif3CKRJk160c9Qf6V+SBPG3fxMN/A+qPdXXYZJ/cusL+oCTeq3/Bzq0/znZULt9cSsgLrZ9uHhIfL5PI6Pj1EoFHBycmKahAhABeDQPWRsbAwejwcjIyPwer1oNptwu92o1+sYHR1FtVrFyMgIKpUKWq0WKpUKqtVqT/0E+ESS3KsUlu57vP1TX6O+4eTnhdcLPZmL+oguf7XapvuDjt/vN5XA8iQuXURGggzaLwFrwrBB7Z7uF3zN+4TZcbvkRwAGcYy+1sV++nFKlpSkFWGY4OMA3SRH0nJwcGCa2AU4S5S3SppcWFiQRBdhaOBSWKulUCi0LXy/1XiZz+drE8GaCWL5cxkt9HwmQXnCMEBjUjxB2Gzhz1v6wov0cfx+v+Edhb+r6O8vU1NTmJqaMnwOBAJSzFsYON2Ik82OHR4eWib66M9SnYpS6MfluUsYJsyKGOnv+LTP7J3eKmFobGysbaxLf2fXhbD6sZmZmT7/1xCGjWKxaCqDdbI2m2eg5Em7ghBWBbeobQqDhWSXu7u7yGQyhvXe3h5SqRSy2awqoA5AiV+j0SgikYhax2IxhEIhLC0tIRQKSYF/4VpBfSGZTKo+kUqlkE6nVX/Y399X54+OjiIUCiEWiyEcDiMajZquQ6EQRkZGBviXCYI9ugTcrLg1F2kAnYtax2IxKaYgDA2VSgXpdNogh+HtfWtrS82nWoliqI3funVLrulCXzg5OcHGxgaeP3+O58+f48MPP8QHH3yAly9fqnmapaUlrK6u4sGDB1hdXVXbiURC2qngGBICrq2t4enTp1hbW0OlUkEkElGC1ydPnuDtt9+W+JArYmdnBz/4wQ/wr//6r5iamsKf/dmf4etf/7o8S10xInW9OkTmenWIxFXohb29PfVM+cEHH+D58+fY2NjAzs4OAGBiYgJvvPEG3njjDTx69Eitb9++jdHR0QH/9sJ1x+x9nJaNjQ015+TxeBCLxSzfxW/fvj3gv0R43anVamp+wGzsKJ1OGyQsfPzIbIz03r17Mv8pDIRqtYqjoyPLcf7NzU0kk0lD3Fansf7l5WXJWReGkkajgWw2i2QyiVQqpQTbtE3zvbqgOBqNKkFxNBpVgmLaf9PiG87Pz23zoHX5hr6PC3EJKu6uxw3pC9/PY0Jv2v8DoX90I+TTY5/39/cNc+WEXszaycLPD4fDMu4uXCnUjp3mWup9IJvNmuYl8zbfTR+gcyORiMxlCn3BLkeSx+vzhSR6dvVf+POOng/Gn3esjsl8iXBd6eW+ws91cl/pZhHBsdAvupV78/ZP9fJ0em33i4uLUqtOuHTsZKxO6lGYwesV0fOPmaBVl7PKWPT1Qa/vq9f81esE81q/fM1r/p6enhpklsVi8UJyPqqHNT09jZGREfh8Ppyfn2N8fFzVa2w0GhgZGUG9Xker1VI1GqleY6lUMq0/5PF4VE4kte+FhYWOsnm5jl8P+iUM1r8jn8+jGz1Lr1K+btezs7Mybv+acdlCyasQTl51u+Zr2hZeD/RnE3oW0Z81qKYUzZUCn9Q7p3qd9AxCzyb0DENtmv4t+je6geow63XMqa4ztU+qwUz1zcl3QDWhddk2nafLtaXuuSAIgiAIwrVHpK6CIAiCIAjCcHORiTQn59qdQxPO3eJ0IuGyz5WJCWHQXGTCu5ufMTuXJl+6xa6Pme27qvNlwmXw2AVzdgpuPj4+tpy4Ngvo9Hg8Sqbq9/vh8/kM8liSqlIAktO+YneMn9NtEAn/W3q9V13Gz5OsWRgsNBFuJsawksDSvmKxiEKhYCtS5oILLk/SRWT6fhJmyPVUGDaciC6dHDfDSQKy3XEJ3hOGDSs5n35PMTsvn8+jXC6bfi8XV9L9Y2ZmBpOTk2376DMtdG+ZmpqS/iIMDWb3jG6kfkdHR5ZyWD2AvJv7Ch0LhUJSXFIYON1ILrtJ2gQ6F8Jw0n9EfikMEl7ogq/5M5fZcXoGKxQKlvMmuryStikJlO+jZyyzZzBBGAZ4m9eFrzwB2uwcKipjNWeh9we+HQwGLWWxfLxMimDcTOr1uiMhrL6P3pvNrt+UOMQLHOnyV32fXuhIxmP7Qy6Xayv+qxe11ovuUGFrKgAciUSUEJbvk0KFwnWhWq0inU4jlUoZhK/ZbFatk8mkoYCt2+3G4uKikryS/Jivw+EwFhcXZR5SGFq42MBK/rq9vW1o+2ZyA70gvFz/hWGgWq0ilUq1CTto4VJjr9eLaDRqKpi5ffs2lpeXZWxauHLS6TSePXuGzc1NrK+vq+3NzU0AnwiRHj58qIRcfBEEO+r1On71q1/hvffew9raGtbW1pDP5xEIBPC5z30O77zzDh4/fozf/M3fhNfrHfSv+1qxv7+Pf/7nf8YPf/hDjI2N4c///M/xta99TQo7XxEidb18ROZ6+YjEVXBKo9HA5uYmnj17ho2NDSVw3djYUEXHgsEgHjx4gIcPH6r1w4cPsbKyIu/lQs+cnp5ie3sbr169wvb2Nra2tvDq1Su1UNyLy+XC0tISbt26hVu3buH27dtq+9atW4hGozIuKlwpuVzOdNyHPm9tbam5LY/Hg7m5OcN4j5nwUhD6Dclad3Z21DzV7u4u0uk0kskk0uk0jo6O1PlutxvhcBjxeBxLS0tKXrm0tGTYJ3HpwjBSLpdt56OoL/Bi+jdVUFypVAzxljyXkuLIeI6lHu9zcnJi+r26eMNMxGq1b3p6us//FYTXnXK5bCnb4zGTelvneWBmefck5LOKTTNb67FtEocvXBV2ucBO9zvJ2eolF1gEekI/ODs760nCqt8bzKDrP4+N12tE8Lh6s5hled4RritOchmtzun1vtKp/sTc3JzEHQhXTq1WM7wz6LkuZrkvfJ9ZTtbk5KR6H+brTvuCwSB8Pt8A/isIrzMXyWM/ODgwFVY6zVc3WyRvvX+QYIzeIUmUygWoVN+Pi1IbjYZBpFqv102Fq1xapsvLnEK1DUlApovI9DXlF3IZ68zMDMrlMtxuN87OzjA6OopSqYTz83ODnI3qf5ZKpbbac2a5jJOTk5YSVjs56+zsrMRYXCF6e9ble0C7dE+X7emSvW6+oxuorepCPrpmkoiP+gG1awAqns7qO0jCRzVA6Tj1F+F6wq/P1AatruW8vVI7pTZO93hq22bfR9dtvS85Rb8e69flq1oLrw+XJQXu5rt6uZb3Sx7M/y2pYywIgiAIgiD0gEhdBUEQBEEQBMEOmgghgRkAVYiZJk1o8tBuwkafYOETNjT5TpPsNDhNE+/dQhMwNCFIg8g0ucgnT+hcmkykiUiagAc+mYCkCUWacKTv499DPy8Ig6JX8eVln3/dJbMSQNA9pVLJUkBmJSjTJWVmjI+Pt8kseYICSch4ooIuwKRrNIfuOXQP4/clar/UpnlQAN2zKMiA7mv8PqkHK9j9fLdQkAu1VQoMAz65F9F9yuPxwO/3G+571Ob5PjrfbB8PyDHbJ/TORZLanMiUuxWR6fslWFUYNnoN6KbjutCB00nO1ymhR4p+C8MCT+7Rn8EKhQKKxSKKxaLaR59JOk4CcitJGQXc0vMYF4/RMxcXxdI+fh4V2ZAAL2HQnJ6edkym5n3DKgHbDKdJ1tQnuKRsamoKk5OTmJycVM/5gjAo9Gcss+euTvusEuuA7t9brPZJMWZhEJTLZdPiY1ZCWF0YWygUcHZ2Zvn9ZqLXXvbRmKcgDAq74k5O3uu7EY33KhtfXFwUIecNwkmbtGuf+/v7pnNfds81TgrFyNjS5VGtVpHNZpFKpbC/v69Er+l0GplMBplMBul0Gvv7+4bxDxrjW1paUvLLUCiEpaUlhEIhRCIRRCIRuWYI1wa92LDZOp1Oq7lNghcctlqLMFAYZswE4LowgT9fer1ezM7O2spf4/G4zJkKA6VWqyGZTGJ7e1stW1tbant3d1c913g8HsTjccTjcSQSCSQSCcPn17VwvDAcHB0dYWNjAxsbG/jwww/x4sULtaZ4wmAwiPv37+P+/ft48OAB7t27h/v37+PevXvw+/0D/guEYaTZbOL58+d4+vQp3nvvPfzkJz/BwcEBfD4f3n77bTx58gSPHz/GF77wBYlluySOjo7wT//0T/jHf/xHNBoN/PEf/zH+4i/+AuFweNC/2muFSF0vD5G5Xh4icRU6Ua1W8fLlSzx79gzr6+t49uyZkrmWy2UAHz/vPXz4EI8ePVJrEmhKrJjQLScnJ+odmBb+bnx4eKjOnZ+fx8rKikHWSsvy8rIU5heuDD4eqctaNzc3TaV/uuyPfxbZtdBv6vU69vf3sbOzg729Pezu7mJ3d1dJK2kf3esBwO/3I5FIGGStkUjEsC8UCklbFoaOSqWi5kipnSeTSWQyGSUpTqfThvY+MTGh2ng8Hkc4HFaC4qWlJbV9HeVyZvGPFEfP4xz5tp6bYpXTaCciMxOx8s/BYFCuH8KlQLneZjkgeryvXU6JVf633+835IpQWyfxqpmUVd8WhMtGlxDn8/m23Chq4/wzHae2b1UbxO/3G+o68JwpnhvF60DobV/mA4Wrxkm+01XUbeiUi07HQqGQxN0J1xLeV3qJw89kMjAr9cz7jNM4fIm/F/pNr7lQnfKhusk74YvIiIXLppvnJ305PDw0rUnSS34VLfPz89dyrG0QXIZ8rJefAT6pndcN/ZKW0Zrqydbr9bYaiDzXmwtYzdZ213Gz8R8a6+TjomZtXXITzLGrB2klQqXaxtROLyJk7QYrmSrVe6Qaxt3IVHUhq913CNcLap9UV7vbmt66JFuv082/T5ew0vd1C7U1aqfUPs1EwXotb7oW623Y6Vq4nlCb4/Xj6bpL13e7er3UT+xqAZvVuqc2323da7198vq7+vWaalJbSbCpzdP1W78X6AJ5QRAEQRAEQbhmiNRVEARBEARBEK4LVxUk0M253dKreNJs30W+gwb/BaGfmE2Q0cQan3SjCTHqa5d9fi+BQAAMkmaaYKMJNC7SpMm2bs/n++g8AEpyRZNyAJSk+nXHTP7KE4f4Zz1pqFPCEEmRdJmSLlIimZJVMtFl42SSmU8kU/vXZelOf576iNm+XqD7C58sttvH273ZPmrrZvu4cN1u302jGzms2bGjoyPLwLJek4z4filyLwwTF+0vnaRkZv3CiVCGfxahsjAsDErk1+0+SbwTBo2TpNRO9xp6xjfDrK90e28R8aUwaLhInBf94HLxYrFoeN/XReSFQsHyvdXj8XSUiZvt44Jyv98vfUQYCJ3uI93ss6Lbe4jZvtnZWYyPj/fxv4wgfEKtVjMIknkRKbq35PP5tvuKfq+h+QIzzATJwWDQcK9wcq+R9/nXn1arhXw+rxaeqG21rZ9v9p48OjpqSNjW5yb484t+jO+XAh3d4UR6mcvlsLe3Z/g5kl4Gg0FL8WU0GlVJfIIwzJyenmJ3dxeZTAapVMqw3t3dRTabxe7uriGB2+12IxQKIRaLqfXi4iIikQjC4bBBkCz3RmEYKRaLSCaTSKVSSKfTSCaTqoA37ctkMur80dFR1daXlpYQj8dVYXoq4B2NRtWctSD0m2azib29PSW42dnZQTKZVCLYZDKpYqqAj59lYrEYlpeXDdLX5eVl1b7l+i1cNrlcziD+2tzcxPr6OjY2NlScF4ltuPyLPlMsrCAAwObmJtbW1vD06VOsra3h2bNnGB0dxVtvvYXHjx/jyZMn+OIXv4j5+flB/6rXmmKxiP/4j//A9773PRSLRXz1q1/FN7/5TUSj0UH/aq8FInW9OCJzvTgicRWs0J/daHtrawutVgtutxvxeNzw3Pbw4UO8+eab8m4sdAWN0dM7Al9onJ6wEmHevn0bd+/elYJ3wqVTr9eRyWSQTCaV1DKVSinRZSqVwu7urkH0QqK/WCymxlxisZgadwmHw5JTIfSNWq2Gw8PDjnOg29vbhhw8mgflc5/8+ktrQRg2uGSby7X5vmw2a4jV5u3drJ1HIhFEIhGMjIwM8C8zh9c9uIhYxoxupEpmxyUfSrgM7Nq40xzA/f19yzxzJzG0dsdEMCNcNuVy2ZBPQfGyvIYCz6ngx3jNBav8PZ/PZ4g55LGxVvGJ/DjVW5DcPeEqoNoYFOd9enqq2jZ9pr6hn0N1RWgpl8um/4bX6zWtH2JWc8Sqb0gfEK4rJPzm+Xk8t4/nXPDz9Bh4s1pNHo9H5VboAjMzob0uO5N+JVw11J7pvsKfnfizlb6mmld2uaxUe0pf83ZudYz6iiBcBvQsxZ+J9BwmvpDMki9m7xHj4+Mqt8lqobZsttyEd+ZB1D2lbbuaFFZ0U8f0ss+96D2fxnq6EQ7z853I5Z2MhfLldcy1dtIu7Y5d9OeBq5UFd3NuP9q10H8u45rs5Byzc7sVTQK4suu0k3NoW7ge9CpQNZNn2wlU9bq3JM4Geq/xTLWWqQ4tCYOBT2o4U01ZqsfMxau6IJvO1UXZunCV/zuCIAiCIAiCIDhCpK6CIAiCIAiCIDiDJg1osoIm0IBPJjJoIoJPYNAEhJlAj09K0EQHnyChCQ6a/OC/Ry/ooslOwj0zaR4JKblwkiY2uNSSJktoMoNPhHCBpSD0k14DQC77/F4CpDh2k+Wdjl/FzwxTny6VSoakJJ7EpC96cCLfb4VVYpJe5H9yclKdOzk5qfaRSHaYCzJcRfu/6L5euEw5+kX2XYdAFUpi0hMueL+gPsUTnXhi4EX6jVWiH+3z+/1ScEcYGvh9hdanp6cdkwdPT0+Ry+VUIqFVkiDw8TM4LdQX6PPU1BRmZmYM55A8hj5Tgsfk5OTQ3J+FmwsX+dkJlzpJmKyeCeid1onEj8RM1F/4vmF6nhVuHo1Gw9A3SqUSTk9PDfcZun9Y3VtOT09RKpVsn8kouDQYDMLv99veR+zuPSK/FAaJE7llJxFmp0I9PPnKiTDZ6vPc3JzcW4S+Ua1W25616N5B++hzp31Wcw/j4+PqPsGfs8z20Xs8bfv9fsO9hOYQBKGf6Mnq3UphuYDc7j5iJRDXpbD8uYo/Z1FfGubxY+Fi0HO7lQiWi+7Nxmvt3pE9Ho+hwBQvLsXHZK3202d5hjFSKpVUkde9vT3s7e0hm80ilUphf39frbPZrOE+Ojk5iWg0isXFRUSjUYRCISW8XFxcRDgcRigUEvGlcC0oFosGySuXv6bTadUXKPaFWFhYQCgUUsLXUCikhK/UP0iSLAjDhFnBe734987OjortAj4p7mIl/B72AuDC602lUkE6nTa0Yy7K2d7eRqlUUudbyXLo88rKihSjES6Fer2OZDJpEL2SPOzVq1c4Pz/H2NgYEomEQfJK8jBpiwIA7O3tYW1tTYleuZSQJK9f+tKXcOvWrUH/qteSUqmEf/u3f8M//MM/4ODgAL//+7+Pv/mbv8Hdu3cH/atda0Tq2jsic+0dkbgKnEajgZ2dHcMz2Pr6Ot5//32V7zQzM4M7d+4YnsEePnyI1dVVjI6ODvgvEK4DuVzO9D10c3MTH330kSHGid5D9ffP27dv4/79+yIMFi6Vk5MTJWXd29tT4tZkMol0Oo10Om0o8DwyMqLGuWOxGKLRqJK1JhIJtS3za0I/KJfL6rrKFz52nclkcHR0ZPg5mq+JRqMIh8NYWlpSa2rbkUjkRhTbF64Xh4eHyGazqo2TYJtfw7PZrEE+QXM10WgUkUgE8Xgc4XAY8XgckUhEzd0P6pn2okLW4+Njg1ScI0JWYZBQ4W/KdaB4Oy4Po7hWfR+P4bMTU+qSPYrR0/NWrcR909PTErsqXBpm13O7/AWrfXbXdaBdMmN3jTfbL/kMwlVQrVYtc94oP4EWPa9BP4fHKeh0ynvT6xPw6z2/N0gfEK4jVvkO+j79uYqOUx6E1XMV5TxQbjXPeeAiVi7o0+WsVE9FEK6CTs9YnZ63jo6OUKvVTL+7W4mfflzenYXLopOo0q7tO82TdiKnNGvzwxp/QbX6zERjlOtH/2147U9dbAa0S8/oO+kc/t2Ub2t3b7WC8tWpvoOZiIzqe+rn8rqgVMeTzqVrE9X/NDt3WHITO7VjOzGr07FQp+Ofw3QtNxPs0X+TyxTslUol1Go12/bfLdQOqdYsb9fU7qg987ZJdWvp/w2vc0v9gWr+8Hq3dE3S5X3C8ELtj9otb9PUJnl7tauzbPYdZvWbzb6D2no3UBs0u75aXbfN2iu1d/37eLvXr+28bqVwPbhswXW3P9/rdbwb2W+vx+zOuQ71WAVBEARBEARBMCBSV0EQBEEQBEEQrieDlsqa/fvdMmipLP/3+c8KQr8wmyjl270cv+jP9DpRSwxSLmt1nK4H3WAme6XC6VxkSQuJLnlyiZ1QiQe7W4kwuAyWgtz5Pkoqed2L49C9iAeIme2jYDOzffw+5nQf3e8uS6xO9yqz+xffR8Fj/L7Fj9P9EvikbVNQJmAUp9P383sr7xuX3X76ETh8EakS356fn5eiE8LAsesznRJ7+efDw0Pbe3c3fcbunMXFRQluFQaKk0SsTv3JLhkLgOk9w+5+YvdZ7jXCoHBSIMLpvaeT3K/bZzOrc+i5VRD6hS7qI0lyqVRSYuV8Pq8+U2EhOsY/W0nVgI/7iZXc0kx2SZ9JvkyFKGZmZuD3+6WghNAXqM33IoSlfZ1E4zTm4aTd+/1+JcOkY/yz3+9XYx6C0C/M+kk3UthCoaDuKVbwe0ggEFD9hQoO0WcaX6bPk5OTShTL+5Pw+nER2T3tt5vfvWhBuNnZ2RuXXNdoNLC/v28Qv5IMlsSve3t7yGQyao6dmJ+fx+LiIhYXF7G0tGQQYS4sLCgZ7MLCgow1CENNpVLB8fGxKhyey+XaiolTgWU+pu31ejE7O2srxAwGg1heXlbzYYIwDGQyGXWdJ+kxrff399VnPiY9MTFhELzSwtt8OBwWkZLQd6xkO/R5a2sLrVYLwMfXbSqIbyZ9vXfvnryrCxemWq3i5cuXSvJKwrFf//rXqjAPtUUuer19+zYePXqESCQy4L9AGBQkLSTJ609/+lNUq1VEIhE8efJEiV7ffvttmZvqglqthv/+7//Gd7/7XWxvb+MP/uAP8Jd/+ZdYXV0d9K92LRGpa/eIzLV7ROIqAB/HWr948cIgb6VtmueORCLqWYo/V926davr2Hvh5tBsNpFOp7Gzs4Pt7W0kk0lsb29je3sbW1tb2NraUrH3o6OjWFpawvLyMm7duoXl5WW1rKysIJFISCyEcGno4xv6WhcK09g0jW+YjU2vrKyoPA5BuCpoPtes3erCVh7PGQwGLedUaFvmVoRhJJfLmbZx3g+SyaTKewc+zqubm5sznUuka3gsFlO54leBEyGrXZzGZQlZzc6RXB+hV0jGp8e36TnSXCBGMdR8n52Uj2KoadFzp/k+LuTT5XzyjipclFwuZ5BNchExLXQO9QGqK8D32cVHe71eVQ+A4j6prVNcJy10Dq8rQHUEAoHAwAUzwutFryJiq3OscJp72ekcyckUriuXkf9sVy/ALJezm352U+O7hf5BOTL0zETvDPRMRWuqzcT38W0zxsbGDGJhembS1yQiNjsm8WzCZcDF27Sdz+eVZFuvUUZtW+8PVgSDQfWeTAuXcPN91ObpvZva+2XNaVCtLV7HkmR8vD4l1SEwE0vqslUu5aNal2aySbN/p1toLMFMEqnL9qjupJ2Yj2p96eeaiVp5XbDrCv1/522X51FS+6Ztq7UZExMTqu3q62Aw2CaZ19e9znlQu+LtkNqt2b6rEKjyGnbd0C+BKvUbasO8np3EaA0nvL3pIlSz67fZNdruO3oRsnaLXZ1hXheY2iC1a97m6Tt4m9WvyXYSVpEFDy/8WkzXYN62ee1cuuby+thmNUmp3fLnDHouMbu+07/b63OJ2bWU2iXVFeXXd12ebXZ9p/Zsdn2nPmJ2fRcEQRAEQRAEQegBkboKgiAIgiAIgiBcJt1KJC8iquy07yLyvV7lkU6P9/pdr0PQjjD88ElrPkFNk9F84pmCLYBPJrV5kAVNVvPJcbN+yiesKRDJ6vfoBd53aGKbB21QYAWfpKb+B5jLM3nf5CJNHqTh8XhwdnaGSqUCl8uFk5MTVCoVNJtNlEollMtlgxSGB2Hy5Ee7oMyJiQmVuEUBmjypkWQxtI8CMUkgQwlgPp9P/T2CPbw92snOne7jfYICQXif4T9DfYoHPV0ksAmApeScAjyAT/pNJ8EsD+bgfaiTYJbLbIGPrzcUZEsyGAri559JskQJk/wzvz6ZYSaLoaTIbuUxN0GwLAw3l5VY6UTo0W0Spdm+ubk5KcQlDAz+3EVFJkqlEs7OzlRizNnZmeF+c3Z2phLJ6BglHzQaDct/y+fzwefzqXuF3++Hz+dTCQM+n0/dd3w+n+G+Qp/5PUkKKwn9hp5JzYpV6EmW9P5CxSuoyAsv6GI1RkPPo7qwjPcT6hfUh2ibnst8Pp96B6L+Iwj9gp6nuhUq88+0vb+/j2azaflvdSsZF6GyMEg6vad000fsitoBvcnG+WcpBCMMisu4h/T6Pt/tZ9on42CvFxeVwzot0iVyWCN6YWar7XQ63VYIcHx83LQgsy7DjMVicj8Thpbz83Ps7+9jf39fyY8zmQz29vawv7+vRMjpdLptjnR+fh6hUAjhcBiRSASLi4tqzcXIi4uLcr8ShoZyudyxGL9enJwLJcyK8tM6FApJWxf6QqVSwc7ODpLJpJL17OzsYHd3V0l8eEzPwsICYrEYYrEY4vE4otEo4vG4Yft1fM4T+gNJenQ52QcffKBiZ4LBoJK8ckHZgwcPZL7thnF2doaf//znePr0KdbW1rC2toZ8Po9AIIDPfe5zeOedd/D48WN87nOfk3coB9TrdfzoRz/C3//93+PDDz/E7/zO7+Db3/42fuM3fmPQv9q1QqSuzsDrpV0AACAASURBVBGZq3NE4npzaTab2NrawsbGBp4/f44PP/wQH374ITY2NpBOpwF8/I55//59rK6uYnV1FW+88QYePHiA1dVVFS8sCJxisYidnR1sbW0hmUyq98GtrS3s7OwgnU6rmDm3241YLIZEImEQt66srGB5eRnxeFxENMKFsRpf4wLXnZ0dQyynleySy/8ikYgUrhSuFCdjw3t7e21z/Xbtl9ZyfRWGjVqthsPDw46CYv16TTECneZDwuFwz7GOZtJKnktAcdF6XicJCkhAYBXXGQwGDaIYLuQj0QAXypidKwjdQO22VCqptspj9nl7NmvjvC9YwePyqZ3yfGXax3OW+T4uaxWEXjk7O7PM8zLLVeHiVd43aJ8V1N4pt4u2KYeF8r943jG1cWr/dI7MMwiXRS6XU22enk94H6A8SKo/QXmPVvlfVlDNDC4d5nUoeNu3EhPznxOE60ihUFC1KbiQj8vsdaE9vSs4kdyTmIVL+KifUV0Yvk9/puIyP0G4CszyvroRfdO+o6Mj27ouZrkDnXIL+DEZyxUuilVb7yaHppN4u9tcGX3/4uIixsbGLlyv8TLqPFJ9uW6xq5vY67Fuz+f1324i9J7An2P4eBA9x+jjRGbjSVZ0kq7SemJiAn6/X4lzp6am4Ha70Ww2r7Tm6FXVIe2mbV5mXxDZ5HBBbclMhErtrpNM1Uxq3a2QtVOdIyt4jUGqG9hJpko1Be1kqhcRsgrDAW9fVP+V17+kdtiNUJXaNa+N6VSoSr9DL/AasXQN5W2U2i2vg0nPD7xt6vLsbgWqN/2ZRBAEQRAEQRCE1waRugqCIAiCIAiCILzO0EQgnxykyTo+gccnB2nSz0qYxyVxNGnY63f1Ck1S88lDLs+jie1uRH38u/hkIE0edvNdgnCVmMllz87OlNiBJvh54IqZCNNMpMkn+80CCXjQAP83ew104fB+x/sV9cHz83NMTU2h1Wqh1WrB4/GgVqspUSZd60iiSwFAZ2dnSo5Jf59d8KTX68X09DS8Xi8mJiYQDAYxPj6OyclJzM7O2kovzI7JdaH/mAUb8u1BHe81cJfoVZLudrtVPxkbG0OlUkGr1YLL5VL9gkTSrVZLCZebzSaq1aoSltVqNRSLRdtAzYtKlejz6yorEK4P3Yhj7M7pRUrWzT1G315YWJACNcJA6EXEZHXMLqkH6Cxj6lZeJs9qQj+hJGcqDMALBVCCM0+E5oUGCoUCyuWyKkJQLpcNkgAzpqenTUWwFxHGSuKL0A86iTC7+dztfeUiz2IiixWuGroPUIEmvXgGfabiTFS4jD7TcZKT2xUMIEE4FRvz+/2G4mP6fYPa/uTkpEqW5KJx6RdCP6CxX70P8Ocu6hNWxf2oj9F4uRlUXIYKMfn9fszMzBiem/R+4vf7VT+iY9Q3+Fi4cH25SHELkcPa41QAa1bw2YkANhgMYnl5WcRWwtBSLpeRzWaRTqcNwte9vT1kMhlks1ns7e3h4ODAcB0ZGRnB4uIiFhYWlAR2cXER4XAYoVCoTQYr7/vCMOCkwH8qlTLIjj0eD+bm5joWPF9eXhb5q3DlHB4eGqSvu7u7SKVSSv6aTqdVjA3wsfg1Go3ail9pzl8QnNBsNrG9vY0XL14okRlt7+zsoNVqYWRkBPF4HHfv3sW9e/dw9+5dw7YUT3n9aTabeP78OZ4+fYr33nsPP/nJT3BwcAC/34+33noLT548wePHj/GFL3xBikbZ0Gq18O677+I73/kOfv7zn+N3f/d38Vd/9Vf4/Oc/P+hf7VogUtfOiMy1MyJxvXkcHh7iww8/bBO3vnz5Us15hUIhrK6u4v79+0ri+sYbb2BlZUXeCQUDuVwOm5ubBiGm/pmgcebbt28rESb/LGMOwkXJ5XKmkla+j89/eL1ezM7OmkpaaZ/MewhXjZOx3N3dXZXHCjgfy00kEjJnIQwVlUoFx8fHHdt8Nps1xJnwuWqrNfUHM3T5AJclUWyM3T4u+bOCYohJxkexxbpwlUsKzMSsguCEy8rRchITbBc/42Tf3NycjJULPeEkr8pp/LsTKZiT9tzpnPn5eRGxCheG5z6RUJXHtJOMlZ5T6DOJW0leTLHwdjVoqNYLz4eiOHe/32+QsdrJiukcud4L1xGze0qv+VYHBwe29WGcPEt1ut+EQiEZPxUuHS7wpndf/i5sJ72nz53E316v1yDx5vcSEhLzewvPL6F8Evo5ud8IF4GE9rzdUl4htWmSVHI5Ny0ktbTKj6KcqKmpKUxMTGBiYgKBQAButxsTExPweDxwu92qxp7X61XXda/Xi1qtBrfb7VhMZiZH4z/bLbo0zEyqR+9PZpIyj8cDv98P4JOagST543UIdXEZl5rRvyNcjG7Hivi+4+NjHB8fI5fL2b5L0/9vn8+H8fFxeDweuFwueL1eJWUcGxvD+fk5XC6Xkml7PB5Vl4vqcNHvrNcWu0j9vW7lvle5T0R8g6WXunZXXUuvUz66HZcptb7Id9D1XRg8l9XGL2tfP67dnY5f1j6pByQIgiAIgiAIgnDpiNRVEARBEARBEARBGBwUdAVcXBDLv4uCJ3ngVq/f1QudBLE8OItPiPYim7X6Lj65yrclGEzoB50CJ/h2N+de1nfYBZ06hYLvOg2tjYyMqGDO0dFRuN1ujI+Pw+12K3EsrSnwjwK2PR4PgsEgAoEAAoEAZmdnlQzALnADgMgyhpxBC2YvO4COcLvdcLvdaLVaGBsbw+joKJrNJkZGRlSfIflyq9VCo9Gw7UNjY2Mq+Nvr9aog2fHxcYyPjxukGdRf5ubmMDU1pZLrFhcXVREB6leC0E+4TIaKZfAE1JOTE5ydnakg8nK5rJJYucyP/4wdXKIUCARUcClJY0hi6VS8xO85gtAveCI2SceoL1Cf4QIzPaG7VCoZzrUTX1KCDpdXTk9PG/qCnlxKz1wzMzMYHx9XP0P3JnnnEvpJJ/mlk209eYmLBcy4bCm5yGOEq8ZMdqkXCimXy4Zts2excrmM09NTNa5oBcn+dJEyf96amJhQidtWz2v0LkTfJUlwwmXDBZjUJ+gZihK++WculKUiOvo7jB30DEXtmwvEKelb7x9m8li9fwnCVcHfLfj9gwoh6LJkXlxK7yedxt1oHkh/L6ECCTQGRm1ff2ehY/Q+Isl315NqtaraDxVbNSu20akIh1VbGx8fV3MN1Ia4bJg+U5vq9HkYKZVK2NvbQzabxcHBgZJgkgBzf38f2WwWmUymbYxtdnZWCTAXFhYQDoexsLCA+fl5LC4uIhQKqWPz8/NqvFsQhgldgmwlQ9almIAzCfLS0hJisZjMswgD5+TkBKlUCplMBul0GplMBqlUCtlsFru7u8hms0ilUoZr/djYGEKhEKLRKMLhMJaWlgzrUCikRMfy/i1cJVaiFtre2dkxFCjj8iAzUcu9e/fk3VhwRLVaVYLXFy9e4OXLl3j58iVevHiBVCoF4OMiyLFYrE32eu/ePdy5c0euj68xm5ubWFtbw9OnT7G2toZnz55hbGwMb775Jh4/fownT57gi1/8Iubn5wf9qw4d5+fn+PGPf4y/+7u/w//+7//i8ePH+Pa3v4133nln0L/a0PCf//mf+Pd//3dDrObGxgYA4MGDB2qfy+XCV7/6VXzlK1/p++84TIjM1RqRuN4M6vU6kskkNjc3sb6+jmfPnqltkmx6PB7EYjE8fPgQjx49UmLNT3/60wiFQgP+C4RhoFKpIJ1OmwpbNzc3kUwmDXlCwWBQtSP93evOnTtDOx4uDDf1el2NUWWzWTWGRWsar93f3zcUB52fn8fS0hLi8TgikQii0ShisRgikYjat7CwMMC/THidqdfrODg4wP7+vhJU7u7uYn9/3zDums1mDfGF4+Pj6trJx1yj0ShCoRBisZiaZxOEYaFer2N/fx+ZTAaZTEbNJ2ezWcN2KpVSOc7Ax++ui4uLqo0vLi4iFosZ1iSDpFiSbmSV9NmJyE+klcJVwuMKreKleG4Ufaa4FcqdotgWu3hbLj6ivA3a5iI+Pb6Fy5MoRnd8fLyP/5WE646ZaJLHltM2tW0urDTLd7K7bnu9Xvh8PgSDQRUbSDlMPH9Pz2GibcoPpLgtEWoLF0HPP+pVQkzb+/v7ttd5MwFxL7lH9FnqJgjXjU75Gjw/gz6TeJJ/diI9pvoLdL/hz1P0mefQcvEkFyOLhFK4KpxKiDvtu6iM2Mk78+zsrLxfCD2jP2/t7e3h9PQUZ2dn2NvbU1JiugccHh6iVquhXC6r63+lUkG9XkexWLSsizU6OorR0VG4XC643W6Mjo625RA1m024XC7U63W0Wi20Wi3be0knKE+Ji1OpTh6vfUe1f7hElXJjKe8WgKkglf4Neo/i59O/xevwCf3BrI4jPddUq1UcHBwoifDBwQFqtZohdzufzyuBb6FQUDUeSf5bLpct606NjIzA5XLB5XKh1WphZGQE5+fntu8hdpi11U7t165NO93H27LZPqF/cLm0WS1CXvv07OxMzYlRbVS9H9RqNZyfnyOfzwMw1jAlUSSvcUr3COCTGqj8d+oFfl3mtQhIfM2vm1xsSvFoXHBN11rejnk9Q6qjw/9NM1E2/dtCf6HrL2Bsv1xaStd03tZ5u+RjndQveBvm/4aVGJX+Dd4fLlLvk54ZgE/aFj0rAJ/U2eRtkNoqb//dPrcAn/QT3g+kfQuCIAiCIAiCINwoROoqCIIgCIIgCIIgCJ24apFer8c7JbU5wUoIOWzbMpEtXDY8SIoHllgFpFBwFQ+k4sEplJzaaDSU9KLZbOL09BTlclkFpDSbTdTrdRVUWK/XVYBWq9VCs9lU/85FcblcKpDP4/GobZLHulwuJcugbQrU5ZJpHpDFg054wAvvrzzohQe78CAXHpzFA12EwWMWaNhJjM7P5cFXVn2Ly9MPDw9Rr9fRbDaRz+fRaDRU36Hfg/5tCsq9KC6XS4lmPR4PRkZGMDo6Cq/Xq4IHJyYmDAHfY2NjKnHW7XYruYzX6zX0Fy7yo8AsHrjF2z7vH7wPibBJ6IRdAm2vYj8KirRCT6LtVtinb0tSk9BP6J5lJl2ipBFKdqLkEF34Vy6XlfCvUql0TFyi556pqSmMj4+r4iEk0SFJORViGB8fx8zMjCN5rCSgC1cNtXFKmCI5GSUPUqEdK6FfL1JyK6m4LiLnfYSe0wKBALxer+pnXq/XcK4gXAU8sZDuEdQX6H5RKpVUQi2NDej94uTkRPUfGnewgt4reEEfO0EsSTPNZJjUp/j7viBcFhcRjZu9zzgZA7/Iu4nZMf5uLwiXBY2f0XMULwBHz04nJyfqGcusOBy/f/DxNTN4cR4uQqZCh1QoTi/+xmXK/D1kfHxcjWMJww1dO0lITAWduAiW3nvp3dfqs10b40U19SKb9JkXhbL6TO2035RKJWQyGUNh3oODA7VkMhkcHh6qz/w5bXR01FL4Sp/n5+eVHFbmP4RhpFgsIp1O4+DgANlsVvUBEh9TIetsNts2DjY3N6faPUkEeJufn59Xn0X8JQyScrlsKjc2W3NEciwMGpIF7O7uYmdnp207lUqp+XPg4+tyNBpFIpFQspd4PI5YLKb2U/yGIJhRq9Wwu7vbJk/b3NzE1taWKqITDAbb5Gm3b9/G6uqqvC++Zuzt7WFtbU2JXrk4kSSvX/rSl3Dr1q1B/6pDxdraGr7zne/gvffew+PHj/Gtb30LX/7yl298zOsvf/lLvPXWW47O/cUvfoE333zzin+j4URkru2IxPX1JpfLtT13rK+vY2NjQ80J6c8etL28vCxzODeYSqWC3d1dpFIp7OzsIJlMYnd3F8lkEslkEjs7O4Y4yOnpaSQSCSwvLyORSCCRSCAej2N5eRnLy8uIRCLSnoSuoPEmWkh8SaLLVCqF/f19ZLNZw8/NzMwYhJehUAjRaBThcBjxeBxLS0tYWlqSOCfh0qlWqzg6OjKMj1pt6zImiqHgY6Jm60gkcuPfe4ThoNFoYH9/X8392klbDw8PDT/r8/mwsLCAYDCImZkZQyzR6OgoxsfH4Xa7UavVcHJyYiuYseIyRKwUUysIHD2WlcccUfxqsVhEuVw2xB9R++X5FBSPxOXdOhQbogtWKW6VBKskBdOlYSQVo+8QhE5Q4X+7eG1q05THQG3fLEeIciGsIKHK5OQkfD6fQYZH8XcUY0cxUlycp4vxeL62IHQDz13j2zz34OTkBJVKRQnByuVy2zZd6yknyA5dNKz3A4otpWPU1ukYb/f0WRCuI3ayyW4+O83ZdiqetPssudrCZaPL+Oi+w5+n9HcLynkgkTHl0lHdHSt0yTC9U9C7RjAYNHym9wz6TPcpyQG62fD6NlxERrmaZoI+qhtFz0mFQgG1Wg0HBweo1+sql6dWq6m8CpKvNptNVKtVNJtNVRuH6kVdBjQmNTY2psSmU1NTcLvd8Hq9qv6anZjMTJ4K2IvJOsnRhKuhU/vltdF4zaZe6kLR+0Oz2cTJyQlqtRoajQYKhQLOz89Rq9VQqVTU9kXaNNVzGhkZwdjYmJIQT0xMYHR0FB6PR9V0oroAlPPs8XjU+7aZRLiTWNiubwhXD89F43nB9GzM26RV+7YTqPLv4v2H/l0rwepF63TSswZvV/z6StdNK8GqnUCVX7P5tZjGlawEq1J3pn9Y1ZvkdfV426c2bCVD5e3drJ84+TfMhKu9wmtC8nbNxzbpeYH3Ad5e+XXWTKhK39WLCF4QBEEQBEEQBEEQBohIXQVBEARBEARBEAThOtMp+AqAQQrBt2kSnwcN8G0+Yc+3m82mSt7g2zwYgG/z3/Ei8KAUvs0n962ksDxYgG/z4Be+zYME+DYX9/FtHoAgAb/CZUFB7MfHx6rQejqdRrlcRrVaxd7engqKPDw8RLFYVAFllERLn0ulUkcZJgXDUHunYLFWq6WCaiigeGRkBPV6XV1nqtWqbYF3p/AgGz24hiefc9EzDzLj/a+fklqh/1gJYzOZjEpI3N/fR7FYRKVSwdHRkUoOKRQKqFQqKui4Wq2iUqmgWq2iVqup9kxLp2BjksS6XC7VXwBcioDWShhLfYD3Ex64xtsnb8O8nVvd63i/sOoLVpJm4frC5TKUuKsXdnAi9uMJ8fRMaQVdU7l0jBIKKcnXSlpGAfkko6EkFN5OBeGqcSJY7kVkxt/lrLCTk3UrMqPtubk5kYoLV4ouKetGGEv3FepX+Xwe1Wq1oywWgEFIxgWwXK5M/YDuJVRQgp9L9xp+rhSkE64Cp6JLp/eXo6MjlaxmhdX9o9O9o9NxKQ4hXBZ6wQc7mTgvQKe/z/AiXnbF5wCoIkPUrqnNc7nl1NQUxsfH28Ti9F7Nf47fYwThMun2XuHkmB293jM6bUsi8fDSrwJVejvqRsitf56fn7/0ggtWYkD6W/nnbDZreK/3er2YnZ1VvyuXA+qfo9GoJB0LQ8fp6akqfr2/v69ksFQM++DgAIeHh9jf38fx8bHhZ8fGxpTgdWFhQclguRSZH5+dnR3QXyncZCqVCo6PjztKDfb29truZVYCWH1fPB5X832CcBnkcjnVLjc3N9u2k8mkoSAib6tLS0u4fft2m4RD5FiCGdVqFalUSoSvNxwSK5Lk9ac//Smq1SoikQiePHmiRK+f/exnZd4EH8tdv//97+Pdd9/Fm2++ia9//ev4oz/6oxs97rG6uoqNjQ3bc+7evYsXL1706Te6Wii20gkic/0Ekbi+flSrVbx8+bJN3Pr++++rvIPp6WncvXtXPTvQ88SDBw8kNvcGUqvVkEqlsLu7i52dHSVv3d7eVttclOnxeBCNRhGLxbC8vIxYLIZ4PI5EIoGVlRXE43EVjy4IneBzAFbjQul0GplMBrwsSzAYtBwXonU8Hlex3YJwGZRKJcOYfSaTQTabxcHBgdpP27q0aXJyEuFwWI3VRyIRLC4utm1Ho1G5FwtDA42F6mP3JHSnPnB0dGSYoyXJhcfjUWvKjyMBB+VPWJXcspqr7nafxEMIhNM4ah73Zhcr10lEQLleXK5HcdA+n0/Fx3EREkn6uDyJPkssg2AHjxFyIlw9PT1FuVxui/vkOW2d8moop9Ln86nrLc9BMxNUkoQyGAyqY1NTUwgEAvD7/SqHUhCcUCqVVA4Yb+tnZ2cq58UsF4b6CW1T/+Hn20G5kzwf02o7GAwa+gW1dboPcCmxIFw3us3X7BRHfXx8bJtf0CnGtZuY16uIcRVuLvo7As/JtHvn4O8m3bxnUF0A/d2Cnqso/59E3/S+EQwGDZ8DgYDkZl4zuCyS587r+7o53st3nZ+fq+et8/NznJ6eXppMVcflcsHtdqu6S16vV9Vj8ng86p16bGwMExMTSjrs9/vhdrsRDAYxNzeHqakpLCwsADDeT/TaaLwOjHC5XFb7u+zjF5VMut1uuN1unJ+fK9kkAFX3iMY9SUBMEmIrqC1TG6frNb0vU5untj03N6fy8BcWFhAOhxEMBrG4uCjjoVdAN9dUJ9uX/X1O8nU7YVU30u7aedHjnX6G198SrgZ+LaTaqLymKZfzmslQeb1UXue0k3CV/xtOhKu9QnGAvDYbb2u83hvNJ/H6i7zWoZVwlf4NJ8JV+jdEmioIgiAIgiAIgiAIjhCpqyAIgiAIgiAIgiAI/YMHMPBtHjzBt3lwBN+2CvThQRGlUklJK/g2D7Lg28ViUQn4+PZF4AI+noTLt3mwMd/mhZlomwdD8OALvq1/DxcC8oAOXcJ3EwtB3TR4Em+xWFSyVx5sXygU2gL2eSC+HqRvB8n3SIREgZt+vx+jo6Pwer3wer0YHR2Fx+OB2+3G6OioCm4eHR1Fq9VSstlqtYqxsTG43W4lmeaBVQAMhXS5xJoncfKAqcvq67w/8QCnYZTU8t9PcE6vYj5K5jo9PTUkXVarVeRyuY4yZAqkdrvdGB8fx9jYmFooCYASAQjqO9R/6vU6ms0mXC4XKpUKms0mRkZGDO3/MgIJgfb7kVU7t2rbF5HLWvU9p7+HcHX0KiCz+hmnwks7scxFBGX8GiwIV4WemE/JDLxIC0nMS6WSEpafnp4qoXmxWFT3n0KhYHgnsoIn95OsLBAI4P+z96axkaVX/f/31m6Xy1Vll7e2u5uZ7p4hiQSIQBQxE0Kk+bEKAUIQhEhQAiQEsUbkRdiUlS0JCgGGJYKwKIIAQixSFMSQRExPWBRIXiQDmaUz3W63d1fZrrJrr/+L/n+fOfepW1X31mKX7fORru5+Xa46z3rPOd9EIoGpqak24bJEImECciiOKUXMWE/L+l1Rho0dQOS3nfF7bS+Cipj5vVb2kRRlUGSiGDsRUqVSMfMAHKdwbu7g4ACVSgWHh4colUqoVCqmXeLzevXL2BZIwUsvIeVUKoV4PI7p6WlXW8K2qZPQsqL0C4PubGFxJgSzkynJ/lilUmnrd8nkTL2Q9myLwbJseInB2vexLybvU5RB4XubbuMNJtHrNvZgAr4gZcMWQmZiCpaJbmOVbuMWHcOPF37mf/zudxOJBV5MshtUFHbQBFqVSsUIXDKJNgUwNzc3sbOzg+3tbSOGyXeiJJlMYmFhwSTOphBmLpczy+zsrDmvSfWUcUMm1w4qggy4hRB6CSEvLi5q0hflRGFb1E3cI5/PY21tra3/Y4tq2oIf3L5y5YrOCSlDYXt7G2tra1hdXcXa2hrW19exurqK9fV1I4gk+1PRaBSLi4u4fPkylpaWsLy8jOXlZSNMvLS0hJWVFfOeVlFU8PXicnR0hP/5n//BU089hZs3b+LmzZsoFAqYnp7GK17xCjz22GN45JFH8IpXvOJCvx/8/Oc/j9/6rd/CRz/6Ubz0pS/F2972NvzgD/5gz3b+7W9/O97whjfgoYceOqFPOnre+9734p3vfGdHH6loNIp3vOMd+IVf+IUT/mTDZ3t7Gz/wAz+Af/qnf+raZqqYq4q4nhfq9Tpu376NZ555Bl/60pfwpS99Cc888wyeeeYZ3L17F8D9Mv7AAw/g4YcfxsMPP4yHHnoIDz30EL7yK78SCwsLp/wfKCdJPp/HrVu3zBiafUfu37592+XDms1mTd+RY2m5ffXqVfX9VHriJdZqr73mcThH2UmolWNlfQelDAt73tFr/pFz73YMj5x39Jpv5Pby8rL6FCinCu2cosMbGxu4ffs2dnd3sbOzY96j0qeNvgYyHZbjOHAcB61Wq6MQazweN74G/YqwJhIJLCwsaF/jgtOvGFin63rFbAX1Re4lHMYyoCheDMu+/YrgAd1t3K/N29sqmq34ZRgxJfb2zs5Oz7jgfur2Xtuzs7MqJqOcKfyIeNuC3oyblL78tmh4NxhnPjU1ZWKzmO+C/sYTExNG9NsWvKd/MkX5uK8ow6Bb29NP/8xvexSkz9Xt3MzMjApOjgAZ9838OPV6HYeHhwC8BSCbzaaZ15b5uWT+LPqFSXGzfp/VDxQ9bbVamJiYQDgcRqvVQjKZNLlOYrGYyXXiOA7q9boRWavVami1WiiXy6jVaqjVal3j4yORiCvGiuKTrN+z2SwmJyeRy+WQyWRM3Hs2mzUi93L/Ivu7+GUcRSXt7V7xJr0IIhSZSCRQq9UQDocRiUSMPYdCIdRqNSOyyjxAx8fHRmSVcb3VahXVatX0obqVQSmyLfswqVTK7NPuGZsry0c2mzXxV1wuKqOyzWHZscyd1i+DiqEG3Q56n87xDJdR163D2vaT+6YXo7LLYdi+xrQqiqIoiqIoiqIoyrlARV0VRVEURVEURVEURVF6MQ6OKH62/Yhp+EU6idj7QbdP8h4NOD1ZpPBFoVAwgq+2cKxM/F8sFlGpVIxArC0aE0SszxaAYbJ+KfDiJRbjRwymUznrdu6sOafZ36fX/jDL67CfLUU/zypHR0col8um/HBbBpjZ2xQs6LRNB99ueIn2ye1kMoloNGqEYcLhsAlIAGDOAfcFVCcnJ1Gv142j3leDawAAIABJREFU7Kjt394fhiMyMDo7Hta2vS8Fai8a+XzeCM1IMTLaRaFQMEELst1h+ZBCTFKgTAYcdUO2HVJoLBaLtQmRMSiCAue2kFksFjNCZ7FYTJMmKSNnkMQDwxS/9Aru7HbMz/Uq1q0MGwbASjEy2daUy+U24UvZ7vS6thedRC5p/+l0GvF43CWenMlkEI1GTdIBjpGi0ahpd9jf0+BSZRgwuDuI7XcSiGX/TD6rF17lwRa77CUQG4vFXGLkvEeDAZV+8Rqj2wKZst0ol8tGXFnOC9jlKOicWSKRMDafSCQ8ywfHJfI+jlG8yoeiDMooxiGHh4cmmUcnNEHa+YT9kEKhgFKpZPoRnfY5j2TvU/i+W4JJJrZIpVLIZDKmT51KpZBOp0292Wk/mUya+Vi7L14qlVwCsDs7O0bwlQuTGO/s7LSNvxOJRJvQq5f4K/dzuZyKBSpjhYrAKucV9ld6CTHcvXsXBwcHrnulXXuJMHA9Pz+vdboyEJVKBbu7u0ZQyUvc5vbt2yiVSuaeRMItUCxFlVRcSSGlUgnPPfccnnvuOTz77LOu7Xv37gG472dw5coVXL9+HdevX8eNGzfM+sEHH9Tx1hmi0Wjg//7v//DUU0/hiSeewKc//Wlsb28jmUzia77ma/Doo4/ikUcewatf/eoLmfD4C1/4An7zN38Tf/mXf4krV67gp3/6p/HjP/7jnjb+5S9/GTdu3MDMzAyeeuop3Lhx4xQ+8fC5desWrl+/3tWv59lnn8X169dP8FMNn93dXbzqVa/C//7v/+IDH/gA3vrWt7Zdc5HFXFXE9WyTz+fbRNxv3bqFp59+2vgpUoDTFnN/6Utf6vJ9U84f9Xodm5ubuHPnDtbW1nD37l3cvn0bd+/exdraGu7cuYONjQ0jYBUOh7G4uIirV69ieXkZKysruHLlClZWVsy2zuEo3Tg+PsbGxgY2NjawtbWFjY0NbG5uYmNjA/fu3cPm5ibu3r2Lra0tl599IpEw49aFhQUsLy+b9fz8PFZWVrCwsID5+fkL65erDJfj4+OOc4Jye3V1tc2PTM6/dBNqvXLlyoVOtK6Mjnw+j1KphKOjI/M+le9XDw4OjP9XqVRCoVDA3t4e8vk89vf3zb30e6lWqz3jPCh0QPEDignMzMyYhXPifE+byWSMUBNFB9QH/2LSryhSp+v8ilMOS3g1kUhgbm5Ok7srBlv4w7bZXjbd7bwfUaZuPlb9+lld5Ng3xR/DsPNB45yG5VOoosPKWWUUsYZBxCaHWf603VGGgVccIXM5MK5D5lhhzhQKEssxtcy30o10Om2E9uinPDk5iVQq5RLho3gffZIZAzU1NdX2jIuMFCqVAqnMSwDAlY9D5m2RsQkyhofikH6eLXNg8HmtVssIU8tr+4X1XSQSMXOErBMBmFjrUCiEdDoN4MX8IsCLQmPNZhOTk5Pm84bDYZTLZYRCITQaDRwfHyMcDqNSqeD4+NgIrR4f3xecZB4Uxj1RjLLX/8fP0klUNej+WY+TleK7MteFtFPmZgLcorsU6AVezLki7S1oeZB/p1PZ6BcZmybtsVMOISmIJ/MH0P4dxzH1nbT1TuWC/fRwOIxYLGb+T8bvhUIhVKvVvoS3e8X8DTKHdBZEthuNhvGFljYnbRFw5wWS35m0Y2lr0galbfrJw+VVNuRnGwRpj/T5CYfDxjdO2non+06lUsYHnLYpbVo+j3kB7OfJ+k8+T/vEg9PJjgfZBtxi08PaHla+LWlDnbZlvTysbWnHLFuyLMg63c6/pSiKoiiKoiiKoiiKMmJU1FVRFEVRFEVRFEVRFOU80o8o3mnfM0yRSmC8xWc7nZOOTBedoEFAfq+VjpedCCI01mn7pBL9S4dU6XzdyTlVOlfbzn/yWdLxWjrQAm7nWPls6SwOuJ0A5WfzK5roF1mGpDMe4HY2lY580mlVOvMB7nIony2dXgG4giul86wtdCIT70nHQuloPgpGEUjnJ0EBEKwM9SPg5+VELG1bBhRI++3m8NrJ6VvabidHb2nT/XyOQZC2LG3PtmvpGC6DD/yWGWmvMqghyLM7lYVx5LQTIQDtwRn9tEtex8YxUEM5H7DO8xJSPjw8RK1Wc4knF4tFVKtVI8LMBE8MeGK/gsf8BhRI8UrWXVI02Ra0lCJlUhSTYmZS2C8Wi2kSNGWoeIlg5vN5U04ODg5M29HrWlnOggT2ptNpRKNRI/g3MTFh+sOdRC4jkYgR0GQZi0ajyGazpv1nu6Niy8ow6Hf83+tav8HEoxYlH/e+sTJ+DGu8bx/z298adKzf7ZjOjSr90u/4vde2XzHlfueLO23ncrkzn1jkrOIn0avf/V71qm0XfhOPhcNh1Ot1kzREJoeyBQTz+Ty2t7fb+jx8li1+2enY0tKSJnVQxgYpAttNADafz2NjY8NVDuPxOGZmZnwJwKrtKydJoVAwQiTr6+vY3NzE1tYW7t27h+3tbZdQifSniEQimJubw9zcHBYXF822vT8/P4+FhQXzPklRgtBqtbC5uYl79+5hbW0Na2truHfvHlZXV7G+vo67d++aupdEo1EsLi7i8uXLuHTpEpaXl7G8vIylpSUsLy9jcXERy8vLF1LcUbn/bn1tbQ23bt1qE4d74YUXzBiMwnBSEO5lL3sZrl27duGTj54Fbt26hZs3b+Kpp57CzZs38fTTTyMSieCrv/qr8cgjj+DRRx8dunjjl770JTz88MNDe96wuXXrFn77t38bf/iHf4jFxUX83M/9HN70pje5/Anf/OY34yMf+QharRZmZmZw8+bNcyPs+vKXvxyf+9zn2sbJjuPga7/2a/HZz372lD7ZcCgUCvimb/omPP3006jVapiZmcHq6qrxszkrYq6f+9zncPnyZeRyuYGfpSKuZ4/NzU0888wzePbZZ9sW+pGm02ncuHEDN27cwEMPPYSHHnrI7Gv7fD6pVCrY3d3F+vo6bt26hXv37rVt37lzxzX/yH4cxTPt7StXrui7IKWNTkKtnCvZ2toycyXSTx247/O6uLjoEma9dOkSFhcXzZpzfooyKJ3mp+3t1dXVNj9hzjl3E2m9dOkSLl++rD4kSmCCiAv02u8Vb0UBVsdx0Gw2UavV2sZ69CHMZDJIp9OYnp7G3NycmRukmPu1a9ewsrKigmcXBL7Ppkgw/V4PDg6M2EuhUDAiSYVCwYgiUUz4+PjYCCbxum7Ql1WKG01MTCCbzZptWwRJiiXxXT3Fks6DSIwyGIyXYGxZPp83PtuHh4fGh4P16/7+PqrVqhHDZpyE9P3mPXYsdieSySTi8bgrVmJ6ehqxWMzYL88zHkLGUtBXhfbNGD0ZF6koEsYAlUolVwwQ7fzg4ADVahUHBwembAS5pxeM//Gy4+npacTjcaRSKZftx+Nxcw/jGeR2Mpk09yvKWWF/fx+VSsXE2cmyxjalUqmYvhX9F6vVqolH8iqLQUT3MpmMaTvsdsRuU9jPkuVtamrK+Iqrv7gyDEaVh8RP/HcQsWE/585K7F1QMchhiKP2++xB6CQoKfOKcL65k4ikzP0gx5KDCrGGQiEjOuxn/sfvdjeCxkH42R5Fzh2boDl4/OQNkcc75cmSf7eX2Kqdw6dfOok6BrHZTmKr0j79/B0vsVU7R09QWD/IPpA9t8T9QqHQJrrNc4y5YC6DTvCzS3Ftzislk8m2uSRbbJvjk2QyaZ4TxN5lzqZOuWpsYdJO4o6yfgxaJmT926lM2PmlBkHal6yHpQ3K3E+ynpXbnWzWSxS4U3noVAY0Brg//OY7k3WmrGOlncly4GdblpWg24Mi68ag29Jeh7XtR7BVURRFURRFURRFURRFaUNFXRVFURRFURRFURRFUZTxQjr9Sec+2ylVOhZKB8JOzlndHLo6OYHZDlfSaVE6KkqHxGEgRfWkY1838UqZbEQ6KdrCfv2ek86GQc6NI73E+gYN2OjFsMViz1rQhsQuY50cLaXDMeB2JpZ1xmmJ1g5KvyKZfkRrAbcDp31O/i1bkNbrXLVaxfHxMVqtFmq1Go6OjlAqlRAOh42TOwXHarWaESNjoF+tVjNBhHSIr9VqrmD2XrCe6SbMJ4XHKFQmxcWi0WibGJkt+HdSBHWO9xMk0q3MdBJZ7rc8Dko/grH9iCfbzz5NYWav4FmvQPZOSR56Bdf2gv+jtHse4/fM/zWTySAcDhtBSwahMPCdon8MquH3oklLlFHAtlgme6jVaiZRD0WX2P7wmCxfUmiWZUoGsfsR92M56dTGMCmKFMiMxWKudqeT+CyPqWCIMiw6jWn8jnd6neuVxI34FbnsVzDzJIJ5lfOJV3sgxyh22+M1zmH70en6XmKAAHqOVSicbPfZ/F6vCbcUP3AexStpnNcxr35Vt2N+2wyv8XmvY15j+27HFMUvLBfs+8i+EI9zu1wum2QP5XIZpVLJlfSKbUexWPQ19picnEQikTAJr9gfYh+ICeqYnNFuD+SYQ5YbthHK6OG888HBAYrForEVO1kt60iZQETajkxI20solvNfMhlaMplEJBJBNBpFOBxGq9VCo9FAs9lEtVpFpVIxgvdMbMK5RxKPx5HL5ZDL5TA/P4+5uTnMzs6aY7Ozs+YYFylMpCinRblcxs7ODra3t7G5uYnt7W3s7Oxga2sLm5ub5hxFIezEMhTapiBmLpczApnz8/Mum+eiiceVUbO7u9sm+rq9vY319XWXTW9ubrbZ9MTEREcBWNr3wsKCqet1HKkE4fj42Ai9SsHXtbU1c2xzc9MlTDw5OWkEd1ZWVoygA/cXFhZw+fJlFSS+QJRKJTz33HN4/vnn8fzzz7u2V1dXzbxCLpfDtWvXcO3aNVy/ft1sX7t2DYuLi6f8XyherK+v4+bNm0bo1RZ3fOSRR/CN3/iN+Iqv+Iq+/8aVK1fwdV/3dXj88cfH2g5u376N3/qt38KHP/xhpFIpvOUtb8Fb3/pWlEolXL161fhycJz/b//2b3jJS15yyp96cH77t38bP//zP982HxKJRPCBD3wAP/3TP31Kn2xwSqUSHnvsMfz3f/+3+f0ikQh+4zd+Az/2Yz92JsRcK5UK3vOe9+DXf/3X8dGPfhTf//3fH/gZKuJ6NigUCnj++eeNoDpF1r/whS8Yn7FYLIaVlRUjqC5F1h944AH1ZThH7O3t4d69e7hz547pv9+9e9ccu3fvHvb29sz1sVgMS0tLWFlZwfLyMi5duoQrV67g0qVLRphtaWlJ/cUUQ7lcxt7enkvwspMI5sbGhmvum+9EKHbZSQDz8uXLKsKj9M3x8bFrPm1nZ8cs8tju7q45JkkkEpibm8PS0pKZT5Pbly5dMvPJc3Nzp/RfKuOI7YsXVHxVbu/t7bW915NI/zr6fIdCIUSjUUQiETQaDVSrVdRqNeOvzvfakomJCeRyOWPXnFeen58371HkvLMmAD+7BI2h83t+Z2fnxESRuD03N6ciCxeUYdjxoDGjfuNFg8aNqsiCYiN9munjyThbeU7G3/QSeLTj2nrRr9iq33t0Lko5C9BPMIiYaqVS8SUUPgrRb1kWpcjqxMSEEbmfmJgwomXqC6X0ixQbPjo6MuWDZUH6W8trZey19NmmYLgfH2z6Tcv2JZlMmnhQ2c+SZYZx1VIMnOUnm80OPS5H5rqQ+RVkDH834b5hiKPKfAO9hFcHQcbmy7wBQcRRZZ6dTsKrw3j2oATNX+Nne3t7u6fd+81ZE0SANZVKoVQqGbuQMUFBhVI72bsfAUs7Z4wfActBkHkmpCBpJ+HToLYp+7nDKAPjwrBzNskYnm7Ytk/7jUQiSKfTSCQSiEQipm8TjUbRarVMvE29XsfU1BSSySQqlYr5Xf3YnCwTQfPCdBJNHRQ5dyBtsJNoqrQradfS9jqVCWm3dp4xaeedxCp1nqM33XL8SLvslP+un5xD3fJvdcrt1S1P0aDYOYKkz9Ug29JGh7Wt4ydFURRFURRFURRFUZRzh4q6KoqiKIqiKIqiKIqiKMow6eYIN0zx2W5OcfLv2J+n2znpdDcs/IrBAm4HuH7P2U7zozjXDf6WQYNNmDRdBpvIIC4Gm/SCifvpOEwBMTod03nWTs6QzWaNgzf/XyksFolEXGJ9iptOTq7DEIwdVIDTrlPsQL5hitMS6eBtC9HKc9JpG0CbCGQ8Hke9Xkej0UA0GoXjOCiXy2g2m8ZJvlarwXEcNJtNHB0dmfqNyUWOj4/NNpOOMMjYD/2Iivm9fmZmxlWXnVUGtddhl4XTEGa2bbkfwVgZeCDbnW7P7iTSXK1Wzfdbr9ddDvEUV3YcxwT5Hh8fo9FouAL4Dw4OUK/XjWhmsVg09VuQ/sL09DQikYhLONZuZ/y0U15tEb/bsyhqrow3gwhf+jk2iAhmkLbH7z0qwqwMCtuW/f19E4jF9kSKL0uhv1qtZpJVlEqlroKZftsdW7xSti9228H2he1qNps1wV18Du9hmWFZ0WBBJSgcF3YrA15lRrYbLGN20iV5vR+kILkUH7dF/2y7Z3+rW1mxy5eieMHxm0xMRPv2SgzmdUyWDa9jfvpZ0u5pw93m0eTYwx7fdJtjU5RucG6eYp/cpi2XSiWUy2Uj/MltOdcs54/t9qMXtGHaPu1aJsaT7YaXoLhsS1h+bJFxZTT0m9yn07leCZ+B+/O8TPQsxWEbjQZqtVqb2CztKZVKIZ1OY25uDjMzM5idncXS0pIRa5OisDLZgqKcBkwqTZFXisBSQNMWiPV6b+cl9MpFCiHLRedllFEhxVNssRS5nc/ncffuXdc7TaBdQIUJr7z2L1++rG2/4ovj42Osr6/j1q1bLkEfuV5dXXXNByYSiTYhH3u9srKifYlzTq1Ww+rqqkuEjsv//d//mXY5Ho9jeXnZJUBHUborV67ovNWYcHBwgP/6r//CE088gZs3b+Kzn/0sKpUKlpaW8Oijj+KRRx7Bo48+iq/92q/1lbD9zp07uHr1KkKhEJLJJD74wQ/iDW94w1gne7937x7e//7344/+6I8wOTmJl7/85fjXf/1XV/13noRdt7a2sLS01JZANRQKYW1tbayFeLtxdHSEb/mWb8F//Md/tCXRTSaTpn/0sz/7s/iZn/kZl9/kuPBf//VfeP3rX4/nnnsOrVYLb37zm/H444/3vE9FXMeXSqWCtbU1fPGLX8TTTz/d1m4C9+uXy5cvt7WVDz74IL7iK75Ck0yeA/L5vOlfy743t9fW1lz+dp363A8++KDZvnr1qvpnKSrUqow1+Xy+TYh1c3PTJdYqj9nzu7FYzMzhUoyV+7lcDgsLC5ifnzfilToPcf45OjoycT9yu1QqGT8GuS3f+/Lc0dERCoUCisWi8Q/qBP3Fp6enkUwmjcgA3+kmk0lks1kz1uA7O/p/UyyKf2d3d9fUyVtbW22+FNlstuO8r9ccsHL60J+/WCwaX0uvmDM7lo0iYfQ7kyJiMn6tF/ShSaVSxodGihtJISQpmkTxMPoX0Hd5cnLS2Li+X7g49OOj79df3494MOBP6Cio2CrPq3CDQvza+iDn/ApW9RPv6LdsqL+wcpYYVNS72z1+YlRV9FsZRziGlGMMrzEE41g4HqGfs7xW5s7gtb2Q+TA6iajKfhbjW+SYRI45pJhWuVw2vtTSr9rPdj/3DLI9KDIWPch2v/cFffa4CU9KsW1ZBniMdu91TIpuy2NSkLhX7orJyUlj64lEwsQpxuNxRCIRTE1NIRKJIBwOI5lMIhwOw3EcTExMGBtPJBJIJBKo1+twHAexWAzVanXotjnMvEOjtLd+trudO6/jOsY2euU9sueVmPeIc6H2PKhdTvyIkVJkNRqNIh6PIxQKIRqNuvLRhMNhRCIROI6DVqtl4hXq9ToikYjJK8M8GtVq1fxW4267w3xWp23NE9CZUfUFRvlsmSNnGAzTNk/ynvNaJyuKoiiKoiiKoiiKoihnBhV1VRRFURRFURRFURRFURSlnW6Cr93O2SKWozgnndqDnBsWMrjDdgocxjlbOFWeo5jl8fExKpUK4vE4jo6OUK1W0Ww2UalUUKvVUC6XTUJ1BmlWKhVUKhUcHx8bAb/9/X3U63WXAJMf+hWz8CPkpyJ9p4cUpAWAQqFgkip1OycFNgG3qKZtV93OSfEVW4BTClJ3O2fXT8NiYmICrVbLBAHIMhkKhYyIAY81Gg1fwdGhUAixWMwEHiQSCRNsw+MM0pmcnHQJMMnEGJOTk4hEIpifnzfXhMNhrKysIBwOG8EaKSiqvMgohZmlffbzbFlOeok0DwvZ7nQSjgXu2+/k5CSazSaazaYJpKH9JxIJVCoVNBoN4zBPoeZQKIR6vY56vY5yuYxWq4VqtWrOU4zZ/m67EY/HMTExYdZSiK9fQebzKsasnD5ewpVSTJntGesAlncpvNxoNFAoFFx9OgaCMomY3zqCZZ32botXZjIZhEIhI7BM0aZuwn+ZTAbhcNj09xhgKvu5ihKEfD5v2gW2lV5il2zX8/m8KRcsDzzHdjefzxuBwaD9SFkumFTPFjDn2IbjHy9hTLtMdSqHitKLbiKwslzY5cirTWJ7w76sbG/8kk6nXe3B9PR0oLajk2isnE/gPYoi4Virm/irtHseo73L9sMWGGc/ze98oz0Pxr4UE1gOIiZrP1tRbLyS/AVNeNntmN9kF37H336P2ed0Dnm49CsWS5vY2dnB3t6eGZMWi8U2UZtecL6UdSATz01NTSGVSmF2dhbLy8tYWFjA7Owsrl69at49qF0oJ40tmGkvtmhmp8TptF+52AnTvc4ryrBhvW7brte+LboJuEWAegkBLCwsaF2tdKTVamFzcxMbGxtYW1tzCQXJ/c3NTVedmk6njZ0tLy+3CVJxW/ppKOeHfD7vEqyjkN3zzz9v3q/aAnZyefjhh/Vd+ilydHSE//mf/8FTTz2Fmzdv4sknn8T+/j7m5+fxile8wgi9vuIVr/BMQPjRj34Ur3/96837dsdx8MpXvhIf+chH8PDDD5/0vxOI7e1t/Oqv/ioef/xxzwSv50nY9TWveQ2efPJJU3eHw2F84zd+Iz75yU+e8ifrj0qlgu/8zu/Epz71Kc+xbzgcxmOPPYa/+qu/Gst57HK5jHe84x143/veZ/w3AOD69et49tln265XEdfxwhY7lwKuL7zwgqkPl5aWjFirFHB9+OGH9f3jGWVrawubm5u4e/cuNjc3sbq6iq2tLayurprjGxsbrnppbm4Oly5dwuXLl7G8vIxLly7hypUrWFpawsrKClZWVlSY8ILDOavt7W3s7OxgY2MDW1tb2Nrawvr6utm+d+9emx9fJpPB4uKiEbmk6OXS0pIRv7x06RLm5ubU504JDAWqe825rq+v4+7du239aTnv6jXfah9bXFzUxNBnFL73L5VK2N/fN36WFLyksCpjaOR2Pp93CTHJa7pBPxjGzqRSKaRSKfMOP51OG3+AdDptxC0nJyeNMGsymcTU1BQymYzxWaD48Pb2thEp5iLralsoIZFIYHZ21ggQS0FiL5Hiubk54yeuDJd+BcL8XtuLfkXCep2fnZ0dKxEeZTTQNz2fzxv/Ky+hL+n3ToFgKejFe6TgkV+fdvpRSaFgKdpFm8xkMojFYkb4mkJeUlyYfon0PZAxMsrFZZQiq0Hqa8C/T9Ug59TulXGnm4/jsMqjH4E9IJgAsop+KyfBIL6/fs7t7e35EgqLx+OewpPst8ViMUxMTBi/z1gsBsdxEI1GTTz+xMSEibmnEGWz2YTjOAOLn/kVOffDaYpIqnDffTiuoOCqjHva2tpCqVQycz5HR0cmdurg4MAV71GpVIytM36qXq/7snkKTdoCk1JoMhKJmLwStGXauOM4CIVCJjbecRyUSqXAvtSdOC0R1H5s/KL2RWXOFcasEplHRdZlMqa1Wq3i3r17JsfC3t6esWWKslLctFQqmRwOzD3UbDZRq9XMNvMN2X7Lo4btB9uEVCplzknbkPWZzGfhOI7Lz0LG17KfBcDE5QZ5NmN6vf5ONpsd4rdw9giS36dbXqBRn7NzFMm8LsMU9QXcNiHj92R9180O5T0yvwLfMRA5ZvBju4A754sUUdecQoqiKIqiKIqiKIqiKIoyNFTUVVEURVEURVEURVEURVGUi0U/gR3jdk4Kag4TOo+2Wi3jLN1oNNBqtYzoJIMMKGxJMT8GGkin7nq9bgRnm82mEfBj8IOf/4GBPLFYDPF43CRylwHWdDKlwAvFLVKpFObm5noGraoAxvnntMpvtVo1IkoMemCQBIMiGDDRarXM8VarhVqtZo63Wq2hBe3YhEIh47DN4CIGFQEvlkGWedYN4XDYiM2Gw2EjRjs5OWmC77oF5PTaH+ReABdWvKFfGw5y7Un8DSlsO0xCoZAJmHMcx7RjbNv8wCAilhW2j7FYzAjdynaLgsyxWAyJRMK0OVKYmYGtFHeiqBOTi6h9K4PAICSvZDsMbGKgKsseg6329/dd4rEyQU8n4T8/sC/WTejSj5hfKBRCNpttE9Rk+bKFqRXFL4MmCPF7j98kIcSvCHk/wuWaOETxSycRcS/R5E7tjZdobLPZdAmby0D2XtiisbY4sh/B8W7tDNsmO/hVudjQRmnPFAjf3983/SUpQk5bZ3lg2fEjJuuHfgVj7f4YbZ1B5Nls1iQq0DKgeNFPUii/x/wmjQI695OGISCby+XOZfKjk4L1pVeC7Hw+j7W1Nezu7mJvbw87OzvY399HsVg0iWfK5TLK5TJqtRrq9brv9yLxeBzRaNS8J2C9mM1mkUqlMDMz01dCWH2XoAwD2r+XAIGXOMHOzo6neGY3wVcvQVgVJVCGDQUEKDCwubnpEoDZ3t42y87OjuveaDTqEhSYm5szogO5XA6zs7MuoQFNGK90ggIvUvjVXt+5c8c1d806VAq+yjXPXblyRcXOzgm24KtcvvzlL5s+ZjabbRO9e9nLXoZr166NpSDjeabs5sJvAAAgAElEQVTRaODzn/88bt68iaeeegqf+tSnsLOzg2Qyia/5mq/Bo48+isceewyPPPIIJiYm8OM//uP4yEc+4ppvj0ajcBwHb3/72/GLv/iLY92Pf/vb344PfOADHeeBIpEIZmdncfPmTVy/fv2EP93w+JM/+RO86U1vcom6fvjDH8Yb3vCGU/5kwalWq/ju7/5u/Mu//EvX96MzMzNYXV01SRnHhc985jN4/etfj9u3b7d9fsdxsL6+jsnJSRVx/f954YUX8Gu/9mv43u/9XnzzN3/zif3darWKu3fveoqW375925Qltl9st9iOveQlLxk721M6s7u7i/X1daytrWFjY8OsKd5KsVbZ1k1OTmJ5eRmLi4tmvbKygqWlJZeAqwppXjwODw+xublpxAC3t7exsbFhtu19uw+SzWaNOKsKtSrDgnOincRZ7f3Nzc02f85EIuFLnDWbzeLy5cuuZNjKeBBU2NLe9jrnx2exm8BSv9vdkvsfHx9jd3cXu7u7pq7tJM7Kxf4fpqenzbypFGPtJNiqCdr9E+S9ddDzfmK8RiUWNjMzo+3yOWVYvqy9rvErhtXLP7UfHw2N67u40FfOjk3wimegTypFkTrd6+WbWq/XcXBw4Osz0aduenoakUgEmUzGxCf0e462Ls8pyjgzbP9XWQ5tv1s/9Osby/gieY7xSel0WuMmlL6gnbMMyPIhy06xWMTu7q6x/aOjIxweHroEK+v1OorFookvp0ifFCjrBgUn+U6U8eDSxyESiZh3CY7joNFomD5fo9Ew84LlcnnsBCeH+axO21IU8Dwixe9YNwNAq9VCoVAw18l4hW4ieRRUrdfrZu6D+UL29vaMjy99fqvVqrmWOUbkthSaDBLb3Q/hcBjxeNwI7KVSKYRCITiOY9qNUChk+mxEthHSXmwhPhnrLW2sm/ieFKiU4n1SlO+82yjQbo92HJm0Yyl8CrTnIJD2atuynENknBvplsNHnuM4g/kI9vf3Tf6PYdaj3WB+hHA43CZATJ95imwnk0mEw2HTZ2I5oE88x+GpVAoTExNDF169KMg61LZRaXejOBdEgNXvuWEhbcauC+U5uz7t95wfcdN+hYIVRVEURVEURVEURVEURbnQqKiroiiKoiiKoiiKoiiKoijKWcR2VJfO572c2O3ANxmAzQBXYjuu207ug94rAy6kU/FJwwAMilNGIhEj3EdBy0gkYhzVHcdBLBYzjr3RaBThcBizs7MIhULGgZ37FL+USTzsoHPp4G47EzPokEjHY+ViIQNCZDBVuVzG6uqqCeQrlUomILBaraJYLJrgAJbN4+Nj1Go11Go1HB0doVqtmmMMIuFSrVZNsJ7fACmWK3uR9zuOE0jMLCh2OZOBVLYDvl3ubDFAOzDQLoe20KYs03bQge3Ub5dxO+Cq2/9xETgpcVqWg2KxaIIM2T5RdJkBhOVyGc1m05QVtmcMOGy1WiawEMDIggsZhEVbo9AsyxuDsSjKHI/HTRs3OTmJRCKBSCSCiYkJpFIpc8/k5GRfIsv9nrsIQYYXiaCCl4OIY/oliABmkHW3Z8t2QFG6Mc4isqMoN53WOsZROjGqMtFvcjrg5MuHJq272HgltRpWwizavZ/kooRjZ45rGejOMTTHt7Rf2jLn0FjfMwjej6CscnGhXctkVrVazZUYa39/H7VaDQcHB8bGKSq+v79vys3h4aFJFMdjfsWTac+pVMokN6Gt0/6Z5CSZTLrEQu0EjkzcKMXFdUzsn0qlgr29PSNuQWGLra0tbG1tIZ/Po1AooFAo4PDw0CRKs+s4+S6AtFot1Gq1nvUh5/lkcj/Wa15CsRSfn56edtkB60SvhIGKYtNL9MA+t7a25imM3Un4oJMIwtzcnPZDlaFQr9eNUMHm5iY2NzddgrDb29tG5IBre3yWSqWQy+UwPz9vBGC5pgCsLQx7kd6jKJ2p1+vY3NzEvXv3sLGxgY2NDayvr2Nrawtra2vY2toyx/jeFbjf5s/Pz2NhYQGXLl0ywkULCwttx1Qo5uxSLBbx/PPPey537twx7/lyuRyuXbvmEn3lsrKycmr1zRNPPIEXXngBP/zDP3yu2+xWq4Wnn34aTz75JG7evIl/+7d/w+rqKmKxGL7u674Ozz//PDY3Nz3vDYVC+Mqv/Er86Z/+Kb7+67/+hD95b/b397G8vNwzQfN5EHY9ODhALpcz4/BoNIqtra0zN/dTq9XwPd/zPfjnf/7nnoktI5EIfuM3fgNvfetbT+jTdef4+BjvfOc78b73vc8k77ZxHAdXrlzB6uoqAOCrvuqr8OpXvxqvec1r8KpXvQozMzMn/bFPjdu3b+Nd73oX/uzP/gyNRgMf+tCH8FM/9VND/RvHx8d47rnnPJfV1VUzR7G8vIzr16+7lhs3buDGjRsq3DrmHB8fmzF8p/Xq6qrLbzYej2NmZsaMzzutl5aW1B/hgiDnhXoJYd69e7dNrIhCWfa8jz1HdOnSJaysrOg7e6Un1WoVe3t72N3dxd7enpnvkQKV9jHbx2tiYsIlSOklUmnP9Widd3L0K7rabduv2GU3MdWgAqyzs7Mmefug34XXPHy3OXqv/82PKPGlS5ewvLw88Oc+i9gCX/IdcJBzfBdcKpWM7zdFlLzem9jId3h8nzs9PW3sqtN5vgO03xHG43HXO0Tl7EM/Bel302g0jG8P7U769khxyUajYWy4WCy2CVFSsEaK0fQik8kYvxv6J9g+mfTNyWazxu+H/g20b/qnUeyFInqpVMoliKGcXzqJq9LeaZ/SlnsJs9r3SmFWv9DHjL5l2WzW+G3QRmnPXrZPIVUvwUf7nKKMK2x/WLbYbniVz04Cqr36VxRR9kNQcdVOYscs39ls1pRn9ZtWJJ3iQO1t9sH29/eNrzRjqxuNhvEjbDQaKJfLphw1Gg1XjOjx8bGJn2ZMNYVQT0Ksj0SjUSNAGY/HjT8hfU3D4fCJC6B2O3ce45z7iU8e9nV+nyFjnVutlrHt04AxzRSYZJxzJBJx5eFgOxCJRFxjErYttPNMJmP6etls1vTX/NrmefLNtoUd7fhAOYa0c8p0EzS150vkHJ6dI0b63Nt9ett2ZWyWLSAZNLbRL1NTU6YuYp4XCrBS+JSiwvF43OQAaDab5t15vV6H4zgmj0C1WkWz2fQVawDA2HUqlTK2LmNn2EeikCrtOh6PY2lpyTW/JH3UE4nEmR+v2DEb0mZt+5a5U7zulXZq39tNZHUU54aBnX9EzsGcxDm/AqxBzimKoiiKoiiKoiiKoiiKopwTVNRVURRFURRFURRFURRFURRFGT9sJ33boV8GEQC9nbeZyL9arZoEMQyuohhAvV43f7dSqaDRaBiBTArzUSCz2WwaB3068Z9WoAtw35maApqhUMgI84VCIePsz4Q2dkA9AyCJHfRoJ5Gw77f37UAXO0DGFq+0g58ZoEls8Uvl9LGFwVgeZRAwRWEY5CPL1tHRUdszhhGsz8AuGZwYCoWQTCbRarUQj8cRCoUQjUbRarUQi8WMuCUDc6LRqCnLvLZaraLVasFxHEQiESPuyQCfIIKjowo4AoIJbwbdH+Wze/2t85oEhLbBpVAoGAEaBhIzWQtFmdkGUZCkWq3i6OjItGkUZKaNVqtVVyBxrVbzlZRIwrZFbrdaLVMmGJw8bEYhHHsS4rT2OU0kMFrYTnRqY2zBPp7nmuWJaz6Ha/YpubYDV/3A4HSumRiJa/bDuKbNcM1+GNesE7m2hf9UAEjpRa/ESba4H9c8zjX7bLKM1Ot1Vxmq1WqB+z7S/qemplzlg0HN09PTrnKVTqeN6J8U/8tmswDgOu44TsdyqSgAPMcjtHG7nbDXdrvCtd3+cG0HmvfCFsrstO50zm5DbIFNe639mItJEKHkYayDiMmepJjyzMzMuUniovjDrxh4N4Hwbtfbc9ndkLYoE08HXfe696L1gTol3e6UhHtvbw/5fN5zHoVjNX6f0WjUJJ6KRCIIhUKuJD8cb8o5ID+Jy7v9pl6/b69j3LbfByjnl2KxaMQxKZTptVBMc3d311NMbHZ2tm2ZmZlBNps1a6/ti1THKKOBdXcngQT7+ObmZttcNeu/TmLG9vHFxcVzl2hSCYZtd/aa5+7cuePq33nZmi26lc1mcfXqVdd7cGW8qVareOGFF1xCr7du3TILk2DGYjE88MADnoKvDz744Eh/8/e+9734pV/6JaysrOCd73wnXve6112Y+Zzbt2/jySefxBNPPIE///M/79rHjkQiaDab+Mmf/En86q/+6lglu3zXu96Fd7/73b7GjJFIBLlcDjdv3sS1a9dO4NMNn+/6ru/Cxz/+cQDAd3zHd+Dv//7vT/kTBaNer+O1r30t/vEf/9H3OH92dhZ37tw5deHNJ598Eq9//etx9+7drp89Go3ipS99Kd75zndeOBFXsra2hve+97348Ic/bJL2xmIxvOUtb8EHP/jBwM+rVCpYW1vDrVu38MUvfhFPP/20aUteeOEF04fNZrOm7XjpS1+Kl73sZXjwwQfx0EMPqd/cGOJHrNUW14zFYpidnVWxVgXlctnMP/YSal1fX28T9Ook0uq1f/nyZUxPT5/Sf6qcBey5825Clf3OwXQ6pvQP30Pt7+8bnx/6zu3v77vexUpxS76zOjw8xPHxMYrFomub/hLdoLhlMplEOp3GxMQEJicnkclkjF9CJpMxvgt8NzExMeE6nslkjBjNSSV739/f7zpfzrlyed5+TxSNRj3nzOfm5tqOUZh4dnb2RP6/USHFUjsJhA0iHsZzfvAS/7KFwWwh1Xg8buyT9haPxzE1NYWpqSnE43Fjy+ojcHYJ4qvS7Z1/r2ttEZFuDPIuP8i19IlUzh/01bV9c1mH0g/R9pEPIsxKX+GgojRSJLiXuCp9FKVosF9h1vMaR6ScDwbxmwzSFgX1OwPQs/0Y9JyK1Y83MnbDjn+SfRk7dl+OB20RP+nrZvu+y3krKQzYbDaRz+eNKN7BwYGJwz86OkKtVkOr1TLxkPxMfvta/cJ4+1AoZOKIQ6EQwuEwYrEYwuGwK84rHA6bmBaON2KxGKLRqBHk4ziRosT0KwTccfMyRt4WPmPcy1ln2GKlJ3kdgLZ52EGJx+NIJBImjp0x7Yxjj0QiaDQaCIfDCIVC5lwoFDJlhIKUMo8F81tUKpVAPviMWZQxjel0GtFoFJOTk2ZsnclkMDU1hVQqhampKcRiMWQyGWPP8hhjUsYx7iRIzP8orh3Vc4bFScRZ1+t1U5fW63VT/9ZqNZfoLO291WqZsQvzt1SrVVQqFbPNuSTG0B8cHPiKbe933B3k3CB++YzflN9Jvzl2hpmfZ9DcPkHi5YIg/eBlHCngznET5Jwcf570OUVRFEVRFEVRFEVRFEVRFGVsUVFXRVEURVEURVEURVEURVEURRkWFHzZ39/vKeRSKBRc+0yMwgBrirxUq1UTZF2pVEwQAsX9/BIOh02gGQMhGCDAwJ9wOGwSznOfwXAMmGCwj+M4cBwHx8fHJujBcRwTWDcqbJFXBhERO6jNdnQH2gPd7GQGDBTvtG8HVwwqjGsnAtHEIO2cZMAzEyD4xa+ghte1DMCjMB/LGYP2gPs2HI/HEY/HXZ8rHA77DuwKuj/os/wkd+qXIAFjvfaH+axBn32Sghr8fWyBvk5CfvaawVZy3Ww2sbe3Z4T/GPzNZCJ+cRwHExMTZg3cD2JlOWg2m2afIinxeBytVguNRsOIL1OMnWXq6OgIjuMgHA672s9QKGQC/ZiQVQatDQvZLtjtlF3n222G3abY+7bt2Pt2m2Xv92rz7DbS3j8vweNBkPYty49dXgC0lRe/QpmdBDP9wjrGS7xvcnKyTbyvk5CmtFfaBu/js2Rf7CLag+IPW9DS7n8FFV/uJLpsl88g+BVZ7iSuzGN8hhyHsO61y489tlEuJvaYpNd4xUsglolYvYRlOwnR+sUuC1yzrLAMsE2R7QL7Df1cq0HqF4t+xu7nTUiWSY2V8wsFwAuFgqmnaZess9m/kckz2b+xE2/a93AsLpO69IJ1bjqdRjgcRiaTMXUz6/wgiTZ5D/s69j1nkWKxiL29vcCLVyKaVCqFmZkZzMzMIJPJIJVKIZVKufqWbCdbrZaZs3ccB5VKxVVPdhMR9jsvN0yhWHk+l8tp8sYzTLlc7ir6yiWfz7vER7z6l9PT0x2FX7uJwp5UAn/l/FGtVl2iC9vb29je3jb7XG9ubpp9u76myFIul0Mul8Pc3JzZpghDLpfD/Py82Zfz/MrFIp/Pu4SWOonAbmxsuMYgiUTCU/DVPrawsKAC2WNOPp93ibzKpZNIn71cvXp1oN/5R3/0R/Gnf/qnxsdjeXkZ73rXu/BDP/RDJ/be8bT5h3/4B3z3d3+3r2sjkQiWlpbwJ3/yJ3jsscdG/Ml6c3h4iKtXr6JQKJgxUy8fICaGPqvCrn/zN3+D1772tQCAj33sY/i+7/u+U/5E/mk0Gnjd616Hv/7rv+465qJfVrVaNfX/hz70IfzUT/3USX1UFwcHB/ilX/ol/O7v/i5CoZCv8eL169fx7LPPnsCnGy92dnbw/ve/Hx/84AfRbDbbxjnf9m3fZkSJbSjcaou2dmoTpGjrgw8+iIcffliF38cEP2Kt9+7dc83B+RVrXVxcdPliKOeHvb09l0ig1/hzZ2cHW1tb2NzcbPNLSiQSmJubw/z8PObn581YdHFxEXNzc237py0Urownh4eH2NvbMzbHba+13LbfmSUSCczMzGB2drZtzcU+nsvlLsz4wy98L+9HaJXiA3zvn8/njR8ARTAPDw9NHIKf90B872CLWyYSCaTTaSMwMz09bQQ7KNTBbYrTTE1NYXp6GolEYmz6KxTH7rR4zWnv7u629e8SiUSbCKsUYrXPzc3NjZVQdr9+80HOBfFZHLV42Fl+33hRGZWw6rB8QIYprCqvUX+n84Xt+237dtMPz/aLtf1hbb9X+oF4+ZsHsWkAbX7gfsVV6dNKP5EgwqyKMm4Mw6/Qb9sU1P92WO2Mn7UdB6T4Y1gCpjJuQQqr2gJnso9tx6/JeFa2IV5/W8Zw2iJww4BxrfweIpEIHMdxxZxTXJKfodFoGOFJv+1YOBxGPB5HJBIx8RVc6KfO+AuOCRzHwfz8vBFeTSaTpg3L5XJIJpOYnJw0MRuZTAaRSGSsxnKA20YA929v25NtC93utYV85bW2vUlbl/ZsP0P6Isoy0U+8Ti9kPCTnNoD2/AAy7pJ9ICJjHu2+eSKRQK1WQ71eRzweR6PRMGLCjuOgWq2iXq+b2FN+J7VaDdVqFbVazczPlMtlNBoN0+eTApN+Y6qH3TZ0uqeX0ORZEjTttm+XjWExrPjxcXhOOp3G4eFhW/w499kmsk6gIHc/5/yUA44/+Dk53shms23n4vE4QqGQqdej0ajJy5BMJlGtVk180uTkJMrlsvHJbjabrrZe1nmAu57rJa46yL29BFKHhR27LgXGgfa6UdYRdn07yL12/WzH0A9yr6IoiqIoiqIoiqIoiqIoiqKcECrqqiiKoiiKoiiKoiiKoiiKoihnnVELXtjroIKNFDtiYF0sFjPCR7FYzARPcDscDiMWiyEWi5nE9bFYzCQ/bDabiEajiEajRoSPQRjA/YBHBl/ZwY92YJkdQGbvewU3ykBIIHjgfFDs4BkZlAa0B0bYAQu2WNQwhHFt0UD7M9nP8LrH/r+87jktaOO2iCUDHTslYmA5YSAS7c8W/APakzsEDe4GXgw0ZAALA5ekEMTExERbsgb+5vK37fQsBjragmV+sP8nWzBXliW7bNqB0XYwtB1oZwcxyXJpl2u7XrADXO0AqV7/xzCxA5vsciMD7e2yaZdtu26wg5e8hOTsQH7789j1C9Au8mmLktJ+pMhrMplsE6+Ua1usj2tbkEzaDe2J19tBcn6wBcaazab5jvldskyFQiHEYjET7FepVMx3E4vFTJBvJBIxQb1cyuWy+S0ajQbq9bp5pt3G2vZm79ttkL1vt1n2/rCRQlS2jdr7dr1v79vtk73fq32z7dXet8uIvW/Xd0Hqv1FhC/LZwn29hDPthEG9hDTt+tIvtngfv3svkUvWM14isaxf2CZJUfROopmaFEuRDCK23Etk2S6HLFN2H8YvrD/9lI1+rrWFMr1ENZWLBW3Yq89Vr9c79tVsQXKWAbYpbKdkueO1/Yx5ggjAel0bpAx0ulY5f9Am5ZhfJiahOKYcm9C2Wc+z32QLLtuCykETbmSzWdPHYf+YdmonT2S9zz4S+160XY7NbDFmtfGLwSgSE3rdE3TsPeyEU53W9lzFSXN8fOxKIE4BTPuYvbAesqE4ZjabRSaTMdudjjGhHROMVatVVyJ4mSjeK5G8TDTP33x/f98ICfdCJgRjwnjWOUykl06nTT3G+qrXOSaXHYfxueKG9ULQZXt727OdZLkPuiwtLZ1q2VfOHl62K8U47WO7u7ttCQD92qsU71QB7IsFhUakbXmJv965c6dN3Il1WzcR2JWVlbF5v6q8SLVaxd27dz0FX5955hnTp4rFYlhZWfEUfL1x40bPebtXvepVuHnzptl3HAeO42BpaQm/8iu/gje+8Y1t4kp3797FZz7zmeH/06fEX/zFX+ATn/iE77G34zhotVp4zWteg9e97nWu90InTaVSwZe//GWXyBBFyff29lx1QigUQjgcRrPZRKPRQCaTwbve9S4sLCyc2ufvh1qthh/5kR9Bq9XCH//xH5+Z9rDVauEP/uAP8OlPfxoAzG/Bd7LhcBjZbBaLi4tGcE8K851WPf35z38ev//7v99XQvc/+qM/Otftyzd8wzdgZWUFwH0xxg996EN43/veZxJ2e/HAAw/gb//2b/Hcc8/h2WefxXPPPWe2Nzc3Ady3hcuXL+P69eu4ceMGrl+/bpZr167pXNwpYvfJvNa3bt1q86fw6o89+OCDrmMLCwsDibQr40WxWDQCrFKgVQqz2sdtP9pkMmnE/ygQyP3FxUWzPT8/j4WFhbERSVTGA9ZXfubWWIft7e15ztPLOQs5ntT5tfv0ek8ij3Xa7na+F53egXi9DwlyfnZ29sz0OY6OjjoKs0oRYnuRvr4klUphZmYGMzMzPcVZeW5U9e8oRVa59uvD3O+7tyDneonDKKdLJ79QW2DSFpq04x66CU9K/wv6FfmBfhH0YeD7P75TpF+DFJykHyrtj+8RaYd8Tzo9Pd0mSqmcXew60OvYqNZBCCq+2M899lpFVpVxwaudsH2xO/lc2zF4dhtlt0m2T59fbL86259OCh07juPpw8c2i+2R7cNn/43zjl8RvlEI+Q3jXD91fS/8CO1xHomx1Iwlk++AwuEwQqEQGo2GefdH0clGo4Fms4lyuYxQKIRKpYJWq+X6v+r1OiqVCmq1WiC/PlkOKJzXy3eVdm/7qnqVKbssdUPG1dkxoXashu2/2E0gdZj3DiLMOkzs/oCM6ZM2aMfr+RVQlc/3K6Bq/8YyVrVeryMajaJUKhkbl+1Dp1g4O/aN+7yP4xopUBmkreD/bY8notGo8cdmTJuMv45GowiHwybWtNFoGGHiaDSKarWKyclJhEIhtFotNJtNIzIpbcSuk6Q92cKQMh7Dzn1gx1/LstRvfGAvbF9OaVt2rJId2yxtyI7plLZn26+XWCltzLZTGeds27f92U96jmMY45ZisYhSqWTKydHRkSvXAReWE795MGjDkUjExIUy94fjOMb2KfzNmGnaOcsH46bp70CBZLZ1vYR7RxnHPyyh3WHeO+izZBugKIqiKIqiKIqiKIqiKIqiKMrAqKiroiiKoiiKoiiKoiiKoiiKoijBsUXEbNGKXutOwjG9BGX8YgtV2OsgImMMCJLBPwz0YVCPFFBi4AmxP7st6GHv20I3dpCfHTBoB6rYwVf2vh0IaAcV2oFeXgFboxayJXYglB245SVCZYtSDkPI0useWxzCj/ilHRTjJX7pOI6nQKwtYulHINYOZORv3+1ZfvEjEMvvleWuk0Asn8XvWZY1W1hpHEUbuonT2uVpEHFa+3caVJzWvt4OCPX6PKPCDiL2IxRt1w92EJqsH+r1OhqNBiYnJ039TKHVUCiEer2OWq2GZrOJSCSCWq1mEraGQiHzvVWrVbRaLdTrdSOKUqvVzH6lUkGlUjHB8ZVKxZS1IAwjgUuQa8vlMhzHQSKRwPT0tKmDSK8AyXHfH7UdDzPYchz3ZXvnJ4FRkGRHQa7pJxC3n3IxrGvs/ody8ZB9ArbnrI9sAVjZBvNattv9XGv3P/wSRCjTvpY27zV26nSt7C+ftvCacvKMug3pdk/Q+QWgvzZlWNfK5CbK2SZokt5B1kH7TsNIHhnkHnmtPSejnD3Y97DFjtlHYb0rE/lyToN9HN7DeRJ5Tz9JtgC4EszZieXsRL+saymkLBN02cmB2Ufieth9//39/TahVy/xV699L/H0ZDLpWwyWx7h0EriiuDWFYI+OjnBwcGB+O85J2kKxtVoN+Xze/Jb83dlP5v/gty/L34+/sRR87XaO81DynJ34Vp5TRgvbw6DLxsaG5zsKJpX3K1ShAptKUAqFgkvER4pMSFE+ecwrOWc6nTYCEjMzM2Ytt7nmdedZWE25z/Hxsafgq31sa2vLNa61675OIrBXrlzR8eWYkM/nPQVfb926hRdeeMGM57LZrKfg64MPPoirV6/i8uXLWF9fb3t+N3HXv/7rv8ZrX/vaE/1/FUVRxo2Pfexj+PZv/3b83u/9Ht7znveYBPfdiEQi5p3z0tISXvayl7XVzS95yUvGzp/lPNNoNLC1tYWtrS3TR1pfX8fGxga2t7extraGra0trK2tufx4IpEI5ufnsby8jKWlJSwvL2NxcdGsV1ZWsLCwgIWFBX13dcYpl73FMWVfWy5ra2ueImQUupRzCZ3mHVZWVnTsphjkvFcnu/Mz58Uxn710m067C3gAACAASURBVP86D4LTvd6r9SO0yvN+/J/lu61BhVbl9szMzLnzJwo6x8vy4CWg1Mneu9l/LyHbfD7fUQSM76nsd1x8z2HHPPB91cHBgUs8o1ar+far53snvmeiQFImk/F1jrbU69xZrwPOE7YYHdAujGoLpPJdqR1PYwviScFWGS9g+6r3gn5eMhaA/vv06bJjA2hnthiel7AqbZPv4dRH4OwxTB8uv2s7ZsoPQf1chuEro35dyrjAfk0noW6gvf2x2xu7fbEF9GzBVTt2yg92LBnbCvr1eMWn2W2SFIfsJThpx0WcJuMkYNrvuVHEknYSLutX4MzPuVqthlqthlAohFAohGq1at6tVatVxGIxtFotEyPWbDbRbDZRqVRMbBr7XI1Gw1OAslvbGfS76bSORqOIxWJIJBLGzykcDpsyxv+PsQCc64zH42g2m2i1WojFYq5YVtKvzQx676jE+waJqRrXewcZ97Fet2OWO9XzbEu4rtVq2N3dBQAz1mYbI9uWZrNpxFsZB+OXSCRiRCQjkQhCoRCi0agRhnQcx5xrtVpwHAehUAiRSMTlS8EY0Uaj4Srz1WoVoVDIFcctkbHCw6ZfGxmWrQ3rOeMmYm7HTQfZ53wh7Z8+rbRfjk9KpZIR0uYx3s94LuB+GWP70Gq1TMxyvV43At5BcBxn5LkcEokEEomEaStsv+hx37fzNMj4NK8cFIqiKIqiKIqiKIqiKIqiKIqiKD5QUVdFURRFURRFURRFURRFURRFUc4ODOzqtZaBYl5rBhszCM1LdImBOUETWxAGfjABBYOWpECon2tsUUtbBKmbIO1JYQtTeiXft4PZ7HtscVmve+zgKfseL3HSYQhZ2jbgdc+osIVgbfFLBp0TaQ+d7rEFM+176vU6EomEEa+s1WpwHAfhcNgEL9frdZf4Ja/hPoMbG42GS+yS95fLZXNMBoEGpZ+kGsM8N2zhjrNCr0BqP9ectXtGGZAriUajpry2Wi0TWA/cr3tk0DUD6eU+Azu5lvWsX2KxGCKRiFmHw2FMTEwgFAqZdTQaxcTEhEnKBMAkXwqHwya5RjqdNoHUTLyRy+VMcoDZ2VmTyAl4Mah3lAGTwxaNHbf9UQuvn1QSAq992h6FlKPRqAnsL5VKiMViaDQapu1pNpsIhUKm3T86OkIkEjFiy2zPmOjMcRwjEmX3GYfVH5RtMM+xLZEimAwoln1C9gVl289n8LuSbTqfoQHIF5sgCdT8XBPk2n4SqwHe9YifflnQ63s9S4Vmzz8nVRa8rlVhWeUk4NxGp7myTgLico7MTrRk9404/+MlOh4E9pVod16C4hwzcO6LfR2ZpGxqaqqrsDjHG+yTyb6WcrbwU/f2I5ps32PPK/YiaILXQdf2vFTQxOly6ZQ4MJHonUCdn2PYwgF+f89+z+3t7fnur8rv3itB/6DnzmMi/1FTrVaRz+dd4iy9trn2mi9LpVLGdmdmZlxrblPwOJ1OI51Om20VbVJ6US57Cwl1E7XY3d31tFXWJ0GEiy9dunQK/7UyavL5vEvo1UsE1kuUKpFItIm9em2r7Zwe5XK5o+DrrVu3zDx1PB5HtVrtOv7i3Nby8jJ++Zd/GW984xvxd3/3d3jta1878qSfJ0GpVEImk3G913ccBzMzM0bI+MqVK1hcXMTly5eNUN/S0hJyudwpfvLhwXf2Z4lPfOITcBwH3/It33LaH8UXTNZ+1r5nUqvVsL29jY2NDWxsbBjRy83NTWxtbeH27dtGAJPvyADgLW95Cx5//PFT/OSjw3Ec/MAP/AA+/vGP4+joyFWH9OJTn/oUHnnkEZf/jTJ8tre3sbW1hY2NDayvr2N7exv37t3D5uYmNjc3sb6+bsRcZXs2MTGBhYUFLC0tYX5+HpcuXcLCwkKb4P3CwoIKSJ1BOJezu7tr1tvb29jZ2cHu7q5rvbW1hZ2dnTY/xHA4jFwuh9nZWbOem5vD3Nyc61gul8P8/DxyuRympqZO6T9WxoV+51y3trba3gX6mWu1x2a5XM6VjH4coI8u3yvx3RGPFYtF4ytAoQ8pgHl4eGgEMPmMYrForvcrdJNOpxGPxzE1NYWpqSnE43Gk02kz55rJZBCPx5FMJpFKpYxAJt8VZbNZ875oenoa8XgcqVSqTQDgolAqlfp+v+A1hzUzM+O5pFIppFIpTE5OYmpqyuUDNTU15RK+HFSANYgYEN8z2uJftBfahRRS5WcPKs6qnC6DikwGXZ+GKOWg7xuV8UPGx7CuY53oRzTYFm3kMzqJAdvijUGgbwbrTdaDtDPaHetQ1pf03fAjMGwLCttxMopyWrCssUzJMsRyaccosr8jy7At9N2r7NrxcX5gOWL5YZntVVbpxyV9s/yWUb8xjrIPZ8ft2fGDdjsrfTLt2EK7TrPHHTJWSJ6zP4OMg2Q/lcg4xH7q0F7YcRUy1lC25/L3AdwiZ3ZMgfQL5e9MpAi7HYMo/avl37ZjE+Xvbv/tQftZfp7BcQPbN4r1BYHCqYyJ4j6FJaPRqBGEjUajLj8pbrdaLRMLFg6HjTBsKBRqE5hljCW/35OKFTqpGKGTvFfa8CiwfwvbL9yOTbb9EDkfwmP7+/umjmEdwj5ZvV7H0dGReadRr9eNECntibG7jUbDiEdSzJSxhtwel/fE9BkG7tcfbHdkmZL1ku3va9crXjHVdly2HR9i24ktlmv7z9t9T3seSd5/UnFcdnsEtNujV3+hV6z/sJ5rt9fdnsv6knU2ANMvYHtM+6XQ9klCW2GdTYFh1u8UyabgcCKRMLbMPhdtnfOaoVDI9MPYz+JclGzP7bH7sPcVRVEURVEURVEURVEURVEURVGUNlTUVVEURVEURVEURVEURVEURVEUxS/DFkXyc428NkiSGdKPmMswr7loYkj9iEWe5Xv6FesaBAZB8zUnA91arZZZGBjHYNNmswnHcRAKhUzAHtetVssE/vUjLAvAiPFRgI/il47jIJlMmnOJRMIE4QH3A+BkIigp/JfNZk1AH6+bm5sDAJOMjInEgPZAU2W0jEsZ9HsPk/c0m00cHR2hXC6b8sEAbpl8guLMpFqtBhYeGxYMyGYZZoArcD8QlmWOgdFMmsDAVwBGGCocDptkCgzmluJRDEhlMiGWTeB+eyZFtO2gf69jvfb53FG0k3aSAHvfDpSW+3YiE3vfTlAihb+89u2ga3vfTlxi79tJWLyE4IeJLZguE5G0Wi2TXIbtCIVWZWLvcDhs2pVoNGraJyZCiMVipv2sVqumzeIxtkdMuOA4Do6Pj80z+xFsBmASYMmA8Ewmg1AoZOxTJjPwElpmYgKZJIHJDKSN22LNMkmMJtK6OMj6wxZLZhIHeY2djETex7GRrIOYzERez/qO7aFdZwWBiTa62buXbcvruwk4e4k1n6TQuXK62EKaXqKYTOozyLWyXbavDQJtnklxZHvRSWTT61qWE2nbvE+WD6/2w29yN2V8GOa8WNCEbIMIi5+UYDLX2jcab2hbdlJwrtl/4ZrX2Yk0OyXD5Vo+I+h4h/0HO1km+y1cs762k2imUikzXqxWq2ZsItsVzi0Ui0UUi0UcHh6iUChgf38fhULBM8k+6/9sNmsEMCmC2Wuby6jnlflb8v9kgvhCoeBKkFepVFzn+Dv5PednToW/hS2qns1mzTnWHfIc+55+zml9c59SqRRIDJbbhULB8/1MLBZzibxKG5bH7WvktYrixeHhIXZ3d12LLWLktbaZmJjAzMwMZmdnXetOIsbcl8kBlbPL4eEh7t275xLxo1Aat7e3t7G+vt6WOHNiYgLz8/NYXFw0YleLi4uYn5/H3NycOZfL5TA3N3dmBQ/PGuvr67h16xb+/d//HW9729t83RMKhdBqtfDAAw/gO77jO/A7v/M7Y5OsdxA2Njbwt3/7ty7B1oWFBbXFMUe+W1HGi0qlYtqLZrOJV77ylaf9kYbOP//zP+Nbv/VbzX44HA70DvyTn/wkXvOa14zio517KIrYTZB+fX0dd+/ebXsPms1mXaKsUvBQHltaWrpQ/mlnGdse/AgGrq+vtz1HCmRKW+h0bGFhwZWQX7kYSKHKQqHgud3pnJcwEn36OO/JRe5nMhkjYCnH4ieRJD+omGo/gqx+3jHKOc5oNOoSwOR7QQpfUuCzl9BqPB5HJpPx9L1S7lMsFnvWp7btc97Tyw+Jc8qTk5NmjjkejxsfCooSAfffB9Bvql6ve76/C/qOmvX8sEUs7WdeVCHf02RYgqlBnhU0/mJYNhfkWaMWnVKC48fmvI71e87rmC2s5JdRiwF7/Q21YeW08COmyn4Iy5f0Y6c/HX1A6MMh/fPYjrCvzP607V/vFylax3fFtl+Hl0hyNBo1vquMWwLu+9QxnqLZbGJqasr0z4D7/nr0X+fcCb8LYosNyrgDOy7AFnCz66pBhFmHiayXbKFTO1ZAivB1EzCV9WAQAVPpq2GLEsq/bX+uYdetpVIJW1tbxt+p1WphbW0NrVYLpVIJ9XodBwcHKBaLaDabKBaLqFarODg4ML5E9Hmi/fD3ppgkfVdbrZbx4WY7V6lUAr2fkvFDfNfF74OxGbyu39iKXti/ie3DGsTO7LFHt3ttOxzVvcMmaPzbae1T5JT9LeB+XUehVNoX63zaGuMTTppOQpKMp2P7wLg6xh+wzYjH4664O1tMMpFIIJ1Ow3EcU8/JGNVBxHi99oflv+YloG2P/ez2zqvt9iNmOqrn2n6F/Tx3lKTTaZf9JxIJEyvXbDaNjXFOqNlsmvg57hNu8/+l0DCFiWu1WqBxO+PL5cI+UjweRyKRwOTkpCuuYWpqypxjXHY8HjfzmOl0uuMYyBb8VRRFURRFURRFURRFURRFURRFUc49KuqqKIqiKIqiKIqiKIqiKIqiKIpylrBFkLwEWhhQ7RV0bgem28HrMtiLgWF28HlQGLTE4FWvIHRe4yUI63VMBpn6PabJIkaLLRDoJTbplZjJvs8rSNEO/Ae8gxDtIEkvES8vO7aTATQaDezv76Ner5vn8XNJ8ViWMxmgWK/X+xaHHQZMGMCAXG4DMIG6DN5l8CQAE7DL4EVeL4VmgfsB3hQGYSKIeDxuRCL4PCkaIfEKvrVFOrzEy+zAcqA9uBzQcj5q7KQcQHu5s8u+XaaLxSL29/dN29NsNpHP503bRpFZll2WsYODA1O22CZRbJZJH+r1Our1ukusFnhRIJNBv9weJxjcDNwvxwwcZtmQomi8bnp62lUGGDBM7KQfQHt5sxNLSCFC+dnsssxkMRJbXNrr73s9a9CgfLuNsPftwHZ7P6iAub1vt0f2vp3oxv77Xm2V3V56tYOjgLYn2zXamzwmkzEC/ZcntjvAiwLpTCzhOI5LfJkJJNgWUIjZFkvnPUw0kUqlzDEmnIhGo5ienkYoFEI8HsfS0pK5/yQSqSqnB8uW7A9yDMWyLds6Xi/LIPuuLOuyneP4ieW8l2BtUDhmkol+WB/LOpd9JDne8hpTeT1H9sPYRnR6jva7zj5BBGDteQF5rT3P0O1alpNByoK0SVl3sy/iJTLea16B9izLhVeZ4zHZZ/LqFymnj5ewOPtl3Wyf9kxbZf1tz7H5eX4QaGeyHqb9dhJOph3KxGy0b9qlTLJNe+dzvQTHlfHiLCW1jsfjZn6IfQrOQXEejcncOV6vVComGV6nJJ1TU1NIp9OYnp42wgiZTAbT09NIp9PIZrNme3p62rVN0cxx6a90++7z+Xygc52uD/L7DSshfq9nnLd2kt+//A38iM/wensunEghGi7yO+62zMzM6FhWMTSbTV/Cr1ykeIdNKBTqKPjqtS33ZXJa5exQqVSwu7vrKbBlC695Ca4lEglPsTUvUS0VXRucJ554Av/v//2/vu9XYUJFUS4qjuPg3e9+N9LpND772c/iP//zP/Hss8+i2Wyad2Re/h6RSASPP/44fuzHfuwUPvV4Ui6Xsbe35ynQKrfX1tba3oWyP2D3EWyxVhXhHG+8xsS9hFo3Nzfb5i78jH3tvuXc3NxQEv8rZ4Mg8y+2WKXXvKO0Ob/zL1wWFxf7nm8cdO6v1/W2b0ongs7xeZ3rdn0ul1OBzD6p1WrY2NjAnTt3sLOzg+3tbTOXUygUUCgUsL+/j4ODAxweHqJYLKJUKuHo6AjHx8eec8PhcNiIvXAcTvGaIL4+w5gr9ru2fduU4eDlC2DHI0gfNjt+wCvGwI5D8IpVsOMZ/EKfAOkbwHecPG6L39kiePRJ5DtWKYpH0Rb6n9lilcrJI22LdiP9Smz/rm7+WtKW/fh+ef2doNCupP8WbZS2Jd/L00693uPTVqVfCu2eturlV6D1p3KSsO738o/08r/001Z4xcfx/TefTT8fL79rP0SjUUxMTJj4GACmDDLOjX6VwP3yWK/XXTE4jMOJx+OoVqtm3iIajZo+Ft+PUxAwGo2auAmOJ4IIEY5S+DSIEN8gIn4neW9QP5FRi0geHR2ZcsJyw758s9k0ts2yQQFitnd8BoXvOH9px8fwWsbRSNHTk0gxGg6Hje07joNIJGLKEssXyxH7YByrTExMIJlMGrHKyclJl+//adidV6yaH7z6vXb/wo4TscfzdiyKXQfYNmbHmdgxk7avrh3fZceI2XWsl6hkJ9+XYUGxRn5v9NmlXcuYqkajYepnKSQZCoXM91qr1eA4TmDRSMK+nPSNm5ycdLUn09PTpn/HZXp6Go7jYHp6GtFoFLlcDsD9eioWi2Fubs70Eb2EvXv5XHn5qfaqk/xcM+7P9YpNHhVB65FBrqF4dbVaRTQaRTgcNjGStVoNkUjE2O/R0RGi0aiJe2Y/i/bP8XwoFDL1A+sB9knou1mr1QLHqnkJ03MMzjELx+Hd/Js5juK4SI6n7HFONpttE1RXFEVRFEVRFEVRFEVRFEVRFEVRlBGioq6KoiiKoiiKoiiKoiiKoiiKoihKMGSAph0oP6jYLINNZdBnt2P94CXc0kvMJajASy/RF69jyvnHT3Cp17FCoYDDw0MjjATcT0JTLBZNQhngfmDq0dGRCdKvVqumrPEYg/zL5bJJzMXnMpifIksM6G82m2a/X5gohmuZJOCkiMfjLpFMWzQa8Bcs6/fYuD4L0OQ9Nn4Eb3js4OAABwcHpg0qlUo4ODgwyTPK5bJp2ygUdXR0ZJ4hRWtYdtl+lsvlvpIEECbiA14UdpYJC5iogNuhUAj1et1VJmRSDyY3oMjhSTJKsWcv0SrZbhNb8BZ4sf0mXkHhXm27l1hov8K4NqNIftDvPcVi0SSCoV0z6Uyz2cTR0ZERwWSyIpYl2p1s1yj6ZCeaGaScBMFuu5gshtvcl8eYQCESicBxHESjUZeAOoVomfwpFAq5ErGFw2HE43Gk02lX8iceo80MIyGEtg+nD8dSXkkFOXbyEqCV5Y/H5HVMViLHZByvyTGVTB4UVMBN4iVu6SUOC3gLbLKels9hYo9ez/Eax+kY62zSrT/W6/ywj9lJsYLilfT1tI7ZfRfl5BlEXLOfe+S1/dbtw7TDQZ5x3oQgzxJMfMw+BfslXLOe5JrztFzTBm0blddwbCCfFYRIJIJWq4VQKGTGG53mmaLRqJkTor0lk0mkUikkk0mk02kkk0nMzs5icnIS8/PzmJ6exsLCAjKZDObm5kxfvNf4bBzY399HvV7H/v6++a75PfO3LRQKaDabKBQKps/I35j32O0i7YF1S5AEjfJ7j8Vipnyz/5bNZs1Ynd8zx+n83dgXZD+RbVwmk0EoFEImk/GcOxhH/AqReInGbm9veyYmtMUTei32tYMIkijnh35FcrxEmYDuwky2IJOKFJ9NaDO9hNy87CQejxsR4G7ir5cuXcLKyooKAnnw4Q9/GD/xEz/RMWEtE8g3m01MTk7i5S9/OR555BHUajV84AMfOPF3dIqiKOOC4zj42Mc+hu///u83x0qlEj73uc/hv//7v43Q63PPPYdWq4V4PG4SYL/tbW/Dr//6r5/ipz8Z8vk87t2759muy2MbGxuu9qSTwLt97PLlyyrGOWYUCgUjgMmFIoJcex2zxwGxWAyzs7OYmZnxXHOxj9vv5JXzx9HREfb3940YJcUp5diy275X33V6ehrZbBaZTMaMJeR2t3O2zeXzeePHyrkxzmMeHBygXq+jUCiYeUzOl+XzeTOnxuv39/dRq9WMH9Hx8bFvoQk5Nx6NRpHJZMwcGefVMpkMotEoUqmUmd9Jp9NmvszPM5TeBH2Pw/Xh4aGx9cPDQxweHhrRVfqO0U+TQjJScMMv9FOh7wnFKSmExPnv6elp1xxrPwKs+q5mcKSfvB8xVc7Bs15h3SN9PTjXPwoxVcK5eNqCnH+nbxHrFfptSJ88zuFzbp91Ef05pEBRKpUyz/fy61NGS78+Fv2e8zo2iI9GkPfTw3zX7eX3pCijZJi+UcViEcVi0bzTp48v+yq2zzuvYzlluxQUKfgYDoc9fXB5DoAR5qNPgBTtY2wMfV1lXMywGQdR02E+y8s3nn0LYgtM2rF7dv/C9vmwBSO9BCFtkcugn8EWsbRFK2u1Gg4PD12iplK0clRQkI/0W1YodkpBYY7lYrGYEUelHzr9WOLxOMLhMKampsw5jhPYV5PjSvrd5nI541PD8ebMzIz5PHa8Rjf/3JMShhzHawbxwfZLv2WfdSiFdIH7QqiMa6BvOevZaDRqyluj0UA0GjXxH4w3oi97OBxGpVIxAsV8N8k6olgswnEclEqlvupp+opzfEJ75HgEgBlHxGIxE887MTGBRqPhEum146smJiaMeCa/L/res5wlEgkVM/W5P6xrTuO5BwcHaDabRkQVeHFszd/Aa0zOcYwdJy+vZd3Ado3PY7vVr1D9MIVU5XifZY5jeo77+V1pjIaiKIqiKIqiKIqiKIqiKIqiKIpygVBRV0VRFEVRFEVRFEVRFEVRFEVRFOXsc5ICL72ODRKMOypRl37uYWCfotgELR/dzlPUgwk/yuWyEYZmchDeZwsA8nylUnElbQhKJBIxSUK4LZORUZiMQrRMlBAOh03QuRT0o1ghk/Q4jmP+ZylqKxNJOI6DWq3mOnYSDFNo0ivI2aseGaXAZqf/qZOoh9f/6iUUehoMImw2qvODJgFgMDcAk1iQC+AuizLpCO0lGo0iGo2ahBDRaNQI+dRqNdf9THgokwDVajWTDIKJgCiYaH/vXr9Fr2N2gplR4ycZgd9jw3zWqJ/vdQ3rYyJ/n3q9jlKphHK5jEKhYNqL7e1tkyiTCaeYkBW4n3iKiRNkMipZRhqNhkuEVibelGK1PM9knK1W68QSagzK/8feucbadpXl/5lrzTXXfe29z9nt6SnlL0EFTVFAQWwK0gAalUjlgyIJGIyQEAUxTU2BpMSAWJVGIUE+ALEh0VDamhSQCEFAI0QgGNFqUqiABXJOz2WffVv3Ndda/w8nzzjvHGuMuea67cvZ7y/Z2WuOeZ9zvOPyjjHfhwFQgKt1FQM4MDgWABPwh9tUKpVDCcixrH1OugDiNHFYBseaJg4rg2QxEInrOMC1wCcy0JbrOPPAtohsb8h2hi9gmWzDyPaCbMewfSSPLdscMnCJbF9MC5KmHC0O0q8wbb0drG4WjoKPQaapiPjBYQeDcgmIy2CKDDgly2Q7QNUix5gVWZayjpZlKstilqeyTGZZLo/BcnyRYyirg/U+8xP/sw3B/8yD/M92SRzHuHjxIvr9Pra2ttDv942IAo/DAGpsm9CvMythGJr2MfOlDG5eLBZNG2JzcxNRFOHUqVMm78kA5q5A6NyX5Sfz4FEMZj6vwEGn00kIl2b5P4vffZoYwTwCBr7/B12vUdiEAhH8bf/n7+3t7US6qz2Rz+extrZmBE7W1tawtraGRqOBRqNhRCDW19fN73q9jnq9jvX1daytrRkxXuXkMRgMjACUFOCRy751tn8NuFonU+yTQq9ShCft7ziIb59kKBCXJv66vb2NH/zgB4lg38CkSJxL/JW/T4pQ9Tve8Q68733vM0GXgauBlcMwxE/+5E/ipS99KV74whfihS98IZ797GebZ/LQQw/hNa95jYq6KopyYnGJurrY39/Hf/zHf+Bzn/scHnnkEWxvb+OOO+7AQw89dEBXujw6nQ4uXbqE8+fP4/Lly7h06RIuXLiACxcu4OLFizh//rxZvnz5cmLfarWKm2++GWfOnMGZM2dw9uxZ3HjjjTh79qxJu/nmm3HDDTeor/uQYT971r/Lly87+4nsL7vaXr6/s2fPqt/3OmVa/pK+HvvvypUrCeEdiZ3P5F+9XjfilPTD0E9WKpVMX3RR/1PWeR6z+pRc69LSVHD1GnIcmmML9AvST0z/MccoOPbM98p3zHEK2/ds+5rlubIi5zNRcMyXlzjPQvqOy+WyEV3d2NhArVbD5uYmGo0Gzpw5g7W1NTztaU8z/rqj6Bs+DqSNw2YZy110/3mFKn2+6LR1WbbJur+O5a4GOd7PMkfO0ZH5hevlnBmWgcxjsgxjuegan+Vx5bHmnefIMS45f5hjuMxDsrzieJdrbJZ1nxyHZd5jnpRjaPZ5FGVZyHlvcRxje3sbrVYr8X3C5cuXzXzP3d1dM7eUInaXL1/GeDxGv99Hv99Hp9Mxttjv942AN+sLfichBfAoqMc04GoZMK8fV36vsAqkmF4ulzMCYTxvrVYz8y+CIEjMyQyCAOvr6+ZY9nwM+zsAe66//KbB/p7BLiPm3XdWcb5ViPkdxDUcxLx3fg/A83DuPwDzXU4YhmYeQBAERhh1OBya37SP0WhkhH3ZBmf9Kr8TmJUwDE2+q9VqRhiS9ZE9j4hzPvL5vMnPpVIJuVwO5XIZURRhMBiYYzYaDSNEHIahOUar1TLbUITSni9rf7/hmlNrb2ML62bdxp6fKOf8+rZZJa7veuxvkVzfHWXZxp4r7poblmUbu5wJw9C8/16vh36/j2q1aubS9Xo98x0Mv4dj3u73VOMFYwAAIABJREFU+2g2m2Y+E79vYJ9X1k3yWzrWJfzN/xRenff7VvltHcW4abO+7+sAGEFuvgMpzM10fsPD7xGCIDDXLb9ROCgRU9f3Zva7deUju+9k12Guvvyqjmt/uzfPcQ+Kw/je27XOFgyfhUX65otua3/fqSiKoiiKoiiKoiiKoiiKoiiKoijKSlBRV0VRFEVRFEVRFEVRFEVRFEVRFEVZNjLAAz92d6XJACpZ0xikKmuaDPoyK/LDVJcgkvwoWX4gKAONyQ+Z+eGgPK78UFR+ECo/KOVH1TKIg0tUUjnZMM/LAAOuQEfSTviRtwxw5BJplsEPeEwZRE4ek8GTXEEMZkHaV7VaNYIhUjSM9lIsFlEqlUzAbgZMLZVKJnBFuVxGGIZG/JJ2WalUMBgMMBgMEqJ8uVzOfBTP8/C+gasfgjNYgGQRUcwsAUQOMhCEC19QqEVEK4/T/kxn4AQGi2CZz/clBTWPi+jsNMGyWX/HcZwI8kKxnziOTcBG4Jq9AVfrxGKxaAQTuY0M/sjAR8u0q+vRbucR+FwkbZFjMagVgES5y8ArDBZRLBaNzQEwbT9Z1/X7ffP8ZQAXHoOBTLkvg7dwPQOLMZDLotPKZJAWtuX4m2K80gYYeAmACcrCe+O+/X5/7gAzs2C/K1fAliwBe1wBT+z97EBlQHaBczv4kOs6XYGNXMLrdrCWo9beZlki223TxGHZLpNtOVlOSfuRZRX3A671yWTbT16DK7jVPMh3IvPINCFZ+Z58QrIyH8q+mu+czAsyb2qg4KOB7GMw/7psQuZX5nlXGnDNdqb5LNL8DvPgEtBkmsx7Pj+By9fgs40s9iCDZWmg4tUyb3DsrGlZt583kDYwv6jxMoSR5bqTLka/bDqdDlqtFvb29nDhwgVsbW1ha2vLCA9Kocy9vT00m00TkFfmtbT2ahAEJtBhEASmjTGvuOxhBog7rLKS7bSdnR2MRiPs7OyY+pFtPtZVfCesH9nem+UYWVlfX0cul8P6+rppk7NtJsV8Xf/Z9ud/7mf/Z922aDu93W57hWB3dnbMH/P6/v4+9vb2sLe3h52dHezv73v7+BSlaDQaRoBFisBSLNYWht3Y2Egs230h5fql0+mkisHayzKPugRhGTiZf1mEYOWf5r2jw+7uLp566ilcunTJiNDxNwXoLl26hIsXL2Jrayuxb7FYxA033IAbb7wRZ86cwebmpvm78cYbE8v8O4781m/9Fh566CGUSiXcdtttePWrX40XvehFeO5zn5sqBqWiroqinHSyirr+67/+K+6//358+tOfxng8xnvf+17cfffdR0JwL45jXLp0CZcvX06Isco6U66z5wtVKhXceOONuOmmm8x//r755ptN2tmzZyfGFpTV0u12vW3htHbylStXnP009rdOnTqVEMtMW+Zv5fqg2+0afxb/pI9L+rrk+p2dHZPm86HW63XUajVUq1Xj3+A8iEKhgCiKzDwnjnNRqGQ4HCKOY69vN6u4ZhZfVprwatZt7PHYk8I8opbL+j8LfE/Mf1J0hnkQgBHJiePYCKQNh0Nzzmaz6fXRFotFrK2tGV/D5ubmRFkq85P8O3PmzIny43PulhxDd6XRV+tK43xnOQZKny59vfTjyvFPnmeefARcGytknpJ+WPrDpYhXpVJJzM9gWUFfL/29ctyT4zrML3J8X+vfxXHN25Vj4Fwv85Zc75pL7Jpr7Js3zDws1y8yfx9wz+GX84mYb1xiq8yTLoFUW2xV5lOexyW2qlz/2GPottCVXcba42l2nrfnVHW7XVy5cgUATJkv64k4jo39xHGM4XCIVqtlbJvXSFvldzJc5jVw2/F4bOxWikKu2jfKtiPtJgzDRBpFLbme9QaF7ljXcO4l6xbOaVlbWzPHKBaLZhyRdivnDsp5MdyX12LPM6QwH6lUKon3W6/XjYig3IfvjNj5JMs2ACb6Afa842n5zxa6tL/jcF3HQYia2mWoPc/DnsdpzxEtlUoIw9Bc53g8RqVSMW3bXC6HYrFo5s8PBgOUy2Uzxwu4midpM51OB8Vi0dgO54flcjnTzgqCwDxbKbK4t7dntp9XBI9tIQCm7mJ9NRqNEvYRRRGGw6ERT6UgKr+X4Zx8mafDMEyUY9ym1+uZ58xvZHq9nnkXWebHLyL+Nw/zzFM/qtvIPFMqlVAqlRLfoHAuH0Wza7Waea+u70M4j4HlfLfbxf7+vhHRpsBps9k08yaZ1m63zbwemZ9tcW32V5jGucuyXpHrj8u4W6VSMfUJyyaWS1JM1dX+s/ssrrnvrjnsrjkurvn2WebNL0t81TUf/3rlIOdjTkubt95dxpzLrGnTtj+pfkJFURRFURRFURRFURRFURRFURRFOYGoqKuiKIqiKIqiKIqiKIqiKIqiKIqinBRcQq+uNAavcaXJgAcyoI38SF1+OM2PLmWgG98x5kUKtMiPa12iLFlEaZdxDJ+wrXIycX2ILO3AFdDJFVBKBriRx5T2J4PqSFt0iZMBk0FQ5kHaoM+WfIJJLgExwC3MJD/ClwFtCoWCOf9wODRBNxiwbzAYmKAeEte9u4R97EAvxCVi5Qrq4RP4dQWZcImGuALJAG6hTN+1HkTgmTRcooY+cTg7sALgFiR0BYIYj8eIogiFQsHYDNNLpZIJvgHAiKsGQWACdgBX35cM0jIajZDL5UzgDQCmbgzD0PwejUZotVrI5XIYj8cmnUEFKbC8SKA21/OQNiefs8/+ZAAg+dtnlzLwgMtGfYJ/rvfrysuuPOuyI5e92DbsElZ0tTNctmPbiMtuXfbpKjPsskG2gciiQfvmwRWAZJq46Hg8NkEnGdQGuFoeM08xaFoul0MURebeGcCmXC6b3wBMcKRcLmfeDevFfD6P0Whk3r+0YwZys393u92FBJ0BmKBQvDcGiMrn8+aZUXwLgAkSNRgMUCgUEsK1vE8+n16vNxEshnme9hfHsQl8xeMwUJAMdLy/vz81b60aVyAmV7ltB++ZV/DWVZa4Au7YAeiAwxW8nUeofFm/s2y7iJChZJrA+bIF02c9z0kKvHTYTPMdzBrA2SVeK48NXGsH+ESWZxXjcyHrRVmGyHLIl+dkWTJNnHlWQWZfH0qZHZl/mP9k3mWek/mJeY/98GUfY1ZkHrQDN8t86xKDlflQ5j2Zf3lMmU9lfmQelHWjq44/aXQ6Hezv72N/f9+IYHB5f3/fiGTItL29PWxtbZnlZrOZ2mdgoEsG+WVfmO+WbeswDE0wUvqemE/ZXx2Px3PlQbYD+c5ZPsq8xTKQ+Yp5kXlKlrV20HtXMHGuc7UvV4EtBM36i/2+7e1tY8M+YVgegzZvt9vsQIhZoY3b//n8+Z/PTz5H2rb9X24XBEHiOPI/n40UFNre3jb3MC2N6Wn+KimC4RPFcKXLtBtvvHGir6JcX7jyVpa/K1eueIMuu/Jalr9yuaziG4fEYDAwgq9PPfUULl68iMuXLyd+b21tGXE72+eXy+UmRF5d4q8yzfaNHAb//d//jXe/+9145JFHAABPf/rTcffdd+MNb3hDqs9CRV0VRTnppIm6jkYjfOYzn8G73/1ufOMb30ChUMBgMEAURfi93/s9/NVf/dXKrovtmvPnz+PcuXMTv+XyxYsXjY+MbGxs4OzZs6ZtcvPNN5tl+ZvLyupoNptGFNP1lybO6vJnsp2ZVYxVLmt/6PiT1uehCOWlS5ewtbVl8pL0O/nGgsIwNL4X4GqbOAgC5PP5hNgVRU7oP8rCxsaG8VHQt0N/JX1Jchv6k+xt1tfXEYZhYhv2+V1jk9cb9D3RJ0U/kxw/oV+Ffmn6YORYCX3Rttglt+Vx6c9yzbmYBv11fId8T/QD8v3Rn0dfMvNAt9tFqVRCr9dDFEVot9vI5XJGjIeiPq1WC+12G/v7+2i1WsbPure355wnBVwTY200GlhbW8PGxob5bf+naKudbo9LHxXkmAPziyttWfOE08b/fHPVsiB9yS5RSvpabP9yFjFVeWz6T3kOl39aycY8cxRWvX4ZAmtZ5zgc5PqTUN8dF+y5jEdtudVqJeaqjsdjYxvA1Xl5nU7HiNWNx2P0+32zzGMeVX+hLX4KwMxT5Dg1x0fL5bIZx6pWqyad9QvbJlzP+eWVSsW0jznGWi6XUa/XjRAm6xC2W4CrdlosFjEcDrG/v2/GNYHJOaqu+aj2/K0s27jmFmTZxi4rD3reoT0n0DX/z97Gnndg72PP33PNM7THbez5hLVazYyfE9oX53OGYYjd3V0zv5XbsC0URRF6vZ4RqOS+3I7vh+dge5zbUZS41+uZthXnq3NMfxHY36NoMHC1XKB4YxAEpuyQ84+WMbcwK/a7d80FsOvFLNu45njac0OzbOOad+raplgsJuyMfWxCUWdZhpfLZSNKKskiMruqNPY9ARifAOc28X4ojMptgGsCwIT5+yhBG6A90BfCNNYpzEcU2qYfBbj2rguFgkkLw9DMmWG5FYahqWvYvgvDELVaDeVyGeVy2djcskR0fWmu+dXKcpD9cFkf02cj63Y5J0i2E+i78X0ryPpGfoczbW7ivN+OybqU+UaWgSz7ZBnMMlzW05xz5JrXyPp92rm0P6QoiqIoiqIoiqIoiqIoiqIoiqIoyiGjoq6KoiiKoiiKoiiKoiiKoiiKoiiKohwNpom1yA9aZSAp+cGq/LiVgajkx6s+kZdZRGkXgR+i+gRhZAAKlwiMDCQlgwhkEbaV6XJ7eRwNVHVykXlc2pr84Fvanfyo3CcSJu1R2rQUPKR9SzuV55c2uywRRJ+QpUu80idYKe1GfpRuB/eQIkvyvPIDdTvYh2+fVbJIAI+D3P8oXKtLzPcgYRAQTvfh/zAMEYahWaYgJYNMye2Zz2WwlDiOTb6TAYqWEZgnl8uZui2fz5tADFJIM5fLJeyAAbQY/EQKazYajYTd8v43NjZMeqlUQqPRMMFUpF3KQE2yDJB18EEJBElcQQZl2QkcHWFc17W6ymhf+8llRy6RyWUI8i0ChZ/JeDw2AXukXUnhVQZyYzp/y3QpEr0oMlgQA6Xw2mVZIQMI2cGEaF9MHw6HKJVK5lisDxkMj+mDwSBRn8lnEoahCUA1HA6NnUkRbTJvuZx1v4MutxcJnrSsQExZ9xuPx+b9sxwpFoumrKCwMtOkPbJsYlC6MAwRx7EpGyh0DiTLH1c7MEs7dBF87T2XQLlsl/mENrOIePoE0mVdY7cB7QB1ynI5DAHlWX/b9fI8HKbYcpZtNdjWdFYlLusKHOcLBsfzuYLGzopLQNaVJ2RZLP1kLgFZWT7LcpjlrU/MludyBXM9ymxvbyfEXykUK8VgKRgjxWOZxkDOaXUqn3+1WjUiGgxCLAUZ2O+s1WqmT8dnziDQAEzQWV/Z6fsPzB9YMK0syrJu1u1d65YZCDPrM1v2/0We+Tz/i8WiCWQ5Go1M+5PtSorgMj+58vre3l5qWbW2toZ6vY56vY5Go4F6vY6NjY3Ecr1eN9tVKhVUq1VsbGwYm6jVaiqkfh3SbrdTxbbk3/b29kSaKwhxsVjE+vo61tfXjQDMxsbGhEiMFIix17NeU1bL9vZ2QiTPJZzHtB/+8IcTPitb/NcWy7PTViUy/fKXvxxf/OIXAVzzT0VRhNe//vW466678OxnP3tiHxV1VRTlpOMSdd3b28MDDzyAP//zP8dTTz2FXC6XGJvJ5XJ47Wtfi7/927/NfJ5ut2sEFn3irNPqGpcYq0uo9cyZM+pXXCKtVitzO9FuK+7u7k5tJ2YRY5XLR0FMXslOp9PBzs4OnnrqKVy8eNH0WS9fvmzKhHa7jVarZfq2zWbTCJ+02210u130ej2vcA/H7aQvZhoUFSmVSkYc1e6jM7+l9eOnbXM99J2XLZ5KH60c56f/lb5djrPPOzeRvjuXkKUtdsnxLm5rC6xyX+mv5XvlsRqNhvE/7+/vm+chBYdtQWLfNpcuXfKK/Nh5z/WXtv7UqVMJEbBFSRsfOug0ex7JLMzqI11lmgoLJZHljhwPcgn4yPUsm+R6OX/HJRYk18vxI673jS/Nw7RxGjkO7xozkus5Hu8bU5Lj9ZwL4BtzskX+lHRscUrXXA57fMeeq2PPW7OPac9Fs8ct7Plr9jw1+5rsOXB2PeuagzfvGFVW8vl8Yi6wnIPFZ1UoFMz8q9FoZOaacByF/VVuv+jcTjnXjPPGeF1Mj6LI/OY8M7ltFEXmOimuHcexmdNWKBSMOCXnk3F8mz5TzjkaDoemfxvH8cT9zTt3zE5zzaNcFcua57XoNpyzJ8UFKeYr5+e2220jpE6azSYKhUJi/KLb7U7MbW+322buYa/XQxRF2NvbM7ZNwWEpbke2trbMb27fbDYTc9h4XDn3ku1S2gy35X5M43gkOUz/uJyHSThPmv09AIn3wm+I5JybIAhQrVbNfGzazubmZsJP5MoXdt+N9a20O/YPgKvtgUKhkDhOv9+fuI9CoWDeB6lUKqYfBSTnq7rmSbns0zX/01Vm28dzlfWueTCuedOuum5Z37HMCgWnyXg8NmWvnKPBdhLzuhTvlfOXpb0s83sB+a0A+6j8XSgUTN+U+ZnzicIwTHwHJ9turDdOnz5tRJZrtZrxtdnfCLA9KOcNab/j+OLq9wDueW+yrSjLEVcfx9W/9vWnZBuVZdEyvi2V891cc+t8/RjX3Djme2kLLqFV7psmtKooiqIoiqIoiqIoiqIoiqIoiqIoiqIYVNRVURRFURRFURRFURRFURRFURRFURRlHpYp5rLsYywaTEniE2NJWzfr72Uf63oIFKikIz/Sl3lffjgv7UB+nC+D28mgHrMIi/kEbmX6ssSgiQxSliYEKz/cT9tHCoz59rHFbnz7yA/+0/a5nnAFewHc5a9PlM4VVMYllukTU3IFanQJcPryoh0cDfAHndne3k4EehkOhyagrAyWMxgMjG3Ke2u328daKIDBMvib9iaDreVyuYQ4A20CgAlIw3Rul8vlTCAdBs8lxWLRBLGRgdIojFuv1xEEgbFX4GpQjdOnTwO4FiSN+AISLiv9MDhMkedlHFPaNQNXMUizFIPtdDro9/uJYEr9fj9xzNFohMFgMCEYOxgM5g7oukpkALIgCCbsijDAEtMpHEbRZ+CqHTHwE4CEeG2pVDL2Nh6PjWA0AJM+Go0QRZEJZtPr9RCGYWJbKY5bq9WMUBnToihCu91OBBSjqK18H0EQTOSF4XA4UW/YwcmyCjpnEYdedvskC66AP7awowy8SlwC2xSUk3Yijx3HMUajEarVqhEEI8wztBUGw2RdOBgMTN7s9/smndswsDbPw98y3c4Hi7JoX2oZx5j3XCpSOx+yzpBtNVcfRtqzDFLm66v4+kcMOCbLGl8/aBmC67K/IIOJyT6FLDdkv0OWHewD+YSZ0/olMjDaSezLzMsyfVvLXO/q18zCMsWLlymsvIrAkvL5uQQVpgkw2NtOC25ti324xBZc68bjMer1uglAG4ah6XvZIsey7GI5J8tElnG89mkCyLyneUROgWSbhoETZZuGZRfvW5Y7LONkucT+lxTWtYMw0o8jzz1P/uGztP8D13xm/G9vx+ds/+f74n/WP/b/rNi2xHtnUFj2sYGrdUIcx8jn8+j1euY/+1hxHJs+WLfbRavVMuk+aJuNRgONRsOIvq6traFWq5llWxB2bW3NLMt9VZzpeEORYd/f7u4u9vb2sL29bX7L/z6x7WKxmBB53djYSIjA2sKwrvXSV6QsB1kf+sRfZdqFCxcmAoy7xPl8grC33HJLJhH65zznOfif//mfifRCoYA4jvHLv/zLuOuuu/CKV7zCrFNRV0VRTjpS1PW73/0uPvzhD+ODH/ygEQXx8bKXvQyPPPJIqjirvU4i+z0+cVYu33LLLSo0sACuvm/WvytXrkz42EmaqGDaX7lcPlJjaycJ9kXZR93e3jZ9WfZd4zjGzs4Oms0mLl++jP39fezt7aHVaqHZbBrByna7jX6/j06ng8FggH6/b4SCpDBKFjguRuESCl2VSiVEUWTESRqNBkqlEk6dOoVyuYwzZ86Y/ER/Av0E9D3YfqXj6N+0/X9ZRC6Xtc284l5pQrjTBC4X3WaWd5xVaHXaNmk+QZ8Qa5p/kH8UIeI9RVE0VXRF+vo4TupKk34uVxqP7UqbB+nfcwmpME36DV1p9BdmTZO+Q1faSWQZc3IP4rdLqGwWDntsw7f+ONZDPrLMBZpHbPKgjjvvtaxa5BRIihaOx+OEcCTrBopEAjBjRrQZOd8IuDrOzfFkzpnK5XIT8z+4Let+Ob+k3+8vZJNBEJixbjluwXKZYqZSuJHXxG3H47F59rzHbrdrhPB4Lxz/p63ZczQHg8HEXNJVYY+1A5PjVa5t5Nh91m3sOUacAyS3ieN4wlcunx+PE0VRYt5WsVhMPFveR6/XS8wVKxQK6HQ6Jk0KpS5zPmKn00m819FolGjPMJ1jXmQ8Hk/1dRxlpEipHC9mmj2fltvLdO4n5w/yOLRHpkdRlBiTBq6+Uwops/1F8UlpVyyn9vf3TdnB7QeDgZm/Q9FWwD332zd3e5ZtXW0K3zyfRdsfy8CePwhM2jvgHvuuVCqm7GO/vFarIQiChKgv52GPx2Nj15VKJWEv3W7XlBcUOw6CIDFfl/0FzgnlOeT83H6/j36/jyAI0Ol0TPr+/v5C9aksF2U/QM6tkuL2nJPgE4iUz1i+A/qv5Nwr+U2JS4TSNedTOdrIukbOK5RlhSxjZL9czsWRcwtnEUGV55FzGbmt75uZWZgmgirztdyWtiP70bJ/I+2M9iK3lTZJ25I24poPrSiKoiiKoiiKoiiKoiiKoiiKoiiKohxZVNRVURRFURRFURRFURRFURRFURRFURTlekUGXJAiF/JjaflhtPzQ2haP8h1LftgtP8i2P6hm0H8gGWBCfoy9SEA0Gxl8wBazlEEzpdiQ/ChbfoAtP6C2A8ZIMU0Z2EAGLpAffmc9lnL9IW1I2mCaEKy0Fd8+tq369pFBFdL2kXboEyOdl6ziSdIuffvYgUB8+9gBEOzgLrI8kLYKJG3aFSxKWQxXme8TG3QFUHUFG/IJH25vb2MwGBgxH6ZvbW2Z7Rh8t9lsIo5jU8+Nx+MJ++N0KxmsazgcGjui6K28Bh5L7s+gP/wvgyQddsCkWSmVShNBzwAYcSEZqDEIgkTgUBmYqdFoJJYZ+IeBVRgQhfjqzsNK9wWlXFb6IrhEp1125AuKI4M1UuCy2Wyi1+slApXJoOfdbtcE2tnd3TU2QvGsdrs9EQhvb28vYWPA1TaqFJFmYEUG62W6DL4FXAvYJQPbHWUoAC3zFoPGAdfEcCnuLAUz8/m8sStuA1wNJESBK65rNBoJ8VumyzpwMBgYQTXC4GQMgEbBM56/UqmY4GTy+mUgTBLH8URgzU6nk6ijGXCQQfMAmDJaXvuyBG99gu0HAd+pDfsjzLtBECTKQeZxtnt8wRhZRzF/yXfEoPM8nl1OLArzCwOXyqCKso9Wq9VM4MRSqZQIcrq5uWnyiwyOJduGQLJdJ9t8dntQlrEymJZ9TSpMm460K58ALf0APgFaWQ/JtqFPgJZ1kU+A1hdUbhnMIwS7yn2y9JlOWh+G+U2+e5mXmH9kHpwmBCADGDIvyvwn85zrXMvIh66ggbLscgUg9P2Wx5C/ZTnp+y3zlvxNsftFxGH5O008h6SJP0wTi7XXnTlzZmo5L8sY5gH53pnvWCbK/MU8xXt0CdDKPEK/EfOlS4B2VuQ7ZF6hz1OWEaxf5bu1BWhlXqK/RPpRXMITPO608oj3x/98PvzP52v/t8VXbJEWHlM+S74zVwDkaVBYl0GjaXuyLZXL5Uy/n+8/rR9SKBRQr9dRLBZRKpXQaDRQqVRQLpdx6tQpnD59GpVKBY1GY2re39zcVPGvYwTz5fb29oTgq/ztWi+3cxGGoRF/XV9f9wrC8q9er6PRaKBer6Ner2NjY8MZSFvJTr/fx+XLl83fhQsXsLW1ZZYvXbqEixcvJtLscr5Wq2FzcxM33ngjNjc3E39Me9Ob3oRLly55ryMMQ8RxjFtvvRV/8Ad/gNe//vX49Kc/raKuiqKcaIIgwH333YfHHnsMDz74IMIwzOQP29zcxOXLlxNpGxsbqeKscp2KemYjiyBr2vplirLKdvfZs2d1HseCsA9m/2dfm31iCq7awqv7+/umj9fr9bC/v492u41Wq4V2u23yRb/fR7fbnbm/54OCQBSIkqKppVIJ9Xod1WoVGxsbqFQqOH36NKrVKm666SbU63XcdNNNWFtbw5kzZ46kcIPLL5Hmx3D5PaSP1/aTSL8Kt+c67jevWAd9ovRNSL8FfRr0S/C9SX8Yy2X2fWjzfE9yPgp9HjynPTaxKnxl4qxirNN8blEUmbFD3j99E1EUIQgCFItFFAoF46ekX6RUKpkx21KplPATZhW0dI3hzUoWcdyDSrsex5akH02O2fh+S1+m77f0p/t+y7whf2fJX4vg86PL3z4fvPSD+n7L8sX3W9YZcgzINV6wSuR8X2ByzolrDMQWiLPfi2temD0nzCVUN+1aZB3luxY55uO7lnl98bPiahfY84TkWNBwODRzPeQcmUqlYuac0U6ZX9gmKpfLZr4Aj2M/F56Xc+vo5+YxOebF+Tf9fh+5XM6MEY1GIwRBgOFw6BRPXQb2XBp7TpWcU0FBv4P0f9nzTmXdMEsa62N7GwoS8lysg2k7bBfxnURRZMYL2MbjOBDTKIRK+P7s89t9ZoqFkkKhgDiOJ+zWZcuueWquNNtefWkHgRQJBa7ePwU7XeK73GY0Gpk2E98T0/k+aSNBEEzML13WtefzeSNEKQWGaVNS2JRzkjjvTIqjSoFV3juFl7kvBXLtfCTzjEznfEA+P8Bfn7vSD6rMTsNl1770o7ot5+uVy+WEECnzOnDV/mQeJeyD873SRlkmUNBQfpvUbreBeqm5AAAgAElEQVSNrcj6Xdbbsv3pa0Muipz/K+tgOT4m56jJNpmso2X5P4sAqk800iVGqRxNZhVH9QmiyjapPX9V2oicv+2zF9mu9vWtFsH+dsclBLwqEVSXTfmEixVFURRFURRFURRFURRFURRFURRFURRlCaioq6IoiqIoiqIoiqIoiqIoiqIoiqIoinJ0mSXQm/170f3nOdYyA0YAcAaDc62bZdt51y1ynOsxaN1Jww4UIQM/pAnByoASvn3swGq+fWTQi7R9ZBAXV9CnZWDncRk8xQ6eJwNJAEnBMTuIhAxgYYuepNnVtHPKADRp51SOJjJPSzuwg9zKoC1XrlwxeZ/iNvwt7YUBjhnoTdoVA8pIoWgGgqHQDqGAEQP4SbtjQN9lQyEdIBlECkBCXIdB0uyA27wvGfhPihsSBk7ziSUeFrME5NL0yfRWq2XsigKjLMftukcKpOzt7WE0GqHVapnAxOTSpUsmXzEgNW1L1o2dTsfY8Wg0MvbCYHU8BtfRrhk0k4Hxli2+aQswM7AfYVBBBvqUNiWDhB4muVwO9Xp9wt5LpVIigCZxCRXZwaeIS3iVQarlMRgMVdbLzAcMDA4gUa5QIIzpzJvlctmIsDItl8uhWCyi1+sl8gADZdr5wpVXmDcl3W7XBF8lDJDP4zDQYq/XS+SN8Xg8EST3oJB5kddE0eFlTo+m4CwpFovm3AyECcCI0pJyuWzqKynSHEVRIlBvpVIxAVplgNVisZhoO1YqFRMgtF6vJ8q5G264wZw7iiIjACcF/E46y/QdHOQ+so23KFn78fP0/Ve5z/UQrFIGSWSARdmedwnIyuCNsm8r+wcuMVsZGNL3Wx5vWf1mX8BS33uWARr5ezgcJupEvvdOp2MCjXe7XVO3sl5iwNh+v49ms2n6QNNEddfX11EsFlGtVo2wJkU1i8Widz2v114fRRHW1tYS97ZsXOVFWtq866ZtbwfEn4WsghjT1i/rOPazneVZZNmn2Wyi3W6j3W6bNObdZQebZruDgkNRFKFYLBohFtpmtVpFuVxGo9FArVYzIkQU9tzc3MTa2hqKxSLOnDljtlX/7tFjZ2fHKwi7u7s7sV7+5zqfLZfLZSP0ur6+nhB+rdfrCVFY/jUajYltj5pw1VFFijCdP38e586dS4gy2WkXLlxICAKkwfp0Y2MDL3/5y/HQQw+pqKuiKCeO0WiEj3/843jd61431/6VSgWf/exnccMNN+CGG27A6dOnl3yFx59ms5naHvG1WXZ2drCzs4O9vT1vvTZNqJ5tWSlsL/80oP81ltHfmfX/rL4t9mtssSD+p6CYj0KhgGq1avpEzBvMCxRfPXv2LBqNxoTgL/uOZ86cWXofKKsAKn1D0pfD58jnKv1G9BW5RFrTtp8VOX+BcwykwAZ9eOxvy/GWZQusHiWGwyEuXryIra0t7O7uYmtrC61WC/v7+9je3jYCxLu7u+h0Omi1Wmg2m2i1WgkfFn1b9vijDcfBpRiX9J1K0T4pyDcvWX0uB72tFDs6LkixHVkeyPlYaSI+vt+ynPX9lv1/3295ffL3Irh8wPbvLHnA53uWv+VcKN9vWWb5fgOTIqMu8VK7frPHdlyi1fZztX3z9vsHsgmpznMt9j2uan6dCzmHDZic40ZRKPms6vV6wg9C8Tng6thHEASmXqEgohzHZxuC8+PiOMZgMEA+nzflJreTz4VjORSw5hgGxym4jRQ9tQVTpZAqj29vJwVIebxlj3fPC+sbQrFvOU4fBIERHuVyPp9HsVhELpcz6fRbk1wuZ8bQ5XbVahVhGJo5FrlcDuVyGePxOPF+KFgr2zWuORbMKxxP4nkJ50/I+QBsn9lCsoPBYKKenkWE0pW2LKG1efEJP8p3NRqNzBjDcDg0+TeKosT8PM7RKxQKifxuzz9iehiGCbvju5PbcT9pq8BV2+c6Od/PtuNFofApj++ag0iYhzkPTR6D5chh4hO8c43F20LF82xrl/ezbuub5yLn0MyzrW8cd5Ztfe06IFkvS5u363lp+3ZdLfts9pxeeW5bPFm27eT8A/vcy5qDIttQPqF7IPls5buW/Ts5h0+29WU7wdc+lNch+5ryOnzzAZXDY97867OdNHHUWQVRfWLCqxZHBbLZiPzuw2cvvj6Xrw+V1Z4VRVEURVEURVEURVEURVEURVEURVEU5YSgoq6KoiiKoiiKoiiKoiiKoiiKoiiKoiiKsmzkh/3yA34ZvEMG2rCDB8hgADLIAJAu8GIHOJDXYQf9kEEG7PPLYAjLJE2EUgYVsIMVpAlo2oFmZNACGVQASBfXtIPFyGu1g7epCObxRQbgsG0izbbsoHhp9mMH6UmzQ/ucMlhI2jmBZJDGZTHN9qRdTLMvGVhkmg3Zy/ax7OBI9nXZ9mufzy5vlKPBPIJni2y7zOPIOrLZbK5M8JXifoRCfnJ9oVAw5Zq9fjweJ5YZ4JdliRQpZJBJGQiZQbN5vzKI4ng8RrfbnQhgGASBad/Ywp97e3sTQetcgUkPG5fQ5zLT08oku/yTuALYAdfyAeuP4XBo3jGDWHKdFP2s1WoJ0djBYIB+v2/EJ2WbkoEpKV5pB6rs9XoYjUbm+ihUBVwTRu50OonAW7Ie7XQ6iUCbrVZrpULKDB4ql+31MrCqvT4MQ2N3FPVkAEYGdCQyyCS3z+fzJoioDLYtA4vK7bvd7tIFv+bFF9CTaTK4LAXE7KC0xWIRxWIx8Y4ZYN++T19A7tFoNNEeZ9BPSa/XM++EtsC8DsAE32SgVuZhBuhl3mb5RgFBbsN7tQOvUgBXLtsBSI8Cdjlt51/mTwYmlfVAGIYJgXXg2nsErtUx/JN1TqFQQLlcTrzDUqmUaL8xwL8Mwshj12o1k49IrVZLtGNrtZoJLLdIUNGjgh2oU/YFZH9DtlmO2j6LYgcKlH0TWX9KH0HaPnYfQ+YVu65e1bZHEekP8v32tZul/8v3W/Zr5W/pB/P9lvlpWXmLQcbH4zGiKDL1fxiGRsw+l8slAkDL4O3ToBg382UYhkZQkwLftVoN5XIZlUrFiCOWSiWcOnUKjUYDxWIRN910kyn71tbWjoxYt3z/tH3pE6EPRpYTfO+yHyCPQ1+M9HHyfUufisxXrnPPA8sIaat8V9Jf4hKdlmUP+wHSbyLLKfoji8UiSqUSms0m6vU6Wq3WRABztkfa7bZ5joPBABcvXkS328XW1hb6/T52d3eNcCwFYdjGpnBAr9cz/T4pijQLsj0QhqEJ4M8A/QzKz796vY58Po+NjQ2T5ymMvL6+jnK5jFOnTiWeNZ+lfOZ8pkcl719vUGxof3/fiL3yN9MptMbl/f19I8jGZZ/98V3OIgrL5Wq1irW1NSM+rIJs19jb21soiPLHPvYx/PZv//YSr0hRFOXoc+7cOTztaU/DS1/6UnzrW9/CU089ZXwV0/oXuVzu0MVXVkGv1zNirKzrm82mqf/lMtO4zP22t7e9QvFsB7DedwmzUnjVJ9h6PYoGsG3PPs60/5wvMu0/+1m+/7NA3xpFsUqlkuk7j8dj01cOgsD4XSmkRH+2S/DKdZ5yuYxarYaNjQ1UKhXTP67VaqhUKqjValhbWzPrpDgr25DsX9t+F1d/cpmCqex7yWfMfq/sC88rjij7p+yzSt8m+5fsc8p+KPtRsr9K/yf9V7OKtB4UqxKopADf1taWGRPY3d01YzO0y36/b/IB10mxPvapF0H63SlmFwQBqtWqEUJjn5p2UCgUsL6+bvwLfJ/yvcv8IedgzLPtUcPnk1zG3IFlb2cvL9NPPosI7iy/KZ7IsT2OGTO/cex0MBgYUT8ACd9SFEVmzC2KIpOnsooaTpvPser9DlJUkc9ZLheLxcTzZJoUTuQ2LNvoS+O4phxLi+PYCKSTwWAwkcbtCH2zwLXx0lwuZ8ZMOW4KXK3zeM0cR+Uz5VwQOT9Bzkuwx9AHg8GRbm/b82iAa+P+LDPpM7XX2eOv9DfL7TjvR25XqVRMG0sK3udyucScH45J0e/OOobX2Ww2zf6k3W6bd0viOEar1ZqoA1ztGHuO8mHgElrM5XJoNBoTfm+O1cj6OwxDVKtVk88J62I5DsRxa8I5A2wrA9fyNd+3nPdgz4GQx6ZN89pY1trC0nKsSgqgAlffhxQ0lQLBtq3Rjm0RYV7LcQhBx/kHMq/K8k/OweG8BMK2Les1aXPVatVsSxvytYOnzZtJSzvobX2ipAeJXWbYdXCagKlsR2UVgASScxbk+ezvMeR4tT23W57bJaq+KPb7kt8zyPdmz3mQ31rYc7jlXEV7DrWctyDnTfjEIX2CkD5xYOXgWMX86WUfY5bjL4NV9ZOkjS7rWL45xYqiKIqiKIqiKIqiKIqiKIqiKIqiKIqiHClU1FVRFEVRFEVRFEVRFEVRFEVRFEVRFEVRlHTShCZnEaCVgVJsoQUZAMoOzDKLAK281jSR22VhB0yxg6TY66ct26Ka9jKQDN6SZdkOumgv20JH05aVo0eakOw0u5Q2Y9ueHbRQ2rBtX2llgW2ndsA3O9jzKmxVYttpFjuz7UYGNgImA63YQrUy4BGwfKHaaWWPsnqkDUyrq2yR5mnL8tjzLNs2NevyvMGms+ITema5UKvVEuKzDCxLGFSR98xlbh+GYaIey+fzKJfLibJQBpZleSoDMAJXy09b3NMWPwVgbJvloBQyBGBElVwBkX3B7+3yWZIW6Gqe4x0V7HIYuCZUzOBwduBsO29IMdF6vZ4Ihsl1DEhsi7MNh8NEsFMGfie2kAMDzTLwqivQprxWBoEnvV5vpXbGoL7EFo2WNmCvj+PYBHMnFDmV2zM4KZ+rfBcu0Wjg6n0zsKzcVj4LBpnd29szgWhp04PBAO12e8KefIGXfTZx0LgCZ47HYzQajYnAyUEQmDYA8zDLNr4TCuYBMAILMiCrFMGlsC2fJcW+5TOXItHANWETTjmXeZvBYOU74PHlNgwCy+thMOfjEhwWmCzPgWt5H0iK5FIwQebtKIoS7515m/Yp6ysK38n8EEVRIu8w2OxRCAbr21aW5bI9lFUIdpX7pLXd0vo6y+QwhGXtfsOqtj1osgQEXcXvZrOJdruN0WiEdruNTqeD4XBoBO4pBMJysdvtLrXMswO/MwC1FNtkkHcG/WfZRDHtMAxRLpeN6CwFFovF4oRwy6IBTaWw6aphnS/9mLRl6bPIKig7TZhWlj1p554H2b+Xdkg/hUtQFrhWBkt/gtyf2/b7fVSrVSNwk8vlcOXKFSNoxGc0GAxMe4wCssPhEO122wje9Pt9I17ENt0q2l4UlwKu9hdlcPVCoWDKp9OnTxtRHJ9gLPuf9MPI503fD5+bLG+PqsjNYdPpdLC9vY3t7W1jF1z2pdnpFCv2wXddKpWwsbGBjY2N1OVp61x9z+MAhQnToCDIeDzGs571LLzyla9EsVjEfffdd2za4IqiKMsmCAJ84hOfwG/+5m/iu9/9Lv7pn/4Jn//85/H5z38eu7u7xl/hYm9vb0Iw8jDodrupIquziLP66ly2lyi0TrFMpnG50WhgY2NjQoxV/j+qcNyD7Xu2R9iOl2Iu9CfY+7CtzD4E9+V/9iPs/7PAtgr/r6+vYzQaoV6vYzQaoVwuYzQaoVQqGTG+fr+faAdQyEn2aweDAXq9HjqdDprN5sQ4lov19XWv6Gq1WjUCdZVKxYgn0c9MYaVarZawMbuvPa0vPk/arLj6tGlpq9reHjubB5nnZJ7O8lv2nX2/Zb/Y9xtI+uR8v6Vg6yLQJ0LfOq9ZCpb5oP+XQqr0wUv/Cd8Ly8JSqYRGo+EsG6vVKjY3N03f9CD9Z7Z/Uz5fOX5iC1fJMWF7Dop8X3Is2y7bfOJa9hwS6StJu95lUCqVEqJYjUbD5An6xLhO+j+r1ap5HqVSCYVCwVwzbdYet6Kfn8+H23W7XZMPgKv5zR6rHgwG6HQ6Cdunb1FiPx9X/eIqBw96jIyieRL6ZUgulzPClYTjJTKNz5HvgwLVTIvjODFuxnFd+kaZNhgMJq6J9aWdZs9ZkoKlvIZerzchWApcfW/2OCX9yIeBHL+S5SOAlV6TFDwFkJjzYQuiAkiMiXG+CMed5TqOyUvb6/f7ifuk8LydDhzuu+C1ueqDcrmMMAwTeSmfz5uySKZXKhUjPMp7CcPQlGFxHJt0Oe9F2oYc82S6bR9sR8rxTqa7cI2D8D1KQUXg2nwLWdfwGMPhEL1ez2mbByW+fJAwL0uBYOZtjgHYczNoF1xPwjBMzDfk+BBFge15H6VSyYwvSDHvfD5v2jDAtfkkbM/IMSH66Ov1OkqlUuJ6fPMXXXM107Y/CWQVRlx0+SCPbc9/XBbzjFEe9Lq07Q5yvPSkYvs47Dlk9ritPTfWbm/LvpDdr7G/qcnap0oTG04TOl4GMj/a5XFW0WBb3FfOHUk7vsz/9vGlHyRNlNieW68oiqIoiqIoiqIoiqIoiqIoiqIoiqIoirICVNRVURRFURRFURRFURRFURRFURRFURRFUZSThx2cME28Mk2A1g7IYQcbtJft7e1lOwDHtGX7PuzlVWCLWU4Tt5y27BPzI7YY5rRlO+iNvWwL0djBRVzXqBwudhAdO+CTbb92UB7bhm27nGZXdtAcOxAtMBnIxw6IaQf+mXZPq2CaUK0t4OwS57K3yWI/rgC1tt26grWt8vzKYtj5edqybQ/Tlm17mnVZ1uGuZbtMACaFbG27XzWuwIQuG7DrXFf+dolF2rZjB74CrtmlFCO17TKO44ljM/iuLHeluAEDkNkB6SqVCprNpnl3UpgpLSCZT6wT8Ad5tttbqzxeWnk+z/GOCrLMBq4FJmbAXHlfFOlkgFwprgsgU/D0RaGgHa9RBnImFLaToqJSiFOKoXJfiqDax8rn885gqgwYLIPOFQoF9Pt9jEYjhGGYECnudDqJa6f4AoNRSygmJ+85l8slyi4ZPNwWoAb8+dUnPOcTQ/Ad/zBhQG8+I/lflkcybzJQstx2NBqZwLl2XuY29vY8/0Hn/bSg3Fy/SvHorNgi0xSe5HuR91EsFlEqlRKBvSkQYtvEQYnY8j3LOkvWdQw2zW37/b65NwblB67aN4UiuW273TblRb/fN+2VYrFoxGG4bVYR2qMoWHtQwrKLbGv3M2zh3EXIIpzCtlCz2USn08HW1ha63S7iOMbFixeNmMPOzo4R22y1WkZMiHmNot29Xs8IbM4itCnLkWULadAuKGLNc8l+q/QPyeCs9vuT71b6gOw2rzy27DdKO8967HnxCcqyzyL7KuxXzSIoy/amzE/ymLLuZz9sGX0f+Qzle6PfgcKqrFfZxmHfhuIALBcpshHHsRGKHY1G6HQ6RmCAIlRZyOVypi6R9bqsG4fD4Vz1JMsPmd+Y19IEY5kH5bPj85L53ZXGfOlKu16g6Fy73Taic+12G81mEzs7O2i322i1Wtjb28Pe3p5Z3tnZQavVQqvVMsJ2LB998N1RpKxarRpRpmq1ikqlYgSZqtUqarUa1tbWzDqKYXPfgxKJ/a//+i8897nP9a7P5XK49dZb8Tu/8zv4jd/4Ddxyyy0AgIceegivec1rVNRVUZQTixR1BYDvfOc7+MhHPoIPf/jD2NnZMf15F9/5znfwzGc+c+5zZxE7t9Ps5StXrqS2gXyi5lnTuHzmzJkJ4aZlwjaubNeyfcp+Kv22bM+69mH7l/597mMLtMo27zxCgWxrsX3H58S+I/uMFDKK49gI1NHfFIYh+v0+oihCp9NBPp9Hv983bV8KWvF+BoMBut0udnZ20Ov10Gq1JsYxXFAcj6JLpVLJ+Bul+BP9huPxGFEUIY5j40ftdrvI5XLmuqTPUfYvFhEHlO1nPlfZH5P9L7avZNuZ70Qeh21s13HkOCTT5HbT5gbI+7b7UHJMTPbz5D6+39JP4/stxw98v2W/cZmCL7Jty+c3Ho9Rr9eNj6xWq5k+Hf3t9ImPRiOT7+mj4LXLPwor93q9qXmc4pxra2uoVCoolUo4deoUqtUqSqWSadeXy2Wzjes3bZcCrlEUTYwf2v5ze/xTjj/a8wvsY9ljUbY9y2MtQyR1GeTz+YS/TPpIKdgsBU+JFC5lf5z3ynEUiixKwTp57WEYIo5j9Hq9xHbdbjdR7uRyOVM38DhxHB/4mJstVBoEwYTIUD6fnxhbos3I/YrFYiJvcD+ZfyhUSn8J0wAYEUs+ewBGTFEKBgJX86U9juCyQ+nXPgwouEz4zHjPMk/k8/nEskvAVI5hSqFMewxFjvmsAtuPQ0FquSyv+ajAMRtCG+WYnHze3FbegxSNBK75BUul0oQ/ejweOwXOKWZqp+VyuQnh3UKhYMZg5bY8F+tbWdYAMOWI/Z5YPtu2s7e3d6TekxxvBq7ZDdtPrnwl3yvLAT4XWS6wruczmNevmgVbbJv1NmF9xHX00xK223mt9Xo9IVIq5+hw3L5YLBohVHmu9fX1xLiRPb/Vnssj21H2fCDXfB3FPe/Z7nPYbSp73NM1p9Nud9lzHlztZ7vfaB9Drp8m8GjPY13lHFF7XNuen5mWT+3xMXusdJXHluO0aQKTyuKkiZfa9mPb5LRvJZYppGpfi21ntt2ues6Snb/lvGu7jJf1gz3HQB7HFiaVx8xqE/bx00RX7bniiqIoiqIoiqIoiqIoiqIoiqIoiqIoiqIoSioq6qooiqIoiqIoiqIoiqIoiqIoiqIoiqIoinK9YwcjWvbyQZxj2vIigWNnwQ5QlEUQKKto0HE4th2UTFkNdlAjO79PE6q1gyjNI1RrB1JyCb24ygI7sJlLQNMOyrTMAMNZOGxR2XnP7woq7RIWcu3rCgroEhJVppOlDnSlrXK/g76mNCHUVXBc69Fl7letVjEcDr1iRD6xzbR18+yzzOONx2Ps7u6adIofyn1k3TYajUygTAZ5l3UJg1LLYJoyGDOFwVyinBQokGnc/yiIYC4DV3BoGTCdy3YQPwoqyP1YFzEoOP8kFIqQx3MJ37oEgxlQWIrhjkYj8y4KhYIJkk7hNwZm5h/fnX1dpVLJ+04pqmvTarUSgejl9mwryQDpvV4vEawyDEMTlF62zxgweTQaYW9vbyKA8mg0SrSPpIBsp9M5cnnTFQDyID9TkGKwxCVuNy1QJQOny7KC+0wTlTgI2HazA6PbIgGu9iXgF6WUQTxpP7lcDo1Gw4hKymPTdqUwbRRFE8G9B4OBEYuhWI08jlymGCmFPmSdIAUKgKt2RvGHfr+faJdQ9If3s7u7u7K8OE/dv+jyMo9JoSDWsf1+3wgesZ5mXy2fzxthRIoKdTod7O/vmz5fr9dDs9nEcDjE/v6+EWTKCusc5iWWycViEaPRyATbpxgIy1cKVeRyOfR6PSMeQGEYCnMtU9BD9gVte/MJxgJJW7P7k3I/ux9p264Mhm0H/ZUBge1gvmnHAWYTgJV+CNk/YH9/lm2lT0LWl65t5yUIApRKJdPWCMPQ5CtZx/I++ZdVEIfCEWz/8L8tJktxo8FgYOociiB0Oh0EQWCEmOdB5iv6MGQa84crTfoNXGnMo640mddcaUcBCuJJcbxpwnm+bacJ6gHXRPXSBPOmreOyHTAcAD7/+c/jl37pl7znl/nuVa96Fd785jfj5S9/OR5++OGliLo++OCDeO1rXwvgajmUJpx7PfGJT3wC9913H771rW+Ze37sscfwnOc855Cv7PqhVqtNBOAPggBra2v4f//v/+H222/H7/7u7+Jnf/ZnD+kKl4frXn185CMfwRvf+MYVX9HJIAgCfPzjH8fm5iY+9KEP4ZOf/CTy+XymuveDH/wgnv70pxvx8Z2dHezv75tlpnGZAuNp7RjWlxsbG6jVaqjVaqjX62g0GlhbWzPLFCCv1+tm2d7P7hdOg/463/8s28yzzzwCNmxXy/a1nTbtP9t87M/Sj8D0VquFXC6HVqtlhCbZv+l0OkZgfjgcYmdnx/RbuT37olnarPSR0Z80Ho8T7VL2bSgut4hIo0v8lGO9cozIJaIq+xppx8kixhoEAbrdLhqNBnK5nDPP2L/T1i26z7zHXhRXHl7Gb/Y1KH7X7XZNHqOIcK/XMyKr4/EY3W7X+ATog+TYcLfbRbvdRrfbNQJW9hixjzAMjaBaFEXm+uz/QRCgXC4b3y/9uhTqY78tiiJTX9LvxD7jaDSaeEd2GdNqtRLjBKsU+5T9bfrRpYihFDZkW533wX0kLE9YXklfOHDV90o/yEHD98Vzs69IKGot/e2j0SjhRwcw0SeXabI8ZZ7hu7WFXqVIJwBTHstnSpHYw6JQKEwILLJc5TL/22kAjBC3K80WMbX3HY1GiTrP9n3TByIFOV2Cpvb5j3qYImlnrnTb5qTQp1wn7ZfrZD1uCw3L40i/JdNte7avUfo3JSy7XdAPelQJwxC1Ws2ZZ4rFYqL8kPmMY4F2fqRvy86jw+EwMRZJHyXPw2cvyxc5pkjY9pLvgOPM9vuV44Su95BFsHxRfG2EZS+veuxDLksxu5OCPe/VNUfPJVo4j0CpPR/QntfnOrfdr7XnDNnj4EBStN51XlsAclXY4yL2/DnXmKo9P88+hr0+TeDRPt80cdRFjq2kk2V+rCsv2zZk251LkNi2GdvOlnUtti1OE1JdJvZYpp0f7fFLu2yflvftebrSFmy7XfW1KIqiKIqiKIqiKIqiKIqiKIqiKIqiKIqiKCcOFXVVFEVRFEVRFEVRFEVRFEVRFEVRFEVRFEVRrh8YWNS3bAeqySJK6QqKZR93XsFL17GXJaa5SlxikK6ganZwHVcwKVso1nXsrAKzru18IhK2eAngDsjjui/XffiEk5TFOA6il6s8/yqDa01jXvHJZex/HM7tE0U7adh1jyuonCtQvx3AMUt96Apq5wr0aAfHcwXQc9mfHSjTFfDSJWh7kPUvkAxUB/gDNdp1MOCvq3zH8AVM9Ym829cGuMWtAbcwNuC3LV/QPFd97qv77ciTJnYAACAASURBVEB9s+Irk115k7jEzYnM97bdxHGMCxcuTLQbef5ut2uEDG2xiP39/YkgkrZgri2uKI/PdFtolwGj7f0YFDmOY3O99jYuUTQZmF0GGr/ep7bbQrkSChVIZDD6SqWSCNoq7a1YLJpA+TIQO49Lm5DpFKCrVquJ9xGGYUL0l0GqbbsfDodOe2U+KhaLiXtl3uP1U7xTBrqmgG+xWDQiKnbeoeCdS8SZ4s124H8KVtplVL/fnxB15r58TjJPyvSjjn2vLkFbKWIjl/nbFTzed5wsAhSSw2rjuiiXy4m8yudRqVScAhsUBJIiB+PxOGEPzJ8URRwMBqnlG4PJ0y5t4R7mewkFIOXyqsvQer2eeFbVatWUDblcbqLtUCwWUS6XE3VEuVw2gfL53Hi/UmSTAgZ89t1u19Q3FKJhecPftuh7GmEYolKpGMGZUqmEKIqM8AyFBqrVqimzuDwYDExZSZHZXC6HYrGIZrNp8gJFkVk2N5tN056x26OyTWm3jaUPytW2XQaziMPK4MZ2+9LuO8k2qd3mnOc4bI9Uq1XEcYx2u41KpYI4jtHtds02fF7sd8h+inz28tlyW9kPYH+Dvjnm052dHVNntVotk1cPIgg9kBTstMVBWI4wjc+fIiTAtXaEFExmHcfyjfmXdTHz8LziumkibAedZgfongeK0u3u7qLZbKLVaqHVamFnZwetVgvtdht7e3vY29szyxT6oyjczs4O2u02Wq3WVLteX19HtVpFpVJBo9HA3t4ennjiiUzXynroWc96Fm6//XY88MADS6svXvGKV+DLX/7yiRB1/cpXvoKXvOQluPvuu/Gud70LFy5cwB133IF//Md/VFHXJfPNb34Tz3/+83HnnXfi0UcfxXA4xOXLl/HVr34VH/jAB/ClL30Jb3jDG/DXf/3XC/XxjwL2vbq444478LrXvU5FXZfAk08+iWc84xmo1+toNpum3T0PaYLcWUS7NzY2EASBEcZetnBq2j62DzTr/aYJpvq2ochfqVQybf9isYg4jhFFEcIwxJUrV4xQI/s4nU7HiJ1RUJVtNPq09vf3E8KU7BsMBgPT32A/ZN73nAXefxiGKJfL5r2Wy2XjcymVSsjn81hfXzdtEPpR5ZiobJNXKhUMh0Mjgtbtdo3vh21ePvcgCNBut02bvd1um36a9M3Lfobtz5TtYulzt/sfMv9IX7k9LmD7/xfBHjeWfRXZB7X7G9JvLP3Z8jlnPTYFOuV2vN9yuYwoiky5UiqVMBgMjJgo+xOyXcj+y87OjmlX8p30+33Tt6UPeDgcGlFW/qcfIas9S18f/1PoEXALOC4TPlf2N2x/i4T9koMmn89PiG7KfhSR/lI7XeKqZ3K5nOlL2vvZAq18R0dBqDEIgomxIt/zoaCrvS1wLR/KcQCm2c+K/VJfXpB+bolvn8PKV/Piyl/SXqUt2/5cYo8R2uK3ch/pH5brXGKUzJurFqpcBNecI+KbXwD4xyJdx2M7g35F6Y+S57LhOAXrIvpBSRRFRiBU0uv1jCg79+P4I32XLvFi+lIlrHd4r2xzSShaL2H7S4qm2mPpq0bW1YD73bjGg+2xbdeYuWvs2uVDssfDXePS88yfs6/bHjP3jc8fN+adO+VKW+axVn181zaueTSrZFah3YPaZ1XHcM2XOWnYc6dmSXP5DrKkZREvtfP+QQmprhJ73pJdhrvqCjuP2vnYVcfZbRW77lq2kKq8r7T2laIoiqIoiqIoiqIoiqIoiqIoiqIoiqIoiqJcZzw8OaNYURRFURRFURRFURRFURRFURRFURRFURRFUY4pdvAbl9DXScEV8GhewVg7ANMqj93r9fDd73439diAW0TPtZ1L8O8gsYMKAm7BOleAQF8wJFfgMVcwwaMocusT6/MJB5bL5Yl7OMl2DSwWOHDR/Vdx7u3tbZw/f35p53GJbh4kswjNzpq+ymMf9LXQjn3bH+cAi/MK1LoCCmYRkXfVc67tfNcG+G324sWLzqDMrjrYJzDmC5R42LYq8Yk6uerwWYVxfXncF3jXlfd9gveNRsMpjOurU2cRxvWJ8fqEcVcpdG23FV3525XPWq0Wtre3E3l1d3c3IZjX7XYn2rT9fh/tdjthExRbkNsweDUZDodot9uJoPz9fn8iGC8F6Gwo8mDblhTglddqB/k/LOwg8IQCBa7tgeMZRN+2NzvAPcsAOxA+09bW1hL3LQPlj0YjVCqViecixZVLpZIRg6DQBvPacDhMBHS3xWrl9UoxZXsbu8znNsPh0Lxr+x54HbK8lKIK9nmJL48cBr7+oqvO9OFrt/hshPvY6yio4kp3HU8+V5bxtjjIMrGD9PtEzYlLEJrpUlCGNkMhSiksCVzLL/I5cJnbTBPOtYnjeG5xVPkO5i3L+L4oykQBTgoBS8GU06dPG+FuikWxzImiCMPh0IhFjUYjUx6wDSKFpiielc/njdhTFEVGLAi42q6RdmGLs+/v75trlgKoAHDlypVEG0+KOdn1tQywbR9nWdjtIim6YPth7GX6T7j/DTfckGiTudqFtqhDuVw2+bbf7xvBcwoRc32/3zcCTRQla7fb5p202220221TL1C8jG0NCspKMTOeg8JmtBlZj6wCKTCby+VMXqfALEVO2u02Op0OdnZ2kM/njfCPFJdiucDr570EQTAh6jwvqxSO3dzcnGnfIAiMqBfFYVutFprNJnZ3d40I2O7uLp544onMoq58Pt/+9rfx7W9/GwDwnve8B+985zsn+juKn4cffhjj8Rhve9vbUKvVUKvV8IMf/GCl56zVanje856HL3/5yys9z1Enn8/jzJkzuPPOO3HnnXfinnvuwV/8xV/gypUrePTRR71tLSU7JyGvdTodPPLIIwCu9a1nbTO//e1vRxRF6PV6CT8Y2zu7u7s4f/48nnjiCQwGA+Nvoz9unvaO9AmxnUF/VblcRj6fR6lUMu2/Wq1m2oVxHJs6plAoYDgcolgsmjYA04Brbelms4lCoWCE3FutVkJcstvtmnYHfSysi1nHU2hymX0S9p1s4XqKxgLXfFYUZSwWi0ZYNp/PI4oi016uVCoTYqxsgxP2SXgOPge2bYbDYcLn4xLQleuazabJe2nbrkrYydUOsX/71m1sbBifxNOe9jQjXse2KJ/3YDAw7yeOY2c/mffK57q/v298GgBMW5b+iVwuZ/Zh/pJ9CuZTtqlpa7RTvkcKoXKZz5h9UQrzkePmt5qGy89AQU4J2+O2KKTsO/hEYLmfT1DTLhNWKbYMXOtr81p5jdJHwfdPkVaZLn1h0s8n+yPy2Gl9rKMsfOlqR0l/jEw7yPkWvvbdcbJL1jsS3hfFv+37lPWTzazjsq51FDznOl4f/Qq8Nqbb7zyKIkRR5MwLUnBcIn13klarZXxgclvWlfSBATA+EqbTXjudTsIHRbt3CdO5xmV5bDnusr+/75wLddBjmllFQu1xTNfYXq1Wm8gj9njhIgKn9hija7zUNYZoj4e6rt033rlqsuYX4ssfLvFDezz94sWLAPzj6a45cEDS5zrt+nzzAFy24ptj4Ls+X9vVNZdg1bjGsF151s57rnF4Vz5mfpR53jX2btuEy55dZfe812rbqm8uw/WCr1/d6XQm7OLcuXNzzzHLmuaygaxpLvt2pdlzhmZJOwjseS+uPG/XV1nES2+55ZbEcV310lERUlUURVEURVEURVEURVEURVEURVEURVEURVEU5fpDRV0VRVEURVEURVEURVEURVEURVEURVEURVEU5TqkUChMBC866WKYLo6aQOa8+1PoYRXnWVUQ6Vk5SoKWB5G+6LGkIO71KJyZBVfedQlgugJc+gQqXQEyfeLRPttxBcj0Hdt3HbOKc/oCfrqCifoClR4WrnyaFozSJcRJfEKaQLooZZqdpgXs8wlpA37hbuKqs3O5HDY2NlJt1yfcCazm/pfNLPbhC9o5q0267MAXtHbWgLg+2/vhD3+YWRjXZ+++IMGHFbDTxSzCuGn2kpbn0/Jumn2mlSO5XA433XSTtyxJCyTtE/glae3xNPtNs0M+u9Fo5MxDpVLJaRPLCAa9SLBqYtssRduazaZTZHRra8t5fRTVlbZFgTt5bODqvbts0CdOQHGbNKaJQ547dy51vUQKtMprcJXhFLKxy04K6hH5LF1iohThca0DMDVoPQVSjjO0a1tYl0iRH5dYRxiGRujDPgafrUzzCR/b+9sCp2n7HyS20IsLKZZzWFD8yXetafeRdn+LBqWnTR6VOnsRWF8yj1IQRy4XCoUJkQkKkHG7m2++2QSuZ1kURZERhWJ6tVpNlHkUKIuiCKPRyGwrhaVof3EcJ4S4aJuFQiEhDlooFDAej42wArd/8sknzX3EcYxWq5UQI5Z1QRAEE/2qVQsa2AIYQRDghhtuSGxjt0Wq1SqCIDDPg89HClsFQWBEXijk3el0EmLLbAvT5vv9PsbjsXmmFH+YV3jaJ0S16POkQC6FKSiuzOPaeZllxmg0MsLGLKN57xSzmwcp/MP+YqlUMm3aU6dOGZGaWcnn8/jxH/9xPP7447j33nvnur6TDAVcT58+fchXovzZn/0Z/uVf/gWf+tSn8OCDD+K1r33tYV/SSvnnf/7nw76E64JyuYynP/3pAOZvx33sYx8zYqpBEBhfRLlcxnA4NOU3BS7pG2BbdDQaIYoitNttU9+yPh2Px2i1WqY+6Xa7CIIAnU7H1C/f//73FxZBlAKHFFKfdX/WjRRWZduoUCggiiLTL6KwWj6fx3A4RBRFCfF22ebg8UajEbrdrmkLdrvdRF90d3d3ITF3H1LAbjweG5FXtjUqlYrJNxSZ47soFAooFApGjLRUKpm2g6ufzudIAdq1tbWEcD3p9/sIgiDRX2Be4TOQPhC2sexzsl0yHo9Ne4c+EVe/VXKcxBOPO7RNl6CqTJPtc0J7Tntfvjb4qoVVs+AT75T+Fzs9S1/d19c+bB/FLMi+rIR+DtnHZhph35pltaRSqUyk8Xgsx3lMW6CL5aOEfkz69m2hTgDY3NxM+L15LiDpax8MBolyPgxDU5+6fLdra2uI49g5NmMLEpNms5k4P3A133CsKZfLJa7HHpsplUpG6NzGNzazjPGty5cvO+tAl59hHiH5VeESWQTc4oWAeyzIN57qG/+56aabnGNGWcerXOdz3YdrzDWLwKnr3tfX173l4TSWIczpGyvKkr+Yv2cRDvXNRfCNh84iHJp1HIoclTFVl6gt4B8bdI0n+sY9feOWPltx2aHv+nw27rKPNKHUafvaNuK7nqPMonP5gKu24BojXcaxj+MxDgqXHS4jTdqDvd2qzrmstLT5GYqiKIqiKIqiKIqiKIqiKIqiKIqiKIqiKIqiKNcDKuqqKIqiKIqiKIqiKIqiKIqiKIqiKIqiKIqiKMqJRQpfEhW/9eMK6OgKgOgLFuoT6/MFS/Sl+wJL+gI6+gTHfMHffEEdfem+AJi+wJM+YbTDxCcklyYilyZql7YuLcBb2jpfgNdp69KELNPWuZ4Jy4c04b00UdG0dUddZNdn1778P6vgpCvw6qzCmr7tiStwMnEJEJJLly55A7qmCV+nBZj0PTfgaJYRPg5S1JbMK2qbVk6klVl2OXjjjTea38fp/mcJHO7Ln7MI4/qCMvvyt6+O9QVgTjs32d7exoULF5zrfHU0ML0s8bVnAH/Zd1RJswtfMGZgeh5eVIi3WCxObJMmxAuk17GLCPFWKhVvu5OChGQ0Gk3k73q9jsFg4Mzf7XYbg8EAhUIhcb8+27FFXniMrMK68vg+AYyDWsfl3d3dIyMUkxYUulgsesUDKTop4ba2kCUZjUZOYQ1CASFXWUNxJJdooBTxiOM4Icpon2swGCSEMnndhPv7BFMoBHzU8T3H48K8gg4H+W5cz3cVYlvXAy5hUqbZ66SQKDApyM3tXWWMTJPtqHw+j263a8RYKdJj5xeKqskyRQq4ymW7zOH+w+EwkQ9Yj81LFgFXKWzkEsOWx5HHcwmgpSGfixR6le9Elrl8blkEufv9vmlvp7WBfW2eNIbDIR5//HEAwF/+5V/irrvumvkYJ5njXJdcbwRBgLe85S342te+hg996EPXrajrW97yFoRhiPe///2HfSnXBaPRyNnPmoXz58+nrqf4nl0HAZMiqvMKk8vj2gKtQFLk04Wsh2Y5fxAECd8B6zq2tQuFgikne70e8vm86adSVI/n5nOK49i0N+T1y/tgHSv7V9PESG3xyyz32uv1Ev3qecTTlaOPq2/lEkl1rV81vjbirMLLq8AWC7XTpA3bY225XM4IftrlV6FQSPix7XKGfrIoipyipC6/If108jooxA3AlCNRFE1c62g0cvo66RfM5/PmeovFYqI9Ph6Pkc/nnb7Hfr/vbbt3Op2JZyPXpfmp0tavYh3Xu4TkFjnuUfWlZxU0WyQ9DEMzDrTsYy/jGHbdWCqVTB7vdrvo9/uIomgi31OY3jf2vmgeXHbeHo/HOHfu3JG6pmnr08alD5qDyrvVatWMxR5E/j+oY6eNW2UhbawyTazWN9cKSM9faWW2PcYrx8nS5kSkjdWmzZdIE69OG+P1jV2TtLkkaWNgaeNxB80sory+/Ooab/WNC8uxZDm/YRbhYd98C98YtmsehWus13XcrGmKoiiKoiiKoiiKoiiKoiiKoiiKoiiKoiiKoijKyUZFXRVFURRFURRFURRFURRFURRFURRFURRFURRFUZRMpAleKYvjCz7oE7TzBST0pacFKfQFKEwL+JgWKDJtXVpQw/Pnzy9dgCwt+GLauqNEmjje+vp6qthTloCgaQJ7JE34j6SJ5JI00UmSReDWd9+nT582v6eJdE8TCAT8QS4laeKdZJpoIDBfwMiDDt591AIu2+tkGXrU7v8okybgPcs2h3GstO1kGXAUrn84HDrzSLFYRKlUShVzHg6H3rKz1+uZIMu2OOhwOJw4bhRFpjxPE9R0CUbKc/qCMB83Id7jbLsustYlaSK4JEvdBVyt26fVuVnqSiDZluj3+878mc/nEUWRN+/2ej2Mx2OUy+WJdoktrhsEgXlenU4n1R58NijzpV02NJvNiWOGYYgwDL1Bx4Gr9ZmvTbW3t5cQapVtJrnOhvbuamOxLS3bViwr0trZxw2KtwB+cZ5SqZQQyLLLq1wuZ/Ifj2GLY+VyuUR+GY1GGA6HiXOGYTghviXLKSkUKaFQL89jt4ftc0lBTnmNPKfvOXAdhbZ8z2xeQbKTiu8ZKsthPB57/Rc2LoFdpvNYvnVyvUs4zrf/MllUYPTf//3f0Ww2M/v5Hn/8cbz97W/Hl770JcRxjJ/5mZ/Bfffd593+0qVLeM973oNPfepTOHfuHNbW1vCSl7wE73rXu/C85z0PAPDoo4/i1a9+tdnne9/7Hu655x587nOfQz6fx2233YYPfOAD+NEf/VGzTa/Xw3vf+1489NBD+P73v49SqYTbb78db3rTm/DKV74y0bbJcg1ZsK+Tdf2LXvQifPWrX53pXHEc4+///u/x0Y9+FI899hh2d3fxYz/2Y3jjG9+It771rabMvv/++/FHf/RHAICvfOUrJu/l83nEcYw/+ZM/wb333gsAuP322/HlL38ZAPDZz34Wv/IrvwLgqm/k8uXLznt4/PHHce+99+ILX/gCrly5Yu5hc3Mz873M8i5WwYtf/GIAwFe/+lUMBgPTbrqe8p4LzWvz57XxeIwvfvGLsz/0GZCi6fa503C1+XzI7Q5SbHo8Hh9J0T3l5BAEgelj2TYWRZFJk228MAydYlRSnNOmUqmYfaSNFQoFk57P583v8Xg8cSwen2Myw+EwcaxyuTwxzsLyg9csrzuXyxlfH0VAub5UKk08jziOE8/Lvv8oijAYDCZ8bK7xoSzjZ2ljiMQWerPh2KUsZ+TvtHMcN79e2hhams95ml8tbZzQd06KcjOvUpBNnrNWqyGOY+fzbzQaiXPGcWxEAila6iKt7s7lcuY64jieeLeVSsVZz7p8zfl8PpGvR6MR9vb2JgSFXWTJ+2n+8WnHSBNGTBMqBNL93Itc01Fkmm86bSw5bWzcJ0xI0uzUvibbJ552TWk2vsg1+cb4XdfCusZ1bfJ5dbtddLtdVKvVVHtJEweVTBO8XJaQqIT24Kojrhch0aNEWj2Vlr+nzdVIG3fy1X25XA4/8iM/4q1v0uaqpNla2lwZnx36yjHfsVz35Hu2Web3KIqiKIqiKIqiKIqiKIqiKIqiKIqiKIqiKIqiKIoyH8H4OMzgVBRFURRFURRFURRFURRFURRFURRFURRFURRFURRFOSDSgmumrUsL2Jm2Lk0AL23dtOC+wPRAqUC2gKDTgv1OC0gKTBf6A7LdU1pAVZIWkPU4UywWJ4Jr25w0IV/JMoR8Z9nORt5Hr9fzBhoeDAYmkLAdyH8wGKSKZTabzUSwdTIej1NFLXO5HMrlciZ7z2o/WYL2Z7FpID1oM8lSzgDThQJIlvJxmc/ieiCLrc4iBDqtbMgirC3JUm7Nci2j0Sg1KHmxWEwtl4bD4cT+uVxu4vl0u91MgbmnPXseS+brMAwnnolPYFhSKBQwGAym5uvRaJQoswqFgvM6u91uqo2zLpnWTuCxWL6m5aG08oLXmLW8OC7B01fBtLaH67lIIQCfuIZ9jDTBDwDOvFitVhOipdOErYbDoVP0Rl6HbbNBECTqXFuQx4UUl3MxGo0m7EEK8wBX659p5+n1eqnlQhzHE20RWwi20+mknmc8HqcKglzvyPeSRZBVCgID1wSnXDZgi4fOIgp6UssjZXmcPXsWTzzxRCYh+P/93//FC1/4QlSrVTzwwAO47bbb8L3vfQ933303vv3tb+P8+fOJdsX58+dx2223odvt4m/+5m/wC7/wC3jyySfx+7//+/ja176GL37xi7jtttvM9r/+67+OT37yk7jzzjtxzz334Kd/+qfxb//2b3jVq16F5zznOfj6179utn3Tm96Ehx9+GA8//DBe/OIXY29vD/fffz/uv/9+fOlLX8Idd9wx1zVkgdfZ6XQS9cIs5/qHf/gH/Nqv/Rr+9E//FG9+85sxHA7x8Y9/HH/4h3+Iu+66C+973/sS56zVanje855nhDRtfOtf8IIX4P/+7/+M0KZ9Dy996Uvxx3/8x/i5n/s5PPbYY7j99tvx1FNPYTAYZL6XrO8CAF72spfhP//zP/GZz3wGP//zPz/1WX/zm9/E85//fNx555149NFHndt0u10jiHTu3DmcPXv2WOY93quPt73tbXj/+98/87FPal5L4/Lly/iJn/gJbG1tZdpeUbIi29ZSUNQWkJfpsk3OZRKGoREHZjuU63O5nOlnSNHO0WhkxEW5LYVB5fHpGxgOh85+Qlrfotfredugrn3pF/UdUz6DOI4Tvn5ff+1675fXarWpfp9isej1e7HPkc/nvaKB9vmCIMBwOPT22UulkrefPhqNEMex0+9kEwRB4pr6/f7EOYvFold8LQxDFAoFbx837fxRFHn7tNKHNRqNEvnQFteV7O3tpQqT0k6z5tk0kU5JVv9zVpHF64ksfmYg23gWkG38bJnCoDZra2teP6otvinp9Xqp4r+VSgW5XM7pCw3DMOH/k2NYQNIHJ+E2vnulUG+z2XTmX9v2s4zZZB0nyjImDWQfA8oqIr0soVMg+71mHYc7jqTZy1ESEgWuil0fBSFRIF1EOM3fH0WR11eUdexPURRFURRFURRFURRFURRFURRFURRFURRFURRFURTlmPPw9NnpiqIoiqIoiqIoiqIoiqIoiqIoiqIoiqIoiqIoiqIoJ4ggCFIDdp46deoAr0ZZFb1eb6pIVZYgve122yveSbIIVmYJ9JslcPdREvIdjUb4/ve/n7rNQQr5ZnnnSpJpQrkusgZRl2QJlG7jCy6cFth8WrB1H1lEhW3sQMzj8XhqWVEqlZDP5zMF6Gfw9TAMvcGZ4zh2lj1BECSCL7fb7anlhhQCTRN4nCYQCFwtG3z2KoVSe73e1LJTXl+r1UotB6TACen3+zh37lymc0iksCgwXQCFtFqtuYQVms1mJoEHm+td9OQgSQtoTqYJnALXBHemMc0m8/k8SqXSTO8367nL5bLJz/MIVvvyeD6fdwahH4/HiOM49drsgPij0SghzBBF0dRyejgcZhJCSDtPFlwCNS7kPfV6vUz7SHyCxGmitVlEjJVrjEajmd7LPOV0FmzRXx8+gVgJhb1mFZGdpy6Zt/7Remv1nD17NvO273znO7Gzs4OPfvSj+MVf/EUAwE/91E/hgQcewDOf+cyJ7d/xjnfgySefxN/93d/hV3/1VwEAt956Kx588EE84xnPwFvf+lZ84xvfmNjvjW98oxFxfMUrXoFXvvKVeOSRR3D58mVsbm4CAL7whS/g1ltvNddRLpfxvve9D5/61KeWcg3zMOu57rjjDrzjHe8wy29961vx9a9/HR/4wAdw7733otFoLOW60rjnnnuMEOaLXvQiU9e/4Q1vyHwvWd8FcK39sUzbdh3rOOc9l4DtW97yloWOfRLzWhrVahUveMEL8LnPfW7u+5nVJ5AFrfMOH+m3SWtHkzShRxdBEEwVEJT9tOFwaMRYZRr/u/paFHj1+ZziOJ5o085yD0C6zyOOY2e/MYuPhKT5EenbSvNFEdc7nOZTPiocl+uUVCqVqT5gVxsgn89nep8uyuXy1Pxr+z6kjyONNBFbSRRFRmTW1QekLa6trXmPJ+15Y2Mjk9AofSrSR+vbzne88XhsxkeCIJgqyGtvN80/knY86dvMOnYQhiGCIFhIqDarEKek3W6bcb95x5OuZ5HNecgqBpkmRClJE7qUpIlsSrKMg1WrVdx8881Tt7se7jWfz2fqLyzrXrP43RVFURRFURRFURRFURRFURRFURRFURRFURRFURRFURTloFBRV0VRFEVRFEVRFEVRFEVRFEVRfh5IAAAAIABJREFUFEVRlP/P3r0HWV3Q/x9/nYXl5iIIcskRvpUKNCSmRsmMNgaSmjhcJkEatERTRtGoqQwza6y84C1LNM2my+SMYJMLI0njjVKDTCydVBYveUVEIpBNBXb3/P7oB+O6KLsLu2fZ83jM7Iz7uZzP+/M5n70cnLNPAAAAyk737t13GZFsTdCSvV9zQr3v1Zo/8F1fX9/iUFzSukhjc4LA79WaP3jenADvezU0NGTTpk0t2idpXsD4vZoTWH6vd955J//6179atE9r/2j7rmLKO/Pmm2+2OCLXmnuczun9Qsd7KsTX3FhUS7+Wk/9FOVoTVmpu2PS93vt11qNHj2ZFQVobnfggxWIx9fX1Lf5+1hz19fW7/P7QtWvXFkeAisVi3nnnnV0G2Fujrq6uTR63oaGhyc/O7UHM3bUnnrudRVrq6+s/8Od9SwPqLdXcr63mho3bKpK6t2lpXBZ2ZevWrc0KRyXJ0qVLkyTHH398o+UHHHBAhg0bltWrVzdaXl1dnYqKikyYMKHR8sGDB2fkyJFZuXJlXnnllRx44IGN1o8ePbrR50OGDEmSrFmzZkdY84QTTshNN92Us88+OzNnzszo0aPTpUuX1NTU7JEZWqMlx5owYUKT7ZLksMMOy29/+9s8+eSTO+KibelTn/rUTpe35Fya+1wkybJly/b4Obz22mtJ/vezcPv90dnvPffaru+1XWnp6/r3EmBtey19jfFu218b7k58d1fPcaFQaNXvZc353b9QKOzyd3noaFobu0zSqn8L7oj69u3b6u87zQ0w7kxzI48707Nnz/To0SPFYrHZ/z68/f8RdO/evdVB3j59+qR3796t2vf9/v2yufbm56ktjtfcc2rOddvd5wYAAAAAAAAAAAAAoKMRdQUAAAAAAAAAAAAA+P969erV6j9ODp1da2K6zVVbW9smAcykdZHi5tq6dWubBCWT1sefm6s1IeHmak1wuLlaE2jeE1oTru4sisViNm7cWOoxSqYt7+fObHcCJp1NfX19Se+hLl26tDqg0xmOX1FR0ezQeFsoFAp7NHTzwAMPZPHixS3ap6KiIsOHD8/TTz+dJ554ollRoy1btmTz5s3p0aNHqqqqmqwfOHBgo6jrli1bdgS6Puh8n3nmmSZRy/duv/17x7t/75w/f37GjBmTX//61xk3blyS5Jhjjsk555yTyZMn7/YMLdXSY23atCnXXHNN7rzzzrzyyitNfq7uThitJfbZZ58my1p6Ls15LtrSQw89lCQZM2ZMKisrO+W9d8MNN+z4b/fa7t9rPXv2TE1NTQqFQj7xiU/kySefbPFrz7lz52bs2LEt2qcl3nrrrWbH9TqTcn6N1Vy7EyOk5Xr06JGePXu26zH322+/dj1er1690r1793Y7Ximij16LAgAAAAAAAAAAAAAArSHqCgAAAAAAAAAAAAAA7FJbRhjaO2IBALA36tmzZ7OjroccckguueSSTJ06NdXV1Zk2bVqzw3Ddu3dP7969s3nz5tTW1jYJu27YsKHJ9n379k1tbW3efvvtdO26Z9+6WigUctppp+W0007Ltm3bsmzZslx99dWZMmVKrrnmmnz9619v8xneraXHOvnkk/Pggw/m+uuvz/Tp07P//vunUCjkxz/+cb72ta+lWCw2Od8PUlFRsdPIfWsi7C09l+Y8F22loaEh8+fPT5Kcd955rZq/pUp977nX9uy9Nnfu3EydOjUrV67MRRddlPvuu69Z4fVhw4bluOOOa/HxAAAAAAAAAAAAAAAAANg7VJR6AAAAAAAAAAAAAAAAAAAAPthHP/rRD1x/4IEHZv78+XnzzTezevXqzJgxI926dWvVsU488cQkydKlSxstX79+fWpqappsP2XKlNTV1eXhhx9usu7KK6/M0KFDU1dX16pZ+vbtm1WrViVJKisrM378+FRXV6dQKGTJkiXtMsN7NfdY9fX1efjhhzN48OBccMEFGTBgwI6Q5ttvv73Tx+7Vq1ejkObw4cNzyy237Pj8Qx/6UF599dVG+6xduzYvvfRSm55L0vznoi3MnTs3jzzySCZPnpxTTjmlVfO3VEe499xre/5eO/LII/PHP/4xW7duzV133ZUjjjjiAwO3Q4cO3a3jAQAAAAAAAAAAAAAAANCxiboCAAAAAAAAAAAAAAAAAHRwo0aNarJs0KBBueKKK7J+/fq8/PLLOffcc9O7d+/dPtZll12Wfv36Zc6cObnnnntSW1ubp556KjNmzEhVVVWT7S+//PIcdNBBmTlzZu6+++5s2rQpGzZsyM0335xLL700V199dbp27drqeWbNmpUnnngiW7Zsybp16zJv3rwUi8WMHTu23WZ4t+Yeq0uXLjn22GOzdu3aXHXVVVm/fn3efvvtPPDAA/nZz36208c+4ogjsnr16rz88stZvnx5nn/++RxzzDE71n/uc5/LmjVrcsMNN6S2tjbPPfdcvvrVr2bgwIFtei7bNee5SJKxY8emf//+WbFiRavmamhoyLp167Jo0aKMGzcu8+bNy8yZM3Pbbbc1CnB29nvPvbbre621KioqctJJJ2XlypV55513cuutt+YjH/lIk+0OO+ywPXI8AAAAAAAAAAAAAAAAADqmQrFYLJZ6CAAAAAAAAAAAAAAAAAAA3l99fX0qKyvTu3fvzJo1K+edd16GDh26y/0WLlyYadOmpaVvJ129enUuvPDC3H///dm2bVs+/vGP53vf+16uu+663HfffUmSM888M7feemuSZMOGDfnRj36U6urqvPzyy+nbt28OP/zwfPOb38xxxx2XJFmxYkXGjBnT6Djf+c538sMf/rBRqDNJTjrppNx11115/PHHc9NNN+XPf/5zXnzxxfTo0SPDhg3LmWeemTPPPLPRfs2ZoTmqq6szefLkJsuXL1+eo446qkXHWr9+fS6++OL84Q9/yNq1a9OvX7+ceOKJGTx4cK644ookyZFHHplHH300SVJTU5OvfOUreeyxx9KvX798+9vfzrnnnrvj8TZt2pRvfOMbWbJkSTZu3Jgjjzwy1113XWbNmpWVK1cmSS688MJMmjSpybVOstP7oLnn0pLn4jOf+Uz++c9/ZsmSJTud492qqqry3//+t9GyQqGQfffdN0OHDs3RRx+ds846K0ccccRO99+b7r2dneugQYOydu3a970+7rUPvvbNUSgUsmDBgkydOnWX227evDk33XRTrrvuurzxxhupq6tr0bEAAAAAAAAAAAAAAAAA2KvcIeoKAAAAAAAAAAAAAAAAALAXqKmpyfDhw1u0T2ujrgCdRUuiru/2n//8J/vtt18bTQUAAAAAAAAAAAAAAABAB3BHRaknAAAAAAAAAAAAAAAAAABg11oadAWg9QRdAQAAAAAAAAAAAAAAADo/UVcAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAoKyIugIAAAAAAAAAAAAAAAAAUBYKhcIuP77//e+XekwAAAAAAAAAAAAAAAAAANpB11IPAAAAAAAAAAAAAAAAAAAA7aFYLJZ6BAAAAAAAAAAAAAAAAAAAOoiKUg8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAANCeRF0BAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgLIi6goAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAlBVRVwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACgrIi6AgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABlRdQVAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACgroq4AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQFkRdQUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAyoqoKwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABQVkRdAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAICyIuoKAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAJQVUVcAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAoKyIugIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAZaVrqQcAAAAAAAAAAAAAAAAAAKBtLVy4sNQjAAAAAAAAAAAAAAAAAABAhyLqCgAAAAAAAAAAAAAAAADQyU2bNq3UIwAAAAAAAAAAAAAAAAAAQIdSKBaLxVIPAQAAAAAAAAAAAAAAAAAAdD5Tp05NkixcuLDEkwAAAAAAAAAAAAAAAAAANHJHRaknAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABoT6KuAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEBZEXUFAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAMqKqCsAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAUFZEXQEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAsiLqCgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACUFVFXAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKCsiLoCAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAGVF1BUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKCuirgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAWRF1BQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADKiqgrAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAFBWRF0BAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgLIi6goAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAlBVRVwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACgrIi6AgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABlRdQVAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACgroq4AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQFkRdQUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAyoqoKwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABQVkRdAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAICyIuoKAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAJQVUVcAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAoKyIugIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAZUXUFQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAoK6KuAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEBZEXUFAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAMqKqCsAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAUFZEXQEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAsiLqCgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACUFVFXAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKCsiLoCAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAGVF1BUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKCtdSz0AAAAAAAAAAAAAAAAAAACw9/vrX/+axx9/vNGy559/Pklyyy23NFo+atSoHHXUUe02GwAAAAAAAAAAAAAAAADAe4m6AgAAAAAAAAAAAAAAAAAAu23dunU555xz0qVLl1RUVCRJisVikmT27NlJkoaGhtTX12fx4sUlmxMAAAAAAAAAAAAAAAAAIEkKxe3vhAQAAAAAAAAAAAAAAAAAAGilbdu2Zf/998+bb775gdv17t0769evT7du3dppMgAAAAAAAAAAAAAAAACAJu6oKPUEAAAAAAAAAAAAAAAAAADA3q+ysjKnnnrqB8ZaKysrM336dEFXAAAAAAAAAAAAAAAAAKDkRF0BAAAAAAAAAAAAAAAAAIA9Yvr06dm6dev7rt+2bVu++MUvtuNEAAAAAAAAAAAAAAAAAAA7VygWi8VSDwEAAAAAAAAAAAAAAAAAAOz9GhoacsABB+T111/f6foBAwZk7dq1qaioaOfJAAAAAAAAAAAAAAAAAAAaucO7HQEAAAAAAAAAAAAAAAAAgD2ioqIiM2bMSLdu3Zqs69atW770pS8JugIAAAAAAAAAAAAAAAAAHYJ3PAIAAAAAAAAAAAAAAAAAAHvM9OnTs3Xr1ibLt27dmunTp5dgIgAAAAAAAAAAAAAAAACApgrFYrFY6iEAAAAAAAAAAAAAAAAAAIDO4+CDD85zzz3XaNn//d//5YUXXijNQAAAAAAAAAAAAAAAAAAAjd1RUeoJAAAAAAAAAAAAAAAAAACAzmXGjBmprKzc8Xm3bt1yxhlnlHAiAAAAAAAAAAAAAAAAAIDGCsVisVjqIQAAAAAAAAAAAAAAAAAAgM7j2WefzSGHHNJoWU1NTYYNG1aiiQAAAAAAAAAAAAAAAAAAGrmjotQTAAAAAAAAAAAAAAAAAAAAncvBBx+cUaNGpVAopFAoZNSoUYKuAAAAAAAAAAAAAAAAAECHIuoKAAAAAAAAAAAAAAAAAADscaeffnq6dOmSLl265PTTTy/1OAAAAAAAAAAAAAAAAAAAjRSKxWKx1EMAAAAAAAAAAAAAAAAAAACdy5o1azJkyJAUi8W89NJLOfDAA0s9EgAAAAAAAAAAAAAAAADAdnd0LfUEAAAAAAAAAAAAAAAAAACwN1m4cGGmTZtW6jH2KkOGDCn1CHuFBQsWZOrUqaUeAwAAAAAAAAAAAAAAAADKgqgrAAAAAAAAAAAAAAAAAAC0woIFC0o9Qod37733plAoZNy4caUepcMTCgYAAAAAAAAAAAAAAACA9iXqCgAAAAAAAAAAAAAAAAAArTB16tRSj9DhbY+59u/fv8STdHyirgAAAAAAAAAAAAAAAADQvkRdAQAAAAAAAAAAAAAAAACANiHmCgAAAAAAAAAAAAAAAAB0VBWlHgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAoD2JugIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAZUXUFQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAoK6KuAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEBZEXUFAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAMqKqCsAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAUFZEXQEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAsiLqCgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACUFVFXAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKCsiLoCAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAGVF1BUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKCuirgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAWRF1BQAAAAAAAAAAAAAAAACAErj99ttTKBRSKBTSo0ePUo/Trqqqqnac+/aPioqK7LfffjnssMNy7rnnZuXKlaUeEwAAAAAAAAAAAAAAAADoxERdAQAAAAAAAAAAAAAAAACgBE499dQUi8WMGzeu1KO0u9ra2vz9739PkkycODHFYjHbtm3LqlWrcumll2bVqlX55Cc/mTPOOCNvvfVWiacFAAAAAAAAAAAAAAAAADojUVcAAAAAAAAAAAAAAAAAAKDkunTpkkGDBmXixIm5//77861vfSu/+tWvMn369BSLxVKPBwAAAAAAAAAAAAAAAAB0MqKuAAAAAAAAAAAAAAAAAABAh3PFFVfk05/+dBYvXpzbb7+91OMAAAAAAAAAAAAAAAAAAJ2MqCsAAAAAAAAAAAAAAAAAANDhFAqFzJ49O0ly4403lngaAAAAAAAAAAAAAAAAAKCzEXUFAAAAAAAAAAAAAAAAAIB2sGrVqkyaNCl9+vTJPvvsk2OOOSYPPfTQ+27/xhtv5IILLsiHP/zhdOvWLQMGDMiUKVPyj3/8Y8c21dXVKRQKOz5eeOGFTJs2LX379k3//v0zYcKEPPfcc40ed8uWLbnkkksyYsSI9OrVK/369cvJJ5+cxYsXp76+vsUztKWjjz46SbJixYps27atRXN19msDAAAAAAAAAAAAAAAAAOweUVcAAAAAAAAAAAAAAAAAAGhjzz77bMaMGZNHH300v/vd7/L666/nxhtvzA9+8IMmYdEkee211zJ69OgsXLgwN954YzZs2JBly5Zlw4YNGTNmTJYvX54kmTRpUorFYiZOnJgkmTNnTubMmZNXX301CxYsyP3335/p06c3euzZs2fnJz/5SX7605/m3//+d55++umMGDEiEydOzIMPPtjiGbYbO3Zs+vfvnxUrVuyx6zZ48OAkSV1dXdavX7/XXhsAAAAAAAAAAAAAAAAAoOMRdQUAAAAAAAAAAAAAAAAAgDZ20UUXZePGjbn++uszfvz4VFVV5dBDD80vf/nLvPbaa022nzt3bl588cVce+21+fznP5+qqqqMHDkyt99+e4rFYs4///ydHuess87KmDFjss8+++S4447LSSedlL/97W87gqhJct9992XkyJEZP358evbsmUGDBuWqq67KsGHDdmuGhoaGFIvFFIvFPXDF/mdnj7U3XhsAAAAAAAAAAAAAAAAAoOMRdQUAAAAAAAAAAAAAAAAAgDa2dOnSJMnxxx/faPkBBxzQJBiaJNXV1amoqMiECRMaLR88eHBGjhyZlStX5pVXXmmy3+jRoxt9PmTIkCTJmjVrdiw74YQT8pe//CVnn312VqxYkfr6+iRJTU1Njj322FbPsGzZsmzYsCFjxox53+vQUtuDt5WVldl///1bNdd2pbw2AAAAAAAAAAAAAAAAAEDHI+oKAAAAAAAAAAAAAAAAAABtaMuWLdm8eXN69OiRqqqqJusHDhzYZPtNmzaloaEhffr0SaFQaPTx2GOPJUmeeeaZJo/Vp0+fRp9369YtSdLQ0LBj2fz58/Ob3/wmzz//fMaNG5d99903J5xwQu688849MsOe9NBDDyVJxowZk8rKStcGAAAAAAAAAAAAAAAAANhjRF0BAAAAAAAAAAAAAAAAAKANde/ePb17984777yT2traJus3bNjQZPu+ffuma9eu2bZtW4rF4k4/PvvZz7ZqnkKhkNNOOy333ntvNm7cmOrq6hSLxUyZMiXXXnttu8zQHA0NDZk/f36S5LzzzmuXufaWawMAAAAAAAAAAAAAAAAA7D5RVwAAAAAAAAAAAAAAAAAAaGMnnnhikmTp0qWNlq9fvz41NTVNtp8yZUrq6ury8MMPN1l35ZVXZujQoamrq2vVLH379s2qVauSJJWVlRk/fnyqq6tTKBSyZMmSdpmhOebOnZtHHnkkkydPzimnnNIuc+0t1wYAAAAAAAAAAAAAAAAA2H2irgAAAAAAAAAAAAAAAAAA0MYuu+yy9OvXL3PmzMk999yT2traPPXUU5kxY0aqqqqabH/55ZfnoIMOysyZM3P33Xdn06ZN2bBhQ26++eZceumlufrqq9O1a9dWzzNr1qw88cQT2bJlS9atW5d58+alWCxm7NixrZ5h7Nix6d+/f1asWNGqmRoaGrJu3bosWrQo48aNy7x58zJz5szcdtttKRQKe/W1AQAAAAAAAAAAAAAAAAA6HlFXAAAAAAAAAAAAAAAAAABoYwcddFCWL1+e0aNH5wtf+EIGDhyYL3/5yzn//PNz6KGHZsuWLSkUCjnrrLOSJAMHDswjjzySSZMmZfbs2RkwYEBGjBiR3//+91m0aFGmTp2aJFmxYkUKhUIWLVqUJOnZs2cuvvjiJEmhUMiVV16ZJDn88MMzYcKEJMmf/vSnjBgxIqeeemr69euXj33sY1m6dGl+/vOf56KLLtoxc3Nn2K6uri7FYjHFYnGX16OqqiqHH354kmTRokUpFArp2rVrhg0blu9+97sZPnx4Vq5cmV/84hfp2bNno333xmsDAAAAAAAAAAAAAAAAAHQ8hWJz3hUJAAAAAAAAAAAAAAAAAAAkSRYuXJhp06Y1K14KzVUoFLJgwQJBWAAAAAAAAAAAAAAAAABoH3dUlHoCAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAID2JOoKAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAJQVUVcAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAoKyIugIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAZUXUFQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAoK6KuAAAA/D/27jREj8Lw4/hv9oqJGxMTN7ElbQUhRoKI0IArFdqkoR7RpCFuzCFUEZEmKfGNENEiFrQeWHoYKCpKQYgbaZPgEbAVxWtRo1JQIx5YSBpydFUSxM0e0zf/Lux/Y9R13cnufD7wvHhmZmd+DPv24QsAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAtSLqCgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADUiqgrAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAFAroq4AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQK2IugIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAtSLqCgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADUiqgrAAAAAAAAAAAAAAAAAAAAAABDJydRAAAgAElEQVQAAAAAAAAAAAAAAFAroq4AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQK2IugIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAtSLqCgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADUiqgrAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAFAroq4AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQK00VT0AAAAAAAAAAAAAAAAAAADGo6Ioqp4AAAAAAAAAAAAAAAAAAMAIFWVZllWPAAAAAAAAAAAAAAAAAACA8WLPnj156aWXqp4xLvzud79Lktxwww0VLxkfLrjggsyZM6fqGQAAAAAAAAAAAAAAAABQB1tFXQEAAAAAAAAAAAAAAAAAgG9FR0dHkqSzs7PiJQAAAAAAAAAAAAAAAAAAQ2xtqHoBAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAMBYEnUFAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAGpF1BUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAqBVRVwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACgVkRdAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIBaEXUFAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAGpF1BUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAqBVRVwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACgVkRdAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIBaEXUFAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAGpF1BUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAqBVRVwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACgVkRdAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIBaEXUFAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAGpF1BUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAqBVRVwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACgVkRdAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIBaEXUFAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAGpF1BUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAqBVRVwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACgVkRdAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIBaEXUFAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAGpF1BUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAqBVRVwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACgVkRdAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIBaEXUFAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAGpF1BUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAqBVRVwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACgVkRdAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIBaEXUFAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAGpF1BUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAqBVRVwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACgVpqqHgAAAAAAAAAAAAAAAAAAAIx/n332WXp6eoYcO3r0aJLk448/HnJ80qRJmTJlyphtAwAAAAAAAAAAAAAAAAD4/0RdAQAAAAAAAAAAAAAAAACAb+yhhx7K+vXrj3luxowZQ77/6U9/yrp168ZiFgAAAAAAAAAAAAAAAADAMRVlWZZVjwAAAAAAAAAAAAAAAAAAAMa3gwcP5jvf+U76+/uPe11jY2P27duXtra2MVoGAAAAAAAAAAAAAAAAADDM1oaqFwAAAAAAAAAAAAAAAAAAAONfW1tbFi5cmMbGxi+8prGxMYsWLRJ0BQAAAAAAAAAAAAAAAAAqJ+oKAAAAAAAAAAAAAAAAAACMirVr16Ysyy88X5Zl1q5dO4aLAAAAAAAAAAAAAAAAAACOrSiP96tIAAAAAAAAAAAAAAAAAACAr+jw4cNpa2tLT0/PMc+3tLTk4MGDOeWUU8Z4GQAAAAAAAAAAAAAAAADAEFsbql4AAAAAAAAAAAAAAAAAAABMDFOnTs2SJUvS3Nw87FxTU1Muv/xyQVcAAAAAAAAAAAAAAAAA4IQg6goAAAAAAAAAAAAAAAAAAIyaNWvWpK+vb9jx/v7+rFmzpoJFAAAAAAAAAAAAAAAAAADDFWVZllWPAAAAAAAAAAAAAAAAAAAAJoajR4/mtNNOy+HDh4ccb21tzaFDhzJp0qSKlgEAAAAAAAAAAAAAAAAADNraUPUCAAAAAAAAAAAAAAAAAABg4mhpacmKFSvS0tIyeKy5uTkdHR2CrgAAAAAAAAAAAAAAAADACUPUFQAAAAAAAAAAAAAAAAAAGFWrV6/O0aNHB7/39vZm9erVFS4CAAAAAAAAAAAAAAAAABiqKMuyrHoEAAAAAAAAAAAAAAAAAAAwcQwMDGT27Nk5dOhQkmTmzJnZv39/GhsbK14GAAAAAAAAAAAAAAAAAJAk2dpQ9QIAAAAAAAAAAAAAAAAAAGBiaWhoyJo1a9LS0pLm5uasXbtW0BUAAAAAAAAAAAAAAAAAOKGIugIAAAAAAAAAAAAAAAAAAKNu1apVOXr0aHp7e7N69eqq5wAAAAAAAAAAAAAAAAAADNFU9QAAAAAAAAAAAAAAAAAAADgRXHHFFVVPmHCmTJmSJLn77rsrXjLxbN26teoJAAAAAAAAAAAAAAAAADCuiboCAAAAAAAAAAAAAAAAAECSxx57LOeff37mzJlT9ZQJ4wc/+EHVEyacPXv2pKurq+oZAAAAAAAAAAAAAAAAADDuiboCAAAAAAAAAAAAAAAAAMD/ueGGG9LR0VH1jAnjrbfeSpLMnz+/4iUTR2dnZ1auXFn1DAAAAAAAAAAAAAAAAAAY90RdAQAAAAAAAAAAAAAAAACAb4WYKwAAAAAAAAAAAAAAAABwomqoegAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAwFgSdQUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAakXUFQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACoFVFXAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKBWRF0BAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgFoRdQUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAakXUFQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACoFVFXAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKBWRF0BAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgFoRdQUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAakXUFQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACoFVFXAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKBWRF0BAAAAAAAAAAAAAAAAAGCUbNmyJUVRpCiKnHTSSVXPGVNPPvlk5s6dm6amplG7Z2tr6+D7/N+noaEhp556as4999z88pe/zK5du0bteQAAAAAAAAAAAAAAAABAfYi6AgAAAAAAAAAAAAAAAADAKLnyyitTlmUWLVpU9ZQx88EHH+Tyyy/Ppk2bsn///lG995EjR/LGG28kSZYuXZqyLNPb25vdu3fntttuy+7du/PDH/4wV199dT777LNRfTYAAAAAAAAAAAAAAAAAMLGJugIAAAAAAAAAAAAAAAAAACN2yy235IILLsiuXbsyderUb/15jY2NmT17dpYuXZpnnnkmN954Yx5++OGsWrUqZVl+688HAAAAAAAAAAAAAAAAACaGpqoHAAAAAAAAAAAAAAAAAAAA49eDDz6YyZMnV/b83/72t3nuueeyY8eObNmyJatWrapsCwAAAAAAAAAAAAAAAAAwfjRUPQAAAAAAAAAAAAAAAAAAABi/qgy6JklRFFm/fn2SZPPmzZVuAQAAAAAAAAAAAAAAAADGD1FXAAAAAAAAAAAAAAAAAAAYod27d2fZsmWZNm1aTj755Fx44YV54YUXvvD6gwcP5le/+lXOOOOMtLS0pK2tLcuXL8+bb745eM22bdtSFMXg56OPPsrKlSszffr0zJw5M0uWLMkHH3ww5L49PT359a9/nXnz5mXKlCmZMWNGLrvssuzYsSP9/f1fe8N486Mf/ShJ0tXVld7e3sHj3jcAAAAAAAAAAAAAAAAA8EVEXQEAAAAAAAAAAAAAAAAAYATef//9tLe357XXXstjjz2W/fv3Z/PmzfnNb34zLAKaJPv27cuCBQvS2dmZzZs3p7u7O88++2y6u7vT3t6el19+OUmybNmylGWZpUuXJkk2btyYjRs3Zu/evXn00UfzzDPPZNWqVUPuvX79+vzhD3/IH//4x/znP//JO++8k3nz5mXp0qV5/vnnv/aGb9PChQszc+bMdHV1jdo9Tz/99CRJX19fDh06lMT7BgAAAAAAAAAAAAAAAACOT9QVAAAAAAAAAAAAAAAAAABG4Kabbsonn3yS3//+91m8eHFaW1tzzjnn5KGHHsq+ffuGXb9p06b861//yr333ptLLrkkra2tmT9/frZs2ZKyLLNhw4ZjPufaa69Ne3t7Tj755Pz0pz/NpZdemldffXUwXpok//jHPzJ//vwsXrw4kydPzuzZs3P33Xdn7ty5o7JhNA0MDKQsy5RlOWr3PNa9vG8AAAAAAAAAAAAAAAAA4HhEXQEAAAAAAAAAAAAAAAAAYAR27tyZJPnZz3425Ph3v/vdYXHPJNm2bVsaGhqyZMmSIcdPP/30zJ8/P7t27cqePXuG/d2CBQuGfP/e976XJPn3v/89eOyiiy7KSy+9lOuuuy5dXV3p7+9Pkrz77rv58Y9//I03jKZnn3023d3daW9vH7V7/i+i29zcnNNOOy2J9w0AAAAAAAAAAAAAAAAAHJ+oKwAAAAAAAAAAAAAAAAAAfE09PT05fPhwTjrppLS2tg47P2vWrGHXf/rppxkYGMi0adNSFMWQz+uvv54kee+994bda9q0aUO+t7S0JEkGBgYGj9133335y1/+kg8//DCLFi3KKaeckosuuih/+9vfRmXDie6FF15IkrS3t6e5udn7BgAAAAAAAAAAAAAAAAC+lKgrAAAAAAAAAAAAAAAAAAB8TZMmTcrUqVPz+eef58iRI8POd3d3D7t++vTpaWpqSm9vb8qyPObnJz/5yYj2FEWRq666Kn//+9/zySefZNu2bSnLMsuXL8+99947JhuqMjAwkPvuuy9Jsm7duiTeNwAAAAAAAAAAAAAAAADw5URdAQAAAAAAAAAAAAAAAABgBC6++OIkyc6dO4ccP3ToUN59991h1y9fvjx9fX158cUXh52788478/3vfz99fX0j2jJ9+vTs3r07SdLc3JzFixdn27ZtKYoiTzzxxJhsqMqmTZvyyiuv5Oc//3muuOKKwePeNwAAAAAAAAAAAAAAAABwPKKuAAAAAAAAAAAAAAAAAAAwArfffntmzJiRjRs35umnn86RI0fy9ttvZ+3atWltbR12/R133JEzzzwz11xzTZ566ql8+umn6e7uzp///Ofcdtttueeee9LU1DTiPddff33++c9/pqenJwcOHMhdd92VsiyzcOHCMdvwVSxcuDAzZ85MV1fXiP5+YGAgBw4cyPbt27No0aLcddddueaaa/LII4+kKIrB67xvAAAAAAAAAAAAAAAAAOB4RF0BAAAAAAAAAAAAAAAAAGAEzjzzzLz88stZsGBBVqxYkVmzZuUXv/hFNmzYkHPOOSc9PT0piiLXXnttkmTWrFl55ZVXsmzZsqxfvz5tbW2ZN29e/vrXv2b79u3p6OhIknR1daUoimzfvj1JMnny5Nx8881JkqIocueddyZJzjvvvCxZsiRJ8txzz2XevHm58sorM2PGjJx99tnZuXNn7r///tx0002Dm7/qhq/j8ccfT1EUKYoie/fuTX9//+D3Bx54YNj1fX19KcsyZVl+6b1bW1tz3nnnJUm2b9+eoijS1NSUuXPn5pZbbslZZ52VXbt25cEHH8zkyZOH/O1Efd8AAAAAAAAAAAAAAAAAwOgoyq/ya0cAAAAAAAAAAAAAAAAAAJjgiqLIo48+KrTJCa2zszMrV678SlFcAAAAAAAAAAAAAAAAAOALbW2oegEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAwFgSdQUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAakXUFQAAAAAAAAAAAAAAAAAAGKIoii/93HrrrVXPBAAAAAAAAAAAAAAAAAAYsaaqBwAAAAAAAAAAAAAAAAAAACeWsiyrngAAAAAAAAAAAAAAAAAA8K1qqHoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAMBYEnUFAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAGpF1BUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAqBVRVwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACgVkRdAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIBaEXUFAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAGpF1BUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAqBVRVwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACgVkRdAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIBaEXUFAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAGpF1BUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAqBVRVwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACgVkRdAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIBaKcqyLKseAQAAAAAAAAAAAAAAAAAAVSuKIueff37mzJlT9RT4Qnv27ElXV1f8TBwAAAAAAAAAAAAAAAAAvpGtTVUvAAAAAAAAAAAAAAAAAACAE8GKFSuqnjDhvPPOO0mSs88+u+IlE8ecOXP8rwIAAAAAAAAAAAAAAADAKCjKsiyrHgEAAAAAAAAAAAAAAAAAAEw8HR0dSZLOzs6KlwAAAAAAAAAAAAAAAAAADLG1oeoFAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABjSdQVAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKgVUVcAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAoFZEXQEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAWhF1BQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABqRdQVAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKgVUVcAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAoFZEXQEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAWhF1BQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABqRdQVAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKgVUVcAAAAAAAAAAAAAAAAAAP7L3v3HWl0Xfhx/fc49F716FQJFqtVaklJkyJStu2URyESlXWKJMsQlsWwFxT+6tB8z2yqtuVWTWWZZGw4uru5la7JVTpfF9QcFbRnMYLVlzqvcNNG6uz8+37+6+94uJt4fHLifx2P7/MH7fO45r/OZ/nn2BAAAAAAAAAAAAAAAAACAShF1BQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAqRdQVAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKgUUVcAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAoFJEXQEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAShF1BQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAqRdQVAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKgUUVcAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAoFJEXQEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAShF1BQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAqRdQVAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKgUUVcAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAoFJEXQEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAShF1BQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAqRdQVAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKgUUVcAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAoFJEXQEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAShF1BQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAqRdQVAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKgUUVcAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAoFJEXQEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAShF1BQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAqRdQVAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKiUeqMHAAAAAAAAAAAAAAAAAAAAJ7+tW7fm3nvvzdDQ0PDZgQMHkiSLFy8ePqvVavnEJz6RtWvXHu+JAAAAAAAAAAAAAAAAAADDirIsy0aPAAAAAAAAAAAAAAAAAAAATm779u3LhRdeeEz37t27NwsWLJjkRQAAAAAAAAAAAAAAAAAAr2lHrdELAAAAAAAAAAAAAAAAAACAk9+CBQty/vnnv+59c+fOFXQFAAAAAAAAAAAAAAAAABpO1BUAAAAAAAAAAAAAAAAAAJgQ69atS3Nz82u+3tzcnOuvv/44LgIAAAAAAAAAAAAAAAAAOLqiLMuy0SMAAAAAAAAAAAAAAAAAAICT36FDhzJ37tz8r58wP/3005k7d+5xXAUAAAAAAAAAAAAAAAAAMMqOWqMXAAAAAAAAAAAAAAAAAAAAU8M73/nOLFy4MEVRjHqtKIpcdNFFgq4AAAAAAAAAAAAAAAAAwAlB1BUAAAAAAAAAAAAAAAAAAJgw1113XZqamkadNzU15brrrmvAIgAAAAAAAAAAAAAAAACA0YqyLMtGjwAAAAAAAAAAAAAAAAAAAKaGnp6evPnNb87Q0NCI81qtlmeeeSZz5sxp0DIAAAAAAAAAAAAAAAAAgGE7ao1eAAAAAAAAAAAAAAAAAAAATB2zZ8/OBz/4wTQ1NQ2fNTU15UMf+pCgKwAAAAAAAAAAAAAAAABwwhB1BQAAAAAAAAAAAAAAAAAAJtS6deuO6QwAAAAAAAAAAAAAAAAAoFGKsizLRo8AAAAAAAAAAAAAAAAAAACmjn/+858566yz0t/fnyRpbm5OT09PZsyY0eBlAAAAAAAAAAAAAAAAAABJkh21Ri8AAAAAAAAAAAAAAAAAAACmljPPPDOXX3556vV66vV6rrjiCkFXAAAAAAAAAAAAAAAAAOCEIuoKAAAAAAAAAAAAAAAAAABMuGuvvTaDg4MZHBzM2rVrGz0HAAAAAAAAAAAAAAAAAGCEeqMHAAAAAAAAAAAAAAAAAADAiaCjo6PRE6aU/v7+TJs2LWVZpq+vz/OdYKtXr270BAAAAAAAAAAAAAAAAAA4qRVlWZaNHgEAAAAAAAAAAAAAAAAAAI1WFEWjJ8Ax8zNxAAAAAAAAAAAAAAAAABiXHbVGLwAAAAAAAAAAAAAAAAAAgBPF9u3bU5ala4KuBx98MLt27Wr4jql0bd++vdH/mwAAAAAAAAAAAAAAAADAlFBv9AAAAAAAAAAAAAAAAAAAAGBquvTSSxs9AQAAAAAAAAAAAAAAAADgqERdAQAAAAAAAAAAAAAAAACASVGv+zkzAAAAAAAAAAAAAAAAAHBiqjV6AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADA8STqCgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABUiqgrAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAFApoq4AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQKWIugIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAlSLqCgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABUiqgrAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAFApoq4AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQKWIugIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAlSLqCgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABUiqgrAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAFApoq4AAAAAAAAAAAAAAAAAADBBtm3blqIoUhRFTj311EbPmXT/+Mc/cvfdd2fJkiWZOXNmWlpa8q53vStr11xWkFkAACAASURBVK7Nvn37xv3+ra2tw8/zP1etVsub3vSmLFiwIJ/+9KezZ8+eCfgmAAAAAAAAAAAAAAAAAEDViLoCAAAAAAAAAAAAAAAAAMAEueaaa1KWZZYuXdroKcfFjTfemE2bNqW9vT1PPfVUDh8+nB/+8IfZu3dvLrroonR2do7r/Y8cOZLf//73SZL29vaUZZn+/v7s378/t912W/bv35+LL744119/fV599dWJ+EoAAAAAAAAAAAAAAAAAQEWIugIAAAAAAAAAAAAAAAAAAGO2fv36fO5zn8ucOXNy2mmn5ZJLLsn999+fwcHB3HTTTRP+eU1NTTnnnHPS3t6ehx56KDfddFPuu+++rFmzJmVZTvjnAQAAAAAAAAAAAAAAAABTU73RAwAAAAAAAAAAAAAAAAAAgJPTD37wg6OeL1iwIC0tLTl48GDKskxRFJO24Rvf+EYeeeSR7Ny5M9u2bcuaNWsm7bMAAAAAAAAAAAAAAAAAgKmj1ugBAAAAAAAAAAAAAAAAAADA1PLKK6/kX//6V9773vdOatA1SYqiyMaNG5MkW7ZsmdTPAgAAAAAAAAAAAAAAAACmDlFXAAAAAAAAAAAAAAAAAAAYo/3792flypWZPn16Tj/99FxyySV59NFHX/P+559/Pp/97Gfzjne8I9OmTcvZZ5+dVatWZe/evcP3dHZ2piiK4esvf/lLrr766syYMSOzZs3KihUrcvDgwRHv29fXly9/+cuZN29eTjvttMycOTMf+chHsnPnzgwODr7hDeO1Y8eOJMkXvvCFCXvP/+UDH/hAkqS7uzv9/f3D51V53gAAAAAAAAAAAAAAAADAGyfqCgAAAAAAAAAAAAAAAAAAY/DnP/85bW1tefLJJ/PAAw/kueeey5YtW/LVr351VAQ0SZ599tksWrQoHR0d2bJlS3p7e/Pwww+nt7c3bW1t2b17d5Jk5cqVKcsy7e3tSZLNmzdn8+bNeeaZZ7J9+/Y89NBDWbNmzYj33rhxY77zne/ku9/9bg4fPpw//elPmTdvXtrb2/PrX//6DW8Yj+eeey6f//zns2HDhqxevXrU60uWLMmsWbPS3d097s/6jzlz5iRJBgYG8sILLySpzvMGAAAAAAAAAAAAAAAAAMZG1BUAAAAAAAAAAAAAAAAAAMbglltuyYsvvphvf/vbWbZsWVpbW3PBBRfkRz/6UZ599tlR9998883561//mjvvvDNXXHFFWltbM3/+/Gzbti1lWWbTpk1H/ZwNGzakra0tp59+ei699NJceeWVeeKJJ4bjpUnyq1/9KvPnz8+yZcvS0tKSc845J9/85jdz3nnnTciGY3X48OEsX748ixcvzt13333Ue4aGhlKWZcqyHNdn/X9He68qPG8AAAAAAAAAAAAAAAAAYOxEXQEAAAAAAAAAAAAAAAAAYAx27dqVJLnssstGnL/lLW8ZFfdMks7OztRqtaxYsWLE+Zw5czJ//vzs2bMnf/vb30b93aJFi0b8+21ve1uS5O9///vw2fLly/Pb3/42n/zkJ9Pd3Z3BwcEkyYEDB7J48eJxbzgWr7zySi677LK85z3vydatW9PU1HTU+x5++OH09vamra1tTJ9zNP+J6DY3N+ess85KMvWfNwAAAAAAAAAAAAAAAAAwPqKuAAAAAAAAAAAAAAAAAADwBvX19eXll1/OqaeemtbW1lGvz549e9T9L730UoaGhjJ9+vQURTHi+t3vfpckefrpp0e91/Tp00f8e9q0aUmSoaGh4bO77rorP/nJT3Lo0KEsXbo0Z555ZpYvX56f/exnE7Lh9QwMDOSqq67KW9/61vz4xz9+zaDrZHn00UeTJG1tbWlubp7yzxsAAAAAAAAAAAAAAAAAGD9RVwAAAAAAAAAAAAAAAAAAeINOOeWUnHHGGfn3v/+dI0eOjHq9t7d31P0zZsxIvV5Pf39/yrI86vXhD394THuKosi6devyy1/+Mi+++GI6OztTlmVWrVqVO++8c9I33HDDDenr60tHR0fq9frw+dy5c9Pd3T2m73SshoaGctdddyVJPvOZzySZ+s8bAAAAAAAAAAAAAAAAABg/UVcAAAAAAAAAAAAAAAAAABiDyy+/PEmya9euEecvvPBCDhw4MOr+VatWZWBgIL/5zW9GvXb77bfn7W9/ewYGBsa0ZcaMGdm/f3+SpLm5OcuWLUtnZ2eKosjPf/7zSd1w66235o9//GO6urpyyimnjGn/eNx88815/PHH89GPfjRXXXXV8PlUfd4AAAAAAAAAAAAAAAAAwMQQdQUAAAAAAAAAAAAAAAAAgDH42te+lpkzZ2bz5s35xS9+kSNHjuSpp57Ktddem9bW1lH3f/3rX8+5556b9evX58EHH8xLL72U3t7efO9738ttt92Wb33rW6nX62Pe86lPfSp/+MMf0tfXl56entxxxx0pyzJLliyZtA333XdfvvKVr+Sxxx7LGWeckaIoRlwHDx4c9TdLlizJrFmz0t3dPabvOTQ0lJ6ennR1dWXp0qW54447sn79+mzdujVFUUzad/1vjXjeAAAAAAAAAAAAAAAAAMDEEXUFAAAAAAAAAAAAAAAAAIAxOPfcc7N79+4sWrQoH/vYxzJ79ux8/OMfz6ZNm3LBBRekr68vRVFkw4YNSZLZs2fn8ccfz8qVK7Nx48acffbZmTdvXn7605+mq6srq1evTpJ0d3enKIp0dXUlSVpaWvLFL34xSVIURW6//fYkycKFC7NixYokySOPPJJ58+blmmuuycyZM/Pud787u3btyj333JNbbrllePOxbjhWDzzwwBt+bgMDAynLMmVZvu69ra2tWbhwYZKkq6srRVGkXq/nvPPOy5e+9KWcf/752bNnT+699960tLSM+Nup+LwBAAAAAAAAAAAAAAAAgIlTlMfya0cAAAAAAAAAAAAAAAAAAJjiiqLI9u3bhTY5oXV0dOTqq68+piguAAAAAAAAAAAAAAAAAPCadtQavQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA4HgSdQUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKkXUFQAAAAAAAAAAAAAAAAAAGKEoite9br311kbPBAAAAAAAAAAAAAAAAAAYs3qjBwAAAAAAAAAAAAAAAAAAACeWsiwbPQEAAAAAAAAAAAAAAAAAYFLVGj0AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAOB4EnUFAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACpF1BUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAqBRRVwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACgUkRdAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIBKEXUFAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACpF1BUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAqBRRVwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACgUkRdAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIBKEXUFAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACpF1BUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAqBRRVwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACgUkRdAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIBKqTd6AAAAAAAAAAAAAAAAAAAAnCh2797d6AnwP/lvFAAAAAAAAAAAAAAAAAAmRlGWZdnoEQAAAAAAAAAAAAAAAAAA0GhFUTR6AhwzPxMHAAAAAAAAAAAAAAAAgHHZUW/0AgAAAAAAAAAAAAAAAAAAOBGIZE681atXJ0k6OjoavAQAAAAAAAAAAAAAAAAAYKRaowcAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABxPoq4AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQKWIugIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAlSLqCgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABUiqgrAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAFApoq4AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQKWIugIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAlSLqCgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABUiqgrAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAFApoq4AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQKWIugIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAlSLqCgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABUiqgrAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAFApoq4AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQKWIugIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAlSLqCgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABUiqgrAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAFApoq4AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQKWIugIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAlSLqCgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABUiqgrAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAFApoq4AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQKWIugIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAlSLqCgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABUiqgrAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAFApoq4AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQKWIugIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAlSLqCgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABUiqgrAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAFApoq4AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQKWIugIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAlSLqCgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABUiqgrAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAFAp9UYPAAAAAAAAAAAAAAAAAAAATn6PPfZY9u3bN+Ls0KFDSZLvf//7I87f97735f3vf/9x2wYAAAAAAAAAAAAAAAAA8N9EXQEAAAAAAAAAAAAAAAAAgHHr6enJDTfckKamptRqtSRJWZZJko0bNyZJhoaGMjg4mJ07dzZsJwAAAP/X3v3HWl3XfwB/fi73XgMuAl5+XBz0YxrCWKAl1V32Q24WJQ1lol5Hls5aKwJq1MJRW45lGjPJZMsfm3PlrOsKXC3bkLQibhKV9UdoYaYCF4I7TCm593o/3z++X+52Bt8yu5fDvefx2M4f5/V5nc/7uc+9/372BAAAAAAAAAAAAACSpCiPvQkJAAAAAAAAAAAAAAAAAADwKvX29mbSpEn5+9///i/3xo0bl4MHD6axsfEkJQMAAAAAAAAAAAAAAAAAOE5HXbUTAAAAAAAAAAAAAAAAAAAAw19DQ0OuvPLKf1nW2tDQkPb2doWuAAAAAAAAAAAAAAAAAEDVKXUFAAAAAAAAAAAAAAAAAAAGRXt7e3p6ev7f6729vbnqqqtOYiIAAAAAAAAAAAAAAAAAgBMryrIsqx0CAAAAAAAAAAAAAAAAAAAY/vr7+3PmmWdm//79J7w+efLkdHV1pa6u7iQnAwAAAAAAAAAAAAAAAACo0OFtRwAAAAAAAAAAAAAAAAAAYFDU1dVl2bJlaWxsPO5aY2NjPvKRjyh0BQAAAAAAAAAAAAAAAABOCd54BAAAAAAAAAAAAAAAAAAABk17e3t6enqOm/f09KS9vb0KiQAAAAAAAAAAAAAAAAAAjleUZVlWOwQAAAAAAAAAAAAAAAAAADBynH322dm9e3fF7HWve12efvrp6gQCAAAAAAAAAAAAAAAAAKjUUVftBAAAAAAAAAAAAAAAAAAAwMiybNmyNDQ0DHxvbGzMNddcU8VEAAAAAAAAAAAAAAAAAACVirIsy2qHAAAAAAAAAAAAAAAAAAAARo4///nPeeMb31gxe+KJJzJz5swqJQIAAAAAAAAAAAAAAAAAqNBRV+0EAAAAAAAAAAAAAAAAAADAyHL22Wdn7ty5KYoiRVFk7ty5Cl0BAAAAAAAAAAAAAAAAgFOKUlcAAAAAAAAAAAAAAAAAAGDQXX311Rk1alRGjRqVq6++utpxAAAAAAAAAAAAAAAAAAAqFGVZltUOAQAAAAAAAAAAAAAAAAAAjCx79+7NjBkzUpZlnnnmmUyfPr3akQAAAAAAAAAAAAAAAAAAjumor3YCAAAAAAAAAAAAAAAAAAA41S1dujQPPPBAtWMMWzNmzKh2hGHnsssuS0dHR7VjAAAAAAAAAAAAAAAAAMCIpdQVAAAAAAAAAAAAAAAAAABegbe//e35zGc+U+0Yw8qWLVtSFEXa2tqqHWVY+frXv17tCAAAAAAAAAAAAAAAAAAw4il1BQAAAAAAAAAAAAAAAACAV2D69Om5/PLLqx1jWDlW5trc3FzlJMNLR0dHtSMAAAAAAAAAAAAAAAAAwIin1BUAAAAAAAAAAAAAAAAAABgSylwBAAAAAAAAAAAAAAAAgFNVXbUDAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACcTEpdAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAICaotQVAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKgpSl0BAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgJqi1BUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAqClKXQEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAmqLUFQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACoKUpdAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAICaotQVAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKgpSl0BAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgJqi1BUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAqClKXQEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAmqLUFQAAAAAAAAAAAAAAAAAAqKqmpqYURVHxqaury8SJEzNv3rx88pOfzM6dO6sdEwAAAAAAAAAAAAAAAAAYQZS6AgAAAAAAAAAAAAAAAAAAVfXiiy/mt7/9bZJk8eLFKcsyvb292bVrV2644Ybs2rUr559/fq655pr84x//qHJaAAAAAAAAAAAAAAAAAGAkUOoKAAAAAAAAAAAAAAAAAADDUFNTUy644IIRe/6oUaMyderULF68OFu3bs3nP//53HPPPWlvb09ZlkN2LgAAAAAAAAAAAAAAAABQG5S6AgAAAAAAAAAAAAAAAAAAp7yvfvWredvb3pYHH3ww999/f7XjAAAAAAAAAAAAAAAAAADDnFJXAAAAAAAAAAAAAAAAAADglFcURZYvX54k2bhxY5XTAAAAAAAAAAAAAAAAAADDnVJXAAAAAAAAAAAAAAAAAAAYIocOHcpnP/vZnHXWWWlsbMzEiRPzgQ98ID/96U8HdtatW5eiKFIURS644IKB+UMPPTQwnzRp0sB8/fr1KYoiR44cybZt2wZ26uvrK64XRZHp06dnx44daWtry7hx4zJmzJhceOGF2bZt25CdP5SO5evs7Exvb+/A/G9/+1tWrFiR17/+9WlsbMzkyZOzZMmS/O53vxvY2bRp00DWoijy9NNP54orrsiECRPS3NycRYsWZffu3RXnHT16NF/60pcya9asjBkzJmeccUY+9KEP5cEHH8zLL79csftKMgAAAAAAAAAAAAAAAAAApw6lrgAAAAAAAAAAAAAAAAAAMAS6uroyf/783HfffdmwYUMOHjyYX/3qVxkzZkza2tpy1113JUnWrl2bsiwzduzYit8vXLgwZVnmLW95S8V89erVA/vveMc7UpZlyrJMX19fxfV58+bl8OHDWblyZdatW5eurq787Gc/S3d3dxYsWJBHH310SM4/ZsGCBWlubk5nZ+d//zD/T0tLS5Kkr68vBw8eTJLs27cv8+fPz/e+971s3Lgx3d3deeSRR9Ld3Z3W1tZs3749SXLJJZekLMssXrw4SbJq1aqsWrUqe/bsyXe/+91s3bo17e3tFectX7483/jGN3Lbbbfl0KFD+eMf/5hZs2Zl8eLF+fnPfz6w90ozAAAAAAAAAAAAAAAAAACnDqWuAAAAAAAAAAAAAAAAAAAwBNasWZO//OUvufXWW7No0aKcfvrpmTlzZu67775MmzYtK1asyP79+4c0w5EjR7Jx48a0trZm7NixOf/88/Ptb387PT09Wbly5ZCe3d/fP1D4OlhOdK81a9bkr3/9a2655ZZ88IMfTFNTU+bMmZP7778/ZVnm05/+9Anvdd111w08l/e+9725+OKLs2PHjoGy2CR5+OGHM2fOnFx00UUZPXp0pk6dmq997WuZOXPmoGQAAAAAAAAAAAAAAAAAAKpHqSsAAAAAAAAAAAAAAAAAAAyBH/zgB0mSiy++uGJ+2mmnpa2tLf/85z/zk5/8ZEgzjB07Nueee27F7E1velPOPPPMPP7449m3b9+Qnf3II4+ku7s7ra2tg3bPY3kbGhoyadKkJMmmTZtSV1eXRYsWVey2tLRkzpw52blzZ5577rnj7jV//vyK7zNmzEiS7N27d2C2cOHC/PKXv8zHP/7xdHZ25uWXX06SPPHEE3nPe94zsPdqMwAAAAAAAAAAAAAAAAAA1aPUFQAAAAAAAAAAAAAAAAAABtnRo0fz/PPP5zWveU3GjRt33PWpU6cmSbq6uoY0x4QJE044nzJlSpLkwIEDQ3r+YPvFL36RJGltbU1DQ8PAc+7v78/48eNTFEXF5ze/+U2S5E9/+tNx9xo/fnzF98bGxiRJf3//wOz222/Pvffem6eeeiptbW05/fTTs3DhwoHC3iT/VQYAAAAAAAAAAAAAAAAAoHqUugIAAAAAAAAAAAAAAAAAwCA77bTTMn78+Lz00kt54YUXjru+f//+JElLS8vArK6uLj09PcftHj58+IRnFEXxb3McOnQoZVkeNz9W5nqs3HWozh9M/f39uf3225Mkn/rUp5L873OeMGFC6uvr09vbm7IsT/i58MILX9WZRVHkwx/+cLZs2ZLDhw9n06ZNKcsyS5YsyS233HJSMgAAAAAAAAAAAAAAAAAAQ0OpKwAAAAAAAAAAAAAAAAAADIFLL700SfKjH/2oYn706NE8/PDDGT16dN7//vcPzKdNm5Y9e/ZU7HZ1deWZZ5454f3HjBlTUcJ6zjnn5I477qjYeemll7Jjx46K2R/+8Ifs3bs38+bNy7Rp04b0/MG0Zs2aPPbYY7n00kuzdOnSgfmSJUvS19eXbdu2Hfebm266Ka997WvT19f3qs6cMGFCdu3alSRpaGjIRRddlE2bNqUoioq/61BmAAAAAAAAAAAAAAAAAACGhlJXAAAAAAAAAAAAAAAAAAAYAjfeeGPe8IY3ZNWqVfnhD3+YF154IU8++WSuuuqq7Nu3Lxs2bMjUqVMH9t/3vvdl7969+eY3v5kXX3wxu3fvzsqVKzNlypQT3v/Nb35znnzyyTz77LPZvn17nnrqqbzzne+s2Bk/fnyuv/76bN++PUeOHMmvf/3rLFu2LI2NjdmwYUPF7mCfv2DBgjQ3N6ezs/NVPb/+/v4cOHAgmzdvTltbW26++eZce+21+c53vpOiKAb2brzxxpx11lm59tpr8+Mf/zjPP/98uru7861vfSs33HBD1q9fn/r6+leVIUk+8YlP5Pe//32OHj2aAwcO5Oabb05ZllmwYMFJywAAAAAAAAAAAAAAAAAADD6lrgAAAAAAAAAAAAAAAAAAMARaWlqyY8eOtLe3Z8WKFWlubs5b3/rWHDlyJFu2bMnHPvaxiv1169bluuuuy1e+8pVMmTIlH/3oR/O5z30uLS0tOXToUIqiyBe+8IWB/VtvvTVz587N7Nmzc8UVV2TDhg2ZPXt2xT2bmppy22235ctf/nKmTZuWd73rXZk4cWK2bt2ad7/73UN6fl9fX8qyTFmW//ZZNTU15bzzzkuSbN68OUVRpL6+PjNnzswXv/jFnHPOOdm5c2fuvvvujB49uuK3U6ZMyWOPPZZLLrkky5cvz+TJkzNr1qx8//vfz+bNm3P55ZcnSTo7O1MURTZv3pwkGT16dNauXZskKYoiN910U5LkvPPOy6JFi5Ikjz76aGbNmpUrr7wyZ5xxRmbPnp2HHnood955Z66//vr/OAMAAAAAAAAAAAAAAAAAcOooylfyFiQAAAAAAAAAAAAAAAAAANSwpUuXJkk6OjqqnOSVO/fcc3Pw4ME899xz1Y7Cf2g4/r8BAAAAAAAAAAAAAAAAwDDTUVftBAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAJ5NSVwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACgpih1BQAAAAAAAAAAAAAAAACAEWT9+vUpiiKPP/549uzZk6Iosnbt2mrHAgAAAAAAAAAALT4e1gAAAv1JREFUAAAAAAA4pdRXOwAAAAAAAAAAAAAAAAAAADB4Vq9endWrV1c7BgAAAAAAAAAAAAAAAADAKa2u2gEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAE4mpa4AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQE1R6goAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA1BSlrgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABATVHqCgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADUFKWuAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEBNUeoKAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAANQUpa4AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQE1R6goAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA1BSlrgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABATVHqCgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADUFKWuAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEBNUeoKAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAANSU+moHAAAAAAAAAAAAAAAAAACA4eCBBx5IURTVjkGNuOyyy6odAQAAAAAAAAAAAAAAAABGtKIsy7LaIQAAAAAAAAAAAAAAAAAA4FS2ffv2PPvss9WOQQ2ZMWNGWltbqx0DAAAAAAAAAAAAAAAAAEaqDqWuAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAt6airdgIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgJNJqSsAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAUFOUugIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAANaU+SUe1QwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAnCSd/wOWUNub8wKOkgAAAABJRU5ErkJggg==", - "text/plain": [ - "" - ] - }, - "execution_count": 15, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# You need to install the dependencies\n", - "tf.keras.utils.plot_model(model)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Training the deep learning model" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We can train our model with `model.fit`. We need to use a Callback to add the validation dataloader." - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "3046/3046 [==============================] - 103s 32ms/step - loss: 0.1932\n" - ] - } - ], - "source": [ - "EPOCHS = 1\n", - "validation_callback = KerasSequenceValidater(valid_dataset_tf)\n", - "start = time.time()\n", - "history = model.fit(train_dataset_tf, callbacks=[validation_callback], epochs=EPOCHS)\n", - "end = time.time() - start\n", - "total_rows = train_dataset_tf.num_rows_processed + valid_dataset_tf.num_rows_processed\n", - "print(f\"run_time: {end} - rows: {total_rows * EPOCHS} - epochs: {EPOCHS} - dl_thru: {(total_rows * EPOCHS) / end}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We save the trained model." - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "INFO:tensorflow:Assets written to: /raid/data/criteo2/test_dask/output/model.savedmodel/assets\n" - ] - } - ], - "source": [ - "model.save(os.path.join(input_path, \"model.savedmodel\"))" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.8" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/examples/scaling-criteo/04-Triton-Inference-with-HugeCTR.ipynb b/examples/scaling-criteo/04-Triton-Inference-with-HugeCTR.ipynb deleted file mode 100644 index 0348233e98f..00000000000 --- a/examples/scaling-criteo/04-Triton-Inference-with-HugeCTR.ipynb +++ /dev/null @@ -1,800 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "# Copyright 2021 NVIDIA Corporation. All Rights Reserved.\n", - "#\n", - "# Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# http://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License.\n", - "# ==============================================================================" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Scaling Criteo: Triton Inference with HugeCTR\n", - "\n", - "## Overview\n", - "\n", - "The last step is to deploy the ETL workflow and saved model to production. In the production setting, we want to transform the input data as during training (ETL). We need to apply the same mean/std for continuous features and use the same categorical mapping to convert the categories to continuous integer before we use the deep learning model for a prediction. Therefore, we deploy the NVTabular workflow with the HugeCTR model as an ensemble model to Triton Inference. The ensemble model guarantees that the same transformation are applied to the raw inputs.\n", - "\n", - "\n", - "\n", - "### Learning objectives\n", - "\n", - "In this notebook, we learn how to deploy our models to production:\n", - "\n", - "- Use **NVTabular** to generate config and model files for Triton Inference Server\n", - "- Deploy an ensemble of NVTabular workflow and HugeCTR model\n", - "- Send example request to Triton Inference Server" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Inference with Triton and HugeCTR\n", - "\n", - "First, we need to generate the Triton Inference Server configurations and save the models in the correct format. In the previous notebooks [02-ETL-with-NVTabular](./02-ETL-with-NVTabular.ipynb) and [03-Training-with-HugeCTR](./03-Training-with-HugeCTR.ipynb) we saved the NVTabular workflow and HugeCTR model to disk. We will load them." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Saving Ensemble Model for Triton Inference Server" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "After training terminates, we can see that two `.model` files are generated. We need to move them inside a temporary folder, like `criteo_hugectr/1`. Let's create these folders." - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "import numpy as np" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now we move our saved `.model` files inside 1 folder. We use only the last snapshot after `9600` iterations." - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "mv: cannot stat '*9600.model': No such file or directory\n" - ] - }, - { - "data": { - "text/plain": [ - "256" - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "os.system(\"mv *9600.model ./criteo_hugectr/1/\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now we can save our models to be deployed at the inference stage. To do so we will use export_hugectr_ensemble method below. With this method, we can generate the config.pbtxt files automatically for each model. In doing so, we should also create a hugectr_params dictionary, and define the parameters like where the amazonreview.json file will be read, slots which corresponds to number of categorical features, `embedding_vector_size`, `max_nnz`, and `n_outputs` which is number of outputs.

\n", - "The script below creates an ensemble triton server model where\n", - "- workflow is the the nvtabular workflow used in preprocessing,\n", - "- hugectr_model_path is the HugeCTR model that should be served. \n", - "- This path includes the .model files.name is the base name of the various triton models\n", - "- output_path is the path where is model will be saved to.\n", - "- cats are the categorical column names\n", - "- conts are the continuous column names" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We need to load the NVTabular workflow first" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/usr/local/lib/python3.8/dist-packages/nvtabular/workflow/workflow.py:373: UserWarning: Loading workflow generated with nvtabular version 0.10.0+124.g0930748e.dirty - but we are running nvtabular 1.2.2+4.gebf56ca0f. This might cause issues\n", - " warnings.warn(\n" - ] - } - ], - "source": [ - "import nvtabular as nvt\n", - "\n", - "BASE_DIR = os.environ.get(\"BASE_DIR\", \"/raid/data/criteo\")\n", - "input_path = os.environ.get(\"INPUT_DATA_DIR\", os.path.join(BASE_DIR, \"test_dask/output\"))\n", - "workflow = nvt.Workflow.load(os.path.join(input_path, \"workflow\"))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Let's clear the directory" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "0" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "os.system(\"rm -rf /tmp/model/*\")" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [], - "source": [ - "from nvtabular.inference.triton import export_hugectr_ensemble\n", - "\n", - "hugectr_params = dict()\n", - "hugectr_params[\"config\"] = \"/tmp/model/criteo/1/criteo.json\"\n", - "hugectr_params[\"slots\"] = 26\n", - "hugectr_params[\"max_nnz\"] = 1\n", - "hugectr_params[\"embedding_vector_size\"] = 128\n", - "hugectr_params[\"n_outputs\"] = 1\n", - "export_hugectr_ensemble(\n", - " workflow=workflow,\n", - " hugectr_model_path=\"./criteo_hugectr/1/\",\n", - " hugectr_params=hugectr_params,\n", - " name=\"criteo\",\n", - " output_path=\"/tmp/model/\",\n", - " label_columns=[\"label\"],\n", - " cats=[\"C\" + str(x) for x in range(1, 27)],\n", - " conts=[\"I\" + str(x) for x in range(1, 14)],\n", - " max_batch_size=64,\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We can take a look at the generated files." - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[01;34m/tmp/model\u001b[00m\n", - "├── \u001b[01;34mcriteo\u001b[00m\n", - "│   ├── \u001b[01;34m1\u001b[00m\n", - "│   │   ├── 0_opt_sparse_9600.model\n", - "│   │   ├── \u001b[01;34m0_sparse_9600.model\u001b[00m\n", - "│   │   │   ├── emb_vector\n", - "│   │   │   ├── key\n", - "│   │   │   └── slot_id\n", - "│   │   ├── _dense_9600.model\n", - "│   │   ├── _opt_dense_9600.model\n", - "│   │   └── criteo.json\n", - "│   └── config.pbtxt\n", - "├── \u001b[01;34mcriteo_ens\u001b[00m\n", - "│   ├── \u001b[01;34m1\u001b[00m\n", - "│   └── config.pbtxt\n", - "└── \u001b[01;34mcriteo_nvt\u001b[00m\n", - " ├── \u001b[01;34m1\u001b[00m\n", - " │   ├── model.py\n", - " │   └── \u001b[01;34mworkflow\u001b[00m\n", - " │   ├── \u001b[01;34mcategories\u001b[00m\n", - " │   │   ├── unique.C1.parquet\n", - " │   │   ├── unique.C10.parquet\n", - " │   │   ├── unique.C11.parquet\n", - " │   │   ├── unique.C12.parquet\n", - " │   │   ├── unique.C13.parquet\n", - " │   │   ├── unique.C14.parquet\n", - " │   │   ├── unique.C15.parquet\n", - " │   │   ├── unique.C16.parquet\n", - " │   │   ├── unique.C17.parquet\n", - " │   │   ├── unique.C18.parquet\n", - " │   │   ├── unique.C19.parquet\n", - " │   │   ├── unique.C2.parquet\n", - " │   │   ├── unique.C20.parquet\n", - " │   │   ├── unique.C21.parquet\n", - " │   │   ├── unique.C22.parquet\n", - " │   │   ├── unique.C23.parquet\n", - " │   │   ├── unique.C24.parquet\n", - " │   │   ├── unique.C25.parquet\n", - " │   │   ├── unique.C26.parquet\n", - " │   │   ├── unique.C3.parquet\n", - " │   │   ├── unique.C4.parquet\n", - " │   │   ├── unique.C5.parquet\n", - " │   │   ├── unique.C6.parquet\n", - " │   │   ├── unique.C7.parquet\n", - " │   │   ├── unique.C8.parquet\n", - " │   │   └── unique.C9.parquet\n", - " │   ├── metadata.json\n", - " │   └── workflow.pkl\n", - " └── config.pbtxt\n", - "\n", - "9 directories, 39 files\n" - ] - } - ], - "source": [ - "!tree /tmp/model" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We need to write a configuration file with the stored model weights and model configuration." - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": { - "tags": [ - "flake8-noqa-cell" - ] - }, - "outputs": [], - "source": [ - "# %%writefile '/tmp/model/ps.json'\n", - "import json\n", - "\n", - "config = json.dumps(\n", - "{\n", - " \"supportlonglong\": \"true\",\n", - " \"models\": [\n", - " {\n", - " \"model\": \"criteo\",\n", - " \"sparse_files\": [\"/tmp/model/criteo/1/0_sparse_9600.model\"],\n", - " \"dense_file\": \"/tmp/model/criteo/1/_dense_9600.model\",\n", - " \"network_file\": \"/tmp/model/criteo/1/criteo.json\",\n", - " \"max_batch_size\": \"64\",\n", - " \"gpucache\": \"true\",\n", - " \"hit_rate_threshold\": \"0.9\",\n", - " \"gpucacheper\": \"0.5\",\n", - " \"num_of_worker_buffer_in_pool\": \"4\",\n", - " \"num_of_refresher_buffer_in_pool\": \"1\",\n", - " \"cache_refresh_percentage_per_iteration\": 0.2,\n", - " \"deployed_device_list\": [\"0\"],\n", - " \"default_value_for_each_table\": [\"0.0\", \"0.0\"],\n", - " \"maxnum_catfeature_query_per_table_per_sample\": [2, 26],\n", - " \"embedding_vecsize_per_table\": [16 for x in range(26)],\n", - " }\n", - " ],\n", - "}\n", - ")\n", - "\n", - "config = json.loads(config)\n", - "with open(\"/tmp/model/ps.json\", \"w\", encoding=\"utf-8\") as f:\n", - " json.dump(config, f)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Loading Ensemble Model with Triton Inference Server\n", - "\n", - "We have only saved the models for Triton Inference Server. We started Triton Inference Server in explicit mode, meaning that we need to send a request that Triton will load the ensemble model." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We connect to the Triton Inference Server." - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "client created.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/usr/local/lib/python3.8/dist-packages/tritonhttpclient/__init__.py:31: DeprecationWarning: The package `tritonhttpclient` is deprecated and will be removed in a future version. Please use instead `tritonclient.http`\n", - " warnings.warn(\n" - ] - } - ], - "source": [ - "import tritonhttpclient\n", - "\n", - "try:\n", - " triton_client = tritonhttpclient.InferenceServerClient(url=\"localhost:8000\", verbose=True)\n", - " print(\"client created.\")\n", - "except Exception as e:\n", - " print(\"channel creation failed: \" + str(e))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We deactivate warnings." - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [], - "source": [ - "import warnings\n", - "\n", - "warnings.filterwarnings(\"ignore\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We check if the server is alive." - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "GET /v2/health/live, headers None\n", - "\n" - ] - }, - { - "data": { - "text/plain": [ - "True" - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "triton_client.is_server_live()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We check the available models in the repositories:\n", - "- criteo_ens: Ensemble \n", - "- criteo_nvt: NVTabular \n", - "- criteo: HugeCTR model" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "POST /v2/repository/index, headers None\n", - "\n", - "\n", - "bytearray(b'[{\"name\":\"criteo\"},{\"name\":\"criteo_ens\"},{\"name\":\"criteo_nvt\"}]')\n" - ] - }, - { - "data": { - "text/plain": [ - "[{'name': 'criteo'}, {'name': 'criteo_ens'}, {'name': 'criteo_nvt'}]" - ] - }, - "execution_count": 12, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "triton_client.get_model_repository_index()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We load the models individually." - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "POST /v2/repository/models/criteo_nvt/load, headers None\n", - "{}\n", - "\n", - "Loaded model 'criteo_nvt'\n", - "CPU times: user 7.84 ms, sys: 14 ms, total: 21.9 ms\n", - "Wall time: 36.6 s\n" - ] - } - ], - "source": [ - "%%time\n", - "\n", - "triton_client.load_model(model_name=\"criteo_nvt\")" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "POST /v2/repository/models/criteo/load, headers None\n", - "{}\n", - "\n", - "Loaded model 'criteo'\n", - "CPU times: user 3.75 ms, sys: 3.62 ms, total: 7.38 ms\n", - "Wall time: 6.04 s\n" - ] - } - ], - "source": [ - "%%time\n", - "\n", - "triton_client.load_model(model_name=\"criteo\")" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "POST /v2/repository/models/criteo_ens/load, headers None\n", - "{}\n", - "\n", - "Loaded model 'criteo_ens'\n", - "CPU times: user 12.1 ms, sys: 9.87 ms, total: 21.9 ms\n", - "Wall time: 37.5 s\n" - ] - } - ], - "source": [ - "%%time\n", - "\n", - "triton_client.load_model(model_name=\"criteo_ens\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Example Request to Triton Inference Server\n", - "\n", - "Now, the models are loaded and we can create a sample request. We read an example **raw batch** for inference." - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - " I1 I2 I3 I4 I5 I6 I7 I8 I9 I10 ... C17 \\\n", - "0 5 110 16 1 0 14 7 1 ... -771205462 \n", - "1 32 3 5 1 0 0 61 5 0 ... -771205462 \n", - "2 233 1 146 1 0 0 99 7 0 ... -771205462 \n", - "\n", - " C18 C19 C20 C21 C22 C23 \\\n", - "0 -1206449222 -1793932789 -1014091992 351689309 632402057 -675152885 \n", - "1 -1578429167 -1793932789 -20981661 -1556988767 -924717482 391309800 \n", - "2 1653545869 -1793932789 -1014091992 351689309 632402057 -675152885 \n", - "\n", - " C24 C25 C26 \n", - "0 2091868316 809724924 -317696227 \n", - "1 1966410890 -1726799382 -1218975401 \n", - "2 883538181 -10139646 -317696227 \n", - "\n", - "[3 rows x 39 columns]\n" - ] - } - ], - "source": [ - "# Get dataframe library - cudf or pandas\n", - "from merlin.core.dispatch import get_lib\n", - "\n", - "df_lib = get_lib()\n", - "\n", - "# read in the workflow (to get input/output schema to call triton with)\n", - "BASE_DIR = os.environ.get(\"BASE_DIR\", \"/raid/data/criteo\")\n", - "batch_path = os.path.join(BASE_DIR, \"converted/criteo\")\n", - "batch = df_lib.read_parquet(os.path.join(batch_path, \"*.parquet\"), num_rows=3)\n", - "batch = batch[[x for x in batch.columns if x != \"label\"]]\n", - "print(batch)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We prepare the batch for inference by using correct column names and data types. We use the same datatypes as defined in our dataframe." - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "I1 int32\n", - "I2 int32\n", - "I3 int32\n", - "I4 int32\n", - "I5 int32\n", - "I6 int32\n", - "I7 int32\n", - "I8 int32\n", - "I9 int32\n", - "I10 int32\n", - "I11 int32\n", - "I12 int32\n", - "I13 int32\n", - "C1 int32\n", - "C2 int32\n", - "C3 int32\n", - "C4 int32\n", - "C5 int32\n", - "C6 int32\n", - "C7 int32\n", - "C8 int32\n", - "C9 int32\n", - "C10 int32\n", - "C11 int32\n", - "C12 int32\n", - "C13 int32\n", - "C14 int32\n", - "C15 int32\n", - "C16 int32\n", - "C17 int32\n", - "C18 int32\n", - "C19 int32\n", - "C20 int32\n", - "C21 int32\n", - "C22 int32\n", - "C23 int32\n", - "C24 int32\n", - "C25 int32\n", - "C26 int32\n", - "dtype: object" - ] - }, - "execution_count": 17, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "batch.dtypes" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "metadata": {}, - "outputs": [], - "source": [ - "import tritonclient.http as httpclient\n", - "from tritonclient.utils import np_to_triton_dtype\n", - "\n", - "inputs = []\n", - "\n", - "col_names = list(batch.columns)\n", - "col_dtypes = [np.int32] * len(col_names)\n", - "\n", - "for i, col in enumerate(batch.columns):\n", - " d = batch[col].fillna(0).values_host.astype(col_dtypes[i])\n", - " d = d.reshape(len(d), 1)\n", - " inputs.append(httpclient.InferInput(col_names[i], d.shape, np_to_triton_dtype(col_dtypes[i])))\n", - " inputs[i].set_data_from_numpy(d)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We send the request to the triton server and collect the last output." - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "POST /v2/models/criteo_ens/infer, headers {'Inference-Header-Content-Length': 3383}\n", - "b'{\"id\":\"1\",\"inputs\":[{\"name\":\"I1\",\"shape\":[3,1],\"datatype\":\"INT32\",\"parameters\":{\"binary_data_size\":12}},{\"name\":\"I2\",\"shape\":[3,1],\"datatype\":\"INT32\",\"parameters\":{\"binary_data_size\":12}},{\"name\":\"I3\",\"shape\":[3,1],\"datatype\":\"INT32\",\"parameters\":{\"binary_data_size\":12}},{\"name\":\"I4\",\"shape\":[3,1],\"datatype\":\"INT32\",\"parameters\":{\"binary_data_size\":12}},{\"name\":\"I5\",\"shape\":[3,1],\"datatype\":\"INT32\",\"parameters\":{\"binary_data_size\":12}},{\"name\":\"I6\",\"shape\":[3,1],\"datatype\":\"INT32\",\"parameters\":{\"binary_data_size\":12}},{\"name\":\"I7\",\"shape\":[3,1],\"datatype\":\"INT32\",\"parameters\":{\"binary_data_size\":12}},{\"name\":\"I8\",\"shape\":[3,1],\"datatype\":\"INT32\",\"parameters\":{\"binary_data_size\":12}},{\"name\":\"I9\",\"shape\":[3,1],\"datatype\":\"INT32\",\"parameters\":{\"binary_data_size\":12}},{\"name\":\"I10\",\"shape\":[3,1],\"datatype\":\"INT32\",\"parameters\":{\"binary_data_size\":12}},{\"name\":\"I11\",\"shape\":[3,1],\"datatype\":\"INT32\",\"parameters\":{\"binary_data_size\":12}},{\"name\":\"I12\",\"shape\":[3,1],\"datatype\":\"INT32\",\"parameters\":{\"binary_data_size\":12}},{\"name\":\"I13\",\"shape\":[3,1],\"datatype\":\"INT32\",\"parameters\":{\"binary_data_size\":12}},{\"name\":\"C1\",\"shape\":[3,1],\"datatype\":\"INT32\",\"parameters\":{\"binary_data_size\":12}},{\"name\":\"C2\",\"shape\":[3,1],\"datatype\":\"INT32\",\"parameters\":{\"binary_data_size\":12}},{\"name\":\"C3\",\"shape\":[3,1],\"datatype\":\"INT32\",\"parameters\":{\"binary_data_size\":12}},{\"name\":\"C4\",\"shape\":[3,1],\"datatype\":\"INT32\",\"parameters\":{\"binary_data_size\":12}},{\"name\":\"C5\",\"shape\":[3,1],\"datatype\":\"INT32\",\"parameters\":{\"binary_data_size\":12}},{\"name\":\"C6\",\"shape\":[3,1],\"datatype\":\"INT32\",\"parameters\":{\"binary_data_size\":12}},{\"name\":\"C7\",\"shape\":[3,1],\"datatype\":\"INT32\",\"parameters\":{\"binary_data_size\":12}},{\"name\":\"C8\",\"shape\":[3,1],\"datatype\":\"INT32\",\"parameters\":{\"binary_data_size\":12}},{\"name\":\"C9\",\"shape\":[3,1],\"datatype\":\"INT32\",\"parameters\":{\"binary_data_size\":12}},{\"name\":\"C10\",\"shape\":[3,1],\"datatype\":\"INT32\",\"parameters\":{\"binary_data_size\":12}},{\"name\":\"C11\",\"shape\":[3,1],\"datatype\":\"INT32\",\"parameters\":{\"binary_data_size\":12}},{\"name\":\"C12\",\"shape\":[3,1],\"datatype\":\"INT32\",\"parameters\":{\"binary_data_size\":12}},{\"name\":\"C13\",\"shape\":[3,1],\"datatype\":\"INT32\",\"parameters\":{\"binary_data_size\":12}},{\"name\":\"C14\",\"shape\":[3,1],\"datatype\":\"INT32\",\"parameters\":{\"binary_data_size\":12}},{\"name\":\"C15\",\"shape\":[3,1],\"datatype\":\"INT32\",\"parameters\":{\"binary_data_size\":12}},{\"name\":\"C16\",\"shape\":[3,1],\"datatype\":\"INT32\",\"parameters\":{\"binary_data_size\":12}},{\"name\":\"C17\",\"shape\":[3,1],\"datatype\":\"INT32\",\"parameters\":{\"binary_data_size\":12}},{\"name\":\"C18\",\"shape\":[3,1],\"datatype\":\"INT32\",\"parameters\":{\"binary_data_size\":12}},{\"name\":\"C19\",\"shape\":[3,1],\"datatype\":\"INT32\",\"parameters\":{\"binary_data_size\":12}},{\"name\":\"C20\",\"shape\":[3,1],\"datatype\":\"INT32\",\"parameters\":{\"binary_data_size\":12}},{\"name\":\"C21\",\"shape\":[3,1],\"datatype\":\"INT32\",\"parameters\":{\"binary_data_size\":12}},{\"name\":\"C22\",\"shape\":[3,1],\"datatype\":\"INT32\",\"parameters\":{\"binary_data_size\":12}},{\"name\":\"C23\",\"shape\":[3,1],\"datatype\":\"INT32\",\"parameters\":{\"binary_data_size\":12}},{\"name\":\"C24\",\"shape\":[3,1],\"datatype\":\"INT32\",\"parameters\":{\"binary_data_size\":12}},{\"name\":\"C25\",\"shape\":[3,1],\"datatype\":\"INT32\",\"parameters\":{\"binary_data_size\":12}},{\"name\":\"C26\",\"shape\":[3,1],\"datatype\":\"INT32\",\"parameters\":{\"binary_data_size\":12}}],\"outputs\":[{\"name\":\"OUTPUT0\",\"parameters\":{\"binary_data\":true}}]}\\x05\\x00\\x00\\x00 \\x00\\x00\\x00\\x00\\x00\\x00\\x00n\\x00\\x00\\x00\\x03\\x00\\x00\\x00\\xe9\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x05\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x10\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x92\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x0e\\x00\\x00\\x00=\\x00\\x00\\x00c\\x00\\x00\\x00\\x07\\x00\\x00\\x00\\x05\\x00\\x00\\x00\\x07\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x002\\x01\\x00\\x00U\\x0c\\x00\\x00\\x1d\\x0c\\x00\\x00\\x00\\x00\\x00\\x00\\x05\\x00\\x00\\x00\\x01\\x00\\x00\\x00y\\rwb\\x8d\\xfd\\xf3\\xe5y\\rwbX]\\x1f\\xe2\\xa6\\xff\\xaa\\xa0\\x03B\\x98\\xad/D\\xea\\xaf\\xd5\\x15\\xaao\\r\\xc6\\xbeb\\xcf\\x7f\\\\\\x94!4\\x8a\\xda\\xeeIl8H\\'\\xb08#\\x9f\\xd6\\xdcL\\xfa>\\xdcL\\xfa>\\xdcL\\xaaV\\x08\\xd2\\xaaV\\x08\\xd2\\xaaV\\x08\\xd2\\xba\\x0b\\x17\\xb8\\x11\\x15\\xeb\\xa1\\x8d\\x1b\\x8fb\\x0b\\xc2\\x12\\x95\\x0b\\xc2\\x12\\x95\\x0b\\xc2\\x12\\x95(/\\x8e\\xc3c\\xd8\\xbf\\xfe(/\\x8e\\xc3]Z\\xf6\\x14\\xa1<2\\xa3]Z\\xf6\\x14\\x89\\xb0\\xb1%V\\xee\\xe1\\xc8\\x89\\xb0\\xb1%\\x0b\\xfc\\xc1\\xd7\\xe8\\xe9R\\x17\\x0b\\xfc\\xc1\\xd7\\x9c`\\xaf|\\x8a\\x0c5u\\x05\\xb9\\xa94\\xfckC0\\xea!\\x13\\x99\\x02He\\xff\\x1dW\\x10\\xedW\\xe9W\\xb7\\x1dW\\x10\\xed'\n", - "\n", - "bytearray(b'{\"id\":\"1\",\"model_name\":\"criteo_ens\",\"model_version\":\"1\",\"parameters\":{\"sequence_id\":0,\"sequence_start\":false,\"sequence_end\":false},\"outputs\":[{\"name\":\"OUTPUT0\",\"datatype\":\"FP32\",\"shape\":[3],\"parameters\":{\"binary_data_size\":12}}]}')\n", - "predicted sigmoid result:\n", - " [0.03639793 0.0345494 0.03440537]\n" - ] - } - ], - "source": [ - "# placeholder variables for the output\n", - "outputs = [httpclient.InferRequestedOutput(\"OUTPUT0\")]\n", - "\n", - "# build a client to connect to our server.\n", - "# This InferenceServerClient object is what we'll be using to talk to Triton.\n", - "# make the request with tritonclient.http.InferInput object\n", - "response = triton_client.infer(\"criteo_ens\", inputs, request_id=\"1\", outputs=outputs)\n", - "\n", - "print(\"predicted sigmoid result:\\n\", response.as_numpy(\"OUTPUT0\"))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Let's unload the model. We need to unload each model." - ] - }, - { - "cell_type": "code", - "execution_count": 20, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "POST /v2/repository/models/criteo_ens/unload, headers None\n", - "{\"parameters\":{\"unload_dependents\":false}}\n", - "\n", - "Loaded model 'criteo_ens'\n", - "POST /v2/repository/models/criteo_nvt/unload, headers None\n", - "{\"parameters\":{\"unload_dependents\":false}}\n", - "\n", - "Loaded model 'criteo_nvt'\n", - "POST /v2/repository/models/criteo/unload, headers None\n", - "{\"parameters\":{\"unload_dependents\":false}}\n", - "\n", - "Loaded model 'criteo'\n" - ] - } - ], - "source": [ - "triton_client.unload_model(model_name=\"criteo_ens\")\n", - "triton_client.unload_model(model_name=\"criteo_nvt\")\n", - "triton_client.unload_model(model_name=\"criteo\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "celltoolbar": "Tags", - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.10" - }, - "vscode": { - "interpreter": { - "hash": "916dbcbb3f70747c44a77c7bcd40155683ae19c65e1c03b4aa3499c5328201f1" - } - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/examples/scaling-criteo/04-Triton-Inference-with-TF.ipynb b/examples/scaling-criteo/04-Triton-Inference-with-TF.ipynb deleted file mode 100644 index 16940424ea8..00000000000 --- a/examples/scaling-criteo/04-Triton-Inference-with-TF.ipynb +++ /dev/null @@ -1,1131 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Copyright 2021 NVIDIA Corporation. All Rights Reserved.\n", - "#\n", - "# Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# http://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License.\n", - "# ==============================================================================" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Scaling Criteo: Triton Inference with TensorFlow\n", - "\n", - "## Overview\n", - "\n", - "The last step is to deploy the ETL workflow and saved model to production. In the production setting, we want to transform the input data as during training (ETL). We need to apply the same mean/std for continuous features and use the same categorical mapping to convert the categories to continuous integer before we use the deep learning model for a prediction. Therefore, we deploy the NVTabular workflow with the TensorFlow model as an ensemble model to Triton Inference. The ensemble model garantuees that the same transformation are applied to the raw inputs.\n", - "\n", - "\n", - "\n", - "### Learning objectives\n", - "\n", - "In this notebook, we learn how to deploy our models to production\n", - "\n", - "- Use **NVTabular** to generate config and model files for Triton Inference Server\n", - "- Deploy an ensemble of NVTabular workflow and TensorFlow model\n", - "- Send example request to Triton Inference Server" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Inference with Triton and TensorFlow\n", - "\n", - "First, we need to generate the Triton Inference Server configurations and save the models in the correct format. In the previous notebooks [02-ETL-with-NVTabular](./02-ETL-with-NVTabular.ipynb) and [03-Training-with-TF](./03-Training-with-TF.ipynb) we saved the NVTabular workflow and TensorFlow model to disk. We will load them." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Saving Ensemble Model for Triton Inference Server" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "\n", - "import tensorflow as tf\n", - "import nvtabular as nvt" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "BASE_DIR = os.environ.get(\"BASE_DIR\", \"/raid/data/criteo\")\n", - "input_path = os.environ.get(\"INPUT_DATA_DIR\", os.path.join(BASE_DIR, \"test_dask/output\"))" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/usr/local/lib/python3.8/dist-packages/nvtabular/workflow/workflow.py:373: UserWarning: Loading workflow generated with nvtabular version 0.10.0+123.g44d3c3e8.dirty - but we are running nvtabular 1.2.2+4.gebf56ca0f. This might cause issues\n", - " warnings.warn(\n" - ] - } - ], - "source": [ - "workflow = nvt.Workflow.load(os.path.join(input_path, \"workflow\"))" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2022-07-14 23:15:34.019787: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 AVX512F FMA\n", - "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", - "2022-07-14 23:15:36.054064: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1532] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 46898 MB memory: -> device: 0, name: Quadro RTX 8000, pci bus id: 0000:15:00.0, compute capability: 7.5\n", - "2022-07-14 23:15:36.054715: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1532] Created device /job:localhost/replica:0/task:0/device:GPU:1 with 46890 MB memory: -> device: 1, name: Quadro RTX 8000, pci bus id: 0000:2d:00.0, compute capability: 7.5\n" - ] - } - ], - "source": [ - "model = tf.keras.models.load_model(os.path.join(input_path, \"model.savedmodel\"))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "TensorFlow expect the Integer as `int32` datatype. Therefore, we need to define the NVTabular output datatypes to `int32` for categorical features." - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [], - "source": [ - "for key in workflow.output_dtypes.keys():\n", - " if key.startswith(\"C\"):\n", - " workflow.output_dtypes[key] = \"int32\"" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "NVTabular provides an easy function to deploy the ensemble model for Triton Inference Server." - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [], - "source": [ - "from nvtabular.inference.triton import export_tensorflow_ensemble" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "WARNING:absl:Function `_wrapped_model` contains input name(s) C1, C10, C11, C12, C13, C14, C15, C16, C17, C18, C19, C2, C20, C21, C22, C23, C24, C25, C26, C3, C4, C5, C6, C7, C8, C9, I1, I10, I11, I12, I13, I2, I3, I4, I5, I6, I7, I8, I9 with unsupported characters which will be renamed to c1, c10, c11, c12, c13, c14, c15, c16, c17, c18, c19, c2, c20, c21, c22, c23, c24, c25, c26, c3, c4, c5, c6, c7, c8, c9, i1, i10, i11, i12, i13, i2, i3, i4, i5, i6, i7, i8, i9 in the SavedModel.\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "INFO:tensorflow:Assets written to: /tmp/model/models/criteo_tf/1/model.savedmodel/assets\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "INFO:tensorflow:Assets written to: /tmp/model/models/criteo_tf/1/model.savedmodel/assets\n", - "WARNING:absl: has the same name 'DenseFeatures' as a built-in Keras object. Consider renaming to avoid naming conflicts when loading with `tf.keras.models.load_model`. If renaming is not possible, pass the object in the `custom_objects` parameter of the load function.\n", - "/usr/local/lib/python3.8/dist-packages/nvtabular/inference/triton/ensemble.py:85: UserWarning: TF model expects int32 for column C1, but workflow is producing type int64. Overriding dtype in NVTabular workflow.\n", - " warnings.warn(\n", - "/usr/local/lib/python3.8/dist-packages/nvtabular/inference/triton/ensemble.py:85: UserWarning: TF model expects int32 for column C10, but workflow is producing type int64. Overriding dtype in NVTabular workflow.\n", - " warnings.warn(\n", - "/usr/local/lib/python3.8/dist-packages/nvtabular/inference/triton/ensemble.py:85: UserWarning: TF model expects int32 for column C11, but workflow is producing type int64. Overriding dtype in NVTabular workflow.\n", - " warnings.warn(\n", - "/usr/local/lib/python3.8/dist-packages/nvtabular/inference/triton/ensemble.py:85: UserWarning: TF model expects int32 for column C12, but workflow is producing type int64. Overriding dtype in NVTabular workflow.\n", - " warnings.warn(\n", - "/usr/local/lib/python3.8/dist-packages/nvtabular/inference/triton/ensemble.py:85: UserWarning: TF model expects int32 for column C13, but workflow is producing type int64. Overriding dtype in NVTabular workflow.\n", - " warnings.warn(\n", - "/usr/local/lib/python3.8/dist-packages/nvtabular/inference/triton/ensemble.py:85: UserWarning: TF model expects int32 for column C14, but workflow is producing type int64. Overriding dtype in NVTabular workflow.\n", - " warnings.warn(\n", - "/usr/local/lib/python3.8/dist-packages/nvtabular/inference/triton/ensemble.py:85: UserWarning: TF model expects int32 for column C15, but workflow is producing type int64. Overriding dtype in NVTabular workflow.\n", - " warnings.warn(\n", - "/usr/local/lib/python3.8/dist-packages/nvtabular/inference/triton/ensemble.py:85: UserWarning: TF model expects int32 for column C16, but workflow is producing type int64. Overriding dtype in NVTabular workflow.\n", - " warnings.warn(\n", - "/usr/local/lib/python3.8/dist-packages/nvtabular/inference/triton/ensemble.py:85: UserWarning: TF model expects int32 for column C17, but workflow is producing type int64. Overriding dtype in NVTabular workflow.\n", - " warnings.warn(\n", - "/usr/local/lib/python3.8/dist-packages/nvtabular/inference/triton/ensemble.py:85: UserWarning: TF model expects int32 for column C18, but workflow is producing type int64. Overriding dtype in NVTabular workflow.\n", - " warnings.warn(\n", - "/usr/local/lib/python3.8/dist-packages/nvtabular/inference/triton/ensemble.py:85: UserWarning: TF model expects int32 for column C19, but workflow is producing type int64. Overriding dtype in NVTabular workflow.\n", - " warnings.warn(\n", - "/usr/local/lib/python3.8/dist-packages/nvtabular/inference/triton/ensemble.py:85: UserWarning: TF model expects int32 for column C2, but workflow is producing type int64. Overriding dtype in NVTabular workflow.\n", - " warnings.warn(\n", - "/usr/local/lib/python3.8/dist-packages/nvtabular/inference/triton/ensemble.py:85: UserWarning: TF model expects int32 for column C20, but workflow is producing type int64. Overriding dtype in NVTabular workflow.\n", - " warnings.warn(\n", - "/usr/local/lib/python3.8/dist-packages/nvtabular/inference/triton/ensemble.py:85: UserWarning: TF model expects int32 for column C21, but workflow is producing type int64. Overriding dtype in NVTabular workflow.\n", - " warnings.warn(\n", - "/usr/local/lib/python3.8/dist-packages/nvtabular/inference/triton/ensemble.py:85: UserWarning: TF model expects int32 for column C22, but workflow is producing type int64. Overriding dtype in NVTabular workflow.\n", - " warnings.warn(\n", - "/usr/local/lib/python3.8/dist-packages/nvtabular/inference/triton/ensemble.py:85: UserWarning: TF model expects int32 for column C23, but workflow is producing type int64. Overriding dtype in NVTabular workflow.\n", - " warnings.warn(\n", - "/usr/local/lib/python3.8/dist-packages/nvtabular/inference/triton/ensemble.py:85: UserWarning: TF model expects int32 for column C24, but workflow is producing type int64. Overriding dtype in NVTabular workflow.\n", - " warnings.warn(\n", - "/usr/local/lib/python3.8/dist-packages/nvtabular/inference/triton/ensemble.py:85: UserWarning: TF model expects int32 for column C25, but workflow is producing type int64. Overriding dtype in NVTabular workflow.\n", - " warnings.warn(\n", - "/usr/local/lib/python3.8/dist-packages/nvtabular/inference/triton/ensemble.py:85: UserWarning: TF model expects int32 for column C26, but workflow is producing type int64. Overriding dtype in NVTabular workflow.\n", - " warnings.warn(\n", - "/usr/local/lib/python3.8/dist-packages/nvtabular/inference/triton/ensemble.py:85: UserWarning: TF model expects int32 for column C3, but workflow is producing type int64. Overriding dtype in NVTabular workflow.\n", - " warnings.warn(\n", - "/usr/local/lib/python3.8/dist-packages/nvtabular/inference/triton/ensemble.py:85: UserWarning: TF model expects int32 for column C4, but workflow is producing type int64. Overriding dtype in NVTabular workflow.\n", - " warnings.warn(\n", - "/usr/local/lib/python3.8/dist-packages/nvtabular/inference/triton/ensemble.py:85: UserWarning: TF model expects int32 for column C5, but workflow is producing type int64. Overriding dtype in NVTabular workflow.\n", - " warnings.warn(\n", - "/usr/local/lib/python3.8/dist-packages/nvtabular/inference/triton/ensemble.py:85: UserWarning: TF model expects int32 for column C6, but workflow is producing type int64. Overriding dtype in NVTabular workflow.\n", - " warnings.warn(\n", - "/usr/local/lib/python3.8/dist-packages/nvtabular/inference/triton/ensemble.py:85: UserWarning: TF model expects int32 for column C7, but workflow is producing type int64. Overriding dtype in NVTabular workflow.\n", - " warnings.warn(\n", - "/usr/local/lib/python3.8/dist-packages/nvtabular/inference/triton/ensemble.py:85: UserWarning: TF model expects int32 for column C8, but workflow is producing type int64. Overriding dtype in NVTabular workflow.\n", - " warnings.warn(\n", - "/usr/local/lib/python3.8/dist-packages/nvtabular/inference/triton/ensemble.py:85: UserWarning: TF model expects int32 for column C9, but workflow is producing type int64. Overriding dtype in NVTabular workflow.\n", - " warnings.warn(\n", - "/usr/local/lib/python3.8/dist-packages/nvtabular/inference/triton/ensemble.py:85: UserWarning: TF model expects float32 for column I1, but workflow is producing type float64. Overriding dtype in NVTabular workflow.\n", - " warnings.warn(\n", - "/usr/local/lib/python3.8/dist-packages/nvtabular/inference/triton/ensemble.py:85: UserWarning: TF model expects float32 for column I10, but workflow is producing type float64. Overriding dtype in NVTabular workflow.\n", - " warnings.warn(\n", - "/usr/local/lib/python3.8/dist-packages/nvtabular/inference/triton/ensemble.py:85: UserWarning: TF model expects float32 for column I11, but workflow is producing type float64. Overriding dtype in NVTabular workflow.\n", - " warnings.warn(\n", - "/usr/local/lib/python3.8/dist-packages/nvtabular/inference/triton/ensemble.py:85: UserWarning: TF model expects float32 for column I12, but workflow is producing type float64. Overriding dtype in NVTabular workflow.\n", - " warnings.warn(\n", - "/usr/local/lib/python3.8/dist-packages/nvtabular/inference/triton/ensemble.py:85: UserWarning: TF model expects float32 for column I13, but workflow is producing type float64. Overriding dtype in NVTabular workflow.\n", - " warnings.warn(\n", - "/usr/local/lib/python3.8/dist-packages/nvtabular/inference/triton/ensemble.py:85: UserWarning: TF model expects float32 for column I2, but workflow is producing type float64. Overriding dtype in NVTabular workflow.\n", - " warnings.warn(\n", - "/usr/local/lib/python3.8/dist-packages/nvtabular/inference/triton/ensemble.py:85: UserWarning: TF model expects float32 for column I3, but workflow is producing type float64. Overriding dtype in NVTabular workflow.\n", - " warnings.warn(\n", - "/usr/local/lib/python3.8/dist-packages/nvtabular/inference/triton/ensemble.py:85: UserWarning: TF model expects float32 for column I4, but workflow is producing type float64. Overriding dtype in NVTabular workflow.\n", - " warnings.warn(\n", - "/usr/local/lib/python3.8/dist-packages/nvtabular/inference/triton/ensemble.py:85: UserWarning: TF model expects float32 for column I5, but workflow is producing type float64. Overriding dtype in NVTabular workflow.\n", - " warnings.warn(\n", - "/usr/local/lib/python3.8/dist-packages/nvtabular/inference/triton/ensemble.py:85: UserWarning: TF model expects float32 for column I6, but workflow is producing type float64. Overriding dtype in NVTabular workflow.\n", - " warnings.warn(\n", - "/usr/local/lib/python3.8/dist-packages/nvtabular/inference/triton/ensemble.py:85: UserWarning: TF model expects float32 for column I7, but workflow is producing type float64. Overriding dtype in NVTabular workflow.\n", - " warnings.warn(\n", - "/usr/local/lib/python3.8/dist-packages/nvtabular/inference/triton/ensemble.py:85: UserWarning: TF model expects float32 for column I8, but workflow is producing type float64. Overriding dtype in NVTabular workflow.\n", - " warnings.warn(\n", - "/usr/local/lib/python3.8/dist-packages/nvtabular/inference/triton/ensemble.py:85: UserWarning: TF model expects float32 for column I9, but workflow is producing type float64. Overriding dtype in NVTabular workflow.\n", - " warnings.warn(\n" - ] - } - ], - "source": [ - "export_tensorflow_ensemble(model, workflow, \"criteo\", \"/tmp/model/models/\", [\"label\"])" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We can take a look on the generated files." - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[01;34m/tmp/model\u001b[00m\n", - "└── \u001b[01;34mmodels\u001b[00m\n", - " ├── \u001b[01;34mcriteo\u001b[00m\n", - " │   ├── \u001b[01;34m1\u001b[00m\n", - " │   └── config.pbtxt\n", - " ├── \u001b[01;34mcriteo_nvt\u001b[00m\n", - " │   ├── \u001b[01;34m1\u001b[00m\n", - " │   │   ├── model.py\n", - " │   │   └── \u001b[01;34mworkflow\u001b[00m\n", - " │   │   ├── \u001b[01;34mcategories\u001b[00m\n", - " │   │   │   ├── unique.C1.parquet\n", - " │   │   │   ├── unique.C10.parquet\n", - " │   │   │   ├── unique.C11.parquet\n", - " │   │   │   ├── unique.C12.parquet\n", - " │   │   │   ├── unique.C13.parquet\n", - " │   │   │   ├── unique.C14.parquet\n", - " │   │   │   ├── unique.C15.parquet\n", - " │   │   │   ├── unique.C16.parquet\n", - " │   │   │   ├── unique.C17.parquet\n", - " │   │   │   ├── unique.C18.parquet\n", - " │   │   │   ├── unique.C19.parquet\n", - " │   │   │   ├── unique.C2.parquet\n", - " │   │   │   ├── unique.C20.parquet\n", - " │   │   │   ├── unique.C21.parquet\n", - " │   │   │   ├── unique.C22.parquet\n", - " │   │   │   ├── unique.C23.parquet\n", - " │   │   │   ├── unique.C24.parquet\n", - " │   │   │   ├── unique.C25.parquet\n", - " │   │   │   ├── unique.C26.parquet\n", - " │   │   │   ├── unique.C3.parquet\n", - " │   │   │   ├── unique.C4.parquet\n", - " │   │   │   ├── unique.C5.parquet\n", - " │   │   │   ├── unique.C6.parquet\n", - " │   │   │   ├── unique.C7.parquet\n", - " │   │   │   ├── unique.C8.parquet\n", - " │   │   │   └── unique.C9.parquet\n", - " │   │   ├── metadata.json\n", - " │   │   └── workflow.pkl\n", - " │   └── config.pbtxt\n", - " └── \u001b[01;34mcriteo_tf\u001b[00m\n", - " ├── \u001b[01;34m1\u001b[00m\n", - " │   └── \u001b[01;34mmodel.savedmodel\u001b[00m\n", - " │   ├── \u001b[01;34massets\u001b[00m\n", - " │   ├── keras_metadata.pb\n", - " │   ├── saved_model.pb\n", - " │   └── \u001b[01;34mvariables\u001b[00m\n", - " │   ├── variables.data-00000-of-00001\n", - " │   └── variables.index\n", - " └── config.pbtxt\n", - "\n", - "12 directories, 36 files\n" - ] - } - ], - "source": [ - "!tree /tmp/model" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Loading Ensemble Model with Triton Inference Server\n", - "\n", - "We have only saved the models for Triton Inference Server. We started Triton Inference Server in explicit mode, meaning that we need to send a request that Triton will load the ensemble model." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "First, we restart this notebook to free the GPU memory." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# import IPython\n", - "\n", - "# app = IPython.Application.instance()\n", - "# app.kernel.do_shutdown(True)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We define the BASE_DIR again." - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "\n", - "BASE_DIR = os.environ.get(\"BASE_DIR\", \"/raid/data/criteo\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We connect to the Triton Inference Server." - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "client created.\n" - ] - } - ], - "source": [ - "import tritonclient.grpc as grpc_client\n", - "\n", - "try:\n", - " triton_client = grpc_client.InferenceServerClient(url=\"localhost:8001\", verbose=True)\n", - " print(\"client created.\")\n", - "except Exception as e:\n", - " print(\"channel creation failed: \" + str(e))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We deactivate warnings." - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "import warnings\n", - "\n", - "warnings.filterwarnings(\"ignore\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We check if the server is alive." - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "is_server_live, metadata ()\n", - "\n", - "live: true\n", - "\n" - ] - }, - { - "data": { - "text/plain": [ - "True" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "triton_client.is_server_live()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We check the available models in the repositories:\n", - "- criteo: Ensemble \n", - "- criteo_nvt: NVTabular \n", - "- criteo_tf: TensorFlow model" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "get_model_repository_index, metadata ()\n", - "\n", - "models {\n", - " name: \"criteo\"\n", - "}\n", - "models {\n", - " name: \"criteo_nvt\"\n", - "}\n", - "models {\n", - " name: \"criteo_tf\"\n", - "}\n", - "\n" - ] - }, - { - "data": { - "text/plain": [ - "models {\n", - " name: \"criteo\"\n", - "}\n", - "models {\n", - " name: \"criteo_nvt\"\n", - "}\n", - "models {\n", - " name: \"criteo_tf\"\n", - "}" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "triton_client.get_model_repository_index()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We load the ensembled model." - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "load_model, metadata ()\n", - "override files omitted:\n", - "model_name: \"criteo\"\n", - "\n", - "Loaded model 'criteo'\n", - "CPU times: user 13.5 ms, sys: 8.86 ms, total: 22.4 ms\n", - "Wall time: 41.9 s\n" - ] - } - ], - "source": [ - "%%time\n", - "\n", - "triton_client.load_model(model_name=\"criteo\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Example Request to Triton Inference Server\n", - "\n", - "Now, the models are loaded and we can create a sample request. We read an example **raw batch** for inference." - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - " I1 I2 I3 I4 I5 I6 I7 I8 I9 I10 ... C17 \\\n", - "0 5 110 16 1 0 14 7 1 ... -771205462 \n", - "1 32 3 5 1 0 0 61 5 0 ... -771205462 \n", - "2 233 1 146 1 0 0 99 7 0 ... -771205462 \n", - "\n", - " C18 C19 C20 C21 C22 C23 \\\n", - "0 -1206449222 -1793932789 -1014091992 351689309 632402057 -675152885 \n", - "1 -1578429167 -1793932789 -20981661 -1556988767 -924717482 391309800 \n", - "2 1653545869 -1793932789 -1014091992 351689309 632402057 -675152885 \n", - "\n", - " C24 C25 C26 \n", - "0 2091868316 809724924 -317696227 \n", - "1 1966410890 -1726799382 -1218975401 \n", - "2 883538181 -10139646 -317696227 \n", - "\n", - "[3 rows x 39 columns]\n" - ] - } - ], - "source": [ - "# Get dataframe library - cudf or pandas\n", - "from merlin.core.dispatch import get_lib\n", - "\n", - "df_lib = get_lib()\n", - "\n", - "# read in the workflow (to get input/output schema to call triton with)\n", - "batch_path = os.path.join(BASE_DIR, \"converted/criteo\")\n", - "# raise(ValueError(f\"{batch_path}\"))\n", - "batch = df_lib.read_parquet(os.path.join(batch_path, \"*.parquet\"), num_rows=3)\n", - "batch = batch[[x for x in batch.columns if x != \"label\"]]\n", - "print(batch)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We prepare the batch for inference by using correct column names and data types. We use the same datatypes as defined in our dataframe." - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "I1 int32\n", - "I2 int32\n", - "I3 int32\n", - "I4 int32\n", - "I5 int32\n", - "I6 int32\n", - "I7 int32\n", - "I8 int32\n", - "I9 int32\n", - "I10 int32\n", - "I11 int32\n", - "I12 int32\n", - "I13 int32\n", - "C1 int32\n", - "C2 int32\n", - "C3 int32\n", - "C4 int32\n", - "C5 int32\n", - "C6 int32\n", - "C7 int32\n", - "C8 int32\n", - "C9 int32\n", - "C10 int32\n", - "C11 int32\n", - "C12 int32\n", - "C13 int32\n", - "C14 int32\n", - "C15 int32\n", - "C16 int32\n", - "C17 int32\n", - "C18 int32\n", - "C19 int32\n", - "C20 int32\n", - "C21 int32\n", - "C22 int32\n", - "C23 int32\n", - "C24 int32\n", - "C25 int32\n", - "C26 int32\n", - "dtype: object" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "batch.dtypes" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [], - "source": [ - "import tritonclient.grpc as httpclient\n", - "from tritonclient.utils import np_to_triton_dtype\n", - "import numpy as np\n", - "\n", - "inputs = []\n", - "\n", - "col_names = list(batch.columns)\n", - "col_dtypes = [np.int32] * len(col_names)\n", - "\n", - "for i, col in enumerate(batch.columns):\n", - " d = batch[col].fillna(0).values_host.astype(col_dtypes[i])\n", - " d = d.reshape(len(d), 1)\n", - " inputs.append(httpclient.InferInput(col_names[i], d.shape, np_to_triton_dtype(col_dtypes[i])))\n", - " inputs[i].set_data_from_numpy(d)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We send the request to the triton server and collect the last output." - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "infer, metadata ()\n", - "model_name: \"criteo\"\n", - "id: \"1\"\n", - "inputs {\n", - " name: \"I1\"\n", - " datatype: \"INT32\"\n", - " shape: 3\n", - " shape: 1\n", - "}\n", - "inputs {\n", - " name: \"I2\"\n", - " datatype: \"INT32\"\n", - " shape: 3\n", - " shape: 1\n", - "}\n", - "inputs {\n", - " name: \"I3\"\n", - " datatype: \"INT32\"\n", - " shape: 3\n", - " shape: 1\n", - "}\n", - "inputs {\n", - " name: \"I4\"\n", - " datatype: \"INT32\"\n", - " shape: 3\n", - " shape: 1\n", - "}\n", - "inputs {\n", - " name: \"I5\"\n", - " datatype: \"INT32\"\n", - " shape: 3\n", - " shape: 1\n", - "}\n", - "inputs {\n", - " name: \"I6\"\n", - " datatype: \"INT32\"\n", - " shape: 3\n", - " shape: 1\n", - "}\n", - "inputs {\n", - " name: \"I7\"\n", - " datatype: \"INT32\"\n", - " shape: 3\n", - " shape: 1\n", - "}\n", - "inputs {\n", - " name: \"I8\"\n", - " datatype: \"INT32\"\n", - " shape: 3\n", - " shape: 1\n", - "}\n", - "inputs {\n", - " name: \"I9\"\n", - " datatype: \"INT32\"\n", - " shape: 3\n", - " shape: 1\n", - "}\n", - "inputs {\n", - " name: \"I10\"\n", - " datatype: \"INT32\"\n", - " shape: 3\n", - " shape: 1\n", - "}\n", - "inputs {\n", - " name: \"I11\"\n", - " datatype: \"INT32\"\n", - " shape: 3\n", - " shape: 1\n", - "}\n", - "inputs {\n", - " name: \"I12\"\n", - " datatype: \"INT32\"\n", - " shape: 3\n", - " shape: 1\n", - "}\n", - "inputs {\n", - " name: \"I13\"\n", - " datatype: \"INT32\"\n", - " shape: 3\n", - " shape: 1\n", - "}\n", - "inputs {\n", - " name: \"C1\"\n", - " datatype: \"INT32\"\n", - " shape: 3\n", - " shape: 1\n", - "}\n", - "inputs {\n", - " name: \"C2\"\n", - " datatype: \"INT32\"\n", - " shape: 3\n", - " shape: 1\n", - "}\n", - "inputs {\n", - " name: \"C3\"\n", - " datatype: \"INT32\"\n", - " shape: 3\n", - " shape: 1\n", - "}\n", - "inputs {\n", - " name: \"C4\"\n", - " datatype: \"INT32\"\n", - " shape: 3\n", - " shape: 1\n", - "}\n", - "inputs {\n", - " name: \"C5\"\n", - " datatype: \"INT32\"\n", - " shape: 3\n", - " shape: 1\n", - "}\n", - "inputs {\n", - " name: \"C6\"\n", - " datatype: \"INT32\"\n", - " shape: 3\n", - " shape: 1\n", - "}\n", - "inputs {\n", - " name: \"C7\"\n", - " datatype: \"INT32\"\n", - " shape: 3\n", - " shape: 1\n", - "}\n", - "inputs {\n", - " name: \"C8\"\n", - " datatype: \"INT32\"\n", - " shape: 3\n", - " shape: 1\n", - "}\n", - "inputs {\n", - " name: \"C9\"\n", - " datatype: \"INT32\"\n", - " shape: 3\n", - " shape: 1\n", - "}\n", - "inputs {\n", - " name: \"C10\"\n", - " datatype: \"INT32\"\n", - " shape: 3\n", - " shape: 1\n", - "}\n", - "inputs {\n", - " name: \"C11\"\n", - " datatype: \"INT32\"\n", - " shape: 3\n", - " shape: 1\n", - "}\n", - "inputs {\n", - " name: \"C12\"\n", - " datatype: \"INT32\"\n", - " shape: 3\n", - " shape: 1\n", - "}\n", - "inputs {\n", - " name: \"C13\"\n", - " datatype: \"INT32\"\n", - " shape: 3\n", - " shape: 1\n", - "}\n", - "inputs {\n", - " name: \"C14\"\n", - " datatype: \"INT32\"\n", - " shape: 3\n", - " shape: 1\n", - "}\n", - "inputs {\n", - " name: \"C15\"\n", - " datatype: \"INT32\"\n", - " shape: 3\n", - " shape: 1\n", - "}\n", - "inputs {\n", - " name: \"C16\"\n", - " datatype: \"INT32\"\n", - " shape: 3\n", - " shape: 1\n", - "}\n", - "inputs {\n", - " name: \"C17\"\n", - " datatype: \"INT32\"\n", - " shape: 3\n", - " shape: 1\n", - "}\n", - "inputs {\n", - " name: \"C18\"\n", - " datatype: \"INT32\"\n", - " shape: 3\n", - " shape: 1\n", - "}\n", - "inputs {\n", - " name: \"C19\"\n", - " datatype: \"INT32\"\n", - " shape: 3\n", - " shape: 1\n", - "}\n", - "inputs {\n", - " name: \"C20\"\n", - " datatype: \"INT32\"\n", - " shape: 3\n", - " shape: 1\n", - "}\n", - "inputs {\n", - " name: \"C21\"\n", - " datatype: \"INT32\"\n", - " shape: 3\n", - " shape: 1\n", - "}\n", - "inputs {\n", - " name: \"C22\"\n", - " datatype: \"INT32\"\n", - " shape: 3\n", - " shape: 1\n", - "}\n", - "inputs {\n", - " name: \"C23\"\n", - " datatype: \"INT32\"\n", - " shape: 3\n", - " shape: 1\n", - "}\n", - "inputs {\n", - " name: \"C24\"\n", - " datatype: \"INT32\"\n", - " shape: 3\n", - " shape: 1\n", - "}\n", - "inputs {\n", - " name: \"C25\"\n", - " datatype: \"INT32\"\n", - " shape: 3\n", - " shape: 1\n", - "}\n", - "inputs {\n", - " name: \"C26\"\n", - " datatype: \"INT32\"\n", - " shape: 3\n", - " shape: 1\n", - "}\n", - "outputs {\n", - " name: \"output\"\n", - "}\n", - "raw_input_contents: \"\\005\\000\\000\\000 \\000\\000\\000\\000\\000\\000\\000\"\n", - "raw_input_contents: \"n\\000\\000\\000\\003\\000\\000\\000\\351\\000\\000\\000\"\n", - "raw_input_contents: \"\\000\\000\\000\\000\\005\\000\\000\\000\\001\\000\\000\\000\"\n", - "raw_input_contents: \"\\020\\000\\000\\000\\000\\000\\000\\000\\222\\000\\000\\000\"\n", - "raw_input_contents: \"\\000\\000\\000\\000\\001\\000\\000\\000\\001\\000\\000\\000\"\n", - "raw_input_contents: \"\\001\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\"\n", - "raw_input_contents: \"\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\"\n", - "raw_input_contents: \"\\016\\000\\000\\000=\\000\\000\\000c\\000\\000\\000\"\n", - "raw_input_contents: \"\\007\\000\\000\\000\\005\\000\\000\\000\\007\\000\\000\\000\"\n", - "raw_input_contents: \"\\001\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\"\n", - "raw_input_contents: \"\\000\\000\\000\\000\\001\\000\\000\\000\\001\\000\\000\\000\"\n", - "raw_input_contents: \"2\\001\\000\\000U\\014\\000\\000\\035\\014\\000\\000\"\n", - "raw_input_contents: \"\\000\\000\\000\\000\\005\\000\\000\\000\\001\\000\\000\\000\"\n", - "raw_input_contents: \"y\\rwb\\215\\375\\363\\345y\\rwb\"\n", - "raw_input_contents: \"X]\\037\\342\\246\\377\\252\\240\\003B\\230\\255\"\n", - "raw_input_contents: \"/D\\352\\257\\325\\025\\252o\\r\\306\\276b\"\n", - "raw_input_contents: \"\\317\\177\\\\\\224!4\\212\\332\\356Il8\"\n", - "raw_input_contents: \"H\\'\\2608#\\237\\326\\334L\\372>\\334L\\372>\\334L\"\n", - "raw_input_contents: \"\\252V\\010\\322\\252V\\010\\322\\252V\\010\\322\"\n", - "raw_input_contents: \"\\272\\013\\027\\270\\021\\025\\353\\241\\215\\033\\217b\"\n", - "raw_input_contents: \"\\013\\302\\022\\225\\013\\302\\022\\225\\013\\302\\022\\225\"\n", - "raw_input_contents: \"(/\\216\\303c\\330\\277\\376(/\\216\\303\"\n", - "raw_input_contents: \"]Z\\366\\024\\241<2\\243]Z\\366\\024\"\n", - "raw_input_contents: \"\\211\\260\\261%V\\356\\341\\310\\211\\260\\261%\"\n", - "raw_input_contents: \"\\013\\374\\301\\327\\350\\351R\\027\\013\\374\\301\\327\"\n", - "raw_input_contents: \"\\234`\\257|\\212\\0145u\\005\\271\\2514\"\n", - "raw_input_contents: \"\\374kC0\\352!\\023\\231\\002He\\377\"\n", - "raw_input_contents: \"\\035W\\020\\355W\\351W\\267\\035W\\020\\355\"\n", - "\n", - "model_name: \"criteo\"\n", - "model_version: \"1\"\n", - "id: \"1\"\n", - "parameters {\n", - " key: \"sequence_end\"\n", - " value {\n", - " bool_param: false\n", - " }\n", - "}\n", - "parameters {\n", - " key: \"sequence_id\"\n", - " value {\n", - " int64_param: 0\n", - " }\n", - "}\n", - "parameters {\n", - " key: \"sequence_start\"\n", - " value {\n", - " bool_param: false\n", - " }\n", - "}\n", - "outputs {\n", - " name: \"output\"\n", - " datatype: \"FP32\"\n", - " shape: 3\n", - " shape: 1\n", - "}\n", - "raw_output_contents: \"Dd\\217<$r\\233<\\241\\231u<\"\n", - "\n", - "predicted softmax result:\n", - " [[0.01750387]\n", - " [0.01897532]\n", - " [0.01499024]]\n" - ] - } - ], - "source": [ - "# placeholder variables for the output\n", - "outputs = [httpclient.InferRequestedOutput(\"output\")]\n", - "\n", - "# build a client to connect to our server.\n", - "# This InferenceServerClient object is what we'll be using to talk to Triton.\n", - "# make the request with tritonclient.http.InferInput object\n", - "response = triton_client.infer(\"criteo\", inputs, request_id=\"1\", outputs=outputs)\n", - "\n", - "print(\"predicted softmax result:\\n\", response.as_numpy(\"output\"))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Let's unload the model. We need to unload each model." - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "unload_model, metadata ()\n", - "model_name: \"criteo\"\n", - "parameters {\n", - " key: \"unload_dependents\"\n", - " value {\n", - " bool_param: false\n", - " }\n", - "}\n", - "\n", - "Unloaded model 'criteo'\n", - "unload_model, metadata ()\n", - "model_name: \"criteo_nvt\"\n", - "parameters {\n", - " key: \"unload_dependents\"\n", - " value {\n", - " bool_param: false\n", - " }\n", - "}\n", - "\n", - "Unloaded model 'criteo_nvt'\n", - "unload_model, metadata ()\n", - "model_name: \"criteo_tf\"\n", - "parameters {\n", - " key: \"unload_dependents\"\n", - " value {\n", - " bool_param: false\n", - " }\n", - "}\n", - "\n", - "Unloaded model 'criteo_tf'\n" - ] - } - ], - "source": [ - "triton_client.unload_model(model_name=\"criteo\")\n", - "triton_client.unload_model(model_name=\"criteo_nvt\")\n", - "triton_client.unload_model(model_name=\"criteo_tf\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3.9.7 64-bit", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.7" - }, - "vscode": { - "interpreter": { - "hash": "916dbcbb3f70747c44a77c7bcd40155683ae19c65e1c03b4aa3499c5328201f1" - } - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/examples/scaling-criteo/README.md b/examples/scaling-criteo/README.md deleted file mode 100644 index ee2eb7f8fcd..00000000000 --- a/examples/scaling-criteo/README.md +++ /dev/null @@ -1,15 +0,0 @@ -# Scaling to Large Datasets with Criteo - -Criteo provides the largest publicly available dataset for recommender systems. -The dataset is 1 TB uncompressed click logs of 4 billion examples. -The example notebooks show how to scale NVTabular in the following ways: - -* Using multiple GPUs and multiple nodes with NVTabular for ETL. -* Training recommender system model with NVTabular dataloader for PyTorch. - -Refer to the following notebooks: - -* [Download and Convert](01-Download-Convert.ipynb) -* [ETL with NVTabular](02-ETL-with-NVTabular.ipynb) -* Training a model: [HugeCTR](03-Training-with-HugeCTR.ipynb) | [TensorFlow](03-Training-with-TF.ipynb) -* Use Triton Inference Server to serve a model: [HugeCTR](04-Triton-Inference-with-HugeCTR.ipynb) | [TensorFlow](04-Triton-Inference-with-TF.ipynb) diff --git a/examples/scaling-criteo/docker-compose-fastai.yml b/examples/scaling-criteo/docker-compose-fastai.yml deleted file mode 100644 index f115d6914f2..00000000000 --- a/examples/scaling-criteo/docker-compose-fastai.yml +++ /dev/null @@ -1,20 +0,0 @@ -# (1) Use this file for docker runtime configurations that are common to both -# development and deployment. - -# `version : '2.3'` lets us use the `runtime=nvidia` configuration so that our -# containers can interact with the GPU(s). -version: '2.3' - -volumes: - models: - -services: - lab: - runtime: nvidia - image: nvcr.io/nvidia/merlin/merlin-pytorch:22.06 - command: "/bin/bash -c 'pip install jupyterlab jupytext pydot && apt-get update && apt-get install -y tree && python -m ipykernel install --user --name=merlin && jupyter notebook --no-browser --allow-root --port=8888 --ip=0.0.0.0 --NotebookApp.token='demotoken' --NotebookApp.allow_origin='*' --notebook-dir=/'" - volumes: - - models:/models - - /raid/:/raid/ - ports: - - 8888:8888 diff --git a/examples/scaling-criteo/docker-compose-hugectr.yml b/examples/scaling-criteo/docker-compose-hugectr.yml deleted file mode 100644 index cd7977be592..00000000000 --- a/examples/scaling-criteo/docker-compose-hugectr.yml +++ /dev/null @@ -1,37 +0,0 @@ -# (1) Use this file for docker runtime configurations that are common to both -# development and deployment. - -# `version : '2.3'` lets us use the `runtime=nvidia` configuration so that our -# containers can interact with the GPU(s). -version: '2.3' - -volumes: - model: - -services: - triton: - command: "/bin/bash -c 'tritonserver --model-repository=/model/ --backend-config=hugectr,ps=/model/ps.json --model-control-mode=explicit'" - image: nvcr.io/nvidia/merlin/merlin-hugectr:22.06 - runtime: nvidia - shm_size: "1g" - ulimits: - memlock: -1 - stack: 67108864 - ports: - - 8000:8000 - - 8001:8001 - - 8002:8002 - volumes: - - model:/model - - lab: - runtime: nvidia - image: nvcr.io/nvidia/merlin/merlin-hugectr:22.06 - command: "/bin/bash -c 'pip install jupyterlab jupytext pydot nvidia-pyindex tritonclient geventhttpclient && apt-get update && apt-get install -y tree && jupyter notebook --no-browser --allow-root --port=8888 --ip=0.0.0.0 --NotebookApp.token='demotoken' --NotebookApp.allow_origin='*' --notebook-dir=/'" - volumes: - - model:/model - - /raid/:/raid/ - links: - - triton - ports: - - 8888:8888 diff --git a/examples/scaling-criteo/docker-compose-tf.yml b/examples/scaling-criteo/docker-compose-tf.yml deleted file mode 100644 index 19ec8ee0705..00000000000 --- a/examples/scaling-criteo/docker-compose-tf.yml +++ /dev/null @@ -1,37 +0,0 @@ -# (1) Use this file for docker runtime configurations that are common to both -# development and deployment. - -# `version : '2.3'` lets us use the `runtime=nvidia` configuration so that our -# containers can interact with the GPU(s). -version: '2.3' - -volumes: - models: - -services: - triton: - command: "/bin/bash -c 'pip install grpcio-channelz && tritonserver --model-repository=/models/ --model-control-mode=explicit'" - image: nvcr.io/nvidia/merlin/merlin-tensorflow:22.06 - runtime: nvidia - shm_size: "1g" - ulimits: - memlock: -1 - stack: 67108864 - ports: - - 8000:8000 - - 8001:8001 - - 8002:8002 - volumes: - - models:/models - - lab: - runtime: nvidia - image: nvcr.io/nvidia/merlin/merlin-tensorflow:22.06 - command: "/bin/bash -c 'pip install jupyterlab jupytext pydot nvidia-pyindex tritonclient geventhttpclient && apt-get update && apt-get install -y tree && python -m ipykernel install --user --name=merlin && jupyter notebook --no-browser --allow-root --port=8888 --ip=0.0.0.0 --NotebookApp.token='demotoken' --NotebookApp.allow_origin='*' --notebook-dir=/'" - volumes: - - models:/models - - /raid/:/raid/ - links: - - triton - ports: - - 8888:8888 diff --git a/examples/scaling-criteo/imgs/dask-dataframe.svg b/examples/scaling-criteo/imgs/dask-dataframe.svg deleted file mode 100644 index 7d371234328..00000000000 --- a/examples/scaling-criteo/imgs/dask-dataframe.svg +++ /dev/null @@ -1,225 +0,0 @@ - - - - - - - - - - - - image/svg+xml - - - - - - - - - - - - January, 2016 - Febrary, 2016 - March, 2016 - April, 2016 - May, 2016 - Pandas DataFrame - } - Dask DataFrame - } - - diff --git a/examples/scaling-criteo/imgs/triton-hugectr.png b/examples/scaling-criteo/imgs/triton-hugectr.png deleted file mode 100644 index 8c372d5d62d..00000000000 Binary files a/examples/scaling-criteo/imgs/triton-hugectr.png and /dev/null differ diff --git a/examples/scaling-criteo/imgs/triton-tf.png b/examples/scaling-criteo/imgs/triton-tf.png deleted file mode 100644 index e68bcd4fefc..00000000000 Binary files a/examples/scaling-criteo/imgs/triton-tf.png and /dev/null differ diff --git a/examples/tensorflow/README.md b/examples/tensorflow/README.md deleted file mode 100644 index 47d02b57ab1..00000000000 --- a/examples/tensorflow/README.md +++ /dev/null @@ -1,17 +0,0 @@ -# Accelerating TensorFlow Tabular Workflows with NVTabular -## TODO: Include data section? -Get Criteo, run preproc notebook, mount volume, make sure you have space for tfrecords, etc. - -## Build container -From root directory -``` -docker build -t $USER/nvtabular-tf-example -f examples/tensorflow/docker/Dockerfile . -``` -## Run container -``` -docker run --rm -it \ - -v /path/to/data:/data -v /path/to/write/tfrecords:/tfrecords \ - -p 8888:8888 -p 6006:6006 \ - --gpus 1 $USER/nvtabular-tf-example -``` -And navigate to `:8888/?token=nvidia` diff --git a/examples/tensorflow/TFRecords-To-Parquet.ipynb b/examples/tensorflow/TFRecords-To-Parquet.ipynb deleted file mode 100644 index 96866d44dc5..00000000000 --- a/examples/tensorflow/TFRecords-To-Parquet.ipynb +++ /dev/null @@ -1,1033 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "id": "1d4a2a17", - "metadata": {}, - "outputs": [], - "source": [ - "# Copyright 2021 NVIDIA Corporation. All Rights Reserved.\n", - "#\n", - "# Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# http://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License.\n", - "# ==============================================================================" - ] - }, - { - "cell_type": "markdown", - "id": "7da4cfc5", - "metadata": {}, - "source": [ - "\n", - "\n", - "# TensorFlow: Convert TFRecords to Parquet files\n", - "\n", - "## TFRecords\n", - "\n", - "[TFRecords](https://www.tensorflow.org/tutorials/load_data/tfrecord) are a popular file format to store data for deep learning training with TensorFlow. It is a \"simple format for storing a sequence of binary records\". In many cases the dataset is too large for the host memory and the dataset is converted into (multiple) tfrecords file to disk. TensorFlow's ecosystem enables to stream the tfrecords from disk to train the model without requiring to load the full dataset.

\n", - "That sounds great, but there are some disadvantages when working with tabular dataset. TFRecords stores the dataset as key, values. In other domains, such as computer vision, this representation is efficient as the key is `image` and the values are a the pixels. For an RGB image with 200x200 resolution, there are 120000 (200x200x3) values. In a tabular dataset, a feature is often a single number and therefore, there is a significant overhead for using a key in each example. **In some of our experiments, we experienced that tfrecords can be ~4-5x larger than `parquet` files for the same dataset.**\n", - "

\n", - "[Parquet](https://en.wikipedia.org/wiki/Apache_Parquet) is another file format to store data. It is a free and open-source data storage format in the Hadoop ecosystem. Many popular systems, such as Spark or Pandas, support to read and write parquet files. \n", - "

\n", - "We developed [NVTabular Data Loaders](https://nvidia-merlin.github.io/NVTabular/main/training/index.html) as a customized data loader, fully operating on the GPU. It reads the data from disk into the GPU memory and prepares the next batch on the GPU. Therefore, we do not have any CPU-GPU communication. Our data loader leverages parquet files to reduce the disk pressure. **In our experiments, we experienced that the native data loader is the bottleneck in training tabular deep learning models and by changing the native data loader to NVTabular Data Loader, we saw a 8-9x speed-up.**\n", - "\n", - "### Convert TFRecords to Parquet files\n", - "That is a lot of background information. In many cases, we saw that users have their dataset stored as tfrecords files. In this notebook, we provide a tfrecords to parquet examples. Users can transform their dataset to parquet and be able to experiment with NVTabular data loader." - ] - }, - { - "cell_type": "markdown", - "id": "096a7716", - "metadata": {}, - "source": [ - "We leverage the library pandas-tfrecords. We install pandas-tfrecords without dependencies, as it would install a specific TensorFlow version." - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "35e6c8d4", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Looking in indexes: https://pypi.org/simple, https://pypi.ngc.nvidia.com\n", - "Requirement already satisfied: pandas-tfrecords==0.1.5 in /usr/local/lib/python3.8/dist-packages (0.1.5)\n", - "\u001b[33mWARNING: You are using pip version 21.0.1; however, version 21.2.4 is available.\n", - "You should consider upgrading via the '/usr/bin/python -m pip install --upgrade pip' command.\u001b[0m\n", - "Looking in indexes: https://pypi.org/simple, https://pypi.ngc.nvidia.com\n", - "Requirement already satisfied: s3fs in /usr/local/lib/python3.8/dist-packages (2021.8.1)\n", - "Requirement already satisfied: fsspec==2021.08.1 in /root/.local/lib/python3.8/site-packages/fsspec-2021.8.1-py3.8.egg (from s3fs) (2021.8.1)\n", - "Requirement already satisfied: aiobotocore~=1.4.0 in /usr/local/lib/python3.8/dist-packages (from s3fs) (1.4.1)\n", - "Requirement already satisfied: wrapt>=1.10.10 in /usr/local/lib/python3.8/dist-packages (from aiobotocore~=1.4.0->s3fs) (1.12.1)\n", - "Requirement already satisfied: aioitertools>=0.5.1 in /usr/local/lib/python3.8/dist-packages (from aiobotocore~=1.4.0->s3fs) (0.8.0)\n", - "Requirement already satisfied: botocore<1.20.107,>=1.20.106 in /usr/local/lib/python3.8/dist-packages (from aiobotocore~=1.4.0->s3fs) (1.20.106)\n", - "Requirement already satisfied: aiohttp>=3.3.1 in /usr/local/lib/python3.8/dist-packages (from aiobotocore~=1.4.0->s3fs) (3.7.4.post0)\n", - "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.8/dist-packages (from aiohttp>=3.3.1->aiobotocore~=1.4.0->s3fs) (1.6.3)\n", - "Requirement already satisfied: typing-extensions>=3.6.5 in /usr/local/lib/python3.8/dist-packages (from aiohttp>=3.3.1->aiobotocore~=1.4.0->s3fs) (3.7.4.3)\n", - "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.8/dist-packages (from aiohttp>=3.3.1->aiobotocore~=1.4.0->s3fs) (21.2.0)\n", - "Requirement already satisfied: chardet<5.0,>=2.0 in /usr/lib/python3/dist-packages (from aiohttp>=3.3.1->aiobotocore~=1.4.0->s3fs) (3.0.4)\n", - "Requirement already satisfied: async-timeout<4.0,>=3.0 in /usr/local/lib/python3.8/dist-packages (from aiohttp>=3.3.1->aiobotocore~=1.4.0->s3fs) (3.0.1)\n", - "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.8/dist-packages (from aiohttp>=3.3.1->aiobotocore~=1.4.0->s3fs) (5.1.0)\n", - "Requirement already satisfied: urllib3<1.27,>=1.25.4 in /usr/lib/python3/dist-packages (from botocore<1.20.107,>=1.20.106->aiobotocore~=1.4.0->s3fs) (1.25.8)\n", - "Requirement already satisfied: python-dateutil<3.0.0,>=2.1 in /usr/local/lib/python3.8/dist-packages (from botocore<1.20.107,>=1.20.106->aiobotocore~=1.4.0->s3fs) (2.8.2)\n", - "Requirement already satisfied: jmespath<1.0.0,>=0.7.1 in /usr/local/lib/python3.8/dist-packages (from botocore<1.20.107,>=1.20.106->aiobotocore~=1.4.0->s3fs) (0.10.0)\n", - "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.8/dist-packages (from python-dateutil<3.0.0,>=2.1->botocore<1.20.107,>=1.20.106->aiobotocore~=1.4.0->s3fs) (1.15.0)\n", - "Requirement already satisfied: idna>=2.0 in /usr/lib/python3/dist-packages (from yarl<2.0,>=1.0->aiohttp>=3.3.1->aiobotocore~=1.4.0->s3fs) (2.8)\n", - "\u001b[33mWARNING: You are using pip version 21.0.1; however, version 21.2.4 is available.\n", - "You should consider upgrading via the '/usr/bin/python -m pip install --upgrade pip' command.\u001b[0m\n" - ] - } - ], - "source": [ - "!pip install --no-deps pandas-tfrecords==0.1.5\n", - "!pip install s3fs" - ] - }, - { - "cell_type": "markdown", - "id": "9a8f4dcd", - "metadata": {}, - "source": [ - "## Create a Synthetic Dataset" - ] - }, - { - "cell_type": "markdown", - "id": "243a5cbd", - "metadata": {}, - "source": [ - "First, we will create a synthetic dataset. Afterwards, we will convert the synthetic data to a tfrecord file. The synthetic dataset contains `continuous features`, `categorical features`, `continuous features in a list with variable length`, `categorical features in a list with variable length` and the `label`.

\n", - "The features of a list have variable length, which are often used in session-based recommender systems. For example, the last page views in a session and sessions have different lengths." - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "id": "58949777", - "metadata": {}, - "outputs": [], - "source": [ - "import numpy as np\n", - "import pandas as pd\n", - "\n", - "import cudf" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "id": "deeafde3", - "metadata": {}, - "outputs": [], - "source": [ - "def create_synthetic_df(\n", - " N_CONT_FEATURES, N_CAT_FEATURES, N_CONT_LIST_FEATURES, N_CAT_LIST_FEATURES, N_ROWS\n", - "):\n", - " dict_features = {}\n", - " for icont in range(N_CONT_FEATURES):\n", - " dict_features[\"cont\" + str(icont)] = np.random.uniform(-1, 1, size=N_ROWS)\n", - " for icat in range(N_CAT_FEATURES):\n", - " dict_features[\"cat\" + str(icat)] = np.random.choice(list(range(10)), size=N_ROWS)\n", - " for icontlist in range(N_CONT_LIST_FEATURES):\n", - " feature_list = []\n", - " for irow in range(N_ROWS):\n", - " n_elements = np.random.choice(list(range(20)))\n", - " feature_list.append(np.random.uniform(-1, 1, size=n_elements).tolist())\n", - " dict_features[\"cont_list\" + str(icontlist)] = feature_list\n", - " for icatlist in range(N_CAT_LIST_FEATURES):\n", - " feature_list = []\n", - " for irow in range(N_ROWS):\n", - " n_elements = np.random.choice(list(range(20)))\n", - " feature_list.append(np.random.choice(list(range(10)), size=n_elements).tolist())\n", - " dict_features[\"cat_list\" + str(icatlist)] = feature_list\n", - " dict_features[\"label\"] = np.random.choice(list(range(2)), size=N_ROWS)\n", - " df = pd.DataFrame(dict_features)\n", - " return df" - ] - }, - { - "cell_type": "markdown", - "id": "fda49c3f", - "metadata": {}, - "source": [ - "We can configure the size of the dataset and numbers of features of the different type. As this is just a example, we use only 20,000 rows." - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "id": "0b141d03", - "metadata": {}, - "outputs": [], - "source": [ - "N_ROWS = 20000\n", - "N_CONT_FEATURES = 5\n", - "N_CAT_FEATURES = 7\n", - "N_CONT_LIST_FEATURES = 2\n", - "N_CAT_LIST_FEATURES = 3" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "id": "6616a87b", - "metadata": {}, - "outputs": [], - "source": [ - "df = create_synthetic_df(\n", - " N_CONT_FEATURES, N_CAT_FEATURES, N_CONT_LIST_FEATURES, N_CAT_LIST_FEATURES, N_ROWS\n", - ")" - ] - }, - { - "cell_type": "markdown", - "id": "22d66e48", - "metadata": {}, - "source": [ - "We can take a look on the dataset." - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "id": "e023dca6", - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
cont0cont1cont2cont3cont4cat0cat1cat2cat3cat4cat5cat6cont_list0cont_list1cat_list0cat_list1cat_list2label
0-0.346288-0.0927840.8788760.990467-0.5050792289024[-0.5329311666886798, -0.7973632802691455, -0....[-0.7527243533757371][7, 5, 1, 9, 5, 6, 5, 7, 1, 6, 0, 7, 8, 1][2, 0, 0, 0, 6, 4, 2, 3][8, 3, 5, 7, 0, 5, 2, 1, 2, 7, 7]1
1-0.336003-0.6659820.9020710.531961-0.0051431269307[0.9805303513847896, -0.1364336119532299, 0.39...[][4, 5, 0, 7, 6, 7][9, 0, 6, 9, 2, 2][1, 2, 0, 6, 2, 4, 9, 4, 3, 3, 7, 4, 1, 5, 7, 9]0
2-0.089536-0.922915-0.636890-0.494594-0.1230657900244[0.9677775916375682, 0.4868478686143529, 0.010...[0.9863213170102452, 0.801522837843786, 0.8203...[4, 5, 3, 5, 2, 5, 3, 4, 1, 8, 0, 4, 5, 3, 0, ...[6, 2][8, 7, 4, 6, 5, 4, 7, 9, 0, 7, 6]0
3-0.2604000.693127-0.8757540.4562870.7629043533173[-0.2644213019104138, -0.09665251017206655, -0...[-0.8362007638643811, 0.1541830950440195, 0.79...[8, 0, 1, 0, 9, 5, 9, 7, 9, 6, 7][0, 8, 9, 5, 9, 7, 8][7, 0, 7, 2, 0, 0, 8, 3, 5]0
40.980959-0.9823290.628736-0.311694-0.8809406608422[-0.34002032148205985, -0.28546136806218714, -...[0.057850173597639776, 0.8166183641925591, -0....[4, 8, 9, 9, 7, 9, 2][4, 3, 5, 9, 0, 3, 8, 5, 4, 0, 3, 1, 4, 8, 0, ...[7, 4, 4, 2, 5, 0, 3, 9, 5, 8, 3, 9, 3, 1, 7, ...0
\n", - "
" - ], - "text/plain": [ - " cont0 cont1 cont2 cont3 cont4 cat0 cat1 cat2 cat3 \\\n", - "0 -0.346288 -0.092784 0.878876 0.990467 -0.505079 2 2 8 9 \n", - "1 -0.336003 -0.665982 0.902071 0.531961 -0.005143 1 2 6 9 \n", - "2 -0.089536 -0.922915 -0.636890 -0.494594 -0.123065 7 9 0 0 \n", - "3 -0.260400 0.693127 -0.875754 0.456287 0.762904 3 5 3 3 \n", - "4 0.980959 -0.982329 0.628736 -0.311694 -0.880940 6 6 0 8 \n", - "\n", - " cat4 cat5 cat6 cont_list0 \\\n", - "0 0 2 4 [-0.5329311666886798, -0.7973632802691455, -0.... \n", - "1 3 0 7 [0.9805303513847896, -0.1364336119532299, 0.39... \n", - "2 2 4 4 [0.9677775916375682, 0.4868478686143529, 0.010... \n", - "3 1 7 3 [-0.2644213019104138, -0.09665251017206655, -0... \n", - "4 4 2 2 [-0.34002032148205985, -0.28546136806218714, -... \n", - "\n", - " cont_list1 \\\n", - "0 [-0.7527243533757371] \n", - "1 [] \n", - "2 [0.9863213170102452, 0.801522837843786, 0.8203... \n", - "3 [-0.8362007638643811, 0.1541830950440195, 0.79... \n", - "4 [0.057850173597639776, 0.8166183641925591, -0.... \n", - "\n", - " cat_list0 \\\n", - "0 [7, 5, 1, 9, 5, 6, 5, 7, 1, 6, 0, 7, 8, 1] \n", - "1 [4, 5, 0, 7, 6, 7] \n", - "2 [4, 5, 3, 5, 2, 5, 3, 4, 1, 8, 0, 4, 5, 3, 0, ... \n", - "3 [8, 0, 1, 0, 9, 5, 9, 7, 9, 6, 7] \n", - "4 [4, 8, 9, 9, 7, 9, 2] \n", - "\n", - " cat_list1 \\\n", - "0 [2, 0, 0, 0, 6, 4, 2, 3] \n", - "1 [9, 0, 6, 9, 2, 2] \n", - "2 [6, 2] \n", - "3 [0, 8, 9, 5, 9, 7, 8] \n", - "4 [4, 3, 5, 9, 0, 3, 8, 5, 4, 0, 3, 1, 4, 8, 0, ... \n", - "\n", - " cat_list2 label \n", - "0 [8, 3, 5, 7, 0, 5, 2, 1, 2, 7, 7] 1 \n", - "1 [1, 2, 0, 6, 2, 4, 9, 4, 3, 3, 7, 4, 1, 5, 7, 9] 0 \n", - "2 [8, 7, 4, 6, 5, 4, 7, 9, 0, 7, 6] 0 \n", - "3 [7, 0, 7, 2, 0, 0, 8, 3, 5] 0 \n", - "4 [7, 4, 4, 2, 5, 0, 3, 9, 5, 8, 3, 9, 3, 1, 7, ... 0 " - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "df.head()" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "id": "6a49b022", - "metadata": {}, - "outputs": [], - "source": [ - "CONTINUOUS_COLUMNS = [\"cont\" + str(i) for i in range(N_CONT_FEATURES)]\n", - "CATEGORICAL_COLUMNS = [\"cat\" + str(i) for i in range(N_CAT_FEATURES)]\n", - "CONTINUOUS_LIST_COLUMNS = [\"cont_list\" + str(i) for i in range(N_CONT_LIST_FEATURES)]\n", - "CATEGORICAL_LIST_COLUMNS = [\"cat_list\" + str(i) for i in range(N_CAT_LIST_FEATURES)]\n", - "LABEL_COLUMNS = [\"label\"]" - ] - }, - { - "cell_type": "markdown", - "id": "bb33cb9b", - "metadata": {}, - "source": [ - "## Convert the Synthetic Dataset into TFRecords" - ] - }, - { - "cell_type": "markdown", - "id": "5a8b05b0", - "metadata": {}, - "source": [ - "After we created the synthetic dataset, we store it to tfrecords." - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "id": "055a8dae", - "metadata": {}, - "outputs": [], - "source": [ - "import tensorflow as tf" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "id": "f8f502ff", - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "import multiprocessing as mp\n", - "from itertools import repeat\n", - "\n", - "\n", - "def transform_tfrecords(\n", - " df,\n", - " PATH,\n", - " CONTINUOUS_COLUMNS,\n", - " CATEGORICAL_COLUMNS,\n", - " CONTINUOUS_LIST_COLUMNS,\n", - " CATEGORICAL_LIST_COLUMNS,\n", - " LABEL_COLUMNS,\n", - "):\n", - " write_dir = os.path.dirname(PATH)\n", - " if not os.path.exists(write_dir):\n", - " os.makedirs(write_dir)\n", - " file_idx, example_idx = 0, 0\n", - " writer = get_writer(write_dir, file_idx)\n", - " column_names = [\n", - " CONTINUOUS_COLUMNS,\n", - " CATEGORICAL_COLUMNS + LABEL_COLUMNS,\n", - " CONTINUOUS_LIST_COLUMNS,\n", - " CATEGORICAL_LIST_COLUMNS,\n", - " ]\n", - " with mp.Pool(8, pool_initializer, column_names) as pool:\n", - " data = []\n", - " for col_names in column_names:\n", - " if len(col_names) == 0:\n", - " data.append(repeat(None))\n", - " else:\n", - " data.append(df[col_names].values)\n", - " data = zip(*data)\n", - " record_map = pool.imap(build_and_serialize_example, data, chunksize=200)\n", - " for record in record_map:\n", - " writer.write(record)\n", - " example_idx += 1\n", - " writer.close()\n", - "\n", - "\n", - "def pool_initializer(num_cols, cat_cols, num_list_cols, cat_list_cols):\n", - " global numeric_columns\n", - " global categorical_columns\n", - " global numeric_list_columns\n", - " global categorical_list_columns\n", - " numeric_columns = num_cols\n", - " categorical_columns = cat_cols\n", - " numeric_list_columns = num_list_cols\n", - " categorical_list_columns = cat_list_cols\n", - "\n", - "\n", - "def build_and_serialize_example(data):\n", - " numeric_values, categorical_values, numeric_list_values, categorical_list_values = data\n", - " feature = {}\n", - " if numeric_values is not None:\n", - " feature.update(\n", - " {\n", - " col: tf.train.Feature(float_list=tf.train.FloatList(value=[val]))\n", - " for col, val in zip(numeric_columns, numeric_values)\n", - " }\n", - " )\n", - " if categorical_values is not None:\n", - " feature.update(\n", - " {\n", - " col: tf.train.Feature(int64_list=tf.train.Int64List(value=[val]))\n", - " for col, val in zip(categorical_columns, categorical_values)\n", - " }\n", - " )\n", - " if numeric_list_values is not None:\n", - " feature.update(\n", - " {\n", - " col: tf.train.Feature(float_list=tf.train.FloatList(value=val))\n", - " for col, val in zip(numeric_list_columns, numeric_list_values)\n", - " }\n", - " )\n", - " if categorical_list_values is not None:\n", - " feature.update(\n", - " {\n", - " col: tf.train.Feature(int64_list=tf.train.Int64List(value=val))\n", - " for col, val in zip(categorical_list_columns, categorical_list_values)\n", - " }\n", - " )\n", - " return tf.train.Example(features=tf.train.Features(feature=feature)).SerializeToString()\n", - "\n", - "\n", - "def get_writer(write_dir, file_idx):\n", - " filename = str(file_idx).zfill(5) + \".tfrecords\"\n", - " return tf.io.TFRecordWriter(os.path.join(write_dir, filename))" - ] - }, - { - "cell_type": "markdown", - "id": "f0430ce5", - "metadata": {}, - "source": [ - "We define the output path." - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "id": "0ca623b3", - "metadata": {}, - "outputs": [], - "source": [ - "PATH = \"/raid/tfrecord-test/\"" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "id": "2619480a", - "metadata": {}, - "outputs": [], - "source": [ - "!rm -rf $PATH\n", - "!mkdir $PATH" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "id": "b88f1b42", - "metadata": {}, - "outputs": [], - "source": [ - "transform_tfrecords(\n", - " df,\n", - " PATH,\n", - " CONTINUOUS_COLUMNS,\n", - " CATEGORICAL_COLUMNS,\n", - " CONTINUOUS_LIST_COLUMNS,\n", - " CATEGORICAL_LIST_COLUMNS,\n", - " LABEL_COLUMNS,\n", - ")" - ] - }, - { - "cell_type": "markdown", - "id": "25ad1044", - "metadata": {}, - "source": [ - "We can check the file." - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "id": "31362c7e", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "00000.tfrecords\r\n" - ] - } - ], - "source": [ - "!ls $PATH" - ] - }, - { - "cell_type": "markdown", - "id": "69fc385f", - "metadata": {}, - "source": [ - "## Convert TFRecords to parquet files" - ] - }, - { - "cell_type": "markdown", - "id": "3aafe8a0", - "metadata": {}, - "source": [ - "Now, we have a dataset in the tfrecords format. Let's use the `convert_tfrecords_to_parquet` function to convert a tfrecord file into parquet." - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "id": "62fa679c", - "metadata": {}, - "outputs": [], - "source": [ - "import glob\n", - "\n", - "from nvtabular.framework_utils.tensorflow.tfrecords_to_parquet import convert_tfrecords_to_parquet" - ] - }, - { - "cell_type": "markdown", - "id": "1e59596b", - "metadata": {}, - "source": [ - "Let's select all TFRecords in the folder." - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "id": "fd930951", - "metadata": {}, - "outputs": [], - "source": [ - "filenames = glob.glob(PATH + \"/*.tfrecords\")" - ] - }, - { - "cell_type": "markdown", - "id": "3eab6554", - "metadata": {}, - "source": [ - "Let's call the `convert_tfrecords_to_parquet`.

\n", - "Some details about the parameters:\n", - "* `compression_type` is the compression type of the tfrecords. Options: `\"\"` (no compression), `\"ZLIB\"`, or `\"GZIP\"`\n", - "* `chunks` defines how many data points per `parquet` file should be saved. It splits a tfrecords into multiple parquet files.\n", - "* `convert_lists` defines, if feature lists should be converted into multiple feature columns. Even single dataframe series are 1 dimensional arrays when converted back from tfrecords to parquet. " - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "id": "d249b965", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "['/raid/tfrecord-test/00000.tfrecords']" - ] - }, - "execution_count": 18, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "filenames" - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "id": "854f2aa3", - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2021-09-22 21:56:53.202269: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA\n", - "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", - "2021-09-22 21:56:54.586055: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1510] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 30681 MB memory: -> device: 0, name: Tesla V100-SXM2-32GB, pci bus id: 0000:0b:00.0, compute capability: 7.0\n", - "2021-09-22 21:56:55.158643: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:185] None of the MLIR Optimization Passes are enabled (registered 2)\n", - "20000it [00:12, 1665.20it/s]\n" - ] - } - ], - "source": [ - "convert_tfrecords_to_parquet(\n", - " filenames=filenames, output_dir=PATH, compression_type=\"\", chunks=1000, convert_lists=True\n", - ")" - ] - }, - { - "cell_type": "markdown", - "id": "c6a3881c", - "metadata": {}, - "source": [ - "## Let's take a look" - ] - }, - { - "cell_type": "markdown", - "id": "897c4ea3", - "metadata": {}, - "source": [ - "We can see that `convert_tfrecords_to_parquet` created multiple files per `tfrecord` depending on the chunk size." - ] - }, - { - "cell_type": "code", - "execution_count": 20, - "id": "dab31264", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "['/raid/tfrecord-test/00000.parquet']" - ] - }, - "execution_count": 20, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "filenames = glob.glob(PATH + \"/*.parquet\")\n", - "filenames" - ] - }, - { - "cell_type": "markdown", - "id": "453e26eb", - "metadata": {}, - "source": [ - "If we load the first file, we can see, that it has the same structure as our original synthetic dataset." - ] - }, - { - "cell_type": "code", - "execution_count": 23, - "id": "0bd30a89", - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
cat0cat1cat2cat3cat4cat5cat6cat_list0cat_list1cat_list2cont0cont1cont2cont3cont4cont_list0cont_list1label
02289024[7, 5, 1, 9, 5, 6, 5, 7, 1, 6, 0, 7, 8, 1][2, 0, 0, 0, 6, 4, 2, 3][8, 3, 5, 7, 0, 5, 2, 1, 2, 7, 7]-0.346288-0.0927840.8788760.990467-0.505079[-0.53293115, -0.7973633, -0.047344275, -0.132...[-0.75272435]1
11269307[4, 5, 0, 7, 6, 7][9, 0, 6, 9, 2, 2][1, 2, 0, 6, 2, 4, 9, 4, 3, 3, 7, 4, 1, 5, 7, 9]-0.336003-0.6659820.9020710.531961-0.005143[0.9805303, -0.13643362, 0.39948544, 0.7434469...[]0
27900244[4, 5, 3, 5, 2, 5, 3, 4, 1, 8, 0, 4, 5, 3, 0, ...[6, 2][8, 7, 4, 6, 5, 4, 7, 9, 0, 7, 6]-0.089536-0.922915-0.636890-0.494594-0.123065[0.9677776, 0.48684788, 0.010608715][0.98632133, 0.80152285, 0.820345, 0.015393688...0
33533173[8, 0, 1, 0, 9, 5, 9, 7, 9, 6, 7][0, 8, 9, 5, 9, 7, 8][7, 0, 7, 2, 0, 0, 8, 3, 5]-0.2604000.693127-0.8757540.4562870.762904[-0.2644213, -0.09665251, -0.92680424, 0.30409...[-0.8362008, 0.15418309, 0.799706, 0.4666645, ...0
46608422[4, 8, 9, 9, 7, 9, 2][4, 3, 5, 9, 0, 3, 8, 5, 4, 0, 3, 1, 4, 8, 0, ...[7, 4, 4, 2, 5, 0, 3, 9, 5, 8, 3, 9, 3, 1, 7, ...0.980959-0.9823290.628736-0.311694-0.880940[-0.34002033, -0.28546137, -0.2595898, -0.5337...[0.057850175, 0.8166184, -0.3719872, -0.703909...0
\n", - "
" - ], - "text/plain": [ - " cat0 cat1 cat2 cat3 cat4 cat5 cat6 \\\n", - "0 2 2 8 9 0 2 4 \n", - "1 1 2 6 9 3 0 7 \n", - "2 7 9 0 0 2 4 4 \n", - "3 3 5 3 3 1 7 3 \n", - "4 6 6 0 8 4 2 2 \n", - "\n", - " cat_list0 \\\n", - "0 [7, 5, 1, 9, 5, 6, 5, 7, 1, 6, 0, 7, 8, 1] \n", - "1 [4, 5, 0, 7, 6, 7] \n", - "2 [4, 5, 3, 5, 2, 5, 3, 4, 1, 8, 0, 4, 5, 3, 0, ... \n", - "3 [8, 0, 1, 0, 9, 5, 9, 7, 9, 6, 7] \n", - "4 [4, 8, 9, 9, 7, 9, 2] \n", - "\n", - " cat_list1 \\\n", - "0 [2, 0, 0, 0, 6, 4, 2, 3] \n", - "1 [9, 0, 6, 9, 2, 2] \n", - "2 [6, 2] \n", - "3 [0, 8, 9, 5, 9, 7, 8] \n", - "4 [4, 3, 5, 9, 0, 3, 8, 5, 4, 0, 3, 1, 4, 8, 0, ... \n", - "\n", - " cat_list2 cont0 cont1 \\\n", - "0 [8, 3, 5, 7, 0, 5, 2, 1, 2, 7, 7] -0.346288 -0.092784 \n", - "1 [1, 2, 0, 6, 2, 4, 9, 4, 3, 3, 7, 4, 1, 5, 7, 9] -0.336003 -0.665982 \n", - "2 [8, 7, 4, 6, 5, 4, 7, 9, 0, 7, 6] -0.089536 -0.922915 \n", - "3 [7, 0, 7, 2, 0, 0, 8, 3, 5] -0.260400 0.693127 \n", - "4 [7, 4, 4, 2, 5, 0, 3, 9, 5, 8, 3, 9, 3, 1, 7, ... 0.980959 -0.982329 \n", - "\n", - " cont2 cont3 cont4 \\\n", - "0 0.878876 0.990467 -0.505079 \n", - "1 0.902071 0.531961 -0.005143 \n", - "2 -0.636890 -0.494594 -0.123065 \n", - "3 -0.875754 0.456287 0.762904 \n", - "4 0.628736 -0.311694 -0.880940 \n", - "\n", - " cont_list0 \\\n", - "0 [-0.53293115, -0.7973633, -0.047344275, -0.132... \n", - "1 [0.9805303, -0.13643362, 0.39948544, 0.7434469... \n", - "2 [0.9677776, 0.48684788, 0.010608715] \n", - "3 [-0.2644213, -0.09665251, -0.92680424, 0.30409... \n", - "4 [-0.34002033, -0.28546137, -0.2595898, -0.5337... \n", - "\n", - " cont_list1 label \n", - "0 [-0.75272435] 1 \n", - "1 [] 0 \n", - "2 [0.98632133, 0.80152285, 0.820345, 0.015393688... 0 \n", - "3 [-0.8362008, 0.15418309, 0.799706, 0.4666645, ... 0 \n", - "4 [0.057850175, 0.8166184, -0.3719872, -0.703909... 0 " - ] - }, - "execution_count": 23, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "df = cudf.read_parquet(filenames[0])\n", - "df.head()" - ] - }, - { - "cell_type": "code", - "execution_count": 24, - "id": "b2ce99e0", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "(20000, 18)" - ] - }, - "execution_count": 24, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "df.shape" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.10" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/examples/tensorflow/accelerating-tensorflow.ipynb b/examples/tensorflow/accelerating-tensorflow.ipynb deleted file mode 100644 index c56e3991698..00000000000 --- a/examples/tensorflow/accelerating-tensorflow.ipynb +++ /dev/null @@ -1,1682 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "# Copyright 2021 NVIDIA Corporation. All Rights Reserved.\n", - "#\n", - "# Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# http://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License.\n", - "# ==============================================================================" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Training Tabular Deep Learning Models with Keras on GPU\n", - "Deep learning has revolutionized the fields of computer vision (CV) and natural language processing (NLP) in the last few years, providing a fast and general framework for solving a host of difficult problems with unprecedented accuracy. Part and parcel of this revolution has been the development of APIs like [Keras](https://www.tensorflow.org/api_docs/python/tf/keras) for NVIDIA GPUs, allowing practitioners to quickly iterate on new and interesting ideas and receive feedback on their efficacy in shorter and shorter intervals.\n", - "\n", - "One class of problem which has remained largely immune to this revolution, however, is the class involving tabular data. Part of this difficulty is that, unlike CV or NLP, where different datasets are underlied by similar phenomena and therefore can be solved with similar mechanisms, \"tabular datasets\" span a vast array of phenomena, semantic meanings, and problem statements, from product and video recommendation to particle discovery and loan default prediction. This diversity makes universally useful components difficult to find or even define, and is only exacerbated by the notorious lack of standard, industrial-scale benchmark datasets in the tabular space. As a result, deep learning models are frequently bested by their machine learning analogues on these important tasks, particularly on smaller scale datasets.\n", - "\n", - "Yet this diversity is also what makes tools like Keras all the more valuable. Architecture components can be quickly swapped in and out for different tasks like the implementation details they are, and new components can be built and tested with ease. Importantly, domain experts can interact with models at a high level and build *a priori* knowledge into model architectures, without having to spend their time becoming Python programming wizrds.\n", - "\n", - "However, most out-of-the-box APIs suffer from a lack of acceleration that reduces the rate at which new components can be tested and makes production deployment of deep learning systems cost-prohibitive. In this example, we will walk through some recent advancements made by NVIDIA's [NVTabular](https://github.com/nvidia/nvtabular) data loading library that can alleviate existing bottlenecks and bring to bear the full power of GPU acceleration.\n", - "\n", - "#### What to Keep an Eye Out For\n", - "The point of this walkthrough will be to show how common components of existing TensorFlow tabular-learning pipelines can be drop-in replaced by NVTabular components for cheap-as-free acceleration with minimal overhead. To do this, we'll start by examining a pipeline for fitting the [DLRM](https://arxiv.org/abs/1906.00091) architecture on the [Criteo Terabyte Dataset](https://ailab.criteo.com/download-criteo-1tb-click-logs-dataset/) using Keras/TensorFlow's native tools on both on CPU and GPU, and discuss why the acceleration we observe on GPU is not particularly impressive. Then we'll examine what an identical pipeline would look like using NVTabular and why it overcomes those bottlenecks.\n", - "\n", - "Since the Criteo Terabyte Dataset is large, and you and I both have better things to do than sit around for hours waiting to train a model we have no intention of ever using, I'll restrict the training to 1000 steps in order to illustrate the similarities in convergence and the expected acceleration. Of course, there may well exist alternative choices of architectures and hyperparameters that will lead to better or faster convergence, but I trust that you, clever data scientist that you are, are more than capable of finding these yourself should you wish. I intend only to demonstrate how NVTabular can help you achieve that convergence more quickly, in the hopes that you will find it easy to apply the same methods to the dataset that really matters: your own.\n", - "\n", - "I will assume at least some familiarity with the relevant tabular deep learning methods (in particular what I mean by \"tabular data\" and how it is distinct from, say, image data; continuous vs. categorical variables; learned categorical embeddings; and online vs. offline preprocessing) and a passing familiarity with TensorFlow and Keras. If you are green or rusty on any of this points, it won't make this discussion illegible, but I'll put links in the relevant places just in case.\n", - "\n", - "The structure will be building, step-by-step, the necessary functions that a dataset-agnostic pipeline might need in order to train a model in Keras. In each function, we'll include an `accelerated` kwarg that will be used to show the difference between what such a function might look like in native TensorFlow vs. using NVTabular. Let's start here by doing our imports and defining some hyperparameters for training (which won't change from one implementation to the next)." - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "from itertools import filterfalse\n", - "import re\n", - "\n", - "import tensorflow as tf\n", - "from tensorflow.keras.mixed_precision import experimental as mixed_precision\n", - "\n", - "# this is a good habit to get in now: TensorFlow's default behavior\n", - "# is to claim all of the GPU memory that it can for itself. This\n", - "# is a problem when it needs to run alongside another GPU library\n", - "# like NVTabular. To get around this, NVTabular will configure\n", - "# TensorFlow to use this fraction of available GPU memory up front.\n", - "# Make sure, however, that you do this before you do anything\n", - "# with TensorFlow: as soon as it's initialized, that memory is gone\n", - "# for good\n", - "os.environ[\"TF_MEMORY_ALLOCATION\"] = \"0.5\"\n", - "import nvtabular as nvt\n", - "from nvtabular.loader.tensorflow import KerasSequenceLoader\n", - "from nvtabular.framework_utils.tensorflow import layers, make_feature_column_workflow\n", - "\n", - "# import custom callback for monitoring throughput\n", - "from callbacks import ThroughputLogger" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "DATA_DIR = os.environ.get(\"DATA_DIR\", \"/data\")\n", - "TFRECORD_DIR = os.environ.get(\"TFRECORD_DIR\", \"/tfrecords\")\n", - "LOG_DIR = os.environ.get(\"LOG_DIR\", \"logs/\")\n", - "\n", - "TFRECORDS = os.path.join(TFRECORD_DIR, \"train\", \"*.tfrecords\")\n", - "PARQUETS = os.path.join(DATA_DIR, \"train\", \"*.parquet\")\n", - "\n", - "# TODO: reimplement the preproc from criteo-example here\n", - "# Alternatively, make criteo its own folder, and split preproc\n", - "# and training into separate notebooks, then execute the\n", - "# preproc notebook from here?\n", - "NUMERIC_FEATURE_NAMES = [f\"I{i}\" for i in range(1, 14)]\n", - "CATEGORICAL_FEATURE_NAMES = [f\"C{i}\" for i in range(1, 27)]\n", - "CATEGORY_COUNTS = [\n", - " 7599500,\n", - " 33521,\n", - " 17022,\n", - " 7339,\n", - " 20046,\n", - " 3,\n", - " 7068,\n", - " 1377,\n", - " 63,\n", - " 5345303,\n", - " 561810,\n", - " 242827,\n", - " 11,\n", - " 2209,\n", - " 10616,\n", - " 100,\n", - " 4,\n", - " 968,\n", - " 14,\n", - " 7838519,\n", - " 2580502,\n", - " 6878028,\n", - " 298771,\n", - " 11951,\n", - " 97,\n", - " 35,\n", - "]\n", - "LABEL_NAME = \"label\"\n", - "\n", - "# optimization params\n", - "BATCH_SIZE = 65536\n", - "STEPS = 1000\n", - "LEARNING_RATE = 0.001\n", - "\n", - "# architecture params\n", - "EMBEDDING_DIM = 8\n", - "TOP_MLP_HIDDEN_DIMS = [1024, 512, 256]\n", - "BOTTOM_MLP_HIDDEN_DIMS = [1024, 1024, 512, 256]\n", - "\n", - "# I'll get sloppy with warnings because just like\n", - "# Steven Tyler sometimes you gotta live on the edge\n", - "tf.get_logger().setLevel(\"ERROR\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## What Does Your Data Look Like\n", - "As we discussed before, \"tabular data\" is an umbrella term referring to data collected from a vast array of problems and phenomena. Perhaps Bob's dataset has 192 features, 54 of which are continuous variables recorded as 32 bit floating point numbers, and the remainder of which are categorical variables which he has encoded as strings. Alice, on the other hand, may have a dataset consisting of 3271 features, most of which are continuous, but a handful of which are integer IDs which can take on one of millions of possible values. We can't expect the same model to be able to handle this kind of variety unless we give it some description of what sorts of inputs to expect.\n", - "\n", - "Moreover, the format in which the data gets read from disk will rarely be the one the model finds useful. Bob's string categories will be of no use to a neural network which lives in the world of continuous functions of real numbers; they will need to be converted to integer lookup table indices before being ingested. For certain types of these **transformations**, Bob may want to do this conversion once, up front, before training begins, and then be done with it. However, this may not always be possible. Bob may wish to hyperparameter search over the parameters of such a transformation (if, for instance, he is using a hash function to map to indices and wants to play with the number of buckets to use). Or perhaps he wants to retain the pre-transformed values, but finds the cost of storing an entire second dataset of the transformed values prohibitive. In this case, he'll need to perform the transformations *online*, between when the data is read from disk and when it gets fed to the network.\n", - "\n", - "Finally, in the case of categorical variables, these lookup indices will need to, well, *look up* an embedding vector that finally puts us in the continuous space our network prefers. Therefore, we also need to define how large of an embedding vector we want to use for a given feature.\n", - "\n", - "TensorFlow provides a convenient module to record this information about the names of features to expect, their type (categorical or numeric), their data type, common transformations to perform on them, and the size of embedding table to use in the case of categorical variables: the [`feature_column` module](https://www.tensorflow.org/tutorials/structured_data/feature_columns). (Note: as of [TensorFlow 2.3](https://github.com/tensorflow/tensorflow/releases/tag/v2.3.0-rc0) these are being deprecated and replaced with Keras layers with similar functionality. Most of the arguments made here will still apply, the code will just look a bit different.) These objects provide both stateless representations of feature information, as well as the code that performs the transformations and embeddings at train time.\n", - "\n", - "While `feature_column`s are a handy and robust representation format, their transformation and embedding implementations are poorly suited for GPUs. We'll see how this looks in terms of TensorFlow profile traces later, but the upshot comes down to two basic points:\n", - "- Many of the transformations involve ops that either don't have a GPU kernel, or have one which is unoptimized. The involvement of ops without GPU kernels means that you're spending a lot of your train step moving data around to the device which can run the current op. Many of the ops that *do* have a GPU kernel are small and don't involve much math, which drowns the math-hungry parallel computing model of GPUs in kernel launch overhead.\n", - "- The embeddings use sparse tensor machinery that is unoptimized on GPUs and is unnecessary for one-hot categoricals, the only type we'll focus on here. This is a good time to mention that the techniques we'll cover today *do not generalize to multi-hot categorical data*, which isn't currently supported by NVTabular. However, there is active work to support this being done and we hope to have it seamlessly integrated in the near future.\n", - "\n", - "As we'll see later, one difficulty in addressing the second issue is that the same Keras layer which performs the embeddings *also* performs the transformations, so even if you know that all your categoricals are one-hot and want to build an accelerated embedding layer that leverages this information, you would be out of luck on a layer which can just perform whatever transformations you might need. One way to get around this is to move your transformations to NVTabular, which will do them all on the GPU at data-loading time, so that all Keras needs to handle is the embedding using a layer like the `tf.keras.layers.DenseFeatures`, or, even more accelerated, NVTabular's equivalent `layers.DenseFeatures` layer.\n", - "\n", - "The good news is, as of NVTabular 0.2, you don't need to change the feature columns you use to represent your inputs and preprocessing in order to enjoy GPU acceleration. The `make_feature_column_workflow` utility will take care of creating an NVTabular `Workflow` object which will perform all of the requisite preprocessing on the GPU, then pass the preprocessed columns to TensorFlow tensors." - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "def get_feature_columns():\n", - " columns = [tf.feature_column.numeric_column(name, (1,)) for name in NUMERIC_FEATURE_NAMES]\n", - " for feature_name, count in zip(CATEGORICAL_FEATURE_NAMES, CATEGORY_COUNTS):\n", - " categorical_column = tf.feature_column.categorical_column_with_hash_bucket(\n", - " feature_name, int(0.75 * count), dtype=tf.int64\n", - " )\n", - " embedding_column = tf.feature_column.embedding_column(categorical_column, EMBEDDING_DIM)\n", - " columns.append(embedding_column)\n", - " return columns" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## A Data By Any Other Format: TFRecords and Tabular Representation\n", - "By running the Criteo preprocessing example above, we generated a dataset in the parquet data format. Why Parquet? Well, besides the fact that NVTabular can read parquet files exceptionally quickly, parquet is a widely used tabular data format that can be read by libraries like Pandas or CuDF to quickly search, filter, and manipulate data using high level abstractions." - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
I5I4I6I11I2I8I12I13I1I3...C16C2C17C25C3C26C9C13C14label
0-0.898195-1.059381-0.488376-0.9105740.4065060.9915781.0301960.039582-0.3634460.113603...7656111455884123685120
12.5958750.674505-0.4883761.5892690.881184-1.0925830.2118191.1434880.3876890.323043...68324521617465233651420
2-0.1132551.034299-0.4883760.4101450.8989000.9179250.198978-0.2139171.099744-0.156412...5841833457152336211990
3-0.898195-1.059381-0.488376-0.9105740.099380-1.092583-0.4953830.236211-1.3112730.323043...031490616167662400
4-0.898195-1.059381-0.488376-0.9105740.561786-1.092583-0.043296-1.181990-1.311273-1.187559...031490457419636200
..................................................................
999995-0.898195-1.059381-0.488376-0.9105741.146024-1.0925830.327294-1.181990-1.311273-1.187559...010231061135182336100
999996-0.898195-1.059381-0.488376-0.9105740.5749690.733282-0.717885-1.1819900.263033-1.187559...761269916189613915120
999997-0.4029531.149989-0.488376-0.077293-0.0330202.4204491.056442-0.571204-0.837359-0.536978...01524006172901321700
9999980.0922890.988406-0.488376-0.077293-0.267567-0.3334860.4424041.925359-0.2108801.289434...05280618663636500
999999-0.8981950.2723160.407738-0.910574-2.140079-0.333486-0.6846631.3145741.4649031.414765...7624626184736122115120
\n", - "

1000000 rows × 40 columns

\n", - "
" - ], - "text/plain": [ - " I5 I4 I6 I11 I2 I8 I12 \\\n", - "0 -0.898195 -1.059381 -0.488376 -0.910574 0.406506 0.991578 1.030196 \n", - "1 2.595875 0.674505 -0.488376 1.589269 0.881184 -1.092583 0.211819 \n", - "2 -0.113255 1.034299 -0.488376 0.410145 0.898900 0.917925 0.198978 \n", - "3 -0.898195 -1.059381 -0.488376 -0.910574 0.099380 -1.092583 -0.495383 \n", - "4 -0.898195 -1.059381 -0.488376 -0.910574 0.561786 -1.092583 -0.043296 \n", - "... ... ... ... ... ... ... ... \n", - "999995 -0.898195 -1.059381 -0.488376 -0.910574 1.146024 -1.092583 0.327294 \n", - "999996 -0.898195 -1.059381 -0.488376 -0.910574 0.574969 0.733282 -0.717885 \n", - "999997 -0.402953 1.149989 -0.488376 -0.077293 -0.033020 2.420449 1.056442 \n", - "999998 0.092289 0.988406 -0.488376 -0.077293 -0.267567 -0.333486 0.442404 \n", - "999999 -0.898195 0.272316 0.407738 -0.910574 -2.140079 -0.333486 -0.684663 \n", - "\n", - " I13 I1 I3 ... C16 C2 C17 C25 C3 C26 \\\n", - "0 0.039582 -0.363446 0.113603 ... 76 5611 1 45 5884 12 \n", - "1 1.143488 0.387689 0.323043 ... 68 32452 1 61 7465 23 \n", - "2 -0.213917 1.099744 -0.156412 ... 58 4183 3 45 715 23 \n", - "3 0.236211 -1.311273 0.323043 ... 0 3149 0 61 6167 6 \n", - "4 -1.181990 -1.311273 -1.187559 ... 0 3149 0 45 7419 6 \n", - "... ... ... ... ... ... ... ... ... ... ... \n", - "999995 -1.181990 -1.311273 -1.187559 ... 0 10231 0 61 13518 23 \n", - "999996 -1.181990 0.263033 -1.187559 ... 76 12699 1 61 896 13 \n", - "999997 -0.571204 -0.837359 -0.536978 ... 0 15240 0 61 7290 13 \n", - "999998 1.925359 -0.210880 1.289434 ... 0 528 0 61 8663 6 \n", - "999999 1.314574 1.464903 1.414765 ... 76 24626 1 8 4736 12 \n", - "\n", - " C9 C13 C14 label \n", - "0 36 8 512 0 \n", - "1 36 5 142 0 \n", - "2 36 2 1199 0 \n", - "3 62 4 0 0 \n", - "4 36 2 0 0 \n", - "... .. ... ... ... \n", - "999995 36 1 0 0 \n", - "999996 9 1 512 0 \n", - "999997 21 7 0 0 \n", - "999998 36 5 0 0 \n", - "999999 21 1 512 0 \n", - "\n", - "[1000000 rows x 40 columns]" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "import cudf\n", - "import glob\n", - "\n", - "filename = glob.glob(os.path.join(DATA_DIR, \"train\", \"*.parquet\"))[0]\n", - "df = cudf.read_parquet(filename, num_rows=1000000)\n", - "df" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
I5I4I6I11I2I8I12I13I1I3...C16C2C17C25C3C26C9C13C14label
0-0.898195-1.059381-0.488376-0.9105740.4065060.9915781.0301960.039582-0.3634460.113603...7656111455884123685120
3-0.898195-1.059381-0.488376-0.9105740.099380-1.092583-0.4953830.236211-1.3112730.323043...031490616167662400
4-0.898195-1.059381-0.488376-0.9105740.561786-1.092583-0.043296-1.181990-1.311273-1.187559...031490457419636200
9-0.898195-1.059381-0.488376-0.910574-0.1878131.1654542.1951991.871938-1.3112732.065346...0125540813182636600
10-0.898195-0.5429990.407738-0.910574-2.140079-0.574419-0.581672-1.181990-0.837359-1.187559...02499906150791336600
..................................................................
999985-0.898195-1.059381-0.488376-0.9105740.083191-0.5744191.745785-0.213917-0.837359-0.156412...0136130246240621300
999986-0.8981950.0677030.931930-0.9105740.486195-0.765658-1.9644340.9309810.7703051.063082...0745204586651136200
999990-0.898195-1.059381-0.488376-0.910574-0.210349-1.0925831.758998-1.181990-0.837359-1.187559...6022810361168172336616140
9999920.4921241.426266-0.4883760.4101450.322983-1.0925830.473773-0.213917-0.560138-0.156412...0310720613920621500
999993-0.8981951.039733-0.488376-0.9105740.325448-0.2474940.0738570.9309812.1961030.638853...768228145470866265120
\n", - "

499183 rows × 40 columns

\n", - "
" - ], - "text/plain": [ - " I5 I4 I6 I11 I2 I8 I12 \\\n", - "0 -0.898195 -1.059381 -0.488376 -0.910574 0.406506 0.991578 1.030196 \n", - "3 -0.898195 -1.059381 -0.488376 -0.910574 0.099380 -1.092583 -0.495383 \n", - "4 -0.898195 -1.059381 -0.488376 -0.910574 0.561786 -1.092583 -0.043296 \n", - "9 -0.898195 -1.059381 -0.488376 -0.910574 -0.187813 1.165454 2.195199 \n", - "10 -0.898195 -0.542999 0.407738 -0.910574 -2.140079 -0.574419 -0.581672 \n", - "... ... ... ... ... ... ... ... \n", - "999985 -0.898195 -1.059381 -0.488376 -0.910574 0.083191 -0.574419 1.745785 \n", - "999986 -0.898195 0.067703 0.931930 -0.910574 0.486195 -0.765658 -1.964434 \n", - "999990 -0.898195 -1.059381 -0.488376 -0.910574 -0.210349 -1.092583 1.758998 \n", - "999992 0.492124 1.426266 -0.488376 0.410145 0.322983 -1.092583 0.473773 \n", - "999993 -0.898195 1.039733 -0.488376 -0.910574 0.325448 -0.247494 0.073857 \n", - "\n", - " I13 I1 I3 ... C16 C2 C17 C25 C3 C26 \\\n", - "0 0.039582 -0.363446 0.113603 ... 76 5611 1 45 5884 12 \n", - "3 0.236211 -1.311273 0.323043 ... 0 3149 0 61 6167 6 \n", - "4 -1.181990 -1.311273 -1.187559 ... 0 3149 0 45 7419 6 \n", - "9 1.871938 -1.311273 2.065346 ... 0 12554 0 8 13182 6 \n", - "10 -1.181990 -0.837359 -1.187559 ... 0 24999 0 61 5079 13 \n", - "... ... ... ... ... ... ... ... ... ... ... \n", - "999985 -0.213917 -0.837359 -0.156412 ... 0 13613 0 24 6240 6 \n", - "999986 0.930981 0.770305 1.063082 ... 0 7452 0 45 8665 11 \n", - "999990 -1.181990 -0.837359 -1.187559 ... 60 22810 3 61 16817 23 \n", - "999992 -0.213917 -0.560138 -0.156412 ... 0 31072 0 61 3920 6 \n", - "999993 0.930981 2.196103 0.638853 ... 76 8228 1 45 4708 6 \n", - "\n", - " C9 C13 C14 label \n", - "0 36 8 512 0 \n", - "3 62 4 0 0 \n", - "4 36 2 0 0 \n", - "9 36 6 0 0 \n", - "10 36 6 0 0 \n", - "... .. ... ... ... \n", - "999985 21 3 0 0 \n", - "999986 36 2 0 0 \n", - "999990 36 6 1614 0 \n", - "999992 21 5 0 0 \n", - "999993 62 6 512 0 \n", - "\n", - "[499183 rows x 40 columns]" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# do some filtering or whatever\n", - "df[df[\"C18\"] == 228]" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "This is great news for data scientists: formats like parquet are the bread and butter of any sort of data exploration. You almost certainly want to keep at least *one* version of your dataset in a format like this. If your dataset is large enough, and storage gets expensive, it's probably the *only* format you want to keep your dataset in.\n", - "\n", - "Unfortunately, TensorFlow does not have fast native readers for formats like this that can read larger-than-memory datasets in an online fashion. TensorFlow's preferred, and fastest, data format is the [TFRecord](https://www.tensorflow.org/tutorials/load_data/tfrecord), a binary format which associates all field names and their values with every example in your dataset. For tabular data, where small float or int features have a smaller memory footprint than string field names, the memory footprint of such a representation can get really big, really fast.\n", - "\n", - "More importantly, TFRecords require reading and parsing in batches using user-provided data schema descriptions. This makes doing the sorts of manipulations described above difficult, if not near impossible, and requires an enormous amount of work to change the values corresponding to a single field in your dataset. For this reason, you almost never want to use TFRecords as the *only* means of representing your data, which means you have generate and store an entire copy of your dataset every time it needs to update. This can take an enormous amount of time and resources that prolong the time from the conception of a feature to testing it in a model.\n", - "\n", - "The main advantage of TFRecords is the speed with which TensorFlow can read them (and its APIs for doing this online), and their support for multi-hot categorical features. While NVTabular is still working on addressing the latter, we'll show below that reading parquet files in batch using NVTabular is substantially faster than the existing TFRecord readers. In order to do this, we'll need to generate a TFRecord version of the parquet dataset we generated before. I'm going to restrict this to generating just the 1000 steps we'll need to do our training demo, but if you have a few days and a couple terabytes of storage lying around feel free to run the whole thing.\n", - "\n", - "Don't worry too much about the code below: it's a bit dense (and frankly still isn't fully robust to string features) and doesn't have much to do with what follows. I'm sure there are ways to make it cleaner/faster/etc., but If anything, it should make clear how nontrivial the process of building and writing TFRecords is. I'm also going to keep it commented out for now since the disk space required is so high, and the casual user clicking through cells might accidentally exhaust their allotment. If you feel like running the comparisons below to keep me honest, uncomment this cell and run it first.\n", - "\n", - "The last thing I'll note is that the astute and experienced TensorFlow user will at this point object that there exist ways to make reading TFRecords for tabular data faster than what I'm about to present. Among these are pre-batching examples (which, I would point out, more or less enforces a fixed valency for all categorical features) and combining all fixed valency categorical and continuous features into vectorized fields in records which can all be parsed at once. And while it's true that methods like this will accelerate TFRecord reading, they still fail to overtake NVTabular's parquet reader. Perhaps more importantly (at least from my workflow-centric view), they only compound the problems I've outlined so far of the difficulty of doing data analysis with TFRecords, and would almost certainly require the code below to be even more brittle and complicated. And this is actually a point worth emphasizing: with NVTabular data loading, you're getting better performance *and* less programming overhead, the holy grail of GPU-based DL software." - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [], - "source": [ - "# import multiprocessing as mp\n", - "# from glob import glob\n", - "# from itertools import repeat\n", - "# from tqdm.notebook import trange\n", - "\n", - "# def pool_initializer(num_cols, cat_cols):\n", - "# global numeric_columns\n", - "# global categorical_columns\n", - "# numeric_columns = num_cols\n", - "# categorical_columns = cat_cols\n", - "\n", - "# def build_and_serialize_example(data):\n", - "# numeric_values, categorical_values = data\n", - "# feature = {}\n", - "# if numeric_values is not None:\n", - "# feature.update({\n", - "# col: tf.train.Feature(float_list=tf.train.FloatList(value=[float(val)]))\n", - "# for col, val in zip(numeric_columns, numeric_values)\n", - "# })\n", - "# if categorical_values is not None:\n", - "# feature.update({\n", - "# col: tf.train.Feature(int64_list=tf.train.Int64List(value=[int(val)]))\n", - "# for col, val in zip(categorical_columns, categorical_values)\n", - "# })\n", - "# return tf.train.Example(features=tf.train.Features(feature=feature)).SerializeToString()\n", - "\n", - "# def get_writer(write_dir, file_idx):\n", - "# filename = str(file_idx).zfill(5) + '.tfrecords'\n", - "# return tf.io.TFRecordWriter(os.path.join(write_dir, filename))\n", - "\n", - "\n", - "# _EXAMPLES_PER_RECORD = 20000000\n", - "# write_dir = os.path.dirname(TFRECORDS)\n", - "# if not os.path.exists(write_dir):\n", - "# os.makedirs(write_dir)\n", - "# file_idx, example_idx = 0, 0\n", - "# writer = get_writer(write_dir, file_idx)\n", - "\n", - "# do_break = False\n", - "# column_names = [NUMERIC_FEATURE_NAMES, CATEGORICAL_FEATURE_NAMES+[LABEL_NAME]]\n", - "# with mp.Pool(8, pool_initializer, column_names) as pool:\n", - "# fnames = glob(PARQUETS)\n", - "# dataset = nvt.Dataset(fnames)\n", - "# pbar = trange(BATCH_SIZE*STEPS)\n", - "\n", - "# for df in dataset.to_iter():\n", - "# data = []\n", - "# for col_names in column_names:\n", - "# if len(col_names) == 0:\n", - "# data.append(repeat(None))\n", - "# else:\n", - "# data.append(df[col_names].to_pandas().values)\n", - "# data = zip(*data)\n", - "\n", - "# record_map = pool.imap(build_and_serialize_example, data, chunksize=200)\n", - "# for record in record_map:\n", - "# writer.write(record)\n", - "# example_idx += 1\n", - "\n", - "# if example_idx == _EXAMPLES_PER_RECORD:\n", - "# writer.close()\n", - "# file_idx += 1\n", - "# writer = get_writer(file_idx)\n", - "# example_idx = 0\n", - "# pbar.update(1)\n", - "# if pbar.n == BATCH_SIZE*STEPS:\n", - "# do_break = True\n", - "# break\n", - "# if do_break:\n", - "# del df\n", - "# break" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Ok, now that we have our data set up the way that we need it, we're ready to get training! TensorFlow provides a handy utility for building an online dataloader that we'll use to parse the tfrecords. Meanwhile, on the NVTablar side, we'll use the `KerasSequenceLoader` for reading chunks of parquet files. We'll also use a the `make_feature_column_workflow` to build an NVTabular `Workflow` that handles hash bucketing online on the GPU. It will also return a simplified set of feature columns that _don't_ include the preprocessing steps.\n", - "\n", - "Take a look below to see the similarities in the API. What's great about using NVTabular `Workflow`s for online preprocessing is that it makes doing arbitrary preprocessing reasonably simple by using `DFlambda` ops, and the `Op` class API allows for extension to more complicated, stat-driven preprocessing as well.\n", - "\n", - "One potentially important difference between these dataset classes is the way in which shuffling is handled. The TensorFlow data loader maintains a buffer of size `shuffle_buffer_size` from which batch elements are randomly selected, with the buffer then sequentially replenished by the next `batch_size` elements in the TFRecord. Large shuffle buffers, while allowing for better epoch-to-epoch randomness and hence generalization, can be hard to maintain given the slow read times. The limitation this enforces on your buffer size isn't as big a deal for datasets which are uniformly shuffled in the TFRecord and only require one or two epochs to converge, but many datasets are ordered by some feature (whether it's time or some categorical groupby), and in this case the windowed shuffle buffer can lead to biased sampling and hence poorer quality gradients.\n", - "\n", - "On the other hand, the `KerasSequenceLoader` manages shuffling by loading in chunks of data from different parts of the full dataset, concatenating them and then shuffling, then iterating through this super-chunk sequentially in batches. The number of \"parts\" of the dataset that get sample, or \"partitions\", is controlled by the `parts_per_chunk` kwarg, while the size of each one of these parts is controlled by the `buffer_size` kwarg, which refers to a fraction of available GPU memory. Using more chunks leads to better randomness, especially at the epoch level where physically disparate samples can be brought into the same batch, but can impact throughput if you use too many. In any case, the speed of the parquet reader makes feasible buffer sizes much larger.\n", - "\n", - "The key thing to keep in mind is due to the asynchronus nature of the data loader, there will be `parts_per_chunk*buffer_size*3` rows of data floating around the GPU at any one time, so your goal should be to balance `parts_per_chunk` and `buffer_size` in such a way to leverage as much GPU memory as possible without going out-of-memory (OOM) and while still meeting your randomness and throughput needs.\n", - "\n", - "Finally, remember that once the data is loaded, it doesn't just pass to TensorFlow untouched: we also apply concatenation, shuffling, and preprocessing operations which will take memory to execute. The takeaway is that just because TensorFlow is only occupying 50% of the GPU memory, don't expect that this implies that we can algebraically balance `parts_per_chunk` and `buffer_size` to exactly occupy the remaining 50%. This might take a bit of tuning for your workload, but once you know the right combination you can use it forever. (Or at least until you get a bigger GPU!)" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [], - "source": [ - "def make_dataset(file_pattern, columns, accelerated=False):\n", - " # make a tfrecord features dataset\n", - " if not accelerated:\n", - " # feature spec tells us how to parse tfrecords\n", - " # using FixedLenFeatures keeps from using sparse machinery,\n", - " # but obviously wouldn't extend to multi-hot categoricals\n", - " feature_spec = {LABEL_NAME: tf.io.FixedLenFeature((1,), tf.int64)}\n", - " for column in columns:\n", - " column = getattr(column, \"categorical_column\", column)\n", - " dtype = getattr(column, \"dtype\", tf.int64)\n", - " feature_spec[column.name] = tf.io.FixedLenFeature((1,), dtype)\n", - "\n", - " dataset = tf.data.experimental.make_batched_features_dataset(\n", - " file_pattern,\n", - " BATCH_SIZE,\n", - " feature_spec,\n", - " label_key=LABEL_NAME,\n", - " num_epochs=1,\n", - " shuffle=True,\n", - " shuffle_buffer_size=4 * BATCH_SIZE,\n", - " )\n", - "\n", - " # make an nvtabular KerasSequenceLoader and add\n", - " # a hash bucketing workflow for online preproc\n", - " else:\n", - " online_workflow, columns = make_feature_column_workflow(columns, LABEL_NAME)\n", - " train_paths = glob.glob(file_pattern)\n", - " dataset = nvt.Dataset(train_paths, engine=\"parquet\")\n", - " online_workflow.fit(dataset)\n", - " ds = KerasSequenceLoader(\n", - " online_workflow.transform(dataset),\n", - " batch_size=BATCH_SIZE,\n", - " label_names=[LABEL_NAME],\n", - " feature_columns=columns,\n", - " shuffle=True,\n", - " buffer_size=0.06,\n", - " parts_per_chunk=1,\n", - " )\n", - " return ds, columns" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Living In The Continuous World\n", - "So at this point, we have a description of our dataset schema contained in our `feature_column`s, and we have a `dataset` object which can load some particular materialization of this schema (our dataset) in an online fashion (with the bytes encoding that materialization organized according to either the TFRecord or Parquet standard).\n", - "\n", - "Once the data is loaded, it needs to get run through a neural network, which will use them to produce predictions of interaction likelihoods, compare its predictions to the labelled answers, and improve its future guesses using this comparison through the magic of backpropogation. Easy as pie.\n", - "\n", - "Unfortunately, the magic of backpropogation relies on a trick of calculus which, by its nature, requires that the functions represented by the neural network are *continuous*. Whether or not you fully understand exactly what that means, you can probably imagine that this is incongrous with the *categorical* features our dataset contains. Less fundamentally, but from an equally practical standpoint, much of the algebra that our network will perform on our tabular features goes much (read: *MUCH*) faster if we do it in parallel as matrix algebra.\n", - "\n", - "For these reasons, we'll want to convert our tabular continuous and categorical features into purely continuous vectors that can be consumed by the network and processed efficiently. For categorical features, this means using the categorical index to lookup a (typically learned) vector from some lower-dimensional space to pass to the network. The exact mechanism by which your network embeds and combines these values will depend on your choice of architecture. But the fundamental operation of looking up and concatenating (or stacking) is ubiquitous across almost all tabular deep learning architectures.\n", - "\n", - "The go-to Keras layer for doing this sort of operation is the `DenseFeatures` layer, which will also perform any transformations defined by your `feature_column`s. The downside of using the `DenseFeatures` layer, as we'll investigate more fully in a bit, is that its GPU performance is handicapped by the use of lots of small ops for doing things that aren't necessarily worth doing on an accelerator like a GPU e.g. checking for in-range values. This drowns the compute itself in kernel launch overhead. Moreover, `DenseFeatures` has no mechanism for identifying one-hot categorical features, instead using `SparseTensor` machinery for all categorical columns for the sake of robustness. Many sparse TensorFlow ops aren't optimized for GPU, particularly for leveraging those Tensor Cores you're paying for by using mixed precision compute, and this further bottlenecks GPU performance.\n", - "\n", - "Because we're now doing all our transformations in NVTabular, and we *know* all of our categorical features are one-hot, we can use a better-optimized embedding layer, NVTabular's `DenseFeatures` layer, that leverages this information. Below, we'll see how we can use such a layer to implement the input ingestion pattern of the DLRM architecture. Note how the numeric and categorical features are handled entirely separately: this is a peculiarity of DLRM, and it's worth noting that our `DenseFeatures` layer makes no assumptions about the combinations of categorical and continuous inputs. As a helpful exercise, I would encourage the reader to think of *other* input ingestion patterns that might capture information that DLRM's does not, and use these same building blocks to mock up an example." - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [], - "source": [ - "class DLRMEmbedding(tf.keras.layers.Layer):\n", - " def __init__(self, columns, accelerated=False, **kwargs):\n", - " is_cat = lambda col: hasattr(col, \"categorical_column\") # noqa\n", - " embedding_columns = list(filter(is_cat, columns))\n", - " numeric_columns = list(filterfalse(is_cat, columns))\n", - "\n", - " self.categorical_feature_names = [col.categorical_column.name for col in embedding_columns]\n", - " self.numeric_feature_names = [col.name for col in numeric_columns]\n", - "\n", - " if not accelerated:\n", - " # need DenseFeatures layer to perform transformations,\n", - " # so we're stuck with the whole thing\n", - " self.categorical_densifier = tf.keras.layers.DenseFeatures(embedding_columns)\n", - " self.categorical_reshape = tf.keras.layers.Reshape((len(embedding_columns), -1))\n", - " self.numeric_densifier = tf.keras.layers.DenseFeatures(numeric_columns)\n", - " else:\n", - " # otherwise we can do a much faster embedding that\n", - " # doesn't break out the SparseTensor machinery\n", - " self.categorical_densifier = layers.DenseFeatures(\n", - " embedding_columns, aggregation=\"stack\"\n", - " )\n", - " self.categorical_reshape = None\n", - " self.numeric_densifier = layers.DenseFeatures(numeric_columns, aggregation=\"concat\")\n", - " super(DLRMEmbedding, self).__init__(**kwargs)\n", - "\n", - " def call(self, inputs):\n", - " if not isinstance(inputs, dict):\n", - " raise TypeError(\"Expected a dict!\")\n", - "\n", - " categorical_inputs = {name: inputs[name] for name in self.categorical_feature_names}\n", - " numeric_inputs = {name: inputs[name] for name in self.numeric_feature_names}\n", - "\n", - " fm_x = self.categorical_densifier(categorical_inputs)\n", - " dense_x = self.numeric_densifier(numeric_inputs)\n", - " if self.categorical_reshape is not None:\n", - " fm_x = self.categorical_reshape(fm_x)\n", - " return fm_x, dense_x\n", - "\n", - " def get_config(self):\n", - " # I'm going to be lazy here. Sue me.\n", - " return {}" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Putting Our Differences Aside\n", - "As a practical matter, that *does it* for the differences between a typical TensorFlow pipeline and an NVTabular accelerated pipeline. Let's review where they've diverged so far:\n", - "- We needed different feature columns because we're no longer using TensorFlow's transformation code for the hash bucketing\n", - "- We needed a different data loader because we're reading parquet files instead of tfrecords (and using NVTabular to hash that data online)\n", - "- We needed a different embedding layer because the existing one is suboptimal and we don't need most of its functionality\n", - "\n", - "Once the data is ready to be consumed by the network, we really *shouldn't* be doing anything different. So from here on out we'll just define the DLRM architecture using Keras, and then define a training function which uses the components we've built so far to string together a functional training run! Note that we'll use a layer implemented by NVTabular, `DotProductInteraction`, which computes the FM component of the DLRM architecture (and can generalize to parameterized variants of the interactions proposed in the [FibiNet](https://arxiv.org/abs/1905.09433) architecture as well)." - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [], - "source": [ - "class ReLUMLP(tf.keras.layers.Layer):\n", - " def __init__(self, dims, output_activation, **kwargs):\n", - " self.layers = []\n", - " for dim in dims[:-1]:\n", - " self.layers.append(tf.keras.layers.Dense(dim, activation=\"relu\"))\n", - " self.layers.append(tf.keras.layers.Dense(dims[-1], activation=output_activation))\n", - " super(ReLUMLP, self).__init__(**kwargs)\n", - "\n", - " def call(self, x):\n", - " for layer in self.layers:\n", - " x = layer(x)\n", - " return x\n", - "\n", - " def get_config(self):\n", - " return {\n", - " \"dims\": [layer.units for layer in self.layers],\n", - " \"output_activation\": self.layers[-1].activation,\n", - " }\n", - "\n", - "\n", - "class DLRM(tf.keras.layers.Layer):\n", - " def __init__(self, embedding_dim, top_mlp_hidden_dims, bottom_mlp_hidden_dims, **kwargs):\n", - " self.top_mlp = ReLUMLP(top_mlp_hidden_dims + [embedding_dim], \"linear\", name=\"top_mlp\")\n", - " self.bottom_mlp = ReLUMLP(bottom_mlp_hidden_dims + [1], \"linear\", name=\"bottom_mlp\")\n", - " self.interaction = layers.DotProductInteraction()\n", - "\n", - " # adding in an activation layer for stability for mixed precision training\n", - " # not strictly necessary, but worth pointing out\n", - " self.activation = tf.keras.layers.Activation(\"sigmoid\", dtype=\"float32\")\n", - " self.double_check = tf.keras.layers.Lambda(\n", - " lambda x: tf.clip_by_value(x, 0.0, 1.0), dtype=\"float32\"\n", - " )\n", - " super(DLRM, self).__init__(**kwargs)\n", - "\n", - " def call(self, inputs):\n", - " dense_x, fm_x = inputs\n", - " dense_x = self.top_mlp(dense_x)\n", - " dense_x_expanded = tf.expand_dims(dense_x, axis=1)\n", - "\n", - " x = tf.concat([fm_x, dense_x_expanded], axis=1)\n", - " x = self.interaction(x)\n", - " x = tf.concat([x, dense_x], axis=1)\n", - " x = self.bottom_mlp(x)\n", - "\n", - " # stuff I'm adding in for mixed precision stability\n", - " # not actually related to DLRM at all\n", - " x = self.activation(x)\n", - " x = self.double_check(x)\n", - " return x\n", - "\n", - " def get_config(self):\n", - " return {\n", - " \"embedding_dim\": self.top_mlp.layers[-1].units,\n", - " \"top_mlp_hidden_dims\": [layer.units for layer in self.top_mlp.layers[:-1]],\n", - " \"bottom_mlp_hidden_dims\": [layer.units for layer in self.bottom_mlp.layers[:-1]],\n", - " }" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "This is an ugly little function I have for giving a more useful reporting of the model parameter count, since the embedding parameters will dominate the total count yet account for very little of the actual learning capacity. Unless you're curious, just execute the cell and keep moving." - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [], - "source": [ - "def print_param_counts(model):\n", - " # I want to go on record as saying I abhor\n", - " # importing inside a function, but I didn't want to\n", - " # make anyone think these imports were strictly\n", - " # *necessary* for a normal training pipeline\n", - " from functools import reduce\n", - "\n", - " num_embedding_params, num_network_params = 0, 0\n", - " for weight in model.trainable_weights:\n", - " weight_param_count = reduce(lambda x, y: x * y, weight.shape)\n", - " if re.search(\"/embedding_weights:[0-9]+$\", weight.name) is not None:\n", - " num_embedding_params += weight_param_count\n", - " else:\n", - " num_network_params += weight_param_count\n", - "\n", - " print(\"Embedding parameter count: {}\".format(num_embedding_params))\n", - " print(\"Non-embedding parameter count: {}\".format(num_network_params))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We'll also include some callbacks to use TensorFlow's incredible TensorBoard tool, both to track training metrics and to profile our GPU performance to diagnose and remove bottlenecks. We'll also use a custom summary metric to monitor throughput in samples per second, to get a sense for the acceleration our improvements bring us. I'm building a function for this just because, like the function above, it's not strictly *necessary*, particularly the throughput hook, so I don't want to muddle the clarity of the actual training function by doing this there." - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [], - "source": [ - "def get_callbacks(device, accelerated=False):\n", - " run_name = device + \"_\" + (\"accelerated\" if accelerated else \"native\")\n", - " if mixed_precision.global_policy().name == \"mixed_float16\":\n", - " run_name += \"_mixed-precision\"\n", - "\n", - " log_dir = os.path.join(LOG_DIR, run_name)\n", - " file_writer = tf.summary.create_file_writer(os.path.join(log_dir, \"metrics\"))\n", - " file_writer.set_as_default()\n", - "\n", - " # note that we're going to be doing some profiling from batches 90-100, and so\n", - " # should expect to see a throughput dip there (since both the profiling itself\n", - " # and the export of the stats it gathers will eat up time). Thus, as a rule,\n", - " # it's not always necessary or desirable to be profiling every training run\n", - " # you do\n", - " return [\n", - " ThroughputLogger(BATCH_SIZE),\n", - " tf.keras.callbacks.TensorBoard(log_dir, update_freq=20, profile_batch=\"90,100\"),\n", - " ]" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "So, finally, below we will define our training pipeline from end to end. Take a look at the comments to see how each component we've built so far plugs in. What's great about such a pipeline is that it's more or less agnostic to what the schema returned by `get_feature_columns` looks like (subject of course to the constraint that there are no multi-hot categorical or vectorized continuous features, which aren't supported yet). In fact, from a certain point of view it would make sense to make the columns and filenames an *input* to this function (and possibly even the architecture itself as well). But I'll leave that level of robustness to you for when you build your own pipeline.\n", - "\n", - "The last thing I'll mention is that we're just going to do training below. The validation picture gets slightly complicated by the fact that `model.fit` doesn't accept Keras `Sequence` objects as validation data. To support this, we've built an extremely lightweight Keras callback to handle validation, `KerasSequenceValidater`. To see how to use it, consult the [Rossmann Store Sales example notebook](../rossmann-store-sales-example.ipynb) in the directory above this, and consider extending its functionality to support more exotic validation metrics." - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [], - "source": [ - "def fit_a_model(accelerated=False, cpu=False):\n", - " # get our columns to describe our dataset\n", - " columns = get_feature_columns()\n", - "\n", - " # build a dataset from those descriptions\n", - " file_pattern = PARQUETS if accelerated else TFRECORDS\n", - " train_dataset, columns = make_dataset(file_pattern, columns, accelerated=accelerated)\n", - "\n", - " # build our Keras model, using column descriptions to build input tensors\n", - " inputs = {}\n", - " for column in columns:\n", - " column = getattr(column, \"categorical_column\", column)\n", - " dtype = getattr(column, \"dtype\", tf.int64)\n", - " input = tf.keras.Input(name=column.name, shape=(1,), dtype=dtype)\n", - " inputs[column.name] = input\n", - "\n", - " fm_x, dense_x = DLRMEmbedding(columns, accelerated=accelerated)(inputs)\n", - " x = DLRM(EMBEDDING_DIM, TOP_MLP_HIDDEN_DIMS, BOTTOM_MLP_HIDDEN_DIMS)([dense_x, fm_x])\n", - " model = tf.keras.Model(inputs=list(inputs.values()), outputs=x)\n", - "\n", - " # compile our Keras model with our desired loss, optimizer, and metrics\n", - " optimizer = tf.keras.optimizers.Adam(LEARNING_RATE)\n", - " metrics = [tf.keras.metrics.AUC(curve=\"ROC\", name=\"auroc\")]\n", - " model.compile(optimizer, \"binary_crossentropy\", metrics=metrics)\n", - " print_param_counts(model)\n", - "\n", - " # name our run and grab our callbacks\n", - " device = \"cpu\" if cpu else \"gpu\"\n", - " callbacks = get_callbacks(device, accelerated=accelerated)\n", - "\n", - " # now fit the model\n", - " model.fit(train_dataset, epochs=1, steps_per_epoch=STEPS, callbacks=callbacks)\n", - "\n", - " # just because I'm doing multiple runs back-to-back, I'm going to\n", - " # clear the Keras session to free up memory now that we're done.\n", - " # You don't need to do this in a typical training script\n", - " tf.keras.backend.clear_session()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "One particularly cool feature of TensorFlow's TensorBoard tool is that we can embed it directly into this notebook. This way, we can monitor training metrics, including throughput, as well as take a look at the in-depth profiles the most recent versions of TensorBoard can generate, without every having to leave the comfort of this browser tab." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "One particularly cool feature of TensorFlow's TensorBoard tool is that we can embed it directly into this notebook. This way, we can monitor training metrics, including throughput, as well as take a look at the in-depth profiles the most recent versions of TensorBoard can generate, without every having to leave the comfort of this browser tab." - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "Reusing TensorBoard on port 6006 (pid 370), started 0:01:41 ago. (Use '!kill 370' to kill it.)" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "\n", - " \n", - " \n", - " " - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "if not os.path.exists(LOG_DIR):\n", - " os.mkdir(LOG_DIR)\n", - "\n", - "%load_ext tensorboard\n", - "%tensorboard --logdir /home/docker/logs --host 0.0.0.0" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We'll start by doing a training run on CPU using all the default TensorFlow tools. Since I'm less concerned about profiling this run, we'll just note the throughput and then move on." - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Embedding parameter count: 188746160\n", - "Non-embedding parameter count: 2747145\n", - "1000/1000 [==============================] - 2483s 2s/step - loss: 0.1317 - auroc: 0.7485\n" - ] - } - ], - "source": [ - "with tf.device(\"/CPU:0\"):\n", - " fit_a_model(accelerated=False, cpu=True)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Next, let's do the exact same run, but this time on GPU. This will give us some indication of the \"out-of-the-box\" acceleration generated by GPU-based training. To spoil the surprise, we'll find that it's not particularly impressive, and we'll start to get an indication of *why* that is." - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Embedding parameter count: 188746160\n", - "Non-embedding parameter count: 2747145\n", - "1000/1000 [==============================] - 406s 406ms/step - loss: 0.1307 - auroc: 0.7474\n" - ] - } - ], - "source": [ - "fit_a_model(accelerated=False)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "If you look at the \"Throughput\" metric in your TensorBoard instance above, you should see something like this\n", - "\n", - "\n", - "This shows a roughly 3-4x improvement in throughput attained simply by moving native TensorFlow code from CPU to GPU. While this is OK, anyone who has ever trained a convolutional model on both CPU and GPU will be disappointed by that figure. Shouldn't parallel computing be able to help a lot more than that?\n", - "\n", - "To understand why this is, switch to the \"Profile\" tab on Tensorboard and take a look at the trace view for your `gpu_native` model\n", - "\n", - "\n", - "This trace view shows us when individual ops take place during the course of a training step, which piece of hardware (CPU or GPU, aka the \"host\" or \"device\") is used to execute them, and how long that execution takes. This is useful because it not only can show us which ops are taking the longest (and so motivate ways to accelerate or remove them), but also when ops aren't running at all! Let's zoom in on this portion of one training step.\n", - "\n", - "\n", - "Here we see compute being done by the GPU for the first ~120 ms of our training step. Notice anything missing?\n", - "\n", - "The issue here is that many of the ops being implemented by `feature_column`s either don't have GPU kernels, requiring data to be passed back and forth between the host and the GPU, or are so small as to not be worth a kernel launch in the first place. Moreover, the `categorical_column_with_hash_bucket`'s in particular implements a costly string mapping for integer categories before hashing.\n", - "\n", - "Taken together, these deficiencies provide a enormous drag on GPU acceleration. By contrast, NVTabular's fast parquet data loaders get your data on the GPU as soon as possible, and use super fast GPU-based preprocessing operations to keep it their waiting to be consumed by your network. By leveraging this fact to write faster, more efficient embedding layers, we can shift the training bottleneck to the math-heavy matrix algebra GPUs are best at.\n", - "\n", - "With this in mind, let's try training with NVTabular's accelerated tools and get a sense for the speed up we can expect." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Next, let's do the exact same run, but this time on GPU. This will give us some indication of the \"out-of-the-box\" acceleration generated by GPU-based training. We'll see that it's not particularly impressive (around 4x or so), and we'll start to get an indication of *why* that is." - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Embedding parameter count: 188746160\n", - "Non-embedding parameter count: 2747145\n", - "1000/1000 [==============================] - 160s 160ms/step - loss: 0.1290 - auroc: 0.7666\n" - ] - } - ], - "source": [ - "fit_a_model(accelerated=True)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Our \"Throughput\" metric should now look like\n", - "\n", - "\n", - "The first thing to note is that this gets us a 2.5-3x boost over native GPU performance, translating to a ~10x improvement over CPU. That's beginning to get closer to the value we should expect GPU training to bring. To get a picture of why this is, let's take a look at the trace view again\n", - "\n", - "\n", - "There's almost no blank space on the GPU portion of the trace, and the ops that *are* on the trace actually occupy a reasonable amount of time, more effectively leveraging GPU resources. You can see this if you watch the output of `nvidia-smi` during training too: GPU utilization is higher and more consistent when using NVTabular for training, which is great, since usually you're paying for the whole GPU whether you're utilizing it all or not. Think of this as just getting more bang for your buck.\n", - "\n", - "The story doesn't end here, either. If you're using a Volta, T4, or Ampere GPU, you have silicon optimized for FP16 compute called Tensor Cores. This lower precision compute is particularly valuable if the majority of your training time is spent on math heavy ops like matrix multiplications. Since we saw that using NVTabular for data loading and preprocessing moves the training bottleneck from data loading to network compute, we should expect to see some pretty good throughput gains from switching to **mixed precision** training. Luckily, Keras has APIs that make changing this compute style extremely simple." - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "metadata": {}, - "outputs": [], - "source": [ - "# update our precision policy to use mixed\n", - "policy = mixed_precision.Policy(\"mixed_float16\")\n", - "mixed_precision.set_policy(policy)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "So now let's compare the advantage wrought by mixed precision training in both the native and accelerated pipelines. One thing I'll note right now is that this architecture has some stability issues in lower precision, and the loss may diverge or nan-out. Increasing numeric stability across model architectures is an ongoing project for NVIDIA, and coverage for most popular tabular architectures and their components should be there soon. So while from a practical standpoint mixed precision compute may not be able to help you *today*, it's still good to know that it's a powerful options to keep an eye on for the near future." - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Embedding parameter count: 188746160\n", - "Non-embedding parameter count: 2747145\n", - "1000/1000 [==============================] - 394s 394ms/step - loss: 0.6790 - auroc: 0.4979\n" - ] - } - ], - "source": [ - "fit_a_model(accelerated=False)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now our \"Throughput\" metric should show\n", - "\n", - "\n", - "As we expected, adding mixed precision compute to the native pipeline doesn't help much, since our training was bottlenecked by things like CPU compute, data transfer, and kernel overhead, none of which reduced-precision GPU compute does anything to address. Let's see what the gains look like when we remove these bottlenecks using NVTabular." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Looking at the \"Throughput\" metric in the \"Scalars\" tab of TensorBoard, we should something like this:\n", - "\n", - "As we expected, adding mixed precision compute to the native pipeline doesn't help much, since our training was bottlenecked by things like CPU compute, data transfer, and kernel overhead, none of which reduced-precision GPU compute does anything to address. Let's see what the gains look like when we remove these bottlenecks using NVTabular." - ] - }, - { - "cell_type": "code", - "execution_count": 20, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Embedding parameter count: 188746160\n", - "Non-embedding parameter count: 2747145\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/opt/conda/envs/rapids/lib/python3.7/site-packages/tensorflow/python/framework/indexed_slices.py:432: UserWarning: Converting sparse IndexedSlices to a dense Tensor of unknown shape. This may consume a large amount of memory.\n", - " \"Converting sparse IndexedSlices to a dense Tensor of unknown shape. \"\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1000/1000 [==============================] - 82s 82ms/step - loss: 0.2073 - auroc: 0.5284\n" - ] - } - ], - "source": [ - "fit_a_model(accelerated=True)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now our \"Throughput\" metric should look like this:\n", - "\n", - "\n", - "By adding in two lines of code to our accelerated pipeline, we can get an over 2x additional improvement in throughput! And again, this should stand to reason, since removing the data loading and preprocessing bottlenecks now makes the most costly parts of our pipeline the matrix multiplies in the dense layers, which are ripe for acceleration via FP16.\n", - "\n", - "Take for example the matmul in the second layer of the bottom MLP. We can take find it on the trace view and click on it for a timing breakdown at full precision:\n", - "\n", - "\n", - "So it takes around 9 ms to run. Let's take a look at the same measurement when using mixed precision:\n", - "\n", - "That's a factor of over 6x improvement! Not bad for an extra line or two of code.\n", - "\n", - "\n", - "As a final tip for interested mixed precision users, the particularly astute observer might have noticed that the matmul in the first layer of the bottom MLP (the `dense_4` layer) didn't enjoy the same level acceleration as the one in this second layer. Why is that?\n", - "\n", - "This is getting a bit beyond the scope of this tutorial, but it's worth noting here that reduced precision kernels require all relevant dimensions to be multiples of 16 in order to be accelerated. The dimension of the input to the bottom MLP, however, can't be controlled directly and is decided by the size of your data. For example, if you have $N$ categorical features and an embedding dimension of $k$, in the DLRM architecture the dimension of this vector will be $\\frac{(N+1)N}{2} + k$. As an exercise, try padding this vector with 0s to the nearest multiple of 16 and see what sort of acceleration FP16 compute provides then." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Conclusions\n", - "Keras represents an incredibly robust and powerful way to rapidly iterate on new ideas for representing relationships between variables in tabular deep learning models, leading to better learning and, hopefully, to a better understanding of the systems we're trying to model. However, inefficiencies in certain modules related to data loading and preprocessing have so far limited the ability of GPUs to provide useful acceleration to these models. By leveraging NVTabular to replace these modules, we can not only achieve stellar acceleration with minimal coding overhead, but also shift our training bottlenecks in order to introduce the possibility of further acceleration farther down the pipeline." - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.8" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/examples/tensorflow/callbacks.py b/examples/tensorflow/callbacks.py deleted file mode 100644 index 62e7e209475..00000000000 --- a/examples/tensorflow/callbacks.py +++ /dev/null @@ -1,38 +0,0 @@ -# -# Copyright (c) 2021, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import time - -import tensorflow as tf - - -class ThroughputLogger(tf.keras.callbacks.Callback): - def __init__(self, batch_size, window_size=10, **kwargs): - self.batch_size = batch_size - self.window_size = window_size - self.times = [] - super(ThroughputLogger, self).__init__(**kwargs) - - def on_epoch_begin(self, epoch, logs=None): - self.times = [time.time()] - - def on_batch_end(self, batch, logs=None): - self.times.append(time.time()) - if len(self.times) > self.window_size: - del self.times[0] - - time_delta = self.times[-1] - self.times[0] - tf.summary.scalar("throughput", self.batch_size * len(self.times) / time_delta, step=batch) diff --git a/examples/tensorflow/docker/Dockerfile b/examples/tensorflow/docker/Dockerfile deleted file mode 100644 index 219a2283398..00000000000 --- a/examples/tensorflow/docker/Dockerfile +++ /dev/null @@ -1,49 +0,0 @@ -# dev decides whether to copy notebooks in and -# run as root. Root is useful for cupti profiling -ARG dev=false -FROM nvcr.io/nvidia/cuda:10.1-devel-ubuntu18.04 AS base - -# install python and cudf -ADD https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh /miniconda.sh -RUN sh /miniconda.sh -b -p /conda && \ - /conda/bin/conda update -n base conda && \ - /conda/bin/conda create --name nvtabular -c rapidsai-nightly -c nvidia -c numba -c conda-forge \ - -c defaults cudf=0.15 python=3.7 cudatoolkit=10.1 dask-cudf pip nodejs>=10.0.0 - -# set up shell so we can do "source activate" -ENV PATH=${PATH}:/conda/bin -SHELL ["/bin/bash", "-c"] - -# set up nvtabular and example-specific libs -ADD . nvtabular/ -ADD examples/tensorflow/docker/requirements.txt requirements.txt -RUN source activate nvtabular && \ - echo "nvtabular/." >> requirements.txt && \ - pip install -U --no-cache-dir -r requirements.txt && \ - rm -rf nvtabular requirements.txt - -# configure environment -ENV HOME=/home/docker -WORKDIR $HOME -VOLUME $HOME -EXPOSE 8888 6006 - -# configure jupyter notebook -# add arg for login token and enable tensorboard -ARG token=nvidia -RUN source activate nvtabular && \ - jupyter nbextension enable --py widgetsnbextension && \ - jupyter labextension install @jupyter-widgets/jupyterlab-manager - -# add cupti to ld library path for profiling -ENV LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/usr/local/cuda-10.1/extras/CUPTI/lib64/" - -# different images for dev and production -FROM base AS true -ENTRYPOINT source activate nvtabular && jupyter lab --ip=0.0.0.0 --LabApp.token=${token} - -FROM base AS false -COPY examples/tensorflow/ $HOME -ENTRYPOINT source activate nvtabular && jupyter lab --ip=0.0.0.0 --LabApp.token=${token} --allow-root - -FROM ${dev} diff --git a/examples/tensorflow/docker/requirements.txt b/examples/tensorflow/docker/requirements.txt deleted file mode 100644 index b2e697fd779..00000000000 --- a/examples/tensorflow/docker/requirements.txt +++ /dev/null @@ -1,7 +0,0 @@ -grpcio>=1.24.3 -tensorflow>=2.4.0 -tensorboard_plugin_profile -pynvml -jupyterlab -jupyter-tensorboard -ipywidgets diff --git a/examples/tensorflow/imgs/cpu-native_vs_gpu-native.PNG b/examples/tensorflow/imgs/cpu-native_vs_gpu-native.PNG deleted file mode 100644 index f18254296f9..00000000000 Binary files a/examples/tensorflow/imgs/cpu-native_vs_gpu-native.PNG and /dev/null differ diff --git a/examples/tensorflow/imgs/cpu-native_vs_gpu-native_vs_gpu-accelerated.PNG b/examples/tensorflow/imgs/cpu-native_vs_gpu-native_vs_gpu-accelerated.PNG deleted file mode 100644 index 709b97ac5a0..00000000000 Binary files a/examples/tensorflow/imgs/cpu-native_vs_gpu-native_vs_gpu-accelerated.PNG and /dev/null differ diff --git a/examples/tensorflow/imgs/cpu-native_vs_gpu-native_vs_gpu-accelerated_vs_gpu-native-mp.PNG b/examples/tensorflow/imgs/cpu-native_vs_gpu-native_vs_gpu-accelerated_vs_gpu-native-mp.PNG deleted file mode 100644 index b6a144ce375..00000000000 Binary files a/examples/tensorflow/imgs/cpu-native_vs_gpu-native_vs_gpu-accelerated_vs_gpu-native-mp.PNG and /dev/null differ diff --git a/examples/tensorflow/imgs/cpu-native_vs_gpu-native_vs_gpu-accelerated_vs_gpu-native-mp_vs_gpu-accelerated-mp.PNG b/examples/tensorflow/imgs/cpu-native_vs_gpu-native_vs_gpu-accelerated_vs_gpu-native-mp_vs_gpu-accelerated-mp.PNG deleted file mode 100644 index 6c042a6e1da..00000000000 Binary files a/examples/tensorflow/imgs/cpu-native_vs_gpu-native_vs_gpu-accelerated_vs_gpu-native-mp_vs_gpu-accelerated-mp.PNG and /dev/null differ diff --git a/examples/tensorflow/imgs/full-precision-matmul.PNG b/examples/tensorflow/imgs/full-precision-matmul.PNG deleted file mode 100644 index 5e89d52dc1a..00000000000 Binary files a/examples/tensorflow/imgs/full-precision-matmul.PNG and /dev/null differ diff --git a/examples/tensorflow/imgs/gpu-accelerated-trace.PNG b/examples/tensorflow/imgs/gpu-accelerated-trace.PNG deleted file mode 100644 index c3f675897ef..00000000000 Binary files a/examples/tensorflow/imgs/gpu-accelerated-trace.PNG and /dev/null differ diff --git a/examples/tensorflow/imgs/gpu-native-trace-zoom.PNG b/examples/tensorflow/imgs/gpu-native-trace-zoom.PNG deleted file mode 100644 index 8906129a43d..00000000000 Binary files a/examples/tensorflow/imgs/gpu-native-trace-zoom.PNG and /dev/null differ diff --git a/examples/tensorflow/imgs/gpu-native-trace.PNG b/examples/tensorflow/imgs/gpu-native-trace.PNG deleted file mode 100644 index f157a8c176a..00000000000 Binary files a/examples/tensorflow/imgs/gpu-native-trace.PNG and /dev/null differ diff --git a/examples/tensorflow/imgs/mixed-precision-matmul.PNG b/examples/tensorflow/imgs/mixed-precision-matmul.PNG deleted file mode 100644 index 18b6578a66b..00000000000 Binary files a/examples/tensorflow/imgs/mixed-precision-matmul.PNG and /dev/null differ diff --git a/tests/unit/loader/test_torch_dataloader.py b/tests/unit/loader/test_torch_dataloader.py index 62f5b98b79e..8b6d8943bfd 100644 --- a/tests/unit/loader/test_torch_dataloader.py +++ b/tests/unit/loader/test_torch_dataloader.py @@ -293,6 +293,8 @@ def test_gpu_dl_break(tmpdir, df, dataset, batch_size, part_mem_fraction, engine assert idx < len_dl + data_itr.stop() + first_chunk_2 = 0 for idx, chunk in enumerate(data_itr): if idx == 0: diff --git a/tests/unit/test_notebooks.py b/tests/unit/test_notebooks.py deleted file mode 100644 index d2b451531d4..00000000000 --- a/tests/unit/test_notebooks.py +++ /dev/null @@ -1,292 +0,0 @@ -# -# Copyright (c) 2021, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import itertools -import json -import os -import subprocess -import sys -from os.path import dirname, realpath - -import pytest - -pytest.importorskip("cudf") -import cudf # noqa: E402 - -import nvtabular.tools.data_gen as datagen # noqa: E402 -from tests.conftest import get_cuda_cluster # noqa: E402 -from tests.unit.test_triton_inference import TRITON_SERVER_PATH, run_triton_server # noqa: E402 - -TEST_PATH = dirname(dirname(realpath(__file__))) - -# pylint: disable=unused-import,broad-except - - -def test_criteo_tf_notebook(tmpdir): - tor = pytest.importorskip("tensorflow") # noqa - # create a toy dataset in tmpdir, and point environment variables so the notebook - # will read from it - os.system("mkdir -p " + os.path.join(tmpdir, "converted/criteo")) - for i in range(24): - df = _get_random_criteo_data(1000) - df.to_parquet(os.path.join(tmpdir, "converted/criteo", f"day_{i}.parquet")) - os.environ["BASE_DIR"] = str(tmpdir) - - def _nb_modify(line): - # Disable LocalCUDACluster - line = line.replace("client.run(_rmm_pool)", "# client.run(_rmm_pool)") - line = line.replace("if cluster is None:", "if False:") - line = line.replace("client = Client(cluster)", "# client = Client(cluster)") - line = line.replace( - "workflow = nvt.Workflow(features, client=client)", "workflow = nvt.Workflow(features)" - ) - line = line.replace("client", "# client") - line = line.replace("NUM_GPUS = [0, 1, 2, 3, 4, 5, 6, 7]", "NUM_GPUS = [0]") - line = line.replace("part_size = int(part_mem_frac * device_size)", "part_size = '128MB'") - - return line - - _run_notebook( - tmpdir, - os.path.join( - dirname(TEST_PATH), - "examples/scaling-criteo/", - "02-ETL-with-NVTabular.ipynb", - ), - # disable rmm.reinitialize, seems to be causing issues - transform=_nb_modify, - ) - - def _modify_tf_nb(line): - return line.replace( - # don't require grqphviz/pydot - "tf.keras.utils.plot_model(model)", - "# tf.keras.utils.plot_model(model)", - ) - - _run_notebook( - tmpdir, - os.path.join( - dirname(TEST_PATH), - "examples/scaling-criteo/", - "03-Training-with-TF.ipynb", - ), - transform=_modify_tf_nb, - ) - - -def test_optimize_criteo(tmpdir): - input_path = str(tmpdir.mkdir("input")) - _get_random_criteo_data(1000).to_csv(os.path.join(input_path, "day_0"), sep="\t", header=False) - os.environ["INPUT_DATA_DIR"] = input_path - os.environ["OUTPUT_DATA_DIR"] = str(tmpdir.mkdir("output")) - with get_cuda_cluster() as cuda_cluster: - scheduler_port = cuda_cluster.scheduler_address - - def _nb_modify(line): - # Use cuda_cluster "fixture" port rather than allowing notebook - # to deploy a LocalCUDACluster within the subprocess - line = line.replace("download_criteo = True", "download_criteo = False") - line = line.replace("cluster = None", f"cluster = '{scheduler_port}'") - return line - - notebook_path = os.path.join( - dirname(TEST_PATH), - "examples/scaling-criteo/", - "01-Download-Convert.ipynb", - ) - _run_notebook(tmpdir, notebook_path, _nb_modify) - - -def test_movielens_example(tmpdir): - _get_random_movielens_data(tmpdir, 10000, dataset="movie") - _get_random_movielens_data(tmpdir, 10000, dataset="ratings") - _get_random_movielens_data(tmpdir, 5000, dataset="ratings", valid=True) - - triton_model_path = os.path.join(tmpdir, "models") - os.environ["INPUT_DATA_DIR"] = str(tmpdir) - os.environ["MODEL_PATH"] = triton_model_path - - notebook_path = os.path.join( - dirname(TEST_PATH), - "examples/getting-started-movielens/", - "02-ETL-with-NVTabular.ipynb", - ) - _run_notebook(tmpdir, notebook_path) - - def _modify_tf_nb(line): - return line.replace( - # don't require graphviz/pydot - "tf.keras.utils.plot_model(model)", - "# tf.keras.utils.plot_model(model)", - ) - - def _modify_tf_triton(line): - # models are already preloaded - line = line.replace("triton_client.load_model", "# triton_client.load_model") - line = line.replace("triton_client.unload_model", "# triton_client.unload_model") - return line - - notebooks = [] - try: - import torch # noqa - - notebooks.append("03-Training-with-PyTorch.ipynb") - except Exception: - pass - try: - import nvtabular.inference.triton # noqa - import nvtabular.loader.tensorflow # noqa - - notebooks.append("03-Training-with-TF.ipynb") - has_tf = True - - except Exception: - has_tf = False - - for notebook in notebooks: - notebook_path = os.path.join( - dirname(TEST_PATH), - "examples/getting-started-movielens/", - notebook, - ) - if notebook == "03-Training-with-TF.ipynb": - _run_notebook(tmpdir, notebook_path, transform=_modify_tf_nb) - else: - _run_notebook(tmpdir, notebook_path) - - # test out the TF inference movielens notebook if appropriate - if has_tf and TRITON_SERVER_PATH: - notebook = "04-Triton-Inference-with-TF.ipynb" - notebook_path = os.path.join( - dirname(TEST_PATH), - "examples/getting-started-movielens/", - notebook, - ) - with run_triton_server(triton_model_path): - _run_notebook(tmpdir, notebook_path, transform=_modify_tf_triton) - - -def test_multigpu_dask_example(tmpdir): - with get_cuda_cluster() as cuda_cluster: - os.environ["BASE_DIR"] = str(tmpdir) - scheduler_port = cuda_cluster.scheduler_address - - def _nb_modify(line): - # Use cuda_cluster "fixture" port rather than allowing notebook - # to deploy a LocalCUDACluster within the subprocess - line = line.replace("cluster = None", f"cluster = '{scheduler_port}'") - # Use a much smaller "toy" dataset - line = line.replace("write_count = 25", "write_count = 4") - line = line.replace('freq = "1s"', 'freq = "1h"') - # Use smaller partitions for smaller dataset - line = line.replace("part_mem_fraction=0.1", "part_size=1_000_000") - line = line.replace("out_files_per_proc=8", "out_files_per_proc=1") - return line - - notebook_path = os.path.join( - dirname(TEST_PATH), "examples/multi-gpu-toy-example/", "multi-gpu_dask.ipynb" - ) - _run_notebook(tmpdir, notebook_path, _nb_modify) - - -def _run_notebook(tmpdir, notebook_path, transform=None): - # read in the notebook as JSON, and extract a python script from it - notebook = json.load(open(notebook_path, encoding="utf-8")) - source_cells = [cell["source"] for cell in notebook["cells"] if cell["cell_type"] == "code"] - lines = [ - transform(line.rstrip()) if transform else line - for line in itertools.chain(*source_cells) - if not (line.startswith("%") or line.startswith("!")) - ] - - # save the script to a file, and run with the current python executable - # we're doing this in a subprocess to avoid some issues using 'exec' - # that were causing a segfault with globals of the exec'ed function going - # out of scope - script_path = os.path.join(tmpdir, "notebook.py") - with open(script_path, "w") as script: - script.write("\n".join(lines)) - subprocess.check_output([sys.executable, script_path]) - - -def _get_random_criteo_data(rows): - dtypes = {col: float for col in [f"I{x}" for x in range(1, 14)]} - dtypes.update({col: int for col in [f"C{x}" for x in range(1, 27)]}) - dtypes["label"] = bool - ret = cudf.datasets.randomdata(rows, dtypes=dtypes) - # binarize the labels - ret.label = ret.label.astype(int) - return ret - - -def _get_random_movielens_data(tmpdir, rows, dataset="movie", valid=None): - if dataset == "movie": - json_sample_movie = { - "conts": {}, - "cats": { - "genres": { - "dtype": None, - "cardinality": 50, - "min_entry_size": 1, - "max_entry_size": 5, - "multi_min": 2, - "multi_max": 4, - "multi_avg": 3, - }, - "movieId": { - "dtype": None, - "cardinality": 500, - "min_entry_size": 1, - "max_entry_size": 5, - }, - }, - } - cols = datagen._get_cols_from_schema(json_sample_movie) - if dataset == "ratings": - json_sample_ratings = { - "conts": {}, - "cats": { - "movieId": { - "dtype": None, - "cardinality": 500, - "min_entry_size": 1, - "max_entry_size": 5, - }, - "userId": { - "dtype": None, - "cardinality": 500, - "min_entry_size": 1, - "max_entry_size": 5, - }, - }, - "labels": {"rating": {"dtype": None, "cardinality": 5}}, - } - cols = datagen._get_cols_from_schema(json_sample_ratings) - - df_gen = datagen.DatasetGen(datagen.UniformDistro(), gpu_frac=0.1) - target_path = tmpdir - df_gen.full_df_create(rows, cols, output=target_path) - - if dataset == "movie": - movies_converted = cudf.read_parquet(os.path.join(tmpdir, "dataset_0.parquet")) - movies_converted = movies_converted.drop_duplicates(["movieId"], keep="first") - movies_converted.to_parquet(os.path.join(tmpdir, "movies_converted.parquet")) - - elif dataset == "ratings" and not valid: - os.rename(os.path.join(tmpdir, "dataset_0.parquet"), os.path.join(tmpdir, "train.parquet")) - else: - os.rename(os.path.join(tmpdir, "dataset_0.parquet"), os.path.join(tmpdir, "valid.parquet")) diff --git a/tox.ini b/tox.ini index 7f38c1249b4..4fa66330be1 100644 --- a/tox.ini +++ b/tox.ini @@ -84,7 +84,7 @@ commands = changedir = {toxinidir} deps = -rrequirements/docs.txt commands = - python -m sphinx.cmd.build -P -b html docs/source docs/build/html + python -m sphinx.cmd.build -P -b {posargs:html} docs/source docs/build/{posargs:html} [testenv:docs-multi] ; Run the multi-version build that is shown on GitHub Pages.