diff --git a/notebooks/learning-to-rank/01-learning-to-rank.ipynb b/notebooks/learning-to-rank/01-learning-to-rank.ipynb index 1a14bb1f..f0e3d6db 100644 --- a/notebooks/learning-to-rank/01-learning-to-rank.ipynb +++ b/notebooks/learning-to-rank/01-learning-to-rank.ipynb @@ -1,1556 +1,1572 @@ { - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "id": "TKxL_NmZmOpF" - }, - "source": [ - "# How to train and deploy Learning To Rank\n", - "\n", - "TODO: udpate the link to elastic/elasticsearch-labs instead of my fork before merging.\n", - "\n", - "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/afoucret/elasticsearch-labs/blob/ltr-notebook/notebooks/learning-to-rank/01-learning-to-rank.ipynb)\n", - "\n", - "In this notebook we will see example on how to train a Learning To Rank model using [XGBoost](https://xgboost.ai/) and how to deploy it to be used as a rescorer in Elasticsearch." - ] + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "TKxL_NmZmOpF" + }, + "source": [ + "# How to train and deploy Learning To Rank\n", + "\n", + "TODO: udpate the link to elastic/elasticsearch-labs instead of my fork before merging.\n", + "\n", + "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/afoucret/elasticsearch-labs/blob/ltr-notebook/notebooks/learning-to-rank/01-learning-to-rank.ipynb)\n", + "\n", + "In this notebook we will see example on how to train a Learning To Rank model using [XGBoost](https://xgboost.ai/) and how to deploy it to be used as a rescorer in Elasticsearch.\n", + "\n", + "\n", + "**Notes about the Learning To Rank feature:**\n", + "- The Learning To Rank feature is available for Elastic Stack versions 8.12.0 and newer and requires a Platinum subscription or higher.\n", + "- The Learning To rank is experimental and may be changed or removed completely in future releases. Elastic will make a best effort to fix any issues, but experimental features are not supported to the same level as generally available (GA) features.\n", + " \n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Jq6mztWOmOpH" + }, + "source": [ + "## Install required packages\n", + "\n", + "First we will be installing packages required for our example." + ] + }, + { + "cell_type": "code", + "execution_count": 99, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" }, - { - "cell_type": "markdown", - "metadata": { - "id": "Jq6mztWOmOpH" - }, - "source": [ - "## Install required packages\n", - "\n", - "First we will be installing packages required for our example." - ] + "id": "0nCl2nhamOpH", + "outputId": "1e7380e7-4944-430a-db5f-180f1e299615" + }, + "outputs": [], + "source": [ + "# TODO: when eland 8.12.1 is released, we can avoid installing from github main:\n", + "!pip install -qU git+https://github.com/elastic/eland@main\n", + "!pip install -qU elasticsearch \"eland[scikit-learn]\" xgboost tqdm\n", + "\n", + "from tqdm import tqdm\n", + "\n", + "# Setup the progress bar so we can use progress_apply in the notebook.\n", + "tqdm.pandas()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "yks44hf0mOpI" + }, + "source": [ + "## Configure your Elasticsearch deployment\n", + "\n", + "For this example, we will be using an [Elastic Cloud](https://www.elastic.co/guide/en/cloud/current/ec-getting-started.html) deployment (available with a [free trial](https://cloud.elastic.co/registration?utm_source=github&utm_content=elasticsearch-labs-notebook))." + ] + }, + { + "cell_type": "code", + "execution_count": 100, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 71 }, + "id": "IpnP7JUHmOpI", + "outputId": "eb52c692-a773-4863-f930-fdedb5c6e0eb" + }, + "outputs": [ { - "cell_type": "code", - "execution_count": 4, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "0nCl2nhamOpH", - "outputId": "1e7380e7-4944-430a-db5f-180f1e299615" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Collecting git+https://github.com/elastic/eland@main\n", - " Cloning https://github.com/elastic/eland (to revision main) to /private/var/folders/g_/zb4vtmp57f1f1bjvhrg0v3qc0000gn/T/pip-req-build-yf32qvq8\n", - " Running command git clone -q https://github.com/elastic/eland /private/var/folders/g_/zb4vtmp57f1f1bjvhrg0v3qc0000gn/T/pip-req-build-yf32qvq8\n", - " Resolved https://github.com/elastic/eland to commit 2a6a4b1f06b39e79a3c67a450193992bf6c0ac0a\n", - "Requirement already satisfied: elasticsearch<9,>=8.3 in /Users/afoucret/git/elasticsearch-labs/.venv/lib/python3.9/site-packages (from eland==8.12.0) (8.12.0)\n", - "Requirement already satisfied: pandas<2,>=1.5 in /Users/afoucret/git/elasticsearch-labs/.venv/lib/python3.9/site-packages (from eland==8.12.0) (1.5.3)\n", - "Requirement already satisfied: matplotlib>=3.6 in /Users/afoucret/git/elasticsearch-labs/.venv/lib/python3.9/site-packages (from eland==8.12.0) (3.8.2)\n", - "Requirement already satisfied: numpy<2,>=1.2.0 in /Users/afoucret/git/elasticsearch-labs/.venv/lib/python3.9/site-packages (from eland==8.12.0) (1.26.3)\n", - "Requirement already satisfied: packaging in /Users/afoucret/git/elasticsearch-labs/.venv/lib/python3.9/site-packages (from eland==8.12.0) (23.2)\n", - "Requirement already satisfied: elastic-transport<9,>=8 in /Users/afoucret/git/elasticsearch-labs/.venv/lib/python3.9/site-packages (from elasticsearch<9,>=8.3->eland==8.12.0) (8.12.0)\n", - "Requirement already satisfied: urllib3<3,>=1.26.2 in /Users/afoucret/git/elasticsearch-labs/.venv/lib/python3.9/site-packages (from elastic-transport<9,>=8->elasticsearch<9,>=8.3->eland==8.12.0) (2.1.0)\n", - "Requirement already satisfied: certifi in /Users/afoucret/git/elasticsearch-labs/.venv/lib/python3.9/site-packages (from elastic-transport<9,>=8->elasticsearch<9,>=8.3->eland==8.12.0) (2023.11.17)\n", - "Requirement already satisfied: kiwisolver>=1.3.1 in /Users/afoucret/git/elasticsearch-labs/.venv/lib/python3.9/site-packages (from matplotlib>=3.6->eland==8.12.0) (1.4.5)\n", - "Requirement already satisfied: pillow>=8 in /Users/afoucret/git/elasticsearch-labs/.venv/lib/python3.9/site-packages (from matplotlib>=3.6->eland==8.12.0) (10.2.0)\n", - "Requirement already satisfied: pyparsing>=2.3.1 in /Users/afoucret/git/elasticsearch-labs/.venv/lib/python3.9/site-packages (from matplotlib>=3.6->eland==8.12.0) (3.1.1)\n", - "Requirement already satisfied: importlib-resources>=3.2.0 in /Users/afoucret/git/elasticsearch-labs/.venv/lib/python3.9/site-packages (from matplotlib>=3.6->eland==8.12.0) (6.1.1)\n", - "Requirement already satisfied: cycler>=0.10 in /Users/afoucret/git/elasticsearch-labs/.venv/lib/python3.9/site-packages (from matplotlib>=3.6->eland==8.12.0) (0.12.1)\n", - "Requirement already satisfied: python-dateutil>=2.7 in /Users/afoucret/git/elasticsearch-labs/.venv/lib/python3.9/site-packages (from matplotlib>=3.6->eland==8.12.0) (2.8.2)\n", - "Requirement already satisfied: fonttools>=4.22.0 in /Users/afoucret/git/elasticsearch-labs/.venv/lib/python3.9/site-packages (from matplotlib>=3.6->eland==8.12.0) (4.47.2)\n", - "Requirement already satisfied: contourpy>=1.0.1 in /Users/afoucret/git/elasticsearch-labs/.venv/lib/python3.9/site-packages (from matplotlib>=3.6->eland==8.12.0) (1.2.0)\n", - "Requirement already satisfied: zipp>=3.1.0 in /Users/afoucret/git/elasticsearch-labs/.venv/lib/python3.9/site-packages (from importlib-resources>=3.2.0->matplotlib>=3.6->eland==8.12.0) (3.17.0)\n", - "Requirement already satisfied: pytz>=2020.1 in /Users/afoucret/git/elasticsearch-labs/.venv/lib/python3.9/site-packages (from pandas<2,>=1.5->eland==8.12.0) (2023.3.post1)\n", - "Requirement already satisfied: six>=1.5 in /Users/afoucret/git/elasticsearch-labs/.venv/lib/python3.9/site-packages (from python-dateutil>=2.7->matplotlib>=3.6->eland==8.12.0) (1.16.0)\n", - "\u001b[33mWARNING: You are using pip version 21.2.4; however, version 23.3.2 is available.\n", - "You should consider upgrading via the '/Users/afoucret/git/elasticsearch-labs/.venv/bin/python3 -m pip install --upgrade pip' command.\u001b[0m\n", - "Requirement already satisfied: elasticsearch in /Users/afoucret/git/elasticsearch-labs/.venv/lib/python3.9/site-packages (8.12.0)\n", - "Requirement already satisfied: eland[scikit-learn] in /Users/afoucret/git/elasticsearch-labs/.venv/lib/python3.9/site-packages (8.12.0)\n", - "Requirement already satisfied: xgboost in /Users/afoucret/git/elasticsearch-labs/.venv/lib/python3.9/site-packages (2.0.1)\n", - "Requirement already satisfied: tqdm in /Users/afoucret/git/elasticsearch-labs/.venv/lib/python3.9/site-packages (4.66.1)\n", - "Requirement already satisfied: elastic-transport<9,>=8 in /Users/afoucret/git/elasticsearch-labs/.venv/lib/python3.9/site-packages (from elasticsearch) (8.12.0)\n", - "Requirement already satisfied: pandas<2,>=1.5 in /Users/afoucret/git/elasticsearch-labs/.venv/lib/python3.9/site-packages (from eland[scikit-learn]) (1.5.3)\n", - "Requirement already satisfied: matplotlib>=3.6 in /Users/afoucret/git/elasticsearch-labs/.venv/lib/python3.9/site-packages (from eland[scikit-learn]) (3.8.2)\n", - "Requirement already satisfied: numpy<2,>=1.2.0 in /Users/afoucret/git/elasticsearch-labs/.venv/lib/python3.9/site-packages (from eland[scikit-learn]) (1.26.3)\n", - "Requirement already satisfied: packaging in /Users/afoucret/git/elasticsearch-labs/.venv/lib/python3.9/site-packages (from eland[scikit-learn]) (23.2)\n", - "Requirement already satisfied: scikit-learn<1.4,>=1.3 in /Users/afoucret/git/elasticsearch-labs/.venv/lib/python3.9/site-packages (from eland[scikit-learn]) (1.3.2)\n", - "Requirement already satisfied: scipy in /Users/afoucret/git/elasticsearch-labs/.venv/lib/python3.9/site-packages (from xgboost) (1.12.0)\n", - "Requirement already satisfied: urllib3<3,>=1.26.2 in /Users/afoucret/git/elasticsearch-labs/.venv/lib/python3.9/site-packages (from elastic-transport<9,>=8->elasticsearch) (2.1.0)\n", - "Requirement already satisfied: certifi in /Users/afoucret/git/elasticsearch-labs/.venv/lib/python3.9/site-packages (from elastic-transport<9,>=8->elasticsearch) (2023.11.17)\n", - "Requirement already satisfied: contourpy>=1.0.1 in /Users/afoucret/git/elasticsearch-labs/.venv/lib/python3.9/site-packages (from matplotlib>=3.6->eland[scikit-learn]) (1.2.0)\n", - "Requirement already satisfied: pyparsing>=2.3.1 in /Users/afoucret/git/elasticsearch-labs/.venv/lib/python3.9/site-packages (from matplotlib>=3.6->eland[scikit-learn]) (3.1.1)\n", - "Requirement already satisfied: python-dateutil>=2.7 in /Users/afoucret/git/elasticsearch-labs/.venv/lib/python3.9/site-packages (from matplotlib>=3.6->eland[scikit-learn]) (2.8.2)\n", - "Requirement already satisfied: pillow>=8 in /Users/afoucret/git/elasticsearch-labs/.venv/lib/python3.9/site-packages (from matplotlib>=3.6->eland[scikit-learn]) (10.2.0)\n", - "Requirement already satisfied: fonttools>=4.22.0 in /Users/afoucret/git/elasticsearch-labs/.venv/lib/python3.9/site-packages (from matplotlib>=3.6->eland[scikit-learn]) (4.47.2)\n", - "Requirement already satisfied: kiwisolver>=1.3.1 in /Users/afoucret/git/elasticsearch-labs/.venv/lib/python3.9/site-packages (from matplotlib>=3.6->eland[scikit-learn]) (1.4.5)\n", - "Requirement already satisfied: importlib-resources>=3.2.0 in /Users/afoucret/git/elasticsearch-labs/.venv/lib/python3.9/site-packages (from matplotlib>=3.6->eland[scikit-learn]) (6.1.1)\n", - "Requirement already satisfied: cycler>=0.10 in /Users/afoucret/git/elasticsearch-labs/.venv/lib/python3.9/site-packages (from matplotlib>=3.6->eland[scikit-learn]) (0.12.1)\n", - "Requirement already satisfied: zipp>=3.1.0 in /Users/afoucret/git/elasticsearch-labs/.venv/lib/python3.9/site-packages (from importlib-resources>=3.2.0->matplotlib>=3.6->eland[scikit-learn]) (3.17.0)\n", - "Requirement already satisfied: pytz>=2020.1 in /Users/afoucret/git/elasticsearch-labs/.venv/lib/python3.9/site-packages (from pandas<2,>=1.5->eland[scikit-learn]) (2023.3.post1)\n", - "Requirement already satisfied: six>=1.5 in /Users/afoucret/git/elasticsearch-labs/.venv/lib/python3.9/site-packages (from python-dateutil>=2.7->matplotlib>=3.6->eland[scikit-learn]) (1.16.0)\n", - "Requirement already satisfied: joblib>=1.1.1 in /Users/afoucret/git/elasticsearch-labs/.venv/lib/python3.9/site-packages (from scikit-learn<1.4,>=1.3->eland[scikit-learn]) (1.3.2)\n", - "Requirement already satisfied: threadpoolctl>=2.0.0 in /Users/afoucret/git/elasticsearch-labs/.venv/lib/python3.9/site-packages (from scikit-learn<1.4,>=1.3->eland[scikit-learn]) (3.2.0)\n", - "\u001b[33mWARNING: You are using pip version 21.2.4; however, version 23.3.2 is available.\n", - "You should consider upgrading via the '/Users/afoucret/git/elasticsearch-labs/.venv/bin/python3 -m pip install --upgrade pip' command.\u001b[0m\n" - ] - } - ], - "source": [ - "# TODO: when eland 8.12.1 is released, we can avoid installing from github main:\n", - "!pip install git+https://github.com/elastic/eland@main\n", - "!pip install elasticsearch \"eland[scikit-learn]\" xgboost tqdm\n", - "\n", - "from tqdm import tqdm\n", - "# Setup the progress bar so we can use progress_apply in the notebook.\n", - "tqdm.pandas()" + "data": { + "text/plain": [ + "'Successfully connected to cluster bd63f706e18b476aacb5cb0aaeb5f0bd (version 8.12.0)'" ] + }, + "execution_count": 100, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import getpass\n", + "from elasticsearch import Elasticsearch\n", + "\n", + "# Found in the \"Manage Deployment\" page\n", + "try:\n", + " CLOUD_ID\n", + "except NameError:\n", + " CLOUD_ID = getpass.getpass(\"Enter Elastic Cloud ID: \")\n", + "\n", + "# Password for the \"elastic\" user generated by Elasticsearch\n", + "try:\n", + " ELASTIC_PASSWORD\n", + "except NameError:\n", + " ELASTIC_PASSWORD = getpass.getpass(\"Enter Elastic password: \")\n", + "\n", + "# Create the client instance\n", + "es_client = Elasticsearch(cloud_id=CLOUD_ID, basic_auth=(\"elastic\", ELASTIC_PASSWORD))\n", + "\n", + "client_info = es_client.info()\n", + "\n", + "f\"Successfully connected to cluster {client_info['cluster_name']} (version {client_info['version']['number']})\"" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "KLAN6aq_mOpJ" + }, + "source": [ + "## Configuring the dataset\n", + "\n", + "In this example notebook we will use a dataset derived from [MSRD](https://github.com/metarank/msrd/tree/master) (Movie Search Ranking Dataset).\n", + "\n", + "The dataset is available [here](https://github.com/elastic/elasticsearch-labs/tree/main//ltr-notebook/notebooks/learning-to-rank/sample_data/) and contains the following files:\n", + "\n", + "- **movies_corpus.jsonl.gz**: The movies dataset which will be indexed.\n", + "- **movies_judgements.tsv.gz**: A file containing relevance judgments for a set of queries.\n", + "- **movies_index_settings.json**: Settings to be applied to the documents and index." + ] + }, + { + "cell_type": "code", + "execution_count": 101, + "metadata": { + "id": "gFm7i-b7mOpJ" + }, + "outputs": [], + "source": [ + "from urllib.parse import urljoin\n", + "\n", + "# TODO: use elastic/elasticsearch-labs instead of afoucret/elasticsearch-labs before merging the PR.\n", + "\n", + "DATASET_BASE_URL = \"https://raw.githubusercontent.com/afoucret/elasticsearch-labs/ltr-notebook/notebooks/learning-to-rank/sample_data/\"\n", + "\n", + "CORPUS_URL = urljoin(DATASET_BASE_URL, \"movies_corpus.jsonl.gz\")\n", + "JUDGEMENTS_FILE_URL = urljoin(DATASET_BASE_URL, \"movies_judgments.tsv.gz\")\n", + "INDEX_SETTINGS_URL = urljoin(DATASET_BASE_URL, \"movies_index_settings.json\")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "fhO5awX9mOpJ" + }, + "source": [ + " ## Importing the document corpus\n", + "\n", + "This step will import the documents of the corpus into the `movies` index .\n", + "\n", + "Documents contains the following fields:\n", + "\n", + "| Field name | Description |\n", + "|--------------|---------------------------------------------|\n", + "| `id` | Id of the document |\n", + "| `title` | Movie title |\n", + "| `overview` | A short description of the movie |\n", + "| `actors` | List of actors in the movies |\n", + "| `director` | Director of the movie |\n", + "| `characters` | List of characters that appear in the movie |\n", + "| `genres` | Genres of the movie |\n", + "| `year` | Year the movie was released |\n", + "| `budget` | Budget of the movies in USD |\n", + "| `votes` | Number of votes received by the movie |\n", + "| `rating` | Average rating of the movie |\n", + "| `popularity` | Number use to measure the movie popularity |\n", + "| `tags` | A list of tags for the movies |\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 103, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" }, + "id": "v5vhClAHmOpK", + "outputId": "77ee3248-86ad-4cbf-9b3e-9dfdc9cf93f4" + }, + "outputs": [ { - "cell_type": "markdown", - "metadata": { - "id": "yks44hf0mOpI" - }, - "source": [ - "## Configure your Elasticsearch deployment\n", - "\n", - "For this example, we will be using an [Elastic Cloud](https://www.elastic.co/guide/en/cloud/current/ec-getting-started.html) deployment (available with a [free trial](https://cloud.elastic.co/registration?utm_source=github&utm_content=elasticsearch-labs-notebook))." - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "Deleting index if it already exists: movies\n", + "Creating index: movies\n", + "Loading the corpus from https://raw.githubusercontent.com/afoucret/elasticsearch-labs/ltr-notebook/notebooks/learning-to-rank/sample_data/movies_corpus.jsonl.gz\n", + "Indexing the corpus into movies ...\n", + "Indexed 9751 documents into movies\n" + ] + } + ], + "source": [ + "import json\n", + "import elasticsearch.helpers as es_helpers\n", + "import pandas as pd\n", + "from urllib.request import urlopen\n", + "\n", + "MOVIE_INDEX = \"movies\"\n", + "\n", + "# Delete index\n", + "print(\"Deleting index if it already exists:\", MOVIE_INDEX)\n", + "es_client.options(ignore_status=[400, 404]).indices.delete(index=MOVIE_INDEX)\n", + "\n", + "print(\"Creating index:\", MOVIE_INDEX)\n", + "index_settings = json.load(urlopen(INDEX_SETTINGS_URL))\n", + "es_client.indices.create(index=MOVIE_INDEX, **index_settings)\n", + "\n", + "print(f\"Loading the corpus from {CORPUS_URL}\")\n", + "corpus_df = pd.read_json(CORPUS_URL, lines=True)\n", + "\n", + "print(f\"Indexing the corpus into {MOVIE_INDEX} ...\")\n", + "bulk_result = es_helpers.bulk(\n", + " es_client,\n", + " actions=[\n", + " {\"_id\": movie[\"id\"], \"_index\": MOVIE_INDEX, **movie}\n", + " for movie in corpus_df.to_dict(\"records\")\n", + " ],\n", + ")\n", + "print(f\"Indexed {bulk_result[0]} documents into {MOVIE_INDEX}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Loading the judgment list\n", + "\n", + "Judgemnent list provides human judgement that will be used to train our Learning To Rank model.\n", + "\n", + "Each row represents a query-document pair with an associated relevance grade and contains the following columns:\n", + "\n", + "| Column | Description |\n", + "|-----------|------------------------------------------------------------------------|\n", + "| `query_id`| Pair for the same query are grouped together and received a unique id. |\n", + "| `query` | Actual text for the query. |\n", + "| `doc_id` | Id of the document. |\n", + "| `grade` | The relevance grade of the document for the query. |\n", + "\n", + "\n", + "**Note:**\n", + "\n", + "In our notebook the relevance grade is a binary value (relevant or not relavant).\n", + "Instread of a binary judgement, you can also use a number that represent the degree of relevance (e.g. from `0` to `4`)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 424 }, + "id": "XLjiKfYQqM-U", + "outputId": "38df2283-421f-43ea-8bdf-580f1a63ac0d" + }, + "outputs": [ { - "cell_type": "code", - "execution_count": 5, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 71 - }, - "id": "IpnP7JUHmOpI", - "outputId": "eb52c692-a773-4863-f930-fdedb5c6e0eb" - }, - "outputs": [ - { - "data": { - "text/plain": [ - "'Successfully connected to cluster runTask (version 8.13.0-SNAPSHOT)'" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
query_idquerydoc_idgrade
0qid:5141insidious 2 netflix8464330
1qid:5141insidious 2 netflix490181
2qid:5141insidious 2 netflix382340
3qid:5141insidious 2 netflix5676040
4qid:5141insidious 2 netflix2697950
...............
384750qid:33832013 the wolverine2631150
384751qid:33832013 the wolverine259130
384752qid:33832013 the wolverine5676040
384753qid:33832013 the wolverine5335350
384754qid:33832013 the wolverine8763270
\n", + "

384755 rows × 4 columns

\n", + "
" ], - "source": [ - "import getpass\n", - "from elasticsearch import Elasticsearch\n", - "\n", - "# Found in the \"Manage Deployment\" page\n", - "try: CLOUD_ID\n", - "except NameError: CLOUD_ID = getpass.getpass(\"Enter Elastic Cloud ID: \")\n", - "\n", - "# Password for the \"elastic\" user generated by Elasticsearch\n", - "try: ELASTIC_PASSWORD\n", - "except NameError:\n", - " ELASTIC_PASSWORD = getpass.getpass(\"Enter Elastic password: \")\n", - "\n", - "# Create the client instance\n", - "es_client = Elasticsearch(\n", - " cloud_id=CLOUD_ID,\n", - " basic_auth=(\"elastic\", ELASTIC_PASSWORD)\n", - ")\n", - "\n", - "client_info = es_client.info()\n", - "\n", - "f\"Successfully connected to cluster {client_info['cluster_name']} (version {client_info['version']['number']})\"" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "KLAN6aq_mOpJ" - }, - "source": [ - "## Configuring the dataset\n", - "\n", - "In this example notebook we will use a dataset derived from [MSRD](https://github.com/metarank/msrd/tree/master) (Movie Search Ranking Dataset).\n", - "\n", - "The dataset is available [here](https://github.com/elastic/elasticsearch-labs/tree/main//ltr-notebook/notebooks/learning-to-rank/sample_data/) and contains the following files:\n", - "\n", - "- **movies_corpus.jsonl.gz**\n", - "- **movies_judgements.csv.gz**:\n", - "- **movies_index_settings.json**" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": { - "id": "gFm7i-b7mOpJ" - }, - "outputs": [], - "source": [ - "from urllib.parse import urljoin\n", - "\n", - "# TODO: use elastic/elasticsearch-labs instead of afoucret/elasticsearch-labs before merging the PR.\n", - "\n", - "DATASET_BASE_URL = \"https://raw.githubusercontent.com/afoucret/elasticsearch-labs/ltr-notebook/notebooks/learning-to-rank/sample_data/\"\n", - "\n", - "CORPUS_URL = urljoin(DATASET_BASE_URL, \"movies_corpus.jsonl.gz\")\n", - "JUDGEMENTS_FILE_URL = urljoin(DATASET_BASE_URL,\"movies_judgments.csv.gz\")\n", - "INDEX_SETTINGS_URL = urljoin(DATASET_BASE_URL,\"movies_index_settings.json\")\n" + "text/plain": [ + " query_id query doc_id grade\n", + "0 qid:5141 insidious 2 netflix 846433 0\n", + "1 qid:5141 insidious 2 netflix 49018 1\n", + "2 qid:5141 insidious 2 netflix 38234 0\n", + "3 qid:5141 insidious 2 netflix 567604 0\n", + "4 qid:5141 insidious 2 netflix 269795 0\n", + "... ... ... ... ...\n", + "384750 qid:3383 2013 the wolverine 263115 0\n", + "384751 qid:3383 2013 the wolverine 25913 0\n", + "384752 qid:3383 2013 the wolverine 567604 0\n", + "384753 qid:3383 2013 the wolverine 533535 0\n", + "384754 qid:3383 2013 the wolverine 876327 0\n", + "\n", + "[384755 rows x 4 columns]" ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "judgments_df = pd.read_csv(JUDGEMENTS_FILE_URL, delimiter=\"\\t\")\n", + "judgments_df" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Configure feature extraction\n", + "\n", + "Features and the inputs to our model. They represent information about the query alone, a result document alone or a result document in the context of a query, as in the case of BM25 scores.\n", + "\n", + "Features are defined using standard templated queries and the Query DSL.\n", + "\n", + "To simplify defining and iterating on feature extraction during training, we've exposed some primitives directly in `eland`." + ] + }, + { + "cell_type": "code", + "execution_count": 105, + "metadata": { + "id": "LjxAj4lQqEYJ" + }, + "outputs": [], + "source": [ + "from eland.ml.ltr import LTRModelConfig, QueryFeatureExtractor\n", + "\n", + "ltr_config = LTRModelConfig(\n", + " feature_extractors=[\n", + " # For the following field we want to use the score of the match query for the field as a features:\n", + " QueryFeatureExtractor(\n", + " feature_name=\"title_bm25\", query={\"match\": {\"title\": \"{{query}}\"}}\n", + " ),\n", + " QueryFeatureExtractor(\n", + " feature_name=\"actors_bm25\", query={\"match\": {\"actors\": \"{{query}}\"}}\n", + " ),\n", + " # We could also use a more strict matching clause as an additional features. Here we want all the terms of our query to match.\n", + " QueryFeatureExtractor(\n", + " feature_name=\"title_all_terms_bm25\",\n", + " query={\n", + " \"match\": {\n", + " \"title\": {\"query\": \"{{query}}\", \"minimum_should_match\": \"100%\"}\n", + " }\n", + " },\n", + " ),\n", + " QueryFeatureExtractor(\n", + " feature_name=\"actors_all_terms_bm25\",\n", + " query={\n", + " \"match\": {\n", + " \"actors\": {\"query\": \"{{query}}\", \"minimum_should_match\": \"100%\"}\n", + " }\n", + " },\n", + " ),\n", + " # Also we can use a script_score query to get the document field values directly as a feature.\n", + " QueryFeatureExtractor(\n", + " feature_name=\"popularity\",\n", + " query={\n", + " \"script_score\": {\n", + " \"query\": {\"exists\": {\"field\": \"popularity\"}},\n", + " \"script\": {\"source\": \"return doc['popularity'].value;\"},\n", + " }\n", + " },\n", + " ),\n", + " ]\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Building the training dataset\n", + "\n", + "Now that we have our basic datasets loaded, and feature extraction configured, we'll use our judgement list to come up with the final dataset for training. The dataset will consist of rows containing `` pairs, as well as all of the features we need to train the model. To generate this dataset, we'll run each query from the judgement list and add the extracted features as columns for each of the labelled result documents in the judgement list.\n", + "\n", + "For example, if we have a query `q1` with two labelled documents `d3` and `d9`, the training dataset will end up with two rows — one for each of the pairs `` and ``.\n", + "\n", + "Note that because this executes queries on your Elasticsearch cluster, the time to run this operation will vary depending on where the cluster is versus where this notebook runs. For example, if you run the notebook on the same server or host as the Elasticsearch cluster, this operation tends to run very quickly on the sample dataset (< 2 mins)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 615, + "referenced_widgets": [ + "f9cdfbc3972a4b84a557507567ca2965", + "a6d4eb3325444f28b11ba02c3d01ed83", + "bff504814b434aec90b2cf020b08cfa9", + "594c5ebcb9624b63b128536a46594211", + "e95447129da74d9ebc4c1d99165bd534", + "7ed336be71e74521a596a7d624c1e7d1", + "ee1a6943af1e49e6a8851f27c9811c32", + "8c393bd522f6427f9978a89bc7dbdf3b", + "549645ca4e7b48ef86cffdaa5507c56c", + "0ab8ee0c2e1a42658ecbf01b0b28cf92", + "84206a53779249fbb34c78edb17fd1e0" + ] }, + "id": "xbp6_9dqqJkJ", + "outputId": "0aadfe34-d739-4823-e5e2-310bd5fb69d3" + }, + "outputs": [ { - "cell_type": "markdown", - "metadata": { - "id": "fhO5awX9mOpJ" - }, - "source": [ - " ## Importing the document corpus\n", - "\n", - "This step will import the documents of the corpus into the `movies` index .\n", - "\n", - "Documments contains the following fields:\n", - "\n", - "| Field name | Description |\n", - "|--------------|---------------------------------------------|\n", - "| `id` | Id of the document |\n", - "| `title` | Movie title |\n", - "| `overview` | A short description of the movie |\n", - "| `actors` | List of actors in the movies |\n", - "| `director` | Director of the movie |\n", - "| `characters` | List of characters that appear in the movie |\n", - "| `genres` | Genres of the movie |\n", - "| `year` | Year the movie was released |\n", - "| `budget` | Budget of the movies in USD |\n", - "| `votes` | Number of votes received by the movie |\n", - "| `rating` | Average rating of the movie |\n", - "| `popularity` | Number use to measure the movie popularity |\n", - "| `tags` | A list of tags for the movies |\n", - "\n" - ] + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 16279/16279 [01:28<00:00, 183.72it/s]\n" + ] }, { - "cell_type": "code", - "execution_count": 7, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "v5vhClAHmOpK", - "outputId": "77ee3248-86ad-4cbf-9b3e-9dfdc9cf93f4" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Deleting index if it already exists: movies\n", - "Creating index: movies\n", - "Loading the corpus from https://raw.githubusercontent.com/afoucret/elasticsearch-labs/ltr-notebook/notebooks/learning-to-rank/sample_data/movies_corpus.jsonl.gz\n", - "Indexing the corpus into movies ...\n", - "Indexed 9751 documents into movies\n" - ] - } + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
query_idquerydoc_idgradetitle_bm25actors_bm25title_all_terms_bm25popularity
0qid:5141insidious 2 netflix8464330NaN9.555378NaN13.628
1qid:5141insidious 2 netflix4901819.857398NaNNaN64.003
2qid:5141insidious 2 netflix382340NaNNaNNaN143.211
3qid:5141insidious 2 netflix5676040NaNNaNNaN32.913
4qid:5141insidious 2 netflix26979503.809668NaNNaN21.058
...........................
384750qid:33832013 the wolverine2631150NaNNaNNaN68.287
384751qid:33832013 the wolverine259130NaNNaNNaN21.026
384752qid:33832013 the wolverine5676040NaNNaNNaN32.913
384753qid:33832013 the wolverine5335350NaNNaNNaN34.773
384754qid:33832013 the wolverine8763270NaNNaNNaN25.920
\n", + "

384755 rows × 8 columns

\n", + "
" ], - "source": [ - "import json\n", - "import elasticsearch.helpers as es_helpers\n", - "import pandas as pd\n", - "from urllib.request import urlopen\n", - "\n", - "MOVIE_INDEX = \"movies\"\n", - "\n", - "# Delete index\n", - "print(\"Deleting index if it already exists:\", MOVIE_INDEX)\n", - "es_client.options(ignore_status=[400, 404]).indices.delete(index=MOVIE_INDEX)\n", - "\n", - "print(\"Creating index:\", MOVIE_INDEX)\n", - "index_settings = json.load(urlopen(INDEX_SETTINGS_URL))\n", - "es_client.indices.create(index=MOVIE_INDEX, **index_settings)\n", - "\n", - "print(f\"Loading the corpus from {CORPUS_URL}\")\n", - "corpus_df = pd.read_json(CORPUS_URL, lines=True)\n", - "\n", - "print(f\"Indexing the corpus into {MOVIE_INDEX} ...\")\n", - "bulk_result = es_helpers.bulk(\n", - " es_client,\n", - " actions=[{ \"_id\": movie['id'], \"_index\": MOVIE_INDEX, **movie } for movie in corpus_df.to_dict('records')]\n", - ")\n", - "print(f\"Indexed {bulk_result[0]} documents into {MOVIE_INDEX}\")" + "text/plain": [ + " query_id query doc_id grade title_bm25 actors_bm25 \\\n", + "0 qid:5141 insidious 2 netflix 846433 0 NaN 9.555378 \n", + "1 qid:5141 insidious 2 netflix 49018 1 9.857398 NaN \n", + "2 qid:5141 insidious 2 netflix 38234 0 NaN NaN \n", + "3 qid:5141 insidious 2 netflix 567604 0 NaN NaN \n", + "4 qid:5141 insidious 2 netflix 269795 0 3.809668 NaN \n", + "... ... ... ... ... ... ... \n", + "384750 qid:3383 2013 the wolverine 263115 0 NaN NaN \n", + "384751 qid:3383 2013 the wolverine 25913 0 NaN NaN \n", + "384752 qid:3383 2013 the wolverine 567604 0 NaN NaN \n", + "384753 qid:3383 2013 the wolverine 533535 0 NaN NaN \n", + "384754 qid:3383 2013 the wolverine 876327 0 NaN NaN \n", + "\n", + " title_all_terms_bm25 popularity \n", + "0 NaN 13.628 \n", + "1 NaN 64.003 \n", + "2 NaN 143.211 \n", + "3 NaN 32.913 \n", + "4 NaN 21.058 \n", + "... ... ... \n", + "384750 NaN 68.287 \n", + "384751 NaN 21.026 \n", + "384752 NaN 32.913 \n", + "384753 NaN 34.773 \n", + "384754 NaN 25.920 \n", + "\n", + "[384755 rows x 8 columns]" ] - }, + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import numpy\n", + "\n", + "from eland.ml.ltr import FeatureLogger\n", + "\n", + "# First we create a feature logger that will be used to query Elasticsearch to retrieve the features:\n", + "feature_logger = FeatureLogger(es_client, MOVIE_INDEX, ltr_config)\n", + "\n", + "\n", + "# This method will be applied for each group of query in the judgment log:\n", + "def _extract_query_features(query_judgements_group):\n", + " # Retrieve document ids in the query group as strings.\n", + " doc_ids = query_judgements_group[\"doc_id\"].astype(\"str\").to_list()\n", + "\n", + " # Resolve query paras for the current query group (e.g.: {\"query\": \"batman\"}).\n", + " query_params = {\"query\": query_judgements_group[\"query\"].iloc[0]}\n", + "\n", + " # Extract the features for the documents in the query group:\n", + " doc_features = feature_logger.extract_features(query_params, doc_ids)\n", + "\n", + " # Adding a column to the dataframe for each features:\n", + " for feature_index, feature_name in enumerate(feature_logger._model_config.feature_names):\n", + " query_judgements_group[feature_name] = numpy.array([doc_features[doc_id][feature_index] for doc_id in doc_ids])\n", + "\n", + " return query_judgements_group\n", + "\n", + "\n", + "judgments_with_features = judgments_df.groupby(\"query_id\", group_keys=False).progress_apply(_extract_query_features)\n", + "\n", + "judgments_with_features" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Create and train the model\n", + "\n", + "The LTR rescorer supports XGBRanker trained models.\n", + "\n", + "You will find more information on XGBRanker model in the xgboost [documentation](https://xgboost.readthedocs.io/en/latest/tutorials/learning_to_rank.html)." + ] + }, + { + "cell_type": "code", + "execution_count": 125, + "metadata": {}, + "outputs": [ { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Loading the judgment list\n", - "\n", - "Judgemnent list provides human judgement that will be used to train our Learning To Rank model.\n", - "\n", - "Each row represents a query-document pair with an associated relevance grade and contains the following columns:\n", - "\n", - "| Column | Description |\n", - "|-----------|------------------------------------------------------------------------|\n", - "| `query_id`| Pair for the same query are grouped together and received a unique id. |\n", - "| `query` | Actual text for the query. |\n", - "| `doc_id` | Id of the document. |\n", - "| `grade` | The relevance grade of the document for the query. |\n", - "\n", - "\n", - "**Note:**\n", - "\n", - "In our notebook the relevance grade is a binary value (relevant or not relavant).\n", - "Instread of a binary judgement, you can also use a number that represent the degree of relevance (e.g. from `0` to `4`)." - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "[0]\tvalidation_0-ndcg@10:0.86234\n", + "[1]\tvalidation_0-ndcg@10:0.87022\n", + "[2]\tvalidation_0-ndcg@10:0.87147\n", + "[3]\tvalidation_0-ndcg@10:0.87229\n", + "[4]\tvalidation_0-ndcg@10:0.87288\n", + "[5]\tvalidation_0-ndcg@10:0.87311\n", + "[6]\tvalidation_0-ndcg@10:0.87315\n", + "[7]\tvalidation_0-ndcg@10:0.87361\n", + "[8]\tvalidation_0-ndcg@10:0.87451\n", + "[9]\tvalidation_0-ndcg@10:0.87493\n", + "[10]\tvalidation_0-ndcg@10:0.87514\n", + "[11]\tvalidation_0-ndcg@10:0.87553\n", + "[12]\tvalidation_0-ndcg@10:0.87564\n", + "[13]\tvalidation_0-ndcg@10:0.87650\n", + "[14]\tvalidation_0-ndcg@10:0.87653\n", + "[15]\tvalidation_0-ndcg@10:0.87679\n", + "[16]\tvalidation_0-ndcg@10:0.87700\n", + "[17]\tvalidation_0-ndcg@10:0.87749\n", + "[18]\tvalidation_0-ndcg@10:0.87754\n", + "[19]\tvalidation_0-ndcg@10:0.87794\n", + "[20]\tvalidation_0-ndcg@10:0.87796\n", + "[21]\tvalidation_0-ndcg@10:0.87837\n", + "[22]\tvalidation_0-ndcg@10:0.87902\n", + "[23]\tvalidation_0-ndcg@10:0.87904\n", + "[24]\tvalidation_0-ndcg@10:0.87910\n", + "[25]\tvalidation_0-ndcg@10:0.87962\n", + "[26]\tvalidation_0-ndcg@10:0.87962\n", + "[27]\tvalidation_0-ndcg@10:0.87980\n", + "[28]\tvalidation_0-ndcg@10:0.88025\n", + "[29]\tvalidation_0-ndcg@10:0.88025\n", + "[30]\tvalidation_0-ndcg@10:0.88058\n", + "[31]\tvalidation_0-ndcg@10:0.88051\n", + "[32]\tvalidation_0-ndcg@10:0.88058\n", + "[33]\tvalidation_0-ndcg@10:0.88090\n", + "[34]\tvalidation_0-ndcg@10:0.88094\n", + "[35]\tvalidation_0-ndcg@10:0.88090\n", + "[36]\tvalidation_0-ndcg@10:0.88118\n", + "[37]\tvalidation_0-ndcg@10:0.88124\n", + "[38]\tvalidation_0-ndcg@10:0.88145\n", + "[39]\tvalidation_0-ndcg@10:0.88216\n", + "[40]\tvalidation_0-ndcg@10:0.88227\n", + "[41]\tvalidation_0-ndcg@10:0.88239\n", + "[42]\tvalidation_0-ndcg@10:0.88273\n", + "[43]\tvalidation_0-ndcg@10:0.88286\n", + "[44]\tvalidation_0-ndcg@10:0.88317\n", + "[45]\tvalidation_0-ndcg@10:0.88311\n", + "[46]\tvalidation_0-ndcg@10:0.88323\n", + "[47]\tvalidation_0-ndcg@10:0.88335\n", + "[48]\tvalidation_0-ndcg@10:0.88397\n", + "[49]\tvalidation_0-ndcg@10:0.88404\n", + "[50]\tvalidation_0-ndcg@10:0.88404\n", + "[51]\tvalidation_0-ndcg@10:0.88443\n", + "[52]\tvalidation_0-ndcg@10:0.88433\n", + "[53]\tvalidation_0-ndcg@10:0.88464\n", + "[54]\tvalidation_0-ndcg@10:0.88466\n", + "[55]\tvalidation_0-ndcg@10:0.88450\n", + "[56]\tvalidation_0-ndcg@10:0.88476\n", + "[57]\tvalidation_0-ndcg@10:0.88489\n", + "[58]\tvalidation_0-ndcg@10:0.88477\n", + "[59]\tvalidation_0-ndcg@10:0.88486\n", + "[60]\tvalidation_0-ndcg@10:0.88483\n", + "[61]\tvalidation_0-ndcg@10:0.88518\n", + "[62]\tvalidation_0-ndcg@10:0.88529\n", + "[63]\tvalidation_0-ndcg@10:0.88519\n", + "[64]\tvalidation_0-ndcg@10:0.88538\n", + "[65]\tvalidation_0-ndcg@10:0.88544\n", + "[66]\tvalidation_0-ndcg@10:0.88559\n", + "[67]\tvalidation_0-ndcg@10:0.88546\n", + "[68]\tvalidation_0-ndcg@10:0.88557\n", + "[69]\tvalidation_0-ndcg@10:0.88560\n", + "[70]\tvalidation_0-ndcg@10:0.88590\n", + "[71]\tvalidation_0-ndcg@10:0.88592\n", + "[72]\tvalidation_0-ndcg@10:0.88600\n", + "[73]\tvalidation_0-ndcg@10:0.88605\n", + "[74]\tvalidation_0-ndcg@10:0.88602\n", + "[75]\tvalidation_0-ndcg@10:0.88629\n", + "[76]\tvalidation_0-ndcg@10:0.88635\n", + "[77]\tvalidation_0-ndcg@10:0.88624\n", + "[78]\tvalidation_0-ndcg@10:0.88620\n", + "[79]\tvalidation_0-ndcg@10:0.88638\n", + "[80]\tvalidation_0-ndcg@10:0.88658\n", + "[81]\tvalidation_0-ndcg@10:0.88674\n", + "[82]\tvalidation_0-ndcg@10:0.88673\n", + "[83]\tvalidation_0-ndcg@10:0.88677\n", + "[84]\tvalidation_0-ndcg@10:0.88671\n", + "[85]\tvalidation_0-ndcg@10:0.88682\n", + "[86]\tvalidation_0-ndcg@10:0.88693\n", + "[87]\tvalidation_0-ndcg@10:0.88694\n", + "[88]\tvalidation_0-ndcg@10:0.88682\n", + "[89]\tvalidation_0-ndcg@10:0.88687\n", + "[90]\tvalidation_0-ndcg@10:0.88700\n", + "[91]\tvalidation_0-ndcg@10:0.88701\n", + "[92]\tvalidation_0-ndcg@10:0.88705\n", + "[93]\tvalidation_0-ndcg@10:0.88705\n", + "[94]\tvalidation_0-ndcg@10:0.88719\n", + "[95]\tvalidation_0-ndcg@10:0.88720\n", + "[96]\tvalidation_0-ndcg@10:0.88716\n", + "[97]\tvalidation_0-ndcg@10:0.88717\n", + "[98]\tvalidation_0-ndcg@10:0.88707\n", + "[99]\tvalidation_0-ndcg@10:0.88706\n", + "[100]\tvalidation_0-ndcg@10:0.88715\n", + "[101]\tvalidation_0-ndcg@10:0.88731\n", + "[102]\tvalidation_0-ndcg@10:0.88724\n", + "[103]\tvalidation_0-ndcg@10:0.88732\n", + "[104]\tvalidation_0-ndcg@10:0.88738\n", + "[105]\tvalidation_0-ndcg@10:0.88726\n", + "[106]\tvalidation_0-ndcg@10:0.88739\n", + "[107]\tvalidation_0-ndcg@10:0.88728\n", + "[108]\tvalidation_0-ndcg@10:0.88752\n", + "[109]\tvalidation_0-ndcg@10:0.88749\n", + "[110]\tvalidation_0-ndcg@10:0.88766\n", + "[111]\tvalidation_0-ndcg@10:0.88776\n", + "[112]\tvalidation_0-ndcg@10:0.88784\n", + "[113]\tvalidation_0-ndcg@10:0.88774\n", + "[114]\tvalidation_0-ndcg@10:0.88786\n", + "[115]\tvalidation_0-ndcg@10:0.88793\n", + "[116]\tvalidation_0-ndcg@10:0.88813\n", + "[117]\tvalidation_0-ndcg@10:0.88802\n", + "[118]\tvalidation_0-ndcg@10:0.88801\n", + "[119]\tvalidation_0-ndcg@10:0.88804\n", + "[120]\tvalidation_0-ndcg@10:0.88811\n", + "[121]\tvalidation_0-ndcg@10:0.88806\n", + "[122]\tvalidation_0-ndcg@10:0.88803\n", + "[123]\tvalidation_0-ndcg@10:0.88816\n", + "[124]\tvalidation_0-ndcg@10:0.88814\n", + "[125]\tvalidation_0-ndcg@10:0.88824\n", + "[126]\tvalidation_0-ndcg@10:0.88836\n", + "[127]\tvalidation_0-ndcg@10:0.88834\n", + "[128]\tvalidation_0-ndcg@10:0.88835\n", + "[129]\tvalidation_0-ndcg@10:0.88835\n", + "[130]\tvalidation_0-ndcg@10:0.88846\n", + "[131]\tvalidation_0-ndcg@10:0.88850\n", + "[132]\tvalidation_0-ndcg@10:0.88849\n", + "[133]\tvalidation_0-ndcg@10:0.88870\n", + "[134]\tvalidation_0-ndcg@10:0.88861\n", + "[135]\tvalidation_0-ndcg@10:0.88867\n", + "[136]\tvalidation_0-ndcg@10:0.88886\n", + "[137]\tvalidation_0-ndcg@10:0.88898\n", + "[138]\tvalidation_0-ndcg@10:0.88892\n", + "[139]\tvalidation_0-ndcg@10:0.88894\n", + "[140]\tvalidation_0-ndcg@10:0.88882\n", + "[141]\tvalidation_0-ndcg@10:0.88875\n", + "[142]\tvalidation_0-ndcg@10:0.88877\n", + "[143]\tvalidation_0-ndcg@10:0.88879\n", + "[144]\tvalidation_0-ndcg@10:0.88875\n", + "[145]\tvalidation_0-ndcg@10:0.88875\n", + "[146]\tvalidation_0-ndcg@10:0.88875\n", + "[147]\tvalidation_0-ndcg@10:0.88875\n", + "[148]\tvalidation_0-ndcg@10:0.88878\n", + "[149]\tvalidation_0-ndcg@10:0.88892\n", + "[150]\tvalidation_0-ndcg@10:0.88890\n", + "[151]\tvalidation_0-ndcg@10:0.88885\n", + "[152]\tvalidation_0-ndcg@10:0.88887\n", + "[153]\tvalidation_0-ndcg@10:0.88893\n", + "[154]\tvalidation_0-ndcg@10:0.88887\n", + "[155]\tvalidation_0-ndcg@10:0.88888\n", + "[156]\tvalidation_0-ndcg@10:0.88889\n" + ] }, { - "cell_type": "code", - "execution_count": 8, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 424 - }, - "id": "XLjiKfYQqM-U", - "outputId": "38df2283-421f-43ea-8bdf-580f1a63ac0d" - }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
query_idquerydoc_idgrade
0qid:5141insidious 2 netflix8464330
1qid:5141insidious 2 netflix490181
2qid:5141insidious 2 netflix382340
3qid:5141insidious 2 netflix5676040
4qid:5141insidious 2 netflix2697950
...............
384750qid:33832013 the wolverine2631150
384751qid:33832013 the wolverine259130
384752qid:33832013 the wolverine5676040
384753qid:33832013 the wolverine5335350
384754qid:33832013 the wolverine8763270
\n", - "

384755 rows × 4 columns

\n", - "
" - ], - "text/plain": [ - " query_id query doc_id grade\n", - "0 qid:5141 insidious 2 netflix 846433 0\n", - "1 qid:5141 insidious 2 netflix 49018 1\n", - "2 qid:5141 insidious 2 netflix 38234 0\n", - "3 qid:5141 insidious 2 netflix 567604 0\n", - "4 qid:5141 insidious 2 netflix 269795 0\n", - "... ... ... ... ...\n", - "384750 qid:3383 2013 the wolverine 263115 0\n", - "384751 qid:3383 2013 the wolverine 25913 0\n", - "384752 qid:3383 2013 the wolverine 567604 0\n", - "384753 qid:3383 2013 the wolverine 533535 0\n", - "384754 qid:3383 2013 the wolverine 876327 0\n", - "\n", - "[384755 rows x 4 columns]" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } + "data": { + "text/html": [ + "
XGBRanker(base_score=None, booster=None, callbacks=None, colsample_bylevel=None,\n",
+       "          colsample_bynode=None, colsample_bytree=None, device=None,\n",
+       "          early_stopping_rounds=20, enable_categorical=False,\n",
+       "          eval_metric=['ndcg@10'], feature_types=None, gamma=None,\n",
+       "          grow_policy=None, importance_type=None, interaction_constraints=None,\n",
+       "          learning_rate=None, max_bin=None, max_cat_threshold=None,\n",
+       "          max_cat_to_onehot=None, max_delta_step=None, max_depth=None,\n",
+       "          max_leaves=None, min_child_weight=None, missing=nan,\n",
+       "          monotone_constraints=None, multi_strategy=None, n_estimators=200,\n",
+       "          n_jobs=None, num_parallel_tree=None, random_state=None, ...)
In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.
" ], - "source": [ - "judgments_df = pd.read_csv(JUDGEMENTS_FILE_URL, delimiter=\"\\t\")\n", - "judgments_df" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Configure feature extraction\n", - "\n", - "Features are the input data of our model. They represent the document in the context of the query.\n", - "Features are configured using templated queries to extract features.\n", - "\n", - "To define features extraction, you will be using the primitives provided by the Eland API:" + "text/plain": [ + "XGBRanker(base_score=None, booster=None, callbacks=None, colsample_bylevel=None,\n", + " colsample_bynode=None, colsample_bytree=None, device=None,\n", + " early_stopping_rounds=20, enable_categorical=False,\n", + " eval_metric=['ndcg@10'], feature_types=None, gamma=None,\n", + " grow_policy=None, importance_type=None, interaction_constraints=None,\n", + " learning_rate=None, max_bin=None, max_cat_threshold=None,\n", + " max_cat_to_onehot=None, max_delta_step=None, max_depth=None,\n", + " max_leaves=None, min_child_weight=None, missing=nan,\n", + " monotone_constraints=None, multi_strategy=None, n_estimators=200,\n", + " n_jobs=None, num_parallel_tree=None, random_state=None, ...)" ] + }, + "execution_count": 125, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from xgboost import XGBRanker\n", + "from sklearn.model_selection import GroupShuffleSplit\n", + "\n", + "\n", + "# Create the ranker model:\n", + "ranker = XGBRanker(\n", + " objective=\"rank:ndcg\",\n", + " eval_metric=[\"ndcg@10\"],\n", + " early_stopping_rounds=20,\n", + " n_estimators=200,\n", + ")\n", + "\n", + "# Shaping training and eval data in the expected format.\n", + "X = judgments_with_features[ltr_config.feature_names]\n", + "y = judgments_with_features[\"grade\"]\n", + "groups = judgments_with_features[\"query_id\"]\n", + "\n", + "# Split the dataset in two parts respectively used for training and evaluation of the model.\n", + "group_preserving_splitter = GroupShuffleSplit(n_splits=1, train_size=0.7).split(X, y, groups)\n", + "train_idx, eval_idx = next(group_preserving_splitter)\n", + "train_features, eval_features = X.loc[train_idx], X.loc[eval_idx]\n", + "\n", + "train_target, eval_target = y.loc[train_idx], y.loc[eval_idx]\n", + "train_query_groups, eval_query_groups = groups.loc[train_idx], groups.loc[eval_idx]\n", + "\n", + "# Training the model\n", + "ranker.fit(\n", + " X=train_features,\n", + " y=train_target,\n", + " group=train_query_groups.value_counts().sort_index().values,\n", + " eval_set=[(eval_features, eval_target)],\n", + " eval_group=[eval_query_groups.value_counts().sort_index().values],\n", + " verbose=True,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 126, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 490 }, + "id": "3iSx3IuLqq7R", + "outputId": "d81ac47f-99c6-4656-9fc1-b9699a80c458" + }, + "outputs": [ { - "cell_type": "code", - "execution_count": 9, - "metadata": { - "id": "LjxAj4lQqEYJ" - }, - "outputs": [], - "source": [ - "from eland.ml.ltr import LTRModelConfig, QueryFeatureExtractor\n", - "\n", - "ltr_config = LTRModelConfig(\n", - " feature_extractors = [\n", - " # For the following field we want to use the score of the match query for the field as a features:\n", - " QueryFeatureExtractor(\n", - " feature_name=\"title_bm25\",\n", - " query={ \"match\": { \"title\": \"{{query}}\" } }\n", - " ),\n", - " QueryFeatureExtractor(\n", - " feature_name=\"actors_bm25\",\n", - " query={ \"match\": { \"actors\": \"{{query}}\" } }\n", - " ),\n", - " # We could also use a more strict matching clause as an additional features. Here we want all the terms of our query to match.\n", - " QueryFeatureExtractor(\n", - " feature_name=\"title_all_terms_bm25\",\n", - " query={ \"match\": { \"title\": { \"query\": \"{{query}}\", \"minimum_should_match\": \"100%\" } } }\n", - " ),\n", - " # Also we can use a script_score query to get the document field values directly as a feature.\n", - " QueryFeatureExtractor(\n", - " feature_name=\"popularity\",\n", - " query={\n", - " \"script_score\": {\n", - " \"query\": { \"exists\": { \"field\": \"popularity\" } },\n", - " \"script\": { \"source\": \"return doc['popularity'].value;\" }\n", - " }\n", - " }\n", - " )\n", - " ]\n", - ")" + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAsUAAAHHCAYAAABX6yWOAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8g+/7EAAAACXBIWXMAAA9hAAAPYQGoP6dpAABnkUlEQVR4nO3dd1gU1/s28HspS+9SFQEFFRRsqAFrBAGxYQm2JGDUxN4xdkETIVaM+jVRE4ixx9iSoBELdjEWjJXYEI0QG0VAaTvvH77MzxVQFlHAuT/XtZfszJmZZw6L3MyeOSsTBEEAEREREZGEqVV2AURERERElY2hmIiIiIgkj6GYiIiIiCSPoZiIiIiIJI+hmIiIiIgkj6GYiIiIiCSPoZiIiIiIJI+hmIiIiIgkj6GYiIiIiCSPoZiIiKq96OhoyGQyJCUlVXYpRFRNMRQTEVVDRSGwpMeUKVPeyjGPHz+O0NBQpKenv5X9S1lOTg5CQ0MRFxdX2aUQSZZGZRdARETlN2fOHDg4OCgta9So0Vs51vHjxxEWFobg4GAYGxu/lWOU1yeffIJ+/fpBS0urskspl5ycHISFhQEAOnToULnFEEkUQzERUTXWuXNnuLu7V3YZbyQ7Oxt6enpvtA91dXWoq6tXUEXvjkKhQF5eXmWXQUTg8Akiovfa7t270bZtW+jp6cHAwABdunTBpUuXlNr8/fffCA4ORp06daCtrQ0rKyt89tlnePTokdgmNDQUISEhAAAHBwdxqEZSUhKSkpIgk8kQHR1d7PgymQyhoaFK+5HJZLh8+TIGDBgAExMTtGnTRly/bt06NG/eHDo6OjA1NUW/fv1w586d155nSWOK7e3t0bVrV8TFxcHd3R06OjpwdXUVhyhs27YNrq6u0NbWRvPmzXHu3DmlfQYHB0NfXx83b96Er68v9PT0YGNjgzlz5kAQBKW22dnZmDhxImxtbaGlpYX69etj4cKFxdrJZDKMGjUK69evR8OGDaGlpYXvvvsO5ubmAICwsDCxb4v6rSzfnxf79vr16+LVfCMjIwwaNAg5OTnF+mzdunVo2bIldHV1YWJignbt2mHv3r1Kbcry+iF6X/BKMRFRNZaRkYGHDx8qLatRowYA4Oeff0ZQUBB8fX3xzTffICcnBytXrkSbNm1w7tw52NvbAwBiY2Nx8+ZNDBo0CFZWVrh06RJWrVqFS5cu4eTJk5DJZOjVqxf++ecfbNy4EUuWLBGPYW5ujgcPHqhc90cffQQnJyfMmzdPDI5ff/01Zs6cicDAQAwZMgQPHjzAsmXL0K5dO5w7d65cQzauX7+OAQMG4IsvvsDHH3+MhQsXolu3bvjuu+8wbdo0jBgxAgAQHh6OwMBAJCYmQk3t/64XFRYWws/PDx988AHmz5+PPXv2YPbs2SgoKMCcOXMAAIIgoHv37jh48CAGDx6MJk2a4M8//0RISAj+/fdfLFmyRKmmAwcOYMuWLRg1ahRq1KiBxo0bY+XKlRg+fDh69uyJXr16AQDc3NwAlO3786LAwEA4ODggPDwcZ8+exZo1a2BhYYFvvvlGbBMWFobQ0FB4enpizpw5kMvliI+Px4EDB+Dj4wOg7K8foveGQERE1U5UVJQAoMSHIAjCkydPBGNjY2Ho0KFK26WmpgpGRkZKy3Nycortf+PGjQIA4fDhw+KyBQsWCACEW7duKbW9deuWAECIiooqth8AwuzZs8Xns2fPFgAI/fv3V2qXlJQkqKurC19//bXS8gsXLggaGhrFlpfWHy/WZmdnJwAQjh8/Li77888/BQCCjo6OcPv2bXH5999/LwAQDh48KC4LCgoSAAijR48WlykUCqFLly6CXC4XHjx4IAiCIOzYsUMAIHz11VdKNfXp00eQyWTC9evXlfpDTU1NuHTpklLbBw8eFOurImX9/hT17WeffabUtmfPnoKZmZn4/Nq1a4KamprQs2dPobCwUKmtQqEQBEG11w/R+4LDJ4iIqrEVK1YgNjZW6QE8v7qYnp6O/v374+HDh+JDXV0drVq1wsGDB8V96OjoiF8/e/YMDx8+xAcffAAAOHv27Fupe9iwYUrPt23bBoVCgcDAQKV6rays4OTkpFSvKlxcXODh4SE+b9WqFQCgY8eOqF27drHlN2/eLLaPUaNGiV8XDX/Iy8vDvn37AAAxMTFQV1fHmDFjlLabOHEiBEHA7t27lZa3b98eLi4uZT4HVb8/L/dt27Zt8ejRI2RmZgIAduzYAYVCgVmzZildFS86P0C11w/R+4LDJ4iIqrGWLVuWeKPdtWvXADwPfyUxNDQUv378+DHCwsKwadMm3L9/X6ldRkZGBVb7f16eMePatWsQBAFOTk4lttfU1CzXcV4MvgBgZGQEALC1tS1xeVpamtJyNTU11KlTR2lZvXr1AEAcv3z79m3Y2NjAwMBAqZ2zs7O4/kUvn/vrqPr9efmcTUxMADw/N0NDQ9y4cQNqamqvDOaqvH6I3hcMxURE7yGFQgHg+bhQKyurYus1NP7vv//AwEAcP34cISEhaNKkCfT19aFQKODn5yfu51VeHtNapLCwsNRtXrz6WVSvTCbD7t27S5xFQl9f/7V1lKS0GSlKWy68dGPc2/Dyub+Oqt+fijg3VV4/RO8LvqqJiN5DdevWBQBYWFjA29u71HZpaWnYv38/wsLCMGvWLHF50ZXCF5UWfouuRL78oR4vXyF9Xb2CIMDBwUG8ElsVKBQK3Lx5U6mmf/75BwDEG83s7Oywb98+PHnyROlq8dWrV8X1r1Na36ry/SmrunXrQqFQ4PLly2jSpEmpbYDXv36I3iccU0xE9B7y9fWFoaEh5s2bh/z8/GLri2aMKLqq+PJVxMjIyGLbFM0l/HL4NTQ0RI0aNXD48GGl5f/73//KXG+vXr2grq6OsLCwYrUIglBs+rF3afny5Uq1LF++HJqamvDy8gIA+Pv7o7CwUKkdACxZsgQymQydO3d+7TF0dXUBFO9bVb4/ZRUQEAA1NTXMmTOn2JXmouOU9fVD9D7hlWIioveQoaEhVq5ciU8++QTNmjVDv379YG5ujuTkZPzxxx9o3bo1li9fDkNDQ7Rr1w7z589Hfn4+atasib179+LWrVvF9tm8eXMAwPTp09GvXz9oamqiW7du0NPTw5AhQxAREYEhQ4bA3d0dhw8fFq+olkXdunXx1VdfYerUqUhKSkJAQAAMDAxw69YtbN++HZ9//jkmTZpUYf1TVtra2tizZw+CgoLQqlUr7N69G3/88QemTZsmzi3crVs3fPjhh5g+fTqSkpLQuHFj7N27Fzt37sS4cePEq66voqOjAxcXF2zevBn16tWDqakpGjVqhEaNGpX5+1NWjo6OmD59OubOnYu2bduiV69e0NLSwl9//QUbGxuEh4eX+fVD9F6ppFkviIjoDRRNQfbXX3+9st3BgwcFX19fwcjISNDW1hbq1q0rBAcHC6dPnxbb3L17V+jZs6dgbGwsGBkZCR999JFw7969EqcImzt3rlCzZk1BTU1NaQq0nJwcYfDgwYKRkZFgYGAgBAYGCvfv3y91Srai6cxe9uuvvwpt2rQR9PT0BD09PaFBgwbCyJEjhcTExDL1x8tTsnXp0qVYWwDCyJEjlZYVTSu3YMECcVlQUJCgp6cn3LhxQ/Dx8RF0dXUFS0tLYfbs2cWmMnvy5Ikwfvx4wcbGRtDU1BScnJyEBQsWiFOcverYRY4fPy40b95ckMvlSv1W1u9PaX1bUt8IgiD8+OOPQtOmTQUtLS3BxMREaN++vRAbG6vUpiyvH6L3hUwQ3sFdBURERNVMcHAwtm7diqysrMouhYjeAY4pJiIiIiLJYygmIiIiIsljKCYiIiIiyeOYYiIiIiKSPF4pJiIiIiLJYygmIiIiIsnjh3cQlUKhUODevXswMDAo9SNYiYiIqGoRBAFPnjyBjY0N1NTKfv2XoZioFPfu3YOtrW1ll0FERETlcOfOHdSqVavM7RmKiUphYGAAALh16xZMTU0ruZqqLz8/H3v37oWPjw80NTUru5xqgX2mGvaXathfqmOfqaaq9ldmZiZsbW3F3+NlxVBMVIqiIRMGBgYwNDSs5Gqqvvz8fOjq6sLQ0LBK/edYlbHPVMP+Ug37S3XsM9VU9f5Sdegjb7QjIiIiIsljKCYiIiIiyWMoJiIiIiLJYygmIiIiIsljKCYiIiIiyWMoJiIiIiLJYygmIiIiIsljKCYiIiIiyWMoJiIiIiLJYygmIiIiIsljKCYiIiIiyWMoJiIiIiLJYygmIiIiIsljKCYiIiIiyWMoJiIiIiLJYygmIiIiIsljKCYiIiIiyWMoJiIiIiLJYygmIiIiIsljKCYiIiIiyWMoJiIiIiLJYygmIiIiIsljKCYiIiIiyWMoJiIiIiLJYygmIiIiIsljKCYiIiIiyWMoJiIiIiLJYygmIiIiIsljKCYiIiIiyWMoJiIiIiLJYygmIiIiIsljKCYiIiIiyWMoJiIiIiLJYygmIiIiIsljKCYiIiIiyWMoJiIiIiLJYygmIiIiIsljKCYiIiIiyWMoJiIiIiLJYygmIiIiIsljKCYiIiIiyWMoJiIiIiLJYygmIiIiIsljKCYiIiIiyWMoJiIiIiLJYygmIiIiIsljKCYiIiIiyWMoJiIiIiLJYygmIiIiIsljKCYiIiIiyWMoJiIiIiLJYygmIiIiIsljKCYiIiIiydOo7AJIGjp06IAmTZogMjLyjfYTGhqKHTt2ICEhoULqKotW4ftRoKH3zo5XXWmpC5jfEmgU+idyC2WVXU61wD5TDftLNewv1bHPSpYU0QXh4eHYtm0brl69Ch0dHXh6euKrr75Sanfjxg1MmjQJR48eRW5uLvz8/LBs2TJYWlqKbezt7XH79m2l7cLDwzFlyhQAQFxcHJYsWYJTp04hMzMTTk5OCAkJwcCBA19ZY3JyMoYPH46DBw9CT+/57+yCggKVzpNXiqlamTRpEvbv3y8+Dw4ORkBAQOUVREREJAGHDh3CyJEjcfLkScTGxiI/Px9dunTBs2fPAADZ2dnw8fGBTCbDgQMHcOzYMeTl5aFbt25QKBRK+5ozZw5SUlLEx+jRo8V1x48fh5ubG3799Vf8/fffGDRoED799FP8/vvvpdZWWFiILl26IC8vD8ePH8d3330HAPj6669VOkdeKaZqQRAEFBYWQl9fH/r6+pVdDhERkaTs2bNH6Xl0dDQsLCxw48YNAMCxY8eQlJSEc+fOwdDQEADw008/wcTEBAcOHIC3t7e4rYGBAaysrEo8zrRp05Sejx07Fnv37sW2bdvQtWvXErfZu3cvLl++jH379sHS0hJ16tQBAKxZswbh4eGQy+VlOkdeKX7PdejQAaNGjcKoUaNgZGSEGjVqYObMmRAEAQCQlpaGTz/9FCYmJtDV1UXnzp1x7do1cfvo6GgYGxtjx44dcHJygra2Nnx9fXHnzh2xTUlXa8eNG4cOHTqUWtfPP/8Md3d38QdjwIABuH//vrg+Li4OMpkMu3fvRvPmzaGlpYWjR48iNDQUTZo0AfB8KMVPP/2EnTt3QiaTQSaTIS4uDh07dsSoUaOUjvfgwQPI5XKlq8xERERUPhkZGQAgXqjKzc2FTCaDlpaW2EZbWxtqamo4evSo0rYREREwMzND06ZNsWDBgtcOc8jIyICpqWmp60+cOAFXV1elYRoAkJmZiUuXLpX5nBiKJeCnn36ChoYGTp06haVLl2Lx4sVYs2YNgOeB9vTp09i1axdOnDgBQRDg7++P/Px8cfucnBx8/fXXWLt2LY4dO4b09HT069fvjWrKz8/H3Llzcf78eezYsQNJSUkIDg4u1m7KlCmIiIjAlStX4ObmprRu0qRJCAwMhJ+fn/gWjKenJ4YMGYINGzYgNzdXbLtu3TrUrFkTHTt2fKO6iYiIpE6hUGDcuHHw9PSEnZ0dAOCDDz6Anp4evvzyS+Tk5CA7OxuTJk1CYWEhUlJSxG3HjBmDTZs24eDBg/jiiy8wb948TJ48udRjbdmyBX/99RcGDRpUapvU1NRigfjFdWXF4RMSYGtriyVLlkAmk6F+/fq4cOEClixZgg4dOmDXrl04duwYPD09AQDr16+Hra0tduzYgY8++gjA8wC7fPlytGrVCsDzkO3s7IxTp06hZcuW5arps88+E7+uU6cOvv32W7Ro0QJZWVlKwyPmzJmDTp06lbgPfX196OjoIDc3V+ltmF69emHUqFHYuXMnAgMDATy/4h0cHAyZrPQbJ3Jzc5WCdGZmJgBAS02AurpQrvOUEi01Qelfej32mWrYX6phf6mOfVayFy+UAcCoUaNw8eJFxMbG4vLly8jPz4exsTE2btyI0aNH49tvv4Wamhr69u2Lpk2bKu3jxfHDzs7OUFdXx4gRIzBnzhylq8zA83eNBw0ahJUrV6JevXrF6iiiUCggCIK4vrR2r8NQLAEffPCBUhj08PDAokWLcPnyZWhoaIhhFwDMzMxQv359XLlyRVymoaGBFi1aiM8bNGgAY2NjXLlypdyh+MyZMwgNDcX58+eRlpYmDsJPTk6Gi4uL2M7d3V3lfWtra+OTTz7Bjz/+iMDAQJw9exYXL17Erl27XrldeHg4wsLCii2f0VQBXd1CleuQqrnuitc3IiXsM9Wwv1TD/lId+0xZTEyM+PWqVasQHx+PefPm4fLlywCA2NhYcf3ixYuRmZkJNTU16OvrIzg4GG5ubkr7eNGzZ89QUFCAtWvXombNmuLyixcv4quvvsKgQYNgZmZW6vYA8OTJE1y7dk1sk5OTI64rbexySRiK6Y2pqamJY5SLvOqvtOzsbPj6+sLX1xfr16+Hubk5kpOT4evri7y8PKW2RdOqqGrIkCFo0qQJ7t69i6ioKHTs2FF8i6c0U6dOxYQJE8TnmZmZsLW1xVfn1FCgqV6uOqRES03AXHcFZp5WQ66CUxmVBftMNewv1bC/VMc+K9nFUF8IgoBx48YhISEBhw8fhpOTE/Lz8xEbG4tOnTpBU1Oz2HYHDx5ERkYGJk2ahPr165e47w0bNkBNTQ19+vSBiYkJgOczXYSHh+Obb77B8OHDX1ufmpoatm7dCnd3d1hYWIjv9BoaGipdaHsdhmIJiI+PV3p+8uRJODk5wcXFBQUFBYiPjxeHTzx69AiJiYlKL6KCggKcPn1avCqcmJiI9PR0ODs7AwDMzc1x8eJFpWMkJCSU+AMCAFevXsWjR48QEREBW1tbAMDp06fLdW5yuRyFhcWv4rq6usLd3R2rV6/Ghg0bsHz58tfuS0tLq9hbNwCQq5ChgPNVllmuQsb5PVXEPlMN+0s17C/Vsc+UaWpqYsSIEdiwYQN27twJU1NTPHr0CPn5+cjNzYWmpiY0NTURFRUFZ2dnmJub48SJExg7dizGjx+PRo0aAXh+Q1x8fDw+/PBDGBgY4MSJEwgJCcHHH38MCwsLAM+DdI8ePTB27FgEBgbi0aNHAJ7/vi+62W779u2YOnUqrl69CgDw9/eHi4sLPvvsM8yfP1+cEWPIkCEl/l4vDW+0k4Dk5GRMmDABiYmJ2LhxI5YtW4axY8fCyckJPXr0wNChQ3H06FGcP38eH3/8MWrWrIkePXqI22tqamL06NGIj4/HmTNnEBwcjA8++EAMyR07dsTp06exdu1aXLt2DbNnzy4Wkl9Uu3ZtyOVyLFu2DDdv3sSuXbswd+7ccp2bvb09/v77byQmJuLhw4dKV6iHDBmCiIgICIKAnj17lmv/REREBKxcuRIZGRno0KEDrK2tYW1tjdq1ayvNLJGYmIiAgAA4Oztjzpw5mD59OhYuXCiu19LSwqZNm9C+fXs0bNgQX3/9NcaPH49Vq1aJbX766Sfk5OQgPDxcPI61tTV69eoltsnIyEBiYqL4XF1dHb///jvU1dXh4eGBzz//HAAwffp0lc6RoVgCPv30Uzx9+hQtW7bEyJEjMXbsWPEFExUVhebNm6Nr167w8PCAIAiIiYlRusqrq6uLL7/8EgMGDEDr1q2hr6+PzZs3i+t9fX0xc+ZMTJ48GS1atMCTJ0/w6aefllqPubk5oqOj8csvv8DFxQURERFKPzSqGDp0KOrXrw93d3eYm5vj2LFj4rr+/ftDQ0MD/fv3h7a2drn2T0RERM8/L+DlR15eHry8vMQ2ERERSE1NRV5eHv755x9MmDBB6Z6mZs2a4eTJk0hPT8fTp09x+fJlTJ06VelqbnR0dInHiouLE9sEBwcXG7ZpZ2eHmJgY5OTk4ObNmwCe3xOlCg6fkABNTU1ERkZi5cqVxdaZmJhg7dq1r91Hr169lP5Ke1lYWFiJN6kVefHFDDwPrP3791da9uILvEOHDsVe8MDzuYlDQ0PF5+bm5ti7d2+Jx3z48CGePXuGwYMHl1oXEREREcBQTO+h/Px8PHr0CDNmzMAHH3yAZs2avdH+4qd6wczMrIKqe3/l5+cjJiYGF0N9Sx1PTsrYZ6phf6mG/aU69pm0cfgEvXeOHTsGa2tr/PXXX+LnnxMRERG9Cq8Uv+deHragquDg4BI/aa4qK23oBREREVFpeKWYiIiIiCSPoZiIiIiIJI+hmIiIiIgkj6GYiIiIiCSPoZiIiIiIJI+hmIiIiIgkj6GYiIiIiCSPoZiIiIiIJI+hmIiIiIgkj6GYiIiIiCSPoZiIiIiIJI+hmIiIiIgkj6GYiIiIiCSPoZiIiIiIJI+hmIiIiIgkj6GYiIiIiCSPoZiIiIiIJI+hmIiIiIgkj6GYiIiIiCSPoZiIiIiIJI+hmIiIiIgkj6GYiIiIiCSPoZiIiIiIJI+hmIiIiIgkj6GYiIiIiCSPoZiIiIiIJI+hmIiIiIgkj6GYiIiIiCSPoZiIiIiIJI+hmIiIiIgkj6GYiIiIiCSPoZiIiIiIJI+hmIiIiIgkj6GYiIiIiCSPoZiIiIiIJI+hmIiIiIgkj6GYiIiIiCSPoZiIiIiIJI+hmIiIiIgkj6GYiIiIiCSPoZiIiIiIJI+hmIiIiIgkj6GYiIiIiCSPoZiIiIiIJI+hmIiIiIgkj6GYiIiIiCRPo7ILIKrqWoXvR4GGXmWXUeVpqQuY3xJoFPoncgtllV1OtcA+Uw37SzXVvb+SIrpUdgkkMbxSLAFxcXGQyWRIT09/ZTt7e3tERkZWyDFDQ0PRpEmTCtkXERFJU3h4OFq0aAEDAwNYWFggICAAiYmJSm2++OIL1K1bFzo6OjA3N0ePHj1w9epVpTbJycno0qULdHV1YWFhgZCQEBQUFIjrg4ODIZPJIJfLERAQALlcDplMhoYNG76yvr///htt27aFtrY2bG1tMX/+/Io7eXrnGIrfQx06dMC4cePE556enkhJSYGRkREAIDo6GsbGxpVT3BtKSkrC4MGD4eDgAB0dHdStWxezZ89GXl6eUhuZTFbscfLkyUqsnIiIVHXo0CGMHDkSJ0+eRGxsLPLz8+Hj44Ps7GyxTfPmzREVFYUrV67gzz//hCAI8PHxQWFhIQCgsLAQXbp0QV5eHo4fP46ffvoJ0dHRmDVrlriPpUuXIiUlBcnJyYiKisLNmzdhamqKjz76qNTaMjMz4ePjAzs7O5w5cwYLFixAaGgoVq1a9fY6hN4qDp+QALlcDisrq8ouo0JcvXoVCoUC33//PRwdHXHx4kUMHToU2dnZWLhwoVLbffv2Kf2Vb2Zm9q7LJSKiN7Bnzx6l59HR0bCwsMCZM2fQrl07AMDnn38urre3t8dXX32Fxo0bIykpCXXr1sXevXtx+fJl7Nu3D5aWlmjSpAnmzp2LL7/8EqGhoZDL5TAyMoKRkRHy8/NhYmKCM2fOIC0tDYMGDSq1tvXr1yMvLw8//vgj5HI5GjZsiISEBCxevFipJqo+eKX4PRMcHIxDhw5h6dKl4hXS6OhocfhEXFwcBg0ahIyMDHF9aGhoiftKT0/HkCFDYG5uDkNDQ3Ts2BHnz59XqZ7vv/8etra20NXVRWBgIDIyMpRqDQgIwLx582BpaQljY2PMmTMHBQUFCAkJgampKWrVqoWoqChxGz8/P0RFRcHHxwd16tRB9+7dMWnSJGzbtq3Ysc3MzGBlZSU+NDU1VaqdiIiqlqLfIaampiWuz87ORlRUFBwcHGBrawsAOHHiBFxdXWFpaSm28/X1RWZmJi5dulTifqKiouDt7Q07O7tSazlx4gTatWsHuVyutN/ExESkpaWpfG5U+Xil+D2zdOlS/PPPP2jUqBHmzJkDAEo/9J6enoiMjMSsWbPEcVn6+vol7uujjz6Cjo4Odu/eDSMjI3z//ffw8vLCP//8U+p/SC+6fv06tmzZgt9++w2ZmZkYPHgwRowYgfXr14ttDhw4gFq1auHw4cM4duwYBg8ejOPHj6Ndu3aIj4/H5s2b8cUXX6BTp06oVatWicfJyMgosZ7u3bvj2bNnqFevHiZPnozu3bu/st7c3Fzk5uaKzzMzMwEAWmoC1NWF156v1GmpCUr/0uuxz1TD/lJNde+v/Px8pecKhQJjx46Fp6cn6tevr7T+u+++w9SpU5GdnY169eohJiYGMpkM+fn5uHfvHiwsLJTaF/3OuHv3Lho1aqR0zMePH+PPP//E2rVri9XwopSUFNjb25e43zt37pT6u/V9UnTur+qnylDeehiK3zNGRkaQy+XQ1dUVh0y8eMNB0dtEMpnslUMqjh49ilOnTuH+/fvQ0tICACxcuBA7duzA1q1by/TW0LNnz7B27VrUrFkTALBs2TJ06dIFixYtEo9tamqKb7/9Fmpqaqhfvz7mz5+PnJwcTJs2DQAwdepURERE4OjRo+jXr1+xY1y/fh3Lli1TGjqhr6+PRYsWoXXr1lBTU8Ovv/6KgIAA7Nix45XBODw8HGFhYcWWz2iqgK5u4WvPl56b666o7BKqHfaZathfqqmu/RUTE6P0/LvvvsOZM2cQHh5ebJ2ZmRkWLFiAtLQ07NixA126dEFERATkcjmSk5Px4MEDpW2KLoD89ddfUCiU++fAgQPQ1dWFXC4vdpwXPXjwAGpqakpt7ty5AwA4fPgwbt26Vb4Tr4ZiY2MruwQlOTk55dqOoZhKdP78eWRlZRUbh/v06VPcuHGjTPuoXbu2GIgBwMPDAwqFAomJiWIobtiwIdTU/m8Uj6WlpdJf7erq6jAzM8P9+/eL7f/ff/+Fn58fPvroIwwdOlRcXqNGDUyYMEF83qJFC9y7dw8LFix4ZSieOnWq0naZmZmwtbXFV+fUUKCpXqZzljItNQFz3RWYeVoNuYrqN/1TZWCfqYb9pZrq3l8XQ33Fr8eOHYuLFy/i6NGjcHBweOV2Y8eOhYWFBZ49e4aAgACcOnUKv//+O/z9/cU2RYG1a9euaNq0qbg8Ly8Pw4cPx6effooePXq88ji//PILMjMzlfYbFxcHAAgMDISJiUmZz7W6ys/PR2xsLDp16lSlhigWvdOrKoZiKlFWVhasra3FH/AXVeTMFS//EMlkshKXvfyX/L179/Dhhx/C09OzTHf6tmrV6rV/yWppaYlXxV+Uq5ChoBrO8VlZchWyajknamVin6mG/aWa6tpfmpqaEAQBo0ePxs6dOxEXFwcnJ6fXbqdQKCAIAgoLC6GpqYk2bdogIiICaWlpsLCwAPA8vBoaGqJx48ZKv3MOHTqElJQUDB48+LUhr3Xr1pg+fbpYKwAcPHgQ9evXF48jFZqamlUqFJe3Ft5o9x6Sy+XiVDTlWQ8AzZo1Q2pqKjQ0NODo6Kj0qFGjRpnqSE5Oxr1798TnJ0+eFIdJvIl///0XHTp0EKfhefFKc2kSEhJgbW39RsclIqJ3a+TIkVi3bh02bNgAAwMDpKamIjU1FU+fPgUA3Lx5E+Hh4Thz5gySk5Nx/Phx8X6Yoiu4Pj4+cHFxwSeffILz58/jzz//xIwZMzBy5MhiF0KioqJQr149pXcsiyxfvhxeXl7i8wEDBkAul2Pw4MG4dOkSNm/ejKVLlyq940jVC68Uv4fs7e0RHx+PpKQk6OvrF7vKam9vj6ysLOzfvx+NGzeGrq4udHV1ldp4e3vDw8MDAQEBmD9/PurVq4d79+7hjz/+QM+ePeHu7v7aOrS1tREUFISFCxciMzMTY8aMQWBg4BtND1cUiO3s7LBw4UI8ePBAXFe0359++glyuVx8S2zbtm348ccfsWbNmnIfl4iI3r2VK1cCeD7//ouioqIQHBwMbW1tHDlyBJGRkUhLS4OlpSXatWuH48ePi1dr1dXV8fvvv2P48OHw8PCAnp4egoKCxJvRi2RkZGD79u2lTsP28OFDpeGDRkZG2Lt3L0aOHInmzZujRo0amDVrFqdjq8YYit9DkyZNQlBQEFxcXPD06VOlKc2A5zNQDBs2DH379sWjR48we/bsYtOyyWQyxMTEYPr06Rg0aBAePHgAKysrtGvXTmlam1dxdHREr1694O/vj8ePH6Nr16743//+90bnFhsbi+vXr+P69evFZqMQhP+7w3ru3Lm4ffs2NDQ00KBBA2zevBl9+vR5o2MTEdG79eL/6yWxsbF55c1wRezs7F7bzsjICBkZGaW2Cw0NLfa70s3NDUeOHHnt8al6kAmve8URSVRmZiaMjIzw8OFDfvBHGeTn5yMmJgb+/v5VamxZVcY+Uw37SzXsL9Wxz1RTVfur6Pd3RkYGDA0Ny7wdxxQTERERkeQxFFO5NGzYEPr6+iU+XvxwDiIiIqLqgGOKqVxiYmJK/cSYso45JiIiIqoqGIqpXF71efBERERE1Q2HTxARERGR5DEUExEREZHkMRQTERERkeQxFBMRERGR5DEUExEREZHkMRQTERERkeQxFBMRERGR5DEUExEREZHkMRQTERERkeQxFBMRERGR5DEUExEREZHkMRQTERERkeQxFBMRERGR5DEUExEREZHkMRQTERERkeQxFBMRERGR5DEUExEREZHkMRQTERERkeQxFBMRERGR5DEUExEREZHkMRQTERERkeQxFBMRERGR5DEUExEREZHkMRQTERERkeRVWChOT0+vqF0REREREb1T5QrF33zzDTZv3iw+DwwMhJmZGWrWrInz589XWHFERERERO9CuULxd999B1tbWwBAbGwsYmNjsXv3bnTu3BkhISEVWiARERER0dumUZ6NUlNTxVD8+++/IzAwED4+PrC3t0erVq0qtEAiIiIioretXFeKTUxMcOfOHQDAnj174O3tDQAQBAGFhYUVVx0RERER0TtQrivFvXr1woABA+Dk5IRHjx6hc+fOAIBz587B0dGxQgskIiIiInrbyhWKlyxZAnt7e9y5cwfz58+Hvr4+ACAlJQUjRoyo0AKJiIiIiN62coViTU1NTJo0qdjy8ePHv3FBRERERETvWrnnKf7555/Rpk0b2NjY4Pbt2wCAyMhI7Ny5s8KKIyIiIiJ6F8oVileuXIkJEyagc+fOSE9PF2+uMzY2RmRkZEXWR0RERET01pUrFC9btgyrV6/G9OnToa6uLi53d3fHhQsXKqw4IiIiIqJ3oVyh+NatW2jatGmx5VpaWsjOzn7jooiIiIiI3qVyhWIHBwckJCQUW75nzx44Ozu/aU1ERERERO9UuWafmDBhAkaOHIlnz55BEAScOnUKGzduRHh4ONasWVPRNRIRERERvVXlCsVDhgyBjo4OZsyYgZycHAwYMAA2NjZYunQp+vXrV9E1EhERERG9VSqH4oKCAmzYsAG+vr4YOHAgcnJykJWVBQsLi7dRHxERERHRW6fymGINDQ0MGzYMz549AwDo6uoyEBMRERFRtVauG+1atmyJc+fOVXQtRERERESVolxjikeMGIGJEyfi7t27aN68OfT09JTWu7m5VUhxRERERETvQrlCcdHNdGPGjBGXyWQyCIIAmUwmfsIdEREREVF1UK5QfOvWrYqug4iIiIio0pQrFNvZ2VV0HURVVqvw/SjQ0Ht9Q4nTUhcwvyXQKPRP5BbKKrWWpIgusLe3x+3bt4utGzFiBFasWIFVq1Zhw4YNOHv2LJ48eYK0tDQYGxsrtT179iy+/PJL/PXXX1BXV0fv3r2xePFi6Ovrl3psQRAwe/ZsrF69Gunp6WjdujVWrlwJJyenij5NIiKqQOUKxWvXrn3l+k8//bRcxdD7JTg4GOnp6dixY0dll0IS9NdffykN5bp48SI6deqEjz76CACQk5MDPz8/+Pn5YerUqcW2v3fvHry9vdG3b18sX74cmZmZGDduHIKDg7F169ZSjzt//nx8++23+Omnn+Dg4ICZM2fC19cXly9fhra2dsWfKBERVYhyheKxY8cqPc/Pz0dOTg7kcjl0dXUZiitZaGgoduzYUeJHcb8Pzp8/j4iICBw9ehQPHz6Evb09hg0bpvS6jIuLw4cfflhs25SUFFhZWb3LcqmSmJubKz2PiIhA3bp10b59ewDAuHHjADx/rZTk999/h6amJlasWAE1tecT9Xz33Xdwc3PD9evX4ejoWGwbQRAQGRmJGTNmoEePHgCeX0SwtLTEjh07+OFGRERVWLlCcVpaWrFl165dw/DhwxESEvLGRVHVkJeXB7lcXtllFHPmzBlYWFhg3bp1sLW1xfHjx/H5559DXV0do0aNUmqbmJgIQ0ND8Tnn1JamvLw8rFu3DhMmTIBMVrahHbm5uZDL5WIgBgAdHR0AwNGjR0sMxbdu3UJqaiq8vb3FZUZGRmjVqhVOnDjBUExEVIWVa57ikjg5OSEiIqLYVWQqnz179qBNmzYwNjaGmZkZunbtihs3bojr7969i/79+8PU1BR6enpwd3dHfHw8oqOjERYWhvPnz0Mmk0EmkyE6OhoAkJycjB49ekBfXx+GhoYIDAzEf//9J+4zNDQUTZo0wZo1a+Dg4CC+1bt161a4urpCR0cHZmZm8Pb2RnZ2dpnPJSwsDObm5jA0NMSwYcOQl5cnruvQoQNGjx6NcePGwcTEBJaWlli9ejWys7MxaNAgGBgYwNHREbt37xa3+eyzz7B06VK0b98ederUwccff4xBgwZh27ZtxY5tYWEBKysr8fFiwCHp2LFjB9LT0xEcHFzmbTp27IjU1FQsWLAAeXl5SEtLw5QpUwA8f8ehJKmpqQAAS0tLpeWWlpbiOiIiqprKdaW41J1paODevXsVuUvJys7OxoQJE+Dm5oasrCzMmjULPXv2REJCAnJyctC+fXvUrFkTu3btgpWVFc6ePQuFQoG+ffvi4sWL2LNnD/bt2wfg+ZUqhUIhBuJDhw6hoKAAI0eORN++fZXePr5+/Tp+/fVXbNu2Derq6khJSUH//v0xf/589OzZE0+ePMGRI0cgCEKZzmP//v3Q1tZGXFwckpKSMGjQIJiZmeHrr78W2/z000+YPHkyTp06hc2bN2P48OHYvn07evbsiWnTpmHJkiX45JNPkJycDF1d3RKPk5GRAVNT02LLmzRpgtzcXDRq1AihoaFo3bp1qbXm5uYiNzdXfJ6ZmQkA0FIToK5etvOVMi01QenfypSfn6/0fM2aNfD19YW5uXmxdQUFBeI2L66rV68efvjhB0yePBlTp04V34mwtLSEIAjF9vOqfSkUCshksmLbFD0vaV9UHPtLNewv1bHPVFNV+6u89ciEsqabF+zatUvpuSAISElJwfLly2Fra6t0VY8qxsOHD2Fubo4LFy7g+PHjmDRpEpKSkkoMgiWNKY6NjUXnzp1x69Yt2NraAgAuX76Mhg0b4tSpU2jRogVCQ0Mxb948/Pvvv+J4zLNnz6J58+ZISkpSedaR4OBg/Pbbb7hz544YZr/77juEhIQgIyMDampq6NChAwoLC3HkyBEAQGFhIYyMjNCrVy/xhs7U1FRYW1vjxIkT+OCDD4od5/jx42jfvj3++OMP+Pj4AHg+bCIuLg7u7u7Izc3FmjVr8PPPPyM+Ph7NmjUrsd7Q0FCEhYUVW75hw4ZSwzhVfffv38ewYcPw5ZdfolWrVsXWX7hwATNnzsS6detKnVUiPT0dWlpakMlkGDBgACZOnFjiH1ipqakYNmwYFi9ejDp16ojLp0+fDgcHBwwZMqTiToyIiEqUk5ODAQMGICMjQ2kI5euU60pxQECA0nOZTAZzc3N07NgRixYtKs8u6SXXrl3DrFmzEB8fj4cPH0KhUAB4PgQiISEBTZs2LTEQl+bKlSuwtbUVAzEAuLi4wNjYGFeuXEGLFi0APJ9u78UblBo3bgwvLy+4urrC19cXPj4+6NOnD0xMTMp03MaNGysFSg8PD2RlZeHOnTtiyH7xExDV1dVhZmYGV1dXcVnRW9H3798vtv+LFy+iR48emD17thiIAaB+/fqoX7+++NzT0xM3btzAkiVL8PPPP5dY69SpUzFhwgTxeWZmJmxtbfHVOTUUaKqX6XylTEtNwFx3BWaeVkOuonKnZLsY6it+PWfOHFhYWGDmzJnQ0Cj+X17RJ3L6+PgUm5LtZdHR0dDW1kZISEiJbQVBQGhoKPLz8+Hv7w/g+evo+vXrmDJlirisSH5+PmJjY9GpUydoamqqeJbSw/5SDftLdewz1VTV/ip6p1dV5QrFRQGN3p5u3brBzs4Oq1evho2NDRQKBRo1aoS8vDzxZp+34eWP7FZXV0dsbCyOHz+OvXv3YtmyZZg+fTri4+Ph4OBQIcd8+QdJJpMpLSu6Merl193ly5fh5eWFzz//HDNmzHjtcVq2bImjR4+Wul5LSwtaWlrFlucqZCio5Hl3q5NchazS5ykuev0oFAqsXbsWQUFBxX5uUlNTkZqaiqSkJADA1atXYWBggNq1a4t/cC5fvhyenp7Q19dHbGwsQkJCEBERofSHY4MGDRAeHo6ePXsCeD6rRXh4OBo0aCBOyWZjY4M+ffqU+ktDU1OzSv1CqerYX6phf6mOfaaaqtZf5a2lXHcdzZkzBzk5OcWWP336FHPmzClXIfR/Hj16hMTERMyYMQNeXl5wdnZWmvHDzc0NCQkJePz4cYnby+XyYh+17ezsjDt37uDOnTvissuXLyM9PR0uLi6vrEcmk6F169YICwvDuXPnIJfLsX379jKdy/nz5/H06VPx+cmTJ6Gvr690xbo8Ll26hA8//BBBQUFK45NfJSEhAdbW1m90XKpe9u3bh+TkZHz22WfF1n333Xdo2rQphg4dCgBo164dmjZtqjQ87NSpU+jUqRNcXV2xatUqfP/990ofbw88H6qTkZEhPp88eTJGjx6Nzz//HC1atEBWVhb27NnDOYqJiKq4cl0pDgsLw7Bhw4qNs8zJyUFYWBhmzZpVIcVJlYmJCczMzLBq1SpYW1sjOTlZvOsdAPr374958+YhICAA4eHhsLa2xrlz52BjYwMPDw/Y29vj1q1bSEhIQK1atWBgYABvb2+4urpi4MCBiIyMREFBAUaMGIH27dvD3d291Fri4+Oxf/9++Pj4wMLCAvHx8Xjw4AGcnZ3LdC55eXkYPHgwZsyYgaSkJMyePRujRo16o1kgLl68iI4dO8LX1xcTJkwQ7+pXV1cXr+BFRkbCwcEBDRs2xLNnz7BmzRocOHAAe/fuLfdxqfrx8fEp9abQ0NBQhIaGvnL7131QEYBi+5fJZJgzZw4vEBARVTPlSiaCIJQ41+f58+dVGudKJVNTU8OmTZtw5swZNGrUCOPHj8eCBQvE9XK5HHv37oWFhQX8/f3h6uqKiIgIqKs/H/fau3dv+Pn54cMPP4S5uTk2btwImUyGnTt3wsTEBO3atYO3tzfq1KmDzZs3v7IWQ0NDHD58GP7+/qhXrx5mzJiBRYsWoXPnzmU6Fy8vLzg5OaFdu3bo27cvunfv/tog8jpbt27FgwcPsG7dOlhbW4uPonHRwPMwPnHiRLi6uqJ9+/Y4f/489u3bBy8vrzc6NhEREb2fVJp9wsTEBDKZTLyb78VgXFhYiKysLAwbNgwrVqx4K8USvUuZmZkwMjLCw4cPYWZmVtnlVHn5+fmIiYmBv79/lRpbVpWxz1TD/lIN+0t17DPVVNX+Kvr9/VZnn4iMjIQgCPjss88QFhYGIyMjcZ1cLoe9vT08PDxU2SURERERUaVTKRQHBQUBABwcHODp6Vml/iqgd6+0OV0BYPfu3Wjbtu07rIaIiIio/Mp1o1379u3Fr589e6b0sb0AVLpUTdXXix8O8rKaNWu+u0KIiIiI3lC5QnFOTg4mT56MLVu24NGjR8XWvzwdGL2fHB0dK7sEIiIiogpRrtknQkJCcODAAaxcuRJaWlpYs2YNwsLCYGNjU6YpjIiIiIiIqpJyXSn+7bffsHbtWnTo0AGDBg1C27Zt4ejoCDs7O6xfvx4DBw6s6DqJiIiIiN6acl0pfvz4MerUqQPg+fjhok9Wa9OmDQ4fPlxx1RERERERvQPlCsV16tTBrVu3AAANGjTAli1bADy/gmxsbFxhxRERERERvQvlCsWDBg3C+fPnAQBTpkzBihUroK2tjfHjxyMkJKRCCyQiIiIietvKNaZ4/Pjx4tfe3t64evUqzpw5A0dHR7i5uVVYcURERERE70K5QvGLnj17Bjs7O9jZ2VVEPURERERE71y5hk8UFhZi7ty5qFmzJvT19XHz5k0AwMyZM/HDDz9UaIFERERERG9buULx119/jejoaMyfPx9yuVxc3qhRI6xZs6bCiiMiIiIiehfKFYrXrl2LVatWYeDAgVBXVxeXN27cGFevXq2w4oiIiIiI3oVyheJ///23xI/4VSgUyM/Pf+OiiIiIiIjepXKFYhcXFxw5cqTY8q1bt6Jp06ZvXBQRERER0btUrtknZs2ahaCgIPz7779QKBTYtm0bEhMTsXbtWvz+++8VXSMRERER0Vul0pXimzdvQhAE9OjRA7/99hv27dsHPT09zJo1C1euXMFvv/2GTp06va1aiYiIiIjeCpWuFDs5OSElJQUWFhZo27YtTE1NceHCBVhaWr6t+oiIiIiI3jqVrhQLgqD0fPfu3cjOzq7QgoiIiIiI3rVy3WhX5OWQTERERERUHakUimUyGWQyWbFlRERERETVmUpjigVBQHBwMLS0tAAAz549w7Bhw6Cnp6fUbtu2bRVXIRERERHRW6ZSKA4KClJ6/vHHH1doMURERERElUGlUBwVFfW26iAiIiIiqjRvdKMdEREREdH7gKGYiIiIiCSPoZiIiIiIJI+hmIiIiIgkj6GYiIiIiCSPoZiIiIiIJI+hmIiIiIgkj6GYiIiIiCSPoZiIiIiIJI+hmIiIiIgkj6GYiIiIiCSPoZiIiIiIJI+hmIiIiIgkj6GYiIiIiCSPoZiIiIiIJI+hmIiIiIgkj6GYiIiIiCSPoZiIiIiIJI+hmIiIiIgkj6GYiIiIiCSPoZiIiIiIJI+hmIiIiIgkj6GYiIiIiCRPo7ILIKrqWoXvR4GGXmWX8VYlRXSp7BKIiIgq1Xt1pTguLg4ymQzp6emvbGdvb4/IyMh3UlNJx5PJZNixY8c7O35lkcp5vi9WrlwJNzc3GBoawtDQEB4eHti9ezcAICkpCTKZrMTHL7/8Iu7j2rVr8PX1hbGxMUxMTODr64vz58+/8rjPnj3DyJEjYWZmBn19ffTu3Rv//fffWz1XIiKil1XrUNyhQweMGzdOfO7p6YmUlBQYGRkBAKKjo2FsbFw5xVWAoiCSkJBQ2aVUKatXr0bbtm1hYmICExMTeHt749SpU0ptgoODi4U3Pz+/Sqq4eqhVqxYiIiJw5swZnD59Gh07dkSPHj1w6dIl2NraIiUlRekRFhYGfX19dO7cGQCQlZWFOXPmwNbWFvHx8Th69CgMDAzg6+uL/Pz8Uo87fvx4/Pbbb/jll19w6NAh3Lt3D7169XpXp01ERATgPRs+IZfLYWVlVdllVEn5+fnQ1NSs7DIqRFxcHPr37w9PT09oa2vjm2++gY+PDy5duoSaNWuK7fz8/BAVFSU+19LSqoxyq41u3bopPf/666+xcuVKnDx5Eg0bNiz2s7V9+3YEBgZCX18fAJCYmIgnT55g9uzZqFOnDgBg9uzZcHNzw+3bt+Ho6FjsmBkZGfjhhx+wYcMGdOzYEQAQFRUFZ2dnnDx5Eh988MHbOFUiIqJiqu2V4uDgYBw6dAhLly4VrwRGR0eLwyfi4uIwaNAgZGRkiOtDQ0NL3Fd6ejqGDBkCc3NzGBoaomPHjq99y7fIjRs30KNHD1haWkJfXx8tWrTAvn37KuQcHRwcAABNmzaFTCZDhw4dxHVr1qyBs7MztLW10aBBA/zvf/8T1xVdYd68eTPat28PbW1trF+/HsHBwQgICMC8efNgaWkJY2NjzJkzBwUFBQgJCYGpqSlq1aqlFCTz8vIwatQoWFtbQ1tbG3Z2dggPDy/zOaSkpKBz587Q0dFBnTp1sHXr1mJ1btmyBW3btoWOjg5atGiBf/75B3/99Rfc3d3FK5EPHjwQt1u/fj1GjBiBJk2aoEGDBlizZg0UCgX279+vdGwtLS1YWVmJDxMTkzLXLXWFhYXYtGkTsrOz4eHhUWz9mTNnkJCQgMGDB4vL6tWrBwMDA0RFRSEvLw9Pnz7FDz/8AGdnZ9jb25d4nDNnziA/Px/e3t7isgYNGqB27do4ceJEhZ8XERFRaaptKF66dCk8PDwwdOhQ8e1cW1tbcb2npyciIyNhaGgorp80aVKJ+/roo49w//597N69G2fOnEGzZs3g5eWFx48fv7aOrKws+Pv7Y//+/Th37hz8/PzQrVs3JCcnv/E5Fg0J2LdvH1JSUrBt2zYAz0PhrFmz8PXXX+PKlSuYN28eZs6ciZ9++klp+ylTpmDs2LG4cuUKfH19AQAHDhzAvXv3cPjwYSxevBizZ89G165dYWJigvj4eAwbNgxffPEF7t69CwD49ttvsWvXLmzZsgWJiYlYv359qQGnJDNnzkTv3r1x/vx5DBw4EP369cOVK1eU2syePRszZszA2bNnoaGhgQEDBmDy5MlYunQpjhw5guvXr2PWrFmlHiMnJwf5+fkwNTVVWh4XFwcLCwvUr18fw4cPx6NHj8pct1RduHAB+vr60NLSwrBhw7B9+3a4uLgUa1cUdj09PcVlBgYG+Oqrr7Bx40bo6OhAX18fe/bswe7du6GhUfKbUqmpqZDL5cWGOVlaWiI1NbVCz42IiOhVqu3wCSMjI8jlcujq6opv6169elVcL5fLYWRkBJlM9sohFUePHsWpU6dw//598e31hQsXYseOHdi6dSs+//zzV9bRuHFjNG7cWHw+d+5cbN++Hbt27cKoUaPe5BRhbm4OADAzM1M6h9mzZ2PRokXiuEsHBwdcvnwZ33//PYKCgsR248aNKzY209TUFN9++y3U1NRQv359zJ8/Hzk5OZg2bRoAYOrUqYiIiMDRo0fRr18/JCcnw8nJCW3atIFMJoOdnZ1K5/DRRx9hyJAhAJ73TWxsLJYtW6Z0ZXvSpEliaB87diz69++P/fv3o3Xr1gCAwYMHIzo6utRjfPnll7CxsVG62ujn54devXrBwcEBN27cwLRp09C5c2ecOHEC6urqJe4nNzcXubm54vPMzEwAgJaaAHV1QaXzrm6KxvzWqVMHf/31FzIzM/Hrr78iKCgI+/btUwrGT58+xYYNGzBt2jSlscKZmZlYvnw5WrVqhZ9//hmFhYVYvHgx/P39ceLECejo6BQ7bkFBgdLxiwiCgMLCwleORX4fFJ3f+36eFYX9pRr2l+rYZ6qpqv1V3nqqbSiuKOfPn0dWVhbMzMyUlj99+hQ3btx47fZZWVkIDQ3FH3/8gZSUFBQUFODp06cVcqW4JNnZ2bhx4wYGDx6MoUOHissLCgrEGwyLuLu7F9u+YcOGUFP7vzcILC0t0ahRI/G5uro6zMzMcP/+fQDPh6l06tQJ9evXh5+fH7p27QofH58y1/vyW+8eHh7Fbhx0c3NTqgcAXF1dlZYV1fOyiIgIbNq0CXFxcdDW1haX9+vXT/za1dUVbm5uqFu3LuLi4uDl5VXivsLDwxEWFlZs+YymCujqFpZyhu+HmJiYYstat26NP//8E5MnT8aIESPE5QcPHkR2djasrKyUtouNjcX9+/fRp08f8fs1YMAAfPzxx5gzZw7atm1b7Bi3b99GXl4etmzZIo5NLlqelpZWYl3vo9jY2MouoVphf6mG/aU69plqqlp/5eTklGs7yYfirKwsWFtbIy4urti6ssxcMWnSJMTGxmLhwoVwdHSEjo4O+vTpg7y8vIovFs/rBZ7PwNCqVSuldS9fAdXTKz637ss328lkshKXKRQKAECzZs1w69Yt7N69G/v27UNgYCC8vb2Vxga/qRePL5PJSlxWVM+LFi5ciIiICOzbt08pWJekTp06qFGjBq5fv15qKJ46dSomTJggPs/MzIStrS2+OqeGAs2Sry6/Ly6G+pa4PDIyEpaWlvD39xeXLV68GN26dUP//v2V2l6/fh1qamrw8fGBXC4H8PyPNQ0NDbi5uSnto0jr1q0xd+5caGhoiOsTExPx4MEDDBo0qNhr/H2Tn5+P2NhYdOrU6b25EfZtYn+phv2lOvaZaqpqfxW906uqah2K5XI5CgtLv4L3uvXA89CXmpoKDQ0NlcbKFjl27BiCg4PRs2dPAM9Da1JSksr7KUlRsHjxHCwtLWFjY4ObN29i4MCBFXKc1zE0NETfvn3Rt29f9OnTB35+fnj8+HGxMbwlOXnyJD799FOl502bNn3jmubPn4+vv/4af/75Z4lXxF929+5dPHr0CNbW1qW20dLSKnGGilyFDAWFsjeqt6rT1NTE1KlT0blzZ9SuXRtPnjzBhg0bcOjQIfz555/if3bXr1/HkSNHEBMTU+w/QB8fH0ydOhUTJ07E2LFjoVAoEBERAQ0NDfE/zH///RdeXl5Yu3YtWrZsiRo1amDw4MGYPHkyLCwsYGhoiNGjR8PDwwNt2rSpjK6oFJqamlXqF0pVx/5SDftLdewz1VS1/ipvLdU6FNvb2yM+Ph5JSUnQ19cvdjXR3t4eWVlZ2L9/Pxo3bgxdXV3o6uoqtfH29oaHhwcCAgIwf/581KtXD/fu3cMff/yBnj17vjZwOTk5Ydu2bejWrRtkMhlmzpxZ4lXN8rCwsICOjg727NmDWrVqQVtbG0ZGRggLC8OYMWNgZGQEPz8/5Obm4vTp00hLS1O60lkRFi9eDGtrazRt2hRqamr45ZdfYGVlVeb5n3/55Re4u7ujTZs2WL9+PU6dOoUffvjhjWr65ptvMGvWLGzYsAH29vbiDVn6+vrQ19dHVlYWwsLC0Lt3b1hZWeHGjRuYPHkyHB0dxbHLVNz9+/fx6aefinN9u7m54c8//0SnTp3ENj/++CNq1apV4hCaBg0aYPr06di7dy88PDygpqaGpk2bYs+ePeIfI/n5+UhMTFR6a2vJkiVQU1ND7969kZubC19fX6Ux50RERO9CtZ19Ang+dEFdXR0uLi4wNzcvNo7X09MTw4YNQ9++fWFubo758+cX24dMJkNMTAzatWuHQYMGoV69eujXrx9u374tjm99lcWLF8PExASenp7o1q0bfH190axZswo5Pw0NDXz77bf4/vvvYWNjgx49egAAhgwZgjVr1iAqKgqurq5o3749oqOjxSncKpKBgQHmz58Pd3d3tGjRAklJSYiJiVEal/wqYWFh2LRpE9zc3LB27Vps3LixxNkMVLFy5Urk5eWhT58+sLa2Fh8LFy4E8HwYyd9//43u3bujXr16GDx4MJo3b44jR45wruJX+OGHH5CUlITc3Fzcv38f+/btUwrEADBv3jwkJyeX+v1v0qQJ4uLikJ6ejsePH2P//v1Kcw3b29tDEASl6QW1tbWxYsUKPH78GNnZ2di2bRvnGyciondOJgjC+31bPVE5ZWZmwsjICHUnbkaBRvHx2e+TpIgub7yP/Px8xMTEwN/fv0q9jVaVsc9Uw/5SDftLdewz1VTV/ir6/Z2RkQFDQ8Myb1eth08QvQvxU72KzU5CRERE75dqPXziXWjYsKE4VvXlx/r1699o3/PmzSt13507d66gM3g71q9fX2rtDRs2rOzyiIiIiFTCK8WvERMTU+ok0GUZc/wqw4YNQ2BgYInrSvqgg6qke/fupU6XVZXeQiEiIiIqC4bi11D1E9xUYWpqWqZpzaoiAwMDGBgYVHYZRERERBWCwyeIiIiISPIYiomIiIhI8hiKiYiIiEjyGIqJiIiISPIYiomIiIhI8hiKiYiIiEjyGIqJiIiISPIYiomIiIhI8hiKiYiIiEjyGIqJiIiISPIYiomIiIhI8hiKiYiIiEjyGIqJiIiISPIYiomIiIhI8hiKiYiIiEjyGIqJiIiISPIYiomIiIhI8hiKiYiIiEjyGIqJiIiISPIYiomIiIhI8hiKiYiIiEjyGIqJiIiISPIYiomIiIhI8hiKiYiIiEjyGIqJiIiISPIYiomIiIhI8hiKiYiIiEjyGIqJiIiISPIYiomIiIhI8hiKiYiIiEjyGIqJiIiISPIYiomIiIhI8hiKiYiIiEjyGIqJiIiISPIYiomIiIhI8hiKiYiIiEjyGIqJiIiISPIYiomIiIhI8hiKiYiIiEjyGIqJiIiISPIYiomIiIhI8hiKiYiIiEjyGIqJiIiISPIYiomIiIhI8hiKiYiIiEjyNCq7AKKqrlX4fhRo6JW5fVJEFwBAYWEhQkNDsW7dOqSmpsLGxgbBwcGYMWMGZDIZAOC///7Dl19+ib179yI9PR3t2rXDsmXL4OTk9Mpj/PLLL5g5cyaSkpLg5OSEb775Bv7+/uU/SSIiIonjleJ3QCaTYceOHQCApKQkyGQyJCQkVGpNb5tUzvNVvvnmG6xcuRLLly/HlStX8M0332D+/PlYtmwZAEAQBAQEBODmzZvYuXMnzp07Bzs7O3h7eyM7O7vU/R4/fhz9+/fH4MGDce7cOQQEBCAgIAAXL158V6dGRET03qm2oTg0NBRNmjSp7DLeqri4OMhkMqSnp1d2KVVKeHg4WrRoAQMDA1hYWCAgIACJiYlKbTp06ACZTKb0GDZs2Dut8/jx4+jRowe6dOkCe3t79OnTBz4+Pjh16hQA4Nq1azh58iRWrlyJFi1aoH79+li5ciWePn2KjRs3lrrfpUuXws/PDyEhIXB2dsbcuXPRrFkzLF++/F2dGhER0Xun2obiipKXl1fZJbx1giCgoKCgssuoMIcOHcLIkSNx8uRJxMbGIj8/Hz4+PsWurg4dOhQpKSniY/78+e+0Tk9PT+zfvx///PMPAOD8+fM4evQoOnfuDADIzc0FAGhra4vbqKmpQUtLC0ePHi11vydOnIC3t7fSMl9fX5w4caKiT4GIiEgyKjUU79mzB23atIGxsTHMzMzQtWtX3LhxQ1x/9+5d9O/fH6amptDT04O7uzvi4+MRHR2NsLAwnD9/XrwKGB0dDQBITk5Gjx49oK+vD0NDQwQGBuK///4T91l0hXnNmjVwcHAQA8nWrVvh6uoKHR0dmJmZvfYt7CJ//fUXOnXqhBo1asDIyAjt27fH2bNn37hvkpKS8OGHHwIATExMIJPJEBwcDABQKBQIDw+Hg4MDdHR00LhxY2zdulXctugK8+7du9G8eXMxZHXo0AGjR4/GuHHjYGJiAktLS6xevRrZ2dkYNGgQDAwM4OjoiN27d4v7SktLw8CBA2Fubg4dHR04OTkhKiqqzOdx9epVeHp6QltbG40aNcKhQ4eK1fnnn3+iadOm0NHRQceOHXH//n3s3r0bzs7OMDQ0xIABA5CTkyNut2fPHgQHB6Nhw4Zo3LgxoqOjkZycjDNnzigdW1dXF1ZWVuLD0NBQpe/Bm5oyZQr69euHBg0aQFNTE02bNsW4ceMwcOBAAECDBg1Qu3ZtTJ06FWlpacjLy8M333yDu3fvIiUlpdT9pqamwtLSUmmZpaUlUlNT3+r5EBERvc8q9Ua77OxsTJgwAW5ubsjKysKsWbPQs2dPJCQkICcnB+3bt0fNmjWxa9cuWFlZ4ezZs1AoFOjbty8uXryIPXv2YN++fQAAIyMjKBQKMRAfOnQIBQUFGDlyJPr27Yu4uDjxuNevX8evv/6Kbdu2QV1dHSkpKejfvz/mz5+Pnj174smTJzhy5AgEQXjtOTx58gRBQUFYtmwZBEHAokWL4O/vj2vXrsHAwKDcfWNra4tff/0VvXv3RmJiIgwNDaGjowPg+fCBdevW4bvvvoOTkxMOHz6Mjz/+GObm5mjfvr24jylTpmDhwoWoU6cOTExMAAA//fQTJk+ejFOnTmHz5s0YPnw4tm/fjp49e2LatGlYsmQJPvnkEyQnJ0NXVxczZ87E5cuXsXv3btSoUQPXr1/H06dPy3weISEhiIyMhIuLCxYvXoxu3brh1q1bMDMzE9uEhoZi+fLl0NXVRWBgIAIDA6GlpYUNGzYgKysLPXv2xLJly/Dll1+WeIyMjAwAgKmpqdLy9evXY926dbCyskK3bt0wc+ZM6Orqllprbm6uePUWADIzMwEAWmoC1NVf/1ookp+fDwDYvHkz1q9fj7Vr18LFxQXnz5/HpEmTYGFhgU8//RQAsGXLFnz++ecwNTWFuro6vLy84OfnB0EQxP2UpKCgQGl9YWGh0rErQ9GxK7OG6oZ9phr2l2rYX6pjn6mmqvZXeeuRCWVJfu/Iw4cPYW5ujgsXLuD48eOYNGkSkpKSioUd4HmQ2rFjh9KNXLGxsejcuTNu3boFW1tbAMDly5fRsGFDnDp1Ci1atEBoaCjmzZuHf//9F+bm5gCAs2fPonnz5khKSoKdnd0bnYNCoYCxsTE2bNiArl27Anh+o9327dsREBCApKQkODg44Ny5c68dEx0XF4cPP/wQaWlpMDY2BvA8uJmammLfvn3w8PAQ2w4ZMgQ5OTnYsGGDuN2OHTvQo0cPsU2HDh1QWFiII0eOAHgepIyMjNCrVy+sXbsWwPOrkNbW1jhx4gQ++OADdO/eHTVq1MCPP/6oUj8UnWdERIQYZgsKCuDg4IDRo0dj8uTJYp379u2Dl5cXACAiIgJTp07FjRs3UKdOHQDAsGHDkJSUhD179pTY3927d0d6errSkINVq1bBzs4ONjY2+Pvvv/Hll1+iZcuW2LZtW6k1h4aGIiwsrNjyDRs2vDJMl2bw4MHo3bu30qwQW7ZswaFDh7BixQqlttnZ2SgoKICRkRFCQkLg6OiIL774osT9DhkyBN27d0f37t3FZRs3bkR8fDwiIyNVrpOIiOh9kpOTgwEDBiAjI0Old4kr9UrxtWvXMGvWLMTHx+Phw4dQKBQAng+BSEhIQNOmTUsMxKW5cuUKbG1txUAMAC4uLjA2NsaVK1fQokULAICdnZ0YiAGgcePG8PLygqurK3x9feHj44M+ffqIV1df5b///sOMGTMQFxeH+/fvo7CwEDk5OUhOTi5z3aq4fv06cnJy0KlTJ6XleXl5aNq0qdIyd3f3Ytu7ubmJX6urq8PMzAyurq7isqK35e/fvw8AGD58OHr37o2zZ8/Cx8cHAQEB8PT0LHO9LwZ3DQ0NuLu748qVK6XWZGlpCV1dXTEQFy0rujntZSNHjsTFixeLjcH9/PPPxa9dXV1hbW0NLy8v3LhxA3Xr1i1xX1OnTsWECRPE55mZmbC1tcVX59RQoKlehrN97mKoL4DnY7ldXV2VQvGFCxdw6tSpUqdPu3btGm7cuIHIyMhi3+MiHTp0QGpqqtI+IiIi0KlTp0qdli0/Px+xsbHo1KkTNDU1K62O6oR9phr2l2rYX6pjn6mmqvZX0Tu9qqrUUNytWzfY2dlh9erVsLGxgUKhQKNGjZCXlycOFXgb9PSU55xVV1dHbGwsjh8/jr1792LZsmWYPn064uPj4eDg8Mp9BQUF4dGjR1i6dCns7OygpaUFDw+Pt3YDX1ZWFgDgjz/+QM2aNZXWaWlpKT1/+TwBFHvRymQypWVF8+cW/YHSuXNn3L59GzExMYiNjYWXlxdGjhyJhQsXvvnJlFDTy/UULSuq50WjRo3C77//jsOHD6NWrVqvPEarVq0APP+jorRQrKWlVawPASBXIUNBoey151GkqP5u3bohIiICDg4OaNiwIc6dO4elS5fis88+E9v88ssvMDc3R+3atXHhwgWMHTsWAQEBSuH2008/Rc2aNREeHg4AGD9+PNq3b49vv/0WXbp0waZNm3DmzBmsXr26SvynpKmpWSXqqE7YZ6phf6mG/aU69plqqlp/lbeWSrvR7tGjR0hMTMSMGTPg5eUFZ2dnpKWlievd3NyQkJCAx48fl7i9XC4Xx1EWcXZ2xp07d3Dnzh1x2eXLl5Geng4XF5dX1iOTydC6dWuEhYXh3LlzkMvl2L59+2vP49ixYxgzZgz8/f3RsGFDaGlp4eHDh6/drizkcjkAKJ2ni4sLtLS0kJycDEdHR6XHi1fIK5K5uTmCgoKwbt06REZGYtWqVWXe9uTJk+LXBQUFOHPmDJydnd+oHkEQMGrUKGzfvh0HDhx47R8uAMRhNtbW1m90bFUsW7YMffr0wYgRI+Ds7IxJkybhiy++wNy5c8U2KSkp+OSTT9CgQQOMGTMGn3zySbHp2JKTk5VuvPP09MSGDRuwatUq8SbLHTt2oFGjRu/s3IiIiN43lXal2MTEBGZmZli1ahWsra2RnJyMKVOmiOv79++PefPmISAgAOHh4bC2tsa5c+dgY2MDDw8P2Nvb49atW0hISECtWrVgYGAAb29vuLq6YuDAgYiMjERBQQFGjBiB9u3blziUoEh8fDz2798PHx8fWFhYID4+Hg8ePChTeHNycsLPP/8Md3d3ZGZmIiQkpMKuctvZ2UEmk+H333+Hv78/dHR0YGBggEmTJmH8+PFQKBRo06YNMjIycOzYMRgaGiIoKKhCjl1k1qxZaN68ORo2bIjc3Fz8/vvvKoXaFStWwMnJCc7OzliyZAnS0tLw2WefvVFNI0eOxIYNG7Bz504YGBiIsy4YGRlBR0cHN27cwIYNG+Dv7w8zMzP8/fffGD9+PNq1a6c0VONtMzAwQGRk5CvH+Y4ZMwZjxox55X5evEm0yEcffYSPPvroDSskIiKiIpV2pVhNTU1827dRo0YYP348FixYIK6Xy+XYu3cvLCws4O/vD1dXV0REREBd/fnYzt69e8PPzw8ffvghzM3NsXHjRshkMuzcuRMmJiZo164dvL29UadOHWzevPmVtRgaGuLw4cPw9/dHvXr1MGPGDCxatEicT/ZVfvjhB6SlpaFZs2b45JNPMGbMGFhYWLxZ5/x/NWvWRFhYGKZMmQJLS0uMGjUKADB37lzMnDkT4eHhcHZ2hp+fH/74448yXTFVlVwux9SpU+Hm5oZ27dpBXV0dmzZtKvP2ERERiIiIQOPGjXH06FHs2rULNWrUeKOaVq5ciYyMDHTo0AHW1tbio+j7LJfLsW/fPvj4+KBBgwaYOHEievfujd9+++2NjktERETvryo1+wRRVZKZmQkjIyM8fPhQaQo5Kll+fj5iYmLg7+9fpcaWVWXsM9Wwv1TD/lId+0w1VbW/in5/qzr7hOQ/0Y6IiIiIiKH4NfT19Ut9FM33W17Dhg0rdd/Dhg2roDN4O+bNm1dq7WUZdkJERERUlVTqlGzVwYsfDvKyl6dEU9WcOXMwadKkEte9648kVtWwYcMQGBhY4rq3OZ0eERER0dvAUPwajo6Ob23fFhYWFXZT3rtmamqq0gerEBEREVVlHD5BRERERJLHUExEREREksdQTERERESSx1BMRERERJLHUExEREREksdQTERERESSx1BMRERERJLHUExEREREksdQTERERESSx1BMRERERJLHUExEREREksdQTERERESSx1BMRERERJLHUExEREREksdQTERERESSx1BMRERERJLHUExEREREksdQTERERESSx1BMRERERJLHUExEREREksdQTERERESSx1BMRERERJLHUExEREREksdQTERERESSx1BMRERERJLHUExEREREksdQTERERESSx1BMRERERJLHUExEREREksdQTERERESSx1BMRERERJLHUExEREREksdQTERERESSx1BMRERERJLHUExEREREksdQTERERESSx1BMRERERJLHUExEREREksdQTERERESSx1BMRERERJLHUExEREREksdQTERERESSx1BMRERERJLHUExEREREksdQTERERESSx1BMRERERJLHUExEREREksdQTERERESSx1BMRERERJLHUExEREREkqdR2QUQVVWCIAAAnjx5Ak1NzUqupurLz89HTk4OMjMz2V9lxD5TDftLNewv1bHPVFNV+yszMxPA//0eLyuGYqJSPHr0CADg4OBQyZUQERGRqp48eQIjI6Myt2coJiqFqakpACA5OVmlHyqpyszMhK2tLe7cuQNDQ8PKLqdaYJ+phv2lGvaX6thnqqmq/SUIAp48eQIbGxuVtmMoJiqFmtrzIfdGRkZV6oe9qjM0NGR/qYh9phr2l2rYX6pjn6mmKvZXeS5m8UY7IiIiIpI8hmIiIiIikjyGYqJSaGlpYfbs2dDS0qrsUqoF9pfq2GeqYX+phv2lOvaZat63/pIJqs5XQURERET0nuGVYiIiIiKSPIZiIiIiIpI8hmIiIiIikjyGYiIiIiKSPIZiohKsWLEC9vb20NbWRqtWrXDq1KnKLqlShIaGQiaTKT0aNGggrn/27BlGjhwJMzMz6Ovro3fv3vjvv/+U9pGcnIwuXbpAV1cXFhYWCAkJQUFBwbs+lbfm8OHD6NatG2xsbCCTybBjxw6l9YIgYNasWbC2toaOjg68vb1x7do1pTaPHz/GwIEDYWhoCGNjYwwePBhZWVlKbf7++2+0bdsW2trasLW1xfz589/2qb0Vr+uv4ODgYq85Pz8/pTZS6q/w8HC0aNECBgYGsLCwQEBAABITE5XaVNTPYVxcHJo1awYtLS04OjoiOjr6bZ9ehStLf3Xo0KHYa2zYsGFKbaTSXwCwcuVKuLm5iR/A4eHhgd27d4vrJfX6EohIyaZNmwS5XC78+OOPwqVLl4ShQ4cKxsbGwn///VfZpb1zs2fPFho2bCikpKSIjwcPHojrhw0bJtja2gr79+8XTp8+LXzwwQeCp6enuL6goEBo1KiR4O3tLZw7d06IiYkRatSoIUydOrUyTuetiImJEaZPny5s27ZNACBs375daX1ERIRgZGQk7NixQzh//rzQvXt3wcHBQXj69KnYxs/PT2jcuLFw8uRJ4ciRI4Kjo6PQv39/cX1GRoZgaWkpDBw4ULh48aKwceNGQUdHR/j+++/f1WlWmNf1V1BQkODn56f0mnv8+LFSGyn1l6+vrxAVFSVcvHhRSEhIEPz9/YXatWsLWVlZYpuK+Dm8efOmoKurK0yYMEG4fPmysGzZMkFdXV3Ys2fPOz3fN1WW/mrfvr0wdOhQpddYRkaGuF5K/SUIgrBr1y7hjz/+EP755x8hMTFRmDZtmqCpqSlcvHhREARpvb4Yiole0rJlS2HkyJHi88LCQsHGxkYIDw+vxKoqx+zZs4XGjRuXuC49PV3Q1NQUfvnlF3HZlStXBADCiRMnBEF4HoDU1NSE1NRUsc3KlSsFQ0NDITc3963WXhleDnkKhUKwsrISFixYIC5LT08XtLS0hI0bNwqCIAiXL18WAAh//fWX2Gb37t2CTCYT/v33X0EQBOF///ufYGJiotRnX375pVC/fv23fEZvV2mhuEePHqVuI+X+EgRBuH//vgBAOHTokCAIFfdzOHnyZKFhw4ZKx+rbt6/g6+v7tk/prXq5vwTheSgeO3ZsqdtIub+KmJiYCGvWrJHc64vDJ4hekJeXhzNnzsDb21tcpqamBm9vb5w4caISK6s8165dg42NDerUqYOBAwciOTkZAHDmzBnk5+cr9VWDBg1Qu3Ztsa9OnDgBV1dXWFpaim18fX2RmZmJS5cuvdsTqQS3bt1CamqqUh8ZGRmhVatWSn1kbGwMd3d3sY23tzfU1NQQHx8vtmnXrh3kcrnYxtfXF4mJiUhLS3tHZ/PuxMXFwcLCAvXr18fw4cPx6NEjcZ3U+ysjIwMAYGpqCqDifg5PnDihtI+iNtX9/72X+6vI+vXrUaNGDTRq1AhTp05FTk6OuE7K/VVYWIhNmzYhOzsbHh4eknt9aVR2AURVycOHD1FYWKj0ww0AlpaWuHr1aiVVVXlatWqF6Oho1K9fHykpKQgLC0Pbtm1x8eJFpKamQi6Xw9jYWGkbS0tLpKamAgBSU1NL7Muide+7onMsqQ9e7CMLCwul9RoaGjA1NVVq4+DgUGwfRetMTEzeSv2Vwc/PD7169YKDgwNu3LiBadOmoXPnzjhx4gTU1dUl3V8KhQLjxo1D69at0ahRIwCosJ/D0tpkZmbi6dOn0NHReRun9FaV1F8AMGDAANjZ2cHGxgZ///03vvzySyQmJmLbtm0ApNlfFy5cgIeHB549ewZ9fX1s374dLi4uSEhIkNTri6GYiErVuXNn8Ws3Nze0atUKdnZ22LJlS5X5T4zeL/369RO/dnV1hZubG+rWrYu4uDh4eXlVYmWVb+TIkbh48SKOHj1a2aVUC6X11+effy5+7erqCmtra3h5eeHGjRuoW7fuuy6zSqhfvz4SEhKQkZGBrVu3IigoCIcOHarsst45Dp8gekGNGjWgrq5e7M7a//77D1ZWVpVUVdVhbGyMevXq4fr167CyskJeXh7S09OV2rzYV1ZWViX2ZdG6913ROb7q9WRlZYX79+8rrS8oKMDjx4/ZjwDq1KmDGjVq4Pr16wCk21+jRo3C77//joMHD6JWrVri8or6OSytjaGhYbX8A7i0/ipJq1atAEDpNSa1/pLL5XB0dETz5s0RHh6Oxo0bY+nSpZJ7fTEUE71ALpejefPm2L9/v7hMoVBg//798PDwqMTKqoasrCzcuHED1tbWaN68OTQ1NZX6KjExEcnJyWJfeXh44MKFC0ohJjY2FoaGhnBxcXnn9b9rDg4OsLKyUuqjzMxMxMfHK/VReno6zpw5I7Y5cOAAFAqF+Mvaw8MDhw8fRn5+vtgmNjYW9evXr7ZDAcrq7t27ePToEaytrQFIr78EQcCoUaOwfft2HDhwoNiwkIr6OfTw8FDaR1Gb6vb/3uv6qyQJCQkAoPQak0p/lUahUCA3N1d6r6/KvtOPqKrZtGmToKWlJURHRwuXL18WPv/8c8HY2FjpzlqpmDhxohAXFyfcunVLOHbsmODt7S3UqFFDuH//viAIz6fqqV27tnDgwAHh9OnTgoeHh+Dh4SFuXzRVj4+Pj5CQkCDs2bNHMDc3f6+mZHvy5Ilw7tw54dy5cwIAYfHixcK5c+eE27dvC4LwfEo2Y2NjYefOncLff/8t9OjRo8Qp2Zo2bSrEx8cLR48eFZycnJSmGEtPTxcsLS2FTz75RLh48aKwadMmQVdXt1pOMfaq/nry5IkwadIk4cSJE8KtW7eEffv2Cc2aNROcnJyEZ8+eifuQUn8NHz5cMDIyEuLi4pSmEMvJyRHbVMTPYdGUWSEhIcKVK1eEFStWVMkps17ndf11/fp1Yc6cOcLp06eFW7duCTt37hTq1KkjtGvXTtyHlPpLEARhypQpwqFDh4Rbt24Jf//9tzBlyhRBJpMJe/fuFQRBWq8vhmKiEixbtkyoXbu2IJfLhZYtWwonT56s7JIqRd++fQVra2tBLpcLNWvWFPr27Stcv35dXP/06VNhxIgRgomJiaCrqyv07NlTSElJUdpHUlKS0LlzZ0FHR0eoUaOGMHHiRCE/P/9dn8pbc/DgQQFAsUdQUJAgCM+nZZs5c6ZgaWkpaGlpCV5eXkJiYqLSPh49eiT0799f0NfXFwwNDYVBgwYJT548UWpz/vx5oU2bNoKWlpZQs2ZNISIi4l2dYoV6VX/l5OQIPj4+grm5uaCpqSnY2dkJQ4cOLfYHqZT6q6S+AiBERUWJbSrq5/DgwYNCkyZNBLlcLtSpU0fpGNXF6/orOTlZaNeunWBqaipoaWkJjo6OQkhIiNI8xYIgnf4SBEH47LPPBDs7O0Eulwvm5uaCl5eXGIgFQVqvL5kgCMK7uy5NRERERFT1cEwxEREREUkeQzERERERSR5DMRERERFJHkMxEREREUkeQzERERERSR5DMRERERFJHkMxEREREUkeQzERERERSR5DMRERVUnBwcGQyWTFHtevX6/s0ojoPaRR2QUQERGVxs/PD1FRUUrLzM3NK6kaZfn5+dDU1KzsMoiogvBKMRERVVlaWlqwsrJSeqirq5fY9vbt2+jWrRtMTEygp6eHhg0bIiYmRlx/6dIldO3aFYaGhjAwMEDbtm1x48YNAIBCocCcOXNQq1YtaGlpoUmTJtizZ4+4bVJSEmQyGTZv3oz27dtDW1sb69evBwCsWbMGzs7O0NbWRoMGDfC///3vLfYIEb0tvFJMRETvhZEjRyIvLw+HDx+Gnp4eLl++DH19fQDAv//+i3bt2qFDhw44cOAADA0NcezYMRQUFAAAli5dikWLFuH7779H06ZN8eOPP6J79+64dOkSnJycxGNMmTIFixYtQtOmTcVgPGvWLCxfvhxNmzbFuXPnMHToUOjp6SEoKKhS+oGIykcmCIJQ2UUQERG9LDg4GOvWrYO2tra4rHPnzvjll19KbO/m5obevXtj9uzZxdZNmzYNmzZtQmJiYolDHmrWrImRI0di2rRp4rKWLVuiRYsWWLFiBZKSkuDg4IDIyEiMHTtWbOPo6Ii5c+eif//+4rKvvvoKMTExOH78eLnOm4gqB68UExFRlfXhhx9i5cqV4nM9Pb1S244ZMwbDhw/H3r174e3tjd69e8PNzQ0AkJCQgLZt25YYiDMzM3Hv3j20bt1aaXnr1q1x/vx5pWXu7u7i19nZ2bhx4wYGDx6MoUOHissLCgpgZGSk2okSUaVjKCYioipLT08Pjo6OZWo7ZMgQ+Pr64o8//sDevXsRHh6ORYsWYfTo0dDR0amweopkZWUBAFavXo1WrVoptStt3DMRVV280Y6IiN4btra2GDZsGLZt24aJEydi9erVAJ4PrThy5Ajy8/OLbWNoaAgbGxscO3ZMafmxY8fg4uJS6rEsLS1hY2ODmzdvwtHRUenh4OBQsSdGRG8drxQTEdF7Ydy4cejcuTPq1auHtLQ0HDx4EM7OzgCAUaNGYdmyZejXrx+mTp0KIyMjnDx5Ei1btkT9+vUREhKC2bNno27dumjSpAmioqKQkJAgzjBRmrCwMIwZMwZGRkbw8/NDbm4uTp8+jbS0NEyYMOFdnDYRVRCGYiIiei8UFhZi5MiRuHv3LgwNDeHn54clS5YAAMzMzHDgwAGEhISgffv2UFdXR5MmTcRxxGPGjEFGRgYmTpyI+/fvw8XFBbt27VKaeaIkQ4YMga6uLhYsWICQkBDo6enB1dUV48aNe9unS0QVjLNPEBEREZHkcUwxEREREUkeQzERERERSR5DMRERERFJHkMxEREREUkeQzERERERSR5DMRERERFJHkMxEREREUkeQzERERERSR5DMRERERFJHkMxEREREUkeQzERERERSR5DMRERERFJ3v8DXqNZ/RorfxMAAAAASUVORK5CYII=", + "text/plain": [ + "
" ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from xgboost import plot_importance\n", + "\n", + "plot_importance(ranker, importance_type=\"weight\");" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Importing the model to Elasticsearch\n", + "\n", + "Once the model is trained you will be able to use Eland to send it to Elasticsearch.\n", + "\n", + "Please note that the `MLModel.import_ltr_model` method contains the `LTRModelConfig` object in order to associate the feature extraction with the model.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 127, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" }, + "id": "zAMwvqYlq9py", + "outputId": "c0f60ce3-fb07-47a5-9e37-fccbd1f30bcc" + }, + "outputs": [ { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Adding features to the judgement list\n", - "\n", - "During this step we will add features to our judgmennt list. The resuling dataframe will be used to train our model.\n", - "\n", - "**Note** This operation is quite fast if your Elasticsearch instance is local or close (around 1 min 30 sec.) but can be much longer if it is not the case (more than 10 minutes). When using Google Collab it is difficult to control where your notebook is executed and it is likely that you can get in the later case." - ] + "name": "stderr", + "output_type": "stream", + "text": [ + "/Users/afoucret/git/elasticsearch-labs/.venv/lib/python3.9/site-packages/eland/ml/ml_model.py:550: ElasticsearchWarning: The default [remove_binary] value of 'false' is deprecated and will be set to 'true' in a future release. Set [remove_binary] explicitly to 'true' or 'false' to ensure no behavior change.\n", + " self._client.options(ignore_status=404).ml.delete_trained_model(\n" + ] }, { - "cell_type": "code", - "execution_count": 10, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 615, - "referenced_widgets": [ - "f9cdfbc3972a4b84a557507567ca2965", - "a6d4eb3325444f28b11ba02c3d01ed83", - "bff504814b434aec90b2cf020b08cfa9", - "594c5ebcb9624b63b128536a46594211", - "e95447129da74d9ebc4c1d99165bd534", - "7ed336be71e74521a596a7d624c1e7d1", - "ee1a6943af1e49e6a8851f27c9811c32", - "8c393bd522f6427f9978a89bc7dbdf3b", - "549645ca4e7b48ef86cffdaa5507c56c", - "0ab8ee0c2e1a42658ecbf01b0b28cf92", - "84206a53779249fbb34c78edb17fd1e0" - ] - }, - "id": "xbp6_9dqqJkJ", - "outputId": "0aadfe34-d739-4823-e5e2-310bd5fb69d3" - }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|██████████| 16279/16279 [01:28<00:00, 183.72it/s]\n" - ] - }, - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
query_idquerydoc_idgradetitle_bm25actors_bm25title_all_terms_bm25popularity
0qid:5141insidious 2 netflix8464330NaN9.555378NaN13.628
1qid:5141insidious 2 netflix4901819.857398NaNNaN64.003
2qid:5141insidious 2 netflix382340NaNNaNNaN143.211
3qid:5141insidious 2 netflix5676040NaNNaNNaN32.913
4qid:5141insidious 2 netflix26979503.809668NaNNaN21.058
...........................
384750qid:33832013 the wolverine2631150NaNNaNNaN68.287
384751qid:33832013 the wolverine259130NaNNaNNaN21.026
384752qid:33832013 the wolverine5676040NaNNaNNaN32.913
384753qid:33832013 the wolverine5335350NaNNaNNaN34.773
384754qid:33832013 the wolverine8763270NaNNaNNaN25.920
\n", - "

384755 rows × 8 columns

\n", - "
" - ], - "text/plain": [ - " query_id query doc_id grade title_bm25 actors_bm25 \\\n", - "0 qid:5141 insidious 2 netflix 846433 0 NaN 9.555378 \n", - "1 qid:5141 insidious 2 netflix 49018 1 9.857398 NaN \n", - "2 qid:5141 insidious 2 netflix 38234 0 NaN NaN \n", - "3 qid:5141 insidious 2 netflix 567604 0 NaN NaN \n", - "4 qid:5141 insidious 2 netflix 269795 0 3.809668 NaN \n", - "... ... ... ... ... ... ... \n", - "384750 qid:3383 2013 the wolverine 263115 0 NaN NaN \n", - "384751 qid:3383 2013 the wolverine 25913 0 NaN NaN \n", - "384752 qid:3383 2013 the wolverine 567604 0 NaN NaN \n", - "384753 qid:3383 2013 the wolverine 533535 0 NaN NaN \n", - "384754 qid:3383 2013 the wolverine 876327 0 NaN NaN \n", - "\n", - " title_all_terms_bm25 popularity \n", - "0 NaN 13.628 \n", - "1 NaN 64.003 \n", - "2 NaN 143.211 \n", - "3 NaN 32.913 \n", - "4 NaN 21.058 \n", - "... ... ... \n", - "384750 NaN 68.287 \n", - "384751 NaN 21.026 \n", - "384752 NaN 32.913 \n", - "384753 NaN 34.773 \n", - "384754 NaN 25.920 \n", - "\n", - "[384755 rows x 8 columns]" - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "import numpy\n", - "\n", - "from eland.ml.ltr import FeatureLogger\n", - "\n", - "# First we create a feature logger that will be used to query Elasticsearch to retrieve the features:\n", - "feature_logger = FeatureLogger(es_client, MOVIE_INDEX, ltr_config)\n", - "\n", - "# This method will be applied for each group of query in the judgment log:\n", - "def _extract_query_features(query_judgements_group):\n", - " # Retrieve document ids in the query group as strings.\n", - " doc_ids = query_judgements_group['doc_id'].astype('str').to_list()\n", - "\n", - " # Resolve query paras for the current query group (e.g.: {\"query\": \"batman\"}).\n", - " query_params = { 'query': query_judgements_group['query'].iloc[0] }\n", - "\n", - " # Extract the features for the documents in the query group:\n", - " doc_features = feature_logger.extract_features(query_params, doc_ids)\n", - "\n", - " # Adding a column to the dataframe for each features:\n", - " for feature_index, feature_name in enumerate(feature_logger._model_config.feature_names):\n", - " query_judgements_group[feature_name] = numpy.array([doc_features[doc_id][feature_index] for doc_id in doc_ids])\n", - "\n", - " return query_judgements_group\n", - "\n", - "judgments_with_features = judgments_df.groupby('query_id', group_keys=False).progress_apply(_extract_query_features)\n", - "\n", - "judgments_with_features" + "data": { + "text/plain": [ + "" ] + }, + "execution_count": 127, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from eland.ml import MLModel\n", + "\n", + "LEARNING_TO_RANK_MODEL_ID=\"ltr-model-xgboost\"\n", + "\n", + "MLModel.import_ltr_model(\n", + " es_client=es_client,\n", + " model=ranker,\n", + " model_id=LEARNING_TO_RANK_MODEL_ID,\n", + " ltr_model_config=ltr_config,\n", + " es_if_exists=\"replace\",\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Using the rescorer\n", + "\n", + "Once the model is uploaded to Elasticsearch, you will be able to use it as a rescorer in the _search API, as shown in this example:\n", + "\n", + "```\n", + "POST /_search\n", + "{\n", + " \"query\" : {\n", + " \"multi_match\" : {\n", + " \"query\": \"star wars\",\n", + " \"field\": [\"title\", \"overview\", \"actors\", \"director\", \"tags\", \"characters\"]\n", + " }\n", + " },\n", + " \"rescore\" : {\n", + " \"window_size\" : 50,\n", + " \"learning_to_rank\" : {\n", + " \"model_id\": \"ltr-model-xgboost\",\n", + " \"params\": { \n", + " \"query\": \"star wars\"\n", + " }\n", + " }\n", + " }\n", + "}\n", + "```" + ] + }, + { + "cell_type": "code", + "execution_count": 140, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" }, + "id": "Xgr5MWWIrEk9", + "outputId": "e296cf37-afd1-43fb-e839-6c65cb65c072" + }, + "outputs": [ { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [], - "source": [ - "# This step will separate the dataset in two different parts one used for the training and one used for the evaluation of the model.\n", - "#\n", - "# We are not using sklearn.model_selection.train_test_split because it is ignoring query group during the split.\n", - "# In theory it should be possible to use it if you have enough pairs for each query in your judgment list.\n", - "\n", - "import random\n", - "\n", - "def train_test_split(df, test_size=0.3, group_field='query_id'):\n", - " def _add_split(query_judgements_group):\n", - " split, = random.choices(['train', 'eval'], [1 - test_size, test_size])\n", - " query_judgements_group['split'] = split\n", - " return query_judgements_group\n", - " df_with_split = df.groupby(group_field, group_keys=False).apply(_add_split)\n", - " return (\n", - " df_with_split.query('split == \"train\"').drop(columns='split'),\n", - " df_with_split.query('split == \"eval\"').drop(columns='split')\n", - " )\n", - "\n", - "train_judgments_df, eval_judgments_df = train_test_split(judgments_with_features)" + "data": { + "text/plain": [ + "[('Star Wars', 10.972473, '11'),\n", + " ('Star Wars: The Clone Wars', 9.924128, '12180'),\n", + " ('After Porn Ends 2', 9.613241, '440249'),\n", + " ('Andor: A Disney+ Day Special Look', 8.982841, '1022100'),\n", + " (\"Family Guy Presents: It's a Trap!\", 8.840657, '278427'),\n", + " ('Star Wars: The Rise of Skywalker', 8.053794, '181812'),\n", + " ('Star Wars: The Force Awakens', 8.053794, '140607'),\n", + " ('Star Wars: The Last Jedi', 8.053794, '181808'),\n", + " ('Solo: A Star Wars Story', 8.053794, '348350'),\n", + " ('The Star Wars Holiday Special', 8.053794, '74849')]" ] - }, + }, + "execution_count": 140, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "query = \"star wars\"\n", + "\n", + "# First let's display the result when not using the rescorer:\n", + "search_fields = [\"title\", \"overview\", \"actors\", \"director\", \"tags\", \"characters\"]\n", + "bm25_query = { \"multi_match\": { \"query\": query, \"fields\": search_fields } }\n", + "\n", + "bm25_search_response = es_client.search(index=MOVIE_INDEX, query=bm25_query)\n", + "\n", + "[\n", + " (movie[\"_source\"][\"title\"], movie[\"_score\"], movie[\"_id\"])\n", + " for movie in bm25_search_response['hits']['hits']\n", + "]" + ] + }, + { + "cell_type": "code", + "execution_count": 141, + "metadata": {}, + "outputs": [ { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Create and train the model\n", - "\n", - "The LTR rescorer supports XGBRanker trained models.\n", - "\n", - "You will find more information on XGBRanker model in the xgboost [documentation](https://xgboost.readthedocs.io/en/latest/tutorials/learning_to_rank.html)." + "data": { + "text/plain": [ + "[('Star Wars: The Clone Wars', 3.809828, '12180'),\n", + " ('Star Wars', 3.4305632, '11'),\n", + " ('Star Wars: The Last Jedi', 2.3990567, '181808'),\n", + " ('Solo: A Star Wars Story', 2.044759, '348350'),\n", + " ('Star Wars: The Force Awakens', 2.0258214, '140607'),\n", + " ('Star Wars: The Rise of Skywalker', 1.9873005, '181812'),\n", + " ('LEGO Star Wars Summer Vacation', 1.9347491, '980804'),\n", + " ('LEGO Star Wars Terrifying Tales', 1.495373, '857702'),\n", + " ('LEGO Star Wars Holiday Special', 1.3972183, '732670'),\n", + " ('Rogue One: A Star Wars Story', 1.0395032, '330459')]" ] + }, + "execution_count": 141, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Now let's display result when using the rescorer:\n", + "\n", + "ltr_rescorer = {\n", + " \"learning_to_rank\": {\n", + " \"model_id\": LEARNING_TO_RANK_MODEL_ID,\n", + " \"params\": {\"query\": query},\n", + " },\n", + " \"window_size\": 100,\n", + "}\n", + "\n", + "rescored_search_response = es_client.search(index=MOVIE_INDEX, query=bm25_query, rescore=ltr_rescorer)\n", + "\n", + "[\n", + " (movie[\"_source\"][\"title\"], movie[\"_score\"], movie[\"_id\"])\n", + " for movie in rescored_search_response[\"hits\"][\"hits\"]\n", + "]" + ] + } + ], + "metadata": { + "colab": { + "provenance": [] + }, + "kernelspec": { + "display_name": ".venv", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.6" + }, + "widgets": { + "application/vnd.jupyter.widget-state+json": { + "0ab8ee0c2e1a42658ecbf01b0b28cf92": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[0]\tvalidation_0-ndcg@10:0.86390\n", - "[1]\tvalidation_0-ndcg@10:0.87184\n", - "[2]\tvalidation_0-ndcg@10:0.87372\n", - "[3]\tvalidation_0-ndcg@10:0.87504\n", - "[4]\tvalidation_0-ndcg@10:0.87537\n", - "[5]\tvalidation_0-ndcg@10:0.87582\n", - "[6]\tvalidation_0-ndcg@10:0.87628\n", - "[7]\tvalidation_0-ndcg@10:0.87691\n", - "[8]\tvalidation_0-ndcg@10:0.87771\n", - "[9]\tvalidation_0-ndcg@10:0.87769\n", - "[10]\tvalidation_0-ndcg@10:0.87768\n", - "[11]\tvalidation_0-ndcg@10:0.87793\n", - "[12]\tvalidation_0-ndcg@10:0.87802\n", - "[13]\tvalidation_0-ndcg@10:0.87792\n", - "[14]\tvalidation_0-ndcg@10:0.87805\n", - "[15]\tvalidation_0-ndcg@10:0.87905\n", - "[16]\tvalidation_0-ndcg@10:0.87952\n", - "[17]\tvalidation_0-ndcg@10:0.87964\n", - "[18]\tvalidation_0-ndcg@10:0.88025\n", - "[19]\tvalidation_0-ndcg@10:0.88012\n", - "[20]\tvalidation_0-ndcg@10:0.88032\n", - "[21]\tvalidation_0-ndcg@10:0.88083\n", - "[22]\tvalidation_0-ndcg@10:0.88153\n", - "[23]\tvalidation_0-ndcg@10:0.88206\n", - "[24]\tvalidation_0-ndcg@10:0.88152\n", - "[25]\tvalidation_0-ndcg@10:0.88240\n", - "[26]\tvalidation_0-ndcg@10:0.88185\n", - "[27]\tvalidation_0-ndcg@10:0.88204\n", - "[28]\tvalidation_0-ndcg@10:0.88222\n", - "[29]\tvalidation_0-ndcg@10:0.88209\n", - "[30]\tvalidation_0-ndcg@10:0.88219\n", - "[31]\tvalidation_0-ndcg@10:0.88270\n", - "[32]\tvalidation_0-ndcg@10:0.88294\n", - "[33]\tvalidation_0-ndcg@10:0.88306\n", - "[34]\tvalidation_0-ndcg@10:0.88316\n", - "[35]\tvalidation_0-ndcg@10:0.88348\n", - "[36]\tvalidation_0-ndcg@10:0.88352\n", - "[37]\tvalidation_0-ndcg@10:0.88369\n", - "[38]\tvalidation_0-ndcg@10:0.88421\n", - "[39]\tvalidation_0-ndcg@10:0.88417\n", - "[40]\tvalidation_0-ndcg@10:0.88421\n", - "[41]\tvalidation_0-ndcg@10:0.88413\n", - "[42]\tvalidation_0-ndcg@10:0.88443\n", - "[43]\tvalidation_0-ndcg@10:0.88428\n", - "[44]\tvalidation_0-ndcg@10:0.88415\n", - "[45]\tvalidation_0-ndcg@10:0.88426\n", - "[46]\tvalidation_0-ndcg@10:0.88428\n", - "[47]\tvalidation_0-ndcg@10:0.88465\n", - "[48]\tvalidation_0-ndcg@10:0.88453\n", - "[49]\tvalidation_0-ndcg@10:0.88495\n", - "[50]\tvalidation_0-ndcg@10:0.88539\n", - "[51]\tvalidation_0-ndcg@10:0.88573\n", - "[52]\tvalidation_0-ndcg@10:0.88552\n", - "[53]\tvalidation_0-ndcg@10:0.88572\n", - "[54]\tvalidation_0-ndcg@10:0.88579\n", - "[55]\tvalidation_0-ndcg@10:0.88584\n", - "[56]\tvalidation_0-ndcg@10:0.88593\n", - "[57]\tvalidation_0-ndcg@10:0.88596\n", - "[58]\tvalidation_0-ndcg@10:0.88611\n", - "[59]\tvalidation_0-ndcg@10:0.88611\n", - "[60]\tvalidation_0-ndcg@10:0.88601\n", - "[61]\tvalidation_0-ndcg@10:0.88621\n", - "[62]\tvalidation_0-ndcg@10:0.88624\n", - "[63]\tvalidation_0-ndcg@10:0.88593\n", - "[64]\tvalidation_0-ndcg@10:0.88595\n", - "[65]\tvalidation_0-ndcg@10:0.88585\n", - "[66]\tvalidation_0-ndcg@10:0.88603\n", - "[67]\tvalidation_0-ndcg@10:0.88630\n", - "[68]\tvalidation_0-ndcg@10:0.88635\n", - "[69]\tvalidation_0-ndcg@10:0.88660\n", - "[70]\tvalidation_0-ndcg@10:0.88664\n", - "[71]\tvalidation_0-ndcg@10:0.88658\n", - "[72]\tvalidation_0-ndcg@10:0.88674\n", - "[73]\tvalidation_0-ndcg@10:0.88662\n", - "[74]\tvalidation_0-ndcg@10:0.88710\n", - "[75]\tvalidation_0-ndcg@10:0.88731\n", - "[76]\tvalidation_0-ndcg@10:0.88732\n", - "[77]\tvalidation_0-ndcg@10:0.88739\n", - "[78]\tvalidation_0-ndcg@10:0.88748\n", - "[79]\tvalidation_0-ndcg@10:0.88727\n", - "[80]\tvalidation_0-ndcg@10:0.88756\n", - "[81]\tvalidation_0-ndcg@10:0.88790\n", - "[82]\tvalidation_0-ndcg@10:0.88785\n", - "[83]\tvalidation_0-ndcg@10:0.88784\n", - "[84]\tvalidation_0-ndcg@10:0.88792\n", - "[85]\tvalidation_0-ndcg@10:0.88801\n", - "[86]\tvalidation_0-ndcg@10:0.88803\n", - "[87]\tvalidation_0-ndcg@10:0.88803\n", - "[88]\tvalidation_0-ndcg@10:0.88813\n", - "[89]\tvalidation_0-ndcg@10:0.88811\n", - "[90]\tvalidation_0-ndcg@10:0.88810\n", - "[91]\tvalidation_0-ndcg@10:0.88814\n", - "[92]\tvalidation_0-ndcg@10:0.88841\n", - "[93]\tvalidation_0-ndcg@10:0.88870\n", - "[94]\tvalidation_0-ndcg@10:0.88887\n", - "[95]\tvalidation_0-ndcg@10:0.88888\n", - "[96]\tvalidation_0-ndcg@10:0.88877\n", - "[97]\tvalidation_0-ndcg@10:0.88869\n", - "[98]\tvalidation_0-ndcg@10:0.88855\n", - "[99]\tvalidation_0-ndcg@10:0.88865\n" - ] - }, - { - "data": { - "text/html": [ - "
XGBRanker(base_score=None, booster=None, callbacks=None, colsample_bylevel=None,\n",
-              "          colsample_bynode=None, colsample_bytree=None, device=None,\n",
-              "          early_stopping_rounds=20, enable_categorical=False,\n",
-              "          eval_metric=['ndcg@10'], feature_types=None, gamma=None,\n",
-              "          grow_policy=None, importance_type=None, interaction_constraints=None,\n",
-              "          learning_rate=None, max_bin=None, max_cat_threshold=None,\n",
-              "          max_cat_to_onehot=None, max_delta_step=None, max_depth=None,\n",
-              "          max_leaves=None, min_child_weight=None, missing=nan,\n",
-              "          monotone_constraints=None, multi_strategy=None, n_estimators=None,\n",
-              "          n_jobs=None, num_parallel_tree=None, random_state=None, ...)
In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.
" - ], - "text/plain": [ - "XGBRanker(base_score=None, booster=None, callbacks=None, colsample_bylevel=None,\n", - " colsample_bynode=None, colsample_bytree=None, device=None,\n", - " early_stopping_rounds=20, enable_categorical=False,\n", - " eval_metric=['ndcg@10'], feature_types=None, gamma=None,\n", - " grow_policy=None, importance_type=None, interaction_constraints=None,\n", - " learning_rate=None, max_bin=None, max_cat_threshold=None,\n", - " max_cat_to_onehot=None, max_delta_step=None, max_depth=None,\n", - " max_leaves=None, min_child_weight=None, missing=nan,\n", - " monotone_constraints=None, multi_strategy=None, n_estimators=None,\n", - " n_jobs=None, num_parallel_tree=None, random_state=None, ...)" - ] - }, - "execution_count": 12, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "import xgboost as xgb\n", - "\n", - "# Create the ranker model:\n", - "ranker = xgb.XGBRanker(\n", - " objective=\"rank:ndcg\",\n", - " eval_metric=[\"ndcg@10\"],\n", - " early_stopping_rounds=20,\n", - ")\n", - "\n", - "# Shaping training and eval data in the expected format.\n", - "train_query_groups = train_judgments_df['query_id'].value_counts().sort_index().values\n", - "train_target = train_judgments_df['grade'].values\n", - "train_features = train_judgments_df[ltr_config.feature_names]\n", - "\n", - "eval_query_groups = eval_judgments_df['query_id'].value_counts().sort_index().values\n", - "eval_target = eval_judgments_df['grade'].values\n", - "eval_features = eval_judgments_df[ltr_config.feature_names]\n", - "\n", - "# Training the model\n", - "ranker.fit(\n", - " X=train_features,\n", - " y=train_target,\n", - " group=train_query_groups,\n", - " eval_set=[(eval_features, eval_target)],\n", - " eval_group=[eval_query_groups],\n", - " verbose=True\n", - ")" - ] + "549645ca4e7b48ef86cffdaa5507c56c": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 490 - }, - "id": "3iSx3IuLqq7R", - "outputId": "d81ac47f-99c6-4656-9fc1-b9699a80c458" - }, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 13, - "metadata": {}, - "output_type": "execute_result" - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAArUAAAHHCAYAAAChoqAWAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8g+/7EAAAACXBIWXMAAA9hAAAPYQGoP6dpAABV/UlEQVR4nO3deVwV9f7H8fcB2XcQBFwx931Bzd3UBDT3cslKzOVqWplpN8sUtNLrctW0zLI0b6a2qpWSuJA3FyxTyyVyI83lahoi4sIyvz98cH4eQQVEYfD1fDzOQ87Md2Y+c74Ib77nO3MshmEYAgAAAEzMrrALAAAAAO4UoRYAAACmR6gFAACA6RFqAQAAYHqEWgAAAJgeoRYAAACmR6gFAACA6RFqAQAAYHqEWgAAAJgeoRYAUOgWLVoki8WixMTEwi4FgEkRagGgEGSFuJweL7/88l055pYtWxQVFaWkpKS7sv/7WWpqqqKiohQXF1fYpQD3rRKFXQAA3M8mTpyokJAQm2W1atW6K8fasmWLoqOjFRkZKW9v77tyjPx68skn1adPHzk5ORV2KfmSmpqq6OhoSVKbNm0KtxjgPkWoBYBCFBERodDQ0MIu445cvHhRbm5ud7QPe3t72dvbF1BF905mZqauXr1a2GUAENMPAKBIW7NmjVq2bCk3Nzd5eHioU6dO2rt3r02bX375RZGRkapYsaKcnZ0VGBiop59+WmfPnrW2iYqK0pgxYyRJISEh1qkOiYmJSkxMlMVi0aJFi7Id32KxKCoqymY/FotF+/bt0+OPPy4fHx+1aNHCuv7jjz9Ww4YN5eLiIl9fX/Xp00fHjh277XnmNKe2QoUKeuSRRxQXF6fQ0FC5uLiodu3a1rf4v/zyS9WuXVvOzs5q2LChdu7cabPPyMhIubu76/DhwwoLC5Obm5uCg4M1ceJEGYZh0/bixYt68cUXVbZsWTk5Oalq1aqaPn16tnYWi0UjRozQkiVLVLNmTTk5Oendd9+Vv7+/JCk6Otr62ma9brnpn+tf24MHD1pH0728vDRgwAClpqZme80+/vhjNW7cWK6urvLx8VGrVq20du1amza5+f4BigtGagGgEJ0/f15//fWXzbKSJUtKkv7zn/+of//+CgsL07/+9S+lpqZq3rx5atGihXbu3KkKFSpIkmJjY3X48GENGDBAgYGB2rt3r9577z3t3btX27Ztk8ViUY8ePfT7779r6dKlmjlzpvUY/v7+OnPmTJ7rfuyxx1S5cmW9+eab1uD3xhtv6LXXXlOvXr00aNAgnTlzRnPmzFGrVq20c+fOfE15OHjwoB5//HH94x//0BNPPKHp06erc+fOevfdd/XKK6/omWeekSRNnjxZvXr1UkJCguzs/n+8JiMjQ+Hh4XrwwQc1depUxcTEaMKECUpPT9fEiRMlSYZhqEuXLtq4caMGDhyoevXq6bvvvtOYMWN0/PhxzZw506amDRs26NNPP9WIESNUsmRJ1a1bV/PmzdOwYcPUvXt39ejRQ5JUp04dSbnrn+v16tVLISEhmjx5sn7++WctWLBAAQEB+te//mVtEx0draioKDVr1kwTJ06Uo6Oj4uPjtWHDBnXo0EFS7r9/gGLDAADccwsXLjQk5fgwDMO4cOGC4e3tbQwePNhmu1OnThleXl42y1NTU7Ptf+nSpYYkY9OmTdZl06ZNMyQZR44csWl75MgRQ5KxcOHCbPuRZEyYMMH6fMKECYYko2/fvjbtEhMTDXt7e+ONN96wWf7rr78aJUqUyLb8Zq/H9bWVL1/ekGRs2bLFuuy7774zJBkuLi7GH3/8YV0+f/58Q5KxceNG67L+/fsbkoxnn33WuiwzM9Po1KmT4ejoaJw5c8YwDMNYsWKFIcl4/fXXbWp69NFHDYvFYhw8eNDm9bCzszP27t1r0/bMmTPZXqssue2frNf26aeftmnbvXt3w8/Pz/r8wIEDhp2dndG9e3cjIyPDpm1mZqZhGHn7/gGKC6YfAEAhevvttxUbG2vzkK6N7iUlJalv377666+/rA97e3s1adJEGzdutO7DxcXF+vXly5f1119/6cEHH5Qk/fzzz3el7qFDh9o8//LLL5WZmalevXrZ1BsYGKjKlSvb1JsXNWrUUNOmTa3PmzRpIklq27atypUrl2354cOHs+1jxIgR1q+zpg9cvXpV69atkyStXr1a9vb2eu6552y2e/HFF2UYhtasWWOzvHXr1qpRo0auzyGv/XPja9uyZUudPXtWycnJkqQVK1YoMzNT48ePtxmVzjo/KW/fP0BxwfQDAChEjRs3zvFCsQMHDki6Ft5y4unpaf363Llzio6O1rJly3T69GmbdufPny/Aav/fjXdsOHDggAzDUOXKlXNs7+DgkK/jXB9cJcnLy0uSVLZs2RyX//333zbL7ezsVLFiRZtlVapUkSTr/N0//vhDwcHB8vDwsGlXvXp16/rr3Xjut5PX/rnxnH18fCRdOzdPT08dOnRIdnZ2twzWefn+AYoLQi0AFEGZmZmSrs2LDAwMzLa+RIn///Hdq1cvbdmyRWPGjFG9evXk7u6uzMxMhYeHW/dzKzfO6cySkZFx022uH33MqtdisWjNmjU53sXA3d39tnXk5GZ3RLjZcuOGC7vuhhvP/Xby2j8FcW55+f4Bigu+qwGgCHrggQckSQEBAWrfvv1N2/39999av369oqOjNX78eOvyrJG6690svGaNBN74oQw3jlDerl7DMBQSEmIdCS0KMjMzdfjwYZuafv/9d0myXihVvnx5rVu3ThcuXLAZrf3tt9+s62/nZq9tXvontx544AFlZmZq3759qlev3k3bSLf//gGKE+bUAkARFBYWJk9PT7355ptKS0vLtj7rjgVZo3o3juLNmjUr2zZZ95K9Mbx6enqqZMmS2rRpk83yd955J9f19ujRQ/b29oqOjs5Wi2EY2W5fdS/NnTvXppa5c+fKwcFB7dq1kyR17NhRGRkZNu0kaebMmbJYLIqIiLjtMVxdXSVlf23z0j+51a1bN9nZ2WnixInZRnqzjpPb7x+gOGGkFgCKIE9PT82bN09PPvmkGjRooD59+sjf319Hjx7Vt99+q+bNm2vu3Lny9PRUq1atNHXqVKWlpal06dJau3atjhw5km2fDRs2lCS9+uqr6tOnjxwcHNS5c2e5ublp0KBBmjJligYNGqTQ0FBt2rTJOqKZGw888IBef/11jR07VomJierWrZs8PDx05MgRffXVVxoyZIhGjx5dYK9Pbjk7OysmJkb9+/dXkyZNtGbNGn377bd65ZVXrPeW7dy5sx566CG9+uqrSkxMVN26dbV27VqtXLlSI0eOtI563oqLi4tq1Kih5cuXq0qVKvL19VWtWrVUq1atXPdPblWqVEmvvvqqJk2apJYtW6pHjx5ycnLSjz/+qODgYE2ePDnX3z9AsVJId10AgPta1i2sfvzxx1u227hxoxEWFmZ4eXkZzs7OxgMPPGBERkYaP/30k7XNn3/+aXTv3t3w9vY2vLy8jMcee8w4ceJEjreYmjRpklG6dGnDzs7O5hZaqampxsCBAw0vLy/Dw8PD6NWrl3H69Omb3tIr63ZYN/riiy+MFi1aGG5uboabm5tRrVo1Y/jw4UZCQkKuXo8bb+nVqVOnbG0lGcOHD7dZlnVbsmnTplmX9e/f33BzczMOHTpkdOjQwXB1dTVKlSplTJgwIdutsC5cuGC88MILRnBwsOHg4GBUrlzZmDZtmvUWWbc6dpYtW7YYDRs2NBwdHW1et9z2z81e25xeG8MwjA8//NCoX7++4eTkZPj4+BitW7c2YmNjbdrk5vsHKC4shnEPZtUDAHCPRUZG6vPPP1dKSkphlwLgHmBOLQAAAEyPUAsAAADTI9QCAADA9JhTCwAAANNjpBYAAACmR6gFAACA6fHhCyi2MjMzdeLECXl4eNz0IywBAEDRYhiGLly4oODgYNnZ5X78lVCLYuvEiRMqW7ZsYZcBAADy4dixYypTpkyu2xNqUWx5eHhIko4cOSJfX99CrgbXS0tL09q1a9WhQwc5ODgUdjm4Af1TdNE3RRd9U3CSk5NVtmxZ6+/x3CLUotjKmnLg4eEhT0/PQq4G10tLS5Orq6s8PT354V8E0T9FF31TdNE3BS+vUwe5UAwAAACmR6gFAACA6RFqAQAAYHqEWgAAAJgeoRYAAACmR6gFAACA6RFqAQAAYHqEWgAAAJgeoRYAAACmR6gFAACA6RFqAQAAYHqEWgAAAJgeoRYAAACmR6gFAACA6RFqAQAAYHqEWgAAAJgeoRYAAACmR6gFAACA6RFqAQAAYHqEWgAAAJgeoRYAAACmR6gFAACA6RFqAQAAYHqEWgAAAJgeoRYAAACmR6gFAACA6RFqAQAAYHqEWgAAAJgeoRYAAACmR6gFAACA6RFqAQAAYHqEWgAAAJgeoRYAAACmR6gFAACA6RFqAQAAYHqEWgAAAJgeoRYAAACmR6gFAACA6RFqAQAAYHqEWgAAAJgeoRYAAACmR6gFAACA6RFqAQAAYHqEWgAAAJgeoRYAAACmR6gFAACA6RFqAQAAYHqEWgAAAJgeoRYAAACmR6gFAACA6RFqAQAAYHqEWgAAAJgeoRYAAACmR6gFAACA6RFqAQAAYHqEWgAAAJgeoRYAAACmR6gFAACA6RFqAQAAYHqEWgAAAJgeoRYAAACmR6gFAACA6RFqAQAAYHqEWgAAAJiexTAMo7CLAO6G5ORkeXl56YEXlyu9hFthl4PrONkbmto4Qy9tt9eVDEthl4Mb0D9FF31TdBXlvkmc0qmwS8iTrN/f58+fl6enZ663Y6QWAACgmJs8ebIaNWokDw8PBQQEqFu3bkpISLBp895776lNmzby9PSUxWJRUlJStv38/vvv6tq1q0qWLClPT0+1aNFCGzdutGnz3HPPqWHDhnJyclK9evVyVd/ly5c1fPhw+fn5KTg4WJJ0+vTpPJ0joRa51qZNG40cOfKO9xMVFZXrb3IAAHDnvv/+ew0fPlzbtm1TbGys0tLS1KFDB128eNHaJjU1VeHh4XrllVduup9HHnlE6enp2rBhg3bs2KG6devqkUce0alTp2zaPf300+rdu3eu63vhhRf09ddf67PPPtO3334rSXriiSfydI4l8tQaKACjR4/Ws88+a30eGRmppKQkrVixovCKAgCgGIuJibF5vmjRIgUEBGjHjh1q1aqVJFkHruLi4nLcx19//aUDBw7ogw8+UJ06dSRJU6ZM0TvvvKM9e/YoMDBQkvTWW29Jks6cOaNffvnltrWdP39eH3zwgT755BO1bdtWycnJkqT4+Hht27ZNDz74YK7OkZFa3DOGYSg9PV3u7u7y8/Mr7HIAALhvnT9/XpLk6+ub6238/PxUtWpVLV68WBcvXlR6errmz5+vgIAANWzYMN+17NixQ2lpaWrfvr3N8rJly2rr1q253g+h1gTatGmjESNGaMSIEfLy8lLJkiX12muvKesav7///ltPPfWUfHx85OrqqoiICB04cMC6/aJFi+Tt7a0VK1aocuXKcnZ2VlhYmI4dO2ZtExkZqW7dutkcd+TIkWrTps1N6/rPf/6j0NBQeXh4KDAwUI8//rjN/Je4uDhZLBatWbPGOrfmhx9+sJl+EBUVpY8++kgrV66UxWKRxWJRXFyc2rZtqxEjRtgc78yZM3J0dNT69evz+UoCAIDMzEyNHDlSzZs3V61atXK9ncVi0bp167Rz5055eHjI2dlZ//73vxUTEyMfH59813Pq1Ck5OjrK29vbZrm/v3+2aQ23wvQDk/joo480cOBAbd++XT/99JOGDBmicuXKafDgwYqMjNSBAwe0atUqeXp66p///Kc6duyoffv2ycHBQdK1eTJvvPGGFi9eLEdHRz3zzDPq06ePNm/enO+a0tLSNGnSJFWtWlWnT5/WqFGjFBkZqdWrV9u0e/nllzV9+nRVrFhRPj4+Nm9rjB49Wvv371dycrIWLlwo6dpfjYMGDdKIESM0Y8YMOTk5SZI+/vhjlS5dWm3bts2xnitXrujKlSvW51lvXzjZGbK35yYfRYmTnWHzL4oW+qfoom+KrqLcN2lpaTbPR4wYoT179mjjxo3Z1klSenq6dbvr1xuGoWHDhsnf318bN26Ui4uLPvzwQ3Xu3FlbtmxRUFCQzX4yMjJkGEaOx7jZ8XKqN7cItSZRtmxZzZw5UxaLRVWrVtWvv/6qmTNnqk2bNlq1apU2b96sZs2aSZKWLFmismXLasWKFXrsscckXfsGmTt3rpo0aSLpWkiuXr26tm/frsaNG+erpqefftr6dcWKFfXWW2+pUaNGSklJkbu7u3XdxIkT9fDDD+e4D3d3d7m4uOjKlSvWuTiS1KNHD40YMUIrV65Ur169JF0bcY6MjJTFkvOtUiZPnqzo6Ohsy8fVz5Sra0a+zhF316TQzMIuAbdA/xRd9E3RVRT75vrBpvfee0/x8fF688039csvv+Q45/XXX3+VJK1du9bm9/nu3bu1evVqffzxx0pKSlJSUpIiIiK0atUqjRs3Tj179rTZz4EDB5ScnJxtsOtGf/zxh65evapPP/1U7u7uSk1NlXTtHdrrs8HtEGpN4sEHH7QJc02bNtWMGTO0b98+lShRwhpWpf+f87J//37rshIlSqhRo0bW59WqVZO3t7f279+f71C7Y8cORUVFaffu3fr777+VmXntP/LRo0dVo0YNa7vQ0NA879vZ2VlPPvmkPvzwQ/Xq1Us///yz9uzZo1WrVt10m7Fjx2rUqFHW58nJySpbtqxe32mndAf7PNeAu8fJztCk0Ey99pOdrmQWrfs5gv4pyuiboqso982eqDAZhqGRI0dq165d2rRpkypXrnzT9m5u1+7t3qFDB5spAVm/58PDw23Crru7uypXrqyOHTva7Oenn37S/v37sy2/UfPmzTVp0iSVKFFCHTt2tL7TeuzYMTVt2jTX50mohSTJzs5ON34Ox62G/y9evKiwsDCFhYVpyZIl8vf319GjRxUWFqarV6/atM36z5FXgwYNUr169fTnn39q4cKFatu2rcqXL3/T9k5OTtapCte7kmlRehG7ETauuZJpKXI3Kcf/o3+KLvqm6CqKfePg4KBnnnlGn3zyiVauXClfX1+dPXtWkuTl5SUXFxdJ1+a2njp1SomJiZKk3377TR4eHipXrpx8fX3VsmVL+fj4aNCgQRo/frxcXFz0/vvvKzExUV26dLFOeTx48KBSUlJ05swZXb58WXv37pUk1ahRQ46Ojjp+/LjatWunxYsXq3HjxipZsqQGDhyol156SQEBAbKzu3bJV+PGjXN95wOJUGsa8fHxNs+3bdumypUrq0aNGkpPT1d8fLx1+sHZs2eVkJBgM1qanp6un376yToqm5CQoKSkJFWvXl3StcnYe/bssTnGrl27rN+gN/rtt9909uxZTZkyRWXLlpV07S+y/HB0dFRGRvbpAbVr11ZoaKjef/99ffLJJ5o7d26+9g8AwP1u3rx5kpTtAvCFCxcqMjJSkvTuu+/aTOPLutVXVpuSJUsqJiZGr776qtq2bau0tDTVrFlTK1euVN26da3bDRo0SN9//731ef369SVJR44cUYUKFZSWlqaEhATrNANJmjlzpuzs7NSzZ0/r9TEff/xxns6Rux+YxNGjRzVq1CglJCRo6dKlmjNnjp5//nlVrlxZXbt21eDBg/XDDz9o9+7deuKJJ1S6dGl17drVur2Dg4OeffZZxcfHa8eOHYqMjNSDDz5oDblt27bVTz/9pMWLF+vAgQOaMGFCtpB7vXLlysnR0VFz5szR4cOHtWrVKk2aNClf51ahQgX98ssvSkhI0F9//WUzQjxo0CBNmTJFhmGoe/fu+do/AAD3O8MwcnxkBVrp2h2JbtcmNDRU3333nc6ePavk5GRt3bpVERERNseKi4vLcT8VKlSQdO33vmEYNgHb2dlZb7/9ts6dO6eTJ09KkkqVKpWncyTUmsRTTz2lS5cuqXHjxho+fLief/55DRkyRNK1v6AaNmyoRx55RE2bNpVhGFq9erXNKKurq6v++c9/6vHHH1fz5s3l7u6u5cuXW9eHhYXptdde00svvaRGjRrpwoULeuqpp25aj7+/vxYtWqTPPvtMNWrU0JQpUzR9+vR8ndvgwYNVtWpVhYaGyt/f3+aODH379lWJEiXUt29fOTs752v/AACg+LMYN06kRJHTpk0b1atXT7NmzcrX9osWLdLIkSNz/Aznoi4xMVEPPPCAfvzxRzVo0CBP2yYnJ8vLy0sPvLhc6SXyN68Xd4eTvaGpjTP00nb7Ijf3DPRPUUbfFF1FuW8Sp3Qq7BLyJOv39/nz5+Xp6Znr7ZhTiyIpLS1NZ8+e1bhx4/Tggw/mOdBeL35sOz7BrIhJS0vT6tWrtScq7KbztlF46J+ii74puuibwsf0AxRJmzdvVlBQkH788Ue9++67hV0OAAAo4hipNYHrP4ErPyIjI20meZtBmzZtst1iDAAA4GYYqQUAAIDpEWoBAABgeoRaAAAAmB6hFgAAAKZHqAUAAIDpEWoBAABgeoRaAAAAmB6hFgAAAKZHqAUAAIDpEWoBAABgeoRaAAAAmB6hFgAAAKZHqAUAAIDpEWoBAABgeoRaAAAAmB6hFgAAAKZHqAUAAIDpEWoBAABgeoRaAAAAmB6hFgAAAKZHqAUAAIDpEWoBAABgeoRaAAAAmB6hFgAAAKZHqAUAAIDpEWoBAABgeoRaAAAAmB6hFgAAAKZHqAUAAIDpEWoBAABgeoRaAAAAmB6hFgAAAKZHqAUAAIDpEWoBAABgeoRaAAAAmB6hFgAAAKZHqAUAAIDpEWoBAABgeoRaAAAAmB6hFgAAAKZHqAUAAIDpEWoBAABgeoRaAAAAmB6hFgAAAKZHqAUAAIDpEWoBAABgeoRaAAAAmB6hFgAAAKZHqAUAAIDpEWoBAABgeoRaAAAAmB6hFgAAAKZHqAUAAIDpEWoBAABgeoRaAAAAmB6hFgAAAKZHqAUAAIDplSjsAoC7rcnk9Uov4VbYZeA6TvaGpjaWakV9pysZlsIuBzegf4qu+61vEqd0KuwSYCKM1AIAgCJt06ZN6ty5s4KDg2WxWLRixQqb9RaLJcfHtGnTrG1+//13de3aVSVLlpSnp6datGihjRs35ni8s2fPqkyZMrJYLEpKSrplbefOnVO/fv3k5+enxx9/XEOGDFFKSsqdnjLygVBrAnFxcbn6j1WhQgXNmjWrQI4ZFRWlevXqFci+AAC4ExcvXlTdunX19ttv57j+5MmTNo8PP/xQFotFPXv2tLZ55JFHlJ6erg0bNmjHjh2qW7euHnnkEZ06dSrb/gYOHKg6derkqrZ+/fpp7969WrNmjcaNG6cffvhBQ4YMyd+J4o4QaougNm3aaOTIkdbnzZo108mTJ+Xl5SVJWrRokby9vQunuDuUmJiogQMHKiQkRC4uLnrggQc0YcIEXb161aZNTn9xb9u2rRArBwAUloiICL3++uvq3r17jusDAwNtHitXrtRDDz2kihUrSpL++usvHThwQC+//LLq1KmjypUra8qUKUpNTdWePXts9jVv3jwlJSVp9OjRt61r//79iomJ0YIFC9S4cWPVqFFDM2fO1LJly3TixIk7P3HkCXNqTcDR0VGBgYGFXUaB+O2335SZman58+erUqVK2rNnjwYPHqyLFy9q+vTpNm3XrVunmjVrWp/7+fnd63IBACbzv//9T99++60++ugj6zI/Pz9VrVpVixcvVoMGDeTk5KT58+crICBADRs2tLbbt2+fJk6cqPj4eB0+fPi2x9q6dau8vb0VGhqqtLQ0SVK7du1kZ2en+Pj4m4Zw3B2M1BYxkZGR+v777zV79mzrCOWiRYus0w/i4uI0YMAAnT9/3ro+Kioqx30lJSVp0KBB8vf3l6enp9q2bavdu3fnqZ758+erbNmycnV1Va9evXT+/HmbWrt166Y333xTpUqVkre3tyZOnKj09HSNGTNGvr6+KlOmjBYuXGjdJjw8XAsXLlSHDh1UsWJFdenSRaNHj9aXX36Z7dh+fn42f3k7ODjkqXYAwP3no48+koeHh3r06GFdZrFYtG7dOu3cuVMeHh5ydnbWv//9b8XExMjHx0eSdOXKFfXt21fTpk1TuXLlcnWsU6dOKSAgwGZZiRIl5Ovrm+O0BtxdjNQWMbNnz9bvv/+uWrVqaeLEiZKkvXv3Wtc3a9ZMs2bN0vjx45WQkCBJcnd3z3Ffjz32mFxcXLRmzRp5eXlp/vz5ateunX7//Xf5+vretpaDBw/q008/1ddff63k5GQNHDhQzzzzjJYsWWJts2HDBpUpU0abNm3S5s2bNXDgQG3ZskWtWrVSfHy8li9frn/84x96+OGHVaZMmRyPc/78+Rzr6dKliy5fvqwqVaropZdeUpcuXW5Z75UrV3TlyhXr8+TkZEmSk50he3vjtueLe8fJzrD5F0UL/VN03W99kzX6eaP09PSbrvvggw/Ut29f2dvbW9sYhqFhw4bJ399fGzdulIuLiz788EN17txZW7ZsUVBQkP75z3+qatWq6t27t9LS0pSenm6t4WbHysjIkGEYNm2y/s3IyLjpdri1/L5uhNoixsvLS46OjnJ1dbVOOfjtt9+s6x0dHeXl5SWLxXLLKQk//PCDtm/frtOnT8vJyUmSNH36dK1YsUKff/55riaxX758WYsXL1bp0qUlSXPmzFGnTp00Y8YM67F9fX311ltvyc7OTlWrVtXUqVOVmpqqV155RZI0duxYTZkyRT/88IP69OmT7RgHDx7UnDlzbKYeuLu7a8aMGWrevLns7Oz0xRdfqFu3blqxYsUtg+3kyZMVHR2dbfm4+plydc247fni3psUmlnYJeAW6J+i637pm9WrV+e4fMeOHTm+e7d37179/vvvGjZsmM22u3fv1urVq/Xxxx8rKSlJSUlJioiI0KpVqzRu3Dj17NlTK1eu1NGjR/XFF1/Y7DMwMFCPPfaY+vbtm+14p0+f1okTJ2yOFRMTo7Nnz+r48eM3rR+3lpqamq/tCLXF1O7du5WSkpJtHuqlS5d06NChXO2jXLly1kArSU2bNlVmZqYSEhKsobZmzZqys/v/WSylSpVSrVq1rM/t7e3l5+en06dPZ9v/8ePHFR4erscee0yDBw+2Li9ZsqRGjRplfd6oUSOdOHFC06ZNu2WoHTt2rM12ycnJKlu2rF7faad0B/tcnTPuDSc7Q5NCM/XaT3a6kln877VpNvRP0XW/9c2eqLAclzds2FAdO3bMtvyLL75QgwYNNHz4cJvlmZnX/ggIDw+3eXfT3d1dlStXVseOHVW1alVdunTJum7Hjh0aPHiw4uLiVLFixWzTDCQpJCREc+fOVWBgoGrXrq3Y2FiVKFFChmFo6NChCg4Oztd53++y3mnNK0JtMZWSkqKgoCDFxcVlW1eQd0648S9li8WS47KsHyhZTpw4oYceekjNmjXTe++9d9vjNGnSRLGxsbds4+TkZB2Vvt6VTIvS74OblJvRlUzLfXEDebOif4qu+6Vvsn6fpKSk6ODBg9blx44d0969e+Xr62ud/5qcnKwvvvhCM2bMyPZ7qGXLlvLx8dGgQYM0fvx4ubi46P3331diYqK6dOkiBwcHVatWzWabrGtIateubf29uX37dj311FNav369SpcurTp16ig8PFzDhg3T3LlztX//fn3wwQfq06ePypcvf7delmIvv9fQEGqLIEdHR2Vk3Pzt8tutl6QGDRro1KlTKlGihCpUqJCvOo4ePaoTJ05Y/9Lctm2bdZrBnTh+/LgeeughNWzYUAsXLrQZ6b2ZXbt2KSgo6I6OCwAwp59++kkPPfSQ9XnWu3L9+/fXokWLJEnLli2TYRg5ThMoWbKkYmJi9Oqrr6pt27ZKS0tTzZo1tXLlStWtWzfXdaSmpiohIcFmzueSJUs0YsQIhYWFKTMzU4899pjmzp2bzzPFnSDUFkEVKlRQfHy8EhMT5e7unm2Us0KFCkpJSdH69etVt25dubq6ytXV1aZN+/bt1bRpU3Xr1k1Tp05VlSpVdOLECX377bfq3r27QkNDb1uHs7Oz+vfvr+nTpys5OVnPPfecevXqdUe3Fzt+/LjatGmj8uXLa/r06Tpz5ox1XdZ+P/roIzk6Oqp+/fqSpC+//FIffvihFixYkO/jAgDMq02bNjKMW18cN2TIkFteLxIaGqrvvvvujo6Z0zJfX1998sknSktL0+rVq9WxY0fu1lNIuKVXETR69GjZ29urRo0a8vf319GjR23WN2vWTEOHDlXv3r3l7++vqVOnZtuHxWLR6tWr1apVKw0YMEBVqlRRnz599Mcff6hUqVK5qqNSpUrq0aOHOnbsqA4dOqhOnTp655137ujcYmNjdfDgQa1fv15lypRRUFCQ9XG9SZMmqWHDhmrSpIlWrlyp5cuXa8CAAXd0bAAAUHxZjNv96QOYVHJysry8vPTAi8uVXsKtsMvBdZzsDU1tnKGXttvfF/MCzYb+Kbrut75JnNKpsEvINUZqC07W7+/z58/L09Mz19sx/QDFXvzYdnwaWRGT9cN/T1QYP/yLIPqn6KJvgJtj+sF9qmbNmnJ3d8/xcf2HKwAAAJgBI7X3qdWrV9/0EztyO+cWAACgqCiwUJuUlFSg9z/F3cX98wAAQHGSr+kH//rXv7R8+XLr8169esnPz0+lS5fW7t27C6w4AAAAIDfyFWrfffddlS1bVtK1WzTFxsZqzZo1ioiI0JgxYwq0QAAAAOB28jX94NSpU9ZQ+80336hXr17q0KGDKlSooCZNmhRogQAAAMDt5Guk1sfHR8eOHZMkxcTEqH379pIkwzBu+/GtAAAAQEHL10htjx499Pjjj6ty5co6e/asIiIiJEk7d+5UpUqVCrRAAAAA4HbyFWpnzpypChUq6NixY5o6darc3d0lSSdPntQzzzxToAUCAAAAt5OvUOvg4KDRo0dnW/7CCy/ccUEAAABAXuX7E8X+85//qEWLFgoODtYff/whSZo1a5ZWrlxZYMUBAAAAuZGvUDtv3jyNGjVKERERSkpKsl4c5u3trVmzZhVkfQAAAMBt5SvUzpkzR++//75effVV2dvbW5eHhobq119/LbDiAAAAgNzIV6g9cuSI6tevn225k5OTLl68eMdFAQAAAHmRr1AbEhKiXbt2ZVseExOj6tWr32lNAAAAQJ7k6+4Ho0aN0vDhw3X58mUZhqHt27dr6dKlmjx5shYsWFDQNQIAAAC3lK9QO2jQILm4uGjcuHFKTU3V448/ruDgYM2ePVt9+vQp6BoBAACAW8pzqE1PT9cnn3yisLAw9evXT6mpqUpJSVFAQMDdqA8AAAC4rTzPqS1RooSGDh2qy5cvS5JcXV0JtAAAAChU+bpQrHHjxtq5c2dB1wIAAADkS77m1D7zzDN68cUX9eeff6phw4Zyc3OzWV+nTp0CKQ4AAADIjXyF2qyLwZ577jnrMovFIsMwZLFYrJ8wBgAAANwL+Qq1R44cKeg6AAAAgHzLV6gtX758QdcBAAAA5Fu+Qu3ixYtvuf6pp57KVzEAAABAfuQr1D7//PM2z9PS0pSamipHR0e5uroSagEAAHBP5euWXn///bfNIyUlRQkJCWrRooWWLl1a0DUCAAAAt5SvUJuTypUra8qUKdlGcQEAAIC7rcBCrXTt08ZOnDhRkLsEAAAAbitfc2pXrVpl89wwDJ08eVJz585V8+bNC6QwAAAAILfyFWq7detm89xiscjf319t27bVjBkzCqIuAAAAINfyFWozMzMLug4AAAAg3/I1p3bixIlKTU3NtvzSpUuaOHHiHRcFAAAA5EW+Qm10dLRSUlKyLU9NTVV0dPQdFwUAAADkRb5CrWEYslgs2Zbv3r1bvr6+d1wUAAAAkBd5mlPr4+Mji8Uii8WiKlWq2ATbjIwMpaSkaOjQoQVeJAAAAHAreQq1s2bNkmEYevrppxUdHS0vLy/rOkdHR1WoUEFNmzYt8CIBAACAW8lTqO3fv78kKSQkRM2aNZODg8NdKQoAAADIi3zd0qt169bWry9fvqyrV6/arPf09LyzqgAAAIA8yNeFYqmpqRoxYoQCAgLk5uYmHx8fmwcAAABwL+Ur1I4ZM0YbNmzQvHnz5OTkpAULFig6OlrBwcFavHhxQdcIAAAA3FK+ph98/fXXWrx4sdq0aaMBAwaoZcuWqlSpksqXL68lS5aoX79+BV0nAAAAcFP5Gqk9d+6cKlasKOna/Nlz585Jklq0aKFNmzYVXHUAAABALuQr1FasWFFHjhyRJFWrVk2ffvqppGsjuN7e3gVWHAAAAJAb+Qq1AwYM0O7duyVJL7/8st5++205OzvrhRde0JgxYwq0QAAAAOB28jWn9oUXXrB+3b59e/3222/asWOHKlWqpDp16hRYcQAAAEBu5CvUXu/y5csqX768ypcvXxD1AAAAAHmWr+kHGRkZmjRpkkqXLi13d3cdPnxYkvTaa6/pgw8+KNACAQAAgNvJV6h94403tGjRIk2dOlWOjo7W5bVq1dKCBQsKrDgAAAAgN/IVahcvXqz33ntP/fr1k729vXV53bp19dtvvxVYcQAAAEBu5CvUHj9+XJUqVcq2PDMzU2lpaXdcFAAAAJAX+Qq1NWrU0H//+99syz///HPVr1//josCAAAA8iJfdz8YP368+vfvr+PHjyszM1NffvmlEhIStHjxYn3zzTcFXSMAAABwS3kaqT18+LAMw1DXrl319ddfa926dXJzc9P48eO1f/9+ff3113r44YfvVq0AAABAjvI0Ulu5cmWdPHlSAQEBatmypXx9ffXrr7+qVKlSd6s+AAAA4LbyNFJrGIbN8zVr1ujixYsFWhAAAACQV/m6UCzLjSEXAAAAKAx5CrUWi0UWiyXbMgAAAKAw5WlOrWEYioyMlJOTkyTp8uXLGjp0qNzc3GzaffnllwVXIXCHmkxer/QSbrdviHvGyd7Q1MZSrajvdCXj5n8YJ07pdA+rAgCYWZ5Cbf/+/W2eP/HEEwVaDAAAAJAfeQq1CxcuvFt1AECOoqKiFB0dbbOsatWq2T6S2zAMdezYUTExMfrqq6/UrVs367qjR49q2LBh2rhxo9zd3dW/f39NnjxZJUrc/EfguXPn9Oyzz+rrr7+WnZ2devbsqdmzZ8vd3b1Azw8AUDDy9eELuD9ERkYqKSlJK1asKOxScJ+rWbOm1q1bZ32eUxidNWtWjnP8MzIy1KlTJwUGBmrLli06efKknnrqKTk4OOjNN9+86TH79eunkydPKjY2VmlpaRowYICGDBmiTz75pGBOCgBQoO7o7ge4O6KiolSvXr3CLuOu2b17t/r27auyZcvKxcVF1atX1+zZs23axMXFWS9MvP5x6tSpQqoahalEiRIKDAy0PkqWLGmzfteuXZoxY4Y+/PDDbNuuXbtW+/bt08cff6x69eopIiJCkyZN0ttvv62rV6/meLz9+/crJiZGCxYsUJMmTdSiRQvNmTNHy5Yt04kTJ+7KOQIA7gyhthi72S/swrZjxw4FBATo448/1t69e/Xqq69q7Nixmjt3bra2CQkJOnnypPUREBBQCBWjsB04cEDBwcGqWLGi+vXrp6NHj1rXpaam6vHHH9fbb7+twMDAbNtu3bpVtWvXtvmQmLCwMCUnJ2vv3r05Hm/r1q3y9vZWaGiodVn79u1lZ2en+Pj4AjwzAEBBIdTeJTExMWrRooW8vb3l5+enRx55RIcOHbKu//PPP9W3b1/5+vrKzc1NoaGhio+P16JFixQdHa3du3dbRycXLVok6dq8wK5du8rd3V2enp7q1auX/ve//1n3mTXCu2DBAoWEhMjZ2VmS9Pnnn6t27dpycXGRn5+f2rdvn6cPzYiOjpa/v788PT01dOhQm7Dcpk0bPfvssxo5cqR8fHxUqlQpvf/++7p48aIGDBggDw8PVapUSWvWrLFu8/TTT2v27Nlq3bq1KlasqCeeeEIDBgzI8a4ZAQEBNiN0dnZ8y95vmjRpokWLFikmJkbz5s3TkSNH1LJlS124cEGS9MILL6hZs2bq2rVrjtufOnUq26ceZj2/2cj/qVOnsv0BVaJECfn6+vJuAQAUUcypvUsuXryoUaNGqU6dOkpJSdH48ePVvXt37dq1S6mpqWrdurVKly6tVatWKTAwUD///LMyMzPVu3dv7dmzRzExMdY5hF5eXsrMzLQG2u+//17p6ekaPny4evfurbi4OOtxDx48qC+++EJffvml7O3tdfLkSfXt21dTp05V9+7ddeHCBf33v//N9QdnrF+/Xs7OzoqLi1NiYqIGDBggPz8/vfHGG9Y2H330kV566SVt375dy5cv17Bhw/TVV1+pe/fueuWVVzRz5kw9+eSTOnr0qFxdXXM8zvnz5+Xr65tteb169XTlyhXVqlVLUVFRat68+U1rvXLliq5cuWJ9npycLElysjNkb88HhRQlTnaGzb83k5aWpvbt21ufV69eXQ0aNFClSpW0dOlSlSxZUhs2bND27duVlpZmbZeenm59npmZKcMwbNZnfX19u+tlZGRk2+b6dTktL06yzq+4n6cZ0TdFF31TcPL7GhJq75KePXvaPP/www/l7++vffv2acuWLTpz5ox+/PFHa5CrVKmSta27u7t1DmGW2NhY/frrrzpy5IjKli0rSVq8eLFq1qypH3/8UY0aNZJ0bcrB4sWL5e/vL0n6+eeflZ6erh49eqh8+fKSpNq1a+f6PBwdHfXhhx/K1dVVNWvW1MSJEzVmzBhNmjTJOmpat25djRs3TpI0duxYTZkyRSVLltTgwYMlSePHj9e8efP0yy+/6MEHH8x2jC1btmj58uX69ttvrcuCgoL07rvvKjQ0VFeuXNGCBQvUpk0bxcfHq0GDBjnWOnny5GxXyUvSuPqZcnXNyPU5496ZFJp5y/WrV6/OcXlAQIDWrl2rK1eu6NChQ9nm2Pbu3VvVq1fXG2+8oQsXLujAgQM2+8p6h+PgwYM5HuP06dM6ceKEzbqMjAydPXtWx48fv2ldxU1sbGxhl4CboG+KLvrmzqWmpuZrO0LtXXLgwAGNHz9e8fHx+uuvv5SZee2X99GjR7Vr1y7Vr18/x5HJm9m/f7/Kli1rDbSSVKNGDXl7e2v//v3WUFu+fHlroJWuBc527dqpdu3aCgsLU4cOHfToo4/Kx8cnV8etW7euzehq06ZNlZKSomPHjllDcp06dazr7e3t5efnZxOcs97qPX36dLb979mzR127dtWECRPUoUMH6/KqVauqatWq1ufNmjXToUOHNHPmTP3nP//JsdaxY8dq1KhR1ufJyckqW7asXt9pp3QH+1ydL+4NJztDk0Iz9dpPdrqSefMPX9gTFZZtWUpKis6ePavmzZvr0Ucf1V9//WWzvkGDBpo+fbo6deqkkJAQ2dnZ6fPPP1doaKh1SsGCBQvk6empwYMHWz9M5nohISGaO3euAgMDrX9ExcbGyjAMDR06VMHBwXdy+kVeWlqaYmNj9fDDD8vBwaGwy8F16Juii74pOFnvtOYVofYu6dy5s8qXL6/3339fwcHByszMVK1atXT16lW5uLjctePe+Olu9vb2io2N1ZYtW7R27VrNmTNHr776quLj4xUSElIgx7zxP6/FYrFZlnWbpaxgn2Xfvn1q166dhgwZYh3pvZXGjRvrhx9+uOl6JyenHAPKlUyL0m/xqVUoPFcyLbf8RDEHBweNHj3a+v/pxIkTmjBhguzt7fXEE0/I39/f5g+9LCEhIapSpYokqWPHjqpRo4aefvppTZ06VadOndKECRM0fPhw6z1nt2/frqeeekrr169X6dKlVadOHYWHh2vYsGF69913lZaWppEjR6pPnz7WP+buBw4ODvxyLqLom6KLvrlz+X39uOrmLjh79qwSEhI0btw4tWvXTtWrV9fff/9tXV+nTh3t2rVL586dy3F7R0dHZWTYvl1evXp1HTt2TMeOHbMu27dvn5KSklSjRo1b1mOxWNS8eXNFR0dr586dcnR01FdffZWrc9m9e7cuXbpkfb5t2za5u7vnGCTyYu/evXrooYfUv39/m/m5t7Jr1y4FBQXd0XFhPlkXVVatWlW9evWSn5+ftm3bZvOOxK3Y29vrm2++kb29vZo2baonnnhCTz31lCZOnGhtk5qaqoSEBJt5XEuWLFG1atXUrl07dezYUS1atNB7771X4OcHACgYjNTeBT4+PvLz89N7772noKAgHT16VC+//LJ1fd++ffXmm2+qW7dumjx5soKCgrRz504FBweradOmqlChgo4cOaJdu3apTJky8vDwUPv27VW7dm3169dPs2bNUnp6up555hm1bt3a5rZDN4qPj9f69evVoUMHBQQEKD4+XmfOnFH16tVzdS5Xr17VwIEDNW7cOCUmJmrChAkaMWLEHd2FYM+ePWrbtq3CwsI0atQo69Xk9vb21qAya9YshYSEqGbNmrp8+bIWLFigDRs2aO3atfk+Lsxp2bJleWqf00WQ5cuXv+U82DZt2mTbztfXlw9aAAATYaT2LrCzs9OyZcu0Y8cO1apVSy+88IKmTZtmXe/o6Ki1a9cqICBAHTt2VO3atTVlyhTZ21+b99mzZ0+Fh4froYcekr+/v5YuXSqLxaKVK1fKx8dHrVq1Uvv27VWxYkUtX778lrV4enpq06ZN6tixo6pUqaJx48ZpxowZioiIyNW5tGvXTpUrV1arVq3Uu3dvdenSRVFRUfl+baRrtxg7c+aMPv74YwUFBVkfWfOCpWth+sUXX1Tt2rXVunVr7d69W+vWrVO7du3u6NgAAKB4shi5vbcTYDLJycny8vLSAy8uV3oJt9tvgHvGyd7Q1MYZemm7/S3n1CZO6XQPq0KWtLQ0rV69Wh07dmRuYBFD3xRd9E3Byfr9ff78eXl6euZ6O6YfoNiLH9tOfn5+hV0GrpP1w39PVBg//AEABYLpB/cxd3f3mz7++9//FnZ5AAAAucZI7X1s165dN11XunTpe1cIAADAHSLU3seu/xQzAAAAM2P6AQAAAEyPUAsAAADTI9QCAADA9Ai1AAAAMD1CLQAAAEyPUAsAAADTI9QCAADA9Ai1AAAAMD1CLQAAAEyPUAsAAADTI9QCAADA9Ai1AAAAMD1CLQAAAEyPUAsAAADTI9QCAADA9Ai1AAAAMD1CLQAAAEyPUAsAAADTI9QCAADA9Ai1AAAAMD1CLQAAAEyPUAsAAADTI9QCAADA9Ai1AAAAMD1CLQAAAEyPUAsAAADTI9QCAADA9Ai1AAAAMD1CLQAAAEyPUAsAAADTI9QCAADA9Ai1AAAAMD1CLQAAAEyPUAsAAADTI9QCAADA9Ai1AAAAMD1CLQAAAEyPUAsAAADTI9QCAADA9Ai1AAAAMD1CLQAAAEyPUAsAAADTI9QCAADA9Ai1AAAAMD1CLQAAAEyPUAsAAADTI9QCAADA9Ai1AAAAMD1CLQAAAEyPUAsAAADTI9QCAADA9Ai1AAAAMD1CLQAAAEyPUAsAAADTI9QCAADA9EoUdgHA3dZk8nqll3Ar7DJyLXFKp8IuAQAA02GkFgAAAKZHqAWKoMmTJ6tRo0by8PBQQECAunXrpoSEBJs27733ntq0aSNPT09ZLBYlJSXluK9vv/1WTZo0kYuLi3x8fNStW7dbHtswDI0fP15BQUFycXFR+/btdeDAgQI6MwAA7o4iFWrj4uJu+cs5S4UKFTRr1qx7UlNOx7NYLFqxYsU9O35huV/Osyj6/vvvNXz4cG3btk2xsbFKS0tThw4ddPHiRWub1NRUhYeH65VXXrnpfr744gs9+eSTGjBggHbv3q3Nmzfr8ccfv+Wxp06dqrfeekvvvvuu4uPj5ebmprCwMF2+fLnAzg8AgIJWqKG2TZs2GjlypPV5s2bNdPLkSXl5eUmSFi1aJG9v78IprgAkJibKYrFo165dhV1KkfL++++rZcuW8vHxkY+Pj9q3b6/t27fbtImMjJTFYrF5hIeHF1LF915MTIwiIyNVs2ZN1a1bV4sWLdLRo0e1Y8cOa5uRI0fq5Zdf1oMPPpjjPtLT0/X8889r2rRpGjp0qKpUqaIaNWqoV69eNz2uYRiaNWuWxo0bp65du6pOnTpavHixTpw4wR84AIAirUiN1Do6OiowMFAWi6WwSyly0tLSCruEAhMXF6e+fftq48aN2rp1q8qWLasOHTro+PHjNu3Cw8N18uRJ62Pp0qWFVHHhO3/+vCTJ19c319v8/PPPOn78uOzs7FS/fn0FBQUpIiJCe/bsuek2R44c0alTp9S+fXvrMi8vLzVp0kRbt27N/wkAAHCXFVqojYyM1Pfff6/Zs2dbR+IWLVpknX4QFxenAQMG6Pz589b1UVFROe4rKSlJgwYNkr+/vzw9PdW2bVvt3r07V3UcOnRIXbt2ValSpeTu7q5GjRpp3bp1BXKOISEhkqT69evLYrGoTZs21nULFixQ9erV5ezsrGrVqumdd96xrssa4V2+fLlat24tZ2dnLVmyRJGRkerWrZvefPNNlSpVSt7e3po4caLS09M1ZswY+fr6qkyZMlq4cKF1X1evXtWIESMUFBQkZ2dnlS9fXpMnT871OZw8eVIRERFycXFRxYoV9fnnn2er89NPP1XLli3l4uKiRo0a6ffff9ePP/6o0NBQubu7KyIiQmfOnLFut2TJEj3zzDOqV6+eqlWrpgULFigzM1Pr16+3ObaTk5MCAwOtDx8fn1zXXZxkZmZq5MiRat68uWrVqpXr7Q4fPixJioqK0rhx4/TNN9/Ix8dHbdq00blz53Lc5tSpU5KkUqVK2SwvVaqUdR0AAEVRod3Sa/bs2fr9999Vq1YtTZw4UZK0d+9e6/pmzZpp1qxZGj9+vPUCGXd39xz39dhjj8nFxUVr1qyRl5eX5s+fr3bt2un333+/7chWSkqKOnbsqDfeeENOTk5avHixOnfurISEBJUrV+6OznH79u1q3Lix1q1bp5o1a8rR0VHStVA3fvx4zZ07V/Xr19fOnTs1ePBgubm5qX///tbtX375Zc2YMUP169eXs7Oz4uLitGHDBpUpU0abNm3S5s2bNXDgQG3ZskWtWrVSfHy8li9frn/84x96+OGHVaZMGb311ltatWqVPv30U5UrV07Hjh3TsWPHcn0Or732mqZMmaLZs2frP//5j/r06aNff/1V1atXt7aZMGGCZs2apXLlyunpp5/W448/Lg8PD82ePVuurq7q1auXxo8fr3nz5uV4jNTUVKWlpWXrq7i4OAUEBMjHx0dt27bV66+/Lj8/v5vWeuXKFV25csX6PDk5WZLkZGfI3t7I9TkXthtH5UeMGKE9e/Zo48aNOY7Yp6enW7e7fv3Vq1clXfs+6tKli6RrF5eFhIRo2bJlGjx4cK73lZmZKYvFUmDvGGTtpzi9A1Gc0D9FF31TdNE3BSe/r2GhhVovLy85OjrK1dVVgYGBkqTffvvNut7R0VFeXl6yWCzW9Tn54YcftH37dp0+fVpOTk6SpOnTp2vFihX6/PPPNWTIkFvWUbduXdWtW9f6fNKkSfrqq6+0atUqjRgx4k5OUf7+/pIkPz8/m3OYMGGCZsyYoR49eki6NqK7b98+zZ8/3ybUjhw50tomi6+vr9566y3Z2dmpatWqmjp1qlJTU60XC40dO1ZTpkzRDz/8oD59+ujo0aOqXLmyWrRoIYvFovLly+fpHB577DENGjRI0rXXJjY2VnPmzLEZWR49erTCwsIkSc8//7z69u2r9evXq3nz5pKkgQMHatGiRTc9xj//+U8FBwfbvOUdHh6uHj16KCQkRIcOHdIrr7yiiIgIbd26Vfb29jnuZ/LkyYqOjs62fFz9TLm6ZuTpvAvT6tWrrV+/9957io+P15tvvqlffvlFv/zyS7b2v/76qyRp7dq1Nn/4HT16VNK1dzKu36ePj482btyo0qVLZ9tX1mjsF198oYoVK1qX//bbbwoJCbHZT0GIjY0t0P2hYNE/RRd9U3TRN3cuNTU1X9uZ/sMXdu/erZSUlGwjeJcuXdKhQ4duu31KSoqioqL07bff6uTJk0pPT9elS5esgaCgXbx4UYcOHdLAgQNtRsrS09OtF8hlCQ0NzbZ9zZo1ZWf3/7NGSpUqZfOWtL29vfz8/HT69GlJ16Z5PPzww6patarCw8P1yCOPqEOHDrmut2nTptme33jhW506dWzqkaTatWvbLMuq50ZTpkzRsmXLFBcXJ2dnZ+vyPn36WL+uXbu26tSpowceeEBxcXFq165djvsaO3asRo0aZX2enJyssmXL6vWddkp3yDkIF0V7osJkGIZGjhypXbt2adOmTapcufJN27u5XftgiQ4dOthcWNmiRQvr6HbHjh0lXfvr9/z582rbtq112fUMw1BUVJTS0tKs65OTk3Xw4EG9/PLLOW6TH2lpaYqNjdXDDz8sBweHAtknCg79U3TRN0UXfVNwst5pzSvTh9qUlBQFBQUpLi4u27rc3Dlh9OjRio2N1fTp01WpUiW5uLjo0Ucftb51W9BSUlIkXbsDQJMmTWzW3TgCmRVWrnfjfxSLxZLjsszMTElSgwYNdOTIEa1Zs0br1q1Tr1691L59e5u5sXfq+uNnXeR347Kseq43ffp0TZkyRevWrbMJxjmpWLGiSpYsqYMHD9401Do5OVlH6693JdOi9AzzXHzo4OCgZ555Rp988olWrlwpX19fnT17VtK1dzhcXFwkXRtVPXXqlBITEyVdG0318PBQuXLl5OvrKz8/Pw0dOlQTJ05UhQoVVL58eU2bNk3StT8asvqoWrVqmjx5srp37y7p2jsEkydPVrVq1RQSEqLXXntNwcHBevTRRwv8B7WDgwM//Isw+qfoom+KLvrmzuX39SvUUOvo6KiMjJu/LXy79dK10Hbq1CmVKFFCFSpUyHMNmzdvVmRkpPUXekpKijUk3KmsObTXn0OpUqUUHBysw4cPq1+/fgVynNvx9PRU79691bt3bz366KMKDw/XuXPncnUl/bZt2/TUU0/ZPK9fv/4d1zR16lS98cYb+u6773Ickb7Rn3/+qbNnzyooKOiOj20GWfOPr7+4UJIWLlyoyMhISdK7775rM92iVatW2dpMmzZNJUqU0JNPPqlLly6pSZMm2rBhg81FdwkJCda7K0jSSy+9pIsXL2rIkCFKSkpSixYtFBMTYzOSDgBAUVOoobZChQqKj49XYmKi3N3ds43mVahQQSkpKVq/fr3q1q0rV1dXubq62rRp3769mjZtqm7dumnq1KmqUqWKTpw4oW+//Vbdu3e/bWCqXLmyvvzyS3Xu3FkWi0WvvfZajqOK+REQECAXFxfFxMSoTJkycnZ2lpeXl6Kjo/Xcc8/Jy8tL4eHhunLlin766Sf9/fffNm+fF4R///vfCgoKUv369WVnZ6fPPvtMgYGBub7/72effabQ0FC1aNFCS5Ys0fbt2/XBBx/cUU3/+te/NH78eH3yySeqUKGCdR6nu7u73N3dlZKSoujoaPXs2VOBgYE6dOiQXnrpJVWqVMk6d7e4M4zbX9gWFRV10zuCZHFwcND06dM1ffr0XB/LYrFo4sSJ1gs4AQAwg0K9T+3o0aNlb2+vGjVqyN/fP9s81mbNmmno0KHq3bu3/P39NXXq1Gz7sFgsWr16tVq1aqUBAwaoSpUq6tOnj/74449styXKyb///W/5+PioWbNm6ty5s8LCwtSgQYMCOb8SJUrorbfe0vz58xUcHKyuXbtKkgYNGqQFCxZo4cKFql27tlq3bq1FixZZbwFWkDw8PDR16lSFhoaqUaNGSkxM1OrVq23m5d5KdHS0li1bZr0J/9KlS1WjRo07qmnevHm6evWqHn30UQUFBVkfWcHL3t5ev/zyi7p06aIqVapo4MCBatiwof773//mOL0AAADAYuRmSAgwoeTkZHl5eemBF5crvUT2+clFVeKUToVdwl2Xlpam1atXq2PHjsw9K4Lon6KLvim66JuCk/X7+/z58/L09Mz1dqa/UAy4nfix7W55f1sAAGB+Repjcu+GmjVrWudq3vhYsmTJHe37zTffvOm+IyIiCugM7o4lS5bctPaaNWsWdnkAAAB5UuxHalevXn3TT6bIzZzbWxk6dKh69eqV47qs2y4VVV26dMl2S7EsvG0CAADMptiH2rx+glZe+Pr65uq2WEWRh4eHPDw8CrsMAACAAlHspx8AAACg+CPUAgAAwPQItQAAADA9Qi0AAABMj1ALAAAA0yPUAgAAwPQItQAAADA9Qi0AAABMj1ALAAAA0yPUAgAAwPQItQAAADA9Qi0AAABMj1ALAAAA0yPUAgAAwPQItQAAADA9Qi0AAABMj1ALAAAA0yPUAgAAwPQItQAAADA9Qi0AAABMj1ALAAAA0yPUAgAAwPQItQAAADA9Qi0AAABMj1ALAAAA0yPUAgAAwPQItQAAADA9Qi0AAABMj1ALAAAA0yPUAgAAwPQItQAAADA9Qi0AAABMj1ALAAAA0yPUAgAAwPQItQAAADA9Qi0AAABMj1ALAAAA0yPUAgAAwPQItQAAADA9Qi0AAABMj1ALAAAA0yPUAgAAwPQItQAAADA9Qi0AAABMj1ALAAAA0yPUAgAAwPQItQAAADA9Qi0AAABMj1ALAAAA0yPUAgAAwPQItQAAADA9Qi0AAABMj1ALAAAA0yPUAgAAwPQItQAAADA9Qi0AAABMj1ALAAAA0yPUAgAAwPQItQAAADA9Qi0AAABMj1ALAAAA0yPUAgAAwPRKFHYBwN1iGIYk6cKFC3JwcCjkanC9tLQ0paamKjk5mb4pguifoou+Kbrom4KTnJws6f9/j+cWoRbF1tmzZyVJISEhhVwJAADIqwsXLsjLyyvX7Qm1KLZ8fX0lSUePHs3TfwrcfcnJySpbtqyOHTsmT0/Pwi4HN6B/ii76puiibwqOYRi6cOGCgoOD87QdoRbFlp3dtSnjXl5e/IApojw9PembIoz+Kbrom6KLvikY+RmM4kIxAAAAmB6hFgAAAKZHqEWx5eTkpAkTJsjJyamwS8EN6Juijf4puuiboou+KXwWI6/3SwAAAACKGEZqAQAAYHqEWgAAAJgeoRYAAACmR6gFAACA6RFqUSy9/fbbqlChgpydndWkSRNt3769sEsq9qKiomSxWGwe1apVs66/fPmyhg8fLj8/P7m7u6tnz5763//+Z7OPo0ePqlOnTnJ1dVVAQIDGjBmj9PT0e30qxcKmTZvUuXNnBQcHy2KxaMWKFTbrDcPQ+PHjFRQUJBcXF7Vv314HDhywaXPu3Dn169dPnp6e8vb21sCBA5WSkmLT5pdfflHLli3l7OyssmXLaurUqXf71Ezvdn0TGRmZ7f9SeHi4TRv65u6YPHmyGjVqJA8PDwUEBKhbt25KSEiwaVNQP8vi4uLUoEEDOTk5qVKlSlq0aNHdPr1ij1CLYmf58uUaNWqUJkyYoJ9//ll169ZVWFiYTp8+XdilFXs1a9bUyZMnrY8ffvjBuu6FF17Q119/rc8++0zff/+9Tpw4oR49eljXZ2RkqFOnTrp69aq2bNmijz76SIsWLdL48eML41RM7+LFi6pbt67efvvtHNdPnTpVb731lt59913Fx8fLzc1NYWFhunz5srVNv379tHfvXsXGxuqbb77Rpk2bNGTIEOv65ORkdejQQeXLl9eOHTs0bdo0RUVF6b333rvr52dmt+sbSQoPD7f5v7R06VKb9fTN3fH9999r+PDh2rZtm2JjY5WWlqYOHTro4sWL1jYF8bPsyJEj6tSpkx566CHt2rVLI0eO1KBBg/Tdd9/d0/MtdgygmGncuLExfPhw6/OMjAwjODjYmDx5ciFWVfxNmDDBqFu3bo7rkpKSDAcHB+Ozzz6zLtu/f78hydi6dathGIaxevVqw87Ozjh16pS1zbx58wxPT0/jypUrd7X24k6S8dVXX1mfZ2ZmGoGBgca0adOsy5KSkgwnJydj6dKlhmEYxr59+wxJxo8//mhts2bNGsNisRjHjx83DMMw3nnnHcPHx8emf/75z38aVatWvctnVHzc2DeGYRj9+/c3unbtetNt6Jt75/Tp04Yk4/vvvzcMo+B+lr300ktGzZo1bY7Vu3dvIyws7G6fUrHGSC2KlatXr2rHjh1q3769dZmdnZ3at2+vrVu3FmJl94cDBw4oODhYFStWVL9+/XT06FFJ0o4dO5SWlmbTL9WqVVO5cuWs/bJ161bVrl1bpUqVsrYJCwtTcnKy9u7de29PpJg7cuSITp06ZdMfXl5eatKkiU1/eHt7KzQ01Nqmffv2srOzU3x8vLVNq1at5OjoaG0TFhamhIQE/f333/fobIqnuLg4BQQEqGrVqho2bJjOnj1rXUff3Dvnz5+XJPn6+koquJ9lW7dutdlHVht+T90ZQi2Klb/++ksZGRk2P0wkqVSpUjp16lQhVXV/aNKkiRYtWqSYmBjNmzdPR44cUcuWLXXhwgWdOnVKjo6O8vb2ttnm+n45depUjv2WtQ4FJ+v1vNX/k1OnTikgIMBmfYkSJeTr60uf3WXh4eFavHix1q9fr3/961/6/vvvFRERoYyMDEn0zb2SmZmpkSNHqnnz5qpVq5YkFdjPspu1SU5O1qVLl+7G6dwXShR2AQCKh4iICOvXderUUZMmTVS+fHl9+umncnFxKcTKAHPp06eP9evatWurTp06euCBBxQXF6d27doVYmX3l+HDh2vPnj021wagaGOkFsVKyZIlZW9vn+1K1P/9738KDAwspKruT97e3qpSpYoOHjyowMBAXb16VUlJSTZtru+XwMDAHPstax0KTtbreav/J4GBgdkurkxPT9e5c+fos3usYsWKKlmypA4ePCiJvrkXRowYoW+++UYbN25UmTJlrMsL6mfZzdp4enoyCHAHCLUoVhwdHdWwYUOtX7/euiwzM1Pr169X06ZNC7Gy+09KSooOHTqkoKAgNWzYUA4ODjb9kpCQoKNHj1r7pWnTpvr1119tflnHxsbK09NTNWrUuOf1F2chISEKDAy06Y/k5GTFx8fb9EdSUpJ27NhhbbNhwwZlZmaqSZMm1jabNm1SWlqatU1sbKyqVq0qHx+fe3Q2xd+ff/6ps2fPKigoSBJ9czcZhqERI0boq6++0oYNGxQSEmKzvqB+ljVt2tRmH1lt+D11hwr7SjWgoC1btsxwcnIyFi1aZOzbt88YMmSI4e3tbXMlKgreiy++aMTFxRlHjhwxNm/ebLRv394oWbKkcfr0acMwDGPo0KFGuXLljA0bNhg//fST0bRpU6Np06bW7dPT041atWoZHTp0MHbt2mXExMQY/v7+xtixYwvrlEztwoULxs6dO42dO3cakox///vfxs6dO40//vjDMAzDmDJliuHt7W2sXLnS+OWXX4yuXbsaISEhxqVLl6z7CA8PN+rXr2/Ex8cbP/zwg1G5cmWjb9++1vVJSUlGqVKljCeffNLYs2ePsWzZMsPV1dWYP3/+PT9fM7lV31y4cMEYPXq0sXXrVuPIkSPGunXrjAYNGhiVK1c2Ll++bN0HfXN3DBs2zPDy8jLi4uKMkydPWh+pqanWNgXxs+zw4cOGq6urMWbMGGP//v3G22+/bdjb2xsxMTH39HyLG0ItiqU5c+YY5cqVMxwdHY3GjRsb27ZtK+ySir3evXsbQUFBhqOjo1G6dGmjd+/exsGDB63rL126ZDzzzDOGj4+P4erqanTv3t04efKkzT4SExONiIgIw8XFxShZsqTx4osvGmlpaff6VIqFjRs3GpKyPfr3728YxrXber322mtGqVKlDCcnJ6Ndu3ZGQkKCzT7Onj1r9O3b13B3dzc8PT2NAQMGGBcuXLBps3v3bqNFixaGk5OTUbp0aWPKlCn36hRN61Z9k5qaanTo0MHw9/c3HBwcjPLlyxuDBw/O9kc5fXN35NQvkoyFCxda2xTUz7KNGzca9erVMxwdHY2KFSvaHAP5YzEMw7jXo8MAAABAQWJOLQAAAEyPUAsAAADTI9QCAADA9Ai1AAAAMD1CLQAAAEyPUAsAAADTI9QCAADA9Ai1AAAAMD1CLQDgroiMjJTFYsn2OHjwYGGXBqAYKlHYBQAAiq/w8HAtXLjQZpm/v38hVWMrLS1NDg4OhV0GgALCSC0A4K5xcnJSYGCgzcPe3j7Htn/88Yc6d+4sHx8fubm5qWbNmlq9erV1/d69e/XII4/I09NTHh4eatmypQ4dOiRJyszM1MSJE1WmTBk5OTmpXr16iomJsW6bmJgoi8Wi5cuXq3Xr1nJ2dtaSJUskSQsWLFD16tXl7OysatWq6Z133rmLrwiAu4WRWgBAkTB8+HBdvXpVmzZtkpubm/bt2yd3d3dJ0vHjx9WqVSu1adNGGzZskKenpzZv3qz09HRJ0uzZszVjxgzNnz9f9evX14cffqguXbpo7969qly5svUYL7/8smbMmKH69etbg+348eM1d+5c1a9fXzt37tTgwYPl5uam/v37F8rrACB/LIZhGIVdBACg+ImMjNTHH38sZ2dn67KIiAh99tlnObavU6eOevbsqQkTJmRb98orr2jZsmVKSEjIccpA6dKlNXz4cL3yyivWZY0bN1ajRo309ttvKzExUSEhIZo1a5aef/55a5tKlSpp0qRJ6tu3r3XZ66+/rtWrV2vLli35Om8AhYORWgDAXfPQQw9p3rx51udubm43bfvcc89p2LBhWrt2rdq3b6+ePXuqTp06kqRdu3apZcuWOQba5ORknThxQs2bN7dZ3rx5c+3evdtmWWhoqPXrixcv6tChQxo4cKAGDx5sXZ6eni4vL6+8nSiAQkeoBQDcNW5ubqpUqVKu2g4aNEhhYWH69ttvtXbtWk2ePFkzZszQs88+KxcXlwKrJ0tKSook6f3331eTJk1s2t1s3i+AoosLxQAARUbZsmU1dOhQffnll3rxxRf1/vvvS7o2NeG///2v0tLSsm3j6emp4OBgbd682Wb55s2bVaNGjZseq1SpUgoODtbhw4dVqVIlm0dISEjBnhiAu46RWgBAkTBy5EhFRESoSpUq+vvvv7Vx40ZVr15dkjRixAjNmTNHffr00dixY+Xl5aVt27apcePGqlq1qsaMGaMJEybogQceUL169bRw4ULt2rXLeoeDm4mOjtZzzz0nLy8vhYeH68qVK/rpp5/0999/a9SoUffitAEUEEItAKBIyMjI0PDhw/Xnn3/K09NT4eHhmjlzpiTJz89PGzZs0JgxY9S6dWvZ29urXr161nm0zz33nM6fP68XX3xRp0+fVo0aNbRq1SqbOx/kZNCgQXJ1ddW0adM0ZswYubm5qXbt2ho5cuTdPl0ABYy7HwAAAMD0mFMLAAAA0yPUAgAAwPQItQAAADA9Qi0AAABMj1ALAAAA0yPUAgAAwPQItQAAADA9Qi0AAABMj1ALAAAA0yPUAgAAwPQItQAAADA9Qi0AAABM7/8AyOW8sYShOpQAAAAASUVORK5CYII=", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "xgb.plot_importance(ranker, importance_type='weight')\n" - ] + "594c5ebcb9624b63b128536a46594211": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_0ab8ee0c2e1a42658ecbf01b0b28cf92", + "placeholder": "​", + "style": "IPY_MODEL_84206a53779249fbb34c78edb17fd1e0", + "value": " 4233/4233 [05:50<00:00, 11.77it/s]" + } }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Importing the model to Elasticsearch\n", - "\n", - "Once the model is trained you will be able to use Eland to send it to Elasticsearch.\n", - "\n", - "Please note that the `MLModel.import_ltr_model` method contains the LTRModelConfig object, so you do not need to send it separately to configure feature extraction.\n" - ] + "7ed336be71e74521a596a7d624c1e7d1": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "zAMwvqYlq9py", - "outputId": "c0f60ce3-fb07-47a5-9e37-fccbd1f30bcc" - }, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 14, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "from eland.ml import MLModel\n", - "\n", - "MLModel.import_ltr_model(\n", - " es_client=es_client,\n", - " model=ranker,\n", - " model_id='ltr-model-xgboost',\n", - " ltr_model_config=ltr_config,\n", - " es_if_exists = 'replace'\n", - ")" - ] + "84206a53779249fbb34c78edb17fd1e0": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Using the rescorer\n", - "Once the model is uploaded to ES, you will be able to use it as a rescorer into the _search API, as shown in the example after:\n", - "\n", - "```\n", - "POST /_search\n", - "{\n", - " \"query\" : {\n", - " \"multi_match\" : {\n", - " \"query\": \"star wars\",\n", - " \"field\": [\"title\", \"overview\", \"actors\", \"director\", \"tags\", \"characters\"]\n", - " }\n", - " },\n", - " \"rescore\" : {\n", - " \"window_size\" : 50,\n", - " \"learning_to_rank\" : {\n", - " \"model_id\": \"ltr-model-xgboost\",\n", - " \"params\": { \n", - " \"query\": \"star wars\"\n", - " }\n", - " }\n", - " }\n", - "}\n", - "```" - ] + "8c393bd522f6427f9978a89bc7dbdf3b": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "Xgr5MWWIrEk9", - "outputId": "e296cf37-afd1-43fb-e839-6c65cb65c072" - }, - "outputs": [ - { - "data": { - "text/plain": [ - "[('Star Wars', 10.972473, '11'),\n", - " ('Star Wars: The Clone Wars', 9.924128, '12180'),\n", - " ('After Porn Ends 2', 9.613241, '440249'),\n", - " ('Andor: A Disney+ Day Special Look', 8.982841, '1022100'),\n", - " (\"Family Guy Presents: It's a Trap!\", 8.840657, '278427'),\n", - " ('Star Wars: The Rise of Skywalker', 8.053794, '181812'),\n", - " ('Star Wars: The Force Awakens', 8.053794, '140607'),\n", - " ('Star Wars: The Last Jedi', 8.053794, '181808'),\n", - " ('Solo: A Star Wars Story', 8.053794, '348350'),\n", - " ('The Star Wars Holiday Special', 8.053794, '74849')]" - ] - }, - "execution_count": 15, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "query = 'star wars'\n", - "\n", - "# First let's display the result when not using the rescorer:\n", - "[\n", - " (movie['_source']['title'], movie['_score'], movie['_id']) for movie in es_client.search(\n", - " index=MOVIE_INDEX,\n", - " query={ \"multi_match\": { \"query\": query, \"fields\": [\"title\", \"overview\", \"actors\", \"director\", \"tags\", \"characters\"] } }\n", - " )['hits']['hits']\n", - "]" - ] + "a6d4eb3325444f28b11ba02c3d01ed83": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_7ed336be71e74521a596a7d624c1e7d1", + "placeholder": "​", + "style": "IPY_MODEL_ee1a6943af1e49e6a8851f27c9811c32", + "value": "100%" + } }, - { - "cell_type": "code", - "execution_count": 19, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[('Star Wars', 4.580671, '11'),\n", - " ('LEGO Star Wars Holiday Special', 1.9806126, '732670'),\n", - " ('Star Wars: The Clone Wars', 1.8576434, '12180'),\n", - " ('Star Wars: The Last Jedi', 1.7370756, '181808'),\n", - " ('LEGO Star Wars Summer Vacation', 1.6153007, '980804'),\n", - " ('Rogue One: A Star Wars Story', 1.5883299, '330459'),\n", - " ('Star Wars: The Rise of Skywalker', 1.5681647, '181812'),\n", - " ('Star Wars: The Force Awakens', 1.4801544, '140607'),\n", - " ('LEGO Star Wars Terrifying Tales', 1.4480213, '857702'),\n", - " ('Solo: A Star Wars Story', 1.1000854, '348350')]" - ] - }, - "execution_count": 19, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# Now let's display result using the rescorer:\n", - "[\n", - " (movie['_source']['title'], movie['_score'], movie['_id']) for movie in es_client.search(\n", - " index=MOVIE_INDEX,\n", - " query={ \"multi_match\": { \"query\": query, \"type\": \"best_fields\", \"fields\": [\"title\", \"overview\", \"actors\", \"director\", \"tags\", \"characters\"] } },\n", - " rescore={ \"learning_to_rank\": { \"model_id\": \"ltr-model-xgboost\", \"params\": {\"query\": query} }, \"window_size\": 100 }\n", - " )['hits']['hits']\n", - "]" - ] - } - ], - "metadata": { - "colab": { - "provenance": [] + "bff504814b434aec90b2cf020b08cfa9": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_8c393bd522f6427f9978a89bc7dbdf3b", + "max": 4233, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_549645ca4e7b48ef86cffdaa5507c56c", + "value": 4233 + } }, - "kernelspec": { - "display_name": ".venv", - "language": "python", - "name": "python3" + "e95447129da74d9ebc4c1d99165bd534": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.6" + "ee1a6943af1e49e6a8851f27c9811c32": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } }, - "widgets": { - "application/vnd.jupyter.widget-state+json": { - "0ab8ee0c2e1a42658ecbf01b0b28cf92": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "549645ca4e7b48ef86cffdaa5507c56c": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "ProgressStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "" - } - }, - "594c5ebcb9624b63b128536a46594211": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_0ab8ee0c2e1a42658ecbf01b0b28cf92", - "placeholder": "​", - "style": "IPY_MODEL_84206a53779249fbb34c78edb17fd1e0", - "value": " 4233/4233 [05:50<00:00, 11.77it/s]" - } - }, - "7ed336be71e74521a596a7d624c1e7d1": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "84206a53779249fbb34c78edb17fd1e0": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "8c393bd522f6427f9978a89bc7dbdf3b": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "a6d4eb3325444f28b11ba02c3d01ed83": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_7ed336be71e74521a596a7d624c1e7d1", - "placeholder": "​", - "style": "IPY_MODEL_ee1a6943af1e49e6a8851f27c9811c32", - "value": "100%" - } - }, - "bff504814b434aec90b2cf020b08cfa9": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "FloatProgressModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_8c393bd522f6427f9978a89bc7dbdf3b", - "max": 4233, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_549645ca4e7b48ef86cffdaa5507c56c", - "value": 4233 - } - }, - "e95447129da74d9ebc4c1d99165bd534": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "ee1a6943af1e49e6a8851f27c9811c32": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "f9cdfbc3972a4b84a557507567ca2965": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HBoxModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_a6d4eb3325444f28b11ba02c3d01ed83", - "IPY_MODEL_bff504814b434aec90b2cf020b08cfa9", - "IPY_MODEL_594c5ebcb9624b63b128536a46594211" - ], - "layout": "IPY_MODEL_e95447129da74d9ebc4c1d99165bd534" - } - } - } + "f9cdfbc3972a4b84a557507567ca2965": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_a6d4eb3325444f28b11ba02c3d01ed83", + "IPY_MODEL_bff504814b434aec90b2cf020b08cfa9", + "IPY_MODEL_594c5ebcb9624b63b128536a46594211" + ], + "layout": "IPY_MODEL_e95447129da74d9ebc4c1d99165bd534" + } } - }, - "nbformat": 4, - "nbformat_minor": 0 + } + } + }, + "nbformat": 4, + "nbformat_minor": 0 } diff --git a/notebooks/learning-to-rank/sample_data/movies_judgments.csv.gz b/notebooks/learning-to-rank/sample_data/movies_judgments.tsv.gz similarity index 100% rename from notebooks/learning-to-rank/sample_data/movies_judgments.csv.gz rename to notebooks/learning-to-rank/sample_data/movies_judgments.tsv.gz