lecture_12_generalized_matrix_factorization.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "verified-accommodation",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The autoreload extension is already loaded. To reload it, use:\n",
      "  %reload_ext autoreload\n"
     ]
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from IPython.display import Markdown, display, HTML\n",
    "from collections import defaultdict, deque\n",
    "\n",
    "from livelossplot import PlotLosses\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "\n",
    "# Fix the dying kernel problem (only a problem in some installations - you can remove it, if it works without it)\n",
    "import os\n",
    "os.environ['KMP_DUPLICATE_LIB_OK'] = 'True'\n",
    "os.environ['CUDA_LAUNCH_BLOCKING'] = '1'"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "educated-tourist",
   "metadata": {},
   "source": [
    "# Load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "prepared-fraction",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>item_id</th>\n",
       "      <th>title</th>\n",
       "      <th>genres</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>118</th>\n",
       "      <td>145</td>\n",
       "      <td>Bad Boys (1995)</td>\n",
       "      <td>Action|Comedy|Crime|Drama|Thriller</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>143</th>\n",
       "      <td>171</td>\n",
       "      <td>Jeffrey (1995)</td>\n",
       "      <td>Comedy|Drama</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>194</th>\n",
       "      <td>228</td>\n",
       "      <td>Destiny Turns on the Radio (1995)</td>\n",
       "      <td>Comedy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199</th>\n",
       "      <td>233</td>\n",
       "      <td>Exotica (1994)</td>\n",
       "      <td>Drama</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>230</th>\n",
       "      <td>267</td>\n",
       "      <td>Major Payne (1995)</td>\n",
       "      <td>Comedy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>313</th>\n",
       "      <td>355</td>\n",
       "      <td>Flintstones, The (1994)</td>\n",
       "      <td>Children|Comedy|Fantasy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>379</th>\n",
       "      <td>435</td>\n",
       "      <td>Coneheads (1993)</td>\n",
       "      <td>Comedy|Sci-Fi</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>419</th>\n",
       "      <td>481</td>\n",
       "      <td>Kalifornia (1993)</td>\n",
       "      <td>Drama|Thriller</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>615</th>\n",
       "      <td>780</td>\n",
       "      <td>Independence Day (a.k.a. ID4) (1996)</td>\n",
       "      <td>Action|Adventure|Sci-Fi|Thriller</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>737</th>\n",
       "      <td>959</td>\n",
       "      <td>Of Human Bondage (1934)</td>\n",
       "      <td>Drama</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of interactions left: 1170\n"
     ]
    }
   ],
   "source": [
    "ml_ratings_df = pd.read_csv(os.path.join(\"data\", \"movielens_small\", \"ratings.csv\")).rename(columns={'userId': 'user_id', 'movieId': 'item_id'})\n",
    "ml_movies_df = pd.read_csv(os.path.join(\"data\", \"movielens_small\", \"movies.csv\")).rename(columns={'movieId': 'item_id'})\n",
    "ml_df = pd.merge(ml_ratings_df, ml_movies_df, on='item_id')\n",
    "\n",
    "# Filter the data to reduce the number of movies\n",
    "seed = 6789\n",
    "rng = np.random.RandomState(seed=seed)\n",
    "left_ids = rng.choice(ml_movies_df['item_id'], size=100, replace=False)\n",
    "\n",
    "ml_ratings_df = ml_ratings_df.loc[ml_ratings_df['item_id'].isin(left_ids)]\n",
    "ml_movies_df = ml_movies_df.loc[ml_movies_df['item_id'].isin(left_ids)]\n",
    "ml_df = ml_df.loc[ml_df['item_id'].isin(left_ids)]\n",
    "\n",
    "display(HTML(ml_movies_df.head(10).to_html()))\n",
    "\n",
    "print(\"Number of interactions left: {}\".format(len(ml_ratings_df)))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "opponent-prediction",
   "metadata": {},
   "source": [
    "# Generalized Matrix Factorization (GMF)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "fancy-return",
   "metadata": {},
   "outputs": [],
   "source": [
    "from recommenders.recommender import Recommender\n",
    "\n",
    "\n",
    "class GMFModel(nn.Module):\n",
    "    def __init__(self, n_items, n_users, embedding_dim, seed):\n",
    "        super().__init__()\n",
    "\n",
    "        self.seed = torch.manual_seed(seed)\n",
    "        self.item_embedding = nn.Embedding(n_items, embedding_dim)\n",
    "        self.user_embedding = nn.Embedding(n_users, embedding_dim)\n",
    "        self.fc = nn.Linear(embedding_dim, 1, bias=False)\n",
    "\n",
    "    def forward(self, x):\n",
    "        user_ids = x[:, 0]\n",
    "        item_ids = x[:, 1]\n",
    "        user_embedding = self.user_embedding(user_ids)\n",
    "        item_embedding = self.item_embedding(item_ids)\n",
    "        x = self.fc(user_embedding * item_embedding)\n",
    "        x = torch.sigmoid(x)\n",
    "\n",
    "        return x\n",
    "\n",
    "\n",
    "class GMFRecommender(Recommender):\n",
    "    \"\"\"\n",
    "    General Matrix Factorization recommender as described in:\n",
    "    - He X., Liao L., Zhang H., Nie L., Hu X., Chua T., Neural Collaborative Filtering, WWW Conference, 2017\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self, seed=6789, n_neg_per_pos=5, print_type=None, **params):\n",
    "        super().__init__()\n",
    "        self.recommender_df = pd.DataFrame(columns=['user_id', 'item_id', 'score'])\n",
    "        self.interactions_df = None\n",
    "        self.item_id_mapping = None\n",
    "        self.user_id_mapping = None\n",
    "        self.item_id_reverse_mapping = None\n",
    "        self.user_id_reverse_mapping = None\n",
    "        self.r = None\n",
    "        self.most_popular_items = None\n",
    "        \n",
    "        self.nn_model = None\n",
    "        self.optimizer = None\n",
    "        \n",
    "        self.n_neg_per_pos = n_neg_per_pos\n",
    "        if 'n_epochs' in params:  # number of epochs (each epoch goes through the entire training set)\n",
    "            self.n_epochs = params['n_epochs']\n",
    "        else:\n",
    "            self.n_epochs = 10\n",
    "        if 'lr' in params:  # learning rate\n",
    "            self.lr = params['lr']\n",
    "        else:\n",
    "            self.lr = 0.01\n",
    "        if 'weight_decay' in params:  # weight decay (L2 regularization)\n",
    "            self.weight_decay = params['weight_decay']\n",
    "        else:\n",
    "            self.weight_decay = 0.001\n",
    "        if 'embedding_dim' in params:\n",
    "            self.embedding_dim = params['embedding_dim']\n",
    "        else:\n",
    "            self.embedding_dim = 4\n",
    "        if 'batch_size' in params:\n",
    "            self.batch_size = params['batch_size']\n",
    "        else:\n",
    "            self.batch_size = 64\n",
    "        if 'device' in params:\n",
    "            self.device = params['device']\n",
    "        else:\n",
    "            self.device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "        \n",
    "        if 'should_recommend_already_bought' in params:\n",
    "            self.should_recommend_already_bought = params['should_recommend_already_bought']\n",
    "        else:\n",
    "            self.should_recommend_already_bought = False\n",
    "        \n",
    "        if 'train' in params:\n",
    "            self.train = params['train']\n",
    "        else:\n",
    "            self.train = False\n",
    "        self.validation_set_size = 0.2\n",
    "        \n",
    "        self.seed = seed\n",
    "        self.rng = np.random.RandomState(seed=seed)\n",
    "        torch.manual_seed(seed)\n",
    "        \n",
    "        if 'should_save_model' in params:\n",
    "            self.should_save_model = params['should_save_model']\n",
    "        self.print_type = print_type\n",
    "\n",
    "    def fit(self, interactions_df, users_df, items_df):\n",
    "        \"\"\"\n",
    "        Training of the recommender.\n",
    "\n",
    "        :param pd.DataFrame interactions_df: DataFrame with recorded interactions between users and items\n",
    "            defined by user_id, item_id and features of the interaction.\n",
    "        :param pd.DataFrame users_df: DataFrame with users and their features defined by\n",
    "            user_id and the user feature columns.\n",
    "        :param pd.DataFrame items_df: DataFrame with items and their features defined\n",
    "            by item_id and the item feature columns.\n",
    "        \"\"\"\n",
    "\n",
    "        del users_df, items_df\n",
    "\n",
    "        # Shift item ids and user ids so that they are consecutive\n",
    "\n",
    "        unique_item_ids = interactions_df['item_id'].unique()\n",
    "        self.item_id_mapping = dict(zip(unique_item_ids, list(range(len(unique_item_ids)))))\n",
    "        self.item_id_reverse_mapping = dict(zip(list(range(len(unique_item_ids))), unique_item_ids))\n",
    "        unique_user_ids = interactions_df['user_id'].unique()\n",
    "        self.user_id_mapping = dict(zip(unique_user_ids, list(range(len(unique_user_ids)))))\n",
    "        self.user_id_reverse_mapping = dict(zip(list(range(len(unique_user_ids))), unique_user_ids))\n",
    "\n",
    "        interactions_df = interactions_df.copy()\n",
    "        interactions_df['item_id'] = interactions_df['item_id'].map(self.item_id_mapping)\n",
    "        interactions_df['user_id'] = interactions_df['user_id'].map(self.user_id_mapping)\n",
    "\n",
    "        # Get the number of items and users\n",
    "\n",
    "        self.interactions_df = interactions_df.copy()\n",
    "        n_users = np.max(interactions_df['user_id']) + 1\n",
    "        n_items = np.max(interactions_df['item_id']) + 1\n",
    "\n",
    "        # Get the user-item interaction matrix (mapping to int is necessary because of how iterrows works)\n",
    "        r = np.zeros(shape=(n_users, n_items))\n",
    "        for idx, interaction in interactions_df.iterrows():\n",
    "            r[int(interaction['user_id'])][int(interaction['item_id'])] = 1\n",
    "\n",
    "        self.r = r\n",
    "        \n",
    "        # Indicate positive interactions\n",
    "        \n",
    "        interactions_df.loc[:, 'interacted'] = 1\n",
    "\n",
    "        # Generate negative interactions\n",
    "        negative_interactions = []\n",
    "\n",
    "        i = 0\n",
    "        while i < self.n_neg_per_pos * len(interactions_df):\n",
    "            sample_size = 1000\n",
    "            user_ids = self.rng.choice(np.arange(n_users), size=sample_size)\n",
    "            item_ids = self.rng.choice(np.arange(n_items), size=sample_size)\n",
    "\n",
    "            j = 0\n",
    "            while j < sample_size and i < self.n_neg_per_pos * len(interactions_df):\n",
    "                if r[user_ids[j]][item_ids[j]] == 0:\n",
    "                    negative_interactions.append([user_ids[j], item_ids[j], 0])\n",
    "                    i += 1\n",
    "                j += 1\n",
    "        \n",
    "        interactions_df = pd.concat(\n",
    "            [interactions_df, pd.DataFrame(negative_interactions, columns=['user_id', 'item_id', 'interacted'])])\n",
    "        interactions_df = interactions_df.reset_index(drop=True)\n",
    "        \n",
    "        # Initialize losses and loss visualization\n",
    "        \n",
    "        if self.print_type is not None and self.print_type == 'live':\n",
    "            liveloss = PlotLosses()\n",
    "\n",
    "        training_losses = deque(maxlen=50)\n",
    "        training_avg_losses = []\n",
    "        training_epoch_losses = []\n",
    "        validation_losses = deque(maxlen=50)\n",
    "        validation_avg_losses = []\n",
    "        validation_epoch_losses = []\n",
    "        last_training_total_loss = 0.0\n",
    "        last_validation_total_loss = 0.0\n",
    "        \n",
    "        # Initialize the network\n",
    "        \n",
    "        self.nn_model = GMFModel(n_items, n_users, self.embedding_dim, self.seed)\n",
    "        self.nn_model.train()\n",
    "        self.nn_model.to(self.device)\n",
    "        self.optimizer = optim.Adam(self.nn_model.parameters(), lr=self.lr, weight_decay=self.weight_decay)\n",
    "        \n",
    "        # Split the data\n",
    "        \n",
    "        if self.train:\n",
    "            interaction_ids = self.rng.permutation(len(interactions_df))\n",
    "            train_validation_slice_idx = int(len(interactions_df) * (1 - self.validation_set_size))\n",
    "            training_ids = interaction_ids[:train_validation_slice_idx]\n",
    "            validation_ids = interaction_ids[train_validation_slice_idx:]\n",
    "        else:\n",
    "            interaction_ids = self.rng.permutation(len(interactions_df))\n",
    "            training_ids = interaction_ids\n",
    "            validation_ids = []\n",
    "        \n",
    "        # Train the model\n",
    "        \n",
    "        for epoch in range(self.n_epochs):\n",
    "            if self.print_type is not None and self.print_type == 'live':\n",
    "                logs = {}\n",
    "                \n",
    "            # Train\n",
    "            \n",
    "            training_losses.clear()\n",
    "            training_total_loss = 0.0\n",
    "            \n",
    "            self.rng.shuffle(training_ids)\n",
    "            \n",
    "            batch_idx = 0\n",
    "            n_batches = int(np.ceil(len(training_ids) / self.batch_size))\n",
    "            \n",
    "            for batch_idx in range(n_batches):\n",
    "                \n",
    "                batch_ids = training_ids[(batch_idx * self.batch_size):((batch_idx + 1) * self.batch_size)]\n",
    "                \n",
    "                batch = interactions_df.loc[batch_ids]\n",
    "                batch_input = torch.from_numpy(batch.loc[:, ['user_id', 'item_id']].values).long().to(self.device)\n",
    "                y_target = torch.from_numpy(batch.loc[:, ['interacted']].values).float().to(self.device)\n",
    "                \n",
    "                # Create responses\n",
    "\n",
    "                y = self.nn_model(batch_input).clip(0.000001, 0.999999)\n",
    "\n",
    "                # Define loss and backpropagate\n",
    "\n",
    "                self.optimizer.zero_grad()\n",
    "                loss = -(y_target * y.log() + (1 - y_target) * (1 - y).log()).sum()\n",
    "                \n",
    "                loss.backward()\n",
    "                self.optimizer.step()\n",
    "                \n",
    "                training_total_loss += loss.item()\n",
    "                \n",
    "                if self.print_type is not None and self.print_type == 'text':\n",
    "                    print(\"\\rEpoch: {}\\tBatch: {}\\tLast epoch - avg training loss: {:.2f} avg validation loss: {:.2f} loss: {}\".format(\n",
    "                        epoch, batch_idx, last_training_total_loss, last_validation_total_loss, loss), end=\"\")\n",
    "                \n",
    "                training_losses.append(loss.item())\n",
    "                training_avg_losses.append(np.mean(training_losses))\n",
    "                \n",
    "            # Validate\n",
    "            \n",
    "            if self.train:\n",
    "\n",
    "                validation_total_loss = 0.0\n",
    "\n",
    "                batch = interactions_df.loc[validation_ids]\n",
    "                batch_input = torch.from_numpy(batch.loc[:, ['user_id', 'item_id']].values).long().to(self.device)\n",
    "                y_target = torch.from_numpy(batch.loc[:, ['interacted']].values).float().to(self.device)\n",
    "\n",
    "                # Create responses\n",
    "\n",
    "                y = self.nn_model(batch_input).clip(0.000001, 0.999999)\n",
    "\n",
    "                # Calculate validation loss\n",
    "\n",
    "                loss = -(y_target * y.log() + (1 - y_target) * (1 - y).log()).sum()\n",
    "                validation_total_loss += loss.item()\n",
    "                \n",
    "            # Save and print epoch losses\n",
    "            \n",
    "            training_last_avg_loss = training_total_loss / len(training_ids)\n",
    "            \n",
    "            if self.train:\n",
    "                validation_last_avg_loss = validation_total_loss / len(validation_ids)\n",
    "\n",
    "            if self.print_type is not None and self.print_type == 'live' and epoch >= 0:\n",
    "                # A bound on epoch prevents showing extremely high losses in the first epochs\n",
    "                logs['loss'] = training_last_avg_loss\n",
    "                if self.train:\n",
    "                    logs['val_loss'] = validation_last_avg_loss\n",
    "                liveloss.update(logs)\n",
    "                liveloss.send()\n",
    "\n",
    "        # Find the most popular items for the cold start problem\n",
    "\n",
    "        offers_count = interactions_df.loc[:, ['item_id', 'user_id']].groupby(by='item_id').count()\n",
    "        offers_count = offers_count.sort_values('user_id', ascending=False)\n",
    "        self.most_popular_items = offers_count.index\n",
    "\n",
    "    def recommend(self, users_df, items_df, n_recommendations=1):\n",
    "        \"\"\"\n",
    "        Serving of recommendations. Scores items in items_df for each user in users_df and returns\n",
    "        top n_recommendations for each user.\n",
    "\n",
    "        :param pd.DataFrame users_df: DataFrame with users and their features for which\n",
    "            recommendations should be generated.\n",
    "        :param pd.DataFrame items_df: DataFrame with items and their features which should be scored.\n",
    "        :param int n_recommendations: Number of recommendations to be returned for each user.\n",
    "        :return: DataFrame with user_id, item_id and score as columns returning n_recommendations top recommendations\n",
    "            for each user.\n",
    "        :rtype: pd.DataFrame\n",
    "        \"\"\"\n",
    "\n",
    "        # Clean previous recommendations (iloc could be used alternatively)\n",
    "        self.recommender_df = self.recommender_df[:0]\n",
    "\n",
    "        # Handle users not in the training data\n",
    "\n",
    "        # Map item ids\n",
    "\n",
    "        items_df = items_df.copy()\n",
    "        items_df = items_df.loc[items_df['item_id'].isin(self.item_id_mapping)]\n",
    "        items_df['item_id'] = items_df['item_id'].map(self.item_id_mapping)\n",
    "\n",
    "        # Generate recommendations\n",
    "\n",
    "        for idx, user in users_df.iterrows():\n",
    "            recommendations = []\n",
    "\n",
    "            user_id = user['user_id']\n",
    "\n",
    "            if user_id in self.user_id_mapping:\n",
    "                \n",
    "                mapped_user_id = self.user_id_mapping[user_id]\n",
    "                \n",
    "                ids_list = items_df['item_id'].tolist()\n",
    "                id_to_pos = np.array([0]*len(ids_list))\n",
    "                for k in range(len(ids_list)):\n",
    "                    id_to_pos[ids_list[k]] = k\n",
    "                \n",
    "                net_input = torch.tensor(list(zip([mapped_user_id]*len(ids_list), ids_list))).to(self.device)\n",
    "                \n",
    "                scores = self.nn_model(net_input).flatten().detach().cpu().numpy()\n",
    "                \n",
    "                # Choose n recommendations based on highest scores\n",
    "                if not self.should_recommend_already_bought:\n",
    "                    x_list = self.interactions_df.loc[\n",
    "                        self.interactions_df['user_id'] == mapped_user_id]['item_id'].tolist()\n",
    "                    scores[id_to_pos[x_list]] = -np.inf\n",
    "\n",
    "                chosen_pos = np.argsort(-scores)[:n_recommendations]\n",
    "\n",
    "                for item_pos in chosen_pos:\n",
    "                    recommendations.append(\n",
    "                        {\n",
    "                            'user_id': self.user_id_reverse_mapping[mapped_user_id],\n",
    "                            'item_id': self.item_id_reverse_mapping[ids_list[item_pos]],\n",
    "                            'score': scores[item_pos]\n",
    "                        }\n",
    "                    )\n",
    "            else:  # For new users recommend most popular items\n",
    "                for i in range(n_recommendations):\n",
    "                    recommendations.append(\n",
    "                        {\n",
    "                            'user_id': user['user_id'],\n",
    "                            'item_id': self.item_id_reverse_mapping[self.most_popular_items[i]],\n",
    "                            'score': 1.0\n",
    "                        }\n",
    "                    )\n",
    "\n",
    "            user_recommendations = pd.DataFrame(recommendations)\n",
    "\n",
    "            self.recommender_df = pd.concat([self.recommender_df, user_recommendations])\n",
    "\n",
    "        return self.recommender_df\n",
    "    \n",
    "    def get_user_repr(self, user_id):\n",
    "        mapped_user_id = self.user_id_mapping[user_id]\n",
    "        return self.nn_model.user_embedding(torch.tensor(mapped_user_id).to(self.device)).detach().cpu().numpy()\n",
    "    \n",
    "    def get_item_repr(self, item_id):\n",
    "        mapped_item_id = self.item_id_mapping[item_id]\n",
    "        return self.nn_model.item_embedding(torch.tensor(mapped_item_id).to(self.device)).detach().cpu().numpy()\n",
    "\n",
    "    \n",
    "class MLPModel(nn.Module):\n",
    "    def __init__(self, n_items, n_users, embedding_dim, hidden_dim, seed):\n",
    "        super().__init__()\n",
    "\n",
    "        self.seed = torch.manual_seed(seed)\n",
    "        self.item_embedding = nn.Embedding(n_items, embedding_dim)\n",
    "        self.user_embedding = nn.Embedding(n_users, embedding_dim)\n",
    "        self.fc1 = nn.Linear(2 * embedding_dim, hidden_dim, bias=False)\n",
    "        self.fc2 = nn.Linear(hidden_dim, int(hidden_dim / 2), bias=True)\n",
    "        self.fc3 = nn.Linear(int(hidden_dim / 2), 1, bias=True)\n",
    "\n",
    "    def forward(self, x):\n",
    "        user = x[:, 0]\n",
    "        item = x[:, 1]\n",
    "        user_embedding = self.user_embedding(user)\n",
    "        item_embedding = self.item_embedding(item)\n",
    "        x = torch.cat([user_embedding, item_embedding], dim=1)\n",
    "        x = torch.relu(self.fc1(x))\n",
    "        x = torch.relu(self.fc2(x))\n",
    "        x = torch.sigmoid(self.fc3(x))\n",
    "\n",
    "        return x\n",
    "\n",
    "    \n",
    "class NeuMFModel(nn.Module):\n",
    "    def __init__(self, n_items, n_users, gmf_embedding_dim, mlp_embedding_dim, hidden_dim, seed):\n",
    "        super().__init__()\n",
    "\n",
    "        self.seed = torch.manual_seed(seed)\n",
    "\n",
    "        # GMF\n",
    "\n",
    "        self.gmf_user_embedding = nn.Embedding(n_users, gmf_embedding_dim)\n",
    "        self.gmf_item_embedding = nn.Embedding(n_items, gmf_embedding_dim)\n",
    "\n",
    "        # MLP\n",
    "\n",
    "        self.mlp_user_embedding = nn.Embedding(n_users, mlp_embedding_dim)\n",
    "        self.mlp_item_embedding = nn.Embedding(n_items, mlp_embedding_dim)\n",
    "        self.mlp_fc1 = nn.Linear(2 * mlp_embedding_dim, hidden_dim, bias=False)\n",
    "        self.mlp_fc2 = nn.Linear(hidden_dim, int(hidden_dim / 2), bias=False)\n",
    "\n",
    "        # Merge\n",
    "\n",
    "        self.fc = nn.Linear(gmf_embedding_dim + int(hidden_dim / 2), 1, bias=False)\n",
    "\n",
    "    def forward(self, x):\n",
    "        user = x[:, 0]\n",
    "        item = x[:, 1]\n",
    "\n",
    "        # GMF\n",
    "\n",
    "        gmf_user_embedding = self.gmf_user_embedding(user)\n",
    "        gmf_item_embedding = self.gmf_item_embedding(item)\n",
    "        gmf_x = gmf_user_embedding * gmf_item_embedding\n",
    "\n",
    "        # MLP\n",
    "\n",
    "        mlp_user_embedding = self.mlp_user_embedding(user)\n",
    "        mlp_item_embedding = self.mlp_item_embedding(item)\n",
    "        mlp_x = torch.cat([mlp_user_embedding, mlp_item_embedding], dim=1)\n",
    "        mlp_x = torch.relu(self.mlp_fc1(mlp_x))\n",
    "        mlp_x = torch.relu(self.mlp_fc2(mlp_x))\n",
    "\n",
    "        # Final score\n",
    "\n",
    "        x = torch.cat([gmf_x, mlp_x], dim=1)\n",
    "        x = torch.sigmoid(self.fc(x))\n",
    "\n",
    "        return x"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "expensive-offering",
   "metadata": {},
   "source": [
    "## Quick test of the recommender (training)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "nonprofit-roads",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAbkAAAI4CAYAAAD3UJfIAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAABTCUlEQVR4nO3dd3zV5d3/8dd1shMCZIcMCJsMwgqIIkMEBLeIgnW3zuptW7usd62td3vfHdafHdoWR+tGRRG1qIgLUVbYJIBsCCODlTCyr98fJ1LULMLZ5/18PM4jZ1zf7/kcorz5XucaxlqLiIhIIHJ4uwARERF3UciJiEjAUsiJiEjAUsiJiEjAUsiJiEjAUsiJiEjAUsiJiEjAUsiJeJAxZocxZoK36xAJFgo5EREJWAo5ES8zxkQYYx41xuxtuj1qjIloei3RGPO2MeawMeagMeZTY4yj6bWfGmP2GGOqjDGbjDHne/eTiPieUG8XICL8NzASGAxYYC7wc+AB4IdACZDU1HYkYI0x/YG7geHW2r3GmCwgxLNli/g+XcmJeN+1wEPW2jJrbTnwK+D6ptfqgG5AD2ttnbX2U+tccLYBiAByjDFh1tod1tqtXqlexIcp5ES8Lw3YecrjnU3PAfwB2ALMN8ZsM8bcB2Ct3QJ8H/glUGaMmWWMSUNEvkIhJ+J9e4Eepzzu3vQc1toqa+0PrbW9gEuBe7/87s1a+6K19tymYy3wO8+WLeL7FHIinhdmjIn88ga8BPzcGJNkjEkEfgE8D2CMudgY08cYY4AjOLspG40x/Y0x45sGqFQDJ4BG73wcEd+lkBPxvHk4Q+nLWyRQCKwF1gErgV83te0LLACOAouBx621H+H8Pu63QAWwH0gGfua5jyDiH4w2TRURkUClKzkREQlYCjkREQlYCjkREQlYCjkREQlYXlvWKzEx0WZlZXnr7UVEJACsWLGiwlqb1NLrXgu5rKwsCgsLvfX2IiISAIwxO1t7Xd2VIiISsBRyIiISsBRyIiISsBRyIiISsBRyIiISsBRyIiISsBRyIiISsBRyIiISsBRyIiISsBRyIiISsBRyIiISsBRyIiISsBRyIiISsBRyIiISsBRyIiISsBRyIiISsBRyIiISsBRyIiISsBRyIiISsBRyIiISsPw65GrrG3lrzV6std4uRUREfJBfh9yrK3bzXy+t4kevrqW6rsHb5YiIiI8J9XYBZ+Ka4d0pr6rh0QWb2VJWxd+vH0a3LlHeLktERHyEX1/JORyG70/ox8zrh7Gl7CiX/GURy7Yf9HZZIiLiI9oVcsaYycaYTcaYLcaY+5p5/f8ZY1Y33b4wxhx2eaWtmJSbyht3jSI2MoxvPbGE5xbv0Pd0IiLSdsgZY0KAx4ApQA5wjTEm59Q21tofWGsHW2sHA38BXndDra3qmxLLG3eNYky/JB6YW8R9r62jpl7f04mIBLP2XMmNALZYa7dZa2uBWcBlrbS/BnjJFcWdri5RYTx5QwF3n9eHlwt3M2PmEkorq71RioiI+ID2hFw6sPuUxyVNz32DMaYH0BP4sIXXbzPGFBpjCsvLy0+31nZxOAw/uqA/f7t2KJv2V3HxXxaxYqe+pxMRCUauHngyA5htrW22n9BaO9NaW2CtLUhKSnLxW3/VlIHdmPPdUUSHhzBj5hJeXLrLre8nIiK+pz0htwfIPOVxRtNzzZmBJ7sqK7bAZ3+CDW9BaTHUHv/Ky/1TY3nzrnM5u3ci989Zx/1z1lFb3+ix8kRExLvaM09uOdDXGNMTZ7jNAL719UbGmAFAHLDYpRW2Zs8KeP8XX30uNg0SekN8T4jvRZf43vxzSk8eTU7jL4t2sWl/FY9cPYiMuGhCHMZjpYqIiOeZ9gy1N8ZcCDwKhABPW2t/Y4x5CCi01r7Z1OaXQKS19htTDJpTUFBgCwsLO1r3f5w4DIe2w4GtcHA7HNwGB7c6fx776vd+JyKTWX8igdLGzlgMYaGhhIeGEh7mvEWEhxIZFkpEmPNnZHgYURHONsY4wDjAGMA4f5687/jmfeMARxiEhEJIeNv3HWEQHgORXSGqK0R2gZCwM//zEREJYMaYFdbaghZf99Z8MpeFXGuqK5tC78vg20516RfUVFbQ0NhIY0MDjY2NNDY2YhsbsLYRg8VBI87IasSBJYRGHIam1+zJ+/+5gYNGwGKs8ziDC/5cwzt9NfRO3m/6GRUH8b0gaQB0TmsKWhGR4NFWyPn1sl5tiuwMaYOdty+faro1x1pLZXU95VU1ztvRGiqafh6trud4bQPHa7/+s+l+TQPHautpPJltllAaCKWBcOoJpYGEKENyjIOkaAeJUQ4SohwkRhniIw1xkZAS1Ui38GocNUecV6jVh7/689AO2Nd0v+7YV4uP6AxJ/Ztu2c7gS+oPXTIUfiIStAI75E6TMYYuUWF0iQqjT3Kn0z7eWktNfSPHaxuoqq6j4mgtFUdrnLeq/9zfc7SG1RU1VByt5WhN/SlnCCEyLIIBqZnkpnUmJ60zubldGJAaS2RYyFffrL4WThyEA1ugfCOUbXT+/OI9WPX8f9qFdzol+PpDt0GQdS44vnY+EZEAFNjdlX6guq6B8ipn+G0rP0bR3kqK9x2haG8lVdXOAAxxGHonxZCb1oWcbp1PBmDX6PDmT3rsgDPwTr2VbYRjZc7XO2fAkGth8LUQ18NDn1RExPWC+zs5P2atpeTQCYr2HqF4byVFTbf9p6zgkhEXxYTsFK4cmkFeemdMW92Sxw/C9k9g5XOwtWm+fq+xMPQGGHAxhEa48ROJiLieQi7AHDhaQ/E+Z+Ct2nWIjzaVU1vfSN/kTkwdmsHlQ9Lat93Q4d2w+gVY9QIc2eUcxJI/HYZcD6l57v8gIiIuoJALcEeO1/Hvdft4fWUJhTsPYQyM6p3I1KHpXJCbSkxEG1+7NjbC9o9h5bOw8d/QUAtpQ2Ho9ZA3zTl4R0TERynkgsjOA8d4feUeXl9Vwu6DJ4gOD2FyXirThmYwslcCjrYmvx87AGtfhlXPQVkxhEZB7hVwzt2QkuuZDyEichoUckHIWkvhzkO8vrKEt9fso6qmnrQukVw+JJ0rh2XQO6mNkaPWOleTWfksrH8NGupgym9h2M2ajiAiPkUhF+Sq6xpYsKGU11aUsHBzBdZabjqnJz+c1K/trkyAYxXw+m2w9QMYeBVc/ChEnP70ChERd1DIyUllVdX8+YPNvLB0F906R/LQZXlMyElp+8DGRlj0R/jofyGhD1z1DKTktH2ciIibtRVyrt5qR3xYcmwkv758ILPvOIfYyDBuebaQO59f0fbGsg4HjPkx3DDXudrKE+OdozJFRHycQi4IDesRx1v/dS4/vqA/H2wsY8IfP+G5JTtpbGzjqr7nGLhjEWQUwNzvwht3fWN7IxERX6KQC1LhoQ7uOq8P878/hvzMLjzwxnqm/f1zNu2vav3A2BTnFd2Ynzjn2T15PlRs9kzRIiKnSSEX5LISY3j+O2fxyNWD2F5xjIv+/Cm/f3cj1XXNbu7u5AiB8f8N182Go6Uwcxysm+2xmkVE2kshJxhjmDo0gw9+OI7LBqfz+MdbueDRhSzaXNH6gX0mwO2fQkoevPYdePsHUNfG93siIh6kkJOT4mPC+ePVg3jxlrMwwHVPLeUHL6/mwNGalg/qkg43vQ2jvgeFT8NTE53794mI+ACFnHzDOX0Seff7Y/iv8X14e+1ervzb55S1NgIzJAwmPgTXzILDu+AfY6FE00NExPsUctKsyLAQfjipPy/dOpKyqhqufXJp61d0AP2nwB2fOhd7fvl650RyEREvUshJqwqy4nnqxuHsOnic655axuHjta0f0LU7TH/euaHr7G9DYysDWERE3EwhJ206u3cCM28oYGvZUW58ehmV1XWtH9AtHy76o3Pvug9/7ZkiRUSaoZCTdhnbL4nHrh1K0d5Kvv3P5RyrqW/9gCHXwdAbYdEjzi18RES8QCEn7TYxJ4U/zRjCyl2HuOWZwtbn0gFM+T10Gwxz7oQDWz1So4jIqRRyclouyu/Gw1cNYsn2A9z+3Apq6lsJurBImP6cc+3LV27QEmAi4nEKOTltU4dm8L9XDOSTL8q5+8VV1DU0tty4a3eY+iSUFsG/73XuVSci4iEKOemQa0Z055eX5PB+cSk/eHk1Da0t7tx3Aoy7D9a85JwwLiLiIe3YNVOkeTeN6kl1fSO/fWcjEaEh/GFaPg5HCzuHj/mJc4L4u/c5v6fLGObRWkUkOOlKTs7IHWN78/0JfXltZQkPzF1Pi5vwOhwwdSbEpjq/nzt2wLOFikhQUsjJGfve+X25Y2xvXli6i/95e0PLQRcdD1c/C8fKnQs6a6K4iLiZQk7OmDGGn07uz03nZPH0Z9v5w3ubWg66tCFw4R9g20fw8f95tlARCTr6Tk5cwhjDg5fkUFPfyOMfb6VnYgxXFWQ233jYjVCyDBb+AdILoP9kzxYrIkFDV3LiMsYYfnN5HiOy4vmft4spq2pl54ILH4bUfJhzGxzc7rkiRSSoKOTEpRwOw/9dOZDq+kZ++WZRyw3DopwTxTHwyvVQd8JjNYpI8FDIicv1TurE987vy7x1+3mvaH/LDeOyYOoTsH8dzPuRx+oTkeChkBO3uG1ML7K7deaBN9Zz5EQruxb0mwTn/gBWPa+NVkXE5RRy4hZhIQ5+d+VAKo7W8Nt3NrbeePSPIDpB2/KIiMsp5MRt8jO6csvoXry0bBdLtrUy+TuiE5x7r3NawY5FnitQRAKeQk7c6gcT+tE9Ppr7Xlvb+tY8w78DnVKdV3NaxFlEXEQhJ24VFR7C/00dyI4Dx3l0weaWG4ZFwZgfwa7FsPUDzxUoIgFNISduN6pPIlcXZPDEp9tYv+dIyw2H3ghduutqTkRcRiEnHvHfF+YQHxPOT19bS31L+8+FhsO4n8LeVbBpnmcLFJGApJATj+gSHcZDl+ZStLeSJz5tZYWT/BkQ3xs+/A00trIZq4hIOyjkxGOmDOzGBbkpPLrgC7ZXHGu+UUgonHc/lBVB8RzPFigiAUchJx710GV5hIc6uO+1tTS2tJt47lRIzoGP/hca6j1boIgEFIWceFRK50j++8Jslm4/yMuFu5tv5HA4r+YObIG1L3u2QBEJKAo58bjpwzMZ2Sue/523gdLKFnYqGHAxdBsEn/wW6ms9W6CIBAyFnHicMYbfTs2ntr6RB95Y3/wGq8bA+Afg8C5Y9ZznixSRgKCQE6/ISozhBxP7Mb+4lHfWt7BTQZ8JkDnSubmqtuIRkQ5QyInX3HJuT/LSO/OLuUUcPt5Ml6QxMP7nULUPCv/p+QJFxO8p5MRrQkMc/O7KfA4dr+U3/97QfKOeo6HnWFj0CNQc9WyBIuL3FHLiVblpXbjl3J68uqKELWUthNj4n8Oxclg207PFiYjfU8iJ1902phcRoQ6eWLit+QaZI6DvBfDZn+DEYY/WJiL+TSEnXpfQKYKrCzKZs2pPy1MKzrsfqg/Dksc9WpuI+DeFnPiEW0b3pL6xkX9+tqP5BmmDIftSWPw4HGtlA1YRkVMo5MQn9EiIYcrAbrywZCdV1XXNNzrvfqg9Cp//ybPFiYjfUsiJz7h9TC+qaup5admu5hskZ8PAq2DpTKgq9WxxIuKXFHLiM/IzunJO7wSeWrSd2voWttkZdx801DqnFIiItEEhJz7l9rG9Ka2sYe7qPc03SOgNQ66FwqfhSIlnixMRv6OQE58ypm8iA1JjmblwW8tb8Yz5ifPnokc9VpeI+CeFnPgUYwx3jO3N5rKjfPxFWfONumY695xbMwtqW9h8VUQEhZz4oIvyu5HWJZK/f9LC5HCAgpuhtgrWv+65wkTE7yjkxOeEhTj4zuheLNt+kJW7DjXfKPMsSBoAK/7l0dpExL8o5MQnzRieSZeoMGa2dDVnDAy7CfYUwv51Hq1NRPyHQk58UkxEKNeP7MF7xfvZVt7Cws350yEkQldzItIihZz4rBvPySIsxMETn25vvkF0POReAWtf0QAUEWmWQk58VlJsBNOGZfDayhLKqlpYuHnYTVBTCUVzPFqbiPgHhZz4tFtH96KuoZFnPt/RfIPuIyGxv3YOF5FmKeTEp/VMjGFybirPLd7J0Zr6bzbQABQRaYVCTnzebWN6UVldz6yWFm4eNKNpAMozni1MRHyeQk583pDucZzVM56nFm2nrqGZhZuj4yH3clj7MtQe93h9IuK7FHLiF+4Y25t9R6p5a83e5hucHICiFVBE5D8UcuIXxvVPon9KLP/4ZBvWNrNwc/ezIbGf5syJyFe0K+SMMZONMZuMMVuMMfe10OZqY0yxMabIGPOia8uUYGeM4dYxvdhUWsXHX5Q318B5NVeyHPav93h9IuKb2gw5Y0wI8BgwBcgBrjHG5HytTV/gZ8Aoa20u8H3XlyrB7tJBaaR2juQfn2xtvsGga5wDUFZqAIqIOLXnSm4EsMVau81aWwvMAi77WptbgcestYcArLUt7JEi0nHhoQ6+c25Plmw7yJrdh7/ZIDoeci6DNRqAIiJO7Qm5dGD3KY9Lmp47VT+gnzHmM2PMEmPMZFcVKHKqGSMyiY0MZebCFhZuHnYT1BzRCigiArhu4Eko0BcYB1wDPGGM6fr1RsaY24wxhcaYwvLyZr5XEWlDbGQY143swTvr97Gjopn1KnucowEoInJSe0JuD5B5yuOMpudOVQK8aa2ts9ZuB77AGXpfYa2daa0tsNYWJCUldbRmCXI3n5NFqMPBPz9rZuHmkwNQlkFpkcdrExHf0p6QWw70Ncb0NMaEAzOAN7/W5g2cV3EYYxJxdl+2sq2zSMcld45kUm4Kc9fspba+mcnhg66BkHCtgCIibYectbYeuBt4D9gAvGKtLTLGPGSMubSp2XvAAWNMMfAR8GNr7QF3FS0ydWg6h4/X8Ulz0wlODkCZpQEoIkGuXd/JWWvnWWv7WWt7W2t/0/TcL6y1bzbdt9bae621OdbagdbaWe4sWmR03yQSYsKZs6qk+QZfDkApfsOTZYmIj9GKJ+KXwkIcXDIojQUbyjhyou6bDXqMgoS+GoAiEuQUcuK3rhiSTm19I/PW7fvmi18OQNm9FEqLPV6biPgGhZz4rfyMLvRKimHOqq8P9m1ycgDKvzxal4j4DoWc+C1jDFOHpLNs+0F2H2xmgElMAmRfCms1AEUkWCnkxK9dNti5+M7c1S1czQ27CaqPQPFczxUlIj5DISd+LTM+mhFZ8by+ak/zW/BknQsJfdRlKRKkFHLi964Yms628mOs23Pkmy+eHICyBMo2eLw2EfEuhZz4vQsHdiM8xMHrK1sagPItDUARCVIKOfF7XaLCOD87mbfW7KWuoZllvmISIPsSWPMS1J3wfIEi4jUKOQkIVwxJ58CxWj7d3MLuFkNvdA5A2fhvzxYmIl6lkJOAMK5/MnHRYcxZtbf5BlmjITYN1r/m2cJExKsUchIQwkMdXJyfxvyi/VRVN7PMl8MBeVNh8/tw4pDnCxQRr1DIScC4fEg6NfWNvLN+f/MN8qZCYx1seMuzhYmI1yjkJGAM7d6VrIRo5rQ0yjJtKMT1VJelSBBRyEnAMMZw+ZB0lmw/wN7DzYyiNAYGToPtC6Gq1PMFiojHKeQkoFwxJB1rYe7qFgag5E0D26h95kSChEJOAkqPhBiGdu/KnFUlzS/zlTwAUvJg3WzPFyciHqeQk4BzxdAMvig9SvG+yuYb5E2FkmVwaKdnCxMRj1PIScC5eGA3wkJMywNQ8q50/ix63XNFiYhXKOQk4MTFhHNe/2TmrtlLfXPLfMVlQcZwWKdRliKBTiEnAemKIemUV9Xw2dYDzTfImwal66B8k2cLExGPUshJQBqfnUznyFDeWNVCl2XuFWAcGoAiEuAUchKQIkJDuCg/jXfX7+dYTf03G8SmODdUXT8bmhuFKSIBQSEnAeuKIemcqGvgvaKWlvmaBge3wb7VHq1LRDxHIScBq6BHHBlxUcxpqcsy51JwhKnLUiSAKeQkYDkchiuGpPPZlgpKK6u/2SAqDvpMgKI50NjMKEwR8XsKOQlolw9Jp9HCmy0t8zVwGlTugV2LPVuYiHiEQk4CWu+kTgzK6MLrLXVZ9p8CYdHOASgiEnAUchLwrhiSzoZ9lWzc38wyX+Ex0G8yFM+FhmY2WxURv6aQk4B3yaA0Qh2m5QEoA6fB8QOw7RPPFiYibqeQk4CX0CmCsf2SmLtqLw2NzcyJ6zMBIruoy1IkACnkJChcPiSd/ZXVLN3ezDJfoRGQfQlseBvqmtlsVUT8lkJOgsL52clEhDqYX9TCjuB506C2CjbP92xhIuJWCjkJCtHhoYzum8T7xaXNb6aaNRpikmC9diYQCSQKOQkak3JT2HP4BEV7mxllGRLqXLT5i/eguoXNVkXE7yjkJGhMyE7BYWh9Lcv6atg0z7OFiYjbKOQkaMTHhDM8K77l7+UyR0CX7lrLUiSAKOQkqEzKTWVTaRU7Ko5980VjIG8qbPsIjrWw2aqI+BWFnASVSTkpAMwvbqnL8kporIcNcz1YlYi4i0JOgkpmfDQ53Tq33GWZOhAS+8E6jbIUCQQKOQk6k3JTWLHrEOVVNd980RjnAJSdn0FlCzsXiIjfUMhJ0LkgNxVrYcGGFq7mBk4DLKx/3aN1iYjrKeQk6AxIjSUzPor5LU0lSOgN3QZrLUuRAKCQk6BjjOGCnFQ+23KAozX1zTfKuxL2roIDWz1bnIi4lEJOgtKk3FRqGxr5eFNZ8w3ypjp/qstSxK8p5CQoDesRR0JMeMujLLtkQPdz1GUp4ucUchKUQhyGCdkpfLSxjNr6xuYb5U2F8o1QtsGzxYmIyyjkJGhNyk2hqqaexdtaWN0k+1IwDiia49nCRMRlFHIStEb1SSQ6PKTlUZaxKdBjlDPkmtueR0R8nkJOglZkWAjj+jv3mGtsbCHEcq+Aii+grNizxYmISyjkJKhNykmlrKqG1SWHm2+gLksRv6aQk6B23oBkQh2m5VGWnZKcu4ary1LELynkJKh1iQrj7N4JzC/aj20pxHKvgANboHS9Z4sTkTOmkJOgNyknhW0Vx9hafrT5BtmXgAlRl6WIH1LISdCbmJMKwHstdVnGJELPMeqyFPFDCjkJeqldIhmU2bXlqQTg7LI8uA32r/VcYSJyxhRyIji7LNeUHGHfkRPNN8i+BByh6rIU8TMKORGce8wBvF/cQpdldDz0GqcuSxE/o5ATAfokd6JXUkzLUwnA2WV5aAfsW+2pskTkDCnkRJpckJvKkm0HOHK8rvkGAy4CR5i23xHxIwo5kSaTclKob7R8uKmFq7moOOh9HhS9oS5LET+hkBNpMiijK8mxEW13WR7ZBXtWeq4wEekwhZxIE4fDMCk3hU++KKe6rqH5Rv0vdHZZFqnLUsQfKORETjEpJ5XjtQ0s2lzRfIOortDnfHVZivgJhZzIKUb2SiA2MpT5xW1MDK8sgZJCzxUmIh2ikBM5RXiog/EDklmwoYyGlvaY6z8FQsI1MVzEDyjkRL5mUk4qB4/VUrjjYPMNIrtAnwlQ/AY0Nnq0NhE5PQo5ka8Z2z+J8FAH81ta/QSauiz3QMlyzxUmIqdNISfyNZ0iQjm3TyLvtbbHXL/JEBKhLksRH6eQE2nGpJwUSg6dYMO+quYbRHaGvhPVZSni4xRyIs2YkJOCMbQ9yrJqH+xe6rnCROS0KOREmpHYKYKCHnEtb6QK0O8CCI1Ul6WID2tXyBljJhtjNhljthhj7mvm9ZuMMeXGmNVNt1tcX6qIZ03MSWHDvkr2HG5hj7mI2KYuy7nQ2MIKKSLiVW2GnDEmBHgMmALkANcYY3KaafqytXZw0+1JF9cp4nHjB6QA8OGGNkZZHt0Pu5Z4qCoROR3tuZIbAWyx1m6z1tYCs4DL3FuWiPf1ToohKyGaDzaWtdyo7wUQGqUuSxEf1Z6QSwd2n/K4pOm5r7vSGLPWGDPbGJPZ3ImMMbcZYwqNMYXl5eUdKFfEc4wxnJ+dwudbD3C8tr75RhGdoN8kdVmK+ChXDTx5C8iy1uYD7wPPNNfIWjvTWltgrS1ISkpy0VuLuM/5A5KprW9secFmcHZZHiuDnZ97rjARaZf2hNwe4NQrs4ym506y1h6w1tY0PXwSGOaa8kS8a3jPeGIjQvlgQ2tdlpMgLFpdliI+qD0htxzoa4zpaYwJB2YAb57awBjT7ZSHlwIbXFeiiPeEhTgY0z+JDzeV0djSgs3hMc7pBBvehIYWujVFxCvaDDlrbT1wN/AezvB6xVpbZIx5yBhzaVOze4wxRcaYNcA9wE3uKljE0yZkJ1NeVcO6PUdabpQ7FY6Vw85FnitMRNoU2p5G1tp5wLyvPfeLU+7/DPiZa0sT8Q3j+iXjMPDBxjIGZXZtvlHfiRAW4+yy7DXOk+WJSCu04olIG+JiwhnWI44PWpsvFxbl3GeuWF2WIr5EISfSDudnp1C0t5J9R1pY/QScoyxPHIQdCz1XmIi0SiEn0g7nD0gG4MPWJob3mQDhsbD+NQ9VJSJtUciJtEOf5E50j49ufSpBWCQMuAiK34L6mpbbiYjHKORE2sEYw/gByXy2pYITta2sbDLwKqg5AlsWeK44EWmRQk6knSZkp1BT38hnW1pZ/aTXWIhOhHWveq4wEWmRQk6knUb0jKdTRGjrCzaHhEHu5bDpXahpYVdxEfEYhZxIO4WHOhjTL5EPN5ZibQurn4Czy7L+BGyc13IbEfEIhZzIaRg/IIXSyhqK9la23ChjBHTpri5LER+gkBM5Def1T8IYWNDaxHCHA/KmwtYP4Vgr39+JiNsp5EROQ0KnCIZ2j2t9KgE4uyxtAxS/4ZG6RKR5CjmR0zR+QDLr9hyhtLK65UYpuZA0ANbN9lxhIvINCjmR0zQhOwVoY/UTY2DgNNi1GA7v9lBlIvJ1CjmR09QvpRPpXaPa7rLMm+b8qWW+RLxGISdymowxTMhOZtGWcqrrWln9JL4npBeoy1LEixRyIh0wPjuF6rpGFm890HrDgVdB6Too2+iZwkTkKxRyIh0wslc80eEhrU8lAOf2O8YB63U1J+INCjmRDogIDWF030Q+3FjW+uonsSnQc4yzy7K1diLiFgo5kQ46PzuFfUeqKd7Xyuon4OyyPLQd9qz0TGEicpJCTqSDzuufjDG0Pcoy+xIIidAyXyJeoJAT6aCk2AgGZXRtfVcCgMgu0HciFL0Oja2MxhQRl1PIiZyBCdnJrNl9mLKqVlY/AWeX5dFS2PGpZwoTEUAhJ3JGxg9wrn7y8cby1hv2uwDCY9VlKeJhCjmRM5DdLZa0LpFtTyUIi4Lsi6H4Laiv8UxxIqKQEzkTxhjGZyezaEtF66ufgHMty5ojsPl9zxQnIgo5kTN1fnYKx2sbWLKtjdVPeo6D6ER1WYp4kEJO5Ayd3SuBqLCQtqcShIQ6V0D54l2oqfJMcSJBTiEncoYiw0I4tz2rn4BzlGV9NWz8t2eKEwlyCjkRF5iQncyewyfYuL+NK7TMEdClu3YmEPEQhZyIC5zXPxloYyNVaNpM9UrY+iEcq/BAZSLBTSEn4gLJnSMZlNGl7akE4OyytA1QNMf9hYkEOYWciIuMH5DC6t2HqTjaxjy4lFxIytaO4SIeoJATcZHzs5OxFj5qq8sSnHPmdi2Gw7vcX5hIEFPIibhIblpnurVn9ROAvCudP3U1J+JWCjkRFzHGMCE7hYVftGP1k/iekDEc1inkRNxJISfiQhNzUjhR18Cize0YOTnwKihdB2Ub3F+YSJBSyIm40MheCcRGhPJ+cTu6LHOvAOPQnDkRN1LIibhQeKiDcQOS+WBjKQ2Nbax+0ikZeo6F9bOhrZVSRKRDFHIiLjYxJ4WKo7Ws3n2o7cYDr4JDO2D3MrfXJRKMFHIiLjaufxJhIYb5Re3ossy5FMI7wcpn3V+YSBBSyIm4WOfIMEb2Smjf93IRsc7pBOtfg+oj7i9OJMgo5ETcYFJOCtsqjrGl7GjbjYfdCPUntM+ciBuEersAkUA0ISeFB+YW8X5xKX2SO7XeOG0opAyEFc/A8Fs8U6C4XV1dHSUlJVRXV3u7lIAQGRlJRkYGYWFhp3WcQk7EDbp1iWJgehfmF+/nznG9W29sjPNqbt6PYO8qSBvimSLFrUpKSoiNjSUrKwtjjLfL8WvWWg4cOEBJSQk9e/Y8rWPVXSniJpNynAs2l1W141/yA6+C0Cjn1ZwEhOrqahISEhRwLmCMISEhoUNXxQo5ETeZmJuCtfDBhnYs2BzVFXIvd04Mr2nH93jiFxRwrtPRP0uFnIib9E+JJTM+qn2jLAGG3gi1VdpnTlzi8OHDPP7446d93IUXXsjhw4dbbfOLX/yCBQsWdLAyz1LIibiJMYaJ2aks2lLBsZr6tg/oPhIS+8NKdVnKmWsp5OrrW/9vcd68eXTt2rXVNg899BATJkw4k/I8RiEn4kaTclOorW9k4RflbTf+cgBKyXIoLXJ/cRLQ7rvvPrZu3crgwYMZPnw4o0eP5tJLLyUnJweAyy+/nGHDhpGbm8vMmTNPHpeVlUVFRQU7duwgOzubW2+9ldzcXCZNmsSJEycAuOmmm5g9e/bJ9g8++CBDhw5l4MCBbNy4EYDy8nImTpxIbm4ut9xyCz169KCioh0Ll7uYRleKuFFBjzi6RofxfnEpUwZ2a/uA/Bmw4JfOASgX/t7t9Yln/OqtIor3Vrr0nDlpnXnwktwWX//tb3/L+vXrWb16NR9//DEXXXQR69evPzk68emnnyY+Pp4TJ04wfPhwrrzyShISEr5yjs2bN/PSSy/xxBNPcPXVV/Paa69x3XXXfeO9EhMTWblyJY8//jgPP/wwTz75JL/61a8YP348P/vZz3j33Xd56qmnXPr520tXciJuFBriYPyAZD7YWEZdQ2PbB8QkQPYlsHYW1J1wf4ESNEaMGPGV4fd//vOfGTRoECNHjmT37t1s3rz5G8f07NmTwYMHAzBs2DB27NjR7LmnTp36jTaLFi1ixowZAEyePJm4uDjXfZjToCs5ETeblJPC6yv3sHzHQc7pndj2AUNvdC7zVfwmDJru/gLF7Vq74vKUmJiYk/c//vhjFixYwOLFi4mOjmbcuHHNDs+PiIg4eT8kJORkd2VL7UJCQtr8zs/TdCUn4mZj+iUREepo/yjLrNEQ11MDUOSMxMbGUlVV1exrR44cIS4ujujoaDZu3MiSJUtc/v6jRo3ilVdeAWD+/PkcOtSOXTncQCEn4mbR4aGc2yeR94tLse3ZN87hgKE3wM7PoOKbXUgi7ZGQkMCoUaPIy8vjxz/+8Vdemzx5MvX19WRnZ3PfffcxcuRIl7//gw8+yPz588nLy+PVV18lNTWV2NhYl79PW0y7/qdzg4KCAltYWOiV9xbxtFnLdnHf6+uYd89octI6t31AVSn8vxwYeSdM+rX7CxSX27BhA9nZ2d4uw2tqamoICQkhNDSUxYsXc+edd7J69eozOmdzf6bGmBXW2oKWjtF3ciIecH52Csas4/3i0vaFXGwK9JsMq1+E8Q9AaETbx4j4kF27dnH11VfT2NhIeHg4TzzxhFfqUMiJeEBSbARDu8fx/ob9fG9C3/YdNOxm2Pg2bPw35E11b4EiLta3b19WrVrl7TL0nZyIp0zMSWH9nkr2Hm7n1IDe50GXTA1AETkDCjkRD5mYkwLQ/lGWjhAYcj1s+xgObndfYSIBTCEn4iG9kzrROymm/SEHMOQ6MA5Y9Zz7ChMJYAo5EQ+amJPKkm0HOHKirn0HdEmHPhNh1QvQ4FuTbEX8gUJOxIMm5qRQ32j5eFM79pj70rAb4eh+2Pye+wqToNepUycA9u7dy7Rp05ptM27cONqa+vXoo49y/Pjxk4/bs3WPOynkRDxoSGZXEjtFMP90uiz7XgCdUrVruHhEWlrayR0GOuLrIdeerXvcSSEn4kEOh2FiTjKfbCqnpr6hfQeFhMKQa2HL+3CkxL0FSsC47777eOyxx04+/uUvf8mvf/1rzj///JPb4sydO/cbx+3YsYO8vDwATpw4wYwZM8jOzuaKK674ytqVd955JwUFBeTm5vLggw8CzkWf9+7dy3nnncd5550H/GfrHoBHHnmEvLw88vLyePTRR0++X0tb+riC5smJeNjEnBReWrabJdsOMrZfUvsOGnI9fPpHWPU8jLvPvQWK671zH+xf59pzpg6EKb9t8eXp06fz/e9/n7vuuguAV155hffee4977rmHzp07U1FRwciRI7n00ksxxjR7jr/97W9ER0ezYcMG1q5dy9ChQ0++9pvf/Ib4+HgaGho4//zzWbt2Lffccw+PPPIIH330EYmJX12MfMWKFfzzn/9k6dKlWGs566yzGDt2LHFxce3e0qcjdCUn4mHn9E4kOjyE+UX7239QfE/odR6sfA4a23kFKEFtyJAhlJWVsXfvXtasWUNcXBypqancf//95OfnM2HCBPbs2UNpactd5wsXLjwZNvn5+eTn55987ZVXXmHo0KEMGTKEoqIiiouLW61n0aJFXHHFFcTExNCpUyemTp3Kp59+CrR/S5+O0JWciIdFhoUwpm8SCzaU8j+X5eFwNP+v6G8YdiO8ehNs/RD6TnRrjeJirVxxudNVV13F7Nmz2b9/P9OnT+eFF16gvLycFStWEBYWRlZWVrNb7LRl+/btPPzwwyxfvpy4uDhuuummDp3nS+3d0qcjdCUn4gWTclMoraxh3Z4j7T+o/0UQnQgr/uW2uiSwTJ8+nVmzZjF79myuuuoqjhw5QnJyMmFhYXz00Ufs3Lmz1ePHjBnDiy++CMD69etZu3YtAJWVlcTExNClSxdKS0t55513Th7T0hY/o0eP5o033uD48eMcO3aMOXPmMHr0aBd+2ubpSk7EC8YPSCbEYXi/uJRBmV3bd1BoOAy+Bpb8zblLQWyKW2sU/5ebm0tVVRXp6el069aNa6+9lksuuYSBAwdSUFDAgAEDWj3+zjvv5OabbyY7O5vs7GyGDRsGwKBBgxgyZAgDBgwgMzOTUaNGnTzmtttuY/LkyaSlpfHRRx+dfH7o0KHcdNNNjBgxAoBbbrmFIUOGuLRrsjnaakfES2bMXMzBY7XM/8HY9h9UsRn+WgDnPwij73VfcXLGgn2rHXfoyFY76q4U8ZJJOal8UXqUnQeOtf+gxL7QY5Szy1IDUETapJAT8ZLTXrD5S2fdAYd3woY33VCVSGBpV8gZYyYbYzYZY7YYY1qcpGOMudIYY40xLV46iohTZnw0A1JjT2/1E4ABF0F8L/jsz+ClrxtE/EWbIWeMCQEeA6YAOcA1xpicZtrFAt8Dlrq6SJFANSknhcIdBzlwtKb9BzlC4Oy7YO9K2Pm5+4qTM+atMQ+BqKN/lu25khsBbLHWbrPW1gKzgMuaafc/wO+Ajk+WEAkyF+Sl0mjhvaLTvJobfC1EJ8Dnf3ZPYXLGIiMjOXDggILOBay1HDhwgMjIyNM+tj1TCNKB3ac8LgHOOrWBMWYokGmt/bcx5sctncgYcxtwG0D37t1Pu1iRQJPTrTO9EmN4e+1evnXWafw/ERYFI26Dj/8PyjZCcutDwcXzMjIyKCkpoby83NulBITIyEgyMjJO+7gznidnjHEAjwA3tdXWWjsTmAnOKQRn+t4i/s4Yw8X53fjrR1soq6omOfY0/qU6/BZY9P9g8V/gssfabi8eFRYWRs+ePb1dRtBrT3flHiDzlMcZTc99KRbIAz42xuwARgJvavCJSPtcMiiNRgvvrDuNtSwBYhKd3ZZrX4Gq0zxWJEi0J+SWA32NMT2NMeHADODk2GVr7RFrbaK1NstamwUsAS611mqmt0g79E2JpX9KLG+v3Xv6B599FzTUwdJ/uL4wkQDQZshZa+uBu4H3gA3AK9baImPMQ8aYS91doEgwuDi/G8t3HGLfkdNcmDahN2RfAoVPQc031wsUCXbtmidnrZ1nre1nre1trf1N03O/sNZ+YzaqtXacruJETs/Fg9IA+Pfafad/8Dn3QPUR515zIvIVWvFExAf0TIwhL70zb3Uk5DKHQ/ezYfHj0FDv+uJE/JhCTsRHXJyfxprdh9l98PjpH3zOPXBkFxS/4fK6RPyZQk7ER1w0sBsAb3VkAEq/yZDQ1zk5XJOPRU5SyIn4iMz4aIZ078rbazrQZelwwDl3w741sH2h64sT8VMKOREfcnF+GsX7KtlafvT0D86fATFJWupL5BQKOREfctHAbhhDx67mwiJhxO2wZQGUFru+OBE/pJAT8SGpXSIZnhXfsYnhAMO/A2HR8PlfXFuYiJ9SyIn4mEvyu7G57Cib9ndgcnd0PAy5Hta9CpUdDEqRAKKQE/ExUwZ2w2HgrTUdDKmzvwu2AZb+3bWFifghhZyIj0nsFME5vRN5e+3eju1FFpcFOZdB4T+hutLl9Yn4E4WciA+6OL8bOw4cp2hvB0PqnHugphJWPuPawkT8jEJOxAdNzksl1GE63mWZPhR6nAtL/ubcpUAkSCnkRHxQ1+hwRvdN5O21+zrWZQkw6h6o3APrX3dtcSJ+RCEn4qMuzk9jz+ETrNp9uGMn6DMRkgZoqS8Jago5ER81MTeF8FBHx7ssHQ44+24oXQ9bP3RtcSJ+QiEn4qM6R4Yxrl8S89bto7Gxg1di+VdDp1RNDpegpZAT8WEXD0qjtLKG5TsOduwEoRFw1u2w7SPYt9a1xYn4AYWciA+bkJ1MVFhIx7bf+VLBzRAeC5/8znWFifgJhZyID4sOD2V8djLvrNtPfUNjx04SFQfnfg82vg07F7u2QBEfp5AT8XGX5HfjwLFalmzrYJclwMi7ILYbvP+ARlpKUFHIifi4cf2T6RQR2vFRlgDh0XDe/VCyHDa86briRHycQk7Ex0WGhTAxJ4V3i/ZTW9/BLkuAQd+CpGxY8EutgiJBQyEn4gcuzu/GkRN1fLalouMnCQmFib+Cg9tgxb9cVpuIL1PIifiB0X2T6Bx5hl2WAH0nQdZo+Pi32qFAgoJCTsQPhIc6mJyXyvziUqrrGjp+ImOcV3PHK5zLfYkEOIWciJ+4OD+NozX1fPJF+ZmdKH0Y5F0Jn/8VKve5pjgRH6WQE/ET5/ROID4m/My7LAHGPwCN9fDx/575uUR8mEJOxE+EhjiYkpfKBxvKOF5bf2Yni+8JI26FVc9D2QbXFCjigxRyIn7k4vw0TtQ18OHGsjM/2ZgfO5f7WvDLMz+XiI9SyIn4kRE940mOjXBNl2V0PIz+AXzxLmz/9MzPJ+KDFHIifiTEYbhkUBofbizj4LHaMz/hWXdA53Tncl+NZzDRXMRHKeRE/MzVBZnUNVheX1ly5icLi4LxP4e9q6B4zpmfT8THKORE/Ez/1FiGdO/Ky8t3Y12x2HL+dEjJgwW/gvqaMz+fiA9RyIn4oRnDM9lcdpRVuw+f+ckcIc4J4od3QuHTZ34+ER+ikBPxQxflpxEdHsLLy3a75oS9z4de4+CT38OJw645p4gPUMiJ+KFOEaFckp/GW2v3crTmDOfMQdNyXw/BiYPw2aNnfj4RH6GQE/FT00dkcry2gbddMZ0AoNsg5/dzS/4GR1wwqEXEByjkRPzUkMyu9EvpxKzlLuqyBOdIS9sIH2m5LwkMCjkRP2WMYfrw7qzefZhN+6tcc9Ku3eGs22H1i7B/vWvOKeJFCjkRP3bFkHTCQxy87MqrudE/hMgusOBB151TxEsUciJ+LD4mnIm5Kby+qoSa+jPYZ+5UUXEw5kewZQFseMs15xTxEoWciJ+bMTyTw8frmF9U6rqTnnUHpA6Et++F4wddd14RD1PIifi5Ub0TSe8a5douy5AwuPxvzikF837suvOKeJhCTsTPORyG6cMzWbSlgt0Hj7vuxKkDYcxPYP1sdVuK31LIiQSAacMycBh4pdCFV3MAo+9Vt6X4NYWcSABI6xrFmH5JvFpYQkOjCxZt/pK6LcXPKeREAsSM4Znsr6xm4Rflrj2xui3FjynkRALE+AEpJHYKZ9byXa4/ubotxU8p5EQCRHiogyuHZvDBhjLKqqpde3J1W4qfUsiJBJCrh2dS32h5feUe159c3ZbihxRyIgGkd1InhmfFuW7X8K9Tt6X4GYWcSICZPrw72yuOsWy7G0Lo1G7Ld37i+vOLuJhCTiTAXDgwldiIUF529Zy5L33ZbbnuVdjwtnveQ8RFFHIiASY6PJRLB6cxb90+jpyoc8+bnOy2/IG6LcWnKeREAtCM4d2prmvkTVftGv516rYUP6GQEwlAeemdyenWmZfdMWfuS+q2FD+gkBMJQM5dwzNZv6eS9XuOuO+N1G0pPk4hJxKgLh+cTnioi3cN/zp1W4qPU8iJBKgu0WFcmJfKG6v3UF3nol3Dm/OVbktNEhffopATCWDTh3enqrqed9bvc+8bjb4XUvNh7t1Qsdm97yVyGhRyIgFsZK94shKimbXMjV2W4Oy2nP6c8+cL0+BYhXvfT6SdFHIiAcwYw1UFmSzdfpBt5Ufd+2ZxWXDNLKjaDy/NgLoT7n0/kXZQyIkEuKuGZRDiMLyw1I3TCb6UUQBTZ0JJIcy5Axob3f+eIq1QyIkEuOTOkVyc341Zy3a5bwWUU+VcBhMfguI34INfuf/9RFqhkBMJAreN6cWx2gZeWLrTM294zn9Bwbfhs0dhxb88854izVDIiQSB3LQujO6byD8/20FNvRunE3zJGJjyB+gzwbktz5YP3P+eIs1QyIkEidvH9Ka8qoY3VrlhQ9XmhITCtH9Ccja8ciOUFnnmfUVOoZATCRKj+iSQm9aZfyzcRmOjGzZUbU5kZ/jWKxDRCV64GirdPF9P5GsUciJBwhjD7WN7s638GAs2lHrujbukw7dehhOH4KXpUOPmqQwip1DIiQSRC/NSyYiLYubCbZ59426D4Kp/wv518Not0OiB7wVFUMiJBJXQEAe3nNuTwp2HKNzh4V0D+l0AU34PX7wD793v2feWoNWukDPGTDbGbDLGbDHG3NfM63cYY9YZY1YbYxYZY3JcX6qIuMLVwzPpGh3GPzx9NQcw4lYYeRcs/Tss+bvn31+CTpshZ4wJAR4DpgA5wDXNhNiL1tqB1trBwO+BR1xdqIi4RnR4KDeM7MGCDaVsKfPC92OT/gcGXAzv3gcb53n+/SWotOdKbgSwxVq7zVpbC8wCLju1gbW28pSHMYCHhm6JSEfccE4W4SEOnvzUC1dzjhDn0l9pg+G178DuZZ6vQYJGe0IuHTh1CfOSpue+whhzlzFmK84ruXuaO5Ex5jZjTKExprC8vLwj9YqICyR2iuCqggxeX7mHsspqzxcQHgPXvAydUuDZy2Dz+56vQYKCywaeWGsfs9b2Bn4K/LyFNjOttQXW2oKkpCRXvbWIdMAt5/airrGRf32+wzsFxKbAd+ZDYl94cTqsfsk7dUhAa0/I7QEyT3mc0fRcS2YBl59BTSLiAVmJMUzJS+W5JTs5WlPvnSI6JcONb0PWufDGHfDZn71ThwSs9oTccqCvMaanMSYcmAG8eWoDY0zfUx5eBGhrYBE/cPuY3lRV1zNrmQe24WlJZGe49lXIvQLefwDe+29t0SMu02bIWWvrgbuB94ANwCvW2iJjzEPGmEubmt1tjCkyxqwG7gVudFfBIuI6gzK7clbPeJ5atJ26Bi8GS2gEXPk0jLgdFv8V5twO9bXeq0cCRmh7Gllr5wHzvvbcL065/z0X1yUiHnLH2N7c/K/lvLVmL1OHZnivEIcDpvzO2YX54f/A8QNw9bPOdS9FOkgrnogEuXH9k+ifEsvMhduw1suzf4yBMT+CS/8C2z6CZy6BYxXerUn8mkJOJMgZY7h1TC827q/iky98ZGrP0Btg+gtQVgxPXwCHPLTZqwQchZyIcOmgNFI7R/KPT7wwObwlAy6EG+bCsXJ4ahLsX+/tisQPKeREhPBQB985tyeLtx1gbclhb5fzH91HwrffA+OAf14IOz7zdkXiZxRyIgLAjBGZxEaEemfh5tYkZzsnjcemwHNXwIa3vF2R+BGFnIgAEBsZxrUje/DOun3sPHDM2+V8VddM5xVdt3x4+Tp4936o88JyZOJ3FHIictLNo7IIdTh48tPt3i7lm6Lj4YY3YfgtsOQxmDkW9q72dlXi4xRyInJSSudILh+SxqsrdnPgaI23y/mm8Gi46I9w3WtQfQSePB8++QM0eGlZMvF5CjkR+YrbxvSiuq6RZxf78LD9PhPgu4sh53L46NfOaQYVW7xdlfgghZyIfEWf5FgmZKfw7OIdHPPWws3tERUH056CaU/DgS3w93Nh2RPg7Qnt4lMUciLyDd89rzeHjtfxhDc2VT1deVfCd5dA1iiY9yPnCMzKvd6uSnyEQk5EvmFo9zguHJjKzIXbKKvyg1GMnbvBtbPhokdg91J4fCSsm62rOlHIiUjzfnLBAOoaGvl/7/vJzlnGwPDvwB2LILE/vPYdmH0zHD/o7crEixRyItKsrMQYrj2rBy8v38UXpVXeLqf9EnrDze/A+b+ADW/D42dD0Rvaoy5IKeREpEX3nN+XmPBQfvvORm+XcnpCQmH0D+HWDyE6AV690TkwZf3r0Njg7erEgxRyItKi+Jhw7hrfhw83lvH5Vj/c8qZbPty+EKY+AY31zu7Lx0fCmpc1ty5IKOREpFU3nZNFetco/nfeBhob/XAgR0go5F/tHIF51b8gJBzm3AZ/LYCVz0FDnbcrFDdSyIlIqyLDQvjRBf1Yv6eSuWv2eLucjnM4IPcKuP1TmPEiRHaBN++GPw+F5U9BvQ+u8CJnTCEnIm26bFA6eemdefi9L6iu8/PvtBwOGHAR3Paxc9pBbCr8+17402BY8neoO+HtCsWFFHIi0iaHw3D/hdnsOXyCf32+w9vluIYx0HeicxufG+ZCfE9496fwaD589mdNPQgQCjkRaZdzeicyfkAyj324hYPHar1djusYA73Gwc3z4KZ5kJID7z8Af+gDz011fm+nwPNbCjkRabefTRnAsdp6/vyBn0wQP11Zo5xXdbd/CqPugYNbnd/bPdwXnp8Gq56HE4e8XaWcBmO9tOxNQUGBLSws9Mp7i0jH/ez1dbxauJv37x1Lz8QYb5fjXtbCvtVQNMd5O7wLHGHQ+zznIJb+F0JUV29XGdSMMSustQUtvq6QE5HTUVZVzbg/fMzYfkn87bph3i7Hc6yFvSudq6cUvQFHmgKvz/lNgTfFOWJTPKqtkAv1ZDEi4v+SYyO5bUwvHl2wmRU7DzKsR7y3S/IMYyB9mPM28SHYsxKKXncG3hfvgnFAci5kFEDGcMgcAfG9naM5xWt0JScip+14bT3j/vAxGXFRvHbnORhjvF2S9zQ2wp4VsGUBlCyHkkKoOeJ8LbJrU+iNaPpZoKs9F9OVnIi4XHR4KPdO7Md9r6/jnfX7uXBgN2+X5D0OB2QOd97AGXoHNsPuZf8JvY//D7CAgaT+/wm+lFzo2gNiEp1XiuJyupITkQ5paLRM+dNCauobef8HYwkPVbdci6ornd/n7V7eFHzL4cQp0xLCYqBrd4jrAXFZzuCL6/GfnxGxXivdpayFuuPOKRn11ZDY94xPqSs5EXGLEIfhZ1Oyuflfy3lh6U5uHtXT2yX5rsjOzrl4vcY5H1sLB7dBxWY4vBMO7YRDO5z3d3wGtV/b2igqvin0ukNUHER0dp4zovMp92O/+XxouGs/R2OjM5zqTjjD6suftcecUytOHHT+PH7Qef/4QThx+JT7B6GhaY5l1x7w/bWura8ZCjkR6bBx/ZMY1SeBP32wmalDM+gSFebtkvyDMc597xJ6f/M1a51B8WXonRqApcVQfQRqKp1h05bQSAiLBkfoKbeQFn6GgglxPm6s/0+I1R7/z/36di555giD6HhnOEfFQXwv54CdU5+LTT2dP7EOU8iJSIcZ47yau+Svi3j84y38bEq2t0vyf8Y4wyA6HtKHttyuvhZqqpyDXKorm+5XNt2v/M/92mNgG5z76DU2OAPsy5tt/OrjL18PCYPIbhAW5QzJsKiv3W/mZ1Sc8xYdD+GdfOY7RoWciJyRvPQuXDE4nX9+toPrR/YgIy7a2yUFh9BwCE2AmARvV+LT9E2xiJyxH17QH4DfvbvJy5WIfJVCTkTOWHrXKO4c25u31uxlftF+b5cjcpJCTkRc4q7z+pDTrTP3z1kXWLsUiF9TyImIS4SHOnj4qkEcOVHHL+au93Y5IoBCTkRcKCetM/eM78vba/fx77X7vF2OiEJORFzrznG9yc/owgNz11NxtMbb5UiQU8iJiEuFhjj441WDOFpdz3/PWYe3lg4UAYWciLhB35RY7p3Uj/eKSnlzzV5vlyNBTCEnIm5x6+heDOnelV/MLaK0sh1LUIm4gUJORNwixGF4+KpBVNc1cP/r6rYU71DIiYjb9E7qxE8mD+CDjWXMXlHi7XIkCCnkRMStbj4nixFZ8Tz0VjH7jrRzFXsRF1HIiYhbORyGP1yVT32j5Sez16rbUjxKIScibtcjIYb7LxzAp5srmLV8t7fLkSCikBMRj7j2rB6M6pPAr98uZvfB494uR4KEQk5EPMLhMPzuynyMMfxk9loaG9VtKe6nkBMRj8mIi+bnF2WzeNsBnl+609vlSBBQyImIR00fnsnYfkn837yN7Kg45u1yJMAp5ETEo4wx/PbKgYSGGH48e426LcWtFHIi4nHdukTxy0tyWb7jEI99tMXb5UgAU8iJiFdMHZrOFUPS+eP7X/Be0X5vlyMBSiEnIl5hjOH/pg5kUGZXfvDyajbsq/R2SRKAFHIi4jWRYSE8cf0wOkeGccszhdpkVVxOISciXpXcOZKZNwyj4mgNdz6/gtr6Rm+XJAFEISciXpef0ZWHrxrE8h2HeOCN9VrfUlwm1NsFiIgAXDIojS9Kq/jLh1vonxrLt8/t6e2SJADoSk5EfMYPJvTjgtwUfv3vYj75otzb5UgAUMiJiM9wOAyPXD2Yfimx3P3iSraWH/V2SeLnFHIi4lNiIkJ58sYCwkMc3PpMIUeO13m7JPFjCjkR8TkZcdH8/fph7D50nLtfWkl9g0ZcSsco5ETEJw3PiufXl+fx6eYKfjNvg7fLET+l0ZUi4rOmD+/Opv1Hefqz7fRPiWXGiO7eLkn8jK7kRMSn3X/hAMb0S+KBuetZuu2At8sRP6OQExGfFhri4C/XDCEzLpo7X1jJ7oPHvV2S+BGFnIj4vC5RYTx5YwH1DY3c8kwhh4/Xersk8RMKORHxC72SOvH4tcPYfuAY1zyxlIPHFHTSNoWciPiNc/sm8uQNBWwrP8q3nliiXQukTQo5EfErY/ol8fRNw9lx4BjXzFxCWVW1t0sSH6aQExG/M6pPIv+6eQR7Dp9gxswllFYq6KR5CjkR8UsjeyXwzLdHUHqkmun/WMzewye8XZL4oHaFnDFmsjFmkzFmizHmvmZev9cYU2yMWWuM+cAY08P1pYqIfNXwrHieu+UsDhytZfrMxZQc0vQC+ao2Q84YEwI8BkwBcoBrjDE5X2u2Ciiw1uYDs4Hfu7pQEZHmDO0ex/O3nMWR43VM/8cSdh1Q0Ml/tOdKbgSwxVq7zVpbC8wCLju1gbX2I2vtl/9lLQEyXFumiEjLBmV25cVbR3Kstp7pMxezo+KYt0sSH9GekEsHdp/yuKTpuZZ8B3inuReMMbcZYwqNMYXl5doQUURcJy+9Cy/eMpKa+kamz1ysvegEcPHAE2PMdUAB8IfmXrfWzrTWFlhrC5KSklz51iIi5KR15qVbR9LQaJkxcwmbS6u8XZJ4WXtCbg+QecrjjKbnvsIYMwH4b+BSa61maIqIV/RPjWXWbSMBmDFzCZv2K+iCWXtCbjnQ1xjT0xgTDswA3jy1gTFmCPAPnAFX5voyRUTar09yLC/fNpKwEAczZi6maO8Rb5ckXtJmyFlr64G7gfeADcAr1toiY8xDxphLm5r9AegEvGqMWW2MebOF04mIeESvpE68fPtIosJCuOrvi/n32n3eLkm8wFhrvfLGBQUFtrCw0CvvLSLBo7SymjufX8HKXYe5fUwvfnxBf0JDtA5GoDDGrLDWFrT0un7TIhLQUjpHMuu2s7luZHf+sXAbN/5zmXYwCCIKOREJeOGhDn59+UB+Py2f5TsOcclfFrGuRN/TBQOFnIgEjasLMpl9x9lYa7ny75/zauHutg8Sv6aQE5Ggkp/Rlbf+61wKesTx49lreeCN9dTWN3q7LHEThZyIBJ2EThE8++0R3D6mF88t2cmMmYu1XU+AUsiJSFAKDXHwswuz+eu3hrBxfxUX/2URy3cc9HZZ4mIKOREJahfnpzHnu6OICQ/hmplLeObzHXhrapW4nkJORIJe/9RY5t59LmP7JfHgm0X88NU1HK+t93ZZ4gIKORERoEtUGE/cUMAPJvRjzqo9XPDoQj7bUuHtsuQMKeRERJo4HIbvTejLrFtHEupwcO2TS/np7LUcOVHn7dKkgxRyIiJfc1avBN753mjuHNeb2StLmPjIJ7y7fr+3y5IOUMiJiDQjMiyEn04ewNy7RpHYKYI7nl/Bd19YQVmVphr4E4WciEgr8tK7MPfuUfxkcn8WbChj4iMLebVwt0Zg+gmFnIhIG8JCHHx3XB/e+d5o+qfE8uPZa7nh6WXsPnjc26VJGxRyIiLt1DupE7NuG8n/XJ7Hyp2HmPT/FvL0ou00NOqqzlcp5EREToPDYbh+ZA/ev3csZ/dO4KG3i5n298/5orTK26VJMxRyIiIdkNY1iqduLOBPMwazo+IYF/7pU37+xjoNTPExCjkRkQ4yxnDZ4HQW3DuWb53VnVnLdjP29x/zyPxNVFVrbp0vMN4aIVRQUGALCwu98t4iIu6wo+IYD8/fxNtr9xEfE85/je/Dt87qTkRoiLdLC1jGmBXW2oKWXteVnIiIi2QlxvDXbw3lzbtHMSA1ll+9VcyERz5h7uo9NGpwilco5EREXCw/oysv3HIWz357BLERYXxv1mou/ssiFn5Rrvl1HqaQExFxA2MMY/ol8fZ/ncufZgymsrqOG55exnVPLWVtyWFvlxc0FHIiIm7kcDgHp3zww7E8eEkOG/ZVcelfP+PuF1eytfyot8sLeBp4IiLiQVXVdTyxcBtPfLqd6voGJmSncPuYXhRkxXu7NL/U1sAThZyIiBdUHK3h2c938OySnRw+XsfQ7l25bUwvJuakEuIw3i7PbyjkRER82PHaemavKOHJT7ez6+BxshKi+c7oXkwbmkFUuKYetEUhJyLiBxoaLe8V7ecfC7exZvdh4mPCuX5kD244uwcJnSK8XZ7PUsiJiPgRay3Ldxxi5sKtLNhQRkSog2nDMrhldC96JsZ4uzyf01bIhXqyGBERaZ0xhhE94xnRM54tZVU8+el2Xi0s4cVlu5iUk8INZ2dxdq8EHPrerl10JSci4uPKqqp59vOdPL/UOUglKyGaa0Z0Z9qwjKDvylR3pYhIgKiua+Cd9ft4aelulu04SHiIgwvyUrlmRCZn90rAmOC7ulPIiYgEoM2lVby4bBevrSihsrqenokxXDMik2nDMomPCfd2eR6jkBMRCWDVdQ3MW7ePF5fuonDnIcJDHEzOS+VbZ3XnrJ7xAX91p5ATEQkSm/ZX8dKyXby2soSq6np6JcVwzfDuXDo4jZTOkd4uzy0UciIiQeZEbQP/XrePF5fuZOWuwxgD5/RO4PLB6UzOSyU2MszbJbqMQk5EJIhtKz/KG6v3Mnf1HnYeOE5EqIMJ2SlcNjiNcf2TCQ/173X6FXIiIoK1llW7DzN31R7eXruPA8dq6RodxoUDu3H54HQKesT55dw7hZyIiHxFXUMjizZX8MbqPcwvKuVEXQPpXaO4bHAalw9Jp19KrLdLbDeFnIiItOhYTT3zi/fzxqq9LNpSQUOjpX9KLFMGpnLhwG70Te7k0yM0FXIiItIu5VU1vL12L++s28/ynQexFnonxTAlrxtTBqaS062zzwWeQk5ERE5bWVU17xWV8s66fSzZdoBGCz0Sop2Bl5dKfkYXnwg8hZyIiJyRA0dreL+4lHnr9/P5lgrqGy3pXaOYkpfKlIHdGJLZ1WuDVhRyIiLiMoeP1/J+cSnvrt/Pp5srqG1oJLVzJBNykpmQncLZvROICPXcZq8KORERcYvK6jo+3FDGu+v3s3BzOcdrG4gJD2F03yQm5KQwfkCy29fRVMiJiIjbVdc1sHjbARYUl7JgQymllTU4DAzrEceE7BQm5KTQO6mTy99XISciIh5lrWX9nkre31DKguJSivdVAtArMYYJOSlMyE5haPeuhIac+WorCjkREfGqPYdP8OGGUt7fUMbirRXUNVgy4qJY+OPzznjASlshF3pGZxcREWlDetcorj87i+vPzqKquo5PN1dQWlntkRGZCjkREfGY2Ejnepme4t/LT4uIiLRCISciIgFLISciIgFLISciIgFLISciIgFLISciIgFLISciIgFLISciIgFLISciIgFLISciIgFLISciIgFLISciIgFLISciIgFLISciIgFLISciIgFLISciIgFLISciIgFLISciIgFLISciIgFLISciIgFLISciIgHLWGu988bGlAM7XXCqRKDCBefxV8H8+fXZg1Mwf3YI7s/f3GfvYa1NaukAr4WcqxhjCq21Bd6uw1uC+fPrs+uzB6Ng/vwd+ezqrhQRkYClkBMRkYAVCCE309sFeFkwf3599uAUzJ8dgvvzn/Zn9/vv5ERERFoSCFdyIiIizVLIiYhIwPLrkDPGTDbGbDLGbDHG3OftejzJGLPDGLPOGLPaGFPo7XrczRjztDGmzBiz/pTn4o0x7xtjNjf9jPNmje7Swmf/pTFmT9Pvf7Ux5kJv1uguxphMY8xHxphiY0yRMeZ7Tc8H/O++lc8e8L97Y0ykMWaZMWZN02f/VdPzPY0xS5v+zn/ZGBPe5rn89Ts5Y0wI8AUwESgBlgPXWGuLvVqYhxhjdgAF1tqgmBRqjBkDHAWetdbmNT33e+Cgtfa3Tf/IibPW/tSbdbpDC5/9l8BRa+3D3qzN3Ywx3YBu1tqVxphYYAVwOXATAf67b+WzX02A/+6NMQaIsdYeNcaEAYuA7wH3Aq9ba2cZY/4OrLHW/q21c/nzldwIYIu1dpu1thaYBVzm5ZrETay1C4GDX3v6MuCZpvvP4PwLIOC08NmDgrV2n7V2ZdP9KmADkE4Q/O5b+ewBzzodbXoY1nSzwHhgdtPz7fq9+3PIpQO7T3lcQpD8B9DEAvONMSuMMbd5uxgvSbHW7mu6vx9I8WYxXnC3MWZtU3dmwHXXfZ0xJgsYAiwlyH73X/vsEAS/e2NMiDFmNVAGvA9sBQ5ba+ubmrTr73x/Drlgd661digwBbirqUsraFlnv7t/9r13zN+A3sBgYB/wR69W42bGmE7Aa8D3rbWVp74W6L/7Zj57UPzurbUN1trBQAbOnrsBHTmPP4fcHiDzlMcZTc8FBWvtnqafZcAcnP8RBJvSpu8tvvz+oszL9XiMtba06S+BRuAJAvj33/SdzGvAC9ba15ueDorffXOfPZh+9wDW2sPAR8DZQFdjTGjTS+36O9+fQ2450LdptE04MAN408s1eYQxJqbpi2iMMTHAJGB960cFpDeBG5vu3wjM9WItHvXlX/BNriBAf/9NAxCeAjZYax855aWA/9239NmD4XdvjEkyxnRtuh+Fc4DhBpxhN62pWbt+7347uhKgaejso0AI8LS19jfercgzjDG9cF69AYQCLwb6ZzfGvASMw7nVRinwIPAG8ArQHee2TVdbawNugEYLn30czu4qC+wAbj/lO6qAYYw5F/gUWAc0Nj19P87vpgL6d9/KZ7+GAP/dG2PycQ4sCcF5MfaKtfahpr/7ZgHxwCrgOmttTavn8ueQExERaY0/d1eKiIi0SiEnIiIBSyEnIiIBSyEnIiIBSyEnIiIBSyEn4seMMeOMMW97uw4RX6WQExGRgKWQE/EAY8x1TftjrTbG/KNp8dmjxpj/17Rf1gfGmKSmtoONMUuaFuCd8+UCvMaYPsaYBU17bK00xvRuOn0nY8xsY8xGY8wLTStliAgKORG3M8ZkA9OBUU0LzjYA1wIxQKG1Nhf4BOdKJgDPAj+11ubjXO3iy+dfAB6z1g4CzsG5OC84V6f/PpAD9AJGufkjifiN0LabiMgZOh8YBixvusiKwrmgcCPwclOb54HXjTFdgK7W2k+ann8GeLVprdJ0a+0cAGttNUDT+ZZZa0uaHq8GsnBuMikS9BRyIu5ngGestT/7ypPGPPC1dh1dY+/Utfsa0P/XIiepu1LE/T4AphljkgGMMfHGmB44///7ckX1bwGLrLVHgEPGmNFNz18PfNK0M3SJMebypnNEGGOiPfkhRPyR/sUn4mbW2mJjzM9x7uTuAOqAu4BjwIim18pwfm8Hzi1E/t4UYtuAm5uevx74hzHmoaZzXOXBjyHil7QLgYiXGGOOWms7ebsOkUCm7koREQlYupITEZGApSs5EREJWAo5EREJWAo5EREJWAo5EREJWAo5EREJWP8fHmkvPNwDavgAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 864x576 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loss\n",
      "\ttraining         \t (min:    0.135, max:    0.707, cur:    0.135)\n",
      "\tvalidation       \t (min:    0.256, max:    0.696, cur:    0.258)\n"
     ]
    }
   ],
   "source": [
    "gmf_recommender = GMFRecommender(print_type='live', n_neg_per_pos=10, batch_size=16, \n",
    "                                 embedding_dim=6, lr=0.001, weight_decay=0.0001, n_epochs=30, seed=1, train=True)\n",
    "gmf_recommender.fit(ml_ratings_df, None, ml_movies_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "incorporated-messaging",
   "metadata": {},
   "source": [
    "## Quick test of the recommender (recommending)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "accessible-value",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Recommendations\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>item_id</th>\n",
       "      <th>score</th>\n",
       "      <th>title</th>\n",
       "      <th>genres</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>41566</td>\n",
       "      <td>0.978991</td>\n",
       "      <td>Chronicles of Narnia: The Lion, the Witch and the Wardrobe, The (2005)</td>\n",
       "      <td>Adventure|Children|Fantasy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>6502</td>\n",
       "      <td>0.770488</td>\n",
       "      <td>28 Days Later (2002)</td>\n",
       "      <td>Action|Horror|Sci-Fi</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>4896</td>\n",
       "      <td>0.736372</td>\n",
       "      <td>Harry Potter and the Sorcerer's Stone (a.k.a. Harry Potter and the Philosopher's Stone) (2001)</td>\n",
       "      <td>Adventure|Children|Fantasy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>1914</td>\n",
       "      <td>0.571401</td>\n",
       "      <td>Smoke Signals (1998)</td>\n",
       "      <td>Comedy|Drama</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>5673</td>\n",
       "      <td>0.534286</td>\n",
       "      <td>Punch-Drunk Love (2002)</td>\n",
       "      <td>Comedy|Drama|Romance</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1</td>\n",
       "      <td>3342</td>\n",
       "      <td>0.386565</td>\n",
       "      <td>Birdy (1984)</td>\n",
       "      <td>Drama|War</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>1</td>\n",
       "      <td>104218</td>\n",
       "      <td>0.350539</td>\n",
       "      <td>Grown Ups 2 (2013)</td>\n",
       "      <td>Comedy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>1</td>\n",
       "      <td>267</td>\n",
       "      <td>0.347575</td>\n",
       "      <td>Major Payne (1995)</td>\n",
       "      <td>Comedy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>1</td>\n",
       "      <td>67295</td>\n",
       "      <td>0.332892</td>\n",
       "      <td>Kung Fu Panda: Secrets of the Furious Five (2008)</td>\n",
       "      <td>Action|Animation|Children|Comedy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>1</td>\n",
       "      <td>3089</td>\n",
       "      <td>0.296297</td>\n",
       "      <td>Bicycle Thieves (a.k.a. The Bicycle Thief) (a.k.a. The Bicycle Thieves) (Ladri di biciclette) (1948)</td>\n",
       "      <td>Drama</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>4</td>\n",
       "      <td>233</td>\n",
       "      <td>0.901040</td>\n",
       "      <td>Exotica (1994)</td>\n",
       "      <td>Drama</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>4</td>\n",
       "      <td>5673</td>\n",
       "      <td>0.838174</td>\n",
       "      <td>Punch-Drunk Love (2002)</td>\n",
       "      <td>Comedy|Drama|Romance</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>4</td>\n",
       "      <td>41566</td>\n",
       "      <td>0.793882</td>\n",
       "      <td>Chronicles of Narnia: The Lion, the Witch and the Wardrobe, The (2005)</td>\n",
       "      <td>Adventure|Children|Fantasy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>4</td>\n",
       "      <td>3089</td>\n",
       "      <td>0.589797</td>\n",
       "      <td>Bicycle Thieves (a.k.a. The Bicycle Thief) (a.k.a. The Bicycle Thieves) (Ladri di biciclette) (1948)</td>\n",
       "      <td>Drama</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>4</td>\n",
       "      <td>435</td>\n",
       "      <td>0.533054</td>\n",
       "      <td>Coneheads (1993)</td>\n",
       "      <td>Comedy|Sci-Fi</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>4</td>\n",
       "      <td>780</td>\n",
       "      <td>0.458692</td>\n",
       "      <td>Independence Day (a.k.a. ID4) (1996)</td>\n",
       "      <td>Action|Adventure|Sci-Fi|Thriller</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>4</td>\n",
       "      <td>6502</td>\n",
       "      <td>0.237901</td>\n",
       "      <td>28 Days Later (2002)</td>\n",
       "      <td>Action|Horror|Sci-Fi</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>4</td>\n",
       "      <td>2275</td>\n",
       "      <td>0.213286</td>\n",
       "      <td>Six-String Samurai (1998)</td>\n",
       "      <td>Action|Adventure|Sci-Fi</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>4</td>\n",
       "      <td>118270</td>\n",
       "      <td>0.187004</td>\n",
       "      <td>Hellbenders (2012)</td>\n",
       "      <td>Comedy|Horror|Thriller</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>4</td>\n",
       "      <td>55444</td>\n",
       "      <td>0.186750</td>\n",
       "      <td>Control (2007)</td>\n",
       "      <td>Drama</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>6</td>\n",
       "      <td>3089</td>\n",
       "      <td>0.864304</td>\n",
       "      <td>Bicycle Thieves (a.k.a. The Bicycle Thief) (a.k.a. The Bicycle Thieves) (Ladri di biciclette) (1948)</td>\n",
       "      <td>Drama</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>6</td>\n",
       "      <td>6502</td>\n",
       "      <td>0.855338</td>\n",
       "      <td>28 Days Later (2002)</td>\n",
       "      <td>Action|Horror|Sci-Fi</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>6</td>\n",
       "      <td>1914</td>\n",
       "      <td>0.791815</td>\n",
       "      <td>Smoke Signals (1998)</td>\n",
       "      <td>Comedy|Drama</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>6</td>\n",
       "      <td>118270</td>\n",
       "      <td>0.663633</td>\n",
       "      <td>Hellbenders (2012)</td>\n",
       "      <td>Comedy|Horror|Thriller</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>6</td>\n",
       "      <td>41566</td>\n",
       "      <td>0.611798</td>\n",
       "      <td>Chronicles of Narnia: The Lion, the Witch and the Wardrobe, The (2005)</td>\n",
       "      <td>Adventure|Children|Fantasy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>6</td>\n",
       "      <td>3714</td>\n",
       "      <td>0.600252</td>\n",
       "      <td>Clara's Heart (1988)</td>\n",
       "      <td>Drama</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>6</td>\n",
       "      <td>233</td>\n",
       "      <td>0.465912</td>\n",
       "      <td>Exotica (1994)</td>\n",
       "      <td>Drama</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>6</td>\n",
       "      <td>3551</td>\n",
       "      <td>0.462927</td>\n",
       "      <td>Marathon Man (1976)</td>\n",
       "      <td>Crime|Drama|Thriller</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>6</td>\n",
       "      <td>3614</td>\n",
       "      <td>0.420233</td>\n",
       "      <td>Honeymoon in Vegas (1992)</td>\n",
       "      <td>Comedy|Romance</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>6</td>\n",
       "      <td>47382</td>\n",
       "      <td>0.381753</td>\n",
       "      <td>Step Up (2006)</td>\n",
       "      <td>Drama|Romance</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "recommendations = gmf_recommender.recommend(pd.DataFrame([[1], [4], [6]], columns=['user_id']), ml_movies_df, 10)\n",
    "\n",
    "recommendations = pd.merge(recommendations, ml_movies_df, on='item_id', how='left')\n",
    "print(\"Recommendations\")\n",
    "display(HTML(recommendations.to_html()))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "documentary-barcelona",
   "metadata": {},
   "source": [
    "## User and item representations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "balanced-detective",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "User id=1\n",
      "[ 1.583236    1.928833    0.5076821   0.02077339 -0.7576393   0.720875  ]\n",
      "\n",
      "User watched\n",
      "['Independence Day (a.k.a. ID4) (1996)', 'Grosse Pointe Blank (1997)', 'Ladyhawke (1985)']\n",
      "\n",
      "User history item representations\n",
      "Item id = 780\titem title = Independence Day (a.k.a. ID4) (1996)\n",
      "[-0.08688126 -0.5049163  -0.5540194  -1.7089794   0.36428154 -0.8702819 ]\n",
      "Scalar product=-2.331578\n",
      "Score=0.864368\n",
      "\n",
      "Item id = 1500\titem title = Grosse Pointe Blank (1997)\n",
      "[-0.99034846 -1.2885239  -0.03223765  0.10811037 -1.2086896  -0.84467167]\n",
      "Scalar product=-3.760575\n",
      "Score=0.395192\n",
      "\n",
      "Item id = 3479\titem title = Ladyhawke (1985)\n",
      "[-0.13638315  1.1943792   1.1057804   0.9315033  -0.71073234 -1.4644285 ]\n",
      "Scalar product=2.151376\n",
      "Score=0.056009\n",
      "\n",
      "===============\n",
      "Item id = 145\titem title = Bad Boys (1995)\n",
      "[-0.4696549   0.4609577   0.35850123 -1.0623587   0.9433856   0.8181307 ]\n",
      "Scalar product=0.180496\n",
      "Score=0.155046\n",
      "\n",
      "Item id = 171\titem title = Jeffrey (1995)\n",
      "[ 0.5504582  -0.712692    1.8113137  -0.12408835 -1.2910577   0.67041284]\n",
      "Scalar product=1.875275\n",
      "Score=0.148060\n"
     ]
    }
   ],
   "source": [
    "user_id = 1\n",
    "user_repr = gmf_recommender.get_user_repr(user_id=user_id)\n",
    "print(\"User id={}\".format(user_id))\n",
    "print(user_repr)\n",
    "print()\n",
    "\n",
    "print(\"User watched\")\n",
    "print(ml_df.loc[ml_df['user_id'] == user_id, 'title'].tolist())\n",
    "print()\n",
    "\n",
    "print('User history item representations')\n",
    "for item_id in ml_df.loc[ml_df['user_id'] == user_id, 'item_id'].tolist():\n",
    "    item_repr = gmf_recommender.get_item_repr(item_id=item_id)\n",
    "    print(\"Item id = {}\\titem title = {}\".format(\n",
    "        item_id, ml_movies_df.loc[ml_movies_df['item_id'] == item_id, 'title'].iloc[0]))\n",
    "    print(item_repr)\n",
    "    scalar_product = np.dot(user_repr, item_repr)\n",
    "    print(\"Scalar product={:.6f}\".format(scalar_product))\n",
    "    score = gmf_recommender.nn_model(\n",
    "        torch.tensor([[gmf_recommender.user_id_mapping[user_id], \n",
    "                       gmf_recommender.item_id_mapping[item_id]]]).to(gmf_recommender.device)).flatten().detach().cpu().item()\n",
    "    print(\"Score={:.6f}\".format(score))\n",
    "    print()\n",
    "\n",
    "print(\"===============\")\n",
    "    \n",
    "item_id = 145\n",
    "item_repr = gmf_recommender.get_item_repr(item_id=item_id)\n",
    "print(\"Item id = {}\\titem title = {}\".format(item_id, ml_movies_df.loc[ml_movies_df['item_id'] == item_id, 'title'].iloc[0]))\n",
    "print(item_repr)\n",
    "score = np.dot(user_repr, item_repr)\n",
    "print(\"Scalar product={:.6f}\".format(score))\n",
    "score = gmf_recommender.nn_model(\n",
    "    torch.tensor([[gmf_recommender.user_id_mapping[user_id], \n",
    "                   gmf_recommender.item_id_mapping[item_id]]]).to(gmf_recommender.device)).flatten().detach().cpu().item()\n",
    "print(\"Score={:.6f}\".format(score))\n",
    "print()\n",
    "\n",
    "item_id = 171\n",
    "item_repr = gmf_recommender.get_item_repr(item_id=item_id)\n",
    "print(\"Item id = {}\\titem title = {}\".format(item_id, ml_movies_df.loc[ml_movies_df['item_id'] == item_id, 'title'].iloc[0]))\n",
    "print(item_repr)\n",
    "score = np.dot(user_repr, item_repr)\n",
    "print(\"Scalar product={:.6f}\".format(score))\n",
    "score = gmf_recommender.nn_model(\n",
    "    torch.tensor([[gmf_recommender.user_id_mapping[user_id], \n",
    "                   gmf_recommender.item_id_mapping[item_id]]]).to(gmf_recommender.device)).flatten().detach().cpu().item()\n",
    "print(\"Score={:.6f}\".format(score))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "framed-negative",
   "metadata": {},
   "source": [
    "# Training-test split evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "amended-future",
   "metadata": {},
   "outputs": [],
   "source": [
    "from evaluation_and_testing.testing import evaluate_train_test_split_implicit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "unsigned-video",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Recommender</th>\n",
       "      <th>HR@1</th>\n",
       "      <th>HR@3</th>\n",
       "      <th>HR@5</th>\n",
       "      <th>HR@10</th>\n",
       "      <th>NDCG@1</th>\n",
       "      <th>NDCG@3</th>\n",
       "      <th>NDCG@5</th>\n",
       "      <th>NDCG@10</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>GMFRecommender</td>\n",
       "      <td>0.233766</td>\n",
       "      <td>0.402597</td>\n",
       "      <td>0.506494</td>\n",
       "      <td>0.675325</td>\n",
       "      <td>0.233766</td>\n",
       "      <td>0.330935</td>\n",
       "      <td>0.37255</td>\n",
       "      <td>0.42848</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "gmf_recommender = GMFRecommender(n_neg_per_pos=10, batch_size=16, \n",
    "                                 embedding_dim=6, lr=0.001, weight_decay=0.0001, n_epochs=25)\n",
    "\n",
    "gmf_tts_results = [['GMFRecommender'] + list(evaluate_train_test_split_implicit(\n",
    "    gmf_recommender, ml_ratings_df.loc[:, ['user_id', 'item_id']], ml_movies_df))]\n",
    "\n",
    "gmf_tts_results = pd.DataFrame(\n",
    "    gmf_tts_results, columns=['Recommender', 'HR@1', 'HR@3', 'HR@5', 'HR@10', 'NDCG@1', 'NDCG@3', 'NDCG@5', 'NDCG@10'])\n",
    "\n",
    "display(HTML(gmf_tts_results.to_html()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "romantic-music",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Recommender</th>\n",
       "      <th>HR@1</th>\n",
       "      <th>HR@3</th>\n",
       "      <th>HR@5</th>\n",
       "      <th>HR@10</th>\n",
       "      <th>NDCG@1</th>\n",
       "      <th>NDCG@3</th>\n",
       "      <th>NDCG@5</th>\n",
       "      <th>NDCG@10</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>NetflixRecommender</td>\n",
       "      <td>0.292208</td>\n",
       "      <td>0.538961</td>\n",
       "      <td>0.733766</td>\n",
       "      <td>0.948052</td>\n",
       "      <td>0.292208</td>\n",
       "      <td>0.434289</td>\n",
       "      <td>0.514203</td>\n",
       "      <td>0.583217</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from recommenders.netflix_recommender import NetflixRecommender\n",
    "\n",
    "netflix_recommender = NetflixRecommender(n_epochs=150)\n",
    "\n",
    "netflix_tts_results = [['NetflixRecommender'] + list(evaluate_train_test_split_implicit(\n",
    "    netflix_recommender, ml_ratings_df.loc[:, ['user_id', 'item_id']], ml_movies_df))]\n",
    "\n",
    "netflix_tts_results = pd.DataFrame(\n",
    "    netflix_tts_results, columns=['Recommender', 'HR@1', 'HR@3', 'HR@5', 'HR@10', 'NDCG@1', 'NDCG@3', 'NDCG@5', 'NDCG@10'])\n",
    "\n",
    "display(HTML(netflix_tts_results.to_html()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "standing-tiffany",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Recommender</th>\n",
       "      <th>HR@1</th>\n",
       "      <th>HR@3</th>\n",
       "      <th>HR@5</th>\n",
       "      <th>HR@10</th>\n",
       "      <th>NDCG@1</th>\n",
       "      <th>NDCG@3</th>\n",
       "      <th>NDCG@5</th>\n",
       "      <th>NDCG@10</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>AmazonRecommender</td>\n",
       "      <td>0.181818</td>\n",
       "      <td>0.311688</td>\n",
       "      <td>0.402597</td>\n",
       "      <td>0.551948</td>\n",
       "      <td>0.181818</td>\n",
       "      <td>0.257806</td>\n",
       "      <td>0.294682</td>\n",
       "      <td>0.34147</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from recommenders.amazon_recommender import AmazonRecommender\n",
    "\n",
    "amazon_recommender = AmazonRecommender()\n",
    "\n",
    "amazon_tts_results = [['AmazonRecommender'] + list(evaluate_train_test_split_implicit(\n",
    "    amazon_recommender, ml_ratings_df.loc[:, ['user_id', 'item_id']], ml_movies_df))]\n",
    "\n",
    "amazon_tts_results = pd.DataFrame(\n",
    "    amazon_tts_results, columns=['Recommender', 'HR@1', 'HR@3', 'HR@5', 'HR@10', 'NDCG@1', 'NDCG@3', 'NDCG@5', 'NDCG@10'])\n",
    "\n",
    "display(HTML(amazon_tts_results.to_html()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "saving-harrison",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Recommender</th>\n",
       "      <th>HR@1</th>\n",
       "      <th>HR@3</th>\n",
       "      <th>HR@5</th>\n",
       "      <th>HR@10</th>\n",
       "      <th>NDCG@1</th>\n",
       "      <th>NDCG@3</th>\n",
       "      <th>NDCG@5</th>\n",
       "      <th>NDCG@10</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>TFIDFRecommender</td>\n",
       "      <td>0.025974</td>\n",
       "      <td>0.090909</td>\n",
       "      <td>0.136364</td>\n",
       "      <td>0.318182</td>\n",
       "      <td>0.025974</td>\n",
       "      <td>0.064393</td>\n",
       "      <td>0.083685</td>\n",
       "      <td>0.140799</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from recommenders.tfidf_recommender import TFIDFRecommender\n",
    "\n",
    "tfidf_recommender = TFIDFRecommender()\n",
    "\n",
    "tfidf_tts_results = [['TFIDFRecommender'] + list(evaluate_train_test_split_implicit(\n",
    "    tfidf_recommender, ml_ratings_df.loc[:, ['user_id', 'item_id']], ml_movies_df))]\n",
    "\n",
    "tfidf_tts_results = pd.DataFrame(\n",
    "    tfidf_tts_results, columns=['Recommender', 'HR@1', 'HR@3', 'HR@5', 'HR@10', 'NDCG@1', 'NDCG@3', 'NDCG@5', 'NDCG@10'])\n",
    "\n",
    "display(HTML(tfidf_tts_results.to_html()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "random-source",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Recommender</th>\n",
       "      <th>HR@1</th>\n",
       "      <th>HR@3</th>\n",
       "      <th>HR@5</th>\n",
       "      <th>HR@10</th>\n",
       "      <th>NDCG@1</th>\n",
       "      <th>NDCG@3</th>\n",
       "      <th>NDCG@5</th>\n",
       "      <th>NDCG@10</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>GMFRecommender</td>\n",
       "      <td>0.233766</td>\n",
       "      <td>0.402597</td>\n",
       "      <td>0.506494</td>\n",
       "      <td>0.675325</td>\n",
       "      <td>0.233766</td>\n",
       "      <td>0.330935</td>\n",
       "      <td>0.372550</td>\n",
       "      <td>0.428480</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>NetflixRecommender</td>\n",
       "      <td>0.292208</td>\n",
       "      <td>0.538961</td>\n",
       "      <td>0.733766</td>\n",
       "      <td>0.948052</td>\n",
       "      <td>0.292208</td>\n",
       "      <td>0.434289</td>\n",
       "      <td>0.514203</td>\n",
       "      <td>0.583217</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>AmazonRecommender</td>\n",
       "      <td>0.181818</td>\n",
       "      <td>0.311688</td>\n",
       "      <td>0.402597</td>\n",
       "      <td>0.551948</td>\n",
       "      <td>0.181818</td>\n",
       "      <td>0.257806</td>\n",
       "      <td>0.294682</td>\n",
       "      <td>0.341470</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>TFIDFRecommender</td>\n",
       "      <td>0.025974</td>\n",
       "      <td>0.090909</td>\n",
       "      <td>0.136364</td>\n",
       "      <td>0.318182</td>\n",
       "      <td>0.025974</td>\n",
       "      <td>0.064393</td>\n",
       "      <td>0.083685</td>\n",
       "      <td>0.140799</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "tts_results = pd.concat([gmf_tts_results, netflix_tts_results, amazon_tts_results, tfidf_tts_results]).reset_index(drop=True)\n",
    "display(HTML(tts_results.to_html()))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "continued-harassment",
   "metadata": {},
   "source": [
    "# Leave-one-out evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "exact-stuff",
   "metadata": {},
   "outputs": [],
   "source": [
    "from evaluation_and_testing.testing import evaluate_leave_one_out_implicit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "divided-resistance",
   "metadata": {},
   "outputs": [],
   "source": [
    "gmf_recommender = GMFRecommender(n_neg_per_pos=10, batch_size=16, \n",
    "                                 embedding_dim=6, lr=0.001, weight_decay=0.0001, n_epochs=25)\n",
    "\n",
    "gmf_loo_results = [['GMFRecommender'] + list(evaluate_leave_one_out_implicit(\n",
    "    gmf_recommender, ml_ratings_df.loc[:, ['user_id', 'item_id']], ml_movies_df, max_evals=300, seed=6789))]\n",
    "\n",
    "gmf_loo_results = pd.DataFrame(\n",
    "    gmf_loo_results, columns=['Recommender', 'HR@1', 'HR@3', 'HR@5', 'HR@10', 'NDCG@1', 'NDCG@3', 'NDCG@5', 'NDCG@10'])\n",
    "\n",
    "display(HTML(gmf_loo_results.to_html()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "prerequisite-lounge",
   "metadata": {},
   "outputs": [],
   "source": [
    "netflix_recommender = NetflixRecommender(n_epochs=150)\n",
    "\n",
    "netflix_loo_results = [['NetflixRecommender'] + list(evaluate_leave_one_out_implicit(\n",
    "    netflix_recommender, ml_ratings_df.loc[:, ['user_id', 'item_id']], ml_movies_df, max_evals=300, seed=6789))]\n",
    "\n",
    "netflix_loo_results = pd.DataFrame(\n",
    "    netflix_loo_results, columns=['Recommender', 'HR@1', 'HR@3', 'HR@5', 'HR@10', 'NDCG@1', 'NDCG@3', 'NDCG@5', 'NDCG@10'])\n",
    "\n",
    "display(HTML(netflix_loo_results.to_html()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "social-escape",
   "metadata": {},
   "outputs": [],
   "source": [
    "from recommenders.amazon_recommender import AmazonRecommender\n",
    "\n",
    "amazon_recommender = AmazonRecommender()\n",
    "\n",
    "amazon_loo_results = [['AmazonRecommender'] + list(evaluate_leave_one_out_implicit(\n",
    "    amazon_recommender, ml_ratings_df.loc[:, ['user_id', 'item_id']], ml_movies_df, max_evals=300, seed=6789))]\n",
    "\n",
    "amazon_loo_results = pd.DataFrame(\n",
    "    amazon_loo_results, columns=['Recommender', 'HR@1', 'HR@3', 'HR@5', 'HR@10', 'NDCG@1', 'NDCG@3', 'NDCG@5', 'NDCG@10'])\n",
    "\n",
    "display(HTML(amazon_loo_results.to_html()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "behind-cambodia",
   "metadata": {},
   "outputs": [],
   "source": [
    "tfidf_recommender = TFIDFRecommender()\n",
    "\n",
    "tfidf_loo_results = [['TFIDFRecommender'] + list(evaluate_leave_one_out_implicit(\n",
    "    tfidf_recommender, ml_ratings_df.loc[:, ['user_id', 'item_id']], ml_movies_df, max_evals=300, seed=6789))]\n",
    "\n",
    "tfidf_loo_results = pd.DataFrame(\n",
    "    tfidf_loo_results, columns=['Recommender', 'HR@1', 'HR@3', 'HR@5', 'HR@10', 'NDCG@1', 'NDCG@3', 'NDCG@5', 'NDCG@10'])\n",
    "\n",
    "display(HTML(tfidf_loo_results.to_html()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "lightweight-password",
   "metadata": {},
   "outputs": [],
   "source": [
    "loo_results = pd.concat([gmf_loo_results, netflix_loo_results, amazon_loo_results, tfidf_loo_results]).reset_index(drop=True)\n",
    "display(HTML(loo_results.to_html()))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}