Amazon-last-enhanced-data-Copy1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c3e2f4d9-7994-4d24-9062-b10fc9f79168",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "b7dedcce-4b97-490c-a8db-89ed9b518456",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/cevher/.local/lib/python3.8/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "from torch_geometric.data import Data, DataLoader\n",
    "from torch_geometric.nn import GCNConv, GATConv, SAGEConv, global_mean_pool\n",
    "from datasets import load_dataset\n",
    "import networkx as nx\n",
    "from sklearn.model_selection import train_test_split\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "d60bf3cb-8446-4926-8523-6120ea7269a4",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = load_dataset(\"McAuley-Lab/Amazon-Reviews-2023\", \"raw_review_All_Beauty\", trust_remote_code=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "b97596fb-f4fd-4bc6-9d2a-0a3ea2736f23",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    full: Dataset({\n",
       "        features: ['rating', 'title', 'text', 'images', 'asin', 'parent_asin', 'user_id', 'timestamp', 'helpful_vote', 'verified_purchase'],\n",
       "        num_rows: 701528\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "bfcd2ccc-eeec-4ead-ac07-1b19a18a1096",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/cevher/.local/lib/python3.8/site-packages/torch_geometric/deprecation.py:26: UserWarning: 'data.DataLoader' is deprecated, use 'loader.DataLoader' instead\n",
      "  warnings.warn(out)\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "from torch_geometric.data import Data\n",
    "\n",
    "def create_graph_data(review):\n",
    "    # Her bir incelemeyi bir embedding vektörüne dönüştürme\n",
    "    review_vector = torch.randn(128)  # Her düğüm için 768 boyutlu özellik vektörü\n",
    "\n",
    "    # Basit bir self-loop kenar yapısı oluşturuyoruz (her node kendisine bağlı)\n",
    "    edge_index = torch.tensor([[0], [0]], dtype=torch.long)  # [2, num_edges] boyutunda olmalı\n",
    "\n",
    "    # Node'u oluşturmak\n",
    "    node = Data(x=review_vector.unsqueeze(0), edge_index=edge_index, y=torch.tensor([review['rating']], dtype=torch.float))\n",
    "    \n",
    "    return node\n",
    "\n",
    "# İlk 1000 örneği alıyoruz\n",
    "reviews = dataset['full'].select(range(100000))  # Dataset'in ilk 1000 örneğini seçiyoruz\n",
    "\n",
    "# Her bir incelemeyi işleyerek graph verisi oluşturuyoruz\n",
    "graph_data = [create_graph_data(review) for review in reviews]\n",
    "\n",
    "train_data, test_data = train_test_split(graph_data, test_size=0.2, random_state=42)\n",
    "\n",
    "train_loader = DataLoader(train_data, batch_size=32, shuffle=True)\n",
    "test_loader = DataLoader(test_data, batch_size=32, shuffle=False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "12b75371-55d9-4abb-a4de-b93cf9b453ec",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "from torch_geometric.nn import GCNConv, GATConv, global_mean_pool, SAGEConv\n",
    "from torch_geometric.data import Data, DataLoader\n",
    "\n",
    "class Encoder(nn.Module):\n",
    "    def __init__(self, in_channels, hidden_channels):\n",
    "        super(Encoder, self).__init__()\n",
    "        self.conv1 = SAGEConv(in_channels, hidden_channels)\n",
    "        self.conv2 = SAGEConv(hidden_channels, hidden_channels)\n",
    "\n",
    "    def forward(self, x, edge_index):\n",
    "        x = self.conv1(x, edge_index)\n",
    "        x = torch.relu(x)\n",
    "        x = self.conv2(x, edge_index)\n",
    "        return x\n",
    "\n",
    "class Decoder(nn.Module):\n",
    "    def __init__(self, hidden_channels, out_channels):\n",
    "        super(Decoder, self).__init__()\n",
    "        self.linear = nn.Linear(hidden_channels, out_channels)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.linear(x)\n",
    "        return x\n",
    "\n",
    "class GNNModel(nn.Module):\n",
    "    def __init__(self, in_channels, hidden_channels, out_channels):\n",
    "        super(GNNModel, self).__init__()\n",
    "        self.encoder = Encoder(in_channels, hidden_channels)\n",
    "        self.decoder = Decoder(hidden_channels, out_channels)\n",
    "\n",
    "    def forward(self, x, edge_index, batch):\n",
    "        x = self.encoder(x, edge_index)\n",
    "        x = global_mean_pool(x, batch)  # Global mean pooling over all nodes\n",
    "        x = self.decoder(x)\n",
    "        return x\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "489e9f7d-291f-4336-8e56-89bd98a1e32a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1, Train Loss: 2.4107, Test Loss: 2.4393\n",
      "Epoch 2, Train Loss: 2.2395, Test Loss: 2.2248\n",
      "Epoch 3, Train Loss: 2.0304, Test Loss: 2.0389\n",
      "Epoch 4, Train Loss: 1.9872, Test Loss: 1.9742\n",
      "Epoch 5, Train Loss: 2.0345, Test Loss: 2.0408\n",
      "Epoch 6, Train Loss: 2.1779, Test Loss: 2.1528\n",
      "Epoch 7, Train Loss: 1.9807, Test Loss: 1.9825\n",
      "Epoch 8, Train Loss: 2.0326, Test Loss: 2.0160\n",
      "Epoch 9, Train Loss: 1.9867, Test Loss: 1.9792\n",
      "Epoch 10, Train Loss: 2.2441, Test Loss: 2.2003\n",
      "Epoch 11, Train Loss: 1.9463, Test Loss: 1.9468\n",
      "Epoch 12, Train Loss: 1.9679, Test Loss: 1.9595\n",
      "Epoch 13, Train Loss: 2.0463, Test Loss: 2.0065\n",
      "Epoch 14, Train Loss: 1.9722, Test Loss: 1.9740\n",
      "Epoch 15, Train Loss: 2.0611, Test Loss: 2.0340\n",
      "Epoch 16, Train Loss: 2.0284, Test Loss: 2.0032\n",
      "Epoch 17, Train Loss: 1.9978, Test Loss: 1.9929\n",
      "Epoch 18, Train Loss: 1.9447, Test Loss: 1.9443\n",
      "Epoch 19, Train Loss: 1.9546, Test Loss: 1.9509\n",
      "Epoch 20, Train Loss: 2.0902, Test Loss: 2.0731\n",
      "Epoch 21, Train Loss: 1.9410, Test Loss: 1.9443\n",
      "Epoch 22, Train Loss: 1.9494, Test Loss: 1.9533\n",
      "Epoch 23, Train Loss: 2.0260, Test Loss: 1.9822\n",
      "Epoch 24, Train Loss: 1.9571, Test Loss: 1.9883\n",
      "Epoch 25, Train Loss: 1.9415, Test Loss: 1.9449\n",
      "Epoch 26, Train Loss: 1.9400, Test Loss: 1.9443\n",
      "Epoch 27, Train Loss: 1.9330, Test Loss: 1.9380\n",
      "Epoch 28, Train Loss: 2.0246, Test Loss: 2.0211\n",
      "Epoch 29, Train Loss: 1.9561, Test Loss: 1.9408\n",
      "Epoch 30, Train Loss: 1.9367, Test Loss: 1.9449\n",
      "Epoch 31, Train Loss: 1.9320, Test Loss: 1.9362\n",
      "Epoch 32, Train Loss: 1.9368, Test Loss: 1.9401\n",
      "Epoch 33, Train Loss: 1.9365, Test Loss: 1.9356\n",
      "Epoch 34, Train Loss: 1.9316, Test Loss: 1.9360\n",
      "Epoch 35, Train Loss: 1.9428, Test Loss: 1.9514\n",
      "Epoch 36, Train Loss: 1.9336, Test Loss: 1.9362\n",
      "Epoch 37, Train Loss: 1.9458, Test Loss: 1.9503\n",
      "Epoch 38, Train Loss: 1.9344, Test Loss: 1.9380\n",
      "Epoch 39, Train Loss: 1.9342, Test Loss: 1.9378\n",
      "Epoch 40, Train Loss: 1.9359, Test Loss: 1.9424\n",
      "Epoch 41, Train Loss: 1.9326, Test Loss: 1.9364\n",
      "Epoch 42, Train Loss: 1.9319, Test Loss: 1.9368\n",
      "Epoch 43, Train Loss: 1.9313, Test Loss: 1.9358\n",
      "Epoch 44, Train Loss: 1.9341, Test Loss: 1.9373\n",
      "Epoch 45, Train Loss: 1.9317, Test Loss: 1.9371\n",
      "Epoch 46, Train Loss: 1.9328, Test Loss: 1.9367\n",
      "Epoch 47, Train Loss: 1.9313, Test Loss: 1.9358\n",
      "Epoch 48, Train Loss: 1.9388, Test Loss: 1.9412\n",
      "Epoch 49, Train Loss: 1.9314, Test Loss: 1.9411\n",
      "Epoch 50, Train Loss: 1.9385, Test Loss: 1.9434\n",
      "Epoch 51, Train Loss: 1.9336, Test Loss: 1.9393\n",
      "Epoch 52, Train Loss: 1.9311, Test Loss: 1.9368\n",
      "Epoch 53, Train Loss: 1.9460, Test Loss: 1.9531\n",
      "Epoch 54, Train Loss: 1.9329, Test Loss: 1.9381\n",
      "Epoch 55, Train Loss: 1.9424, Test Loss: 1.9433\n",
      "Epoch 56, Train Loss: 1.9367, Test Loss: 1.9395\n",
      "Epoch 57, Train Loss: 1.9318, Test Loss: 1.9368\n",
      "Epoch 58, Train Loss: 1.9362, Test Loss: 1.9420\n",
      "Epoch 59, Train Loss: 1.9313, Test Loss: 1.9360\n",
      "Epoch 60, Train Loss: 1.9332, Test Loss: 1.9370\n",
      "Epoch 61, Train Loss: 2.0305, Test Loss: 2.0153\n",
      "Epoch 62, Train Loss: 1.9317, Test Loss: 1.9358\n",
      "Epoch 63, Train Loss: 1.9786, Test Loss: 1.9887\n",
      "Epoch 64, Train Loss: 1.9325, Test Loss: 1.9364\n",
      "Epoch 65, Train Loss: 1.9316, Test Loss: 1.9365\n",
      "Epoch 66, Train Loss: 1.9342, Test Loss: 1.9394\n",
      "Epoch 67, Train Loss: 1.9312, Test Loss: 1.9359\n",
      "Epoch 68, Train Loss: 1.9314, Test Loss: 1.9356\n",
      "Epoch 69, Train Loss: 1.9452, Test Loss: 1.9507\n",
      "Epoch 70, Train Loss: 1.9319, Test Loss: 1.9370\n",
      "Epoch 71, Train Loss: 1.9322, Test Loss: 1.9362\n",
      "Epoch 72, Train Loss: 1.9317, Test Loss: 1.9360\n",
      "Epoch 73, Train Loss: 1.9359, Test Loss: 1.9415\n",
      "Epoch 74, Train Loss: 1.9318, Test Loss: 1.9359\n",
      "Epoch 75, Train Loss: 1.9322, Test Loss: 1.9361\n",
      "Epoch 76, Train Loss: 1.9313, Test Loss: 1.9363\n",
      "Epoch 77, Train Loss: 1.9313, Test Loss: 1.9359\n",
      "Epoch 78, Train Loss: 1.9563, Test Loss: 1.9640\n",
      "Epoch 79, Train Loss: 1.9360, Test Loss: 1.9402\n",
      "Epoch 80, Train Loss: 1.9344, Test Loss: 1.9378\n",
      "Epoch 81, Train Loss: 1.9320, Test Loss: 1.9373\n",
      "Epoch 82, Train Loss: 1.9327, Test Loss: 1.9379\n",
      "Epoch 83, Train Loss: 1.9313, Test Loss: 1.9359\n",
      "Epoch 84, Train Loss: 1.9323, Test Loss: 1.9376\n",
      "Epoch 85, Train Loss: 1.9354, Test Loss: 1.9414\n",
      "Epoch 86, Train Loss: 1.9316, Test Loss: 1.9354\n",
      "Epoch 87, Train Loss: 1.9424, Test Loss: 1.9488\n",
      "Epoch 88, Train Loss: 1.9321, Test Loss: 1.9372\n",
      "Epoch 89, Train Loss: 1.9314, Test Loss: 1.9362\n",
      "Epoch 90, Train Loss: 1.9319, Test Loss: 1.9360\n",
      "Epoch 91, Train Loss: 1.9320, Test Loss: 1.9370\n",
      "Epoch 92, Train Loss: 1.9335, Test Loss: 1.9389\n",
      "Epoch 93, Train Loss: 1.9346, Test Loss: 1.9402\n",
      "Epoch 94, Train Loss: 1.9312, Test Loss: 1.9360\n",
      "Epoch 95, Train Loss: 1.9336, Test Loss: 1.9373\n",
      "Epoch 96, Train Loss: 1.9318, Test Loss: 1.9360\n",
      "Epoch 97, Train Loss: 1.9517, Test Loss: 1.9542\n",
      "Epoch 98, Train Loss: 1.9325, Test Loss: 1.9377\n",
      "Epoch 99, Train Loss: 1.9314, Test Loss: 1.9358\n"
     ]
    }
   ],
   "source": [
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "model = GNNModel(in_channels=128, hidden_channels=128, out_channels=1).to(device)  # 768 input features, 128 hidden units, 1 output (rating)\n",
    "optimizer = optim.Adam(model.parameters(), lr=0.01)\n",
    "criterion = nn.MSELoss()  # Mean Squared Error Loss, çünkü rating tahmin ediyoruz\n",
    "\n",
    "def train():\n",
    "    model.train()\n",
    "    for data in train_loader:\n",
    "        data = data.to(device)\n",
    "        optimizer.zero_grad()\n",
    "        out = model(data.x, data.edge_index, data.batch)\n",
    "        loss = criterion(out.squeeze(), data.y)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "def test(loader):\n",
    "    model.eval()\n",
    "    total_loss = 0\n",
    "    with torch.no_grad():\n",
    "        for data in loader:\n",
    "            data = data.to(device)\n",
    "            out = model(data.x, data.edge_index, data.batch)\n",
    "            loss = criterion(out.squeeze(), data.y)\n",
    "            total_loss += loss.item()\n",
    "    return total_loss / len(loader)\n",
    "\n",
    "for epoch in range(1, 100):  # 100 epoch boyunca eğitiyoruz\n",
    "    train()\n",
    "    train_loss = test(train_loader)\n",
    "    test_loss = test(test_loader)\n",
    "    print(f'Epoch {epoch}, Train Loss: {train_loss:.4f}, Test Loss: {test_loss:.4f}')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "0f36ce25-7b90-4dde-be54-ca4276f065a1",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Encoder katmanlarındaki ağırlıkları kaydetme\n",
    "encoder_state_dict = {\n",
    "    'conv1': model.encoder.conv1.state_dict(),\n",
    "    'conv2': model.encoder.conv2.state_dict(),\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "74f9e4d2-bda6-47c9-b694-7d97c404f8a1",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "lin_l.weight: torch.Size([128, 128])\n",
      "lin_l.bias: torch.Size([128])\n",
      "lin_r.weight: torch.Size([128, 128])\n"
     ]
    }
   ],
   "source": [
    "for param_name, param_tensor in encoder_state_dict['conv2'].items():\n",
    "    print(f\"{param_name}: {param_tensor.shape}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3d349bea-d3b4-415b-8ba9-d49c43cdac90",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "80f04655-8f41-49c3-9a68-85a3493d0e28",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4163866c-854b-4b60-8344-dc87d65db230",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "c18ed08e-ccf1-472e-b4d8-ce6ae63a557c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.3.0+cu121\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "from torch import Tensor\n",
    "from torch.nn.functional import normalize\n",
    "from torch_geometric_temporal.nn.recurrent import GConvGRU\n",
    "print(torch.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "91598954-5ee4-4a44-8b86-d2c32797e71f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.cuda.is_available()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "7c450a26-510a-4408-97e9-3dc22cc22212",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Install required packages.\n",
    "import os\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "os.environ['TORCH'] = torch.__version__\n",
    "\n",
    "# !pip install torch-scatter -f https://data.pyg.org/whl/torch-${TORCH}.html\n",
    "# !pip install torch-sparse -f https://data.pyg.org/whl/torch-${TORCH}.html\n",
    "# !pip install pyg-lib -f https://data.pyg.org/whl/nightly/torch-${TORCH}.html\n",
    "# !pip install git+https://github.com/pyg-team/pytorch_geometric.git"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "773e57f6-6192-49b5-9ec8-5b97dbf4b2a8",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/home/cevher/movielens'"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "os.getcwd()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8cd0abcf-bb90-4d04-be9f-13f784276aca",
   "metadata": {},
   "source": [
    "### Link Prediction on MovieLens"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d4e42911-bc10-475c-bb5c-b3143c681a94",
   "metadata": {},
   "source": [
    "### Heterogeneous Graph Creation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "5bb6f401-304e-47c3-85f9-a031e6b04d99",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "movies.csv:\n",
      "===========\n",
      "   movieId                                             genres\n",
      "0        1  ['Adventure', 'Animation', 'Children', 'Comedy...\n",
      "1        2               ['Adventure', 'Children', 'Fantasy']\n",
      "2        3                              ['Comedy', 'Romance']\n",
      "3        4                     ['Comedy', 'Drama', 'Romance']\n",
      "4        5                                         ['Comedy']\n",
      "\n",
      "ratings.csv:\n",
      "============\n",
      "   userId  movieId\n",
      "0       1        1\n",
      "1       1        3\n",
      "2       1        6\n",
      "3       1       47\n",
      "4       1       50\n"
     ]
    }
   ],
   "source": [
    "movies_path = './ml-latest-small/movies_cleaned.csv'\n",
    "ratings_path = './ml-latest-small/ratings_cleaned.csv'\n",
    "import pandas as pd\n",
    "\n",
    "print('movies.csv:')\n",
    "print('===========')\n",
    "movies = pd.read_csv(movies_path)\n",
    "print(movies[[\"movieId\", \"genres\"]].head())\n",
    "print()\n",
    "print('ratings.csv:')\n",
    "print('============')\n",
    "ratings_df = pd.read_csv(ratings_path)\n",
    "print(ratings_df[[\"userId\", \"movieId\"]].head())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "20aaa2f4-b7f3-474a-a117-c568901708f8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>Unnamed: 0.1</th>\n",
       "      <th>Unnamed: 0.1.1</th>\n",
       "      <th>movieId</th>\n",
       "      <th>title</th>\n",
       "      <th>genres</th>\n",
       "      <th>wikipedia_page_name</th>\n",
       "      <th>movie_plot</th>\n",
       "      <th>token_ids</th>\n",
       "      <th>plot_ids</th>\n",
       "      <th>mean_tokens</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>Toy Story (1995)</td>\n",
       "      <td>['Adventure', 'Animation', 'Children', 'Comedy...</td>\n",
       "      <td>Toy Story</td>\n",
       "      <td>A group of sentient toys, who pretend to be li...</td>\n",
       "      <td>[1037, 2177, 1997, 2741, 11638, 10899, 1010, 2...</td>\n",
       "      <td>[ 1037.  2177.  1997.  2741. 11638. 10899.  10...</td>\n",
       "      <td>4984.423077</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>Jumanji (1995)</td>\n",
       "      <td>['Adventure', 'Children', 'Fantasy']</td>\n",
       "      <td>Jumanji</td>\n",
       "      <td>In 1969, Alan Parrish lives in Brantford, New ...</td>\n",
       "      <td>[1999, 3440, 1010, 5070, 11968, 18774, 3268, 1...</td>\n",
       "      <td>[ 1999.  3440.  1010.  5070. 11968. 18774.  32...</td>\n",
       "      <td>4644.158750</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>Grumpier Old Men (1995)</td>\n",
       "      <td>['Comedy', 'Romance']</td>\n",
       "      <td>Grumpier Old Men</td>\n",
       "      <td>The feud between Max and John has cooled and t...</td>\n",
       "      <td>[1996, 13552, 2090, 4098, 1998, 2198, 2038, 12...</td>\n",
       "      <td>[ 1996. 13552.  2090.  4098.  1998.  2198.  20...</td>\n",
       "      <td>4044.667638</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>Waiting to Exhale (1995)</td>\n",
       "      <td>['Comedy', 'Drama', 'Romance']</td>\n",
       "      <td>Waiting to Exhale</td>\n",
       "      <td>Four friends (Savannah, Robin, Bernadine, and ...</td>\n",
       "      <td>[2176, 2814, 1006, 10891, 1010, 5863, 1010, 16...</td>\n",
       "      <td>[ 2176.  2814.  1006. 10891.  1010.  5863.  10...</td>\n",
       "      <td>4013.604545</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>Father of the Bride Part II (1995)</td>\n",
       "      <td>['Comedy']</td>\n",
       "      <td>Father of the Bride Part II</td>\n",
       "      <td>Four years after the events of the first film,...</td>\n",
       "      <td>[2176, 2086, 2044, 1996, 2824, 1997, 1996, 203...</td>\n",
       "      <td>[ 2176.  2086.  2044.  1996.  2824.  1997.  19...</td>\n",
       "      <td>4053.462801</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0  Unnamed: 0.1  Unnamed: 0.1.1  movieId  \\\n",
       "0           0             0               0        1   \n",
       "1           1             1               1        2   \n",
       "2           2             2               2        3   \n",
       "3           3             3               3        4   \n",
       "4           4             4               4        5   \n",
       "\n",
       "                                title  \\\n",
       "0                    Toy Story (1995)   \n",
       "1                      Jumanji (1995)   \n",
       "2             Grumpier Old Men (1995)   \n",
       "3            Waiting to Exhale (1995)   \n",
       "4  Father of the Bride Part II (1995)   \n",
       "\n",
       "                                              genres  \\\n",
       "0  ['Adventure', 'Animation', 'Children', 'Comedy...   \n",
       "1               ['Adventure', 'Children', 'Fantasy']   \n",
       "2                              ['Comedy', 'Romance']   \n",
       "3                     ['Comedy', 'Drama', 'Romance']   \n",
       "4                                         ['Comedy']   \n",
       "\n",
       "           wikipedia_page_name  \\\n",
       "0                    Toy Story   \n",
       "1                      Jumanji   \n",
       "2             Grumpier Old Men   \n",
       "3            Waiting to Exhale   \n",
       "4  Father of the Bride Part II   \n",
       "\n",
       "                                          movie_plot  \\\n",
       "0  A group of sentient toys, who pretend to be li...   \n",
       "1  In 1969, Alan Parrish lives in Brantford, New ...   \n",
       "2  The feud between Max and John has cooled and t...   \n",
       "3  Four friends (Savannah, Robin, Bernadine, and ...   \n",
       "4  Four years after the events of the first film,...   \n",
       "\n",
       "                                           token_ids  \\\n",
       "0  [1037, 2177, 1997, 2741, 11638, 10899, 1010, 2...   \n",
       "1  [1999, 3440, 1010, 5070, 11968, 18774, 3268, 1...   \n",
       "2  [1996, 13552, 2090, 4098, 1998, 2198, 2038, 12...   \n",
       "3  [2176, 2814, 1006, 10891, 1010, 5863, 1010, 16...   \n",
       "4  [2176, 2086, 2044, 1996, 2824, 1997, 1996, 203...   \n",
       "\n",
       "                                            plot_ids  mean_tokens  \n",
       "0  [ 1037.  2177.  1997.  2741. 11638. 10899.  10...  4984.423077  \n",
       "1  [ 1999.  3440.  1010.  5070. 11968. 18774.  32...  4644.158750  \n",
       "2  [ 1996. 13552.  2090.  4098.  1998.  2198.  20...  4044.667638  \n",
       "3  [ 2176.  2814.  1006. 10891.  1010.  5863.  10...  4013.604545  \n",
       "4  [ 2176.  2086.  2044.  1996.  2824.  1997.  19...  4053.462801  "
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "movies.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "509d2ca1-232e-4e42-b77e-d004aeb1c017",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>userId</th>\n",
       "      <th>movieId</th>\n",
       "      <th>rating</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>month</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>4.0</td>\n",
       "      <td>964982703</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>4.0</td>\n",
       "      <td>964981247</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>4.0</td>\n",
       "      <td>964982224</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>47</td>\n",
       "      <td>5.0</td>\n",
       "      <td>964983815</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>50</td>\n",
       "      <td>5.0</td>\n",
       "      <td>964982931</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0  userId  movieId  rating  timestamp  month\n",
       "0           0       1        1     4.0  964982703      7\n",
       "1           1       1        3     4.0  964981247      7\n",
       "2           2       1        6     4.0  964982224      7\n",
       "3           3       1       47     5.0  964983815      7\n",
       "4           4       1       50     5.0  964982931      7"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ratings_df.head()"
   ]
  },
  {
   "cell_type": "raw",
   "id": "df7ad1a8-6fe4-4ad7-8094-6704aa38db38",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "307193c3-8066-4ea5-8e0b-e4b71a73240e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>Unnamed: 0.1</th>\n",
       "      <th>Unnamed: 0.1.1</th>\n",
       "      <th>movieId</th>\n",
       "      <th>title_x</th>\n",
       "      <th>genres_x</th>\n",
       "      <th>wikipedia_page_name</th>\n",
       "      <th>movie_plot</th>\n",
       "      <th>token_ids</th>\n",
       "      <th>plot_ids</th>\n",
       "      <th>mean_tokens</th>\n",
       "      <th>title_y</th>\n",
       "      <th>genres_y</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>Toy Story (1995)</td>\n",
       "      <td>['Adventure', 'Animation', 'Children', 'Comedy...</td>\n",
       "      <td>Toy Story</td>\n",
       "      <td>A group of sentient toys, who pretend to be li...</td>\n",
       "      <td>[1037, 2177, 1997, 2741, 11638, 10899, 1010, 2...</td>\n",
       "      <td>[ 1037.  2177.  1997.  2741. 11638. 10899.  10...</td>\n",
       "      <td>4984.423077</td>\n",
       "      <td>Toy Story (1995)</td>\n",
       "      <td>Adventure|Animation|Children|Comedy|Fantasy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>Jumanji (1995)</td>\n",
       "      <td>['Adventure', 'Children', 'Fantasy']</td>\n",
       "      <td>Jumanji</td>\n",
       "      <td>In 1969, Alan Parrish lives in Brantford, New ...</td>\n",
       "      <td>[1999, 3440, 1010, 5070, 11968, 18774, 3268, 1...</td>\n",
       "      <td>[ 1999.  3440.  1010.  5070. 11968. 18774.  32...</td>\n",
       "      <td>4644.158750</td>\n",
       "      <td>Jumanji (1995)</td>\n",
       "      <td>Adventure|Children|Fantasy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>Grumpier Old Men (1995)</td>\n",
       "      <td>['Comedy', 'Romance']</td>\n",
       "      <td>Grumpier Old Men</td>\n",
       "      <td>The feud between Max and John has cooled and t...</td>\n",
       "      <td>[1996, 13552, 2090, 4098, 1998, 2198, 2038, 12...</td>\n",
       "      <td>[ 1996. 13552.  2090.  4098.  1998.  2198.  20...</td>\n",
       "      <td>4044.667638</td>\n",
       "      <td>Grumpier Old Men (1995)</td>\n",
       "      <td>Comedy|Romance</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>Waiting to Exhale (1995)</td>\n",
       "      <td>['Comedy', 'Drama', 'Romance']</td>\n",
       "      <td>Waiting to Exhale</td>\n",
       "      <td>Four friends (Savannah, Robin, Bernadine, and ...</td>\n",
       "      <td>[2176, 2814, 1006, 10891, 1010, 5863, 1010, 16...</td>\n",
       "      <td>[ 2176.  2814.  1006. 10891.  1010.  5863.  10...</td>\n",
       "      <td>4013.604545</td>\n",
       "      <td>Waiting to Exhale (1995)</td>\n",
       "      <td>Comedy|Drama|Romance</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>Father of the Bride Part II (1995)</td>\n",
       "      <td>['Comedy']</td>\n",
       "      <td>Father of the Bride Part II</td>\n",
       "      <td>Four years after the events of the first film,...</td>\n",
       "      <td>[2176, 2086, 2044, 1996, 2824, 1997, 1996, 203...</td>\n",
       "      <td>[ 2176.  2086.  2044.  1996.  2824.  1997.  19...</td>\n",
       "      <td>4053.462801</td>\n",
       "      <td>Father of the Bride Part II (1995)</td>\n",
       "      <td>Comedy</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0  Unnamed: 0.1  Unnamed: 0.1.1  movieId  \\\n",
       "0           0             0               0        1   \n",
       "1           1             1               1        2   \n",
       "2           2             2               2        3   \n",
       "3           3             3               3        4   \n",
       "4           4             4               4        5   \n",
       "\n",
       "                              title_x  \\\n",
       "0                    Toy Story (1995)   \n",
       "1                      Jumanji (1995)   \n",
       "2             Grumpier Old Men (1995)   \n",
       "3            Waiting to Exhale (1995)   \n",
       "4  Father of the Bride Part II (1995)   \n",
       "\n",
       "                                            genres_x  \\\n",
       "0  ['Adventure', 'Animation', 'Children', 'Comedy...   \n",
       "1               ['Adventure', 'Children', 'Fantasy']   \n",
       "2                              ['Comedy', 'Romance']   \n",
       "3                     ['Comedy', 'Drama', 'Romance']   \n",
       "4                                         ['Comedy']   \n",
       "\n",
       "           wikipedia_page_name  \\\n",
       "0                    Toy Story   \n",
       "1                      Jumanji   \n",
       "2             Grumpier Old Men   \n",
       "3            Waiting to Exhale   \n",
       "4  Father of the Bride Part II   \n",
       "\n",
       "                                          movie_plot  \\\n",
       "0  A group of sentient toys, who pretend to be li...   \n",
       "1  In 1969, Alan Parrish lives in Brantford, New ...   \n",
       "2  The feud between Max and John has cooled and t...   \n",
       "3  Four friends (Savannah, Robin, Bernadine, and ...   \n",
       "4  Four years after the events of the first film,...   \n",
       "\n",
       "                                           token_ids  \\\n",
       "0  [1037, 2177, 1997, 2741, 11638, 10899, 1010, 2...   \n",
       "1  [1999, 3440, 1010, 5070, 11968, 18774, 3268, 1...   \n",
       "2  [1996, 13552, 2090, 4098, 1998, 2198, 2038, 12...   \n",
       "3  [2176, 2814, 1006, 10891, 1010, 5863, 1010, 16...   \n",
       "4  [2176, 2086, 2044, 1996, 2824, 1997, 1996, 203...   \n",
       "\n",
       "                                            plot_ids  mean_tokens  \\\n",
       "0  [ 1037.  2177.  1997.  2741. 11638. 10899.  10...  4984.423077   \n",
       "1  [ 1999.  3440.  1010.  5070. 11968. 18774.  32...  4644.158750   \n",
       "2  [ 1996. 13552.  2090.  4098.  1998.  2198.  20...  4044.667638   \n",
       "3  [ 2176.  2814.  1006. 10891.  1010.  5863.  10...  4013.604545   \n",
       "4  [ 2176.  2086.  2044.  1996.  2824.  1997.  19...  4053.462801   \n",
       "\n",
       "                              title_y  \\\n",
       "0                    Toy Story (1995)   \n",
       "1                      Jumanji (1995)   \n",
       "2             Grumpier Old Men (1995)   \n",
       "3            Waiting to Exhale (1995)   \n",
       "4  Father of the Bride Part II (1995)   \n",
       "\n",
       "                                      genres_y  \n",
       "0  Adventure|Animation|Children|Comedy|Fantasy  \n",
       "1                   Adventure|Children|Fantasy  \n",
       "2                               Comedy|Romance  \n",
       "3                         Comedy|Drama|Romance  \n",
       "4                                       Comedy  "
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "movies2 = pd.read_csv('./ml-latest-small/movies.csv')\n",
    "movies_df = pd.merge(movies, movies2, on='movieId', how='left')\n",
    "movies_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "079a25d9-3ee3-492c-9a64-c268f161eba6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0.1.1</th>\n",
       "      <th>movieId</th>\n",
       "      <th>wikipedia_page_name</th>\n",
       "      <th>movie_plot</th>\n",
       "      <th>token_ids</th>\n",
       "      <th>plot_ids</th>\n",
       "      <th>mean_tokens</th>\n",
       "      <th>title</th>\n",
       "      <th>genres</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>Toy Story</td>\n",
       "      <td>A group of sentient toys, who pretend to be li...</td>\n",
       "      <td>[1037, 2177, 1997, 2741, 11638, 10899, 1010, 2...</td>\n",
       "      <td>[ 1037.  2177.  1997.  2741. 11638. 10899.  10...</td>\n",
       "      <td>4984.423077</td>\n",
       "      <td>Toy Story (1995)</td>\n",
       "      <td>Adventure|Animation|Children|Comedy|Fantasy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>Jumanji</td>\n",
       "      <td>In 1969, Alan Parrish lives in Brantford, New ...</td>\n",
       "      <td>[1999, 3440, 1010, 5070, 11968, 18774, 3268, 1...</td>\n",
       "      <td>[ 1999.  3440.  1010.  5070. 11968. 18774.  32...</td>\n",
       "      <td>4644.158750</td>\n",
       "      <td>Jumanji (1995)</td>\n",
       "      <td>Adventure|Children|Fantasy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>Grumpier Old Men</td>\n",
       "      <td>The feud between Max and John has cooled and t...</td>\n",
       "      <td>[1996, 13552, 2090, 4098, 1998, 2198, 2038, 12...</td>\n",
       "      <td>[ 1996. 13552.  2090.  4098.  1998.  2198.  20...</td>\n",
       "      <td>4044.667638</td>\n",
       "      <td>Grumpier Old Men (1995)</td>\n",
       "      <td>Comedy|Romance</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>Waiting to Exhale</td>\n",
       "      <td>Four friends (Savannah, Robin, Bernadine, and ...</td>\n",
       "      <td>[2176, 2814, 1006, 10891, 1010, 5863, 1010, 16...</td>\n",
       "      <td>[ 2176.  2814.  1006. 10891.  1010.  5863.  10...</td>\n",
       "      <td>4013.604545</td>\n",
       "      <td>Waiting to Exhale (1995)</td>\n",
       "      <td>Comedy|Drama|Romance</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>Father of the Bride Part II</td>\n",
       "      <td>Four years after the events of the first film,...</td>\n",
       "      <td>[2176, 2086, 2044, 1996, 2824, 1997, 1996, 203...</td>\n",
       "      <td>[ 2176.  2086.  2044.  1996.  2824.  1997.  19...</td>\n",
       "      <td>4053.462801</td>\n",
       "      <td>Father of the Bride Part II (1995)</td>\n",
       "      <td>Comedy</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0.1.1  movieId          wikipedia_page_name  \\\n",
       "0               0        1                    Toy Story   \n",
       "1               1        2                      Jumanji   \n",
       "2               2        3             Grumpier Old Men   \n",
       "3               3        4            Waiting to Exhale   \n",
       "4               4        5  Father of the Bride Part II   \n",
       "\n",
       "                                          movie_plot  \\\n",
       "0  A group of sentient toys, who pretend to be li...   \n",
       "1  In 1969, Alan Parrish lives in Brantford, New ...   \n",
       "2  The feud between Max and John has cooled and t...   \n",
       "3  Four friends (Savannah, Robin, Bernadine, and ...   \n",
       "4  Four years after the events of the first film,...   \n",
       "\n",
       "                                           token_ids  \\\n",
       "0  [1037, 2177, 1997, 2741, 11638, 10899, 1010, 2...   \n",
       "1  [1999, 3440, 1010, 5070, 11968, 18774, 3268, 1...   \n",
       "2  [1996, 13552, 2090, 4098, 1998, 2198, 2038, 12...   \n",
       "3  [2176, 2814, 1006, 10891, 1010, 5863, 1010, 16...   \n",
       "4  [2176, 2086, 2044, 1996, 2824, 1997, 1996, 203...   \n",
       "\n",
       "                                            plot_ids  mean_tokens  \\\n",
       "0  [ 1037.  2177.  1997.  2741. 11638. 10899.  10...  4984.423077   \n",
       "1  [ 1999.  3440.  1010.  5070. 11968. 18774.  32...  4644.158750   \n",
       "2  [ 1996. 13552.  2090.  4098.  1998.  2198.  20...  4044.667638   \n",
       "3  [ 2176.  2814.  1006. 10891.  1010.  5863.  10...  4013.604545   \n",
       "4  [ 2176.  2086.  2044.  1996.  2824.  1997.  19...  4053.462801   \n",
       "\n",
       "                                title  \\\n",
       "0                    Toy Story (1995)   \n",
       "1                      Jumanji (1995)   \n",
       "2             Grumpier Old Men (1995)   \n",
       "3            Waiting to Exhale (1995)   \n",
       "4  Father of the Bride Part II (1995)   \n",
       "\n",
       "                                        genres  \n",
       "0  Adventure|Animation|Children|Comedy|Fantasy  \n",
       "1                   Adventure|Children|Fantasy  \n",
       "2                               Comedy|Romance  \n",
       "3                         Comedy|Drama|Romance  \n",
       "4                                       Comedy  "
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "movies_df = movies_df.drop(columns=['Unnamed: 0', 'Unnamed: 0.1', 'title_x', 'genres_x'])\n",
    "movies_df.rename(columns={'title_y':'title', 'genres_y':'genres' },inplace=True)\n",
    "movies_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "74874eca-976f-4a6d-8cf9-aaba1762a854",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "There are 0 NaN movie plots\n"
     ]
    }
   ],
   "source": [
    "print(f'There are {movies_df[\"movie_plot\"].isna().sum()} NaN movie plots')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "57cf6e76-9394-487a-a16d-8a56dde61946",
   "metadata": {},
   "outputs": [],
   "source": [
    "movies_plots = movies_df.index[movies_df['movie_plot'] == ''].tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "e73cd185-a42c-4757-ae74-98d31bc16fc1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8168"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(movies_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "32e93fa5-e115-4a77-a2dd-515fdc35bd34",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0.1.1</th>\n",
       "      <th>movieId</th>\n",
       "      <th>wikipedia_page_name</th>\n",
       "      <th>movie_plot</th>\n",
       "      <th>token_ids</th>\n",
       "      <th>plot_ids</th>\n",
       "      <th>mean_tokens</th>\n",
       "      <th>title</th>\n",
       "      <th>genres</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>Toy Story</td>\n",
       "      <td>A group of sentient toys, who pretend to be li...</td>\n",
       "      <td>[1037, 2177, 1997, 2741, 11638, 10899, 1010, 2...</td>\n",
       "      <td>[ 1037.  2177.  1997.  2741. 11638. 10899.  10...</td>\n",
       "      <td>4984.423077</td>\n",
       "      <td>Toy Story (1995)</td>\n",
       "      <td>Adventure|Animation|Children|Comedy|Fantasy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>Jumanji</td>\n",
       "      <td>In 1969, Alan Parrish lives in Brantford, New ...</td>\n",
       "      <td>[1999, 3440, 1010, 5070, 11968, 18774, 3268, 1...</td>\n",
       "      <td>[ 1999.  3440.  1010.  5070. 11968. 18774.  32...</td>\n",
       "      <td>4644.158750</td>\n",
       "      <td>Jumanji (1995)</td>\n",
       "      <td>Adventure|Children|Fantasy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>Grumpier Old Men</td>\n",
       "      <td>The feud between Max and John has cooled and t...</td>\n",
       "      <td>[1996, 13552, 2090, 4098, 1998, 2198, 2038, 12...</td>\n",
       "      <td>[ 1996. 13552.  2090.  4098.  1998.  2198.  20...</td>\n",
       "      <td>4044.667638</td>\n",
       "      <td>Grumpier Old Men (1995)</td>\n",
       "      <td>Comedy|Romance</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>Waiting to Exhale</td>\n",
       "      <td>Four friends (Savannah, Robin, Bernadine, and ...</td>\n",
       "      <td>[2176, 2814, 1006, 10891, 1010, 5863, 1010, 16...</td>\n",
       "      <td>[ 2176.  2814.  1006. 10891.  1010.  5863.  10...</td>\n",
       "      <td>4013.604545</td>\n",
       "      <td>Waiting to Exhale (1995)</td>\n",
       "      <td>Comedy|Drama|Romance</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>Father of the Bride Part II</td>\n",
       "      <td>Four years after the events of the first film,...</td>\n",
       "      <td>[2176, 2086, 2044, 1996, 2824, 1997, 1996, 203...</td>\n",
       "      <td>[ 2176.  2086.  2044.  1996.  2824.  1997.  19...</td>\n",
       "      <td>4053.462801</td>\n",
       "      <td>Father of the Bride Part II (1995)</td>\n",
       "      <td>Comedy</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0.1.1  movieId          wikipedia_page_name  \\\n",
       "0               0        1                    Toy Story   \n",
       "1               1        2                      Jumanji   \n",
       "2               2        3             Grumpier Old Men   \n",
       "3               3        4            Waiting to Exhale   \n",
       "4               4        5  Father of the Bride Part II   \n",
       "\n",
       "                                          movie_plot  \\\n",
       "0  A group of sentient toys, who pretend to be li...   \n",
       "1  In 1969, Alan Parrish lives in Brantford, New ...   \n",
       "2  The feud between Max and John has cooled and t...   \n",
       "3  Four friends (Savannah, Robin, Bernadine, and ...   \n",
       "4  Four years after the events of the first film,...   \n",
       "\n",
       "                                           token_ids  \\\n",
       "0  [1037, 2177, 1997, 2741, 11638, 10899, 1010, 2...   \n",
       "1  [1999, 3440, 1010, 5070, 11968, 18774, 3268, 1...   \n",
       "2  [1996, 13552, 2090, 4098, 1998, 2198, 2038, 12...   \n",
       "3  [2176, 2814, 1006, 10891, 1010, 5863, 1010, 16...   \n",
       "4  [2176, 2086, 2044, 1996, 2824, 1997, 1996, 203...   \n",
       "\n",
       "                                            plot_ids  mean_tokens  \\\n",
       "0  [ 1037.  2177.  1997.  2741. 11638. 10899.  10...  4984.423077   \n",
       "1  [ 1999.  3440.  1010.  5070. 11968. 18774.  32...  4644.158750   \n",
       "2  [ 1996. 13552.  2090.  4098.  1998.  2198.  20...  4044.667638   \n",
       "3  [ 2176.  2814.  1006. 10891.  1010.  5863.  10...  4013.604545   \n",
       "4  [ 2176.  2086.  2044.  1996.  2824.  1997.  19...  4053.462801   \n",
       "\n",
       "                                title  \\\n",
       "0                    Toy Story (1995)   \n",
       "1                      Jumanji (1995)   \n",
       "2             Grumpier Old Men (1995)   \n",
       "3            Waiting to Exhale (1995)   \n",
       "4  Father of the Bride Part II (1995)   \n",
       "\n",
       "                                        genres  \n",
       "0  Adventure|Animation|Children|Comedy|Fantasy  \n",
       "1                   Adventure|Children|Fantasy  \n",
       "2                               Comedy|Romance  \n",
       "3                         Comedy|Drama|Romance  \n",
       "4                                       Comedy  "
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "movies_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7661e3af-bed8-4fba-921c-91d2da0c0c3f",
   "metadata": {},
   "source": [
    "#### ÖNEMLİ::: Wikipedia'da plot'ı olmayan filmleri çıkartmıştık. Bunlar ratings'de duruyor. Ratings'den bunları da temizlememiz gerekiyor\n",
    "\n",
    "Ya da # ratings_df = pd.read_csv('./ml-latest-small/ratings_cleanest.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "631029f8-47f1-4fe6-88ea-ef6c8b64dd63",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>Unnamed: 0.1.1</th>\n",
       "      <th>movieId</th>\n",
       "      <th>wikipedia_page_name</th>\n",
       "      <th>movie_plot</th>\n",
       "      <th>token_ids</th>\n",
       "      <th>plot_ids</th>\n",
       "      <th>mean_tokens</th>\n",
       "      <th>title</th>\n",
       "      <th>genres</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>Toy Story</td>\n",
       "      <td>A group of sentient toys, who pretend to be li...</td>\n",
       "      <td>[1037, 2177, 1997, 2741, 11638, 10899, 1010, 2...</td>\n",
       "      <td>[ 1037.  2177.  1997.  2741. 11638. 10899.  10...</td>\n",
       "      <td>4984.423077</td>\n",
       "      <td>Toy Story (1995)</td>\n",
       "      <td>Adventure|Animation|Children|Comedy|Fantasy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>Jumanji</td>\n",
       "      <td>In 1969, Alan Parrish lives in Brantford, New ...</td>\n",
       "      <td>[1999, 3440, 1010, 5070, 11968, 18774, 3268, 1...</td>\n",
       "      <td>[ 1999.  3440.  1010.  5070. 11968. 18774.  32...</td>\n",
       "      <td>4644.158750</td>\n",
       "      <td>Jumanji (1995)</td>\n",
       "      <td>Adventure|Children|Fantasy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>Grumpier Old Men</td>\n",
       "      <td>The feud between Max and John has cooled and t...</td>\n",
       "      <td>[1996, 13552, 2090, 4098, 1998, 2198, 2038, 12...</td>\n",
       "      <td>[ 1996. 13552.  2090.  4098.  1998.  2198.  20...</td>\n",
       "      <td>4044.667638</td>\n",
       "      <td>Grumpier Old Men (1995)</td>\n",
       "      <td>Comedy|Romance</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>Waiting to Exhale</td>\n",
       "      <td>Four friends (Savannah, Robin, Bernadine, and ...</td>\n",
       "      <td>[2176, 2814, 1006, 10891, 1010, 5863, 1010, 16...</td>\n",
       "      <td>[ 2176.  2814.  1006. 10891.  1010.  5863.  10...</td>\n",
       "      <td>4013.604545</td>\n",
       "      <td>Waiting to Exhale (1995)</td>\n",
       "      <td>Comedy|Drama|Romance</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>Father of the Bride Part II</td>\n",
       "      <td>Four years after the events of the first film,...</td>\n",
       "      <td>[2176, 2086, 2044, 1996, 2824, 1997, 1996, 203...</td>\n",
       "      <td>[ 2176.  2086.  2044.  1996.  2824.  1997.  19...</td>\n",
       "      <td>4053.462801</td>\n",
       "      <td>Father of the Bride Part II (1995)</td>\n",
       "      <td>Comedy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8163</th>\n",
       "      <td>8163</td>\n",
       "      <td>9734</td>\n",
       "      <td>193571</td>\n",
       "      <td>Silver Spoon (manga)</td>\n",
       "      <td>After failing to pass the entrance examination...</td>\n",
       "      <td>[2044, 7989, 2000, 3413, 1996, 4211, 14912, 20...</td>\n",
       "      <td>[ 2044.  7989.  2000.  3413.  1996.  4211. 149...</td>\n",
       "      <td>3927.593220</td>\n",
       "      <td>Silver Spoon (2014)</td>\n",
       "      <td>Comedy|Drama</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8164</th>\n",
       "      <td>8164</td>\n",
       "      <td>9735</td>\n",
       "      <td>193573</td>\n",
       "      <td>Love Live! The School Idol Movie</td>\n",
       "      <td>The movie begins with a scene from the second ...</td>\n",
       "      <td>[1996, 3185, 4269, 2007, 1037, 3496, 2013, 199...</td>\n",
       "      <td>[1996. 3185. 4269. ... 2345. 2836. 1012.]</td>\n",
       "      <td>4155.394251</td>\n",
       "      <td>Love Live! The School Idol Movie (2015)</td>\n",
       "      <td>Animation</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8165</th>\n",
       "      <td>8165</td>\n",
       "      <td>9737</td>\n",
       "      <td>193581</td>\n",
       "      <td>Black Butler: Book of the Atlantic</td>\n",
       "      <td>The story is based on the sinking of the Titan...</td>\n",
       "      <td>[1996, 2466, 2003, 2241, 2006, 1996, 10186, 19...</td>\n",
       "      <td>[ 1996.  2466.  2003.  2241.  2006.  1996. 101...</td>\n",
       "      <td>5494.832941</td>\n",
       "      <td>Black Butler: Book of the Atlantic (2017)</td>\n",
       "      <td>Action|Animation|Comedy|Fantasy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8166</th>\n",
       "      <td>8166</td>\n",
       "      <td>9738</td>\n",
       "      <td>193583</td>\n",
       "      <td>No Game No Life: Zero</td>\n",
       "      <td>In the present, Izuna and Tet play a game of c...</td>\n",
       "      <td>[1999, 1996, 2556, 1010, 1045, 9759, 2532, 199...</td>\n",
       "      <td>[ 1999.  1996.  2556.  1010.  1045.  9759.  25...</td>\n",
       "      <td>4360.566018</td>\n",
       "      <td>No Game No Life: Zero (2017)</td>\n",
       "      <td>Animation|Comedy|Fantasy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8167</th>\n",
       "      <td>8167</td>\n",
       "      <td>9740</td>\n",
       "      <td>193587</td>\n",
       "      <td>Bungo Stray Dogs: Dead Apple</td>\n",
       "      <td>Two years before the Dark Era, the most violen...</td>\n",
       "      <td>[2048, 2086, 2077, 1996, 2601, 3690, 1010, 199...</td>\n",
       "      <td>[ 2048.  2086.  2077.  1996.  2601.  3690.  10...</td>\n",
       "      <td>5319.952886</td>\n",
       "      <td>Bungo Stray Dogs: Dead Apple (2018)</td>\n",
       "      <td>Action|Animation</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8168 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      index  Unnamed: 0.1.1  movieId                 wikipedia_page_name  \\\n",
       "0         0               0        1                           Toy Story   \n",
       "1         1               1        2                             Jumanji   \n",
       "2         2               2        3                    Grumpier Old Men   \n",
       "3         3               3        4                   Waiting to Exhale   \n",
       "4         4               4        5         Father of the Bride Part II   \n",
       "...     ...             ...      ...                                 ...   \n",
       "8163   8163            9734   193571                Silver Spoon (manga)   \n",
       "8164   8164            9735   193573    Love Live! The School Idol Movie   \n",
       "8165   8165            9737   193581  Black Butler: Book of the Atlantic   \n",
       "8166   8166            9738   193583               No Game No Life: Zero   \n",
       "8167   8167            9740   193587        Bungo Stray Dogs: Dead Apple   \n",
       "\n",
       "                                             movie_plot  \\\n",
       "0     A group of sentient toys, who pretend to be li...   \n",
       "1     In 1969, Alan Parrish lives in Brantford, New ...   \n",
       "2     The feud between Max and John has cooled and t...   \n",
       "3     Four friends (Savannah, Robin, Bernadine, and ...   \n",
       "4     Four years after the events of the first film,...   \n",
       "...                                                 ...   \n",
       "8163  After failing to pass the entrance examination...   \n",
       "8164  The movie begins with a scene from the second ...   \n",
       "8165  The story is based on the sinking of the Titan...   \n",
       "8166  In the present, Izuna and Tet play a game of c...   \n",
       "8167  Two years before the Dark Era, the most violen...   \n",
       "\n",
       "                                              token_ids  \\\n",
       "0     [1037, 2177, 1997, 2741, 11638, 10899, 1010, 2...   \n",
       "1     [1999, 3440, 1010, 5070, 11968, 18774, 3268, 1...   \n",
       "2     [1996, 13552, 2090, 4098, 1998, 2198, 2038, 12...   \n",
       "3     [2176, 2814, 1006, 10891, 1010, 5863, 1010, 16...   \n",
       "4     [2176, 2086, 2044, 1996, 2824, 1997, 1996, 203...   \n",
       "...                                                 ...   \n",
       "8163  [2044, 7989, 2000, 3413, 1996, 4211, 14912, 20...   \n",
       "8164  [1996, 3185, 4269, 2007, 1037, 3496, 2013, 199...   \n",
       "8165  [1996, 2466, 2003, 2241, 2006, 1996, 10186, 19...   \n",
       "8166  [1999, 1996, 2556, 1010, 1045, 9759, 2532, 199...   \n",
       "8167  [2048, 2086, 2077, 1996, 2601, 3690, 1010, 199...   \n",
       "\n",
       "                                               plot_ids  mean_tokens  \\\n",
       "0     [ 1037.  2177.  1997.  2741. 11638. 10899.  10...  4984.423077   \n",
       "1     [ 1999.  3440.  1010.  5070. 11968. 18774.  32...  4644.158750   \n",
       "2     [ 1996. 13552.  2090.  4098.  1998.  2198.  20...  4044.667638   \n",
       "3     [ 2176.  2814.  1006. 10891.  1010.  5863.  10...  4013.604545   \n",
       "4     [ 2176.  2086.  2044.  1996.  2824.  1997.  19...  4053.462801   \n",
       "...                                                 ...          ...   \n",
       "8163  [ 2044.  7989.  2000.  3413.  1996.  4211. 149...  3927.593220   \n",
       "8164          [1996. 3185. 4269. ... 2345. 2836. 1012.]  4155.394251   \n",
       "8165  [ 1996.  2466.  2003.  2241.  2006.  1996. 101...  5494.832941   \n",
       "8166  [ 1999.  1996.  2556.  1010.  1045.  9759.  25...  4360.566018   \n",
       "8167  [ 2048.  2086.  2077.  1996.  2601.  3690.  10...  5319.952886   \n",
       "\n",
       "                                          title  \\\n",
       "0                              Toy Story (1995)   \n",
       "1                                Jumanji (1995)   \n",
       "2                       Grumpier Old Men (1995)   \n",
       "3                      Waiting to Exhale (1995)   \n",
       "4            Father of the Bride Part II (1995)   \n",
       "...                                         ...   \n",
       "8163                        Silver Spoon (2014)   \n",
       "8164    Love Live! The School Idol Movie (2015)   \n",
       "8165  Black Butler: Book of the Atlantic (2017)   \n",
       "8166               No Game No Life: Zero (2017)   \n",
       "8167        Bungo Stray Dogs: Dead Apple (2018)   \n",
       "\n",
       "                                           genres  \n",
       "0     Adventure|Animation|Children|Comedy|Fantasy  \n",
       "1                      Adventure|Children|Fantasy  \n",
       "2                                  Comedy|Romance  \n",
       "3                            Comedy|Drama|Romance  \n",
       "4                                          Comedy  \n",
       "...                                           ...  \n",
       "8163                                 Comedy|Drama  \n",
       "8164                                    Animation  \n",
       "8165              Action|Animation|Comedy|Fantasy  \n",
       "8166                     Animation|Comedy|Fantasy  \n",
       "8167                             Action|Animation  \n",
       "\n",
       "[8168 rows x 10 columns]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "movies_df.reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "4d0049ce-713a-4191-85d5-f6c6b65048e8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8152"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "valid_movie_ids = movies_df['movieId'].tolist()\n",
    "valid_ratings_movie_ids = ratings_df['movieId'].tolist()\n",
    "movies_df = movies_df[movies_df['movieId'].isin(valid_ratings_movie_ids)]\n",
    "# ratings_df'deki movieId değerlerini filtreleyelim\n",
    "\n",
    "len(movies_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "cf7d8d55-7871-402f-8e62-809a9e59c863",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(93589, 6)"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ratings_df = ratings_df[ratings_df['movieId'].isin(valid_movie_ids)]\n",
    "ratings_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "52a0cc0b-ac9d-4b2b-8b6f-72cbd67673a7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8152"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(ratings_df['movieId'].unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "4699671b-4fb7-4a9f-b956-2070326b6ddf",
   "metadata": {},
   "outputs": [],
   "source": [
    "def normalize_timestamps(df, movie_col='movieId', timestamp_col='timestamp'):\n",
    "    \"\"\"\n",
    "    Normalize the timestamp values within the dataframe for each unique movieId.\n",
    "    \n",
    "    Parameters:\n",
    "    df (pd.DataFrame): DataFrame containing the data\n",
    "    movie_col (str): Name of the column containing movie IDs\n",
    "    timestamp_col (str): Name of the column containing timestamp values\n",
    "    \n",
    "    Returns:\n",
    "    pd.DataFrame: DataFrame with an additional column 'normalized_timestamp'\n",
    "    \"\"\"\n",
    "    min_timestamp = df[timestamp_col].min()\n",
    "    max_timestamp = df[timestamp_col].max()\n",
    "    \n",
    "    # Compute normalized timestamp for each row\n",
    "    df['normalized_timestamp'] = (df[timestamp_col] - min_timestamp) / (max_timestamp - min_timestamp)\n",
    "    \n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "7a877bb7-9a3f-408a-ab26-5d1e5ee63007",
   "metadata": {},
   "outputs": [],
   "source": [
    "avg_timestamps = normalize_timestamps(ratings_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "28cfa4a4-9100-4459-aa2f-14e47a025d0f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8152"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# movieId'ye göre gruplama ve timestamp'lerin ortalamasını hesaplama\n",
    "grouped = ratings_df.groupby('movieId')['normalized_timestamp'].mean()\n",
    "\n",
    "# SeriesGroupBy nesnesini DataFrame'e dönüştürme\n",
    "normalized_timestamps_df = grouped.reset_index()\n",
    "len(normalized_timestamps_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "7f63614c-1469-4916-893f-dc8ffb5ae883",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>movieId</th>\n",
       "      <th>normalized_timestamp</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>8147</th>\n",
       "      <td>193571</td>\n",
       "      <td>0.999073</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8148</th>\n",
       "      <td>193573</td>\n",
       "      <td>0.999074</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8149</th>\n",
       "      <td>193581</td>\n",
       "      <td>0.999087</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8150</th>\n",
       "      <td>193583</td>\n",
       "      <td>0.999088</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8151</th>\n",
       "      <td>193587</td>\n",
       "      <td>0.999088</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      movieId  normalized_timestamp\n",
       "8147   193571              0.999073\n",
       "8148   193573              0.999074\n",
       "8149   193581              0.999087\n",
       "8150   193583              0.999088\n",
       "8151   193587              0.999088"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "normalized_timestamps_df.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "9a61f2fb-123b-4300-abb9-adafc1cceb37",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>userId</th>\n",
       "      <th>movieId</th>\n",
       "      <th>rating</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>month</th>\n",
       "      <th>normalized_timestamp</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>100831</th>\n",
       "      <td>100831</td>\n",
       "      <td>610</td>\n",
       "      <td>166534</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1493848402</td>\n",
       "      <td>5</td>\n",
       "      <td>0.938125</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100832</th>\n",
       "      <td>100832</td>\n",
       "      <td>610</td>\n",
       "      <td>168248</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1493850091</td>\n",
       "      <td>5</td>\n",
       "      <td>0.938127</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100833</th>\n",
       "      <td>100833</td>\n",
       "      <td>610</td>\n",
       "      <td>168250</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1494273047</td>\n",
       "      <td>5</td>\n",
       "      <td>0.938723</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100834</th>\n",
       "      <td>100834</td>\n",
       "      <td>610</td>\n",
       "      <td>168252</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1493846352</td>\n",
       "      <td>5</td>\n",
       "      <td>0.938122</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100835</th>\n",
       "      <td>100835</td>\n",
       "      <td>610</td>\n",
       "      <td>170875</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1493846415</td>\n",
       "      <td>5</td>\n",
       "      <td>0.938122</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        Unnamed: 0  userId  movieId  rating   timestamp  month  \\\n",
       "100831      100831     610   166534     4.0  1493848402      5   \n",
       "100832      100832     610   168248     5.0  1493850091      5   \n",
       "100833      100833     610   168250     5.0  1494273047      5   \n",
       "100834      100834     610   168252     5.0  1493846352      5   \n",
       "100835      100835     610   170875     3.0  1493846415      5   \n",
       "\n",
       "        normalized_timestamp  \n",
       "100831              0.938125  \n",
       "100832              0.938127  \n",
       "100833              0.938723  \n",
       "100834              0.938122  \n",
       "100835              0.938122  "
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ratings_df.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "3649ce6f-8251-4929-828a-f45da9413e55",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0.1.1</th>\n",
       "      <th>movieId</th>\n",
       "      <th>wikipedia_page_name</th>\n",
       "      <th>movie_plot</th>\n",
       "      <th>token_ids</th>\n",
       "      <th>plot_ids</th>\n",
       "      <th>mean_tokens</th>\n",
       "      <th>title</th>\n",
       "      <th>genres</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>8163</th>\n",
       "      <td>9734</td>\n",
       "      <td>193571</td>\n",
       "      <td>Silver Spoon (manga)</td>\n",
       "      <td>After failing to pass the entrance examination...</td>\n",
       "      <td>[2044, 7989, 2000, 3413, 1996, 4211, 14912, 20...</td>\n",
       "      <td>[ 2044.  7989.  2000.  3413.  1996.  4211. 149...</td>\n",
       "      <td>3927.593220</td>\n",
       "      <td>Silver Spoon (2014)</td>\n",
       "      <td>Comedy|Drama</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8164</th>\n",
       "      <td>9735</td>\n",
       "      <td>193573</td>\n",
       "      <td>Love Live! The School Idol Movie</td>\n",
       "      <td>The movie begins with a scene from the second ...</td>\n",
       "      <td>[1996, 3185, 4269, 2007, 1037, 3496, 2013, 199...</td>\n",
       "      <td>[1996. 3185. 4269. ... 2345. 2836. 1012.]</td>\n",
       "      <td>4155.394251</td>\n",
       "      <td>Love Live! The School Idol Movie (2015)</td>\n",
       "      <td>Animation</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8165</th>\n",
       "      <td>9737</td>\n",
       "      <td>193581</td>\n",
       "      <td>Black Butler: Book of the Atlantic</td>\n",
       "      <td>The story is based on the sinking of the Titan...</td>\n",
       "      <td>[1996, 2466, 2003, 2241, 2006, 1996, 10186, 19...</td>\n",
       "      <td>[ 1996.  2466.  2003.  2241.  2006.  1996. 101...</td>\n",
       "      <td>5494.832941</td>\n",
       "      <td>Black Butler: Book of the Atlantic (2017)</td>\n",
       "      <td>Action|Animation|Comedy|Fantasy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8166</th>\n",
       "      <td>9738</td>\n",
       "      <td>193583</td>\n",
       "      <td>No Game No Life: Zero</td>\n",
       "      <td>In the present, Izuna and Tet play a game of c...</td>\n",
       "      <td>[1999, 1996, 2556, 1010, 1045, 9759, 2532, 199...</td>\n",
       "      <td>[ 1999.  1996.  2556.  1010.  1045.  9759.  25...</td>\n",
       "      <td>4360.566018</td>\n",
       "      <td>No Game No Life: Zero (2017)</td>\n",
       "      <td>Animation|Comedy|Fantasy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8167</th>\n",
       "      <td>9740</td>\n",
       "      <td>193587</td>\n",
       "      <td>Bungo Stray Dogs: Dead Apple</td>\n",
       "      <td>Two years before the Dark Era, the most violen...</td>\n",
       "      <td>[2048, 2086, 2077, 1996, 2601, 3690, 1010, 199...</td>\n",
       "      <td>[ 2048.  2086.  2077.  1996.  2601.  3690.  10...</td>\n",
       "      <td>5319.952886</td>\n",
       "      <td>Bungo Stray Dogs: Dead Apple (2018)</td>\n",
       "      <td>Action|Animation</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      Unnamed: 0.1.1  movieId                 wikipedia_page_name  \\\n",
       "8163            9734   193571                Silver Spoon (manga)   \n",
       "8164            9735   193573    Love Live! The School Idol Movie   \n",
       "8165            9737   193581  Black Butler: Book of the Atlantic   \n",
       "8166            9738   193583               No Game No Life: Zero   \n",
       "8167            9740   193587        Bungo Stray Dogs: Dead Apple   \n",
       "\n",
       "                                             movie_plot  \\\n",
       "8163  After failing to pass the entrance examination...   \n",
       "8164  The movie begins with a scene from the second ...   \n",
       "8165  The story is based on the sinking of the Titan...   \n",
       "8166  In the present, Izuna and Tet play a game of c...   \n",
       "8167  Two years before the Dark Era, the most violen...   \n",
       "\n",
       "                                              token_ids  \\\n",
       "8163  [2044, 7989, 2000, 3413, 1996, 4211, 14912, 20...   \n",
       "8164  [1996, 3185, 4269, 2007, 1037, 3496, 2013, 199...   \n",
       "8165  [1996, 2466, 2003, 2241, 2006, 1996, 10186, 19...   \n",
       "8166  [1999, 1996, 2556, 1010, 1045, 9759, 2532, 199...   \n",
       "8167  [2048, 2086, 2077, 1996, 2601, 3690, 1010, 199...   \n",
       "\n",
       "                                               plot_ids  mean_tokens  \\\n",
       "8163  [ 2044.  7989.  2000.  3413.  1996.  4211. 149...  3927.593220   \n",
       "8164          [1996. 3185. 4269. ... 2345. 2836. 1012.]  4155.394251   \n",
       "8165  [ 1996.  2466.  2003.  2241.  2006.  1996. 101...  5494.832941   \n",
       "8166  [ 1999.  1996.  2556.  1010.  1045.  9759.  25...  4360.566018   \n",
       "8167  [ 2048.  2086.  2077.  1996.  2601.  3690.  10...  5319.952886   \n",
       "\n",
       "                                          title  \\\n",
       "8163                        Silver Spoon (2014)   \n",
       "8164    Love Live! The School Idol Movie (2015)   \n",
       "8165  Black Butler: Book of the Atlantic (2017)   \n",
       "8166               No Game No Life: Zero (2017)   \n",
       "8167        Bungo Stray Dogs: Dead Apple (2018)   \n",
       "\n",
       "                               genres  \n",
       "8163                     Comedy|Drama  \n",
       "8164                        Animation  \n",
       "8165  Action|Animation|Comedy|Fantasy  \n",
       "8166         Animation|Comedy|Fantasy  \n",
       "8167                 Action|Animation  "
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "movies_df.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "02d36fb3-0ee4-4a3e-b300-7c28cd252783",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0.1.1</th>\n",
       "      <th>wikipedia_page_name</th>\n",
       "      <th>movie_plot</th>\n",
       "      <th>token_ids</th>\n",
       "      <th>plot_ids</th>\n",
       "      <th>mean_tokens</th>\n",
       "      <th>title</th>\n",
       "      <th>genres</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>movieId</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>Toy Story</td>\n",
       "      <td>A group of sentient toys, who pretend to be li...</td>\n",
       "      <td>[1037, 2177, 1997, 2741, 11638, 10899, 1010, 2...</td>\n",
       "      <td>[ 1037.  2177.  1997.  2741. 11638. 10899.  10...</td>\n",
       "      <td>4984.423077</td>\n",
       "      <td>Toy Story (1995)</td>\n",
       "      <td>Adventure|Animation|Children|Comedy|Fantasy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>Jumanji</td>\n",
       "      <td>In 1969, Alan Parrish lives in Brantford, New ...</td>\n",
       "      <td>[1999, 3440, 1010, 5070, 11968, 18774, 3268, 1...</td>\n",
       "      <td>[ 1999.  3440.  1010.  5070. 11968. 18774.  32...</td>\n",
       "      <td>4644.158750</td>\n",
       "      <td>Jumanji (1995)</td>\n",
       "      <td>Adventure|Children|Fantasy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2</td>\n",
       "      <td>Grumpier Old Men</td>\n",
       "      <td>The feud between Max and John has cooled and t...</td>\n",
       "      <td>[1996, 13552, 2090, 4098, 1998, 2198, 2038, 12...</td>\n",
       "      <td>[ 1996. 13552.  2090.  4098.  1998.  2198.  20...</td>\n",
       "      <td>4044.667638</td>\n",
       "      <td>Grumpier Old Men (1995)</td>\n",
       "      <td>Comedy|Romance</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3</td>\n",
       "      <td>Waiting to Exhale</td>\n",
       "      <td>Four friends (Savannah, Robin, Bernadine, and ...</td>\n",
       "      <td>[2176, 2814, 1006, 10891, 1010, 5863, 1010, 16...</td>\n",
       "      <td>[ 2176.  2814.  1006. 10891.  1010.  5863.  10...</td>\n",
       "      <td>4013.604545</td>\n",
       "      <td>Waiting to Exhale (1995)</td>\n",
       "      <td>Comedy|Drama|Romance</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>4</td>\n",
       "      <td>Father of the Bride Part II</td>\n",
       "      <td>Four years after the events of the first film,...</td>\n",
       "      <td>[2176, 2086, 2044, 1996, 2824, 1997, 1996, 203...</td>\n",
       "      <td>[ 2176.  2086.  2044.  1996.  2824.  1997.  19...</td>\n",
       "      <td>4053.462801</td>\n",
       "      <td>Father of the Bride Part II (1995)</td>\n",
       "      <td>Comedy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>193571</th>\n",
       "      <td>9734</td>\n",
       "      <td>Silver Spoon (manga)</td>\n",
       "      <td>After failing to pass the entrance examination...</td>\n",
       "      <td>[2044, 7989, 2000, 3413, 1996, 4211, 14912, 20...</td>\n",
       "      <td>[ 2044.  7989.  2000.  3413.  1996.  4211. 149...</td>\n",
       "      <td>3927.593220</td>\n",
       "      <td>Silver Spoon (2014)</td>\n",
       "      <td>Comedy|Drama</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>193573</th>\n",
       "      <td>9735</td>\n",
       "      <td>Love Live! The School Idol Movie</td>\n",
       "      <td>The movie begins with a scene from the second ...</td>\n",
       "      <td>[1996, 3185, 4269, 2007, 1037, 3496, 2013, 199...</td>\n",
       "      <td>[1996. 3185. 4269. ... 2345. 2836. 1012.]</td>\n",
       "      <td>4155.394251</td>\n",
       "      <td>Love Live! The School Idol Movie (2015)</td>\n",
       "      <td>Animation</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>193581</th>\n",
       "      <td>9737</td>\n",
       "      <td>Black Butler: Book of the Atlantic</td>\n",
       "      <td>The story is based on the sinking of the Titan...</td>\n",
       "      <td>[1996, 2466, 2003, 2241, 2006, 1996, 10186, 19...</td>\n",
       "      <td>[ 1996.  2466.  2003.  2241.  2006.  1996. 101...</td>\n",
       "      <td>5494.832941</td>\n",
       "      <td>Black Butler: Book of the Atlantic (2017)</td>\n",
       "      <td>Action|Animation|Comedy|Fantasy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>193583</th>\n",
       "      <td>9738</td>\n",
       "      <td>No Game No Life: Zero</td>\n",
       "      <td>In the present, Izuna and Tet play a game of c...</td>\n",
       "      <td>[1999, 1996, 2556, 1010, 1045, 9759, 2532, 199...</td>\n",
       "      <td>[ 1999.  1996.  2556.  1010.  1045.  9759.  25...</td>\n",
       "      <td>4360.566018</td>\n",
       "      <td>No Game No Life: Zero (2017)</td>\n",
       "      <td>Animation|Comedy|Fantasy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>193587</th>\n",
       "      <td>9740</td>\n",
       "      <td>Bungo Stray Dogs: Dead Apple</td>\n",
       "      <td>Two years before the Dark Era, the most violen...</td>\n",
       "      <td>[2048, 2086, 2077, 1996, 2601, 3690, 1010, 199...</td>\n",
       "      <td>[ 2048.  2086.  2077.  1996.  2601.  3690.  10...</td>\n",
       "      <td>5319.952886</td>\n",
       "      <td>Bungo Stray Dogs: Dead Apple (2018)</td>\n",
       "      <td>Action|Animation</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8152 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         Unnamed: 0.1.1                 wikipedia_page_name  \\\n",
       "movieId                                                       \n",
       "1                     0                           Toy Story   \n",
       "2                     1                             Jumanji   \n",
       "3                     2                    Grumpier Old Men   \n",
       "4                     3                   Waiting to Exhale   \n",
       "5                     4         Father of the Bride Part II   \n",
       "...                 ...                                 ...   \n",
       "193571             9734                Silver Spoon (manga)   \n",
       "193573             9735    Love Live! The School Idol Movie   \n",
       "193581             9737  Black Butler: Book of the Atlantic   \n",
       "193583             9738               No Game No Life: Zero   \n",
       "193587             9740        Bungo Stray Dogs: Dead Apple   \n",
       "\n",
       "                                                movie_plot  \\\n",
       "movieId                                                      \n",
       "1        A group of sentient toys, who pretend to be li...   \n",
       "2        In 1969, Alan Parrish lives in Brantford, New ...   \n",
       "3        The feud between Max and John has cooled and t...   \n",
       "4        Four friends (Savannah, Robin, Bernadine, and ...   \n",
       "5        Four years after the events of the first film,...   \n",
       "...                                                    ...   \n",
       "193571   After failing to pass the entrance examination...   \n",
       "193573   The movie begins with a scene from the second ...   \n",
       "193581   The story is based on the sinking of the Titan...   \n",
       "193583   In the present, Izuna and Tet play a game of c...   \n",
       "193587   Two years before the Dark Era, the most violen...   \n",
       "\n",
       "                                                 token_ids  \\\n",
       "movieId                                                      \n",
       "1        [1037, 2177, 1997, 2741, 11638, 10899, 1010, 2...   \n",
       "2        [1999, 3440, 1010, 5070, 11968, 18774, 3268, 1...   \n",
       "3        [1996, 13552, 2090, 4098, 1998, 2198, 2038, 12...   \n",
       "4        [2176, 2814, 1006, 10891, 1010, 5863, 1010, 16...   \n",
       "5        [2176, 2086, 2044, 1996, 2824, 1997, 1996, 203...   \n",
       "...                                                    ...   \n",
       "193571   [2044, 7989, 2000, 3413, 1996, 4211, 14912, 20...   \n",
       "193573   [1996, 3185, 4269, 2007, 1037, 3496, 2013, 199...   \n",
       "193581   [1996, 2466, 2003, 2241, 2006, 1996, 10186, 19...   \n",
       "193583   [1999, 1996, 2556, 1010, 1045, 9759, 2532, 199...   \n",
       "193587   [2048, 2086, 2077, 1996, 2601, 3690, 1010, 199...   \n",
       "\n",
       "                                                  plot_ids  mean_tokens  \\\n",
       "movieId                                                                   \n",
       "1        [ 1037.  2177.  1997.  2741. 11638. 10899.  10...  4984.423077   \n",
       "2        [ 1999.  3440.  1010.  5070. 11968. 18774.  32...  4644.158750   \n",
       "3        [ 1996. 13552.  2090.  4098.  1998.  2198.  20...  4044.667638   \n",
       "4        [ 2176.  2814.  1006. 10891.  1010.  5863.  10...  4013.604545   \n",
       "5        [ 2176.  2086.  2044.  1996.  2824.  1997.  19...  4053.462801   \n",
       "...                                                    ...          ...   \n",
       "193571   [ 2044.  7989.  2000.  3413.  1996.  4211. 149...  3927.593220   \n",
       "193573           [1996. 3185. 4269. ... 2345. 2836. 1012.]  4155.394251   \n",
       "193581   [ 1996.  2466.  2003.  2241.  2006.  1996. 101...  5494.832941   \n",
       "193583   [ 1999.  1996.  2556.  1010.  1045.  9759.  25...  4360.566018   \n",
       "193587   [ 2048.  2086.  2077.  1996.  2601.  3690.  10...  5319.952886   \n",
       "\n",
       "                                             title  \\\n",
       "movieId                                              \n",
       "1                                 Toy Story (1995)   \n",
       "2                                   Jumanji (1995)   \n",
       "3                          Grumpier Old Men (1995)   \n",
       "4                         Waiting to Exhale (1995)   \n",
       "5               Father of the Bride Part II (1995)   \n",
       "...                                            ...   \n",
       "193571                         Silver Spoon (2014)   \n",
       "193573     Love Live! The School Idol Movie (2015)   \n",
       "193581   Black Butler: Book of the Atlantic (2017)   \n",
       "193583                No Game No Life: Zero (2017)   \n",
       "193587         Bungo Stray Dogs: Dead Apple (2018)   \n",
       "\n",
       "                                              genres  \n",
       "movieId                                               \n",
       "1        Adventure|Animation|Children|Comedy|Fantasy  \n",
       "2                         Adventure|Children|Fantasy  \n",
       "3                                     Comedy|Romance  \n",
       "4                               Comedy|Drama|Romance  \n",
       "5                                             Comedy  \n",
       "...                                              ...  \n",
       "193571                                  Comedy|Drama  \n",
       "193573                                     Animation  \n",
       "193581               Action|Animation|Comedy|Fantasy  \n",
       "193583                      Animation|Comedy|Fantasy  \n",
       "193587                              Action|Animation  \n",
       "\n",
       "[8152 rows x 8 columns]"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# movies = new_movies\n",
    "# # movies = movies.drop(columns=[\"Unnamed: 0\"])\n",
    "movies_df.set_index(\"movieId\")\n",
    "\n",
    "# movies[\"movies_id\"] = movies[\"movie_id\"] +1\n",
    "# movies.to_csv(movies_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "d7e6d423-95e5-43c5-a7ab-a6310b496895",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "movies.csv:\n",
      "===========\n",
      "   movieId                                       genres\n",
      "0        1  Adventure|Animation|Children|Comedy|Fantasy\n",
      "1        2                   Adventure|Children|Fantasy\n",
      "2        3                               Comedy|Romance\n",
      "3        4                         Comedy|Drama|Romance\n",
      "4        5                                       Comedy\n",
      "\n",
      "ratings.csv:\n",
      "============\n",
      "   userId  movieId\n",
      "0       1        1\n",
      "1       1        3\n",
      "2       1        6\n",
      "3       1       47\n",
      "4       1       50\n"
     ]
    }
   ],
   "source": [
    "print('movies.csv:')\n",
    "print('===========')\n",
    "\n",
    "print(movies_df[[\"movieId\", \"genres\"]].head())\n",
    "print()\n",
    "print('ratings.csv:')\n",
    "print('============')\n",
    "\n",
    "print(ratings_df[[\"userId\", \"movieId\"]].head())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "a03d42f2-6c0b-409c-b9b7-371602d9deda",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   Action  Adventure  Drama  Horror\n",
      "0       0          1      0       0\n",
      "1       0          1      0       0\n",
      "2       0          0      0       0\n",
      "3       0          0      1       0\n",
      "4       0          0      0       0\n"
     ]
    }
   ],
   "source": [
    "# Split genres and convert into indicator variables:\n",
    "genres = movies_df['genres'].str.get_dummies('|')\n",
    "print(genres[[\"Action\", \"Adventure\", \"Drama\", \"Horror\"]].head())\n",
    "\n",
    "# Use genres as movie input features:\n",
    "movie_feat_1 = torch.from_numpy(genres.values).to(torch.float)\n",
    "# assert movie_feat_1.size() == (9742, 20)  # 20 genres in total."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "48a764e6-acf1-4d18-9fa3-eab4de3d11c0",
   "metadata": {},
   "outputs": [],
   "source": [
    "# def convert_strings_to_floats(input_array):\n",
    "#     output_array = []\n",
    "#     for element in input_array:\n",
    "#         converted_float = float(element)\n",
    "#         output_array.append(converted_float)\n",
    "#     return output_array\n",
    "# movies_df['token_ids'] = output_array"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "1ab2c6ae-1a12-4e7d-86d6-044eb965ba7b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# token_ids = movies_df['token_ids'].str.strip(\"[]\").str.split(\", \").map(np.array)\n",
    "\n",
    "movie_feat_2 = torch.from_numpy(movies_df['mean_tokens'].values).to(torch.float)\n",
    "movie_feat_2 = normalize(movie_feat_2,p=1.0, dim = 0)\n",
    "movie_feat_3 = torch.from_numpy(normalized_timestamps_df['normalized_timestamp'].values).to(torch.float)\n",
    "# movie_feat_3 = normalize(movie_feat_2,p=1.0, dim = 0)\n",
    "# movie_feat_3 = torch.from_numpy(movies_df['month'].values).to(torch.float)\n",
    "# movie_feat_3 = normalize(movie_feat_3,p=1.0, dim = 0)\n",
    "# movie_feat = torch.column_stack([movie_feat_1, movie_feat_2, movie_feat_3])\n",
    "movie_feat = torch.column_stack([movie_feat_1, movie_feat_2, movie_feat_3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "483d25d3-8ce3-47d6-885d-d227f9493937",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([8152, 22])"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "movie_feat.shape\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "b250487a-5800-4078-966f-41e6cf52fc17",
   "metadata": {},
   "outputs": [],
   "source": [
    "# movie_feat = torch.column_stack([movie_feat_1, movie_feat_2, movie_feat_3])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "59684dd4-571a-4981-b3c3-b190a491c298",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(False)"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "movie_feat = torch.nan_to_num(movie_feat, nan=2.0, posinf=1.0)\n",
    "movie_feat.isnan().any()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "81ad96c8-3dcd-4215-a4de-8c3895c17520",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8152"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "unique_movie_id = ratings_df['movieId'].unique()\n",
    "len(unique_movie_id)\n",
    "len(movies_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "7db03757-4788-4c8d-8ce0-868c549b77bd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>movieId</th>\n",
       "      <th>wikipedia_page_name</th>\n",
       "      <th>movie_plot</th>\n",
       "      <th>token_ids</th>\n",
       "      <th>plot_ids</th>\n",
       "      <th>mean_tokens</th>\n",
       "      <th>title</th>\n",
       "      <th>genres</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>Toy Story</td>\n",
       "      <td>A group of sentient toys, who pretend to be li...</td>\n",
       "      <td>[1037, 2177, 1997, 2741, 11638, 10899, 1010, 2...</td>\n",
       "      <td>[ 1037.  2177.  1997.  2741. 11638. 10899.  10...</td>\n",
       "      <td>4984.423077</td>\n",
       "      <td>Toy Story (1995)</td>\n",
       "      <td>Adventure|Animation|Children|Comedy|Fantasy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>Jumanji</td>\n",
       "      <td>In 1969, Alan Parrish lives in Brantford, New ...</td>\n",
       "      <td>[1999, 3440, 1010, 5070, 11968, 18774, 3268, 1...</td>\n",
       "      <td>[ 1999.  3440.  1010.  5070. 11968. 18774.  32...</td>\n",
       "      <td>4644.158750</td>\n",
       "      <td>Jumanji (1995)</td>\n",
       "      <td>Adventure|Children|Fantasy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>Grumpier Old Men</td>\n",
       "      <td>The feud between Max and John has cooled and t...</td>\n",
       "      <td>[1996, 13552, 2090, 4098, 1998, 2198, 2038, 12...</td>\n",
       "      <td>[ 1996. 13552.  2090.  4098.  1998.  2198.  20...</td>\n",
       "      <td>4044.667638</td>\n",
       "      <td>Grumpier Old Men (1995)</td>\n",
       "      <td>Comedy|Romance</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>Waiting to Exhale</td>\n",
       "      <td>Four friends (Savannah, Robin, Bernadine, and ...</td>\n",
       "      <td>[2176, 2814, 1006, 10891, 1010, 5863, 1010, 16...</td>\n",
       "      <td>[ 2176.  2814.  1006. 10891.  1010.  5863.  10...</td>\n",
       "      <td>4013.604545</td>\n",
       "      <td>Waiting to Exhale (1995)</td>\n",
       "      <td>Comedy|Drama|Romance</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>Father of the Bride Part II</td>\n",
       "      <td>Four years after the events of the first film,...</td>\n",
       "      <td>[2176, 2086, 2044, 1996, 2824, 1997, 1996, 203...</td>\n",
       "      <td>[ 2176.  2086.  2044.  1996.  2824.  1997.  19...</td>\n",
       "      <td>4053.462801</td>\n",
       "      <td>Father of the Bride Part II (1995)</td>\n",
       "      <td>Comedy</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   movieId          wikipedia_page_name  \\\n",
       "0        1                    Toy Story   \n",
       "1        2                      Jumanji   \n",
       "2        3             Grumpier Old Men   \n",
       "3        4            Waiting to Exhale   \n",
       "4        5  Father of the Bride Part II   \n",
       "\n",
       "                                          movie_plot  \\\n",
       "0  A group of sentient toys, who pretend to be li...   \n",
       "1  In 1969, Alan Parrish lives in Brantford, New ...   \n",
       "2  The feud between Max and John has cooled and t...   \n",
       "3  Four friends (Savannah, Robin, Bernadine, and ...   \n",
       "4  Four years after the events of the first film,...   \n",
       "\n",
       "                                           token_ids  \\\n",
       "0  [1037, 2177, 1997, 2741, 11638, 10899, 1010, 2...   \n",
       "1  [1999, 3440, 1010, 5070, 11968, 18774, 3268, 1...   \n",
       "2  [1996, 13552, 2090, 4098, 1998, 2198, 2038, 12...   \n",
       "3  [2176, 2814, 1006, 10891, 1010, 5863, 1010, 16...   \n",
       "4  [2176, 2086, 2044, 1996, 2824, 1997, 1996, 203...   \n",
       "\n",
       "                                            plot_ids  mean_tokens  \\\n",
       "0  [ 1037.  2177.  1997.  2741. 11638. 10899.  10...  4984.423077   \n",
       "1  [ 1999.  3440.  1010.  5070. 11968. 18774.  32...  4644.158750   \n",
       "2  [ 1996. 13552.  2090.  4098.  1998.  2198.  20...  4044.667638   \n",
       "3  [ 2176.  2814.  1006. 10891.  1010.  5863.  10...  4013.604545   \n",
       "4  [ 2176.  2086.  2044.  1996.  2824.  1997.  19...  4053.462801   \n",
       "\n",
       "                                title  \\\n",
       "0                    Toy Story (1995)   \n",
       "1                      Jumanji (1995)   \n",
       "2             Grumpier Old Men (1995)   \n",
       "3            Waiting to Exhale (1995)   \n",
       "4  Father of the Bride Part II (1995)   \n",
       "\n",
       "                                        genres  \n",
       "0  Adventure|Animation|Children|Comedy|Fantasy  \n",
       "1                   Adventure|Children|Fantasy  \n",
       "2                               Comedy|Romance  \n",
       "3                         Comedy|Drama|Romance  \n",
       "4                                       Comedy  "
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "movies_df = movies_df.drop(columns=['Unnamed: 0.1.1'])\n",
    "movies_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "3a4b9478-0178-4293-920e-ec2eff03087d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>userId</th>\n",
       "      <th>movieId</th>\n",
       "      <th>rating</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>month</th>\n",
       "      <th>normalized_timestamp</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>4.0</td>\n",
       "      <td>964982703</td>\n",
       "      <td>7</td>\n",
       "      <td>0.192858</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>4.0</td>\n",
       "      <td>964981247</td>\n",
       "      <td>7</td>\n",
       "      <td>0.192856</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>4.0</td>\n",
       "      <td>964982224</td>\n",
       "      <td>7</td>\n",
       "      <td>0.192857</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>47</td>\n",
       "      <td>5.0</td>\n",
       "      <td>964983815</td>\n",
       "      <td>7</td>\n",
       "      <td>0.192859</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>50</td>\n",
       "      <td>5.0</td>\n",
       "      <td>964982931</td>\n",
       "      <td>7</td>\n",
       "      <td>0.192858</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   userId  movieId  rating  timestamp  month  normalized_timestamp\n",
       "0       1        1     4.0  964982703      7              0.192858\n",
       "1       1        3     4.0  964981247      7              0.192856\n",
       "2       1        6     4.0  964982224      7              0.192857\n",
       "3       1       47     5.0  964983815      7              0.192859\n",
       "4       1       50     5.0  964982931      7              0.192858"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ratings_df = ratings_df.drop(columns=['Unnamed: 0'])\n",
    "ratings_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "97b5f86c-d281-471d-9711-ea5f6bf51dd3",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "def get_positional_encoding(d, position):\n",
    "    \"\"\"\n",
    "    Generate positional encoding for a given position.\n",
    "    \"\"\"\n",
    "    pe = np.zeros(d)\n",
    "    for i in range(d):\n",
    "        if i % 2 == 0:\n",
    "            pe[i] = np.sin(position / (10000 ** (i / d)))\n",
    "        else:\n",
    "            pe[i] = np.cos(position / (10000 ** (i / d)))\n",
    "    return pe\n",
    "\n",
    "# Let's assume 16-dimensional positional encoding\n",
    "d = 16\n",
    "\n",
    "# Apply positional encoding to movie and user ids\n",
    "movies_df['positional_encoding'] = movies_df['movieId'].apply(lambda x: get_positional_encoding(d, x))\n",
    "ratings_df['positional_encoding'] = ratings_df['userId'].apply(lambda x: get_positional_encoding(d, x))\n",
    "\n",
    "# Convert these into tensor form\n",
    "movie_pe = torch.tensor(np.vstack(movies_df['positional_encoding'].values), dtype=torch.float)\n",
    "user_pe = torch.tensor(np.vstack(ratings_df['positional_encoding'].values), dtype=torch.float)\n",
    "\n",
    "# Now we can include these in the final feature vector\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "eff9619f-2d2c-4663-84bf-17df31c20798",
   "metadata": {},
   "outputs": [],
   "source": [
    "import networkx as nx\n",
    "\n",
    "# Öncelikle grafı oluşturalım\n",
    "G = nx.Graph()\n",
    "\n",
    "# Kenarları ekleyelim\n",
    "edges = list(zip(ratings_df['userId'], ratings_df['movieId']))\n",
    "G.add_edges_from(edges)\n",
    "\n",
    "# Topolojik özellikleri hesaplayalım\n",
    "degree_centrality = nx.degree_centrality(G)\n",
    "closeness_centrality = nx.closeness_centrality(G)\n",
    "betweenness_centrality = nx.betweenness_centrality(G)\n",
    "\n",
    "# Bunları DataFrame'e ekleyelim\n",
    "movies_df['degree_centrality'] = movies_df['movieId'].map(degree_centrality)\n",
    "movies_df['closeness_centrality'] = movies_df['movieId'].map(closeness_centrality)\n",
    "movies_df['betweenness_centrality'] = movies_df['movieId'].map(betweenness_centrality)\n",
    "\n",
    "# Eksik değerleri sıfırla doldur\n",
    "movies_df.fillna(0, inplace=True)\n",
    "\n",
    "# Bu özellikleri tensor olarak ekleyelim\n",
    "topological_features = torch.tensor(movies_df[['degree_centrality', 'closeness_centrality', 'betweenness_centrality']].values, dtype=torch.float)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "105253d0-8920-4cab-b744-a195e1740d0d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# # Rating değerlerini ikili hale getirme\n",
    "mean_rating = ratings_df['rating'].mean()\n",
    "ratings_df['binary_rating'] = (ratings_df['rating'] >= mean_rating).astype(int)\n",
    "\n",
    "## Burası yeni _____ exponential weight decay ile yakın zamanlı değerlendirmelerin ağırlıkları fazla olacak\n",
    "ratings_df['timestamp'] = (ratings_df['timestamp'] - ratings_df['timestamp'].min()) / \\\n",
    "                        (ratings_df['timestamp'].max() - ratings_df['timestamp'].min())\n",
    "\n",
    "# Zaman faktörünü exponential decay ile ağırlık olarak kullanacağız\n",
    "ratings_df['weight'] = np.exp(-ratings_df['timestamp'])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "8de4715d-2896-4727-ac0e-77c66ed2d9a6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.00012038040207054291 0.29782111472252315\n",
      "0.2480812304016724 0.5383668178872326\n",
      "0.0 0.13506609853040205\n"
     ]
    }
   ],
   "source": [
    "# normalizasyon gerekli mi kontrol ediyoruz, sonuç gerekli değil\n",
    "print(movies_df['degree_centrality'].min(), movies_df['degree_centrality'].max())\n",
    "print(movies_df['closeness_centrality'].min(), movies_df['closeness_centrality'].max())\n",
    "print(movies_df['betweenness_centrality'].min(), movies_df['betweenness_centrality'].max())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "74118afc-9cb9-4567-ac1a-c066d083f8e0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([8152, 41])\n"
     ]
    }
   ],
   "source": [
    "# Nihai film özelliklerine ekleyelim\n",
    "movie_features = torch.column_stack([movie_feat, movie_pe, topological_features])\n",
    "\n",
    "print(movie_features.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c817c1ec-2aaf-4157-bd40-844fdee5200d",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fb0793ce-189d-46f7-bf39-116970fa6573",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "00e5ee66-54a1-4097-aa16-f3683e6cd152",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e6120343-4c4a-46a7-b38f-b3d08908a2c6",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "640c3638-09f1-4a64-82c6-8937993a504f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mapping of user IDs to consecutive values:\n",
      "==========================================\n",
      "   userId  mappedID\n",
      "0       1         0\n",
      "1       2         1\n",
      "2       3         2\n",
      "3       4         3\n",
      "4       5         4\n",
      "\n",
      "Mapping of movie IDs to consecutive values:\n",
      "===========================================\n",
      "   movieId  mappedID\n",
      "0        1         0\n",
      "1        3         1\n",
      "2        6         2\n",
      "3       47         3\n",
      "4       50         4\n",
      "\n",
      "Final edge indices pointing from users to movies:\n",
      "=================================================\n",
      "tensor([[   0,    0,    0,  ...,  609,  609,  609],\n",
      "        [   0,    1,    2,  ..., 2827, 1290, 2605]])\n"
     ]
    }
   ],
   "source": [
    "# Load the entire ratings data frame into memory:\n",
    "# ratings_df = pd.read_csv(ratings_path)\n",
    "\n",
    "# Create a mapping from unique user indices to range [0, num_user_nodes):\n",
    "# Create a mapping from unique user indices to range [0, num_user_nodes):\n",
    "unique_user_id = ratings_df['userId'].unique()\n",
    "unique_user_id = pd.DataFrame(data={\n",
    "    'userId': unique_user_id,\n",
    "    'mappedID': pd.RangeIndex(len(unique_user_id)),\n",
    "})\n",
    "print(\"Mapping of user IDs to consecutive values:\")\n",
    "print(\"==========================================\")\n",
    "print(unique_user_id.head())\n",
    "print()\n",
    "# Create a mapping from unique movie indices to range [0, num_movie_nodes):\n",
    "unique_movie_id = ratings_df['movieId'].unique()\n",
    "unique_movie_id = pd.DataFrame(data={\n",
    "    'movieId': unique_movie_id,\n",
    "    'mappedID': pd.RangeIndex(len(movies_df)),\n",
    "})\n",
    "print(\"Mapping of movie IDs to consecutive values:\")\n",
    "print(\"===========================================\")\n",
    "print(unique_movie_id.head())\n",
    "\n",
    "# Perform merge to obtain the edges from users and movies:\n",
    "ratings_user_id = pd.merge(ratings_df['userId'], unique_user_id,\n",
    "                            left_on='userId', right_on='userId', how='left')\n",
    "ratings_user_id = torch.from_numpy(ratings_user_id['mappedID'].values)\n",
    "ratings_movie_id = pd.merge(ratings_df['movieId'], unique_movie_id,\n",
    "                            left_on='movieId', right_on='movieId', how='left')\n",
    "ratings_movie_id = torch.from_numpy(ratings_movie_id['mappedID'].values)\n",
    "\n",
    "# With this, we are ready to construct our `edge_index` in COO format\n",
    "# following PyG semantics:\n",
    "edge_index_user_to_movie = torch.stack([ratings_user_id, ratings_movie_id], dim=0)\n",
    "# assert edge_index_user_to_movie.size() == (2, 100836)\n",
    "\n",
    "print()\n",
    "print(\"Final edge indices pointing from users to movies:\")\n",
    "print(\"=================================================\")\n",
    "print(edge_index_user_to_movie)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "db0ab34d-ffd1-4270-bd8e-8a8efac7172c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(False)"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.isnan(edge_index_user_to_movie).any()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "69906a18-8201-4858-ac4d-f6000e63911c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8152"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(unique_movie_id) \n",
    "# len(movies_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "939763c7-70f4-49e9-9b95-5137e34bb3fd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "HeteroData(\n",
      "  user={ node_id=[610] },\n",
      "  movie={\n",
      "    node_id=[8152],\n",
      "    x=[8152, 41],\n",
      "  },\n",
      "  (user, rates, movie)={\n",
      "    edge_index=[2, 93589],\n",
      "    edge_attr=[93589],\n",
      "  },\n",
      "  (movie, rev_rates, user)={\n",
      "    edge_index=[2, 93589],\n",
      "    edge_attr=[93589],\n",
      "  }\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "from torch_geometric.data import HeteroData\n",
    "import torch_geometric.transforms as T\n",
    "\n",
    "data = HeteroData()\n",
    "\n",
    "# Save node indices:\n",
    "data[\"user\"].node_id = torch.arange(len(unique_user_id))\n",
    "data[\"movie\"].node_id = torch.arange(len(unique_movie_id))\n",
    "weight = torch.tensor(ratings_df['weight'].values, dtype=torch.float)\n",
    "# Add the node features and edge indices:\n",
    "data[\"movie\"].x = movie_features\n",
    "data[\"user\", \"rates\", \"movie\"].edge_index = edge_index_user_to_movie.long()\n",
    "# data[\"user\", \"rates\", \"movie\"].edge_attr = ratings_month.long()\n",
    "data['user', 'rates', 'movie'].edge_attr = weight\n",
    "\n",
    "# We also need to make sure to add the reverse edges from movies to users\n",
    "# in order to let a GNN be able to pass messages in both directions.\n",
    "# We can leverage the `T.ToUndirected()` transform for this from PyG:\n",
    "data = T.ToUndirected()(data)\n",
    "\n",
    "print(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "bd59ec9b-5523-4c2c-b57b-484963855922",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training data:\n",
      "==============\n",
      "HeteroData(\n",
      "  user={ node_id=[610] },\n",
      "  movie={\n",
      "    node_id=[8152],\n",
      "    x=[8152, 41],\n",
      "  },\n",
      "  (user, rates, movie)={\n",
      "    edge_index=[2, 52412],\n",
      "    edge_attr=[52412],\n",
      "    edge_label=[22461],\n",
      "    edge_label_index=[2, 22461],\n",
      "  },\n",
      "  (movie, rev_rates, user)={\n",
      "    edge_index=[2, 52412],\n",
      "    edge_attr=[52412],\n",
      "  }\n",
      ")\n",
      "\n",
      "Validation data:\n",
      "================\n",
      "HeteroData(\n",
      "  user={ node_id=[610] },\n",
      "  movie={\n",
      "    node_id=[8152],\n",
      "    x=[8152, 41],\n",
      "  },\n",
      "  (user, rates, movie)={\n",
      "    edge_index=[2, 74873],\n",
      "    edge_attr=[74873],\n",
      "    edge_label=[28074],\n",
      "    edge_label_index=[2, 28074],\n",
      "  },\n",
      "  (movie, rev_rates, user)={\n",
      "    edge_index=[2, 74873],\n",
      "    edge_attr=[74873],\n",
      "  }\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "# For this, we first split the set of edges into\n",
    "# training (80%), validation (10%), and testing edges (10%).\n",
    "# Across the training edges, we use 70% of edges for message passing,\n",
    "# and 30% of edges for supervision.\n",
    "# We further want to generate fixed negative edges for evaluation with a ratio of 2:1.\n",
    "# Negative edges during training will be generated on-the-fly.\n",
    "# We can leverage the `RandomLinkSplit()` transform for this from PyG:\n",
    "transform = T.RandomLinkSplit(\n",
    "    num_val=0.1,\n",
    "    num_test=0.1,\n",
    "    disjoint_train_ratio=0.3,\n",
    "    neg_sampling_ratio=2.0,\n",
    "    add_negative_train_samples=False,\n",
    "    edge_types=(\"user\", \"rates\", \"movie\"),\n",
    "    rev_edge_types=(\"movie\", \"rev_rates\", \"user\"),\n",
    ")\n",
    "\n",
    "train_data, val_data, test_data = transform(data)\n",
    "print(\"Training data:\")\n",
    "print(\"==============\")\n",
    "print(train_data)\n",
    "print()\n",
    "print(\"Validation data:\")\n",
    "print(\"================\")\n",
    "print(val_data)\n",
    "\n",
    "# assert train_data[\"user\", \"rates\", \"movie\"].num_edges == 56469\n",
    "# assert train_data[\"user\", \"rates\", \"movie\"].edge_label_index.size(1) == 24201\n",
    "# assert train_data[\"movie\", \"rev_rates\", \"user\"].num_edges == 56469\n",
    "# # No negative edges added:\n",
    "# assert train_data[\"user\", \"rates\", \"movie\"].edge_label.min() == 1\n",
    "# assert train_data[\"user\", \"rates\", \"movie\"].edge_label.max() == 1\n",
    "\n",
    "# assert val_data[\"user\", \"rates\", \"movie\"].num_edges == 80670\n",
    "# assert val_data[\"user\", \"rates\", \"movie\"].edge_label_index.size(1) == 30249\n",
    "# assert val_data[\"movie\", \"rev_rates\", \"user\"].num_edges == 80670\n",
    "# # Negative edges with ratio 2:1:\n",
    "# assert val_data[\"user\", \"rates\", \"movie\"].edge_label.long().bincount().tolist() == [20166, 10083]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "2cde198b-6b00-47ce-b80b-609e2ed31215",
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch_geometric.data import Data\n",
    "edge_label_index = train_data[\"user\", \"rates\", \"movie\"].edge_label_index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "62bc6dab-7f55-4309-b1f0-ed53da2e3f61",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([2, 22461])"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "edge_label_index.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "94f479f8-7a7b-449e-a498-152479ee76af",
   "metadata": {},
   "source": [
    "### Mini-batch Loaders\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "8bd2941b-070e-4e21-be7f-876b54d74220",
   "metadata": {},
   "outputs": [],
   "source": [
    "# In the first hop, we sample at most 20 neighbors.\n",
    "# In the second hop, we sample at most 10 neighbors.\n",
    "# In addition, during training, we want to sample negative edges on-the-fly with\n",
    "# a ratio of 2:1.\n",
    "# We can make use of the `loader.LinkNeighborLoader` from PyG:\n",
    "from torch_geometric.loader import LinkNeighborLoader, NeighborLoader, DataLoader\n",
    "\n",
    "# Define seed edges:\n",
    "edge_label_index = train_data[\"user\", \"rates\", \"movie\"].edge_label_index\n",
    "\n",
    "edge_label = train_data[\"user\", \"rates\", \"movie\"].edge_label\n",
    "\n",
    "if edge_label_index.dtype != torch.long:\n",
    "    edge_label_index = edge_label_index.long()\n",
    "\n",
    "if edge_label.dtype != torch.long:\n",
    "    edge_label = edge_label.long()\n",
    "    \n",
    "train_loader = LinkNeighborLoader(\n",
    "    data=train_data,\n",
    "    num_neighbors=[22, 10],\n",
    "    neg_sampling_ratio=2.0,\n",
    "    edge_label_index=((\"user\", \"rates\", \"movie\"), edge_label_index),\n",
    "    edge_label=edge_label,\n",
    "    batch_size=64,\n",
    "    shuffle=True,\n",
    ")\n",
    "\n",
    "# Inspect a sample:\n",
    "\n",
    "\n",
    "\n",
    "# assert sampled_data[\"user\", \"rates\", \"movie\"].edge_label_index.size(1) == 3 * 128\n",
    "# assert sampled_data[\"user\", \"rates\", \"movie\"].edge_label.min() == 0\n",
    "# assert sampled_data[\"user\", \"rates\", \"movie\"].edge_label.max() == 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "a652db4e-4e5d-4068-b27c-bb64a52932df",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sampled mini-batch:\n",
      "===================\n",
      "HeteroData(\n",
      "  user={\n",
      "    node_id=[606],\n",
      "    n_id=[606],\n",
      "    num_sampled_nodes=[3],\n",
      "  },\n",
      "  movie={\n",
      "    node_id=[2300],\n",
      "    x=[2300, 41],\n",
      "    n_id=[2300],\n",
      "    num_sampled_nodes=[3],\n",
      "  },\n",
      "  (user, rates, movie)={\n",
      "    edge_index=[2, 13173],\n",
      "    edge_attr=[13173],\n",
      "    edge_label=[192],\n",
      "    edge_label_index=[2, 192],\n",
      "    e_id=[13173],\n",
      "    num_sampled_edges=[2],\n",
      "    input_id=[64],\n",
      "  },\n",
      "  (movie, rev_rates, user)={\n",
      "    edge_index=[2, 6208],\n",
      "    edge_attr=[6208],\n",
      "    e_id=[6208],\n",
      "    num_sampled_edges=[2],\n",
      "  }\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "sampled_data = next(iter(train_loader))\n",
    "print(\"Sampled mini-batch:\")\n",
    "print(\"===================\")\n",
    "print(sampled_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "700224dc-2699-4ac6-9c23-ca1b03c1189b",
   "metadata": {},
   "source": [
    "### Heterogeneous Link-level GNN\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "id": "56e51477-8ac5-4ac5-8fe4-531b383301e5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model(\n",
      "  (movie_lin): Linear(in_features=41, out_features=128, bias=True)\n",
      "  (user_emb): Embedding(610, 128)\n",
      "  (movie_emb): Embedding(8152, 128)\n",
      "  (gnn): GraphModule(\n",
      "    (conv1): ModuleDict(\n",
      "      (user__rates__movie): SAGEConv(128, 128, aggr=mean)\n",
      "      (movie__rev_rates__user): SAGEConv(128, 128, aggr=mean)\n",
      "    )\n",
      "    (conv2): ModuleDict(\n",
      "      (user__rates__movie): SAGEConv(128, 128, aggr=mean)\n",
      "      (movie__rev_rates__user): SAGEConv(128, 128, aggr=mean)\n",
      "    )\n",
      "  )\n",
      "  (classifier): Classifier()\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "from torch_geometric.nn import SAGEConv, to_hetero\n",
    "import torch.nn.functional as F\n",
    "import coord_transforms\n",
    "from torch_geometric_temporal.nn.recurrent import DCRNN\n",
    "class GNN(torch.nn.Module):\n",
    "    def __init__(self, hidden_channels):\n",
    "        super().__init__()\n",
    "         \n",
    "        self.conv1 = SAGEConv(hidden_channels, hidden_channels)\n",
    "        self.conv2 = SAGEConv(hidden_channels, hidden_channels)\n",
    "        \n",
    "         \n",
    "\n",
    "    def forward(self, x: Tensor, edge_index: Tensor) -> Tensor:\n",
    "        x = F.relu(self.conv1(x, edge_index))\n",
    "        x = self.conv2(x, edge_index)\n",
    "        return x\n",
    "\n",
    "# Our final classifier applies the dot-product between source and destination\n",
    "# node embeddings to derive edge-level predictions:\n",
    "class Classifier(torch.nn.Module):\n",
    "    def forward(self, x_user: Tensor, x_movie: Tensor, edge_label_index: Tensor) -> Tensor:\n",
    "        # Convert node embeddings to edge-level representations:\n",
    "        edge_feat_user = x_user[edge_label_index[0]]\n",
    "        edge_feat_movie = x_movie[edge_label_index[1]]\n",
    "\n",
    "        # Apply dot-product to get a prediction per supervision edge:\n",
    "        return (edge_feat_user * edge_feat_movie).sum(dim=-1)\n",
    "\n",
    "\n",
    "class Model(torch.nn.Module):\n",
    "    def __init__(self, hidden_channels):\n",
    "        super().__init__()\n",
    "        # Since the dataset does not come with rich features, we also learn two\n",
    "        # embedding matrices for users and movies:\n",
    "        self.movie_lin = torch.nn.Linear(41, hidden_channels)\n",
    "        self.user_emb = torch.nn.Embedding(data[\"user\"].num_nodes, hidden_channels)\n",
    "        self.movie_emb = torch.nn.Embedding(data[\"movie\"].num_nodes, hidden_channels)\n",
    "\n",
    "        # Instantiate homogeneous GNN:\n",
    "        self.gnn = GNN(hidden_channels)\n",
    "\n",
    "        # Convert GNN model into a heterogeneous variant:\n",
    "        self.gnn = to_hetero(self.gnn, metadata=data.metadata())\n",
    "\n",
    "        self.classifier = Classifier()\n",
    "\n",
    "    def forward(self, data: HeteroData) -> Tensor:\n",
    "        x_dict = {\n",
    "          \"user\": self.user_emb(data[\"user\"].node_id),\n",
    "          \"movie\": self.movie_lin(data[\"movie\"].x) + self.movie_emb(data[\"movie\"].node_id),\n",
    "        }\n",
    "\n",
    "        # `x_dict` holds feature matrices of all node types\n",
    "        # `edge_index_dict` holds all edge indices of all edge types\n",
    "        x_dict = self.gnn(x_dict, data.edge_index_dict)\n",
    "        pred = self.classifier(\n",
    "            x_dict[\"user\"],\n",
    "            x_dict[\"movie\"],\n",
    "            data[\"user\", \"rates\", \"movie\"].edge_label_index,\n",
    "        )\n",
    "\n",
    "        return pred\n",
    "\n",
    "\n",
    "model = Model(hidden_channels=128)\n",
    "\n",
    "print(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "cc044b1a-e8d5-4046-9fc2-3615f8d1ec3b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<All keys matched successfully>"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# model.gnn.conv1.user__rates__movie.load_state_dict(encoder_state_dict['conv1'])\n",
    "# model.gnn.conv2.user__rates__movie.load_state_dict(encoder_state_dict['conv2'])\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e9bbf55c-feba-4946-9493-34a03123aa4a",
   "metadata": {},
   "source": [
    "### Training GNN\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "id": "7561e7ac-f62d-48b9-a4ab-ca150649d013",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Device: 'cuda'\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████| 351/351 [00:17<00:00, 20.26it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 001, Loss: 0.4039\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████| 351/351 [00:15<00:00, 22.45it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 002, Loss: 0.3221\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████| 351/351 [00:14<00:00, 24.29it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 003, Loss: 0.2920\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████| 351/351 [00:15<00:00, 22.58it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 004, Loss: 0.2721\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████| 351/351 [00:16<00:00, 20.90it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 005, Loss: 0.2544\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████| 351/351 [00:15<00:00, 23.19it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 006, Loss: 0.2417\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████| 351/351 [00:19<00:00, 17.83it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 007, Loss: 0.2267\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████| 351/351 [00:14<00:00, 23.92it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 008, Loss: 0.2175\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████| 351/351 [00:14<00:00, 24.86it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 009, Loss: 0.2004\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████| 351/351 [00:15<00:00, 23.11it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 010, Loss: 0.1970\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████| 351/351 [00:15<00:00, 23.25it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 011, Loss: 0.1870\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████| 351/351 [00:16<00:00, 21.36it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 012, Loss: 0.1834\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████| 351/351 [00:15<00:00, 22.21it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 013, Loss: 0.1759\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████| 351/351 [00:14<00:00, 23.85it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 014, Loss: 0.1700\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████| 351/351 [00:14<00:00, 23.67it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 015, Loss: 0.1647\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████| 351/351 [00:14<00:00, 23.66it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 016, Loss: 0.1602\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████| 351/351 [00:16<00:00, 21.69it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 017, Loss: 0.1558\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████| 351/351 [00:13<00:00, 25.22it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 018, Loss: 0.1549\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████| 351/351 [00:14<00:00, 23.50it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 019, Loss: 0.1534\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████| 351/351 [00:15<00:00, 23.27it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 020, Loss: 0.1532\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████| 351/351 [00:25<00:00, 13.68it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 021, Loss: 0.1478\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████| 351/351 [00:25<00:00, 13.83it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 022, Loss: 0.1504\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████| 351/351 [00:27<00:00, 12.73it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 023, Loss: 0.1462\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████| 351/351 [00:26<00:00, 13.34it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 024, Loss: 0.1454\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████| 351/351 [00:23<00:00, 14.78it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 025, Loss: 0.1379\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████| 351/351 [00:16<00:00, 21.67it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 026, Loss: 0.1368\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████| 351/351 [00:21<00:00, 16.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 027, Loss: 0.1387\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████| 351/351 [00:15<00:00, 22.70it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 028, Loss: 0.1383\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████| 351/351 [00:21<00:00, 16.03it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 029, Loss: 0.1334\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████| 351/351 [00:17<00:00, 19.70it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 030, Loss: 0.1326\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████| 351/351 [00:16<00:00, 21.85it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 031, Loss: 0.1338\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████| 351/351 [00:16<00:00, 21.38it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 032, Loss: 0.1302\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████| 351/351 [00:20<00:00, 16.95it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 033, Loss: 0.1331\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████| 351/351 [00:19<00:00, 17.65it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 034, Loss: 0.1322\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████| 351/351 [00:16<00:00, 20.85it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 035, Loss: 0.1308\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████| 351/351 [00:19<00:00, 18.15it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 036, Loss: 0.1289\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████| 351/351 [00:19<00:00, 17.72it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 037, Loss: 0.1267\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████| 351/351 [00:16<00:00, 20.94it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 038, Loss: 0.1231\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████| 351/351 [00:16<00:00, 21.63it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 039, Loss: 0.1250\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████| 351/351 [00:15<00:00, 22.23it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 040, Loss: 0.1231\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████| 351/351 [00:15<00:00, 23.20it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 041, Loss: 0.1266\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████| 351/351 [00:15<00:00, 22.66it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 042, Loss: 0.1225\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████| 351/351 [00:15<00:00, 22.72it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 043, Loss: 0.1213\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████| 351/351 [00:14<00:00, 23.58it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 044, Loss: 0.1199\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████| 351/351 [00:15<00:00, 23.02it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 045, Loss: 0.1222\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████| 351/351 [00:15<00:00, 23.29it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 046, Loss: 0.1152\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████| 351/351 [00:14<00:00, 23.40it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 047, Loss: 0.1201\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████| 351/351 [00:15<00:00, 22.58it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 048, Loss: 0.1179\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████| 351/351 [00:14<00:00, 24.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 049, Loss: 0.1169\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "import tqdm\n",
    "import torch.nn.functional as F\n",
    "\n",
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "print(f\"Device: '{device}'\")\n",
    "\n",
    "model = model.to(device)\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=0.001)\n",
    "\n",
    "for epoch in range(1, 50): # 50, 100, 200 dene\n",
    "    total_loss = total_examples = 0\n",
    "    for sampled_data in tqdm.tqdm(train_loader):\n",
    "        optimizer.zero_grad()\n",
    "\n",
    "        sampled_data.to(device)\n",
    "        pred = model(sampled_data)\n",
    "\n",
    "        ground_truth = sampled_data[\"user\", \"rates\", \"movie\"].edge_label\n",
    "        ground_truth = ground_truth.float()\n",
    "        loss = F.binary_cross_entropy_with_logits(pred, ground_truth)\n",
    "\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        total_loss += float(loss) * pred.numel()\n",
    "        total_examples += pred.numel()\n",
    "    print(f\"Epoch: {epoch:03d}, Loss: {total_loss / total_examples:.4f}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e4e0f1b1-f9f2-4b7c-9f0c-ce2a985d14f9",
   "metadata": {},
   "source": [
    "### Evaluating a Heterogeneous Link-level GNN\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "id": "655a34bf-bd2e-4d7f-bfdf-4ff7b3f97cdb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sampled mini-batch:\n",
      "===================\n",
      "HeteroData(\n",
      "  user={\n",
      "    node_id=[609],\n",
      "    n_id=[609],\n",
      "    num_sampled_nodes=[3],\n",
      "  },\n",
      "  movie={\n",
      "    node_id=[2640],\n",
      "    x=[2640, 41],\n",
      "    n_id=[2640],\n",
      "    num_sampled_nodes=[3],\n",
      "  },\n",
      "  (user, rates, movie)={\n",
      "    edge_index=[2, 19536],\n",
      "    edge_attr=[19536],\n",
      "    edge_label=[384],\n",
      "    edge_label_index=[2, 384],\n",
      "    e_id=[19536],\n",
      "    num_sampled_edges=[2],\n",
      "    input_id=[384],\n",
      "  },\n",
      "  (movie, rev_rates, user)={\n",
      "    edge_index=[2, 8293],\n",
      "    edge_attr=[8293],\n",
      "    e_id=[8293],\n",
      "    num_sampled_edges=[2],\n",
      "  }\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "# Define the validation seed edges:\n",
    "edge_label_index = val_data[\"user\", \"rates\", \"movie\"].edge_label_index\n",
    "edge_label = val_data[\"user\", \"rates\", \"movie\"].edge_label\n",
    "\n",
    "val_loader = LinkNeighborLoader(\n",
    "    data=val_data,\n",
    "    num_neighbors=[22, 10],\n",
    "    edge_label_index=((\"user\", \"rates\", \"movie\"), edge_label_index),\n",
    "    edge_label=edge_label,\n",
    "    batch_size=3 * 128,\n",
    "    shuffle=False,\n",
    ")\n",
    "\n",
    "sampled_data = next(iter(val_loader))\n",
    "\n",
    "print(\"Sampled mini-batch:\")\n",
    "print(\"===================\")\n",
    "print(sampled_data)\n",
    "\n",
    "# assert sampled_data[\"user\", \"rates\", \"movie\"].edge_label_index.size(1) == 3 * 128\n",
    "# assert sampled_data[\"user\", \"rates\", \"movie\"].edge_label.min() >= 0\n",
    "# assert sampled_data[\"user\", \"rates\", \"movie\"].edge_label.max() <= 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "id": "a90ed97c-cfd9-4fa8-8c75-f11e862a8e4e",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████████| 74/74 [00:03<00:00, 23.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Validation AUC: 0.9094\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import roc_auc_score\n",
    "\n",
    "preds = []\n",
    "ground_truths = []\n",
    "for sampled_data in tqdm.tqdm(val_loader):\n",
    "    with torch.no_grad():\n",
    "        sampled_data.to(device)\n",
    "        preds.append(model(sampled_data))\n",
    "        ground_truths.append(sampled_data[\"user\", \"rates\", \"movie\"].edge_label)\n",
    "\n",
    "pred = torch.cat(preds, dim=0).cpu().numpy()\n",
    "ground_truth = torch.cat(ground_truths, dim=0).cpu().numpy()\n",
    "auc = roc_auc_score(ground_truth, pred)\n",
    "print()\n",
    "print(f\"Validation AUC: {auc:.4f}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "id": "eef7168e-08c0-43e0-92e7-62c7dd7d6fd7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.8097\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "accuracy = accuracy_score(ground_truth, (pred > 0.5).astype(int))\n",
    "print(f\"Validation Accuracy: {accuracy:.4f}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "id": "add839d7-5784-42ff-bf0a-a5f2657d6967",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.8097\n",
      "Validation Precision: 0.8882\n",
      "Validation Recall: 0.4909\n",
      "Validation F1 Score: 0.6323\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import precision_score, recall_score, f1_score\n",
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "accuracy = accuracy_score(ground_truth, (pred > 0.5).astype(int))\n",
    "\n",
    "precision = precision_score(ground_truth, (pred > 0.5).astype(int))\n",
    "recall = recall_score(ground_truth, (pred > 0.5).astype(int))\n",
    "f1 = f1_score(ground_truth, (pred > 0.5).astype(int))\n",
    "\n",
    "print(f\"Validation Accuracy: {accuracy:.4f}\")\n",
    "print(f\"Validation Precision: {precision:.4f}\")\n",
    "print(f\"Validation Recall: {recall:.4f}\")\n",
    "print(f\"Validation F1 Score: {f1:.4f}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "id": "51c17c36-af89-4493-96c3-056421144f15",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.7275\n",
      "Validation Precision: 0.9348\n",
      "Validation Recall: 0.1961\n",
      "Validation F1 Score: 0.3242\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import precision_score, recall_score, f1_score\n",
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "accuracy = accuracy_score(ground_truth, (pred > 3.0).astype(int))\n",
    "\n",
    "precision = precision_score(ground_truth, (pred > 3.0).astype(int))\n",
    "recall = recall_score(ground_truth, (pred > 3.0).astype(int))\n",
    "f1 = f1_score(ground_truth, (pred > 3.0).astype(int))\n",
    "\n",
    "print(f\"Validation Accuracy: {accuracy:.4f}\")\n",
    "print(f\"Validation Precision: {precision:.4f}\")\n",
    "print(f\"Validation Recall: {recall:.4f}\")\n",
    "print(f\"Validation F1 Score: {f1:.4f}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e269ef5b-b3a4-44b2-bfcc-9d2e6a4d068b",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}