train.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import AutoModelForCausalLM, AutoTokenizer\n",
    "from LoraHelpers import convert_model_to_lora_model, change_lora_alpha\n",
    "from data_loader import load_dataset_from_list, load_json_data\n",
    "from tqdm.notebook import tqdm\n",
    "import torch\n",
    "from torch.nn import CrossEntropyLoss\n",
    "from torch.optim import AdamW\n",
    "\n",
    "model_path = \"./llama-3.2-1b-inst\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = AutoTokenizer.from_pretrained(model_path)\n",
    "model = AutoModelForCausalLM.from_pretrained(model_path)\n",
    "convert_model_to_lora_model(model)\n",
    "change_lora_alpha(model, 0.125)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "Crterion = CrossEntropyLoss(reduction=\"mean\")\n",
    "optimizer = AdamW(model.parameters(), lr=1e-4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_samples = load_json_data()\n",
    "data = load_dataset_from_list(data_samples)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Starting from v4.46, the `logits` model output will have the same type as the model (except at train time, where it will always be FP32)\n"
     ]
    }
   ],
   "source": [
    "a = model.forward(torch.tensor(data[0][\"input_tokens\"]).unsqueeze(0)).logits\n",
    "# a = torch.softmax(a, dim=-1)\n",
    "a = a.softmax(dim=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(torch.Size([1, 49, 128256]), torch.Size([1, 49]), 10)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a.shape, torch.tensor(data[0][\"output_tokens\"]).unsqueeze(0).shape, data[0][\"sep_token_index\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([  3513,   6261,     26,   5127,    775,    374,   2736,   4529,     13,\n",
       "           765,   9681,     25,   2570,   1395,  53246,    726,    518,    364,\n",
       "         61887,   9339,  17453,  78342,    518,    364,     71,  36224,     88,\n",
       "           518,    364,   1354,   5682,   1697,    518,    364,  34965,  45213,\n",
       "          2844,  83528,   7063,   2695,  35376,    518,    364,   3022,   3502,\n",
       "          7164,  58182,    663, 128009])"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.tensor(data[0][\"output_tokens\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(11.3790, grad_fn=<NllLossBackward0>)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Crterion(a.squeeze()[10:], torch.tensor(data[0][\"output_tokens\"][10:]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"#ach, be else is already taken.\\n - Unknown: selfself yourselfself','selfgames okins','selfappinessy', 'authpirationational','motunderuted',car-wil','self',spiror']\\n\\n |\\n\""
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.decode(a.argmax(dim=-1)[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_losses = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "epochs = 5\n",
    "for epoch in tqdm(range(epochs)):\n",
    "    pbar = tqdm(range(len(data)), leave=False)\n",
    "    for i in pbar:\n",
    "        optimizer.zero_grad()\n",
    "        mask_idx = data[i][\"sep_token_index\"]\n",
    "        outs = model.forward(torch.tensor(data[i]['input_tokens']).unsqueeze(0)).logits.softmax(dim=-1)\n",
    "        sample_loss = Crterion(outs.squeeze()[mask_idx:], torch.tensor(data[i][\"output_tokens\"][mask_idx:]))\n",
    "        train_losses.append(sample_loss.detach())\n",
    "        sample_loss.backward()\n",
    "        optimizer.step()\n",
    "        del outs\n",
    "        del sample_loss\n",
    "        pbar.set_description(\"\" if len(train_losses)==0 else f\"loss : {train_losses[-1]:.3f}\")\n",
    "    change_lora_alpha(model, 0.125*2**epoch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_losses_bkp = [float(item) for item in train_losses]\n",
    "#train_losses_bkp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiwAAAGgCAYAAACJ7TzXAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAABT8ElEQVR4nO3dd3gUVdsG8HuTkA0tCYQSEhKKVCkhUgNBQBCMETsKIsX64oeK8lpABcRCsFcUseGrAooioiCIQOgtQGiBUEMiSWghhQCp8/0BWXaTLTO7MztnN/fvurgusjs7c3Z2yjOnPMcgSZIEIiIiIoH56F0AIiIiIkcYsBAREZHwGLAQERGR8BiwEBERkfAYsBAREZHwGLAQERGR8BiwEBERkfAYsBAREZHwGLAQERGR8BiwEBERkfAUByzr1q3D0KFDERYWBoPBgMWLF1u8v2jRIgwePBghISEwGAxITk6Wtd4PP/wQbdu2Rc2aNREREYFnn30Wly9fVlo8IiIi8kJ+Sj9QWFiIqKgoPPzww7j77rutvh8bG4v77rsPjz32mKx1zps3D5MmTcI333yD3r1749ChQxg7diwMBgPef/99WesoLy9HZmYm6tatC4PBoOg7ERERkT4kSUJBQQHCwsLg42O7HkVxwBIXF4e4uDib748aNQoAkJaWJnudmzZtQp8+ffDAAw8AAJo3b44RI0Zg69atsteRmZmJiIgI2csTERGRODIyMtC0aVOb7ysOWLTQu3dv/PDDD9i2bRt69OiBY8eOYdmyZabgx5qioiIUFRWZ/q6YdDojIwOBgYGal5mIiIhcl5+fj4iICNStW9fuckIELA888ADOnj2L2NhYSJKE0tJSjBs3Di+99JLNzyQkJGD69OlVXg8MDGTAQkRE5GEcdecQYpRQYmIiZsyYgc8++ww7d+7EokWLsHTpUrz++us2PzN58mTk5eWZ/mVkZLixxEREROROQtSwTJkyBaNGjcKjjz4KAOjUqRMKCwvx+OOP4+WXX7baCcdoNMJoNLq7qERERKQDIWpYLl68WCUo8fX1BXCtbwoRERFVX4prWC5cuIAjR46Y/j5+/DiSk5NRv359REZGIicnB+np6cjMzAQApKamAgBCQ0MRGhoKABg9ejTCw8ORkJAAABg6dCjef/99REdHo2fPnjhy5AimTJmCoUOHmgIXIiIiqr4UByxJSUkYMGCA6e+JEycCAMaMGYO5c+diyZIleOihh0zvDx8+HAAwbdo0vPrqqwCA9PR0ixqVV155BQaDAa+88gpOnjyJhg0bYujQoXjzzTed+lJERETkXQySl7S55OfnIygoCHl5eRwlRERE5CHk3r+F6MNCREREZA8DFiIiIhIeAxYiIiISHgMWIiIiEh4DFiIiIhIeAxYiIiISHgMWIiIigTz7UzK+XHcMZeVekXVENQxYiIiIBLHvZB5+23USby47gOteWoZtx3P0LpIwGLAQEREJIv9SicXfE39O1qcgAmLAQkREJAhfH4PF396Ri14dDFiIiIgE4edrcLxQNcWAhYiISBC+Prwt28I9Q0REJAg/H9aw2MKAhYiISBA+BgYstjBgISIiEgTjFdsYsBAREQlK4jAhEwYsREREgsrMu4zzhcVYsjsTp/Mv610cXfnpXQAiIiKyLfr1lQCARnWN2PbyIJ1Lox/WsBAREXmA0wVFehdBVwxYiIiIBMEuK7YxYCEiIiLhMWAhIiLyEOcuVN9mIQYsREREHuKl3/bqXQTdMGAhIiLyEClZ+XoXQTcMWIiIiEh4DFiIiIgEIcH+MKHqPIqIAQsREZGHYMBCREREwjuZe0nvIuiGAQsREREJjwELERERCY8BCxERkSCqcx8VRxiwEBERkfAYsBAREZHwGLAQERGR8BiwEBERkfAYsBAREZHwGLAQERGR8BiwEBERkfAYsBAREZHwGLAQERGR8BiwEBERkfAYsBAREZHwGLAQEREJgnMJ2caAhYiIiITHgIWIiIiEx4CFiIiIhMeAhYiIiITHgIWIiIiEx4CFiIhIEBI4TMgWBixEREQkPAYsREREJDwGLERERCQ8BixEREQkPAYsREREJDwGLERERILgXEK2MWAhIiIi4TFgISIiIuExYCEiIiLhMWAhIiIi4TFgISIiIuExYCEiIhIEBwnZxoCFiIiIhMeAhYiIiITHgIWIiIiEx4CFiIiIhMeAhYiISBASc/PbxICFiIiIhKc4YFm3bh2GDh2KsLAwGAwGLF682OL9RYsWYfDgwQgJCYHBYEBycrKs9ebm5mL8+PFo0qQJjEYj2rRpg2XLliktHhEREXkhxQFLYWEhoqKiMGvWLJvvx8bG4q233pK9zuLiYtx8881IS0vDL7/8gtTUVHz55ZcIDw9XWjwiIiLyQn5KPxAXF4e4uDib748aNQoAkJaWJnud33zzDXJycrBp0ybUqFEDANC8eXO7nykqKkJRUZHp7/z8fNnbIyIiIs8iRB+WJUuWICYmBuPHj0fjxo3RsWNHzJgxA2VlZTY/k5CQgKCgINO/iIgIN5aYiIiI3EmIgOXYsWP45ZdfUFZWhmXLlmHKlCl477338MYbb9j8zOTJk5GXl2f6l5GR4cYSExERqY9jhGxT3CSkhfLycjRq1Ahz5syBr68vunbtipMnT+Kdd97BtGnTrH7GaDTCaDS6uaRERET6ad2ojt5F0I0QNSxNmjRBmzZt4Ovra3qtffv2yM7ORnFxsY4lIyIi0l/H8EAAwKDrG+tcEv0IEbD06dMHR44cQXl5uem1Q4cOoUmTJvD399exZERERPrr2SJE7yLoTnHAcuHCBSQnJ5vyqxw/fhzJyclIT08HAOTk5CA5ORkpKSkAgNTUVCQnJyM7O9u0jtGjR2Py5Mmmv5944gnk5ORgwoQJOHToEJYuXYoZM2Zg/Pjxrnw3IiIi8hKKA5akpCRER0cjOjoaADBx4kRER0dj6tSpAK6M+ImOjkZ8fDwAYPjw4YiOjsbs2bNN60hPT0dWVpbp74iICKxYsQLbt29H586d8fTTT2PChAmYNGmSS1+OiIiIvIPiTrf9+/e3O9fB2LFjMXbsWLvrSExMrPJaTEwMtmzZorQ45KHOXSjCEz/uxP3dInBP16Z6F4fIYxQWlWLtoTPo16YhahuFGDdBV0mShLeWp6JVozq418nrGqcSso1HO+ni3b8PYdvxHGw7nsOAhUiBZ39Kxt8ppxDfqQlmjbxB7+KQmW3HczB77VEAcDpgIduE6HRL3iP/coms2UbzL5e4oTRE3ufvlFMAgKV7sxwsSe52/qL217XqXAPDgIVUs/noOXR+9W+89Ns+vYtCRORVDHoXQAAMWEg1H/xzCAAwf1u6ziUhIiJvw4CFiIiIhMeAhYiISBjVuJOKAwxYiIiISHgMWIiIiEh4DFiIiIhIeAxYSD1seiUi0pRUjS+0DFiIiIgEZ2AiFgYsREREoqjOmWwdYcBCRESkCkYbWmLAQuphlSUREWmEAQsREVUb6ecuori0XO9ikBMYsJDu8i6WYOy32/B78km9i0LksuX7svHAl1uQnXdZ76JQJZuPnsON76zBPZ9v0rso5AQGLIIoL/eCtk8nv8JHqw4jMfUMJixIVrU4RHoY98MObDp6DtOWcNZy0SzckQEA2HsyT+eSuMALbhXOYsAigCfn7UTft9fgUnGZ3kXRxfmLxXoXgUh15y+W6F0E8kC24hEDxzUzYBHBn3uycDL3ElYfPK13UdynGj8lEBGRcgxYiIiISHgMWIiIiEh4DFiIiIhUIW4/k1lrjmD6H/v1LoZL/PQuAFVT4p7XROpgPy1yglap+d9ZkQoAeKBHJFo3rqvNRjTGGhbyOLkXi3G5pHqOqCKi6s3VeOZyiecmzWPAIhCDASjzhnwsCkkKHinOFxajy2sr0ePNfzQskbZ2nMjB7Z9uwI4T5/UuChF5CFZKM2ARykf/HEbHaStw+FSB6bXycglbjp1D/mXncjqUl0tIyczXJBCSm+yutKwcm46eRWFRqcvb3JVx5Saff9n1dSkhSRL2ncxDQaXf4fjZQuzPrJqEatPRs/hmw/EqwZgkSbjn883Y82+eZtk2i0rLMGfdUaRmFzheGMoCRm+QU1iMj/45jH/PX9S7KEI4U1CEP3ZnCpOuvrxc8uBj0jPLXVYuoaRMjN/fHgYsAkk9VYBLJWWY+ddB02s/JWVg+JwtuGvWRqfW+eE/h3Drx+vx0qK9ahUTAPDxqsOImv43jpy+4HDZ2WuP4oEvt+KhudtVLYM7rTt8Frd9sgE3v7/O4vUB7yYi/uMNOHehyOL1B77citf+TMH6w2ctXl+6N0vzss5Zewwzlh3EkA/XOVxWkiSM/GorRn291YNvEspMWLALH/xzCPd/sUXvogjhjk834Kn5uzBrzRFV1ne64DLynEiaV14uIaewGHEfrcfYbz33WqGmBnX8Nd+GJEm46b1E9Jm5GqWCBy0MWBy4WFyKKYv3YeORs44X1kDF/DpHzxQ69fmPV1+5CP2UlKFamQDg/ZWHUFBUahFc2TJ/25Vtbzueo2oZ3Omvq4FGdr71+WEyc62/nlHpKX67G/bB7n/lpx0/XVCETUfPYf3hs8i7VD0ys1acyydzL+lcEjFkXp3zaGXKKdNr/56/iGm/78OJc8quOwWXS9DjzVWIeu1vxeV4aO523PD6SqSeKsDaQ2cUf14PR89cwMVi7Wp7b+schg/uj8Lq//bTbBtFpeU4ce4iThcUIUvw+a8YsDjweeJRfL/lBEZ+tVXvongVuc0VpC3zShWDh7eSi1ZDJHlo8wAAPPTtdny3+QSGz1FWC3XinPPNbJ4SpFTYmX4eA99bi5veXavqes2PY4MBuCu6KVo2rKPqNjwVAxYH0nM8p50772IJth47p+jCrddFPpfzrJCKXvhlN27+YB1Hj6nk8NWmXtGfuCuUlUuYs+4oPll1uEo/M60s35cNwHatqxo8/SFCbQxYBORsCHHrx+tx/5wt+G3XSVnLPzlvJ275cL1qna08+YnSEc47Jrafk/7FkdMXLJo1qPr4OSkDM5YdxHsrD+HR75L0Lg5phAGLF6lok1+2N1vW8n/uyULqqQJsOXZOy2J5BUcVUQxoyBP94yUBXkpmvun/W+31E/OCZyrRmj7diQGLgKrbva/6nn5E+kr464Dp/0qC7oS/DuD9lYc0KBHZpNKNYcuxc7igQooJPTBgIdUoa29lmELkiU7lX8YXa4/h41WHq/QZYk2j+N5cdsDmIBLRK28YsAjI3ceM6AdphUvFZfhi7THFnzt+thC9E1bhf5vTnN62p16Iq8tQZRF5a4dJURLMOUXmT7JgW7pba5DmrDuKIR+sQ0pmvluu/7szct2wFfUxYCGX/HPgFFYdcKYdXPnF/MN/Dtlvn7Zh2pL9yMy7jKm/e/ZMpc6Imv43ktI8N/+NUiIFlt7cCd3bTVq0Fx+vOmw1i7VcxaXluPWj9Xh+4W67yyWl5WDGsoNIPVWAOz9zLkFodcGAxQs5e9FesjsT2524uT1ytVe+3Av0iXOFOFspM6wcSU7OvSN69katzXaiVspTeUptoSdZuicLl4r1GS4uSZLsKUC0UODCFCAbjpxBSlY+Fu741+5y+05eC4o8uvbKDRiwCEiPh8QDWfl4ev4uDJu9WfNt/ef7HRZ/C/RQTFf9nnwSn6w6rHcxcPTMBY9LKOapbD3ojJ+3E1N/3ydvHSqfzaO/2YaB76/1yBt5uQpFrhPg5/pKvAgDFgGZP0+444lRApChYYK8ysPwjp91bpqB6sb5nvyuHzQTFiTjvZWHkKxzW/fA99ZizDfbPLbN3Zai0jJ8tf6YxUSnIvt1p/1aAq2sP3wWx88WYs+/ubps3x5rw4vVvl4/fmNLzbfhSRiwOGF/Zh7eWXGwyg1lyAfrMODdROxMP4/529IhSRJyLxbrVEr1FZWW4fU/U5Cw7ECV915ZvNcie21pWTm+Wn8M4+ftRJZZJsgNh9Wbk6liniVzJVe3a56XwZy96uWJPyWj+aSl+Gr9MexKv9b89E/KKdN8SO7wx+5MXPfSMnSctgKv/5mCjtNW4LH/JVmU/c89mZj+x35MWLALv16tcl6081+8+Mse/HPgtMX6/jlwCpuOWu73tLOFePS77VhmNhnju3+nQpIkHMy+tu/OO3H8ZuZewut/piDdSpr235NP4v2r21Hiu01paD5pKUZ9vRWTft1jmr/lVP5lPLNgl2k587LLkZSWgxd+2Y2cQvXP0+1p5/H6nylWZ0r/ct0xvLH0AG7+YB0+WHnI7v44duYCnl+4W7VA/6v1x7B0T9VJOPedzJeVRLJyUYtLyzH9j/1YvOskxn67zeK9zxKP4PPEow7XaW2yxKw8+XM9WWuOXn/4DF7/M8Vm7czkRXssrhOSJFUpx+RFe6tM8FpwuQRfbzhumoMJABZfTdb51z55ObDsqZiotlWjOqhjvFbDUrn2asuxc5j06x5M/DnZolnJ3MHsfLz8216ctpKNt/mkpXjtjxRMWXyt9kyChB+3nsD7f6e6/D20wPomJ8R/vAEAUFhUhldv7wAAOHK6AKlXn5bu/mwTgCvzEKXnXMS3D3XHgLaNFG/nckmZU51MtfLdpjR8veG41fd+2JJu8fesNUfxwT9Ve9k/+HXV4XRyb12VL+oTFiSjeUhtREUEm177fvMJvLH0SkCVNjO+yjp+3HoCo2KaV3n9fGExFl296FT+/KP/cz5z5u6MXIzs2ezad5DxmafmX7sBV+zvlSmnsHx/Nm7t1AQA8OS8a8v8npyJ2NYNMPFn2537Hvhyq+n7XCouQ/93EwHAIrj5fssJ3Bkdjns+3ySjlLY98l0SDmTlY/m+bGycdJPFexMWJAMAYls3RI8W9WWvs+K3qZj9OqSOP54f0g7PLdxtMSP2rDVH8fyQdrLXe+/VJtDi0nJ8ODxa9ufk+nrDcXQIC8TdNzS1eH1Xeq7p/x+tOozuzesjtnUDAFcCz8mL9mLq0OsxsmczDJ+zBacLirDxyFlsmjzQpfIcyMo3Hd/xneOrHI/L9mbhji7hitY5b+sJfLsxrcrruReL8fbyKze+B3tFom5ADZvreO6Xqseu+WzaX60/joZ1jWgWUlt2uUZ9fSV4ahIUgEf7XqmpOGoWfMzfloH52zJM58XU3/fj+y0n8O3Y7qZljp8txD2fb8LuaYMBAGcvFKHbG/9U2dYzPyXjzuhwLNmdKbt8thyTGZiaz/O0aOdJq9e7Wz5cDwBIszGJ5Tcbq17PX/7tSgBza+cmaBcaKKss7sIaFhfM3ZRm+v8vO6o+7VfMQ/SBk8Pj1Dj45ZAkCXL6tWXkyH/i+WHrCRdKJF/lE3Gfg179S/dWfbIErsxYqoWfk9SrSj9npxZASZBxrtB2h+e8S67XNBzIuvLUaj4b8vnCYtzy4TrT367WaFTMjq1WrcNxFybtc+RUvuMO5uad0Cf+vBtFpeV4+bd9SErLwemCK+9l5l3GKRfnrXG03505D2zNJG9es1FaZv8CY22kofk8bsv3Z+Om95ybZPDf89eOw3wrnWglScLLv+3F91uuXLPeWWFZu2CeGmC2jNoi03qVFrQSNXsDHchS3vR4wYUOx1phwKKS2WvlH8iOVByo1qqStfDod0kY98MOxwuSsMwvyiL6fO1RHOQM3YpVfmiZvy3dxpLqKCottzqBZLl0pdl39cFTVTrnVtzotebs9fBicandpvm9J/Pw41Z5+1XHAUtu99P2DEz9fZ9QUwEwYHFAj9/K3ZsslXkWekNeCa1+T3tDycf/uBOTF+3RZsMeokjlWZRFuohq6X+b3RMMVJiyeB+ipv9t9b0ftqTj4bn6TizYfNJSrJMxasy8dufnpH/R5bWVNjuxX9RpyLboFu74F//bfAKJqeKM0mPAUo0UlfLE1MPSvVmYvy0DpWXlXj2Eu6xc8sjhp55O7TwljpqF9I4VR3+zzeEybV75q8prx200XWnFE891a8PSRcqWzYDFATUyZ6p14JaWlZtGRzij2+tVO4uR88yf8uVcxPu9k4jvXHhiFv0CeN1Ly6zeKLRgECmlrY7+STmFztP/xt/7nRud4i17Uc38L6IfWtWjbtE6BiwOqPE0odYB1ithNa6fusJh7gZb51uBWZWoJw63drbq1vw3NB+d4arX/kxRtLx5J1RSh1o3l90ZuVigcf8Qc85eEyp3hHz0f0m4UFSKx7+X1wfNnbUj5ptK/jcXG4+ol9JAS+7cRyVl5SiUeV1zRyAlerM/AxY32PNvnirVthUjCW7+YJ2DJR17a/lBu+/nXSrBNxuOWx2/rxc1RoQUq5im33wop+hPZXJp18dH3R2kRR+WSYv2Cp/I7SsbaQUcuVhcihd/2YO1h047XtiBDU4EHg99ux0jv9qq+/XE2nGj1akr5wjt/05ilVFJFbzlmqIm5mFxQK2D5ted/2JYtwh1VuaCQ6cK0KZxXYejSib9ugd/7cvGj1tP4LuHe+BAVoHubdfO4omvv8o3ClF/kzMXitC6cV1N1i1JEsb9sAO1/d1/2f088Sh+SlIn+eFLv+11+rOnC4rQKDBAlXKYk1szYG2pyq+589j0hFpXkWpdWMPigFo36ZUpzsxobJuzk5ENllk7s+pqQrGjZwoR+9YaPPa/JNW/QwW5+9jaYtWpL0M1+qpe6d/zl7Bi/yks2nXS7Z2Tbd0YXb28yR1haM1P29Ox4+qEptXpPCbnMWDRyfJ92S5VP7efutzmjMd/p5zCIg3m/qhIYEVVqT3pmyi881vpo9xTqyjt6Dhthc337H3dTUfO4sVf97qcVZmqFwYsDqgV+P+dcsqUkGnrsXMY98MOl/uirLAzMsBemnZvUl3ycZB13hooerujLvZHO2Ej1bwj7jxaeGSqjwGLA2reD9+92rlqv42J+cj9ysslfL/lhCmdPGmjcpW/J8WZF4pKsWJ/ttUMsEqYf+fKD0JsEVFmy7Fzqq2Lu/4a0c9Ldrp1o7WHzuAVvQvhJmpfBH7cesJqu7+rbd+/7TppMVtpdVf5giX49cstxv+4E2sPncGwrk3xzrAovYvjMZScmt5YU+r6XELWd6C7d5VIPw1rWBzQ88mHkf8Vx85cMM0gqjbWdpE5azeJtVdTwS/c4Vy/MNaeiMHW76DkfqzniBkeRgxYHBIpuvQkau42V2f2Jfm0url64xO0XNa+epWaLC/cPZ76nRhgiosBi85sTcgFyLvpe+uNIP9yidd+N2cV2jlWPE11vylU9+9fmZ7DmitvmZcdSyIdqwxYHJDzY8mZPdSWUV9vdfqzC7ZloMtrK7E7I9fpdVizOyNX1YywjlQe7bQ9LQedX/0bL/zieIZjRzfxM3aGYp/Kv+xUEJCZe6lKMHXpaofMzNxLKNNoDvoZyw5iZ/p5ZORcdOrzJTJ+U7WLXlxajqy8S4pvSJIkYWf6eZvvHz93UVZAW1YuIbNSDhJ7ZVFS5Z+Rc7HK8ZOVp38iMKsPQVa+ltqTJspRWi5dOX7Nfjs1zhc5q1ixP9vu9UAOe6PSzleqCbbWSftMQZFbcvCUlUs4pGLWZlEeHhmwOODod9p09Kys2UNtcWVum70n85B3qQRPL9hl9f3l+7KcWu8dszY6XSZnVJ4d9uNVhwHI6zMweZHtrJsz/zqI7m/+g01Hq44oyM67jJ4zVtnN/jl3o/U06L1nrsZLlfrU3PP5JqxJPY3eM1dj7LfOHw+O3P3ZJvR9e41Tn739U8e/62P/S3K4zIQFu/DI3O2yLmJtXvkLMQmrsfvfXDlFNBn4/lrc/ZntHB27M3LxjdnUCObMp5145Lvt6D1zNRJTXU9Jb+7I6Qvo+/Ya9J652vTa8n1ZiElYbedTtqn1FPv28oPoOG0F/trr+Nx/ar7164Za7vqs6vH28Nzt6Pv2Gqw8cO33GOPE9bNy4DBvq+N5oD5ZfUT2vD3OiH59pcXf5vu3uLQc932xGd3f/AdDPpSXziLHhfnenvkp2SJJqNxmdWtn9G+7MtFzxirsUXgOa4EBi4uenKf+Sb/64Gl8uvqw7OVtXevG/bBTnQJ5qNlrj9p8b+txx8MiX/3D9uSG861MlDf36g10/WExJ3lTY+h2cWk5fk/OxKqDp5GRI782QWlgfuyM4zwbH648ZPX1zxOPmmY1T0y9Uvv53aY00/v2Ai25eV0qAqC8SyWm1z5LtH28ucOag6dNZXi58sg3K19rqYygxhVZeVXnDaq4cZrXSjszN5GWtGgCScnKx7bjOQDkz4nmSm3QH7sznf5sZesOncHpgiL834/6308YsDhQ+eCtfBBp1SH03b8PId/sYmhP2rmLKCrV7snBnS6XlAl1w9c6E+f//bgDbyic9dkVruavMG8yEWmOkcp0aO3Q3aRFtptQK8/yTO5VVq5eM5CW5529WE2rpm4lGLA4cDr/WoBSWlaON5a67+Yip89BhVmrj2hYEvf5WaUJ2tRSMdeJVpbtzTbNwHu+sNilieXkGD5nC85ekB9k27uAidCsLVKHQCXcue/KyyX8bWMeMDk1WaL7207Gb2/iqce6mhiwOFBQdK2WQwJwTsHF3lV/7pFfZbvlanWjp3NmUkdn03SLZtqS/bLa4l11Kr9qVb1cTIXvPHsxipYBjDs70LubJEl4/PsdeheD3ISZbgUmt60T0KfHvyhSswswft5O3NklXO+iuOTI6Qt6F8GqsnIJJWXlCKjha/G63kec3tsnHV2Nm5/5KVnXYsglQm2kN2ANi8CUHONJGjddiOyLdcew72Q+3lh6QO+ieKU7Z21E+6nLkX+5hNXSdijZNZX7IaixXz35pqj461/9rr8nq9e5lMTHgEVkHnwBcpYzX9k83wGbLNS392QeJAnYcvSczZuit3T6diVwqIanq1fQM2mdaEQ/hhUHLOvWrcPQoUMRFhYGg8GAxYsXW7y/aNEiDB48GCEhITAYDEhOTla0/gULFsBgMODOO+9UWjThKT4YPPg8UqvoP21PVzaJmvCnnPfYeOQsikvL8crivWj7ynJdynDinLwkenKPCi1qKfS4H3rwpcPtKg9z1+KhhzGROhQHLIWFhYiKisKsWbNsvh8bG4u33npLcWHS0tLw3HPPoW/fvoo/qxXzg1eS3HvgVcdj/J0VqRZ/v/jrXvyx237nY0+uCvc05sf/K4v34dU/9uOHLdp3FLbG3rQWatFqKKfWNYGedkqIVF57Dz3OPhBZu0alODnxanW+3inudBsXF4e4uDib748aNQrAleBDibKyMowcORLTp0/H+vXrkZubq7RowlN6iaqcAbY6sHaDmGuW9EtkLk8n74ERqjtGNenl41WH8Vmi/HQBjn4+UdKbewUPPFcqe+HX3YqW17u5W4TDV5g+LK+99hoaNWqERx55RNbyRUVFyM/Pt/gnusOCjAJxx4XztItzdpB2lFz2Ks97oiQ3kEicudS/v/IQLpdo8321brrMKSyuMrcNyWMtMKhInbD/pLj3mSmL92GDQEk3tSBEwLJhwwZ8/fXX+PLLL2V/JiEhAUFBQaZ/ERERGpbwCoPB9SjTHdXYjtz3xWa9i6AqAQJ/AJ7z0Kdkf1XOcRH/8QZ1C1MNWLtmuONptf+7idpvxAvI6XTb751ElJSVY1uac/mu3HGN+n7LCTzowmS6nkD3gKWgoACjRo3Cl19+iQYNGsj+3OTJk5GXl2f6l5GhTYbUyk9CR8+4VktSVqb/7XV7WvUdAg0A3248jiKNnpw9gSs3SyW5gewRNbjTslnO3aNR8mRO7SECUY8Hc67ULlr7fno38Vhjd54tAYqre+K4o0ePIi0tDUOHDjW9Vn513gU/Pz+kpqbiuuuuq/I5o9EIo9HotnJWsDahF3mW6X+koElQgN7FsHAwO99tbcQiXHj0ZFCjqtRMUWkZjH6+DpdjHxYx8XfxHLrXsLRr1w579+5FcnKy6d/tt9+OAQMGIDk52S1NPfaIGAWT60QLPG/5cL1HPRG7KvdSCf77825sOupZbe4VfRnKzW5y6w8p/w6FTkxB4Uh1uu2WlkkuT+RpSzkDGGEprmG5cOECjhy51nP++PHjSE5ORv369REZGYmcnBykp6cjM/NKBsLU1CvDVENDQxEaGgoAGD16NMLDw5GQkICAgAB07NjRYhvBwcEAUOV1vfE4FpO3PCGdzL2kdxHcZuZfB5FTWIxfd/6LtJnxOJl7Cc/9vBsXS8owJb69KtswPy7kHiEnzhVCkoCY60Ksvt/vnUQk3N0JM5YddKlsuzNyXfp8dTdtyT6UqNS8Xrmpbr+Tw429QcZ5sa9BigOWpKQkDBgwwPT3xIkTAQBjxozB3LlzsWTJEjz00EOm94cPHw4AmDZtGl599VUAQHp6Onx8dK/cIaqWRIjvcsxGsHyy6jDeW3nI9Pe9s9XpFG5rqgZ7Ae6Lv16ZLXvZ07ZzQU1epP6M2rvSc7F410m8ctv1qq/bG6kVrCghSm27lukuPl51WLN1q0FxwNK/f3+7J/zYsWMxduxYu+tITEy0+/7cuXOVFksz5y9yaCCRlsyDFTV9veG4059NyXLvU3ZFrqG0c0lu3S65hxrhVUVF0Pxt6RjUvpEKa7SyDU3Wqh5Wczjwr9pVZKIfER5OlKcgkVX3TreaUWHHnhS8Sp7E8N+FypLOeQsGLAqocaHPyuMFyVsJ0NJCenLQ1ibn+OBcWN7J6rBmAR8c7B19IjQlM2Bxs1s+XK93EbwaL/hkjZ73BhFvTKIp1WjOJjWp/TuKEABUJvoABgYsCgj+WxLJUh2OY1fvLaJfuImqI90TxxF5Cz5IV3MOHsH1OD4uFZdhxjLro6VImUm/7nH6s9byPrlSG6zVsSR6mM4aFgXu+myj3kUgcll1aKKofOE1/1tO64OSNPqP/i8JiamnFZdJbdYqhT5dcxjfbzmh8Zarh8XJmU5/9qn5u1QsSfXFgEUBb0oolH9Z/0kYSR+Z1ShBXWUVmWrVNvbb7aqsR+38IkdPa/N9vYmnBPB6F1OE/cSApZpips3qa/ofKU59zt2T92nh7AV5eZXKyp1IzsV+Lx6JP9s1ou8LBizk8UQ/ybxFmQeM5FBLRcZbNXy25spUJlofp14QT1YbZeUS9p10vsZeq4cH0c9wBixEJMvrfzpXM1Nt2LiJ5F8uxen8y5on+2Lg7px/z190+zZXHTil2bpLyrRL3a83BixEJEt1mpxRbS/+uofNsIKa+LP7s8YWaNiH8Ma312i2br0xYCFSCR9wxaV3a8nxs+z8Su5hbQi1GkSowWPAQkReT7LyPyK98Ch0DgMW8ip6Tn647tAZ3bZNRJ5j45Gzyj+kc6/q7Hxtam6UYMBCHo/zB1FlejcB6YPngaf4bddJvYtgnQjtPnYwYCGvknepRO8iEBFpqnoG5JxLiLyAeTPQPxoOFyTPYe058fstJzBn3VG3l8V9quttjKoLBiw6eI35LFTFJiGqrPJsyyfOXcSUxfs03ebFIttDVfXKuZem0VQE5J12/5undxHsYpMQEXmdJ+dZTjbnjmHFh09fsPleeo72ycnOXiiq8trB7ALNt0vkLgxYyOMJ3k+MdLCWI7bIi1XXaRgYsBAREZHwGLAQEREJrppWqlhgwEJERETCY8BCHs9eZ0ciIvIODFiIiIg8yNkLxXoXQRcMWIiIiEh4DFiIiIhIeAxYiIiISHgMWIiIiARXXZPFmWPAQkRERMJjwEJEREQOles1i+dVDFiIiIjIoSW7M3XdPgMWIiIicmjDkbO6bp8BCxEREQmPAQsREREJjwELERGR4Aycr5kBCxEREYmPAQsREREJjwELEREROaR3oxQDFiIiInJI37RxDFiIiIjIAzBgISIiIofYJERERER2cbZmBixERETkARiwEBERkfAYsBAREZFDO9PP67p9BixERESCO3ehSO8i4OiZQl23z4CFiIhIcAWXS/Uugu4YsBAREZHwGLAQERGJjsOaGbAQERGJziBIxJJ2Vr9+LAxYiIiISJaM8xd12zYDFiIiIsEx0y0DFiIiIvIADFiIiIhIeAxYiIiISHgMWIiIiAQnShcWSdJv2wxYiIiIBMdOtwxYiIiIyAMwYCEiIiJZ9KzpYcBCREQkOFEy3eqJAQsREZHg2IeFAQsRERHJxFFCREREZBNrWBiwEBERkQdgwEJERCQ8VrEwYCEiIiLhKQ5Y1q1bh6FDhyIsLAwGgwGLFy+2eH/RokUYPHgwQkJCYDAYkJyc7HCdX375Jfr27Yt69eqhXr16GDRoELZt26a0aERERKQhj8rDUlhYiKioKMyaNcvm+7GxsXjrrbdkrzMxMREjRozAmjVrsHnzZkRERGDw4ME4efKk0uIRERGRRvQcJeSn9ANxcXGIi4uz+f6oUaMAAGlpabLX+eOPP1r8/dVXX+HXX3/FqlWrMHr0aKufKSoqQlFRkenv/Px82dsjIiLyJBwlJGgflosXL6KkpAT169e3uUxCQgKCgoJM/yIiItxYQiIiIvdhvCJowPLiiy8iLCwMgwYNsrnM5MmTkZeXZ/qXkZHhxhISERGROyluEtLazJkzsWDBAiQmJiIgIMDmckajEUaj0Y0lIyIi0gebhAQLWN59913MnDkT//zzDzp37qx3cYiIiEgQwgQsb7/9Nt58802sWLEC3bp107s4REREwuBszU4ELBcuXMCRI0dMfx8/fhzJycmoX78+IiMjkZOTg/T0dGRmZgIAUlNTAQChoaEIDQ0FAIwePRrh4eFISEgAALz11luYOnUq5s2bh+bNmyM7OxsAUKdOHdSpU8e1b0hEREQeT3Gn26SkJERHRyM6OhoAMHHiRERHR2Pq1KkAgCVLliA6Ohrx8fEAgOHDhyM6OhqzZ882rSM9PR1ZWVmmvz///HMUFxfj3nvvRZMmTUz/3n33XZe+HBEREXkHxTUs/fv3h2Qnc8zYsWMxduxYu+tITEy0+FtJzhYiIiKqfoQc1kxERETX+LALCwMWIiIiEh8DFiIiIpJFx6mEGLAQERGR+BiwEBERkfAYsDgwbej1eheBiIiqOT2bYszp2feXAYsDD/ZqpncRiIiomsspLNa7CLpjwOKAL2ecIiIinf25J8vxQl6OAQsRERHJwlFCRERERHYwYCEiIiLhMWAhIiIi4TFgISIiIuExYHGAg4SIiIj0x4CFiIiIZJEk/cYJMWAhIiIiWTismYiIiMSnY8TCgIWIiIiEx4CFiIiIZJF0rGJhwOKAgcOEiIiIdMeAhYiIiGTRcZAQAxYiIiISHwMWIiIikoU1LERERER2MGAhIiIiWZg4joiIiMgOBixEREQkC+cSIiIiIuGxSYiIiIjIDgYsREREJDwGLERERCQL87AQERER2cGAhYiIiGTiKCGhPdgrUu8iEBER6Y5NQkRERCQ8DmsWnAEGTdbbs0V9TdZLROROsa0a6F0EqgYYsMhwR5cwTdYbXq+mJuslInKn4T0i9C4CVQMMWGTo1pw1IUREtujZr4Hci31Yqiue5EQkMH9f3iLIksRRQkREJJraRl9Zyxm06eZHAmINSzXw/SM9LP4e27u5PgUhIlKZVgMTiMwxYJHpmUGtXfp839YNLf5+9fYOLq2PiIjI3Tis2QM8M6iNauuKaRmi2rqIiPSmZ78Gci9JxzYhBixu9EDPKxlzX7ilbZX3ataQ11ZMROQuDENIJH56F6A6mXFXJ7wS3x61/K/sdvOLATutEZGnur5JoN5FoGqANSwKTLzZ9WahimClMsYrRCQaudelyPq1NC0HiYOjhDzE0wNd63hLREREzmHAoqOa/uy3QmTP+AHX6V0EIjLDxHFe5r5uTWUt9+ygNugUHoQ37uyocYmIiIhcxyYhLzN1qLwcKw3rGvHHU7F4sFczjUtERETkOgYsXqaOUfngq1s6NtGgJERE2jNwmGO1wcRxXqRxoNGpz712h+dlvu3buoHeRSAiL9ajeX29i0ACYcAiiNoKamVmP9jV5e25OtUAAHwyItrldZDn47xYnq1daF2X16FF/conI6Lx5ZhuGqxZnnfu7azbtsk6BixuoHYW2yZBAS6vw9/P9Z8+uJa/y+sgsRn9fDA5rp3dZYZGhamyreeHVM0ATdp7d1gUmoWIl0dlaFQYgmrW0G37w7pF6LZta8JUuO6rgan5CQAwXeaEiCF1/Jmoidwi9Y04tG5cR7ftd/PwJgF/X7EvsQPaNkTH8CCseOZGq+8zNb84fHzE6CfEPixexJXgc0zv5ujVUt4FeoKTSewGtW+EN+9yfRi1UYUaGhLbV6PlVcer1d+y8nq+Hdsd/ds0tL6whxC9L6rR70rtb4CLtcDm33NQ+0YurYus07O2yQJHCZFSd0WHY/aDN2DjpJtkf8bHAHw1pjtG9nR9GPV3D/dweR3kGaIj6umy3QHtGnH0icbMd2+LBrVVWecT/Vupsh6yFNMyRO8iAGDiOK/iruurj48Bt3RsgvDgmrI/o9bF/75uTdFLkJOHtFevNvsqOcuTmlT6eXhtlhKN6hrxf/1tZ1FmfyrbmIeF7IrvpH6OFlcOuuE9ItUrCBF5FEc1MaJUigXUsH172/rSQLxwSzs0qFM1DcWWyQMxfgBriUTEgEVgy5/pi69Gd8OMuztZvK5nhAvIH8LI4a7e4/fxfdAxPFDvYpCG9Bz9oYV5j/Wy+V5FbfPzQ9q4qzheQ8858BiwqMzVc95gFg60Cw3EoOsbI6hmDdzZxfWho2o9+MhtWnIm4y+Jw/xQjooIxtyHrPdbEuSBWkyCxwBqxCjNKw2JFuV4uCGynsMm87uiq877JkoNUWWiHErNQtTp6+QMBiyCsdWhSe1hzO44KfXsnGWLGjlsqitbNze1fmWDMLc6quDoF3l+SFus+m9/ResMrlUDn4+8we4y7poQVo18VHLcHR3u8jpEqQBjHhYvYi0QUGMYsaNtyBFp9iTkyjHH20r1FFjTuRqzni08O5eKSwQ/WXx9lRfQ/BP+vj7wrZQfRE4NbO/rbE/r0TjQ6HUTwjbmg5IqGLBo7PaoMNx9Q9VqRz20D2UfBHKe0c8XmybdhE0KhtIDrtfAhAbyYq+VAD99+iME1aqBpFcGoW3jqtMC1GMGbbKBAYtG3r8vCr1a1serMrPXuoWNB58P7o9Cn1YhVi8ecvm5mIXxucHu6fzm6ZlT9RYWXBNhCobSK2HrwVzUPgWyCFKNr5W6AVdq3ZxJmdCgjhG1jNcCprfv6YzQwAB8cH8XtYrnFC0Ot8HXN9ZgrfpQMu+d2hiwaOTuG5piweMxqK8wh4Ue7fh3RTfFj4/2QnCtqpkUrb0m99qk5LvEaTB025o37ujoVRcPEXhyPKG1LpHBehdBtnu6Vq0JthVvJdzdCbd1buJU7bGt4+W+7hHY8tJAtG+iXk2wU/0tNDigoyNdT774cGxz1wuigjYuPNi6igGLylz9Md3VUbVyu7Mt3z3UAzdXusHbCkRmVhp+7QqlQcVdMju1BdWqgWcGec5QRnvJrQBg/QsD3FQS7Xlb4LNx0k0e1ZzVuWkwtkweaPq7cWDVHCUAAAMwokckPn3gBoedVh1N2Odtv7mWmtarhb+ftT7nkxL28tOITnHJ161bh6FDhyIsLAwGgwGLFy+2eH/RokUYPHgwQkJCYDAYkJycLGu9CxcuRLt27RAQEIBOnTph2bJlSoumu+HdI/DesChN1t1W5f4nD/SMRMsGtfGffi0dLvvJiGgseNx2ToMKlRPKKQm+zC9cgQF+mOVgFEFlEV46GaS96ldfH4PXfm9vEB5c0+NahELNAoyeLZzLZm1+LrdrEohO4UE2l/W0/aM3uQ+a9ogy2sgZigOWwsJCREVFYdasWTbfj42NxVtvvSV7nZs2bcKIESPwyCOPYNeuXbjzzjtx5513Yt++fUqLp6uZ93RGI42eqG5sY7tXvVyWQUENrH6uPybHtb/2vo1zIaCGr0Uqfq37FLQNrYsaVma5vVGl1OEe3SfCA8mdPiLQxuRunnyB9US/j++DB3tFyp493h4DeL6RehT3nomLi0NcXJzN90eNGgUASEtLk73Ojz76CLfccguef/55AMDrr7+OlStX4tNPP8Xs2bOtfqaoqAhFRUWmv/Pz82VvTwtDOmjbL8LP59oNXPQp67XiqHq5sgFtG2JN6pkqr3vbBfSTEdF4av4u3bYfbGdUR2T9Wphy2/X4bddJu+u4rXMT3HNDU0xetFft4pFCURHBiIoIdvrzIp1f1joDOxpcwHxA4hLizrd582YMGjTI4rUhQ4Zg8+bNNj+TkJCAoKAg07+IiAiti2nTF6O6at6zvaa/L54e2Br/6dfS6VocRz35rT3JupJoTe0T//7u6vzG3nZBGhplPQvyLR1C3bJ9e3PLfDS8i8OO5/Gdm9jtDyHSDdDbONNnTrSfo2YN+UOz5z/WCwufiNGwNOLz5ApLIQKW7OxsNG5sWUPRuHFjZGdn2/zM5MmTkZeXZ/qXkZGhdTGtatWoDoZ0CEUtf+2Hek28uY1FE447WAuO9LqBKO1pr9bs1J7o6ZtaIba1682IctkKmmT9Bh56BZUzo+9/bnTcR8zbmD8QSACa1rNsEjQ/JiqCaiWzzldWeVCAPTHXhSAwwLLpcc6orhbNzdX4siE8IQIWZxiNRgQGBlr804Or+UfcSbW5hKysqbmDGVxdXX+FimahLpWqrOtZG35ta/2e85PZZT5kM7aV+4ITJbxkV1s1zMow4Mo6hgdh//QhOJ5wq0s3Za24o7Zx+u22M30/EtsCX47uhiVP9nF6/a6ez4M7hOKD+7QZLCHHgLYN8ekD0bKWtdWfa0SlAQ/21DKbvPCpm1rhHkESm8ohRMASGhqKU6dOWbx26tQphIa6p0pbiWcHtUHLhvpN/qQluQ+6zcxS/P/5VCziOzXBV6O7aVOoShKfH4C9rw6ukh9mRI9I3NbZMpeL0guZnBuQqF6vPPeKwWD3+7cLdTz8Xk4Oi4o5XxwNQ38l3nbNoJJmiZo1fPHR8C6mv/UMAuSWurbRT7Pavoj6rn1/Z5qEGta1Mdz5KvOvari6/Iy7rKc88PP1wc3XN0ZIHfvrtMXRsH+5zPeCu4Psbx/qgds6uz65rVw3t792rt7UrhF6X+fcaDA9CBGwxMTEYNWqVRavrVy5EjEx4rU1ThjUGqvNJvvSM4mOHOZDCl29Zu6bPgTJU2+2GGrbMTwIs0beoGoNiz3+fj6oG1C1NiWghi8+fUDeUGhbu8FP0M7ML99q/WZvfhNUWtMX11GdRH0VAUPlYLFCRRGHdVOn/9H+6UNwR5dwrHt+AB7sFYl5j/VUZb2eatETfSwCOC19/0gP3NimId51InWD2vmlAmr4oH2TQPx3sONmOVe8dc+VQMtWwO3+48/6flRyba8695Mr5XEvxR0vLly4gCNHjpj+Pn78OJKTk1G/fn1ERkYiJycH6enpyMzMBACkpqYCuFKLUlFjMnr0aISHhyMhIQEAMGHCBPTr1w/vvfce4uPjsWDBAiQlJWHOnDkuf0Gt/PFkLH7ZkeFRScjqOEip7Oi4dfR5Vyh9Ai0ts37ixrZqgA1HzgKw/STorhla1aLFBUWtG0jFehz9fkE2hiwDQHcF0yX4XL3YRobUwht3qpeoUC93dAnD78mZTn++YV0j7ugSjgkLkp36vKMmIV+z37Vv64bo29r11AJKOsnacuC1WwBcOe6aBGlXy3Z/90jEdw6D0c8Hbyw9AODKKKrdGbkAgA5htnPMkPoUX7mTkpIQHR2N6OgrbW4TJ05EdHQ0pk6dCgBYsmQJoqOjER8fDwAYPnw4oqOjLYYnp6enIysry/R37969MW/ePMyZMwdRUVH45ZdfsHjxYnTs6J4pxp3RqWkQpt/REfUUpt53RMnFWw6DAXh3WBR6tKiv+dOIaqxcQxMqZdG11Wb71Zhu+E+/lhjUvhFeGNLO4r3X77iSV6JZSG0M7x6BsKAAfDu2uzpl1khggLzmBEetN3V1mv+jWci1mrfxA65V3x95Mw5fjOqKV+LbY5SDmXkrZu7t08p61fW9OjXlqRFHav1w66j5xlHgOmd0NzSo4+9yLY75jf2LUV1dWhdwJVCpOC+euqkVhnVt6jDdg7MZXis/qKmZwkKtaQgkCXhvWBRiWzXAdw/3AHBlvziqDbuuUR1F29E6fYcjiq9i/fv3t9u2PXbsWIwdO9buOhITE6u8NmzYMAwbNkxpcbzO+AGtEFzLHwPaOv8kY/TzQVFpuenve7s21e2i7hQrh1flACW+cxMs2d0YK/afwoSBrU2vB9TwtTqS6t6uTTEqprnp75n3dJZdnDmjuuLx73fIXr7C0wNb4+NVhxV/ztx93SPww5Z05R+sdI6+d1+UxXfQOhnbzik3o6i0zGbNip+vD4bIHHY9rt916N68vs2MqSN6ROCXHf86XVZXbJk8EL0SVjle0AatB0hte2kg+r2TiPSci059vmuzetj+8iCX++B0iQjGt2O7I6J+LbRSeJN0pLbRD+8Mi8L5iyX458CVvpAVTZXm96p1z9uexsL8fLD2Xf18DGjTuA4uXC5FRD1l2aUr+u/8+VQsbvtkg+n1qIhg/PZEb0Xrsueerk1N80EdeiPOVJP83MLdFsv1b9sQr9x2PYpKyqqMmLLm5VvbI6hmDTQMNKK/Ssk7naXftItkVUANXzwS28KldTQJCkDauSsXKFduTHfLnJ9HDSF1lNdUzX6wKzJyLsnqeChnP8jpiCpXm8Z1MPHmNi4HLEY/29XnjjrFRjUNNv3fR6OGaltNCkon/bTH18eAHi1s1zza2w392jTE2kNVkweqQYJlKns11fL3xcXiMpfXo0ZnX1fWYf7RAe0auVwWV8jNX2Xt2xoMBiyfcCPKJQl/p5yyskRV9Wv7Y9OkmxBwtQmsY6WA2+jnY2riVJutZu95j/ZEzHVXps2R28T/mEBD8z2rMZ/c5qvR3TDDhckM/X19ZPe9iGoaZDfSt7Ueg8GAyJBaLl+UlzzZBy/e0g4je1pvZnImmFJz9IpT385gQMfwICwcF4MNLw6A0Ynq8LYqBnB6qaged5UazRgkLjl9unx8DFU75tv5WLdm9UzBilpcrRnt3aqBR+enYg2LF1j6dCzyLpXg2JlCzPzrIN6/vwvu/myTS+scpHC25MomDGqNyyXynhCvD6vUjuvm86lz02B0NquNMDe2d3N0baZuvyJ3qugTFVapY6Kc6555/xM5fH0MKCuX4EGpiWRrZKUviJZfU6t1923dAPmXSrD73zyNtiDGqJOKZHXO3JwdfcRW0BAeXBP5l0rwzrAopGTl46HezRVvWw+Va0MXPN4Lw+ds0ak09rGGxQt0CAtC7+sa4MFezbBn2mDcoDAjbIVx/dTJaQBcaa5Qq5+Ente/p25q5dTn1HyKGdTe9Y5uPj4GNHAy14Vcv4yLQddm9fCrjXZ5T54SQaunUl8b69Wqb8uYmOb4/clYjdaur8fNmi7eVzpVigo7/NcnemPv9CG4pWMoJt7cRvUBGYDtYtrqkC5HvzYNLTL9RkcG4y43dgdQggGLl6ncJqrkOqtfE4D1QlYMf2wXqk8WY3NKJ5xU8/YWGVIL218eZHcZpcOUtbj9RkdeCVZsTaGgdi4OTzYprh3Cg2viORnp/V1lfg1Qu8Or1e1ZHF3uC1J7tKiPXVNudjmrsKPA2iJppXmSPJ3i8Vs7hSK+k/N5lQwGgyozc7sDAxYyCQuuiem3d8B7TiSG0sLvT/bBfd2aYs5o9/Uf+HpMNzxtNuqowrzHeqJ5iLLRAWpyNDxVKVFDh4oU7TcqGI3g6LtUzorsrK7NnKu5rGA+hHVcv+uwcdJNCHNzpl53JXjUS73a/k7VhtUy64DqqL9X7+tC8HCfFnhbwUjDCuaBgVGlfFBDOoR6dL8UJRiwkIUxvZubhsa5ytlzqOJjbRrXxdv3RqGpwmGErhjYvjHGxFTNC9KteX0k2hgWqdY8Po/1bYEFj/cypbsfqGBURVhwTYsLoLURT+a/R3m5+0MWOU1CnZsGY+eUmzFXxfw4D/dxbdQdcOWY/O7hHpj94LXgWWkCwpn3dMKYmGb4a0JfxdtX+wm4IohzNKWCM9S+d8bbyKKspjpGP8x9qDv+93APhx1lDQYDpg69Hvc5MXu8+YPHG5Wn07jqoT7NAQDDrazfvFn3qZtaIaZliMOs1RUpLVrbqV0zL5efjw9q+qvbWVgt7HRLuvP0h4PPHrwBnV/92+I1Z77Ty/HXAwB6tqiP7s3rO5yzatrQazexGr4+2D1tMA5k5SMlKx9xHe3nOCkTuIOR0iHRWueUAYC6AX6oY/TDLR1D8fodHVBaLiG4lrJyNqhjxPQ7bCfDrBvgh4LLpQCq7tYxvZtj2pL9Sott05r/9sfh0xfQvblrtUbu8MnwaEyJvx7j5+3EjhPnNdtO/7ZODLt24diz1am9z3UN8MygNggMqHp7rl/bH/Mf64Wa/r5VJoC15e17OuOeG5qiQ7jtpvU6Rj/8M7Efavga4OtjwMSb22DfyTzc70RQpiUGLF6urSBzHcV3boKle7IcL+ghvhjVFesPn8FDfVrISr6khMFgkNWfaEylUQgBNXwRHVnPZh8Sc2Vq1bCI2rakkrfv6YwzF4rQsuG1p1PzBIRq2PbyQOReLMGzPyVjf2a+zeX8fX1QXFZu830l6tX2t5vbxhXmwXqHyiMAneDjY0BoUICsyTj1pFbsbjDYn8oiRuFkhT4+BlmfMe/f1KCOEUsE7JzNgMVL/flULH7bddJqfww99GoZomvAoqTDp3mVsNFG9XBYUE27c9k0Vzgk2N0cBSwv3OIh0zhozJlqf6Ua1Q1Ao7qWSc3EvjXbZ/TzxbKn+2L1wVN4tK84ScdE4OGVybpjwOKlOoYHVcms6G5xHZvgk9VHHGairXwS691EVNvoh1lXZ352ZsLHUb2a4Zmb5U2K6e/ng+LScjRzc4deRwGLtRmxrRLgCqzGUzy5buLNbZBTWIwWVzv2VsmvRLLoff0TGTvdkmauDwvExkk3YeWz/VDPbKRG7+tCTBMRiiq+cxO7nf3sXVRev7Oj7EBnePcI/PhoTywZ797qVz061Wl1Ia5t9EPKa0NkLfvHk7EW82r9+kRvU5Kxyqx1eiTbnh7YGq9qODzWmZEwojcjWVNdRvw4gwELaSo8uCYCavgirmMTDO8egbfv7Yx5j/VSvR+AI+5KWlY5OZujWZJ9DAb0adUAQQqH3t7RJUxx2cyNu1G9JIFytdGwP1Utf8v9bDDAas1ep6ZBqG0WrHVtVs8iYGlgNg1Dr5bOJ+Nyhvl9Ss2jlbc/zzCiRwSimgapNurQG7FJiDQRVakHu6+PweYMyRUX6opOhZ5wwlZ+cGvfJBAHsvJxe5RlILHo/3rji3XHbM4m7MzD1Nv3dsZ93ZQ//TcJCsCZgiIAsAiQpg29HtP/SFFeEABNFeQRua1zE5y/WCx7dIOz/HwM2Dd9CPyuJlHs27oB3l95yPS3PU8PbI2pv7s2GmfNc/1VnfxRjucGt8G7fx+yeM1WzZGncmaiSa1rK/x8r62/hozkkpWvi+YS7lae16W6YcBCqkp8rj8OnSpA39bKpyFf83x/bDxyFnd2UT8ttNZZVuc92hMbj56tkka/deO6eHdYlEXAMqJHJOZvS3d6W6EyZ52t7JMR0Zjy+348UWkKBkkCFo6Lweaj5/D+ykM2Pm3p1ydicCq/CK0V1JoYDAaMdlPNmnnH6ejIevjzqVhVJ6S0p4UOydmevKm1RcAyvHsEnujv/lo0Lb06tAPKyiSMsDFJqR5qG/3w8q3tUSZJslLxhwXXROJz/e2OAiLbGLCQqpo3qO10Ns3w4JpO1RyIoF5tf9zWWXkzjZL5fWY/2BUHsvLRt7VzNVDNQmrjfzZmL+7e/EruF7kBi6dNCKl3B3SlXAmvY1s1sKjNVHvGYL00rGvEbAFnzX7sRmUjobw927CW2IeFdOeJk+IprWl+wMZToZIsrLd0DMWzN7dhpzwHPHX3uDo55c//icGtnULxzjDLpoWPhkejZcPa+GREtEvrJ9IbAxbSnafeYJToZmMeGlFTYIcHO9fsJAKlA0NcDZjVmmNqxl2dENuqAb4Z282pEvVoUR+fjeyKJkGWTV9tQ+ti9X/7Y2iUax21ifTGgIV08/TA1ggNDMCTA1ppvq1m9dWthlV6U7yzSzheiW+PxeP7qFoOtf3v4R544Za2GOBMmnIPUcvOyC3zDsFy+iQAwIpnb3S1SACu9G/44dGeuKmd9fl9Brb33t+ESA72YSHdTLy5DZ4d1FrTJo4Fj/fCypRT+E8/fTNu+vgYTFk/f9qeoWtZKgsxG8p7Y5uGimZK9kTj+l2H7cdzTEPDzTtkd24ajI+Gd0FKVj5ulNlXyOjnnlqyt+7tjJ4t6uNVJ0d0EXk6BiykK637Y/RqGeL2fBqe4uMR0UhKy3Gqs7AnC6pZA7880dvm+3d0CccdCkeq9WpZH1uO5aClBh0qK0Y3BQbUwNg+LRiwULXFgIXIzQa1b4T529IdJpXT2u1RYVXyxpBzPn3gBszbmo5h3Zo6XlihRf9nO7giqk4YsBC52U3tGuGXcTEWMwCTftQYpdagjlGziUYbO5l3h8jbMGAhcjODwYBuzT0rjwmRJwqowXEl3oS/JhEReaVPH7gBLRrUxqcPMAeNN2ANCxF5FTahUIX2TQKx5rn+eheDVMIaFiLyCr+Mi0Fsqwb49qHuehfFZUzfTlQVa1iIFLixTUOcyruM9k3kT/pH7tGteX388GhPvYuhis9HdsVbyw/icYXz1BB5MwYsRAp891B3SNKVRHDkHUScGiIypBZmjbxB72IQCYVNQkQKGAwGBiukq4+GdwEATL+9g74FIXIz1rAQEXmQO7qEY/D1ocJOnEmkFdawEBF5GAYrVB0xYCGiao1zTRF5BjYJEVG19p9+LVG/tj9ubO3ds1QTeToGLERUrRn9fPFgr2Z6F4OIHGCTEBEREQmPAQsREREJjwELERERCY8BCxEREQmPAQsREREJjwELERERCY8BCxEREQmPAQsREREJjwELERERCY8BCxEREQmPAQsREREJjwELERERCY8BCxEREQnPa2ZrliQJAJCfn69zSYiIiEiuivt2xX3cFq8JWAoKCgAAEREROpeEiIiIlCooKEBQUJDN9w2So5DGQ5SXlyMzMxN169aFwWBQbb35+fmIiIhARkYGAgMDVVtvdcR9qQ7uR/VwX6qD+1E91XFfSpKEgoIChIWFwcfHdk8Vr6lh8fHxQdOmTTVbf2BgYLU5eLTGfakO7kf1cF+qg/tRPdVtX9qrWanATrdEREQkPAYsREREJDwGLA4YjUZMmzYNRqNR76J4PO5LdXA/qof7Uh3cj+rhvrTNazrdEhERkfdiDQsREREJjwELERERCY8BCxEREQmPAQsREREJjwELERERCY8BiwOzZs1C8+bNERAQgJ49e2Lbtm16F0k3CQkJ6N69O+rWrYtGjRrhzjvvRGpqqsUyly9fxvjx4xESEoI6dergnnvuwalTpyyWSU9PR3x8PGrVqoVGjRrh+eefR2lpqcUyiYmJuOGGG2A0GtGqVSvMnTtX66+nm5kzZ8JgMOCZZ54xvcb9KN/Jkyfx4IMPIiQkBDVr1kSnTp2QlJRkel+SJEydOhVNmjRBzZo1MWjQIBw+fNhiHTk5ORg5ciQCAwMRHByMRx55BBcuXLBYZs+ePejbty8CAgIQERGBt99+2y3fzx3KysowZcoUtGjRAjVr1sR1112H119/3WIyOu5H69atW4ehQ4ciLCwMBoMBixcvtnjfnftt4cKFaNeuHQICAtCpUycsW7ZM9e+rK4lsWrBggeTv7y9988030v79+6XHHntMCg4Olk6dOqV30XQxZMgQ6dtvv5X27dsnJScnS7feeqsUGRkpXbhwwbTMuHHjpIiICGnVqlVSUlKS1KtXL6l3796m90tLS6WOHTtKgwYNknbt2iUtW7ZMatCggTR58mTTMseOHZNq1aolTZw4UUpJSZE++eQTydfXV1q+fLlbv687bNu2TWrevLnUuXNnacKECabXuR/lycnJkZo1ayaNHTtW2rp1q3Ts2DFpxYoV0pEjR0zLzJw5UwoKCpIWL14s7d69W7r99tulFi1aSJcuXTItc8stt0hRUVHSli1bpPXr10utWrWSRowYYXo/Ly9Paty4sTRy5Ehp37590vz586WaNWtKX3zxhVu/r1befPNNKSQkRPrzzz+l48ePSwsXLpTq1KkjffTRR6ZluB+tW7ZsmfTyyy9LixYtkgBIv/32m8X77tpvGzdulHx9faW3335bSklJkV555RWpRo0a0t69ezXfB+7CgMWOHj16SOPHjzf9XVZWJoWFhUkJCQk6lkocp0+flgBIa9eulSRJknJzc6UaNWpICxcuNC1z4MABCYC0efNmSZKunNw+Pj5Sdna2aZnPP/9cCgwMlIqKiiRJkqQXXnhB6tChg8W27r//fmnIkCFafyW3KigokFq3bi2tXLlS6tevnylg4X6U78UXX5RiY2Ntvl9eXi6FhoZK77zzjum13NxcyWg0SvPnz5ckSZJSUlIkANL27dtNy/z111+SwWCQTp48KUmSJH322WdSvXr1TPu2Yttt27ZV+yvpIj4+Xnr44YctXrv77rulkSNHSpLE/ShX5YDFnfvtvvvuk+Lj4y3K07NnT+k///mPqt9RT2wSsqG4uBg7duzAoEGDTK/5+Phg0KBB2Lx5s44lE0deXh4AoH79+gCAHTt2oKSkxGKftWvXDpGRkaZ9tnnzZnTq1AmNGzc2LTNkyBDk5+dj//79pmXM11GxjLft9/HjxyM+Pr7Kd+V+lG/JkiXo1q0bhg0bhkaNGiE6Ohpffvml6f3jx48jOzvbYj8EBQWhZ8+eFvsyODgY3bp1My0zaNAg+Pj4YOvWraZlbrzxRvj7+5uWGTJkCFJTU3H+/Hmtv6bmevfujVWrVuHQoUMAgN27d2PDhg2Ii4sDwP3oLHfut+pwvjNgseHs2bMoKyuzuCEAQOPGjZGdna1TqcRRXl6OZ555Bn369EHHjh0BANnZ2fD390dwcLDFsub7LDs72+o+rXjP3jL5+fm4dOmSFl/H7RYsWICdO3ciISGhynvcj/IdO3YMn3/+OVq3bo0VK1bgiSeewNNPP43vvvsOwLV9Ye88zs7ORqNGjSze9/PzQ/369RXtb082adIkDB8+HO3atUONGjUQHR2NZ555BiNHjgTA/egsd+43W8t4037107sA5JnGjx+Pffv2YcOGDXoXxeNkZGRgwoQJWLlyJQICAvQujkcrLy9Ht27dMGPGDABAdHQ09u3bh9mzZ2PMmDE6l85z/Pzzz/jxxx8xb948dOjQAcnJyXjmmWcQFhbG/UjCYA2LDQ0aNICvr2+VkRmnTp1CaGioTqUSw5NPPok///wTa9asQdOmTU2vh4aGori4GLm5uRbLm++z0NBQq/u04j17ywQGBqJmzZpqfx2327FjB06fPo0bbrgBfn5+8PPzw9q1a/Hxxx/Dz88PjRs35n6UqUmTJrj++ustXmvfvj3S09MBXNsX9s7j0NBQnD592uL90tJS5OTkKNrfnuz555831bJ06tQJo0aNwrPPPmuqAeR+dI4795utZbxpvzJgscHf3x9du3bFqlWrTK+Vl5dj1apViImJ0bFk+pEkCU8++SR+++03rF69Gi1atLB4v2vXrqhRo4bFPktNTUV6erppn8XExGDv3r0WJ+jKlSsRGBhouvHExMRYrKNiGW/Z7wMHDsTevXuRnJxs+tetWzeMHDnS9H/uR3n69OlTZWj9oUOH0KxZMwBAixYtEBoaarEf8vPzsXXrVot9mZubix07dpiWWb16NcrLy9GzZ0/TMuvWrUNJSYlpmZUrV6Jt27aoV6+eZt/PXS5evAgfH8vbga+vL8rLywFwPzrLnfutOpzvHCVkx4IFCySj0SjNnTtXSklJkR5//HEpODjYYmRGdfLEE09IQUFBUmJiopSVlWX6d/HiRdMy48aNkyIjI6XVq1dLSUlJUkxMjBQTE2N6v2I47uDBg6Xk5GRp+fLlUsOGDa0Ox33++eelAwcOSLNmzfK64biVmY8SkiTuR7m2bdsm+fn5SW+++aZ0+PBh6ccff5Rq1aol/fDDD6ZlZs6cKQUHB0u///67tGfPHumOO+6wOqw0Ojpa2rp1q7RhwwapdevWFsNKc3NzpcaNG0ujRo2S9u3bJy1YsECqVauWRw/HNTdmzBgpPDzcNKx50aJFUoMGDaQXXnjBtAz3o3UFBQXSrl27pF27dkkApPfff1/atWuXdOLECUmS3LffNm7cKPn5+UnvvvuudODAAWnatGkc1lzdfPLJJ1JkZKTk7+8v9ejRQ9qyZYveRdINAKv/vv32W9Myly5dkv7v//5PqlevnlSrVi3prrvukrKysizWk5aWJsXFxUk1a9aUGjRoIP33v/+VSkpKLJZZs2aN1KVLF8nf319q2bKlxTa8UeWAhftRvj/++EPq2LGjZDQapXbt2klz5syxeL+8vFyaMmWK1LhxY8loNEoDBw6UUlNTLZY5d+6cNGLECKlOnTpSYGCg9NBDD0kFBQUWy+zevVuKjY2VjEajFB4eLs2cOVPz7+Yu+fn50oQJE6TIyEgpICBAatmypfTyyy9bDKPlfrRuzZo1Vq+LY8aMkSTJvfvt559/ltq0aSP5+/tLHTp0kJYuXarZ99aDQZLMUhkSERERCYh9WIiIiEh4DFiIiIhIeAxYiIiISHgMWIiIiEh4DFiIiIhIeAxYiIiISHgMWIiIiEh4DFiIiIhIeAxYiIiISHgMWIiIiEh4DFiIiIhIeP8PhG6IoJnnRzoAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "plt.plot(train_losses_bkp)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "change_lora_alpha(model, 0.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
      "Setting `pad_token_id` to `eos_token_id`:None for open-end generation.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<|begin_of_text|>Mischief managed | tags: mischief']<|eot_id|>\n"
     ]
    }
   ],
   "source": [
    "s = tokenizer(\"Mischief managed | tags:\", return_tensors='pt')['input_ids']\n",
    "a = model.generate(s, max_new_tokens=200)\n",
    "print(\"\".join([tokenizer.decode(token) for token in a[0]]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
      "Setting `pad_token_id` to `eos_token_id`:None for open-end generation.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<|begin_of_text|>Don't cry because its over, smile because it happened | tags:', 'humor', 'smile']<|eot_id|>\n"
     ]
    }
   ],
   "source": [
    "s = tokenizer(\"Don't cry because its over, smile because it happened | tags:\", return_tensors='pt')['input_ids']\n",
    "a = model.generate(s, max_new_tokens=200)\n",
    "print(\"\".join([tokenizer.decode(token) for token in a[0]]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
      "Setting `pad_token_id` to `eos_token_id`:None for open-end generation.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<|begin_of_text|>row, row, row, your boat. | tags::humor', 'rows', 'row', 'rows', 'rows', 'rows', '\n"
     ]
    }
   ],
   "source": [
    "s = tokenizer(\"row, row, row, your boat. | tags:\", return_tensors='pt')['input_ids']\n",
    "a = model.generate(s, max_new_tokens=20)\n",
    "print(\"\".join([tokenizer.decode(token) for token in a[0]]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "change_lora_alpha(model, 0.)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
      "Setting `pad_token_id` to `eos_token_id`:None for open-end generation.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<|begin_of_text|>Mischief managed | tags: fiction, fantasy, character development, relationships\n",
      "\n",
      "It's been a while since I've had a chance\n"
     ]
    }
   ],
   "source": [
    "s = tokenizer(\"Mischief managed | tags:\", return_tensors='pt')['input_ids']\n",
    "a = model.generate(s, max_new_tokens=20)\n",
    "print(\"\".join([tokenizer.decode(token) for token in a[0]]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
      "Setting `pad_token_id` to `eos_token_id`:None for open-end generation.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<|begin_of_text|>Don't cry because its over, smile because it happened | tags: #relaxation, #selfcare, #mentalhealthmatters, #mindfulness, #\n"
     ]
    }
   ],
   "source": [
    "s = tokenizer(\"Don't cry because its over, smile because it happened | tags:\", return_tensors='pt')['input_ids']\n",
    "a = model.generate(s, max_new_tokens=20)\n",
    "print(\"\".join([tokenizer.decode(token) for token in a[0]]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
      "Setting `pad_token_id` to `eos_token_id`:None for open-end generation.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<|begin_of_text|>row row row your boat | tags: poetry, boat, rowing, rowing poem\n",
      "\n",
      "The classic nursery rhyme \"Row, Row,\n"
     ]
    }
   ],
   "source": [
    "s = tokenizer(\"row row row your boat | tags:\", return_tensors='pt')['input_ids']\n",
    "a = model.generate(s, max_new_tokens=20)\n",
    "print(\"\".join([tokenizer.decode(token) for token in a[0]]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "with open(\"lora_model.pkl\", 'wb') as f:\n",
    "    pickle.dump(model, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"lora_model.pkl\", 'rb') as f:\n",
    "    tmp_model = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "change_lora_alpha(tmp_model, 0.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
      "Setting `pad_token_id` to `eos_token_id`:None for open-end generation.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<|begin_of_text|>Thank God it's Friday | tags: friday', 'god', 'humor', 'weekends']<|eot_id|>\n"
     ]
    }
   ],
   "source": [
    "s = tokenizer(\"Thank God it's Friday | tags:\", return_tensors='pt')['input_ids']\n",
    "a = tmp_model.generate(s, max_new_tokens=20)\n",
    "print(\"\".join([tokenizer.decode(token) for token in a[0]]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "del tmp_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "llm",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.3"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}