LSTM/LSTM.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# data는 e9t(Lucy Park)님께서 github에 공유해주신 네이버 영화평점 데이터를 사용하였습니다.\n",
    "# https://github.com/e9t/nsmc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import defaultdict\n",
    "\n",
    "import torch.nn as nn\n",
    "import torch\n",
    "from torch.autograd import Variable\n",
    "import torch.nn.functional as F\n",
    "import random\n",
    "import numpy as np\n",
    "\n",
    "# data를 읽어옴\n",
    "def read_txt(path_to_file):\n",
    "    txt_ls = []\n",
    "    label_ls = []\n",
    "\n",
    "    with open(path_to_file) as f:\n",
    "        for i, line in enumerate(f.readlines()[1:]):\n",
    "            id_num, txt, label = line.split('\\t')\n",
    "            txt_ls.append(txt)\n",
    "            label_ls.append(int(label.replace('\\n','')))\n",
    "    return txt_ls, label_ls"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 불러오기"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import Counter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train, y_train = read_txt('../ratings_train.txt')\n",
    "x_test, y_test = read_txt('../ratings_test.txt')\n",
    "\n",
    "x_train = [x.split() for x in x_train]\n",
    "x_test = [x.split() for x in x_test]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['아', '더빙..', '진짜', '짜증나네요', '목소리']"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_train[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Counter({0: 75173, 1: 74827})"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Counter(y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 비어있는 리뷰 제거"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_empty_review(X, Y):\n",
    "    empty_idx_ls = []\n",
    "    \n",
    "    for idx, review in enumerate(X):\n",
    "        if len(review) == 0:\n",
    "            empty_idx_ls.append(idx)\n",
    "    \n",
    "    # idx 값이 큰 것부터 제거 (앞으로 밀리는 것을 방지)\n",
    "    empty_idx_ls = sorted(empty_idx_ls, reverse = True)\n",
    "    \n",
    "    for empty_idx in empty_idx_ls:\n",
    "        del X[empty_idx], Y[empty_idx]\n",
    "    \n",
    "    return X, Y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train, y_train = remove_empty_review(x_train, y_train)\n",
    "x_test, y_test = remove_empty_review(x_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['아', '더빙..', '진짜', '짜증나네요', '목소리']"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_train[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Add Padding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Sequence Length를 맞추기 위한 padding\n",
    "def add_padding(token_ls, max_len):\n",
    "    pad = '<PAD>'\n",
    "    seq_length_ls = []\n",
    "    \n",
    "    for i, tokens in enumerate(token_ls):\n",
    "        seq_length = len(tokens)\n",
    "        \n",
    "        # 짧으면 padding을 추가\n",
    "        if seq_length < max_len:\n",
    "            seq_length_ls.append(seq_length)\n",
    "            token_ls[i] += [pad] * (max_len - seq_length)\n",
    "        \n",
    "        # 길이가 길면, max_len까지의 token만 사용\n",
    "        elif seq_length >= max_len:\n",
    "            seq_length_ls.append(max_len)\n",
    "            token_ls[i] = tokens[:max_len]\n",
    "            \n",
    "    return token_ls, seq_length_ls\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "max_sequence_length = 30\n",
    "x_train, x_train_seq_length = add_padding(x_train, max_sequence_length)\n",
    "x_test, x_test_seq_length = add_padding(x_test, max_sequence_length)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Converting token to index "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 단어에 대한 idx 부여\n",
    "def convert_token_to_idx(token_ls):\n",
    "    for tokens in token_ls:\n",
    "        yield [token2idx[token] for token in tokens]\n",
    "    return"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "token2idx = defaultdict(lambda: len(token2idx))\n",
    "pad = token2idx['<PAD>']\n",
    "\n",
    "x_train = list(convert_token_to_idx(x_train))\n",
    "x_test = list(convert_token_to_idx(x_test))\n",
    "\n",
    "idx2token = {val : key for key,val in token2idx.items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'아 더빙.. 진짜 짜증나네요 목소리 <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD>'"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "' '.join([idx2token[x] for x in x_train[0]])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Packing을 위해, 긴 문장이 위로 오도록 순서를 sorting and Variable로 변환"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sort_by_sequence_length(x, y, seq_len):\n",
    "    sorted_idx = np.argsort(np.array(seq_len))[::-1]\n",
    "    \n",
    "    x = Variable(torch.LongTensor(np.array(x)[sorted_idx]))\n",
    "    y = Variable(torch.LongTensor(np.array(y)[sorted_idx]))\n",
    "    seq_len = Variable(torch.LongTensor(np.array(seq_len)[sorted_idx]))\n",
    "    \n",
    "    return x, y, seq_len"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train, y_train, x_train_seq_length = sort_by_sequence_length(x_train, y_train, x_train_seq_length)\n",
    "x_test, y_test, x_test_seq_length = sort_by_sequence_length(x_test, y_test, x_test_seq_length)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# LSTM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "class LSTM(nn.Module):\n",
    "    def __init__(self, token2idx, max_sequence, vocab_size, embed_size, hid_size, n_layers, dropout, bidirectional, n_category):\n",
    "        super(LSTM, self).__init__() \n",
    "        self.vocab_size = vocab_size                # 고유한 단어의 수\n",
    "        self.embed_size = embed_size                # 임베딩 차원의 크기\n",
    "        self.padding_index = token2idx['<PAD>']     # 패딩 토큰\n",
    "        \n",
    "        self.embed = nn.Embedding(\n",
    "            num_embeddings=vocab_size, \n",
    "            embedding_dim=embed_size, \n",
    "            padding_idx=self.padding_index\n",
    "        )\n",
    "        \n",
    "        self.max_sequence = max_sequence           # 한 문장의 최대 길이\n",
    "        self.hid_size = hid_size                   # LSTM 뉴런의 갯수\n",
    "        self.n_layers = n_layers                   # LSTM layer의 수\n",
    "        self.drouput = dropout                     # 드롭아웃\n",
    "        self.n_category = n_category               # 분류 카테고리 갯수\n",
    "        self.bidirectional = bidirectional         # optinal, True : bidirectional \n",
    "        \n",
    "        self.lstm = nn.LSTM(embed_size, hid_size, n_layers, batch_first=True, bidirectional=True)\n",
    "        \n",
    "        if bidirectional:\n",
    "            input_dim = 2*hid_size*max_sequence\n",
    "        else:\n",
    "            input_dim = hid_size*max_sequence\n",
    "\n",
    "        self.lin = nn.Linear(input_dim, n_category)\n",
    "        self.outputs = []\n",
    "        \n",
    "    def init_hidden(self, batch_size):\n",
    "        # 최초에 h_0와 c_0의 초기값 부여 (n_layers, batch_size, hid_size(n_neuron))\n",
    "        # bidirectional일 경우, (2*n_layers, batch_size, hid_size)\n",
    "        if self.bidirectional:\n",
    "            h_0 = Variable(torch.randn(2*self.n_layers, batch_size, self.hid_size)) * 0.1\n",
    "            c_0 = Variable(torch.randn(2*self.n_layers, batch_size, self.hid_size)) * 0.1\n",
    "        else:\n",
    "            h_0 = Variable(torch.randn(self.n_layers, batch_size, self.hid_size)) * 0.1\n",
    "            c_0 = Variable(torch.randn(self.n_layers, batch_size, self.hid_size)) * 0.1\n",
    "        return (h_0, c_0)    \n",
    "    \n",
    "    def forward(self, x, x_sequence_length):\n",
    "        # init h randomly\n",
    "        batch_size = x.size(0)\n",
    "        self.h_c = self.init_hidden(batch_size)\n",
    "        \n",
    "        # embedding\n",
    "        x = self.embed(x) # sequence_length(max_len), batch_size, embed_size\n",
    "        \n",
    "        # packing for LSTM\n",
    "        x = torch.nn.utils.rnn.pack_padded_sequence(x, x_sequence_length, batch_first=True)\n",
    "        \n",
    "        # LSTM\n",
    "        output, self.h_c = self.lstm(x, self.h_c)\n",
    "        \n",
    "        # unpack\n",
    "        # unpacking 과정에서, size(1) S가 가장 긴 sequence 길이를 갖는 data에 맞춰진다.\n",
    "        # S가 max_sequence_length로 고정되는 것이 아님을 주의\n",
    "        x, _ = torch.nn.utils.rnn.pad_packed_sequence(output, batch_first=True)\n",
    "        #print(x.shape)\n",
    "        \n",
    "        # flatten\n",
    "        x = x.contiguous()\n",
    "        x = x.view(batch_size, -1)\n",
    "        \n",
    "        # fully-connect\n",
    "        logit = self.lin(x)\n",
    "        \n",
    "        return logit\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "params = {\n",
    "    'token2idx' : token2idx,\n",
    "    'max_sequence' : 30,\n",
    "    'vocab_size' : len(token2idx),\n",
    "    'embed_size' : 32,\n",
    "    'hid_size' : 32,\n",
    "    'n_layers' : 2,\n",
    "    'dropout' : 0.5,\n",
    "    'bidirectional' : True,\n",
    "    'n_category' : 2,\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = LSTM(**params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LSTM(\n",
       "  (embed): Embedding(448188, 32, padding_idx=0)\n",
       "  (lstm): LSTM(32, 32, num_layers=8, batch_first=True, bidirectional=True)\n",
       "  (lin): Sequential(\n",
       "    (0): Linear(in_features=1920, out_features=2, bias=True)\n",
       "    (1): Tanh()\n",
       "    (2): Dropout(p=0.5)\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "def adjust_learning_rate(optimizer, epoch, init_lr=0.001, lr_decay_epoch=10):\n",
    "    \"\"\"Decay learning rate by a factor of 0.1 every lr_decay_epoch epochs.\"\"\"\n",
    "    lr = init_lr * (0.1**(epoch // lr_decay_epoch))\n",
    "\n",
    "    if epoch % lr_decay_epoch == 0:\n",
    "        print('LR is set to %s'%(lr))\n",
    "\n",
    "    for param_group in optimizer.param_groups:\n",
    "        param_group['lr'] = lr\n",
    "\n",
    "    return optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "epochs = 50\n",
    "lr = 0.01\n",
    "batch_size = 10000\n",
    "\n",
    "train_idx = np.arange(x_train.size(0))\n",
    "test_idx = np.arange(x_test.size(0))\n",
    "optimizer = torch.optim.Adam(model.parameters(),lr)\n",
    "criterion = nn.CrossEntropyLoss(reduction='sum')\n",
    "\n",
    "loss_ls = []\n",
    "\n",
    "for epoch in range(1, epochs+1):\n",
    "    model.train()\n",
    "    \n",
    "    # input 데이터 순서 섞기\n",
    "    random.shuffle(train_idx)\n",
    "    x_train, y_train = x_train[train_idx], y_train[train_idx]\n",
    "    x_train_seq_length = x_train_seq_length[train_idx]\n",
    "    \n",
    "    train_loss = 0\n",
    "\n",
    "    for start_idx, end_idx in zip(range(0, x_train.size(0), batch_size),\n",
    "                                  range(batch_size, x_train.size(0)+1, batch_size)):\n",
    "        # batch 뽑기\n",
    "        x_batch = x_train[start_idx : end_idx]\n",
    "        y_batch = y_train[start_idx : end_idx].long()\n",
    "        x_batch_seq_length = x_train_seq_length[start_idx: end_idx]\n",
    "        \n",
    "        # sequence 순서대로 정렬하기\n",
    "        x_batch, y_batch, x_batch_seq_length = sort_by_sequence_length(x_batch, y_batch, x_batch_seq_length)\n",
    "        \n",
    "        scores = model(x_batch, x_batch_seq_length)\n",
    "        predict = F.softmax(scores, dim=1).argmax(dim = 1)\n",
    "        \n",
    "        acc = (predict == y_batch).sum().item() / batch_size\n",
    "        \n",
    "        loss = criterion(scores, y_batch)\n",
    "        train_loss += loss.item()\n",
    "\n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        \n",
    "    print('Train epoch : %s,  loss : %s,  accuracy :%.3f'%(epoch, train_loss / batch_size, acc))\n",
    "    print('=================================================================================================')\n",
    "    \n",
    "    loss_ls.append(train_loss)\n",
    "    optimizer = adjust_learning_rate(optimizer, epoch, lr, 10) # adjust learning_rate while training\n",
    "    \n",
    "    if (epoch) % 5 == 0:\n",
    "        model.eval()\n",
    "        scores = model(x_test, x_test_seq_length)\n",
    "        predict = F.softmax(scores, dim=1).argmax(dim = 1)\n",
    "        \n",
    "        acc = (predict == y_test.long()).sum().item() / y_test.size(0)\n",
    "        loss = criterion(scores, y_test.long())\n",
    "        \n",
    "        print('*************************************************************************************************')\n",
    "        print('*************************************************************************************************')\n",
    "        print('Test Epoch : %s, Test Loss : %.03f , Test Accuracy : %.03f'%(epoch, loss.item()/y_test.size(0), acc))\n",
    "        print('*************************************************************************************************')\n",
    "        print('*************************************************************************************************')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "fininsight_python_3.5",
   "language": "python",
   "name": "fininsight_python3.5"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}