notebook.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<torch._C.Generator at 0x7fc60012af00>"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch\n",
    "from torch.autograd import Variable\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "import json\n",
    "import pickle\n",
    "import random\n",
    "import time\n",
    "import math\n",
    "import numpy as np\n",
    "from konlpy.tag import Mecab;tagger=Mecab()\n",
    "from collections import Counter\n",
    "from torch.nn.utils.rnn import pad_packed_sequence, pack_padded_sequence\n",
    "import numpy as np\n",
    "torch.manual_seed(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "USE_CUDA = torch.cuda.is_available()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def prepare_sequence(seq, to_ix):\n",
    "    idxs = list(map(lambda w: to_ix[w] if w in to_ix.keys() else to_ix[\"<UNK>\"], seq))\n",
    "    tensor = Variable(torch.LongTensor(idxs)).cuda() if USE_CUDA else Variable(torch.LongTensor(idxs))\n",
    "    return tensor"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* 트레이닝 : 286,817\n",
    "* 벨리데이션 : 13,368\n",
    "* 테스트 : 11,487\n",
    "* 인풋 : 800토큰, 아웃풋 100 토큰\n",
    "* 인풋 : 헤드라인, 바이라인(필자 등), 본문을 스페셜 토큰으로 조인"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "raw = open('insight_social.txt','r',encoding='utf-8').readlines()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data=[]\n",
    "temp=[]\n",
    "for r in raw:\n",
    "    if r !='\\n':\n",
    "        temp.append(r[:-1])\n",
    "    else:\n",
    "        data.append(temp)\n",
    "        temp=[]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_set=[]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for i, d in enumerate(data):\n",
    "    try:\n",
    "        temp = []\n",
    "        temp.append(d[0].split(\"제목 : \")[1])\n",
    "        temp.append(d[2].split(\"본문 : \")[1].replace(\"\\'\",\"\"))\n",
    "        input_seq = \"ssSEPARATEss\".join(temp)\n",
    "        train_set.append((input_seq, d[1].split(\"요약 : \")[1]))\n",
    "    except:\n",
    "        pass\n",
    "        #print(i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_set = [[tagger.morphs(t[0]),tagger.morphs(t[1])] for t in train_set if len(tagger.morphs(t[0]))<500]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "flatten = lambda l: [item for sublist in l for item in sublist]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X,y = zip(*train_set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "input2index = {\"<PAD>\":0,\"<SOS>\":1,\"<EOS>\":2,\"<UNK>\":3}\n",
    "output2index = {\"<PAD>\":0,\"<SOS>\":1,\"<EOS>\":2,\"<UNK>\":3}\n",
    "\n",
    "for token in flatten(X): \n",
    "    if token not in input2index.keys():\n",
    "        input2index[token]=len(input2index)\n",
    "\n",
    "index2input = {v:k for k,v in input2index.items()}\n",
    "\n",
    "for token in flatten(y): \n",
    "    if token not in output2index.keys():\n",
    "        output2index[token]=len(output2index)\n",
    "\n",
    "index2output = {v:k for k,v in output2index.items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "18007\n",
      "6315\n"
     ]
    }
   ],
   "source": [
    "print(len(input2index))\n",
    "print(len(output2index))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_p=[]\n",
    "y_p=[]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "for i in range(len(X)):\n",
    "\n",
    "    if len(X[i])<300:\n",
    "        num_pad = 300-len(X[i])\n",
    "        temp_x = X[i]+[\"<PAD>\"]*num_pad\n",
    "    else:\n",
    "        temp_x  = X[i][:300]\n",
    "    X_p.append(prepare_sequence(temp_x,input2index).view(1,-1))\n",
    "\n",
    "    if len(y[i])<50:\n",
    "        num_pad = 50-len(y[i])\n",
    "        temp_y = y[i] +[\"<PAD>\"]*num_pad\n",
    "    else:\n",
    "        temp_y = y[i][:50]\n",
    "    y_p.append(prepare_sequence(temp_y,output2index).view(1,-1))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_data = list(zip(X_p,y_p))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import random"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def getBatch(batch_size,train_data):\n",
    "    random.shuffle(train_data)\n",
    "    sindex=0\n",
    "    eindex=batch_size\n",
    "    while eindex < len(train_data):\n",
    "        batch = train_data[sindex:eindex]\n",
    "        temp = eindex\n",
    "        eindex = eindex+batch_size\n",
    "        sindex = temp\n",
    "\n",
    "        yield batch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class EncoderRNN(nn.Module):\n",
    "    def __init__(self, input_size,embedding_size, hidden_size ,n_layers=1):\n",
    "        super(EncoderRNN, self).__init__()\n",
    "        \n",
    "        self.input_size = input_size\n",
    "        self.embedding_size = embedding_size\n",
    "        self.hidden_size = hidden_size\n",
    "        self.n_layers = n_layers\n",
    "        \n",
    "        self.embedding = nn.Embedding(input_size, embedding_size)\n",
    "        self.lstm = nn.LSTM(embedding_size, hidden_size, n_layers, batch_first=True,bidirectional=True)\n",
    "    \n",
    "    def init_hidden(self,input):\n",
    "        hidden = Variable(torch.randn(self.n_layers*2, input.size(0), self.hidden_size)).cuda() if USE_CUDA else Variable(torch.randn(self.n_layers*2, input.size(0), self.hidden_size))\n",
    "        context = Variable(torch.randn(self.n_layers*2, input.size(0), self.hidden_size)).cuda() if USE_CUDA else Variable(torch.randn(self.n_layers*2, input.size(0), self.hidden_size))\n",
    "        return (hidden,context)\n",
    "    \n",
    "        \n",
    "    def forward(self, input,input_masking):\n",
    "        \"\"\"\n",
    "        input : B,T (LongTensor)\n",
    "        input_masking : B,T (PAD 마스킹한 ByteTensor)\n",
    "        \n",
    "        <PAD> 제외한 리얼 Context를 다시 만들어서 아웃풋으로\n",
    "        \"\"\"\n",
    "        \n",
    "        self.hidden = self.init_hidden(input)\n",
    "        embedded = self.embedding(input)\n",
    "        output, self.hidden = self.lstm(embedded, self.hidden)\n",
    "        \n",
    "        real_context=[]\n",
    "        \n",
    "        for i,o in enumerate(output): # B,T,D\n",
    "            real_length = input_masking[i].data.tolist().count(0) # 실제 길이\n",
    "            real_context.append(o[real_length-1])\n",
    "            \n",
    "        return output, torch.cat(real_context).view(input.size(0),-1).unsqueeze(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class DecoderRNN(nn.Module):\n",
    "    \n",
    "    def __init__(self,output_size,embedding_size,hidden_size,max_len=50,n_layers=1,dropout_p=0.1):\n",
    "        super(DecoderRNN, self).__init__()\n",
    "        \n",
    "        self.hidden_size = hidden_size\n",
    "        self.output_size = output_size\n",
    "        self.n_layers = n_layers\n",
    "        self.dropout_p = dropout_p\n",
    "        self.max_len=max_len\n",
    "        self.embedding_size = embedding_size\n",
    "        self.attn_e = nn.Linear(self.hidden_size,self.hidden_size) # encoder Attention\n",
    "        self.attn_d = nn.Linear(self.hidden_size,self.hidden_size) # decoder Attention\n",
    "        self.sigmoid = nn.Sigmoid()\n",
    "        self.temp_attn_scores=[]\n",
    "\n",
    "        # Define the layers\n",
    "        self.embedding = nn.Embedding(self.output_size, self.embedding_size) #TODO encoder와 공유하도록 하고 학습되지 않게..\n",
    "\n",
    "        #self.dropout = nn.Dropout(self.dropout_p)\n",
    "        self.lstm = nn.LSTM(self.embedding_size, hidden_size, n_layers, batch_first=True)\n",
    "        \n",
    "        self.switch = nn.Linear(self.hidden_size*3, 1) # Gen or Copy\n",
    "        self.out = nn.Linear(self.hidden_size*3, self.output_size)\n",
    "    \n",
    "    def init_hidden(self,input):\n",
    "        hidden = Variable(torch.randn(self.n_layers*1, input.size(0), self.hidden_size)).cuda() if USE_CUDA else Variable(torch.randn(self.n_layers*2, input.size(0), self.hidden_size))\n",
    "        context = Variable(torch.randn(self.n_layers*1, input.size(0), self.hidden_size)).cuda() if USE_CUDA else Variable(torch.randn(self.n_layers*2, input.size(0), self.hidden_size))\n",
    "        return (hidden,context)\n",
    "    \n",
    "    \n",
    "    def IntraEncAttention(self, hidden, encoder_outputs, encoder_masking):\n",
    "        \"\"\"\n",
    "        hidden : 1,B,D\n",
    "        encoder_outputs : B,T,D\n",
    "        encoder_masking : B,T # ByteTensor\n",
    "        \"\"\"\n",
    "        hidden = hidden.squeeze(0).unsqueeze(2)  # 히든 : (1,배치,차원) -> (배치,차원,1)\n",
    "        \n",
    "        batch_size = encoder_outputs.size(0) # B\n",
    "        encoder_len = encoder_outputs.size(1) # T : 300\n",
    "        \n",
    "        e_tis=[]\n",
    "        for i, e_i in enumerate(encoder_outputs.transpose(0,1)):\n",
    "            e_i = self.attn_e(e_i) # BxD\n",
    "            \n",
    "            ## PAD MASKING\n",
    "            e_masking = encoder_masking.transpose(0,1)[i].unsqueeze(1).expand_as(e_i) # BxD\n",
    "            e_i.masked_fill(e_masking,-1e12) # masking\n",
    "            \n",
    "            temporal_score = e_i.unsqueeze(1).bmm(hidden).squeeze(1).exp() # Bx1\n",
    "            self.temp_attn_scores.append(temporal_score)\n",
    "            \n",
    "            if len(self.temp_attn_scores)>1:\n",
    "                temporal_score = temporal_score/torch.sum(torch.cat(self.temp_attn_scores[:-1],1),1)\n",
    "            \n",
    "            e_tis.append(temporal_score)\n",
    "        \n",
    "        contexts=[]\n",
    "        normalized_attn_scores=[]\n",
    "        for i,e_i in enumerate(encoder_outputs.transpose(0,1)):\n",
    "            normalized_attn_score = e_tis[i]/torch.sum(torch.cat(e_tis,1),1)\n",
    "            normalized_attn_scores.append(normalized_attn_score)\n",
    "            context = normalized_attn_score.unsqueeze(2).bmm(e_i.unsqueeze(1)) # Bx1x1 , Bx1xD = Bx1xD\n",
    "            contexts.append(context)\n",
    "        \n",
    "        _, copy_attn_index = torch.max(F.softmax(torch.cat(normalized_attn_scores,1)),1)\n",
    "        \n",
    "        enc_context = torch.sum(torch.cat(contexts,1),1)\n",
    "        \n",
    "        del e_tis\n",
    "        del contexts\n",
    "        del normalized_attn_scores\n",
    "        \n",
    "        return  enc_context.unsqueeze(1), copy_attn_index.unsqueeze(1) # B,1,D\n",
    "    \n",
    "    def IntraDecAttention(self,hidden, decoder_outputs):\n",
    "        \"\"\"\n",
    "        hidden : 1,B,D\n",
    "        decoder_outputs : B,t,D\n",
    "        \"\"\"\n",
    "        hidden = hidden.squeeze(0).unsqueeze(2)  # 히든 : (1,배치,차원) -> (배치,차원,1)\n",
    "        \n",
    "        batch_size = decoder_outputs.size(0) # B\n",
    "        max_len = decoder_outputs.size(1) # t\n",
    "        \n",
    "        d_tis=[]\n",
    "        for d_i in decoder_outputs.transpose(0,1):\n",
    "            d_i = self.attn_d(d_i) # BxD\n",
    "            temporal_score = d_i.unsqueeze(1).bmm(hidden).squeeze(1).exp() # Bx1\n",
    "            d_tis.append(temporal_score)\n",
    "        \n",
    "        contexts=[]\n",
    "        for i,d_i in enumerate(decoder_outputs.transpose(0,1)):\n",
    "            normalized_attn_score = d_tis[i]/torch.sum(torch.cat(d_tis,1),1)\n",
    "            context = normalized_attn_score.unsqueeze(2).bmm(d_i.unsqueeze(1)) # Bx1x1 , Bx1xD = Bx1xD\n",
    "            contexts.append(context)\n",
    "        \n",
    "        dec_context = torch.sum(torch.cat(contexts,1),1) # B,1,D\n",
    "        \n",
    "        del d_tis\n",
    "        del contexts\n",
    "        \n",
    "        return dec_context.unsqueeze(1)\n",
    "    \n",
    "    def forward(self, input,teacher_forcing,context,encoder_outputs,encoder_maskings,training=True):\n",
    "        \"\"\"\n",
    "        input : B,1 (Start_token)\n",
    "        teacher_forcing : B,T # 수렴을 빠르게 할 목적으로 그 다음 인풋을 Ground_Truth로 알려준다\n",
    "        context : B,1,D\n",
    "        encoder_outpouts : B,T,D\n",
    "        \"\"\"\n",
    "        # Get the embedding of the current input word\n",
    "        embedded = self.embedding(input)\n",
    "        self.hidden = self.init_hidden(input)\n",
    "        #embedded = self.dropout(embedded)\n",
    "        \n",
    "        decode=[]\n",
    "        \n",
    "        enc_context = context\n",
    "        dec_context = Variable(torch.zeros(context.size())).cuda() if USE_CUDA else Variable(torch.zeros(context.size()))\n",
    "        decoder_outputs=[]\n",
    "        \n",
    "        \n",
    "        for i in range(self.max_len):\n",
    "\n",
    "            _, self.hidden = self.lstm(embedded, self.hidden)\n",
    "            decoder_outputs.append(self.hidden[0])\n",
    "            concated = torch.cat((self.hidden[0],enc_context.transpose(0,1),dec_context.transpose(0,1)),2)\n",
    "            \n",
    "            #TODO switch = torch.round(self.sigmoid(self.switch(concated.squeeze(0)))).squeeze(1) # 0 or 1 copy할지 decode할지 결정\n",
    "            score = self.out(concated.squeeze(0))\n",
    "            softmaxed = F.log_softmax(score)\n",
    "            decode.append(softmaxed)\n",
    "            _, dec_index = torch.max(softmaxed,1)\n",
    "            \n",
    "            \n",
    "            # encoder Context & Copy Index when switch= 1\n",
    "            enc_context, copy_attn_index = self.IntraEncAttention(self.hidden[0], encoder_outputs,encoder_maskings) \n",
    "\n",
    "            # decoder Context\n",
    "            if len(decoder_outputs)>=1:\n",
    "                dec_context = self.IntraDecAttention(self.hidden[0],torch.cat(decoder_outputs).transpose(0,1))\n",
    "            \n",
    "            # teacher forcing !!\n",
    "            if training: \n",
    "                input = teacher_forcing.transpose(0,1)[i].unsqueeze(1) # T,B [i] ==> B,1\n",
    "                embedded = self.embedding(input)\n",
    "                \n",
    "            # decode or copy\n",
    "            else:\n",
    "                embedded = self.embedding(dec_index.unsqueeze(1))\n",
    "                \n",
    "                #TODO : u_t의 ground-truth 어떻게 줄지 고민 후 ㅠㅠ\n",
    "#                 input=[]\n",
    "#                 for i,s in enumerate(switch):\n",
    "#                     if s==1:\n",
    "#                         input.append(copy_attn_index[i]) # copy\n",
    "#                     else:\n",
    "#                         input.append(dec_index[i]) # decode\n",
    "\n",
    "#                 embedded = self.embedding(torch.cat(input).unsqueeze(1))\n",
    "        \n",
    "        \n",
    "        # 요고 주의! time-step을 column-wise concat한 후, reshape!!\n",
    "        scores = torch.cat(decode,1)\n",
    "        \n",
    "        del decode\n",
    "        del decoder_outputs\n",
    "        \n",
    "        return scores.view(input.size(0)*self.max_len,-1) #TODO , switch # 0 or 1 학습"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "EMBEDDING_SIZE = 100\n",
    "HIDDEN_SIZE = 100\n",
    "BATCH_SIZE=8\n",
    "INPUT_LEN = 300\n",
    "OUTPUT_LEN = 50\n",
    "LEARNING_RATE = 0.001\n",
    "STEP_SIZE=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "encoder = EncoderRNN(len(input2index),EMBEDDING_SIZE,HIDDEN_SIZE)\n",
    "decoder = DecoderRNN(len(output2index),EMBEDDING_SIZE,HIDDEN_SIZE*2,OUTPUT_LEN)\n",
    "\n",
    "if USE_CUDA:\n",
    "    encoder = encoder.cuda()\n",
    "    decoder = decoder.cuda()\n",
    "    \n",
    "loss_function = nn.CrossEntropyLoss(ignore_index=0)\n",
    "enc_optim= optim.Adam(encoder.parameters(), lr=LEARNING_RATE)\n",
    "dec_optim = optim.Adam(decoder.parameters(),lr=LEARNING_RATE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "encoder go!\n",
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "5\n",
      "6\n",
      "7\n",
      "8\n",
      "9\n",
      "10\n",
      "11\n",
      "12\n",
      "13\n",
      "14\n",
      "15\n",
      "16\n",
      "17\n",
      "18\n",
      "19\n",
      "20\n",
      "21\n",
      "22\n",
      "23\n",
      "24\n",
      "25\n",
      "26\n",
      "27\n",
      "28\n",
      "29\n",
      "30\n",
      "31\n",
      "32\n",
      "33\n",
      "34\n",
      "35\n",
      "36\n",
      "37\n",
      "38\n",
      "39\n",
      "40\n",
      "41\n",
      "42\n",
      "43\n",
      "44\n",
      "45\n",
      "46\n",
      "47\n",
      "48\n",
      "49\n",
      "decoder go!\n"
     ]
    }
   ],
   "source": [
    "for step in range(STEP_SIZE):\n",
    "    losses=[]\n",
    "    for i, batch in enumerate(getBatch(BATCH_SIZE,train_data)):\n",
    "        x,y = zip(*batch)\n",
    "        \n",
    "        inputs = torch.cat(x)\n",
    "        targets = torch.cat(y)\n",
    "        masks = torch.cat([Variable(torch.ByteTensor(tuple(map(lambda s: s ==0, t.data)))).cuda() if USE_CUDA else Variable(torch.ByteTensor(tuple(map(lambda s: s ==0, t.data)))) for t in inputs]).view(BATCH_SIZE,-1)\n",
    "        \n",
    "        encoder.zero_grad()\n",
    "        decoder.zero_grad()\n",
    "        \n",
    "        enc_output, context = encoder(inputs,masks)\n",
    "        print(\"encoder go!\")\n",
    "        decoder_input = Variable(torch.LongTensor([[output2index['<SOS>']]*BATCH_SIZE])).cuda().transpose(1,0) if USE_CUDA else Variable(torch.LongTensor([[output2index['<SOS>']]*BATCH_SIZE])).transpose(1,0)\n",
    "                \n",
    "        decoding_score = decoder(decoder_input,targets,context,enc_output,masks)\n",
    "        print(\"decoder go!\")\n",
    "        loss = loss_function(decoding_score,targets.view(-1))\n",
    "        losses.append(loss)\n",
    "        loss.backward()\n",
    "        \n",
    "        torch.nn.utils.clip_grad_norm(encoder.parameters(), 5.0)\n",
    "        torch.nn.utils.clip_grad_norm(decoder.parameters(), 5.0)\n",
    "        \n",
    "        enc_optim.step()\n",
    "        dec_optim.step()\n",
    "        \n",
    "        if i % 10==0:\n",
    "            print(\"Step\",step,\" epoch\",i,\" : \",np.mean(losses))\n",
    "            losses=[]        "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Problem "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* Intra-attention  forward, backward 연산량이 말도 안되게 많음... epoch 한번 당 5~10분 걸리는듯\n",
    "* 혹은 잘못 짰거나 "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# TODO"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "1. 우선 teacher forcing (MLE)부터 해서 어텐션 및 마스킹이 제대로 들어갔는지부터 검증\n",
    "2. switch score의 ground Truth를 훈련시키기 위해 NER로 만드는거 고민\n",
    "3. 디코딩 액션을 선택하기 위해 log-prob을 어떻게 만들지... (policy gradient) \n",
    "4. mixed objective 최적화 (RL+MLE)\n",
    "5. softmax 연산의 부하를 줄이기 위한 방법 (negative sampling, sampled softmax 등)\n",
    "6. 나중에 teacher forcing anealing?! 확률적으로 동작하도록... \n",
    "7. GloVe로 임베딩 매트릭스 초기화 및 고정, 인코더-디코더-프로젝션 웨이트 공유"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}