bert_baseline_interactive.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import gc\n",
    "import torch\n",
    "from torch.utils.data import Dataset\n",
    "import numpy as np\n",
    "import random\n",
    "from transformers import BertModel, BertTokenizer\n",
    "from torch.utils.data import TensorDataset\n",
    "from transformers import glue_processors as processors\n",
    "from transformers import glue_output_modes as output_modes\n",
    "from transformers import glue_convert_examples_to_features as convert_examples_to_features\n",
    "import logging\n",
    "\n",
    "from torch import nn\n",
    "from torch.nn import functional as F\n",
    "from torch.utils.data import TensorDataset, DataLoader, RandomSampler\n",
    "from torch.optim import Adam\n",
    "from torch.nn import CrossEntropyLoss\n",
    "from transformers import BertForSequenceClassification\n",
    "from copy import deepcopy\n",
    "import gc\n",
    "from sklearn.metrics import accuracy_score\n",
    "import torch\n",
    "import numpy as np\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class TrainingArgs:\n",
    "    def __init__(self):\n",
    "        self.bert_model = 'bert-base-uncased'\n",
    "        self.local_rank = -1\n",
    "        self.data_dir = \"glue_data\"\n",
    "        self.overwrite_cache = True\n",
    "        self.num_labels = 2\n",
    "        self.batch_size = 8\n",
    "        self.update_lr  = 5e-5\n",
    "        self.epochs = 3\n",
    "        self.output_dir = \"bert_models&results\"\n",
    "        self.train_sample_per_task = 500\n",
    "        self.eval_sample_per_task = 100\n",
    "        self.seed = 2020\n",
    "args = TrainingArgs()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "logger=logging.getLogger()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "class BertTask_Baseline(Dataset):\n",
    "    ''' \n",
    "    Before running this script, please makes sure all 8 GLUE datasets are downloaded in local by running python3 ../../utils/download_glue_data.py\n",
    "    Modified MetaTask takes all 10 GLUE tasks, namely cola, mnli, mnli-mm, mrpc, sst-2, sts-b, \n",
    "    qqp, qnli, rte and wnli and convert them from raw test into features. \n",
    "    '''\n",
    "    \n",
    "    def __init__(self, args, tokenizer, max_seq_length, task, evaluate=False,sample=False):\n",
    "        \"\"\"\n",
    "        :param num_task: number of training tasks.\n",
    "        :param tokenizer: tokenizer uses to tokenzie from word to sequence\n",
    "        :param max_seq_length: length of the tokenzier vector\n",
    "        :param evaluate: indicate whether the dataset is from training/ evaluate sets\n",
    "        \"\"\"\n",
    "\n",
    "        self.tokenizer       = tokenizer\n",
    "        self.max_seq_length  = max_seq_length\n",
    "        self.evaluate        = evaluate\n",
    "        self.local_rank      = args.local_rank\n",
    "        self.data_dir        = args.data_dir\n",
    "        self.bert_model      = args.bert_model\n",
    "        self.overwrite_cache = args.overwrite_cache\n",
    "        self.sample = sample\n",
    "        self.task = task\n",
    "        self.create_batch()\n",
    "        \n",
    "        \n",
    "    def create_batch(self):\n",
    "        '''\n",
    "        Randomly select number of examples from each task into supports (meta training dataset) and queries (meta evaluating dataset)\n",
    "        '''\n",
    "        \n",
    "        # 1. randomly select num_task GLUE tasks \n",
    "        task = self.task \n",
    "\n",
    "        self.dataset = self.load_and_cache_examples(task, self.tokenizer, self.evaluate, self.sample) # map style dataset \n",
    "\n",
    "\n",
    "    def load_and_cache_examples(self, task, tokenizer, evaluate=False, sample=False):\n",
    "        '''\n",
    "        Copied from official loading and cache scripts from Huggingface Transformer load_and_cache_examples\n",
    "        https://github.com/huggingface/transformers/blob/master/examples/run_glue.py#L334\n",
    "        '''\n",
    "        folder_name = {'cola': 'CoLA', 'mnli-mm':'MNLI'}\n",
    "        if task in folder_name:\n",
    "            task_data_path = folder_name[task]\n",
    "        else:\n",
    "            task_data_path = task.upper()\n",
    "\n",
    "\n",
    "        if self.local_rank not in [-1, 0] and not evaluate:\n",
    "            torch.distributed.barrier()  # Make sure only the first process in distributed training process the dataset, and the others will use the cache\n",
    "\n",
    "        processor = processors[task]()\n",
    "        output_mode = output_modes[task]\n",
    "        cached_downloaded_file = os.path.join(self.data_dir, task_data_path)\n",
    "        # print(cached_downloaded_file)\n",
    "\n",
    "        logger.info(f\"Creating features from dataset file at {cached_downloaded_file}\")\n",
    "        label_list = processor.get_labels()\n",
    "\n",
    "        examples = (\n",
    "                processor.get_dev_examples(cached_downloaded_file) if evaluate else processor.get_train_examples(cached_downloaded_file)\n",
    "            )\n",
    "        if sample:\n",
    "            if args.seed:\n",
    "                random.seed(args.seed)\n",
    "            examples = random.sample(examples, sample)\n",
    "\n",
    "        features = convert_examples_to_features(\n",
    "            examples, tokenizer, max_length=self.max_seq_length, label_list=label_list, output_mode=output_mode,\n",
    "        )\n",
    "\n",
    "        if self.local_rank == 0 and not evaluate:\n",
    "            torch.distributed.barrier()  # Make sure only the first process in distributed training process the dataset, and the others will use the cache\n",
    "\n",
    "        # Convert to Tensors and build dataset\n",
    "        all_input_ids = torch.tensor([f.input_ids for f in features], dtype=torch.long)\n",
    "        all_attention_mask = torch.tensor([f.attention_mask for f in features], dtype=torch.long)\n",
    "        all_token_type_ids = torch.tensor([f.token_type_ids for f in features], dtype=torch.long)\n",
    "        if output_mode == \"classification\":\n",
    "            all_labels = torch.tensor([f.label for f in features], dtype=torch.long)\n",
    "        elif output_mode == \"regression\":\n",
    "            all_labels = torch.tensor([f.label for f in features], dtype=torch.float)\n",
    "\n",
    "        dataset = TensorDataset(all_input_ids, all_attention_mask, all_token_type_ids, all_labels)\n",
    "        return dataset\n",
    "\n",
    "    def __getitem__(self, index):\n",
    "        dataset_set = self.dataset[index]\n",
    "        return dataset_set\n",
    "\n",
    "    def __len__(self):\n",
    "        # as we have built up to batchsz of sets, you can sample some small batch size of sets.\n",
    "        return len(self.dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Bert_trainer(nn.Module):\n",
    "    \"\"\"\n",
    "    Meta Learner\n",
    "    \"\"\"\n",
    "    def __init__(self, args):\n",
    "        \"\"\"\n",
    "        :param args:\n",
    "        \"\"\"\n",
    "        super(Bert_trainer, self).__init__()\n",
    "\n",
    "        self.num_labels = args.num_labels\n",
    "        self.batch_size = args.batch_size\n",
    "        self.update_lr  = args.update_lr\n",
    "    \n",
    "\n",
    "        self.bert_model = args.bert_model\n",
    "        self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "        \n",
    "        self.model = BertForSequenceClassification.from_pretrained(self.bert_model, num_labels = self.num_labels)\n",
    "        self.outer_optimizer = Adam(self.model.parameters(), lr=self.update_lr)\n",
    "        self.model.train()\n",
    "\n",
    "    def forward(self, datasets,training=True):\n",
    "        \"\"\"\n",
    "        batch_tasks = TensorDataset(all_input_ids, all_attention_mask, all_segment_ids, all_label_ids)\n",
    "        \"\"\"\n",
    "\n",
    "        num_task = len(datasets)\n",
    "        self.model.to(self.device)\n",
    "        acc = None\n",
    "\n",
    "        \n",
    "        if training:\n",
    "            dataloader = DataLoader(datasets,batch_size=self.batch_size)\n",
    "            for data in dataloader:\n",
    "                all_loss = []\n",
    "                batch = tuple(t.to(self.device) for t in data)\n",
    "                input_ids, attention_mask, segment_ids, label_id = batch\n",
    "                outputs = self.model(input_ids, attention_mask, segment_ids, labels = label_id)\n",
    "                loss = outputs[0]              \n",
    "                loss.backward()\n",
    "                self.outer_optimizer.step()\n",
    "                self.outer_optimizer.zero_grad()\n",
    "                all_loss.append(loss.item())\n",
    "            self.model.to(torch.device('cpu'))\n",
    "            return outputs\n",
    "        else:\n",
    "            \n",
    "            with torch.no_grad():\n",
    "                correct = 0\n",
    "                total = 0\n",
    "                self.model.to(torch.device(self.device))\n",
    "                dataloader = DataLoader(datasets,batch_size=self.batch_size)\n",
    "                for data in dataloader:\n",
    "                    \n",
    "                    query_batch = iter(dataloader).next()\n",
    "                    query_batch = tuple(t.to(self.device) for t in query_batch)\n",
    "                    q_input_ids, q_attention_mask, q_segment_ids, q_label_id = query_batch\n",
    "                    q_outputs = self.model(q_input_ids, q_attention_mask, q_segment_ids, labels = q_label_id)\n",
    "                \n",
    "                    q_logits = F.softmax(q_outputs[1],dim=1)\n",
    "                    pre_label_id = torch.argmax(q_logits,dim=1)\n",
    "                    # pre_label_id = pre_label_id.detach().cpu().numpy().tolist()\n",
    "                    # label_id = q_label_id.detach().cpu().numpy().tolist()\n",
    "                    total += q_label_id.size(0)\n",
    "                    correct += pre_label_id.eq(q_label_id.to(self.device).view_as(pre_label_id)).sum().item()\n",
    "                acc = correct/total\n",
    "                self.model.to(torch.device('cpu'))\n",
    "        return acc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training on the cola task\n",
      "_____***Saving Model***___cola\n",
      "_____Evalating on the cola\n",
      "_____Finishing evalating on the cola\n",
      "Finishing training on the cola task\n",
      "Training on the sst-2 task\n",
      "_____***Saving Model***___sst-2\n",
      "_____Evalating on the cola\n",
      "_____Finishing evalating on the cola\n",
      "_____Evalating on the sst-2\n",
      "_____Finishing evalating on the sst-2\n",
      "Finishing training on the sst-2 task\n",
      "Training on the mrpc task\n",
      "_____***Saving Model***___mrpc\n",
      "_____Evalating on the cola\n",
      "_____Finishing evalating on the cola\n",
      "_____Evalating on the sst-2\n",
      "_____Finishing evalating on the sst-2\n",
      "_____Evalating on the mrpc\n",
      "_____Finishing evalating on the mrpc\n",
      "Finishing training on the mrpc task\n",
      "Training on the qqp task\n",
      "_____***Saving Model***___qqp\n",
      "_____Evalating on the cola\n",
      "_____Finishing evalating on the cola\n",
      "_____Evalating on the sst-2\n",
      "_____Finishing evalating on the sst-2\n",
      "_____Evalating on the mrpc\n",
      "_____Finishing evalating on the mrpc\n",
      "_____Evalating on the qqp\n",
      "_____Finishing evalating on the qqp\n",
      "Finishing training on the qqp task\n",
      "Training on the qnli task\n",
      "_____***Saving Model***___qnli\n",
      "_____Evalating on the cola\n",
      "_____Finishing evalating on the cola\n",
      "_____Evalating on the sst-2\n",
      "_____Finishing evalating on the sst-2\n",
      "_____Evalating on the mrpc\n",
      "_____Finishing evalating on the mrpc\n",
      "_____Evalating on the qqp\n",
      "_____Finishing evalating on the qqp\n",
      "_____Evalating on the qnli\n",
      "_____Finishing evalating on the qnli\n",
      "Finishing training on the qnli task\n",
      "Training on the rte task\n",
      "_____***Saving Model***___rte\n",
      "_____Evalating on the cola\n",
      "_____Finishing evalating on the cola\n",
      "_____Evalating on the sst-2\n",
      "_____Finishing evalating on the sst-2\n",
      "_____Evalating on the mrpc\n",
      "_____Finishing evalating on the mrpc\n",
      "_____Evalating on the qqp\n",
      "_____Finishing evalating on the qqp\n",
      "_____Evalating on the qnli\n",
      "_____Finishing evalating on the qnli\n",
      "_____Evalating on the rte\n",
      "_____Finishing evalating on the rte\n",
      "Finishing training on the rte task\n"
     ]
    }
   ],
   "source": [
    "random.seed(2020)\n",
    "tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case = True)\n",
    "task_lists = [\"cola\", \"sst-2\", \"mrpc\",\"qqp\",\"qnli\",\"rte\"]\n",
    "\n",
    "my_Bert = Bert_trainer(args)\n",
    "acc_results = []\n",
    "saving_path = os.path.join(args.output_dir,\"model\")\n",
    "if  not os.path.exists(saving_path):\n",
    "    os.makedirs(saving_path)\n",
    "\n",
    "for i,task in enumerate(task_lists):\n",
    "    train_data = BertTask_Baseline(args, tokenizer,128,task,sample=args.train_sample_per_task)\n",
    "    print(\"Training on the {} task\".format(task))\n",
    "    for epoch in range(args.epochs):\n",
    "        outputs = my_Bert(train_data)\n",
    "    print(\"_____***Saving Model***___{}\".format(task))\n",
    "    torch.save(my_Bert.state_dict(), os.path.join(saving_path,\"{}_params.pkl\".format(task)))\n",
    "    del train_data\n",
    "    _ = gc.collect()\n",
    "    ### Evaluating\n",
    "    accs = []\n",
    "    for j in range(i+1):\n",
    "        eval_task = task_lists[j]\n",
    "        print(\"_____Evalating on the {}\".format(eval_task))\n",
    "        eval_data = BertTask_Baseline(args, tokenizer,128,eval_task,evaluate=True,sample=args.eval_sample_per_task)\n",
    "        acc = my_Bert(eval_data,training=False)\n",
    "        accs.append(acc)\n",
    "        del eval_data\n",
    "        _ = gc.collect()\n",
    "        print(\"_____Finishing evalating on the {}\".format(eval_task))\n",
    "    acc_results.append(accs)\n",
    "    print(\"Finishing training on the {} task\".format(task))\n",
    "\n",
    "\n",
    "acc_results_pad = [line+[\"\"]*(6-len(line)) for line in acc_results]\n",
    "final_acc = \"\\n\".join([\",\".join(list(map(str,line))) for line in acc_results_pad])\n",
    "    \n",
    "with open(os.path.join(args.output_dir,\"results.txt\"),\"w\") as f:\n",
    "    f.write(\",\".join(task_lists))\n",
    "    f.write(\"\\n\")\n",
    "    f.write(final_acc)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>cola</th>\n",
       "      <th>sst-2</th>\n",
       "      <th>mrpc</th>\n",
       "      <th>qqp</th>\n",
       "      <th>qnli</th>\n",
       "      <th>rte</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.730769</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0.663462</td>\n",
       "      <td>0.961538</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0.644231</td>\n",
       "      <td>0.509615</td>\n",
       "      <td>0.298077</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0.250000</td>\n",
       "      <td>0.250000</td>\n",
       "      <td>0.394231</td>\n",
       "      <td>0.586538</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>0.625000</td>\n",
       "      <td>0.750000</td>\n",
       "      <td>0.105769</td>\n",
       "      <td>0.346154</td>\n",
       "      <td>0.682692</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>0.750000</td>\n",
       "      <td>0.682692</td>\n",
       "      <td>0.615385</td>\n",
       "      <td>0.394231</td>\n",
       "      <td>0.461538</td>\n",
       "      <td>0.653846</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       cola     sst-2      mrpc       qqp      qnli       rte\n",
       "0  0.730769       NaN       NaN       NaN       NaN       NaN\n",
       "1  0.663462  0.961538       NaN       NaN       NaN       NaN\n",
       "2  0.644231  0.509615  0.298077       NaN       NaN       NaN\n",
       "3  0.250000  0.250000  0.394231  0.586538       NaN       NaN\n",
       "4  0.625000  0.750000  0.105769  0.346154  0.682692       NaN\n",
       "5  0.750000  0.682692  0.615385  0.394231  0.461538  0.653846"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "acc_mtx = pd.read_csv(os.path.join(args.output_dir,\"results.txt\"))\n",
    "acc_mtx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}