inception cifar sup large batch.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import argparse\n",
    "import logging\n",
    "logging.basicConfig(level=logging.INFO)\n",
    "from common import find_mxnet, data, fit\n",
    "from common.util import download_file\n",
    "import mxnet as mx\n",
    "import numpy as np\n",
    "from mxnet.symbol import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def download_cifar100():\n",
    "    data_dir=\"/efs/data/cifar-100-mxnet\"\n",
    "    #data_dir=\"/data/cifar-100-mxnet\"\n",
    "    fnames = (os.path.join(data_dir, \"train.rec\"),\n",
    "              os.path.join(data_dir, \"test.rec\"))\n",
    "    return fnames"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "(train_fname, val_fname) = download_cifar100()\n",
    "\n",
    "# parse args\n",
    "parser = argparse.ArgumentParser(description=\"train cifar100\",\n",
    "                                 formatter_class=argparse.ArgumentDefaultsHelpFormatter)\n",
    "fit.add_fit_args(parser)\n",
    "data.add_data_args(parser)\n",
    "data.add_data_aug_args(parser)\n",
    "data.set_data_aug_level(parser, 2)\n",
    "\n",
    "num_unsup_examples = 50000\n",
    "subset_factor = 5\n",
    "\n",
    "parser.set_defaults(\n",
    "    # data\n",
    "    data_train     = train_fname,\n",
    "    data_val       = val_fname,\n",
    "    num_classes    = 100,\n",
    "    image_shape    = '3,28,28',\n",
    " #   model_prefix   = '/efs/checkpoints/inceptionStandard',\n",
    "    pad_size       = 4,\n",
    "    # train\n",
    "    batch_size     = 100,\n",
    "    num_epochs     = 130,\n",
    "    lr_step_epochs = '80,120',\n",
    "    gpus           = '0,1,2,3',\n",
    "    loss_gpu       = 0,\n",
    "    disp_batches   = 100,\n",
    "    num_examples   = num_unsup_examples / subset_factor, \n",
    "    wd             = 1e-4,\n",
    "    lr             = .05,\n",
    "    lr_factor      = .33,\n",
    "    nembeddings    = 256,\n",
    "    optimizer      = 'sgd'\n",
    ")\n",
    "\n",
    "args = parser.parse_args(\"\")\n",
    "unsup_multiplier = 0\n",
    "labeled_per_class = 500 / subset_factor\n",
    "sample_seed = 47\n",
    "val_interval = 1\n",
    "num_sup_examples = labeled_per_class * args.num_classes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1  times more unsup data than sup data\n"
     ]
    }
   ],
   "source": [
    "from common.multi_iterator import Multi_iterator\n",
    "from common.data import get_partial_rec_iter\n",
    "\n",
    "(train_sup, val) = get_partial_rec_iter(args, get_val=True, devide_by=subset_factor, shuffle=True)\n",
    "(train_unsup, _) = get_partial_rec_iter(args, get_val=False, devide_by=1, shuffle=True)\n",
    "    \n",
    "train = Multi_iterator(train_sup, train_unsup, unsup_multiplier, num_unsup_examples, num_sup_examples)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[8000.0, 12000.0]\n"
     ]
    }
   ],
   "source": [
    "from symbols import inception_cifar as base_net\n",
    "from common.lba import compute_semisup_loss, logit_loss\n",
    "from common.wrapper_module import get_embedding_shapes\n",
    "\n",
    "# kvstore\n",
    "kv = mx.kvstore.create(args.kv_store)\n",
    "\n",
    "t_nb = args.batch_size * unsup_multiplier\n",
    "initializer = mx.init.Xavier(rnd_type='gaussian', factor_type=\"avg\", magnitude=2.34)\n",
    "initializer = mx.init.Uniform(0.01)\n",
    "\n",
    "lr, lr_scheduler = fit._get_lr_scheduler(args, kv)\n",
    "\n",
    "def buildEmbeddingModule():\n",
    "    dataSup = mx.symbol.Variable(name=\"dataSup\")\n",
    "    dataUnsup = []\n",
    "    for i in range(unsup_multiplier):\n",
    "        dataUnsup.append(Variable(name=\"dataUnsup\"+str(i)))\n",
    "\n",
    "    # concat data, feed both through the network\n",
    "    # then split it up again\n",
    "    data = concat(dataSup, *dataUnsup, dim=0)\n",
    "\n",
    "    embeddings = base_net.build_embeddings(data, nembeddings=args.nembeddings)\n",
    "    splitted = split(embeddings, num_outputs=(unsup_multiplier+1), axis=0, name='split')\n",
    "    \n",
    "    # devices for training\n",
    "    devs = mx.cpu() if args.gpus is None or args.gpus is '' else [\n",
    "        mx.gpu(int(i)) for i in args.gpus.split(',')]\n",
    "\n",
    "    data_names = ['dataSup'] + ['dataUnsup'+str(i) for i in range(unsup_multiplier)]\n",
    "        \n",
    "    # create model\n",
    "    model = mx.mod.Module(\n",
    "        context       = devs,\n",
    "        symbol        = splitted,\n",
    "        data_names    = data_names,\n",
    "        label_names   = None)\n",
    "    \n",
    "    model.bind(data_shapes=train.provide_data)         \n",
    "    model.init_params(initializer)    \n",
    "    model.init_optimizer(optimizer=args.optimizer, optimizer_params=(\n",
    "        ('learning_rate', lr), \n",
    "        ('wd', 1e-4),\n",
    "        ('momentum', args.mom),\n",
    "        #('rescale_grad', 0.01),\n",
    "        ('lr_scheduler', lr_scheduler)))\n",
    "\n",
    "    return model\n",
    "\n",
    "\n",
    "def buildLossModule():\n",
    "    supEmbeddings = Variable(name=\"embeddings_sup\")\n",
    "    labelSup = mx.symbol.Variable(name='labelSup')\n",
    "    overall_loss = []\n",
    "    \n",
    "    if unsup_multiplier >= 1:\n",
    "        unsupEmbeddings = []\n",
    "        for i in range(unsup_multiplier):\n",
    "            unsupEmbeddings.append(Variable(name=\"embeddings_unsup\"+str(i)))\n",
    "    \n",
    "        unsupEmbeddings = concat(*unsupEmbeddings, dim=0)\n",
    "            \n",
    "        (walker_loss, visit_loss) = compute_semisup_loss(supEmbeddings, unsupEmbeddings, labelSup, t_nb, \n",
    "                                                     walker_weight=1.0, visit_weight=1.0)\n",
    "        overall_loss = [walker_loss, visit_loss]\n",
    "        \n",
    "    overall_loss = [logit_loss(supEmbeddings, labelSup, args.num_classes)] + overall_loss\n",
    "    \n",
    "    # todo maybe use gpu\n",
    "    devs = mx.cpu() if args.loss_gpu is None or args.loss_gpu is '' else mx.gpu(args.loss_gpu)\n",
    "\n",
    "    # create module\n",
    "    model = mx.mod.Module(\n",
    "        context = devs,\n",
    "        symbol  = Group(overall_loss),\n",
    "        data_names = ['embeddings_sup']+['embeddings_unsup'+str(i) for i in range(unsup_multiplier)],\n",
    "        label_names = ['labelSup'])\n",
    "    \n",
    "    # allocate memory by given the input data and label shapes\n",
    "    model.bind(data_shapes=get_embedding_shapes(args.batch_size, args.nembeddings, unsup_multiplier), \n",
    "               label_shapes=train.provide_label,\n",
    "               inputs_need_grad=True)\n",
    "            \n",
    "    model.init_params(initializer)    \n",
    "    model.init_optimizer(optimizer=args.optimizer, optimizer_params=(\n",
    "        ('learning_rate', lr), \n",
    "        #('rescale_grad', 0.01),\n",
    "        ('momentum', args.mom),\n",
    "        ('wd', 1e-4),\n",
    "        ('lr_scheduler', lr_scheduler)))\n",
    "    \n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from common.wrapper_module import WrapperModule\n",
    "#eval_metrics = Multi_Accuracy(num= 3 if unsup_multiplier >= 1 else 1)\n",
    "                    \n",
    "def fit_model(args, data, **kwargs):\n",
    "    \"\"\"\n",
    "    train a model\n",
    "    args : argparse returns\n",
    "    data_loader : function that returns the train and val data iterators\n",
    "    \"\"\"\n",
    "    # logging\n",
    "    head = '%(asctime)-15s Node[' + str(kv.rank) + '] %(message)s'\n",
    "    logging.basicConfig(level=logging.DEBUG, format=head)\n",
    "    logging.info('start with arguments %s', args)\n",
    "    \n",
    "    # data iterators\n",
    "    (train, val) = data\n",
    "\n",
    "    # load model\n",
    "    if 'arg_params' in kwargs and 'aux_params' in kwargs:\n",
    "        arg_params = kwargs['arg_params']\n",
    "        aux_params = kwargs['aux_params']\n",
    "    else: \n",
    "        sym, arg_params, aux_params = fit._load_model(args, kv.rank)\n",
    "        \n",
    "    # save model\n",
    "    checkpoint = fit._save_model(args, kv.rank)\n",
    "\n",
    "    model = WrapperModule(buildEmbeddingModule, buildLossModule)\n",
    "    \n",
    "    def validate_model(epoch, *args):\n",
    "        if epoch % val_interval != 0: \n",
    "            return\n",
    "        res = model.score(val)\n",
    "        #TODO: pull this into default\n",
    "        print('Epoch[%d] Validation-accuracy=%f' % (epoch,  res))\n",
    "\n",
    "    #monitor = mx.mon.Monitor(interval=1000, pattern='.*aba_backward.*') \n",
    "    monitor = mx.mon.Monitor(interval=1000, pattern='.*') \n",
    "    \n",
    "    # callbacks that run after each batch\n",
    "    batch_end_callbacks = [mx.callback.Speedometer(args.batch_size, args.disp_batches)]\n",
    "    \n",
    "    epoch_end_callbacks = validate_model\n",
    "\n",
    "    # run\n",
    "    model.fit(train,\n",
    "        begin_epoch        = args.load_epoch if args.load_epoch else 0,\n",
    "        num_epoch          = args.num_epochs,\n",
    "        #eval_data          = val,\n",
    "        #eval_metric        = eval_metrics,\n",
    "        kvstore            = kv,\n",
    "        arg_params         = arg_params,\n",
    "        aux_params         = aux_params,\n",
    "        batch_end_callback = batch_end_callbacks,\n",
    "        epoch_end_callback = epoch_end_callbacks,\n",
    "        allow_missing      = True\n",
    "        #monitor            = monitor\n",
    "             )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:root:start with arguments Namespace(batch_size=100, benchmark=0, data_nthreads=4, data_train='/efs/data/cifar-100-mxnet/train.rec', data_val='/efs/data/cifar-100-mxnet/test.rec', disp_batches=100, dtype='float32', gpus='0,1,2,3', image_shape='3,28,28', kv_store='device', load_epoch=None, loss_gpu=0, lr=0.05, lr_factor=0.33, lr_step_epochs='80,120', max_random_aspect_ratio=0, max_random_h=36, max_random_l=50, max_random_rotate_angle=0, max_random_s=50, max_random_scale=1, max_random_shear_ratio=0, min_random_scale=1, model_prefix=None, mom=0.9, monitor=0, nembeddings=256, network=None, num_classes=100, num_epochs=130, num_examples=10000.0, num_layers=None, optimizer='sgd', pad_size=4, random_crop=1, random_mirror=1, rgb_mean='123.68,116.779,103.939', test_io=0, top_k=0, wd=0.0001)\n",
      "INFO:root:Epoch[0] Train-accuracy=nan\n",
      "INFO:root:Epoch[0] Time cost=24.905\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch[0] Validation-accuracy=0.040800\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:root:Epoch[1] Train-accuracy=nan\n",
      "INFO:root:Epoch[1] Time cost=24.730\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch[1] Validation-accuracy=0.038000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:root:Epoch[2] Train-accuracy=nan\n",
      "INFO:root:Epoch[2] Time cost=24.752\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch[2] Validation-accuracy=0.047600\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:root:Epoch[3] Train-accuracy=nan\n",
      "INFO:root:Epoch[3] Time cost=24.671\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch[3] Validation-accuracy=0.057500\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:root:Epoch[4] Train-accuracy=nan\n",
      "INFO:root:Epoch[4] Time cost=25.020\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch[4] Validation-accuracy=0.054600\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:root:Epoch[5] Train-accuracy=nan\n",
      "INFO:root:Epoch[5] Time cost=25.202\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch[5] Validation-accuracy=0.099900\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:root:Epoch[6] Train-accuracy=nan\n",
      "INFO:root:Epoch[6] Time cost=24.729\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch[6] Validation-accuracy=0.095500\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:root:Epoch[7] Train-accuracy=nan\n",
      "INFO:root:Epoch[7] Time cost=24.816\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch[7] Validation-accuracy=0.136800\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:root:Epoch[8] Train-accuracy=nan\n",
      "INFO:root:Epoch[8] Time cost=24.825\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch[8] Validation-accuracy=0.141400\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:root:Epoch[9] Train-accuracy=nan\n",
      "INFO:root:Epoch[9] Time cost=24.780\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch[9] Validation-accuracy=0.145800\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:root:Epoch[10] Train-accuracy=nan\n",
      "INFO:root:Epoch[10] Time cost=24.806\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch[10] Validation-accuracy=0.158800\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:root:Epoch[11] Train-accuracy=nan\n",
      "INFO:root:Epoch[11] Time cost=24.822\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch[11] Validation-accuracy=0.148100\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:root:Epoch[12] Train-accuracy=nan\n",
      "INFO:root:Epoch[12] Time cost=24.780\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch[12] Validation-accuracy=0.181600\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:root:Epoch[13] Train-accuracy=nan\n",
      "INFO:root:Epoch[13] Time cost=24.746\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch[13] Validation-accuracy=0.177500\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:root:Epoch[14] Train-accuracy=nan\n",
      "INFO:root:Epoch[14] Time cost=24.763\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch[14] Validation-accuracy=0.220400\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:root:Epoch[15] Train-accuracy=nan\n",
      "INFO:root:Epoch[15] Time cost=24.735\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch[15] Validation-accuracy=0.195400\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:root:Epoch[16] Train-accuracy=nan\n",
      "INFO:root:Epoch[16] Time cost=25.104\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch[16] Validation-accuracy=0.238300\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:root:Epoch[17] Train-accuracy=nan\n",
      "INFO:root:Epoch[17] Time cost=24.990\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch[17] Validation-accuracy=0.248500\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:root:Epoch[18] Train-accuracy=nan\n",
      "INFO:root:Epoch[18] Time cost=24.733\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch[18] Validation-accuracy=0.252200\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:root:Epoch[19] Train-accuracy=nan\n",
      "INFO:root:Epoch[19] Time cost=24.775\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch[19] Validation-accuracy=0.277300\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:root:Epoch[20] Train-accuracy=nan\n",
      "INFO:root:Epoch[20] Time cost=24.703\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch[20] Validation-accuracy=0.269600\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:root:Epoch[21] Train-accuracy=nan\n",
      "INFO:root:Epoch[21] Time cost=24.788\n"
     ]
    }
   ],
   "source": [
    "train.reset()\n",
    "import time\n",
    "\n",
    "start = time.time()\n",
    "fit_model(args, (train, val))\n",
    "print(time.time() - start)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "128"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val.reset()\n",
    "val.next().label[0].shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class Multi_Accuracy(mx.metric.EvalMetric):\n",
    "    \"\"\"Calculate accuracies of multi label\"\"\"\n",
    "\n",
    "    def __init__(self, num=None):\n",
    "        super(Multi_Accuracy, self).__init__('multi-accuracy', num)\n",
    "\n",
    "    def update(self, labels, preds):\n",
    "\n",
    "        #for i in range(len(preds)):\n",
    "        for i in range(1):\n",
    "            pred_label = mx.nd.argmax_channel(preds[i]).asnumpy().astype('int32')\n",
    "            label = labels[0].asnumpy().astype('int32')\n",
    "\n",
    "            #mx.metric.check_label_shapes(label, pred_label)\n",
    "            \n",
    "            #print((pred_label.flat == label.flat).sum())\n",
    "            #print(len(pred_label.flat))\n",
    "\n",
    "            \n",
    "            self.sum_metric[i] += (pred_label.flat == label.flat).sum()\n",
    "            self.num_inst[i] += len(pred_label.flat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# batch 10 train 0.389, valid: 0.40"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.4.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}